2005-04-07 21:19:42 +02:00
// plasmaWordIndex.java
2006-10-13 03:19:26 +02:00
// (C) 2005, 2006 by Michael Peter Christen; mc@anomic.de, Frankfurt a. M., Germany
// first published 2005 on http://www.anomic.de
//
// This is a part of YaCy, a peer-to-peer based web search engine
2005-09-20 17:43:31 +02:00
//
// $LastChangedDate$
// $LastChangedRevision$
2007-07-18 00:06:06 +02:00
// $LastChangedBy$
2005-04-07 21:19:42 +02:00
//
2006-10-13 03:19:26 +02:00
// LICENSE
2006-12-22 20:26:01 +01:00
//
2005-04-07 21:19:42 +02:00
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation; either version 2 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
package de.anomic.plasma ;
2005-05-05 07:32:19 +02:00
import java.io.File ;
2007-02-26 16:49:23 +01:00
import java.util.ArrayList ;
2006-09-30 00:27:20 +02:00
import java.util.Date ;
2006-09-11 00:36:47 +02:00
import java.util.HashMap ;
2006-09-30 00:27:20 +02:00
import java.util.HashSet ;
2005-05-05 07:32:19 +02:00
import java.util.Iterator ;
2006-01-22 01:07:00 +01:00
import java.util.Map ;
2005-10-12 14:28:49 +02:00
import java.util.Set ;
2006-03-15 17:01:42 +01:00
import java.util.TreeSet ;
2005-05-05 07:32:19 +02:00
2006-01-30 01:42:38 +01:00
import de.anomic.htmlFilter.htmlFilterContentScraper ;
2006-08-06 00:22:14 +02:00
import de.anomic.index.indexCollectionRI ;
2006-05-28 03:09:31 +02:00
import de.anomic.index.indexContainer ;
2006-07-26 13:21:51 +02:00
import de.anomic.index.indexContainerOrder ;
2006-11-06 11:15:05 +01:00
import de.anomic.index.indexRAMRI ;
2006-05-26 11:32:50 +02:00
import de.anomic.index.indexRI ;
2007-11-07 23:38:09 +01:00
import de.anomic.index.indexRWIRowEntry ;
2006-08-02 22:01:59 +02:00
import de.anomic.index.indexURLEntry ;
2006-01-04 01:39:00 +01:00
import de.anomic.kelondro.kelondroBase64Order ;
2008-01-11 01:12:01 +01:00
import de.anomic.kelondro.kelondroByteOrder ;
2007-03-08 17:15:40 +01:00
import de.anomic.kelondro.kelondroCloneableIterator ;
2006-02-14 01:12:07 +01:00
import de.anomic.kelondro.kelondroMergeIterator ;
2006-03-16 17:44:29 +01:00
import de.anomic.kelondro.kelondroOrder ;
2007-03-08 17:15:40 +01:00
import de.anomic.kelondro.kelondroRotateIterator ;
2008-02-03 13:40:40 +01:00
import de.anomic.kelondro.kelondroRowSet ;
2006-09-30 00:27:20 +02:00
import de.anomic.plasma.urlPattern.plasmaURLPattern ;
2007-03-11 00:22:37 +01:00
import de.anomic.server.serverMemory ;
2007-12-27 18:56:59 +01:00
import de.anomic.server.logging.serverLog ;
2006-09-14 02:51:02 +02:00
import de.anomic.yacy.yacyDHTAction ;
2007-05-14 12:00:23 +02:00
import de.anomic.yacy.yacySeedDB ;
2007-09-05 11:01:35 +02:00
import de.anomic.yacy.yacyURL ;
2005-04-07 21:19:42 +02:00
2006-10-28 02:22:10 +02:00
public final class plasmaWordIndex implements indexRI {
2005-09-20 17:43:31 +02:00
2007-03-03 01:55:51 +01:00
// environment constants
public static final long wCacheMaxAge = 1000 * 60 * 30 ; // milliseconds; 30 minutes
2007-10-31 12:32:40 +01:00
public static final int wCacheMaxChunk = 400 ; // maximum number of references for each urlhash
2007-09-28 03:21:31 +02:00
public static final int lowcachedivisor = 320 ;
2007-11-22 00:14:57 +01:00
public static final int maxCollectionPartition = 7 ; // should be 7
2007-03-03 01:55:51 +01:00
2008-01-11 01:12:01 +01:00
private final kelondroByteOrder indexOrder = kelondroBase64Order . enhancedCoder ;
2006-12-05 03:47:51 +01:00
private final indexRAMRI dhtOutCache , dhtInCache ;
private final indexCollectionRI collections ; // new database structure to replace AssortmentCluster and FileCluster
public boolean busyCacheFlush ; // shows if a cache flush is currently performed
2007-02-27 14:01:22 +01:00
private int flushsize ;
2006-12-05 03:47:51 +01:00
public final plasmaCrawlLURL loadedURL ;
2006-02-14 01:12:07 +01:00
2008-02-19 10:14:07 +01:00
public plasmaWordIndex ( File indexPrimaryRoot , File indexSecondaryRoot , serverLog log ) {
2007-03-24 16:28:17 +01:00
File textindexcache = new File ( indexPrimaryRoot , " PUBLIC/TEXT/RICACHE " ) ;
2006-11-19 21:05:25 +01:00
if ( ! ( textindexcache . exists ( ) ) ) textindexcache . mkdirs ( ) ;
2007-11-07 23:38:09 +01:00
this . dhtOutCache = new indexRAMRI ( textindexcache , indexRWIRowEntry . urlEntryRow , wCacheMaxChunk , wCacheMaxAge , " dump1.array " , log ) ;
this . dhtInCache = new indexRAMRI ( textindexcache , indexRWIRowEntry . urlEntryRow , wCacheMaxChunk , wCacheMaxAge , " dump2.array " , log ) ;
2006-08-06 00:22:14 +02:00
// create collections storage path
2007-03-24 16:28:17 +01:00
File textindexcollections = new File ( indexPrimaryRoot , " PUBLIC/TEXT/RICOLLECTION " ) ;
2006-11-19 21:05:25 +01:00
if ( ! ( textindexcollections . exists ( ) ) ) textindexcollections . mkdirs ( ) ;
2008-02-19 10:14:07 +01:00
this . collections = new indexCollectionRI ( textindexcollections , " collection " , maxCollectionPartition , indexRWIRowEntry . urlEntryRow ) ;
2006-12-05 03:47:51 +01:00
// create LURL-db
2008-02-19 10:14:07 +01:00
loadedURL = new plasmaCrawlLURL ( indexSecondaryRoot ) ;
2006-08-07 23:49:39 +02:00
2006-12-05 03:47:51 +01:00
// performance settings
2006-08-07 23:49:39 +02:00
busyCacheFlush = false ;
2007-02-27 14:01:22 +01:00
this . flushsize = 2000 ;
2005-04-07 21:19:42 +02:00
}
2005-09-20 17:43:31 +02:00
2006-12-22 13:54:56 +01:00
public int minMem ( ) {
2007-08-20 19:36:43 +02:00
return 1024 * 1024 /* indexing overhead */ + dhtOutCache . minMem ( ) + dhtInCache . minMem ( ) + collections . minMem ( ) ;
2006-12-22 13:54:56 +01:00
}
2006-12-22 20:26:01 +01:00
2006-09-14 02:51:02 +02:00
public int maxURLinDHTOutCache ( ) {
return dhtOutCache . maxURLinCache ( ) ;
2005-04-25 01:15:40 +02:00
}
2005-09-20 17:43:31 +02:00
2006-09-14 02:51:02 +02:00
public long minAgeOfDHTOutCache ( ) {
return dhtOutCache . minAgeOfCache ( ) ;
2006-03-09 12:31:17 +01:00
}
2006-09-14 02:51:02 +02:00
public long maxAgeOfDHTOutCache ( ) {
return dhtOutCache . maxAgeOfCache ( ) ;
2006-02-25 22:05:19 +01:00
}
2006-09-14 02:51:02 +02:00
public int maxURLinDHTInCache ( ) {
return dhtInCache . maxURLinCache ( ) ;
2006-03-13 11:43:12 +01:00
}
2006-09-14 02:51:02 +02:00
public long minAgeOfDHTInCache ( ) {
return dhtInCache . minAgeOfCache ( ) ;
2006-03-13 11:43:12 +01:00
}
2006-09-14 02:51:02 +02:00
public long maxAgeOfDHTInCache ( ) {
return dhtInCache . maxAgeOfCache ( ) ;
2006-03-13 11:43:12 +01:00
}
2006-09-14 02:51:02 +02:00
public int dhtOutCacheSize ( ) {
return dhtOutCache . size ( ) ;
}
public int dhtInCacheSize ( ) {
return dhtInCache . size ( ) ;
2005-04-25 01:15:40 +02:00
}
2007-10-19 23:47:07 +02:00
public long dhtCacheSizeBytes ( boolean in ) {
// calculate the real size in bytes of DHT-In/Out-Cache
long cacheBytes = 0 ;
2007-11-07 23:38:09 +01:00
long entryBytes = indexRWIRowEntry . urlEntryRow . objectsize ;
2007-10-19 23:47:07 +02:00
indexRAMRI cache = ( in ? dhtInCache : dhtOutCache ) ;
2007-10-20 01:36:40 +02:00
synchronized ( cache ) {
2008-01-11 01:12:01 +01:00
Iterator < indexContainer > it = cache . wordContainers ( null , false ) ;
while ( it . hasNext ( ) ) cacheBytes + = it . next ( ) . size ( ) * entryBytes ;
2007-10-19 23:47:07 +02:00
}
return cacheBytes ;
}
2005-09-20 17:43:31 +02:00
2006-03-13 11:43:12 +01:00
public void setMaxWordCount ( int maxWords ) {
2006-09-14 02:51:02 +02:00
dhtOutCache . setMaxWordCount ( maxWords ) ;
2006-09-22 12:58:58 +02:00
dhtInCache . setMaxWordCount ( maxWords ) ;
}
2007-02-27 14:01:22 +01:00
public void setWordFlushSize ( int flushsize ) {
this . flushsize = flushsize ;
2006-06-27 13:14:30 +02:00
}
2007-08-15 13:36:59 +02:00
public void dhtFlushControl ( indexRAMRI theCache ) {
2006-02-25 17:10:31 +01:00
// check for forced flush
2007-11-12 02:14:51 +01:00
int count = - 1 ;
2007-08-15 13:36:59 +02:00
synchronized ( theCache ) {
2007-10-31 15:21:58 +01:00
if ( ( theCache . maxURLinCache ( ) > wCacheMaxChunk ) | |
2007-08-15 13:36:59 +02:00
( theCache . size ( ) > theCache . getMaxWordCount ( ) ) | |
2007-03-11 00:22:37 +01:00
( serverMemory . available ( ) < collections . minMem ( ) ) ) {
2007-11-12 02:14:51 +01:00
count = theCache . size ( ) + flushsize - theCache . getMaxWordCount ( ) ;
2006-09-14 02:51:02 +02:00
}
2006-02-14 01:12:07 +01:00
}
2007-11-12 02:14:51 +01:00
if ( count > = 0 ) flushCache ( theCache , ( count > 0 ) ? count : 1 ) ;
2006-02-25 17:10:31 +01:00
}
2006-10-28 02:22:10 +02:00
public long getUpdateTime ( String wordHash ) {
2007-09-04 01:43:55 +02:00
indexContainer entries = getContainer ( wordHash , null ) ;
2006-10-28 02:22:10 +02:00
if ( entries = = null ) return 0 ;
return entries . updated ( ) ;
}
2007-08-25 01:12:59 +02:00
public static indexContainer emptyContainer ( String wordHash , int elementCount ) {
2007-11-07 23:38:09 +01:00
return new indexContainer ( wordHash , indexRWIRowEntry . urlEntryRow , elementCount ) ;
2006-11-05 20:07:19 +01:00
}
2006-11-19 21:05:25 +01:00
2008-02-19 15:13:35 +01:00
public void addEntry ( String wordHash , indexRWIRowEntry entry , long updateTime , boolean dhtInCase ) {
2006-09-14 02:51:02 +02:00
// set dhtInCase depending on wordHash
if ( ( ! dhtInCase ) & & ( yacyDHTAction . shallBeOwnWord ( wordHash ) ) ) dhtInCase = true ;
// add the entry
2007-04-27 14:26:36 +02:00
if ( dhtInCase ) {
2006-09-14 02:51:02 +02:00
dhtInCache . addEntry ( wordHash , entry , updateTime , true ) ;
2007-08-15 13:36:59 +02:00
dhtFlushControl ( this . dhtInCache ) ;
2007-04-27 14:26:36 +02:00
} else {
2006-09-14 02:51:02 +02:00
dhtOutCache . addEntry ( wordHash , entry , updateTime , false ) ;
2007-08-15 13:36:59 +02:00
dhtFlushControl ( this . dhtOutCache ) ;
2006-09-14 02:51:02 +02:00
}
2006-02-25 17:10:31 +01:00
}
2008-02-03 00:53:39 +01:00
public void addEntries ( indexContainer entries ) {
addEntries ( entries , false ) ;
}
public void addEntries ( indexContainer entries , boolean dhtInCase ) {
2007-11-07 23:38:09 +01:00
assert ( entries . row ( ) . objectsize = = indexRWIRowEntry . urlEntryRow . objectsize ) ;
2006-11-19 21:05:25 +01:00
2006-09-14 02:51:02 +02:00
// set dhtInCase depending on wordHash
if ( ( ! dhtInCase ) & & ( yacyDHTAction . shallBeOwnWord ( entries . getWordHash ( ) ) ) ) dhtInCase = true ;
// add the entry
2007-04-27 14:26:36 +02:00
if ( dhtInCase ) {
2008-02-03 00:53:39 +01:00
dhtInCache . addEntries ( entries ) ;
2007-08-15 13:36:59 +02:00
dhtFlushControl ( this . dhtInCache ) ;
2007-04-27 14:26:36 +02:00
} else {
2008-02-03 00:53:39 +01:00
dhtOutCache . addEntries ( entries ) ;
2007-08-15 13:36:59 +02:00
dhtFlushControl ( this . dhtOutCache ) ;
2006-09-14 02:51:02 +02:00
}
2005-11-07 13:33:02 +01:00
}
2005-09-20 17:43:31 +02:00
2007-02-27 14:01:22 +01:00
public void flushCacheSome ( ) {
2007-05-02 16:20:43 +02:00
flushCache ( dhtOutCache , ( dhtOutCache . size ( ) > 3 * flushsize ) ? flushsize : Math . min ( flushsize , Math . max ( 1 , dhtOutCache . size ( ) / lowcachedivisor ) ) ) ;
flushCache ( dhtInCache , ( dhtInCache . size ( ) > 3 * flushsize ) ? flushsize : Math . min ( flushsize , Math . max ( 1 , dhtInCache . size ( ) / lowcachedivisor ) ) ) ;
2006-02-25 09:42:45 +01:00
}
2006-11-06 11:15:05 +01:00
private void flushCache ( indexRAMRI ram , int count ) {
2007-11-04 11:36:25 +01:00
if ( count < = 0 ) return ;
2006-08-21 03:29:26 +02:00
busyCacheFlush = true ;
String wordHash ;
2008-01-11 01:12:01 +01:00
ArrayList < indexContainer > containerList = new ArrayList < indexContainer > ( ) ;
2007-11-12 02:14:51 +01:00
count = Math . min ( 5000 , Math . min ( count , ram . size ( ) ) ) ;
boolean collectMax = true ;
indexContainer c ;
while ( collectMax ) {
synchronized ( ram ) {
2007-02-28 12:13:23 +01:00
wordHash = ram . maxScoreWordHash ( ) ;
2007-09-04 01:43:55 +02:00
c = ram . getContainer ( wordHash , null ) ;
2007-03-03 01:55:51 +01:00
if ( ( c ! = null ) & & ( c . size ( ) > wCacheMaxChunk ) ) {
2007-02-28 12:13:23 +01:00
containerList . add ( ram . deleteContainer ( wordHash ) ) ;
2007-03-11 00:22:37 +01:00
if ( serverMemory . available ( ) < collections . minMem ( ) ) break ; // protect memory during flush
2007-02-28 12:13:23 +01:00
} else {
collectMax = false ;
}
}
2007-11-12 02:14:51 +01:00
}
count = count - containerList . size ( ) ;
for ( int i = 0 ; i < count ; i + + ) { // possible position of outOfMemoryError ?
synchronized ( ram ) {
2007-02-26 16:49:23 +01:00
if ( ram . size ( ) = = 0 ) break ;
2007-03-11 00:22:37 +01:00
if ( serverMemory . available ( ) < collections . minMem ( ) ) break ; // protect memory during flush
2007-11-12 02:14:51 +01:00
2007-02-26 16:49:23 +01:00
// select one word to flush
2006-09-14 02:51:02 +02:00
wordHash = ram . bestFlushWordHash ( ) ;
2006-08-11 18:27:56 +02:00
2007-02-26 16:49:23 +01:00
// move one container from ram to flush list
2007-02-28 12:13:23 +01:00
c = ram . deleteContainer ( wordHash ) ;
2006-08-07 23:49:39 +02:00
}
2007-11-12 02:14:51 +01:00
if ( c ! = null ) containerList . add ( c ) ;
2006-02-25 09:42:45 +01:00
}
2007-03-03 01:55:51 +01:00
// flush the containers
collections . addMultipleEntries ( containerList ) ;
2006-08-23 13:36:09 +02:00
//System.out.println("DEBUG-Finished flush of " + count + " entries from RAM to DB in " + (System.currentTimeMillis() - start) + " milliseconds");
2006-08-21 03:29:26 +02:00
busyCacheFlush = false ;
2006-02-25 09:42:45 +01:00
}
2005-11-07 13:33:02 +01:00
private static final int hour = 3600000 ;
private static final int day = 86400000 ;
public static int microDateDays ( Date modified ) {
2006-01-19 15:13:39 +01:00
return microDateDays ( modified . getTime ( ) ) ;
2005-11-07 13:33:02 +01:00
}
2006-01-19 15:13:39 +01:00
public static int microDateDays ( long modified ) {
// this calculates a virtual age from a given date
// the purpose is to have an age in days of a given modified date
// from a fixed standpoint in the past
// one day has 60*60*24 seconds = 86400 seconds
// we take mod 64**3 = 262144, this is the mask of the storage
return ( int ) ( ( modified / day ) % 262144 ) ;
}
2005-11-07 13:33:02 +01:00
public static String microDateHoursStr ( long time ) {
2006-01-19 15:13:39 +01:00
return kelondroBase64Order . enhancedCoder . encodeLong ( microDateHoursInt ( time ) , 3 ) ;
2005-11-07 13:33:02 +01:00
}
public static int microDateHoursInt ( long time ) {
2006-01-19 15:13:39 +01:00
return ( int ) ( ( time / hour ) % 262144 ) ;
2005-11-07 13:33:02 +01:00
}
public static int microDateHoursAge ( String mdhs ) {
2006-01-04 01:39:00 +01:00
return microDateHoursInt ( System . currentTimeMillis ( ) ) - ( int ) kelondroBase64Order . enhancedCoder . decodeLong ( mdhs ) ;
2005-10-13 02:05:30 +02:00
}
2006-01-19 15:13:39 +01:00
public static long reverseMicroDateDays ( int microDateDays ) {
return ( ( long ) microDateDays ) * ( ( long ) day ) ;
}
2007-09-05 11:01:35 +02:00
public int addPageIndex ( yacyURL url , Date urlModified , int size , plasmaParserDocument document , plasmaCondenser condenser , String language , char doctype , int outlinksSame , int outlinksOther ) {
2005-10-13 02:05:30 +02:00
// this is called by the switchboard to put in a new page into the index
2006-01-30 01:42:38 +01:00
// use all the words in one condenser object to simultanous create index entries
2006-12-08 03:14:56 +01:00
int wordCount = 0 ;
2007-07-19 17:32:10 +02:00
int urlLength = url . toNormalform ( true , true ) . length ( ) ;
2006-12-08 03:14:56 +01:00
int urlComps = htmlFilterContentScraper . urlComps ( url . toString ( ) ) . length ;
// iterate over all words of context text
2008-01-11 01:12:01 +01:00
Iterator < Map . Entry < String , plasmaCondenser . wordStatProp > > i = condenser . words ( ) . entrySet ( ) . iterator ( ) ;
Map . Entry < String , plasmaCondenser . wordStatProp > wentry ;
2006-01-19 15:13:39 +01:00
String word ;
2008-02-19 15:13:35 +01:00
indexRWIRowEntry ientry ;
2006-01-22 01:07:00 +01:00
plasmaCondenser . wordStatProp wprop ;
2006-01-19 15:13:39 +01:00
while ( i . hasNext ( ) ) {
2008-01-11 01:12:01 +01:00
wentry = i . next ( ) ;
word = wentry . getKey ( ) ;
wprop = wentry . getValue ( ) ;
2006-12-08 03:14:56 +01:00
assert ( wprop . flags ! = null ) ;
2007-11-07 23:38:09 +01:00
ientry = new indexRWIRowEntry ( url . hash ( ) ,
2008-01-22 12:51:43 +01:00
urlLength , urlComps , ( document = = null ) ? urlLength : document . dc_title ( ) . length ( ) ,
2006-11-19 21:05:25 +01:00
wprop . count ,
2006-12-19 04:10:46 +01:00
condenser . words ( ) . size ( ) ,
condenser . sentences ( ) . size ( ) ,
2006-11-19 21:05:25 +01:00
wprop . posInText ,
wprop . posInPhrase ,
wprop . numOfPhrase ,
urlModified . getTime ( ) ,
System . currentTimeMillis ( ) ,
language ,
doctype ,
outlinksSame , outlinksOther ,
2008-02-19 15:13:35 +01:00
wprop . flags ) ;
2006-12-08 03:14:56 +01:00
addEntry ( plasmaCondenser . word2hash ( word ) , ientry , System . currentTimeMillis ( ) , false ) ;
wordCount + + ;
2006-01-19 15:13:39 +01:00
}
2006-12-08 03:14:56 +01:00
return wordCount ;
2005-10-13 02:05:30 +02:00
}
2006-01-30 01:42:38 +01:00
2007-01-08 14:13:30 +01:00
public boolean hasContainer ( String wordHash ) {
if ( dhtOutCache . hasContainer ( wordHash ) ) return true ;
if ( dhtInCache . hasContainer ( wordHash ) ) return true ;
if ( collections . hasContainer ( wordHash ) ) return true ;
return false ;
}
2008-01-11 01:12:01 +01:00
public indexContainer getContainer ( String wordHash , Set < String > urlselection ) {
2007-05-14 12:00:23 +02:00
if ( ( wordHash = = null ) | | ( wordHash . length ( ) ! = yacySeedDB . commonHashLength ) ) {
// wrong input
return null ;
}
2006-11-19 21:05:25 +01:00
// get from cache
2007-04-05 16:58:29 +02:00
indexContainer container ;
2008-03-15 01:49:16 +01:00
container = dhtOutCache . getContainer ( wordHash , urlselection ) ;
if ( container = = null ) {
container = dhtInCache . getContainer ( wordHash , urlselection ) ;
} else {
container . addAllUnique ( dhtInCache . getContainer ( wordHash , urlselection ) ) ;
2007-04-05 16:58:29 +02:00
}
2006-11-19 21:05:25 +01:00
// get from collection index
2008-03-15 01:49:16 +01:00
if ( container = = null ) {
container = collections . getContainer ( wordHash , urlselection ) ;
} else {
container . addAllUnique ( collections . getContainer ( wordHash , urlselection ) ) ;
2006-11-19 21:05:25 +01:00
}
2008-02-03 13:40:40 +01:00
2008-02-03 22:47:27 +01:00
if ( container = = null ) return null ;
2008-02-03 13:40:40 +01:00
// check doubles
int beforeDouble = container . size ( ) ;
ArrayList < kelondroRowSet > d = container . removeDoubles ( ) ;
kelondroRowSet set ;
for ( int i = 0 ; i < d . size ( ) ; i + + ) {
// for each element in the double-set, take that one that is the most recent one
set = d . get ( i ) ;
indexRWIRowEntry e , elm = null ;
long lm = 0 ;
for ( int j = 0 ; j < set . size ( ) ; j + + ) {
e = new indexRWIRowEntry ( set . get ( j ) ) ;
if ( ( elm = = null ) | | ( e . lastModified ( ) > lm ) ) {
elm = e ;
lm = e . lastModified ( ) ;
}
}
container . addUnique ( elm . toKelondroEntry ( ) ) ;
}
if ( container . size ( ) < beforeDouble ) System . out . println ( " *** DEBUG DOUBLECHECK - removed " + ( beforeDouble - container . size ( ) ) + " index entries from word container " + container . getWordHash ( ) ) ;
2006-11-19 21:05:25 +01:00
return container ;
2006-01-30 01:42:38 +01:00
}
2006-01-30 13:42:06 +01:00
2008-02-27 16:16:47 +01:00
public HashMap < String , indexContainer > getContainers ( Set < String > wordHashes , Set < String > urlselection , boolean deleteIfEmpty , boolean interruptIfEmpty ) {
2006-09-11 13:12:42 +02:00
// return map of wordhash:indexContainer
2006-01-30 01:42:38 +01:00
// retrieve entities that belong to the hashes
2008-02-27 16:16:47 +01:00
HashMap < String , indexContainer > containers = new HashMap < String , indexContainer > ( wordHashes . size ( ) ) ;
2006-01-30 01:42:38 +01:00
String singleHash ;
2006-05-28 03:09:31 +02:00
indexContainer singleContainer ;
2007-12-27 18:56:59 +01:00
Iterator < String > i = wordHashes . iterator ( ) ;
2006-08-12 00:07:59 +02:00
while ( i . hasNext ( ) ) {
2006-01-30 01:42:38 +01:00
2006-09-11 13:12:42 +02:00
// get next word hash:
2006-08-12 00:07:59 +02:00
singleHash = ( String ) i . next ( ) ;
2006-01-30 01:42:38 +01:00
2006-08-12 00:07:59 +02:00
// retrieve index
2007-09-04 01:43:55 +02:00
singleContainer = getContainer ( singleHash , urlselection ) ;
2006-01-30 01:42:38 +01:00
2006-08-12 00:07:59 +02:00
// check result
2008-02-27 16:16:47 +01:00
if ( ( ( singleContainer = = null ) | | ( singleContainer . size ( ) = = 0 ) ) & & ( interruptIfEmpty ) ) return new HashMap < String , indexContainer > ( 0 ) ;
2006-01-30 01:42:38 +01:00
2006-09-11 00:36:47 +02:00
containers . put ( singleHash , singleContainer ) ;
2006-08-12 00:07:59 +02:00
}
2006-01-30 01:42:38 +01:00
return containers ;
}
2008-01-19 01:40:19 +01:00
@SuppressWarnings ( " unchecked " )
2008-02-27 16:16:47 +01:00
public HashMap < String , indexContainer > [ ] localSearchContainers ( plasmaSearchQuery query , Set < String > urlselection ) {
2007-11-17 02:53:02 +01:00
// search for the set of hashes and return a map of of wordhash:indexContainer containing the seach result
// retrieve entities that belong to the hashes
2008-02-27 16:16:47 +01:00
HashMap < String , indexContainer > inclusionContainers = ( query . queryHashes . size ( ) = = 0 ) ? new HashMap < String , indexContainer > ( 0 ) : getContainers (
2007-11-17 02:53:02 +01:00
query . queryHashes ,
urlselection ,
true ,
true ) ;
2008-02-27 16:16:47 +01:00
if ( ( inclusionContainers . size ( ) ! = 0 ) & & ( inclusionContainers . size ( ) < query . queryHashes . size ( ) ) ) inclusionContainers = new HashMap < String , indexContainer > ( 0 ) ; // prevent that only a subset is returned
HashMap < String , indexContainer > exclusionContainers = ( inclusionContainers . size ( ) = = 0 ) ? new HashMap < String , indexContainer > ( 0 ) : getContainers (
2007-11-17 02:53:02 +01:00
query . excludeHashes ,
urlselection ,
true ,
true ) ;
2008-02-27 16:16:47 +01:00
return new HashMap [ ] { inclusionContainers , exclusionContainers } ;
2007-11-17 02:53:02 +01:00
}
2005-05-07 23:11:18 +02:00
public int size ( ) {
2006-12-02 20:34:59 +01:00
return java . lang . Math . max ( collections . size ( ) , java . lang . Math . max ( dhtInCache . size ( ) , dhtOutCache . size ( ) ) ) ;
2005-04-07 21:19:42 +02:00
}
2005-09-20 17:43:31 +02:00
2006-02-14 01:12:07 +01:00
public int indexSize ( String wordHash ) {
int size = 0 ;
2006-11-19 21:05:25 +01:00
size + = dhtInCache . indexSize ( wordHash ) ;
size + = dhtOutCache . indexSize ( wordHash ) ;
2006-12-02 20:34:59 +01:00
size + = collections . indexSize ( wordHash ) ;
2006-02-14 01:12:07 +01:00
return size ;
2005-04-07 21:19:42 +02:00
}
2005-09-20 17:43:31 +02:00
2006-12-05 03:47:51 +01:00
public void close ( ) {
2007-04-26 17:15:40 +02:00
dhtInCache . close ( ) ;
dhtOutCache . close ( ) ;
2008-03-15 01:49:16 +01:00
collections . close ( ) ;
loadedURL . close ( ) ;
2005-04-07 21:19:42 +02:00
}
2005-09-20 17:43:31 +02:00
2006-08-12 01:04:36 +02:00
public indexContainer deleteContainer ( String wordHash ) {
2007-08-25 01:12:59 +02:00
indexContainer c = new indexContainer (
wordHash ,
2007-11-07 23:38:09 +01:00
indexRWIRowEntry . urlEntryRow ,
2007-08-25 01:12:59 +02:00
dhtInCache . sizeContainer ( wordHash ) + dhtOutCache . sizeContainer ( wordHash ) + collections . indexSize ( wordHash )
) ;
2008-03-15 01:49:16 +01:00
c . addAllUnique ( dhtInCache . deleteContainer ( wordHash ) ) ;
c . addAllUnique ( dhtOutCache . deleteContainer ( wordHash ) ) ;
c . addAllUnique ( collections . deleteContainer ( wordHash ) ) ;
2006-11-19 21:05:25 +01:00
return c ;
2005-04-07 21:19:42 +02:00
}
2006-02-14 01:12:07 +01:00
2006-12-06 13:51:46 +01:00
public boolean removeEntry ( String wordHash , String urlHash ) {
2006-09-18 02:37:02 +02:00
boolean removed = false ;
2008-03-15 01:49:16 +01:00
removed = removed | ( dhtInCache . removeEntry ( wordHash , urlHash ) ) ;
removed = removed | ( dhtOutCache . removeEntry ( wordHash , urlHash ) ) ;
removed = removed | ( collections . removeEntry ( wordHash , urlHash ) ) ;
2006-11-19 21:05:25 +01:00
return removed ;
2006-08-01 12:30:55 +02:00
}
2007-12-28 04:39:36 +01:00
public int removeEntryMultiple ( Set < String > wordHashes , String urlHash ) {
2007-08-26 20:18:35 +02:00
// remove the same url hashes for multiple words
// this is mainly used when correcting a index after a search
2008-01-11 01:12:01 +01:00
Iterator < String > i = wordHashes . iterator ( ) ;
2007-08-26 20:18:35 +02:00
int count = 0 ;
while ( i . hasNext ( ) ) {
if ( removeEntry ( ( String ) i . next ( ) , urlHash ) ) count + + ;
}
return count ;
}
2007-12-28 04:39:36 +01:00
public int removeEntries ( String wordHash , Set < String > urlHashes ) {
2006-08-16 21:49:31 +02:00
int removed = 0 ;
2008-03-15 01:49:16 +01:00
removed + = dhtInCache . removeEntries ( wordHash , urlHashes ) ;
removed + = dhtOutCache . removeEntries ( wordHash , urlHashes ) ;
removed + = collections . removeEntries ( wordHash , urlHashes ) ;
2006-10-13 01:14:41 +02:00
return removed ;
}
2007-12-28 04:39:36 +01:00
public String removeEntriesExpl ( String wordHash , Set < String > urlHashes ) {
2006-10-13 01:14:41 +02:00
String removed = " " ;
2008-03-15 01:49:16 +01:00
removed + = dhtInCache . removeEntries ( wordHash , urlHashes ) + " , " ;
removed + = dhtOutCache . removeEntries ( wordHash , urlHashes ) + " , " ;
removed + = collections . removeEntries ( wordHash , urlHashes ) ;
2006-02-14 01:12:07 +01:00
return removed ;
}
2007-12-28 04:39:36 +01:00
public void removeEntriesMultiple ( Set < String > wordHashes , Set < String > urlHashes ) {
2007-08-26 20:18:35 +02:00
// remove the same url hashes for multiple words
// this is mainly used when correcting a index after a search
2008-01-11 01:12:01 +01:00
Iterator < String > i = wordHashes . iterator ( ) ;
2007-08-26 20:18:35 +02:00
while ( i . hasNext ( ) ) {
removeEntries ( ( String ) i . next ( ) , urlHashes ) ;
2007-03-13 23:18:36 +01:00
}
}
2007-12-28 04:39:36 +01:00
public int removeWordReferences ( Set < String > words , String urlhash ) {
2007-03-13 23:18:36 +01:00
// sequentially delete all word references
// returns number of deletions
2008-01-11 01:12:01 +01:00
Iterator < String > iter = words . iterator ( ) ;
2007-03-13 23:18:36 +01:00
int count = 0 ;
while ( iter . hasNext ( ) ) {
// delete the URL reference in this word index
2007-08-26 20:18:35 +02:00
if ( removeEntry ( plasmaCondenser . word2hash ( ( String ) iter . next ( ) ) , urlhash ) ) count + + ;
2006-12-06 13:51:46 +01:00
}
return count ;
}
2006-09-14 02:51:02 +02:00
2006-04-20 16:00:47 +02:00
public int tryRemoveURLs ( String urlHash ) {
2006-03-09 11:46:02 +01:00
// this tries to delete an index from the cache that has this
// urlHash assigned. This can only work if the entry is really fresh
// and can be found in the RAM cache
// this returns the number of deletion that had been possible
2007-04-03 14:10:12 +02:00
int d = 0 ;
2007-04-26 17:15:40 +02:00
d = dhtInCache . tryRemoveURLs ( urlHash ) ;
if ( d > 0 ) return d ; else return dhtOutCache . tryRemoveURLs ( urlHash ) ;
2006-03-09 11:46:02 +01:00
}
2008-01-11 01:12:01 +01:00
public synchronized TreeSet < indexContainer > indexContainerSet ( String startHash , boolean ram , boolean rot , int count ) {
2006-07-26 13:21:51 +02:00
// creates a set of indexContainers
2006-09-14 02:51:02 +02:00
// this does not use the dhtInCache
2008-01-11 01:12:01 +01:00
kelondroOrder < indexContainer > containerOrder = new indexContainerOrder ( indexOrder . clone ( ) ) ;
containerOrder . rotate ( emptyContainer ( startHash , 0 ) ) ;
TreeSet < indexContainer > containers = new TreeSet < indexContainer > ( containerOrder ) ;
Iterator < indexContainer > i = wordContainers ( startHash , ram , rot ) ;
2006-12-05 03:47:51 +01:00
if ( ram ) count = Math . min ( dhtOutCache . size ( ) , count ) ;
2006-09-14 02:51:02 +02:00
indexContainer container ;
2007-04-13 15:40:19 +02:00
// this loop does not terminate using the i.hasNex() predicate when rot == true
// because then the underlying iterator is a rotating iterator without termination
// in this case a termination must be ensured with a counter
// It must also be ensured that the counter is in/decreased every loop
2006-09-14 02:51:02 +02:00
while ( ( count > 0 ) & & ( i . hasNext ( ) ) ) {
2008-01-11 01:12:01 +01:00
container = i . next ( ) ;
2006-09-14 02:51:02 +02:00
if ( ( container ! = null ) & & ( container . size ( ) > 0 ) ) {
containers . add ( container ) ;
2006-03-18 17:14:24 +01:00
}
2007-04-13 15:40:19 +02:00
count - - ; // decrease counter even if the container was null or empty to ensure termination
2006-09-14 02:51:02 +02:00
}
2007-04-13 15:40:19 +02:00
return containers ; // this may return less containers as demanded
2006-03-15 00:22:49 +01:00
}
2006-12-05 03:47:51 +01:00
2008-01-11 01:12:01 +01:00
public synchronized kelondroCloneableIterator < indexContainer > wordContainers ( String startHash , boolean ram , boolean rot ) {
kelondroCloneableIterator < indexContainer > i = wordContainers ( startHash , ram ) ;
2007-03-08 17:15:40 +01:00
if ( rot ) {
2008-01-25 12:44:27 +01:00
return new kelondroRotateIterator < indexContainer > ( i , new String ( kelondroBase64Order . zero ( startHash . length ( ) ) ) , dhtOutCache . size ( ) + ( ( ram ) ? 0 : collections . size ( ) ) ) ;
2007-03-08 17:15:40 +01:00
} else {
return i ;
}
2006-02-14 01:12:07 +01:00
}
2008-01-11 01:12:01 +01:00
public synchronized kelondroCloneableIterator < indexContainer > wordContainers ( String startWordHash , boolean ram ) {
kelondroOrder < indexContainer > containerOrder = new indexContainerOrder ( indexOrder . clone ( ) ) ;
containerOrder . rotate ( emptyContainer ( startWordHash , 0 ) ) ;
2006-12-05 03:47:51 +01:00
if ( ram ) {
2006-09-14 02:51:02 +02:00
return dhtOutCache . wordContainers ( startWordHash , false ) ;
2006-12-05 03:47:51 +01:00
} else {
2008-01-11 01:12:01 +01:00
return new kelondroMergeIterator < indexContainer > (
2006-09-14 02:51:02 +02:00
dhtOutCache . wordContainers ( startWordHash , false ) ,
2006-08-06 00:22:14 +02:00
collections . wordContainers ( startWordHash , false ) ,
2006-11-21 01:36:41 +01:00
containerOrder ,
2006-09-12 13:13:27 +02:00
indexContainer . containerMergeMethod ,
2006-08-06 00:22:14 +02:00
true ) ;
2006-12-05 03:47:51 +01:00
}
2005-04-07 21:19:42 +02:00
}
2006-02-12 00:19:01 +01:00
2006-03-08 23:06:11 +01:00
// The Cleaner class was provided as "UrldbCleaner" by Hydrox
2007-04-26 17:15:40 +02:00
public synchronized Cleaner makeCleaner ( plasmaCrawlLURL lurl , String startHash ) {
2006-03-08 23:06:11 +01:00
return new Cleaner ( lurl , startHash ) ;
}
public class Cleaner extends Thread {
private String startHash ;
private boolean run = true ;
private boolean pause = false ;
public int rwiCountAtStart = 0 ;
public String wordHashNow = " " ;
public String lastWordHash = " " ;
public int lastDeletionCounter = 0 ;
private plasmaCrawlLURL lurl ;
public Cleaner ( plasmaCrawlLURL lurl , String startHash ) {
this . lurl = lurl ;
this . startHash = startHash ;
this . rwiCountAtStart = size ( ) ;
}
public void run ( ) {
2006-03-09 13:23:05 +01:00
serverLog . logInfo ( " INDEXCLEANER " , " IndexCleaner-Thread started " ) ;
2006-07-26 13:21:51 +02:00
indexContainer container = null ;
2008-01-30 01:15:43 +01:00
indexRWIRowEntry entry = null ;
2007-09-05 11:01:35 +02:00
yacyURL url = null ;
2008-01-11 01:12:01 +01:00
HashSet < String > urlHashs = new HashSet < String > ( ) ;
Iterator < indexContainer > indexContainerIterator = indexContainerSet ( startHash , false , false , 100 ) . iterator ( ) ;
2006-12-02 20:34:59 +01:00
while ( indexContainerIterator . hasNext ( ) & & run ) {
waiter ( ) ;
container = ( indexContainer ) indexContainerIterator . next ( ) ;
2008-01-11 01:12:01 +01:00
Iterator < indexRWIRowEntry > containerIterator = container . entries ( ) ;
2006-12-02 20:34:59 +01:00
wordHashNow = container . getWordHash ( ) ;
while ( containerIterator . hasNext ( ) & & run ) {
2006-03-08 23:06:11 +01:00
waiter ( ) ;
2008-01-30 01:15:43 +01:00
entry = containerIterator . next ( ) ;
2006-12-02 20:34:59 +01:00
// System.out.println("Wordhash: "+wordHash+" UrlHash:
// "+entry.getUrlHash());
2007-11-16 15:48:09 +01:00
indexURLEntry ue = lurl . load ( entry . urlHash ( ) , entry , 0 ) ;
2006-12-02 20:34:59 +01:00
if ( ue = = null ) {
urlHashs . add ( entry . urlHash ( ) ) ;
} else {
url = ue . comp ( ) . url ( ) ;
if ( ( url = = null ) | | ( plasmaSwitchboard . urlBlacklist . isListed ( plasmaURLPattern . BLACKLIST_CRAWLER , url ) = = true ) ) {
2006-09-07 20:24:39 +02:00
urlHashs . add ( entry . urlHash ( ) ) ;
2006-03-08 23:06:11 +01:00
}
2006-03-17 21:52:43 +01:00
}
2006-12-02 20:34:59 +01:00
}
if ( urlHashs . size ( ) > 0 ) {
2006-12-06 13:51:46 +01:00
int removed = removeEntries ( container . getWordHash ( ) , urlHashs ) ;
2006-12-02 20:34:59 +01:00
serverLog . logFine ( " INDEXCLEANER " , container . getWordHash ( ) + " : " + removed + " of " + container . size ( ) + " URL-entries deleted " ) ;
lastWordHash = container . getWordHash ( ) ;
lastDeletionCounter = urlHashs . size ( ) ;
urlHashs . clear ( ) ;
}
if ( ! containerIterator . hasNext ( ) ) {
// We may not be finished yet, try to get the next chunk of wordHashes
2008-01-11 01:12:01 +01:00
TreeSet < indexContainer > containers = indexContainerSet ( container . getWordHash ( ) , false , false , 100 ) ;
2006-12-02 20:34:59 +01:00
indexContainerIterator = containers . iterator ( ) ;
// Make sure we don't get the same wordhash twice, but don't skip a word
if ( ( indexContainerIterator . hasNext ( ) ) & & ( ! container . getWordHash ( ) . equals ( ( ( indexContainer ) indexContainerIterator . next ( ) ) . getWordHash ( ) ) ) ) {
2006-07-26 13:21:51 +02:00
indexContainerIterator = containers . iterator ( ) ;
2006-03-18 00:39:10 +01:00
}
2006-03-08 23:06:11 +01:00
}
}
serverLog . logInfo ( " INDEXCLEANER " , " IndexCleaner-Thread stopped " ) ;
}
public void abort ( ) {
synchronized ( this ) {
run = false ;
this . notifyAll ( ) ;
}
}
public void pause ( ) {
2006-12-22 20:17:23 +01:00
synchronized ( this ) {
if ( ! pause ) {
2006-03-08 23:06:11 +01:00
pause = true ;
2006-12-22 20:17:23 +01:00
serverLog . logInfo ( " INDEXCLEANER " , " IndexCleaner-Thread paused " ) ;
2006-03-08 23:06:11 +01:00
}
}
}
public void endPause ( ) {
2006-12-22 20:17:23 +01:00
synchronized ( this ) {
if ( pause ) {
2006-03-08 23:06:11 +01:00
pause = false ;
this . notifyAll ( ) ;
serverLog . logInfo ( " INDEXCLEANER " , " IndexCleaner-Thread resumed " ) ;
}
}
}
2006-12-22 20:17:23 +01:00
2006-03-08 23:06:11 +01:00
public void waiter ( ) {
2006-12-22 20:17:23 +01:00
synchronized ( this ) {
2006-03-08 23:06:11 +01:00
if ( this . pause ) {
try {
this . wait ( ) ;
} catch ( InterruptedException e ) {
this . run = false ;
return ;
}
}
}
}
}
2007-04-26 17:15:40 +02:00
2005-04-07 21:19:42 +02:00
}