2007-08-06 02:56:56 +02:00
// plasmaSearchEvent.java
// (C) 2005 by Michael Peter Christen; mc@yacy.net, Frankfurt a. M., Germany
// first published 10.10.2005 on http://yacy.net
2005-10-10 02:32:15 +02:00
//
2007-08-06 02:56:56 +02:00
// This is a part of YaCy, a peer-to-peer based web search engine
//
// $LastChangedDate: 2006-04-02 22:40:07 +0200 (So, 02 Apr 2006) $
// $LastChangedRevision: 1986 $
// $LastChangedBy: orbiter $
//
// LICENSE
//
2005-10-10 02:32:15 +02:00
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation; either version 2 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
package de.anomic.plasma ;
2007-08-28 14:15:46 +02:00
import java.io.UnsupportedEncodingException ;
2007-08-26 20:18:35 +02:00
import java.util.ArrayList ;
2007-08-28 14:15:46 +02:00
import java.util.Date ;
2007-08-25 01:12:59 +02:00
import java.util.HashMap ;
2005-10-11 09:06:33 +02:00
import java.util.Iterator ;
2006-09-11 00:36:47 +02:00
import java.util.Map ;
2007-09-07 13:45:38 +02:00
import java.util.Set ;
2006-09-11 12:39:25 +02:00
import java.util.TreeMap ;
2007-08-28 14:15:46 +02:00
import java.util.TreeSet ;
2005-10-12 14:28:49 +02:00
2006-09-30 00:27:20 +02:00
import de.anomic.index.indexContainer ;
2007-08-26 20:18:35 +02:00
import de.anomic.index.indexRWIEntry ;
2008-02-21 11:06:57 +01:00
import de.anomic.index.indexRWIVarEntry ;
2007-08-28 14:15:46 +02:00
import de.anomic.index.indexURLEntry ;
import de.anomic.kelondro.kelondroBitfield ;
2006-09-12 13:13:27 +02:00
import de.anomic.kelondro.kelondroMSetTools ;
2008-02-21 11:06:57 +01:00
import de.anomic.kelondro.kelondroSortStack ;
import de.anomic.kelondro.kelondroSortStore ;
2008-01-18 18:14:02 +01:00
import de.anomic.plasma.plasmaSnippetCache.MediaSnippet ;
2007-11-17 02:53:02 +01:00
import de.anomic.server.serverProfiling ;
2005-10-12 14:28:49 +02:00
import de.anomic.server.logging.serverLog ;
2006-09-16 02:07:09 +02:00
import de.anomic.yacy.yacyCore ;
2007-08-28 14:15:46 +02:00
import de.anomic.yacy.yacyDHTAction ;
2005-10-13 15:57:15 +02:00
import de.anomic.yacy.yacySearch ;
2007-08-28 14:15:46 +02:00
import de.anomic.yacy.yacySeed ;
2007-09-05 11:01:35 +02:00
import de.anomic.yacy.yacyURL ;
2005-10-10 02:32:15 +02:00
2007-08-15 13:36:59 +02:00
public final class plasmaSearchEvent {
2005-10-10 02:32:15 +02:00
2007-11-17 02:53:02 +01:00
public static final String COLLECTION = " collection " ;
public static final String JOIN = " join " ;
public static final String PRESORT = " presort " ;
public static final String URLFETCH = " urlfetch " ;
public static final String NORMALIZING = " normalizing " ;
2008-02-02 01:16:00 +01:00
public static int workerThreadCount = 10 ;
2007-08-26 20:18:35 +02:00
public static String lastEventID = " " ;
2008-01-08 21:12:31 +01:00
private static HashMap < String , plasmaSearchEvent > lastEvents = new HashMap < String , plasmaSearchEvent > ( ) ; // a cache for objects from this class: re-use old search requests
2007-08-25 01:12:59 +02:00
public static final long eventLifetime = 600000 ; // the time an event will stay in the cache, 10 Minutes
2007-11-07 23:38:09 +01:00
private static final int max_results_preparation = 200 ;
2007-08-25 01:12:59 +02:00
private long eventTime ;
2008-03-04 23:45:45 +01:00
plasmaSearchQuery query ;
2005-10-12 14:28:49 +02:00
private plasmaWordIndex wordIndex ;
2008-03-04 23:45:45 +01:00
plasmaSearchRankingProcess rankedCache ; // ordered search results, grows dynamically as all the query threads enrich this container
2008-01-17 19:43:01 +01:00
private Map < String , TreeMap < String , String > > rcAbstracts ; // cache for index abstracts; word:TreeMap mapping where the embedded TreeMap is a urlhash:peerlist relation
2006-09-13 19:13:28 +02:00
private yacySearch [ ] primarySearchThreads , secondarySearchThreads ;
2007-09-28 03:36:22 +02:00
private Thread localSearchThread ;
2008-01-18 18:14:02 +01:00
private TreeMap < String , String > preselectedPeerHashes ;
2007-09-07 13:45:38 +02:00
//private Object[] references;
2008-01-17 19:43:01 +01:00
public TreeMap < String , String > IAResults ;
2008-01-11 01:12:01 +01:00
public TreeMap < String , Integer > IACount ;
2007-08-28 14:15:46 +02:00
public String IAmaxcounthash , IAneardhthash ;
2007-09-04 01:43:55 +02:00
private resultWorker [ ] workerThreads ;
2008-03-04 23:45:45 +01:00
kelondroSortStore < ResultEntry > result ;
kelondroSortStore < plasmaSnippetCache . MediaSnippet > images ; // container to sort images by size
HashMap < String , String > failedURLs ; // a mapping from a urlhash to a fail reason string
2008-01-18 18:14:02 +01:00
TreeSet < String > snippetFetchWordHashes ; // a set of word hashes that are used to match with the snippets
2008-03-04 23:45:45 +01:00
long urlRetrievalAllTime ;
long snippetComputationAllTime ;
2005-10-12 14:28:49 +02:00
2008-01-18 18:14:02 +01:00
@SuppressWarnings ( " unchecked " )
2007-08-25 01:12:59 +02:00
private plasmaSearchEvent ( plasmaSearchQuery query ,
2008-01-18 19:43:56 +01:00
plasmaWordIndex wordIndex ,
2008-01-18 18:14:02 +01:00
TreeMap < String , String > preselectedPeerHashes ,
2008-01-18 19:43:56 +01:00
boolean generateAbstracts ) {
2007-08-25 01:12:59 +02:00
this . eventTime = System . currentTimeMillis ( ) ; // for lifetime check
2005-10-12 14:28:49 +02:00
this . wordIndex = wordIndex ;
2005-10-11 09:06:33 +02:00
this . query = query ;
2008-01-17 19:43:01 +01:00
this . rcAbstracts = ( query . queryHashes . size ( ) > 1 ) ? new TreeMap < String , TreeMap < String , String > > ( ) : null ; // generate abstracts only for combined searches
2006-09-13 19:13:28 +02:00
this . primarySearchThreads = null ;
this . secondarySearchThreads = null ;
2007-04-26 11:51:51 +02:00
this . preselectedPeerHashes = preselectedPeerHashes ;
2008-01-17 19:43:01 +01:00
this . IAResults = new TreeMap < String , String > ( ) ;
this . IACount = new TreeMap < String , Integer > ( ) ;
2007-08-28 14:15:46 +02:00
this . IAmaxcounthash = null ;
this . IAneardhthash = null ;
2007-09-26 12:11:50 +02:00
this . urlRetrievalAllTime = 0 ;
this . snippetComputationAllTime = 0 ;
2007-09-04 01:43:55 +02:00
this . workerThreads = null ;
2008-02-01 00:40:47 +01:00
this . localSearchThread = null ;
2008-02-21 11:06:57 +01:00
this . result = new kelondroSortStore < ResultEntry > ( - 1 ) ; // this is the result, enriched with snippets, ranked and ordered by ranking
2008-02-21 15:53:51 +01:00
this . images = new kelondroSortStore < plasmaSnippetCache . MediaSnippet > ( - 1 ) ;
2008-01-18 18:14:02 +01:00
this . failedURLs = new HashMap < String , String > ( ) ; // a map of urls to reason strings where a worker thread tried to work on, but failed.
2007-09-04 01:43:55 +02:00
// snippets do not need to match with the complete query hashes,
// only with the query minus the stopwords which had not been used for the search
2008-01-18 18:14:02 +01:00
final TreeSet < String > filtered = kelondroMSetTools . joinConstructive ( query . queryHashes , plasmaSwitchboard . stopwords ) ;
this . snippetFetchWordHashes = ( TreeSet < String > ) query . queryHashes . clone ( ) ;
2007-09-04 01:43:55 +02:00
if ( ( filtered ! = null ) & & ( filtered . size ( ) > 0 ) ) {
kelondroMSetTools . excludeDestructive ( this . snippetFetchWordHashes , plasmaSwitchboard . stopwords ) ;
}
2007-08-25 01:12:59 +02:00
long start = System . currentTimeMillis ( ) ;
if ( ( query . domType = = plasmaSearchQuery . SEARCHDOM_GLOBALDHT ) | |
( query . domType = = plasmaSearchQuery . SEARCHDOM_CLUSTERALL ) ) {
2007-11-17 02:53:02 +01:00
// do a global search
2008-03-05 22:46:55 +01:00
this . rankedCache = new plasmaSearchRankingProcess ( wordIndex , query , max_results_preparation , 16 ) ;
2007-09-04 01:43:55 +02:00
2008-02-03 00:53:39 +01:00
int fetchpeers = 30 ;
2007-08-25 01:12:59 +02:00
// the result of the fetch is then in the rcGlobal
2007-12-04 21:19:13 +01:00
long timer = System . currentTimeMillis ( ) ;
2007-09-04 01:43:55 +02:00
serverLog . logFine ( " SEARCH_EVENT " , " STARTING " + fetchpeers + " THREADS TO CATCH EACH " + query . displayResults ( ) + " URLs " ) ;
2007-09-08 13:50:19 +02:00
this . primarySearchThreads = yacySearch . primaryRemoteSearches (
2007-08-25 01:12:59 +02:00
plasmaSearchQuery . hashSet2hashString ( query . queryHashes ) ,
plasmaSearchQuery . hashSet2hashString ( query . excludeHashes ) ,
" " ,
query . prefer ,
query . urlMask ,
2007-09-04 01:43:55 +02:00
query . displayResults ( ) ,
2007-08-25 01:12:59 +02:00
query . maxDistance ,
wordIndex ,
2007-09-04 01:43:55 +02:00
rankedCache ,
2007-08-25 01:12:59 +02:00
rcAbstracts ,
fetchpeers ,
plasmaSwitchboard . urlBlacklist ,
2008-01-08 21:12:31 +01:00
query . ranking ,
2007-08-25 01:12:59 +02:00
query . constraint ,
( query . domType = = plasmaSearchQuery . SEARCHDOM_GLOBALDHT ) ? null : preselectedPeerHashes ) ;
2007-12-12 19:57:43 +01:00
serverProfiling . update ( " SEARCH " , new plasmaProfiling . searchEvent ( query . id ( true ) , " remote search thread start " , this . primarySearchThreads . length , System . currentTimeMillis ( ) - timer ) ) ;
2007-09-04 01:43:55 +02:00
2007-08-25 01:12:59 +02:00
// meanwhile do a local search
2007-09-28 03:36:22 +02:00
localSearchThread = new localSearchProcess ( ) ;
localSearchThread . start ( ) ;
2007-09-04 01:43:55 +02:00
2007-08-25 01:12:59 +02:00
// finished searching
serverLog . logFine ( " SEARCH_EVENT " , " SEARCH TIME AFTER GLOBAL-TRIGGER TO " + primarySearchThreads . length + " PEERS: " + ( ( System . currentTimeMillis ( ) - start ) / 1000 ) + " seconds " ) ;
} else {
2007-11-17 02:53:02 +01:00
// do a local search
2008-03-05 22:46:55 +01:00
this . rankedCache = new plasmaSearchRankingProcess ( wordIndex , query , max_results_preparation , 2 ) ;
2008-01-30 22:58:30 +01:00
this . rankedCache . execQuery ( ) ;
2007-11-22 00:14:57 +01:00
//plasmaWordIndex.Finding finding = wordIndex.retrieveURLs(query, false, 2, ranking, process);
2007-08-25 01:12:59 +02:00
2007-08-28 14:15:46 +02:00
if ( generateAbstracts ) {
// compute index abstracts
2007-12-04 21:19:13 +01:00
long timer = System . currentTimeMillis ( ) ;
2008-01-18 18:14:02 +01:00
Iterator < Map . Entry < String , indexContainer > > ci = this . rankedCache . searchContainerMaps ( ) [ 0 ] . entrySet ( ) . iterator ( ) ;
Map . Entry < String , indexContainer > entry ;
2007-08-28 14:15:46 +02:00
int maxcount = - 1 ;
double mindhtdistance = 1 . 1 , d ;
String wordhash ;
while ( ci . hasNext ( ) ) {
2008-01-18 18:14:02 +01:00
entry = ci . next ( ) ;
wordhash = entry . getKey ( ) ;
indexContainer container = entry . getValue ( ) ;
2007-08-28 14:15:46 +02:00
assert ( container . getWordHash ( ) . equals ( wordhash ) ) ;
if ( container . size ( ) > maxcount ) {
IAmaxcounthash = wordhash ;
maxcount = container . size ( ) ;
}
2007-10-01 14:30:23 +02:00
d = yacyDHTAction . dhtDistance ( yacyCore . seedDB . mySeed ( ) . hash , wordhash ) ;
2007-08-28 14:15:46 +02:00
if ( d < mindhtdistance ) {
// calculate the word hash that is closest to our dht position
mindhtdistance = d ;
IAneardhthash = wordhash ;
}
IACount . put ( wordhash , new Integer ( container . size ( ) ) ) ;
2007-11-17 02:53:02 +01:00
IAResults . put ( wordhash , indexContainer . compressIndex ( container , null , 1000 ) . toString ( ) ) ;
2007-08-28 14:15:46 +02:00
}
2007-12-12 19:57:43 +01:00
serverProfiling . update ( " SEARCH " , new plasmaProfiling . searchEvent ( query . id ( true ) , " abstract generation " , this . rankedCache . searchContainerMaps ( ) [ 0 ] . size ( ) , System . currentTimeMillis ( ) - timer ) ) ;
2007-08-28 14:15:46 +02:00
}
2007-08-25 01:12:59 +02:00
}
2007-09-05 11:01:35 +02:00
2007-09-04 01:43:55 +02:00
if ( query . onlineSnippetFetch ) {
// start worker threads to fetch urls and snippets
this . workerThreads = new resultWorker [ workerThreadCount ] ;
for ( int i = 0 ; i < workerThreadCount ; i + + ) {
2008-02-03 00:53:39 +01:00
this . workerThreads [ i ] = new resultWorker ( i , 10000 ) ;
2007-09-04 01:43:55 +02:00
this . workerThreads [ i ] . start ( ) ;
}
} else {
// prepare result vector directly without worker threads
2007-12-04 21:19:13 +01:00
long timer = System . currentTimeMillis ( ) ;
2007-11-16 15:48:09 +01:00
indexURLEntry uentry ;
2007-11-07 23:38:09 +01:00
ResultEntry resultEntry ;
2007-11-23 10:26:35 +01:00
yacyURL url ;
2007-11-07 23:38:09 +01:00
synchronized ( rankedCache ) {
2008-02-21 11:06:57 +01:00
while ( ( rankedCache . size ( ) > 0 ) & & ( ( uentry = rankedCache . bestURL ( true ) ) ! = null ) & & ( result . size ( ) < ( query . neededResults ( ) ) ) ) {
2007-11-23 10:26:35 +01:00
url = uentry . comp ( ) . url ( ) ;
if ( url = = null ) continue ;
2007-12-12 15:18:42 +01:00
//System.out.println("***DEBUG*** SEARCH RESULT URL=" + url.toNormalform(false, false));
2007-09-07 13:45:38 +02:00
2007-11-16 15:48:09 +01:00
resultEntry = obtainResultEntry ( uentry , ( snippetComputationAllTime < 300 ) ? 1 : 0 ) ;
2007-11-07 23:38:09 +01:00
if ( resultEntry = = null ) continue ; // the entry had some problems, cannot be used
urlRetrievalAllTime + = resultEntry . dbRetrievalTime ;
snippetComputationAllTime + = resultEntry . snippetComputationTime ;
2007-09-04 01:43:55 +02:00
2007-11-07 23:38:09 +01:00
// place the result to the result vector
2008-02-22 00:40:38 +01:00
result . push ( resultEntry , new Long ( rankedCache . getOrder ( ) . cardinal ( resultEntry . word ( ) ) ) ) ;
2007-09-04 01:43:55 +02:00
2007-11-07 23:38:09 +01:00
// add references
synchronized ( rankedCache ) {
rankedCache . addReferences ( resultEntry ) ;
}
2007-09-04 01:43:55 +02:00
}
}
2008-02-21 11:06:57 +01:00
serverProfiling . update ( " SEARCH " , new plasmaProfiling . searchEvent ( query . id ( true ) , " offline snippet fetch " , result . size ( ) , System . currentTimeMillis ( ) - timer ) ) ;
2007-09-04 01:43:55 +02:00
}
2007-08-25 01:12:59 +02:00
2007-09-07 13:45:38 +02:00
// clean up events
2007-11-15 13:49:13 +01:00
cleanupEvents ( false ) ;
2007-08-25 01:12:59 +02:00
// store this search to a cache so it can be re-used
2007-12-12 19:57:43 +01:00
lastEvents . put ( query . id ( false ) , this ) ;
lastEventID = query . id ( false ) ;
2005-10-12 14:28:49 +02:00
}
2007-09-28 03:36:22 +02:00
private class localSearchProcess extends Thread {
public localSearchProcess ( ) {
}
public void run ( ) {
// do a local search
// sort the local containers and truncate it to a limited count,
// so following sortings together with the global results will be fast
synchronized ( rankedCache ) {
2008-01-30 22:58:30 +01:00
rankedCache . execQuery ( ) ;
2007-09-28 03:36:22 +02:00
}
}
}
2007-09-04 01:43:55 +02:00
2007-11-15 13:49:13 +01:00
public static void cleanupEvents ( boolean all ) {
2007-09-07 13:45:38 +02:00
// remove old events in the event cache
2008-01-18 18:14:02 +01:00
Iterator < plasmaSearchEvent > i = lastEvents . values ( ) . iterator ( ) ;
2007-09-07 13:45:38 +02:00
plasmaSearchEvent cleanEvent ;
while ( i . hasNext ( ) ) {
2008-01-18 18:14:02 +01:00
cleanEvent = i . next ( ) ;
2007-11-15 13:49:13 +01:00
if ( ( all ) | | ( cleanEvent . eventTime + eventLifetime < System . currentTimeMillis ( ) ) ) {
2007-09-07 13:45:38 +02:00
// execute deletion of failed words
2008-01-18 18:14:02 +01:00
Set < String > removeWords = cleanEvent . query . queryHashes ;
2007-09-07 13:45:38 +02:00
removeWords . addAll ( cleanEvent . query . excludeHashes ) ;
cleanEvent . wordIndex . removeEntriesMultiple ( removeWords , cleanEvent . failedURLs . keySet ( ) ) ;
2007-12-12 19:57:43 +01:00
serverLog . logInfo ( " SearchEvents " , " cleaning up event " + cleanEvent . query . id ( true ) + " , removed " + cleanEvent . failedURLs . size ( ) + " URL references on " + removeWords . size ( ) + " words " ) ;
2007-09-07 13:45:38 +02:00
// remove the event
i . remove ( ) ;
}
}
}
2008-03-04 23:45:45 +01:00
ResultEntry obtainResultEntry ( indexURLEntry page , int snippetFetchMode ) {
2007-09-04 01:43:55 +02:00
// a search result entry needs some work to produce a result Entry:
// - check if url entry exists in LURL-db
// - check exclusions, constraints, masks, media-domains
// - load snippet (see if page exists) and check if snippet contains searched word
2007-10-03 18:42:11 +02:00
// Snippet Fetching can has 3 modes:
// 0 - do not fetch snippets
// 1 - fetch snippets offline only
// 2 - online snippet fetch
2007-09-04 01:43:55 +02:00
// load only urls if there was not yet a root url of that hash
// find the url entry
2007-09-05 11:01:35 +02:00
2007-09-26 12:11:50 +02:00
long startTime = System . currentTimeMillis ( ) ;
2007-09-04 01:43:55 +02:00
indexURLEntry . Components comp = page . comp ( ) ;
2008-01-30 22:58:30 +01:00
String pagetitle = comp . dc_title ( ) . toLowerCase ( ) ;
2007-09-04 01:43:55 +02:00
if ( comp . url ( ) = = null ) {
2007-09-07 13:45:38 +02:00
registerFailure ( page . hash ( ) , " url corrupted (null) " ) ;
2007-09-04 01:43:55 +02:00
return null ; // rare case where the url is corrupted
}
String pageurl = comp . url ( ) . toString ( ) . toLowerCase ( ) ;
2008-01-30 22:58:30 +01:00
String pageauthor = comp . dc_creator ( ) . toLowerCase ( ) ;
2007-09-26 12:11:50 +02:00
long dbRetrievalTime = System . currentTimeMillis ( ) - startTime ;
2007-09-04 01:43:55 +02:00
// check exclusion
2007-09-07 13:45:38 +02:00
if ( ( plasmaSearchQuery . matches ( pagetitle , query . excludeHashes ) ) | |
( plasmaSearchQuery . matches ( pageurl , query . excludeHashes ) ) | |
( plasmaSearchQuery . matches ( pageauthor , query . excludeHashes ) ) ) {
2007-09-04 01:43:55 +02:00
return null ;
}
// check url mask
if ( ! ( pageurl . matches ( query . urlMask ) ) ) {
return null ;
}
// check constraints
2007-11-22 00:14:57 +01:00
if ( ( query . constraint ! = null ) & &
2007-09-04 01:43:55 +02:00
( query . constraint . get ( plasmaCondenser . flag_cat_indexof ) ) & &
2008-01-30 22:58:30 +01:00
( ! ( comp . dc_title ( ) . startsWith ( " Index of " ) ) ) ) {
2008-01-18 18:14:02 +01:00
final Iterator < String > wi = query . queryHashes . iterator ( ) ;
2007-09-04 01:43:55 +02:00
while ( wi . hasNext ( ) ) wordIndex . removeEntry ( ( String ) wi . next ( ) , page . hash ( ) ) ;
2007-10-03 16:34:05 +02:00
registerFailure ( page . hash ( ) , " index-of constraint not fullfilled " ) ;
2007-09-04 01:43:55 +02:00
return null ;
}
if ( ( query . contentdom = = plasmaSearchQuery . CONTENTDOM_AUDIO ) & & ( page . laudio ( ) = = 0 ) ) {
2007-10-03 16:34:05 +02:00
registerFailure ( page . hash ( ) , " contentdom-audio constraint not fullfilled " ) ;
2007-09-04 01:43:55 +02:00
return null ;
}
if ( ( query . contentdom = = plasmaSearchQuery . CONTENTDOM_VIDEO ) & & ( page . lvideo ( ) = = 0 ) ) {
2007-10-03 16:34:05 +02:00
registerFailure ( page . hash ( ) , " contentdom-video constraint not fullfilled " ) ;
2007-09-04 01:43:55 +02:00
return null ;
}
if ( ( query . contentdom = = plasmaSearchQuery . CONTENTDOM_IMAGE ) & & ( page . limage ( ) = = 0 ) ) {
2007-10-03 16:34:05 +02:00
registerFailure ( page . hash ( ) , " contentdom-image constraint not fullfilled " ) ;
2007-09-04 01:43:55 +02:00
return null ;
}
if ( ( query . contentdom = = plasmaSearchQuery . CONTENTDOM_APP ) & & ( page . lapp ( ) = = 0 ) ) {
2007-10-03 16:34:05 +02:00
registerFailure ( page . hash ( ) , " contentdom-app constraint not fullfilled " ) ;
2007-09-04 01:43:55 +02:00
return null ;
}
2007-10-03 18:42:11 +02:00
if ( snippetFetchMode = = 0 ) {
return new ResultEntry ( page , wordIndex , null , null , dbRetrievalTime , 0 ) ; // result without snippet
}
2007-09-04 01:43:55 +02:00
// load snippet
if ( query . contentdom = = plasmaSearchQuery . CONTENTDOM_TEXT ) {
// attach text snippet
2007-09-26 12:11:50 +02:00
startTime = System . currentTimeMillis ( ) ;
2008-01-30 01:15:43 +01:00
plasmaSnippetCache . TextSnippet snippet = plasmaSnippetCache . retrieveTextSnippet ( comp , snippetFetchWordHashes , ( snippetFetchMode = = 2 ) , ( ( query . constraint ! = null ) & & ( query . constraint . get ( plasmaCondenser . flag_cat_indexof ) ) ) , 180 , 3000 , ( snippetFetchMode = = 2 ) ? Integer . MAX_VALUE : 100000 ) ;
2007-09-26 12:11:50 +02:00
long snippetComputationTime = System . currentTimeMillis ( ) - startTime ;
2007-10-03 17:45:12 +02:00
serverLog . logInfo ( " SEARCH_EVENT " , " text snippet load time for " + comp . url ( ) + " : " + snippetComputationTime + " , " + ( ( snippet . getErrorCode ( ) < 11 ) ? " snippet found " : ( " no snippet found ( " + snippet . getError ( ) + " ) " ) ) ) ;
2007-09-26 12:11:50 +02:00
2007-09-04 01:43:55 +02:00
if ( snippet . getErrorCode ( ) < 11 ) {
// we loaded the file and found the snippet
2007-09-26 12:11:50 +02:00
return new ResultEntry ( page , wordIndex , snippet , null , dbRetrievalTime , snippetComputationTime ) ; // result with snippet attached
2007-10-03 18:42:11 +02:00
} else if ( snippetFetchMode = = 1 ) {
2007-09-04 01:43:55 +02:00
// we did not demand online loading, therefore a failure does not mean that the missing snippet causes a rejection of this result
// this may happen during a remote search, because snippet loading is omitted to retrieve results faster
2007-09-26 12:11:50 +02:00
return new ResultEntry ( page , wordIndex , null , null , dbRetrievalTime , snippetComputationTime ) ; // result without snippet
2007-09-04 01:43:55 +02:00
} else {
// problems with snippet fetch
2007-09-07 13:45:38 +02:00
registerFailure ( page . hash ( ) , " no text snippet for URL " + comp . url ( ) ) ;
2007-12-12 19:57:43 +01:00
plasmaSnippetCache . failConsequences ( snippet , query . id ( false ) ) ;
2007-09-04 01:43:55 +02:00
return null ;
}
} else {
// attach media information
2007-09-26 12:11:50 +02:00
startTime = System . currentTimeMillis ( ) ;
2008-01-18 18:14:02 +01:00
ArrayList < MediaSnippet > mediaSnippets = plasmaSnippetCache . retrieveMediaSnippets ( comp . url ( ) , snippetFetchWordHashes , query . contentdom , ( snippetFetchMode = = 2 ) , 6000 ) ;
2007-09-26 12:11:50 +02:00
long snippetComputationTime = System . currentTimeMillis ( ) - startTime ;
2007-10-03 16:34:05 +02:00
serverLog . logInfo ( " SEARCH_EVENT " , " media snippet load time for " + comp . url ( ) + " : " + snippetComputationTime ) ;
2007-09-26 12:11:50 +02:00
2007-09-04 01:43:55 +02:00
if ( ( mediaSnippets ! = null ) & & ( mediaSnippets . size ( ) > 0 ) ) {
// found media snippets, return entry
2007-09-26 12:11:50 +02:00
return new ResultEntry ( page , wordIndex , null , mediaSnippets , dbRetrievalTime , snippetComputationTime ) ;
2007-10-03 18:42:11 +02:00
} else if ( snippetFetchMode = = 1 ) {
2007-09-26 12:11:50 +02:00
return new ResultEntry ( page , wordIndex , null , null , dbRetrievalTime , snippetComputationTime ) ;
2007-09-04 01:43:55 +02:00
} else {
// problems with snippet fetch
2007-09-07 13:45:38 +02:00
registerFailure ( page . hash ( ) , " no media snippet for URL " + comp . url ( ) ) ;
2007-09-04 01:43:55 +02:00
return null ;
}
}
// finished, no more actions possible here
}
private boolean anyWorkerAlive ( ) {
if ( this . workerThreads = = null ) return false ;
for ( int i = 0 ; i < workerThreadCount ; i + + ) {
2007-12-12 15:18:42 +01:00
if ( ( this . workerThreads [ i ] ! = null ) & &
( this . workerThreads [ i ] . isAlive ( ) ) & &
( this . workerThreads [ i ] . busytime ( ) < 3000 ) ) return true ;
2007-09-04 01:43:55 +02:00
}
return false ;
}
2005-10-12 14:28:49 +02:00
2008-03-04 23:45:45 +01:00
boolean anyRemoteSearchAlive ( ) {
2007-09-08 13:50:19 +02:00
// check primary search threads
if ( ( this . primarySearchThreads ! = null ) & & ( this . primarySearchThreads . length ! = 0 ) ) {
for ( int i = 0 ; i < this . primarySearchThreads . length ; i + + ) {
if ( ( this . primarySearchThreads [ i ] ! = null ) & & ( this . primarySearchThreads [ i ] . isAlive ( ) ) ) return true ;
}
}
2007-12-12 15:18:42 +01:00
// maybe a secondary search thread is alive, check this
2007-09-08 13:50:19 +02:00
if ( ( this . secondarySearchThreads ! = null ) & & ( this . secondarySearchThreads . length ! = 0 ) ) {
2007-10-04 10:50:33 +02:00
for ( int i = 0 ; i < this . secondarySearchThreads . length ; i + + ) {
2007-09-08 13:50:19 +02:00
if ( ( this . secondarySearchThreads [ i ] ! = null ) & & ( this . secondarySearchThreads [ i ] . isAlive ( ) ) ) return true ;
}
}
return false ;
}
2008-02-01 00:40:47 +01:00
private int countFinishedRemoteSearch ( ) {
int count = 0 ;
// check only primary search threads
if ( ( this . primarySearchThreads ! = null ) & & ( this . primarySearchThreads . length ! = 0 ) ) {
for ( int i = 0 ; i < this . primarySearchThreads . length ; i + + ) {
if ( ( this . primarySearchThreads [ i ] = = null ) | | ( ! ( this . primarySearchThreads [ i ] . isAlive ( ) ) ) ) count + + ;
}
}
return count ;
}
2005-10-24 02:34:15 +02:00
public plasmaSearchQuery getQuery ( ) {
return query ;
}
2006-09-13 19:13:28 +02:00
public yacySearch [ ] getPrimarySearchThreads ( ) {
return primarySearchThreads ;
}
2007-11-07 23:38:09 +01:00
2006-09-13 19:13:28 +02:00
public yacySearch [ ] getSecondarySearchThreads ( ) {
return secondarySearchThreads ;
2005-10-24 02:34:15 +02:00
}
2008-01-30 22:58:30 +01:00
public plasmaSearchRankingProcess getRankingResult ( ) {
return this . rankedCache ;
2007-01-15 02:50:57 +01:00
}
2007-09-26 12:11:50 +02:00
public long getURLRetrievalTime ( ) {
return this . urlRetrievalAllTime ;
}
public long getSnippetComputationTime ( ) {
return this . snippetComputationAllTime ;
}
2007-08-16 13:44:18 +02:00
2007-08-26 20:18:35 +02:00
public static plasmaSearchEvent getEvent ( String eventID ) {
2007-09-06 03:28:35 +02:00
synchronized ( lastEvents ) {
return ( plasmaSearchEvent ) lastEvents . get ( eventID ) ;
}
2007-08-26 20:18:35 +02:00
}
2007-08-25 01:12:59 +02:00
public static plasmaSearchEvent getEvent ( plasmaSearchQuery query ,
plasmaSearchRankingProfile ranking ,
plasmaWordIndex wordIndex ,
2008-01-18 18:14:02 +01:00
TreeMap < String , String > preselectedPeerHashes ,
boolean generateAbstracts ) {
2007-09-06 03:28:35 +02:00
synchronized ( lastEvents ) {
2007-12-12 19:57:43 +01:00
plasmaSearchEvent event = ( plasmaSearchEvent ) lastEvents . get ( query . id ( false ) ) ;
2007-09-06 03:28:35 +02:00
if ( event = = null ) {
2008-01-18 18:14:02 +01:00
event = new plasmaSearchEvent ( query , wordIndex , preselectedPeerHashes , generateAbstracts ) ;
2007-09-06 03:28:35 +02:00
} else {
//re-new the event time for this event, so it is not deleted next time too early
event . eventTime = System . currentTimeMillis ( ) ;
// replace the query, because this contains the current result offset
event . query = query ;
}
2007-09-04 01:43:55 +02:00
2008-02-22 00:40:38 +01:00
// if a local crawl is ongoing, do another local search to enrich the current results with more
// entries that can possibly come out of the running crawl
if ( plasmaSwitchboard . getSwitchboard ( ) . crawlQueues . noticeURL . size ( ) > 0 ) {
synchronized ( event . rankedCache ) {
event . rankedCache . execQuery ( ) ;
}
}
2007-09-06 03:28:35 +02:00
// if worker threads had been alive, but did not succeed, start them again to fetch missing links
if ( ( query . onlineSnippetFetch ) & &
( ! event . anyWorkerAlive ( ) ) & &
2008-02-21 15:53:51 +01:00
( ( ( query . contentdom = = plasmaSearchQuery . CONTENTDOM_IMAGE ) & & ( event . images . size ( ) + 30 < query . neededResults ( ) ) ) | |
( event . result . size ( ) < query . neededResults ( ) + 10 ) ) & &
2008-02-21 11:06:57 +01:00
( event . getRankingResult ( ) . getLocalResourceSize ( ) + event . getRankingResult ( ) . getRemoteResourceSize ( ) > event . result . size ( ) ) ) {
2007-09-06 03:28:35 +02:00
// set new timeout
event . eventTime = System . currentTimeMillis ( ) ;
// start worker threads to fetch urls and snippets
event . workerThreads = new resultWorker [ workerThreadCount ] ;
for ( int i = 0 ; i < workerThreadCount ; i + + ) {
2008-02-03 00:53:39 +01:00
event . workerThreads [ i ] = event . deployWorker ( i , 10000 ) ;
2007-09-06 03:28:35 +02:00
}
2007-08-25 01:12:59 +02:00
}
2007-09-06 03:28:35 +02:00
return event ;
2007-08-25 01:12:59 +02:00
}
2007-09-04 01:43:55 +02:00
}
private resultWorker deployWorker ( int id , long lifetime ) {
resultWorker worker = new resultWorker ( id , lifetime ) ;
worker . start ( ) ;
return worker ;
2006-04-03 17:36:53 +02:00
}
2007-08-28 14:15:46 +02:00
2007-09-04 01:43:55 +02:00
private class resultWorker extends Thread {
2007-08-28 14:15:46 +02:00
2007-09-04 01:43:55 +02:00
private long timeout ; // the date until this thread should try to work
2007-12-12 15:18:42 +01:00
private long lastLifeSign ; // when the last time the run()-loop was executed
2007-09-04 01:43:55 +02:00
private int id ;
2007-08-28 14:15:46 +02:00
2007-12-12 15:18:42 +01:00
public resultWorker ( int id , long maxlifetime ) {
2007-09-04 01:43:55 +02:00
this . id = id ;
2007-12-12 15:18:42 +01:00
this . lastLifeSign = System . currentTimeMillis ( ) ;
2007-12-22 03:58:38 +01:00
this . timeout = System . currentTimeMillis ( ) + Math . max ( 1000 , maxlifetime ) ;
2008-02-02 12:30:47 +01:00
//this.sleeptime = Math.min(300, maxlifetime / 10 * id);
2007-09-04 01:43:55 +02:00
}
public void run ( ) {
// start fetching urls and snippets
2007-11-22 00:14:57 +01:00
indexURLEntry page ;
2007-11-23 02:08:16 +01:00
while ( System . currentTimeMillis ( ) < this . timeout ) {
2007-12-12 15:18:42 +01:00
this . lastLifeSign = System . currentTimeMillis ( ) ;
2008-02-21 15:53:51 +01:00
// check if we have enough
if ( ( query . contentdom = = plasmaSearchQuery . CONTENTDOM_IMAGE ) & & ( images . size ( ) > = query . neededResults ( ) + 30 ) ) break ;
if ( ( query . contentdom ! = plasmaSearchQuery . CONTENTDOM_IMAGE ) & & ( result . size ( ) > = query . neededResults ( ) + 10 /*+ query.displayResults()*/ ) ) break ;
2007-12-12 15:18:42 +01:00
2007-11-23 02:08:16 +01:00
// get next entry
page = rankedCache . bestURL ( true ) ;
if ( page = = null ) {
2007-12-12 15:18:42 +01:00
if ( ! anyRemoteSearchAlive ( ) ) break ; // we cannot expect more results
2007-11-23 02:08:16 +01:00
// if we did not get another entry, sleep some time and try again
try { Thread . sleep ( 100 ) ; } catch ( InterruptedException e1 ) { }
continue ;
}
2007-11-22 00:14:57 +01:00
if ( anyResultWith ( page . hash ( ) ) ) continue ;
if ( anyFailureWith ( page . hash ( ) ) ) continue ;
2007-09-04 01:43:55 +02:00
// try secondary search
2007-09-07 13:45:38 +02:00
prepareSecondarySearch ( ) ; // will be executed only once
2007-09-04 01:43:55 +02:00
2007-10-03 18:42:11 +02:00
ResultEntry resultEntry = obtainResultEntry ( page , 2 ) ;
2007-09-04 01:43:55 +02:00
if ( resultEntry = = null ) continue ; // the entry had some problems, cannot be used
2007-09-26 12:11:50 +02:00
urlRetrievalAllTime + = resultEntry . dbRetrievalTime ;
snippetComputationAllTime + = resultEntry . snippetComputationTime ;
2007-11-23 02:08:16 +01:00
//System.out.println("+++DEBUG-resultWorker+++ fetched " + resultEntry.urlstring());
2007-09-04 01:43:55 +02:00
// place the result to the result vector
2008-02-21 11:06:57 +01:00
if ( ! result . exists ( resultEntry ) ) {
2008-02-22 00:40:38 +01:00
result . push ( resultEntry , new Long ( rankedCache . getOrder ( ) . cardinal ( resultEntry . word ( ) ) ) ) ;
2007-09-04 01:43:55 +02:00
rankedCache . addReferences ( resultEntry ) ;
}
2007-11-23 02:08:16 +01:00
//System.out.println("DEBUG SNIPPET_LOADING: thread " + id + " got " + resultEntry.url());
2007-09-04 01:43:55 +02:00
}
2007-09-08 13:50:19 +02:00
serverLog . logInfo ( " SEARCH " , " resultWorker thread " + id + " terminated " ) ;
2007-09-04 01:43:55 +02:00
}
2007-08-28 14:15:46 +02:00
2007-09-04 01:43:55 +02:00
private boolean anyResultWith ( String urlhash ) {
2008-02-21 11:06:57 +01:00
return result . exists ( urlhash . hashCode ( ) ) ;
2007-09-04 01:43:55 +02:00
}
2007-08-28 14:15:46 +02:00
2007-09-04 01:43:55 +02:00
private boolean anyFailureWith ( String urlhash ) {
return ( failedURLs . get ( urlhash ) ! = null ) ;
}
2007-12-12 15:18:42 +01:00
public long busytime ( ) {
return System . currentTimeMillis ( ) - this . lastLifeSign ;
}
2007-09-04 01:43:55 +02:00
}
private void registerFailure ( String urlhash , String reason ) {
this . failedURLs . put ( urlhash , reason ) ;
serverLog . logInfo ( " search " , " sorted out hash " + urlhash + " during search: " + reason ) ;
}
public ResultEntry oneResult ( int item ) {
2008-02-21 15:53:51 +01:00
// check if we already retrieved this item (happens if a search pages is accessed a second time)
2008-02-21 11:06:57 +01:00
if ( this . result . sizeStore ( ) > item ) {
// we have the wanted result already in the result array .. return that
return this . result . element ( item ) . element ;
}
2008-02-02 12:30:47 +01:00
if ( ( query . domType = = plasmaSearchQuery . SEARCHDOM_GLOBALDHT ) | |
( query . domType = = plasmaSearchQuery . SEARCHDOM_CLUSTERALL ) ) {
// this is a search using remote search threads. Also the local search thread is started as background process
if ( ( localSearchThread ! = null ) & & ( localSearchThread . isAlive ( ) ) ) {
// in case that the local search takes longer than some other remote search requests,
// do some sleeps to give the local process a chance to contribute
2008-02-06 21:29:22 +01:00
try { Thread . sleep ( item * 100 ) ; } catch ( InterruptedException e ) { }
2008-02-02 12:30:47 +01:00
}
// now wait until as many remote worker threads have finished, as we want to display results
while ( ( this . primarySearchThreads ! = null ) & & ( this . primarySearchThreads . length > item ) & & ( anyWorkerAlive ( ) ) & &
2008-02-21 11:06:57 +01:00
( ( result . size ( ) < = item ) | | ( countFinishedRemoteSearch ( ) < = item ) ) ) {
2008-02-02 12:30:47 +01:00
try { Thread . sleep ( 100 ) ; } catch ( InterruptedException e ) { }
}
2008-02-06 21:29:22 +01:00
}
// finally wait until enough results are there produced from the snippet fetch process
2008-02-21 11:06:57 +01:00
while ( ( anyWorkerAlive ( ) ) & & ( result . size ( ) < = item ) ) {
2008-02-06 21:29:22 +01:00
try { Thread . sleep ( 100 ) ; } catch ( InterruptedException e ) { }
2007-09-04 01:43:55 +02:00
}
2008-02-01 00:40:47 +01:00
2007-09-04 01:43:55 +02:00
// finally, if there is something, return the result
2008-02-21 11:06:57 +01:00
if ( this . result . size ( ) < = item ) return null ;
return this . result . element ( item ) . element ;
2007-09-04 01:43:55 +02:00
}
2008-02-21 11:06:57 +01:00
2008-02-21 15:53:51 +01:00
private int resultCounter = 0 ;
public ResultEntry nextResult ( ) {
ResultEntry re = oneResult ( resultCounter ) ;
resultCounter + + ;
return re ;
}
public plasmaSnippetCache . MediaSnippet oneImage ( int item ) {
// check if we already retrieved this item (happens if a search pages is accessed a second time)
if ( this . images . sizeStore ( ) > item ) {
// we have the wanted result already in the result array .. return that
return this . images . element ( item ) . element ;
}
// feed some results from the result stack into the image stack
int count = Math . min ( 5 , Math . max ( 1 , 10 * this . result . size ( ) / ( item + 1 ) ) ) ;
for ( int i = 0 ; i < count ; i + + ) {
// generate result object
plasmaSearchEvent . ResultEntry result = nextResult ( ) ;
plasmaSnippetCache . MediaSnippet ms ;
if ( result ! = null ) {
// iterate over all images in the result
ArrayList < plasmaSnippetCache . MediaSnippet > imagemedia = result . mediaSnippets ( ) ;
if ( imagemedia ! = null ) {
for ( int j = 0 ; j < imagemedia . size ( ) ; j + + ) {
ms = imagemedia . get ( j ) ;
2008-02-22 00:40:38 +01:00
images . push ( ms , new Long ( ms . ranking ) ) ;
2008-02-21 15:53:51 +01:00
}
}
}
}
// now take the specific item from the image stack
if ( this . images . size ( ) < = item ) return null ;
return this . images . element ( item ) . element ;
}
2008-02-21 11:06:57 +01:00
public ArrayList < kelondroSortStack < ResultEntry > . stackElement > completeResults ( long waitingtime ) {
2007-09-04 01:43:55 +02:00
long timeout = System . currentTimeMillis ( ) + waitingtime ;
2008-02-21 11:06:57 +01:00
while ( ( result . size ( ) < query . neededResults ( ) ) & & ( anyWorkerAlive ( ) ) & & ( System . currentTimeMillis ( ) < timeout ) ) {
2008-02-03 00:53:39 +01:00
try { Thread . sleep ( 100 ) ; } catch ( InterruptedException e ) { }
2007-11-23 02:08:16 +01:00
//System.out.println("+++DEBUG-completeResults+++ sleeping " + 200);
2007-09-04 01:43:55 +02:00
}
2008-02-21 11:06:57 +01:00
return this . result . list ( this . result . size ( ) ) ;
2007-09-04 01:43:55 +02:00
}
boolean secondarySearchStartet = false ;
2007-08-25 01:12:59 +02:00
2008-03-04 23:45:45 +01:00
void prepareSecondarySearch ( ) {
2008-02-01 00:40:47 +01:00
if ( secondarySearchStartet ) return ; // don't do this twice
2006-09-13 19:13:28 +02:00
2007-09-04 01:43:55 +02:00
if ( ( rcAbstracts = = null ) | | ( rcAbstracts . size ( ) ! = query . queryHashes . size ( ) ) ) return ; // secondary search not possible (yet)
this . secondarySearchStartet = true ;
2006-09-13 19:13:28 +02:00
2007-12-22 03:58:38 +01:00
/ *
2007-09-04 01:43:55 +02:00
// catch up index abstracts and join them; then call peers again to submit their urls
System . out . println ( " DEBUG-INDEXABSTRACT: " + rcAbstracts . size ( ) + " word references catched, " + query . queryHashes . size ( ) + " needed " ) ;
2006-09-13 19:13:28 +02:00
Iterator i = rcAbstracts . entrySet ( ) . iterator ( ) ;
Map . Entry entry ;
while ( i . hasNext ( ) ) {
entry = ( Map . Entry ) i . next ( ) ;
System . out . println ( " DEBUG-INDEXABSTRACT: hash " + ( String ) entry . getKey ( ) + " : " + ( ( query . queryHashes . contains ( ( String ) entry . getKey ( ) ) ) ? " NEEDED " : " NOT NEEDED " ) + " ; " + ( ( TreeMap ) entry . getValue ( ) ) . size ( ) + " entries " ) ;
}
2007-12-22 03:58:38 +01:00
* /
2008-01-18 18:14:02 +01:00
TreeMap < String , String > abstractJoin = ( rcAbstracts . size ( ) = = query . queryHashes . size ( ) ) ? kelondroMSetTools . joinConstructive ( rcAbstracts . values ( ) , true ) : new TreeMap < String , String > ( ) ;
2006-09-13 19:13:28 +02:00
if ( abstractJoin . size ( ) = = 0 ) {
2007-12-22 03:58:38 +01:00
//System.out.println("DEBUG-INDEXABSTRACT: no success using index abstracts from remote peers");
2006-09-13 19:13:28 +02:00
} else {
2007-12-22 03:58:38 +01:00
//System.out.println("DEBUG-INDEXABSTRACT: index abstracts delivered " + abstractJoin.size() + " additional results for secondary search");
2006-09-13 19:13:28 +02:00
// generate query for secondary search
2008-01-18 18:14:02 +01:00
TreeMap < String , String > secondarySearchURLs = new TreeMap < String , String > ( ) ; // a (peerhash:urlhash-liststring) mapping
Iterator < Map . Entry < String , String > > i1 = abstractJoin . entrySet ( ) . iterator ( ) ;
Map . Entry < String , String > entry1 ;
2006-09-13 19:13:28 +02:00
String url , urls , peer , peers ;
2007-10-01 14:30:23 +02:00
String mypeerhash = yacyCore . seedDB . mySeed ( ) . hash ;
2006-09-16 02:07:09 +02:00
boolean mypeerinvolved = false ;
2006-10-31 03:45:41 +01:00
int mypeercount ;
2006-09-13 19:13:28 +02:00
while ( i1 . hasNext ( ) ) {
2008-01-18 18:14:02 +01:00
entry1 = i1 . next ( ) ;
url = entry1 . getKey ( ) ;
peers = entry1 . getValue ( ) ;
2007-12-22 03:58:38 +01:00
//System.out.println("DEBUG-INDEXABSTRACT: url " + url + ": from peers " + peers);
2006-10-31 03:45:41 +01:00
mypeercount = 0 ;
2006-09-13 19:13:28 +02:00
for ( int j = 0 ; j < peers . length ( ) ; j = j + 12 ) {
peer = peers . substring ( j , j + 12 ) ;
2006-10-31 03:45:41 +01:00
if ( ( peer . equals ( mypeerhash ) ) & & ( mypeercount + + > 1 ) ) continue ;
//if (peers.indexOf(peer) < j) continue; // avoid doubles that may appear in the abstractJoin
2006-09-13 19:13:28 +02:00
urls = ( String ) secondarySearchURLs . get ( peer ) ;
urls = ( urls = = null ) ? url : urls + url ;
secondarySearchURLs . put ( peer , urls ) ;
}
2006-10-31 03:45:41 +01:00
if ( mypeercount = = 1 ) mypeerinvolved = true ;
2006-09-13 19:13:28 +02:00
}
// compute words for secondary search and start the secondary searches
i1 = secondarySearchURLs . entrySet ( ) . iterator ( ) ;
String words ;
2006-09-16 02:07:09 +02:00
secondarySearchThreads = new yacySearch [ ( mypeerinvolved ) ? secondarySearchURLs . size ( ) - 1 : secondarySearchURLs . size ( ) ] ;
2006-09-13 19:13:28 +02:00
int c = 0 ;
while ( i1 . hasNext ( ) ) {
2008-01-18 18:14:02 +01:00
entry1 = i1 . next ( ) ;
peer = entry1 . getKey ( ) ;
2006-09-16 02:07:09 +02:00
if ( peer . equals ( mypeerhash ) ) continue ; // we dont need to ask ourself
2006-09-13 19:13:28 +02:00
urls = ( String ) entry1 . getValue ( ) ;
words = wordsFromPeer ( peer , urls ) ;
2007-12-22 03:58:38 +01:00
//System.out.println("DEBUG-INDEXABSTRACT ***: peer " + peer + " has urls: " + urls);
//System.out.println("DEBUG-INDEXABSTRACT ***: peer " + peer + " from words: " + words);
2006-09-13 19:13:28 +02:00
secondarySearchThreads [ c + + ] = yacySearch . secondaryRemoteSearch (
2007-09-04 01:43:55 +02:00
words , " " , urls , wordIndex , this . rankedCache , peer , plasmaSwitchboard . urlBlacklist ,
2008-01-08 21:12:31 +01:00
query . ranking , query . constraint , preselectedPeerHashes ) ;
2006-09-13 19:13:28 +02:00
}
}
}
private String wordsFromPeer ( String peerhash , String urls ) {
2008-01-18 18:14:02 +01:00
Map . Entry < String , TreeMap < String , String > > entry ;
2006-09-13 19:13:28 +02:00
String word , peerlist , url , wordlist = " " ;
2008-01-18 18:14:02 +01:00
TreeMap < String , String > urlPeerlist ;
2006-09-13 19:13:28 +02:00
int p ;
boolean hasURL ;
synchronized ( rcAbstracts ) {
2008-01-18 18:14:02 +01:00
Iterator < Map . Entry < String , TreeMap < String , String > > > i = rcAbstracts . entrySet ( ) . iterator ( ) ;
2006-09-13 19:13:28 +02:00
while ( i . hasNext ( ) ) {
2008-01-18 18:14:02 +01:00
entry = i . next ( ) ;
word = entry . getKey ( ) ;
urlPeerlist = entry . getValue ( ) ;
2006-09-13 19:13:28 +02:00
hasURL = true ;
for ( int j = 0 ; j < urls . length ( ) ; j = j + 12 ) {
url = urls . substring ( j , j + 12 ) ;
peerlist = ( String ) urlPeerlist . get ( url ) ;
p = ( peerlist = = null ) ? - 1 : peerlist . indexOf ( peerhash ) ;
if ( ( p < 0 ) | | ( p % 12 ! = 0 ) ) {
hasURL = false ;
break ;
}
}
if ( hasURL ) wordlist + = word ;
}
}
return wordlist ;
}
2007-08-26 20:18:35 +02:00
public void remove ( String urlhash ) {
// removes the url hash reference from last search result
2007-09-04 01:43:55 +02:00
/*indexRWIEntry e =*/ this . rankedCache . remove ( urlhash ) ;
//assert e != null;
2007-08-26 20:18:35 +02:00
}
2006-09-13 19:13:28 +02:00
2008-01-11 01:12:01 +01:00
public Set < String > references ( int count ) {
2007-09-08 13:50:19 +02:00
// returns a set of words that are computed as toplist
2007-09-07 13:45:38 +02:00
return this . rankedCache . getReferences ( count ) ;
2007-08-28 14:15:46 +02:00
}
2007-09-04 01:43:55 +02:00
public static class ResultEntry {
2007-09-26 12:11:50 +02:00
// payload objects
2007-08-28 14:15:46 +02:00
private indexURLEntry urlentry ;
private indexURLEntry . Components urlcomps ; // buffer for components
private String alternative_urlstring ;
private String alternative_urlname ;
2007-09-04 01:43:55 +02:00
private plasmaSnippetCache . TextSnippet textSnippet ;
2008-01-11 01:12:01 +01:00
private ArrayList < plasmaSnippetCache . MediaSnippet > mediaSnippets ;
2007-08-28 14:15:46 +02:00
2007-09-26 12:11:50 +02:00
// statistic objects
public long dbRetrievalTime , snippetComputationTime ;
2008-01-11 01:12:01 +01:00
public ResultEntry ( indexURLEntry urlentry , plasmaWordIndex wordIndex ,
plasmaSnippetCache . TextSnippet textSnippet ,
ArrayList < plasmaSnippetCache . MediaSnippet > mediaSnippets ,
2007-09-26 12:11:50 +02:00
long dbRetrievalTime , long snippetComputationTime ) {
2007-08-28 14:15:46 +02:00
this . urlentry = urlentry ;
this . urlcomps = urlentry . comp ( ) ;
this . alternative_urlstring = null ;
this . alternative_urlname = null ;
2007-09-04 01:43:55 +02:00
this . textSnippet = textSnippet ;
this . mediaSnippets = mediaSnippets ;
2007-09-26 12:11:50 +02:00
this . dbRetrievalTime = dbRetrievalTime ;
this . snippetComputationTime = snippetComputationTime ;
2007-08-28 14:15:46 +02:00
String host = urlcomps . url ( ) . getHost ( ) ;
if ( host . endsWith ( " .yacyh " ) ) {
// translate host into current IP
int p = host . indexOf ( " . " ) ;
String hash = yacySeed . hexHash2b64Hash ( host . substring ( p + 1 , host . length ( ) - 6 ) ) ;
yacySeed seed = yacyCore . seedDB . getConnected ( hash ) ;
String filename = urlcomps . url ( ) . getFile ( ) ;
String address = null ;
if ( ( seed = = null ) | | ( ( address = seed . getPublicAddress ( ) ) = = null ) ) {
// seed is not known from here
try {
wordIndex . removeWordReferences (
plasmaCondenser . getWords (
( " yacyshare " +
filename . replace ( '?' , ' ' ) +
" " +
2008-01-30 22:58:30 +01:00
urlcomps . dc_title ( ) ) . getBytes ( ) , " UTF-8 " ) . keySet ( ) ,
2007-08-28 14:15:46 +02:00
urlentry . hash ( ) ) ;
wordIndex . loadedURL . remove ( urlentry . hash ( ) ) ; // clean up
throw new RuntimeException ( " index void " ) ;
} catch ( UnsupportedEncodingException e ) {
throw new RuntimeException ( " parser failed: " + e . getMessage ( ) ) ;
}
}
alternative_urlstring = " http:// " + address + " / " + host . substring ( 0 , p ) + filename ;
alternative_urlname = " http://share. " + seed . getName ( ) + " .yacy " + filename ;
if ( ( p = alternative_urlname . indexOf ( " ? " ) ) > 0 ) alternative_urlname = alternative_urlname . substring ( 0 , p ) ;
}
}
2008-02-21 11:06:57 +01:00
public int hashCode ( ) {
return urlentry . hash ( ) . hashCode ( ) ;
}
2007-08-28 14:15:46 +02:00
public String hash ( ) {
return urlentry . hash ( ) ;
}
2007-09-05 11:01:35 +02:00
public yacyURL url ( ) {
2007-08-28 14:15:46 +02:00
return urlcomps . url ( ) ;
}
public kelondroBitfield flags ( ) {
return urlentry . flags ( ) ;
}
public String urlstring ( ) {
return ( alternative_urlstring = = null ) ? urlcomps . url ( ) . toNormalform ( false , true ) : alternative_urlstring ;
}
public String urlname ( ) {
return ( alternative_urlname = = null ) ? urlcomps . url ( ) . toNormalform ( false , true ) : alternative_urlname ;
}
public String title ( ) {
2008-01-30 22:58:30 +01:00
return urlcomps . dc_title ( ) ;
2007-08-28 14:15:46 +02:00
}
2007-09-04 01:43:55 +02:00
public plasmaSnippetCache . TextSnippet textSnippet ( ) {
return this . textSnippet ;
2007-08-28 14:15:46 +02:00
}
2008-01-11 01:12:01 +01:00
public ArrayList < plasmaSnippetCache . MediaSnippet > mediaSnippets ( ) {
2007-09-04 01:43:55 +02:00
return this . mediaSnippets ;
2007-08-28 14:15:46 +02:00
}
public Date modified ( ) {
return urlentry . moddate ( ) ;
}
public int filesize ( ) {
return urlentry . size ( ) ;
}
2007-09-08 13:50:19 +02:00
public int limage ( ) {
return urlentry . limage ( ) ;
}
public int laudio ( ) {
return urlentry . laudio ( ) ;
}
public int lvideo ( ) {
return urlentry . lvideo ( ) ;
}
public int lapp ( ) {
return urlentry . lapp ( ) ;
}
2008-02-21 11:06:57 +01:00
public indexRWIVarEntry word ( ) {
indexRWIEntry word = urlentry . word ( ) ;
assert word instanceof indexRWIVarEntry ;
return ( indexRWIVarEntry ) word ;
2007-08-28 14:15:46 +02:00
}
2007-09-04 01:43:55 +02:00
public boolean hasTextSnippet ( ) {
return ( this . textSnippet ! = null ) & & ( this . textSnippet . getErrorCode ( ) < 11 ) ;
2007-08-28 14:15:46 +02:00
}
2007-09-04 01:43:55 +02:00
public boolean hasMediaSnippets ( ) {
return ( this . mediaSnippets ! = null ) & & ( this . mediaSnippets . size ( ) > 0 ) ;
2007-08-28 14:15:46 +02:00
}
public String resource ( ) {
// generate transport resource
2007-09-04 01:43:55 +02:00
if ( ( textSnippet ! = null ) & & ( textSnippet . exists ( ) ) ) {
return urlentry . toString ( textSnippet . getLineRaw ( ) ) ;
2007-08-28 14:15:46 +02:00
} else {
return urlentry . toString ( ) ;
}
}
}
2005-10-10 02:32:15 +02:00
}