2005-10-10 02:32:15 +02:00
// plasmaSearchEvent.java
// -----------------------
// part of YACY
// (C) by Michael Peter Christen; mc@anomic.de
// first published on http://www.anomic.de
// Frankfurt, Germany, 2005
// Created: 10.10.2005
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation; either version 2 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
//
// Using this software in any meaning (reading, learning, copying, compiling,
// running) means that you agree that the Author(s) is (are) not responsible
// for cost, loss of data or any harm that may be caused directly or indirectly
// by usage of this softare or this documentation. The usage of this software
// is on your own risk. The installation and usage (starting/running) of this
// software may allow other people or application to access your computer and
// any attached devices and is highly dependent on the configuration of the
// software which must be done by the user of the software; the author(s) is
// (are) also not responsible for proper configuration and usage of the
// software, even if provoked by documentation provided together with
// the software.
//
// Any changes to this file according to the GPL as documented in the file
// gpl.txt aside this file in the shipment you received can be done to the
// lines that follows this copyright notice here, but changes must not be
// done inside the copyright notive above. A re-distribution must contain
// the intact and unchanged copyright notice.
// Contributions and changes to the program code must be marked as such.
package de.anomic.plasma ;
2005-10-11 09:06:33 +02:00
import java.util.Iterator ;
2005-10-12 14:28:49 +02:00
import java.util.Set ;
2005-12-15 14:21:42 +01:00
import java.util.HashSet ;
2005-10-12 14:28:49 +02:00
import java.io.IOException ;
import de.anomic.kelondro.kelondroException ;
import de.anomic.server.logging.serverLog ;
2005-10-13 15:57:15 +02:00
import de.anomic.server.serverInstantThread ;
import de.anomic.yacy.yacySearch ;
2006-05-28 03:09:31 +02:00
import de.anomic.index.indexContainer ;
2006-08-02 21:59:28 +02:00
import de.anomic.index.indexEntry ;
2006-08-03 01:20:03 +02:00
import de.anomic.index.indexRowSetContainer ;
2005-10-10 02:32:15 +02:00
2005-12-15 14:21:42 +01:00
public final class plasmaSearchEvent extends Thread implements Runnable {
2005-10-10 02:32:15 +02:00
2005-10-24 02:34:15 +02:00
public static plasmaSearchEvent lastEvent = null ;
2005-12-15 14:21:42 +01:00
private static HashSet flushThreads = new HashSet ( ) ;
2005-10-24 02:34:15 +02:00
2005-10-12 14:28:49 +02:00
private serverLog log ;
2005-10-11 09:06:33 +02:00
private plasmaSearchQuery query ;
2006-02-05 00:51:00 +01:00
private plasmaSearchRankingProfile ranking ;
2005-10-12 14:28:49 +02:00
private plasmaWordIndex wordIndex ;
private plasmaCrawlLURL urlStore ;
private plasmaSnippetCache snippetCache ;
2006-05-28 03:09:31 +02:00
private indexContainer rcLocal , rcGlobal ; // caches for results
2006-02-04 15:13:54 +01:00
private int rcGlobalCount ;
2006-02-05 00:11:31 +01:00
private plasmaSearchTimingProfile profileLocal , profileGlobal ;
2005-10-13 15:57:15 +02:00
private yacySearch [ ] searchThreads ;
2005-10-12 14:28:49 +02:00
2006-02-05 00:51:00 +01:00
public plasmaSearchEvent ( plasmaSearchQuery query ,
plasmaSearchRankingProfile ranking ,
plasmaSearchTimingProfile localTiming ,
plasmaSearchTimingProfile remoteTiming ,
serverLog log ,
plasmaWordIndex wordIndex ,
plasmaCrawlLURL urlStore ,
plasmaSnippetCache snippetCache ) {
2005-10-12 14:28:49 +02:00
this . log = log ;
this . wordIndex = wordIndex ;
2005-10-11 09:06:33 +02:00
this . query = query ;
2006-02-05 00:51:00 +01:00
this . ranking = ranking ;
2005-10-12 14:28:49 +02:00
this . urlStore = urlStore ;
this . snippetCache = snippetCache ;
2006-08-03 01:20:03 +02:00
this . rcLocal = new indexRowSetContainer ( null ) ;
this . rcGlobal = new indexRowSetContainer ( null ) ;
2006-02-04 15:13:54 +01:00
this . rcGlobalCount = 0 ;
2006-02-05 00:51:00 +01:00
this . profileLocal = localTiming ;
this . profileGlobal = remoteTiming ;
2005-10-13 15:57:15 +02:00
this . searchThreads = null ;
2005-10-12 14:28:49 +02:00
}
2005-10-24 02:34:15 +02:00
public plasmaSearchQuery getQuery ( ) {
return query ;
}
2006-02-05 00:51:00 +01:00
public plasmaSearchTimingProfile getLocalTiming ( ) {
2005-10-27 02:42:08 +02:00
return profileLocal ;
}
2005-10-24 02:34:15 +02:00
public yacySearch [ ] getSearchThreads ( ) {
return searchThreads ;
}
2005-10-13 15:57:15 +02:00
public plasmaSearchResult search ( ) {
// combine all threads
2006-04-02 20:51:18 +02:00
// we synchronize with flushThreads to allow only one local search at a time,
// so all search tasks are queued
synchronized ( flushThreads ) {
2006-04-03 17:36:53 +02:00
if ( query . domType = = plasmaSearchQuery . SEARCHDOM_GLOBALDHT ) {
int fetchpeers = ( int ) ( query . maximumTime / 500L ) ; // number of target peers; means 10 peers in 10 seconds
if ( fetchpeers > 50 ) fetchpeers = 50 ;
if ( fetchpeers < 30 ) fetchpeers = 30 ;
// remember time
long start = System . currentTimeMillis ( ) ;
// first trigger a local search within a separate thread
serverInstantThread . oneTimeJob ( this , " localSearch " , log , 0 ) ;
// do a global search
int globalContributions = globalSearch ( fetchpeers ) ;
log . logFine ( " SEARCH TIME AFTER GLOBAL-TRIGGER TO " + fetchpeers + " PEERS: " + ( ( System . currentTimeMillis ( ) - start ) / 1000 ) + " seconds " ) ;
// combine the result and order
plasmaSearchResult result = order ( ) ;
result . globalContributions = globalContributions ;
result . localContributions = rcLocal . size ( ) ;
flushGlobalResults ( ) ; // make these values available for immediate next search
// flush results in a separate thread
this . start ( ) ; // start to flush results
// serverInstantThread.oneTimeJob(this, "flushResults", log, 0);
// clean up
rcLocal = null ;
// return search result
log . logFine ( " SEARCHRESULT: " + profileLocal . reportToString ( ) ) ;
lastEvent = this ;
return result ;
} else {
localSearch ( ) ;
plasmaSearchResult result = order ( ) ;
result . localContributions = rcLocal . size ( ) ;
// clean up
rcLocal = null ;
// return search result
log . logFine ( " SEARCHRESULT: " + profileLocal . reportToString ( ) ) ;
lastEvent = this ;
return result ;
2006-04-02 20:51:18 +02:00
}
2006-04-03 17:36:53 +02:00
}
}
public int localSearch ( ) {
// search for the set of hashes and return an array of urlEntry elements
2006-04-02 20:51:18 +02:00
2006-04-03 17:36:53 +02:00
// retrieve entities that belong to the hashes
profileLocal . startTimer ( ) ;
Set containers = wordIndex . getContainers (
query . queryHashes ,
true ,
true ,
profileLocal . getTargetTime ( plasmaSearchTimingProfile . PROCESS_COLLECTION ) ) ;
if ( containers . size ( ) < query . size ( ) ) containers = null ; // prevent that only a subset is returned
profileLocal . setYieldTime ( plasmaSearchTimingProfile . PROCESS_COLLECTION ) ;
profileLocal . setYieldCount ( plasmaSearchTimingProfile . PROCESS_COLLECTION , ( containers = = null ) ? 0 : containers . size ( ) ) ;
2006-04-02 20:51:18 +02:00
2006-04-03 17:36:53 +02:00
// since this is a conjunction we return an empty entity if any word
// is not known
if ( containers = = null ) {
2006-08-03 01:20:03 +02:00
rcLocal = new indexRowSetContainer ( null ) ;
2006-04-03 17:36:53 +02:00
return 0 ;
2005-10-13 15:57:15 +02:00
}
2006-04-03 17:36:53 +02:00
// join the result
profileLocal . startTimer ( ) ;
2006-08-03 01:20:03 +02:00
rcLocal = indexRowSetContainer . joinContainer ( containers ,
2006-04-03 17:36:53 +02:00
profileLocal . getTargetTime ( plasmaSearchTimingProfile . PROCESS_JOIN ) ,
query . maxDistance ) ;
profileLocal . setYieldTime ( plasmaSearchTimingProfile . PROCESS_JOIN ) ;
profileLocal . setYieldCount ( plasmaSearchTimingProfile . PROCESS_JOIN , rcLocal . size ( ) ) ;
return rcLocal . size ( ) ;
2005-10-10 02:32:15 +02:00
}
2005-10-23 19:50:27 +02:00
public int globalSearch ( int fetchpeers ) {
2005-10-13 15:57:15 +02:00
// do global fetching
// the result of the fetch is then in the rcGlobal
if ( fetchpeers < 10 ) fetchpeers = 10 ;
2005-10-23 19:50:27 +02:00
2006-02-05 00:11:31 +01:00
log . logFine ( " STARTING " + fetchpeers + " THREADS TO CATCH EACH " + profileGlobal . getTargetCount ( plasmaSearchTimingProfile . PROCESS_POSTSORT ) + " URLs WITHIN " + ( profileGlobal . duetime ( ) / 1000 ) + " SECONDS " ) ;
2006-02-26 12:30:37 +01:00
2005-10-23 19:50:27 +02:00
long timeout = System . currentTimeMillis ( ) + profileGlobal . duetime ( ) + 4000 ;
2006-04-20 12:15:00 +02:00
searchThreads = yacySearch . searchHashes ( query . queryHashes , query . prefer , query . urlMask , query . maxDistance , urlStore , rcGlobal , fetchpeers , plasmaSwitchboard . urlBlacklist , snippetCache , profileGlobal , ranking ) ;
2006-02-26 12:30:37 +01:00
2005-10-13 15:57:15 +02:00
// wait until wanted delay passed or wanted result appeared
while ( System . currentTimeMillis ( ) < timeout ) {
// check if all threads have been finished or results so far are enough
2006-02-05 00:11:31 +01:00
if ( rcGlobal . size ( ) > = profileGlobal . getTargetCount ( plasmaSearchTimingProfile . PROCESS_POSTSORT ) * 5 ) break ; // we have enough
2005-10-13 15:57:15 +02:00
if ( yacySearch . remainingWaiting ( searchThreads ) = = 0 ) break ; // we cannot expect more
// wait a little time ..
2005-12-05 15:24:13 +01:00
try { Thread . sleep ( 100 ) ; } catch ( InterruptedException e ) { }
2005-10-13 15:57:15 +02:00
}
return rcGlobal . size ( ) ;
}
2006-01-30 01:42:38 +01:00
public plasmaSearchResult order ( ) {
2005-10-13 15:57:15 +02:00
// we collect the urlhashes and construct a list with urlEntry objects
// attention: if minEntries is too high, this method will not terminate within the maxTime
2005-10-12 14:28:49 +02:00
2006-08-03 01:20:03 +02:00
indexContainer searchResult = new indexRowSetContainer ( null ) ;
2006-02-05 00:11:31 +01:00
long preorderTime = profileLocal . getTargetTime ( plasmaSearchTimingProfile . PROCESS_PRESORT ) ;
2005-10-23 19:50:27 +02:00
profileLocal . startTimer ( ) ;
2006-04-02 22:40:07 +02:00
long pst = System . currentTimeMillis ( ) ;
2006-04-04 01:26:08 +02:00
searchResult . add ( rcLocal , preorderTime ) ;
searchResult . add ( rcGlobal , preorderTime ) ;
2006-04-02 22:40:07 +02:00
preorderTime = preorderTime - ( System . currentTimeMillis ( ) - pst ) ;
2006-04-04 01:26:08 +02:00
if ( preorderTime < 0 ) preorderTime = 200 ;
2006-02-05 00:51:00 +01:00
plasmaSearchPreOrder preorder = new plasmaSearchPreOrder ( query , ranking ) ;
2006-01-30 01:42:38 +01:00
preorder . addContainer ( searchResult , preorderTime ) ;
2006-02-05 00:11:31 +01:00
profileLocal . setYieldTime ( plasmaSearchTimingProfile . PROCESS_PRESORT ) ;
profileLocal . setYieldCount ( plasmaSearchTimingProfile . PROCESS_PRESORT , rcLocal . size ( ) ) ;
2005-10-23 19:50:27 +02:00
2006-04-04 01:26:08 +02:00
// start url-fetch
long postorderTime = profileLocal . getTargetTime ( plasmaSearchTimingProfile . PROCESS_POSTSORT ) ;
long postorderLimitTime = ( postorderTime < 0 ) ? Long . MAX_VALUE : ( System . currentTimeMillis ( ) + postorderTime ) ;
2005-10-23 19:50:27 +02:00
profileLocal . startTimer ( ) ;
2006-02-05 00:51:00 +01:00
plasmaSearchResult acc = new plasmaSearchResult ( query , ranking ) ;
2006-04-04 01:26:08 +02:00
//if (searchResult == null) return acc; // strange case where searchResult is not proper: acc is then empty
//if (searchResult.size() == 0) return acc; // case that we have nothing to do
2006-08-02 21:59:28 +02:00
indexEntry entry ;
2005-10-12 14:28:49 +02:00
plasmaCrawlLURL . Entry page ;
2006-02-05 00:11:31 +01:00
int minEntries = profileLocal . getTargetCount ( plasmaSearchTimingProfile . PROCESS_POSTSORT ) ;
2005-12-15 11:31:00 +01:00
try {
while ( preorder . hasNext ( ) ) {
2006-04-02 22:52:34 +02:00
if ( ( acc . sizeFetched ( ) > = 50 ) & & ( ( acc . sizeFetched ( ) > = minEntries ) | | ( System . currentTimeMillis ( ) > = postorderLimitTime ) ) ) break ;
2005-12-07 00:51:29 +01:00
entry = preorder . next ( ) ;
2005-10-12 14:28:49 +02:00
// find the url entry
2005-12-15 11:31:00 +01:00
try {
2006-07-24 00:39:41 +02:00
page = urlStore . getEntry ( entry . urlHash ( ) , entry ) ;
2005-12-15 11:31:00 +01:00
// add a result
acc . addResult ( entry , page ) ;
} catch ( IOException e ) {
// result was not found
}
}
} catch ( kelondroException ee ) {
serverLog . logSevere ( " PLASMA " , " Database Failure during plasmaSearch.order: " + ee . getMessage ( ) , ee ) ;
}
2006-02-05 00:11:31 +01:00
profileLocal . setYieldTime ( plasmaSearchTimingProfile . PROCESS_URLFETCH ) ;
profileLocal . setYieldCount ( plasmaSearchTimingProfile . PROCESS_URLFETCH , acc . sizeFetched ( ) ) ;
2005-10-23 19:50:27 +02:00
// start postsorting
profileLocal . startTimer ( ) ;
2005-10-12 14:28:49 +02:00
acc . sortResults ( ) ;
2006-02-05 00:11:31 +01:00
profileLocal . setYieldTime ( plasmaSearchTimingProfile . PROCESS_POSTSORT ) ;
profileLocal . setYieldCount ( plasmaSearchTimingProfile . PROCESS_POSTSORT , acc . sizeOrdered ( ) ) ;
2005-10-27 02:42:08 +02:00
// apply filter
profileLocal . startTimer ( ) ;
2005-11-22 16:17:05 +01:00
//acc.removeRedundant();
2006-03-04 01:10:49 +01:00
acc . removeDoubleDom ( ) ;
2006-02-05 00:11:31 +01:00
profileLocal . setYieldTime ( plasmaSearchTimingProfile . PROCESS_FILTER ) ;
profileLocal . setYieldCount ( plasmaSearchTimingProfile . PROCESS_FILTER , acc . sizeOrdered ( ) ) ;
2005-10-27 02:42:08 +02:00
2005-10-23 19:50:27 +02:00
return acc ;
2005-10-12 14:28:49 +02:00
}
2005-12-15 14:21:42 +01:00
public void run ( ) {
flushThreads . add ( this ) ; // this will care that the search event object is referenced from somewhere while it is still alive
2006-01-31 02:20:28 +01:00
2005-10-13 15:57:15 +02:00
// put all new results into wordIndex
// this must be called after search results had been computed
2006-01-31 02:20:28 +01:00
// it is wise to call this within a separate thread because
// this method waits until all threads are finished
2005-10-23 19:50:27 +02:00
2005-10-13 15:57:15 +02:00
int remaining ;
long starttime = System . currentTimeMillis ( ) ;
2006-01-31 02:20:28 +01:00
while ( ( searchThreads ! = null ) & & ( ( remaining = yacySearch . remainingWaiting ( searchThreads ) ) > 0 ) ) {
2006-02-04 15:13:54 +01:00
flushGlobalResults ( ) ;
2006-01-31 02:20:28 +01:00
2005-10-23 19:50:27 +02:00
// wait a little bit before trying again
2005-12-05 15:24:13 +01:00
try { Thread . sleep ( 3000 ) ; } catch ( InterruptedException e ) { }
2005-10-13 15:57:15 +02:00
if ( System . currentTimeMillis ( ) - starttime > 90000 ) {
yacySearch . interruptAlive ( searchThreads ) ;
2005-12-15 14:21:42 +01:00
log . logFine ( " SEARCH FLUSH: " + remaining + " PEERS STILL BUSY; ABANDONED; SEARCH WAS " + query . queryWords ) ;
2005-10-13 15:57:15 +02:00
break ;
}
2005-12-15 14:21:42 +01:00
log . logFine ( " FINISHED FLUSH RESULTS PROCESS for query " + query . hashes ( " , " ) ) ;
2005-10-13 15:57:15 +02:00
}
2006-02-04 15:13:54 +01:00
serverLog . logFine ( " PLASMA " , " FINISHED FLUSHING " + rcGlobalCount + " GLOBAL SEARCH RESULTS FOR SEARCH " + query . queryWords ) ;
2006-01-31 02:20:28 +01:00
2005-10-13 15:57:15 +02:00
// finally delete the temporary index
rcGlobal = null ;
2006-01-31 02:20:28 +01:00
flushThreads . remove ( this ) ;
}
2006-02-04 15:13:54 +01:00
public void flushGlobalResults ( ) {
2006-01-31 02:20:28 +01:00
// flush the rcGlobal as much as is there so far
// this must be called sometime after search results had been computed
int count = 0 ;
if ( ( rcGlobal ! = null ) & & ( rcGlobal . size ( ) > 0 ) ) {
synchronized ( rcGlobal ) {
String wordHash ;
Iterator hashi = query . queryHashes . iterator ( ) ;
while ( hashi . hasNext ( ) ) {
wordHash = ( String ) hashi . next ( ) ;
rcGlobal . setWordHash ( wordHash ) ;
2006-03-13 11:43:12 +01:00
wordIndex . addEntries ( rcGlobal , System . currentTimeMillis ( ) , false ) ;
2006-01-31 02:20:28 +01:00
log . logFine ( " FLUSHED " + wordHash + " : " + rcGlobal . size ( ) + " url entries " ) ;
}
// the rcGlobal was flushed, empty it
count + = rcGlobal . size ( ) ;
rcGlobal . clear ( ) ;
}
}
2006-02-04 15:13:54 +01:00
rcGlobalCount + = count ;
2005-10-13 15:57:15 +02:00
}
2005-10-10 02:32:15 +02:00
}