Marown/yacy_search_server
1
0
Fork 0
mirror of https://github.com/yacy/yacy_search_server.git synced 2024-09-21 00:00:13 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity

finals;

Browse Source 
cleaned;
Properties;

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@770 6c8d7289-2bf4-0310-a012-ef5d649a1542
This commit is contained in:
borg-0300 2005-09-21 23:18:12 +00:00
parent 979a3ee3c0
commit a1f5027a88
1 changed files with 65 additions and 56 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

85
source/de/anomic/yacy/yacySearch.java
View File

@ -3,7 +3,10 @@
// (C) by Michael Peter Christen; mc@anomic.de // (C) by Michael Peter Christen; mc@anomic.de
// first published on http://www.anomic.de // first published on http://www.anomic.de
// Frankfurt, Germany, 2004 // Frankfurt, Germany, 2004
// last major change: 13.06.2004 //
// $LastChangedDate$
// $LastChangedRevision$
// $LastChangedBy$
// //
// This program is free software; you can redistribute it and/or modify // This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by // it under the terms of the GNU General Public License as published by
@ -54,16 +57,16 @@ import de.anomic.server.logging.serverLog;
public class yacySearch extends Thread { public class yacySearch extends Thread {
private Set wordhashes; final private Set wordhashes;
private int count; final private int count;
private boolean global; final private boolean global;
private plasmaCrawlLURL urlManager; final private plasmaCrawlLURL urlManager;
private plasmaSearch searchManager; final private plasmaSearch searchManager;
private plasmaURLPattern blacklist; final private plasmaURLPattern blacklist;
private plasmaSnippetCache snippetCache; final private plasmaSnippetCache snippetCache;
private yacySeed targetPeer; final private yacySeed targetPeer;
private int links; private int links;
private long duetime; final private long duetime;
public yacySearch(Set wordhashes, int count, boolean global, yacySeed targetPeer, public yacySearch(Set wordhashes, int count, boolean global, yacySeed targetPeer,
plasmaCrawlLURL urlManager, plasmaSearch searchManager, plasmaURLPattern blacklist, plasmaSnippetCache snippetCache, long duetime) { plasmaCrawlLURL urlManager, plasmaSearch searchManager, plasmaURLPattern blacklist, plasmaSnippetCache snippetCache, long duetime) {
@ -91,8 +94,8 @@ public class yacySearch extends Thread {
public static String set2string(Set hashes) { public static String set2string(Set hashes) {
String wh = ""; String wh = "";
Iterator i = hashes.iterator(); final Iterator iter = hashes.iterator();
while (i.hasNext()) wh = wh + (String) i.next(); while (iter.hasNext()) { wh = wh + (String) iter.next(); }
return wh; return wh;
} }
@ -104,27 +107,27 @@ public class yacySearch extends Thread {
// find out a specific number of seeds, that would be relevant for the given word hash(es) // find out a specific number of seeds, that would be relevant for the given word hash(es)
// the result is ordered by relevance: [0] is most relevant // the result is ordered by relevance: [0] is most relevant
// the seedcount is the maximum number of wanted results // the seedcount is the maximum number of wanted results
if (yacyCore.seedDB == null) return null; if (yacyCore.seedDB == null) { return null; }
if (seedcount > yacyCore.seedDB.sizeConnected()) seedcount = yacyCore.seedDB.sizeConnected(); if (seedcount > yacyCore.seedDB.sizeConnected()) { seedcount = yacyCore.seedDB.sizeConnected(); }
// put in seeds according to dht // put in seeds according to dht
kelondroMScoreCluster ranking = new kelondroMScoreCluster(); final kelondroMScoreCluster ranking = new kelondroMScoreCluster();
HashMap seeds = new HashMap(); final HashMap seeds = new HashMap();
yacySeed seed; yacySeed seed;
Enumeration dhtEnum; Enumeration dhtEnum;
Iterator i = wordhashes.iterator();
int c; int c;
String wordhash; String wordhash;
double distance; double distance;
while (i.hasNext()) { Iterator iter = wordhashes.iterator();
wordhash = (String) i.next(); while (iter.hasNext()) {
wordhash = (String) iter.next();
dhtEnum = yacyCore.dhtAgent.getDHTSeeds(true, wordhash); dhtEnum = yacyCore.dhtAgent.getDHTSeeds(true, wordhash);
c = seedcount; c = seedcount;
while ((dhtEnum.hasMoreElements()) && (c > 0)) { while (dhtEnum.hasMoreElements() && c > 0) {
seed = (yacySeed) dhtEnum.nextElement(); seed = (yacySeed) dhtEnum.nextElement();
if (seed == null) continue; if (seed == null) { continue; }
distance = yacyDHTAction.dhtDistance(seed.hash, wordhash); distance = yacyDHTAction.dhtDistance(seed.hash, wordhash);
if (distance > 0.9) continue; // catch bug in peer selection if (distance > 0.9) { continue; } // catch bug in peer selection
serverLog.logFine("PLASMA", "selectPeers/DHTorder: " + seed.hash + ":" + seed.getName() + "/" + distance + " for wordhash " + wordhash + ", score " + c); serverLog.logFine("PLASMA", "selectPeers/DHTorder: " + seed.hash + ":" + seed.getName() + "/" + distance + " for wordhash " + wordhash + ", score " + c);
ranking.addScore(seed.hash, c--); ranking.addScore(seed.hash, c--);
seeds.put(seed.hash, seed); seeds.put(seed.hash, seed);
@ -135,10 +138,10 @@ public class yacySearch extends Thread {
dhtEnum = yacyCore.seedDB.seedsSortedConnected(false, "ICount"); dhtEnum = yacyCore.seedDB.seedsSortedConnected(false, "ICount");
c = seedcount; c = seedcount;
int score; int score;
if (c > yacyCore.seedDB.sizeConnected()) c = yacyCore.seedDB.sizeConnected(); if (c > yacyCore.seedDB.sizeConnected()) { c = yacyCore.seedDB.sizeConnected(); }
while ((dhtEnum.hasMoreElements()) && (c > 0)) { while (dhtEnum.hasMoreElements() && c > 0) {
seed = (yacySeed) dhtEnum.nextElement(); seed = (yacySeed) dhtEnum.nextElement();
if (seed == null) continue; if (seed == null) { continue; }
score = (int) Math.round(Math.random() * ((c / 3) + 3)); score = (int) Math.round(Math.random() * ((c / 3) + 3));
serverLog.logFine("PLASMA", "selectPeers/RWIcount: " + seed.hash + ":" + seed.getName() + ", RWIcount=" + seed.get("ICount","") + ", score " + score); serverLog.logFine("PLASMA", "selectPeers/RWIcount: " + seed.hash + ":" + seed.getName() + ", RWIcount=" + seed.get("ICount","") + ", score " + score);
ranking.addScore(seed.hash, score); ranking.addScore(seed.hash, score);
@ -147,12 +150,12 @@ public class yacySearch extends Thread {
} }
// evaluate the ranking score and select seeds // evaluate the ranking score and select seeds
if (ranking.size() < seedcount) seedcount = ranking.size(); if (ranking.size() < seedcount) { seedcount = ranking.size(); }
yacySeed[] result = new yacySeed[seedcount]; yacySeed[] result = new yacySeed[seedcount];
Iterator e = ranking.scores(false); // higher are better
c = 0; c = 0;
while ((e.hasNext()) && (c < result.length)) { iter = ranking.scores(false); // higher are better
seed = (yacySeed) seeds.get((String) e.next()); while (iter.hasNext() && c < result.length) {
seed = (yacySeed) seeds.get((String) iter.next());
seed.selectscore = c; seed.selectscore = c;
serverLog.logFine("PLASMA", "selectPeers/_lineup_: " + seed.hash + ":" + seed.getName() + " is choice " + c); serverLog.logFine("PLASMA", "selectPeers/_lineup_: " + seed.hash + ":" + seed.getName() + " is choice " + c);
result[c++] = seed; result[c++] = seed;
@ -165,21 +168,21 @@ public class yacySearch extends Thread {
public static int searchHashes(Set wordhashes, plasmaCrawlLURL urlManager, plasmaSearch searchManager, public static int searchHashes(Set wordhashes, plasmaCrawlLURL urlManager, plasmaSearch searchManager,
int count, int targets, plasmaURLPattern blacklist, plasmaSnippetCache snippetCache, long waitingtime) { int count, int targets, plasmaURLPattern blacklist, plasmaSnippetCache snippetCache, long waitingtime) {
// check own peer status // check own peer status
if ((yacyCore.seedDB.mySeed == null) || (yacyCore.seedDB.mySeed.getAddress() == null)) return 0; if (yacyCore.seedDB.mySeed == null || yacyCore.seedDB.mySeed.getAddress() == null) { return 0; }
// start delay control // start delay control
long start = System.currentTimeMillis(); final long start = System.currentTimeMillis();
// set a duetime for clients // set a duetime for clients
long duetime = waitingtime - 4000; // subtract network traffic overhead, guessed 4 seconds long duetime = waitingtime - 4000; // subtract network traffic overhead, guessed 4 seconds
if (duetime < 1000) duetime = 1000; if (duetime < 1000) { duetime = 1000; }
// prepare seed targets and threads // prepare seed targets and threads
//Set wordhashes = plasmaSearch.words2hashes(querywords); //Set wordhashes = plasmaSearch.words2hashes(querywords);
yacySeed[] targetPeers = selectPeers(wordhashes, targets); final yacySeed[] targetPeers = selectPeers(wordhashes, targets);
if (targetPeers == null) return 0; if (targetPeers == null) { return 0; }
targets = targetPeers.length; targets = targetPeers.length;
if (targets == 0) return 0; if (targets == 0) { return 0; }
yacySearch[] searchThreads = new yacySearch[targets]; yacySearch[] searchThreads = new yacySearch[targets];
for (int i = 0; i < targets; i++) { for (int i = 0; i < targets; i++) {
searchThreads[i]= new yacySearch(wordhashes, count, true, targetPeers[i], searchThreads[i]= new yacySearch(wordhashes, count, true, targetPeers[i],
@ -200,20 +203,26 @@ public class yacySearch extends Thread {
c = 0; c = 0;
anyIdle = false; anyIdle = false;
for (int i = 0; i < targets; i++) { for (int i = 0; i < targets; i++) {
if (searchThreads[i].links() < 0) anyIdle = true; else c = c + searchThreads[i].links(); if (searchThreads[i].links() < 0) {
anyIdle = true;
} else {
c = c + searchThreads[i].links();
}
} }
if ((c >= count * 3) && ((System.currentTimeMillis() - start) > (waitingtime * 2 / 3))) { if ((c >= count * 3) && ((System.currentTimeMillis() - start) > (waitingtime * 2 / 3))) {
yacyCore.log.logFine("DEBUG yacySearch: c=" + c + ", count=" + count + ", waitingtime=" + waitingtime); yacyCore.log.logFine("DEBUG yacySearch: c=" + c + ", count=" + count + ", waitingtime=" + waitingtime);
break; // we have enough break; // we have enough
} }
if (c >= count * 5) break; if (c >= count * 5) { break; }
// wait a little time .. // wait a little time ..
try {Thread.currentThread().sleep(100);} catch (InterruptedException e) {} try {Thread.currentThread().sleep(100);} catch (InterruptedException e) {}
} }
// collect results // collect results
c = 0; c = 0;
for (int i = 0; i < targets; i++) c = c + ((searchThreads[i].links() > 0) ? searchThreads[i].links() : 0); for (int i = 0; i < targets; i++) {
c = c + ((searchThreads[i].links() > 0) ? searchThreads[i].links() : 0);
}
return c; return c;
} }