mirror of
https://github.com/yacy/yacy_search_server.git
synced 2024-09-21 00:00:13 +02:00
finals;
cleaned; Properties; git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@770 6c8d7289-2bf4-0310-a012-ef5d649a1542
This commit is contained in:
parent
979a3ee3c0
commit
a1f5027a88
|
@ -3,7 +3,10 @@
|
||||||
// (C) by Michael Peter Christen; mc@anomic.de
|
// (C) by Michael Peter Christen; mc@anomic.de
|
||||||
// first published on http://www.anomic.de
|
// first published on http://www.anomic.de
|
||||||
// Frankfurt, Germany, 2004
|
// Frankfurt, Germany, 2004
|
||||||
// last major change: 13.06.2004
|
//
|
||||||
|
// $LastChangedDate$
|
||||||
|
// $LastChangedRevision$
|
||||||
|
// $LastChangedBy$
|
||||||
//
|
//
|
||||||
// This program is free software; you can redistribute it and/or modify
|
// This program is free software; you can redistribute it and/or modify
|
||||||
// it under the terms of the GNU General Public License as published by
|
// it under the terms of the GNU General Public License as published by
|
||||||
|
@ -54,19 +57,19 @@ import de.anomic.server.logging.serverLog;
|
||||||
|
|
||||||
public class yacySearch extends Thread {
|
public class yacySearch extends Thread {
|
||||||
|
|
||||||
private Set wordhashes;
|
final private Set wordhashes;
|
||||||
private int count;
|
final private int count;
|
||||||
private boolean global;
|
final private boolean global;
|
||||||
private plasmaCrawlLURL urlManager;
|
final private plasmaCrawlLURL urlManager;
|
||||||
private plasmaSearch searchManager;
|
final private plasmaSearch searchManager;
|
||||||
private plasmaURLPattern blacklist;
|
final private plasmaURLPattern blacklist;
|
||||||
private plasmaSnippetCache snippetCache;
|
final private plasmaSnippetCache snippetCache;
|
||||||
private yacySeed targetPeer;
|
final private yacySeed targetPeer;
|
||||||
private int links;
|
private int links;
|
||||||
private long duetime;
|
final private long duetime;
|
||||||
|
|
||||||
public yacySearch(Set wordhashes, int count, boolean global, yacySeed targetPeer,
|
public yacySearch(Set wordhashes, int count, boolean global, yacySeed targetPeer,
|
||||||
plasmaCrawlLURL urlManager, plasmaSearch searchManager, plasmaURLPattern blacklist, plasmaSnippetCache snippetCache, long duetime) {
|
plasmaCrawlLURL urlManager, plasmaSearch searchManager, plasmaURLPattern blacklist, plasmaSnippetCache snippetCache, long duetime) {
|
||||||
super("yacySearch_" + targetPeer.getName());
|
super("yacySearch_" + targetPeer.getName());
|
||||||
this.wordhashes = wordhashes;
|
this.wordhashes = wordhashes;
|
||||||
this.count = count;
|
this.count = count;
|
||||||
|
@ -88,100 +91,100 @@ public class yacySearch extends Thread {
|
||||||
yacyCore.seedDB.mySeed.incRU(links);
|
yacyCore.seedDB.mySeed.incRU(links);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public static String set2string(Set hashes) {
|
public static String set2string(Set hashes) {
|
||||||
String wh = "";
|
String wh = "";
|
||||||
Iterator i = hashes.iterator();
|
final Iterator iter = hashes.iterator();
|
||||||
while (i.hasNext()) wh = wh + (String) i.next();
|
while (iter.hasNext()) { wh = wh + (String) iter.next(); }
|
||||||
return wh;
|
return wh;
|
||||||
}
|
}
|
||||||
|
|
||||||
public int links() {
|
public int links() {
|
||||||
return this.links;
|
return this.links;
|
||||||
}
|
}
|
||||||
|
|
||||||
private static yacySeed[] selectPeers(Set wordhashes, int seedcount) {
|
private static yacySeed[] selectPeers(Set wordhashes, int seedcount) {
|
||||||
// find out a specific number of seeds, that would be relevant for the given word hash(es)
|
// find out a specific number of seeds, that would be relevant for the given word hash(es)
|
||||||
// the result is ordered by relevance: [0] is most relevant
|
// the result is ordered by relevance: [0] is most relevant
|
||||||
// the seedcount is the maximum number of wanted results
|
// the seedcount is the maximum number of wanted results
|
||||||
if (yacyCore.seedDB == null) return null;
|
if (yacyCore.seedDB == null) { return null; }
|
||||||
if (seedcount > yacyCore.seedDB.sizeConnected()) seedcount = yacyCore.seedDB.sizeConnected();
|
if (seedcount > yacyCore.seedDB.sizeConnected()) { seedcount = yacyCore.seedDB.sizeConnected(); }
|
||||||
|
|
||||||
// put in seeds according to dht
|
// put in seeds according to dht
|
||||||
kelondroMScoreCluster ranking = new kelondroMScoreCluster();
|
final kelondroMScoreCluster ranking = new kelondroMScoreCluster();
|
||||||
HashMap seeds = new HashMap();
|
final HashMap seeds = new HashMap();
|
||||||
yacySeed seed;
|
yacySeed seed;
|
||||||
Enumeration dhtEnum;
|
Enumeration dhtEnum;
|
||||||
Iterator i = wordhashes.iterator();
|
|
||||||
int c;
|
int c;
|
||||||
String wordhash;
|
String wordhash;
|
||||||
double distance;
|
double distance;
|
||||||
while (i.hasNext()) {
|
Iterator iter = wordhashes.iterator();
|
||||||
wordhash = (String) i.next();
|
while (iter.hasNext()) {
|
||||||
|
wordhash = (String) iter.next();
|
||||||
dhtEnum = yacyCore.dhtAgent.getDHTSeeds(true, wordhash);
|
dhtEnum = yacyCore.dhtAgent.getDHTSeeds(true, wordhash);
|
||||||
c = seedcount;
|
c = seedcount;
|
||||||
while ((dhtEnum.hasMoreElements()) && (c > 0)) {
|
while (dhtEnum.hasMoreElements() && c > 0) {
|
||||||
seed = (yacySeed) dhtEnum.nextElement();
|
seed = (yacySeed) dhtEnum.nextElement();
|
||||||
if (seed == null) continue;
|
if (seed == null) { continue; }
|
||||||
distance = yacyDHTAction.dhtDistance(seed.hash, wordhash);
|
distance = yacyDHTAction.dhtDistance(seed.hash, wordhash);
|
||||||
if (distance > 0.9) continue; // catch bug in peer selection
|
if (distance > 0.9) { continue; } // catch bug in peer selection
|
||||||
serverLog.logFine("PLASMA", "selectPeers/DHTorder: " + seed.hash + ":" + seed.getName() + "/" + distance + " for wordhash " + wordhash + ", score " + c);
|
serverLog.logFine("PLASMA", "selectPeers/DHTorder: " + seed.hash + ":" + seed.getName() + "/" + distance + " for wordhash " + wordhash + ", score " + c);
|
||||||
ranking.addScore(seed.hash, c--);
|
ranking.addScore(seed.hash, c--);
|
||||||
seeds.put(seed.hash, seed);
|
seeds.put(seed.hash, seed);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// put in seeds according to size of peer
|
// put in seeds according to size of peer
|
||||||
dhtEnum = yacyCore.seedDB.seedsSortedConnected(false, "ICount");
|
dhtEnum = yacyCore.seedDB.seedsSortedConnected(false, "ICount");
|
||||||
c = seedcount;
|
c = seedcount;
|
||||||
int score;
|
int score;
|
||||||
if (c > yacyCore.seedDB.sizeConnected()) c = yacyCore.seedDB.sizeConnected();
|
if (c > yacyCore.seedDB.sizeConnected()) { c = yacyCore.seedDB.sizeConnected(); }
|
||||||
while ((dhtEnum.hasMoreElements()) && (c > 0)) {
|
while (dhtEnum.hasMoreElements() && c > 0) {
|
||||||
seed = (yacySeed) dhtEnum.nextElement();
|
seed = (yacySeed) dhtEnum.nextElement();
|
||||||
if (seed == null) continue;
|
if (seed == null) { continue; }
|
||||||
score = (int) Math.round(Math.random() * ((c / 3) + 3));
|
score = (int) Math.round(Math.random() * ((c / 3) + 3));
|
||||||
serverLog.logFine("PLASMA", "selectPeers/RWIcount: " + seed.hash + ":" + seed.getName() + ", RWIcount=" + seed.get("ICount","") + ", score " + score);
|
serverLog.logFine("PLASMA", "selectPeers/RWIcount: " + seed.hash + ":" + seed.getName() + ", RWIcount=" + seed.get("ICount","") + ", score " + score);
|
||||||
ranking.addScore(seed.hash, score);
|
ranking.addScore(seed.hash, score);
|
||||||
seeds.put(seed.hash, seed);
|
seeds.put(seed.hash, seed);
|
||||||
c--;
|
c--;
|
||||||
}
|
}
|
||||||
|
|
||||||
// evaluate the ranking score and select seeds
|
// evaluate the ranking score and select seeds
|
||||||
if (ranking.size() < seedcount) seedcount = ranking.size();
|
if (ranking.size() < seedcount) { seedcount = ranking.size(); }
|
||||||
yacySeed[] result = new yacySeed[seedcount];
|
yacySeed[] result = new yacySeed[seedcount];
|
||||||
Iterator e = ranking.scores(false); // higher are better
|
|
||||||
c = 0;
|
c = 0;
|
||||||
while ((e.hasNext()) && (c < result.length)) {
|
iter = ranking.scores(false); // higher are better
|
||||||
seed = (yacySeed) seeds.get((String) e.next());
|
while (iter.hasNext() && c < result.length) {
|
||||||
|
seed = (yacySeed) seeds.get((String) iter.next());
|
||||||
seed.selectscore = c;
|
seed.selectscore = c;
|
||||||
serverLog.logFine("PLASMA", "selectPeers/_lineup_: " + seed.hash + ":" + seed.getName() + " is choice " + c);
|
serverLog.logFine("PLASMA", "selectPeers/_lineup_: " + seed.hash + ":" + seed.getName() + " is choice " + c);
|
||||||
result[c++] = seed;
|
result[c++] = seed;
|
||||||
}
|
}
|
||||||
|
|
||||||
//System.out.println("DEBUG yacySearch.selectPeers = " + seedcount + " seeds:"); for (int i = 0; i < seedcount; i++) System.out.println(" #" + i + ":" + result[i]); // debug
|
// System.out.println("DEBUG yacySearch.selectPeers = " + seedcount + " seeds:"); for (int i = 0; i < seedcount; i++) System.out.println(" #" + i + ":" + result[i]); // debug
|
||||||
return result;
|
return result;
|
||||||
}
|
}
|
||||||
|
|
||||||
public static int searchHashes(Set wordhashes, plasmaCrawlLURL urlManager, plasmaSearch searchManager,
|
public static int searchHashes(Set wordhashes, plasmaCrawlLURL urlManager, plasmaSearch searchManager,
|
||||||
int count, int targets, plasmaURLPattern blacklist, plasmaSnippetCache snippetCache, long waitingtime) {
|
int count, int targets, plasmaURLPattern blacklist, plasmaSnippetCache snippetCache, long waitingtime) {
|
||||||
// check own peer status
|
// check own peer status
|
||||||
if ((yacyCore.seedDB.mySeed == null) || (yacyCore.seedDB.mySeed.getAddress() == null)) return 0;
|
if (yacyCore.seedDB.mySeed == null || yacyCore.seedDB.mySeed.getAddress() == null) { return 0; }
|
||||||
|
|
||||||
// start delay control
|
// start delay control
|
||||||
long start = System.currentTimeMillis();
|
final long start = System.currentTimeMillis();
|
||||||
|
|
||||||
// set a duetime for clients
|
// set a duetime for clients
|
||||||
long duetime = waitingtime - 4000; // subtract network traffic overhead, guessed 4 seconds
|
long duetime = waitingtime - 4000; // subtract network traffic overhead, guessed 4 seconds
|
||||||
if (duetime < 1000) duetime = 1000;
|
if (duetime < 1000) { duetime = 1000; }
|
||||||
|
|
||||||
// prepare seed targets and threads
|
// prepare seed targets and threads
|
||||||
//Set wordhashes = plasmaSearch.words2hashes(querywords);
|
//Set wordhashes = plasmaSearch.words2hashes(querywords);
|
||||||
yacySeed[] targetPeers = selectPeers(wordhashes, targets);
|
final yacySeed[] targetPeers = selectPeers(wordhashes, targets);
|
||||||
if (targetPeers == null) return 0;
|
if (targetPeers == null) { return 0; }
|
||||||
targets = targetPeers.length;
|
targets = targetPeers.length;
|
||||||
if (targets == 0) return 0;
|
if (targets == 0) { return 0; }
|
||||||
yacySearch[] searchThreads = new yacySearch[targets];
|
yacySearch[] searchThreads = new yacySearch[targets];
|
||||||
for (int i = 0; i < targets; i++) {
|
for (int i = 0; i < targets; i++) {
|
||||||
searchThreads[i]= new yacySearch(wordhashes, count, true, targetPeers[i],
|
searchThreads[i]= new yacySearch(wordhashes, count, true, targetPeers[i],
|
||||||
urlManager, searchManager, blacklist, snippetCache, duetime);
|
urlManager, searchManager, blacklist, snippetCache, duetime);
|
||||||
searchThreads[i].start();
|
searchThreads[i].start();
|
||||||
|
@ -200,20 +203,26 @@ public class yacySearch extends Thread {
|
||||||
c = 0;
|
c = 0;
|
||||||
anyIdle = false;
|
anyIdle = false;
|
||||||
for (int i = 0; i < targets; i++) {
|
for (int i = 0; i < targets; i++) {
|
||||||
if (searchThreads[i].links() < 0) anyIdle = true; else c = c + searchThreads[i].links();
|
if (searchThreads[i].links() < 0) {
|
||||||
|
anyIdle = true;
|
||||||
|
} else {
|
||||||
|
c = c + searchThreads[i].links();
|
||||||
|
}
|
||||||
}
|
}
|
||||||
if ((c >= count * 3) && ((System.currentTimeMillis() - start) > (waitingtime * 2 / 3))) {
|
if ((c >= count * 3) && ((System.currentTimeMillis() - start) > (waitingtime * 2 / 3))) {
|
||||||
yacyCore.log.logFine("DEBUG yacySearch: c=" + c + ", count=" + count + ", waitingtime=" + waitingtime);
|
yacyCore.log.logFine("DEBUG yacySearch: c=" + c + ", count=" + count + ", waitingtime=" + waitingtime);
|
||||||
break; // we have enough
|
break; // we have enough
|
||||||
}
|
}
|
||||||
if (c >= count * 5) break;
|
if (c >= count * 5) { break; }
|
||||||
// wait a little time ..
|
// wait a little time ..
|
||||||
try {Thread.currentThread().sleep(100);} catch (InterruptedException e) {}
|
try {Thread.currentThread().sleep(100);} catch (InterruptedException e) {}
|
||||||
}
|
}
|
||||||
|
|
||||||
// collect results
|
// collect results
|
||||||
c = 0;
|
c = 0;
|
||||||
for (int i = 0; i < targets; i++) c = c + ((searchThreads[i].links() > 0) ? searchThreads[i].links() : 0);
|
for (int i = 0; i < targets; i++) {
|
||||||
|
c = c + ((searchThreads[i].links() > 0) ? searchThreads[i].links() : 0);
|
||||||
|
}
|
||||||
return c;
|
return c;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
Loading…
Reference in New Issue
Block a user