mirror of
https://github.com/yacy/yacy_search_server.git
synced 2024-09-21 00:00:13 +02:00
ce1adf9955
high-performance search query situations as seen in yacy-metager integration showed deadlock situation caused by synchronization effects inside of sun.java code. It appears that the logger is not completely safe against deadlock situations in concurrent calls of the logger. One possible solution would be a outside-synchronization with 'synchronized' statements, but that would further apply blocking on all high-efficient methods that call the logger. It is much better to do a non-blocking hand-over of logging lines and work off log entries with a concurrent log writer. This also disconnects IO operations from logging, which can also cause IO operation when a log is written to a file. This commit not only moves the logger from kelondro to yacy.logging, it also inserts the concurrency methods to realize non-blocking logging. git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@6078 6c8d7289-2bf4-0310-a012-ef5d649a1542
392 lines
16 KiB
Java
Executable File
392 lines
16 KiB
Java
Executable File
// PeerSelection.java
|
|
// -------------------------------------
|
|
// (C) by Michael Peter Christen; mc@yacy.net
|
|
// first published 05.11.2008 on http://yacy.net
|
|
// Frankfurt, Germany, 2008
|
|
//
|
|
// $LastChangedDate: 2008-09-03 02:30:21 +0200 (Mi, 03 Sep 2008) $
|
|
// $LastChangedRevision: 5102 $
|
|
// $LastChangedBy: orbiter $
|
|
//
|
|
// This program is free software; you can redistribute it and/or modify
|
|
// it under the terms of the GNU General Public License as published by
|
|
// the Free Software Foundation; either version 2 of the License, or
|
|
// (at your option) any later version.
|
|
//
|
|
// This program is distributed in the hope that it will be useful,
|
|
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
// GNU General Public License for more details.
|
|
//
|
|
// You should have received a copy of the GNU General Public License
|
|
// along with this program; if not, write to the Free Software
|
|
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
|
|
|
package de.anomic.yacy.dht;
|
|
|
|
import java.util.ArrayList;
|
|
import java.util.HashMap;
|
|
import java.util.HashSet;
|
|
import java.util.Iterator;
|
|
|
|
import de.anomic.kelondro.util.DateFormatter;
|
|
import de.anomic.kelondro.util.kelondroException;
|
|
import de.anomic.kelondro.order.Base64Order;
|
|
import de.anomic.kelondro.order.Digest;
|
|
import de.anomic.kelondro.util.ScoreCluster;
|
|
import de.anomic.yacy.yacyCore;
|
|
import de.anomic.yacy.yacySeed;
|
|
import de.anomic.yacy.yacySeedDB;
|
|
import de.anomic.yacy.yacyVersion;
|
|
import de.anomic.yacy.dht.PartitionScheme;
|
|
import de.anomic.yacy.logging.Log;
|
|
|
|
|
|
/*
|
|
* this package is a collection of peer selection iterations that had been
|
|
* part of yacyPeerActions, yacyDHTActions and yacySeedDB
|
|
*/
|
|
|
|
public class PeerSelection {
|
|
|
|
public static void selectDHTPositions(
|
|
final yacySeedDB seedDB,
|
|
byte[] wordhash,
|
|
int redundancy,
|
|
HashMap<String, yacySeed> regularSeeds,
|
|
ScoreCluster<String> ranking) {
|
|
// this method is called from the search target computation
|
|
long[] dhtVerticalTargets = seedDB.scheme.dhtPositions(wordhash);
|
|
yacySeed seed;
|
|
for (int v = 0; v < dhtVerticalTargets.length; v++) {
|
|
wordhash = FlatWordPartitionScheme.positionToHash(dhtVerticalTargets[v]);
|
|
Iterator<yacySeed> dhtEnum = getAcceptRemoteIndexSeeds(seedDB, wordhash, redundancy, false);
|
|
int c = Math.min(seedDB.sizeConnected(), redundancy);
|
|
int cc = 3; // select a maximum of 3, this is enough redundancy
|
|
while (dhtEnum.hasNext() && c > 0 && cc-- > 0) {
|
|
seed = dhtEnum.next();
|
|
if (seed == null || seed.hash == null) continue;
|
|
if (!seed.getFlagAcceptRemoteIndex()) continue; // probably a robinson peer
|
|
if (Log.isFine("PLASMA")) Log.logFine("PLASMA", "selectPeers/DHTorder: " + seed.hash + ":" + seed.getName() + "/ score " + c);
|
|
ranking.addScore(seed.hash, 2 * c);
|
|
regularSeeds.put(seed.hash, seed);
|
|
c--;
|
|
}
|
|
}
|
|
}
|
|
|
|
private static int guessedOwn = 0;
|
|
//private static int guessedNotOwn = 0;
|
|
private static int verifiedOwn = 0;
|
|
private static int verifiedNotOwn = 0;
|
|
|
|
public static boolean shallBeOwnWord(final yacySeedDB seedDB, final byte[] wordhash, String urlhash, int redundancy, PartitionScheme scheme) {
|
|
// the guessIfOwnWord is a fast method that should only fail in case that a 'true' may be incorrect, but a 'false' shall always be correct
|
|
if (guessIfOwnWord(seedDB, wordhash, urlhash)) {
|
|
// this case must be verified, because it can be wrong.
|
|
guessedOwn++;
|
|
if (verifyIfOwnWord(seedDB, wordhash, urlhash, redundancy)) {
|
|
// this is the correct case, but does not need to be an average case
|
|
verifiedOwn++;
|
|
//System.out.println("*** DEBUG shallBeOwnWord: true. guessed: true. verified/guessed ration = " + verifiedOwn + "/" + guessedOwn);
|
|
return true;
|
|
} else {
|
|
// this may happen, but can be corrected
|
|
verifiedNotOwn++;
|
|
//System.out.println("*** DEBUG shallBeOwnWord: false. guessed: true. verified/guessed ration = " + verifiedNotOwn + "/" + guessedNotOwn);
|
|
return false;
|
|
}
|
|
} else {
|
|
return false;
|
|
/*
|
|
// this should mean that the guessing should not be wrong
|
|
guessedNotOwn++;
|
|
if (yacyPeerSelection.verifyIfOwnWord(seedDB, wordhash, redundancy)) {
|
|
// this should never happen
|
|
verifiedOwn++;
|
|
System.out.println("*** DEBUG shallBeOwnWord: true. guessed: false. verified/guessed ration = " + verifiedOwn + "/" + guessedOwn);
|
|
return true;
|
|
} else {
|
|
// this should always happen
|
|
verifiedNotOwn++;
|
|
//System.out.println("*** DEBUG shallBeOwnWord: false. guessed: false. verified/guessed ration = " + verifiedNotOwn + "/" + guessedNotOwn);
|
|
return false;
|
|
}
|
|
*/
|
|
}
|
|
|
|
}
|
|
|
|
public static boolean guessIfOwnWord(final yacySeedDB seedDB, final byte[] wordhash, final String urlhash) {
|
|
if (seedDB == null) return false;
|
|
int connected = seedDB.sizeConnected();
|
|
if (connected == 0) return true;
|
|
final long target = seedDB.scheme.dhtPosition(wordhash, urlhash);
|
|
final long mypos = seedDB.scheme.dhtPosition(seedDB.mySeed().hash.getBytes(), urlhash);
|
|
long distance = FlatWordPartitionScheme.dhtDistance(target, mypos);
|
|
if (distance <= 0) return false;
|
|
if (distance <= Long.MAX_VALUE / connected * 2) return true;
|
|
return false;
|
|
}
|
|
|
|
public static boolean verifyIfOwnWord(final yacySeedDB seedDB, byte[] wordhash, String urlhash, int redundancy) {
|
|
String myHash = seedDB.mySeed().hash;
|
|
wordhash = FlatWordPartitionScheme.positionToHash(seedDB.scheme.dhtPosition(wordhash, urlhash));
|
|
Iterator<yacySeed> dhtEnum = getAcceptRemoteIndexSeeds(seedDB, wordhash, redundancy, true);
|
|
while (dhtEnum.hasNext()) {
|
|
if (dhtEnum.next().hash.equals(myHash)) return true;
|
|
}
|
|
return false;
|
|
}
|
|
|
|
public static byte[] selectTransferStart() {
|
|
return Base64Order.enhancedCoder.encode(Digest.encodeMD5Raw(Long.toString(System.currentTimeMillis()))).substring(2, 2 + yacySeedDB.commonHashLength).getBytes();
|
|
}
|
|
|
|
public static byte[] limitOver(final yacySeedDB seedDB, final byte[] startHash) {
|
|
Iterator<yacySeed> seeds = getAcceptRemoteIndexSeeds(seedDB, startHash, 1, false);
|
|
if (seeds.hasNext()) return seeds.next().hash.getBytes();
|
|
return null;
|
|
}
|
|
|
|
public static ArrayList<yacySeed> getAcceptRemoteIndexSeedsList(
|
|
yacySeedDB seedDB,
|
|
final byte[] starthash,
|
|
int max,
|
|
boolean alsoMyOwn) {
|
|
final Iterator<yacySeed> seedIter = PeerSelection.getAcceptRemoteIndexSeeds(seedDB, starthash, max, alsoMyOwn);
|
|
ArrayList<yacySeed> targets = new ArrayList<yacySeed>();
|
|
while (seedIter.hasNext() && max-- > 0) targets.add(seedIter.next());
|
|
return targets;
|
|
}
|
|
|
|
/**
|
|
* returns an enumeration of yacySeed-Objects that have the AcceptRemoteIndex-Flag set
|
|
* the seeds are enumerated in the right order according to the DHT
|
|
* @param seedDB
|
|
* @param starthash
|
|
* @param max
|
|
* @param alsoMyOwn
|
|
* @return
|
|
*/
|
|
public static Iterator<yacySeed> getAcceptRemoteIndexSeeds(yacySeedDB seedDB, final byte[] starthash, int max, boolean alsoMyOwn) {
|
|
return new acceptRemoteIndexSeedEnum(seedDB, starthash, Math.min(max, seedDB.sizeConnected()), alsoMyOwn);
|
|
}
|
|
|
|
private static class acceptRemoteIndexSeedEnum implements Iterator<yacySeed> {
|
|
|
|
private Iterator<yacySeed> se;
|
|
private yacySeed nextSeed;
|
|
private yacySeedDB seedDB;
|
|
private HashSet<String> doublecheck;
|
|
private int remaining;
|
|
private boolean alsoMyOwn;
|
|
|
|
public acceptRemoteIndexSeedEnum(yacySeedDB seedDB, final byte[] starthash, int max, boolean alsoMyOwn) {
|
|
this.seedDB = seedDB;
|
|
this.se = getDHTSeeds(seedDB, starthash, yacyVersion.YACY_HANDLES_COLLECTION_INDEX);
|
|
this.remaining = max;
|
|
this.doublecheck = new HashSet<String>();
|
|
this.nextSeed = nextInternal();
|
|
this.alsoMyOwn = alsoMyOwn && nextSeed != null && (Base64Order.enhancedCoder.compare(seedDB.mySeed().hash.getBytes(), nextSeed.hash.getBytes()) > 0);
|
|
}
|
|
|
|
public boolean hasNext() {
|
|
return nextSeed != null;
|
|
}
|
|
|
|
private yacySeed nextInternal() {
|
|
if (this.remaining <= 0) return null;
|
|
yacySeed s;
|
|
try {
|
|
while (se.hasNext()) {
|
|
s = se.next();
|
|
if (s == null) return null;
|
|
if (doublecheck.contains(s.hash)) return null;
|
|
this.doublecheck.add(s.hash);
|
|
if (s.getFlagAcceptRemoteIndex()) {
|
|
this.remaining--;
|
|
return s;
|
|
}
|
|
}
|
|
} catch (final kelondroException e) {
|
|
System.out.println("DEBUG acceptRemoteIndexSeedEnum:" + e.getMessage());
|
|
yacyCore.log.logSevere("database inconsistency (" + e.getMessage() + "), re-set of db.");
|
|
seedDB.resetActiveTable();
|
|
return null;
|
|
}
|
|
return null;
|
|
}
|
|
|
|
public yacySeed next() {
|
|
if (alsoMyOwn && Base64Order.enhancedCoder.compare(seedDB.mySeed().hash.getBytes(), nextSeed.hash.getBytes()) < 0) {
|
|
// take my own seed hash instead the enumeration result
|
|
alsoMyOwn = false;
|
|
return seedDB.mySeed();
|
|
} else {
|
|
final yacySeed next = nextSeed;
|
|
nextSeed = nextInternal();
|
|
return next;
|
|
}
|
|
}
|
|
|
|
public void remove() {
|
|
throw new UnsupportedOperationException();
|
|
}
|
|
|
|
}
|
|
|
|
/**
|
|
* enumerate seeds for DHT target positions
|
|
* @param seedDB
|
|
* @param firstHash
|
|
* @param minVersion
|
|
* @return
|
|
*/
|
|
private static Iterator<yacySeed> getDHTSeeds(yacySeedDB seedDB, final byte[] firstHash, final float minVersion) {
|
|
// enumerates seed-type objects: all seeds with starting point in the middle, rotating at the end/beginning
|
|
return new seedDHTEnum(seedDB, firstHash, minVersion);
|
|
}
|
|
|
|
private static class seedDHTEnum implements Iterator<yacySeed> {
|
|
|
|
Iterator<yacySeed> e1, e2;
|
|
int steps;
|
|
float minVersion;
|
|
yacySeedDB seedDB;
|
|
|
|
public seedDHTEnum(yacySeedDB seedDB, final byte[] firstHash, final float minVersion) {
|
|
this.seedDB = seedDB;
|
|
this.steps = seedDB.sizeConnected();
|
|
this.minVersion = minVersion;
|
|
this.e1 = seedDB.seedsConnected(true, false, firstHash, minVersion);
|
|
this.e2 = null;
|
|
}
|
|
|
|
public boolean hasNext() {
|
|
return (steps > 0) && ((e2 == null) || (e2.hasNext()));
|
|
}
|
|
|
|
public yacySeed next() {
|
|
if (steps == 0) return null;
|
|
steps--;
|
|
|
|
if (e1 == null || !e1.hasNext()) {
|
|
if (e2 == null) {
|
|
e1 = null;
|
|
e2 = seedDB.seedsConnected(true, false, null, minVersion);
|
|
}
|
|
return e2.next();
|
|
}
|
|
|
|
final yacySeed n = e1.next();
|
|
if (!(e1.hasNext())) {
|
|
e1 = null;
|
|
e2 = seedDB.seedsConnected(true, false, null, minVersion);
|
|
}
|
|
return n;
|
|
}
|
|
|
|
public void remove() {
|
|
throw new UnsupportedOperationException();
|
|
}
|
|
}
|
|
|
|
/**
|
|
* enumerate peers that provide remote crawl urls
|
|
* @param seedDB
|
|
* @return an iterator of seed objects
|
|
*/
|
|
public static Iterator<yacySeed> getProvidesRemoteCrawlURLs(yacySeedDB seedDB) {
|
|
return new providesRemoteCrawlURLsEnum(seedDB);
|
|
}
|
|
|
|
private static class providesRemoteCrawlURLsEnum implements Iterator<yacySeed> {
|
|
|
|
Iterator<yacySeed> se;
|
|
yacySeed nextSeed;
|
|
yacySeedDB seedDB;
|
|
|
|
public providesRemoteCrawlURLsEnum(yacySeedDB seedDB) {
|
|
this.seedDB = seedDB;
|
|
se = getDHTSeeds(seedDB, null, yacyVersion.YACY_POVIDES_REMOTECRAWL_LISTS);
|
|
nextSeed = nextInternal();
|
|
}
|
|
|
|
public boolean hasNext() {
|
|
return nextSeed != null;
|
|
}
|
|
|
|
private yacySeed nextInternal() {
|
|
yacySeed s;
|
|
try {
|
|
while (se.hasNext()) {
|
|
s = se.next();
|
|
if (s == null) return null;
|
|
if (s.getLong(yacySeed.RCOUNT, 0) > 0) return s;
|
|
}
|
|
} catch (final kelondroException e) {
|
|
System.out.println("DEBUG providesRemoteCrawlURLsEnum:" + e.getMessage());
|
|
yacyCore.log.logSevere("database inconsistency (" + e.getMessage() + "), re-set of db.");
|
|
seedDB.resetActiveTable();
|
|
return null;
|
|
}
|
|
return null;
|
|
}
|
|
|
|
public yacySeed next() {
|
|
final yacySeed next = nextSeed;
|
|
nextSeed = nextInternal();
|
|
return next;
|
|
}
|
|
|
|
public void remove() {
|
|
throw new UnsupportedOperationException();
|
|
}
|
|
|
|
}
|
|
|
|
/**
|
|
* get either the youngest or oldest peers from the seed db. Count as many as requested
|
|
* @param seedDB
|
|
* @param up if up = true then get the most recent peers, if up = false then get oldest
|
|
* @param count number of wanted peers
|
|
* @return a hash map of peer hashes to seed object
|
|
*/
|
|
public static HashMap<String, yacySeed> seedsByAge(yacySeedDB seedDB, final boolean up, int count) {
|
|
|
|
if (count > seedDB.sizeConnected()) count = seedDB.sizeConnected();
|
|
|
|
// fill a score object
|
|
final ScoreCluster<String> seedScore = new ScoreCluster<String>();
|
|
yacySeed ys;
|
|
long absage;
|
|
final Iterator<yacySeed> s = seedDB.seedsConnected(true, false, null, (float) 0.0);
|
|
int searchcount = 1000;
|
|
if (searchcount > seedDB.sizeConnected()) searchcount = seedDB.sizeConnected();
|
|
try {
|
|
while ((s.hasNext()) && (searchcount-- > 0)) {
|
|
ys = s.next();
|
|
if ((ys != null) && (ys.get(yacySeed.LASTSEEN, "").length() > 10)) try {
|
|
absage = Math.abs(System.currentTimeMillis() + DateFormatter.dayMillis - ys.getLastSeenUTC());
|
|
seedScore.addScore(ys.hash, (int) absage); // the higher absage, the older is the peer
|
|
} catch (final Exception e) {}
|
|
}
|
|
|
|
// result is now in the score object; create a result vector
|
|
final HashMap<String, yacySeed> result = new HashMap<String, yacySeed>();
|
|
final Iterator<String> it = seedScore.scores(up);
|
|
int c = 0;
|
|
while ((c < count) && (it.hasNext())) {
|
|
c++;
|
|
ys = seedDB.getConnected(it.next());
|
|
if ((ys != null) && (ys.hash != null)) result.put(ys.hash, ys);
|
|
}
|
|
return result;
|
|
} catch (final kelondroException e) {
|
|
yacyCore.log.logSevere("Internal Error at yacySeedDB.seedsByAge: " + e.getMessage(), e);
|
|
return null;
|
|
}
|
|
}
|
|
}
|