mirror of
https://github.com/yacy/yacy_search_server.git
synced 2024-09-21 00:00:13 +02:00
01e0669264
and replaced old fist hash computation by new method that tries to find a gap in the current dht to do this, it is necessary that the network bootstraping is done before the own hash is computed this made further redesigns in peer initialization order necessary git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@4117 6c8d7289-2bf4-0310-a012-ef5d649a1542
1000 lines
42 KiB
Java
1000 lines
42 KiB
Java
// yacySeedDB.java
|
|
// -------------------------------------
|
|
// (C) by Michael Peter Christen; mc@anomic.de
|
|
// first published on http://www.anomic.de
|
|
// Frankfurt, Germany, 2004, 2005
|
|
//
|
|
// $LastChangedDate$
|
|
// $LastChangedRevision$
|
|
// $LastChangedBy$
|
|
//
|
|
// This program is free software; you can redistribute it and/or modify
|
|
// it under the terms of the GNU General Public License as published by
|
|
// the Free Software Foundation; either version 2 of the License, or
|
|
// (at your option) any later version.
|
|
//
|
|
// This program is distributed in the hope that it will be useful,
|
|
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
// GNU General Public License for more details.
|
|
//
|
|
// You should have received a copy of the GNU General Public License
|
|
// along with this program; if not, write to the Free Software
|
|
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
|
//
|
|
// Using this software in any meaning (reading, learning, copying, compiling,
|
|
// running) means that you agree that the Author(s) is (are) not responsible
|
|
// for cost, loss of data or any harm that may be caused directly or indirectly
|
|
// by usage of this softare or this documentation. The usage of this software
|
|
// is on your own risk. The installation and usage (starting/running) of this
|
|
// software may allow other people or application to access your computer and
|
|
// any attached devices and is highly dependent on the configuration of the
|
|
// software which must be done by the user of the software; the author(s) is
|
|
// (are) also not responsible for proper configuration and usage of the
|
|
// software, even if provoked by documentation provided together with
|
|
// the software.
|
|
//
|
|
// Any changes to this file according to the GPL as documented in the file
|
|
// gpl.txt aside this file in the shipment you received can be done to the
|
|
// lines that follows this copyright notice here, but changes must not be
|
|
// done inside the copyright notive above. A re-distribution must contain
|
|
// the intact and unchanged copyright notice.
|
|
// Contributions and changes to the program code must be marked as such.
|
|
|
|
package de.anomic.yacy;
|
|
|
|
import java.io.BufferedWriter;
|
|
import java.io.File;
|
|
import java.io.FileWriter;
|
|
import java.io.IOException;
|
|
import java.io.PrintWriter;
|
|
import java.lang.ref.SoftReference;
|
|
import java.lang.reflect.Method;
|
|
import java.net.InetAddress;
|
|
import java.net.UnknownHostException;
|
|
import java.util.ArrayList;
|
|
import java.util.HashMap;
|
|
import java.util.Hashtable;
|
|
import java.util.Iterator;
|
|
import java.util.Map;
|
|
import java.util.TreeMap;
|
|
|
|
import de.anomic.http.httpHeader;
|
|
import de.anomic.http.httpc;
|
|
import de.anomic.http.httpd;
|
|
import de.anomic.kelondro.kelondroDyn;
|
|
import de.anomic.kelondro.kelondroException;
|
|
import de.anomic.kelondro.kelondroMScoreCluster;
|
|
import de.anomic.kelondro.kelondroMapObjects;
|
|
import de.anomic.kelondro.kelondroBase64Order;
|
|
import de.anomic.plasma.plasmaHTCache;
|
|
import de.anomic.plasma.plasmaSwitchboard;
|
|
import de.anomic.server.serverCore;
|
|
import de.anomic.server.serverDomains;
|
|
import de.anomic.server.serverDate;
|
|
import de.anomic.server.serverFileUtils;
|
|
import de.anomic.server.serverSwitch;
|
|
import de.anomic.server.logging.serverLog;
|
|
import de.anomic.tools.nxTools;
|
|
|
|
public final class yacySeedDB {
|
|
|
|
// global statics
|
|
|
|
/**
|
|
* this is the lenght(12) of the hash key that is used:<br>
|
|
* - for seed hashes (this Object)<br>
|
|
* - for word hashes (plasmaIndexEntry.wordHashLength)<br>
|
|
* - for L-URL hashes (plasmaLURL.urlHashLength)<br><br>
|
|
* these hashes all shall be generated by base64.enhancedCoder
|
|
*/
|
|
public static final int commonHashLength = 12;
|
|
public static final int dhtActivityMagic = 32;
|
|
|
|
public static final String[] sortFields = new String[] {yacySeed.LCOUNT, yacySeed.ICOUNT, yacySeed.UPTIME, yacySeed.VERSION, yacySeed.LASTSEEN};
|
|
public static final String[] longaccFields = new String[] {yacySeed.LCOUNT, yacySeed.ICOUNT, yacySeed.ISPEED};
|
|
public static final String[] doubleaccFields = new String[] {yacySeed.RSPEED};
|
|
|
|
// class objects
|
|
protected File seedActiveDBFile, seedPassiveDBFile, seedPotentialDBFile;
|
|
|
|
protected kelondroMapObjects seedActiveDB, seedPassiveDB, seedPotentialDB;
|
|
private long preloadTime;
|
|
|
|
private final plasmaSwitchboard sb;
|
|
private yacySeed mySeed; // my own seed
|
|
|
|
private final Hashtable nameLookupCache;
|
|
private final Hashtable ipLookupCache;
|
|
|
|
public yacySeedDB(plasmaSwitchboard sb,
|
|
File seedActiveDBFile,
|
|
File seedPassiveDBFile,
|
|
File seedPotentialDBFile,
|
|
long preloadTime) {
|
|
|
|
this.seedActiveDBFile = seedActiveDBFile;
|
|
this.seedPassiveDBFile = seedPassiveDBFile;
|
|
this.seedPotentialDBFile = seedPotentialDBFile;
|
|
this.mySeed = null; // my own seed
|
|
this.sb = sb;
|
|
this.preloadTime = preloadTime;
|
|
|
|
// set up seed database
|
|
seedActiveDB = openSeedTable(seedActiveDBFile);
|
|
seedPassiveDB = openSeedTable(seedPassiveDBFile);
|
|
seedPotentialDB = openSeedTable(seedPotentialDBFile);
|
|
|
|
// start our virtual DNS service for yacy peers with empty cache
|
|
nameLookupCache = new Hashtable();
|
|
|
|
// cache for reverse name lookup
|
|
ipLookupCache = new Hashtable();
|
|
|
|
// check if we are in the seedCaches: this can happen if someone else published our seed
|
|
removeMySeed();
|
|
}
|
|
|
|
private synchronized void initMySeed() {
|
|
if (this.mySeed != null) return;
|
|
|
|
// create or init own seed
|
|
File myOwnSeedFile = sb.getOwnSeedFile();
|
|
if (myOwnSeedFile.length() > 0) try {
|
|
// load existing identity
|
|
mySeed = yacySeed.load(myOwnSeedFile);
|
|
} catch (IOException e) {
|
|
// create new identity
|
|
mySeed = yacySeed.genLocalSeed(sb);
|
|
try {
|
|
mySeed.save(myOwnSeedFile);
|
|
} catch (IOException ee) {
|
|
ee.printStackTrace();
|
|
System.exit(-1);
|
|
}
|
|
} else {
|
|
// create new identity
|
|
mySeed = yacySeed.genLocalSeed(sb);
|
|
try {
|
|
mySeed.save(myOwnSeedFile);
|
|
} catch (IOException ee) {
|
|
ee.printStackTrace();
|
|
System.exit(-1);
|
|
}
|
|
}
|
|
|
|
if (sb.getConfig("portForwardingEnabled","false").equalsIgnoreCase("true")) {
|
|
mySeed.put(yacySeed.PORT, sb.getConfig("portForwardingPort","8080"));
|
|
mySeed.put(yacySeed.IP, sb.getConfig("portForwardingHost","localhost"));
|
|
} else {
|
|
mySeed.put(yacySeed.IP, ""); // we delete the old information to see what we have now
|
|
mySeed.put(yacySeed.PORT, Integer.toString(serverCore.getPortNr(sb.getConfig("port", "8080")))); // set my seed's correct port number
|
|
}
|
|
mySeed.put(yacySeed.PEERTYPE, yacySeed.PEERTYPE_VIRGIN); // markup startup condition
|
|
|
|
}
|
|
|
|
public boolean mySeedIsDefined() {
|
|
return this.mySeed != null;
|
|
}
|
|
|
|
public yacySeed mySeed() {
|
|
if (this.mySeed == null) {
|
|
if (this.sizeConnected() == 0) try {Thread.sleep(5000);} catch (InterruptedException e) {} // wait for init
|
|
initMySeed();
|
|
}
|
|
return this.mySeed;
|
|
}
|
|
|
|
public synchronized void removeMySeed() {
|
|
if ((seedActiveDB.size() == 0) && (seedPassiveDB.size() == 0) && (seedPotentialDB.size() == 0)) return; // avoid that the own seed is initialized too early
|
|
if (this.mySeed == null) initMySeed();
|
|
try {
|
|
seedActiveDB.remove(mySeed.hash);
|
|
seedPassiveDB.remove(mySeed.hash);
|
|
seedPotentialDB.remove(mySeed.hash);
|
|
} catch (IOException e) {}
|
|
}
|
|
|
|
public boolean noDHTActivity() {
|
|
// for small networks, we don't perform DHT transmissions, because it is possible to search over all peers
|
|
return this.sizeConnected() <= dhtActivityMagic;
|
|
}
|
|
|
|
private synchronized kelondroMapObjects openSeedTable(File seedDBFile) {
|
|
final boolean usetree = false;
|
|
new File(seedDBFile.getParent()).mkdirs();
|
|
Class[] args;
|
|
try {
|
|
args = new Class[]{"".getClass(), Class.forName("java.util.Map")};
|
|
} catch (ClassNotFoundException e2){
|
|
e2.printStackTrace();
|
|
args = null;
|
|
}
|
|
Method initializeHandlerMethod;
|
|
try {
|
|
initializeHandlerMethod = this.getClass().getMethod("initializeHandler", args);
|
|
} catch (SecurityException e1) {
|
|
e1.printStackTrace();
|
|
initializeHandlerMethod = null;
|
|
} catch (NoSuchMethodException e1) {
|
|
e1.printStackTrace();
|
|
initializeHandlerMethod = null;
|
|
}
|
|
try {
|
|
return new kelondroMapObjects(new kelondroDyn(seedDBFile, true, true, preloadTime / 3, commonHashLength, 480, '#', kelondroBase64Order.enhancedCoder, usetree, false, true), 500, sortFields, longaccFields, doubleaccFields, initializeHandlerMethod, this);
|
|
} catch (Exception e) {
|
|
// try again
|
|
kelondroDyn.delete(seedDBFile, usetree);
|
|
return new kelondroMapObjects(new kelondroDyn(seedDBFile, true, true, preloadTime / 3, commonHashLength, 480, '#', kelondroBase64Order.enhancedCoder, usetree, false, true), 500, sortFields, longaccFields, doubleaccFields, initializeHandlerMethod, this);
|
|
}
|
|
}
|
|
|
|
protected synchronized kelondroMapObjects resetSeedTable(kelondroMapObjects seedDB, File seedDBFile) {
|
|
// this is an emergency function that should only be used if any problem with the
|
|
// seed.db is detected
|
|
yacyCore.log.logFine("seed-db " + seedDBFile.toString() + " reset (on-the-fly)");
|
|
seedDB.close();
|
|
seedDBFile.delete();
|
|
// create new seed database
|
|
seedDB = openSeedTable(seedDBFile);
|
|
return seedDB;
|
|
}
|
|
|
|
public synchronized void resetActiveTable() { seedActiveDB = resetSeedTable(seedActiveDB, seedActiveDBFile); }
|
|
public synchronized void resetPassiveTable() { seedPassiveDB = resetSeedTable(seedPassiveDB, seedPassiveDBFile); }
|
|
public synchronized void resetPotentialTable() { seedPotentialDB = resetSeedTable(seedPotentialDB, seedPotentialDBFile); }
|
|
|
|
public void close() {
|
|
if (seedActiveDB != null) seedActiveDB.close();
|
|
if (seedPassiveDB != null) seedPassiveDB.close();
|
|
if (seedPotentialDB != null) seedPotentialDB.close();
|
|
}
|
|
|
|
public void initializeHandler(String mapname, Map map) {
|
|
// this is used to set up a lastSeen lookup table
|
|
|
|
}
|
|
|
|
public Iterator seedsSortedConnected(boolean up, String field) {
|
|
// enumerates seed-type objects: all seeds sequentially ordered by field
|
|
return new seedEnum(up, field, seedActiveDB);
|
|
}
|
|
|
|
public Iterator seedsSortedDisconnected(boolean up, String field) {
|
|
// enumerates seed-type objects: all seeds sequentially ordered by field
|
|
return new seedEnum(up, field, seedPassiveDB);
|
|
}
|
|
|
|
public Iterator seedsSortedPotential(boolean up, String field) {
|
|
// enumerates seed-type objects: all seeds sequentially ordered by field
|
|
return new seedEnum(up, field, seedPotentialDB);
|
|
}
|
|
|
|
public TreeMap /* peer-b64-hashes/ipport */ clusterHashes(String clusterdefinition) {
|
|
// collects seeds according to cluster definition string, which consists of
|
|
// comma-separated .yacy or .yacyh-domains
|
|
// the domain may be extended by an alternative address specification of the form
|
|
// <ip> or <ip>:<port>. The port must be identical to the port specified in the peer seed,
|
|
// therefore it is optional. The address specification is separated by a '='; the complete
|
|
// address has therefore the form
|
|
// address ::= (<peername>'.yacy'|<peerhexhash>'.yacyh'){'='<ip>{':'<port}}
|
|
// clusterdef ::= {address}{','address}*
|
|
String[] addresses = (clusterdefinition.length() == 0) ? new String[0] : clusterdefinition.split(",");
|
|
TreeMap clustermap = new TreeMap(kelondroBase64Order.enhancedCoder);
|
|
yacySeed seed;
|
|
String hash, yacydom, ipport;
|
|
int p;
|
|
for (int i = 0; i < addresses.length; i++) {
|
|
p = addresses[i].indexOf('=');
|
|
if (p >= 0) {
|
|
yacydom = addresses[i].substring(0, p);
|
|
ipport = addresses[i].substring(p + 1);
|
|
} else {
|
|
yacydom = addresses[i];
|
|
ipport = null;
|
|
}
|
|
if (yacydom.endsWith(".yacyh")) {
|
|
// find a peer with its hexhash
|
|
hash = yacySeed.hexHash2b64Hash(yacydom.substring(0, yacydom.length() - 6));
|
|
seed = get(hash);
|
|
if (seed == null) {
|
|
yacyCore.log.logWarning("cluster peer '" + yacydom + "' was not found.");
|
|
} else {
|
|
clustermap.put(hash, ipport);
|
|
}
|
|
} else if (yacydom.endsWith(".yacy")) {
|
|
// find a peer with its name
|
|
seed = lookupByName(yacydom.substring(0, yacydom.length() - 5));
|
|
if (seed == null) {
|
|
yacyCore.log.logWarning("cluster peer '" + yacydom + "' was not found.");
|
|
} else {
|
|
clustermap.put(seed.hash, ipport);
|
|
}
|
|
} else {
|
|
yacyCore.log.logWarning("cluster peer '" + addresses[i] + "' has wrong syntax. the name must end with .yacy or .yacyh");
|
|
}
|
|
}
|
|
return clustermap;
|
|
}
|
|
|
|
public Iterator seedsConnected(boolean up, boolean rot, String firstHash, float minVersion) {
|
|
// enumerates seed-type objects: all seeds sequentially without order
|
|
return new seedEnum(up, rot, (firstHash == null) ? null : firstHash.getBytes(), null, seedActiveDB, minVersion);
|
|
}
|
|
|
|
public Iterator seedsDisconnected(boolean up, boolean rot, String firstHash, float minVersion) {
|
|
// enumerates seed-type objects: all seeds sequentially without order
|
|
return new seedEnum(up, rot, (firstHash == null) ? null : firstHash.getBytes(), null, seedPassiveDB, minVersion);
|
|
}
|
|
|
|
public Iterator seedsPotential(boolean up, boolean rot, String firstHash, float minVersion) {
|
|
// enumerates seed-type objects: all seeds sequentially without order
|
|
return new seedEnum(up, rot, (firstHash == null) ? null : firstHash.getBytes(), null, seedPotentialDB, minVersion);
|
|
}
|
|
|
|
public yacySeed anySeedVersion(float minVersion) {
|
|
// return just any seed that has a specific minimum version number
|
|
Iterator e = seedsConnected(true, true, yacySeed.randomHash(), minVersion);
|
|
return (yacySeed) e.next();
|
|
}
|
|
|
|
public HashMap seedsByAge(boolean up, int count) {
|
|
// returns a peerhash/yacySeed relation
|
|
// to get most recent peers, set up = true; for oldest peers, set up = false
|
|
|
|
if (count > sizeConnected()) count = sizeConnected();
|
|
|
|
// fill a score object
|
|
kelondroMScoreCluster seedScore = new kelondroMScoreCluster();
|
|
yacySeed ys;
|
|
long absage;
|
|
Iterator s = seedsConnected(true, false, null, (float) 0.0);
|
|
int searchcount = 1000;
|
|
if (searchcount > sizeConnected()) searchcount = sizeConnected();
|
|
try {
|
|
while ((s.hasNext()) && (searchcount-- > 0)) {
|
|
ys = (yacySeed) s.next();
|
|
if ((ys != null) && (ys.get(yacySeed.LASTSEEN, "").length() > 10)) try {
|
|
absage = Math.abs(System.currentTimeMillis() + serverDate.dayMillis - ys.getLastSeenUTC());
|
|
seedScore.addScore(ys.hash, (int) absage); // the higher absage, the older is the peer
|
|
} catch (Exception e) {}
|
|
}
|
|
|
|
// result is now in the score object; create a result vector
|
|
HashMap result = new HashMap();
|
|
Iterator it = seedScore.scores(up);
|
|
int c = 0;
|
|
while ((c < count) && (it.hasNext())) {
|
|
c++;
|
|
ys = getConnected((String) it.next());
|
|
if ((ys != null) && (ys.hash != null)) result.put(ys.hash, ys);
|
|
}
|
|
return result;
|
|
} catch (kelondroException e) {
|
|
seedActiveDB = resetSeedTable(seedActiveDB, seedActiveDBFile);
|
|
yacyCore.log.logFine("Internal Error at yacySeedDB.seedsByAge: " + e.getMessage(), e);
|
|
return null;
|
|
}
|
|
}
|
|
|
|
public int sizeConnected() {
|
|
return seedActiveDB.size();
|
|
/*
|
|
Enumeration e = seedsConnected(true, false, null);
|
|
int c = 0; while (e.hasMoreElements()) {c++; e.nextElement();}
|
|
return c;
|
|
*/
|
|
}
|
|
|
|
public int sizeDisconnected() {
|
|
return seedPassiveDB.size();
|
|
/*
|
|
Enumeration e = seedsDisconnected(true, false, null);
|
|
int c = 0; while (e.hasMoreElements()) {c++; e.nextElement();}
|
|
return c;
|
|
*/
|
|
}
|
|
|
|
public int sizePotential() {
|
|
return seedPotentialDB.size();
|
|
/*
|
|
Enumeration e = seedsPotential(true, false, null);
|
|
int c = 0; while (e.hasMoreElements()) {c++; e.nextElement();}
|
|
return c;
|
|
*/
|
|
}
|
|
|
|
public long countActiveURL() { return seedActiveDB.getLongAcc(yacySeed.LCOUNT); }
|
|
public long countActiveRWI() { return seedActiveDB.getLongAcc(yacySeed.ICOUNT); }
|
|
public long countActivePPM() { return seedActiveDB.getLongAcc(yacySeed.ISPEED); }
|
|
public double countActiveQPM() { return seedActiveDB.getDoubleAcc(yacySeed.RSPEED); }
|
|
public long countPassiveURL() { return seedPassiveDB.getLongAcc(yacySeed.LCOUNT); }
|
|
public long countPassiveRWI() { return seedPassiveDB.getLongAcc(yacySeed.ICOUNT); }
|
|
public long countPotentialURL() { return seedPotentialDB.getLongAcc(yacySeed.LCOUNT); }
|
|
public long countPotentialRWI() { return seedPotentialDB.getLongAcc(yacySeed.ICOUNT); }
|
|
|
|
public synchronized void addConnected(yacySeed seed) {
|
|
if ((seed == null) || (seed.isProper() != null)) return;
|
|
//seed.put(yacySeed.LASTSEEN, yacyCore.shortFormatter.format(new Date(yacyCore.universalTime())));
|
|
try {
|
|
nameLookupCache.put(seed.getName(), seed);
|
|
Map seedPropMap = seed.getMap();
|
|
synchronized(seedPropMap) {
|
|
seedActiveDB.set(seed.hash, seedPropMap);
|
|
}
|
|
seedPassiveDB.remove(seed.hash);
|
|
seedPotentialDB.remove(seed.hash);
|
|
} catch (IOException e){
|
|
yacyCore.log.logSevere("ERROR add: seed.db corrupt (" + e.getMessage() + "); resetting seed.db", e);
|
|
resetActiveTable();
|
|
} catch (kelondroException e){
|
|
yacyCore.log.logSevere("ERROR add: seed.db corrupt (" + e.getMessage() + "); resetting seed.db", e);
|
|
resetActiveTable();
|
|
} catch (IllegalArgumentException e) {
|
|
yacyCore.log.logSevere("ERROR add: seed.db corrupt (" + e.getMessage() + "); resetting seed.db", e);
|
|
resetActiveTable();
|
|
}
|
|
}
|
|
|
|
public synchronized void addDisconnected(yacySeed seed) {
|
|
if (seed == null) return;
|
|
try {
|
|
nameLookupCache.remove(seed.getName());
|
|
seedActiveDB.remove(seed.hash);
|
|
seedPotentialDB.remove(seed.hash);
|
|
} catch (Exception e) {}
|
|
//seed.put(yacySeed.LASTSEEN, yacyCore.shortFormatter.format(new Date(yacyCore.universalTime())));
|
|
try {
|
|
Map seedPropMap = seed.getMap();
|
|
synchronized(seedPropMap) {
|
|
seedPassiveDB.set(seed.hash, seedPropMap);
|
|
}
|
|
} catch (IOException e) {
|
|
yacyCore.log.logSevere("ERROR add: seed.db corrupt (" + e.getMessage() + "); resetting seed.db", e);
|
|
resetPassiveTable();
|
|
} catch (kelondroException e) {
|
|
yacyCore.log.logSevere("ERROR add: seed.db corrupt (" + e.getMessage() + "); resetting seed.db", e);
|
|
resetPassiveTable();
|
|
} catch (IllegalArgumentException e) {
|
|
yacyCore.log.logSevere("ERROR add: seed.db corrupt (" + e.getMessage() + "); resetting seed.db", e);
|
|
resetPassiveTable();
|
|
}
|
|
}
|
|
|
|
public synchronized void addPotential(yacySeed seed) {
|
|
if (seed == null) return;
|
|
try {
|
|
nameLookupCache.remove(seed.getName());
|
|
seedActiveDB.remove(seed.hash);
|
|
seedPassiveDB.remove(seed.hash);
|
|
} catch (Exception e) {}
|
|
if (seed.isProper() != null) return;
|
|
//seed.put(yacySeed.LASTSEEN, yacyCore.shortFormatter.format(new Date(yacyCore.universalTime())));
|
|
try {
|
|
Map seedPropMap = seed.getMap();
|
|
synchronized(seedPropMap) {
|
|
seedPotentialDB.set(seed.hash, seedPropMap);
|
|
}
|
|
} catch (IOException e) {
|
|
yacyCore.log.logSevere("ERROR add: seed.db corrupt (" + e.getMessage() + "); resetting seed.db", e);
|
|
resetPotentialTable();
|
|
} catch (kelondroException e) {
|
|
yacyCore.log.logSevere("ERROR add: seed.db corrupt (" + e.getMessage() + "); resetting seed.db", e);
|
|
resetPotentialTable();
|
|
} catch (IllegalArgumentException e) {
|
|
yacyCore.log.logSevere("ERROR add: seed.db corrupt (" + e.getMessage() + "); resetting seed.db", e);
|
|
resetPotentialTable();
|
|
}
|
|
}
|
|
|
|
public synchronized void removeDisconnected(String peerHash) {
|
|
if(peerHash == null) return;
|
|
try {
|
|
seedPassiveDB.remove(peerHash);
|
|
} catch (IOException e) { }
|
|
}
|
|
|
|
public synchronized void removePotential(String peerHash) {
|
|
if(peerHash == null) return;
|
|
try {
|
|
seedPotentialDB.remove(peerHash);
|
|
} catch (IOException e) { }
|
|
}
|
|
|
|
public boolean hasConnected(String hash) {
|
|
try {
|
|
return (seedActiveDB.get(hash) != null);
|
|
} catch (IOException e) {
|
|
return false;
|
|
}
|
|
}
|
|
|
|
public boolean hasDisconnected(String hash) {
|
|
try {
|
|
return (seedPassiveDB.get(hash) != null);
|
|
} catch (IOException e) {
|
|
return false;
|
|
}
|
|
}
|
|
|
|
public boolean hasPotential(String hash) {
|
|
try {
|
|
return (seedPotentialDB.get(hash) != null);
|
|
} catch (IOException e) {
|
|
return false;
|
|
}
|
|
}
|
|
|
|
private yacySeed get(String hash, kelondroMapObjects database) {
|
|
if (hash == null) return null;
|
|
if ((this.mySeed != null) && (hash.equals(mySeed.hash))) return mySeed;
|
|
Map entry = database.getMap(hash);
|
|
if (entry == null) return null;
|
|
return new yacySeed(hash, entry);
|
|
}
|
|
|
|
public yacySeed getConnected(String hash) {
|
|
return get(hash, seedActiveDB);
|
|
}
|
|
|
|
public yacySeed getDisconnected(String hash) {
|
|
return get(hash, seedPassiveDB);
|
|
}
|
|
|
|
public yacySeed getPotential(String hash) {
|
|
return get(hash, seedPotentialDB);
|
|
}
|
|
|
|
public yacySeed get(String hash) {
|
|
yacySeed seed = getConnected(hash);
|
|
if (seed == null) seed = getDisconnected(hash);
|
|
if (seed == null) seed = getPotential(hash);
|
|
return seed;
|
|
}
|
|
|
|
public void update(String hash, yacySeed seed) {
|
|
if (this.mySeed == null) initMySeed();
|
|
if (hash.equals(mySeed.hash)) {
|
|
mySeed = seed;
|
|
return;
|
|
}
|
|
|
|
yacySeed s = get(hash, seedActiveDB);
|
|
if (s != null) try { seedActiveDB.set(hash, seed.getMap()); return;} catch (IOException e) {}
|
|
|
|
s = get(hash, seedPassiveDB);
|
|
if (s != null) try { seedPassiveDB.set(hash, seed.getMap()); return;} catch (IOException e) {}
|
|
|
|
s = get(hash, seedPotentialDB);
|
|
if (s != null) try { seedPotentialDB.set(hash, seed.getMap()); return;} catch (IOException e) {}
|
|
}
|
|
|
|
public yacySeed lookupByName(String peerName) {
|
|
// reads a seed by searching by name
|
|
|
|
// local peer?
|
|
if (peerName.equals("localpeer")) {
|
|
if (this.mySeed == null) initMySeed();
|
|
return mySeed;
|
|
}
|
|
|
|
// then try to use the cache
|
|
yacySeed seed = (yacySeed) nameLookupCache.get(peerName);
|
|
if (seed != null) return seed;
|
|
|
|
// enumerate the cache and simultanous insert values
|
|
String name;
|
|
for (int table = 0; table < 2; table++) {
|
|
Iterator e = (table == 0) ? seedsConnected(true, false, null, (float) 0.0) : seedsDisconnected(true, false, null, (float) 0.0);
|
|
while (e.hasNext()) {
|
|
seed = (yacySeed) e.next();
|
|
if (seed != null) {
|
|
name = seed.getName().toLowerCase();
|
|
if (seed.isProper() == null) nameLookupCache.put(name, seed);
|
|
if (name.equals(peerName)) return seed;
|
|
}
|
|
}
|
|
}
|
|
// check local seed
|
|
if (this.mySeed == null) initMySeed();
|
|
name = mySeed.getName().toLowerCase();
|
|
if (mySeed.isProper() == null) nameLookupCache.put(name, mySeed);
|
|
if (name.equals(peerName)) return mySeed;
|
|
// nothing found
|
|
return null;
|
|
}
|
|
|
|
public yacySeed lookupByIP(
|
|
InetAddress peerIP,
|
|
boolean lookupConnected,
|
|
boolean lookupDisconnected,
|
|
boolean lookupPotential
|
|
) {
|
|
|
|
if (peerIP == null) return null;
|
|
yacySeed seed = null;
|
|
|
|
// local peer?
|
|
if (httpd.isThisHostIP(peerIP)) {
|
|
if (this.mySeed == null) initMySeed();
|
|
return mySeed;
|
|
}
|
|
|
|
// then try to use the cache
|
|
SoftReference ref = (SoftReference) ipLookupCache.get(peerIP);
|
|
if (ref != null) {
|
|
seed = (yacySeed) ref.get();
|
|
if (seed != null) return seed;
|
|
}
|
|
|
|
int pos = -1;
|
|
String addressStr = null;
|
|
InetAddress seedIPAddress = null;
|
|
|
|
if (lookupConnected) {
|
|
// enumerate the cache and simultanous insert values
|
|
Iterator e = seedsConnected(true, false, null, (float) 0.0);
|
|
|
|
while (e.hasNext()) {
|
|
try {
|
|
seed = (yacySeed) e.next();
|
|
if (seed != null) {
|
|
addressStr = seed.getPublicAddress();
|
|
if (addressStr == null) {
|
|
serverLog.logWarning("YACY","lookupByIP: address of seed " + seed.getName() + " is null.");
|
|
continue;
|
|
}
|
|
if ((pos = addressStr.indexOf(":"))!= -1) {
|
|
addressStr = addressStr.substring(0,pos);
|
|
}
|
|
seedIPAddress = InetAddress.getByName(addressStr);
|
|
if (seed.isProper() == null) ipLookupCache.put(seedIPAddress, new SoftReference(seed));
|
|
if (seedIPAddress.equals(peerIP)) return seed;
|
|
}
|
|
} catch (UnknownHostException ex) {}
|
|
}
|
|
}
|
|
|
|
if (lookupDisconnected) {
|
|
// enumerate the cache and simultanous insert values
|
|
Iterator e = seedsDisconnected(true, false, null, (float) 0.0);
|
|
|
|
while (e.hasNext()) {
|
|
try {
|
|
seed = (yacySeed) e.next();
|
|
if (seed != null) {
|
|
addressStr = seed.getPublicAddress();
|
|
if (addressStr == null) {
|
|
serverLog.logWarning("YACY","lookupByIP: address of seed " + seed.getName() + " is null.");
|
|
continue;
|
|
}
|
|
if ((pos = addressStr.indexOf(":"))!= -1) {
|
|
addressStr = addressStr.substring(0,pos);
|
|
}
|
|
seedIPAddress = InetAddress.getByName(addressStr);
|
|
if (seed.isProper() == null) ipLookupCache.put(seedIPAddress, new SoftReference(seed));
|
|
if (seedIPAddress.equals(peerIP)) return seed;
|
|
}
|
|
} catch (UnknownHostException ex) {}
|
|
}
|
|
}
|
|
|
|
if (lookupPotential) {
|
|
// enumerate the cache and simultanous insert values
|
|
Iterator e = seedsPotential(true, false, null, (float) 0.0);
|
|
|
|
while (e.hasNext()) {
|
|
try {
|
|
seed = (yacySeed) e.next();
|
|
if ((seed != null) && ((addressStr = seed.getPublicAddress()) != null)) {
|
|
if ((pos = addressStr.indexOf(":"))!= -1) {
|
|
addressStr = addressStr.substring(0,pos);
|
|
}
|
|
seedIPAddress = InetAddress.getByName(addressStr);
|
|
if (seed.isProper() == null) ipLookupCache.put(seedIPAddress, new SoftReference(seed));
|
|
if (seedIPAddress.equals(peerIP)) return seed;
|
|
}
|
|
} catch (UnknownHostException ex) {}
|
|
}
|
|
}
|
|
|
|
try {
|
|
// check local seed
|
|
if (this.mySeed == null) return null;
|
|
addressStr = mySeed.getPublicAddress();
|
|
if (addressStr == null) return null;
|
|
if ((pos = addressStr.indexOf(":"))!= -1) {
|
|
addressStr = addressStr.substring(0,pos);
|
|
}
|
|
seedIPAddress = InetAddress.getByName(addressStr);
|
|
if (mySeed.isProper() == null) ipLookupCache.put(seedIPAddress, new SoftReference(mySeed));
|
|
if (seedIPAddress.equals(peerIP)) return mySeed;
|
|
// nothing found
|
|
return null;
|
|
} catch (UnknownHostException e2) {
|
|
return null;
|
|
}
|
|
}
|
|
|
|
public ArrayList storeCache(File seedFile) throws IOException {
|
|
return storeCache(seedFile, false);
|
|
}
|
|
|
|
private ArrayList storeCache(File seedFile, boolean addMySeed) throws IOException {
|
|
PrintWriter pw = null;
|
|
ArrayList v = new ArrayList(seedActiveDB.size()+1);
|
|
try {
|
|
|
|
pw = new PrintWriter(new BufferedWriter(new FileWriter(seedFile)));
|
|
|
|
// store own seed
|
|
String line;
|
|
if (this.mySeed == null) initMySeed();
|
|
if (addMySeed) {
|
|
line = mySeed.genSeedStr(null);
|
|
v.add(line);
|
|
pw.print(line + serverCore.crlfString);
|
|
}
|
|
|
|
// store other seeds
|
|
yacySeed ys;
|
|
Iterator se = seedsConnected(true, false, null, (float) 0.0);
|
|
while (se.hasNext()) {
|
|
ys = (yacySeed) se.next();
|
|
if (ys != null) {
|
|
line = ys.genSeedStr(null);
|
|
v.add(line);
|
|
pw.print(line + serverCore.crlfString);
|
|
}
|
|
}
|
|
pw.flush();
|
|
} finally {
|
|
if (pw != null) try { pw.close(); } catch (Exception e) {}
|
|
}
|
|
return v;
|
|
}
|
|
|
|
public String uploadCache(yacySeedUploader uploader,
|
|
serverSwitch sb,
|
|
yacySeedDB seedDB,
|
|
// String seedFTPServer,
|
|
// String seedFTPAccount,
|
|
// String seedFTPPassword,
|
|
// File seedFTPPath,
|
|
yacyURL seedURL) throws Exception {
|
|
|
|
// upload a seed file, if possible
|
|
if (seedURL == null) throw new NullPointerException("UPLOAD - Error: URL not given");
|
|
|
|
String log = null;
|
|
File seedFile = null;
|
|
try {
|
|
// create a seed file which for uploading ...
|
|
seedFile = File.createTempFile("seedFile",".txt", plasmaHTCache.cachePath);
|
|
seedFile.deleteOnExit();
|
|
serverLog.logFine("YACY","SaveSeedList: Storing seedlist into tempfile " + seedFile.toString());
|
|
ArrayList uv = storeCache(seedFile, true);
|
|
|
|
// uploading the seed file
|
|
serverLog.logFine("YACY","SaveSeedList: Trying to upload seed-file, " + seedFile.length() + " bytes, " + uv.size() + " entries.");
|
|
log = uploader.uploadSeedFile(sb,seedDB,seedFile);
|
|
|
|
// test download
|
|
serverLog.logFine("YACY","SaveSeedList: Trying to download seed-file '" + seedURL + "'.");
|
|
ArrayList check = downloadSeedFile(seedURL);
|
|
|
|
// Comparing if local copy and uploaded copy are equal
|
|
String errorMsg = checkCache(uv, check);
|
|
if (errorMsg == null)
|
|
log = log + "UPLOAD CHECK - Success: the result vectors are equal" + serverCore.crlfString;
|
|
else {
|
|
throw new Exception("UPLOAD CHECK - Error: the result vector is different. " + errorMsg + serverCore.crlfString);
|
|
}
|
|
} finally {
|
|
if (seedFile != null) try { seedFile.delete(); } catch (Exception e) {/* ignore this */}
|
|
}
|
|
|
|
return log;
|
|
}
|
|
|
|
private ArrayList downloadSeedFile(yacyURL seedURL) throws IOException {
|
|
httpc remote = null;
|
|
try {
|
|
// init httpc
|
|
remote = new httpc(
|
|
seedURL.getHost(),
|
|
seedURL.getHost(),
|
|
seedURL.getPort(),
|
|
10000,
|
|
seedURL.getProtocol().equalsIgnoreCase("https"),
|
|
sb.remoteProxyConfig,
|
|
null, null);
|
|
|
|
// Configure http headers
|
|
httpHeader reqHeader = new httpHeader();
|
|
reqHeader.put(httpHeader.PRAGMA, "no-cache");
|
|
reqHeader.put(httpHeader.CACHE_CONTROL, "no-cache"); // httpc uses HTTP/1.0 is this necessary?
|
|
|
|
// send request
|
|
httpc.response res = remote.GET(seedURL.getFile(), reqHeader);
|
|
|
|
// check response code
|
|
if (res.statusCode != 200) {
|
|
throw new IOException("Server returned status: " + res.status);
|
|
}
|
|
|
|
// read byte array
|
|
byte[] content = serverFileUtils.read(res.getContentInputStream());
|
|
remote.close();
|
|
|
|
// uncompress it if it is gzipped
|
|
content = serverFileUtils.uncompressGZipArray(content);
|
|
|
|
// convert it into an array
|
|
return nxTools.strings(content,"UTF-8");
|
|
} catch (Exception e) {
|
|
throw new IOException("Unable to download seed file '" + seedURL + "'. " + e.getMessage());
|
|
}
|
|
}
|
|
|
|
private String checkCache(ArrayList uv, ArrayList check) {
|
|
if ((check == null) || (uv == null) || (uv.size() != check.size())) {
|
|
serverLog.logFine("YACY","SaveSeedList: Local and uploades seed-list " +
|
|
"contains varying numbers of entries." +
|
|
"\n\tLocal seed-list: " + ((uv == null) ? "null" : Integer.toString(uv.size())) + " entries" +
|
|
"\n\tRemote seed-list: " + ((check == null) ? "null" : Integer.toString(check.size())) + " enties");
|
|
return "Entry count is different: uv.size() = " + ((uv == null) ? "null" : Integer.toString(uv.size())) + ", check = " + ((check == null) ? "null" : Integer.toString(check.size()));
|
|
}
|
|
|
|
serverLog.logFine("YACY","SaveSeedList: Comparing local and uploades seed-list entries ...");
|
|
int i;
|
|
for (i = 0; i < uv.size(); i++) {
|
|
if (!(((String) uv.get(i)).equals((String) check.get(i)))) return "Element at position " + i + " is different.";
|
|
}
|
|
|
|
// no difference found
|
|
return null;
|
|
}
|
|
|
|
public String resolveYacyAddress(String host) {
|
|
yacySeed seed;
|
|
int p;
|
|
String subdom = null;
|
|
if (host.endsWith(".yacyh")) {
|
|
// this is not functional at the moment
|
|
// caused by lowecasing of hashes at the browser client
|
|
p = host.indexOf(".");
|
|
if ((p > 0) && (p != (host.length() - 6))) {
|
|
subdom = host.substring(0, p);
|
|
host = host.substring(p + 1);
|
|
}
|
|
// check if we have a b64-hash or a hex-hash
|
|
String hash = host.substring(0, host.length() - 6);
|
|
if (hash.length() > commonHashLength) {
|
|
// this is probably a hex-hash
|
|
hash = yacySeed.hexHash2b64Hash(hash);
|
|
}
|
|
// check remote seeds
|
|
seed = getConnected(hash); // checks only remote, not local
|
|
// check local seed
|
|
if (seed == null) {
|
|
if (this.mySeed == null) initMySeed();
|
|
if (hash.equals(mySeed.hash))
|
|
seed = mySeed;
|
|
else return null;
|
|
}
|
|
return seed.getPublicAddress() + ((subdom == null) ? "" : ("/" + subdom));
|
|
} else if (host.endsWith(".yacy")) {
|
|
// identify subdomain
|
|
p = host.indexOf(".");
|
|
if ((p > 0) && (p != (host.length() - 5))) {
|
|
subdom = host.substring(0, p); // no double-dot attack possible, the subdom cannot have ".." in it
|
|
host = host.substring(p + 1); // if ever, the double-dots are here but do not harm
|
|
}
|
|
// identify domain
|
|
String domain = host.substring(0, host.length() - 5).toLowerCase();
|
|
seed = lookupByName(domain);
|
|
if (seed == null) return null;
|
|
if (this.mySeed == null) initMySeed();
|
|
if ((seed == mySeed) && (!(seed.isOnline()))) {
|
|
// take local ip instead of external
|
|
return serverDomains.myPublicIP() + ":" + serverCore.getPortNr(sb.getConfig("port", "8080")) + ((subdom == null) ? "" : ("/" + subdom));
|
|
}
|
|
return seed.getPublicAddress() + ((subdom == null) ? "" : ("/" + subdom));
|
|
} else {
|
|
return null;
|
|
}
|
|
}
|
|
|
|
class seedEnum implements Iterator {
|
|
|
|
kelondroMapObjects.mapIterator it;
|
|
yacySeed nextSeed;
|
|
kelondroMapObjects database;
|
|
float minVersion;
|
|
|
|
public seedEnum(boolean up, boolean rot, byte[] firstKey, byte[] secondKey, kelondroMapObjects database, float minVersion) {
|
|
this.database = database;
|
|
this.minVersion = minVersion;
|
|
try {
|
|
it = (firstKey == null) ? database.maps(up, rot) : database.maps(up, rot, firstKey, secondKey);
|
|
while (true) {
|
|
nextSeed = internalNext();
|
|
if (nextSeed == null) break;
|
|
if (nextSeed.getVersion() >= this.minVersion) break;
|
|
}
|
|
} catch (IOException e) {
|
|
e.printStackTrace();
|
|
yacyCore.log.logSevere("ERROR seedLinEnum: seed.db corrupt (" + e.getMessage() + "); resetting seed.db", e);
|
|
if (database == seedActiveDB) seedActiveDB = resetSeedTable(seedActiveDB, seedActiveDBFile);
|
|
if (database == seedPassiveDB) seedPassiveDB = resetSeedTable(seedPassiveDB, seedPassiveDBFile);
|
|
it = null;
|
|
} catch (kelondroException e) {
|
|
e.printStackTrace();
|
|
yacyCore.log.logSevere("ERROR seedLinEnum: seed.db corrupt (" + e.getMessage() + "); resetting seed.db", e);
|
|
if (database == seedActiveDB) seedActiveDB = resetSeedTable(seedActiveDB, seedActiveDBFile);
|
|
if (database == seedPassiveDB) seedPassiveDB = resetSeedTable(seedPassiveDB, seedPassiveDBFile);
|
|
it = null;
|
|
}
|
|
}
|
|
|
|
public seedEnum(boolean up, String field, kelondroMapObjects database) {
|
|
this.database = database;
|
|
try {
|
|
it = database.maps(up, field);
|
|
nextSeed = internalNext();
|
|
} catch (kelondroException e) {
|
|
e.printStackTrace();
|
|
yacyCore.log.logSevere("ERROR seedLinEnum: seed.db corrupt (" + e.getMessage() + "); resetting seed.db", e);
|
|
if (database == seedActiveDB) seedActiveDB = resetSeedTable(seedActiveDB, seedActiveDBFile);
|
|
if (database == seedPassiveDB) seedPassiveDB = resetSeedTable(seedPassiveDB, seedPassiveDBFile);
|
|
if (database == seedPotentialDB) seedPotentialDB = resetSeedTable(seedPotentialDB, seedPotentialDBFile);
|
|
it = null;
|
|
}
|
|
}
|
|
|
|
public boolean hasNext() {
|
|
return (nextSeed != null);
|
|
}
|
|
|
|
public yacySeed internalNext() {
|
|
if ((it == null) || (!(it.hasNext()))) return null;
|
|
Map dna = (Map) it.next();
|
|
if (dna == null) return null;
|
|
String hash = (String) dna.remove("key");
|
|
//while (hash.length() < commonHashLength) { hash = hash + "_"; }
|
|
return new yacySeed(hash, dna);
|
|
}
|
|
|
|
public Object next() {
|
|
yacySeed seed = nextSeed;
|
|
try {while (true) {
|
|
nextSeed = internalNext();
|
|
if (nextSeed == null) break;
|
|
if (nextSeed.getVersion() >= this.minVersion) break;
|
|
}} catch (kelondroException e) {
|
|
e.printStackTrace();
|
|
// eergency reset
|
|
yacyCore.log.logSevere("seed-db emergency reset", e);
|
|
try {
|
|
database.reset();
|
|
nextSeed = null;
|
|
return null;
|
|
} catch (IOException e1) {
|
|
// no recovery possible
|
|
e1.printStackTrace();
|
|
System.exit(-1);
|
|
}
|
|
}
|
|
return seed;
|
|
}
|
|
|
|
public void remove() {
|
|
throw new UnsupportedOperationException();
|
|
}
|
|
|
|
}
|
|
|
|
}
|