mirror of
https://github.com/yacy/yacy_search_server.git
synced 2024-09-21 00:00:13 +02:00
d2ba1fd2ab
This change is inspired by the need to see a network connected to the index it creates in a indexing team. It is not possible to divide the network and the index. Therefore all control files for the network was moved to the network within the INDEX/<network-name> subfolder. The remaining YACYDB is superfluous and can be deleted. The yacyDB and yacyNews data structures are now part of plasmaWordIndex. Therefore all methods, using static access to yacySeedDB had to be rewritten. A special problem had been all the port forwarding methods which had been tightly mixed with seed construction. It was not possible to move the port forwarding functions to the place, meaning and usage of plasmaWordIndex. Therefore the port forwarding had been deleted (I guess nobody used it and it can be simulated by methods outside of YaCy). The mySeed.txt is automatically moved to the current network position. A new effect causes that every network will create a different local seed file, which is ok, since the seed identifies the peer only against the network (it is the purpose of the seed hash to give a peer a location within the DHT). No other functional change has been made. The next steps to enable network switcing are: - shift of crawler tables from PLASMADB into the network (crawls are also network-specific) - possibly shift of plasmaWordIndex code into yacy package (index management is network-specific) - servlet to switch networks git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@4765 6c8d7289-2bf4-0310-a012-ef5d649a1542
1041 lines
44 KiB
Java
1041 lines
44 KiB
Java
// yacySeedDB.java
|
|
// -------------------------------------
|
|
// (C) by Michael Peter Christen; mc@anomic.de
|
|
// first published on http://www.anomic.de
|
|
// Frankfurt, Germany, 2004, 2005
|
|
//
|
|
// $LastChangedDate$
|
|
// $LastChangedRevision$
|
|
// $LastChangedBy$
|
|
//
|
|
// This program is free software; you can redistribute it and/or modify
|
|
// it under the terms of the GNU General Public License as published by
|
|
// the Free Software Foundation; either version 2 of the License, or
|
|
// (at your option) any later version.
|
|
//
|
|
// This program is distributed in the hope that it will be useful,
|
|
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
// GNU General Public License for more details.
|
|
//
|
|
// You should have received a copy of the GNU General Public License
|
|
// along with this program; if not, write to the Free Software
|
|
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
|
//
|
|
// Using this software in any meaning (reading, learning, copying, compiling,
|
|
// running) means that you agree that the Author(s) is (are) not responsible
|
|
// for cost, loss of data or any harm that may be caused directly or indirectly
|
|
// by usage of this softare or this documentation. The usage of this software
|
|
// is on your own risk. The installation and usage (starting/running) of this
|
|
// software may allow other people or application to access your computer and
|
|
// any attached devices and is highly dependent on the configuration of the
|
|
// software which must be done by the user of the software; the author(s) is
|
|
// (are) also not responsible for proper configuration and usage of the
|
|
// software, even if provoked by documentation provided together with
|
|
// the software.
|
|
//
|
|
// Any changes to this file according to the GPL as documented in the file
|
|
// gpl.txt aside this file in the shipment you received can be done to the
|
|
// lines that follows this copyright notice here, but changes must not be
|
|
// done inside the copyright notive above. A re-distribution must contain
|
|
// the intact and unchanged copyright notice.
|
|
// Contributions and changes to the program code must be marked as such.
|
|
|
|
package de.anomic.yacy;
|
|
|
|
import java.io.BufferedWriter;
|
|
import java.io.File;
|
|
import java.io.FileWriter;
|
|
import java.io.IOException;
|
|
import java.io.PrintWriter;
|
|
import java.lang.ref.SoftReference;
|
|
import java.lang.reflect.Method;
|
|
import java.net.InetAddress;
|
|
import java.net.UnknownHostException;
|
|
import java.util.ArrayList;
|
|
import java.util.HashMap;
|
|
import java.util.HashSet;
|
|
import java.util.Hashtable;
|
|
import java.util.Iterator;
|
|
import java.util.Map;
|
|
import java.util.TreeMap;
|
|
|
|
import de.anomic.http.JakartaCommonsHttpClient;
|
|
import de.anomic.http.JakartaCommonsHttpResponse;
|
|
import de.anomic.http.httpHeader;
|
|
import de.anomic.http.httpd;
|
|
import de.anomic.http.httpdAlternativeDomainNames;
|
|
import de.anomic.kelondro.kelondroBase64Order;
|
|
import de.anomic.kelondro.kelondroDyn;
|
|
import de.anomic.kelondro.kelondroException;
|
|
import de.anomic.kelondro.kelondroMScoreCluster;
|
|
import de.anomic.kelondro.kelondroMapObjects;
|
|
import de.anomic.plasma.plasmaHTCache;
|
|
import de.anomic.server.serverCore;
|
|
import de.anomic.server.serverDate;
|
|
import de.anomic.server.serverDomains;
|
|
import de.anomic.server.serverFileUtils;
|
|
import de.anomic.server.serverSwitch;
|
|
import de.anomic.server.logging.serverLog;
|
|
import de.anomic.tools.nxTools;
|
|
|
|
public final class yacySeedDB implements httpdAlternativeDomainNames {
|
|
|
|
// global statics
|
|
|
|
/**
|
|
* this is the lenght(12) of the hash key that is used:<br>
|
|
* - for seed hashes (this Object)<br>
|
|
* - for word hashes (plasmaIndexEntry.wordHashLength)<br>
|
|
* - for L-URL hashes (plasmaLURL.urlHashLength)<br><br>
|
|
* these hashes all shall be generated by base64.enhancedCoder
|
|
*/
|
|
public static final int commonHashLength = 12;
|
|
public static final int dhtActivityMagic = 32;
|
|
|
|
public static final String[] sortFields = new String[] {yacySeed.LCOUNT, yacySeed.ICOUNT, yacySeed.UPTIME, yacySeed.VERSION, yacySeed.LASTSEEN};
|
|
public static final String[] longaccFields = new String[] {yacySeed.LCOUNT, yacySeed.ICOUNT, yacySeed.ISPEED};
|
|
public static final String[] doubleaccFields = new String[] {yacySeed.RSPEED};
|
|
|
|
// class objects
|
|
protected File seedActiveDBFile, seedPassiveDBFile, seedPotentialDBFile;
|
|
protected File myOwnSeedFile;
|
|
protected kelondroMapObjects seedActiveDB, seedPassiveDB, seedPotentialDB;
|
|
|
|
public int lastSeedUpload_seedDBSize = 0;
|
|
public long lastSeedUpload_timeStamp = System.currentTimeMillis();
|
|
public String lastSeedUpload_myIP = "";
|
|
|
|
private yacySeed mySeed; // my own seed
|
|
|
|
private final Hashtable<String, yacySeed> nameLookupCache;
|
|
private final Hashtable<InetAddress, SoftReference<yacySeed>> ipLookupCache;
|
|
|
|
public yacySeedDB(
|
|
File seedActiveDBFile,
|
|
File seedPassiveDBFile,
|
|
File seedPotentialDBFile,
|
|
File myOwnSeedFile) {
|
|
this.seedActiveDBFile = seedActiveDBFile;
|
|
this.seedPassiveDBFile = seedPassiveDBFile;
|
|
this.seedPotentialDBFile = seedPotentialDBFile;
|
|
this.mySeed = null; // my own seed
|
|
this.myOwnSeedFile = myOwnSeedFile;
|
|
|
|
// set up seed database
|
|
seedActiveDB = openSeedTable(seedActiveDBFile);
|
|
seedPassiveDB = openSeedTable(seedPassiveDBFile);
|
|
seedPotentialDB = openSeedTable(seedPotentialDBFile);
|
|
|
|
// start our virtual DNS service for yacy peers with empty cache
|
|
nameLookupCache = new Hashtable<String, yacySeed>();
|
|
|
|
// cache for reverse name lookup
|
|
ipLookupCache = new Hashtable<InetAddress, SoftReference<yacySeed>>();
|
|
|
|
// check if we are in the seedCaches: this can happen if someone else published our seed
|
|
removeMySeed();
|
|
|
|
lastSeedUpload_seedDBSize = sizeConnected();
|
|
|
|
// tell the httpdProxy how to find this table as address resolver
|
|
httpd.alternativeResolver = this;
|
|
}
|
|
|
|
private synchronized void initMySeed() {
|
|
if (this.mySeed != null) return;
|
|
|
|
// create or init own seed
|
|
if (myOwnSeedFile.length() > 0) try {
|
|
// load existing identity
|
|
mySeed = yacySeed.load(myOwnSeedFile);
|
|
} catch (IOException e) {
|
|
// create new identity
|
|
mySeed = yacySeed.genLocalSeed(this);
|
|
try {
|
|
mySeed.save(myOwnSeedFile);
|
|
} catch (IOException ee) {
|
|
ee.printStackTrace();
|
|
System.exit(-1);
|
|
}
|
|
} else {
|
|
// create new identity
|
|
mySeed = yacySeed.genLocalSeed(this);
|
|
try {
|
|
mySeed.save(myOwnSeedFile);
|
|
} catch (IOException ee) {
|
|
ee.printStackTrace();
|
|
System.exit(-1);
|
|
}
|
|
}
|
|
|
|
mySeed.put(yacySeed.IP, ""); // we delete the old information to see what we have now
|
|
mySeed.put(yacySeed.PEERTYPE, yacySeed.PEERTYPE_VIRGIN); // markup startup condition
|
|
}
|
|
|
|
public boolean mySeedIsDefined() {
|
|
return this.mySeed != null;
|
|
}
|
|
|
|
public yacySeed mySeed() {
|
|
if (this.mySeed == null) {
|
|
if (this.sizeConnected() == 0) try {Thread.sleep(5000);} catch (InterruptedException e) {} // wait for init
|
|
initMySeed();
|
|
}
|
|
return this.mySeed;
|
|
}
|
|
|
|
public String myAlternativeAddress() {
|
|
return mySeed().getName() + ".yacy";
|
|
}
|
|
|
|
public String myIP() {
|
|
return mySeed().getIP();
|
|
}
|
|
|
|
public int myPort() {
|
|
return mySeed().getPort();
|
|
}
|
|
|
|
public String myName() {
|
|
return mySeed.getName();
|
|
}
|
|
|
|
public String myID() {
|
|
return mySeed.hash;
|
|
}
|
|
|
|
public synchronized void removeMySeed() {
|
|
if ((seedActiveDB.size() == 0) && (seedPassiveDB.size() == 0) && (seedPotentialDB.size() == 0)) return; // avoid that the own seed is initialized too early
|
|
if (this.mySeed == null) initMySeed();
|
|
try {
|
|
seedActiveDB.remove(mySeed.hash);
|
|
seedPassiveDB.remove(mySeed.hash);
|
|
seedPotentialDB.remove(mySeed.hash);
|
|
} catch (IOException e) {}
|
|
}
|
|
|
|
public boolean noDHTActivity() {
|
|
// for small networks, we don't perform DHT transmissions, because it is possible to search over all peers
|
|
return this.sizeConnected() <= dhtActivityMagic;
|
|
}
|
|
|
|
@SuppressWarnings("unchecked")
|
|
private synchronized kelondroMapObjects openSeedTable(File seedDBFile) {
|
|
final boolean usetree = false;
|
|
new File(seedDBFile.getParent()).mkdirs();
|
|
Class[] args;
|
|
try {
|
|
args = new Class[]{"".getClass(), Class.forName("java.util.Map")};
|
|
} catch (ClassNotFoundException e2){
|
|
e2.printStackTrace();
|
|
args = null;
|
|
}
|
|
Method initializeHandlerMethod;
|
|
try {
|
|
initializeHandlerMethod = this.getClass().getMethod("initializeHandler", args);
|
|
} catch (SecurityException e1) {
|
|
e1.printStackTrace();
|
|
initializeHandlerMethod = null;
|
|
} catch (NoSuchMethodException e1) {
|
|
e1.printStackTrace();
|
|
initializeHandlerMethod = null;
|
|
}
|
|
try {
|
|
return new kelondroMapObjects(new kelondroDyn(seedDBFile, true, true, commonHashLength, 480, '#', kelondroBase64Order.enhancedCoder, usetree, false, true), 500, sortFields, longaccFields, doubleaccFields, initializeHandlerMethod, this);
|
|
} catch (Exception e) {
|
|
// try again
|
|
kelondroDyn.delete(seedDBFile);
|
|
return new kelondroMapObjects(new kelondroDyn(seedDBFile, true, true, commonHashLength, 480, '#', kelondroBase64Order.enhancedCoder, usetree, false, true), 500, sortFields, longaccFields, doubleaccFields, initializeHandlerMethod, this);
|
|
}
|
|
}
|
|
|
|
protected synchronized kelondroMapObjects resetSeedTable(kelondroMapObjects seedDB, File seedDBFile) {
|
|
// this is an emergency function that should only be used if any problem with the
|
|
// seed.db is detected
|
|
yacyCore.log.logFine("seed-db " + seedDBFile.toString() + " reset (on-the-fly)");
|
|
seedDB.close();
|
|
seedDBFile.delete();
|
|
// create new seed database
|
|
seedDB = openSeedTable(seedDBFile);
|
|
return seedDB;
|
|
}
|
|
|
|
public synchronized void resetActiveTable() { seedActiveDB = resetSeedTable(seedActiveDB, seedActiveDBFile); }
|
|
public synchronized void resetPassiveTable() { seedPassiveDB = resetSeedTable(seedPassiveDB, seedPassiveDBFile); }
|
|
public synchronized void resetPotentialTable() { seedPotentialDB = resetSeedTable(seedPotentialDB, seedPotentialDBFile); }
|
|
|
|
public void close() {
|
|
if (seedActiveDB != null) seedActiveDB.close();
|
|
if (seedPassiveDB != null) seedPassiveDB.close();
|
|
if (seedPotentialDB != null) seedPotentialDB.close();
|
|
}
|
|
|
|
@SuppressWarnings("unchecked")
|
|
public void initializeHandler(String mapname, Map map) {
|
|
// this is used to set up a lastSeen lookup table
|
|
|
|
}
|
|
|
|
public Iterator<yacySeed> seedsSortedConnected(boolean up, String field) {
|
|
// enumerates seed-type objects: all seeds sequentially ordered by field
|
|
return new seedEnum(up, field, seedActiveDB);
|
|
}
|
|
|
|
public Iterator<yacySeed> seedsSortedDisconnected(boolean up, String field) {
|
|
// enumerates seed-type objects: all seeds sequentially ordered by field
|
|
return new seedEnum(up, field, seedPassiveDB);
|
|
}
|
|
|
|
public Iterator<yacySeed> seedsSortedPotential(boolean up, String field) {
|
|
// enumerates seed-type objects: all seeds sequentially ordered by field
|
|
return new seedEnum(up, field, seedPotentialDB);
|
|
}
|
|
|
|
public TreeMap<String, String> /* peer-b64-hashes/ipport */ clusterHashes(String clusterdefinition) {
|
|
// collects seeds according to cluster definition string, which consists of
|
|
// comma-separated .yacy or .yacyh-domains
|
|
// the domain may be extended by an alternative address specification of the form
|
|
// <ip> or <ip>:<port>. The port must be identical to the port specified in the peer seed,
|
|
// therefore it is optional. The address specification is separated by a '='; the complete
|
|
// address has therefore the form
|
|
// address ::= (<peername>'.yacy'|<peerhexhash>'.yacyh'){'='<ip>{':'<port}}
|
|
// clusterdef ::= {address}{','address}*
|
|
String[] addresses = (clusterdefinition.length() == 0) ? new String[0] : clusterdefinition.split(",");
|
|
TreeMap<String, String> clustermap = new TreeMap<String, String>(kelondroBase64Order.enhancedComparator);
|
|
yacySeed seed;
|
|
String hash, yacydom, ipport;
|
|
int p;
|
|
for (int i = 0; i < addresses.length; i++) {
|
|
p = addresses[i].indexOf('=');
|
|
if (p >= 0) {
|
|
yacydom = addresses[i].substring(0, p);
|
|
ipport = addresses[i].substring(p + 1);
|
|
} else {
|
|
yacydom = addresses[i];
|
|
ipport = null;
|
|
}
|
|
if (yacydom.endsWith(".yacyh")) {
|
|
// find a peer with its hexhash
|
|
hash = yacySeed.hexHash2b64Hash(yacydom.substring(0, yacydom.length() - 6));
|
|
seed = get(hash);
|
|
if (seed == null) {
|
|
yacyCore.log.logWarning("cluster peer '" + yacydom + "' was not found.");
|
|
} else {
|
|
clustermap.put(hash, ipport);
|
|
}
|
|
} else if (yacydom.endsWith(".yacy")) {
|
|
// find a peer with its name
|
|
seed = lookupByName(yacydom.substring(0, yacydom.length() - 5));
|
|
if (seed == null) {
|
|
yacyCore.log.logWarning("cluster peer '" + yacydom + "' was not found.");
|
|
} else {
|
|
clustermap.put(seed.hash, ipport);
|
|
}
|
|
} else {
|
|
yacyCore.log.logWarning("cluster peer '" + addresses[i] + "' has wrong syntax. the name must end with .yacy or .yacyh");
|
|
}
|
|
}
|
|
return clustermap;
|
|
}
|
|
|
|
public Iterator<yacySeed> seedsConnected(boolean up, boolean rot, String firstHash, float minVersion) {
|
|
// enumerates seed-type objects: all seeds sequentially without order
|
|
return new seedEnum(up, rot, (firstHash == null) ? null : firstHash.getBytes(), null, seedActiveDB, minVersion);
|
|
}
|
|
|
|
public Iterator<yacySeed> seedsDisconnected(boolean up, boolean rot, String firstHash, float minVersion) {
|
|
// enumerates seed-type objects: all seeds sequentially without order
|
|
return new seedEnum(up, rot, (firstHash == null) ? null : firstHash.getBytes(), null, seedPassiveDB, minVersion);
|
|
}
|
|
|
|
public Iterator<yacySeed> seedsPotential(boolean up, boolean rot, String firstHash, float minVersion) {
|
|
// enumerates seed-type objects: all seeds sequentially without order
|
|
return new seedEnum(up, rot, (firstHash == null) ? null : firstHash.getBytes(), null, seedPotentialDB, minVersion);
|
|
}
|
|
|
|
public yacySeed anySeedVersion(float minVersion) {
|
|
// return just any seed that has a specific minimum version number
|
|
Iterator<yacySeed> e = seedsConnected(true, true, yacySeed.randomHash(), minVersion);
|
|
return (yacySeed) e.next();
|
|
}
|
|
|
|
public HashMap<String, yacySeed> seedsByAge(boolean up, int count) {
|
|
// returns a peerhash/yacySeed relation
|
|
// to get most recent peers, set up = true; for oldest peers, set up = false
|
|
|
|
if (count > sizeConnected()) count = sizeConnected();
|
|
|
|
// fill a score object
|
|
kelondroMScoreCluster<String> seedScore = new kelondroMScoreCluster<String>();
|
|
yacySeed ys;
|
|
long absage;
|
|
Iterator<yacySeed> s = seedsConnected(true, false, null, (float) 0.0);
|
|
int searchcount = 1000;
|
|
if (searchcount > sizeConnected()) searchcount = sizeConnected();
|
|
try {
|
|
while ((s.hasNext()) && (searchcount-- > 0)) {
|
|
ys = s.next();
|
|
if ((ys != null) && (ys.get(yacySeed.LASTSEEN, "").length() > 10)) try {
|
|
absage = Math.abs(System.currentTimeMillis() + serverDate.dayMillis - ys.getLastSeenUTC());
|
|
seedScore.addScore(ys.hash, (int) absage); // the higher absage, the older is the peer
|
|
} catch (Exception e) {}
|
|
}
|
|
|
|
// result is now in the score object; create a result vector
|
|
HashMap<String, yacySeed> result = new HashMap<String, yacySeed>();
|
|
Iterator<String> it = seedScore.scores(up);
|
|
int c = 0;
|
|
while ((c < count) && (it.hasNext())) {
|
|
c++;
|
|
ys = getConnected((String) it.next());
|
|
if ((ys != null) && (ys.hash != null)) result.put(ys.hash, ys);
|
|
}
|
|
return result;
|
|
} catch (kelondroException e) {
|
|
seedActiveDB = resetSeedTable(seedActiveDB, seedActiveDBFile);
|
|
yacyCore.log.logFine("Internal Error at yacySeedDB.seedsByAge: " + e.getMessage(), e);
|
|
return null;
|
|
}
|
|
}
|
|
|
|
public int sizeConnected() {
|
|
return seedActiveDB.size();
|
|
/*
|
|
Enumeration e = seedsConnected(true, false, null);
|
|
int c = 0; while (e.hasMoreElements()) {c++; e.nextElement();}
|
|
return c;
|
|
*/
|
|
}
|
|
|
|
public int sizeDisconnected() {
|
|
return seedPassiveDB.size();
|
|
/*
|
|
Enumeration e = seedsDisconnected(true, false, null);
|
|
int c = 0; while (e.hasMoreElements()) {c++; e.nextElement();}
|
|
return c;
|
|
*/
|
|
}
|
|
|
|
public int sizePotential() {
|
|
return seedPotentialDB.size();
|
|
/*
|
|
Enumeration e = seedsPotential(true, false, null);
|
|
int c = 0; while (e.hasMoreElements()) {c++; e.nextElement();}
|
|
return c;
|
|
*/
|
|
}
|
|
|
|
public long countActiveURL() { return seedActiveDB.getLongAcc(yacySeed.LCOUNT); }
|
|
public long countActiveRWI() { return seedActiveDB.getLongAcc(yacySeed.ICOUNT); }
|
|
public long countActivePPM() { return seedActiveDB.getLongAcc(yacySeed.ISPEED); }
|
|
public double countActiveQPM() { return seedActiveDB.getDoubleAcc(yacySeed.RSPEED); }
|
|
public long countPassiveURL() { return seedPassiveDB.getLongAcc(yacySeed.LCOUNT); }
|
|
public long countPassiveRWI() { return seedPassiveDB.getLongAcc(yacySeed.ICOUNT); }
|
|
public long countPotentialURL() { return seedPotentialDB.getLongAcc(yacySeed.LCOUNT); }
|
|
public long countPotentialRWI() { return seedPotentialDB.getLongAcc(yacySeed.ICOUNT); }
|
|
|
|
public synchronized void addConnected(yacySeed seed) {
|
|
if ((seed == null) || (seed.isProper() != null)) return;
|
|
//seed.put(yacySeed.LASTSEEN, yacyCore.shortFormatter.format(new Date(yacyCore.universalTime())));
|
|
try {
|
|
nameLookupCache.put(seed.getName(), seed);
|
|
HashMap<String, String> seedPropMap = seed.getMap();
|
|
synchronized(seedPropMap) {
|
|
seedActiveDB.set(seed.hash, seedPropMap);
|
|
}
|
|
seedPassiveDB.remove(seed.hash);
|
|
seedPotentialDB.remove(seed.hash);
|
|
} catch (IOException e){
|
|
yacyCore.log.logSevere("ERROR add: seed.db corrupt (" + e.getMessage() + "); resetting seed.db", e);
|
|
resetActiveTable();
|
|
} catch (kelondroException e){
|
|
yacyCore.log.logSevere("ERROR add: seed.db corrupt (" + e.getMessage() + "); resetting seed.db", e);
|
|
resetActiveTable();
|
|
} catch (IllegalArgumentException e) {
|
|
yacyCore.log.logSevere("ERROR add: seed.db corrupt (" + e.getMessage() + "); resetting seed.db", e);
|
|
resetActiveTable();
|
|
}
|
|
}
|
|
|
|
public synchronized void addDisconnected(yacySeed seed) {
|
|
if (seed == null) return;
|
|
try {
|
|
nameLookupCache.remove(seed.getName());
|
|
seedActiveDB.remove(seed.hash);
|
|
seedPotentialDB.remove(seed.hash);
|
|
} catch (Exception e) {}
|
|
//seed.put(yacySeed.LASTSEEN, yacyCore.shortFormatter.format(new Date(yacyCore.universalTime())));
|
|
try {
|
|
HashMap<String, String> seedPropMap = seed.getMap();
|
|
synchronized(seedPropMap) {
|
|
seedPassiveDB.set(seed.hash, seedPropMap);
|
|
}
|
|
} catch (IOException e) {
|
|
yacyCore.log.logSevere("ERROR add: seed.db corrupt (" + e.getMessage() + "); resetting seed.db", e);
|
|
resetPassiveTable();
|
|
} catch (kelondroException e) {
|
|
yacyCore.log.logSevere("ERROR add: seed.db corrupt (" + e.getMessage() + "); resetting seed.db", e);
|
|
resetPassiveTable();
|
|
} catch (IllegalArgumentException e) {
|
|
yacyCore.log.logSevere("ERROR add: seed.db corrupt (" + e.getMessage() + "); resetting seed.db", e);
|
|
resetPassiveTable();
|
|
}
|
|
}
|
|
|
|
public synchronized void addPotential(yacySeed seed) {
|
|
if (seed == null) return;
|
|
try {
|
|
nameLookupCache.remove(seed.getName());
|
|
seedActiveDB.remove(seed.hash);
|
|
seedPassiveDB.remove(seed.hash);
|
|
} catch (Exception e) {}
|
|
if (seed.isProper() != null) return;
|
|
//seed.put(yacySeed.LASTSEEN, yacyCore.shortFormatter.format(new Date(yacyCore.universalTime())));
|
|
try {
|
|
HashMap<String, String> seedPropMap = seed.getMap();
|
|
synchronized(seedPropMap) {
|
|
seedPotentialDB.set(seed.hash, seedPropMap);
|
|
}
|
|
} catch (IOException e) {
|
|
yacyCore.log.logSevere("ERROR add: seed.db corrupt (" + e.getMessage() + "); resetting seed.db", e);
|
|
resetPotentialTable();
|
|
} catch (kelondroException e) {
|
|
yacyCore.log.logSevere("ERROR add: seed.db corrupt (" + e.getMessage() + "); resetting seed.db", e);
|
|
resetPotentialTable();
|
|
} catch (IllegalArgumentException e) {
|
|
yacyCore.log.logSevere("ERROR add: seed.db corrupt (" + e.getMessage() + "); resetting seed.db", e);
|
|
resetPotentialTable();
|
|
}
|
|
}
|
|
|
|
public synchronized void removeDisconnected(String peerHash) {
|
|
if(peerHash == null) return;
|
|
try {
|
|
seedPassiveDB.remove(peerHash);
|
|
} catch (IOException e) { }
|
|
}
|
|
|
|
public synchronized void removePotential(String peerHash) {
|
|
if(peerHash == null) return;
|
|
try {
|
|
seedPotentialDB.remove(peerHash);
|
|
} catch (IOException e) { }
|
|
}
|
|
|
|
public boolean hasConnected(String hash) {
|
|
try {
|
|
return (seedActiveDB.get(hash) != null);
|
|
} catch (IOException e) {
|
|
return false;
|
|
}
|
|
}
|
|
|
|
public boolean hasDisconnected(String hash) {
|
|
try {
|
|
return (seedPassiveDB.get(hash) != null);
|
|
} catch (IOException e) {
|
|
return false;
|
|
}
|
|
}
|
|
|
|
public boolean hasPotential(String hash) {
|
|
try {
|
|
return (seedPotentialDB.get(hash) != null);
|
|
} catch (IOException e) {
|
|
return false;
|
|
}
|
|
}
|
|
|
|
private yacySeed get(String hash, kelondroMapObjects database) {
|
|
if (hash == null) return null;
|
|
if ((this.mySeed != null) && (hash.equals(mySeed.hash))) return mySeed;
|
|
HashMap<String, String> entry = database.getMap(hash);
|
|
if (entry == null) return null;
|
|
return new yacySeed(hash, entry);
|
|
}
|
|
|
|
public yacySeed getConnected(String hash) {
|
|
return get(hash, seedActiveDB);
|
|
}
|
|
|
|
public yacySeed getDisconnected(String hash) {
|
|
return get(hash, seedPassiveDB);
|
|
}
|
|
|
|
public yacySeed getPotential(String hash) {
|
|
return get(hash, seedPotentialDB);
|
|
}
|
|
|
|
public yacySeed get(String hash) {
|
|
yacySeed seed = getConnected(hash);
|
|
if (seed == null) seed = getDisconnected(hash);
|
|
if (seed == null) seed = getPotential(hash);
|
|
return seed;
|
|
}
|
|
|
|
public void update(String hash, yacySeed seed) {
|
|
if (this.mySeed == null) initMySeed();
|
|
if (hash.equals(mySeed.hash)) {
|
|
mySeed = seed;
|
|
return;
|
|
}
|
|
|
|
yacySeed s = get(hash, seedActiveDB);
|
|
if (s != null) try { seedActiveDB.set(hash, seed.getMap()); return;} catch (IOException e) {}
|
|
|
|
s = get(hash, seedPassiveDB);
|
|
if (s != null) try { seedPassiveDB.set(hash, seed.getMap()); return;} catch (IOException e) {}
|
|
|
|
s = get(hash, seedPotentialDB);
|
|
if (s != null) try { seedPotentialDB.set(hash, seed.getMap()); return;} catch (IOException e) {}
|
|
}
|
|
|
|
public yacySeed lookupByName(String peerName) {
|
|
// reads a seed by searching by name
|
|
|
|
// local peer?
|
|
if (peerName.equals("localpeer")) {
|
|
if (this.mySeed == null) initMySeed();
|
|
return mySeed;
|
|
}
|
|
|
|
// then try to use the cache
|
|
yacySeed seed = (yacySeed) nameLookupCache.get(peerName);
|
|
if (seed != null) return seed;
|
|
|
|
// enumerate the cache and simultanous insert values
|
|
String name;
|
|
for (int table = 0; table < 2; table++) {
|
|
Iterator<yacySeed> e = (table == 0) ? seedsConnected(true, false, null, (float) 0.0) : seedsDisconnected(true, false, null, (float) 0.0);
|
|
while (e.hasNext()) {
|
|
seed = (yacySeed) e.next();
|
|
if (seed != null) {
|
|
name = seed.getName().toLowerCase();
|
|
if (seed.isProper() == null) nameLookupCache.put(name, seed);
|
|
if (name.equals(peerName)) return seed;
|
|
}
|
|
}
|
|
}
|
|
// check local seed
|
|
if (this.mySeed == null) initMySeed();
|
|
name = mySeed.getName().toLowerCase();
|
|
if (mySeed.isProper() == null) nameLookupCache.put(name, mySeed);
|
|
if (name.equals(peerName)) return mySeed;
|
|
// nothing found
|
|
return null;
|
|
}
|
|
|
|
public yacySeed lookupByIP(
|
|
InetAddress peerIP,
|
|
boolean lookupConnected,
|
|
boolean lookupDisconnected,
|
|
boolean lookupPotential
|
|
) {
|
|
|
|
if (peerIP == null) return null;
|
|
yacySeed seed = null;
|
|
|
|
// local peer?
|
|
if (httpd.isThisHostIP(peerIP)) {
|
|
if (this.mySeed == null) initMySeed();
|
|
return mySeed;
|
|
}
|
|
|
|
// then try to use the cache
|
|
SoftReference<yacySeed> ref = ipLookupCache.get(peerIP);
|
|
if (ref != null) {
|
|
seed = (yacySeed) ref.get();
|
|
if (seed != null) return seed;
|
|
}
|
|
|
|
int pos = -1;
|
|
String addressStr = null;
|
|
InetAddress seedIPAddress = null;
|
|
HashSet<String> badPeerHashes = new HashSet<String>();
|
|
|
|
if (lookupConnected) {
|
|
// enumerate the cache and simultanous insert values
|
|
Iterator<yacySeed> e = seedsConnected(true, false, null, (float) 0.0);
|
|
while (e.hasNext()) {
|
|
try {
|
|
seed = (yacySeed) e.next();
|
|
if (seed != null) {
|
|
addressStr = seed.getPublicAddress();
|
|
if (addressStr == null) {
|
|
serverLog.logWarning("YACY","lookupByIP/Connected: address of seed " + seed.getName() + "/" + seed.hash + " is null.");
|
|
badPeerHashes.add(seed.hash);
|
|
continue;
|
|
}
|
|
if ((pos = addressStr.indexOf(":"))!= -1) {
|
|
addressStr = addressStr.substring(0,pos);
|
|
}
|
|
seedIPAddress = InetAddress.getByName(addressStr);
|
|
if (seed.isProper() == null) ipLookupCache.put(seedIPAddress, new SoftReference<yacySeed>(seed));
|
|
if (seedIPAddress.equals(peerIP)) return seed;
|
|
}
|
|
} catch (UnknownHostException ex) {}
|
|
}
|
|
// delete bad peers
|
|
Iterator<String> i = badPeerHashes.iterator();
|
|
while (i.hasNext()) try {seedActiveDB.remove(i.next());} catch (IOException e1) {e1.printStackTrace();}
|
|
badPeerHashes.clear();
|
|
}
|
|
|
|
if (lookupDisconnected) {
|
|
// enumerate the cache and simultanous insert values
|
|
Iterator<yacySeed>e = seedsDisconnected(true, false, null, (float) 0.0);
|
|
|
|
while (e.hasNext()) {
|
|
try {
|
|
seed = (yacySeed) e.next();
|
|
if (seed != null) {
|
|
addressStr = seed.getPublicAddress();
|
|
if (addressStr == null) {
|
|
serverLog.logWarning("YACY","lookupByIPDisconnected: address of seed " + seed.getName() + "/" + seed.hash + " is null.");
|
|
badPeerHashes.add(seed.hash);
|
|
continue;
|
|
}
|
|
if ((pos = addressStr.indexOf(":"))!= -1) {
|
|
addressStr = addressStr.substring(0,pos);
|
|
}
|
|
seedIPAddress = InetAddress.getByName(addressStr);
|
|
if (seed.isProper() == null) ipLookupCache.put(seedIPAddress, new SoftReference<yacySeed>(seed));
|
|
if (seedIPAddress.equals(peerIP)) return seed;
|
|
}
|
|
} catch (UnknownHostException ex) {}
|
|
}
|
|
// delete bad peers
|
|
Iterator<String> i = badPeerHashes.iterator();
|
|
while (i.hasNext()) try {seedActiveDB.remove(i.next());} catch (IOException e1) {e1.printStackTrace();}
|
|
badPeerHashes.clear();
|
|
}
|
|
|
|
if (lookupPotential) {
|
|
// enumerate the cache and simultanous insert values
|
|
Iterator<yacySeed> e = seedsPotential(true, false, null, (float) 0.0);
|
|
|
|
while (e.hasNext()) {
|
|
try {
|
|
seed = (yacySeed) e.next();
|
|
if ((seed != null) && ((addressStr = seed.getPublicAddress()) != null)) {
|
|
if ((pos = addressStr.indexOf(":"))!= -1) {
|
|
addressStr = addressStr.substring(0,pos);
|
|
}
|
|
seedIPAddress = InetAddress.getByName(addressStr);
|
|
if (seed.isProper() == null) ipLookupCache.put(seedIPAddress, new SoftReference<yacySeed>(seed));
|
|
if (seedIPAddress.equals(peerIP)) return seed;
|
|
}
|
|
} catch (UnknownHostException ex) {}
|
|
}
|
|
}
|
|
|
|
try {
|
|
// check local seed
|
|
if (this.mySeed == null) return null;
|
|
addressStr = mySeed.getPublicAddress();
|
|
if (addressStr == null) return null;
|
|
if ((pos = addressStr.indexOf(":"))!= -1) {
|
|
addressStr = addressStr.substring(0,pos);
|
|
}
|
|
seedIPAddress = InetAddress.getByName(addressStr);
|
|
if (mySeed.isProper() == null) ipLookupCache.put(seedIPAddress, new SoftReference<yacySeed>(mySeed));
|
|
if (seedIPAddress.equals(peerIP)) return mySeed;
|
|
// nothing found
|
|
return null;
|
|
} catch (UnknownHostException e2) {
|
|
return null;
|
|
}
|
|
}
|
|
|
|
public ArrayList<String> storeCache(File seedFile) throws IOException {
|
|
return storeCache(seedFile, false);
|
|
}
|
|
|
|
private ArrayList<String> storeCache(File seedFile, boolean addMySeed) throws IOException {
|
|
PrintWriter pw = null;
|
|
ArrayList<String> v = new ArrayList<String>(seedActiveDB.size() + 1);
|
|
try {
|
|
|
|
pw = new PrintWriter(new BufferedWriter(new FileWriter(seedFile)));
|
|
|
|
// store own seed
|
|
String line;
|
|
if (this.mySeed == null) initMySeed();
|
|
if (addMySeed) {
|
|
line = mySeed.genSeedStr(null);
|
|
v.add(line);
|
|
pw.print(line + serverCore.CRLF_STRING);
|
|
}
|
|
|
|
// store other seeds
|
|
yacySeed ys;
|
|
Iterator<yacySeed> se = seedsConnected(true, false, null, (float) 0.0);
|
|
while (se.hasNext()) {
|
|
ys = (yacySeed) se.next();
|
|
if (ys != null) {
|
|
line = ys.genSeedStr(null);
|
|
v.add(line);
|
|
pw.print(line + serverCore.CRLF_STRING);
|
|
}
|
|
}
|
|
pw.flush();
|
|
} finally {
|
|
if (pw != null) try { pw.close(); } catch (Exception e) {}
|
|
}
|
|
return v;
|
|
}
|
|
|
|
public String uploadCache(yacySeedUploader uploader,
|
|
serverSwitch<?> sb,
|
|
yacySeedDB seedDB,
|
|
// String seedFTPServer,
|
|
// String seedFTPAccount,
|
|
// String seedFTPPassword,
|
|
// File seedFTPPath,
|
|
yacyURL seedURL) throws Exception {
|
|
|
|
// upload a seed file, if possible
|
|
if (seedURL == null) throw new NullPointerException("UPLOAD - Error: URL not given");
|
|
|
|
String log = null;
|
|
File seedFile = null;
|
|
try {
|
|
// create a seed file which for uploading ...
|
|
seedFile = File.createTempFile("seedFile",".txt", plasmaHTCache.cachePath);
|
|
seedFile.deleteOnExit();
|
|
serverLog.logFine("YACY","SaveSeedList: Storing seedlist into tempfile " + seedFile.toString());
|
|
ArrayList<String> uv = storeCache(seedFile, true);
|
|
|
|
// uploading the seed file
|
|
serverLog.logFine("YACY","SaveSeedList: Trying to upload seed-file, " + seedFile.length() + " bytes, " + uv.size() + " entries.");
|
|
log = uploader.uploadSeedFile(sb,seedDB,seedFile);
|
|
|
|
// test download
|
|
serverLog.logFine("YACY","SaveSeedList: Trying to download seed-file '" + seedURL + "'.");
|
|
ArrayList<String> check = downloadSeedFile(seedURL);
|
|
|
|
// Comparing if local copy and uploaded copy are equal
|
|
String errorMsg = checkCache(uv, check);
|
|
if (errorMsg == null)
|
|
log = log + "UPLOAD CHECK - Success: the result vectors are equal" + serverCore.CRLF_STRING;
|
|
else {
|
|
throw new Exception("UPLOAD CHECK - Error: the result vector is different. " + errorMsg + serverCore.CRLF_STRING);
|
|
}
|
|
} finally {
|
|
if (seedFile != null) try { seedFile.delete(); } catch (Exception e) {/* ignore this */}
|
|
}
|
|
|
|
return log;
|
|
}
|
|
|
|
private ArrayList<String> downloadSeedFile(yacyURL seedURL) throws IOException {
|
|
// Configure http headers
|
|
httpHeader reqHeader = new httpHeader();
|
|
reqHeader.put(httpHeader.PRAGMA, "no-cache");
|
|
reqHeader.put(httpHeader.CACHE_CONTROL, "no-cache"); // httpc uses HTTP/1.0 is this necessary?
|
|
|
|
// init http-client
|
|
JakartaCommonsHttpClient client = new JakartaCommonsHttpClient(10000, reqHeader, null);
|
|
byte[] content = null;
|
|
JakartaCommonsHttpResponse res = null;
|
|
try {
|
|
// send request
|
|
res = client.GET(seedURL.toString());
|
|
|
|
// check response code
|
|
if (res.getStatusCode() != 200) {
|
|
throw new IOException("Server returned status: " + res.getStatusLine());
|
|
}
|
|
|
|
// read byte array
|
|
content = res.getData();
|
|
} finally {
|
|
if(res != null) {
|
|
res.closeStream();
|
|
}
|
|
}
|
|
|
|
try {
|
|
// uncompress it if it is gzipped
|
|
content = serverFileUtils.uncompressGZipArray(content);
|
|
|
|
// convert it into an array
|
|
return nxTools.strings(content,"UTF-8");
|
|
} catch (Exception e) {
|
|
throw new IOException("Unable to download seed file '" + seedURL + "'. " + e.getMessage());
|
|
}
|
|
}
|
|
|
|
private String checkCache(ArrayList<String> uv, ArrayList<String> check) {
|
|
if ((check == null) || (uv == null) || (uv.size() != check.size())) {
|
|
serverLog.logFine("YACY","SaveSeedList: Local and uploades seed-list " +
|
|
"contains varying numbers of entries." +
|
|
"\n\tLocal seed-list: " + ((uv == null) ? "null" : Integer.toString(uv.size())) + " entries" +
|
|
"\n\tRemote seed-list: " + ((check == null) ? "null" : Integer.toString(check.size())) + " enties");
|
|
return "Entry count is different: uv.size() = " + ((uv == null) ? "null" : Integer.toString(uv.size())) + ", check = " + ((check == null) ? "null" : Integer.toString(check.size()));
|
|
}
|
|
|
|
serverLog.logFine("YACY","SaveSeedList: Comparing local and uploades seed-list entries ...");
|
|
int i;
|
|
for (i = 0; i < uv.size(); i++) {
|
|
if (!(((String) uv.get(i)).equals((String) check.get(i)))) return "Element at position " + i + " is different.";
|
|
}
|
|
|
|
// no difference found
|
|
return null;
|
|
}
|
|
|
|
/**
|
|
* resolve a yacy address
|
|
*/
|
|
public String resolve(String host) {
|
|
yacySeed seed;
|
|
int p;
|
|
String subdom = null;
|
|
if (host.endsWith(".yacyh")) {
|
|
// this is not functional at the moment
|
|
// caused by lowecasing of hashes at the browser client
|
|
p = host.indexOf(".");
|
|
if ((p > 0) && (p != (host.length() - 6))) {
|
|
subdom = host.substring(0, p);
|
|
host = host.substring(p + 1);
|
|
}
|
|
// check if we have a b64-hash or a hex-hash
|
|
String hash = host.substring(0, host.length() - 6);
|
|
if (hash.length() > commonHashLength) {
|
|
// this is probably a hex-hash
|
|
hash = yacySeed.hexHash2b64Hash(hash);
|
|
}
|
|
// check remote seeds
|
|
seed = getConnected(hash); // checks only remote, not local
|
|
// check local seed
|
|
if (seed == null) {
|
|
if (this.mySeed == null) initMySeed();
|
|
if (hash.equals(mySeed.hash))
|
|
seed = mySeed;
|
|
else return null;
|
|
}
|
|
return seed.getPublicAddress() + ((subdom == null) ? "" : ("/" + subdom));
|
|
} else if (host.endsWith(".yacy")) {
|
|
// identify subdomain
|
|
p = host.indexOf(".");
|
|
if ((p > 0) && (p != (host.length() - 5))) {
|
|
subdom = host.substring(0, p); // no double-dot attack possible, the subdom cannot have ".." in it
|
|
host = host.substring(p + 1); // if ever, the double-dots are here but do not harm
|
|
}
|
|
// identify domain
|
|
String domain = host.substring(0, host.length() - 5).toLowerCase();
|
|
seed = lookupByName(domain);
|
|
if (seed == null) return null;
|
|
if (this.mySeed == null) initMySeed();
|
|
if ((seed == mySeed) && (!(seed.isOnline()))) {
|
|
// take local ip instead of external
|
|
return serverDomains.myPublicIP() + ":8080" + ((subdom == null) ? "" : ("/" + subdom));
|
|
}
|
|
return seed.getPublicAddress() + ((subdom == null) ? "" : ("/" + subdom));
|
|
} else {
|
|
return null;
|
|
}
|
|
}
|
|
|
|
class seedEnum implements Iterator<yacySeed> {
|
|
|
|
kelondroMapObjects.mapIterator it;
|
|
yacySeed nextSeed;
|
|
kelondroMapObjects database;
|
|
float minVersion;
|
|
|
|
public seedEnum(boolean up, boolean rot, byte[] firstKey, byte[] secondKey, kelondroMapObjects database, float minVersion) {
|
|
this.database = database;
|
|
this.minVersion = minVersion;
|
|
try {
|
|
it = (firstKey == null) ? database.maps(up, rot) : database.maps(up, rot, firstKey, secondKey);
|
|
while (true) {
|
|
nextSeed = internalNext();
|
|
if (nextSeed == null) break;
|
|
if (nextSeed.getVersion() >= this.minVersion) break;
|
|
}
|
|
} catch (IOException e) {
|
|
e.printStackTrace();
|
|
yacyCore.log.logSevere("ERROR seedLinEnum: seed.db corrupt (" + e.getMessage() + "); resetting seed.db", e);
|
|
if (database == seedActiveDB) seedActiveDB = resetSeedTable(seedActiveDB, seedActiveDBFile);
|
|
if (database == seedPassiveDB) seedPassiveDB = resetSeedTable(seedPassiveDB, seedPassiveDBFile);
|
|
it = null;
|
|
} catch (kelondroException e) {
|
|
e.printStackTrace();
|
|
yacyCore.log.logSevere("ERROR seedLinEnum: seed.db corrupt (" + e.getMessage() + "); resetting seed.db", e);
|
|
if (database == seedActiveDB) seedActiveDB = resetSeedTable(seedActiveDB, seedActiveDBFile);
|
|
if (database == seedPassiveDB) seedPassiveDB = resetSeedTable(seedPassiveDB, seedPassiveDBFile);
|
|
it = null;
|
|
}
|
|
}
|
|
|
|
public seedEnum(boolean up, String field, kelondroMapObjects database) {
|
|
this.database = database;
|
|
try {
|
|
it = database.maps(up, field);
|
|
nextSeed = internalNext();
|
|
} catch (kelondroException e) {
|
|
e.printStackTrace();
|
|
yacyCore.log.logSevere("ERROR seedLinEnum: seed.db corrupt (" + e.getMessage() + "); resetting seed.db", e);
|
|
if (database == seedActiveDB) seedActiveDB = resetSeedTable(seedActiveDB, seedActiveDBFile);
|
|
if (database == seedPassiveDB) seedPassiveDB = resetSeedTable(seedPassiveDB, seedPassiveDBFile);
|
|
if (database == seedPotentialDB) seedPotentialDB = resetSeedTable(seedPotentialDB, seedPotentialDBFile);
|
|
it = null;
|
|
}
|
|
}
|
|
|
|
public boolean hasNext() {
|
|
return (nextSeed != null);
|
|
}
|
|
|
|
public yacySeed internalNext() {
|
|
if ((it == null) || (!(it.hasNext()))) return null;
|
|
try {
|
|
HashMap<String, String> dna = it.next();
|
|
if (dna == null) return null;
|
|
String hash = (String) dna.remove("key");
|
|
//while (hash.length() < commonHashLength) { hash = hash + "_"; }
|
|
return new yacySeed(hash, dna);
|
|
} catch (Exception e) {
|
|
e.printStackTrace();
|
|
yacyCore.log.logSevere("ERROR internalNext: seed.db corrupt (" + e.getMessage() + "); resetting seed.db", e);
|
|
if (database == seedActiveDB) seedActiveDB = resetSeedTable(seedActiveDB, seedActiveDBFile);
|
|
if (database == seedPassiveDB) seedPassiveDB = resetSeedTable(seedPassiveDB, seedPassiveDBFile);
|
|
if (database == seedPotentialDB) seedPotentialDB = resetSeedTable(seedPotentialDB, seedPotentialDBFile);
|
|
return null;
|
|
}
|
|
}
|
|
|
|
public yacySeed next() {
|
|
yacySeed seed = nextSeed;
|
|
try {while (true) {
|
|
nextSeed = internalNext();
|
|
if (nextSeed == null) break;
|
|
if (nextSeed.getVersion() >= this.minVersion) break;
|
|
}} catch (kelondroException e) {
|
|
e.printStackTrace();
|
|
// eergency reset
|
|
yacyCore.log.logSevere("seed-db emergency reset", e);
|
|
try {
|
|
database.reset();
|
|
nextSeed = null;
|
|
return null;
|
|
} catch (IOException e1) {
|
|
// no recovery possible
|
|
e1.printStackTrace();
|
|
System.exit(-1);
|
|
}
|
|
}
|
|
return seed;
|
|
}
|
|
|
|
public void remove() {
|
|
throw new UnsupportedOperationException();
|
|
}
|
|
|
|
}
|
|
|
|
}
|