2011-09-14 23:19:02 +02:00
// yacySeedDB.java
2005-04-07 21:19:42 +02:00
// -------------------------------------
2008-07-20 19:14:51 +02:00
// (C) by Michael Peter Christen; mc@yacy.net
2005-04-07 21:19:42 +02:00
// first published on http://www.anomic.de
// Frankfurt, Germany, 2004, 2005
2005-10-17 17:46:12 +02:00
//
// $LastChangedDate$
// $LastChangedRevision$
2007-04-19 18:16:54 +02:00
// $LastChangedBy$
2005-04-07 21:19:42 +02:00
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation; either version 2 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
2011-09-25 18:59:06 +02:00
package net.yacy.peers ;
2005-04-07 21:19:42 +02:00
2005-05-17 10:25:04 +02:00
import java.io.BufferedWriter ;
2005-05-05 07:32:19 +02:00
import java.io.File ;
import java.io.FileWriter ;
import java.io.IOException ;
import java.io.PrintWriter ;
2005-10-12 10:17:43 +02:00
import java.net.InetAddress ;
2005-10-05 12:45:33 +02:00
import java.util.ArrayList ;
2012-02-02 06:43:57 +01:00
import java.util.Collection ;
2012-01-31 18:17:25 +01:00
import java.util.HashSet ;
2005-05-05 07:32:19 +02:00
import java.util.Iterator ;
import java.util.Map ;
2011-04-04 01:39:45 +02:00
import java.util.Set ;
2007-04-30 00:05:34 +02:00
import java.util.TreeMap ;
2010-02-15 16:57:35 +01:00
import java.util.concurrent.ConcurrentHashMap ;
2011-09-14 23:19:02 +02:00
import java.util.concurrent.ConcurrentMap ;
2006-09-30 00:27:20 +02:00
2011-05-27 10:24:54 +02:00
import net.yacy.cora.document.ASCII ;
2011-04-12 07:02:36 +02:00
import net.yacy.cora.document.UTF8 ;
2011-04-26 15:35:29 +02:00
import net.yacy.cora.protocol.ClientIdentification ;
2010-09-14 17:27:27 +02:00
import net.yacy.cora.protocol.Domains ;
2010-08-23 14:32:02 +02:00
import net.yacy.cora.protocol.HeaderFramework ;
import net.yacy.cora.protocol.RequestHeader ;
2010-08-23 00:32:39 +02:00
import net.yacy.cora.protocol.http.HTTPClient ;
2009-10-10 02:43:25 +02:00
import net.yacy.kelondro.blob.MapDataMining ;
2009-10-11 02:12:19 +02:00
import net.yacy.kelondro.data.meta.DigestURI ;
import net.yacy.kelondro.data.word.Word ;
2010-04-19 18:42:37 +02:00
import net.yacy.kelondro.index.RowSpaceExceededException ;
2009-10-10 01:13:30 +02:00
import net.yacy.kelondro.logging.Log ;
2009-10-10 01:22:22 +02:00
import net.yacy.kelondro.order.Base64Order ;
2009-10-10 03:14:19 +02:00
import net.yacy.kelondro.util.FileUtils ;
import net.yacy.kelondro.util.kelondroException ;
2011-09-25 18:59:06 +02:00
import net.yacy.peers.dht.PartitionScheme ;
import net.yacy.peers.dht.VerticalWordPartitionScheme ;
import net.yacy.peers.operation.yacySeedUploader ;
import net.yacy.search.Switchboard ;
2009-07-19 22:37:44 +02:00
import de.anomic.http.server.AlternativeDomainNames ;
2011-01-03 21:52:54 +01:00
import de.anomic.http.server.HTTPDemon ;
2005-05-05 07:32:19 +02:00
import de.anomic.server.serverCore ;
2005-05-17 10:25:04 +02:00
import de.anomic.server.serverSwitch ;
2005-04-07 21:19:42 +02:00
2011-10-04 11:06:24 +02:00
public final class SeedDB implements AlternativeDomainNames {
2011-09-14 23:19:02 +02:00
2005-04-07 21:19:42 +02:00
// global statics
2006-02-21 00:15:39 +01:00
2010-04-19 18:42:37 +02:00
private static final int dhtActivityMagic = 32 ;
2011-09-14 23:19:02 +02:00
2008-05-14 22:30:44 +02:00
/ * *
* < p > < code > public static final String < strong > DBFILE_OWN_SEED < / strong > = " mySeed.txt " < / code > < / p >
* < p > Name of the file containing the database holding this peer ' s seed < / p >
* /
public static final String DBFILE_OWN_SEED = " mySeed.txt " ;
2011-09-14 23:19:02 +02:00
2011-10-04 11:06:24 +02:00
public static final String [ ] sortFields = new String [ ] { Seed . LCOUNT , Seed . RCOUNT , Seed . ICOUNT , Seed . UPTIME , Seed . VERSION , Seed . LASTSEEN } ;
public static final String [ ] longaccFields = new String [ ] { Seed . LCOUNT , Seed . ICOUNT , Seed . ISPEED } ;
public static final String [ ] doubleaccFields = new String [ ] { Seed . RSPEED } ;
2011-09-14 23:19:02 +02:00
2005-04-07 21:19:42 +02:00
// class objects
2010-04-19 18:42:37 +02:00
private File seedActiveDBFile , seedPassiveDBFile , seedPotentialDBFile ;
private File myOwnSeedFile ;
private MapDataMining seedActiveDB , seedPassiveDB , seedPotentialDB ;
2011-09-14 23:19:02 +02:00
2010-04-19 18:42:37 +02:00
protected int lastSeedUpload_seedDBSize = 0 ;
2008-05-06 01:13:47 +02:00
public long lastSeedUpload_timeStamp = System . currentTimeMillis ( ) ;
2010-04-19 18:42:37 +02:00
protected String lastSeedUpload_myIP = " " ;
2009-01-23 16:32:27 +01:00
2011-10-04 11:06:24 +02:00
public PeerActions peerActions ;
public NewsPool newsPool ;
2011-09-14 23:19:02 +02:00
2010-01-18 01:07:20 +01:00
private int netRedundancy ;
replaced old DHT transmission method with new method. Many things have changed! some of them:
- after a index selection is made, the index is splitted into its vertical components
- from differrent index selctions the splitted components can be accumulated before they are placed into the transmission queue
- each splitted chunk gets its own transmission thread
- multiple transmission threads are started concurrently
- the process can be monitored with the blocking queue servlet
To implement that, a new package de.anomic.yacy.dht was created. Some old files have been removed.
The new index distribution model using a vertical DHT was implemented. An abstraction of this model
is implemented in the new dht package as interface. The freeworld network has now a configuration
of two vertial partitions; sixteen partitions are planned and will be configured if the process is bug-free.
This modification has three main targets:
- enhance the DHT transmission speed
- with a vertical DHT, a search will speed up. With two partitions, two times. With sixteen, sixteen times.
- the vertical DHT will apply a semi-dht for URLs, and peers will receive a fraction of the overall URLs they received before.
with two partitions, the fractions will be halve. With sixteen partitions, a 1/16 of the previous number of URLs.
BE CAREFULL, THIS IS A MAJOR CODE CHANGE, POSSIBLY FULL OF BUGS AND HARMFUL THINGS.
git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@5586 6c8d7289-2bf4-0310-a012-ef5d649a1542
2009-02-10 01:06:59 +01:00
public PartitionScheme scheme ;
2011-09-14 23:19:02 +02:00
2011-10-04 11:06:24 +02:00
private Seed mySeed ; // my own seed
2011-09-14 23:19:02 +02:00
private final Set < String > myBotIDs ; // list of id's that this bot accepts as robots.txt identification
2011-10-04 11:06:24 +02:00
public SeedDB (
replaced old DHT transmission method with new method. Many things have changed! some of them:
- after a index selection is made, the index is splitted into its vertical components
- from differrent index selctions the splitted components can be accumulated before they are placed into the transmission queue
- each splitted chunk gets its own transmission thread
- multiple transmission threads are started concurrently
- the process can be monitored with the blocking queue servlet
To implement that, a new package de.anomic.yacy.dht was created. Some old files have been removed.
The new index distribution model using a vertical DHT was implemented. An abstraction of this model
is implemented in the new dht package as interface. The freeworld network has now a configuration
of two vertial partitions; sixteen partitions are planned and will be configured if the process is bug-free.
This modification has three main targets:
- enhance the DHT transmission speed
- with a vertical DHT, a search will speed up. With two partitions, two times. With sixteen, sixteen times.
- the vertical DHT will apply a semi-dht for URLs, and peers will receive a fraction of the overall URLs they received before.
with two partitions, the fractions will be halve. With sixteen partitions, a 1/16 of the previous number of URLs.
BE CAREFULL, THIS IS A MAJOR CODE CHANGE, POSSIBLY FULL OF BUGS AND HARMFUL THINGS.
git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@5586 6c8d7289-2bf4-0310-a012-ef5d649a1542
2009-02-10 01:06:59 +01:00
final File networkRoot ,
final String seedActiveDBFileName ,
final String seedPassiveDBFileName ,
final String seedPotentialDBFileName ,
2011-09-14 23:19:02 +02:00
final File myOwnSeedFile ,
2009-01-23 16:32:27 +01:00
final int redundancy ,
2009-09-07 22:30:57 +02:00
final int partitionExponent ,
final boolean useTailCache ,
final boolean exceed134217727 ) {
replaced old DHT transmission method with new method. Many things have changed! some of them:
- after a index selection is made, the index is splitted into its vertical components
- from differrent index selctions the splitted components can be accumulated before they are placed into the transmission queue
- each splitted chunk gets its own transmission thread
- multiple transmission threads are started concurrently
- the process can be monitored with the blocking queue servlet
To implement that, a new package de.anomic.yacy.dht was created. Some old files have been removed.
The new index distribution model using a vertical DHT was implemented. An abstraction of this model
is implemented in the new dht package as interface. The freeworld network has now a configuration
of two vertial partitions; sixteen partitions are planned and will be configured if the process is bug-free.
This modification has three main targets:
- enhance the DHT transmission speed
- with a vertical DHT, a search will speed up. With two partitions, two times. With sixteen, sixteen times.
- the vertical DHT will apply a semi-dht for URLs, and peers will receive a fraction of the overall URLs they received before.
with two partitions, the fractions will be halve. With sixteen partitions, a 1/16 of the previous number of URLs.
BE CAREFULL, THIS IS A MAJOR CODE CHANGE, POSSIBLY FULL OF BUGS AND HARMFUL THINGS.
git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@5586 6c8d7289-2bf4-0310-a012-ef5d649a1542
2009-02-10 01:06:59 +01:00
this . seedActiveDBFile = new File ( networkRoot , seedActiveDBFileName ) ;
this . seedPassiveDBFile = new File ( networkRoot , seedPassiveDBFileName ) ;
this . seedPotentialDBFile = new File ( networkRoot , seedPotentialDBFileName ) ;
2005-04-07 21:19:42 +02:00
this . mySeed = null ; // my own seed
2008-05-06 01:13:47 +02:00
this . myOwnSeedFile = myOwnSeedFile ;
2011-04-04 01:39:45 +02:00
this . myBotIDs = new HashSet < String > ( ) ;
this . myBotIDs . add ( " yacy " ) ;
this . myBotIDs . add ( " yacybot " ) ;
2011-04-04 14:20:20 +02:00
this . myBotIDs . add ( " yacyproxy " ) ;
2009-01-23 16:32:27 +01:00
this . netRedundancy = redundancy ;
replaced old DHT transmission method with new method. Many things have changed! some of them:
- after a index selection is made, the index is splitted into its vertical components
- from differrent index selctions the splitted components can be accumulated before they are placed into the transmission queue
- each splitted chunk gets its own transmission thread
- multiple transmission threads are started concurrently
- the process can be monitored with the blocking queue servlet
To implement that, a new package de.anomic.yacy.dht was created. Some old files have been removed.
The new index distribution model using a vertical DHT was implemented. An abstraction of this model
is implemented in the new dht package as interface. The freeworld network has now a configuration
of two vertial partitions; sixteen partitions are planned and will be configured if the process is bug-free.
This modification has three main targets:
- enhance the DHT transmission speed
- with a vertical DHT, a search will speed up. With two partitions, two times. With sixteen, sixteen times.
- the vertical DHT will apply a semi-dht for URLs, and peers will receive a fraction of the overall URLs they received before.
with two partitions, the fractions will be halve. With sixteen partitions, a 1/16 of the previous number of URLs.
BE CAREFULL, THIS IS A MAJOR CODE CHANGE, POSSIBLY FULL OF BUGS AND HARMFUL THINGS.
git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@5586 6c8d7289-2bf4-0310-a012-ef5d649a1542
2009-02-10 01:06:59 +01:00
this . scheme = new VerticalWordPartitionScheme ( partitionExponent ) ;
2011-09-14 23:19:02 +02:00
2005-04-07 21:19:42 +02:00
// set up seed database
2011-09-14 23:19:02 +02:00
this . seedActiveDB = openSeedTable ( this . seedActiveDBFile ) ;
this . seedPassiveDB = openSeedTable ( this . seedPassiveDBFile ) ;
this . seedPotentialDB = openSeedTable ( this . seedPotentialDBFile ) ;
2007-10-01 14:30:23 +02:00
// check if we are in the seedCaches: this can happen if someone else published our seed
removeMySeed ( ) ;
2011-09-14 23:19:02 +02:00
2010-01-18 01:07:20 +01:00
this . lastSeedUpload_seedDBSize = sizeConnected ( ) ;
2008-05-06 01:13:47 +02:00
// tell the httpdProxy how to find this table as address resolver
2009-07-19 22:37:44 +02:00
HTTPDemon . setAlternativeResolver ( this ) ;
2011-09-14 23:19:02 +02:00
replaced old DHT transmission method with new method. Many things have changed! some of them:
- after a index selection is made, the index is splitted into its vertical components
- from differrent index selctions the splitted components can be accumulated before they are placed into the transmission queue
- each splitted chunk gets its own transmission thread
- multiple transmission threads are started concurrently
- the process can be monitored with the blocking queue servlet
To implement that, a new package de.anomic.yacy.dht was created. Some old files have been removed.
The new index distribution model using a vertical DHT was implemented. An abstraction of this model
is implemented in the new dht package as interface. The freeworld network has now a configuration
of two vertial partitions; sixteen partitions are planned and will be configured if the process is bug-free.
This modification has three main targets:
- enhance the DHT transmission speed
- with a vertical DHT, a search will speed up. With two partitions, two times. With sixteen, sixteen times.
- the vertical DHT will apply a semi-dht for URLs, and peers will receive a fraction of the overall URLs they received before.
with two partitions, the fractions will be halve. With sixteen partitions, a 1/16 of the previous number of URLs.
BE CAREFULL, THIS IS A MAJOR CODE CHANGE, POSSIBLY FULL OF BUGS AND HARMFUL THINGS.
git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@5586 6c8d7289-2bf4-0310-a012-ef5d649a1542
2009-02-10 01:06:59 +01:00
// create or init news database
2011-10-04 11:06:24 +02:00
this . newsPool = new NewsPool ( networkRoot , useTailCache , exceed134217727 ) ;
2011-09-14 23:19:02 +02:00
2010-01-18 01:07:20 +01:00
// deploy peer actions
2011-10-04 11:06:24 +02:00
this . peerActions = new PeerActions ( this , this . newsPool ) ;
2010-01-18 01:07:20 +01:00
}
2011-09-14 23:19:02 +02:00
2010-01-18 01:07:20 +01:00
public void relocate (
2011-09-14 23:19:02 +02:00
final File newNetworkRoot ,
2010-01-18 01:07:20 +01:00
final int redundancy ,
final int partitionExponent ,
final boolean useTailCache ,
final boolean exceed134217727 ) {
2011-09-14 23:19:02 +02:00
2010-01-18 01:07:20 +01:00
// close old databases
this . seedActiveDB . close ( ) ;
this . seedPassiveDB . close ( ) ;
this . seedPotentialDB . close ( ) ;
this . newsPool . close ( ) ;
this . peerActions . close ( ) ;
2011-09-14 23:19:02 +02:00
2010-01-18 01:07:20 +01:00
// open new according to the newNetworkRoot
2011-09-14 23:19:02 +02:00
this . seedActiveDBFile = new File ( newNetworkRoot , this . seedActiveDBFile . getName ( ) ) ;
this . seedPassiveDBFile = new File ( newNetworkRoot , this . seedPassiveDBFile . getName ( ) ) ;
this . seedPotentialDBFile = new File ( newNetworkRoot , this . seedPotentialDBFile . getName ( ) ) ;
2011-03-20 00:52:09 +01:00
2011-04-04 01:39:45 +02:00
// replace my (old) seed with new seed definition from other network
// but keep the seed name
2011-09-14 23:19:02 +02:00
final String peername = myName ( ) ;
2010-01-18 01:07:20 +01:00
this . mySeed = null ; // my own seed
2011-10-04 11:06:24 +02:00
this . myOwnSeedFile = new File ( newNetworkRoot , SeedDB . DBFILE_OWN_SEED ) ;
2011-04-04 01:39:45 +02:00
initMySeed ( ) ;
this . mySeed . setName ( peername ) ;
2011-09-14 23:19:02 +02:00
2010-01-18 01:07:20 +01:00
this . netRedundancy = redundancy ;
this . scheme = new VerticalWordPartitionScheme ( partitionExponent ) ;
2011-09-14 23:19:02 +02:00
2010-01-18 01:07:20 +01:00
// set up seed database
2011-09-14 23:19:02 +02:00
this . seedActiveDB = openSeedTable ( this . seedActiveDBFile ) ;
this . seedPassiveDB = openSeedTable ( this . seedPassiveDBFile ) ;
this . seedPotentialDB = openSeedTable ( this . seedPotentialDBFile ) ;
2010-01-18 01:07:20 +01:00
// check if we are in the seedCaches: this can happen if someone else published our seed
removeMySeed ( ) ;
2011-09-14 23:19:02 +02:00
2010-01-18 01:07:20 +01:00
this . lastSeedUpload_seedDBSize = sizeConnected ( ) ;
// tell the httpdProxy how to find this table as address resolver
HTTPDemon . setAlternativeResolver ( this ) ;
2011-09-14 23:19:02 +02:00
2010-01-18 01:07:20 +01:00
// create or init news database
2011-10-04 11:06:24 +02:00
this . newsPool = new NewsPool ( newNetworkRoot , useTailCache , exceed134217727 ) ;
2011-09-14 23:19:02 +02:00
replaced old DHT transmission method with new method. Many things have changed! some of them:
- after a index selection is made, the index is splitted into its vertical components
- from differrent index selctions the splitted components can be accumulated before they are placed into the transmission queue
- each splitted chunk gets its own transmission thread
- multiple transmission threads are started concurrently
- the process can be monitored with the blocking queue servlet
To implement that, a new package de.anomic.yacy.dht was created. Some old files have been removed.
The new index distribution model using a vertical DHT was implemented. An abstraction of this model
is implemented in the new dht package as interface. The freeworld network has now a configuration
of two vertial partitions; sixteen partitions are planned and will be configured if the process is bug-free.
This modification has three main targets:
- enhance the DHT transmission speed
- with a vertical DHT, a search will speed up. With two partitions, two times. With sixteen, sixteen times.
- the vertical DHT will apply a semi-dht for URLs, and peers will receive a fraction of the overall URLs they received before.
with two partitions, the fractions will be halve. With sixteen partitions, a 1/16 of the previous number of URLs.
BE CAREFULL, THIS IS A MAJOR CODE CHANGE, POSSIBLY FULL OF BUGS AND HARMFUL THINGS.
git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@5586 6c8d7289-2bf4-0310-a012-ef5d649a1542
2009-02-10 01:06:59 +01:00
// deploy peer actions
2011-10-04 11:06:24 +02:00
this . peerActions = new PeerActions ( this , this . newsPool ) ;
2007-10-01 14:30:23 +02:00
}
2011-09-14 23:19:02 +02:00
2007-10-01 14:30:23 +02:00
private synchronized void initMySeed ( ) {
if ( this . mySeed ! = null ) return ;
2011-09-14 23:19:02 +02:00
2005-04-07 21:19:42 +02:00
// create or init own seed
2011-09-14 23:19:02 +02:00
if ( this . myOwnSeedFile . length ( ) > 0 ) try {
2005-06-23 13:00:26 +02:00
// load existing identity
2011-10-04 11:06:24 +02:00
this . mySeed = Seed . load ( this . myOwnSeedFile ) ;
2011-09-14 23:19:02 +02:00
if ( this . mySeed = = null ) throw new IOException ( " current seed is null " ) ;
2008-08-02 14:12:04 +02:00
} catch ( final IOException e ) {
2005-12-13 00:59:58 +01:00
// create new identity
2011-09-14 23:19:02 +02:00
Log . logSevere ( " SEEDDB " , " could not load stored mySeed.txt from " + this . myOwnSeedFile . toString ( ) + " : " + e . getMessage ( ) + " . creating new seed. " , e ) ;
2011-10-04 11:06:24 +02:00
this . mySeed = Seed . genLocalSeed ( this ) ;
2005-12-13 00:59:58 +01:00
try {
2011-09-14 23:19:02 +02:00
this . mySeed . save ( this . myOwnSeedFile ) ;
2008-08-02 14:12:04 +02:00
} catch ( final IOException ee ) {
2011-09-14 23:19:02 +02:00
Log . logSevere ( " SEEDDB " , " error saving mySeed.txt (1) to " + this . myOwnSeedFile . toString ( ) + " : " + ee . getMessage ( ) , ee ) ;
2009-11-05 21:28:37 +01:00
Log . logException ( ee ) ;
2005-12-13 00:59:58 +01:00
System . exit ( - 1 ) ;
}
2005-06-23 13:00:26 +02:00
} else {
// create new identity
2011-09-14 23:19:02 +02:00
Log . logInfo ( " SEEDDB " , " could not find stored mySeed.txt at " + this . myOwnSeedFile . toString ( ) + " : " + " . creating new seed. " ) ;
2011-10-04 11:06:24 +02:00
this . mySeed = Seed . genLocalSeed ( this ) ;
2005-12-13 00:59:58 +01:00
try {
2011-09-14 23:19:02 +02:00
this . mySeed . save ( this . myOwnSeedFile ) ;
2008-08-02 14:12:04 +02:00
} catch ( final IOException ee ) {
2011-09-14 23:19:02 +02:00
Log . logSevere ( " SEEDDB " , " error saving mySeed.txt (2) to " + this . myOwnSeedFile . toString ( ) + " : " + ee . getMessage ( ) , ee ) ;
2009-11-05 21:28:37 +01:00
Log . logException ( ee ) ;
2005-12-13 00:59:58 +01:00
System . exit ( - 1 ) ;
}
2005-06-23 13:00:26 +02:00
}
2011-04-04 01:39:45 +02:00
this . myBotIDs . add ( this . mySeed . getName ( ) + " .yacy " ) ;
this . myBotIDs . add ( this . mySeed . hash + " .yacyh " ) ;
2011-09-14 23:19:02 +02:00
this . mySeed . setIP ( " " ) ; // we delete the old information to see what we have now
2011-10-04 11:06:24 +02:00
this . mySeed . put ( Seed . PEERTYPE , Seed . PEERTYPE_VIRGIN ) ; // markup startup condition
2007-10-01 14:30:23 +02:00
}
2011-04-04 01:39:45 +02:00
public Set < String > myBotIDs ( ) {
return this . myBotIDs ;
}
2011-09-14 23:19:02 +02:00
replaced old DHT transmission method with new method. Many things have changed! some of them:
- after a index selection is made, the index is splitted into its vertical components
- from differrent index selctions the splitted components can be accumulated before they are placed into the transmission queue
- each splitted chunk gets its own transmission thread
- multiple transmission threads are started concurrently
- the process can be monitored with the blocking queue servlet
To implement that, a new package de.anomic.yacy.dht was created. Some old files have been removed.
The new index distribution model using a vertical DHT was implemented. An abstraction of this model
is implemented in the new dht package as interface. The freeworld network has now a configuration
of two vertial partitions; sixteen partitions are planned and will be configured if the process is bug-free.
This modification has three main targets:
- enhance the DHT transmission speed
- with a vertical DHT, a search will speed up. With two partitions, two times. With sixteen, sixteen times.
- the vertical DHT will apply a semi-dht for URLs, and peers will receive a fraction of the overall URLs they received before.
with two partitions, the fractions will be halve. With sixteen partitions, a 1/16 of the previous number of URLs.
BE CAREFULL, THIS IS A MAJOR CODE CHANGE, POSSIBLY FULL OF BUGS AND HARMFUL THINGS.
git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@5586 6c8d7289-2bf4-0310-a012-ef5d649a1542
2009-02-10 01:06:59 +01:00
public int redundancy ( ) {
if ( this . mySeed . isJunior ( ) ) return 1 ;
return this . netRedundancy ;
}
2011-09-14 23:19:02 +02:00
2007-10-01 14:30:23 +02:00
public boolean mySeedIsDefined ( ) {
return this . mySeed ! = null ;
}
2011-09-14 23:19:02 +02:00
2011-10-04 11:06:24 +02:00
public Seed mySeed ( ) {
2007-10-01 14:30:23 +02:00
if ( this . mySeed = = null ) {
2011-09-14 23:19:02 +02:00
if ( sizeConnected ( ) = = 0 ) try { Thread . sleep ( 5000 ) ; } catch ( final InterruptedException e ) { } // wait for init
2007-10-01 14:30:23 +02:00
initMySeed ( ) ;
2011-04-29 12:58:12 +02:00
// check if my seed has an IP assigned
2011-09-14 23:19:02 +02:00
if ( myIP ( ) = = null | | myIP ( ) . length ( ) = = 0 ) {
2011-04-29 12:58:12 +02:00
this . mySeed . setIP ( Domains . myPublicLocalIP ( ) . getHostAddress ( ) ) ;
}
2007-10-01 14:30:23 +02:00
}
return this . mySeed ;
2005-04-07 21:19:42 +02:00
}
2011-09-14 23:19:02 +02:00
public void setMyName ( final String name ) {
2011-04-04 01:39:45 +02:00
this . myBotIDs . remove ( this . mySeed . getName ( ) + " .yacy " ) ;
this . mySeed . setName ( name ) ;
this . myBotIDs . add ( name + " .yacy " ) ;
}
2011-09-14 23:19:02 +02:00
2011-12-15 15:15:53 +01:00
@Override
2008-05-06 01:13:47 +02:00
public String myAlternativeAddress ( ) {
return mySeed ( ) . getName ( ) + " .yacy " ;
}
2011-09-14 23:19:02 +02:00
2011-12-15 15:15:53 +01:00
@Override
2008-05-06 01:13:47 +02:00
public String myIP ( ) {
return mySeed ( ) . getIP ( ) ;
}
2011-09-14 23:19:02 +02:00
2011-12-15 15:15:53 +01:00
@Override
2008-05-06 01:13:47 +02:00
public int myPort ( ) {
return mySeed ( ) . getPort ( ) ;
}
2011-09-14 23:19:02 +02:00
2011-12-15 15:15:53 +01:00
@Override
2008-05-06 01:13:47 +02:00
public String myName ( ) {
2011-09-14 23:19:02 +02:00
return this . mySeed . getName ( ) ;
2008-05-06 01:13:47 +02:00
}
2011-09-14 23:19:02 +02:00
2011-12-15 15:15:53 +01:00
@Override
2008-05-06 01:13:47 +02:00
public String myID ( ) {
2011-09-14 23:19:02 +02:00
return this . mySeed . hash ;
2008-05-06 01:13:47 +02:00
}
2011-09-14 23:19:02 +02:00
2005-04-07 21:19:42 +02:00
public synchronized void removeMySeed ( ) {
2011-09-14 23:19:02 +02:00
if ( this . seedActiveDB . isEmpty ( ) & & this . seedPassiveDB . isEmpty ( ) & & this . seedPotentialDB . isEmpty ( ) ) return ; // avoid that the own seed is initialized too early
2007-10-01 14:30:23 +02:00
if ( this . mySeed = = null ) initMySeed ( ) ;
2005-04-07 21:19:42 +02:00
try {
2011-09-14 23:19:02 +02:00
final byte [ ] mySeedHash = ASCII . getBytes ( this . mySeed . hash ) ;
this . seedActiveDB . delete ( mySeedHash ) ;
this . seedPassiveDB . delete ( mySeedHash ) ;
this . seedPotentialDB . delete ( mySeedHash ) ;
2009-01-31 00:33:47 +01:00
} catch ( final IOException e ) { Log . logWarning ( " yacySeedDB " , " could not remove hash ( " + e . getClass ( ) + " ): " + e . getMessage ( ) ) ; }
2005-04-07 21:19:42 +02:00
}
2011-09-14 23:19:02 +02:00
2011-03-20 00:52:09 +01:00
public void saveMySeed ( ) {
2008-05-14 22:30:44 +02:00
try {
2011-09-14 23:19:02 +02:00
mySeed ( ) . save ( this . myOwnSeedFile ) ;
} catch ( final IOException e ) { Log . logWarning ( " yacySeedDB " , " could not save mySeed ' " + this . myOwnSeedFile + " ': " + e . getMessage ( ) ) ; }
2008-05-14 22:30:44 +02:00
}
2011-09-14 23:19:02 +02:00
2007-08-27 00:35:26 +02:00
public boolean noDHTActivity ( ) {
// for small networks, we don't perform DHT transmissions, because it is possible to search over all peers
2011-09-14 23:19:02 +02:00
return sizeConnected ( ) < = dhtActivityMagic ;
2007-08-27 00:35:26 +02:00
}
2011-09-14 23:19:02 +02:00
2009-01-30 23:44:20 +01:00
private synchronized MapDataMining openSeedTable ( final File seedDBFile ) {
2011-09-14 23:19:02 +02:00
final File parentDir = new File ( seedDBFile . getParent ( ) ) ;
2008-08-06 21:43:12 +02:00
if ( ! parentDir . exists ( ) ) {
if ( ! parentDir . mkdirs ( ) )
2009-01-31 00:33:47 +01:00
Log . logWarning ( " yacySeedDB " , " could not create directories for " + seedDBFile . getParent ( ) ) ;
2008-08-06 21:43:12 +02:00
}
2006-11-07 02:09:02 +01:00
try {
2010-05-15 01:50:07 +02:00
return new MapDataMining ( seedDBFile , Word . commonHashLength , Base64Order . enhancedCoder , 1024 * 512 , 500 , sortFields , longaccFields , doubleaccFields , this ) ;
2008-08-02 14:12:04 +02:00
} catch ( final Exception e ) {
2006-11-07 02:09:02 +01:00
// try again
2009-03-30 17:31:25 +02:00
FileUtils . deletedelete ( seedDBFile ) ;
2008-10-10 16:40:02 +02:00
try {
2010-05-15 01:50:07 +02:00
return new MapDataMining ( seedDBFile , Word . commonHashLength , Base64Order . enhancedCoder , 1024 * 512 , 500 , sortFields , longaccFields , doubleaccFields , this ) ;
2011-09-14 23:19:02 +02:00
} catch ( final IOException e1 ) {
2009-11-05 21:28:37 +01:00
Log . logException ( e1 ) ;
2008-10-10 16:40:02 +02:00
System . exit ( - 1 ) ;
return null ;
}
2006-11-07 02:09:02 +01:00
}
2005-04-07 21:19:42 +02:00
}
2011-09-14 23:19:02 +02:00
2010-04-19 18:42:37 +02:00
private synchronized MapDataMining resetSeedTable ( MapDataMining seedDB , final File seedDBFile ) {
2005-04-07 21:19:42 +02:00
// this is an emergency function that should only be used if any problem with the
// seed.db is detected
2011-10-04 11:06:24 +02:00
Network . log . logWarning ( " seed-db " + seedDBFile . toString ( ) + " reset (on-the-fly) " ) ;
2007-03-09 09:48:47 +01:00
seedDB . close ( ) ;
2009-03-30 17:31:25 +02:00
FileUtils . deletedelete ( seedDBFile ) ;
if ( seedDBFile . exists ( ) )
2009-01-31 00:33:47 +01:00
Log . logWarning ( " yacySeedDB " , " could not delete file " + seedDBFile ) ;
2007-03-09 09:48:47 +01:00
// create new seed database
seedDB = openSeedTable ( seedDBFile ) ;
2005-04-07 21:19:42 +02:00
return seedDB ;
}
2011-09-14 23:19:02 +02:00
public synchronized void resetActiveTable ( ) { this . seedActiveDB = resetSeedTable ( this . seedActiveDB , this . seedActiveDBFile ) ; }
private synchronized void resetPassiveTable ( ) { this . seedPassiveDB = resetSeedTable ( this . seedPassiveDB , this . seedPassiveDBFile ) ; }
private synchronized void resetPotentialTable ( ) { this . seedPotentialDB = resetSeedTable ( this . seedPotentialDB , this . seedPotentialDBFile ) ; }
2005-04-07 21:19:42 +02:00
public void close ( ) {
2011-09-14 23:19:02 +02:00
if ( this . seedActiveDB ! = null ) this . seedActiveDB . close ( ) ;
if ( this . seedPassiveDB ! = null ) this . seedPassiveDB . close ( ) ;
if ( this . seedPotentialDB ! = null ) this . seedPotentialDB . close ( ) ;
this . newsPool . close ( ) ;
this . peerActions . close ( ) ;
2005-04-07 21:19:42 +02:00
}
2011-09-14 23:19:02 +02:00
2011-10-04 11:06:24 +02:00
public Iterator < Seed > seedsSortedConnected ( final boolean up , final String field ) {
2005-04-07 21:19:42 +02:00
// enumerates seed-type objects: all seeds sequentially ordered by field
2011-09-14 23:19:02 +02:00
return new seedEnum ( up , field , this . seedActiveDB ) ;
2005-04-07 21:19:42 +02:00
}
2011-09-14 23:19:02 +02:00
2011-10-04 11:06:24 +02:00
public Iterator < Seed > seedsSortedDisconnected ( final boolean up , final String field ) {
2005-04-07 21:19:42 +02:00
// enumerates seed-type objects: all seeds sequentially ordered by field
2011-09-14 23:19:02 +02:00
return new seedEnum ( up , field , this . seedPassiveDB ) ;
2005-04-07 21:19:42 +02:00
}
2011-09-14 23:19:02 +02:00
2011-10-04 11:06:24 +02:00
public Iterator < Seed > seedsSortedPotential ( final boolean up , final String field ) {
2005-04-07 21:19:42 +02:00
// enumerates seed-type objects: all seeds sequentially ordered by field
2011-09-14 23:19:02 +02:00
return new seedEnum ( up , field , this . seedPotentialDB ) ;
2005-04-07 21:19:42 +02:00
}
2011-09-14 23:19:02 +02:00
2009-04-16 17:29:00 +02:00
public TreeMap < byte [ ] , String > /* peer-b64-hashes/ipport */ clusterHashes ( final String clusterdefinition ) {
2007-04-26 11:51:51 +02:00
// collects seeds according to cluster definition string, which consists of
// comma-separated .yacy or .yacyh-domains
2007-04-30 00:05:34 +02:00
// the domain may be extended by an alternative address specification of the form
// <ip> or <ip>:<port>. The port must be identical to the port specified in the peer seed,
// therefore it is optional. The address specification is separated by a '='; the complete
// address has therefore the form
// address ::= (<peername>'.yacy'|<peerhexhash>'.yacyh'){'='<ip>{':'<port}}
// clusterdef ::= {address}{','address}*
2008-08-02 14:12:04 +02:00
final String [ ] addresses = ( clusterdefinition . length ( ) = = 0 ) ? new String [ 0 ] : clusterdefinition . split ( " , " ) ;
2009-04-16 17:29:00 +02:00
final TreeMap < byte [ ] , String > clustermap = new TreeMap < byte [ ] , String > ( Base64Order . enhancedCoder ) ;
2011-10-04 11:06:24 +02:00
Seed seed ;
2007-04-30 00:05:34 +02:00
String hash , yacydom , ipport ;
int p ;
2011-09-14 23:19:02 +02:00
for ( final String addresse : addresses ) {
p = addresse . indexOf ( '=' ) ;
2007-04-30 00:05:34 +02:00
if ( p > = 0 ) {
2011-09-14 23:19:02 +02:00
yacydom = addresse . substring ( 0 , p ) ;
ipport = addresse . substring ( p + 1 ) ;
2007-04-30 00:05:34 +02:00
} else {
2011-09-14 23:19:02 +02:00
yacydom = addresse ;
2007-04-30 00:05:34 +02:00
ipport = null ;
}
if ( yacydom . endsWith ( " .yacyh " ) ) {
2007-04-26 11:51:51 +02:00
// find a peer with its hexhash
2011-10-04 11:06:24 +02:00
hash = Seed . hexHash2b64Hash ( yacydom . substring ( 0 , yacydom . length ( ) - 6 ) ) ;
2007-04-26 11:51:51 +02:00
seed = get ( hash ) ;
if ( seed = = null ) {
2011-10-04 11:06:24 +02:00
Network . log . logWarning ( " cluster peer ' " + yacydom + " ' was not found. " ) ;
2007-04-26 11:51:51 +02:00
} else {
2011-05-27 10:24:54 +02:00
clustermap . put ( ASCII . getBytes ( hash ) , ipport ) ;
2007-04-26 11:51:51 +02:00
}
2007-04-30 00:05:34 +02:00
} else if ( yacydom . endsWith ( " .yacy " ) ) {
2007-04-26 11:51:51 +02:00
// find a peer with its name
2007-04-30 00:05:34 +02:00
seed = lookupByName ( yacydom . substring ( 0 , yacydom . length ( ) - 5 ) ) ;
2007-04-26 11:51:51 +02:00
if ( seed = = null ) {
2011-10-04 11:06:24 +02:00
Network . log . logWarning ( " cluster peer ' " + yacydom + " ' was not found. " ) ;
2007-04-26 11:51:51 +02:00
} else {
2011-05-27 10:24:54 +02:00
clustermap . put ( ASCII . getBytes ( seed . hash ) , ipport ) ;
2007-04-26 11:51:51 +02:00
}
} else {
2011-10-04 11:06:24 +02:00
Network . log . logWarning ( " cluster peer ' " + addresse + " ' has wrong syntax. the name must end with .yacy or .yacyh " ) ;
2007-04-26 11:51:51 +02:00
}
}
2007-04-30 00:05:34 +02:00
return clustermap ;
2007-04-26 11:51:51 +02:00
}
2011-09-14 23:19:02 +02:00
2011-10-04 11:06:24 +02:00
public Iterator < Seed > seedsConnected ( final boolean up , final boolean rot , final byte [ ] firstHash , final float minVersion ) {
2005-04-07 21:19:42 +02:00
// enumerates seed-type objects: all seeds sequentially without order
2011-09-14 23:19:02 +02:00
return new seedEnum ( up , rot , ( firstHash = = null ) ? null : firstHash , null , this . seedActiveDB , minVersion ) ;
2005-04-07 21:19:42 +02:00
}
2011-09-14 23:19:02 +02:00
2011-10-04 11:06:24 +02:00
private Iterator < Seed > seedsDisconnected ( final boolean up , final boolean rot , final byte [ ] firstHash , final float minVersion ) {
2005-04-07 21:19:42 +02:00
// enumerates seed-type objects: all seeds sequentially without order
2011-09-14 23:19:02 +02:00
return new seedEnum ( up , rot , ( firstHash = = null ) ? null : firstHash , null , this . seedPassiveDB , minVersion ) ;
2005-04-07 21:19:42 +02:00
}
2011-09-14 23:19:02 +02:00
2011-10-04 11:06:24 +02:00
public Seed anySeedVersion ( final float minVersion ) {
2005-11-11 00:48:20 +01:00
// return just any seed that has a specific minimum version number
2011-10-04 11:06:24 +02:00
final Iterator < Seed > e = seedsConnected ( true , true , Seed . randomHash ( ) , minVersion ) ;
2008-06-06 18:01:27 +02:00
return e . next ( ) ;
2005-04-07 21:19:42 +02:00
}
2010-05-22 01:27:32 +02:00
/ * *
* count the number of peers that had been seed within the time limit
* @param limit the time limit in minutes . 1440 minutes is a day
* @return the number of peers seen in the given time
* /
2011-09-14 23:19:02 +02:00
public int sizeActiveSince ( final long limit ) {
int c = this . seedActiveDB . size ( ) ;
2011-10-04 11:06:24 +02:00
Seed seed ;
Iterator < Seed > i = seedsSortedDisconnected ( false , Seed . LASTSEEN ) ;
2010-05-22 01:27:32 +02:00
while ( i . hasNext ( ) ) {
seed = i . next ( ) ;
if ( seed ! = null ) {
if ( Math . abs ( ( System . currentTimeMillis ( ) - seed . getLastSeenUTC ( ) ) / 1000 / 60 ) > limit ) break ;
c + + ;
}
}
2011-10-04 11:06:24 +02:00
i = seedsSortedPotential ( false , Seed . LASTSEEN ) ;
2010-05-22 01:27:32 +02:00
while ( i . hasNext ( ) ) {
seed = i . next ( ) ;
if ( seed ! = null ) {
if ( Math . abs ( ( System . currentTimeMillis ( ) - seed . getLastSeenUTC ( ) ) / 1000 / 60 ) > limit ) break ;
c + + ;
}
}
return c ;
}
2011-09-14 23:19:02 +02:00
2010-05-22 01:27:32 +02:00
public int sizeConnected ( ) {
2011-09-14 23:19:02 +02:00
return this . seedActiveDB . size ( ) ;
2005-04-07 21:19:42 +02:00
}
2011-09-14 23:19:02 +02:00
2005-04-07 21:19:42 +02:00
public int sizeDisconnected ( ) {
2011-09-14 23:19:02 +02:00
return this . seedPassiveDB . size ( ) ;
2005-04-07 21:19:42 +02:00
}
2011-09-14 23:19:02 +02:00
2005-04-07 21:19:42 +02:00
public int sizePotential ( ) {
2011-09-14 23:19:02 +02:00
return this . seedPotentialDB . size ( ) ;
2005-04-07 21:19:42 +02:00
}
2011-09-14 23:19:02 +02:00
2011-10-04 11:06:24 +02:00
public long countActiveURL ( ) { return this . seedActiveDB . getLongAcc ( Seed . LCOUNT ) ; }
public long countActiveRWI ( ) { return this . seedActiveDB . getLongAcc ( Seed . ICOUNT ) ; }
public long countActivePPM ( ) { return this . seedActiveDB . getLongAcc ( Seed . ISPEED ) ; }
public float countActiveQPM ( ) { return this . seedActiveDB . getFloatAcc ( Seed . RSPEED ) ; }
public long countPassiveURL ( ) { return this . seedPassiveDB . getLongAcc ( Seed . LCOUNT ) ; }
public long countPassiveRWI ( ) { return this . seedPassiveDB . getLongAcc ( Seed . ICOUNT ) ; }
public long countPotentialURL ( ) { return this . seedPotentialDB . getLongAcc ( Seed . LCOUNT ) ; }
public long countPotentialRWI ( ) { return this . seedPotentialDB . getLongAcc ( Seed . ICOUNT ) ; }
2011-09-14 23:19:02 +02:00
2011-10-04 11:06:24 +02:00
public void addConnected ( final Seed seed ) {
2008-08-24 13:30:00 +02:00
if ( seed . isProper ( false ) ! = null ) return ;
2005-10-17 17:46:12 +02:00
//seed.put(yacySeed.LASTSEEN, yacyCore.shortFormatter.format(new Date(yacyCore.universalTime())));
2011-09-14 23:19:02 +02:00
synchronized ( this ) {
try {
final ConcurrentMap < String , String > seedPropMap = seed . getMap ( ) ;
this . seedActiveDB . insert ( ASCII . getBytes ( seed . hash ) , seedPropMap ) ;
this . seedPassiveDB . delete ( ASCII . getBytes ( seed . hash ) ) ;
this . seedPotentialDB . delete ( ASCII . getBytes ( seed . hash ) ) ;
} catch ( final Exception e ) {
2011-10-04 11:06:24 +02:00
Network . log . logSevere ( " ERROR add: seed.db corrupt ( " + e . getMessage ( ) + " ); resetting seed.db " , e ) ;
2011-09-14 23:19:02 +02:00
resetActiveTable ( ) ;
2005-10-28 09:15:00 +02:00
}
2005-09-10 07:59:12 +02:00
}
2005-04-07 21:19:42 +02:00
}
2008-08-24 13:30:00 +02:00
2011-10-04 11:06:24 +02:00
protected void addDisconnected ( final Seed seed ) {
2008-08-24 13:30:00 +02:00
if ( seed . isProper ( false ) ! = null ) return ;
2011-09-14 23:19:02 +02:00
synchronized ( this ) {
try {
this . seedActiveDB . delete ( ASCII . getBytes ( seed . hash ) ) ;
this . seedPotentialDB . delete ( ASCII . getBytes ( seed . hash ) ) ;
} catch ( final Exception e ) { Log . logWarning ( " yacySeedDB " , " could not remove hash ( " + e . getClass ( ) + " ): " + e . getMessage ( ) ) ; }
//seed.put(yacySeed.LASTSEEN, yacyCore.shortFormatter.format(new Date(yacyCore.universalTime())));
try {
final ConcurrentMap < String , String > seedPropMap = seed . getMap ( ) ;
this . seedPassiveDB . insert ( ASCII . getBytes ( seed . hash ) , seedPropMap ) ;
} catch ( final Exception e ) {
2011-10-04 11:06:24 +02:00
Network . log . logSevere ( " ERROR add: seed.db corrupt ( " + e . getMessage ( ) + " ); resetting seed.db " , e ) ;
2011-09-14 23:19:02 +02:00
resetPassiveTable ( ) ;
2005-10-28 09:15:00 +02:00
}
}
2005-04-07 21:19:42 +02:00
}
2008-08-24 13:30:00 +02:00
2011-10-04 11:06:24 +02:00
protected void addPotential ( final Seed seed ) {
2008-08-24 13:30:00 +02:00
if ( seed . isProper ( false ) ! = null ) return ;
2011-09-14 23:19:02 +02:00
synchronized ( this ) {
try {
this . seedActiveDB . delete ( ASCII . getBytes ( seed . hash ) ) ;
this . seedPassiveDB . delete ( ASCII . getBytes ( seed . hash ) ) ;
} catch ( final Exception e ) { Log . logWarning ( " yacySeedDB " , " could not remove hash ( " + e . getClass ( ) + " ): " + e . getMessage ( ) ) ; }
//seed.put(yacySeed.LASTSEEN, yacyCore.shortFormatter.format(new Date(yacyCore.universalTime())));
try {
final ConcurrentMap < String , String > seedPropMap = seed . getMap ( ) ;
this . seedPotentialDB . insert ( ASCII . getBytes ( seed . hash ) , seedPropMap ) ;
} catch ( final Exception e ) {
2011-10-04 11:06:24 +02:00
Network . log . logSevere ( " ERROR add: seed.db corrupt ( " + e . getMessage ( ) + " ); resetting seed.db " , e ) ;
2011-09-14 23:19:02 +02:00
resetPotentialTable ( ) ;
2005-10-28 09:15:00 +02:00
}
2008-08-24 13:30:00 +02:00
}
2005-04-07 21:19:42 +02:00
}
2008-08-24 13:30:00 +02:00
2008-08-02 14:12:04 +02:00
public synchronized void removeDisconnected ( final String peerHash ) {
2011-09-14 23:19:02 +02:00
if ( peerHash = = null ) return ;
2007-06-22 11:16:25 +02:00
try {
2011-09-14 23:19:02 +02:00
this . seedPassiveDB . delete ( ASCII . getBytes ( peerHash ) ) ;
2009-01-31 00:33:47 +01:00
} catch ( final IOException e ) { Log . logWarning ( " yacySeedDB " , " could not remove hash ( " + e . getClass ( ) + " ): " + e . getMessage ( ) ) ; }
2007-06-22 11:16:25 +02:00
}
2011-09-14 23:19:02 +02:00
2008-08-02 14:12:04 +02:00
public synchronized void removePotential ( final String peerHash ) {
2011-09-14 23:19:02 +02:00
if ( peerHash = = null ) return ;
2007-06-22 11:16:25 +02:00
try {
2011-09-14 23:19:02 +02:00
this . seedPotentialDB . delete ( ASCII . getBytes ( peerHash ) ) ;
2009-01-31 00:33:47 +01:00
} catch ( final IOException e ) { Log . logWarning ( " yacySeedDB " , " could not remove hash ( " + e . getClass ( ) + " ): " + e . getMessage ( ) ) ; }
2007-06-22 11:16:25 +02:00
}
2011-09-14 23:19:02 +02:00
2010-04-19 18:42:37 +02:00
public boolean hasConnected ( final byte [ ] hash ) {
2011-09-14 23:19:02 +02:00
return this . seedActiveDB . containsKey ( hash ) ;
2005-04-07 21:19:42 +02:00
}
2010-04-19 18:42:37 +02:00
public boolean hasDisconnected ( final byte [ ] hash ) {
2011-09-14 23:19:02 +02:00
return this . seedPassiveDB . containsKey ( hash ) ;
2005-04-07 21:19:42 +02:00
}
2011-09-14 23:19:02 +02:00
2010-04-19 18:42:37 +02:00
public boolean hasPotential ( final byte [ ] hash ) {
2011-09-14 23:19:02 +02:00
return this . seedPotentialDB . containsKey ( hash ) ;
2005-04-07 21:19:42 +02:00
}
2011-09-14 23:19:02 +02:00
2011-10-04 11:06:24 +02:00
private Seed get ( final String hash , final MapDataMining database ) {
2011-06-06 00:58:17 +02:00
if ( hash = = null | | hash . length ( ) = = 0 ) return null ;
2011-09-14 23:19:02 +02:00
if ( ( this . mySeed ! = null ) & & ( hash . equals ( this . mySeed . hash ) ) ) return this . mySeed ;
final ConcurrentHashMap < String , String > entry = new ConcurrentHashMap < String , String > ( ) ;
2008-07-11 09:15:46 +02:00
try {
2011-09-14 23:19:02 +02:00
final Map < String , String > map = database . get ( ASCII . getBytes ( hash ) ) ;
2010-02-15 23:06:13 +01:00
if ( map = = null ) return null ;
entry . putAll ( map ) ;
2008-08-02 14:12:04 +02:00
} catch ( final IOException e ) {
2010-02-15 23:06:13 +01:00
Log . logException ( e ) ;
return null ;
2011-09-14 23:19:02 +02:00
} catch ( final RowSpaceExceededException e ) {
2010-06-15 21:44:05 +02:00
Log . logException ( e ) ;
return null ;
2008-07-11 09:15:46 +02:00
}
2011-10-04 11:06:24 +02:00
return new Seed ( hash , entry ) ;
2005-04-07 21:19:42 +02:00
}
2011-09-14 23:19:02 +02:00
2011-10-04 11:06:24 +02:00
public Seed getConnected ( final String hash ) {
2011-09-14 23:19:02 +02:00
return get ( hash , this . seedActiveDB ) ;
2005-04-07 21:19:42 +02:00
}
2011-10-04 11:06:24 +02:00
public Seed getDisconnected ( final String hash ) {
2011-09-14 23:19:02 +02:00
return get ( hash , this . seedPassiveDB ) ;
2005-04-07 21:19:42 +02:00
}
2011-09-14 23:19:02 +02:00
2011-10-04 11:06:24 +02:00
public Seed getPotential ( final String hash ) {
2011-09-14 23:19:02 +02:00
return get ( hash , this . seedPotentialDB ) ;
2005-08-01 03:12:02 +02:00
}
2011-09-14 23:19:02 +02:00
2011-10-04 11:06:24 +02:00
public Seed get ( final String hash ) {
Seed seed = getConnected ( hash ) ;
2005-08-01 03:12:02 +02:00
if ( seed = = null ) seed = getDisconnected ( hash ) ;
if ( seed = = null ) seed = getPotential ( hash ) ;
return seed ;
}
2011-09-14 23:19:02 +02:00
2011-10-04 11:06:24 +02:00
public void update ( final String hash , final Seed seed ) {
2007-10-01 14:30:23 +02:00
if ( this . mySeed = = null ) initMySeed ( ) ;
2011-09-14 23:19:02 +02:00
if ( hash . equals ( this . mySeed . hash ) ) {
this . mySeed = seed ;
2006-05-19 00:24:51 +02:00
return ;
}
2011-09-14 23:19:02 +02:00
final byte [ ] hashb = ASCII . getBytes ( hash ) ;
2011-10-04 11:06:24 +02:00
Seed s = get ( hash , this . seedActiveDB ) ;
2011-09-14 23:19:02 +02:00
if ( s ! = null ) try { this . seedActiveDB . insert ( hashb , seed . getMap ( ) ) ; return ; } catch ( final Exception e ) { Log . logException ( e ) ; }
s = get ( hash , this . seedPassiveDB ) ;
if ( s ! = null ) try { this . seedPassiveDB . insert ( hashb , seed . getMap ( ) ) ; return ; } catch ( final Exception e ) { Log . logException ( e ) ; }
s = get ( hash , this . seedPotentialDB ) ;
if ( s ! = null ) try { this . seedPotentialDB . insert ( hashb , seed . getMap ( ) ) ; return ; } catch ( final Exception e ) { Log . logException ( e ) ; }
2006-05-19 00:24:51 +02:00
}
2011-09-14 23:19:02 +02:00
2011-10-04 11:06:24 +02:00
public Seed lookupByName ( String peerName ) {
2005-04-07 21:19:42 +02:00
// reads a seed by searching by name
2009-10-05 22:11:41 +02:00
if ( peerName . endsWith ( " .yacy " ) ) peerName = peerName . substring ( 0 , peerName . length ( ) - 5 ) ;
2007-04-26 11:51:51 +02:00
2005-04-07 21:19:42 +02:00
// local peer?
2007-10-01 14:30:23 +02:00
if ( peerName . equals ( " localpeer " ) ) {
if ( this . mySeed = = null ) initMySeed ( ) ;
2011-09-14 23:19:02 +02:00
return this . mySeed ;
2007-10-01 14:30:23 +02:00
}
2011-09-14 23:19:02 +02:00
2011-12-25 10:14:15 +01:00
peerName = peerName . toLowerCase ( ) ;
2011-10-04 11:06:24 +02:00
Seed seed ;
2011-09-14 23:19:02 +02:00
2011-12-25 10:14:15 +01:00
// enumerate the cache
String name = Seed . checkPeerName ( peerName ) ;
2012-01-31 18:17:25 +01:00
synchronized ( this ) { try {
2012-02-02 06:43:57 +01:00
Collection < byte [ ] > idx = this . seedActiveDB . select ( Seed . NAME , name ) ;
for ( byte [ ] pk : idx ) {
seed = this . getConnected ( ASCII . String ( pk ) ) ;
2011-12-25 10:14:15 +01:00
if ( seed = = null ) continue ;
//System.out.println("*** found lookupByName in seedActiveDB: " + peerName);
return seed ;
}
} catch ( IOException e ) {
2012-01-31 18:17:25 +01:00
} }
synchronized ( this ) { try {
2012-02-02 06:43:57 +01:00
Collection < byte [ ] > idx = this . seedPassiveDB . select ( Seed . NAME , name ) ;
for ( byte [ ] pk : idx ) {
seed = this . getDisconnected ( ASCII . String ( pk ) ) ;
2011-12-25 10:14:15 +01:00
if ( seed = = null ) continue ;
//System.out.println("*** found lookupByName in seedPassiveDB: " + peerName);
return seed ;
}
} catch ( IOException e ) {
2012-01-31 18:17:25 +01:00
} }
2005-04-07 21:19:42 +02:00
// check local seed
2007-10-01 14:30:23 +02:00
if ( this . mySeed = = null ) initMySeed ( ) ;
2011-09-14 23:19:02 +02:00
name = this . mySeed . getName ( ) . toLowerCase ( ) ;
if ( name . equals ( peerName ) ) return this . mySeed ;
2005-04-07 21:19:42 +02:00
// nothing found
return null ;
}
2011-09-14 23:19:02 +02:00
2011-10-04 11:06:24 +02:00
public Seed lookupByIP (
2011-09-14 23:19:02 +02:00
final InetAddress peerIP ,
2011-12-21 01:14:43 +01:00
final int port ,
2011-09-14 23:19:02 +02:00
final boolean lookupConnected ,
2008-08-02 14:12:04 +02:00
final boolean lookupDisconnected ,
final boolean lookupPotential
2005-10-12 10:17:43 +02:00
) {
2011-09-14 23:19:02 +02:00
2006-12-20 02:07:49 +01:00
if ( peerIP = = null ) return null ;
2011-09-14 23:19:02 +02:00
2005-10-12 10:17:43 +02:00
// local peer?
2010-11-29 20:18:44 +01:00
if ( Domains . isThisHostIP ( peerIP ) ) {
2007-10-01 14:30:23 +02:00
if ( this . mySeed = = null ) initMySeed ( ) ;
2011-09-14 23:19:02 +02:00
return this . mySeed ;
2007-10-01 14:30:23 +02:00
}
2011-09-14 23:19:02 +02:00
2005-10-12 10:17:43 +02:00
// then try to use the cache
2011-12-21 01:14:43 +01:00
Seed seed = null ;
String ipString = peerIP . getHostAddress ( ) ;
2011-12-25 10:14:15 +01:00
2012-01-31 18:17:25 +01:00
if ( lookupConnected ) synchronized ( this ) {
2011-12-21 01:14:43 +01:00
try {
2012-02-02 06:43:57 +01:00
Collection < byte [ ] > idx = this . seedActiveDB . select ( Seed . IP , ipString ) ;
for ( byte [ ] pk : idx ) {
seed = this . getConnected ( ASCII . String ( pk ) ) ;
2011-12-21 01:14:43 +01:00
if ( seed = = null ) continue ;
2012-02-02 06:43:57 +01:00
if ( seed . getPort ( ) ! = port ) continue ;
2011-12-21 01:14:43 +01:00
//System.out.println("*** found lookupByIP in connected: " + peerIP.toString() + " -> " + seed.getName());
return seed ;
2010-09-14 17:27:27 +02:00
}
2011-12-21 01:14:43 +01:00
} catch ( IOException e ) {
2005-10-12 10:17:43 +02:00
}
}
2011-09-14 23:19:02 +02:00
2012-01-31 18:17:25 +01:00
if ( lookupDisconnected ) synchronized ( this ) {
2011-12-21 01:14:43 +01:00
try {
2012-02-02 06:43:57 +01:00
Collection < byte [ ] > idx = this . seedPassiveDB . select ( Seed . IP , ipString ) ;
for ( byte [ ] pk : idx ) {
seed = this . getDisconnected ( ASCII . String ( pk ) ) ;
2011-12-21 01:14:43 +01:00
if ( seed = = null ) continue ;
2012-02-02 06:43:57 +01:00
if ( seed . getPort ( ) ! = port ) continue ;
2011-12-21 01:14:43 +01:00
//System.out.println("*** found lookupByIP in disconnected: " + peerIP.toString() + " -> " + seed.getName());
return seed ;
2010-09-14 17:27:27 +02:00
}
2011-12-21 01:14:43 +01:00
} catch ( IOException e ) {
2005-10-12 10:17:43 +02:00
}
}
2011-09-14 23:19:02 +02:00
2012-01-31 18:17:25 +01:00
if ( lookupPotential ) synchronized ( this ) {
2011-12-21 01:14:43 +01:00
try {
2012-02-02 06:43:57 +01:00
Collection < byte [ ] > idx = this . seedPotentialDB . select ( Seed . IP , ipString ) ;
for ( byte [ ] pk : idx ) {
seed = this . getPotential ( ASCII . String ( pk ) ) ;
2011-12-21 01:14:43 +01:00
if ( seed = = null ) continue ;
2012-02-02 06:43:57 +01:00
if ( seed . getPort ( ) ! = port ) continue ;
2011-12-21 01:14:43 +01:00
//System.out.println("*** found lookupByIP in potential: " + peerIP.toString() + " -> " + seed.getName());
return seed ;
2010-09-14 17:27:27 +02:00
}
2011-12-21 01:14:43 +01:00
} catch ( IOException e ) {
2005-10-12 10:17:43 +02:00
}
}
2011-09-14 23:19:02 +02:00
2010-09-14 17:27:27 +02:00
// check local seed
if ( this . mySeed = = null ) return null ;
2011-12-21 01:14:43 +01:00
String s = this . mySeed . getIP ( ) ;
if ( s = = null | | ! ipString . equals ( s ) ) return null ;
int p = this . mySeed . getPort ( ) ;
if ( p ! = port ) return null ;
//System.out.println("*** found lookupByIP as my seed: " + peerIP.toString() + " -> " + this.mySeed.getName());
return this . mySeed ;
2005-10-12 10:17:43 +02:00
}
2005-04-07 21:19:42 +02:00
2011-01-27 00:21:33 +01:00
private ArrayList < String > storeSeedList ( final File seedFile , final boolean addMySeed ) throws IOException {
2005-05-17 10:25:04 +02:00
PrintWriter pw = null ;
2011-09-14 23:19:02 +02:00
final ArrayList < String > v = new ArrayList < String > ( this . seedActiveDB . size ( ) + 1 ) ;
2005-05-17 10:25:04 +02:00
try {
2011-09-14 23:19:02 +02:00
2005-05-17 10:25:04 +02:00
pw = new PrintWriter ( new BufferedWriter ( new FileWriter ( seedFile ) ) ) ;
2011-09-14 23:19:02 +02:00
2011-01-27 00:21:33 +01:00
// store own peer seed
2005-05-17 10:25:04 +02:00
String line ;
2007-10-01 14:30:23 +02:00
if ( this . mySeed = = null ) initMySeed ( ) ;
if ( addMySeed ) {
2011-09-14 23:19:02 +02:00
line = this . mySeed . genSeedStr ( null ) ;
2005-05-17 10:25:04 +02:00
v . add ( line ) ;
2007-12-14 20:17:54 +01:00
pw . print ( line + serverCore . CRLF_STRING ) ;
2005-05-17 10:25:04 +02:00
}
2011-09-14 23:19:02 +02:00
2011-01-27 00:21:33 +01:00
// store active peer seeds
2011-10-04 11:06:24 +02:00
Seed ys ;
Iterator < Seed > se = seedsConnected ( true , false , null , ( float ) 0 . 0 ) ;
2007-10-01 14:30:23 +02:00
while ( se . hasNext ( ) ) {
2008-06-06 18:01:27 +02:00
ys = se . next ( ) ;
2005-05-17 10:25:04 +02:00
if ( ys ! = null ) {
line = ys . genSeedStr ( null ) ;
v . add ( line ) ;
2007-12-14 20:17:54 +01:00
pw . print ( line + serverCore . CRLF_STRING ) ;
2005-05-17 10:25:04 +02:00
}
}
2011-09-14 23:19:02 +02:00
2011-01-27 00:21:33 +01:00
// store some of the not-so-old passive peer seeds (limit: 1 day)
se = seedsDisconnected ( true , false , null , ( float ) 0 . 0 ) ;
2011-09-14 23:19:02 +02:00
final long timeout = System . currentTimeMillis ( ) - ( 1000L * 60L * 60L * 24L ) ;
2011-01-27 00:21:33 +01:00
while ( se . hasNext ( ) ) {
ys = se . next ( ) ;
if ( ys ! = null ) {
2011-09-14 23:19:02 +02:00
if ( ys . getLastSeenUTC ( ) < timeout ) continue ;
2011-01-27 00:21:33 +01:00
line = ys . genSeedStr ( null ) ;
v . add ( line ) ;
pw . print ( line + serverCore . CRLF_STRING ) ;
}
}
2005-05-17 10:25:04 +02:00
pw . flush ( ) ;
} finally {
2008-08-02 14:12:04 +02:00
if ( pw ! = null ) try { pw . close ( ) ; } catch ( final Exception e ) { }
2005-05-17 10:25:04 +02:00
}
return v ;
2005-04-07 21:19:42 +02:00
}
2011-09-14 23:19:02 +02:00
protected String uploadSeedList ( final yacySeedUploader uploader ,
2009-07-17 15:59:21 +02:00
final serverSwitch sb ,
2011-10-04 11:06:24 +02:00
final SeedDB seedDB ,
2009-10-11 02:12:19 +02:00
final DigestURI seedURL ) throws Exception {
2011-09-14 23:19:02 +02:00
2005-05-17 10:25:04 +02:00
// upload a seed file, if possible
if ( seedURL = = null ) throw new NullPointerException ( " UPLOAD - Error: URL not given " ) ;
2011-09-14 23:19:02 +02:00
String log = null ;
2005-05-17 10:25:04 +02:00
File seedFile = null ;
2011-09-14 23:19:02 +02:00
try {
// create a seed file which for uploading ...
2009-07-19 22:37:44 +02:00
seedFile = File . createTempFile ( " seedFile " , " .txt " , seedDB . myOwnSeedFile . getParentFile ( ) ) ;
2005-10-24 10:54:46 +02:00
seedFile . deleteOnExit ( ) ;
2009-01-31 00:33:47 +01:00
if ( Log . isFine ( " YACY " ) ) Log . logFine ( " YACY " , " SaveSeedList: Storing seedlist into tempfile " + seedFile . toString ( ) ) ;
2011-09-14 23:19:02 +02:00
final ArrayList < String > uv = storeSeedList ( seedFile , true ) ;
2005-05-17 10:25:04 +02:00
// uploading the seed file
2009-01-31 00:33:47 +01:00
if ( Log . isFine ( " YACY " ) ) Log . logFine ( " YACY " , " SaveSeedList: Trying to upload seed-file, " + seedFile . length ( ) + " bytes, " + uv . size ( ) + " entries. " ) ;
2011-01-27 00:21:33 +01:00
log = uploader . uploadSeedFile ( sb , seedFile ) ;
2011-09-14 23:19:02 +02:00
2006-10-19 12:10:53 +02:00
// test download
2009-01-31 00:33:47 +01:00
if ( Log . isFine ( " YACY " ) ) Log . logFine ( " YACY " , " SaveSeedList: Trying to download seed-file ' " + seedURL + " '. " ) ;
2009-10-05 22:11:41 +02:00
final Iterator < String > check = downloadSeedFile ( seedURL ) ;
2011-09-14 23:19:02 +02:00
2006-10-19 12:10:53 +02:00
// Comparing if local copy and uploaded copy are equal
2008-08-02 14:12:04 +02:00
final String errorMsg = checkCache ( uv , check ) ;
2006-10-19 12:10:53 +02:00
if ( errorMsg = = null )
2007-12-14 20:17:54 +01:00
log = log + " UPLOAD CHECK - Success: the result vectors are equal " + serverCore . CRLF_STRING ;
2005-05-23 12:10:51 +02:00
else {
2007-12-14 20:17:54 +01:00
throw new Exception ( " UPLOAD CHECK - Error: the result vector is different. " + errorMsg + serverCore . CRLF_STRING ) ;
2005-05-23 12:10:51 +02:00
}
2005-05-17 10:25:04 +02:00
} finally {
2008-08-06 21:43:12 +02:00
if ( seedFile ! = null )
try {
2009-03-30 17:31:25 +02:00
FileUtils . deletedelete ( seedFile ) ;
2008-08-06 21:43:12 +02:00
} catch ( final Exception e ) {
/* ignore this */
}
2005-05-17 10:25:04 +02:00
}
2011-09-14 23:19:02 +02:00
2005-05-17 10:25:04 +02:00
return log ;
2005-04-07 21:19:42 +02:00
}
2011-09-14 23:19:02 +02:00
2009-10-11 02:12:19 +02:00
private Iterator < String > downloadSeedFile ( final DigestURI seedURL ) throws IOException {
2008-04-05 15:17:16 +02:00
// Configure http headers
2009-07-19 22:37:44 +02:00
final RequestHeader reqHeader = new RequestHeader ( ) ;
reqHeader . put ( HeaderFramework . PRAGMA , " no-cache " ) ;
reqHeader . put ( HeaderFramework . CACHE_CONTROL , " no-cache " ) ; // httpc uses HTTP/1.0 is this necessary?
2011-04-26 15:35:29 +02:00
reqHeader . put ( HeaderFramework . USER_AGENT , ClientIdentification . getUserAgent ( ) ) ;
2011-09-14 23:19:02 +02:00
2010-08-23 00:32:39 +02:00
final HTTPClient client = new HTTPClient ( ) ;
2010-07-23 01:08:37 +02:00
client . setHeader ( reqHeader . entrySet ( ) ) ;
2008-04-05 15:17:16 +02:00
byte [ ] content = null ;
2006-10-19 12:10:53 +02:00
try {
// send request
2011-04-21 15:58:49 +02:00
content = client . GETbytes ( seedURL ) ;
2010-07-23 01:08:37 +02:00
} catch ( final Exception e ) {
throw new IOException ( " Unable to download seed file ' " + seedURL + " '. " + e . getMessage ( ) ) ;
}
2011-09-14 23:19:02 +02:00
2010-07-23 01:08:37 +02:00
// check response code
if ( client . getHttpResponse ( ) . getStatusLine ( ) . getStatusCode ( ) ! = 200 ) {
throw new IOException ( " Server returned status: " + client . getHttpResponse ( ) . getStatusLine ( ) ) ;
2008-04-05 15:17:16 +02:00
}
2011-09-14 23:19:02 +02:00
2008-04-05 15:17:16 +02:00
try {
2006-10-19 12:10:53 +02:00
// uncompress it if it is gzipped
2009-01-31 02:06:56 +01:00
content = FileUtils . uncompressGZipArray ( content ) ;
2006-10-19 12:10:53 +02:00
// convert it into an array
2009-10-05 22:11:41 +02:00
return FileUtils . strings ( content ) ;
2008-08-02 14:12:04 +02:00
} catch ( final Exception e ) {
2010-07-23 01:08:37 +02:00
throw new IOException ( " Unable to uncompress seed file ' " + seedURL + " '. " + e . getMessage ( ) ) ;
2007-09-23 22:49:52 +02:00
}
2006-10-19 12:10:53 +02:00
}
2005-04-07 21:19:42 +02:00
2011-09-14 23:19:02 +02:00
private String checkCache ( final ArrayList < String > uv , final Iterator < String > check ) {
2009-10-05 22:11:41 +02:00
if ( ( check = = null ) | | ( uv = = null ) ) {
if ( Log . isFine ( " YACY " ) ) Log . logFine ( " YACY " , " SaveSeedList: Local and uploades seed-list are different " ) ;
return " Entry count is different: uv.size() = " + ( ( uv = = null ) ? " null " : Integer . toString ( uv . size ( ) ) ) ;
}
2011-09-14 23:19:02 +02:00
2009-01-31 00:33:47 +01:00
if ( Log . isFine ( " YACY " ) ) Log . logFine ( " YACY " , " SaveSeedList: Comparing local and uploades seed-list entries ... " ) ;
2009-10-05 22:11:41 +02:00
int i = 0 ;
while ( check . hasNext ( ) & & i < uv . size ( ) ) {
if ( ! ( ( uv . get ( i ) ) . equals ( check . next ( ) ) ) ) return " Element at position " + i + " is different. " ;
i + + ;
2005-04-07 21:19:42 +02:00
}
2011-09-14 23:19:02 +02:00
2006-10-19 12:10:53 +02:00
// no difference found
return null ;
2005-04-07 21:19:42 +02:00
}
2008-05-06 01:13:47 +02:00
/ * *
* resolve a yacy address
* /
public String resolve ( String host ) {
2011-10-04 11:06:24 +02:00
Seed seed ;
2005-04-07 21:19:42 +02:00
int p ;
String subdom = null ;
if ( host . endsWith ( " .yacyh " ) ) {
// this is not functional at the moment
// caused by lowecasing of hashes at the browser client
2010-01-13 01:23:07 +01:00
p = host . indexOf ( '.' ) ;
2005-04-07 21:19:42 +02:00
if ( ( p > 0 ) & & ( p ! = ( host . length ( ) - 6 ) ) ) {
subdom = host . substring ( 0 , p ) ;
host = host . substring ( p + 1 ) ;
}
2006-02-15 23:12:53 +01:00
// check if we have a b64-hash or a hex-hash
String hash = host . substring ( 0 , host . length ( ) - 6 ) ;
2009-10-11 02:12:19 +02:00
if ( hash . length ( ) > Word . commonHashLength ) {
2006-02-15 23:12:53 +01:00
// this is probably a hex-hash
2011-10-04 11:06:24 +02:00
hash = Seed . hexHash2b64Hash ( hash ) ;
2006-02-15 23:12:53 +01:00
}
2005-04-07 21:19:42 +02:00
// check remote seeds
2006-02-15 23:12:53 +01:00
seed = getConnected ( hash ) ; // checks only remote, not local
2005-04-07 21:19:42 +02:00
// check local seed
if ( seed = = null ) {
2007-10-01 14:30:23 +02:00
if ( this . mySeed = = null ) initMySeed ( ) ;
2011-09-14 23:19:02 +02:00
if ( hash . equals ( this . mySeed . hash ) )
seed = this . mySeed ;
2005-04-07 21:19:42 +02:00
else return null ;
}
2007-04-30 00:05:34 +02:00
return seed . getPublicAddress ( ) + ( ( subdom = = null ) ? " " : ( " / " + subdom ) ) ;
2005-04-07 21:19:42 +02:00
} else if ( host . endsWith ( " .yacy " ) ) {
// identify subdomain
2010-01-13 01:23:07 +01:00
p = host . indexOf ( '.' ) ;
2005-04-07 21:19:42 +02:00
if ( ( p > 0 ) & & ( p ! = ( host . length ( ) - 5 ) ) ) {
subdom = host . substring ( 0 , p ) ; // no double-dot attack possible, the subdom cannot have ".." in it
host = host . substring ( p + 1 ) ; // if ever, the double-dots are here but do not harm
}
// identify domain
2008-08-02 14:12:04 +02:00
final String domain = host . substring ( 0 , host . length ( ) - 5 ) . toLowerCase ( ) ;
2005-04-07 21:19:42 +02:00
seed = lookupByName ( domain ) ;
if ( seed = = null ) return null ;
2007-10-01 14:30:23 +02:00
if ( this . mySeed = = null ) initMySeed ( ) ;
2011-09-14 23:19:02 +02:00
if ( ( seed = = this . mySeed ) & & ( ! ( seed . isOnline ( ) ) ) ) {
2005-04-07 21:19:42 +02:00
// take local ip instead of external
2011-01-28 11:54:13 +01:00
return Switchboard . getSwitchboard ( ) . myPublicIP ( ) + " : " + Switchboard . getSwitchboard ( ) . getConfig ( " port " , " 8090 " ) + ( ( subdom = = null ) ? " " : ( " / " + subdom ) ) ;
2005-04-07 21:19:42 +02:00
}
2007-04-30 00:05:34 +02:00
return seed . getPublicAddress ( ) + ( ( subdom = = null ) ? " " : ( " / " + subdom ) ) ;
2005-04-07 21:19:42 +02:00
} else {
return null ;
}
}
2011-05-18 16:26:28 +02:00
public String targetAddress ( final String targetHash ) {
2011-09-14 23:19:02 +02:00
// find target address
2011-05-18 16:26:28 +02:00
String address ;
2011-09-14 23:19:02 +02:00
if ( targetHash . equals ( mySeed ( ) . hash ) ) {
address = mySeed ( ) . getClusterAddress ( ) ;
2011-05-18 16:26:28 +02:00
} else {
2011-10-04 11:06:24 +02:00
final Seed targetSeed = getConnected ( targetHash ) ;
2011-05-18 16:26:28 +02:00
if ( targetSeed = = null ) { return null ; }
address = targetSeed . getClusterAddress ( ) ;
}
if ( address = = null ) address = " localhost:8090 " ;
return address ;
}
2011-09-14 23:19:02 +02:00
2011-10-04 11:06:24 +02:00
private class seedEnum implements Iterator < Seed > {
2011-09-14 23:19:02 +02:00
2011-12-15 15:15:53 +01:00
private Iterator < Map . Entry < byte [ ] , Map < String , String > > > it ;
2011-10-04 11:06:24 +02:00
private Seed nextSeed ;
2011-09-14 23:19:02 +02:00
private final MapDataMining database ;
2010-04-19 18:42:37 +02:00
private float minVersion ;
2011-09-14 23:19:02 +02:00
2010-04-19 18:42:37 +02:00
private seedEnum ( final boolean up , final boolean rot , final byte [ ] firstKey , final byte [ ] secondKey , final MapDataMining database , final float minVersion ) {
2005-04-07 21:19:42 +02:00
this . database = database ;
2006-11-20 03:46:53 +01:00
this . minVersion = minVersion ;
2005-10-24 16:07:43 +02:00
try {
2011-12-15 15:15:53 +01:00
this . it = ( firstKey = = null ) ? database . entries ( up , rot ) : database . entries ( up , rot , firstKey , secondKey ) ;
2010-06-30 10:42:29 +02:00
float version ;
2006-11-20 03:46:53 +01:00
while ( true ) {
2011-09-14 23:19:02 +02:00
this . nextSeed = internalNext ( ) ;
if ( this . nextSeed = = null ) break ;
version = this . nextSeed . getVersion ( ) ;
2010-06-30 10:42:29 +02:00
if ( version > = this . minVersion | | version = = 0 . 0 ) break ; // include 0.0 to access always developer peers
2006-11-20 03:46:53 +01:00
}
2008-08-02 14:12:04 +02:00
} catch ( final IOException e ) {
2009-11-05 21:28:37 +01:00
Log . logException ( e ) ;
2011-10-04 11:06:24 +02:00
Network . log . logSevere ( " ERROR seedLinEnum: seed.db corrupt ( " + e . getMessage ( ) + " ); resetting seed.db " , e ) ;
if ( database = = SeedDB . this . seedActiveDB ) SeedDB . this . seedActiveDB = resetSeedTable ( SeedDB . this . seedActiveDB , SeedDB . this . seedActiveDBFile ) ;
if ( database = = SeedDB . this . seedPassiveDB ) SeedDB . this . seedPassiveDB = resetSeedTable ( SeedDB . this . seedPassiveDB , SeedDB . this . seedPassiveDBFile ) ;
2011-09-14 23:19:02 +02:00
this . it = null ;
2008-08-02 14:12:04 +02:00
} catch ( final kelondroException e ) {
2009-11-05 21:28:37 +01:00
Log . logException ( e ) ;
2011-10-04 11:06:24 +02:00
Network . log . logSevere ( " ERROR seedLinEnum: seed.db corrupt ( " + e . getMessage ( ) + " ); resetting seed.db " , e ) ;
if ( database = = SeedDB . this . seedActiveDB ) SeedDB . this . seedActiveDB = resetSeedTable ( SeedDB . this . seedActiveDB , SeedDB . this . seedActiveDBFile ) ;
if ( database = = SeedDB . this . seedPassiveDB ) SeedDB . this . seedPassiveDB = resetSeedTable ( SeedDB . this . seedPassiveDB , SeedDB . this . seedPassiveDBFile ) ;
2011-09-14 23:19:02 +02:00
this . it = null ;
2005-10-24 16:07:43 +02:00
}
2005-10-17 17:46:12 +02:00
}
2011-09-14 23:19:02 +02:00
2010-04-19 18:42:37 +02:00
private seedEnum ( final boolean up , final String field , final MapDataMining database ) {
2005-04-07 21:19:42 +02:00
this . database = database ;
2005-10-24 16:07:43 +02:00
try {
2011-12-15 15:15:53 +01:00
this . it = database . entries ( up , field ) ;
2011-09-14 23:19:02 +02:00
this . nextSeed = internalNext ( ) ;
2008-08-02 14:12:04 +02:00
} catch ( final kelondroException e ) {
2009-11-05 21:28:37 +01:00
Log . logException ( e ) ;
2011-10-04 11:06:24 +02:00
Network . log . logSevere ( " ERROR seedLinEnum: seed.db corrupt ( " + e . getMessage ( ) + " ); resetting seed.db " , e ) ;
if ( database = = SeedDB . this . seedActiveDB ) SeedDB . this . seedActiveDB = resetSeedTable ( SeedDB . this . seedActiveDB , SeedDB . this . seedActiveDBFile ) ;
if ( database = = SeedDB . this . seedPassiveDB ) SeedDB . this . seedPassiveDB = resetSeedTable ( SeedDB . this . seedPassiveDB , SeedDB . this . seedPassiveDBFile ) ;
if ( database = = SeedDB . this . seedPotentialDB ) SeedDB . this . seedPotentialDB = resetSeedTable ( SeedDB . this . seedPotentialDB , SeedDB . this . seedPotentialDBFile ) ;
2011-09-14 23:19:02 +02:00
this . it = null ;
2005-10-24 16:07:43 +02:00
}
2005-10-17 17:46:12 +02:00
}
2011-09-14 23:19:02 +02:00
2007-10-01 14:30:23 +02:00
public boolean hasNext ( ) {
2011-09-14 23:19:02 +02:00
return ( this . nextSeed ! = null ) ;
2005-08-02 18:03:35 +02:00
}
2011-09-14 23:19:02 +02:00
2011-10-04 11:06:24 +02:00
private Seed internalNext ( ) {
2011-09-14 23:19:02 +02:00
if ( this . it = = null | | ! ( this . it . hasNext ( ) ) ) return null ;
2008-03-24 00:47:41 +01:00
try {
2011-12-15 15:15:53 +01:00
Map . Entry < byte [ ] , Map < String , String > > dna0 ;
2010-02-15 16:57:35 +01:00
ConcurrentHashMap < String , String > dna ;
2011-09-14 23:19:02 +02:00
while ( this . it . hasNext ( ) ) {
2011-03-07 02:45:11 +01:00
try {
2011-09-14 23:19:02 +02:00
dna0 = this . it . next ( ) ;
} catch ( final OutOfMemoryError e ) {
2011-03-07 02:45:11 +01:00
Log . logException ( e ) ;
dna0 = null ;
}
2011-02-28 07:28:29 +01:00
assert dna0 ! = null ;
if ( dna0 = = null ) continue ;
2011-12-15 15:15:53 +01:00
if ( dna0 . getValue ( ) instanceof ConcurrentHashMap ) {
dna = ( ConcurrentHashMap < String , String > ) dna0 . getValue ( ) ;
2011-03-03 12:30:04 +01:00
} else {
dna = new ConcurrentHashMap < String , String > ( ) ;
2011-12-15 15:15:53 +01:00
dna . putAll ( dna0 . getValue ( ) ) ;
2011-03-03 12:30:04 +01:00
}
2011-12-15 15:15:53 +01:00
if ( dna0 . getKey ( ) = = null ) continue ; // bad seed
final String hash = UTF8 . String ( dna0 . getKey ( ) ) ;
2011-10-04 11:06:24 +02:00
return new Seed ( hash , dna ) ;
2008-12-02 00:08:27 +01:00
}
2010-02-01 23:18:56 +01:00
return null ;
2008-08-02 14:12:04 +02:00
} catch ( final Exception e ) {
2009-11-05 21:28:37 +01:00
Log . logException ( e ) ;
2011-10-04 11:06:24 +02:00
Network . log . logSevere ( " ERROR internalNext: seed.db corrupt ( " + e . getMessage ( ) + " ); resetting seed.db " , e ) ;
if ( this . database = = SeedDB . this . seedActiveDB ) SeedDB . this . seedActiveDB = resetSeedTable ( SeedDB . this . seedActiveDB , SeedDB . this . seedActiveDBFile ) ;
if ( this . database = = SeedDB . this . seedPassiveDB ) SeedDB . this . seedPassiveDB = resetSeedTable ( SeedDB . this . seedPassiveDB , SeedDB . this . seedPassiveDBFile ) ;
if ( this . database = = SeedDB . this . seedPotentialDB ) SeedDB . this . seedPotentialDB = resetSeedTable ( SeedDB . this . seedPotentialDB , SeedDB . this . seedPotentialDBFile ) ;
2008-03-24 00:47:41 +01:00
return null ;
}
2005-10-24 16:07:43 +02:00
}
2011-09-14 23:19:02 +02:00
2011-10-04 11:06:24 +02:00
public Seed next ( ) {
final Seed seed = this . nextSeed ;
2010-06-30 10:42:29 +02:00
float version ;
2007-04-05 12:14:48 +02:00
try { while ( true ) {
2011-09-14 23:19:02 +02:00
this . nextSeed = internalNext ( ) ;
if ( this . nextSeed = = null ) break ;
version = this . nextSeed . getVersion ( ) ;
2010-06-30 10:42:29 +02:00
if ( version > = this . minVersion | | version = = 0 . 0 ) break ; // include 0.0 to access always developer peers
2008-08-02 14:12:04 +02:00
} } catch ( final kelondroException e ) {
2009-11-05 21:28:37 +01:00
Log . logException ( e ) ;
2010-06-30 10:42:29 +02:00
// emergency reset
2011-10-04 11:06:24 +02:00
Network . log . logSevere ( " seed-db emergency reset " , e ) ;
2011-09-14 23:19:02 +02:00
this . database . clear ( ) ;
this . nextSeed = null ;
2010-08-24 14:36:56 +02:00
return null ;
2006-11-20 03:46:53 +01:00
}
2005-10-24 16:07:43 +02:00
return seed ;
}
2007-10-01 14:30:23 +02:00
public void remove ( ) {
throw new UnsupportedOperationException ( ) ;
}
2011-09-14 23:19:02 +02:00
2005-04-07 21:19:42 +02:00
}
2006-01-25 14:25:48 +01:00
}