mirror of
https://github.com/yacy/yacy_search_server.git
synced 2024-09-19 00:01:41 +02:00
8da3174867
Especially for Turkish speaking users using "tr" as their system default locale : strings for technical stuff (URLs, tag names, constants...) must not be lower cased with the default locale, as 'I' doesn't becomes 'i' like in other locales such as "en", but becomes 'ı'.
679 lines
31 KiB
Java
679 lines
31 KiB
Java
// yacyCore.java
|
|
// -------------------------------------
|
|
// (C) by Michael Peter Christen; mc@yacy.net
|
|
// first published on http://www.anomic.de
|
|
// Frankfurt, Germany, 2004
|
|
//
|
|
// $LastChangedDate$
|
|
// $LastChangedRevision$
|
|
// $LastChangedBy$
|
|
//
|
|
// This program is free software; you can redistribute it and/or modify
|
|
// it under the terms of the GNU General Public License as published by
|
|
// the Free Software Foundation; either version 2 of the License, or
|
|
// (at your option) any later version.
|
|
//
|
|
// This program is distributed in the hope that it will be useful,
|
|
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
// GNU General Public License for more details.
|
|
//
|
|
// You should have received a copy of the GNU General Public License
|
|
// along with this program; if not, write to the Free Software
|
|
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
|
|
|
/*
|
|
the yacy process of getting in touch of other peers starts as follows:
|
|
- init seed cache. It is needed to determine the right peer for the Hello-Process
|
|
- create a own seed. This can be a new one or one loaded from a file
|
|
- The httpd must start up then first
|
|
- the own seed is completed by performing the 'yacyHello' process. This
|
|
process will result in a request back to the own peer to check if it runs
|
|
in server mode. This is the reason that the httpd must be started in advance.
|
|
|
|
*/
|
|
|
|
// contributions:
|
|
// principal peer status via file generation by Alexander Schier [AS]
|
|
|
|
package net.yacy.peers;
|
|
|
|
import java.net.MalformedURLException;
|
|
import java.util.Collections;
|
|
import java.util.Date;
|
|
import java.util.HashMap;
|
|
import java.util.Iterator;
|
|
import java.util.LinkedList;
|
|
import java.util.List;
|
|
import java.util.Locale;
|
|
import java.util.Map;
|
|
import java.util.Set;
|
|
import java.util.concurrent.ConcurrentHashMap;
|
|
import net.yacy.cora.date.GenericFormatter;
|
|
import net.yacy.cora.document.encoding.ASCII;
|
|
import net.yacy.cora.document.feed.RSSFeed;
|
|
import net.yacy.cora.document.feed.RSSMessage;
|
|
import net.yacy.cora.document.id.DigestURL;
|
|
import net.yacy.cora.protocol.Domains;
|
|
import net.yacy.cora.util.ConcurrentLog;
|
|
import net.yacy.peers.operation.yacySeedUploadFile;
|
|
import net.yacy.peers.operation.yacySeedUploadFtp;
|
|
import net.yacy.peers.operation.yacySeedUploadScp;
|
|
import net.yacy.peers.operation.yacySeedUploader;
|
|
import net.yacy.search.Switchboard;
|
|
import net.yacy.search.SwitchboardConstants;
|
|
import net.yacy.server.serverCore;
|
|
|
|
public class Network
|
|
{
|
|
|
|
// statics
|
|
public static final ThreadGroup publishThreadGroup = new ThreadGroup("publishThreadGroup");
|
|
public static final HashMap<String, String> seedUploadMethods = new HashMap<String, String>();
|
|
public static final ConcurrentLog log = new ConcurrentLog("YACY");
|
|
/** pseudo-random key derived from a time-interval while YaCy startup */
|
|
public static long speedKey = 0;
|
|
public static long magic = System.currentTimeMillis();
|
|
public static final Map<String, Accessible> amIAccessibleDB = new ConcurrentHashMap<String, Accessible>(); // Holds PeerHash / yacyAccessible Relations
|
|
// constants for PeerPing behavior
|
|
private static final int PING_INITIAL = 20;
|
|
private static final int PING_MAX_RUNNING = 3;
|
|
private static final int PING_MIN_RUNNING = 1;
|
|
private static final int PING_MIN_DBSIZE = 5;
|
|
private static final int PING_MIN_PEERSEEN = 1; // min. accessible to force senior
|
|
private static final long PING_MAX_DBAGE = 15 * 60 * 1000; // in milliseconds
|
|
|
|
// public static yacyShare shareManager = null;
|
|
// public static boolean terminate = false;
|
|
|
|
// class variables
|
|
Switchboard sb;
|
|
|
|
public Network(final Switchboard sb) {
|
|
final long time = System.currentTimeMillis();
|
|
|
|
this.sb = sb;
|
|
sb.setConfig("yacyStatus", "");
|
|
|
|
// create a peer news channel
|
|
final RSSFeed peernews = EventChannel.channels(EventChannel.PEERNEWS);
|
|
peernews.addMessage(new RSSMessage("YaCy started", "", ""));
|
|
|
|
// ensure that correct IP is used
|
|
final String staticIP = sb.getConfig(SwitchboardConstants.SERVER_STATICIP, "");
|
|
if (staticIP.length() != 0 && Seed.isProperIP(staticIP)) {
|
|
serverCore.useStaticIP = true;
|
|
sb.peers.mySeed().setIP(staticIP);
|
|
log.info("staticIP set to " + staticIP);
|
|
} else {
|
|
serverCore.useStaticIP = false;
|
|
}
|
|
|
|
loadSeedUploadMethods();
|
|
|
|
log.config("CORE INITIALIZED");
|
|
// ATTENTION, VERY IMPORTANT: before starting the thread, the httpd yacy server must be running!
|
|
|
|
speedKey = System.currentTimeMillis() - time;
|
|
}
|
|
|
|
public final void publishSeedList() {
|
|
if (log.isFine()) log.fine("yacyCore.publishSeedList: Triggered Seed Publish");
|
|
|
|
/*
|
|
if (oldIPStamp.equals((String) seedDB.mySeed.get(yacySeed.IP, "127.0.0.1")))
|
|
yacyCore.log.logDebug("***DEBUG publishSeedList: oldIP is equal");
|
|
if (seedCacheSizeStamp == seedDB.sizeConnected())
|
|
yacyCore.log.logDebug("***DEBUG publishSeedList: sizeConnected is equal");
|
|
if (canReachMyself())
|
|
yacyCore.log.logDebug("***DEBUG publishSeedList: I can reach myself");
|
|
*/
|
|
|
|
if ((this.sb.peers.mySeed().getIPs().contains(this.sb.peers.lastSeedUpload_myIP))
|
|
&& (this.sb.peers.lastSeedUpload_seedDBSize == this.sb.peers.sizeConnected())
|
|
&& (System.currentTimeMillis() - this.sb.peers.lastSeedUpload_timeStamp < 1000 * 60 * 60 * 24)
|
|
&& (this.sb.peers.mySeed().isPrincipal()) ) {
|
|
if (log.isFine()) log.fine("yacyCore.publishSeedList: not necessary to publish: oldIP is equal, sizeConnected is equal and I can reach myself under the old IP.");
|
|
return;
|
|
}
|
|
|
|
// getting the seed upload method that should be used ...
|
|
final String seedUploadMethod = this.sb.getConfig("seedUploadMethod", "");
|
|
|
|
if ( (!seedUploadMethod.equalsIgnoreCase("none"))
|
|
|| ((seedUploadMethod.equals("")) && (this.sb.getConfig("seedFTPPassword", "").length() > 0))
|
|
|| ((seedUploadMethod.equals("")) && (this.sb.getConfig("seedFilePath", "").length() > 0)) ) {
|
|
if ( seedUploadMethod.equals("") ) {
|
|
if ( this.sb.getConfig("seedFTPPassword", "").length() > 0 ) {
|
|
this.sb.setConfig("seedUploadMethod", "Ftp");
|
|
}
|
|
if ( this.sb.getConfig("seedFilePath", "").length() > 0 ) {
|
|
this.sb.setConfig("seedUploadMethod", "File");
|
|
}
|
|
}
|
|
// we want to be a principal...
|
|
saveSeedList(this.sb);
|
|
} else {
|
|
if ( seedUploadMethod.equals("") ) {
|
|
this.sb.setConfig("seedUploadMethod", "none");
|
|
}
|
|
if (log.isFine()) log.fine("yacyCore.publishSeedList: No uploading method configured");
|
|
return;
|
|
}
|
|
}
|
|
|
|
public final void peerPing() {
|
|
if ( (this.sb.isRobinsonMode())
|
|
&& (this.sb.getConfig(SwitchboardConstants.CLUSTER_MODE, "").equals(SwitchboardConstants.CLUSTER_MODE_PRIVATE_PEER)) ) {
|
|
// in case this peer is a privat peer we omit the peer ping
|
|
// all other robinson peer types do a peer ping:
|
|
// the privatecluster does the ping to the other cluster members
|
|
// the publiccluster does the ping to all peers, but prefers the own peer
|
|
// the publicpeer does the ping to all peers
|
|
return;
|
|
}
|
|
|
|
// before publishing, update some seed data
|
|
this.sb.updateMySeed();
|
|
|
|
// publish own seed to other peer, this can every peer, but makes only sense for senior peers
|
|
if ( this.sb.peers.sizeConnected() == 0 ) {
|
|
// reload the seed lists
|
|
this.sb.loadSeedLists();
|
|
log.info("re-initialized seed list. received "
|
|
+ this.sb.peers.sizeConnected()
|
|
+ " new peer(s)");
|
|
}
|
|
publishMySeed();
|
|
}
|
|
|
|
// use our own formatter to prevent concurrency locks with other processes
|
|
private final static GenericFormatter my_SHORT_SECOND_FORMATTER = new GenericFormatter(
|
|
GenericFormatter.FORMAT_SHORT_SECOND,
|
|
GenericFormatter.time_second);
|
|
|
|
protected class publishThread extends Thread
|
|
{
|
|
private Map<String, String> result;
|
|
private final Seed seed;
|
|
|
|
public publishThread(final ThreadGroup tg, final Seed seed) {
|
|
super(tg, "PublishSeed_" + seed.getName());
|
|
this.seed = seed;
|
|
this.result = null;
|
|
}
|
|
|
|
@Override
|
|
public final void run() {
|
|
try {
|
|
for (String ip: this.seed.getIPs()) {
|
|
this.result = Protocol.hello(Network.this.sb.peers.mySeed(), Network.this.sb.peers.peerActions, this.seed.getPublicAddress(ip), this.seed.hash);
|
|
if ( this.result == null ) {
|
|
// no or wrong response, delete that address
|
|
final String cause = "peer ping to peer resulted in error response (added < 0)";
|
|
log.info("publish: disconnected " + this.seed.get(Seed.PEERTYPE, Seed.PEERTYPE_SENIOR) + " peer '" + this.seed.getName() + "' from " + this.seed.getIPs() + ": " + cause);
|
|
Network.this.sb.peers.peerActions.interfaceDeparture(this.seed, ip);
|
|
continue;
|
|
}
|
|
// success! we have published our peer to a senior peer
|
|
// update latest news from the other peer
|
|
log.info("publish: handshaked "+ this.seed.get(Seed.PEERTYPE, Seed.PEERTYPE_SENIOR) + " peer '" + this.seed.getName() + "' at " + this.seed.getIPs());
|
|
// check if seed's lastSeen has been updated
|
|
final Seed newSeed = Network.this.sb.peers.getConnected(this.seed.hash);
|
|
if ( newSeed != null ) {
|
|
if ( !newSeed.isOnline() ) {
|
|
if ( log.isFine() ) {
|
|
log.fine("publish: recently handshaked " + this.seed.get(Seed.PEERTYPE, Seed.PEERTYPE_SENIOR) + " peer '" + this.seed.getName() + "' at " + this.seed.getIPs() + " is not online." + " Removing Peer from connected");
|
|
}
|
|
Network.this.sb.peers.peerActions.interfaceDeparture(newSeed, ip);
|
|
continue;
|
|
} else if ( newSeed.getLastSeenUTC() < (System.currentTimeMillis() - 10000) ) {
|
|
// update last seed date
|
|
if ( newSeed.getLastSeenUTC() >= this.seed.getLastSeenUTC() ) {
|
|
if ( log.isFine() ) {
|
|
log.fine("publish: recently handshaked " + this.seed.get(Seed.PEERTYPE, Seed.PEERTYPE_SENIOR) + " peer '" + this.seed.getName() + "' at " + this.seed.getIPs() + " with old LastSeen: '" + my_SHORT_SECOND_FORMATTER.format(new Date(newSeed.getLastSeenUTC())) + "'");
|
|
}
|
|
newSeed.setLastSeenUTC();
|
|
Network.this.sb.peers.peerActions.peerArrival(newSeed, true);
|
|
} else {
|
|
if ( log.isFine() ) {
|
|
log.fine("publish: recently handshaked " + this.seed.get(Seed.PEERTYPE, Seed.PEERTYPE_SENIOR) + " peer '" + this.seed.getName() + "' at " + this.seed.getIPs() + " with old LastSeen: '" + my_SHORT_SECOND_FORMATTER.format(new Date(newSeed.getLastSeenUTC())) + "', this is more recent: '" + my_SHORT_SECOND_FORMATTER.format(new Date(this.seed.getLastSeenUTC())) + "'");
|
|
}
|
|
this.seed.setLastSeenUTC();
|
|
Network.this.sb.peers.peerActions.peerArrival(this.seed, true);
|
|
}
|
|
}
|
|
} else {
|
|
if ( log.isFine() ) {
|
|
log.fine("publish: recently handshaked " + this.seed.get(Seed.PEERTYPE, Seed.PEERTYPE_SENIOR) + " peer '" + this.seed.getName() + "' at " + this.seed.getIPs() + " not in connectedDB");
|
|
}
|
|
}
|
|
}
|
|
} catch (final Exception e ) {
|
|
ConcurrentLog.logException(e);
|
|
log.severe(
|
|
"publishThread: error with target seed " + this.seed.toString() + ": " + e.getMessage(),
|
|
e);
|
|
}
|
|
}
|
|
}
|
|
|
|
private boolean publishMySeed() {
|
|
try {
|
|
// call this after the httpd was started up
|
|
|
|
// we need to find out our own ip
|
|
// This is not always easy, since the application may
|
|
// live behind a firewall or nat.
|
|
// the normal way to do this is either measure the value that java gives us,
|
|
// but this is not correct if the peer lives behind a NAT/Router or has several
|
|
// addresses and not the right one can be found out.
|
|
// We have several alternatives:
|
|
// 1. ask another peer. This should be normal and the default method.
|
|
// but if no other peer lives, or we don't know them, we cannot do that
|
|
// 2. ask own NAT. This is only an option if the NAT is a DI604, because this is the
|
|
// only supported for address retrieval
|
|
// 3. ask ip respond services in the internet. There are several, and they are all
|
|
// probed until we get a valid response.
|
|
|
|
// init yacyHello-process
|
|
Map<String, Seed> seeds; // hash/yacySeed relation
|
|
|
|
int attempts = this.sb.peers.sizeConnected();
|
|
|
|
// getting a list of peers to contact
|
|
if ( this.sb.peers.mySeed().get(Seed.PEERTYPE, Seed.PEERTYPE_VIRGIN).equals(Seed.PEERTYPE_VIRGIN) ) {
|
|
if (attempts > PING_INITIAL) attempts = PING_INITIAL;
|
|
final Set<byte[]> ch = Switchboard.getSwitchboard().clusterhashes;
|
|
seeds = DHTSelection.seedsByAge(this.sb.peers, true, Math.max(1, attempts - ((ch == null) ? 0 : ch.size()))); // best for fast connection
|
|
// add also all peers from cluster if this is a public robinson cluster
|
|
if ( ch != null ) {
|
|
String hash;
|
|
Seed seed;
|
|
for (byte[] hashb: ch) {
|
|
hash = ASCII.String(hashb);
|
|
seed = seeds.get(hash);
|
|
if (seed == null) {
|
|
seed = this.sb.peers.get(hash);
|
|
if (seed == null) continue;
|
|
}
|
|
seeds.put(hash, seed);
|
|
}
|
|
}
|
|
} else {
|
|
int diff = PING_MIN_DBSIZE - amIAccessibleDB.size();
|
|
if ( diff > PING_MIN_RUNNING ) {
|
|
diff = Math.min(diff, PING_MAX_RUNNING);
|
|
if ( attempts > diff ) {
|
|
attempts = diff;
|
|
}
|
|
} else {
|
|
if ( attempts > PING_MAX_RUNNING ) {
|
|
attempts = PING_MAX_RUNNING;
|
|
}
|
|
}
|
|
seeds = DHTSelection.seedsByAge(this.sb.peers, false, attempts); // best for seed list maintenance/cleaning
|
|
}
|
|
|
|
if (seeds == null || seeds.isEmpty()) return false;
|
|
if (seeds.size() < attempts) attempts = seeds.size();
|
|
|
|
// This will try to get Peers that are not currently in amIAccessibleDB
|
|
final Iterator<Seed> si = seeds.values().iterator();
|
|
Seed seed;
|
|
|
|
// include a YaCyNews record to my seed
|
|
try {
|
|
final NewsDB.Record record = this.sb.peers.newsPool.myPublication();
|
|
if ( record == null ) {
|
|
this.sb.peers.mySeed().put(Seed.NEWS, "");
|
|
} else {
|
|
this.sb.peers.mySeed().put(Seed.NEWS, net.yacy.utils.crypt.simpleEncode(record.toString()));
|
|
}
|
|
} catch (final Exception e ) {
|
|
log.severe("publishMySeed: problem with news encoding", e);
|
|
}
|
|
this.sb.peers.mySeed().setUnusedFlags();
|
|
//if (seeds.length > 1) {
|
|
// holding a reference to all started threads
|
|
final List<publishThread> syncList = Collections.synchronizedList(new LinkedList<publishThread>()); // memory for threads
|
|
|
|
// go through the peer list and starting a new publisher thread for each peer
|
|
int i = 0;
|
|
while ( si.hasNext() ) {
|
|
seed = si.next();
|
|
if ( seed == null || seed.hash.equals(this.sb.peers.mySeed().hash)) {
|
|
continue;
|
|
}
|
|
i++;
|
|
|
|
String ip = seed.getIP();
|
|
final String address = seed.getPublicAddress(ip);
|
|
if ( log.isFine() ) log.fine("HELLO #" + i + " to peer '" + seed.get(Seed.NAME, "") + "' at " + address); // debug
|
|
final String seederror = seed.isProper(false);
|
|
if ( (address == null) || (seederror != null) ) {
|
|
// we don't like that address, delete it
|
|
this.sb.peers.peerActions.interfaceDeparture(seed, ip);
|
|
} else {
|
|
// starting a new publisher thread
|
|
publishThread t = new publishThread(Network.publishThreadGroup, seed);
|
|
t.start();
|
|
syncList.add(t);
|
|
}
|
|
}
|
|
|
|
// receiving the result of all started publisher threads
|
|
for (publishThread t: syncList) {
|
|
// waiting for the next thread to finish
|
|
t.join();
|
|
}
|
|
|
|
int accessible = 0;
|
|
int notaccessible = 0;
|
|
final long cutofftime = System.currentTimeMillis() - PING_MAX_DBAGE;
|
|
final int dbSize;
|
|
synchronized ( amIAccessibleDB ) {
|
|
dbSize = amIAccessibleDB.size();
|
|
final Iterator<String> ai = amIAccessibleDB.keySet().iterator();
|
|
while ( ai.hasNext() ) {
|
|
final Accessible ya = amIAccessibleDB.get(ai.next());
|
|
if ( ya.lastUpdated < cutofftime ) {
|
|
ai.remove();
|
|
} else {
|
|
if ( ya.IWasAccessed ) {
|
|
accessible++;
|
|
} else {
|
|
notaccessible++;
|
|
}
|
|
}
|
|
}
|
|
if (log.isFine()) log.fine("DBSize before -> after Cleanup: " + dbSize + " -> " + amIAccessibleDB.size());
|
|
}
|
|
log.info("PeerPing: I am accessible for " + accessible + " peer(s), not accessible for " + notaccessible + " peer(s).");
|
|
|
|
if ( (accessible + notaccessible) > 0 ) {
|
|
final String newPeerType;
|
|
// At least one other Peer told us our type
|
|
if ( (accessible >= PING_MIN_PEERSEEN) || (accessible >= notaccessible) ) {
|
|
// We can be reached from a majority of other Peers
|
|
if ( this.sb.peers.mySeed().isPrincipal() ) {
|
|
newPeerType = Seed.PEERTYPE_PRINCIPAL;
|
|
} else {
|
|
newPeerType = Seed.PEERTYPE_SENIOR;
|
|
}
|
|
} else {
|
|
// We cannot be reached from the outside
|
|
newPeerType = Seed.PEERTYPE_JUNIOR;
|
|
}
|
|
if ( this.sb.peers.mySeed().orVirgin().equals(newPeerType) ) {
|
|
log.info("PeerPing: myType is " + this.sb.peers.mySeed().orVirgin());
|
|
} else {
|
|
log.info("PeerPing: changing myType from '" + this.sb.peers.mySeed().orVirgin() + "' to '" + newPeerType + "'");
|
|
this.sb.peers.mySeed().put(Seed.PEERTYPE, newPeerType);
|
|
}
|
|
} else {
|
|
log.info("PeerPing: No data, staying at myType: " + this.sb.peers.mySeed().orVirgin());
|
|
}
|
|
|
|
// success! we have published our peer to a senior peer
|
|
// update latest news from the other peer
|
|
// log.logInfo("publish: handshaked " + t.seed.get(yacySeed.PEERTYPE, yacySeed.PEERTYPE_SENIOR) + " peer '" + t.seed.getName() + "' at " + t.seed.getAddress());
|
|
this.sb.peers.saveMySeed();
|
|
|
|
// if we have an address, we do nothing
|
|
if (this.sb.peers.mySeed().isProper(true) == null) return true;
|
|
|
|
// still no success
|
|
final String ip = this.sb.getConfig(SwitchboardConstants.SERVER_STATICIP, "");
|
|
|
|
if (Seed.isProperIP(ip)) {
|
|
this.sb.peers.mySeed().setIP(ip);
|
|
}
|
|
if ( this.sb.peers.mySeed().get(Seed.PEERTYPE, Seed.PEERTYPE_JUNIOR).equals(Seed.PEERTYPE_JUNIOR) ) {
|
|
this.sb.peers.mySeed().put(Seed.PEERTYPE, Seed.PEERTYPE_SENIOR); // to start bootstraping, we need to be recognised as PEERTYPE_SENIOR peer
|
|
}
|
|
log.info("publish: no recipient found, our address is " + this.sb.peers.mySeed().getIPs());
|
|
this.sb.peers.saveMySeed();
|
|
return false;
|
|
} catch (final InterruptedException e ) {
|
|
try {
|
|
log.info("publish: Interruption detected while publishing my seed.");
|
|
|
|
// consuming the theads interrupted signal
|
|
Thread.interrupted();
|
|
|
|
// interrupt all already started publishThreads
|
|
log.info("publish: Signaling shutdown to "
|
|
+ Network.publishThreadGroup.activeCount()
|
|
+ " remaining publishing threads ...");
|
|
Network.publishThreadGroup.interrupt();
|
|
|
|
// waiting some time for the publishThreads to finish execution
|
|
try {
|
|
Thread.sleep(500);
|
|
} catch (final InterruptedException ex ) {
|
|
}
|
|
|
|
// getting the amount of remaining publishing threads
|
|
int threadCount = Network.publishThreadGroup.activeCount();
|
|
final Thread[] threadList = new Thread[threadCount];
|
|
threadCount = Network.publishThreadGroup.enumerate(threadList);
|
|
|
|
// we need to use a timeout here because of missing interruptable session threads ...
|
|
if ( log.isFine() ) {
|
|
log.fine("publish: Waiting for "
|
|
+ Network.publishThreadGroup.activeCount()
|
|
+ " remaining publishing threads to finish shutdown ...");
|
|
}
|
|
for ( int currentThreadIdx = 0; currentThreadIdx < threadCount; currentThreadIdx++ ) {
|
|
final Thread currentThread = threadList[currentThreadIdx];
|
|
|
|
if ( currentThread.isAlive() ) {
|
|
if ( log.isFine() ) {
|
|
log.fine("publish: Waiting for remaining publishing thread '"
|
|
+ currentThread.getName()
|
|
+ "' to finish shutdown");
|
|
}
|
|
try {
|
|
currentThread.join(500);
|
|
} catch (final InterruptedException ex ) {
|
|
}
|
|
}
|
|
}
|
|
|
|
log.info("publish: Shutdown off all remaining publishing thread finished.");
|
|
|
|
} catch (final Exception ee ) {
|
|
log.warn("publish: Unexpected error while trying to shutdown all remaining publishing threads.", e);
|
|
}
|
|
|
|
return false;
|
|
}
|
|
}
|
|
|
|
@SuppressWarnings("unchecked")
|
|
public static HashMap<String, String> getSeedUploadMethods() {
|
|
synchronized ( Network.seedUploadMethods ) {
|
|
return (HashMap<String, String>) Network.seedUploadMethods.clone();
|
|
}
|
|
}
|
|
|
|
public static yacySeedUploader getSeedUploader(final String methodname) {
|
|
String className = null;
|
|
synchronized ( Network.seedUploadMethods ) {
|
|
if ( Network.seedUploadMethods.containsKey(methodname) ) {
|
|
className = Network.seedUploadMethods.get(methodname);
|
|
}
|
|
}
|
|
|
|
if ( className == null ) {
|
|
return null;
|
|
}
|
|
try {
|
|
final Class<?> uploaderClass = Class.forName(className);
|
|
final Object uploader = uploaderClass.newInstance();
|
|
return (yacySeedUploader) uploader;
|
|
} catch (final Exception e ) {
|
|
return null;
|
|
}
|
|
}
|
|
|
|
public static void loadSeedUploadMethods() {
|
|
yacySeedUploader uploader;
|
|
uploader = new yacySeedUploadFile();
|
|
Network.seedUploadMethods.put(uploader
|
|
.getClass()
|
|
.getSimpleName()
|
|
.substring("yacySeedUpload".length()), uploader.getClass().getCanonicalName());
|
|
uploader = new yacySeedUploadFtp();
|
|
Network.seedUploadMethods.put(uploader
|
|
.getClass()
|
|
.getSimpleName()
|
|
.substring("yacySeedUpload".length()), uploader.getClass().getCanonicalName());
|
|
uploader = new yacySeedUploadScp();
|
|
Network.seedUploadMethods.put(uploader
|
|
.getClass()
|
|
.getSimpleName()
|
|
.substring("yacySeedUpload".length()), uploader.getClass().getCanonicalName());
|
|
}
|
|
|
|
public static boolean changeSeedUploadMethod(final String method) {
|
|
if ( method == null || method.isEmpty() ) {
|
|
return false;
|
|
}
|
|
|
|
if ( method.equalsIgnoreCase("none") ) {
|
|
return true;
|
|
}
|
|
|
|
synchronized ( Network.seedUploadMethods ) {
|
|
return Network.seedUploadMethods.containsKey(method);
|
|
}
|
|
}
|
|
|
|
public static final String saveSeedList(final Switchboard sb) {
|
|
try {
|
|
// return an error if this is not successful, and NULL if everything is fine
|
|
String logt;
|
|
|
|
// be shure that we have something to say
|
|
if ( sb.peers.mySeed().getPublicAddress(sb.peers.mySeed().getIP()) == null ) {
|
|
final String errorMsg = "We have no valid IP address until now";
|
|
log.warn("SaveSeedList: " + errorMsg);
|
|
return errorMsg;
|
|
}
|
|
|
|
// getting the configured seed uploader
|
|
String seedUploadMethod = sb.getConfig("seedUploadMethod", "");
|
|
|
|
// for backward compatiblity ....
|
|
if ( seedUploadMethod.equalsIgnoreCase("Ftp")
|
|
|| (seedUploadMethod.equals("") && sb.getConfig("seedFTPPassword", "").length() > 0) ) {
|
|
|
|
seedUploadMethod = "Ftp";
|
|
sb.setConfig("seedUploadMethod", seedUploadMethod);
|
|
|
|
} else if ( seedUploadMethod.equalsIgnoreCase("File")
|
|
|| (seedUploadMethod.equals("") && sb.getConfig("seedFilePath", "").length() > 0) ) {
|
|
|
|
seedUploadMethod = "File";
|
|
sb.setConfig("seedUploadMethod", seedUploadMethod);
|
|
}
|
|
|
|
// determine the seed uploader that should be used ...
|
|
if ( seedUploadMethod.equalsIgnoreCase("none") ) {
|
|
return "no uploader specified";
|
|
}
|
|
|
|
final yacySeedUploader uploader = getSeedUploader(seedUploadMethod);
|
|
if ( uploader == null ) {
|
|
final String errorMsg =
|
|
"Unable to get the proper uploader-class for seed uploading method '"
|
|
+ seedUploadMethod
|
|
+ "'.";
|
|
log.warn("SaveSeedList: " + errorMsg);
|
|
return errorMsg;
|
|
}
|
|
|
|
// ensure that the seed file url is configured properly
|
|
DigestURL seedURL;
|
|
try {
|
|
final String seedURLStr = sb.peers.mySeed().get(Seed.SEEDLISTURL, "");
|
|
if ( seedURLStr.isEmpty() ) {
|
|
throw new MalformedURLException("The seed-file url must not be empty.");
|
|
}
|
|
if ( !(seedURLStr.toLowerCase(Locale.ROOT).startsWith("http://") || seedURLStr.toLowerCase(Locale.ROOT).startsWith(
|
|
"https://")) ) {
|
|
throw new MalformedURLException("Unsupported protocol.");
|
|
}
|
|
seedURL = new DigestURL(seedURLStr);
|
|
final String host = seedURL.getHost();
|
|
if (Domains.isLocalhost(host) || (Domains.isIntranet(host) && !sb.isIntranetMode())) { // check seedlist reacheable
|
|
// TODO: this does not prevent setting a local hostname e.g. "http://testhost_8090/seedlist.txt" (fyi: never did)
|
|
// but the or part allows to setup a principal peer in intranet environment
|
|
final String errorMsg = "seedURL in local network rejected (local hosts can't be reached from outside)";
|
|
log.warn("SaveSeedList: " + errorMsg);
|
|
return errorMsg;
|
|
}
|
|
} catch (final MalformedURLException e ) {
|
|
final String errorMsg =
|
|
"Malformed seed file URL '"
|
|
+ sb.peers.mySeed().get(Seed.SEEDLISTURL, "")
|
|
+ "'. "
|
|
+ e.getMessage();
|
|
log.warn("SaveSeedList: " + errorMsg);
|
|
return errorMsg;
|
|
}
|
|
|
|
// upload the seed-list using the configured uploader class
|
|
String prevStatus = sb.peers.mySeed().get(Seed.PEERTYPE, Seed.PEERTYPE_JUNIOR);
|
|
if ( prevStatus.equals(Seed.PEERTYPE_PRINCIPAL) ) {
|
|
prevStatus = Seed.PEERTYPE_SENIOR;
|
|
}
|
|
|
|
try {
|
|
sb.peers.mySeed().put(Seed.PEERTYPE, Seed.PEERTYPE_PRINCIPAL); // this information shall also be uploaded
|
|
|
|
if ( log.isFine() ) {
|
|
log.fine("SaveSeedList: Using seed uploading method '"
|
|
+ seedUploadMethod
|
|
+ "' for seed-list uploading."
|
|
+ "\n\tPrevious peerType is '"
|
|
+ sb.peers.mySeed().get(Seed.PEERTYPE, Seed.PEERTYPE_JUNIOR)
|
|
+ "'.");
|
|
}
|
|
|
|
logt = sb.peers.uploadSeedList(uploader, sb, sb.peers, seedURL);
|
|
if ( logt != null ) {
|
|
if ( logt.indexOf("Error", 0) >= 0 ) {
|
|
sb.peers.mySeed().put(Seed.PEERTYPE, prevStatus);
|
|
final String errorMsg =
|
|
"SaveSeedList: seed upload failed using "
|
|
+ uploader.getClass().getName()
|
|
+ " (error): "
|
|
+ logt.substring(logt.indexOf("Error", 0) + 6);
|
|
log.severe(errorMsg);
|
|
return errorMsg;
|
|
}
|
|
log.info(logt);
|
|
}
|
|
|
|
// finally, set the principal status
|
|
sb.setConfig("yacyStatus", Seed.PEERTYPE_PRINCIPAL);
|
|
return null;
|
|
} catch (final Exception e ) {
|
|
sb.peers.mySeed().put(Seed.PEERTYPE, prevStatus);
|
|
sb.setConfig("yacyStatus", prevStatus);
|
|
final String errorMsg = "SaveSeedList: Seed upload failed (IO error): " + e.getMessage();
|
|
log.info(errorMsg, e);
|
|
return errorMsg;
|
|
}
|
|
} finally {
|
|
sb.peers.lastSeedUpload_seedDBSize = sb.peers.sizeConnected();
|
|
sb.peers.lastSeedUpload_timeStamp = System.currentTimeMillis();
|
|
sb.peers.lastSeedUpload_myIP = sb.peers.mySeed().getIP();
|
|
}
|
|
}
|
|
|
|
}
|