mirror of
https://github.com/yacy/yacy_search_server.git
synced 2024-09-19 00:01:41 +02:00
seedDB helpers update/cleanup
git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@4461 6c8d7289-2bf4-0310-a012-ef5d649a1542
This commit is contained in:
parent
dd215e7f6b
commit
77ba446332
|
@ -62,25 +62,34 @@ import de.anomic.yacy.yacySeed;
|
|||
|
||||
public final class crawlReceipt {
|
||||
|
||||
|
||||
/*
|
||||
* this is used to respond on a remote crawling request
|
||||
*/
|
||||
|
||||
public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch env) {
|
||||
// return variable that accumulates replacements
|
||||
plasmaSwitchboard switchboard = (plasmaSwitchboard) env;
|
||||
serverObjects prop = new serverObjects();
|
||||
if ((post == null) || (env == null)) return prop;
|
||||
if (!yacyNetwork.authentifyRequest(post, env)) return prop;
|
||||
|
||||
serverLog log = switchboard.getLog();
|
||||
final serverObjects prop = new serverObjects();
|
||||
// defaults
|
||||
prop.put("delay", "3600");
|
||||
|
||||
if (post == null || env == null || !yacyNetwork.authentifyRequest(post, env)) {
|
||||
return prop;
|
||||
}
|
||||
|
||||
// seed hash of requester
|
||||
String ohash = post.get("iam", "");
|
||||
if (ohash == null || ohash.length() == 0 ) return prop;
|
||||
yacySeed oseed = yacyCore.seedDB.get(ohash);
|
||||
if (oseed == null) return prop;
|
||||
oseed.setFlagDirectConnect(true);
|
||||
oseed.setLastSeenUTC();
|
||||
|
||||
final plasmaSwitchboard sb = (plasmaSwitchboard) env;
|
||||
serverLog log = sb.getLog();
|
||||
|
||||
//int proxyPrefetchDepth = Integer.parseInt(env.getConfig("proxyPrefetchDepth", "0"));
|
||||
//int crawlingDepth = Integer.parseInt(env.getConfig("crawlingDepth", "0"));
|
||||
|
||||
// request values
|
||||
String iam = post.get("iam", ""); // seed hash of requester
|
||||
String youare = post.get("youare", ""); // seed hash of the target peer, needed for network stability
|
||||
//String process = post.get("process", ""); // process type
|
||||
String key = post.get("key", ""); // transmission key
|
||||
|
@ -111,59 +120,50 @@ public final class crawlReceipt {
|
|||
|
||||
*/
|
||||
|
||||
final yacySeed otherPeer = yacyCore.seedDB.get(iam);
|
||||
if (otherPeer == null) {
|
||||
prop.put("delay", "3600");
|
||||
return prop;
|
||||
} else {
|
||||
otherPeer.setLastSeenUTC();
|
||||
}
|
||||
final String otherPeerName = iam + ":" + otherPeer.getName() + "/" + otherPeer.getVersion();
|
||||
|
||||
|
||||
if ((yacyCore.seedDB.mySeed() == null) || (!(yacyCore.seedDB.mySeed().hash.equals(youare)))) {
|
||||
// no yacy connection / unknown peers
|
||||
prop.put("delay", "3600");
|
||||
return prop;
|
||||
}
|
||||
|
||||
if (propStr == null) {
|
||||
// error with url / wrong key
|
||||
prop.put("delay", "3600");
|
||||
return prop;
|
||||
}
|
||||
|
||||
if ((switchboard.isRobinsonMode()) && (!switchboard.isInMyCluster(otherPeer))) {
|
||||
if ((sb.isRobinsonMode()) && (!sb.isInMyCluster(oseed))) {
|
||||
// we reject urls that are from outside our cluster
|
||||
prop.put("delay", "9999");
|
||||
return prop; // ???
|
||||
}
|
||||
|
||||
// generating a new loaded URL entry
|
||||
indexURLEntry entry = switchboard.wordIndex.loadedURL.newEntry(propStr);
|
||||
indexURLEntry entry = sb.wordIndex.loadedURL.newEntry(propStr);
|
||||
if (entry == null) {
|
||||
log.logWarning("crawlReceipt: RECEIVED wrong RECEIPT (entry null) from peer " + iam + "\n\tURL properties: "+ propStr);
|
||||
prop.put("delay", "3600");
|
||||
log.logWarning("crawlReceipt: RECEIVED wrong RECEIPT (entry null) from peer " + ohash + "\n\tURL properties: "+ propStr);
|
||||
return prop;
|
||||
}
|
||||
|
||||
indexURLEntry.Components comp = entry.comp();
|
||||
if (comp.url() == null) {
|
||||
log.logWarning("crawlReceipt: RECEIVED wrong RECEIPT (url null) for hash " + entry.hash() + " from peer " + iam + "\n\tURL properties: "+ propStr);
|
||||
prop.put("delay", "3600");
|
||||
log.logWarning("crawlReceipt: RECEIVED wrong RECEIPT (url null) for hash " + entry.hash() + " from peer " + ohash + "\n\tURL properties: "+ propStr);
|
||||
return prop;
|
||||
}
|
||||
|
||||
// check if the entry is in our network domain
|
||||
if (!switchboard.acceptURL(comp.url())) {
|
||||
log.logWarning("crawlReceipt: RECEIVED wrong RECEIPT (url outside of our domain) for hash " + entry.hash() + " from peer " + iam + "\n\tURL properties: "+ propStr);
|
||||
if (!sb.acceptURL(comp.url())) {
|
||||
log.logWarning("crawlReceipt: RECEIVED wrong RECEIPT (url outside of our domain) for hash " + entry.hash() + " from peer " + ohash + "\n\tURL properties: "+ propStr);
|
||||
prop.put("delay", "9999");
|
||||
return prop;
|
||||
}
|
||||
|
||||
if (result.equals("fill")) try {
|
||||
// put new entry into database
|
||||
switchboard.wordIndex.loadedURL.store(entry);
|
||||
switchboard.wordIndex.loadedURL.stack(entry, youare, iam, 1);
|
||||
switchboard.crawlQueues.delegatedURL.remove(entry.hash()); // the delegated work has been done
|
||||
sb.wordIndex.loadedURL.store(entry);
|
||||
sb.wordIndex.loadedURL.stack(entry, youare, ohash, 1);
|
||||
sb.crawlQueues.delegatedURL.remove(entry.hash()); // the delegated work has been done
|
||||
final String otherPeerName = ohash + ":" + oseed.getName() + "/" + oseed.getVersion();
|
||||
log.logInfo("crawlReceipt: RECEIVED RECEIPT from " + otherPeerName + " for URL " + entry.hash() + ":" + comp.url().toNormalform(false, true));
|
||||
|
||||
// ready for more
|
||||
|
@ -171,16 +171,14 @@ public final class crawlReceipt {
|
|||
return prop;
|
||||
} catch (IOException e) {
|
||||
e.printStackTrace();
|
||||
prop.put("delay", "3600");
|
||||
return prop;
|
||||
}
|
||||
|
||||
switchboard.crawlQueues.delegatedURL.remove(entry.hash()); // the delegated work is transformed into an error case
|
||||
plasmaCrawlZURL.Entry ee = switchboard.crawlQueues.errorURL.newEntry(entry.toBalancerEntry(), youare, null, 0, result + ":" + reason);
|
||||
sb.crawlQueues.delegatedURL.remove(entry.hash()); // the delegated work is transformed into an error case
|
||||
plasmaCrawlZURL.Entry ee = sb.crawlQueues.errorURL.newEntry(entry.toBalancerEntry(), youare, null, 0, result + ":" + reason);
|
||||
ee.store();
|
||||
switchboard.crawlQueues.errorURL.push(ee);
|
||||
sb.crawlQueues.errorURL.push(ee);
|
||||
//switchboard.noticeURL.remove(receivedUrlhash);
|
||||
prop.put("delay", "3600");
|
||||
return prop;
|
||||
|
||||
// return rewrite properties
|
||||
|
|
|
@ -65,10 +65,9 @@ import de.anomic.yacy.yacyVersion;
|
|||
public final class hello {
|
||||
|
||||
public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch env) throws InterruptedException {
|
||||
plasmaSwitchboard sb = (plasmaSwitchboard) env;
|
||||
// return variable that accumulates replacements
|
||||
serverObjects prop = new serverObjects();
|
||||
prop.put("message", "none");
|
||||
if ((post == null) || (env == null)) {
|
||||
if (post == null || env == null) {
|
||||
prop.put("message", "no post or no enviroment");
|
||||
return prop;
|
||||
}
|
||||
|
@ -76,7 +75,8 @@ public final class hello {
|
|||
prop.put("message", "not in my network");
|
||||
return prop;
|
||||
}
|
||||
|
||||
prop.put("message", "none");
|
||||
|
||||
// final String iam = (String) post.get("iam", ""); // complete seed of the requesting peer
|
||||
// final String mytime = (String) post.get(MYTIME, ""); //
|
||||
final String key = post.get("key", ""); // transmission key for response
|
||||
|
@ -90,14 +90,14 @@ public final class hello {
|
|||
prop.put("message", "your seed is too long (" + seed.length() + ")");
|
||||
return prop;
|
||||
}
|
||||
final yacySeed remoteSeed = yacySeed.genRemoteSeed(seed, key, false);
|
||||
final yacySeed oseed = yacySeed.genRemoteSeed(seed, key, false);
|
||||
|
||||
// System.out.println("YACYHELLO: REMOTESEED=" + ((remoteSeed == null) ? "NULL" : remoteSeed.toString()));
|
||||
if ((remoteSeed == null) || (remoteSeed.hash == null)) {
|
||||
if (oseed == null || oseed.hash == null) {
|
||||
prop.put("message", "cannot parse your seed");
|
||||
return prop;
|
||||
}
|
||||
|
||||
|
||||
// final String properTest = remoteSeed.isProper();
|
||||
// The remote peer might not know its IP yet, so don't abort if the IP check fails
|
||||
// if ((properTest != null) && (! properTest.substring(0,1).equals("IP"))) { return null; }
|
||||
|
@ -110,18 +110,19 @@ public final class hello {
|
|||
return prop;
|
||||
}
|
||||
final String userAgent = (String) header.get(httpHeader.USER_AGENT, "<unknown>");
|
||||
final String reportedip = remoteSeed.get(yacySeed.IP, "");
|
||||
final String reportedPeerType = remoteSeed.get(yacySeed.PEERTYPE, yacySeed.PEERTYPE_JUNIOR);
|
||||
final float clientversion = remoteSeed.getVersion();
|
||||
final String reportedip = oseed.get(yacySeed.IP, "");
|
||||
final String reportedPeerType = oseed.get(yacySeed.PEERTYPE, yacySeed.PEERTYPE_JUNIOR);
|
||||
final float clientversion = oseed.getVersion();
|
||||
|
||||
if ((sb.isRobinsonMode()) && (!sb.isPublicRobinson())) {
|
||||
// if we are a robinson cluster, answer only if this client is known by our network definition
|
||||
final plasmaSwitchboard sb = (plasmaSwitchboard) env;
|
||||
if (sb.isRobinsonMode() && !sb.isPublicRobinson()) {
|
||||
// if we are a robinson cluster, answer only if this client is known by our network definition
|
||||
prop.put("message", "I am robinson, I do not answer");
|
||||
return prop;
|
||||
}
|
||||
|
||||
|
||||
int urls = -1;
|
||||
if (sb.clusterhashes != null) remoteSeed.setAlternativeAddress((String) sb.clusterhashes.get(remoteSeed.hash));
|
||||
if (sb.clusterhashes != null) oseed.setAlternativeAddress((String) sb.clusterhashes.get(oseed.hash));
|
||||
|
||||
// if the remote client has reported its own IP address and the client supports
|
||||
// the port forwarding feature (if client version >= 0.383) then we try to
|
||||
|
@ -131,8 +132,8 @@ public final class hello {
|
|||
|
||||
// try first the reportedip, since this may be a connect from a port-forwarding host
|
||||
prop.put("yourip", reportedip);
|
||||
remoteSeed.put(yacySeed.IP, reportedip);
|
||||
urls = yacyClient.queryUrlCount(remoteSeed);
|
||||
oseed.put(yacySeed.IP, reportedip);
|
||||
urls = yacyClient.queryUrlCount(oseed);
|
||||
} else {
|
||||
prop.put("yourip", "unknown");
|
||||
}
|
||||
|
@ -150,41 +151,41 @@ public final class hello {
|
|||
serverCore.checkInterruption();
|
||||
|
||||
prop.put("yourip", clientip);
|
||||
remoteSeed.put(yacySeed.IP, clientip);
|
||||
urls = yacyClient.queryUrlCount(remoteSeed);
|
||||
oseed.put(yacySeed.IP, clientip);
|
||||
urls = yacyClient.queryUrlCount(oseed);
|
||||
}
|
||||
}
|
||||
|
||||
// System.out.println("YACYHELLO: YOUR IP=" + clientip);
|
||||
// set lastseen value (we have seen that peer, it contacted us!)
|
||||
remoteSeed.setLastSeenUTC();
|
||||
oseed.setLastSeenUTC();
|
||||
|
||||
// assign status
|
||||
if (urls >= 0) {
|
||||
if (remoteSeed.get(yacySeed.PEERTYPE, yacySeed.PEERTYPE_SENIOR) == null) {
|
||||
if (oseed.get(yacySeed.PEERTYPE, yacySeed.PEERTYPE_SENIOR) == null) {
|
||||
prop.put(yacySeed.YOURTYPE, yacySeed.PEERTYPE_SENIOR);
|
||||
remoteSeed.put(yacySeed.PEERTYPE, yacySeed.PEERTYPE_SENIOR);
|
||||
} else if (remoteSeed.get(yacySeed.PEERTYPE, yacySeed.PEERTYPE_PRINCIPAL).equals(yacySeed.PEERTYPE_PRINCIPAL)) {
|
||||
oseed.put(yacySeed.PEERTYPE, yacySeed.PEERTYPE_SENIOR);
|
||||
} else if (oseed.get(yacySeed.PEERTYPE, yacySeed.PEERTYPE_PRINCIPAL).equals(yacySeed.PEERTYPE_PRINCIPAL)) {
|
||||
prop.put(yacySeed.YOURTYPE, yacySeed.PEERTYPE_PRINCIPAL);
|
||||
} else {
|
||||
prop.put(yacySeed.YOURTYPE, yacySeed.PEERTYPE_SENIOR);
|
||||
remoteSeed.put(yacySeed.PEERTYPE, yacySeed.PEERTYPE_SENIOR);
|
||||
oseed.put(yacySeed.PEERTYPE, yacySeed.PEERTYPE_SENIOR);
|
||||
}
|
||||
// connect the seed
|
||||
yacyCore.peerActions.peerArrival(remoteSeed, true);
|
||||
yacyCore.peerActions.peerArrival(oseed, true);
|
||||
} else {
|
||||
prop.put(yacySeed.YOURTYPE, yacySeed.PEERTYPE_JUNIOR);
|
||||
yacyCore.peerActions.juniorConnects++; // update statistics
|
||||
remoteSeed.put(yacySeed.PEERTYPE, yacySeed.PEERTYPE_JUNIOR);
|
||||
yacyCore.log.logInfo("hello: responded remote junior peer '" + remoteSeed.getName() + "' from " + reportedip);
|
||||
oseed.put(yacySeed.PEERTYPE, yacySeed.PEERTYPE_JUNIOR);
|
||||
yacyCore.log.logInfo("hello: responded remote junior peer '" + oseed.getName() + "' from " + reportedip);
|
||||
// no connection here, instead store junior in connection cache
|
||||
if ((remoteSeed.hash != null) && (remoteSeed.isProper() == null)) {
|
||||
yacyCore.peerActions.peerPing(remoteSeed);
|
||||
if ((oseed.hash != null) && (oseed.isProper() == null)) {
|
||||
yacyCore.peerActions.peerPing(oseed);
|
||||
}
|
||||
}
|
||||
yacyCore.peerActions.setUserAgent(clientip, userAgent);
|
||||
if (!((String)prop.get(yacySeed.YOURTYPE)).equals(reportedPeerType)) {
|
||||
yacyCore.log.logInfo("hello: changing remote peer '" + remoteSeed.getName() +
|
||||
yacyCore.log.logInfo("hello: changing remote peer '" + oseed.getName() +
|
||||
"' [" + reportedip +
|
||||
"] peerType from '" + reportedPeerType +
|
||||
"' to '" + prop.get(yacySeed.YOURTYPE) + "'.");
|
||||
|
|
|
@ -73,16 +73,17 @@ public final class message {
|
|||
}
|
||||
|
||||
public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch env) {
|
||||
if (post == null || env == null) { return null; }
|
||||
|
||||
// return variable that accumulates replacements
|
||||
plasmaSwitchboard sb = (plasmaSwitchboard) env;
|
||||
serverObjects prop = new serverObjects();
|
||||
if ((post == null) || (env == null)) return prop;
|
||||
if (!yacyNetwork.authentifyRequest(post, env)) return prop;
|
||||
final serverObjects prop = new serverObjects();
|
||||
// defaults
|
||||
prop.put("response", "-1"); // request rejected
|
||||
|
||||
if (post == null || env == null || !yacyNetwork.authentifyRequest(post, env)) {
|
||||
return prop;
|
||||
}
|
||||
|
||||
String process = post.get("process", "permission");
|
||||
String key = post.get("key", "");
|
||||
String key = post.get("key", "");
|
||||
|
||||
int messagesize = 10240;
|
||||
int attachmentsize = 0;
|
||||
|
@ -94,18 +95,17 @@ public final class message {
|
|||
// check if we are the right target and requester has correct information about this peer
|
||||
if ((yacyCore.seedDB.mySeed() == null) || (!(yacyCore.seedDB.mySeed().hash.equals(youare)))) {
|
||||
// this request has a wrong target
|
||||
prop.put("response", "-1"); // request rejected
|
||||
return prop;
|
||||
}
|
||||
|
||||
if ((sb.isRobinsonMode()) &&
|
||||
(!((sb.isPublicRobinson()) ||
|
||||
(sb.isInMyCluster((String)header.get(httpHeader.CONNECTION_PROP_CLIENTIP)))))) {
|
||||
// if we are a robinson cluster, answer only if this client is known by our network definition
|
||||
prop.put("response", "-1"); // request rejected
|
||||
final plasmaSwitchboard sb = (plasmaSwitchboard) env;
|
||||
if (sb.isRobinsonMode() &&
|
||||
!(sb.isPublicRobinson() ||
|
||||
sb.isInMyCluster((String)header.get(httpHeader.CONNECTION_PROP_CLIENTIP)))) {
|
||||
// if we are a robinson cluster, answer only if this client is known by our network definition
|
||||
return prop;
|
||||
}
|
||||
|
||||
|
||||
prop.put("messagesize", Integer.toString(messagesize));
|
||||
prop.put("attachmentsize", Integer.toString(attachmentsize));
|
||||
|
||||
|
|
|
@ -64,29 +64,32 @@ public final class profile {
|
|||
|
||||
public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch env) {
|
||||
// return variable that accumulates replacements
|
||||
serverObjects prop = new serverObjects();
|
||||
plasmaSwitchboard sb = (plasmaSwitchboard) env;
|
||||
if ((post == null) || (env == null)) return prop;
|
||||
if (!yacyNetwork.authentifyRequest(post, env)) return prop;
|
||||
final serverObjects prop = new serverObjects();
|
||||
// defaults
|
||||
prop.put("list", "0");
|
||||
|
||||
if ((sb.isRobinsonMode()) &&
|
||||
(!sb.isPublicRobinson()) &&
|
||||
(!sb.isInMyCluster((String)header.get(httpHeader.CONNECTION_PROP_CLIENTIP)))) {
|
||||
// if we are a robinson cluster, answer only if this client is known by our network definition
|
||||
prop.put("list", "0");
|
||||
if (post == null || env == null || !yacyNetwork.authentifyRequest(post, env)) {
|
||||
return prop;
|
||||
}
|
||||
|
||||
|
||||
final plasmaSwitchboard sb = (plasmaSwitchboard) env;
|
||||
if (sb.isRobinsonMode() &&
|
||||
!sb.isPublicRobinson() &&
|
||||
!sb.isInMyCluster((String) header.get(httpHeader.CONNECTION_PROP_CLIENTIP))) {
|
||||
// if we are a robinson cluster, answer only if this client is known by our network definition
|
||||
return prop;
|
||||
}
|
||||
|
||||
Properties profile = new Properties();
|
||||
int count=0;
|
||||
String key="";
|
||||
String value="";
|
||||
int count = 0;
|
||||
String key = "";
|
||||
String value = "";
|
||||
|
||||
FileInputStream fileIn = null;
|
||||
try {
|
||||
fileIn = new FileInputStream(new File("DATA/SETTINGS/profile.txt"));
|
||||
profile.load(fileIn);
|
||||
} catch(IOException e) {
|
||||
profile.load(fileIn);
|
||||
} catch (IOException e) {
|
||||
} finally {
|
||||
if (fileIn != null) try { fileIn.close(); fileIn = null; } catch (Exception e) {}
|
||||
}
|
||||
|
|
|
@ -57,41 +57,41 @@ import de.anomic.yacy.yacySeed;
|
|||
|
||||
public final class query {
|
||||
|
||||
public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch ss) {
|
||||
if (post == null || ss == null) { return null; }
|
||||
|
||||
public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch env) {
|
||||
// return variable that accumulates replacements
|
||||
final plasmaSwitchboard sb = (plasmaSwitchboard) ss;
|
||||
final serverObjects prop = new serverObjects();
|
||||
if ((post == null) || (ss == null)) return prop;
|
||||
if (!yacyNetwork.authentifyRequest(post, ss)) return prop;
|
||||
|
||||
|
||||
if ((sb.isRobinsonMode()) &&
|
||||
(!sb.isPublicRobinson()) &&
|
||||
(!sb.isInMyCluster((String)header.get(httpHeader.CONNECTION_PROP_CLIENTIP)))) {
|
||||
// if we are a robinson cluster, answer only if we are public robinson peers,
|
||||
// or we are a private cluster and the requester is in our cluster.
|
||||
// if we don't answer, the remote peer will recognize us as junior peer,
|
||||
// what would mean that our peer ping does not work
|
||||
prop.put("response", "-1"); // request rejected
|
||||
|
||||
if (post == null || env == null || !yacyNetwork.authentifyRequest(post, env)) {
|
||||
return prop;
|
||||
}
|
||||
|
||||
|
||||
final plasmaSwitchboard sb = (plasmaSwitchboard) env;
|
||||
if (sb.isRobinsonMode() &&
|
||||
!sb.isPublicRobinson() &&
|
||||
!sb.isInMyCluster((String) header.get(httpHeader.CONNECTION_PROP_CLIENTIP))) {
|
||||
// if we are a robinson cluster, answer only if we are public robinson peers,
|
||||
// or we are a private cluster and the requester is in our cluster.
|
||||
// if we don't answer, the remote peer will recognize us as junior peer,
|
||||
// what would mean that our peer ping does not work
|
||||
prop.put("response", "-1"); // request rejected
|
||||
return prop;
|
||||
}
|
||||
|
||||
// System.out.println("YACYQUERY: RECEIVED POST = " + ((post == null) ? "NULL" : post.toString()));
|
||||
|
||||
final String iam = post.get("iam", ""); // complete seed of the requesting peer
|
||||
final String ohash = post.get("iam", ""); // complete seed of the requesting peer
|
||||
final String youare = post.get("youare", ""); // seed hash of the target peer, used for testing network stability
|
||||
// final String key = post.get("key", ""); // transmission key for response
|
||||
final String obj = post.get("object", ""); // keyword for query subject
|
||||
final String env = post.get("env", ""); // argument to query
|
||||
final String qenv = post.get("env", ""); // argument to query
|
||||
|
||||
final yacySeed otherPeer = yacyCore.seedDB.get(iam);
|
||||
if (otherPeer == null) {
|
||||
final yacySeed oseed = yacyCore.seedDB.get(ohash);
|
||||
if (oseed == null) {
|
||||
prop.put("response", "0");
|
||||
return prop;
|
||||
} else {
|
||||
otherPeer.setLastSeenUTC();
|
||||
oseed.setFlagDirectConnect(true);
|
||||
oseed.setLastSeenUTC();
|
||||
}
|
||||
|
||||
prop.put("mytime", serverDate.formatShortSecond());
|
||||
|
@ -107,7 +107,7 @@ public final class query {
|
|||
if (obj.equals("rwiurlcount")) {
|
||||
// the total number of different urls in the rwi is returned
|
||||
// <env> shall contain a word hash, the number of assigned lurls to this hash is returned
|
||||
prop.put("response", sb.wordIndex.indexSize(env));
|
||||
prop.put("response", sb.wordIndex.indexSize(qenv));
|
||||
return prop;
|
||||
}
|
||||
|
||||
|
|
|
@ -59,7 +59,7 @@ public final class search {
|
|||
|
||||
public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch env) {
|
||||
// return variable that accumulates replacements
|
||||
serverObjects prop = new serverObjects();
|
||||
final serverObjects prop = new serverObjects();
|
||||
// defaults
|
||||
prop.put("links", "");
|
||||
prop.put("linkcount", "0");
|
||||
|
@ -86,8 +86,8 @@ public final class search {
|
|||
return prop;
|
||||
}
|
||||
|
||||
final String query = post.get("query", ""); // a string of word hashes that shall be searched and combined
|
||||
String abstracts = post.get("abstracts", ""); // a string of word hashes for abstracts that shall be generated, or 'auto' (for maxcount-word), or '' (for none)
|
||||
final String query = post.get("query", ""); // a string of word hashes that shall be searched and combined
|
||||
final String abstracts = post.get("abstracts", ""); // a string of word hashes for abstracts that shall be generated, or 'auto' (for maxcount-word), or '' (for none)
|
||||
if ((query == null || query.length() == 0) & (abstracts == null || abstracts.length() == 0)) {
|
||||
return prop;
|
||||
}
|
||||
|
@ -96,9 +96,8 @@ public final class search {
|
|||
sb.intermissionAllThreads(3000);
|
||||
sb.remoteSearchLastAccess = System.currentTimeMillis();
|
||||
|
||||
final String oseed = post.get("myseed", ""); // complete seed of the requesting peer
|
||||
final String key = post.get("key", ""); // transmission key for response
|
||||
final yacySeed opeer = yacySeed.genRemoteSeed(oseed, key, true);
|
||||
// myseed = complete seed of the requesting peer, key = transmission key for response
|
||||
final yacySeed opeer = yacySeed.genRemoteSeed(post.get("myseed", ""), post.get("key", ""), true);
|
||||
// store accessing peer
|
||||
if (yacyCore.seedDB == null) {
|
||||
yacyCore.log.logSevere("yacy.search: seed cache not initialized");
|
||||
|
|
|
@ -61,11 +61,13 @@ import de.anomic.yacy.yacySeed;
|
|||
public final class transfer {
|
||||
|
||||
public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch env) {
|
||||
plasmaSwitchboard sb = (plasmaSwitchboard) env;
|
||||
serverObjects prop = new serverObjects();
|
||||
if ((post == null) || (env == null)) return prop;
|
||||
if (!yacyNetwork.authentifyRequest(post, env)) return prop;
|
||||
|
||||
// return variable that accumulates replacements
|
||||
final serverObjects prop = new serverObjects();
|
||||
|
||||
if (post == null || env == null || !yacyNetwork.authentifyRequest(post, env)) {
|
||||
return prop;
|
||||
}
|
||||
|
||||
String process = post.get("process", ""); // permission or store
|
||||
// String key = post.get("key", ""); // a transmission key from the client
|
||||
String ohash = post.get("iam", ""); // identification of the client (a peer-hash)
|
||||
|
@ -81,22 +83,24 @@ public final class transfer {
|
|||
prop.put("process_path", "");
|
||||
prop.put("process_maxsize", "0");
|
||||
|
||||
final plasmaSwitchboard sb = (plasmaSwitchboard) env;
|
||||
if (sb.isRobinsonMode() || !sb.rankingOn) {
|
||||
// in a robinson environment, do not answer. We do not do any transfer in a robinson cluster.
|
||||
return prop;
|
||||
}
|
||||
|
||||
final yacySeed opeer = yacyCore.seedDB.get(ohash);
|
||||
if (opeer == null) {
|
||||
final yacySeed oseed = yacyCore.seedDB.get(ohash);
|
||||
if (oseed == null) {
|
||||
// reject unknown peers: this does not appear fair, but anonymous senders are dangerous
|
||||
sb.getLog().logFine("RankingTransmission: rejected unknown peer '" + ohash + "', current IP " + header.get(httpHeader.CONNECTION_PROP_CLIENTIP, "unknown"));
|
||||
return prop;
|
||||
}
|
||||
opeer.setLastSeenUTC();
|
||||
oseed.setFlagDirectConnect(true);
|
||||
oseed.setLastSeenUTC();
|
||||
|
||||
if (filename.indexOf("..") >= 0) {
|
||||
// reject paths that contain '..' because they are dangerous
|
||||
sb.getLog().logFine("RankingTransmission: rejected wrong path '" + filename + "' from peer " + opeer.getName() + "/" + opeer.getPublicAddress()+ ", current IP " + header.get(httpHeader.CONNECTION_PROP_CLIENTIP, "unknown"));
|
||||
sb.getLog().logFine("RankingTransmission: rejected wrong path '" + filename + "' from peer " + oseed.getName() + "/" + oseed.getPublicAddress()+ ", current IP " + header.get(httpHeader.CONNECTION_PROP_CLIENTIP, "unknown"));
|
||||
return prop;
|
||||
}
|
||||
|
||||
|
@ -114,7 +118,7 @@ public final class transfer {
|
|||
prop.put("process_path", ""); // currently empty; the store process will find a path
|
||||
prop.put("process_maxsize", "-1"); // if response is too big we return the size of the file
|
||||
sb.rankingPermissions.put(serverCodings.encodeMD5Hex(kelondroBase64Order.standardCoder.encodeString(access)), filename);
|
||||
sb.getLog().logFine("RankingTransmission: granted peer " + opeer.hash + ":" + opeer.getName() + " to send CR file " + filename);
|
||||
sb.getLog().logFine("RankingTransmission: granted peer " + oseed.hash + ":" + oseed.getName() + " to send CR file " + filename);
|
||||
}
|
||||
return prop;
|
||||
}
|
||||
|
@ -132,7 +136,7 @@ public final class transfer {
|
|||
if ((grantedFile == null) || (!(grantedFile.equals(filename)))) {
|
||||
// fraud-access of this interface
|
||||
prop.put("response", "denied");
|
||||
sb.getLog().logFine("RankingTransmission: denied " + opeer.hash + ":" + opeer.getName() + " to send CR file " + filename + ": wrong access code");
|
||||
sb.getLog().logFine("RankingTransmission: denied " + oseed.hash + ":" + oseed.getName() + " to send CR file " + filename + ": wrong access code");
|
||||
} else {
|
||||
sb.rankingPermissions.remove(accesscode); // not needed any more
|
||||
File path = new File(sb.rankingPath, plasmaRankingDistribution.CR_OTHER);
|
||||
|
@ -144,10 +148,10 @@ public final class transfer {
|
|||
String md5t = serverCodings.encodeMD5Hex(file);
|
||||
if (md5t.equals(md5)) {
|
||||
prop.put("response", "ok");
|
||||
sb.getLog().logFine("RankingTransmission: received from peer " + opeer.hash + ":" + opeer.getName() + " CR file " + filename);
|
||||
sb.getLog().logFine("RankingTransmission: received from peer " + oseed.hash + ":" + oseed.getName() + " CR file " + filename);
|
||||
} else {
|
||||
prop.put("response", "transfer failure");
|
||||
sb.getLog().logFine("RankingTransmission: transfer failure from peer " + opeer.hash + ":" + opeer.getName() + " for CR file " + filename);
|
||||
sb.getLog().logFine("RankingTransmission: transfer failure from peer " + oseed.hash + ":" + oseed.getName() + " for CR file " + filename);
|
||||
}
|
||||
}else{
|
||||
//exploit?
|
||||
|
@ -163,7 +167,7 @@ public final class transfer {
|
|||
}
|
||||
|
||||
// wrong access
|
||||
sb.getLog().logFine("RankingTransmission: rejected unknown process " + process + ":" + purpose + " from peer " + opeer.hash + ":" + opeer.getName());
|
||||
sb.getLog().logFine("RankingTransmission: rejected unknown process " + process + ":" + purpose + " from peer " + oseed.hash + ":" + oseed.getName());
|
||||
return prop;
|
||||
}
|
||||
|
||||
|
|
|
@ -67,55 +67,59 @@ import de.anomic.yacy.yacySeed;
|
|||
public final class transferRWI {
|
||||
|
||||
public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch env) throws InterruptedException {
|
||||
|
||||
// return variable that accumulates replacements
|
||||
final plasmaSwitchboard sb = (plasmaSwitchboard) env;
|
||||
final serverObjects prop = new serverObjects();
|
||||
if ((post == null) || (env == null)) return prop;
|
||||
if (!yacyNetwork.authentifyRequest(post, env)) return prop;
|
||||
|
||||
if (post == null || env == null || !yacyNetwork.authentifyRequest(post, env)) {
|
||||
return prop;
|
||||
}
|
||||
|
||||
if (!post.containsKey("wordc")) return prop;
|
||||
if (!post.containsKey("entryc")) return prop;
|
||||
|
||||
|
||||
// request values
|
||||
final String iam = post.get("iam", ""); // seed hash of requester
|
||||
final String youare = post.get("youare", ""); // seed hash of the target peer, needed for network stability
|
||||
// final String key = (String) post.get("key", ""); // transmission key
|
||||
final int wordc = post.getInt("wordc", 0); // number of different words
|
||||
final int entryc = post.getInt("entryc", 0); // number of entries in indexes
|
||||
byte[] indexes = post.get("indexes", "").getBytes(); // the indexes, as list of word entries
|
||||
final String ohash = post.get("iam", ""); // seed hash of requester
|
||||
final String youare = post.get("youare", ""); // seed hash of the target peer, needed for network stability
|
||||
// final String key = (String) post.get("key", ""); // transmission key
|
||||
final int wordc = post.getInt("wordc", 0); // number of different words
|
||||
final int entryc = post.getInt("entryc", 0); // number of entries in indexes
|
||||
byte[] indexes = post.get("indexes", "").getBytes(); // the indexes, as list of word entries
|
||||
|
||||
final plasmaSwitchboard sb = (plasmaSwitchboard) env;
|
||||
boolean granted = sb.getConfig("allowReceiveIndex", "false").equals("true");
|
||||
boolean blockBlacklist = sb.getConfig("indexReceiveBlockBlacklist", "false").equals("true");
|
||||
boolean checkLimit = sb.getConfigBool("indexDistribution.transferRWIReceiptLimitEnabled", true);
|
||||
final long cachelimit = sb.getConfigLong("indexDistribution.dhtReceiptLimit", 10000);
|
||||
|
||||
final yacySeed otherPeer = yacyCore.seedDB.get(iam);
|
||||
if (otherPeer == null) {
|
||||
final yacySeed oseed = yacyCore.seedDB.get(ohash);
|
||||
if (oseed == null) {
|
||||
prop.put("unknownURL", "");
|
||||
prop.put("result", "busy");
|
||||
prop.put("pause", "120000");
|
||||
return prop;
|
||||
} else {
|
||||
otherPeer.setLastSeenUTC();
|
||||
oseed.setFlagDirectConnect(true);
|
||||
oseed.setLastSeenUTC();
|
||||
}
|
||||
final String otherPeerName = iam + ":" + otherPeer.getName() + "/" + otherPeer.getVersion();
|
||||
final String oname = ohash + ":" + oseed.getName() + "/" + oseed.getVersion();
|
||||
|
||||
// response values
|
||||
String result = "ok";
|
||||
StringBuffer unknownURLs = new StringBuffer();
|
||||
int pause = 10000;
|
||||
|
||||
|
||||
if ((youare == null) || (!youare.equals(yacyCore.seedDB.mySeed().hash))) {
|
||||
sb.getLog().logInfo("Rejecting RWIs from peer " + otherPeerName + ". Wrong target. Wanted peer=" + youare + ", iam=" + yacyCore.seedDB.mySeed().hash);
|
||||
sb.getLog().logInfo("Rejecting RWIs from peer " + oname + ". Wrong target. Wanted peer=" + youare + ", iam=" + yacyCore.seedDB.mySeed().hash);
|
||||
result = "wrong_target";
|
||||
pause = 0;
|
||||
} else if ((!granted) || (sb.isRobinsonMode())) {
|
||||
// we dont want to receive indexes
|
||||
sb.getLog().logInfo("Rejecting RWIs from peer " + otherPeerName + ". Not granted.");
|
||||
sb.getLog().logInfo("Rejecting RWIs from peer " + oname + ". Not granted.");
|
||||
result = "not_granted";
|
||||
pause = 0;
|
||||
} else if (checkLimit && sb.wordIndex.dhtInCacheSize() > cachelimit) {
|
||||
// we are too busy to receive indexes
|
||||
sb.getLog().logInfo("Rejecting RWIs from peer " + otherPeerName + ". We are too busy (buffersize=" + sb.wordIndex.dhtInCacheSize() + ").");
|
||||
sb.getLog().logInfo("Rejecting RWIs from peer " + oname + ". We are too busy (buffersize=" + sb.wordIndex.dhtInCacheSize() + ").");
|
||||
granted = false; // don't accept more words if there are too many words to flush
|
||||
result = "busy";
|
||||
pause = 60000;
|
||||
|
@ -128,7 +132,7 @@ public final class transferRWI {
|
|||
} */ else {
|
||||
// we want and can receive indexes
|
||||
// log value status (currently added to find outOfMemory error
|
||||
sb.getLog().logFine("Processing " + indexes.length + " bytes / " + wordc + " words / " + entryc + " entries from " + otherPeerName);
|
||||
sb.getLog().logFine("Processing " + indexes.length + " bytes / " + wordc + " words / " + entryc + " entries from " + oname);
|
||||
final long startProcess = System.currentTimeMillis();
|
||||
|
||||
// decode request
|
||||
|
@ -171,7 +175,7 @@ public final class transferRWI {
|
|||
// block blacklisted entries
|
||||
if ((blockBlacklist) && (plasmaSwitchboard.urlBlacklist.hashInBlacklistedCache(plasmaURLPattern.BLACKLIST_DHT, urlHash))) {
|
||||
int deleted = sb.wordIndex.tryRemoveURLs(urlHash);
|
||||
yacyCore.log.logFine("transferRWI: blocked blacklisted URLHash '" + urlHash + "' from peer " + otherPeerName + "; deleted " + deleted + " URL entries from RWIs");
|
||||
yacyCore.log.logFine("transferRWI: blocked blacklisted URLHash '" + urlHash + "' from peer " + oname + "; deleted " + deleted + " URL entries from RWIs");
|
||||
blocked++;
|
||||
continue;
|
||||
}
|
||||
|
@ -205,10 +209,10 @@ public final class transferRWI {
|
|||
}
|
||||
if (unknownURLs.length() > 0) { unknownURLs.delete(0, 1); }
|
||||
if ((wordhashes.length == 0) || (received == 0)) {
|
||||
sb.getLog().logInfo("Received 0 RWIs from " + otherPeerName + ", processed in " + (System.currentTimeMillis() - startProcess) + " milliseconds, requesting " + unknownURL.size() + " URLs, blocked " + blocked + " RWIs");
|
||||
sb.getLog().logInfo("Received 0 RWIs from " + oname + ", processed in " + (System.currentTimeMillis() - startProcess) + " milliseconds, requesting " + unknownURL.size() + " URLs, blocked " + blocked + " RWIs");
|
||||
} else {
|
||||
final double avdist = (yacyDHTAction.dhtDistance(yacyCore.seedDB.mySeed().hash, wordhashes[0]) + yacyDHTAction.dhtDistance(yacyCore.seedDB.mySeed().hash, wordhashes[received - 1])) / 2.0;
|
||||
sb.getLog().logInfo("Received " + received + " Entries " + wordc + " Words [" + wordhashes[0] + " .. " + wordhashes[received - 1] + "]/" + avdist + " from " + otherPeerName + ", processed in " + (System.currentTimeMillis() - startProcess) + " milliseconds, requesting " + unknownURL.size() + "/" + receivedURL + " URLs, blocked " + blocked + " RWIs");
|
||||
sb.getLog().logInfo("Received " + received + " Entries " + wordc + " Words [" + wordhashes[0] + " .. " + wordhashes[received - 1] + "]/" + avdist + " from " + oname + ", processed in " + (System.currentTimeMillis() - startProcess) + " milliseconds, requesting " + unknownURL.size() + "/" + receivedURL + " URLs, blocked " + blocked + " RWIs");
|
||||
}
|
||||
result = "ok";
|
||||
|
||||
|
|
|
@ -62,45 +62,48 @@ import de.anomic.yacy.yacySeed;
|
|||
|
||||
public final class transferURL {
|
||||
|
||||
|
||||
public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch env) throws InterruptedException {
|
||||
// return variable that accumulates replacements
|
||||
final serverObjects prop = new serverObjects();
|
||||
|
||||
if (post == null || env == null || !yacyNetwork.authentifyRequest(post, env)) {
|
||||
return prop;
|
||||
}
|
||||
|
||||
long start = System.currentTimeMillis();
|
||||
long freshdate = 0;
|
||||
try {freshdate = serverDate.parseShortDay("20061101").getTime();} catch (ParseException e1) {}
|
||||
|
||||
// return variable that accumulates replacements
|
||||
final plasmaSwitchboard sb = (plasmaSwitchboard) env;
|
||||
final serverObjects prop = new serverObjects();
|
||||
if ((post == null) || (env == null)) return prop;
|
||||
if (!yacyNetwork.authentifyRequest(post, env)) return prop;
|
||||
|
||||
// request values
|
||||
final String iam = post.get("iam", ""); // seed hash of requester
|
||||
final String youare = post.get("youare", ""); // seed hash of the target peer, needed for network stability
|
||||
// final String key = post.get("key", ""); // transmission key
|
||||
final int urlc = post.getInt("urlc", 0); // number of transported urls
|
||||
final String ohash = post.get("iam", ""); // seed hash of requester
|
||||
final String youare = post.get("youare", ""); // seed hash of the target peer, needed for network stability
|
||||
// final String key = post.get("key", ""); // transmission key
|
||||
final int urlc = post.getInt("urlc", 0); // number of transported urls
|
||||
|
||||
final plasmaSwitchboard sb = (plasmaSwitchboard) env;
|
||||
final boolean granted = sb.getConfig("allowReceiveIndex", "false").equals("true");
|
||||
final boolean blockBlacklist = sb.getConfig("indexReceiveBlockBlacklist", "false").equals("true");
|
||||
|
||||
final yacySeed otherPeer = yacyCore.seedDB.get(iam);
|
||||
if (otherPeer == null) {
|
||||
final yacySeed oseed = yacyCore.seedDB.get(ohash);
|
||||
if (oseed == null) {
|
||||
prop.put("result", "error_not_granted");
|
||||
prop.put("pause", "120000");
|
||||
return prop;
|
||||
} else {
|
||||
otherPeer.setLastSeenUTC();
|
||||
oseed.setFlagDirectConnect(true);
|
||||
oseed.setLastSeenUTC();
|
||||
}
|
||||
final String otherPeerName = iam + ":" + otherPeer.getName() + "/" + otherPeer.getVersion();
|
||||
final String oname = ohash + ":" + oseed.getName() + "/" + oseed.getVersion();
|
||||
|
||||
// response values
|
||||
String result = "";
|
||||
String doublevalues = "0";
|
||||
|
||||
if ((youare == null) || (!youare.equals(yacyCore.seedDB.mySeed().hash))) {
|
||||
sb.getLog().logInfo("Rejecting URLs from peer " + otherPeerName + ". Wrong target. Wanted peer=" + youare + ", iam=" + yacyCore.seedDB.mySeed().hash);
|
||||
sb.getLog().logInfo("Rejecting URLs from peer " + oname + ". Wrong target. Wanted peer=" + youare + ", iam=" + yacyCore.seedDB.mySeed().hash);
|
||||
result = "wrong_target";
|
||||
} else if ((!granted) || (sb.isRobinsonMode())) {
|
||||
sb.getLog().logInfo("Rejecting URLs from peer " + otherPeerName + ". Not granted.");
|
||||
sb.getLog().logInfo("Rejecting URLs from peer " + oname + ". Not granted.");
|
||||
result = "error_not_granted";
|
||||
} else {
|
||||
int received = 0;
|
||||
|
@ -115,7 +118,7 @@ public final class transferURL {
|
|||
// read new lurl-entry
|
||||
urls = (String) post.get("url" + i);
|
||||
if (urls == null) {
|
||||
yacyCore.log.logFine("transferURL: got null URL-string from peer " + otherPeerName);
|
||||
yacyCore.log.logFine("transferURL: got null URL-string from peer " + oname);
|
||||
blocked++;
|
||||
continue;
|
||||
}
|
||||
|
@ -123,7 +126,7 @@ public final class transferURL {
|
|||
// parse new lurl-entry
|
||||
lEntry = sb.wordIndex.loadedURL.newEntry(urls);
|
||||
if (lEntry == null) {
|
||||
yacyCore.log.logWarning("transferURL: received invalid URL (entry null) from peer " + otherPeerName + "\n\tURL Property: " + urls);
|
||||
yacyCore.log.logWarning("transferURL: received invalid URL (entry null) from peer " + oname + "\n\tURL Property: " + urls);
|
||||
blocked++;
|
||||
continue;
|
||||
}
|
||||
|
@ -131,14 +134,14 @@ public final class transferURL {
|
|||
// check if entry is well-formed
|
||||
indexURLEntry.Components comp = lEntry.comp();
|
||||
if (comp.url() == null) {
|
||||
yacyCore.log.logWarning("transferURL: received invalid URL from peer " + otherPeerName + "\n\tURL Property: " + urls);
|
||||
yacyCore.log.logWarning("transferURL: received invalid URL from peer " + oname + "\n\tURL Property: " + urls);
|
||||
blocked++;
|
||||
continue;
|
||||
}
|
||||
|
||||
// check whether entry is too old
|
||||
if (lEntry.freshdate().getTime() <= freshdate) {
|
||||
yacyCore.log.logFine("transerURL: received too old URL from peer " + otherPeerName + ": " + lEntry.freshdate());
|
||||
yacyCore.log.logFine("transerURL: received too old URL from peer " + oname + ": " + lEntry.freshdate());
|
||||
blocked++;
|
||||
continue;
|
||||
}
|
||||
|
@ -146,7 +149,7 @@ public final class transferURL {
|
|||
// check if the entry is blacklisted
|
||||
if ((blockBlacklist) && (plasmaSwitchboard.urlBlacklist.isListed(plasmaURLPattern.BLACKLIST_DHT, comp.url()))) {
|
||||
int deleted = sb.wordIndex.tryRemoveURLs(lEntry.hash());
|
||||
yacyCore.log.logFine("transferURL: blocked blacklisted URL '" + comp.url().toNormalform(false, true) + "' from peer " + otherPeerName + "; deleted " + deleted + " URL entries from RWIs");
|
||||
yacyCore.log.logFine("transferURL: blocked blacklisted URL '" + comp.url().toNormalform(false, true) + "' from peer " + oname + "; deleted " + deleted + " URL entries from RWIs");
|
||||
lEntry = null;
|
||||
blocked++;
|
||||
continue;
|
||||
|
@ -154,7 +157,7 @@ public final class transferURL {
|
|||
|
||||
// check if the entry is in our network domain
|
||||
if (!sb.acceptURL(comp.url())) {
|
||||
yacyCore.log.logFine("transferURL: blocked URL outside of our domain '" + comp.url().toNormalform(false, true) + "' from peer " + otherPeerName);
|
||||
yacyCore.log.logFine("transferURL: blocked URL outside of our domain '" + comp.url().toNormalform(false, true) + "' from peer " + oname);
|
||||
lEntry = null;
|
||||
blocked++;
|
||||
continue;
|
||||
|
@ -163,8 +166,8 @@ public final class transferURL {
|
|||
// write entry to database
|
||||
try {
|
||||
sb.wordIndex.loadedURL.store(lEntry);
|
||||
sb.wordIndex.loadedURL.stack(lEntry, iam, iam, 3);
|
||||
yacyCore.log.logFine("transferURL: received URL '" + comp.url().toNormalform(false, true) + "' from peer " + otherPeerName);
|
||||
sb.wordIndex.loadedURL.stack(lEntry, ohash, ohash, 3);
|
||||
yacyCore.log.logFine("transferURL: received URL '" + comp.url().toNormalform(false, true) + "' from peer " + oname);
|
||||
received++;
|
||||
} catch (IOException e) {
|
||||
e.printStackTrace();
|
||||
|
@ -176,8 +179,8 @@ public final class transferURL {
|
|||
// return rewrite properties
|
||||
final int more = sb.wordIndex.loadedURL.size() - sizeBefore;
|
||||
doublevalues = Integer.toString(received - more);
|
||||
sb.getLog().logInfo("Received " + received + " URLs from peer " + otherPeerName + " in " + (System.currentTimeMillis() - start) + " ms, Blocked " + blocked + " URLs");
|
||||
if ((received - more) > 0) sb.getLog().logSevere("Received " + doublevalues + " double URLs from peer " + otherPeerName);
|
||||
sb.getLog().logInfo("Received " + received + " URLs from peer " + oname + " in " + (System.currentTimeMillis() - start) + " ms, Blocked " + blocked + " URLs");
|
||||
if ((received - more) > 0) sb.getLog().logSevere("Received " + doublevalues + " double URLs from peer " + oname);
|
||||
result = "ok";
|
||||
}
|
||||
|
||||
|
|
|
@ -259,6 +259,7 @@ public final class yacyClient {
|
|||
);
|
||||
|
||||
if (result == null || result.size() == 0) { return null; }
|
||||
target.setFlagDirectConnect(true);
|
||||
target.setLastSeenUTC();
|
||||
//final Date remoteTime = yacyCore.parseUniversalDate((String) result.get(yacySeed.MYTIME)); // read remote time
|
||||
return yacySeed.genRemoteSeed((String) result.get("response"), post.get("key", ""), true);
|
||||
|
@ -294,6 +295,7 @@ public final class yacyClient {
|
|||
if (resp == null) {
|
||||
return -1;
|
||||
} else try {
|
||||
target.setFlagDirectConnect(true);
|
||||
target.setLastSeenUTC();
|
||||
return Integer.parseInt(resp);
|
||||
} catch (NumberFormatException e) {
|
||||
|
@ -334,6 +336,7 @@ public final class yacyClient {
|
|||
if (resp == null) {
|
||||
return -1;
|
||||
} else try {
|
||||
target.setFlagDirectConnect(true);
|
||||
target.setLastSeenUTC();
|
||||
return Integer.parseInt(resp);
|
||||
} catch (NumberFormatException e) {
|
||||
|
@ -862,7 +865,7 @@ public final class yacyClient {
|
|||
}
|
||||
}
|
||||
|
||||
public static HashMap<String, Object> transferIndex(yacySeed targetSeed, indexContainer[] indexes, HashMap<String, indexURLEntry> urlCache, boolean gzipBody, int timeout) {
|
||||
public static HashMap<String, Object> transferIndex(yacySeed target, indexContainer[] indexes, HashMap<String, indexURLEntry> urlCache, boolean gzipBody, int timeout) {
|
||||
|
||||
HashMap<String, Object> resultObj = new HashMap<String, Object>();
|
||||
int payloadSize = 0;
|
||||
|
@ -882,7 +885,7 @@ public final class yacyClient {
|
|||
}
|
||||
|
||||
// transfer the RWI without the URLs
|
||||
HashMap<String, String> in = transferRWI(targetSeed, indexes, gzipBody, timeout);
|
||||
HashMap<String, String> in = transferRWI(target, indexes, gzipBody, timeout);
|
||||
resultObj.put("resultTransferRWI", in);
|
||||
|
||||
if (in == null) {
|
||||
|
@ -897,11 +900,9 @@ public final class yacyClient {
|
|||
return resultObj;
|
||||
}
|
||||
|
||||
targetSeed.setLastSeenUTC();
|
||||
|
||||
if (!(result.equals("ok"))) {
|
||||
targetSeed.setFlagAcceptRemoteIndex(false);
|
||||
yacyCore.seedDB.update(targetSeed.hash, targetSeed);
|
||||
target.setFlagAcceptRemoteIndex(false);
|
||||
yacyCore.seedDB.update(target.hash, target);
|
||||
resultObj.put("result", result);
|
||||
return resultObj;
|
||||
}
|
||||
|
@ -926,7 +927,7 @@ public final class yacyClient {
|
|||
}
|
||||
}
|
||||
|
||||
in = transferURL(targetSeed, urls, gzipBody, timeout);
|
||||
in = transferURL(target, urls, gzipBody, timeout);
|
||||
resultObj.put("resultTransferURL", in);
|
||||
|
||||
if (in == null) {
|
||||
|
@ -941,8 +942,8 @@ public final class yacyClient {
|
|||
return resultObj;
|
||||
}
|
||||
if (!(result.equals("ok"))) {
|
||||
targetSeed.setFlagAcceptRemoteIndex(false);
|
||||
yacyCore.seedDB.update(targetSeed.hash, targetSeed);
|
||||
target.setFlagAcceptRemoteIndex(false);
|
||||
yacyCore.seedDB.update(target.hash, target);
|
||||
resultObj.put("result",result);
|
||||
return resultObj;
|
||||
}
|
||||
|
|
|
@ -93,11 +93,11 @@ public class yacyCore {
|
|||
public static File yacyDBPath;
|
||||
public static final Map<String, yacyAccessible> amIAccessibleDB = Collections.synchronizedMap(new HashMap<String, yacyAccessible>()); // Holds PeerHash / yacyAccessible Relations
|
||||
// constants for PeerPing behaviour
|
||||
private static final int PING_INITIAL = 18;
|
||||
private static final int PING_INITIAL = 12;
|
||||
private static final int PING_MAX_RUNNING = 6;
|
||||
private static final int PING_MIN_RUNNING = 3;
|
||||
private static final int PING_MIN_DBSIZE = 5;
|
||||
private static final int PING_MIN_PEERSEEN = 1; // min. accessible to force senior
|
||||
private static final int PING_MIN_DBSIZE = 3;
|
||||
private static final int PING_MIN_PEERSEEN = 2; // min. accessible to force senior
|
||||
private static final long PING_MAX_DBAGE = 15 * 60 * 1000; // in milliseconds
|
||||
|
||||
// public static yacyShare shareManager = null;
|
||||
|
@ -438,7 +438,7 @@ public class yacyCore {
|
|||
|
||||
// going through the peer list and starting a new publisher thread for each peer
|
||||
int i = 0;
|
||||
while (si. hasNext()) {
|
||||
while (si.hasNext()) {
|
||||
seed = (yacySeed) si.next();
|
||||
if (seed == null) {
|
||||
sync.P();
|
||||
|
|
|
@ -815,7 +815,7 @@ public class yacySeed {
|
|||
|
||||
public final String genSeedStr(String key) {
|
||||
// use a default encoding
|
||||
return this.genSeedStr('b', key);
|
||||
return this.genSeedStr('z', key);
|
||||
}
|
||||
|
||||
public final synchronized String genSeedStr(char method, String key) {
|
||||
|
@ -838,7 +838,7 @@ public class yacySeed {
|
|||
}
|
||||
|
||||
public final void save(File f) throws IOException {
|
||||
final String out = this.genSeedStr('p', null);
|
||||
final String out = this.genSeedStr('z', null);
|
||||
final FileWriter fw = new FileWriter(f);
|
||||
fw.write(out, 0, out.length());
|
||||
fw.close();
|
||||
|
|
Loading…
Reference in New Issue
Block a user