Marown/yacy_search_server
1
0
Fork 0
mirror of https://github.com/yacy/yacy_search_server.git synced 2024-09-19 00:01:41 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity

seedDB helpers update/cleanup

Browse Source 
git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@4461 6c8d7289-2bf4-0310-a012-ef5d649a1542
This commit is contained in:
borg-0300 2008-02-08 14:06:34 +00:00
parent dd215e7f6b
commit 77ba446332
12 changed files with 214 additions and 201 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

68
htroot/yacy/crawlReceipt.java
View File

@ -62,25 +62,34 @@ import de.anomic.yacy.yacySeed;
public final class crawlReceipt {
/*
* this is used to respond on a remote crawling request
*/
public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch env) {
// return variable that accumulates replacements
plasmaSwitchboard switchboard = (plasmaSwitchboard) env;
serverObjects prop = new serverObjects();
if ((post == null) || (env == null)) return prop;
if (!yacyNetwork.authentifyRequest(post, env)) return prop;
serverLog log = switchboard.getLog();
final serverObjects prop = new serverObjects();
// defaults
prop.put("delay", "3600");
if (post == null || env == null || !yacyNetwork.authentifyRequest(post, env)) {
return prop;
}
// seed hash of requester
String ohash = post.get("iam", "");
if (ohash == null || ohash.length() == 0 ) return prop;
yacySeed oseed = yacyCore.seedDB.get(ohash);
if (oseed == null) return prop;
oseed.setFlagDirectConnect(true);
oseed.setLastSeenUTC();
final plasmaSwitchboard sb = (plasmaSwitchboard) env;
serverLog log = sb.getLog();
//int proxyPrefetchDepth = Integer.parseInt(env.getConfig("proxyPrefetchDepth", "0"));
//int crawlingDepth = Integer.parseInt(env.getConfig("crawlingDepth", "0"));
// request values
String iam = post.get("iam", ""); // seed hash of requester
String youare = post.get("youare", ""); // seed hash of the target peer, needed for network stability
//String process = post.get("process", ""); // process type
String key = post.get("key", ""); // transmission key
@ -111,59 +120,50 @@ public final class crawlReceipt {
*/
final yacySeed otherPeer = yacyCore.seedDB.get(iam);
if (otherPeer == null) {
prop.put("delay", "3600");
return prop;
} else {
otherPeer.setLastSeenUTC();
}
final String otherPeerName = iam + ":" + otherPeer.getName() + "/" + otherPeer.getVersion();
if ((yacyCore.seedDB.mySeed() == null) || (!(yacyCore.seedDB.mySeed().hash.equals(youare)))) {
// no yacy connection / unknown peers
prop.put("delay", "3600");
return prop;
}
if (propStr == null) {
// error with url / wrong key
prop.put("delay", "3600");
return prop;
}
if ((switchboard.isRobinsonMode()) && (!switchboard.isInMyCluster(otherPeer))) {
if ((sb.isRobinsonMode()) && (!sb.isInMyCluster(oseed))) {
// we reject urls that are from outside our cluster
prop.put("delay", "9999");
return prop; // ???
}
// generating a new loaded URL entry
indexURLEntry entry = switchboard.wordIndex.loadedURL.newEntry(propStr);
indexURLEntry entry = sb.wordIndex.loadedURL.newEntry(propStr);
if (entry == null) {
log.logWarning("crawlReceipt: RECEIVED wrong RECEIPT (entry null) from peer " + iam + "\n\tURL properties: "+ propStr);
prop.put("delay", "3600");
log.logWarning("crawlReceipt: RECEIVED wrong RECEIPT (entry null) from peer " + ohash + "\n\tURL properties: "+ propStr);
return prop;
}
indexURLEntry.Components comp = entry.comp();
if (comp.url() == null) {
log.logWarning("crawlReceipt: RECEIVED wrong RECEIPT (url null) for hash " + entry.hash() + " from peer " + iam + "\n\tURL properties: "+ propStr);
prop.put("delay", "3600");
log.logWarning("crawlReceipt: RECEIVED wrong RECEIPT (url null) for hash " + entry.hash() + " from peer " + ohash + "\n\tURL properties: "+ propStr);
return prop;
}
// check if the entry is in our network domain
if (!switchboard.acceptURL(comp.url())) {
log.logWarning("crawlReceipt: RECEIVED wrong RECEIPT (url outside of our domain) for hash " + entry.hash() + " from peer " + iam + "\n\tURL properties: "+ propStr);
if (!sb.acceptURL(comp.url())) {
log.logWarning("crawlReceipt: RECEIVED wrong RECEIPT (url outside of our domain) for hash " + entry.hash() + " from peer " + ohash + "\n\tURL properties: "+ propStr);
prop.put("delay", "9999");
return prop;
}
if (result.equals("fill")) try {
// put new entry into database
switchboard.wordIndex.loadedURL.store(entry);
switchboard.wordIndex.loadedURL.stack(entry, youare, iam, 1);
switchboard.crawlQueues.delegatedURL.remove(entry.hash()); // the delegated work has been done
sb.wordIndex.loadedURL.store(entry);
sb.wordIndex.loadedURL.stack(entry, youare, ohash, 1);
sb.crawlQueues.delegatedURL.remove(entry.hash()); // the delegated work has been done
final String otherPeerName = ohash + ":" + oseed.getName() + "/" + oseed.getVersion();
log.logInfo("crawlReceipt: RECEIVED RECEIPT from " + otherPeerName + " for URL " + entry.hash() + ":" + comp.url().toNormalform(false, true));
// ready for more
@ -171,16 +171,14 @@ public final class crawlReceipt {
return prop;
} catch (IOException e) {
e.printStackTrace();
prop.put("delay", "3600");
return prop;
}
switchboard.crawlQueues.delegatedURL.remove(entry.hash()); // the delegated work is transformed into an error case
plasmaCrawlZURL.Entry ee = switchboard.crawlQueues.errorURL.newEntry(entry.toBalancerEntry(), youare, null, 0, result + ":" + reason);
sb.crawlQueues.delegatedURL.remove(entry.hash()); // the delegated work is transformed into an error case
plasmaCrawlZURL.Entry ee = sb.crawlQueues.errorURL.newEntry(entry.toBalancerEntry(), youare, null, 0, result + ":" + reason);
ee.store();
switchboard.crawlQueues.errorURL.push(ee);
sb.crawlQueues.errorURL.push(ee);
//switchboard.noticeURL.remove(receivedUrlhash);
prop.put("delay", "3600");
return prop;
// return rewrite properties

59
htroot/yacy/hello.java
View File

@ -65,10 +65,9 @@ import de.anomic.yacy.yacyVersion;
public final class hello {
public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch env) throws InterruptedException {
plasmaSwitchboard sb = (plasmaSwitchboard) env;
// return variable that accumulates replacements
serverObjects prop = new serverObjects();
prop.put("message", "none");
if ((post == null) || (env == null)) {
if (post == null || env == null) {
prop.put("message", "no post or no enviroment");
return prop;
}
@ -76,7 +75,8 @@ public final class hello {
prop.put("message", "not in my network");
return prop;
}
prop.put("message", "none");
// final String iam = (String) post.get("iam", ""); // complete seed of the requesting peer
// final String mytime = (String) post.get(MYTIME, ""); //
final String key = post.get("key", ""); // transmission key for response
@ -90,14 +90,14 @@ public final class hello {
prop.put("message", "your seed is too long (" + seed.length() + ")");
return prop;
}
final yacySeed remoteSeed = yacySeed.genRemoteSeed(seed, key, false);
final yacySeed oseed = yacySeed.genRemoteSeed(seed, key, false);
// System.out.println("YACYHELLO: REMOTESEED=" + ((remoteSeed == null) ? "NULL" : remoteSeed.toString()));
if ((remoteSeed == null) || (remoteSeed.hash == null)) {
if (oseed == null || oseed.hash == null) {
prop.put("message", "cannot parse your seed");
return prop;
}
// final String properTest = remoteSeed.isProper();
// The remote peer might not know its IP yet, so don't abort if the IP check fails
// if ((properTest != null) && (! properTest.substring(0,1).equals("IP"))) { return null; }
@ -110,18 +110,19 @@ public final class hello {
return prop;
}
final String userAgent = (String) header.get(httpHeader.USER_AGENT, "<unknown>");
final String reportedip = remoteSeed.get(yacySeed.IP, "");
final String reportedPeerType = remoteSeed.get(yacySeed.PEERTYPE, yacySeed.PEERTYPE_JUNIOR);
final float clientversion = remoteSeed.getVersion();
final String reportedip = oseed.get(yacySeed.IP, "");
final String reportedPeerType = oseed.get(yacySeed.PEERTYPE, yacySeed.PEERTYPE_JUNIOR);
final float clientversion = oseed.getVersion();
if ((sb.isRobinsonMode()) && (!sb.isPublicRobinson())) {
// if we are a robinson cluster, answer only if this client is known by our network definition
final plasmaSwitchboard sb = (plasmaSwitchboard) env;
if (sb.isRobinsonMode() && !sb.isPublicRobinson()) {
// if we are a robinson cluster, answer only if this client is known by our network definition
prop.put("message", "I am robinson, I do not answer");
return prop;
}
int urls = -1;
if (sb.clusterhashes != null) remoteSeed.setAlternativeAddress((String) sb.clusterhashes.get(remoteSeed.hash));
if (sb.clusterhashes != null) oseed.setAlternativeAddress((String) sb.clusterhashes.get(oseed.hash));
// if the remote client has reported its own IP address and the client supports
// the port forwarding feature (if client version >= 0.383) then we try to
@ -131,8 +132,8 @@ public final class hello {
// try first the reportedip, since this may be a connect from a port-forwarding host
prop.put("yourip", reportedip);
remoteSeed.put(yacySeed.IP, reportedip);
urls = yacyClient.queryUrlCount(remoteSeed);
oseed.put(yacySeed.IP, reportedip);
urls = yacyClient.queryUrlCount(oseed);
} else {
prop.put("yourip", "unknown");
}
@ -150,41 +151,41 @@ public final class hello {
serverCore.checkInterruption();
prop.put("yourip", clientip);
remoteSeed.put(yacySeed.IP, clientip);
urls = yacyClient.queryUrlCount(remoteSeed);
oseed.put(yacySeed.IP, clientip);
urls = yacyClient.queryUrlCount(oseed);
}
}
// System.out.println("YACYHELLO: YOUR IP=" + clientip);
// set lastseen value (we have seen that peer, it contacted us!)
remoteSeed.setLastSeenUTC();
oseed.setLastSeenUTC();
// assign status
if (urls >= 0) {
if (remoteSeed.get(yacySeed.PEERTYPE, yacySeed.PEERTYPE_SENIOR) == null) {
if (oseed.get(yacySeed.PEERTYPE, yacySeed.PEERTYPE_SENIOR) == null) {
prop.put(yacySeed.YOURTYPE, yacySeed.PEERTYPE_SENIOR);
remoteSeed.put(yacySeed.PEERTYPE, yacySeed.PEERTYPE_SENIOR);
} else if (remoteSeed.get(yacySeed.PEERTYPE, yacySeed.PEERTYPE_PRINCIPAL).equals(yacySeed.PEERTYPE_PRINCIPAL)) {
oseed.put(yacySeed.PEERTYPE, yacySeed.PEERTYPE_SENIOR);
} else if (oseed.get(yacySeed.PEERTYPE, yacySeed.PEERTYPE_PRINCIPAL).equals(yacySeed.PEERTYPE_PRINCIPAL)) {
prop.put(yacySeed.YOURTYPE, yacySeed.PEERTYPE_PRINCIPAL);
} else {
prop.put(yacySeed.YOURTYPE, yacySeed.PEERTYPE_SENIOR);
remoteSeed.put(yacySeed.PEERTYPE, yacySeed.PEERTYPE_SENIOR);
oseed.put(yacySeed.PEERTYPE, yacySeed.PEERTYPE_SENIOR);
}
// connect the seed
yacyCore.peerActions.peerArrival(remoteSeed, true);
yacyCore.peerActions.peerArrival(oseed, true);
} else {
prop.put(yacySeed.YOURTYPE, yacySeed.PEERTYPE_JUNIOR);
yacyCore.peerActions.juniorConnects++; // update statistics
remoteSeed.put(yacySeed.PEERTYPE, yacySeed.PEERTYPE_JUNIOR);
yacyCore.log.logInfo("hello: responded remote junior peer '" + remoteSeed.getName() + "' from " + reportedip);
oseed.put(yacySeed.PEERTYPE, yacySeed.PEERTYPE_JUNIOR);
yacyCore.log.logInfo("hello: responded remote junior peer '" + oseed.getName() + "' from " + reportedip);
// no connection here, instead store junior in connection cache
if ((remoteSeed.hash != null) && (remoteSeed.isProper() == null)) {
yacyCore.peerActions.peerPing(remoteSeed);
if ((oseed.hash != null) && (oseed.isProper() == null)) {
yacyCore.peerActions.peerPing(oseed);
}
}
yacyCore.peerActions.setUserAgent(clientip, userAgent);
if (!((String)prop.get(yacySeed.YOURTYPE)).equals(reportedPeerType)) {
yacyCore.log.logInfo("hello: changing remote peer '" + remoteSeed.getName() +
yacyCore.log.logInfo("hello: changing remote peer '" + oseed.getName() +
"' [" + reportedip +
"] peerType from '" + reportedPeerType +
"' to '" + prop.get(yacySeed.YOURTYPE) + "'.");

28
htroot/yacy/message.java
View File

@ -73,16 +73,17 @@ public final class message {
}
public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch env) {
if (post == null || env == null) { return null; }
// return variable that accumulates replacements
plasmaSwitchboard sb = (plasmaSwitchboard) env;
serverObjects prop = new serverObjects();
if ((post == null) || (env == null)) return prop;
if (!yacyNetwork.authentifyRequest(post, env)) return prop;
final serverObjects prop = new serverObjects();
// defaults
prop.put("response", "-1"); // request rejected
if (post == null || env == null || !yacyNetwork.authentifyRequest(post, env)) {
return prop;
}
String process = post.get("process", "permission");
String key = post.get("key", "");
String key = post.get("key", "");
int messagesize = 10240;
int attachmentsize = 0;
@ -94,18 +95,17 @@ public final class message {
// check if we are the right target and requester has correct information about this peer
if ((yacyCore.seedDB.mySeed() == null) || (!(yacyCore.seedDB.mySeed().hash.equals(youare)))) {
// this request has a wrong target
prop.put("response", "-1"); // request rejected
return prop;
}
if ((sb.isRobinsonMode()) &&
(!((sb.isPublicRobinson()) ||
(sb.isInMyCluster((String)header.get(httpHeader.CONNECTION_PROP_CLIENTIP)))))) {
// if we are a robinson cluster, answer only if this client is known by our network definition
prop.put("response", "-1"); // request rejected
final plasmaSwitchboard sb = (plasmaSwitchboard) env;
if (sb.isRobinsonMode() &&
!(sb.isPublicRobinson() ||
sb.isInMyCluster((String)header.get(httpHeader.CONNECTION_PROP_CLIENTIP)))) {
// if we are a robinson cluster, answer only if this client is known by our network definition
return prop;
}
prop.put("messagesize", Integer.toString(messagesize));
prop.put("attachmentsize", Integer.toString(attachmentsize));

33
htroot/yacy/profile.java
View File

@ -64,29 +64,32 @@ public final class profile {
public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch env) {
// return variable that accumulates replacements
serverObjects prop = new serverObjects();
plasmaSwitchboard sb = (plasmaSwitchboard) env;
if ((post == null) || (env == null)) return prop;
if (!yacyNetwork.authentifyRequest(post, env)) return prop;
final serverObjects prop = new serverObjects();
// defaults
prop.put("list", "0");
if ((sb.isRobinsonMode()) &&
(!sb.isPublicRobinson()) &&
(!sb.isInMyCluster((String)header.get(httpHeader.CONNECTION_PROP_CLIENTIP)))) {
// if we are a robinson cluster, answer only if this client is known by our network definition
prop.put("list", "0");
if (post == null || env == null || !yacyNetwork.authentifyRequest(post, env)) {
return prop;
}
final plasmaSwitchboard sb = (plasmaSwitchboard) env;
if (sb.isRobinsonMode() &&
!sb.isPublicRobinson() &&
!sb.isInMyCluster((String) header.get(httpHeader.CONNECTION_PROP_CLIENTIP))) {
// if we are a robinson cluster, answer only if this client is known by our network definition
return prop;
}
Properties profile = new Properties();
int count=0;
String key="";
String value="";
int count = 0;
String key = "";
String value = "";
FileInputStream fileIn = null;
try {
fileIn = new FileInputStream(new File("DATA/SETTINGS/profile.txt"));
profile.load(fileIn);
} catch(IOException e) {
profile.load(fileIn);
} catch (IOException e) {
} finally {
if (fileIn != null) try { fileIn.close(); fileIn = null; } catch (Exception e) {}
}

46
htroot/yacy/query.java
View File

@ -57,41 +57,41 @@ import de.anomic.yacy.yacySeed;
public final class query {
public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch ss) {
if (post == null || ss == null) { return null; }
public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch env) {
// return variable that accumulates replacements
final plasmaSwitchboard sb = (plasmaSwitchboard) ss;
final serverObjects prop = new serverObjects();
if ((post == null) || (ss == null)) return prop;
if (!yacyNetwork.authentifyRequest(post, ss)) return prop;
if ((sb.isRobinsonMode()) &&
(!sb.isPublicRobinson()) &&
(!sb.isInMyCluster((String)header.get(httpHeader.CONNECTION_PROP_CLIENTIP)))) {
// if we are a robinson cluster, answer only if we are public robinson peers,
// or we are a private cluster and the requester is in our cluster.
// if we don't answer, the remote peer will recognize us as junior peer,
// what would mean that our peer ping does not work
prop.put("response", "-1"); // request rejected
if (post == null || env == null || !yacyNetwork.authentifyRequest(post, env)) {
return prop;
}
final plasmaSwitchboard sb = (plasmaSwitchboard) env;
if (sb.isRobinsonMode() &&
!sb.isPublicRobinson() &&
!sb.isInMyCluster((String) header.get(httpHeader.CONNECTION_PROP_CLIENTIP))) {
// if we are a robinson cluster, answer only if we are public robinson peers,
// or we are a private cluster and the requester is in our cluster.
// if we don't answer, the remote peer will recognize us as junior peer,
// what would mean that our peer ping does not work
prop.put("response", "-1"); // request rejected
return prop;
}
// System.out.println("YACYQUERY: RECEIVED POST = " + ((post == null) ? "NULL" : post.toString()));
final String iam = post.get("iam", ""); // complete seed of the requesting peer
final String ohash = post.get("iam", ""); // complete seed of the requesting peer
final String youare = post.get("youare", ""); // seed hash of the target peer, used for testing network stability
// final String key = post.get("key", ""); // transmission key for response
final String obj = post.get("object", ""); // keyword for query subject
final String env = post.get("env", ""); // argument to query
final String qenv = post.get("env", ""); // argument to query
final yacySeed otherPeer = yacyCore.seedDB.get(iam);
if (otherPeer == null) {
final yacySeed oseed = yacyCore.seedDB.get(ohash);
if (oseed == null) {
prop.put("response", "0");
return prop;
} else {
otherPeer.setLastSeenUTC();
oseed.setFlagDirectConnect(true);
oseed.setLastSeenUTC();
}
prop.put("mytime", serverDate.formatShortSecond());
@ -107,7 +107,7 @@ public final class query {
if (obj.equals("rwiurlcount")) {
// the total number of different urls in the rwi is returned
// <env> shall contain a word hash, the number of assigned lurls to this hash is returned
prop.put("response", sb.wordIndex.indexSize(env));
prop.put("response", sb.wordIndex.indexSize(qenv));
return prop;
}

11
htroot/yacy/search.java
View File

@ -59,7 +59,7 @@ public final class search {
public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch env) {
// return variable that accumulates replacements
serverObjects prop = new serverObjects();
final serverObjects prop = new serverObjects();
// defaults
prop.put("links", "");
prop.put("linkcount", "0");
@ -86,8 +86,8 @@ public final class search {
return prop;
}
final String query = post.get("query", ""); // a string of word hashes that shall be searched and combined
String abstracts = post.get("abstracts", ""); // a string of word hashes for abstracts that shall be generated, or 'auto' (for maxcount-word), or '' (for none)
final String query = post.get("query", ""); // a string of word hashes that shall be searched and combined
final String abstracts = post.get("abstracts", ""); // a string of word hashes for abstracts that shall be generated, or 'auto' (for maxcount-word), or '' (for none)
if ((query == null || query.length() == 0) & (abstracts == null || abstracts.length() == 0)) {
return prop;
}
@ -96,9 +96,8 @@ public final class search {
sb.intermissionAllThreads(3000);
sb.remoteSearchLastAccess = System.currentTimeMillis();
final String oseed = post.get("myseed", ""); // complete seed of the requesting peer
final String key = post.get("key", ""); // transmission key for response
final yacySeed opeer = yacySeed.genRemoteSeed(oseed, key, true);
// myseed = complete seed of the requesting peer, key = transmission key for response
final yacySeed opeer = yacySeed.genRemoteSeed(post.get("myseed", ""), post.get("key", ""), true);
// store accessing peer
if (yacyCore.seedDB == null) {
yacyCore.log.logSevere("yacy.search: seed cache not initialized");

32
htroot/yacy/transfer.java
View File

@ -61,11 +61,13 @@ import de.anomic.yacy.yacySeed;
public final class transfer {
public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch env) {
plasmaSwitchboard sb = (plasmaSwitchboard) env;
serverObjects prop = new serverObjects();
if ((post == null) || (env == null)) return prop;
if (!yacyNetwork.authentifyRequest(post, env)) return prop;
// return variable that accumulates replacements
final serverObjects prop = new serverObjects();
if (post == null || env == null || !yacyNetwork.authentifyRequest(post, env)) {
return prop;
}
String process = post.get("process", ""); // permission or store
// String key = post.get("key", ""); // a transmission key from the client
String ohash = post.get("iam", ""); // identification of the client (a peer-hash)
@ -81,22 +83,24 @@ public final class transfer {
prop.put("process_path", "");
prop.put("process_maxsize", "0");
final plasmaSwitchboard sb = (plasmaSwitchboard) env;
if (sb.isRobinsonMode() || !sb.rankingOn) {
// in a robinson environment, do not answer. We do not do any transfer in a robinson cluster.
return prop;
}
final yacySeed opeer = yacyCore.seedDB.get(ohash);
if (opeer == null) {
final yacySeed oseed = yacyCore.seedDB.get(ohash);
if (oseed == null) {
// reject unknown peers: this does not appear fair, but anonymous senders are dangerous
sb.getLog().logFine("RankingTransmission: rejected unknown peer '" + ohash + "', current IP " + header.get(httpHeader.CONNECTION_PROP_CLIENTIP, "unknown"));
return prop;
}
opeer.setLastSeenUTC();
oseed.setFlagDirectConnect(true);
oseed.setLastSeenUTC();
if (filename.indexOf("..") >= 0) {
// reject paths that contain '..' because they are dangerous
sb.getLog().logFine("RankingTransmission: rejected wrong path '" + filename + "' from peer " + opeer.getName() + "/" + opeer.getPublicAddress()+ ", current IP " + header.get(httpHeader.CONNECTION_PROP_CLIENTIP, "unknown"));
sb.getLog().logFine("RankingTransmission: rejected wrong path '" + filename + "' from peer " + oseed.getName() + "/" + oseed.getPublicAddress()+ ", current IP " + header.get(httpHeader.CONNECTION_PROP_CLIENTIP, "unknown"));
return prop;
}
@ -114,7 +118,7 @@ public final class transfer {
prop.put("process_path", ""); // currently empty; the store process will find a path
prop.put("process_maxsize", "-1"); // if response is too big we return the size of the file
sb.rankingPermissions.put(serverCodings.encodeMD5Hex(kelondroBase64Order.standardCoder.encodeString(access)), filename);
sb.getLog().logFine("RankingTransmission: granted peer " + opeer.hash + ":" + opeer.getName() + " to send CR file " + filename);
sb.getLog().logFine("RankingTransmission: granted peer " + oseed.hash + ":" + oseed.getName() + " to send CR file " + filename);
}
return prop;
}
@ -132,7 +136,7 @@ public final class transfer {
if ((grantedFile == null) || (!(grantedFile.equals(filename)))) {
// fraud-access of this interface
prop.put("response", "denied");
sb.getLog().logFine("RankingTransmission: denied " + opeer.hash + ":" + opeer.getName() + " to send CR file " + filename + ": wrong access code");
sb.getLog().logFine("RankingTransmission: denied " + oseed.hash + ":" + oseed.getName() + " to send CR file " + filename + ": wrong access code");
} else {
sb.rankingPermissions.remove(accesscode); // not needed any more
File path = new File(sb.rankingPath, plasmaRankingDistribution.CR_OTHER);
@ -144,10 +148,10 @@ public final class transfer {
String md5t = serverCodings.encodeMD5Hex(file);
if (md5t.equals(md5)) {
prop.put("response", "ok");
sb.getLog().logFine("RankingTransmission: received from peer " + opeer.hash + ":" + opeer.getName() + " CR file " + filename);
sb.getLog().logFine("RankingTransmission: received from peer " + oseed.hash + ":" + oseed.getName() + " CR file " + filename);
} else {
prop.put("response", "transfer failure");
sb.getLog().logFine("RankingTransmission: transfer failure from peer " + opeer.hash + ":" + opeer.getName() + " for CR file " + filename);
sb.getLog().logFine("RankingTransmission: transfer failure from peer " + oseed.hash + ":" + oseed.getName() + " for CR file " + filename);
}
}else{
//exploit?
@ -163,7 +167,7 @@ public final class transfer {
}
// wrong access
sb.getLog().logFine("RankingTransmission: rejected unknown process " + process + ":" + purpose + " from peer " + opeer.hash + ":" + opeer.getName());
sb.getLog().logFine("RankingTransmission: rejected unknown process " + process + ":" + purpose + " from peer " + oseed.hash + ":" + oseed.getName());
return prop;
}

50
htroot/yacy/transferRWI.java
View File

@ -67,55 +67,59 @@ import de.anomic.yacy.yacySeed;
public final class transferRWI {
public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch env) throws InterruptedException {
// return variable that accumulates replacements
final plasmaSwitchboard sb = (plasmaSwitchboard) env;
final serverObjects prop = new serverObjects();
if ((post == null) || (env == null)) return prop;
if (!yacyNetwork.authentifyRequest(post, env)) return prop;
if (post == null || env == null || !yacyNetwork.authentifyRequest(post, env)) {
return prop;
}
if (!post.containsKey("wordc")) return prop;
if (!post.containsKey("entryc")) return prop;
// request values
final String iam = post.get("iam", ""); // seed hash of requester
final String youare = post.get("youare", ""); // seed hash of the target peer, needed for network stability
// final String key = (String) post.get("key", ""); // transmission key
final int wordc = post.getInt("wordc", 0); // number of different words
final int entryc = post.getInt("entryc", 0); // number of entries in indexes
byte[] indexes = post.get("indexes", "").getBytes(); // the indexes, as list of word entries
final String ohash = post.get("iam", ""); // seed hash of requester
final String youare = post.get("youare", ""); // seed hash of the target peer, needed for network stability
// final String key = (String) post.get("key", ""); // transmission key
final int wordc = post.getInt("wordc", 0); // number of different words
final int entryc = post.getInt("entryc", 0); // number of entries in indexes
byte[] indexes = post.get("indexes", "").getBytes(); // the indexes, as list of word entries
final plasmaSwitchboard sb = (plasmaSwitchboard) env;
boolean granted = sb.getConfig("allowReceiveIndex", "false").equals("true");
boolean blockBlacklist = sb.getConfig("indexReceiveBlockBlacklist", "false").equals("true");
boolean checkLimit = sb.getConfigBool("indexDistribution.transferRWIReceiptLimitEnabled", true);
final long cachelimit = sb.getConfigLong("indexDistribution.dhtReceiptLimit", 10000);
final yacySeed otherPeer = yacyCore.seedDB.get(iam);
if (otherPeer == null) {
final yacySeed oseed = yacyCore.seedDB.get(ohash);
if (oseed == null) {
prop.put("unknownURL", "");
prop.put("result", "busy");
prop.put("pause", "120000");
return prop;
} else {
otherPeer.setLastSeenUTC();
oseed.setFlagDirectConnect(true);
oseed.setLastSeenUTC();
}
final String otherPeerName = iam + ":" + otherPeer.getName() + "/" + otherPeer.getVersion();
final String oname = ohash + ":" + oseed.getName() + "/" + oseed.getVersion();
// response values
String result = "ok";
StringBuffer unknownURLs = new StringBuffer();
int pause = 10000;
if ((youare == null) || (!youare.equals(yacyCore.seedDB.mySeed().hash))) {
sb.getLog().logInfo("Rejecting RWIs from peer " + otherPeerName + ". Wrong target. Wanted peer=" + youare + ", iam=" + yacyCore.seedDB.mySeed().hash);
sb.getLog().logInfo("Rejecting RWIs from peer " + oname + ". Wrong target. Wanted peer=" + youare + ", iam=" + yacyCore.seedDB.mySeed().hash);
result = "wrong_target";
pause = 0;
} else if ((!granted) || (sb.isRobinsonMode())) {
// we dont want to receive indexes
sb.getLog().logInfo("Rejecting RWIs from peer " + otherPeerName + ". Not granted.");
sb.getLog().logInfo("Rejecting RWIs from peer " + oname + ". Not granted.");
result = "not_granted";
pause = 0;
} else if (checkLimit && sb.wordIndex.dhtInCacheSize() > cachelimit) {
// we are too busy to receive indexes
sb.getLog().logInfo("Rejecting RWIs from peer " + otherPeerName + ". We are too busy (buffersize=" + sb.wordIndex.dhtInCacheSize() + ").");
sb.getLog().logInfo("Rejecting RWIs from peer " + oname + ". We are too busy (buffersize=" + sb.wordIndex.dhtInCacheSize() + ").");
granted = false; // don't accept more words if there are too many words to flush
result = "busy";
pause = 60000;
@ -128,7 +132,7 @@ public final class transferRWI {
} */ else {
// we want and can receive indexes
// log value status (currently added to find outOfMemory error
sb.getLog().logFine("Processing " + indexes.length + " bytes / " + wordc + " words / " + entryc + " entries from " + otherPeerName);
sb.getLog().logFine("Processing " + indexes.length + " bytes / " + wordc + " words / " + entryc + " entries from " + oname);
final long startProcess = System.currentTimeMillis();
// decode request
@ -171,7 +175,7 @@ public final class transferRWI {
// block blacklisted entries
if ((blockBlacklist) && (plasmaSwitchboard.urlBlacklist.hashInBlacklistedCache(plasmaURLPattern.BLACKLIST_DHT, urlHash))) {
int deleted = sb.wordIndex.tryRemoveURLs(urlHash);
yacyCore.log.logFine("transferRWI: blocked blacklisted URLHash '" + urlHash + "' from peer " + otherPeerName + "; deleted " + deleted + " URL entries from RWIs");
yacyCore.log.logFine("transferRWI: blocked blacklisted URLHash '" + urlHash + "' from peer " + oname + "; deleted " + deleted + " URL entries from RWIs");
blocked++;
continue;
}
@ -205,10 +209,10 @@ public final class transferRWI {
}
if (unknownURLs.length() > 0) { unknownURLs.delete(0, 1); }
if ((wordhashes.length == 0) || (received == 0)) {
sb.getLog().logInfo("Received 0 RWIs from " + otherPeerName + ", processed in " + (System.currentTimeMillis() - startProcess) + " milliseconds, requesting " + unknownURL.size() + " URLs, blocked " + blocked + " RWIs");
sb.getLog().logInfo("Received 0 RWIs from " + oname + ", processed in " + (System.currentTimeMillis() - startProcess) + " milliseconds, requesting " + unknownURL.size() + " URLs, blocked " + blocked + " RWIs");
} else {
final double avdist = (yacyDHTAction.dhtDistance(yacyCore.seedDB.mySeed().hash, wordhashes[0]) + yacyDHTAction.dhtDistance(yacyCore.seedDB.mySeed().hash, wordhashes[received - 1])) / 2.0;
sb.getLog().logInfo("Received " + received + " Entries " + wordc + " Words [" + wordhashes[0] + " .. " + wordhashes[received - 1] + "]/" + avdist + " from " + otherPeerName + ", processed in " + (System.currentTimeMillis() - startProcess) + " milliseconds, requesting " + unknownURL.size() + "/" + receivedURL + " URLs, blocked " + blocked + " RWIs");
sb.getLog().logInfo("Received " + received + " Entries " + wordc + " Words [" + wordhashes[0] + " .. " + wordhashes[received - 1] + "]/" + avdist + " from " + oname + ", processed in " + (System.currentTimeMillis() - startProcess) + " milliseconds, requesting " + unknownURL.size() + "/" + receivedURL + " URLs, blocked " + blocked + " RWIs");
}
result = "ok";

57
htroot/yacy/transferURL.java
View File

@ -62,45 +62,48 @@ import de.anomic.yacy.yacySeed;
public final class transferURL {
public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch env) throws InterruptedException {
// return variable that accumulates replacements
final serverObjects prop = new serverObjects();
if (post == null || env == null || !yacyNetwork.authentifyRequest(post, env)) {
return prop;
}
long start = System.currentTimeMillis();
long freshdate = 0;
try {freshdate = serverDate.parseShortDay("20061101").getTime();} catch (ParseException e1) {}
// return variable that accumulates replacements
final plasmaSwitchboard sb = (plasmaSwitchboard) env;
final serverObjects prop = new serverObjects();
if ((post == null) || (env == null)) return prop;
if (!yacyNetwork.authentifyRequest(post, env)) return prop;
// request values
final String iam = post.get("iam", ""); // seed hash of requester
final String youare = post.get("youare", ""); // seed hash of the target peer, needed for network stability
// final String key = post.get("key", ""); // transmission key
final int urlc = post.getInt("urlc", 0); // number of transported urls
final String ohash = post.get("iam", ""); // seed hash of requester
final String youare = post.get("youare", ""); // seed hash of the target peer, needed for network stability
// final String key = post.get("key", ""); // transmission key
final int urlc = post.getInt("urlc", 0); // number of transported urls
final plasmaSwitchboard sb = (plasmaSwitchboard) env;
final boolean granted = sb.getConfig("allowReceiveIndex", "false").equals("true");
final boolean blockBlacklist = sb.getConfig("indexReceiveBlockBlacklist", "false").equals("true");
final yacySeed otherPeer = yacyCore.seedDB.get(iam);
if (otherPeer == null) {
final yacySeed oseed = yacyCore.seedDB.get(ohash);
if (oseed == null) {
prop.put("result", "error_not_granted");
prop.put("pause", "120000");
return prop;
} else {
otherPeer.setLastSeenUTC();
oseed.setFlagDirectConnect(true);
oseed.setLastSeenUTC();
}
final String otherPeerName = iam + ":" + otherPeer.getName() + "/" + otherPeer.getVersion();
final String oname = ohash + ":" + oseed.getName() + "/" + oseed.getVersion();
// response values
String result = "";
String doublevalues = "0";
if ((youare == null) || (!youare.equals(yacyCore.seedDB.mySeed().hash))) {
sb.getLog().logInfo("Rejecting URLs from peer " + otherPeerName + ". Wrong target. Wanted peer=" + youare + ", iam=" + yacyCore.seedDB.mySeed().hash);
sb.getLog().logInfo("Rejecting URLs from peer " + oname + ". Wrong target. Wanted peer=" + youare + ", iam=" + yacyCore.seedDB.mySeed().hash);
result = "wrong_target";
} else if ((!granted) || (sb.isRobinsonMode())) {
sb.getLog().logInfo("Rejecting URLs from peer " + otherPeerName + ". Not granted.");
sb.getLog().logInfo("Rejecting URLs from peer " + oname + ". Not granted.");
result = "error_not_granted";
} else {
int received = 0;
@ -115,7 +118,7 @@ public final class transferURL {
// read new lurl-entry
urls = (String) post.get("url" + i);
if (urls == null) {
yacyCore.log.logFine("transferURL: got null URL-string from peer " + otherPeerName);
yacyCore.log.logFine("transferURL: got null URL-string from peer " + oname);
blocked++;
continue;
}
@ -123,7 +126,7 @@ public final class transferURL {
// parse new lurl-entry
lEntry = sb.wordIndex.loadedURL.newEntry(urls);
if (lEntry == null) {
yacyCore.log.logWarning("transferURL: received invalid URL (entry null) from peer " + otherPeerName + "\n\tURL Property: " + urls);
yacyCore.log.logWarning("transferURL: received invalid URL (entry null) from peer " + oname + "\n\tURL Property: " + urls);
blocked++;
continue;
}
@ -131,14 +134,14 @@ public final class transferURL {
// check if entry is well-formed
indexURLEntry.Components comp = lEntry.comp();
if (comp.url() == null) {
yacyCore.log.logWarning("transferURL: received invalid URL from peer " + otherPeerName + "\n\tURL Property: " + urls);
yacyCore.log.logWarning("transferURL: received invalid URL from peer " + oname + "\n\tURL Property: " + urls);
blocked++;
continue;
}
// check whether entry is too old
if (lEntry.freshdate().getTime() <= freshdate) {
yacyCore.log.logFine("transerURL: received too old URL from peer " + otherPeerName + ": " + lEntry.freshdate());
yacyCore.log.logFine("transerURL: received too old URL from peer " + oname + ": " + lEntry.freshdate());
blocked++;
continue;
}
@ -146,7 +149,7 @@ public final class transferURL {
// check if the entry is blacklisted
if ((blockBlacklist) && (plasmaSwitchboard.urlBlacklist.isListed(plasmaURLPattern.BLACKLIST_DHT, comp.url()))) {
int deleted = sb.wordIndex.tryRemoveURLs(lEntry.hash());
yacyCore.log.logFine("transferURL: blocked blacklisted URL '" + comp.url().toNormalform(false, true) + "' from peer " + otherPeerName + "; deleted " + deleted + " URL entries from RWIs");
yacyCore.log.logFine("transferURL: blocked blacklisted URL '" + comp.url().toNormalform(false, true) + "' from peer " + oname + "; deleted " + deleted + " URL entries from RWIs");
lEntry = null;
blocked++;
continue;
@ -154,7 +157,7 @@ public final class transferURL {
// check if the entry is in our network domain
if (!sb.acceptURL(comp.url())) {
yacyCore.log.logFine("transferURL: blocked URL outside of our domain '" + comp.url().toNormalform(false, true) + "' from peer " + otherPeerName);
yacyCore.log.logFine("transferURL: blocked URL outside of our domain '" + comp.url().toNormalform(false, true) + "' from peer " + oname);
lEntry = null;
blocked++;
continue;
@ -163,8 +166,8 @@ public final class transferURL {
// write entry to database
try {
sb.wordIndex.loadedURL.store(lEntry);
sb.wordIndex.loadedURL.stack(lEntry, iam, iam, 3);
yacyCore.log.logFine("transferURL: received URL '" + comp.url().toNormalform(false, true) + "' from peer " + otherPeerName);
sb.wordIndex.loadedURL.stack(lEntry, ohash, ohash, 3);
yacyCore.log.logFine("transferURL: received URL '" + comp.url().toNormalform(false, true) + "' from peer " + oname);
received++;
} catch (IOException e) {
e.printStackTrace();
@ -176,8 +179,8 @@ public final class transferURL {
// return rewrite properties
final int more = sb.wordIndex.loadedURL.size() - sizeBefore;
doublevalues = Integer.toString(received - more);
sb.getLog().logInfo("Received " + received + " URLs from peer " + otherPeerName + " in " + (System.currentTimeMillis() - start) + " ms, Blocked " + blocked + " URLs");
if ((received - more) > 0) sb.getLog().logSevere("Received " + doublevalues + " double URLs from peer " + otherPeerName);
sb.getLog().logInfo("Received " + received + " URLs from peer " + oname + " in " + (System.currentTimeMillis() - start) + " ms, Blocked " + blocked + " URLs");
if ((received - more) > 0) sb.getLog().logSevere("Received " + doublevalues + " double URLs from peer " + oname);
result = "ok";
}

19
source/de/anomic/yacy/yacyClient.java
View File

@ -259,6 +259,7 @@ public final class yacyClient {
);
if (result == null || result.size() == 0) { return null; }
target.setFlagDirectConnect(true);
target.setLastSeenUTC();
//final Date remoteTime = yacyCore.parseUniversalDate((String) result.get(yacySeed.MYTIME)); // read remote time
return yacySeed.genRemoteSeed((String) result.get("response"), post.get("key", ""), true);
@ -294,6 +295,7 @@ public final class yacyClient {
if (resp == null) {
return -1;
} else try {
target.setFlagDirectConnect(true);
target.setLastSeenUTC();
return Integer.parseInt(resp);
} catch (NumberFormatException e) {
@ -334,6 +336,7 @@ public final class yacyClient {
if (resp == null) {
return -1;
} else try {
target.setFlagDirectConnect(true);
target.setLastSeenUTC();
return Integer.parseInt(resp);
} catch (NumberFormatException e) {
@ -862,7 +865,7 @@ public final class yacyClient {
}
}
public static HashMap<String, Object> transferIndex(yacySeed targetSeed, indexContainer[] indexes, HashMap<String, indexURLEntry> urlCache, boolean gzipBody, int timeout) {
public static HashMap<String, Object> transferIndex(yacySeed target, indexContainer[] indexes, HashMap<String, indexURLEntry> urlCache, boolean gzipBody, int timeout) {
HashMap<String, Object> resultObj = new HashMap<String, Object>();
int payloadSize = 0;
@ -882,7 +885,7 @@ public final class yacyClient {
}
// transfer the RWI without the URLs
HashMap<String, String> in = transferRWI(targetSeed, indexes, gzipBody, timeout);
HashMap<String, String> in = transferRWI(target, indexes, gzipBody, timeout);
resultObj.put("resultTransferRWI", in);
if (in == null) {
@ -897,11 +900,9 @@ public final class yacyClient {
return resultObj;
}
targetSeed.setLastSeenUTC();
if (!(result.equals("ok"))) {
targetSeed.setFlagAcceptRemoteIndex(false);
yacyCore.seedDB.update(targetSeed.hash, targetSeed);
target.setFlagAcceptRemoteIndex(false);
yacyCore.seedDB.update(target.hash, target);
resultObj.put("result", result);
return resultObj;
}
@ -926,7 +927,7 @@ public final class yacyClient {
}
}
in = transferURL(targetSeed, urls, gzipBody, timeout);
in = transferURL(target, urls, gzipBody, timeout);
resultObj.put("resultTransferURL", in);
if (in == null) {
@ -941,8 +942,8 @@ public final class yacyClient {
return resultObj;
}
if (!(result.equals("ok"))) {
targetSeed.setFlagAcceptRemoteIndex(false);
yacyCore.seedDB.update(targetSeed.hash, targetSeed);
target.setFlagAcceptRemoteIndex(false);
yacyCore.seedDB.update(target.hash, target);
resultObj.put("result",result);
return resultObj;
}

8
source/de/anomic/yacy/yacyCore.java
View File

@ -93,11 +93,11 @@ public class yacyCore {
public static File yacyDBPath;
public static final Map<String, yacyAccessible> amIAccessibleDB = Collections.synchronizedMap(new HashMap<String, yacyAccessible>()); // Holds PeerHash / yacyAccessible Relations
// constants for PeerPing behaviour
private static final int PING_INITIAL = 18;
private static final int PING_INITIAL = 12;
private static final int PING_MAX_RUNNING = 6;
private static final int PING_MIN_RUNNING = 3;
private static final int PING_MIN_DBSIZE = 5;
private static final int PING_MIN_PEERSEEN = 1; // min. accessible to force senior
private static final int PING_MIN_DBSIZE = 3;
private static final int PING_MIN_PEERSEEN = 2; // min. accessible to force senior
private static final long PING_MAX_DBAGE = 15 * 60 * 1000; // in milliseconds
// public static yacyShare shareManager = null;
@ -438,7 +438,7 @@ public class yacyCore {
// going through the peer list and starting a new publisher thread for each peer
int i = 0;
while (si. hasNext()) {
while (si.hasNext()) {
seed = (yacySeed) si.next();
if (seed == null) {
sync.P();

4
source/de/anomic/yacy/yacySeed.java
View File

@ -815,7 +815,7 @@ public class yacySeed {
public final String genSeedStr(String key) {
// use a default encoding
return this.genSeedStr('b', key);
return this.genSeedStr('z', key);
}
public final synchronized String genSeedStr(char method, String key) {
@ -838,7 +838,7 @@ public class yacySeed {
}
public final void save(File f) throws IOException {
final String out = this.genSeedStr('p', null);
final String out = this.genSeedStr('z', null);
final FileWriter fw = new FileWriter(f);
fw.write(out, 0, out.length());
fw.close();