Marown/yacy_search_server
1
0
Fork 0
mirror of https://github.com/yacy/yacy_search_server.git synced 2024-09-19 00:01:41 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity

explicit double-check in transferURL

Browse Source
This commit is contained in:
orbiter 2012-08-18 13:18:51 +02:00
parent ee01c12e56
commit 6f01542aaa
1 changed files with 16 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
htroot/yacy/transferURL.java
View File

@ -84,7 +84,7 @@ public final class transferURL {
} else {
int received = 0;
int blocked = 0;
final int sizeBefore = sb.index.fulltext().size();
int doublecheck = 0;
// read the urls from the other properties and store
String urls;
URIMetadata lEntry;
@ -102,14 +102,14 @@ public final class transferURL {
// parse new lurl-entry
lEntry = URIMetadataRow.importEntry(urls);
if (lEntry == null) {
Network.log.logWarning("transferURL: received invalid URL (entry null) from peer " + otherPeerName + "\n\tURL Property: " + urls);
if (Network.log.isWarning()) Network.log.logWarning("transferURL: received invalid URL (entry null) from peer " + otherPeerName + "\n\tURL Property: " + urls);
blocked++;
continue;
}
// check if entry is well-formed
if (lEntry.url() == null) {
Network.log.logWarning("transferURL: received invalid URL from peer " + otherPeerName + "\n\tURL Property: " + urls);
if (Network.log.isWarning()) Network.log.logWarning("transferURL: received invalid URL from peer " + otherPeerName + "\n\tURL Property: " + urls);
blocked++;
continue;
}
@ -123,7 +123,7 @@ public final class transferURL {
// check if the entry is blacklisted
if ((blockBlacklist) && (Switchboard.urlBlacklist.isListed(BlacklistType.DHT, lEntry))) {
Network.log.logFine("transferURL: blocked blacklisted URL '" + lEntry.url().toNormalform(false, true) + "' from peer " + otherPeerName);
if (Network.log.isFine()) Network.log.logFine("transferURL: blocked blacklisted URL '" + lEntry.url().toNormalform(false, true) + "' from peer " + otherPeerName);
lEntry = null;
blocked++;
continue;
@ -138,6 +138,14 @@ public final class transferURL {
continue;
}
// doublecheck
if (sb.index.exists(lEntry.hash())) {
if (Network.log.isFine()) Network.log.logFine("transferURL: double URL '" + lEntry.url() + "' from peer " + otherPeerName);
lEntry = null;
doublecheck++;
continue;
}
// write entry to database
if (Network.log.isFine()) Network.log.logFine("Accepting URL " + i + "/" + urlc + " from peer " + otherPeerName + ": " + lEntry.url().toNormalform(true, false));
try {
@ -153,11 +161,12 @@ public final class transferURL {
sb.peers.mySeed().incRU(received);
// return rewrite properties
final int more = sb.index.fulltext().size() - sizeBefore;
doublevalues = Integer.toString(received - more);
Network.log.logInfo("Received " + received + " URLs from peer " + otherPeerName + " in " + (System.currentTimeMillis() - start) + " ms, blocked " + blocked + " URLs");
EventChannel.channels(EventChannel.DHTRECEIVE).addMessage(new RSSMessage("Received " + received + ", blocked " + blocked + " URLs from peer " + otherPeerName, "", otherPeer.hash));
if ((received - more) > 0) Network.log.logSevere("Received " + doublevalues + " double URLs from peer " + otherPeerName);
if (doublecheck > 0) {
Network.log.logSevere("Received " + doublecheck + " double URLs from peer " + otherPeerName);
doublevalues = Integer.toString(doublecheck);
}
result = "ok";
}