Marown/yacy_search_server
1
0
Fork 0
mirror of https://github.com/yacy/yacy_search_server.git synced 2024-09-19 00:01:41 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity

network functions for robinson peers: server-side protection

Browse Source 
git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@3591 6c8d7289-2bf4-0310-a012-ef5d649a1542
This commit is contained in:
orbiter 2007-04-24 15:11:12 +00:00
parent 8253a5bf82
commit 657585fe0d
12 changed files with 84 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
htroot/yacy/crawlOrder.java
View File

@ -99,7 +99,7 @@ public final class crawlOrder {
String reason = "false-input";
String delay = "5";
String lurl = "";
boolean granted = switchboard.getConfig("crawlResponse", "false").equals("true");
boolean granted = switchboard.getConfigBool("crawlResponse", false);
int acceptDepth = Integer.parseInt(switchboard.getConfig("crawlResponseDepth", "0"));
int ppm = yacyCore.seedDB.mySeed.getPPM();
int acceptDelay = (ppm == 0) ? 10 : (2 + 60 / yacyCore.seedDB.mySeed.getPPM());
@ -108,11 +108,17 @@ public final class crawlOrder {
// check if requester is authorized
if ((yacyCore.seedDB.mySeed == null) || (!(yacyCore.seedDB.mySeed.hash.equals(youare)))) {
// this request has a wrong target
// this request has a wrong target
response = "denied";
reason = "authentify-problem";
delay = "3600"; // may request one hour later again
} else if (orderDepth > 0) {
} else if ((switchboard.isRobinsonMode()) && (!switchboard.isInMyCluster(iam))) {
// check network environment, if we are a robinson peer or in a robinson cluster
// then the request must come from a peer that is in the same cluster as we are
reason = "not in my cluster";
response = "denied";
delay = "9999";
} else if (orderDepth > 0) {
response = "denied";
reason = "order depth must be 0";
delay = "3600"; // may request one hour later again

5
htroot/yacy/crawlReceipt.java
View File

@ -124,6 +124,11 @@ public final class crawlReceipt {
return prop;
}
if ((switchboard.isRobinsonMode()) && (!switchboard.isInMyCluster(otherPeer))) {
// we reject urls that are from outside our cluster
prop.putASIS("delay", "9999");
}
// generating a new loaded URL entry
indexURLEntry entry = switchboard.wordIndex.loadedURL.newEntry(propStr);
if (entry == null) {

8
htroot/yacy/hello.java
View File

@ -49,6 +49,7 @@
import java.util.Date;
import de.anomic.http.httpHeader;
import de.anomic.plasma.plasmaSwitchboard;
import de.anomic.server.serverCore;
import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch;
@ -62,6 +63,7 @@ public final class hello {
public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch ss) throws InterruptedException {
if (post == null || ss == null || yacyCore.seedDB == null || yacyCore.seedDB.mySeed == null) { return null; }
plasmaSwitchboard sb = (plasmaSwitchboard) ss;
// return variable that accumulates replacements
final serverObjects prop = new serverObjects();
if (prop == null) { return null; }
@ -80,6 +82,7 @@ public final class hello {
// System.out.println("YACYHELLO: REMOTESEED=" + ((remoteSeed == null) ? "NULL" : remoteSeed.toString()));
if (remoteSeed == null) { return null; }
// final String properTest = remoteSeed.isProper();
// The remote peer might not know its IP yet, so don't abort if the IP check fails
// if ((properTest != null) && (! properTest.substring(0,1).equals("IP"))) { return null; }
@ -91,6 +94,11 @@ public final class hello {
final String reportedPeerType = remoteSeed.get(yacySeed.PEERTYPE, yacySeed.PEERTYPE_JUNIOR);
final float clientversion = remoteSeed.getVersion();
if ((sb.isRobinsonMode()) && (!sb.isOpenRobinsonCluster())) {
// if we are a robinson cluster, answer only if this client is known by our network definition
return null;
}
int urls = -1;
// if the remote client has reported its own IP address and the client supports
// the port forwarding feature (if client version >= 0.383) then we try to

13
htroot/yacy/list.java
View File

@ -57,6 +57,7 @@ import de.anomic.data.listManager;
import de.anomic.data.wikiCode;
import de.anomic.http.httpHeader;
import de.anomic.net.URL;
import de.anomic.plasma.plasmaSwitchboard;
import de.anomic.server.serverCore;
import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch;
@ -69,6 +70,7 @@ public final class list {
public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch ss) {
if (post == null || ss == null)
throw new NullPointerException("post: " + post + ", sb: " + ss);
plasmaSwitchboard sb = (plasmaSwitchboard) ss;
// return variable that accumulates replacements
final serverObjects prop = new serverObjects();
@ -83,6 +85,11 @@ public final class list {
}
if (otherPeerName == null) otherPeerName = (String)header.get(httpHeader.CONNECTION_PROP_CLIENTIP);
if ((sb.isRobinsonMode()) && (!sb.isInMyCluster(otherPeerName))) {
// if we are a robinson cluster, answer only if this client is known by our network definition
return null;
}
if (col.equals("black")) {
final StringBuffer out = new StringBuffer();
@ -109,16 +116,16 @@ public final class list {
final int count = Math.min(post.getInt("count", 50), CrawlURLFetchStack_p.maxURLsPerFetch);
if (count > 0 && db.size() > 0) {
final StringBuffer sb = new StringBuffer();
final StringBuffer b = new StringBuffer();
URL url;
int cnt = 0;
for (int i=0; i<count; i++) {
if ((url = db.pop()) == null) continue;
sb.append(wikiCode.deReplaceHTMLEntities(url.toNormalform())).append("\n");
b.append(wikiCode.deReplaceHTMLEntities(url.toNormalform())).append("\n");
cnt++;
}
prop.put("list", sb);
prop.put("list", b);
CrawlURLFetchStack_p.fetchMap.put(
otherPeerName,
new Integer(((CrawlURLFetchStack_p.fetchMap.get(otherPeerName) == null)

13
htroot/yacy/message.java
View File

@ -79,8 +79,6 @@ public final class message {
serverObjects prop = new serverObjects();
if (prop == null || sb == null) { return null; }
// System.out.println("yacy/message:post=" + post.toString());
String process = post.get("process", "permission");
String key = post.get("key", "");
@ -90,17 +88,22 @@ public final class message {
prop.putASIS("messagesize", "0");
prop.putASIS("attachmentsize", "0");
// System.out.println("DEBUG yacy/message: message post values = " + post.toString());
String youare = post.get("youare", ""); // seed hash of the target peer, needed for network stability
// check if we are the right target and requester has correct information about this peer
if ((yacyCore.seedDB.mySeed == null) || (!(yacyCore.seedDB.mySeed.hash.equals(youare)))) {
// this request has a wrong target
// System.out.println("DEBUG yacy/message: authenticate failed");
prop.putASIS("response", "-1"); // request rejected
return prop;
}
if ((sb.isRobinsonMode()) &&
(!((sb.isOpenRobinsonCluster()) ||
(sb.isInMyCluster((String)header.get(httpHeader.CONNECTION_PROP_CLIENTIP)))))) {
// if we are a robinson cluster, answer only if this client is known by our network definition
prop.putASIS("response", "-1"); // request rejected
return prop;
}
prop.putASIS("messagesize", Integer.toString(messagesize));
prop.putASIS("attachmentsize", Integer.toString(attachmentsize));

10
htroot/yacy/profile.java
View File

@ -56,6 +56,7 @@ import java.util.Map;
import java.util.Properties;
import de.anomic.http.httpHeader;
import de.anomic.plasma.plasmaSwitchboard;
import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch;
@ -64,8 +65,17 @@ public final class profile {
public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch ss) {
// return variable that accumulates replacements
serverObjects prop = new serverObjects();
plasmaSwitchboard sb = (plasmaSwitchboard) ss;
if (prop == null) { return null; }
if ((sb.isRobinsonMode()) &&
(!((sb.isOpenRobinsonCluster()) ||
(sb.isInMyCluster((String)header.get(httpHeader.CONNECTION_PROP_CLIENTIP)))))) {
// if we are a robinson cluster, answer only if this client is known by our network definition
prop.put("list", 0);
return prop;
}
Properties profile = new Properties();
int count=0;
String key="";

8
htroot/yacy/query.java
View File

@ -64,6 +64,14 @@ public final class query {
final serverObjects prop = new serverObjects();
if (prop == null || sb == null) { return null; }
if ((sb.isRobinsonMode()) &&
(!((sb.isOpenRobinsonCluster()) ||
(sb.isInMyCluster((String)header.get(httpHeader.CONNECTION_PROP_CLIENTIP)))))) {
// if we are a robinson cluster, answer only if this client is known by our network definition
prop.putASIS("response", "-1"); // request rejected
return prop;
}
// System.out.println("YACYQUERY: RECEIVED POST = " + ((post == null) ? "NULL" : post.toString()));
// final String iam = post.get("iam", ""); // complete seed of the requesting peer

13
htroot/yacy/search.java
View File

@ -114,6 +114,18 @@ public final class search {
// http://localhost:8080/yacy/search.html?query=4galTpdpDM5Qgh8DKIhGKXws&abstracts=auto (search for linux and book, generate abstract automatically)
// http://localhost:8080/yacy/search.html?query=&abstracts=4galTpdpDM5Q (only abstracts for linux)
serverObjects prop = new serverObjects();
if ((sb.isRobinsonMode()) &&
(!((sb.isOpenRobinsonCluster()) ||
(sb.isInMyCluster((String)header.get(httpHeader.CONNECTION_PROP_CLIENTIP)))))) {
// if we are a robinson cluster, answer only if this client is known by our network definition
prop.putASIS("links", "");
prop.putASIS("linkcount", "0");
prop.putASIS("references", "");
return prop;
}
// tell all threads to do nothing for a specific time
sb.intermissionAllThreads(2 * duetime);
@ -130,7 +142,6 @@ public final class search {
final TreeSet queryhashes = plasmaSearchQuery.hashes2Set(query);
final TreeSet excludehashes = (exclude.length() == 0) ? new TreeSet(kelondroBase64Order.enhancedCoder) : plasmaSearchQuery.hashes2Set(exclude);
final long timestamp = System.currentTimeMillis();
serverObjects prop = new serverObjects();
// prepare an abstract result
StringBuffer indexabstract = new StringBuffer();

5
htroot/yacy/transfer.java
View File

@ -79,6 +79,11 @@ public final class transfer {
prop.putASIS("process_path", "");
prop.putASIS("process_maxsize", "0");
if (sb.isRobinsonMode()) {
// in a robinson environment, do not answer. We do not do any transfer in a robinson cluster.
return prop;
}
if (!sb.rankingOn) { return prop; }
yacySeed otherseed = yacyCore.seedDB.get(otherpeer);

2
htroot/yacy/transferRWI.java
View File

@ -105,7 +105,7 @@ public final class transferRWI {
}
*/
if (!granted) {
if ((!granted) || (sb.isRobinsonMode())) {
// we dont want to receive indexes
sb.getLog().logInfo("Rejecting RWIs from peer " + otherPeerName + ". Not granted.");
result = "not_granted";

8
htroot/yacy/transferURL.java
View File

@ -89,7 +89,10 @@ public final class transferURL {
final yacySeed otherPeer = yacyCore.seedDB.get(iam);
final String otherPeerName = iam + ":" + ((otherPeer == null) ? "NULL" : (otherPeer.getName() + "/" + otherPeer.getVersion()));
if (granted) {
if ((!granted) || (sb.isRobinsonMode())) {
sb.getLog().logInfo("Rejecting URLs from peer " + otherPeerName + ". Not granted.");
result = "error_not_granted";
} else {
int received = 0;
int blocked = 0;
final int sizeBefore = sb.wordIndex.loadedURL.size();
@ -154,9 +157,6 @@ public final class transferURL {
sb.getLog().logInfo("Received " + received + " URLs from peer " + otherPeerName + " in " + (System.currentTimeMillis() - start) + " ms, Blocked " + blocked + " URLs");
if ((received - more) > 0) sb.getLog().logSevere("Received " + doublevalues + " double URLs from peer " + otherPeerName);
result = "ok";
} else {
sb.getLog().logInfo("Rejecting URLs from peer " + otherPeerName + ". Not granted.");
result = "error_not_granted";
}
prop.putASIS("double", doublevalues);

6
source/de/anomic/plasma/plasmaSwitchboard.java
View File

@ -1331,17 +1331,18 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser
return getConfigBool(plasmaSwitchboard.INDEX_DIST_ALLOW, false) && !getConfigBool(plasmaSwitchboard.INDEX_RECEIVE_ALLOW, false);
}
public boolean isClosedRobinsonCluster() {
public boolean isOpenRobinsonCluster() {
// robinson peers may be member of robinson clusters, which can be public or private
// this does not check the robinson attribute, only the specific subtype of the cluster
String clustermode = getConfig("cluster.mode", "publicpeer");
return (clustermode.equals("privatecluster")) || (clustermode.equals("privatepeer"));
return (clustermode.equals("publiccluster")) || (clustermode.equals("publicepeer"));
}
public boolean isInMyCluster(String peer) {
// check if the given peer is in the own network, if this is a robinson cluster
// depending on the robinson cluster type, the peer String may be a peerhash (b64-hash)
// or a ip:port String or simply a ip String
// if this robinson mode does not define a cluster membership, false is returned
if (!isRobinsonMode()) return false;
String clustermode = getConfig("cluster.mode", "publicpeer");
if (clustermode.equals("privatecluster")) {
@ -1368,6 +1369,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser
public boolean isInMyCluster(yacySeed seed) {
// check if the given peer is in the own network, if this is a robinson cluster
// if this robinson mode does not define a cluster membership, false is returned
if (seed == null) return false;
if (!isRobinsonMode()) return false;
String clustermode = getConfig("cluster.mode", "publicpeer");