From 17e004599d12259ad0b659db12e17857cd8e1d83 Mon Sep 17 00:00:00 2001 From: luccioman Date: Fri, 15 Dec 2017 11:28:46 +0100 Subject: [PATCH] Started implementing optional https preference for protocol operations Introduced through the new configurable setting network.unit.protocol.https.preferred, defaulting to false for now. Let choose to prefer using https when available on remote peers to perform YaCy protocol operations including notably hello or transferRWI. Not yet implemented for every YaCy protocol operations. --- defaults/yacy.init | 4 + htroot/MessageSend_p.java | 91 ++++++++-- htroot/Network.java | 51 ++++-- htroot/yacy/hello.java | 29 +++- source/net/yacy/peers/Network.java | 43 ++++- source/net/yacy/peers/Protocol.java | 164 +++++++++++------- source/net/yacy/peers/Seed.java | 24 ++- .../net/yacy/search/SwitchboardConstants.java | 7 + 8 files changed, 301 insertions(+), 112 deletions(-) diff --git a/defaults/yacy.init b/defaults/yacy.init index 31196f8ed..1d8c67449 100644 --- a/defaults/yacy.init +++ b/defaults/yacy.init @@ -127,6 +127,10 @@ network.unit.domain.nocheck = false # that means it is not usable in YaCy p2p-configurations, only in private portal configurations network.unit.tenant.agent = +# Prefer https for in-protocol operations when available on remote peers +# A distinct general setting is available to control whether https sould be used for remote search queries : remotesearch.https.preferred +network.unit.protocol.https.preferred = false + # Update process properties # The update server location is given in the network.unit.definition, # but the settings for update processing and cycles are individual. diff --git a/htroot/MessageSend_p.java b/htroot/MessageSend_p.java index 52870ae78..57271bc00 100644 --- a/htroot/MessageSend_p.java +++ b/htroot/MessageSend_p.java @@ -26,6 +26,7 @@ //if the shell's current path is HTROOT import java.io.IOException; +import java.net.MalformedURLException; import java.util.Date; import java.util.Map; @@ -34,6 +35,7 @@ import org.apache.http.entity.mime.content.ContentBody; import net.yacy.cora.date.GenericFormatter; import net.yacy.cora.document.encoding.ASCII; import net.yacy.cora.document.encoding.UTF8; +import net.yacy.cora.document.id.MultiProtocolURL; import net.yacy.cora.protocol.RequestHeader; import net.yacy.kelondro.util.FileUtils; import net.yacy.peers.Network; @@ -42,6 +44,7 @@ import net.yacy.peers.Protocol.Post; import net.yacy.peers.Seed; import net.yacy.peers.SeedDB; import net.yacy.search.Switchboard; +import net.yacy.search.SwitchboardConstants; import net.yacy.server.serverObjects; import net.yacy.server.serverSwitch; import net.yacy.utils.crypt; @@ -77,8 +80,35 @@ public class MessageSend_p { // first ask if the other peer is online, and also what kind of document it accepts Seed seed = sb.peers.getConnected(ASCII.getBytes(hash)); if (seed != null) { - for (String ip : seed.getIPs()) { - final Map result = Protocol.permissionMessage(seed.getPublicAddress(ip), hash); + for (final String ip : seed.getIPs()) { + Map result = null; + MultiProtocolURL targetBaseURL = null; + final String targetBaseURLStr = seed.getPublicURL(ip, + sb.getConfigBool(SwitchboardConstants.NETWORK_PROTOCOL_HTTPS_PREFERRED, + SwitchboardConstants.NETWORK_PROTOCOL_HTTPS_PREFERRED_DEFAULT)); + try { + targetBaseURL = new MultiProtocolURL(targetBaseURLStr); + result = Protocol.permissionMessage(targetBaseURL, seed, sb); + } catch(final MalformedURLException e) { + Network.log.warn("yacyClient.permissionMessage malformed target peer URL :" + targetBaseURLStr); + } catch(final Exception e) { + // most probably a network time-out exception + Network.log.warn("yacyClient.permissionMessage error:" + e.getMessage()); + if(targetBaseURL.isHTTPS()) { + try { + /* Request made over https : retry using http on the same IP as a fallback */ + targetBaseURL = seed.getPublicMultiprotocolURL(ip, false); + result = Protocol.permissionMessage(targetBaseURL, seed, sb); + if(result != null) { + /* Got a successfull result with http : mark now SSl as not available ont the target peer */ + seed.setFlagSSLAvailable(false); + sb.peers.updateConnected(seed); + } + } catch (final IOException e2) { + Network.log.warn("yacyClient.postMessage error:" + e2.getMessage()); + } + } + } //System.out.println("DEBUG: permission request result = " + result.toString()); String peerName; Seed targetPeer = null; @@ -102,6 +132,7 @@ public class MessageSend_p { sb.peers.peerActions.interfaceDeparture(targetPeer, ip); } } else { + prop.put("mode_permission", "1"); // write input form @@ -146,28 +177,56 @@ public class MessageSend_p { final String salt = crypt.randomSalt(); // send request - final Map parts = Protocol.basicRequestParts(Switchboard.getSwitchboard(), hash, salt); + final Map parts = Protocol.basicRequestParts(sb, hash, salt); parts.put("process", UTF8.StringBody("post")); parts.put("myseed", UTF8.StringBody(seedDB.mySeed().genSeedStr(salt))); parts.put("subject", UTF8.StringBody(subject)); parts.put("message", UTF8.StringBody(mb)); - Seed seed = seedDB.getConnected(ASCII.getBytes(hash)); + final Seed seed = seedDB.getConnected(ASCII.getBytes(hash)); + boolean preferHttps = sb.getConfigBool(SwitchboardConstants.NETWORK_PROTOCOL_HTTPS_PREFERRED, + SwitchboardConstants.NETWORK_PROTOCOL_HTTPS_PREFERRED_DEFAULT); Post post1 = null; - for (String ip: seed.getIPs()) { - try { - post1 = new Post(seed.getPublicAddress(ip), hash, "/yacy/message.html", parts, 20000); - } catch (IOException e) { - Network.log.warn("yacyClient.postMessage error:" + e.getMessage()); - post1 = null; - } - if (post1 != null) break; - seedDB.peerActions.interfaceDeparture(seed, ip); + for(final String ip : seed.getIPs()) { + MultiProtocolURL targetBaseURL = null; + try { + targetBaseURL = seed.getPublicMultiprotocolURL(ip, preferHttps); + post1 = new Post(targetBaseURL, seed.hash, "/yacy/message.html", parts, 20000); + } catch(final MalformedURLException e) { + Network.log.warn("yacyClient.postMessage malformed target peer URL when using ip " + ip); + } catch (final IOException e) { + Network.log.warn("yacyClient.postMessage error:" + e.getMessage()); + if(targetBaseURL.isHTTPS()) { + try { + /* Request made over https : retry using http on the same IP as a fallback */ + targetBaseURL = seed.getPublicMultiprotocolURL(ip, false); + post1 = new Post(targetBaseURL, seed.hash, "/yacy/message.html", parts, 20000); + if(post1 != null) { + /* Got a successfull result with http : mark now SSl as not available ont the target peer */ + seed.setFlagSSLAvailable(false); + } + } catch (final IOException e2) { + Network.log.warn("yacyClient.postMessage error:" + e2.getMessage()); + } + } + } + + if (post1 != null) { + break; + } + seedDB.peerActions.interfaceDeparture(seed, ip); } - final Map result1 = post1 == null ? null : FileUtils.table(post1.result); + final Map result1 = post1 == null ? null : FileUtils.table(post1.getResult()); final Map result = result1; - //message has been sent - prop.put("mode_status_response", result.get("response")); + if(result != null) { + // message has been sent + prop.put("mode_status_response", result.get("response")); + } else { + prop.put("mode_status", "1"); + + // "unresolved pattern", the remote peer is alive but had an exception + prop.putXML("mode_status_message", message); + } } catch (final NumberFormatException e) { prop.put("mode_status", "1"); diff --git a/htroot/Network.java b/htroot/Network.java index ee5539dd8..11416b7e9 100644 --- a/htroot/Network.java +++ b/htroot/Network.java @@ -27,6 +27,7 @@ // javac -classpath .:../classes Network.java // if the shell's current path is HTROOT +import java.net.MalformedURLException; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; @@ -39,6 +40,7 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.regex.PatternSyntaxException; +import net.yacy.cora.document.id.MultiProtocolURL; import net.yacy.cora.protocol.ClientIdentification; import net.yacy.cora.protocol.Domains; import net.yacy.cora.protocol.HeaderFramework; @@ -223,29 +225,40 @@ public class Network { map.put(Seed.IP, challengeIP); map.put(Seed.PORT, challengePort); Seed peer = post.get("peerHash") == null ? null : new Seed(post.get("peerHash"), map); - String challengeAddress = peer.getPublicAddress(challengeIP); sb.updateMySeed(); Seed mySeed = sb.peers.mySeed(); - final Map response = Protocol.hello(mySeed, sb.peers.peerActions, challengeAddress, peer.hash); - - if (response == null) { - Seed peerd = sb.peers.get(peer.hash); - if (peerd != null) peer = peerd; + final String challengeURLStr = peer.getPublicURL(challengeIP, false); + try { + final MultiProtocolURL challengeURL = new MultiProtocolURL(challengeURLStr); + final Map response = Protocol.hello(mySeed, sb.peers.peerActions, challengeURL, peer.hash); + + if (response == null) { + Seed peerd = sb.peers.get(peer.hash); + if (peerd != null) peer = peerd; + sb.peers.peerActions.interfaceDeparture(peer, challengeIP); + prop.put("table_comment",1); + prop.put("table_comment_status", "publish: no response from peer '" + peer.getName() + "/" + post.get("peerHash") + "' from " + challengeURL + ""); + } else { + String yourtype = response.get("yourtype"); + String yourip = response.get("yourip"); + peer = sb.peers.getConnected(peer.hash); + if (peer == null) { + prop.put("table_comment",1); + prop.put("table_comment_status","publish: disconnected peer 'UNKNOWN/" + post.get("peerHash") + "' from " + challengeURL + ", yourtype = " + yourtype + ", yourip = " + yourip); + } else { + prop.put("table_comment",2); + prop.put("table_comment_status","publish: handshaked " + peer.get(Seed.PEERTYPE, Seed.PEERTYPE_SENIOR) + " peer '" + peer.getName() + "' at " + challengeURL + ", yourtype = " + yourtype + ", yourip = " + yourip); + prop.putHTML("table_comment_details",peer.toString()); + } + } + } catch(final MalformedURLException e) { + final Seed peerd = sb.peers.get(peer.hash); + if (peerd != null) { + peer = peerd; + } sb.peers.peerActions.interfaceDeparture(peer, challengeIP); prop.put("table_comment",1); - prop.put("table_comment_status", "publish: no response from peer '" + peer.getName() + "/" + post.get("peerHash") + "' from " + challengeAddress + ""); - } else { - String yourtype = response.get("yourtype"); - String yourip = response.get("yourip"); - peer = sb.peers.getConnected(peer.hash); - if (peer == null) { - prop.put("table_comment",1); - prop.put("table_comment_status","publish: disconnected peer 'UNKNOWN/" + post.get("peerHash") + "' from " + challengeAddress + ", yourtype = " + yourtype + ", yourip = " + yourip); - } else { - prop.put("table_comment",2); - prop.put("table_comment_status","publish: handshaked " + peer.get(Seed.PEERTYPE, Seed.PEERTYPE_SENIOR) + " peer '" + peer.getName() + "' at " + challengeAddress + ", yourtype = " + yourtype + ", yourip = " + yourip); - prop.putHTML("table_comment_details",peer.toString()); - } + prop.put("table_comment_status", "publish: malformed URL for peer '" + peer.getName() + "/" + post.get("peerHash") + "' : " + challengeURLStr); } prop.putHTML("table_peerHash",post.get("peerHash")); diff --git a/htroot/yacy/hello.java b/htroot/yacy/hello.java index 5bed791be..a153204e9 100644 --- a/htroot/yacy/hello.java +++ b/htroot/yacy/hello.java @@ -29,10 +29,12 @@ import java.io.IOException; import java.net.InetAddress; +import java.net.MalformedURLException; import java.util.Iterator; import java.util.Set; import java.util.concurrent.ConcurrentMap; +import net.yacy.cora.document.id.MultiProtocolURL; import net.yacy.cora.protocol.Domains; import net.yacy.cora.protocol.HeaderFramework; import net.yacy.cora.protocol.RequestHeader; @@ -43,6 +45,7 @@ import net.yacy.peers.Seed; import net.yacy.peers.graphics.ProfilingGraph; import net.yacy.search.EventTracker; import net.yacy.search.Switchboard; +import net.yacy.search.SwitchboardConstants; import net.yacy.server.serverCore; import net.yacy.server.serverObjects; import net.yacy.server.serverSwitch; @@ -153,11 +156,14 @@ public final class hello { } final int connectedBefore = sb.peers.sizeConnected(); //ConcurrentLog.info("**hello-DEBUG**", "peer " + remoteSeed.getName() + " challenged us with IPs " + reportedips); + final boolean preferHttps = sb.getConfigBool(SwitchboardConstants.NETWORK_PROTOCOL_HTTPS_PREFERRED, + SwitchboardConstants.NETWORK_PROTOCOL_HTTPS_PREFERRED_DEFAULT); + final int totalTimeout = preferHttps ? 13000 : 6500; int callbackRemain = Math.min(5, reportedips.size()); - long callbackStart = System.currentTimeMillis(); - if (callbackRemain > 0 && reportedips.size() > 0) { + final long callbackStart = System.currentTimeMillis(); + if (callbackRemain > 0 && reportedips.size() > 0) { for (String reportedip: reportedips) { - int partialtimeout = ((int) (callbackStart + 6500 - System.currentTimeMillis())) / callbackRemain; // bad hack until a concurrent version is implemented + int partialtimeout = ((int) (callbackStart + totalTimeout - System.currentTimeMillis())) / callbackRemain; // bad hack until a concurrent version is implemented if (partialtimeout <= 0) break; //ConcurrentLog.info("**hello-DEBUG**", "reportedip = " + reportedip + " is handled"); if (Seed.isProperIP(reportedip)) { @@ -165,11 +171,24 @@ public final class hello { prop.put("yourip", reportedip); remoteSeed.setIP(reportedip); time = System.currentTimeMillis(); - callback = Protocol.queryRWICount(remoteSeed.getPublicAddress(reportedip), remoteSeed.hash, partialtimeout); + try { + MultiProtocolURL remoteBaseURL = remoteSeed.getPublicMultiprotocolURL(reportedip, preferHttps); + callback = Protocol.queryRWICount(remoteBaseURL, remoteSeed, partialtimeout); + if (callback[0] < 0 && remoteBaseURL.isHTTPS()) { + /* Failed using https : retry using http */ + remoteBaseURL = remoteSeed.getPublicMultiprotocolURL(reportedip, false); + callback = Protocol.queryRWICount(remoteBaseURL, remoteSeed, partialtimeout); + } + } catch(final MalformedURLException e) { + callback = new long[] {-1, -1}; + } //ConcurrentLog.info("**hello-DEBUG**", "reportedip = " + reportedip + " returns callback " + (callback == null ? "NULL" : callback[0])); time_backping = System.currentTimeMillis() - time; backping_method = "reportedip=" + reportedip; - if (callback[0] >= 0) { success = true; break; } + if (callback[0] >= 0) { + success = true; + break; + } if (--callbackRemain <= 0) break; // no more tries left / restrict to a limited number of ips } } diff --git a/source/net/yacy/peers/Network.java b/source/net/yacy/peers/Network.java index 32a26fc59..53bff58c6 100644 --- a/source/net/yacy/peers/Network.java +++ b/source/net/yacy/peers/Network.java @@ -54,6 +54,7 @@ import net.yacy.cora.document.encoding.ASCII; import net.yacy.cora.document.feed.RSSFeed; import net.yacy.cora.document.feed.RSSMessage; import net.yacy.cora.document.id.DigestURL; +import net.yacy.cora.document.id.MultiProtocolURL; import net.yacy.cora.protocol.Domains; import net.yacy.cora.util.ConcurrentLog; import net.yacy.peers.operation.yacySeedUploadFile; @@ -194,27 +195,51 @@ public class Network protected class publishThread extends Thread { - private Map result; private final Seed seed; public publishThread(final ThreadGroup tg, final Seed seed) { super(tg, "PublishSeed_" + seed.getName()); this.seed = seed; - this.result = null; } @Override public final void run() { + Map result = null; try { - for (String ip: this.seed.getIPs()) { - this.result = Protocol.hello(Network.this.sb.peers.mySeed(), Network.this.sb.peers.peerActions, this.seed.getPublicAddress(ip), this.seed.hash); - if ( this.result == null ) { - // no or wrong response, delete that address - final String cause = "peer ping to peer resulted in error response (added < 0)"; + final boolean preferHttps = Network.this.sb.getConfigBool( + SwitchboardConstants.NETWORK_PROTOCOL_HTTPS_PREFERRED, + SwitchboardConstants.NETWORK_PROTOCOL_HTTPS_PREFERRED_DEFAULT); + for (final String ip: this.seed.getIPs()) { + try { + MultiProtocolURL targetBaseURL = this.seed.getPublicMultiprotocolURL(ip, preferHttps); + result = Protocol.hello(Network.this.sb.peers.mySeed(), Network.this.sb.peers.peerActions, targetBaseURL, this.seed.hash); + if (result == null && targetBaseURL.isHTTPS()) { + /* Failed with https : retry with http on the same address */ + targetBaseURL = this.seed.getPublicMultiprotocolURL(ip, false); + result = Protocol.hello(Network.this.sb.peers.mySeed(), Network.this.sb.peers.peerActions, + targetBaseURL, this.seed.hash); + if (result != null) { + /* Got a result using http : mark SSL as unavailable on the peer */ + log.info("publish: SSL/TLS unavailable on " + this.seed.get(Seed.PEERTYPE, Seed.PEERTYPE_SENIOR) + " peer '" + + this.seed.getName() + "' : can be reached using http but not https on address " + + ip); + this.seed.setFlagSSLAvailable(false); + Network.this.sb.peers.updateConnected(this.seed); + } + } + if(result == null) { + // no or wrong response, delete that address + final String cause = "peer ping to peer resulted in error response (added < 0)"; + log.info("publish: disconnected " + this.seed.get(Seed.PEERTYPE, Seed.PEERTYPE_SENIOR) + " peer '" + this.seed.getName() + "' from " + this.seed.getIPs() + ": " + cause); + Network.this.sb.peers.peerActions.interfaceDeparture(this.seed, ip); + continue; + } + } catch(final MalformedURLException e) { + final String cause = "malformed peer URL"; log.info("publish: disconnected " + this.seed.get(Seed.PEERTYPE, Seed.PEERTYPE_SENIOR) + " peer '" + this.seed.getName() + "' from " + this.seed.getIPs() + ": " + cause); Network.this.sb.peers.peerActions.interfaceDeparture(this.seed, ip); - continue; - } + continue; + } // success! we have published our peer to a senior peer // update latest news from the other peer log.info("publish: handshaked "+ this.seed.get(Seed.PEERTYPE, Seed.PEERTYPE_SENIOR) + " peer '" + this.seed.getName() + "' at " + this.seed.getIPs()); diff --git a/source/net/yacy/peers/Protocol.java b/source/net/yacy/peers/Protocol.java index 94f9db8ed..8f5345681 100644 --- a/source/net/yacy/peers/Protocol.java +++ b/source/net/yacy/peers/Protocol.java @@ -58,7 +58,6 @@ import java.util.LinkedHashMap; import java.util.LinkedHashSet; import java.util.List; import java.util.Map; -import java.util.Set; import java.util.TreeMap; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; @@ -138,32 +137,37 @@ public final class Protocol { */ public static class Post { - public byte[] result; // contains the result from a successful post or null if no attempt was successful - public Set unsuccessfulAddresses; // contains a set of addresses which had been tested for submission was without success - public String successfulAddress; // contains the address which had been successfully used or null if no success with any Address + /** Contains the result from a successful post or null if no attempt was successful */ + private byte[] result; + /** + * @param targetBaseURL the base target URL + * @param targetHash the hash of the target peer + * @param path the path on the base URL + * @param parts the body content + * @param timeout the timeout in milliseconds + * @param httpFallback when true, retry as http when a https request failed + * @throws IOException + */ public Post( - final String targetAddress, - final String targetPeerHash, + final MultiProtocolURL targetBaseURL, + final String targetHash, final String path, final Map parts, final int timeout) throws IOException { final HTTPClient httpClient = new HTTPClient(ClientIdentification.yacyInternetCrawlerAgent); httpClient.setTimout(timeout); - this.result = httpClient.POSTbytes( - new MultiProtocolURL("http://" + targetAddress + path), - Seed.b64Hash2hexHash(targetPeerHash) + ".yacyh", - parts, - false, true); - this.unsuccessfulAddresses = new HashSet<>(); - if (this.result == null) { - this.unsuccessfulAddresses.add(targetAddress); - this.successfulAddress = null; - } else { - this.successfulAddress = targetAddress; - } + MultiProtocolURL targetURL = new MultiProtocolURL(targetBaseURL, path); + this.result = httpClient.POSTbytes(targetURL, Seed.b64Hash2hexHash(targetHash) + ".yacyh", parts, false, + true); } - + + /** + * @return the result from a successful post or null if no attempt was successful + */ + public byte[] getResult() { + return this.result; + } } /** @@ -182,7 +186,7 @@ public final class Protocol { public static Map hello( final Seed mySeed, final PeerActions peerActions, - final String targetAddress, + final MultiProtocolURL targetBaseURL, final String targetHash) { Map result = null; @@ -202,7 +206,7 @@ public final class Protocol { final HTTPClient httpClient = new HTTPClient(ClientIdentification.yacyInternetCrawlerAgent, 30000); content = httpClient.POSTbytes( - new MultiProtocolURL("http://" + targetAddress + "/yacy/hello.html"), + new MultiProtocolURL(targetBaseURL, "/yacy/hello.html"), Seed.b64Hash2hexHash(targetHash) + ".yacyh", parts, false, true); @@ -212,7 +216,7 @@ public final class Protocol { if ( Thread.currentThread().isInterrupted() ) {Network.log.info("yacyClient.hello thread '" + Thread.currentThread().getName() + "' interrupted."); return null; } - Network.log.info("yacyClient.hello thread '" + Thread.currentThread().getName() + "', peer " + targetAddress + "; exception: " + e.getMessage()); + Network.log.info("yacyClient.hello thread '" + Thread.currentThread().getName() + "', peer " + targetBaseURL + "; exception: " + e.getMessage()); // try again (go into loop) result = null; } @@ -222,7 +226,7 @@ public final class Protocol { + ((result == null) ? "result null" : ("result=" + result.toString()))); return null; } - Network.log.info("yacyClient.hello thread '" + Thread.currentThread().getName() + "' contacted peer at " + targetAddress + ", received " + ((content == null) ? "null" : content.length) + " bytes, time = " + responseTime + " milliseconds"); + Network.log.info("yacyClient.hello thread '" + Thread.currentThread().getName() + "' contacted peer at " + targetBaseURL + ", received " + ((content == null) ? "null" : content.length) + " bytes, time = " + responseTime + " milliseconds"); // check consistency with expectation Seed otherPeer = null; @@ -233,7 +237,7 @@ public final class Protocol { } else { try { // patch the remote peer address to avoid that remote peers spoof the network with wrong addresses - String host = Domains.stripToHostName(targetAddress); + String host = targetBaseURL.getHost(); InetAddress ie = Domains.dnsResolve(host); otherPeer = Seed.genRemoteSeed(seed, false, ie.getHostAddress()); if ( !otherPeer.hash.equals(targetHash) ) { @@ -342,7 +346,7 @@ public final class Protocol { } else { try { if ( i == 1 ) { - String host = Domains.stripToHostName(targetAddress); + String host = targetBaseURL.getHost(); InetAddress ia = Domains.dnsResolve(host); if (ia == null) continue; host = ia.getHostAddress(); // the actual address of the target as we had been successful when contacting them is patched here @@ -364,23 +368,24 @@ public final class Protocol { return result; } - public static long[] queryRWICount(final String targetAddress, final String targetHash, int timeout) { + public static long[] queryRWICount(final MultiProtocolURL targetBaseURL, final Seed target, int timeout) { // prepare request final String salt = crypt.randomSalt(); // send request try { - final Map parts = basicRequestParts(Switchboard.getSwitchboard(), targetHash, salt); + final Map parts = basicRequestParts(Switchboard.getSwitchboard(), target.hash, salt); parts.put("object", UTF8.StringBody("rwicount")); parts.put("env", UTF8.StringBody("")); - //ConcurrentLog.info("**hello-DEBUG**queryRWICount**", "posting request to " + targetAddress); - final Post post = new Post(targetAddress, targetHash, "/yacy/query.html", parts, timeout); - //ConcurrentLog.info("**hello-DEBUG**queryRWICount**", "received CONTENT from requesting " + targetAddress + (post.result == null ? "NULL" : (": length = " + post.result.length))); + //ConcurrentLog.info("**hello-DEBUG**queryRWICount**", "posting request to " + targetBaseURL); + final Post post = new Post(targetBaseURL, target.hash, "/yacy/query.html", parts, timeout); + + //ConcurrentLog.info("**hello-DEBUG**queryRWICount**", "received CONTENT from requesting " + targetBaseURL + (post.result == null ? "NULL" : (": length = " + post.result.length))); final Map result = FileUtils.table(post.result); if (result == null || result.isEmpty()) return new long[] {-1, -1}; - //ConcurrentLog.info("**hello-DEBUG**queryRWICount**", "received RESULT from requesting " + targetAddress + " : result = " + result.toString()); + //ConcurrentLog.info("**hello-DEBUG**queryRWICount**", "received RESULT from requesting " + targetBaseURL + " : result = " + result.toString()); final String resp = result.get("response"); - //ConcurrentLog.info("**hello-DEBUG**queryRWICount**", "received RESPONSE from requesting " + targetAddress + " : response = " + resp); + //ConcurrentLog.info("**hello-DEBUG**queryRWICount**", "received RESPONSE from requesting " + targetBaseURL + " : response = " + resp); if (resp == null) return new long[] {-1, -1}; String magic = result.get("magic"); if (magic == null) magic = "0"; @@ -390,7 +395,7 @@ public final class Protocol { return new long[] {-1, -1}; } } catch (final Exception e ) { - //ConcurrentLog.info("**hello-DEBUG**queryRWICount**", "received EXCEPTION from requesting " + targetAddress + ": " + e.getMessage()); + //ConcurrentLog.info("**hello-DEBUG**queryRWICount**", "received EXCEPTION from requesting " + targetBaseURL + ": " + e.getMessage()); if (Network.log.isFine()) Network.log.fine("yacyClient.queryRWICount error:" + e.getMessage()); return new long[] {-1, -1}; } @@ -518,7 +523,8 @@ public final class Protocol { Network.log.info("SEARCH failed, Peer: " + target.hash + ":" + target.getName() + " (" + e.getMessage() + ")"); if(targetBaseURL.startsWith("https")) { /* First mark https unavailable on this peer before removing any interface */ - target.setFlagSSLAvailable(false); + target.setFlagSSLAvailable(false); + event.peers.updateConnected(target); } else { event.peers.peerActions.interfaceDeparture(target, ip); } @@ -608,7 +614,8 @@ public final class Protocol { Network.log.info("SEARCH failed, Peer: " + target.hash + ":" + target.getName() + " (" + e.getMessage() + ")"); if(targetBaseURL.startsWith("https")) { /* First mark https unavailable on this peer before removing any interface */ - target.setFlagSSLAvailable(false); + target.setFlagSSLAvailable(false); + event.peers.updateConnected(target); } else { event.peers.peerActions.interfaceDeparture(target, ip); } @@ -1222,11 +1229,11 @@ public final class Protocol { if(targetBaseURL.startsWith("https")) { /* First mark https unavailable on this peer before removing anything else */ target.setFlagSSLAvailable(false); + event.peers.updateConnected(target); } else { target.setFlagSolrAvailable(false); } } - target.setFlagSolrAvailable(false || myseed); return -1; } } catch(InterruptedException e) { @@ -1477,26 +1484,34 @@ public final class Protocol { return true; } - public static Map permissionMessage(final String targetAddress, final String targetHash) { - // ask for allowed message size and attachment size - // if this replies null, the peer does not answer - + /** + * Post a request asking for allowed message size and attachment size to the + * target peer on the selected target ip. All parameters must not be null. + * + * @param targetBaseURL + * the public base URL of the target peer on one of its reported IP + * addresses in {@link Seed#getIPs()} + * @param target + * the target peer + * @param sb + * the switchboard instance + * @return the result of the request + * @throws IOException + * when the peer doesn't answer on this IP or any other error + * occurred + */ + public static Map permissionMessage(final MultiProtocolURL targetBaseURL, final Seed target, + final Switchboard sb) throws IOException { // prepare request final String salt = crypt.randomSalt(); // send request - try { - final Map parts = - basicRequestParts(Switchboard.getSwitchboard(), targetHash, salt); - parts.put("process", UTF8.StringBody("permission")); - final Post post = new Post(targetAddress, targetAddress, "/yacy/message.html", parts, 6000); - final Map result = FileUtils.table(post.result); - return result; - } catch (final Exception e ) { - // most probably a network time-out exception - Network.log.warn("yacyClient.permissionMessage error:" + e.getMessage()); - return null; - } + final Map parts = basicRequestParts(sb, target.hash, salt); + parts.put("process", UTF8.StringBody("permission")); + final Post post = new Post(targetBaseURL, target.hash, "/yacy/message.html", parts, 6000); + + final Map result = FileUtils.table(post.result); + return result; } public static Map crawlReceipt( @@ -1702,12 +1717,21 @@ public final class Protocol { final ReferenceContainerCache indexes, boolean gzipBody, final int timeout) { - for (String ip : targetSeed.getIPs()) { + final boolean preferHttps = Switchboard.getSwitchboard().getConfigBool(SwitchboardConstants.NETWORK_PROTOCOL_HTTPS_PREFERRED, SwitchboardConstants.NETWORK_PROTOCOL_HTTPS_PREFERRED_DEFAULT); + for (final String ip : targetSeed.getIPs()) { if (ip == null) { Network.log.warn("no address for transferRWI"); return null; } - final String address = targetSeed.getPublicAddress(ip); + MultiProtocolURL targetBaseURL = null; + try { + targetBaseURL = targetSeed.getPublicMultiprotocolURL(ip, preferHttps); + } catch(final MalformedURLException e) { + Network.log.info("yacyClient.transferRWI malformed target URL : " + targetBaseURL); + // disconnect unavailable peer ip + Switchboard.getSwitchboard().peers.peerActions.interfaceDeparture(targetSeed, ip); + continue; + } // prepare post values final String salt = crypt.randomSalt(); @@ -1746,12 +1770,25 @@ public final class Protocol { parts.put("entryc", UTF8.StringBody(Integer.toString(indexcount))); parts.put("indexes", UTF8.StringBody(entrypost.toString())); final HTTPClient httpClient = new HTTPClient(ClientIdentification.yacyInternetCrawlerAgent, timeout); - final byte[] content = - httpClient.POSTbytes( - new MultiProtocolURL("http://" + address + "/yacy/transferRWI.html"), - targetSeed.getHexHash() + ".yacyh", - parts, - gzipBody, true); + byte[] content = null; + try { + content = httpClient.POSTbytes(new MultiProtocolURL(targetBaseURL, "/yacy/transferRWI.html"), + targetSeed.getHexHash() + ".yacyh", parts, gzipBody, true); + } catch(final IOException e) { + if(targetBaseURL.isHTTPS()) { + targetBaseURL = targetSeed.getPublicMultiprotocolURL(ip, false); + /* Failed with https : retry with http on the same address */ + content = httpClient.POSTbytes(new MultiProtocolURL(targetBaseURL, "/yacy/transferRWI.html"), + targetSeed.getHexHash() + ".yacyh", parts, gzipBody, true); + if(content != null) { + /* Success with http : mark SSL as unavailable on the target peer */ + Network.log.info("yacyClient.transferRWI SSL unavailable on address " + ip); + Switchboard.getSwitchboard().peers.updateConnected(targetSeed); + } + } else { + throw e; + } + } final Iterator v = FileUtils.strings(content); // this should return a list of urlhashes that are unknown @@ -1761,7 +1798,7 @@ public final class Protocol { result.put(Seed.IP, ip); // add used ip to result for error handling (in case no "result" key was received) return result; } catch (final Exception e ) { - Network.log.info("yacyClient.transferRWI to " + address + " error: " + e.getMessage()); + Network.log.info("yacyClient.transferRWI to " + targetBaseURL + " error: " + e.getMessage()); // disconnect unavailable peer ip Switchboard.getSwitchboard().peers.peerActions.interfaceDeparture(targetSeed, ip); } @@ -1908,7 +1945,10 @@ public final class Protocol { final Map parts = basicRequestParts(Switchboard.getSwitchboard(), target.hash, salt); parts.put("object", UTF8.StringBody("host")); - final Post post = new Post(target.getPublicAddress(target.getIP()), target.hash, "/yacy/idx.json", parts, 30000); + final String remoteBaseURL = target.getPublicURL(target.getIP(), + Switchboard.getSwitchboard().getConfigBool(SwitchboardConstants.NETWORK_PROTOCOL_HTTPS_PREFERRED, + SwitchboardConstants.NETWORK_PROTOCOL_HTTPS_PREFERRED_DEFAULT)); + final Post post = new Post(new MultiProtocolURL(remoteBaseURL), target.hash, "/yacy/idx.json", parts, 30000); if ( post.result == null || post.result.length == 0 ) { Network.log.warn("yacyClient.loadIDXHosts error: empty result"); return null; diff --git a/source/net/yacy/peers/Seed.java b/source/net/yacy/peers/Seed.java index 5ec1ce52c..562b0eb93 100644 --- a/source/net/yacy/peers/Seed.java +++ b/source/net/yacy/peers/Seed.java @@ -68,6 +68,7 @@ import net.yacy.cora.date.AbstractFormatter; import net.yacy.cora.date.GenericFormatter; import net.yacy.cora.document.encoding.ASCII; import net.yacy.cora.document.encoding.UTF8; +import net.yacy.cora.document.id.MultiProtocolURL; import net.yacy.cora.federate.yacy.Distribution; import net.yacy.cora.order.Base64Order; import net.yacy.cora.order.Digest; @@ -791,7 +792,7 @@ public class Seed implements Cloneable, Comparable, Comparator * @return an URL string for the given peer ip * @throws RuntimeException when the ip parameter is null */ - public final String getPublicURL(final String ip, final boolean preferHTTPS) { + public final String getPublicURL(final String ip, final boolean preferHTTPS) throws RuntimeException { if (ip == null) { throw new RuntimeException("ip == NULL"); // that should not happen in Peer-to-Peer mode (but can in Intranet mode) } @@ -822,6 +823,27 @@ public class Seed implements Cloneable, Comparable, Comparator } return sb.toString(); } + + /** + * Generate a public URL Multiprotocol instance using a given ip. This combines + * the ip with the http(s) port and encloses the ip with square brackets if the + * ip is of typeIPv6 + * + * @param ip + * a host name or ip address + * @param preferHTTPS + * when true and https is available on this Seed, use it as the + * scheme part of the url + * @return an MultiProtocolURL instance for the given peer ip + * @throws RuntimeException + * when the ip parameter is null + * @throws MalformedURLException + * when the ip and port could not make a well formed URL + */ + public final MultiProtocolURL getPublicMultiprotocolURL(final String ip, final boolean preferHTTPS) + throws RuntimeException, MalformedURLException { + return new MultiProtocolURL(getPublicURL(ip, preferHTTPS)); + } /** @return the port number of this seed or -1 if not present */ public final int getPort() { diff --git a/source/net/yacy/search/SwitchboardConstants.java b/source/net/yacy/search/SwitchboardConstants.java index 88860ef77..71b91cab9 100644 --- a/source/net/yacy/search/SwitchboardConstants.java +++ b/source/net/yacy/search/SwitchboardConstants.java @@ -520,6 +520,13 @@ public final class SwitchboardConstants { public static final String NETWORK_BOOTSTRAP_SEEDLIST_STUB = "network.unit.bootstrap.seedlist"; public static final String NETWORK_SEARCHVERIFY = "network.unit.inspection.searchverify"; + + /** Key of the setting controlling whether https should be preferred for in-protocol operations when available on remote peers. + * A distinct general setting is available to control whether https sould be used for remote search queries : see {@link #REMOTESEARCH_HTTPS_PREFERRED} */ + public static final String NETWORK_PROTOCOL_HTTPS_PREFERRED = "network.unit.protocol.https.preferred"; + + /** Default setting value controlling whether https should be preferred for in-protocol operations when available on remote peers */ + public static final boolean NETWORK_PROTOCOL_HTTPS_PREFERRED_DEFAULT = false; /** * appearance