mirror of
https://github.com/yacy/yacy_search_server.git
synced 2024-09-19 00:01:41 +02:00
use DigestURL in UrlProxyServlet as parameter to pass requested url to
handler. UrlProxyServlet splits url in parts to pass it on as parameter and HeaderFramework constructs a url from param parts. This is obsolete if already created url is used (makes HeaderFramework.getRequestURL obsolete = removed)
This commit is contained in:
reger
parent
3630fcc458
commit
4eeb448eb3
|
|
@ -24,12 +24,10 @@ import java.io.BufferedReader;
|
|||
import java.io.File;
|
||||
import java.io.FileOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.net.MalformedURLException;
|
||||
import java.text.ParseException;
|
||||
import java.text.SimpleDateFormat;
|
||||
import java.util.Calendar;
|
||||
import java.util.Date;
|
||||
import java.util.HashMap;
|
||||
import java.util.Iterator;
|
||||
import java.util.Locale;
|
||||
import java.util.Map;
|
||||
|
|
@ -40,10 +38,8 @@ import java.util.concurrent.ConcurrentHashMap;
|
|||
|
||||
import net.yacy.cora.document.encoding.ASCII;
|
||||
import net.yacy.cora.document.encoding.UTF8;
|
||||
import net.yacy.cora.document.id.DigestURL;
|
||||
import net.yacy.cora.util.CommonPattern;
|
||||
import net.yacy.cora.util.ConcurrentLog;
|
||||
import net.yacy.cora.util.NumberTools;
|
||||
|
||||
|
||||
/**
|
||||
|
|
@ -200,7 +196,6 @@ public class HeaderFramework extends TreeMap<String, String> implements Map<Stri
|
|||
public static final String CONNECTION_PROP_METHOD = "METHOD";
|
||||
public static final String CONNECTION_PROP_PATH = "PATH";
|
||||
public static final String CONNECTION_PROP_EXT = "EXT";
|
||||
public static final String CONNECTION_PROP_URL = "URL";
|
||||
public static final String CONNECTION_PROP_ARGS = "ARGS";
|
||||
public static final String CONNECTION_PROP_CLIENTIP = "CLIENTIP";
|
||||
public static final String CONNECTION_PROP_PERSISTENT = "PERSISTENT";
|
||||
|
|
@ -208,6 +203,7 @@ public class HeaderFramework extends TreeMap<String, String> implements Map<Stri
|
|||
public static final String CONNECTION_PROP_REQUEST_END = "REQUEST_END";
|
||||
|
||||
/* PROPERTIES: Client -> Proxy */
|
||||
public static final String CONNECTION_PROP_DIGESTURL = "URL"; // value DigestURL object
|
||||
public static final String CONNECTION_PROP_CLIENT_HTTPSERVLETREQUEST = "CLIENT_HTTPSERVLETREQUEST";
|
||||
|
||||
/* PROPERTIES: Proxy -> Client */
|
||||
|
|
@ -565,36 +561,6 @@ public class HeaderFramework extends TreeMap<String, String> implements Map<Stri
|
|||
theHeader.append("\r\n");
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate a url from Header properties
|
||||
* @param conProp containing host, path, query and protocol (defaults to http if missing)
|
||||
* @return url
|
||||
* @throws MalformedURLException
|
||||
*/
|
||||
public static DigestURL getRequestURL(final HashMap<String, Object> conProp) throws MalformedURLException {
|
||||
String host = (String) conProp.get(HeaderFramework.CONNECTION_PROP_HOST);
|
||||
final String path = (String) conProp.get(HeaderFramework.CONNECTION_PROP_PATH); // always starts with leading '/'
|
||||
final String args = (String) conProp.get(HeaderFramework.CONNECTION_PROP_ARGS); // may be null if no args were given
|
||||
String protocol = (String) conProp.get(HeaderFramework.CONNECTION_PROP_PROTOCOL);
|
||||
if (protocol == null) protocol = "http";
|
||||
//String ip = conProp.getProperty(httpHeader.CONNECTION_PROP_CLIENTIP); // the ip from the connecting peer
|
||||
|
||||
int port, pos;
|
||||
if ((pos = host.lastIndexOf(':')) < 0) {
|
||||
port = Domains.stripToPort(protocol + "://" + host); // use stripToPort to get default ports
|
||||
} else {
|
||||
if (pos > host.indexOf(']')) { // check for ipv6
|
||||
port = NumberTools.parseIntDecSubstring(host, pos + 1);
|
||||
host = host.substring(0, pos);
|
||||
} else {
|
||||
port = Domains.stripToPort(protocol + "://" + host); // use stripToPort to get default ports
|
||||
}
|
||||
}
|
||||
|
||||
final DigestURL url = new DigestURL(protocol, host, port, (args == null) ? path : path + "?" + args);
|
||||
return url;
|
||||
}
|
||||
|
||||
/**
|
||||
* Reading http headers from a reader class and building up a httpHeader object
|
||||
* @param reader the {@link BufferedReader} that is used to read the http header lines
|
||||
|
|
|
|||
|
|
@ -5,7 +5,6 @@ import java.io.ByteArrayOutputStream;
|
|||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.net.MalformedURLException;
|
||||
import java.net.URL;
|
||||
import java.net.URLDecoder;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.HashMap;
|
||||
|
|
@ -122,7 +121,7 @@ public class UrlProxyServlet extends HttpServlet implements Servlet {
|
|||
return;
|
||||
}
|
||||
// 2 - get target url
|
||||
URL proxyurl = null;
|
||||
DigestURL proxyurl = null;
|
||||
final String strUrl = request.getParameter("url");
|
||||
if (strUrl == null) {
|
||||
response.sendError(HttpServletResponse.SC_NOT_FOUND,"url parameter missing");
|
||||
|
|
@ -130,9 +129,9 @@ public class UrlProxyServlet extends HttpServlet implements Servlet {
|
|||
}
|
||||
|
||||
try {
|
||||
proxyurl = new URL(strUrl);
|
||||
proxyurl = new DigestURL(strUrl);
|
||||
} catch (final MalformedURLException e) {
|
||||
proxyurl = new URL(URLDecoder.decode(strUrl, StandardCharsets.UTF_8.name()));
|
||||
proxyurl = new DigestURL(URLDecoder.decode(strUrl, StandardCharsets.UTF_8.name()));
|
||||
}
|
||||
|
||||
if (proxyurl == null) {
|
||||
|
|
@ -151,11 +150,8 @@ public class UrlProxyServlet extends HttpServlet implements Servlet {
|
|||
|
||||
final HashMap<String, Object> prop = new HashMap<String, Object>();
|
||||
prop.put(HeaderFramework.CONNECTION_PROP_HTTP_VER, HeaderFramework.HTTP_VERSION_1_1);
|
||||
prop.put(HeaderFramework.CONNECTION_PROP_PROTOCOL, proxyurl.getProtocol());
|
||||
prop.put(HeaderFramework.CONNECTION_PROP_HOST, hostwithport);
|
||||
prop.put(HeaderFramework.CONNECTION_PROP_PATH, proxyurl.getPath().replaceAll(" ", "%20"));
|
||||
prop.put(HeaderFramework.CONNECTION_PROP_DIGESTURL, proxyurl);
|
||||
prop.put(HeaderFramework.CONNECTION_PROP_METHOD, request.getMethod()); // only needed for HTTPDeamon errormsg in case of blacklisted url
|
||||
if (proxyurl.getQuery() != null) prop.put(HeaderFramework.CONNECTION_PROP_ARGS, proxyurl.getQuery());
|
||||
prop.put(HeaderFramework.CONNECTION_PROP_CLIENTIP, Domains.LOCALHOST);
|
||||
prop.put(HeaderFramework.CONNECTION_PROP_CLIENT_HTTPSERVLETREQUEST, request);
|
||||
|
||||
|
|
|
|||
|
|
@ -6,7 +6,6 @@ import java.io.IOException;
|
|||
import java.io.InputStream;
|
||||
import java.io.StringWriter;
|
||||
import java.net.MalformedURLException;
|
||||
import java.net.URL;
|
||||
import java.net.URLDecoder;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.HashMap;
|
||||
|
|
@ -91,7 +90,7 @@ public class YaCyProxyServlet extends HttpServlet implements Servlet {
|
|||
response.sendError(HttpServletResponse.SC_GATEWAY_TIMEOUT); // Need better test that isInitial
|
||||
return;
|
||||
}
|
||||
URL proxyurl = null;
|
||||
DigestURL proxyurl = null;
|
||||
String strARGS = request.getQueryString();
|
||||
if (strARGS == null) {
|
||||
response.sendError(HttpServletResponse.SC_NOT_FOUND,"url parameter missing");
|
||||
|
|
@ -102,10 +101,9 @@ public class YaCyProxyServlet extends HttpServlet implements Servlet {
|
|||
final String strUrl = strARGS.substring(4); // strip "url="
|
||||
|
||||
try {
|
||||
proxyurl = new URL(strUrl);
|
||||
proxyurl = new DigestURL(strUrl);
|
||||
} catch (final MalformedURLException e) {
|
||||
proxyurl = new URL(URLDecoder.decode(strUrl, StandardCharsets.UTF_8.name()));
|
||||
|
||||
proxyurl = new DigestURL(URLDecoder.decode(strUrl, StandardCharsets.UTF_8.name()));
|
||||
}
|
||||
}
|
||||
if (proxyurl == null) {
|
||||
|
|
@ -123,9 +121,7 @@ public class YaCyProxyServlet extends HttpServlet implements Servlet {
|
|||
|
||||
final HashMap<String, Object> prop = new HashMap<String, Object>();
|
||||
prop.put(HeaderFramework.CONNECTION_PROP_HTTP_VER, HeaderFramework.HTTP_VERSION_1_1);
|
||||
prop.put(HeaderFramework.CONNECTION_PROP_PROTOCOL, proxyurl.getProtocol());
|
||||
prop.put(HeaderFramework.CONNECTION_PROP_HOST, hostwithport);
|
||||
prop.put(HeaderFramework.CONNECTION_PROP_PATH, proxyurl.getPath().replaceAll(" ", "%20"));
|
||||
prop.put(HeaderFramework.CONNECTION_PROP_DIGESTURL, proxyurl);
|
||||
prop.put(HeaderFramework.CONNECTION_PROP_CLIENTIP, Domains.LOCALHOST);
|
||||
prop.put(HeaderFramework.CONNECTION_PROP_CLIENT_HTTPSERVLETREQUEST, request);
|
||||
|
||||
|
|
|
|||
|
|
@ -277,10 +277,8 @@ public final class HTTPDProxyHandler {
|
|||
|
||||
final String ip = (String) conProp.get(HeaderFramework.CONNECTION_PROP_CLIENTIP); // the ip from the connecting peer
|
||||
|
||||
DigestURL url = null;
|
||||
try {
|
||||
url = HeaderFramework.getRequestURL(conProp);
|
||||
if (log.isFine()) log.fine(reqID +" GET "+ url);
|
||||
DigestURL url = (DigestURL) conProp.get(HeaderFramework.CONNECTION_PROP_DIGESTURL);
|
||||
if (log.isFine()) log.fine(reqID +" GET "+ url.toString());
|
||||
if (log.isFinest()) log.finest(reqID +" header: "+ requestHeader);
|
||||
|
||||
//redirector
|
||||
|
|
@ -293,25 +291,17 @@ public final class HTTPDProxyHandler {
|
|||
if (!newUrl.equals("")) {
|
||||
try {
|
||||
url = new DigestURL(newUrl);
|
||||
} catch(final MalformedURLException e){}//just keep the old one
|
||||
} catch (final MalformedURLException e) {
|
||||
}//just keep the old one
|
||||
}
|
||||
if (log.isFinest()) log.finest(reqID +" using redirector to "+ url);
|
||||
conProp.put(HeaderFramework.CONNECTION_PROP_HOST, url.getHost()+":"+url.getPort());
|
||||
conProp.put(HeaderFramework.CONNECTION_PROP_PATH, url.getPath());
|
||||
if (log.isFinest()) {
|
||||
log.finest(reqID + " using redirector to " + url);
|
||||
}
|
||||
conProp.put(HeaderFramework.CONNECTION_PROP_DIGESTURL, url);
|
||||
requestHeader.put(HeaderFramework.HOST, url.getHost() + ":" + url.getPort());
|
||||
requestHeader.put(HeaderFramework.CONNECTION_PROP_PATH, url.getPath());
|
||||
}
|
||||
} catch (final MalformedURLException e) {
|
||||
// get header info for error logging
|
||||
final String host = (String) conProp.get(HeaderFramework.CONNECTION_PROP_HOST);
|
||||
final String path = (String) conProp.get(HeaderFramework.CONNECTION_PROP_PATH); // always starts with leading '/'
|
||||
final String args = (String) conProp.get(HeaderFramework.CONNECTION_PROP_ARGS); // may be null if no args were given
|
||||
final String errorMsg = "ERROR: internal error with url generation: host=" +
|
||||
host + ", path=" + path + ", args=" + args;
|
||||
log.severe(errorMsg);
|
||||
HTTPDemon.sendRespondError(conProp,countedRespond,4,501,null,errorMsg,e);
|
||||
return;
|
||||
}
|
||||
|
||||
|
||||
// check the blacklist
|
||||
// blacklist idea inspired by [AS]:
|
||||
|
|
@ -1049,14 +1039,13 @@ public final class HTTPDProxyHandler {
|
|||
}));
|
||||
|
||||
// getting some connection properties
|
||||
String orgHostPort = "80";
|
||||
String orgHostName = (String) conProp.get(HeaderFramework.CONNECTION_PROP_HOST);
|
||||
DigestURL orgurl = (DigestURL) conProp.get(HeaderFramework.CONNECTION_PROP_DIGESTURL);
|
||||
int orgHostPort = orgurl.getPort();
|
||||
String orgHostName = orgurl.getHost();
|
||||
if (orgHostName == null) orgHostName = "unknown";
|
||||
orgHostName = orgHostName.toLowerCase();
|
||||
orgHostPort = Integer.toString(Domains.stripToPort(orgHostName));
|
||||
orgHostName = Domains.stripToHostName(orgHostName);
|
||||
String orgHostPath = (String) conProp.get(HeaderFramework.CONNECTION_PROP_PATH); if (orgHostPath == null) orgHostPath = "";
|
||||
String orgHostArgs = (String) conProp.get(HeaderFramework.CONNECTION_PROP_ARGS); if (orgHostArgs == null) orgHostArgs = "";
|
||||
String orgHostPath = orgurl.getPath(); if (orgHostPath == null) orgHostPath = "";
|
||||
String orgHostArgs = orgurl.getSearchpart();; if (orgHostArgs == null) orgHostArgs = "";
|
||||
if (orgHostArgs.length() > 0) orgHostArgs = "?" + orgHostArgs;
|
||||
detailedErrorMsgMap.put("hostName", orgHostName);
|
||||
|
||||
|
|
@ -1183,13 +1172,8 @@ public final class HTTPDProxyHandler {
|
|||
logMessage.append(' ');
|
||||
|
||||
// URL
|
||||
final String requestURL = (String) conProp.get(HeaderFramework.CONNECTION_PROP_URL);
|
||||
final String requestArgs = (String) conProp.get(HeaderFramework.CONNECTION_PROP_ARGS);
|
||||
logMessage.append(requestURL);
|
||||
if (requestArgs != null) {
|
||||
logMessage.append("?")
|
||||
.append(requestArgs);
|
||||
}
|
||||
final DigestURL requestURL = (DigestURL) conProp.get(HeaderFramework.CONNECTION_PROP_DIGESTURL);
|
||||
logMessage.append(requestURL.toString());
|
||||
logMessage.append(' ');
|
||||
|
||||
// Rfc931
|
||||
|
|
@ -1197,9 +1181,8 @@ public final class HTTPDProxyHandler {
|
|||
logMessage.append(' ');
|
||||
|
||||
// Peerstatus/Peerhost
|
||||
final String host = (String) conProp.get(HeaderFramework.CONNECTION_PROP_HOST);
|
||||
logMessage.append("DIRECT/");
|
||||
logMessage.append(host);
|
||||
logMessage.append(requestURL.getHost());
|
||||
logMessage.append(' ');
|
||||
|
||||
// Type
|
||||
|
|
|
|||
|
|
@ -30,14 +30,12 @@ import java.io.FileInputStream;
|
|||
import java.io.IOException;
|
||||
import java.io.OutputStream;
|
||||
import java.io.PrintStream;
|
||||
import java.net.MalformedURLException;
|
||||
import java.util.Date;
|
||||
import java.util.HashMap;
|
||||
import java.util.Iterator;
|
||||
import java.util.Map.Entry;
|
||||
import net.yacy.cora.document.encoding.UTF8;
|
||||
import net.yacy.cora.document.id.DigestURL;
|
||||
import net.yacy.cora.protocol.Domains;
|
||||
import net.yacy.cora.protocol.HeaderFramework;
|
||||
import net.yacy.cora.protocol.RequestHeader;
|
||||
import net.yacy.cora.protocol.ResponseHeader;
|
||||
|
|
@ -138,20 +136,9 @@ public final class HTTPDemon {
|
|||
}
|
||||
|
||||
// generating the desired request url
|
||||
String host = (String) conProp.get(HeaderFramework.CONNECTION_PROP_HOST);
|
||||
String path = (String) conProp.get(HeaderFramework.CONNECTION_PROP_PATH); if (path == null) path = "/";
|
||||
final String args = (String) conProp.get(HeaderFramework.CONNECTION_PROP_ARGS);
|
||||
|
||||
final String method = (String) conProp.get(HeaderFramework.CONNECTION_PROP_METHOD);
|
||||
|
||||
final int port = Domains.stripToPort(host);
|
||||
host = Domains.stripToHostName(host);
|
||||
|
||||
String urlString;
|
||||
try {
|
||||
urlString = (new DigestURL((method.equals(HeaderFramework.METHOD_CONNECT)?"https":"http"), host, port, (args == null) ? path : path + "?" + args)).toString();
|
||||
} catch (final MalformedURLException e) {
|
||||
urlString = "invalid URL";
|
||||
}
|
||||
DigestURL url = (DigestURL) conProp.get(HeaderFramework.CONNECTION_PROP_DIGESTURL);
|
||||
|
||||
// set rewrite values
|
||||
final serverObjects tp = new serverObjects();
|
||||
|
|
@ -159,8 +146,8 @@ public final class HTTPDemon {
|
|||
tp.put("peerName", (switchboard.peers == null) ? "" : switchboard.peers.myName());
|
||||
tp.put("errorMessageType", Integer.toString(errorcase));
|
||||
tp.put("httpStatus", Integer.toString(httpStatusCode) + " " + httpStatusText);
|
||||
tp.put("requestMethod", (String) conProp.get(HeaderFramework.CONNECTION_PROP_METHOD));
|
||||
tp.put("requestURL", urlString);
|
||||
tp.put("requestMethod", method);
|
||||
tp.put("requestURL", url.toString());
|
||||
|
||||
switch (errorcase) {
|
||||
case ERRORCASE_FILE:
|
||||
|
|
|
|||
Loading…
Reference in New Issue
Block a user