mirror of
https://github.com/yacy/yacy_search_server.git
synced 2024-09-19 00:01:41 +02:00
ce87025462
following comment "use of properties as header values is discouraged" in case where (proxy)HTTPClient overwrites values with supplied url. Use defined request.referer procedure in response class.
1210 lines
55 KiB
Java
1210 lines
55 KiB
Java
// HTTPDProxyHandler.java
|
|
// (C) 2004 by Michael Peter Christen; mc@yacy.net, Frankfurt a. M., Germany
|
|
// first published 2004 on http://yacy.net
|
|
//
|
|
// $LastChangedDate$
|
|
// $LastChangedRevision$
|
|
// $LastChangedBy$
|
|
//
|
|
// LICENSE
|
|
//
|
|
// This program is free software; you can redistribute it and/or modify
|
|
// it under the terms of the GNU General Public License as published by
|
|
// the Free Software Foundation; either version 2 of the License, or
|
|
// (at your option) any later version.
|
|
//
|
|
// This program is distributed in the hope that it will be useful,
|
|
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
// GNU General Public License for more details.
|
|
//
|
|
// You should have received a copy of the GNU General Public License
|
|
// along with this program; if not, write to the Free Software
|
|
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
|
|
|
// Contributions:
|
|
// [AS] Alexander Schier: Blacklist (404 response for AGIS hosts)
|
|
// [TL] Timo Leise: url-wildcards for blacklists
|
|
|
|
/*
|
|
Class documentation:
|
|
This class is a servlet to the httpd daemon. It is accessed each time
|
|
an URL in a GET, HEAD or POST command contains the whole host information
|
|
or a host is given in the header host field of an HTTP/1.0 / HTTP/1.1
|
|
command.
|
|
Transparency is maintained, whenever appropriate. We change header
|
|
attributes if necessary for the indexing mechanism; i.e. we do not
|
|
support gzip-ed encoding. We also do not support unrealistic
|
|
'expires' values that would force a cache to be flushed immediately
|
|
pragma non-cache attributes are supported
|
|
*/
|
|
|
|
|
|
package net.yacy.server.http;
|
|
|
|
import java.io.BufferedReader;
|
|
import java.io.ByteArrayOutputStream;
|
|
import java.io.File;
|
|
import java.io.IOException;
|
|
import java.io.InputStreamReader;
|
|
import java.io.OutputStream;
|
|
import java.io.PrintWriter;
|
|
import java.net.BindException;
|
|
import java.net.ConnectException;
|
|
import java.net.InetAddress;
|
|
import java.net.MalformedURLException;
|
|
import java.net.NoRouteToHostException;
|
|
import java.net.SocketException;
|
|
import java.net.SocketTimeoutException;
|
|
import java.net.UnknownHostException;
|
|
import java.util.Arrays;
|
|
import java.util.Date;
|
|
import java.util.HashMap;
|
|
import java.util.HashSet;
|
|
import java.util.Iterator;
|
|
import java.util.logging.FileHandler;
|
|
import java.util.logging.Level;
|
|
import java.util.logging.LogManager;
|
|
import java.util.logging.Logger;
|
|
import javax.servlet.http.HttpServletRequest;
|
|
import javax.servlet.http.HttpServletResponse;
|
|
|
|
import net.yacy.cora.document.id.DigestURL;
|
|
import net.yacy.cora.protocol.ClientIdentification;
|
|
import net.yacy.cora.protocol.Domains;
|
|
import net.yacy.cora.protocol.HeaderFramework;
|
|
import net.yacy.cora.protocol.RequestHeader;
|
|
import net.yacy.cora.protocol.ResponseHeader;
|
|
import net.yacy.cora.protocol.http.HTTPClient;
|
|
import net.yacy.cora.util.ConcurrentLog;
|
|
import net.yacy.crawler.data.Cache;
|
|
import net.yacy.crawler.retrieval.Request;
|
|
import net.yacy.crawler.retrieval.Response;
|
|
import net.yacy.document.TextParser;
|
|
import net.yacy.kelondro.io.ByteCountOutputStream;
|
|
import net.yacy.kelondro.util.FileUtils;
|
|
import net.yacy.peers.operation.yacyBuildProperties;
|
|
import net.yacy.repository.Blacklist.BlacklistType;
|
|
import net.yacy.search.Switchboard;
|
|
import net.yacy.server.serverObjects;
|
|
|
|
public final class HTTPDProxyHandler {
|
|
|
|
|
|
private static final String yacyProxyUserAgent = "yacyproxy (" + ClientIdentification.yacySystem +") http://yacy.net/bot.html";
|
|
|
|
// static variables
|
|
// can only be instantiated upon first instantiation of this class object
|
|
private static Switchboard sb = null;
|
|
private static final HashSet<String> yellowList;
|
|
private static int timeout = 60000;
|
|
private static Process redirectorProcess = null;
|
|
private static boolean redirectorEnabled = false;
|
|
private static PrintWriter redirectorWriter = null;
|
|
private static BufferedReader redirectorReader = null;
|
|
|
|
//private Properties connectionProperties = null;
|
|
// creating a logger
|
|
private static final ConcurrentLog log = new ConcurrentLog("PROXY");
|
|
|
|
/**
|
|
* Do logging configuration for special proxy access log file
|
|
*/
|
|
static {
|
|
|
|
// get a switchboard
|
|
sb = Switchboard.getSwitchboard();
|
|
if (sb != null) {
|
|
|
|
// set timeout
|
|
timeout = sb.getConfigInt("proxy.clientTimeout", 60000);
|
|
|
|
// do logger initialization
|
|
try {
|
|
log.info("Configuring proxy access logging ...");
|
|
|
|
// getting the logging manager
|
|
final LogManager manager = LogManager.getLogManager();
|
|
final String className = HTTPDProxyHandler.class.getName();
|
|
|
|
// determining if proxy access logging is enabled
|
|
final String enabled = manager.getProperty(className + ".logging.enabled");
|
|
if ("true".equalsIgnoreCase(enabled)) {
|
|
|
|
// reading out some needed configuration properties
|
|
int limit = 1024*1024, count = 20;
|
|
String pattern = manager.getProperty(className + ".logging.FileHandler.pattern");
|
|
if (pattern == null) pattern = "DATA/LOG/proxyAccess%u%g.log";
|
|
// make pattern absolute
|
|
if (!new File(pattern).isAbsolute()) pattern = new File(sb.getDataPath(), pattern).getAbsolutePath();
|
|
|
|
final String limitStr = manager.getProperty(className + ".logging.FileHandler.limit");
|
|
if (limitStr != null) try { limit = Integer.parseInt(limitStr); } catch (final NumberFormatException e) {}
|
|
|
|
final String countStr = manager.getProperty(className + ".logging.FileHandler.count");
|
|
if (countStr != null) try { count = Integer.parseInt(countStr); } catch (final NumberFormatException e) {}
|
|
|
|
// creating the proxy access logger
|
|
final Logger proxyLogger = Logger.getLogger("PROXY.access");
|
|
proxyLogger.setUseParentHandlers(false);
|
|
proxyLogger.setLevel(Level.FINEST);
|
|
|
|
final FileHandler txtLog = new FileHandler(pattern, limit, count, true);
|
|
txtLog.setFormatter(new ProxyLogFormatter());
|
|
txtLog.setLevel(Level.FINEST);
|
|
proxyLogger.addHandler(txtLog);
|
|
|
|
log.info("Proxy access logging configuration done." +
|
|
"\n\tFilename: " + pattern +
|
|
"\n\tLimit: " + limitStr +
|
|
"\n\tCount: " + countStr);
|
|
} else {
|
|
log.info("Proxy access logging is deactivated.");
|
|
}
|
|
} catch (final Exception e) {
|
|
log.severe("Unable to configure proxy access logging.",e);
|
|
}
|
|
|
|
// load the yellow-list
|
|
final String f = sb.getConfig("proxyYellowList", null);
|
|
if (f != null) {
|
|
yellowList = FileUtils.loadList(new File(f));
|
|
log.config("loaded yellow-list from file " + f + ", " + yellowList.size() + " entries");
|
|
} else {
|
|
yellowList = null;
|
|
}
|
|
|
|
final String redirectorPath = sb.getConfig("externalRedirector", "");
|
|
if (redirectorPath.length() > 0 && !redirectorEnabled) {
|
|
try {
|
|
redirectorProcess=Runtime.getRuntime().exec(redirectorPath);
|
|
redirectorWriter = new PrintWriter(redirectorProcess.getOutputStream());
|
|
redirectorReader = new BufferedReader(new InputStreamReader(redirectorProcess.getInputStream()));
|
|
redirectorEnabled=true;
|
|
} catch (final IOException e) {
|
|
System.out.println("redirector not Found");
|
|
}
|
|
}
|
|
} else {
|
|
yellowList = null;
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Special logger instance for proxy access logging much similar
|
|
* to the squid access.log file
|
|
*/
|
|
public static final ConcurrentLog proxyLog = new ConcurrentLog("PROXY.access");
|
|
|
|
/**
|
|
* Reusable {@link StringBuilder} for logging
|
|
*/
|
|
private static final StringBuilder logMessage = new StringBuilder();
|
|
|
|
/**
|
|
* Reusable {@link StringBuilder} to generate the useragent string
|
|
*/
|
|
private static final StringBuilder userAgentStr = new StringBuilder();
|
|
|
|
private static void handleOutgoingCookies(final RequestHeader requestHeader, final String targethost, final String clienthost) {
|
|
/*
|
|
The syntax for the header is:
|
|
|
|
cookie = "Cookie:" cookie-version
|
|
1*((";" | ",") cookie-value)
|
|
cookie-value = NAME "=" VALUE [";" path] [";" domain]
|
|
cookie-version = "$Version" "=" value
|
|
NAME = attr
|
|
VALUE = value
|
|
path = "$Path" "=" value
|
|
domain = "$Domain" "=" value
|
|
*/
|
|
if (sb.getConfigBool("proxy.monitorCookies", false)) {
|
|
if (requestHeader.containsKey(RequestHeader.COOKIE)) {
|
|
final Object[] entry = new Object[]{new Date(), clienthost, requestHeader.getCookies()};
|
|
synchronized(sb.outgoingCookies) {
|
|
sb.outgoingCookies.put(targethost, entry);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
private static void handleIncomingCookies(final ResponseHeader respondHeader, final String serverhost, final String targetclient) {
|
|
/*
|
|
The syntax for the Set-Cookie response header is
|
|
|
|
set-cookie = "Set-Cookie:" cookies
|
|
cookies = 1#cookie
|
|
cookie = NAME "=" VALUE *(";" cookie-av)
|
|
NAME = attr
|
|
VALUE = value
|
|
cookie-av = "Comment" "=" value
|
|
| "Domain" "=" value
|
|
| "Max-Age" "=" value
|
|
| "Path" "=" value
|
|
| "Secure"
|
|
| "Version" "=" 1*DIGIT
|
|
*/
|
|
if (sb.getConfigBool("proxy.monitorCookies", false)) {
|
|
if (respondHeader.containsKey(HeaderFramework.SET_COOKIE)) {
|
|
final Object[] entry = new Object[]{new Date(), targetclient, respondHeader.getMultiple(HeaderFramework.SET_COOKIE)};
|
|
synchronized(sb.incomingCookies) {
|
|
sb.incomingCookies.put(serverhost, entry);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Get a page from the remote server as proxy request for the client.
|
|
* The target url is expected in the conProp CONNECTION_PROP_DIGESTURL
|
|
*
|
|
* @param conProp a collection of properties about the connection, like URL
|
|
* @param requestHeader The header lines of the connection from the request
|
|
* @param respond the OutputStream to the client
|
|
* @see de.anomic.http.httpdHandler#doGet(java.util.Properties, net.yacy.cora.protocol.HeaderFramework, java.io.OutputStream)
|
|
*/
|
|
public static void doGet(final HashMap<String, Object> conProp, final RequestHeader requestHeader, final OutputStream respond, final ClientIdentification.Agent agent) {
|
|
ByteCountOutputStream countedRespond = null;
|
|
try {
|
|
final int reqID = requestHeader.hashCode();
|
|
// remembering the starting time of the request
|
|
final Date requestDate = new Date(); // remember the time...
|
|
conProp.put(HeaderFramework.CONNECTION_PROP_REQUEST_START, Long.valueOf(requestDate.getTime()));
|
|
sb.proxyLastAccess = System.currentTimeMillis();
|
|
|
|
// using an ByteCount OutputStream to count the send bytes (needed for the logfile)
|
|
countedRespond = new ByteCountOutputStream(respond, "PROXY");
|
|
|
|
final String ip = (String) conProp.get(HeaderFramework.CONNECTION_PROP_CLIENTIP); // the ip from the connecting peer
|
|
|
|
DigestURL url = (DigestURL) conProp.get(HeaderFramework.CONNECTION_PROP_DIGESTURL);
|
|
if (log.isFine()) log.fine(reqID +" GET "+ url.toString());
|
|
if (log.isFinest()) log.finest(reqID +" header: "+ requestHeader);
|
|
|
|
//redirector
|
|
if (redirectorEnabled) {
|
|
synchronized (redirectorProcess) {
|
|
redirectorWriter.println(url.toNormalform(true));
|
|
redirectorWriter.flush();
|
|
}
|
|
final String newUrl = redirectorReader.readLine();
|
|
if (!newUrl.equals("")) {
|
|
try {
|
|
url = new DigestURL(newUrl);
|
|
} catch (final MalformedURLException e) {
|
|
}//just keep the old one
|
|
}
|
|
if (log.isFinest()) {
|
|
log.finest(reqID + " using redirector to " + url);
|
|
}
|
|
conProp.put(HeaderFramework.CONNECTION_PROP_DIGESTURL, url);
|
|
}
|
|
|
|
|
|
// check the blacklist
|
|
// blacklist idea inspired by [AS]:
|
|
// respond a 404 for all AGIS ("all you get is shit") servers
|
|
if (Switchboard.urlBlacklist.isListed(BlacklistType.PROXY, url)) {
|
|
log.info("AGIS blocking of host '" + url.getHost() + "'");
|
|
HTTPDemon.sendRespondError(conProp,countedRespond,4,403,null,
|
|
"URL '" + url.getHost() + "' blocked by yacy proxy (blacklisted)",null);
|
|
return;
|
|
}
|
|
|
|
// handle outgoing cookies
|
|
handleOutgoingCookies(requestHeader, url.getHost(), ip);
|
|
prepareRequestHeader(conProp, requestHeader, url.getHost().toLowerCase());
|
|
final ResponseHeader cachedResponseHeader = Cache.getResponseHeader(url.hash());
|
|
|
|
// why are files unzipped upon arrival? why not zip all files in cache?
|
|
// This follows from the following premises
|
|
// (a) no file shall be unzip-ed more than once to prevent unnecessary computing time
|
|
// (b) old cache entries shall be comparable with refill-entries to detect/distinguish case 3+4
|
|
// (c) the indexing mechanism needs files unzip-ed, a schedule could do that later
|
|
// case b and c contradicts, if we use a scheduler, because files in a stale cache would be unzipped
|
|
// and the newly arrival would be zipped and would have to be unzipped upon load. But then the
|
|
// scheduler is superfluous. Therefore the only reminding case is
|
|
// (d) cached files shall be either all zipped or unzipped
|
|
// case d contradicts with a, because files need to be unzipped for indexing. Therefore
|
|
// the only remaining case is to unzip files right upon load. Thats what we do here.
|
|
|
|
// finally use existing cache if appropriate
|
|
// here we must decide weather or not to save the data
|
|
// to a cache
|
|
// we distinguish four CACHE STATE cases:
|
|
// 1. cache fill
|
|
// 2. cache fresh - no refill
|
|
// 3. cache stale - refill - necessary
|
|
// 4. cache stale - refill - superfluous
|
|
// in two of these cases we trigger a scheduler to handle newly arrived files:
|
|
// case 1 and case 3
|
|
if (cachedResponseHeader == null) {
|
|
if (log.isFinest()) log.finest(reqID + " page not in cache: fulfill request from web");
|
|
fulfillRequestFromWeb(conProp, url, requestHeader, cachedResponseHeader, countedRespond, agent);
|
|
} else {
|
|
final Request request = new Request(
|
|
null,
|
|
url,
|
|
requestHeader.referer() == null ? null : requestHeader.referer().hash(),
|
|
"",
|
|
cachedResponseHeader.lastModified(),
|
|
sb.crawler.defaultProxyProfile.handle(),
|
|
0,
|
|
sb.crawler.defaultProxyProfile.timezoneOffset());
|
|
final Response response = new Response(
|
|
request,
|
|
requestHeader,
|
|
cachedResponseHeader,
|
|
sb.crawler.defaultProxyProfile,
|
|
true,
|
|
null
|
|
);
|
|
final byte[] cacheContent = Cache.getContent(url.hash());
|
|
if (cacheContent != null && response.isFreshForProxy()) {
|
|
if (log.isFinest()) log.finest(reqID + " fulfill request from cache");
|
|
fulfillRequestFromCache(conProp, url, requestHeader, cachedResponseHeader, cacheContent, countedRespond);
|
|
} else {
|
|
if (log.isFinest()) log.finest(reqID + " fulfill request from web");
|
|
fulfillRequestFromWeb(conProp, url, requestHeader, cachedResponseHeader, countedRespond, agent);
|
|
}
|
|
}
|
|
|
|
|
|
} catch (final Exception e) {
|
|
try {
|
|
final String exTxt = e.getMessage();
|
|
if ((exTxt!=null)&&(exTxt.startsWith("Socket closed"))) {
|
|
forceConnectionClose(conProp);
|
|
} else if (!conProp.containsKey(HeaderFramework.CONNECTION_PROP_PROXY_RESPOND_HEADER)) {
|
|
final String errorMsg = "Unexpected Error. " + e.getClass().getName() + ": " + e.getMessage();
|
|
HTTPDemon.sendRespondError(conProp,countedRespond,4,501,null,errorMsg,e);
|
|
log.severe(errorMsg);
|
|
} else {
|
|
forceConnectionClose(conProp);
|
|
}
|
|
} catch (final Exception ee) {
|
|
forceConnectionClose(conProp);
|
|
}
|
|
} finally {
|
|
try { if(countedRespond != null) countedRespond.flush(); else if(respond != null) respond.flush(); } catch (final Exception e) {}
|
|
if (countedRespond != null) countedRespond.finish();
|
|
|
|
conProp.put(HeaderFramework.CONNECTION_PROP_REQUEST_END, Long.valueOf(System.currentTimeMillis()));
|
|
conProp.put(HeaderFramework.CONNECTION_PROP_PROXY_RESPOND_SIZE,(countedRespond != null) ? Long.toString(countedRespond.getCount()) : -1L);
|
|
logProxyAccess(conProp);
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Get requested proxied page from the web
|
|
*
|
|
* @param conProp
|
|
* @param url requested target url
|
|
* @param requestHeader
|
|
* @param cachedResponseHeader
|
|
* @param respond
|
|
* @param agent
|
|
*/
|
|
private static void fulfillRequestFromWeb(final HashMap<String, Object> conProp, final DigestURL url, final RequestHeader requestHeader, final ResponseHeader cachedResponseHeader, final OutputStream respond, final ClientIdentification.Agent agent) {
|
|
try {
|
|
final int reqID = requestHeader.hashCode();
|
|
|
|
final String host = url.getHost(); // conProp.get(HeaderFramework.CONNECTION_PROP_HOST);
|
|
final String path = url.getPath(); // conProp.get(HeaderFramework.CONNECTION_PROP_PATH); // always starts with leading '/'
|
|
final String args = url.getSearchpart(); // conProp.get(HeaderFramework.CONNECTION_PROP_ARGS); // may be null if no args were given
|
|
final String ip = (String) conProp.get(HeaderFramework.CONNECTION_PROP_CLIENTIP); // the ip from the connecting peer
|
|
final String httpVer = (String) conProp.get(HeaderFramework.CONNECTION_PROP_HTTP_VER); // the http version for the request
|
|
final String clienthttpVer; // the http version of the client connection
|
|
final HttpServletRequest clientservletrequest = (HttpServletRequest) conProp.get(HeaderFramework.CONNECTION_PROP_CLIENT_HTTPSERVLETREQUEST);
|
|
if (clientservletrequest != null) {
|
|
clienthttpVer = clientservletrequest.getProtocol();
|
|
} else {
|
|
clienthttpVer = null;
|
|
}
|
|
final int port = url.getPort();
|
|
|
|
// resolve yacy and yacyh domains
|
|
String yAddress = resolveYacyDomains(host);
|
|
|
|
// re-calc the url path
|
|
final String remotePath = (args == null) ? path : (path + "?" + args); // with leading '/'
|
|
|
|
// remove yacy-subdomain-path, when accessing /env
|
|
if ((yAddress != null)
|
|
&& (remotePath.startsWith("/env"))
|
|
&& (yAddress.indexOf('/') != -1)
|
|
) yAddress = yAddress.substring(0, yAddress.indexOf('/'));
|
|
|
|
modifyProxyHeaders(requestHeader, httpVer);
|
|
|
|
final String connectHost = hostPart(host, port, yAddress);
|
|
final String getUrl = url.getProtocol() +"://"+ connectHost + remotePath;
|
|
|
|
requestHeader.remove(HeaderFramework.HOST);
|
|
|
|
final HTTPClient client = setupHttpClient(requestHeader, agent);
|
|
|
|
// send request
|
|
try {
|
|
client.GET(getUrl, false);
|
|
if (log.isFinest()) log.finest(reqID +" response status: "+ client.getHttpResponse().getStatusLine());
|
|
|
|
int statusCode = client.getHttpResponse().getStatusLine().getStatusCode();
|
|
final ResponseHeader responseHeader = new ResponseHeader(statusCode, client.getHttpResponse().getAllHeaders());
|
|
// determine if it's an internal error of the httpc
|
|
if (responseHeader.isEmpty()) {
|
|
throw new Exception(client.getHttpResponse().getStatusLine().toString());
|
|
}
|
|
|
|
ChunkedOutputStream chunkedOut = setTransferEncoding(conProp, responseHeader, statusCode, respond);
|
|
|
|
// the cache does either not exist or is (supposed to be) stale
|
|
long sizeBeforeDelete = -1;
|
|
if (cachedResponseHeader != null) {
|
|
// delete the cache
|
|
final ResponseHeader rh = Cache.getResponseHeader(url.hash());
|
|
if (rh != null && (sizeBeforeDelete = rh.getContentLength()) == 0) {
|
|
final byte[] b = Cache.getContent(url.hash());
|
|
if (b != null) sizeBeforeDelete = b.length;
|
|
}
|
|
Cache.delete(url.hash());
|
|
conProp.put(HeaderFramework.CONNECTION_PROP_PROXY_RESPOND_CODE, "TCP_REFRESH_MISS");
|
|
}
|
|
|
|
// reserver cache entry
|
|
final Request request = new Request(
|
|
null,
|
|
url,
|
|
requestHeader.referer() == null ? null : requestHeader.referer().hash(),
|
|
"",
|
|
responseHeader.lastModified(),
|
|
sb.crawler.defaultProxyProfile.handle(),
|
|
0,
|
|
sb.crawler.defaultProxyProfile.timezoneOffset());
|
|
|
|
// handle incoming cookies
|
|
handleIncomingCookies(responseHeader, host, ip);
|
|
|
|
// prepareResponseHeader(responseHeader, res.getHttpVer());
|
|
prepareResponseHeader(responseHeader, client.getHttpResponse().getProtocolVersion().toString());
|
|
|
|
// sending the respond header back to the client
|
|
if (chunkedOut != null) {
|
|
responseHeader.put(HeaderFramework.TRANSFER_ENCODING, "chunked");
|
|
}
|
|
|
|
if (log.isFinest()) log.finest(reqID +" sending response header: "+ responseHeader);
|
|
HTTPDemon.sendRespondHeader(
|
|
conProp,
|
|
respond,
|
|
clienthttpVer,
|
|
statusCode,
|
|
client.getHttpResponse().getStatusLine().toString(), // status text
|
|
responseHeader);
|
|
|
|
if (hasBody(client.getHttpResponse().getStatusLine().getStatusCode())) {
|
|
|
|
OutputStream outStream = chunkedOut != null ? chunkedOut : respond;
|
|
final Response response = new Response(
|
|
request,
|
|
requestHeader,
|
|
responseHeader,
|
|
sb.crawler.defaultProxyProfile,
|
|
true,
|
|
null
|
|
);
|
|
final String storeError = response.shallStoreCacheForProxy();
|
|
final boolean storeHTCache = response.profile().storeHTCache();
|
|
final String supportError = TextParser.supports(response.url(), response.getMimeType());
|
|
|
|
if (
|
|
/*
|
|
* Now we store the response into the htcache directory if
|
|
* a) the response is cacheable AND
|
|
*/
|
|
(storeError == null) &&
|
|
/*
|
|
* b) the user has configured to use the htcache OR
|
|
* c) the content should be indexed
|
|
*/
|
|
((storeHTCache) || (supportError != null))
|
|
) {
|
|
// we don't write actually into a file, only to RAM, and schedule writing the file.
|
|
// int l = res.getResponseHeader().size();
|
|
final int l = responseHeader.size();
|
|
final ByteArrayOutputStream byteStream = new ByteArrayOutputStream((l < 32) ? 32 : l);
|
|
|
|
final OutputStream toClientAndMemory = new MultiOutputStream(new OutputStream[] {outStream, byteStream});
|
|
// FileUtils.copy(res.getDataAsStream(), toClientAndMemory);
|
|
client.writeTo(toClientAndMemory);
|
|
// cached bytes
|
|
byte[] cacheArray;
|
|
if (byteStream.size() > 0) {
|
|
cacheArray = byteStream.toByteArray();
|
|
} else {
|
|
cacheArray = null;
|
|
}
|
|
if (log.isFine()) log.fine(reqID +" writeContent of " + url + " produced cacheArray = " + ((cacheArray == null) ? "null" : ("size=" + cacheArray.length)));
|
|
|
|
if (sizeBeforeDelete == -1) {
|
|
// totally fresh file
|
|
response.setContent(cacheArray);
|
|
try {
|
|
Cache.store(response.url(), response.getResponseHeader(), cacheArray);
|
|
sb.toIndexer(response);
|
|
} catch (final IOException e) {
|
|
log.warn("cannot write " + response.url() + " to Cache (1): " + e.getMessage(), e);
|
|
}
|
|
conProp.put(HeaderFramework.CONNECTION_PROP_PROXY_RESPOND_CODE, "TCP_MISS");
|
|
} else if (cacheArray != null && sizeBeforeDelete == cacheArray.length) {
|
|
// before we came here we deleted a cache entry
|
|
cacheArray = null;
|
|
//cacheManager.push(cacheEntry); // unnecessary update
|
|
conProp.put(HeaderFramework.CONNECTION_PROP_PROXY_RESPOND_CODE, "TCP_REF_FAIL_HIT");
|
|
} else {
|
|
// before we came here we deleted a cache entry
|
|
response.setContent(cacheArray);
|
|
try {
|
|
Cache.store(response.url(), response.getResponseHeader(), cacheArray);
|
|
sb.toIndexer(response);
|
|
} catch (final IOException e) {
|
|
log.warn("cannot write " + response.url() + " to Cache (2): " + e.getMessage(), e);
|
|
}
|
|
conProp.put(HeaderFramework.CONNECTION_PROP_PROXY_RESPOND_CODE, "TCP_REFRESH_MISS");
|
|
}
|
|
} else {
|
|
// no caching
|
|
if (log.isFine()) log.fine(reqID +" "+ url.toNormalform(false) + " not cached." +
|
|
" StoreError=" + ((storeError==null)?"None":storeError) +
|
|
" StoreHTCache=" + storeHTCache +
|
|
" SupportError=" + supportError);
|
|
|
|
if (statusCode == HttpServletResponse.SC_OK) { // continue to serve header to client e.g. HttpStatus = 302 (while skiping content)
|
|
client.writeTo(outStream); // may throw exception on httpStatus=302 while gzip encoded inputstream
|
|
}
|
|
|
|
conProp.put(HeaderFramework.CONNECTION_PROP_PROXY_RESPOND_CODE,"TCP_MISS");
|
|
}
|
|
|
|
outStream.close();
|
|
|
|
if (chunkedOut != null) {
|
|
chunkedOut.finish();
|
|
chunkedOut.flush();
|
|
}
|
|
} // end hasBody
|
|
} catch(final SocketException se) {
|
|
// if opened ...
|
|
// if(res != null) {
|
|
// // client cut proxy connection, abort download
|
|
// res.abort();
|
|
// }
|
|
client.finish();
|
|
handleProxyException(se,conProp,respond,url);
|
|
} finally {
|
|
// if opened ...
|
|
// if(res != null) {
|
|
// // ... close connection
|
|
// res.closeStream();
|
|
// }
|
|
client.finish();
|
|
}
|
|
} catch (final Exception e) {
|
|
handleProxyException(e,conProp,respond,url);
|
|
}
|
|
}
|
|
|
|
/**
|
|
* determines if the response should have a body
|
|
*
|
|
* @param statusCode
|
|
* @param responseHeader
|
|
* @return
|
|
*/
|
|
private static boolean hasBody(final int statusCode) {
|
|
// "All 1xx (informational), 204 (no content), and 304 (not modified) responses MUST NOT
|
|
// include a message-body."
|
|
// [RFC 2616 HTTP/1.1, Sect. 4.3] and like [RFC 1945 HTTP/1.0, Sect. 7.2]
|
|
if((statusCode >= 100 && statusCode < 200) || statusCode == 204 || statusCode == 304) {
|
|
return false;
|
|
}
|
|
return true;
|
|
}
|
|
|
|
private static void fulfillRequestFromCache(
|
|
final HashMap<String, Object> conProp,
|
|
final DigestURL url,
|
|
final RequestHeader requestHeader,
|
|
final ResponseHeader cachedResponseHeader,
|
|
final byte[] cacheEntry,
|
|
OutputStream respond
|
|
) throws IOException {
|
|
|
|
final String httpVer = (String) conProp.get(HeaderFramework.CONNECTION_PROP_HTTP_VER);
|
|
final String clienthttpVer; // the http version of the client connection
|
|
final HttpServletRequest clientservletrequest = (HttpServletRequest) conProp.get(HeaderFramework.CONNECTION_PROP_CLIENT_HTTPSERVLETREQUEST);
|
|
if (clientservletrequest != null) {
|
|
clienthttpVer = clientservletrequest.getProtocol();
|
|
} else {
|
|
clienthttpVer = null;
|
|
}
|
|
// we respond on the request by using the cache, the cache is fresh
|
|
try {
|
|
prepareResponseHeader(cachedResponseHeader, httpVer);
|
|
|
|
// replace date field in old header by actual date, this is according to RFC
|
|
cachedResponseHeader.put(HeaderFramework.DATE, HeaderFramework.formatRFC1123(new Date()));
|
|
|
|
// check if we can send a 304 instead the complete content
|
|
if (requestHeader.containsKey(RequestHeader.IF_MODIFIED_SINCE)) {
|
|
// conditional request: freshness of cache for that condition was already
|
|
// checked within shallUseCache(). Now send only a 304 response
|
|
log.info("CACHE HIT/304 " + url.toNormalform(false));
|
|
conProp.put(HeaderFramework.CONNECTION_PROP_PROXY_RESPOND_CODE,"TCP_REFRESH_HIT");
|
|
|
|
// setting the content length header to 0
|
|
cachedResponseHeader.put(HeaderFramework.CONTENT_LENGTH, Integer.toString(0));
|
|
|
|
// send cached header with replaced date and added length
|
|
HTTPDemon.sendRespondHeader(conProp,respond,clienthttpVer,304,cachedResponseHeader);
|
|
//respondHeader(respond, "304 OK", cachedResponseHeader); // respond with 'not modified'
|
|
} else {
|
|
// unconditional request: send content of cache
|
|
log.info("CACHE HIT/203 " + url.toNormalform(false));
|
|
conProp.put(HeaderFramework.CONNECTION_PROP_PROXY_RESPOND_CODE,"TCP_HIT");
|
|
|
|
// setting the content header to the proper length
|
|
cachedResponseHeader.put(HeaderFramework.CONTENT_LENGTH, Long.toString(cacheEntry.length));
|
|
|
|
// send cached header with replaced date and added length
|
|
HTTPDemon.sendRespondHeader(conProp,respond,clienthttpVer,203,cachedResponseHeader);
|
|
//respondHeader(respond, "203 OK", cachedResponseHeader); // respond with 'non-authoritative'
|
|
|
|
// send also the complete body now from the cache
|
|
// simply read the file and transfer to out socket
|
|
FileUtils.copy(cacheEntry, respond);
|
|
}
|
|
// that's it!
|
|
} catch (final Exception e) {
|
|
// this happens if the client stops loading the file
|
|
// we do nothing here
|
|
if (conProp.containsKey(HeaderFramework.CONNECTION_PROP_PROXY_RESPOND_HEADER)) {
|
|
log.warn("Error while trying to send cached message body.");
|
|
conProp.put(HeaderFramework.CONNECTION_PROP_PERSISTENT,"close");
|
|
} else {
|
|
HTTPDemon.sendRespondError(conProp,respond,4,503,"socket error: " + e.getMessage(),"socket error: " + e.getMessage(), e);
|
|
}
|
|
} finally {
|
|
try { respond.flush(); respond.close(); } catch (final Exception e) {}
|
|
}
|
|
}
|
|
|
|
/**
|
|
* resolve yacy and yacyh domains
|
|
*
|
|
* @param host
|
|
* @return
|
|
*/
|
|
private static String resolveYacyDomains(final String host) {
|
|
return (sb.peers == null) ? null : sb.peers.resolve(host);
|
|
}
|
|
|
|
/**
|
|
* @param host
|
|
* @param port
|
|
* @param yAddress
|
|
* @return
|
|
*/
|
|
private static String hostPart(final String host, final int port, final String yAddress) {
|
|
final String connectHost = (yAddress == null) ? host +":"+ port : yAddress;
|
|
return connectHost;
|
|
}
|
|
|
|
/**
|
|
* @param conProp
|
|
* @param requestHeader
|
|
* @param hostlow
|
|
*/
|
|
private static void prepareRequestHeader(final HashMap<String, Object> conProp, final RequestHeader requestHeader, final String hostlow) {
|
|
// set another userAgent, if not yellow-listed
|
|
if ((yellowList != null) && (!(yellowList.contains(domain(hostlow))))) {
|
|
// change the User-Agent
|
|
requestHeader.put(HeaderFramework.USER_AGENT, generateUserAgent(requestHeader));
|
|
}
|
|
|
|
// only gzip-encoding is supported, remove other encodings (e. g. deflate)
|
|
if ((requestHeader.get(HeaderFramework.ACCEPT_ENCODING,"")).indexOf(HeaderFramework.CONTENT_ENCODING_GZIP,0) != -1) {
|
|
requestHeader.put(HeaderFramework.ACCEPT_ENCODING, HeaderFramework.CONTENT_ENCODING_GZIP);
|
|
} else {
|
|
requestHeader.put(HeaderFramework.ACCEPT_ENCODING, "");
|
|
}
|
|
|
|
addXForwardedForHeader(conProp, requestHeader);
|
|
}
|
|
|
|
private static String domain(final String host) {
|
|
String domain = host;
|
|
int pos = domain.lastIndexOf('.');
|
|
if (pos >= 0) {
|
|
// truncate from last part
|
|
domain = domain.substring(0, pos);
|
|
pos = domain.lastIndexOf('.');
|
|
if (pos >= 0) {
|
|
// truncate from first part
|
|
domain = domain.substring(pos + 1);
|
|
}
|
|
}
|
|
return domain;
|
|
}
|
|
|
|
/**
|
|
* creates a new HttpClient and sets parameters according to proxy needs
|
|
*
|
|
* @param requestHeader
|
|
* @return
|
|
*/
|
|
private static HTTPClient setupHttpClient(final RequestHeader requestHeader, final ClientIdentification.Agent agent) {
|
|
// setup HTTP-client
|
|
final HTTPClient client = new HTTPClient(agent, timeout);
|
|
client.setHeader(requestHeader.entrySet());
|
|
client.setRedirecting(false);
|
|
return client;
|
|
}
|
|
|
|
/**
|
|
* determines in which form the response should be send and sets header accordingly
|
|
* if the content length is not set we need to use chunked content encoding
|
|
* Implemented:
|
|
* if !content-length
|
|
* switch httpVer
|
|
* case 0.9:
|
|
* case 1.0:
|
|
* close connection after transfer
|
|
* break;
|
|
* default:
|
|
* new ChunkedStream around respond
|
|
* end if
|
|
*
|
|
* @param conProp
|
|
* @param responseHeader
|
|
* @param statusCode
|
|
* @param respond
|
|
* @return
|
|
*/
|
|
private static ChunkedOutputStream setTransferEncoding(
|
|
final HashMap<String, Object> conProp, final ResponseHeader responseHeader,
|
|
final int statusCode, final OutputStream respond) {
|
|
final String httpVer = (String) conProp.get(HeaderFramework.CONNECTION_PROP_HTTP_VER);
|
|
ChunkedOutputStream chunkedOut = null;
|
|
// gzipped response is ungzipped an therefor the length is unknown
|
|
if (responseHeader.gzip() || responseHeader.getContentLength() < 0) {
|
|
// according to http://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html
|
|
// a 204,304 message must not contain a message body.
|
|
// Therefore we need to set the content-length to 0.
|
|
if (statusCode == 204 || statusCode == 304) {
|
|
responseHeader.put(HeaderFramework.CONTENT_LENGTH, "0");
|
|
} else {
|
|
if (httpVer.equals(HeaderFramework.HTTP_VERSION_0_9) || httpVer.equals(HeaderFramework.HTTP_VERSION_1_0)) {
|
|
forceConnectionClose(conProp);
|
|
} else {
|
|
chunkedOut = new ChunkedOutputStream(respond);
|
|
}
|
|
responseHeader.remove(HeaderFramework.CONTENT_LENGTH);
|
|
}
|
|
}
|
|
return chunkedOut;
|
|
}
|
|
|
|
/**
|
|
* @param res
|
|
* @param responseHeader
|
|
*/
|
|
private static void prepareResponseHeader(final ResponseHeader responseHeader, final String httpVer) {
|
|
modifyProxyHeaders(responseHeader, httpVer);
|
|
correctContentEncoding(responseHeader);
|
|
}
|
|
|
|
/**
|
|
* @param responseHeader
|
|
*/
|
|
private static void correctContentEncoding(final ResponseHeader responseHeader) {
|
|
// TODO gzip again? set "correct" encoding?
|
|
if(responseHeader.gzip()) {
|
|
responseHeader.remove(HeaderFramework.CONTENT_ENCODING);
|
|
responseHeader.remove(HeaderFramework.CONTENT_LENGTH); // remove gziped length
|
|
}
|
|
}
|
|
|
|
/**
|
|
* adds the client-IP of conProp to the requestHeader
|
|
*
|
|
* @param conProp
|
|
* @param requestHeader
|
|
*/
|
|
private static void addXForwardedForHeader(final HashMap<String, Object> conProp, final RequestHeader requestHeader) {
|
|
// setting the X-Forwarded-For Header
|
|
if (sb.getConfigBool("proxy.sendXForwardedForHeader", true)) {
|
|
requestHeader.put(HeaderFramework.X_FORWARDED_FOR, (String) conProp.get(HeaderFramework.CONNECTION_PROP_CLIENTIP));
|
|
}
|
|
}
|
|
|
|
/**
|
|
* removing hop by hop headers and adding additional headers
|
|
*
|
|
* @param requestHeader
|
|
* @param httpVer
|
|
*/
|
|
public static void modifyProxyHeaders(final HeaderFramework requestHeader, final String httpVer) {
|
|
removeHopByHopHeaders(requestHeader);
|
|
setViaHeader(requestHeader, httpVer);
|
|
}
|
|
|
|
private static void removeHopByHopHeaders(final HeaderFramework headers) {
|
|
/*
|
|
- Trailers
|
|
*/
|
|
|
|
headers.remove(RequestHeader.CONNECTION);
|
|
headers.remove(RequestHeader.KEEP_ALIVE);
|
|
headers.remove(RequestHeader.UPGRADE);
|
|
headers.remove(RequestHeader.TE);
|
|
headers.remove(RequestHeader.PROXY_CONNECTION);
|
|
headers.remove(RequestHeader.PROXY_AUTHENTICATE);
|
|
headers.remove(RequestHeader.PROXY_AUTHORIZATION);
|
|
|
|
// special headers inserted by squid
|
|
headers.remove(RequestHeader.X_CACHE);
|
|
headers.remove(RequestHeader.X_CACHE_LOOKUP);
|
|
|
|
// remove transfer encoding header
|
|
headers.remove(HeaderFramework.TRANSFER_ENCODING);
|
|
}
|
|
|
|
private static void setViaHeader(final HeaderFramework header, final String httpVer) {
|
|
if (!sb.getConfigBool("proxy.sendViaHeader", true)) return;
|
|
final String myAddress = (sb.peers == null) ? null : sb.peers.myAlternativeAddress();
|
|
if (myAddress != null) {
|
|
|
|
// getting header set by other proxies in the chain
|
|
final StringBuilder viaValue = new StringBuilder(80);
|
|
if (header.containsKey(HeaderFramework.VIA)) viaValue.append(header.get(HeaderFramework.VIA));
|
|
if (viaValue.length() > 0) viaValue.append(", ");
|
|
|
|
// appending info about this peer
|
|
viaValue
|
|
.append(httpVer).append(" ")
|
|
.append(myAddress).append(" ")
|
|
.append("(YaCy ").append(yacyBuildProperties.getVersion()).append(")");
|
|
|
|
// storing header back
|
|
header.put(HeaderFramework.VIA, viaValue.toString());
|
|
}
|
|
}
|
|
|
|
private static void handleProxyException(final Exception e, final HashMap<String, Object> conProp, final OutputStream respond, final DigestURL url) {
|
|
// this may happen if
|
|
// - the targeted host does not exist
|
|
// - anything with the remote server was wrong.
|
|
// - the client unexpectedly closed the connection ...
|
|
try {
|
|
|
|
|
|
// doing some errorhandling ...
|
|
int httpStatusCode = 404;
|
|
String httpStatusText = null;
|
|
String errorMessage = null;
|
|
Exception errorExc = null;
|
|
boolean unknownError = false;
|
|
|
|
// for customized error messages
|
|
boolean detailedErrorMsg = false;
|
|
String detailedErrorMsgFile = null;
|
|
serverObjects detailedErrorMsgMap = null;
|
|
|
|
if (e instanceof ConnectException) {
|
|
httpStatusCode = 403; httpStatusText = "Connection refused";
|
|
errorMessage = "Connection refused by destination host";
|
|
} else if (e instanceof BindException) {
|
|
errorMessage = "Unable to establish a connection to the destination host";
|
|
} else if (e instanceof NoRouteToHostException) {
|
|
errorMessage = "No route to destination host";
|
|
} else if (e instanceof UnknownHostException) {
|
|
//errorMessage = "IP address of the destination host could not be determined";
|
|
try {
|
|
detailedErrorMsgMap = unknownHostHandling(conProp);
|
|
httpStatusText = "Unknown Host";
|
|
detailedErrorMsg = true;
|
|
detailedErrorMsgFile = "proxymsg/unknownHost.inc";
|
|
} catch (final Exception e1) {
|
|
errorMessage = "IP address of the destination host could not be determined";
|
|
}
|
|
} else if (e instanceof SocketTimeoutException) {
|
|
errorMessage = "Unable to establish a connection to the destination host. Connect timed out.";
|
|
} else {
|
|
final String exceptionMsg = e.getMessage();
|
|
if ((exceptionMsg != null) && (exceptionMsg.indexOf("Corrupt GZIP trailer",0) >= 0)) {
|
|
// just do nothing, we leave it this way
|
|
if (log.isFine()) log.fine("ignoring bad gzip trail for URL " + url + " (" + e.getMessage() + ")");
|
|
forceConnectionClose(conProp);
|
|
} else if ((exceptionMsg != null) && (exceptionMsg.indexOf("Connection reset",0)>= 0)) {
|
|
errorMessage = "Connection reset";
|
|
} else if ((exceptionMsg != null) && (exceptionMsg.indexOf("unknown host",0)>=0)) {
|
|
try {
|
|
detailedErrorMsgMap = unknownHostHandling(conProp);
|
|
httpStatusText = "Unknown Host";
|
|
detailedErrorMsg = true;
|
|
detailedErrorMsgFile = "proxymsg/unknownHost.inc";
|
|
} catch (final Exception e1) {
|
|
errorMessage = "IP address of the destination host could not be determined";
|
|
}
|
|
} else if ((exceptionMsg != null) &&
|
|
(
|
|
(exceptionMsg.indexOf("socket write error",0)>=0) ||
|
|
(exceptionMsg.indexOf("Read timed out",0) >= 0) ||
|
|
(exceptionMsg.indexOf("Broken pipe",0) >= 0) ||
|
|
(exceptionMsg.indexOf("server has closed connection",0) >= 0)
|
|
)) {
|
|
errorMessage = exceptionMsg;
|
|
ConcurrentLog.logException(e);
|
|
} else {
|
|
errorMessage = "Unexpected Error. " + e.getClass().getName() + ": " + e.getMessage();
|
|
unknownError = true;
|
|
errorExc = e;
|
|
}
|
|
}
|
|
|
|
// sending back an error message to the client
|
|
if (!conProp.containsKey(HeaderFramework.CONNECTION_PROP_PROXY_RESPOND_HEADER)) {
|
|
if (detailedErrorMsg) {
|
|
HTTPDemon.sendRespondError(conProp,respond, httpStatusCode, httpStatusText, new File(detailedErrorMsgFile), detailedErrorMsgMap, errorExc);
|
|
} else {
|
|
HTTPDemon.sendRespondError(conProp,respond,4,httpStatusCode,httpStatusText,errorMessage,errorExc);
|
|
}
|
|
} else {
|
|
if (unknownError) {
|
|
log.severe("Unknown Error while processing request 'PROXY':" +
|
|
"\n" + Thread.currentThread().getName() +
|
|
"\n" + errorMessage,e);
|
|
} else {
|
|
log.warn("Error while processing request 'PROXY':" +
|
|
"\n" + Thread.currentThread().getName() +
|
|
"\n" + errorMessage);
|
|
}
|
|
forceConnectionClose(conProp);
|
|
}
|
|
} catch (final Exception ee) {
|
|
forceConnectionClose(conProp);
|
|
}
|
|
|
|
}
|
|
|
|
private static void forceConnectionClose(final HashMap<String, Object> conProp) {
|
|
if (conProp != null) {
|
|
conProp.put(HeaderFramework.CONNECTION_PROP_PERSISTENT,"close");
|
|
}
|
|
}
|
|
|
|
private static serverObjects unknownHostHandling(final HashMap<String, Object> conProp) throws Exception {
|
|
final serverObjects detailedErrorMsgMap = new serverObjects();
|
|
|
|
// generic toplevel domains
|
|
final HashSet<String> topLevelDomains = new HashSet<String>(Arrays.asList(new String[]{
|
|
"aero", // Fluggesellschaften/Luftfahrt
|
|
"arpa", // Einrichtung des ARPANet
|
|
"biz", // Business
|
|
"com", // Commercial
|
|
"coop", // genossenschaftliche Unternehmen
|
|
"edu", // Education
|
|
"gov", // Government
|
|
"info", // Informationsangebote
|
|
"int", // International
|
|
"jobs", // Jobangebote von Unternemen
|
|
"mil", // Military (US-Militaer)
|
|
// "museum", // Museen
|
|
"name", // Privatpersonen
|
|
"nato", // NATO (veraltet)
|
|
"net", // Net (Netzwerkbetreiber)
|
|
"org", // Organization (Nichtkommerzielle Organisation)
|
|
"pro", // Professionals
|
|
"travel", // Touristikindustrie
|
|
|
|
// some country tlds
|
|
"de",
|
|
"at",
|
|
"ch",
|
|
"it",
|
|
"uk"
|
|
}));
|
|
|
|
// getting some connection properties
|
|
DigestURL orgurl = (DigestURL) conProp.get(HeaderFramework.CONNECTION_PROP_DIGESTURL);
|
|
int orgHostPort = orgurl.getPort();
|
|
String orgHostName = orgurl.getHost();
|
|
if (orgHostName == null) orgHostName = "unknown";
|
|
orgHostName = orgHostName.toLowerCase();
|
|
String orgHostPath = orgurl.getPath(); if (orgHostPath == null) orgHostPath = "";
|
|
String orgHostArgs = orgurl.getSearchpart();; if (orgHostArgs == null) orgHostArgs = "";
|
|
if (orgHostArgs.length() > 0) orgHostArgs = "?" + orgHostArgs;
|
|
detailedErrorMsgMap.put("hostName", orgHostName);
|
|
|
|
// guessing hostnames
|
|
final HashSet<String> testHostNames = new HashSet<String>();
|
|
String testHostName = null;
|
|
if (!orgHostName.startsWith("www.")) {
|
|
testHostName = "www." + orgHostName;
|
|
final InetAddress addr = Domains.dnsResolve(testHostName);
|
|
if (addr != null) testHostNames.add(testHostName);
|
|
} else if (orgHostName.startsWith("www.")) {
|
|
testHostName = orgHostName.substring(4);
|
|
final InetAddress addr = Domains.dnsResolve(testHostName);
|
|
if (addr != null) testHostNames.add(testHostName);
|
|
}
|
|
if (orgHostName.length()>4 && orgHostName.startsWith("www") && (orgHostName.charAt(3) != '.')) {
|
|
testHostName = orgHostName.substring(0,3) + "." + orgHostName.substring(3);
|
|
final InetAddress addr = Domains.dnsResolve(testHostName);
|
|
if (addr != null) testHostNames.add(testHostName);
|
|
}
|
|
|
|
int pos = orgHostName.lastIndexOf('.');
|
|
if (pos != -1) {
|
|
final Iterator<String> iter = topLevelDomains.iterator();
|
|
while (iter.hasNext()) {
|
|
final String topLevelDomain = iter.next();
|
|
testHostName = orgHostName.substring(0,pos) + "." + topLevelDomain;
|
|
final InetAddress addr = Domains.dnsResolve(testHostName);
|
|
if (addr != null) testHostNames.add(testHostName);
|
|
}
|
|
}
|
|
|
|
int hostNameCount = 0;
|
|
final Iterator<String> iter = testHostNames.iterator();
|
|
while (iter.hasNext()) {
|
|
testHostName = iter.next();
|
|
detailedErrorMsgMap.put("list_" + hostNameCount + "_hostName",testHostName);
|
|
detailedErrorMsgMap.put("list_" + hostNameCount + "_hostPort",orgHostPort);
|
|
detailedErrorMsgMap.put("list_" + hostNameCount + "_hostPath",orgHostPath);
|
|
detailedErrorMsgMap.put("list_" + hostNameCount + "_hostArgs",orgHostArgs);
|
|
hostNameCount++;
|
|
}
|
|
|
|
detailedErrorMsgMap.put("list", hostNameCount);
|
|
|
|
if (hostNameCount != 0) {
|
|
detailedErrorMsgMap.put("showList", 1);
|
|
} else {
|
|
detailedErrorMsgMap.put("showList", 0);
|
|
}
|
|
|
|
return detailedErrorMsgMap;
|
|
}
|
|
|
|
private static synchronized String generateUserAgent(final HeaderFramework requestHeaders) {
|
|
userAgentStr.setLength(0);
|
|
|
|
final String browserUserAgent = requestHeaders.get(HeaderFramework.USER_AGENT, yacyProxyUserAgent);
|
|
final int pos = browserUserAgent.lastIndexOf(')');
|
|
if (pos >= 0) {
|
|
userAgentStr
|
|
.append(browserUserAgent.substring(0,pos))
|
|
.append("; YaCy ")
|
|
.append(yacyBuildProperties.getVersion())
|
|
.append("; yacy.net")
|
|
.append(browserUserAgent.substring(pos));
|
|
} else {
|
|
userAgentStr.append(browserUserAgent);
|
|
}
|
|
|
|
return userAgentStr.toString();
|
|
}
|
|
|
|
/**
|
|
* This function is used to generate a logging message according to the
|
|
* <a href="http://www.squid-cache.org/Doc/FAQ/FAQ-6.html">squid logging format</a>.<p>
|
|
* e.g.<br>
|
|
* <code>1117528623.857 178 192.168.1.201 TCP_MISS/200 1069 GET http://www.yacy.de/ - DIRECT/81.169.145.74 text/html</code>
|
|
*/
|
|
private final static synchronized void logProxyAccess(final HashMap<String, Object> conProp) {
|
|
|
|
logMessage.setLength(0);
|
|
|
|
// Timestamp
|
|
final String currentTimestamp = Long.toString(System.currentTimeMillis());
|
|
final int offset = currentTimestamp.length()-3;
|
|
|
|
logMessage.append(currentTimestamp.substring(0,offset));
|
|
logMessage.append('.');
|
|
logMessage.append(currentTimestamp.substring(offset));
|
|
logMessage.append(' ');
|
|
|
|
// Elapsed time
|
|
final Long requestStart = (Long) conProp.get(HeaderFramework.CONNECTION_PROP_REQUEST_START);
|
|
final Long requestEnd = (Long) conProp.get(HeaderFramework.CONNECTION_PROP_REQUEST_END);
|
|
final String elapsed = Long.toString(requestEnd.longValue()-requestStart.longValue());
|
|
|
|
for (int i=0; i<6-elapsed.length(); i++) logMessage.append(' ');
|
|
logMessage.append(elapsed);
|
|
logMessage.append(' ');
|
|
|
|
// Remote Host
|
|
final String clientIP = (String) conProp.get(HeaderFramework.CONNECTION_PROP_CLIENTIP);
|
|
logMessage.append(clientIP);
|
|
logMessage.append(' ');
|
|
|
|
// Code/Status
|
|
final String respondStatus = (String) conProp.get(HeaderFramework.CONNECTION_PROP_PROXY_RESPOND_STATUS);
|
|
String respondCode = (String) conProp.get(HeaderFramework.CONNECTION_PROP_PROXY_RESPOND_CODE);
|
|
if (respondCode == null) respondCode = "UNKNOWN";
|
|
logMessage.append(respondCode);
|
|
logMessage.append("/");
|
|
logMessage.append(respondStatus);
|
|
logMessage.append(' ');
|
|
|
|
// Bytes
|
|
final String bytes = (String) conProp.get(HeaderFramework.CONNECTION_PROP_PROXY_RESPOND_SIZE);
|
|
logMessage.append(bytes.toString());
|
|
logMessage.append(' ');
|
|
|
|
// Method
|
|
HttpServletRequest origrequest = (HttpServletRequest) conProp.get(HeaderFramework.CONNECTION_PROP_CLIENT_HTTPSERVLETREQUEST);
|
|
final String requestMethod = origrequest.getMethod();
|
|
logMessage.append(requestMethod);
|
|
logMessage.append(' ');
|
|
|
|
// URL
|
|
final DigestURL requestURL = (DigestURL) conProp.get(HeaderFramework.CONNECTION_PROP_DIGESTURL);
|
|
logMessage.append(requestURL.toString());
|
|
logMessage.append(' ');
|
|
|
|
// Rfc931
|
|
logMessage.append("-");
|
|
logMessage.append(' ');
|
|
|
|
// Peerstatus/Peerhost
|
|
logMessage.append("DIRECT/");
|
|
logMessage.append(requestURL.getHost());
|
|
logMessage.append(' ');
|
|
|
|
// Type
|
|
String mime = "-";
|
|
if (conProp.containsKey(HeaderFramework.CONNECTION_PROP_PROXY_RESPOND_HEADER)) {
|
|
final HeaderFramework proxyRespondHeader = (HeaderFramework) conProp.get(HeaderFramework.CONNECTION_PROP_PROXY_RESPOND_HEADER);
|
|
mime = proxyRespondHeader.mime();
|
|
}
|
|
logMessage.append(mime);
|
|
|
|
// sending the logging message to the logger
|
|
if (proxyLog.isFine()) proxyLog.fine(logMessage.toString());
|
|
}
|
|
|
|
}
|
|
|
|
/*
|
|
proxy test:
|
|
|
|
http://www.chipchapin.com/WebTools/cookietest.php?
|
|
http://xlists.aza.org/moderator/cookietest/cookietest1.php
|
|
http://vancouver-webpages.com/proxy/cache-test.html
|
|
|
|
*/
|