mirror of
https://github.com/yacy/yacy_search_server.git
synced 2024-09-21 00:00:13 +02:00
aab0b680c3
try i.e. http://localhost:8090/solr/select?q=*:*&start=0&rows=10&wt=xslt&tr=json.xsl - added servlet-side mime-type configuration for streamed servlets. this is used for the result formatters in solr result formats
1643 lines
82 KiB
Java
1643 lines
82 KiB
Java
// HTTPDFileHandler.java
|
|
// -----------------------
|
|
// (C) by Michael Peter Christen; mc@yacy.net
|
|
// first published on http://www.anomic.de
|
|
// Frankfurt, Germany, 2004, 2005
|
|
//
|
|
// $LastChangedDate$
|
|
// $LastChangedRevision$
|
|
// $LastChangedBy$
|
|
//
|
|
// This program is free software; you can redistribute it and/or modify
|
|
// it under the terms of the GNU General Public License as published by
|
|
// the Free Software Foundation; either version 2 of the License, or
|
|
// (at your option) any later version.
|
|
//
|
|
// This program is distributed in the hope that it will be useful,
|
|
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
// GNU General Public License for more details.
|
|
//
|
|
// You should have received a copy of the GNU General Public License
|
|
// along with this program; if not, write to the Free Software
|
|
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
|
|
|
/*
|
|
Class documentation:
|
|
this class provides a file servlet and CGI interface
|
|
for the httpd server.
|
|
Whenever this server is addressed to load a local file,
|
|
this class searches for the file in the local path as
|
|
configured in the setting property 'rootPath'
|
|
The servlet loads the file and returns it to the client.
|
|
Every file can also act as an template for the built-in
|
|
CGI interface. There is no specific path for CGI functions.
|
|
CGI functionality is triggered, if for the file to-be-served
|
|
'template.html' also a file 'template.class' exists. Then,
|
|
the class file is called with the GET/POST properties that
|
|
are attached to the http call.
|
|
Possible variable hand-over are:
|
|
- form method GET
|
|
- form method POST, enctype text/plain
|
|
- form method POST, enctype multipart/form-data
|
|
The class that creates the CGI respond must have at least one
|
|
static method of the form
|
|
public static java.util.Hashtable respond(java.util.HashMap, serverSwitch)
|
|
In the HashMap, the GET/POST variables are handed over.
|
|
The return value is a Property object that contains replacement
|
|
key/value pairs for the patterns in the template file.
|
|
The templates must have the form
|
|
either '#['<name>']#' for single attributes, or
|
|
'#{'<enumname>'}#' and '#{/'<enumname>'}#' for enumerations of
|
|
values '#['<value>']#'.
|
|
A single value in repetitions/enumerations in the template has
|
|
the property key '_'<enumname><count>'_'<value>
|
|
Please see also the example files 'test.html' and 'test.java'
|
|
*/
|
|
|
|
package de.anomic.http.server;
|
|
|
|
import java.awt.Image;
|
|
import java.awt.image.BufferedImage;
|
|
import java.io.BufferedInputStream;
|
|
import java.io.BufferedOutputStream;
|
|
import java.io.BufferedReader;
|
|
import java.io.ByteArrayInputStream;
|
|
import java.io.ByteArrayOutputStream;
|
|
import java.io.File;
|
|
import java.io.FileInputStream;
|
|
import java.io.FileNotFoundException;
|
|
import java.io.FileReader;
|
|
import java.io.IOException;
|
|
import java.io.InputStream;
|
|
import java.io.OutputStream;
|
|
import java.io.StringWriter;
|
|
import java.lang.ref.SoftReference;
|
|
import java.lang.reflect.InvocationTargetException;
|
|
import java.lang.reflect.Method;
|
|
import java.net.MalformedURLException;
|
|
import java.net.URL;
|
|
import java.net.URLDecoder;
|
|
import java.util.ArrayList;
|
|
import java.util.Arrays;
|
|
import java.util.Date;
|
|
import java.util.HashMap;
|
|
import java.util.Iterator;
|
|
import java.util.List;
|
|
import java.util.Map;
|
|
import java.util.concurrent.ConcurrentHashMap;
|
|
import java.util.regex.Matcher;
|
|
import java.util.regex.Pattern;
|
|
import java.util.zip.GZIPOutputStream;
|
|
|
|
import net.yacy.cora.date.GenericFormatter;
|
|
import net.yacy.cora.document.ASCII;
|
|
import net.yacy.cora.document.Classification;
|
|
import net.yacy.cora.document.MultiProtocolURI;
|
|
import net.yacy.cora.document.UTF8;
|
|
import net.yacy.cora.protocol.Domains;
|
|
import net.yacy.cora.protocol.HeaderFramework;
|
|
import net.yacy.cora.protocol.RequestHeader;
|
|
import net.yacy.cora.protocol.ResponseHeader;
|
|
import net.yacy.cora.util.NumberTools;
|
|
import net.yacy.document.parser.htmlParser;
|
|
import net.yacy.document.parser.html.ContentScraper;
|
|
import net.yacy.document.parser.html.ScraperInputStream;
|
|
import net.yacy.kelondro.data.meta.DigestURI;
|
|
import net.yacy.kelondro.logging.Log;
|
|
import net.yacy.kelondro.order.Digest;
|
|
import net.yacy.kelondro.util.ByteBuffer;
|
|
import net.yacy.kelondro.util.FileUtils;
|
|
import net.yacy.kelondro.util.MemoryControl;
|
|
import net.yacy.peers.Seed;
|
|
import net.yacy.peers.graphics.EncodedImage;
|
|
import net.yacy.peers.operation.yacyBuildProperties;
|
|
import net.yacy.search.Switchboard;
|
|
import net.yacy.search.SwitchboardConstants;
|
|
import net.yacy.visualization.RasterPlotter;
|
|
import de.anomic.data.UserDB;
|
|
import de.anomic.server.serverClassLoader;
|
|
import de.anomic.server.serverCore;
|
|
import de.anomic.server.serverObjects;
|
|
import de.anomic.server.serverSwitch;
|
|
import de.anomic.server.servletProperties;
|
|
|
|
public final class HTTPDFileHandler {
|
|
|
|
// create a class loader
|
|
private static final serverClassLoader provider = new serverClassLoader(/*this.getClass().getClassLoader()*/);
|
|
private static serverSwitch switchboard = null;
|
|
private static Switchboard sb = Switchboard.getSwitchboard();
|
|
private final static byte[] UNRESOLVED_PATTERN = ASCII.getBytes("-UNRESOLVED_PATTERN-");
|
|
|
|
|
|
private static File htRootPath = null;
|
|
private static File htDocsPath = null;
|
|
private static String[] defaultFiles = null;
|
|
private static File htDefaultPath = null;
|
|
private static File htLocalePath = null;
|
|
public static String indexForward = "";
|
|
|
|
protected static final class TemplateCacheEntry {
|
|
Date lastModified;
|
|
byte[] content;
|
|
}
|
|
private static final ConcurrentHashMap<File, SoftReference<TemplateCacheEntry>> templateCache;
|
|
private static final ConcurrentHashMap<File, SoftReference<Method>> templateMethodCache;
|
|
|
|
public static final boolean useTemplateCache;
|
|
|
|
//private Properties connectionProperties = null;
|
|
// creating a logger
|
|
private static final Log theLogger = new Log("FILEHANDLER");
|
|
|
|
static {
|
|
final serverSwitch theSwitchboard = Switchboard.getSwitchboard();
|
|
useTemplateCache = theSwitchboard.getConfig("enableTemplateCache","true").equalsIgnoreCase("true");
|
|
templateCache = (useTemplateCache)? new ConcurrentHashMap<File, SoftReference<TemplateCacheEntry>>() : new ConcurrentHashMap<File, SoftReference<TemplateCacheEntry>>(0);
|
|
templateMethodCache = new ConcurrentHashMap<File, SoftReference<Method>>();
|
|
|
|
if (switchboard == null) {
|
|
switchboard = theSwitchboard;
|
|
|
|
if (Classification.countMimes() == 0) {
|
|
// load the mime table
|
|
final String mimeTablePath = theSwitchboard.getConfig("mimeTable","");
|
|
Log.logConfig("HTTPDFiles", "Loading mime mapping file " + mimeTablePath);
|
|
Classification.init(new File(theSwitchboard.getAppPath(), mimeTablePath));
|
|
}
|
|
|
|
// create default files array
|
|
initDefaultPath();
|
|
|
|
// create a htRootPath: system pages
|
|
if (htRootPath == null) {
|
|
htRootPath = new File(theSwitchboard.getAppPath(), theSwitchboard.getConfig(SwitchboardConstants.HTROOT_PATH, SwitchboardConstants.HTROOT_PATH_DEFAULT));
|
|
if (!(htRootPath.exists())) htRootPath.mkdir();
|
|
}
|
|
|
|
// create a htDocsPath: user defined pages
|
|
if (htDocsPath == null) {
|
|
htDocsPath = theSwitchboard.getDataPath(SwitchboardConstants.HTDOCS_PATH, SwitchboardConstants.HTDOCS_PATH_DEFAULT);
|
|
if (!(htDocsPath.exists())) htDocsPath.mkdirs();
|
|
}
|
|
|
|
// create a repository path
|
|
final File repository = new File(htDocsPath, "repository");
|
|
if (!repository.exists()) repository.mkdirs();
|
|
|
|
// create htLocaleDefault, htLocalePath
|
|
if (htDefaultPath == null) htDefaultPath = theSwitchboard.getAppPath("htDefaultPath", "htroot");
|
|
if (htLocalePath == null) htLocalePath = theSwitchboard.getDataPath("locale.translated_html", "DATA/LOCALE/htroot");
|
|
}
|
|
|
|
}
|
|
|
|
public static final void initDefaultPath() {
|
|
// create default files array
|
|
defaultFiles = switchboard.getConfig(SwitchboardConstants.BROWSER_DEFAULT,"index.html").split(",");
|
|
if (defaultFiles.length == 0) defaultFiles = new String[] {"index.html"};
|
|
indexForward = switchboard.getConfig(SwitchboardConstants.INDEX_FORWARD, "");
|
|
if (indexForward.startsWith("/")) indexForward = indexForward.substring(1);
|
|
}
|
|
|
|
/** Returns a path to the localized or default file according to the locale.language (from he switchboard)
|
|
* @param path relative from htroot */
|
|
public static File getLocalizedFile(final String path){
|
|
return getLocalizedFile(path, switchboard.getConfig("locale.language","default"));
|
|
}
|
|
|
|
/** Returns a path to the localized or default file according to the parameter localeSelection
|
|
* @param path relative from htroot
|
|
* @param localeSelection language of localized file; locale.language from switchboard is used if localeSelection.equals("") */
|
|
public static File getLocalizedFile(final String path, final String localeSelection){
|
|
//if (htDefaultPath == null) htDefaultPath = switchboard.getConfigPath("htDefaultPath", "htroot");
|
|
//if (htLocalePath == null) htLocalePath = switchboard.getConfigPath("locale.translated_html", "DATA/LOCALE/htroot");
|
|
//if (htDocsPath == null) htDocsPath = switchboard.getConfigPath(plasmaSwitchboardConstants.HTDOCS_PATH, plasmaSwitchboardConstants.HTDOCS_PATH_DEFAULT);
|
|
|
|
if (path.startsWith("/repository/"))
|
|
return new File(switchboard.getConfig("repositoryPath", "DATA/HTDOCS/repository"), path.substring(11));
|
|
if (!(localeSelection.equals("default"))) {
|
|
final File localePath = new File(htLocalePath, localeSelection + '/' + path);
|
|
if (localePath.exists()) return localePath; // avoid "NoSuchFile" troubles if the "localeSelection" is misspelled
|
|
}
|
|
|
|
final File docsPath = new File(htDocsPath, path);
|
|
if (docsPath.exists()) return docsPath;
|
|
return new File(htDefaultPath, path);
|
|
}
|
|
|
|
private static final ResponseHeader getDefaultHeaders(final String path) {
|
|
final ResponseHeader headers = new ResponseHeader(200);
|
|
String ext;
|
|
int pos;
|
|
if ((pos = path.lastIndexOf('.')) < 0) {
|
|
ext = "";
|
|
} else {
|
|
ext = path.substring(pos + 1).toLowerCase();
|
|
}
|
|
headers.put(HeaderFramework.SERVER, "AnomicHTTPD (www.anomic.de)");
|
|
headers.put(HeaderFramework.DATE, HeaderFramework.formatRFC1123(new Date()));
|
|
if(!(Classification.isMediaExtension(ext))){
|
|
headers.put(HeaderFramework.PRAGMA, "no-cache");
|
|
}
|
|
return headers;
|
|
}
|
|
|
|
public static void doGet(final HashMap<String, Object> conProp, final RequestHeader requestHeader, final OutputStream response) {
|
|
doResponse(conProp, requestHeader, response, null);
|
|
}
|
|
|
|
public static void doHead(final HashMap<String, Object> conProp, final RequestHeader requestHeader, final OutputStream response) {
|
|
doResponse(conProp, requestHeader, response, null);
|
|
}
|
|
|
|
public static void doPost(final HashMap<String, Object> conProp, final RequestHeader requestHeader, final OutputStream response, final InputStream body) {
|
|
doResponse(conProp, requestHeader, response, body);
|
|
}
|
|
|
|
public static void doResponse(final HashMap<String, Object> conProp, final RequestHeader requestHeader, final OutputStream out, final InputStream body) {
|
|
|
|
String path = null;
|
|
try {
|
|
// getting some connection properties
|
|
final String method = (String) conProp.get(HeaderFramework.CONNECTION_PROP_METHOD);
|
|
path = (String) conProp.get(HeaderFramework.CONNECTION_PROP_PATH);
|
|
String argsString = (String) conProp.get(HeaderFramework.CONNECTION_PROP_ARGS); // is null if no args were given
|
|
final String httpVersion = (String) conProp.get(HeaderFramework.CONNECTION_PROP_HTTP_VER);
|
|
String clientIP = (String) conProp.get(HeaderFramework.CONNECTION_PROP_CLIENTIP); if (clientIP == null) clientIP = "unknown-host";
|
|
|
|
// check hack attacks in path
|
|
if (path.indexOf("..",0) >= 0) {
|
|
HTTPDemon.sendRespondError(conProp,out,4,403,null,"Access not allowed",null);
|
|
return;
|
|
}
|
|
|
|
path = UTF8.decodeURL(path);
|
|
|
|
// check against hack attacks in path
|
|
if (path.indexOf("..",0) >= 0) {
|
|
HTTPDemon.sendRespondError(conProp,out,4,403,null,"Access not allowed",null);
|
|
return;
|
|
}
|
|
|
|
// allow proper access to current peer via virtual directory
|
|
if (path.startsWith("/currentyacypeer/")) {
|
|
path = path.substring(16);
|
|
}
|
|
|
|
// cache settings
|
|
boolean nocache = path.contains("?") || body != null;
|
|
|
|
// a bad patch to map the /xml/ path to /api/
|
|
if (path.startsWith("/xml/")) {
|
|
path = "/api/" + path.substring(5);
|
|
}
|
|
// another bad patch to map the /util/ path to /api/util/ to support old yacybars
|
|
if (path.startsWith("/util/")) {
|
|
path = "/api/util/" + path.substring(6);
|
|
}
|
|
// one more for bookmarks
|
|
if (path.startsWith("/bookmarks/")) {
|
|
path = "/api/bookmarks/" + path.substring(11);
|
|
}
|
|
|
|
// these are the 5 cases where an access granted:
|
|
// (the alternative is that we deliver a 401 to request authorization)
|
|
|
|
// -1- the page is not protected; or
|
|
final boolean protectedPage = path.indexOf("_p.",0) > 0;
|
|
boolean accessGranted = !protectedPage;
|
|
|
|
// -2- a password is not configured; or
|
|
final String adminAccountBase64MD5 = switchboard.getConfig(SwitchboardConstants.ADMIN_ACCOUNT_B64MD5, "");
|
|
if (!accessGranted) {
|
|
accessGranted = adminAccountBase64MD5.isEmpty();
|
|
}
|
|
|
|
// -3- access from localhost is granted and access comes from localhost; or
|
|
final String refererHost = requestHeader.refererHost();
|
|
if (!accessGranted) {
|
|
final boolean adminAccountForLocalhost = sb.getConfigBool("adminAccountForLocalhost", false);
|
|
final boolean accessFromLocalhost = Domains.isLocalhost(clientIP) && (refererHost == null || refererHost.isEmpty() || Domains.isLocalhost(refererHost));
|
|
accessGranted = adminAccountForLocalhost && accessFromLocalhost;
|
|
}
|
|
|
|
// -4- a password is configured and access comes from localhost
|
|
// and the realm-value of a http-authentify String is equal to the stored base64MD5; or
|
|
String realmProp = requestHeader.get(RequestHeader.AUTHORIZATION);
|
|
if (realmProp != null && realmProp.isEmpty()) realmProp = null;
|
|
final String realmValue = realmProp == null ? null : realmProp.substring(6);
|
|
if (!accessGranted) {
|
|
final boolean accessFromLocalhost = Domains.isLocalhost(clientIP) && (refererHost == null || refererHost.isEmpty() || Domains.isLocalhost(refererHost));
|
|
accessGranted = accessFromLocalhost && realmValue != null && realmProp.length() > 6 && (adminAccountBase64MD5.equals(realmValue));
|
|
Log.logInfo("HTTPDFileHandler", "access from localhost blocked, clientIP=" + clientIP);
|
|
}
|
|
|
|
// -5- a password is configured and access comes with matching http-authentify
|
|
if (!accessGranted) {
|
|
accessGranted = realmProp != null && realmValue != null && (sb.userDB.hasAdminRight(realmProp, requestHeader.getHeaderCookies()) || adminAccountBase64MD5.equals(Digest.encodeMD5Hex(realmValue)));
|
|
}
|
|
|
|
// in case that we are still not granted we ask for a password
|
|
if (!accessGranted) {
|
|
Log.logInfo("HTTPD", "Wrong log-in for path '" + path + "' from host '" + clientIP + "'");
|
|
final Integer attempts = serverCore.bfHost.get(clientIP);
|
|
if (attempts == null)
|
|
serverCore.bfHost.put(clientIP, Integer.valueOf(1));
|
|
else
|
|
serverCore.bfHost.put(clientIP, Integer.valueOf(attempts.intValue() + 1));
|
|
|
|
final ResponseHeader responseHeader = getDefaultHeaders(path);
|
|
responseHeader.put(RequestHeader.WWW_AUTHENTICATE, "Basic realm=\"" + serverObjects.ADMIN_AUTHENTICATE_MSG + "\"");
|
|
final servletProperties tp=new servletProperties();
|
|
tp.put("returnto", path);
|
|
HTTPDemon.sendRespondError(conProp, out, 5, 401, "Wrong Authentication", "", new File("proxymsg/authfail.inc"), tp, null, responseHeader);
|
|
return;
|
|
}
|
|
|
|
// Authentication successful. remove brute-force flag
|
|
serverCore.bfHost.remove(conProp.get(HeaderFramework.CONNECTION_PROP_CLIENTIP));
|
|
|
|
// parse arguments
|
|
serverObjects args = new serverObjects();
|
|
int argc = 0;
|
|
if (argsString == null) {
|
|
// no args here, maybe a POST with multipart extension
|
|
final int length = requestHeader.getContentLength();
|
|
//System.out.println("HEADER: " + requestHeader.toString()); // DEBUG
|
|
|
|
/* don't parse body in case of a POST CGI call since it has to be
|
|
* handed over to the CGI script unaltered and parsed by the script
|
|
*/
|
|
if (method.equals(HeaderFramework.METHOD_POST) &&
|
|
!(switchboard.getConfigBool("cgi.allow", false) &&
|
|
matchesSuffix(path, switchboard.getConfig("cgi.suffixes", null)))
|
|
) {
|
|
|
|
// if its a POST, it can be either multipart or as args in the body
|
|
if ((requestHeader.containsKey(HeaderFramework.CONTENT_TYPE)) &&
|
|
(requestHeader.get(HeaderFramework.CONTENT_TYPE).toLowerCase().startsWith("multipart"))) {
|
|
// parse multipart
|
|
final Map<String, byte[]> files = HTTPDemon.parseMultipart(requestHeader, args, body);
|
|
// integrate these files into the args
|
|
if (files != null) {
|
|
final Iterator<Map.Entry<String, byte[]>> fit = files.entrySet().iterator();
|
|
Map.Entry<String, byte[]> entry;
|
|
while (fit.hasNext()) {
|
|
entry = fit.next();
|
|
args.put(entry.getKey() + "$file", entry.getValue());
|
|
}
|
|
}
|
|
argc = Integer.parseInt(requestHeader.get("ARGC"));
|
|
} else {
|
|
// parse args in body
|
|
argc = HTTPDemon.parseArgs(args, body, length);
|
|
}
|
|
} else {
|
|
// no args
|
|
argsString = null;
|
|
args = null;
|
|
argc = 0;
|
|
}
|
|
} else {
|
|
// simple args in URL (stuff after the "?")
|
|
argc = HTTPDemon.parseArgs(args, argsString);
|
|
}
|
|
|
|
// check for cross site scripting - attacks in request arguments
|
|
if (args != null && argc > 0) {
|
|
// check all values for occurrences of script values
|
|
final Iterator<String> e = args.values().iterator(); // enumeration of values
|
|
String val;
|
|
while (e.hasNext()) {
|
|
val = e.next();
|
|
if ((val != null) && (val.indexOf("<script",0) >= 0) && !path.equals("/Crawler_p.html")) {
|
|
// deny request
|
|
HTTPDemon.sendRespondError(conProp,out,4,403,null,"bad post values",null);
|
|
return;
|
|
}
|
|
}
|
|
}
|
|
|
|
if (args != null) nocache = true;
|
|
|
|
// we are finished with parsing
|
|
// the result of value hand-over is in args and argc
|
|
if (path.isEmpty()) {
|
|
HTTPDemon.sendRespondError(conProp,out,4,400,null,"Bad Request",null);
|
|
out.flush();
|
|
return;
|
|
}
|
|
File targetClass = null;
|
|
|
|
// locate the file
|
|
if (!path.isEmpty() && path.charAt(0) != '/' && path.charAt(0) != '\\') {
|
|
path = "/" + path; // attach leading slash
|
|
}
|
|
if (path.endsWith("index.html")) {
|
|
path = path.substring(0, path.length() - 10);
|
|
}
|
|
|
|
// a different language can be desired (by i.e. ConfigBasic.html) than the one stored in the locale.language
|
|
String localeSelection = switchboard.getConfig("locale.language","default");
|
|
if (args != null && (args.containsKey("language"))) {
|
|
// TODO 9.11.06 Bost: a class with information about available languages is needed.
|
|
// the indexOf(".") is just a workaround because there from ConfigLanguage.html commes "de.lng" and
|
|
// from ConfigBasic.html comes just "de" in the "language" parameter
|
|
localeSelection = args.get("language", localeSelection);
|
|
if (localeSelection.indexOf('.') != -1)
|
|
localeSelection = localeSelection.substring(0, localeSelection.indexOf('.'));
|
|
}
|
|
|
|
File targetFile = getLocalizedFile(path, localeSelection);
|
|
String targetExt = (String) conProp.get("EXT"); if (targetExt == null) targetExt = "";
|
|
targetClass = rewriteClassFile(new File(htDefaultPath, path));
|
|
if (path.endsWith("/") || path.endsWith("\\")) {
|
|
String testpath;
|
|
// look for indexForward setting
|
|
if (indexForward.length() > 0 && (targetFile = getOverlayedFile(path + indexForward)).exists()) {
|
|
testpath = path + indexForward;
|
|
targetClass = getOverlayedClass(testpath);
|
|
path = testpath;
|
|
} else {
|
|
// attach default file name(s)
|
|
for (final String defaultFile : defaultFiles) {
|
|
testpath = path + defaultFile;
|
|
targetFile = getOverlayedFile(testpath);
|
|
targetClass = getOverlayedClass(testpath);
|
|
if (targetFile.exists()) {
|
|
path = testpath;
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
targetFile = getLocalizedFile(path, localeSelection);
|
|
|
|
//no defaultfile, send a dirlisting
|
|
if (targetFile == null || !targetFile.exists() || (targetFile.exists() && targetFile.isDirectory())) {
|
|
final StringBuilder aBuffer = new StringBuilder();
|
|
aBuffer.append("<html>\n<head>\n</head>\n<body>\n<h1>Index of " + path + "</h1>\n <ul>\n");
|
|
String[] list = targetFile.list();
|
|
if (list == null) list = new String[0]; // should not occur!
|
|
File f;
|
|
String size;
|
|
long sz;
|
|
String headline, author, description, publisher;
|
|
int images, links;
|
|
ContentScraper scraper;
|
|
for (final String element : list) {
|
|
f = new File(targetFile, element);
|
|
if (f.isDirectory()) {
|
|
aBuffer.append(" <li><a href=\"" + path + element + "/\">" + element + "/</a><br/></li>\n");
|
|
} else {
|
|
if (element.endsWith("html") || (element.endsWith("htm"))) {
|
|
scraper = ContentScraper.parseResource(f, 10000);
|
|
headline = scraper.getTitle();
|
|
author = scraper.getAuthor();
|
|
publisher = scraper.getPublisher();
|
|
description = scraper.getDescription();
|
|
images = scraper.getImages().size();
|
|
links = scraper.getAnchors().size();
|
|
} else {
|
|
headline = null;
|
|
author = null;
|
|
publisher = null;
|
|
description = null;
|
|
images = 0;
|
|
links = 0;
|
|
}
|
|
sz = f.length();
|
|
if (sz < 1024) {
|
|
size = sz + " bytes";
|
|
} else if (sz < 1024 * 1024) {
|
|
size = (sz / 1024) + " KB";
|
|
} else {
|
|
size = (sz / 1024 / 1024) + " MB";
|
|
}
|
|
aBuffer.append(" <li>");
|
|
if (headline != null && headline.length() > 0) aBuffer.append("<a href=\"" + element + "\"><b>" + headline + "</b></a><br/>");
|
|
aBuffer.append("<a href=\"" + path + element + "\">" + element + "</a><br/>");
|
|
if (author != null && author.length() > 0) aBuffer.append("Author: " + author + "<br/>");
|
|
if (publisher != null && publisher.length() > 0) aBuffer.append("Publisher: " + publisher + "<br/>");
|
|
if (description != null && description.length() > 0) aBuffer.append("Description: " + description + "<br/>");
|
|
aBuffer.append(GenericFormatter.SHORT_DAY_FORMATTER.format(new Date(f.lastModified())) + ", " + size + ((images > 0) ? ", " + images + " images" : "") + ((links > 0) ? ", " + links + " links" : "") + "<br/></li>\n");
|
|
}
|
|
}
|
|
aBuffer.append(" </ul>\n</body>\n</html>\n");
|
|
|
|
// write the list to the client
|
|
HTTPDemon.sendRespondHeader(conProp, out, httpVersion, 200, null, "text/html; charset=UTF-8", aBuffer.length(), new Date(targetFile.lastModified()), null, new ResponseHeader(200), null, null, true);
|
|
if (!method.equals(HeaderFramework.METHOD_HEAD)) {
|
|
out.write(UTF8.getBytes(aBuffer.toString()));
|
|
}
|
|
return;
|
|
}
|
|
} else {
|
|
//XXX: you cannot share a .png/.gif file with a name like a class in htroot.
|
|
if ( !(targetFile.exists()) &&
|
|
!((path.endsWith("png")||path.endsWith("gif") || path.indexOf('.') < 0 ||
|
|
matchesSuffix(path, switchboard.getConfig("cgi.suffixes", null)) ||
|
|
path.endsWith(".stream")) &&
|
|
targetClass!=null ) ){
|
|
targetFile = new File(htDocsPath, path);
|
|
targetClass = rewriteClassFile(new File(htDocsPath, path));
|
|
}
|
|
}
|
|
|
|
// implement proxy via url (not in servlet, because we need binary access on ouputStream)
|
|
if (path.equals("/proxy.html")) {
|
|
final List<Pattern> urlProxyAccess = Domains.makePatterns(sb.getConfig("proxyURL.access", Domains.LOCALHOST));
|
|
final UserDB.Entry user = sb.userDB.getUser(requestHeader);
|
|
final boolean user_may_see_proxyurl = Domains.matchesList(clientIP, urlProxyAccess) || (user!=null && user.hasRight(UserDB.AccessRight.PROXY_RIGHT));
|
|
if (sb.getConfigBool("proxyURL", false) && user_may_see_proxyurl) {
|
|
doURLProxy(conProp, requestHeader, out);
|
|
return;
|
|
}
|
|
HTTPDemon.sendRespondError(conProp,out,3,403,"Access denied",null,null);
|
|
}
|
|
|
|
// track all files that had been accessed so far
|
|
if (targetFile != null && targetFile.exists()) {
|
|
if (args != null && !args.isEmpty()) sb.setConfig("server.servlets.submitted", appendPath(sb.getConfig("server.servlets.submitted", ""), path));
|
|
}
|
|
|
|
//File targetClass = rewriteClassFile(targetFile);
|
|
//We need tp here
|
|
servletProperties templatePatterns = null;
|
|
Date targetDate;
|
|
|
|
if ((targetClass != null) && (path.endsWith("png"))) {
|
|
// call an image-servlet to produce an on-the-fly - generated image
|
|
Object img = null;
|
|
requestHeader.put(HeaderFramework.CONNECTION_PROP_CLIENTIP, (String) conProp.get(HeaderFramework.CONNECTION_PROP_CLIENTIP));
|
|
requestHeader.put(HeaderFramework.CONNECTION_PROP_PATH, path);
|
|
requestHeader.put(HeaderFramework.CONNECTION_PROP_EXT, "png");
|
|
// in case that there are no args given, args = null or empty hashmap
|
|
img = invokeServlet(targetClass, requestHeader, args, null);
|
|
if (img == null) {
|
|
// error with image generation; send file-not-found
|
|
HTTPDemon.sendRespondError(conProp, out, 3, 404, "File not Found", null, null);
|
|
} else {
|
|
if (img instanceof RasterPlotter) {
|
|
final RasterPlotter yp = (RasterPlotter) img;
|
|
// send an image to client
|
|
targetDate = new Date(System.currentTimeMillis());
|
|
nocache = true;
|
|
final String mimeType = Classification.ext2mime(targetExt, "text/html");
|
|
final ByteBuffer result = RasterPlotter.exportImage(yp.getImage(), targetExt);
|
|
|
|
// write the array to the client
|
|
HTTPDemon.sendRespondHeader(conProp, out, httpVersion, 200, null, mimeType, result.length(), targetDate, null, null, null, null, nocache);
|
|
if (!method.equals(HeaderFramework.METHOD_HEAD)) {
|
|
result.writeTo(out);
|
|
}
|
|
}
|
|
if (img instanceof EncodedImage) {
|
|
final EncodedImage yp = (EncodedImage) img;
|
|
// send an image to client
|
|
targetDate = new Date(System.currentTimeMillis());
|
|
nocache = true;
|
|
final String mimeType = Classification.ext2mime(targetExt, "text/html");
|
|
final ByteBuffer result = yp.getImage();
|
|
|
|
// write the array to the client
|
|
HTTPDemon.sendRespondHeader(conProp, out, httpVersion, 200, null, mimeType, result.length(), targetDate, null, null, null, null, nocache);
|
|
if (!method.equals(HeaderFramework.METHOD_HEAD)) {
|
|
result.writeTo(out);
|
|
}
|
|
}
|
|
/*
|
|
if (img instanceof BufferedImage) {
|
|
final BufferedImage i = (BufferedImage) img;
|
|
// send an image to client
|
|
targetDate = new Date(System.currentTimeMillis());
|
|
nocache = true;
|
|
final String mimeType = MimeTable.ext2mime(targetExt, "text/html");
|
|
|
|
// generate an byte array from the generated image
|
|
int width = i.getWidth(); if (width < 0) width = 96; // bad hack
|
|
int height = i.getHeight(); if (height < 0) height = 96; // bad hack
|
|
final ByteBuffer result = RasterPlotter.exportImage(i, targetExt);
|
|
|
|
// write the array to the client
|
|
HTTPDemon.sendRespondHeader(conProp, out, httpVersion, 200, null, mimeType, result.length(), targetDate, null, null, null, null, nocache);
|
|
if (!method.equals(HeaderFramework.METHOD_HEAD)) {
|
|
result.writeTo(out);
|
|
}
|
|
}
|
|
*/
|
|
if (img instanceof Image) {
|
|
final Image i = (Image) img;
|
|
// send an image to client
|
|
targetDate = new Date(System.currentTimeMillis());
|
|
nocache = true;
|
|
final String mimeType = Classification.ext2mime(targetExt, "text/html");
|
|
|
|
// generate an byte array from the generated image
|
|
int width = i.getWidth(null); if (width < 0) width = 96; // bad hack
|
|
int height = i.getHeight(null); if (height < 0) height = 96; // bad hack
|
|
final BufferedImage bi = new BufferedImage(width, height, BufferedImage.TYPE_INT_RGB);
|
|
bi.createGraphics().drawImage(i, 0, 0, width, height, null);
|
|
final ByteBuffer result = RasterPlotter.exportImage(bi, targetExt);
|
|
|
|
// write the array to the client
|
|
HTTPDemon.sendRespondHeader(conProp, out, httpVersion, 200, null, mimeType, result.length(), targetDate, null, null, null, null, nocache);
|
|
if (!method.equals(HeaderFramework.METHOD_HEAD)) {
|
|
result.writeTo(out);
|
|
}
|
|
}
|
|
}
|
|
// old-school CGI execution
|
|
} else if ((switchboard.getConfigBool("cgi.allow", false) // check if CGI execution is allowed in config
|
|
&& matchesSuffix(path, switchboard.getConfig("cgi.suffixes", null)) // "right" file extension?
|
|
&& path.substring(0, path.indexOf(targetFile.getName())).toUpperCase().contains("/CGI-BIN/") // file in right directory?
|
|
&& targetFile.exists())
|
|
) {
|
|
|
|
if (!targetFile.canExecute()) {
|
|
HTTPDemon.sendRespondError(
|
|
conProp,
|
|
out,
|
|
-1,
|
|
403,
|
|
null,
|
|
HeaderFramework.http1_1.get(
|
|
Integer.toString(403)),
|
|
null);
|
|
Log.logWarning(
|
|
"HTTPD",
|
|
"CGI script " + targetFile.getPath()
|
|
+ " could not be executed due to "
|
|
+ "insufficient access rights.");
|
|
} else {
|
|
String mimeType = "text/html";
|
|
int statusCode = 200;
|
|
|
|
final ProcessBuilder pb =
|
|
new ProcessBuilder(assembleCommandFromShebang(targetFile));
|
|
pb.directory(targetFile.getParentFile());
|
|
|
|
final String fileSeparator =
|
|
System.getProperty("file.separator", "/");
|
|
|
|
// set environment variables
|
|
final Map<String, String> env = pb.environment();
|
|
env.put(
|
|
"SERVER_SOFTWARE",
|
|
getDefaultHeaders(path).get(HeaderFramework.SERVER));
|
|
env.put("SERVER_NAME", sb.peers.mySeed().getName());
|
|
env.put("GATEWAY_INTERFACE", "CGI/1.1");
|
|
if (httpVersion != null) {
|
|
env.put("SERVER_PROTOCOL", httpVersion);
|
|
}
|
|
env.put("SERVER_PORT", switchboard.getConfig("port", "8090"));
|
|
env.put("REQUEST_METHOD", method);
|
|
// env.put("PATH_INFO", ""); // TODO: implement
|
|
// env.put("PATH_TRANSLATED", ""); // TODO: implement
|
|
env.put("SCRIPT_NAME", path);
|
|
if (argsString != null) {
|
|
env.put("QUERY_STRING", argsString);
|
|
}
|
|
env.put("REMOTE_ADDR", clientIP);
|
|
// env.put("AUTH_TYPE", ""); // TODO: implement
|
|
// env.put("REMOTE_USER", ""); // TODO: implement
|
|
// env.put("REMOTE_IDENT", ""); // I don't think we need this
|
|
env.put(
|
|
"DOCUMENT_ROOT",
|
|
switchboard.getAppPath().getAbsolutePath()
|
|
+ fileSeparator + switchboard.getConfig("htDocsPath", "DATA/HTDOCS"));
|
|
if (requestHeader.getContentType() != null) {
|
|
env.put("CONTENT_TYPE", requestHeader.getContentType());
|
|
}
|
|
if (method.equalsIgnoreCase(HeaderFramework.METHOD_POST)
|
|
&& body != null) {
|
|
env.put(
|
|
"CONTENT_LENGTH",
|
|
Integer.toString(requestHeader.getContentLength()));
|
|
}
|
|
|
|
/* add values from request header to environment
|
|
* (see: http://hoohoo.ncsa.uiuc.edu/cgi/env.html#headers) */
|
|
for (final Map.Entry<String, String> requestHeaderEntry
|
|
: requestHeader.entrySet()) {
|
|
env.put("HTTP_"
|
|
+ requestHeaderEntry.getKey().toUpperCase().replace("-", "_"),
|
|
requestHeaderEntry.getValue());
|
|
}
|
|
|
|
int exitValue = 0;
|
|
String cgiBody = null;
|
|
final StringBuilder error = new StringBuilder(256);
|
|
|
|
try {
|
|
// start execution of script
|
|
final Process p = pb.start();
|
|
|
|
final OutputStream os =
|
|
new BufferedOutputStream(p.getOutputStream());
|
|
|
|
if (method.equalsIgnoreCase(
|
|
HeaderFramework.METHOD_POST) && body != null) {
|
|
final byte[] buffer = new byte[1024];
|
|
int len = requestHeader.getContentLength();
|
|
while (len > 0) {
|
|
body.read(buffer);
|
|
len = len - buffer.length;
|
|
os.write(buffer);
|
|
}
|
|
}
|
|
|
|
os.close();
|
|
|
|
try {
|
|
p.waitFor();
|
|
} catch (final InterruptedException ex) {
|
|
|
|
}
|
|
|
|
exitValue = p.exitValue();
|
|
|
|
final InputStream is =
|
|
new BufferedInputStream(p.getInputStream());
|
|
|
|
final InputStream es =
|
|
new BufferedInputStream(p.getErrorStream());
|
|
|
|
final StringBuilder processOutput =
|
|
new StringBuilder(1024);
|
|
|
|
while (is.available() > 0) {
|
|
processOutput.append((char) is.read());
|
|
}
|
|
|
|
while (es.available() > 0) {
|
|
error.append((char) es.read());
|
|
}
|
|
|
|
int indexOfDelimiter = processOutput.indexOf("\n\n", 0);
|
|
final String[] cgiHeader;
|
|
if (indexOfDelimiter > -1) {
|
|
cgiHeader =
|
|
processOutput.substring(
|
|
0, indexOfDelimiter).split("\n");
|
|
} else {
|
|
cgiHeader = new String[0];
|
|
}
|
|
cgiBody = processOutput.substring(indexOfDelimiter + 1);
|
|
|
|
String key;
|
|
String value;
|
|
for (final String element : cgiHeader) {
|
|
indexOfDelimiter = element.indexOf(':');
|
|
key = element.substring(0, indexOfDelimiter).trim();
|
|
value = element.substring(indexOfDelimiter + 1).trim();
|
|
conProp.put(key, value);
|
|
if ("Cache-Control".equals(key)
|
|
&& "no-cache".equals(value)) {
|
|
nocache = true;
|
|
} else if ("Content-type".equals(key)) {
|
|
mimeType = value;
|
|
} else if ("Status".equals(key)) {
|
|
if (key.length() > 2) {
|
|
try {
|
|
statusCode =
|
|
Integer.parseInt(
|
|
value.substring(0, 3));
|
|
} catch (final NumberFormatException ex) {
|
|
Log.logWarning(
|
|
"HTTPD",
|
|
"CGI script " + targetFile.getPath()
|
|
+ " returned illegal status code \""
|
|
+ value + "\".");
|
|
}
|
|
}
|
|
}
|
|
}
|
|
} catch (final IOException ex) {
|
|
exitValue = -1;
|
|
}
|
|
|
|
/* did the script return an exit value != 0
|
|
* and still there is supposed to be
|
|
* everything right with the HTTP status?
|
|
* -> change status to 500 since 200 would
|
|
* be a lie
|
|
*/
|
|
if (exitValue != 0 && statusCode == 200) {
|
|
statusCode = 500;
|
|
}
|
|
|
|
targetDate = new Date(System.currentTimeMillis());
|
|
|
|
if (cgiBody != null && !cgiBody.isEmpty()) {
|
|
HTTPDemon.sendRespondHeader(
|
|
conProp,
|
|
out,
|
|
httpVersion,
|
|
statusCode,
|
|
null,
|
|
mimeType,
|
|
cgiBody.length(),
|
|
targetDate,
|
|
null,
|
|
null,
|
|
null,
|
|
null,
|
|
nocache);
|
|
out.write(UTF8.getBytes(cgiBody));
|
|
} else {
|
|
HTTPDemon.sendRespondError(
|
|
conProp,
|
|
out,
|
|
exitValue,
|
|
statusCode,
|
|
null,
|
|
HeaderFramework.http1_1.get(
|
|
Integer.toString(statusCode)),
|
|
null);
|
|
Log.logWarning(
|
|
"HTTPD",
|
|
"CGI script " + targetFile.getPath()
|
|
+ " returned exit value " + exitValue
|
|
+ ", body empty: "
|
|
+ (cgiBody == null || cgiBody.isEmpty()));
|
|
if (error.length() > 0) {
|
|
Log.logWarning("HTTPD", "Reported error: " + error);
|
|
}
|
|
}
|
|
}
|
|
} else if (targetClass != null && (path.endsWith(".stream") || path.indexOf('.') < 0)) {
|
|
// call rewrite-class
|
|
requestHeader.put(HeaderFramework.CONNECTION_PROP_CLIENTIP, (String) conProp.get(HeaderFramework.CONNECTION_PROP_CLIENTIP));
|
|
requestHeader.put(HeaderFramework.CONNECTION_PROP_PATH, path);
|
|
requestHeader.put(HeaderFramework.CONNECTION_PROP_EXT, path.endsWith(".stream") ? "stream" : "");
|
|
//requestHeader.put(httpHeader.CONNECTION_PROP_INPUTSTREAM, body);
|
|
//requestHeader.put(httpHeader.CONNECTION_PROP_OUTPUTSTREAM, out);
|
|
ResponseHeader header = new ResponseHeader(200);
|
|
header.put(HeaderFramework.CONTENT_TYPE, getMimeFromServlet(targetClass, requestHeader, args, "text/xml"));
|
|
conProp.remove(HeaderFramework.CONNECTION_PROP_PERSISTENT);
|
|
HTTPDemon.sendRespondHeader(conProp, out, httpVersion, 200, header);
|
|
invokeServlet(targetClass, requestHeader, args, out);
|
|
forceConnectionClose(conProp);
|
|
return;
|
|
} else if (targetFile.exists() && targetFile.isFile() && targetFile.canRead()) {
|
|
// we have found a file that can be written to the client
|
|
// if this file uses templates, then we use the template
|
|
// re-write - method to create an result
|
|
String mimeType = Classification.ext2mime(targetExt, "text/html");
|
|
String ext = (String) conProp.get("EXT"); if (ext == null) ext = "";
|
|
final boolean zipContent = requestHeader.acceptGzip() && HTTPDemon.shallTransportZipped("." + ext);
|
|
if (path.endsWith("html") ||
|
|
path.endsWith("htm") ||
|
|
path.endsWith("xml") ||
|
|
path.endsWith("json") ||
|
|
path.endsWith("rdf") ||
|
|
path.endsWith("rss") ||
|
|
path.endsWith("csv") ||
|
|
path.endsWith("pac") ||
|
|
path.endsWith("src") ||
|
|
path.endsWith("vcf") ||
|
|
path.endsWith("kml") ||
|
|
path.endsWith("gpx") ||
|
|
path.endsWith("css") ||
|
|
path.endsWith("/") ||
|
|
path.equals("/robots.txt")) {
|
|
|
|
/*targetFile = getLocalizedFile(path);
|
|
if (!(targetFile.exists())) {
|
|
// try to find that file in the htDocsPath
|
|
File trialFile = new File(htDocsPath, path);
|
|
if (trialFile.exists()) targetFile = trialFile;
|
|
}*/
|
|
|
|
|
|
// call rewrite-class
|
|
|
|
if (targetClass != null) {
|
|
requestHeader.put(HeaderFramework.CONNECTION_PROP_CLIENTIP, (String) conProp.get(HeaderFramework.CONNECTION_PROP_CLIENTIP));
|
|
requestHeader.put(HeaderFramework.CONNECTION_PROP_PATH, path);
|
|
final int ep = path.lastIndexOf(".");
|
|
requestHeader.put(HeaderFramework.CONNECTION_PROP_EXT, path.substring(ep + 1));
|
|
// in case that there are no args given, args = null or empty hashmap
|
|
final Object tmp = invokeServlet(targetClass, requestHeader, args, null);
|
|
if (tmp == null) {
|
|
// if no args given, then tp will be an empty Hashtable object (not null)
|
|
templatePatterns = new servletProperties();
|
|
} else if (tmp instanceof servletProperties) {
|
|
templatePatterns = (servletProperties) tmp;
|
|
} else {
|
|
templatePatterns = new servletProperties((serverObjects) tmp);
|
|
}
|
|
// check if the servlets requests authentication
|
|
if (templatePatterns.containsKey(serverObjects.ACTION_AUTHENTICATE)) {
|
|
// handle brute-force protection
|
|
if (realmProp != null) {
|
|
Log.logInfo("HTTPD", "dynamic log-in for account 'admin' in http file handler for path '" + path + "' from host '" + clientIP + "'");
|
|
final Integer attempts = serverCore.bfHost.get(clientIP);
|
|
if (attempts == null)
|
|
serverCore.bfHost.put(clientIP, Integer.valueOf(1));
|
|
else
|
|
serverCore.bfHost.put(clientIP, Integer.valueOf(attempts.intValue() + 1));
|
|
}
|
|
// send authentication request to browser
|
|
final ResponseHeader headers = getDefaultHeaders(path);
|
|
headers.put(RequestHeader.WWW_AUTHENTICATE,"Basic realm=\"" + templatePatterns.get(serverObjects.ACTION_AUTHENTICATE, "") + "\"");
|
|
HTTPDemon.sendRespondHeader(conProp,out,httpVersion,401,headers);
|
|
return;
|
|
} else if (templatePatterns.containsKey(serverObjects.ACTION_LOCATION)) {
|
|
String location = templatePatterns.get(serverObjects.ACTION_LOCATION, "");
|
|
if (location.isEmpty()) location = path;
|
|
|
|
final ResponseHeader headers = getDefaultHeaders(path);
|
|
headers.setAdditionalHeaderProperties(templatePatterns.getOutgoingHeader().getAdditionalHeaderProperties()); //put the cookies into the new header TODO: can we put all headerlines, without trouble?
|
|
headers.put(HeaderFramework.LOCATION,location);
|
|
HTTPDemon.sendRespondHeader(conProp,out,httpVersion,302,headers);
|
|
return;
|
|
}
|
|
// add the application version, the uptime and the client name to every rewrite table
|
|
templatePatterns.put(servletProperties.PEER_STAT_VERSION, yacyBuildProperties.getVersion());
|
|
templatePatterns.put(servletProperties.PEER_STAT_UPTIME, ((System.currentTimeMillis() - serverCore.startupTime) / 1000) / 60); // uptime in minutes
|
|
templatePatterns.putHTML(servletProperties.PEER_STAT_CLIENTNAME, sb.peers.mySeed().getName());
|
|
templatePatterns.putHTML(servletProperties.PEER_STAT_CLIENTID, ((Switchboard) switchboard).peers.myID());
|
|
templatePatterns.put(servletProperties.PEER_STAT_MYTIME, GenericFormatter.SHORT_SECOND_FORMATTER.format());
|
|
final Seed myPeer = sb.peers.mySeed();
|
|
templatePatterns.put("newpeer", myPeer.getAge() >= 1 ? 0 : 1);
|
|
templatePatterns.putHTML("newpeer_peerhash", myPeer.hash);
|
|
//System.out.println("respond props: " + ((tp == null) ? "null" : tp.toString())); // debug
|
|
nocache = true;
|
|
}
|
|
|
|
targetDate = new Date(targetFile.lastModified());
|
|
Date expireDate = null;
|
|
if (templatePatterns == null) {
|
|
// if the file will not be changed, cache it in the browser
|
|
expireDate = new Date(new Date().getTime() + (31l * 24 * 60 * 60 * 1000));
|
|
}
|
|
|
|
|
|
// rewrite the file
|
|
InputStream fis = null;
|
|
|
|
// read the file/template
|
|
TemplateCacheEntry templateCacheEntry = null;
|
|
final long fileSize = targetFile.length();
|
|
if (useTemplateCache && fileSize <= 512 * 1024) {
|
|
// read from cache
|
|
SoftReference<TemplateCacheEntry> ref = templateCache.get(targetFile);
|
|
if (ref != null) {
|
|
templateCacheEntry = ref.get();
|
|
if (templateCacheEntry == null) templateCache.remove(targetFile);
|
|
}
|
|
|
|
final Date targetFileDate = new Date(targetFile.lastModified());
|
|
if (templateCacheEntry == null || targetFileDate.after(templateCacheEntry.lastModified)) {
|
|
// loading the content of the template file into
|
|
// a byte array
|
|
templateCacheEntry = new TemplateCacheEntry();
|
|
templateCacheEntry.lastModified = targetFileDate;
|
|
templateCacheEntry.content = FileUtils.read(targetFile);
|
|
|
|
// storing the content into the cache
|
|
ref = new SoftReference<TemplateCacheEntry>(templateCacheEntry);
|
|
if (MemoryControl.shortStatus()) templateCache.clear();
|
|
templateCache.put(targetFile, ref);
|
|
if (theLogger.isFinest()) theLogger.logFinest("Cache MISS for file " + targetFile);
|
|
} else {
|
|
if (theLogger.isFinest()) theLogger.logFinest("Cache HIT for file " + targetFile);
|
|
}
|
|
|
|
// creating an inputstream needed by the template
|
|
// rewrite function
|
|
fis = new ByteArrayInputStream(templateCacheEntry.content);
|
|
templateCacheEntry = null;
|
|
} else if (fileSize <= Math.min(4 * 1024 * 1204, MemoryControl.available() / 100)) {
|
|
// read file completely into ram, avoid that too many files are open at the same time
|
|
fis = new ByteArrayInputStream(FileUtils.read(targetFile));
|
|
} else {
|
|
fis = new BufferedInputStream(new FileInputStream(targetFile));
|
|
}
|
|
|
|
if (mimeType.startsWith("text")) {
|
|
// every text-file distributed by yacy is UTF-8
|
|
if (!path.startsWith("/repository")) {
|
|
mimeType = mimeType + "; charset=UTF-8";
|
|
} else {
|
|
// detect charset of html-files
|
|
if ((path.endsWith("html") || path.endsWith("htm"))) {
|
|
// save position
|
|
fis.mark(1000);
|
|
// scrape document to look up charset
|
|
final ScraperInputStream htmlFilter = new ScraperInputStream(fis, "UTF-8", new DigestURI("http://localhost"), null, false, 10);
|
|
final String charset = htmlParser.patchCharsetEncoding(htmlFilter.detectCharset());
|
|
htmlFilter.close();
|
|
if (charset != null) mimeType = mimeType + "; charset="+charset;
|
|
// reset position
|
|
fis.reset();
|
|
}
|
|
}
|
|
}
|
|
|
|
// write the array to the client
|
|
// we can do that either in standard mode (whole thing completely) or in chunked mode
|
|
// since yacy clients do not understand chunked mode (yet), we use this only for communication with the administrator
|
|
final boolean yacyClient = requestHeader.userAgent().startsWith("yacy");
|
|
final boolean chunked = !method.equals(HeaderFramework.METHOD_HEAD) && !yacyClient && httpVersion.equals(HeaderFramework.HTTP_VERSION_1_1);
|
|
if (chunked) {
|
|
// send page in chunks and parse SSIs
|
|
final ByteBuffer o = new ByteBuffer();
|
|
// apply templates
|
|
TemplateEngine.writeTemplate(fis, o, templatePatterns, UNRESOLVED_PATTERN);
|
|
fis.close();
|
|
ResponseHeader rh = (templatePatterns == null) ? new ResponseHeader(200) : templatePatterns.getOutgoingHeader();
|
|
HTTPDemon.sendRespondHeader(conProp, out,
|
|
httpVersion, rh.getStatusCode(), null, mimeType, -1,
|
|
targetDate, expireDate, rh,
|
|
null, "chunked", nocache);
|
|
// send the content in chunked parts, see RFC 2616 section 3.6.1
|
|
final ChunkedOutputStream chos = new ChunkedOutputStream(out);
|
|
// GZIPOutputStream does not implement flush (this is a bug IMHO)
|
|
// so we can't compress this stuff, without loosing the cool SSI trickle feature
|
|
ServerSideIncludes.writeSSI(o, chos, realmProp, clientIP, requestHeader);
|
|
//chos.write(result);
|
|
chos.finish();
|
|
} else {
|
|
// send page as whole thing, SSIs are not possible
|
|
final String contentEncoding = (zipContent) ? "gzip" : null;
|
|
// apply templates
|
|
final ByteBuffer o1 = new ByteBuffer();
|
|
TemplateEngine.writeTemplate(fis, o1, templatePatterns, ASCII.getBytes("-UNRESOLVED_PATTERN-"));
|
|
fis.close();
|
|
final ByteBuffer o = new ByteBuffer();
|
|
|
|
if (zipContent) {
|
|
GZIPOutputStream zippedOut = new GZIPOutputStream(o);
|
|
ServerSideIncludes.writeSSI(o1, zippedOut, realmProp, clientIP, requestHeader);
|
|
//httpTemplate.writeTemplate(fis, zippedOut, tp, "-UNRESOLVED_PATTERN-".getBytes("UTF-8"));
|
|
zippedOut.finish();
|
|
zippedOut.flush();
|
|
zippedOut.close();
|
|
zippedOut = null;
|
|
} else {
|
|
ServerSideIncludes.writeSSI(o1, o, realmProp, clientIP, requestHeader);
|
|
//httpTemplate.writeTemplate(fis, o, tp, "-UNRESOLVED_PATTERN-".getBytes("UTF-8"));
|
|
}
|
|
ResponseHeader rh = (templatePatterns == null) ? new ResponseHeader(200) : templatePatterns.getOutgoingHeader();
|
|
if (method.equals(HeaderFramework.METHOD_HEAD)) {
|
|
HTTPDemon.sendRespondHeader(conProp, out,
|
|
httpVersion, rh.getStatusCode(), null, mimeType, o.length(),
|
|
targetDate, expireDate, rh,
|
|
contentEncoding, null, nocache);
|
|
} else {
|
|
final byte[] result = o.getBytes(); // this interrupts streaming (bad idea!)
|
|
HTTPDemon.sendRespondHeader(conProp, out,
|
|
httpVersion, rh.getStatusCode(), null, mimeType, result.length,
|
|
targetDate, expireDate, rh,
|
|
contentEncoding, null, nocache);
|
|
FileUtils.copy(result, out);
|
|
}
|
|
}
|
|
} else { // no html
|
|
|
|
int statusCode = 200;
|
|
int rangeStartOffset = 0;
|
|
final ResponseHeader header = new ResponseHeader(statusCode);
|
|
|
|
// adding the accept ranges header
|
|
header.put(HeaderFramework.ACCEPT_RANGES, "bytes");
|
|
|
|
// reading the files md5 hash if availabe and use it as ETAG of the resource
|
|
String targetMD5 = null;
|
|
final File targetMd5File = new File(targetFile + ".md5");
|
|
try {
|
|
if (targetMd5File.exists()) {
|
|
//String description = null;
|
|
targetMD5 = UTF8.String(FileUtils.read(targetMd5File));
|
|
final int pos = targetMD5.indexOf('\n');
|
|
if (pos >= 0) {
|
|
//description = targetMD5.substring(pos + 1);
|
|
targetMD5 = targetMD5.substring(0, pos);
|
|
}
|
|
|
|
// using the checksum as ETAG header
|
|
header.put(HeaderFramework.ETAG, targetMD5);
|
|
}
|
|
} catch (final IOException e) {
|
|
Log.logException(e);
|
|
}
|
|
|
|
if (requestHeader.containsKey(HeaderFramework.RANGE)) {
|
|
final Object ifRange = requestHeader.ifRange();
|
|
if ((ifRange == null)||
|
|
(ifRange instanceof Date && targetFile.lastModified() == ((Date)ifRange).getTime()) ||
|
|
(ifRange instanceof String && ifRange.equals(targetMD5))) {
|
|
final String rangeHeaderVal = requestHeader.get(HeaderFramework.RANGE).trim();
|
|
if (rangeHeaderVal.startsWith("bytes=")) {
|
|
final String rangesVal = rangeHeaderVal.substring("bytes=".length());
|
|
final String[] ranges = rangesVal.split(",");
|
|
if ((ranges.length == 1)&&(ranges[0].endsWith("-"))) {
|
|
rangeStartOffset = NumberTools.parseIntDecSubstring(ranges[0], 0, ranges[0].length() - 1);
|
|
statusCode = 206;
|
|
header.put(HeaderFramework.CONTENT_RANGE, "bytes " + rangeStartOffset + "-" + (targetFile.length()-1) + "/" + targetFile.length());
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// write the file to the client
|
|
targetDate = new Date(targetFile.lastModified());
|
|
// cache file for one month in browser (but most browsers won't cache for that long)
|
|
final Date expireDate = new Date(new Date().getTime() + (31l * 24 * 60 * 60 * 1000));
|
|
final long contentLength = (zipContent)?-1:targetFile.length()-rangeStartOffset;
|
|
final String contentEncoding = (zipContent) ? "gzip" : null;
|
|
final String transferEncoding = (httpVersion.equals(HeaderFramework.HTTP_VERSION_1_1) && zipContent) ? "chunked" : null;
|
|
if (!httpVersion.equals(HeaderFramework.HTTP_VERSION_1_1) && zipContent) forceConnectionClose(conProp);
|
|
|
|
HTTPDemon.sendRespondHeader(conProp, out, httpVersion, statusCode, null, mimeType, contentLength, targetDate, expireDate, header, contentEncoding, transferEncoding, nocache);
|
|
|
|
if (!method.equals(HeaderFramework.METHOD_HEAD)) {
|
|
ChunkedOutputStream chunkedOut = null;
|
|
GZIPOutputStream zipped = null;
|
|
OutputStream newOut = out;
|
|
|
|
if (transferEncoding != null) {
|
|
chunkedOut = new ChunkedOutputStream(newOut);
|
|
newOut = chunkedOut;
|
|
}
|
|
if (contentEncoding != null) {
|
|
zipped = new GZIPOutputStream(newOut);
|
|
newOut = zipped;
|
|
}
|
|
|
|
FileUtils.copyRange(targetFile, newOut, rangeStartOffset);
|
|
|
|
if (zipped != null) {
|
|
zipped.flush();
|
|
zipped.finish();
|
|
}
|
|
if (chunkedOut != null) {
|
|
chunkedOut.finish();
|
|
}
|
|
|
|
// flush all
|
|
try {newOut.flush();}catch (final Exception e) {}
|
|
}
|
|
}
|
|
} else {
|
|
HTTPDemon.sendRespondError(conProp,out,3,404,"File not Found",null,null);
|
|
return;
|
|
}
|
|
} catch (final Exception e) {
|
|
try {
|
|
// error handling
|
|
if (e instanceof NullPointerException) {
|
|
Log.logException(e);
|
|
}
|
|
int httpStatusCode = 400;
|
|
final String httpStatusText = null;
|
|
final StringBuilder errorMessage = new StringBuilder(2000);
|
|
Exception errorExc = null;
|
|
|
|
final String errorMsg = e.getMessage();
|
|
if (
|
|
(e instanceof InterruptedException) ||
|
|
((errorMsg != null) && (errorMsg.startsWith("Socket closed")) && (Thread.currentThread().isInterrupted()))
|
|
) {
|
|
errorMessage.append("Interruption detected while processing query.");
|
|
httpStatusCode = 503;
|
|
} else {
|
|
if ((errorMsg != null) &&
|
|
(
|
|
errorMsg.contains("Broken pipe") ||
|
|
errorMsg.contains("Connection reset") ||
|
|
errorMsg.contains("Read timed out") ||
|
|
errorMsg.contains("Connection timed out") ||
|
|
errorMsg.contains("Software caused connection abort")
|
|
)) {
|
|
// client closed the connection, so we just end silently
|
|
errorMessage.append("Client unexpectedly closed connection while processing query.");
|
|
} else {
|
|
errorMessage.append("Unexpected error while processing query.");
|
|
httpStatusCode = 500;
|
|
errorExc = e;
|
|
}
|
|
}
|
|
|
|
errorMessage.append("\nSession: ").append(Thread.currentThread().getName())
|
|
.append("\nQuery: ").append(path)
|
|
.append("\nClient: ").append(conProp.get(HeaderFramework.CONNECTION_PROP_CLIENTIP))
|
|
.append("\nReason: ").append(e.getMessage());
|
|
|
|
if (!conProp.containsKey(HeaderFramework.CONNECTION_PROP_PROXY_RESPOND_HEADER)) {
|
|
// sending back an error message to the client
|
|
// if we have not already send an http header
|
|
HTTPDemon.sendRespondError(conProp,out, 4, httpStatusCode, httpStatusText, errorMessage.toString(), errorExc);
|
|
} else {
|
|
// otherwise we close the connection
|
|
forceConnectionClose(conProp);
|
|
}
|
|
|
|
// if it is an unexpected error we log it
|
|
if (httpStatusCode == 500) {
|
|
theLogger.logWarning(errorMessage.toString(), e);
|
|
}
|
|
|
|
} catch (final Exception ee) {
|
|
forceConnectionClose(conProp);
|
|
}
|
|
|
|
} finally {
|
|
try {out.flush();}catch (final Exception e) {}
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Returns a list which contains parts of command
|
|
* which is used to start external process for
|
|
* CGI scripts.
|
|
* @param targetFile file to run
|
|
* @return list of parts of command
|
|
* @throws FileNotFoundException
|
|
* @throws IOException if file can not be accessed
|
|
*/
|
|
private static List<String> assembleCommandFromShebang(final File targetFile) throws FileNotFoundException {
|
|
final List<String > ret = new ArrayList<String>();
|
|
final BufferedReader br = new BufferedReader(new FileReader(targetFile), 512);
|
|
String line;
|
|
try {
|
|
line = br.readLine();
|
|
if (line.startsWith("#!")) {
|
|
ret.addAll(Arrays.asList(line.substring(2).split(" ")));
|
|
}
|
|
ret.add(targetFile.getAbsolutePath());
|
|
} catch (IOException e) {
|
|
Log.logException(e);
|
|
} finally {
|
|
try {br.close();} catch (IOException e) {}
|
|
}
|
|
return ret;
|
|
}
|
|
|
|
private static final String appendPath(final String proplist, final String path) {
|
|
if (proplist.isEmpty()) return path;
|
|
if (proplist.indexOf(path) >= 0) return proplist;
|
|
return proplist + "," + path;
|
|
}
|
|
|
|
public static final File getOverlayedClass(final String path) {
|
|
File targetClass;
|
|
targetClass = rewriteClassFile(new File(htDefaultPath, path)); //works for default and localized files
|
|
if (targetClass == null || !targetClass.exists()) {
|
|
//works for htdocs
|
|
targetClass=rewriteClassFile(new File(htDocsPath, path));
|
|
}
|
|
return targetClass;
|
|
}
|
|
|
|
public static final File getOverlayedFile(final String path) {
|
|
File targetFile;
|
|
targetFile = getLocalizedFile(path);
|
|
if (!targetFile.exists()) {
|
|
targetFile = new File(htDocsPath, path);
|
|
}
|
|
return targetFile;
|
|
}
|
|
|
|
private static final void forceConnectionClose(final HashMap<String, Object> conprop) {
|
|
if (conprop != null) {
|
|
conprop.put(HeaderFramework.CONNECTION_PROP_PERSISTENT,"close");
|
|
}
|
|
}
|
|
|
|
private static final File rewriteClassFile(final File template) {
|
|
try {
|
|
String f = template.getCanonicalPath();
|
|
final int p = f.lastIndexOf('.');
|
|
f = p < 0 ? f + ".class" : f.substring(0, p) + ".class";
|
|
final File cf = new File(f);
|
|
if (cf.exists()) return cf;
|
|
return null;
|
|
} catch (final IOException e) {
|
|
return null;
|
|
}
|
|
}
|
|
|
|
private static final Method rewriteMethod(final File classFile, final String methodName) throws InvocationTargetException {
|
|
Method m = null;
|
|
// now make a class out of the stream
|
|
try {
|
|
if (templateMethodCache != null && "respond".equals(methodName)) {
|
|
final SoftReference<Method> ref = templateMethodCache.get(classFile);
|
|
if (ref != null) {
|
|
m = ref.get();
|
|
if (m == null) {
|
|
templateMethodCache.remove(classFile);
|
|
} else {
|
|
return m;
|
|
}
|
|
}
|
|
}
|
|
|
|
final Class<?> c = provider.loadClass(classFile);
|
|
Class<?>[] params = new Class[] {
|
|
RequestHeader.class,
|
|
serverObjects.class,
|
|
serverSwitch.class };
|
|
try {
|
|
m = c.getMethod(methodName, params);
|
|
} catch (NoSuchMethodException e) {
|
|
params = new Class[] {
|
|
RequestHeader.class,
|
|
serverObjects.class,
|
|
serverSwitch.class,
|
|
OutputStream.class};
|
|
m = c.getMethod(methodName, params);
|
|
}
|
|
|
|
if (MemoryControl.shortStatus()) {
|
|
templateMethodCache.clear();
|
|
} else {
|
|
// store the method into the cache
|
|
if (templateMethodCache != null && "respond".equals(methodName)) {
|
|
templateMethodCache.put(classFile, new SoftReference<Method>(m));
|
|
}
|
|
}
|
|
|
|
} catch (final ClassNotFoundException e) {
|
|
Log.logSevere("HTTPDFileHandler", "class " + classFile + " is missing:" + e.getMessage());
|
|
throw new InvocationTargetException(e, "class " + classFile + " is missing:" + e.getMessage());
|
|
} catch (final NoSuchMethodException e) {
|
|
Log.logSevere("HTTPDFileHandler", "method 'respond' not found in class " + classFile + ": " + e.getMessage());
|
|
throw new InvocationTargetException(e, "method 'respond' not found in class " + classFile + ": " + e.getMessage());
|
|
}
|
|
return m;
|
|
}
|
|
|
|
private static final Object invokeServlet(final File targetClass, final RequestHeader request, final serverObjects args, final OutputStream os) {
|
|
try {
|
|
if (os == null) {
|
|
return rewriteMethod(targetClass, "respond").invoke(null, new Object[] {request, args, switchboard});
|
|
}
|
|
return rewriteMethod(targetClass, "respond").invoke(null, new Object[] {request, args, switchboard, os});
|
|
} catch (final Throwable e) {
|
|
theLogger.logSevere("INTERNAL ERROR: " + e.toString() + ":" +
|
|
e.getMessage() +
|
|
" target exception at " + targetClass + ": " +
|
|
"; java.awt.graphicsenv='" + System.getProperty("java.awt.graphicsenv","") + "'");
|
|
Log.logException(e);
|
|
Log.logException(e.getCause());
|
|
if (e instanceof InvocationTargetException) Log.logException(((InvocationTargetException) e).getTargetException());
|
|
return null;
|
|
}
|
|
}
|
|
|
|
private static final String getMimeFromServlet(final File targetClass, final RequestHeader request, final serverObjects args, final String dflt) {
|
|
try {
|
|
return (String) rewriteMethod(targetClass, "mime").invoke(null, new Object[] {request, args, switchboard});
|
|
} catch (final Throwable e) {
|
|
theLogger.logSevere("INTERNAL ERROR: " + e.toString() + ":" +
|
|
e.getMessage() +
|
|
" target exception at " + targetClass + ": " +
|
|
"; java.awt.graphicsenv='" + System.getProperty("java.awt.graphicsenv","") + "'");
|
|
Log.logException(e);
|
|
Log.logException(e.getCause());
|
|
if (e instanceof InvocationTargetException) Log.logException(((InvocationTargetException) e).getTargetException());
|
|
return dflt;
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Tells if a filename ends with a suffix from a given list.
|
|
* @param filename the filename
|
|
* @param suffixList the list of suffixes which is a string of suffixes separated by commas
|
|
* @return true if the filename ends with a suffix from the list, else false
|
|
*/
|
|
private static boolean matchesSuffix(final String name, final String suffixList) {
|
|
boolean ret = false;
|
|
|
|
if (suffixList != null && name != null) {
|
|
final String[] suffixes = suffixList.split(",");
|
|
find:
|
|
for (int i = 0; i < suffixes.length; i++) {
|
|
if (name.endsWith("." + suffixes[i].trim())) {
|
|
ret = true;
|
|
break find;
|
|
}
|
|
}
|
|
}
|
|
|
|
return ret;
|
|
}
|
|
|
|
/**
|
|
* do a proxy request for document
|
|
* extracts url from GET-parameter url
|
|
* not in separete servlet, because we need access to binary outstream
|
|
* @throws IOException
|
|
*/
|
|
private static void doURLProxy(final HashMap<String, Object> conProp, final RequestHeader requestHeader, final OutputStream out) throws IOException {
|
|
final String httpVersion = (String) conProp.get(HeaderFramework.CONNECTION_PROP_HTTP_VER);
|
|
URL proxyurl = null;
|
|
String action = "";
|
|
|
|
if(conProp != null && conProp.containsKey("ARGS")) {
|
|
String strARGS = (String) conProp.get("ARGS");
|
|
if(strARGS.startsWith("action=")) {
|
|
int detectnextargument = strARGS.indexOf("&");
|
|
action = strARGS.substring (7, detectnextargument);
|
|
strARGS = strARGS.substring(detectnextargument+1);
|
|
}
|
|
if(strARGS.startsWith("url=")) {
|
|
final String strUrl = strARGS.substring(4); // strip url=
|
|
|
|
try {
|
|
proxyurl = new URL(strUrl);
|
|
} catch (MalformedURLException e) {
|
|
proxyurl = new URL (URLDecoder.decode(strUrl, UTF8.charset.name()));
|
|
|
|
}
|
|
}
|
|
}
|
|
|
|
if (proxyurl==null) {
|
|
throw new IOException("no url as argument supplied");
|
|
}
|
|
String host = proxyurl.getHost();
|
|
if (proxyurl.getPort() != -1) {
|
|
host += ":" + proxyurl.getPort();
|
|
}
|
|
|
|
// set properties for proxy connection
|
|
final HashMap<String, Object> prop = new HashMap<String, Object>();
|
|
prop.put(HeaderFramework.CONNECTION_PROP_HTTP_VER, HeaderFramework.HTTP_VERSION_1_1);
|
|
prop.put(HeaderFramework.CONNECTION_PROP_HOST, host);
|
|
prop.put(HeaderFramework.CONNECTION_PROP_PATH, proxyurl.getFile().replaceAll(" ", "%20"));
|
|
prop.put(HeaderFramework.CONNECTION_PROP_REQUESTLINE, "PROXY");
|
|
prop.put("CLIENTIP", "0:0:0:0:0:0:0:1");
|
|
|
|
// remove some stuff from request header, so it isn't send to the server
|
|
requestHeader.remove("CLIENTIP");
|
|
requestHeader.remove("EXT");
|
|
requestHeader.remove("PATH");
|
|
requestHeader.remove("Authorization");
|
|
requestHeader.remove("Connection");
|
|
requestHeader.put(HeaderFramework.HOST, proxyurl.getHost());
|
|
|
|
// temporarily add argument to header to pass it on to augmented browsing
|
|
requestHeader.put("YACYACTION", action);
|
|
|
|
final ByteArrayOutputStream o = new ByteArrayOutputStream();
|
|
HTTPDProxyHandler.doGet(prop, requestHeader, o);
|
|
|
|
// reparse header to extract content-length and mimetype
|
|
final ResponseHeader outgoingHeader = new ResponseHeader(200);
|
|
final InputStream in = new ByteArrayInputStream(o.toByteArray());
|
|
String line = readLine(in);
|
|
while(line != null && !line.equals("")) {
|
|
int p;
|
|
if ((p = line.indexOf(':')) >= 0) {
|
|
// store a property
|
|
outgoingHeader.add(line.substring(0, p).trim(), line.substring(p + 1).trim());
|
|
}
|
|
line = readLine(in);
|
|
}
|
|
if (line==null) {
|
|
HTTPDemon.sendRespondError(conProp,out,3,500,"null",null,null);
|
|
return;
|
|
}
|
|
|
|
final int httpStatus = Integer.parseInt((String) prop.get(HeaderFramework.CONNECTION_PROP_PROXY_RESPOND_STATUS));
|
|
|
|
String directory = "";
|
|
if (proxyurl.getPath().lastIndexOf('/') > 0)
|
|
directory = proxyurl.getPath().substring(0, proxyurl.getPath().lastIndexOf('/'));
|
|
|
|
String location = "";
|
|
|
|
if (outgoingHeader.containsKey("Location")) {
|
|
// rewrite location header
|
|
location = outgoingHeader.get("Location");
|
|
if (location.startsWith("http")) {
|
|
location = "/proxy.html?action="+action+"&url=" + location;
|
|
} else {
|
|
location = "/proxy.html?action="+action+"&url=http://" + proxyurl.getHost() + "/" + location;
|
|
}
|
|
outgoingHeader.put("Location", location);
|
|
}
|
|
|
|
final String mimeType = outgoingHeader.getContentType();
|
|
if ((mimeType.startsWith("text/html") || mimeType.startsWith("text"))) {
|
|
final StringWriter buffer = new StringWriter();
|
|
|
|
if (outgoingHeader.containsKey(HeaderFramework.TRANSFER_ENCODING)) {
|
|
FileUtils.copy(new ChunkedInputStream(in), buffer, UTF8.charset);
|
|
} else {
|
|
FileUtils.copy(in, buffer, UTF8.charset);
|
|
}
|
|
|
|
final String sbuffer = buffer.toString();
|
|
|
|
final Pattern p = Pattern.compile("(href=\"|src=\")([^\"]+)|(href='|src=')([^']+)|(url\\(')([^']+)|(url\\(\")([^\"]+)|(url\\()([^\\)]+)");
|
|
final Matcher m = p.matcher(sbuffer);
|
|
final StringBuffer result = new StringBuffer(80);
|
|
String init, url;
|
|
MultiProtocolURI target;
|
|
while (m.find()) {
|
|
init = null;
|
|
if(m.group(1) != null) init = m.group(1);
|
|
if(m.group(3) != null) init = m.group(3);
|
|
if(m.group(5) != null) init = m.group(5);
|
|
if(m.group(7) != null) init = m.group(7);
|
|
if(m.group(9) != null) init = m.group(9);
|
|
url = null;
|
|
if(m.group(2) != null) url = m.group(2);
|
|
if(m.group(4) != null) url = m.group(4);
|
|
if(m.group(6) != null) url = m.group(6);
|
|
if(m.group(8) != null) url = m.group(8);
|
|
if(m.group(10) != null) url = m.group(10);
|
|
if (url.startsWith("data:") || url.startsWith("#") || url.startsWith("mailto:") || url.startsWith("javascript:")) {
|
|
String newurl = init + url;
|
|
newurl = newurl.replaceAll("\\$","\\\\\\$");
|
|
m.appendReplacement(result, newurl);
|
|
|
|
} else if (url.startsWith("http")) {
|
|
// absoulte url of form href="http://domain.com/path"
|
|
if (sb.getConfig("proxyURL.rewriteURLs", "all").equals("domainlist")) {
|
|
if (sb.crawlStacker.urlInAcceptedDomain(new DigestURI(url)) != null) {
|
|
continue;
|
|
}
|
|
}
|
|
|
|
String newurl = init + "/proxy.html?url=" + url;
|
|
newurl = newurl.replaceAll("\\$","\\\\\\$");
|
|
m.appendReplacement(result, newurl);
|
|
|
|
} else if (url.startsWith("//")) {
|
|
// absoulte url but same protocol of form href="//domain.com/path"
|
|
final String complete_url = proxyurl.getProtocol() + ":" + url;
|
|
if (sb.getConfig("proxyURL.rewriteURLs", "all").equals("domainlist")) {
|
|
if (sb.crawlStacker.urlInAcceptedDomain(new DigestURI(complete_url)) != null) {
|
|
continue;
|
|
}
|
|
}
|
|
|
|
String newurl = init + "/proxy.html?url=" + complete_url;
|
|
newurl = newurl.replaceAll("\\$","\\\\\\$");
|
|
m.appendReplacement(result, newurl);
|
|
|
|
} else if (url.startsWith("/")) {
|
|
// absolute path of form href="/absolute/path/to/linked/page"
|
|
String newurl = init + "/proxy.html?url=http://" + host + url;
|
|
newurl = newurl.replaceAll("\\$","\\\\\\$");
|
|
m.appendReplacement(result, newurl);
|
|
|
|
} else {
|
|
// relative path of form href="relative/path"
|
|
try {
|
|
target = new MultiProtocolURI("http://" + host + directory + "/" + url);
|
|
String newurl = init + "/proxy.html?url=" + target.toString();
|
|
newurl = newurl.replaceAll("\\$","\\\\\\$");
|
|
m.appendReplacement(result, newurl);
|
|
}
|
|
catch (final MalformedURLException e) {}
|
|
|
|
}
|
|
}
|
|
m.appendTail(result);
|
|
|
|
final byte[] sbb = UTF8.getBytes(result.toString());
|
|
|
|
if (outgoingHeader.containsKey(HeaderFramework.TRANSFER_ENCODING)) {
|
|
HTTPDemon.sendRespondHeader(conProp, out, httpVersion, httpStatus, outgoingHeader);
|
|
final ChunkedOutputStream cos = new ChunkedOutputStream(out);
|
|
cos.write(sbb);
|
|
cos.finish();
|
|
cos.close();
|
|
} else {
|
|
outgoingHeader.put(HeaderFramework.CONTENT_LENGTH, Integer.toString(sbb.length));
|
|
HTTPDemon.sendRespondHeader(conProp, out, httpVersion, httpStatus, outgoingHeader);
|
|
out.write(sbb);
|
|
}
|
|
} else {
|
|
if (!outgoingHeader.containsKey(HeaderFramework.CONTENT_LENGTH))
|
|
outgoingHeader.put(HeaderFramework.CONTENT_LENGTH, (String) prop.get(HeaderFramework.CONNECTION_PROP_PROXY_RESPOND_SIZE));
|
|
HTTPDemon.sendRespondHeader(conProp, out, httpVersion, httpStatus, outgoingHeader);
|
|
FileUtils.copy(in, out);
|
|
}
|
|
return;
|
|
}
|
|
|
|
private static String readLine(final InputStream in) throws IOException {
|
|
final ByteArrayOutputStream buf = new ByteArrayOutputStream();
|
|
int b;
|
|
while ((b=in.read()) != '\r' && b != -1) {
|
|
buf.write(b);
|
|
}
|
|
if (b == -1) return null;
|
|
b = in.read(); // read \n
|
|
if (b == -1) return null;
|
|
return buf.toString("UTF-8");
|
|
}
|
|
|
|
}
|