mirror of
https://github.com/yacy/yacy_search_server.git
synced 2024-09-21 00:00:13 +02:00
711641f167
there are now two time-outs, one for the complete connection time, and one for an idle time connections that are idle for more than 2 minutes are closed, and connections that are alive since more than one hour are also closed if the complete number of connections exceeds 64, all connections more than 64 and have most idle time are also closed During normal operation of peers these forced closings should never appear, but the existence of the idle connection check ensures the availability of the peer and the usability of the host. git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@4134 6c8d7289-2bf4-0310-a012-ef5d649a1542
1607 lines
64 KiB
Java
1607 lines
64 KiB
Java
// httpc.java
|
|
// -------------------------------------
|
|
// (C) by Michael Peter Christen; mc@anomic.de
|
|
// first published on http://www.anomic.de
|
|
// Frankfurt, Germany, 2004
|
|
// last major change: 26.02.2004
|
|
//
|
|
// This program is free software; you can redistribute it and/or modify
|
|
// it under the terms of the GNU General Public License as published by
|
|
// the Free Software Foundation; either version 2 of the License, or
|
|
// (at your option) any later version.
|
|
//
|
|
// This program is distributed in the hope that it will be useful,
|
|
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
// GNU General Public License for more details.
|
|
//
|
|
// You should have received a copy of the GNU General Public License
|
|
// along with this program; if not, write to the Free Software
|
|
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
|
//
|
|
// Using this software in any meaning (reading, learning, copying, compiling,
|
|
// running) means that you agree that the Author(s) is (are) not responsible
|
|
// for cost, loss of data or any harm that may be caused directly or indirectly
|
|
// by usage of this softare or this documentation. The usage of this software
|
|
// is on your own risk. The installation and usage (starting/running) of this
|
|
// software may allow other people or application to access your computer and
|
|
// any attached devices and is highly dependent on the configuration of the
|
|
// software which must be done by the user of the software; the author(s) is
|
|
// (are) also not responsible for proper configuration and usage of the
|
|
// software, even if provoked by documentation provided together with
|
|
// the software.
|
|
//
|
|
// Any changes to this file according to the GPL as documented in the file
|
|
// gpl.txt aside this file in the shipment you received can be done to the
|
|
// lines that follows this copyright notice here, but changes must not be
|
|
// done inside the copyright notive above. A re-distribution must contain
|
|
// the intact and unchanged copyright notice.
|
|
// Contributions and changes to the program code must be marked as such.
|
|
|
|
package de.anomic.http;
|
|
|
|
import java.io.File;
|
|
import java.io.FileNotFoundException;
|
|
import java.io.FileOutputStream;
|
|
import java.io.IOException;
|
|
import java.io.InputStream;
|
|
import java.io.InputStreamReader;
|
|
import java.io.OutputStream;
|
|
import java.io.OutputStreamWriter;
|
|
import java.io.PushbackInputStream;
|
|
import java.io.Writer;
|
|
import java.net.InetAddress;
|
|
import java.net.InetSocketAddress;
|
|
import java.net.MalformedURLException;
|
|
import java.net.Socket;
|
|
import java.net.SocketException;
|
|
import java.net.UnknownHostException;
|
|
import java.text.SimpleDateFormat;
|
|
import java.util.ArrayList;
|
|
import java.util.Arrays;
|
|
import java.util.Comparator;
|
|
import java.util.Date;
|
|
import java.util.Enumeration;
|
|
import java.util.GregorianCalendar;
|
|
import java.util.HashMap;
|
|
import java.util.HashSet;
|
|
import java.util.Iterator;
|
|
import java.util.Locale;
|
|
import java.util.Map;
|
|
import java.util.TimeZone;
|
|
import java.util.zip.GZIPInputStream;
|
|
import java.util.zip.GZIPOutputStream;
|
|
|
|
import javax.net.ssl.HostnameVerifier;
|
|
import javax.net.ssl.HttpsURLConnection;
|
|
import javax.net.ssl.SSLContext;
|
|
import javax.net.ssl.SSLSocketFactory;
|
|
import javax.net.ssl.TrustManager;
|
|
import javax.net.ssl.X509TrustManager;
|
|
|
|
import de.anomic.kelondro.kelondroBase64Order;
|
|
import de.anomic.server.serverByteBuffer;
|
|
import de.anomic.server.serverCore;
|
|
import de.anomic.server.serverDomains;
|
|
import de.anomic.server.serverFileUtils;
|
|
import de.anomic.server.serverObjects;
|
|
import de.anomic.server.logging.serverLog;
|
|
import de.anomic.tools.nxTools;
|
|
import de.anomic.yacy.yacyURL;
|
|
|
|
/**
|
|
* This class implements an http client. While http access is built-in in java
|
|
* libraries, it is still necessary to implement the network interface since
|
|
* otherwise there is no access to the HTTP/1.0 / HTTP/1.1 header information
|
|
* that comes along each connection.
|
|
*/
|
|
|
|
public final class httpc {
|
|
|
|
// some constants
|
|
/**
|
|
* Specifies that the httpc is allowed to use gzip content encoding for
|
|
* http post requests
|
|
* @see #POST(String, httpHeader, serverObjects, HashMap)
|
|
*/
|
|
public static final String GZIP_POST_BODY = "GZIP_POST_BODY";
|
|
|
|
// final statics
|
|
private static final String vDATE = "20040602";
|
|
private static final int terminalMaxLength = 30000;
|
|
private static final TimeZone GMTTimeZone = TimeZone.getTimeZone("GMT");
|
|
private static final SimpleDateFormat HTTPGMTFormatter = new SimpleDateFormat("EEE, dd MMM yyyy HH:mm:ss zzz", Locale.US);
|
|
private static final HashMap reverseMappingCache = new HashMap();
|
|
private static final HashSet activeConnections = new HashSet(); // all connections are stored here and deleted when they are finished
|
|
private static final long minimumTime_before_activeConnections_cleanup = 3600000; // 1 Hour
|
|
private static final long minimumTime_before_idleConnections_cleanup = 120000; // 2 Minutes
|
|
private static final int activeConnections_maximum = 64;
|
|
public static final connectionTimeComparator connectionTimeComparatorInstance = new connectionTimeComparator();
|
|
|
|
private static int objCounter = 0; // will be increased with each object and is use to return a hash code
|
|
|
|
// defined during set-up of switchboard
|
|
public static boolean yacyDebugMode = false;
|
|
|
|
// statics to be defined in static section below
|
|
private static SSLSocketFactory theSSLSockFactory = null;
|
|
public static String systemOST;
|
|
public static String userAgent;
|
|
|
|
static {
|
|
// set the time zone
|
|
HTTPGMTFormatter.setTimeZone(GMTTimeZone); // The GMT standard date format used in the HTTP protocol
|
|
|
|
// set time-out of InetAddress.getByName cache ttl
|
|
java.security.Security.setProperty("networkaddress.cache.ttl" , "60");
|
|
java.security.Security.setProperty("networkaddress.cache.negative.ttl" , "0");
|
|
|
|
// initializing a dummy trustManager to enable https connections
|
|
|
|
// Create a trust manager that does not validate certificate chains
|
|
TrustManager[] trustAllCerts = new TrustManager[] { new X509TrustManager() {
|
|
public java.security.cert.X509Certificate[] getAcceptedIssuers() {
|
|
return null;
|
|
}
|
|
|
|
public void checkClientTrusted(
|
|
java.security.cert.X509Certificate[] certs, String authType) {
|
|
}
|
|
|
|
public void checkServerTrusted(
|
|
java.security.cert.X509Certificate[] certs, String authType) {
|
|
}
|
|
} };
|
|
|
|
// Install the all-trusting trust manager
|
|
try {
|
|
SSLContext sc = SSLContext.getInstance("SSL");
|
|
// Create empty HostnameVerifier
|
|
HostnameVerifier hv = new HostnameVerifier() {
|
|
public boolean verify(String urlHostName, javax.net.ssl.SSLSession session) {
|
|
// logger.info("Warning: URL Host: "+urlHostName+"
|
|
// vs."+session.getPeerHost());
|
|
return true;
|
|
}
|
|
};
|
|
|
|
sc.init(null, trustAllCerts, new java.security.SecureRandom());
|
|
HttpsURLConnection.setDefaultSSLSocketFactory(theSSLSockFactory = sc.getSocketFactory());
|
|
HttpsURLConnection.setDefaultHostnameVerifier(hv);
|
|
} catch (Exception e) {
|
|
}
|
|
|
|
// provide system information for client identification
|
|
String loc = System.getProperty("user.timezone", "nowhere");
|
|
int p = loc.indexOf("/");
|
|
if (p > 0) loc = loc.substring(0,p);
|
|
loc = loc + "/" + System.getProperty("user.language", "dumb");
|
|
systemOST =
|
|
System.getProperty("os.arch", "no-os-arch") + " " +
|
|
System.getProperty("os.name", "no-os-name") + " " +
|
|
System.getProperty("os.version", "no-os-version") + "; " +
|
|
"java " + System.getProperty("java.version", "no-java-version") + "; " + loc;
|
|
userAgent = "yacy (www.yacy.net; v" + vDATE + "; " + systemOST + ")";
|
|
}
|
|
|
|
|
|
// class variables
|
|
private Socket socket = null; // client socket for commands
|
|
public String adressed_host = null;
|
|
public int adressed_port = 80;
|
|
private String target_virtual_host = null;
|
|
|
|
// output and input streams for client control connection
|
|
private PushbackInputStream clientInput = null;
|
|
private OutputStream clientOutput = null;
|
|
|
|
private httpdByteCountInputStream clientInputByteCount = null;
|
|
private httpdByteCountOutputStream clientOutputByteCount = null;
|
|
|
|
private boolean remoteProxyUse = false;
|
|
private httpRemoteProxyConfig remoteProxyConfig = null;
|
|
|
|
private String requestPath = null;
|
|
private boolean allowContentEncoding = true;
|
|
|
|
public boolean ssl;
|
|
public long initTime, lastIO;
|
|
public String command;
|
|
public int timeout;
|
|
|
|
private int hashIndex;
|
|
|
|
|
|
/**
|
|
* Initialize the httpc-instance with the given data.
|
|
*
|
|
* @param remoteProxyHost
|
|
* @param remoteProxyPort
|
|
* @throws IOException
|
|
*/
|
|
public httpc(
|
|
String server,
|
|
String vhost,
|
|
int port,
|
|
int timeout,
|
|
boolean ssl,
|
|
httpRemoteProxyConfig theRemoteProxyConfig,
|
|
String incomingByteCountAccounting,
|
|
String outgoingByteCountAccounting
|
|
) throws IOException {
|
|
|
|
// remove old connections
|
|
// do NOT remove this check; in case that everything works fine this call does nothing
|
|
// but if in any arror case connections stay open, this will ensure that the peer keeps running and the host server is not blocked from working
|
|
checkIdleConnections();
|
|
|
|
// register new connection
|
|
this.hashIndex = objCounter;
|
|
|
|
// register new connection
|
|
this.hashIndex = objCounter;
|
|
objCounter++;
|
|
synchronized (activeConnections) {activeConnections.add(this);}
|
|
//System.out.println("*** DEBUG init httpc: " + activeConnections.size() + " connections online");
|
|
|
|
this.ssl = ssl;
|
|
this.initTime = Long.MAX_VALUE;
|
|
this.lastIO = Long.MAX_VALUE;
|
|
this.command = null;
|
|
this.timeout = timeout;
|
|
|
|
if ((theRemoteProxyConfig == null) ||
|
|
(!theRemoteProxyConfig.useProxy())) {
|
|
initN(
|
|
server,
|
|
vhost,
|
|
port,
|
|
timeout,
|
|
ssl,
|
|
incomingByteCountAccounting,
|
|
outgoingByteCountAccounting
|
|
);
|
|
return;
|
|
}
|
|
|
|
if (port == -1) {
|
|
port = (ssl)? 443 : 80;
|
|
}
|
|
|
|
String remoteProxyHost = theRemoteProxyConfig.getProxyHost();
|
|
int remoteProxyPort = theRemoteProxyConfig.getProxyPort();
|
|
|
|
this.initN(
|
|
remoteProxyHost,
|
|
vhost,
|
|
remoteProxyPort,
|
|
timeout,
|
|
ssl,
|
|
incomingByteCountAccounting,
|
|
outgoingByteCountAccounting);
|
|
|
|
this.remoteProxyUse = true;
|
|
this.adressed_host = server;
|
|
this.adressed_port = port;
|
|
this.target_virtual_host = vhost;
|
|
this.remoteProxyConfig = theRemoteProxyConfig;
|
|
}
|
|
|
|
/**
|
|
* Convert the status of this class into an String object to output it.
|
|
*/
|
|
public String toString() {
|
|
return (this.adressed_host == null) ? "Disconnected" : "Connected to " + this.adressed_host +
|
|
((this.remoteProxyUse) ? " via " + adressed_host : "");
|
|
}
|
|
|
|
/**
|
|
* Sets wether the content is allowed to be unzipped while getting?
|
|
* FIXME: The name of this method seems misleading, if I read the usage of
|
|
* this method correctly?
|
|
*
|
|
* @param status true, if the content is allowed to be decoded on the fly?
|
|
*/
|
|
public void setAllowContentEncoding(boolean status) {
|
|
this.allowContentEncoding = status;
|
|
}
|
|
|
|
/**
|
|
* Check wether the connection of this instance is closed.
|
|
*
|
|
* @return true if the connection is no longer open.
|
|
*/
|
|
public boolean isClosed() {
|
|
if (this.socket == null) return true;
|
|
return (!this.socket.isConnected()) || (this.socket.isClosed());
|
|
}
|
|
|
|
/**
|
|
* Returns the given date in an HTTP-usable format.
|
|
* (according to RFC822)
|
|
*
|
|
* @param date The Date-Object to be converted.
|
|
* @return String with the date.
|
|
*/
|
|
public static String dateString(Date date) {
|
|
if (date == null) return "";
|
|
|
|
/*
|
|
* This synchronized is needed because SimpleDateFormat
|
|
* is not thread-safe.
|
|
* See: http://bugs.sun.com/bugdatabase/view_bug.do?bug_id=6231579
|
|
*/
|
|
synchronized(HTTPGMTFormatter) {
|
|
return HTTPGMTFormatter.format(date);
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Returns the current date as Date-Object.
|
|
*
|
|
* @return Date-object with the current time.
|
|
*/
|
|
public static Date nowDate() {
|
|
return new GregorianCalendar(GMTTimeZone).getTime();
|
|
}
|
|
|
|
public int hashCode() {
|
|
// return a hash code so it is possible to store objects of httpc objects in a HashSet
|
|
return this.hashIndex;
|
|
}
|
|
|
|
/**
|
|
* Initialize the https-instance with the given data. Opens the sockets to
|
|
* the remote server and creats input and output streams.
|
|
*
|
|
* @param server Hostname of the server to connect to.
|
|
* @param port On which port should we connect.
|
|
* @param timeout How long do we wait for answers?
|
|
* @param ssl Wether we should use SSL.
|
|
* @throws IOException
|
|
*/
|
|
private void initN(
|
|
String server,
|
|
String vhost,
|
|
int port,
|
|
int timeout,
|
|
boolean ssl,
|
|
String incomingByteCountAccounting,
|
|
String outgoingByteCountAccounting
|
|
) throws IOException {
|
|
//serverLog.logDebug("HTTPC", handle + " initialized");
|
|
this.remoteProxyUse = false;
|
|
|
|
try {
|
|
if (port == -1) {
|
|
port = (ssl)? 443 : 80;
|
|
}
|
|
|
|
this.adressed_host = server;
|
|
this.adressed_port = port;
|
|
this.target_virtual_host = vhost;
|
|
|
|
// creating a socket
|
|
this.socket = (ssl)
|
|
? theSSLSockFactory.createSocket()
|
|
: new Socket();
|
|
|
|
// creating a socket address
|
|
InetSocketAddress address = null;
|
|
if (!this.remoteProxyUse) {
|
|
// only try to resolve the address if we are not using a proxy
|
|
InetAddress hostip = serverDomains.dnsResolve(server);
|
|
if (hostip == null) throw new UnknownHostException(server);
|
|
address = new InetSocketAddress(hostip, port);
|
|
} else {
|
|
address = new InetSocketAddress(server,port);
|
|
}
|
|
|
|
// trying to establish a connection to the address
|
|
this.initTime = System.currentTimeMillis();
|
|
this.lastIO = System.currentTimeMillis();
|
|
this.socket.setKeepAlive(false);
|
|
this.socket.connect(address, timeout);
|
|
// setting socket timeout and keep alive behaviour
|
|
this.socket.setSoTimeout(timeout); // waiting time for read
|
|
|
|
if (incomingByteCountAccounting != null) {
|
|
this.clientInputByteCount = new httpdByteCountInputStream(this.socket.getInputStream(),incomingByteCountAccounting);
|
|
}
|
|
if (outgoingByteCountAccounting != null) {
|
|
this.clientOutputByteCount = new httpdByteCountOutputStream(this.socket.getOutputStream(),outgoingByteCountAccounting);
|
|
}
|
|
|
|
// getting input and output streams
|
|
this.clientInput = new PushbackInputStream((this.clientInputByteCount!=null)?
|
|
this.clientInputByteCount:
|
|
this.socket.getInputStream());
|
|
this.clientOutput = this.socket.getOutputStream();
|
|
|
|
// if we reached this point, we should have a connection
|
|
} catch (UnknownHostException e) {
|
|
if (this.socket != null) {
|
|
// no need to track this, the socket cannot be established
|
|
synchronized (activeConnections) {activeConnections.remove(this);}
|
|
}
|
|
this.socket = null;
|
|
throw new IOException("unknown host: " + server);
|
|
} catch (IOException e) {
|
|
// There was an error while connecting the socket, probably a SocketTimeoutException
|
|
// we have to close the httpc, otherwise it would stay in activeConnections forever
|
|
serverLog.logFine("HTTPC", "Couldn't open socket to: " + this.adressed_host + ":" + this.adressed_port);
|
|
close();
|
|
|
|
// TODO do we need to hand it over to the caller?
|
|
throw e;
|
|
}
|
|
}
|
|
|
|
public long getInputStreamByteCount() {
|
|
return (this.clientInputByteCount == null)?0:this.clientInputByteCount.getCount();
|
|
}
|
|
|
|
public long getOutputStreamByteCount() {
|
|
return (this.clientOutputByteCount == null)?0:this.clientOutputByteCount.getCount();
|
|
}
|
|
|
|
public static int checkIdleConnections() {
|
|
// try to find and close all connections that did not find a target server and are idle waiting for a server socket
|
|
|
|
httpc[] a = allConnections(); // put set into array to avoid ConcurrentModificationExceptions
|
|
int tbd = 0;
|
|
int c = 0;
|
|
if (a.length > activeConnections_maximum) {
|
|
// delete some connections; choose the oldest
|
|
Arrays.sort(a, httpc.connectionTimeComparatorInstance);
|
|
tbd = a.length - activeConnections_maximum;
|
|
for (int i = 0; i < tbd; i++) {
|
|
if (a[i] != null) {
|
|
a[i].close();
|
|
c++;
|
|
}
|
|
}
|
|
}
|
|
for (int i = tbd; i < a.length; i++) {
|
|
httpc clientConnection = a[i];
|
|
if (clientConnection != null) {
|
|
if ((clientConnection.initTime != Long.MAX_VALUE) &&
|
|
(clientConnection.initTime + Math.max(minimumTime_before_activeConnections_cleanup, clientConnection.timeout) < System.currentTimeMillis())) {
|
|
// the time-out limit is reached. close the connection
|
|
clientConnection.close();
|
|
c++;
|
|
}
|
|
if ((clientConnection.lastIO != Long.MAX_VALUE) &&
|
|
(clientConnection.lastIO + Math.max(minimumTime_before_idleConnections_cleanup, clientConnection.timeout) < System.currentTimeMillis())) {
|
|
// the time-out limit is reached. close the connection
|
|
clientConnection.close();
|
|
c++;
|
|
}
|
|
}
|
|
}
|
|
return c;
|
|
}
|
|
|
|
public static int closeAllConnections() {
|
|
httpc[] a = allConnections(); // put set into array to avoid ConcurrentModificationExceptions
|
|
int c = 0;
|
|
for (int i = 0; i < a.length; i++) {
|
|
httpc clientConnection = a[i];
|
|
if (clientConnection != null) {
|
|
clientConnection.close();
|
|
c++;
|
|
}
|
|
}
|
|
return c;
|
|
}
|
|
|
|
public static httpc[] allConnections() {
|
|
httpc[] a = null;
|
|
synchronized (activeConnections) {
|
|
a = new httpc[activeConnections.size()];
|
|
Iterator i = httpc.activeConnections.iterator();
|
|
int c = 0;
|
|
while (i.hasNext()) {
|
|
a[c++] = (httpc) i.next();
|
|
}
|
|
}
|
|
return a;
|
|
}
|
|
|
|
public static class connectionTimeComparator implements Comparator {
|
|
|
|
public connectionTimeComparator() {
|
|
super();
|
|
}
|
|
|
|
public int compare(Object o1, Object o2) {
|
|
httpc c1 = (httpc) o1;
|
|
httpc c2 = (httpc) o2;
|
|
long l1 = System.currentTimeMillis() - c1.lastIO;
|
|
long l2 = System.currentTimeMillis() - c2.lastIO;
|
|
if (l1 < l2) return 1;
|
|
if (l1 > l2) return -1;
|
|
return 0;
|
|
}
|
|
}
|
|
|
|
public void finalize() {
|
|
this.close();
|
|
}
|
|
|
|
public void close() {
|
|
synchronized (activeConnections) {activeConnections.remove(this);}
|
|
//System.out.println("*** DEBUG close httpc: " + activeConnections.size() + " connections online");
|
|
|
|
if (this.clientInput != null) {
|
|
try {this.clientInput.close();} catch (Exception e) {}
|
|
this.clientInput = null;
|
|
}
|
|
|
|
if (this.clientOutput != null) {
|
|
try {this.clientOutput.close();} catch (Exception e) {}
|
|
this.clientOutput = null;
|
|
}
|
|
|
|
if (this.socket != null) {
|
|
try {this.socket.close();} catch (Exception e) {}
|
|
this.socket = null;
|
|
}
|
|
|
|
if (this.clientInputByteCount != null) {
|
|
this.clientInputByteCount.finish();
|
|
this.clientInputByteCount = null;
|
|
}
|
|
|
|
if (this.clientOutputByteCount != null) {
|
|
this.clientOutputByteCount.finish();
|
|
this.clientOutputByteCount = null;
|
|
}
|
|
|
|
this.adressed_host = null;
|
|
this.target_virtual_host = null;
|
|
|
|
this.remoteProxyConfig = null;
|
|
this.requestPath = null;
|
|
}
|
|
|
|
/**
|
|
* This method invokes a call to the given server.
|
|
*
|
|
* @param method
|
|
* Which method should be called? GET, POST, HEAD or CONNECT
|
|
* @param path
|
|
* String with the path on the server to be get.
|
|
* @param header
|
|
* The prefilled header (if available) from the calling browser.
|
|
* @param zipped
|
|
* Is encoded content (gzip) allowed or not?
|
|
* @throws IOException
|
|
*/
|
|
private void send(String method, String path, httpHeader header, boolean zipped) throws IOException {
|
|
// scheduled request through request-response objects/threads
|
|
|
|
// check and correct path
|
|
if ((path == null) || (path.length() == 0)) path = "/";
|
|
|
|
// for debuggug:
|
|
this.requestPath = path;
|
|
|
|
// prepare header
|
|
if (header == null) header = new httpHeader();
|
|
|
|
// set some standard values
|
|
if (!(header.containsKey(httpHeader.ACCEPT)))
|
|
header.put(httpHeader.ACCEPT, "text/xml,application/xml,application/xhtml+xml,text/html;q=0.9,text/plain;q=0.8,image/png,*/*;q=0.5");
|
|
if (!(header.containsKey(httpHeader.ACCEPT_CHARSET)))
|
|
header.put(httpHeader.ACCEPT_CHARSET, "ISO-8859-1,utf-8;q=0.7,*;q=0.7");
|
|
if (!(header.containsKey(httpHeader.ACCEPT_LANGUAGE)))
|
|
header.put(httpHeader.ACCEPT_LANGUAGE, "en-us,en;q=0.5");
|
|
if (!(header.containsKey(httpHeader.KEEP_ALIVE)))
|
|
header.put(httpHeader.KEEP_ALIVE, "300");
|
|
|
|
// set user agent. The user agent is only set if the value does not yet exists.
|
|
// this gives callers the opportunity, to change the user agent themselves, and
|
|
// it will not be changed.
|
|
if (!(header.containsKey(httpHeader.USER_AGENT))) header.put(httpHeader.USER_AGENT, userAgent);
|
|
|
|
// set the host attribute. This is in particular necessary, if we contact another proxy
|
|
// the host is mandatory, if we use HTTP/1.1
|
|
if (!(header.containsKey(httpHeader.HOST))) {
|
|
if (this.remoteProxyUse) {
|
|
header.put(httpHeader.HOST, this.adressed_host);
|
|
} else {
|
|
header.put(httpHeader.HOST, this.target_virtual_host);
|
|
}
|
|
}
|
|
|
|
if (this.remoteProxyUse) {
|
|
String remoteProxyUser = this.remoteProxyConfig.getProxyUser();
|
|
String remoteProxyPwd = this.remoteProxyConfig.getProxyPwd();
|
|
if ((remoteProxyUser!=null)&&(remoteProxyUser.length()>0)) {
|
|
header.put(httpHeader.PROXY_AUTHORIZATION,"Basic " + kelondroBase64Order.standardCoder.encodeString(remoteProxyUser + ":" + remoteProxyPwd));
|
|
}
|
|
}
|
|
|
|
if (!(header.containsKey(httpHeader.CONNECTION))) {
|
|
header.put(httpHeader.CONNECTION, "close");
|
|
}
|
|
|
|
// stimulate zipping or not
|
|
// we can unzip, and we will return it always as unzipped, unless not wanted
|
|
if (header.containsKey(httpHeader.ACCEPT_ENCODING)) {
|
|
String encoding = (String) header.get(httpHeader.ACCEPT_ENCODING);
|
|
if (zipped) {
|
|
if (encoding.indexOf("gzip") < 0) {
|
|
// add the gzip encoding
|
|
//System.out.println("!!! adding gzip encoding");
|
|
header.put(httpHeader.ACCEPT_ENCODING, "gzip,deflate" + ((encoding.length() == 0) ? "" : (";" + encoding)));
|
|
}
|
|
} else {
|
|
int pos = encoding.indexOf("gzip");
|
|
if (pos >= 0) {
|
|
// remove the gzip encoding
|
|
//System.out.println("!!! removing gzip encoding");
|
|
// ex: "gzip,deflate" => pos == 0, but we need to remove the "," as well => substring(pos+5),
|
|
// ex: "gzip" => pos == 0, but substring(pos+5) would exceed boundaries
|
|
String enc = encoding.substring(0, pos) + (encoding.length() > (pos+5) ? encoding.substring(pos + 5) : "");
|
|
header.put(httpHeader.ACCEPT_ENCODING, enc);
|
|
}
|
|
}
|
|
} else {
|
|
if (zipped) header.put(httpHeader.ACCEPT_ENCODING, "gzip,deflate");
|
|
}
|
|
|
|
//header = new httpHeader(); header.put("Host", this.host); // debug
|
|
|
|
StringBuffer sb = new StringBuffer();
|
|
// send request
|
|
if ((this.remoteProxyUse) && (!(method.equals(httpHeader.METHOD_CONNECT))))
|
|
path = ((this.adressed_port == 443) ? "https://" : "http://") + this.adressed_host + ":" + this.adressed_port + path;
|
|
sb.append(method + " " + path + " HTTP/1.0" + serverCore.crlfString); // TODO if set to HTTP/1.1, servers give time-outs?
|
|
|
|
// send header
|
|
//System.out.println("***HEADER for path " + path + ": PROXY TO SERVER = " + header.toString()); // DEBUG
|
|
Iterator i = header.keySet().iterator();
|
|
String key;
|
|
int count;
|
|
char tag;
|
|
while (i.hasNext()) {
|
|
key = (String) i.next();
|
|
tag = key.charAt(0);
|
|
if ((tag != '*') && (tag != '#')) {
|
|
count = header.keyCount(key);
|
|
for (int j = 0; j < count; j++) {
|
|
sb.append(key + ": " + ((String) header.getSingle(key, j)).trim() + serverCore.crlfString);
|
|
}
|
|
//System.out.println("#" + key + ": " + value);
|
|
}
|
|
}
|
|
|
|
// add terminating line
|
|
sb.append(serverCore.crlfString);
|
|
serverCore.send(this.clientOutput, sb.toString());
|
|
this.clientOutput.flush();
|
|
|
|
// this is the place where www.stern.de refuses to answer ..???
|
|
}
|
|
|
|
/**
|
|
* This method GETs a page from the server.
|
|
*
|
|
* @param path The path to the page which should be GET.
|
|
* @param requestHeader Prefilled httpHeader.
|
|
* @return Instance of response with the content.
|
|
* @throws IOException
|
|
*/
|
|
public response GET(String path, httpHeader requestHeader) throws IOException {
|
|
//serverLog.logDebug("HTTPC", handle + " requested GET '" + path + "', time = " + (System.currentTimeMillis() - handle));
|
|
this.command = "GET " + path;
|
|
try {
|
|
boolean zipped = (!this.allowContentEncoding) ? false : httpd.shallTransportZipped(path);
|
|
send(httpHeader.METHOD_GET, path, requestHeader, zipped);
|
|
response r = new response(zipped);
|
|
//serverLog.logDebug("HTTPC", handle + " returned GET '" + path + "', time = " + (System.currentTimeMillis() - handle));
|
|
return r;
|
|
} catch (Exception e) {
|
|
if (e.getMessage().indexOf("heap space") > 0) {
|
|
e.printStackTrace();
|
|
}
|
|
throw new IOException(e.getMessage());
|
|
}
|
|
}
|
|
|
|
/**
|
|
* This method gets only the header of a page.
|
|
*
|
|
* @param path The path to the page whose header should be get.
|
|
* @param requestHeader Prefilled httpHeader.
|
|
* @return Instance of response with the content.
|
|
* @throws IOException
|
|
*/
|
|
public response HEAD(String path, httpHeader requestHeader) throws IOException {
|
|
this.command = "HEAD " + path;
|
|
try {
|
|
send(httpHeader.METHOD_HEAD, path, requestHeader, false);
|
|
return new response(false);
|
|
// in this case the caller should not read the response body,
|
|
// since there is none...
|
|
} catch (SocketException e) {
|
|
throw new IOException(e.getMessage());
|
|
}
|
|
}
|
|
|
|
/**
|
|
* This method POSTs some data to a page.
|
|
*
|
|
* @param path The path to the page which the post is sent to.
|
|
* @param requestHeader Prefilled httpHeader.
|
|
* @param ins InputStream with the data to be posted to the server.
|
|
* @return Instance of response with the content.
|
|
* @throws IOException
|
|
*/
|
|
public response POST(String path, httpHeader requestHeader, InputStream ins) throws IOException {
|
|
this.command = "POST " + path;
|
|
try {
|
|
send(httpHeader.METHOD_POST, path, requestHeader, false);
|
|
// if there is a body to the call, we would have a CONTENT-LENGTH tag in the requestHeader
|
|
String cl = (String) requestHeader.get(httpHeader.CONTENT_LENGTH);
|
|
int len, c;
|
|
byte[] buffer = new byte[512];
|
|
if (cl != null) {
|
|
len = Integer.parseInt(cl);
|
|
// transfer len bytes from ins to the server
|
|
while ((len > 0) && ((c = ins.read(buffer)) >= 0)) {
|
|
this.clientOutput.write(buffer, 0, c);
|
|
len -= c;
|
|
}
|
|
} else {
|
|
len = 0;
|
|
while ((c = ins.read(buffer)) >= 0) {
|
|
this.clientOutput.write(buffer, 0, c);
|
|
len += c;
|
|
}
|
|
|
|
// TODO: we can not set the header here. This ist too late
|
|
requestHeader.put(httpHeader.CONTENT_LENGTH, Integer.toString(len));
|
|
}
|
|
this.clientOutput.flush();
|
|
return new response(false);
|
|
} catch (SocketException e) {
|
|
throw new IOException(e.getMessage());
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Call the server with the CONNECT-method.
|
|
* This is used to establish https-connections through a https-proxy
|
|
*
|
|
* @param host To which host should a connection be made?
|
|
* @param port Which port should be connected?
|
|
* @param requestHeader prefilled httpHeader.
|
|
* @return Instance of response with the content.
|
|
*/
|
|
|
|
public response CONNECT(String remotehost, int remoteport, httpHeader requestHeader) throws IOException {
|
|
this.command = "CONNECT " + remotehost + ":" + remoteport;
|
|
try {
|
|
send(httpHeader.METHOD_CONNECT, remotehost + ":" + remoteport, requestHeader, false);
|
|
return new response(false);
|
|
} catch (SocketException e) {
|
|
throw new IOException(e.getMessage());
|
|
}
|
|
}
|
|
|
|
/**
|
|
* This method sends several files at once via a POST request. Only those
|
|
* files in the Hashtable files are written whose names are contained in
|
|
* args.
|
|
*
|
|
* @param path The path to the page which the post is sent to.
|
|
* @param requestHeader Prefilled httpHeader.
|
|
* @param args serverObjects with the names of the files to send.
|
|
* @param files HashMap with the names of the files as key and the content
|
|
* of the files as value.
|
|
* @return Instance of response with the content.
|
|
* @throws IOException
|
|
*/
|
|
public response POST(String path, httpHeader requestHeader, serverObjects args, HashMap files) throws IOException {
|
|
// make shure, the header has a boundary information like
|
|
// CONTENT-TYPE=multipart/form-data; boundary=----------0xKhTmLbOuNdArY
|
|
if (requestHeader == null) requestHeader = new httpHeader();
|
|
String boundary = (String) requestHeader.get(httpHeader.CONTENT_TYPE);
|
|
if (boundary == null) {
|
|
// create a boundary
|
|
boundary = "multipart/form-data; boundary=----------" + java.lang.System.currentTimeMillis();
|
|
requestHeader.put(httpHeader.CONTENT_TYPE, boundary);
|
|
}
|
|
// extract the boundary string
|
|
int pos = boundary.toUpperCase().indexOf("BOUNDARY=");
|
|
if (pos < 0) {
|
|
// again, create a boundary
|
|
boundary = "multipart/form-data; boundary=----------" + java.lang.System.currentTimeMillis();
|
|
requestHeader.put(httpHeader.CONTENT_TYPE, boundary);
|
|
pos = boundary.indexOf("boundary=");
|
|
}
|
|
boundary = "--" + boundary.substring(pos + "boundary=".length());
|
|
|
|
boolean zipContent = args.containsKey(GZIP_POST_BODY);
|
|
args.remove(GZIP_POST_BODY);
|
|
|
|
OutputStream out;
|
|
GZIPOutputStream zippedOut;
|
|
serverByteBuffer buf = new serverByteBuffer();
|
|
if (zipContent) {
|
|
zippedOut = new GZIPOutputStream(buf);
|
|
out = zippedOut;
|
|
} else {
|
|
out = buf;
|
|
}
|
|
|
|
// in contrast to GET and HEAD, this method also transports a message body
|
|
// the body consists of repeated boundaries and values in between
|
|
if (args.size() != 0) {
|
|
// we have values for the POST, start with one boundary
|
|
String key, value;
|
|
Enumeration e = args.keys();
|
|
while (e.hasMoreElements()) {
|
|
// start with a boundary
|
|
out.write(boundary.getBytes("UTF-8"));
|
|
out.write(serverCore.crlf);
|
|
// write value
|
|
key = (String) e.nextElement();
|
|
value = args.get(key, "");
|
|
if ((files != null) && (files.containsKey(key))) {
|
|
// we are about to write a file
|
|
out.write(("Content-Disposition: form-data; name=" + '"' + key + '"' + "; filename=" + '"' + value + '"').getBytes("UTF-8"));
|
|
out.write(serverCore.crlf);
|
|
out.write(serverCore.crlf);
|
|
out.write((byte[]) files.get(key));
|
|
out.write(serverCore.crlf);
|
|
} else {
|
|
// write a single value
|
|
out.write(("Content-Disposition: form-data; name=" + '"' + key + '"').getBytes("UTF-8"));
|
|
out.write(serverCore.crlf);
|
|
out.write(serverCore.crlf);
|
|
out.write(value.getBytes("UTF-8"));
|
|
out.write(serverCore.crlf);
|
|
}
|
|
}
|
|
// finish with a boundary
|
|
out.write(boundary.getBytes("UTF-8"));
|
|
out.write(serverCore.crlf);
|
|
}
|
|
// create body array
|
|
out.close();
|
|
byte[] body = buf.toByteArray();
|
|
buf = null; out = null;
|
|
|
|
//System.out.println("DEBUG: PUT BODY=" + new String(body));
|
|
if (zipContent) {
|
|
requestHeader.put(httpHeader.CONTENT_ENCODING, "gzip");
|
|
|
|
//TODO: should we also set the content length here?
|
|
} else {
|
|
// size of that body
|
|
requestHeader.put(httpHeader.CONTENT_LENGTH, Integer.toString(body.length));
|
|
}
|
|
|
|
// send the header
|
|
send(httpHeader.METHOD_POST, path, requestHeader, false);
|
|
|
|
// send the body
|
|
serverCore.send(this.clientOutput, body);
|
|
|
|
return new response(false);
|
|
}
|
|
|
|
public static byte[] singleGET(
|
|
String realhost,
|
|
String virtualhost,
|
|
int port,
|
|
String path,
|
|
int timeout,
|
|
String user,
|
|
String password,
|
|
boolean ssl,
|
|
httpRemoteProxyConfig theRemoteProxyConfig,
|
|
httpHeader requestHeader,
|
|
File download
|
|
) throws IOException {
|
|
// if download == null, the get result is stored to a byte[] and returned,
|
|
// otherwise the get is streamed to the file and null is returned
|
|
if (requestHeader == null) requestHeader = new httpHeader();
|
|
|
|
// setting host authorization header
|
|
if ((user != null) && (password != null) && (user.length() != 0)) {
|
|
requestHeader.put(httpHeader.AUTHORIZATION, kelondroBase64Order.standardCoder.encodeString(user + ":" + password));
|
|
}
|
|
|
|
httpc con = new httpc(realhost, virtualhost, port, timeout, ssl, theRemoteProxyConfig, null, null);
|
|
|
|
httpc.response res = con.GET(path, requestHeader);
|
|
if (res.status.startsWith("2")) {
|
|
if (download == null) {
|
|
// stream to byte[]
|
|
serverByteBuffer sbb = new serverByteBuffer();
|
|
res.writeContent(sbb, null);
|
|
con.close();
|
|
return sbb.getBytes();
|
|
} else {
|
|
// stream to file and return null
|
|
res.writeContent(null, download);
|
|
con.close();
|
|
return null;
|
|
}
|
|
}
|
|
return res.status.getBytes();
|
|
}
|
|
|
|
public static byte[] singleGET(
|
|
yacyURL u,
|
|
String vhost,
|
|
int timeout,
|
|
String user,
|
|
String password,
|
|
httpRemoteProxyConfig theRemoteProxyConfig,
|
|
File download
|
|
) throws IOException {
|
|
int port = u.getPort();
|
|
boolean ssl = u.getProtocol().equals("https");
|
|
if (port < 0) port = (ssl) ? 443: 80;
|
|
String path = u.getPath();
|
|
String query = u.getQuery();
|
|
if ((query != null) && (query.length() > 0)) path = path + "?" + query;
|
|
return singleGET(u.getHost(), vhost, port, path, timeout, user, password, ssl, theRemoteProxyConfig, null, download);
|
|
}
|
|
|
|
public static byte[] singlePOST(
|
|
String realhost,
|
|
String virtualhost,
|
|
int port,
|
|
String path,
|
|
int timeout,
|
|
String user,
|
|
String password,
|
|
boolean ssl,
|
|
httpRemoteProxyConfig theRemoteProxyConfig,
|
|
httpHeader requestHeader,
|
|
serverObjects props,
|
|
HashMap files
|
|
) throws IOException {
|
|
|
|
if (requestHeader == null) requestHeader = new httpHeader();
|
|
if ((user != null) && (password != null) && (user.length() != 0)) {
|
|
requestHeader.put(httpHeader.AUTHORIZATION, kelondroBase64Order.standardCoder.encodeString(user + ":" + password));
|
|
}
|
|
|
|
httpc con = new httpc(realhost, virtualhost, port, timeout, ssl, theRemoteProxyConfig, null, null);
|
|
httpc.response res = con.POST(path, requestHeader, props, files);
|
|
|
|
//System.out.println("response=" + res.toString());
|
|
if (!(res.status.startsWith("2"))) {
|
|
byte[] status = res.status.getBytes();
|
|
con.close();
|
|
return status;
|
|
}
|
|
|
|
// read connection body and return body
|
|
serverByteBuffer sbb = new serverByteBuffer();
|
|
res.writeContent(sbb, null);
|
|
con.close();
|
|
return sbb.getBytes();
|
|
}
|
|
|
|
public static byte[] singlePOST(
|
|
yacyURL u,
|
|
String vhost,
|
|
int timeout,
|
|
String user,
|
|
String password,
|
|
httpRemoteProxyConfig theRemoteProxyConfig,
|
|
serverObjects props,
|
|
HashMap files
|
|
) throws IOException {
|
|
int port = u.getPort();
|
|
boolean ssl = u.getProtocol().equals("https");
|
|
if (port < 0) port = (ssl) ? 443 : 80;
|
|
String path = u.getPath();
|
|
String query = u.getQuery();
|
|
if ((query != null) && (query.length() > 0)) path = path + "?" + query;
|
|
return singlePOST(
|
|
u.getHost(),
|
|
vhost,
|
|
port,
|
|
path,
|
|
timeout,
|
|
user,
|
|
password,
|
|
ssl,
|
|
theRemoteProxyConfig,
|
|
null,
|
|
props,
|
|
files
|
|
);
|
|
}
|
|
|
|
public static byte[] wget(
|
|
yacyURL url,
|
|
String vhost,
|
|
int timeout,
|
|
String user,
|
|
String password,
|
|
httpRemoteProxyConfig theRemoteProxyConfig,
|
|
httpHeader requestHeader,
|
|
File download
|
|
) throws IOException {
|
|
|
|
int port = url.getPort();
|
|
boolean ssl = url.getProtocol().equals("https");
|
|
if (port < 0) port = (ssl) ? 443: 80;
|
|
String path = url.getPath();
|
|
String query = url.getQuery();
|
|
if ((query != null) && (query.length() > 0)) path = path + "?" + query;
|
|
|
|
// splitting of the byte array into lines
|
|
byte[] a = singleGET(
|
|
url.getHost(),
|
|
vhost,
|
|
port,
|
|
path,
|
|
timeout,
|
|
user,
|
|
password,
|
|
ssl,
|
|
theRemoteProxyConfig,
|
|
requestHeader,
|
|
download
|
|
);
|
|
|
|
if (a == null) return null;
|
|
|
|
// support of gzipped data (requested by roland)
|
|
a = serverFileUtils.uncompressGZipArray(a);
|
|
|
|
// return result
|
|
return a;
|
|
}
|
|
|
|
public static Map loadHashMap(yacyURL url, httpRemoteProxyConfig proxy) {
|
|
try {
|
|
// should we use the proxy?
|
|
boolean useProxy = (proxy != null) &&
|
|
(proxy.useProxy()) &&
|
|
(proxy.useProxy4Yacy());
|
|
|
|
// sending request
|
|
final HashMap result = nxTools.table(
|
|
httpc.wget(
|
|
url,
|
|
url.getHost(),
|
|
8000,
|
|
null,
|
|
null,
|
|
(useProxy) ? proxy : null,
|
|
null,
|
|
null
|
|
)
|
|
, "UTF-8");
|
|
|
|
if (result == null) return new HashMap();
|
|
return result;
|
|
} catch (Exception e) {
|
|
return new HashMap();
|
|
}
|
|
}
|
|
|
|
public static httpHeader whead(
|
|
yacyURL url,
|
|
String vhost,
|
|
int timeout,
|
|
String user,
|
|
String password,
|
|
httpRemoteProxyConfig theRemoteProxyConfig
|
|
) throws IOException {
|
|
return whead(url,vhost,timeout,user,password,theRemoteProxyConfig,null);
|
|
}
|
|
|
|
public static httpHeader whead(
|
|
yacyURL url,
|
|
String vhost,
|
|
int timeout,
|
|
String user,
|
|
String password,
|
|
httpRemoteProxyConfig theRemoteProxyConfig,
|
|
httpHeader requestHeader
|
|
) throws IOException {
|
|
// generate request header
|
|
if (requestHeader == null) requestHeader = new httpHeader();
|
|
if ((user != null) && (password != null) && (user.length() != 0)) {
|
|
requestHeader.put(httpHeader.AUTHORIZATION, kelondroBase64Order.standardCoder.encodeString(user + ":" + password));
|
|
}
|
|
// parse query
|
|
|
|
int port = url.getPort();
|
|
boolean ssl = url.getProtocol().equals("https");
|
|
if (port < 0) port = (ssl) ? 443 : 80;
|
|
String path = url.getPath();
|
|
String query = url.getQuery();
|
|
if ((query != null) && (query.length() > 0)) path = path + "?" + query;
|
|
String realhost = url.getHost();
|
|
|
|
// start connection
|
|
httpc con = new httpc(realhost, vhost, port, timeout, ssl, theRemoteProxyConfig, null, null);
|
|
httpc.response res = con.HEAD(path, requestHeader);
|
|
httpHeader h = res.responseHeader;
|
|
con.close();
|
|
return h;
|
|
}
|
|
|
|
public static byte[] wput(
|
|
yacyURL url,
|
|
String vhost,
|
|
int timeout,
|
|
String user,
|
|
String password,
|
|
httpRemoteProxyConfig theRemoteProxyConfig,
|
|
serverObjects props,
|
|
HashMap files
|
|
) throws IOException {
|
|
// splitting of the byte array into lines
|
|
byte[] a = singlePOST(
|
|
url,
|
|
vhost,
|
|
timeout,
|
|
user,
|
|
password,
|
|
theRemoteProxyConfig,
|
|
props,
|
|
files
|
|
);
|
|
|
|
if (a == null) return null;
|
|
|
|
// support of gzipped data
|
|
a = serverFileUtils.uncompressGZipArray(a);
|
|
|
|
// return result
|
|
return a;
|
|
|
|
//System.out.println("wput-out=" + new String(a));
|
|
//return nxTools.strings(a);
|
|
}
|
|
|
|
public static void main(String[] args) {
|
|
System.out.println("ANOMIC.DE HTTP CLIENT v" + vDATE);
|
|
String url = args[0];
|
|
if (!(url.toUpperCase().startsWith("HTTP://"))) url = "http://" + url;
|
|
ArrayList text = new ArrayList();
|
|
if (args.length == 4) {
|
|
int timeout = Integer.parseInt(args[1]);
|
|
String proxyHost = args[2];
|
|
int proxyPort = Integer.parseInt(args[3]);
|
|
|
|
httpRemoteProxyConfig theRemoteProxyConfig = httpRemoteProxyConfig.init(proxyHost,proxyPort);
|
|
try {
|
|
yacyURL u = new yacyURL(url, null);
|
|
text = nxTools.strings(wget(u, u.getHost(), timeout, null, null, theRemoteProxyConfig, null, null));
|
|
} catch (MalformedURLException e) {
|
|
System.out.println("The url '" + url + "' is wrong.");
|
|
} catch (IOException e) {
|
|
System.out.println("Error loading url '" + url + "': " + e.getMessage());
|
|
}
|
|
} /*else {
|
|
serverObjects post = new serverObjects();
|
|
int p;
|
|
for (int i = 1; i < args.length; i++) {
|
|
p = args[i].indexOf("=");
|
|
if (p > 0) post.put(args[i].substring(0, p), args[i].substring(p + 1));
|
|
}
|
|
text = wput(url, post);
|
|
}*/
|
|
Iterator i = text.listIterator();
|
|
while (i.hasNext()) System.out.println((String) i.next());
|
|
}
|
|
|
|
/**
|
|
* Inner Class to get the response of an http-request and parse it.
|
|
*/
|
|
public final class response {
|
|
// Response-Header = Date | Pragma | Allow | Content-Encoding | Content-Length | Content-Type |
|
|
// Expires | Last-Modified | HTTP-header
|
|
/*
|
|
Status-Line = HTTP-Version SP Status-Code SP Reason-Phrase CRLF
|
|
1xx: Informational - Not used, but reserved for future use
|
|
2xx: Success - The action was successfully received, understood, and accepted.
|
|
3xx: Redirection - Further action must be taken in order to complete the request
|
|
4xx: Client Error - The request contains bad syntax or cannot be fulfilled
|
|
5xx: Server Error - The server failed to fulfill an apparently valid request
|
|
*/
|
|
|
|
// header information
|
|
public httpHeader responseHeader = null;
|
|
public String httpVer = "HTTP/0.9";
|
|
public String status; // the success/failure response string starting with status-code
|
|
public int statusCode = 503;
|
|
public String statusText = "internal error";
|
|
private boolean gzip; // for gunzipping on-the-fly
|
|
private long gzippedLength = -1; // reported content length if content-encoding is set
|
|
|
|
/**
|
|
* Constructor for this class. Reads in the content for the given outer
|
|
* instance and parses it.
|
|
*
|
|
* @param zipped true, if the content of this response is gzipped.
|
|
* @throws IOException
|
|
*/
|
|
public response(boolean zipped) throws IOException {
|
|
|
|
// lets start with worst-case attributes as set-up
|
|
this.responseHeader = new httpHeader(reverseMappingCache);
|
|
this.statusCode = 503;
|
|
this.statusText = "internal httpc error";
|
|
this.status = Integer.toString(this.statusCode) + " " + this.statusText;
|
|
this.gzip = false;
|
|
|
|
// check connection status
|
|
if (httpc.this.clientInput == null) {
|
|
// the server has meanwhile disconnected
|
|
this.statusCode = 503;
|
|
this.statusText = "lost connection to server";
|
|
this.status = Integer.toString(this.statusCode) + " " + this.statusText;
|
|
return; // in bad mood
|
|
}
|
|
|
|
// reads in the http header, right now, right here
|
|
byte[] b = serverCore.receive(httpc.this.clientInput, terminalMaxLength, false);
|
|
if (b == null) {
|
|
// the server has meanwhile disconnected
|
|
this.statusCode = 503;
|
|
this.statusText = "server has closed connection";
|
|
this.status = Integer.toString(this.statusCode) + " " + this.statusText;
|
|
return; // in bad mood
|
|
}
|
|
|
|
// parsing the response status line
|
|
String buffer = new String(b);
|
|
Object[] responseInfo = httpHeader.parseResponseLine(buffer);
|
|
this.httpVer = (String) responseInfo[0];
|
|
this.statusCode = ((Integer)responseInfo[1]).intValue();
|
|
this.statusText = (String) responseInfo[2];
|
|
this.status = this.statusCode + " " + this.statusText;
|
|
|
|
if ((this.statusCode==500)&&(this.statusText.equals("status line parse error"))) {
|
|
// flush in anything that comes without parsing
|
|
while ((b != null) && (b.length != 0)) b = serverCore.receive(httpc.this.clientInput, terminalMaxLength, false);
|
|
return; // in bad mood
|
|
}
|
|
|
|
// check validity
|
|
if (this.statusCode == 400) {
|
|
// bad request
|
|
// flush in anything that comes without parsing
|
|
while ((b = serverCore.receive(httpc.this.clientInput, terminalMaxLength, false)).length != 0) {}
|
|
return; // in bad mood
|
|
}
|
|
|
|
// at this point we should have a valid response. read in the header properties
|
|
String key = "";
|
|
while ((b = serverCore.receive(httpc.this.clientInput, terminalMaxLength, false)) != null) {
|
|
if (b.length == 0) break;
|
|
buffer = new String(b);
|
|
buffer=buffer.trim();
|
|
//System.out.println("#H#" + buffer); // debug
|
|
if (buffer.charAt(0) <= 32) {
|
|
// use old entry
|
|
if (key.length() == 0) throw new IOException("header corrupted - input error");
|
|
// attach new line
|
|
if (!(this.responseHeader.containsKey(key))) throw new IOException("header corrupted - internal error");
|
|
this.responseHeader.put(key, (String) this.responseHeader.get(key) + " " + buffer.trim());
|
|
} else {
|
|
// create new entry
|
|
int p = buffer.indexOf(":");
|
|
if (p > 0) {
|
|
this.responseHeader.add(buffer.substring(0, p).trim(), buffer.substring(p + 1).trim());
|
|
} else {
|
|
serverLog.logSevere("HTTPC", "RESPONSE PARSE ERROR: HOST='" + httpc.this.adressed_host + "', PATH='" + httpc.this.requestPath + "', STATUS='" + this.status + "'");
|
|
serverLog.logSevere("HTTPC", "..............BUFFER: " + buffer);
|
|
throw new IOException(this.status);
|
|
}
|
|
}
|
|
}
|
|
// finished with reading header
|
|
|
|
// we will now manipulate the header if the content is gzip encoded, because
|
|
// reading the content with "writeContent" will gunzip on-the-fly
|
|
this.gzip = ((zipped) && (this.responseHeader.gzip()));
|
|
|
|
if (this.gzip) {
|
|
if (this.responseHeader.containsKey(httpHeader.CONTENT_LENGTH)) {
|
|
this.gzippedLength = this.responseHeader.contentLength();
|
|
}
|
|
this.responseHeader.remove(httpHeader.CONTENT_ENCODING); // we fake that we don't have encoding, since what comes out does not have gzip and we also don't know what was encoded
|
|
this.responseHeader.remove(httpHeader.CONTENT_LENGTH); // we cannot use the length during gunzippig yet; still we can hope that it works
|
|
}
|
|
}
|
|
|
|
public long getGzippedLength() {
|
|
return this.gzippedLength;
|
|
}
|
|
|
|
public boolean isGzipped() {
|
|
return this.gzip;
|
|
}
|
|
|
|
/**
|
|
* Converts an instance of this class into a readable string.
|
|
*
|
|
* @return String with some information about this instance.
|
|
*/
|
|
public String toString() {
|
|
StringBuffer toStringBuffer = new StringBuffer();
|
|
toStringBuffer.append((this.status == null) ? "Status: Unknown" : "Status: " + this.status)
|
|
.append(" | Headers: ")
|
|
.append((this.responseHeader == null) ? "none" : this.responseHeader.toString());
|
|
return new String(toStringBuffer);
|
|
}
|
|
|
|
/**
|
|
* Returns wether this request was successful or not. Stati beginning
|
|
* with 2 or 3 are considered successful.
|
|
*
|
|
* @return True, if the request was successful.
|
|
*/
|
|
public boolean success() {
|
|
return ((this.status.charAt(0) == '2') || (this.status.charAt(0) == '3'));
|
|
}
|
|
|
|
/**
|
|
* If the response was encoded using <code>Content-Encoding: gzip</code>
|
|
* a {@link GZIPInputStream} is returned. If the <code>Content-Length</code> header was set,
|
|
* a {@link httpContentLengthInputStream} is returned which returns <code>-1</code> if the end of the
|
|
* response body was reached.
|
|
*
|
|
* @return a {@link InputStream} to read the response body
|
|
* @throws IOException
|
|
*/
|
|
public InputStream getContentInputStream() throws IOException {
|
|
if (this.gzip) {
|
|
// use a gzip input stream for Content-Encoding: gzip
|
|
return new GZIPInputStream(httpc.this.clientInput);
|
|
} else if (this.responseHeader.contentLength() != -1) {
|
|
// use a httpContentLengthInputStream to read until the end of the response body is reached
|
|
return new httpContentLengthInputStream(httpc.this.clientInput,this.responseHeader.contentLength());
|
|
}
|
|
// no Content-Lenght was set. In this case we can read until EOF
|
|
return httpc.this.clientInput;
|
|
}
|
|
|
|
/**
|
|
* This method outputs the found content into an byte-array and
|
|
* additionally outputs it to procOS.
|
|
*
|
|
* @param procOS
|
|
* @return the found content
|
|
* @throws IOException
|
|
*/
|
|
public byte[] writeContent(Object procOS, boolean returnByteArray) throws IOException {
|
|
serverByteBuffer sbb = null;
|
|
|
|
if (returnByteArray) {
|
|
int contentLength = (int) this.responseHeader.contentLength();
|
|
sbb = new serverByteBuffer((contentLength==-1)?8192:contentLength);
|
|
}
|
|
|
|
if (procOS instanceof OutputStream) {
|
|
writeX(this.getContentInputStream(), (OutputStream) procOS, sbb);
|
|
} else if (procOS instanceof Writer) {
|
|
String charSet = this.responseHeader.getCharacterEncoding();
|
|
if (charSet == null) charSet = httpHeader.DEFAULT_CHARSET;
|
|
writeX(this.getContentInputStream(), charSet, (Writer) procOS, sbb, charSet);
|
|
} else {
|
|
throw new IllegalArgumentException("Invalid procOS object type '" + procOS.getClass().getName() + "'");
|
|
}
|
|
|
|
return (sbb==null)?null:sbb.getBytes();
|
|
}
|
|
|
|
/**
|
|
* This method writes the input stream to either another output stream
|
|
* or a file or both.
|
|
* In case that an exception occurrs, the stream reading is just teminated
|
|
* and content received so far is returned
|
|
*
|
|
* @param procOS
|
|
* @param file
|
|
*/
|
|
public void writeContent(Object procOS, File file) {
|
|
// this writes the input stream to either another output stream or
|
|
// a file or both.
|
|
FileOutputStream bufferOS = null;
|
|
if (file != null) try {
|
|
bufferOS = new FileOutputStream(file);
|
|
} catch (FileNotFoundException e) {
|
|
file = null;
|
|
}
|
|
try {
|
|
InputStream is = this.getContentInputStream();
|
|
if (procOS == null) {
|
|
writeX(is, null, bufferOS);
|
|
} else if (procOS instanceof OutputStream) {
|
|
writeX(is, (OutputStream) procOS, bufferOS);
|
|
//writeContentX(httpc.this.clientInput, this.gzip, this.responseHeader.contentLength(), procOS, bufferOS);
|
|
} else if (procOS instanceof Writer) {
|
|
String charSet = this.responseHeader.getCharacterEncoding();
|
|
if (charSet == null) charSet = httpHeader.DEFAULT_CHARSET;
|
|
writeX(is, charSet, (Writer) procOS, bufferOS, charSet);
|
|
} else {
|
|
throw new IllegalArgumentException("Invalid procOS object type '" + procOS.getClass().getName() + "'");
|
|
}
|
|
} catch (IOException e) {}
|
|
|
|
if (bufferOS != null) {
|
|
try {
|
|
bufferOS.flush();
|
|
bufferOS.close();
|
|
} catch (IOException e) {}
|
|
if (file.length() == 0) file.delete();
|
|
}
|
|
}
|
|
|
|
|
|
public void writeX(InputStream source, OutputStream procOS, OutputStream bufferOS) {
|
|
byte[] buffer = new byte[2048];
|
|
int l, c = 0;
|
|
|
|
while (true) try {
|
|
l = source.read(buffer, 0, buffer.length);
|
|
if (l <= 0) break;
|
|
lastIO = System.currentTimeMillis();
|
|
c += l;
|
|
if (procOS != null) procOS.write(buffer, 0, l);
|
|
if (bufferOS != null) bufferOS.write(buffer, 0, l);
|
|
} catch (IOException e) {
|
|
//System.out.println("*** DEBUG: writeX/IOStream terminated with IOException, processed " + c + " bytes.");
|
|
break;
|
|
}
|
|
|
|
// flush the streams
|
|
if (procOS != null) try { procOS.flush(); } catch (IOException e) {}
|
|
if (bufferOS != null) try { bufferOS.flush(); } catch (IOException e) {}
|
|
buffer = null;
|
|
}
|
|
|
|
public void writeX(InputStream source, String inputCharset, Writer procOS, OutputStream bufferOS, String outputCharset) {
|
|
try {
|
|
InputStreamReader sourceReader = new InputStreamReader(source, inputCharset);
|
|
OutputStreamWriter bufferOSWriter = (bufferOS == null) ? null : new OutputStreamWriter(bufferOS,outputCharset);
|
|
char[] buffer = new char[2048];
|
|
int l, c= 0;
|
|
|
|
while (true) try{
|
|
l = sourceReader.read(buffer, 0, buffer.length);
|
|
if (l <= 0) break;
|
|
lastIO = System.currentTimeMillis();
|
|
c += l;
|
|
if (procOS != null) procOS.write(buffer, 0, l);
|
|
if (bufferOSWriter != null) bufferOSWriter.write(buffer, 0, l);
|
|
} catch (IOException e) {
|
|
//System.out.println("*** DEBUG: writeX/ReaderWriter terminated with IOException, processed " + c + " bytes.");
|
|
break;
|
|
}
|
|
|
|
// flush the streams
|
|
if (procOS != null) try { procOS.flush(); } catch (IOException e) {}
|
|
if (bufferOSWriter != null) try { bufferOSWriter.flush(); } catch (IOException e) {}
|
|
buffer = null;
|
|
} catch (IOException e) {}
|
|
}
|
|
|
|
/**
|
|
* This method outputs a logline to the serverlog with the current
|
|
* status of this instance.
|
|
*/
|
|
public void print() {
|
|
serverLog.logInfo("HTTPC", "RESPONSE: status=" + this.status + ", header=" + this.responseHeader.toString());
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
/*
|
|
import java.net.*;
|
|
import java.io.*;
|
|
import javax.net.ssl.*;
|
|
import javax.security.cert.X509Certificate;
|
|
import java.security.KeyStore;
|
|
|
|
|
|
//The application can be modified to connect to a server outside
|
|
//the firewall by following SSLSocketClientWithTunneling.java.
|
|
|
|
public class SSLSocketClientWithClientAuth {
|
|
|
|
public static void main(String[] args) throws Exception {
|
|
String host = null;
|
|
int port = -1;
|
|
String path = null;
|
|
for (int i = 0; i < args.length; i++)
|
|
System.out.println(args[i]);
|
|
|
|
if (args.length < 3) {
|
|
System.out.println(
|
|
"USAGE: java SSLSocketClientWithClientAuth " +
|
|
"host port requestedfilepath");
|
|
System.exit(-1);
|
|
}
|
|
|
|
try {
|
|
host = args[0];
|
|
port = Integer.parseInt(args[1]);
|
|
path = args[2];
|
|
} catch (IllegalArgumentException e) {
|
|
System.out.println("USAGE: java SSLSocketClientWithClientAuth " +
|
|
"host port requestedfilepath");
|
|
System.exit(-1);
|
|
}
|
|
|
|
try {
|
|
|
|
SSLSocketFactory factory = null;
|
|
try {
|
|
SSLContext ctx;
|
|
KeyManagerFactory kmf;
|
|
KeyStore ks;
|
|
char[] passphrase = "passphrase".toCharArray();
|
|
|
|
ctx = SSLContext.getInstance("TLS");
|
|
kmf = KeyManagerFactory.getInstance("SunX509");
|
|
ks = KeyStore.getInstance("JKS");
|
|
|
|
ks.load(new FileInputStream("testkeys"), passphrase);
|
|
|
|
kmf.init(ks, passphrase);
|
|
ctx.init(kmf.getKeyManagers(), null, null);
|
|
|
|
factory = ctx.getSocketFactory();
|
|
} catch (Exception e) {
|
|
throw new IOException(e.getMessage());
|
|
}
|
|
|
|
SSLSocket socket = (SSLSocket)factory.createSocket(host, port);
|
|
|
|
socket.startHandshake();
|
|
|
|
PrintWriter out = new PrintWriter(
|
|
new BufferedWriter(
|
|
new OutputStreamWriter(
|
|
socket.getOutputStream())));
|
|
out.println("GET " + path + " HTTP/1.1");
|
|
out.println();
|
|
out.flush();
|
|
|
|
if (out.checkError())
|
|
System.out.println(
|
|
"SSLSocketClient: java.io.PrintWriter error");
|
|
|
|
BufferedReader in = new BufferedReader(
|
|
new InputStreamReader(
|
|
socket.getInputStream()));
|
|
|
|
String inputLine;
|
|
|
|
while ((inputLine = in.readLine()) != null)
|
|
System.out.println(inputLine);
|
|
|
|
in.close();
|
|
out.close();
|
|
socket.close();
|
|
|
|
} catch (Exception e) {
|
|
e.printStackTrace();
|
|
}
|
|
}
|
|
}
|
|
*/
|