2005-06-18 03:48:11 +02:00
// httpc.java
2005-04-07 21:19:42 +02:00
// -------------------------------------
// (C) by Michael Peter Christen; mc@anomic.de
// first published on http://www.anomic.de
// Frankfurt, Germany, 2004
// last major change: 26.02.2004
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation; either version 2 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
//
// Using this software in any meaning (reading, learning, copying, compiling,
// running) means that you agree that the Author(s) is (are) not responsible
// for cost, loss of data or any harm that may be caused directly or indirectly
// by usage of this softare or this documentation. The usage of this software
// is on your own risk. The installation and usage (starting/running) of this
// software may allow other people or application to access your computer and
// any attached devices and is highly dependent on the configuration of the
// software which must be done by the user of the software; the author(s) is
// (are) also not responsible for proper configuration and usage of the
// software, even if provoked by documentation provided together with
// the software.
//
// Any changes to this file according to the GPL as documented in the file
// gpl.txt aside this file in the shipment you received can be done to the
// lines that follows this copyright notice here, but changes must not be
// done inside the copyright notive above. A re-distribution must contain
// the intact and unchanged copyright notice.
// Contributions and changes to the program code must be marked as such.
package de.anomic.http ;
2005-05-05 07:32:19 +02:00
import java.io.File ;
import java.io.FileOutputStream ;
import java.io.IOException ;
import java.io.InputStream ;
import java.io.OutputStream ;
import java.io.PushbackInputStream ;
import java.net.InetAddress ;
2005-08-19 08:47:34 +02:00
import java.net.InetSocketAddress ;
2005-05-05 07:32:19 +02:00
import java.net.MalformedURLException ;
import java.net.Socket ;
import java.net.SocketException ;
import java.net.URL ;
import java.net.UnknownHostException ;
import java.text.SimpleDateFormat ;
2005-07-07 15:58:54 +02:00
import java.util.ArrayList ;
2005-05-05 07:32:19 +02:00
import java.util.Date ;
import java.util.Enumeration ;
import java.util.GregorianCalendar ;
import java.util.HashMap ;
import java.util.Iterator ;
import java.util.Locale ;
import java.util.TimeZone ;
import java.util.zip.GZIPInputStream ;
2005-09-22 12:30:55 +02:00
import java.util.zip.GZIPOutputStream ;
2005-04-19 08:55:57 +02:00
2005-05-05 07:32:19 +02:00
import javax.net.ssl.SSLSocketFactory ;
2005-04-07 21:19:42 +02:00
2005-05-05 07:32:19 +02:00
import de.anomic.server.serverByteBuffer ;
import de.anomic.server.serverCodings ;
import de.anomic.server.serverCore ;
import de.anomic.server.serverObjects ;
2005-06-09 11:56:41 +02:00
import de.anomic.server.logging.serverLog ;
2005-05-05 07:32:19 +02:00
import de.anomic.server.serverCore.Session ;
2005-07-06 16:48:41 +02:00
import org.apache.commons.pool.impl.GenericObjectPool ;
2005-06-09 11:56:41 +02:00
2005-08-07 17:16:33 +02:00
/ * *
2005-08-07 18:27:38 +02:00
* This class implements an http client . While http access is built - in in java
* libraries , it is still necessary to implement the network interface since
* otherwise there is no access to the HTTP / 1 . 0 / HTTP / 1 . 1 header information
* that comes along each connection .
* FIXME : Add some information about the usage of the threadpool .
2005-08-07 17:16:33 +02:00
* /
2005-04-19 08:55:57 +02:00
public final class httpc {
2005-08-07 17:16:33 +02:00
2005-09-22 12:30:55 +02:00
// some constants
/ * *
* Specifies that the httpc is allowed to use gzip content encoding for
* http post requests
2005-10-05 12:45:33 +02:00
* @see # POST ( String , httpHeader , serverObjects , HashMap )
2005-09-22 12:30:55 +02:00
* /
public static final String GZIP_POST_BODY = " GZIP_POST_BODY " ;
2005-04-07 21:19:42 +02:00
// statics
private static final String vDATE = " 20040602 " ;
private static String userAgent ;
private static final int terminalMaxLength = 30000 ;
2005-04-19 08:55:57 +02:00
private static final TimeZone GMTTimeZone = TimeZone . getTimeZone ( " PST " ) ;
2005-08-07 18:27:38 +02:00
/ * *
* This string is initialized on loading of this class and contains
* information about the current OS .
* /
public static String systemOST ;
2005-08-07 17:16:33 +02:00
2005-04-07 21:19:42 +02:00
// --- The GMT standard date format used in the HTTP protocol
2005-04-19 08:55:57 +02:00
private static final SimpleDateFormat HTTPGMTFormatter = new SimpleDateFormat ( " EEE, dd MMM yyyy HH:mm:ss 'GMT' " , Locale . US ) ;
private static final SimpleDateFormat EMLFormatter = new SimpleDateFormat ( " dd MMM yyyy HH:mm:ss " , Locale . US ) ;
private static final SimpleDateFormat ShortFormatter = new SimpleDateFormat ( " yyyyMMddHHmmss " ) ;
2005-06-18 03:48:11 +02:00
//Mo 06 Sep 2004 23:32
2005-10-09 06:43:07 +02:00
static final HashMap reverseMappingCache = new HashMap ( ) ;
2005-08-07 17:16:33 +02:00
2005-06-18 03:48:11 +02:00
// the dns cache
private static final HashMap nameCacheHit = new HashMap ( ) ;
//private static HashSet nameCacheMiss = new HashSet();
2005-08-07 17:16:33 +02:00
2005-06-18 03:48:11 +02:00
/ * *
* A Object Pool containing all pooled httpc - objects .
* @see httpcPool
* /
private static final httpcPool theHttpcPool ;
2005-08-07 17:16:33 +02:00
2005-04-07 21:19:42 +02:00
// class variables
private Socket socket = null ; // client socket for commands
2005-07-07 23:00:11 +02:00
private Thread socketOwner = null ;
2005-10-09 06:43:07 +02:00
String host = null ;
2005-04-07 21:19:42 +02:00
private long timeout ;
private long handle ;
2005-08-07 17:16:33 +02:00
2005-04-07 21:19:42 +02:00
// output and input streams for client control connection
2005-07-07 15:58:54 +02:00
PushbackInputStream clientInput = null ;
2005-06-18 03:48:11 +02:00
private OutputStream clientOutput = null ;
2005-08-07 17:16:33 +02:00
2005-04-07 21:19:42 +02:00
private boolean remoteProxyUse = false ;
private String savedRemoteHost = null ;
2005-10-09 06:43:07 +02:00
String requestPath = null ;
2005-07-04 13:09:48 +02:00
private boolean allowContentEncoding = true ;
2005-09-03 15:40:32 +02:00
static boolean useYacyReferer = true ;
2005-09-05 12:34:34 +02:00
public static boolean yacyDebugMode = false ;
2005-08-07 17:16:33 +02:00
2005-04-12 00:44:40 +02:00
static {
2005-06-18 03:48:11 +02:00
// set time-out of InetAddress.getByName cache ttl
java . security . Security . setProperty ( " networkaddress.cache.ttl " , " 60 " ) ;
2005-09-02 10:21:33 +02:00
java . security . Security . setProperty ( " networkaddress.cache.negative.ttl " , " 0 " ) ;
2005-04-12 00:44:40 +02:00
}
2005-08-07 17:16:33 +02:00
2005-04-20 09:39:40 +02:00
/ * *
* Indicates if the current object was removed from pool because the maximum limit
* was exceeded .
* /
2005-04-19 12:42:48 +02:00
boolean removedFromPool = false ;
2005-08-07 17:16:33 +02:00
2005-04-20 09:39:40 +02:00
// Configuring the httpc object pool
2005-04-19 08:55:57 +02:00
static {
// implementation of session thread pool
GenericObjectPool . Config config = new GenericObjectPool . Config ( ) ;
2005-08-07 17:16:33 +02:00
2005-04-19 08:55:57 +02:00
// The maximum number of active connections that can be allocated from pool at the same time,
// 0 for no limit
config . maxActive = 150 ;
2005-08-07 17:16:33 +02:00
2005-04-19 08:55:57 +02:00
// The maximum number of idle connections connections in the pool
2005-06-18 03:48:11 +02:00
// 0 = no limit.
2005-04-19 08:55:57 +02:00
config . maxIdle = 75 ;
2005-06-18 03:48:11 +02:00
config . minIdle = 10 ;
2005-08-07 17:16:33 +02:00
2005-06-18 03:48:11 +02:00
config . whenExhaustedAction = GenericObjectPool . WHEN_EXHAUSTED_BLOCK ;
config . minEvictableIdleTimeMillis = 30000 ;
2005-08-07 17:16:33 +02:00
2005-06-18 03:48:11 +02:00
theHttpcPool = new httpcPool ( new httpcFactory ( ) , config ) ;
2005-04-19 08:55:57 +02:00
}
2005-08-07 17:16:33 +02:00
2005-04-20 09:39:40 +02:00
/ * *
* A reusable readline buffer
* @see serverByteBuffer
* /
2005-06-17 03:26:51 +02:00
final serverByteBuffer readLineBuffer = new serverByteBuffer ( 100 ) ;
2005-08-07 17:16:33 +02:00
2005-10-05 12:45:33 +02:00
private static final HashMap openSocketLookupTable = new HashMap ( ) ;
2005-08-07 17:16:33 +02:00
2005-08-07 18:27:38 +02:00
/ * *
* Convert the status of this class into an String object to output it .
* /
2005-06-09 11:56:41 +02:00
public String toString ( ) {
2005-06-18 03:48:11 +02:00
return ( this . savedRemoteHost = = null ) ? " Disconnected " : " Connected to " + this . savedRemoteHost +
( ( this . remoteProxyUse ) ? " via " + this . host : " " ) ;
2005-06-09 11:56:41 +02:00
}
2005-08-07 17:16:33 +02:00
2005-08-07 18:27:38 +02:00
/ * *
* This method gets a new httpc instance from the object pool and
* initializes it with the given parameters . Use this method if you have to
* use a proxy to access the pages .
*
* @param server
* @param port
* @param timeout
* @param ssl
* @param remoteProxyHost
* @param remoteProxyPort
* @throws IOException
* @see httpc # init
* /
2005-04-20 09:39:40 +02:00
public static httpc getInstance (
2005-06-18 03:48:11 +02:00
String server ,
int port ,
int timeout ,
2005-04-20 09:39:40 +02:00
boolean ssl ,
2005-06-18 03:48:11 +02:00
String remoteProxyHost ,
2005-04-20 09:39:40 +02:00
int remoteProxyPort
2005-06-18 03:48:11 +02:00
) throws IOException {
2005-08-07 17:16:33 +02:00
2005-04-19 12:42:48 +02:00
httpc newHttpc ;
2005-04-19 08:55:57 +02:00
try {
// fetching a new httpc from the object pool
2005-06-18 03:48:11 +02:00
newHttpc = ( httpc ) httpc . theHttpcPool . borrowObject ( ) ;
2005-04-19 08:55:57 +02:00
} catch ( Exception e ) {
throw new IOException ( " Unable to initialize a new httpc. " + e . getMessage ( ) ) ;
2005-06-18 03:48:11 +02:00
}
2005-08-07 17:16:33 +02:00
2005-04-19 12:42:48 +02:00
// initialize it
2005-08-09 11:07:19 +02:00
try {
newHttpc . init ( server , port , timeout , ssl , remoteProxyHost , remoteProxyPort ) ;
} catch ( IOException e ) {
try { httpc . theHttpcPool . returnObject ( newHttpc ) ; } catch ( Exception e1 ) { }
throw e ;
}
2005-04-19 12:42:48 +02:00
return newHttpc ;
2005-04-19 08:55:57 +02:00
}
2005-08-07 17:16:33 +02:00
2005-08-07 18:27:38 +02:00
/ * *
* This method gets a new httpc instance from the object pool and
* initializes it with the given parameters .
*
* @param server
* @param port
* @param timeout
* @param ssl
* @throws IOException
* @see httpc # init
* /
2005-04-19 08:55:57 +02:00
public static httpc getInstance ( String server , int port , int timeout , boolean ssl ) throws IOException {
2005-08-07 17:16:33 +02:00
2005-04-21 12:31:40 +02:00
httpc newHttpc = null ;
2005-04-26 12:38:35 +02:00
// fetching a new httpc from the object pool
2005-04-19 08:55:57 +02:00
try {
2005-04-21 12:31:40 +02:00
newHttpc = ( httpc ) httpc . theHttpcPool . borrowObject ( ) ;
2005-08-07 17:16:33 +02:00
2005-04-19 08:55:57 +02:00
} catch ( Exception e ) {
2005-05-13 10:30:30 +02:00
throw new IOException ( " Unable to fetch a new httpc from pool. " + e . getMessage ( ) ) ;
2005-06-18 03:48:11 +02:00
}
2005-08-07 17:16:33 +02:00
2005-04-21 12:31:40 +02:00
// initialize it
2005-04-26 12:38:35 +02:00
try {
2005-06-18 03:48:11 +02:00
newHttpc . init ( server , port , timeout , ssl ) ;
2005-04-26 12:38:35 +02:00
} catch ( IOException e ) {
try { httpc . theHttpcPool . returnObject ( newHttpc ) ; } catch ( Exception e1 ) { }
throw e ;
}
2005-04-21 12:31:40 +02:00
return newHttpc ;
2005-08-07 17:16:33 +02:00
2005-06-18 03:48:11 +02:00
}
2005-08-07 17:16:33 +02:00
2005-08-07 18:27:38 +02:00
/ * *
* Put back a used instance into the instance pool of httpc .
*
* @param theHttpc The instance of httpc which should be returned to the pool
* /
2005-04-19 08:55:57 +02:00
public static void returnInstance ( httpc theHttpc ) {
try {
theHttpc . reset ( ) ;
httpc . theHttpcPool . returnObject ( theHttpc ) ;
} catch ( Exception e ) {
2005-06-18 03:48:11 +02:00
// we could ignore this error
2005-04-26 12:38:35 +02:00
}
2005-04-19 08:55:57 +02:00
}
2005-08-07 17:16:33 +02:00
2005-08-07 18:27:38 +02:00
/ * *
* Sets wether the content is allowed to be unzipped while getting ?
* FIXME : The name of this method seems misleading , if I read the usage of
* this method correctly ?
*
* @param status true , if the content is allowed to be decoded on the fly ?
* /
2005-07-04 13:09:48 +02:00
public void setAllowContentEncoding ( boolean status ) {
this . allowContentEncoding = status ;
}
2005-08-07 17:16:33 +02:00
2005-08-07 18:27:38 +02:00
/ * *
* Check wether the connection of this instance is closed .
*
* @return true if the connection is no longer open .
* /
2005-06-23 13:00:26 +02:00
public boolean isClosed ( ) {
if ( this . socket = = null ) return true ;
2005-10-09 06:43:07 +02:00
return ( ! this . socket . isConnected ( ) ) | | ( this . socket . isClosed ( ) ) ;
2005-06-23 13:00:26 +02:00
}
2005-08-07 17:16:33 +02:00
2005-08-07 18:27:38 +02:00
/ * *
* Does an DNS - Check to resolve a hostname to an IP .
*
* @param host Hostname of the host in demand .
* @return String with the ip . null , if the host could not be resolved .
* /
2005-04-13 13:16:31 +02:00
public static String dnsResolve ( String host ) {
2005-04-07 21:19:42 +02:00
String ip = ( String ) nameCacheHit . get ( host ) ;
if ( ip ! = null ) return ip ;
// if (nameCacheMiss.contains(host)) return null;
try {
ip = InetAddress . getByName ( host ) . getHostAddress ( ) ;
if ( ( ip ! = null ) & & ( ! ( ip . equals ( " 127.0.0.1 " ) ) ) & & ( ! ( ip . equals ( " localhost " ) ) ) ) {
nameCacheHit . put ( host , ip ) ;
return ip ;
}
2005-10-09 06:43:07 +02:00
return null ;
2005-04-07 21:19:42 +02:00
} catch ( UnknownHostException e ) {
//nameCacheMiss.add(host);
}
return null ;
}
2005-08-07 17:16:33 +02:00
2005-08-07 18:27:38 +02:00
/ * *
* Checks wether an hostname already is in the DNS - cache .
* FIXME : This method should use dnsResolve , as the code is 90 % identical ?
*
* @param host Searched for hostname .
* @return true , if the hostname already is in the cache .
* /
2005-04-07 21:19:42 +02:00
public static boolean dnsFetch ( String host ) {
if ( ( nameCacheHit . get ( host ) ! = null ) /*|| (nameCacheMiss.contains(host)) */ ) return false ;
try {
String ip = InetAddress . getByName ( host ) . getHostAddress ( ) ;
if ( ( ip ! = null ) & & ( ! ( ip . equals ( " 127.0.0.1 " ) ) ) & & ( ! ( ip . equals ( " localhost " ) ) ) ) {
nameCacheHit . put ( host , ip ) ;
return true ;
}
2005-10-09 06:43:07 +02:00
return false ;
2005-04-07 21:19:42 +02:00
} catch ( UnknownHostException e ) {
//nameCacheMiss.add(host);
return false ;
}
}
2005-06-18 03:48:11 +02:00
2005-08-07 18:27:38 +02:00
/ * *
* Returns the given date in an HTTP - usable format .
*
* @param date The Date - Object to be converted .
* @return String with the date .
* /
2005-06-18 03:48:11 +02:00
public static String dateString ( Date date ) {
if ( date = = null ) return " " ; else return HTTPGMTFormatter . format ( date ) ;
}
2005-08-07 17:16:33 +02:00
2005-08-07 18:27:38 +02:00
/ * *
* Returns the current date as Date - Object .
*
* @return Date - object with the current time .
* /
2005-06-18 03:48:11 +02:00
public static Date nowDate ( ) {
return new GregorianCalendar ( GMTTimeZone ) . getTime ( ) ;
2005-04-19 08:55:57 +02:00
}
2005-08-07 17:16:33 +02:00
2005-08-07 18:27:38 +02:00
// FIXME: Why weren't all static parts put together? They are run one after
// each other on class loading? Hopefully. So why not put them into one
// static block?
2005-06-18 03:48:11 +02:00
static {
// provide system information for client identification
String loc = System . getProperty ( " user.timezone " , " nowhere " ) ;
int p = loc . indexOf ( " / " ) ;
if ( p > 0 ) loc = loc . substring ( 0 , p ) ;
loc = loc + " / " + System . getProperty ( " user.language " , " dumb " ) ;
systemOST =
System . getProperty ( " os.arch " , " no-os-arch " ) + " " + System . getProperty ( " os.name " , " no-os-arch " ) + " " +
System . getProperty ( " os.version " , " no-os-version " ) + " ; " +
" java " + System . getProperty ( " java.version " , " no-java-version " ) + " ; " + loc ;
userAgent = " yacy (www.yacy.net; v " + vDATE + " ; " + systemOST + " ) " ;
}
2005-08-07 18:27:38 +02:00
/ * *
* Initialize the httpc - instance with the given data . This method is used ,
* if you have to use a proxy to access the pages . This just calls init
* without proxy information and adds the proxy information .
*
* @param remoteProxyHost
* @param remoteProxyPort
* @throws IOException
* /
2005-04-19 08:55:57 +02:00
void init ( String server , int port , int timeout , boolean ssl ,
2005-06-18 03:48:11 +02:00
String remoteProxyHost , int remoteProxyPort ) throws IOException {
2005-09-07 11:54:11 +02:00
if ( port = = - 1 ) {
port = ( ssl ) ? 443 : 80 ;
}
2005-06-18 03:48:11 +02:00
this . init ( remoteProxyHost , remoteProxyPort , timeout , ssl ) ;
this . remoteProxyUse = true ;
this . savedRemoteHost = server + ( ( port = = 80 ) ? " " : ( " : " + port ) ) ;
2005-04-07 21:19:42 +02:00
}
2005-08-07 17:16:33 +02:00
2005-08-07 18:27:38 +02:00
/ * *
* Initialize the https - instance with the given data . Opens the sockets to
* the remote server and creats input and output streams .
*
* @param server Hostname of the server to connect to .
* @param port On which port should we connect .
* @param timeout How long do we wait for answers ?
* @param ssl Wether we should use SSL .
* @throws IOException
* /
2005-04-19 08:55:57 +02:00
void init ( String server , int port , int timeout , boolean ssl ) throws IOException {
2005-10-09 06:43:07 +02:00
this . handle = System . currentTimeMillis ( ) ;
2005-04-07 21:19:42 +02:00
//serverLog.logDebug("HTTPC", handle + " initialized");
2005-05-23 12:10:51 +02:00
this . remoteProxyUse = false ;
this . timeout = timeout ;
2005-09-03 15:40:32 +02:00
if ( yacyDebugMode ) {
2005-08-22 16:02:34 +02:00
this . timeout = 60000 ;
}
2005-05-23 12:10:51 +02:00
this . savedRemoteHost = server ;
2005-08-07 17:16:33 +02:00
2005-05-23 12:10:51 +02:00
try {
2005-09-07 11:54:11 +02:00
if ( port = = - 1 ) {
port = ( ssl ) ? 443 : 80 ;
}
2005-05-23 12:10:51 +02:00
this . host = server + ( ( port = = 80 ) ? " " : ( " : " + port ) ) ;
String hostip ;
if ( ( server . equals ( " localhost " ) ) | | ( server . equals ( " 127.0.0.1 " ) ) | | ( server . startsWith ( " 192.168. " ) ) | | ( server . startsWith ( " 10. " ) ) ) {
2005-04-19 08:55:57 +02:00
hostip = server ;
2005-05-23 12:10:51 +02:00
} else {
hostip = dnsResolve ( server ) ;
if ( hostip = = null ) throw new UnknownHostException ( server ) ;
}
2005-08-07 17:16:33 +02:00
2005-08-19 08:47:34 +02:00
// creating a socket
this . socket = ( ssl ) ? SSLSocketFactory . getDefault ( ) . createSocket ( )
: new Socket ( ) ;
// creating a socket address
InetSocketAddress address = new InetSocketAddress ( hostip , port ) ;
2005-08-07 17:16:33 +02:00
2005-08-19 10:23:12 +02:00
// trying to establish a connection to the address
2005-08-19 08:47:34 +02:00
this . socket . connect ( address , timeout ) ;
2005-07-07 15:58:54 +02:00
// registering the socket
2005-08-19 08:47:34 +02:00
this . socketOwner = this . registerOpenSocket ( this . socket ) ;
2005-08-07 17:16:33 +02:00
2005-07-07 15:58:54 +02:00
// setting socket timeout and keep alive behaviour
2005-08-19 08:47:34 +02:00
this . socket . setSoTimeout ( timeout ) ; // waiting time for read
//socket.setSoLinger(true, timeout);
this . socket . setKeepAlive ( true ) ; //
2005-08-07 17:16:33 +02:00
2005-07-07 15:58:54 +02:00
// getting input and output streams
2005-08-19 08:47:34 +02:00
this . clientInput = new PushbackInputStream ( this . socket . getInputStream ( ) ) ;
this . clientOutput = this . socket . getOutputStream ( ) ;
2005-05-23 12:10:51 +02:00
// if we reached this point, we should have a connection
} catch ( UnknownHostException e ) {
2005-07-07 15:58:54 +02:00
if ( this . socket ! = null ) {
2005-08-19 08:47:34 +02:00
httpc . unregisterOpenSocket ( this . socket , this . socketOwner ) ;
2005-07-07 15:58:54 +02:00
}
this . socket = null ;
2005-07-07 23:00:11 +02:00
this . socketOwner = null ;
2005-05-23 12:10:51 +02:00
throw new IOException ( " unknown host: " + server ) ;
2005-08-02 18:03:35 +02:00
}
2005-04-07 21:19:42 +02:00
}
2005-08-07 17:16:33 +02:00
2005-08-07 18:27:38 +02:00
/ * *
* This method resets an httpc - instance , so that it can be used for the next
* connection . This is called before the instance is put back to the pool .
* All streams and sockets are closed and set to null .
* /
2005-06-18 03:48:11 +02:00
void reset ( ) {
if ( this . clientInput ! = null ) {
try { this . clientInput . close ( ) ; } catch ( Exception e ) { }
this . clientInput = null ;
}
if ( this . clientOutput ! = null ) {
try { this . clientOutput . close ( ) ; } catch ( Exception e ) { }
this . clientOutput = null ;
}
if ( this . socket ! = null ) {
try { this . socket . close ( ) ; } catch ( Exception e ) { }
2005-10-09 06:43:07 +02:00
httpc . unregisterOpenSocket ( this . socket , this . socketOwner ) ;
2005-06-18 03:48:11 +02:00
this . socket = null ;
2005-07-07 23:00:11 +02:00
this . socketOwner = null ;
2005-06-18 03:48:11 +02:00
}
2005-08-07 17:16:33 +02:00
2005-06-18 03:48:11 +02:00
this . host = null ;
this . timeout = 0 ;
this . handle = 0 ;
2005-08-07 17:16:33 +02:00
2005-06-18 03:48:11 +02:00
this . remoteProxyUse = false ;
this . savedRemoteHost = null ;
this . requestPath = null ;
2005-08-07 17:16:33 +02:00
2005-07-04 13:09:48 +02:00
this . allowContentEncoding = true ;
2005-08-07 17:16:33 +02:00
2005-06-18 03:48:11 +02:00
// shrink readlinebuffer if it is to large
this . readLineBuffer . reset ( 80 ) ;
2005-04-07 21:19:42 +02:00
}
2005-08-07 18:27:38 +02:00
/ * *
* Just calls reset to close all connections .
* /
2005-06-18 03:48:11 +02:00
public void close ( ) {
reset ( ) ;
2005-04-07 21:19:42 +02:00
}
2005-08-07 18:27:38 +02:00
/ * *
* If this instance is garbage - collected we check if the object was returned
* to the pool . if not , we invalidate the object from the pool .
*
* @see httpcPool # invalidateObject
* /
2005-06-18 03:48:11 +02:00
protected void finalize ( ) throws Throwable {
if ( ! ( this . removedFromPool ) ) {
System . err . println ( " Httpc object was not returned to object pool. " ) ;
httpc . theHttpcPool . invalidateObject ( this ) ;
}
this . reset ( ) ;
2005-04-07 21:19:42 +02:00
}
2005-08-10 23:50:17 +02:00
/ * *
* This method invokes a call to the given server .
*
* @param method Which method should be called ? GET , POST , HEAD or CONNECT
* @param path String with the path on the server to be get .
* @param header The prefilled header ( if available ) from the calling
* browser .
* @param zipped Is encoded content ( gzip ) allowed or not ?
* @throws IOException
* /
2005-04-07 21:19:42 +02:00
private void send ( String method , String path , httpHeader header , boolean zipped ) throws IOException {
2005-06-09 11:56:41 +02:00
// scheduled request through request-response objects/threads
2005-08-07 17:16:33 +02:00
2005-06-09 11:56:41 +02:00
// check and correct path
if ( ( path = = null ) | | ( path . length ( ) = = 0 ) ) path = " / " ;
2005-08-07 17:16:33 +02:00
2005-06-09 11:56:41 +02:00
// for debuggug:
2005-10-09 06:43:07 +02:00
this . requestPath = path ;
2005-08-07 17:16:33 +02:00
2005-06-09 11:56:41 +02:00
// prepare header
if ( header = = null ) header = new httpHeader ( ) ;
2005-08-07 17:16:33 +02:00
2005-04-07 21:19:42 +02:00
// set some standard values
2005-05-13 11:14:12 +02:00
if ( ! ( header . containsKey ( httpHeader . ACCEPT ) ) )
header . put ( httpHeader . ACCEPT , " text/xml,application/xml,application/xhtml+xml,text/html;q=0.9,text/plain;q=0.8,image/png,*/*;q=0.5 " ) ;
if ( ! ( header . containsKey ( httpHeader . ACCEPT_CHARSET ) ) )
header . put ( httpHeader . ACCEPT_CHARSET , " ISO-8859-1,utf-8;q=0.7,*;q=0.7 " ) ;
if ( ! ( header . containsKey ( httpHeader . ACCEPT_LANGUAGE ) ) )
header . put ( httpHeader . ACCEPT_LANGUAGE , " en-us,en;q=0.5 " ) ;
if ( ! ( header . containsKey ( httpHeader . KEEP_ALIVE ) ) )
header . put ( httpHeader . KEEP_ALIVE , " 300 " ) ;
2005-08-07 17:16:33 +02:00
2005-06-09 11:56:41 +02:00
// set user agent. The user agent is only set if the value does not yet exists.
// this gives callers the opportunity, to change the user agent themselves, and
// it will not be changed.
if ( ! ( header . containsKey ( httpHeader . USER_AGENT ) ) ) header . put ( httpHeader . USER_AGENT , userAgent ) ;
2005-08-07 17:16:33 +02:00
2005-06-09 11:56:41 +02:00
// set the host attribute. This is in particular necessary, if we contact another proxy
// the host is mandatory, if we use HTTP/1.1
if ( ! ( header . containsKey ( httpHeader . HOST ) ) ) {
if ( this . remoteProxyUse )
2005-10-09 06:43:07 +02:00
header . put ( httpHeader . HOST , this . savedRemoteHost ) ;
2005-06-09 11:56:41 +02:00
else
header . put ( httpHeader . HOST , this . host ) ;
}
2005-08-07 17:16:33 +02:00
2005-06-09 11:56:41 +02:00
if ( ! ( header . containsKey ( httpHeader . CONNECTION ) ) ) {
header . put ( httpHeader . CONNECTION , " close " ) ;
}
2005-08-07 17:16:33 +02:00
2005-06-09 11:56:41 +02:00
// advertise a little bit...
2005-09-02 10:21:33 +02:00
if ( ( ! ( header . containsKey ( httpHeader . REFERER ) ) ) | | ( ( ( String ) header . get ( httpHeader . REFERER ) ) . trim ( ) . length ( ) = = 0 ) & & useYacyReferer ) {
2005-05-13 11:14:12 +02:00
header . put ( httpHeader . REFERER ,
2005-06-09 11:56:41 +02:00
( ( ( System . currentTimeMillis ( ) > > 10 ) & 1 ) = = 0 ) ?
2005-06-18 03:48:11 +02:00
" http://www.anomic.de " :
" http://www.yacy.net/yacy " ) ;
2005-04-07 21:19:42 +02:00
}
2005-08-07 17:16:33 +02:00
2005-06-09 11:56:41 +02:00
// stimulate zipping or not
// we can unzip, and we will return it always as unzipped, unless not wanted
if ( header . containsKey ( httpHeader . ACCEPT_ENCODING ) ) {
String encoding = ( String ) header . get ( httpHeader . ACCEPT_ENCODING ) ;
if ( zipped ) {
if ( encoding . indexOf ( " gzip " ) < 0 ) {
// add the gzip encoding
//System.out.println("!!! adding gzip encoding");
header . put ( httpHeader . ACCEPT_ENCODING , " gzip,deflate " + ( ( encoding . length ( ) = = 0 ) ? " " : ( " ; " + encoding ) ) ) ;
}
} else {
int pos = encoding . indexOf ( " gzip " ) ;
if ( pos > = 0 ) {
// remove the gzip encoding
//System.out.println("!!! removing gzip encoding");
header . put ( httpHeader . ACCEPT_ENCODING , encoding . substring ( 0 , pos ) + encoding . substring ( pos + 4 ) ) ;
}
}
} else {
if ( zipped ) header . put ( httpHeader . ACCEPT_ENCODING , " gzip,deflate " ) ;
}
2005-08-07 17:16:33 +02:00
2005-06-09 11:56:41 +02:00
//header = new httpHeader(); header.put("Host", this.host); // debug
2005-08-07 17:16:33 +02:00
2005-06-09 11:56:41 +02:00
// send request
if ( ( this . remoteProxyUse ) & & ( ! ( method . equals ( httpHeader . METHOD_CONNECT ) ) ) )
path = " http:// " + this . savedRemoteHost + path ;
2005-10-09 06:43:07 +02:00
serverCore . send ( this . clientOutput , method + " " + path + " HTTP/1.0 " ) ; // if set to HTTP/1.1, servers give time-outs?
2005-08-07 17:16:33 +02:00
2005-04-07 21:19:42 +02:00
// send header
2005-06-09 11:56:41 +02:00
//System.out.println("***HEADER for path " + path + ": PROXY TO SERVER = " + header.toString()); // DEBUG
Iterator i = header . keySet ( ) . iterator ( ) ;
String key ;
int count ;
2005-04-30 03:22:46 +02:00
char tag ;
2005-06-09 11:56:41 +02:00
while ( i . hasNext ( ) ) {
key = ( String ) i . next ( ) ;
2005-04-30 03:22:46 +02:00
tag = key . charAt ( 0 ) ;
if ( ( tag ! = '*' ) & & ( tag ! = '#' ) ) {
count = header . keyCount ( key ) ;
for ( int j = 0 ; j < count ; j + + ) {
2005-06-09 11:56:41 +02:00
serverCore . send ( this . clientOutput , key + " : " + ( ( String ) header . getSingle ( key , j ) ) . trim ( ) ) ;
2005-04-30 03:22:46 +02:00
}
//System.out.println("#" + key + ": " + value);
2005-06-09 11:56:41 +02:00
}
}
2005-08-07 17:16:33 +02:00
2005-06-09 11:56:41 +02:00
// send terminating line
serverCore . send ( this . clientOutput , " " ) ;
this . clientOutput . flush ( ) ;
2005-08-07 17:16:33 +02:00
2005-06-09 11:56:41 +02:00
// this is the place where www.stern.de refuses to answer ..???
2005-04-07 21:19:42 +02:00
}
2005-08-07 17:16:33 +02:00
2005-08-10 23:50:17 +02:00
/ * *
* This method GETs a page from the server .
*
* @param path The path to the page which should be GET .
* @param requestHeader Prefilled httpHeader .
* @param return Instance of response with the content .
* @throws IOException
* /
2005-04-07 21:19:42 +02:00
public response GET ( String path , httpHeader requestHeader ) throws IOException {
//serverLog.logDebug("HTTPC", handle + " requested GET '" + path + "', time = " + (System.currentTimeMillis() - handle));
2005-06-09 11:56:41 +02:00
try {
2005-07-04 13:25:25 +02:00
boolean zipped = ( ! this . allowContentEncoding ) ? false : httpd . shallTransportZipped ( path ) ;
2005-06-09 11:56:41 +02:00
send ( httpHeader . METHOD_GET , path , requestHeader , zipped ) ;
response r = new response ( zipped ) ;
2005-04-07 21:19:42 +02:00
//serverLog.logDebug("HTTPC", handle + " returned GET '" + path + "', time = " + (System.currentTimeMillis() - handle));
return r ;
2005-06-09 11:56:41 +02:00
} catch ( Exception e ) {
throw new IOException ( e . getMessage ( ) ) ;
}
2005-04-07 21:19:42 +02:00
}
2005-08-07 17:16:33 +02:00
2005-08-10 23:50:17 +02:00
/ * *
* This method gets only the header of a page .
*
* @param path The path to the page whose header should be get .
* @param requestHeader Prefilled httpHeader .
* @param return Instance of response with the content .
* @throws IOException
* /
2005-04-07 21:19:42 +02:00
public response HEAD ( String path , httpHeader requestHeader ) throws IOException {
2005-06-18 03:48:11 +02:00
try {
send ( httpHeader . METHOD_HEAD , path , requestHeader , false ) ;
return new response ( false ) ;
// in this case the caller should not read the response body,
// since there is none...
} catch ( SocketException e ) {
throw new IOException ( e . getMessage ( ) ) ;
}
2005-04-07 21:19:42 +02:00
}
2005-08-07 17:16:33 +02:00
2005-08-10 23:50:17 +02:00
/ * *
* This method POSTs some data to a page .
*
* @param path The path to the page which the post is sent to .
* @param requestHeader Prefilled httpHeader .
* @param ins InputStream with the data to be posted to the server .
* @param return Instance of response with the content .
* @throws IOException
* /
2005-04-07 21:19:42 +02:00
public response POST ( String path , httpHeader requestHeader , InputStream ins ) throws IOException {
2005-06-18 03:48:11 +02:00
try {
send ( httpHeader . METHOD_POST , path , requestHeader , false ) ;
// if there is a body to the call, we would have a CONTENT-LENGTH tag in the requestHeader
String cl = ( String ) requestHeader . get ( httpHeader . CONTENT_LENGTH ) ;
2005-04-07 21:19:42 +02:00
int len , c ;
byte [ ] buffer = new byte [ 512 ] ;
2005-06-18 03:48:11 +02:00
if ( cl ! = null ) {
len = Integer . parseInt ( cl ) ;
// transfer len bytes from ins to the server
while ( ( len > 0 ) & & ( ( c = ins . read ( buffer ) ) > = 0 ) ) {
2005-10-09 06:43:07 +02:00
this . clientOutput . write ( buffer , 0 , c ) ;
2005-04-07 21:19:42 +02:00
len - = c ;
}
2005-06-18 03:48:11 +02:00
} else {
2005-04-07 21:19:42 +02:00
len = 0 ;
2005-06-18 03:48:11 +02:00
while ( ( c = ins . read ( buffer ) ) > = 0 ) {
2005-10-09 06:43:07 +02:00
this . clientOutput . write ( buffer , 0 , c ) ;
2005-04-07 21:19:42 +02:00
len + = c ;
}
2005-08-03 04:02:39 +02:00
requestHeader . put ( httpHeader . CONTENT_LENGTH , Integer . toString ( len ) ) ;
2005-04-07 21:19:42 +02:00
}
2005-10-09 06:43:07 +02:00
this . clientOutput . flush ( ) ;
2005-06-18 03:48:11 +02:00
return new response ( false ) ;
} catch ( SocketException e ) {
throw new IOException ( e . getMessage ( ) ) ;
}
2005-04-07 21:19:42 +02:00
}
2005-08-07 17:16:33 +02:00
2005-08-10 23:50:17 +02:00
/ * *
* Call the server with the CONNECT - method .
*
* @param host To which host should a connection be made ?
* @param port Which port should be connected ?
* @param requestHeader prefilled httpHeader .
* @return Instance of response with the content .
* /
2005-04-07 21:19:42 +02:00
public response CONNECT ( String host , int port , httpHeader requestHeader ) throws IOException {
2005-06-18 03:48:11 +02:00
try {
send ( httpHeader . METHOD_CONNECT , host + " : " + port , requestHeader , false ) ;
return new response ( false ) ;
} catch ( SocketException e ) {
throw new IOException ( e . getMessage ( ) ) ;
}
2005-04-07 21:19:42 +02:00
}
2005-08-07 17:16:33 +02:00
2005-08-10 23:50:17 +02:00
/ * *
* This method sends several files at once via a POST request . Only those
* files in the Hashtable files are written whose names are contained in
* args .
*
* @param path The path to the page which the post is sent to .
* @param requestHeader Prefilled httpHeader .
* @param args serverObjects with the names of the files to send .
2005-10-05 12:45:33 +02:00
* @param files HashMap with the names of the files as key and the content
2005-08-10 23:50:17 +02:00
* of the files as value .
* @return Instance of response with the content .
* @throws IOException
* /
2005-10-05 12:45:33 +02:00
public response POST ( String path , httpHeader requestHeader , serverObjects args , HashMap files ) throws IOException {
2005-06-18 03:48:11 +02:00
// make shure, the header has a boundary information like
// CONTENT-TYPE=multipart/form-data; boundary=----------0xKhTmLbOuNdArY
if ( requestHeader = = null ) requestHeader = new httpHeader ( ) ;
String boundary = ( String ) requestHeader . get ( httpHeader . CONTENT_TYPE ) ;
if ( boundary = = null ) {
// create a boundary
boundary = " multipart/form-data; boundary=---------- " + java . lang . System . currentTimeMillis ( ) ;
requestHeader . put ( httpHeader . CONTENT_TYPE , boundary ) ;
}
// extract the boundary string
int pos = boundary . toUpperCase ( ) . indexOf ( " BOUNDARY= " ) ;
if ( pos < 0 ) {
// again, create a boundary
boundary = " multipart/form-data; boundary=---------- " + java . lang . System . currentTimeMillis ( ) ;
requestHeader . put ( httpHeader . CONTENT_TYPE , boundary ) ;
pos = boundary . indexOf ( " boundary= " ) ;
}
boundary = " -- " + boundary . substring ( pos + " boundary= " . length ( ) ) ;
2005-08-07 17:16:33 +02:00
2005-09-22 12:30:55 +02:00
boolean zipContent = args . containsKey ( GZIP_POST_BODY ) ;
2005-09-23 01:05:34 +02:00
args . remove ( GZIP_POST_BODY ) ;
2005-09-22 12:30:55 +02:00
OutputStream out ;
GZIPOutputStream zippedOut ;
2005-10-05 12:45:33 +02:00
serverByteBuffer buf = new serverByteBuffer ( ) ;
2005-09-22 12:30:55 +02:00
if ( zipContent ) {
zippedOut = new GZIPOutputStream ( buf ) ;
out = zippedOut ;
} else {
out = buf ;
}
2005-06-18 03:48:11 +02:00
// in contrast to GET and HEAD, this method also transports a message body
// the body consists of repeated boundaries and values in between
if ( args . size ( ) ! = 0 ) {
// we have values for the POST, start with one boundary
String key , value ;
Enumeration e = args . keys ( ) ;
while ( e . hasMoreElements ( ) ) {
// start with a boundary
2005-09-22 12:30:55 +02:00
out . write ( boundary . getBytes ( ) ) ;
out . write ( serverCore . crlf ) ;
2005-04-07 21:19:42 +02:00
// write value
2005-06-18 03:48:11 +02:00
key = ( String ) e . nextElement ( ) ;
2005-10-05 12:45:33 +02:00
value = args . get ( key , " " ) ;
2005-06-18 03:48:11 +02:00
if ( ( files ! = null ) & & ( files . containsKey ( key ) ) ) {
// we are about to write a file
2005-09-22 12:30:55 +02:00
out . write ( ( " Content-Disposition: form-data; name= " + '"' + key + '"' + " ; filename= " + '"' + value + '"' ) . getBytes ( ) ) ;
out . write ( serverCore . crlf ) ;
out . write ( serverCore . crlf ) ;
out . write ( ( byte [ ] ) files . get ( key ) ) ;
out . write ( serverCore . crlf ) ;
2005-06-18 03:48:11 +02:00
} else {
// write a single value
2005-09-22 12:30:55 +02:00
out . write ( ( " Content-Disposition: form-data; name= " + '"' + key + '"' ) . getBytes ( ) ) ;
out . write ( serverCore . crlf ) ;
out . write ( serverCore . crlf ) ;
out . write ( value . getBytes ( ) ) ;
out . write ( serverCore . crlf ) ;
2005-04-07 21:19:42 +02:00
}
2005-06-18 03:48:11 +02:00
}
// finish with a boundary
2005-09-22 12:30:55 +02:00
out . write ( boundary . getBytes ( ) ) ;
out . write ( serverCore . crlf ) ;
2005-06-18 03:48:11 +02:00
//buf.write("" + serverCore.crlfString);
}
2005-04-07 21:19:42 +02:00
// create body array
2005-09-22 12:30:55 +02:00
out . close ( ) ;
2005-04-07 21:19:42 +02:00
byte [ ] body = buf . toByteArray ( ) ;
2005-10-05 12:45:33 +02:00
buf = null ; out = null ;
2005-04-07 21:19:42 +02:00
//System.out.println("DEBUG: PUT BODY=" + new String(body));
2005-09-22 12:30:55 +02:00
if ( zipContent ) {
requestHeader . put ( httpHeader . CONTENT_ENCODING , " gzip " ) ;
} else {
// size of that body
requestHeader . put ( httpHeader . CONTENT_LENGTH , Integer . toString ( body . length ) ) ;
}
2005-06-18 03:48:11 +02:00
// send the header
//System.out.println("header=" + requestHeader);
send ( httpHeader . METHOD_POST , path , requestHeader , false ) ;
// send the body
//System.out.println("body=" + buf.toString());
2005-10-09 06:43:07 +02:00
serverCore . send ( this . clientOutput , body ) ;
2005-08-07 17:16:33 +02:00
2005-06-18 03:48:11 +02:00
return new response ( false ) ;
2005-04-07 21:19:42 +02:00
}
2005-08-07 17:16:33 +02:00
2005-04-07 21:19:42 +02:00
/ *
DEBUG : PUT BODY = - - - - - - - - - - - - 1090358578442
Content - Disposition : form - data ; name = " youare "
2005-08-07 17:16:33 +02:00
2005-04-07 21:19:42 +02:00
Ty2F86ekSWM5
- - - - - - - - - - - - 1090358578442
Content - Disposition : form - data ; name = " key "
2005-08-07 17:16:33 +02:00
2005-04-07 21:19:42 +02:00
6EkPPOl7
- - - - - - - - - - - - 1090358578442
Content - Disposition : form - data ; name = " iam "
2005-08-07 17:16:33 +02:00
2005-04-07 21:19:42 +02:00
HnTvzwV7SCJR
- - - - - - - - - - - - 1090358578442
Content - Disposition : form - data ; name = " process "
2005-08-07 17:16:33 +02:00
2005-04-07 21:19:42 +02:00
permission
- - - - - - - - - - - - 1090358578442
2005-08-07 17:16:33 +02:00
2005-04-07 21:19:42 +02:00
* /
2005-08-07 17:16:33 +02:00
2005-04-07 21:19:42 +02:00
/ *
- - - - - - - - - - - - 0xKhTmLbOuNdArY
Content - Disposition : form - data ; name = " file1 " ; filename = " dir.gif "
Content - Type : image / gif
2005-08-07 17:16:33 +02:00
2005-04-07 21:19:42 +02:00
GIF89
- - - - - - - - - - - - 0xKhTmLbOuNdArY
Content - Disposition : form - data ; name = " file2 " ; filename = " "
2005-08-07 17:16:33 +02:00
2005-04-07 21:19:42 +02:00
- - - - - - - - - - - - 0xKhTmLbOuNdArY
Content - Disposition : form - data ; name = " upload "
2005-08-07 17:16:33 +02:00
2005-04-07 21:19:42 +02:00
do upload
- - - - - - - - - - - - 0xKhTmLbOuNdArY - -
2005-08-07 17:16:33 +02:00
2005-04-07 21:19:42 +02:00
# # # # # # Listing Properties # # # # # #
# METHOD = POST
# # # Header Values :
# EXT = html
# HTTP = HTTP / 1 . 1
# ACCEPT - ENCODING = gzip , deflate ; q = 1 . 0 , identity ; q = 0 . 5 , * ; q = 0
# HOST = localhost : 8080
# PATH = / testcgi / doit . html
# CONTENT - LENGTH = 474
# CONTENT - TYPE = multipart / form - data ; boundary = - - - - - - - - - - 0xKhTmLbOuNdArY
# ARGC = 0
# CONNECTION = close
# USER - AGENT = Mozilla / 5 . 0 ( Macintosh ; U ; PPC Mac OS X ; de - de ) AppleWebKit / 103u ( KHTML , like Gecko ) Safari / 100 . 1
# # # Call Properties :
# # # # # # End OfList # # # # # #
2005-06-18 03:48:11 +02:00
* /
2005-08-07 17:16:33 +02:00
2005-04-07 21:19:42 +02:00
public static byte [ ] singleGET ( String host , int port , String path , int timeout ,
2005-06-18 03:48:11 +02:00
String user , String password , boolean ssl ,
String proxyHost , int proxyPort ,
httpHeader requestHeader ) throws IOException {
if ( requestHeader = = null ) requestHeader = new httpHeader ( ) ;
if ( ( user ! = null ) & & ( password ! = null ) & & ( user . length ( ) ! = 0 ) ) {
requestHeader . put ( httpHeader . AUTHORIZATION , serverCodings . standardCoder . encodeBase64String ( user + " : " + password ) ) ;
}
2005-08-07 17:16:33 +02:00
2005-04-19 08:55:57 +02:00
httpc con = null ;
2005-06-18 03:48:11 +02:00
try {
2005-08-07 17:16:33 +02:00
2005-04-19 08:55:57 +02:00
if ( ( proxyHost = = null ) | | ( proxyPort = = 0 ) ) {
con = httpc . getInstance ( host , port , timeout , ssl ) ;
2005-06-18 03:48:11 +02:00
} else {
2005-04-19 08:55:57 +02:00
con = httpc . getInstance ( host , port , timeout , ssl , proxyHost , proxyPort ) ;
}
2005-08-07 17:16:33 +02:00
2005-06-18 03:48:11 +02:00
httpc . response res = con . GET ( path , null ) ;
if ( res . status . startsWith ( " 2 " ) ) {
2005-10-05 12:45:33 +02:00
return res . writeContent ( ) ;
2005-06-18 03:48:11 +02:00
}
2005-10-09 06:43:07 +02:00
return res . status . getBytes ( ) ;
2005-04-19 08:55:57 +02:00
} catch ( Exception e ) {
throw new IOException ( e . getMessage ( ) ) ;
} finally {
if ( con ! = null ) httpc . returnInstance ( con ) ;
}
2005-08-07 17:16:33 +02:00
2005-04-07 21:19:42 +02:00
}
2005-08-07 17:16:33 +02:00
2005-04-07 21:19:42 +02:00
public static byte [ ] singleGET ( URL u , int timeout ,
2005-06-18 03:48:11 +02:00
String user , String password ,
String proxyHost , int proxyPort ) throws IOException {
int port = u . getPort ( ) ;
2005-04-07 21:19:42 +02:00
boolean ssl = u . getProtocol ( ) . equals ( " https " ) ;
2005-06-18 03:48:11 +02:00
if ( port < 0 ) port = ( ssl ) ? 443 : 80 ;
String path = u . getPath ( ) ;
String query = u . getQuery ( ) ;
if ( ( query ! = null ) & & ( query . length ( ) > 0 ) ) path = path + " ? " + query ;
return singleGET ( u . getHost ( ) , port , path , timeout , user , password , ssl , proxyHost , proxyPort , null ) ;
2005-04-07 21:19:42 +02:00
}
2005-08-07 17:16:33 +02:00
2005-04-07 21:19:42 +02:00
/ *
public static byte [ ] singleGET ( String url , int timeout ) throws IOException {
2005-06-18 03:48:11 +02:00
try {
return singleGET ( new URL ( url ) , timeout , null , null , null , 0 ) ;
} catch ( MalformedURLException e ) {
throw new IOException ( " Malformed URL: " + e . getMessage ( ) ) ;
}
2005-04-07 21:19:42 +02:00
}
2005-06-18 03:48:11 +02:00
* /
2005-08-07 17:16:33 +02:00
2005-04-07 21:19:42 +02:00
public static byte [ ] singlePOST ( String host , int port , String path , int timeout ,
2005-06-18 03:48:11 +02:00
String user , String password , boolean ssl ,
String proxyHost , int proxyPort ,
httpHeader requestHeader , serverObjects props ) throws IOException {
2005-08-07 17:16:33 +02:00
2005-06-18 03:48:11 +02:00
if ( requestHeader = = null ) requestHeader = new httpHeader ( ) ;
if ( ( user ! = null ) & & ( password ! = null ) & & ( user . length ( ) ! = 0 ) ) {
requestHeader . put ( httpHeader . AUTHORIZATION , serverCodings . standardCoder . encodeBase64String ( user + " : " + password ) ) ;
}
2005-08-07 17:16:33 +02:00
2005-06-18 03:48:11 +02:00
httpc con = null ;
2005-04-19 08:55:57 +02:00
try {
2005-06-18 03:48:11 +02:00
if ( ( proxyHost = = null ) | | ( proxyPort = = 0 ) )
con = httpc . getInstance ( host , port , timeout , ssl ) ;
else
con = httpc . getInstance ( host , port , timeout , ssl , proxyHost , proxyPort ) ;
httpc . response res = con . POST ( path , null , props , null ) ;
2005-08-07 17:16:33 +02:00
2005-06-18 03:48:11 +02:00
//System.out.println("response=" + res.toString());
if ( res . status . startsWith ( " 2 " ) ) {
2005-10-05 12:45:33 +02:00
return res . writeContent ( ) ;
2005-06-18 03:48:11 +02:00
}
2005-10-09 06:43:07 +02:00
return res . status . getBytes ( ) ;
2005-04-19 08:55:57 +02:00
} catch ( Exception e ) {
throw new IOException ( e . getMessage ( ) ) ;
} finally {
if ( con ! = null ) httpc . returnInstance ( con ) ;
}
2005-08-07 17:16:33 +02:00
2005-04-07 21:19:42 +02:00
}
2005-08-07 17:16:33 +02:00
2005-04-07 21:19:42 +02:00
public static byte [ ] singlePOST ( URL u , int timeout ,
2005-06-18 03:48:11 +02:00
String user , String password ,
String proxyHost , int proxyPort ,
serverObjects props ) throws IOException {
int port = u . getPort ( ) ;
2005-04-07 21:19:42 +02:00
boolean ssl = u . getProtocol ( ) . equals ( " https " ) ;
if ( port < 0 ) port = ( ssl ) ? 443 : 80 ;
2005-06-18 03:48:11 +02:00
String path = u . getPath ( ) ;
String query = u . getQuery ( ) ;
if ( ( query ! = null ) & & ( query . length ( ) > 0 ) ) path = path + " ? " + query ;
return singlePOST ( u . getHost ( ) , port , path , timeout , user , password , ssl , proxyHost , proxyPort , null , props ) ;
2005-04-07 21:19:42 +02:00
}
2005-08-07 17:16:33 +02:00
2005-04-07 21:19:42 +02:00
public static byte [ ] singlePOST ( String url , int timeout , serverObjects props ) throws IOException {
2005-06-18 03:48:11 +02:00
try {
return singlePOST ( new URL ( url ) , timeout , null , null , null , 0 , props ) ;
} catch ( MalformedURLException e ) {
throw new IOException ( " Malformed URL: " + e . getMessage ( ) ) ;
}
2005-04-07 21:19:42 +02:00
}
2005-08-07 17:16:33 +02:00
2005-10-05 12:45:33 +02:00
public static ArrayList wget ( URL url , int timeout , String user , String password , String proxyHost , int proxyPort ) throws IOException {
2005-06-18 03:48:11 +02:00
// splitting of the byte array into lines
byte [ ] a = singleGET ( url , timeout , user , password , proxyHost , proxyPort ) ;
2005-04-07 21:19:42 +02:00
if ( a = = null ) return null ;
2005-06-18 03:48:11 +02:00
int s = 0 ;
int e ;
2005-10-05 12:45:33 +02:00
ArrayList v = new ArrayList ( ) ;
2005-06-18 03:48:11 +02:00
while ( s < a . length ) {
e = s ; while ( e < a . length ) if ( a [ e + + ] < 32 ) { e - - ; break ; }
v . add ( new String ( a , s , e - s ) ) ;
s = e ; while ( s < a . length ) if ( a [ s + + ] > = 32 ) { s - - ; break ; }
}
return v ;
2005-04-07 21:19:42 +02:00
}
2005-08-07 17:16:33 +02:00
2005-04-07 21:19:42 +02:00
public static httpHeader whead ( URL url , int timeout , String user , String password , String proxyHost , int proxyPort ) throws IOException {
2005-04-19 08:55:57 +02:00
// generate request header
2005-04-07 21:19:42 +02:00
httpHeader requestHeader = new httpHeader ( ) ;
2005-06-18 03:48:11 +02:00
if ( ( user ! = null ) & & ( password ! = null ) & & ( user . length ( ) ! = 0 ) ) {
requestHeader . put ( httpHeader . AUTHORIZATION , serverCodings . standardCoder . encodeBase64String ( user + " : " + password ) ) ;
}
2005-04-07 21:19:42 +02:00
// parse query
2005-08-07 17:16:33 +02:00
2005-04-07 21:19:42 +02:00
int port = url . getPort ( ) ;
boolean ssl = url . getProtocol ( ) . equals ( " https " ) ;
2005-06-18 03:48:11 +02:00
if ( port < 0 ) port = ( ssl ) ? 443 : 80 ;
String path = url . getPath ( ) ;
String query = url . getQuery ( ) ;
if ( ( query ! = null ) & & ( query . length ( ) > 0 ) ) path = path + " ? " + query ;
String host = url . getHost ( ) ;
2005-08-07 17:16:33 +02:00
2005-04-07 21:19:42 +02:00
// start connection
2005-04-19 08:55:57 +02:00
httpc con = null ;
2005-06-18 03:48:11 +02:00
try {
2005-04-19 08:55:57 +02:00
if ( ( proxyHost = = null ) | | ( proxyPort = = 0 ) )
2005-06-18 03:48:11 +02:00
con = httpc . getInstance ( host , port , timeout , ssl ) ;
2005-04-19 08:55:57 +02:00
else con = httpc . getInstance ( host , port , timeout , ssl , proxyHost , proxyPort ) ;
2005-08-07 17:16:33 +02:00
2005-06-18 03:48:11 +02:00
httpc . response res = con . HEAD ( path , requestHeader ) ;
if ( res . status . startsWith ( " 2 " ) ) {
// success
return res . responseHeader ;
}
2005-10-09 06:43:07 +02:00
// fail
return res . responseHeader ;
2005-04-19 08:55:57 +02:00
} catch ( Exception e ) {
throw new IOException ( e . getMessage ( ) ) ;
} finally {
if ( con ! = null ) httpc . returnInstance ( con ) ;
}
2005-04-07 21:19:42 +02:00
}
2005-08-07 17:16:33 +02:00
2005-04-07 21:19:42 +02:00
/ *
public static Vector wget ( String url ) {
2005-06-18 03:48:11 +02:00
try {
return wget ( new URL ( url ) , 5000 , null , null , null , 0 ) ;
} catch ( IOException e ) {
Vector ll = new Vector ( ) ;
ll . add ( " 503 " + e . getMessage ( ) ) ;
return ll ;
}
2005-04-07 21:19:42 +02:00
}
2005-06-18 03:48:11 +02:00
* /
2005-08-07 17:16:33 +02:00
2005-10-05 12:45:33 +02:00
public static ArrayList wput ( URL url , int timeout , String user , String password , String proxyHost , int proxyPort , serverObjects props ) throws IOException {
2005-06-18 03:48:11 +02:00
// splitting of the byte array into lines
byte [ ] a = singlePOST ( url , timeout , user , password , proxyHost , proxyPort , props ) ;
//System.out.println("wput-out=" + new String(a));
int s = 0 ;
int e ;
2005-10-05 12:45:33 +02:00
ArrayList v = new ArrayList ( ) ;
2005-06-18 03:48:11 +02:00
while ( s < a . length ) {
e = s ; while ( e < a . length ) if ( a [ e + + ] < 32 ) { e - - ; break ; }
v . add ( new String ( a , s , e - s ) ) ;
s = e ; while ( s < a . length ) if ( a [ s + + ] > = 32 ) { s - - ; break ; }
}
return v ;
2005-04-07 21:19:42 +02:00
}
2005-08-07 17:16:33 +02:00
2005-04-07 21:19:42 +02:00
/ *
public static Vector wput ( String url , serverObjects props ) {
2005-06-18 03:48:11 +02:00
try {
return wput ( url , 5000 , null , null , null , 0 , props ) ;
} catch ( IOException e ) {
2005-08-15 01:35:18 +02:00
serverLog . logError ( " HTTPC " , " wput exception for URL " + url + " : " + e . getMessage ( ) , e ) ;
2005-06-18 03:48:11 +02:00
Vector ll = new Vector ( ) ;
ll . add ( " 503 " + e . getMessage ( ) ) ;
return ll ;
}
2005-04-07 21:19:42 +02:00
}
2005-06-18 03:48:11 +02:00
* /
2005-08-07 17:16:33 +02:00
2005-04-07 21:19:42 +02:00
public static void main ( String [ ] args ) {
2005-06-18 03:48:11 +02:00
System . out . println ( " ANOMIC.DE HTTP CLIENT v " + vDATE ) ;
String url = args [ 0 ] ;
if ( ! ( url . toUpperCase ( ) . startsWith ( " HTTP:// " ) ) ) url = " http:// " + url ;
2005-10-05 12:45:33 +02:00
ArrayList text = new ArrayList ( ) ;
2005-06-18 03:48:11 +02:00
if ( args . length = = 4 ) {
2005-04-07 21:19:42 +02:00
int timeout = Integer . parseInt ( args [ 1 ] ) ;
String proxyHost = args [ 2 ] ;
int proxyPort = Integer . parseInt ( args [ 3 ] ) ;
try {
text = wget ( new URL ( url ) , timeout , null , null , proxyHost , proxyPort ) ;
} catch ( MalformedURLException e ) {
System . out . println ( " The url ' " + url + " ' is wrong. " ) ;
} catch ( IOException e ) {
System . out . println ( " Error loading url ' " + url + " ': " + e . getMessage ( ) ) ;
}
2005-06-18 03:48:11 +02:00
} / * else {
serverObjects post = new serverObjects ( ) ;
int p ;
for ( int i = 1 ; i < args . length ; i + + ) {
p = args [ i ] . indexOf ( " = " ) ;
if ( p > 0 ) post . put ( args [ i ] . substring ( 0 , p ) , args [ i ] . substring ( p + 1 ) ) ;
}
text = wput ( url , post ) ;
} * /
2005-10-05 12:45:33 +02:00
Iterator i = text . listIterator ( ) ;
while ( i . hasNext ( ) ) System . out . println ( ( String ) i . next ( ) ) ;
2005-04-07 21:19:42 +02:00
}
2005-08-07 17:16:33 +02:00
2005-07-07 15:58:54 +02:00
/ * *
* To register an open socket .
2005-08-07 17:16:33 +02:00
* This adds the socket to the list of open sockets where the current thread
2005-08-10 23:50:17 +02:00
* is the owner .
2005-07-07 15:58:54 +02:00
* @param openedSocket the socket that should be registered
2005-08-07 17:16:33 +02:00
* @return the id of the current thread
2005-07-07 15:58:54 +02:00
* /
2005-07-07 23:00:11 +02:00
private Thread registerOpenSocket ( Socket openedSocket ) {
Thread currentThread = Thread . currentThread ( ) ;
2005-07-07 15:58:54 +02:00
synchronized ( openSocketLookupTable ) {
ArrayList openSockets = null ;
2005-07-07 23:00:11 +02:00
if ( openSocketLookupTable . containsKey ( currentThread ) ) {
openSockets = ( ArrayList ) openSocketLookupTable . get ( currentThread ) ;
2005-07-07 15:58:54 +02:00
} else {
openSockets = new ArrayList ( 1 ) ;
2005-07-07 23:00:11 +02:00
openSocketLookupTable . put ( currentThread , openSockets ) ;
2005-07-07 15:58:54 +02:00
}
synchronized ( openSockets ) {
openSockets . add ( openedSocket ) ;
}
2005-07-07 23:00:11 +02:00
return currentThread ;
2005-08-07 17:16:33 +02:00
}
2005-07-07 15:58:54 +02:00
}
2005-08-07 17:16:33 +02:00
2005-07-07 15:58:54 +02:00
/ * *
* Closing all sockets that were opened in the context of the thread
2005-08-07 17:16:33 +02:00
* with the given thread id
2005-07-07 15:58:54 +02:00
* @param threadId
* /
2005-07-07 23:00:11 +02:00
public static int closeOpenSockets ( Thread thread ) {
2005-08-07 17:16:33 +02:00
// getting all still opened sockets
2005-07-07 23:00:11 +02:00
ArrayList openSockets = ( ArrayList ) httpc . getRegisteredOpenSockets ( thread ) . clone ( ) ;
2005-07-07 15:58:54 +02:00
int closedSocketCount = 0 ;
2005-08-07 17:16:33 +02:00
2005-07-07 23:00:11 +02:00
// looping through the list of sockets and close each one
for ( int socketCount = 0 ; socketCount < openSockets . size ( ) ; socketCount + + ) {
Socket openSocket = ( Socket ) openSockets . get ( 0 ) ;
try {
// closing the socket
if ( ! openSocket . isClosed ( ) ) {
openSocket . close ( ) ;
closedSocketCount + + ;
}
// unregistering the socket
httpc . unregisterOpenSocket ( openSocket , thread ) ;
} catch ( Exception ex ) { }
2005-07-07 15:58:54 +02:00
}
2005-08-07 17:16:33 +02:00
2005-07-07 15:58:54 +02:00
return closedSocketCount ;
}
2005-08-07 17:16:33 +02:00
2005-07-07 15:58:54 +02:00
/ * *
2005-08-07 17:16:33 +02:00
* Unregistering the socket .
2005-07-07 15:58:54 +02:00
* The socket will be removed from the list of sockets where the thread with the
* given thread id is the owner .
* @param closedSocket the socket that should be unregistered
* @param threadId the id of the owner thread
* /
2005-07-07 23:00:11 +02:00
public static void unregisterOpenSocket ( Socket closedSocket , Thread thread ) {
2005-07-07 15:58:54 +02:00
synchronized ( openSocketLookupTable ) {
ArrayList openSockets = null ;
2005-07-07 23:00:11 +02:00
if ( openSocketLookupTable . containsKey ( thread ) ) {
openSockets = ( ArrayList ) openSocketLookupTable . get ( thread ) ;
2005-07-07 15:58:54 +02:00
synchronized ( openSockets ) {
openSockets . remove ( closedSocket ) ;
if ( openSockets . size ( ) = = 0 ) {
2005-07-07 23:00:11 +02:00
openSocketLookupTable . remove ( thread ) ;
2005-07-07 15:58:54 +02:00
}
}
}
2005-08-07 17:16:33 +02:00
}
2005-07-07 15:58:54 +02:00
}
2005-08-07 17:16:33 +02:00
2005-07-07 15:58:54 +02:00
/ * *
* Getting a list of open sockets where the current thread is
2005-08-07 17:16:33 +02:00
* the owner
2005-07-07 15:58:54 +02:00
* @return the list of open sockets
* /
public static ArrayList getRegisteredOpenSockets ( ) {
2005-07-07 23:00:11 +02:00
Thread currentThread = Thread . currentThread ( ) ;
2005-08-07 17:16:33 +02:00
return getRegisteredOpenSockets ( currentThread ) ;
2005-07-07 15:58:54 +02:00
}
2005-08-07 17:16:33 +02:00
2005-07-07 15:58:54 +02:00
/ * *
* Getting a list of open sockets where the thread with the given
* thread id is the owner
* @param threadId the thread id of the owner thread
* @return the list of open sockets
* /
2005-07-07 23:00:11 +02:00
public static ArrayList getRegisteredOpenSockets ( Thread thread ) {
2005-07-07 15:58:54 +02:00
synchronized ( openSocketLookupTable ) {
ArrayList openSockets = null ;
2005-07-07 23:00:11 +02:00
if ( openSocketLookupTable . containsKey ( thread ) ) {
openSockets = ( ArrayList ) openSocketLookupTable . get ( thread ) ;
2005-07-07 15:58:54 +02:00
} else {
openSockets = new ArrayList ( 0 ) ;
}
return openSockets ;
2005-08-07 17:16:33 +02:00
}
}
2005-08-10 23:50:17 +02:00
/ * *
* Inner Class to get the response of an http - request and parse it .
* /
public final class response {
// Response-Header = Date | Pragma | Allow | Content-Encoding | Content-Length | Content-Type |
// Expires | Last-Modified | HTTP-header
/ *
Status - Line = HTTP - Version SP Status - Code SP Reason - Phrase CRLF
1xx : Informational - Not used , but reserved for future use
2xx : Success - The action was successfully received , understood , and accepted .
3xx : Redirection - Further action must be taken in order to complete the request
4xx : Client Error - The request contains bad syntax or cannot be fulfilled
5xx : Server Error - The server failed to fulfill an apparently valid request
* /
// header information
public httpHeader responseHeader = null ;
public String httpVer = " HTTP/0.9 " ;
public String status ; // the success/failure response string starting with status-code
2005-09-05 10:01:54 +02:00
public int statusCode = 503 ;
public String statusText = " internal error " ;
2005-08-10 23:50:17 +02:00
private boolean gzip ; // for gunzipping on-the-fly
private String encoding ;
/ * *
* Constructor for this class . Reads in the content for the given outer
* instance and parses it .
*
* @param zipped true , if the content of this response is gzipped .
* @throws IOException
* /
public response ( boolean zipped ) throws IOException {
// lets start with worst-case attributes as set-up
2005-10-09 06:43:07 +02:00
this . responseHeader = new httpHeader ( reverseMappingCache ) ;
this . statusCode = 503 ;
this . statusText = " internal httpc error " ;
this . status = Integer . toString ( this . statusCode ) + " " + this . statusText ;
this . gzip = false ;
2005-08-10 23:50:17 +02:00
// check connection status
2005-10-09 06:43:07 +02:00
if ( httpc . this . clientInput = = null ) {
2005-08-10 23:50:17 +02:00
// the server has meanwhile disconnected
2005-10-09 06:43:07 +02:00
this . statusCode = 503 ;
this . statusText = " lost connection to server " ;
this . status = Integer . toString ( this . statusCode ) + " " + this . statusText ;
2005-08-10 23:50:17 +02:00
return ; // in bad mood
}
// reads in the http header, right now, right here
2005-10-09 06:43:07 +02:00
byte [ ] b = serverCore . receive ( httpc . this . clientInput , httpc . this . readLineBuffer , terminalMaxLength , false ) ;
2005-08-10 23:50:17 +02:00
if ( b = = null ) {
// the server has meanwhile disconnected
2005-10-09 06:43:07 +02:00
this . statusCode = 503 ;
this . statusText = " server has closed connection " ;
this . status = Integer . toString ( this . statusCode ) + " " + this . statusText ;
2005-08-10 23:50:17 +02:00
return ; // in bad mood
}
2005-09-05 10:01:54 +02:00
2005-09-20 23:49:47 +02:00
// parsing the response status line
String buffer = new String ( b ) ;
Object [ ] responseInfo = httpHeader . parseResponseLine ( buffer ) ;
this . httpVer = ( String ) responseInfo [ 0 ] ;
this . statusCode = ( ( Integer ) responseInfo [ 1 ] ) . intValue ( ) ;
this . statusText = ( String ) responseInfo [ 2 ] ;
this . status = this . statusCode + " " + this . statusText ;
2005-09-05 10:01:54 +02:00
2005-09-20 23:49:47 +02:00
if ( ( this . statusCode = = 500 ) & & ( this . statusText . equals ( " status line parse error " ) ) ) {
// flush in anything that comes without parsing
2005-10-09 06:43:07 +02:00
while ( ( b ! = null ) & & ( b . length ! = 0 ) ) b = serverCore . receive ( httpc . this . clientInput , httpc . this . readLineBuffer , terminalMaxLength , false ) ;
2005-09-20 23:49:47 +02:00
return ; // in bad mood
2005-09-05 10:01:54 +02:00
}
2005-09-20 23:49:47 +02:00
2005-08-10 23:50:17 +02:00
// check validity
2005-09-20 23:49:47 +02:00
if ( this . statusCode = = 400 ) {
2005-08-10 23:50:17 +02:00
// bad request
// flush in anything that comes without parsing
2005-10-09 06:43:07 +02:00
while ( ( b = serverCore . receive ( httpc . this . clientInput , httpc . this . readLineBuffer , terminalMaxLength , false ) ) . length ! = 0 ) { }
2005-08-10 23:50:17 +02:00
return ; // in bad mood
}
// at this point we should have a valid response. read in the header properties
String key = " " ;
2005-10-09 06:43:07 +02:00
while ( ( b = serverCore . receive ( httpc . this . clientInput , httpc . this . readLineBuffer , terminalMaxLength , false ) ) ! = null ) {
2005-08-10 23:50:17 +02:00
if ( b . length = = 0 ) break ;
buffer = new String ( b ) ;
//System.out.println("#H#" + buffer); // debug
if ( buffer . charAt ( 0 ) < = 32 ) {
// use old entry
if ( key . length ( ) = = 0 ) throw new IOException ( " header corrupted - input error " ) ;
// attach new line
2005-10-09 06:43:07 +02:00
if ( ! ( this . responseHeader . containsKey ( key ) ) ) throw new IOException ( " header corrupted - internal error " ) ;
this . responseHeader . put ( key , ( String ) this . responseHeader . get ( key ) + " " + buffer . trim ( ) ) ;
2005-08-10 23:50:17 +02:00
} else {
// create new entry
2005-09-20 23:49:47 +02:00
int p = buffer . indexOf ( " : " ) ;
2005-08-10 23:50:17 +02:00
if ( p > 0 ) {
2005-10-09 06:43:07 +02:00
this . responseHeader . add ( buffer . substring ( 0 , p ) . trim ( ) , buffer . substring ( p + 1 ) . trim ( ) ) ;
2005-08-10 23:50:17 +02:00
} else {
2005-10-09 06:43:07 +02:00
serverLog . logSevere ( " HTTPC " , " RESPONSE PARSE ERROR: HOST=' " + httpc . this . host + " ', PATH=' " + httpc . this . requestPath + " ', STATUS=' " + this . status + " ' " ) ;
2005-08-30 23:32:59 +02:00
serverLog . logSevere ( " HTTPC " , " ..............BUFFER: " + buffer ) ;
2005-08-10 23:50:17 +02:00
}
}
}
// finished with reading header
// we will now manipulate the header if the content is gzip encoded, because
// reading the content with "writeContent" will gunzip on-the-fly
2005-10-09 06:43:07 +02:00
this . gzip = ( ( zipped ) & & ( this . responseHeader . gzip ( ) ) ) ;
2005-08-10 23:50:17 +02:00
2005-10-09 06:43:07 +02:00
if ( this . gzip ) {
this . responseHeader . remove ( httpHeader . CONTENT_ENCODING ) ; // we fake that we don't have encoding, since what comes out does not have gzip and we also don't know what was encoded
this . responseHeader . remove ( httpHeader . CONTENT_LENGTH ) ; // we cannot use the length during gunzippig yet; still we can hope that it works
2005-08-10 23:50:17 +02:00
}
}
/ * *
* Converts an instance of this class into a readable string .
*
* @return String with some information about this instance .
* /
public String toString ( ) {
StringBuffer toStringBuffer = new StringBuffer ( ) ;
toStringBuffer . append ( ( this . status = = null ) ? " Status: Unknown " : " Status: " + this . status )
. append ( " | Headers: " )
. append ( ( this . responseHeader = = null ) ? " none " : this . responseHeader . toString ( ) ) ;
return toStringBuffer . toString ( ) ;
}
/ * *
* Returns wether this request was successful or not . Stati beginning
* with 2 or 3 are considered successful .
*
* @return True , if the request was successfull .
* /
public boolean success ( ) {
2005-10-09 06:43:07 +02:00
return ( ( this . status . charAt ( 0 ) = = '2' ) | | ( this . status . charAt ( 0 ) = = '3' ) ) ;
2005-08-10 23:50:17 +02:00
}
/ * *
* This method just output the found content into an byte - array and
* returns it .
*
* @return
* @throws IOException
* /
public byte [ ] writeContent ( ) throws IOException {
int contentLength = ( int ) this . responseHeader . contentLength ( ) ;
serverByteBuffer sbb = new serverByteBuffer ( ( contentLength = = - 1 ) ? 8192 : contentLength ) ;
2005-10-09 06:43:07 +02:00
writeContentX ( null , sbb , httpc . this . clientInput ) ;
2005-08-10 23:50:17 +02:00
return sbb . getBytes ( ) ;
}
/ * *
* This method outputs the found content into an byte - array and
* additionally outputs it to procOS .
*
* @param procOS
* @return
* @throws IOException
* /
public byte [ ] writeContent ( OutputStream procOS ) throws IOException {
int contentLength = ( int ) this . responseHeader . contentLength ( ) ;
serverByteBuffer sbb = new serverByteBuffer ( ( contentLength = = - 1 ) ? 8192 : contentLength ) ;
2005-10-09 06:43:07 +02:00
writeContentX ( procOS , sbb , httpc . this . clientInput ) ;
2005-08-10 23:50:17 +02:00
return sbb . getBytes ( ) ;
}
/ * *
* This method writes the input stream to either another output stream
* or a file or both .
*
* @param procOS
* @param file
* @throws IOException
* /
public void writeContent ( OutputStream procOS , File file ) throws IOException {
// this writes the input stream to either another output stream or
// a file or both.
FileOutputStream bufferOS = null ;
try {
if ( file ! = null ) bufferOS = new FileOutputStream ( file ) ;
2005-10-09 06:43:07 +02:00
writeContentX ( procOS , bufferOS , httpc . this . clientInput ) ;
2005-08-10 23:50:17 +02:00
} finally {
if ( bufferOS ! = null ) {
bufferOS . close ( ) ;
if ( file . length ( ) = = 0 ) file . delete ( ) ;
}
}
}
/ * *
* This method outputs the input stream to either an output socket or an
* file or both . If the length of the input stream is given in the
* header , exactly that lenght is written . Otherwise the stream is
* written , till it is closed . If this instance is zipped , stream the
* input stream through gzip to unzip it on the fly .
*
* @param procOS OutputStream where the stream is to be written . If null
* no write happens .
* @param bufferOS OutputStream where the stream is to be written too .
* If null no write happens .
* @param clientInput InputStream where the content is to be read from .
* @throws IOException
* /
public void writeContentX ( OutputStream procOS , OutputStream bufferOS , InputStream clientInput ) throws IOException {
// we write length bytes, but if length == -1 (or < 0) then we
// write until the input stream closes
// procOS == null -> no write to procOS
// file == null -> no write to file
// If the Content-Encoding is gzip, we gunzip on-the-fly
// and change the Content-Encoding and Content-Length attributes in the header
byte [ ] buffer = new byte [ 2048 ] ;
int l ;
long len = 0 ;
// find out length
long length = this . responseHeader . contentLength ( ) ;
// using the proper intput stream
InputStream dis = ( this . gzip ) ? ( InputStream ) new GZIPInputStream ( clientInput ) : ( InputStream ) clientInput ;
// we have three methods of reading: length-based, length-based gzip and connection-close-based
try {
if ( length > 0 ) {
// we read exactly 'length' bytes
while ( ( len < length ) & & ( ( l = dis . read ( buffer ) ) > = 0 ) ) {
if ( procOS ! = null ) procOS . write ( buffer , 0 , l ) ;
if ( bufferOS ! = null ) bufferOS . write ( buffer , 0 , l ) ;
len + = l ;
}
} else {
// no content-length was given, thus we read until the connection closes
while ( ( l = dis . read ( buffer , 0 , buffer . length ) ) > = 0 ) {
if ( procOS ! = null ) procOS . write ( buffer , 0 , l ) ;
if ( bufferOS ! = null ) bufferOS . write ( buffer , 0 , l ) ;
}
}
} catch ( java . net . SocketException e ) {
throw new IOException ( " Socket exception: " + e . getMessage ( ) ) ;
} catch ( java . net . SocketTimeoutException e ) {
throw new IOException ( " Socket time-out: " + e . getMessage ( ) ) ;
} finally {
// close the streams
if ( procOS ! = null ) {
if ( procOS instanceof httpChunkedOutputStream )
( ( httpChunkedOutputStream ) procOS ) . finish ( ) ;
procOS . flush ( ) ;
}
if ( bufferOS ! = null ) bufferOS . flush ( ) ;
buffer = null ;
}
}
/ * *
* This method outputs a logline to the serverlog with the current
* status of this instance .
* /
public void print ( ) {
2005-10-09 06:43:07 +02:00
serverLog . logInfo ( " HTTPC " , " RESPONSE: status= " + this . status + " , header= " + this . responseHeader . toString ( ) ) ;
2005-08-10 23:50:17 +02:00
}
}
2005-04-07 21:19:42 +02:00
}
/ *
import java.net.* ;
import java.io.* ;
import javax.net.ssl.* ;
import javax.security.cert.X509Certificate ;
import java.security.KeyStore ;
2005-08-07 17:16:33 +02:00
2005-04-07 21:19:42 +02:00
//The application can be modified to connect to a server outside
//the firewall by following SSLSocketClientWithTunneling.java.
2005-08-07 17:16:33 +02:00
2005-04-07 21:19:42 +02:00
public class SSLSocketClientWithClientAuth {
2005-08-07 17:16:33 +02:00
2005-04-07 21:19:42 +02:00
public static void main ( String [ ] args ) throws Exception {
2005-06-18 03:48:11 +02:00
String host = null ;
int port = - 1 ;
String path = null ;
for ( int i = 0 ; i < args . length ; i + + )
System . out . println ( args [ i ] ) ;
2005-08-07 17:16:33 +02:00
2005-06-18 03:48:11 +02:00
if ( args . length < 3 ) {
System . out . println (
" USAGE: java SSLSocketClientWithClientAuth " +
" host port requestedfilepath " ) ;
System . exit ( - 1 ) ;
}
2005-08-07 17:16:33 +02:00
2005-06-18 03:48:11 +02:00
try {
host = args [ 0 ] ;
port = Integer . parseInt ( args [ 1 ] ) ;
path = args [ 2 ] ;
} catch ( IllegalArgumentException e ) {
System . out . println ( " USAGE: java SSLSocketClientWithClientAuth " +
" host port requestedfilepath " ) ;
System . exit ( - 1 ) ;
}
2005-08-07 17:16:33 +02:00
2005-06-18 03:48:11 +02:00
try {
2005-08-07 17:16:33 +02:00
2005-06-18 03:48:11 +02:00
SSLSocketFactory factory = null ;
try {
SSLContext ctx ;
KeyManagerFactory kmf ;
KeyStore ks ;
char [ ] passphrase = " passphrase " . toCharArray ( ) ;
2005-08-07 17:16:33 +02:00
2005-06-18 03:48:11 +02:00
ctx = SSLContext . getInstance ( " TLS " ) ;
kmf = KeyManagerFactory . getInstance ( " SunX509 " ) ;
ks = KeyStore . getInstance ( " JKS " ) ;
2005-08-07 17:16:33 +02:00
2005-06-18 03:48:11 +02:00
ks . load ( new FileInputStream ( " testkeys " ) , passphrase ) ;
2005-08-07 17:16:33 +02:00
2005-06-18 03:48:11 +02:00
kmf . init ( ks , passphrase ) ;
ctx . init ( kmf . getKeyManagers ( ) , null , null ) ;
2005-08-07 17:16:33 +02:00
2005-06-18 03:48:11 +02:00
factory = ctx . getSocketFactory ( ) ;
} catch ( Exception e ) {
throw new IOException ( e . getMessage ( ) ) ;
}
2005-08-07 17:16:33 +02:00
2005-06-18 03:48:11 +02:00
SSLSocket socket = ( SSLSocket ) factory . createSocket ( host , port ) ;
2005-08-07 17:16:33 +02:00
2005-06-18 03:48:11 +02:00
socket . startHandshake ( ) ;
2005-08-07 17:16:33 +02:00
2005-06-18 03:48:11 +02:00
PrintWriter out = new PrintWriter (
new BufferedWriter (
new OutputStreamWriter (
socket . getOutputStream ( ) ) ) ) ;
out . println ( " GET " + path + " HTTP/1.1 " ) ;
out . println ( ) ;
out . flush ( ) ;
2005-08-07 17:16:33 +02:00
2005-06-18 03:48:11 +02:00
if ( out . checkError ( ) )
System . out . println (
" SSLSocketClient: java.io.PrintWriter error " ) ;
2005-08-07 17:16:33 +02:00
2005-06-18 03:48:11 +02:00
BufferedReader in = new BufferedReader (
new InputStreamReader (
socket . getInputStream ( ) ) ) ;
2005-08-07 17:16:33 +02:00
2005-06-18 03:48:11 +02:00
String inputLine ;
2005-08-07 17:16:33 +02:00
2005-06-18 03:48:11 +02:00
while ( ( inputLine = in . readLine ( ) ) ! = null )
System . out . println ( inputLine ) ;
2005-08-07 17:16:33 +02:00
2005-06-18 03:48:11 +02:00
in . close ( ) ;
out . close ( ) ;
socket . close ( ) ;
2005-08-07 17:16:33 +02:00
2005-06-18 03:48:11 +02:00
} catch ( Exception e ) {
e . printStackTrace ( ) ;
}
2005-04-07 21:19:42 +02:00
}
}
2005-06-18 03:48:11 +02:00
* /
2005-04-19 08:55:57 +02:00
final class httpcFactory implements org . apache . commons . pool . PoolableObjectFactory {
2005-08-07 17:16:33 +02:00
2005-04-19 08:55:57 +02:00
public httpcFactory ( ) {
2005-06-18 03:48:11 +02:00
super ( ) ;
2005-04-19 08:55:57 +02:00
}
2005-08-07 17:16:33 +02:00
2005-04-19 08:55:57 +02:00
/ * *
* @see org . apache . commons . pool . PoolableObjectFactory # makeObject ( )
* /
public Object makeObject ( ) throws Exception {
return new httpc ( ) ;
2005-06-18 03:48:11 +02:00
}
2005-08-07 17:16:33 +02:00
2005-06-18 03:48:11 +02:00
/ * *
2005-04-19 08:55:57 +02:00
* @see org . apache . commons . pool . PoolableObjectFactory # destroyObject ( java . lang . Object )
* /
public void destroyObject ( Object obj ) {
if ( obj instanceof httpc ) {
httpc theHttpc = ( httpc ) obj ;
2005-08-07 17:16:33 +02:00
2005-04-19 12:42:48 +02:00
theHttpc . removedFromPool = true ;
2005-04-19 08:55:57 +02:00
}
}
2005-08-07 17:16:33 +02:00
2005-04-19 08:55:57 +02:00
/ * *
* @see org . apache . commons . pool . PoolableObjectFactory # validateObject ( java . lang . Object )
* /
public boolean validateObject ( Object obj ) {
2005-06-18 03:48:11 +02:00
if ( obj instanceof httpc ) {
2005-04-19 08:55:57 +02:00
httpc theHttpc = ( httpc ) obj ;
return true ;
}
return true ;
}
2005-08-07 17:16:33 +02:00
2005-04-19 08:55:57 +02:00
/ * *
2005-06-18 03:48:11 +02:00
* @param obj
*
2005-04-19 08:55:57 +02:00
* /
public void activateObject ( Object obj ) {
//log.debug(" activateObject...");
}
2005-08-07 17:16:33 +02:00
2005-04-19 08:55:57 +02:00
/ * *
2005-06-18 03:48:11 +02:00
* @param obj
*
2005-04-19 08:55:57 +02:00
* /
2005-06-18 03:48:11 +02:00
public void passivateObject ( Object obj ) {
2005-04-19 08:55:57 +02:00
//log.debug(" passivateObject..." + obj);
if ( obj instanceof Session ) {
2005-06-18 03:48:11 +02:00
httpc theHttpc = ( httpc ) obj ;
2005-04-19 08:55:57 +02:00
}
}
2005-06-18 03:48:11 +02:00
}
2005-04-19 08:55:57 +02:00
final class httpcPool extends GenericObjectPool {
/ * *
* First constructor .
* @param objFactory
2005-06-18 03:48:11 +02:00
* /
2005-04-19 08:55:57 +02:00
public httpcPool ( httpcFactory objFactory ) {
super ( objFactory ) ;
this . setMaxIdle ( 75 ) ; // Maximum idle threads.
this . setMaxActive ( 150 ) ; // Maximum active threads.
this . setMinEvictableIdleTimeMillis ( 30000 ) ; //Evictor runs every 30 secs.
//this.setMaxWait(1000); // Wait 1 second till a thread is available
}
2005-08-07 17:16:33 +02:00
2005-04-19 08:55:57 +02:00
public httpcPool ( httpcFactory objFactory ,
2005-06-18 03:48:11 +02:00
GenericObjectPool . Config config ) {
2005-04-19 08:55:57 +02:00
super ( objFactory , config ) ;
}
2005-08-07 17:16:33 +02:00
2005-04-19 08:55:57 +02:00
/ * *
* @see org . apache . commons . pool . impl . GenericObjectPool # borrowObject ( )
* /
public Object borrowObject ( ) throws Exception {
2005-06-18 03:48:11 +02:00
return super . borrowObject ( ) ;
2005-04-19 08:55:57 +02:00
}
2005-08-07 17:16:33 +02:00
2005-04-19 08:55:57 +02:00
/ * *
* @see org . apache . commons . pool . impl . GenericObjectPool # returnObject ( java . lang . Object )
* /
public void returnObject ( Object obj ) throws Exception {
super . returnObject ( obj ) ;
2005-06-18 03:48:11 +02:00
}
}