Marown/yacy_search_server
1
0
Fork 0
mirror of https://github.com/yacy/yacy_search_server.git synced 2024-09-19 00:01:41 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity

code cleanup in http clieant

Browse Source
This commit is contained in:
Michael Peter Christen 2022-10-24 23:34:39 +02:00
parent 62d177bf59
commit 5ddc794bb9
1 changed files with 441 additions and 441 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

58
source/net/yacy/cora/protocol/http/HTTPClient.java
View File

@ -186,7 +186,7 @@ public class HTTPClient implements Closeable {
super(); super();
this.timeout = agent.clientTimeout; this.timeout = agent.clientTimeout;
clientBuilder.setUserAgent(agent.userAgent); clientBuilder.setUserAgent(agent.userAgent);
reqConfBuilder = RequestConfig.copy(DFLTREQUESTCONFIG); this.reqConfBuilder = RequestConfig.copy(DFLTREQUESTCONFIG);
setTimout(agent.clientTimeout); setTimout(agent.clientTimeout);
} }
@ -194,7 +194,7 @@ public class HTTPClient implements Closeable {
super(); super();
this.timeout = timeout; this.timeout = timeout;
clientBuilder.setUserAgent(agent.userAgent); clientBuilder.setUserAgent(agent.userAgent);
reqConfBuilder = RequestConfig.copy(DFLTREQUESTCONFIG); this.reqConfBuilder = RequestConfig.copy(DFLTREQUESTCONFIG);
setTimout(timeout); setTimout(timeout);
} }
@ -284,7 +284,7 @@ public class HTTPClient implements Closeable {
* @throws IllegalArgumentException * @throws IllegalArgumentException
* when pool is null or when maxConnections is lower than 1 * when pool is null or when maxConnections is lower than 1
*/ */
public static void initPoolMaxConnections(final PoolingHttpClientConnectionManager pool, int maxConnections) { public static void initPoolMaxConnections(final PoolingHttpClientConnectionManager pool, final int maxConnections) {
if (pool == null) { if (pool == null) {
throw new IllegalArgumentException("pool parameter must not be null"); throw new IllegalArgumentException("pool parameter must not be null");
} }
@ -340,9 +340,9 @@ public class HTTPClient implements Closeable {
* @param timeout in milliseconds * @param timeout in milliseconds
*/ */
public void setTimout(final int timeout) { public void setTimout(final int timeout) {
reqConfBuilder.setSocketTimeout(timeout); this.reqConfBuilder.setSocketTimeout(timeout);
reqConfBuilder.setConnectTimeout(timeout); this.reqConfBuilder.setConnectTimeout(timeout);
reqConfBuilder.setConnectionRequestTimeout(timeout); this.reqConfBuilder.setConnectionRequestTimeout(timeout);
} }
/** /**
@ -369,8 +369,8 @@ public class HTTPClient implements Closeable {
* @param redirecting * @param redirecting
*/ */
public void setRedirecting(final boolean redirecting) { public void setRedirecting(final boolean redirecting) {
reqConfBuilder.setRedirectsEnabled(redirecting); this.reqConfBuilder.setRedirectsEnabled(redirecting);
reqConfBuilder.setRelativeRedirectsAllowed(redirecting); this.reqConfBuilder.setRelativeRedirectsAllowed(redirecting);
} }
/** /**
@ -438,7 +438,7 @@ public class HTTPClient implements Closeable {
try { try {
this.currentRequest = new HttpGet(urix); this.currentRequest = new HttpGet(urix);
} catch (IllegalArgumentException e) { } catch (final IllegalArgumentException e) {
throw new IOException(e.getMessage()); // can be caused at java.net.URI.create() throw new IOException(e.getMessage()); // can be caused at java.net.URI.create()
} }
if (!localhost) setHost(url.getHost()); // overwrite resolved IP, needed for shared web hosting DO NOT REMOVE, see http://en.wikipedia.org/wiki/Shared_web_hosting_service if (!localhost) setHost(url.getHost()); // overwrite resolved IP, needed for shared web hosting DO NOT REMOVE, see http://en.wikipedia.org/wiki/Shared_web_hosting_service
@ -446,7 +446,7 @@ public class HTTPClient implements Closeable {
return getContentBytes(maxBytes, concurrent); return getContentBytes(maxBytes, concurrent);
} }
CredentialsProvider credsProvider = new BasicCredentialsProvider(); final CredentialsProvider credsProvider = new BasicCredentialsProvider();
credsProvider.setCredentials( credsProvider.setCredentials(
new AuthScope("localhost", url.getPort()), new AuthScope("localhost", url.getPort()),
new UsernamePasswordCredentials(username, pass)); new UsernamePasswordCredentials(username, pass));
@ -454,7 +454,7 @@ public class HTTPClient implements Closeable {
try (final CloseableHttpClient httpclient = clientBuilder.setDefaultCredentialsProvider(credsProvider) try (final CloseableHttpClient httpclient = clientBuilder.setDefaultCredentialsProvider(credsProvider)
.setDefaultAuthSchemeRegistry(AUTHSCHEMEREGISTRY).build()) { .setDefaultAuthSchemeRegistry(AUTHSCHEMEREGISTRY).build()) {
this.httpResponse = httpclient.execute(this.currentRequest); this.httpResponse = httpclient.execute(this.currentRequest);
HttpEntity httpEntity = this.httpResponse.getEntity(); final HttpEntity httpEntity = this.httpResponse.getEntity();
if (httpEntity != null) { if (httpEntity != null) {
if (getStatusCode() == HttpStatus.SC_OK) { if (getStatusCode() == HttpStatus.SC_OK) {
if (maxBytes >= 0 && httpEntity.getContentLength() > maxBytes) { if (maxBytes >= 0 && httpEntity.getContentLength() > maxBytes) {
@ -498,7 +498,7 @@ public class HTTPClient implements Closeable {
try { try {
this.currentRequest = new HttpGet(urix); this.currentRequest = new HttpGet(urix);
} catch (IllegalArgumentException e) { } catch (final IllegalArgumentException e) {
throw new IOException(e.getMessage()); // can be caused at java.net.URI.create() throw new IOException(e.getMessage()); // can be caused at java.net.URI.create()
} }
setHost(url.getHost()); // overwrite resolved IP, needed for shared web hosting DO NOT REMOVE, see http://en.wikipedia.org/wiki/Shared_web_hosting_service setHost(url.getHost()); // overwrite resolved IP, needed for shared web hosting DO NOT REMOVE, see http://en.wikipedia.org/wiki/Shared_web_hosting_service
@ -642,7 +642,7 @@ public class HTTPClient implements Closeable {
try (final CloseableHttpClient httpclient = clientBuilder.setDefaultCredentialsProvider(credsProvider) try (final CloseableHttpClient httpclient = clientBuilder.setDefaultCredentialsProvider(credsProvider)
.setDefaultAuthSchemeRegistry(AUTHSCHEMEREGISTRY).build()) { .setDefaultAuthSchemeRegistry(AUTHSCHEMEREGISTRY).build()) {
this.httpResponse = httpclient.execute(this.currentRequest); this.httpResponse = httpclient.execute(this.currentRequest);
HttpEntity httpEntity = this.httpResponse.getEntity(); final HttpEntity httpEntity = this.httpResponse.getEntity();
if (httpEntity != null) { if (httpEntity != null) {
if (getStatusCode() == HttpStatus.SC_OK) { if (getStatusCode() == HttpStatus.SC_OK) {
return getByteArray(httpEntity, Integer.MAX_VALUE); return getByteArray(httpEntity, Integer.MAX_VALUE);
@ -703,7 +703,7 @@ public class HTTPClient implements Closeable {
String mimeType = null; String mimeType = null;
if (this.httpResponse != null) { if (this.httpResponse != null) {
Header contentType = this.httpResponse.getFirstHeader(HttpHeaders.CONTENT_TYPE); final Header contentType = this.httpResponse.getFirstHeader(HttpHeaders.CONTENT_TYPE);
if (contentType != null) { if (contentType != null) {
@ -731,11 +731,11 @@ public class HTTPClient implements Closeable {
String charsetName = null; String charsetName = null;
if (this.httpResponse != null) { if (this.httpResponse != null) {
Header contentTypeHeader = this.httpResponse.getFirstHeader(HttpHeaders.CONTENT_TYPE); final Header contentTypeHeader = this.httpResponse.getFirstHeader(HttpHeaders.CONTENT_TYPE);
if (contentTypeHeader != null) { if (contentTypeHeader != null) {
String contentType = contentTypeHeader.getValue(); final String contentType = contentTypeHeader.getValue();
if (contentType != null) { if (contentType != null) {
@ -818,7 +818,7 @@ public class HTTPClient implements Closeable {
this.httpResponse.close(); this.httpResponse.close();
} }
if (this.client != null) { if (this.client != null) {
client.close(); this.client.close();
} }
} finally { } finally {
if (this.currentRequest != null) { if (this.currentRequest != null) {
@ -854,7 +854,7 @@ public class HTTPClient implements Closeable {
private void execute(final boolean concurrent) throws IOException { private void execute(final boolean concurrent) throws IOException {
final HttpClientContext context = HttpClientContext.create(); final HttpClientContext context = HttpClientContext.create();
context.setRequestConfig(reqConfBuilder.build()); context.setRequestConfig(this.reqConfBuilder.build());
if (this.host != null) if (this.host != null)
context.setTargetHost(new HttpHost(this.host)); context.setTargetHost(new HttpHost(this.host));
@ -879,25 +879,25 @@ public class HTTPClient implements Closeable {
this.client = clientBuilder.build(); this.client = clientBuilder.build();
if (concurrent) { if (concurrent) {
FutureTask<CloseableHttpResponse> t = new FutureTask<CloseableHttpResponse>(new Callable<CloseableHttpResponse>() { final FutureTask<CloseableHttpResponse> t = new FutureTask<CloseableHttpResponse>(new Callable<CloseableHttpResponse>() {
@Override @Override
public CloseableHttpResponse call() throws ClientProtocolException, IOException { public CloseableHttpResponse call() throws ClientProtocolException, IOException {
CloseableHttpResponse response = client.execute(currentRequest, context); final CloseableHttpResponse response = HTTPClient.this.client.execute(HTTPClient.this.currentRequest, context);
return response; return response;
} }
}); });
executor.execute(t); executor.execute(t);
try { try {
this.httpResponse = t.get(this.timeout, TimeUnit.MILLISECONDS); this.httpResponse = t.get(this.timeout, TimeUnit.MILLISECONDS);
} catch (ExecutionException e) { } catch (final ExecutionException e) {
throw e.getCause(); throw e.getCause();
} catch (Throwable e) {} } catch (final Throwable e) {}
try {t.cancel(true);} catch (Throwable e) {} try {t.cancel(true);} catch (final Throwable e) {}
if (this.httpResponse == null) { if (this.httpResponse == null) {
throw new IOException("timout to client after " + this.timeout + "ms" + " for url " + uri); throw new IOException("timout to client after " + this.timeout + "ms" + " for url " + uri);
} }
} else { } else {
this.httpResponse = client.execute(this.currentRequest, context); this.httpResponse = this.client.execute(this.currentRequest, context);
} }
this.httpResponse.setHeader(HeaderFramework.RESPONSE_TIME_MILLIS, Long.toString(System.currentTimeMillis() - time)); this.httpResponse.setHeader(HeaderFramework.RESPONSE_TIME_MILLIS, Long.toString(System.currentTimeMillis() - time));
} catch (final Throwable e) { } catch (final Throwable e) {
@ -923,7 +923,7 @@ public class HTTPClient implements Closeable {
if (instream == null) { if (instream == null) {
return null; return null;
} }
long contentLength = entity.getContentLength(); final long contentLength = entity.getContentLength();
/* /*
* When no maxBytes is specified, the default limit is * When no maxBytes is specified, the default limit is
* Integer.MAX_VALUE as a byte array size can not be over * Integer.MAX_VALUE as a byte array size can not be over
@ -944,7 +944,7 @@ public class HTTPClient implements Closeable {
initialSize = 4096; initialSize = 4096;
} }
final ByteArrayBuffer buffer = new ByteArrayBuffer(initialSize); final ByteArrayBuffer buffer = new ByteArrayBuffer(initialSize);
byte[] tmp = new byte[4096]; final byte[] tmp = new byte[4096];
int l = 0; int l = 0;
/* Sum is a long to enable check against Integer.MAX_VALUE */ /* Sum is a long to enable check against Integer.MAX_VALUE */
long sum = 0; long sum = 0;
@ -1025,7 +1025,7 @@ public class HTTPClient implements Closeable {
new NoopHostnameVerifier()) { new NoopHostnameVerifier()) {
@Override @Override
protected void prepareSocket(SSLSocket socket) throws IOException { protected void prepareSocket(final SSLSocket socket) throws IOException {
if(!ENABLE_SNI_EXTENSION.get()) { if(!ENABLE_SNI_EXTENSION.get()) {
/* Set the SSLParameters server names to empty so we don't use SNI extension. /* Set the SSLParameters server names to empty so we don't use SNI extension.
* See https://docs.oracle.com/javase/8/docs/technotes/guides/security/jsse/JSSERefGuide.html#ClientSNIExamples */ * See https://docs.oracle.com/javase/8/docs/technotes/guides/security/jsse/JSSERefGuide.html#ClientSNIExamples */
@ -1047,8 +1047,8 @@ public class HTTPClient implements Closeable {
private static ConnectionKeepAliveStrategy customKeepAliveStrategy() { private static ConnectionKeepAliveStrategy customKeepAliveStrategy() {
return new DefaultConnectionKeepAliveStrategy() { return new DefaultConnectionKeepAliveStrategy() {
@Override @Override
public long getKeepAliveDuration(HttpResponse response, HttpContext context) { public long getKeepAliveDuration(final HttpResponse response, final HttpContext context) {
long keepAlive = super.getKeepAliveDuration(response, context); final long keepAlive = super.getKeepAliveDuration(response, context);
return Math.min(Math.max(keepAlive, 5000), 25000); return Math.min(Math.max(keepAlive, 5000), 25000);
} }
}; };