Marown/yacy_search_server
1
0
Fork 0
mirror of https://github.com/yacy/yacy_search_server.git synced 2024-09-19 00:01:41 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity

If available, check content length before downloading. Check also

Browse Source 
content length is not over Integer.MAX_VALUE.
This commit is contained in:
luc 2015-11-18 10:11:38 +01:00
parent 5bbb2e1730
commit 3c4c77099d
1 changed files with 56 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

81
source/net/yacy/cora/protocol/http/HTTPClient.java
View File

@ -712,31 +712,62 @@ public class HTTPClient {
}
}
private static byte[] getByteArray(final HttpEntity entity, final int maxBytes) throws IOException {
final InputStream instream = entity.getContent();
if (instream == null) {
return null;
}
try {
int i = maxBytes < 0 ? (int)entity.getContentLength() : Math.min(maxBytes, (int)entity.getContentLength());
if (i < 0) {
i = 4096;
}
final ByteArrayBuffer buffer = new ByteArrayBuffer(i);
byte[] tmp = new byte[4096];
int l, sum = 0;
while((l = instream.read(tmp)) != -1) {
sum += l;
if (maxBytes >= 0 && sum > maxBytes) throw new IOException("Download exceeded maximum value of " + maxBytes + " bytes");
buffer.append(tmp, 0, l);
}
return buffer.toByteArray();
} catch (final OutOfMemoryError e) {
throw new IOException(e.toString());
} finally {
instream.close();
}
}
/**
* Return entity content loaded as a byte array
* @param entity HTTP entity
* @param maxBytes maximum bytes to read. -1 means no maximum limit.
* @return content bytes or null when entity content is null.
* @throws IOException when a read error occured or content length is over maxBytes
*/
public static byte[] getByteArray(final HttpEntity entity, int maxBytes) throws IOException {
final InputStream instream = entity.getContent();
if (instream == null) {
return null;
}
try {
long contentLength = entity.getContentLength();
/*
* When no maxBytes is specified, the default limit is
* Integer.MAX_VALUE as a byte array size can not be over
*/
if (maxBytes < 0) {
maxBytes = Integer.MAX_VALUE;
}
/*
* Content length may already be known now : check it before
* downloading
*/
if (contentLength > maxBytes) {
throw new IOException("Content to download exceed maximum value of " + maxBytes + " bytes");
}
int initialSize = Math.min(maxBytes, (int) contentLength);
/* ContentLenght may be negative because unknown for now */
if (initialSize < 0) {
initialSize = 4096;
}
final ByteArrayBuffer buffer = new ByteArrayBuffer(initialSize);
byte[] tmp = new byte[4096];
int l = 0;
/* Sum is a long to enable check against Integer.MAX_VALUE */
long sum = 0;
while ((l = instream.read(tmp)) != -1) {
sum += l;
/*
* Check total length while downloading as content lenght might
* not be known at beginning
*/
if (sum > maxBytes) {
throw new IOException("Download exceeded maximum value of " + maxBytes + " bytes");
}
buffer.append(tmp, 0, l);
}
return buffer.toByteArray();
} catch (final OutOfMemoryError e) {
throw new IOException(e.toString());
} finally {
instream.close();
}
}
private void setHeaders(final HttpUriRequest httpUriRequest) {
if (this.headers != null) {