Marown/yacy_search_server
1
0
Fork 0
mirror of https://github.com/yacy/yacy_search_server.git synced 2024-09-19 00:01:41 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity

healing for some OOM problems

Browse Source 
git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@7502 6c8d7289-2bf4-0310-a012-ef5d649a1542
This commit is contained in:
orbiter 2011-02-21 00:38:49 +00:00
parent 4aa406fb0f
commit d84b4a072e
8 changed files with 43 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
source/de/anomic/server/serverCore.java
View File

@ -726,8 +726,22 @@ public final class serverCore extends AbstractBusyThread implements BusyThread {
parameter[0] = this.request.trim();
}
}
//long commandStart = System.currentTimeMillis();
Object result = commandMethod.invoke(this.commandObj, parameter);
Object result = null;
try {
result = commandMethod.invoke(this.commandObj, parameter);
} catch (OutOfMemoryError e) {
// try again
terminateOldSessions(2000);
try {
result = commandMethod.invoke(this.commandObj, parameter);
} catch (OutOfMemoryError e2) {
// try again
Thread.sleep(1000);
result = commandMethod.invoke(this.commandObj, parameter);
}
}
//announceMoreExecTime(commandStart - System.currentTimeMillis()); // shall be negative!
//this.log.logDebug("* session " + handle + " completed command '" + request + "'. time = " + (System.currentTimeMillis() - handle));
this.out.flush();

6
source/net/yacy/cora/protocol/http/HTTPClient.java
View File

@ -445,8 +445,12 @@ public class HTTPClient {
// get the response body
final HttpEntity httpEntity = httpResponse.getEntity();
if (httpEntity != null) {
if (getStatusCode() == 200 && httpEntity.getContentLength() < maxBytes) {
if (getStatusCode() == 200 && httpEntity.getContentLength() < maxBytes) {
try {
content = EntityUtils.toByteArray(httpEntity);
} catch (OutOfMemoryError e) {
throw new IOException(e.toString());
}
}
// Ensures that the entity content is fully consumed and the content stream, if exists, is closed.
EntityUtils.consume(httpEntity);

3
source/net/yacy/kelondro/blob/BLOB.java
View File

@ -189,8 +189,9 @@ public interface BLOB {
* of the input
* @param b
* @return an array that is equal or smaller in size than b
* @throws RowSpaceExceededException
*/
public byte[] rewrite(byte[] b);
public byte[] rewrite(byte[] b) throws RowSpaceExceededException;
}

3
source/net/yacy/kelondro/blob/HeapModifier.java
View File

@ -28,6 +28,7 @@ import java.io.File;
import java.io.IOException;
import java.util.SortedMap;
import net.yacy.kelondro.index.RowSpaceExceededException;
import net.yacy.kelondro.io.CachedFileWriter;
import net.yacy.kelondro.logging.Log;
import net.yacy.kelondro.order.ByteOrder;
@ -242,7 +243,7 @@ public class HeapModifier extends HeapReader implements BLOB {
throw new UnsupportedOperationException();
}
public int reduce(byte[] key, final Reducer reducer) throws IOException {
public int reduce(byte[] key, final Reducer reducer) throws IOException, RowSpaceExceededException {
key = normalizeKey(key);
assert key.length == this.keylength;

2
source/net/yacy/kelondro/index/HandleSet.java
View File

@ -64,7 +64,7 @@ public final class HandleSet implements Iterable<byte[]>, Cloneable {
this.index = index;
}
public HandleSet(Row rowdef, byte[] b) {
public HandleSet(Row rowdef, byte[] b) throws RowSpaceExceededException {
this.rowdef = rowdef;
this.index = RowSet.importRowSet(b, this.rowdef);
}

11
source/net/yacy/kelondro/index/RowSet.java
View File

@ -32,6 +32,7 @@ import net.yacy.kelondro.logging.Log;
import net.yacy.kelondro.order.Base64Order;
import net.yacy.kelondro.order.CloneableIterator;
import net.yacy.kelondro.order.NaturalOrder;
import net.yacy.kelondro.util.MemoryControl;
public class RowSet extends RowCollection implements Index, Iterable<Row.Entry> {
@ -68,7 +69,7 @@ public class RowSet extends RowCollection implements Index, Iterable<Row.Entry>
assert rowdef.objectOrder != null;
}
public final static RowSet importRowSet(final byte[] b, final Row rowdef) {
public final static RowSet importRowSet(final byte[] b, final Row rowdef) throws RowSpaceExceededException {
assert b.length >= exportOverheadSize : "b.length = " + b.length;
if (b.length < exportOverheadSize) return new RowSet(rowdef);
final int size = (int) NaturalOrder.decodeLong(b, 0, 4);
@ -80,7 +81,13 @@ public class RowSet extends RowCollection implements Index, Iterable<Row.Entry>
long alloc = ((long) size) * ((long) rowdef.objectsize);
assert alloc <= Integer.MAX_VALUE : "alloc = " + alloc;
assert alloc == b.length - exportOverheadSize;
final byte[] chunkcache = new byte[(int) alloc];
MemoryControl.request((int) alloc, true);
final byte[] chunkcache;
try {
chunkcache = new byte[(int) alloc];
} catch (OutOfMemoryError e) {
throw new RowSpaceExceededException((int) alloc, "importRowSet");
}
//assert b.length - exportOverheadSize == size * rowdef.objectsize : "b.length = " + b.length + ", size * rowdef.objectsize = " + size * rowdef.objectsize;
if (b.length - exportOverheadSize != alloc) {
Log.logSevere("RowSet", "exportOverheadSize wrong: b.length = " + b.length + ", size * rowdef.objectsize = " + size * rowdef.objectsize);

2
source/net/yacy/kelondro/rwi/ReferenceContainerArray.java
View File

@ -291,7 +291,7 @@ public final class ReferenceContainerArray<ReferenceType extends Reference> {
this.wordHash = wordHash;
}
public byte[] rewrite(byte[] b) {
public byte[] rewrite(byte[] b) throws RowSpaceExceededException {
if (b == null) return null;
ReferenceContainer<ReferenceType> c = rewriter.reduce(new ReferenceContainer<ReferenceType>(factory, this.wordHash, RowSet.importRowSet(b, payloadrow)));
if (c == null) return null;

8
source/net/yacy/kelondro/rwi/ReferenceIterator.java
View File

@ -34,6 +34,7 @@ import java.util.Map;
import net.yacy.kelondro.blob.HeapReader;
import net.yacy.kelondro.index.Row;
import net.yacy.kelondro.index.RowSet;
import net.yacy.kelondro.index.RowSpaceExceededException;
import net.yacy.kelondro.logging.Log;
import net.yacy.kelondro.order.CloneableIterator;
@ -67,7 +68,12 @@ public class ReferenceIterator <ReferenceType extends Reference> implements Clon
public ReferenceContainer<ReferenceType> next() {
Map.Entry<byte[], byte[]> entry = blobs.next();
byte[] payload = entry.getValue();
return new ReferenceContainer<ReferenceType>(factory, entry.getKey(), RowSet.importRowSet(payload, payloadrow));
try {
return new ReferenceContainer<ReferenceType>(factory, entry.getKey(), RowSet.importRowSet(payload, payloadrow));
} catch (RowSpaceExceededException e) {
Log.logSevere("ReferenceIterator", "lost entry '" + entry.getKey() + "' because of too low memory: " + e.toString());
return null;
}
}
public void remove() {