From 77c99e500f07af075acad0ad0c1f8b1f6ffcebfb Mon Sep 17 00:00:00 2001 From: orbiter Date: Tue, 27 Oct 2009 15:25:48 +0000 Subject: [PATCH] added more control over memory allocation should avoid some of the OOMs git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@6436 6c8d7289-2bf4-0310-a012-ef5d649a1542 --- .../http/server/servlets/transferURL.java | 4 +-- source/de/anomic/server/serverCore.java | 16 +++++----- .../yacy/kelondro/index/RowCollection.java | 31 +++++++++++++------ source/net/yacy/kelondro/table/Table.java | 6 ++-- 4 files changed, 35 insertions(+), 22 deletions(-) diff --git a/source/de/anomic/http/server/servlets/transferURL.java b/source/de/anomic/http/server/servlets/transferURL.java index 3e8c3ca84..59b689da0 100644 --- a/source/de/anomic/http/server/servlets/transferURL.java +++ b/source/de/anomic/http/server/servlets/transferURL.java @@ -137,11 +137,11 @@ public final class transferURL { blocked++; continue; } - + // write entry to database yacyCore.log.logInfo("Accepting URL " + i + "/" + urlc + " from peer " + otherPeerName + ": " + lEntry.metadata().url().toNormalform(true, false)); try { - sb.indexSegments.urlMetadata(Segments.Process.DHTIN).store(lEntry); + sb.indexSegments.urlMetadata(Segments.Process.DHTIN).store(lEntry); // OOM here! sb.crawlResults.stack(lEntry, iam, iam, EventOrigin.DHT_TRANSFER); if (yacyCore.log.isFine()) yacyCore.log.logFine("transferURL: received URL '" + metadata.url().toNormalform(false, true) + "' from peer " + otherPeerName); received++; diff --git a/source/de/anomic/server/serverCore.java b/source/de/anomic/server/serverCore.java index b7d2623f3..d29697f41 100644 --- a/source/de/anomic/server/serverCore.java +++ b/source/de/anomic/server/serverCore.java @@ -540,7 +540,7 @@ public final class serverCore extends AbstractBusyThread implements BusyThread { // closing the socket to the client if (this.controlSocket != null) try { this.controlSocket.close(); - serverCore.this.log.logInfo("Closing main socket of thread '" + this.getName() + "'"); + log.logInfo("Closing main socket of thread '" + this.getName() + "'"); //this.controlSocket = null; } catch (final Exception e) {} } @@ -558,7 +558,7 @@ public final class serverCore extends AbstractBusyThread implements BusyThread { } public void log(final boolean outgoing, final String request) { - if (serverCore.this.log.isFine()) serverCore.this.log.logFine(this.userAddress.getHostAddress() + "/" + this.identity + " " + + if (log.isFine()) log.logFine(this.userAddress.getHostAddress() + "/" + this.identity + " " + "[" + ((busySessions == null)? -1 : busySessions.size()) + ", " + this.commandCounter + ((outgoing) ? "] > " : "] < ") + request); @@ -570,7 +570,7 @@ public final class serverCore extends AbstractBusyThread implements BusyThread { } public byte[] readLine() { - return receive(this.in, serverCore.this.commandMaxLength, true); + return receive(this.in, commandMaxLength, true); } /** @@ -653,7 +653,7 @@ public final class serverCore extends AbstractBusyThread implements BusyThread { } finally { if (busySessions != null) { busySessions.remove(this); - if (serverCore.this.log.isFinest()) serverCore.this.log.logFinest("* removed session "+ this.controlSocket.getRemoteSocketAddress() + " " + this.request); + if (log.isFinest()) log.logFinest("* removed session "+ this.controlSocket.getRemoteSocketAddress() + " " + this.request); } this.controlSocket = null; } @@ -664,7 +664,7 @@ public final class serverCore extends AbstractBusyThread implements BusyThread { protected void finalize() { if (busySessions != null && busySessions.contains(this)) { busySessions.remove(this); - if(serverCore.this.log.isFinest()) serverCore.this.log.logFinest("* removed session "+ this.controlSocket.getRemoteSocketAddress() + this.request); + if(log.isFinest()) log.logFinest("* removed session "+ this.controlSocket.getRemoteSocketAddress() + this.request); } this.close(); } @@ -708,7 +708,7 @@ public final class serverCore extends AbstractBusyThread implements BusyThread { if (this.request == null) break; if (reqProtocol.equals("HTTP")) { - this.commandObj = serverCore.this.handlerPrototype.clone(); + this.commandObj = handlerPrototype.clone(); } // initializing the session @@ -772,8 +772,8 @@ public final class serverCore extends AbstractBusyThread implements BusyThread { } catch (final NoSuchMethodException e) { log.logSevere("command execution, method exception " + e.getMessage() + " for client " + this.userAddress.getHostAddress(), e); if (!this.userAddress.isSiteLocalAddress()) { - if (serverCore.this.denyHost != null) { - serverCore.this.denyHost.put((""+this.userAddress.getHostAddress()), "deny"); // block client: hacker attempt + if (denyHost != null) { + denyHost.put((""+this.userAddress.getHostAddress()), "deny"); // block client: hacker attempt } } break; diff --git a/source/net/yacy/kelondro/index/RowCollection.java b/source/net/yacy/kelondro/index/RowCollection.java index 2eb987117..4006f6c5c 100644 --- a/source/net/yacy/kelondro/index/RowCollection.java +++ b/source/net/yacy/kelondro/index/RowCollection.java @@ -49,7 +49,8 @@ import net.yacy.kelondro.util.kelondroException; public class RowCollection implements Iterable { - public static final long growfactor100 = 140L; + public static final long growfactorLarge100 = 140L; + public static final long growfactorSmall100 = 120L; private static final int isortlimit = 20; private static int availableCPU = Runtime.getRuntime().availableProcessors(); @@ -202,13 +203,25 @@ public class RowCollection implements Iterable { return this.rowdef; } - protected final void ensureSize(final int elements) { + protected final long neededSpaceForEnsuredSize(final int elements, boolean forcegc) { assert elements > 0 : "elements = " + elements; final long needed = elements * rowdef.objectsize; - if (chunkcache.length >= needed) return; + if (chunkcache.length >= needed) return 0; assert needed > 0 : "needed = " + needed; - assert needed * growfactor100 / 100L > 0 : "elements = " + elements + ", new = " + (needed * growfactor100 / 100L); - byte[] newChunkcache = new byte[(int) (needed * growfactor100 / 100L)]; // increase space + long allocram = needed * growfactorLarge100 / 100L; + assert allocram > 0 : "elements = " + elements + ", new = " + allocram; + if (MemoryControl.request(allocram, false)) return allocram; + allocram = needed * growfactorSmall100 / 100L; + assert allocram > 0 : "elements = " + elements + ", new = " + allocram; + if (MemoryControl.request(allocram, forcegc)) return allocram; + return needed; + } + + protected final void ensureSize(final int elements) { + long allocram = neededSpaceForEnsuredSize(elements, true); + if (allocram == 0) return; + assert allocram > chunkcache.length : "wrong alloc computation: allocram = " + allocram + ", chunkcache.length = " + chunkcache.length; + byte[] newChunkcache = new byte[(int) allocram]; // increase space System.arraycopy(chunkcache, 0, newChunkcache, 0, chunkcache.length); chunkcache = newChunkcache; } @@ -220,13 +233,13 @@ public class RowCollection implements Iterable { * @return */ public final long memoryNeededForGrow() { - return (((long) (chunkcount + 1)) * ((long) rowdef.objectsize)) * growfactor100 / 100L; + return neededSpaceForEnsuredSize(chunkcount + 1, false); } public synchronized void trim(final boolean plusGrowFactor) { if (chunkcache.length == 0) return; - int needed = chunkcount * rowdef.objectsize; - if (plusGrowFactor) needed = (int) (((long) needed) * growfactor100 / 100L); + long needed = chunkcount * rowdef.objectsize; + if (plusGrowFactor) needed = neededSpaceForEnsuredSize(chunkcount, false); if (needed >= chunkcache.length) return; // in case that the growfactor causes that the cache would // grow instead of shrink, simply ignore the growfactor @@ -234,7 +247,7 @@ public class RowCollection implements Iterable { return; // if the swap buffer is not available, we must give up. // This is not critical. Otherwise we provoke a serious // problem with OOM - byte[] newChunkcache = new byte[needed]; + byte[] newChunkcache = new byte[(int) needed]; System.arraycopy(chunkcache, 0, newChunkcache, 0, Math.min( chunkcache.length, newChunkcache.length)); chunkcache = newChunkcache; diff --git a/source/net/yacy/kelondro/table/Table.java b/source/net/yacy/kelondro/table/Table.java index ec3123a34..c24877ef2 100644 --- a/source/net/yacy/kelondro/table/Table.java +++ b/source/net/yacy/kelondro/table/Table.java @@ -268,9 +268,9 @@ public class Table implements ObjectIndex, Iterable { final HashMap map = new HashMap(); map.put("tableSize", Integer.toString(index.size())); map.put("tableKeyChunkSize", Integer.toString(index.row().objectsize)); - map.put("tableKeyMem", Integer.toString((int) (((long) index.row().objectsize) * ((long) index.size()) * RowCollection.growfactor100 / 100L))); + map.put("tableKeyMem", Integer.toString(index.row().objectsize * index.size())); map.put("tableValueChunkSize", (table == null) ? "0" : Integer.toString(table.row().objectsize)); - map.put("tableValueMem", (table == null) ? "0" : Integer.toString((int) (((long) table.row().objectsize) * ((long) table.size()) * RowCollection.growfactor100 / 100L))); + map.put("tableValueMem", (table == null) ? "0" : Integer.toString(table.row().objectsize * table.size())); return map; } @@ -279,7 +279,7 @@ public class Table implements ObjectIndex, Iterable { } public static int staticRAMIndexNeed(final File f, final Row rowdef) throws IOException { - return (int) (((long)(rowdef.primaryKeyLength + 4)) * tableSize(f, rowdef.objectsize) * RowCollection.growfactor100 / 100L); + return (int) (((long)(rowdef.primaryKeyLength + 4)) * tableSize(f, rowdef.objectsize) * RowCollection.growfactorLarge100 / 100L); } public synchronized void addUnique(final Entry row) throws IOException {