Marown/yacy_search_server
1
0
Fork 0
mirror of https://github.com/yacy/yacy_search_server.git synced 2024-09-19 00:01:41 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity

added more control over memory allocation

Browse Source 
should avoid some of the OOMs

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@6436 6c8d7289-2bf4-0310-a012-ef5d649a1542
This commit is contained in:
orbiter 2009-10-27 15:25:48 +00:00
parent 52470d0de4
commit 77c99e500f
4 changed files with 35 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
source/de/anomic/http/server/servlets/transferURL.java
View File

@ -137,11 +137,11 @@ public final class transferURL {
blocked++;
continue;
}
// write entry to database
yacyCore.log.logInfo("Accepting URL " + i + "/" + urlc + " from peer " + otherPeerName + ": " + lEntry.metadata().url().toNormalform(true, false));
try {
sb.indexSegments.urlMetadata(Segments.Process.DHTIN).store(lEntry);
sb.indexSegments.urlMetadata(Segments.Process.DHTIN).store(lEntry); // OOM here!
sb.crawlResults.stack(lEntry, iam, iam, EventOrigin.DHT_TRANSFER);
if (yacyCore.log.isFine()) yacyCore.log.logFine("transferURL: received URL '" + metadata.url().toNormalform(false, true) + "' from peer " + otherPeerName);
received++;

16
source/de/anomic/server/serverCore.java
View File

@ -540,7 +540,7 @@ public final class serverCore extends AbstractBusyThread implements BusyThread {
// closing the socket to the client
if (this.controlSocket != null) try {
this.controlSocket.close();
serverCore.this.log.logInfo("Closing main socket of thread '" + this.getName() + "'");
log.logInfo("Closing main socket of thread '" + this.getName() + "'");
//this.controlSocket = null;
} catch (final Exception e) {}
}
@ -558,7 +558,7 @@ public final class serverCore extends AbstractBusyThread implements BusyThread {
}
public void log(final boolean outgoing, final String request) {
if (serverCore.this.log.isFine()) serverCore.this.log.logFine(this.userAddress.getHostAddress() + "/" + this.identity + " " +
if (log.isFine()) log.logFine(this.userAddress.getHostAddress() + "/" + this.identity + " " +
"[" + ((busySessions == null)? -1 : busySessions.size()) + ", " + this.commandCounter +
((outgoing) ? "] > " : "] < ") +
request);
@ -570,7 +570,7 @@ public final class serverCore extends AbstractBusyThread implements BusyThread {
}
public byte[] readLine() {
return receive(this.in, serverCore.this.commandMaxLength, true);
return receive(this.in, commandMaxLength, true);
}
/**
@ -653,7 +653,7 @@ public final class serverCore extends AbstractBusyThread implements BusyThread {
} finally {
if (busySessions != null) {
busySessions.remove(this);
if (serverCore.this.log.isFinest()) serverCore.this.log.logFinest("* removed session "+ this.controlSocket.getRemoteSocketAddress() + " " + this.request);
if (log.isFinest()) log.logFinest("* removed session "+ this.controlSocket.getRemoteSocketAddress() + " " + this.request);
}
this.controlSocket = null;
}
@ -664,7 +664,7 @@ public final class serverCore extends AbstractBusyThread implements BusyThread {
protected void finalize() {
if (busySessions != null && busySessions.contains(this)) {
busySessions.remove(this);
if(serverCore.this.log.isFinest()) serverCore.this.log.logFinest("* removed session "+ this.controlSocket.getRemoteSocketAddress() + this.request);
if(log.isFinest()) log.logFinest("* removed session "+ this.controlSocket.getRemoteSocketAddress() + this.request);
}
this.close();
}
@ -708,7 +708,7 @@ public final class serverCore extends AbstractBusyThread implements BusyThread {
if (this.request == null) break;
if (reqProtocol.equals("HTTP")) {
this.commandObj = serverCore.this.handlerPrototype.clone();
this.commandObj = handlerPrototype.clone();
}
// initializing the session
@ -772,8 +772,8 @@ public final class serverCore extends AbstractBusyThread implements BusyThread {
} catch (final NoSuchMethodException e) {
log.logSevere("command execution, method exception " + e.getMessage() + " for client " + this.userAddress.getHostAddress(), e);
if (!this.userAddress.isSiteLocalAddress()) {
if (serverCore.this.denyHost != null) {
serverCore.this.denyHost.put((""+this.userAddress.getHostAddress()), "deny"); // block client: hacker attempt
if (denyHost != null) {
denyHost.put((""+this.userAddress.getHostAddress()), "deny"); // block client: hacker attempt
}
}
break;

31
source/net/yacy/kelondro/index/RowCollection.java
View File

@ -49,7 +49,8 @@ import net.yacy.kelondro.util.kelondroException;
public class RowCollection implements Iterable<Row.Entry> {
public static final long growfactor100 = 140L;
public static final long growfactorLarge100 = 140L;
public static final long growfactorSmall100 = 120L;
private static final int isortlimit = 20;
private static int availableCPU = Runtime.getRuntime().availableProcessors();
@ -202,13 +203,25 @@ public class RowCollection implements Iterable<Row.Entry> {
return this.rowdef;
}
protected final void ensureSize(final int elements) {
protected final long neededSpaceForEnsuredSize(final int elements, boolean forcegc) {
assert elements > 0 : "elements = " + elements;
final long needed = elements * rowdef.objectsize;
if (chunkcache.length >= needed) return;
if (chunkcache.length >= needed) return 0;
assert needed > 0 : "needed = " + needed;
assert needed * growfactor100 / 100L > 0 : "elements = " + elements + ", new = " + (needed * growfactor100 / 100L);
byte[] newChunkcache = new byte[(int) (needed * growfactor100 / 100L)]; // increase space
long allocram = needed * growfactorLarge100 / 100L;
assert allocram > 0 : "elements = " + elements + ", new = " + allocram;
if (MemoryControl.request(allocram, false)) return allocram;
allocram = needed * growfactorSmall100 / 100L;
assert allocram > 0 : "elements = " + elements + ", new = " + allocram;
if (MemoryControl.request(allocram, forcegc)) return allocram;
return needed;
}
protected final void ensureSize(final int elements) {
long allocram = neededSpaceForEnsuredSize(elements, true);
if (allocram == 0) return;
assert allocram > chunkcache.length : "wrong alloc computation: allocram = " + allocram + ", chunkcache.length = " + chunkcache.length;
byte[] newChunkcache = new byte[(int) allocram]; // increase space
System.arraycopy(chunkcache, 0, newChunkcache, 0, chunkcache.length);
chunkcache = newChunkcache;
}
@ -220,13 +233,13 @@ public class RowCollection implements Iterable<Row.Entry> {
* @return
*/
public final long memoryNeededForGrow() {
return (((long) (chunkcount + 1)) * ((long) rowdef.objectsize)) * growfactor100 / 100L;
return neededSpaceForEnsuredSize(chunkcount + 1, false);
}
public synchronized void trim(final boolean plusGrowFactor) {
if (chunkcache.length == 0) return;
int needed = chunkcount * rowdef.objectsize;
if (plusGrowFactor) needed = (int) (((long) needed) * growfactor100 / 100L);
long needed = chunkcount * rowdef.objectsize;
if (plusGrowFactor) needed = neededSpaceForEnsuredSize(chunkcount, false);
if (needed >= chunkcache.length)
return; // in case that the growfactor causes that the cache would
// grow instead of shrink, simply ignore the growfactor
@ -234,7 +247,7 @@ public class RowCollection implements Iterable<Row.Entry> {
return; // if the swap buffer is not available, we must give up.
// This is not critical. Otherwise we provoke a serious
// problem with OOM
byte[] newChunkcache = new byte[needed];
byte[] newChunkcache = new byte[(int) needed];
System.arraycopy(chunkcache, 0, newChunkcache, 0, Math.min(
chunkcache.length, newChunkcache.length));
chunkcache = newChunkcache;

6
source/net/yacy/kelondro/table/Table.java
View File

@ -268,9 +268,9 @@ public class Table implements ObjectIndex, Iterable<Row.Entry> {
final HashMap<String, String> map = new HashMap<String, String>();
map.put("tableSize", Integer.toString(index.size()));
map.put("tableKeyChunkSize", Integer.toString(index.row().objectsize));
map.put("tableKeyMem", Integer.toString((int) (((long) index.row().objectsize) * ((long) index.size()) * RowCollection.growfactor100 / 100L)));
map.put("tableKeyMem", Integer.toString(index.row().objectsize * index.size()));
map.put("tableValueChunkSize", (table == null) ? "0" : Integer.toString(table.row().objectsize));
map.put("tableValueMem", (table == null) ? "0" : Integer.toString((int) (((long) table.row().objectsize) * ((long) table.size()) * RowCollection.growfactor100 / 100L)));
map.put("tableValueMem", (table == null) ? "0" : Integer.toString(table.row().objectsize * table.size()));
return map;
}
@ -279,7 +279,7 @@ public class Table implements ObjectIndex, Iterable<Row.Entry> {
}
public static int staticRAMIndexNeed(final File f, final Row rowdef) throws IOException {
return (int) (((long)(rowdef.primaryKeyLength + 4)) * tableSize(f, rowdef.objectsize) * RowCollection.growfactor100 / 100L);
return (int) (((long)(rowdef.primaryKeyLength + 4)) * tableSize(f, rowdef.objectsize) * RowCollection.growfactorLarge100 / 100L);
}
public synchronized void addUnique(final Entry row) throws IOException {