From ebfaf753b7fc3d3bde506ddb739f4cefb8178eac Mon Sep 17 00:00:00 2001 From: Michael Peter Christen Date: Tue, 28 Jan 2014 12:39:58 +0100 Subject: [PATCH 1/2] - faster initialization of index files - removal of not used space if index files shrink (rare, but possible) --- .../yacy/kelondro/index/RowCollection.java | 41 +++++++++++++++---- 1 file changed, 32 insertions(+), 9 deletions(-) diff --git a/source/net/yacy/kelondro/index/RowCollection.java b/source/net/yacy/kelondro/index/RowCollection.java index 66f8ca737..47c3c9645 100644 --- a/source/net/yacy/kelondro/index/RowCollection.java +++ b/source/net/yacy/kelondro/index/RowCollection.java @@ -238,16 +238,16 @@ public class RowCollection implements Sortable, Iterable, return this.rowdef; } - private final long neededSpaceForEnsuredSize(final int elements, final boolean forcegc) { + private final long neededSpaceForEnsuredSize(final int elements) { assert elements > 0 : "elements = " + elements; final long needed = elements * this.rowdef.objectsize; if (this.chunkcache.length >= needed) return 0; assert needed > 0 : "needed = " + needed; - long allocram = needed * growfactorLarge100 / 100L; + long allocram = Math.max(1024, (needed * growfactorLarge100) / 100L); allocram -= allocram % this.rowdef.objectsize; assert allocram > 0 : "elements = " + elements + ", new = " + allocram; - if (allocram <= Integer.MAX_VALUE && MemoryControl.request(allocram, forcegc)) return allocram; - allocram = needed * growfactorSmall100 / 100L; + if (allocram <= Integer.MAX_VALUE && MemoryControl.request(allocram, false)) return allocram; + allocram = (needed * growfactorSmall100) / 100L; allocram -= allocram % this.rowdef.objectsize; assert allocram >= 0 : "elements = " + elements + ", new = " + allocram; return allocram; @@ -255,7 +255,7 @@ public class RowCollection implements Sortable, Iterable, private final void ensureSize(final int elements) throws SpaceExceededException { if (elements == 0) return; - final long allocram = neededSpaceForEnsuredSize(elements, true); + final long allocram = neededSpaceForEnsuredSize(elements); if (allocram == 0) return; assert this.chunkcache.length < elements * this.rowdef.objectsize : "wrong alloc computation (1): elements * rowdef.objectsize = " + (elements * this.rowdef.objectsize) + ", chunkcache.length = " + this.chunkcache.length; assert allocram > this.chunkcache.length : "wrong alloc computation (2): allocram = " + allocram + ", chunkcache.length = " + this.chunkcache.length; @@ -277,6 +277,7 @@ public class RowCollection implements Sortable, Iterable, } } } + /** * compute the needed memory in case of a cache extension. That is, if the cache is full and must @@ -285,7 +286,7 @@ public class RowCollection implements Sortable, Iterable, * @return */ protected final long memoryNeededForGrow() { - return neededSpaceForEnsuredSize(this.chunkcount + 1, false); + return neededSpaceForEnsuredSize(this.chunkcount + 1); } @Override @@ -307,6 +308,11 @@ public class RowCollection implements Sortable, Iterable, System.arraycopy(swapspace, 0, this.chunkcache, this.rowdef.objectsize * j, this.rowdef.objectsize); } + private final void checkShrink() { + final long allocram = this.rowdef.objectsize * this.chunkcount; + if (allocram < this.chunkcache.length / 2 && MemoryControl.request(allocram + 32, true)) trim(); + } + protected synchronized void trim() { if (this.chunkcache.length == 0) return; final long needed = this.chunkcount * this.rowdef.objectsize; @@ -318,9 +324,20 @@ public class RowCollection implements Sortable, Iterable, return; // if the swap buffer is not available, we must give up. // This is not critical. Otherwise we provoke a serious // problem with OOM - final byte[] newChunkcache = new byte[(int) needed]; - System.arraycopy(this.chunkcache, 0, newChunkcache, 0, Math.min(this.chunkcache.length, newChunkcache.length)); - this.chunkcache = newChunkcache; + try { + final byte[] newChunkcache = new byte[(int) needed]; + System.arraycopy(this.chunkcache, 0, newChunkcache, 0, newChunkcache.length); + this.chunkcache = newChunkcache; + } catch (final OutOfMemoryError e) { + // lets try again after a forced gc() + System.gc(); + try { + final byte[] newChunkcache = new byte[(int) needed]; + System.arraycopy(this.chunkcache, 0, newChunkcache, 0, newChunkcache.length); + this.chunkcache = newChunkcache; + } catch (final OutOfMemoryError ee) { + } + } } public final long lastWrote() { @@ -486,6 +503,9 @@ public class RowCollection implements Sortable, Iterable, } this.chunkcount--; this.lastTimeWrote = System.currentTimeMillis(); + + // check if the chunkcache can shrink + checkShrink(); } @@ -504,6 +524,9 @@ public class RowCollection implements Sortable, Iterable, if (this.chunkcount == this.sortBound) this.sortBound--; this.chunkcount--; this.lastTimeWrote = System.currentTimeMillis(); + + // check if the chunkcache can shrink + checkShrink(); return r; } From c589ee8c6e5ea3738a1cb25a8c038a3ec8a0f1db Mon Sep 17 00:00:00 2001 From: reger Date: Tue, 28 Jan 2014 22:39:45 +0100 Subject: [PATCH 2/2] URLproxy access check too tight respect config ip pattern (was own ip) --- source/net/yacy/http/servlets/YaCyProxyServlet.java | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/source/net/yacy/http/servlets/YaCyProxyServlet.java b/source/net/yacy/http/servlets/YaCyProxyServlet.java index fa710e164..b125a6c85 100644 --- a/source/net/yacy/http/servlets/YaCyProxyServlet.java +++ b/source/net/yacy/http/servlets/YaCyProxyServlet.java @@ -83,13 +83,11 @@ public class YaCyProxyServlet extends ProxyServlet implements Servlet { final String remoteHost = req.getRemoteHost(); if (!Domains.isThisHostIP(remoteHost)) { - response.sendError(HttpServletResponse.SC_FORBIDDEN, - "proxy use not granted for IP " + remoteHost); - return; - } else if (!proxyippatternmatch(remoteHost)) { - response.sendError(HttpServletResponse.SC_FORBIDDEN, - "proxy use not granted for IP " + remoteHost); - return; + if (!proxyippatternmatch(remoteHost)) { + response.sendError(HttpServletResponse.SC_FORBIDDEN, + "proxy use not granted for IP " + remoteHost); + return; + } } if ("CONNECT".equalsIgnoreCase(request.getMethod())) {