Marown/yacy_search_server
1
0
Fork 0
mirror of https://github.com/yacy/yacy_search_server.git synced 2024-09-19 00:01:41 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity

- faster initialization of index files

Browse Source 
- removal of not used space if index files shrink (rare, but possible)
This commit is contained in:
Michael Peter Christen 2014-01-28 12:39:58 +01:00
parent 322854a5f8
commit ebfaf753b7
1 changed files with 32 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

41
source/net/yacy/kelondro/index/RowCollection.java
View File

@ -238,16 +238,16 @@ public class RowCollection implements Sortable<Row.Entry>, Iterable<Row.Entry>,
return this.rowdef;
}
private final long neededSpaceForEnsuredSize(final int elements, final boolean forcegc) {
private final long neededSpaceForEnsuredSize(final int elements) {
assert elements > 0 : "elements = " + elements;
final long needed = elements * this.rowdef.objectsize;
if (this.chunkcache.length >= needed) return 0;
assert needed > 0 : "needed = " + needed;
long allocram = needed * growfactorLarge100 / 100L;
long allocram = Math.max(1024, (needed * growfactorLarge100) / 100L);
allocram -= allocram % this.rowdef.objectsize;
assert allocram > 0 : "elements = " + elements + ", new = " + allocram;
if (allocram <= Integer.MAX_VALUE && MemoryControl.request(allocram, forcegc)) return allocram;
allocram = needed * growfactorSmall100 / 100L;
if (allocram <= Integer.MAX_VALUE && MemoryControl.request(allocram, false)) return allocram;
allocram = (needed * growfactorSmall100) / 100L;
allocram -= allocram % this.rowdef.objectsize;
assert allocram >= 0 : "elements = " + elements + ", new = " + allocram;
return allocram;
@ -255,7 +255,7 @@ public class RowCollection implements Sortable<Row.Entry>, Iterable<Row.Entry>,
private final void ensureSize(final int elements) throws SpaceExceededException {
if (elements == 0) return;
final long allocram = neededSpaceForEnsuredSize(elements, true);
final long allocram = neededSpaceForEnsuredSize(elements);
if (allocram == 0) return;
assert this.chunkcache.length < elements * this.rowdef.objectsize : "wrong alloc computation (1): elements * rowdef.objectsize = " + (elements * this.rowdef.objectsize) + ", chunkcache.length = " + this.chunkcache.length;
assert allocram > this.chunkcache.length : "wrong alloc computation (2): allocram = " + allocram + ", chunkcache.length = " + this.chunkcache.length;
@ -277,6 +277,7 @@ public class RowCollection implements Sortable<Row.Entry>, Iterable<Row.Entry>,
}
}
}
/**
* compute the needed memory in case of a cache extension. That is, if the cache is full and must
@ -285,7 +286,7 @@ public class RowCollection implements Sortable<Row.Entry>, Iterable<Row.Entry>,
* @return
*/
protected final long memoryNeededForGrow() {
return neededSpaceForEnsuredSize(this.chunkcount + 1, false);
return neededSpaceForEnsuredSize(this.chunkcount + 1);
}
@Override
@ -307,6 +308,11 @@ public class RowCollection implements Sortable<Row.Entry>, Iterable<Row.Entry>,
System.arraycopy(swapspace, 0, this.chunkcache, this.rowdef.objectsize * j, this.rowdef.objectsize);
}
private final void checkShrink() {
final long allocram = this.rowdef.objectsize * this.chunkcount;
if (allocram < this.chunkcache.length / 2 && MemoryControl.request(allocram + 32, true)) trim();
}
protected synchronized void trim() {
if (this.chunkcache.length == 0) return;
final long needed = this.chunkcount * this.rowdef.objectsize;
@ -318,9 +324,20 @@ public class RowCollection implements Sortable<Row.Entry>, Iterable<Row.Entry>,
return; // if the swap buffer is not available, we must give up.
// This is not critical. Otherwise we provoke a serious
// problem with OOM
final byte[] newChunkcache = new byte[(int) needed];
System.arraycopy(this.chunkcache, 0, newChunkcache, 0, Math.min(this.chunkcache.length, newChunkcache.length));
this.chunkcache = newChunkcache;
try {
final byte[] newChunkcache = new byte[(int) needed];
System.arraycopy(this.chunkcache, 0, newChunkcache, 0, newChunkcache.length);
this.chunkcache = newChunkcache;
} catch (final OutOfMemoryError e) {
// lets try again after a forced gc()
System.gc();
try {
final byte[] newChunkcache = new byte[(int) needed];
System.arraycopy(this.chunkcache, 0, newChunkcache, 0, newChunkcache.length);
this.chunkcache = newChunkcache;
} catch (final OutOfMemoryError ee) {
}
}
}
public final long lastWrote() {
@ -486,6 +503,9 @@ public class RowCollection implements Sortable<Row.Entry>, Iterable<Row.Entry>,
}
this.chunkcount--;
this.lastTimeWrote = System.currentTimeMillis();
// check if the chunkcache can shrink
checkShrink();
}
@ -504,6 +524,9 @@ public class RowCollection implements Sortable<Row.Entry>, Iterable<Row.Entry>,
if (this.chunkcount == this.sortBound) this.sortBound--;
this.chunkcount--;
this.lastTimeWrote = System.currentTimeMillis();
// check if the chunkcache can shrink
checkShrink();
return r;
}