mirror of
https://github.com/yacy/yacy_search_server.git
synced 2024-09-19 00:01:41 +02:00
- less restrictions for usage of Table RAM copy
- new limit to use the table copy (instead of flag): 400MB available. If less is available, then a copy is never used. If more is available, then it can be used if there is a remaining space of at least 200MB - flush caches more often: flush the Digest cache
This commit is contained in:
parent
b8f56a9803
commit
41c02cb10e
|
@ -623,10 +623,11 @@ cleanup.failedSearchURLtimeout = 86400000
|
|||
javastart_Xmx=Xmx600m
|
||||
javastart_Xms=Xms600m
|
||||
|
||||
# YaCy is able to use RAM copies of database tables. This needs a lot of RAM
|
||||
# To switch copying of file tables int RAM on, use this property
|
||||
# this value is automatically set to true, if more than one gigabyte RAM is available
|
||||
ramcopy=false
|
||||
# YaCy is able to use RAM copies of database tables. This needs a lot of RAM.
|
||||
# To switch on copying of file tables int RAM, there must be enough memory
|
||||
# The memory that is available at startup time is used to switch the feature on
|
||||
# The tableCachingLimit is the amount of free RAM at startup time to switch on the feature
|
||||
tableCachingLimit=419430400
|
||||
|
||||
# some java versions may be limited to a specific array size
|
||||
# of 134217727 entries. To prevent that tables of that size are generated,
|
||||
|
|
|
@ -60,12 +60,19 @@ public class Digest {
|
|||
private static ARC<String, byte[]> md5Cache = null;
|
||||
static {
|
||||
try {
|
||||
md5Cache = new ConcurrentARC<String, byte[]>(md5CacheSize, Math.max(32, 4 * Runtime.getRuntime().availableProcessors()));
|
||||
md5Cache = new ConcurrentARC<String, byte[]>(md5CacheSize, Math.max(8, 2 * Runtime.getRuntime().availableProcessors()));
|
||||
} catch (final OutOfMemoryError e) {
|
||||
md5Cache = new ConcurrentARC<String, byte[]>(1000, Math.max(8, 2 * Runtime.getRuntime().availableProcessors()));
|
||||
md5Cache = new ConcurrentARC<String, byte[]>(1000, Math.max(2, Runtime.getRuntime().availableProcessors()));
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* clean the md5 cache
|
||||
*/
|
||||
public static void cleanup() {
|
||||
md5Cache.clear();
|
||||
}
|
||||
|
||||
public static String encodeHex(final long in, final int length) {
|
||||
String s = Long.toHexString(in);
|
||||
while (s.length() < length) s = "0" + s;
|
||||
|
@ -114,7 +121,7 @@ public class Digest {
|
|||
// generate a hex representation from the md5 of a byte-array
|
||||
return encodeHex(encodeMD5Raw(b));
|
||||
}
|
||||
|
||||
|
||||
public static byte[] encodeMD5Raw(final String key) {
|
||||
|
||||
byte[] h = md5Cache.get(key);
|
||||
|
|
|
@ -93,7 +93,7 @@ public class Table implements Index, Iterable<Row.Entry> {
|
|||
|
||||
this.rowdef = rowdef;
|
||||
this.buffersize = buffersize;
|
||||
this.minmemremaining = Math.max(400 * 1024 * 1024, MemoryControl.available() / 10);
|
||||
this.minmemremaining = Math.max(200L * 1024L * 1024L, MemoryControl.available() / 10);
|
||||
//this.fail = 0;
|
||||
// define the taildef, a row like the rowdef but without the first column
|
||||
final Column[] cols = new Column[rowdef.columns() - 1];
|
||||
|
@ -123,19 +123,18 @@ public class Table implements Index, Iterable<Row.Entry> {
|
|||
|
||||
// initialize index and copy table
|
||||
final int records = Math.max(fileSize, initialSpace);
|
||||
final long neededRAM4table = (records) * ((rowdef.objectsize) + 4L) * 3L;
|
||||
this.table = ((exceed134217727 || neededRAM4table < maxarraylength) &&
|
||||
(useTailCache && MemoryControl.available() > neededRAM4table + 200 * 1024 * 1024)) ?
|
||||
final long neededRAM4table = 200L * 1024L * 1024L + records * (rowdef.objectsize + rowdef.primaryKeyLength) * 3L / 2L;
|
||||
this.table = ((exceed134217727 || neededRAM4table < maxarraylength) && useTailCache && MemoryControl.available() > neededRAM4table) ?
|
||||
new RowSet(this.taildef, records) : null;
|
||||
Log.logInfo("TABLE", "initialization of " + tablefile.getName() + ". table copy: " + ((this.table == null) ? "no" : "yes") + ", available RAM: " + (MemoryControl.available() / 1024 / 1024) + "MB, needed: " + (neededRAM4table/1024/1024 + 200) + "MB, allocating space for " + records + " entries");
|
||||
final long neededRAM4index = 400 * 1024 * 1024 + records * (rowdef.primaryKeyLength + 4) * 3 / 2;
|
||||
if (!MemoryControl.request(neededRAM4index, false)) {
|
||||
final long neededRAM4index = 200L * 1024L * 1024L + records * rowdef.primaryKeyLength * 3L / 2L;
|
||||
if (!MemoryControl.request(neededRAM4index, true)) {
|
||||
// despite calculations seemed to show that there is enough memory for the table AND the index
|
||||
// there is now not enough memory left for the index. So delete the table again to free the memory
|
||||
// for the index
|
||||
Log.logSevere("TABLE", tablefile.getName() + ": not enough RAM (" + (MemoryControl.available() / 1024 / 1024) + "MB) left for index, deleting allocated table space to enable index space allocation (needed: " + (neededRAM4index / 1024 / 1024) + "MB)");
|
||||
Log.logSevere("TABLE", tablefile.getName() + ": not enough RAM (" + (MemoryControl.available() / 1024L / 1024L) + "MB) left for index, deleting allocated table space to enable index space allocation (needed: " + (neededRAM4index / 1024 / 1024) + "MB)");
|
||||
this.table = null; System.gc();
|
||||
Log.logSevere("TABLE", tablefile.getName() + ": RAM after releasing the table: " + (MemoryControl.available() / 1024 / 1024) + "MB");
|
||||
Log.logSevere("TABLE", tablefile.getName() + ": RAM after releasing the table: " + (MemoryControl.available() / 1024L / 1024L) + "MB");
|
||||
}
|
||||
this.index = new HandleMap(rowdef.primaryKeyLength, rowdef.objectOrder, 4, records, tablefile.getAbsolutePath());
|
||||
final HandleMap errors = new HandleMap(rowdef.primaryKeyLength, NaturalOrder.naturalOrder, 4, records, tablefile.getAbsolutePath() + ".errors");
|
||||
|
@ -178,14 +177,14 @@ public class Table implements Index, Iterable<Row.Entry> {
|
|||
this.table = null;
|
||||
break;
|
||||
}
|
||||
if (abandonTable()) {
|
||||
this.table = null;
|
||||
break;
|
||||
}
|
||||
} else {
|
||||
errors.putUnique(key, i++);
|
||||
}
|
||||
}
|
||||
Runtime.getRuntime().gc();
|
||||
if (abandonTable()) {
|
||||
this.table = null;
|
||||
}
|
||||
}
|
||||
this.index.trim();
|
||||
|
||||
|
|
|
@ -302,12 +302,12 @@ public final class Switchboard extends serverSwitch
|
|||
initRemoteProxy();
|
||||
|
||||
// memory configuration
|
||||
this.useTailCache = getConfigBool("ramcopy", true);
|
||||
if ( MemoryControl.available() > 1024 * 1024 * 1024 * 1 ) {
|
||||
long tableCachingLimit = getConfigLong("tableCachingLimit", 419430400L);
|
||||
if ( MemoryControl.available() > tableCachingLimit ) {
|
||||
this.useTailCache = true;
|
||||
}
|
||||
this.exceed134217727 = getConfigBool("exceed134217727", true);
|
||||
if ( MemoryControl.available() > 1024 * 1024 * 1024 * 2 ) {
|
||||
if ( MemoryControl.available() > 1024L * 1024L * 1024L * 2L ) {
|
||||
this.exceed134217727 = true;
|
||||
}
|
||||
|
||||
|
@ -1911,6 +1911,7 @@ public final class Switchboard extends serverSwitch
|
|||
try {
|
||||
// flush the document compressor cache
|
||||
Cache.commit();
|
||||
Digest.cleanup(); // don't let caches become permanent memory leaks
|
||||
|
||||
// clear caches if necessary
|
||||
if ( !MemoryControl.request(8000000L, false) ) {
|
||||
|
|
|
@ -97,6 +97,7 @@ public final class MetadataRepository implements /*Metadata,*/ Iterable<byte[]>
|
|||
|
||||
public void clearCache() {
|
||||
if (this.urlIndexFile instanceof Cache) ((Cache) this.urlIndexFile).clearCache();
|
||||
if (this.statsDump != null) this.statsDump.clear();
|
||||
this.statsDump = null;
|
||||
}
|
||||
|
||||
|
|
Loading…
Reference in New Issue
Block a user