Marown/yacy_search_server
1
0
Fork 0
mirror of https://github.com/yacy/yacy_search_server.git synced 2024-09-19 00:01:41 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity

better RAM protection using eco tables

Browse Source 
git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@4345 6c8d7289-2bf4-0310-a012-ef5d649a1542
This commit is contained in:
orbiter 2008-01-19 01:50:24 +00:00
parent f6cfb97b7f
commit d4d07802ac
4 changed files with 30 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
source/de/anomic/kelondro/kelondroCollectionIndex.java
View File

@ -154,12 +154,17 @@ public class kelondroCollectionIndex {
}
}
serverLog.logFine("STARTUP", "STARTED INITIALIZATION OF NEW COLLECTION INDEX WITH " + initialSpace + " ENTRIES. THIS WILL TAKE SOME TIME");
kelondroRow indexRowdef = indexRow(keyLength, indexOrder);
long necessaryRAM4fullTable = minimumRAM4Eco + (indexRowdef.objectsize + 4) * initialSpace * 3 / 2;
long necessaryRAM4fullIndex = minimumRAM4Eco + (indexRowdef.primaryKeyLength + 4) * initialSpace * 3 / 2;
// initialize (new generation) index table from file
if (serverMemory.request(minimumRAM4Eco, false)) {
index = new kelondroEcoTable(f, indexRow(keyLength, indexOrder), true, EcoFSBufferSize);
if (serverMemory.request(necessaryRAM4fullTable, false)) {
index = new kelondroEcoTable(f, indexRowdef, true, EcoFSBufferSize);
} else if (serverMemory.request(necessaryRAM4fullIndex, false)) {
index = new kelondroEcoTable(f, indexRowdef, false, EcoFSBufferSize);
} else {
index = new kelondroFlexTable(path, filenameStub + ".index", preloadTime, indexRow(keyLength, indexOrder), initialSpace, true);
index = new kelondroFlexTable(path, filenameStub + ".index", preloadTime, indexRowdef, initialSpace, true);
}
// open array files
@ -233,9 +238,11 @@ public class kelondroCollectionIndex {
long preloadTime, int loadfactor, kelondroRow rowdef, int initialSpace) throws IOException {
// open/create index table
File f = new File(path, filenameStub + ".index");
kelondroRow indexRowdef = indexRow(keylength, indexOrder);
if (f.isDirectory()) {
// use a flextable
kelondroIndex theindex = new kelondroCache(new kelondroFlexTable(path, filenameStub + ".index", preloadTime, indexRow(keylength, indexOrder), initialSpace, true));
kelondroIndex theindex = new kelondroCache(new kelondroFlexTable(path, filenameStub + ".index", preloadTime, indexRowdef, initialSpace, true));
// save/check property file for this array
File propfile = propertyFile(path, filenameStub, loadfactor, rowdef.objectsize);
@ -255,7 +262,9 @@ public class kelondroCollectionIndex {
return theindex;
} else {
// open a ecotable
return new kelondroEcoTable(f, indexRow(keylength, indexOrder), true, EcoFSBufferSize);
long records = f.length() / indexRowdef.objectsize;
long necessaryRAM4fullTable = minimumRAM4Eco + (indexRowdef.objectsize + 4) * records * 3 / 2;
return new kelondroEcoTable(f, indexRowdef, serverMemory.request(necessaryRAM4fullTable, false), EcoFSBufferSize);
}
}

2
source/de/anomic/kelondro/kelondroEcoTable.java
View File

@ -90,7 +90,7 @@ public class kelondroEcoTable implements kelondroIndex {
// initialize index and copy table
int records = file.size();
long neededRAM4table = records * taildef.objectsize * 3 / 2;
long neededRAM4table = 10 * 1024 * 1024 + records * (rowdef.objectsize + 4) * 3 / 2;
table = ((useTailCache) && (serverMemory.request(neededRAM4table, true))) ? new kelondroRowSet(taildef, records + 1) : null;
index = new kelondroBytesIntMap(rowdef.primaryKeyLength, rowdef.objectOrder, records + 1);

6
source/de/anomic/plasma/plasmaCrawlBalancer.java
View File

@ -80,6 +80,7 @@ public class plasmaCrawlBalancer {
private File cacheStacksPath;
private String stackname;
private boolean top; // to alternate between top and bottom of the file stack
private boolean fullram;
public static class domaccess {
long time;
@ -100,7 +101,7 @@ public class plasmaCrawlBalancer {
}
}
public plasmaCrawlBalancer(File cachePath, String stackname) {
public plasmaCrawlBalancer(File cachePath, String stackname, boolean fullram) {
this.cacheStacksPath = cachePath;
this.stackname = stackname;
File stackFile = new File(cachePath, stackname + stackSuffix);
@ -108,6 +109,7 @@ public class plasmaCrawlBalancer {
this.domainStacks = new HashMap<String, LinkedList<String>>();
this.urlRAMStack = new ArrayList<String>();
this.top = true;
this.fullram = fullram;
// create a stack for newly entered entries
if (!(cachePath.exists())) cachePath.mkdir(); // make the path
@ -140,7 +142,7 @@ public class plasmaCrawlBalancer {
private void openFileIndex() {
cacheStacksPath.mkdirs();
urlFileIndex = new kelondroEcoTable(new File(cacheStacksPath, stackname + indexSuffix), plasmaCrawlEntry.rowdef, true, EcoFSBufferSize);
urlFileIndex = new kelondroEcoTable(new File(cacheStacksPath, stackname + indexSuffix), plasmaCrawlEntry.rowdef, fullram, EcoFSBufferSize);
}
private void resetFileIndex() {

20
source/de/anomic/plasma/plasmaCrawlNURL.java
View File

@ -74,10 +74,10 @@ public class plasmaCrawlNURL {
public plasmaCrawlNURL(File cachePath) {
super();
coreStack = new plasmaCrawlBalancer(cachePath, "urlNoticeCoreStack");
limitStack = new plasmaCrawlBalancer(cachePath, "urlNoticeLimitStack");
coreStack = new plasmaCrawlBalancer(cachePath, "urlNoticeCoreStack", true);
limitStack = new plasmaCrawlBalancer(cachePath, "urlNoticeLimitStack", false);
//overhangStack = new plasmaCrawlBalancer(overhangStackFile);
remoteStack = new plasmaCrawlBalancer(cachePath, "urlNoticeRemoteStack");
remoteStack = new plasmaCrawlBalancer(cachePath, "urlNoticeRemoteStack", true);
}
public void close() {
@ -213,7 +213,7 @@ public class plasmaCrawlNURL {
private plasmaCrawlEntry[] top(plasmaCrawlBalancer balancer, int count) {
// this is a filo - top
if (count > balancer.size()) count = balancer.size();
ArrayList list = new ArrayList(count);
ArrayList<plasmaCrawlEntry> list = new ArrayList<plasmaCrawlEntry>(count);
for (int i = 0; i < count; i++) {
try {
plasmaCrawlEntry entry = balancer.top(i);
@ -226,15 +226,15 @@ public class plasmaCrawlNURL {
return (plasmaCrawlEntry[]) list.toArray(new plasmaCrawlEntry[list.size()]);
}
public Iterator iterator(int stackType) {
public Iterator<plasmaCrawlEntry> iterator(int stackType) {
// returns an iterator of plasmaCrawlBalancerEntry Objects
try {switch (stackType) {
case STACK_TYPE_CORE: return coreStack.iterator();
case STACK_TYPE_LIMIT: return limitStack.iterator();
case STACK_TYPE_REMOTE: return remoteStack.iterator();
default: return null;
case STACK_TYPE_CORE: return coreStack.iterator();
case STACK_TYPE_LIMIT: return limitStack.iterator();
case STACK_TYPE_REMOTE: return remoteStack.iterator();
default: return null;
}} catch (IOException e) {
return new HashSet().iterator();
return new HashSet<plasmaCrawlEntry>().iterator();
}
}