Marown/yacy_search_server
1
0
Fork 0
mirror of https://github.com/yacy/yacy_search_server.git synced 2024-09-19 00:01:41 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity

fixed some problems with eco tables

Browse Source 
git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@4346 6c8d7289-2bf4-0310-a012-ef5d649a1542
This commit is contained in:
orbiter 2008-01-19 12:23:56 +00:00
parent d4d07802ac
commit 58a1f518f8
11 changed files with 75 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
source/dbtest.java
View File

@ -215,7 +215,7 @@ public class dbtest {
table = new kelondroSplitTable(tablepath, new File(tablename).getName(), preload, testRow, true);
}
if (dbe.equals("kelondroEcoTable")) {
table = new kelondroEcoTable(new File(tablename), testRow, true, 100);
table = new kelondroEcoTable(new File(tablename), testRow, kelondroEcoTable.tailCacheDenyUsage /*kelondroEcoTable.tailCacheForceUsage*/, 100);
}
if (dbe.equals("mysql")) {
table = new kelondroSQLTable("mysql", testRow);
@ -377,6 +377,16 @@ public class dbtest {
//
// args: <number-of-writes> <number-of-reads-per-write> <random-startpoint>
// example: kelondroFlexTable stressThreaded /Users/admin/dbtest 500 50 0
/* result with svn 4346
kelondroFlex:
removed: 70, size of jcontrol set: 354, size of kcontrol set: 354
Database size = 354 unique entries.
Execution time: open=1329, command=36234, close=17, total=37580
kelondroEco:
removed: 70, size of jcontrol set: 354, size of kcontrol set: 354
Database size = 354 unique entries.
Execution time: open=1324, command=34032, close=1, total=35357
*/
long writeCount = Long.parseLong(args[3]);
long readCount = Long.parseLong(args[4]);
long randomstart = Long.parseLong(args[5]);

2
source/de/anomic/kelondro/kelondroBytesIntMap.java
View File

@ -76,7 +76,7 @@ public class kelondroBytesIntMap {
public synchronized int removei(byte[] key) throws IOException {
assert (key != null);
kelondroRow.Entry indexentry = index.remove(key, false);
kelondroRow.Entry indexentry = index.remove(key, true); // keeping the order will prevent multiple re-sorts
if (indexentry == null) return -1;
return (int) indexentry.getColLong(1);
}

11
source/de/anomic/kelondro/kelondroCollectionIndex.java
View File

@ -160,9 +160,9 @@ public class kelondroCollectionIndex {
// initialize (new generation) index table from file
if (serverMemory.request(necessaryRAM4fullTable, false)) {
index = new kelondroEcoTable(f, indexRowdef, true, EcoFSBufferSize);
index = new kelondroEcoTable(f, indexRowdef, kelondroEcoTable.tailCacheUsageAuto, EcoFSBufferSize);
} else if (serverMemory.request(necessaryRAM4fullIndex, false)) {
index = new kelondroEcoTable(f, indexRowdef, false, EcoFSBufferSize);
index = new kelondroEcoTable(f, indexRowdef, kelondroEcoTable.tailCacheDenyUsage, EcoFSBufferSize);
} else {
index = new kelondroFlexTable(path, filenameStub + ".index", preloadTime, indexRowdef, initialSpace, true);
}
@ -221,7 +221,8 @@ public class kelondroCollectionIndex {
ientry.setCol(idx_col_indexpos, aentry.index());
ientry.setCol(idx_col_lastread, t);
ientry.setCol(idx_col_lastwrote, t);
index.addUnique(ientry); // FIXME: this should avoid doubles
//index.addUnique(ientry); // FIXME: this should avoid doubles
index.put(ientry);
count++;
// write a log
@ -264,7 +265,7 @@ public class kelondroCollectionIndex {
// open a ecotable
long records = f.length() / indexRowdef.objectsize;
long necessaryRAM4fullTable = minimumRAM4Eco + (indexRowdef.objectsize + 4) * records * 3 / 2;
return new kelondroEcoTable(f, indexRowdef, serverMemory.request(necessaryRAM4fullTable, false), EcoFSBufferSize);
return new kelondroEcoTable(f, indexRowdef, (serverMemory.request(necessaryRAM4fullTable, false)) ? kelondroEcoTable.tailCacheUsageAuto : kelondroEcoTable.tailCacheDenyUsage, EcoFSBufferSize);
}
}
@ -978,7 +979,7 @@ public class kelondroCollectionIndex {
kelondroRowSet collection = new kelondroRowSet(this.payloadrow, arrayrow, 1); // FIXME: this does not yet work with different rowdef in case of several rowdef.objectsize()
if ((!(index.row().objectOrder.wellformed(indexkey))) || (index.row().objectOrder.compare(arraykey, indexkey) != 0)) {
// check if we got the right row; this row is wrong. Fix it:
index.remove(indexkey, true); // the wrong row cannot be fixed
index.remove(indexkey, false); // the wrong row cannot be fixed
// store the row number in the index; this may be a double-entry, but better than nothing
kelondroRow.Entry indexEntry = index.row().newEntry();
indexEntry.setCol(idx_col_key, arrayrow.getColBytes(0));

4
source/de/anomic/kelondro/kelondroDyn.java
View File

@ -99,10 +99,10 @@ public class kelondroDyn {
if (file.isDirectory()) {
fbi = new kelondroFlexTable(file.getParentFile(), file.getName(), 10000, rowdef, 0, resetOnFail);
} else {
fbi = new kelondroEcoTable(file, rowdef, false, EcoFSBufferSize);
fbi = new kelondroEcoTable(file, rowdef, kelondroEcoTable.tailCacheUsageAuto, EcoFSBufferSize);
}
} else {
fbi = new kelondroEcoTable(file, rowdef, false, EcoFSBufferSize);
fbi = new kelondroEcoTable(file, rowdef, kelondroEcoTable.tailCacheUsageAuto, EcoFSBufferSize);
}
}
this.index = (useObjectCache) ? (kelondroIndex) new kelondroCache(fbi) : fbi;

9
source/de/anomic/kelondro/kelondroEcoFS.java
View File

@ -485,7 +485,14 @@ public class kelondroEcoFS {
System.out.println("size = " + t.size());
t.clean(t.size() - 2);
t.cleanLast();
System.out.println("size = " + t.size());
long start = System.currentTimeMillis();
long c = 0;
for (int i = 0; i < 100000; i++) {
c = t.size();
}
System.out.println("size() needs " + ((System.currentTimeMillis() - start) / 100) + " nanoseconds");
System.out.println("size = " + c);
t.close();
} catch (IOException e) {
e.printStackTrace();

55
source/de/anomic/kelondro/kelondroEcoTable.java
View File

@ -54,13 +54,17 @@ public class kelondroEcoTable implements kelondroIndex {
// static tracker objects
private static TreeMap<String, kelondroEcoTable> tableTracker = new TreeMap<String, kelondroEcoTable>();
public static final int tailCacheDenyUsage = 0;
public static final int tailCacheForceUsage = 1;
public static final int tailCacheUsageAuto = 2;
private kelondroRowSet table;
private kelondroBytesIntMap index;
private kelondroBufferedEcoFS file;
private kelondroRow rowdef, taildef;
private int buffersize;
public kelondroEcoTable(File tablefile, kelondroRow rowdef, boolean useTailCache, int buffersize) {
public kelondroEcoTable(File tablefile, kelondroRow rowdef, int useTailCache, int buffersize) {
this.rowdef = rowdef;
this.buffersize = buffersize;
assert rowdef.primaryKeyIndex == 0;
@ -91,7 +95,9 @@ public class kelondroEcoTable implements kelondroIndex {
// initialize index and copy table
int records = file.size();
long neededRAM4table = 10 * 1024 * 1024 + records * (rowdef.objectsize + 4) * 3 / 2;
table = ((useTailCache) && (serverMemory.request(neededRAM4table, true))) ? new kelondroRowSet(taildef, records + 1) : null;
table = ((useTailCache == tailCacheForceUsage) ||
((useTailCache == tailCacheUsageAuto) && (serverMemory.request(neededRAM4table, true)))) ?
new kelondroRowSet(taildef, records + 1) : null;
index = new kelondroBytesIntMap(rowdef.primaryKeyLength, rowdef.objectOrder, records + 1);
// read all elements from the file into the copy table
@ -104,6 +110,7 @@ public class kelondroEcoTable implements kelondroIndex {
// write the key into the index table
System.arraycopy(record, 0, key, 0, rowdef.primaryKeyLength);
index.addi(key, i);
//index.puti(key, i);
// write the tail into the table
if (table != null) table.addUnique(taildef.newEntry(record, rowdef.primaryKeyLength, true));
@ -116,6 +123,11 @@ public class kelondroEcoTable implements kelondroIndex {
e.printStackTrace();
throw new kelondroException(e.getMessage());
}
try {
assert file.size() == index.size() : "file.size() = " + file.size() + ", index.size() = " + index.size();
} catch (IOException e) {
e.printStackTrace();
}
// track this table
tableTracker.put(tablefile.toString(), this);
@ -156,7 +168,7 @@ public class kelondroEcoTable implements kelondroIndex {
}
public synchronized void addUnique(Entry row) throws IOException {
assert (file.size() == index.size());
assert file.size() == index.size() : "file.size() = " + file.size() + ", index.size() = " + index.size();
assert ((table == null) || (table.size() == index.size()));
int i = file.size();
index.addi(row.getPrimaryKeyBytes(), i);
@ -165,13 +177,16 @@ public class kelondroEcoTable implements kelondroIndex {
table.addUnique(taildef.newEntry(row.bytes(), rowdef.primaryKeyLength, true));
}
file.put(i, row.bytes(), 0);
assert file.size() == index.size() : "file.size() = " + file.size() + ", index.size() = " + index.size();
}
public synchronized void addUniqueMultiple(List<Entry> rows) throws IOException {
assert file.size() == index.size() : "file.size() = " + file.size() + ", index.size() = " + index.size();
Iterator<Entry> i = rows.iterator();
while (i.hasNext()) {
addUnique(i.next());
}
assert file.size() == index.size() : "file.size() = " + file.size() + ", index.size() = " + index.size();
}
public void close() {
@ -188,7 +203,7 @@ public class kelondroEcoTable implements kelondroIndex {
}
public synchronized Entry get(byte[] key) throws IOException {
assert (file.size() == index.size());
assert file.size() == index.size() : "file.size() = " + file.size() + ", index.size() = " + index.size();
assert ((table == null) || (table.size() == index.size()));
int i = index.geti(key);
if (i == -1) return null;
@ -204,13 +219,13 @@ public class kelondroEcoTable implements kelondroIndex {
System.arraycopy(key, 0, b, 0, key.length);
System.arraycopy(v.bytes(), 0, b, rowdef.primaryKeyLength, rowdef.objectsize - rowdef.primaryKeyLength);
}
assert (file.size() == index.size());
assert file.size() == index.size() : "file.size() = " + file.size() + ", index.size() = " + index.size();
assert ((table == null) || (table.size() == index.size()));
return rowdef.newEntry(b);
}
public synchronized boolean has(byte[] key) throws IOException {
assert (file.size() == index.size());
assert file.size() == index.size() : "file.size() = " + file.size() + ", index.size() = " + index.size();
assert ((table == null) || (table.size() == index.size()));
return index.geti(key) >= 0;
}
@ -224,7 +239,7 @@ public class kelondroEcoTable implements kelondroIndex {
}
public synchronized Entry put(Entry row) throws IOException {
assert (file.size() == index.size());
assert file.size() == index.size() : "file.size() = " + file.size() + ", index.size() = " + index.size();
assert ((table == null) || (table.size() == index.size()));
int i = index.geti(row.getPrimaryKeyBytes());
if (i == -1) {
@ -247,7 +262,7 @@ public class kelondroEcoTable implements kelondroIndex {
table.set(i, taildef.newEntry(row.bytes(), rowdef.primaryKeyLength, true));
file.put(i, row.bytes(), 0);
}
assert (file.size() == index.size());
assert file.size() == index.size() : "file.size() = " + file.size() + ", index.size() = " + index.size();
assert ((table == null) || (table.size() == index.size()));
// return old value
return rowdef.newEntry(b);
@ -258,14 +273,16 @@ public class kelondroEcoTable implements kelondroIndex {
}
public synchronized void putMultiple(List<Entry> rows) throws IOException {
assert file.size() == index.size() : "file.size() = " + file.size() + ", index.size() = " + index.size();
Iterator<Entry> i = rows.iterator();
while (i.hasNext()) {
put(i.next());
}
assert file.size() == index.size() : "file.size() = " + file.size() + ", index.size() = " + index.size();
}
public synchronized Entry remove(byte[] key, boolean keepOrder) throws IOException {
assert (file.size() == index.size());
assert file.size() == index.size() : "file.size() = " + file.size() + ", index.size() = " + index.size();
assert ((table == null) || (table.size() == index.size()));
assert keepOrder == false; // this class cannot keep the order during a remove
int i = index.geti(key);
@ -311,13 +328,13 @@ public class kelondroEcoTable implements kelondroIndex {
assert ((table == null) || (table.size() == index.size())) : "table.size() = " + table.size() + ", index.size() = " + index.size();
}
}
assert (file.size() == index.size());
assert file.size() == index.size() : "file.size() = " + file.size() + ", index.size() = " + index.size();
assert ((table == null) || (table.size() == index.size()));
return rowdef.newEntry(b);
}
public synchronized Entry removeOne() throws IOException {
assert (file.size() == index.size());
assert file.size() == index.size() : "file.size() = " + file.size() + ", index.size() = " + index.size();
assert ((table == null) || (table.size() == index.size()));
byte[] le = new byte[rowdef.objectsize];
file.cleanLast(le, 0);
@ -325,6 +342,7 @@ public class kelondroEcoTable implements kelondroIndex {
int i = index.removei(lr.getPrimaryKeyBytes());
assert i >= 0;
table.removeRow(i, false);
assert file.size() == index.size() : "file.size() = " + file.size() + ", index.size() = " + index.size();
return lr;
}
@ -428,10 +446,10 @@ public class kelondroEcoTable implements kelondroIndex {
}
public static kelondroIndex testTable(File f, String testentities) throws IOException {
public static kelondroIndex testTable(File f, String testentities, int testcase) throws IOException {
if (f.exists()) f.delete();
kelondroRow rowdef = new kelondroRow("byte[] a-4, byte[] b-4", kelondroNaturalOrder.naturalOrder, 0);
kelondroIndex tt = new kelondroEcoTable(f, rowdef, true, 100);
kelondroIndex tt = new kelondroEcoTable(f, rowdef, testcase, 100);
byte[] b;
kelondroRow.Entry row = rowdef.newEntry();
for (int i = 0; i < testentities.length(); i++) {
@ -443,7 +461,7 @@ public class kelondroEcoTable implements kelondroIndex {
return tt;
}
public static void bigtest(int elements, File testFile) {
public static void bigtest(int elements, File testFile, int testcase) {
System.out.println("starting big test with " + elements + " elements:");
long start = System.currentTimeMillis();
String[] s = kelondroTree.permutations(elements);
@ -452,13 +470,13 @@ public class kelondroEcoTable implements kelondroIndex {
for (int i = 0; i < s.length; i++) {
System.out.println("*** probing tree " + i + " for permutation " + s[i]);
// generate tree and delete elements
tt = testTable(testFile, s[i]);
tt = testTable(testFile, s[i], testcase);
if (kelondroTree.countElements(tt) != tt.size()) {
System.out.println("wrong size for " + s[i]);
}
tt.close();
for (int j = 0; j < s.length; j++) {
tt = testTable(testFile, s[i]);
tt = testTable(testFile, s[i], testcase);
// delete by permutation j
for (int elt = 0; elt < s[j].length(); elt++) {
tt.remove(kelondroTree.testWord(s[j].charAt(elt)), false);
@ -479,7 +497,10 @@ public class kelondroEcoTable implements kelondroIndex {
public static void main(String[] args) {
// open a file, add one entry and exit
File f = new File(args[0]);
bigtest(5, f);
System.out.println("========= Testcase: no tail cache:");
bigtest(5, f, tailCacheDenyUsage);
System.out.println("========= Testcase: with tail cache:");
bigtest(5, f, tailCacheForceUsage);
/*
kelondroRow row = new kelondroRow("byte[] key-4, byte[] x-5", kelondroNaturalOrder.naturalOrder, 0);
try {

1
source/de/anomic/kelondro/kelondroRAMIndex.java
View File

@ -123,6 +123,7 @@ public class kelondroRAMIndex implements kelondroIndex {
}
public synchronized kelondroRow.Entry remove(byte[] key, boolean keepOrder) throws IOException {
assert keepOrder == true; // if this is false, the index must be re-ordered so many times which will cause a major CPU usage
finishInitialization();
// if the new entry is within the initialization part, just delete it
kelondroRow.Entry indexentry = index0.remove(key, keepOrder);

8
source/de/anomic/kelondro/kelondroSplitTable.java
View File

@ -118,7 +118,7 @@ public class kelondroSplitTable implements kelondroIndex {
// this is a kelonodroFlex table
table = new kelondroCache(new kelondroFlexTable(path, maxf, preloadTime, rowdef, 0, resetOnFail));
} else {
table = new kelondroEcoTable(f, rowdef, false, EcoFSBufferSize);
table = new kelondroEcoTable(f, rowdef, kelondroEcoTable.tailCacheDenyUsage, EcoFSBufferSize);
}
tables.put(date, table);
}
@ -228,13 +228,13 @@ public class kelondroSplitTable implements kelondroIndex {
table = new kelondroFlexTable(path, tablename + "." + suffix, -1, rowdef, 0, true);
} else {
// open a eco table
table = new kelondroEcoTable(f, rowdef, false, EcoFSBufferSize);
table = new kelondroEcoTable(f, rowdef, kelondroEcoTable.tailCacheDenyUsage, EcoFSBufferSize);
}
} else {
// make new table
if (serverMemory.request(minimumRAM4Eco, true)) {
// enough memory for a ecoTable
table = new kelondroEcoTable(f, rowdef, false, EcoFSBufferSize);
table = new kelondroEcoTable(f, rowdef, kelondroEcoTable.tailCacheDenyUsage, EcoFSBufferSize);
} else {
// use the flex table
table = new kelondroFlexTable(path, tablename + "." + suffix, -1, rowdef, 0, true);
@ -272,7 +272,7 @@ public class kelondroSplitTable implements kelondroIndex {
// make new table
if (serverMemory.request(minimumRAM4Eco, true)) {
// enough memory for a ecoTable
table = new kelondroEcoTable(new File(path, tablename + "." + suffix), rowdef, false, EcoFSBufferSize);
table = new kelondroEcoTable(new File(path, tablename + "." + suffix), rowdef, kelondroEcoTable.tailCacheDenyUsage, EcoFSBufferSize);
} else {
// use the flex table
table = new kelondroFlexTable(path, tablename + "." + suffix, -1, rowdef, 0, true);

2
source/de/anomic/plasma/plasmaCrawlBalancer.java
View File

@ -142,7 +142,7 @@ public class plasmaCrawlBalancer {
private void openFileIndex() {
cacheStacksPath.mkdirs();
urlFileIndex = new kelondroEcoTable(new File(cacheStacksPath, stackname + indexSuffix), plasmaCrawlEntry.rowdef, fullram, EcoFSBufferSize);
urlFileIndex = new kelondroEcoTable(new File(cacheStacksPath, stackname + indexSuffix), plasmaCrawlEntry.rowdef, (fullram) ? kelondroEcoTable.tailCacheUsageAuto : kelondroEcoTable.tailCacheDenyUsage, EcoFSBufferSize);
}
private void resetFileIndex() {

4
source/de/anomic/plasma/plasmaCrawlStacker.java
View File

@ -302,7 +302,7 @@ public final class plasmaCrawlStacker extends Thread {
cacheStacksPath.mkdirs();
File f = new File(cacheStacksPath, stackfile);
try {
this.urlEntryCache = new kelondroEcoTable(f, plasmaCrawlEntry.rowdef, true, EcoFSBufferSize);
this.urlEntryCache = new kelondroEcoTable(f, plasmaCrawlEntry.rowdef, kelondroEcoTable.tailCacheUsageAuto, EcoFSBufferSize);
//this.urlEntryCache = new kelondroCache(new kelondroFlexTable(cacheStacksPath, newCacheName, preloadTime, plasmaCrawlEntry.rowdef, 0, true));
} catch (Exception e) {
e.printStackTrace();
@ -310,7 +310,7 @@ public final class plasmaCrawlStacker extends Thread {
f.delete();
//kelondroFlexTable.delete(cacheStacksPath, newCacheName);
try {
this.urlEntryCache = new kelondroEcoTable(f, plasmaCrawlEntry.rowdef, true, EcoFSBufferSize);
this.urlEntryCache = new kelondroEcoTable(f, plasmaCrawlEntry.rowdef, kelondroEcoTable.tailCacheUsageAuto, EcoFSBufferSize);
//this.urlEntryCache = new kelondroCache(new kelondroFlexTable(cacheStacksPath, newCacheName, preloadTime, plasmaCrawlEntry.rowdef, 0, true));
} catch (Exception ee) {
ee.printStackTrace();

2
source/de/anomic/plasma/plasmaCrawlZURL.java
View File

@ -69,7 +69,7 @@ public class plasmaCrawlZURL {
if (f.isDirectory()) kelondroFlexTable.delete(cachePath, tablename); else f.delete();
}
}
urlIndex = new kelondroEcoTable(f, rowdef, true, EcoFSBufferSize);
urlIndex = new kelondroEcoTable(f, rowdef, kelondroEcoTable.tailCacheUsageAuto, EcoFSBufferSize);
//urlIndex = new kelondroFlexTable(cachePath, tablename, -1, rowdef, 0, true);
}