yacy_search_server/source/dbtest.java
orbiter 8444357291 added new row interator in kelondro tables files that enumerates rows
without an order by the primary key. The result is a very fast enumeration of the Eco table data structure. Other table data types are not affected.
The new enumerator is used for the url export function that can be accessed from the online interface (Index Administration -> URL References -> Export). This export should now be much faster, if all url database files are from type Eco
The new enumeration is also used at other functions in YaCy, i.e. the initialization of the crawl balancer and the initialization of YaCy News.

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@5647 6c8d7289-2bf4-0310-a012-ef5d649a1542
2009-02-24 10:40:20 +00:00

560 lines
27 KiB
Java

// test application
// shall be used to compare the kelondroDB with other databases
// with relevance to yacy-specific use cases
import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashSet;
import java.util.Iterator;
import java.util.Random;
import javax.imageio.ImageIO;
import de.anomic.kelondro.index.Row;
import de.anomic.kelondro.index.RowSet;
import de.anomic.kelondro.index.ObjectIndex;
import de.anomic.kelondro.index.IntBytesMap;
import de.anomic.kelondro.order.Base64Order;
import de.anomic.kelondro.order.CloneableIterator;
import de.anomic.kelondro.order.NaturalOrder;
import de.anomic.kelondro.table.EcoTable;
import de.anomic.kelondro.table.Tree;
import de.anomic.kelondro.table.SQLTable;
import de.anomic.kelondro.table.SplitTable;
import de.anomic.kelondro.util.MemoryControl;
import de.anomic.server.serverInstantBusyThread;
import de.anomic.ymage.ymageChart;
public class dbtest {
public final static int keylength = 12;
public final static int valuelength = 223; // sum of all data length as defined in plasmaURL
private static final byte[] dummyvalue2 = new byte[valuelength];
static {
// fill the dummy value
for (int i = 0; i < valuelength; i++) dummyvalue2[i] = '-';
dummyvalue2[0] = '{';
dummyvalue2[valuelength - 1] = '}';
}
public static byte[] randomHash(final long r0, final long r1) {
// a long can have 64 bit, but a 12-byte hash can have 6 * 12 = 72 bits
// so we construct a generic Hash using two long values
final String s = (Base64Order.enhancedCoder.encodeLong(Math.abs(r0), 6) +
Base64Order.enhancedCoder.encodeLong(Math.abs(r1), 6));
return s.getBytes();
}
public static byte[] randomHash(final Random r) {
return randomHash(r.nextLong(), r.nextLong());
}
public final static class STEntry {
private final byte[] key;
private final byte[] value;
public STEntry(final long aSource) {
this.key = randomHash(aSource, aSource);
this.value = new byte[valuelength];
for (int i = 0; i < valuelength; i++) this.value[i] = 0;
final byte[] tempKey = String.valueOf(aSource).getBytes();
System.arraycopy(tempKey, 0, this.value, 0, tempKey.length);
}
public STEntry(final byte[] aKey, final byte[] aValue) {
this.key = aKey;
this.value = aValue;
}
public boolean isValid() {
final String s = new String(this.value).trim();
if (s.length() == 0) return false;
final long source = Long.valueOf(s).longValue();
return new String(this.key).equals(new String(randomHash(source, source)));
}
public byte[] getKey() {
return this.key;
}
public byte[] getValue() {
return this.value;
}
public String toString() {
return new String(this.key) + "#" + new String(this.value);
}
}
public static abstract class STJob implements Runnable {
private final ObjectIndex table_test, table_reference;
private final long source;
public STJob(final ObjectIndex table_test, final ObjectIndex table_reference, final long aSource) {
this.table_test = table_test;
this.table_reference = table_reference;
this.source = aSource;
}
public ObjectIndex getTable_test() {
return this.table_test;
}
public ObjectIndex getTable_reference() {
return this.table_reference;
}
public abstract void run();
public long getSource() {
return this.source;
}
}
public static final class WriteJob extends STJob {
public WriteJob(final ObjectIndex table_test, final ObjectIndex table_reference, final long aSource) {
super(table_test, table_reference, aSource);
}
public void run() {
final STEntry entry = new STEntry(this.getSource());
System.out.println("write: " + NaturalOrder.arrayList(entry.getKey(), 0, entry.getKey().length));
try {
getTable_test().put(getTable_test().row().newEntry(new byte[][] { entry.getKey(), entry.getValue() , entry.getValue() }));
if (getTable_reference() != null) getTable_reference().put(getTable_test().row().newEntry(new byte[][] { entry.getKey(), entry.getValue() , entry.getValue() }));
} catch (final IOException e) {
System.err.println(e);
e.printStackTrace();
System.exit(0);
}
}
}
public static final class RemoveJob extends STJob {
public RemoveJob(final ObjectIndex table_test, final ObjectIndex table_reference, final long aSource) {
super(table_test, table_reference, aSource);
}
public void run() {
final STEntry entry = new STEntry(this.getSource());
System.out.println("remove: " + NaturalOrder.arrayList(entry.getKey(), 0, entry.getKey().length));
try {
getTable_test().remove(entry.getKey());
if (getTable_reference() != null) getTable_reference().remove(entry.getKey());
} catch (final IOException e) {
System.err.println(e);
e.printStackTrace();
System.exit(0);
}
}
}
public static final class ReadJob extends STJob {
public ReadJob(final ObjectIndex table_test, final ObjectIndex table_reference, final long aSource) {
super(table_test, table_reference, aSource);
}
public void run() {
final STEntry entry = new STEntry(this.getSource());
try {
Row.Entry entryBytes = getTable_test().get(entry.getKey());
if (entryBytes != null) {
final STEntry dbEntry = new STEntry(entryBytes.getColBytes(0), entryBytes.getColBytes(1));
if (!dbEntry.isValid()) {
System.out.println("INVALID table_test: " + dbEntry);
} /* else {
System.out.println("_VALID_ table_test: " + dbEntry);
getTable().remove(entry.getKey(), true);
} */
}
if (getTable_reference() != null) {
entryBytes = getTable_reference().get(entry.getKey());
if (entryBytes != null) {
final STEntry dbEntry = new STEntry(entryBytes.getColBytes(0), entryBytes.getColBytes(1));
if (!dbEntry.isValid()) {
System.out.println("INVALID table_reference: " + dbEntry);
} /* else {
System.out.println("_VALID_ table_reference: " + dbEntry);
getTable().remove(entry.getKey(), true);
} */
}
}
} catch (final IOException e) {
System.err.println(e);
e.printStackTrace();
System.exit(0);
}
}
}
public static ObjectIndex selectTableType(final String dbe, final String tablename, final Row testRow) throws Exception {
if (dbe.equals("kelondroRowSet")) {
return new RowSet(testRow, 0);
}
if (dbe.equals("kelondroSplitTable")) {
final File tablepath = new File(tablename).getParentFile();
return new SplitTable(tablepath, new File(tablename).getName(), testRow, true);
}
if (dbe.equals("kelondroEcoTable")) {
return new EcoTable(new File(tablename), testRow, EcoTable.tailCacheForceUsage, 1000, 0);
}
if (dbe.equals("mysql")) {
return new SQLTable("mysql", testRow);
}
if (dbe.equals("pgsql")) {
return new SQLTable("pgsql", testRow);
}
return null;
}
public static boolean checkEquivalence(final ObjectIndex test, final ObjectIndex reference) throws IOException {
if (reference == null) return true;
if (test.size() == reference.size()) {
System.out.println("* Testing equivalence of test table to reference table, " + test.size() + " entries");
} else {
System.out.println("* Testing equivalence of test table to reference table: FAILED! the tables have different sizes: test.size() = " + test.size() + ", reference.size() = " + reference.size());
return false;
}
boolean eq = true;
Row.Entry test_entry, reference_entry;
Iterator<Row.Entry> i = test.rows();
System.out.println("* Testing now by enumeration over test table");
final long ts = System.currentTimeMillis();
while (i.hasNext()) {
test_entry = i.next();
reference_entry = reference.get(test_entry.getPrimaryKeyBytes());
if (!test_entry.equals(reference_entry)) {
System.out.println("* FAILED: test entry with key '" + new String(test_entry.getPrimaryKeyBytes()) + "' has no equivalent entry in reference");
eq = false;
}
}
i = reference.rows();
System.out.println("* Testing now by enumeration over reference table");
final long rs = System.currentTimeMillis();
while (i.hasNext()) {
reference_entry = i.next();
test_entry = test.get(reference_entry.getPrimaryKeyBytes());
if (!test_entry.equals(reference_entry)) {
System.out.println("* FAILED: reference entry with key '" + new String(test_entry.getPrimaryKeyBytes()) + "' has no equivalent entry in test");
eq = false;
}
}
System.out.println("* finished checkEquivalence; test-enumeration = " + ((rs - ts) / 1000) + " seconds, reference-enumeration = "+ ((System.currentTimeMillis() - rs) / 1000) + " seconds");
if (eq) System.out.println("* SUCCESS! the tables are equal."); else System.out.println("* FAILED! the tables are not equal.");
return eq;
}
public static void main(final String[] args) {
System.setProperty("java.awt.headless", "true");
System.out.println("*** YaCy Database Test");
// print out command line
boolean assertionenabled = false;
assert assertionenabled = true;
if (assertionenabled) System.out.println("*** Asserts are enabled"); else System.out.println("*** HINT: YOU SHOULD ENABLE ASSERTS! (include -ea in start arguments");
final long mb = MemoryControl.available() / 1024 / 1024;
System.out.println("*** RAM = " + mb + " MB");
System.out.print(">java " +
((assertionenabled) ? "-ea " : "") +
((mb > 100) ? ("-Xmx" + (mb + 11) + "m ") : " ") +
"-classpath classes dbtest ");
for (int i = 0; i < args.length; i++) System.out.print(args[i] + " ");
System.out.println();
// read command options
final String command = args[0]; // test command
final String dbe_test = args[1]; // the database engine that shall be tested
String dbe_reference = args[2]; // the reference database engine (may be null)
final String tablename_test = args[3]; // name of test-table
if (dbe_reference.toLowerCase().equals("null")) dbe_reference = null;
final String tablename_reference = (dbe_reference == null) ? null : (tablename_test + ".reference");
// start test
final long startup = System.currentTimeMillis();
try {
// create a memory profiler
final memprofiler profiler = new memprofiler(1024, 320, 120, new File(tablename_test + ".profile.png"));
profiler.start();
// create the database access
final Row testRow = new Row("byte[] key-" + keylength + ", byte[] dummy-" + keylength + ", value-" + valuelength, Base64Order.enhancedCoder, 0);
final ObjectIndex table_test = selectTableType(dbe_test, tablename_test, testRow);
final ObjectIndex table_reference = (dbe_reference == null) ? null : selectTableType(dbe_reference, tablename_reference, testRow);
final long afterinit = System.currentTimeMillis();
System.out.println("Test for db-engine " + dbe_test + " started to create file " + tablename_test + " with test " + command);
// execute command
if (command.equals("create")) {
// do nothing, since opening of the database access must handle this
System.out.println("Database created");
}
if (command.equals("fill")) {
// fill database with random entries;
// args: <number-of-entries> <random-startpoint>
// example: java -ea -Xmx200m fill kelondroEcoTable kelondroFlexTable filldbtest 50000 0
final long count = Long.parseLong(args[4]);
final long randomstart = Long.parseLong(args[5]);
final Random random = new Random(randomstart);
byte[] key;
for (int i = 0; i < count; i++) {
key = randomHash(random);
table_test.put(testRow.newEntry(new byte[][]{key, key, dummyvalue2}));
if (table_reference != null) table_reference.put(testRow.newEntry(new byte[][]{key, key, dummyvalue2}));
if (i % 1000 == 0) {
System.out.println(i + " entries. ");
}
}
}
if (command.equals("read")) {
// read the database and compare with random entries;
// args: <number-of-entries> <random-startpoint>
final long count = Long.parseLong(args[4]);
final long randomstart = Long.parseLong(args[5]);
final Random random = new Random(randomstart);
Row.Entry entry;
byte[] key;
for (int i = 0; i < count; i++) {
key = randomHash(random);
entry = table_test.get(key);
if (entry == null)
System.out.println("missing value for entry " + new String(key) + " in test table");
else
if (!(new String(entry.getColBytes(1)).equals(new String(key)))) System.out.println("wrong value for entry " + new String(key) + ": " + new String(entry.getColBytes(1)) + " in test table");
if (table_reference != null) {
entry = table_reference.get(key);
if (entry == null)
System.out.println("missing value for entry " + new String(key) + " in reference table");
else
if (!(new String(entry.getColBytes(1)).equals(new String(key)))) System.out.println("wrong value for entry " + new String(key) + ": " + new String(entry.getColBytes(1)) + " in reference table");
}
if (i % 1000 == 0) {
System.out.println(i + " entries processed so far.");
}
}
}
if (command.equals("ramtest")) {
// fill database with random entries and delete them again;
// this is repeated without termination; after each loop
// the current ram is printed out
// args: <number-of-entries> <random-startpoint>
final long count = Long.parseLong(args[4]);
final long randomstart = Long.parseLong(args[5]);
byte[] key;
Random random;
long start, write, remove;
int loop = 0;
while (true) {
// write
random = new Random(randomstart);
start = System.currentTimeMillis();
for (int i = 0; i < count; i++) {
key = randomHash(random);
table_test.put(table_test.row().newEntry(new byte[][]{key, key, dummyvalue2}));
}
write = System.currentTimeMillis() - start;
// delete
random = new Random(randomstart);
start = System.currentTimeMillis();
for (int i = 0; i < count; i++) {
key = randomHash(random);
table_test.remove(key);
}
remove = System.currentTimeMillis() - start;
System.out.println("Loop " + loop + ": Write = " + write + ", Remove = " + remove);
System.out.println(" bevore GC: " +
"free = " + MemoryControl.free() +
", max = " + MemoryControl.max() +
", total = " + MemoryControl.total());
System.gc();
System.out.println(" after GC: " +
"free = " + MemoryControl.free() +
", max = " + MemoryControl.max() +
", total = " + MemoryControl.total());
loop++;
}
}
if (command.equals("list")) {
CloneableIterator<Row.Entry> i = null;
if (table_test instanceof Tree) i = ((Tree) table_test).rows();
if (table_test instanceof SQLTable) i = ((SQLTable) table_test).rows();
if(i != null) {
Row.Entry row;
while (i.hasNext()) {
row = i.next();
for (int j = 0; j < row.columns(); j++) System.out.print(row.getColString(j, null) + ",");
System.out.println();
}
}
}
if (command.equals("stressThreaded")) {
//
// args: <number-of-writes> <number-of-reads-per-write> <random-startpoint>
// example: java -ea -classpath classes dbtest stressThreaded kelondroEcoTable kelondroFlexTable stressthreadedtest 500 50 0
/* result with svn 4346
kelondroFlex:
removed: 70, size of jcontrol set: 354, size of kcontrol set: 354
Database size = 354 unique entries.
Execution time: open=1329, command=36234, close=17, total=37580
kelondroEco:
removed: 70, size of jcontrol set: 354, size of kcontrol set: 354
Database size = 354 unique entries.
Execution time: open=1324, command=34032, close=1, total=35357
*/
final long writeCount = Long.parseLong(args[4]);
final long readCount = Long.parseLong(args[5]);
final long randomstart = Long.parseLong(args[6]);
final Random random = new Random(randomstart);
long r;
Long R;
int p, rc=0;
final ArrayList<Long> ra = new ArrayList<Long>();
final HashSet<Long> jcontrol = new HashSet<Long>();
final IntBytesMap kcontrol = new IntBytesMap(1, 0);
for (int i = 0; i < writeCount; i++) {
r = Math.abs(random.nextLong() % 1000);
jcontrol.add(Long.valueOf(r));
kcontrol.putb((int) r, "x".getBytes());
serverInstantBusyThread.oneTimeJob(new WriteJob(table_test, table_reference, r), 0, 50);
if (random.nextLong() % 5 == 0) ra.add(Long.valueOf(r));
for (int j = 0; j < readCount; j++) {
serverInstantBusyThread.oneTimeJob(new ReadJob(table_test, table_reference, random.nextLong() % writeCount), random.nextLong() % 1000, 20);
}
if ((ra.size() > 0) && (random.nextLong() % 7 == 0)) {
rc++;
p = Math.abs(random.nextInt() % ra.size());
R = ra.get(p);
jcontrol.remove(R);
kcontrol.removeb((int) R.longValue());
System.out.println("remove: " + R.longValue());
serverInstantBusyThread.oneTimeJob(new RemoveJob(table_test, table_reference, (ra.remove(p)).longValue()), 0, 50);
}
}
System.out.println("removed: " + rc + ", size of jcontrol set: " + jcontrol.size() + ", size of kcontrol set: " + kcontrol.size());
while (serverInstantBusyThread.instantThreadCounter > 0) {
try {Thread.sleep(1000);} catch (final InterruptedException e) {} // wait for all tasks to finish
System.out.println("count: " + serverInstantBusyThread.instantThreadCounter + ", jobs: " + serverInstantBusyThread.jobs.toString());
}
try {Thread.sleep(6000);} catch (final InterruptedException e) {}
}
if (command.equals("stressSequential")) {
//
// args: <number-of-writes> <number-of-reads> <random-startpoint>
final long writeCount = Long.parseLong(args[4]);
final long readCount = Long.parseLong(args[5]);
final long randomstart = Long.parseLong(args[6]);
final Random random = new Random(randomstart);
long r;
Long R;
int p, rc=0;
final ArrayList<Long> ra = new ArrayList<Long>();
final HashSet<Long> jcontrol = new HashSet<Long>();
final IntBytesMap kcontrol = new IntBytesMap(1, 0);
for (int i = 0; i < writeCount; i++) {
//if (i == 30) random = new Random(randomstart);
r = Math.abs(random.nextLong() % 1000);
jcontrol.add(Long.valueOf(r));
kcontrol.putb((int) r, "x".getBytes());
new WriteJob(table_test, table_reference, r).run();
if (random.nextLong() % 5 == 0) ra.add(Long.valueOf(r));
for (int j = 0; j < readCount; j++) {
new ReadJob(table_test, table_reference, random.nextLong() % writeCount).run();
}
if ((ra.size() > 0) && (random.nextLong() % 7 == 0)) {
rc++;
p = Math.abs(random.nextInt() % ra.size());
R = ra.get(p);
jcontrol.remove(R);
kcontrol.removeb((int) R.longValue());
new RemoveJob(table_test, table_reference, (ra.remove(p)).longValue()).run();
}
}
try {Thread.sleep(1000);} catch (final InterruptedException e) {}
System.out.println("removed: " + rc + ", size of jcontrol set: " + jcontrol.size() + ", size of kcontrol set: " + kcontrol.size());
}
final long aftercommand = System.currentTimeMillis();
checkEquivalence(table_test, table_reference);
final long afterequiv = System.currentTimeMillis();
// final report
System.out.println("Database size = " + table_test.size() + " unique entries.");
// finally close the database/table
table_test.close();
if (table_reference != null) table_reference.close();
final long afterclose = System.currentTimeMillis();
System.out.println("Execution time: open=" + (afterinit - startup) +
", command=" + (aftercommand - afterinit) +
", equiv=" + (afterequiv - aftercommand) +
", close=" + (afterclose - afterequiv) +
", total=" + (afterclose - startup));
profiler.terminate();
} catch (final Exception e) {
e.printStackTrace();
}
}
}
final class memprofiler extends Thread {
ymageChart memChart;
boolean run;
File outputFile;
long start;
public memprofiler(final int width, final int height, final int expectedTimeSeconds, final File outputFile) {
this.outputFile = outputFile;
final int expectedKilobytes = 20 * 1024;//(Runtime.getRuntime().totalMemory() / 1024);
memChart = new ymageChart(width, height, "FFFFFF", "000000", "000000", 50, 20, 20, 20, "MEMORY CHART FROM EXECUTION AT " + new Date(), null);
final int timescale = 10; // steps with each 10 seconds
final int memscale = 1024;
memChart.declareDimension(ymageChart.DIMENSION_BOTTOM, timescale, (width - 40) * timescale / expectedTimeSeconds, 0, "FFFFFF", "555555", "SECONDS");
memChart.declareDimension(ymageChart.DIMENSION_LEFT, memscale, (height - 40) * memscale / expectedKilobytes, 0, "FFFFFF", "555555", "KILOBYTES");
run = true;
start = System.currentTimeMillis();
}
public void run() {
int seconds0 = 0, kilobytes0 = 0;
int seconds1 = 0, kilobytes1 = 0;
while(run) {
memChart.setColor("FF0000");
seconds1 = (int) ((System.currentTimeMillis() - start) / 1000);
kilobytes1 = (int) (MemoryControl.used() / 1024);
memChart.chartLine(ymageChart.DIMENSION_BOTTOM, ymageChart.DIMENSION_LEFT, seconds0, kilobytes0, seconds1, kilobytes1);
seconds0 = seconds1;
kilobytes0 = kilobytes1;
try {Thread.sleep(100);} catch (final InterruptedException e) {}
}
try {
ImageIO.write(memChart.getImage(), "png", outputFile);
} catch (final IOException e) {}
}
public void terminate() {
run = false;
while (this.isAlive()) {
try {Thread.sleep(1000);} catch (final InterruptedException e) {}
}
}
}