package net.yacy; // test application // shall be used to compare the kelondroDB with other databases // with relevance to yacy-specific use cases import java.io.File; import java.io.IOException; import java.util.Date; import java.util.Iterator; import java.util.Random; import javax.imageio.ImageIO; import net.yacy.kelondro.index.ObjectIndex; import net.yacy.kelondro.index.Row; import net.yacy.kelondro.index.RowSet; import net.yacy.kelondro.index.RowSpaceExceededException; import net.yacy.kelondro.logging.Log; import net.yacy.kelondro.order.Base64Order; import net.yacy.kelondro.order.CloneableIterator; import net.yacy.kelondro.order.NaturalOrder; import net.yacy.kelondro.table.SQLTable; import net.yacy.kelondro.table.SplitTable; import net.yacy.kelondro.table.Table; import net.yacy.kelondro.util.MemoryControl; import net.yacy.visualization.ChartPlotter; public class dbtest { public final static int keylength = 12; public final static int valuelength = 223; // sum of all data length as defined in plasmaURL private static final byte[] dummyvalue2 = new byte[valuelength]; static { // fill the dummy value for (int i = 0; i < valuelength; i++) dummyvalue2[i] = '-'; dummyvalue2[0] = '{'; dummyvalue2[valuelength - 1] = '}'; } public static byte[] randomHash(final long r0, final long r1) { // a long can have 64 bit, but a 12-byte hash can have 6 * 12 = 72 bits // so we construct a generic Hash using two long values final String s = (Base64Order.enhancedCoder.encodeLong(Math.abs(r0), 6) + Base64Order.enhancedCoder.encodeLong(Math.abs(r1), 6)); return s.getBytes(); } public static byte[] randomHash(final Random r) { return randomHash(r.nextLong(), r.nextLong()); } public final static class STEntry { private final byte[] key; private final byte[] value; public STEntry(final long aSource) { this.key = randomHash(aSource, aSource); this.value = new byte[valuelength]; for (int i = 0; i < valuelength; i++) this.value[i] = 0; final byte[] tempKey = String.valueOf(aSource).getBytes(); System.arraycopy(tempKey, 0, this.value, 0, tempKey.length); } public STEntry(final byte[] aKey, final byte[] aValue) { this.key = aKey; this.value = aValue; } public boolean isValid() { final String s = new String(this.value).trim(); if (s.length() == 0) return false; final long source = Long.parseLong(s); return new String(this.key).equals(new String(randomHash(source, source))); } public byte[] getKey() { return this.key; } public byte[] getValue() { return this.value; } public String toString() { return new String(this.key) + "#" + new String(this.value); } } public static abstract class STJob implements Runnable { private final ObjectIndex table_test, table_reference; private final long source; public STJob(final ObjectIndex table_test, final ObjectIndex table_reference, final long aSource) { this.table_test = table_test; this.table_reference = table_reference; this.source = aSource; } public ObjectIndex getTable_test() { return this.table_test; } public ObjectIndex getTable_reference() { return this.table_reference; } public abstract void run(); public long getSource() { return this.source; } } public static final class WriteJob extends STJob { public WriteJob(final ObjectIndex table_test, final ObjectIndex table_reference, final long aSource) { super(table_test, table_reference, aSource); } public void run() { final STEntry entry = new STEntry(this.getSource()); System.out.println("write: " + NaturalOrder.arrayList(entry.getKey(), 0, entry.getKey().length)); try { getTable_test().put(getTable_test().row().newEntry(new byte[][] { entry.getKey(), entry.getValue() , entry.getValue() })); if (getTable_reference() != null) getTable_reference().put(getTable_test().row().newEntry(new byte[][] { entry.getKey(), entry.getValue() , entry.getValue() })); } catch (final IOException e) { System.err.println(e); Log.logException(e); System.exit(0); } catch (RowSpaceExceededException e) { System.err.println(e); Log.logException(e); System.exit(0); } } } public static final class RemoveJob extends STJob { public RemoveJob(final ObjectIndex table_test, final ObjectIndex table_reference, final long aSource) { super(table_test, table_reference, aSource); } public void run() { final STEntry entry = new STEntry(this.getSource()); System.out.println("remove: " + NaturalOrder.arrayList(entry.getKey(), 0, entry.getKey().length)); try { getTable_test().remove(entry.getKey()); if (getTable_reference() != null) getTable_reference().delete(entry.getKey()); } catch (final IOException e) { System.err.println(e); Log.logException(e); System.exit(0); } } } public static final class ReadJob extends STJob { public ReadJob(final ObjectIndex table_test, final ObjectIndex table_reference, final long aSource) { super(table_test, table_reference, aSource); } public void run() { final STEntry entry = new STEntry(this.getSource()); try { Row.Entry entryBytes = getTable_test().get(entry.getKey()); if (entryBytes != null) { final STEntry dbEntry = new STEntry(entryBytes.getColBytes(0, true), entryBytes.getColBytes(1, true)); if (!dbEntry.isValid()) { System.out.println("INVALID table_test: " + dbEntry); } /* else { System.out.println("_VALID_ table_test: " + dbEntry); getTable().remove(entry.getKey(), true); } */ } if (getTable_reference() != null) { entryBytes = getTable_reference().get(entry.getKey()); if (entryBytes != null) { final STEntry dbEntry = new STEntry(entryBytes.getColBytes(0, true), entryBytes.getColBytes(1, true)); if (!dbEntry.isValid()) { System.out.println("INVALID table_reference: " + dbEntry); } /* else { System.out.println("_VALID_ table_reference: " + dbEntry); getTable().remove(entry.getKey(), true); } */ } } } catch (final IOException e) { System.err.println(e); Log.logException(e); System.exit(0); } } } public static ObjectIndex selectTableType(final String dbe, final String tablename, final Row testRow) throws Exception { if (dbe.equals("kelondroRowSet")) { return new RowSet(testRow, 0); } if (dbe.equals("kelondroSplitTable")) { final File tablepath = new File(tablename).getParentFile(); return new SplitTable(tablepath, new File(tablename).getName(), testRow, true, true); } if (dbe.equals("kelondroEcoTable")) { return new Table(new File(tablename), testRow, 1000, 0, true, true); } if (dbe.equals("mysql")) { return new SQLTable("mysql", testRow); } if (dbe.equals("pgsql")) { return new SQLTable("pgsql", testRow); } return null; } public static boolean checkEquivalence(final ObjectIndex test, final ObjectIndex reference) throws IOException { if (reference == null) return true; if (test.size() == reference.size()) { System.out.println("* Testing equivalence of test table to reference table, " + test.size() + " entries"); } else { System.out.println("* Testing equivalence of test table to reference table: FAILED! the tables have different sizes: test.size() = " + test.size() + ", reference.size() = " + reference.size()); return false; } boolean eq = true; Row.Entry test_entry, reference_entry; Iterator i = test.rows(); System.out.println("* Testing now by enumeration over test table"); final long ts = System.currentTimeMillis(); while (i.hasNext()) { test_entry = i.next(); reference_entry = reference.get(test_entry.getPrimaryKeyBytes()); if (!test_entry.equals(reference_entry)) { System.out.println("* FAILED: test entry with key '" + new String(test_entry.getPrimaryKeyBytes()) + "' has no equivalent entry in reference"); eq = false; } } i = reference.rows(); System.out.println("* Testing now by enumeration over reference table"); final long rs = System.currentTimeMillis(); while (i.hasNext()) { reference_entry = i.next(); test_entry = test.get(reference_entry.getPrimaryKeyBytes()); if (!test_entry.equals(reference_entry)) { System.out.println("* FAILED: reference entry with key '" + new String(test_entry.getPrimaryKeyBytes()) + "' has no equivalent entry in test"); eq = false; } } System.out.println("* finished checkEquivalence; test-enumeration = " + ((rs - ts) / 1000) + " seconds, reference-enumeration = "+ ((System.currentTimeMillis() - rs) / 1000) + " seconds"); if (eq) System.out.println("* SUCCESS! the tables are equal."); else System.out.println("* FAILED! the tables are not equal."); return eq; } public static void main(final String[] args) { System.setProperty("java.awt.headless", "true"); System.out.println("*** YaCy Database Test"); // print out command line boolean assertionenabled = false; assert assertionenabled = true; if (assertionenabled) System.out.println("*** Asserts are enabled"); else System.out.println("*** HINT: YOU SHOULD ENABLE ASSERTS! (include -ea in start arguments"); final long mb = MemoryControl.available() / 1024 / 1024; System.out.println("*** RAM = " + mb + " MB"); System.out.print(">java " + ((assertionenabled) ? "-ea " : "") + ((mb > 100) ? ("-Xmx" + (mb + 11) + "m ") : " ") + "-classpath classes dbtest "); for (int i = 0; i < args.length; i++) System.out.print(args[i] + " "); System.out.println(); // read command options final String command = args[0]; // test command final String dbe_test = args[1]; // the database engine that shall be tested String dbe_reference = args[2]; // the reference database engine (may be null) final String tablename_test = args[3]; // name of test-table if (dbe_reference.toLowerCase().equals("null")) dbe_reference = null; final String tablename_reference = (dbe_reference == null) ? null : (tablename_test + ".reference"); // start test final long startup = System.currentTimeMillis(); try { // create a memory profiler final memprofiler profiler = new memprofiler(1024, 320, 120, new File(tablename_test + ".profile.png")); profiler.start(); // create the database access final Row testRow = new Row("byte[] key-" + keylength + ", byte[] dummy-" + keylength + ", value-" + valuelength, Base64Order.enhancedCoder); final ObjectIndex table_test = selectTableType(dbe_test, tablename_test, testRow); final ObjectIndex table_reference = (dbe_reference == null) ? null : selectTableType(dbe_reference, tablename_reference, testRow); final long afterinit = System.currentTimeMillis(); System.out.println("Test for db-engine " + dbe_test + " started to create file " + tablename_test + " with test " + command); // execute command if (command.equals("create")) { // do nothing, since opening of the database access must handle this System.out.println("Database created"); } if (command.equals("fill")) { // fill database with random entries; // args: // example: java -ea -Xmx200m fill kelondroEcoTable kelondroFlexTable filldbtest 50000 0 final long count = Long.parseLong(args[4]); final long randomstart = Long.parseLong(args[5]); final Random random = new Random(randomstart); byte[] key; for (int i = 0; i < count; i++) { key = randomHash(random); table_test.put(testRow.newEntry(new byte[][]{key, key, dummyvalue2})); if (table_reference != null) table_reference.put(testRow.newEntry(new byte[][]{key, key, dummyvalue2})); if (i % 1000 == 0) { System.out.println(i + " entries. "); } } } if (command.equals("read")) { // read the database and compare with random entries; // args: final long count = Long.parseLong(args[4]); final long randomstart = Long.parseLong(args[5]); final Random random = new Random(randomstart); Row.Entry entry; byte[] key; for (int i = 0; i < count; i++) { key = randomHash(random); entry = table_test.get(key); if (entry == null) System.out.println("missing value for entry " + new String(key) + " in test table"); else if (!(new String(entry.getColBytes(1, false)).equals(new String(key)))) System.out.println("wrong value for entry " + new String(key) + ": " + new String(entry.getColBytes(1, false)) + " in test table"); if (table_reference != null) { entry = table_reference.get(key); if (entry == null) System.out.println("missing value for entry " + new String(key) + " in reference table"); else if (!(new String(entry.getColBytes(1, false)).equals(new String(key)))) System.out.println("wrong value for entry " + new String(key) + ": " + new String(entry.getColBytes(1, false)) + " in reference table"); } if (i % 1000 == 0) { System.out.println(i + " entries processed so far."); } } } if (command.equals("ramtest")) { // fill database with random entries and delete them again; // this is repeated without termination; after each loop // the current ram is printed out // args: final long count = Long.parseLong(args[4]); final long randomstart = Long.parseLong(args[5]); byte[] key; Random random; long start, write, remove; int loop = 0; while (true) { // write random = new Random(randomstart); start = System.currentTimeMillis(); for (int i = 0; i < count; i++) { key = randomHash(random); table_test.put(table_test.row().newEntry(new byte[][]{key, key, dummyvalue2})); } write = System.currentTimeMillis() - start; // delete random = new Random(randomstart); start = System.currentTimeMillis(); for (int i = 0; i < count; i++) { key = randomHash(random); table_test.delete(key); } remove = System.currentTimeMillis() - start; System.out.println("Loop " + loop + ": Write = " + write + ", Remove = " + remove); System.out.println(" bevore GC: " + "free = " + MemoryControl.free() + ", max = " + MemoryControl.maxMemory + ", total = " + MemoryControl.total()); System.gc(); System.out.println(" after GC: " + "free = " + MemoryControl.free() + ", max = " + MemoryControl.maxMemory + ", total = " + MemoryControl.total()); loop++; } } if (command.equals("list")) { CloneableIterator i = null; if (table_test instanceof SQLTable) i = ((SQLTable) table_test).rows(); if(i != null) { Row.Entry row; while (i.hasNext()) { row = i.next(); for (int j = 0; j < row.columns(); j++) System.out.print(row.getColString(j, null) + ","); System.out.println(); } } } final long aftercommand = System.currentTimeMillis(); checkEquivalence(table_test, table_reference); final long afterequiv = System.currentTimeMillis(); // final report System.out.println("Database size = " + table_test.size() + " unique entries."); // finally close the database/table table_test.close(); if (table_reference != null) table_reference.close(); final long afterclose = System.currentTimeMillis(); System.out.println("Execution time: open=" + (afterinit - startup) + ", command=" + (aftercommand - afterinit) + ", equiv=" + (afterequiv - aftercommand) + ", close=" + (afterclose - afterequiv) + ", total=" + (afterclose - startup)); profiler.terminate(); } catch (final Exception e) { Log.logException(e); } } } final class memprofiler extends Thread { ChartPlotter memChart; boolean run; File outputFile; long start; public memprofiler(final int width, final int height, final int expectedTimeSeconds, final File outputFile) { this.outputFile = outputFile; final int expectedKilobytes = 20 * 1024;//(Runtime.getRuntime().totalMemory() / 1024); memChart = new ChartPlotter(width, height, "FFFFFF", "000000", "000000", 50, 20, 20, 20, "MEMORY CHART FROM EXECUTION AT " + new Date(), null); final int timescale = 10; // steps with each 10 seconds final int memscale = 1024; memChart.declareDimension(ChartPlotter.DIMENSION_BOTTOM, timescale, (width - 40) * timescale / expectedTimeSeconds, 0, "FFFFFF", "555555", "SECONDS"); memChart.declareDimension(ChartPlotter.DIMENSION_LEFT, memscale, (height - 40) * memscale / expectedKilobytes, 0, "FFFFFF", "555555", "KILOBYTES"); run = true; start = System.currentTimeMillis(); } public void run() { try { int seconds0 = 0, kilobytes0 = 0; int seconds1 = 0, kilobytes1 = 0; while (run) { memChart.setColor("FF0000"); seconds1 = (int) ((System.currentTimeMillis() - start) / 1000); kilobytes1 = (int) (MemoryControl.used() / 1024); memChart.chartLine(ChartPlotter.DIMENSION_BOTTOM, ChartPlotter.DIMENSION_LEFT, seconds0, kilobytes0, seconds1, kilobytes1); seconds0 = seconds1; kilobytes0 = kilobytes1; try {Thread.sleep(100);} catch (final InterruptedException e) {} } } catch (final Exception e) { Log.logException(e); } try { ImageIO.write(memChart.getImage(), "png", outputFile); } catch (final IOException e) { // do noting } catch (final Exception e) { Log.logException(e); } } public void terminate() { run = false; while (this.isAlive()) { try {Thread.sleep(1000);} catch (final InterruptedException e) {} } } }