diff --git a/build.properties b/build.properties index db29be5d3..7a49e807a 100644 --- a/build.properties +++ b/build.properties @@ -3,7 +3,7 @@ javacSource=1.4 javacTarget=1.4 # Release Configuration -releaseVersion=0.402 +releaseVersion=0.403 releaseFile=yacy_dev_v${releaseVersion}_${DSTAMP}_${releaseNr}.tar.gz #releaseFile=yacy_v${releaseVersion}_${DSTAMP}_${releaseNr}.tar.gz releaseDir=yacy_dev_v${releaseVersion}_${DSTAMP}_${releaseNr} diff --git a/source/de/anomic/kelondro/kelondroAbstractRA.java b/source/de/anomic/kelondro/kelondroAbstractRA.java index f71f09ad4..5c8a8bb9b 100644 --- a/source/de/anomic/kelondro/kelondroAbstractRA.java +++ b/source/de/anomic/kelondro/kelondroAbstractRA.java @@ -66,7 +66,12 @@ abstract class kelondroAbstractRA implements kelondroRA { abstract public void seek(long pos) throws IOException; abstract public void close() throws IOException; - // derivated methods: + // derived methods: + public void readFully(byte[] b, int off, int len) throws IOException { + int r = read(b, off, len); + if (r < len) readFully(b, off + r, len - r); + } + public byte readByte() throws IOException { int ch = this.read(); if (ch < 0) throw new IOException(); @@ -112,7 +117,7 @@ abstract class kelondroAbstractRA implements kelondroRA { this.write((int) (v >>> 24) & 0xFF); this.write((int) (v >>> 16) & 0xFF); this.write((int) (v >>> 8) & 0xFF); this.write((int) (v >>> 0) & 0xFF); } - + public void write(byte[] b) throws IOException { this.write(b, 0, b.length); } diff --git a/source/de/anomic/kelondro/kelondroArray.java b/source/de/anomic/kelondro/kelondroArray.java index 8aad33c92..7a597ff59 100644 --- a/source/de/anomic/kelondro/kelondroArray.java +++ b/source/de/anomic/kelondro/kelondroArray.java @@ -67,16 +67,22 @@ public class kelondroArray extends kelondroRecords { } public synchronized byte[][] set(int index, byte[][] row) throws IOException { - if (row.length != columns()) throw new IllegalArgumentException("set: wrong row length " + row.length + "; must be " + columns()); + if (row.length != columns()) + throw new IllegalArgumentException("set: wrong row length " + row.length + "; must be " + columns()); // make room for element - if (size() <= index) while (newNode() <= index) {} + Node n; + while (size() <= index) { + n = newNode(); + n.commit(CP_NONE); + } // get the node at position index - Node n = getNode(new Handle(index)); + n = getNode(new Handle(index)); // write the row byte[][] before = n.setValues(row); + n.commit(CP_NONE); return before; } @@ -103,7 +109,7 @@ public class kelondroArray extends kelondroRecords { for (int i = 0; i < size(); i++) { System.out.print("row " + i + ": "); row = get(i); - for (int j = 0; j < columns(); j++) System.out.print(new String(row[j]) + ", "); + for (int j = 0; j < columns(); j++) System.out.print(((row[j] == null) ? "NULL" : new String(row[j])) + ", "); System.out.println(); } System.out.println("EndOfTable"); @@ -123,9 +129,7 @@ public class kelondroArray extends kelondroRecords { // create File f = new File(args[1]); if (f.exists()) f.delete(); - int[] lens = new int[1]; - lens[0] = Integer.parseInt(args[2]); - kelondroArray fm = new kelondroArray(f, lens, 2); + kelondroArray fm = new kelondroArray(f, new int[]{Integer.parseInt(args[2])}, 2); fm.close(); } else if ((args.length == 2) && (args[0].equals("-v"))) { @@ -149,7 +153,17 @@ public class kelondroArray extends kelondroRecords { byte[][] row = new byte[][]{args[3].getBytes()}; fm.set(Integer.parseInt(args[2]), row); fm.close(); - } else { + } else + if ((args.length == 1) && (args[0].equals("-test"))) { + File testfile = new File("test.array"); + if (testfile.exists()) testfile.delete(); + kelondroArray fm = new kelondroArray(testfile, new int[]{30, 50}, 9); + for (int i = 0; i < 100; i++) { + fm.set(i, new byte[][]{("name" + i).getBytes(), ("value" + i).getBytes()}); + } + fm.close(); + } else + { System.err.println("usage: kelondroArray -c|-v|-s|-g [file]|[index [value]] "); System.err.println("( create, view, set, get)"); System.exit(0); diff --git a/source/de/anomic/kelondro/kelondroBufferedRA.java b/source/de/anomic/kelondro/kelondroBufferedRA.java index 2d849f8a7..1a5861837 100644 --- a/source/de/anomic/kelondro/kelondroBufferedRA.java +++ b/source/de/anomic/kelondro/kelondroBufferedRA.java @@ -3,8 +3,8 @@ // part of The Kelondro Database // (C) by Michael Peter Christen; mc@anomic.de // first published on http://www.anomic.de -// Frankfurt, Germany, 2004 -// last major change: 06.10.2004 +// Frankfurt, Germany, 2005 +// last major change: 13.09.2005 // // This program is free software; you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by @@ -43,124 +43,105 @@ package de.anomic.kelondro; import java.io.FileNotFoundException; import java.io.IOException; -import java.util.HashMap; -import java.util.Iterator; public class kelondroBufferedRA extends kelondroAbstractRA implements kelondroRA { + private static final int bufferSizeExp = 10; + private static final int bufferSize = 1 << bufferSizeExp; + private static final int bufferOffsetFilter = bufferSize - 1; + protected kelondroRA ra; - protected kelondroMScoreCluster bufferScore; - protected HashMap bufferMemory; - private int bufferMaxElements; - private int bufferElementSize; + protected byte[] buffer; + protected int bufferPage; + protected boolean bufferWritten; private long seekpos; - public kelondroBufferedRA(kelondroRA ra, int buffersize, int elementsize) throws FileNotFoundException { + public kelondroBufferedRA(kelondroRA ra) throws FileNotFoundException { this.ra = ra; this.name = ra.name(); - this.bufferMemory = new HashMap(); - this.bufferScore = new kelondroMScoreCluster(); - this.bufferElementSize = elementsize; - this.bufferMaxElements = (int) (buffersize / bufferElementSize); + this.buffer = new byte[bufferSize]; this.seekpos = 0; + this.bufferPage = -1; + this.bufferWritten = true; } - private int bufferElementNumber(long address) { - return (int) address / bufferElementSize; + private void readBuffer(int newPageNr) throws IOException { + if (newPageNr == bufferPage) return; + bufferPage = newPageNr; + ra.seek(bufferPage << bufferSizeExp); + ra.readFully(buffer, 0, bufferSize); + bufferWritten = true; + } + + private void writeBuffer() throws IOException { + if ((bufferWritten) || (bufferPage < 0)) return; + ra.seek(bufferPage << bufferSizeExp); + ra.write(buffer, 0, bufferSize); + bufferWritten = true; } - private int bufferElementOffset(long address) { - return (int) address % bufferElementSize; - } - - private byte[] readBuffer(int bufferNr) throws IOException { - Integer bufferNrI = new Integer(bufferNr); - byte[] buffer = (byte[]) bufferMemory.get(bufferNrI); - if (buffer == null) { - if (bufferMemory.size() >= bufferMaxElements) { - // delete elements in buffer if buffer too big - Iterator it = bufferScore.scores(true); - Integer element = (Integer) it.next(); - writeBuffer((byte[]) bufferMemory.get(element), element.intValue()); - bufferMemory.remove(element); - int age = bufferScore.deleteScore(element); - de.anomic.server.logging.serverLog.logFine("CACHE: " + name, "GC; age=" + ((((int) (0xFFFFFFFFL & System.currentTimeMillis())) - age) / 1000)); - } - // add new element - buffer = new byte[bufferElementSize]; - //System.out.println("buffernr=" + bufferNr + ", elSize=" + bufferElementSize); - ra.seek(bufferNr * bufferElementSize); - ra.read(buffer, 0, bufferElementSize); - bufferMemory.put(bufferNrI, buffer); + private void updateToBuffer(int newPageNr) throws IOException { + if (newPageNr != bufferPage) { + writeBuffer(); + readBuffer(newPageNr); } - bufferScore.setScore(bufferNrI, (int) (0xFFFFFFFFL & System.currentTimeMillis())); - return buffer; - } - - private void writeBuffer(byte[] buffer, int bufferNr) throws IOException { - if (buffer == null) return; - Integer bufferNrI = new Integer(bufferNr); - ra.seek(bufferNr * bufferElementSize); - ra.write(buffer, 0, bufferElementSize); - bufferScore.setScore(bufferNrI, (int) (0xFFFFFFFFL & System.currentTimeMillis())); } // pseudo-native method read public int read() throws IOException { - int bn = bufferElementNumber(seekpos); - int offset = bufferElementOffset(seekpos); + int bn = (int) seekpos >> bufferSizeExp; // buffer page number + int offset = (int) seekpos & bufferOffsetFilter; // buffer page offset seekpos++; - return 0xFF & readBuffer(bn)[offset]; + updateToBuffer(bn); + return 0xFF & buffer[offset]; } // pseudo-native method write public void write(int b) throws IOException { - int bn = bufferElementNumber(seekpos); - int offset = bufferElementOffset(seekpos); - byte[] buffer = readBuffer(bn); + int bn = (int) seekpos >> bufferSizeExp; // buffer page number + int offset = (int) seekpos & bufferOffsetFilter; // buffer page offset seekpos++; + updateToBuffer(bn); buffer[offset] = (byte) b; - //writeBuffer(buffer, bn); + bufferWritten = false; } public int read(byte[] b, int off, int len) throws IOException { - int bn1 = bufferElementNumber(seekpos); - int bn2 = bufferElementNumber(seekpos + len - 1); - int offset = bufferElementOffset(seekpos); - byte[] buffer = readBuffer(bn1); + int bn1 = (int) seekpos >> bufferSizeExp; // buffer page number, first position + int bn2 = (int) (seekpos + len - 1) >> bufferSizeExp; // buffer page number, last position + int offset = (int) seekpos & bufferOffsetFilter; // buffer page offset + updateToBuffer(bn1); if (bn1 == bn2) { // simple case - //System.out.println("C1: bn1=" + bn1 + ", offset=" + offset + ", off=" + off + ", len=" + len); System.arraycopy(buffer, offset, b, off, len); seekpos += len; return len; } else { // do recursively - int thislen = bufferElementSize - offset; - //System.out.println("C2: bn1=" + bn1 + ", bn2=" + bn2 +", offset=" + offset + ", off=" + off + ", len=" + len + ", thislen=" + thislen); + int thislen = bufferSize - offset; System.arraycopy(buffer, offset, b, off, thislen); seekpos += thislen; - return thislen + read(b, thislen, len - thislen); + return thislen + read(b, off + thislen, len - thislen); } } public void write(byte[] b, int off, int len) throws IOException { - int bn1 = bufferElementNumber(seekpos); - int bn2 = bufferElementNumber(seekpos + len - 1); - int offset = bufferElementOffset(seekpos); - byte[] buffer = readBuffer(bn1); + int bn1 = (int) seekpos >> bufferSizeExp; // buffer page number, first position + int bn2 = (int) (seekpos + len - 1) >> bufferSizeExp; // buffer page number, last position + int offset = (int) seekpos & bufferOffsetFilter; // buffer page offset + updateToBuffer(bn1); if (bn1 == bn2) { // simple case System.arraycopy(b, off, buffer, offset, len); + bufferWritten = false; seekpos += len; - //writeBuffer(buffer, bn1); } else { // do recursively - int thislen = bufferElementSize - offset; + int thislen = bufferSize - offset; System.arraycopy(b, off, buffer, offset, thislen); + bufferWritten = false; seekpos += thislen; - //writeBuffer(buffer, bn1); - write(b, thislen, len - thislen); + write(b, off + thislen, len - thislen); } } @@ -169,16 +150,30 @@ public class kelondroBufferedRA extends kelondroAbstractRA implements kelondroRA } public void close() throws IOException { - // write all unwritten buffers - Iterator it = bufferScore.scores(true); - while (it.hasNext()) { - Integer element = (Integer) it.next(); - writeBuffer((byte[]) bufferMemory.get(element), element.intValue()); - bufferMemory.remove(element); - } + // write unwritten buffer + if (buffer == null) return; + writeBuffer(); ra.close(); - bufferScore = null; - bufferMemory = null; + buffer = null; + } + + public void finalize() { + try { + close(); + } catch (IOException e) {} } + public static void main(String[] args) { + try { + kelondroRA file = new kelondroBufferedRA(new kelondroFileRA("testx")); + file.seek(bufferSize - 2); + byte[] b = new byte[]{65, 66, 77, 88}; + file.write(b); + file.seek(bufferSize * 2 - 30); + for (int i = 65; i < 150; i++) file.write(i); + file.close(); + } catch (IOException e) { + e.printStackTrace(); + } + } } diff --git a/source/de/anomic/kelondro/kelondroCachedRA.java b/source/de/anomic/kelondro/kelondroCachedRA.java new file mode 100644 index 000000000..a94601496 --- /dev/null +++ b/source/de/anomic/kelondro/kelondroCachedRA.java @@ -0,0 +1,191 @@ +// kelondroCachedRA.java +// ----------------------- +// part of The Kelondro Database +// (C) by Michael Peter Christen; mc@anomic.de +// first published on http://www.anomic.de +// Frankfurt, Germany, 2004, 2005 +// last major change: 06.10.2004 +// this file was previously named kelondroBufferedRA +// +// This program is free software; you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation; either version 2 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +// +// Using this software in any meaning (reading, learning, copying, compiling, +// running) means that you agree that the Author(s) is (are) not responsible +// for cost, loss of data or any harm that may be caused directly or indirectly +// by usage of this softare or this documentation. The usage of this software +// is on your own risk. The installation and usage (starting/running) of this +// software may allow other people or application to access your computer and +// any attached devices and is highly dependent on the configuration of the +// software which must be done by the user of the software; the author(s) is +// (are) also not responsible for proper configuration and usage of the +// software, even if provoked by documentation provided together with +// the software. +// +// Any changes to this file according to the GPL as documented in the file +// gpl.txt aside this file in the shipment you received can be done to the +// lines that follows this copyright notice here, but changes must not be +// done inside the copyright notive above. A re-distribution must contain +// the intact and unchanged copyright notice. +// Contributions and changes to the program code must be marked as such. + +package de.anomic.kelondro; + +import java.io.FileNotFoundException; +import java.io.IOException; +import java.util.HashMap; +import java.util.Iterator; + +public class kelondroCachedRA extends kelondroAbstractRA implements kelondroRA { + + protected kelondroRA ra; + protected kelondroMScoreCluster cacheScore; + protected HashMap cacheMemory; + private int cacheMaxElements; + private int cacheElementSize; + private long seekpos; + + public kelondroCachedRA(kelondroRA ra, int cachesize, int elementsize) throws FileNotFoundException { + this.ra = ra; + this.name = ra.name(); + this.cacheMemory = new HashMap(); + this.cacheScore = new kelondroMScoreCluster(); + this.cacheElementSize = elementsize; + this.cacheMaxElements = (int) (cachesize / cacheElementSize); + this.seekpos = 0; + } + + private int cacheElementNumber(long address) { + return (int) address / cacheElementSize; + } + + private int cacheElementOffset(long address) { + return (int) address % cacheElementSize; + } + + private byte[] readCache(int cacheNr) throws IOException { + Integer cacheNrI = new Integer(cacheNr); + byte[] cache = (byte[]) cacheMemory.get(cacheNrI); + if (cache == null) { + if (cacheMemory.size() >= cacheMaxElements) { + // delete elements in buffer if buffer too big + Iterator it = cacheScore.scores(true); + Integer element = (Integer) it.next(); + writeCache((byte[]) cacheMemory.get(element), element.intValue()); + cacheMemory.remove(element); + int age = cacheScore.deleteScore(element); + de.anomic.server.logging.serverLog.logFine("CACHE: " + name, "GC; age=" + ((((int) (0xFFFFFFFFL & System.currentTimeMillis())) - age) / 1000)); + } + // add new element + cache = new byte[cacheElementSize]; + //System.out.println("buffernr=" + bufferNr + ", elSize=" + bufferElementSize); + ra.seek(cacheNr * cacheElementSize); + ra.read(cache, 0, cacheElementSize); + cacheMemory.put(cacheNrI, cache); + } + cacheScore.setScore(cacheNrI, (int) (0xFFFFFFFFL & System.currentTimeMillis())); + return cache; + } + + private void writeCache(byte[] cache, int cacheNr) throws IOException { + if (cache == null) return; + Integer cacheNrI = new Integer(cacheNr); + ra.seek(cacheNr * cacheElementSize); + ra.write(cache, 0, cacheElementSize); + cacheScore.setScore(cacheNrI, (int) (0xFFFFFFFFL & System.currentTimeMillis())); + } + + // pseudo-native method read + public int read() throws IOException { + int bn = cacheElementNumber(seekpos); + int offset = cacheElementOffset(seekpos); + seekpos++; + return 0xFF & readCache(bn)[offset]; + } + + // pseudo-native method write + public void write(int b) throws IOException { + int bn = cacheElementNumber(seekpos); + int offset = cacheElementOffset(seekpos); + byte[] cache = readCache(bn); + seekpos++; + cache[offset] = (byte) b; + //writeBuffer(buffer, bn); + } + + public int read(byte[] b, int off, int len) throws IOException { + int bn1 = cacheElementNumber(seekpos); + int bn2 = cacheElementNumber(seekpos + len - 1); + int offset = cacheElementOffset(seekpos); + byte[] buffer = readCache(bn1); + if (bn1 == bn2) { + // simple case + //System.out.println("C1: bn1=" + bn1 + ", offset=" + offset + ", off=" + off + ", len=" + len); + System.arraycopy(buffer, offset, b, off, len); + seekpos += len; + return len; + } else { + // do recursively + int thislen = cacheElementSize - offset; + //System.out.println("C2: bn1=" + bn1 + ", bn2=" + bn2 +", offset=" + offset + ", off=" + off + ", len=" + len + ", thislen=" + thislen); + System.arraycopy(buffer, offset, b, off, thislen); + seekpos += thislen; + return thislen + read(b, off + thislen, len - thislen); + } + } + + public void write(byte[] b, int off, int len) throws IOException { + int bn1 = cacheElementNumber(seekpos); + int bn2 = cacheElementNumber(seekpos + len - 1); + int offset = cacheElementOffset(seekpos); + byte[] cache = readCache(bn1); + if (bn1 == bn2) { + // simple case + System.arraycopy(b, off, cache, offset, len); + seekpos += len; + //writeBuffer(buffer, bn1); + } else { + // do recursively + int thislen = cacheElementSize - offset; + System.arraycopy(b, off, cache, offset, thislen); + seekpos += thislen; + //writeBuffer(buffer, bn1); + write(b, off + thislen, len - thislen); + } + } + + public void seek(long pos) throws IOException { + seekpos = pos; + } + + public void close() throws IOException { + // write all unwritten buffers + if (cacheMemory == null) return; + Iterator it = cacheScore.scores(true); + while (it.hasNext()) { + Integer element = (Integer) it.next(); + writeCache((byte[]) cacheMemory.get(element), element.intValue()); + cacheMemory.remove(element); + } + ra.close(); + cacheScore = null; + cacheMemory = null; + } + + public void finalize() { + try { + close(); + } catch (IOException e) {} + } +} diff --git a/source/de/anomic/kelondro/kelondroDyn.java b/source/de/anomic/kelondro/kelondroDyn.java index 74a1e2ecc..682483a4f 100644 --- a/source/de/anomic/kelondro/kelondroDyn.java +++ b/source/de/anomic/kelondro/kelondroDyn.java @@ -228,7 +228,13 @@ public class kelondroDyn extends kelondroTree { } else { byte[] buf = getValueCached(dynKey(key, reccnt)); if (buf == null) return null; - //System.out.println("read: buf.length="+buf.length+",recpos="+recpos+",len="+len); + if (buf.length < reclen) { + byte[] buff = new byte[reclen]; + System.arraycopy(buf, 0, buff, 0, buf.length); + buf = buff; + buff = null; + } + //System.out.println("read: buf.length="+buf.length+",recpos="+recpos+",len="+len); if (len < (reclen - recpos)) { segment1 = new byte[len]; System.arraycopy(buf, recpos, segment1, 0, len); @@ -269,7 +275,14 @@ public class kelondroDyn extends kelondroTree { } } else { buf = getValueCached(dynKey(key, reccnt)); - if (buf == null) buf = new byte[reclen]; + if (buf == null) { + buf = new byte[reclen]; + } else if (buf.length < reclen) { + byte[] buff = new byte[reclen]; + System.arraycopy(buf, 0, buff, 0, buf.length); + buf = buff; + buff = null; + } //System.out.println("write: b.length="+b.length+",off="+off+",len="+(reclen-recpos)); if (len < (reclen - recpos)) System.arraycopy(b, off, buf, recpos, len); diff --git a/source/de/anomic/kelondro/kelondroFileRA.java b/source/de/anomic/kelondro/kelondroFileRA.java index 258be8d7e..840ea467f 100644 --- a/source/de/anomic/kelondro/kelondroFileRA.java +++ b/source/de/anomic/kelondro/kelondroFileRA.java @@ -45,38 +45,55 @@ import java.io.File; import java.io.FileNotFoundException; import java.io.IOException; import java.io.RandomAccessFile; +import java.io.FileDescriptor; +import java.io.SyncFailedException; import java.util.Map; import java.util.Properties; public class kelondroFileRA extends kelondroAbstractRA implements kelondroRA { protected RandomAccessFile RAFile; + protected FileDescriptor RADescriptor; - public kelondroFileRA(String file) throws FileNotFoundException { + public kelondroFileRA(String file) throws IOException, FileNotFoundException { this(new File(file)); } - public kelondroFileRA(File file) throws FileNotFoundException { + public kelondroFileRA(File file) throws IOException, FileNotFoundException { this.name = file.getName(); RAFile = new RandomAccessFile(file, "rw"); + RADescriptor = RAFile.getFD(); } + private void sync() throws IOException { + try { + RADescriptor.sync(); + //try {Thread.currentThread().sleep(8);} catch (InterruptedException e) {return;} + } catch (SyncFailedException e) { + throw new IOException(e.getMessage()); + } + } + // pseudo-native method read public int read() throws IOException { + sync(); return RAFile.read(); } // pseudo-native method write public void write(int b) throws IOException { - RAFile.write(b); + RAFile.write(b); } public int read(byte[] b, int off, int len) throws IOException { - return RAFile.read(b, off, len); + sync(); + RAFile.read(b, off, len); + return len; } public void write(byte[] b, int off, int len) throws IOException { - RAFile.write(b, off, len); + // write to file + RAFile.write(b, off, len); } public void seek(long pos) throws IOException { @@ -84,8 +101,9 @@ public class kelondroFileRA extends kelondroAbstractRA implements kelondroRA { } public void close() throws IOException { - RAFile.close(); - RAFile = null; + sync(); + RAFile.close(); + RAFile = null; } protected void finalize() throws Throwable { diff --git a/source/de/anomic/kelondro/kelondroRA.java b/source/de/anomic/kelondro/kelondroRA.java index d67ca1b74..1f4f1e8df 100644 --- a/source/de/anomic/kelondro/kelondroRA.java +++ b/source/de/anomic/kelondro/kelondroRA.java @@ -64,6 +64,7 @@ public interface kelondroRA { public void write(int b) throws IOException; public int read(byte[] b, int off, int len) throws IOException; + public void readFully(byte[] b, int off, int len) throws IOException; public void write(byte[] b, int off, int len) throws IOException; public void seek(long pos) throws IOException; diff --git a/source/de/anomic/kelondro/kelondroRecords.java b/source/de/anomic/kelondro/kelondroRecords.java index 09adce8f8..1073902ac 100644 --- a/source/de/anomic/kelondro/kelondroRecords.java +++ b/source/de/anomic/kelondro/kelondroRecords.java @@ -73,14 +73,22 @@ import java.io.IOException; import java.util.HashMap; import java.util.NoSuchElementException; import java.util.StringTokenizer; +import java.util.Map; +import java.util.Iterator; public class kelondroRecords { // constants private static final int NUL = Integer.MIN_VALUE; // the meta value for the kelondroRecords' NUL abstraction - public static final long memBlock = 5000000; // do not fill cache further if the amount of available memory is less that this + public static final long memBlock = 500000; // do not fill cache further if the amount of available memory is less that this public static final long memKcolb = 10000000; // if the amount of available memory is greater than this, do not use cache size to block, simply use memory + // caching flags + protected static final int CP_NONE = -1; // cache priority none; entry shall not be cached + protected static final int CP_LOW = 0; // cache priority low; entry may be cached + protected static final int CP_MEDIUM = 1; // cache priority medium; entry shall be cached + protected static final int CP_HIGH = 2; // cache priority high; entry must be cached + // static seek pointers private static long POS_MAGIC = 0; // 1 byte, byte: file type magic private static long POS_BUSY = POS_MAGIC + 1; // 1 byte, byte: marker for synchronization @@ -108,6 +116,8 @@ public class kelondroRecords { protected String filename; // the database's file name protected kelondroRA entryFile; // the database file private int overhead; // OHBYTEC + 4 * OHHANDLEC = size of additional control bytes + private int headchunksize;// overheadsize + key element column size + private int tailchunksize;// sum(all: COLWIDTHS) minus the size of the key element colum private int recordsize; // (overhead + sum(all: COLWIDTHS)) = the overall size of a record // dynamic run-time seek pointers @@ -146,7 +156,7 @@ public class kelondroRecords { if (file.exists()) throw new IOException("kelondroRecords: file " + file + " already exist"); this.filename = file.getCanonicalPath(); kelondroRA raf = new kelondroFileRA(this.filename); - //kelondroRA raf = new kelondroBufferedRA(new kelondroFileRA(this.filename), 5000000, 1000); + //kelondroRA raf = new kelondroBufferedRA(new kelondroFileRA(this.filename)); //kelondroRA raf = new kelondroNIOFileRA(this.filename, false, 10000); init(raf, ohbytec, ohhandlec, columns, FHandles, txtProps, txtPropWidth); this.XcacheSize = (int) (buffersize / ((long) (overhead + columns[0]))); @@ -186,6 +196,8 @@ public class kelondroRecords { this.overhead = ohbytec + 4 * ohhandlec; this.recordsize = this.overhead; for (int i = 0; i < columns.length; i++) this.recordsize += columns[i]; + this.headchunksize = overhead + columns[0]; + this.tailchunksize = this.recordsize - this.headchunksize; // store dynamic run-time seek pointers POS_HANDLES = POS_COLWIDTHS + columns.length * 4; @@ -256,7 +268,8 @@ public class kelondroRecords { this.filename = file.getCanonicalPath(); kelondroRA raf = new kelondroFileRA(this.filename); - //kelondroRA raf = new kelondroBufferedRA(new kelondroFileRA(this.filename), 5000000, 1000); + //kelondroRA raf = new kelondroBufferedRA(new kelondroFileRA(this.filename)); + //kelondroRA raf = new kelondroCachedRA(new kelondroFileRA(this.filename), 5000000, 1000); //kelondroRA raf = new kelondroNIOFileRA(this.filename, (file.length() < 4000000), 10000); init(raf); this.XcacheSize = (int) (buffersize / ((long) (overhead + COLWIDTHS[0]))); @@ -286,7 +299,8 @@ public class kelondroRecords { this.XcacheStartup = System.currentTimeMillis(); } - private void init(kelondroRA ra) throws IOException{ + private void init(kelondroRA ra) throws IOException { + // assign values that are only present at run-time this.entryFile = ra; @@ -303,6 +317,8 @@ public class kelondroRecords { entryFile.seek(POS_TXTPROPC); this.TXTPROPS = new byte[entryFile.readInt()][]; entryFile.seek(POS_TXTPROPW); this.TXTPROPW = entryFile.readInt(); + if (COLWIDTHS.length == 0) throw new kelondroException(filename, "init: zero columns; strong failure"); + // calculate dynamic run-time seek pointers POS_HANDLES = POS_COLWIDTHS + COLWIDTHS.length * 4; POS_TXTPROPS = POS_HANDLES + HANDLES.length * 4; @@ -325,38 +341,23 @@ public class kelondroRecords { // assign remaining values that are only present at run-time this.overhead = OHBYTEC + 4 * OHHANDLEC; - this.recordsize = this.overhead; for (int i = 0; i < COLWIDTHS.length; i++) this.recordsize += COLWIDTHS[i]; + this.recordsize = this.overhead; + for (int i = 0; i < COLWIDTHS.length; i++) this.recordsize += COLWIDTHS[i]; + this.headchunksize = this.overhead + COLWIDTHS[0]; + this.tailchunksize = this.recordsize - this.headchunksize; } - protected int newNode() { - Node n = new Node(); - return USEDC + FREEC; + protected Node newNode() throws IOException { + return new Node(); } - protected Node newNode(byte[][] v) { - return new Node(v); - } - - protected Node getNode(Handle handle) { + protected Node getNode(Handle handle) throws IOException { return getNode(handle, null, 0); } - protected Node getNode(Handle handle, Node parentNode, int referenceInParent) { - if (XcacheSize == 0) return new Node(handle, parentNode, referenceInParent); - synchronized (XcacheHeaders) { - Node n = (Node) XcacheHeaders.get(handle); - if (n == null) { - n = new Node(handle, parentNode, referenceInParent); - checkCacheSpace(); - n.updateNodeCache(); - return n; - } else { - //System.out.println("read from cache " + n.toString()); - XcacheScore.setScore(handle, (int) ((System.currentTimeMillis() - XcacheStartup) / 1000)); - return n; - } - } + protected Node getNode(Handle handle, Node parentNode, int referenceInParent) throws IOException { + return new Node(handle, parentNode, referenceInParent); } protected void deleteNode(Handle handle) throws IOException { @@ -397,6 +398,401 @@ public class kelondroRecords { } } + public class Node { + // an Node holds all information of one row of data. This includes the key to the entry + // which is stored as entry element at position 0 + // an Node object can be created in two ways: + // 1. instantiation with an index number. After creation the Object does not hold any + // value information until such is retrieved using the getValue() method + // 2. instantiation with a value array. the values are not directly written into the + // file. Expanding the tree structure is then done using the save() method. at any + // time it is possible to verify the save state using the saved() predicate. + // Therefore an entry object has three modes: + // a: holding an index information only (saved() = true) + // b: holding value information only (saved() = false) + // c: holding index and value information at the same time (saved() = true) + // which can be the result of one of the two processes as follow: + // (i) created with index and after using the getValue() method, or + // (ii) created with values and after calling the save() method + // the method will therefore throw an IllegalStateException when the following + // process step is performed: + // - create the Node with index and call then the save() method + // this case can be decided with + // ((index != NUL) && (values == null)) + // The save() method represents the insert function for the tree. Balancing functions + // are applied automatically. While balancing, the Node does never change its index key, + // but its parent/child keys. + //private byte[] ohBytes = null; // the overhead bytes, OHBYTEC values + //private Handle[] ohHandle= null; // the overhead handles, OHHANDLEC values + //private byte[][] values = null; // an array of byte[] nodes is the value vector + private Handle handle = null; // index of the entry, by default NUL means undefined + private byte[] headChunk = null; // contains ohBytes, ohHandles and the key value + private byte[] tailChunk = null; // contains all values except the key value + private boolean headChanged = false; + private boolean tailChanged = false; + + private Node() throws IOException { + // create a new empty node and reserve empty space in file for it + // use this method only if you want to extend the file with new entries + // without the need to have content in it. + this.handle = new Handle(); + + // create empty chunks + this.headChunk = new byte[headchunksize]; + this.tailChunk = new byte[tailchunksize]; + for (int i = 0; i < headchunksize; i++) this.headChunk[i] = 0; + for (int i = 0; i < tailchunksize; i++) this.tailChunk[i] = 0; + this.headChanged = true; + this.tailChanged = true; + } + + private Node(Handle handle) throws IOException { + // this creates an entry with an pre-reserved entry position + // values can be written using the setValues() method + // but we expect that values are already there in the file ready to be read which we do not here + if (handle == null) throw new IllegalArgumentException("INTERNAL ERROR: node handle is null."); + if (handle.index >= USEDC + FREEC) throw new kelondroException(filename, "INTERNAL ERROR: node handle index exceeds size."); + + // use given handle + this.handle = new Handle(handle.index); + + // init the content + initContent(); + } + + private Node(Handle handle, Node parentNode, int referenceInParent) throws IOException { + // this creates an entry with an pre-reserved entry position + // values can be written using the setValues() method + // but we expect that values are already there in the file ready to be read which we do not here + if (handle == null) throw new IllegalArgumentException("INTERNAL ERROR: node handle is null."); + + // the parentNode can be given if an auto-fix in the following case is wanted + if (handle.index >= USEDC + FREEC) { + if (parentNode == null) { + throw new kelondroException(filename, "INTERNAL ERROR, Node/init: node handle index exceeds size. No auto-fix node was submitted. This is a serious failure."); + } else { + try { + Handle[] handles = parentNode.getOHHandles(); + handles[referenceInParent] = null; + parentNode.setOHHandles(handles); + parentNode.commit(CP_NONE); + throw new kelondroException(filename, "INTERNAL ERROR, Node/init: node handle index " + handle.index + " exceeds size. The bad node has been auto-fixed"); + } catch (IOException ee) { + throw new kelondroException(filename, "INTERNAL ERROR, Node/init: node handle index " + handle.index + " exceeds size. It was tried to fix the bad node, but failed with an IOException: " + ee.getMessage()); + } + } + } + + // use given handle + this.handle = new Handle(handle.index); + + // init the content + initContent(); + } + + private void initContent() throws IOException { + // create chunks; read them from file or cache + this.tailChunk = null; + if (XcacheSize == 0) { + // read overhead and key + //System.out.println("**NO CACHE for " + this.handle.index + "**"); + this.headChunk = new byte[headchunksize]; + synchronized (entryFile) { + entryFile.seek(seekpos(this.handle)); + entryFile.readFully(this.headChunk, 0, this.headChunk.length); + } + this.headChanged = false; + } else synchronized(XcacheHeaders) { + byte[] cacheEntry = (byte[]) XcacheHeaders.get(this.handle); + if (cacheEntry == null) { + // read overhead and key + //System.out.println("**CACHE miss for " + this.handle.index + "**"); + this.headChunk = new byte[headchunksize]; + //this.tailChunk = new byte[tailchunksize]; + synchronized (entryFile) { + entryFile.seek(seekpos(this.handle)); + entryFile.readFully(this.headChunk, 0, this.headChunk.length); + //entryFile.read(this.tailChunk, 0, this.tailChunk.length); + } + this.headChanged = true; // provoke a cache store + checkCacheSpace(); + updateNodeCache(); + } else { + //System.out.println("**CACHE HIT for " + this.handle.index + "**"); + // copy cache entry + this.headChunk = new byte[headchunksize]; + System.arraycopy(cacheEntry, 0, this.headChunk, 0, headchunksize); + // update cache scores to announce this cache hit + XcacheScore.setScore(this.handle, (int) ((System.currentTimeMillis() - XcacheStartup) / 1000)); + this.headChanged = false; + } + } + } + + private void setValue(byte[] value, int valuewidth, byte[] targetarray, int targetoffset) { + if (value == null) { + while (valuewidth-- > 0) targetarray[targetoffset + valuewidth] = 0; + } else { + System.arraycopy(value, 0, targetarray, targetoffset, Math.min(value.length, valuewidth)); // error? + if (value.length < valuewidth) + while (valuewidth-- > value.length) targetarray[targetoffset + valuewidth] = 0; + } + } + + protected Handle handle() { + // if this entry has an index, return it + if (this.handle.index == NUL) throw new kelondroException(filename, "the entry has no index assigned"); + return new Handle(this.handle.index); + } + + protected void setOHBytes(byte[] b) throws IOException { + if (b == null) throw new IllegalArgumentException("setOHByte: setting null value does not make any sense"); + if (b.length != OHBYTEC) throw new IllegalArgumentException("setOHByte: wrong array size"); + if (this.handle.index == NUL) throw new kelondroException(filename, "setOHByte: no handle assigned"); + System.arraycopy(b, 0, this.headChunk, 0, b.length); + this.headChanged = true; + } + + protected void setOHHandles(Handle[] handles) throws IOException { + if (handles == null) throw new IllegalArgumentException("setOHint: setting null value does not make any sense"); + if (handles.length != OHHANDLEC) throw new IllegalArgumentException("setOHHandle: wrong array size"); + if (this.handle.index == NUL) throw new kelondroException(filename, "setOHHandle: no handle assigned"); + int offset = OHBYTEC; + for (int i = 0; i < handles.length; i++) { + if (handles[i] == null) { + NUL2bytes(this.headChunk, offset); + } else { + if (handles[i].index > USEDC + FREEC) throw new kelondroException(filename, "INTERNAL ERROR, setOHHandles: handle " + i + " exceeds file size (" + handles[i].index + " > " + (USEDC + FREEC) + ")"); + int2bytes(handles[i].index, this.headChunk, offset); + } + offset += 4; + } + this.headChanged = true; + } + + protected byte[] getOHBytes() throws IOException { + if (this.handle.index == NUL) throw new kelondroException(filename, "Cannot load OH values"); + byte[] b = new byte[OHBYTEC]; + System.arraycopy(this.headChunk, 0, b, 0, OHBYTEC); + return b; + } + + protected Handle[] getOHHandles() throws IOException { + if (this.handle.index == NUL) throw new kelondroException(filename, "Cannot load OH values"); + Handle[] handles = new Handle[OHHANDLEC]; + int offset = OHBYTEC; + int i; + for (int j = 0; j < handles.length; j++) { + i = bytes2int(this.headChunk, offset); + handles[j] = (i == NUL) ? null : new Handle(i); + offset += 4; + } + return handles; + } + + public byte[][] setValues(byte[][] row) throws IOException { + // if the index is defined, then write values directly to the file, else only to the object + byte[][] result = getValues(); // previous value (this loads the values if not already happened) + + // set values + if (this.handle.index != NUL) { + setValue(row[0], COLWIDTHS[0], headChunk, overhead); + int offset = 0; + for (int i = 1; i < row.length; i++) { + setValue(row[i], COLWIDTHS[i], tailChunk, offset); + offset +=COLWIDTHS[i]; + } + } + this.headChanged = true; + this.tailChanged = true; + return result; // return previous value + } + + public byte[] getKey() throws IOException { + // read key + return trimCopy(headChunk, overhead, COLWIDTHS[0]); + } + + public byte[][] getValues() throws IOException { + if (this.tailChunk == null) { + // load all values from the database file + this.tailChunk = new byte[tailchunksize]; + // read values + synchronized (entryFile) { + entryFile.seek(seekpos(this.handle) + headchunksize); + entryFile.read(this.tailChunk, 0, this.tailChunk.length); + } + } + + // create return value + byte[][] values = new byte[COLWIDTHS.length][]; + + // read key + values[0] = trimCopy(headChunk, overhead, COLWIDTHS[0]); + + // read remaining values + int offset = 0; + for (int i = 1; i < COLWIDTHS.length; i++) { + values[i] = trimCopy(tailChunk, offset, COLWIDTHS[i]); + offset += COLWIDTHS[i]; + } + + return values; + } + + public synchronized void commit(int cachePriority) throws IOException { + // this must be called after all write operations to the node are finished + + // place the data to the file + + if (this.headChunk == null) { + // there is nothing to save + throw new kelondroException(filename, "no values to save (header missing)"); + } + + /* + if (this.tailChunk == null) { + // there is nothing to save + throw new kelondroException(filename, "no values to save (tail missing)"); + } + */ + + // save head + if (this.headChanged) { + synchronized (entryFile) { + entryFile.seek(seekpos(this.handle)); + //System.out.print("#write "); printChunk(this.handle, this.headChunk); System.out.println(); + entryFile.write(this.headChunk); + } + updateNodeCache(); + } + + // save tail + if ((this.tailChunk != null) && (this.tailChanged)) synchronized (entryFile) { + entryFile.seek(seekpos(this.handle) + headchunksize); + entryFile.write(this.tailChunk); + } + } + + public synchronized void collapse() { + // this must be called after all write and read operations to the node are finished + this.headChunk = null; + this.tailChunk = null; + this.handle = null; + } + + /* + public void finalize() { + try { + commit(CP_NONE); + collapse(); + } catch (IOException e) {} + } + */ + + private byte[] trimCopy(byte[] a, int offset, int length) { + if (length > a.length - offset) length = a.length - offset; + while ((length > 0) && (a[offset + length - 1] == 0)) length--; + if (length == 0) return null; + byte[] b = new byte[length]; + System.arraycopy(a, offset, b, 0, length); + return b; + } + + public String toString() { + if (this.handle.index == NUL) return "NULL"; + String s = Integer.toHexString(this.handle.index); + while (s.length() < 4) s = "0" + s; + try { + byte[] b = getOHBytes(); + for (int i = 0; i < b.length; i++) s = s + ":b" + b[i]; + Handle[] h = getOHHandles(); + for (int i = 0; i < h.length; i++) if (h[i] == null) s = s + ":hNULL"; else s = s + ":h" + h[i].toString(); + byte[][] content = getValues(); + for (int i = 0; i < content.length; i++) s = s + ":" + ((content[i] == null) ? "NULL" : (new String(content[i])).trim()); + } catch (IOException e) { + s = s + ":***LOAD ERROR***:" + e.getMessage(); + } + return s; + } + + private void updateNodeCache() { + if (this.handle == null) return; + if (this.headChunk == null) return; + + if (XcacheSize != 0) { + synchronized (XcacheHeaders) { + // remember size to evaluate a cache size check need + int sizeBefore = XcacheHeaders.size(); + //long memBefore = Runtime.getRuntime().freeMemory(); + // generate cache entry + byte[] cacheEntry = new byte[headchunksize]; + System.arraycopy(headChunk, 0, cacheEntry, 0, headchunksize); + Handle cacheHandle = new Handle(this.handle.index); + + // store the cache entry + //XcacheHeaders.remove(cacheHandle); + XcacheHeaders.put(cacheHandle, cacheEntry); + XcacheScore.setScore(cacheHandle, (int) ((System.currentTimeMillis() - XcacheStartup) / 1000)); + + // delete the cache entry buffer + cacheEntry = null; + cacheHandle = null; + //System.out.println("kelondroRecords cache4" + filename + ": cache record size = " + (memBefore - Runtime.getRuntime().freeMemory()) + " bytes" + ((newentry) ? " new" : "")); + // check cache size + if (XcacheHeaders.size() > sizeBefore) checkCacheSpace(); + //System.out.println("kelondroRecords cache4" + filename + ": " + XcacheHeaders.size() + " entries, " + XcacheSize + " allowed."); + //printCache(); + } + } + } + } + + protected void printCache() { + if (XcacheSize == 0) { + System.out.println("### file report: " + size() + " entries"); + for (int i = 0; i < size() + 3; i++) { + // print from file to compare + System.out.print("#F " + i + ": "); + try {synchronized (entryFile) { + entryFile.seek(seekpos(new Handle(i))); + for (int j = 0; j < headchunksize; j++) System.out.print(entryFile.readByte() + ","); + }} catch (IOException e) {} + + System.out.println(); + } + } else { + System.out.println("### cache report: " + XcacheHeaders.size() + " entries"); + Iterator i = XcacheHeaders.entrySet().iterator(); + Map.Entry entry; + byte[] b; + while (i.hasNext()) { + entry = (Map.Entry) i.next(); + + // print from cache + System.out.print("#C "); + printChunk((Handle) entry.getKey(), (byte[]) entry.getValue()); + System.out.println(); + + // print from file to compare + System.out.print("#F " + ((Handle) entry.getKey()).index + ": "); + try {synchronized (entryFile) { + entryFile.seek(seekpos((Handle) entry.getKey())); + for (int j = 0; j < headchunksize; j++) System.out.print(entryFile.readByte() + ","); + }} catch (IOException e) {} + + System.out.println(); + } + } + System.out.println("### end report"); + } + + private void printChunk(Handle handle, byte[] chunk) { + System.out.print(handle.index + ": "); + for (int j = 0; j < chunk.length; j++) System.out.print(chunk[j] + ","); + } + + /* public class Node { // an Node holds all information of one row of data. This includes the key to the entry // which is stored as entry element at position 0 @@ -746,7 +1142,8 @@ public class kelondroRecords { } } } - + */ + public synchronized int columns() { return this.COLWIDTHS.length; } @@ -876,8 +1273,30 @@ public class kelondroRecords { for (int i = 0; i < b.length; i++) x = (x << 8) | (0xff & (int) b[i]); return x; } + + public static void NUL2bytes(byte[] b, int offset) { + b[offset ] = (byte) (0XFF & (NUL >> 24)); + b[offset + 1] = (byte) (0XFF & (NUL >> 16)); + b[offset + 2] = (byte) (0XFF & (NUL >> 8)); + b[offset + 3] = (byte) (0XFF & NUL); + } + + public static void int2bytes(long i, byte[] b, int offset) { + b[offset ] = (byte) (0XFF & (i >> 24)); + b[offset + 1] = (byte) (0XFF & (i >> 16)); + b[offset + 2] = (byte) (0XFF & (i >> 8)); + b[offset + 3] = (byte) (0XFF & i); + } + + public static int bytes2int(byte[] b, int offset) { + return ( + ((b[offset ] & 0xff) << 24) | + ((b[offset + 1] & 0xff) << 16) | + ((b[offset + 2] & 0xff) << 8) | + (b[offset + 3] & 0xff)); + } - public void print(boolean records) { + public void print(boolean records) throws IOException { System.out.println("REPORT FOR FILE '" + this.filename + "':"); System.out.println("--"); System.out.println("CONTROL DATA"); @@ -905,7 +1324,9 @@ public class kelondroRecords { System.out.println(" Overhead : " + this.overhead + " bytes ("+ OHBYTEC + " OH bytes, " + OHHANDLEC + " OH Handles)"); System.out.println(" Recordsize : " + this.recordsize + " bytes"); System.out.println("--"); - + printCache(); + System.out.println("--"); + if (!(records)) return; // print also all records for (int i = 0; i < USEDC + FREEC; i++) System.out.println("NODE: " + new Node(new Handle(i), null, 0).toString()); diff --git a/source/de/anomic/kelondro/kelondroStack.java b/source/de/anomic/kelondro/kelondroStack.java index 501c9e0a2..d90468605 100644 --- a/source/de/anomic/kelondro/kelondroStack.java +++ b/source/de/anomic/kelondro/kelondroStack.java @@ -101,11 +101,11 @@ public class kelondroStack extends kelondroRecords { public Object next() { Handle ret = nextHandle; try { - nextHandle = getNode(nextHandle, null, 0).getOHHandle()[right]; + nextHandle = getNode(nextHandle, null, 0).getOHHandles()[right]; + return getNode(ret, null, 0); } catch (IOException e) { - System.err.println("IO error at Counter:next()"); + throw new kelondroException(filename, "IO error at Counter:next()"); } - return getNode(ret, null, 0); } public void remove() { throw new UnsupportedOperationException("no remove here.."); @@ -119,21 +119,23 @@ public class kelondroStack extends kelondroRecords { if (getHandle(toor) == null) { if (getHandle(root) != null) throw new RuntimeException("push: internal organisation of root and toor"); // create node - Node n = newNode(row); - n.save(); - n.setOHHandle(new Handle[] {null, null}); - n.setValues(row); + Node n = newNode(); + n.setValues(row); + n.setOHHandles(new Handle[] {null, null}); + n.commit(CP_NONE); // assign handles setHandle(root, n.handle()); setHandle(toor, n.handle()); // thats it } else { // expand the list at the end - Node n = newNode(row); - n.save(); - n.setOHHandle(new Handle[] {getHandle(toor), null}); + Node n = newNode(); + n.setValues(row); + n.setOHHandles(new Handle[] {getHandle(toor), null}); + n.commit(CP_NONE); Node n1 = getNode(getHandle(toor), null, 0); - n1.setOHHandle(new Handle[] {n1.getOHHandle()[left], n.handle()}); + n1.setOHHandles(new Handle[] {n1.getOHHandles()[left], n.handle()}); + n1.commit(CP_NONE); // assign handles setHandle(toor, n.handle()); // thats it @@ -212,8 +214,8 @@ public class kelondroStack extends kelondroRecords { private void unlinkNode(Node n) throws IOException { // join chaines over node - Handle l = n.getOHHandle()[left]; - Handle r = n.getOHHandle()[right]; + Handle l = n.getOHHandles()[left]; + Handle r = n.getOHHandles()[right]; // look left if (l == null) { // reached the root on left side @@ -221,7 +223,8 @@ public class kelondroStack extends kelondroRecords { } else { // un-link the previous record Node k = getNode(l, null, 0); - k.setOHHandle(new Handle[] {k.getOHHandle()[left], r}); + k.setOHHandles(new Handle[] {k.getOHHandles()[left], r}); + k.commit(CP_NONE); } // look right if (r == null) { @@ -230,7 +233,8 @@ public class kelondroStack extends kelondroRecords { } else { // un-link the following record Node k = getNode(r, null, 0); - k.setOHHandle(new Handle[] {l, k.getOHHandle()[right]}); + k.setOHHandles(new Handle[] {l, k.getOHHandles()[right]}); + k.commit(CP_NONE); } } @@ -249,7 +253,7 @@ public class kelondroStack extends kelondroRecords { Handle h = getHandle(side); if (h == null) return null; if (dist >= size()) return null; // that would exceed the stack - while (dist-- > 0) h = getNode(h, null, 0).getOHHandle()[dir]; // track through elements + while (dist-- > 0) h = getNode(h, null, 0).getOHHandles()[dir]; // track through elements return getNode(h, null, 0); } @@ -313,7 +317,7 @@ public class kelondroStack extends kelondroRecords { if (h == null) return "NULL"; else return h.toString(); } - public void print() { + public void print() throws IOException { super.print(false); Node n; try { @@ -322,7 +326,7 @@ public class kelondroStack extends kelondroRecords { n = (Node) it.next(); //n = getNode(h, null, 0); System.out.println("> NODE " + hp(n.handle()) + - "; left " + hp(n.getOHHandle()[left]) + ", right " + hp(n.getOHHandle()[right])); + "; left " + hp(n.getOHHandles()[left]) + ", right " + hp(n.getOHHandles()[right])); System.out.print(" KEY:'" + (new String(n.getValues()[0])).trim() + "'"); for (int j = 1; j < columns(); j++) System.out.print(", V[" + j + "]:'" + (new String(n.getValues()[j])).trim() + "'"); diff --git a/source/de/anomic/kelondro/kelondroTree.java b/source/de/anomic/kelondro/kelondroTree.java index 3224560ed..0bd104afd 100644 --- a/source/de/anomic/kelondro/kelondroTree.java +++ b/source/de/anomic/kelondro/kelondroTree.java @@ -212,34 +212,38 @@ public class kelondroTree extends kelondroRecords implements Comparator { if (visitedNodeKeys.containsKey(otherkey)) { // we have loops in the database. // to fix this, all affected nodes must be patched - thenode.setOHByte(new byte[] {1, 0}); - thenode.setOHHandle(new Handle[] {null, null, null}); + thenode.setOHBytes(new byte[] {1, 0}); + thenode.setOHHandles(new Handle[] {null, null, null}); Iterator fix = visitedNodeKeys.entrySet().iterator(); Map.Entry entry; while (fix.hasNext()) { entry = (Map.Entry) fix.next(); thenode = (Node) entry.getValue(); - thenode.setOHByte(new byte[] {1, 0}); - thenode.setOHHandle(new Handle[] {null, null, null}); + thenode.setOHBytes(new byte[] {1, 0}); + thenode.setOHHandles(new Handle[] {null, null, null}); } + thenode.commit(CP_NONE); + //printCache(); throw new kelondroException(filename, "database contains loops; the loop-nodes have been auto-fixed"); } - //System.out.print("Comparing key = '" + new String(key) + "' with '" + otherkey + "':"); // debug + //System.out.println("Comparing key = '" + new String(key) + "' with '" + otherkey + "':"); // debug c = compare(key, thenode.getKey()); //System.out.println(c); // debug if (c == 0) { found = true; + //System.out.println("DEBUG: search for " + new String(key) + " ended with status=" + ((found) ? "found" : "not-found") + ", node=" + ((thenode == null) ? "NULL" : thenode.toString()) + ", parent=" + ((parentnode == null) ? "NULL" : parentnode.toString())); return; } else if (c < 0) { child = -1; - thisHandle = thenode.getOHHandle()[leftchild]; + thisHandle = thenode.getOHHandles()[leftchild]; } else { child = 1; - thisHandle = thenode.getOHHandle()[rightchild]; + thisHandle = thenode.getOHHandles()[rightchild]; } visitedNodeKeys.put(otherkey, thenode); } } + //System.out.println("DEBUG: search for " + new String(key) + " ended with status=" + ((found) ? "found" : "not-found") + ", node=" + ((thenode == null) ? "NULL" : thenode.toString()) + ", parent=" + ((parentnode == null) ? "NULL" : parentnode.toString())); // we reached a node where we must insert the new value // the parent of this new value can be obtained by getParent() // all values are set, just return @@ -276,7 +280,7 @@ public class kelondroTree extends kelondroRecords implements Comparator { public boolean isChild(Node childn, Node parentn, int child) throws IOException { if (childn == null) throw new IllegalArgumentException("isLeftChild: Node parameter is NULL"); - Handle lc = parentn.getOHHandle()[child]; + Handle lc = parentn.getOHHandles()[child]; if (lc == null) return false; return (lc.equals(childn.handle())); } @@ -305,11 +309,12 @@ public class kelondroTree extends kelondroRecords implements Comparator { public synchronized byte[][] put(byte[][] newrow) throws IOException { if (newrow.length != columns()) throw new IllegalArgumentException("put: wrong row length " + newrow.length + "; must be " + columns()); // first try to find the key element in the database - Search searchResult = new Search(newrow[0]); - if (searchResult.found()) { + Search searchResult = new Search(newrow[0]); + if (searchResult.found()) { // a node with this key exist. simply overwrite the content and return old content Node e = searchResult.getMatcher(); byte[][] result = e.setValues(newrow); + e.commit(CP_MEDIUM); searchResult = null; return result; } else if (searchResult.isRoot()) { @@ -318,13 +323,14 @@ public class kelondroTree extends kelondroRecords implements Comparator { if (getHandle(root) != null) throw new kelondroException(filename, "tried to create root node twice"); // we dont have any Nodes in the file, so start here to create one - Node e = newNode(newrow); - e.save(); + Node e = newNode(); + e.setValues(newrow); // write the propetries - e.setOHByte(new byte[] {1, 0}); // {magic, balance} - e.setOHHandle(new Handle[] {null, null, null}); // {parent, leftchild, rightchild} + e.setOHBytes(new byte[] {1, 0}); // {magic, balance} + e.setOHHandles(new Handle[] {null, null, null}); // {parent, leftchild, rightchild} // do updates - setHandle(root, e.handle()); + e.commit(CP_LOW); + setHandle(root, e.handle()); searchResult = null; return null; } else { @@ -336,16 +342,16 @@ public class kelondroTree extends kelondroRecords implements Comparator { // that side, but not if the assigned position is appropriate. // create new node and assign values - Node theNode = newNode(newrow); theNode.save(); - Node parentNode = searchResult.getParent(); - Handle[] parentOHHandle; - byte[] parentOHByte; - - theNode.setOHByte(new byte[] {1, 0}); // fresh {magic, balance} - theNode.setOHHandle(new Handle[] {parentNode.handle(), null, null}); // {parent, leftchild, rightchild} - + Node parentNode = searchResult.getParent(); + Node theNode = newNode(); + theNode.setValues(newrow); + theNode.setOHBytes(new byte[] {1, 0}); // fresh {magic, balance} + theNode.setOHHandles(new Handle[] {parentNode.handle(), null, null}); // {parent, leftchild, rightchild} + theNode.commit(CP_LOW); + // check consistency and link new node to parent node - parentOHHandle = parentNode.getOHHandle(); // {parent, leftchild, rightchild} + byte[] parentOHByte; + Handle[] parentOHHandle = parentNode.getOHHandles(); // {parent, leftchild, rightchild} if (searchResult.isLeft()) { if (parentOHHandle[leftchild] != null) throw new kelondroException(filename, "tried to create leftchild node twice"); parentOHHandle[leftchild] = theNode.handle(); @@ -355,8 +361,9 @@ public class kelondroTree extends kelondroRecords implements Comparator { } else { throw new kelondroException(filename, "neither left nor right child"); } - parentNode.setOHHandle(parentOHHandle); - + parentNode.setOHHandles(parentOHHandle); + parentNode.commit(((parentOHHandle[leftchild] == null) || (parentOHHandle[rightchild] == null)) ? CP_MEDIUM : CP_HIGH); + // now update recursively the node balance of the parentNode // what do we have: // - new Node, called 'theNode' @@ -369,8 +376,8 @@ public class kelondroTree extends kelondroRecords implements Comparator { while (increasedHight) { // update balance - parentOHByte = parentNode.getOHByte(); // {magic, balance} - parentOHHandle = parentNode.getOHHandle(); // {parent, leftchild, rightchild} + parentOHByte = parentNode.getOHBytes(); // {magic, balance} + parentOHHandle = parentNode.getOHHandles(); // {parent, leftchild, rightchild} prevHight = parentOHByte[balance]; if ((parentOHHandle[leftchild] != null) && (parentOHHandle[leftchild].equals(theNode.handle()))) { //isLeftchild @@ -382,7 +389,8 @@ public class kelondroTree extends kelondroRecords implements Comparator { path = "R" + path; } increasedHight = ((java.lang.Math.abs((int) parentOHByte[balance]) - java.lang.Math.abs((int) prevHight)) > 0); - parentNode.setOHByte(parentOHByte); + parentNode.setOHBytes(parentOHByte); + parentNode.commit(((parentOHHandle[leftchild] == null) || (parentOHHandle[rightchild] == null)) ? CP_MEDIUM : CP_HIGH); // here we either stop because we had no increased hight, // or we have a balance greater then 1 or less than -1 and we do rotation @@ -403,16 +411,16 @@ public class kelondroTree extends kelondroRecords implements Comparator { } if (path.startsWith("RL")) { Handle parentHandle = parentNode.handle(); - LL_RightRotation(theNode, getNode(theNode.getOHHandle()[leftchild], theNode, leftchild)); + LL_RightRotation(theNode, getNode(theNode.getOHHandles()[leftchild], theNode, leftchild)); parentNode = getNode(parentHandle, null, 0); // reload the parent node - RR_LeftRotation(parentNode, getNode(parentNode.getOHHandle()[rightchild], parentNode, rightchild)); + RR_LeftRotation(parentNode, getNode(parentNode.getOHHandles()[rightchild], parentNode, rightchild)); break; } if (path.startsWith("LR")) { Handle parentHandle = parentNode.handle(); - RR_LeftRotation(theNode, getNode(theNode.getOHHandle()[rightchild], theNode, rightchild)); + RR_LeftRotation(theNode, getNode(theNode.getOHHandles()[rightchild], theNode, rightchild)); parentNode = getNode(parentHandle, null, 0); // reload the parent node - LL_RightRotation(parentNode, getNode(parentNode.getOHHandle()[leftchild], parentNode, leftchild)); + LL_RightRotation(parentNode, getNode(parentNode.getOHHandles()[leftchild], parentNode, leftchild)); break; } break; @@ -423,23 +431,26 @@ public class kelondroTree extends kelondroRecords implements Comparator { break; } else { theNode = parentNode; - parentNode = getNode(parentOHHandle[parent] /*previous handles*/, null, 0); + parentNode = getNode(parentOHHandle[parent], null, 0); } } } + return null; // that means: no previous stored value present } } private void assignChild(Node parentNode, Node childNode, int childType) throws IOException { - Handle[] parentHandle = parentNode.getOHHandle(); - Handle[] childHandle = childNode.getOHHandle(); + Handle[] parentOHHandle = parentNode.getOHHandles(); + Handle[] childOHHandle = childNode.getOHHandles(); - parentHandle[childType] = childNode.handle(); - childHandle[parent] = parentNode.handle(); + parentOHHandle[childType] = childNode.handle(); + childOHHandle[parent] = parentNode.handle(); - parentNode.setOHHandle(parentHandle); - childNode.setOHHandle(childHandle); + parentNode.setOHHandles(parentOHHandle); + parentNode.commit(((parentOHHandle[leftchild] == null) || (parentOHHandle[rightchild] == null)) ? CP_MEDIUM : CP_HIGH); + childNode.setOHHandles(childOHHandle); + childNode.commit(((childOHHandle[leftchild] == null) || (childOHHandle[rightchild] == null)) ? CP_MEDIUM : CP_HIGH); } private void replace(Node oldNode, Node oldNodeParent, Node newNode) throws IOException { @@ -447,18 +458,19 @@ public class kelondroTree extends kelondroRecords implements Comparator { // the anchor's link to the oldNode by the newNode-link // the new link gets the anchor as parent link assigned // the oldNode will not be updated, so this must be done outside this routine - Handle[] oldHandle = oldNode.getOHHandle(); // {parent, leftchild, rightchild} + Handle[] oldHandle = oldNode.getOHHandles(); // {parent, leftchild, rightchild} // distinguish case where the oldNode is the root node if (oldNodeParent == null) { // this is the root, update root setHandle(root, newNode.handle()); // update new Node - Handle[] newHandle = newNode.getOHHandle(); + Handle[] newHandle = newNode.getOHHandles(); newHandle[parent] = null; - newNode.setOHHandle(newHandle); + newNode.setOHHandles(newHandle); + newNode.commit(CP_HIGH); } else { // not the root, find parent - Handle[] parentHandle = oldNodeParent.getOHHandle(); + Handle[] parentHandle = oldNodeParent.getOHHandles(); // ok, we have the parent, but for updating the child link we must know // if the oldNode was left or right child if ((parentHandle[leftchild] != null) && (parentHandle[leftchild].equals(oldNode.handle()))) { @@ -469,11 +481,13 @@ public class kelondroTree extends kelondroRecords implements Comparator { // update right node from parent parentHandle[rightchild] = newNode.handle(); } - oldNodeParent.setOHHandle(parentHandle); + oldNodeParent.setOHHandles(parentHandle); + oldNodeParent.commit(((parentHandle[leftchild] == null) || (parentHandle[rightchild] == null)) ? CP_MEDIUM : CP_HIGH); // update new Node - Handle[] newHandle = newNode.getOHHandle(); + Handle[] newHandle = newNode.getOHHandles(); newHandle[parent] = oldNodeParent.handle(); - newNode.setOHHandle(newHandle); + newNode.setOHHandles(newHandle); + newNode.commit(((newHandle[leftchild] == null) || (newHandle[rightchild] == null)) ? CP_MEDIUM : CP_HIGH); } // finished. remember that we did not set the links to the oldNode // we have also not set the children of the newNode. @@ -491,16 +505,16 @@ public class kelondroTree extends kelondroRecords implements Comparator { private void LL_RightRotation(Node parentNode, Node childNode) throws IOException { // replace the parent node; the parent is afterwards unlinked - Handle p2Handle = parentNode.getOHHandle()[parent]; + Handle p2Handle = parentNode.getOHHandles()[parent]; Node p2Node = (p2Handle == null) ? null : getNode(p2Handle, null, 0); replace(parentNode, p2Node, childNode); // set the left son of the parent to the right son of the childNode - Handle childOfChild = childNode.getOHHandle()[rightchild]; + Handle childOfChild = childNode.getOHHandles()[rightchild]; if (childOfChild == null) { - Handle[] parentHandle = parentNode.getOHHandle(); + Handle[] parentHandle = parentNode.getOHHandles(); parentHandle[leftchild] = null; - parentNode.setOHHandle(parentHandle); + parentNode.setOHHandles(parentHandle); } else { assignChild(parentNode, getNode(childOfChild, childNode, rightchild), leftchild); } @@ -510,28 +524,30 @@ public class kelondroTree extends kelondroRecords implements Comparator { // - newBal(parent) = oldBal(parent) - 1 - max(oldBal(leftChild), 0) // - newBal(leftChild) = oldBal(leftChild) - 1 + min(newBal(parent), 0) - byte[] parentBytes = parentNode.getOHByte(); - byte[] childBytes = childNode.getOHByte(); + byte[] parentBytes = parentNode.getOHBytes(); + byte[] childBytes = childNode.getOHBytes(); byte oldBalParent = parentBytes[balance]; byte oldBalChild = childBytes[balance]; parentBytes[balance] = (byte) (oldBalParent - 1 - max0(oldBalChild)); childBytes[balance] = (byte) (oldBalChild - 1 + min0(parentBytes[balance])); - parentNode.setOHByte(parentBytes); - childNode.setOHByte(childBytes); + parentNode.setOHBytes(parentBytes); + childNode.setOHBytes(childBytes); + parentNode.commit(CP_NONE); + childNode.commit(CP_NONE); } private void RR_LeftRotation(Node parentNode, Node childNode) throws IOException { // replace the parent node; the parent is afterwards unlinked - Handle p2Handle = parentNode.getOHHandle()[parent]; + Handle p2Handle = parentNode.getOHHandles()[parent]; Node p2Node = (p2Handle == null) ? null : getNode(p2Handle, null, 0); replace(parentNode, p2Node, childNode); // set the left son of the parent to the right son of the childNode - Handle childOfChild = childNode.getOHHandle()[leftchild]; + Handle childOfChild = childNode.getOHHandles()[leftchild]; if (childOfChild == null) { - Handle[] parentHandle = parentNode.getOHHandle(); + Handle[] parentHandle = parentNode.getOHHandles(); parentHandle[rightchild] = null; - parentNode.setOHHandle(parentHandle); + parentNode.setOHHandles(parentHandle); } else { assignChild(parentNode, getNode(childOfChild, childNode, leftchild), rightchild); } @@ -541,14 +557,16 @@ public class kelondroTree extends kelondroRecords implements Comparator { // - newBal(parent) = oldBal(parent) + 1 - min(oldBal(rightChild), 0) // - newBal(rightChild) = oldBal(rightChild) + 1 + max(newBal(parent), 0) - byte[] parentBytes = parentNode.getOHByte(); - byte[] childBytes = childNode.getOHByte(); + byte[] parentBytes = parentNode.getOHBytes(); + byte[] childBytes = childNode.getOHBytes(); byte oldBalParent = parentBytes[balance]; byte oldBalChild = childBytes[balance]; parentBytes[balance] = (byte) (oldBalParent + 1 - min0(oldBalChild)); childBytes[balance] = (byte) (oldBalChild + 1 + max0(parentBytes[balance])); - parentNode.setOHByte(parentBytes); - childNode.setOHByte(childBytes); + parentNode.setOHBytes(parentBytes); + childNode.setOHBytes(childBytes); + parentNode.commit(CP_NONE); + childNode.commit(CP_NONE); } // Associates the specified value with the specified key in this map @@ -586,7 +604,7 @@ public class kelondroTree extends kelondroRecords implements Comparator { // by the greatest node of the left child or the smallest // node of the right child - Handle[] handles = node.getOHHandle(); + Handle[] handles = node.getOHHandles(); Node childnode; if ((handles[leftchild] == null) && (handles[rightchild] == null)) { // easy case: the node is a leaf @@ -594,10 +612,11 @@ public class kelondroTree extends kelondroRecords implements Comparator { // this is the root! setHandle(root, null); } else { - Handle[] h = parentOfNode.getOHHandle(); + Handle[] h = parentOfNode.getOHHandles(); if ((h[leftchild] != null) && (h[leftchild].equals(node.handle()))) h[leftchild] = null; if ((h[rightchild] != null) && (h[rightchild].equals(node.handle()))) h[rightchild] = null; - parentOfNode.setOHHandle(h); + parentOfNode.setOHHandles(h); + parentOfNode.commit(((h[leftchild] == null) && (h[rightchild] == null)) ? CP_LOW : CP_MEDIUM); } } else if ((handles[leftchild] != null) && (handles[rightchild] == null)) { replace(node, parentOfNode, getNode(handles[leftchild], node, leftchild)); @@ -610,17 +629,18 @@ public class kelondroTree extends kelondroRecords implements Comparator { // we remove that replacement node and put it where the node was // this seems to be recursive, but is not since the replacement // node cannot have two children (it would not have been the smallest or greatest) - Handle[] replha = repl.getOHHandle(); + Handle[] replha = repl.getOHHandles(); Node n; Handle[] h; // remove leaf if ((replha[leftchild] == null) && (replha[rightchild] == null)) { // the replacement cannot be the root, so simply remove from parent node n = getNode(replha[parent], null, 0); // parent node of replacement node - h = n.getOHHandle(); + h = n.getOHHandles(); if ((h[leftchild] != null) && (h[leftchild].equals(repl.handle()))) h[leftchild] = null; if ((h[rightchild] != null) && (h[rightchild].equals(repl.handle()))) h[rightchild] = null; - n.setOHHandle(h); + n.setOHHandles(h); + n.commit(((h[leftchild] == null) && (h[rightchild] == null)) ? CP_LOW : CP_MEDIUM); } else if ((replha[leftchild] != null) && (replha[rightchild] == null)) { try { childnode = getNode(replha[leftchild], repl, leftchild); @@ -628,10 +648,11 @@ public class kelondroTree extends kelondroRecords implements Comparator { } catch (IllegalArgumentException e) { // now treat the situation as if that link had been null before n = getNode(replha[parent], null, 0); // parent node of replacement node - h = n.getOHHandle(); + h = n.getOHHandles(); if ((h[leftchild] != null) && (h[leftchild].equals(repl.handle()))) h[leftchild] = null; if ((h[rightchild] != null) && (h[rightchild].equals(repl.handle()))) h[rightchild] = null; - n.setOHHandle(h); + n.setOHHandles(h); + n.commit(((h[leftchild] == null) && (h[rightchild] == null)) ? CP_LOW : CP_MEDIUM); } } else if ((replha[leftchild] == null) && (replha[rightchild] != null)) { try { @@ -640,10 +661,11 @@ public class kelondroTree extends kelondroRecords implements Comparator { } catch (IllegalArgumentException e) { // now treat the situation as if that link had been null before n = getNode(replha[parent], null, 0); // parent node of replacement node - h = n.getOHHandle(); + h = n.getOHHandles(); if ((h[leftchild] != null) && (h[leftchild].equals(repl.handle()))) h[leftchild] = null; if ((h[rightchild] != null) && (h[rightchild].equals(repl.handle()))) h[rightchild] = null; - n.setOHHandle(h); + n.setOHHandles(h); + n.commit(((h[leftchild] == null) && (h[rightchild] == null)) ? CP_LOW : CP_MEDIUM); } } //System.out.println("node before reload is " + node.toString()); @@ -651,25 +673,29 @@ public class kelondroTree extends kelondroRecords implements Comparator { //System.out.println("node after reload is " + node.toString()); // now plant in the replha node - byte[] b = node.getOHByte(); // save bytes of disappearing node - handles = node.getOHHandle(); // save handles of disappearing node + byte[] b = node.getOHBytes(); // save bytes of disappearing node + handles = node.getOHHandles(); // save handles of disappearing node replace(node, parentOfNode, repl); - repl.setOHByte(b); // restore bytes - repl.setOHHandle(handles); // restore handles + repl.setOHBytes(b); // restore bytes + repl.setOHHandles(handles); // restore handles + repl.commit(((handles[leftchild] == null) || (handles[rightchild] == null)) ? CP_MEDIUM : CP_HIGH); // last thing to do: change uplinks of children to this new node if (handles[leftchild] != null) { n = getNode(handles[leftchild], node, leftchild); - h = n.getOHHandle(); + h = n.getOHHandles(); h[parent] = repl.handle(); - n.setOHHandle(h); + n.setOHHandles(h); + n.commit(((h[leftchild] == null) || (h[rightchild] == null)) ? CP_MEDIUM : CP_HIGH); } if (handles[rightchild] != null) { n = getNode(handles[rightchild], node, rightchild); - h = n.getOHHandle(); + h = n.getOHHandles(); h[parent] = repl.handle(); - n.setOHHandle(h); + n.setOHHandles(h); + n.commit(((h[leftchild] == null) || (h[rightchild] == null)) ? CP_MEDIUM : CP_HIGH); } } + // move node to recycling queue deleteNode(node.handle()); } @@ -681,7 +707,7 @@ public class kelondroTree extends kelondroRecords implements Comparator { private Node firstNode(Node node) throws IOException { if (node == null) throw new IllegalArgumentException("firstNode: node=null"); - Handle h = node.getOHHandle()[leftchild]; + Handle h = node.getOHHandles()[leftchild]; while (h != null) { try { node = getNode(h, node, leftchild); @@ -689,7 +715,7 @@ public class kelondroTree extends kelondroRecords implements Comparator { // return what we have return node; } - h = node.getOHHandle()[leftchild]; + h = node.getOHHandles()[leftchild]; } return node; } @@ -702,7 +728,7 @@ public class kelondroTree extends kelondroRecords implements Comparator { private Node lastNode(Node node) throws IOException { if (node == null) throw new IllegalArgumentException("lastNode: node=null"); - Handle h = node.getOHHandle()[rightchild]; + Handle h = node.getOHHandles()[rightchild]; while (h != null) { try { node = getNode(h, node, rightchild); @@ -710,7 +736,7 @@ public class kelondroTree extends kelondroRecords implements Comparator { // return what we have return node; } - h = node.getOHHandle()[rightchild]; + h = node.getOHHandles()[rightchild]; } return node; } @@ -823,7 +849,7 @@ public class kelondroTree extends kelondroRecords implements Comparator { nodeStack.addLast(new Object[]{searchNode, new Integer(ct)}); // go to next node - searchHandle = searchNode.getOHHandle()[ct]; + searchHandle = searchNode.getOHHandles()[ct]; if (searchHandle == null) throw new kelondroException(filename, "start node does not exist (handle null)"); searchNode = getNode(searchHandle, searchNode, ct); if (searchNode == null) throw new kelondroException(filename, "start node does not exist (node null)"); @@ -850,14 +876,14 @@ public class kelondroTree extends kelondroRecords implements Comparator { try { int childtype = (up) ? rightchild : leftchild; - Handle childHandle = nextNode.getOHHandle()[childtype]; + Handle childHandle = nextNode.getOHHandles()[childtype]; if (childHandle != null) { //System.out.println("go to other leg, stack size=" + nodeStack.size()); // we have walked one leg of the tree; now go to the other one: step down to next child nodeStack.addLast(new Object[]{nextNode, new Integer(childtype)}); nextNode = getNode(childHandle, nextNode, childtype); childtype = (up) ? leftchild : rightchild; - while ((childHandle = nextNode.getOHHandle()[childtype]) != null) { + while ((childHandle = nextNode.getOHHandles()[childtype]) != null) { try { nodeStack.addLast(new Object[]{nextNode, new Integer(childtype)}); nextNode = getNode(childHandle, nextNode, childtype); @@ -1056,7 +1082,7 @@ public class kelondroTree extends kelondroRecords implements Comparator { private int height(Node node) throws IOException { if (node == null) return 0; - Handle[] childs = node.getOHHandle(); + Handle[] childs = node.getOHHandles(); int hl = (childs[leftchild] == null) ? 0 : height(getNode(childs[leftchild], node, leftchild)); int hr = (childs[rightchild] == null) ? 0 : height(getNode(childs[rightchild], node, rightchild)); if (hl > hr) return hl + 1; else return hr + 1; @@ -1098,7 +1124,7 @@ public class kelondroTree extends kelondroRecords implements Comparator { nextline.add(null); nextline.add(null); } else { - childs = node.getOHHandle(); + childs = node.getOHHandles(); nextline.add(childs[leftchild]); nextline.add(childs[rightchild]); } @@ -1317,9 +1343,9 @@ public class kelondroTree extends kelondroRecords implements Comparator { } public static void main(String[] args) { - cmd(args); + //cmd(args); //bigtest(Integer.parseInt(args[0])); - //randomtest(Integer.parseInt(args[0])); + randomtest(Integer.parseInt(args[0])); //smalltest(); } @@ -1348,17 +1374,6 @@ public class kelondroTree extends kelondroRecords implements Comparator { return new byte[]{(byte) c, 32, 32, 32}; } - public static kelondroTree testTree(File f, String testentities) throws IOException { - if (f.exists()) f.delete(); - kelondroTree tt = new kelondroTree(f, 0, 4, 4); - byte[] b; - for (int i = 0; i < testentities.length(); i++) { - b = testWord(testentities.charAt(i)); - tt.put(b, b); - } - return tt; - } - public static void randomtest(int elements) { System.out.println("random " + elements + ":"); String s = "ABCDEFGHIJKLMNOPQRSTUVWXYZ".substring(0, elements); @@ -1371,7 +1386,7 @@ public class kelondroTree extends kelondroRecords implements Comparator { int steps = 0; while (true) { if (testFile.exists()) testFile.delete(); - tt = new kelondroTree(testFile, 20000, 4 ,4); + tt = new kelondroTree(testFile, 0, 4 ,4); steps = 10 + ((int) System.currentTimeMillis() % 7) * (((int) System.currentTimeMillis() + 17) % 11); t = s; d = ""; @@ -1394,31 +1409,35 @@ public class kelondroTree extends kelondroRecords implements Comparator { t = t + c; System.out.println("removed " + new String(b)); } + //tt.printCache(); //tt.print(); + if (countElements(tt) != tt.size()) { - System.out.println("wrong size for "); + System.out.println("wrong size for this table:"); tt.print(); } + // check all words within for (int j = 0; j < d.length(); j++) { if (tt.get(testWord(d.charAt(j))) == null) { - System.out.println("missing entry " + d.charAt(j)); + System.out.println("missing entry " + d.charAt(j) + " in this table:"); tt.print(); } } // check all words outside for (int j = 0; j < t.length(); j++) { if (tt.get(testWord(t.charAt(j))) != null) { - System.out.println("superfluous entry " + t.charAt(j)); + System.out.println("superfluous entry " + t.charAt(j) + " in this table:"); tt.print(); } } if (tt.get(testWord('z')) != null) { - System.out.println("superfluous entry z"); + System.out.println("superfluous entry z in this table:"); tt.print(); } + } - tt.print(); + //tt.print(); tt.close(); } @@ -1433,27 +1452,44 @@ public class kelondroTree extends kelondroRecords implements Comparator { File f = new File("test.db"); if (f.exists()) f.delete(); try { - kelondroTree tt = new kelondroTree(f, 1000, 4, 4); + kelondroTree tt = new kelondroTree(f, 0, 4, 4); byte[] b; - b = testWord('b'); tt.put(b, b); - b = testWord('c'); tt.put(b, b); - b = testWord('a'); tt.put(b, b); + b = testWord('B'); tt.put(b, b); tt.print(); + b = testWord('C'); tt.put(b, b); tt.print(); + b = testWord('D'); tt.put(b, b); tt.print(); + b = testWord('A'); tt.put(b, b); tt.print(); + b = testWord('D'); tt.remove(b); tt.print(); + b = testWord('B'); tt.remove(b); tt.print(); + b = testWord('B'); + tt.put(b, b); tt.print(); System.out.println("elements: " + countElements(tt)); - tt.print(); + System.out.println("TERMINATED"); } catch (IOException e) { e.printStackTrace(); } } - public static void bigtest(int elements) { - System.out.println("perm " + elements + ":"); + public static kelondroTree testTree(File f, String testentities) throws IOException { + if (f.exists()) f.delete(); + kelondroTree tt = new kelondroTree(f, 200000, 4, 4); + byte[] b; + for (int i = 0; i < testentities.length(); i++) { + b = testWord(testentities.charAt(i)); + tt.put(b, b); + } + return tt; + } + + public static void bigtest(int elements) { + System.out.println("starting big test with " + elements + " elements:"); + long start = System.currentTimeMillis(); String[] s = permutations(elements); kelondroTree tt; File testFile = new File("test.db"); byte[] b; try { for (int i = 0; i < s.length; i++) { - System.out.println("probing tree " + i + " for permutation " + s[i]); + System.out.println("*** probing tree " + i + " for permutation " + s[i]); // generate tree and delete elements tt = testTree(testFile, s[i]); //tt.print(); @@ -1491,17 +1527,23 @@ public class kelondroTree extends kelondroRecords implements Comparator { tt.close(); } } - System.out.println("FINISHED"); + System.out.println("FINISHED test after " + ((System.currentTimeMillis() - start) / 1000) + " seconds."); } catch (Exception e) { e.printStackTrace(); System.out.println("TERMINATED"); } } - public static int countElements(kelondroTree t) { + public static int countElements(kelondroTree t) throws IOException { int count = 0; Iterator iter = t.nodeIterator(true, false); - while (iter.hasNext()) {count++; if (iter.next() == null) System.out.println("ERROR! null element found");} + Node n; + while (iter.hasNext()) { + count++; + n = (Node) iter.next(); + if (n == null) System.out.println("ERROR! null element found"); + //else System.out.println("counted element: " + new String(n.getKey())); + } return count; } } diff --git a/source/de/anomic/plasma/plasmaCrawlRobotsTxt.java b/source/de/anomic/plasma/plasmaCrawlRobotsTxt.java index 3f7c7bbaf..772e354f0 100644 --- a/source/de/anomic/plasma/plasmaCrawlRobotsTxt.java +++ b/source/de/anomic/plasma/plasmaCrawlRobotsTxt.java @@ -56,6 +56,7 @@ import java.util.LinkedList; import java.util.Map; import de.anomic.kelondro.kelondroDyn; import de.anomic.kelondro.kelondroMap; +import de.anomic.kelondro.kelondroException; import de.anomic.server.logging.serverLog; public class plasmaCrawlRobotsTxt { @@ -65,7 +66,13 @@ public class plasmaCrawlRobotsTxt { public plasmaCrawlRobotsTxt(File robotsTableFile) throws IOException { this.robotsTableFile = robotsTableFile; if (robotsTableFile.exists()) { - robotsTable = new kelondroMap(new kelondroDyn(robotsTableFile, 32000)); + try { + robotsTable = new kelondroMap(new kelondroDyn(robotsTableFile, 32000)); + } catch (kelondroException e) { + robotsTableFile.delete(); + robotsTableFile.getParentFile().mkdirs(); + robotsTable = new kelondroMap(new kelondroDyn(robotsTableFile, 32000, 256, 512)); + } } else { robotsTableFile.getParentFile().mkdirs(); robotsTable = new kelondroMap(new kelondroDyn(robotsTableFile, 32000, 256, 512)); diff --git a/source/de/anomic/plasma/plasmaWordIndexAssortment.java b/source/de/anomic/plasma/plasmaWordIndexAssortment.java index c74202593..bbff454c4 100644 --- a/source/de/anomic/plasma/plasmaWordIndexAssortment.java +++ b/source/de/anomic/plasma/plasmaWordIndexAssortment.java @@ -113,7 +113,9 @@ public final class plasmaWordIndexAssortment { if (log != null) log.logConfig("Opened Assortment Database, " + assortments.size() + " entries, width " + assortmentLength + ", " + bufferkb + "kb buffer"); return; } catch (IOException e){ - serverLog.logSevere("PLASMA", "unable to open assortment database, creating new: " + e.getMessage(), e); + serverLog.logSevere("PLASMA", "unable to open assortment database " + assortmentLength + ", creating new: " + e.getMessage(), e); + } catch (kelondroException e) { + serverLog.logSevere("PLASMA", "assortment database " + assortmentLength + " corupted, creating new: " + e.getMessage(), e); } assortmentFile.delete(); // make space for new one } diff --git a/source/de/anomic/yacy/yacyNewsQueue.java b/source/de/anomic/yacy/yacyNewsQueue.java index 7c40ba626..37d550e29 100644 --- a/source/de/anomic/yacy/yacyNewsQueue.java +++ b/source/de/anomic/yacy/yacyNewsQueue.java @@ -50,6 +50,7 @@ import java.util.Date; import de.anomic.htmlFilter.htmlFilterContentScraper; import de.anomic.http.httpHeader; import de.anomic.kelondro.kelondroStack; +import de.anomic.kelondro.kelondroException; import de.anomic.server.serverCodings; import de.anomic.yacy.yacySeedDB; @@ -64,7 +65,12 @@ public class yacyNewsQueue { this.newsDB = newsDB; if (path.exists()) { - queueStack = new kelondroStack(path, 0); + try { + queueStack = new kelondroStack(path, 0); + } catch (kelondroException e) { + path.delete(); + queueStack = createStack(path); + } } else queueStack = createStack(path); }