Marown/yacy_search_server
1
0
Fork 0
mirror of https://github.com/yacy/yacy_search_server.git synced 2024-09-19 00:01:41 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity

replaced kelondro storage core; much less objects in kelondro cache now; less IO from DB

Browse Source 
git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@724 6c8d7289-2bf4-0310-a012-ef5d649a1542
This commit is contained in:
orbiter 2005-09-14 10:10:49 +00:00
parent dff96601fe
commit 2148c0cf49
14 changed files with 991 additions and 272 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
build.properties
View File

@ -3,7 +3,7 @@ javacSource=1.4
javacTarget=1.4
# Release Configuration
releaseVersion=0.402
releaseVersion=0.403
releaseFile=yacy_dev_v${releaseVersion}_${DSTAMP}_${releaseNr}.tar.gz
#releaseFile=yacy_v${releaseVersion}_${DSTAMP}_${releaseNr}.tar.gz
releaseDir=yacy_dev_v${releaseVersion}_${DSTAMP}_${releaseNr}

7
source/de/anomic/kelondro/kelondroAbstractRA.java
View File

@ -66,7 +66,12 @@ abstract class kelondroAbstractRA implements kelondroRA {
abstract public void seek(long pos) throws IOException;
abstract public void close() throws IOException;
// derivated methods:
// derived methods:
public void readFully(byte[] b, int off, int len) throws IOException {
int r = read(b, off, len);
if (r < len) readFully(b, off + r, len - r);
}
public byte readByte() throws IOException {
int ch = this.read();
if (ch < 0) throw new IOException();

30
source/de/anomic/kelondro/kelondroArray.java
View File

@ -67,16 +67,22 @@ public class kelondroArray extends kelondroRecords {
}
public synchronized byte[][] set(int index, byte[][] row) throws IOException {
if (row.length != columns()) throw new IllegalArgumentException("set: wrong row length " + row.length + "; must be " + columns());
if (row.length != columns())
throw new IllegalArgumentException("set: wrong row length " + row.length + "; must be " + columns());
// make room for element
if (size() <= index) while (newNode() <= index) {}
Node n;
while (size() <= index) {
n = newNode();
n.commit(CP_NONE);
}
// get the node at position index
Node n = getNode(new Handle(index));
n = getNode(new Handle(index));
// write the row
byte[][] before = n.setValues(row);
n.commit(CP_NONE);
return before;
}
@ -103,7 +109,7 @@ public class kelondroArray extends kelondroRecords {
for (int i = 0; i < size(); i++) {
System.out.print("row " + i + ": ");
row = get(i);
for (int j = 0; j < columns(); j++) System.out.print(new String(row[j]) + ", ");
for (int j = 0; j < columns(); j++) System.out.print(((row[j] == null) ? "NULL" : new String(row[j])) + ", ");
System.out.println();
}
System.out.println("EndOfTable");
@ -123,9 +129,7 @@ public class kelondroArray extends kelondroRecords {
// create <filename> <valuelen>
File f = new File(args[1]);
if (f.exists()) f.delete();
int[] lens = new int[1];
lens[0] = Integer.parseInt(args[2]);
kelondroArray fm = new kelondroArray(f, lens, 2);
kelondroArray fm = new kelondroArray(f, new int[]{Integer.parseInt(args[2])}, 2);
fm.close();
} else
if ((args.length == 2) && (args[0].equals("-v"))) {
@ -149,7 +153,17 @@ public class kelondroArray extends kelondroRecords {
byte[][] row = new byte[][]{args[3].getBytes()};
fm.set(Integer.parseInt(args[2]), row);
fm.close();
} else {
} else
if ((args.length == 1) && (args[0].equals("-test"))) {
File testfile = new File("test.array");
if (testfile.exists()) testfile.delete();
kelondroArray fm = new kelondroArray(testfile, new int[]{30, 50}, 9);
for (int i = 0; i < 100; i++) {
fm.set(i, new byte[][]{("name" + i).getBytes(), ("value" + i).getBytes()});
}
fm.close();
} else
{
System.err.println("usage: kelondroArray -c|-v|-s|-g [file]|[index [value]] <db-file>");
System.err.println("( create, view, set, get)");
System.exit(0);

151
source/de/anomic/kelondro/kelondroBufferedRA.java
View File

@ -3,8 +3,8 @@
// part of The Kelondro Database
// (C) by Michael Peter Christen; mc@anomic.de
// first published on http://www.anomic.de
// Frankfurt, Germany, 2004
// last major change: 06.10.2004
// Frankfurt, Germany, 2005
// last major change: 13.09.2005
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
@ -43,124 +43,105 @@ package de.anomic.kelondro;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.util.HashMap;
import java.util.Iterator;
public class kelondroBufferedRA extends kelondroAbstractRA implements kelondroRA {
private static final int bufferSizeExp = 10;
private static final int bufferSize = 1 << bufferSizeExp;
private static final int bufferOffsetFilter = bufferSize - 1;
protected kelondroRA ra;
protected kelondroMScoreCluster bufferScore;
protected HashMap bufferMemory;
private int bufferMaxElements;
private int bufferElementSize;
protected byte[] buffer;
protected int bufferPage;
protected boolean bufferWritten;
private long seekpos;
public kelondroBufferedRA(kelondroRA ra, int buffersize, int elementsize) throws FileNotFoundException {
public kelondroBufferedRA(kelondroRA ra) throws FileNotFoundException {
this.ra = ra;
this.name = ra.name();
this.bufferMemory = new HashMap();
this.bufferScore = new kelondroMScoreCluster();
this.bufferElementSize = elementsize;
this.bufferMaxElements = (int) (buffersize / bufferElementSize);
this.buffer = new byte[bufferSize];
this.seekpos = 0;
this.bufferPage = -1;
this.bufferWritten = true;
}
private int bufferElementNumber(long address) {
return (int) address / bufferElementSize;
private void readBuffer(int newPageNr) throws IOException {
if (newPageNr == bufferPage) return;
bufferPage = newPageNr;
ra.seek(bufferPage << bufferSizeExp);
ra.readFully(buffer, 0, bufferSize);
bufferWritten = true;
}
private int bufferElementOffset(long address) {
return (int) address % bufferElementSize;
private void writeBuffer() throws IOException {
if ((bufferWritten) || (bufferPage < 0)) return;
ra.seek(bufferPage << bufferSizeExp);
ra.write(buffer, 0, bufferSize);
bufferWritten = true;
}
private byte[] readBuffer(int bufferNr) throws IOException {
Integer bufferNrI = new Integer(bufferNr);
byte[] buffer = (byte[]) bufferMemory.get(bufferNrI);
if (buffer == null) {
if (bufferMemory.size() >= bufferMaxElements) {
// delete elements in buffer if buffer too big
Iterator it = bufferScore.scores(true);
Integer element = (Integer) it.next();
writeBuffer((byte[]) bufferMemory.get(element), element.intValue());
bufferMemory.remove(element);
int age = bufferScore.deleteScore(element);
de.anomic.server.logging.serverLog.logFine("CACHE: " + name, "GC; age=" + ((((int) (0xFFFFFFFFL & System.currentTimeMillis())) - age) / 1000));
}
// add new element
buffer = new byte[bufferElementSize];
//System.out.println("buffernr=" + bufferNr + ", elSize=" + bufferElementSize);
ra.seek(bufferNr * bufferElementSize);
ra.read(buffer, 0, bufferElementSize);
bufferMemory.put(bufferNrI, buffer);
private void updateToBuffer(int newPageNr) throws IOException {
if (newPageNr != bufferPage) {
writeBuffer();
readBuffer(newPageNr);
}
bufferScore.setScore(bufferNrI, (int) (0xFFFFFFFFL & System.currentTimeMillis()));
return buffer;
}
private void writeBuffer(byte[] buffer, int bufferNr) throws IOException {
if (buffer == null) return;
Integer bufferNrI = new Integer(bufferNr);
ra.seek(bufferNr * bufferElementSize);
ra.write(buffer, 0, bufferElementSize);
bufferScore.setScore(bufferNrI, (int) (0xFFFFFFFFL & System.currentTimeMillis()));
}
// pseudo-native method read
public int read() throws IOException {
int bn = bufferElementNumber(seekpos);
int offset = bufferElementOffset(seekpos);
int bn = (int) seekpos >> bufferSizeExp; // buffer page number
int offset = (int) seekpos & bufferOffsetFilter; // buffer page offset
seekpos++;
return 0xFF & readBuffer(bn)[offset];
updateToBuffer(bn);
return 0xFF & buffer[offset];
}
// pseudo-native method write
public void write(int b) throws IOException {
int bn = bufferElementNumber(seekpos);
int offset = bufferElementOffset(seekpos);
byte[] buffer = readBuffer(bn);
int bn = (int) seekpos >> bufferSizeExp; // buffer page number
int offset = (int) seekpos & bufferOffsetFilter; // buffer page offset
seekpos++;
updateToBuffer(bn);
buffer[offset] = (byte) b;
//writeBuffer(buffer, bn);
bufferWritten = false;
}
public int read(byte[] b, int off, int len) throws IOException {
int bn1 = bufferElementNumber(seekpos);
int bn2 = bufferElementNumber(seekpos + len - 1);
int offset = bufferElementOffset(seekpos);
byte[] buffer = readBuffer(bn1);
int bn1 = (int) seekpos >> bufferSizeExp; // buffer page number, first position
int bn2 = (int) (seekpos + len - 1) >> bufferSizeExp; // buffer page number, last position
int offset = (int) seekpos & bufferOffsetFilter; // buffer page offset
updateToBuffer(bn1);
if (bn1 == bn2) {
// simple case
//System.out.println("C1: bn1=" + bn1 + ", offset=" + offset + ", off=" + off + ", len=" + len);
System.arraycopy(buffer, offset, b, off, len);
seekpos += len;
return len;
} else {
// do recursively
int thislen = bufferElementSize - offset;
//System.out.println("C2: bn1=" + bn1 + ", bn2=" + bn2 +", offset=" + offset + ", off=" + off + ", len=" + len + ", thislen=" + thislen);
int thislen = bufferSize - offset;
System.arraycopy(buffer, offset, b, off, thislen);
seekpos += thislen;
return thislen + read(b, thislen, len - thislen);
return thislen + read(b, off + thislen, len - thislen);
}
}
public void write(byte[] b, int off, int len) throws IOException {
int bn1 = bufferElementNumber(seekpos);
int bn2 = bufferElementNumber(seekpos + len - 1);
int offset = bufferElementOffset(seekpos);
byte[] buffer = readBuffer(bn1);
int bn1 = (int) seekpos >> bufferSizeExp; // buffer page number, first position
int bn2 = (int) (seekpos + len - 1) >> bufferSizeExp; // buffer page number, last position
int offset = (int) seekpos & bufferOffsetFilter; // buffer page offset
updateToBuffer(bn1);
if (bn1 == bn2) {
// simple case
System.arraycopy(b, off, buffer, offset, len);
bufferWritten = false;
seekpos += len;
//writeBuffer(buffer, bn1);
} else {
// do recursively
int thislen = bufferElementSize - offset;
int thislen = bufferSize - offset;
System.arraycopy(b, off, buffer, offset, thislen);
bufferWritten = false;
seekpos += thislen;
//writeBuffer(buffer, bn1);
write(b, thislen, len - thislen);
write(b, off + thislen, len - thislen);
}
}
@ -169,16 +150,30 @@ public class kelondroBufferedRA extends kelondroAbstractRA implements kelondroRA
}
public void close() throws IOException {
// write all unwritten buffers
Iterator it = bufferScore.scores(true);
while (it.hasNext()) {
Integer element = (Integer) it.next();
writeBuffer((byte[]) bufferMemory.get(element), element.intValue());
bufferMemory.remove(element);
}
// write unwritten buffer
if (buffer == null) return;
writeBuffer();
ra.close();
bufferScore = null;
bufferMemory = null;
buffer = null;
}
public void finalize() {
try {
close();
} catch (IOException e) {}
}
public static void main(String[] args) {
try {
kelondroRA file = new kelondroBufferedRA(new kelondroFileRA("testx"));
file.seek(bufferSize - 2);
byte[] b = new byte[]{65, 66, 77, 88};
file.write(b);
file.seek(bufferSize * 2 - 30);
for (int i = 65; i < 150; i++) file.write(i);
file.close();
} catch (IOException e) {
e.printStackTrace();
}
}
}

191
source/de/anomic/kelondro/kelondroCachedRA.java Normal file
View File

@ -0,0 +1,191 @@
// kelondroCachedRA.java
// -----------------------
// part of The Kelondro Database
// (C) by Michael Peter Christen; mc@anomic.de
// first published on http://www.anomic.de
// Frankfurt, Germany, 2004, 2005
// last major change: 06.10.2004
// this file was previously named kelondroBufferedRA
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation; either version 2 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
//
// Using this software in any meaning (reading, learning, copying, compiling,
// running) means that you agree that the Author(s) is (are) not responsible
// for cost, loss of data or any harm that may be caused directly or indirectly
// by usage of this softare or this documentation. The usage of this software
// is on your own risk. The installation and usage (starting/running) of this
// software may allow other people or application to access your computer and
// any attached devices and is highly dependent on the configuration of the
// software which must be done by the user of the software; the author(s) is
// (are) also not responsible for proper configuration and usage of the
// software, even if provoked by documentation provided together with
// the software.
//
// Any changes to this file according to the GPL as documented in the file
// gpl.txt aside this file in the shipment you received can be done to the
// lines that follows this copyright notice here, but changes must not be
// done inside the copyright notive above. A re-distribution must contain
// the intact and unchanged copyright notice.
// Contributions and changes to the program code must be marked as such.
package de.anomic.kelondro;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.util.HashMap;
import java.util.Iterator;
public class kelondroCachedRA extends kelondroAbstractRA implements kelondroRA {
protected kelondroRA ra;
protected kelondroMScoreCluster cacheScore;
protected HashMap cacheMemory;
private int cacheMaxElements;
private int cacheElementSize;
private long seekpos;
public kelondroCachedRA(kelondroRA ra, int cachesize, int elementsize) throws FileNotFoundException {
this.ra = ra;
this.name = ra.name();
this.cacheMemory = new HashMap();
this.cacheScore = new kelondroMScoreCluster();
this.cacheElementSize = elementsize;
this.cacheMaxElements = (int) (cachesize / cacheElementSize);
this.seekpos = 0;
}
private int cacheElementNumber(long address) {
return (int) address / cacheElementSize;
}
private int cacheElementOffset(long address) {
return (int) address % cacheElementSize;
}
private byte[] readCache(int cacheNr) throws IOException {
Integer cacheNrI = new Integer(cacheNr);
byte[] cache = (byte[]) cacheMemory.get(cacheNrI);
if (cache == null) {
if (cacheMemory.size() >= cacheMaxElements) {
// delete elements in buffer if buffer too big
Iterator it = cacheScore.scores(true);
Integer element = (Integer) it.next();
writeCache((byte[]) cacheMemory.get(element), element.intValue());
cacheMemory.remove(element);
int age = cacheScore.deleteScore(element);
de.anomic.server.logging.serverLog.logFine("CACHE: " + name, "GC; age=" + ((((int) (0xFFFFFFFFL & System.currentTimeMillis())) - age) / 1000));
}
// add new element
cache = new byte[cacheElementSize];
//System.out.println("buffernr=" + bufferNr + ", elSize=" + bufferElementSize);
ra.seek(cacheNr * cacheElementSize);
ra.read(cache, 0, cacheElementSize);
cacheMemory.put(cacheNrI, cache);
}
cacheScore.setScore(cacheNrI, (int) (0xFFFFFFFFL & System.currentTimeMillis()));
return cache;
}
private void writeCache(byte[] cache, int cacheNr) throws IOException {
if (cache == null) return;
Integer cacheNrI = new Integer(cacheNr);
ra.seek(cacheNr * cacheElementSize);
ra.write(cache, 0, cacheElementSize);
cacheScore.setScore(cacheNrI, (int) (0xFFFFFFFFL & System.currentTimeMillis()));
}
// pseudo-native method read
public int read() throws IOException {
int bn = cacheElementNumber(seekpos);
int offset = cacheElementOffset(seekpos);
seekpos++;
return 0xFF & readCache(bn)[offset];
}
// pseudo-native method write
public void write(int b) throws IOException {
int bn = cacheElementNumber(seekpos);
int offset = cacheElementOffset(seekpos);
byte[] cache = readCache(bn);
seekpos++;
cache[offset] = (byte) b;
//writeBuffer(buffer, bn);
}
public int read(byte[] b, int off, int len) throws IOException {
int bn1 = cacheElementNumber(seekpos);
int bn2 = cacheElementNumber(seekpos + len - 1);
int offset = cacheElementOffset(seekpos);
byte[] buffer = readCache(bn1);
if (bn1 == bn2) {
// simple case
//System.out.println("C1: bn1=" + bn1 + ", offset=" + offset + ", off=" + off + ", len=" + len);
System.arraycopy(buffer, offset, b, off, len);
seekpos += len;
return len;
} else {
// do recursively
int thislen = cacheElementSize - offset;
//System.out.println("C2: bn1=" + bn1 + ", bn2=" + bn2 +", offset=" + offset + ", off=" + off + ", len=" + len + ", thislen=" + thislen);
System.arraycopy(buffer, offset, b, off, thislen);
seekpos += thislen;
return thislen + read(b, off + thislen, len - thislen);
}
}
public void write(byte[] b, int off, int len) throws IOException {
int bn1 = cacheElementNumber(seekpos);
int bn2 = cacheElementNumber(seekpos + len - 1);
int offset = cacheElementOffset(seekpos);
byte[] cache = readCache(bn1);
if (bn1 == bn2) {
// simple case
System.arraycopy(b, off, cache, offset, len);
seekpos += len;
//writeBuffer(buffer, bn1);
} else {
// do recursively
int thislen = cacheElementSize - offset;
System.arraycopy(b, off, cache, offset, thislen);
seekpos += thislen;
//writeBuffer(buffer, bn1);
write(b, off + thislen, len - thislen);
}
}
public void seek(long pos) throws IOException {
seekpos = pos;
}
public void close() throws IOException {
// write all unwritten buffers
if (cacheMemory == null) return;
Iterator it = cacheScore.scores(true);
while (it.hasNext()) {
Integer element = (Integer) it.next();
writeCache((byte[]) cacheMemory.get(element), element.intValue());
cacheMemory.remove(element);
}
ra.close();
cacheScore = null;
cacheMemory = null;
}
public void finalize() {
try {
close();
} catch (IOException e) {}
}
}

17
source/de/anomic/kelondro/kelondroDyn.java
View File

@ -228,7 +228,13 @@ public class kelondroDyn extends kelondroTree {
} else {
byte[] buf = getValueCached(dynKey(key, reccnt));
if (buf == null) return null;
//System.out.println("read: buf.length="+buf.length+",recpos="+recpos+",len="+len);
if (buf.length < reclen) {
byte[] buff = new byte[reclen];
System.arraycopy(buf, 0, buff, 0, buf.length);
buf = buff;
buff = null;
}
//System.out.println("read: buf.length="+buf.length+",recpos="+recpos+",len="+len);
if (len < (reclen - recpos)) {
segment1 = new byte[len];
System.arraycopy(buf, recpos, segment1, 0, len);
@ -269,7 +275,14 @@ public class kelondroDyn extends kelondroTree {
}
} else {
buf = getValueCached(dynKey(key, reccnt));
if (buf == null) buf = new byte[reclen];
if (buf == null) {
buf = new byte[reclen];
} else if (buf.length < reclen) {
byte[] buff = new byte[reclen];
System.arraycopy(buf, 0, buff, 0, buf.length);
buf = buff;
buff = null;
}
//System.out.println("write: b.length="+b.length+",off="+off+",len="+(reclen-recpos));
if (len < (reclen - recpos))
System.arraycopy(b, off, buf, recpos, len);

32
source/de/anomic/kelondro/kelondroFileRA.java
View File

@ -45,38 +45,55 @@ import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.RandomAccessFile;
import java.io.FileDescriptor;
import java.io.SyncFailedException;
import java.util.Map;
import java.util.Properties;
public class kelondroFileRA extends kelondroAbstractRA implements kelondroRA {
protected RandomAccessFile RAFile;
protected FileDescriptor RADescriptor;
public kelondroFileRA(String file) throws FileNotFoundException {
public kelondroFileRA(String file) throws IOException, FileNotFoundException {
this(new File(file));
}
public kelondroFileRA(File file) throws FileNotFoundException {
public kelondroFileRA(File file) throws IOException, FileNotFoundException {
this.name = file.getName();
RAFile = new RandomAccessFile(file, "rw");
RADescriptor = RAFile.getFD();
}
private void sync() throws IOException {
try {
RADescriptor.sync();
//try {Thread.currentThread().sleep(8);} catch (InterruptedException e) {return;}
} catch (SyncFailedException e) {
throw new IOException(e.getMessage());
}
}
// pseudo-native method read
public int read() throws IOException {
sync();
return RAFile.read();
}
// pseudo-native method write
public void write(int b) throws IOException {
RAFile.write(b);
RAFile.write(b);
}
public int read(byte[] b, int off, int len) throws IOException {
return RAFile.read(b, off, len);
sync();
RAFile.read(b, off, len);
return len;
}
public void write(byte[] b, int off, int len) throws IOException {
RAFile.write(b, off, len);
// write to file
RAFile.write(b, off, len);
}
public void seek(long pos) throws IOException {
@ -84,8 +101,9 @@ public class kelondroFileRA extends kelondroAbstractRA implements kelondroRA {
}
public void close() throws IOException {
RAFile.close();
RAFile = null;
sync();
RAFile.close();
RAFile = null;
}
protected void finalize() throws Throwable {

1
source/de/anomic/kelondro/kelondroRA.java
View File

@ -64,6 +64,7 @@ public interface kelondroRA {
public void write(int b) throws IOException;
public int read(byte[] b, int off, int len) throws IOException;
public void readFully(byte[] b, int off, int len) throws IOException;
public void write(byte[] b, int off, int len) throws IOException;
public void seek(long pos) throws IOException;

479
source/de/anomic/kelondro/kelondroRecords.java
View File

@ -73,14 +73,22 @@ import java.io.IOException;
import java.util.HashMap;
import java.util.NoSuchElementException;
import java.util.StringTokenizer;
import java.util.Map;
import java.util.Iterator;
public class kelondroRecords {
// constants
private static final int NUL = Integer.MIN_VALUE; // the meta value for the kelondroRecords' NUL abstraction
public static final long memBlock = 5000000; // do not fill cache further if the amount of available memory is less that this
public static final long memBlock = 500000; // do not fill cache further if the amount of available memory is less that this
public static final long memKcolb = 10000000; // if the amount of available memory is greater than this, do not use cache size to block, simply use memory
// caching flags
protected static final int CP_NONE = -1; // cache priority none; entry shall not be cached
protected static final int CP_LOW = 0; // cache priority low; entry may be cached
protected static final int CP_MEDIUM = 1; // cache priority medium; entry shall be cached
protected static final int CP_HIGH = 2; // cache priority high; entry must be cached
// static seek pointers
private static long POS_MAGIC = 0; // 1 byte, byte: file type magic
private static long POS_BUSY = POS_MAGIC + 1; // 1 byte, byte: marker for synchronization
@ -108,6 +116,8 @@ public class kelondroRecords {
protected String filename; // the database's file name
protected kelondroRA entryFile; // the database file
private int overhead; // OHBYTEC + 4 * OHHANDLEC = size of additional control bytes
private int headchunksize;// overheadsize + key element column size
private int tailchunksize;// sum(all: COLWIDTHS) minus the size of the key element colum
private int recordsize; // (overhead + sum(all: COLWIDTHS)) = the overall size of a record
// dynamic run-time seek pointers
@ -146,7 +156,7 @@ public class kelondroRecords {
if (file.exists()) throw new IOException("kelondroRecords: file " + file + " already exist");
this.filename = file.getCanonicalPath();
kelondroRA raf = new kelondroFileRA(this.filename);
//kelondroRA raf = new kelondroBufferedRA(new kelondroFileRA(this.filename), 5000000, 1000);
//kelondroRA raf = new kelondroBufferedRA(new kelondroFileRA(this.filename));
//kelondroRA raf = new kelondroNIOFileRA(this.filename, false, 10000);
init(raf, ohbytec, ohhandlec, columns, FHandles, txtProps, txtPropWidth);
this.XcacheSize = (int) (buffersize / ((long) (overhead + columns[0])));
@ -186,6 +196,8 @@ public class kelondroRecords {
this.overhead = ohbytec + 4 * ohhandlec;
this.recordsize = this.overhead;
for (int i = 0; i < columns.length; i++) this.recordsize += columns[i];
this.headchunksize = overhead + columns[0];
this.tailchunksize = this.recordsize - this.headchunksize;
// store dynamic run-time seek pointers
POS_HANDLES = POS_COLWIDTHS + columns.length * 4;
@ -256,7 +268,8 @@ public class kelondroRecords {
this.filename = file.getCanonicalPath();
kelondroRA raf = new kelondroFileRA(this.filename);
//kelondroRA raf = new kelondroBufferedRA(new kelondroFileRA(this.filename), 5000000, 1000);
//kelondroRA raf = new kelondroBufferedRA(new kelondroFileRA(this.filename));
//kelondroRA raf = new kelondroCachedRA(new kelondroFileRA(this.filename), 5000000, 1000);
//kelondroRA raf = new kelondroNIOFileRA(this.filename, (file.length() < 4000000), 10000);
init(raf);
this.XcacheSize = (int) (buffersize / ((long) (overhead + COLWIDTHS[0])));
@ -286,7 +299,8 @@ public class kelondroRecords {
this.XcacheStartup = System.currentTimeMillis();
}
private void init(kelondroRA ra) throws IOException{
private void init(kelondroRA ra) throws IOException {
// assign values that are only present at run-time
this.entryFile = ra;
@ -303,6 +317,8 @@ public class kelondroRecords {
entryFile.seek(POS_TXTPROPC); this.TXTPROPS = new byte[entryFile.readInt()][];
entryFile.seek(POS_TXTPROPW); this.TXTPROPW = entryFile.readInt();
if (COLWIDTHS.length == 0) throw new kelondroException(filename, "init: zero columns; strong failure");
// calculate dynamic run-time seek pointers
POS_HANDLES = POS_COLWIDTHS + COLWIDTHS.length * 4;
POS_TXTPROPS = POS_HANDLES + HANDLES.length * 4;
@ -325,38 +341,23 @@ public class kelondroRecords {
// assign remaining values that are only present at run-time
this.overhead = OHBYTEC + 4 * OHHANDLEC;
this.recordsize = this.overhead; for (int i = 0; i < COLWIDTHS.length; i++) this.recordsize += COLWIDTHS[i];
this.recordsize = this.overhead;
for (int i = 0; i < COLWIDTHS.length; i++) this.recordsize += COLWIDTHS[i];
this.headchunksize = this.overhead + COLWIDTHS[0];
this.tailchunksize = this.recordsize - this.headchunksize;
}
protected int newNode() {
Node n = new Node();
return USEDC + FREEC;
protected Node newNode() throws IOException {
return new Node();
}
protected Node newNode(byte[][] v) {
return new Node(v);
}
protected Node getNode(Handle handle) {
protected Node getNode(Handle handle) throws IOException {
return getNode(handle, null, 0);
}
protected Node getNode(Handle handle, Node parentNode, int referenceInParent) {
if (XcacheSize == 0) return new Node(handle, parentNode, referenceInParent);
synchronized (XcacheHeaders) {
Node n = (Node) XcacheHeaders.get(handle);
if (n == null) {
n = new Node(handle, parentNode, referenceInParent);
checkCacheSpace();
n.updateNodeCache();
return n;
} else {
//System.out.println("read from cache " + n.toString());
XcacheScore.setScore(handle, (int) ((System.currentTimeMillis() - XcacheStartup) / 1000));
return n;
}
}
protected Node getNode(Handle handle, Node parentNode, int referenceInParent) throws IOException {
return new Node(handle, parentNode, referenceInParent);
}
protected void deleteNode(Handle handle) throws IOException {
@ -397,6 +398,401 @@ public class kelondroRecords {
}
}
public class Node {
// an Node holds all information of one row of data. This includes the key to the entry
// which is stored as entry element at position 0
// an Node object can be created in two ways:
// 1. instantiation with an index number. After creation the Object does not hold any
// value information until such is retrieved using the getValue() method
// 2. instantiation with a value array. the values are not directly written into the
// file. Expanding the tree structure is then done using the save() method. at any
// time it is possible to verify the save state using the saved() predicate.
// Therefore an entry object has three modes:
// a: holding an index information only (saved() = true)
// b: holding value information only (saved() = false)
// c: holding index and value information at the same time (saved() = true)
// which can be the result of one of the two processes as follow:
// (i) created with index and after using the getValue() method, or
// (ii) created with values and after calling the save() method
// the method will therefore throw an IllegalStateException when the following
// process step is performed:
// - create the Node with index and call then the save() method
// this case can be decided with
// ((index != NUL) && (values == null))
// The save() method represents the insert function for the tree. Balancing functions
// are applied automatically. While balancing, the Node does never change its index key,
// but its parent/child keys.
//private byte[] ohBytes = null; // the overhead bytes, OHBYTEC values
//private Handle[] ohHandle= null; // the overhead handles, OHHANDLEC values
//private byte[][] values = null; // an array of byte[] nodes is the value vector
private Handle handle = null; // index of the entry, by default NUL means undefined
private byte[] headChunk = null; // contains ohBytes, ohHandles and the key value
private byte[] tailChunk = null; // contains all values except the key value
private boolean headChanged = false;
private boolean tailChanged = false;
private Node() throws IOException {
// create a new empty node and reserve empty space in file for it
// use this method only if you want to extend the file with new entries
// without the need to have content in it.
this.handle = new Handle();
// create empty chunks
this.headChunk = new byte[headchunksize];
this.tailChunk = new byte[tailchunksize];
for (int i = 0; i < headchunksize; i++) this.headChunk[i] = 0;
for (int i = 0; i < tailchunksize; i++) this.tailChunk[i] = 0;
this.headChanged = true;
this.tailChanged = true;
}
private Node(Handle handle) throws IOException {
// this creates an entry with an pre-reserved entry position
// values can be written using the setValues() method
// but we expect that values are already there in the file ready to be read which we do not here
if (handle == null) throw new IllegalArgumentException("INTERNAL ERROR: node handle is null.");
if (handle.index >= USEDC + FREEC) throw new kelondroException(filename, "INTERNAL ERROR: node handle index exceeds size.");
// use given handle
this.handle = new Handle(handle.index);
// init the content
initContent();
}
private Node(Handle handle, Node parentNode, int referenceInParent) throws IOException {
// this creates an entry with an pre-reserved entry position
// values can be written using the setValues() method
// but we expect that values are already there in the file ready to be read which we do not here
if (handle == null) throw new IllegalArgumentException("INTERNAL ERROR: node handle is null.");
// the parentNode can be given if an auto-fix in the following case is wanted
if (handle.index >= USEDC + FREEC) {
if (parentNode == null) {
throw new kelondroException(filename, "INTERNAL ERROR, Node/init: node handle index exceeds size. No auto-fix node was submitted. This is a serious failure.");
} else {
try {
Handle[] handles = parentNode.getOHHandles();
handles[referenceInParent] = null;
parentNode.setOHHandles(handles);
parentNode.commit(CP_NONE);
throw new kelondroException(filename, "INTERNAL ERROR, Node/init: node handle index " + handle.index + " exceeds size. The bad node has been auto-fixed");
} catch (IOException ee) {
throw new kelondroException(filename, "INTERNAL ERROR, Node/init: node handle index " + handle.index + " exceeds size. It was tried to fix the bad node, but failed with an IOException: " + ee.getMessage());
}
}
}
// use given handle
this.handle = new Handle(handle.index);
// init the content
initContent();
}
private void initContent() throws IOException {
// create chunks; read them from file or cache
this.tailChunk = null;
if (XcacheSize == 0) {
// read overhead and key
//System.out.println("**NO CACHE for " + this.handle.index + "**");
this.headChunk = new byte[headchunksize];
synchronized (entryFile) {
entryFile.seek(seekpos(this.handle));
entryFile.readFully(this.headChunk, 0, this.headChunk.length);
}
this.headChanged = false;
} else synchronized(XcacheHeaders) {
byte[] cacheEntry = (byte[]) XcacheHeaders.get(this.handle);
if (cacheEntry == null) {
// read overhead and key
//System.out.println("**CACHE miss for " + this.handle.index + "**");
this.headChunk = new byte[headchunksize];
//this.tailChunk = new byte[tailchunksize];
synchronized (entryFile) {
entryFile.seek(seekpos(this.handle));
entryFile.readFully(this.headChunk, 0, this.headChunk.length);
//entryFile.read(this.tailChunk, 0, this.tailChunk.length);
}
this.headChanged = true; // provoke a cache store
checkCacheSpace();
updateNodeCache();
} else {
//System.out.println("**CACHE HIT for " + this.handle.index + "**");
// copy cache entry
this.headChunk = new byte[headchunksize];
System.arraycopy(cacheEntry, 0, this.headChunk, 0, headchunksize);
// update cache scores to announce this cache hit
XcacheScore.setScore(this.handle, (int) ((System.currentTimeMillis() - XcacheStartup) / 1000));
this.headChanged = false;
}
}
}
private void setValue(byte[] value, int valuewidth, byte[] targetarray, int targetoffset) {
if (value == null) {
while (valuewidth-- > 0) targetarray[targetoffset + valuewidth] = 0;
} else {
System.arraycopy(value, 0, targetarray, targetoffset, Math.min(value.length, valuewidth)); // error?
if (value.length < valuewidth)
while (valuewidth-- > value.length) targetarray[targetoffset + valuewidth] = 0;
}
}
protected Handle handle() {
// if this entry has an index, return it
if (this.handle.index == NUL) throw new kelondroException(filename, "the entry has no index assigned");
return new Handle(this.handle.index);
}
protected void setOHBytes(byte[] b) throws IOException {
if (b == null) throw new IllegalArgumentException("setOHByte: setting null value does not make any sense");
if (b.length != OHBYTEC) throw new IllegalArgumentException("setOHByte: wrong array size");
if (this.handle.index == NUL) throw new kelondroException(filename, "setOHByte: no handle assigned");
System.arraycopy(b, 0, this.headChunk, 0, b.length);
this.headChanged = true;
}
protected void setOHHandles(Handle[] handles) throws IOException {
if (handles == null) throw new IllegalArgumentException("setOHint: setting null value does not make any sense");
if (handles.length != OHHANDLEC) throw new IllegalArgumentException("setOHHandle: wrong array size");
if (this.handle.index == NUL) throw new kelondroException(filename, "setOHHandle: no handle assigned");
int offset = OHBYTEC;
for (int i = 0; i < handles.length; i++) {
if (handles[i] == null) {
NUL2bytes(this.headChunk, offset);
} else {
if (handles[i].index > USEDC + FREEC) throw new kelondroException(filename, "INTERNAL ERROR, setOHHandles: handle " + i + " exceeds file size (" + handles[i].index + " > " + (USEDC + FREEC) + ")");
int2bytes(handles[i].index, this.headChunk, offset);
}
offset += 4;
}
this.headChanged = true;
}
protected byte[] getOHBytes() throws IOException {
if (this.handle.index == NUL) throw new kelondroException(filename, "Cannot load OH values");
byte[] b = new byte[OHBYTEC];
System.arraycopy(this.headChunk, 0, b, 0, OHBYTEC);
return b;
}
protected Handle[] getOHHandles() throws IOException {
if (this.handle.index == NUL) throw new kelondroException(filename, "Cannot load OH values");
Handle[] handles = new Handle[OHHANDLEC];
int offset = OHBYTEC;
int i;
for (int j = 0; j < handles.length; j++) {
i = bytes2int(this.headChunk, offset);
handles[j] = (i == NUL) ? null : new Handle(i);
offset += 4;
}
return handles;
}
public byte[][] setValues(byte[][] row) throws IOException {
// if the index is defined, then write values directly to the file, else only to the object
byte[][] result = getValues(); // previous value (this loads the values if not already happened)
// set values
if (this.handle.index != NUL) {
setValue(row[0], COLWIDTHS[0], headChunk, overhead);
int offset = 0;
for (int i = 1; i < row.length; i++) {
setValue(row[i], COLWIDTHS[i], tailChunk, offset);
offset +=COLWIDTHS[i];
}
}
this.headChanged = true;
this.tailChanged = true;
return result; // return previous value
}
public byte[] getKey() throws IOException {
// read key
return trimCopy(headChunk, overhead, COLWIDTHS[0]);
}
public byte[][] getValues() throws IOException {
if (this.tailChunk == null) {
// load all values from the database file
this.tailChunk = new byte[tailchunksize];
// read values
synchronized (entryFile) {
entryFile.seek(seekpos(this.handle) + headchunksize);
entryFile.read(this.tailChunk, 0, this.tailChunk.length);
}
}
// create return value
byte[][] values = new byte[COLWIDTHS.length][];
// read key
values[0] = trimCopy(headChunk, overhead, COLWIDTHS[0]);
// read remaining values
int offset = 0;
for (int i = 1; i < COLWIDTHS.length; i++) {
values[i] = trimCopy(tailChunk, offset, COLWIDTHS[i]);
offset += COLWIDTHS[i];
}
return values;
}
public synchronized void commit(int cachePriority) throws IOException {
// this must be called after all write operations to the node are finished
// place the data to the file
if (this.headChunk == null) {
// there is nothing to save
throw new kelondroException(filename, "no values to save (header missing)");
}
/*
if (this.tailChunk == null) {
// there is nothing to save
throw new kelondroException(filename, "no values to save (tail missing)");
}
*/
// save head
if (this.headChanged) {
synchronized (entryFile) {
entryFile.seek(seekpos(this.handle));
//System.out.print("#write "); printChunk(this.handle, this.headChunk); System.out.println();
entryFile.write(this.headChunk);
}
updateNodeCache();
}
// save tail
if ((this.tailChunk != null) && (this.tailChanged)) synchronized (entryFile) {
entryFile.seek(seekpos(this.handle) + headchunksize);
entryFile.write(this.tailChunk);
}
}
public synchronized void collapse() {
// this must be called after all write and read operations to the node are finished
this.headChunk = null;
this.tailChunk = null;
this.handle = null;
}
/*
public void finalize() {
try {
commit(CP_NONE);
collapse();
} catch (IOException e) {}
}
*/
private byte[] trimCopy(byte[] a, int offset, int length) {
if (length > a.length - offset) length = a.length - offset;
while ((length > 0) && (a[offset + length - 1] == 0)) length--;
if (length == 0) return null;
byte[] b = new byte[length];
System.arraycopy(a, offset, b, 0, length);
return b;
}
public String toString() {
if (this.handle.index == NUL) return "NULL";
String s = Integer.toHexString(this.handle.index);
while (s.length() < 4) s = "0" + s;
try {
byte[] b = getOHBytes();
for (int i = 0; i < b.length; i++) s = s + ":b" + b[i];
Handle[] h = getOHHandles();
for (int i = 0; i < h.length; i++) if (h[i] == null) s = s + ":hNULL"; else s = s + ":h" + h[i].toString();
byte[][] content = getValues();
for (int i = 0; i < content.length; i++) s = s + ":" + ((content[i] == null) ? "NULL" : (new String(content[i])).trim());
} catch (IOException e) {
s = s + ":***LOAD ERROR***:" + e.getMessage();
}
return s;
}
private void updateNodeCache() {
if (this.handle == null) return;
if (this.headChunk == null) return;
if (XcacheSize != 0) {
synchronized (XcacheHeaders) {
// remember size to evaluate a cache size check need
int sizeBefore = XcacheHeaders.size();
//long memBefore = Runtime.getRuntime().freeMemory();
// generate cache entry
byte[] cacheEntry = new byte[headchunksize];
System.arraycopy(headChunk, 0, cacheEntry, 0, headchunksize);
Handle cacheHandle = new Handle(this.handle.index);
// store the cache entry
//XcacheHeaders.remove(cacheHandle);
XcacheHeaders.put(cacheHandle, cacheEntry);
XcacheScore.setScore(cacheHandle, (int) ((System.currentTimeMillis() - XcacheStartup) / 1000));
// delete the cache entry buffer
cacheEntry = null;
cacheHandle = null;
//System.out.println("kelondroRecords cache4" + filename + ": cache record size = " + (memBefore - Runtime.getRuntime().freeMemory()) + " bytes" + ((newentry) ? " new" : ""));
// check cache size
if (XcacheHeaders.size() > sizeBefore) checkCacheSpace();
//System.out.println("kelondroRecords cache4" + filename + ": " + XcacheHeaders.size() + " entries, " + XcacheSize + " allowed.");
//printCache();
}
}
}
}
protected void printCache() {
if (XcacheSize == 0) {
System.out.println("### file report: " + size() + " entries");
for (int i = 0; i < size() + 3; i++) {
// print from file to compare
System.out.print("#F " + i + ": ");
try {synchronized (entryFile) {
entryFile.seek(seekpos(new Handle(i)));
for (int j = 0; j < headchunksize; j++) System.out.print(entryFile.readByte() + ",");
}} catch (IOException e) {}
System.out.println();
}
} else {
System.out.println("### cache report: " + XcacheHeaders.size() + " entries");
Iterator i = XcacheHeaders.entrySet().iterator();
Map.Entry entry;
byte[] b;
while (i.hasNext()) {
entry = (Map.Entry) i.next();
// print from cache
System.out.print("#C ");
printChunk((Handle) entry.getKey(), (byte[]) entry.getValue());
System.out.println();
// print from file to compare
System.out.print("#F " + ((Handle) entry.getKey()).index + ": ");
try {synchronized (entryFile) {
entryFile.seek(seekpos((Handle) entry.getKey()));
for (int j = 0; j < headchunksize; j++) System.out.print(entryFile.readByte() + ",");
}} catch (IOException e) {}
System.out.println();
}
}
System.out.println("### end report");
}
private void printChunk(Handle handle, byte[] chunk) {
System.out.print(handle.index + ": ");
for (int j = 0; j < chunk.length; j++) System.out.print(chunk[j] + ",");
}
/*
public class Node {
// an Node holds all information of one row of data. This includes the key to the entry
// which is stored as entry element at position 0
@ -746,6 +1142,7 @@ public class kelondroRecords {
}
}
}
*/
public synchronized int columns() {
return this.COLWIDTHS.length;
@ -877,7 +1274,29 @@ public class kelondroRecords {
return x;
}
public void print(boolean records) {
public static void NUL2bytes(byte[] b, int offset) {
b[offset ] = (byte) (0XFF & (NUL >> 24));
b[offset + 1] = (byte) (0XFF & (NUL >> 16));
b[offset + 2] = (byte) (0XFF & (NUL >> 8));
b[offset + 3] = (byte) (0XFF & NUL);
}
public static void int2bytes(long i, byte[] b, int offset) {
b[offset ] = (byte) (0XFF & (i >> 24));
b[offset + 1] = (byte) (0XFF & (i >> 16));
b[offset + 2] = (byte) (0XFF & (i >> 8));
b[offset + 3] = (byte) (0XFF & i);
}
public static int bytes2int(byte[] b, int offset) {
return (
((b[offset ] & 0xff) << 24) |
((b[offset + 1] & 0xff) << 16) |
((b[offset + 2] & 0xff) << 8) |
(b[offset + 3] & 0xff));
}
public void print(boolean records) throws IOException {
System.out.println("REPORT FOR FILE '" + this.filename + "':");
System.out.println("--");
System.out.println("CONTROL DATA");
@ -905,6 +1324,8 @@ public class kelondroRecords {
System.out.println(" Overhead : " + this.overhead + " bytes ("+ OHBYTEC + " OH bytes, " + OHHANDLEC + " OH Handles)");
System.out.println(" Recordsize : " + this.recordsize + " bytes");
System.out.println("--");
printCache();
System.out.println("--");
if (!(records)) return;
// print also all records

40
source/de/anomic/kelondro/kelondroStack.java
View File

@ -101,11 +101,11 @@ public class kelondroStack extends kelondroRecords {
public Object next() {
Handle ret = nextHandle;
try {
nextHandle = getNode(nextHandle, null, 0).getOHHandle()[right];
nextHandle = getNode(nextHandle, null, 0).getOHHandles()[right];
return getNode(ret, null, 0);
} catch (IOException e) {
System.err.println("IO error at Counter:next()");
throw new kelondroException(filename, "IO error at Counter:next()");
}
return getNode(ret, null, 0);
}
public void remove() {
throw new UnsupportedOperationException("no remove here..");
@ -119,21 +119,23 @@ public class kelondroStack extends kelondroRecords {
if (getHandle(toor) == null) {
if (getHandle(root) != null) throw new RuntimeException("push: internal organisation of root and toor");
// create node
Node n = newNode(row);
n.save();
n.setOHHandle(new Handle[] {null, null});
n.setValues(row);
Node n = newNode();
n.setValues(row);
n.setOHHandles(new Handle[] {null, null});
n.commit(CP_NONE);
// assign handles
setHandle(root, n.handle());
setHandle(toor, n.handle());
// thats it
} else {
// expand the list at the end
Node n = newNode(row);
n.save();
n.setOHHandle(new Handle[] {getHandle(toor), null});
Node n = newNode();
n.setValues(row);
n.setOHHandles(new Handle[] {getHandle(toor), null});
n.commit(CP_NONE);
Node n1 = getNode(getHandle(toor), null, 0);
n1.setOHHandle(new Handle[] {n1.getOHHandle()[left], n.handle()});
n1.setOHHandles(new Handle[] {n1.getOHHandles()[left], n.handle()});
n1.commit(CP_NONE);
// assign handles
setHandle(toor, n.handle());
// thats it
@ -212,8 +214,8 @@ public class kelondroStack extends kelondroRecords {
private void unlinkNode(Node n) throws IOException {
// join chaines over node
Handle l = n.getOHHandle()[left];
Handle r = n.getOHHandle()[right];
Handle l = n.getOHHandles()[left];
Handle r = n.getOHHandles()[right];
// look left
if (l == null) {
// reached the root on left side
@ -221,7 +223,8 @@ public class kelondroStack extends kelondroRecords {
} else {
// un-link the previous record
Node k = getNode(l, null, 0);
k.setOHHandle(new Handle[] {k.getOHHandle()[left], r});
k.setOHHandles(new Handle[] {k.getOHHandles()[left], r});
k.commit(CP_NONE);
}
// look right
if (r == null) {
@ -230,7 +233,8 @@ public class kelondroStack extends kelondroRecords {
} else {
// un-link the following record
Node k = getNode(r, null, 0);
k.setOHHandle(new Handle[] {l, k.getOHHandle()[right]});
k.setOHHandles(new Handle[] {l, k.getOHHandles()[right]});
k.commit(CP_NONE);
}
}
@ -249,7 +253,7 @@ public class kelondroStack extends kelondroRecords {
Handle h = getHandle(side);
if (h == null) return null;
if (dist >= size()) return null; // that would exceed the stack
while (dist-- > 0) h = getNode(h, null, 0).getOHHandle()[dir]; // track through elements
while (dist-- > 0) h = getNode(h, null, 0).getOHHandles()[dir]; // track through elements
return getNode(h, null, 0);
}
@ -313,7 +317,7 @@ public class kelondroStack extends kelondroRecords {
if (h == null) return "NULL"; else return h.toString();
}
public void print() {
public void print() throws IOException {
super.print(false);
Node n;
try {
@ -322,7 +326,7 @@ public class kelondroStack extends kelondroRecords {
n = (Node) it.next();
//n = getNode(h, null, 0);
System.out.println("> NODE " + hp(n.handle()) +
"; left " + hp(n.getOHHandle()[left]) + ", right " + hp(n.getOHHandle()[right]));
"; left " + hp(n.getOHHandles()[left]) + ", right " + hp(n.getOHHandles()[right]));
System.out.print(" KEY:'" + (new String(n.getValues()[0])).trim() + "'");
for (int j = 1; j < columns(); j++)
System.out.print(", V[" + j + "]:'" + (new String(n.getValues()[j])).trim() + "'");

278
source/de/anomic/kelondro/kelondroTree.java
View File

@ -212,34 +212,38 @@ public class kelondroTree extends kelondroRecords implements Comparator {
if (visitedNodeKeys.containsKey(otherkey)) {
// we have loops in the database.
// to fix this, all affected nodes must be patched
thenode.setOHByte(new byte[] {1, 0});
thenode.setOHHandle(new Handle[] {null, null, null});
thenode.setOHBytes(new byte[] {1, 0});
thenode.setOHHandles(new Handle[] {null, null, null});
Iterator fix = visitedNodeKeys.entrySet().iterator();
Map.Entry entry;
while (fix.hasNext()) {
entry = (Map.Entry) fix.next();
thenode = (Node) entry.getValue();
thenode.setOHByte(new byte[] {1, 0});
thenode.setOHHandle(new Handle[] {null, null, null});
thenode.setOHBytes(new byte[] {1, 0});
thenode.setOHHandles(new Handle[] {null, null, null});
}
thenode.commit(CP_NONE);
//printCache();
throw new kelondroException(filename, "database contains loops; the loop-nodes have been auto-fixed");
}
//System.out.print("Comparing key = '" + new String(key) + "' with '" + otherkey + "':"); // debug
//System.out.println("Comparing key = '" + new String(key) + "' with '" + otherkey + "':"); // debug
c = compare(key, thenode.getKey());
//System.out.println(c); // debug
if (c == 0) {
found = true;
//System.out.println("DEBUG: search for " + new String(key) + " ended with status=" + ((found) ? "found" : "not-found") + ", node=" + ((thenode == null) ? "NULL" : thenode.toString()) + ", parent=" + ((parentnode == null) ? "NULL" : parentnode.toString()));
return;
} else if (c < 0) {
child = -1;
thisHandle = thenode.getOHHandle()[leftchild];
thisHandle = thenode.getOHHandles()[leftchild];
} else {
child = 1;
thisHandle = thenode.getOHHandle()[rightchild];
thisHandle = thenode.getOHHandles()[rightchild];
}
visitedNodeKeys.put(otherkey, thenode);
}
}
//System.out.println("DEBUG: search for " + new String(key) + " ended with status=" + ((found) ? "found" : "not-found") + ", node=" + ((thenode == null) ? "NULL" : thenode.toString()) + ", parent=" + ((parentnode == null) ? "NULL" : parentnode.toString()));
// we reached a node where we must insert the new value
// the parent of this new value can be obtained by getParent()
// all values are set, just return
@ -276,7 +280,7 @@ public class kelondroTree extends kelondroRecords implements Comparator {
public boolean isChild(Node childn, Node parentn, int child) throws IOException {
if (childn == null) throw new IllegalArgumentException("isLeftChild: Node parameter is NULL");
Handle lc = parentn.getOHHandle()[child];
Handle lc = parentn.getOHHandles()[child];
if (lc == null) return false;
return (lc.equals(childn.handle()));
}
@ -305,11 +309,12 @@ public class kelondroTree extends kelondroRecords implements Comparator {
public synchronized byte[][] put(byte[][] newrow) throws IOException {
if (newrow.length != columns()) throw new IllegalArgumentException("put: wrong row length " + newrow.length + "; must be " + columns());
// first try to find the key element in the database
Search searchResult = new Search(newrow[0]);
if (searchResult.found()) {
Search searchResult = new Search(newrow[0]);
if (searchResult.found()) {
// a node with this key exist. simply overwrite the content and return old content
Node e = searchResult.getMatcher();
byte[][] result = e.setValues(newrow);
e.commit(CP_MEDIUM);
searchResult = null;
return result;
} else if (searchResult.isRoot()) {
@ -318,13 +323,14 @@ public class kelondroTree extends kelondroRecords implements Comparator {
if (getHandle(root) != null)
throw new kelondroException(filename, "tried to create root node twice");
// we dont have any Nodes in the file, so start here to create one
Node e = newNode(newrow);
e.save();
Node e = newNode();
e.setValues(newrow);
// write the propetries
e.setOHByte(new byte[] {1, 0}); // {magic, balance}
e.setOHHandle(new Handle[] {null, null, null}); // {parent, leftchild, rightchild}
e.setOHBytes(new byte[] {1, 0}); // {magic, balance}
e.setOHHandles(new Handle[] {null, null, null}); // {parent, leftchild, rightchild}
// do updates
setHandle(root, e.handle());
e.commit(CP_LOW);
setHandle(root, e.handle());
searchResult = null;
return null;
} else {
@ -336,16 +342,16 @@ public class kelondroTree extends kelondroRecords implements Comparator {
// that side, but not if the assigned position is appropriate.
// create new node and assign values
Node theNode = newNode(newrow); theNode.save();
Node parentNode = searchResult.getParent();
Handle[] parentOHHandle;
byte[] parentOHByte;
theNode.setOHByte(new byte[] {1, 0}); // fresh {magic, balance}
theNode.setOHHandle(new Handle[] {parentNode.handle(), null, null}); // {parent, leftchild, rightchild}
Node parentNode = searchResult.getParent();
Node theNode = newNode();
theNode.setValues(newrow);
theNode.setOHBytes(new byte[] {1, 0}); // fresh {magic, balance}
theNode.setOHHandles(new Handle[] {parentNode.handle(), null, null}); // {parent, leftchild, rightchild}
theNode.commit(CP_LOW);
// check consistency and link new node to parent node
parentOHHandle = parentNode.getOHHandle(); // {parent, leftchild, rightchild}
byte[] parentOHByte;
Handle[] parentOHHandle = parentNode.getOHHandles(); // {parent, leftchild, rightchild}
if (searchResult.isLeft()) {
if (parentOHHandle[leftchild] != null) throw new kelondroException(filename, "tried to create leftchild node twice");
parentOHHandle[leftchild] = theNode.handle();
@ -355,7 +361,8 @@ public class kelondroTree extends kelondroRecords implements Comparator {
} else {
throw new kelondroException(filename, "neither left nor right child");
}
parentNode.setOHHandle(parentOHHandle);
parentNode.setOHHandles(parentOHHandle);
parentNode.commit(((parentOHHandle[leftchild] == null) || (parentOHHandle[rightchild] == null)) ? CP_MEDIUM : CP_HIGH);
// now update recursively the node balance of the parentNode
// what do we have:
@ -369,8 +376,8 @@ public class kelondroTree extends kelondroRecords implements Comparator {
while (increasedHight) {
// update balance
parentOHByte = parentNode.getOHByte(); // {magic, balance}
parentOHHandle = parentNode.getOHHandle(); // {parent, leftchild, rightchild}
parentOHByte = parentNode.getOHBytes(); // {magic, balance}
parentOHHandle = parentNode.getOHHandles(); // {parent, leftchild, rightchild}
prevHight = parentOHByte[balance];
if ((parentOHHandle[leftchild] != null) && (parentOHHandle[leftchild].equals(theNode.handle()))) {
//isLeftchild
@ -382,7 +389,8 @@ public class kelondroTree extends kelondroRecords implements Comparator {
path = "R" + path;
}
increasedHight = ((java.lang.Math.abs((int) parentOHByte[balance]) - java.lang.Math.abs((int) prevHight)) > 0);
parentNode.setOHByte(parentOHByte);
parentNode.setOHBytes(parentOHByte);
parentNode.commit(((parentOHHandle[leftchild] == null) || (parentOHHandle[rightchild] == null)) ? CP_MEDIUM : CP_HIGH);
// here we either stop because we had no increased hight,
// or we have a balance greater then 1 or less than -1 and we do rotation
@ -403,16 +411,16 @@ public class kelondroTree extends kelondroRecords implements Comparator {
}
if (path.startsWith("RL")) {
Handle parentHandle = parentNode.handle();
LL_RightRotation(theNode, getNode(theNode.getOHHandle()[leftchild], theNode, leftchild));
LL_RightRotation(theNode, getNode(theNode.getOHHandles()[leftchild], theNode, leftchild));
parentNode = getNode(parentHandle, null, 0); // reload the parent node
RR_LeftRotation(parentNode, getNode(parentNode.getOHHandle()[rightchild], parentNode, rightchild));
RR_LeftRotation(parentNode, getNode(parentNode.getOHHandles()[rightchild], parentNode, rightchild));
break;
}
if (path.startsWith("LR")) {
Handle parentHandle = parentNode.handle();
RR_LeftRotation(theNode, getNode(theNode.getOHHandle()[rightchild], theNode, rightchild));
RR_LeftRotation(theNode, getNode(theNode.getOHHandles()[rightchild], theNode, rightchild));
parentNode = getNode(parentHandle, null, 0); // reload the parent node
LL_RightRotation(parentNode, getNode(parentNode.getOHHandle()[leftchild], parentNode, leftchild));
LL_RightRotation(parentNode, getNode(parentNode.getOHHandles()[leftchild], parentNode, leftchild));
break;
}
break;
@ -423,23 +431,26 @@ public class kelondroTree extends kelondroRecords implements Comparator {
break;
} else {
theNode = parentNode;
parentNode = getNode(parentOHHandle[parent] /*previous handles*/, null, 0);
parentNode = getNode(parentOHHandle[parent], null, 0);
}
}
}
return null; // that means: no previous stored value present
}
}
private void assignChild(Node parentNode, Node childNode, int childType) throws IOException {
Handle[] parentHandle = parentNode.getOHHandle();
Handle[] childHandle = childNode.getOHHandle();
Handle[] parentOHHandle = parentNode.getOHHandles();
Handle[] childOHHandle = childNode.getOHHandles();
parentHandle[childType] = childNode.handle();
childHandle[parent] = parentNode.handle();
parentOHHandle[childType] = childNode.handle();
childOHHandle[parent] = parentNode.handle();
parentNode.setOHHandle(parentHandle);
childNode.setOHHandle(childHandle);
parentNode.setOHHandles(parentOHHandle);
parentNode.commit(((parentOHHandle[leftchild] == null) || (parentOHHandle[rightchild] == null)) ? CP_MEDIUM : CP_HIGH);
childNode.setOHHandles(childOHHandle);
childNode.commit(((childOHHandle[leftchild] == null) || (childOHHandle[rightchild] == null)) ? CP_MEDIUM : CP_HIGH);
}
private void replace(Node oldNode, Node oldNodeParent, Node newNode) throws IOException {
@ -447,18 +458,19 @@ public class kelondroTree extends kelondroRecords implements Comparator {
// the anchor's link to the oldNode by the newNode-link
// the new link gets the anchor as parent link assigned
// the oldNode will not be updated, so this must be done outside this routine
Handle[] oldHandle = oldNode.getOHHandle(); // {parent, leftchild, rightchild}
Handle[] oldHandle = oldNode.getOHHandles(); // {parent, leftchild, rightchild}
// distinguish case where the oldNode is the root node
if (oldNodeParent == null) {
// this is the root, update root
setHandle(root, newNode.handle());
// update new Node
Handle[] newHandle = newNode.getOHHandle();
Handle[] newHandle = newNode.getOHHandles();
newHandle[parent] = null;
newNode.setOHHandle(newHandle);
newNode.setOHHandles(newHandle);
newNode.commit(CP_HIGH);
} else {
// not the root, find parent
Handle[] parentHandle = oldNodeParent.getOHHandle();
Handle[] parentHandle = oldNodeParent.getOHHandles();
// ok, we have the parent, but for updating the child link we must know
// if the oldNode was left or right child
if ((parentHandle[leftchild] != null) && (parentHandle[leftchild].equals(oldNode.handle()))) {
@ -469,11 +481,13 @@ public class kelondroTree extends kelondroRecords implements Comparator {
// update right node from parent
parentHandle[rightchild] = newNode.handle();
}
oldNodeParent.setOHHandle(parentHandle);
oldNodeParent.setOHHandles(parentHandle);
oldNodeParent.commit(((parentHandle[leftchild] == null) || (parentHandle[rightchild] == null)) ? CP_MEDIUM : CP_HIGH);
// update new Node
Handle[] newHandle = newNode.getOHHandle();
Handle[] newHandle = newNode.getOHHandles();
newHandle[parent] = oldNodeParent.handle();
newNode.setOHHandle(newHandle);
newNode.setOHHandles(newHandle);
newNode.commit(((newHandle[leftchild] == null) || (newHandle[rightchild] == null)) ? CP_MEDIUM : CP_HIGH);
}
// finished. remember that we did not set the links to the oldNode
// we have also not set the children of the newNode.
@ -491,16 +505,16 @@ public class kelondroTree extends kelondroRecords implements Comparator {
private void LL_RightRotation(Node parentNode, Node childNode) throws IOException {
// replace the parent node; the parent is afterwards unlinked
Handle p2Handle = parentNode.getOHHandle()[parent];
Handle p2Handle = parentNode.getOHHandles()[parent];
Node p2Node = (p2Handle == null) ? null : getNode(p2Handle, null, 0);
replace(parentNode, p2Node, childNode);
// set the left son of the parent to the right son of the childNode
Handle childOfChild = childNode.getOHHandle()[rightchild];
Handle childOfChild = childNode.getOHHandles()[rightchild];
if (childOfChild == null) {
Handle[] parentHandle = parentNode.getOHHandle();
Handle[] parentHandle = parentNode.getOHHandles();
parentHandle[leftchild] = null;
parentNode.setOHHandle(parentHandle);
parentNode.setOHHandles(parentHandle);
} else {
assignChild(parentNode, getNode(childOfChild, childNode, rightchild), leftchild);
}
@ -510,28 +524,30 @@ public class kelondroTree extends kelondroRecords implements Comparator {
// - newBal(parent) = oldBal(parent) - 1 - max(oldBal(leftChild), 0)
// - newBal(leftChild) = oldBal(leftChild) - 1 + min(newBal(parent), 0)
byte[] parentBytes = parentNode.getOHByte();
byte[] childBytes = childNode.getOHByte();
byte[] parentBytes = parentNode.getOHBytes();
byte[] childBytes = childNode.getOHBytes();
byte oldBalParent = parentBytes[balance];
byte oldBalChild = childBytes[balance];
parentBytes[balance] = (byte) (oldBalParent - 1 - max0(oldBalChild));
childBytes[balance] = (byte) (oldBalChild - 1 + min0(parentBytes[balance]));
parentNode.setOHByte(parentBytes);
childNode.setOHByte(childBytes);
parentNode.setOHBytes(parentBytes);
childNode.setOHBytes(childBytes);
parentNode.commit(CP_NONE);
childNode.commit(CP_NONE);
}
private void RR_LeftRotation(Node parentNode, Node childNode) throws IOException {
// replace the parent node; the parent is afterwards unlinked
Handle p2Handle = parentNode.getOHHandle()[parent];
Handle p2Handle = parentNode.getOHHandles()[parent];
Node p2Node = (p2Handle == null) ? null : getNode(p2Handle, null, 0);
replace(parentNode, p2Node, childNode);
// set the left son of the parent to the right son of the childNode
Handle childOfChild = childNode.getOHHandle()[leftchild];
Handle childOfChild = childNode.getOHHandles()[leftchild];
if (childOfChild == null) {
Handle[] parentHandle = parentNode.getOHHandle();
Handle[] parentHandle = parentNode.getOHHandles();
parentHandle[rightchild] = null;
parentNode.setOHHandle(parentHandle);
parentNode.setOHHandles(parentHandle);
} else {
assignChild(parentNode, getNode(childOfChild, childNode, leftchild), rightchild);
}
@ -541,14 +557,16 @@ public class kelondroTree extends kelondroRecords implements Comparator {
// - newBal(parent) = oldBal(parent) + 1 - min(oldBal(rightChild), 0)
// - newBal(rightChild) = oldBal(rightChild) + 1 + max(newBal(parent), 0)
byte[] parentBytes = parentNode.getOHByte();
byte[] childBytes = childNode.getOHByte();
byte[] parentBytes = parentNode.getOHBytes();
byte[] childBytes = childNode.getOHBytes();
byte oldBalParent = parentBytes[balance];
byte oldBalChild = childBytes[balance];
parentBytes[balance] = (byte) (oldBalParent + 1 - min0(oldBalChild));
childBytes[balance] = (byte) (oldBalChild + 1 + max0(parentBytes[balance]));
parentNode.setOHByte(parentBytes);
childNode.setOHByte(childBytes);
parentNode.setOHBytes(parentBytes);
childNode.setOHBytes(childBytes);
parentNode.commit(CP_NONE);
childNode.commit(CP_NONE);
}
// Associates the specified value with the specified key in this map
@ -586,7 +604,7 @@ public class kelondroTree extends kelondroRecords implements Comparator {
// by the greatest node of the left child or the smallest
// node of the right child
Handle[] handles = node.getOHHandle();
Handle[] handles = node.getOHHandles();
Node childnode;
if ((handles[leftchild] == null) && (handles[rightchild] == null)) {
// easy case: the node is a leaf
@ -594,10 +612,11 @@ public class kelondroTree extends kelondroRecords implements Comparator {
// this is the root!
setHandle(root, null);
} else {
Handle[] h = parentOfNode.getOHHandle();
Handle[] h = parentOfNode.getOHHandles();
if ((h[leftchild] != null) && (h[leftchild].equals(node.handle()))) h[leftchild] = null;
if ((h[rightchild] != null) && (h[rightchild].equals(node.handle()))) h[rightchild] = null;
parentOfNode.setOHHandle(h);
parentOfNode.setOHHandles(h);
parentOfNode.commit(((h[leftchild] == null) && (h[rightchild] == null)) ? CP_LOW : CP_MEDIUM);
}
} else if ((handles[leftchild] != null) && (handles[rightchild] == null)) {
replace(node, parentOfNode, getNode(handles[leftchild], node, leftchild));
@ -610,17 +629,18 @@ public class kelondroTree extends kelondroRecords implements Comparator {
// we remove that replacement node and put it where the node was
// this seems to be recursive, but is not since the replacement
// node cannot have two children (it would not have been the smallest or greatest)
Handle[] replha = repl.getOHHandle();
Handle[] replha = repl.getOHHandles();
Node n;
Handle[] h;
// remove leaf
if ((replha[leftchild] == null) && (replha[rightchild] == null)) {
// the replacement cannot be the root, so simply remove from parent node
n = getNode(replha[parent], null, 0); // parent node of replacement node
h = n.getOHHandle();
h = n.getOHHandles();
if ((h[leftchild] != null) && (h[leftchild].equals(repl.handle()))) h[leftchild] = null;
if ((h[rightchild] != null) && (h[rightchild].equals(repl.handle()))) h[rightchild] = null;
n.setOHHandle(h);
n.setOHHandles(h);
n.commit(((h[leftchild] == null) && (h[rightchild] == null)) ? CP_LOW : CP_MEDIUM);
} else if ((replha[leftchild] != null) && (replha[rightchild] == null)) {
try {
childnode = getNode(replha[leftchild], repl, leftchild);
@ -628,10 +648,11 @@ public class kelondroTree extends kelondroRecords implements Comparator {
} catch (IllegalArgumentException e) {
// now treat the situation as if that link had been null before
n = getNode(replha[parent], null, 0); // parent node of replacement node
h = n.getOHHandle();
h = n.getOHHandles();
if ((h[leftchild] != null) && (h[leftchild].equals(repl.handle()))) h[leftchild] = null;
if ((h[rightchild] != null) && (h[rightchild].equals(repl.handle()))) h[rightchild] = null;
n.setOHHandle(h);
n.setOHHandles(h);
n.commit(((h[leftchild] == null) && (h[rightchild] == null)) ? CP_LOW : CP_MEDIUM);
}
} else if ((replha[leftchild] == null) && (replha[rightchild] != null)) {
try {
@ -640,10 +661,11 @@ public class kelondroTree extends kelondroRecords implements Comparator {
} catch (IllegalArgumentException e) {
// now treat the situation as if that link had been null before
n = getNode(replha[parent], null, 0); // parent node of replacement node
h = n.getOHHandle();
h = n.getOHHandles();
if ((h[leftchild] != null) && (h[leftchild].equals(repl.handle()))) h[leftchild] = null;
if ((h[rightchild] != null) && (h[rightchild].equals(repl.handle()))) h[rightchild] = null;
n.setOHHandle(h);
n.setOHHandles(h);
n.commit(((h[leftchild] == null) && (h[rightchild] == null)) ? CP_LOW : CP_MEDIUM);
}
}
//System.out.println("node before reload is " + node.toString());
@ -651,25 +673,29 @@ public class kelondroTree extends kelondroRecords implements Comparator {
//System.out.println("node after reload is " + node.toString());
// now plant in the replha node
byte[] b = node.getOHByte(); // save bytes of disappearing node
handles = node.getOHHandle(); // save handles of disappearing node
byte[] b = node.getOHBytes(); // save bytes of disappearing node
handles = node.getOHHandles(); // save handles of disappearing node
replace(node, parentOfNode, repl);
repl.setOHByte(b); // restore bytes
repl.setOHHandle(handles); // restore handles
repl.setOHBytes(b); // restore bytes
repl.setOHHandles(handles); // restore handles
repl.commit(((handles[leftchild] == null) || (handles[rightchild] == null)) ? CP_MEDIUM : CP_HIGH);
// last thing to do: change uplinks of children to this new node
if (handles[leftchild] != null) {
n = getNode(handles[leftchild], node, leftchild);
h = n.getOHHandle();
h = n.getOHHandles();
h[parent] = repl.handle();
n.setOHHandle(h);
n.setOHHandles(h);
n.commit(((h[leftchild] == null) || (h[rightchild] == null)) ? CP_MEDIUM : CP_HIGH);
}
if (handles[rightchild] != null) {
n = getNode(handles[rightchild], node, rightchild);
h = n.getOHHandle();
h = n.getOHHandles();
h[parent] = repl.handle();
n.setOHHandle(h);
n.setOHHandles(h);
n.commit(((h[leftchild] == null) || (h[rightchild] == null)) ? CP_MEDIUM : CP_HIGH);
}
}
// move node to recycling queue
deleteNode(node.handle());
}
@ -681,7 +707,7 @@ public class kelondroTree extends kelondroRecords implements Comparator {
private Node firstNode(Node node) throws IOException {
if (node == null) throw new IllegalArgumentException("firstNode: node=null");
Handle h = node.getOHHandle()[leftchild];
Handle h = node.getOHHandles()[leftchild];
while (h != null) {
try {
node = getNode(h, node, leftchild);
@ -689,7 +715,7 @@ public class kelondroTree extends kelondroRecords implements Comparator {
// return what we have
return node;
}
h = node.getOHHandle()[leftchild];
h = node.getOHHandles()[leftchild];
}
return node;
}
@ -702,7 +728,7 @@ public class kelondroTree extends kelondroRecords implements Comparator {
private Node lastNode(Node node) throws IOException {
if (node == null) throw new IllegalArgumentException("lastNode: node=null");
Handle h = node.getOHHandle()[rightchild];
Handle h = node.getOHHandles()[rightchild];
while (h != null) {
try {
node = getNode(h, node, rightchild);
@ -710,7 +736,7 @@ public class kelondroTree extends kelondroRecords implements Comparator {
// return what we have
return node;
}
h = node.getOHHandle()[rightchild];
h = node.getOHHandles()[rightchild];
}
return node;
}
@ -823,7 +849,7 @@ public class kelondroTree extends kelondroRecords implements Comparator {
nodeStack.addLast(new Object[]{searchNode, new Integer(ct)});
// go to next node
searchHandle = searchNode.getOHHandle()[ct];
searchHandle = searchNode.getOHHandles()[ct];
if (searchHandle == null) throw new kelondroException(filename, "start node does not exist (handle null)");
searchNode = getNode(searchHandle, searchNode, ct);
if (searchNode == null) throw new kelondroException(filename, "start node does not exist (node null)");
@ -850,14 +876,14 @@ public class kelondroTree extends kelondroRecords implements Comparator {
try {
int childtype = (up) ? rightchild : leftchild;
Handle childHandle = nextNode.getOHHandle()[childtype];
Handle childHandle = nextNode.getOHHandles()[childtype];
if (childHandle != null) {
//System.out.println("go to other leg, stack size=" + nodeStack.size());
// we have walked one leg of the tree; now go to the other one: step down to next child
nodeStack.addLast(new Object[]{nextNode, new Integer(childtype)});
nextNode = getNode(childHandle, nextNode, childtype);
childtype = (up) ? leftchild : rightchild;
while ((childHandle = nextNode.getOHHandle()[childtype]) != null) {
while ((childHandle = nextNode.getOHHandles()[childtype]) != null) {
try {
nodeStack.addLast(new Object[]{nextNode, new Integer(childtype)});
nextNode = getNode(childHandle, nextNode, childtype);
@ -1056,7 +1082,7 @@ public class kelondroTree extends kelondroRecords implements Comparator {
private int height(Node node) throws IOException {
if (node == null) return 0;
Handle[] childs = node.getOHHandle();
Handle[] childs = node.getOHHandles();
int hl = (childs[leftchild] == null) ? 0 : height(getNode(childs[leftchild], node, leftchild));
int hr = (childs[rightchild] == null) ? 0 : height(getNode(childs[rightchild], node, rightchild));
if (hl > hr) return hl + 1; else return hr + 1;
@ -1098,7 +1124,7 @@ public class kelondroTree extends kelondroRecords implements Comparator {
nextline.add(null);
nextline.add(null);
} else {
childs = node.getOHHandle();
childs = node.getOHHandles();
nextline.add(childs[leftchild]);
nextline.add(childs[rightchild]);
}
@ -1317,9 +1343,9 @@ public class kelondroTree extends kelondroRecords implements Comparator {
}
public static void main(String[] args) {
cmd(args);
//cmd(args);
//bigtest(Integer.parseInt(args[0]));
//randomtest(Integer.parseInt(args[0]));
randomtest(Integer.parseInt(args[0]));
//smalltest();
}
@ -1348,17 +1374,6 @@ public class kelondroTree extends kelondroRecords implements Comparator {
return new byte[]{(byte) c, 32, 32, 32};
}
public static kelondroTree testTree(File f, String testentities) throws IOException {
if (f.exists()) f.delete();
kelondroTree tt = new kelondroTree(f, 0, 4, 4);
byte[] b;
for (int i = 0; i < testentities.length(); i++) {
b = testWord(testentities.charAt(i));
tt.put(b, b);
}
return tt;
}
public static void randomtest(int elements) {
System.out.println("random " + elements + ":");
String s = "ABCDEFGHIJKLMNOPQRSTUVWXYZ".substring(0, elements);
@ -1371,7 +1386,7 @@ public class kelondroTree extends kelondroRecords implements Comparator {
int steps = 0;
while (true) {
if (testFile.exists()) testFile.delete();
tt = new kelondroTree(testFile, 20000, 4 ,4);
tt = new kelondroTree(testFile, 0, 4 ,4);
steps = 10 + ((int) System.currentTimeMillis() % 7) * (((int) System.currentTimeMillis() + 17) % 11);
t = s;
d = "";
@ -1394,31 +1409,35 @@ public class kelondroTree extends kelondroRecords implements Comparator {
t = t + c;
System.out.println("removed " + new String(b));
}
//tt.printCache();
//tt.print();
if (countElements(tt) != tt.size()) {
System.out.println("wrong size for ");
System.out.println("wrong size for this table:");
tt.print();
}
// check all words within
for (int j = 0; j < d.length(); j++) {
if (tt.get(testWord(d.charAt(j))) == null) {
System.out.println("missing entry " + d.charAt(j));
System.out.println("missing entry " + d.charAt(j) + " in this table:");
tt.print();
}
}
// check all words outside
for (int j = 0; j < t.length(); j++) {
if (tt.get(testWord(t.charAt(j))) != null) {
System.out.println("superfluous entry " + t.charAt(j));
System.out.println("superfluous entry " + t.charAt(j) + " in this table:");
tt.print();
}
}
if (tt.get(testWord('z')) != null) {
System.out.println("superfluous entry z");
System.out.println("superfluous entry z in this table:");
tt.print();
}
}
tt.print();
//tt.print();
tt.close();
}
@ -1433,27 +1452,44 @@ public class kelondroTree extends kelondroRecords implements Comparator {
File f = new File("test.db");
if (f.exists()) f.delete();
try {
kelondroTree tt = new kelondroTree(f, 1000, 4, 4);
kelondroTree tt = new kelondroTree(f, 0, 4, 4);
byte[] b;
b = testWord('b'); tt.put(b, b);
b = testWord('c'); tt.put(b, b);
b = testWord('a'); tt.put(b, b);
b = testWord('B'); tt.put(b, b); tt.print();
b = testWord('C'); tt.put(b, b); tt.print();
b = testWord('D'); tt.put(b, b); tt.print();
b = testWord('A'); tt.put(b, b); tt.print();
b = testWord('D'); tt.remove(b); tt.print();
b = testWord('B'); tt.remove(b); tt.print();
b = testWord('B');
tt.put(b, b); tt.print();
System.out.println("elements: " + countElements(tt));
tt.print();
System.out.println("TERMINATED");
} catch (IOException e) {
e.printStackTrace();
}
}
public static void bigtest(int elements) {
System.out.println("perm " + elements + ":");
public static kelondroTree testTree(File f, String testentities) throws IOException {
if (f.exists()) f.delete();
kelondroTree tt = new kelondroTree(f, 200000, 4, 4);
byte[] b;
for (int i = 0; i < testentities.length(); i++) {
b = testWord(testentities.charAt(i));
tt.put(b, b);
}
return tt;
}
public static void bigtest(int elements) {
System.out.println("starting big test with " + elements + " elements:");
long start = System.currentTimeMillis();
String[] s = permutations(elements);
kelondroTree tt;
File testFile = new File("test.db");
byte[] b;
try {
for (int i = 0; i < s.length; i++) {
System.out.println("probing tree " + i + " for permutation " + s[i]);
System.out.println("*** probing tree " + i + " for permutation " + s[i]);
// generate tree and delete elements
tt = testTree(testFile, s[i]);
//tt.print();
@ -1491,17 +1527,23 @@ public class kelondroTree extends kelondroRecords implements Comparator {
tt.close();
}
}
System.out.println("FINISHED");
System.out.println("FINISHED test after " + ((System.currentTimeMillis() - start) / 1000) + " seconds.");
} catch (Exception e) {
e.printStackTrace();
System.out.println("TERMINATED");
}
}
public static int countElements(kelondroTree t) {
public static int countElements(kelondroTree t) throws IOException {
int count = 0;
Iterator iter = t.nodeIterator(true, false);
while (iter.hasNext()) {count++; if (iter.next() == null) System.out.println("ERROR! null element found");}
Node n;
while (iter.hasNext()) {
count++;
n = (Node) iter.next();
if (n == null) System.out.println("ERROR! null element found");
//else System.out.println("counted element: " + new String(n.getKey()));
}
return count;
}
}

9
source/de/anomic/plasma/plasmaCrawlRobotsTxt.java
View File

@ -56,6 +56,7 @@ import java.util.LinkedList;
import java.util.Map;
import de.anomic.kelondro.kelondroDyn;
import de.anomic.kelondro.kelondroMap;
import de.anomic.kelondro.kelondroException;
import de.anomic.server.logging.serverLog;
public class plasmaCrawlRobotsTxt {
@ -65,7 +66,13 @@ public class plasmaCrawlRobotsTxt {
public plasmaCrawlRobotsTxt(File robotsTableFile) throws IOException {
this.robotsTableFile = robotsTableFile;
if (robotsTableFile.exists()) {
robotsTable = new kelondroMap(new kelondroDyn(robotsTableFile, 32000));
try {
robotsTable = new kelondroMap(new kelondroDyn(robotsTableFile, 32000));
} catch (kelondroException e) {
robotsTableFile.delete();
robotsTableFile.getParentFile().mkdirs();
robotsTable = new kelondroMap(new kelondroDyn(robotsTableFile, 32000, 256, 512));
}
} else {
robotsTableFile.getParentFile().mkdirs();
robotsTable = new kelondroMap(new kelondroDyn(robotsTableFile, 32000, 256, 512));

4
source/de/anomic/plasma/plasmaWordIndexAssortment.java
View File

@ -113,7 +113,9 @@ public final class plasmaWordIndexAssortment {
if (log != null) log.logConfig("Opened Assortment Database, " + assortments.size() + " entries, width " + assortmentLength + ", " + bufferkb + "kb buffer");
return;
} catch (IOException e){
serverLog.logSevere("PLASMA", "unable to open assortment database, creating new: " + e.getMessage(), e);
serverLog.logSevere("PLASMA", "unable to open assortment database " + assortmentLength + ", creating new: " + e.getMessage(), e);
} catch (kelondroException e) {
serverLog.logSevere("PLASMA", "assortment database " + assortmentLength + " corupted, creating new: " + e.getMessage(), e);
}
assortmentFile.delete(); // make space for new one
}

8
source/de/anomic/yacy/yacyNewsQueue.java
View File

@ -50,6 +50,7 @@ import java.util.Date;
import de.anomic.htmlFilter.htmlFilterContentScraper;
import de.anomic.http.httpHeader;
import de.anomic.kelondro.kelondroStack;
import de.anomic.kelondro.kelondroException;
import de.anomic.server.serverCodings;
import de.anomic.yacy.yacySeedDB;
@ -64,7 +65,12 @@ public class yacyNewsQueue {
this.newsDB = newsDB;
if (path.exists()) {
queueStack = new kelondroStack(path, 0);
try {
queueStack = new kelondroStack(path, 0);
} catch (kelondroException e) {
path.delete();
queueStack = createStack(path);
}
} else
queueStack = createStack(path);
}