diff --git a/source/de/anomic/kelondro/kelondroObjectCache.java b/source/de/anomic/kelondro/kelondroObjectCache.java new file mode 100644 index 000000000..0ef9450f4 --- /dev/null +++ b/source/de/anomic/kelondro/kelondroObjectCache.java @@ -0,0 +1,186 @@ +// kelondroObjectCache.java +// ------------------------ +// (C) by Michael Peter Christen; mc@anomic.de +// first published on http://www.anomic.de +// Frankfurt, Germany, 2006 +// +// This is a part of the kelondro database, which is a part of YaCy +// +// $LastChangedDate: 2006-04-02 22:40:07 +0200 (So, 02 Apr 2006) $ +// $LastChangedRevision: 1986 $ +// $LastChangedBy: orbiter $ +// +// +// LICENSE +// +// This program is free software; you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation; either version 2 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +// +// +// A NOTE FROM THE AUTHOR TO THE USERS: +// +// Using this software in any meaning (reading, learning, copying, compiling, +// running) means that you agree that the Author(s) is (are) not responsible +// for cost, loss of data or any harm that may be caused directly or indirectly +// by usage of this softare or this documentation. The usage of this software +// is on your own risk. The installation and usage (starting/running) of this +// software may allow other people or application to access your computer and +// any attached devices and is highly dependent on the configuration of the +// software which must be done by the user of the software; the author(s) is +// (are) also not responsible for proper configuration and usage of the +// software, even if provoked by documentation provided together with +// the software. +// +// +// A NOTE FROM THE AUTHOR TO DEVELOPERS: +// +// Contributions and changes to the program code should be marked as such: +// Please enter your own (C) notice below; they must be compatible with the GPL. +// Please mark also all changes in the code; if you don't mark them then they +// can't be identified; thus all unmarked code belong to the copyright holder +// as mentioned above. A good documentation of code authorities will also help +// to maintain the code and the project. +// A re-distribution must contain the intact and unchanged copyright statement. + + +package de.anomic.kelondro; + +import java.util.TreeMap; + +public class kelondroObjectCache { + + private final TreeMap cache; + private final kelondroMScoreCluster ages; + private long startTime; + private int maxSize; + private long maxAge; + private long minMem; + private int readHit, readMiss, writeUnique, writeDouble; + + public kelondroObjectCache(int maxSize, long maxAge, long minMem) { + this.cache = new TreeMap(); + this.ages = new kelondroMScoreCluster(); + this.startTime = System.currentTimeMillis(); + this.maxAge = maxAge; + this.minMem = minMem; + this.readHit = 0; + this.readMiss = 0; + this.writeUnique = 0; + this.writeDouble = 0; + } + + public void setMaxAge(long maxAge) { + this.maxAge = maxAge; + } + + public void setMaxSize(int maxSize) { + this.maxSize = maxSize; + } + + public void setMinMem(int minMem) { + this.minMem = minMem; + } + + public long minAge() { + if (ages.size() == 0) return 0; + return System.currentTimeMillis() - longEmit(ages.getMaxScore()); + } + + public long maxAge() { + if (ages.size() == 0) return 0; + return System.currentTimeMillis() - longEmit(ages.getMinScore()); + } + + public int size() { + return cache.size(); + } + + private int intTime(long longTime) { + return (int) Math.max(0, ((longTime - startTime) / 1000)); + } + + private long longEmit(int intTime) { + return (((long) intTime) * (long) 1000) + startTime; + } + + public void put(byte[] key, Object value) { + if (key != null) put(new String(key), value); + } + + public void put(String key, Object value) { + if (key == null) return; + Object prev = null; + synchronized(cache) { + prev = cache.put(key, value); + ages.setScore(key, intTime(System.currentTimeMillis())); + } + if (prev == null) this.writeUnique++; else this.writeDouble++; + flush(); + } + + public Object get(byte[] key) { + if (key == null) return null; + Object r = cache.get(new String(key)); + flush(); + if (r == null) this.readMiss++; else this.readHit++; + return r; + } + + public Object get(String key) { + if (key == null) return null; + Object r = cache.get(key); + flush(); + if (r == null) this.readMiss++; else this.readHit++; + return r; + } + + public void remove(byte[] key) { + remove(new String(key)); + } + + public void remove(String key) { + if (key == null) return; + synchronized(cache) { + cache.remove(key); + ages.deleteScore(key); + } + flush(); + } + + public void flush() { + String k; + synchronized(cache) { + while ((ages.size() > 0) && + ((size() > maxSize) || + (longEmit(ages.getMinScore()) > maxAge) || + (Runtime.getRuntime().freeMemory() < minMem)) && + ((k = bestFlush()) != null)) { + cache.remove(k); + ages.deleteScore(k); + if (Runtime.getRuntime().freeMemory() < minMem) System.gc(); // prevent unnecessary loops + } + } + } + + public String bestFlush() { + if (cache.size() == 0) return null; + try { + synchronized (cache) { + return (String) ages.getMinObject(); // flush oldest entries + } + } catch (Exception e) {} + return null; + } + +} diff --git a/source/de/anomic/kelondro/kelondroRecords.java b/source/de/anomic/kelondro/kelondroRecords.java index f1a73ace8..d24e48f9e 100644 --- a/source/de/anomic/kelondro/kelondroRecords.java +++ b/source/de/anomic/kelondro/kelondroRecords.java @@ -127,6 +127,7 @@ public class kelondroRecords { private int headchunksize;// overheadsize + key element column size private int tailchunksize;// sum(all: COLWIDTHS) minus the size of the key element colum private int recordsize; // (overhead + sum(all: COLWIDTHS)) = the overall size of a record + protected int objectsize; // sum(all: COLWIDTHS)) = the size of all data fields // dynamic run-time seek pointers private long POS_HANDLES = 0; // starts after end of POS_COLWIDHS which is POS_COLWIDTHS + COLWIDTHS.length * 4 @@ -247,8 +248,9 @@ public class kelondroRecords { // store dynamic run-time data this.overhead = ohbytec + 4 * ohhandlec; - this.recordsize = this.overhead; - for (int i = 0; i < columns.length; i++) this.recordsize += columns[i]; + this.objectsize = 0; + for (int i = 0; i < columns.length; i++) this.objectsize += columns[i]; + this.recordsize = this.overhead + this.objectsize; this.headchunksize = overhead + columns[0]; this.tailchunksize = this.recordsize - this.headchunksize; @@ -412,7 +414,9 @@ public class kelondroRecords { // assign remaining values that are only present at run-time this.overhead = OHBYTEC + 4 * OHHANDLEC; this.recordsize = this.overhead; - for (int i = 0; i < COLWIDTHS.length; i++) this.recordsize += COLWIDTHS[i]; + this.objectsize = 0; + for (int i = 0; i < COLWIDTHS.length; i++) this.objectsize += COLWIDTHS[i]; + this.recordsize = this.overhead + this.objectsize; this.headchunksize = this.overhead + COLWIDTHS[0]; this.tailchunksize = this.recordsize - this.headchunksize; } diff --git a/source/de/anomic/kelondro/kelondroTree.java b/source/de/anomic/kelondro/kelondroTree.java index 9f144a2c2..448f89488 100644 --- a/source/de/anomic/kelondro/kelondroTree.java +++ b/source/de/anomic/kelondro/kelondroTree.java @@ -80,11 +80,17 @@ public class kelondroTree extends kelondroRecords implements kelondroIndex { private static int root = 0; // pointer for FHandles-array: pointer to root node + // calibration of cache + private static int nodeCachePercent = 90; + private static int objectCachePercent = 10; + + // class variables private Search writeSearchObj = new Search(); protected kelondroOrder objectOrder = new kelondroNaturalOrder(true); private final kelondroOrder loopDetectionOrder = new kelondroNaturalOrder(true); private int readAheadChunkSize = 100; private long lastIteratorCount = readAheadChunkSize; + private kelondroObjectCache objectCache; public kelondroTree(File file, long buffersize, int key, int value, boolean exitOnFail) { this(file, buffersize, new int[] { key, value }, new kelondroNaturalOrder(true), 1, 8, exitOnFail); @@ -101,7 +107,10 @@ public class kelondroTree extends kelondroRecords implements kelondroIndex { public kelondroTree(File file, long buffersize, int[] columns, kelondroOrder objectOrder, int txtProps, int txtPropsWidth, boolean exitOnFail) { // this creates a new tree file - super(file, buffersize, thisOHBytes, thisOHHandles, columns, thisFHandles, txtProps, txtPropsWidth, exitOnFail); + super(file, + nodeCachePercent * buffersize / (nodeCachePercent + objectCachePercent), + thisOHBytes, thisOHHandles, columns, + thisFHandles, txtProps, txtPropsWidth, exitOnFail); try { setHandle(root, null); // define the root value } catch (IOException e) { @@ -112,6 +121,9 @@ public class kelondroTree extends kelondroRecords implements kelondroIndex { this.objectOrder = objectOrder; writeOrderType(); super.setLogger(log); + long objectbuffersize = objectCachePercent * buffersize / (nodeCachePercent + objectCachePercent); + long nodecachesize = objectbuffersize / (super.objectsize + 8 * columns.length); + this.objectCache = new kelondroObjectCache((int) nodecachesize, nodecachesize * 300 , 4*1024*1024); } public kelondroTree(kelondroRA ra, long buffersize, int[] columns, boolean exitOnFail) { @@ -121,8 +133,10 @@ public class kelondroTree extends kelondroRecords implements kelondroIndex { public kelondroTree(kelondroRA ra, long buffersize, int[] columns, kelondroOrder objectOrder, int txtProps, int txtPropsWidth, boolean exitOnFail) { // this creates a new tree within a kelondroRA - super(ra, buffersize, thisOHBytes, thisOHHandles, columns, - thisFHandles, txtProps, txtPropsWidth, exitOnFail); + super(ra, + nodeCachePercent * buffersize / (nodeCachePercent + objectCachePercent), + thisOHBytes, thisOHHandles, columns, + thisFHandles, txtProps, txtPropsWidth, exitOnFail); try { setHandle(root, null); // define the root value } catch (IOException e) { @@ -133,20 +147,29 @@ public class kelondroTree extends kelondroRecords implements kelondroIndex { this.objectOrder = objectOrder; writeOrderType(); super.setLogger(log); + long objectbuffersize = objectCachePercent * buffersize / (nodeCachePercent + objectCachePercent); + long nodecachesize = objectbuffersize / (super.objectsize + 8 * columns.length); + this.objectCache = new kelondroObjectCache((int) nodecachesize, nodecachesize * 300 , 4*1024*1024); } public kelondroTree(File file, long buffersize) throws IOException { // this opens a file with an existing tree file - super(file, buffersize); + super(file, nodeCachePercent * buffersize / (nodeCachePercent + objectCachePercent)); readOrderType(); super.setLogger(log); + long objectbuffersize = objectCachePercent * buffersize / (nodeCachePercent + objectCachePercent); + long nodecachesize = objectbuffersize / (super.objectsize + 8 * super.columns()); + this.objectCache = new kelondroObjectCache((int) nodecachesize, nodecachesize * 300 , 4*1024*1024); } public kelondroTree(kelondroRA ra, long buffersize) throws IOException { // this opens a file with an existing tree in a kelondroRA - super(ra, buffersize); + super(ra, nodeCachePercent * buffersize / (nodeCachePercent + objectCachePercent)); readOrderType(); super.setLogger(log); + long objectbuffersize = objectCachePercent * buffersize / (nodeCachePercent + objectCachePercent); + long nodecachesize = objectbuffersize / (super.objectsize + 8 * super.columns()); + this.objectCache = new kelondroObjectCache((int) nodecachesize, nodecachesize * 300 , 4*1024*1024); } private void writeOrderType() { @@ -193,7 +216,8 @@ public class kelondroTree extends kelondroRecords implements kelondroIndex { // Returns the value to which this map maps the specified key. public byte[][] get(byte[] key) throws IOException { // System.out.println("kelondroTree.get " + new String(key) + " in " + filename); - byte[][] result = null; + byte[][] result = (byte[][]) objectCache.get(key); + if (result != null) return result; // writeLock.stay(2000, 1000); synchronized (writeSearchObj) { writeSearchObj.process(key); @@ -202,6 +226,7 @@ public class kelondroTree extends kelondroRecords implements kelondroIndex { } else { result = null; } + objectCache.put(key, result); } // writeLock.release(); return result; @@ -378,6 +403,7 @@ public class kelondroTree extends kelondroRecords implements kelondroIndex { if (newrow.length != columns()) throw new IllegalArgumentException("put: wrong row length " + newrow.length + "; must be " + columns()); // first try to find the key element in the database synchronized(writeSearchObj) { + objectCache.put(newrow[0], newrow); writeSearchObj.process(newrow[0]); if (writeSearchObj.found()) { // a node with this key exist. simply overwrite the content and return old content @@ -632,16 +658,17 @@ public class kelondroTree extends kelondroRecords implements kelondroIndex { // Associates the specified value with the specified key in this map public byte[] put(byte[] key, byte[] value) throws IOException { - byte[][] row = new byte[2][]; - row[0] = key; - row[1] = value; - byte[][] ret = put(row); - if (ret == null) return null; else return ret[1]; + byte[][] row = new byte[2][]; + row[0] = key; + row[1] = value; + byte[][] ret = put(row); + if (ret == null) return null; else return ret[1]; } // Removes the mapping for this key from this map if present (optional operation). public byte[][] remove(byte[] key) throws IOException { synchronized(writeSearchObj) { + objectCache.remove(key); writeSearchObj.process(key); if (writeSearchObj.found()) { Node result = writeSearchObj.getMatcher();