Marown/yacy_search_server
1
0
Fork 0
mirror of https://github.com/yacy/yacy_search_server.git synced 2024-09-19 00:01:41 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity

added very experimental object cache for all kelondroTree

Browse Source 
databases. This cache now takes up 10% of all cache sizes,
90% goes to the node routing cache we had so far.

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@2069 6c8d7289-2bf4-0310-a012-ef5d649a1542
This commit is contained in:
orbiter 2006-05-08 13:37:27 +00:00
parent 0dfef19559
commit bab9ee38e7
3 changed files with 231 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

186
source/de/anomic/kelondro/kelondroObjectCache.java Normal file
View File

@ -0,0 +1,186 @@
// kelondroObjectCache.java
// ------------------------
// (C) by Michael Peter Christen; mc@anomic.de
// first published on http://www.anomic.de
// Frankfurt, Germany, 2006
//
// This is a part of the kelondro database, which is a part of YaCy
//
// $LastChangedDate: 2006-04-02 22:40:07 +0200 (So, 02 Apr 2006) $
// $LastChangedRevision: 1986 $
// $LastChangedBy: orbiter $
//
//
// LICENSE
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation; either version 2 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
//
//
// A NOTE FROM THE AUTHOR TO THE USERS:
//
// Using this software in any meaning (reading, learning, copying, compiling,
// running) means that you agree that the Author(s) is (are) not responsible
// for cost, loss of data or any harm that may be caused directly or indirectly
// by usage of this softare or this documentation. The usage of this software
// is on your own risk. The installation and usage (starting/running) of this
// software may allow other people or application to access your computer and
// any attached devices and is highly dependent on the configuration of the
// software which must be done by the user of the software; the author(s) is
// (are) also not responsible for proper configuration and usage of the
// software, even if provoked by documentation provided together with
// the software.
//
//
// A NOTE FROM THE AUTHOR TO DEVELOPERS:
//
// Contributions and changes to the program code should be marked as such:
// Please enter your own (C) notice below; they must be compatible with the GPL.
// Please mark also all changes in the code; if you don't mark them then they
// can't be identified; thus all unmarked code belong to the copyright holder
// as mentioned above. A good documentation of code authorities will also help
// to maintain the code and the project.
// A re-distribution must contain the intact and unchanged copyright statement.
package de.anomic.kelondro;
import java.util.TreeMap;
public class kelondroObjectCache {
private final TreeMap cache;
private final kelondroMScoreCluster ages;
private long startTime;
private int maxSize;
private long maxAge;
private long minMem;
private int readHit, readMiss, writeUnique, writeDouble;
public kelondroObjectCache(int maxSize, long maxAge, long minMem) {
this.cache = new TreeMap();
this.ages = new kelondroMScoreCluster();
this.startTime = System.currentTimeMillis();
this.maxAge = maxAge;
this.minMem = minMem;
this.readHit = 0;
this.readMiss = 0;
this.writeUnique = 0;
this.writeDouble = 0;
}
public void setMaxAge(long maxAge) {
this.maxAge = maxAge;
}
public void setMaxSize(int maxSize) {
this.maxSize = maxSize;
}
public void setMinMem(int minMem) {
this.minMem = minMem;
}
public long minAge() {
if (ages.size() == 0) return 0;
return System.currentTimeMillis() - longEmit(ages.getMaxScore());
}
public long maxAge() {
if (ages.size() == 0) return 0;
return System.currentTimeMillis() - longEmit(ages.getMinScore());
}
public int size() {
return cache.size();
}
private int intTime(long longTime) {
return (int) Math.max(0, ((longTime - startTime) / 1000));
}
private long longEmit(int intTime) {
return (((long) intTime) * (long) 1000) + startTime;
}
public void put(byte[] key, Object value) {
if (key != null) put(new String(key), value);
}
public void put(String key, Object value) {
if (key == null) return;
Object prev = null;
synchronized(cache) {
prev = cache.put(key, value);
ages.setScore(key, intTime(System.currentTimeMillis()));
}
if (prev == null) this.writeUnique++; else this.writeDouble++;
flush();
}
public Object get(byte[] key) {
if (key == null) return null;
Object r = cache.get(new String(key));
flush();
if (r == null) this.readMiss++; else this.readHit++;
return r;
}
public Object get(String key) {
if (key == null) return null;
Object r = cache.get(key);
flush();
if (r == null) this.readMiss++; else this.readHit++;
return r;
}
public void remove(byte[] key) {
remove(new String(key));
}
public void remove(String key) {
if (key == null) return;
synchronized(cache) {
cache.remove(key);
ages.deleteScore(key);
}
flush();
}
public void flush() {
String k;
synchronized(cache) {
while ((ages.size() > 0) &&
((size() > maxSize) ||
(longEmit(ages.getMinScore()) > maxAge) ||
(Runtime.getRuntime().freeMemory() < minMem)) &&
((k = bestFlush()) != null)) {
cache.remove(k);
ages.deleteScore(k);
if (Runtime.getRuntime().freeMemory() < minMem) System.gc(); // prevent unnecessary loops
}
}
}
public String bestFlush() {
if (cache.size() == 0) return null;
try {
synchronized (cache) {
return (String) ages.getMinObject(); // flush oldest entries
}
} catch (Exception e) {}
return null;
}
}

10
source/de/anomic/kelondro/kelondroRecords.java
View File

@ -127,6 +127,7 @@ public class kelondroRecords {
private int headchunksize;// overheadsize + key element column size
private int tailchunksize;// sum(all: COLWIDTHS) minus the size of the key element colum
private int recordsize; // (overhead + sum(all: COLWIDTHS)) = the overall size of a record
protected int objectsize; // sum(all: COLWIDTHS)) = the size of all data fields
// dynamic run-time seek pointers
private long POS_HANDLES = 0; // starts after end of POS_COLWIDHS which is POS_COLWIDTHS + COLWIDTHS.length * 4
@ -247,8 +248,9 @@ public class kelondroRecords {
// store dynamic run-time data
this.overhead = ohbytec + 4 * ohhandlec;
this.recordsize = this.overhead;
for (int i = 0; i < columns.length; i++) this.recordsize += columns[i];
this.objectsize = 0;
for (int i = 0; i < columns.length; i++) this.objectsize += columns[i];
this.recordsize = this.overhead + this.objectsize;
this.headchunksize = overhead + columns[0];
this.tailchunksize = this.recordsize - this.headchunksize;
@ -412,7 +414,9 @@ public class kelondroRecords {
// assign remaining values that are only present at run-time
this.overhead = OHBYTEC + 4 * OHHANDLEC;
this.recordsize = this.overhead;
for (int i = 0; i < COLWIDTHS.length; i++) this.recordsize += COLWIDTHS[i];
this.objectsize = 0;
for (int i = 0; i < COLWIDTHS.length; i++) this.objectsize += COLWIDTHS[i];
this.recordsize = this.overhead + this.objectsize;
this.headchunksize = this.overhead + COLWIDTHS[0];
this.tailchunksize = this.recordsize - this.headchunksize;
}

49
source/de/anomic/kelondro/kelondroTree.java
View File

@ -80,11 +80,17 @@ public class kelondroTree extends kelondroRecords implements kelondroIndex {
private static int root = 0; // pointer for FHandles-array: pointer to root node
// calibration of cache
private static int nodeCachePercent = 90;
private static int objectCachePercent = 10;
// class variables
private Search writeSearchObj = new Search();
protected kelondroOrder objectOrder = new kelondroNaturalOrder(true);
private final kelondroOrder loopDetectionOrder = new kelondroNaturalOrder(true);
private int readAheadChunkSize = 100;
private long lastIteratorCount = readAheadChunkSize;
private kelondroObjectCache objectCache;
public kelondroTree(File file, long buffersize, int key, int value, boolean exitOnFail) {
this(file, buffersize, new int[] { key, value }, new kelondroNaturalOrder(true), 1, 8, exitOnFail);
@ -101,7 +107,10 @@ public class kelondroTree extends kelondroRecords implements kelondroIndex {
public kelondroTree(File file, long buffersize, int[] columns, kelondroOrder objectOrder, int txtProps, int txtPropsWidth, boolean exitOnFail) {
// this creates a new tree file
super(file, buffersize, thisOHBytes, thisOHHandles, columns, thisFHandles, txtProps, txtPropsWidth, exitOnFail);
super(file,
nodeCachePercent * buffersize / (nodeCachePercent + objectCachePercent),
thisOHBytes, thisOHHandles, columns,
thisFHandles, txtProps, txtPropsWidth, exitOnFail);
try {
setHandle(root, null); // define the root value
} catch (IOException e) {
@ -112,6 +121,9 @@ public class kelondroTree extends kelondroRecords implements kelondroIndex {
this.objectOrder = objectOrder;
writeOrderType();
super.setLogger(log);
long objectbuffersize = objectCachePercent * buffersize / (nodeCachePercent + objectCachePercent);
long nodecachesize = objectbuffersize / (super.objectsize + 8 * columns.length);
this.objectCache = new kelondroObjectCache((int) nodecachesize, nodecachesize * 300 , 4*1024*1024);
}
public kelondroTree(kelondroRA ra, long buffersize, int[] columns, boolean exitOnFail) {
@ -121,8 +133,10 @@ public class kelondroTree extends kelondroRecords implements kelondroIndex {
public kelondroTree(kelondroRA ra, long buffersize, int[] columns, kelondroOrder objectOrder, int txtProps, int txtPropsWidth, boolean exitOnFail) {
// this creates a new tree within a kelondroRA
super(ra, buffersize, thisOHBytes, thisOHHandles, columns,
thisFHandles, txtProps, txtPropsWidth, exitOnFail);
super(ra,
nodeCachePercent * buffersize / (nodeCachePercent + objectCachePercent),
thisOHBytes, thisOHHandles, columns,
thisFHandles, txtProps, txtPropsWidth, exitOnFail);
try {
setHandle(root, null); // define the root value
} catch (IOException e) {
@ -133,20 +147,29 @@ public class kelondroTree extends kelondroRecords implements kelondroIndex {
this.objectOrder = objectOrder;
writeOrderType();
super.setLogger(log);
long objectbuffersize = objectCachePercent * buffersize / (nodeCachePercent + objectCachePercent);
long nodecachesize = objectbuffersize / (super.objectsize + 8 * columns.length);
this.objectCache = new kelondroObjectCache((int) nodecachesize, nodecachesize * 300 , 4*1024*1024);
}
public kelondroTree(File file, long buffersize) throws IOException {
// this opens a file with an existing tree file
super(file, buffersize);
super(file, nodeCachePercent * buffersize / (nodeCachePercent + objectCachePercent));
readOrderType();
super.setLogger(log);
long objectbuffersize = objectCachePercent * buffersize / (nodeCachePercent + objectCachePercent);
long nodecachesize = objectbuffersize / (super.objectsize + 8 * super.columns());
this.objectCache = new kelondroObjectCache((int) nodecachesize, nodecachesize * 300 , 4*1024*1024);
}
public kelondroTree(kelondroRA ra, long buffersize) throws IOException {
// this opens a file with an existing tree in a kelondroRA
super(ra, buffersize);
super(ra, nodeCachePercent * buffersize / (nodeCachePercent + objectCachePercent));
readOrderType();
super.setLogger(log);
long objectbuffersize = objectCachePercent * buffersize / (nodeCachePercent + objectCachePercent);
long nodecachesize = objectbuffersize / (super.objectsize + 8 * super.columns());
this.objectCache = new kelondroObjectCache((int) nodecachesize, nodecachesize * 300 , 4*1024*1024);
}
private void writeOrderType() {
@ -193,7 +216,8 @@ public class kelondroTree extends kelondroRecords implements kelondroIndex {
// Returns the value to which this map maps the specified key.
public byte[][] get(byte[] key) throws IOException {
// System.out.println("kelondroTree.get " + new String(key) + " in " + filename);
byte[][] result = null;
byte[][] result = (byte[][]) objectCache.get(key);
if (result != null) return result;
// writeLock.stay(2000, 1000);
synchronized (writeSearchObj) {
writeSearchObj.process(key);
@ -202,6 +226,7 @@ public class kelondroTree extends kelondroRecords implements kelondroIndex {
} else {
result = null;
}
objectCache.put(key, result);
}
// writeLock.release();
return result;
@ -378,6 +403,7 @@ public class kelondroTree extends kelondroRecords implements kelondroIndex {
if (newrow.length != columns()) throw new IllegalArgumentException("put: wrong row length " + newrow.length + "; must be " + columns());
// first try to find the key element in the database
synchronized(writeSearchObj) {
objectCache.put(newrow[0], newrow);
writeSearchObj.process(newrow[0]);
if (writeSearchObj.found()) {
// a node with this key exist. simply overwrite the content and return old content
@ -632,16 +658,17 @@ public class kelondroTree extends kelondroRecords implements kelondroIndex {
// Associates the specified value with the specified key in this map
public byte[] put(byte[] key, byte[] value) throws IOException {
byte[][] row = new byte[2][];
row[0] = key;
row[1] = value;
byte[][] ret = put(row);
if (ret == null) return null; else return ret[1];
byte[][] row = new byte[2][];
row[0] = key;
row[1] = value;
byte[][] ret = put(row);
if (ret == null) return null; else return ret[1];
}
// Removes the mapping for this key from this map if present (optional operation).
public byte[][] remove(byte[] key) throws IOException {
synchronized(writeSearchObj) {
objectCache.remove(key);
writeSearchObj.process(key);
if (writeSearchObj.found()) {
Node result = writeSearchObj.getMatcher();