From 1b86d06d1edfef231dc88f67a456323cb108b33d Mon Sep 17 00:00:00 2001 From: orbiter Date: Wed, 26 Oct 2011 10:07:16 +0000 Subject: [PATCH] fix for http://bugs.yacy.net/view.php?id=62 git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@8004 6c8d7289-2bf4-0310-a012-ef5d649a1542 --- htroot/PerformanceMemory_p.java | 55 ++--- source/de/anomic/crawler/Balancer.java | 2 +- source/de/anomic/crawler/NoticedURL.java | 197 +++++++++--------- source/net/yacy/kelondro/index/Cache.java | 63 ++++-- .../net/yacy/kelondro/order/Base64Order.java | 6 - source/net/yacy/kelondro/order/ByteOrder.java | 4 - source/net/yacy/kelondro/table/Table.java | 21 +- 7 files changed, 182 insertions(+), 166 deletions(-) diff --git a/htroot/PerformanceMemory_p.java b/htroot/PerformanceMemory_p.java index bf9f10bb6..a10ba0d2a 100644 --- a/htroot/PerformanceMemory_p.java +++ b/htroot/PerformanceMemory_p.java @@ -100,25 +100,25 @@ public class PerformanceMemory_p { // write table for Table index sizes Iterator i = Table.filenames(); String filename; - Map map; + Map mapx; int p, c = 0; long mem, totalmem = 0; while (i.hasNext()) { filename = i.next(); - map = Table.memoryStats(filename); + mapx = Table.memoryStats(filename); prop.put("EcoList_" + c + "_tableIndexPath", ((p = filename.indexOf("DATA")) < 0) ? filename : filename.substring(p)); - prop.putNum("EcoList_" + c + "_tableSize", map.get("tableSize")); + prop.putNum("EcoList_" + c + "_tableSize", mapx.get(Table.StatKeys.tableSize)); - assert map.get("tableKeyMem") != null : map; - mem = Long.parseLong(map.get("tableKeyMem")); + assert mapx.get(Table.StatKeys.tableKeyMem) != null : mapx; + mem = Long.parseLong(mapx.get(Table.StatKeys.tableKeyMem)); totalmem += mem; prop.put("EcoList_" + c + "_tableKeyMem", Formatter.bytesToString(mem)); - prop.put("EcoList_" + c + "_tableKeyChunkSize", map.get("tableKeyChunkSize")); + prop.put("EcoList_" + c + "_tableKeyChunkSize", mapx.get(Table.StatKeys.tableKeyChunkSize)); - mem = Long.parseLong(map.get("tableValueMem")); + mem = Long.parseLong(mapx.get(Table.StatKeys.tableValueMem)); totalmem += mem; prop.put("EcoList_" + c + "_tableValueMem", Formatter.bytesToString(mem)); - prop.put("EcoList_" + c + "_tableValueChunkSize", map.get("tableValueChunkSize")); + prop.put("EcoList_" + c + "_tableValueChunkSize", mapx.get(Table.StatKeys.tableValueChunkSize)); c++; } @@ -159,36 +159,37 @@ public class PerformanceMemory_p { c = 0; long missmem, totalmissmem = 0; totalhitmem = 0; + Map mapy; while (i.hasNext()) { filename = i.next(); - map = Cache.memoryStats(filename); + mapy = Cache.memoryStats(filename); prop.put("ObjectList_" + c + "_objectCachePath", ((p = filename.indexOf("DATA")) < 0) ? filename : filename.substring(p)); // hit cache - hitmem = Long.parseLong(map.get("objectHitMem")); + hitmem = Long.parseLong(mapy.get(Cache.StatKeys.objectHitMem)); totalhitmem += hitmem; - prop.put("ObjectList_" + c + "_objectHitChunkSize", map.get("objectHitChunkSize")); - prop.putNum("ObjectList_" + c + "_objectHitCacheCount", map.get("objectHitCacheCount")); + prop.put("ObjectList_" + c + "_objectHitChunkSize", mapy.get(Cache.StatKeys.objectHitChunkSize)); + prop.putNum("ObjectList_" + c + "_objectHitCacheCount", mapy.get(Cache.StatKeys.objectHitCacheCount)); prop.put("ObjectList_" + c + "_objectHitCacheMem", Formatter.bytesToString(hitmem)); - prop.putNum("ObjectList_" + c + "_objectHitCacheReadHit", map.get("objectHitCacheReadHit")); - prop.putNum("ObjectList_" + c + "_objectHitCacheReadMiss", map.get("objectHitCacheReadMiss")); - prop.putNum("ObjectList_" + c + "_objectHitCacheWriteUnique", map.get("objectHitCacheWriteUnique")); - prop.putNum("ObjectList_" + c + "_objectHitCacheWriteDouble", map.get("objectHitCacheWriteDouble")); - prop.putNum("ObjectList_" + c + "_objectHitCacheDeletes", map.get("objectHitCacheDeletes")); - prop.putNum("ObjectList_" + c + "_objectHitCacheFlushes", map.get("objectHitCacheFlushes")); + prop.putNum("ObjectList_" + c + "_objectHitCacheReadHit", mapy.get(Cache.StatKeys.objectHitCacheReadHit)); + prop.putNum("ObjectList_" + c + "_objectHitCacheReadMiss", mapy.get(Cache.StatKeys.objectHitCacheReadMiss)); + prop.putNum("ObjectList_" + c + "_objectHitCacheWriteUnique", mapy.get(Cache.StatKeys.objectHitCacheWriteUnique)); + prop.putNum("ObjectList_" + c + "_objectHitCacheWriteDouble", mapy.get(Cache.StatKeys.objectHitCacheWriteDouble)); + prop.putNum("ObjectList_" + c + "_objectHitCacheDeletes", mapy.get(Cache.StatKeys.objectHitCacheDeletes)); + prop.putNum("ObjectList_" + c + "_objectHitCacheFlushes", mapy.get(Cache.StatKeys.objectHitCacheFlushes)); // miss cache - missmem = Long.parseLong(map.get("objectMissMem")); + missmem = Long.parseLong(mapy.get(Cache.StatKeys.objectMissMem)); totalmissmem += missmem; - prop.put("ObjectList_" + c + "_objectMissChunkSize", map.get("objectMissChunkSize")); - prop.putNum("ObjectList_" + c + "_objectMissCacheCount", map.get("objectMissCacheCount")); + prop.put("ObjectList_" + c + "_objectMissChunkSize", mapy.get(Cache.StatKeys.objectMissChunkSize)); + prop.putNum("ObjectList_" + c + "_objectMissCacheCount", mapy.get(Cache.StatKeys.objectMissCacheCount)); prop.putHTML("ObjectList_" + c + "_objectMissCacheMem", Formatter.bytesToString(missmem)); - prop.putNum("ObjectList_" + c + "_objectMissCacheReadHit", map.get("objectMissCacheReadHit")); - prop.putNum("ObjectList_" + c + "_objectMissCacheReadMiss", map.get("objectMissCacheReadMiss")); - prop.putNum("ObjectList_" + c + "_objectMissCacheWriteUnique", map.get("objectMissCacheWriteUnique")); - prop.putNum("ObjectList_" + c + "_objectMissCacheWriteDouble", map.get("objectMissCacheWriteDouble")); - prop.putNum("ObjectList_" + c + "_objectMissCacheDeletes", map.get("objectMissCacheDeletes")); - //prop.put("ObjectList_" + c + "_objectMissCacheFlushes", map.get("objectMissCacheFlushes")); + prop.putNum("ObjectList_" + c + "_objectMissCacheReadHit", mapy.get(Cache.StatKeys.objectMissCacheReadHit)); + prop.putNum("ObjectList_" + c + "_objectMissCacheReadMiss", mapy.get(Cache.StatKeys.objectMissCacheReadMiss)); + prop.putNum("ObjectList_" + c + "_objectMissCacheWriteUnique", mapy.get(Cache.StatKeys.objectMissCacheWriteUnique)); + prop.putNum("ObjectList_" + c + "_objectMissCacheWriteDouble", mapy.get(Cache.StatKeys.objectMissCacheWriteDouble)); + prop.putNum("ObjectList_" + c + "_objectMissCacheDeletes", mapy.get(Cache.StatKeys.objectMissCacheDeletes)); + //prop.put("ObjectList_" + c + "_objectMissCacheFlushes", mapy.get(Cache.StatKeys.objectMissCacheFlushes)); c++; } diff --git a/source/de/anomic/crawler/Balancer.java b/source/de/anomic/crawler/Balancer.java index 1a0714cb9..3e27dd0cc 100644 --- a/source/de/anomic/crawler/Balancer.java +++ b/source/de/anomic/crawler/Balancer.java @@ -165,7 +165,7 @@ public class Balancer { // returns number of deletions // first find a list of url hashes that shall be deleted - final HandleSet urlHashes = Base64Order.enhancedCoder.getHandleSet(this.urlFileIndex.row().primaryKeyLength, 100); + final HandleSet urlHashes = new HandleSet(this.urlFileIndex.row().primaryKeyLength, Base64Order.enhancedCoder, 100); final long terminate = (timeout > 0) ? System.currentTimeMillis() + timeout : Long.MAX_VALUE; synchronized (this) { final Iterator i = this.urlFileIndex.rows(); diff --git a/source/de/anomic/crawler/NoticedURL.java b/source/de/anomic/crawler/NoticedURL.java index 2cc6bbf99..8d64937fd 100755 --- a/source/de/anomic/crawler/NoticedURL.java +++ b/source/de/anomic/crawler/NoticedURL.java @@ -1,4 +1,4 @@ -// NoticedURL.java +// NoticedURL.java // ----------------------- // part of YaCy // (C) by Michael Peter Christen; mc@yacy.net @@ -38,23 +38,22 @@ import net.yacy.kelondro.index.HandleSet; import net.yacy.kelondro.index.RowSpaceExceededException; import net.yacy.kelondro.logging.Log; import net.yacy.kelondro.order.Base64Order; - import de.anomic.crawler.retrieval.Request; public class NoticedURL { - + public enum StackType { NULL, CORE, LIMIT, OVERHANG, REMOTE, NOLOAD, IMAGE, MOVIE, MUSIC; } public static final long minimumLocalDeltaInit = 10; // the minimum time difference between access of the same local domain public static final long minimumGlobalDeltaInit = 500; // the minimum time difference between access of the same global domain - + private Balancer coreStack; // links found by crawling to depth-1 private Balancer limitStack; // links found by crawling at target depth private Balancer remoteStack; // links from remote crawl orders private Balancer noloadStack; // links that are not passed to a loader; the index will be generated from the Request entry - + public NoticedURL( final File cachePath, final Set myAgentIDs, @@ -71,98 +70,98 @@ public class NoticedURL { public long getMinimumLocalDelta() { return this.coreStack.getMinimumLocalDelta(); } - + public long getMinimumGlobalDelta() { return this.coreStack.getMinimumGlobalDelta(); } - + public void setMinimumDelta(final long minimumLocalDelta, final long minimumGlobalDelta) { this.coreStack.setMinimumDelta(minimumLocalDelta, minimumGlobalDelta); this.limitStack.setMinimumDelta(minimumLocalDelta, minimumGlobalDelta); this.remoteStack.setMinimumDelta(minimumLocalDelta, minimumGlobalDelta); this.noloadStack.setMinimumDelta(minimumLocalDelta, minimumGlobalDelta); } - + public void clear() { Log.logInfo("NoticedURL", "CLEARING ALL STACKS"); - coreStack.clear(); - limitStack.clear(); - remoteStack.clear(); - noloadStack.clear(); + this.coreStack.clear(); + this.limitStack.clear(); + this.remoteStack.clear(); + this.noloadStack.clear(); } - + public void close() { Log.logInfo("NoticedURL", "CLOSING ALL STACKS"); - if (coreStack != null) { - coreStack.close(); - coreStack = null; + if (this.coreStack != null) { + this.coreStack.close(); + this.coreStack = null; } - if (limitStack != null) { - limitStack.close(); - limitStack = null; + if (this.limitStack != null) { + this.limitStack.close(); + this.limitStack = null; } //overhangStack.close(); - if (remoteStack != null) { - remoteStack.close(); - remoteStack = null; + if (this.remoteStack != null) { + this.remoteStack.close(); + this.remoteStack = null; } - if (noloadStack != null) { - noloadStack.close(); - noloadStack = null; + if (this.noloadStack != null) { + this.noloadStack.close(); + this.noloadStack = null; } } - + @Override protected void finalize() throws Throwable { - if ((coreStack != null) || (limitStack != null) || (remoteStack != null)) { + if ((this.coreStack != null) || (this.limitStack != null) || (this.remoteStack != null)) { Log.logWarning("plasmaCrawlNURL", "NURL stack closed by finalizer"); close(); } super.finalize(); } - + public boolean notEmpty() { - return coreStack.notEmpty() || limitStack.notEmpty() || remoteStack.notEmpty() || noloadStack.notEmpty(); + return this.coreStack.notEmpty() || this.limitStack.notEmpty() || this.remoteStack.notEmpty() || this.noloadStack.notEmpty(); } - + public boolean notEmptyLocal() { - return coreStack.notEmpty() || limitStack.notEmpty() || noloadStack.notEmpty(); + return this.coreStack.notEmpty() || this.limitStack.notEmpty() || this.noloadStack.notEmpty(); } - + public int size() { // this does not count the overhang stack size - return ((coreStack == null) ? 0 : coreStack.size()) + ((limitStack == null) ? 0 : limitStack.size()) + ((remoteStack == null) ? 0 : remoteStack.size()); + return ((this.coreStack == null) ? 0 : this.coreStack.size()) + ((this.limitStack == null) ? 0 : this.limitStack.size()) + ((this.remoteStack == null) ? 0 : this.remoteStack.size()); } public boolean isEmpty() { - if (coreStack == null) return true; - if (!coreStack.isEmpty()) return false; - if (!limitStack.isEmpty()) return false; - if (!remoteStack.isEmpty()) return false; - if (!noloadStack.isEmpty()) return false; + if (this.coreStack == null) return true; + if (!this.coreStack.isEmpty()) return false; + if (!this.limitStack.isEmpty()) return false; + if (!this.remoteStack.isEmpty()) return false; + if (!this.noloadStack.isEmpty()) return false; return true; } - + public int stackSize(final StackType stackType) { switch (stackType) { - case NOLOAD: return (noloadStack == null) ? 0 : noloadStack.size(); - case CORE: return (coreStack == null) ? 0 : coreStack.size(); - case LIMIT: return (limitStack == null) ? 0 : limitStack.size(); + case NOLOAD: return (this.noloadStack == null) ? 0 : this.noloadStack.size(); + case CORE: return (this.coreStack == null) ? 0 : this.coreStack.size(); + case LIMIT: return (this.limitStack == null) ? 0 : this.limitStack.size(); case OVERHANG: return 0; - case REMOTE: return (remoteStack == null) ? 0 : remoteStack.size(); + case REMOTE: return (this.remoteStack == null) ? 0 : this.remoteStack.size(); default: return -1; } } public boolean existsInStack(final byte[] urlhashb) { return - coreStack.has(urlhashb) || - limitStack.has(urlhashb) || - //overhangStack.has(urlhashb) || - remoteStack.has(urlhashb) || - noloadStack.has(urlhashb); + this.coreStack.has(urlhashb) || + this.limitStack.has(urlhashb) || + //overhangStack.has(urlhashb) || + this.remoteStack.has(urlhashb) || + this.noloadStack.has(urlhashb); } - + /** * push a crawl request on one of the different crawl stacks * @param stackType @@ -173,13 +172,13 @@ public class NoticedURL { try { switch (stackType) { case CORE: - return coreStack.push(entry); + return this.coreStack.push(entry); case LIMIT: - return limitStack.push(entry); + return this.limitStack.push(entry); case REMOTE: - return remoteStack.push(entry); + return this.remoteStack.push(entry); case NOLOAD: - return noloadStack.push(entry); + return this.noloadStack.push(entry); default: return "stack type unknown"; } @@ -191,13 +190,13 @@ public class NoticedURL { public Request get(final byte[] urlhash) { Request entry = null; - try {if ((entry = noloadStack.get(urlhash)) != null) return entry;} catch (final IOException e) {} - try {if ((entry = coreStack.get(urlhash)) != null) return entry;} catch (final IOException e) {} - try {if ((entry = limitStack.get(urlhash)) != null) return entry;} catch (final IOException e) {} - try {if ((entry = remoteStack.get(urlhash)) != null) return entry;} catch (final IOException e) {} + try {if ((entry = this.noloadStack.get(urlhash)) != null) return entry;} catch (final IOException e) {} + try {if ((entry = this.coreStack.get(urlhash)) != null) return entry;} catch (final IOException e) {} + try {if ((entry = this.limitStack.get(urlhash)) != null) return entry;} catch (final IOException e) {} + try {if ((entry = this.remoteStack.get(urlhash)) != null) return entry;} catch (final IOException e) {} return null; } - + /** * remove a CrawlEntry by a given hash. Usage of this method is not encouraged, * because the underlying data structure (crawl stacks) cannot handle removals very good. @@ -206,54 +205,54 @@ public class NoticedURL { */ public boolean removeByURLHash(final byte[] urlhashBytes) { try { - final HandleSet urlHashes = Base64Order.enhancedCoder.getHandleSet(12, 1); + final HandleSet urlHashes = new HandleSet(12, Base64Order.enhancedCoder, 1); urlHashes.put(urlhashBytes); boolean ret = false; - try {ret |= noloadStack.remove(urlHashes) > 0;} catch (final IOException e) {} - try {ret |= coreStack.remove(urlHashes) > 0;} catch (final IOException e) {} - try {ret |= limitStack.remove(urlHashes) > 0;} catch (final IOException e) {} - try {ret |= remoteStack.remove(urlHashes) > 0;} catch (final IOException e) {} + try {ret |= this.noloadStack.remove(urlHashes) > 0;} catch (final IOException e) {} + try {ret |= this.coreStack.remove(urlHashes) > 0;} catch (final IOException e) {} + try {ret |= this.limitStack.remove(urlHashes) > 0;} catch (final IOException e) {} + try {ret |= this.remoteStack.remove(urlHashes) > 0;} catch (final IOException e) {} return ret; - } catch (RowSpaceExceededException e) { + } catch (final RowSpaceExceededException e) { Log.logException(e); return false; } } - + public int removeByProfileHandle(final String handle, final long timeout) throws RowSpaceExceededException { int removed = 0; - try {removed += noloadStack.removeAllByProfileHandle(handle, timeout);} catch (final IOException e) {} - try {removed += coreStack.removeAllByProfileHandle(handle, timeout);} catch (final IOException e) {} - try {removed += limitStack.removeAllByProfileHandle(handle, timeout);} catch (final IOException e) {} - try {removed += remoteStack.removeAllByProfileHandle(handle, timeout);} catch (final IOException e) {} + try {removed += this.noloadStack.removeAllByProfileHandle(handle, timeout);} catch (final IOException e) {} + try {removed += this.coreStack.removeAllByProfileHandle(handle, timeout);} catch (final IOException e) {} + try {removed += this.limitStack.removeAllByProfileHandle(handle, timeout);} catch (final IOException e) {} + try {removed += this.remoteStack.removeAllByProfileHandle(handle, timeout);} catch (final IOException e) {} return removed; } - + public List top(final StackType stackType, final int count) { switch (stackType) { - case CORE: return top(coreStack, count); - case LIMIT: return top(limitStack, count); - case REMOTE: return top(remoteStack, count); - case NOLOAD: return top(noloadStack, count); - default: return null; - } - } - - public Request pop(final StackType stackType, final boolean delay, CrawlSwitchboard cs) throws IOException { - switch (stackType) { - case CORE: return pop(coreStack, delay, cs); - case LIMIT: return pop(limitStack, delay, cs); - case REMOTE: return pop(remoteStack, delay, cs); - case NOLOAD: return pop(noloadStack, false, cs); + case CORE: return top(this.coreStack, count); + case LIMIT: return top(this.limitStack, count); + case REMOTE: return top(this.remoteStack, count); + case NOLOAD: return top(this.noloadStack, count); default: return null; } } - public void shift(final StackType fromStack, final StackType toStack, CrawlSwitchboard cs) { + public Request pop(final StackType stackType, final boolean delay, final CrawlSwitchboard cs) throws IOException { + switch (stackType) { + case CORE: return pop(this.coreStack, delay, cs); + case LIMIT: return pop(this.limitStack, delay, cs); + case REMOTE: return pop(this.remoteStack, delay, cs); + case NOLOAD: return pop(this.noloadStack, false, cs); + default: return null; + } + } + + public void shift(final StackType fromStack, final StackType toStack, final CrawlSwitchboard cs) { try { final Request entry = pop(fromStack, false, cs); if (entry != null) { - String warning = push(toStack, entry); + final String warning = push(toStack, entry); if (warning != null) { Log.logWarning("NoticedURL", "shift from " + fromStack + " to " + toStack + ": " + warning); } @@ -266,15 +265,15 @@ public class NoticedURL { public void clear(final StackType stackType) { Log.logInfo("NoticedURL", "CLEARING STACK " + stackType); switch (stackType) { - case CORE: coreStack.clear(); break; - case LIMIT: limitStack.clear(); break; - case REMOTE: remoteStack.clear(); break; - case NOLOAD: noloadStack.clear(); break; + case CORE: this.coreStack.clear(); break; + case LIMIT: this.limitStack.clear(); break; + case REMOTE: this.remoteStack.clear(); break; + case NOLOAD: this.noloadStack.clear(); break; default: return; } } - - private Request pop(final Balancer balancer, final boolean delay, CrawlSwitchboard cs) throws IOException { + + private Request pop(final Balancer balancer, final boolean delay, final CrawlSwitchboard cs) throws IOException { // this is a filo - pop int s; Request entry; @@ -295,24 +294,24 @@ public class NoticedURL { } return null; } - + private List top(final Balancer balancer, int count) { // this is a filo - top if (count > balancer.size()) count = balancer.size(); return balancer.top(count); } - + public Iterator iterator(final StackType stackType) { // returns an iterator of plasmaCrawlBalancerEntry Objects try {switch (stackType) { - case CORE: return coreStack.iterator(); - case LIMIT: return limitStack.iterator(); - case REMOTE: return remoteStack.iterator(); - case NOLOAD: return noloadStack.iterator(); + case CORE: return this.coreStack.iterator(); + case LIMIT: return this.limitStack.iterator(); + case REMOTE: return this.remoteStack.iterator(); + case NOLOAD: return this.noloadStack.iterator(); default: return null; }} catch (final IOException e) { return new HashSet().iterator(); } } - + } diff --git a/source/net/yacy/kelondro/index/Cache.java b/source/net/yacy/kelondro/index/Cache.java index c1e77155a..120aca368 100644 --- a/source/net/yacy/kelondro/index/Cache.java +++ b/source/net/yacy/kelondro/index/Cache.java @@ -132,7 +132,28 @@ public final class Cache implements Index, Iterable { return objectTracker.keySet().iterator(); } - public static final Map memoryStats(final String filename) { + public enum StatKeys { + objectHitChunkSize, + objectHitCacheCount, + objectHitMem, + objectHitCacheReadHit, + objectHitCacheReadMiss, + objectHitCacheWriteUnique, + objectHitCacheWriteDouble, + objectHitCacheDeletes, + objectHitCacheFlushes, + objectMissChunkSize, + objectMissCacheCount, + objectMissMem, + objectMissCacheReadHit, + objectMissCacheReadMiss, + objectMissCacheWriteUnique, + objectMissCacheWriteDouble, + objectMissCacheDeletes, + objectMissCacheFlushes; + } + + public static final Map memoryStats(final String filename) { // returns a map for each file in the tracker; // the map represents properties for each record oobjects, // i.e. for cache memory allocation @@ -140,28 +161,28 @@ public final class Cache implements Index, Iterable { return theObjectsCache.memoryStats(); } - private final Map memoryStats() { + private final Map memoryStats() { // returns statistical data about this object - final HashMap map = new HashMap(20); - map.put("objectHitChunkSize", (this.readHitCache == null) ? "0" : Integer.toString(this.readHitCache.rowdef.objectsize)); - map.put("objectHitCacheCount", (this.readHitCache == null) ? "0" : Integer.toString(this.readHitCache.size())); - map.put("objectHitMem", (this.readHitCache == null) ? "0" : Long.toString(this.readHitCache.rowdef.objectsize * this.readHitCache.size())); - map.put("objectHitCacheReadHit", Integer.toString(this.readHit)); - map.put("objectHitCacheReadMiss", Integer.toString(this.readMiss)); - map.put("objectHitCacheWriteUnique", Integer.toString(this.writeUnique)); - map.put("objectHitCacheWriteDouble", Integer.toString(this.writeDouble)); - map.put("objectHitCacheDeletes", Integer.toString(this.cacheDelete)); - map.put("objectHitCacheFlushes", Integer.toString(this.cacheFlush)); + final HashMap map = new HashMap(20); + map.put(StatKeys.objectHitChunkSize, (this.readHitCache == null) ? "0" : Integer.toString(this.readHitCache.rowdef.objectsize)); + map.put(StatKeys.objectHitCacheCount, (this.readHitCache == null) ? "0" : Integer.toString(this.readHitCache.size())); + map.put(StatKeys.objectHitMem, (this.readHitCache == null) ? "0" : Long.toString(this.readHitCache.rowdef.objectsize * this.readHitCache.size())); + map.put(StatKeys.objectHitCacheReadHit, Integer.toString(this.readHit)); + map.put(StatKeys.objectHitCacheReadMiss, Integer.toString(this.readMiss)); + map.put(StatKeys.objectHitCacheWriteUnique, Integer.toString(this.writeUnique)); + map.put(StatKeys.objectHitCacheWriteDouble, Integer.toString(this.writeDouble)); + map.put(StatKeys.objectHitCacheDeletes, Integer.toString(this.cacheDelete)); + map.put(StatKeys.objectHitCacheFlushes, Integer.toString(this.cacheFlush)); - map.put("objectMissChunkSize", (this.readMissCache == null) ? "0" : Integer.toString(this.readMissCache.rowdef.objectsize)); - map.put("objectMissCacheCount", (this.readMissCache == null) ? "0" : Integer.toString(this.readMissCache.size())); - map.put("objectMissMem", (this.readMissCache == null) ? "0" : Long.toString(this.readMissCache.rowdef.objectsize * this.readMissCache.size())); - map.put("objectMissCacheReadHit", Integer.toString(this.hasnotHit)); - map.put("objectMissCacheReadMiss", Integer.toString(this.hasnotMiss)); - map.put("objectMissCacheWriteUnique", Integer.toString(this.hasnotUnique)); - map.put("objectMissCacheWriteDouble", Integer.toString(this.hasnotDouble)); - map.put("objectMissCacheDeletes", Integer.toString(this.hasnotDelete)); - map.put("objectMissCacheFlushes", "0"); // a miss cache flush can only happen if we have a deletion cache (which we dont have) + map.put(StatKeys.objectMissChunkSize, (this.readMissCache == null) ? "0" : Integer.toString(this.readMissCache.rowdef.objectsize)); + map.put(StatKeys.objectMissCacheCount, (this.readMissCache == null) ? "0" : Integer.toString(this.readMissCache.size())); + map.put(StatKeys.objectMissMem, (this.readMissCache == null) ? "0" : Long.toString(this.readMissCache.rowdef.objectsize * this.readMissCache.size())); + map.put(StatKeys.objectMissCacheReadHit, Integer.toString(this.hasnotHit)); + map.put(StatKeys.objectMissCacheReadMiss, Integer.toString(this.hasnotMiss)); + map.put(StatKeys.objectMissCacheWriteUnique, Integer.toString(this.hasnotUnique)); + map.put(StatKeys.objectMissCacheWriteDouble, Integer.toString(this.hasnotDouble)); + map.put(StatKeys.objectMissCacheDeletes, Integer.toString(this.hasnotDelete)); + map.put(StatKeys.objectMissCacheFlushes, "0"); // a miss cache flush can only happen if we have a deletion cache (which we dont have) // future feature .. map.put("objectElderTimeRead", index.profile().) return map; diff --git a/source/net/yacy/kelondro/order/Base64Order.java b/source/net/yacy/kelondro/order/Base64Order.java index e9e86f7dd..5b462a24f 100644 --- a/source/net/yacy/kelondro/order/Base64Order.java +++ b/source/net/yacy/kelondro/order/Base64Order.java @@ -32,8 +32,6 @@ import java.util.Comparator; import net.yacy.cora.document.UTF8; import net.yacy.cora.ranking.AbstractOrder; import net.yacy.cora.ranking.Order; -import net.yacy.kelondro.index.HandleSet; -import net.yacy.kelondro.index.RowSpaceExceededException; public class Base64Order extends AbstractOrder implements ByteOrder, Comparator, Cloneable { @@ -86,10 +84,6 @@ public class Base64Order extends AbstractOrder implements ByteOrder, Com } } - public HandleSet getHandleSet(final int keylength, final int space) throws RowSpaceExceededException { - return new HandleSet(keylength, this, space); - } - public static byte[] zero(int length) { final byte[] z = new byte[length]; while (length > 0) { diff --git a/source/net/yacy/kelondro/order/ByteOrder.java b/source/net/yacy/kelondro/order/ByteOrder.java index 6178fa73c..630aab691 100644 --- a/source/net/yacy/kelondro/order/ByteOrder.java +++ b/source/net/yacy/kelondro/order/ByteOrder.java @@ -28,8 +28,6 @@ package net.yacy.kelondro.order; import net.yacy.cora.ranking.Order; -import net.yacy.kelondro.index.HandleSet; -import net.yacy.kelondro.index.RowSpaceExceededException; public interface ByteOrder extends Order { @@ -50,8 +48,6 @@ public interface ByteOrder extends Order { public long cardinal(final byte[] a, int off, int len); - public HandleSet getHandleSet(final int keylength, final int space) throws RowSpaceExceededException; - public byte[] smallest(byte[] a, byte[] b); public byte[] largest(byte[] a, byte[] b); diff --git a/source/net/yacy/kelondro/table/Table.java b/source/net/yacy/kelondro/table/Table.java index 0f77eb8b6..98711b91e 100644 --- a/source/net/yacy/kelondro/table/Table.java +++ b/source/net/yacy/kelondro/table/Table.java @@ -303,7 +303,7 @@ public class Table implements Index, Iterable { return tableTracker.keySet().iterator(); } - public static final Map memoryStats(final String filename) { + public static final Map memoryStats(final String filename) { // returns a map for each file in the tracker; // the map represents properties for each record objects, // i.e. for cache memory allocation @@ -311,18 +311,22 @@ public class Table implements Index, Iterable { return theTABLE.memoryStats(); } - private final Map memoryStats() { + public enum StatKeys { + tableSize, tableKeyChunkSize, tableKeyMem, tableValueChunkSize, tableValueMem + } + + private final Map memoryStats() { // returns statistical data about this object synchronized (this) { assert this.table == null || this.table.size() == this.index.size() : "table.size() = " + this.table.size() + ", index.size() = " + this.index.size(); } - final HashMap map = new HashMap(8); + final HashMap map = new HashMap(8); if (this.index == null) return map; // possibly closed or beeing closed - map.put("tableSize", Integer.toString(this.index.size())); - map.put("tableKeyChunkSize", Integer.toString(this.index.row().objectsize)); - map.put("tableKeyMem", Integer.toString(this.index.row().objectsize * this.index.size())); - map.put("tableValueChunkSize", (this.table == null) ? "0" : Integer.toString(this.table.row().objectsize)); - map.put("tableValueMem", (this.table == null) ? "0" : Integer.toString(this.table.row().objectsize * this.table.size())); + map.put(StatKeys.tableSize, Integer.toString(this.index.size())); + map.put(StatKeys.tableKeyChunkSize, Integer.toString(this.index.row().objectsize)); + map.put(StatKeys.tableKeyMem, Integer.toString(this.index.row().objectsize * this.index.size())); + map.put(StatKeys.tableValueChunkSize, (this.table == null) ? "0" : Integer.toString(this.table.row().objectsize)); + map.put(StatKeys.tableValueMem, (this.table == null) ? "0" : Integer.toString(this.table.row().objectsize * this.table.size())); return map; } @@ -814,6 +818,7 @@ public class Table implements Index, Iterable { this.file.close(); this.file = null; FileUtils.deletedelete(f); + tableTracker.remove(f.getName()); // make new file FileOutputStream fos = null;