From fe0c08455b7150163af46a4969ed0382e6da7cea Mon Sep 17 00:00:00 2001 From: orbiter Date: Mon, 30 May 2011 08:53:58 +0000 Subject: [PATCH] more concurrency (enhancement) hacks git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@7759 6c8d7289-2bf4-0310-a012-ef5d649a1542 --- source/de/anomic/search/TextSnippet.java | 4 ++-- source/net/yacy/cora/storage/ARC.java | 9 ++++++++ .../net/yacy/cora/storage/ConcurrentARC.java | 23 ++++++++++++++++--- source/net/yacy/cora/storage/SimpleARC.java | 19 +++++++++++++++ .../net/yacy/kelondro/blob/MapDataMining.java | 17 ++++++++++++++ source/net/yacy/kelondro/blob/MapHeap.java | 20 ++++++++-------- source/net/yacy/kelondro/data/word/Word.java | 14 +++++++---- source/net/yacy/kelondro/util/FileUtils.java | 6 ++++- 8 files changed, 92 insertions(+), 20 deletions(-) diff --git a/source/de/anomic/search/TextSnippet.java b/source/de/anomic/search/TextSnippet.java index 01a72171b..e7352a815 100644 --- a/source/de/anomic/search/TextSnippet.java +++ b/source/de/anomic/search/TextSnippet.java @@ -87,8 +87,8 @@ public class TextSnippet implements Comparable, Comparator extends Iterable> { */ public void insertIfAbsent(K s, V v); + /** + * put a value to the cache if there was not an entry before + * return a previous content value + * @param s + * @param v + * @return the value before inserting the new value + */ + public V putIfAbsent(K s, V v); + /** * put a value to the cache. * @param s diff --git a/source/net/yacy/cora/storage/ConcurrentARC.java b/source/net/yacy/cora/storage/ConcurrentARC.java index 7bb7dd377..4de97774f 100644 --- a/source/net/yacy/cora/storage/ConcurrentARC.java +++ b/source/net/yacy/cora/storage/ConcurrentARC.java @@ -101,6 +101,17 @@ public final class ConcurrentARC extends AbstractMap implements Map< public void insertIfAbsent(K s, V v) { this.arc[getPartition(s)].insertIfAbsent(s, v); } + + /** + * put a value to the cache if there was not an entry before + * return a previous content value + * @param s + * @param v + * @return the value before inserting the new value + */ + public V putIfAbsent(K s, V v) { + return this.arc[getPartition(s)].putIfAbsent(s, v); + } /** * put a value to the cache. @@ -206,18 +217,24 @@ public final class ConcurrentARC extends AbstractMap implements Map< public int hashCode() { return this.arc.hashCode(); } + + //private static String latestObject = ""; /** * return in which partition the Object belongs * This function uses the objects hashCode() function * except for byte[] keys - * @return the partitrion number + * @return the partition number */ private int getPartition(final Object x) { if (x instanceof byte[]) { int h = 0; for (byte c: (byte[])x) h = 31 * h + (c & 0xFF); - return h & mask; + int p = h & mask; + //String o = UTF8.String((byte[]) x); try { if (o.equals(latestObject)) throw new RuntimeException("ConcurrentARC: p = " + p + ", objectb = " + o); } catch (Exception e) { Log.logException(e); } latestObject = o; + return p; } - return x.hashCode() & mask; + int p = x.hashCode() & mask; + //String o = x.toString(); try { if (o.equals(latestObject)) throw new RuntimeException("ConcurrentARC: p = " + p + ", objecto = " + o); } catch (Exception e) { Log.logException(e); } latestObject = o; + return p; } } diff --git a/source/net/yacy/cora/storage/SimpleARC.java b/source/net/yacy/cora/storage/SimpleARC.java index fe801a6c2..40c20fc5b 100644 --- a/source/net/yacy/cora/storage/SimpleARC.java +++ b/source/net/yacy/cora/storage/SimpleARC.java @@ -91,6 +91,25 @@ abstract class SimpleARC extends AbstractMap implements Map, I } } + /** + * put a value to the cache if there was not an entry before + * return a previous content value + * @param s + * @param v + * @return the value before inserting the new value + */ + public V putIfAbsent(K s, V v) { + synchronized (this) { + V o = this.levelB.get(s); + if (o != null) return o; + o = this.levelA.get(s); + if (o != null) return o; + this.levelA.put(s, v); + assert (this.levelA.size() <= cacheSize); // the cache should shrink automatically + return null; + } + } + /** * put a value to the cache. * @param s diff --git a/source/net/yacy/kelondro/blob/MapDataMining.java b/source/net/yacy/kelondro/blob/MapDataMining.java index 1e261b677..e2697fc6e 100644 --- a/source/net/yacy/kelondro/blob/MapDataMining.java +++ b/source/net/yacy/kelondro/blob/MapDataMining.java @@ -279,6 +279,23 @@ public class MapDataMining extends MapHeap { super.delete(key); } +/* would be better but does not work (recursion) + @Override + public synchronized void delete(final byte[] key) throws IOException { + if (key == null) return; + + // update elementCount + Map map = super.remove(key); + if (map != null && (sortfields != null || longaccfields != null || floataccfields != null)) { + // update accumulators (subtract) + if ((longaccfields != null) || (floataccfields != null)) updateAcc(map, false); + + // remove from sortCluster + if (sortfields != null) deleteSortCluster(UTF8.String(key)); + } + } +*/ + private void deleteSortCluster(final String key) { if (key == null) return; ScoreMap cluster; diff --git a/source/net/yacy/kelondro/blob/MapHeap.java b/source/net/yacy/kelondro/blob/MapHeap.java index 18ff267da..a69db4761 100644 --- a/source/net/yacy/kelondro/blob/MapHeap.java +++ b/source/net/yacy/kelondro/blob/MapHeap.java @@ -69,7 +69,7 @@ public class MapHeap implements Map> { final int cachesize, char fillchar) throws IOException { this.blob = new Heap(heapFile, keylength, ordering, buffermax); - this.cache = new ConcurrentARC>(cachesize, Runtime.getRuntime().availableProcessors(), ordering); + this.cache = new ConcurrentARC>(cachesize, Math.max(32, 4 * Runtime.getRuntime().availableProcessors()), ordering); this.fillchar = fillchar; } @@ -285,6 +285,10 @@ public class MapHeap implements Map> { if (cache == null) return null; // case may appear during shutdown key = normalizeKey(key); + if (MemoryControl.shortStatus()) { + cache.clear(); + } + Map map; if (storeCache) { synchronized (this) { @@ -300,16 +304,12 @@ public class MapHeap implements Map> { throw new IOException(e.getMessage()); } - if (MemoryControl.shortStatus()) { - cache.clear(); - } else { - // write map to cache - cache.insert(key, map); - } + // write map to cache + cache.insert(key, map); + + // return value + return map; } - - // return value - return map; } else { byte[] b; synchronized (this) { diff --git a/source/net/yacy/kelondro/data/word/Word.java b/source/net/yacy/kelondro/data/word/Word.java index 58e820754..e55d17d33 100644 --- a/source/net/yacy/kelondro/data/word/Word.java +++ b/source/net/yacy/kelondro/data/word/Word.java @@ -30,7 +30,6 @@ import java.util.HashSet; import java.util.Iterator; import java.util.Locale; import java.util.Set; - import net.yacy.cora.storage.ARC; import net.yacy.cora.storage.ConcurrentARC; import net.yacy.document.LargeNumberCache; @@ -54,16 +53,22 @@ public class Word { * these hashes all shall be generated by base64.enhancedCoder */ public static final int commonHashLength = 12; - + private static final int hashCacheSize = Math.max(200000, Math.min(10000000, (int) (MemoryControl.available() / 20000L))); private static ARC hashCache = null; static { try { - hashCache = new ConcurrentARC(hashCacheSize, 2 * Runtime.getRuntime().availableProcessors()); + hashCache = new ConcurrentARC(hashCacheSize, Math.max(32, 4 * Runtime.getRuntime().availableProcessors())); } catch (OutOfMemoryError e) { - hashCache = new ConcurrentARC(1000, Runtime.getRuntime().availableProcessors()); + hashCache = new ConcurrentARC(1000, Math.max(8, 2 * Runtime.getRuntime().availableProcessors())); } } + /* + private static ConcurrentHashMap hashCache = null; + static { + hashCache = new ConcurrentHashMap(); + } + */ // object carries statistics for words and sentences public int count; // number of occurrences @@ -122,6 +127,7 @@ public class Word { if (MemoryControl.shortStatus()) { hashCache.clear(); } else { + //hashCache.putIfAbsent(wordlc, h); // prevent expensive MD5 computation and encoding hashCache.insertIfAbsent(wordlc, h); // prevent expensive MD5 computation and encoding } return h; diff --git a/source/net/yacy/kelondro/util/FileUtils.java b/source/net/yacy/kelondro/util/FileUtils.java index cd0c1c223..d79fe40ce 100644 --- a/source/net/yacy/kelondro/util/FileUtils.java +++ b/source/net/yacy/kelondro/util/FileUtils.java @@ -142,7 +142,10 @@ public final class FileUtils { } public static int copy(final Reader source, final Writer dest) throws IOException { + assert source != null; + assert dest != null; if (source == null) throw new IOException("source is null"); + if (dest == null) throw new IOException("dest is null"); final char[] buffer = new char[DEFAULT_BUFFER_SIZE]; int count = 0; int n = 0; @@ -153,9 +156,10 @@ public final class FileUtils { } dest.flush(); } catch (final Exception e) { + assert e != null; // an "sun.io.MalformedInputException: Missing byte-order mark" - exception may occur here //Log.logException(e); - throw new IOException(e == null ? "null" : e.getMessage() == null ? e.toString() : e.getMessage(), e); + //throw new IOException(e == null ? "null" : e.getMessage() == null ? e.toString() : e.getMessage(), e); } return count; }