mirror of
https://github.com/yacy/yacy_search_server.git
synced 2024-09-19 00:01:41 +02:00
more concurrency (enhancement) hacks
git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@7759 6c8d7289-2bf4-0310-a012-ef5d649a1542
This commit is contained in:
parent
0e9a99cb05
commit
fe0c08455b
|
@ -87,8 +87,8 @@ public class TextSnippet implements Comparable<TextSnippet>, Comparator<TextSnip
|
|||
public void put(final String wordhashes, final String urlhash, final String snippet) {
|
||||
// generate key
|
||||
final String key = urlhash + wordhashes;
|
||||
|
||||
// do nothing if snippet is known or learn new snippet
|
||||
|
||||
// do nothing if snippet is known or otherwise learn new snippet
|
||||
cache.insertIfAbsent(key, snippet);
|
||||
}
|
||||
|
||||
|
|
|
@ -63,6 +63,15 @@ public interface ARC<K, V> extends Iterable<Map.Entry<K, V>> {
|
|||
*/
|
||||
public void insertIfAbsent(K s, V v);
|
||||
|
||||
/**
|
||||
* put a value to the cache if there was not an entry before
|
||||
* return a previous content value
|
||||
* @param s
|
||||
* @param v
|
||||
* @return the value before inserting the new value
|
||||
*/
|
||||
public V putIfAbsent(K s, V v);
|
||||
|
||||
/**
|
||||
* put a value to the cache.
|
||||
* @param s
|
||||
|
|
|
@ -101,6 +101,17 @@ public final class ConcurrentARC<K, V> extends AbstractMap<K, V> implements Map<
|
|||
public void insertIfAbsent(K s, V v) {
|
||||
this.arc[getPartition(s)].insertIfAbsent(s, v);
|
||||
}
|
||||
|
||||
/**
|
||||
* put a value to the cache if there was not an entry before
|
||||
* return a previous content value
|
||||
* @param s
|
||||
* @param v
|
||||
* @return the value before inserting the new value
|
||||
*/
|
||||
public V putIfAbsent(K s, V v) {
|
||||
return this.arc[getPartition(s)].putIfAbsent(s, v);
|
||||
}
|
||||
|
||||
/**
|
||||
* put a value to the cache.
|
||||
|
@ -206,18 +217,24 @@ public final class ConcurrentARC<K, V> extends AbstractMap<K, V> implements Map<
|
|||
public int hashCode() {
|
||||
return this.arc.hashCode();
|
||||
}
|
||||
|
||||
//private static String latestObject = "";
|
||||
/**
|
||||
* return in which partition the Object belongs
|
||||
* This function uses the objects hashCode() function
|
||||
* except for byte[] keys
|
||||
* @return the partitrion number
|
||||
* @return the partition number
|
||||
*/
|
||||
private int getPartition(final Object x) {
|
||||
if (x instanceof byte[]) {
|
||||
int h = 0;
|
||||
for (byte c: (byte[])x) h = 31 * h + (c & 0xFF);
|
||||
return h & mask;
|
||||
int p = h & mask;
|
||||
//String o = UTF8.String((byte[]) x); try { if (o.equals(latestObject)) throw new RuntimeException("ConcurrentARC: p = " + p + ", objectb = " + o); } catch (Exception e) { Log.logException(e); } latestObject = o;
|
||||
return p;
|
||||
}
|
||||
return x.hashCode() & mask;
|
||||
int p = x.hashCode() & mask;
|
||||
//String o = x.toString(); try { if (o.equals(latestObject)) throw new RuntimeException("ConcurrentARC: p = " + p + ", objecto = " + o); } catch (Exception e) { Log.logException(e); } latestObject = o;
|
||||
return p;
|
||||
}
|
||||
}
|
||||
|
|
|
@ -91,6 +91,25 @@ abstract class SimpleARC<K, V> extends AbstractMap<K, V> implements Map<K, V>, I
|
|||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* put a value to the cache if there was not an entry before
|
||||
* return a previous content value
|
||||
* @param s
|
||||
* @param v
|
||||
* @return the value before inserting the new value
|
||||
*/
|
||||
public V putIfAbsent(K s, V v) {
|
||||
synchronized (this) {
|
||||
V o = this.levelB.get(s);
|
||||
if (o != null) return o;
|
||||
o = this.levelA.get(s);
|
||||
if (o != null) return o;
|
||||
this.levelA.put(s, v);
|
||||
assert (this.levelA.size() <= cacheSize); // the cache should shrink automatically
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* put a value to the cache.
|
||||
* @param s
|
||||
|
|
|
@ -279,6 +279,23 @@ public class MapDataMining extends MapHeap {
|
|||
super.delete(key);
|
||||
}
|
||||
|
||||
/* would be better but does not work (recursion)
|
||||
@Override
|
||||
public synchronized void delete(final byte[] key) throws IOException {
|
||||
if (key == null) return;
|
||||
|
||||
// update elementCount
|
||||
Map<String, String> map = super.remove(key);
|
||||
if (map != null && (sortfields != null || longaccfields != null || floataccfields != null)) {
|
||||
// update accumulators (subtract)
|
||||
if ((longaccfields != null) || (floataccfields != null)) updateAcc(map, false);
|
||||
|
||||
// remove from sortCluster
|
||||
if (sortfields != null) deleteSortCluster(UTF8.String(key));
|
||||
}
|
||||
}
|
||||
*/
|
||||
|
||||
private void deleteSortCluster(final String key) {
|
||||
if (key == null) return;
|
||||
ScoreMap<String> cluster;
|
||||
|
|
|
@ -69,7 +69,7 @@ public class MapHeap implements Map<byte[], Map<String, String>> {
|
|||
final int cachesize,
|
||||
char fillchar) throws IOException {
|
||||
this.blob = new Heap(heapFile, keylength, ordering, buffermax);
|
||||
this.cache = new ConcurrentARC<byte[], Map<String, String>>(cachesize, Runtime.getRuntime().availableProcessors(), ordering);
|
||||
this.cache = new ConcurrentARC<byte[], Map<String, String>>(cachesize, Math.max(32, 4 * Runtime.getRuntime().availableProcessors()), ordering);
|
||||
this.fillchar = fillchar;
|
||||
}
|
||||
|
||||
|
@ -285,6 +285,10 @@ public class MapHeap implements Map<byte[], Map<String, String>> {
|
|||
if (cache == null) return null; // case may appear during shutdown
|
||||
key = normalizeKey(key);
|
||||
|
||||
if (MemoryControl.shortStatus()) {
|
||||
cache.clear();
|
||||
}
|
||||
|
||||
Map<String, String> map;
|
||||
if (storeCache) {
|
||||
synchronized (this) {
|
||||
|
@ -300,16 +304,12 @@ public class MapHeap implements Map<byte[], Map<String, String>> {
|
|||
throw new IOException(e.getMessage());
|
||||
}
|
||||
|
||||
if (MemoryControl.shortStatus()) {
|
||||
cache.clear();
|
||||
} else {
|
||||
// write map to cache
|
||||
cache.insert(key, map);
|
||||
}
|
||||
// write map to cache
|
||||
cache.insert(key, map);
|
||||
|
||||
// return value
|
||||
return map;
|
||||
}
|
||||
|
||||
// return value
|
||||
return map;
|
||||
} else {
|
||||
byte[] b;
|
||||
synchronized (this) {
|
||||
|
|
|
@ -30,7 +30,6 @@ import java.util.HashSet;
|
|||
import java.util.Iterator;
|
||||
import java.util.Locale;
|
||||
import java.util.Set;
|
||||
|
||||
import net.yacy.cora.storage.ARC;
|
||||
import net.yacy.cora.storage.ConcurrentARC;
|
||||
import net.yacy.document.LargeNumberCache;
|
||||
|
@ -54,16 +53,22 @@ public class Word {
|
|||
* these hashes all shall be generated by base64.enhancedCoder
|
||||
*/
|
||||
public static final int commonHashLength = 12;
|
||||
|
||||
|
||||
private static final int hashCacheSize = Math.max(200000, Math.min(10000000, (int) (MemoryControl.available() / 20000L)));
|
||||
private static ARC<String, byte[]> hashCache = null;
|
||||
static {
|
||||
try {
|
||||
hashCache = new ConcurrentARC<String, byte[]>(hashCacheSize, 2 * Runtime.getRuntime().availableProcessors());
|
||||
hashCache = new ConcurrentARC<String, byte[]>(hashCacheSize, Math.max(32, 4 * Runtime.getRuntime().availableProcessors()));
|
||||
} catch (OutOfMemoryError e) {
|
||||
hashCache = new ConcurrentARC<String, byte[]>(1000, Runtime.getRuntime().availableProcessors());
|
||||
hashCache = new ConcurrentARC<String, byte[]>(1000, Math.max(8, 2 * Runtime.getRuntime().availableProcessors()));
|
||||
}
|
||||
}
|
||||
/*
|
||||
private static ConcurrentHashMap<String, byte[]> hashCache = null;
|
||||
static {
|
||||
hashCache = new ConcurrentHashMap<String, byte[]>();
|
||||
}
|
||||
*/
|
||||
|
||||
// object carries statistics for words and sentences
|
||||
public int count; // number of occurrences
|
||||
|
@ -122,6 +127,7 @@ public class Word {
|
|||
if (MemoryControl.shortStatus()) {
|
||||
hashCache.clear();
|
||||
} else {
|
||||
//hashCache.putIfAbsent(wordlc, h); // prevent expensive MD5 computation and encoding
|
||||
hashCache.insertIfAbsent(wordlc, h); // prevent expensive MD5 computation and encoding
|
||||
}
|
||||
return h;
|
||||
|
|
|
@ -142,7 +142,10 @@ public final class FileUtils {
|
|||
}
|
||||
|
||||
public static int copy(final Reader source, final Writer dest) throws IOException {
|
||||
assert source != null;
|
||||
assert dest != null;
|
||||
if (source == null) throw new IOException("source is null");
|
||||
if (dest == null) throw new IOException("dest is null");
|
||||
final char[] buffer = new char[DEFAULT_BUFFER_SIZE];
|
||||
int count = 0;
|
||||
int n = 0;
|
||||
|
@ -153,9 +156,10 @@ public final class FileUtils {
|
|||
}
|
||||
dest.flush();
|
||||
} catch (final Exception e) {
|
||||
assert e != null;
|
||||
// an "sun.io.MalformedInputException: Missing byte-order mark" - exception may occur here
|
||||
//Log.logException(e);
|
||||
throw new IOException(e == null ? "null" : e.getMessage() == null ? e.toString() : e.getMessage(), e);
|
||||
//throw new IOException(e == null ? "null" : e.getMessage() == null ? e.toString() : e.getMessage(), e);
|
||||
}
|
||||
return count;
|
||||
}
|
||||
|
|
Loading…
Reference in New Issue
Block a user