mirror of
https://github.com/yacy/yacy_search_server.git
synced 2024-09-19 00:01:41 +02:00
ConcurrentLinkedQueue has a VERY long return time on the .size() method.
See http://docs.oracle.com/javase/6/docs/api/java/util/concurrent/ConcurrentLinkedQueue.html and the following test programm: public class QueueLengthTimeTest { public static long countTest(Queue<Integer> q, int c) { long t = System.currentTimeMillis(); for (int i = 0; i < c; i++) { q.add(q.size()); } return System.currentTimeMillis() - t; } public static void main(String[] args) { int c = 1; for (int i = 0; i < 100; i++) { Runtime.getRuntime().gc(); long t1 = countTest(new ArrayBlockingQueue<Integer>(c), c); Runtime.getRuntime().gc(); long t2 = countTest(new LinkedBlockingQueue<Integer>(), c); Runtime.getRuntime().gc(); long t3 = countTest(new ConcurrentLinkedQueue<Integer>(), c); System.out.println("count = " + c + ": ArrayBlockingQueue = " + t1 + ", LinkedBlockingQueue = " + t2 + ", ConcurrentLinkedQueue = " + t3); c = c * 2; } } }
This commit is contained in:
parent
8aba045ba1
commit
2fc8ecee36
|
@ -33,7 +33,7 @@ import java.util.Locale;
|
|||
import java.util.Map;
|
||||
import java.util.Map.Entry;
|
||||
import java.util.TreeSet;
|
||||
import java.util.concurrent.ConcurrentLinkedQueue;
|
||||
import java.util.concurrent.LinkedBlockingQueue;
|
||||
|
||||
import net.yacy.cora.protocol.Domains;
|
||||
import net.yacy.cora.protocol.HeaderFramework;
|
||||
|
@ -52,7 +52,7 @@ public class AccessTracker_p {
|
|||
private static SimpleDateFormat SimpleFormatter = new SimpleDateFormat("yyyy/MM/dd HH:mm:ss", Locale.US);
|
||||
|
||||
private static Collection<Track> listclone (final Collection<Track> m) {
|
||||
final Collection<Track> accessClone = new ConcurrentLinkedQueue<Track>();
|
||||
final Collection<Track> accessClone = new LinkedBlockingQueue<Track>();
|
||||
try {
|
||||
accessClone.addAll(m);
|
||||
} catch (final ConcurrentModificationException e) {}
|
||||
|
|
|
@ -27,9 +27,10 @@
|
|||
package de.anomic.crawler;
|
||||
|
||||
import java.util.Map;
|
||||
import java.util.Queue;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
import java.util.concurrent.ConcurrentLinkedQueue;
|
||||
import java.util.concurrent.ConcurrentMap;
|
||||
import java.util.concurrent.LinkedBlockingQueue;
|
||||
|
||||
import net.yacy.cora.document.MultiProtocolURI;
|
||||
import net.yacy.document.Document;
|
||||
|
@ -43,10 +44,10 @@ public class ResultImages {
|
|||
// we maintain two different queues for private and public crawls and divide both into two halves:
|
||||
// such images that appear to be good quality for a image monitor bacause their size is known, and other images
|
||||
// that are not declared with sizes.
|
||||
private static final ConcurrentLinkedQueue<OriginEntry> privateImageQueueHigh = new ConcurrentLinkedQueue<OriginEntry>();
|
||||
private static final ConcurrentLinkedQueue<OriginEntry> privateImageQueueLow = new ConcurrentLinkedQueue<OriginEntry>();
|
||||
private static final ConcurrentLinkedQueue<OriginEntry> publicImageQueueHigh = new ConcurrentLinkedQueue<OriginEntry>();
|
||||
private static final ConcurrentLinkedQueue<OriginEntry> publicImageQueueLow = new ConcurrentLinkedQueue<OriginEntry>();
|
||||
private static final Queue<OriginEntry> privateImageQueueHigh = new LinkedBlockingQueue<OriginEntry>();
|
||||
private static final Queue<OriginEntry> privateImageQueueLow = new LinkedBlockingQueue<OriginEntry>();
|
||||
private static final Queue<OriginEntry> publicImageQueueHigh = new LinkedBlockingQueue<OriginEntry>();
|
||||
private static final Queue<OriginEntry> publicImageQueueLow = new LinkedBlockingQueue<OriginEntry>();
|
||||
|
||||
// we also check all links for a double-check so we don't get the same image more than once in any queue
|
||||
// image links may appear double here even if the pages where the image links are embedded already are checked for double-occurrence:
|
||||
|
|
|
@ -31,12 +31,13 @@ import java.io.IOException;
|
|||
import java.util.ArrayList;
|
||||
import java.util.Date;
|
||||
import java.util.Iterator;
|
||||
import java.util.concurrent.ConcurrentLinkedQueue;
|
||||
import java.util.Queue;
|
||||
import java.util.concurrent.LinkedBlockingQueue;
|
||||
|
||||
import net.yacy.cora.document.ASCII;
|
||||
import net.yacy.cora.document.UTF8;
|
||||
import net.yacy.cora.services.federated.solr.SolrShardingConnector;
|
||||
import net.yacy.cora.services.federated.solr.SolrConnector;
|
||||
import net.yacy.cora.services.federated.solr.SolrShardingConnector;
|
||||
import net.yacy.kelondro.data.meta.DigestURI;
|
||||
import net.yacy.kelondro.data.word.Word;
|
||||
import net.yacy.kelondro.index.Index;
|
||||
|
@ -76,7 +77,7 @@ public class ZURL implements Iterable<ZURL.Entry> {
|
|||
|
||||
// the class object
|
||||
private Index urlIndex;
|
||||
private final ConcurrentLinkedQueue<byte[]> stack;
|
||||
private final Queue<byte[]> stack;
|
||||
private final SolrConnector solrConnector;
|
||||
|
||||
public ZURL(
|
||||
|
@ -105,14 +106,14 @@ public class ZURL implements Iterable<ZURL.Entry> {
|
|||
}
|
||||
}
|
||||
//urlIndex = new kelondroFlexTable(cachePath, tablename, -1, rowdef, 0, true);
|
||||
this.stack = new ConcurrentLinkedQueue<byte[]>();
|
||||
this.stack = new LinkedBlockingQueue<byte[]>();
|
||||
}
|
||||
|
||||
public ZURL(final SolrShardingConnector solrConnector) {
|
||||
this.solrConnector = solrConnector;
|
||||
// creates a new ZUR in RAM
|
||||
this.urlIndex = new RowSet(rowdef);
|
||||
this.stack = new ConcurrentLinkedQueue<byte[]>();
|
||||
this.stack = new LinkedBlockingQueue<byte[]>();
|
||||
}
|
||||
|
||||
public void clear() throws IOException {
|
||||
|
@ -163,6 +164,7 @@ public class ZURL implements Iterable<ZURL.Entry> {
|
|||
while (this.stack.size() > maxStackSize) this.stack.poll();
|
||||
}
|
||||
|
||||
@Override
|
||||
public Iterator<ZURL.Entry> iterator() {
|
||||
return new EntryIterator();
|
||||
}
|
||||
|
@ -185,14 +187,17 @@ public class ZURL implements Iterable<ZURL.Entry> {
|
|||
public EntryIterator() {
|
||||
this.hi = ZURL.this.stack.iterator();
|
||||
}
|
||||
@Override
|
||||
public boolean hasNext() {
|
||||
return this.hi.hasNext();
|
||||
}
|
||||
|
||||
@Override
|
||||
public ZURL.Entry next() {
|
||||
return get(this.hi.next());
|
||||
}
|
||||
|
||||
@Override
|
||||
public void remove() {
|
||||
this.hi.remove();
|
||||
}
|
||||
|
@ -326,11 +331,13 @@ public class ZURL implements Iterable<ZURL.Entry> {
|
|||
this.error = false;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean hasNext() {
|
||||
if (this.error) return false;
|
||||
return this.i.hasNext();
|
||||
}
|
||||
|
||||
@Override
|
||||
public Entry next() throws RuntimeException {
|
||||
final Row.Entry e = this.i.next();
|
||||
if (e == null) return null;
|
||||
|
@ -341,6 +348,7 @@ public class ZURL implements Iterable<ZURL.Entry> {
|
|||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void remove() {
|
||||
this.i.remove();
|
||||
}
|
||||
|
|
|
@ -26,9 +26,10 @@
|
|||
|
||||
package de.anomic.data;
|
||||
|
||||
import java.util.Queue;
|
||||
import java.util.Random;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
import java.util.concurrent.ConcurrentLinkedQueue;
|
||||
import java.util.concurrent.LinkedBlockingQueue;
|
||||
|
||||
import net.yacy.kelondro.data.meta.DigestURI;
|
||||
|
||||
|
@ -42,13 +43,13 @@ public class URLLicense {
|
|||
|
||||
private final Random random;
|
||||
private final ConcurrentHashMap<String, DigestURI> permissions;
|
||||
private final ConcurrentLinkedQueue<String> aging;
|
||||
private final Queue<String> aging;
|
||||
private long lastCheck;
|
||||
private final int keylen;
|
||||
|
||||
public URLLicense(final int keylen) {
|
||||
this.permissions = new ConcurrentHashMap<String, DigestURI>();
|
||||
this.aging = new ConcurrentLinkedQueue<String>();
|
||||
this.aging = new LinkedBlockingQueue<String>();
|
||||
this.lastCheck = System.currentTimeMillis();
|
||||
this.random = new Random(System.currentTimeMillis());
|
||||
this.keylen = keylen;
|
||||
|
@ -56,20 +57,20 @@ public class URLLicense {
|
|||
|
||||
public String aquireLicense(final DigestURI url) {
|
||||
// generate license key
|
||||
StringBuilder stringBuilder = new StringBuilder(keylen * 2);
|
||||
StringBuilder stringBuilder = new StringBuilder(this.keylen * 2);
|
||||
if (url == null) return stringBuilder.toString();
|
||||
while (stringBuilder.length() < keylen) stringBuilder.append(Integer.toHexString(random.nextInt()));
|
||||
String license = stringBuilder.substring(0, keylen);
|
||||
while (stringBuilder.length() < this.keylen) stringBuilder.append(Integer.toHexString(this.random.nextInt()));
|
||||
String license = stringBuilder.substring(0, this.keylen);
|
||||
// store reference to url with license key
|
||||
permissions.put(license, url);
|
||||
aging.add(license);
|
||||
this.permissions.put(license, url);
|
||||
this.aging.add(license);
|
||||
if (System.currentTimeMillis() - this.lastCheck > minCheck) {
|
||||
// check aging
|
||||
this.lastCheck = System.currentTimeMillis();
|
||||
String s;
|
||||
while (aging.size() > maxQueue) {
|
||||
s = aging.poll();
|
||||
if (s != null) permissions.remove(s);
|
||||
while (this.aging.size() > maxQueue) {
|
||||
s = this.aging.poll();
|
||||
if (s != null) this.permissions.remove(s);
|
||||
}
|
||||
}
|
||||
// return the license key
|
||||
|
@ -78,7 +79,7 @@ public class URLLicense {
|
|||
|
||||
public DigestURI releaseLicense(final String license) {
|
||||
DigestURI url = null;
|
||||
url = permissions.remove(license);
|
||||
url = this.permissions.remove(license);
|
||||
return url;
|
||||
}
|
||||
|
||||
|
|
|
@ -24,8 +24,9 @@ import java.util.Collection;
|
|||
import java.util.Iterator;
|
||||
import java.util.Map;
|
||||
import java.util.NoSuchElementException;
|
||||
import java.util.Queue;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
import java.util.concurrent.ConcurrentLinkedQueue;
|
||||
import java.util.concurrent.LinkedBlockingQueue;
|
||||
|
||||
public class serverAccessTracker {
|
||||
|
||||
|
@ -34,7 +35,7 @@ public class serverAccessTracker {
|
|||
private final long maxTrackingTime;
|
||||
private final int maxTrackingCount;
|
||||
private final int maxHostCount;
|
||||
private final ConcurrentHashMap<String, ConcurrentLinkedQueue<Track>> accessTracker; // mappings from requesting host to an ArrayList of serverTrack-entries
|
||||
private final ConcurrentHashMap<String, Queue<Track>> accessTracker; // mappings from requesting host to an ArrayList of serverTrack-entries
|
||||
private long lastCleanup;
|
||||
|
||||
public static class Track {
|
||||
|
@ -56,7 +57,7 @@ public class serverAccessTracker {
|
|||
this.maxTrackingTime = maxTrackingTime;
|
||||
this.maxTrackingCount = maxTrackingCount;
|
||||
this.maxHostCount = maxTrackingHostCount;
|
||||
this.accessTracker = new ConcurrentHashMap<String, ConcurrentLinkedQueue<Track>>();
|
||||
this.accessTracker = new ConcurrentHashMap<String, Queue<Track>>();
|
||||
}
|
||||
|
||||
/*
|
||||
|
@ -70,8 +71,8 @@ public class serverAccessTracker {
|
|||
}
|
||||
|
||||
// clear entries which had no entry for the maxTrackingTime time
|
||||
final Iterator<Map.Entry<String, ConcurrentLinkedQueue<Track>>> i = this.accessTracker.entrySet().iterator();
|
||||
ConcurrentLinkedQueue<Track> track;
|
||||
final Iterator<Map.Entry<String, Queue<Track>>> i = this.accessTracker.entrySet().iterator();
|
||||
Queue<Track> track;
|
||||
while (i.hasNext()) {
|
||||
track = i.next().getValue();
|
||||
clearTooOldAccess(track);
|
||||
|
@ -111,7 +112,7 @@ public class serverAccessTracker {
|
|||
return c;
|
||||
}
|
||||
|
||||
private void clearTooOldAccess(final ConcurrentLinkedQueue<Track> access) {
|
||||
private void clearTooOldAccess(final Queue<Track> access) {
|
||||
final long time = System.currentTimeMillis() - this.maxTrackingTime;
|
||||
final Iterator<Track> e = access.iterator();
|
||||
Track l;
|
||||
|
@ -130,9 +131,9 @@ public class serverAccessTracker {
|
|||
|
||||
// learn that a specific host has accessed a specific path
|
||||
if (accessPath == null) accessPath="NULL";
|
||||
ConcurrentLinkedQueue<Track> track = this.accessTracker.get(host);
|
||||
Queue<Track> track = this.accessTracker.get(host);
|
||||
if (track == null) {
|
||||
track = new ConcurrentLinkedQueue<Track>();
|
||||
track = new LinkedBlockingQueue<Track>();
|
||||
track.add(new Track(System.currentTimeMillis(), accessPath));
|
||||
// add to tracker
|
||||
this.accessTracker.put(host, track);
|
||||
|
@ -145,7 +146,7 @@ public class serverAccessTracker {
|
|||
public Collection<Track> accessTrack(final String host) {
|
||||
// returns mapping from Long(accesstime) to path
|
||||
|
||||
final ConcurrentLinkedQueue<Track> access = this.accessTracker.get(host);
|
||||
final Queue<Track> access = this.accessTracker.get(host);
|
||||
if (access == null) return null;
|
||||
// clear too old entries
|
||||
clearTooOldAccess(access);
|
||||
|
@ -157,7 +158,7 @@ public class serverAccessTracker {
|
|||
|
||||
public Iterator<String> accessHosts() {
|
||||
// returns an iterator of hosts in tracker (String)
|
||||
final Map<String, ConcurrentLinkedQueue<Track>> accessTrackerClone = new ConcurrentHashMap<String, ConcurrentLinkedQueue<Track>>();
|
||||
final Map<String, Queue<Track>> accessTrackerClone = new ConcurrentHashMap<String, Queue<Track>>();
|
||||
accessTrackerClone.putAll(this.accessTracker);
|
||||
return accessTrackerClone.keySet().iterator();
|
||||
}
|
||||
|
|
|
@ -27,7 +27,8 @@
|
|||
package net.yacy.kelondro.data.image;
|
||||
|
||||
import java.util.Collection;
|
||||
import java.util.concurrent.ConcurrentLinkedQueue;
|
||||
import java.util.Queue;
|
||||
import java.util.concurrent.LinkedBlockingQueue;
|
||||
|
||||
import net.yacy.kelondro.index.Row.Entry;
|
||||
import net.yacy.kelondro.order.Bitfield;
|
||||
|
@ -54,7 +55,7 @@ public class ImageReferenceVars extends AbstractReference implements ImageRefere
|
|||
posinphrase, posofphrase,
|
||||
urlcomps, urllength, virtualAge,
|
||||
wordsintext, wordsintitle;
|
||||
private final ConcurrentLinkedQueue<Integer> positions;
|
||||
private final Queue<Integer> positions;
|
||||
public double termFrequency;
|
||||
|
||||
public ImageReferenceVars(
|
||||
|
@ -65,7 +66,7 @@ public class ImageReferenceVars extends AbstractReference implements ImageRefere
|
|||
final int hitcount, // how often appears this word in the text
|
||||
final int wordcount, // total number of words
|
||||
final int phrasecount, // total number of phrases
|
||||
final ConcurrentLinkedQueue<Integer> ps, // positions of words that are joined into the reference
|
||||
final Queue<Integer> ps, // positions of words that are joined into the reference
|
||||
final int posinphrase, // position of word in its phrase
|
||||
final int posofphrase, // number of the phrase where word appears
|
||||
final long lastmodified, // last-modified time of the document where word appears
|
||||
|
@ -90,7 +91,7 @@ public class ImageReferenceVars extends AbstractReference implements ImageRefere
|
|||
this.llocal = outlinksSame;
|
||||
this.lother = outlinksOther;
|
||||
this.phrasesintext = phrasecount;
|
||||
this.positions = new ConcurrentLinkedQueue<Integer>();
|
||||
this.positions = new LinkedBlockingQueue<Integer>();
|
||||
for (Integer i: ps) this.positions.add(i);
|
||||
this.posinphrase = posinphrase;
|
||||
this.posofphrase = posofphrase;
|
||||
|
@ -113,7 +114,7 @@ public class ImageReferenceVars extends AbstractReference implements ImageRefere
|
|||
this.llocal = e.llocal();
|
||||
this.lother = e.lother();
|
||||
this.phrasesintext = e.phrasesintext();
|
||||
this.positions = new ConcurrentLinkedQueue<Integer>();
|
||||
this.positions = new LinkedBlockingQueue<Integer>();
|
||||
for (Integer i: e.positions()) this.positions.add(i);
|
||||
this.posinphrase = e.posinphrase();
|
||||
this.posofphrase = e.posofphrase();
|
||||
|
@ -184,113 +185,134 @@ public class ImageReferenceVars extends AbstractReference implements ImageRefere
|
|||
this.termFrequency = this.termFrequency + v.termFrequency;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Bitfield flags() {
|
||||
return flags;
|
||||
return this.flags;
|
||||
}
|
||||
/*
|
||||
public long freshUntil() {
|
||||
return freshUntil;
|
||||
}
|
||||
*/
|
||||
@Override
|
||||
public String getLanguage() {
|
||||
return language;
|
||||
return this.language;
|
||||
}
|
||||
|
||||
@Override
|
||||
public char getType() {
|
||||
return type;
|
||||
return this.type;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int hitcount() {
|
||||
return hitcount;
|
||||
return this.hitcount;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean isOlder(final Reference other) {
|
||||
assert false; // should not be used
|
||||
return false;
|
||||
}
|
||||
|
||||
@Override
|
||||
public long lastModified() {
|
||||
return lastModified;
|
||||
return this.lastModified;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int llocal() {
|
||||
return llocal;
|
||||
return this.llocal;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int lother() {
|
||||
return lother;
|
||||
return this.lother;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int phrasesintext() {
|
||||
return phrasesintext;
|
||||
return this.phrasesintext;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int posinphrase() {
|
||||
return posinphrase;
|
||||
return this.posinphrase;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Collection<Integer> positions() {
|
||||
return this.positions;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int posofphrase() {
|
||||
return posofphrase;
|
||||
return this.posofphrase;
|
||||
}
|
||||
|
||||
public ImageReferenceRow toRowEntry() {
|
||||
return new ImageReferenceRow(
|
||||
urlHash,
|
||||
urllength, // byte-length of complete URL
|
||||
urlcomps, // number of path components
|
||||
wordsintitle, // length of description/length (longer are better?)
|
||||
hitcount, // how often appears this word in the text
|
||||
wordsintext, // total number of words
|
||||
phrasesintext, // total number of phrases
|
||||
positions.iterator().next(), // position of word in all words
|
||||
posinphrase, // position of word in its phrase
|
||||
posofphrase, // number of the phrase where word appears
|
||||
lastModified, // last-modified time of the document where word appears
|
||||
this.urlHash,
|
||||
this.urllength, // byte-length of complete URL
|
||||
this.urlcomps, // number of path components
|
||||
this.wordsintitle, // length of description/length (longer are better?)
|
||||
this.hitcount, // how often appears this word in the text
|
||||
this.wordsintext, // total number of words
|
||||
this.phrasesintext, // total number of phrases
|
||||
this.positions.iterator().next(), // position of word in all words
|
||||
this.posinphrase, // position of word in its phrase
|
||||
this.posofphrase, // number of the phrase where word appears
|
||||
this.lastModified, // last-modified time of the document where word appears
|
||||
System.currentTimeMillis(), // update time;
|
||||
language, // (guessed) language of document
|
||||
type, // type of document
|
||||
llocal, // outlinks to same domain
|
||||
lother, // outlinks to other domain
|
||||
flags // attributes to the url and to the word according the url
|
||||
this.language, // (guessed) language of document
|
||||
this.type, // type of document
|
||||
this.llocal, // outlinks to same domain
|
||||
this.lother, // outlinks to other domain
|
||||
this.flags // attributes to the url and to the word according the url
|
||||
);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Entry toKelondroEntry() {
|
||||
return toRowEntry().toKelondroEntry();
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toPropertyForm() {
|
||||
return toRowEntry().toPropertyForm();
|
||||
}
|
||||
|
||||
@Override
|
||||
public byte[] urlhash() {
|
||||
return urlHash;
|
||||
return this.urlHash;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int urlcomps() {
|
||||
return urlcomps;
|
||||
return this.urlcomps;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int urllength() {
|
||||
return urllength;
|
||||
return this.urllength;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int virtualAge() {
|
||||
return virtualAge;
|
||||
return this.virtualAge;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int wordsintext() {
|
||||
return wordsintext;
|
||||
return this.wordsintext;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int wordsintitle() {
|
||||
return wordsintitle;
|
||||
return this.wordsintitle;
|
||||
}
|
||||
|
||||
@Override
|
||||
public double termFrequency() {
|
||||
if (this.termFrequency == 0.0) this.termFrequency = (((double) this.hitcount()) / ((double) (this.wordsintext() + this.wordsintitle() + 1)));
|
||||
return this.termFrequency;
|
||||
|
@ -340,6 +362,7 @@ public class ImageReferenceVars extends AbstractReference implements ImageRefere
|
|||
if (this.termFrequency < (d = other.termFrequency)) this.termFrequency = d;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void join(final Reference r) {
|
||||
// joins two entries into one entry
|
||||
|
||||
|
|
|
@ -28,8 +28,8 @@ package net.yacy.kelondro.data.word;
|
|||
|
||||
import java.util.Collection;
|
||||
import java.util.Comparator;
|
||||
import java.util.Queue;
|
||||
import java.util.concurrent.BlockingQueue;
|
||||
import java.util.concurrent.ConcurrentLinkedQueue;
|
||||
import java.util.concurrent.LinkedBlockingQueue;
|
||||
import java.util.concurrent.Semaphore;
|
||||
|
||||
|
@ -66,7 +66,7 @@ public class WordReferenceVars extends AbstractReference implements WordReferenc
|
|||
urlcomps, urllength,
|
||||
wordsintext, wordsintitle;
|
||||
private int virtualAge;
|
||||
private final ConcurrentLinkedQueue<Integer> positions;
|
||||
private final Queue<Integer> positions;
|
||||
public double termFrequency;
|
||||
|
||||
public WordReferenceVars(
|
||||
|
@ -77,7 +77,7 @@ public class WordReferenceVars extends AbstractReference implements WordReferenc
|
|||
final int hitcount, // how often appears this word in the text
|
||||
final int wordcount, // total number of words
|
||||
final int phrasecount, // total number of phrases
|
||||
final ConcurrentLinkedQueue<Integer> ps, // positions of words that are joined into the reference
|
||||
final Queue<Integer> ps, // positions of words that are joined into the reference
|
||||
final int posinphrase, // position of word in its phrase
|
||||
final int posofphrase, // number of the phrase where word appears
|
||||
final long lastmodified, // last-modified time of the document where word appears
|
||||
|
@ -101,7 +101,7 @@ public class WordReferenceVars extends AbstractReference implements WordReferenc
|
|||
this.llocal = outlinksSame;
|
||||
this.lother = outlinksOther;
|
||||
this.phrasesintext = phrasecount;
|
||||
this.positions = new ConcurrentLinkedQueue<Integer>();
|
||||
this.positions = new LinkedBlockingQueue<Integer>();
|
||||
if (ps.size() > 0) for (final Integer i: ps) this.positions.add(i);
|
||||
this.posinphrase = posinphrase;
|
||||
this.posofphrase = posofphrase;
|
||||
|
@ -124,7 +124,7 @@ public class WordReferenceVars extends AbstractReference implements WordReferenc
|
|||
this.llocal = e.llocal();
|
||||
this.lother = e.lother();
|
||||
this.phrasesintext = e.phrasesintext();
|
||||
this.positions = new ConcurrentLinkedQueue<Integer>();
|
||||
this.positions = new LinkedBlockingQueue<Integer>();
|
||||
if (e.positions().size() > 0) for (final Integer i: e.positions()) this.positions.add(i);
|
||||
this.posinphrase = e.posinphrase();
|
||||
this.posofphrase = e.posofphrase();
|
||||
|
@ -195,46 +195,57 @@ public class WordReferenceVars extends AbstractReference implements WordReferenc
|
|||
this.termFrequency = this.termFrequency + v.termFrequency;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Bitfield flags() {
|
||||
return this.flags;
|
||||
}
|
||||
|
||||
@Override
|
||||
public byte[] getLanguage() {
|
||||
return this.language;
|
||||
}
|
||||
|
||||
@Override
|
||||
public char getType() {
|
||||
return this.type;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int hitcount() {
|
||||
return this.hitcount;
|
||||
}
|
||||
|
||||
@Override
|
||||
public long lastModified() {
|
||||
return this.lastModified;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int llocal() {
|
||||
return this.llocal;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int lother() {
|
||||
return this.lother;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int phrasesintext() {
|
||||
return this.phrasesintext;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int posinphrase() {
|
||||
return this.posinphrase;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Collection<Integer> positions() {
|
||||
return this.positions;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int posofphrase() {
|
||||
return this.posofphrase;
|
||||
}
|
||||
|
@ -261,14 +272,17 @@ public class WordReferenceVars extends AbstractReference implements WordReferenc
|
|||
);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Entry toKelondroEntry() {
|
||||
return toRowEntry().toKelondroEntry();
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toPropertyForm() {
|
||||
return toRowEntry().toPropertyForm();
|
||||
}
|
||||
|
||||
@Override
|
||||
public byte[] urlhash() {
|
||||
return this.urlHash;
|
||||
}
|
||||
|
@ -279,28 +293,34 @@ public class WordReferenceVars extends AbstractReference implements WordReferenc
|
|||
return this.hostHash;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int urlcomps() {
|
||||
return this.urlcomps;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int urllength() {
|
||||
return this.urllength;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int virtualAge() {
|
||||
if (this.virtualAge > 0) return this.virtualAge;
|
||||
this.virtualAge = MicroDate.microDateDays(this.lastModified);
|
||||
return this.virtualAge;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int wordsintext() {
|
||||
return this.wordsintext;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int wordsintitle() {
|
||||
return this.wordsintitle;
|
||||
}
|
||||
|
||||
@Override
|
||||
public double termFrequency() {
|
||||
if (this.termFrequency == 0.0) this.termFrequency = (((double) hitcount()) / ((double) (wordsintext() + wordsintitle() + 1)));
|
||||
return this.termFrequency;
|
||||
|
@ -350,6 +370,7 @@ public class WordReferenceVars extends AbstractReference implements WordReferenc
|
|||
if (this.termFrequency < (d = other.termFrequency)) this.termFrequency = d;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void join(final Reference r) {
|
||||
// joins two entries into one entry
|
||||
|
||||
|
@ -378,10 +399,12 @@ public class WordReferenceVars extends AbstractReference implements WordReferenc
|
|||
return ByteArray.hashCode(this.urlHash);
|
||||
}
|
||||
|
||||
@Override
|
||||
public int compareTo(final WordReferenceVars o) {
|
||||
return Base64Order.enhancedCoder.compare(this.urlHash, o.urlhash());
|
||||
}
|
||||
|
||||
@Override
|
||||
public int compare(final WordReferenceVars o1, final WordReferenceVars o2) {
|
||||
return o1.compareTo(o2);
|
||||
}
|
||||
|
|
|
@ -28,8 +28,9 @@ package net.yacy.kelondro.util;
|
|||
|
||||
import java.util.Iterator;
|
||||
import java.util.Map;
|
||||
import java.util.Queue;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
import java.util.concurrent.ConcurrentLinkedQueue;
|
||||
import java.util.concurrent.LinkedBlockingQueue;
|
||||
|
||||
import net.yacy.peers.graphics.ProfilingGraph;
|
||||
|
||||
|
@ -49,7 +50,7 @@ public class EventTracker {
|
|||
SEARCH;
|
||||
}
|
||||
|
||||
private final static Map<EClass, ConcurrentLinkedQueue<Event>> historyMaps = new ConcurrentHashMap<EClass, ConcurrentLinkedQueue<Event>>();
|
||||
private final static Map<EClass, Queue<Event>> historyMaps = new ConcurrentHashMap<EClass, Queue<Event>>();
|
||||
private final static Map<EClass, Long> eventAccess = new ConcurrentHashMap<EClass, Long>(); // value: last time when this was accessed
|
||||
|
||||
public final static void delete(final EClass eventName) {
|
||||
|
@ -73,11 +74,11 @@ public class EventTracker {
|
|||
}
|
||||
|
||||
// get event history container
|
||||
ConcurrentLinkedQueue<Event> history = historyMaps.get(eventName);
|
||||
Queue<Event> history = historyMaps.get(eventName);
|
||||
|
||||
// create history
|
||||
if (history == null) {
|
||||
history = new ConcurrentLinkedQueue<Event>();
|
||||
history = new LinkedBlockingQueue<Event>();
|
||||
|
||||
// update entry
|
||||
history.offer(new Event(eventPayload));
|
||||
|
@ -109,7 +110,7 @@ public class EventTracker {
|
|||
}
|
||||
|
||||
public final static Iterator<Event> getHistory(final EClass eventName) {
|
||||
final ConcurrentLinkedQueue<Event> list = historyMaps.get(eventName);
|
||||
final Queue<Event> list = historyMaps.get(eventName);
|
||||
if (list == null) return null;
|
||||
return list.iterator();
|
||||
}
|
||||
|
|
Loading…
Reference in New Issue
Block a user