Marown/yacy_search_server
1
0
Fork 0
mirror of https://github.com/yacy/yacy_search_server.git synced 2024-09-19 00:01:41 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity

ConcurrentLinkedQueue has a VERY long return time on the .size() method.

Browse Source 
See
http://docs.oracle.com/javase/6/docs/api/java/util/concurrent/ConcurrentLinkedQueue.html

and the following test programm:

public class QueueLengthTimeTest {


    public static long countTest(Queue<Integer> q, int c) {
        long t = System.currentTimeMillis();
        for (int i = 0; i < c; i++) {
            q.add(q.size());
        }
        return System.currentTimeMillis() - t;
    }

    public static void main(String[] args) {
        int c = 1;
        for (int i = 0; i < 100; i++) {
            Runtime.getRuntime().gc();
            long t1 = countTest(new ArrayBlockingQueue<Integer>(c), c);
            Runtime.getRuntime().gc();
            long t2 = countTest(new LinkedBlockingQueue<Integer>(), c);
            Runtime.getRuntime().gc();
            long t3 = countTest(new ConcurrentLinkedQueue<Integer>(),
c);

            System.out.println("count = " + c + ": ArrayBlockingQueue =
" + t1 + ", LinkedBlockingQueue = " + t2 + ", ConcurrentLinkedQueue = "
+ t3);
            c = c * 2;
        }
    }
}
This commit is contained in:
Michael Peter Christen 2012-02-27 00:42:32 +01:00
parent 8aba045ba1
commit 2fc8ecee36
8 changed files with 186 additions and 128 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
htroot/AccessTracker_p.java
View File

@ -33,7 +33,7 @@ import java.util.Locale;
import java.util.Map; import java.util.Map;
import java.util.Map.Entry; import java.util.Map.Entry;
import java.util.TreeSet; import java.util.TreeSet;
import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.LinkedBlockingQueue;
import net.yacy.cora.protocol.Domains; import net.yacy.cora.protocol.Domains;
import net.yacy.cora.protocol.HeaderFramework; import net.yacy.cora.protocol.HeaderFramework;
@ -52,7 +52,7 @@ public class AccessTracker_p {
private static SimpleDateFormat SimpleFormatter = new SimpleDateFormat("yyyy/MM/dd HH:mm:ss", Locale.US); private static SimpleDateFormat SimpleFormatter = new SimpleDateFormat("yyyy/MM/dd HH:mm:ss", Locale.US);
private static Collection<Track> listclone (final Collection<Track> m) { private static Collection<Track> listclone (final Collection<Track> m) {
final Collection<Track> accessClone = new ConcurrentLinkedQueue<Track>(); final Collection<Track> accessClone = new LinkedBlockingQueue<Track>();
try { try {
accessClone.addAll(m); accessClone.addAll(m);
} catch (final ConcurrentModificationException e) {} } catch (final ConcurrentModificationException e) {}

11
source/de/anomic/crawler/ResultImages.java
View File

@ -27,9 +27,10 @@
package de.anomic.crawler; package de.anomic.crawler;
import java.util.Map; import java.util.Map;
import java.util.Queue;
import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentLinkedQueue;
import java.util.concurrent.ConcurrentMap; import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.LinkedBlockingQueue;
import net.yacy.cora.document.MultiProtocolURI; import net.yacy.cora.document.MultiProtocolURI;
import net.yacy.document.Document; import net.yacy.document.Document;
@ -43,10 +44,10 @@ public class ResultImages {
// we maintain two different queues for private and public crawls and divide both into two halves: // we maintain two different queues for private and public crawls and divide both into two halves:
// such images that appear to be good quality for a image monitor bacause their size is known, and other images // such images that appear to be good quality for a image monitor bacause their size is known, and other images
// that are not declared with sizes. // that are not declared with sizes.
private static final ConcurrentLinkedQueue<OriginEntry> privateImageQueueHigh = new ConcurrentLinkedQueue<OriginEntry>(); private static final Queue<OriginEntry> privateImageQueueHigh = new LinkedBlockingQueue<OriginEntry>();
private static final ConcurrentLinkedQueue<OriginEntry> privateImageQueueLow = new ConcurrentLinkedQueue<OriginEntry>(); private static final Queue<OriginEntry> privateImageQueueLow = new LinkedBlockingQueue<OriginEntry>();
private static final ConcurrentLinkedQueue<OriginEntry> publicImageQueueHigh = new ConcurrentLinkedQueue<OriginEntry>(); private static final Queue<OriginEntry> publicImageQueueHigh = new LinkedBlockingQueue<OriginEntry>();
private static final ConcurrentLinkedQueue<OriginEntry> publicImageQueueLow = new ConcurrentLinkedQueue<OriginEntry>(); private static final Queue<OriginEntry> publicImageQueueLow = new LinkedBlockingQueue<OriginEntry>();
// we also check all links for a double-check so we don't get the same image more than once in any queue // we also check all links for a double-check so we don't get the same image more than once in any queue
// image links may appear double here even if the pages where the image links are embedded already are checked for double-occurrence: // image links may appear double here even if the pages where the image links are embedded already are checked for double-occurrence:

18
source/de/anomic/crawler/ZURL.java
View File

@ -31,12 +31,13 @@ import java.io.IOException;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Date; import java.util.Date;
import java.util.Iterator; import java.util.Iterator;
import java.util.concurrent.ConcurrentLinkedQueue; import java.util.Queue;
import java.util.concurrent.LinkedBlockingQueue;
import net.yacy.cora.document.ASCII; import net.yacy.cora.document.ASCII;
import net.yacy.cora.document.UTF8; import net.yacy.cora.document.UTF8;
import net.yacy.cora.services.federated.solr.SolrShardingConnector;
import net.yacy.cora.services.federated.solr.SolrConnector; import net.yacy.cora.services.federated.solr.SolrConnector;
import net.yacy.cora.services.federated.solr.SolrShardingConnector;
import net.yacy.kelondro.data.meta.DigestURI; import net.yacy.kelondro.data.meta.DigestURI;
import net.yacy.kelondro.data.word.Word; import net.yacy.kelondro.data.word.Word;
import net.yacy.kelondro.index.Index; import net.yacy.kelondro.index.Index;
@ -76,7 +77,7 @@ public class ZURL implements Iterable<ZURL.Entry> {
// the class object // the class object
private Index urlIndex; private Index urlIndex;
private final ConcurrentLinkedQueue<byte[]> stack; private final Queue<byte[]> stack;
private final SolrConnector solrConnector; private final SolrConnector solrConnector;
public ZURL( public ZURL(
@ -105,14 +106,14 @@ public class ZURL implements Iterable<ZURL.Entry> {
} }
} }
//urlIndex = new kelondroFlexTable(cachePath, tablename, -1, rowdef, 0, true); //urlIndex = new kelondroFlexTable(cachePath, tablename, -1, rowdef, 0, true);
this.stack = new ConcurrentLinkedQueue<byte[]>(); this.stack = new LinkedBlockingQueue<byte[]>();
} }
public ZURL(final SolrShardingConnector solrConnector) { public ZURL(final SolrShardingConnector solrConnector) {
this.solrConnector = solrConnector; this.solrConnector = solrConnector;
// creates a new ZUR in RAM // creates a new ZUR in RAM
this.urlIndex = new RowSet(rowdef); this.urlIndex = new RowSet(rowdef);
this.stack = new ConcurrentLinkedQueue<byte[]>(); this.stack = new LinkedBlockingQueue<byte[]>();
} }
public void clear() throws IOException { public void clear() throws IOException {
@ -163,6 +164,7 @@ public class ZURL implements Iterable<ZURL.Entry> {
while (this.stack.size() > maxStackSize) this.stack.poll(); while (this.stack.size() > maxStackSize) this.stack.poll();
} }
@Override
public Iterator<ZURL.Entry> iterator() { public Iterator<ZURL.Entry> iterator() {
return new EntryIterator(); return new EntryIterator();
} }
@ -185,14 +187,17 @@ public class ZURL implements Iterable<ZURL.Entry> {
public EntryIterator() { public EntryIterator() {
this.hi = ZURL.this.stack.iterator(); this.hi = ZURL.this.stack.iterator();
} }
@Override
public boolean hasNext() { public boolean hasNext() {
return this.hi.hasNext(); return this.hi.hasNext();
} }
@Override
public ZURL.Entry next() { public ZURL.Entry next() {
return get(this.hi.next()); return get(this.hi.next());
} }
@Override
public void remove() { public void remove() {
this.hi.remove(); this.hi.remove();
} }
@ -326,11 +331,13 @@ public class ZURL implements Iterable<ZURL.Entry> {
this.error = false; this.error = false;
} }
@Override
public boolean hasNext() { public boolean hasNext() {
if (this.error) return false; if (this.error) return false;
return this.i.hasNext(); return this.i.hasNext();
} }
@Override
public Entry next() throws RuntimeException { public Entry next() throws RuntimeException {
final Row.Entry e = this.i.next(); final Row.Entry e = this.i.next();
if (e == null) return null; if (e == null) return null;
@ -341,6 +348,7 @@ public class ZURL implements Iterable<ZURL.Entry> {
} }
} }
@Override
public void remove() { public void remove() {
this.i.remove(); this.i.remove();
} }

25
source/de/anomic/data/URLLicense.java
View File

@ -26,9 +26,10 @@
package de.anomic.data; package de.anomic.data;
import java.util.Queue;
import java.util.Random; import java.util.Random;
import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.LinkedBlockingQueue;
import net.yacy.kelondro.data.meta.DigestURI; import net.yacy.kelondro.data.meta.DigestURI;
@ -42,13 +43,13 @@ public class URLLicense {
private final Random random; private final Random random;
private final ConcurrentHashMap<String, DigestURI> permissions; private final ConcurrentHashMap<String, DigestURI> permissions;
private final ConcurrentLinkedQueue<String> aging; private final Queue<String> aging;
private long lastCheck; private long lastCheck;
private final int keylen; private final int keylen;
public URLLicense(final int keylen) { public URLLicense(final int keylen) {
this.permissions = new ConcurrentHashMap<String, DigestURI>(); this.permissions = new ConcurrentHashMap<String, DigestURI>();
this.aging = new ConcurrentLinkedQueue<String>(); this.aging = new LinkedBlockingQueue<String>();
this.lastCheck = System.currentTimeMillis(); this.lastCheck = System.currentTimeMillis();
this.random = new Random(System.currentTimeMillis()); this.random = new Random(System.currentTimeMillis());
this.keylen = keylen; this.keylen = keylen;
@ -56,20 +57,20 @@ public class URLLicense {
public String aquireLicense(final DigestURI url) { public String aquireLicense(final DigestURI url) {
// generate license key // generate license key
StringBuilder stringBuilder = new StringBuilder(keylen * 2); StringBuilder stringBuilder = new StringBuilder(this.keylen * 2);
if (url == null) return stringBuilder.toString(); if (url == null) return stringBuilder.toString();
while (stringBuilder.length() < keylen) stringBuilder.append(Integer.toHexString(random.nextInt())); while (stringBuilder.length() < this.keylen) stringBuilder.append(Integer.toHexString(this.random.nextInt()));
String license = stringBuilder.substring(0, keylen); String license = stringBuilder.substring(0, this.keylen);
// store reference to url with license key // store reference to url with license key
permissions.put(license, url); this.permissions.put(license, url);
aging.add(license); this.aging.add(license);
if (System.currentTimeMillis() - this.lastCheck > minCheck) { if (System.currentTimeMillis() - this.lastCheck > minCheck) {
// check aging // check aging
this.lastCheck = System.currentTimeMillis(); this.lastCheck = System.currentTimeMillis();
String s; String s;
while (aging.size() > maxQueue) { while (this.aging.size() > maxQueue) {
s = aging.poll(); s = this.aging.poll();
if (s != null) permissions.remove(s); if (s != null) this.permissions.remove(s);
} }
} }
// return the license key // return the license key
@ -78,7 +79,7 @@ public class URLLicense {
public DigestURI releaseLicense(final String license) { public DigestURI releaseLicense(final String license) {
DigestURI url = null; DigestURI url = null;
url = permissions.remove(license); url = this.permissions.remove(license);
return url; return url;
} }

21
source/de/anomic/server/serverAccessTracker.java
View File

@ -24,8 +24,9 @@ import java.util.Collection;
import java.util.Iterator; import java.util.Iterator;
import java.util.Map; import java.util.Map;
import java.util.NoSuchElementException; import java.util.NoSuchElementException;
import java.util.Queue;
import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.LinkedBlockingQueue;
public class serverAccessTracker { public class serverAccessTracker {
@ -34,7 +35,7 @@ public class serverAccessTracker {
private final long maxTrackingTime; private final long maxTrackingTime;
private final int maxTrackingCount; private final int maxTrackingCount;
private final int maxHostCount; private final int maxHostCount;
private final ConcurrentHashMap<String, ConcurrentLinkedQueue<Track>> accessTracker; // mappings from requesting host to an ArrayList of serverTrack-entries private final ConcurrentHashMap<String, Queue<Track>> accessTracker; // mappings from requesting host to an ArrayList of serverTrack-entries
private long lastCleanup; private long lastCleanup;
public static class Track { public static class Track {
@ -56,7 +57,7 @@ public class serverAccessTracker {
this.maxTrackingTime = maxTrackingTime; this.maxTrackingTime = maxTrackingTime;
this.maxTrackingCount = maxTrackingCount; this.maxTrackingCount = maxTrackingCount;
this.maxHostCount = maxTrackingHostCount; this.maxHostCount = maxTrackingHostCount;
this.accessTracker = new ConcurrentHashMap<String, ConcurrentLinkedQueue<Track>>(); this.accessTracker = new ConcurrentHashMap<String, Queue<Track>>();
} }
/* /*
@ -70,8 +71,8 @@ public class serverAccessTracker {
} }
// clear entries which had no entry for the maxTrackingTime time // clear entries which had no entry for the maxTrackingTime time
final Iterator<Map.Entry<String, ConcurrentLinkedQueue<Track>>> i = this.accessTracker.entrySet().iterator(); final Iterator<Map.Entry<String, Queue<Track>>> i = this.accessTracker.entrySet().iterator();
ConcurrentLinkedQueue<Track> track; Queue<Track> track;
while (i.hasNext()) { while (i.hasNext()) {
track = i.next().getValue(); track = i.next().getValue();
clearTooOldAccess(track); clearTooOldAccess(track);
@ -111,7 +112,7 @@ public class serverAccessTracker {
return c; return c;
} }
private void clearTooOldAccess(final ConcurrentLinkedQueue<Track> access) { private void clearTooOldAccess(final Queue<Track> access) {
final long time = System.currentTimeMillis() - this.maxTrackingTime; final long time = System.currentTimeMillis() - this.maxTrackingTime;
final Iterator<Track> e = access.iterator(); final Iterator<Track> e = access.iterator();
Track l; Track l;
@ -130,9 +131,9 @@ public class serverAccessTracker {
// learn that a specific host has accessed a specific path // learn that a specific host has accessed a specific path
if (accessPath == null) accessPath="NULL"; if (accessPath == null) accessPath="NULL";
ConcurrentLinkedQueue<Track> track = this.accessTracker.get(host); Queue<Track> track = this.accessTracker.get(host);
if (track == null) { if (track == null) {
track = new ConcurrentLinkedQueue<Track>(); track = new LinkedBlockingQueue<Track>();
track.add(new Track(System.currentTimeMillis(), accessPath)); track.add(new Track(System.currentTimeMillis(), accessPath));
// add to tracker // add to tracker
this.accessTracker.put(host, track); this.accessTracker.put(host, track);
@ -145,7 +146,7 @@ public class serverAccessTracker {
public Collection<Track> accessTrack(final String host) { public Collection<Track> accessTrack(final String host) {
// returns mapping from Long(accesstime) to path // returns mapping from Long(accesstime) to path
final ConcurrentLinkedQueue<Track> access = this.accessTracker.get(host); final Queue<Track> access = this.accessTracker.get(host);
if (access == null) return null; if (access == null) return null;
// clear too old entries // clear too old entries
clearTooOldAccess(access); clearTooOldAccess(access);
@ -157,7 +158,7 @@ public class serverAccessTracker {
public Iterator<String> accessHosts() { public Iterator<String> accessHosts() {
// returns an iterator of hosts in tracker (String) // returns an iterator of hosts in tracker (String)
final Map<String, ConcurrentLinkedQueue<Track>> accessTrackerClone = new ConcurrentHashMap<String, ConcurrentLinkedQueue<Track>>(); final Map<String, Queue<Track>> accessTrackerClone = new ConcurrentHashMap<String, Queue<Track>>();
accessTrackerClone.putAll(this.accessTracker); accessTrackerClone.putAll(this.accessTracker);
return accessTrackerClone.keySet().iterator(); return accessTrackerClone.keySet().iterator();
} }

97
source/net/yacy/kelondro/data/image/ImageReferenceVars.java
View File

@ -27,7 +27,8 @@
package net.yacy.kelondro.data.image; package net.yacy.kelondro.data.image;
import java.util.Collection; import java.util.Collection;
import java.util.concurrent.ConcurrentLinkedQueue; import java.util.Queue;
import java.util.concurrent.LinkedBlockingQueue;
import net.yacy.kelondro.index.Row.Entry; import net.yacy.kelondro.index.Row.Entry;
import net.yacy.kelondro.order.Bitfield; import net.yacy.kelondro.order.Bitfield;
@ -54,7 +55,7 @@ public class ImageReferenceVars extends AbstractReference implements ImageRefere
posinphrase, posofphrase, posinphrase, posofphrase,
urlcomps, urllength, virtualAge, urlcomps, urllength, virtualAge,
wordsintext, wordsintitle; wordsintext, wordsintitle;
private final ConcurrentLinkedQueue<Integer> positions; private final Queue<Integer> positions;
public double termFrequency; public double termFrequency;
public ImageReferenceVars( public ImageReferenceVars(
@ -65,7 +66,7 @@ public class ImageReferenceVars extends AbstractReference implements ImageRefere
final int hitcount, // how often appears this word in the text final int hitcount, // how often appears this word in the text
final int wordcount, // total number of words final int wordcount, // total number of words
final int phrasecount, // total number of phrases final int phrasecount, // total number of phrases
final ConcurrentLinkedQueue<Integer> ps, // positions of words that are joined into the reference final Queue<Integer> ps, // positions of words that are joined into the reference
final int posinphrase, // position of word in its phrase final int posinphrase, // position of word in its phrase
final int posofphrase, // number of the phrase where word appears final int posofphrase, // number of the phrase where word appears
final long lastmodified, // last-modified time of the document where word appears final long lastmodified, // last-modified time of the document where word appears
@ -90,7 +91,7 @@ public class ImageReferenceVars extends AbstractReference implements ImageRefere
this.llocal = outlinksSame; this.llocal = outlinksSame;
this.lother = outlinksOther; this.lother = outlinksOther;
this.phrasesintext = phrasecount; this.phrasesintext = phrasecount;
this.positions = new ConcurrentLinkedQueue<Integer>(); this.positions = new LinkedBlockingQueue<Integer>();
for (Integer i: ps) this.positions.add(i); for (Integer i: ps) this.positions.add(i);
this.posinphrase = posinphrase; this.posinphrase = posinphrase;
this.posofphrase = posofphrase; this.posofphrase = posofphrase;
@ -113,7 +114,7 @@ public class ImageReferenceVars extends AbstractReference implements ImageRefere
this.llocal = e.llocal(); this.llocal = e.llocal();
this.lother = e.lother(); this.lother = e.lother();
this.phrasesintext = e.phrasesintext(); this.phrasesintext = e.phrasesintext();
this.positions = new ConcurrentLinkedQueue<Integer>(); this.positions = new LinkedBlockingQueue<Integer>();
for (Integer i: e.positions()) this.positions.add(i); for (Integer i: e.positions()) this.positions.add(i);
this.posinphrase = e.posinphrase(); this.posinphrase = e.posinphrase();
this.posofphrase = e.posofphrase(); this.posofphrase = e.posofphrase();
@ -184,113 +185,134 @@ public class ImageReferenceVars extends AbstractReference implements ImageRefere
this.termFrequency = this.termFrequency + v.termFrequency; this.termFrequency = this.termFrequency + v.termFrequency;
} }
@Override
public Bitfield flags() { public Bitfield flags() {
return flags; return this.flags;
} }
/* /*
public long freshUntil() { public long freshUntil() {
return freshUntil; return freshUntil;
} }
*/ */
@Override
public String getLanguage() { public String getLanguage() {
return language; return this.language;
} }
@Override
public char getType() { public char getType() {
return type; return this.type;
} }
@Override
public int hitcount() { public int hitcount() {
return hitcount; return this.hitcount;
} }
@Override
public boolean isOlder(final Reference other) { public boolean isOlder(final Reference other) {
assert false; // should not be used assert false; // should not be used
return false; return false;
} }
@Override
public long lastModified() { public long lastModified() {
return lastModified; return this.lastModified;
} }
@Override
public int llocal() { public int llocal() {
return llocal; return this.llocal;
} }
@Override
public int lother() { public int lother() {
return lother; return this.lother;
} }
@Override
public int phrasesintext() { public int phrasesintext() {
return phrasesintext; return this.phrasesintext;
} }
@Override
public int posinphrase() { public int posinphrase() {
return posinphrase; return this.posinphrase;
} }
@Override
public Collection<Integer> positions() { public Collection<Integer> positions() {
return this.positions; return this.positions;
} }
@Override
public int posofphrase() { public int posofphrase() {
return posofphrase; return this.posofphrase;
} }
public ImageReferenceRow toRowEntry() { public ImageReferenceRow toRowEntry() {
return new ImageReferenceRow( return new ImageReferenceRow(
urlHash, this.urlHash,
urllength, // byte-length of complete URL this.urllength, // byte-length of complete URL
urlcomps, // number of path components this.urlcomps, // number of path components
wordsintitle, // length of description/length (longer are better?) this.wordsintitle, // length of description/length (longer are better?)
hitcount, // how often appears this word in the text this.hitcount, // how often appears this word in the text
wordsintext, // total number of words this.wordsintext, // total number of words
phrasesintext, // total number of phrases this.phrasesintext, // total number of phrases
positions.iterator().next(), // position of word in all words this.positions.iterator().next(), // position of word in all words
posinphrase, // position of word in its phrase this.posinphrase, // position of word in its phrase
posofphrase, // number of the phrase where word appears this.posofphrase, // number of the phrase where word appears
lastModified, // last-modified time of the document where word appears this.lastModified, // last-modified time of the document where word appears
System.currentTimeMillis(), // update time; System.currentTimeMillis(), // update time;
language, // (guessed) language of document this.language, // (guessed) language of document
type, // type of document this.type, // type of document
llocal, // outlinks to same domain this.llocal, // outlinks to same domain
lother, // outlinks to other domain this.lother, // outlinks to other domain
flags // attributes to the url and to the word according the url this.flags // attributes to the url and to the word according the url
); );
} }
@Override
public Entry toKelondroEntry() { public Entry toKelondroEntry() {
return toRowEntry().toKelondroEntry(); return toRowEntry().toKelondroEntry();
} }
@Override
public String toPropertyForm() { public String toPropertyForm() {
return toRowEntry().toPropertyForm(); return toRowEntry().toPropertyForm();
} }
@Override
public byte[] urlhash() { public byte[] urlhash() {
return urlHash; return this.urlHash;
} }
@Override
public int urlcomps() { public int urlcomps() {
return urlcomps; return this.urlcomps;
} }
@Override
public int urllength() { public int urllength() {
return urllength; return this.urllength;
} }
@Override
public int virtualAge() { public int virtualAge() {
return virtualAge; return this.virtualAge;
} }
@Override
public int wordsintext() { public int wordsintext() {
return wordsintext; return this.wordsintext;
} }
@Override
public int wordsintitle() { public int wordsintitle() {
return wordsintitle; return this.wordsintitle;
} }
@Override
public double termFrequency() { public double termFrequency() {
if (this.termFrequency == 0.0) this.termFrequency = (((double) this.hitcount()) / ((double) (this.wordsintext() + this.wordsintitle() + 1))); if (this.termFrequency == 0.0) this.termFrequency = (((double) this.hitcount()) / ((double) (this.wordsintext() + this.wordsintitle() + 1)));
return this.termFrequency; return this.termFrequency;
@ -340,6 +362,7 @@ public class ImageReferenceVars extends AbstractReference implements ImageRefere
if (this.termFrequency < (d = other.termFrequency)) this.termFrequency = d; if (this.termFrequency < (d = other.termFrequency)) this.termFrequency = d;
} }
@Override
public void join(final Reference r) { public void join(final Reference r) {
// joins two entries into one entry // joins two entries into one entry

33
source/net/yacy/kelondro/data/word/WordReferenceVars.java
View File

@ -28,8 +28,8 @@ package net.yacy.kelondro.data.word;
import java.util.Collection; import java.util.Collection;
import java.util.Comparator; import java.util.Comparator;
import java.util.Queue;
import java.util.concurrent.BlockingQueue; import java.util.concurrent.BlockingQueue;
import java.util.concurrent.ConcurrentLinkedQueue;
import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.Semaphore; import java.util.concurrent.Semaphore;
@ -66,7 +66,7 @@ public class WordReferenceVars extends AbstractReference implements WordReferenc
urlcomps, urllength, urlcomps, urllength,
wordsintext, wordsintitle; wordsintext, wordsintitle;
private int virtualAge; private int virtualAge;
private final ConcurrentLinkedQueue<Integer> positions; private final Queue<Integer> positions;
public double termFrequency; public double termFrequency;
public WordReferenceVars( public WordReferenceVars(
@ -77,7 +77,7 @@ public class WordReferenceVars extends AbstractReference implements WordReferenc
final int hitcount, // how often appears this word in the text final int hitcount, // how often appears this word in the text
final int wordcount, // total number of words final int wordcount, // total number of words
final int phrasecount, // total number of phrases final int phrasecount, // total number of phrases
final ConcurrentLinkedQueue<Integer> ps, // positions of words that are joined into the reference final Queue<Integer> ps, // positions of words that are joined into the reference
final int posinphrase, // position of word in its phrase final int posinphrase, // position of word in its phrase
final int posofphrase, // number of the phrase where word appears final int posofphrase, // number of the phrase where word appears
final long lastmodified, // last-modified time of the document where word appears final long lastmodified, // last-modified time of the document where word appears
@ -101,7 +101,7 @@ public class WordReferenceVars extends AbstractReference implements WordReferenc
this.llocal = outlinksSame; this.llocal = outlinksSame;
this.lother = outlinksOther; this.lother = outlinksOther;
this.phrasesintext = phrasecount; this.phrasesintext = phrasecount;
this.positions = new ConcurrentLinkedQueue<Integer>(); this.positions = new LinkedBlockingQueue<Integer>();
if (ps.size() > 0) for (final Integer i: ps) this.positions.add(i); if (ps.size() > 0) for (final Integer i: ps) this.positions.add(i);
this.posinphrase = posinphrase; this.posinphrase = posinphrase;
this.posofphrase = posofphrase; this.posofphrase = posofphrase;
@ -124,7 +124,7 @@ public class WordReferenceVars extends AbstractReference implements WordReferenc
this.llocal = e.llocal(); this.llocal = e.llocal();
this.lother = e.lother(); this.lother = e.lother();
this.phrasesintext = e.phrasesintext(); this.phrasesintext = e.phrasesintext();
this.positions = new ConcurrentLinkedQueue<Integer>(); this.positions = new LinkedBlockingQueue<Integer>();
if (e.positions().size() > 0) for (final Integer i: e.positions()) this.positions.add(i); if (e.positions().size() > 0) for (final Integer i: e.positions()) this.positions.add(i);
this.posinphrase = e.posinphrase(); this.posinphrase = e.posinphrase();
this.posofphrase = e.posofphrase(); this.posofphrase = e.posofphrase();
@ -195,46 +195,57 @@ public class WordReferenceVars extends AbstractReference implements WordReferenc
this.termFrequency = this.termFrequency + v.termFrequency; this.termFrequency = this.termFrequency + v.termFrequency;
} }
@Override
public Bitfield flags() { public Bitfield flags() {
return this.flags; return this.flags;
} }
@Override
public byte[] getLanguage() { public byte[] getLanguage() {
return this.language; return this.language;
} }
@Override
public char getType() { public char getType() {
return this.type; return this.type;
} }
@Override
public int hitcount() { public int hitcount() {
return this.hitcount; return this.hitcount;
} }
@Override
public long lastModified() { public long lastModified() {
return this.lastModified; return this.lastModified;
} }
@Override
public int llocal() { public int llocal() {
return this.llocal; return this.llocal;
} }
@Override
public int lother() { public int lother() {
return this.lother; return this.lother;
} }
@Override
public int phrasesintext() { public int phrasesintext() {
return this.phrasesintext; return this.phrasesintext;
} }
@Override
public int posinphrase() { public int posinphrase() {
return this.posinphrase; return this.posinphrase;
} }
@Override
public Collection<Integer> positions() { public Collection<Integer> positions() {
return this.positions; return this.positions;
} }
@Override
public int posofphrase() { public int posofphrase() {
return this.posofphrase; return this.posofphrase;
} }
@ -261,14 +272,17 @@ public class WordReferenceVars extends AbstractReference implements WordReferenc
); );
} }
@Override
public Entry toKelondroEntry() { public Entry toKelondroEntry() {
return toRowEntry().toKelondroEntry(); return toRowEntry().toKelondroEntry();
} }
@Override
public String toPropertyForm() { public String toPropertyForm() {
return toRowEntry().toPropertyForm(); return toRowEntry().toPropertyForm();
} }
@Override
public byte[] urlhash() { public byte[] urlhash() {
return this.urlHash; return this.urlHash;
} }
@ -279,28 +293,34 @@ public class WordReferenceVars extends AbstractReference implements WordReferenc
return this.hostHash; return this.hostHash;
} }
@Override
public int urlcomps() { public int urlcomps() {
return this.urlcomps; return this.urlcomps;
} }
@Override
public int urllength() { public int urllength() {
return this.urllength; return this.urllength;
} }
@Override
public int virtualAge() { public int virtualAge() {
if (this.virtualAge > 0) return this.virtualAge; if (this.virtualAge > 0) return this.virtualAge;
this.virtualAge = MicroDate.microDateDays(this.lastModified); this.virtualAge = MicroDate.microDateDays(this.lastModified);
return this.virtualAge; return this.virtualAge;
} }
@Override
public int wordsintext() { public int wordsintext() {
return this.wordsintext; return this.wordsintext;
} }
@Override
public int wordsintitle() { public int wordsintitle() {
return this.wordsintitle; return this.wordsintitle;
} }
@Override
public double termFrequency() { public double termFrequency() {
if (this.termFrequency == 0.0) this.termFrequency = (((double) hitcount()) / ((double) (wordsintext() + wordsintitle() + 1))); if (this.termFrequency == 0.0) this.termFrequency = (((double) hitcount()) / ((double) (wordsintext() + wordsintitle() + 1)));
return this.termFrequency; return this.termFrequency;
@ -350,6 +370,7 @@ public class WordReferenceVars extends AbstractReference implements WordReferenc
if (this.termFrequency < (d = other.termFrequency)) this.termFrequency = d; if (this.termFrequency < (d = other.termFrequency)) this.termFrequency = d;
} }
@Override
public void join(final Reference r) { public void join(final Reference r) {
// joins two entries into one entry // joins two entries into one entry
@ -378,10 +399,12 @@ public class WordReferenceVars extends AbstractReference implements WordReferenc
return ByteArray.hashCode(this.urlHash); return ByteArray.hashCode(this.urlHash);
} }
@Override
public int compareTo(final WordReferenceVars o) { public int compareTo(final WordReferenceVars o) {
return Base64Order.enhancedCoder.compare(this.urlHash, o.urlhash()); return Base64Order.enhancedCoder.compare(this.urlHash, o.urlhash());
} }
@Override
public int compare(final WordReferenceVars o1, final WordReferenceVars o2) { public int compare(final WordReferenceVars o1, final WordReferenceVars o2) {
return o1.compareTo(o2); return o1.compareTo(o2);
} }

11
source/net/yacy/kelondro/util/EventTracker.java
View File

@ -28,8 +28,9 @@ package net.yacy.kelondro.util;
import java.util.Iterator; import java.util.Iterator;
import java.util.Map; import java.util.Map;
import java.util.Queue;
import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.LinkedBlockingQueue;
import net.yacy.peers.graphics.ProfilingGraph; import net.yacy.peers.graphics.ProfilingGraph;
@ -49,7 +50,7 @@ public class EventTracker {
SEARCH; SEARCH;
} }
private final static Map<EClass, ConcurrentLinkedQueue<Event>> historyMaps = new ConcurrentHashMap<EClass, ConcurrentLinkedQueue<Event>>(); private final static Map<EClass, Queue<Event>> historyMaps = new ConcurrentHashMap<EClass, Queue<Event>>();
private final static Map<EClass, Long> eventAccess = new ConcurrentHashMap<EClass, Long>(); // value: last time when this was accessed private final static Map<EClass, Long> eventAccess = new ConcurrentHashMap<EClass, Long>(); // value: last time when this was accessed
public final static void delete(final EClass eventName) { public final static void delete(final EClass eventName) {
@ -73,11 +74,11 @@ public class EventTracker {
} }
// get event history container // get event history container
ConcurrentLinkedQueue<Event> history = historyMaps.get(eventName); Queue<Event> history = historyMaps.get(eventName);
// create history // create history
if (history == null) { if (history == null) {
history = new ConcurrentLinkedQueue<Event>(); history = new LinkedBlockingQueue<Event>();
// update entry // update entry
history.offer(new Event(eventPayload)); history.offer(new Event(eventPayload));
@ -109,7 +110,7 @@ public class EventTracker {
} }
public final static Iterator<Event> getHistory(final EClass eventName) { public final static Iterator<Event> getHistory(final EClass eventName) {
final ConcurrentLinkedQueue<Event> list = historyMaps.get(eventName); final Queue<Event> list = historyMaps.get(eventName);
if (list == null) return null; if (list == null) return null;
return list.iterator(); return list.iterator();
} }