mirror of
https://github.com/yacy/yacy_search_server.git
synced 2024-09-19 00:01:41 +02:00
ConcurrentLinkedQueue has a VERY long return time on the .size() method.
See http://docs.oracle.com/javase/6/docs/api/java/util/concurrent/ConcurrentLinkedQueue.html and the following test programm: public class QueueLengthTimeTest { public static long countTest(Queue<Integer> q, int c) { long t = System.currentTimeMillis(); for (int i = 0; i < c; i++) { q.add(q.size()); } return System.currentTimeMillis() - t; } public static void main(String[] args) { int c = 1; for (int i = 0; i < 100; i++) { Runtime.getRuntime().gc(); long t1 = countTest(new ArrayBlockingQueue<Integer>(c), c); Runtime.getRuntime().gc(); long t2 = countTest(new LinkedBlockingQueue<Integer>(), c); Runtime.getRuntime().gc(); long t3 = countTest(new ConcurrentLinkedQueue<Integer>(), c); System.out.println("count = " + c + ": ArrayBlockingQueue = " + t1 + ", LinkedBlockingQueue = " + t2 + ", ConcurrentLinkedQueue = " + t3); c = c * 2; } } }
This commit is contained in:
Michael Peter Christen
parent
8aba045ba1
commit
2fc8ecee36
|
|
@ -9,7 +9,7 @@
|
|||
// $LastChangedBy$
|
||||
//
|
||||
// LICENSE
|
||||
//
|
||||
//
|
||||
// This program is free software; you can redistribute it and/or modify
|
||||
// it under the terms of the GNU General Public License as published by
|
||||
// the Free Software Foundation; either version 2 of the License, or
|
||||
|
|
@ -33,7 +33,7 @@ import java.util.Locale;
|
|||
import java.util.Map;
|
||||
import java.util.Map.Entry;
|
||||
import java.util.TreeSet;
|
||||
import java.util.concurrent.ConcurrentLinkedQueue;
|
||||
import java.util.concurrent.LinkedBlockingQueue;
|
||||
|
||||
import net.yacy.cora.protocol.Domains;
|
||||
import net.yacy.cora.protocol.HeaderFramework;
|
||||
|
|
@ -48,20 +48,20 @@ import de.anomic.server.serverObjects;
|
|||
import de.anomic.server.serverSwitch;
|
||||
|
||||
public class AccessTracker_p {
|
||||
|
||||
|
||||
private static SimpleDateFormat SimpleFormatter = new SimpleDateFormat("yyyy/MM/dd HH:mm:ss", Locale.US);
|
||||
|
||||
private static Collection<Track> listclone (final Collection<Track> m) {
|
||||
final Collection<Track> accessClone = new ConcurrentLinkedQueue<Track>();
|
||||
final Collection<Track> accessClone = new LinkedBlockingQueue<Track>();
|
||||
try {
|
||||
accessClone.addAll(m);
|
||||
} catch (final ConcurrentModificationException e) {}
|
||||
return accessClone;
|
||||
}
|
||||
|
||||
|
||||
public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) {
|
||||
final Switchboard sb = (Switchboard) env;
|
||||
|
||||
|
||||
// return variable that accumulates replacements
|
||||
final serverObjects prop = new serverObjects();
|
||||
prop.setLocalized(!(header.get(HeaderFramework.CONNECTION_PROP_PATH)).endsWith(".xml"));
|
||||
|
|
@ -70,7 +70,7 @@ public class AccessTracker_p {
|
|||
page = post.getInt("page", 0);
|
||||
}
|
||||
prop.put("page", page);
|
||||
|
||||
|
||||
final int maxCount = 1000;
|
||||
boolean dark = true;
|
||||
if (page == 0) {
|
||||
|
|
@ -90,7 +90,7 @@ public class AccessTracker_p {
|
|||
} catch (final ConcurrentModificationException e) {} // we don't want to synchronize this
|
||||
prop.put("page_list", entCount);
|
||||
prop.put("page_num", entCount);
|
||||
|
||||
|
||||
entCount = 0;
|
||||
try {
|
||||
for (final Map.Entry<String, Integer> bfe: serverCore.bfHost.entrySet()) {
|
||||
|
|
@ -152,7 +152,7 @@ public class AccessTracker_p {
|
|||
long stimeSum1 = 0;
|
||||
long rtimeSum1 = 0;
|
||||
int m = 0;
|
||||
|
||||
|
||||
while (ai.hasNext()) {
|
||||
try {
|
||||
query = ai.next();
|
||||
|
|
@ -198,7 +198,7 @@ public class AccessTracker_p {
|
|||
prop.put("page_list", m);
|
||||
prop.put("page_num", m);
|
||||
prop.put("page_resultcount", rcount);
|
||||
|
||||
|
||||
// Put -1 instead of NaN as result for empty search list and return the safe HTML blank char for table output
|
||||
if (m == 0) {
|
||||
m = -1;
|
||||
|
|
@ -249,7 +249,7 @@ public class AccessTracker_p {
|
|||
entry = i.next();
|
||||
host = entry.getKey();
|
||||
handles = entry.getValue();
|
||||
|
||||
|
||||
int dateCount = 0;
|
||||
final Iterator<Long> ii = handles.iterator();
|
||||
while (ii.hasNext()) {
|
||||
|
|
@ -262,7 +262,7 @@ public class AccessTracker_p {
|
|||
final int qph = handles.tailSet(Long.valueOf(System.currentTimeMillis() - 1000 * 60 * 60)).size();
|
||||
qphSum += qph;
|
||||
prop.put("page_list_" + m + "_qph", qph);
|
||||
|
||||
|
||||
prop.put("page_list_" + m + "_dark", ((dark) ? 1 : 0) ); dark =! dark;
|
||||
prop.putHTML("page_list_" + m + "_host", host);
|
||||
if (page == 5) {
|
||||
|
|
@ -276,7 +276,7 @@ public class AccessTracker_p {
|
|||
}
|
||||
} catch (final ConcurrentModificationException e) {} // we dont want to synchronize this
|
||||
// return empty values to not break the table view if no results can be listed
|
||||
if (m==0) {
|
||||
if (m==0) {
|
||||
prop.put("page_list", 1);
|
||||
prop.put("page_list_0_dates_0_date", "");
|
||||
prop.put("page_list_0_dates", 1);
|
||||
|
|
@ -295,5 +295,5 @@ public class AccessTracker_p {
|
|||
// return rewrite properties
|
||||
return prop;
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -9,7 +9,7 @@
|
|||
// $LastChangedBy$
|
||||
//
|
||||
// LICENSE
|
||||
//
|
||||
//
|
||||
// This program is free software; you can redistribute it and/or modify
|
||||
// it under the terms of the GNU General Public License as published by
|
||||
// the Free Software Foundation; either version 2 of the License, or
|
||||
|
|
@ -27,9 +27,10 @@
|
|||
package de.anomic.crawler;
|
||||
|
||||
import java.util.Map;
|
||||
import java.util.Queue;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
import java.util.concurrent.ConcurrentLinkedQueue;
|
||||
import java.util.concurrent.ConcurrentMap;
|
||||
import java.util.concurrent.LinkedBlockingQueue;
|
||||
|
||||
import net.yacy.cora.document.MultiProtocolURI;
|
||||
import net.yacy.document.Document;
|
||||
|
|
@ -43,30 +44,30 @@ public class ResultImages {
|
|||
// we maintain two different queues for private and public crawls and divide both into two halves:
|
||||
// such images that appear to be good quality for a image monitor bacause their size is known, and other images
|
||||
// that are not declared with sizes.
|
||||
private static final ConcurrentLinkedQueue<OriginEntry> privateImageQueueHigh = new ConcurrentLinkedQueue<OriginEntry>();
|
||||
private static final ConcurrentLinkedQueue<OriginEntry> privateImageQueueLow = new ConcurrentLinkedQueue<OriginEntry>();
|
||||
private static final ConcurrentLinkedQueue<OriginEntry> publicImageQueueHigh = new ConcurrentLinkedQueue<OriginEntry>();
|
||||
private static final ConcurrentLinkedQueue<OriginEntry> publicImageQueueLow = new ConcurrentLinkedQueue<OriginEntry>();
|
||||
private static final Queue<OriginEntry> privateImageQueueHigh = new LinkedBlockingQueue<OriginEntry>();
|
||||
private static final Queue<OriginEntry> privateImageQueueLow = new LinkedBlockingQueue<OriginEntry>();
|
||||
private static final Queue<OriginEntry> publicImageQueueHigh = new LinkedBlockingQueue<OriginEntry>();
|
||||
private static final Queue<OriginEntry> publicImageQueueLow = new LinkedBlockingQueue<OriginEntry>();
|
||||
|
||||
// we also check all links for a double-check so we don't get the same image more than once in any queue
|
||||
// image links may appear double here even if the pages where the image links are embedded already are checked for double-occurrence:
|
||||
// the same images may be linked from different pages
|
||||
private static final ConcurrentMap<MultiProtocolURI, Long> doubleCheck = new ConcurrentHashMap<MultiProtocolURI, Long>(); // (url, time) when the url appeared first
|
||||
|
||||
|
||||
public static void registerImages(final DigestURI source, final Document document, final boolean privateEntry) {
|
||||
if (document == null) return;
|
||||
if (source == null) return;
|
||||
|
||||
if (MemoryControl.shortStatus()) clearQueues();
|
||||
limitQueues(1000);
|
||||
|
||||
|
||||
final Map<MultiProtocolURI, ImageEntry> images = document.getImages();
|
||||
for (final ImageEntry image: images.values()) {
|
||||
// do a double-check; attention: this can be time-consuming since this possibly needs a DNS-lookup
|
||||
if (image == null || image.url() == null) continue;
|
||||
if (doubleCheck.containsKey(image.url())) continue;
|
||||
doubleCheck.put(image.url(), System.currentTimeMillis());
|
||||
|
||||
|
||||
final String name = image.url().getFile();
|
||||
boolean good = false;
|
||||
if (image.width() > 120 &&
|
||||
|
|
@ -76,7 +77,7 @@ public class ResultImages {
|
|||
name.lastIndexOf(".gif") == -1) {
|
||||
// && ((urlString.lastIndexOf(".jpg") != -1)) ||
|
||||
// ((urlString.lastIndexOf(".png") != -1)){
|
||||
|
||||
|
||||
good = true;
|
||||
float ratio;
|
||||
if (image.width() > image.height()) {
|
||||
|
|
@ -101,7 +102,7 @@ public class ResultImages {
|
|||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
public static OriginEntry next(final boolean privateEntryOnly) {
|
||||
OriginEntry e = null;
|
||||
if (privateEntryOnly) {
|
||||
|
|
@ -115,7 +116,7 @@ public class ResultImages {
|
|||
}
|
||||
return e;
|
||||
}
|
||||
|
||||
|
||||
public static int queueSize(final boolean privateEntryOnly) {
|
||||
int publicSize = 0;
|
||||
if (!privateEntryOnly) {
|
||||
|
|
@ -123,19 +124,19 @@ public class ResultImages {
|
|||
}
|
||||
return privateImageQueueHigh.size() + privateImageQueueLow.size() + publicSize;
|
||||
}
|
||||
|
||||
|
||||
public static int privateQueueHighSize() {
|
||||
return privateImageQueueHigh.size();
|
||||
}
|
||||
|
||||
|
||||
public static int privateQueueLowSize() {
|
||||
return privateImageQueueLow.size();
|
||||
}
|
||||
|
||||
|
||||
public static int publicQueueHighSize() {
|
||||
return publicImageQueueHigh.size();
|
||||
}
|
||||
|
||||
|
||||
public static int publicQueueLowSize() {
|
||||
return publicImageQueueLow.size();
|
||||
}
|
||||
|
|
@ -147,14 +148,14 @@ public class ResultImages {
|
|||
publicImageQueueLow.clear();
|
||||
doubleCheck.clear();
|
||||
}
|
||||
|
||||
|
||||
public static void limitQueues(int limit) {
|
||||
while (privateImageQueueHigh.size() > limit) privateImageQueueHigh.poll();
|
||||
while (privateImageQueueLow.size() > limit) privateImageQueueLow.poll();
|
||||
while (publicImageQueueHigh.size() > limit) publicImageQueueHigh.poll();
|
||||
while (publicImageQueueLow.size() > limit) publicImageQueueLow.poll();
|
||||
}
|
||||
|
||||
|
||||
public static class OriginEntry {
|
||||
public ImageEntry imageEntry;
|
||||
public MultiProtocolURI baseURL;
|
||||
|
|
@ -163,5 +164,5 @@ public class ResultImages {
|
|||
this.baseURL = baseURL;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -31,12 +31,13 @@ import java.io.IOException;
|
|||
import java.util.ArrayList;
|
||||
import java.util.Date;
|
||||
import java.util.Iterator;
|
||||
import java.util.concurrent.ConcurrentLinkedQueue;
|
||||
import java.util.Queue;
|
||||
import java.util.concurrent.LinkedBlockingQueue;
|
||||
|
||||
import net.yacy.cora.document.ASCII;
|
||||
import net.yacy.cora.document.UTF8;
|
||||
import net.yacy.cora.services.federated.solr.SolrShardingConnector;
|
||||
import net.yacy.cora.services.federated.solr.SolrConnector;
|
||||
import net.yacy.cora.services.federated.solr.SolrShardingConnector;
|
||||
import net.yacy.kelondro.data.meta.DigestURI;
|
||||
import net.yacy.kelondro.data.word.Word;
|
||||
import net.yacy.kelondro.index.Index;
|
||||
|
|
@ -76,7 +77,7 @@ public class ZURL implements Iterable<ZURL.Entry> {
|
|||
|
||||
// the class object
|
||||
private Index urlIndex;
|
||||
private final ConcurrentLinkedQueue<byte[]> stack;
|
||||
private final Queue<byte[]> stack;
|
||||
private final SolrConnector solrConnector;
|
||||
|
||||
public ZURL(
|
||||
|
|
@ -105,14 +106,14 @@ public class ZURL implements Iterable<ZURL.Entry> {
|
|||
}
|
||||
}
|
||||
//urlIndex = new kelondroFlexTable(cachePath, tablename, -1, rowdef, 0, true);
|
||||
this.stack = new ConcurrentLinkedQueue<byte[]>();
|
||||
this.stack = new LinkedBlockingQueue<byte[]>();
|
||||
}
|
||||
|
||||
public ZURL(final SolrShardingConnector solrConnector) {
|
||||
this.solrConnector = solrConnector;
|
||||
// creates a new ZUR in RAM
|
||||
this.urlIndex = new RowSet(rowdef);
|
||||
this.stack = new ConcurrentLinkedQueue<byte[]>();
|
||||
this.stack = new LinkedBlockingQueue<byte[]>();
|
||||
}
|
||||
|
||||
public void clear() throws IOException {
|
||||
|
|
@ -163,6 +164,7 @@ public class ZURL implements Iterable<ZURL.Entry> {
|
|||
while (this.stack.size() > maxStackSize) this.stack.poll();
|
||||
}
|
||||
|
||||
@Override
|
||||
public Iterator<ZURL.Entry> iterator() {
|
||||
return new EntryIterator();
|
||||
}
|
||||
|
|
@ -185,14 +187,17 @@ public class ZURL implements Iterable<ZURL.Entry> {
|
|||
public EntryIterator() {
|
||||
this.hi = ZURL.this.stack.iterator();
|
||||
}
|
||||
@Override
|
||||
public boolean hasNext() {
|
||||
return this.hi.hasNext();
|
||||
}
|
||||
|
||||
@Override
|
||||
public ZURL.Entry next() {
|
||||
return get(this.hi.next());
|
||||
}
|
||||
|
||||
@Override
|
||||
public void remove() {
|
||||
this.hi.remove();
|
||||
}
|
||||
|
|
@ -326,11 +331,13 @@ public class ZURL implements Iterable<ZURL.Entry> {
|
|||
this.error = false;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean hasNext() {
|
||||
if (this.error) return false;
|
||||
return this.i.hasNext();
|
||||
}
|
||||
|
||||
@Override
|
||||
public Entry next() throws RuntimeException {
|
||||
final Row.Entry e = this.i.next();
|
||||
if (e == null) return null;
|
||||
|
|
@ -341,6 +348,7 @@ public class ZURL implements Iterable<ZURL.Entry> {
|
|||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void remove() {
|
||||
this.i.remove();
|
||||
}
|
||||
|
|
|
|||
|
|
@ -9,7 +9,7 @@
|
|||
// $LastChangedBy$
|
||||
//
|
||||
// LICENSE
|
||||
//
|
||||
//
|
||||
// This program is free software; you can redistribute it and/or modify
|
||||
// it under the terms of the GNU General Public License as published by
|
||||
// the Free Software Foundation; either version 2 of the License, or
|
||||
|
|
@ -26,9 +26,10 @@
|
|||
|
||||
package de.anomic.data;
|
||||
|
||||
import java.util.Queue;
|
||||
import java.util.Random;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
import java.util.concurrent.ConcurrentLinkedQueue;
|
||||
import java.util.concurrent.LinkedBlockingQueue;
|
||||
|
||||
import net.yacy.kelondro.data.meta.DigestURI;
|
||||
|
||||
|
|
@ -39,47 +40,47 @@ public class URLLicense {
|
|||
// it is used in case of snippet- and preview-Image-fetching to grant also non-authorized users the usage of a image-fetcher servlet
|
||||
private static final int maxQueue = 500;
|
||||
private static final long minCheck = 5000;
|
||||
|
||||
|
||||
private final Random random;
|
||||
private final ConcurrentHashMap<String, DigestURI> permissions;
|
||||
private final ConcurrentLinkedQueue<String> aging;
|
||||
private final Queue<String> aging;
|
||||
private long lastCheck;
|
||||
private final int keylen;
|
||||
|
||||
|
||||
public URLLicense(final int keylen) {
|
||||
this.permissions = new ConcurrentHashMap<String, DigestURI>();
|
||||
this.aging = new ConcurrentLinkedQueue<String>();
|
||||
this.aging = new LinkedBlockingQueue<String>();
|
||||
this.lastCheck = System.currentTimeMillis();
|
||||
this.random = new Random(System.currentTimeMillis());
|
||||
this.keylen = keylen;
|
||||
}
|
||||
|
||||
|
||||
public String aquireLicense(final DigestURI url) {
|
||||
// generate license key
|
||||
StringBuilder stringBuilder = new StringBuilder(keylen * 2);
|
||||
StringBuilder stringBuilder = new StringBuilder(this.keylen * 2);
|
||||
if (url == null) return stringBuilder.toString();
|
||||
while (stringBuilder.length() < keylen) stringBuilder.append(Integer.toHexString(random.nextInt()));
|
||||
String license = stringBuilder.substring(0, keylen);
|
||||
while (stringBuilder.length() < this.keylen) stringBuilder.append(Integer.toHexString(this.random.nextInt()));
|
||||
String license = stringBuilder.substring(0, this.keylen);
|
||||
// store reference to url with license key
|
||||
permissions.put(license, url);
|
||||
aging.add(license);
|
||||
this.permissions.put(license, url);
|
||||
this.aging.add(license);
|
||||
if (System.currentTimeMillis() - this.lastCheck > minCheck) {
|
||||
// check aging
|
||||
this.lastCheck = System.currentTimeMillis();
|
||||
String s;
|
||||
while (aging.size() > maxQueue) {
|
||||
s = aging.poll();
|
||||
if (s != null) permissions.remove(s);
|
||||
while (this.aging.size() > maxQueue) {
|
||||
s = this.aging.poll();
|
||||
if (s != null) this.permissions.remove(s);
|
||||
}
|
||||
}
|
||||
// return the license key
|
||||
return license;
|
||||
}
|
||||
|
||||
|
||||
public DigestURI releaseLicense(final String license) {
|
||||
DigestURI url = null;
|
||||
url = permissions.remove(license);
|
||||
url = this.permissions.remove(license);
|
||||
return url;
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -24,8 +24,9 @@ import java.util.Collection;
|
|||
import java.util.Iterator;
|
||||
import java.util.Map;
|
||||
import java.util.NoSuchElementException;
|
||||
import java.util.Queue;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
import java.util.concurrent.ConcurrentLinkedQueue;
|
||||
import java.util.concurrent.LinkedBlockingQueue;
|
||||
|
||||
public class serverAccessTracker {
|
||||
|
||||
|
|
@ -34,7 +35,7 @@ public class serverAccessTracker {
|
|||
private final long maxTrackingTime;
|
||||
private final int maxTrackingCount;
|
||||
private final int maxHostCount;
|
||||
private final ConcurrentHashMap<String, ConcurrentLinkedQueue<Track>> accessTracker; // mappings from requesting host to an ArrayList of serverTrack-entries
|
||||
private final ConcurrentHashMap<String, Queue<Track>> accessTracker; // mappings from requesting host to an ArrayList of serverTrack-entries
|
||||
private long lastCleanup;
|
||||
|
||||
public static class Track {
|
||||
|
|
@ -56,7 +57,7 @@ public class serverAccessTracker {
|
|||
this.maxTrackingTime = maxTrackingTime;
|
||||
this.maxTrackingCount = maxTrackingCount;
|
||||
this.maxHostCount = maxTrackingHostCount;
|
||||
this.accessTracker = new ConcurrentHashMap<String, ConcurrentLinkedQueue<Track>>();
|
||||
this.accessTracker = new ConcurrentHashMap<String, Queue<Track>>();
|
||||
}
|
||||
|
||||
/*
|
||||
|
|
@ -70,8 +71,8 @@ public class serverAccessTracker {
|
|||
}
|
||||
|
||||
// clear entries which had no entry for the maxTrackingTime time
|
||||
final Iterator<Map.Entry<String, ConcurrentLinkedQueue<Track>>> i = this.accessTracker.entrySet().iterator();
|
||||
ConcurrentLinkedQueue<Track> track;
|
||||
final Iterator<Map.Entry<String, Queue<Track>>> i = this.accessTracker.entrySet().iterator();
|
||||
Queue<Track> track;
|
||||
while (i.hasNext()) {
|
||||
track = i.next().getValue();
|
||||
clearTooOldAccess(track);
|
||||
|
|
@ -111,7 +112,7 @@ public class serverAccessTracker {
|
|||
return c;
|
||||
}
|
||||
|
||||
private void clearTooOldAccess(final ConcurrentLinkedQueue<Track> access) {
|
||||
private void clearTooOldAccess(final Queue<Track> access) {
|
||||
final long time = System.currentTimeMillis() - this.maxTrackingTime;
|
||||
final Iterator<Track> e = access.iterator();
|
||||
Track l;
|
||||
|
|
@ -130,9 +131,9 @@ public class serverAccessTracker {
|
|||
|
||||
// learn that a specific host has accessed a specific path
|
||||
if (accessPath == null) accessPath="NULL";
|
||||
ConcurrentLinkedQueue<Track> track = this.accessTracker.get(host);
|
||||
Queue<Track> track = this.accessTracker.get(host);
|
||||
if (track == null) {
|
||||
track = new ConcurrentLinkedQueue<Track>();
|
||||
track = new LinkedBlockingQueue<Track>();
|
||||
track.add(new Track(System.currentTimeMillis(), accessPath));
|
||||
// add to tracker
|
||||
this.accessTracker.put(host, track);
|
||||
|
|
@ -145,7 +146,7 @@ public class serverAccessTracker {
|
|||
public Collection<Track> accessTrack(final String host) {
|
||||
// returns mapping from Long(accesstime) to path
|
||||
|
||||
final ConcurrentLinkedQueue<Track> access = this.accessTracker.get(host);
|
||||
final Queue<Track> access = this.accessTracker.get(host);
|
||||
if (access == null) return null;
|
||||
// clear too old entries
|
||||
clearTooOldAccess(access);
|
||||
|
|
@ -157,7 +158,7 @@ public class serverAccessTracker {
|
|||
|
||||
public Iterator<String> accessHosts() {
|
||||
// returns an iterator of hosts in tracker (String)
|
||||
final Map<String, ConcurrentLinkedQueue<Track>> accessTrackerClone = new ConcurrentHashMap<String, ConcurrentLinkedQueue<Track>>();
|
||||
final Map<String, Queue<Track>> accessTrackerClone = new ConcurrentHashMap<String, Queue<Track>>();
|
||||
accessTrackerClone.putAll(this.accessTracker);
|
||||
return accessTrackerClone.keySet().iterator();
|
||||
}
|
||||
|
|
|
|||
|
|
@ -9,7 +9,7 @@
|
|||
// $LastChangedBy$
|
||||
//
|
||||
// LICENSE
|
||||
//
|
||||
//
|
||||
// This program is free software; you can redistribute it and/or modify
|
||||
// it under the terms of the GNU General Public License as published by
|
||||
// the Free Software Foundation; either version 2 of the License, or
|
||||
|
|
@ -27,7 +27,8 @@
|
|||
package net.yacy.kelondro.data.image;
|
||||
|
||||
import java.util.Collection;
|
||||
import java.util.concurrent.ConcurrentLinkedQueue;
|
||||
import java.util.Queue;
|
||||
import java.util.concurrent.LinkedBlockingQueue;
|
||||
|
||||
import net.yacy.kelondro.index.Row.Entry;
|
||||
import net.yacy.kelondro.order.Bitfield;
|
||||
|
|
@ -43,8 +44,8 @@ public class ImageReferenceVars extends AbstractReference implements ImageRefere
|
|||
* object for termination of concurrent blocking queue processing
|
||||
*/
|
||||
public static final ImageReferenceVars poison = new ImageReferenceVars();
|
||||
|
||||
|
||||
|
||||
|
||||
public Bitfield flags;
|
||||
public long lastModified;
|
||||
public byte[] urlHash;
|
||||
|
|
@ -54,9 +55,9 @@ public class ImageReferenceVars extends AbstractReference implements ImageRefere
|
|||
posinphrase, posofphrase,
|
||||
urlcomps, urllength, virtualAge,
|
||||
wordsintext, wordsintitle;
|
||||
private final ConcurrentLinkedQueue<Integer> positions;
|
||||
private final Queue<Integer> positions;
|
||||
public double termFrequency;
|
||||
|
||||
|
||||
public ImageReferenceVars(
|
||||
final byte[] urlHash,
|
||||
final int urlLength, // byte-length of complete URL
|
||||
|
|
@ -65,7 +66,7 @@ public class ImageReferenceVars extends AbstractReference implements ImageRefere
|
|||
final int hitcount, // how often appears this word in the text
|
||||
final int wordcount, // total number of words
|
||||
final int phrasecount, // total number of phrases
|
||||
final ConcurrentLinkedQueue<Integer> ps, // positions of words that are joined into the reference
|
||||
final Queue<Integer> ps, // positions of words that are joined into the reference
|
||||
final int posinphrase, // position of word in its phrase
|
||||
final int posofphrase, // number of the phrase where word appears
|
||||
final long lastmodified, // last-modified time of the document where word appears
|
||||
|
|
@ -90,7 +91,7 @@ public class ImageReferenceVars extends AbstractReference implements ImageRefere
|
|||
this.llocal = outlinksSame;
|
||||
this.lother = outlinksOther;
|
||||
this.phrasesintext = phrasecount;
|
||||
this.positions = new ConcurrentLinkedQueue<Integer>();
|
||||
this.positions = new LinkedBlockingQueue<Integer>();
|
||||
for (Integer i: ps) this.positions.add(i);
|
||||
this.posinphrase = posinphrase;
|
||||
this.posofphrase = posofphrase;
|
||||
|
|
@ -101,7 +102,7 @@ public class ImageReferenceVars extends AbstractReference implements ImageRefere
|
|||
this.wordsintitle = titleLength;
|
||||
this.termFrequency = termfrequency;
|
||||
}
|
||||
|
||||
|
||||
public ImageReferenceVars(final ImageReference e) {
|
||||
this.flags = e.flags();
|
||||
//this.freshUntil = e.freshUntil();
|
||||
|
|
@ -113,7 +114,7 @@ public class ImageReferenceVars extends AbstractReference implements ImageRefere
|
|||
this.llocal = e.llocal();
|
||||
this.lother = e.lother();
|
||||
this.phrasesintext = e.phrasesintext();
|
||||
this.positions = new ConcurrentLinkedQueue<Integer>();
|
||||
this.positions = new LinkedBlockingQueue<Integer>();
|
||||
for (Integer i: e.positions()) this.positions.add(i);
|
||||
this.posinphrase = e.posinphrase();
|
||||
this.posofphrase = e.posofphrase();
|
||||
|
|
@ -124,7 +125,7 @@ public class ImageReferenceVars extends AbstractReference implements ImageRefere
|
|||
this.wordsintitle = e.wordsintitle();
|
||||
this.termFrequency = e.termFrequency();
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* initializer for special poison object
|
||||
*/
|
||||
|
|
@ -148,7 +149,7 @@ public class ImageReferenceVars extends AbstractReference implements ImageRefere
|
|||
this.wordsintitle = 0;
|
||||
this.termFrequency = 0.0;
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public ImageReferenceVars clone() {
|
||||
final ImageReferenceVars c = new ImageReferenceVars(
|
||||
|
|
@ -172,7 +173,7 @@ public class ImageReferenceVars extends AbstractReference implements ImageRefere
|
|||
this.termFrequency);
|
||||
return c;
|
||||
}
|
||||
|
||||
|
||||
public void join(final ImageReferenceVars v) {
|
||||
// combine the distance
|
||||
this.positions.addAll(v.positions);
|
||||
|
|
@ -184,118 +185,139 @@ public class ImageReferenceVars extends AbstractReference implements ImageRefere
|
|||
this.termFrequency = this.termFrequency + v.termFrequency;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Bitfield flags() {
|
||||
return flags;
|
||||
return this.flags;
|
||||
}
|
||||
/*
|
||||
public long freshUntil() {
|
||||
return freshUntil;
|
||||
}
|
||||
*/
|
||||
@Override
|
||||
public String getLanguage() {
|
||||
return language;
|
||||
return this.language;
|
||||
}
|
||||
|
||||
@Override
|
||||
public char getType() {
|
||||
return type;
|
||||
return this.type;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int hitcount() {
|
||||
return hitcount;
|
||||
return this.hitcount;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean isOlder(final Reference other) {
|
||||
assert false; // should not be used
|
||||
return false;
|
||||
}
|
||||
|
||||
@Override
|
||||
public long lastModified() {
|
||||
return lastModified;
|
||||
return this.lastModified;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int llocal() {
|
||||
return llocal;
|
||||
return this.llocal;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int lother() {
|
||||
return lother;
|
||||
return this.lother;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int phrasesintext() {
|
||||
return phrasesintext;
|
||||
return this.phrasesintext;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int posinphrase() {
|
||||
return posinphrase;
|
||||
return this.posinphrase;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Collection<Integer> positions() {
|
||||
return this.positions;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int posofphrase() {
|
||||
return posofphrase;
|
||||
return this.posofphrase;
|
||||
}
|
||||
|
||||
|
||||
public ImageReferenceRow toRowEntry() {
|
||||
return new ImageReferenceRow(
|
||||
urlHash,
|
||||
urllength, // byte-length of complete URL
|
||||
urlcomps, // number of path components
|
||||
wordsintitle, // length of description/length (longer are better?)
|
||||
hitcount, // how often appears this word in the text
|
||||
wordsintext, // total number of words
|
||||
phrasesintext, // total number of phrases
|
||||
positions.iterator().next(), // position of word in all words
|
||||
posinphrase, // position of word in its phrase
|
||||
posofphrase, // number of the phrase where word appears
|
||||
lastModified, // last-modified time of the document where word appears
|
||||
this.urlHash,
|
||||
this.urllength, // byte-length of complete URL
|
||||
this.urlcomps, // number of path components
|
||||
this.wordsintitle, // length of description/length (longer are better?)
|
||||
this.hitcount, // how often appears this word in the text
|
||||
this.wordsintext, // total number of words
|
||||
this.phrasesintext, // total number of phrases
|
||||
this.positions.iterator().next(), // position of word in all words
|
||||
this.posinphrase, // position of word in its phrase
|
||||
this.posofphrase, // number of the phrase where word appears
|
||||
this.lastModified, // last-modified time of the document where word appears
|
||||
System.currentTimeMillis(), // update time;
|
||||
language, // (guessed) language of document
|
||||
type, // type of document
|
||||
llocal, // outlinks to same domain
|
||||
lother, // outlinks to other domain
|
||||
flags // attributes to the url and to the word according the url
|
||||
this.language, // (guessed) language of document
|
||||
this.type, // type of document
|
||||
this.llocal, // outlinks to same domain
|
||||
this.lother, // outlinks to other domain
|
||||
this.flags // attributes to the url and to the word according the url
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public Entry toKelondroEntry() {
|
||||
return toRowEntry().toKelondroEntry();
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toPropertyForm() {
|
||||
return toRowEntry().toPropertyForm();
|
||||
}
|
||||
|
||||
@Override
|
||||
public byte[] urlhash() {
|
||||
return urlHash;
|
||||
return this.urlHash;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int urlcomps() {
|
||||
return urlcomps;
|
||||
return this.urlcomps;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int urllength() {
|
||||
return urllength;
|
||||
return this.urllength;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int virtualAge() {
|
||||
return virtualAge;
|
||||
return this.virtualAge;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int wordsintext() {
|
||||
return wordsintext;
|
||||
return this.wordsintext;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int wordsintitle() {
|
||||
return wordsintitle;
|
||||
return this.wordsintitle;
|
||||
}
|
||||
|
||||
@Override
|
||||
public double termFrequency() {
|
||||
if (this.termFrequency == 0.0) this.termFrequency = (((double) this.hitcount()) / ((double) (this.wordsintext() + this.wordsintitle() + 1)));
|
||||
return this.termFrequency;
|
||||
}
|
||||
|
||||
|
||||
public final void min(final ImageReferenceVars other) {
|
||||
if (other == null) return;
|
||||
int v;
|
||||
|
|
@ -317,7 +339,7 @@ public class ImageReferenceVars extends AbstractReference implements ImageRefere
|
|||
if (this.wordsintitle > (v = other.wordsintitle)) this.wordsintitle = v;
|
||||
if (this.termFrequency > (d = other.termFrequency)) this.termFrequency = d;
|
||||
}
|
||||
|
||||
|
||||
public final void max(final ImageReferenceVars other) {
|
||||
if (other == null) return;
|
||||
int v;
|
||||
|
|
@ -340,11 +362,12 @@ public class ImageReferenceVars extends AbstractReference implements ImageRefere
|
|||
if (this.termFrequency < (d = other.termFrequency)) this.termFrequency = d;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void join(final Reference r) {
|
||||
// joins two entries into one entry
|
||||
|
||||
|
||||
// combine the distance
|
||||
ImageReference oe = (ImageReference) r;
|
||||
ImageReference oe = (ImageReference) r;
|
||||
for (Integer i: r.positions()) this.positions.add(i);
|
||||
this.posinphrase = (this.posofphrase == oe.posofphrase()) ? Math.min(this.posinphrase, oe.posinphrase()) : 0;
|
||||
this.posofphrase = Math.min(this.posofphrase, oe.posofphrase());
|
||||
|
|
@ -362,5 +385,5 @@ public class ImageReferenceVars extends AbstractReference implements ImageRefere
|
|||
public void addPosition(int position) {
|
||||
this.positions.add(position);
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -28,8 +28,8 @@ package net.yacy.kelondro.data.word;
|
|||
|
||||
import java.util.Collection;
|
||||
import java.util.Comparator;
|
||||
import java.util.Queue;
|
||||
import java.util.concurrent.BlockingQueue;
|
||||
import java.util.concurrent.ConcurrentLinkedQueue;
|
||||
import java.util.concurrent.LinkedBlockingQueue;
|
||||
import java.util.concurrent.Semaphore;
|
||||
|
||||
|
|
@ -66,7 +66,7 @@ public class WordReferenceVars extends AbstractReference implements WordReferenc
|
|||
urlcomps, urllength,
|
||||
wordsintext, wordsintitle;
|
||||
private int virtualAge;
|
||||
private final ConcurrentLinkedQueue<Integer> positions;
|
||||
private final Queue<Integer> positions;
|
||||
public double termFrequency;
|
||||
|
||||
public WordReferenceVars(
|
||||
|
|
@ -77,7 +77,7 @@ public class WordReferenceVars extends AbstractReference implements WordReferenc
|
|||
final int hitcount, // how often appears this word in the text
|
||||
final int wordcount, // total number of words
|
||||
final int phrasecount, // total number of phrases
|
||||
final ConcurrentLinkedQueue<Integer> ps, // positions of words that are joined into the reference
|
||||
final Queue<Integer> ps, // positions of words that are joined into the reference
|
||||
final int posinphrase, // position of word in its phrase
|
||||
final int posofphrase, // number of the phrase where word appears
|
||||
final long lastmodified, // last-modified time of the document where word appears
|
||||
|
|
@ -101,7 +101,7 @@ public class WordReferenceVars extends AbstractReference implements WordReferenc
|
|||
this.llocal = outlinksSame;
|
||||
this.lother = outlinksOther;
|
||||
this.phrasesintext = phrasecount;
|
||||
this.positions = new ConcurrentLinkedQueue<Integer>();
|
||||
this.positions = new LinkedBlockingQueue<Integer>();
|
||||
if (ps.size() > 0) for (final Integer i: ps) this.positions.add(i);
|
||||
this.posinphrase = posinphrase;
|
||||
this.posofphrase = posofphrase;
|
||||
|
|
@ -124,7 +124,7 @@ public class WordReferenceVars extends AbstractReference implements WordReferenc
|
|||
this.llocal = e.llocal();
|
||||
this.lother = e.lother();
|
||||
this.phrasesintext = e.phrasesintext();
|
||||
this.positions = new ConcurrentLinkedQueue<Integer>();
|
||||
this.positions = new LinkedBlockingQueue<Integer>();
|
||||
if (e.positions().size() > 0) for (final Integer i: e.positions()) this.positions.add(i);
|
||||
this.posinphrase = e.posinphrase();
|
||||
this.posofphrase = e.posofphrase();
|
||||
|
|
@ -195,46 +195,57 @@ public class WordReferenceVars extends AbstractReference implements WordReferenc
|
|||
this.termFrequency = this.termFrequency + v.termFrequency;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Bitfield flags() {
|
||||
return this.flags;
|
||||
}
|
||||
|
||||
@Override
|
||||
public byte[] getLanguage() {
|
||||
return this.language;
|
||||
}
|
||||
|
||||
@Override
|
||||
public char getType() {
|
||||
return this.type;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int hitcount() {
|
||||
return this.hitcount;
|
||||
}
|
||||
|
||||
@Override
|
||||
public long lastModified() {
|
||||
return this.lastModified;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int llocal() {
|
||||
return this.llocal;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int lother() {
|
||||
return this.lother;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int phrasesintext() {
|
||||
return this.phrasesintext;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int posinphrase() {
|
||||
return this.posinphrase;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Collection<Integer> positions() {
|
||||
return this.positions;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int posofphrase() {
|
||||
return this.posofphrase;
|
||||
}
|
||||
|
|
@ -261,14 +272,17 @@ public class WordReferenceVars extends AbstractReference implements WordReferenc
|
|||
);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Entry toKelondroEntry() {
|
||||
return toRowEntry().toKelondroEntry();
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toPropertyForm() {
|
||||
return toRowEntry().toPropertyForm();
|
||||
}
|
||||
|
||||
@Override
|
||||
public byte[] urlhash() {
|
||||
return this.urlHash;
|
||||
}
|
||||
|
|
@ -279,28 +293,34 @@ public class WordReferenceVars extends AbstractReference implements WordReferenc
|
|||
return this.hostHash;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int urlcomps() {
|
||||
return this.urlcomps;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int urllength() {
|
||||
return this.urllength;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int virtualAge() {
|
||||
if (this.virtualAge > 0) return this.virtualAge;
|
||||
this.virtualAge = MicroDate.microDateDays(this.lastModified);
|
||||
return this.virtualAge;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int wordsintext() {
|
||||
return this.wordsintext;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int wordsintitle() {
|
||||
return this.wordsintitle;
|
||||
}
|
||||
|
||||
@Override
|
||||
public double termFrequency() {
|
||||
if (this.termFrequency == 0.0) this.termFrequency = (((double) hitcount()) / ((double) (wordsintext() + wordsintitle() + 1)));
|
||||
return this.termFrequency;
|
||||
|
|
@ -350,6 +370,7 @@ public class WordReferenceVars extends AbstractReference implements WordReferenc
|
|||
if (this.termFrequency < (d = other.termFrequency)) this.termFrequency = d;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void join(final Reference r) {
|
||||
// joins two entries into one entry
|
||||
|
||||
|
|
@ -378,10 +399,12 @@ public class WordReferenceVars extends AbstractReference implements WordReferenc
|
|||
return ByteArray.hashCode(this.urlHash);
|
||||
}
|
||||
|
||||
@Override
|
||||
public int compareTo(final WordReferenceVars o) {
|
||||
return Base64Order.enhancedCoder.compare(this.urlHash, o.urlhash());
|
||||
}
|
||||
|
||||
@Override
|
||||
public int compare(final WordReferenceVars o1, final WordReferenceVars o2) {
|
||||
return o1.compareTo(o2);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -28,8 +28,9 @@ package net.yacy.kelondro.util;
|
|||
|
||||
import java.util.Iterator;
|
||||
import java.util.Map;
|
||||
import java.util.Queue;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
import java.util.concurrent.ConcurrentLinkedQueue;
|
||||
import java.util.concurrent.LinkedBlockingQueue;
|
||||
|
||||
import net.yacy.peers.graphics.ProfilingGraph;
|
||||
|
||||
|
|
@ -49,7 +50,7 @@ public class EventTracker {
|
|||
SEARCH;
|
||||
}
|
||||
|
||||
private final static Map<EClass, ConcurrentLinkedQueue<Event>> historyMaps = new ConcurrentHashMap<EClass, ConcurrentLinkedQueue<Event>>();
|
||||
private final static Map<EClass, Queue<Event>> historyMaps = new ConcurrentHashMap<EClass, Queue<Event>>();
|
||||
private final static Map<EClass, Long> eventAccess = new ConcurrentHashMap<EClass, Long>(); // value: last time when this was accessed
|
||||
|
||||
public final static void delete(final EClass eventName) {
|
||||
|
|
@ -73,11 +74,11 @@ public class EventTracker {
|
|||
}
|
||||
|
||||
// get event history container
|
||||
ConcurrentLinkedQueue<Event> history = historyMaps.get(eventName);
|
||||
Queue<Event> history = historyMaps.get(eventName);
|
||||
|
||||
// create history
|
||||
if (history == null) {
|
||||
history = new ConcurrentLinkedQueue<Event>();
|
||||
history = new LinkedBlockingQueue<Event>();
|
||||
|
||||
// update entry
|
||||
history.offer(new Event(eventPayload));
|
||||
|
|
@ -109,7 +110,7 @@ public class EventTracker {
|
|||
}
|
||||
|
||||
public final static Iterator<Event> getHistory(final EClass eventName) {
|
||||
final ConcurrentLinkedQueue<Event> list = historyMaps.get(eventName);
|
||||
final Queue<Event> list = historyMaps.get(eventName);
|
||||
if (list == null) return null;
|
||||
return list.iterator();
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue
Block a user