2010-12-29 02:54:27 +01:00
|
|
|
/**
|
|
|
|
* AccessTracker
|
|
|
|
* an interface for Adaptive Replacement Caches
|
|
|
|
* Copyright 2009 by Michael Peter Christen, mc@yacy.net, Frankfurt a. M., Germany
|
|
|
|
* First released 29.08.2009 at http://yacy.net
|
|
|
|
*
|
2011-03-08 02:51:51 +01:00
|
|
|
* $LastChangedDate$
|
|
|
|
* $LastChangedRevision$
|
|
|
|
* $LastChangedBy$
|
|
|
|
*
|
2010-12-29 02:54:27 +01:00
|
|
|
* This library is free software; you can redistribute it and/or
|
|
|
|
* modify it under the terms of the GNU Lesser General Public
|
|
|
|
* License as published by the Free Software Foundation; either
|
|
|
|
* version 2.1 of the License, or (at your option) any later version.
|
2011-09-24 03:58:54 +02:00
|
|
|
*
|
2010-12-29 02:54:27 +01:00
|
|
|
* This library is distributed in the hope that it will be useful,
|
|
|
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
|
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
|
|
|
* Lesser General Public License for more details.
|
2011-09-24 03:58:54 +02:00
|
|
|
*
|
2010-12-29 02:54:27 +01:00
|
|
|
* You should have received a copy of the GNU Lesser General Public License
|
|
|
|
* along with this program in the file lgpl21.txt
|
|
|
|
* If not, see <http://www.gnu.org/licenses/>.
|
|
|
|
*/
|
|
|
|
|
2011-09-25 18:59:06 +02:00
|
|
|
package net.yacy.search.query;
|
2010-12-29 02:54:27 +01:00
|
|
|
|
2010-12-29 15:30:25 +01:00
|
|
|
import java.io.File;
|
|
|
|
import java.io.FileNotFoundException;
|
|
|
|
import java.io.IOException;
|
|
|
|
import java.io.RandomAccessFile;
|
|
|
|
import java.util.ArrayList;
|
2011-05-22 22:12:13 +02:00
|
|
|
import java.util.Date;
|
2010-12-29 02:54:27 +01:00
|
|
|
import java.util.Iterator;
|
|
|
|
import java.util.LinkedList;
|
|
|
|
|
2011-01-03 21:52:54 +01:00
|
|
|
import net.yacy.cora.date.GenericFormatter;
|
2012-09-20 19:38:22 +02:00
|
|
|
import net.yacy.cora.document.WordCache;
|
2013-09-15 00:30:23 +02:00
|
|
|
import net.yacy.cora.document.encoding.UTF8;
|
2013-07-09 14:28:25 +02:00
|
|
|
import net.yacy.cora.util.ConcurrentLog;
|
2011-05-27 13:18:22 +02:00
|
|
|
import net.yacy.kelondro.util.MemoryControl;
|
2010-12-29 15:30:25 +01:00
|
|
|
|
2010-12-29 02:54:27 +01:00
|
|
|
public class AccessTracker {
|
|
|
|
|
2012-08-22 23:50:40 +02:00
|
|
|
private final static long DUMP_PERIOD = 60000L;
|
|
|
|
|
2012-11-05 03:19:28 +01:00
|
|
|
private static final int minSize = 100;
|
|
|
|
private static final int maxSize = 1000;
|
|
|
|
private static final int maxAge = 24 * 60 * 60 * 1000;
|
2011-09-24 03:58:54 +02:00
|
|
|
|
2010-12-29 02:54:27 +01:00
|
|
|
public enum Location {local, remote}
|
2011-09-24 03:58:54 +02:00
|
|
|
|
2012-08-22 23:50:40 +02:00
|
|
|
private static final LinkedList<QueryParams> localSearches = new LinkedList<QueryParams>();
|
|
|
|
private static final LinkedList<QueryParams> remoteSearches = new LinkedList<QueryParams>();
|
|
|
|
private static final ArrayList<String> log = new ArrayList<String>();
|
|
|
|
private static long lastLogDump = System.currentTimeMillis();
|
|
|
|
private static File dumpFile = null;
|
|
|
|
|
|
|
|
public static void setDumpFile(File f) {
|
|
|
|
dumpFile = f;
|
|
|
|
}
|
2011-09-24 03:58:54 +02:00
|
|
|
|
2013-02-26 17:16:31 +01:00
|
|
|
public static void add(final Location location, final QueryParams query, int resultCount) {
|
|
|
|
if (location == Location.local) synchronized (localSearches) {add(localSearches, query, resultCount);}
|
|
|
|
if (location == Location.remote) synchronized (remoteSearches) {add(remoteSearches, query, resultCount);}
|
2010-12-29 02:54:27 +01:00
|
|
|
}
|
2011-09-24 03:58:54 +02:00
|
|
|
|
2013-02-26 17:16:31 +01:00
|
|
|
private static void add(final LinkedList<QueryParams> list, final QueryParams query, int resultCount) {
|
2011-05-27 13:18:22 +02:00
|
|
|
// learn that this word can be a word completion for the DidYouMeanLibrary
|
2012-12-15 00:05:46 +01:00
|
|
|
String queryString = query.getQueryGoal().getOriginalQueryString(false);
|
2013-02-26 17:16:31 +01:00
|
|
|
if (resultCount > 10 && queryString != null && queryString.length() > 0) {
|
2012-12-15 00:05:46 +01:00
|
|
|
final StringBuilder sb = new StringBuilder(queryString);
|
|
|
|
sb.append(queryString);
|
2011-11-09 15:42:55 +01:00
|
|
|
WordCache.learn(sb);
|
|
|
|
}
|
2011-09-24 03:58:54 +02:00
|
|
|
|
2011-05-27 13:18:22 +02:00
|
|
|
// add query to statistics list
|
2010-12-29 02:54:27 +01:00
|
|
|
list.add(query);
|
2011-05-27 13:18:22 +02:00
|
|
|
|
|
|
|
// shrink dump list but keep essentials in dump
|
2012-07-10 22:59:03 +02:00
|
|
|
while (list.size() > maxSize || (!list.isEmpty() && MemoryControl.shortStatus())) {
|
2011-05-27 13:18:22 +02:00
|
|
|
synchronized (list) {
|
2013-02-26 17:16:31 +01:00
|
|
|
if (!list.isEmpty()) addToDump(list.removeFirst(), resultCount); else break;
|
2011-05-27 13:18:22 +02:00
|
|
|
}
|
2010-12-29 02:54:27 +01:00
|
|
|
}
|
2011-09-24 03:58:54 +02:00
|
|
|
|
2011-05-27 13:18:22 +02:00
|
|
|
// if the list is small we can terminate
|
|
|
|
if (list.size() <= minSize) return;
|
2011-09-24 03:58:54 +02:00
|
|
|
|
2011-05-27 13:18:22 +02:00
|
|
|
// if the list is large we look for too old entries
|
2011-09-24 03:58:54 +02:00
|
|
|
final long timeout = System.currentTimeMillis() - maxAge;
|
2012-07-10 22:59:03 +02:00
|
|
|
while (!list.isEmpty()) {
|
2011-09-24 03:58:54 +02:00
|
|
|
final QueryParams q = list.getFirst();
|
2012-06-04 15:37:39 +02:00
|
|
|
if (q.starttime > timeout) break;
|
2013-02-26 17:16:31 +01:00
|
|
|
addToDump(list.removeFirst(), resultCount);
|
2010-12-29 02:54:27 +01:00
|
|
|
}
|
|
|
|
}
|
2011-09-24 03:58:54 +02:00
|
|
|
|
|
|
|
public static Iterator<QueryParams> get(final Location location) {
|
2010-12-29 02:54:27 +01:00
|
|
|
if (location == Location.local) return localSearches.descendingIterator();
|
|
|
|
if (location == Location.remote) return remoteSearches.descendingIterator();
|
|
|
|
return null;
|
|
|
|
}
|
2011-09-24 03:58:54 +02:00
|
|
|
|
|
|
|
public static int size(final Location location) {
|
2010-12-29 02:54:27 +01:00
|
|
|
if (location == Location.local) synchronized (localSearches) {return localSearches.size();}
|
|
|
|
if (location == Location.remote) synchronized (remoteSearches) {return remoteSearches.size();}
|
|
|
|
return 0;
|
|
|
|
}
|
2011-09-24 03:58:54 +02:00
|
|
|
|
2013-02-26 17:16:31 +01:00
|
|
|
private static void addToDump(final QueryParams query, int resultCount) {
|
2012-12-15 00:05:46 +01:00
|
|
|
String queryString = query.getQueryGoal().getOriginalQueryString(false);
|
|
|
|
if (queryString == null || queryString.isEmpty()) return;
|
2013-02-26 17:16:31 +01:00
|
|
|
addToDump(queryString, Integer.toString(resultCount), new Date(query.starttime));
|
2012-08-22 23:50:40 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
public static void addToDump(String querystring, String resultcount) {
|
|
|
|
addToDump(querystring, resultcount, new Date());
|
|
|
|
if (lastLogDump + DUMP_PERIOD < System.currentTimeMillis()) {
|
|
|
|
lastLogDump = System.currentTimeMillis();
|
|
|
|
dumpLog();
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
private static void addToDump(String querystring, String resultcount, Date d) {
|
|
|
|
//if (query.resultcount == 0) return;
|
|
|
|
if (querystring == null || querystring.isEmpty()) return;
|
2011-09-24 03:58:54 +02:00
|
|
|
final StringBuilder sb = new StringBuilder(40);
|
2012-08-22 23:50:40 +02:00
|
|
|
sb.append(GenericFormatter.SHORT_SECOND_FORMATTER.format(d));
|
2010-12-29 15:30:25 +01:00
|
|
|
sb.append(' ');
|
2012-08-22 23:50:40 +02:00
|
|
|
sb.append(resultcount);
|
2010-12-29 15:30:25 +01:00
|
|
|
sb.append(' ');
|
2012-08-22 23:50:40 +02:00
|
|
|
sb.append(querystring);
|
|
|
|
synchronized (log) {
|
|
|
|
log.add(sb.toString());
|
|
|
|
}
|
2010-12-29 15:30:25 +01:00
|
|
|
}
|
2011-09-24 03:58:54 +02:00
|
|
|
|
2012-08-22 23:50:40 +02:00
|
|
|
public static void dumpLog() {
|
2012-07-10 22:59:03 +02:00
|
|
|
while (!localSearches.isEmpty()) {
|
2013-02-26 17:16:31 +01:00
|
|
|
addToDump(localSearches.removeFirst(), 0);
|
2010-12-29 15:30:25 +01:00
|
|
|
}
|
2012-08-22 23:50:40 +02:00
|
|
|
Thread t = new Thread() {
|
|
|
|
@Override
|
|
|
|
public void run() {
|
|
|
|
ArrayList<String> logCopy = new ArrayList<String>();
|
|
|
|
synchronized (log) {
|
|
|
|
logCopy.addAll(log);
|
|
|
|
log.clear();
|
|
|
|
}
|
|
|
|
RandomAccessFile raf = null;
|
|
|
|
try {
|
|
|
|
raf = new RandomAccessFile(dumpFile, "rw");
|
|
|
|
raf.seek(raf.length());
|
|
|
|
for (final String s: logCopy) {
|
|
|
|
raf.write(UTF8.getBytes(s));
|
|
|
|
raf.writeByte(10);
|
|
|
|
}
|
|
|
|
logCopy.clear();
|
|
|
|
} catch (final FileNotFoundException e) {
|
2013-07-09 14:28:25 +02:00
|
|
|
ConcurrentLog.logException(e);
|
2012-08-22 23:50:40 +02:00
|
|
|
} catch (final IOException e) {
|
2013-07-09 14:28:25 +02:00
|
|
|
ConcurrentLog.logException(e);
|
2012-08-22 23:50:40 +02:00
|
|
|
} finally {
|
2013-07-17 18:31:30 +02:00
|
|
|
if (raf != null) try {raf.close();} catch (final IOException e) {}
|
2012-08-22 23:50:40 +02:00
|
|
|
}
|
2010-12-29 15:30:25 +01:00
|
|
|
}
|
2012-08-22 23:50:40 +02:00
|
|
|
};
|
|
|
|
t.start();
|
2010-12-29 15:30:25 +01:00
|
|
|
}
|
2010-12-29 02:54:27 +01:00
|
|
|
}
|