mirror of
https://github.com/yacy/yacy_search_server.git
synced 2024-09-21 00:00:13 +02:00
fed26f33a8
to support the new time parser and search functions in YaCy a high precision detection of date and time on the day is necessary. That requires that the time zone of the document content and the time zone of the user, doing a search, is detected. The time zone of the search request is done automatically using the browsers time zone offset which is delivered to the search request automatically and invisible to the user. The time zone for the content of web pages cannot be detected automatically and must be an attribute of crawl starts. The advanced crawl start now provides an input field to set the time zone in minutes as an offset number. All parsers must get a time zone offset passed, so this required the change of the parser java api. A lot of other changes had been made which corrects the wrong handling of dates in YaCy which was to add a correction based on the time zone of the server. Now no correction is added and all dates in YaCy are UTC/GMT time zone, a normalized time zone for all peers.
340 lines
12 KiB
Java
340 lines
12 KiB
Java
/**
|
|
* BEncodedHeapBag
|
|
* Copyright 2011 by Michael Peter Christen, mc@yacy.net, Frankfurt am Main, Germany
|
|
* First released 16.12.2011 at http://yacy.net
|
|
*
|
|
* $LastChangedDate: 2011-04-14 00:04:23 +0200 (Do, 14 Apr 2011) $
|
|
* $LastChangedRevision: 7653 $
|
|
* $LastChangedBy: orbiter $
|
|
*
|
|
* This library is free software; you can redistribute it and/or
|
|
* modify it under the terms of the GNU Lesser General Public
|
|
* License as published by the Free Software Foundation; either
|
|
* version 2.1 of the License, or (at your option) any later version.
|
|
*
|
|
* This library is distributed in the hope that it will be useful,
|
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
|
* Lesser General Public License for more details.
|
|
*
|
|
* You should have received a copy of the GNU Lesser General Public License
|
|
* along with this program in the file lgpl21.txt
|
|
* If not, see <http://www.gnu.org/licenses/>.
|
|
*/
|
|
|
|
package net.yacy.kelondro.blob;
|
|
|
|
import java.io.File;
|
|
import java.io.IOException;
|
|
import java.text.ParseException;
|
|
import java.util.ArrayList;
|
|
import java.util.Date;
|
|
import java.util.HashMap;
|
|
import java.util.Iterator;
|
|
import java.util.List;
|
|
import java.util.Map;
|
|
|
|
import net.yacy.cora.date.GenericFormatter;
|
|
import net.yacy.cora.document.encoding.UTF8;
|
|
import net.yacy.cora.order.Base64Order;
|
|
import net.yacy.cora.order.ByteOrder;
|
|
import net.yacy.cora.order.CloneableIterator;
|
|
import net.yacy.cora.storage.AbstractMapStore;
|
|
import net.yacy.cora.storage.MapStore;
|
|
import net.yacy.cora.util.ConcurrentLog;
|
|
import net.yacy.kelondro.data.word.Word;
|
|
import net.yacy.kelondro.util.FileUtils;
|
|
import net.yacy.kelondro.util.MergeIterator;
|
|
|
|
public class BEncodedHeapBag extends AbstractMapStore implements MapStore {
|
|
|
|
private Map<String, BEncodedHeap> bag; // a map from a date string to a kelondroIndex object
|
|
private final File baseDir;
|
|
private final String prefix;
|
|
private final int keylength, buffermax;
|
|
private final ByteOrder entryOrder;
|
|
private String current;
|
|
private final long fileAgeLimit;
|
|
private final long fileSizeLimit;
|
|
|
|
public BEncodedHeapBag(
|
|
final File path,
|
|
final String prefix,
|
|
final int keylength,
|
|
final ByteOrder ordering,
|
|
final int buffermax,
|
|
final long fileAgeLimit,
|
|
final long fileSizeLimit) {
|
|
this.baseDir = path;
|
|
this.prefix = prefix;
|
|
this.keylength = keylength;
|
|
this.buffermax = buffermax;
|
|
this.entryOrder = ordering;
|
|
this.fileAgeLimit = fileAgeLimit;
|
|
this.fileSizeLimit = fileSizeLimit;
|
|
init();
|
|
}
|
|
|
|
private void init() {
|
|
this.current = null;
|
|
|
|
// initialized tables map
|
|
this.bag = new HashMap<String, BEncodedHeap>();
|
|
if (!(this.baseDir.exists())) this.baseDir.mkdirs();
|
|
String[] tablefile = this.baseDir.list();
|
|
|
|
// first pass: find tables
|
|
final HashMap<String, Long> t = new HashMap<String, Long>();
|
|
long ram, time, maxtime = 0;
|
|
Date d;
|
|
File f;
|
|
for (final String element : tablefile) {
|
|
if ((element.startsWith(this.prefix)) &&
|
|
(element.length() > this.prefix.length()) &&
|
|
(element.charAt(this.prefix.length()) == '.') &&
|
|
(element.length() == this.prefix.length() + 23)) {
|
|
f = new File(this.baseDir, element);
|
|
try {
|
|
d = GenericFormatter.SHORT_MILSEC_FORMATTER.parse(element.substring(this.prefix.length() + 1, this.prefix.length() + 18), 0).getTime();
|
|
} catch (final ParseException e) {
|
|
ConcurrentLog.severe("BEncodedHeapBag", "", e);
|
|
continue;
|
|
}
|
|
time = d.getTime();
|
|
if (time > maxtime) {
|
|
this.current = element;
|
|
assert this.current != null;
|
|
maxtime = time;
|
|
}
|
|
|
|
t.put(element, f.length());
|
|
}
|
|
}
|
|
|
|
// second pass: open tables
|
|
Iterator<Map.Entry<String, Long>> i;
|
|
Map.Entry<String, Long> entry;
|
|
String maxf;
|
|
long maxram;
|
|
while (!t.isEmpty()) {
|
|
// find maximum table
|
|
maxram = 0;
|
|
maxf = null;
|
|
i = t.entrySet().iterator();
|
|
while (i.hasNext()) {
|
|
entry = i.next();
|
|
ram = entry.getValue().longValue();
|
|
if (maxf == null || ram > maxram) {
|
|
maxf = entry.getKey();
|
|
maxram = ram;
|
|
}
|
|
}
|
|
|
|
// open next biggest table
|
|
t.remove(maxf);
|
|
f = new File(this.baseDir, maxf);
|
|
try {
|
|
ConcurrentLog.info("BEncodedHeapBag", "opening partial heap " + f);
|
|
BEncodedHeap heap = new BEncodedHeap(f, this.keylength, this.entryOrder, this.buffermax);
|
|
this.bag.put(maxf, heap);
|
|
} catch (final IOException e) {
|
|
ConcurrentLog.severe("BEncodedHeapBag", "error opening partial heap " + f);
|
|
}
|
|
}
|
|
}
|
|
|
|
@Override
|
|
public synchronized void close() {
|
|
if (this.bag == null) return;
|
|
|
|
final Iterator<BEncodedHeap> i = this.bag.values().iterator();
|
|
while (i.hasNext()) {
|
|
i.next().close();
|
|
}
|
|
this.bag = null;
|
|
}
|
|
|
|
@Override
|
|
public void clear() {
|
|
close();
|
|
final String[] l = this.baseDir.list();
|
|
for (final String element : l) {
|
|
if (element.startsWith(this.prefix)) {
|
|
final File f = new File(this.baseDir, element);
|
|
if (!f.isDirectory()) FileUtils.deletedelete(f);
|
|
}
|
|
}
|
|
init();
|
|
}
|
|
|
|
private MapStore keeperOf(final byte[] key) {
|
|
if (key == null) return null;
|
|
if (this.bag == null) return null;
|
|
for (final MapStore oi: this.bag.values()) {
|
|
if (oi.containsKey(key)) return oi;
|
|
}
|
|
return null;
|
|
}
|
|
|
|
private String newFilename() {
|
|
return this.prefix + "." + GenericFormatter.SHORT_MILSEC_FORMATTER.format() + ".heap";
|
|
}
|
|
|
|
private MapStore newHeap() {
|
|
this.current = newFilename();
|
|
final File f = new File(this.baseDir, this.current);
|
|
BEncodedHeap heap;
|
|
try {
|
|
heap = new BEncodedHeap(f, this.keylength, this.entryOrder, this.buffermax);
|
|
} catch (final IOException e) {
|
|
ConcurrentLog.severe("BEncodedHeapBag", "unable to open new heap file: " + e.getMessage(), e);
|
|
return null;
|
|
}
|
|
this.bag.put(this.current, heap);
|
|
return heap;
|
|
}
|
|
|
|
private MapStore checkHeap(final BEncodedHeap heap) {
|
|
// check size and age of given table; in case it is too large or too old
|
|
// create a new table
|
|
assert heap != null;
|
|
long t = System.currentTimeMillis();
|
|
if (((t / 1000) % 10) != 0) return heap; // we check only every 10 seconds because all these file and parser operations are very expensive
|
|
final String name = heap.getFile().getName();
|
|
long d;
|
|
try {
|
|
d = GenericFormatter.SHORT_MILSEC_FORMATTER.parse(name.substring(this.prefix.length() + 1, this.prefix.length() + 18), 0).getTime().getTime();
|
|
} catch (final ParseException e) {
|
|
ConcurrentLog.severe("BEncodedHeapBag", "", e);
|
|
d = 0;
|
|
}
|
|
if (d + this.fileAgeLimit < t || new File(this.baseDir, name).length() >= this.fileSizeLimit) {
|
|
return newHeap();
|
|
}
|
|
return heap;
|
|
}
|
|
|
|
@Override
|
|
public boolean containsKey(Object key) {
|
|
if (key == null || !(key instanceof byte[])) return false;
|
|
synchronized (this.bag) {
|
|
return keeperOf((byte[]) key) != null;
|
|
}
|
|
}
|
|
|
|
@Override
|
|
public Map<String, byte[]> put(byte[] key, Map<String, byte[]> map) {
|
|
if (this.bag == null) return null;
|
|
MapStore keeper = null;
|
|
synchronized (this.bag) {
|
|
keeper = keeperOf(key);
|
|
}
|
|
if (keeper != null) {
|
|
return keeper.put(key, map);
|
|
}
|
|
synchronized (this.bag) {
|
|
keeper = keeperOf(key); // we must check that again because it could have changed in between
|
|
if (keeper != null) return keeper.put(key, map);
|
|
if (this.current == null) {
|
|
keeper = newHeap();
|
|
return keeper.put(key, map);
|
|
}
|
|
keeper = checkHeap(this.bag.get(this.current));
|
|
}
|
|
return keeper.put(key, map);
|
|
}
|
|
|
|
@Override
|
|
public Map<String, byte[]> get(Object key) {
|
|
if (key == null || !(key instanceof byte[])) return null;
|
|
synchronized (this.bag) {
|
|
return keeperOf((byte[]) key).get(key);
|
|
}
|
|
}
|
|
|
|
@Override
|
|
public boolean isEmpty() {
|
|
final Iterator<BEncodedHeap> i = this.bag.values().iterator();
|
|
while (i.hasNext()) if (!i.next().isEmpty()) return false;
|
|
return true;
|
|
}
|
|
|
|
@Override
|
|
public int size() {
|
|
final Iterator<BEncodedHeap> i = this.bag.values().iterator();
|
|
int s = 0;
|
|
while (i.hasNext()) s += i.next().size();
|
|
return s;
|
|
}
|
|
|
|
@Override
|
|
public Map<String, byte[]> remove(Object key) {
|
|
if (key == null || !(key instanceof byte[])) return null;
|
|
final MapStore heap;
|
|
synchronized (this.bag) {
|
|
heap = keeperOf((byte[]) key);
|
|
}
|
|
if (heap == null) return null;
|
|
return heap.remove(key);
|
|
}
|
|
|
|
@Override
|
|
public ByteOrder getOrdering() {
|
|
return this.entryOrder;
|
|
}
|
|
|
|
@Override
|
|
public CloneableIterator<byte[]> keyIterator() {
|
|
final List<CloneableIterator<byte[]>> c = new ArrayList<CloneableIterator<byte[]>>(this.bag.size());
|
|
final Iterator<BEncodedHeap> i = this.bag.values().iterator();
|
|
CloneableIterator<byte[]> k;
|
|
while (i.hasNext()) {
|
|
k = i.next().keyIterator();
|
|
if (k != null && k.hasNext()) c.add(k);
|
|
}
|
|
return MergeIterator.cascade(c, this.entryOrder, MergeIterator.simpleMerge, true);
|
|
}
|
|
|
|
protected static Map<String, byte[]> testMap(int i) {
|
|
HashMap<String, byte[]> t = new HashMap<String, byte[]>();
|
|
t.put("rdf:about", UTF8.getBytes("http://abc.de/testmap#" + i));
|
|
t.put("dc:title", UTF8.getBytes("test nr " + i));
|
|
return t;
|
|
}
|
|
|
|
private static BEncodedHeapBag testHeapBag(File f) {
|
|
return new BEncodedHeapBag(
|
|
f,
|
|
"testbag",
|
|
12,
|
|
Base64Order.enhancedCoder,
|
|
10,
|
|
ArrayStack.oneMonth, 100 /*Integer.MAX_VALUE*/);
|
|
}
|
|
|
|
public static void main(String[] args) {
|
|
File f = new File("/tmp");
|
|
BEncodedHeapBag hb = testHeapBag(f);
|
|
for (int i = 0; i < 10000; i++) {
|
|
hb.put(Word.word2hash(Integer.toString(i)), testMap(i));
|
|
}
|
|
System.out.println("test size after put = " + hb.size());
|
|
hb.close();
|
|
hb = testHeapBag(f);
|
|
Iterator<Map.Entry<byte[], Map<String, byte[]>>> mi = hb.iterator();
|
|
int c = 1000;
|
|
Map.Entry<byte[], Map<String, byte[]>> entry;
|
|
while (mi.hasNext() && c-- > 0) {
|
|
entry = mi.next();
|
|
System.out.println(UTF8.String(entry.getKey()) + ": " + AbstractMapStore.map2String(entry.getValue()));
|
|
}
|
|
for (int i = 10000; i > 0; i--) {
|
|
hb.remove(Word.word2hash(Integer.toString(i - 1)));
|
|
}
|
|
System.out.println("test size after remove = " + hb.size());
|
|
hb.close();
|
|
ConcurrentLog.shutdown();
|
|
}
|
|
|
|
}
|