mirror of
https://github.com/yacy/yacy_search_server.git
synced 2024-09-19 00:01:41 +02:00
- renamed Stack to RecordStack to avoid name confusion with new classes
- added new Stack class that implements a stack on BLOB files - added new Stacks class that can be used for a set of Stacks (a 'Stack Database') - added methods to other classes to support the new stacks git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@6169 6c8d7289-2bf4-0310-a012-ef5d649a1542
This commit is contained in:
parent
ec2970cc76
commit
24cb6d68bc
|
@ -39,7 +39,7 @@ import de.anomic.http.httpResponseHeader;
|
|||
import de.anomic.kelondro.index.Row;
|
||||
import de.anomic.kelondro.order.Base64Order;
|
||||
import de.anomic.kelondro.order.NaturalOrder;
|
||||
import de.anomic.kelondro.table.Stack;
|
||||
import de.anomic.kelondro.table.RecordStack;
|
||||
import de.anomic.kelondro.text.MetadataRepository;
|
||||
import de.anomic.kelondro.text.metadataPrototype.URLMetadataRow;
|
||||
import de.anomic.kelondro.util.DateFormatter;
|
||||
|
@ -53,7 +53,7 @@ import de.anomic.yacy.logging.Log;
|
|||
public class IndexingStack {
|
||||
|
||||
protected final CrawlProfile profiles;
|
||||
protected final Stack sbQueueStack;
|
||||
protected final RecordStack sbQueueStack;
|
||||
protected final yacySeedDB peers;
|
||||
protected final ConcurrentHashMap<String, QueueEntry> queueInProcess;
|
||||
|
||||
|
@ -64,7 +64,7 @@ public class IndexingStack {
|
|||
this.profiles = profiles;
|
||||
this.peers = peers;
|
||||
this.queueInProcess = new ConcurrentHashMap<String, QueueEntry>();
|
||||
this.sbQueueStack = Stack.open(sbQueueStackPath, rowdef);
|
||||
this.sbQueueStack = RecordStack.open(sbQueueStackPath, rowdef);
|
||||
}
|
||||
|
||||
public static final Row rowdef = new Row(
|
||||
|
|
|
@ -234,6 +234,54 @@ public class HeapReader {
|
|||
return this.ordering;
|
||||
}
|
||||
|
||||
/**
|
||||
* find a special key in the heap: the one with the smallest key
|
||||
* this method is useful if the entries are ordered using their keys.
|
||||
* then the key with the smallest key denotes the first entry
|
||||
* @return the smallest key in the heap
|
||||
* @throws IOException
|
||||
*/
|
||||
protected synchronized byte[] firstKey() throws IOException {
|
||||
return index.smallestKey();
|
||||
}
|
||||
|
||||
/**
|
||||
* find a special blob in the heap: one that has the smallest key
|
||||
* this method is useful if the entries are ordered using their keys.
|
||||
* then the key with the smallest key denotes the first entry
|
||||
* @return the entry which key is the smallest in the heap
|
||||
* @throws IOException
|
||||
*/
|
||||
protected synchronized byte[] first() throws IOException {
|
||||
byte[] key = index.smallestKey();
|
||||
if (key == null) return null;
|
||||
return get(key);
|
||||
}
|
||||
|
||||
/**
|
||||
* find a special key in the heap: the one with the largest key
|
||||
* this method is useful if the entries are ordered using their keys.
|
||||
* then the key with the largest key denotes the last entry
|
||||
* @return the largest key in the heap
|
||||
* @throws IOException
|
||||
*/
|
||||
protected synchronized byte[] lastKey() throws IOException {
|
||||
return index.largestKey();
|
||||
}
|
||||
|
||||
/**
|
||||
* find a special blob in the heap: one that has the largest key
|
||||
* this method is useful if the entries are ordered using their keys.
|
||||
* then the key with the largest key denotes the last entry
|
||||
* @return the entry which key is the smallest in the heap
|
||||
* @throws IOException
|
||||
*/
|
||||
protected synchronized byte[] last() throws IOException {
|
||||
byte[] key = index.largestKey();
|
||||
if (key == null) return null;
|
||||
return get(key);
|
||||
}
|
||||
|
||||
/**
|
||||
* read a blob from the heap
|
||||
* @param key
|
||||
|
|
239
source/de/anomic/kelondro/blob/Stack.java
Normal file
239
source/de/anomic/kelondro/blob/Stack.java
Normal file
|
@ -0,0 +1,239 @@
|
|||
// Stack.java
|
||||
// (C) 2009 by Michael Peter Christen; mc@yacy.net, Frankfurt a. M., Germany
|
||||
// first published 03.07.2009 on http://yacy.net
|
||||
//
|
||||
// $LastChangedDate: 2008-03-14 01:16:04 +0100 (Fr, 14 Mrz 2008) $
|
||||
// $LastChangedRevision: 4558 $
|
||||
// $LastChangedBy: orbiter $
|
||||
//
|
||||
// LICENSE
|
||||
//
|
||||
// This program is free software; you can redistribute it and/or modify
|
||||
// it under the terms of the GNU General Public License as published by
|
||||
// the Free Software Foundation; either version 2 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU General Public License
|
||||
// along with this program; if not, write to the Free Software
|
||||
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
||||
|
||||
package de.anomic.kelondro.blob;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.util.Iterator;
|
||||
|
||||
import de.anomic.kelondro.order.NaturalOrder;
|
||||
|
||||
public class Stack {
|
||||
|
||||
private Heap stack;
|
||||
private long lastHandle;
|
||||
|
||||
/**
|
||||
* create a new stack object.
|
||||
* a stack object is backed by a blob file that contains the stack entries.
|
||||
* all stack entries can be accessed with a long handle; the handle is
|
||||
* represented as b256-encoded byte[] as key in the blob.
|
||||
* The handle is created using the current time. That means that the top
|
||||
* element on the stack has the maximum time as key handle and the element
|
||||
* at the bottom of the stack has the minimum time as key handle
|
||||
* @param stackFile
|
||||
* @throws IOException
|
||||
*/
|
||||
public Stack(final File stackFile) throws IOException {
|
||||
this.stack = new Heap(stackFile, 8, NaturalOrder.naturalOrder, 0);
|
||||
this.lastHandle = 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* create a new time handle. In case that the method is called
|
||||
* within a single millisecond twice, a new handle is created using
|
||||
* an increment of the previous handle to avoid handle collisions.
|
||||
* This method must be called in an synchronized environment
|
||||
* @return a unique handle for this stack
|
||||
*/
|
||||
private long nextHandle() {
|
||||
long h = System.currentTimeMillis();
|
||||
if (h <= this.lastHandle) h = lastHandle + 1;
|
||||
lastHandle = h;
|
||||
return h;
|
||||
}
|
||||
|
||||
/**
|
||||
* Iterate all handles from the stack as Long numbers
|
||||
* @return an iterator of all handles of the stack
|
||||
* @throws IOException
|
||||
*/
|
||||
public synchronized Iterator<Long> handles() throws IOException {
|
||||
return NaturalOrder.LongIterator(this.stack.keys(true, false));
|
||||
}
|
||||
|
||||
/**
|
||||
* get the size of a stack
|
||||
* @return the number of entries on the stack
|
||||
*/
|
||||
public synchronized int size() {
|
||||
return this.stack.size();
|
||||
}
|
||||
|
||||
/**
|
||||
* push a new element on the top of the stack
|
||||
* @param b the new stack element
|
||||
* @return the handle used to store the new element
|
||||
* @throws IOException
|
||||
*/
|
||||
public synchronized long push(byte[] b) throws IOException {
|
||||
long handle = nextHandle();
|
||||
this.stack.put(NaturalOrder.encodeLong(handle, 8), b);
|
||||
return handle;
|
||||
}
|
||||
|
||||
/**
|
||||
* push a new element on the top of the stack using a entry object
|
||||
* this is only useful for internal processes where a special handle
|
||||
* is created
|
||||
* @param b the new stack element
|
||||
* @return the handle used to store the new element
|
||||
* @throws IOException
|
||||
*/
|
||||
protected synchronized void push(Entry e) throws IOException {
|
||||
this.stack.put(NaturalOrder.encodeLong(e.h, 8), e.b);
|
||||
}
|
||||
|
||||
/**
|
||||
* get an element from the stack using the handle
|
||||
* @param handle
|
||||
* @return the object that belongs to the handle
|
||||
* or null if no such element exists
|
||||
* @throws IOException
|
||||
*/
|
||||
public synchronized byte[] get(long handle) throws IOException {
|
||||
byte[] k = NaturalOrder.encodeLong(handle, 8);
|
||||
byte[] b = this.stack.get(k);
|
||||
if (b == null) return null;
|
||||
return b;
|
||||
}
|
||||
|
||||
/**
|
||||
* remove an element from the stack using the entry handle
|
||||
* @param handle
|
||||
* @return the removed element
|
||||
* @throws IOException
|
||||
*/
|
||||
public synchronized byte[] remove(long handle) throws IOException {
|
||||
byte[] k = NaturalOrder.encodeLong(handle, 8);
|
||||
byte[] b = this.stack.get(k);
|
||||
if (b == null) return null;
|
||||
this.stack.remove(k);
|
||||
return b;
|
||||
}
|
||||
|
||||
/**
|
||||
* remove the top element from the stack
|
||||
* @return the top element or null if the stack is empty
|
||||
* @throws IOException
|
||||
*/
|
||||
public synchronized Entry pop() throws IOException {
|
||||
return po(this.stack.lastKey(), true);
|
||||
}
|
||||
|
||||
/**
|
||||
* return the top element of the stack.
|
||||
* The element is not removed from the stack.
|
||||
* Successive calls to this method will always return the same element
|
||||
* @return the element on the top of the stack or null, if stack is empty
|
||||
* @throws IOException
|
||||
*/
|
||||
public synchronized Entry top() throws IOException {
|
||||
return po(this.stack.lastKey(), false);
|
||||
}
|
||||
|
||||
/**
|
||||
* remove the bottom element from the stack
|
||||
* @return the bottom element or null if the stack is empty
|
||||
* @throws IOException
|
||||
*/
|
||||
public synchronized Entry pot() throws IOException {
|
||||
return po(this.stack.firstKey(), true);
|
||||
}
|
||||
|
||||
/**
|
||||
* return the bottom element of the stack.
|
||||
* The element is not removed from the stack.
|
||||
* Successive calls to this method will always return the same element
|
||||
* @return the element on the bottom of the stack or null, if stack is empty
|
||||
* @throws IOException
|
||||
*/
|
||||
public synchronized Entry bot() throws IOException {
|
||||
return po(this.stack.firstKey(), false);
|
||||
}
|
||||
|
||||
private Entry po(byte[] k, boolean remove) throws IOException {
|
||||
if (k == null) return null;
|
||||
assert k.length == 8;
|
||||
byte[] b = this.stack.get(k);
|
||||
assert b != null;
|
||||
if (b == null) return null;
|
||||
if (remove) this.stack.remove(k);
|
||||
return new Entry(k, b);
|
||||
}
|
||||
|
||||
public class Entry {
|
||||
|
||||
private long h;
|
||||
private byte[] b;
|
||||
|
||||
/**
|
||||
* create a new entry object using a long handle
|
||||
* @param h
|
||||
* @param b
|
||||
*/
|
||||
public Entry(long h, byte[] b) {
|
||||
this.h = h;
|
||||
this.b = b;
|
||||
}
|
||||
|
||||
/**
|
||||
* create a new entry object using the byte[] encoded handle
|
||||
* @param k
|
||||
* @param b
|
||||
*/
|
||||
public Entry(byte[] k, byte[] b) {
|
||||
this.h = NaturalOrder.decodeLong(k);
|
||||
this.b = b;
|
||||
}
|
||||
|
||||
/**
|
||||
* get the handle
|
||||
* @return the handle
|
||||
*/
|
||||
public long handle() {
|
||||
return h;
|
||||
}
|
||||
|
||||
/**
|
||||
* get the blob entry
|
||||
* @return the blob
|
||||
*/
|
||||
public byte[] blob() {
|
||||
return b;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* close the stack file and write a handle index
|
||||
*/
|
||||
public synchronized void close() {
|
||||
this.stack.close(true);
|
||||
}
|
||||
|
||||
public void finalize() {
|
||||
this.close();
|
||||
}
|
||||
}
|
274
source/de/anomic/kelondro/blob/Stacks.java
Normal file
274
source/de/anomic/kelondro/blob/Stacks.java
Normal file
|
@ -0,0 +1,274 @@
|
|||
// Stacks.java
|
||||
// (C) 2009 by Michael Peter Christen; mc@yacy.net, Frankfurt a. M., Germany
|
||||
// first published 03.07.2009 on http://yacy.net
|
||||
//
|
||||
// $LastChangedDate: 2008-03-14 01:16:04 +0100 (Fr, 14 Mrz 2008) $
|
||||
// $LastChangedRevision: 4558 $
|
||||
// $LastChangedBy: orbiter $
|
||||
//
|
||||
// LICENSE
|
||||
//
|
||||
// This program is free software; you can redistribute it and/or modify
|
||||
// it under the terms of the GNU General Public License as published by
|
||||
// the Free Software Foundation; either version 2 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU General Public License
|
||||
// along with this program; if not, write to the Free Software
|
||||
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
||||
|
||||
package de.anomic.kelondro.blob;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.util.Iterator;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
|
||||
public class Stacks {
|
||||
|
||||
private File stacksLocation;
|
||||
private String stacksPrefix;
|
||||
private ConcurrentHashMap<String, StackInstance> stacks;
|
||||
|
||||
/**
|
||||
* create a stack organizing object.
|
||||
* Stacks can be created on-the-fly in the given stacksLocation directory
|
||||
* using simple push operations that create first entries in the stack
|
||||
* Stacks that do not contain any element upon the close() operation are removed
|
||||
* @param stackFile
|
||||
* @throws IOException
|
||||
*/
|
||||
public Stacks(final File stacksLocation, String stacksPrefix) {
|
||||
if (!stacksLocation.exists()) stacksLocation.mkdirs();
|
||||
assert stacksLocation.isDirectory();
|
||||
this.stacksLocation = stacksLocation;
|
||||
this.stacksPrefix = stacksPrefix;
|
||||
|
||||
// initialize the stacks map
|
||||
this.stacks = new ConcurrentHashMap<String, StackInstance>();
|
||||
String[] sl = this.stacksLocation.list();
|
||||
for (String s: sl) {
|
||||
if (!s.startsWith(this.stacksPrefix + "_")) continue;
|
||||
StackInstance si;
|
||||
try {
|
||||
si = new StackInstance(new File(this.stacksLocation, s));
|
||||
this.stacks.put(si.name, si);
|
||||
} catch (IOException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private class StackInstance {
|
||||
private String name;
|
||||
private File location;
|
||||
private Stack stack;
|
||||
|
||||
public StackInstance(File location) throws IOException {
|
||||
String filename = location.getName();
|
||||
assert filename.startsWith(stacksPrefix + "_");
|
||||
assert filename.endsWith(".bstack");
|
||||
this.name = filename.substring(stacksPrefix.length() + 1, filename.length() - 7);
|
||||
this.location = location;
|
||||
this.stack = new Stack(location);
|
||||
}
|
||||
|
||||
public StackInstance(String stack) throws IOException {
|
||||
this.location = new File(stacksLocation, stacksPrefix + "_" + stack + ".bstack");
|
||||
this.name = stack;
|
||||
this.stack = new Stack(location);
|
||||
}
|
||||
|
||||
public String name() {
|
||||
return this.name;
|
||||
}
|
||||
|
||||
public Stack stack() {
|
||||
return this.stack;
|
||||
}
|
||||
|
||||
public File location() {
|
||||
return this.location;
|
||||
}
|
||||
}
|
||||
|
||||
private Stack getStack(String stack) {
|
||||
StackInstance si = this.stacks.get(stack);
|
||||
if (si == null) {
|
||||
// create a new Stack on the fly
|
||||
try {
|
||||
si = new StackInstance(stack);
|
||||
this.stacks.put(stack, si);
|
||||
return si.stack();
|
||||
} catch (IOException e) {
|
||||
e.printStackTrace();
|
||||
return null;
|
||||
}
|
||||
}
|
||||
return si.stack();
|
||||
}
|
||||
|
||||
/**
|
||||
* get the number of stacks organized by this object
|
||||
* @return the number of stack objects
|
||||
*/
|
||||
public int size() {
|
||||
return this.stacks.size();
|
||||
}
|
||||
|
||||
/**
|
||||
* iterate all stack names
|
||||
* @return an iterator for the stack names
|
||||
*/
|
||||
public Iterator<String> stacks() {
|
||||
return this.stacks.keySet().iterator();
|
||||
}
|
||||
|
||||
/**
|
||||
* get the size of a stack
|
||||
* @param stack the name of the stack
|
||||
* @return the number of entries on the stack
|
||||
*/
|
||||
public int size(String stack) {
|
||||
Stack s = getStack(stack);
|
||||
if (s == null) return -1;
|
||||
return s.size();
|
||||
}
|
||||
|
||||
/**
|
||||
* Iterate all handles from a stack as Long numbers
|
||||
* @param stack the name of the stack
|
||||
* @return an iterator of all handles of the stack
|
||||
* @throws IOException
|
||||
*/
|
||||
public synchronized Iterator<Long> handles(String stack) throws IOException {
|
||||
Stack s = getStack(stack);
|
||||
if (s == null) return null;
|
||||
return s.handles();
|
||||
}
|
||||
|
||||
/**
|
||||
* push a new element on the top of the stack
|
||||
* @param stack the name of the stack
|
||||
* @param b the new stack element
|
||||
* @return the handle used to store the new element
|
||||
* @throws IOException
|
||||
*/
|
||||
public long push(String stack, byte[] b) throws IOException {
|
||||
Stack s = getStack(stack);
|
||||
if (s == null) return -1;
|
||||
return s.push(b);
|
||||
}
|
||||
|
||||
/**
|
||||
* push a new element on the top of the stack using a entry object
|
||||
* this is only useful for internal processes where a special handle
|
||||
* is created
|
||||
* @param stack the name of the stack
|
||||
* @param b the new stack element
|
||||
* @return the handle used to store the new element
|
||||
* @throws IOException
|
||||
*/
|
||||
protected void push(String stack, Stack.Entry e) throws IOException {
|
||||
Stack s = getStack(stack);
|
||||
if (s == null) return;
|
||||
s.push(e);
|
||||
}
|
||||
|
||||
/**
|
||||
* get an element from the stack using the handle
|
||||
* @param stack the name of the stack
|
||||
* @param handle
|
||||
* @return the object that belongs to the handle
|
||||
* or null if no such element exists
|
||||
* @throws IOException
|
||||
*/
|
||||
public byte[] get(String stack, long handle) throws IOException {
|
||||
Stack s = getStack(stack);
|
||||
if (s == null) return null;
|
||||
return s.get(handle);
|
||||
}
|
||||
|
||||
/**
|
||||
* remove an element from the stack using the entry handle
|
||||
* @param stack the name of the stack
|
||||
* @param handle
|
||||
* @return the removed element
|
||||
* @throws IOException
|
||||
*/
|
||||
public byte[] remove(String stack, long handle) throws IOException {
|
||||
Stack s = getStack(stack);
|
||||
if (s == null) return null;
|
||||
return s.remove(handle);
|
||||
}
|
||||
|
||||
/**
|
||||
* remove the top element from the stack
|
||||
* @param stack the name of the stack
|
||||
* @return the top element or null if the stack is empty
|
||||
* @throws IOException
|
||||
*/
|
||||
public Stack.Entry pop(String stack) throws IOException {
|
||||
Stack s = getStack(stack);
|
||||
if (s == null) return null;
|
||||
return s.pop();
|
||||
}
|
||||
|
||||
/**
|
||||
* return the top element of the stack.
|
||||
* The element is not removed from the stack.
|
||||
* Successive calls to this method will always return the same element
|
||||
* @param stack the name of the stack
|
||||
* @return the element on the top of the stack or null, if stack is empty
|
||||
* @throws IOException
|
||||
*/
|
||||
public Stack.Entry top(String stack) throws IOException {
|
||||
Stack s = getStack(stack);
|
||||
if (s == null) return null;
|
||||
return s.top();
|
||||
}
|
||||
|
||||
/**
|
||||
* remove the bottom element from the stack
|
||||
* @param stack the name of the stack
|
||||
* @return the bottom element or null if the stack is empty
|
||||
* @throws IOException
|
||||
*/
|
||||
public Stack.Entry pot(String stack) throws IOException {
|
||||
Stack s = getStack(stack);
|
||||
if (s == null) return null;
|
||||
return s.pot();
|
||||
}
|
||||
|
||||
/**
|
||||
* return the bottom element of the stack.
|
||||
* The element is not removed from the stack.
|
||||
* Successive calls to this method will always return the same element
|
||||
* @param stack the name of the stack
|
||||
* @return the element on the bottom of the stack or null, if stack is empty
|
||||
* @throws IOException
|
||||
*/
|
||||
public Stack.Entry bot(String stack) throws IOException {
|
||||
Stack s = getStack(stack);
|
||||
if (s == null) return null;
|
||||
return s.bot();
|
||||
}
|
||||
|
||||
/**
|
||||
* close all stack files
|
||||
*/
|
||||
public synchronized void close() {
|
||||
for (StackInstance se: this.stacks.values()) {
|
||||
se.stack.close();
|
||||
}
|
||||
}
|
||||
|
||||
public void finalize() {
|
||||
this.close();
|
||||
}
|
||||
}
|
|
@ -163,6 +163,14 @@ public class HandleMap implements Iterable<Row.Entry> {
|
|||
index.clear();
|
||||
}
|
||||
|
||||
public synchronized byte[] smallestKey() {
|
||||
return index.smallestKey();
|
||||
}
|
||||
|
||||
public synchronized byte[] largestKey() {
|
||||
return index.largestKey();
|
||||
}
|
||||
|
||||
public synchronized boolean has(final byte[] key) {
|
||||
assert (key != null);
|
||||
return index.has(key);
|
||||
|
|
|
@ -72,6 +72,24 @@ public class ObjectIndexCache implements ObjectIndex {
|
|||
}
|
||||
}
|
||||
|
||||
public synchronized byte[] smallestKey() {
|
||||
byte[] b0 = index0.smallestKey();
|
||||
if (b0 == null) return null;
|
||||
if (index1 == null) return b0;
|
||||
byte[] b1 = index0.smallestKey();
|
||||
if (b1 == null || rowdef.objectOrder.compare(b1, b0) > 0) return b0;
|
||||
return b1;
|
||||
}
|
||||
|
||||
public synchronized byte[] largestKey() {
|
||||
byte[] b0 = index0.largestKey();
|
||||
if (b0 == null) return null;
|
||||
if (index1 == null) return b0;
|
||||
byte[] b1 = index0.largestKey();
|
||||
if (b1 == null || rowdef.objectOrder.compare(b0, b1) > 0) return b0;
|
||||
return b1;
|
||||
}
|
||||
|
||||
public synchronized Row.Entry get(final byte[] key) {
|
||||
assert (key != null);
|
||||
finishInitialization();
|
||||
|
|
|
@ -388,6 +388,22 @@ public class RowCollection implements Iterable<Row.Entry> {
|
|||
return r;
|
||||
}
|
||||
|
||||
public synchronized byte[] smallestKey() {
|
||||
if (chunkcount == 0) return null;
|
||||
this.sort();
|
||||
final Row.Entry r = get(0, false);
|
||||
byte[] b = r.getPrimaryKeyBytes();
|
||||
return b;
|
||||
}
|
||||
|
||||
public synchronized byte[] largestKey() {
|
||||
if (chunkcount == 0) return null;
|
||||
this.sort();
|
||||
final Row.Entry r = get(chunkcount - 1, false);
|
||||
byte[] b = r.getPrimaryKeyBytes();
|
||||
return b;
|
||||
}
|
||||
|
||||
public synchronized void clear() {
|
||||
if (this.chunkcache.length == 0) return;
|
||||
this.chunkcache = new byte[0];
|
||||
|
|
|
@ -26,7 +26,9 @@
|
|||
|
||||
package de.anomic.kelondro.order;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.Comparator;
|
||||
import java.util.Iterator;
|
||||
|
||||
public final class NaturalOrder extends AbstractOrder<byte[]> implements ByteOrder, Comparator<byte[]>, Cloneable {
|
||||
|
||||
|
@ -236,6 +238,35 @@ public final class NaturalOrder extends AbstractOrder<byte[]> implements ByteOrd
|
|||
return sb.toString();
|
||||
}
|
||||
|
||||
public static Iterator<Long> LongIterator(Iterator<byte[]> b256Iterator) throws IOException {
|
||||
return new LongIter(b256Iterator);
|
||||
}
|
||||
|
||||
public static class LongIter implements Iterator<Long> {
|
||||
|
||||
private Iterator<byte[]> b256Iterator;
|
||||
|
||||
public LongIter(Iterator<byte[]> b256Iterator) {
|
||||
this.b256Iterator = b256Iterator;
|
||||
}
|
||||
|
||||
public boolean hasNext() {
|
||||
return this.b256Iterator.hasNext();
|
||||
}
|
||||
|
||||
public Long next() {
|
||||
byte[] b = this.b256Iterator.next();
|
||||
assert (b != null);
|
||||
if (b == null) return null;
|
||||
return new Long(decodeLong(b));
|
||||
}
|
||||
|
||||
public void remove() {
|
||||
this.b256Iterator.remove();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
public static void main(final String[] args) {
|
||||
final byte[] t = new byte[12];
|
||||
for (int i = 0; i < 12; i++) t[i] = (byte) 255;
|
||||
|
|
|
@ -40,7 +40,7 @@ import de.anomic.kelondro.util.FileUtils;
|
|||
import de.anomic.kelondro.util.kelondroException;
|
||||
import de.anomic.yacy.logging.Log;
|
||||
|
||||
public final class Stack extends Records {
|
||||
public final class RecordStack extends Records {
|
||||
|
||||
// define the Over-Head-Array
|
||||
private static short thisOHBytes = 0; // our record definition does not need extra bytes
|
||||
|
@ -53,7 +53,7 @@ public final class Stack extends Records {
|
|||
protected static final int root = 0; // pointer for FHandles-array: pointer to root node
|
||||
protected static final int toor = 1; // pointer for FHandles-array: pointer to root node
|
||||
|
||||
public Stack(final File file, final Row rowdef) throws IOException {
|
||||
public RecordStack(final File file, final Row rowdef) throws IOException {
|
||||
// this creates a new stack
|
||||
super(file, thisOHBytes, thisOHHandles, rowdef, thisFHandles, rowdef.columns() /* txtProps */, 80 /* txtPropWidth */);
|
||||
if (super.fileExisted) {
|
||||
|
@ -64,15 +64,15 @@ public final class Stack extends Records {
|
|||
}
|
||||
}
|
||||
|
||||
public static final Stack open(final File file, final Row rowdef) {
|
||||
public static final RecordStack open(final File file, final Row rowdef) {
|
||||
try {
|
||||
return new Stack(file, rowdef);
|
||||
return new RecordStack(file, rowdef);
|
||||
} catch (final IOException e) {
|
||||
Log.logSevere("Stack", "Stack file open failed, deleting stack file " + file.toString());
|
||||
e.printStackTrace();
|
||||
FileUtils.deletedelete(file);
|
||||
try {
|
||||
return new Stack(file, rowdef);
|
||||
return new RecordStack(file, rowdef);
|
||||
} catch (final IOException ee) {
|
||||
System.out.println("kelondroStack: cannot open or create file " + file.toString());
|
||||
e.printStackTrace();
|
||||
|
@ -319,12 +319,12 @@ public final class Stack extends Records {
|
|||
System.err.println("( create, push, view, (g)pop, imp, shell)");
|
||||
System.exit(0);
|
||||
} else if (args.length == 2) {
|
||||
Stack fm = new Stack(new File(args[1]), lens);
|
||||
RecordStack fm = new RecordStack(new File(args[1]), lens);
|
||||
if (args[0].equals("-v")) {
|
||||
fm.print();
|
||||
ret = null;
|
||||
} else if (args[0].equals("-g")) {
|
||||
fm = new Stack(new File(args[1]), lens);
|
||||
fm = new RecordStack(new File(args[1]), lens);
|
||||
final Row.Entry ret2 = fm.pop();
|
||||
ret = ((ret2 == null) ? null : ret2.getColBytes(1));
|
||||
fm.close();
|
||||
|
@ -332,7 +332,7 @@ public final class Stack extends Records {
|
|||
fm.close();
|
||||
} else if (args.length == 3) {
|
||||
if (args[0].equals("-i")) {
|
||||
final Stack fm = new Stack(new File(args[2]), lens);
|
||||
final RecordStack fm = new RecordStack(new File(args[2]), lens);
|
||||
final int i = fm.imp(new File(args[1]),";");
|
||||
fm.close();
|
||||
ret = (i + " records imported").getBytes();
|
||||
|
@ -356,7 +356,7 @@ public final class Stack extends Records {
|
|||
if (f != null) try {f.close();}catch(final Exception e) {}
|
||||
}
|
||||
} else if (args[0].equals("-g")) {
|
||||
final Stack fm = new Stack(new File(args[2]), lens);
|
||||
final RecordStack fm = new RecordStack(new File(args[2]), lens);
|
||||
final Row.Entry ret2 = fm.pop();
|
||||
ret = ((ret2 == null) ? null : ret2.getColBytes(1));
|
||||
fm.close();
|
||||
|
@ -366,10 +366,10 @@ public final class Stack extends Records {
|
|||
// create <keylen> <valuelen> <filename>
|
||||
final File f = new File(args[3]);
|
||||
if (f.exists()) FileUtils.deletedelete(f);
|
||||
final Stack fm = new Stack(f, lens);
|
||||
final RecordStack fm = new RecordStack(f, lens);
|
||||
fm.close();
|
||||
} else if (args[0].equals("-p")) {
|
||||
final Stack fm = new Stack(new File(args[3]), lens);
|
||||
final RecordStack fm = new RecordStack(new File(args[3]), lens);
|
||||
fm.push(fm.row().newEntry(new byte[][] {args[1].getBytes(), args[2].getBytes()}));
|
||||
fm.close();
|
||||
}
|
|
@ -53,14 +53,14 @@ import java.util.Iterator;
|
|||
import de.anomic.kelondro.index.Column;
|
||||
import de.anomic.kelondro.index.Row;
|
||||
import de.anomic.kelondro.order.NaturalOrder;
|
||||
import de.anomic.kelondro.table.Stack;
|
||||
import de.anomic.kelondro.table.RecordStack;
|
||||
import de.anomic.kelondro.util.DateFormatter;
|
||||
import de.anomic.kelondro.util.FileUtils;
|
||||
|
||||
public class yacyNewsQueue {
|
||||
|
||||
private final File path;
|
||||
Stack queueStack;
|
||||
RecordStack queueStack;
|
||||
private final yacyNewsDB newsDB;
|
||||
|
||||
public static final Row rowdef = new Row(new Column[]{
|
||||
|
@ -73,13 +73,13 @@ public class yacyNewsQueue {
|
|||
public yacyNewsQueue(final File path, final yacyNewsDB newsDB) {
|
||||
this.path = path;
|
||||
this.newsDB = newsDB;
|
||||
this.queueStack = Stack.open(path, rowdef);
|
||||
this.queueStack = RecordStack.open(path, rowdef);
|
||||
}
|
||||
|
||||
private void resetDB() {
|
||||
try {close();} catch (final Exception e) {}
|
||||
if (path.exists()) FileUtils.deletedelete(path);
|
||||
queueStack = Stack.open(path, rowdef);
|
||||
queueStack = RecordStack.open(path, rowdef);
|
||||
}
|
||||
|
||||
public void clear() {
|
||||
|
|
Loading…
Reference in New Issue
Block a user