mirror of
https://github.com/yacy/yacy_search_server.git
synced 2024-09-19 00:01:41 +02:00
- renamed Stack to RecordStack to avoid name confusion with new classes
- added new Stack class that implements a stack on BLOB files - added new Stacks class that can be used for a set of Stacks (a 'Stack Database') - added methods to other classes to support the new stacks git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@6169 6c8d7289-2bf4-0310-a012-ef5d649a1542
This commit is contained in:
parent
ec2970cc76
commit
24cb6d68bc
|
@ -39,7 +39,7 @@ import de.anomic.http.httpResponseHeader;
|
||||||
import de.anomic.kelondro.index.Row;
|
import de.anomic.kelondro.index.Row;
|
||||||
import de.anomic.kelondro.order.Base64Order;
|
import de.anomic.kelondro.order.Base64Order;
|
||||||
import de.anomic.kelondro.order.NaturalOrder;
|
import de.anomic.kelondro.order.NaturalOrder;
|
||||||
import de.anomic.kelondro.table.Stack;
|
import de.anomic.kelondro.table.RecordStack;
|
||||||
import de.anomic.kelondro.text.MetadataRepository;
|
import de.anomic.kelondro.text.MetadataRepository;
|
||||||
import de.anomic.kelondro.text.metadataPrototype.URLMetadataRow;
|
import de.anomic.kelondro.text.metadataPrototype.URLMetadataRow;
|
||||||
import de.anomic.kelondro.util.DateFormatter;
|
import de.anomic.kelondro.util.DateFormatter;
|
||||||
|
@ -53,7 +53,7 @@ import de.anomic.yacy.logging.Log;
|
||||||
public class IndexingStack {
|
public class IndexingStack {
|
||||||
|
|
||||||
protected final CrawlProfile profiles;
|
protected final CrawlProfile profiles;
|
||||||
protected final Stack sbQueueStack;
|
protected final RecordStack sbQueueStack;
|
||||||
protected final yacySeedDB peers;
|
protected final yacySeedDB peers;
|
||||||
protected final ConcurrentHashMap<String, QueueEntry> queueInProcess;
|
protected final ConcurrentHashMap<String, QueueEntry> queueInProcess;
|
||||||
|
|
||||||
|
@ -64,7 +64,7 @@ public class IndexingStack {
|
||||||
this.profiles = profiles;
|
this.profiles = profiles;
|
||||||
this.peers = peers;
|
this.peers = peers;
|
||||||
this.queueInProcess = new ConcurrentHashMap<String, QueueEntry>();
|
this.queueInProcess = new ConcurrentHashMap<String, QueueEntry>();
|
||||||
this.sbQueueStack = Stack.open(sbQueueStackPath, rowdef);
|
this.sbQueueStack = RecordStack.open(sbQueueStackPath, rowdef);
|
||||||
}
|
}
|
||||||
|
|
||||||
public static final Row rowdef = new Row(
|
public static final Row rowdef = new Row(
|
||||||
|
|
|
@ -234,6 +234,54 @@ public class HeapReader {
|
||||||
return this.ordering;
|
return this.ordering;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* find a special key in the heap: the one with the smallest key
|
||||||
|
* this method is useful if the entries are ordered using their keys.
|
||||||
|
* then the key with the smallest key denotes the first entry
|
||||||
|
* @return the smallest key in the heap
|
||||||
|
* @throws IOException
|
||||||
|
*/
|
||||||
|
protected synchronized byte[] firstKey() throws IOException {
|
||||||
|
return index.smallestKey();
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* find a special blob in the heap: one that has the smallest key
|
||||||
|
* this method is useful if the entries are ordered using their keys.
|
||||||
|
* then the key with the smallest key denotes the first entry
|
||||||
|
* @return the entry which key is the smallest in the heap
|
||||||
|
* @throws IOException
|
||||||
|
*/
|
||||||
|
protected synchronized byte[] first() throws IOException {
|
||||||
|
byte[] key = index.smallestKey();
|
||||||
|
if (key == null) return null;
|
||||||
|
return get(key);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* find a special key in the heap: the one with the largest key
|
||||||
|
* this method is useful if the entries are ordered using their keys.
|
||||||
|
* then the key with the largest key denotes the last entry
|
||||||
|
* @return the largest key in the heap
|
||||||
|
* @throws IOException
|
||||||
|
*/
|
||||||
|
protected synchronized byte[] lastKey() throws IOException {
|
||||||
|
return index.largestKey();
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* find a special blob in the heap: one that has the largest key
|
||||||
|
* this method is useful if the entries are ordered using their keys.
|
||||||
|
* then the key with the largest key denotes the last entry
|
||||||
|
* @return the entry which key is the smallest in the heap
|
||||||
|
* @throws IOException
|
||||||
|
*/
|
||||||
|
protected synchronized byte[] last() throws IOException {
|
||||||
|
byte[] key = index.largestKey();
|
||||||
|
if (key == null) return null;
|
||||||
|
return get(key);
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* read a blob from the heap
|
* read a blob from the heap
|
||||||
* @param key
|
* @param key
|
||||||
|
|
239
source/de/anomic/kelondro/blob/Stack.java
Normal file
239
source/de/anomic/kelondro/blob/Stack.java
Normal file
|
@ -0,0 +1,239 @@
|
||||||
|
// Stack.java
|
||||||
|
// (C) 2009 by Michael Peter Christen; mc@yacy.net, Frankfurt a. M., Germany
|
||||||
|
// first published 03.07.2009 on http://yacy.net
|
||||||
|
//
|
||||||
|
// $LastChangedDate: 2008-03-14 01:16:04 +0100 (Fr, 14 Mrz 2008) $
|
||||||
|
// $LastChangedRevision: 4558 $
|
||||||
|
// $LastChangedBy: orbiter $
|
||||||
|
//
|
||||||
|
// LICENSE
|
||||||
|
//
|
||||||
|
// This program is free software; you can redistribute it and/or modify
|
||||||
|
// it under the terms of the GNU General Public License as published by
|
||||||
|
// the Free Software Foundation; either version 2 of the License, or
|
||||||
|
// (at your option) any later version.
|
||||||
|
//
|
||||||
|
// This program is distributed in the hope that it will be useful,
|
||||||
|
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||||
|
// GNU General Public License for more details.
|
||||||
|
//
|
||||||
|
// You should have received a copy of the GNU General Public License
|
||||||
|
// along with this program; if not, write to the Free Software
|
||||||
|
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
||||||
|
|
||||||
|
package de.anomic.kelondro.blob;
|
||||||
|
|
||||||
|
import java.io.File;
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.util.Iterator;
|
||||||
|
|
||||||
|
import de.anomic.kelondro.order.NaturalOrder;
|
||||||
|
|
||||||
|
public class Stack {
|
||||||
|
|
||||||
|
private Heap stack;
|
||||||
|
private long lastHandle;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* create a new stack object.
|
||||||
|
* a stack object is backed by a blob file that contains the stack entries.
|
||||||
|
* all stack entries can be accessed with a long handle; the handle is
|
||||||
|
* represented as b256-encoded byte[] as key in the blob.
|
||||||
|
* The handle is created using the current time. That means that the top
|
||||||
|
* element on the stack has the maximum time as key handle and the element
|
||||||
|
* at the bottom of the stack has the minimum time as key handle
|
||||||
|
* @param stackFile
|
||||||
|
* @throws IOException
|
||||||
|
*/
|
||||||
|
public Stack(final File stackFile) throws IOException {
|
||||||
|
this.stack = new Heap(stackFile, 8, NaturalOrder.naturalOrder, 0);
|
||||||
|
this.lastHandle = 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* create a new time handle. In case that the method is called
|
||||||
|
* within a single millisecond twice, a new handle is created using
|
||||||
|
* an increment of the previous handle to avoid handle collisions.
|
||||||
|
* This method must be called in an synchronized environment
|
||||||
|
* @return a unique handle for this stack
|
||||||
|
*/
|
||||||
|
private long nextHandle() {
|
||||||
|
long h = System.currentTimeMillis();
|
||||||
|
if (h <= this.lastHandle) h = lastHandle + 1;
|
||||||
|
lastHandle = h;
|
||||||
|
return h;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Iterate all handles from the stack as Long numbers
|
||||||
|
* @return an iterator of all handles of the stack
|
||||||
|
* @throws IOException
|
||||||
|
*/
|
||||||
|
public synchronized Iterator<Long> handles() throws IOException {
|
||||||
|
return NaturalOrder.LongIterator(this.stack.keys(true, false));
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* get the size of a stack
|
||||||
|
* @return the number of entries on the stack
|
||||||
|
*/
|
||||||
|
public synchronized int size() {
|
||||||
|
return this.stack.size();
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* push a new element on the top of the stack
|
||||||
|
* @param b the new stack element
|
||||||
|
* @return the handle used to store the new element
|
||||||
|
* @throws IOException
|
||||||
|
*/
|
||||||
|
public synchronized long push(byte[] b) throws IOException {
|
||||||
|
long handle = nextHandle();
|
||||||
|
this.stack.put(NaturalOrder.encodeLong(handle, 8), b);
|
||||||
|
return handle;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* push a new element on the top of the stack using a entry object
|
||||||
|
* this is only useful for internal processes where a special handle
|
||||||
|
* is created
|
||||||
|
* @param b the new stack element
|
||||||
|
* @return the handle used to store the new element
|
||||||
|
* @throws IOException
|
||||||
|
*/
|
||||||
|
protected synchronized void push(Entry e) throws IOException {
|
||||||
|
this.stack.put(NaturalOrder.encodeLong(e.h, 8), e.b);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* get an element from the stack using the handle
|
||||||
|
* @param handle
|
||||||
|
* @return the object that belongs to the handle
|
||||||
|
* or null if no such element exists
|
||||||
|
* @throws IOException
|
||||||
|
*/
|
||||||
|
public synchronized byte[] get(long handle) throws IOException {
|
||||||
|
byte[] k = NaturalOrder.encodeLong(handle, 8);
|
||||||
|
byte[] b = this.stack.get(k);
|
||||||
|
if (b == null) return null;
|
||||||
|
return b;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* remove an element from the stack using the entry handle
|
||||||
|
* @param handle
|
||||||
|
* @return the removed element
|
||||||
|
* @throws IOException
|
||||||
|
*/
|
||||||
|
public synchronized byte[] remove(long handle) throws IOException {
|
||||||
|
byte[] k = NaturalOrder.encodeLong(handle, 8);
|
||||||
|
byte[] b = this.stack.get(k);
|
||||||
|
if (b == null) return null;
|
||||||
|
this.stack.remove(k);
|
||||||
|
return b;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* remove the top element from the stack
|
||||||
|
* @return the top element or null if the stack is empty
|
||||||
|
* @throws IOException
|
||||||
|
*/
|
||||||
|
public synchronized Entry pop() throws IOException {
|
||||||
|
return po(this.stack.lastKey(), true);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* return the top element of the stack.
|
||||||
|
* The element is not removed from the stack.
|
||||||
|
* Successive calls to this method will always return the same element
|
||||||
|
* @return the element on the top of the stack or null, if stack is empty
|
||||||
|
* @throws IOException
|
||||||
|
*/
|
||||||
|
public synchronized Entry top() throws IOException {
|
||||||
|
return po(this.stack.lastKey(), false);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* remove the bottom element from the stack
|
||||||
|
* @return the bottom element or null if the stack is empty
|
||||||
|
* @throws IOException
|
||||||
|
*/
|
||||||
|
public synchronized Entry pot() throws IOException {
|
||||||
|
return po(this.stack.firstKey(), true);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* return the bottom element of the stack.
|
||||||
|
* The element is not removed from the stack.
|
||||||
|
* Successive calls to this method will always return the same element
|
||||||
|
* @return the element on the bottom of the stack or null, if stack is empty
|
||||||
|
* @throws IOException
|
||||||
|
*/
|
||||||
|
public synchronized Entry bot() throws IOException {
|
||||||
|
return po(this.stack.firstKey(), false);
|
||||||
|
}
|
||||||
|
|
||||||
|
private Entry po(byte[] k, boolean remove) throws IOException {
|
||||||
|
if (k == null) return null;
|
||||||
|
assert k.length == 8;
|
||||||
|
byte[] b = this.stack.get(k);
|
||||||
|
assert b != null;
|
||||||
|
if (b == null) return null;
|
||||||
|
if (remove) this.stack.remove(k);
|
||||||
|
return new Entry(k, b);
|
||||||
|
}
|
||||||
|
|
||||||
|
public class Entry {
|
||||||
|
|
||||||
|
private long h;
|
||||||
|
private byte[] b;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* create a new entry object using a long handle
|
||||||
|
* @param h
|
||||||
|
* @param b
|
||||||
|
*/
|
||||||
|
public Entry(long h, byte[] b) {
|
||||||
|
this.h = h;
|
||||||
|
this.b = b;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* create a new entry object using the byte[] encoded handle
|
||||||
|
* @param k
|
||||||
|
* @param b
|
||||||
|
*/
|
||||||
|
public Entry(byte[] k, byte[] b) {
|
||||||
|
this.h = NaturalOrder.decodeLong(k);
|
||||||
|
this.b = b;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* get the handle
|
||||||
|
* @return the handle
|
||||||
|
*/
|
||||||
|
public long handle() {
|
||||||
|
return h;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* get the blob entry
|
||||||
|
* @return the blob
|
||||||
|
*/
|
||||||
|
public byte[] blob() {
|
||||||
|
return b;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* close the stack file and write a handle index
|
||||||
|
*/
|
||||||
|
public synchronized void close() {
|
||||||
|
this.stack.close(true);
|
||||||
|
}
|
||||||
|
|
||||||
|
public void finalize() {
|
||||||
|
this.close();
|
||||||
|
}
|
||||||
|
}
|
274
source/de/anomic/kelondro/blob/Stacks.java
Normal file
274
source/de/anomic/kelondro/blob/Stacks.java
Normal file
|
@ -0,0 +1,274 @@
|
||||||
|
// Stacks.java
|
||||||
|
// (C) 2009 by Michael Peter Christen; mc@yacy.net, Frankfurt a. M., Germany
|
||||||
|
// first published 03.07.2009 on http://yacy.net
|
||||||
|
//
|
||||||
|
// $LastChangedDate: 2008-03-14 01:16:04 +0100 (Fr, 14 Mrz 2008) $
|
||||||
|
// $LastChangedRevision: 4558 $
|
||||||
|
// $LastChangedBy: orbiter $
|
||||||
|
//
|
||||||
|
// LICENSE
|
||||||
|
//
|
||||||
|
// This program is free software; you can redistribute it and/or modify
|
||||||
|
// it under the terms of the GNU General Public License as published by
|
||||||
|
// the Free Software Foundation; either version 2 of the License, or
|
||||||
|
// (at your option) any later version.
|
||||||
|
//
|
||||||
|
// This program is distributed in the hope that it will be useful,
|
||||||
|
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||||
|
// GNU General Public License for more details.
|
||||||
|
//
|
||||||
|
// You should have received a copy of the GNU General Public License
|
||||||
|
// along with this program; if not, write to the Free Software
|
||||||
|
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
||||||
|
|
||||||
|
package de.anomic.kelondro.blob;
|
||||||
|
|
||||||
|
import java.io.File;
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.util.Iterator;
|
||||||
|
import java.util.concurrent.ConcurrentHashMap;
|
||||||
|
|
||||||
|
public class Stacks {
|
||||||
|
|
||||||
|
private File stacksLocation;
|
||||||
|
private String stacksPrefix;
|
||||||
|
private ConcurrentHashMap<String, StackInstance> stacks;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* create a stack organizing object.
|
||||||
|
* Stacks can be created on-the-fly in the given stacksLocation directory
|
||||||
|
* using simple push operations that create first entries in the stack
|
||||||
|
* Stacks that do not contain any element upon the close() operation are removed
|
||||||
|
* @param stackFile
|
||||||
|
* @throws IOException
|
||||||
|
*/
|
||||||
|
public Stacks(final File stacksLocation, String stacksPrefix) {
|
||||||
|
if (!stacksLocation.exists()) stacksLocation.mkdirs();
|
||||||
|
assert stacksLocation.isDirectory();
|
||||||
|
this.stacksLocation = stacksLocation;
|
||||||
|
this.stacksPrefix = stacksPrefix;
|
||||||
|
|
||||||
|
// initialize the stacks map
|
||||||
|
this.stacks = new ConcurrentHashMap<String, StackInstance>();
|
||||||
|
String[] sl = this.stacksLocation.list();
|
||||||
|
for (String s: sl) {
|
||||||
|
if (!s.startsWith(this.stacksPrefix + "_")) continue;
|
||||||
|
StackInstance si;
|
||||||
|
try {
|
||||||
|
si = new StackInstance(new File(this.stacksLocation, s));
|
||||||
|
this.stacks.put(si.name, si);
|
||||||
|
} catch (IOException e) {
|
||||||
|
e.printStackTrace();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private class StackInstance {
|
||||||
|
private String name;
|
||||||
|
private File location;
|
||||||
|
private Stack stack;
|
||||||
|
|
||||||
|
public StackInstance(File location) throws IOException {
|
||||||
|
String filename = location.getName();
|
||||||
|
assert filename.startsWith(stacksPrefix + "_");
|
||||||
|
assert filename.endsWith(".bstack");
|
||||||
|
this.name = filename.substring(stacksPrefix.length() + 1, filename.length() - 7);
|
||||||
|
this.location = location;
|
||||||
|
this.stack = new Stack(location);
|
||||||
|
}
|
||||||
|
|
||||||
|
public StackInstance(String stack) throws IOException {
|
||||||
|
this.location = new File(stacksLocation, stacksPrefix + "_" + stack + ".bstack");
|
||||||
|
this.name = stack;
|
||||||
|
this.stack = new Stack(location);
|
||||||
|
}
|
||||||
|
|
||||||
|
public String name() {
|
||||||
|
return this.name;
|
||||||
|
}
|
||||||
|
|
||||||
|
public Stack stack() {
|
||||||
|
return this.stack;
|
||||||
|
}
|
||||||
|
|
||||||
|
public File location() {
|
||||||
|
return this.location;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private Stack getStack(String stack) {
|
||||||
|
StackInstance si = this.stacks.get(stack);
|
||||||
|
if (si == null) {
|
||||||
|
// create a new Stack on the fly
|
||||||
|
try {
|
||||||
|
si = new StackInstance(stack);
|
||||||
|
this.stacks.put(stack, si);
|
||||||
|
return si.stack();
|
||||||
|
} catch (IOException e) {
|
||||||
|
e.printStackTrace();
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return si.stack();
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* get the number of stacks organized by this object
|
||||||
|
* @return the number of stack objects
|
||||||
|
*/
|
||||||
|
public int size() {
|
||||||
|
return this.stacks.size();
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* iterate all stack names
|
||||||
|
* @return an iterator for the stack names
|
||||||
|
*/
|
||||||
|
public Iterator<String> stacks() {
|
||||||
|
return this.stacks.keySet().iterator();
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* get the size of a stack
|
||||||
|
* @param stack the name of the stack
|
||||||
|
* @return the number of entries on the stack
|
||||||
|
*/
|
||||||
|
public int size(String stack) {
|
||||||
|
Stack s = getStack(stack);
|
||||||
|
if (s == null) return -1;
|
||||||
|
return s.size();
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Iterate all handles from a stack as Long numbers
|
||||||
|
* @param stack the name of the stack
|
||||||
|
* @return an iterator of all handles of the stack
|
||||||
|
* @throws IOException
|
||||||
|
*/
|
||||||
|
public synchronized Iterator<Long> handles(String stack) throws IOException {
|
||||||
|
Stack s = getStack(stack);
|
||||||
|
if (s == null) return null;
|
||||||
|
return s.handles();
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* push a new element on the top of the stack
|
||||||
|
* @param stack the name of the stack
|
||||||
|
* @param b the new stack element
|
||||||
|
* @return the handle used to store the new element
|
||||||
|
* @throws IOException
|
||||||
|
*/
|
||||||
|
public long push(String stack, byte[] b) throws IOException {
|
||||||
|
Stack s = getStack(stack);
|
||||||
|
if (s == null) return -1;
|
||||||
|
return s.push(b);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* push a new element on the top of the stack using a entry object
|
||||||
|
* this is only useful for internal processes where a special handle
|
||||||
|
* is created
|
||||||
|
* @param stack the name of the stack
|
||||||
|
* @param b the new stack element
|
||||||
|
* @return the handle used to store the new element
|
||||||
|
* @throws IOException
|
||||||
|
*/
|
||||||
|
protected void push(String stack, Stack.Entry e) throws IOException {
|
||||||
|
Stack s = getStack(stack);
|
||||||
|
if (s == null) return;
|
||||||
|
s.push(e);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* get an element from the stack using the handle
|
||||||
|
* @param stack the name of the stack
|
||||||
|
* @param handle
|
||||||
|
* @return the object that belongs to the handle
|
||||||
|
* or null if no such element exists
|
||||||
|
* @throws IOException
|
||||||
|
*/
|
||||||
|
public byte[] get(String stack, long handle) throws IOException {
|
||||||
|
Stack s = getStack(stack);
|
||||||
|
if (s == null) return null;
|
||||||
|
return s.get(handle);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* remove an element from the stack using the entry handle
|
||||||
|
* @param stack the name of the stack
|
||||||
|
* @param handle
|
||||||
|
* @return the removed element
|
||||||
|
* @throws IOException
|
||||||
|
*/
|
||||||
|
public byte[] remove(String stack, long handle) throws IOException {
|
||||||
|
Stack s = getStack(stack);
|
||||||
|
if (s == null) return null;
|
||||||
|
return s.remove(handle);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* remove the top element from the stack
|
||||||
|
* @param stack the name of the stack
|
||||||
|
* @return the top element or null if the stack is empty
|
||||||
|
* @throws IOException
|
||||||
|
*/
|
||||||
|
public Stack.Entry pop(String stack) throws IOException {
|
||||||
|
Stack s = getStack(stack);
|
||||||
|
if (s == null) return null;
|
||||||
|
return s.pop();
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* return the top element of the stack.
|
||||||
|
* The element is not removed from the stack.
|
||||||
|
* Successive calls to this method will always return the same element
|
||||||
|
* @param stack the name of the stack
|
||||||
|
* @return the element on the top of the stack or null, if stack is empty
|
||||||
|
* @throws IOException
|
||||||
|
*/
|
||||||
|
public Stack.Entry top(String stack) throws IOException {
|
||||||
|
Stack s = getStack(stack);
|
||||||
|
if (s == null) return null;
|
||||||
|
return s.top();
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* remove the bottom element from the stack
|
||||||
|
* @param stack the name of the stack
|
||||||
|
* @return the bottom element or null if the stack is empty
|
||||||
|
* @throws IOException
|
||||||
|
*/
|
||||||
|
public Stack.Entry pot(String stack) throws IOException {
|
||||||
|
Stack s = getStack(stack);
|
||||||
|
if (s == null) return null;
|
||||||
|
return s.pot();
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* return the bottom element of the stack.
|
||||||
|
* The element is not removed from the stack.
|
||||||
|
* Successive calls to this method will always return the same element
|
||||||
|
* @param stack the name of the stack
|
||||||
|
* @return the element on the bottom of the stack or null, if stack is empty
|
||||||
|
* @throws IOException
|
||||||
|
*/
|
||||||
|
public Stack.Entry bot(String stack) throws IOException {
|
||||||
|
Stack s = getStack(stack);
|
||||||
|
if (s == null) return null;
|
||||||
|
return s.bot();
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* close all stack files
|
||||||
|
*/
|
||||||
|
public synchronized void close() {
|
||||||
|
for (StackInstance se: this.stacks.values()) {
|
||||||
|
se.stack.close();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public void finalize() {
|
||||||
|
this.close();
|
||||||
|
}
|
||||||
|
}
|
|
@ -163,6 +163,14 @@ public class HandleMap implements Iterable<Row.Entry> {
|
||||||
index.clear();
|
index.clear();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public synchronized byte[] smallestKey() {
|
||||||
|
return index.smallestKey();
|
||||||
|
}
|
||||||
|
|
||||||
|
public synchronized byte[] largestKey() {
|
||||||
|
return index.largestKey();
|
||||||
|
}
|
||||||
|
|
||||||
public synchronized boolean has(final byte[] key) {
|
public synchronized boolean has(final byte[] key) {
|
||||||
assert (key != null);
|
assert (key != null);
|
||||||
return index.has(key);
|
return index.has(key);
|
||||||
|
|
|
@ -72,6 +72,24 @@ public class ObjectIndexCache implements ObjectIndex {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public synchronized byte[] smallestKey() {
|
||||||
|
byte[] b0 = index0.smallestKey();
|
||||||
|
if (b0 == null) return null;
|
||||||
|
if (index1 == null) return b0;
|
||||||
|
byte[] b1 = index0.smallestKey();
|
||||||
|
if (b1 == null || rowdef.objectOrder.compare(b1, b0) > 0) return b0;
|
||||||
|
return b1;
|
||||||
|
}
|
||||||
|
|
||||||
|
public synchronized byte[] largestKey() {
|
||||||
|
byte[] b0 = index0.largestKey();
|
||||||
|
if (b0 == null) return null;
|
||||||
|
if (index1 == null) return b0;
|
||||||
|
byte[] b1 = index0.largestKey();
|
||||||
|
if (b1 == null || rowdef.objectOrder.compare(b0, b1) > 0) return b0;
|
||||||
|
return b1;
|
||||||
|
}
|
||||||
|
|
||||||
public synchronized Row.Entry get(final byte[] key) {
|
public synchronized Row.Entry get(final byte[] key) {
|
||||||
assert (key != null);
|
assert (key != null);
|
||||||
finishInitialization();
|
finishInitialization();
|
||||||
|
|
|
@ -388,6 +388,22 @@ public class RowCollection implements Iterable<Row.Entry> {
|
||||||
return r;
|
return r;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public synchronized byte[] smallestKey() {
|
||||||
|
if (chunkcount == 0) return null;
|
||||||
|
this.sort();
|
||||||
|
final Row.Entry r = get(0, false);
|
||||||
|
byte[] b = r.getPrimaryKeyBytes();
|
||||||
|
return b;
|
||||||
|
}
|
||||||
|
|
||||||
|
public synchronized byte[] largestKey() {
|
||||||
|
if (chunkcount == 0) return null;
|
||||||
|
this.sort();
|
||||||
|
final Row.Entry r = get(chunkcount - 1, false);
|
||||||
|
byte[] b = r.getPrimaryKeyBytes();
|
||||||
|
return b;
|
||||||
|
}
|
||||||
|
|
||||||
public synchronized void clear() {
|
public synchronized void clear() {
|
||||||
if (this.chunkcache.length == 0) return;
|
if (this.chunkcache.length == 0) return;
|
||||||
this.chunkcache = new byte[0];
|
this.chunkcache = new byte[0];
|
||||||
|
|
|
@ -26,7 +26,9 @@
|
||||||
|
|
||||||
package de.anomic.kelondro.order;
|
package de.anomic.kelondro.order;
|
||||||
|
|
||||||
|
import java.io.IOException;
|
||||||
import java.util.Comparator;
|
import java.util.Comparator;
|
||||||
|
import java.util.Iterator;
|
||||||
|
|
||||||
public final class NaturalOrder extends AbstractOrder<byte[]> implements ByteOrder, Comparator<byte[]>, Cloneable {
|
public final class NaturalOrder extends AbstractOrder<byte[]> implements ByteOrder, Comparator<byte[]>, Cloneable {
|
||||||
|
|
||||||
|
@ -236,6 +238,35 @@ public final class NaturalOrder extends AbstractOrder<byte[]> implements ByteOrd
|
||||||
return sb.toString();
|
return sb.toString();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public static Iterator<Long> LongIterator(Iterator<byte[]> b256Iterator) throws IOException {
|
||||||
|
return new LongIter(b256Iterator);
|
||||||
|
}
|
||||||
|
|
||||||
|
public static class LongIter implements Iterator<Long> {
|
||||||
|
|
||||||
|
private Iterator<byte[]> b256Iterator;
|
||||||
|
|
||||||
|
public LongIter(Iterator<byte[]> b256Iterator) {
|
||||||
|
this.b256Iterator = b256Iterator;
|
||||||
|
}
|
||||||
|
|
||||||
|
public boolean hasNext() {
|
||||||
|
return this.b256Iterator.hasNext();
|
||||||
|
}
|
||||||
|
|
||||||
|
public Long next() {
|
||||||
|
byte[] b = this.b256Iterator.next();
|
||||||
|
assert (b != null);
|
||||||
|
if (b == null) return null;
|
||||||
|
return new Long(decodeLong(b));
|
||||||
|
}
|
||||||
|
|
||||||
|
public void remove() {
|
||||||
|
this.b256Iterator.remove();
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
public static void main(final String[] args) {
|
public static void main(final String[] args) {
|
||||||
final byte[] t = new byte[12];
|
final byte[] t = new byte[12];
|
||||||
for (int i = 0; i < 12; i++) t[i] = (byte) 255;
|
for (int i = 0; i < 12; i++) t[i] = (byte) 255;
|
||||||
|
|
|
@ -40,7 +40,7 @@ import de.anomic.kelondro.util.FileUtils;
|
||||||
import de.anomic.kelondro.util.kelondroException;
|
import de.anomic.kelondro.util.kelondroException;
|
||||||
import de.anomic.yacy.logging.Log;
|
import de.anomic.yacy.logging.Log;
|
||||||
|
|
||||||
public final class Stack extends Records {
|
public final class RecordStack extends Records {
|
||||||
|
|
||||||
// define the Over-Head-Array
|
// define the Over-Head-Array
|
||||||
private static short thisOHBytes = 0; // our record definition does not need extra bytes
|
private static short thisOHBytes = 0; // our record definition does not need extra bytes
|
||||||
|
@ -53,7 +53,7 @@ public final class Stack extends Records {
|
||||||
protected static final int root = 0; // pointer for FHandles-array: pointer to root node
|
protected static final int root = 0; // pointer for FHandles-array: pointer to root node
|
||||||
protected static final int toor = 1; // pointer for FHandles-array: pointer to root node
|
protected static final int toor = 1; // pointer for FHandles-array: pointer to root node
|
||||||
|
|
||||||
public Stack(final File file, final Row rowdef) throws IOException {
|
public RecordStack(final File file, final Row rowdef) throws IOException {
|
||||||
// this creates a new stack
|
// this creates a new stack
|
||||||
super(file, thisOHBytes, thisOHHandles, rowdef, thisFHandles, rowdef.columns() /* txtProps */, 80 /* txtPropWidth */);
|
super(file, thisOHBytes, thisOHHandles, rowdef, thisFHandles, rowdef.columns() /* txtProps */, 80 /* txtPropWidth */);
|
||||||
if (super.fileExisted) {
|
if (super.fileExisted) {
|
||||||
|
@ -64,15 +64,15 @@ public final class Stack extends Records {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public static final Stack open(final File file, final Row rowdef) {
|
public static final RecordStack open(final File file, final Row rowdef) {
|
||||||
try {
|
try {
|
||||||
return new Stack(file, rowdef);
|
return new RecordStack(file, rowdef);
|
||||||
} catch (final IOException e) {
|
} catch (final IOException e) {
|
||||||
Log.logSevere("Stack", "Stack file open failed, deleting stack file " + file.toString());
|
Log.logSevere("Stack", "Stack file open failed, deleting stack file " + file.toString());
|
||||||
e.printStackTrace();
|
e.printStackTrace();
|
||||||
FileUtils.deletedelete(file);
|
FileUtils.deletedelete(file);
|
||||||
try {
|
try {
|
||||||
return new Stack(file, rowdef);
|
return new RecordStack(file, rowdef);
|
||||||
} catch (final IOException ee) {
|
} catch (final IOException ee) {
|
||||||
System.out.println("kelondroStack: cannot open or create file " + file.toString());
|
System.out.println("kelondroStack: cannot open or create file " + file.toString());
|
||||||
e.printStackTrace();
|
e.printStackTrace();
|
||||||
|
@ -319,12 +319,12 @@ public final class Stack extends Records {
|
||||||
System.err.println("( create, push, view, (g)pop, imp, shell)");
|
System.err.println("( create, push, view, (g)pop, imp, shell)");
|
||||||
System.exit(0);
|
System.exit(0);
|
||||||
} else if (args.length == 2) {
|
} else if (args.length == 2) {
|
||||||
Stack fm = new Stack(new File(args[1]), lens);
|
RecordStack fm = new RecordStack(new File(args[1]), lens);
|
||||||
if (args[0].equals("-v")) {
|
if (args[0].equals("-v")) {
|
||||||
fm.print();
|
fm.print();
|
||||||
ret = null;
|
ret = null;
|
||||||
} else if (args[0].equals("-g")) {
|
} else if (args[0].equals("-g")) {
|
||||||
fm = new Stack(new File(args[1]), lens);
|
fm = new RecordStack(new File(args[1]), lens);
|
||||||
final Row.Entry ret2 = fm.pop();
|
final Row.Entry ret2 = fm.pop();
|
||||||
ret = ((ret2 == null) ? null : ret2.getColBytes(1));
|
ret = ((ret2 == null) ? null : ret2.getColBytes(1));
|
||||||
fm.close();
|
fm.close();
|
||||||
|
@ -332,7 +332,7 @@ public final class Stack extends Records {
|
||||||
fm.close();
|
fm.close();
|
||||||
} else if (args.length == 3) {
|
} else if (args.length == 3) {
|
||||||
if (args[0].equals("-i")) {
|
if (args[0].equals("-i")) {
|
||||||
final Stack fm = new Stack(new File(args[2]), lens);
|
final RecordStack fm = new RecordStack(new File(args[2]), lens);
|
||||||
final int i = fm.imp(new File(args[1]),";");
|
final int i = fm.imp(new File(args[1]),";");
|
||||||
fm.close();
|
fm.close();
|
||||||
ret = (i + " records imported").getBytes();
|
ret = (i + " records imported").getBytes();
|
||||||
|
@ -356,7 +356,7 @@ public final class Stack extends Records {
|
||||||
if (f != null) try {f.close();}catch(final Exception e) {}
|
if (f != null) try {f.close();}catch(final Exception e) {}
|
||||||
}
|
}
|
||||||
} else if (args[0].equals("-g")) {
|
} else if (args[0].equals("-g")) {
|
||||||
final Stack fm = new Stack(new File(args[2]), lens);
|
final RecordStack fm = new RecordStack(new File(args[2]), lens);
|
||||||
final Row.Entry ret2 = fm.pop();
|
final Row.Entry ret2 = fm.pop();
|
||||||
ret = ((ret2 == null) ? null : ret2.getColBytes(1));
|
ret = ((ret2 == null) ? null : ret2.getColBytes(1));
|
||||||
fm.close();
|
fm.close();
|
||||||
|
@ -366,10 +366,10 @@ public final class Stack extends Records {
|
||||||
// create <keylen> <valuelen> <filename>
|
// create <keylen> <valuelen> <filename>
|
||||||
final File f = new File(args[3]);
|
final File f = new File(args[3]);
|
||||||
if (f.exists()) FileUtils.deletedelete(f);
|
if (f.exists()) FileUtils.deletedelete(f);
|
||||||
final Stack fm = new Stack(f, lens);
|
final RecordStack fm = new RecordStack(f, lens);
|
||||||
fm.close();
|
fm.close();
|
||||||
} else if (args[0].equals("-p")) {
|
} else if (args[0].equals("-p")) {
|
||||||
final Stack fm = new Stack(new File(args[3]), lens);
|
final RecordStack fm = new RecordStack(new File(args[3]), lens);
|
||||||
fm.push(fm.row().newEntry(new byte[][] {args[1].getBytes(), args[2].getBytes()}));
|
fm.push(fm.row().newEntry(new byte[][] {args[1].getBytes(), args[2].getBytes()}));
|
||||||
fm.close();
|
fm.close();
|
||||||
}
|
}
|
|
@ -53,14 +53,14 @@ import java.util.Iterator;
|
||||||
import de.anomic.kelondro.index.Column;
|
import de.anomic.kelondro.index.Column;
|
||||||
import de.anomic.kelondro.index.Row;
|
import de.anomic.kelondro.index.Row;
|
||||||
import de.anomic.kelondro.order.NaturalOrder;
|
import de.anomic.kelondro.order.NaturalOrder;
|
||||||
import de.anomic.kelondro.table.Stack;
|
import de.anomic.kelondro.table.RecordStack;
|
||||||
import de.anomic.kelondro.util.DateFormatter;
|
import de.anomic.kelondro.util.DateFormatter;
|
||||||
import de.anomic.kelondro.util.FileUtils;
|
import de.anomic.kelondro.util.FileUtils;
|
||||||
|
|
||||||
public class yacyNewsQueue {
|
public class yacyNewsQueue {
|
||||||
|
|
||||||
private final File path;
|
private final File path;
|
||||||
Stack queueStack;
|
RecordStack queueStack;
|
||||||
private final yacyNewsDB newsDB;
|
private final yacyNewsDB newsDB;
|
||||||
|
|
||||||
public static final Row rowdef = new Row(new Column[]{
|
public static final Row rowdef = new Row(new Column[]{
|
||||||
|
@ -73,13 +73,13 @@ public class yacyNewsQueue {
|
||||||
public yacyNewsQueue(final File path, final yacyNewsDB newsDB) {
|
public yacyNewsQueue(final File path, final yacyNewsDB newsDB) {
|
||||||
this.path = path;
|
this.path = path;
|
||||||
this.newsDB = newsDB;
|
this.newsDB = newsDB;
|
||||||
this.queueStack = Stack.open(path, rowdef);
|
this.queueStack = RecordStack.open(path, rowdef);
|
||||||
}
|
}
|
||||||
|
|
||||||
private void resetDB() {
|
private void resetDB() {
|
||||||
try {close();} catch (final Exception e) {}
|
try {close();} catch (final Exception e) {}
|
||||||
if (path.exists()) FileUtils.deletedelete(path);
|
if (path.exists()) FileUtils.deletedelete(path);
|
||||||
queueStack = Stack.open(path, rowdef);
|
queueStack = RecordStack.open(path, rowdef);
|
||||||
}
|
}
|
||||||
|
|
||||||
public void clear() {
|
public void clear() {
|
||||||
|
|
Loading…
Reference in New Issue
Block a user