yacy_search_server/source/de/anomic/kelondro/kelondroBytesLongMap.java
orbiter 1e6d12f146 Major update to BLOB data structures:
- introduced a new BLOB file format: kelondroBLOBHeap. This is a flat file with an index in RAM.
  very similar to the eco-tables, but with flexible value sizes. It will replace the kelondroBLOBTree,
  which is based on a kelondroTree, a file-AVL-based index data structure.
- the HTCACHE header file was replaced by the new blob heap file structure
- the robots.txt file was replaced by the new blob heap file structure
- the robots parser was enhanced (bugfixing for double-loading of the same robots.txt)
- other BLOB-dependent data structures were prepared to use also the new BLOB heap
- fixed a bug in the snippet fetch process: the file header was not written to the header index
There should now be less IO during snippet fetch and during crawling


git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@4978 6c8d7289-2bf4-0310-a012-ef5d649a1542
2008-07-10 00:47:37 +00:00

136 lines
4.6 KiB
Java

// kelondroBytesLongMap.java
// (C) 2008 by Michael Peter Christen; mc@yacy.net, Frankfurt a. M., Germany
// first published 08.04.2008 on http://yacy.net
//
// $LastChangedDate: 2006-04-02 22:40:07 +0200 (So, 02 Apr 2006) $
// $LastChangedRevision: 1986 $
// $LastChangedBy: orbiter $
//
// LICENSE
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation; either version 2 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
package de.anomic.kelondro;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Iterator;
public class kelondroBytesLongMap {
private kelondroRow rowdef;
private kelondroIndex index;
public kelondroBytesLongMap(kelondroIndex ki) {
assert (ki.row().columns() == 2); // must be a key/index relation
assert (ki.row().width(1) == 8); // the value must be a b256-encoded int, 4 bytes long
this.index = ki;
this.rowdef = ki.row();
}
public kelondroBytesLongMap(int keylength, kelondroByteOrder objectOrder, int space) {
this.rowdef = new kelondroRow(new kelondroColumn[]{new kelondroColumn("key", kelondroColumn.celltype_binary, kelondroColumn.encoder_bytes, keylength, "key"), new kelondroColumn("long c-8 {b256}")}, objectOrder, 0);
this.index = new kelondroRAMIndex(rowdef, space);
}
public kelondroRow row() {
return index.row();
}
public void clear() throws IOException {
index.clear();
}
public synchronized long getl(byte[] key) throws IOException {
assert (key != null);
kelondroRow.Entry indexentry = index.get(key);
if (indexentry == null) return -1;
return indexentry.getColLong(1);
}
public synchronized long putl(byte[] key, long l) throws IOException {
assert l >= 0 : "l = " + l;
assert (key != null);
kelondroRow.Entry newentry = index.row().newEntry();
newentry.setCol(0, key);
newentry.setCol(1, l);
kelondroRow.Entry oldentry = index.put(newentry);
if (oldentry == null) return -1;
return oldentry.getColLong(1);
}
public synchronized boolean addl(byte[] key, long l) throws IOException {
assert l >= 0 : "l = " + l;
assert (key != null);
kelondroRow.Entry newentry = this.rowdef.newEntry();
newentry.setCol(0, key);
newentry.setCol(1, l);
return index.addUnique(newentry);
}
public synchronized ArrayList<Long[]> removeDoubles() throws IOException {
ArrayList<kelondroRowCollection> indexreport = index.removeDoubles();
ArrayList<Long[]> report = new ArrayList<Long[]>();
Long[] is;
Iterator<kelondroRow.Entry> ei;
int c;
for (kelondroRowCollection rowset: indexreport) {
is = new Long[rowset.size()];
ei = rowset.rows();
c = 0;
while (ei.hasNext()) {
is[c++] = new Long(ei.next().getColLong(1));
}
report.add(is);
}
return report;
}
public synchronized long removel(byte[] key) throws IOException {
assert (key != null);
kelondroRow.Entry indexentry = index.remove(key);
if (indexentry == null) return -1;
return indexentry.getColLong(1);
}
public synchronized long removeonel() throws IOException {
kelondroRow.Entry indexentry = index.removeOne();
if (indexentry == null) return -1;
return indexentry.getColLong(1);
}
public synchronized int size() {
return index.size();
}
public synchronized kelondroCloneableIterator<byte[]> keys(boolean up, byte[] firstKey) throws IOException {
return index.keys(up, firstKey);
}
public synchronized kelondroCloneableIterator<kelondroRow.Entry> rows(boolean up, byte[] firstKey) throws IOException {
return index.rows(up, firstKey);
}
public kelondroProfile profile() {
return index.profile();
}
public synchronized void close() {
index.close();
index = null;
}
}