mirror of
https://github.com/yacy/yacy_search_server.git
synced 2024-09-19 00:01:41 +02:00
some more enhancements to R/W Head path optimization
git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@3415 6c8d7289-2bf4-0310-a012-ef5d649a1542
This commit is contained in:
parent
f4cfd19835
commit
10a3c20b8d
|
@ -571,6 +571,10 @@ final class dbTable implements kelondroIndex {
|
|||
throw new UnsupportedOperationException();
|
||||
}
|
||||
|
||||
public synchronized void addUniqueMultiple(List rows, Date entryDate) throws IOException {
|
||||
throw new UnsupportedOperationException();
|
||||
}
|
||||
|
||||
public kelondroRow.Entry remove(byte[] key) throws IOException {
|
||||
try {
|
||||
|
||||
|
|
|
@ -540,6 +540,11 @@ public class kelondroCache implements kelondroIndex {
|
|||
}
|
||||
}
|
||||
|
||||
public synchronized void addUniqueMultiple(List rows, Date entryDate) throws IOException {
|
||||
Iterator i = rows.iterator();
|
||||
while (i.hasNext()) addUnique((Entry) i.next(), entryDate);
|
||||
}
|
||||
|
||||
public synchronized Entry remove(byte[] key) throws IOException {
|
||||
|
||||
checkMissSpace();
|
||||
|
|
|
@ -278,11 +278,23 @@ public class kelondroCollectionIndex {
|
|||
return 2 * m * this.payloadrow.objectsize;
|
||||
}
|
||||
|
||||
private kelondroRow.Entry array_new(byte[] key, kelondroRowCollection collection) throws IOException {
|
||||
private void array_remove(
|
||||
int oldPartitionNumber, int serialNumber, int chunkSize,
|
||||
int oldRownumber) throws IOException {
|
||||
// we need a new slot, that means we must first delete the old entry
|
||||
// find array file
|
||||
kelondroFixedWidthArray array = getArray(oldPartitionNumber, serialNumber, chunkSize);
|
||||
|
||||
// delete old entry
|
||||
array.remove(oldRownumber, true);
|
||||
}
|
||||
|
||||
private kelondroRow.Entry array_new(
|
||||
byte[] key, kelondroRowCollection collection) throws IOException {
|
||||
// the collection is new
|
||||
int newPartitionNumber = arrayIndex(collection.size());
|
||||
int partitionNumber = arrayIndex(collection.size());
|
||||
kelondroRow.Entry indexrow = index.row().newEntry();
|
||||
kelondroFixedWidthArray array = getArray(newPartitionNumber, serialNumber, this.payloadrow.objectsize());
|
||||
kelondroFixedWidthArray array = getArray(partitionNumber, serialNumber, this.payloadrow.objectsize());
|
||||
|
||||
// define row
|
||||
kelondroRow.Entry arrayEntry = array.row().newEntry();
|
||||
|
@ -296,7 +308,7 @@ public class kelondroCollectionIndex {
|
|||
indexrow.setCol(idx_col_key, key);
|
||||
indexrow.setCol(idx_col_chunksize, this.payloadrow.objectsize());
|
||||
indexrow.setCol(idx_col_chunkcount, collection.size());
|
||||
indexrow.setCol(idx_col_clusteridx, (byte) newPartitionNumber);
|
||||
indexrow.setCol(idx_col_clusteridx, (byte) partitionNumber);
|
||||
indexrow.setCol(idx_col_flags, (byte) 0);
|
||||
indexrow.setCol(idx_col_indexpos, (long) newRowNumber);
|
||||
indexrow.setCol(idx_col_lastread, kelondroRowCollection.daysSince2000(System.currentTimeMillis()));
|
||||
|
@ -306,17 +318,6 @@ public class kelondroCollectionIndex {
|
|||
return indexrow;
|
||||
}
|
||||
|
||||
private void array_remove(
|
||||
int oldPartitionNumber, int serialNumber, int chunkSize,
|
||||
int oldRownumber) throws IOException {
|
||||
// we need a new slot, that means we must first delete the old entry
|
||||
// find array file
|
||||
kelondroFixedWidthArray array = getArray(oldPartitionNumber, serialNumber, chunkSize);
|
||||
|
||||
// delete old entry
|
||||
array.remove(oldRownumber, true);
|
||||
}
|
||||
|
||||
private void array_add(
|
||||
byte[] key, kelondroRowCollection collection, kelondroRow.Entry indexrow,
|
||||
int partitionNumber, int serialNumber, int chunkSize) throws IOException {
|
||||
|
@ -450,7 +451,7 @@ public class kelondroCollectionIndex {
|
|||
// merge existing containers
|
||||
i = existingContainer.iterator();
|
||||
Object[] record;
|
||||
ArrayList indexrows = new ArrayList();
|
||||
ArrayList indexrows_existing = new ArrayList();
|
||||
kelondroRowCollection collection;
|
||||
while (i.hasNext()) {
|
||||
record = (Object[]) i.next(); // {byte[], indexContainer, kelondroRow.Entry}
|
||||
|
@ -495,21 +496,23 @@ public class kelondroCollectionIndex {
|
|||
newPartitionNumber, oldSerialNumber, this.payloadrow.objectsize()); // modifies indexrow
|
||||
}
|
||||
arrayResolveRemoved(); // remove all to-be-removed marked entries
|
||||
indexrows.add(indexrow); // indexrows are collected and written later as block
|
||||
indexrows_existing.add(indexrow); // indexrows are collected and written later as block
|
||||
}
|
||||
|
||||
// write new containers
|
||||
i = newContainer.iterator();
|
||||
ArrayList indexrows_new = new ArrayList();
|
||||
while (i.hasNext()) {
|
||||
record = (Object[]) i.next(); // {byte[], indexContainer}
|
||||
key = (byte[]) record[0];
|
||||
collection = (indexContainer) record[1];
|
||||
indexrow = array_new(key, collection); // modifies indexrow
|
||||
index.addUnique(indexrow); // write modified indexrow
|
||||
indexrows_new.add(indexrow); // collect new index rows
|
||||
}
|
||||
|
||||
// write index entries
|
||||
index.putMultiple(indexrows, new Date()); // write modified indexrows in optimized manner
|
||||
index.putMultiple(indexrows_existing, new Date()); // write modified indexrows in optimized manner
|
||||
index.addUniqueMultiple(indexrows_new, new Date()); // write new indexrows in optimized manner
|
||||
}
|
||||
|
||||
public synchronized void merge(indexContainer container) throws IOException, kelondroOutOfLimitsException {
|
||||
|
|
|
@ -49,6 +49,8 @@ package de.anomic.kelondro;
|
|||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.util.Iterator;
|
||||
import java.util.Map;
|
||||
import java.util.TreeMap;
|
||||
import java.util.TreeSet;
|
||||
|
||||
public class kelondroFixedWidthArray extends kelondroRecords implements kelondroArray {
|
||||
|
@ -114,6 +116,15 @@ public class kelondroFixedWidthArray extends kelondroRecords implements kelondro
|
|||
// the OHbytes and OHhandles are zero.
|
||||
}
|
||||
|
||||
public synchronized void setMultiple(TreeMap /* of Integer/kelondroRow.Entry */ rows) throws IOException {
|
||||
Iterator i = rows.entrySet().iterator();
|
||||
Map.Entry entry;
|
||||
while (i.hasNext()) {
|
||||
entry = (Map.Entry) i.next();
|
||||
set(((Integer) entry.getKey()).intValue(), (kelondroRow.Entry) entry.getValue());
|
||||
}
|
||||
}
|
||||
|
||||
public synchronized kelondroRow.Entry get(int index) throws IOException {
|
||||
return row().newEntry(getNode(new Handle(index), true).getValueRow());
|
||||
}
|
||||
|
|
|
@ -239,6 +239,11 @@ public class kelondroFlexSplitTable implements kelondroIndex {
|
|||
table.addUnique(row, entryDate);
|
||||
}
|
||||
|
||||
public synchronized void addUniqueMultiple(List rows, Date entryDate) throws IOException {
|
||||
Iterator i = rows.iterator();
|
||||
while (i.hasNext()) addUnique((kelondroRow.Entry) i.next(), entryDate);
|
||||
}
|
||||
|
||||
public synchronized kelondroRow.Entry remove(byte[] key) throws IOException {
|
||||
Iterator i = tables.values().iterator();
|
||||
kelondroIndex table;
|
||||
|
|
|
@ -27,9 +27,11 @@ package de.anomic.kelondro;
|
|||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Date;
|
||||
import java.util.Iterator;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.TreeMap;
|
||||
|
||||
import de.anomic.server.logging.serverLog;
|
||||
|
@ -187,20 +189,25 @@ public class kelondroFlexTable extends kelondroFlexWidthArray implements kelondr
|
|||
// this should save R/W head positioning time
|
||||
Iterator i = rows.iterator();
|
||||
kelondroRow.Entry row;
|
||||
TreeMap ordered = new TreeMap();
|
||||
int pos;
|
||||
byte[] key;
|
||||
TreeMap old_rows_ordered = new TreeMap();
|
||||
ArrayList new_rows_sequential = new ArrayList();
|
||||
while (i.hasNext()) {
|
||||
row = (kelondroRow.Entry) i.next();
|
||||
key = row.getColBytes(0);
|
||||
pos = index.geti(key);
|
||||
if (pos < 0) {
|
||||
index.puti(key, super.add(row));
|
||||
new_rows_sequential.add(row);
|
||||
} else {
|
||||
ordered.put(new Integer(pos), row);
|
||||
old_rows_ordered.put(new Integer(pos), row);
|
||||
}
|
||||
}
|
||||
super.setMultiple(ordered);
|
||||
// overwrite existing entries in index
|
||||
super.setMultiple(old_rows_ordered);
|
||||
|
||||
// write new entries to index
|
||||
addUniqueMultiple(new_rows_sequential, entryDate);
|
||||
}
|
||||
|
||||
public synchronized kelondroRow.Entry put(kelondroRow.Entry row, Date entryDate) throws IOException {
|
||||
|
@ -230,6 +237,20 @@ public class kelondroFlexTable extends kelondroFlexWidthArray implements kelondr
|
|||
index.addi(row.getColBytes(0), super.add(row));
|
||||
}
|
||||
|
||||
public synchronized void addUniqueMultiple(List rows, Date entryDate) throws IOException {
|
||||
// add a list of entries in a ordered way.
|
||||
// this should save R/W head positioning time
|
||||
TreeMap indexed_result = super.addMultiple(rows);
|
||||
// indexed_result is a Integer/byte[] relation
|
||||
// that is used here to store the index
|
||||
Iterator i = indexed_result.entrySet().iterator();
|
||||
Map.Entry entry;
|
||||
while (i.hasNext()) {
|
||||
entry = (Map.Entry) i.next();
|
||||
index.puti((byte[]) entry.getValue(), ((Integer) entry.getKey()).intValue());
|
||||
}
|
||||
}
|
||||
|
||||
public synchronized kelondroRow.Entry remove(byte[] key) throws IOException {
|
||||
int i = index.removei(key);
|
||||
if (i < 0) return null;
|
||||
|
|
|
@ -29,6 +29,7 @@ import java.io.File;
|
|||
import java.io.IOException;
|
||||
import java.util.HashMap;
|
||||
import java.util.Iterator;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.TreeMap;
|
||||
|
||||
|
@ -188,7 +189,7 @@ public class kelondroFlexWidthArray implements kelondroArray {
|
|||
// a R/W head path-optimized option to write a set of entries
|
||||
Iterator i;
|
||||
Map.Entry entry;
|
||||
kelondroRow.Entry rowentry, e0;
|
||||
kelondroRow.Entry rowentry, e;
|
||||
int c = 0, index, lastcol;
|
||||
synchronized (col) {
|
||||
// go across each file
|
||||
|
@ -201,11 +202,11 @@ public class kelondroFlexWidthArray implements kelondroArray {
|
|||
rowentry = (kelondroRow.Entry) entry.getValue();
|
||||
assert rowentry.bytes().length == this.rowdef.objectsize;
|
||||
|
||||
e0 = col[c].row().newEntry(
|
||||
e = col[c].row().newEntry(
|
||||
rowentry.bytes(),
|
||||
rowdef.colstart[c],
|
||||
rowdef.colstart[lastcol] - rowdef.colstart[c] + rowdef.width(lastcol));
|
||||
col[c].set(index, e0);
|
||||
col[c].set(index, e);
|
||||
}
|
||||
c = c + col[c].row().columns();
|
||||
}
|
||||
|
@ -215,16 +216,16 @@ public class kelondroFlexWidthArray implements kelondroArray {
|
|||
public void set(int index, kelondroRow.Entry rowentry) throws IOException {
|
||||
assert rowentry.bytes().length == this.rowdef.objectsize;
|
||||
int c = 0;
|
||||
kelondroRow.Entry e0;
|
||||
kelondroRow.Entry e;
|
||||
int lastcol;
|
||||
synchronized (col) {
|
||||
while (c < rowdef.columns()) {
|
||||
lastcol = c + col[c].row().columns() - 1;
|
||||
e0 = col[c].row().newEntry(
|
||||
e = col[c].row().newEntry(
|
||||
rowentry.bytes(),
|
||||
rowdef.colstart[c],
|
||||
rowdef.colstart[lastcol] - rowdef.colstart[c] + rowdef.width(lastcol));
|
||||
col[c].set(index, e0);
|
||||
col[c].set(index, e);
|
||||
c = c + col[c].row().columns();
|
||||
}
|
||||
}
|
||||
|
@ -253,6 +254,51 @@ public class kelondroFlexWidthArray implements kelondroArray {
|
|||
return index;
|
||||
}
|
||||
|
||||
protected TreeMap addMultiple(List rows) throws IOException {
|
||||
// result is a Integer/byte[] relation
|
||||
// of newly added rows (index, key)
|
||||
TreeMap indexref = new TreeMap();
|
||||
Iterator i;
|
||||
kelondroRow.Entry rowentry;
|
||||
// prepare storage for other columns
|
||||
TreeMap[] colm = new TreeMap[col.length];
|
||||
for (int j = 0; j < col.length; j++) {
|
||||
if (col[j] == null) colm[j] = null; else colm[j] = new TreeMap();
|
||||
}
|
||||
i = rows.iterator();
|
||||
while (i.hasNext()) {
|
||||
rowentry = (kelondroRow.Entry) i.next();
|
||||
assert rowentry.bytes().length == this.rowdef.objectsize;
|
||||
|
||||
kelondroRow.Entry e;
|
||||
int index = -1;
|
||||
int lastcol;
|
||||
synchronized (col) {
|
||||
e = col[0].row().newEntry(rowentry.bytes(), 0, rowdef.width(0));
|
||||
index = col[0].add(e);
|
||||
int c = col[0].row().columns();
|
||||
|
||||
while (c < rowdef.columns()) {
|
||||
lastcol = c + col[c].row().columns() - 1;
|
||||
e = col[c].row().newEntry(
|
||||
rowentry.bytes(),
|
||||
rowdef.colstart[c],
|
||||
rowdef.colstart[lastcol] + rowdef.width(lastcol) - rowdef.colstart[c]);
|
||||
// remember write to column, but do not write directly
|
||||
colm[c].put(new Integer(index), e); // col[c].set(index,e);
|
||||
c = c + col[c].row().columns();
|
||||
}
|
||||
}
|
||||
indexref.put(new Integer(index), rowentry.getColBytes(0));
|
||||
}
|
||||
// write the other columns
|
||||
for (int j = 1; j < col.length; j++) {
|
||||
if (col[j] != null) col[j].setMultiple(colm[j]);
|
||||
}
|
||||
// retrun references to entries with key
|
||||
return indexref;
|
||||
}
|
||||
|
||||
public kelondroRow.Entry get(int index) throws IOException {
|
||||
int r = 0;
|
||||
kelondroRow.Entry e, p;
|
||||
|
|
|
@ -67,6 +67,7 @@ public interface kelondroIndex {
|
|||
public void putMultiple(List /* of kelondroRow.Entry*/ rows, Date entryDate) throws IOException; // for R/W head path optimization
|
||||
public void addUnique(kelondroRow.Entry row) throws IOException; // no double-check
|
||||
public void addUnique(kelondroRow.Entry row, Date entryDate) throws IOException; // no double-check
|
||||
public void addUniqueMultiple(List /* of kelondroRow.Entry*/ rows, Date entryDate) throws IOException; // no double-check
|
||||
public kelondroRow.Entry remove(byte[] key) throws IOException;
|
||||
public kelondroRow.Entry removeOne() throws IOException;
|
||||
public Iterator rows(boolean up, boolean rotating, byte[] firstKey) throws IOException;
|
||||
|
|
|
@ -24,8 +24,10 @@
|
|||
|
||||
package de.anomic.kelondro;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.Date;
|
||||
import java.util.Iterator;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
|
||||
import de.anomic.server.logging.serverLog;
|
||||
|
@ -216,6 +218,11 @@ public class kelondroRowCollection {
|
|||
addUnique(row);
|
||||
}
|
||||
|
||||
public synchronized void addUniqueMultiple(List rows, Date entryDate) throws IOException {
|
||||
Iterator i = rows.iterator();
|
||||
while (i.hasNext()) addUnique((kelondroRow.Entry) i.next());
|
||||
}
|
||||
|
||||
public void add(byte[] a) {
|
||||
addUnique(a, 0, a.length);
|
||||
}
|
||||
|
|
|
@ -31,7 +31,6 @@ import java.util.List;
|
|||
import java.util.Random;
|
||||
import java.util.TreeMap;
|
||||
|
||||
import de.anomic.kelondro.kelondroRow.Entry;
|
||||
import de.anomic.server.logging.serverLog;
|
||||
|
||||
public class kelondroRowSet extends kelondroRowCollection implements kelondroIndex {
|
||||
|
@ -102,9 +101,18 @@ public class kelondroRowSet extends kelondroRowCollection implements kelondroInd
|
|||
}
|
||||
}
|
||||
|
||||
public void addUniqueMultiple(List rows, Date entryDate) throws IOException {
|
||||
if (removeMarker.size() == 0) {
|
||||
super.addUniqueMultiple(rows, entryDate);
|
||||
} else {
|
||||
Iterator i = rows.iterator();
|
||||
while (i.hasNext()) addUnique((kelondroRow.Entry) i.next(), entryDate);
|
||||
}
|
||||
}
|
||||
|
||||
public synchronized void putMultiple(List rows, Date entryDate) throws IOException {
|
||||
Iterator i = rows.iterator();
|
||||
while (i.hasNext()) put ((Entry) i.next(), entryDate);
|
||||
while (i.hasNext()) put((kelondroRow.Entry) i.next(), entryDate);
|
||||
}
|
||||
|
||||
public kelondroRow.Entry put(kelondroRow.Entry row, Date entryDate) {
|
||||
|
|
|
@ -139,6 +139,10 @@ public class kelondroSplittedTree implements kelondroIndex {
|
|||
throw new UnsupportedOperationException();
|
||||
}
|
||||
|
||||
public synchronized void addUniqueMultiple(List rows, Date entryDate) throws IOException {
|
||||
throw new UnsupportedOperationException();
|
||||
}
|
||||
|
||||
public kelondroRow.Entry put(kelondroRow.Entry row) throws IOException {
|
||||
return ktfs[partition(row.getColBytes(0))].put(row);
|
||||
}
|
||||
|
@ -274,4 +278,5 @@ public class kelondroSplittedTree implements kelondroIndex {
|
|||
// a collection of different node cache status values
|
||||
return new int[]{0,0,0,0,0,0,0,0,0,0};
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
@ -486,6 +486,11 @@ public class kelondroTree extends kelondroRecords implements kelondroIndex {
|
|||
this.put(row, entryDate);
|
||||
}
|
||||
|
||||
public synchronized void addUniqueMultiple(List rows, Date entryDate) throws IOException {
|
||||
Iterator i = rows.iterator();
|
||||
while (i.hasNext()) addUnique((kelondroRow.Entry) i.next(), entryDate);
|
||||
}
|
||||
|
||||
private void assignChild(Node parentNode, Node childNode, int childType) throws IOException {
|
||||
parentNode.setOHHandle(childType, childNode.handle());
|
||||
childNode.setOHHandle(parent, parentNode.handle());
|
||||
|
|
|
@ -1018,6 +1018,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser
|
|||
|
||||
int wordInCacheMaxCount = (int) getConfigLong(INDEX_DIST_DHT_RECEIPT_LIMIT, 1000);
|
||||
wordIndex.setInMaxWordCount(wordInCacheMaxCount);
|
||||
wordIndex.setWordFlushSize((int) getConfigLong("wordFlushSize", 1000));
|
||||
|
||||
// set a minimum amount of memory for the indexer thread
|
||||
setConfig(INDEXER_MEMPREREQ, Math.max(getConfigLong(INDEXER_MEMPREREQ, 0), wordIndex.minMem()));
|
||||
|
|
Loading…
Reference in New Issue
Block a user