Marown/yacy_search_server
1
0
Fork 0
mirror of https://github.com/yacy/yacy_search_server.git synced 2024-09-19 00:01:41 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity

bugfixes and performance hacks for tabe index

Browse Source 
git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@7957 6c8d7289-2bf4-0310-a012-ef5d649a1542
This commit is contained in:
orbiter 2011-09-15 11:17:02 +00:00
parent dad5b586a4
commit 2c4a672fe2
5 changed files with 119 additions and 91 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
source/de/anomic/yacy/yacyClient.java
View File

@ -490,7 +490,9 @@ public final class yacyClient {
}
// insert results to containers
int term = count;
for (final URIMetadataRow urlEntry: result.links) {
if (term-- <= 0) break; // do not process more that requested (in case that evil peers fill us up with rubbish)
// get one single search result
if (urlEntry == null) continue;
assert (urlEntry.hash().length == 12) : "urlEntry.hash() = " + ASCII.String(urlEntry.hash());

6
source/net/yacy/kelondro/index/Row.java
View File

@ -379,7 +379,7 @@ public final class Row {
}
public final byte[] bytes() {
if ((this.offset == 0) && (this.rowinstance.length == Row.this.objectsize)) {
if (this.offset == 0 && this.rowinstance.length == Row.this.objectsize) {
return this.rowinstance;
}
final byte[] tmp = new byte[Row.this.objectsize];
@ -518,6 +518,10 @@ public final class Row {
}
public final byte[] getPrimaryKeyBytes() {
if (Row.this.columns() == 1 && this.offset == 0 && this.rowinstance.length == Row.this.primaryKeyLength) {
// avoid memory allocation in case that the row consists in only the primary key
return this.rowinstance;
}
final byte[] c = new byte[Row.this.primaryKeyLength];
System.arraycopy(this.rowinstance, this.offset, c, 0, Row.this.primaryKeyLength);
return c;

48
source/net/yacy/kelondro/index/RowSet.java
View File

@ -158,10 +158,12 @@ public class RowSet extends RowCollection implements Index, Iterable<Row.Entry>
*/
public final boolean put(final Row.Entry entry) throws RowSpaceExceededException {
assert (entry != null);
assert (entry.getPrimaryKeyBytes() != null);
final byte[] key = entry.getPrimaryKeyBytes();
assert (key != null);
final byte[] entrybytes = entry.bytes();
assert entrybytes.length >= this.rowdef.primaryKeyLength;
synchronized (this) {
assert entry.bytes().length >= this.rowdef.primaryKeyLength;
final int index = find(entry.bytes(), 0);
final int index = find(key, 0);
if (index < 0) {
super.addUnique(entry);
return true;
@ -174,26 +176,30 @@ public class RowSet extends RowCollection implements Index, Iterable<Row.Entry>
}
}
public final synchronized Row.Entry replace(final Row.Entry entry) throws RowSpaceExceededException {
public final Row.Entry replace(final Row.Entry entry) throws RowSpaceExceededException {
assert (entry != null);
assert (entry.getPrimaryKeyBytes() != null);
int index = -1;
Row.Entry oldentry = null;
// when reaching a specific amount of un-sorted entries, re-sort all
if ((this.chunkcount - this.sortBound) > collectionReSortLimit) {
sort();
final byte[] key = entry.getPrimaryKeyBytes();
assert (key != null);
final byte[] entrybytes = entry.bytes();
assert entrybytes.length >= this.rowdef.primaryKeyLength;
synchronized (this) {
int index = -1;
Row.Entry oldentry = null;
// when reaching a specific amount of un-sorted entries, re-sort all
if ((this.chunkcount - this.sortBound) > collectionReSortLimit) {
sort();
}
index = find(key, 0);
if (index < 0) {
super.addUnique(entry);
} else {
oldentry = get(index, true);
final int sb = this.sortBound; // save the sortBound, because it is not altered (we replace at the same place)
set(index, entry); // this may alter the sortBound, which we will revert in the next step
this.sortBound = sb; // revert a sortBound altering
}
return oldentry;
}
assert entry.bytes().length >= this.rowdef.primaryKeyLength;
index = find(entry.bytes(), 0);
if (index < 0) {
super.addUnique(entry);
} else {
oldentry = get(index, true);
final int sb = this.sortBound; // save the sortBound, because it is not altered (we replace at the same place)
set(index, entry); // this may alter the sortBound, which we will revert in the next step
this.sortBound = sb; // revert a sortBound altering
}
return oldentry;
}
public final synchronized long inc(final byte[] key, final int col, final long add, final Row.Entry initrow) throws RowSpaceExceededException {

7
source/net/yacy/kelondro/table/SplitTable.java
View File

@ -388,8 +388,13 @@ public class SplitTable implements Index, Iterable<Row.Entry> {
assert row.objectsize() <= this.rowdef.objectsize;
final byte[] key = row.getPrimaryKeyBytes();
if (this.tables == null) return true;
Index keeper = null;
synchronized (this.tables) {
Index keeper = keeperOf(key);
keeper = keeperOf(key);
}
if (keeper != null) return keeper.put(row);
synchronized (this.tables) {
keeper = keeperOf(key); // we must check that again because it could have changed in between
if (keeper != null) return keeper.put(row);
assert this.current == null || this.tables.get(this.current) != null : "this.current = " + this.current;
keeper = (this.current == null) ? newTable() : checkTable(this.tables.get(this.current));

147
source/net/yacy/kelondro/table/Table.java
View File

@ -355,16 +355,17 @@ public class Table implements Index, Iterable<Row.Entry> {
// try again with less memory
this.index.putUnique(row.getPrimaryKeyBytes(), i);
}
final byte[] rowbytes = row.bytes();
if (this.table != null) {
assert this.table.size() == i;
try {
this.table.addUnique(this.taildef.newEntry(row.bytes(), this.rowdef.primaryKeyLength, true));
this.table.addUnique(this.taildef.newEntry(rowbytes, this.rowdef.primaryKeyLength, true));
} catch (final RowSpaceExceededException e) {
this.table = null;
}
if (abandonTable()) this.table = null;
}
this.file.add(row.bytes(), 0);
this.file.add(rowbytes, 0);
assert this.file.size() == this.index.size() : "file.size() = " + this.file.size() + ", index.size() = " + this.index.size();
}
@ -511,49 +512,54 @@ public class Table implements Index, Iterable<Row.Entry> {
return this.index.keys(up, firstKey);
}
public synchronized Entry replace(final Entry row) throws IOException, RowSpaceExceededException {
assert this.file.size() == this.index.size() : "file.size() = " + this.file.size() + ", index.size() = " + this.index.size();
assert this.table == null || this.table.size() == this.index.size() : "table.size() = " + this.table.size() + ", index.size() = " + this.index.size();
public Entry replace(final Entry row) throws IOException, RowSpaceExceededException {
assert row != null;
assert row.bytes() != null;
if (row == null || row.bytes() == null) return null;
final int i = (int) this.index.get(row.getPrimaryKeyBytes());
if (i == -1) {
try {
addUnique(row);
} catch (final RowSpaceExceededException e) {
if (this.table == null) throw e;
this.table = null;
addUnique(row);
if (this.file == null || row == null) return null;
final byte[] rowb = row.bytes();
assert rowb != null;
if (rowb == null) return null;
final byte[] key = row.getPrimaryKeyBytes();
synchronized (this) {
assert this.file.size() == this.index.size() : "file.size() = " + this.file.size() + ", index.size() = " + this.index.size();
assert this.table == null || this.table.size() == this.index.size() : "table.size() = " + this.table.size() + ", index.size() = " + this.index.size();
final int i = (int) this.index.get(key);
if (i == -1) {
try {
addUnique(row);
} catch (final RowSpaceExceededException e) {
if (this.table == null) throw e;
this.table = null;
addUnique(row);
}
return null;
}
return null;
}
final byte[] b = new byte[this.rowdef.objectsize];
Row.Entry cacherow;
if (this.table == null || (cacherow = this.table.get(i, false)) == null) {
// read old value
this.file.get(i, b, 0);
// write new value
this.file.put(i, row.bytes(), 0);
} else {
// read old value
assert cacherow != null;
System.arraycopy(row.getPrimaryKeyBytes(), 0, b, 0, this.rowdef.primaryKeyLength);
System.arraycopy(cacherow.bytes(), 0, b, this.rowdef.primaryKeyLength, this.rowdef.objectsize - this.rowdef.primaryKeyLength);
// write new value
try {
this.table.set(i, this.taildef.newEntry(row.bytes(), this.rowdef.primaryKeyLength, true));
} catch (final RowSpaceExceededException e) {
this.table = null;
final byte[] b = new byte[this.rowdef.objectsize];
Row.Entry cacherow;
if (this.table == null || (cacherow = this.table.get(i, false)) == null) {
// read old value
this.file.get(i, b, 0);
// write new value
this.file.put(i, rowb, 0);
} else {
// read old value
assert cacherow != null;
System.arraycopy(key, 0, b, 0, this.rowdef.primaryKeyLength);
System.arraycopy(cacherow.bytes(), 0, b, this.rowdef.primaryKeyLength, this.rowdef.objectsize - this.rowdef.primaryKeyLength);
// write new value
try {
this.table.set(i, this.taildef.newEntry(rowb, this.rowdef.primaryKeyLength, true));
} catch (final RowSpaceExceededException e) {
this.table = null;
}
if (abandonTable()) this.table = null;
this.file.put(i, rowb, 0);
}
if (abandonTable()) this.table = null;
this.file.put(i, row.bytes(), 0);
assert this.file.size() == this.index.size() : "file.size() = " + this.file.size() + ", index.size() = " + this.index.size();
assert this.table == null || this.table.size() == this.index.size() : "table.size() = " + this.table.size() + ", index.size() = " + this.index.size();
// return old value
return this.rowdef.newEntry(b);
}
assert this.file.size() == this.index.size() : "file.size() = " + this.file.size() + ", index.size() = " + this.index.size();
assert this.table == null || this.table.size() == this.index.size() : "table.size() = " + this.table.size() + ", index.size() = " + this.index.size();
// return old value
return this.rowdef.newEntry(b);
}
/**
@ -563,39 +569,44 @@ public class Table implements Index, Iterable<Row.Entry> {
* @throws IOException
* @throws RowSpaceExceededException
*/
public synchronized boolean put(final Entry row) throws IOException, RowSpaceExceededException {
assert this.file == null || this.file.size() == this.index.size() : "file.size() = " + this.file.size() + ", index.size() = " + this.index.size() + ", file = " + filename();
assert this.table == null || this.table.size() == this.index.size() : "table.size() = " + this.table.size() + ", index.size() = " + this.index.size() + ", file = " + filename();
public boolean put(final Entry row) throws IOException, RowSpaceExceededException {
assert row != null;
assert row.bytes() != null;
if (this.file == null || row == null || row.bytes() == null) return true;
final int i = (int) this.index.get(row.getPrimaryKeyBytes());
if (i == -1) {
try {
addUnique(row);
} catch (final RowSpaceExceededException e) {
if (this.table == null) throw e;
this.table = null;
addUnique(row);
if (this.file == null || row == null) return true;
final byte[] rowb = row.bytes();
assert rowb != null;
if (rowb == null) return true;
final byte[] key = row.getPrimaryKeyBytes();
synchronized (this) {
assert this.file == null || this.file.size() == this.index.size() : "file.size() = " + this.file.size() + ", index.size() = " + this.index.size() + ", file = " + filename();
assert this.table == null || this.table.size() == this.index.size() : "table.size() = " + this.table.size() + ", index.size() = " + this.index.size() + ", file = " + filename();
final int i = (int) this.index.get(key);
if (i == -1) {
try {
addUnique(row);
} catch (final RowSpaceExceededException e) {
if (this.table == null) throw e;
this.table = null;
addUnique(row);
}
return true;
}
return true;
}
if (this.table == null) {
// write new value
this.file.put(i, row.bytes(), 0);
} else {
// write new value
this.file.put(i, row.bytes(), 0);
if (abandonTable()) this.table = null; else try {
this.table.set(i, this.taildef.newEntry(row.bytes(), this.rowdef.primaryKeyLength, true));
} catch (final RowSpaceExceededException e) {
this.table = null;
if (this.table == null) {
// write new value
this.file.put(i, rowb, 0);
} else {
// write new value
this.file.put(i, rowb, 0);
if (abandonTable()) this.table = null; else try {
this.table.set(i, this.taildef.newEntry(rowb, this.rowdef.primaryKeyLength, true));
} catch (final RowSpaceExceededException e) {
this.table = null;
}
}
assert this.file.size() == this.index.size() : "file.size() = " + this.file.size() + ", index.size() = " + this.index.size();
assert this.table == null || this.table.size() == this.index.size() : "table.size() = " + this.table.size() + ", index.size() = " + this.index.size();
return false;
}
assert this.file.size() == this.index.size() : "file.size() = " + this.file.size() + ", index.size() = " + this.index.size();
assert this.table == null || this.table.size() == this.index.size() : "table.size() = " + this.table.size() + ", index.size() = " + this.index.size();
return false;
}
public Entry put(final Entry row, final Date entryDate) throws IOException, RowSpaceExceededException {