mirror of
https://github.com/yacy/yacy_search_server.git
synced 2024-09-21 00:00:13 +02:00
turned back NURL usage of kelondroTree, kelondroFlexTable has still problems with deleted entries
git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@2454 6c8d7289-2bf4-0310-a012-ef5d649a1542
This commit is contained in:
parent
db1eae0227
commit
c26da4893b
|
@ -176,6 +176,7 @@ public class kelondroRecords {
|
||||||
entryFile.writeInt(POS_USEDC, USEDC);
|
entryFile.writeInt(POS_USEDC, USEDC);
|
||||||
entryFile.writeInt(POS_FREEC, FREEC);
|
entryFile.writeInt(POS_FREEC, FREEC);
|
||||||
entryFile.writeInt(POS_FREEH, FREEH.index);
|
entryFile.writeInt(POS_FREEH, FREEH.index);
|
||||||
|
entryFile.commit();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -777,6 +778,8 @@ public class kelondroRecords {
|
||||||
entryFile.write(seekpos(this.handle) + headchunksize, this.tailChunk);
|
entryFile.write(seekpos(this.handle) + headchunksize, this.tailChunk);
|
||||||
this.tailChanged = false;
|
this.tailChanged = false;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
entryFile.commit();
|
||||||
}
|
}
|
||||||
|
|
||||||
public synchronized void collapse() {
|
public synchronized void collapse() {
|
||||||
|
|
|
@ -55,10 +55,10 @@ import java.util.Iterator;
|
||||||
import de.anomic.index.indexURL;
|
import de.anomic.index.indexURL;
|
||||||
import de.anomic.kelondro.kelondroBase64Order;
|
import de.anomic.kelondro.kelondroBase64Order;
|
||||||
import de.anomic.kelondro.kelondroException;
|
import de.anomic.kelondro.kelondroException;
|
||||||
import de.anomic.kelondro.kelondroFlexTable;
|
|
||||||
import de.anomic.kelondro.kelondroRecords;
|
import de.anomic.kelondro.kelondroRecords;
|
||||||
import de.anomic.kelondro.kelondroStack;
|
import de.anomic.kelondro.kelondroStack;
|
||||||
import de.anomic.kelondro.kelondroRow;
|
import de.anomic.kelondro.kelondroRow;
|
||||||
|
import de.anomic.kelondro.kelondroTree;
|
||||||
import de.anomic.server.logging.serverLog;
|
import de.anomic.server.logging.serverLog;
|
||||||
import de.anomic.tools.bitfield;
|
import de.anomic.tools.bitfield;
|
||||||
|
|
||||||
|
@ -147,6 +147,7 @@ public class plasmaCrawlNURL extends indexURL {
|
||||||
}
|
}
|
||||||
|
|
||||||
private void openHashCache() {
|
private void openHashCache() {
|
||||||
|
/*
|
||||||
String newCacheName = "urlNotice3.table";
|
String newCacheName = "urlNotice3.table";
|
||||||
cacheStacksPath.mkdirs();
|
cacheStacksPath.mkdirs();
|
||||||
try {
|
try {
|
||||||
|
@ -155,22 +156,12 @@ public class plasmaCrawlNURL extends indexURL {
|
||||||
e.printStackTrace();
|
e.printStackTrace();
|
||||||
System.exit(-1);
|
System.exit(-1);
|
||||||
}
|
}
|
||||||
/*
|
*/
|
||||||
|
|
||||||
File oldCacheFile = new File(cacheStacksPath, "urlNotice1.db");
|
File oldCacheFile = new File(cacheStacksPath, "urlNotice1.db");
|
||||||
if (oldCacheFile.exists()) try {
|
oldCacheFile.getParentFile().mkdirs();
|
||||||
// open existing cache
|
urlHashCache = kelondroTree.open(oldCacheFile, bufferkb * 0x400, preloadTime, kelondroTree.defaultObjectCachePercent, rowdef);
|
||||||
kelondroTree tree = new kelondroTree(oldCacheFile, bufferkb * 0x400, preloadTime, kelondroTree.defaultObjectCachePercent);
|
|
||||||
tree.assignRowdef(rowdef);
|
|
||||||
urlHashCache = tree;
|
|
||||||
} catch (IOException e) {
|
|
||||||
oldCacheFile.delete();
|
|
||||||
urlHashCache = new kelondroTree(oldCacheFile, bufferkb * 0x400, preloadTime, kelondroTree.defaultObjectCachePercent, rowdef, true);
|
|
||||||
} else {
|
|
||||||
// create new cache
|
|
||||||
oldCacheFile.getParentFile().mkdirs();
|
|
||||||
urlHashCache = new kelondroTree(oldCacheFile, bufferkb * 0x400, preloadTime, kelondroTree.defaultObjectCachePercent, rowdef, true);
|
|
||||||
}
|
|
||||||
*/
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private void resetHashCache() {
|
private void resetHashCache() {
|
||||||
|
@ -387,7 +378,9 @@ public class plasmaCrawlNURL extends indexURL {
|
||||||
private Entry pop(plasmaCrawlBalancer balancer) throws IOException {
|
private Entry pop(plasmaCrawlBalancer balancer) throws IOException {
|
||||||
// this is a filo - pop
|
// this is a filo - pop
|
||||||
if (balancer.size() > 0) {
|
if (balancer.size() > 0) {
|
||||||
Entry e = new Entry(new String(balancer.get()));
|
String hash = new String(balancer.get());
|
||||||
|
if (hash == null) throw new IOException("hash is null");
|
||||||
|
Entry e = new Entry(hash);
|
||||||
stackIndex.remove(e.hash);
|
stackIndex.remove(e.hash);
|
||||||
return e;
|
return e;
|
||||||
} else {
|
} else {
|
||||||
|
@ -480,6 +473,7 @@ public class plasmaCrawlNURL extends indexURL {
|
||||||
// - look into the filed properties
|
// - look into the filed properties
|
||||||
// if the url cannot be found, this returns null
|
// if the url cannot be found, this returns null
|
||||||
this.hash = hash;
|
this.hash = hash;
|
||||||
|
if (hash == null) throw new IOException("hash is null");
|
||||||
kelondroRow.Entry entry = urlHashCache.get(hash.getBytes());
|
kelondroRow.Entry entry = urlHashCache.get(hash.getBytes());
|
||||||
if (entry != null) {
|
if (entry != null) {
|
||||||
insertEntry(entry);
|
insertEntry(entry);
|
||||||
|
|
|
@ -988,7 +988,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser
|
||||||
synchronized (sbQueue) {
|
synchronized (sbQueue) {
|
||||||
|
|
||||||
if (sbQueue.size() == 0) {
|
if (sbQueue.size() == 0) {
|
||||||
log.logFine("deQueue: nothing to do, queue is emtpy");
|
//log.logFine("deQueue: nothing to do, queue is emtpy");
|
||||||
return doneSomething; // nothing to do
|
return doneSomething; // nothing to do
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
Loading…
Reference in New Issue
Block a user