Marown/yacy_search_server
1
0
Fork 0
mirror of https://github.com/yacy/yacy_search_server.git synced 2024-09-19 00:01:41 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity

shifted index deletion-on-exit rule to the class where the errors are produced

Browse Source 
git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@5141 6c8d7289-2bf4-0310-a012-ef5d649a1542
This commit is contained in:
orbiter 2008-09-12 11:51:48 +00:00
parent ba76995d2c
commit 1eb813bd43
21 changed files with 86 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
source/de/anomic/index/indexCollectionRI.java
View File

@ -67,10 +67,6 @@ public class indexCollectionRI implements indexRI {
}
}
public void deleteIndexOnExit() {
collectionIndex.deleteIndexOnExit();
}
public long getUpdateTime(final String wordHash) {
final indexContainer entries = getContainer(wordHash, null);
if (entries == null) return 0;

4
source/de/anomic/kelondro/kelondroAbstractRA.java
View File

@ -238,4 +238,8 @@ abstract class kelondroAbstractRA implements kelondroRA {
return b;
}
public void deleteOnExit() {
if (this.file != null) this.file.deleteOnExit();
}
}

4
source/de/anomic/kelondro/kelondroAbstractRecords.java
View File

@ -1024,6 +1024,10 @@ public abstract class kelondroAbstractRecords implements kelondroRecords {
return b;
}
public void deleteOnExit() {
this.entryFile.deleteOnExit();
}
public abstract kelondroNode newNode(kelondroHandle handle, byte[] bulk, int offset) throws IOException;
}

4
source/de/anomic/kelondro/kelondroBufferedEcoFS.java
View File

@ -128,4 +128,8 @@ public class kelondroBufferedEcoFS {
efs.cleanLast();
}
public void deleteOnExit() {
efs.deleteOnExit();
}
}

3
source/de/anomic/kelondro/kelondroBufferedIOChunks.java
View File

@ -169,4 +169,7 @@ public final class kelondroBufferedIOChunks extends kelondroAbstractIOChunks imp
super.finalize();
}
public void deleteOnExit() {
this.ra.deleteOnExit();
}
}

4
source/de/anomic/kelondro/kelondroCache.java
View File

@ -460,4 +460,8 @@ public class kelondroCache implements kelondroIndex {
init();
}
public void deleteOnExit() {
this.index.deleteOnExit();
}
}

42
source/de/anomic/kelondro/kelondroCollectionIndex.java
View File

@ -53,8 +53,9 @@ public class kelondroCollectionIndex {
private static final int serialNumber = 0;
private static final long minimumRAM4Eco = 20 * 1024 * 1024;
private static final int EcoFSBufferSize = 1000;
private static final int errorLimit = 500; // if the index exceeds this number of errors, it is re-built next time the application starts
kelondroIndex index;
private kelondroIndex index;
private final int keylength;
private final File path;
private final String filenameStub;
@ -63,6 +64,7 @@ public class kelondroCollectionIndex {
private Map<String, kelondroFixedWidthArray> arrays; // Map of (partitionNumber"-"chunksize)/kelondroFixedWidthArray - Objects
private final kelondroRow payloadrow; // definition of the payload (chunks inside the collections)
private final int maxPartitions; // this is the maxmimum number of array files
private int indexErrors; // counter for exceptions when index returned wrong value
private static final int idx_col_key = 0; // the index
private static final int idx_col_chunksize = 1; // chunksize (number of bytes in a single chunk, needed for migration option)
@ -113,6 +115,7 @@ public class kelondroCollectionIndex {
public kelondroCollectionIndex(final File path, final String filenameStub, final int keyLength, final kelondroByteOrder indexOrder,
final int loadfactor, final int maxpartitions, final kelondroRow rowdef) throws IOException {
// the buffersize is number of bytes that are only used if the kelondroFlexTable is backed up with a kelondroTree
indexErrors = 0;
this.path = path;
this.filenameStub = filenameStub;
this.keylength = keyLength;
@ -179,7 +182,7 @@ public class kelondroCollectionIndex {
public void deleteIndexOnExit() {
// will be rebuilt on next start
new File(this.path, this.filenameStub + ".index").deleteOnExit();
this.index.deleteOnExit();
}
private void openAllArrayFiles(final boolean indexGeneration, final kelondroByteOrder indexOrder) throws IOException {
@ -501,9 +504,11 @@ public class kelondroCollectionIndex {
newPartitionNumber, serialNumber, this.payloadrow.objectsize); // modifies indexrow
}
if ((int) indexrow.getColLong(idx_col_chunkcount) != collection.size())
serverLog.logSevere("kelondroCollectionIndex", "UPDATE (put) ERROR: array has different chunkcount than index after merge: index = " + (int) indexrow.getColLong(idx_col_chunkcount) + ", collection.size() = " + collection.size());
if ((int) indexrow.getColLong(idx_col_chunkcount) != collection.size()) {
this.indexErrors++;
if (this.indexErrors == errorLimit) deleteIndexOnExit(); // delete index on exit for rebuild
serverLog.logSevere("kelondroCollectionIndex", "UPDATE (put) ERROR: array has different chunkcount than index after merge: index = " + (int) indexrow.getColLong(idx_col_chunkcount) + ", collection.size() = " + collection.size() + " (error #" + indexErrors + ")");
}
index.put(indexrow); // write modified indexrow
}
@ -562,9 +567,11 @@ public class kelondroCollectionIndex {
final int collectionsize = collection.size(); // extra variable for easier debugging
final int indexrowcount = (int) indexrow.getColLong(idx_col_chunkcount);
if (indexrowcount != collectionsize)
serverLog.logSevere("kelondroCollectionIndex", "UPDATE (merge) ERROR: array has different chunkcount than index after merge: index = " + indexrowcount + ", collection.size() = " + collectionsize);
if (indexrowcount != collectionsize) {
this.indexErrors++;
if (this.indexErrors == errorLimit) deleteIndexOnExit(); // delete index on exit for rebuild
serverLog.logSevere("kelondroCollectionIndex", "UPDATE (merge) ERROR: array has different chunkcount than index after merge: index = " + indexrowcount + ", collection.size() = " + collectionsize + " (error #" + indexErrors + ")");
}
index.put(indexrow); // write modified indexrow
}
}
@ -750,7 +757,12 @@ public class kelondroCollectionIndex {
// open array entry
final kelondroFixedWidthArray array = getArray(clusteridx, serialnumber, index.row().objectOrder, chunksize);
final kelondroRow.Entry arrayrow = array.get(rownumber);
if (arrayrow == null) throw new kelondroException(arrayFile(this.path, this.filenameStub, this.loadfactor, chunksize, clusteridx, serialnumber).toString(), "array does not contain expected row");
if (arrayrow == null) {
// the index appears to be corrupted
this.indexErrors++;
if (this.indexErrors == errorLimit) deleteIndexOnExit(); // delete index on exit for rebuild
throw new kelondroException(arrayFile(this.path, this.filenameStub, this.loadfactor, chunksize, clusteridx, serialnumber).toString(), "array does not contain expected row (error #" + indexErrors + ")");
}
// read the row and define a collection
final byte[] indexkey = indexrow.getColBytes(idx_col_key);
@ -759,7 +771,9 @@ public class kelondroCollectionIndex {
// cleanup for a bad bug that corrupted the database
index.remove(indexkey); // the RowCollection must be considered lost
array.remove(rownumber); // loose the RowCollection (we don't know how much is lost)
serverLog.logSevere("kelondroCollectionIndex." + array.filename, "lost a RowCollection because of a bad arraykey");
this.indexErrors++;
if (this.indexErrors == errorLimit) deleteIndexOnExit(); // delete index on exit for rebuild
serverLog.logSevere("kelondroCollectionIndex." + array.filename, "lost a RowCollection because of a bad arraykey (error #" + indexErrors + ")");
return new kelondroRowSet(this.payloadrow, 0);
}
final kelondroRowSet collection = new kelondroRowSet(this.payloadrow, arrayrow, 1); // FIXME: this does not yet work with different rowdef in case of several rowdef.objectsize()
@ -777,14 +791,18 @@ public class kelondroCollectionIndex {
indexEntry.setCol(idx_col_lastread, kelondroRowCollection.daysSince2000(System.currentTimeMillis()));
indexEntry.setCol(idx_col_lastwrote, kelondroRowCollection.daysSince2000(System.currentTimeMillis()));
index.put(indexEntry);
serverLog.logSevere("kelondroCollectionIndex." + array.filename, "array contains wrong row '" + new String(arrayrow.getColBytes(0)) + "', expected is '" + new String(indexrow.getColBytes(idx_col_key)) + "', the row has been fixed");
this.indexErrors++;
if (this.indexErrors == errorLimit) deleteIndexOnExit(); // delete index on exit for rebuild
serverLog.logSevere("kelondroCollectionIndex." + array.filename, "array contains wrong row '" + new String(arrayrow.getColBytes(0)) + "', expected is '" + new String(indexrow.getColBytes(idx_col_key)) + "', the row has been fixed (error #" + indexErrors + ")");
}
final int chunkcountInArray = collection.size();
if (chunkcountInArray != chunkcount) {
// fix the entry in index
indexrow.setCol(idx_col_chunkcount, chunkcountInArray);
index.put(indexrow);
array.logFailure("INCONSISTENCY (get) in " + arrayFile(this.path, this.filenameStub, this.loadfactor, chunksize, clusteridx, serialnumber).toString() + ": array has different chunkcount than index: index = " + chunkcount + ", array = " + chunkcountInArray + "; the index has been auto-fixed");
this.indexErrors++;
if (this.indexErrors == errorLimit) deleteIndexOnExit(); // delete index on exit for rebuild
array.logFailure("INCONSISTENCY (get) in " + arrayFile(this.path, this.filenameStub, this.loadfactor, chunksize, clusteridx, serialnumber).toString() + ": array has different chunkcount than index: index = " + chunkcount + ", array = " + chunkcountInArray + "; the index has been auto-fixed (error #" + indexErrors + ")");
}
if (remove) array.remove(rownumber); // index is removed in calling method
return collection;

4
source/de/anomic/kelondro/kelondroEcoFS.java
View File

@ -666,4 +666,8 @@ public class kelondroEcoFS {
}
}
public void deleteOnExit() {
this.tablefile.deleteOnExit();
}
}

4
source/de/anomic/kelondro/kelondroEcoTable.java
View File

@ -715,4 +715,8 @@ public class kelondroEcoTable implements kelondroIndex {
*/
}
public void deleteOnExit() {
this.file.deleteOnExit();
}
}

4
source/de/anomic/kelondro/kelondroFlexWidthArray.java
View File

@ -417,4 +417,8 @@ public class kelondroFlexWidthArray implements kelondroArray {
e.printStackTrace();
}
}
public void deleteOnExit() {
for (int i = 0; i < this.col.length; i++) this.col[i].deleteOnExit();
}
}

2
source/de/anomic/kelondro/kelondroIOChunks.java
View File

@ -56,4 +56,6 @@ public interface kelondroIOChunks {
public void write(long pos, byte[] b) throws IOException;
public kelondroProfile profile();
public void deleteOnExit();
}

1
source/de/anomic/kelondro/kelondroIndex.java
View File

@ -54,6 +54,7 @@ public interface kelondroIndex {
public kelondroRow.Entry removeOne() throws IOException;
public kelondroCloneableIterator<byte[]> keys(boolean up, byte[] firstKey) throws IOException; // iterates only the key
public kelondroCloneableIterator<kelondroRow.Entry> rows(boolean up, byte[] firstKey) throws IOException; // iterates the whole row
public void deleteOnExit();
public void clear() throws IOException;
public void close();
}

2
source/de/anomic/kelondro/kelondroRA.java
View File

@ -79,4 +79,6 @@ public interface kelondroRA {
public void writeArray(byte[] b) throws IOException;
public byte[] readArray() throws IOException;
public void deleteOnExit();
}

3
source/de/anomic/kelondro/kelondroRAIOChunks.java
View File

@ -74,4 +74,7 @@ public final class kelondroRAIOChunks extends kelondroAbstractIOChunks implement
super.finalize();
}
public void deleteOnExit() {
this.ra.deleteOnExit();
}
}

4
source/de/anomic/kelondro/kelondroRAMIndex.java
View File

@ -232,4 +232,8 @@ public class kelondroRAMIndex implements kelondroIndex {
return null; // this does not have a file name
}
public void deleteOnExit() {
// do nothing, there is no file
}
}

3
source/de/anomic/kelondro/kelondroRowSet.java
View File

@ -551,4 +551,7 @@ public class kelondroRowSet extends kelondroRowCollection implements kelondroInd
return null;
}
public void deleteOnExit() {
// do nothing, there is no file
}
}

5
source/de/anomic/kelondro/kelondroSQLTable.java
View File

@ -297,7 +297,10 @@ public class kelondroSQLTable implements kelondroIndex {
}
public void clear() {
// TODO Auto-generated method stub
// do nothing
}
public void deleteOnExit() {
// do nothing
}
}

4
source/de/anomic/kelondro/kelondroSplitTable.java
View File

@ -435,4 +435,8 @@ public class kelondroSplitTable implements kelondroIndex {
System.out.println(dateSuffix(new Date()));
}
public void deleteOnExit() {
for (kelondroIndex i: this.tables.values()) i.deleteOnExit();
}
}

3
source/de/anomic/plasma/plasmaDHTChunk.java
View File

@ -66,7 +66,6 @@ public class plasmaDHTChunk {
private long selectionEndTime = 0;
private int transferFailedCounter = 0;
private static int kelondroExceptionCounter = 0;
public indexContainer firstContainer() {
return indexContainers[0];
@ -271,8 +270,6 @@ public class plasmaDHTChunk {
indexContainers = new indexContainer[0];
urlCache = new HashMap<String, indexURLReference>();
this.status = chunkStatus_FAILED;
kelondroExceptionCounter++;
if (kelondroExceptionCounter > 500) wordIndex.deleteIndexOnExit(); // delete index on exit for rebuild
return 0;
}
}

4
source/de/anomic/plasma/plasmaWordIndex.java
View File

@ -219,10 +219,6 @@ public final class plasmaWordIndex implements indexRI {
queuePreStack.clear();
}
public void deleteIndexOnExit() {
collections.deleteIndexOnExit();
}
private void initActiveCrawlProfiles() {
this.defaultProxyProfile = null;
this.defaultRemoteProfile = null;

2
source/de/anomic/yacy/yacySearch.java
View File

@ -191,7 +191,7 @@ public class yacySearch extends Thread {
c = seedcount;
while (dhtEnum.hasNext() && c > 0) {
seed = dhtEnum.next();
if (seed == null) continue;
if (seed == null || seed.hash == null) continue;
distance = yacyDHTAction.dhtDistance(seed.hash, wordhash);
if (distance > 0.2) continue; // catch bug in peer selection
if (!seed.getFlagAcceptRemoteIndex()) continue; // probably a robinson peer