shifted index deletion-on-exit rule to the class where the errors are produced

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@5141 6c8d7289-2bf4-0310-a012-ef5d649a1542

source/de/anomic/index/indexCollectionRI.java

@@ -67,10 +67,6 @@ public class indexCollectionRI implements indexRI {
         }
     }
     
-    public void deleteIndexOnExit() {
-    	collectionIndex.deleteIndexOnExit();
-    }
-    
     public long getUpdateTime(final String wordHash) {
         final indexContainer entries = getContainer(wordHash, null);
         if (entries == null) return 0;

source/de/anomic/kelondro/kelondroAbstractRA.java

@@ -238,4 +238,8 @@ abstract class kelondroAbstractRA implements kelondroRA {
         return b;
     }
     
+    public void deleteOnExit() {
+        if (this.file != null) this.file.deleteOnExit();
+    }
+
 }

source/de/anomic/kelondro/kelondroAbstractRecords.java

@@ -1024,6 +1024,10 @@ public abstract class kelondroAbstractRecords implements kelondroRecords {
         return b;
     }
     
+    public void deleteOnExit() {
+        this.entryFile.deleteOnExit();
+    }
+    
     public abstract kelondroNode newNode(kelondroHandle handle, byte[] bulk, int offset) throws IOException;
     
 }

source/de/anomic/kelondro/kelondroBufferedEcoFS.java

@@ -128,4 +128,8 @@ public class kelondroBufferedEcoFS {
         efs.cleanLast();
     }
     
+    public void deleteOnExit() {
+        efs.deleteOnExit();
+    }
+    
 }

source/de/anomic/kelondro/kelondroBufferedIOChunks.java

@@ -169,4 +169,7 @@ public final class kelondroBufferedIOChunks extends kelondroAbstractIOChunks imp
         super.finalize();
     }
     
+    public void deleteOnExit() {
+        this.ra.deleteOnExit();
+    }
 }

source/de/anomic/kelondro/kelondroCache.java

@@ -460,4 +460,8 @@ public class kelondroCache implements kelondroIndex {
 		init();
 	}
 
+    public void deleteOnExit() {
+        this.index.deleteOnExit();
+    }
+
 }

source/de/anomic/kelondro/kelondroCollectionIndex.java

@@ -53,8 +53,9 @@ public class kelondroCollectionIndex {
     private static final int serialNumber = 0;
     private static final long minimumRAM4Eco = 20 * 1024 * 1024;
     private static final int EcoFSBufferSize = 1000;
+    private static final int errorLimit = 500; // if the index exceeds this number of errors, it is re-built next time the application starts
     
-    kelondroIndex index;
+    private kelondroIndex index;
     private final int           keylength;
     private final File          path;
     private final String        filenameStub;
@@ -63,6 +64,7 @@ public class kelondroCollectionIndex {
     private Map<String, kelondroFixedWidthArray> arrays; // Map of (partitionNumber"-"chunksize)/kelondroFixedWidthArray - Objects
     private final kelondroRow   payloadrow; // definition of the payload (chunks inside the collections)
     private final int           maxPartitions;  // this is the maxmimum number of array files
+    private int                 indexErrors; // counter for exceptions when index returned wrong value
     
     private static final int idx_col_key        = 0;  // the index
     private static final int idx_col_chunksize  = 1;  // chunksize (number of bytes in a single chunk, needed for migration option)
@@ -113,6 +115,7 @@ public class kelondroCollectionIndex {
     public kelondroCollectionIndex(final File path, final String filenameStub, final int keyLength, final kelondroByteOrder indexOrder,
                                    final int loadfactor, final int maxpartitions, final kelondroRow rowdef) throws IOException {
         // the buffersize is number of bytes that are only used if the kelondroFlexTable is backed up with a kelondroTree
+        indexErrors = 0;
         this.path = path;
         this.filenameStub = filenameStub;
         this.keylength = keyLength;
@@ -179,7 +182,7 @@ public class kelondroCollectionIndex {
     
     public void deleteIndexOnExit() {
     	// will be rebuilt on next start
-    	new File(this.path, this.filenameStub + ".index").deleteOnExit();
+    	this.index.deleteOnExit();
     }
     
     private void openAllArrayFiles(final boolean indexGeneration, final kelondroByteOrder indexOrder) throws IOException {
@@ -501,9 +504,11 @@ public class kelondroCollectionIndex {
                     newPartitionNumber, serialNumber, this.payloadrow.objectsize); // modifies indexrow
         }
         
-        if ((int) indexrow.getColLong(idx_col_chunkcount) != collection.size())
+        if ((int) indexrow.getColLong(idx_col_chunkcount) != collection.size()) {
-        	serverLog.logSevere("kelondroCollectionIndex", "UPDATE (put) ERROR: array has different chunkcount than index after merge: index = " + (int) indexrow.getColLong(idx_col_chunkcount) + ", collection.size() = " + collection.size());
+            this.indexErrors++;
-        
+            if (this.indexErrors == errorLimit) deleteIndexOnExit(); // delete index on exit for rebuild
+        	serverLog.logSevere("kelondroCollectionIndex", "UPDATE (put) ERROR: array has different chunkcount than index after merge: index = " + (int) indexrow.getColLong(idx_col_chunkcount) + ", collection.size() = " + collection.size() + " (error #" + indexErrors + ")");
+        }
         index.put(indexrow); // write modified indexrow
     }
     
@@ -562,9 +567,11 @@ public class kelondroCollectionIndex {
             
             final int collectionsize = collection.size(); // extra variable for easier debugging
             final int indexrowcount = (int) indexrow.getColLong(idx_col_chunkcount);
-            if (indexrowcount != collectionsize)
+            if (indexrowcount != collectionsize) {
-            	serverLog.logSevere("kelondroCollectionIndex", "UPDATE (merge) ERROR: array has different chunkcount than index after merge: index = " + indexrowcount + ", collection.size() = " + collectionsize);
+                this.indexErrors++;
-            
+                if (this.indexErrors == errorLimit) deleteIndexOnExit(); // delete index on exit for rebuild
+            	serverLog.logSevere("kelondroCollectionIndex", "UPDATE (merge) ERROR: array has different chunkcount than index after merge: index = " + indexrowcount + ", collection.size() = " + collectionsize + " (error #" + indexErrors + ")");
+            }
             index.put(indexrow); // write modified indexrow
         }
     }
@@ -750,7 +757,12 @@ public class kelondroCollectionIndex {
         // open array entry
         final kelondroFixedWidthArray array = getArray(clusteridx, serialnumber, index.row().objectOrder, chunksize);
         final kelondroRow.Entry arrayrow = array.get(rownumber);
-        if (arrayrow == null) throw new kelondroException(arrayFile(this.path, this.filenameStub, this.loadfactor, chunksize, clusteridx, serialnumber).toString(), "array does not contain expected row");
+        if (arrayrow == null) {
+            // the index appears to be corrupted
+            this.indexErrors++;
+            if (this.indexErrors == errorLimit) deleteIndexOnExit(); // delete index on exit for rebuild
+            throw new kelondroException(arrayFile(this.path, this.filenameStub, this.loadfactor, chunksize, clusteridx, serialnumber).toString(), "array does not contain expected row (error #" + indexErrors + ")");
+        }
 
         // read the row and define a collection
         final byte[] indexkey = indexrow.getColBytes(idx_col_key);
@@ -759,7 +771,9 @@ public class kelondroCollectionIndex {
             // cleanup for a bad bug that corrupted the database
             index.remove(indexkey); // the RowCollection must be considered lost
             array.remove(rownumber); // loose the RowCollection (we don't know how much is lost)
-            serverLog.logSevere("kelondroCollectionIndex." + array.filename, "lost a RowCollection because of a bad arraykey");
+            this.indexErrors++;
+            if (this.indexErrors == errorLimit) deleteIndexOnExit(); // delete index on exit for rebuild
+            serverLog.logSevere("kelondroCollectionIndex." + array.filename, "lost a RowCollection because of a bad arraykey (error #" + indexErrors + ")");
             return new kelondroRowSet(this.payloadrow, 0);
         }
         final kelondroRowSet collection = new kelondroRowSet(this.payloadrow, arrayrow, 1); // FIXME: this does not yet work with different rowdef in case of several rowdef.objectsize()
@@ -777,14 +791,18 @@ public class kelondroCollectionIndex {
             indexEntry.setCol(idx_col_lastread, kelondroRowCollection.daysSince2000(System.currentTimeMillis()));
             indexEntry.setCol(idx_col_lastwrote, kelondroRowCollection.daysSince2000(System.currentTimeMillis()));
             index.put(indexEntry);
-            serverLog.logSevere("kelondroCollectionIndex." + array.filename, "array contains wrong row '" + new String(arrayrow.getColBytes(0)) + "', expected is '" + new String(indexrow.getColBytes(idx_col_key)) + "', the row has been fixed");
+            this.indexErrors++;
+            if (this.indexErrors == errorLimit) deleteIndexOnExit(); // delete index on exit for rebuild
+            serverLog.logSevere("kelondroCollectionIndex." + array.filename, "array contains wrong row '" + new String(arrayrow.getColBytes(0)) + "', expected is '" + new String(indexrow.getColBytes(idx_col_key)) + "', the row has been fixed (error #" + indexErrors + ")");
         }
         final int chunkcountInArray = collection.size();
         if (chunkcountInArray != chunkcount) {
             // fix the entry in index
             indexrow.setCol(idx_col_chunkcount, chunkcountInArray);
             index.put(indexrow);
-            array.logFailure("INCONSISTENCY (get) in " + arrayFile(this.path, this.filenameStub, this.loadfactor, chunksize, clusteridx, serialnumber).toString() + ": array has different chunkcount than index: index = " + chunkcount + ", array = " + chunkcountInArray + "; the index has been auto-fixed");
+            this.indexErrors++;
+            if (this.indexErrors == errorLimit) deleteIndexOnExit(); // delete index on exit for rebuild
+            array.logFailure("INCONSISTENCY (get) in " + arrayFile(this.path, this.filenameStub, this.loadfactor, chunksize, clusteridx, serialnumber).toString() + ": array has different chunkcount than index: index = " + chunkcount + ", array = " + chunkcountInArray + "; the index has been auto-fixed (error #" + indexErrors + ")");
         }
         if (remove) array.remove(rownumber); // index is removed in calling method
         return collection;

source/de/anomic/kelondro/kelondroEcoFS.java

@@ -666,4 +666,8 @@ public class kelondroEcoFS {
         }
     }
     
+    public void deleteOnExit() {
+        this.tablefile.deleteOnExit();
+    }
+
 }

source/de/anomic/kelondro/kelondroEcoTable.java

@@ -715,4 +715,8 @@ public class kelondroEcoTable implements kelondroIndex {
         */
     }
 
+    public void deleteOnExit() {
+        this.file.deleteOnExit();
+    }
+
 }

source/de/anomic/kelondro/kelondroFlexWidthArray.java

@@ -417,4 +417,8 @@ public class kelondroFlexWidthArray implements kelondroArray {
             e.printStackTrace();
         }
     }
+    
+    public void deleteOnExit() {
+        for (int i = 0; i < this.col.length; i++) this.col[i].deleteOnExit();
+    }
 }

source/de/anomic/kelondro/kelondroIOChunks.java

@@ -56,4 +56,6 @@ public interface kelondroIOChunks {
     public void write(long pos, byte[] b) throws IOException;
 
     public kelondroProfile profile();
+    
+    public void deleteOnExit();
 }

source/de/anomic/kelondro/kelondroIndex.java

@@ -54,6 +54,7 @@ public interface kelondroIndex {
     public kelondroRow.Entry removeOne() throws IOException;
     public kelondroCloneableIterator<byte[]> keys(boolean up, byte[] firstKey) throws IOException; // iterates only the key
     public kelondroCloneableIterator<kelondroRow.Entry> rows(boolean up, byte[] firstKey) throws IOException; // iterates the whole row
+    public void deleteOnExit();
     public void clear() throws IOException;
     public void close();
 }

source/de/anomic/kelondro/kelondroRA.java

@@ -79,4 +79,6 @@ public interface kelondroRA {
 
     public void writeArray(byte[] b) throws IOException;
     public byte[] readArray() throws IOException;
+    
+    public void deleteOnExit();
 }

source/de/anomic/kelondro/kelondroRAIOChunks.java

@@ -74,4 +74,7 @@ public final class kelondroRAIOChunks extends kelondroAbstractIOChunks implement
         super.finalize();
     }
 
+    public void deleteOnExit() {
+        this.ra.deleteOnExit();
+    }
 }

source/de/anomic/kelondro/kelondroRAMIndex.java

@@ -232,4 +232,8 @@ public class kelondroRAMIndex implements kelondroIndex {
 		return null; // this does not have a file name
 	}
 	
+	public void deleteOnExit() {
+        // do nothing, there is no file
+    }
+
 }

source/de/anomic/kelondro/kelondroRowSet.java

@@ -551,4 +551,7 @@ public class kelondroRowSet extends kelondroRowCollection implements kelondroInd
         return null;
     }
 
+    public void deleteOnExit() {
+        // do nothing, there is no file
+    }
 }

source/de/anomic/kelondro/kelondroSQLTable.java

@@ -297,7 +297,10 @@ public class kelondroSQLTable implements kelondroIndex {
     }
 
     public void clear() {
-        // TODO Auto-generated method stub
+        // do nothing
+    }
 
+    public void deleteOnExit() {
+        // do nothing
     }
 }

source/de/anomic/kelondro/kelondroSplitTable.java

@@ -435,4 +435,8 @@ public class kelondroSplitTable implements kelondroIndex {
         System.out.println(dateSuffix(new Date()));
     }
 
+    public void deleteOnExit() {
+        for (kelondroIndex i: this.tables.values()) i.deleteOnExit();
+    }
+    
 }

source/de/anomic/plasma/plasmaDHTChunk.java

@@ -66,7 +66,6 @@ public class plasmaDHTChunk {
     private long selectionEndTime = 0;
     
     private int transferFailedCounter = 0;
-    private static int kelondroExceptionCounter = 0;
     
     public indexContainer firstContainer() {
         return indexContainers[0];
@@ -271,8 +270,6 @@ public class plasmaDHTChunk {
             indexContainers = new indexContainer[0];
             urlCache = new HashMap<String, indexURLReference>();
             this.status = chunkStatus_FAILED;
-            kelondroExceptionCounter++;
-            if (kelondroExceptionCounter > 500) wordIndex.deleteIndexOnExit(); // delete index on exit for rebuild
             return 0;
         }
     }

source/de/anomic/plasma/plasmaWordIndex.java

@@ -219,10 +219,6 @@ public final class plasmaWordIndex implements indexRI {
         queuePreStack.clear();
     }
     
-    public void deleteIndexOnExit() {
-    	collections.deleteIndexOnExit();
-    }
-    
     private void initActiveCrawlProfiles() {
         this.defaultProxyProfile = null;
         this.defaultRemoteProfile = null;

source/de/anomic/yacy/yacySearch.java

@@ -191,7 +191,7 @@ public class yacySearch extends Thread {
             c = seedcount;
             while (dhtEnum.hasNext() && c > 0) {
                 seed = dhtEnum.next();
-                if (seed == null) continue;
+                if (seed == null || seed.hash == null) continue;
                 distance = yacyDHTAction.dhtDistance(seed.hash, wordhash);
                 if (distance > 0.2) continue; // catch bug in peer selection
                 if (!seed.getFlagAcceptRemoteIndex()) continue; // probably a robinson peer