From e20450e798d053ac404fd16819fe7c02eedd23b0 Mon Sep 17 00:00:00 2001 From: Michael Peter Christen Date: Fri, 7 Jun 2013 12:52:03 +0200 Subject: [PATCH] patch in HTCache and CitationIndex loading in case that a file is broken: do not crash; instead ignore the file and delete it. --- source/net/yacy/crawler/data/Cache.java | 2 +- source/net/yacy/kelondro/blob/ArrayStack.java | 26 +++++++++++++------ .../yacy/kelondro/io/CachedFileWriter.java | 2 +- .../kelondro/rwi/ReferenceContainerArray.java | 1 + 4 files changed, 21 insertions(+), 10 deletions(-) diff --git a/source/net/yacy/crawler/data/Cache.java b/source/net/yacy/crawler/data/Cache.java index c0c45ed50..36e4526f8 100644 --- a/source/net/yacy/crawler/data/Cache.java +++ b/source/net/yacy/crawler/data/Cache.java @@ -89,7 +89,7 @@ public final class Cache { } // open the cache file try { - fileDBunbuffered = new ArrayStack(new File(cachePath, FILE_DB_NAME), prefix, Base64Order.enhancedCoder, 12, 1024 * 1024 * 2, false); + fileDBunbuffered = new ArrayStack(new File(cachePath, FILE_DB_NAME), prefix, Base64Order.enhancedCoder, 12, 1024 * 1024 * 2, false, true); fileDBunbuffered.setMaxSize(maxCacheSize); fileDB = new Compressor(fileDBunbuffered, 6 * 1024 * 1024); } catch (final IOException e) { diff --git a/source/net/yacy/kelondro/blob/ArrayStack.java b/source/net/yacy/kelondro/blob/ArrayStack.java index 09aed8c74..5eb5636fa 100644 --- a/source/net/yacy/kelondro/blob/ArrayStack.java +++ b/source/net/yacy/kelondro/blob/ArrayStack.java @@ -111,7 +111,8 @@ public class ArrayStack implements BLOB { final ByteOrder ordering, final int keylength, final int buffersize, - final boolean trimall) throws IOException { + final boolean trimall, + final boolean deleteonfail) throws IOException { this.keylength = keylength; this.prefix = prefix; this.ordering = ordering; @@ -200,13 +201,22 @@ public class ArrayStack implements BLOB { d = my_SHORT_MILSEC_FORMATTER.parse(file.substring(this.prefix.length() + 1, this.prefix.length() + 18)); f = new File(heapLocation, file); time = d.getTime(); - if (time == maxtime && !trimall) { - oneBlob = new Heap(f, keylength, ordering, buffersize); - } else { - oneBlob = new HeapModifier(f, keylength, ordering); - oneBlob.trim(); // no writings here, can be used with minimum memory + try { + if (time == maxtime && !trimall) { + oneBlob = new Heap(f, keylength, ordering, buffersize); + } else { + oneBlob = new HeapModifier(f, keylength, ordering); + oneBlob.trim(); // no writings here, can be used with minimum memory + } + sortedItems.put(Long.valueOf(time), new blobItem(d, f, oneBlob)); + } catch (IOException e) { + if (deleteonfail) { + Log.logWarning("ArrayStack", "cannot read file " + f.getName() + ", deleting it (smart fail; alternative would be: crash; required user action would be same as deletion)"); + f.delete(); + } else { + throw new IOException(e.getMessage(), e); + } } - sortedItems.put(Long.valueOf(time), new blobItem(d, f, oneBlob)); } catch (final ParseException e) {continue;} } } @@ -1155,7 +1165,7 @@ public class ArrayStack implements BLOB { final File f = new File("/Users/admin/blobarraytest"); try { //f.delete(); - final ArrayStack heap = new ArrayStack(f, "test", NaturalOrder.naturalOrder, 12, 512 * 1024, false); + final ArrayStack heap = new ArrayStack(f, "test", NaturalOrder.naturalOrder, 12, 512 * 1024, false, true); heap.insert("aaaaaaaaaaaa".getBytes(), "eins zwei drei".getBytes()); heap.insert("aaaaaaaaaaab".getBytes(), "vier fuenf sechs".getBytes()); heap.insert("aaaaaaaaaaac".getBytes(), "sieben acht neun".getBytes()); diff --git a/source/net/yacy/kelondro/io/CachedFileWriter.java b/source/net/yacy/kelondro/io/CachedFileWriter.java index aa8785479..292fa92f4 100644 --- a/source/net/yacy/kelondro/io/CachedFileWriter.java +++ b/source/net/yacy/kelondro/io/CachedFileWriter.java @@ -87,7 +87,7 @@ public final class CachedFileWriter extends AbstractWriter implements Writer { long available = this.RAFile.length() - seek; if (available == -seek) return; // we don't know how this happens but we just silently ignore it by now TODO:fixme //System.out.println("*** available = " + available); - if (available < len) throw new IOException("EOF, available = " + available + ", requested = " + len + ", this.RAFile.length() = " + this.RAFile.length() + ", seek = " + seek); + if (available < len) throw new IOException("EOF in " + this.file.getName() + ", available = " + available + ", requested = " + len + ", this.RAFile.length() = " + this.RAFile.length() + ", seek = " + seek); if (this.cachestart + this.cachelen == seek && this.cache.length - this.cachelen >= len) { this.RAFile.readFully(this.cache, this.cachelen, len); //System.out.println("*** DEBUG FileRA " + this.file.getName() + ": append fill " + len + " bytes"); diff --git a/source/net/yacy/kelondro/rwi/ReferenceContainerArray.java b/source/net/yacy/kelondro/rwi/ReferenceContainerArray.java index 0b95a403d..72b8a58ad 100644 --- a/source/net/yacy/kelondro/rwi/ReferenceContainerArray.java +++ b/source/net/yacy/kelondro/rwi/ReferenceContainerArray.java @@ -71,6 +71,7 @@ public final class ReferenceContainerArray { termOrder, termSize, 0, + true, true); }