From acdf24877fcf365c20ef48b8c90df5503a8d8bcd Mon Sep 17 00:00:00 2001 From: orbiter Date: Fri, 11 Aug 2006 16:27:56 +0000 Subject: [PATCH] more synchronization against outOfMemoryError in wordIndex git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@2381 6c8d7289-2bf4-0310-a012-ef5d649a1542 --- source/de/anomic/plasma/plasmaWordIndex.java | 100 ++++++++++--------- 1 file changed, 54 insertions(+), 46 deletions(-) diff --git a/source/de/anomic/plasma/plasmaWordIndex.java b/source/de/anomic/plasma/plasmaWordIndex.java index e7dddc7ad..6ff2a53ab 100644 --- a/source/de/anomic/plasma/plasmaWordIndex.java +++ b/source/de/anomic/plasma/plasmaWordIndex.java @@ -170,72 +170,78 @@ public final class plasmaWordIndex extends indexAbstractRI implements indexRI { public void flushControl() { // check for forced flush - synchronized (this) { ramCache.shiftK2W(); } - flushCache(ramCache.maxURLinWCache() - ramCache.wCacheReferenceLimit); - if (ramCache.wSize() > ramCache.getMaxWordCount()) { - flushCache(ramCache.wSize() + 500 - ramCache.getMaxWordCount()); + synchronized (ramCache) { + ramCache.shiftK2W(); + flushCache(ramCache.maxURLinWCache() - ramCache.wCacheReferenceLimit); + if (ramCache.wSize() > ramCache.getMaxWordCount()) { + flushCache(ramCache.wSize() + 500 - ramCache.getMaxWordCount()); + } } } public indexContainer addEntry(String wordHash, indexEntry entry, long updateTime, boolean dhtCase) { indexContainer c; - if ((c = ramCache.addEntry(wordHash, entry, updateTime, dhtCase)) == null) { - if (!dhtCase) flushControl(); - return null; + synchronized (ramCache) { + if ((c = ramCache.addEntry(wordHash, entry, updateTime, dhtCase)) == null) { + if (!dhtCase) flushControl(); + return null; + } } return c; } public indexContainer addEntries(indexContainer entries, long updateTime, boolean dhtCase) { - indexContainer added = ramCache.addEntries(entries, updateTime, dhtCase); - - // force flush - if (!dhtCase) flushControl(); - return added; + synchronized (ramCache) { + indexContainer added = ramCache.addEntries(entries, updateTime, dhtCase); + // force flush + if (!dhtCase) flushControl(); + return added; + } } public void flushCacheSome() { - System.out.println("DEBUG-A"); // some temporary debug lines to identify the outOfMemoryError position - synchronized (this) { ramCache.shiftK2W(); } - System.out.println("DEBUG-B"); int flushCount = ramCache.wSize() / 500; if (flushCount > 80) flushCount = 80; if (flushCount < 10) flushCount = Math.min(10, ramCache.wSize()); - System.out.println("DEBUG-C"); - flushCache(flushCount); - System.out.println("DEBUG-D"); + flushCache(flushCount); } public void flushCache(int count) { if (count <= 0) return; synchronized (ramCache) { + ramCache.shiftK2W(); busyCacheFlush = true; - for (int i = 0; i < count; i++) { + String wordHash; + System.out.println("DEBUG-Started flush of " + count + " entries from RAM to DB"); + long start = System.currentTimeMillis(); + for (int i = 0; i < count; i++) { // possible position of outOfMemoryError ? if (ramCache.wSize() == 0) break; - synchronized (this) { flushCache(ramCache.bestFlushWordHash()); } + wordHash = ramCache.bestFlushWordHash(); + + // flush the wordHash + indexContainer c = ramCache.deleteContainer(wordHash); + if (c != null) { + if (useCollectionIndex) { + indexContainer feedback = collections.addEntries(c, c.updated(), false); + if (feedback != null) { + throw new RuntimeException("indexCollectionRI shall not return feedback entries; feedback = " + feedback.toString()); + } + } else { + indexContainer feedback = assortmentCluster.addEntries(c, c.updated(), false); + if (feedback != null) { + backend.addEntries(feedback, System.currentTimeMillis(), true); + } + } + } + + // pause to next loop to give other processes a chance to use IO try {Thread.sleep(8);} catch (InterruptedException e) {} } + System.out.println("DEBUG-Finished flush of " + count + " entries from RAM to DB in " + (System.currentTimeMillis() - start) + " milliseconds"); busyCacheFlush = false; } } - private synchronized void flushCache(String wordHash) { - indexContainer c = ramCache.deleteContainer(wordHash); - if (c != null) { - if (useCollectionIndex) { - indexContainer feedback = collections.addEntries(c, c.updated(), false); - if (feedback != null) { - throw new RuntimeException("indexCollectionRI shall not return feedback entries; feedback = " + feedback.toString()); - } - } else { - indexContainer feedback = assortmentCluster.addEntries(c, c.updated(), false); - if (feedback != null) { - backend.addEntries(feedback, System.currentTimeMillis(), true); - } - } - } - } - private static final int hour = 3600000; private static final int day = 86400000; @@ -289,7 +295,7 @@ public final class plasmaWordIndex extends indexAbstractRI implements indexRI { // if ((s.length() > 4) && (c > 1)) System.out.println("# " + s + ":" + c); wordHash = indexEntryAttribute.word2hash(word); ientry = new indexURLEntry(urlHash, - urlLength, urlComps, (document == null) ? urlLength : document.longTitle.length(), + urlLength, urlComps, (document == null) ? urlLength : document.longTitle.length(), wprop.count, condenser.RESULT_SIMI_WORDS, condenser.RESULT_SIMI_SENTENCES, @@ -412,16 +418,18 @@ public final class plasmaWordIndex extends indexAbstractRI implements indexRI { } public synchronized indexContainer deleteContainer(String wordHash) { - indexContainer c = ramCache.deleteContainer(wordHash); - if (c == null) c = new indexRowSetContainer(wordHash); - if (useCollectionIndex) c.add(collections.deleteContainer(wordHash), -1); - c.add(assortmentCluster.deleteContainer(wordHash), -1); - c.add(backend.deleteContainer(wordHash), -1); - return c; + synchronized (ramCache) { + indexContainer c = ramCache.deleteContainer(wordHash); + if (c == null) c = new indexRowSetContainer(wordHash); + if (useCollectionIndex) c.add(collections.deleteContainer(wordHash), -1); + c.add(assortmentCluster.deleteContainer(wordHash), -1); + c.add(backend.deleteContainer(wordHash), -1); + return c; + } } public boolean removeEntry(String wordHash, String urlHash, boolean deleteComplete) { - synchronized (this) { + synchronized (ramCache) { if (ramCache.removeEntry(wordHash, urlHash, deleteComplete)) return true; if (useCollectionIndex) {if (collections.removeEntry(wordHash, urlHash, deleteComplete)) return true;} if (assortmentCluster.removeEntry(wordHash, urlHash, deleteComplete)) return true; @@ -431,7 +439,7 @@ public final class plasmaWordIndex extends indexAbstractRI implements indexRI { public int removeEntries(String wordHash, Set urlHashes, boolean deleteComplete) { int removed = 0;; - synchronized (this) { + synchronized (ramCache) { removed += ramCache.removeEntries(wordHash, urlHashes, deleteComplete); if (removed == urlHashes.size()) return removed; if (useCollectionIndex) {