From b4acbdaa97baae33c3b005509bc060db43f04d94 Mon Sep 17 00:00:00 2001 From: theli Date: Mon, 4 Sep 2006 05:17:37 +0000 Subject: [PATCH] *) better handling of server shutdown See: e.g. http://www.yacy-forum.de/viewtopic.php?p=25234 git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@2470 6c8d7289-2bf4-0310-a012-ef5d649a1542 --- .../plasma/plasmaRankingDistribution.java | 22 ++++-- .../de/anomic/plasma/plasmaSwitchboard.java | 79 +++++++++++-------- source/de/anomic/yacy/yacyNewsPool.java | 16 ++-- 3 files changed, 73 insertions(+), 44 deletions(-) diff --git a/source/de/anomic/plasma/plasmaRankingDistribution.java b/source/de/anomic/plasma/plasmaRankingDistribution.java index 0531c4023..b405078a4 100644 --- a/source/de/anomic/plasma/plasmaRankingDistribution.java +++ b/source/de/anomic/plasma/plasmaRankingDistribution.java @@ -106,7 +106,7 @@ public final class plasmaRankingDistribution { if ((sourcePath.exists()) && (sourcePath.isDirectory())) return sourcePath.list().length; else return 0; } - public boolean transferRanking(int count) { + public boolean transferRanking(int count) throws InterruptedException { if (method == METHOD_NONE) { log.logFine("no ranking distribution: no transfer method given"); @@ -140,6 +140,10 @@ public final class plasmaRankingDistribution { File crfile = null; for (int i = 0; i < count; i++) { + // check for interruption + if (Thread.currentThread().isInterrupted()) throw new InterruptedException("Shutdown in progress"); + + // getting the next file to transfer crfile = new File(sourcePath, outfiles[i]); if ((method == METHOD_ANYSENIOR) || (method == METHOD_ANYPRINCIPAL)) { @@ -161,9 +165,13 @@ public final class plasmaRankingDistribution { return false; } - private boolean transferRankingAnySeed(File crfile, int trycount) { + private boolean transferRankingAnySeed(File crfile, int trycount) throws InterruptedException { yacySeed target = null; for (int j = 0; j < trycount; j++) { + // check for interruption + if (Thread.currentThread().isInterrupted()) throw new InterruptedException("Shutdown in progress"); + + // get next target target = yacyCore.seedDB.anySeedVersion(yacyVersion.YACY_ACCEPTS_RANKING_TRANSMISSION); if (target == null) continue; @@ -173,10 +181,14 @@ public final class plasmaRankingDistribution { return false; } - private boolean transferRankingAddress(File crfile) { + private boolean transferRankingAddress(File crfile) throws InterruptedException { // try all addresses - for (int i = 0; i < address.length; i++) { - if (transferRankingAddress(crfile, address[i])) return true; + for (int i = 0; i < this.address.length; i++) { + // check for interruption + if (Thread.currentThread().isInterrupted()) throw new InterruptedException("Shutdown in progress"); + + // try to transfer ranking address using the next address + if (transferRankingAddress(crfile, this.address[i])) return true; } return false; } diff --git a/source/de/anomic/plasma/plasmaSwitchboard.java b/source/de/anomic/plasma/plasmaSwitchboard.java index 67cf41cf4..a333a7406 100644 --- a/source/de/anomic/plasma/plasmaSwitchboard.java +++ b/source/de/anomic/plasma/plasmaSwitchboard.java @@ -542,7 +542,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser // clean up profiles this.log.logConfig("Cleaning Profiles"); - cleanProfiles(); + try { cleanProfiles(); } catch (InterruptedException e) { /* Ignore this here */ } // init ranking transmission /* @@ -769,13 +769,17 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser initProfiles(); } - public boolean cleanProfiles() { + public boolean cleanProfiles() throws InterruptedException { if ((sbQueue.size() > 0) || (cacheLoader.size() > 0) || (urlPool.noticeURL.stackSize() > 0)) return false; final Iterator iter = profiles.profiles(true); plasmaCrawlProfile.entry entry; boolean hasDoneSomething = false; try { while (iter.hasNext()) { + // check for interruption + if (Thread.currentThread().isInterrupted()) throw new InterruptedException("Shutdown in progress"); + + // getting next profile entry = (plasmaCrawlProfile.entry) iter.next(); if (!((entry.name().equals("proxy")) || (entry.name().equals("remote")))) { iter.remove(); @@ -1085,40 +1089,49 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser } public boolean cleanupJob() { - - boolean hasDoneSomething = false; - - // do transmission of cr-files - int count = rankingOwnDistribution.size() / 100; - if (count == 0) count = 1; - if (count > 5) count = 5; - rankingOwnDistribution.transferRanking(count); - rankingOtherDistribution.transferRanking(1); - - // clean up error stack - if ((urlPool.errorURL.stackSize() > 1000)) { - log.logFine("Cleaning Error-URLs report stack, " + urlPool.errorURL.stackSize() + " entries on stack"); - urlPool.errorURL.clearStack(); - hasDoneSomething = true; - } - // clean up loadedURL stack - for (int i = 1; i <= 6; i++) { - if (urlPool.loadedURL.getStackSize(i) > 1000) { - log.logFine("Cleaning Loaded-URLs report stack, " + urlPool.loadedURL.getStackSize(i) + " entries on stack " + i); - urlPool.loadedURL.clearStack(i); + try { + boolean hasDoneSomething = false; + + // do transmission of cr-files + checkInterruption(); + int count = rankingOwnDistribution.size() / 100; + if (count == 0) count = 1; + if (count > 5) count = 5; + rankingOwnDistribution.transferRanking(count); + rankingOtherDistribution.transferRanking(1); + + // clean up error stack + checkInterruption(); + if ((urlPool.errorURL.stackSize() > 1000)) { + log.logFine("Cleaning Error-URLs report stack, " + urlPool.errorURL.stackSize() + " entries on stack"); + urlPool.errorURL.clearStack(); hasDoneSomething = true; } - } - // clean up profiles - if (cleanProfiles()) hasDoneSomething = true; + // clean up loadedURL stack + for (int i = 1; i <= 6; i++) { + checkInterruption(); + if (urlPool.loadedURL.getStackSize(i) > 1000) { + log.logFine("Cleaning Loaded-URLs report stack, " + urlPool.loadedURL.getStackSize(i) + " entries on stack " + i); + urlPool.loadedURL.clearStack(i); + hasDoneSomething = true; + } + } + // clean up profiles + checkInterruption(); + if (cleanProfiles()) hasDoneSomething = true; - // clean up news - try { - log.logFine("Cleaning Incoming News, " + yacyCore.newsPool.size(yacyNewsPool.INCOMING_DB) + " entries on stack"); - if (yacyCore.newsPool.automaticProcess() > 0) hasDoneSomething = true; - } catch (IOException e) {} - - return hasDoneSomething; + // clean up news + checkInterruption(); + try { + log.logFine("Cleaning Incoming News, " + yacyCore.newsPool.size(yacyNewsPool.INCOMING_DB) + " entries on stack"); + if (yacyCore.newsPool.automaticProcess() > 0) hasDoneSomething = true; + } catch (IOException e) {} + + return hasDoneSomething; + } catch (InterruptedException e) { + this.log.logInfo("cleanupJob: Shutdown detected"); + return false; + } } /** diff --git a/source/de/anomic/yacy/yacyNewsPool.java b/source/de/anomic/yacy/yacyNewsPool.java index b14827166..86926f9c5 100644 --- a/source/de/anomic/yacy/yacyNewsPool.java +++ b/source/de/anomic/yacy/yacyNewsPool.java @@ -160,17 +160,21 @@ public class yacyNewsPool { return switchQueue(dbKey).size(); } - public int automaticProcess() throws IOException { + public int automaticProcess() throws IOException, InterruptedException { // processes news in the incoming-db // returns number of processes yacyNewsRecord record; int pc = 0; - synchronized (incomingNews) { - for (int i = incomingNews.size() - 1; i >= 0; i--) { - record = incomingNews.top(i); + synchronized (this.incomingNews) { + for (int i = this.incomingNews.size() - 1; i >= 0; i--) { + // check for interruption + if (Thread.currentThread().isInterrupted()) throw new InterruptedException("Shutdown in progress"); + + // get next news record + record = this.incomingNews.top(i); if ((i > 500) || (automaticProcessP(record))) { - incomingNews.pop(i); - processedNews.push(record); + this.incomingNews.pop(i); + this.processedNews.push(record); //newsDB.remove(id); pc++; }