mirror of
https://github.com/yacy/yacy_search_server.git
synced 2024-09-19 00:01:41 +02:00
option to disable dht by memory limit:
memory.acceptDHT in kbytes not yet pre-enabled, will clear on every startup please review since this could break dht in freeworld git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@6459 6c8d7289-2bf4-0310-a012-ef5d649a1542
This commit is contained in:
parent
4431b9767e
commit
6edc168cfe
|
@ -899,6 +899,9 @@ disk.free = 3000
|
||||||
# minimum for DHT or 1/5th of former, whatever is max
|
# minimum for DHT or 1/5th of former, whatever is max
|
||||||
disk.free.hardlimit = 1000
|
disk.free.hardlimit = 1000
|
||||||
|
|
||||||
|
# minimum memory to accept dht-in (KB)
|
||||||
|
#memory.acceptDHT = 10000
|
||||||
|
|
||||||
# setting if execution of CGI files is allowed or not
|
# setting if execution of CGI files is allowed or not
|
||||||
cgi.allow = false
|
cgi.allow = false
|
||||||
cgi.suffixes = cgi,pl
|
cgi.suffixes = cgi,pl
|
||||||
|
|
|
@ -30,6 +30,7 @@ import java.util.Map;
|
||||||
|
|
||||||
import net.yacy.kelondro.logging.Log;
|
import net.yacy.kelondro.logging.Log;
|
||||||
import net.yacy.kelondro.util.DiskSpace;
|
import net.yacy.kelondro.util.DiskSpace;
|
||||||
|
import net.yacy.kelondro.util.MemoryControl;
|
||||||
|
|
||||||
import de.anomic.search.Switchboard;
|
import de.anomic.search.Switchboard;
|
||||||
import de.anomic.search.SwitchboardConstants;
|
import de.anomic.search.SwitchboardConstants;
|
||||||
|
@ -105,6 +106,8 @@ public final class ResourceObserver {
|
||||||
* checks the resources and pauses crawls if necessary
|
* checks the resources and pauses crawls if necessary
|
||||||
*/
|
*/
|
||||||
public void resourceObserverJob() {
|
public void resourceObserverJob() {
|
||||||
|
MemoryControl.setDHTkbytes(sb.getConfigLong(SwitchboardConstants.MEMORY_ACCEPTDHT, 0));
|
||||||
|
|
||||||
checkDiskUsageCount++;
|
checkDiskUsageCount++;
|
||||||
checkMemoryUsageCount++;
|
checkMemoryUsageCount++;
|
||||||
int tmpDisksFree = HIGH;
|
int tmpDisksFree = HIGH;
|
||||||
|
@ -121,15 +124,15 @@ public final class ResourceObserver {
|
||||||
}
|
}
|
||||||
|
|
||||||
if (tmpDisksFree < HIGH || tmpMemoryFree < HIGH) {
|
if (tmpDisksFree < HIGH || tmpMemoryFree < HIGH) {
|
||||||
if (!sb.crawlJobIsPaused(SwitchboardConstants.CRAWLJOB_LOCAL_CRAWL)) {
|
if (tmpDisksFree < HIGH && !sb.crawlJobIsPaused(SwitchboardConstants.CRAWLJOB_LOCAL_CRAWL)) {
|
||||||
log.logInfo("pausing local crawls");
|
log.logInfo("pausing local crawls");
|
||||||
sb.pauseCrawlJob(SwitchboardConstants.CRAWLJOB_LOCAL_CRAWL);
|
sb.pauseCrawlJob(SwitchboardConstants.CRAWLJOB_LOCAL_CRAWL);
|
||||||
}
|
}
|
||||||
if (!sb.crawlJobIsPaused(SwitchboardConstants.CRAWLJOB_REMOTE_TRIGGERED_CRAWL)) {
|
if (tmpDisksFree < HIGH && !sb.crawlJobIsPaused(SwitchboardConstants.CRAWLJOB_REMOTE_TRIGGERED_CRAWL)) {
|
||||||
log.logInfo("pausing remote triggered crawls");
|
log.logInfo("pausing remote triggered crawls");
|
||||||
sb.pauseCrawlJob(SwitchboardConstants.CRAWLJOB_REMOTE_TRIGGERED_CRAWL);
|
sb.pauseCrawlJob(SwitchboardConstants.CRAWLJOB_REMOTE_TRIGGERED_CRAWL);
|
||||||
}
|
}
|
||||||
if (tmpDisksFree == LOW && sb.getConfigBool(SwitchboardConstants.INDEX_RECEIVE_ALLOW, false)) {
|
if ((tmpDisksFree == LOW || tmpMemoryFree < HIGH) && sb.getConfigBool(SwitchboardConstants.INDEX_RECEIVE_ALLOW, false)) {
|
||||||
log.logInfo("disabling index receive");
|
log.logInfo("disabling index receive");
|
||||||
sb.setConfig(SwitchboardConstants.INDEX_RECEIVE_ALLOW, false);
|
sb.setConfig(SwitchboardConstants.INDEX_RECEIVE_ALLOW, false);
|
||||||
sb.peers.mySeed().setFlagAcceptRemoteIndex(false);
|
sb.peers.mySeed().setFlagAcceptRemoteIndex(false);
|
||||||
|
@ -176,7 +179,7 @@ public final class ResourceObserver {
|
||||||
* @return <ul>
|
* @return <ul>
|
||||||
* <li><code>HIGH</code> if disk space is available</li>
|
* <li><code>HIGH</code> if disk space is available</li>
|
||||||
* <li><code>MEDIUM</code> if low disk space is available</li>
|
* <li><code>MEDIUM</code> if low disk space is available</li>
|
||||||
* <li><code>LOW</code> if lower than 100MB or 1/5 disk space is available</li>
|
* <li><code>LOW</code> if lower than hardlimit or 1/5 disk space is available (the max)</li>
|
||||||
* </ul>
|
* </ul>
|
||||||
*/
|
*/
|
||||||
private int checkDisks() {
|
private int checkDisks() {
|
||||||
|
@ -202,6 +205,7 @@ public final class ResourceObserver {
|
||||||
}
|
}
|
||||||
|
|
||||||
private int checkMemory() {
|
private int checkMemory() {
|
||||||
|
if(!MemoryControl.getDHTallowed()) return LOW;
|
||||||
return HIGH;
|
return HIGH;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -392,6 +392,7 @@ public final class SwitchboardConstants {
|
||||||
public static final String DISK_FREE = "disk.free";
|
public static final String DISK_FREE = "disk.free";
|
||||||
public static final String DISK_FREE_HARDLIMIT = "disk.free.hardlimit";
|
public static final String DISK_FREE_HARDLIMIT = "disk.free.hardlimit";
|
||||||
|
|
||||||
|
public static final String MEMORY_ACCEPTDHT = "memory.acceptDHT";
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* Some constants
|
* Some constants
|
||||||
|
|
|
@ -43,6 +43,9 @@ public class MemoryControl {
|
||||||
|
|
||||||
private static long lastGC = 0l;
|
private static long lastGC = 0l;
|
||||||
|
|
||||||
|
private static long DHTkbytes = 0;
|
||||||
|
private static boolean allowDHT = true;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Runs the garbage collector if last garbage collection is more than last millis ago
|
* Runs the garbage collector if last garbage collection is more than last millis ago
|
||||||
* @param last time which must be passed since lased gc
|
* @param last time which must be passed since lased gc
|
||||||
|
@ -154,6 +157,7 @@ public class MemoryControl {
|
||||||
+ " KB (requested/available/average: "
|
+ " KB (requested/available/average: "
|
||||||
+ (size >> 10) + " / " + (avail >> 10) + " / " + (avg >> 10) + " KB)");
|
+ (size >> 10) + " / " + (avail >> 10) + " / " + (avg >> 10) + " KB)");
|
||||||
}
|
}
|
||||||
|
checkDHTrule(avail);
|
||||||
return avail >= size;
|
return avail >= size;
|
||||||
} else {
|
} else {
|
||||||
if (log.isFine()) log.logFine("former GCs indicate to not be able to free enough memory (requested/available/average: "
|
if (log.isFine()) log.logFine("former GCs indicate to not be able to free enough memory (requested/available/average: "
|
||||||
|
@ -170,6 +174,18 @@ public class MemoryControl {
|
||||||
return total() - free();
|
return total() - free();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public static boolean getDHTallowed() {
|
||||||
|
return allowDHT;
|
||||||
|
}
|
||||||
|
|
||||||
|
public static void setDHTkbytes(long kbytes) {
|
||||||
|
DHTkbytes = kbytes;
|
||||||
|
}
|
||||||
|
|
||||||
|
private static void checkDHTrule(long available) {
|
||||||
|
if ((available >> 10) < DHTkbytes) allowDHT = false;
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* main
|
* main
|
||||||
* @param args
|
* @param args
|
||||||
|
|
Loading…
Reference in New Issue
Block a user