mirror of
https://github.com/yacy/yacy_search_server.git
synced 2024-09-19 00:01:41 +02:00
configurable max-load values for YaCy-Threads:
try lower values on smal systems like a Pi
This commit is contained in:
parent
6d8c023a5e
commit
40a4030b55
|
@ -598,33 +598,42 @@ collection=user
|
|||
20_dhtdistribution_idlesleep=30000
|
||||
20_dhtdistribution_busysleep=15000
|
||||
20_dhtdistribution_memprereq=12582912
|
||||
20_dhtdistribution_loadprereq=2.0
|
||||
30_peerping_idlesleep=30000
|
||||
30_peerping_busysleep=30000
|
||||
30_peerping_memprereq=2097152
|
||||
30_peerping_loadprereq=2.0
|
||||
40_peerseedcycle_idlesleep=1800000
|
||||
40_peerseedcycle_busysleep=1200000
|
||||
40_peerseedcycle_memprereq=4194304
|
||||
40_peerseedcycle_loadprereq=2.0
|
||||
50_localcrawl_idlesleep=2000
|
||||
50_localcrawl_busysleep=10
|
||||
50_localcrawl_memprereq=12582912
|
||||
50_localcrawl_loadprereq=2.0
|
||||
50_localcrawl_isPaused=false
|
||||
60_remotecrawlloader_idlesleep=4000
|
||||
60_remotecrawlloader_busysleep=800
|
||||
60_remotecrawlloader_memprereq=12582912
|
||||
60_remotecrawlloader_loadprereq=2.0
|
||||
60_remotecrawlloader_isPaused=false
|
||||
62_remotetriggeredcrawl_idlesleep=2000
|
||||
62_remotetriggeredcrawl_busysleep=200
|
||||
62_remotetriggeredcrawl_memprereq=12582912
|
||||
62_remotetriggeredcrawl_loadprereq=2.0
|
||||
62_remotetriggeredcrawl_isPaused=false
|
||||
70_surrogates_idlesleep=10000
|
||||
70_surrogates_busysleep=0
|
||||
70_surrogates_memprereq=12582912
|
||||
70_surrogates_loadprereq=2.0
|
||||
80_searchresult_idlesleep=10000
|
||||
80_searchresult_busysleep=200
|
||||
80_searchresult_memprereq=0
|
||||
80_searchresult_loadprereq=2.0
|
||||
90_cleanup_idlesleep=300000
|
||||
90_cleanup_busysleep=300000
|
||||
90_cleanup_memprereq=0
|
||||
90_cleanup_loadprereq=3.0
|
||||
|
||||
# additional attributes:
|
||||
# performanceIO is a percent-value. a value of 10 means, that 10% of the busysleep time
|
||||
|
|
|
@ -29,6 +29,7 @@
|
|||
<td>Delay between<br />idle loops</td>
|
||||
<td>Delay between<br />busy loops</td>
|
||||
<td>Minimum of<br />Required Memory</td>
|
||||
<td>Maximum of<br />System-Load</td>
|
||||
<td>Full Description</td>
|
||||
</tr>
|
||||
#{table}#
|
||||
|
@ -51,6 +52,7 @@
|
|||
<td align="right"><input name="#[threadname]#_idlesleep" size="7" maxlength="7" value="#[idlesleep]#" #(disabled)#:: disabled="disabled"#(/disabled)# /> milliseconds</td>
|
||||
<td align="right"><input name="#[threadname]#_busysleep" size="7" maxlength="7" value="#[busysleep]#" #(disabled)#:: disabled="disabled"#(/disabled)# /> milliseconds</td>
|
||||
<td align="right"><input name="#[threadname]#_memprereq" size="7" maxlength="10" value="#[memprereq]#" #(disabled)#:: disabled="disabled"#(/disabled)# /> kbytes</td>
|
||||
<td align="right"><input name="#[threadname]#_loadprereq" size="7" maxlength="7" value="#[loadprereq]#" #(disabled)#:: disabled="disabled"#(/disabled)# /> load</td>
|
||||
<td align="left">#[longdescr]##(recommendation)#::<br/>recommended: #[value]# kbytes#(/recommendation)#</td>
|
||||
</tr>
|
||||
#{/table}#
|
||||
|
|
|
@ -119,6 +119,7 @@ public class PerformanceQueues_p {
|
|||
// set templates for latest news from the threads
|
||||
long blocktime, sleeptime, exectime;
|
||||
long idlesleep, busysleep, memuse, memprereq;
|
||||
double loadprereq;
|
||||
int queuesize;
|
||||
threads = sb.threadNames();
|
||||
int c = 0;
|
||||
|
@ -177,18 +178,21 @@ public class PerformanceQueues_p {
|
|||
idlesleep = sb.getConfigLong(threadName + "_idlesleep" , 1000);
|
||||
busysleep = sb.getConfigLong(threadName + "_busysleep", 100);
|
||||
memprereq = sb.getConfigLong(threadName + "_memprereq", 0);
|
||||
loadprereq = sb.getConfigFloat(threadName + "_loadprereq", 9);
|
||||
if (setDelay && post != null) {
|
||||
// load with new values
|
||||
idlesleep = post.getLong(threadName + "_idlesleep", idlesleep);
|
||||
busysleep = post.getLong(threadName + "_busysleep", busysleep);
|
||||
memprereq = post.getLong(threadName + "_memprereq", memprereq) * 1024l;
|
||||
if (memprereq == 0) memprereq = sb.getConfigLong(threadName + "_memprereq", 0);
|
||||
loadprereq = post.getDouble(threadName + "_loadprereq", loadprereq);
|
||||
if (loadprereq == 0) loadprereq = sb.getConfigFloat(threadName + "_loadprereq", 9);
|
||||
|
||||
// check values to prevent short-cut loops
|
||||
if (idlesleep < 1000) idlesleep = 1000;
|
||||
if (threadName.equals("10_httpd")) { idlesleep = 0; busysleep = 0; memprereq = 0; }
|
||||
if (threadName.equals("10_httpd")) { idlesleep = 0; busysleep = 0; memprereq = 0; loadprereq = 9; }
|
||||
|
||||
sb.setThreadPerformance(threadName, idlesleep, busysleep, memprereq);
|
||||
sb.setThreadPerformance(threadName, idlesleep, busysleep, memprereq, loadprereq);
|
||||
idlesleep = sb.getConfigLong(threadName + "_idlesleep", idlesleep);
|
||||
busysleep = sb.getConfigLong(threadName + "_busysleep", busysleep);
|
||||
}
|
||||
|
@ -205,14 +209,15 @@ public class PerformanceQueues_p {
|
|||
|
||||
// check values to prevent short-cut loops
|
||||
if (idlesleep < 1000) idlesleep = 1000;
|
||||
if (threadName.equals("10_httpd")) { idlesleep = 0; busysleep = 0; memprereq = 0; }
|
||||
if (threadName.equals("10_httpd")) { idlesleep = 0; busysleep = 0; memprereq = 0; loadprereq = 9; }
|
||||
//if (threadName.equals(plasmaSwitchboardConstants.CRAWLJOB_LOCAL_CRAWL) && (busysleep < 50)) busysleep = 50;
|
||||
sb.setThreadPerformance(threadName, idlesleep, busysleep, memprereq);
|
||||
sb.setThreadPerformance(threadName, idlesleep, busysleep, memprereq, loadprereq);
|
||||
}
|
||||
}
|
||||
prop.put("table_" + c + "_idlesleep", idlesleep);
|
||||
prop.put("table_" + c + "_busysleep", busysleep);
|
||||
prop.put("table_" + c + "_memprereq", memprereq / 1024);
|
||||
prop.put("table_" + c + "_loadprereq", loadprereq);
|
||||
// disallow setting of memprereq for indexer to prevent db from throwing OOMs
|
||||
// prop.put("table_" + c + "_disabled", /*(threadName.endsWith("_indexing")) ? 1 :*/ "0");
|
||||
prop.put("table_" + c + "_disabled", threadName.equals("10_httpd") ? "1" : "0" ); // httpd hardcoded defaults
|
||||
|
|
|
@ -20,6 +20,7 @@
|
|||
<delayIdleLoop>#[idlesleep]#</delayIdleLoop>
|
||||
<delayBusyLoop>#[busysleep]#</delayBusyLoop>
|
||||
<minMemory>#[memprereq]#</minMemory>
|
||||
<minMemory>#[loadprereq]#</minMemory>
|
||||
<description><![CDATA[#[longdescr]##(recommendation)#::<br/>recommended: #[value]##(/recommendation)#]]></description>
|
||||
</Task>
|
||||
#{/table}#</Tasks>
|
||||
|
|
|
@ -25,6 +25,8 @@
|
|||
|
||||
package net.yacy.kelondro.util;
|
||||
|
||||
import java.lang.management.ManagementFactory;
|
||||
|
||||
/**
|
||||
* Use this to get information about memory usage or try to free some memory
|
||||
*/
|
||||
|
@ -175,6 +177,14 @@ public class MemoryControl {
|
|||
getStrategy().setProperMbyte(mbyte);
|
||||
}
|
||||
|
||||
/**
|
||||
* get the system load within the last minute
|
||||
* @return the system load or a negative number if the load is not available
|
||||
*/
|
||||
public static double load() {
|
||||
return ManagementFactory.getOperatingSystemMXBean().getSystemLoadAverage();
|
||||
}
|
||||
|
||||
/**
|
||||
* main
|
||||
* @param args use 'force' to request by force, use 'std' / 'gen' to specify strategy
|
||||
|
|
|
@ -37,6 +37,7 @@ public abstract class AbstractBusyThread extends AbstractThread implements BusyT
|
|||
private long startup = 0, intermission = 0, idlePause = 0, busyPause = 0;
|
||||
private long idletime = 0, memprereq = 0;
|
||||
private long idleCycles = 0, busyCycles = 0, outofmemoryCycles = 0;
|
||||
private double loadprereq = 9;
|
||||
private boolean intermissionObedient = true;
|
||||
private final Object syncObject = new Object();
|
||||
|
||||
|
@ -84,6 +85,11 @@ public abstract class AbstractBusyThread extends AbstractThread implements BusyT
|
|||
memprereq = freeBytes;
|
||||
}
|
||||
|
||||
public void setLoadPreReqisite(final double load) {
|
||||
// sets minimum required amount of memory for the job execution
|
||||
loadprereq = load;
|
||||
}
|
||||
|
||||
public void setObeyIntermission(final boolean obey) {
|
||||
// defines if the thread should obey the intermission command
|
||||
intermissionObedient = obey;
|
||||
|
@ -158,6 +164,11 @@ public abstract class AbstractBusyThread extends AbstractThread implements BusyT
|
|||
ratz(this.idlePause);
|
||||
idletime += System.currentTimeMillis() - timestamp;
|
||||
//} else if ((memnow = serverMemory.available()) > memprereq) try {
|
||||
} else if (MemoryControl.load() > loadprereq) {
|
||||
logSystem("Thread '" + this.getName() + "' runs high load cycle. current: " + MemoryControl.load() + " max.: " + loadprereq);
|
||||
timestamp = System.currentTimeMillis();
|
||||
ratz(this.idlePause);
|
||||
idletime += System.currentTimeMillis() - timestamp;
|
||||
} else if (MemoryControl.request(memprereq, false)) try {
|
||||
// do job
|
||||
timestamp = System.currentTimeMillis();
|
||||
|
|
|
@ -65,6 +65,12 @@ public interface BusyThread extends WorkflowThread {
|
|||
*/
|
||||
public void setMemPreReqisite(long freeBytes);
|
||||
|
||||
/**
|
||||
* sets maximimum load for the job execution
|
||||
* @param load
|
||||
*/
|
||||
public void setLoadPreReqisite(final double load);
|
||||
|
||||
/**
|
||||
* defines if the thread should obey the intermission command
|
||||
* @param obey
|
||||
|
|
|
@ -165,6 +165,7 @@ public final class InstantBusyThread extends AbstractBusyThread implements BusyT
|
|||
thread.setIdleSleep(-1);
|
||||
thread.setBusySleep(-1);
|
||||
thread.setMemPreReqisite(0);
|
||||
thread.setLoadPreReqisite(3);
|
||||
thread.start();
|
||||
return thread;
|
||||
}
|
||||
|
|
|
@ -1094,7 +1094,8 @@ public final class Switchboard extends serverSwitch {
|
|||
60000,
|
||||
Long.parseLong(getConfig(SwitchboardConstants.INDEX_DIST_IDLESLEEP, "5000")),
|
||||
Long.parseLong(getConfig(SwitchboardConstants.INDEX_DIST_BUSYSLEEP, "0")),
|
||||
Long.parseLong(getConfig(SwitchboardConstants.INDEX_DIST_MEMPREREQ, "1000000")));
|
||||
Long.parseLong(getConfig(SwitchboardConstants.INDEX_DIST_MEMPREREQ, "1000000")),
|
||||
Double.parseDouble(getConfig(SwitchboardConstants.INDEX_DIST_LOADPREREQ, "9.0")));
|
||||
|
||||
// content control: initialize list sync thread
|
||||
deployThread(
|
||||
|
|
|
@ -62,6 +62,7 @@ public final class SwitchboardConstants {
|
|||
public static final String INDEX_DIST_METHOD_JOBCOUNT = null;
|
||||
public static final String INDEX_DIST_METHOD_FREEMEM = null;
|
||||
public static final String INDEX_DIST_MEMPREREQ = "20_dhtdistribution_memprereq";
|
||||
public static final String INDEX_DIST_LOADPREREQ = "20_dhtdistribution_loadprereq";
|
||||
public static final String INDEX_DIST_IDLESLEEP = "20_dhtdistribution_idlesleep";
|
||||
public static final String INDEX_DIST_BUSYSLEEP = "20_dhtdistribution_busysleep";
|
||||
// 30_peerping
|
||||
|
@ -132,6 +133,7 @@ public final class SwitchboardConstants {
|
|||
*/
|
||||
public static final String SURROGATES = "70_surrogates";
|
||||
public static final String SURROGATES_MEMPREREQ = "70_surrogates_memprereq";
|
||||
public static final String SURROGATES_LOADPREREQ = "70_surrogates_loadprereq";
|
||||
public static final String SURROGATES_IDLESLEEP = "70_surrogates_idlesleep";
|
||||
public static final String SURROGATES_BUSYSLEEP = "70_surrogates_busysleep";
|
||||
public static final String SURROGATES_METHOD_START = "surrogateProcess";
|
||||
|
@ -144,6 +146,7 @@ public final class SwitchboardConstants {
|
|||
*/
|
||||
public static final String SEARCHRESULT = "80_searchresult";
|
||||
public static final String SEARCHRESULT_MEMPREREQ = "80_searchresult_memprereq";
|
||||
public static final String SEARCHRESULT_LOADPREREQ = "80_searchresult_loadprereq";
|
||||
public static final String SEARCHRESULT_IDLESLEEP = "80_searchresult_idlesleep";
|
||||
public static final String SEARCHRESULT_BUSYSLEEP = "80_searchresult_busysleep";
|
||||
public static final String SEARCHRESULT_METHOD_START = "searchresultProcess";
|
||||
|
|
|
@ -344,7 +344,8 @@ public class serverSwitch
|
|||
startupDelay,
|
||||
Long.parseLong(getConfig(threadName + "_idlesleep", "100")),
|
||||
Long.parseLong(getConfig(threadName + "_busysleep", "1000")),
|
||||
Long.parseLong(getConfig(threadName + "_memprereq", "1000000")));
|
||||
Long.parseLong(getConfig(threadName + "_memprereq", "1000000")),
|
||||
Double.parseDouble(getConfig(threadName + "_loadprereq", "9.0")));
|
||||
}
|
||||
|
||||
public void deployThread(
|
||||
|
@ -356,7 +357,8 @@ public class serverSwitch
|
|||
final long startupDelay,
|
||||
final long initialIdleSleep,
|
||||
final long initialBusySleep,
|
||||
final long initialMemoryPreRequisite) {
|
||||
final long initialMemoryPreRequisite,
|
||||
final double initialLoadPreRequisite) {
|
||||
if ( newThread.isAlive() ) {
|
||||
throw new RuntimeException(
|
||||
"undeployed threads must not live; they are started as part of the deployment");
|
||||
|
@ -384,6 +386,13 @@ public class serverSwitch
|
|||
newThread.setMemPreReqisite(initialMemoryPreRequisite);
|
||||
setConfig(threadName + "_memprereq", initialMemoryPreRequisite);
|
||||
}
|
||||
try {
|
||||
final double load = Double.parseDouble(getConfig(threadName + "_loadprereq", "novalue"));
|
||||
newThread.setLoadPreReqisite(load);
|
||||
} catch (final NumberFormatException e ) {
|
||||
newThread.setLoadPreReqisite(initialLoadPreRequisite);
|
||||
setConfig(threadName + "_loadprereq", (float)initialLoadPreRequisite);
|
||||
}
|
||||
newThread.setDescription(threadShortDescription, threadLongDescription, threadMonitorURL);
|
||||
this.workerThreads.put(threadName, newThread);
|
||||
// start the thread
|
||||
|
@ -400,13 +409,16 @@ public class serverSwitch
|
|||
final String threadName,
|
||||
final long idleMillis,
|
||||
final long busyMillis,
|
||||
final long memprereqBytes) {
|
||||
final long memprereqBytes,
|
||||
final double loadprereq) {
|
||||
final BusyThread thread = this.workerThreads.get(threadName);
|
||||
if ( thread != null ) {
|
||||
setConfig(threadName + "_idlesleep", thread.setIdleSleep(idleMillis));
|
||||
setConfig(threadName + "_busysleep", thread.setBusySleep(busyMillis));
|
||||
setConfig(threadName + "_memprereq", memprereqBytes);
|
||||
thread.setMemPreReqisite(memprereqBytes);
|
||||
setConfig(threadName + "_loadprereq", (float)loadprereq);
|
||||
thread.setLoadPreReqisite(loadprereq);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
Loading…
Reference in New Issue
Block a user