Marown/yacy_search_server
1
0
Fork 0
mirror of https://github.com/yacy/yacy_search_server.git synced 2024-09-19 00:01:41 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity

configurable max-load values for YaCy-Threads:

Browse Source 
try lower values on smal systems like a Pi
This commit is contained in:
sixcooler 2014-01-21 17:04:22 +01:00
parent 6d8c023a5e
commit 40a4030b55
11 changed files with 69 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
defaults/yacy.init
View File

@ -598,33 +598,42 @@ collection=user
20_dhtdistribution_idlesleep=30000
20_dhtdistribution_busysleep=15000
20_dhtdistribution_memprereq=12582912
20_dhtdistribution_loadprereq=2.0
30_peerping_idlesleep=30000
30_peerping_busysleep=30000
30_peerping_memprereq=2097152
30_peerping_loadprereq=2.0
40_peerseedcycle_idlesleep=1800000
40_peerseedcycle_busysleep=1200000
40_peerseedcycle_memprereq=4194304
40_peerseedcycle_loadprereq=2.0
50_localcrawl_idlesleep=2000
50_localcrawl_busysleep=10
50_localcrawl_memprereq=12582912
50_localcrawl_loadprereq=2.0
50_localcrawl_isPaused=false
60_remotecrawlloader_idlesleep=4000
60_remotecrawlloader_busysleep=800
60_remotecrawlloader_memprereq=12582912
60_remotecrawlloader_loadprereq=2.0
60_remotecrawlloader_isPaused=false
62_remotetriggeredcrawl_idlesleep=2000
62_remotetriggeredcrawl_busysleep=200
62_remotetriggeredcrawl_memprereq=12582912
62_remotetriggeredcrawl_loadprereq=2.0
62_remotetriggeredcrawl_isPaused=false
70_surrogates_idlesleep=10000
70_surrogates_busysleep=0
70_surrogates_memprereq=12582912
70_surrogates_loadprereq=2.0
80_searchresult_idlesleep=10000
80_searchresult_busysleep=200
80_searchresult_memprereq=0
80_searchresult_loadprereq=2.0
90_cleanup_idlesleep=300000
90_cleanup_busysleep=300000
90_cleanup_memprereq=0
90_cleanup_loadprereq=3.0
# additional attributes:
# performanceIO is a percent-value. a value of 10 means, that 10% of the busysleep time

2
htroot/PerformanceQueues_p.html
View File

@ -29,6 +29,7 @@
<td>Delay between<br />idle loops</td>
<td>Delay between<br />busy loops</td>
<td>Minimum of<br />Required Memory</td>
<td>Maximum of<br />System-Load</td>
<td>Full Description</td>
</tr>
#{table}#
@ -51,6 +52,7 @@
<td align="right"><input name="#[threadname]#_idlesleep" size="7" maxlength="7" value="#[idlesleep]#" #(disabled)#:: disabled="disabled"#(/disabled)# /> milliseconds</td>
<td align="right"><input name="#[threadname]#_busysleep" size="7" maxlength="7" value="#[busysleep]#" #(disabled)#:: disabled="disabled"#(/disabled)# /> milliseconds</td>
<td align="right"><input name="#[threadname]#_memprereq" size="7" maxlength="10" value="#[memprereq]#" #(disabled)#:: disabled="disabled"#(/disabled)# /> kbytes</td>
<td align="right"><input name="#[threadname]#_loadprereq" size="7" maxlength="7" value="#[loadprereq]#" #(disabled)#:: disabled="disabled"#(/disabled)# /> load</td>
<td align="left">#[longdescr]##(recommendation)#::<br/>recommended: #[value]# kbytes#(/recommendation)#</td>
</tr>
#{/table}#

13
htroot/PerformanceQueues_p.java
View File

@ -119,6 +119,7 @@ public class PerformanceQueues_p {
// set templates for latest news from the threads
long blocktime, sleeptime, exectime;
long idlesleep, busysleep, memuse, memprereq;
double loadprereq;
int queuesize;
threads = sb.threadNames();
int c = 0;
@ -177,18 +178,21 @@ public class PerformanceQueues_p {
idlesleep = sb.getConfigLong(threadName + "_idlesleep" , 1000);
busysleep = sb.getConfigLong(threadName + "_busysleep", 100);
memprereq = sb.getConfigLong(threadName + "_memprereq", 0);
loadprereq = sb.getConfigFloat(threadName + "_loadprereq", 9);
if (setDelay && post != null) {
// load with new values
idlesleep = post.getLong(threadName + "_idlesleep", idlesleep);
busysleep = post.getLong(threadName + "_busysleep", busysleep);
memprereq = post.getLong(threadName + "_memprereq", memprereq) * 1024l;
if (memprereq == 0) memprereq = sb.getConfigLong(threadName + "_memprereq", 0);
loadprereq = post.getDouble(threadName + "_loadprereq", loadprereq);
if (loadprereq == 0) loadprereq = sb.getConfigFloat(threadName + "_loadprereq", 9);
// check values to prevent short-cut loops
if (idlesleep < 1000) idlesleep = 1000;
if (threadName.equals("10_httpd")) { idlesleep = 0; busysleep = 0; memprereq = 0; }
if (threadName.equals("10_httpd")) { idlesleep = 0; busysleep = 0; memprereq = 0; loadprereq = 9; }
sb.setThreadPerformance(threadName, idlesleep, busysleep, memprereq);
sb.setThreadPerformance(threadName, idlesleep, busysleep, memprereq, loadprereq);
idlesleep = sb.getConfigLong(threadName + "_idlesleep", idlesleep);
busysleep = sb.getConfigLong(threadName + "_busysleep", busysleep);
}
@ -205,14 +209,15 @@ public class PerformanceQueues_p {
// check values to prevent short-cut loops
if (idlesleep < 1000) idlesleep = 1000;
if (threadName.equals("10_httpd")) { idlesleep = 0; busysleep = 0; memprereq = 0; }
if (threadName.equals("10_httpd")) { idlesleep = 0; busysleep = 0; memprereq = 0; loadprereq = 9; }
//if (threadName.equals(plasmaSwitchboardConstants.CRAWLJOB_LOCAL_CRAWL) && (busysleep < 50)) busysleep = 50;
sb.setThreadPerformance(threadName, idlesleep, busysleep, memprereq);
sb.setThreadPerformance(threadName, idlesleep, busysleep, memprereq, loadprereq);
}
}
prop.put("table_" + c + "_idlesleep", idlesleep);
prop.put("table_" + c + "_busysleep", busysleep);
prop.put("table_" + c + "_memprereq", memprereq / 1024);
prop.put("table_" + c + "_loadprereq", loadprereq);
// disallow setting of memprereq for indexer to prevent db from throwing OOMs
// prop.put("table_" + c + "_disabled", /*(threadName.endsWith("_indexing")) ? 1 :*/ "0");
prop.put("table_" + c + "_disabled", threadName.equals("10_httpd") ? "1" : "0" ); // httpd hardcoded defaults

1
htroot/PerformanceQueues_p.xml
View File

@ -20,6 +20,7 @@
<delayIdleLoop>#[idlesleep]#</delayIdleLoop>
<delayBusyLoop>#[busysleep]#</delayBusyLoop>
<minMemory>#[memprereq]#</minMemory>
<minMemory>#[loadprereq]#</minMemory>
<description><![CDATA[#[longdescr]##(recommendation)#::<br/>recommended: #[value]##(/recommendation)#]]></description>
</Task>
#{/table}#</Tasks>

10
source/net/yacy/kelondro/util/MemoryControl.java
View File

@ -25,6 +25,8 @@
package net.yacy.kelondro.util;
import java.lang.management.ManagementFactory;
/**
* Use this to get information about memory usage or try to free some memory
*/
@ -174,6 +176,14 @@ public class MemoryControl {
public static void setProperMbyte(final long mbyte) {
getStrategy().setProperMbyte(mbyte);
}
/**
* get the system load within the last minute
* @return the system load or a negative number if the load is not available
*/
public static double load() {
return ManagementFactory.getOperatingSystemMXBean().getSystemLoadAverage();
}
/**
* main

11
source/net/yacy/kelondro/workflow/AbstractBusyThread.java
View File

@ -37,6 +37,7 @@ public abstract class AbstractBusyThread extends AbstractThread implements BusyT
private long startup = 0, intermission = 0, idlePause = 0, busyPause = 0;
private long idletime = 0, memprereq = 0;
private long idleCycles = 0, busyCycles = 0, outofmemoryCycles = 0;
private double loadprereq = 9;
private boolean intermissionObedient = true;
private final Object syncObject = new Object();
@ -84,6 +85,11 @@ public abstract class AbstractBusyThread extends AbstractThread implements BusyT
memprereq = freeBytes;
}
public void setLoadPreReqisite(final double load) {
// sets minimum required amount of memory for the job execution
loadprereq = load;
}
public void setObeyIntermission(final boolean obey) {
// defines if the thread should obey the intermission command
intermissionObedient = obey;
@ -158,6 +164,11 @@ public abstract class AbstractBusyThread extends AbstractThread implements BusyT
ratz(this.idlePause);
idletime += System.currentTimeMillis() - timestamp;
//} else if ((memnow = serverMemory.available()) > memprereq) try {
} else if (MemoryControl.load() > loadprereq) {
logSystem("Thread '" + this.getName() + "' runs high load cycle. current: " + MemoryControl.load() + " max.: " + loadprereq);
timestamp = System.currentTimeMillis();
ratz(this.idlePause);
idletime += System.currentTimeMillis() - timestamp;
} else if (MemoryControl.request(memprereq, false)) try {
// do job
timestamp = System.currentTimeMillis();

6
source/net/yacy/kelondro/workflow/BusyThread.java
View File

@ -64,6 +64,12 @@ public interface BusyThread extends WorkflowThread {
* @param freeBytes
*/
public void setMemPreReqisite(long freeBytes);
/**
* sets maximimum load for the job execution
* @param load
*/
public void setLoadPreReqisite(final double load);
/**
* defines if the thread should obey the intermission command

1
source/net/yacy/kelondro/workflow/InstantBusyThread.java
View File

@ -165,6 +165,7 @@ public final class InstantBusyThread extends AbstractBusyThread implements BusyT
thread.setIdleSleep(-1);
thread.setBusySleep(-1);
thread.setMemPreReqisite(0);
thread.setLoadPreReqisite(3);
thread.start();
return thread;
}

3
source/net/yacy/search/Switchboard.java
View File

@ -1094,7 +1094,8 @@ public final class Switchboard extends serverSwitch {
60000,
Long.parseLong(getConfig(SwitchboardConstants.INDEX_DIST_IDLESLEEP, "5000")),
Long.parseLong(getConfig(SwitchboardConstants.INDEX_DIST_BUSYSLEEP, "0")),
Long.parseLong(getConfig(SwitchboardConstants.INDEX_DIST_MEMPREREQ, "1000000")));
Long.parseLong(getConfig(SwitchboardConstants.INDEX_DIST_MEMPREREQ, "1000000")),
Double.parseDouble(getConfig(SwitchboardConstants.INDEX_DIST_LOADPREREQ, "9.0")));
// content control: initialize list sync thread
deployThread(

3
source/net/yacy/search/SwitchboardConstants.java
View File

@ -62,6 +62,7 @@ public final class SwitchboardConstants {
public static final String INDEX_DIST_METHOD_JOBCOUNT = null;
public static final String INDEX_DIST_METHOD_FREEMEM = null;
public static final String INDEX_DIST_MEMPREREQ = "20_dhtdistribution_memprereq";
public static final String INDEX_DIST_LOADPREREQ = "20_dhtdistribution_loadprereq";
public static final String INDEX_DIST_IDLESLEEP = "20_dhtdistribution_idlesleep";
public static final String INDEX_DIST_BUSYSLEEP = "20_dhtdistribution_busysleep";
// 30_peerping
@ -132,6 +133,7 @@ public final class SwitchboardConstants {
*/
public static final String SURROGATES = "70_surrogates";
public static final String SURROGATES_MEMPREREQ = "70_surrogates_memprereq";
public static final String SURROGATES_LOADPREREQ = "70_surrogates_loadprereq";
public static final String SURROGATES_IDLESLEEP = "70_surrogates_idlesleep";
public static final String SURROGATES_BUSYSLEEP = "70_surrogates_busysleep";
public static final String SURROGATES_METHOD_START = "surrogateProcess";
@ -144,6 +146,7 @@ public final class SwitchboardConstants {
*/
public static final String SEARCHRESULT = "80_searchresult";
public static final String SEARCHRESULT_MEMPREREQ = "80_searchresult_memprereq";
public static final String SEARCHRESULT_LOADPREREQ = "80_searchresult_loadprereq";
public static final String SEARCHRESULT_IDLESLEEP = "80_searchresult_idlesleep";
public static final String SEARCHRESULT_BUSYSLEEP = "80_searchresult_busysleep";
public static final String SEARCHRESULT_METHOD_START = "searchresultProcess";

18
source/net/yacy/server/serverSwitch.java
View File

@ -344,7 +344,8 @@ public class serverSwitch
startupDelay,
Long.parseLong(getConfig(threadName + "_idlesleep", "100")),
Long.parseLong(getConfig(threadName + "_busysleep", "1000")),
Long.parseLong(getConfig(threadName + "_memprereq", "1000000")));
Long.parseLong(getConfig(threadName + "_memprereq", "1000000")),
Double.parseDouble(getConfig(threadName + "_loadprereq", "9.0")));
}
public void deployThread(
@ -356,7 +357,8 @@ public class serverSwitch
final long startupDelay,
final long initialIdleSleep,
final long initialBusySleep,
final long initialMemoryPreRequisite) {
final long initialMemoryPreRequisite,
final double initialLoadPreRequisite) {
if ( newThread.isAlive() ) {
throw new RuntimeException(
"undeployed threads must not live; they are started as part of the deployment");
@ -384,6 +386,13 @@ public class serverSwitch
newThread.setMemPreReqisite(initialMemoryPreRequisite);
setConfig(threadName + "_memprereq", initialMemoryPreRequisite);
}
try {
final double load = Double.parseDouble(getConfig(threadName + "_loadprereq", "novalue"));
newThread.setLoadPreReqisite(load);
} catch (final NumberFormatException e ) {
newThread.setLoadPreReqisite(initialLoadPreRequisite);
setConfig(threadName + "_loadprereq", (float)initialLoadPreRequisite);
}
newThread.setDescription(threadShortDescription, threadLongDescription, threadMonitorURL);
this.workerThreads.put(threadName, newThread);
// start the thread
@ -400,13 +409,16 @@ public class serverSwitch
final String threadName,
final long idleMillis,
final long busyMillis,
final long memprereqBytes) {
final long memprereqBytes,
final double loadprereq) {
final BusyThread thread = this.workerThreads.get(threadName);
if ( thread != null ) {
setConfig(threadName + "_idlesleep", thread.setIdleSleep(idleMillis));
setConfig(threadName + "_busysleep", thread.setBusySleep(busyMillis));
setConfig(threadName + "_memprereq", memprereqBytes);
thread.setMemPreReqisite(memprereqBytes);
setConfig(threadName + "_loadprereq", (float)loadprereq);
thread.setLoadPreReqisite(loadprereq);
}
}