configurable max-load values for YaCy-Threads:

try lower values on smal systems like a Pi

defaults/yacy.init

@@ -598,33 +598,42 @@ collection=user
 20_dhtdistribution_idlesleep=30000
 20_dhtdistribution_busysleep=15000
 20_dhtdistribution_memprereq=12582912
+20_dhtdistribution_loadprereq=2.0
 30_peerping_idlesleep=30000
 30_peerping_busysleep=30000
 30_peerping_memprereq=2097152
+30_peerping_loadprereq=2.0
 40_peerseedcycle_idlesleep=1800000
 40_peerseedcycle_busysleep=1200000
 40_peerseedcycle_memprereq=4194304
+40_peerseedcycle_loadprereq=2.0
 50_localcrawl_idlesleep=2000
 50_localcrawl_busysleep=10
 50_localcrawl_memprereq=12582912
+50_localcrawl_loadprereq=2.0
 50_localcrawl_isPaused=false
 60_remotecrawlloader_idlesleep=4000
 60_remotecrawlloader_busysleep=800
 60_remotecrawlloader_memprereq=12582912
+60_remotecrawlloader_loadprereq=2.0
 60_remotecrawlloader_isPaused=false
 62_remotetriggeredcrawl_idlesleep=2000
 62_remotetriggeredcrawl_busysleep=200
 62_remotetriggeredcrawl_memprereq=12582912
+62_remotetriggeredcrawl_loadprereq=2.0
 62_remotetriggeredcrawl_isPaused=false
 70_surrogates_idlesleep=10000
 70_surrogates_busysleep=0
 70_surrogates_memprereq=12582912
+70_surrogates_loadprereq=2.0
 80_searchresult_idlesleep=10000
 80_searchresult_busysleep=200
 80_searchresult_memprereq=0
+80_searchresult_loadprereq=2.0
 90_cleanup_idlesleep=300000
 90_cleanup_busysleep=300000
 90_cleanup_memprereq=0
+90_cleanup_loadprereq=3.0
 
 # additional attributes:
 # performanceIO is a percent-value. a value of 10 means, that 10% of the busysleep time

htroot/PerformanceQueues_p.html

@@ -29,6 +29,7 @@
 	        <td>Delay between<br />idle loops</td>
 	        <td>Delay between<br />busy loops</td>
 	        <td>Minimum of<br />Required Memory</td>
+	        <td>Maximum of<br />System-Load</td>
 	        <td>Full Description</td>
 	      </tr>
 	      #{table}#
@@ -51,6 +52,7 @@
 	        <td align="right"><input name="#[threadname]#_idlesleep" size="7" maxlength="7" value="#[idlesleep]#" #(disabled)#:: disabled="disabled"#(/disabled)# /> milliseconds</td>
 	        <td align="right"><input name="#[threadname]#_busysleep" size="7" maxlength="7" value="#[busysleep]#" #(disabled)#:: disabled="disabled"#(/disabled)# /> milliseconds</td>
 	        <td align="right"><input name="#[threadname]#_memprereq" size="7" maxlength="10" value="#[memprereq]#" #(disabled)#:: disabled="disabled"#(/disabled)# /> kbytes</td>
+	        <td align="right"><input name="#[threadname]#_loadprereq" size="7" maxlength="7" value="#[loadprereq]#" #(disabled)#:: disabled="disabled"#(/disabled)# /> load</td>
 	        <td align="left">#[longdescr]##(recommendation)#::<br/>recommended: #[value]# kbytes#(/recommendation)#</td>
 	      </tr>
 	      #{/table}#

htroot/PerformanceQueues_p.java

@@ -119,6 +119,7 @@ public class PerformanceQueues_p {
         // set templates for latest news from the threads
         long blocktime, sleeptime, exectime;
         long idlesleep, busysleep, memuse, memprereq;
+        double loadprereq;
         int queuesize;
         threads = sb.threadNames();
         int c = 0;
@@ -177,18 +178,21 @@ public class PerformanceQueues_p {
             idlesleep = sb.getConfigLong(threadName + "_idlesleep" , 1000);
             busysleep = sb.getConfigLong(threadName + "_busysleep",   100);
             memprereq = sb.getConfigLong(threadName + "_memprereq",     0);
+            loadprereq = sb.getConfigFloat(threadName + "_loadprereq",  9);
             if (setDelay && post != null) {
                 // load with new values
                 idlesleep = post.getLong(threadName + "_idlesleep", idlesleep);
                 busysleep = post.getLong(threadName + "_busysleep", busysleep);
                 memprereq = post.getLong(threadName + "_memprereq", memprereq) * 1024l;
                 if (memprereq == 0) memprereq = sb.getConfigLong(threadName + "_memprereq", 0);
+                loadprereq = post.getDouble(threadName + "_loadprereq", loadprereq);
+                if (loadprereq == 0) loadprereq = sb.getConfigFloat(threadName + "_loadprereq",  9);
 
                 // check values to prevent short-cut loops
                 if (idlesleep < 1000) idlesleep = 1000;
-                if (threadName.equals("10_httpd")) { idlesleep = 0; busysleep = 0; memprereq = 0; }
+                if (threadName.equals("10_httpd")) { idlesleep = 0; busysleep = 0; memprereq = 0; loadprereq = 9; }
 
-                sb.setThreadPerformance(threadName, idlesleep, busysleep, memprereq);
+                sb.setThreadPerformance(threadName, idlesleep, busysleep, memprereq, loadprereq);
                 idlesleep = sb.getConfigLong(threadName + "_idlesleep", idlesleep);
                 busysleep = sb.getConfigLong(threadName + "_busysleep", busysleep);
             }
@@ -205,14 +209,15 @@ public class PerformanceQueues_p {
 
                     // check values to prevent short-cut loops
                     if (idlesleep < 1000) idlesleep = 1000;
-                    if (threadName.equals("10_httpd")) { idlesleep = 0; busysleep = 0; memprereq = 0; }
+                    if (threadName.equals("10_httpd")) { idlesleep = 0; busysleep = 0; memprereq = 0; loadprereq = 9; }
                     //if (threadName.equals(plasmaSwitchboardConstants.CRAWLJOB_LOCAL_CRAWL) && (busysleep < 50)) busysleep = 50;
-                    sb.setThreadPerformance(threadName, idlesleep, busysleep, memprereq);
+                    sb.setThreadPerformance(threadName, idlesleep, busysleep, memprereq, loadprereq);
                 }
             }
             prop.put("table_" + c + "_idlesleep", idlesleep);
             prop.put("table_" + c + "_busysleep", busysleep);
             prop.put("table_" + c + "_memprereq", memprereq / 1024);
+            prop.put("table_" + c + "_loadprereq", loadprereq);
             // disallow setting of memprereq for indexer to prevent db from throwing OOMs
             // prop.put("table_" + c + "_disabled", /*(threadName.endsWith("_indexing")) ? 1 :*/ "0");
             prop.put("table_" + c + "_disabled", threadName.equals("10_httpd") ? "1" : "0" ); // httpd hardcoded defaults

htroot/PerformanceQueues_p.xml

@@ -20,6 +20,7 @@
 	        <delayIdleLoop>#[idlesleep]#</delayIdleLoop>
 	        <delayBusyLoop>#[busysleep]#</delayBusyLoop>
 	        <minMemory>#[memprereq]#</minMemory>
+	        <minMemory>#[loadprereq]#</minMemory>
 	        <description><![CDATA[#[longdescr]##(recommendation)#::<br/>recommended: #[value]##(/recommendation)#]]></description>
         </Task>
 	#{/table}#</Tasks>

source/net/yacy/kelondro/util/MemoryControl.java

@@ -25,6 +25,8 @@
 
 package net.yacy.kelondro.util;
 
+import java.lang.management.ManagementFactory;
+
 /**
  * Use this to get information about memory usage or try to free some memory
  */
@@ -175,6 +177,14 @@ public class MemoryControl {
     	getStrategy().setProperMbyte(mbyte);
     }
     
+    /**
+     * get the system load within the last minute
+     * @return the system load or a negative number if the load is not available
+     */
+    public static double load() {
+        return ManagementFactory.getOperatingSystemMXBean().getSystemLoadAverage();
+    }
+
     /**
      * main
      * @param args use 'force' to request by force, use 'std' / 'gen' to specify strategy

source/net/yacy/kelondro/workflow/AbstractBusyThread.java

@@ -37,6 +37,7 @@ public abstract class AbstractBusyThread extends AbstractThread implements BusyT
     private long startup = 0, intermission = 0, idlePause = 0, busyPause = 0;
     private long idletime = 0, memprereq = 0;
     private long idleCycles = 0, busyCycles = 0, outofmemoryCycles = 0;
+    private double loadprereq = 9;
     private boolean intermissionObedient = true;
     private final Object syncObject = new Object();
     
@@ -84,6 +85,11 @@ public abstract class AbstractBusyThread extends AbstractThread implements BusyT
         memprereq = freeBytes;
     }
     
+    public void setLoadPreReqisite(final double load) {
+        // sets minimum required amount of memory for the job execution
+        loadprereq = load;
+    }
+    
     public void setObeyIntermission(final boolean obey) {
         // defines if the thread should obey the intermission command
         intermissionObedient = obey;
@@ -158,6 +164,11 @@ public abstract class AbstractBusyThread extends AbstractThread implements BusyT
                 ratz(this.idlePause);
                 idletime += System.currentTimeMillis() - timestamp;
             //} else if ((memnow = serverMemory.available()) > memprereq) try {
+            } else if (MemoryControl.load() > loadprereq) {
+            	logSystem("Thread '" + this.getName() + "' runs high load cycle. current: " + MemoryControl.load() + " max.: " + loadprereq);
+                timestamp = System.currentTimeMillis();
+                ratz(this.idlePause);
+                idletime += System.currentTimeMillis() - timestamp;
             } else if (MemoryControl.request(memprereq, false)) try {
                 // do job
                 timestamp = System.currentTimeMillis();

source/net/yacy/kelondro/workflow/BusyThread.java

@@ -65,6 +65,12 @@ public interface BusyThread extends WorkflowThread {
      */
     public void setMemPreReqisite(long freeBytes);
     
+    /**
+     * sets maximimum load for the job execution
+     * @param load
+     */
+    public void setLoadPreReqisite(final double load);
+ 
     /**
      * defines if the thread should obey the intermission command
      * @param obey

source/net/yacy/kelondro/workflow/InstantBusyThread.java

@@ -165,6 +165,7 @@ public final class InstantBusyThread extends AbstractBusyThread implements BusyT
         thread.setIdleSleep(-1);
         thread.setBusySleep(-1);
         thread.setMemPreReqisite(0);
+        thread.setLoadPreReqisite(3);
         thread.start();
         return thread;
     }

source/net/yacy/search/Switchboard.java

@@ -1094,7 +1094,8 @@ public final class Switchboard extends serverSwitch {
             60000,
             Long.parseLong(getConfig(SwitchboardConstants.INDEX_DIST_IDLESLEEP, "5000")),
             Long.parseLong(getConfig(SwitchboardConstants.INDEX_DIST_BUSYSLEEP, "0")),
-            Long.parseLong(getConfig(SwitchboardConstants.INDEX_DIST_MEMPREREQ, "1000000")));
+            Long.parseLong(getConfig(SwitchboardConstants.INDEX_DIST_MEMPREREQ, "1000000")),
+            Double.parseDouble(getConfig(SwitchboardConstants.INDEX_DIST_LOADPREREQ, "9.0")));
 
         // content control: initialize list sync thread
         deployThread(

source/net/yacy/search/SwitchboardConstants.java

@@ -62,6 +62,7 @@ public final class SwitchboardConstants {
     public static final String INDEX_DIST_METHOD_JOBCOUNT   = null;
     public static final String INDEX_DIST_METHOD_FREEMEM    = null;
     public static final String INDEX_DIST_MEMPREREQ         = "20_dhtdistribution_memprereq";
+    public static final String INDEX_DIST_LOADPREREQ        = "20_dhtdistribution_loadprereq";
     public static final String INDEX_DIST_IDLESLEEP         = "20_dhtdistribution_idlesleep";
     public static final String INDEX_DIST_BUSYSLEEP         = "20_dhtdistribution_busysleep";
     // 30_peerping
@@ -132,6 +133,7 @@ public final class SwitchboardConstants {
      */
     public static final String SURROGATES                      = "70_surrogates";
     public static final String SURROGATES_MEMPREREQ            = "70_surrogates_memprereq";
+    public static final String SURROGATES_LOADPREREQ           = "70_surrogates_loadprereq";
     public static final String SURROGATES_IDLESLEEP            = "70_surrogates_idlesleep";
     public static final String SURROGATES_BUSYSLEEP            = "70_surrogates_busysleep";
     public static final String SURROGATES_METHOD_START         = "surrogateProcess";
@@ -144,6 +146,7 @@ public final class SwitchboardConstants {
      */
     public static final String SEARCHRESULT                      = "80_searchresult";
     public static final String SEARCHRESULT_MEMPREREQ            = "80_searchresult_memprereq";
+    public static final String SEARCHRESULT_LOADPREREQ           = "80_searchresult_loadprereq";
     public static final String SEARCHRESULT_IDLESLEEP            = "80_searchresult_idlesleep";
     public static final String SEARCHRESULT_BUSYSLEEP            = "80_searchresult_busysleep";
     public static final String SEARCHRESULT_METHOD_START         = "searchresultProcess";

source/net/yacy/server/serverSwitch.java

@@ -344,7 +344,8 @@ public class serverSwitch
             startupDelay,
             Long.parseLong(getConfig(threadName + "_idlesleep", "100")),
             Long.parseLong(getConfig(threadName + "_busysleep", "1000")),
-            Long.parseLong(getConfig(threadName + "_memprereq", "1000000")));
+            Long.parseLong(getConfig(threadName + "_memprereq", "1000000")),
+            Double.parseDouble(getConfig(threadName + "_loadprereq", "9.0")));
     }
 
     public void deployThread(
@@ -356,7 +357,8 @@ public class serverSwitch
         final long startupDelay,
         final long initialIdleSleep,
         final long initialBusySleep,
-        final long initialMemoryPreRequisite) {
+        final long initialMemoryPreRequisite,
+        final double initialLoadPreRequisite) {
         if ( newThread.isAlive() ) {
             throw new RuntimeException(
                 "undeployed threads must not live; they are started as part of the deployment");
@@ -384,6 +386,13 @@ public class serverSwitch
             newThread.setMemPreReqisite(initialMemoryPreRequisite);
             setConfig(threadName + "_memprereq", initialMemoryPreRequisite);
         }
+        try {
+            final double load = Double.parseDouble(getConfig(threadName + "_loadprereq", "novalue"));
+            newThread.setLoadPreReqisite(load);
+        } catch (final NumberFormatException e ) {
+            newThread.setLoadPreReqisite(initialLoadPreRequisite);
+            setConfig(threadName + "_loadprereq", (float)initialLoadPreRequisite);
+        }
         newThread.setDescription(threadShortDescription, threadLongDescription, threadMonitorURL);
         this.workerThreads.put(threadName, newThread);
         // start the thread
@@ -400,13 +409,16 @@ public class serverSwitch
         final String threadName,
         final long idleMillis,
         final long busyMillis,
-        final long memprereqBytes) {
+        final long memprereqBytes,
+        final double loadprereq) {
         final BusyThread thread = this.workerThreads.get(threadName);
         if ( thread != null ) {
             setConfig(threadName + "_idlesleep", thread.setIdleSleep(idleMillis));
             setConfig(threadName + "_busysleep", thread.setBusySleep(busyMillis));
             setConfig(threadName + "_memprereq", memprereqBytes);
             thread.setMemPreReqisite(memprereqBytes);
+            setConfig(threadName + "_loadprereq", (float)loadprereq);
+            thread.setLoadPreReqisite(loadprereq);
         }
     }