mirror of
https://github.com/yacy/yacy_search_server.git
synced 2024-09-19 00:01:41 +02:00
*) allow pausing/resuming of crawlJob Threads separately
- pausing/resuming localCrawls - pausing/resuming remoteTriggeredCrawls - pausing/resuming globalCrawlTrigger See: http://www.yacy-forum.de/viewtopic.php?t=1591 git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@1723 6c8d7289-2bf4-0310-a012-ef5d649a1542
This commit is contained in:
parent
af1da090bb
commit
2336f0f013
|
@ -281,12 +281,12 @@ public class IndexCreate_p {
|
|||
|
||||
|
||||
if (post.containsKey("pausecrawlqueue")) {
|
||||
switchboard.pauseCrawling();
|
||||
switchboard.pauseCrawlJob(plasmaSwitchboard.CRAWLJOB_LOCAL_CRAWL);
|
||||
prop.put("info", 4);//crawling paused
|
||||
}
|
||||
|
||||
if (post.containsKey("continuecrawlqueue")) {
|
||||
switchboard.continueCrawling();
|
||||
switchboard.continueCrawlJob(plasmaSwitchboard.CRAWLJOB_LOCAL_CRAWL);
|
||||
prop.put("info", 5);//crawling continued
|
||||
}
|
||||
}
|
||||
|
@ -454,7 +454,7 @@ public class IndexCreate_p {
|
|||
}
|
||||
|
||||
|
||||
prop.put("crawler-paused",(switchboard.crawlingIsPaused())?0:1);
|
||||
prop.put("crawler-paused",(switchboard.crawlJobIsPaused(plasmaSwitchboard.CRAWLJOB_LOCAL_CRAWL))?0:1);
|
||||
|
||||
// return rewrite properties
|
||||
return prop;
|
||||
|
|
|
@ -74,7 +74,9 @@ public class ScreenSaver {
|
|||
public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch env) {
|
||||
|
||||
plasmaSwitchboard sb = (plasmaSwitchboard)env;
|
||||
boolean crawlingStarted = false;
|
||||
boolean localCrawlStarted = false;
|
||||
boolean remoteTriggeredCrawlStarted = false;
|
||||
boolean globalCrawlTriggerStarted = false;
|
||||
try {
|
||||
InputStream input = (InputStream) header.get("INPUTSTREAM");
|
||||
OutputStream output = (OutputStream) header.get("OUTPUTSTREAM");
|
||||
|
@ -110,10 +112,18 @@ public class ScreenSaver {
|
|||
|
||||
outputWriter.println(currentURL);
|
||||
} else if (line.equals("CONTINUECRAWLING")) {
|
||||
if (sb.crawlingIsPaused()) {
|
||||
crawlingStarted = true;
|
||||
sb.continueCrawling();
|
||||
if (sb.crawlJobIsPaused(plasmaSwitchboard.CRAWLJOB_LOCAL_CRAWL)) {
|
||||
localCrawlStarted = true;
|
||||
sb.continueCrawlJob(plasmaSwitchboard.CRAWLJOB_LOCAL_CRAWL);
|
||||
}
|
||||
if (sb.crawlJobIsPaused(plasmaSwitchboard.CRAWLJOB_REMOTE_TRIGGERED_CRAWL)) {
|
||||
remoteTriggeredCrawlStarted = true;
|
||||
sb.continueCrawlJob(plasmaSwitchboard.CRAWLJOB_REMOTE_TRIGGERED_CRAWL);
|
||||
}
|
||||
if (sb.crawlJobIsPaused(plasmaSwitchboard.CRAWLJOB_GLOBAL_CRAWL_TRIGGER)) {
|
||||
globalCrawlTriggerStarted = true;
|
||||
sb.continueCrawlJob(plasmaSwitchboard.CRAWLJOB_GLOBAL_CRAWL_TRIGGER);
|
||||
}
|
||||
} else if (line.equals("EXIT")) {
|
||||
outputWriter.println("OK");
|
||||
outputWriter.flush();
|
||||
|
@ -128,9 +138,15 @@ public class ScreenSaver {
|
|||
} catch (Exception e) {
|
||||
return null;
|
||||
} finally {
|
||||
if (crawlingStarted) {
|
||||
sb.pauseCrawling();
|
||||
if (localCrawlStarted) {
|
||||
sb.pauseCrawlJob(plasmaSwitchboard.CRAWLJOB_LOCAL_CRAWL);
|
||||
}
|
||||
if (remoteTriggeredCrawlStarted) {
|
||||
sb.pauseCrawlJob(plasmaSwitchboard.CRAWLJOB_REMOTE_TRIGGERED_CRAWL);
|
||||
}
|
||||
if (globalCrawlTriggerStarted) {
|
||||
sb.pauseCrawlJob(plasmaSwitchboard.CRAWLJOB_GLOBAL_CRAWL_TRIGGER);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
@ -78,10 +78,22 @@ public class Status {
|
|||
prop.put("LOCATION","");
|
||||
}
|
||||
} else {
|
||||
if (post.containsKey("pausecrawlqueue")) {
|
||||
((plasmaSwitchboard)env).pauseCrawling();
|
||||
} else if (post.containsKey("continuecrawlqueue")) {
|
||||
((plasmaSwitchboard)env).continueCrawling();
|
||||
if (post.containsKey("pauseCrawlJob")) {
|
||||
String jobType = (String) post.get("jobType");
|
||||
if (jobType.equals("localCrawl"))
|
||||
((plasmaSwitchboard)env).pauseCrawlJob(plasmaSwitchboard.CRAWLJOB_LOCAL_CRAWL);
|
||||
else if (jobType.equals("remoteTriggeredCrawl"))
|
||||
((plasmaSwitchboard)env).pauseCrawlJob(plasmaSwitchboard.CRAWLJOB_REMOTE_TRIGGERED_CRAWL);
|
||||
else if (jobType.equals("globalCrawlTrigger"))
|
||||
((plasmaSwitchboard)env).pauseCrawlJob(plasmaSwitchboard.CRAWLJOB_GLOBAL_CRAWL_TRIGGER);
|
||||
} else if (post.containsKey("continueCrawlJob")) {
|
||||
String jobType = (String) post.get("jobType");
|
||||
if (jobType.equals("localCrawl"))
|
||||
((plasmaSwitchboard)env).continueCrawlJob(plasmaSwitchboard.CRAWLJOB_LOCAL_CRAWL);
|
||||
else if (jobType.equals("remoteTriggeredCrawl"))
|
||||
((plasmaSwitchboard)env).continueCrawlJob(plasmaSwitchboard.CRAWLJOB_REMOTE_TRIGGERED_CRAWL);
|
||||
else if (jobType.equals("globalCrawlTrigger"))
|
||||
((plasmaSwitchboard)env).continueCrawlJob(plasmaSwitchboard.CRAWLJOB_GLOBAL_CRAWL_TRIGGER);
|
||||
} else if (post.containsKey("ResetTraffic")) {
|
||||
httpdByteCountInputStream.resetCount();
|
||||
httpdByteCountOutputStream.resetCount();
|
||||
|
@ -278,13 +290,18 @@ public class Status {
|
|||
prop.put("indexingQueueMax", Integer.toString(plasmaSwitchboard.indexingSlots));
|
||||
|
||||
prop.put("loaderQueueSize", Integer.toString(sb.cacheLoader.size()));
|
||||
prop.put("loaderQueueMax", Integer.toString(plasmaSwitchboard.crawlSlots));
|
||||
prop.put("loaderPaused",sb.crawlingIsPaused()?1:0);
|
||||
prop.put("loaderQueueMax", Integer.toString(plasmaSwitchboard.crawlSlots));
|
||||
|
||||
prop.put("localCrawlQueueSize", Integer.toString(sb.getThread("50_localcrawl").getJobCount()));
|
||||
prop.put("localCrawlQueueSize", Integer.toString(sb.getThread(plasmaSwitchboard.CRAWLJOB_LOCAL_CRAWL).getJobCount()));
|
||||
prop.put("localCrawlPaused",sb.crawlJobIsPaused(plasmaSwitchboard.CRAWLJOB_LOCAL_CRAWL)?1:0);
|
||||
|
||||
prop.put("remoteTriggeredCrawlQueueSize", Integer.toString(sb.getThread(plasmaSwitchboard.CRAWLJOB_REMOTE_TRIGGERED_CRAWL).getJobCount()));
|
||||
prop.put("remoteTriggeredCrawlPaused",sb.crawlJobIsPaused(plasmaSwitchboard.CRAWLJOB_REMOTE_TRIGGERED_CRAWL)?1:0);
|
||||
|
||||
prop.put("globalCrawlTriggerQueueSize", Integer.toString(sb.getThread(plasmaSwitchboard.CRAWLJOB_GLOBAL_CRAWL_TRIGGER).getJobCount()));
|
||||
prop.put("globalCrawlTriggerPaused",sb.crawlJobIsPaused(plasmaSwitchboard.CRAWLJOB_GLOBAL_CRAWL_TRIGGER)?1:0);
|
||||
|
||||
prop.put("stackCrawlQueueSize", Integer.toString(sb.sbStackCrawlThread.size()));
|
||||
prop.put("remoteCrawlQueueSize", Integer.toString(sb.getThread("62_remotetriggeredcrawl").getJobCount()));
|
||||
prop.put("remoteCrawlTriggerQueueSize", Integer.toString(sb.getThread("61_globalcrawltrigger").getJobCount()));
|
||||
|
||||
// return rewrite properties
|
||||
prop.put("date",(new Date()).toString());
|
||||
|
|
|
@ -1,4 +1,4 @@
|
|||
<table border="0" cellpadding="2" cellspacing="1" width="100%">
|
||||
<table border="1" cellpadding="2" cellspacing="1" width="100%">
|
||||
<tr class="TableHeader">
|
||||
<td colspan="3"><b>Private System Properties</b></td>
|
||||
</tr>
|
||||
|
@ -71,17 +71,42 @@
|
|||
</tr>
|
||||
<tr class="TableCellLight">
|
||||
<td>Loader Queue</td>
|
||||
<td>#[loaderQueueSize]# | #[loaderQueueMax]# #(loaderPaused)#::(paused)#(/loaderPaused)# <a href="Status.html?#(loaderPaused)#pausecrawlqueue::continuecrawlqueue#(/loaderPaused)#=" title="#(loaderPaused)#pause crawling::continue crawling#(/loaderPaused)#"><img src="env/grafics/#(loaderPaused)#stop.gif::start.gif#(/loaderPaused)#" border="0" width="12" height="12"></a></td>
|
||||
<td>#[loaderQueueSize]# | #[loaderQueueMax]# </td>
|
||||
<td>[<a href="IndexCreateLoaderQueue_p.html">Details</a>]</td>
|
||||
</tr>
|
||||
<tr class="TableCellDark">
|
||||
<td>Crawler Queue</td>
|
||||
<td>Enqueued from: local=#[localCrawlQueueSize]# remote=#[remoteCrawlQueueSize]# | Pending: #[stackCrawlQueueSize]#</td>
|
||||
<td>[<a href="IndexCreateWWWLocalQueue_p.html">Details</a>]</td>
|
||||
</tr>
|
||||
<tr class="TableCellLight">
|
||||
<td>Remote Crawl Trigger Queue</td>
|
||||
<td>#[remoteCrawlTriggerQueueSize]#</td>
|
||||
<td>[<a href="IndexCreateWWWGlobalQueue_p.html">Details</a>]</td>
|
||||
<td>Crawler Queues</td>
|
||||
<td>
|
||||
<table>
|
||||
<tr>
|
||||
<td>Local Crawl</td>
|
||||
<td>#[localCrawlQueueSize]#</td>
|
||||
<td><a href="Status.html?#(localCrawlPaused)#pauseCrawlJob::continueCrawlJob#(/localCrawlPaused)#=&jobType=localCrawl" title="#(localCrawlPaused)#pause local crawl::continue local crawl#(/localCrawlPaused)#"><img src="env/grafics/#(localCrawlPaused)#stop.gif::start.gif#(/localCrawlPaused)#" border="0" width="12" height="12"></a></td>
|
||||
<td>#(localCrawlPaused)# ::(paused)#(/localCrawlPaused)#</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>Remote triggered Crawl</td>
|
||||
<td>#[remoteTriggeredCrawlQueueSize]#</td>
|
||||
<td><a href="Status.html?#(remoteTriggeredCrawlPaused)#pauseCrawlJob::continueCrawlJob#(/remoteTriggeredCrawlPaused)#=&jobType=remoteTriggeredCrawl" title="#(remoteTriggeredCrawlPaused)#pause remote triggered crawl::continue remote triggered crawl#(/remoteTriggeredCrawlPaused)#"><img src="env/grafics/#(remoteTriggeredCrawlPaused)#stop.gif::start.gif#(/remoteTriggeredCrawlPaused)#" border="0" width="12" height="12"></a></td>
|
||||
<td>#(remoteTriggeredCrawlPaused)# ::(paused)#(/remoteTriggeredCrawlPaused)#</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>Global Crawl Trigger</td>
|
||||
<td>#[globalCrawlTriggerQueueSize]#</td>
|
||||
<td><a href="Status.html?#(globalCrawlTriggerPaused)#pauseCrawlJob::continueCrawlJob#(/globalCrawlTriggerPaused)#=&jobType=globalCrawlTrigger" title="#(globalCrawlTriggerPaused)#pause global crawl trigger::continue global crawl trigger#(/globalCrawlTriggerPaused)#"><img src="env/grafics/#(globalCrawlTriggerPaused)#stop.gif::start.gif#(/globalCrawlTriggerPaused)#" border="0" width="12" height="12"></a></td>
|
||||
<td>#(globalCrawlTriggerPaused)# ::(paused)#(/globalCrawlTriggerPaused)#</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>Pending Crawl</td>
|
||||
<td>#[stackCrawlQueueSize]#</td>
|
||||
<td> </td>
|
||||
<td> </td>
|
||||
</tr>
|
||||
</table>
|
||||
</td>
|
||||
<td valign="top">[<a href="IndexCreateWWWLocalQueue_p.html">Details</a>]<br>
|
||||
<br>
|
||||
[<a href="IndexCreateWWWGlobalQueue_p.html">Details</a>]
|
||||
</td>
|
||||
</tr>
|
||||
</table>
|
|
@ -530,7 +530,8 @@ public final class plasmaCrawlWorker extends Thread {
|
|||
} else if ((errorMsg != null) && (errorMsg.indexOf("There is not enough space on the disk") >= 0)) {
|
||||
log.logSevere("CRAWLER Not enough space on the disk detected while crawling '" + url.toString() + "'. " +
|
||||
"Pausing crawlers. ");
|
||||
plasmaCrawlLoader.switchboard.pauseCrawling();
|
||||
plasmaCrawlLoader.switchboard.pauseCrawlJob(plasmaSwitchboard.CRAWLJOB_LOCAL_CRAWL);
|
||||
plasmaCrawlLoader.switchboard.pauseCrawlJob(plasmaSwitchboard.CRAWLJOB_REMOTE_TRIGGERED_CRAWL);
|
||||
} else if ((errorMsg != null) && (errorMsg.indexOf("Network is unreachable") >=0)) {
|
||||
log.logSevere("CRAWLER Network is unreachable while trying to crawl URL '" + url.toString() + "'. ");
|
||||
} else if ((errorMsg != null) && (errorMsg.indexOf("No trusted certificate found")>= 0)) {
|
||||
|
|
|
@ -113,6 +113,7 @@ import java.text.SimpleDateFormat;
|
|||
import java.util.ArrayList;
|
||||
import java.util.Date;
|
||||
import java.util.HashMap;
|
||||
import java.util.Hashtable;
|
||||
import java.util.Iterator;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
|
@ -228,8 +229,17 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser
|
|||
private serverSemaphore shutdownSync = new serverSemaphore(0);
|
||||
private boolean terminate = false;
|
||||
|
||||
private Object crawlingPausedSync = new Object();
|
||||
private boolean crawlingIsPaused = false;
|
||||
//private Object crawlingPausedSync = new Object();
|
||||
//private boolean crawlingIsPaused = false;
|
||||
|
||||
public static final String CRAWLJOB_LOCAL_CRAWL = "50_localcrawl";
|
||||
public static final String CRAWLJOB_REMOTE_TRIGGERED_CRAWL = "62_remotetriggeredcrawl";
|
||||
public static final String CRAWLJOB_GLOBAL_CRAWL_TRIGGER = "61_globalcrawltrigger";
|
||||
private static final int CRAWLJOB_SYNC = 0;
|
||||
private static final int CRAWLJOB_STATUS = 1;
|
||||
|
||||
private Hashtable crawlJobsStatus = new Hashtable();
|
||||
|
||||
private static plasmaSwitchboard sb;
|
||||
|
||||
public plasmaSwitchboard(String rootPath, String initPath, String configPath) {
|
||||
|
@ -421,11 +431,25 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser
|
|||
|
||||
// start a loader
|
||||
log.logConfig("Starting Crawl Loader");
|
||||
|
||||
crawlSlots = Integer.parseInt(getConfig("crawler.MaxActiveThreads", "10"));
|
||||
this.crawlingIsPaused = Boolean.valueOf(getConfig("crawler.isPaused", "false")).booleanValue();
|
||||
plasmaCrawlLoader.switchboard = this;
|
||||
this.cacheLoader = new plasmaCrawlLoader(this.cacheManager, this.log);
|
||||
|
||||
/*
|
||||
* Creating sync objects and loading status for the crawl jobs
|
||||
* a) local crawl
|
||||
* b) remote triggered crawl
|
||||
* c) global crawl trigger
|
||||
*/
|
||||
this.crawlJobsStatus.put(CRAWLJOB_LOCAL_CRAWL, new Object[]{
|
||||
new Object(),
|
||||
Boolean.valueOf(getConfig(CRAWLJOB_LOCAL_CRAWL + "_isPaused", "false"))});
|
||||
this.crawlJobsStatus.put(CRAWLJOB_REMOTE_TRIGGERED_CRAWL, new Object[]{
|
||||
new Object(),
|
||||
Boolean.valueOf(getConfig(CRAWLJOB_REMOTE_TRIGGERED_CRAWL + "_isPaused", "false"))});
|
||||
this.crawlJobsStatus.put(CRAWLJOB_GLOBAL_CRAWL_TRIGGER, new Object[]{
|
||||
new Object(),
|
||||
Boolean.valueOf(getConfig(CRAWLJOB_GLOBAL_CRAWL_TRIGGER + "_isPaused", "false"))});
|
||||
|
||||
// starting board
|
||||
initMessages(ramMessage);
|
||||
|
@ -956,32 +980,35 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser
|
|||
/**
|
||||
* With this function the crawling process can be paused
|
||||
*/
|
||||
public void pauseCrawling() {
|
||||
synchronized(this.crawlingPausedSync) {
|
||||
this.crawlingIsPaused = true;
|
||||
public void pauseCrawlJob(String jobType) {
|
||||
Object[] status = (Object[])this.crawlJobsStatus.get(jobType);
|
||||
synchronized(status[CRAWLJOB_SYNC]) {
|
||||
status[CRAWLJOB_STATUS] = Boolean.TRUE;
|
||||
}
|
||||
setConfig("crawler.isPaused", "true");
|
||||
}
|
||||
setConfig(jobType + "_isPaused", "true");
|
||||
}
|
||||
|
||||
/**
|
||||
* Continue the previously paused crawling
|
||||
*/
|
||||
public void continueCrawling() {
|
||||
synchronized(this.crawlingPausedSync) {
|
||||
if (this.crawlingIsPaused) {
|
||||
this.crawlingIsPaused = false;
|
||||
this.crawlingPausedSync.notifyAll();
|
||||
public void continueCrawlJob(String jobType) {
|
||||
Object[] status = (Object[])this.crawlJobsStatus.get(jobType);
|
||||
synchronized(status[CRAWLJOB_SYNC]) {
|
||||
if (((Boolean)status[CRAWLJOB_STATUS]).booleanValue()) {
|
||||
status[CRAWLJOB_STATUS] = Boolean.FALSE;
|
||||
status[CRAWLJOB_SYNC].notifyAll();
|
||||
}
|
||||
}
|
||||
setConfig("crawler.isPaused", "false");
|
||||
}
|
||||
setConfig(jobType + "_isPaused", "false");
|
||||
}
|
||||
|
||||
/**
|
||||
* @return <code>true</code> if crawling was paused or <code>false</code> otherwise
|
||||
*/
|
||||
public boolean crawlingIsPaused() {
|
||||
synchronized(this.crawlingPausedSync) {
|
||||
return this.crawlingIsPaused;
|
||||
public boolean crawlJobIsPaused(String jobType) {
|
||||
Object[] status = (Object[])this.crawlJobsStatus.get(jobType);
|
||||
synchronized(status[CRAWLJOB_SYNC]) {
|
||||
return ((Boolean)status[CRAWLJOB_STATUS]).booleanValue();
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -1012,10 +1039,11 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser
|
|||
//if (!(cacheManager.idle())) try {Thread.currentThread().sleep(2000);} catch (InterruptedException e) {}
|
||||
|
||||
// if crawling was paused we have to wait until we wer notified to continue
|
||||
synchronized(this.crawlingPausedSync) {
|
||||
if (this.crawlingIsPaused) {
|
||||
Object[] status = (Object[])this.crawlJobsStatus.get(CRAWLJOB_LOCAL_CRAWL);
|
||||
synchronized(status[CRAWLJOB_SYNC]) {
|
||||
if (((Boolean)status[CRAWLJOB_STATUS]).booleanValue()) {
|
||||
try {
|
||||
this.crawlingPausedSync.wait();
|
||||
status[CRAWLJOB_SYNC].wait();
|
||||
}
|
||||
catch (InterruptedException e){ return false;}
|
||||
}
|
||||
|
@ -1090,10 +1118,11 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser
|
|||
//if (!(cacheManager.idle())) try {Thread.currentThread().sleep(2000);} catch (InterruptedException e) {}
|
||||
|
||||
// if crawling was paused we have to wait until we wer notified to continue
|
||||
synchronized(this.crawlingPausedSync) {
|
||||
if (this.crawlingIsPaused) {
|
||||
Object[] status = (Object[])this.crawlJobsStatus.get(CRAWLJOB_GLOBAL_CRAWL_TRIGGER);
|
||||
synchronized(status[CRAWLJOB_SYNC]) {
|
||||
if (((Boolean)status[CRAWLJOB_STATUS]).booleanValue()) {
|
||||
try {
|
||||
this.crawlingPausedSync.wait();
|
||||
status[CRAWLJOB_SYNC].wait();
|
||||
}
|
||||
catch (InterruptedException e){ return false;}
|
||||
}
|
||||
|
@ -1152,12 +1181,13 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser
|
|||
}
|
||||
|
||||
// if crawling was paused we have to wait until we wer notified to continue
|
||||
synchronized(this.crawlingPausedSync) {
|
||||
if (this.crawlingIsPaused) {
|
||||
Object[] status = (Object[])this.crawlJobsStatus.get(CRAWLJOB_REMOTE_TRIGGERED_CRAWL);
|
||||
synchronized(status[CRAWLJOB_SYNC]) {
|
||||
if (((Boolean)status[CRAWLJOB_STATUS]).booleanValue()) {
|
||||
try {
|
||||
this.crawlingPausedSync.wait();
|
||||
status[CRAWLJOB_SYNC].wait();
|
||||
}
|
||||
catch (InterruptedException e){ return false; }
|
||||
catch (InterruptedException e){ return false;}
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
@ -429,12 +429,15 @@ xpstopw=true
|
|||
50_localcrawl_idlesleep=10000
|
||||
50_localcrawl_busysleep=200
|
||||
50_localcrawl_memprereq=1048576
|
||||
50_localcrawl_isPaused=false
|
||||
61_globalcrawltrigger_idlesleep=10000
|
||||
61_globalcrawltrigger_busysleep=200
|
||||
61_globalcrawltrigger_memprereq=1048576
|
||||
61_globalcrawltrigger_isPaused=false
|
||||
62_remotetriggeredcrawl_idlesleep=10000
|
||||
62_remotetriggeredcrawl_busysleep=200
|
||||
62_remotetriggeredcrawl_memprereq=1048576
|
||||
62_remotetriggeredcrawl_isPaused=false
|
||||
70_cachemanager_idlesleep=5000
|
||||
70_cachemanager_busysleep=0
|
||||
70_cachemanager_memprereq=1048576
|
||||
|
@ -561,7 +564,6 @@ onlineCautionDelay=30000
|
|||
crawler.acceptLanguage=en-us,en;q=0.5
|
||||
crawler.acceptCharset=ISO-8859-1,utf-8;q=0.7,*;q=0.7
|
||||
crawler.clientTimeout=9000
|
||||
crawler.isPaused=false
|
||||
|
||||
# maximum number of crawler threads
|
||||
crawler.MaxActiveThreads = 10
|
||||
|
|
Loading…
Reference in New Issue
Block a user