Marown/yacy_search_server
1
0
Fork 0
mirror of https://github.com/yacy/yacy_search_server.git synced 2024-09-19 00:01:41 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity

*) Making CrawlStacker pool configurable via GUI and config file

Browse Source 
See: http://www.yacy-forum.de/viewtopic.php?t=1448

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@1087 6c8d7289-2bf4-0310-a012-ef5d649a1542
This commit is contained in:
theli 2005-11-15 12:46:22 +00:00
parent 3631cb1f6d
commit d0dfccdb77
3 changed files with 63 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
htroot/PerformanceQueues_p.java
View File

@ -224,6 +224,25 @@ public class PerformanceQueues_p {
switchboard.setConfig("httpdMaxActiveSessions",maxActive);
switchboard.setConfig("httpdMaxIdleSessions",maxIdle);
switchboard.setConfig("httpdMinIdleSessions",minIdle);
/*
* Configuring the crawlStacker pool
*/
GenericObjectPool.Config stackerPoolConfig = switchboard.sbStackCrawlThread.getPoolConfig();
maxActive = Integer.parseInt(post.get("CrawlStacker Session Pool_maxActive","10"));
maxIdle = Integer.parseInt(post.get("CrawlStacker Session Pool_maxIdle","10"));
minIdle = Integer.parseInt(post.get("CrawlStacker Session Pool_minIdle","5"));
stackerPoolConfig.minIdle = (minIdle > maxIdle) ? maxIdle/2 : minIdle;
stackerPoolConfig.maxIdle = (maxIdle > maxActive) ? maxActive/2 : maxIdle;
stackerPoolConfig.maxActive = maxActive;
switchboard.sbStackCrawlThread.setPoolConfig(stackerPoolConfig);
// storing the new values into configfile
switchboard.setConfig("stacker.MaxActiveThreads",maxActive);
switchboard.setConfig("stacker.MaxIdleThreads",maxIdle);
switchboard.setConfig("stacker.MinIdleThreads",minIdle);
}
if ((post != null) && (post.containsKey("proxyControlSubmit"))) {
@ -266,9 +285,14 @@ public class PerformanceQueues_p {
prop.put("pool_1_name","httpd Session Pool");
prop.put("pool_1_maxActive",httpdPoolConfig.maxActive);
prop.put("pool_1_maxIdle",httpdPoolConfig.maxIdle);
prop.put("pool_1_minIdle",httpdPoolConfig.minIdle);
prop.put("pool",2);
prop.put("pool_1_minIdle",httpdPoolConfig.minIdle);
GenericObjectPool.Config stackerPoolConfig = switchboard.sbStackCrawlThread.getPoolConfig();
prop.put("pool_2_name","CrawlStacker Session Pool");
prop.put("pool_2_maxActive",stackerPoolConfig.maxActive);
prop.put("pool_2_maxIdle",stackerPoolConfig.maxIdle);
prop.put("pool_2_minIdle",stackerPoolConfig.minIdle);
prop.put("pool",3);
// return rewrite values for templates
return prop;

33
source/de/anomic/plasma/plasmaCrawlStacker.java
View File

@ -72,6 +72,7 @@ import de.anomic.yacy.yacyCore;
public final class plasmaCrawlStacker {
final WorkerPool theWorkerPool;
private GenericObjectPool.Config theWorkerPoolConfig = null;
final ThreadGroup theWorkerThreadGroup = new ThreadGroup("stackCrawlThreadGroup");
final serverLog log = new serverLog("STACKCRAWL");
final plasmaSwitchboard sb;
@ -85,10 +86,40 @@ public final class plasmaCrawlStacker {
this.log.logInfo(this.queue.size() + " entries in the stackCrawl queue.");
this.log.logInfo("STACKCRAWL thread initialized.");
this.theWorkerPool = new WorkerPool(new WorkterFactory(this.theWorkerThreadGroup));
// configuring the thread pool
// implementation of session thread pool
this.theWorkerPoolConfig = new GenericObjectPool.Config();
// The maximum number of active connections that can be allocated from pool at the same time,
// 0 for no limit
this.theWorkerPoolConfig.maxActive = Integer.parseInt(sb.getConfig("stacker.MaxActiveThreads","50"));
// The maximum number of idle connections connections in the pool
// 0 = no limit.
this.theWorkerPoolConfig.maxIdle = Integer.parseInt(sb.getConfig("stacker.MaxIdleThreads","10"));
this.theWorkerPoolConfig.minIdle = Integer.parseInt(sb.getConfig("stacker.MinIdleThreads","5"));
// block undefinitely
this.theWorkerPoolConfig.maxWait = -1;
// Action to take in case of an exhausted DBCP statement pool
// 0 = fail, 1 = block, 2= grow
this.theWorkerPoolConfig.whenExhaustedAction = GenericObjectPool.WHEN_EXHAUSTED_BLOCK;
this.theWorkerPoolConfig.minEvictableIdleTimeMillis = 30000;
// creating worker pool
this.theWorkerPool = new WorkerPool(new WorkterFactory(this.theWorkerThreadGroup),this.theWorkerPoolConfig);
}
public GenericObjectPool.Config getPoolConfig() {
return this.theWorkerPoolConfig;
}
public void setPoolConfig(GenericObjectPool.Config newConfig) {
this.theWorkerPool.setConfig(newConfig);
}
public void close() {
try {
this.log.logFine("Shutdown. Terminationg worker threads.");

5
yacy.init
View File

@ -559,6 +559,11 @@ crawler.MaxActiveThreads = 10
crawler.MaxIdleThreads = 7
crawler.MinIdleThreads = 5
# maximum number of crawl-stacker threads
stacker.MaxActiveThreads = 50
stacker.MaxIdleThreads = 10
stacker.MinIdleThreads = 5
# maximum size of indexing queue
indexer.slots = 100