Marown/yacy_search_server
1
0
Fork 0
mirror of https://github.com/yacy/yacy_search_server.git synced 2024-09-19 00:01:41 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity

*) indexing queue slots are now configurable via config file

Browse Source 
See: http://www.yacy-forum.de/viewtopic.php?t=1480

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@1081 6c8d7289-2bf4-0310-a012-ef5d649a1542
This commit is contained in:
theli 2005-11-15 08:25:46 +00:00
parent 3c11d7b81c
commit 86a9210264
2 changed files with 20 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

48
source/de/anomic/plasma/plasmaSwitchboard.java
View File

@ -151,6 +151,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser
public static int crawlSlots = 10; public static int crawlSlots = 10;
public static int indexingSlots = 100; public static int indexingSlots = 100;
public static int stackCrawlSlots = 10000; public static int stackCrawlSlots = 10000;
public static int maxCRLDump = 300000; public static int maxCRLDump = 300000;
public static int maxCRGDump = 100000; public static int maxCRGDump = 100000;
@ -241,36 +242,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser
this.remoteProxyConfig = httpRemoteProxyConfig.init(this); this.remoteProxyConfig = httpRemoteProxyConfig.init(this);
this.log.logConfig("Remote proxy configuration:\n" + this.remoteProxyConfig.toString()); this.log.logConfig("Remote proxy configuration:\n" + this.remoteProxyConfig.toString());
// // reading the proxy host name // setting timestamp of last proxy access
// this.remoteProxyHost = getConfig("remoteProxyHost", "");
//
// // reading the proxy host port
// try {
// this.remoteProxyPort = Integer.parseInt(getConfig("remoteProxyPort", "3128"));
// } catch (NumberFormatException e) {
// this.remoteProxyPort = 3128;
// }
//
// // determining if remote proxy should be used for yacy -> yacy communication
// this.remoteProxyUse4Yacy = getConfig("remoteProxyUse4Yacy", "true").equalsIgnoreCase("true");
//
// // determining addresses for which the remote proxy should not be used
// this.remoteProxyNoProxy = getConfig("remoteProxyNoProxy","");
// this.remoteProxyNoProxyPatterns = this.remoteProxyNoProxy.split(",");
//
// // determining if remote Proxy should be used
// if (getConfig("remoteProxyUse", "false").equalsIgnoreCase("true")) {
// this.remoteProxyUse = true;
// this.log.logConfig("Using remote proxy:" +
// "\n\tHost: " + this.remoteProxyHost +
// "\n\tPort: " + this.remoteProxyPort +
// "\n\tUseProxy4Yacy: " + Boolean.toString(this.remoteProxyUse4Yacy)
// );
// } else {
// this.remoteProxyUse = false;
// this.remoteProxyHost = null;
// this.remoteProxyPort = 0;
// }
this.proxyLastAccess = System.currentTimeMillis() - 60000; this.proxyLastAccess = System.currentTimeMillis() - 60000;
crg = new StringBuffer(maxCRGDump); crg = new StringBuffer(maxCRGDump);
//crl = new StringBuffer(maxCRLDump); //crl = new StringBuffer(maxCRLDump);
@ -377,9 +349,17 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser
log.logConfig("Starting Parser"); log.logConfig("Starting Parser");
this.parser = new plasmaParser(); this.parser = new plasmaParser();
// initialize switchboard queue /* ======================================================================
sbQueue = new plasmaSwitchboardQueue(this.cacheManager, urlPool.loadedURL, new File(plasmaPath, "switchboardQueue1.stack"), 10, profiles); * initialize switchboard queue
indexingTasksInProcess = new HashMap(); * ====================================================================== */
// create queue
this.sbQueue = new plasmaSwitchboardQueue(this.cacheManager, this.urlPool.loadedURL, new File(this.plasmaPath, "switchboardQueue1.stack"), 10, this.profiles);
// setting the indexing queue slots
indexingSlots = (int) getConfigLong("indexer.slots", 100);
// create in process list
this.indexingTasksInProcess = new HashMap();
// going through the sbQueue Entries and registering all content files as in use // going through the sbQueue Entries and registering all content files as in use
int count = 0; int count = 0;
@ -391,7 +371,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser
count++; count++;
} }
} }
log.logConfig(count + " files in htcache reported to the cachemanager as in use."); this.log.logConfig(count + " files in htcache reported to the cachemanager as in use.");
// define an extension-blacklist // define an extension-blacklist
log.logConfig("Parser: Initializing Extension Mappings for Media/Parser"); log.logConfig("Parser: Initializing Extension Mappings for Media/Parser");

6
yacy.init
View File

@ -559,7 +559,13 @@ crawler.MaxActiveThreads = 10
crawler.MaxIdleThreads = 7 crawler.MaxIdleThreads = 7
crawler.MinIdleThreads = 5 crawler.MinIdleThreads = 5
# maximum size of indexing queue
indexer.slots = 100
# specifies if yacy should set it's own referer if no referer URL
# was set by the client.
useYacyReferer = true useYacyReferer = true
#allow only 443(https-port) for https-proxy? #allow only 443(https-port) for https-proxy?
#if you want to tunnel other protokols, set to false #if you want to tunnel other protokols, set to false
secureHttps = true secureHttps = true