mirror of
https://github.com/yacy/yacy_search_server.git
synced 2024-09-19 00:01:41 +02:00
- fix for online caution delay settings, see
http://forum.yacy-websuche.de/viewtopic.php?f=6&t=738&p=4723#p4723 - removed remote search limitation for non-dht-peers according to discussion in http://forum.yacy-websuche.de/viewtopic.php?f=15&t=793&hilit=&p=5277#p5277 git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@4438 6c8d7289-2bf4-0310-a012-ef5d649a1542
This commit is contained in:
orbiter
parent
f35a3794e0
commit
3c7b94c119
|
|
@ -3,7 +3,7 @@ javacSource=1.5
|
|||
javacTarget=1.5
|
||||
|
||||
# Release Configuration
|
||||
releaseVersion=0.566
|
||||
releaseVersion=0.567
|
||||
stdReleaseFile=yacy_v${releaseVersion}_${DSTAMP}_${releaseNr}.tar.gz
|
||||
embReleaseFile=yacy_emb_v${releaseVersion}_${DSTAMP}_${releaseNr}.tar.gz
|
||||
proReleaseFile=yacy_pro_v${releaseVersion}_${DSTAMP}_${releaseNr}.tar.gz
|
||||
|
|
|
|||
|
|
@ -205,14 +205,20 @@
|
|||
<form action="PerformanceQueues_p.html" method="post" enctype="multipart/form-data">
|
||||
<fieldset><legend>Proxy Performance Settings</legend>
|
||||
<p>
|
||||
This is the time that the crawler idles when the proxy is accessed. The delay is extended by this time
|
||||
each time the proxy is accessed afterwards. This shall improve performance of the proxy throughput.
|
||||
(current delta is #[onlineCautionDelayCurrent]# milliseconds since last proxy access.)
|
||||
This is the time that the crawler idles when the proxy is accessed, or a local or remote search is done.
|
||||
The delay is extended by this time each time the proxy is accessed afterwards.
|
||||
This shall improve performance of the affected process (proxy or search).
|
||||
(current delta is #[crawlPauseProxyCurrent]#/#[crawlPauseLocalsearchCurrent]#/#[crawlPauseRemotesearchCurrent]#
|
||||
seconds since last proxy/local-search/remote-search access.)
|
||||
</p>
|
||||
<dl>
|
||||
<dt><label for="onlineCautionDelay">Online Caution Delay (milliseconds)</label>:</dt>
|
||||
<dd><input id="onlineCautionDelay" name="onlineCautionDelay" type="text" size="20" maxlength="100" value="#[onlineCautionDelay]#" /></dd>
|
||||
<dt><input type="submit" name="proxyControlSubmit" value="Enter New Parameters" /></dt>
|
||||
<dt><label for="crawlPauseProxy">Online Caution Delay for Proxy (milliseconds)</label>:</dt>
|
||||
<dd><input id="crawlPauseProxy" name="crawlPauseProxy" type="text" size="20" maxlength="100" value="#[crawlPauseProxy]#" /></dd>
|
||||
<dt><label for="crawlPauseLocalsearch">Online Caution Delay for Local Search (milliseconds)</label>:</dt>
|
||||
<dd><input id="crawlPauseLocalsearch" name="crawlPauseLocalsearch" type="text" size="20" maxlength="100" value="#[crawlPauseLocalsearch]#" /></dd>
|
||||
<dt><label for="crawlPauseRemotesearch">Online Caution Delay for Remote Search (milliseconds)</label>:</dt>
|
||||
<dd><input id="crawlPauseRemotesearch" name="crawlPauseRemotesearch" type="text" size="20" maxlength="100" value="#[crawlPauseRemotesearch]#" /></dd>
|
||||
<dt><input type="submit" name="onlineCautionSubmit" value="Enter New Parameters" /></dt>
|
||||
<dd>Changes take effect immediately</dd>
|
||||
</dl>
|
||||
</fieldset>
|
||||
|
|
|
|||
|
|
@ -227,9 +227,10 @@ public class PerformanceQueues_p {
|
|||
switchboard.setConfig("javastart_priority",post.get("YaCyPriority","0"));
|
||||
}
|
||||
|
||||
if ((post != null) && (post.containsKey("proxyControlSubmit"))) {
|
||||
int onlineCautionDelay = post.getInt("onlineCautionDelay", 30000);
|
||||
switchboard.setConfig("onlineCautionDelay", Integer.toString(onlineCautionDelay));
|
||||
if ((post != null) && (post.containsKey("onlineCautionSubmit"))) {
|
||||
switchboard.setConfig(plasmaSwitchboard.PROXY_ONLINE_CAUTION_DELAY, Integer.toString(post.getInt("crawlPauseProxy", 30000)));
|
||||
switchboard.setConfig(plasmaSwitchboard.LOCALSEACH_ONLINE_CAUTION_DELAY, Integer.toString(post.getInt("crawlPauseLocalsearch", 30000)));
|
||||
switchboard.setConfig(plasmaSwitchboard.REMOTESEARCH_ONLINE_CAUTION_DELAY, Integer.toString(post.getInt("crawlPauseRemotesearch", 30000)));
|
||||
}
|
||||
|
||||
// table cache settings
|
||||
|
|
@ -248,8 +249,12 @@ public class PerformanceQueues_p {
|
|||
prop.put("wordCacheMaxCount", switchboard.getConfigLong(plasmaSwitchboard.WORDCACHE_MAX_COUNT, 20000));
|
||||
prop.put("wordCacheInitCount", switchboard.getConfigLong(plasmaSwitchboard.WORDCACHE_INIT_COUNT, 30000));
|
||||
prop.put("wordFlushSize", switchboard.getConfigLong("wordFlushSize", 2000));
|
||||
prop.put("onlineCautionDelay", switchboard.getConfigLong("onlineCautionDelay", 30000));
|
||||
prop.putNum("onlineCautionDelayCurrent", System.currentTimeMillis() - switchboard.proxyLastAccess);
|
||||
prop.put("crawlPauseProxy", switchboard.getConfigLong(plasmaSwitchboard.PROXY_ONLINE_CAUTION_DELAY, 30000));
|
||||
prop.put("crawlPauseLocalsearch", switchboard.getConfigLong(plasmaSwitchboard.LOCALSEACH_ONLINE_CAUTION_DELAY, 30000));
|
||||
prop.put("crawlPauseRemotesearch", switchboard.getConfigLong(plasmaSwitchboard.REMOTESEARCH_ONLINE_CAUTION_DELAY, 30000));
|
||||
prop.putNum("crawlPauseProxyCurrent", (System.currentTimeMillis() - switchboard.proxyLastAccess) / 1000);
|
||||
prop.putNum("crawlPauseLocalsearchCurrent", (System.currentTimeMillis() - switchboard.localSearchLastAccess) / 1000);
|
||||
prop.putNum("crawlPauseRemotesearchCurrent", (System.currentTimeMillis() - switchboard.remoteSearchLastAccess) / 1000);
|
||||
|
||||
// table thread pool settings
|
||||
prop.put("pool_0_name","Crawler Pool");
|
||||
|
|
|
|||
|
|
@ -68,26 +68,8 @@ public class index {
|
|||
|
||||
final boolean indexDistributeGranted = sb.getConfigBool(plasmaSwitchboard.INDEX_DIST_ALLOW, true);
|
||||
final boolean indexReceiveGranted = sb.getConfigBool(plasmaSwitchboard.INDEX_RECEIVE_ALLOW, true);
|
||||
global = global && indexDistributeGranted && indexReceiveGranted;
|
||||
/*
|
||||
final String referer = (String) header.get(httpHeader.REFERER);
|
||||
if (referer != null) {
|
||||
yacyURL url;
|
||||
try {
|
||||
url = new yacyURL(referer, null);
|
||||
} catch (MalformedURLException e) {
|
||||
url = null;
|
||||
}
|
||||
if ((url != null) && (!url.isLocal())) {
|
||||
final HashMap referrerprop = new HashMap();
|
||||
referrerprop.put("count", "1");
|
||||
referrerprop.put("clientip", header.get(httpHeader.CONNECTION_PROP_CLIENTIP));
|
||||
referrerprop.put("useragent", header.get(httpHeader.USER_AGENT));
|
||||
referrerprop.put("date", (new serverDate()).toShortString(false));
|
||||
if (sb.facilityDB != null) try {sb.facilityDB.update("backlinks", referer, referrerprop);} catch (IOException e) {}
|
||||
}
|
||||
}
|
||||
*/
|
||||
//global = global && indexDistributeGranted && indexReceiveGranted;
|
||||
|
||||
// search domain
|
||||
int contentdom = plasmaSearchQuery.CONTENTDOM_TEXT;
|
||||
String cds = (post == null) ? "text" : post.get("contentdom", "text");
|
||||
|
|
|
|||
|
|
@ -91,22 +91,6 @@ public class yacysearch {
|
|||
|
||||
boolean rss = (post == null) ? false : post.get("rss", "false").equals("true");
|
||||
if ((post == null) || (env == null) || (querystring.length() == 0) || (!searchAllowed)) {
|
||||
/*
|
||||
// save referrer
|
||||
final String referer = (String) header.get("Referer");
|
||||
if (referer != null) {
|
||||
yacyURL url;
|
||||
try { url = new yacyURL(referer, null); } catch (MalformedURLException e) { url = null; }
|
||||
if ((url != null) && (!url.isLocal())) {
|
||||
final HashMap referrerprop = new HashMap();
|
||||
referrerprop.put("count", "1");
|
||||
referrerprop.put("clientip", header.get("CLIENTIP"));
|
||||
referrerprop.put("useragent", header.get("User-Agent"));
|
||||
referrerprop.put("date", (new serverDate()).toShortString(false));
|
||||
if (sb.facilityDB != null) try { sb.facilityDB.update("backlinks", referer, referrerprop); } catch (IOException e) {}
|
||||
}
|
||||
}
|
||||
*/
|
||||
// we create empty entries for template strings
|
||||
final serverObjects prop = new serverObjects();
|
||||
prop.put("searchagain", "0");
|
||||
|
|
@ -169,13 +153,13 @@ public class yacysearch {
|
|||
}
|
||||
|
||||
// SEARCH
|
||||
final boolean indexDistributeGranted = sb.getConfig(plasmaSwitchboard.INDEX_DIST_ALLOW, "true").equals("true");
|
||||
final boolean indexReceiveGranted = sb.getConfig("allowReceiveIndex", "true").equals("true");
|
||||
final boolean offline = yacyCore.seedDB.mySeed().isVirgin();
|
||||
//final boolean indexDistributeGranted = sb.getConfig(plasmaSwitchboard.INDEX_DIST_ALLOW, "true").equals("true");
|
||||
//final boolean indexReceiveGranted = sb.getConfig("allowReceiveIndex", "true").equals("true");
|
||||
//final boolean offline = yacyCore.seedDB.mySeed().isVirgin();
|
||||
final boolean clustersearch = sb.isRobinsonMode() &&
|
||||
(sb.getConfig("cluster.mode", "").equals("privatecluster") ||
|
||||
sb.getConfig("cluster.mode", "").equals("publiccluster"));
|
||||
if (offline || !indexDistributeGranted || !indexReceiveGranted) { global = false; }
|
||||
//if (offline || !indexDistributeGranted || !indexReceiveGranted) { global = false; }
|
||||
if (clustersearch) global = true; // switches search on, but search target is limited to cluster nodes
|
||||
|
||||
// find search domain
|
||||
|
|
|
|||
|
|
@ -50,6 +50,8 @@ import java.util.Hashtable;
|
|||
import java.util.Properties;
|
||||
import java.util.Vector;
|
||||
|
||||
import de.anomic.server.logging.serverLog;
|
||||
|
||||
public final class serverSystem {
|
||||
|
||||
// constants for system identification
|
||||
|
|
@ -330,6 +332,7 @@ public final class serverSystem {
|
|||
try {
|
||||
Runtime.getRuntime().exec("chmod 755 " + scriptFile.getAbsolutePath().replaceAll(" ", "\\ ")).waitFor();
|
||||
} catch (InterruptedException e) {
|
||||
serverLog.logSevere("DEPLOY", "deploy of script file failed. file = " + scriptFile.getAbsolutePath(), e);
|
||||
throw new IOException(e.getMessage());
|
||||
}
|
||||
}
|
||||
|
|
|
|||
25
yacy.init
25
yacy.init
|
|
@ -725,11 +725,11 @@ crawler.http.acceptEncoding=gzip
|
|||
crawler.http.acceptLanguage=en-us,en;q=0.5
|
||||
crawler.http.acceptCharset=ISO-8859-1,utf-8;q=0.7,*;q=0.7
|
||||
crawler.http.maxFileSize=262144
|
||||
crawler.http.maxFileSize__pro=1048576
|
||||
crawler.http.maxFileSize__pro=262144
|
||||
|
||||
# ftp crawler specific settings; size in bytes
|
||||
crawler.ftp.maxFileSize=262144
|
||||
crawler.ftp.maxFileSize__pro=1048576
|
||||
crawler.ftp.maxFileSize__pro=262144
|
||||
|
||||
# maximum number of crawler threads
|
||||
crawler.MaxActiveThreads = 30
|
||||
|
|
@ -903,27 +903,6 @@ tableTypeForPreNURL=0
|
|||
publicSurftips = true
|
||||
publicSearchpage = true
|
||||
|
||||
# a Java Properties file containig a list of SOAP services that should deployed
|
||||
# additionally to the default services. E.g.
|
||||
# soap.serviceDeploymentList = DATA/SETTINGS/myServices.properties
|
||||
#
|
||||
# One entry in the property file should must have the following format:
|
||||
# servicename=fullClassname
|
||||
#
|
||||
# e.g.
|
||||
# test=org.myservices.test
|
||||
#
|
||||
# Servicename is the name that shoud be used to access the service, e.g.
|
||||
# if the service name is "test" then the service can be reached using
|
||||
# http://localhost:8080/soap/test
|
||||
#
|
||||
# The WSDL document that belongs to the deployed service could be reached
|
||||
# using
|
||||
# http://localhost:8080/soap/test?wsdl
|
||||
#
|
||||
soap.serviceDeploymentList =
|
||||
|
||||
|
||||
# Wiki access rights
|
||||
# the built-in wiki system allows by default only that the administrator is allowed to make changes
|
||||
# this can be changed. There are three options:
|
||||
|
|
|
|||
Loading…
Reference in New Issue
Block a user