mirror of
https://github.com/yacy/yacy_search_server.git
synced 2024-09-19 00:01:41 +02:00
removed ContentControl servlet and functinality
This was not used at all (as I know) and was blocking a smooth integration of ivy in the context of an existing JSON parser.
This commit is contained in:
parent
b54f4ad35f
commit
fc98ca7a9c
123
.classpath
123
.classpath
|
@ -3,119 +3,6 @@
|
|||
<classpathentry excluding="api/|env/|processing/domaingraph/applet/|yacy/|api/bookmarks/|api/bookmarks/posts/|api/bookmarks/tags/|api/bookmarks/xbel/|solr/|gsa/|solr/collection1/|api/blacklists/|proxymsg/|p2p/" kind="src" path="htroot"/>
|
||||
<classpathentry excluding="bookmarks/|bookmarks/posts/|bookmarks/tags/|bookmarks/xbel/|blacklists/" kind="src" path="htroot/api"/>
|
||||
<classpathentry excluding="posts/|tags/|xbel/" kind="src" path="htroot/api/bookmarks"/>
|
||||
<classpathentry kind="con" path="org.eclipse.jdt.junit.JUNIT_CONTAINER/4"/>
|
||||
<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER">
|
||||
<attributes>
|
||||
<attribute name="module" value="true"/>
|
||||
</attributes>
|
||||
</classpathentry>
|
||||
<classpathentry kind="lib" path="lib/apache-mime4j-0.6.jar"/>
|
||||
<classpathentry kind="lib" path="lib/avatica-core-1.13.0.jar"/>
|
||||
<classpathentry kind="lib" path="lib/calcite-core-1.18.0.jar"/>
|
||||
<classpathentry kind="lib" path="lib/calcite-linq4j-1.18.0.jar"/>
|
||||
<classpathentry kind="lib" path="lib/jchardet-1.0.jar"/>
|
||||
<classpathentry kind="lib" path="lib/common-image-3.3.2.jar"/>
|
||||
<classpathentry kind="lib" path="lib/common-io-3.3.2.jar"/>
|
||||
<classpathentry kind="lib" path="lib/common-lang-3.3.2.jar"/>
|
||||
<classpathentry kind="lib" path="lib/commons-codec-1.14.jar"/>
|
||||
<classpathentry kind="lib" path="lib/commons-fileupload-1.4.jar"/>
|
||||
<classpathentry kind="lib" path="lib/commons-io-2.7.jar"/>
|
||||
<classpathentry kind="lib" path="lib/commons-lang-2.6.jar"/>
|
||||
<classpathentry kind="lib" path="lib/commons-lang3-3.12.0.jar"/>
|
||||
<classpathentry kind="lib" path="lib/commons-logging-1.2.jar"/>
|
||||
<classpathentry kind="lib" path="lib/commons-math3-3.4.1.jar"/>
|
||||
<classpathentry kind="lib" path="lib/fontbox-2.0.15.jar"/>
|
||||
<classpathentry kind="lib" path="lib/http2-client-9.4.34.v20201102.jar"/>
|
||||
<classpathentry kind="lib" path="lib/http2-common-9.4.34.v20201102.jar"/>
|
||||
<classpathentry kind="lib" path="lib/http2-http-client-transport-9.4.34.v20201102.jar"/>
|
||||
<classpathentry kind="lib" path="lib/httpclient-4.5.12.jar"/>
|
||||
<classpathentry kind="lib" path="lib/httpcore-4.4.13.jar"/>
|
||||
<classpathentry kind="lib" path="lib/httpmime-4.5.12.jar"/>
|
||||
<classpathentry kind="lib" path="lib/icu4j-63.1.jar"/>
|
||||
<classpathentry kind="lib" path="lib/imageio-bmp-3.3.2.jar"/>
|
||||
<classpathentry kind="lib" path="lib/imageio-core-3.3.2.jar"/>
|
||||
<classpathentry kind="lib" path="lib/imageio-metadata-3.3.2.jar"/>
|
||||
<classpathentry kind="lib" path="lib/imageio-tiff-3.3.2.jar"/>
|
||||
<classpathentry kind="lib" path="lib/jackson-annotations-2.11.2.jar"/>
|
||||
<classpathentry kind="lib" path="lib/jackson-core-2.11.2.jar"/>
|
||||
<classpathentry kind="lib" path="lib/jackson-databind-2.11.2.jar"/>
|
||||
<classpathentry kind="lib" path="lib/oro-2.0.8.jar"/>
|
||||
<classpathentry kind="lib" path="lib/jaudiotagger-2.2.5.jar"/>
|
||||
<classpathentry kind="lib" path="lib/javax.servlet-api-3.1.0.jar"/>
|
||||
<classpathentry kind="lib" path="lib/jcifs-1.3.17.jar"/>
|
||||
<classpathentry kind="lib" path="lib/jcl-over-slf4j-1.7.25.jar"/>
|
||||
<classpathentry kind="lib" path="lib/jetty-client-9.4.35.v20201120.jar"/>
|
||||
<classpathentry kind="lib" path="lib/jetty-continuation-9.4.35.v20201120.jar"/>
|
||||
<classpathentry kind="lib" path="lib/jetty-deploy-9.4.35.v20201120.jar"/>
|
||||
<classpathentry kind="lib" path="lib/jetty-http-9.4.35.v20201120.jar"/>
|
||||
<classpathentry kind="lib" path="lib/jetty-io-9.4.35.v20201120.jar"/>
|
||||
<classpathentry kind="lib" path="lib/jetty-jmx-9.4.35.v20201120.jar"/>
|
||||
<classpathentry kind="lib" path="lib/jetty-proxy-9.4.35.v20201120.jar"/>
|
||||
<classpathentry kind="lib" path="lib/jetty-security-9.4.35.v20201120.jar"/>
|
||||
<classpathentry kind="lib" path="lib/jetty-server-9.4.35.v20201120.jar"/>
|
||||
<classpathentry kind="lib" path="lib/jetty-servlet-9.4.35.v20201120.jar"/>
|
||||
<classpathentry kind="lib" path="lib/jetty-servlets-9.4.35.v20201120.jar"/>
|
||||
<classpathentry kind="lib" path="lib/jetty-util-9.4.35.v20201120.jar"/>
|
||||
<classpathentry kind="lib" path="lib/jetty-webapp-9.4.35.v20201120.jar"/>
|
||||
<classpathentry kind="lib" path="lib/jetty-xml-9.4.35.v20201120.jar"/>
|
||||
<classpathentry kind="lib" path="lib/jsch-0.1.54.jar"/>
|
||||
<classpathentry kind="lib" path="lib/json-simple-1.1.1.jar"/>
|
||||
<classpathentry kind="lib" path="lib/jsonic-1.3.10.jar"/>
|
||||
<classpathentry kind="lib" path="lib/jwat-archive-common-1.1.1.jar"/>
|
||||
<classpathentry kind="lib" path="lib/jwat-common-1.1.1.jar"/>
|
||||
<classpathentry kind="lib" path="lib/jwat-gzip-1.1.1.jar"/>
|
||||
<classpathentry kind="lib" path="lib/jwat-warc-1.1.1.jar"/>
|
||||
<classpathentry kind="lib" path="lib/lucene-analyzers-common-8.8.1.jar"/>
|
||||
<classpathentry kind="lib" path="lib/lucene-analyzers-phonetic-8.8.1.jar"/>
|
||||
<classpathentry kind="lib" path="lib/lucene-backward-codecs-8.8.1.jar"/>
|
||||
<classpathentry kind="lib" path="lib/lucene-classification-8.8.1.jar"/>
|
||||
<classpathentry kind="lib" path="lib/lucene-codecs-8.8.1.jar"/>
|
||||
<classpathentry kind="lib" path="lib/lucene-core-8.8.1.jar"/>
|
||||
<classpathentry kind="lib" path="lib/lucene-grouping-8.8.1.jar"/>
|
||||
<classpathentry kind="lib" path="lib/lucene-highlighter-8.8.1.jar"/>
|
||||
<classpathentry kind="lib" path="lib/lucene-join-8.8.1.jar"/>
|
||||
<classpathentry kind="lib" path="lib/lucene-memory-8.8.1.jar"/>
|
||||
<classpathentry kind="lib" path="lib/lucene-misc-8.8.1.jar"/>
|
||||
<classpathentry kind="lib" path="lib/lucene-queries-8.8.1.jar"/>
|
||||
<classpathentry kind="lib" path="lib/lucene-queryparser-8.8.1.jar"/>
|
||||
<classpathentry kind="lib" path="lib/lucene-spatial-extras-8.8.1.jar"/>
|
||||
<classpathentry kind="lib" path="lib/lucene-suggest-8.8.1.jar"/>
|
||||
<classpathentry kind="lib" path="lib/opentracing-api-0.33.0.jar"/>
|
||||
<classpathentry kind="lib" path="lib/opentracing-noop-0.33.0.jar"/>
|
||||
<classpathentry kind="lib" path="lib/opentracing-util-0.33.0.jar"/>
|
||||
<classpathentry kind="lib" path="lib/metadata-extractor-2.11.0.jar"/>
|
||||
<classpathentry kind="lib" path="lib/metrics-core-3.2.2.jar"/>
|
||||
<classpathentry kind="lib" path="lib/metrics-jmx-4.1.5.jar"/>
|
||||
<classpathentry kind="lib" path="lib/org.restlet.jar"/>
|
||||
<classpathentry kind="lib" path="lib/pdfbox-2.0.15.jar"/>
|
||||
<classpathentry kind="lib" path="lib/poi-3.17.jar"/>
|
||||
<classpathentry kind="lib" path="lib/poi-scratchpad-3.17.jar"/>
|
||||
<classpathentry kind="lib" path="lib/rrd4j-3.2.jar"/>
|
||||
<classpathentry kind="lib" path="lib/solr-core-8.8.1.jar"/>
|
||||
<classpathentry kind="lib" path="lib/solr-solrj-8.8.1.jar"/>
|
||||
<classpathentry kind="lib" path="lib/spatial4j-0.6.jar"/>
|
||||
<classpathentry kind="lib" path="lib/stax2-api-3.1.4.jar"/>
|
||||
<classpathentry kind="lib" path="lib/weupnp-0.1.4.jar"/>
|
||||
<classpathentry kind="lib" path="lib/woodstox-core-asl-4.4.1.jar"/>
|
||||
<classpathentry kind="lib" path="lib/xml-apis-1.4.01.jar"/>
|
||||
<classpathentry kind="lib" path="lib/xmpcore-5.1.3.jar"/>
|
||||
<classpathentry kind="lib" path="lib/xz-1.8.jar"/>
|
||||
<classpathentry kind="lib" path="lib/zookeeper-3.4.14.jar"/>
|
||||
<classpathentry kind="lib" path="libt/hamcrest-2.2.jar"/>
|
||||
<classpathentry kind="lib" path="libt/hamcrest-core-2.2.jar"/>
|
||||
<classpathentry kind="lib" path="libt/hamcrest-library-2.2.jar"/>
|
||||
<classpathentry kind="lib" path="lib/commons-collections4-4.4.jar"/>
|
||||
<classpathentry kind="lib" path="lib/guava-25.1-jre.jar"/>
|
||||
<classpathentry kind="lib" path="lib/hazelcast-4.2.jar"/>
|
||||
<classpathentry kind="lib" path="lib/commons-compress-1.21.jar"/>
|
||||
<classpathentry kind="lib" path="lib/bcmail-jdk15on-1.69.jar"/>
|
||||
<classpathentry kind="lib" path="lib/bcpkix-jdk15on-1.69.jar"/>
|
||||
<classpathentry kind="lib" path="lib/bcprov-jdk15on-1.69.jar"/>
|
||||
<classpathentry kind="lib" path="lib/jsoup-1.14.2.jar"/>
|
||||
<classpathentry kind="lib" path="lib/log4j-over-slf4j-1.7.32.jar"/>
|
||||
<classpathentry kind="lib" path="lib/slf4j-api-1.7.32.jar"/>
|
||||
<classpathentry kind="lib" path="lib/slf4j-jdk14-1.7.32.jar"/>
|
||||
<classpathentry kind="lib" path="lib/langdetect-1.1-20120112.jar"/>
|
||||
<classpathentry kind="src" path="htroot/api/blacklists"/>
|
||||
<classpathentry kind="src" path="htroot/api/bookmarks/posts"/>
|
||||
<classpathentry kind="src" path="htroot/api/bookmarks/tags"/>
|
||||
|
@ -126,6 +13,16 @@
|
|||
<classpathentry kind="src" path="htroot/yacy"/>
|
||||
<classpathentry kind="src" path="source"/>
|
||||
<classpathentry kind="src" path="test/java"/>
|
||||
<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER">
|
||||
<attributes>
|
||||
<attribute name="module" value="true"/>
|
||||
</attributes>
|
||||
</classpathentry>
|
||||
<classpathentry kind="con" path="org.apache.ivyde.eclipse.cpcontainer.IVYDE_CONTAINER/?project=yacy&ivyXmlPath=ivy.xml&confs=compile">
|
||||
<attributes>
|
||||
<attribute name="module" value="true"/>
|
||||
</attributes>
|
||||
</classpathentry>
|
||||
<classpathentry kind="lib" path="lib/J7Zip-modified-1.0.2.jar"/>
|
||||
<classpathentry kind="output" path="gen"/>
|
||||
</classpath>
|
||||
|
|
1
.project
1
.project
|
@ -23,5 +23,6 @@
|
|||
</buildSpec>
|
||||
<natures>
|
||||
<nature>org.eclipse.jdt.core.javanature</nature>
|
||||
<nature>org.apache.ivyde.eclipse.ivynature</nature>
|
||||
</natures>
|
||||
</projectDescription>
|
||||
|
|
|
@ -1370,16 +1370,6 @@ core.service.webgraph.tmp = false
|
|||
parserAugmentation = false
|
||||
parserAugmentation.RDFa = false
|
||||
|
||||
# Content control settings
|
||||
contentcontrol.enabled = false
|
||||
contentcontrol.bookmarklist = contentcontrol
|
||||
contentcontrol.mandatoryfilterlist = yacy
|
||||
contentcontrol.smwimport.enabled = false
|
||||
contentcontrol.smwimport.baseurl =
|
||||
contentcontrol.smwimport.purgelistoninit = true
|
||||
contentcontrol.smwimport.targetlist = contentcontrol
|
||||
contentcontrol.smwimport.defaultcategory = yacy
|
||||
|
||||
# host browser settings
|
||||
# Allow the administrator to stack URLs to the local crawl queue from the host browser page, automatically (when a path is unknown) or manually through a "load and index" link
|
||||
browser.autoload = false
|
||||
|
|
|
@ -1,95 +0,0 @@
|
|||
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
|
||||
<html xmlns="http://www.w3.org/1999/xhtml">
|
||||
<head>
|
||||
<title>YaCy '#[clientname]#': Content Control</title>
|
||||
#%env/templates/metas.template%#
|
||||
</head>
|
||||
<body id="Settings">
|
||||
#%env/templates/header.template%#
|
||||
#%env/templates/submenuBlacklist.template%#
|
||||
|
||||
<h2>Content Control</h2>
|
||||
|
||||
<form id="contentcontrolsettings" action="ContentControl_p.html" method="post" enctype="multipart/form-data">
|
||||
|
||||
<fieldset><legend id="augmentation">Peer Content Control URL Filter</legend>
|
||||
<p>
|
||||
With this settings you can activate or deactivate content control on this peer.
|
||||
</p>
|
||||
|
||||
|
||||
<dl>
|
||||
|
||||
<dt><label for="content">Use content control filtering:</label></dt>
|
||||
<dd>
|
||||
<input type="checkbox" name="contentcontrolenabled" id="contentcontrolenabled" #(contentcontrolenabled_checked)#:: checked="checked"#(/contentcontrolenabled_checked)# />Enabled<br/>
|
||||
<p class="help">
|
||||
Enables or disables content control.
|
||||
</p>
|
||||
</dd>
|
||||
|
||||
|
||||
<dt><label for="content">Use this table to create filter:</label></dt>
|
||||
<dd>
|
||||
<input type="text" name="contentcontrolbml" value="#[contentcontrolbml]#" size="60" /><br/><br/>
|
||||
<p class="help">
|
||||
Define a table. Default: contentcontrol
|
||||
</p>
|
||||
</dd>
|
||||
<dt></dt>
|
||||
<dd><input type="submit" name="contentcontrolSettings" value="Submit" class="btn btn-primary"/></dd>
|
||||
</dl>
|
||||
</fieldset>
|
||||
</form>
|
||||
|
||||
|
||||
|
||||
<form id="contentcontrolExtraSettings" action="ContentControl_p.html" method="post" enctype="multipart/form-data">
|
||||
<fieldset><legend id="urlproxy">Content Control SMW Import Settings</legend>
|
||||
<p>
|
||||
With this settings you can define the content control import settings. You can define a <a href="http://wiki.sciety.org/mediawiki/extensions/yacy-smwextension/" target="_blank">Semantic Media Wiki with the appropriate extensions.</a>
|
||||
</p>
|
||||
|
||||
<dl>
|
||||
|
||||
<dt><label for="content">SMW import to content control list:</label></dt>
|
||||
<dd>
|
||||
<input type="checkbox" name="ccsmwimport" id="ccsmwimport" #(ccsmwimport_checked)#:: checked="checked"#(/ccsmwimport_checked)# />Enabled<br/>
|
||||
<p class="help">
|
||||
Enable or disable constant background synchronization of content control list from SMW (Semantic Mediawiki). Requires restart!
|
||||
</p>
|
||||
</dd>
|
||||
|
||||
<dt><label for="content">SMW import base URL:</label></dt>
|
||||
<dd>
|
||||
<input type="text" name="ccsmwimporturl" value="#[ccsmwimporturl]#" size="60" /><br/><br/>
|
||||
<p class="help">
|
||||
Define base URL for SMW special page "Ask". Example: http://my.wiki.cc/wiki/Special:Ask
|
||||
</p>
|
||||
</dd>
|
||||
|
||||
<dt><label for="content">SMW import target table:</label></dt>
|
||||
<dd>
|
||||
<input type="text" name="ccsmwimportlist" value="#[ccsmwimportlist]#" size="60" /><br/><br/>
|
||||
<p class="help">
|
||||
Define import target table. Default: contentcontrol
|
||||
</p>
|
||||
</dd>
|
||||
|
||||
<dt><label for="content">Purge content control list on initial sync:</label></dt>
|
||||
<dd>
|
||||
<input type="checkbox" name="ccsmwpurge" id="ccsmwpurge" #(ccsmwpurge_checked)#:: checked="checked"#(/ccsmwpurge_checked)# />Enabled<br/>
|
||||
<p class="help">
|
||||
Purge content control list on initial synchronisation after startup.
|
||||
</p>
|
||||
</dd>
|
||||
<dt></dt>
|
||||
<dd><input type="submit" name="contentcontrolExtraSettings" value="Submit" class="btn btn-primary"/></dd>
|
||||
</dl>
|
||||
</fieldset>
|
||||
</form>
|
||||
|
||||
|
||||
#%env/templates/footer.template%#
|
||||
</body>
|
||||
</html>
|
|
@ -1,68 +0,0 @@
|
|||
import net.yacy.cora.protocol.RequestHeader;
|
||||
import net.yacy.server.serverObjects;
|
||||
import net.yacy.server.serverSwitch;
|
||||
|
||||
public final class ContentControl_p {
|
||||
|
||||
public static serverObjects respond(@SuppressWarnings("unused") final RequestHeader header,
|
||||
final serverObjects post, final serverSwitch env) {
|
||||
|
||||
final serverObjects prop = new serverObjects();
|
||||
|
||||
if (post != null) {
|
||||
|
||||
if (post.containsKey("contentcontrolExtraSettings")) {
|
||||
|
||||
env.setConfig("contentcontrol.smwimport.baseurl",
|
||||
post.get("ccsmwimporturl"));
|
||||
|
||||
env.setConfig("contentcontrol.smwimport.enabled",
|
||||
"on".equals(post.get("ccsmwimport")) ? true : false);
|
||||
|
||||
env.setConfig("contentcontrol.smwimport.purgelistoninit",
|
||||
"on".equals(post.get("ccsmwpurge")) ? true : false);
|
||||
|
||||
env.setConfig("contentcontrol.smwimport.targetlist",
|
||||
post.get("ccsmwimportlist"));
|
||||
|
||||
|
||||
}
|
||||
|
||||
if (post.containsKey("contentcontrolSettings")) {
|
||||
|
||||
env.setConfig("contentcontrol.enabled",
|
||||
"on".equals(post.get("contentcontrolenabled")) ? true : false);
|
||||
|
||||
|
||||
env.setConfig("contentcontrol.bookmarklist",
|
||||
post.get("contentcontrolbml"));
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
prop.putHTML("ccsmwimportlist",
|
||||
env.getConfig("contentcontrol.smwimport.targetlist", "contentcontrol"));
|
||||
|
||||
prop.put("ccsmwpurge_checked", env.getConfigBool(
|
||||
"contentcontrol.smwimport.purgelistoninit", false) ? "1" : "0");
|
||||
|
||||
prop.putHTML("ccsmwimporturl",
|
||||
env.getConfig("contentcontrol.smwimport.baseurl", ""));
|
||||
|
||||
prop.put("ccsmwimport_checked", env.getConfigBool(
|
||||
"contentcontrol.smwimport.enabled", false) ? "1" : "0");
|
||||
|
||||
|
||||
prop.put("contentcontrolenabled_checked",
|
||||
env.getConfigBool("contentcontrol.enabled", false) ? "1" : "0");
|
||||
|
||||
prop.putHTML("contentcontrolbml",
|
||||
env.getConfig("contentcontrol.bookmarklist", ""));
|
||||
|
||||
// return rewrite properties
|
||||
return prop;
|
||||
}
|
||||
|
||||
}
|
|
@ -5,6 +5,5 @@
|
|||
<li><a href="BlacklistCleaner_p.html" class="MenuItemLink #(authorized)#lock::unlock#(/authorized)#">Blacklist Cleaner</a></li>
|
||||
<li><a href="BlacklistTest_p.html" class="MenuItemLink #(authorized)#lock::unlock#(/authorized)#">Blacklist Test</a></li>
|
||||
<li><a href="BlacklistImpExp_p.html" class="MenuItemLink #(authorized)#lock::unlock#(/authorized)#">Import/Export</a></li>
|
||||
<li><a href="ContentControl_p.html" class="MenuItemLink #(authorized)#lock::unlock#(/authorized)#">Content Control</a></li>
|
||||
</ul>
|
||||
</div>
|
2
ivy.xml
2
ivy.xml
|
@ -13,7 +13,6 @@
|
|||
<dependency org="com.cybozu.labs" name="langdetect" rev="1.1-20120112" conf="compile->master"/>
|
||||
<dependency org="com.drewnoakes" name="metadata-extractor" rev="2.11.0" />
|
||||
<dependency org="com.fasterxml.jackson.core" name="jackson-databind" rev="2.11.2"/>
|
||||
<dependency org="com.googlecode.json-simple" name="json-simple" rev="1.1.1" conf="compile->master" />
|
||||
<dependency org="com.google.guava" name="guava" rev="25.1-jre" conf="compile->master"/>
|
||||
<dependency org="com.hazelcast" name="hazelcast" rev="4.2" />
|
||||
<dependency org="com.ibm.icu" name="icu4j" rev="63.1"/>
|
||||
|
@ -94,7 +93,6 @@
|
|||
<!-- This does not match langdetect.jar from pre-ivy -->
|
||||
<dependency org="org.tukaani" name="xz" rev="1.8"/>
|
||||
<dependency org="oro" name="oro" rev="2.0.8"/>
|
||||
<dependency org="xml-apis" name="xml-apis" rev="1.4.01"/>
|
||||
|
||||
<dependency org="junit" name="junit" rev="4.13" conf="test->default"/>
|
||||
<dependency org="org.hamcrest" name="hamcrest" rev="2.2" conf="test->default"/>
|
||||
|
|
|
@ -953,30 +953,6 @@ Duration==Dauer
|
|||
#ID==ID
|
||||
#-----------------------------
|
||||
|
||||
#File: ContentControl_p.html
|
||||
#---------------------------
|
||||
Content Control<==Inhaltskontrolle<
|
||||
Peer Content Control URL Filter==Peer Inhaltskontrolle URL Filter
|
||||
With this settings you can activate or deactivate content control on this peer.==Mit dieser Einstellung kann die Inhaltskontrolle auf diesem Peer an- oder abgeschalten werden.
|
||||
Use content control filtering:==Verwende Inhaltskontrollfilter:
|
||||
>Enabled<==>Aktiviert<
|
||||
Enables or disables content control.==Schaltet Inhaltskontrolle an- oder ab.
|
||||
Use this table to create filter:==Verwenden Sie diese Tabelle, um Filter zu erzeugen:
|
||||
Define a table. Default:==Definieren Sie ein Tabelle. Standardeinstellung:
|
||||
Content Control SMW Import Settings==Inhaltskontrolle SMW Importeinstellungen
|
||||
With this settings you can define the content control import settings. You can define a==Mit diesen Einstellungen können Sie die Importeinstellungen für die Inhaltskontrolle definieren. Definieren Sie ein
|
||||
Semantic Media Wiki with the appropriate extensions.==Semantisches Media Wiki mit den passenden Erweiterungen.
|
||||
SMW import to content control list:==SMW Import für die Inhalts-Kontroll-Liste:
|
||||
Enable or disable constant background synchronization of content control list from SMW (Semantic Mediawiki). Requires restart!==Konstante Synchronisation der Inhalts-Kontroll-Liste vom SMW (Semantisches Medienwiki) im Hintergrund. Benötigt Neustart!
|
||||
SMW import base URL:==SMW Import Basis URL:
|
||||
Define base URL for SMW special page "Ask". Example: ==Definiere Basis URL für SMW Spezialseite "Ask". Beispiel:
|
||||
SMW import target table:==SMW Import Ziele Tabelle:
|
||||
Define import target table. Default: contentcontrol==Definieren Import Ziel Tabelle. Standardeinstellung: contentcontrol
|
||||
Purge content control list on initial sync:==Verwerfe Inhalts-Kontroll-Listen bei der ersten Synchronisation:
|
||||
Purge content control list on initial synchronisation after startup.==Verwerfe Inhalts-Kontroll-Listen bei der ersten Synchronisation nach dem Start.
|
||||
"Submit"=="Absenden"
|
||||
#-----------------------------
|
||||
|
||||
|
||||
#File: CookieMonitorIncoming_p.html
|
||||
#---------------------------
|
||||
|
|
|
@ -530,13 +530,6 @@ Duration==Duración
|
|||
ID==ID
|
||||
#-----------------------------
|
||||
|
||||
#File: ContentControl_p.html
|
||||
#---------------------------
|
||||
Content Control<==Control de contenido<
|
||||
>Enabled<==>Habilitado
|
||||
"Submit"=="Enviar"
|
||||
#-----------------------------
|
||||
|
||||
|
||||
#File: CookieMonitorIncoming_p.html
|
||||
#---------------------------
|
||||
|
|
|
@ -510,13 +510,6 @@ Duration==Durata
|
|||
ID==ID
|
||||
#-----------------------------
|
||||
|
||||
#File: ContentControl_p.html
|
||||
#---------------------------
|
||||
Content Control<==Controllo dei contenuti<
|
||||
>Enabled<==>Abilitato
|
||||
"Submit"=="Invia"
|
||||
#-----------------------------
|
||||
|
||||
|
||||
#File: CookieMonitorIncoming_p.html
|
||||
#---------------------------
|
||||
|
|
|
@ -714,13 +714,6 @@ Last Deploy==最後の展開
|
|||
Connection Tracking==接続の追跡
|
||||
#-----------------------------
|
||||
|
||||
#File: ContentControl_p.html
|
||||
#---------------------------
|
||||
Content Control<==コンテントの制御<
|
||||
"Submit"=="確定する"
|
||||
#-----------------------------
|
||||
|
||||
|
||||
#File: CookieMonitorIncoming_p.html
|
||||
#---------------------------
|
||||
Incoming Cookies Monitor==着信したCookieのモニター
|
||||
|
|
|
@ -2429,71 +2429,6 @@
|
|||
</body>
|
||||
</file>
|
||||
|
||||
<file original="ContentControl_p.html" source-language="en" datatype="html">
|
||||
<body>
|
||||
<trans-unit id="3f3b9286" xml:space="preserve" approved="no" translate="yes">
|
||||
<source>Content Control<</source>
|
||||
</trans-unit>
|
||||
<trans-unit id="d21676d1" xml:space="preserve" approved="no" translate="yes">
|
||||
<source>Peer Content Control URL Filter</source>
|
||||
</trans-unit>
|
||||
<trans-unit id="542e1ecb" xml:space="preserve" approved="no" translate="yes">
|
||||
<source>With this settings you can activate or deactivate content control on this peer.</source>
|
||||
</trans-unit>
|
||||
<trans-unit id="2bd01413" xml:space="preserve" approved="no" translate="yes">
|
||||
<source>Use content control filtering:</source>
|
||||
</trans-unit>
|
||||
<trans-unit id="4e4f2379" xml:space="preserve" approved="no" translate="yes">
|
||||
<source>>Enabled<</source>
|
||||
</trans-unit>
|
||||
<trans-unit id="ff54fe20" xml:space="preserve" approved="no" translate="yes">
|
||||
<source>Enables or disables content control.</source>
|
||||
</trans-unit>
|
||||
<trans-unit id="81cdc1a8" xml:space="preserve" approved="no" translate="yes">
|
||||
<source>Use this table to create filter:</source>
|
||||
</trans-unit>
|
||||
<trans-unit id="2a641f75" xml:space="preserve" approved="no" translate="yes">
|
||||
<source>Define a table. Default:</source>
|
||||
</trans-unit>
|
||||
<trans-unit id="c3a262b1" xml:space="preserve" approved="no" translate="yes">
|
||||
<source>Content Control SMW Import Settings</source>
|
||||
</trans-unit>
|
||||
<trans-unit id="fe0fc485" xml:space="preserve" approved="no" translate="yes">
|
||||
<source>With this settings you can define the content control import settings. You can define a</source>
|
||||
</trans-unit>
|
||||
<trans-unit id="a00319d4" xml:space="preserve" approved="no" translate="yes">
|
||||
<source>Semantic Media Wiki with the appropriate extensions.</source>
|
||||
</trans-unit>
|
||||
<trans-unit id="3f00f0c5" xml:space="preserve" approved="no" translate="yes">
|
||||
<source>SMW import to content control list:</source>
|
||||
</trans-unit>
|
||||
<trans-unit id="446815ef" xml:space="preserve" approved="no" translate="yes">
|
||||
<source>Enable or disable constant background synchronization of content control list from SMW (Semantic Mediawiki). Requires restart!</source>
|
||||
</trans-unit>
|
||||
<trans-unit id="d9bff282" xml:space="preserve" approved="no" translate="yes">
|
||||
<source>SMW import base URL:</source>
|
||||
</trans-unit>
|
||||
<trans-unit id="ecfbe3e8" xml:space="preserve" approved="no" translate="yes">
|
||||
<source>Define base URL for SMW special page "Ask". Example: </source>
|
||||
</trans-unit>
|
||||
<trans-unit id="d0d7e963" xml:space="preserve" approved="no" translate="yes">
|
||||
<source>SMW import target table:</source>
|
||||
</trans-unit>
|
||||
<trans-unit id="84acd3e4" xml:space="preserve" approved="no" translate="yes">
|
||||
<source>Define import target table. Default: contentcontrol</source>
|
||||
</trans-unit>
|
||||
<trans-unit id="70ed825" xml:space="preserve" approved="no" translate="yes">
|
||||
<source>Purge content control list on initial sync:</source>
|
||||
</trans-unit>
|
||||
<trans-unit id="642de9e8" xml:space="preserve" approved="no" translate="yes">
|
||||
<source>Purge content control list on initial synchronisation after startup.</source>
|
||||
</trans-unit>
|
||||
<trans-unit id="bfcc5088" xml:space="preserve" approved="no" translate="yes">
|
||||
<source>"Submit"</source>
|
||||
</trans-unit>
|
||||
</body>
|
||||
</file>
|
||||
|
||||
<file original="ContentIntegrationPHPBB3_p.html" source-language="en" datatype="html">
|
||||
<body>
|
||||
<trans-unit id="c7bfa2ca" xml:space="preserve" approved="no" translate="yes">
|
||||
|
|
|
@ -1059,30 +1059,6 @@ Duration==Длительность
|
|||
#ID==ID
|
||||
#-----------------------------
|
||||
|
||||
#File: ContentControl_p.html
|
||||
#---------------------------
|
||||
Content Control<==Управление контентом<
|
||||
Peer Content Control URL Filter==Управление контентом узла
|
||||
With this settings you can activate or deactivate content control on this peer.==Эти настройки позволяют включить или отключить управление контентом для вашего узла.
|
||||
Use content control filtering:==Использовать фильтр управления контентом:
|
||||
>Enabled<==>Включить<
|
||||
Enables or disables content control.==Включение или отключение управления контентом.
|
||||
Use this table to create filter:==Использовать это поле для создания фильтра:
|
||||
Define a table. Default:==Задать значение поля. По-умолчанию:
|
||||
Content Control SMW Import Settings==Импорт настроек управления контентом SMW
|
||||
With this settings you can define the content control import settings. You can define a==Эти настройки позволяют задать параметры импорта настроек управления контентом
|
||||
Semantic Media Wiki with the appropriate extensions.==Semantic Media Wiki с соответствующими расширениями.
|
||||
SMW import to content control list:== Импорт SMW в список управления контентом:
|
||||
Enable or disable constant background synchronization of content control list from SMW (Semantic Mediawiki). Requires restart!==Включение или отключение постоянной фоновой синхронизации списка управления контентом из SMW (Semantic Mediawiki). Потребуется перезапуск программы!
|
||||
SMW import base URL:==Ссылка на импортируемую базу SMW:
|
||||
Define base URL for SMW special page "Ask". Example: ==Укажите ссылку на базу SMW на специальной странице "Ask". Например:
|
||||
SMW import target table:==Поле назначения импорта SMW:
|
||||
Define import target table. Default: contentcontrol==Укажите поле назначения импорта. По-умолчанию: contentcontrol
|
||||
Purge content control list on initial sync:==Удалить список управления контентом в начале синхронизации:
|
||||
Purge content control list on initial synchronisation after startup.==Удалить список управления контентом в начале синхронизации после запуска программы.
|
||||
"Submit"=="Сохранить"
|
||||
#-----------------------------
|
||||
|
||||
|
||||
#File: CookieMonitorIncoming_p.html
|
||||
#---------------------------
|
||||
|
|
|
@ -1033,31 +1033,6 @@ For minTokenLen = 2 the quantRate value should not be below 0.24; for minTokenLe
|
|||
The quantRate is a measurement for the number of words that take part in a signature computation. The higher the number==quantRate是参与签名计算的单词数量的度量。 数字越高,越少
|
||||
#-----------------------------
|
||||
|
||||
#File: ContentControl_p.html
|
||||
#---------------------------
|
||||
Content Control<==内容控制<
|
||||
Peer Content Control URL Filter==节点内容控制地址过滤器
|
||||
With this settings you can activate or deactivate content control on this peer==使用此设置,你可以激活或取消激活此YaCy节点上的内容控制
|
||||
Use content control filtering:==使用内容控制过滤:
|
||||
>Enabled<==>已启用<
|
||||
Enables or disables content control==启用或禁用内容控制
|
||||
Use this table to create filter:==使用此表创建过滤器:
|
||||
Define a table. Default:==定义一个表格. 默认:
|
||||
Content Control SMW Import Settings==内容控制SMW导入设置
|
||||
With this settings you can define the content control import settings. You can define a==使用此设置,你可以定义内容控制导入设置. 你可以定义一个
|
||||
Semantic Media Wiki with the appropriate extensions==语义媒体百科与适当的扩展
|
||||
SMW import to content control list:==SMW导入到内容控制列表:
|
||||
Enable or disable constant background synchronization of content control list from SMW (Semantic Mediawiki). Requires restart!==启用或禁用来自SMW(Semantic Mediawiki)的内容控制列表的恒定后台同步。 需要重启!
|
||||
SMW import base URL:==SMW导入基URL:
|
||||
Define base URL for SMW special page "Ask". Example: ==为SMW特殊页面“Ask”定义基础地址.例:
|
||||
SMW import target table:==SMW导入目标表:
|
||||
Define import target table. Default: contentcontrol==定义导入目标表. 默认值:contentcontrol
|
||||
Purge content control list on initial sync:==在初始同步时清除内容控制列表:
|
||||
Purge content control list on initial synchronisation after startup.==重启后,清除初始同步的内容控制列表.
|
||||
"Submit"=="提交"
|
||||
Define base URL for SMW special page "Ask". Example:==为SMW特殊页面“Ask”定义基础地址.例:
|
||||
#-----------------------------
|
||||
|
||||
#File: ContentIntegrationPHPBB3_p.html
|
||||
#---------------------------
|
||||
Content Integration: Retrieval from phpBB3 Databases==内容集成: 从phpBB3数据库中导入
|
||||
|
|
|
@ -1,90 +0,0 @@
|
|||
package net.yacy.contentcontrol;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.Iterator;
|
||||
|
||||
import net.yacy.kelondro.blob.Tables;
|
||||
import net.yacy.kelondro.blob.Tables.Row;
|
||||
import net.yacy.repository.FilterEngine;
|
||||
import net.yacy.search.Switchboard;
|
||||
|
||||
public class ContentControlFilterUpdateThread implements Runnable {
|
||||
|
||||
private final Switchboard sb;
|
||||
|
||||
private Boolean locked = false;
|
||||
|
||||
private static FilterEngine networkfilter;
|
||||
|
||||
public ContentControlFilterUpdateThread(final Switchboard sb) {
|
||||
|
||||
this.sb = sb;
|
||||
|
||||
}
|
||||
|
||||
@Override
|
||||
public final void run() {
|
||||
|
||||
if (!this.locked) {
|
||||
|
||||
this.locked = true;
|
||||
|
||||
if (this.sb.getConfigBool("contentcontrol.enabled", false) == true) {
|
||||
|
||||
if (SMWListSyncThread.dirty) {
|
||||
|
||||
networkfilter = updateFilter();
|
||||
|
||||
SMWListSyncThread.dirty = false;
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
this.locked = false;
|
||||
|
||||
}
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
private static FilterEngine updateFilter () {
|
||||
|
||||
FilterEngine newfilter = new FilterEngine();
|
||||
|
||||
Switchboard sb = Switchboard.getSwitchboard();
|
||||
|
||||
Iterator<Tables.Row> it;
|
||||
try {
|
||||
it = sb.tables.iterator(sb.getConfig("contentcontrol.bookmarklist",
|
||||
"contentcontrol"));
|
||||
|
||||
while (it.hasNext()) {
|
||||
Row b = it.next();
|
||||
|
||||
if (!b.get("filter", "").equals("")) {
|
||||
|
||||
newfilter.add(b.get("filter", ""), null);
|
||||
}
|
||||
}
|
||||
|
||||
} catch (final IOException e) {
|
||||
// TODO Auto-generated catch block
|
||||
e.printStackTrace();
|
||||
}
|
||||
|
||||
return newfilter;
|
||||
}
|
||||
|
||||
|
||||
public static FilterEngine getNetworkFilter() {
|
||||
FilterEngine f = networkfilter;
|
||||
|
||||
if (f != null && f.size() > 0)
|
||||
return f;
|
||||
|
||||
return null;
|
||||
|
||||
}
|
||||
|
||||
}
|
|
@ -1,163 +0,0 @@
|
|||
package net.yacy.contentcontrol;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.Reader;
|
||||
import java.util.HashMap;
|
||||
import java.util.Map.Entry;
|
||||
import java.util.concurrent.ArrayBlockingQueue;
|
||||
|
||||
import net.yacy.cora.util.ConcurrentLog;
|
||||
|
||||
import org.json.simple.parser.ContentHandler;
|
||||
import org.json.simple.parser.JSONParser;
|
||||
import org.json.simple.parser.ParseException;
|
||||
|
||||
public class SMWListImporter implements Runnable, ContentHandler{
|
||||
|
||||
// Importer Variables
|
||||
private final ArrayBlockingQueue<SMWListRow> listEntries;
|
||||
private final Reader importFile;
|
||||
|
||||
private SMWListRow row;
|
||||
private final JSONParser parser;
|
||||
|
||||
// Parser Variables
|
||||
private final StringBuilder value;
|
||||
private final StringBuilder key;
|
||||
private final HashMap<String,String> obj;
|
||||
|
||||
private Boolean isElement;
|
||||
|
||||
public SMWListImporter(final Reader importFile, final int queueSize) {
|
||||
this.listEntries = new ArrayBlockingQueue<SMWListRow>(queueSize);
|
||||
this.importFile = importFile;
|
||||
|
||||
this.row = new SMWListRow();
|
||||
|
||||
this.parser = new JSONParser();
|
||||
|
||||
this.value = new StringBuilder(128);
|
||||
this.key = new StringBuilder(16);
|
||||
this.obj = new HashMap<String,String>();
|
||||
|
||||
this.isElement = false;
|
||||
|
||||
}
|
||||
|
||||
@Override
|
||||
public void startJSON() throws ParseException, IOException {
|
||||
}
|
||||
|
||||
@Override
|
||||
public void endJSON() throws ParseException, IOException {
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean startArray() throws ParseException, IOException {
|
||||
final String key = this.key.toString();
|
||||
|
||||
if (key.equals("items")) {
|
||||
|
||||
this.isElement = true;
|
||||
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean endArray() throws ParseException, IOException {
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean startObject() throws ParseException, IOException {
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean endObject() throws ParseException, IOException {
|
||||
|
||||
if(this.isElement) {
|
||||
|
||||
for (Entry<String, String> e: this.obj.entrySet()) {
|
||||
this.row.add (e.getKey(), e.getValue());
|
||||
}
|
||||
try {
|
||||
this.listEntries.put(this.row);
|
||||
//this.count++;
|
||||
} catch (final InterruptedException e) {
|
||||
ConcurrentLog.logException(e);
|
||||
}
|
||||
this.obj.clear();
|
||||
this.row = new SMWListRow();
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean startObjectEntry(String key) throws ParseException, IOException {
|
||||
this.key.setLength(0);
|
||||
this.key.append(key);
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean primitive(Object value) throws ParseException, IOException {
|
||||
|
||||
this.value.setLength(0);
|
||||
if(value instanceof java.lang.String) {
|
||||
this.value.append((String)value);
|
||||
} else if(value instanceof java.lang.Boolean) {
|
||||
this.value.append(value);
|
||||
} else if(value instanceof java.lang.Number) {
|
||||
this.value.append(value);
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean endObjectEntry() throws ParseException, IOException {
|
||||
|
||||
final String key = this.key.toString();
|
||||
final String value = this.value.toString();
|
||||
|
||||
this.obj.put(key, value);
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void run() {
|
||||
try {
|
||||
ConcurrentLog.info("SMWLISTSYNC", "Importer run()");
|
||||
this.parser.parse(this.importFile, this, true);
|
||||
|
||||
} catch (final IOException e) {
|
||||
ConcurrentLog.logException(e);
|
||||
} catch (final ParseException e) {
|
||||
ConcurrentLog.logException(e);
|
||||
} finally {
|
||||
|
||||
try {
|
||||
ConcurrentLog.info("SMWLISTSYNC", "Importer inserted poison pill in queue");
|
||||
this.listEntries.put(SMWListRow.POISON);
|
||||
} catch (final InterruptedException e) {
|
||||
ConcurrentLog.logException(e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public SMWListRow take() {
|
||||
try {
|
||||
return this.listEntries.take();
|
||||
} catch (final InterruptedException e) {
|
||||
ConcurrentLog.logException(e);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
}
|
|
@ -1,117 +0,0 @@
|
|||
package net.yacy.contentcontrol;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.Reader;
|
||||
import java.util.Iterator;
|
||||
import java.util.concurrent.ArrayBlockingQueue;
|
||||
|
||||
import net.yacy.cora.util.ConcurrentLog;
|
||||
|
||||
import org.json.simple.JSONArray;
|
||||
import org.json.simple.JSONObject;
|
||||
import org.json.simple.parser.JSONParser;
|
||||
import org.json.simple.parser.ParseException;
|
||||
|
||||
public class SMWListImporterFormatObsolete implements Runnable{
|
||||
|
||||
private final ArrayBlockingQueue<SMWListRow> listEntries;
|
||||
private final Reader importFile;
|
||||
private final JSONParser parser;
|
||||
|
||||
public SMWListImporterFormatObsolete(final Reader importFile, final int queueSize) {
|
||||
this.listEntries = new ArrayBlockingQueue<SMWListRow>(queueSize);
|
||||
this.importFile = importFile;
|
||||
this.parser = new JSONParser();
|
||||
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public void run() {
|
||||
try {
|
||||
ConcurrentLog.info("SMWLISTSYNC", "Importer run()");
|
||||
Object obj = this.parser.parse(this.importFile);
|
||||
|
||||
JSONObject jsonObject = (JSONObject) obj;
|
||||
|
||||
JSONArray items = (JSONArray) jsonObject.get("items");
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
Iterator<JSONObject> iterator = items.iterator();
|
||||
while (iterator.hasNext()) {
|
||||
this.parseItem (iterator.next());
|
||||
}
|
||||
|
||||
} catch (final IOException e) {
|
||||
ConcurrentLog.logException(e);
|
||||
} catch (final ParseException e) {
|
||||
ConcurrentLog.logException(e);
|
||||
} finally {
|
||||
|
||||
try {
|
||||
ConcurrentLog.info("SMWLISTSYNC", "Importer inserted poison pill in queue");
|
||||
this.listEntries.put(SMWListRow.POISON);
|
||||
} catch (final InterruptedException e) {
|
||||
ConcurrentLog.logException(e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void parseItem(JSONObject jsonObject) {
|
||||
|
||||
try {
|
||||
SMWListRow row = new SMWListRow();
|
||||
@SuppressWarnings("unchecked")
|
||||
Iterator<String> iterator = jsonObject.keySet().iterator();
|
||||
|
||||
while (iterator.hasNext()) {
|
||||
String entryKey = iterator.next();
|
||||
|
||||
Object value = jsonObject.get (entryKey);
|
||||
String valueKey = "";
|
||||
|
||||
if (value instanceof java.lang.String) {
|
||||
valueKey = value.toString();
|
||||
} else if (value instanceof JSONArray) {
|
||||
valueKey = jsonListAll ((JSONArray) value);
|
||||
}
|
||||
|
||||
row.add (entryKey, valueKey);
|
||||
}
|
||||
|
||||
this.listEntries.put(row);
|
||||
|
||||
} catch (final Exception e) {
|
||||
ConcurrentLog.info("SMWLISTSYNC", "import of entry failed");
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
private String jsonListAll(JSONArray value) {
|
||||
String res = "";
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
Iterator<Object> iterator = value.listIterator();
|
||||
while (iterator.hasNext()) {
|
||||
Object val = iterator.next();
|
||||
res += val.toString()+",";
|
||||
}
|
||||
|
||||
if (res.endsWith (",")) {
|
||||
res = res.substring (0, res.length()-1);
|
||||
}
|
||||
|
||||
return res;
|
||||
}
|
||||
|
||||
|
||||
public SMWListRow take() {
|
||||
try {
|
||||
return this.listEntries.take();
|
||||
} catch (final InterruptedException e) {
|
||||
ConcurrentLog.logException(e);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
}
|
|
@ -1,24 +0,0 @@
|
|||
package net.yacy.contentcontrol;
|
||||
|
||||
import net.yacy.kelondro.blob.Tables;
|
||||
|
||||
public class SMWListRow {
|
||||
|
||||
private Tables.Data data;
|
||||
|
||||
public static final SMWListRow POISON = new SMWListRow();
|
||||
public static final SMWListRow EMPTY = new SMWListRow();
|
||||
|
||||
public SMWListRow() {
|
||||
this.data = new Tables.Data();
|
||||
}
|
||||
|
||||
public void add (String key, String value) {
|
||||
this.data.put(key, value);
|
||||
}
|
||||
|
||||
public Tables.Data getData() {
|
||||
return this.data;
|
||||
}
|
||||
|
||||
}
|
|
@ -1,201 +0,0 @@
|
|||
package net.yacy.contentcontrol;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.InputStreamReader;
|
||||
import java.net.MalformedURLException;
|
||||
import java.net.URL;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
|
||||
import net.yacy.cora.document.encoding.UTF8;
|
||||
import net.yacy.cora.protocol.ClientIdentification;
|
||||
import net.yacy.cora.protocol.http.HTTPClient;
|
||||
import net.yacy.cora.util.CommonPattern;
|
||||
import net.yacy.cora.util.ConcurrentLog;
|
||||
import net.yacy.search.Switchboard;
|
||||
|
||||
public class SMWListSyncThread implements Runnable {
|
||||
|
||||
private final Switchboard sb;
|
||||
private Boolean locked = false;
|
||||
private String lastsync = "1900-01-01T01:00:00";
|
||||
private String currenttimestamp = "1900-01-01T01:00:00";
|
||||
private long offset = 0;
|
||||
private final long limit = 500;
|
||||
private long currentmax = 0;
|
||||
private boolean runningjob = false;
|
||||
|
||||
private String targetList;
|
||||
private String parameters;
|
||||
private String query;
|
||||
|
||||
public static Boolean dirty = false;
|
||||
|
||||
public SMWListSyncThread(final Switchboard sb, final String targetList, final String query, final String parameters, final Boolean purgeOnInit) {
|
||||
this.sb = sb;
|
||||
this.targetList = targetList;
|
||||
this.parameters = parameters;
|
||||
this.query = query;
|
||||
if (purgeOnInit) {
|
||||
this.sb.tables.clear(targetList);
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
private final String wikiurlify (String s) {
|
||||
String ret = s;
|
||||
ret = ret.replace("-", "-2D");
|
||||
ret = ret.replace("+", "-2B");
|
||||
ret = ret.replace(" ", "-20");
|
||||
ret = ret.replace("[", "-5B");
|
||||
ret = ret.replace("]", "-5D");
|
||||
ret = ret.replace(":", "-3A");
|
||||
ret = ret.replace(">", "-3E");
|
||||
ret = ret.replace("?", "-3F");
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
@Override
|
||||
public final void run() {
|
||||
|
||||
if (!this.locked) {
|
||||
this.locked = true;
|
||||
if (this.sb.getConfigBool("contentcontrol.smwimport.enabled", false) == true) {
|
||||
|
||||
if (!this.runningjob) {
|
||||
|
||||
// we have to count all new elements first
|
||||
try {
|
||||
if (!this.sb.getConfig("contentcontrol.smwimport.baseurl","").equals("")) {
|
||||
URL urlCount;
|
||||
|
||||
urlCount = new URL(
|
||||
this.sb.getConfig(
|
||||
"contentcontrol.smwimport.baseurl",
|
||||
"")
|
||||
+ wikiurlify ("/[["+this.query+"]] [[Modification date::>" +this.lastsync+ "]]")
|
||||
|
||||
+ wikiurlify (this.parameters)
|
||||
|
||||
+ "/mainlabel%3D"
|
||||
+ "/offset%3D0"
|
||||
+ "/limit%3D200000"
|
||||
+ "/format%3Dystat");
|
||||
|
||||
String reply = UTF8.String(new HTTPClient(ClientIdentification.yacyInternetCrawlerAgent).GETbytes(urlCount.toString(), null, null, false));
|
||||
String overallcount = CommonPattern.COMMA.split(reply)[0];
|
||||
String lastsyncstring = CommonPattern.COMMA.split(reply)[1];
|
||||
this.currentmax = Integer.parseInt(overallcount);
|
||||
|
||||
if (this.currentmax > 0) {
|
||||
ConcurrentLog.info("SMWLISTSYNC",
|
||||
"import job counts "
|
||||
+ this.currentmax
|
||||
+ " new elements between "
|
||||
+ this.lastsync + " and "
|
||||
+ this.currenttimestamp);
|
||||
|
||||
this.currenttimestamp = this.lastsync;
|
||||
|
||||
this.runningjob = true;
|
||||
this.lastsync = lastsyncstring;
|
||||
this.offset = 0;
|
||||
}
|
||||
} else {
|
||||
ConcurrentLog.warn("SMWLISTSYNC",
|
||||
"No SMWimport URL defined");
|
||||
}
|
||||
} catch (final MalformedURLException e) {
|
||||
// TODO Auto-generated catch block
|
||||
e.printStackTrace();
|
||||
} catch (final IOException e) {
|
||||
// TODO Auto-generated catch block
|
||||
e.printStackTrace();
|
||||
}
|
||||
|
||||
|
||||
} else {
|
||||
|
||||
// there are new elements to be imported
|
||||
ConcurrentLog.info("SMWLISTSYNC",
|
||||
"importing max. " + this.limit
|
||||
+ " elements at " + this.offset + " of "
|
||||
+ this.currentmax + ", since "
|
||||
+ this.currenttimestamp);
|
||||
URL urlImport;
|
||||
try {
|
||||
if (!this.sb.getConfig("contentcontrol.smwimport.baseurl","").equals("")) {
|
||||
urlImport = new URL(
|
||||
this.sb.getConfig(
|
||||
"contentcontrol.smwimport.baseurl",
|
||||
"")
|
||||
+ wikiurlify ("/[["+this.query+"]] [[Modification date::>" +this.currenttimestamp+ "]]")
|
||||
|
||||
+ wikiurlify (this.parameters)
|
||||
|
||||
+ "/mainlabel%3D"
|
||||
+ "/syntax%3Dobsolete"
|
||||
+ "/offset%3D" + this.offset
|
||||
+ "/limit%3D" + this.limit
|
||||
+ "/format%3Djson");
|
||||
|
||||
this.offset += this.limit;
|
||||
if (this.offset > this.currentmax) {
|
||||
this.runningjob = false;
|
||||
}
|
||||
|
||||
InputStreamReader reader = null;
|
||||
try {
|
||||
reader = new InputStreamReader(
|
||||
urlImport.openStream(), StandardCharsets.UTF_8);
|
||||
} catch (final Exception e) {
|
||||
ConcurrentLog.logException(e);
|
||||
this.runningjob = false;
|
||||
}
|
||||
|
||||
if (reader != null) {
|
||||
SMWListImporterFormatObsolete smwListImporter = null;
|
||||
try {
|
||||
smwListImporter = new SMWListImporterFormatObsolete(
|
||||
reader, 200);
|
||||
} catch (final Exception e) {
|
||||
// TODO: display an error message
|
||||
ConcurrentLog.logException(e);
|
||||
this.runningjob = false;
|
||||
}
|
||||
Thread t;
|
||||
SMWListRow row;
|
||||
t = new Thread(smwListImporter,"SMW List Importer");
|
||||
t.start();
|
||||
while ((row = smwListImporter.take()) != SMWListRow.POISON) {
|
||||
if (row == SMWListRow.EMPTY) {
|
||||
this.runningjob = false;
|
||||
} else {
|
||||
try {
|
||||
this.sb.tables.insert(targetList, row.getData());
|
||||
|
||||
dirty = true;
|
||||
|
||||
} catch (final Exception e) {
|
||||
// TODO Auto-generated catch block
|
||||
e.printStackTrace();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
} catch (final MalformedURLException e2) {
|
||||
// TODO Auto-generated catch block
|
||||
e2.printStackTrace();
|
||||
}
|
||||
|
||||
}
|
||||
this.locked = false;
|
||||
}
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
}
|
|
@ -26,31 +26,31 @@ import java.util.ArrayList;
|
|||
import java.util.Collection;
|
||||
import java.util.List;
|
||||
|
||||
import org.apache.solr.client.solrj.SolrClient;
|
||||
import org.apache.solr.client.solrj.SolrServerException;
|
||||
import org.apache.solr.client.solrj.embedded.EmbeddedSolrServer;
|
||||
import org.apache.solr.client.solrj.impl.XMLResponseParser;
|
||||
import org.apache.solr.client.solrj.request.ContentStreamUpdateRequest;
|
||||
import org.apache.solr.client.solrj.request.LukeRequest;
|
||||
import org.apache.solr.client.solrj.request.UpdateRequest;
|
||||
import org.apache.solr.client.solrj.response.LukeResponse;
|
||||
import org.apache.solr.client.solrj.response.LukeResponse.FieldInfo;
|
||||
import org.apache.solr.client.solrj.response.QueryResponse;
|
||||
import org.apache.solr.common.SolrDocumentList;
|
||||
import org.apache.solr.common.SolrException;
|
||||
import org.apache.solr.common.SolrInputDocument;
|
||||
import org.apache.solr.common.params.CommonParams;
|
||||
import org.apache.solr.common.params.ModifiableSolrParams;
|
||||
import org.apache.solr.common.util.NamedList;
|
||||
|
||||
import net.yacy.cora.federate.solr.instance.ServerShard;
|
||||
import net.yacy.cora.util.ConcurrentLog;
|
||||
import net.yacy.search.schema.CollectionSchema;
|
||||
|
||||
import org.apache.solr.common.SolrDocumentList;
|
||||
import org.apache.solr.common.SolrException;
|
||||
import org.apache.solr.common.SolrInputDocument;
|
||||
import org.apache.solr.common.params.ModifiableSolrParams;
|
||||
import org.apache.solr.common.util.NamedList;
|
||||
import org.apache.solr.client.solrj.embedded.EmbeddedSolrServer;
|
||||
import org.apache.solr.client.solrj.impl.XMLResponseParser;
|
||||
import org.apache.solr.client.solrj.SolrClient;
|
||||
import org.apache.solr.client.solrj.SolrServerException;
|
||||
import org.apache.solr.client.solrj.request.ContentStreamUpdateRequest;
|
||||
import org.apache.solr.client.solrj.request.LukeRequest;
|
||||
import org.apache.solr.client.solrj.request.UpdateRequest;
|
||||
import org.apache.solr.client.solrj.response.LukeResponse.FieldInfo;
|
||||
import org.apache.solr.client.solrj.response.LukeResponse;
|
||||
import org.apache.solr.client.solrj.response.QueryResponse;
|
||||
import org.apache.solr.common.params.CommonParams;
|
||||
|
||||
public abstract class SolrServerConnector extends AbstractSolrConnector implements SolrConnector {
|
||||
|
||||
protected final static ConcurrentLog log = new ConcurrentLog(SolrServerConnector.class.getName());
|
||||
public final static org.apache.lucene.analysis.CharArrayMap<Byte> classLoaderSynchro = new org.apache.lucene.analysis.CharArrayMap<Byte>(0, true);
|
||||
public final static org.apache.lucene.analysis.CharArrayMap<Byte> classLoaderSynchro = new org.apache.lucene.analysis.CharArrayMap<>(0, true);
|
||||
// pre-instantiate this object to prevent sun.misc.Launcher$AppClassLoader deadlocks
|
||||
// this is a very nasty problem; solr instantiates objects dynamically which can cause deadlocks
|
||||
static {
|
||||
|
@ -69,7 +69,7 @@ public abstract class SolrServerConnector extends AbstractSolrConnector implemen
|
|||
public SolrClient getServer() {
|
||||
return this.server;
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public void commit(final boolean softCommit) {
|
||||
if (this.server == null) return;
|
||||
|
@ -158,8 +158,8 @@ public abstract class SolrServerConnector extends AbstractSolrConnector implemen
|
|||
@Override
|
||||
public void deleteByIds(final Collection<String> ids) throws IOException {
|
||||
if (this.server == null) return;
|
||||
List<String> l = new ArrayList<String>();
|
||||
for (String s: ids) l.add(s);
|
||||
final List<String> l = new ArrayList<>();
|
||||
for (final String s: ids) l.add(s);
|
||||
synchronized (this.server) {
|
||||
try {
|
||||
this.server.deleteById(l, -1);
|
||||
|
@ -247,7 +247,7 @@ public abstract class SolrServerConnector extends AbstractSolrConnector implemen
|
|||
@Override
|
||||
public void add(final Collection<SolrInputDocument> solrdocs) throws IOException, SolrException {
|
||||
if (this.server == null) return;
|
||||
for (SolrInputDocument solrdoc : solrdocs) {
|
||||
for (final SolrInputDocument solrdoc : solrdocs) {
|
||||
if (solrdoc.containsKey("_version_")) solrdoc.setField("_version_",0L); // prevent Solr "version conflict"
|
||||
}
|
||||
synchronized (this.server) {
|
||||
|
@ -278,8 +278,8 @@ public abstract class SolrServerConnector extends AbstractSolrConnector implemen
|
|||
this.server.add(solrdocs, -1);
|
||||
} catch (final Throwable ee) {
|
||||
ConcurrentLog.logException(ee);
|
||||
List<String> ids = new ArrayList<String>();
|
||||
for (SolrInputDocument solrdoc : solrdocs) ids.add((String) solrdoc.getFieldValue(CollectionSchema.id.getSolrFieldName()));
|
||||
final List<String> ids = new ArrayList<>();
|
||||
for (final SolrInputDocument solrdoc : solrdocs) ids.add((String) solrdoc.getFieldValue(CollectionSchema.id.getSolrFieldName()));
|
||||
log.warn(e.getMessage() + " IDs=" + ids.toString());
|
||||
throw new IOException(ee);
|
||||
}
|
||||
|
@ -300,11 +300,11 @@ public abstract class SolrServerConnector extends AbstractSolrConnector implemen
|
|||
public SolrDocumentList getDocumentListByParams(ModifiableSolrParams params) throws IOException {
|
||||
if (this.server == null) throw new IOException("server disconnected");
|
||||
// during the solr query we set the thread name to the query string to get more debugging info in thread dumps
|
||||
String q = params.get(CommonParams.Q);
|
||||
String fq = params.get(CommonParams.FQ);
|
||||
String sort = params.get(CommonParams.SORT);
|
||||
String fl = params.get(CommonParams.FL);
|
||||
String threadname = Thread.currentThread().getName();
|
||||
final String q = params.get(CommonParams.Q);
|
||||
final String fq = params.get(CommonParams.FQ);
|
||||
final String sort = params.get(CommonParams.SORT);
|
||||
final String fl = params.get(CommonParams.FL);
|
||||
final String threadname = Thread.currentThread().getName();
|
||||
QueryResponse rsp;
|
||||
int retry = 0;
|
||||
Throwable error = null;
|
||||
|
@ -322,13 +322,13 @@ public abstract class SolrServerConnector extends AbstractSolrConnector implemen
|
|||
clearCaches(); // prevent further OOM if this was caused by OOM
|
||||
}
|
||||
ConcurrentLog.severe("SolrServerConnector", "Failed to query remote Solr: " + error.getMessage() + ", query:" + q + (fq == null ? "" : ", fq = " + fq));
|
||||
try {Thread.sleep(1000);} catch (InterruptedException e) {}
|
||||
try {Thread.sleep(1000);} catch (final InterruptedException e) {}
|
||||
}
|
||||
throw new IOException("Error executing query", error);
|
||||
}
|
||||
|
||||
|
||||
// luke requests: these do not work for attached SolrCloud Server
|
||||
|
||||
|
||||
public Collection<FieldInfo> getFields() throws SolrServerException {
|
||||
// get all fields contained in index
|
||||
return getIndexBrowser(false).getFieldInfo().values();
|
||||
|
@ -342,10 +342,10 @@ public abstract class SolrServerConnector extends AbstractSolrConnector implemen
|
|||
public int getSegmentCount() {
|
||||
if (this.server == null) return 0;
|
||||
try {
|
||||
LukeResponse lukeResponse = getIndexBrowser(false);
|
||||
NamedList<Object> info = lukeResponse.getIndexInfo();
|
||||
final LukeResponse lukeResponse = getIndexBrowser(false);
|
||||
final NamedList<Object> info = lukeResponse.getIndexInfo();
|
||||
if (info == null) return 0;
|
||||
Integer segmentCount = (Integer) info.get("segmentCount");
|
||||
final Integer segmentCount = (Integer) info.get("segmentCount");
|
||||
if (segmentCount == null) return 1;
|
||||
return segmentCount.intValue();
|
||||
} catch (final Throwable e) {
|
||||
|
@ -356,31 +356,31 @@ public abstract class SolrServerConnector extends AbstractSolrConnector implemen
|
|||
}
|
||||
|
||||
private int useluke = 0; // 3-value logic: 1=yes, -1=no, 0=dontknow
|
||||
|
||||
|
||||
@Override
|
||||
public long getSize() {
|
||||
if (this.server == null) return 0;
|
||||
if (this.server instanceof ServerShard) {
|
||||
// the server can be a single shard; we don't know here
|
||||
// to test that, we submit requests to bots variants
|
||||
if (useluke == 1) return getSizeLukeRequest();
|
||||
if (useluke == -1) return getSizeQueryRequest();
|
||||
long ls = getSizeLukeRequest();
|
||||
long qs = getSizeQueryRequest();
|
||||
if (this.useluke == 1) return getSizeLukeRequest();
|
||||
if (this.useluke == -1) return getSizeQueryRequest();
|
||||
final long ls = getSizeLukeRequest();
|
||||
final long qs = getSizeQueryRequest();
|
||||
if (ls == 0 && qs == 0) {
|
||||
// we don't know if this is caused by an error or not; don't change the useluke value
|
||||
return 0;
|
||||
}
|
||||
if (ls == qs) {
|
||||
useluke = 1;
|
||||
this.useluke = 1;
|
||||
return ls;
|
||||
}
|
||||
useluke = -1;
|
||||
this.useluke = -1;
|
||||
return qs;
|
||||
}
|
||||
return getSizeLukeRequest();
|
||||
}
|
||||
|
||||
|
||||
private long getSizeQueryRequest() {
|
||||
if (this.server == null) return 0;
|
||||
try {
|
||||
|
@ -394,13 +394,13 @@ public abstract class SolrServerConnector extends AbstractSolrConnector implemen
|
|||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
private long getSizeLukeRequest() {
|
||||
if (this.server == null) return 0;
|
||||
try {
|
||||
LukeResponse lukeResponse = getIndexBrowser(false);
|
||||
final LukeResponse lukeResponse = getIndexBrowser(false);
|
||||
if (lukeResponse == null) return 0;
|
||||
Integer numDocs = lukeResponse.getNumDocs();
|
||||
final Integer numDocs = lukeResponse.getNumDocs();
|
||||
if (numDocs == null) return 0;
|
||||
return numDocs.longValue();
|
||||
} catch (final Throwable e) {
|
||||
|
@ -409,7 +409,7 @@ public abstract class SolrServerConnector extends AbstractSolrConnector implemen
|
|||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
private LukeResponse getIndexBrowser(final boolean showSchema) throws SolrServerException {
|
||||
// get all fields contained in index
|
||||
final LukeRequest lukeRequest = new LukeRequest();
|
||||
|
@ -419,7 +419,7 @@ public abstract class SolrServerConnector extends AbstractSolrConnector implemen
|
|||
LukeResponse lukeResponse = null;
|
||||
try {
|
||||
lukeResponse = lukeRequest.process(this.server);
|
||||
} catch (IOException e) {
|
||||
} catch (final IOException e) {
|
||||
throw new SolrServerException(e.getMessage());
|
||||
}
|
||||
return lukeResponse;
|
||||
|
|
|
@ -27,10 +27,6 @@ import java.util.Collection;
|
|||
import java.util.Map;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
|
||||
import net.yacy.cora.document.encoding.ASCII;
|
||||
import net.yacy.cora.util.ConcurrentLog;
|
||||
import net.yacy.kelondro.util.MemoryControl;
|
||||
|
||||
import org.apache.solr.client.solrj.SolrClient;
|
||||
import org.apache.solr.client.solrj.embedded.EmbeddedSolrServer;
|
||||
import org.apache.solr.core.CoreContainer;
|
||||
|
@ -38,17 +34,21 @@ import org.apache.solr.core.SolrCore;
|
|||
|
||||
import com.google.common.io.Files;
|
||||
|
||||
import net.yacy.cora.document.encoding.ASCII;
|
||||
import net.yacy.cora.util.ConcurrentLog;
|
||||
import net.yacy.kelondro.util.MemoryControl;
|
||||
|
||||
public class EmbeddedInstance implements SolrInstance {
|
||||
|
||||
private final static String[] confFiles = {"solrconfig.xml", "schema.xml", "stopwords.txt", "synonyms.txt", "protwords.txt", "currency.xml", "elevate.xml", "xslt/example.xsl", "xslt/json.xsl", "lang/"};
|
||||
// additional a optional solrcore.properties (or solrcore.x86.properties for 32bit systems is copied
|
||||
// additional a optional solrcore.properties (or solrcore.x86.properties for 32bit systems is copied
|
||||
private CoreContainer coreContainer;
|
||||
private String defaultCoreName;
|
||||
private SolrCore defaultCore;
|
||||
private SolrClient defaultCoreServer;
|
||||
private File containerPath;
|
||||
private Map<String, SolrCore> cores;
|
||||
private Map<String, SolrClient> server;
|
||||
private final String defaultCoreName;
|
||||
private final SolrCore defaultCore;
|
||||
private final SolrClient defaultCoreServer;
|
||||
private final File containerPath;
|
||||
private final Map<String, SolrCore> cores;
|
||||
private final Map<String, SolrClient> server;
|
||||
|
||||
public EmbeddedInstance(final File solr_config, final File containerPath, String givenDefaultCoreName, String[] initializeCoreNames) throws IOException {
|
||||
super();
|
||||
|
@ -56,30 +56,30 @@ public class EmbeddedInstance implements SolrInstance {
|
|||
this.containerPath = containerPath;
|
||||
|
||||
// ensure that default core path exists
|
||||
File defaultCorePath = new File(containerPath, givenDefaultCoreName);
|
||||
final File defaultCorePath = new File(containerPath, givenDefaultCoreName);
|
||||
if (!defaultCorePath.exists()) defaultCorePath.mkdirs();
|
||||
|
||||
// migrate old conf directory
|
||||
File oldConf = new File(containerPath, "conf");
|
||||
File confDir = new File(defaultCorePath, "conf");
|
||||
final File oldConf = new File(containerPath, "conf");
|
||||
final File confDir = new File(defaultCorePath, "conf");
|
||||
if (oldConf.exists()) oldConf.renameTo(confDir);
|
||||
|
||||
// migrate old data directory
|
||||
File oldData = new File(containerPath, "data");
|
||||
File dataDir = new File(defaultCorePath, "data");
|
||||
final File oldData = new File(containerPath, "data");
|
||||
final File dataDir = new File(defaultCorePath, "data");
|
||||
if (oldData.exists()) oldData.renameTo(dataDir);
|
||||
|
||||
// create index subdirectory in data if it does not exist
|
||||
File indexDir = new File(dataDir, "index");
|
||||
final File indexDir = new File(dataDir, "index");
|
||||
if (!indexDir.exists()) indexDir.mkdirs();
|
||||
|
||||
// initialize the cores' configuration
|
||||
for (String coreName: initializeCoreNames) {
|
||||
for (final String coreName: initializeCoreNames) {
|
||||
initializeCoreConf(solr_config, containerPath, coreName);
|
||||
}
|
||||
|
||||
// initialize the coreContainer
|
||||
File configFile = new File(solr_config, "solr.xml"); // the configuration file for all cores
|
||||
final File configFile = new File(solr_config, "solr.xml"); // the configuration file for all cores
|
||||
this.coreContainer = CoreContainer.createAndLoad(containerPath.toPath(), configFile.toPath()); // this may take indefinitely long if solr files are broken
|
||||
if (this.coreContainer == null) throw new IOException("cannot create core container dir = " + containerPath + ", configFile = " + configFile);
|
||||
|
||||
|
@ -94,9 +94,9 @@ public class EmbeddedInstance implements SolrInstance {
|
|||
this.defaultCoreServer = new EmbeddedSolrServer(this.coreContainer, this.defaultCoreName);
|
||||
|
||||
// initialize core cache
|
||||
this.cores = new ConcurrentHashMap<String, SolrCore>();
|
||||
this.cores = new ConcurrentHashMap<>();
|
||||
this.cores.put(this.defaultCoreName, this.defaultCore);
|
||||
this.server = new ConcurrentHashMap<String, SolrClient>();
|
||||
this.server = new ConcurrentHashMap<>();
|
||||
this.server.put(this.defaultCoreName, this.defaultCoreServer);
|
||||
}
|
||||
|
||||
|
@ -113,42 +113,42 @@ public class EmbeddedInstance implements SolrInstance {
|
|||
private static void initializeCoreConf(final File solr_config, final File containerPath, String coreName) {
|
||||
|
||||
// ensure that default core path exists
|
||||
File corePath = new File(containerPath, coreName);
|
||||
final File corePath = new File(containerPath, coreName);
|
||||
if (!corePath.exists()) corePath.mkdirs();
|
||||
|
||||
// check if core.properties exists in each path (thats new in Solr 5.0)
|
||||
File core_properties = new File(corePath, "core.properties");
|
||||
final File core_properties = new File(corePath, "core.properties");
|
||||
if (!core_properties.exists()) {
|
||||
// create the file
|
||||
try (
|
||||
/* Automatically closed by this try-with-resources statement */
|
||||
FileOutputStream fos = new FileOutputStream(core_properties);
|
||||
) {
|
||||
/* Automatically closed by this try-with-resources statement */
|
||||
FileOutputStream fos = new FileOutputStream(core_properties);
|
||||
) {
|
||||
fos.write(ASCII.getBytes("name=" + coreName + "\n"));
|
||||
fos.write(ASCII.getBytes("shard=${shard:}\n"));
|
||||
fos.write(ASCII.getBytes("collection=${collection:" + coreName + "}\n"));
|
||||
fos.write(ASCII.getBytes("config=${solrconfig:solrconfig.xml}\n"));
|
||||
fos.write(ASCII.getBytes("schema=${schema:schema.xml}\n"));
|
||||
fos.write(ASCII.getBytes("coreNodeName=${coreNodeName:}\n"));
|
||||
} catch (IOException e) {
|
||||
} catch (final IOException e) {
|
||||
ConcurrentLog.logException(e);
|
||||
}
|
||||
}
|
||||
|
||||
// ensure necessary subpaths exist
|
||||
File conf = new File(corePath, "conf");
|
||||
final File conf = new File(corePath, "conf");
|
||||
conf.mkdirs();
|
||||
File data = new File(corePath, "data");
|
||||
final File data = new File(corePath, "data");
|
||||
data.mkdirs();
|
||||
|
||||
// (over-)write configuration into conf path
|
||||
File source, target;
|
||||
for (String cf: confFiles) {
|
||||
for (final String cf: confFiles) {
|
||||
source = new File(solr_config, cf);
|
||||
if (source.isDirectory()) {
|
||||
target = new File(conf, cf);
|
||||
target.mkdirs();
|
||||
for (String cfl: source.list()) {
|
||||
for (final String cfl: source.list()) {
|
||||
try {
|
||||
Files.copy(new File(source, cfl), new File(target, cfl));
|
||||
} catch (final IOException e) {
|
||||
|
@ -168,7 +168,7 @@ public class EmbeddedInstance implements SolrInstance {
|
|||
|
||||
// copy the solrcore.properties
|
||||
// for 32bit systems (os.arch name not containing '64') take the solrcore.x86.properties as solrcore.properties if exists
|
||||
String os = System.getProperty("os.arch");
|
||||
final String os = System.getProperty("os.arch");
|
||||
if (os.contains("64")) {
|
||||
source = new File(solr_config, "solrcore.properties");
|
||||
} else {
|
||||
|
@ -242,7 +242,7 @@ public class EmbeddedInstance implements SolrInstance {
|
|||
|
||||
@Override
|
||||
public synchronized void close() {
|
||||
for (SolrCore core: cores.values()) core.close();
|
||||
for (final SolrCore core: this.cores.values()) core.close();
|
||||
if (this.coreContainer != null) try {
|
||||
this.coreContainer.shutdown();
|
||||
this.coreContainer = null;
|
||||
|
|
|
@ -36,7 +36,6 @@ import java.util.Set;
|
|||
import java.util.concurrent.BlockingQueue;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
|
||||
import net.yacy.contentcontrol.ContentControlFilterUpdateThread;
|
||||
import net.yacy.cora.date.ISO8601Formatter;
|
||||
import net.yacy.cora.document.encoding.ASCII;
|
||||
import net.yacy.cora.document.encoding.UTF8;
|
||||
|
@ -360,13 +359,13 @@ public final class CrawlStacker implements WorkflowTask<Request>{
|
|||
final boolean proxy = (entry.initiator() == null || entry.initiator().length == 0 || ASCII.String(entry.initiator()).equals("------------")) && profile.handle().equals(this.crawler.defaultProxyProfile.handle());
|
||||
final boolean remote = profile.handle().equals(this.crawler.defaultRemoteProfile.handle());
|
||||
final boolean global =
|
||||
(profile.remoteIndexing()) /* granted */ &&
|
||||
(entry.depth() == profile.depth()) /* leaf node */ &&
|
||||
//(initiatorHash.equals(yacyCore.seedDB.mySeed.hash)) /* not proxy */ &&
|
||||
(
|
||||
(this.peers.mySeed().isSenior()) ||
|
||||
(this.peers.mySeed().isPrincipal())
|
||||
) /* qualified */;
|
||||
(profile.remoteIndexing()) /* granted */ &&
|
||||
(entry.depth() == profile.depth()) /* leaf node */ &&
|
||||
//(initiatorHash.equals(yacyCore.seedDB.mySeed.hash)) /* not proxy */ &&
|
||||
(
|
||||
(this.peers.mySeed().isSenior()) ||
|
||||
(this.peers.mySeed().isPrincipal())
|
||||
) /* qualified */;
|
||||
|
||||
if (!local && !global && !remote && !proxy) {
|
||||
error = "URL '" + entry.url().toString() + "' cannot be crawled. initiator = " + ((entry.initiator() == null) ? "" : ASCII.String(entry.initiator())) + ", profile.handle = " + profile.handle();
|
||||
|
@ -424,7 +423,7 @@ public final class CrawlStacker implements WorkflowTask<Request>{
|
|||
if (dbocc != null) {
|
||||
return CRAWL_REJECT_REASON_DOUBLE_IN_PREFIX + ": " + dbocc.name();
|
||||
}
|
||||
String urls = url.toNormalform(false);
|
||||
final String urls = url.toNormalform(false);
|
||||
final long oldDate = this.indexSegment.getLoadTime(url.hash());
|
||||
|
||||
// deny urls that exceed allowed number of occurrences
|
||||
|
@ -441,7 +440,7 @@ public final class CrawlStacker implements WorkflowTask<Request>{
|
|||
if (this.log.isFine()) this.log.fine("URL '" + urlstring + "' appeared too often in result stack, a maximum of " + maxAllowedPagesPerDomain + " is allowed.");
|
||||
return "result stack domain counter exceeded (test by domainCount)";
|
||||
}
|
||||
*/
|
||||
*/
|
||||
}
|
||||
|
||||
//final Long oldDate = oldEntry == null ? null : oldEntry.date;
|
||||
|
@ -453,7 +452,7 @@ public final class CrawlStacker implements WorkflowTask<Request>{
|
|||
if (recrawl) {
|
||||
if (CrawlStacker.log.isFine())
|
||||
CrawlStacker.log.fine("RE-CRAWL of URL '" + urlstring + "': this url was crawled " +
|
||||
((System.currentTimeMillis() - oldDate) / 60000 / 60 / 24) + " days ago.");
|
||||
((System.currentTimeMillis() - oldDate) / 60000 / 60 / 24) + " days ago.");
|
||||
} else {
|
||||
return CRAWL_REJECT_REASON_DOUBLE_IN_PREFIX + ": local index, recrawl rejected. Document date = "
|
||||
+ ISO8601Formatter.FORMATTER.format(new Date(oldDate)) + " is not older than crawl profile recrawl minimum date = "
|
||||
|
@ -574,26 +573,6 @@ public final class CrawlStacker implements WorkflowTask<Request>{
|
|||
}
|
||||
}
|
||||
|
||||
if (Switchboard.getSwitchboard().getConfigBool(
|
||||
"contentcontrol.enabled", false) == true) {
|
||||
|
||||
if (!Switchboard.getSwitchboard()
|
||||
.getConfig("contentcontrol.mandatoryfilterlist", "")
|
||||
.equals("")) {
|
||||
final FilterEngine f = ContentControlFilterUpdateThread.getNetworkFilter();
|
||||
if (f != null) {
|
||||
if (!f.isListed(url, null)) {
|
||||
|
||||
return "the url '"
|
||||
+ url
|
||||
+ "' does not belong to the network mandatory filter list";
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
final boolean local = url.isLocal();
|
||||
if (this.acceptLocalURLs && local) return null;
|
||||
if (this.acceptGlobalURLs && !local) return null;
|
||||
|
@ -604,8 +583,8 @@ public final class CrawlStacker implements WorkflowTask<Request>{
|
|||
//assert local == yacyURL.isLocalDomain(url.hash()); // TODO: remove the dnsResolve above!
|
||||
final InetAddress ia = Domains.dnsResolve(host);
|
||||
return (local) ?
|
||||
("the host '" + host + "' is local, but local addresses are not accepted: " + ((ia == null) ? "DNS lookup resulted in null (unknown host name)" : ia.getHostAddress())) :
|
||||
("the host '" + host + "' is global, but global addresses are not accepted: " + ((ia == null) ? "null" : ia.getHostAddress()));
|
||||
("the host '" + host + "' is local, but local addresses are not accepted: " + ((ia == null) ? "DNS lookup resulted in null (unknown host name)" : ia.getHostAddress())) :
|
||||
("the host '" + host + "' is global, but global addresses are not accepted: " + ((ia == null) ? "null" : ia.getHostAddress()));
|
||||
}
|
||||
|
||||
public String urlInAcceptedDomainHash(final byte[] urlhash) {
|
||||
|
@ -617,8 +596,8 @@ public final class CrawlStacker implements WorkflowTask<Request>{
|
|||
if (this.acceptLocalURLs && local) return null;
|
||||
if (this.acceptGlobalURLs && !local) return null;
|
||||
return (local) ?
|
||||
("the urlhash '" + ASCII.String(urlhash) + "' is local, but local addresses are not accepted") :
|
||||
("the urlhash '" + ASCII.String(urlhash) + "' is global, but global addresses are not accepted");
|
||||
("the urlhash '" + ASCII.String(urlhash) + "' is local, but local addresses are not accepted") :
|
||||
("the urlhash '" + ASCII.String(urlhash) + "' is global, but global addresses are not accepted");
|
||||
}
|
||||
|
||||
public boolean acceptLocalURLs() {
|
||||
|
|
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
Loading…
Reference in New Issue
Block a user