diff --git a/defaults/yacy.init b/defaults/yacy.init
index 980f8147e..751aa379e 100644
--- a/defaults/yacy.init
+++ b/defaults/yacy.init
@@ -1056,7 +1056,6 @@ color_searchurlhover = #008000
# - to check whats in solr after indexing, open http://localhost:8983/solr/admin/
federated.service.solr.indexing.enabled = false
federated.service.solr.indexing.url = http://127.0.0.1:8983/solr
-federated.service.solr.indexing.commitWithinMs = -1
federated.service.solr.indexing.sharding = MODULO_HOST_MD5
federated.service.solr.indexing.schemefile = solr.keys.default.list
# the lazy attribute causes that fields containing "" or 0 are not added and not written
diff --git a/htroot/IndexFederated_p.java b/htroot/IndexFederated_p.java
index 2a5c050a2..6af0118b6 100644
--- a/htroot/IndexFederated_p.java
+++ b/htroot/IndexFederated_p.java
@@ -79,7 +79,6 @@ public class IndexFederated_p {
final boolean previous_core_fulltext = sb.index.fulltext().connectedLocalSolr() && env.getConfigBool(SwitchboardConstants.CORE_SERVICE_FULLTEXT, false);
env.setConfig(SwitchboardConstants.CORE_SERVICE_FULLTEXT, post_core_fulltext);
- final int commitWithinMs = post.getInt("solr.indexing.commitWithinMs", env.getConfigInt(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_COMMITWITHINMS, -1));
if (previous_core_fulltext && !post_core_fulltext) {
// switch off
sb.index.fulltext().disconnectLocalSolr();
@@ -88,7 +87,7 @@ public class IndexFederated_p {
if (!previous_core_fulltext && post_core_fulltext) {
// switch on
sb.index.connectUrlDb(sb.useTailCache, sb.exceed134217727);
- try { sb.index.fulltext().connectLocalSolr(commitWithinMs); } catch (IOException e) { Log.logException(e); }
+ try { sb.index.fulltext().connectLocalSolr(); } catch (IOException e) { Log.logException(e); }
}
// solr
@@ -114,7 +113,6 @@ public class IndexFederated_p {
}
solrurls = s.toString().trim();
env.setConfig(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_URL, solrurls);
- env.setConfig(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_COMMITWITHINMS, commitWithinMs);
env.setConfig(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_LAZY, lazy);
env.setConfig(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_SHARDING, post.get("solr.indexing.sharding", env.getConfig(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_SHARDING, "modulo-host-md5")));
final String schemename = post.get("solr.indexing.schemefile", env.getConfig(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_SCHEMEFILE, "solr.keys.default.list"));
@@ -136,7 +134,6 @@ public class IndexFederated_p {
try {
if (usesolr) {
SolrConnector solr = new ShardSolrConnector(solrurls, ShardSelection.Method.MODULO_HOST_MD5, 10000, true);
- if (commitWithinMs >= 0) solr.setCommitWithinMs(commitWithinMs);
sb.index.fulltext().connectRemoteSolr(solr);
} else {
sb.index.fulltext().disconnectRemoteSolr();
@@ -225,7 +222,6 @@ public class IndexFederated_p {
prop.put(SwitchboardConstants.CORE_SERVICE_CITATION + ".checked", env.getConfigBool(SwitchboardConstants.CORE_SERVICE_CITATION, false) ? 1 : 0);
prop.put("solr.indexing.solrremote.checked", env.getConfigBool(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_ENABLED, false) ? 1 : 0);
prop.put("solr.indexing.url", env.getConfig(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_URL, "http://127.0.0.1:8983/solr").replace(",", "\n"));
- prop.put("solr.indexing.commitWithinMs", env.getConfigInt(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_COMMITWITHINMS, 180000));
prop.put("solr.indexing.lazy.checked", env.getConfigBool(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_LAZY, true) ? 1 : 0);
prop.put("solr.indexing.sharding", env.getConfig(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_SHARDING, "modulo-host-md5"));
prop.put("solr.indexing.schemefile", schemename);
diff --git a/htroot/yacysearch.html b/htroot/yacysearch.html
index 80b206005..842b0e4ff 100644
--- a/htroot/yacysearch.html
+++ b/htroot/yacysearch.html
@@ -69,10 +69,9 @@
-
This search result can also be retrieved as RSS/opensearch output.
The query format is similar to SRU.
Click the API icon to see an example call to the search rss API.
diff --git a/source/net/yacy/cora/federate/solr/connector/MirrorSolrConnector.java b/source/net/yacy/cora/federate/solr/connector/MirrorSolrConnector.java
index 0769ff205..e4788f0ec 100644
--- a/source/net/yacy/cora/federate/solr/connector/MirrorSolrConnector.java
+++ b/source/net/yacy/cora/federate/solr/connector/MirrorSolrConnector.java
@@ -148,23 +148,6 @@ public class MirrorSolrConnector extends AbstractSolrConnector implements SolrCo
this.documentCache.clear();
}
- @Override
- public int getCommitWithinMs() {
- if (this.solr0 != null) this.solr0.getCommitWithinMs();
- if (this.solr1 != null) this.solr1.getCommitWithinMs();
- return -1;
- }
-
- /**
- * set the solr autocommit delay
- * @param c the maximum waiting time after a solr command until it is transported to the server
- */
- @Override
- public void setCommitWithinMs(int c) {
- if (this.solr0 != null) this.solr0.setCommitWithinMs(c);
- if (this.solr1 != null) this.solr1.setCommitWithinMs(c);
- }
-
@Override
public void commit(boolean softCommit) {
if (this.solr0 != null) this.solr0.commit(softCommit);
diff --git a/source/net/yacy/cora/federate/solr/connector/MultipleSolrConnector.java b/source/net/yacy/cora/federate/solr/connector/MultipleSolrConnector.java
index 8bad620e6..f36fa0927 100644
--- a/source/net/yacy/cora/federate/solr/connector/MultipleSolrConnector.java
+++ b/source/net/yacy/cora/federate/solr/connector/MultipleSolrConnector.java
@@ -36,13 +36,11 @@ public class MultipleSolrConnector extends AbstractSolrConnector implements Solr
private final ArrayBlockingQueue queue;
private final AddWorker[] worker;
private final SolrConnector solr;
- private int commitWithinMs;
public MultipleSolrConnector(final String url, final int connections) throws IOException {
this.solr = new RemoteSolrConnector(url);
this.queue = new ArrayBlockingQueue(1000);
this.worker = new AddWorker[connections];
- this.commitWithinMs = -1;
for (int i = 0; i < connections; i++) {
this.worker[i] = new AddWorker(url);
this.worker[i].start();
@@ -53,7 +51,6 @@ public class MultipleSolrConnector extends AbstractSolrConnector implements Solr
private final SolrConnector solr;
public AddWorker(final String url) throws IOException {
this.solr = new RemoteSolrConnector(url);
- if (MultipleSolrConnector.this.commitWithinMs >= 0 ) this.solr.setCommitWithinMs(MultipleSolrConnector.this.commitWithinMs);
}
@Override
public void run() {
@@ -75,22 +72,6 @@ public class MultipleSolrConnector extends AbstractSolrConnector implements Solr
}
}
- @Override
- public int getCommitWithinMs() {
- return this.commitWithinMs;
- }
-
- /**
- * set the solr autocommit delay
- * @param c the maximum waiting time after a solr command until it is transported to the server
- */
- @Override
- public void setCommitWithinMs(final int c) {
- this.commitWithinMs = c;
- this.solr.setCommitWithinMs(c);
- for (AddWorker w: this.worker) w.solr.setCommitWithinMs(c);
- }
-
@Override
public void commit(boolean softCommit) {
this.solr.commit(softCommit);
diff --git a/source/net/yacy/cora/federate/solr/connector/RetrySolrConnector.java b/source/net/yacy/cora/federate/solr/connector/RetrySolrConnector.java
index 2efa0680f..7f7c2a138 100644
--- a/source/net/yacy/cora/federate/solr/connector/RetrySolrConnector.java
+++ b/source/net/yacy/cora/federate/solr/connector/RetrySolrConnector.java
@@ -38,20 +38,6 @@ public class RetrySolrConnector extends AbstractSolrConnector implements SolrCon
this.retryMaxTime = retryMaxTime;
}
- @Override
- public int getCommitWithinMs() {
- return this.solrConnector.getCommitWithinMs();
- }
-
- /**
- * set the solr autocommit delay
- * @param c the maximum waiting time after a solr command until it is transported to the server
- */
- @Override
- public void setCommitWithinMs(int c) {
- this.solrConnector.setCommitWithinMs(c);
- }
-
@Override
public void commit(boolean softCommit) {
this.solrConnector.commit(softCommit);
diff --git a/source/net/yacy/cora/federate/solr/connector/ShardSolrConnector.java b/source/net/yacy/cora/federate/solr/connector/ShardSolrConnector.java
index a3dcbf9fb..6244e3497 100644
--- a/source/net/yacy/cora/federate/solr/connector/ShardSolrConnector.java
+++ b/source/net/yacy/cora/federate/solr/connector/ShardSolrConnector.java
@@ -58,20 +58,6 @@ public class ShardSolrConnector extends AbstractSolrConnector implements SolrCon
this.sharding = new ShardSelection(method, this.urls.length);
}
- @Override
- public int getCommitWithinMs() {
- return this.connectors.get(0).getCommitWithinMs();
- }
-
- /**
- * set the solr autocommit delay
- * @param c the maximum waiting time after a solr command until it is transported to the server
- */
- @Override
- public void setCommitWithinMs(int c) {
- for (final SolrConnector connector: this.connectors) connector.setCommitWithinMs(c);
- }
-
@Override
public void commit(boolean softCommit) {
for (final SolrConnector connector: this.connectors) connector.commit(softCommit);
diff --git a/source/net/yacy/cora/federate/solr/connector/SolrConnector.java b/source/net/yacy/cora/federate/solr/connector/SolrConnector.java
index 5329abb4f..2d4e0a9d2 100644
--- a/source/net/yacy/cora/federate/solr/connector/SolrConnector.java
+++ b/source/net/yacy/cora/federate/solr/connector/SolrConnector.java
@@ -36,18 +36,6 @@ import org.apache.solr.common.params.ModifiableSolrParams;
public interface SolrConnector extends Iterable /* Iterable of document IDs */ {
- /**
- * get the solr autocommit delay
- * @return the maximum waiting time after a solr command until it is transported to the server
- */
- public int getCommitWithinMs();
-
- /**
- * set the solr autocommit delay
- * @param c the maximum waiting time after a solr command until it is transported to the server
- */
- public void setCommitWithinMs(int c);
-
/**
* force a commit
*/
diff --git a/source/net/yacy/cora/federate/solr/connector/SolrServerConnector.java b/source/net/yacy/cora/federate/solr/connector/SolrServerConnector.java
index a6a212e1d..3b272908f 100644
--- a/source/net/yacy/cora/federate/solr/connector/SolrServerConnector.java
+++ b/source/net/yacy/cora/federate/solr/connector/SolrServerConnector.java
@@ -56,11 +56,9 @@ public abstract class SolrServerConnector extends AbstractSolrConnector implemen
protected final static Logger log = Logger.getLogger(SolrServerConnector.class);
protected SolrServer server;
- protected int commitWithinMs; // max time (in ms) before a commit will happen
protected SolrServerConnector() {
this.server = null;
- this.commitWithinMs = 180000;
}
protected void init(SolrServer server) {
@@ -71,25 +69,6 @@ public abstract class SolrServerConnector extends AbstractSolrConnector implemen
return this.server;
}
- /**
- * get the solr autocommit delay
- * @return the maximum waiting time after a solr command until it is transported to the server
- */
- @Override
- public int getCommitWithinMs() {
- return this.commitWithinMs;
- }
-
- /**
- * set the solr autocommit delay
- * when doing continuous inserts, don't set this value because it would cause continuous commits
- * @param c the maximum waiting time after a solr command until it is transported to the server
- */
- @Override
- public void setCommitWithinMs(int c) {
- this.commitWithinMs = c;
- }
-
@Override
public synchronized void commit(final boolean softCommit) {
//if (this.server instanceof HttpSolrServer) ((HttpSolrServer) this.server).getHttpClient().getConnectionManager().closeExpiredConnections();
@@ -209,7 +188,7 @@ public abstract class SolrServerConnector extends AbstractSolrConnector implemen
public void delete(final String id) throws IOException {
try {
synchronized (this.server) {
- this.server.deleteById(id, this.commitWithinMs);
+ this.server.deleteById(id, -1);
}
} catch (final Throwable e) {
throw new IOException(e);
@@ -220,7 +199,7 @@ public abstract class SolrServerConnector extends AbstractSolrConnector implemen
public void delete(final List ids) throws IOException {
try {
synchronized (this.server) {
- this.server.deleteById(ids, this.commitWithinMs);
+ this.server.deleteById(ids, -1);
}
} catch (final Throwable e) {
throw new IOException(e);
@@ -236,7 +215,7 @@ public abstract class SolrServerConnector extends AbstractSolrConnector implemen
public void deleteByQuery(final String querystring) throws IOException {
try {
synchronized (this.server) {
- this.server.deleteByQuery(querystring, this.commitWithinMs);
+ this.server.deleteByQuery(querystring, -1);
}
} catch (final Throwable e) {
throw new IOException(e);
@@ -249,7 +228,7 @@ public abstract class SolrServerConnector extends AbstractSolrConnector implemen
up.setParam("literal.id", solrId);
up.setParam("uprefix", "attr_");
up.setParam("fmap.content", "attr_content");
- up.setCommitWithin(this.commitWithinMs);
+ up.setCommitWithin(-1);
//up.setAction(AbstractUpdateRequest.ACTION.COMMIT, true, true);
try {
synchronized (this.server) {
@@ -266,7 +245,7 @@ public abstract class SolrServerConnector extends AbstractSolrConnector implemen
try {
if (solrdoc.containsKey("_version_")) solrdoc.setField("_version_",0L); // prevent Solr "version conflict"
synchronized (this.server) {
- this.server.add(solrdoc, this.commitWithinMs);
+ this.server.add(solrdoc, -1);
}
} catch (Throwable e) {
// catches "version conflict for": try this again and delete the document in advance
@@ -275,7 +254,7 @@ public abstract class SolrServerConnector extends AbstractSolrConnector implemen
} catch (SolrServerException e1) {}
try {
synchronized (this.server) {
- this.server.add(solrdoc, this.commitWithinMs);
+ this.server.add(solrdoc, -1);
}
} catch (Throwable ee) {
log.warn(e.getMessage() + " DOC=" + solrdoc.toString());
diff --git a/source/net/yacy/data/URLLicense.java b/source/net/yacy/data/URLLicense.java
index c63c2aeca..63153fabe 100644
--- a/source/net/yacy/data/URLLicense.java
+++ b/source/net/yacy/data/URLLicense.java
@@ -42,6 +42,7 @@ public class URLLicense {
private static final Map permissions = Collections.synchronizedMap(new SizeLimitedMap(maxQueue));
public static String aquireLicense(final DigestURI url) {
+ if (url == null) return "";
// generate license key
String license = ASCII.String(url.hash());
// store reference to url with license key
diff --git a/source/net/yacy/search/Switchboard.java b/source/net/yacy/search/Switchboard.java
index b3177d69a..72af0545f 100644
--- a/source/net/yacy/search/Switchboard.java
+++ b/source/net/yacy/search/Switchboard.java
@@ -428,12 +428,11 @@ public final class Switchboard extends serverSwitch {
ReferenceContainer.maxReferences = getConfigInt("index.maxReferences", 0);
final File segmentsPath = new File(new File(indexPath, networkName), "SEGMENTS");
this.index = new Segment(this.log, new File(segmentsPath, "default"), solrScheme);
- final int connectWithinMs = this.getConfigInt(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_COMMITWITHINMS, -1);
if (this.getConfigBool(SwitchboardConstants.CORE_SERVICE_RWI, true)) this.index.connectRWI(wordCacheMaxCount, fileSizeMax);
if (this.getConfigBool(SwitchboardConstants.CORE_SERVICE_CITATION, true)) this.index.connectCitation(wordCacheMaxCount, fileSizeMax);
if (this.getConfigBool(SwitchboardConstants.CORE_SERVICE_FULLTEXT, true)) {
this.index.connectUrlDb(this.useTailCache, this.exceed134217727);
- this.index.fulltext().connectLocalSolr(connectWithinMs);
+ this.index.fulltext().connectLocalSolr();
}
// set up the solr interface
@@ -446,7 +445,6 @@ public final class Switchboard extends serverSwitch {
solrurls,
ShardSelection.Method.MODULO_HOST_MD5,
10000, true);
- solr.setCommitWithinMs(connectWithinMs);
this.index.fulltext().connectRemoteSolr(solr);
} catch ( final IOException e ) {
Log.logException(e);
@@ -1282,11 +1280,10 @@ public final class Switchboard extends serverSwitch {
this.useTailCache,
this.exceed134217727);
this.index = new Segment(this.log, new File(new File(new File(indexPrimaryPath, networkName), "SEGMENTS"), "default"), solrScheme);
- final int connectWithinMs = this.getConfigInt(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_COMMITWITHINMS, -1);
if (this.getConfigBool(SwitchboardConstants.CORE_SERVICE_RWI, true)) this.index.connectRWI(wordCacheMaxCount, fileSizeMax);
if (this.getConfigBool(SwitchboardConstants.CORE_SERVICE_CITATION, true)) this.index.connectCitation(wordCacheMaxCount, fileSizeMax);
if (this.getConfigBool(SwitchboardConstants.CORE_SERVICE_FULLTEXT, true)) {
- this.index.fulltext().connectLocalSolr(connectWithinMs);
+ this.index.fulltext().connectLocalSolr();
this.index.connectUrlDb(this.useTailCache, this.exceed134217727);
}
@@ -1300,7 +1297,6 @@ public final class Switchboard extends serverSwitch {
solrurls,
ShardSelection.Method.MODULO_HOST_MD5,
10000, true);
- if (connectWithinMs >= 0) solr.setCommitWithinMs(connectWithinMs);
this.index.fulltext().connectRemoteSolr(solr);
} catch ( final IOException e ) {
Log.logException(e);
diff --git a/source/net/yacy/search/SwitchboardConstants.java b/source/net/yacy/search/SwitchboardConstants.java
index 655d579ff..fbba77be0 100644
--- a/source/net/yacy/search/SwitchboardConstants.java
+++ b/source/net/yacy/search/SwitchboardConstants.java
@@ -294,7 +294,6 @@ public final class SwitchboardConstants {
public static final String FEDERATED_SERVICE_SOLR_INDEXING_ENABLED = "federated.service.solr.indexing.enabled";
public static final String FEDERATED_SERVICE_SOLR_INDEXING_URL = "federated.service.solr.indexing.url";
- public static final String FEDERATED_SERVICE_SOLR_INDEXING_COMMITWITHINMS = "federated.service.solr.indexing.commitWithinMs";
public static final String FEDERATED_SERVICE_SOLR_INDEXING_SHARDING = "federated.service.solr.indexing.sharding";
public static final String FEDERATED_SERVICE_SOLR_INDEXING_SCHEMEFILE = "federated.service.solr.indexing.schemefile";
public static final String FEDERATED_SERVICE_SOLR_INDEXING_LAZY = "federated.service.solr.indexing.lazy";
diff --git a/source/net/yacy/search/index/DocumentIndex.java b/source/net/yacy/search/index/DocumentIndex.java
index bdebf8c30..f0d79e641 100644
--- a/source/net/yacy/search/index/DocumentIndex.java
+++ b/source/net/yacy/search/index/DocumentIndex.java
@@ -70,7 +70,7 @@ public class DocumentIndex extends Segment {
false, // useTailCache
false // exceed134217727
);
- super.fulltext().connectLocalSolr(-1);
+ super.fulltext().connectLocalSolr();
final int cores = Runtime.getRuntime().availableProcessors() + 1;
this.callback = callback;
this.queue = new LinkedBlockingQueue(cores * 300);
diff --git a/source/net/yacy/search/index/Fulltext.java b/source/net/yacy/search/index/Fulltext.java
index 24fe458de..77f88bbe5 100644
--- a/source/net/yacy/search/index/Fulltext.java
+++ b/source/net/yacy/search/index/Fulltext.java
@@ -142,7 +142,7 @@ public final class Fulltext {
return this.solr.isConnected0();
}
- public void connectLocalSolr(final int commitWithin) throws IOException {
+ public void connectLocalSolr() throws IOException {
File baseLocation = this.location;
if (baseLocation.getName().equals("default")) baseLocation = baseLocation.getParentFile();
File solrLocation = new File(baseLocation, SOLR_PATH);
@@ -152,7 +152,6 @@ public final class Fulltext {
if (oldLocation.exists()) oldLocation.renameTo(solrLocation);
}
EmbeddedSolrConnector esc = new EmbeddedSolrConnector(solrLocation, new File(new File(Switchboard.getSwitchboard().appPath, "defaults"), "solr"));
- if (commitWithin >= 0) esc.setCommitWithinMs(commitWithin);
Version luceneVersion = esc.getConfig().getLuceneVersion("luceneMatchVersion");
String lvn = luceneVersion.name();
Log.logInfo("Fulltext", "using lucene version " + lvn);
@@ -230,10 +229,6 @@ public final class Fulltext {
}
this.solr.close();
}
-
- public int getCommitWithinMs() {
- return this.solr.getCommitWithinMs();
- }
public void commit(boolean softCommit) {
this.solr.commit(softCommit);
@@ -571,7 +566,6 @@ public final class Fulltext {
*/
public File dumpSolr() {
EmbeddedSolrConnector esc = (EmbeddedSolrConnector) this.solr.getSolr0();
- int commitWithin = esc.getCommitWithinMs();
File storagePath = esc.getStoragePath();
File zipOut = new File(storagePath.toString() + "_" + GenericFormatter.SHORT_DAY_FORMATTER.format() + ".zip");
synchronized (this.solr) {
@@ -582,7 +576,7 @@ public final class Fulltext {
Log.logException(e);
} finally {
try {
- this.connectLocalSolr(commitWithin);
+ this.connectLocalSolr();
} catch (IOException e) {
Log.logException(e);
}
@@ -597,7 +591,6 @@ public final class Fulltext {
*/
public void restoreSolr(File solrDumpZipFile) {
EmbeddedSolrConnector esc = (EmbeddedSolrConnector) this.solr.getSolr0();
- int commitWithin = esc.getCommitWithinMs();
File storagePath = esc.getStoragePath();
synchronized (this.solr) {
this.disconnectLocalSolr();
@@ -607,7 +600,7 @@ public final class Fulltext {
Log.logException(e);
} finally {
try {
- this.connectLocalSolr(commitWithin);
+ this.connectLocalSolr();
} catch (IOException e) {
Log.logException(e);
}