diff --git a/defaults/yacy.init b/defaults/yacy.init index 980f8147e..751aa379e 100644 --- a/defaults/yacy.init +++ b/defaults/yacy.init @@ -1056,7 +1056,6 @@ color_searchurlhover = #008000 # - to check whats in solr after indexing, open http://localhost:8983/solr/admin/ federated.service.solr.indexing.enabled = false federated.service.solr.indexing.url = http://127.0.0.1:8983/solr -federated.service.solr.indexing.commitWithinMs = -1 federated.service.solr.indexing.sharding = MODULO_HOST_MD5 federated.service.solr.indexing.schemefile = solr.keys.default.list # the lazy attribute causes that fields containing "" or 0 are not added and not written diff --git a/htroot/IndexFederated_p.java b/htroot/IndexFederated_p.java index 2a5c050a2..6af0118b6 100644 --- a/htroot/IndexFederated_p.java +++ b/htroot/IndexFederated_p.java @@ -79,7 +79,6 @@ public class IndexFederated_p { final boolean previous_core_fulltext = sb.index.fulltext().connectedLocalSolr() && env.getConfigBool(SwitchboardConstants.CORE_SERVICE_FULLTEXT, false); env.setConfig(SwitchboardConstants.CORE_SERVICE_FULLTEXT, post_core_fulltext); - final int commitWithinMs = post.getInt("solr.indexing.commitWithinMs", env.getConfigInt(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_COMMITWITHINMS, -1)); if (previous_core_fulltext && !post_core_fulltext) { // switch off sb.index.fulltext().disconnectLocalSolr(); @@ -88,7 +87,7 @@ public class IndexFederated_p { if (!previous_core_fulltext && post_core_fulltext) { // switch on sb.index.connectUrlDb(sb.useTailCache, sb.exceed134217727); - try { sb.index.fulltext().connectLocalSolr(commitWithinMs); } catch (IOException e) { Log.logException(e); } + try { sb.index.fulltext().connectLocalSolr(); } catch (IOException e) { Log.logException(e); } } // solr @@ -114,7 +113,6 @@ public class IndexFederated_p { } solrurls = s.toString().trim(); env.setConfig(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_URL, solrurls); - env.setConfig(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_COMMITWITHINMS, commitWithinMs); env.setConfig(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_LAZY, lazy); env.setConfig(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_SHARDING, post.get("solr.indexing.sharding", env.getConfig(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_SHARDING, "modulo-host-md5"))); final String schemename = post.get("solr.indexing.schemefile", env.getConfig(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_SCHEMEFILE, "solr.keys.default.list")); @@ -136,7 +134,6 @@ public class IndexFederated_p { try { if (usesolr) { SolrConnector solr = new ShardSolrConnector(solrurls, ShardSelection.Method.MODULO_HOST_MD5, 10000, true); - if (commitWithinMs >= 0) solr.setCommitWithinMs(commitWithinMs); sb.index.fulltext().connectRemoteSolr(solr); } else { sb.index.fulltext().disconnectRemoteSolr(); @@ -225,7 +222,6 @@ public class IndexFederated_p { prop.put(SwitchboardConstants.CORE_SERVICE_CITATION + ".checked", env.getConfigBool(SwitchboardConstants.CORE_SERVICE_CITATION, false) ? 1 : 0); prop.put("solr.indexing.solrremote.checked", env.getConfigBool(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_ENABLED, false) ? 1 : 0); prop.put("solr.indexing.url", env.getConfig(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_URL, "http://127.0.0.1:8983/solr").replace(",", "\n")); - prop.put("solr.indexing.commitWithinMs", env.getConfigInt(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_COMMITWITHINMS, 180000)); prop.put("solr.indexing.lazy.checked", env.getConfigBool(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_LAZY, true) ? 1 : 0); prop.put("solr.indexing.sharding", env.getConfig(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_SHARDING, "modulo-host-md5")); prop.put("solr.indexing.schemefile", schemename); diff --git a/htroot/yacysearch.html b/htroot/yacysearch.html index 80b206005..842b0e4ff 100644 --- a/htroot/yacysearch.html +++ b/htroot/yacysearch.html @@ -69,10 +69,9 @@
-API This search result can also be retrieved as RSS/opensearch output. The query format is similar to SRU. Click the API icon to see an example call to the search rss API. diff --git a/source/net/yacy/cora/federate/solr/connector/MirrorSolrConnector.java b/source/net/yacy/cora/federate/solr/connector/MirrorSolrConnector.java index 0769ff205..e4788f0ec 100644 --- a/source/net/yacy/cora/federate/solr/connector/MirrorSolrConnector.java +++ b/source/net/yacy/cora/federate/solr/connector/MirrorSolrConnector.java @@ -148,23 +148,6 @@ public class MirrorSolrConnector extends AbstractSolrConnector implements SolrCo this.documentCache.clear(); } - @Override - public int getCommitWithinMs() { - if (this.solr0 != null) this.solr0.getCommitWithinMs(); - if (this.solr1 != null) this.solr1.getCommitWithinMs(); - return -1; - } - - /** - * set the solr autocommit delay - * @param c the maximum waiting time after a solr command until it is transported to the server - */ - @Override - public void setCommitWithinMs(int c) { - if (this.solr0 != null) this.solr0.setCommitWithinMs(c); - if (this.solr1 != null) this.solr1.setCommitWithinMs(c); - } - @Override public void commit(boolean softCommit) { if (this.solr0 != null) this.solr0.commit(softCommit); diff --git a/source/net/yacy/cora/federate/solr/connector/MultipleSolrConnector.java b/source/net/yacy/cora/federate/solr/connector/MultipleSolrConnector.java index 8bad620e6..f36fa0927 100644 --- a/source/net/yacy/cora/federate/solr/connector/MultipleSolrConnector.java +++ b/source/net/yacy/cora/federate/solr/connector/MultipleSolrConnector.java @@ -36,13 +36,11 @@ public class MultipleSolrConnector extends AbstractSolrConnector implements Solr private final ArrayBlockingQueue queue; private final AddWorker[] worker; private final SolrConnector solr; - private int commitWithinMs; public MultipleSolrConnector(final String url, final int connections) throws IOException { this.solr = new RemoteSolrConnector(url); this.queue = new ArrayBlockingQueue(1000); this.worker = new AddWorker[connections]; - this.commitWithinMs = -1; for (int i = 0; i < connections; i++) { this.worker[i] = new AddWorker(url); this.worker[i].start(); @@ -53,7 +51,6 @@ public class MultipleSolrConnector extends AbstractSolrConnector implements Solr private final SolrConnector solr; public AddWorker(final String url) throws IOException { this.solr = new RemoteSolrConnector(url); - if (MultipleSolrConnector.this.commitWithinMs >= 0 ) this.solr.setCommitWithinMs(MultipleSolrConnector.this.commitWithinMs); } @Override public void run() { @@ -75,22 +72,6 @@ public class MultipleSolrConnector extends AbstractSolrConnector implements Solr } } - @Override - public int getCommitWithinMs() { - return this.commitWithinMs; - } - - /** - * set the solr autocommit delay - * @param c the maximum waiting time after a solr command until it is transported to the server - */ - @Override - public void setCommitWithinMs(final int c) { - this.commitWithinMs = c; - this.solr.setCommitWithinMs(c); - for (AddWorker w: this.worker) w.solr.setCommitWithinMs(c); - } - @Override public void commit(boolean softCommit) { this.solr.commit(softCommit); diff --git a/source/net/yacy/cora/federate/solr/connector/RetrySolrConnector.java b/source/net/yacy/cora/federate/solr/connector/RetrySolrConnector.java index 2efa0680f..7f7c2a138 100644 --- a/source/net/yacy/cora/federate/solr/connector/RetrySolrConnector.java +++ b/source/net/yacy/cora/federate/solr/connector/RetrySolrConnector.java @@ -38,20 +38,6 @@ public class RetrySolrConnector extends AbstractSolrConnector implements SolrCon this.retryMaxTime = retryMaxTime; } - @Override - public int getCommitWithinMs() { - return this.solrConnector.getCommitWithinMs(); - } - - /** - * set the solr autocommit delay - * @param c the maximum waiting time after a solr command until it is transported to the server - */ - @Override - public void setCommitWithinMs(int c) { - this.solrConnector.setCommitWithinMs(c); - } - @Override public void commit(boolean softCommit) { this.solrConnector.commit(softCommit); diff --git a/source/net/yacy/cora/federate/solr/connector/ShardSolrConnector.java b/source/net/yacy/cora/federate/solr/connector/ShardSolrConnector.java index a3dcbf9fb..6244e3497 100644 --- a/source/net/yacy/cora/federate/solr/connector/ShardSolrConnector.java +++ b/source/net/yacy/cora/federate/solr/connector/ShardSolrConnector.java @@ -58,20 +58,6 @@ public class ShardSolrConnector extends AbstractSolrConnector implements SolrCon this.sharding = new ShardSelection(method, this.urls.length); } - @Override - public int getCommitWithinMs() { - return this.connectors.get(0).getCommitWithinMs(); - } - - /** - * set the solr autocommit delay - * @param c the maximum waiting time after a solr command until it is transported to the server - */ - @Override - public void setCommitWithinMs(int c) { - for (final SolrConnector connector: this.connectors) connector.setCommitWithinMs(c); - } - @Override public void commit(boolean softCommit) { for (final SolrConnector connector: this.connectors) connector.commit(softCommit); diff --git a/source/net/yacy/cora/federate/solr/connector/SolrConnector.java b/source/net/yacy/cora/federate/solr/connector/SolrConnector.java index 5329abb4f..2d4e0a9d2 100644 --- a/source/net/yacy/cora/federate/solr/connector/SolrConnector.java +++ b/source/net/yacy/cora/federate/solr/connector/SolrConnector.java @@ -36,18 +36,6 @@ import org.apache.solr.common.params.ModifiableSolrParams; public interface SolrConnector extends Iterable /* Iterable of document IDs */ { - /** - * get the solr autocommit delay - * @return the maximum waiting time after a solr command until it is transported to the server - */ - public int getCommitWithinMs(); - - /** - * set the solr autocommit delay - * @param c the maximum waiting time after a solr command until it is transported to the server - */ - public void setCommitWithinMs(int c); - /** * force a commit */ diff --git a/source/net/yacy/cora/federate/solr/connector/SolrServerConnector.java b/source/net/yacy/cora/federate/solr/connector/SolrServerConnector.java index a6a212e1d..3b272908f 100644 --- a/source/net/yacy/cora/federate/solr/connector/SolrServerConnector.java +++ b/source/net/yacy/cora/federate/solr/connector/SolrServerConnector.java @@ -56,11 +56,9 @@ public abstract class SolrServerConnector extends AbstractSolrConnector implemen protected final static Logger log = Logger.getLogger(SolrServerConnector.class); protected SolrServer server; - protected int commitWithinMs; // max time (in ms) before a commit will happen protected SolrServerConnector() { this.server = null; - this.commitWithinMs = 180000; } protected void init(SolrServer server) { @@ -71,25 +69,6 @@ public abstract class SolrServerConnector extends AbstractSolrConnector implemen return this.server; } - /** - * get the solr autocommit delay - * @return the maximum waiting time after a solr command until it is transported to the server - */ - @Override - public int getCommitWithinMs() { - return this.commitWithinMs; - } - - /** - * set the solr autocommit delay - * when doing continuous inserts, don't set this value because it would cause continuous commits - * @param c the maximum waiting time after a solr command until it is transported to the server - */ - @Override - public void setCommitWithinMs(int c) { - this.commitWithinMs = c; - } - @Override public synchronized void commit(final boolean softCommit) { //if (this.server instanceof HttpSolrServer) ((HttpSolrServer) this.server).getHttpClient().getConnectionManager().closeExpiredConnections(); @@ -209,7 +188,7 @@ public abstract class SolrServerConnector extends AbstractSolrConnector implemen public void delete(final String id) throws IOException { try { synchronized (this.server) { - this.server.deleteById(id, this.commitWithinMs); + this.server.deleteById(id, -1); } } catch (final Throwable e) { throw new IOException(e); @@ -220,7 +199,7 @@ public abstract class SolrServerConnector extends AbstractSolrConnector implemen public void delete(final List ids) throws IOException { try { synchronized (this.server) { - this.server.deleteById(ids, this.commitWithinMs); + this.server.deleteById(ids, -1); } } catch (final Throwable e) { throw new IOException(e); @@ -236,7 +215,7 @@ public abstract class SolrServerConnector extends AbstractSolrConnector implemen public void deleteByQuery(final String querystring) throws IOException { try { synchronized (this.server) { - this.server.deleteByQuery(querystring, this.commitWithinMs); + this.server.deleteByQuery(querystring, -1); } } catch (final Throwable e) { throw new IOException(e); @@ -249,7 +228,7 @@ public abstract class SolrServerConnector extends AbstractSolrConnector implemen up.setParam("literal.id", solrId); up.setParam("uprefix", "attr_"); up.setParam("fmap.content", "attr_content"); - up.setCommitWithin(this.commitWithinMs); + up.setCommitWithin(-1); //up.setAction(AbstractUpdateRequest.ACTION.COMMIT, true, true); try { synchronized (this.server) { @@ -266,7 +245,7 @@ public abstract class SolrServerConnector extends AbstractSolrConnector implemen try { if (solrdoc.containsKey("_version_")) solrdoc.setField("_version_",0L); // prevent Solr "version conflict" synchronized (this.server) { - this.server.add(solrdoc, this.commitWithinMs); + this.server.add(solrdoc, -1); } } catch (Throwable e) { // catches "version conflict for": try this again and delete the document in advance @@ -275,7 +254,7 @@ public abstract class SolrServerConnector extends AbstractSolrConnector implemen } catch (SolrServerException e1) {} try { synchronized (this.server) { - this.server.add(solrdoc, this.commitWithinMs); + this.server.add(solrdoc, -1); } } catch (Throwable ee) { log.warn(e.getMessage() + " DOC=" + solrdoc.toString()); diff --git a/source/net/yacy/data/URLLicense.java b/source/net/yacy/data/URLLicense.java index c63c2aeca..63153fabe 100644 --- a/source/net/yacy/data/URLLicense.java +++ b/source/net/yacy/data/URLLicense.java @@ -42,6 +42,7 @@ public class URLLicense { private static final Map permissions = Collections.synchronizedMap(new SizeLimitedMap(maxQueue)); public static String aquireLicense(final DigestURI url) { + if (url == null) return ""; // generate license key String license = ASCII.String(url.hash()); // store reference to url with license key diff --git a/source/net/yacy/search/Switchboard.java b/source/net/yacy/search/Switchboard.java index b3177d69a..72af0545f 100644 --- a/source/net/yacy/search/Switchboard.java +++ b/source/net/yacy/search/Switchboard.java @@ -428,12 +428,11 @@ public final class Switchboard extends serverSwitch { ReferenceContainer.maxReferences = getConfigInt("index.maxReferences", 0); final File segmentsPath = new File(new File(indexPath, networkName), "SEGMENTS"); this.index = new Segment(this.log, new File(segmentsPath, "default"), solrScheme); - final int connectWithinMs = this.getConfigInt(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_COMMITWITHINMS, -1); if (this.getConfigBool(SwitchboardConstants.CORE_SERVICE_RWI, true)) this.index.connectRWI(wordCacheMaxCount, fileSizeMax); if (this.getConfigBool(SwitchboardConstants.CORE_SERVICE_CITATION, true)) this.index.connectCitation(wordCacheMaxCount, fileSizeMax); if (this.getConfigBool(SwitchboardConstants.CORE_SERVICE_FULLTEXT, true)) { this.index.connectUrlDb(this.useTailCache, this.exceed134217727); - this.index.fulltext().connectLocalSolr(connectWithinMs); + this.index.fulltext().connectLocalSolr(); } // set up the solr interface @@ -446,7 +445,6 @@ public final class Switchboard extends serverSwitch { solrurls, ShardSelection.Method.MODULO_HOST_MD5, 10000, true); - solr.setCommitWithinMs(connectWithinMs); this.index.fulltext().connectRemoteSolr(solr); } catch ( final IOException e ) { Log.logException(e); @@ -1282,11 +1280,10 @@ public final class Switchboard extends serverSwitch { this.useTailCache, this.exceed134217727); this.index = new Segment(this.log, new File(new File(new File(indexPrimaryPath, networkName), "SEGMENTS"), "default"), solrScheme); - final int connectWithinMs = this.getConfigInt(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_COMMITWITHINMS, -1); if (this.getConfigBool(SwitchboardConstants.CORE_SERVICE_RWI, true)) this.index.connectRWI(wordCacheMaxCount, fileSizeMax); if (this.getConfigBool(SwitchboardConstants.CORE_SERVICE_CITATION, true)) this.index.connectCitation(wordCacheMaxCount, fileSizeMax); if (this.getConfigBool(SwitchboardConstants.CORE_SERVICE_FULLTEXT, true)) { - this.index.fulltext().connectLocalSolr(connectWithinMs); + this.index.fulltext().connectLocalSolr(); this.index.connectUrlDb(this.useTailCache, this.exceed134217727); } @@ -1300,7 +1297,6 @@ public final class Switchboard extends serverSwitch { solrurls, ShardSelection.Method.MODULO_HOST_MD5, 10000, true); - if (connectWithinMs >= 0) solr.setCommitWithinMs(connectWithinMs); this.index.fulltext().connectRemoteSolr(solr); } catch ( final IOException e ) { Log.logException(e); diff --git a/source/net/yacy/search/SwitchboardConstants.java b/source/net/yacy/search/SwitchboardConstants.java index 655d579ff..fbba77be0 100644 --- a/source/net/yacy/search/SwitchboardConstants.java +++ b/source/net/yacy/search/SwitchboardConstants.java @@ -294,7 +294,6 @@ public final class SwitchboardConstants { public static final String FEDERATED_SERVICE_SOLR_INDEXING_ENABLED = "federated.service.solr.indexing.enabled"; public static final String FEDERATED_SERVICE_SOLR_INDEXING_URL = "federated.service.solr.indexing.url"; - public static final String FEDERATED_SERVICE_SOLR_INDEXING_COMMITWITHINMS = "federated.service.solr.indexing.commitWithinMs"; public static final String FEDERATED_SERVICE_SOLR_INDEXING_SHARDING = "federated.service.solr.indexing.sharding"; public static final String FEDERATED_SERVICE_SOLR_INDEXING_SCHEMEFILE = "federated.service.solr.indexing.schemefile"; public static final String FEDERATED_SERVICE_SOLR_INDEXING_LAZY = "federated.service.solr.indexing.lazy"; diff --git a/source/net/yacy/search/index/DocumentIndex.java b/source/net/yacy/search/index/DocumentIndex.java index bdebf8c30..f0d79e641 100644 --- a/source/net/yacy/search/index/DocumentIndex.java +++ b/source/net/yacy/search/index/DocumentIndex.java @@ -70,7 +70,7 @@ public class DocumentIndex extends Segment { false, // useTailCache false // exceed134217727 ); - super.fulltext().connectLocalSolr(-1); + super.fulltext().connectLocalSolr(); final int cores = Runtime.getRuntime().availableProcessors() + 1; this.callback = callback; this.queue = new LinkedBlockingQueue(cores * 300); diff --git a/source/net/yacy/search/index/Fulltext.java b/source/net/yacy/search/index/Fulltext.java index 24fe458de..77f88bbe5 100644 --- a/source/net/yacy/search/index/Fulltext.java +++ b/source/net/yacy/search/index/Fulltext.java @@ -142,7 +142,7 @@ public final class Fulltext { return this.solr.isConnected0(); } - public void connectLocalSolr(final int commitWithin) throws IOException { + public void connectLocalSolr() throws IOException { File baseLocation = this.location; if (baseLocation.getName().equals("default")) baseLocation = baseLocation.getParentFile(); File solrLocation = new File(baseLocation, SOLR_PATH); @@ -152,7 +152,6 @@ public final class Fulltext { if (oldLocation.exists()) oldLocation.renameTo(solrLocation); } EmbeddedSolrConnector esc = new EmbeddedSolrConnector(solrLocation, new File(new File(Switchboard.getSwitchboard().appPath, "defaults"), "solr")); - if (commitWithin >= 0) esc.setCommitWithinMs(commitWithin); Version luceneVersion = esc.getConfig().getLuceneVersion("luceneMatchVersion"); String lvn = luceneVersion.name(); Log.logInfo("Fulltext", "using lucene version " + lvn); @@ -230,10 +229,6 @@ public final class Fulltext { } this.solr.close(); } - - public int getCommitWithinMs() { - return this.solr.getCommitWithinMs(); - } public void commit(boolean softCommit) { this.solr.commit(softCommit); @@ -571,7 +566,6 @@ public final class Fulltext { */ public File dumpSolr() { EmbeddedSolrConnector esc = (EmbeddedSolrConnector) this.solr.getSolr0(); - int commitWithin = esc.getCommitWithinMs(); File storagePath = esc.getStoragePath(); File zipOut = new File(storagePath.toString() + "_" + GenericFormatter.SHORT_DAY_FORMATTER.format() + ".zip"); synchronized (this.solr) { @@ -582,7 +576,7 @@ public final class Fulltext { Log.logException(e); } finally { try { - this.connectLocalSolr(commitWithin); + this.connectLocalSolr(); } catch (IOException e) { Log.logException(e); } @@ -597,7 +591,6 @@ public final class Fulltext { */ public void restoreSolr(File solrDumpZipFile) { EmbeddedSolrConnector esc = (EmbeddedSolrConnector) this.solr.getSolr0(); - int commitWithin = esc.getCommitWithinMs(); File storagePath = esc.getStoragePath(); synchronized (this.solr) { this.disconnectLocalSolr(); @@ -607,7 +600,7 @@ public final class Fulltext { Log.logException(e); } finally { try { - this.connectLocalSolr(commitWithin); + this.connectLocalSolr(); } catch (IOException e) { Log.logException(e); }