2011-05-26 12:57:02 +02:00
|
|
|
/**
|
|
|
|
* IndexFederated_p
|
|
|
|
* Copyright 2011 by Michael Peter Christen, mc@yacy.net, Frankfurt am Main, Germany
|
|
|
|
* First released 25.05.2011 at http://yacy.net
|
|
|
|
*
|
|
|
|
* This library is free software; you can redistribute it and/or
|
|
|
|
* modify it under the terms of the GNU Lesser General Public
|
|
|
|
* License as published by the Free Software Foundation; either
|
|
|
|
* version 2.1 of the License, or (at your option) any later version.
|
2011-06-30 17:49:21 +02:00
|
|
|
*
|
2011-05-26 12:57:02 +02:00
|
|
|
* This library is distributed in the hope that it will be useful,
|
|
|
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
|
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
|
|
|
* Lesser General Public License for more details.
|
2011-06-30 17:49:21 +02:00
|
|
|
*
|
2011-05-26 12:57:02 +02:00
|
|
|
* You should have received a copy of the GNU Lesser General Public License
|
|
|
|
* along with this program in the file lgpl21.txt
|
|
|
|
* If not, see <http://www.gnu.org/licenses/>.
|
|
|
|
*/
|
|
|
|
|
2011-07-04 18:35:30 +02:00
|
|
|
import java.io.BufferedReader;
|
|
|
|
import java.io.ByteArrayInputStream;
|
2011-05-26 12:57:02 +02:00
|
|
|
import java.io.IOException;
|
2011-07-04 18:35:30 +02:00
|
|
|
import java.io.InputStreamReader;
|
2013-02-15 01:38:10 +01:00
|
|
|
import java.util.ArrayList;
|
2011-05-26 12:57:02 +02:00
|
|
|
|
2013-01-25 16:06:58 +01:00
|
|
|
import org.apache.solr.common.SolrException;
|
|
|
|
|
2011-07-04 18:35:30 +02:00
|
|
|
import net.yacy.cora.document.UTF8;
|
2012-09-25 21:20:03 +02:00
|
|
|
import net.yacy.cora.federate.solr.connector.RemoteSolrConnector;
|
|
|
|
import net.yacy.cora.federate.solr.connector.SolrConnector;
|
2013-02-21 13:23:55 +01:00
|
|
|
import net.yacy.cora.federate.solr.instance.RemoteInstance;
|
|
|
|
import net.yacy.cora.federate.solr.instance.ShardInstance;
|
2011-05-26 12:57:02 +02:00
|
|
|
import net.yacy.cora.protocol.RequestHeader;
|
|
|
|
import net.yacy.kelondro.logging.Log;
|
2012-07-23 16:28:39 +02:00
|
|
|
import net.yacy.kelondro.util.OS;
|
2011-09-25 18:59:06 +02:00
|
|
|
import net.yacy.search.Switchboard;
|
2012-07-10 12:01:20 +02:00
|
|
|
import net.yacy.search.SwitchboardConstants;
|
2012-09-21 15:48:16 +02:00
|
|
|
import net.yacy.server.serverObjects;
|
|
|
|
import net.yacy.server.serverSwitch;
|
2011-05-26 12:57:02 +02:00
|
|
|
|
|
|
|
public class IndexFederated_p {
|
|
|
|
|
2012-07-05 09:14:04 +02:00
|
|
|
public static serverObjects respond(@SuppressWarnings("unused") final RequestHeader header, final serverObjects post, final serverSwitch env) {
|
2011-05-26 12:57:02 +02:00
|
|
|
// return variable that accumulates replacements
|
|
|
|
final serverObjects prop = new serverObjects();
|
2011-06-30 17:49:21 +02:00
|
|
|
final Switchboard sb = (Switchboard) env;
|
2011-05-26 12:57:02 +02:00
|
|
|
|
|
|
|
if (post != null && post.containsKey("set")) {
|
2013-02-24 18:09:34 +01:00
|
|
|
//yacy
|
2012-07-23 16:28:39 +02:00
|
|
|
boolean post_core_rwi = post.getBoolean(SwitchboardConstants.CORE_SERVICE_RWI);
|
|
|
|
final boolean previous_core_rwi = sb.index.connectedRWI() && env.getConfigBool(SwitchboardConstants.CORE_SERVICE_RWI, false);
|
|
|
|
env.setConfig(SwitchboardConstants.CORE_SERVICE_RWI, post_core_rwi);
|
|
|
|
if (previous_core_rwi && !post_core_rwi) sb.index.disconnectRWI(); // switch off
|
|
|
|
if (!previous_core_rwi && post_core_rwi) try {
|
|
|
|
final int wordCacheMaxCount = (int) sb.getConfigLong(SwitchboardConstants.WORDCACHE_MAX_COUNT, 20000);
|
|
|
|
final long fileSizeMax = (OS.isWindows) ? sb.getConfigLong("filesize.max.win", Integer.MAX_VALUE) : sb.getConfigLong( "filesize.max.other", Integer.MAX_VALUE);
|
|
|
|
sb.index.connectRWI(wordCacheMaxCount, fileSizeMax);
|
|
|
|
} catch (IOException e) { Log.logException(e); } // switch on
|
|
|
|
|
|
|
|
boolean post_core_citation = post.getBoolean(SwitchboardConstants.CORE_SERVICE_CITATION);
|
|
|
|
final boolean previous_core_citation = sb.index.connectedCitation() && env.getConfigBool(SwitchboardConstants.CORE_SERVICE_CITATION, false);
|
|
|
|
env.setConfig(SwitchboardConstants.CORE_SERVICE_CITATION, post_core_citation);
|
|
|
|
if (previous_core_citation && !post_core_citation) sb.index.disconnectCitation(); // switch off
|
|
|
|
if (!previous_core_citation && post_core_citation) try {
|
|
|
|
final int wordCacheMaxCount = (int) sb.getConfigLong(SwitchboardConstants.WORDCACHE_MAX_COUNT, 20000);
|
|
|
|
final long fileSizeMax = (OS.isWindows) ? sb.getConfigLong("filesize.max.win", Integer.MAX_VALUE) : sb.getConfigLong( "filesize.max.other", Integer.MAX_VALUE);
|
|
|
|
sb.index.connectCitation(wordCacheMaxCount, fileSizeMax);
|
|
|
|
} catch (IOException e) { Log.logException(e); } // switch on
|
|
|
|
|
2012-08-16 18:17:47 +02:00
|
|
|
boolean post_core_fulltext = post.getBoolean(SwitchboardConstants.CORE_SERVICE_FULLTEXT);
|
2012-08-17 15:52:33 +02:00
|
|
|
final boolean previous_core_fulltext = sb.index.fulltext().connectedLocalSolr() && env.getConfigBool(SwitchboardConstants.CORE_SERVICE_FULLTEXT, false);
|
2012-08-16 18:17:47 +02:00
|
|
|
env.setConfig(SwitchboardConstants.CORE_SERVICE_FULLTEXT, post_core_fulltext);
|
|
|
|
|
|
|
|
if (previous_core_fulltext && !post_core_fulltext) {
|
|
|
|
// switch off
|
2012-08-17 15:52:33 +02:00
|
|
|
sb.index.fulltext().disconnectLocalSolr();
|
|
|
|
sb.index.fulltext().disconnectUrlDb();
|
2012-08-16 18:17:47 +02:00
|
|
|
}
|
|
|
|
if (!previous_core_fulltext && post_core_fulltext) {
|
|
|
|
// switch on
|
|
|
|
sb.index.connectUrlDb(sb.useTailCache, sb.exceed134217727);
|
2013-02-13 02:29:47 +01:00
|
|
|
try { sb.index.fulltext().connectLocalSolr(); } catch (IOException e) { Log.logException(e); }
|
2012-08-16 18:17:47 +02:00
|
|
|
}
|
2011-06-30 17:49:21 +02:00
|
|
|
|
2013-02-24 18:09:34 +01:00
|
|
|
boolean webgraph = post.getBoolean(SwitchboardConstants.CORE_SERVICE_WEBGRAPH);
|
|
|
|
sb.index.writeWebgraph(webgraph);
|
|
|
|
env.setConfig(SwitchboardConstants.CORE_SERVICE_WEBGRAPH, webgraph);
|
|
|
|
|
|
|
|
boolean jena = post.getBoolean(SwitchboardConstants.CORE_SERVICE_JENA);
|
|
|
|
env.setConfig(SwitchboardConstants.CORE_SERVICE_JENA, jena);
|
|
|
|
|
2011-05-26 12:57:02 +02:00
|
|
|
// solr
|
2012-08-17 15:52:33 +02:00
|
|
|
final boolean solrRemoteWasOn = sb.index.fulltext().connectedRemoteSolr() && env.getConfigBool(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_ENABLED, true);
|
2012-07-10 12:01:20 +02:00
|
|
|
String solrurls = post.get("solr.indexing.url", env.getConfig(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_URL, "http://127.0.0.1:8983/solr"));
|
2013-01-25 16:06:58 +01:00
|
|
|
final boolean solrRemoteIsOnAfterwards = post.getBoolean("solr.indexing.solrremote") & solrurls.length() > 0;
|
|
|
|
env.setConfig(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_ENABLED, solrRemoteIsOnAfterwards);
|
2011-07-04 18:35:30 +02:00
|
|
|
final BufferedReader r = new BufferedReader(new InputStreamReader(new ByteArrayInputStream(UTF8.getBytes(solrurls))));
|
|
|
|
final StringBuilder s = new StringBuilder();
|
|
|
|
String s0;
|
|
|
|
try {
|
|
|
|
while ((s0 = r.readLine()) != null) {
|
|
|
|
s0 = s0.trim();
|
2012-01-10 03:00:55 +01:00
|
|
|
if (s0.length() > 0) {
|
|
|
|
s.append(s0).append(',');
|
|
|
|
}
|
2011-07-04 18:35:30 +02:00
|
|
|
}
|
|
|
|
} catch (final IOException e1) {
|
|
|
|
}
|
2012-01-10 03:00:55 +01:00
|
|
|
if (s.length() > 0) {
|
|
|
|
s.setLength(s.length() - 1);
|
|
|
|
}
|
2011-07-04 18:35:30 +02:00
|
|
|
solrurls = s.toString().trim();
|
2012-07-10 12:01:20 +02:00
|
|
|
env.setConfig(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_URL, solrurls);
|
|
|
|
env.setConfig(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_SHARDING, post.get("solr.indexing.sharding", env.getConfig(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_SHARDING, "modulo-host-md5")));
|
2013-02-15 01:38:10 +01:00
|
|
|
|
2012-07-20 11:40:33 +02:00
|
|
|
if (solrRemoteWasOn && !solrRemoteIsOnAfterwards) {
|
2011-05-26 12:57:02 +02:00
|
|
|
// switch off
|
2012-09-26 13:38:04 +02:00
|
|
|
try {
|
|
|
|
sb.index.fulltext().disconnectRemoteSolr();
|
|
|
|
} catch (Throwable e) {
|
|
|
|
Log.logException(e);
|
|
|
|
}
|
2011-05-26 12:57:02 +02:00
|
|
|
}
|
2011-06-30 17:49:21 +02:00
|
|
|
|
2013-01-25 16:06:58 +01:00
|
|
|
if (solrRemoteIsOnAfterwards) try {
|
2012-09-25 23:59:09 +02:00
|
|
|
if (solrRemoteWasOn) sb.index.fulltext().disconnectRemoteSolr();
|
2011-05-26 12:57:02 +02:00
|
|
|
// switch on
|
2012-07-10 12:01:20 +02:00
|
|
|
final boolean usesolr = sb.getConfigBool(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_ENABLED, false) & solrurls.length() > 0;
|
2011-05-26 12:57:02 +02:00
|
|
|
try {
|
2012-06-25 14:59:46 +02:00
|
|
|
if (usesolr) {
|
2013-03-01 15:27:17 +01:00
|
|
|
ArrayList<RemoteInstance> instances = RemoteInstance.getShardInstances(solrurls, null, null);
|
2013-02-21 13:23:55 +01:00
|
|
|
sb.index.fulltext().connectRemoteSolr(instances);
|
2012-06-25 14:59:46 +02:00
|
|
|
} else {
|
2012-08-17 15:52:33 +02:00
|
|
|
sb.index.fulltext().disconnectRemoteSolr();
|
2012-06-25 14:59:46 +02:00
|
|
|
}
|
2012-09-26 13:38:04 +02:00
|
|
|
} catch (final Throwable e) {
|
2011-05-26 12:57:02 +02:00
|
|
|
Log.logException(e);
|
2012-09-26 13:38:04 +02:00
|
|
|
try {
|
|
|
|
sb.index.fulltext().disconnectRemoteSolr();
|
|
|
|
} catch (Throwable ee) {
|
|
|
|
Log.logException(ee);
|
|
|
|
}
|
2011-05-26 12:57:02 +02:00
|
|
|
}
|
2013-01-25 16:06:58 +01:00
|
|
|
} catch (SolrException e) {
|
|
|
|
Log.logSevere("IndexFederated_p", "change of solr connection failed", e);
|
2011-05-26 12:57:02 +02:00
|
|
|
}
|
2013-02-24 18:09:34 +01:00
|
|
|
|
|
|
|
boolean lazy = post.getBoolean("solr.indexing.lazy");
|
|
|
|
env.setConfig(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_LAZY, lazy);
|
2011-05-26 12:57:02 +02:00
|
|
|
}
|
2011-06-30 17:49:21 +02:00
|
|
|
|
2011-05-26 12:57:02 +02:00
|
|
|
// show solr host table
|
2012-08-17 15:52:33 +02:00
|
|
|
if (!sb.index.fulltext().connectedRemoteSolr()) {
|
2011-05-26 12:57:02 +02:00
|
|
|
prop.put("table", 0);
|
|
|
|
} else {
|
|
|
|
prop.put("table", 1);
|
2013-02-21 13:23:55 +01:00
|
|
|
final SolrConnector solr = sb.index.fulltext().getDefaultRemoteSolrConnector();
|
2013-03-01 15:27:17 +01:00
|
|
|
final long[] size = new long[]{((RemoteSolrConnector) solr).getSize()};
|
|
|
|
final ArrayList<String> urls = ((ShardInstance) ((RemoteSolrConnector) solr).getInstance()).getAdminInterfaces();
|
2011-07-04 18:35:30 +02:00
|
|
|
boolean dark = false;
|
|
|
|
for (int i = 0; i < size.length; i++) {
|
|
|
|
prop.put("table_list_" + i + "_dark", dark ? 1 : 0); dark = !dark;
|
2013-02-21 13:23:55 +01:00
|
|
|
prop.put("table_list_" + i + "_url", urls.get(i));
|
2011-07-04 18:35:30 +02:00
|
|
|
prop.put("table_list_" + i + "_size", size[i]);
|
2011-05-26 12:57:02 +02:00
|
|
|
}
|
2011-07-04 18:35:30 +02:00
|
|
|
prop.put("table_list", size.length);
|
2011-05-26 12:57:02 +02:00
|
|
|
}
|
2011-06-30 17:49:21 +02:00
|
|
|
|
2012-08-16 18:17:47 +02:00
|
|
|
prop.put(SwitchboardConstants.CORE_SERVICE_FULLTEXT + ".checked", env.getConfigBool(SwitchboardConstants.CORE_SERVICE_FULLTEXT, false) ? 1 : 0);
|
2012-07-23 16:28:39 +02:00
|
|
|
prop.put(SwitchboardConstants.CORE_SERVICE_RWI + ".checked", env.getConfigBool(SwitchboardConstants.CORE_SERVICE_RWI, false) ? 1 : 0);
|
|
|
|
prop.put(SwitchboardConstants.CORE_SERVICE_CITATION + ".checked", env.getConfigBool(SwitchboardConstants.CORE_SERVICE_CITATION, false) ? 1 : 0);
|
2013-02-24 18:09:34 +01:00
|
|
|
prop.put(SwitchboardConstants.CORE_SERVICE_WEBGRAPH + ".checked", env.getConfigBool(SwitchboardConstants.CORE_SERVICE_WEBGRAPH, false) ? 1 : 0);
|
|
|
|
prop.put(SwitchboardConstants.CORE_SERVICE_JENA + ".checked", env.getConfigBool(SwitchboardConstants.CORE_SERVICE_JENA, false) ? 1 : 0);
|
2012-07-10 12:01:20 +02:00
|
|
|
prop.put("solr.indexing.solrremote.checked", env.getConfigBool(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_ENABLED, false) ? 1 : 0);
|
|
|
|
prop.put("solr.indexing.url", env.getConfig(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_URL, "http://127.0.0.1:8983/solr").replace(",", "\n"));
|
|
|
|
prop.put("solr.indexing.sharding", env.getConfig(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_SHARDING, "modulo-host-md5"));
|
2011-05-26 12:57:02 +02:00
|
|
|
|
2013-02-24 18:09:34 +01:00
|
|
|
prop.put("solr.indexing.lazy.checked", env.getConfigBool(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_LAZY, true) ? 1 : 0);
|
|
|
|
|
2011-05-26 12:57:02 +02:00
|
|
|
// return rewrite properties
|
|
|
|
return prop;
|
|
|
|
}
|
|
|
|
}
|