Marown/yacy_search_server
1
0
Fork 0
mirror of https://github.com/yacy/yacy_search_server.git synced 2024-09-19 00:01:41 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity

- enhanced solr.add procedure for mass adds

Browse Source 
- removed unused solr access classes
- made snippet generation for documents aus YaCy RWI/DHT concurrent (as
it was before the search process removation)
- reduced the number of remote results in settings file because the
processing of such mass documents add is too CPU-intensive (in Solr)
This commit is contained in:
orbiter 2013-03-01 15:27:17 +01:00
parent 7ff10bdb1b
commit 0f7ea7ad9f
13 changed files with 55 additions and 641 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
defaults/yacy.network.freeworld.unit
View File

@ -18,10 +18,10 @@ network.unit.description = Public YaCy Community
network.unit.domain = global
# maximum search time for remote queries (milliseconds)
network.unit.remotesearch.maxtime = 1000
network.unit.remotesearch.maxtime = 3000
# maximum number of results per remote query
network.unit.remotesearch.maxcount = 100
network.unit.remotesearch.maxcount = 10
# flag to switch on dht transmission
# if the dht transmission is set to 'false' then for a global

4
defaults/yacy.network.metager.unit
View File

@ -18,10 +18,10 @@ network.unit.description = YaCy freeworld enhanced for MetaGer feed
network.unit.domain = global
# maximum search time for remote queries (milliseconds)
network.unit.remotesearch.maxtime = 1000
network.unit.remotesearch.maxtime = 3000
# maximum number of results per remote query
network.unit.remotesearch.maxcount = 100
network.unit.remotesearch.maxcount = 10
# flag to switch on dht transmission
# if the dht transmission is set to 'false' then for a global

7
htroot/IndexFederated_p.java
View File

@ -28,7 +28,6 @@ import org.apache.solr.common.SolrException;
import net.yacy.cora.document.UTF8;
import net.yacy.cora.federate.solr.connector.RemoteSolrConnector;
import net.yacy.cora.federate.solr.connector.ShardSolrConnector;
import net.yacy.cora.federate.solr.connector.SolrConnector;
import net.yacy.cora.federate.solr.instance.RemoteInstance;
import net.yacy.cora.federate.solr.instance.ShardInstance;
@ -130,7 +129,7 @@ public class IndexFederated_p {
final boolean usesolr = sb.getConfigBool(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_ENABLED, false) & solrurls.length() > 0;
try {
if (usesolr) {
ArrayList<RemoteInstance> instances = ShardSolrConnector.getShardInstances(solrurls, null, null);
ArrayList<RemoteInstance> instances = RemoteInstance.getShardInstances(solrurls, null, null);
sb.index.fulltext().connectRemoteSolr(instances);
} else {
sb.index.fulltext().disconnectRemoteSolr();
@ -157,8 +156,8 @@ public class IndexFederated_p {
} else {
prop.put("table", 1);
final SolrConnector solr = sb.index.fulltext().getDefaultRemoteSolrConnector();
final long[] size = (solr instanceof ShardSolrConnector) ? ((ShardSolrConnector) solr).getSizeList() : new long[]{((RemoteSolrConnector) solr).getSize()};
final ArrayList<String> urls = (solr instanceof ShardSolrConnector) ? ((ShardSolrConnector) solr).getAdminInterfaces() : ((ShardInstance) ((RemoteSolrConnector) solr).getInstance()).getAdminInterfaces();
final long[] size = new long[]{((RemoteSolrConnector) solr).getSize()};
final ArrayList<String> urls = ((ShardInstance) ((RemoteSolrConnector) solr).getInstance()).getAdminInterfaces();
boolean dark = false;
for (int i = 0; i < size.length; i++) {
prop.put("table_list_" + i + "_dark", dark ? 1 : 0); dark = !dark;

9
source/net/yacy/cora/federate/solr/connector/AbstractSolrConnector.java
View File

@ -21,7 +21,6 @@
package net.yacy.cora.federate.solr.connector;
import java.io.IOException;
import java.util.Collection;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
@ -44,7 +43,6 @@ import org.apache.solr.client.solrj.response.FacetField.Count;
import org.apache.solr.common.SolrDocument;
import org.apache.solr.common.SolrDocumentList;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.common.params.FacetParams;
import org.apache.solr.common.params.ModifiableSolrParams;
@ -286,11 +284,4 @@ public abstract class AbstractSolrConnector implements SolrConnector {
throw new IOException(e.getMessage(), e);
}
}
@Override
public void add(final Collection<SolrInputDocument> solrdocs) throws IOException, SolrException {
for (SolrInputDocument solrdoc: solrdocs) {
add(solrdoc);
}
}
}

15
source/net/yacy/cora/federate/solr/connector/CachedSolrConnector.java
View File

@ -21,6 +21,7 @@
package net.yacy.cora.federate.solr.connector;
import java.io.IOException;
import java.util.Collection;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
@ -222,6 +223,20 @@ public class CachedSolrConnector extends AbstractSolrConnector implements SolrCo
if (this.solr != null) this.solr.add(solrdoc);
}
@Override
public void add(final Collection<SolrInputDocument> solrdocs) throws IOException, SolrException {
for (SolrInputDocument solrdoc: solrdocs) {
String id = (String) solrdoc.getFieldValue(CollectionSchema.id.getSolrFieldName());
assert id != null;
if (id == null) continue;
SolrDocument doc = ClientUtils.toSolrDocument(solrdoc);
addToCache(doc, true);
this.documentCache.put(id, doc);
this.documentCache_Insert++;
}
if (this.solr != null) this.solr.add(solrdocs);
}
/**
* get a query result from solr
* to get all results set the query String to "*:*"

11
source/net/yacy/cora/federate/solr/connector/MirrorSolrConnector.java
View File

@ -21,13 +21,13 @@
package net.yacy.cora.federate.solr.connector;
import java.io.IOException;
import java.util.Collection;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.atomic.AtomicLong;
import net.yacy.cora.sorting.ReversibleScoreMap;
import net.yacy.search.schema.CollectionSchema;
import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.common.SolrDocument;
@ -168,12 +168,15 @@ public class MirrorSolrConnector extends AbstractSolrConnector implements SolrCo
*/
@Override
public void add(final SolrInputDocument solrdoc) throws IOException {
String id = (String) solrdoc.getFieldValue(CollectionSchema.id.getSolrFieldName());
assert id != null;
if (id == null) return;
if (this.solr0 != null) this.solr0.add(solrdoc);
if (this.solr1 != null) this.solr1.add(solrdoc);
}
@Override
public void add(final Collection<SolrInputDocument> solrdocs) throws IOException, SolrException {
if (this.solr0 != null) this.solr0.add(solrdocs);
if (this.solr1 != null) this.solr1.add(solrdocs);
}
/**
* get a query result from solr

150
source/net/yacy/cora/federate/solr/connector/MultipleSolrConnector.java
View File

@ -1,150 +0,0 @@
/**
* MultipleSolrConnector
* Copyright 2011 by Michael Peter Christen
* First released 08.11.2011 at http://yacy.net
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program in the file lgpl21.txt
* If not, see <http://www.gnu.org/licenses/>.
*/
package net.yacy.cora.federate.solr.connector;
import java.io.IOException;
import java.util.List;
import java.util.concurrent.ArrayBlockingQueue;
import net.yacy.cora.federate.solr.instance.SolrInstance;
import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.common.params.ModifiableSolrParams;
public class MultipleSolrConnector extends AbstractSolrConnector implements SolrConnector {
private final static SolrInputDocument POISON_DOC = new SolrInputDocument();
private final ArrayBlockingQueue<SolrInputDocument> queue;
private final AddWorker[] worker;
private final SolrConnector solr;
public MultipleSolrConnector(final SolrInstance instance, final String corename, final int connections) {
this.solr = new RemoteSolrConnector(instance, corename);
this.queue = new ArrayBlockingQueue<SolrInputDocument>(1000);
this.worker = new AddWorker[connections];
for (int i = 0; i < connections; i++) {
this.worker[i] = new AddWorker(instance, corename);
this.worker[i].start();
}
}
private class AddWorker extends Thread {
private final SolrConnector solr;
public AddWorker(final SolrInstance instance, final String corename) {
this.solr = new RemoteSolrConnector(instance, corename);
}
@Override
public void run() {
SolrInputDocument doc;
try {
while ((doc = MultipleSolrConnector.this.queue.take()) != POISON_DOC) {
try {
this.solr.add(doc);
} catch (SolrException e) {
e.printStackTrace();
} catch (IOException e) {
e.printStackTrace();
}
}
} catch (InterruptedException e) {
} finally {
this.solr.close();
}
}
}
@Override
public void commit(boolean softCommit) {
this.solr.commit(softCommit);
}
/**
* force an explicit merge of segments
* @param maxSegments the maximum number of segments. Set to 1 for maximum optimization
*/
public void optimize(int maxSegments) {
this.solr.optimize(maxSegments);
}
@Override
public void close() {
// send termination signal to worker
for (@SuppressWarnings("unused") AddWorker element : this.worker) {
try {
this.queue.put(POISON_DOC);
} catch (InterruptedException e) {
e.printStackTrace();
}
}
// wait for termination
for (AddWorker element : this.worker) {
try {
element.join();
} catch (InterruptedException e) {
e.printStackTrace();
}
}
this.solr.close();
}
@Override
public void clear() throws IOException {
this.solr.clear();
}
@Override
public void delete(final String id) throws IOException {
this.solr.delete(id);
}
@Override
public void delete(final List<String> ids) throws IOException {
this.solr.delete(ids);
}
@Override
public void deleteByQuery(final String querystring) throws IOException {
this.solr.deleteByQuery(querystring);
}
@Override
public void add(final SolrInputDocument solrdoc) throws IOException, SolrException {
try {
this.queue.put(solrdoc);
} catch (InterruptedException e) {
e.printStackTrace();
}
}
@Override
public QueryResponse query(final ModifiableSolrParams query) throws IOException, SolrException {
return this.solr.query(query);
}
@Override
public long getSize() {
return this.solr.getSize();
}
}

161
source/net/yacy/cora/federate/solr/connector/RetrySolrConnector.java
View File

@ -1,161 +0,0 @@
/**
* SolrRetryConnector
* Copyright 2011 by Michael Peter Christen
* First released 08.11.2011 at http://yacy.net
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program in the file lgpl21.txt
* If not, see <http://www.gnu.org/licenses/>.
*/
package net.yacy.cora.federate.solr.connector;
import java.io.IOException;
import java.util.List;
import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.common.params.ModifiableSolrParams;
public class RetrySolrConnector extends AbstractSolrConnector implements SolrConnector {
private final SolrConnector solrConnector;
private final long retryMaxTime;
public RetrySolrConnector(final SolrConnector solrConnector, final long retryMaxTime) {
this.solrConnector = solrConnector;
this.retryMaxTime = retryMaxTime;
}
@Override
public void commit(boolean softCommit) {
this.solrConnector.commit(softCommit);
}
/**
* force an explicit merge of segments
* @param maxSegments the maximum number of segments. Set to 1 for maximum optimization
*/
public void optimize(int maxSegments) {
this.solrConnector.optimize(maxSegments);
}
@Override
public synchronized void close() {
this.solrConnector.close();
}
@Override
public void clear() throws IOException {
final long t = System.currentTimeMillis() + this.retryMaxTime;
Throwable ee = null;
while (System.currentTimeMillis() < t) try {
this.solrConnector.clear();
return;
} catch (final Throwable e) {
ee = e;
try {Thread.sleep(10);} catch (final InterruptedException e1) {}
continue;
}
if (ee != null) throw (ee instanceof IOException) ? (IOException) ee : new IOException(ee.getMessage());
}
@Override
public void delete(final String id) throws IOException {
final long t = System.currentTimeMillis() + this.retryMaxTime;
Throwable ee = null;
while (System.currentTimeMillis() < t) try {
this.solrConnector.delete(id);
return;
} catch (final Throwable e) {
ee = e;
try {Thread.sleep(10);} catch (final InterruptedException e1) {}
continue;
}
if (ee != null) throw (ee instanceof IOException) ? (IOException) ee : new IOException(ee.getMessage());
}
@Override
public void delete(final List<String> ids) throws IOException {
final long t = System.currentTimeMillis() + this.retryMaxTime;
Throwable ee = null;
while (System.currentTimeMillis() < t) try {
this.solrConnector.delete(ids);
return;
} catch (final Throwable e) {
ee = e;
try {Thread.sleep(10);} catch (final InterruptedException e1) {}
continue;
}
if (ee != null) throw (ee instanceof IOException) ? (IOException) ee : new IOException(ee.getMessage());
}
@Override
public void deleteByQuery(final String querystring) throws IOException {
final long t = System.currentTimeMillis() + this.retryMaxTime;
Throwable ee = null;
while (System.currentTimeMillis() < t) try {
this.solrConnector.deleteByQuery(querystring);
return;
} catch (final Throwable e) {
ee = e;
try {Thread.sleep(10);} catch (final InterruptedException e1) {}
continue;
}
if (ee != null) throw (ee instanceof IOException) ? (IOException) ee : new IOException(ee.getMessage());
}
@Override
public void add(final SolrInputDocument solrdoc) throws IOException, SolrException {
final long t = System.currentTimeMillis() + this.retryMaxTime;
Throwable ee = null;
while (System.currentTimeMillis() < t) try {
this.solrConnector.add(solrdoc);
return;
} catch (final Throwable e) {
ee = e;
try {Thread.sleep(10);} catch (final InterruptedException e1) {}
continue;
}
if (ee != null) throw (ee instanceof IOException) ? (IOException) ee : new IOException(ee.getMessage());
}
@Override
public QueryResponse query(final ModifiableSolrParams query) throws IOException, SolrException {
final long t = System.currentTimeMillis() + this.retryMaxTime;
Throwable ee = null;
while (System.currentTimeMillis() < t) try {
return this.solrConnector.query(query);
} catch (final Throwable e) {
ee = e;
try {Thread.sleep(10);} catch (final InterruptedException e1) {}
continue;
}
if (ee != null) throw (ee instanceof IOException) ? (IOException) ee : new IOException(ee.getMessage());
return null;
}
@Override
public long getSize() {
final long t = System.currentTimeMillis() + this.retryMaxTime;
while (System.currentTimeMillis() < t) try {
return this.solrConnector.getSize();
} catch (final Throwable e) {
try {Thread.sleep(10);} catch (final InterruptedException e1) {}
continue;
}
return 0;
}
}

295
source/net/yacy/cora/federate/solr/connector/ShardSolrConnector.java
View File

@ -1,295 +0,0 @@
/**
* ShardSolrConnector
* Copyright 2011 by Michael Peter Christen
* First released 25.05.2011 at http://yacy.net
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program in the file lgpl21.txt
* If not, see <http://www.gnu.org/licenses/>.
*/
package net.yacy.cora.federate.solr.connector;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentLinkedQueue;
import java.util.concurrent.atomic.AtomicLong;
import net.yacy.cora.sorting.ReversibleScoreMap;
import net.yacy.cora.federate.solr.instance.ResponseAccumulator;
import net.yacy.cora.federate.solr.instance.RemoteInstance;
import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.common.SolrDocument;
import org.apache.solr.common.SolrDocumentList;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.common.params.ModifiableSolrParams;
public class ShardSolrConnector extends AbstractSolrConnector implements SolrConnector {
private final ArrayList<RemoteInstance> instances;
private final ArrayList<SolrConnector> connectors;
private final ShardSelection sharding;
private final ArrayList<String> adminInterfaces;
public ShardSolrConnector(
ArrayList<RemoteInstance> instances,
final ShardSelection.Method method, boolean multipleConnections) {
this.instances = instances;
this.connectors = new ArrayList<SolrConnector>();
SolrConnector s;
this.adminInterfaces = new ArrayList<String>(instances.size());
String defaultCoreName = instances.get(0).getDefaultCoreName();
for (final RemoteInstance instance: instances) {
adminInterfaces.add(instance.getAdminInterface());
s = multipleConnections ? new MultipleSolrConnector(instance, defaultCoreName, 2) : new RemoteSolrConnector(instance, defaultCoreName);
this.connectors.add(s /*new RetrySolrConnector(s, timeout)*/);
}
this.sharding = new ShardSelection(method, this.connectors.size());
}
public static ArrayList<RemoteInstance> getShardInstances(final String urlList, Collection<String> coreNames, String defaultCoreName) throws IOException {
urlList.replace(' ', ',');
String[] urls = urlList.split(",");
ArrayList<RemoteInstance> instances = new ArrayList<RemoteInstance>();
for (final String u: urls) {
RemoteInstance instance = new RemoteInstance(u, coreNames, defaultCoreName);
instances.add(instance);
}
return instances;
}
public ArrayList<RemoteInstance> getInstances() {
return this.instances;
}
@Override
public void commit(boolean softCommit) {
for (final SolrConnector connector: this.connectors) connector.commit(softCommit);
}
/**
* force an explicit merge of segments
* @param maxSegments the maximum number of segments. Set to 1 for maximum optimization
*/
@Override
public void optimize(int maxSegments) {
for (final SolrConnector connector: this.connectors) connector.optimize(maxSegments);
}
@Override
public synchronized void close() {
for (final SolrConnector connector: this.connectors) connector.close();
}
/**
* delete everything in the solr index
* @throws IOException
*/
@Override
public void clear() throws IOException {
for (final SolrConnector connector: this.connectors) connector.clear();
}
/**
* delete an entry from solr
* @param id the url hash of the entry
* @throws IOException
*/
@Override
public void delete(final String id) throws IOException {
for (final SolrConnector connector: this.connectors) connector.delete(id);
}
/**
* delete a set of entries from solr; entries are identified by their url hash
* @param ids a list of url hashes
* @throws IOException
*/
@Override
public void delete(final List<String> ids) throws IOException {
for (final SolrConnector connector: this.connectors) connector.delete(ids);
}
@Override
public void deleteByQuery(final String querystring) throws IOException {
for (final SolrConnector connector: this.connectors) connector.deleteByQuery(querystring);
}
/**
* check if a given id exists in solr
* @param id
* @return true if any entry in solr exists
* @throws IOException
*/
@Override
public boolean exists(final String fieldName, final String key) throws IOException {
for (final SolrConnector connector: this.connectors) {
if (connector.exists(fieldName, key)) return true;
}
return false;
}
@Override
public SolrDocument getById(final String key, final String ... fields) throws IOException {
for (final SolrConnector connector: this.connectors) {
SolrDocument doc = connector.getById(key, fields);
if (doc != null) return doc;
}
return null;
}
/**
* add a Solr document
* @param solrdoc
* @throws IOException
*/
@Override
public void add(final SolrInputDocument solrdoc) throws IOException {
this.connectors.get(this.sharding.select(solrdoc)).add(solrdoc);
}
/**
* add a collection of Solr documents
* @param docs
* @throws IOException
*/
protected void addSolr(final Collection<SolrInputDocument> docs) throws IOException {
for (final SolrInputDocument doc: docs) add(doc);
}
/**
* get a query result from solr
* to get all results set the query String to "*:*"
* @param querystring
* @throws IOException
*/
@Override
public SolrDocumentList query(final String querystring, final int offset, final int count, final String ... fields) throws IOException {
final SolrDocumentList list = new SolrDocumentList();
List<Thread> t = new ArrayList<Thread>();
for (final SolrConnector connector: this.connectors) {
Thread t0 = new Thread() {
@Override
public void run() {
try {
final SolrDocumentList l = connector.query(querystring, offset, count, fields);
for (final SolrDocument d: l) {
list.add(d);
}
} catch (IOException e) {}
}
};
t0.start();
t.add(t0);
}
for (Thread t0: t) {
try {t0.join();} catch (InterruptedException e) {}
}
return list;
}
@Override
public QueryResponse query(final ModifiableSolrParams query) throws IOException, SolrException {
final Collection<QueryResponse> qrl = new ConcurrentLinkedQueue<QueryResponse>();
// concurrently call all shards
List<Thread> t = new ArrayList<Thread>();
for (final SolrConnector connector: this.connectors) {
Thread t0 = new Thread() {
@Override
public void run() {
QueryResponse rsp;
try {
rsp = connector.query(query);
} catch (Throwable e) {return;}
qrl.add(rsp);
}
};
t0.start();
t.add(t0);
}
for (Thread t0: t) {
try {t0.join();} catch (InterruptedException e) {}
}
// prepare combined response
return ResponseAccumulator.combineResponses(qrl);
}
@Override
public long getQueryCount(final String querystring) throws IOException {
final AtomicLong count = new AtomicLong(0);
List<Thread> t = new ArrayList<Thread>();
for (final SolrConnector connector: this.connectors) {
Thread t0 = new Thread() {
@Override
public void run() {
try {
count.addAndGet(connector.getQueryCount(querystring));
} catch (IOException e) {}
}
};
t0.start();
t.add(t0);
}
for (Thread t0: t) {
try {t0.join();} catch (InterruptedException e) {}
}
return count.get();
}
@Override
public Map<String, ReversibleScoreMap<String>> getFacets(String query, int maxresults, final String ... fields) throws IOException {
Map<String, ReversibleScoreMap<String>> facets = new HashMap<String, ReversibleScoreMap<String>>();
for (final SolrConnector connector: this.connectors) {
Map<String, ReversibleScoreMap<String>> peer = connector.getFacets(query, maxresults, fields);
innerloop: for (Map.Entry<String, ReversibleScoreMap<String>> facet: facets.entrySet()) {
ReversibleScoreMap<String> peerfacet = peer.remove(facet.getKey());
if (peerfacet == null) continue innerloop;
for (String key: peerfacet) facet.getValue().inc(key, peerfacet.get(key));
}
for (Map.Entry<String, ReversibleScoreMap<String>> peerfacet: peer.entrySet()) {
facets.put(peerfacet.getKey(), peerfacet.getValue());
}
}
return facets;
}
public long[] getSizeList() {
final long[] size = new long[this.connectors.size()];
int i = 0;
for (final SolrConnector connector: this.connectors) {
size[i++] = connector.getSize();
}
return size;
}
@Override
public long getSize() {
final long[] size = getSizeList();
long s = 0;
for (final long l: size) s += l;
return s;
}
public ArrayList<String> getAdminInterfaces() {
return this.adminInterfaces;
}
}

13
source/net/yacy/cora/federate/solr/instance/RemoteInstance.java
View File

@ -67,11 +67,16 @@ public class RemoteInstance implements SolrInstance {
private final Collection<String> coreNames;
private final Map<String, HttpSolrServer> server;
/*
public RemoteInstance(final String url) throws IOException {
this(url, null, url.endsWith("solr/") || url.endsWith("solr") ? "solr" : CollectionSchema.CORE_NAME);
public static ArrayList<RemoteInstance> getShardInstances(final String urlList, Collection<String> coreNames, String defaultCoreName) throws IOException {
urlList.replace(' ', ',');
String[] urls = urlList.split(",");
ArrayList<RemoteInstance> instances = new ArrayList<RemoteInstance>();
for (final String u: urls) {
RemoteInstance instance = new RemoteInstance(u, coreNames, defaultCoreName);
instances.add(instance);
}
return instances;
}
*/
public RemoteInstance(final String url, final Collection<String> coreNames, final String defaultCoreName) throws IOException {
this.server= new HashMap<String, HttpSolrServer>();

5
source/net/yacy/search/Switchboard.java
View File

@ -100,7 +100,6 @@ import net.yacy.cora.federate.solr.Boost;
import net.yacy.cora.federate.solr.SchemaConfiguration;
import net.yacy.cora.federate.solr.ProcessType;
import net.yacy.cora.federate.solr.connector.AbstractSolrConnector;
import net.yacy.cora.federate.solr.connector.ShardSolrConnector;
import net.yacy.cora.federate.solr.instance.RemoteInstance;
import net.yacy.cora.federate.yacy.CacheStrategy;
import net.yacy.cora.lod.JenaTripleStore;
@ -490,7 +489,7 @@ public final class Switchboard extends serverSwitch {
if (usesolr && solrurls != null && solrurls.length() > 0) {
try {
ArrayList<RemoteInstance> instances = ShardSolrConnector.getShardInstances(solrurls, null, null);
ArrayList<RemoteInstance> instances = RemoteInstance.getShardInstances(solrurls, null, null);
this.index.fulltext().connectRemoteSolr(instances);
} catch ( final IOException e ) {
Log.logException(e);
@ -1329,7 +1328,7 @@ public final class Switchboard extends serverSwitch {
if (usesolr && solrurls != null && solrurls.length() > 0) {
try {
ArrayList<RemoteInstance> instances = ShardSolrConnector.getShardInstances(solrurls, null, null);
ArrayList<RemoteInstance> instances = RemoteInstance.getShardInstances(solrurls, null, null);
this.index.fulltext().connectRemoteSolr(instances);
} catch ( final IOException e ) {
Log.logException(e);

1
source/net/yacy/search/index/Fulltext.java
View File

@ -61,7 +61,6 @@ import net.yacy.document.parser.html.CharacterCoding;
import net.yacy.kelondro.data.meta.DigestURI;
import net.yacy.kelondro.data.meta.URIMetadataNode;
import net.yacy.kelondro.data.meta.URIMetadataRow;
import net.yacy.kelondro.data.word.WordReference;
import net.yacy.kelondro.data.word.WordReferenceVars;
import net.yacy.kelondro.index.Cache;
import net.yacy.kelondro.index.Index;

21
source/net/yacy/search/query/SearchEvent.java
View File

@ -146,7 +146,7 @@ public final class SearchEvent {
private final long maxtime;
private final ConcurrentHashMap<String, WeakPriorityBlockingQueue<WordReferenceVars>> doubleDomCache; // key = domhash (6 bytes); value = like stack
private final int[] flagcount; // flag counter
private final AtomicInteger feedersAlive, feedersTerminated;
private final AtomicInteger feedersAlive, feedersTerminated, snippetFetchAlive;
private boolean addRunning;
private final AtomicInteger receivedRemoteReferences;
private final ReferenceOrder order;
@ -245,6 +245,7 @@ public final class SearchEvent {
}
this.feedersAlive = new AtomicInteger(0);
this.feedersTerminated = new AtomicInteger(0);
this.snippetFetchAlive = new AtomicInteger(0);
this.addRunning = true;
this.receivedRemoteReferences = new AtomicInteger(0);
this.order = new ReferenceOrder(this.query.ranking, UTF8.getBytes(this.query.targetlang));
@ -1084,17 +1085,25 @@ public final class SearchEvent {
addResult(getSnippet(localEntry, null));
success = true;
}
if (localEntry == null) {
if (SearchEvent.this.snippetFetchAlive.get() >= 10) {
// too many concurrent processes
URIMetadataNode p2pEntry = pullOneFilteredFromRWI(true);
if (p2pEntry != null) {
addResult(getSnippet(p2pEntry, null));
success = true;
}
} else {
new Thread() {
final URIMetadataNode p2pEntry = pullOneFilteredFromRWI(true);
if (p2pEntry != null) new Thread() {
public void run() {
URIMetadataNode p2pEntry = pullOneFilteredFromRWI(true);
if (p2pEntry != null) addResult(getSnippet(p2pEntry, null));
SearchEvent.this.oneFeederStarted();
SearchEvent.this.snippetFetchAlive.incrementAndGet();
try {
addResult(getSnippet(p2pEntry, null));
} catch (Throwable e) {} finally {
SearchEvent.this.oneFeederTerminated();
SearchEvent.this.snippetFetchAlive.decrementAndGet();
}
}
}.start();
}
@ -1310,7 +1319,7 @@ public final class SearchEvent {
}
public void oneFeederStarted() {
this.feedersAlive.addAndGet(1);
this.feedersAlive.incrementAndGet();
}
public QueryParams getQuery() {