mirror of
https://github.com/yacy/yacy_search_server.git
synced 2024-09-19 00:01:41 +02:00
typo
git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@8027 6c8d7289-2bf4-0310-a012-ef5d649a1542
This commit is contained in:
parent
9067ab20b2
commit
78ce3b13be
|
@ -1043,5 +1043,5 @@ color_searchurlhover = #008000
|
|||
federated.service.yacy.indexing.enabled = true
|
||||
federated.service.solr.indexing.enabled = false
|
||||
federated.service.solr.indexing.url = http://127.0.0.1:8983/solr
|
||||
federated.service.solr.indexing.charding = MODULO_HOST_MD5
|
||||
federated.service.solr.indexing.sharding = MODULO_HOST_MD5
|
||||
federated.service.solr.indexing.schemefile = solr.keys.default.list
|
||||
|
|
|
@ -53,8 +53,8 @@
|
|||
#(/table)#
|
||||
<dt class="TableCellDark">Solr URL(s)</dt>
|
||||
<dd><textarea rows="5" cols="80" name="solr.indexing.url" id="solr.indexing.url"/>#[solr.indexing.url]#</textarea></dd>
|
||||
<dt class="TableCellDark">Charding Method</dt>
|
||||
<dd><input type="text" size="50" maxlength="50" value="#[solr.indexing.charding]#" name="solr.indexing.charding" id="solr.indexing.charding" disabled="disabled"/></dd>
|
||||
<dt class="TableCellDark">Sharding Method</dt>
|
||||
<dd><input type="text" size="50" maxlength="50" value="#[solr.indexing.sharding]#" name="solr.indexing.sharding" id="solr.indexing.sharding" disabled="disabled"/></dd>
|
||||
<dt class="TableCellDark">Scheme</dt>
|
||||
<dd><input type="text" size="50" maxlength="50" value="#[solr.indexing.schemefile]#" name="solr.indexing.schemefile" id="solr.indexing.schemefile" disabled="disabled"/></dd>
|
||||
</dl>
|
||||
|
|
|
@ -31,10 +31,10 @@ import java.util.Iterator;
|
|||
|
||||
import net.yacy.cora.document.UTF8;
|
||||
import net.yacy.cora.protocol.RequestHeader;
|
||||
import net.yacy.cora.services.federated.solr.SolrChardingConnector;
|
||||
import net.yacy.cora.services.federated.solr.SolrChardingSelection;
|
||||
import net.yacy.cora.services.federated.solr.SolrConnector;
|
||||
import net.yacy.cora.services.federated.solr.SolrScheme;
|
||||
import net.yacy.cora.services.federated.solr.SolrShardingConnector;
|
||||
import net.yacy.cora.services.federated.solr.SolrShardingSelection;
|
||||
import net.yacy.cora.services.federated.solr.SolrSingleConnector;
|
||||
import net.yacy.cora.storage.ConfigurationSet;
|
||||
import net.yacy.kelondro.logging.Log;
|
||||
|
@ -72,7 +72,7 @@ public class IndexFederated_p {
|
|||
if (s.length() > 0) s.setLength(s.length() - 1);
|
||||
solrurls = s.toString().trim();
|
||||
env.setConfig("federated.service.solr.indexing.url", solrurls);
|
||||
env.setConfig("federated.service.solr.indexing.charding", post.get("solr.indexing.charding", env.getConfig("federated.service.solr.indexing.charding", "modulo-host-md5")));
|
||||
env.setConfig("federated.service.solr.indexing.sharding", post.get("solr.indexing.sharding", env.getConfig("federated.service.solr.indexing.sharding", "modulo-host-md5")));
|
||||
final String schemename = post.get("solr.indexing.schemefile", env.getConfig("federated.service.solr.indexing.schemefile", "solr.keys.default.list"));
|
||||
env.setConfig("federated.service.solr.indexing.schemefile", schemename);
|
||||
|
||||
|
@ -88,7 +88,7 @@ public class IndexFederated_p {
|
|||
// switch on
|
||||
final boolean usesolr = sb.getConfigBool("federated.service.solr.indexing.enabled", false) & solrurls.length() > 0;
|
||||
try {
|
||||
sb.indexSegments.segment(Segments.Process.LOCALCRAWLING).connectSolr((usesolr) ? new SolrChardingConnector(solrurls, scheme, SolrChardingSelection.Method.MODULO_HOST_MD5, 10000) : null);
|
||||
sb.indexSegments.segment(Segments.Process.LOCALCRAWLING).connectSolr((usesolr) ? new SolrShardingConnector(solrurls, scheme, SolrShardingSelection.Method.MODULO_HOST_MD5, 10000) : null);
|
||||
} catch (final IOException e) {
|
||||
Log.logException(e);
|
||||
sb.indexSegments.segment(Segments.Process.LOCALCRAWLING).connectSolr(null);
|
||||
|
@ -118,8 +118,8 @@ public class IndexFederated_p {
|
|||
} else {
|
||||
prop.put("table", 1);
|
||||
final SolrConnector solr = sb.indexSegments.segment(Segments.Process.LOCALCRAWLING).getSolr();
|
||||
final long[] size = (solr instanceof SolrChardingConnector) ? ((SolrChardingConnector) solr).getSizeList() : new long[]{((SolrSingleConnector) solr).getSize()};
|
||||
final String[] urls = (solr instanceof SolrChardingConnector) ? ((SolrChardingConnector) solr).getAdminInterfaceList() : new String[]{((SolrSingleConnector) solr).getAdminInterface()};
|
||||
final long[] size = (solr instanceof SolrShardingConnector) ? ((SolrShardingConnector) solr).getSizeList() : new long[]{((SolrSingleConnector) solr).getSize()};
|
||||
final String[] urls = (solr instanceof SolrShardingConnector) ? ((SolrShardingConnector) solr).getAdminInterfaceList() : new String[]{((SolrSingleConnector) solr).getAdminInterface()};
|
||||
boolean dark = false;
|
||||
for (int i = 0; i < size.length; i++) {
|
||||
prop.put("table_list_" + i + "_dark", dark ? 1 : 0); dark = !dark;
|
||||
|
@ -153,7 +153,7 @@ public class IndexFederated_p {
|
|||
prop.put("yacy.indexing.enabled.checked", env.getConfigBool("federated.service.yacy.indexing.enabled", true) ? 1 : 0);
|
||||
prop.put("solr.indexing.enabled.checked", env.getConfigBool("federated.service.solr.indexing.enabled", false) ? 1 : 0);
|
||||
prop.put("solr.indexing.url", env.getConfig("federated.service.solr.indexing.url", "http://127.0.0.1:8983/solr").replace(",", "\n"));
|
||||
prop.put("solr.indexing.charding", env.getConfig("federated.service.solr.indexing.charding", "modulo-host-md5"));
|
||||
prop.put("solr.indexing.sharding", env.getConfig("federated.service.solr.indexing.sharding", "modulo-host-md5"));
|
||||
prop.put("solr.indexing.schemefile", schemename);
|
||||
|
||||
// return rewrite properties
|
||||
|
|
|
@ -35,7 +35,7 @@ import java.util.concurrent.ConcurrentLinkedQueue;
|
|||
|
||||
import net.yacy.cora.document.ASCII;
|
||||
import net.yacy.cora.document.UTF8;
|
||||
import net.yacy.cora.services.federated.solr.SolrChardingConnector;
|
||||
import net.yacy.cora.services.federated.solr.SolrShardingConnector;
|
||||
import net.yacy.cora.services.federated.solr.SolrConnector;
|
||||
import net.yacy.kelondro.data.meta.DigestURI;
|
||||
import net.yacy.kelondro.data.word.Word;
|
||||
|
@ -108,7 +108,7 @@ public class ZURL implements Iterable<ZURL.Entry> {
|
|||
this.stack = new ConcurrentLinkedQueue<byte[]>();
|
||||
}
|
||||
|
||||
public ZURL(final SolrChardingConnector solrConnector) {
|
||||
public ZURL(final SolrShardingConnector solrConnector) {
|
||||
this.solrConnector = solrConnector;
|
||||
// creates a new ZUR in RAM
|
||||
this.urlIndex = new RowSet(rowdef);
|
||||
|
|
|
@ -40,21 +40,21 @@ import org.apache.solr.common.SolrDocumentList;
|
|||
import org.apache.solr.common.SolrInputDocument;
|
||||
|
||||
|
||||
public class SolrChardingConnector implements SolrConnector {
|
||||
public class SolrShardingConnector implements SolrConnector {
|
||||
|
||||
private final List<SolrConnector> connectors;
|
||||
private final SolrScheme scheme;
|
||||
private final SolrChardingSelection charding;
|
||||
private final SolrShardingSelection sharding;
|
||||
private final String[] urls;
|
||||
|
||||
public SolrChardingConnector(final String urlList, final SolrScheme scheme, final SolrChardingSelection.Method method, final long timeout) throws IOException {
|
||||
public SolrShardingConnector(final String urlList, final SolrScheme scheme, final SolrShardingSelection.Method method, final long timeout) throws IOException {
|
||||
urlList.replace(' ', ',');
|
||||
this.urls = urlList.split(",");
|
||||
this.connectors = new ArrayList<SolrConnector>();
|
||||
for (final String u: this.urls) {
|
||||
this.connectors.add(new SolrRetryConnector(new SolrSingleConnector(u.trim(), scheme), timeout));
|
||||
}
|
||||
this.charding = new SolrChardingSelection(method, this.urls.length);
|
||||
this.sharding = new SolrShardingSelection(method, this.urls.length);
|
||||
this.scheme = scheme;
|
||||
}
|
||||
|
||||
|
@ -122,7 +122,7 @@ public class SolrChardingConnector implements SolrConnector {
|
|||
* @throws IOException
|
||||
*/
|
||||
public void add(final SolrInputDocument solrdoc) throws IOException {
|
||||
this.connectors.get(this.charding.select(solrdoc)).add(solrdoc);
|
||||
this.connectors.get(this.sharding.select(solrdoc)).add(solrdoc);
|
||||
}
|
||||
|
||||
/**
|
||||
|
@ -142,7 +142,7 @@ public class SolrChardingConnector implements SolrConnector {
|
|||
* @throws IOException
|
||||
*/
|
||||
public void err(final DigestURI digestURI, final String failReason, final int httpstatus) throws IOException {
|
||||
this.connectors.get(this.charding.selectURL(digestURI.toNormalform(true, false))).err(digestURI, failReason, httpstatus);
|
||||
this.connectors.get(this.sharding.selectURL(digestURI.toNormalform(true, false))).err(digestURI, failReason, httpstatus);
|
||||
}
|
||||
|
||||
|
|
@ -11,12 +11,12 @@
|
|||
* modify it under the terms of the GNU Lesser General Public
|
||||
* License as published by the Free Software Foundation; either
|
||||
* version 2.1 of the License, or (at your option) any later version.
|
||||
*
|
||||
*
|
||||
* This library is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
* Lesser General Public License for more details.
|
||||
*
|
||||
*
|
||||
* You should have received a copy of the GNU Lesser General Public License
|
||||
* along with this program in the file lgpl21.txt
|
||||
* If not, see <http://www.gnu.org/licenses/>.
|
||||
|
@ -34,21 +34,21 @@ import java.util.concurrent.atomic.AtomicLong;
|
|||
|
||||
import org.apache.solr.common.SolrInputDocument;
|
||||
|
||||
public class SolrChardingSelection {
|
||||
|
||||
public class SolrShardingSelection {
|
||||
|
||||
public final static Charset charsetUTF8;
|
||||
static {
|
||||
charsetUTF8 = Charset.forName("UTF-8");
|
||||
}
|
||||
|
||||
private final Method method; // the charding method
|
||||
private AtomicLong chardID; // the next id that shall be given away
|
||||
|
||||
private final Method method; // the sharding method
|
||||
private final AtomicLong chardID; // the next id that shall be given away
|
||||
private final int dimension; // the number of chards
|
||||
public enum Method {
|
||||
MODULO_HOST_MD5, ROUND_ROBIN;
|
||||
}
|
||||
|
||||
public SolrChardingSelection(Method method, int dimension) {
|
||||
|
||||
public SolrShardingSelection(final Method method, final int dimension) {
|
||||
this.method = method;
|
||||
this.dimension = dimension;
|
||||
this.chardID = new AtomicLong(0);
|
||||
|
@ -57,36 +57,36 @@ public class SolrChardingSelection {
|
|||
private int selectRoundRobin() {
|
||||
return (int) (this.chardID.getAndIncrement() % this.dimension);
|
||||
}
|
||||
|
||||
public int select(SolrInputDocument solrdoc) throws IOException {
|
||||
|
||||
public int select(final SolrInputDocument solrdoc) throws IOException {
|
||||
if (this.method == Method.MODULO_HOST_MD5) {
|
||||
String sku = (String) solrdoc.getField("sku").getValue();
|
||||
final String sku = (String) solrdoc.getField("sku").getValue();
|
||||
return selectURL(sku);
|
||||
}
|
||||
|
||||
|
||||
// finally if no method matches use ROUND_ROBIN
|
||||
return selectRoundRobin();
|
||||
}
|
||||
|
||||
public int selectURL(String sku) throws IOException {
|
||||
|
||||
public int selectURL(final String sku) throws IOException {
|
||||
if (this.method == Method.MODULO_HOST_MD5) {
|
||||
try {
|
||||
URL url = new URL(sku);
|
||||
String host = url.getHost();
|
||||
if (host == null) throw new IOException("charding - bad url, host empty: " + sku);
|
||||
final URL url = new URL(sku);
|
||||
final String host = url.getHost();
|
||||
if (host == null) throw new IOException("sharding - bad url, host empty: " + sku);
|
||||
try {
|
||||
MessageDigest digest = MessageDigest.getInstance("MD5");
|
||||
final MessageDigest digest = MessageDigest.getInstance("MD5");
|
||||
digest.update(host.getBytes(charsetUTF8));
|
||||
byte[] md5 = digest.digest();
|
||||
final byte[] md5 = digest.digest();
|
||||
return (0xff & md5[0]) % this.dimension;
|
||||
} catch (NoSuchAlgorithmException e) {
|
||||
throw new IOException("charding - no md5 available: " + e.getMessage());
|
||||
} catch (final NoSuchAlgorithmException e) {
|
||||
throw new IOException("sharding - no md5 available: " + e.getMessage());
|
||||
}
|
||||
} catch (MalformedURLException e) {
|
||||
throw new IOException("charding - bad url: " + sku);
|
||||
} catch (final MalformedURLException e) {
|
||||
throw new IOException("sharding - bad url: " + sku);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
// finally if no method matches use ROUND_ROBIN
|
||||
return (int) (this.chardID.getAndIncrement() % this.dimension);
|
||||
}
|
|
@ -91,8 +91,8 @@ import net.yacy.cora.protocol.ResponseHeader;
|
|||
import net.yacy.cora.protocol.TimeoutRequest;
|
||||
import net.yacy.cora.protocol.http.HTTPClient;
|
||||
import net.yacy.cora.protocol.http.ProxySettings;
|
||||
import net.yacy.cora.services.federated.solr.SolrChardingConnector;
|
||||
import net.yacy.cora.services.federated.solr.SolrChardingSelection;
|
||||
import net.yacy.cora.services.federated.solr.SolrShardingConnector;
|
||||
import net.yacy.cora.services.federated.solr.SolrShardingSelection;
|
||||
import net.yacy.cora.services.federated.solr.SolrScheme;
|
||||
import net.yacy.cora.services.federated.yacy.CacheStrategy;
|
||||
import net.yacy.document.Condenser;
|
||||
|
@ -599,7 +599,7 @@ public final class Switchboard extends serverSwitch {
|
|||
final String solrurls = getConfig("federated.service.solr.indexing.url", "http://127.0.0.1:8983/solr");
|
||||
final boolean usesolr = getConfigBool("federated.service.solr.indexing.enabled", false) & solrurls.length() > 0;
|
||||
try {
|
||||
this.indexSegments.segment(Segments.Process.LOCALCRAWLING).connectSolr((usesolr) ? new SolrChardingConnector(solrurls, workingScheme, SolrChardingSelection.Method.MODULO_HOST_MD5, 10000) : null);
|
||||
this.indexSegments.segment(Segments.Process.LOCALCRAWLING).connectSolr((usesolr) ? new SolrShardingConnector(solrurls, workingScheme, SolrShardingSelection.Method.MODULO_HOST_MD5, 10000) : null);
|
||||
} catch (final IOException e) {
|
||||
Log.logException(e);
|
||||
this.indexSegments.segment(Segments.Process.LOCALCRAWLING).connectSolr(null);
|
||||
|
|
Loading…
Reference in New Issue
Block a user