Marown/yacy_search_server
1
0
Fork 0
mirror of https://github.com/yacy/yacy_search_server.git synced 2024-09-19 00:01:41 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge branch 'master' of git@gitorious.org:yacy/rc1.git

Browse Source
This commit is contained in:
Marc Nause 2013-02-13 19:31:12 +01:00
commit efb6cf7d21
14 changed files with 15 additions and 129 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
defaults/yacy.init
View File

@ -1056,7 +1056,6 @@ color_searchurlhover = #008000
# - to check whats in solr after indexing, open http://localhost:8983/solr/admin/
federated.service.solr.indexing.enabled = false
federated.service.solr.indexing.url = http://127.0.0.1:8983/solr
federated.service.solr.indexing.commitWithinMs = -1
federated.service.solr.indexing.sharding = MODULO_HOST_MD5
federated.service.solr.indexing.schemefile = solr.keys.default.list
# the lazy attribute causes that fields containing "" or 0 are not added and not written

6
htroot/IndexFederated_p.java
View File

@ -79,7 +79,6 @@ public class IndexFederated_p {
final boolean previous_core_fulltext = sb.index.fulltext().connectedLocalSolr() && env.getConfigBool(SwitchboardConstants.CORE_SERVICE_FULLTEXT, false);
env.setConfig(SwitchboardConstants.CORE_SERVICE_FULLTEXT, post_core_fulltext);
final int commitWithinMs = post.getInt("solr.indexing.commitWithinMs", env.getConfigInt(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_COMMITWITHINMS, -1));
if (previous_core_fulltext && !post_core_fulltext) {
// switch off
sb.index.fulltext().disconnectLocalSolr();
@ -88,7 +87,7 @@ public class IndexFederated_p {
if (!previous_core_fulltext && post_core_fulltext) {
// switch on
sb.index.connectUrlDb(sb.useTailCache, sb.exceed134217727);
try { sb.index.fulltext().connectLocalSolr(commitWithinMs); } catch (IOException e) { Log.logException(e); }
try { sb.index.fulltext().connectLocalSolr(); } catch (IOException e) { Log.logException(e); }
}
// solr
@ -114,7 +113,6 @@ public class IndexFederated_p {
}
solrurls = s.toString().trim();
env.setConfig(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_URL, solrurls);
env.setConfig(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_COMMITWITHINMS, commitWithinMs);
env.setConfig(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_LAZY, lazy);
env.setConfig(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_SHARDING, post.get("solr.indexing.sharding", env.getConfig(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_SHARDING, "modulo-host-md5")));
final String schemename = post.get("solr.indexing.schemefile", env.getConfig(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_SCHEMEFILE, "solr.keys.default.list"));
@ -136,7 +134,6 @@ public class IndexFederated_p {
try {
if (usesolr) {
SolrConnector solr = new ShardSolrConnector(solrurls, ShardSelection.Method.MODULO_HOST_MD5, 10000, true);
if (commitWithinMs >= 0) solr.setCommitWithinMs(commitWithinMs);
sb.index.fulltext().connectRemoteSolr(solr);
} else {
sb.index.fulltext().disconnectRemoteSolr();
@ -225,7 +222,6 @@ public class IndexFederated_p {
prop.put(SwitchboardConstants.CORE_SERVICE_CITATION + ".checked", env.getConfigBool(SwitchboardConstants.CORE_SERVICE_CITATION, false) ? 1 : 0);
prop.put("solr.indexing.solrremote.checked", env.getConfigBool(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_ENABLED, false) ? 1 : 0);
prop.put("solr.indexing.url", env.getConfig(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_URL, "http://127.0.0.1:8983/solr").replace(",", "\n"));
prop.put("solr.indexing.commitWithinMs", env.getConfigInt(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_COMMITWITHINMS, 180000));
prop.put("solr.indexing.lazy.checked", env.getConfigBool(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_LAZY, true) ? 1 : 0);
prop.put("solr.indexing.sharding", env.getConfig(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_SHARDING, "modulo-host-md5"));
prop.put("solr.indexing.schemefile", schemename);

3
htroot/yacysearch.html
View File

@ -69,10 +69,9 @@
<div id="api">
<script type="text/javascript">
//<![CDATA[
document.write("\<a href=\"yacysearch.rss?" + window.location.search.substring(1) + "\"\>")
document.write("\<a href=\"yacysearch.rss?" + window.location.search.substring(1) + "\"\><img src=\"env/grafics/api.png\" width=\"60\" height=\"40\" alt=\"API\" /></a>")
//]]>
</script>
<img src="env/grafics/api.png" width="60" height="40" alt="API" /></a>
<span>This search result can also be retrieved as RSS/<a href="http://www.opensearch.org">opensearch</a> output.
The query format is similar to <a href="http://www.loc.gov/standards/sru/">SRU</a>.
Click the API icon to see an example call to the search rss API.

17
source/net/yacy/cora/federate/solr/connector/MirrorSolrConnector.java
View File

@ -148,23 +148,6 @@ public class MirrorSolrConnector extends AbstractSolrConnector implements SolrCo
this.documentCache.clear();
}
@Override
public int getCommitWithinMs() {
if (this.solr0 != null) this.solr0.getCommitWithinMs();
if (this.solr1 != null) this.solr1.getCommitWithinMs();
return -1;
}
/**
* set the solr autocommit delay
* @param c the maximum waiting time after a solr command until it is transported to the server
*/
@Override
public void setCommitWithinMs(int c) {
if (this.solr0 != null) this.solr0.setCommitWithinMs(c);
if (this.solr1 != null) this.solr1.setCommitWithinMs(c);
}
@Override
public void commit(boolean softCommit) {
if (this.solr0 != null) this.solr0.commit(softCommit);

19
source/net/yacy/cora/federate/solr/connector/MultipleSolrConnector.java
View File

@ -36,13 +36,11 @@ public class MultipleSolrConnector extends AbstractSolrConnector implements Solr
private final ArrayBlockingQueue<SolrInputDocument> queue;
private final AddWorker[] worker;
private final SolrConnector solr;
private int commitWithinMs;
public MultipleSolrConnector(final String url, final int connections) throws IOException {
this.solr = new RemoteSolrConnector(url);
this.queue = new ArrayBlockingQueue<SolrInputDocument>(1000);
this.worker = new AddWorker[connections];
this.commitWithinMs = -1;
for (int i = 0; i < connections; i++) {
this.worker[i] = new AddWorker(url);
this.worker[i].start();
@ -53,7 +51,6 @@ public class MultipleSolrConnector extends AbstractSolrConnector implements Solr
private final SolrConnector solr;
public AddWorker(final String url) throws IOException {
this.solr = new RemoteSolrConnector(url);
if (MultipleSolrConnector.this.commitWithinMs >= 0 ) this.solr.setCommitWithinMs(MultipleSolrConnector.this.commitWithinMs);
}
@Override
public void run() {
@ -75,22 +72,6 @@ public class MultipleSolrConnector extends AbstractSolrConnector implements Solr
}
}
@Override
public int getCommitWithinMs() {
return this.commitWithinMs;
}
/**
* set the solr autocommit delay
* @param c the maximum waiting time after a solr command until it is transported to the server
*/
@Override
public void setCommitWithinMs(final int c) {
this.commitWithinMs = c;
this.solr.setCommitWithinMs(c);
for (AddWorker w: this.worker) w.solr.setCommitWithinMs(c);
}
@Override
public void commit(boolean softCommit) {
this.solr.commit(softCommit);

14
source/net/yacy/cora/federate/solr/connector/RetrySolrConnector.java
View File

@ -38,20 +38,6 @@ public class RetrySolrConnector extends AbstractSolrConnector implements SolrCon
this.retryMaxTime = retryMaxTime;
}
@Override
public int getCommitWithinMs() {
return this.solrConnector.getCommitWithinMs();
}
/**
* set the solr autocommit delay
* @param c the maximum waiting time after a solr command until it is transported to the server
*/
@Override
public void setCommitWithinMs(int c) {
this.solrConnector.setCommitWithinMs(c);
}
@Override
public void commit(boolean softCommit) {
this.solrConnector.commit(softCommit);

14
source/net/yacy/cora/federate/solr/connector/ShardSolrConnector.java
View File

@ -58,20 +58,6 @@ public class ShardSolrConnector extends AbstractSolrConnector implements SolrCon
this.sharding = new ShardSelection(method, this.urls.length);
}
@Override
public int getCommitWithinMs() {
return this.connectors.get(0).getCommitWithinMs();
}
/**
* set the solr autocommit delay
* @param c the maximum waiting time after a solr command until it is transported to the server
*/
@Override
public void setCommitWithinMs(int c) {
for (final SolrConnector connector: this.connectors) connector.setCommitWithinMs(c);
}
@Override
public void commit(boolean softCommit) {
for (final SolrConnector connector: this.connectors) connector.commit(softCommit);

12
source/net/yacy/cora/federate/solr/connector/SolrConnector.java
View File

@ -36,18 +36,6 @@ import org.apache.solr.common.params.ModifiableSolrParams;
public interface SolrConnector extends Iterable<String> /* Iterable of document IDs */ {
/**
* get the solr autocommit delay
* @return the maximum waiting time after a solr command until it is transported to the server
*/
public int getCommitWithinMs();
/**
* set the solr autocommit delay
* @param c the maximum waiting time after a solr command until it is transported to the server
*/
public void setCommitWithinMs(int c);
/**
* force a commit
*/

33
source/net/yacy/cora/federate/solr/connector/SolrServerConnector.java
View File

@ -56,11 +56,9 @@ public abstract class SolrServerConnector extends AbstractSolrConnector implemen
protected final static Logger log = Logger.getLogger(SolrServerConnector.class);
protected SolrServer server;
protected int commitWithinMs; // max time (in ms) before a commit will happen
protected SolrServerConnector() {
this.server = null;
this.commitWithinMs = 180000;
}
protected void init(SolrServer server) {
@ -71,25 +69,6 @@ public abstract class SolrServerConnector extends AbstractSolrConnector implemen
return this.server;
}
/**
* get the solr autocommit delay
* @return the maximum waiting time after a solr command until it is transported to the server
*/
@Override
public int getCommitWithinMs() {
return this.commitWithinMs;
}
/**
* set the solr autocommit delay
* when doing continuous inserts, don't set this value because it would cause continuous commits
* @param c the maximum waiting time after a solr command until it is transported to the server
*/
@Override
public void setCommitWithinMs(int c) {
this.commitWithinMs = c;
}
@Override
public synchronized void commit(final boolean softCommit) {
//if (this.server instanceof HttpSolrServer) ((HttpSolrServer) this.server).getHttpClient().getConnectionManager().closeExpiredConnections();
@ -209,7 +188,7 @@ public abstract class SolrServerConnector extends AbstractSolrConnector implemen
public void delete(final String id) throws IOException {
try {
synchronized (this.server) {
this.server.deleteById(id, this.commitWithinMs);
this.server.deleteById(id, -1);
}
} catch (final Throwable e) {
throw new IOException(e);
@ -220,7 +199,7 @@ public abstract class SolrServerConnector extends AbstractSolrConnector implemen
public void delete(final List<String> ids) throws IOException {
try {
synchronized (this.server) {
this.server.deleteById(ids, this.commitWithinMs);
this.server.deleteById(ids, -1);
}
} catch (final Throwable e) {
throw new IOException(e);
@ -236,7 +215,7 @@ public abstract class SolrServerConnector extends AbstractSolrConnector implemen
public void deleteByQuery(final String querystring) throws IOException {
try {
synchronized (this.server) {
this.server.deleteByQuery(querystring, this.commitWithinMs);
this.server.deleteByQuery(querystring, -1);
}
} catch (final Throwable e) {
throw new IOException(e);
@ -249,7 +228,7 @@ public abstract class SolrServerConnector extends AbstractSolrConnector implemen
up.setParam("literal.id", solrId);
up.setParam("uprefix", "attr_");
up.setParam("fmap.content", "attr_content");
up.setCommitWithin(this.commitWithinMs);
up.setCommitWithin(-1);
//up.setAction(AbstractUpdateRequest.ACTION.COMMIT, true, true);
try {
synchronized (this.server) {
@ -266,7 +245,7 @@ public abstract class SolrServerConnector extends AbstractSolrConnector implemen
try {
if (solrdoc.containsKey("_version_")) solrdoc.setField("_version_",0L); // prevent Solr "version conflict"
synchronized (this.server) {
this.server.add(solrdoc, this.commitWithinMs);
this.server.add(solrdoc, -1);
}
} catch (Throwable e) {
// catches "version conflict for": try this again and delete the document in advance
@ -275,7 +254,7 @@ public abstract class SolrServerConnector extends AbstractSolrConnector implemen
} catch (SolrServerException e1) {}
try {
synchronized (this.server) {
this.server.add(solrdoc, this.commitWithinMs);
this.server.add(solrdoc, -1);
}
} catch (Throwable ee) {
log.warn(e.getMessage() + " DOC=" + solrdoc.toString());

1
source/net/yacy/data/URLLicense.java
View File

@ -42,6 +42,7 @@ public class URLLicense {
private static final Map<String, String> permissions = Collections.synchronizedMap(new SizeLimitedMap<String, String>(maxQueue));
public static String aquireLicense(final DigestURI url) {
if (url == null) return "";
// generate license key
String license = ASCII.String(url.hash());
// store reference to url with license key

8
source/net/yacy/search/Switchboard.java
View File

@ -428,12 +428,11 @@ public final class Switchboard extends serverSwitch {
ReferenceContainer.maxReferences = getConfigInt("index.maxReferences", 0);
final File segmentsPath = new File(new File(indexPath, networkName), "SEGMENTS");
this.index = new Segment(this.log, new File(segmentsPath, "default"), solrScheme);
final int connectWithinMs = this.getConfigInt(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_COMMITWITHINMS, -1);
if (this.getConfigBool(SwitchboardConstants.CORE_SERVICE_RWI, true)) this.index.connectRWI(wordCacheMaxCount, fileSizeMax);
if (this.getConfigBool(SwitchboardConstants.CORE_SERVICE_CITATION, true)) this.index.connectCitation(wordCacheMaxCount, fileSizeMax);
if (this.getConfigBool(SwitchboardConstants.CORE_SERVICE_FULLTEXT, true)) {
this.index.connectUrlDb(this.useTailCache, this.exceed134217727);
this.index.fulltext().connectLocalSolr(connectWithinMs);
this.index.fulltext().connectLocalSolr();
}
// set up the solr interface
@ -446,7 +445,6 @@ public final class Switchboard extends serverSwitch {
solrurls,
ShardSelection.Method.MODULO_HOST_MD5,
10000, true);
solr.setCommitWithinMs(connectWithinMs);
this.index.fulltext().connectRemoteSolr(solr);
} catch ( final IOException e ) {
Log.logException(e);
@ -1282,11 +1280,10 @@ public final class Switchboard extends serverSwitch {
this.useTailCache,
this.exceed134217727);
this.index = new Segment(this.log, new File(new File(new File(indexPrimaryPath, networkName), "SEGMENTS"), "default"), solrScheme);
final int connectWithinMs = this.getConfigInt(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_COMMITWITHINMS, -1);
if (this.getConfigBool(SwitchboardConstants.CORE_SERVICE_RWI, true)) this.index.connectRWI(wordCacheMaxCount, fileSizeMax);
if (this.getConfigBool(SwitchboardConstants.CORE_SERVICE_CITATION, true)) this.index.connectCitation(wordCacheMaxCount, fileSizeMax);
if (this.getConfigBool(SwitchboardConstants.CORE_SERVICE_FULLTEXT, true)) {
this.index.fulltext().connectLocalSolr(connectWithinMs);
this.index.fulltext().connectLocalSolr();
this.index.connectUrlDb(this.useTailCache, this.exceed134217727);
}
@ -1300,7 +1297,6 @@ public final class Switchboard extends serverSwitch {
solrurls,
ShardSelection.Method.MODULO_HOST_MD5,
10000, true);
if (connectWithinMs >= 0) solr.setCommitWithinMs(connectWithinMs);
this.index.fulltext().connectRemoteSolr(solr);
} catch ( final IOException e ) {
Log.logException(e);

1
source/net/yacy/search/SwitchboardConstants.java
View File

@ -294,7 +294,6 @@ public final class SwitchboardConstants {
public static final String FEDERATED_SERVICE_SOLR_INDEXING_ENABLED = "federated.service.solr.indexing.enabled";
public static final String FEDERATED_SERVICE_SOLR_INDEXING_URL = "federated.service.solr.indexing.url";
public static final String FEDERATED_SERVICE_SOLR_INDEXING_COMMITWITHINMS = "federated.service.solr.indexing.commitWithinMs";
public static final String FEDERATED_SERVICE_SOLR_INDEXING_SHARDING = "federated.service.solr.indexing.sharding";
public static final String FEDERATED_SERVICE_SOLR_INDEXING_SCHEMEFILE = "federated.service.solr.indexing.schemefile";
public static final String FEDERATED_SERVICE_SOLR_INDEXING_LAZY = "federated.service.solr.indexing.lazy";

2
source/net/yacy/search/index/DocumentIndex.java
View File

@ -70,7 +70,7 @@ public class DocumentIndex extends Segment {
false, // useTailCache
false // exceed134217727
);
super.fulltext().connectLocalSolr(-1);
super.fulltext().connectLocalSolr();
final int cores = Runtime.getRuntime().availableProcessors() + 1;
this.callback = callback;
this.queue = new LinkedBlockingQueue<DigestURI>(cores * 300);

13
source/net/yacy/search/index/Fulltext.java
View File

@ -142,7 +142,7 @@ public final class Fulltext {
return this.solr.isConnected0();
}
public void connectLocalSolr(final int commitWithin) throws IOException {
public void connectLocalSolr() throws IOException {
File baseLocation = this.location;
if (baseLocation.getName().equals("default")) baseLocation = baseLocation.getParentFile();
File solrLocation = new File(baseLocation, SOLR_PATH);
@ -152,7 +152,6 @@ public final class Fulltext {
if (oldLocation.exists()) oldLocation.renameTo(solrLocation);
}
EmbeddedSolrConnector esc = new EmbeddedSolrConnector(solrLocation, new File(new File(Switchboard.getSwitchboard().appPath, "defaults"), "solr"));
if (commitWithin >= 0) esc.setCommitWithinMs(commitWithin);
Version luceneVersion = esc.getConfig().getLuceneVersion("luceneMatchVersion");
String lvn = luceneVersion.name();
Log.logInfo("Fulltext", "using lucene version " + lvn);
@ -230,10 +229,6 @@ public final class Fulltext {
}
this.solr.close();
}
public int getCommitWithinMs() {
return this.solr.getCommitWithinMs();
}
public void commit(boolean softCommit) {
this.solr.commit(softCommit);
@ -571,7 +566,6 @@ public final class Fulltext {
*/
public File dumpSolr() {
EmbeddedSolrConnector esc = (EmbeddedSolrConnector) this.solr.getSolr0();
int commitWithin = esc.getCommitWithinMs();
File storagePath = esc.getStoragePath();
File zipOut = new File(storagePath.toString() + "_" + GenericFormatter.SHORT_DAY_FORMATTER.format() + ".zip");
synchronized (this.solr) {
@ -582,7 +576,7 @@ public final class Fulltext {
Log.logException(e);
} finally {
try {
this.connectLocalSolr(commitWithin);
this.connectLocalSolr();
} catch (IOException e) {
Log.logException(e);
}
@ -597,7 +591,6 @@ public final class Fulltext {
*/
public void restoreSolr(File solrDumpZipFile) {
EmbeddedSolrConnector esc = (EmbeddedSolrConnector) this.solr.getSolr0();
int commitWithin = esc.getCommitWithinMs();
File storagePath = esc.getStoragePath();
synchronized (this.solr) {
this.disconnectLocalSolr();
@ -607,7 +600,7 @@ public final class Fulltext {
Log.logException(e);
} finally {
try {
this.connectLocalSolr(commitWithin);
this.connectLocalSolr();
} catch (IOException e) {
Log.logException(e);
}