Marown/yacy_search_server
1
0
Fork 0
mirror of https://github.com/yacy/yacy_search_server.git synced 2024-09-19 00:01:41 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity

enhanced solr caching:

Browse Source 
- increased cache size which is needed for longer solr commit time
- speed hacks on cache write code
This commit is contained in:
Michael Peter Christen 2012-10-28 20:31:29 +01:00
parent a33e2742cb
commit 799d71bc67
4 changed files with 45 additions and 55 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
htroot/PerformanceMemory_p.java
View File

@ -205,19 +205,19 @@ public class PerformanceMemory_p {
// other caching structures
final MirrorSolrConnector solr = (MirrorSolrConnector) Switchboard.getSwitchboard().index.fulltext().getSolr();
prop.putNum("solrcacheHit.size", solr.nameCacheHitSize());
prop.putNum("solrcacheHit.Hit", solr.cacheHit_Hit);
prop.putNum("solrcacheHit.Miss", solr.cacheHit_Miss);
prop.putNum("solrcacheHit.Insert", solr.cacheHit_Insert);
prop.putNum("solrcacheHit.Hit", solr.hitCache_Hit);
prop.putNum("solrcacheHit.Miss", solr.hitCache_Miss);
prop.putNum("solrcacheHit.Insert", solr.hitCache_Insert);
prop.putNum("solrcacheMiss.size", solr.nameCacheMissSize());
prop.putNum("solrcacheMiss.Hit", solr.cacheMiss_Hit);
prop.putNum("solrcacheMiss.Miss", solr.cacheMiss_Miss);
prop.putNum("solrcacheMiss.Insert", solr.cacheMiss_Insert);
prop.putNum("solrcacheMiss.Hit", solr.missCache_Hit);
prop.putNum("solrcacheMiss.Miss", solr.missCache_Miss);
prop.putNum("solrcacheMiss.Insert", solr.missCache_Insert);
prop.putNum("solrcacheDocument.size", solr.nameCacheDocumentSize());
prop.putNum("solrcacheDocument.Hit", solr.cacheDocument_Hit);
prop.putNum("solrcacheDocument.Miss", solr.cacheDocument_Miss);
prop.putNum("solrcacheDocument.Insert", solr.cacheDocument_Insert);
prop.putNum("solrcacheDocument.Hit", solr.documentCache_Hit);
prop.putNum("solrcacheDocument.Miss", solr.documentCache_Miss);
prop.putNum("solrcacheDocument.Insert", solr.documentCache_Insert);
prop.putNum("namecacheHit.size", Domains.nameCacheHitSize());
prop.putNum("namecacheHit.Hit", Domains.cacheHit_Hit);

76
source/net/yacy/cora/federate/solr/connector/MirrorSolrConnector.java
View File

@ -59,9 +59,9 @@ public class MirrorSolrConnector extends AbstractSolrConnector implements SolrCo
private SolrConnector solr1;
private final ARC<String, Object> hitCache, missCache;
private final ARC<String, SolrDocument> documentCache;
public long cacheHit_Hit = 0, cacheHit_Miss = 0, cacheHit_Insert = 0; // for statistics only; do not write
public long cacheMiss_Hit = 0, cacheMiss_Miss = 0, cacheMiss_Insert = 0; // for statistics only; do not write
public long cacheDocument_Hit = 0, cacheDocument_Miss = 0, cacheDocument_Insert = 0; // for statistics only; do not write
public long hitCache_Hit = 0, hitCache_Miss = 0, hitCache_Insert = 0; // for statistics only; do not write
public long missCache_Hit = 0, missCache_Miss = 0, missCache_Insert = 0; // for statistics only; do not write
public long documentCache_Hit = 0, documentCache_Miss = 0, documentCache_Insert = 0; // for statistics only; do not write
public MirrorSolrConnector(int hitCacheMax, int missCacheMax, int docCacheMax) {
@ -164,7 +164,7 @@ public class MirrorSolrConnector extends AbstractSolrConnector implements SolrCo
public void delete(final String id) throws IOException {
this.hitCache.remove(id);
this.missCache.put(id, EXIST);
cacheMiss_Insert++;
this.missCache_Insert++;
this.documentCache.remove(id);
if (this.solr0 != null) this.solr0.delete(id);
if (this.solr1 != null) this.solr1.delete(id);
@ -180,7 +180,7 @@ public class MirrorSolrConnector extends AbstractSolrConnector implements SolrCo
for (String id: ids) {
this.hitCache.remove(id);
this.missCache.put(id, EXIST);
cacheMiss_Insert++;
this.missCache_Insert++;
this.documentCache.remove(id);
}
if (this.solr0 != null) this.solr0.delete(ids);
@ -203,31 +203,28 @@ public class MirrorSolrConnector extends AbstractSolrConnector implements SolrCo
@Override
public boolean exists(final String id) throws IOException {
if (this.hitCache.containsKey(id)) {
cacheHit_Hit++;
this.hitCache_Hit++;
return true;
}
cacheHit_Miss++;
this.hitCache_Miss++;
if (this.documentCache.containsKey(id)) {
cacheDocument_Hit++;
this.documentCache_Hit++;
return true;
}
cacheDocument_Miss++;
this.documentCache_Miss++;
if (this.missCache.containsKey(id)) {
cacheMiss_Hit++;
this.missCache_Hit++;
return false;
}
cacheMiss_Miss++;
for (SolrConnector solr: new SolrConnector[]{this.solr0, this.solr1}) {
if (solr != null) {
if (solr.exists(id)) {
this.hitCache.put(id, EXIST);
cacheHit_Insert++;
return true;
}
}
this.missCache_Miss++;
if ((solr0 != null && solr0.exists(id)) || (solr1 != null && solr1.exists(id))) {
this.missCache.remove(id);
this.hitCache.put(id, EXIST);
this.hitCache_Insert++;
return true;
}
this.missCache.put(id, EXIST);
cacheMiss_Insert++;
this.missCache_Insert++;
return false;
}
@ -235,30 +232,25 @@ public class MirrorSolrConnector extends AbstractSolrConnector implements SolrCo
public SolrDocument get(String id) throws IOException {
SolrDocument doc = this.documentCache.get(id);
if (doc != null) {
cacheDocument_Hit++;
this.documentCache_Hit++;
return doc;
}
cacheDocument_Miss++;
documentCache_Miss++;
if (this.missCache.containsKey(id)) {
cacheMiss_Hit++;
this.missCache_Hit++;
return null;
}
cacheMiss_Miss++;
for (SolrConnector solr: new SolrConnector[]{this.solr0, this.solr1}) {
if (solr != null) {
doc = solr.get(id);
if (doc != null) {
this.hitCache.put(id, EXIST);
cacheHit_Insert++;
this.documentCache.put(id, doc);
cacheDocument_Insert++;
return doc;
}
}
missCache_Miss++;
if ((solr0 != null && ((doc = solr0.get(id)) != null)) || (solr1 != null && ((doc = solr1.get(id)) != null))) {
this.missCache.remove(id);
this.hitCache.put(id, EXIST);
this.hitCache_Insert++;
this.documentCache.put(id, doc);
this.documentCache_Insert++;
return doc;
}
this.missCache.put(id, EXIST);
cacheMiss_Insert++;
this.missCache_Insert++;
return null;
}
@ -274,11 +266,11 @@ public class MirrorSolrConnector extends AbstractSolrConnector implements SolrCo
if (id == null) return;
this.missCache.remove(id);
this.documentCache.put(id, ClientUtils.toSolrDocument(solrdoc));
cacheDocument_Insert++;
this.documentCache_Insert++;
this.hitCache.put(id, EXIST);
this.hitCache_Insert++;
if (this.solr0 != null) this.solr0.add(solrdoc);
if (this.solr1 != null) this.solr1.add(solrdoc);
this.hitCache.put(id, EXIST);
cacheHit_Insert++;
}
@Override
@ -441,9 +433,9 @@ public class MirrorSolrConnector extends AbstractSolrConnector implements SolrCo
String id = (String) solrdoc.getFieldValue(YaCySchema.id.getSolrFieldName());
if (id != null) {
this.hitCache.put(id, EXIST);
cacheHit_Insert++;
hitCache_Insert++;
this.documentCache.put(id, solrdoc);
cacheDocument_Insert++;
documentCache_Insert++;
}
}
}

4
source/net/yacy/search/Switchboard.java
View File

@ -1512,9 +1512,7 @@ public final class Switchboard extends serverSwitch
// tests if hash occurrs in any database
// if it exists, the name of the database is returned,
// if it not exists, null is returned
if ( this.index.fulltext().exists(hash) ) {
return "loaded";
}
if (this.index.exists(hash)) return "loaded";
return this.crawlQueues.urlExists(hash);
}

2
source/net/yacy/search/index/Fulltext.java
View File

@ -87,7 +87,7 @@ public final class Fulltext implements Iterable<byte[]> {
this.urlIndexFile = null;
this.exportthread = null; // will have a export thread assigned if exporter is running
this.statsDump = null;
this.solr = new MirrorSolrConnector(1000, 1000, 100);
this.solr = new MirrorSolrConnector(10000, 10000, 100);
this.solrScheme = solrScheme;
this.forcedCommitTime = 0;
}