From 799d71bc67efc679c45744a0f36f483b0db6d2b8 Mon Sep 17 00:00:00 2001 From: Michael Peter Christen Date: Sun, 28 Oct 2012 20:31:29 +0100 Subject: [PATCH] enhanced solr caching: - increased cache size which is needed for longer solr commit time - speed hacks on cache write code --- htroot/PerformanceMemory_p.java | 18 ++--- .../solr/connector/MirrorSolrConnector.java | 76 +++++++++---------- source/net/yacy/search/Switchboard.java | 4 +- source/net/yacy/search/index/Fulltext.java | 2 +- 4 files changed, 45 insertions(+), 55 deletions(-) diff --git a/htroot/PerformanceMemory_p.java b/htroot/PerformanceMemory_p.java index 643bf2306..069c5a791 100644 --- a/htroot/PerformanceMemory_p.java +++ b/htroot/PerformanceMemory_p.java @@ -205,19 +205,19 @@ public class PerformanceMemory_p { // other caching structures final MirrorSolrConnector solr = (MirrorSolrConnector) Switchboard.getSwitchboard().index.fulltext().getSolr(); prop.putNum("solrcacheHit.size", solr.nameCacheHitSize()); - prop.putNum("solrcacheHit.Hit", solr.cacheHit_Hit); - prop.putNum("solrcacheHit.Miss", solr.cacheHit_Miss); - prop.putNum("solrcacheHit.Insert", solr.cacheHit_Insert); + prop.putNum("solrcacheHit.Hit", solr.hitCache_Hit); + prop.putNum("solrcacheHit.Miss", solr.hitCache_Miss); + prop.putNum("solrcacheHit.Insert", solr.hitCache_Insert); prop.putNum("solrcacheMiss.size", solr.nameCacheMissSize()); - prop.putNum("solrcacheMiss.Hit", solr.cacheMiss_Hit); - prop.putNum("solrcacheMiss.Miss", solr.cacheMiss_Miss); - prop.putNum("solrcacheMiss.Insert", solr.cacheMiss_Insert); + prop.putNum("solrcacheMiss.Hit", solr.missCache_Hit); + prop.putNum("solrcacheMiss.Miss", solr.missCache_Miss); + prop.putNum("solrcacheMiss.Insert", solr.missCache_Insert); prop.putNum("solrcacheDocument.size", solr.nameCacheDocumentSize()); - prop.putNum("solrcacheDocument.Hit", solr.cacheDocument_Hit); - prop.putNum("solrcacheDocument.Miss", solr.cacheDocument_Miss); - prop.putNum("solrcacheDocument.Insert", solr.cacheDocument_Insert); + prop.putNum("solrcacheDocument.Hit", solr.documentCache_Hit); + prop.putNum("solrcacheDocument.Miss", solr.documentCache_Miss); + prop.putNum("solrcacheDocument.Insert", solr.documentCache_Insert); prop.putNum("namecacheHit.size", Domains.nameCacheHitSize()); prop.putNum("namecacheHit.Hit", Domains.cacheHit_Hit); diff --git a/source/net/yacy/cora/federate/solr/connector/MirrorSolrConnector.java b/source/net/yacy/cora/federate/solr/connector/MirrorSolrConnector.java index cece1360e..e93f3e28d 100644 --- a/source/net/yacy/cora/federate/solr/connector/MirrorSolrConnector.java +++ b/source/net/yacy/cora/federate/solr/connector/MirrorSolrConnector.java @@ -59,9 +59,9 @@ public class MirrorSolrConnector extends AbstractSolrConnector implements SolrCo private SolrConnector solr1; private final ARC hitCache, missCache; private final ARC documentCache; - public long cacheHit_Hit = 0, cacheHit_Miss = 0, cacheHit_Insert = 0; // for statistics only; do not write - public long cacheMiss_Hit = 0, cacheMiss_Miss = 0, cacheMiss_Insert = 0; // for statistics only; do not write - public long cacheDocument_Hit = 0, cacheDocument_Miss = 0, cacheDocument_Insert = 0; // for statistics only; do not write + public long hitCache_Hit = 0, hitCache_Miss = 0, hitCache_Insert = 0; // for statistics only; do not write + public long missCache_Hit = 0, missCache_Miss = 0, missCache_Insert = 0; // for statistics only; do not write + public long documentCache_Hit = 0, documentCache_Miss = 0, documentCache_Insert = 0; // for statistics only; do not write public MirrorSolrConnector(int hitCacheMax, int missCacheMax, int docCacheMax) { @@ -164,7 +164,7 @@ public class MirrorSolrConnector extends AbstractSolrConnector implements SolrCo public void delete(final String id) throws IOException { this.hitCache.remove(id); this.missCache.put(id, EXIST); - cacheMiss_Insert++; + this.missCache_Insert++; this.documentCache.remove(id); if (this.solr0 != null) this.solr0.delete(id); if (this.solr1 != null) this.solr1.delete(id); @@ -180,7 +180,7 @@ public class MirrorSolrConnector extends AbstractSolrConnector implements SolrCo for (String id: ids) { this.hitCache.remove(id); this.missCache.put(id, EXIST); - cacheMiss_Insert++; + this.missCache_Insert++; this.documentCache.remove(id); } if (this.solr0 != null) this.solr0.delete(ids); @@ -203,31 +203,28 @@ public class MirrorSolrConnector extends AbstractSolrConnector implements SolrCo @Override public boolean exists(final String id) throws IOException { if (this.hitCache.containsKey(id)) { - cacheHit_Hit++; + this.hitCache_Hit++; return true; } - cacheHit_Miss++; + this.hitCache_Miss++; if (this.documentCache.containsKey(id)) { - cacheDocument_Hit++; + this.documentCache_Hit++; return true; } - cacheDocument_Miss++; + this.documentCache_Miss++; if (this.missCache.containsKey(id)) { - cacheMiss_Hit++; + this.missCache_Hit++; return false; } - cacheMiss_Miss++; - for (SolrConnector solr: new SolrConnector[]{this.solr0, this.solr1}) { - if (solr != null) { - if (solr.exists(id)) { - this.hitCache.put(id, EXIST); - cacheHit_Insert++; - return true; - } - } + this.missCache_Miss++; + if ((solr0 != null && solr0.exists(id)) || (solr1 != null && solr1.exists(id))) { + this.missCache.remove(id); + this.hitCache.put(id, EXIST); + this.hitCache_Insert++; + return true; } this.missCache.put(id, EXIST); - cacheMiss_Insert++; + this.missCache_Insert++; return false; } @@ -235,30 +232,25 @@ public class MirrorSolrConnector extends AbstractSolrConnector implements SolrCo public SolrDocument get(String id) throws IOException { SolrDocument doc = this.documentCache.get(id); if (doc != null) { - cacheDocument_Hit++; + this.documentCache_Hit++; return doc; } - cacheDocument_Miss++; + documentCache_Miss++; if (this.missCache.containsKey(id)) { - cacheMiss_Hit++; + this.missCache_Hit++; return null; } - cacheMiss_Miss++; - - for (SolrConnector solr: new SolrConnector[]{this.solr0, this.solr1}) { - if (solr != null) { - doc = solr.get(id); - if (doc != null) { - this.hitCache.put(id, EXIST); - cacheHit_Insert++; - this.documentCache.put(id, doc); - cacheDocument_Insert++; - return doc; - } - } + missCache_Miss++; + if ((solr0 != null && ((doc = solr0.get(id)) != null)) || (solr1 != null && ((doc = solr1.get(id)) != null))) { + this.missCache.remove(id); + this.hitCache.put(id, EXIST); + this.hitCache_Insert++; + this.documentCache.put(id, doc); + this.documentCache_Insert++; + return doc; } this.missCache.put(id, EXIST); - cacheMiss_Insert++; + this.missCache_Insert++; return null; } @@ -274,11 +266,11 @@ public class MirrorSolrConnector extends AbstractSolrConnector implements SolrCo if (id == null) return; this.missCache.remove(id); this.documentCache.put(id, ClientUtils.toSolrDocument(solrdoc)); - cacheDocument_Insert++; + this.documentCache_Insert++; + this.hitCache.put(id, EXIST); + this.hitCache_Insert++; if (this.solr0 != null) this.solr0.add(solrdoc); if (this.solr1 != null) this.solr1.add(solrdoc); - this.hitCache.put(id, EXIST); - cacheHit_Insert++; } @Override @@ -441,9 +433,9 @@ public class MirrorSolrConnector extends AbstractSolrConnector implements SolrCo String id = (String) solrdoc.getFieldValue(YaCySchema.id.getSolrFieldName()); if (id != null) { this.hitCache.put(id, EXIST); - cacheHit_Insert++; + hitCache_Insert++; this.documentCache.put(id, solrdoc); - cacheDocument_Insert++; + documentCache_Insert++; } } } diff --git a/source/net/yacy/search/Switchboard.java b/source/net/yacy/search/Switchboard.java index 6be9f24d2..67fc3d5af 100644 --- a/source/net/yacy/search/Switchboard.java +++ b/source/net/yacy/search/Switchboard.java @@ -1512,9 +1512,7 @@ public final class Switchboard extends serverSwitch // tests if hash occurrs in any database // if it exists, the name of the database is returned, // if it not exists, null is returned - if ( this.index.fulltext().exists(hash) ) { - return "loaded"; - } + if (this.index.exists(hash)) return "loaded"; return this.crawlQueues.urlExists(hash); } diff --git a/source/net/yacy/search/index/Fulltext.java b/source/net/yacy/search/index/Fulltext.java index 87f8d8c1e..980c7cddd 100644 --- a/source/net/yacy/search/index/Fulltext.java +++ b/source/net/yacy/search/index/Fulltext.java @@ -87,7 +87,7 @@ public final class Fulltext implements Iterable { this.urlIndexFile = null; this.exportthread = null; // will have a export thread assigned if exporter is running this.statsDump = null; - this.solr = new MirrorSolrConnector(1000, 1000, 100); + this.solr = new MirrorSolrConnector(10000, 10000, 100); this.solrScheme = solrScheme; this.forcedCommitTime = 0; }