mirror of
https://github.com/yacy/yacy_search_server.git
synced 2024-09-19 00:01:41 +02:00
clear solr caches in case that an exception occurrs. The reason behind
this hack is the occurrence of Exceptions like: W 2014/02/11 18:51:33 ConcurrentLog GC overhead limit exceeded java.io.IOException: GC overhead limit exceeded at net.yacy.cora.federate.solr.connector.AbstractSolrConnector.getDocumentById(AbstractSolrConnector.java:334) at net.yacy.cora.federate.solr.connector.MirrorSolrConnector.getDocumentById(MirrorSolrConnector.java:173) at net.yacy.cora.federate.solr.connector.ConcurrentUpdateSolrConnector.getDocumentById(ConcurrentUpdateSolrConnector.java:415) at net.yacy.search.index.Fulltext.getMetadata(Fulltext.java:331) at net.yacy.search.index.Fulltext.getMetadata(Fulltext.java:317) at net.yacy.search.query.SearchEvent.pullOneRWI(SearchEvent.java:1024) at net.yacy.search.query.SearchEvent.pullOneFilteredFromRWI(SearchEvent.java:1047) at net.yacy.search.query.SearchEvent$3.run(SearchEvent.java:1263) Caused by: java.lang.OutOfMemoryError: GC overhead limit exceeded at java.util.Arrays.copyOfRange(Arrays.java:3077) at java.lang.StringCoding.decode(StringCoding.java:196) at java.lang.String.<init>(String.java:491) at java.lang.String.<init>(String.java:547) at org.apache.lucene.codecs.compressing.CompressingStoredFieldsReader.readField(CompressingStoredFieldsReader.java:187) at org.apache.lucene.codecs.compressing.CompressingStoredFieldsReader.visitDocument(CompressingStoredFieldsReader.java:351) at org.apache.lucene.index.SegmentReader.document(SegmentReader.java:276) at org.apache.lucene.index.BaseCompositeReader.document(BaseCompositeReader.java:110) at org.apache.lucene.index.IndexReader.document(IndexReader.java:436) at org.apache.solr.search.SolrIndexSearcher.doc(SolrIndexSearcher.java:657) at net.yacy.cora.federate.solr.connector.EmbeddedSolrConnector.SolrQueryResponse2SolrDocumentList(EmbeddedSolrConnector.java:230) at net.yacy.cora.federate.solr.connector.EmbeddedSolrConnector.getDocumentListByParams(EmbeddedSolrConnector.java:320) at net.yacy.cora.federate.solr.connector.AbstractSolrConnector.getDocumentById(AbstractSolrConnector.java:330) ... 7 more This problem was analysed with the Eclipse Memory Analyser after a heap dump, where the following problem was reported as the main Problem Suspect: One instance of "org.apache.solr.util.ConcurrentLRUCache" loaded by "sun.misc.Launcher$AppClassLoader @ 0x42e940a0" occupies 902.898.256 (61,80%) bytes. The memory is accumulated in one instance of "java.util.concurrent.ConcurrentHashMap$Segment[]" loaded by "<system class loader>". This memory is part of the result cache of Solr. Flushing this cache appears the most appropriate solution to that problem.
This commit is contained in:
parent
bc28247089
commit
14764632b5
|
@ -331,6 +331,7 @@ public abstract class AbstractSolrConnector implements SolrConnector {
|
|||
if (docs == null || docs.isEmpty()) return null;
|
||||
return docs.get(0);
|
||||
} catch (final Throwable e) {
|
||||
clearCaches(); // we clear the in case that this is caused by OOM
|
||||
throw new IOException(e.getMessage(), e);
|
||||
}
|
||||
}
|
||||
|
|
|
@ -77,6 +77,7 @@ public abstract class SolrServerConnector extends AbstractSolrConnector implemen
|
|||
this.server.commit(true, true, softCommit);
|
||||
//if (this.server instanceof HttpSolrServer) ((HttpSolrServer) this.server).shutdown();
|
||||
} catch (final Throwable e) {
|
||||
clearCaches(); // prevent further OOM if this was caused by OOM
|
||||
//Log.logException(e);
|
||||
}
|
||||
}
|
||||
|
@ -93,6 +94,7 @@ public abstract class SolrServerConnector extends AbstractSolrConnector implemen
|
|||
try {
|
||||
this.server.optimize(true, true, maxSegments);
|
||||
} catch (final Throwable e) {
|
||||
clearCaches(); // prevent further OOM if this was caused by OOM
|
||||
ConcurrentLog.logException(e);
|
||||
}
|
||||
}
|
||||
|
@ -112,6 +114,7 @@ public abstract class SolrServerConnector extends AbstractSolrConnector implemen
|
|||
if (segmentCount == null) return 1;
|
||||
return segmentCount.intValue();
|
||||
} catch (final Throwable e) {
|
||||
clearCaches(); // prevent further OOM if this was caused by OOM
|
||||
log.warn(e);
|
||||
return 0;
|
||||
}
|
||||
|
@ -139,6 +142,7 @@ public abstract class SolrServerConnector extends AbstractSolrConnector implemen
|
|||
if (numDocs == null) return 0;
|
||||
return numDocs.longValue();
|
||||
} catch (final Throwable e) {
|
||||
clearCaches(); // prevent further OOM if this was caused by OOM
|
||||
log.warn(e);
|
||||
return 0;
|
||||
}
|
||||
|
@ -156,6 +160,7 @@ public abstract class SolrServerConnector extends AbstractSolrConnector implemen
|
|||
this.server.deleteByQuery(AbstractSolrConnector.CATCHALL_TERM);
|
||||
this.server.commit(true, true, false);
|
||||
} catch (final Throwable e) {
|
||||
clearCaches(); // prevent further OOM if this was caused by OOM
|
||||
throw new IOException(e);
|
||||
}
|
||||
}
|
||||
|
@ -168,6 +173,7 @@ public abstract class SolrServerConnector extends AbstractSolrConnector implemen
|
|||
try {
|
||||
this.server.deleteById(id, -1);
|
||||
} catch (final Throwable e) {
|
||||
clearCaches(); // prevent further OOM if this was caused by OOM
|
||||
throw new IOException(e);
|
||||
}
|
||||
}
|
||||
|
@ -182,6 +188,7 @@ public abstract class SolrServerConnector extends AbstractSolrConnector implemen
|
|||
try {
|
||||
this.server.deleteById(l, -1);
|
||||
} catch (final Throwable e) {
|
||||
clearCaches(); // prevent further OOM if this was caused by OOM
|
||||
throw new IOException(e);
|
||||
}
|
||||
}
|
||||
|
@ -199,6 +206,7 @@ public abstract class SolrServerConnector extends AbstractSolrConnector implemen
|
|||
try {
|
||||
this.server.deleteByQuery(querystring, -1);
|
||||
} catch (final Throwable e) {
|
||||
clearCaches(); // prevent further OOM if this was caused by OOM
|
||||
throw new IOException(e);
|
||||
}
|
||||
}
|
||||
|
@ -215,6 +223,7 @@ public abstract class SolrServerConnector extends AbstractSolrConnector implemen
|
|||
try {
|
||||
this.server.request(up);
|
||||
} catch (final Throwable e) {
|
||||
clearCaches(); // prevent further OOM if this was caused by OOM
|
||||
throw new IOException(e);
|
||||
}
|
||||
}
|
||||
|
@ -227,6 +236,7 @@ public abstract class SolrServerConnector extends AbstractSolrConnector implemen
|
|||
if (solrdoc.containsKey("_version_")) solrdoc.setField("_version_",0L); // prevent Solr "version conflict"
|
||||
this.server.add(solrdoc, -1);
|
||||
} catch (final Throwable e) {
|
||||
clearCaches(); // prevent further OOM if this was caused by OOM
|
||||
ConcurrentLog.logException(e);
|
||||
// catches "version conflict for": try this again and delete the document in advance
|
||||
try {
|
||||
|
@ -265,6 +275,7 @@ public abstract class SolrServerConnector extends AbstractSolrConnector implemen
|
|||
}
|
||||
this.server.add(solrdocs, -1);
|
||||
} catch (final Throwable e) {
|
||||
clearCaches(); // prevent further OOM if this was caused by OOM
|
||||
ConcurrentLog.logException(e);
|
||||
// catches "version conflict for": try this again and delete the document in advance
|
||||
List<String> ids = new ArrayList<String>();
|
||||
|
@ -314,8 +325,10 @@ public abstract class SolrServerConnector extends AbstractSolrConnector implemen
|
|||
if (rsp != null) if (log.isFine()) log.fine(rsp.getResults().getNumFound() + " results for q=" + q);
|
||||
return rsp.getResults();
|
||||
} catch (final SolrServerException e) {
|
||||
clearCaches(); // prevent further OOM if this was caused by OOM
|
||||
throw new SolrException(ErrorCode.UNKNOWN, e);
|
||||
} catch (final Throwable e) {
|
||||
clearCaches(); // prevent further OOM if this was caused by OOM
|
||||
throw new IOException("Error executing query", e);
|
||||
}
|
||||
}
|
||||
|
|
Loading…
Reference in New Issue
Block a user