Marown/yacy_search_server
1
0
Fork 0
mirror of https://github.com/yacy/yacy_search_server.git synced 2024-09-19 00:01:41 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity

better protection against OOM during search flush and fixed missing

Browse Source 
result push
This commit is contained in:
Michael Peter Christen 2013-03-03 23:45:47 +01:00
parent 221ed7d764
commit 0d7b4bc891
2 changed files with 25 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
defaults/yacy.init
View File

@ -605,7 +605,7 @@ collection=user
70_surrogates_busysleep=0
70_surrogates_memprereq=12582912
80_searchresult_idlesleep=10000
80_searchresult_busysleep=500
80_searchresult_busysleep=200
80_searchresult_memprereq=0
90_cleanup_idlesleep=300000
90_cleanup_busysleep=300000

54
source/net/yacy/search/index/Fulltext.java
View File

@ -417,6 +417,14 @@ public final class Fulltext {
}
public void putDocumentLater(final SolrInputDocument doc) {
if (MemoryControl.shortStatus()) {
try {
putDocument(doc);
return;
} catch (IOException ee) {
Log.logException(ee);
}
}
try {
this.pendingCollectionInputDocuments.put(doc);
} catch (InterruptedException e) {
@ -434,17 +442,13 @@ public final class Fulltext {
public int processPendingInputDocuments(int count) throws IOException {
if (count == 0) return 0;
if (count == 1) {
pendingRows2Docs(1);
SolrInputDocument doc = this.pendingCollectionInputDocuments.poll();
if (doc == null) return 0;
this.putDocument(doc);
return 1;
boolean shortMemStatus = MemoryControl.shortStatus();
if (!shortMemStatus || this.pendingCollectionInputDocuments.size() < count) {
pendingRows2Docs(count);
}
pendingRows2Docs(count);
SolrInputDocument doc;
Collection<SolrInputDocument> docs = new ArrayList<SolrInputDocument>(count);
while (count-- > 0 && (doc = this.pendingCollectionInputDocuments.poll()) != null) {
while ((shortMemStatus || count-- > 0) && (doc = this.pendingCollectionInputDocuments.poll()) != null) {
docs.add(doc);
}
if (docs.size() > 0) this.putDocuments(docs);
@ -497,29 +501,19 @@ public final class Fulltext {
}
public void putMetadataLater(final URIMetadataRow entry) throws IOException {
byte[] idb = entry.hash();
String id = ASCII.String(idb);
try {
if (this.urlIndexFile != null) this.urlIndexFile.remove(idb);
// because node entries are richer than metadata entries we must check if they exist to prevent that they are overwritten
SolrDocument sd = this.getDefaultConnector().getById(id);
if (sd == null || (new URIMetadataNode(sd)).isOlder(entry)) {
SolrInputDocument doc = getDefaultConfiguration().metadata2solr(entry);
try {
this.pendingCollectionInputDocuments.put(doc);
} catch (InterruptedException e) {
try {
putDocument(doc);
} catch (IOException ee) {
Log.logException(ee);
}
}
}
} catch (SolrException e) {
throw new IOException(e.getMessage(), e);
if (MemoryControl.shortStatus()) {
putMetadata(entry);
return;
}
try {
this.pendingCollectionInputRows.put(entry);
} catch (InterruptedException e) {
try {
putMetadata(entry);
} catch (IOException ee) {
Log.logException(ee);
}
}
this.statsDump = null;
if (MemoryControl.shortStatus()) clearCache();
}
/**