mirror of
https://github.com/yacy/yacy_search_server.git
synced 2024-09-19 00:01:41 +02:00
adds deleting during recrawl
This commit is contained in:
parent
e0ad8ca9da
commit
80785b785e
|
@ -73,6 +73,9 @@
|
|||
<div class="form-group">
|
||||
<label>Include failed URLs <input type="checkbox" name="includefailedurls" #(includefailedurls)#::checked="checked"#(/includefailedurls)# /></label>
|
||||
</div>
|
||||
<div class="form-group">
|
||||
<label>Delete URLs <input type="checkbox" name="deleteOnRecrawl" #(deleteOnRecrawl)#::checked="checked"#(/deleteOnRecrawl)# /></label>
|
||||
</div>
|
||||
<input type="submit" name="recrawlDefaults" value="Set defaults" class="btn btn-default" title="Reset to default values"/>
|
||||
<input type="submit" name="recrawlnow" value="start recrawl job now" class="btn btn-primary"/>
|
||||
to re-crawl documents selected with the given query.
|
||||
|
@ -91,6 +94,9 @@
|
|||
<div class="form-group">
|
||||
<label>Include failed urls <input type="checkbox" name="includefailedurls" onchange="this.form.submit()" #(includefailedurls)#::checked="checked"#(/includefailedurls)# /></label>
|
||||
</div>
|
||||
<div class="form-group">
|
||||
<label>Delete urls <input type="checkbox" name="deleteOnRecrawl" onchange="this.form.submit()" #(deleteOnRecrawl)#::checked="checked"#(/deleteOnRecrawl)# /></label>
|
||||
</div>
|
||||
<input type="submit" name="stoprecrawl" value="stop recrawl job" class="btn btn-danger"/>
|
||||
</fieldset>
|
||||
#(/recrawljobrunning)#
|
||||
|
|
|
@ -123,6 +123,7 @@ public class IndexReIndexMonitor_p {
|
|||
|
||||
String recrawlQuery = RecrawlBusyThread.DEFAULT_QUERY;
|
||||
boolean inclerrdoc = RecrawlBusyThread.DEFAULT_INCLUDE_FAILED;
|
||||
boolean deleteOnRecrawl = RecrawlBusyThread.DEFAULT_DELETE_ON_RECRAWL;
|
||||
// to signal that a setting shall change the form provides a fixed parameter setup=recrawljob, if not present return status only
|
||||
if (post != null && "recrawljob".equals(post.get("setup"))) { // it's a command to recrawlThread
|
||||
|
||||
|
@ -136,13 +137,17 @@ public class IndexReIndexMonitor_p {
|
|||
if (post.containsKey("includefailedurls")) {
|
||||
inclerrdoc = post.getBoolean("includefailedurls");
|
||||
}
|
||||
|
||||
if (post.containsKey("deleteOnRecrawl")) {
|
||||
deleteOnRecrawl = post.getBoolean("deleteOnRecrawl");
|
||||
}
|
||||
|
||||
if (recrawlbt == null || recrawlbt.shutdownInProgress()) {
|
||||
prop.put("recrawljobrunning_simulationResult", 0);
|
||||
prop.put("recrawljobrunning_error", 0);
|
||||
if (post.containsKey("recrawlnow")) {
|
||||
sb.deployThread(RecrawlBusyThread.THREAD_NAME, "ReCrawl", "recrawl existing documents", null,
|
||||
new RecrawlBusyThread(Switchboard.getSwitchboard(), recrawlQuery, inclerrdoc), 1000);
|
||||
new RecrawlBusyThread(Switchboard.getSwitchboard(), recrawlQuery, inclerrdoc, deleteOnRecrawl), 1000);
|
||||
recrawlbt = sb.getThread(RecrawlBusyThread.THREAD_NAME);
|
||||
|
||||
/* store this call as an api call for easy scheduling possibility */
|
||||
|
@ -192,6 +197,7 @@ public class IndexReIndexMonitor_p {
|
|||
if(post.containsKey("recrawlDefaults")) {
|
||||
recrawlQuery = RecrawlBusyThread.DEFAULT_QUERY;
|
||||
inclerrdoc = RecrawlBusyThread.DEFAULT_INCLUDE_FAILED;
|
||||
deleteOnRecrawl = RecrawlBusyThread.DEFAULT_DELETE_ON_RECRAWL;
|
||||
}
|
||||
} else {
|
||||
if (post.containsKey("stoprecrawl")) {
|
||||
|
@ -204,9 +210,10 @@ public class IndexReIndexMonitor_p {
|
|||
|
||||
if (recrawlbt != null && !recrawlbt.shutdownInProgress()) {
|
||||
if (post.containsKey("updquery") && post.containsKey("recrawlquerytext")) {
|
||||
((RecrawlBusyThread) recrawlbt).setQuery(recrawlQuery, inclerrdoc);
|
||||
((RecrawlBusyThread) recrawlbt).setQuery(recrawlQuery, inclerrdoc, deleteOnRecrawl);
|
||||
} else {
|
||||
((RecrawlBusyThread) recrawlbt).setIncludeFailed(inclerrdoc);
|
||||
((RecrawlBusyThread) recrawlbt).setDeleteOnRecrawl(deleteOnRecrawl);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
@ -219,10 +226,12 @@ public class IndexReIndexMonitor_p {
|
|||
prop.put("recrawljobrunning_docCount", ((RecrawlBusyThread) recrawlbt).getUrlsToRecrawl());
|
||||
prop.put("recrawljobrunning_recrawlquerytext", ((RecrawlBusyThread) recrawlbt).getQuery());
|
||||
prop.put("recrawljobrunning_includefailedurls", ((RecrawlBusyThread) recrawlbt).getIncludeFailed());
|
||||
prop.put("recrawljobrunning_deleteOnRecrawl", ((RecrawlBusyThread) recrawlbt).getDeleteOnRecrawl());
|
||||
} else {
|
||||
prop.put("recrawljobrunning", 0);
|
||||
prop.put("recrawljobrunning_recrawlquerytext", recrawlQuery);
|
||||
prop.put("recrawljobrunning_includefailedurls", inclerrdoc);
|
||||
prop.put("recrawljobrunning_deleteOnRecrawl", deleteOnRecrawl);
|
||||
}
|
||||
|
||||
// return rewrite properties
|
||||
|
|
|
@ -26,8 +26,10 @@ package net.yacy.crawler;
|
|||
import java.io.IOException;
|
||||
import java.net.MalformedURLException;
|
||||
import java.time.LocalDateTime;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Date;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
|
||||
import org.apache.solr.common.SolrDocument;
|
||||
|
@ -44,7 +46,6 @@ import net.yacy.crawler.data.NoticedURL;
|
|||
import net.yacy.crawler.retrieval.Request;
|
||||
import net.yacy.kelondro.workflow.AbstractBusyThread;
|
||||
import net.yacy.search.Switchboard;
|
||||
import net.yacy.search.SwitchboardConstants;
|
||||
import net.yacy.search.schema.CollectionSchema;
|
||||
|
||||
/**
|
||||
|
@ -64,6 +65,9 @@ public class RecrawlBusyThread extends AbstractBusyThread {
|
|||
|
||||
/** Default value for inclusion or not of documents with a https status different from 200 (success) */
|
||||
public static final boolean DEFAULT_INCLUDE_FAILED = false;
|
||||
|
||||
/** The default value whether to delete on Recrawl */
|
||||
public static final boolean DEFAULT_DELETE_ON_RECRAWL = false;
|
||||
|
||||
/** The current query selecting documents to recrawl */
|
||||
private String currentQuery;
|
||||
|
@ -71,6 +75,9 @@ public class RecrawlBusyThread extends AbstractBusyThread {
|
|||
/** flag if docs with httpstatus_i <> 200 shall be recrawled */
|
||||
private boolean includefailed;
|
||||
|
||||
/** flag whether to delete on Recrawl */
|
||||
private boolean deleteOnRecrawl;
|
||||
|
||||
private int chunkstart = 0;
|
||||
private final int chunksize = 100;
|
||||
private final Switchboard sb;
|
||||
|
@ -116,16 +123,17 @@ public class RecrawlBusyThread extends AbstractBusyThread {
|
|||
* set to true when documents with a https status different from 200
|
||||
* (success) must be included
|
||||
*/
|
||||
public RecrawlBusyThread(final Switchboard xsb, final String query, final boolean includeFailed) {
|
||||
public RecrawlBusyThread(final Switchboard xsb, final String query, final boolean includeFailed, final boolean deleteOnRecrawl) {
|
||||
super(3000, 1000); // set lower limits of cycle delay
|
||||
setName(THREAD_NAME);
|
||||
this.setIdleSleep(10*60000); // set actual cycle delays
|
||||
this.setBusySleep(2*60000);
|
||||
this.setPriority(Thread.MIN_PRIORITY);
|
||||
|
||||
this.setLoadPreReqisite(1);
|
||||
this.sb = xsb;
|
||||
this.currentQuery = query;
|
||||
this.includefailed = includeFailed;
|
||||
this.deleteOnRecrawl = deleteOnRecrawl;
|
||||
this.urlstack = new HashSet<DigestURL>();
|
||||
// workaround to prevent solr exception on existing index (not fully reindexed) since intro of schema with docvalues
|
||||
// org.apache.solr.core.SolrCore java.lang.IllegalStateException: unexpected docvalues type NONE for field 'load_date_dt' (expected=NUMERIC). Use UninvertingReader or index with docvalues.
|
||||
|
@ -143,10 +151,12 @@ public class RecrawlBusyThread extends AbstractBusyThread {
|
|||
* and resets the counter to start a fresh query loop
|
||||
* @param q select query
|
||||
* @param includefailedurls true=all http status docs are recrawled, false=httpstatus=200 docs are recrawled
|
||||
* @param deleteOnRecrawl
|
||||
*/
|
||||
public void setQuery(String q, boolean includefailedurls) {
|
||||
public void setQuery(String q, boolean includefailedurls, final boolean deleteOnRecrawl) {
|
||||
this.currentQuery = q;
|
||||
this.includefailed = includefailedurls;
|
||||
this.deleteOnRecrawl = deleteOnRecrawl;
|
||||
this.chunkstart = 0;
|
||||
}
|
||||
|
||||
|
@ -180,6 +190,14 @@ public class RecrawlBusyThread extends AbstractBusyThread {
|
|||
public boolean getIncludeFailed () {
|
||||
return this.includefailed;
|
||||
}
|
||||
|
||||
public void setDeleteOnRecrawl(final boolean deleteOnRecrawl) {
|
||||
this.deleteOnRecrawl = deleteOnRecrawl;
|
||||
}
|
||||
|
||||
public boolean getDeleteOnRecrawl() {
|
||||
return this.deleteOnRecrawl;
|
||||
}
|
||||
|
||||
/**
|
||||
* feed urls to the local crawler
|
||||
|
@ -290,21 +308,27 @@ public class RecrawlBusyThread extends AbstractBusyThread {
|
|||
}
|
||||
|
||||
if (docList != null) {
|
||||
List<String> tobedeletedIDs = new ArrayList<>();
|
||||
for (final SolrDocument doc : docList) {
|
||||
try {
|
||||
this.urlstack.add(new DigestURL((String) doc.getFieldValue(CollectionSchema.sku.getSolrFieldName())));
|
||||
if (deleteOnRecrawl) tobedeletedIDs.add((String) doc.getFieldValue(CollectionSchema.id.getSolrFieldName()));
|
||||
} catch (final MalformedURLException ex) {
|
||||
this.malformedUrlsCount++;
|
||||
try { // if index entry hasn't a valid url (useless), delete it
|
||||
solrConnector.deleteById((String) doc.getFieldValue(CollectionSchema.id.getSolrFieldName()));
|
||||
this.malformedUrlsDeletedCount++;
|
||||
ConcurrentLog.severe(THREAD_NAME, "deleted index document with invalid url " + (String) doc.getFieldValue(CollectionSchema.sku.getSolrFieldName()));
|
||||
} catch (final IOException ex1) {
|
||||
ConcurrentLog.severe(THREAD_NAME, ex1.getMessage());
|
||||
}
|
||||
// if index entry hasn't a valid url (useless), delete it
|
||||
tobedeletedIDs.add((String) doc.getFieldValue(CollectionSchema.id.getSolrFieldName()));
|
||||
this.malformedUrlsDeletedCount++;
|
||||
ConcurrentLog.severe(THREAD_NAME, "deleted index document with invalid url " + (String) doc.getFieldValue(CollectionSchema.sku.getSolrFieldName()));
|
||||
}
|
||||
}
|
||||
this.chunkstart = this.chunkstart + this.chunksize;
|
||||
|
||||
if (!tobedeletedIDs.isEmpty()) try {
|
||||
solrConnector.deleteByIds(tobedeletedIDs);
|
||||
} catch (IOException e) {
|
||||
ConcurrentLog.severe(THREAD_NAME, "error deleting IDs ", e);
|
||||
}
|
||||
|
||||
this.chunkstart = deleteOnRecrawl? 0 : this.chunkstart + this.chunksize;
|
||||
}
|
||||
|
||||
if (docList == null || docList.size() < this.chunksize) {
|
||||
|
|
Loading…
Reference in New Issue
Block a user