mirror of
https://github.com/yacy/yacy_search_server.git
synced 2024-09-21 00:00:13 +02:00
*) Before the crawler retries to download a URL it checks if the server is already doing a shutdown
git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@554 6c8d7289-2bf4-0310-a012-ef5d649a1542
This commit is contained in:
parent
ea26b84eed
commit
ea9a992f05
|
@ -373,8 +373,16 @@ public final class plasmaCrawlWorker extends Thread {
|
|||
remote = null;
|
||||
|
||||
// restart crawling with new url
|
||||
log.logInfo("Redirection detected ('" + res.status + "') for URL " + url.toString() +
|
||||
log.logInfo("CRAWLER Redirection detected ('" + res.status + "') for URL " + url.toString() +
|
||||
"\nRedirecting request to: " + redirectionUrl);
|
||||
|
||||
// if we are already doing a shutdown we don't need to retry crawling
|
||||
if (Thread.currentThread().isInterrupted()) {
|
||||
log.logError("CRAWLER Retry of URL=" + url.toString() + " aborted because of server shutdown.");
|
||||
return;
|
||||
}
|
||||
|
||||
// retry crawling with new url
|
||||
load(redirectionUrl,
|
||||
name,
|
||||
referer,
|
||||
|
@ -426,7 +434,20 @@ public final class plasmaCrawlWorker extends Thread {
|
|||
|
||||
|
||||
if (retryCrawling) {
|
||||
// if we are already doing a shutdown we don't need to retry crawling
|
||||
if (Thread.currentThread().isInterrupted()) {
|
||||
log.logError("CRAWLER Retry of URL=" + url.toString() + " aborted because of server shutdown.");
|
||||
return;
|
||||
}
|
||||
|
||||
// returning the used httpc
|
||||
httpc.returnInstance(remote);
|
||||
remote = null;
|
||||
|
||||
// setting the retry counter to 1
|
||||
if (crawlingRetryCount > 1) crawlingRetryCount = 1;
|
||||
|
||||
// retry crawling
|
||||
load(url,
|
||||
name,
|
||||
referer,
|
||||
|
|
Loading…
Reference in New Issue
Block a user