2008-10-17 14:14:57 +02:00
// CrawlQueues.java
2007-10-29 02:43:20 +01:00
// (C) 2007 by Michael Peter Christen; mc@yacy.net, Frankfurt a. M., Germany
// first published 29.10.2007 on http://yacy.net
//
// This is a part of YaCy, a peer-to-peer based web search engine
//
// $LastChangedDate: 2006-04-02 22:40:07 +0200 (So, 02 Apr 2006) $
// $LastChangedRevision: 1986 $
// $LastChangedBy: orbiter $
//
// LICENSE
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation; either version 2 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
2008-05-06 02:32:41 +02:00
package de.anomic.crawler ;
2007-10-29 02:43:20 +01:00
import java.io.File ;
import java.io.IOException ;
2007-11-29 03:07:37 +01:00
import java.net.MalformedURLException ;
import java.text.ParseException ;
import java.util.ArrayList ;
2007-10-29 02:43:20 +01:00
import java.util.Date ;
import java.util.Iterator ;
2008-01-14 00:10:09 +01:00
import java.util.Map ;
2008-03-17 14:51:56 +01:00
import java.util.concurrent.ConcurrentHashMap ;
2007-10-29 02:43:20 +01:00
2008-08-25 20:11:47 +02:00
import de.anomic.index.indexDocumentMetadata ;
2009-01-30 23:08:08 +01:00
import de.anomic.kelondro.order.DateFormatter ;
import de.anomic.kelondro.table.FlexWidthArray ;
2009-01-31 00:33:47 +01:00
import de.anomic.kelondro.util.Log ;
2007-11-22 02:34:29 +01:00
import de.anomic.plasma.plasmaParser ;
2007-10-29 02:43:20 +01:00
import de.anomic.plasma.plasmaSwitchboard ;
2008-08-02 15:57:00 +02:00
import de.anomic.plasma.plasmaSwitchboardConstants ;
2008-12-17 23:53:06 +01:00
import de.anomic.server.serverProcessorJob ;
2008-04-24 23:31:07 +02:00
import de.anomic.xml.RSSFeed ;
import de.anomic.xml.RSSMessage ;
2007-10-29 02:43:20 +01:00
import de.anomic.yacy.yacyClient ;
2008-11-06 11:07:53 +01:00
import de.anomic.yacy.yacyPeerSelection ;
2007-10-29 02:43:20 +01:00
import de.anomic.yacy.yacySeed ;
import de.anomic.yacy.yacyURL ;
2008-05-06 02:32:41 +02:00
public class CrawlQueues {
2007-10-29 02:43:20 +01:00
2008-05-06 02:32:41 +02:00
plasmaSwitchboard sb ;
2009-01-31 00:33:47 +01:00
Log log ;
2008-03-04 23:45:45 +01:00
Map < Integer , crawlWorker > workers ; // mapping from url hash to Worker thread object
2008-05-06 02:32:41 +02:00
ProtocolLoader loader ;
2008-08-02 14:12:04 +02:00
private final ArrayList < String > remoteCrawlProviderHashes ;
2007-10-29 02:43:20 +01:00
2008-05-06 02:32:41 +02:00
public NoticedURL noticeURL ;
public ZURL errorURL , delegatedURL ;
2007-10-29 02:43:20 +01:00
2008-08-02 14:12:04 +02:00
public CrawlQueues ( final plasmaSwitchboard sb , final File plasmaPath ) {
2007-10-29 02:43:20 +01:00
this . sb = sb ;
2009-01-31 00:33:47 +01:00
this . log = new Log ( " CRAWLER " ) ;
2008-03-17 14:51:56 +01:00
this . workers = new ConcurrentHashMap < Integer , crawlWorker > ( ) ;
2008-05-06 02:32:41 +02:00
this . loader = new ProtocolLoader ( sb , log ) ;
2008-01-10 00:05:52 +01:00
this . remoteCrawlProviderHashes = new ArrayList < String > ( ) ;
2007-10-29 02:43:20 +01:00
// start crawling management
log . logConfig ( " Starting Crawling Management " ) ;
2008-05-06 02:32:41 +02:00
noticeURL = new NoticedURL ( plasmaPath ) ;
2007-10-29 02:43:20 +01:00
//errorURL = new plasmaCrawlZURL(); // fresh error DB each startup; can be hold in RAM and reduces IO;
2008-08-02 14:12:04 +02:00
final File errorDBFile = new File ( plasmaPath , " urlError2.db " ) ;
2008-03-01 10:44:33 +01:00
if ( errorDBFile . exists ( ) ) {
// delete the error db to get a fresh each time on startup
// this is useful because there is currently no re-use of the data in this table.
2009-01-30 23:08:08 +01:00
if ( errorDBFile . isDirectory ( ) ) FlexWidthArray . delete ( plasmaPath , " urlError2.db " ) ; else errorDBFile . delete ( ) ;
2008-03-01 10:44:33 +01:00
}
2008-11-11 22:42:12 +01:00
errorURL = new ZURL ( plasmaPath , " urlError3.db " , false ) ;
delegatedURL = new ZURL ( plasmaPath , " urlDelegated3.db " , true ) ;
2007-10-29 02:43:20 +01:00
}
2008-03-28 12:56:28 +01:00
2008-08-02 14:12:04 +02:00
public String urlExists ( final String hash ) {
2007-10-29 02:43:20 +01:00
// tests if hash occurrs in any database
// if it exists, the name of the database is returned,
// if it not exists, null is returned
if ( noticeURL . existsInStack ( hash ) ) return " crawler " ;
if ( delegatedURL . exists ( hash ) ) return " delegated " ;
if ( errorURL . exists ( hash ) ) return " errors " ;
2008-08-02 14:12:04 +02:00
for ( final crawlWorker worker : workers . values ( ) ) {
2008-03-17 14:51:56 +01:00
if ( worker . entry . url ( ) . hash ( ) . equals ( hash ) ) return " worker " ;
}
2007-10-29 02:43:20 +01:00
return null ;
}
2008-08-02 14:12:04 +02:00
public void urlRemove ( final String hash ) {
2007-10-29 02:43:20 +01:00
noticeURL . removeByURLHash ( hash ) ;
delegatedURL . remove ( hash ) ;
errorURL . remove ( hash ) ;
}
2008-08-02 14:12:04 +02:00
public yacyURL getURL ( final String urlhash ) {
2008-05-06 01:13:47 +02:00
assert urlhash ! = null ;
2008-07-11 12:59:06 +02:00
if ( urlhash = = null | | urlhash . length ( ) = = 0 ) return null ;
2008-08-02 14:12:04 +02:00
final CrawlEntry ne = noticeURL . get ( urlhash ) ;
2007-10-29 02:43:20 +01:00
if ( ne ! = null ) return ne . url ( ) ;
2008-05-06 02:32:41 +02:00
ZURL . Entry ee = delegatedURL . getEntry ( urlhash ) ;
2007-10-29 02:43:20 +01:00
if ( ee ! = null ) return ee . url ( ) ;
ee = errorURL . getEntry ( urlhash ) ;
if ( ee ! = null ) return ee . url ( ) ;
2008-08-02 14:12:04 +02:00
for ( final crawlWorker w : workers . values ( ) ) {
2008-01-14 00:10:09 +01:00
if ( w . entry . url ( ) . hash ( ) . equals ( urlhash ) ) return w . entry . url ( ) ;
}
2007-10-29 02:43:20 +01:00
return null ;
}
2008-06-04 23:34:57 +02:00
public void clear ( ) {
// wait for all workers to finish
2008-08-02 14:12:04 +02:00
for ( final crawlWorker w : workers . values ( ) ) {
2008-06-04 23:34:57 +02:00
w . interrupt ( ) ;
}
// TODO: wait some more time until all threads are finished
workers . clear ( ) ;
remoteCrawlProviderHashes . clear ( ) ;
noticeURL . clear ( ) ;
try {
errorURL . clear ( ) ;
2008-08-02 14:12:04 +02:00
} catch ( final IOException e ) {
2008-06-04 23:34:57 +02:00
e . printStackTrace ( ) ;
}
try {
delegatedURL . clear ( ) ;
2008-08-02 14:12:04 +02:00
} catch ( final IOException e ) {
2008-06-04 23:34:57 +02:00
e . printStackTrace ( ) ;
}
}
2007-10-29 02:43:20 +01:00
public void close ( ) {
// wait for all workers to finish
2008-08-02 14:12:04 +02:00
for ( final crawlWorker w : workers . values ( ) ) {
2008-03-17 14:51:56 +01:00
w . interrupt ( ) ;
}
2007-10-29 02:43:20 +01:00
// TODO: wait some more time until all threads are finished
2007-11-29 03:07:37 +01:00
noticeURL . close ( ) ;
errorURL . close ( ) ;
delegatedURL . close ( ) ;
2007-10-29 02:43:20 +01:00
}
2008-05-06 02:32:41 +02:00
public CrawlEntry [ ] activeWorkerEntries ( ) {
2007-10-29 02:43:20 +01:00
synchronized ( workers ) {
2008-08-02 14:12:04 +02:00
final CrawlEntry [ ] e = new CrawlEntry [ workers . size ( ) ] ;
2008-03-20 00:47:24 +01:00
int i = 0 ;
2008-08-02 14:12:04 +02:00
for ( final crawlWorker w : workers . values ( ) ) e [ i + + ] = w . entry ;
2008-03-20 00:47:24 +01:00
return e ;
2007-10-29 02:43:20 +01:00
}
}
2008-08-02 14:12:04 +02:00
public boolean isSupportedProtocol ( final String protocol ) {
2007-10-29 02:43:20 +01:00
return loader . isSupportedProtocol ( protocol ) ;
}
public int coreCrawlJobSize ( ) {
2008-05-06 02:32:41 +02:00
return noticeURL . stackSize ( NoticedURL . STACK_TYPE_CORE ) ;
2007-10-29 02:43:20 +01:00
}
public boolean coreCrawlJob ( ) {
2007-11-29 03:07:37 +01:00
2008-08-02 14:12:04 +02:00
final boolean robinsonPrivateCase = ( ( sb . isRobinsonMode ( ) ) & &
2008-08-02 15:57:00 +02:00
( ! sb . getConfig ( plasmaSwitchboardConstants . CLUSTER_MODE , " " ) . equals ( plasmaSwitchboardConstants . CLUSTER_MODE_PUBLIC_CLUSTER ) ) & &
( ! sb . getConfig ( plasmaSwitchboardConstants . CLUSTER_MODE , " " ) . equals ( plasmaSwitchboardConstants . CLUSTER_MODE_PRIVATE_CLUSTER ) ) ) ;
2007-11-29 03:07:37 +01:00
2007-12-04 21:19:13 +01:00
if ( ( ( robinsonPrivateCase ) | | ( coreCrawlJobSize ( ) < = 20 ) ) & & ( limitCrawlJobSize ( ) > 0 ) ) {
2007-11-29 03:07:37 +01:00
// move some tasks to the core crawl job so we have something to do
2008-08-02 14:12:04 +02:00
final int toshift = Math . min ( 10 , limitCrawlJobSize ( ) ) ; // this cannot be a big number because the balancer makes a forced waiting if it cannot balance
2007-11-29 03:07:37 +01:00
for ( int i = 0 ; i < toshift ; i + + ) {
2008-10-08 20:26:36 +02:00
noticeURL . shift ( NoticedURL . STACK_TYPE_LIMIT , NoticedURL . STACK_TYPE_CORE , sb . webIndex . profilesActiveCrawls ) ;
2007-11-29 03:07:37 +01:00
}
log . logInfo ( " shifted " + toshift + " jobs from global crawl to local crawl (coreCrawlJobSize()= " + coreCrawlJobSize ( ) +
2008-08-02 15:57:00 +02:00
" , limitCrawlJobSize()= " + limitCrawlJobSize ( ) + " , cluster.mode= " + sb . getConfig ( plasmaSwitchboardConstants . CLUSTER_MODE , " " ) +
2007-11-29 03:07:37 +01:00
" , robinsonMode= " + ( ( sb . isRobinsonMode ( ) ) ? " on " : " off " ) ) ;
}
2008-10-17 12:39:32 +02:00
if ( ! crawlIsPossible ( NoticedURL . STACK_TYPE_CORE , " Core " ) ) return false ;
2007-10-29 02:43:20 +01:00
2008-10-17 12:39:32 +02:00
if ( isPaused ( plasmaSwitchboardConstants . CRAWLJOB_LOCAL_CRAWL ) ) return false ;
2007-10-29 02:43:20 +01:00
// do a local crawl
2008-05-06 02:32:41 +02:00
CrawlEntry urlEntry = null ;
while ( urlEntry = = null & & noticeURL . stackSize ( NoticedURL . STACK_TYPE_CORE ) > 0 ) {
2008-08-02 14:12:04 +02:00
final String stats = " LOCALCRAWL[ " + noticeURL . stackSize ( NoticedURL . STACK_TYPE_CORE ) + " , " + noticeURL . stackSize ( NoticedURL . STACK_TYPE_LIMIT ) + " , " + noticeURL . stackSize ( NoticedURL . STACK_TYPE_OVERHANG ) + " , " + noticeURL . stackSize ( NoticedURL . STACK_TYPE_REMOTE ) + " ] " ;
2007-10-29 02:43:20 +01:00
try {
2008-10-08 20:26:36 +02:00
urlEntry = noticeURL . pop ( NoticedURL . STACK_TYPE_CORE , true , sb . webIndex . profilesActiveCrawls ) ;
2008-10-10 00:31:43 +02:00
if ( urlEntry = = null ) continue ;
2008-08-02 14:12:04 +02:00
final String profileHandle = urlEntry . profileHandle ( ) ;
2007-10-29 02:43:20 +01:00
// System.out.println("DEBUG plasmaSwitchboard.processCrawling:
// profileHandle = " + profileHandle + ", urlEntry.url = " + urlEntry.url());
if ( profileHandle = = null ) {
log . logSevere ( stats + " : NULL PROFILE HANDLE ' " + urlEntry . profileHandle ( ) + " ' for URL " + urlEntry . url ( ) ) ;
return true ;
}
2008-10-17 12:39:32 +02:00
generateCrawl ( urlEntry , stats , profileHandle ) ;
2007-10-29 02:43:20 +01:00
return true ;
2008-08-02 14:12:04 +02:00
} catch ( final IOException e ) {
2007-10-29 02:43:20 +01:00
log . logSevere ( stats + " : CANNOT FETCH ENTRY: " + e . getMessage ( ) , e ) ;
2008-05-06 02:32:41 +02:00
if ( e . getMessage ( ) . indexOf ( " hash is null " ) > 0 ) noticeURL . clear ( NoticedURL . STACK_TYPE_CORE ) ;
2007-10-29 02:43:20 +01:00
}
}
return true ;
}
2008-10-17 12:39:32 +02:00
/ * *
* Make some checks if crawl is valid and start it
*
* @param urlEntry
* @param profileHandle
* @param stats String for log prefixing
* @return
* /
2008-10-17 14:14:57 +02:00
private void generateCrawl ( CrawlEntry urlEntry , final String stats , final String profileHandle ) {
2008-10-17 12:39:32 +02:00
final CrawlProfile . entry profile = sb . webIndex . profilesActiveCrawls . getEntry ( profileHandle ) ;
if ( profile ! = null ) {
// check if the protocol is supported
final yacyURL url = urlEntry . url ( ) ;
final String urlProtocol = url . getProtocol ( ) ;
if ( this . sb . crawlQueues . isSupportedProtocol ( urlProtocol ) ) {
if ( this . log . isFine ( ) )
log . logFine ( stats + " : URL= " + urlEntry . url ( )
+ " , initiator= " + urlEntry . initiator ( )
+ " , crawlOrder= " + ( ( profile . remoteIndexing ( ) ) ? " true " : " false " )
+ " , depth= " + urlEntry . depth ( )
2008-11-14 10:58:56 +01:00
+ " , crawlDepth= " + profile . depth ( )
+ " , must-match= " + profile . mustMatchPattern ( ) . toString ( )
+ " , must-not-match= " + profile . mustNotMatchPattern ( ) . toString ( )
2008-10-17 12:39:32 +02:00
+ " , permission= " + ( ( sb . webIndex . seedDB = = null ) ? " undefined " : ( ( ( sb . webIndex . seedDB . mySeed ( ) . isSenior ( ) ) | | ( sb . webIndex . seedDB . mySeed ( ) . isPrincipal ( ) ) ) ? " true " : " false " ) ) ) ;
processLocalCrawling ( urlEntry , stats ) ;
} else {
this . log . logSevere ( " Unsupported protocol in URL ' " + url . toString ( ) ) ;
}
} else {
log . logWarning ( stats + " : LOST PROFILE HANDLE ' " + urlEntry . profileHandle ( ) + " ' for URL " + urlEntry . url ( ) ) ;
}
}
/ * *
* if crawling was paused we have to wait until we wer notified to continue
* blocks until pause is ended
* @param crawljob
* @return
* /
private boolean isPaused ( String crawljob ) {
final Object [ ] status = sb . crawlJobsStatus . get ( crawljob ) ;
boolean pauseEnded = false ;
synchronized ( status [ plasmaSwitchboardConstants . CRAWLJOB_SYNC ] ) {
if ( ( ( Boolean ) status [ plasmaSwitchboardConstants . CRAWLJOB_STATUS ] ) . booleanValue ( ) ) {
try {
status [ plasmaSwitchboardConstants . CRAWLJOB_SYNC ] . wait ( ) ;
}
catch ( final InterruptedException e ) { pauseEnded = true ; }
}
}
return pauseEnded ;
}
/ * *
* Checks if crawl queue has elements and new crawl will not exceed thread - limit
* @param stackType
* @param type
* @return
* /
private boolean crawlIsPossible ( int stackType , final String type ) {
if ( noticeURL . stackSize ( stackType ) = = 0 ) {
//log.logDebug("GlobalCrawl: queue is empty");
return false ;
}
if ( sb . webIndex . queuePreStack . size ( ) > = ( int ) sb . getConfigLong ( plasmaSwitchboardConstants . INDEXER_SLOTS , 30 ) ) {
if ( this . log . isFine ( ) ) log . logFine ( type + " Crawl: too many processes in indexing queue, dismissed ( " + " sbQueueSize= " + sb . webIndex . queuePreStack . size ( ) + " ) " ) ;
return false ;
}
if ( this . size ( ) > = sb . getConfigLong ( plasmaSwitchboardConstants . CRAWLER_THREADS_ACTIVE_MAX , 10 ) ) {
if ( this . log . isFine ( ) ) log . logFine ( type + " Crawl: too many processes in loader queue, dismissed ( " + " cacheLoader= " + this . size ( ) + " ) " ) ;
return false ;
}
if ( sb . onlineCaution ( ) ) {
if ( this . log . isFine ( ) ) log . logFine ( type + " Crawl: online caution, omitting processing " ) ;
return false ;
}
return true ;
}
2007-10-29 02:43:20 +01:00
2007-11-29 03:07:37 +01:00
public boolean remoteCrawlLoaderJob ( ) {
// check if we are allowed to crawl urls provided by other peers
2008-05-14 23:36:02 +02:00
if ( ! sb . webIndex . seedDB . mySeed ( ) . getFlagAcceptRemoteCrawl ( ) ) {
2007-11-29 03:54:59 +01:00
//this.log.logInfo("remoteCrawlLoaderJob: not done, we are not allowed to do that");
return false ;
}
2007-10-29 02:43:20 +01:00
2007-11-29 03:07:37 +01:00
// check if we are a senior peer
2008-05-14 23:36:02 +02:00
if ( ! sb . webIndex . seedDB . mySeed ( ) . isActive ( ) ) {
2007-11-29 03:54:59 +01:00
//this.log.logInfo("remoteCrawlLoaderJob: not done, this should be a senior or principal peer");
return false ;
}
2007-10-29 02:43:20 +01:00
2008-08-02 15:57:00 +02:00
if ( sb . webIndex . queuePreStack . size ( ) > = ( int ) sb . getConfigLong ( plasmaSwitchboardConstants . INDEXER_SLOTS , 30 ) / 2 ) {
2008-05-14 23:36:02 +02:00
if ( this . log . isFine ( ) ) log . logFine ( " remoteCrawlLoaderJob: too many processes in indexing queue, dismissed ( " + " sbQueueSize= " + sb . webIndex . queuePreStack . size ( ) + " ) " ) ;
2007-11-29 14:58:00 +01:00
return false ;
}
2008-08-02 15:57:00 +02:00
if ( this . size ( ) > = sb . getConfigLong ( plasmaSwitchboardConstants . CRAWLER_THREADS_ACTIVE_MAX , 10 ) ) {
2008-04-08 16:44:39 +02:00
if ( this . log . isFine ( ) ) log . logFine ( " remoteCrawlLoaderJob: too many processes in loader queue, dismissed ( " + " cacheLoader= " + this . size ( ) + " ) " ) ;
2007-11-29 14:58:00 +01:00
return false ;
}
if ( sb . onlineCaution ( ) ) {
2008-04-08 16:44:39 +02:00
if ( this . log . isFine ( ) ) log . logFine ( " remoteCrawlLoaderJob: online caution, omitting processing " ) ;
2007-11-29 14:58:00 +01:00
return false ;
}
2008-11-06 11:07:53 +01:00
if ( remoteTriggeredCrawlJobSize ( ) > 100 ) {
2008-07-03 15:08:37 +02:00
if ( this . log . isFine ( ) ) log . logFine ( " remoteCrawlLoaderJob: the remote-triggered crawl job queue is filled, omitting processing " ) ;
return false ;
}
if ( coreCrawlJobSize ( ) > 0 ) {
if ( this . log . isFine ( ) ) log . logFine ( " remoteCrawlLoaderJob: a local crawl is running, omitting processing " ) ;
return false ;
}
2007-11-29 03:54:59 +01:00
// check if we have an entry in the provider list, otherwise fill the list
2007-11-29 03:07:37 +01:00
yacySeed seed ;
2008-07-03 15:08:37 +02:00
if ( remoteCrawlProviderHashes . size ( ) = = 0 ) {
2008-05-14 23:36:02 +02:00
if ( sb . webIndex . seedDB ! = null & & sb . webIndex . seedDB . sizeConnected ( ) > 0 ) {
2008-11-06 11:07:53 +01:00
final Iterator < yacySeed > e = yacyPeerSelection . getProvidesRemoteCrawlURLs ( sb . webIndex . seedDB ) ;
2007-11-29 03:07:37 +01:00
while ( e . hasNext ( ) ) {
2008-01-10 00:05:52 +01:00
seed = e . next ( ) ;
2007-11-29 03:07:37 +01:00
if ( seed ! = null ) {
remoteCrawlProviderHashes . add ( seed . hash ) ;
}
2007-10-29 02:43:20 +01:00
}
}
}
2007-11-29 03:07:37 +01:00
if ( remoteCrawlProviderHashes . size ( ) = = 0 ) return false ;
2007-10-29 02:43:20 +01:00
2007-11-29 03:07:37 +01:00
// take one entry from the provider list and load the entries from the remote peer
seed = null ;
String hash = null ;
while ( ( seed = = null ) & & ( remoteCrawlProviderHashes . size ( ) > 0 ) ) {
2008-06-06 18:01:27 +02:00
hash = remoteCrawlProviderHashes . remove ( remoteCrawlProviderHashes . size ( ) - 1 ) ;
2008-01-10 00:05:52 +01:00
if ( hash = = null ) continue ;
2008-05-14 23:36:02 +02:00
seed = sb . webIndex . seedDB . get ( hash ) ;
2008-01-10 00:05:52 +01:00
if ( seed = = null ) continue ;
// check if the peer is inside our cluster
if ( ( sb . isRobinsonMode ( ) ) & & ( ! sb . isInMyCluster ( seed ) ) ) {
seed = null ;
continue ;
}
2007-11-29 03:07:37 +01:00
}
if ( seed = = null ) return false ;
// we know a peer which should provide remote crawl entries. load them now.
2009-02-04 17:55:09 +01:00
final RSSFeed feed = yacyClient . queryRemoteCrawlURLs ( sb . webIndex . seedDB , seed , 30 , 60000 ) ;
2008-11-06 13:30:55 +01:00
if ( feed = = null | | feed . size ( ) = = 0 ) {
// something is wrong with this provider. To prevent that we get not stuck with this peer
// we remove it from the peer list
sb . webIndex . peerActions . peerDeparture ( seed , " no results from provided remote crawls " ) ;
2008-11-06 15:14:34 +01:00
// ask another peer
return remoteCrawlLoaderJob ( ) ;
2008-11-06 13:30:55 +01:00
}
2007-11-29 03:07:37 +01:00
// parse the rss
2007-11-29 14:58:00 +01:00
yacyURL url , referrer ;
Date loaddate ;
2008-08-02 14:12:04 +02:00
for ( final RSSMessage item : feed ) {
2007-11-29 03:07:37 +01:00
//System.out.println("URL=" + item.getLink() + ", desc=" + item.getDescription() + ", pubDate=" + item.getPubDate());
2007-10-29 02:43:20 +01:00
2007-11-29 03:07:37 +01:00
// put url on remote crawl stack
try {
url = new yacyURL ( item . getLink ( ) , null ) ;
2008-08-02 14:12:04 +02:00
} catch ( final MalformedURLException e ) {
2007-11-29 03:07:37 +01:00
url = null ;
2007-10-29 02:43:20 +01:00
}
2007-11-29 14:58:00 +01:00
try {
referrer = new yacyURL ( item . getReferrer ( ) , null ) ;
2008-08-02 14:12:04 +02:00
} catch ( final MalformedURLException e ) {
2007-11-29 14:58:00 +01:00
referrer = null ;
}
2007-11-29 03:07:37 +01:00
try {
2009-01-30 16:33:00 +01:00
loaddate = DateFormatter . parseShortSecond ( item . getPubDate ( ) ) ;
2008-08-02 14:12:04 +02:00
} catch ( final ParseException e ) {
2007-11-29 03:07:37 +01:00
loaddate = new Date ( ) ;
}
2008-12-15 01:02:58 +01:00
final String urlRejectReason = sb . crawlStacker . urlInAcceptedDomain ( url ) ;
2008-04-20 23:36:25 +02:00
if ( urlRejectReason = = null ) {
2007-11-29 03:07:37 +01:00
// stack url
2008-09-03 02:30:21 +02:00
if ( sb . getLog ( ) . isFinest ( ) ) sb . getLog ( ) . logFinest ( " crawlOrder: stack: url=' " + url + " ' " ) ;
2008-12-17 23:53:06 +01:00
sb . crawlStacker . enqueueEntry ( new CrawlEntry (
hash ,
url ,
( referrer = = null ) ? null : referrer . hash ( ) ,
item . getDescription ( ) ,
null ,
loaddate ,
sb . webIndex . defaultRemoteProfile . handle ( ) ,
0 ,
0 ,
0
) ) ;
2007-11-29 03:07:37 +01:00
} else {
2008-06-06 18:01:27 +02:00
log . logWarning ( " crawlOrder: Rejected URL ' " + urlToString ( url ) + " ': " + urlRejectReason ) ;
2007-10-29 02:43:20 +01:00
}
}
2007-11-29 03:07:37 +01:00
return true ;
}
2008-06-06 18:01:27 +02:00
/ * *
* @param url
* @return
* /
2008-08-02 14:12:04 +02:00
private String urlToString ( final yacyURL url ) {
2008-06-06 18:01:27 +02:00
return ( url = = null ? " null " : url . toNormalform ( true , false ) ) ;
}
2007-11-29 03:07:37 +01:00
public int limitCrawlJobSize ( ) {
2008-05-06 02:32:41 +02:00
return noticeURL . stackSize ( NoticedURL . STACK_TYPE_LIMIT ) ;
2007-10-29 02:43:20 +01:00
}
public int remoteTriggeredCrawlJobSize ( ) {
2008-05-06 02:32:41 +02:00
return noticeURL . stackSize ( NoticedURL . STACK_TYPE_REMOTE ) ;
2007-10-29 02:43:20 +01:00
}
public boolean remoteTriggeredCrawlJob ( ) {
// work off crawl requests that had been placed by other peers to our crawl stack
// do nothing if either there are private processes to be done
// or there is no global crawl on the stack
2008-10-17 12:39:32 +02:00
if ( ! crawlIsPossible ( NoticedURL . STACK_TYPE_REMOTE , " Global " ) ) return false ;
2007-11-29 14:58:00 +01:00
2008-10-17 12:39:32 +02:00
if ( isPaused ( plasmaSwitchboardConstants . CRAWLJOB_REMOTE_TRIGGERED_CRAWL ) ) return false ;
2007-10-29 02:43:20 +01:00
// we don't want to crawl a global URL globally, since WE are the global part. (from this point of view)
2008-08-02 14:12:04 +02:00
final String stats = " REMOTETRIGGEREDCRAWL[ " + noticeURL . stackSize ( NoticedURL . STACK_TYPE_CORE ) + " , " + noticeURL . stackSize ( NoticedURL . STACK_TYPE_LIMIT ) + " , " + noticeURL . stackSize ( NoticedURL . STACK_TYPE_OVERHANG ) + " , "
2008-05-06 02:32:41 +02:00
+ noticeURL . stackSize ( NoticedURL . STACK_TYPE_REMOTE ) + " ] " ;
2007-10-29 02:43:20 +01:00
try {
2008-10-08 20:26:36 +02:00
final CrawlEntry urlEntry = noticeURL . pop ( NoticedURL . STACK_TYPE_REMOTE , true , sb . webIndex . profilesActiveCrawls ) ;
2008-08-02 14:12:04 +02:00
final String profileHandle = urlEntry . profileHandle ( ) ;
2007-10-29 02:43:20 +01:00
// System.out.println("DEBUG plasmaSwitchboard.processCrawling:
// profileHandle = " + profileHandle + ", urlEntry.url = " +
// urlEntry.url());
2008-10-17 12:39:32 +02:00
generateCrawl ( urlEntry , stats , profileHandle ) ;
2007-10-29 02:43:20 +01:00
return true ;
2008-08-02 14:12:04 +02:00
} catch ( final IOException e ) {
2007-10-29 02:43:20 +01:00
log . logSevere ( stats + " : CANNOT FETCH ENTRY: " + e . getMessage ( ) , e ) ;
2008-05-06 02:32:41 +02:00
if ( e . getMessage ( ) . indexOf ( " hash is null " ) > 0 ) noticeURL . clear ( NoticedURL . STACK_TYPE_REMOTE ) ;
2007-10-29 02:43:20 +01:00
return true ;
}
}
2008-08-02 14:12:04 +02:00
private void processLocalCrawling ( final CrawlEntry entry , final String stats ) {
2007-10-29 02:43:20 +01:00
// work off one Crawl stack entry
if ( ( entry = = null ) | | ( entry . url ( ) = = null ) ) {
log . logInfo ( stats + " : urlEntry = null " ) ;
return ;
}
2008-01-14 00:10:09 +01:00
new crawlWorker ( entry ) ;
2007-10-29 02:43:20 +01:00
log . logInfo ( stats + " : enqueued for load " + entry . url ( ) + " [ " + entry . url ( ) . hash ( ) + " ] " ) ;
return ;
}
2008-08-25 20:11:47 +02:00
public indexDocumentMetadata loadResourceFromWeb (
2008-08-02 14:12:04 +02:00
final yacyURL url ,
final int socketTimeout ,
final boolean keepInMemory ,
final boolean forText ,
final boolean global
2008-11-11 22:33:40 +01:00
) throws IOException {
2007-10-29 02:43:20 +01:00
2008-08-02 14:12:04 +02:00
final CrawlEntry centry = new CrawlEntry (
2008-05-14 23:36:02 +02:00
sb . webIndex . seedDB . mySeed ( ) . hash ,
2007-10-29 02:43:20 +01:00
url ,
2008-05-06 01:13:47 +02:00
" " ,
2007-10-29 02:43:20 +01:00
" " ,
new Date ( ) ,
2008-12-17 23:53:06 +01:00
new Date ( ) ,
2008-04-24 10:42:08 +02:00
( forText ) ?
( ( global ) ?
2008-05-14 23:36:02 +02:00
sb . webIndex . defaultTextSnippetGlobalProfile . handle ( ) :
sb . webIndex . defaultTextSnippetLocalProfile . handle ( ) )
2008-04-24 10:42:08 +02:00
:
( ( global ) ?
2008-05-14 23:36:02 +02:00
sb . webIndex . defaultMediaSnippetGlobalProfile . handle ( ) :
sb . webIndex . defaultMediaSnippetLocalProfile . handle ( ) ) , // crawl profile
2007-10-29 02:43:20 +01:00
0 ,
0 ,
0 ) ;
2007-11-22 02:34:29 +01:00
return loader . load ( centry , ( forText ) ? plasmaParser . PARSER_MODE_CRAWLER : plasmaParser . PARSER_MODE_IMAGE ) ;
2007-10-29 02:43:20 +01:00
}
public int size ( ) {
return workers . size ( ) ;
}
2008-08-06 21:43:12 +02:00
protected final class crawlWorker extends Thread {
2007-10-29 02:43:20 +01:00
2008-05-06 02:32:41 +02:00
public CrawlEntry entry ;
2008-08-02 14:12:04 +02:00
private final Integer code ;
2007-10-29 02:43:20 +01:00
2008-08-02 14:12:04 +02:00
public crawlWorker ( final CrawlEntry entry ) {
2007-10-29 02:43:20 +01:00
this . entry = entry ;
2008-12-17 23:53:06 +01:00
this . entry . setStatus ( " worker-initialized " , serverProcessorJob . STATUS_INITIATED ) ;
2008-08-06 21:43:12 +02:00
this . code = Integer . valueOf ( entry . hashCode ( ) ) ;
2008-01-14 00:10:09 +01:00
if ( ! workers . containsKey ( code ) ) {
workers . put ( code , this ) ;
this . start ( ) ;
}
2007-10-29 02:43:20 +01:00
}
public void run ( ) {
try {
// checking robots.txt for http(s) resources
2008-12-17 23:53:06 +01:00
this . entry . setStatus ( " worker-checkingrobots " , serverProcessorJob . STATUS_STARTED ) ;
2008-07-05 02:35:20 +02:00
if ( ( entry . url ( ) . getProtocol ( ) . equals ( " http " ) | | entry . url ( ) . getProtocol ( ) . equals ( " https " ) ) & & sb . robots . isDisallowed ( entry . url ( ) ) ) {
2008-04-08 16:44:39 +02:00
if ( log . isFine ( ) ) log . logFine ( " Crawling of URL ' " + entry . url ( ) . toString ( ) + " ' disallowed by robots.txt. " ) ;
2008-08-02 14:12:04 +02:00
final ZURL . Entry eentry = errorURL . newEntry (
2008-05-06 01:13:47 +02:00
this . entry ,
2008-05-14 23:36:02 +02:00
sb . webIndex . seedDB . mySeed ( ) . hash ,
2008-05-06 01:13:47 +02:00
new Date ( ) ,
1 ,
" denied by robots.txt " ) ;
2007-10-29 02:43:20 +01:00
eentry . store ( ) ;
errorURL . push ( eentry ) ;
} else {
// starting a load from the internet
2008-12-17 23:53:06 +01:00
this . entry . setStatus ( " worker-loading " , serverProcessorJob . STATUS_RUNNING ) ;
2008-08-02 14:12:04 +02:00
final String result = loader . process ( this . entry , plasmaParser . PARSER_MODE_CRAWLER ) ;
2007-10-29 02:43:20 +01:00
if ( result ! = null ) {
2008-08-02 14:12:04 +02:00
final ZURL . Entry eentry = errorURL . newEntry (
2008-05-06 01:13:47 +02:00
this . entry ,
2008-05-14 23:36:02 +02:00
sb . webIndex . seedDB . mySeed ( ) . hash ,
2008-05-06 01:13:47 +02:00
new Date ( ) ,
1 ,
" cannot load: " + result ) ;
2007-10-29 02:43:20 +01:00
eentry . store ( ) ;
errorURL . push ( eentry ) ;
} else {
2008-12-17 23:53:06 +01:00
this . entry . setStatus ( " worker-processed " , serverProcessorJob . STATUS_FINISHED ) ;
2007-10-29 02:43:20 +01:00
}
}
2008-08-02 14:12:04 +02:00
} catch ( final Exception e ) {
final ZURL . Entry eentry = errorURL . newEntry (
2008-05-06 01:13:47 +02:00
this . entry ,
2008-05-14 23:36:02 +02:00
sb . webIndex . seedDB . mySeed ( ) . hash ,
2008-05-06 01:13:47 +02:00
new Date ( ) ,
1 ,
e . getMessage ( ) + " - in worker " ) ;
2007-10-29 02:43:20 +01:00
eentry . store ( ) ;
errorURL . push ( eentry ) ;
e . printStackTrace ( ) ;
} finally {
2008-01-14 00:10:09 +01:00
workers . remove ( code ) ;
2008-12-17 23:53:06 +01:00
this . entry . setStatus ( " worker-finalized " , serverProcessorJob . STATUS_FINISHED ) ;
2007-10-29 02:43:20 +01:00
}
}
}
}