2005-10-10 01:11:17 +02:00
// plasmaCrawlStacker.java
2005-10-09 17:59:09 +02:00
// -----------------------
// part of YaCy
2008-07-20 19:14:51 +02:00
// (C) by Michael Peter Christen; mc@yacy.net
2005-10-09 17:59:09 +02:00
// first published on http://www.anomic.de
// Frankfurt, Germany, 2005
//
// This file was contributed by Martin Thelian
2007-10-29 02:43:20 +01:00
// ([MC] removed all multithreading and thread pools, this is not necessary here; complete renovation 2007)
2005-10-09 17:59:09 +02:00
//
2005-11-04 14:41:51 +01:00
// $LastChangedDate$
// $LastChangedRevision$
// $LastChangedBy$
2005-10-09 17:59:09 +02:00
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation; either version 2 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
2008-05-06 02:32:41 +02:00
package de.anomic.crawler ;
2005-10-05 12:45:33 +02:00
2007-10-29 02:43:20 +01:00
import java.net.UnknownHostException ;
2005-10-05 12:45:33 +02:00
import java.util.Date ;
2009-10-11 02:12:19 +02:00
import net.yacy.kelondro.data.meta.DigestURI ;
import net.yacy.kelondro.data.meta.URIMetadataRow ;
2009-10-10 01:13:30 +02:00
import net.yacy.kelondro.logging.Log ;
2009-10-11 02:12:19 +02:00
import net.yacy.kelondro.util.Domains ;
import net.yacy.kelondro.workflow.WorkflowProcessor ;
2009-10-21 22:14:30 +02:00
import net.yacy.repository.Blacklist ;
2009-10-10 01:13:30 +02:00
2009-07-15 23:07:46 +02:00
import de.anomic.crawler.retrieval.Request ;
2009-10-11 02:12:19 +02:00
import de.anomic.search.Segment ;
2009-07-19 22:37:44 +02:00
import de.anomic.search.Switchboard ;
2009-05-28 16:26:05 +02:00
import de.anomic.yacy.yacySeedDB ;
2005-10-05 12:45:33 +02:00
2008-12-15 01:02:58 +01:00
public final class CrawlStacker {
2009-03-16 19:08:43 +01:00
private Log log = new Log ( " STACKCRAWL " ) ;
2009-10-11 02:12:19 +02:00
private WorkflowProcessor < Request > fastQueue , slowQueue ;
2008-12-17 23:53:06 +01:00
private long dnsHit , dnsMiss ;
private CrawlQueues nextQueue ;
2009-05-28 16:26:05 +02:00
private CrawlSwitchboard crawler ;
private Segment indexSegment ;
private yacySeedDB peers ;
2008-12-17 23:53:06 +01:00
private boolean acceptLocalURLs , acceptGlobalURLs ;
2009-03-16 19:08:43 +01:00
2008-12-15 01:02:58 +01:00
// this is the process that checks url for double-occurrences and for allowance/disallowance by robots.txt
2009-03-16 19:08:43 +01:00
2009-05-28 16:26:05 +02:00
public CrawlStacker (
CrawlQueues cq ,
CrawlSwitchboard cs ,
Segment indexSegment ,
yacySeedDB peers ,
boolean acceptLocalURLs ,
boolean acceptGlobalURLs ) {
2008-12-15 01:02:58 +01:00
this . nextQueue = cq ;
2009-05-28 16:26:05 +02:00
this . crawler = cs ;
this . indexSegment = indexSegment ;
this . peers = peers ;
2007-11-01 01:57:32 +01:00
this . dnsHit = 0 ;
this . dnsMiss = 0 ;
2008-12-15 01:02:58 +01:00
this . acceptLocalURLs = acceptLocalURLs ;
this . acceptGlobalURLs = acceptGlobalURLs ;
2009-03-16 19:08:43 +01:00
2009-10-11 02:12:19 +02:00
this . fastQueue = new WorkflowProcessor < Request > ( " CrawlStackerFast " , " This process checks new urls before they are enqueued into the balancer (proper, double-check, correct domain, filter) " , new String [ ] { " Balancer " } , this , " job " , 10000 , null , 2 ) ;
this . slowQueue = new WorkflowProcessor < Request > ( " CrawlStackerSlow " , " This is like CrawlStackerFast, but does additionaly a DNS lookup. The CrawlStackerFast does not need this because it can use the DNS cache. " , new String [ ] { " Balancer " } , this , " job " , 1000 , null , 5 ) ;
2009-03-16 19:08:43 +01:00
2007-10-29 02:43:20 +01:00
this . log . logInfo ( " STACKCRAWL thread initialized. " ) ;
}
2005-11-15 13:46:22 +01:00
2008-12-15 01:02:58 +01:00
public int size ( ) {
2008-12-19 00:18:34 +01:00
return this . fastQueue . queueSize ( ) + this . slowQueue . queueSize ( ) ;
2006-10-17 23:01:35 +02:00
}
2009-12-02 01:37:59 +01:00
public boolean isEmpty ( ) {
if ( ! this . fastQueue . queueIsEmpty ( ) ) return false ;
if ( ! this . slowQueue . queueIsEmpty ( ) ) return false ;
return true ;
}
2008-12-15 01:02:58 +01:00
2008-12-17 23:53:06 +01:00
public void clear ( ) {
this . fastQueue . clear ( ) ;
this . slowQueue . clear ( ) ;
2008-06-04 23:34:57 +02:00
}
2009-03-16 19:08:43 +01:00
2008-12-19 16:26:01 +01:00
public void announceClose ( ) {
this . log . logInfo ( " Flushing remaining " + size ( ) + " crawl stacker job entries. " ) ;
this . fastQueue . announceShutdown ( ) ;
this . slowQueue . announceShutdown ( ) ;
}
2009-03-16 19:08:43 +01:00
2005-10-09 18:40:44 +02:00
public void close ( ) {
2008-12-19 16:26:01 +01:00
this . log . logInfo ( " Shutdown. waiting for remaining " + size ( ) + " crawl stacker job entries. please wait. " ) ;
2008-12-19 00:18:34 +01:00
this . fastQueue . announceShutdown ( ) ;
this . slowQueue . announceShutdown ( ) ;
this . fastQueue . awaitShutdown ( 2000 ) ;
this . slowQueue . awaitShutdown ( 2000 ) ;
2009-03-16 19:08:43 +01:00
2008-04-08 16:44:39 +02:00
this . log . logInfo ( " Shutdown. Closing stackCrawl queue. " ) ;
2007-10-29 02:43:20 +01:00
2008-12-17 23:53:06 +01:00
clear ( ) ;
2005-10-05 12:45:33 +02:00
}
2008-12-15 01:02:58 +01:00
private boolean prefetchHost ( final String host ) {
// returns true when the host was known in the dns cache.
// If not, the host is stacked on the fetch stack and false is returned
try {
2009-10-11 02:12:19 +02:00
if ( Domains . dnsResolveFromCache ( host ) ! = null ) return true ; // found entry
2008-12-15 01:02:58 +01:00
} catch ( final UnknownHostException e ) {
2008-12-23 11:06:49 +01:00
// we know that this is unknown
2008-12-15 01:02:58 +01:00
return false ;
}
2008-12-23 11:06:49 +01:00
// we just don't know anything about that host
return false ;
2008-12-15 01:02:58 +01:00
}
2009-03-16 19:08:43 +01:00
2008-12-19 00:18:34 +01:00
/ *
2007-10-29 02:43:20 +01:00
public boolean job ( ) {
2008-12-19 00:18:34 +01:00
if ( this . fastQueue . queueSize ( ) > 0 & & job ( this . fastQueue ) ) return true ;
if ( this . slowQueue . queueSize ( ) = = 0 ) return false ;
2008-12-17 23:53:06 +01:00
return job ( this . slowQueue ) ;
}
2008-12-19 00:18:34 +01:00
* /
2009-03-16 19:08:43 +01:00
2009-07-15 23:07:46 +02:00
public Request job ( Request entry ) {
2008-12-15 01:02:58 +01:00
// this is the method that is called by the busy thread from outside
2008-12-19 00:18:34 +01:00
if ( entry = = null ) return null ;
2007-10-29 02:43:20 +01:00
try {
2008-12-15 01:02:58 +01:00
final String rejectReason = stackCrawl ( entry ) ;
2007-10-29 02:43:20 +01:00
// if the url was rejected we store it into the error URL db
if ( rejectReason ! = null ) {
2009-10-31 12:58:06 +01:00
nextQueue . errorURL . push ( entry , peers . mySeed ( ) . hash , new Date ( ) , 1 , rejectReason ) ;
2006-11-30 00:09:56 +01:00
}
2008-08-02 14:12:04 +02:00
} catch ( final Exception e ) {
2008-05-06 02:32:41 +02:00
CrawlStacker . this . log . logWarning ( " Error while processing stackCrawl entry. \ n " + " Entry: " + entry . toString ( ) + " Error: " + e . toString ( ) , e ) ;
2008-12-19 00:18:34 +01:00
return null ;
2005-10-05 12:45:33 +02:00
}
2008-12-19 00:18:34 +01:00
return null ;
2005-10-05 12:45:33 +02:00
}
2009-03-16 19:08:43 +01:00
2009-07-15 23:07:46 +02:00
public void enqueueEntry ( final Request entry ) {
2009-03-16 19:08:43 +01:00
2008-08-26 18:34:24 +02:00
// DEBUG
2009-03-16 19:08:43 +01:00
if ( log . isFinest ( ) ) log . logFinest ( " ENQUEUE " + entry . url ( ) + " , referer= " + entry . referrerhash ( ) + " , initiator= " + entry . initiator ( ) + " , name= " + entry . name ( ) + " , load= " + entry . loaddate ( ) + " , depth= " + entry . depth ( ) ) ;
2008-06-18 01:56:39 +02:00
2008-12-17 23:53:06 +01:00
if ( prefetchHost ( entry . url ( ) . getHost ( ) ) ) {
2007-10-29 02:43:20 +01:00
try {
2008-12-19 00:18:34 +01:00
this . fastQueue . enQueue ( entry ) ;
2008-12-17 23:53:06 +01:00
this . dnsHit + + ;
} catch ( InterruptedException e ) {
2009-11-05 21:28:37 +01:00
Log . logException ( e ) ;
2008-12-17 23:53:06 +01:00
}
} else {
try {
2008-12-19 00:18:34 +01:00
this . slowQueue . enQueue ( entry ) ;
2009-03-16 19:08:43 +01:00
this . dnsMiss + + ;
2008-12-17 23:53:06 +01:00
} catch ( InterruptedException e ) {
2009-11-05 21:28:37 +01:00
Log . logException ( e ) ;
2007-10-29 02:43:20 +01:00
}
2005-10-05 12:45:33 +02:00
}
}
2009-03-16 19:08:43 +01:00
2009-07-15 23:07:46 +02:00
public String stackCrawl ( final Request entry ) {
2005-10-05 12:45:33 +02:00
// stacks a crawl item. The position can also be remote
// returns null if successful, a reason string if not successful
2006-10-11 02:46:45 +02:00
//this.log.logFinest("stackCrawl: nexturlString='" + nexturlString + "'");
2009-03-16 19:08:43 +01:00
2008-08-02 14:12:04 +02:00
final long startTime = System . currentTimeMillis ( ) ;
2006-09-04 13:46:17 +02:00
// check if the protocol is supported
2008-08-02 14:12:04 +02:00
final String urlProtocol = entry . url ( ) . getProtocol ( ) ;
2009-07-23 23:31:51 +02:00
if ( ! Switchboard . getSwitchboard ( ) . loader . isSupportedProtocol ( urlProtocol ) ) {
2009-03-16 19:08:43 +01:00
this . log . logSevere ( " Unsupported protocol in URL ' " + entry . url ( ) . toString ( ) + " '. " +
" Stack processing time: " + ( System . currentTimeMillis ( ) - startTime ) + " ms " ) ;
return " unsupported protocol " ;
2006-09-04 13:46:17 +02:00
}
2007-10-29 02:43:20 +01:00
2005-10-05 12:45:33 +02:00
// check if ip is local ip address
2008-12-15 01:02:58 +01:00
final String urlRejectReason = urlInAcceptedDomain ( entry . url ( ) ) ;
2008-04-20 23:36:25 +02:00
if ( urlRejectReason ! = null ) {
2009-03-16 19:08:43 +01:00
if ( this . log . isFine ( ) ) this . log . logFine ( " denied_( " + urlRejectReason + " ) Stack processing time: " + ( System . currentTimeMillis ( ) - startTime ) + " ms " ) ;
return " denied_( " + urlRejectReason + " ) " ;
2005-10-05 12:45:33 +02:00
}
2009-03-16 19:08:43 +01:00
2005-10-05 12:45:33 +02:00
// check blacklist
2009-07-19 22:37:44 +02:00
if ( Switchboard . urlBlacklist . isListed ( Blacklist . BLACKLIST_CRAWLER , entry . url ( ) ) ) {
2008-04-08 16:44:39 +02:00
if ( this . log . isFine ( ) ) this . log . logFine ( " URL ' " + entry . url ( ) . toString ( ) + " ' is in blacklist. " +
2009-03-16 19:08:43 +01:00
" Stack processing time: " + ( System . currentTimeMillis ( ) - startTime ) + " ms " ) ;
return " url in blacklist " ;
2007-10-29 02:43:20 +01:00
}
2009-03-16 19:08:43 +01:00
2009-05-28 16:26:05 +02:00
final CrawlProfile . entry profile = crawler . profilesActiveCrawls . getEntry ( entry . profileHandle ( ) ) ;
2007-10-29 02:43:20 +01:00
if ( profile = = null ) {
2008-10-08 20:26:36 +02:00
final String errorMsg = " LOST STACKER PROFILE HANDLE ' " + entry . profileHandle ( ) + " ' for URL " + entry . url ( ) ;
2007-10-29 02:43:20 +01:00
log . logWarning ( errorMsg ) ;
return errorMsg ;
2008-08-02 15:57:00 +02:00
}
2009-03-16 19:08:43 +01:00
2008-11-14 10:58:56 +01:00
// filter with must-match
if ( ( entry . depth ( ) > 0 ) & & ! profile . mustMatchPattern ( ) . matcher ( entry . url ( ) . toString ( ) ) . matches ( ) ) {
if ( this . log . isFine ( ) ) this . log . logFine ( " URL ' " + entry . url ( ) . toString ( ) + " ' does not match must-match crawling filter ' " + profile . mustMatchPattern ( ) . toString ( ) + " '. " +
2009-03-16 19:08:43 +01:00
" Stack processing time: " + ( System . currentTimeMillis ( ) - startTime ) + " ms " ) ;
return " url does not match must-match filter " ;
2005-10-05 12:45:33 +02:00
}
2008-11-14 10:58:56 +01:00
// filter with must-not-match
if ( ( entry . depth ( ) > 0 ) & & profile . mustNotMatchPattern ( ) . matcher ( entry . url ( ) . toString ( ) ) . matches ( ) ) {
if ( this . log . isFine ( ) ) this . log . logFine ( " URL ' " + entry . url ( ) . toString ( ) + " ' does matches do-not-match crawling filter ' " + profile . mustNotMatchPattern ( ) . toString ( ) + " '. " +
2009-03-16 19:08:43 +01:00
" Stack processing time: " + ( System . currentTimeMillis ( ) - startTime ) + " ms " ) ;
return " url matches must-not-match filter " ;
2008-11-14 10:58:56 +01:00
}
2005-10-05 12:45:33 +02:00
// deny cgi
2009-04-20 15:28:28 +02:00
if ( entry . url ( ) . isIndividual ( ) ) {
2009-03-16 19:08:43 +01:00
if ( this . log . isFine ( ) ) this . log . logFine ( " URL ' " + entry . url ( ) . toString ( ) + " ' is CGI URL. " +
" Stack processing time: " + ( System . currentTimeMillis ( ) - startTime ) + " ms " ) ;
return " cgi url not allowed " ;
2005-10-05 12:45:33 +02:00
}
2009-03-16 19:08:43 +01:00
2005-10-05 12:45:33 +02:00
// deny post properties
2008-06-06 18:01:27 +02:00
if ( entry . url ( ) . isPOST ( ) & & ! ( profile . crawlingQ ( ) ) ) {
2009-03-16 19:08:43 +01:00
if ( this . log . isFine ( ) ) this . log . logFine ( " URL ' " + entry . url ( ) . toString ( ) + " ' is post URL. " +
" Stack processing time: " + ( System . currentTimeMillis ( ) - startTime ) + " ms " ) ;
return " post url not allowed " ;
2005-10-05 12:45:33 +02:00
}
2009-03-16 19:08:43 +01:00
2009-10-11 02:12:19 +02:00
final DigestURI referrerURL = ( entry . referrerhash ( ) = = null ) ? null : nextQueue . getURL ( entry . referrerhash ( ) ) ;
2009-03-16 19:08:43 +01:00
2006-03-23 17:05:16 +01:00
// add domain to profile domain list
2006-04-14 01:19:36 +02:00
if ( ( profile . domFilterDepth ( ) ! = Integer . MAX_VALUE ) | | ( profile . domMaxPages ( ) ! = Integer . MAX_VALUE ) ) {
2007-10-29 02:43:20 +01:00
profile . domInc ( entry . url ( ) . getHost ( ) , ( referrerURL = = null ) ? null : referrerURL . getHost ( ) . toLowerCase ( ) , entry . depth ( ) ) ;
2006-03-23 17:05:16 +01:00
}
// deny urls that do not match with the profile domain list
2007-10-29 02:43:20 +01:00
if ( ! ( profile . grantedDomAppearance ( entry . url ( ) . getHost ( ) ) ) ) {
2009-03-16 19:08:43 +01:00
if ( this . log . isFine ( ) ) this . log . logFine ( " URL ' " + entry . url ( ) . toString ( ) + " ' is not listed in granted domains. " +
" Stack processing time: " + ( System . currentTimeMillis ( ) - startTime ) + " ms " ) ;
return " url does not match domain filter " ;
2006-03-23 17:05:16 +01:00
}
// deny urls that exceed allowed number of occurrences
2007-10-29 02:43:20 +01:00
if ( ! ( profile . grantedDomCount ( entry . url ( ) . getHost ( ) ) ) ) {
2009-03-16 19:08:43 +01:00
if ( this . log . isFine ( ) ) this . log . logFine ( " URL ' " + entry . url ( ) . toString ( ) + " ' appeared too often, a maximum of " + profile . domMaxPages ( ) + " is allowed. " +
" Stack processing time: " + ( System . currentTimeMillis ( ) - startTime ) + " ms " ) ;
return " domain counter exceeded " ;
2006-03-23 17:05:16 +01:00
}
2006-09-03 16:59:00 +02:00
// check if the url is double registered
2009-07-30 11:08:44 +02:00
final String dbocc = nextQueue . urlExists ( entry . url ( ) . hash ( ) ) ; // returns the name of the queue if entry exists
2009-10-11 02:12:19 +02:00
URIMetadataRow oldEntry = null ;
2009-07-30 11:08:44 +02:00
if ( dbocc ! = null | | ( oldEntry = indexSegment . urlMetadata ( ) . load ( entry . url ( ) . hash ( ) , null , 0 ) ) ! = null ) {
2009-01-09 01:06:36 +01:00
final boolean recrawl = ( oldEntry ! = null ) & & ( profile . recrawlIfOlder ( ) > oldEntry . loaddate ( ) . getTime ( ) ) ;
// do double-check
if ( ( dbocc ! = null ) & & ( ! recrawl ) ) {
2009-03-16 19:08:43 +01:00
if ( this . log . isFine ( ) ) this . log . logFine ( " URL ' " + entry . url ( ) . toString ( ) + " ' is double registered in ' " + dbocc + " '. " + " Stack processing time: " + ( System . currentTimeMillis ( ) - startTime ) + " ms " ) ;
return " double " + dbocc ;
2009-01-09 01:06:36 +01:00
}
if ( ( oldEntry ! = null ) & & ( ! recrawl ) ) {
2009-03-16 19:08:43 +01:00
if ( this . log . isFine ( ) ) this . log . logFine ( " URL ' " + entry . url ( ) . toString ( ) + " ' is double registered in 'LURL'. " + " Stack processing time: " + ( System . currentTimeMillis ( ) - startTime ) + " ms " ) ;
return " double LURL " ;
2009-01-09 01:06:36 +01:00
}
2009-03-16 19:08:43 +01:00
2009-01-09 01:06:36 +01:00
// show potential re-crawl
if ( recrawl & & oldEntry ! = null ) {
if ( this . log . isFine ( ) ) this . log . logFine ( " RE-CRAWL of URL ' " + entry . url ( ) . toString ( ) + " ': this url was crawled " +
( ( System . currentTimeMillis ( ) - oldEntry . loaddate ( ) . getTime ( ) ) / 60000 / 60 / 24 ) + " days ago. " ) ;
}
2006-03-23 17:05:16 +01:00
}
2009-03-16 19:08:43 +01:00
2005-10-05 12:45:33 +02:00
// store information
2009-05-28 16:26:05 +02:00
final boolean local = entry . initiator ( ) . equals ( peers . mySeed ( ) . hash ) ;
2009-07-16 01:08:35 +02:00
final boolean proxy = ( entry . initiator ( ) = = null | | entry . initiator ( ) . length ( ) = = 0 | | entry . initiator ( ) . equals ( " ------------ " ) ) & & profile . handle ( ) . equals ( crawler . defaultProxyProfile . handle ( ) ) ;
2009-05-28 16:26:05 +02:00
final boolean remote = profile . handle ( ) . equals ( crawler . defaultRemoteProfile . handle ( ) ) ;
2009-03-16 19:08:43 +01:00
final boolean global =
2005-10-05 12:45:33 +02:00
( profile . remoteIndexing ( ) ) /* granted */ & &
2009-03-16 19:08:43 +01:00
( entry . depth ( ) = = profile . depth ( ) ) /* leaf node */ & &
2006-02-24 10:35:54 +01:00
//(initiatorHash.equals(yacyCore.seedDB.mySeed.hash)) /* not proxy */ &&
(
2009-05-28 16:26:05 +02:00
( peers . mySeed ( ) . isSenior ( ) ) | |
( peers . mySeed ( ) . isPrincipal ( ) )
2006-02-24 10:35:54 +01:00
) /* qualified */ ;
2009-03-16 19:08:43 +01:00
2008-09-16 23:56:23 +02:00
if ( ! local & & ! global & & ! remote & & ! proxy ) {
2009-03-31 09:51:32 +02:00
String error = " URL ' " + entry . url ( ) . toString ( ) + " ' cannot be crawled. initiator = " + entry . initiator ( ) + " , profile.handle = " + profile . handle ( ) ;
this . log . logSevere ( error ) ;
return error ;
}
if ( global ) {
// it may be possible that global == true and local == true, so do not check an error case against it
if ( proxy ) this . log . logWarning ( " URL ' " + entry . url ( ) . toString ( ) + " ' has conflicting initiator properties: global = true, proxy = true, initiator = " + entry . initiator ( ) + " , profile.handle = " + profile . handle ( ) ) ;
if ( remote ) this . log . logWarning ( " URL ' " + entry . url ( ) . toString ( ) + " ' has conflicting initiator properties: global = true, remote = true, initiator = " + entry . initiator ( ) + " , profile.handle = " + profile . handle ( ) ) ;
2009-03-31 14:42:12 +02:00
//int b = nextQueue.noticeURL.stackSize(NoticedURL.STACK_TYPE_LIMIT);
2009-03-31 09:51:32 +02:00
nextQueue . noticeURL . push ( NoticedURL . STACK_TYPE_LIMIT , entry ) ;
2009-03-31 14:42:12 +02:00
//assert b < nextQueue.noticeURL.stackSize(NoticedURL.STACK_TYPE_LIMIT);
2009-04-01 22:13:57 +02:00
//this.log.logInfo("stacked/global: " + entry.url().toString() + ", stacksize = " + nextQueue.noticeURL.stackSize(NoticedURL.STACK_TYPE_LIMIT));
2009-03-31 09:51:32 +02:00
} else if ( local ) {
if ( proxy ) this . log . logWarning ( " URL ' " + entry . url ( ) . toString ( ) + " ' has conflicting initiator properties: local = true, proxy = true, initiator = " + entry . initiator ( ) + " , profile.handle = " + profile . handle ( ) ) ;
if ( remote ) this . log . logWarning ( " URL ' " + entry . url ( ) . toString ( ) + " ' has conflicting initiator properties: local = true, remote = true, initiator = " + entry . initiator ( ) + " , profile.handle = " + profile . handle ( ) ) ;
2009-03-31 14:42:12 +02:00
//int b = nextQueue.noticeURL.stackSize(NoticedURL.STACK_TYPE_CORE);
2009-03-31 09:51:32 +02:00
nextQueue . noticeURL . push ( NoticedURL . STACK_TYPE_CORE , entry ) ;
2009-03-31 14:42:12 +02:00
//assert b < nextQueue.noticeURL.stackSize(NoticedURL.STACK_TYPE_CORE);
2009-04-01 22:13:57 +02:00
//this.log.logInfo("stacked/local: " + entry.url().toString() + ", stacksize = " + nextQueue.noticeURL.stackSize(NoticedURL.STACK_TYPE_CORE));
2009-03-31 09:51:32 +02:00
} else if ( proxy ) {
if ( remote ) this . log . logWarning ( " URL ' " + entry . url ( ) . toString ( ) + " ' has conflicting initiator properties: proxy = true, remote = true, initiator = " + entry . initiator ( ) + " , profile.handle = " + profile . handle ( ) ) ;
2009-03-31 14:42:12 +02:00
//int b = nextQueue.noticeURL.stackSize(NoticedURL.STACK_TYPE_CORE);
2009-03-31 09:51:32 +02:00
nextQueue . noticeURL . push ( NoticedURL . STACK_TYPE_CORE , entry ) ;
2009-03-31 14:42:12 +02:00
//assert b < nextQueue.noticeURL.stackSize(NoticedURL.STACK_TYPE_CORE);
2009-04-01 22:13:57 +02:00
//this.log.logInfo("stacked/proxy: " + entry.url().toString() + ", stacksize = " + nextQueue.noticeURL.stackSize(NoticedURL.STACK_TYPE_CORE));
2009-03-31 09:51:32 +02:00
} else if ( remote ) {
2009-03-31 14:42:12 +02:00
//int b = nextQueue.noticeURL.stackSize(NoticedURL.STACK_TYPE_REMOTE);
2009-03-31 09:51:32 +02:00
nextQueue . noticeURL . push ( NoticedURL . STACK_TYPE_REMOTE , entry ) ;
2009-03-31 14:42:12 +02:00
//assert b < nextQueue.noticeURL.stackSize(NoticedURL.STACK_TYPE_REMOTE);
2009-04-01 22:13:57 +02:00
//this.log.logInfo("stacked/remote: " + entry.url().toString() + ", stacksize = " + nextQueue.noticeURL.stackSize(NoticedURL.STACK_TYPE_REMOTE));
2005-10-05 12:45:33 +02:00
}
2009-03-16 19:08:43 +01:00
2005-10-05 12:45:33 +02:00
return null ;
}
2008-12-15 01:02:58 +01:00
/ * *
* Test a url if it can be used for crawling / indexing
* This mainly checks if the url is in the declared domain ( local / global )
* @param url
* @return null if the url can be accepted , a string containing a rejection reason if the url cannot be accepted
* /
2009-10-11 02:12:19 +02:00
public String urlInAcceptedDomain ( final DigestURI url ) {
2008-12-15 01:02:58 +01:00
// returns true if the url can be accepted accoring to network.unit.domain
if ( url = = null ) return " url is null " ;
final String host = url . getHost ( ) ;
if ( host = = null ) return " url.host is null " ;
if ( this . acceptGlobalURLs & & this . acceptLocalURLs ) return null ; // fast shortcut to avoid dnsResolve
// check if this is a local address and we are allowed to index local pages:
//boolean local = hostAddress.isSiteLocalAddress() || hostAddress.isLoopbackAddress();
final boolean local = url . isLocal ( ) ;
//assert local == yacyURL.isLocalDomain(url.hash()); // TODO: remove the dnsResolve above!
if ( ( this . acceptGlobalURLs & & ! local ) | | ( this . acceptLocalURLs & & local ) ) return null ;
return ( local ) ?
( " the host ' " + host + " ' is local, but local addresses are not accepted " ) :
( " the host ' " + host + " ' is global, but global addresses are not accepted " ) ;
}
2009-04-20 08:38:28 +02:00
public String urlInAcceptedDomainHash ( final String urlhash ) {
// returns true if the url can be accepted accoring to network.unit.domain
if ( urlhash = = null ) return " url is null " ;
if ( this . acceptGlobalURLs & & this . acceptLocalURLs ) return null ; // fast shortcut to avoid dnsResolve
// check if this is a local address and we are allowed to index local pages:
//boolean local = hostAddress.isSiteLocalAddress() || hostAddress.isLoopbackAddress();
2009-10-11 02:12:19 +02:00
final boolean local = DigestURI . isLocal ( urlhash ) ;
2009-04-20 08:38:28 +02:00
//assert local == yacyURL.isLocalDomain(url.hash()); // TODO: remove the dnsResolve above!
if ( ( this . acceptGlobalURLs & & ! local ) | | ( this . acceptLocalURLs & & local ) ) return null ;
return ( local ) ?
( " the urlhash ' " + urlhash + " ' is local, but local addresses are not accepted " ) :
( " the urlhash ' " + urlhash + " ' is global, but global addresses are not accepted " ) ;
}
2009-03-16 19:08:43 +01:00
2008-12-15 01:02:58 +01:00
public boolean acceptLocalURLs ( ) {
return this . acceptLocalURLs ;
}
2009-03-16 19:08:43 +01:00
2008-12-15 01:02:58 +01:00
public boolean acceptGlobalURLs ( ) {
return this . acceptGlobalURLs ;
}
2005-12-03 10:58:00 +01:00
}