2005-04-07 21:19:42 +02:00
// crawlReceipt.java
// -----------------------
// part of the AnomicHTTPD caching proxy
2008-07-20 19:14:51 +02:00
// (C) by Michael Peter Christen; mc@yacy.net
2005-04-07 21:19:42 +02:00
// first published on http://www.anomic.de
// Frankfurt, Germany, 2004
2006-01-18 01:03:28 +01:00
//
// $LastChangedDate$
// $LastChangedRevision$
// $LastChangedBy$
2005-04-07 21:19:42 +02:00
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation; either version 2 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
2005-05-12 19:50:45 +02:00
// You must compile this file with
2005-04-07 21:19:42 +02:00
// javac -classpath .:../classes crawlOrder.java
2005-12-11 01:25:02 +01:00
import java.io.IOException ;
2011-05-27 10:24:54 +02:00
import net.yacy.cora.document.ASCII ;
2010-08-23 14:32:02 +02:00
import net.yacy.cora.protocol.RequestHeader ;
2009-10-11 02:12:19 +02:00
import net.yacy.kelondro.data.meta.URIMetadataRow ;
2009-10-10 01:13:30 +02:00
import net.yacy.kelondro.logging.Log ;
2011-02-12 01:01:40 +01:00
import de.anomic.crawler.ResultURLs ;
import de.anomic.crawler.ResultURLs.EventOrigin ;
2011-05-26 12:57:02 +02:00
import de.anomic.crawler.ZURL.FailCategory ;
2009-10-11 02:12:19 +02:00
import de.anomic.search.Segments ;
2009-07-19 22:37:44 +02:00
import de.anomic.search.Switchboard ;
2005-05-05 07:36:42 +02:00
import de.anomic.server.serverObjects ;
import de.anomic.server.serverSwitch ;
import de.anomic.tools.crypt ;
2007-07-05 01:48:52 +02:00
import de.anomic.yacy.yacyNetwork ;
2006-02-09 09:29:07 +01:00
import de.anomic.yacy.yacySeed ;
2005-04-07 21:19:42 +02:00
2005-10-05 12:45:33 +02:00
public final class crawlReceipt {
2005-04-07 21:19:42 +02:00
2008-03-12 01:05:18 +01:00
2005-04-07 21:19:42 +02:00
/ *
* this is used to respond on a remote crawling request
* /
2008-03-12 01:05:18 +01:00
2009-07-19 22:37:44 +02:00
public static serverObjects respond ( final RequestHeader header , final serverObjects post , final serverSwitch env ) {
2005-11-15 09:40:49 +01:00
// return variable that accumulates replacements
2009-07-19 22:37:44 +02:00
final Switchboard sb = ( Switchboard ) env ;
2008-08-02 14:12:04 +02:00
final serverObjects prop = new serverObjects ( ) ;
2010-11-27 01:54:59 +01:00
if ( ( post = = null ) | | ( env = = null ) | | ! yacyNetwork . authentifyRequest ( post , env ) ) {
return prop ;
}
2008-03-12 01:05:18 +01:00
2009-01-31 00:33:47 +01:00
final Log log = sb . getLog ( ) ;
2005-04-07 21:19:42 +02:00
2005-12-05 00:51:28 +01:00
//int proxyPrefetchDepth = Integer.parseInt(env.getConfig("proxyPrefetchDepth", "0"));
//int crawlingDepth = Integer.parseInt(env.getConfig("crawlingDepth", "0"));
2005-04-07 21:19:42 +02:00
2005-12-05 00:51:28 +01:00
// request values
2008-08-02 14:12:04 +02:00
final String iam = post . get ( " iam " , " " ) ; // seed hash of requester
final String youare = post . get ( " youare " , " " ) ; // seed hash of the target peer, needed for network stability
2005-12-07 00:51:29 +01:00
//String process = post.get("process", ""); // process type
2008-08-02 14:12:04 +02:00
final String key = post . get ( " key " , " " ) ; // transmission key
2007-03-16 14:25:56 +01:00
//String receivedUrlhash = post.get("urlhash", ""); // the url hash that has been crawled
2008-08-02 14:12:04 +02:00
final String result = post . get ( " result " , " " ) ; // the result; either "ok" or "fail"
final String reason = post . get ( " reason " , " " ) ; // the reason for that result
2005-12-07 00:51:29 +01:00
//String words = post.get("wordh", ""); // priority word hashes
2008-08-02 14:12:04 +02:00
final String propStr = crypt . simpleDecode ( post . get ( " lurlEntry " , " " ) , key ) ;
2005-04-07 21:19:42 +02:00
/ *
the result can have one of the following values :
negative cases , no retry
unavailable - the resource is not available ( a broken link ) ; not found or interrupted
exception - an exception occurred
robot - a robot - file has denied to crawl that resource
negative cases , retry possible
rejected - the peer has rejected to load the resource
dequeue - peer too busy - rejected to crawl
positive cases with crawling
fill - the resource was loaded and processed
update - the resource was already in database but re - loaded and processed
positive cases without crawling
known - the resource is already in database , believed to be fresh and not reloaded
stale - the resource was reloaded but not processed because source had no changes
* /
2008-02-08 15:06:34 +01:00
2009-05-28 16:26:05 +02:00
final yacySeed otherPeer = sb . peers . get ( iam ) ;
2008-03-12 01:05:18 +01:00
final String otherPeerName = iam + " : " + ( ( otherPeer = = null ) ? " NULL " : ( otherPeer . getName ( ) + " / " + otherPeer . getVersion ( ) ) ) ;
2007-03-16 14:25:56 +01:00
2009-05-28 16:26:05 +02:00
if ( ( sb . peers . mySeed ( ) = = null ) | | ( ! ( sb . peers . mySeed ( ) . hash . equals ( youare ) ) ) ) {
2005-04-07 21:19:42 +02:00
// no yacy connection / unknown peers
2008-03-12 01:05:18 +01:00
prop . put ( " delay " , " 3600 " ) ;
2007-03-16 14:25:56 +01:00
return prop ;
}
if ( propStr = = null ) {
2005-04-07 21:19:42 +02:00
// error with url / wrong key
2008-03-12 01:05:18 +01:00
prop . put ( " delay " , " 3600 " ) ;
2007-03-16 14:25:56 +01:00
return prop ;
}
2008-05-06 01:13:47 +02:00
if ( ( sb . isRobinsonMode ( ) ) & & ( ! sb . isInMyCluster ( otherPeer ) ) ) {
2007-04-24 17:11:12 +02:00
// we reject urls that are from outside our cluster
2007-10-24 23:38:19 +02:00
prop . put ( " delay " , " 9999 " ) ;
2010-02-02 10:42:14 +01:00
return prop ;
2007-04-24 17:11:12 +02:00
}
2007-03-16 14:25:56 +01:00
// generating a new loaded URL entry
2009-10-11 02:12:19 +02:00
final URIMetadataRow entry = URIMetadataRow . importEntry ( propStr ) ;
2007-03-16 14:25:56 +01:00
if ( entry = = null ) {
2010-03-26 19:33:20 +01:00
if ( log . isWarning ( ) ) log . logWarning ( " crawlReceipt: RECEIVED wrong RECEIPT (entry null) from peer " + iam + " \ n \ tURL properties: " + propStr ) ;
2008-03-12 01:05:18 +01:00
prop . put ( " delay " , " 3600 " ) ;
2007-03-16 14:25:56 +01:00
return prop ;
}
2009-10-11 02:12:19 +02:00
final URIMetadataRow . Components metadata = entry . metadata ( ) ;
2009-03-02 12:04:13 +01:00
if ( metadata . url ( ) = = null ) {
2011-05-27 10:24:54 +02:00
if ( log . isWarning ( ) ) log . logWarning ( " crawlReceipt: RECEIVED wrong RECEIPT (url null) for hash " + ASCII . String ( entry . hash ( ) ) + " from peer " + iam + " \ n \ tURL properties: " + propStr ) ;
2008-03-12 01:05:18 +01:00
prop . put ( " delay " , " 3600 " ) ;
2007-03-16 14:25:56 +01:00
return prop ;
}
2007-07-24 02:46:17 +02:00
// check if the entry is in our network domain
2009-03-02 12:04:13 +01:00
final String urlRejectReason = sb . crawlStacker . urlInAcceptedDomain ( metadata . url ( ) ) ;
2008-04-20 23:36:25 +02:00
if ( urlRejectReason ! = null ) {
2011-05-27 10:24:54 +02:00
if ( log . isWarning ( ) ) log . logWarning ( " crawlReceipt: RECEIVED wrong RECEIPT ( " + urlRejectReason + " ) for hash " + ASCII . String ( entry . hash ( ) ) + " from peer " + iam + " \ n \ tURL properties: " + propStr ) ;
2007-10-24 23:38:19 +02:00
prop . put ( " delay " , " 9999 " ) ;
2007-07-24 02:46:17 +02:00
return prop ;
}
2010-11-27 01:54:59 +01:00
if ( " fill " . equals ( result ) ) try {
2007-03-16 14:25:56 +01:00
// put new entry into database
2009-10-09 16:44:20 +02:00
sb . indexSegments . urlMetadata ( Segments . Process . RECEIPTS ) . store ( entry ) ;
2011-02-12 01:01:40 +01:00
ResultURLs . stack ( entry , youare . getBytes ( ) , iam . getBytes ( ) , EventOrigin . REMOTE_RECEIPTS ) ;
2008-05-06 01:13:47 +02:00
sb . crawlQueues . delegatedURL . remove ( entry . hash ( ) ) ; // the delegated work has been done
2011-05-27 10:24:54 +02:00
if ( log . isInfo ( ) ) log . logInfo ( " crawlReceipt: RECEIVED RECEIPT from " + otherPeerName + " for URL " + ASCII . String ( entry . hash ( ) ) + " : " + metadata . url ( ) . toNormalform ( false , true ) ) ;
2007-03-16 14:25:56 +01:00
2005-04-07 21:19:42 +02:00
// ready for more
2007-10-24 23:38:19 +02:00
prop . put ( " delay " , " 10 " ) ;
2007-03-16 14:25:56 +01:00
return prop ;
2008-08-02 14:12:04 +02:00
} catch ( final IOException e ) {
2009-11-05 21:28:37 +01:00
Log . logException ( e ) ;
2008-03-12 01:05:18 +01:00
prop . put ( " delay " , " 3600 " ) ;
2007-03-16 14:25:56 +01:00
return prop ;
2005-04-07 21:19:42 +02:00
}
2007-03-16 14:25:56 +01:00
2008-05-06 01:13:47 +02:00
sb . crawlQueues . delegatedURL . remove ( entry . hash ( ) ) ; // the delegated work is transformed into an error case
2009-10-31 12:58:06 +01:00
sb . crawlQueues . errorURL . push (
2008-05-06 01:13:47 +02:00
entry . toBalancerEntry ( iam ) ,
2010-04-08 02:11:32 +02:00
youare . getBytes ( ) ,
2008-05-06 01:13:47 +02:00
null ,
0 ,
2011-05-26 12:57:02 +02:00
FailCategory . FINAL_LOAD_CONTEXT ,
2011-05-02 16:05:51 +02:00
result + " : " + reason , - 1 ) ;
2007-03-16 14:25:56 +01:00
//switchboard.noticeURL.remove(receivedUrlhash);
2008-03-12 01:05:18 +01:00
prop . put ( " delay " , " 3600 " ) ;
2007-03-16 14:25:56 +01:00
return prop ;
// return rewrite properties
2005-04-07 21:19:42 +02:00
}
}