// Response.java // (C) 2008 by Michael Peter Christen; mc@yacy.net, Frankfurt a. M., Germany // first published 19.08.2008 on http://yacy.net // // This is a part of YaCy, a peer-to-peer based web search engine // // $LastChangedDate$ // $LastChangedRevision$ // $LastChangedBy$ // // LICENSE // // This program is free software; you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by // the Free Software Foundation; either version 2 of the License, or // (at your option) any later version. // // This program is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU General Public License // along with this program; if not, write to the Free Software // Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA package de.anomic.crawler.retrieval; import java.io.ByteArrayInputStream; import java.util.Date; import net.yacy.cora.date.GenericFormatter; import net.yacy.cora.document.UTF8; import net.yacy.cora.protocol.HeaderFramework; import net.yacy.cora.protocol.RequestHeader; import net.yacy.cora.protocol.ResponseHeader; import net.yacy.document.Classification; import net.yacy.document.Document; import net.yacy.document.Parser; import net.yacy.document.TextParser; import net.yacy.kelondro.data.meta.DigestURI; import de.anomic.crawler.CrawlProfile; import de.anomic.crawler.ResultURLs.EventOrigin; public class Response { // doctypes: public static final char DT_PDFPS = 'p'; public static final char DT_TEXT = 't'; public static final char DT_HTML = 'h'; public static final char DT_DOC = 'd'; public static final char DT_IMAGE = 'i'; public static final char DT_MOVIE = 'm'; public static final char DT_FLASH = 'f'; public static final char DT_SHARE = 's'; public static final char DT_AUDIO = 'a'; public static final char DT_BINARY = 'b'; public static final char DT_UNKNOWN = 'u'; // the class objects private final Request request; private final RequestHeader requestHeader; private final ResponseHeader responseHeader; private final String responseStatus; private final CrawlProfile profile; private byte[] content; private int status; // tracker indexing status, see status defs below // doctype calculation public static char docType(final DigestURI url) { final String path = url.getPath().toLowerCase(); // serverLog.logFinest("PLASMA", "docType URL=" + path); char doctype = DT_UNKNOWN; if (path.endsWith(".gif")) { doctype = DT_IMAGE; } else if (path.endsWith(".ico")) { doctype = DT_IMAGE; } else if (path.endsWith(".bmp")) { doctype = DT_IMAGE; } else if (path.endsWith(".jpg")) { doctype = DT_IMAGE; } else if (path.endsWith(".jpeg")) { doctype = DT_IMAGE; } else if (path.endsWith(".png")) { doctype = DT_IMAGE; } else if (path.endsWith(".html")) { doctype = DT_HTML; } else if (path.endsWith(".txt")) { doctype = DT_TEXT; } else if (path.endsWith(".doc")) { doctype = DT_DOC; } else if (path.endsWith(".rtf")) { doctype = DT_DOC; } else if (path.endsWith(".pdf")) { doctype = DT_PDFPS; } else if (path.endsWith(".ps")) { doctype = DT_PDFPS; } else if (path.endsWith(".avi")) { doctype = DT_MOVIE; } else if (path.endsWith(".mov")) { doctype = DT_MOVIE; } else if (path.endsWith(".qt")) { doctype = DT_MOVIE; } else if (path.endsWith(".mpg")) { doctype = DT_MOVIE; } else if (path.endsWith(".md5")) { doctype = DT_SHARE; } else if (path.endsWith(".mpeg")) { doctype = DT_MOVIE; } else if (path.endsWith(".asf")) { doctype = DT_FLASH; } return doctype; } public static char docType(final String mime) { // serverLog.logFinest("PLASMA", "docType mime=" + mime); char doctype = DT_UNKNOWN; if (mime == null) doctype = DT_UNKNOWN; else if (mime.startsWith("image/")) doctype = DT_IMAGE; else if (mime.endsWith("/gif")) doctype = DT_IMAGE; else if (mime.endsWith("/jpeg")) doctype = DT_IMAGE; else if (mime.endsWith("/png")) doctype = DT_IMAGE; else if (mime.endsWith("/html")) doctype = DT_HTML; else if (mime.endsWith("/rtf")) doctype = DT_DOC; else if (mime.endsWith("/pdf")) doctype = DT_PDFPS; else if (mime.endsWith("/octet-stream")) doctype = DT_BINARY; else if (mime.endsWith("/x-shockwave-flash")) doctype = DT_FLASH; else if (mime.endsWith("/msword")) doctype = DT_DOC; else if (mime.endsWith("/mspowerpoint")) doctype = DT_DOC; else if (mime.endsWith("/postscript")) doctype = DT_PDFPS; else if (mime.startsWith("text/")) doctype = DT_TEXT; else if (mime.startsWith("image/")) doctype = DT_IMAGE; else if (mime.startsWith("audio/")) doctype = DT_AUDIO; else if (mime.startsWith("video/")) doctype = DT_MOVIE; //bz2 = application/x-bzip2 //dvi = application/x-dvi //gz = application/gzip //hqx = application/mac-binhex40 //lha = application/x-lzh //lzh = application/x-lzh //pac = application/x-ns-proxy-autoconfig //php = application/x-httpd-php //phtml = application/x-httpd-php //rss = application/xml //tar = application/tar //tex = application/x-tex //tgz = application/tar //torrent = application/x-bittorrent //xhtml = application/xhtml+xml //xla = application/msexcel //xls = application/msexcel //xsl = application/xml //xml = application/xml //Z = application/x-compress //zip = application/zip return doctype; } public static final int QUEUE_STATE_FRESH = 0; public static final int QUEUE_STATE_PARSING = 1; public static final int QUEUE_STATE_CONDENSING = 2; public static final int QUEUE_STATE_STRUCTUREANALYSIS = 3; public static final int QUEUE_STATE_INDEXSTORAGE = 4; public static final int QUEUE_STATE_FINISHED = 5; public Response( final Request request, final RequestHeader requestHeader, final ResponseHeader responseHeader, final String responseStatus, final CrawlProfile profile, final byte[] content) { this.request = request; // request and response headers may be zero in case that we process surrogates this.requestHeader = requestHeader; this.responseHeader = responseHeader; this.responseStatus = responseStatus; this.profile = profile; this.status = QUEUE_STATE_FRESH; this.content = content; } public Response(final Request request, final CrawlProfile profile) { this.request = request; // request and response headers may be zero in case that we process surrogates this.requestHeader = new RequestHeader(); this.responseHeader = new ResponseHeader(); if (request.size() > 0) this.responseHeader.put(HeaderFramework.CONTENT_LENGTH, Long.toString(request.size())); this.responseStatus = "200"; this.profile = profile; this.status = QUEUE_STATE_FRESH; this.content = request.url().toTokens().getBytes(); } public Response( final Request request, final RequestHeader requestHeader, final ResponseHeader responseHeader, final String responseStatus, final CrawlProfile profile) { this(request, requestHeader, responseHeader, responseStatus, profile, null); } public void updateStatus(final int newStatus) { this.status = newStatus; } public ResponseHeader getResponseHeader() { return this.responseHeader; } public int getStatus() { return this.status; } public String name() { // the anchor name; can be either the text inside the anchor tag or the // page description after loading of the page return this.request.name(); } public DigestURI url() { return this.request.url(); } public char docType() { char doctype = docType(getMimeType()); if (doctype == DT_UNKNOWN) doctype = docType(url()); return doctype; } public Date lastModified() { Date docDate = null; if (responseHeader != null) { docDate = responseHeader.lastModified(); if (docDate == null) docDate = responseHeader.date(); } if (docDate == null && request != null) docDate = request.appdate(); if (docDate == null) docDate = new Date(GenericFormatter.correctedUTCTime()); return docDate; } public String language() { // please avoid this method if a condenser document is available, because the condenser has a built-in language detection // this here is only a guess using the TLD return this.url().language(); } public CrawlProfile profile() { return this.profile; } public byte[] initiator() { return this.request.initiator(); } public boolean proxy() { return initiator() == null; } public long size() { if (this.responseHeader != null) { // take the size from the response header return this.responseHeader.getContentLengthLong(); } if (this.content != null) return this.content.length; // the size is unknown return -1; } public int depth() { return this.request.depth(); } public void setContent(final byte[] data) { this.content = data; } public byte[] getContent() { return this.content; } // the following three methods for cache read/write granting shall be as loose // as possible but also as strict as necessary to enable caching of most items /** * @return NULL if the answer is TRUE, in case of FALSE, the reason as * String is returned */ public String shallStoreCacheForProxy() { String crawlerReason = shallStoreCacheForCrawler(); if (crawlerReason != null) return crawlerReason; // check profile (disabled: we will check this in the plasmaSwitchboard) // if (!this.profile.storeHTCache()) { return "storage_not_wanted"; } // decide upon header information if a specific file should be stored to // the cache or not // if the storage was requested by prefetching, the request map is null // -CGI access in request // CGI access makes the page very individual, and therefore not usable // in caches if (this.url().isPOST() && this.profile != null && !this.profile.crawlingQ()) { return "dynamic_post"; } if (this.url().isCGI()) { return "dynamic_cgi"; } if (this.url().isLocal()) { return "local_URL_no_cache_needed"; } if (responseHeader != null) { // -if-modified-since in request // we do not care about if-modified-since, because this case only occurres if the // cache file does not exist, and we need as much info as possible for the indexing // -cookies in request // we do not care about cookies, because that would prevent loading more pages // from one domain once a request resulted in a client-side stored cookie // -set-cookie in response // we do not care about cookies in responses, because that info comes along // any/many pages from a server and does not express the validity of the page // in modes of life-time/expiration or individuality // -pragma in response // if we have a pragma non-cache, we don't cache. usually if this is wanted from // the server, it makes sense String cacheControl = responseHeader.get(HeaderFramework.PRAGMA); if (cacheControl != null && cacheControl.trim().toUpperCase().equals("NO-CACHE")) { return "controlled_no_cache"; } // -expires in response // we do not care about expires, because at the time this is called the data is // obvious valid and that header info is used in the indexing later on // -cache-control in response // the cache-control has many value options. cacheControl = responseHeader.get(HeaderFramework.CACHE_CONTROL); if (cacheControl != null) { cacheControl = cacheControl.trim().toUpperCase(); if (cacheControl.startsWith("MAX-AGE=")) { // we need also the load date final Date date = responseHeader.date(); if (date == null) return "stale_no_date_given_in_response"; try { final long ttl = 1000 * Long.parseLong(cacheControl.substring(8)); // milliseconds to live if (GenericFormatter.correctedUTCTime() - date.getTime() > ttl) { //System.out.println("***not indexed because cache-control"); return "stale_expired"; } } catch (final Exception e) { return "stale_error_" + e.getMessage() + ")"; } } } } return null; } public String shallStoreCacheForCrawler() { // check storage size: all files will be handled in RAM before storage, so they must not exceed // a given size, which we consider as 1MB if (this.size() > 10 * 1024L * 1024L) return "too_large_for_caching_" + this.size(); // check status code if (!validResponseStatus()) { return "bad_status_" + this.responseStatus; } if (requestHeader != null) { // -authorization cases in request // authorization makes pages very individual, and therefore we cannot use the // content in the cache if (requestHeader.containsKey(RequestHeader.AUTHORIZATION)) { return "personalized"; } // -ranges in request and response // we do not cache partial content if (requestHeader.containsKey(HeaderFramework.RANGE)) { return "partial_request"; } } if (responseHeader != null) { // -ranges in request and response // we do not cache partial content if (responseHeader.containsKey(HeaderFramework.CONTENT_RANGE)) { return "partial_response"; } } return null; } /** * decide upon header information if a specific file should be taken from * the cache or not * * @return whether the file should be taken from the cache */ public boolean isFreshForProxy() { // -CGI access in request // CGI access makes the page very individual, and therefore not usable // in caches if (this.url().isPOST()) { return false; } if (this.url().isCGI()) { return false; } String cacheControl; if (requestHeader != null) { // -authorization cases in request if (requestHeader.containsKey(RequestHeader.AUTHORIZATION)) { return false; } // -ranges in request // we do not cache partial content if (requestHeader.containsKey(HeaderFramework.RANGE)) { return false; } // if the client requests a un-cached copy of the resource ... cacheControl = requestHeader.get(HeaderFramework.PRAGMA); if (cacheControl != null && cacheControl.trim().toUpperCase().equals("NO-CACHE")) { return false; } cacheControl = requestHeader.get(HeaderFramework.CACHE_CONTROL); if (cacheControl != null) { cacheControl = cacheControl.trim().toUpperCase(); if (cacheControl.startsWith("NO-CACHE") || cacheControl.startsWith("MAX-AGE=0")) { return false; } } // -if-modified-since in request // The entity has to be transferred only if it has // been modified since the date given by the If-Modified-Since header. if (requestHeader.containsKey(RequestHeader.IF_MODIFIED_SINCE)) { // checking this makes only sense if the cached response contains // a Last-Modified field. If the field does not exist, we go the safe way if (!responseHeader.containsKey(HeaderFramework.LAST_MODIFIED)) { return false; } // parse date Date d1, d2; d2 = responseHeader.lastModified(); if (d2 == null) { d2 = new Date(GenericFormatter.correctedUTCTime()); } d1 = requestHeader.ifModifiedSince(); if (d1 == null) { d1 = new Date(GenericFormatter.correctedUTCTime()); } // finally, we shall treat the cache as stale if the modification time is after the if-.. time if (d2.after(d1)) { return false; } } final String mimeType = getMimeType(); if (!Classification.isPictureMime(mimeType)) { // -cookies in request // unfortunately, we should reload in case of a cookie // but we think that pictures can still be considered as fresh // -set-cookie in cached response // this is a similar case as for COOKIE. if (requestHeader.containsKey(RequestHeader.COOKIE) || responseHeader.containsKey(HeaderFramework.SET_COOKIE) || responseHeader.containsKey(HeaderFramework.SET_COOKIE2)) { return false; // too strong } } } if (responseHeader != null) { // -pragma in cached response // logically, we would not need to care about no-cache pragmas in cached response headers, // because they cannot exist since they are not written to the cache. // So this IF should always fail.. cacheControl = responseHeader.get(HeaderFramework.PRAGMA); if (cacheControl != null && cacheControl.trim().toUpperCase().equals("NO-CACHE")) { return false; } // see for documentation also: // http://www.web-caching.com/cacheability.html // http://vancouver-webpages.com/CacheNow/ // look for freshnes information // if we don't have any freshnes indication, we treat the file as stale. // no handle for freshness control: // -expires in cached response // the expires value gives us a very easy hint when the cache is stale final Date expires = responseHeader.expires(); if (expires != null) { // System.out.println("EXPIRES-TEST: expires=" + expires + ", NOW=" + serverDate.correctedGMTDate() + ", url=" + url); if (expires.before(new Date(GenericFormatter.correctedUTCTime()))) { return false; } } final Date lastModified = responseHeader.lastModified(); cacheControl = responseHeader.get(HeaderFramework.CACHE_CONTROL); if (cacheControl == null && lastModified == null && expires == null) { return false; } // -lastModified in cached response // we can apply a TTL (Time To Live) heuristic here. We call the time delta between the last read // of the file and the last modified date as the age of the file. If we consider the file as // middel-aged then, the maximum TTL would be cache-creation plus age. // This would be a TTL factor of 100% we want no more than 10% TTL, so that a 10 month old cache // file may only be treated as fresh for one more month, not more. Date date = responseHeader.date(); if (lastModified != null) { if (date == null) { date = new Date(GenericFormatter.correctedUTCTime()); } final long age = date.getTime() - lastModified.getTime(); if (age < 0) { return false; } // TTL (Time-To-Live) is age/10 = (d2.getTime() - d1.getTime()) / 10 // the actual living-time is serverDate.correctedGMTDate().getTime() - d2.getTime() // therefore the cache is stale, if serverDate.correctedGMTDate().getTime() - d2.getTime() > age/10 if (GenericFormatter.correctedUTCTime() - date.getTime() > age / 10) { return false; } } // -cache-control in cached response // the cache-control has many value options. if (cacheControl != null) { cacheControl = cacheControl.trim().toUpperCase(); if (cacheControl.startsWith("PRIVATE") || cacheControl.startsWith("NO-CACHE") || cacheControl.startsWith("NO-STORE")) { // easy case return false; // } else if (cacheControl.startsWith("PUBLIC")) { // // ok, do nothing } else if (cacheControl.startsWith("MAX-AGE=")) { // we need also the load date if (date == null) { return false; } try { final long ttl = 1000 * Long.parseLong(cacheControl.substring(8)); // milliseconds to live if (GenericFormatter.correctedUTCTime() - date.getTime() > ttl) { return false; } } catch (final Exception e) { return false; } } } } return true; } /** * decide upon header information if a specific file should be indexed * this method returns null if the answer is 'YES'! * if the answer is 'NO' (do not index), it returns a string with the reason * to reject the crawling demand in clear text * * This function is used by plasmaSwitchboard#processResourceStack */ public final String shallIndexCacheForProxy() { if (profile() == null) { return "shallIndexCacheForProxy: profile() is null !"; } // check profile if (!profile().indexText() && !profile().indexMedia()) { return "indexing not allowed - indexText and indexMedia not set (for proxy = " + profile.name()+ ")"; } // -CGI access in request // CGI access makes the page very individual, and therefore not usable in caches if (!profile().crawlingQ()) { if (url().isPOST()) { return "Dynamic_(POST)"; } if (url().isCGI()) { return "Dynamic_(CGI)"; } } // -authorization cases in request // we checked that in shallStoreCache // -ranges in request // we checked that in shallStoreCache // a picture cannot be indexed /* if (Classification.isMediaExtension(url().getFileExtension())) { return "Media_Content_(forbidden)"; } */ // -cookies in request // unfortunately, we cannot index pages which have been requested with a cookie // because the returned content may be special for the client if (requestWithCookie()) { // System.out.println("***not indexed because cookie"); return "Dynamic_(Requested_With_Cookie)"; } if (responseHeader != null) { // -set-cookie in response // the set-cookie from the server does not indicate that the content is special // thus we do not care about it here for indexing // a picture cannot be indexed final String mimeType = responseHeader.mime(); /* if (Classification.isPictureMime(mimeType)) { return "Media_Content_(Picture)"; } */ String parserError = TextParser.supportsMime(mimeType); if (parserError != null) { return "Media_Content, no parser: " + parserError; } // -if-modified-since in request // if the page is fresh at the very moment we can index it final Date ifModifiedSince = this.requestHeader.ifModifiedSince(); if ((ifModifiedSince != null) && (responseHeader.containsKey(HeaderFramework.LAST_MODIFIED))) { // parse date Date d = responseHeader.lastModified(); if (d == null) { d = new Date(GenericFormatter.correctedUTCTime()); } // finally, we shall treat the cache as stale if the modification time is after the if-.. time if (d.after(ifModifiedSince)) { //System.out.println("***not indexed because if-modified-since"); return "Stale_(Last-Modified>Modified-Since)"; } } // -pragma in cached response if (responseHeader.containsKey(HeaderFramework.PRAGMA) && (responseHeader.get(HeaderFramework.PRAGMA)).toUpperCase().equals("NO-CACHE")) { return "Denied_(pragma_no_cache)"; } // see for documentation also: // http://www.web-caching.com/cacheability.html // look for freshnes information // -expires in cached response // the expires value gives us a very easy hint when the cache is stale // sometimes, the expires date is set to the past to prevent that a page is cached // we use that information to see if we should index it final Date expires = responseHeader.expires(); if (expires != null && expires.before(new Date(GenericFormatter.correctedUTCTime()))) { return "Stale_(Expired)"; } // -lastModified in cached response // this information is too weak to use it to prevent indexing // even if we can apply a TTL heuristic for cache usage // -cache-control in cached response // the cache-control has many value options. String cacheControl = responseHeader.get(HeaderFramework.CACHE_CONTROL); if (cacheControl != null) { cacheControl = cacheControl.trim().toUpperCase(); /* we have the following cases for cache-control: "public" -- can be indexed "private", "no-cache", "no-store" -- cannot be indexed "max-age=" -- stale/fresh dependent on date */ if (cacheControl.startsWith("PRIVATE") || cacheControl.startsWith("NO-CACHE") || cacheControl.startsWith("NO-STORE")) { // easy case return "Stale_(denied_by_cache-control=" + cacheControl + ")"; // } else if (cacheControl.startsWith("PUBLIC")) { // // ok, do nothing } else if (cacheControl.startsWith("MAX-AGE=")) { // we need also the load date final Date date = responseHeader.date(); if (date == null) { return "Stale_(no_date_given_in_response)"; } try { final long ttl = 1000 * Long.parseLong(cacheControl.substring(8)); // milliseconds to live if (GenericFormatter.correctedUTCTime() - date.getTime() > ttl) { //System.out.println("***not indexed because cache-control"); return "Stale_(expired_by_cache-control)"; } } catch (final Exception e) { return "Error_(" + e.getMessage() + ")"; } } } } return null; } /** * decide upon header information if a specific file should be indexed * this method returns null if the answer is 'YES'! * if the answer is 'NO' (do not index), it returns a string with the reason * to reject the crawling demand in clear text * * This function is used by plasmaSwitchboard#processResourceStack */ public final String shallIndexCacheForCrawler() { if (profile() == null) { return "shallIndexCacheForCrawler: profile() is null !"; } // check profile if (!profile().indexText() && !profile().indexMedia()) { return "indexing not allowed - indexText and indexMedia not set (for crawler = " + profile.name() + ")"; } // -CGI access in request // CGI access makes the page very individual, and therefore not usable in caches if (!profile().crawlingQ()) { if (url().isPOST()) { return "Dynamic_(POST)"; } if (url().isCGI()) { return "Dynamic_(CGI)"; } } // -authorization cases in request // we checked that in shallStoreCache // -ranges in request // we checked that in shallStoreCache // check if document can be indexed if (responseHeader != null) { final String mimeType = responseHeader.mime(); String parserError = TextParser.supportsMime(mimeType); if (parserError != null && TextParser.supportsExtension(url()) != null) return "no parser available: " + parserError; } /* if (Classification.isMediaExtension(url().getFileExtension()) && !Classification.isImageExtension((url().getFileExtension()))) { return "Media_Content_(forbidden)"; } */ // -if-modified-since in request // if the page is fresh at the very moment we can index it // -> this does not apply for the crawler // -cookies in request // unfortunately, we cannot index pages which have been requested with a cookie // because the returned content may be special for the client // -> this does not apply for a crawler // -set-cookie in response // the set-cookie from the server does not indicate that the content is special // thus we do not care about it here for indexing // -> this does not apply for a crawler // -pragma in cached response // -> in the crawler we ignore this // look for freshnes information // -expires in cached response // the expires value gives us a very easy hint when the cache is stale // sometimes, the expires date is set to the past to prevent that a page is cached // we use that information to see if we should index it // -> this does not apply for a crawler // -lastModified in cached response // this information is too weak to use it to prevent indexing // even if we can apply a TTL heuristic for cache usage // -cache-control in cached response // the cache-control has many value options. // -> in the crawler we ignore this return null; } public String getMimeType() { if (responseHeader == null) return null; String mimeType = responseHeader.mime(); mimeType = mimeType.trim().toLowerCase(); final int pos = mimeType.indexOf(';'); return ((pos < 0) ? mimeType : mimeType.substring(0, pos)); } public String getCharacterEncoding() { if (responseHeader == null) return null; return responseHeader.getCharacterEncoding(); } public DigestURI referrerURL() { if (requestHeader == null) return null; try { String r = requestHeader.get(RequestHeader.REFERER, null); if (r == null) return null; return new DigestURI(r); } catch (final Exception e) { return null; } } public byte[] referrerHash() { if (requestHeader == null) return null; String u = requestHeader.get(RequestHeader.REFERER, ""); if (u == null || u.length() == 0) return null; try { return new DigestURI(u).hash(); } catch (final Exception e) { return null; } } public boolean validResponseStatus() { return (responseStatus == null) ? false : responseStatus.startsWith("200") || responseStatus.startsWith("203"); } public Date ifModifiedSince() { return (requestHeader == null) ? null : requestHeader.ifModifiedSince(); } public boolean requestWithCookie() { return (requestHeader == null) ? false : requestHeader.containsKey(RequestHeader.COOKIE); } public boolean requestProhibitsIndexing() { return (requestHeader == null) ? false : requestHeader.containsKey(HeaderFramework.X_YACY_INDEX_CONTROL) && (requestHeader.get(HeaderFramework.X_YACY_INDEX_CONTROL)).toUpperCase().equals("NO-INDEX"); } public EventOrigin processCase(String mySeedHash) { // we must distinguish the following cases: resource-load was initiated by // 1) global crawling: the index is extern, not here (not possible here) // 2) result of search queries, some indexes are here (not possible here) // 3) result of index transfer, some of them are here (not possible here) // 4) proxy-load (initiator is "------------") // 5) local prefetch/crawling (initiator is own seedHash) // 6) local fetching for global crawling (other known or unknwon initiator) EventOrigin processCase = EventOrigin.UNKNOWN; // FIXME the equals seems to be incorrect: String.equals(boolean) if ((initiator() == null) || initiator().length == 0 || UTF8.String(initiator()).equals("------------")) { // proxy-load processCase = EventOrigin.PROXY_LOAD; } else if (UTF8.String(initiator()).equals(mySeedHash)) { // normal crawling processCase = EventOrigin.LOCAL_CRAWLING; } else { // this was done for remote peer (a global crawl) processCase = EventOrigin.GLOBAL_CRAWLING; } return processCase; } public Document[] parse() throws Parser.Failure { String supportError = TextParser.supports(url(), this.responseHeader == null ? null : this.responseHeader.mime()); if (supportError != null) throw new Parser.Failure("no parser support:" + supportError, url()); try { return TextParser.parseSource(url(), this.responseHeader == null ? null : this.responseHeader.mime(), this.responseHeader == null ? "UTF-8" : this.responseHeader.getCharacterEncoding(), this.content.length, new ByteArrayInputStream(this.content)); } catch (Exception e) { return null; } } }