mirror of
https://github.com/yacy/yacy_search_server.git
synced 2024-09-21 00:00:13 +02:00
765943a4b7
in intranets and the internet can now choose to appear as Googlebot. This is an essential necessity to be able to compete in the field of commercial search appliances, since most web pages are these days optimized only for Google and no other search platform any more. All commercial search engine providers have a built-in fake-Google User Agent to be able to get the same search index as Google can do. Without the resistance against obeying to robots.txt in this case, no competition is possible any more. YaCy will always obey the robots.txt when it is used for crawling the web in a peer-to-peer network, but to establish a Search Appliance (like a Google Search Appliance, GSA) it is necessary to be able to behave exactly like a Google crawler. With this change, you will be able to switch the user agent when portal or intranet mode is selected on per-crawl-start basis. Every crawl start can have a different user agent.
154 lines
6.1 KiB
Java
154 lines
6.1 KiB
Java
/**
|
|
* FileLoader
|
|
* Copyright 2010 by Michael Peter Christen
|
|
* First released 25.5.2010 at http://yacy.net
|
|
*
|
|
* $LastChangedDate$
|
|
* $LastChangedRevision$
|
|
* $LastChangedBy$
|
|
*
|
|
* This library is free software; you can redistribute it and/or
|
|
* modify it under the terms of the GNU Lesser General Public
|
|
* License as published by the Free Software Foundation; either
|
|
* version 2.1 of the License, or (at your option) any later version.
|
|
*
|
|
* This library is distributed in the hope that it will be useful,
|
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
|
* Lesser General Public License for more details.
|
|
*
|
|
* You should have received a copy of the GNU Lesser General Public License
|
|
* along with this program in the file lgpl21.txt
|
|
* If not, see <http://www.gnu.org/licenses/>.
|
|
*/
|
|
|
|
package net.yacy.crawler.retrieval;
|
|
|
|
import java.io.IOException;
|
|
import java.io.InputStream;
|
|
import java.util.ArrayList;
|
|
import java.util.Date;
|
|
import java.util.List;
|
|
|
|
import net.yacy.cora.document.ASCII;
|
|
import net.yacy.cora.document.MultiProtocolURI;
|
|
import net.yacy.cora.document.UTF8;
|
|
import net.yacy.cora.document.analysis.Classification;
|
|
import net.yacy.cora.protocol.ClientIdentification;
|
|
import net.yacy.cora.protocol.HeaderFramework;
|
|
import net.yacy.cora.protocol.RequestHeader;
|
|
import net.yacy.cora.protocol.ResponseHeader;
|
|
import net.yacy.cora.protocol.ftp.FTPClient;
|
|
import net.yacy.cora.util.ConcurrentLog;
|
|
import net.yacy.crawler.data.CrawlProfile;
|
|
import net.yacy.document.TextParser;
|
|
import net.yacy.kelondro.data.meta.DigestURI;
|
|
import net.yacy.kelondro.util.FileUtils;
|
|
import net.yacy.search.Switchboard;
|
|
|
|
public class FileLoader {
|
|
|
|
private final Switchboard sb;
|
|
private final ConcurrentLog log;
|
|
private final int maxFileSize;
|
|
|
|
public FileLoader(final Switchboard sb, final ConcurrentLog log) {
|
|
this.sb = sb;
|
|
this.log = log;
|
|
this.maxFileSize = (int) sb.getConfigLong("crawler.file.maxFileSize", -1l);
|
|
}
|
|
|
|
public Response load(final Request request, boolean acceptOnlyParseable) throws IOException {
|
|
DigestURI url = request.url();
|
|
if (!url.getProtocol().equals("file")) throw new IOException("wrong loader for FileLoader: " + url.getProtocol());
|
|
|
|
RequestHeader requestHeader = new RequestHeader();
|
|
if (request.referrerhash() != null) {
|
|
DigestURI ur = this.sb.getURL(request.referrerhash());
|
|
if (ur != null) requestHeader.put(RequestHeader.REFERER, ur.toNormalform(true));
|
|
}
|
|
|
|
// process directories: transform them to html with meta robots=noindex (using the ftpc lib)
|
|
String[] l = null;
|
|
try {l = url.list();} catch (final IOException e) {}
|
|
if (l != null) {
|
|
String u = url.toNormalform(true);
|
|
List<String> list = new ArrayList<String>();
|
|
for (String s: l) {
|
|
list.add(u + ((u.endsWith("/") || u.endsWith("\\")) ? "" : "/") + s);
|
|
}
|
|
|
|
StringBuilder content = FTPClient.dirhtml(u, null, null, null, list, true);
|
|
|
|
ResponseHeader responseHeader = new ResponseHeader(200);
|
|
responseHeader.put(HeaderFramework.LAST_MODIFIED, HeaderFramework.formatRFC1123(new Date()));
|
|
responseHeader.put(HeaderFramework.CONTENT_TYPE, "text/html");
|
|
final CrawlProfile profile = this.sb.crawler.getActive(ASCII.getBytes(request.profileHandle()));
|
|
Response response = new Response(
|
|
request,
|
|
requestHeader,
|
|
responseHeader,
|
|
profile,
|
|
false,
|
|
UTF8.getBytes(content.toString()));
|
|
|
|
return response;
|
|
}
|
|
|
|
// create response header
|
|
String mime = Classification.ext2mime(MultiProtocolURI.getFileExtension(url.getFileName()));
|
|
ResponseHeader responseHeader = new ResponseHeader(200);
|
|
responseHeader.put(HeaderFramework.LAST_MODIFIED, HeaderFramework.formatRFC1123(new Date(url.lastModified())));
|
|
responseHeader.put(HeaderFramework.CONTENT_TYPE, mime);
|
|
|
|
// check mime type and availability of parsers
|
|
// and also check resource size and limitation of the size
|
|
long size;
|
|
try {
|
|
size = url.length();
|
|
} catch (final Exception e) {
|
|
size = -1;
|
|
}
|
|
String parserError = null;
|
|
if ((acceptOnlyParseable && (parserError = TextParser.supports(url, mime)) != null) ||
|
|
(size > this.maxFileSize && this.maxFileSize >= 0)) {
|
|
// we know that we cannot process that file before loading
|
|
// only the metadata is returned
|
|
|
|
if (parserError != null) {
|
|
this.log.info("No parser available in File crawler: '" + parserError + "' for URL " + request.url().toString() + ": parsing only metadata");
|
|
} else {
|
|
this.log.info("Too big file in File crawler with size = " + size + " Bytes for URL " + request.url().toString() + ": parsing only metadata");
|
|
}
|
|
|
|
// create response with metadata only
|
|
responseHeader.put(HeaderFramework.CONTENT_TYPE, "text/plain");
|
|
final CrawlProfile profile = this.sb.crawler.getActive(ASCII.getBytes(request.profileHandle()));
|
|
Response response = new Response(
|
|
request,
|
|
requestHeader,
|
|
responseHeader,
|
|
profile,
|
|
false,
|
|
UTF8.getBytes(url.toTokens()));
|
|
return response;
|
|
}
|
|
|
|
// load the resource
|
|
InputStream is = url.getInputStream(ClientIdentification.yacyInternetCrawlerAgent);
|
|
byte[] b = FileUtils.read(is);
|
|
is.close();
|
|
|
|
// create response with loaded content
|
|
final CrawlProfile profile = this.sb.crawler.getActive(ASCII.getBytes(request.profileHandle()));
|
|
Response response = new Response(
|
|
request,
|
|
requestHeader,
|
|
responseHeader,
|
|
profile,
|
|
false,
|
|
b);
|
|
return response;
|
|
}
|
|
}
|