Marown/yacy_search_server
1
0
Fork 0
mirror of https://github.com/yacy/yacy_search_server.git synced 2024-09-19 00:01:41 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity

CrawlCheck_p.html : also display info about disallowed URLs.

Browse Source
This commit is contained in:
luccioman 2016-10-12 11:26:59 +02:00
parent 8b341e9818
commit 54cfcc3f56
1 changed files with 12 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
source/net/yacy/crawler/robots/RobotsTxt.java
View File

@ -393,13 +393,18 @@ public class RobotsTxt {
// try to load the robots
RobotsTxtEntry robotsEntry = getEntry(u, userAgent);
boolean robotsAllowed = robotsEntry == null ? true : !robotsEntry.isDisallowed(u);
if (robotsAllowed) try {
Request request = loader.request(u, true, false);
Response response = loader.load(request, CacheStrategy.NOCACHE, BlacklistType.CRAWLER, userAgent);
out.put(new CheckEntry(u, robotsEntry, response, null));
} catch (final IOException e) {
out.put(new CheckEntry(u, robotsEntry, null, "error response: " + e.getMessage()));
}
if (robotsAllowed) {
try {
Request request = loader.request(u, true, false);
Response response = loader.load(request, CacheStrategy.NOCACHE,
BlacklistType.CRAWLER, userAgent);
out.put(new CheckEntry(u, robotsEntry, response, null));
} catch (final IOException e) {
out.put(new CheckEntry(u, robotsEntry, null, "error response: " + e.getMessage()));
}
} else {
out.put(new CheckEntry(u, robotsEntry, null, null));
}
}
} catch (InterruptedException e) {}
}