Marown/yacy_search_server
1
0
Fork 0
mirror of https://github.com/yacy/yacy_search_server.git synced 2024-09-19 00:01:41 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity

reverted change in robotsParser

Browse Source 
git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@2378 6c8d7289-2bf4-0310-a012-ef5d649a1542
This commit is contained in:
orbiter 2006-08-11 15:29:29 +00:00
parent abf22f6e60
commit cfbacbbf08
1 changed files with 2 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
source/de/anomic/data/robotsParser.java
View File

@ -282,7 +282,7 @@ public final class robotsParser{
}
}
if (robotsTxt4Host.isDisallowed(nexturl.getPath())) {
if (robotsTxt4Host.isDisallowed(nexturl.getFile())) {
return true;
}
return false;
@ -327,7 +327,7 @@ public final class robotsParser{
}
httpc.response res = con.GET(robotsURL.getPath(), reqHeaders);
httpc.response res = con.GET(robotsURL.getFile(), reqHeaders);
if (res.status.startsWith("2")) {
if (!res.responseHeader.mime().startsWith("text/plain")) {
robotsTxt = null;