mirror of
https://github.com/yacy/yacy_search_server.git
synced 2024-09-21 00:00:13 +02:00
038f956821
appeared after the declaration of robots allow/deny for the crawler because the sitemap parser terminated after the allow/deny rules had been found. Now the parser reads the robots.txt until the end to discover also sitemap rules at the end of the file. |
||
---|---|---|
.. | ||
bookmarks | ||
ymarks | ||
blacklists_p.java | ||
blacklists_p.xml | ||
blacklists.java | ||
blacklists.xml | ||
config_p.java | ||
config_p.xml | ||
feed.java | ||
feed.rss | ||
feed.xml | ||
getpageinfo_p.java | ||
getpageinfo_p.xml | ||
getpageinfo.java | ||
getpageinfo.json | ||
getpageinfo.xml | ||
latency_p.java | ||
latency_p.xml | ||
schema.java | ||
schema.xml | ||
status_p.java | ||
status_p.xml | ||
table_p.html | ||
table_p.java | ||
table_p.xml | ||
termlist_p.java | ||
termlist_p.xml | ||
timeline.java | ||
timeline.xml | ||
trail_p.java | ||
trail_p.json | ||
version.java | ||
version.xml | ||
webstructure.java | ||
webstructure.xml | ||
yacydoc.html | ||
yacydoc.java | ||
yacydoc.xml | ||
ynetSearch.java | ||
ynetSearch.json | ||
ynetSearch.xml |