mirror of
https://github.com/yacy/yacy_search_server.git
synced 2024-09-19 00:01:41 +02:00
0ab6a462ee
- changed default robots.txt deny list to include some more interface pages because the loading of such pages are a peer load issue for YaCy when crawlers come by and information on these pages are not useful for public search. git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@7112 6c8d7289-2bf4-0310-a012-ef5d649a1542
67 lines
915 B
Plaintext
67 lines
915 B
Plaintext
# robots.txt for #[clientname]#.yacy
|
|
|
|
User-agent: *
|
|
|
|
#(all)#
|
|
|
|
# dirs
|
|
#{dirs}#
|
|
Disallow: /#[dir]#
|
|
#{/dirs}#
|
|
|
|
|
|
# locked
|
|
#{locked}#
|
|
Disallow: /#[page]#
|
|
#{/locked}#
|
|
|
|
#(bookmarks)#::
|
|
# bookmarks
|
|
Disallow: /Bookmarks.html#(/bookmarks)#
|
|
|
|
#(wiki)#::
|
|
# wiki
|
|
Disallow: /Wiki.html#(/wiki)#
|
|
|
|
#(blog)#::
|
|
# blog
|
|
Disallow: /Blog.html
|
|
Disallow: /Blog.rss
|
|
Disallow: /Blog.xml#(/blog)#
|
|
|
|
#(news)#::
|
|
# news
|
|
Disallow: /News.html#(/news)#
|
|
|
|
#(status)#::
|
|
# status
|
|
Disallow: /Status.html#(/status)#
|
|
|
|
#(network)#::
|
|
# network
|
|
Disallow: /Network.html
|
|
Disallow: /Network.csv
|
|
Disallow: /Network.xml#(/network)#
|
|
|
|
#(homepage)#::
|
|
# homepage
|
|
Disallow: /www#(/homepage)#
|
|
|
|
#(fileshare)#::
|
|
# fileshare
|
|
Disallow: /share#(/fileshare)#
|
|
|
|
#(surftips)#::
|
|
# surftips
|
|
Disallow: /Surftips.html#(/surftips)#
|
|
|
|
#(profile)#::
|
|
# view profile
|
|
Disallow: /ViewProfile.html
|
|
Disallow: /ViewProfile.vcf
|
|
Disallow: /ViewProfile.xml#(/profile)#
|
|
|
|
::
|
|
# all
|
|
Disallow: /
|
|
#(/all)# |