mirror of
https://github.com/yacy/yacy_search_server.git
synced 2024-09-19 00:01:41 +02:00
c0d9a3e9a7
This was required because spiders and bots crawled through this page and created load on the peer without use for the user or the YaCy network.
71 lines
988 B
Plaintext
71 lines
988 B
Plaintext
# robots.txt for #[clientname]#.yacy
|
|
|
|
User-agent: *
|
|
|
|
Disallow: /*query=
|
|
Disallow: /proxy.html
|
|
Disallow: /IndexBrowser_p.html
|
|
|
|
#(all)#
|
|
|
|
# dirs
|
|
#{dirs}#
|
|
Disallow: /#[dir]#
|
|
#{/dirs}#
|
|
|
|
|
|
# locked
|
|
#{locked}#
|
|
Disallow: /#[page]#
|
|
#{/locked}#
|
|
|
|
#(bookmarks)#::
|
|
# bookmarks
|
|
Disallow: /Bookmarks.html#(/bookmarks)#
|
|
|
|
#(wiki)#::
|
|
# wiki
|
|
Disallow: /Wiki.html#(/wiki)#
|
|
|
|
#(blog)#::
|
|
# blog
|
|
Disallow: /Blog.html
|
|
Disallow: /Blog.rss
|
|
Disallow: /Blog.xml#(/blog)#
|
|
|
|
#(news)#::
|
|
# news
|
|
Disallow: /News.html#(/news)#
|
|
|
|
#(status)#::
|
|
# status
|
|
Disallow: /Status.html#(/status)#
|
|
|
|
#(network)#::
|
|
# network
|
|
Disallow: /Network.html
|
|
Disallow: /Network.csv
|
|
Disallow: /Network.xml#(/network)#
|
|
|
|
#(homepage)#::
|
|
# homepage
|
|
Disallow: /www#(/homepage)#
|
|
|
|
#(fileshare)#::
|
|
# fileshare
|
|
Disallow: /share#(/fileshare)#
|
|
|
|
#(surftips)#::
|
|
# surftips
|
|
Disallow: /Surftips.html#(/surftips)#
|
|
|
|
#(profile)#::
|
|
# view profile
|
|
Disallow: /ViewProfile.html
|
|
Disallow: /ViewProfile.vcf
|
|
Disallow: /ViewProfile.xml#(/profile)#
|
|
|
|
::
|
|
# all
|
|
Disallow: /
|
|
#(/all)# |