diff --git a/htroot/ConfigRobotsTxt_p.html b/htroot/ConfigRobotsTxt_p.html index e723d0462..472e85d1a 100644 --- a/htroot/ConfigRobotsTxt_p.html +++ b/htroot/ConfigRobotsTxt_p.html @@ -9,14 +9,17 @@ #%env/templates/submenuConfig.template%#
+ Here you can set robots.txt-settings for all webcrawlers that try to access your peer. + robots.txt is a volunteer agreement most search-engines (including YaCy) follow. + It disallows crawlers to access webpages or even entire domains.
#(error)# ::Unable to access the local file: #[msg]#
::Deletion of htroot/robots.txt failed
#(/error)#