mirror of
https://github.com/yacy/yacy_search_server.git
synced 2024-09-19 00:01:41 +02:00
cced94298a
This makes possbile to set up much more advanced document crawl filters, by filtering on one or more document indexed fields before inserting in the index.
46 lines
2.5 KiB
XML
46 lines
2.5 KiB
XML
<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
|
|
<crawlProfiles>
|
|
#{crawlProfiles}#
|
|
<crawlProfile>
|
|
<handle>#[handle]#</handle>
|
|
<name>#[name]#</name>
|
|
<collections>#[collections]#</collections>
|
|
<agentName>#[agentName]#</agentName>
|
|
<userAgent>#[userAgent]#</userAgent>
|
|
<depth>#[depth]#</depth>
|
|
<directDocByURL>#(directDocByURL)#false::true#(/directDocByURL)#</directDocByURL>
|
|
<recrawlIfOlder>#[recrawlIfOlder]#</recrawlIfOlder>
|
|
<domMaxPages>#[domMaxPages]#</domMaxPages>
|
|
<crawlingQ>#(crawlingQ)#false::true#(/crawlingQ)#</crawlingQ>
|
|
<followFrames>#(followFrames)#false::true#(/followFrames)#</followFrames>
|
|
<obeyHtmlRobotsNoindex>#(obeyHtmlRobotsNoindex)#false::true#(/obeyHtmlRobotsNoindex)#</obeyHtmlRobotsNoindex>
|
|
<obeyHtmlRobotsNofollow>#(obeyHtmlRobotsNofollow)#false::true#(/obeyHtmlRobotsNofollow)#</obeyHtmlRobotsNofollow>
|
|
<indexText>#(indexText)#false::true#(/indexText)#</indexText>
|
|
<indexMedia>#(indexMedia)#false::true#(/indexMedia)#</indexMedia>
|
|
<storeHTCache>#(storeHTCache)#false::true#(/storeHTCache)#</storeHTCache>
|
|
<remoteIndexing>#(remoteIndexing)#false::true#(/remoteIndexing)#</remoteIndexing>
|
|
<cacheStrategy>#[cacheStrategy]#</cacheStrategy>
|
|
<crawlerURLMustMatch>#[crawlerURLMustMatch]#</crawlerURLMustMatch>
|
|
<crawlerURLMustNotMatch>#[crawlerURLMustNotMatch]#</crawlerURLMustNotMatch>
|
|
<crawlerIPMustMatch>#[crawlerIPMustMatch]#</crawlerIPMustMatch>
|
|
<crawlerIPMustNotMatch>#[crawlerIPMustNotMatch]#</crawlerIPMustNotMatch>
|
|
<crawlerCountryMustMatch>#[crawlerCountryMustMatch]#</crawlerCountryMustMatch>
|
|
<crawlerNoLimitURLMustMatch>#[crawlerNoLimitURLMustMatch]#</crawlerNoLimitURLMustMatch>
|
|
<indexURLMustMatch>#[indexURLMustMatch]#</indexURLMustMatch>
|
|
<indexURLMustNotMatch>#[indexURLMustNotMatch]#</indexURLMustNotMatch>
|
|
<indexContentMustMatch>#[indexContentMustMatch]#</indexContentMustMatch>
|
|
<indexContentMustNotMatch>#[indexContentMustNotMatch]#</indexContentMustNotMatch>
|
|
<indexMediaTypeMustMatch>#[indexMediaTypeMustMatch]#</indexMediaTypeMustMatch>
|
|
<indexMediaTypeMustNotMatch>#[indexMediaTypeMustNotMatch]#</indexMediaTypeMustNotMatch>
|
|
<indexSolrQueryMustMatch>#[indexSolrQueryMustMatch]#</indexSolrQueryMustMatch>
|
|
<indexSolrQueryMustNotMatch>#[indexSolrQueryMustNotMatch]#</indexSolrQueryMustNotMatch>
|
|
<status>#(status)#terminated::active::system#(/status)#</status>
|
|
<crawlingDomFilterContent>
|
|
#{crawlingDomFilterContent}#
|
|
<item>#[item]#</item>
|
|
#{/crawlingDomFilterContent}#
|
|
</crawlingDomFilterContent>
|
|
</crawlProfile>
|
|
#{/crawlProfiles}#
|
|
</crawlProfiles>
|