mirror of
https://github.com/yacy/yacy_search_server.git
synced 2024-09-19 00:01:41 +02:00
fcf6b16db4
New "Media Type detection" section in the advanced crawl start page allow to choose between : - not loading URLs with unknown or unsupported file extension without checking the actual Media Type (relying Content-Type header for now). This was the old default behavior, faster, but not really accurate. - always cross check URL file extension against the actual Media Type. This lets properly parse URLs ending with an apparently odd file extension, but which have actually a supported Media Type such as text/html. Sample URLs with misleading file extensions added as documentation in the crawl start page. fixes issue #244
47 lines
2.6 KiB
XML
47 lines
2.6 KiB
XML
<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
|
|
<crawlProfiles>
|
|
#{crawlProfiles}#
|
|
<crawlProfile>
|
|
<handle>#[handle]#</handle>
|
|
<name>#[name]#</name>
|
|
<collections>#[collections]#</collections>
|
|
<agentName>#[agentName]#</agentName>
|
|
<userAgent>#[userAgent]#</userAgent>
|
|
<depth>#[depth]#</depth>
|
|
<directDocByURL>#(directDocByURL)#false::true#(/directDocByURL)#</directDocByURL>
|
|
<recrawlIfOlder>#[recrawlIfOlder]#</recrawlIfOlder>
|
|
<domMaxPages>#[domMaxPages]#</domMaxPages>
|
|
<crawlingQ>#(crawlingQ)#false::true#(/crawlingQ)#</crawlingQ>
|
|
<followFrames>#(followFrames)#false::true#(/followFrames)#</followFrames>
|
|
<obeyHtmlRobotsNoindex>#(obeyHtmlRobotsNoindex)#false::true#(/obeyHtmlRobotsNoindex)#</obeyHtmlRobotsNoindex>
|
|
<obeyHtmlRobotsNofollow>#(obeyHtmlRobotsNofollow)#false::true#(/obeyHtmlRobotsNofollow)#</obeyHtmlRobotsNofollow>
|
|
<indexText>#(indexText)#false::true#(/indexText)#</indexText>
|
|
<indexMedia>#(indexMedia)#false::true#(/indexMedia)#</indexMedia>
|
|
<storeHTCache>#(storeHTCache)#false::true#(/storeHTCache)#</storeHTCache>
|
|
<remoteIndexing>#(remoteIndexing)#false::true#(/remoteIndexing)#</remoteIndexing>
|
|
<cacheStrategy>#[cacheStrategy]#</cacheStrategy>
|
|
<crawlerAlwaysCheckMediaType>#(crawlerAlwaysCheckMediaType)#false::true#(/crawlerAlwaysCheckMediaType)#</crawlerAlwaysCheckMediaType>
|
|
<crawlerURLMustMatch>#[crawlerURLMustMatch]#</crawlerURLMustMatch>
|
|
<crawlerURLMustNotMatch>#[crawlerURLMustNotMatch]#</crawlerURLMustNotMatch>
|
|
<crawlerIPMustMatch>#[crawlerIPMustMatch]#</crawlerIPMustMatch>
|
|
<crawlerIPMustNotMatch>#[crawlerIPMustNotMatch]#</crawlerIPMustNotMatch>
|
|
<crawlerCountryMustMatch>#[crawlerCountryMustMatch]#</crawlerCountryMustMatch>
|
|
<crawlerNoLimitURLMustMatch>#[crawlerNoLimitURLMustMatch]#</crawlerNoLimitURLMustMatch>
|
|
<indexURLMustMatch>#[indexURLMustMatch]#</indexURLMustMatch>
|
|
<indexURLMustNotMatch>#[indexURLMustNotMatch]#</indexURLMustNotMatch>
|
|
<indexContentMustMatch>#[indexContentMustMatch]#</indexContentMustMatch>
|
|
<indexContentMustNotMatch>#[indexContentMustNotMatch]#</indexContentMustNotMatch>
|
|
<indexMediaTypeMustMatch>#[indexMediaTypeMustMatch]#</indexMediaTypeMustMatch>
|
|
<indexMediaTypeMustNotMatch>#[indexMediaTypeMustNotMatch]#</indexMediaTypeMustNotMatch>
|
|
<indexSolrQueryMustMatch>#[indexSolrQueryMustMatch]#</indexSolrQueryMustMatch>
|
|
<indexSolrQueryMustNotMatch>#[indexSolrQueryMustNotMatch]#</indexSolrQueryMustNotMatch>
|
|
<status>#(status)#terminated::active::system#(/status)#</status>
|
|
<crawlingDomFilterContent>
|
|
#{crawlingDomFilterContent}#
|
|
<item>#[item]#</item>
|
|
#{/crawlingDomFilterContent}#
|
|
</crawlingDomFilterContent>
|
|
</crawlProfile>
|
|
#{/crawlProfiles}#
|
|
</crawlProfiles>
|