Marown/yacy_search_server
1
0
Fork 0
mirror of https://github.com/yacy/yacy_search_server.git synced 2024-09-19 00:01:41 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity

renovated the way how search results are count. should be correct now...

Browse Source
This commit is contained in:
Michael Peter Christen 2012-11-05 03:19:28 +01:00
parent 7bec253bb0
commit 8fb370d9f8
19 changed files with 228 additions and 311 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
htroot/AccessTracker_p.java
View File

@ -173,18 +173,18 @@ public class AccessTracker_p {
}
prop.putNum("page_list_" + m + "_querycount", query.itemsPerPage);
prop.putNum("page_list_" + m + "_transmitcount", query.transmitcount);
prop.putNum("page_list_" + m + "_resultcount", query.resultcount);
prop.putNum("page_list_" + m + "_resultcount", query.getResultCount());
prop.putNum("page_list_" + m + "_urltime", query.urlretrievaltime);
prop.putNum("page_list_" + m + "_snippettime", query.snippetcomputationtime);
prop.putNum("page_list_" + m + "_resulttime", query.searchtime);
prop.putHTML("page_list_" + m + "_userAgent", query.userAgent);
qcountSum += query.itemsPerPage;
rcountSum += query.resultcount;
rcountSum += query.getResultCount();
tcountSum += query.transmitcount;
utimeSum += query.urlretrievaltime;
stimeSum += query.snippetcomputationtime;
rtimeSum += query.searchtime;
if (query.resultcount > 0){
if (query.getResultCount() > 0){
rcount++;
utimeSum1 += query.urlretrievaltime;
stimeSum1 += query.snippetcomputationtime;

5
htroot/HostBrowser.java
View File

@ -189,9 +189,12 @@ public class HostBrowser {
prop.putHTML("outbound_host", host);
prop.putHTML("inbound_host", host);
String hosthash = ASCII.String(uri.hash(), 6, 6);
String[] pathparts = uri.getPaths();
// get all files for a specific host from the index
BlockingQueue<SolrDocument> docs = fulltext.getSolr().concurrentQuery(YaCySchema.host_s.name() + ":" + host, 0, 100000, 3000, 100);
String query = YaCySchema.host_s.name() + ":" + host;
for (String pe: pathparts) if (pe.length() > 0) query += " AND " + YaCySchema.url_paths_sxt.name() + ":" + pe;
BlockingQueue<SolrDocument> docs = fulltext.getSolr().concurrentQuery(query, 0, 100000, 3000, 100);
SolrDocument doc;
Set<String> storedDocs = new HashSet<String>();
Set<String> inboundLinks = new HashSet<String>();

56
htroot/IndexControlRWIs_p.java
View File

@ -126,7 +126,7 @@ public class IndexControlRWIs_p {
if ( post.containsKey("keystringsearch") ) {
prop.put("keyhash", keyhash);
final SearchEvent theSearch = genSearchresult(prop, sb, segment, keyhash, null);
if ( theSearch.rankingProcess.rwiAvailableCount() == 0 ) {
if (theSearch.query.local_rwi_available.get() == 0) {
prop.put("searchresult", 1);
prop.putHTML("searchresult_word", keystring);
}
@ -137,7 +137,7 @@ public class IndexControlRWIs_p {
prop.put("keystring", "&lt;" + errmsg + "&gt;");
}
final SearchEvent theSearch = genSearchresult(prop, sb, segment, keyhash, null);
if ( theSearch.rankingProcess.rwiAvailableCount() == 0 ) {
if (theSearch.query.local_rwi_available.get() == 0) {
prop.put("searchresult", 2);
prop.putHTML("searchresult_wordhash", ASCII.String(keyhash));
}
@ -466,7 +466,7 @@ public class IndexControlRWIs_p {
final String keyhashs = ASCII.String(keyhash);
prop.put("genUrlList_keyHash", keyhashs);
if ( theSearch.rankingProcess.rwiAvailableCount() == 0 ) {
if (theSearch.query.local_rwi_stored.get() == 0) {
prop.put("genUrlList", 1);
prop.put("genUrlList_count", 0);
prop.put("searchresult", 2);
@ -533,39 +533,17 @@ public class IndexControlRWIs_p {
prop
.put(
"genUrlList_urlList_" + i + "_urlExists_props",
((entry.word().flags().get(Condenser.flag_cat_indexof))
? "appears on index page, "
: "")
+ ((entry.word().flags().get(Condenser.flag_cat_hasimage))
? "contains images, "
: "")
+ ((entry.word().flags().get(Condenser.flag_cat_hasaudio))
? "contains audio, "
: "")
+ ((entry.word().flags().get(Condenser.flag_cat_hasvideo))
? "contains video, "
: "")
+ ((entry.word().flags().get(Condenser.flag_cat_hasapp))
? "contains applications, "
: "")
+ ((entry.word().flags().get(WordReferenceRow.flag_app_dc_identifier))
? "appears in url, "
: "")
+ ((entry.word().flags().get(WordReferenceRow.flag_app_dc_title))
? "appears in title, "
: "")
+ ((entry.word().flags().get(WordReferenceRow.flag_app_dc_creator))
? "appears in author, "
: "")
+ ((entry.word().flags().get(WordReferenceRow.flag_app_dc_subject))
? "appears in subject, "
: "")
+ ((entry.word().flags().get(WordReferenceRow.flag_app_dc_description))
? "appears in description, "
: "")
+ ((entry.word().flags().get(WordReferenceRow.flag_app_emphasized))
? "appears emphasized, "
: "")
((entry.word().flags().get(Condenser.flag_cat_indexof)) ? "appears on index page, " : "")
+ ((entry.word().flags().get(Condenser.flag_cat_hasimage)) ? "contains images, " : "")
+ ((entry.word().flags().get(Condenser.flag_cat_hasaudio)) ? "contains audio, " : "")
+ ((entry.word().flags().get(Condenser.flag_cat_hasvideo)) ? "contains video, " : "")
+ ((entry.word().flags().get(Condenser.flag_cat_hasapp)) ? "contains applications, " : "")
+ ((entry.word().flags().get(WordReferenceRow.flag_app_dc_identifier)) ? "appears in url, " : "")
+ ((entry.word().flags().get(WordReferenceRow.flag_app_dc_title)) ? "appears in title, " : "")
+ ((entry.word().flags().get(WordReferenceRow.flag_app_dc_creator)) ? "appears in author, " : "")
+ ((entry.word().flags().get(WordReferenceRow.flag_app_dc_subject)) ? "appears in subject, " : "")
+ ((entry.word().flags().get(WordReferenceRow.flag_app_dc_description)) ? "appears in description, " : "")
+ ((entry.word().flags().get(WordReferenceRow.flag_app_emphasized)) ? "appears emphasized, " : "")
+ ((DigestURI.probablyRootURL(entry.word().urlhash())) ? "probably root url" : ""));
if ( Switchboard.urlBlacklist.isListed(BlacklistType.DHT, url) ) {
prop.put("genUrlList_urlList_" + i + "_urlExists_urlhxChecked", "1");
@ -575,7 +553,7 @@ public class IndexControlRWIs_p {
break;
}
}
final Iterator<byte[]> iter = theSearch.rankingProcess.miss(); // iterates url hash strings
final Iterator<byte[]> iter = theSearch.query.misses.iterator(); // iterates url hash strings
byte[] b;
while ( iter.hasNext() ) {
b = iter.next();
@ -677,12 +655,12 @@ public class IndexControlRWIs_p {
final SearchEvent theSearch = SearchEventCache.getEvent(query, sb.peers, sb.tables, null, false, sb.loader, Integer.MAX_VALUE, Long.MAX_VALUE, (int) sb.getConfigLong(SwitchboardConstants.DHT_BURST_ROBINSON, 0), (int) sb.getConfigLong(SwitchboardConstants.DHT_BURST_MULTIWORD, 0));
//theSearch.rankingProcess.run();
RankingProcess ranked = theSearch.rankingProcess;
if ( ranked.rwiAvailableCount() == 0 ) {
if (theSearch.query.local_rwi_available.get() == 0) {
prop.put("searchresult", 2);
prop.put("searchresult_wordhash", keyhash);
} else {
prop.put("searchresult", 3);
prop.put("searchresult_allurl", ranked.rwiAvailableCount());
prop.put("searchresult_allurl", theSearch.query.local_rwi_available.get());
prop
.put("searchresult_description", ranked.flagCount()[WordReferenceRow.flag_app_dc_description]);
prop.put("searchresult_title", ranked.flagCount()[WordReferenceRow.flag_app_dc_title]);

13
htroot/yacy/search.java
View File

@ -213,7 +213,6 @@ public final class search {
// prepare an abstract result
final StringBuilder indexabstract = new StringBuilder(6000);
int indexabstractContainercount = 0;
int joincount = 0;
QueryParams theQuery = null;
SearchEvent theSearch = null;
ArrayList<WeakPriorityBlockingQueue.Element<ResultEntry>> accu = null;
@ -324,12 +323,11 @@ public final class search {
theSearch = SearchEventCache.getEvent(theQuery, sb.peers, sb.tables, null, abstracts.length() > 0, sb.loader, count, maxtime, (int) sb.getConfigLong(SwitchboardConstants.DHT_BURST_ROBINSON, 0), (int) sb.getConfigLong(SwitchboardConstants.DHT_BURST_MULTIWORD, 0));
// set statistic details of search result and find best result index set
joincount = theSearch.rankingProcess.rwiAvailableCount() - theSearch.rankingProcess.getMissCount() - theSearch.getSortOutCount();
prop.put("joincount", Integer.toString(joincount));
if (joincount != 0) {
prop.put("joincount", Integer.toString(theQuery.getResultCount()));
if (theQuery.getResultCount() > 0) {
accu = theSearch.completeResults(maxtime);
}
if (joincount <= 0 || abstracts.isEmpty()) {
if (theQuery.getResultCount() <= 0 || abstracts.isEmpty()) {
prop.put("indexcount", "");
} else {
// attach information about index abstracts
@ -391,7 +389,7 @@ public final class search {
prop.put("indexabstract", indexabstract.toString());
// prepare result
if (joincount == 0 || accu == null || accu.isEmpty()) {
if (theQuery.getResultCount() == 0 || accu == null || accu.isEmpty()) {
// no results
prop.put("links", "");
@ -419,7 +417,6 @@ public final class search {
// prepare search statistics
theQuery.remotepeer = client == null ? null : sb.peers.lookupByIP(Domains.dnsResolve(client), -1, true, false, false);
theQuery.resultcount = (theSearch == null) ? 0 : joincount;
theQuery.searchtime = System.currentTimeMillis() - timestamp;
theQuery.urlretrievaltime = (theSearch == null) ? 0 : theSearch.getURLRetrievalTime();
theQuery.snippetcomputationtime = (theSearch == null) ? 0 : theSearch.getSnippetComputationTime();
@ -436,7 +433,7 @@ public final class search {
// log
Network.log.logInfo("EXIT HASH SEARCH: " +
QueryParams.anonymizedQueryHashes(theQuery.query_include_hashes) + " - " + joincount + " links found, " +
QueryParams.anonymizedQueryHashes(theQuery.query_include_hashes) + " - " + theQuery.getResultCount() + " links found, " +
prop.get("linkcount", "?") + " links selected, " +
indexabstractContainercount + " index abstracts, " +
(System.currentTimeMillis() - timestamp) + " milliseconds");

68
htroot/yacysearch.java
View File

@ -831,27 +831,18 @@ public class yacysearch {
Log.logInfo("LOCAL_SEARCH", "EXIT WORD SEARCH: "
+ theQuery.queryString
+ " - "
+ "local-unfiltered("
+ theSearch.rankingProcess.rwiAvailableCount()
+ "), "
+ "local_miss("
+ theSearch.rankingProcess.getMissCount()
+ "), "
+ "local_sortout("
+ theSearch.getSortOutCount()
+ "), "
+ "remote("
+ theSearch.rankingProcess.getRemoteResourceSize()
+ ") links found, "
+ "local_rwi_available(" + theSearch.query.local_rwi_available.get() + "), "
+ "local_rwi_stored(" + theSearch.query.local_rwi_stored.get() + "), "
+ "local_solr_available(" + theSearch.query.local_solr_available.get() + "), "
+ "local_solr_stored(" + theSearch.query.local_solr_stored.get() + "), "
+ "remote_available(" + theSearch.query.remote_available.get() + "), "
+ "remote_stored(" + theSearch.query.remote_stored.get() + "), "
+ "remote_peerCount(" + theSearch.query.remote_peerCount.get() + "), "
+ "local_sortout(" + theSearch.query.misses.size() + "), "
+ (System.currentTimeMillis() - timestamp)
+ " ms");
// prepare search statistics
theQuery.resultcount =
theSearch.rankingProcess.rwiAvailableCount()
- theSearch.getSortOutCount()
- theSearch.rankingProcess.getMissCount()
+ theSearch.rankingProcess.getRemoteIndexCount();
theQuery.searchtime = System.currentTimeMillis() - timestamp;
theQuery.urlretrievaltime = theSearch.getURLRetrievalTime();
theQuery.snippetcomputationtime = theSearch.getSnippetComputationTime();
@ -932,37 +923,16 @@ public class yacysearch {
Log.logException(e);
}
final int indexcount =
theSearch.rankingProcess.rwiAvailableCount()
- theSearch.getSortOutCount()
- theSearch.rankingProcess.getMissCount()
+ theSearch.rankingProcess.getRemoteIndexCount();
prop.put("num-results_offset", startRecord == 0 ? 0 : startRecord + 1);
prop.put("num-results_itemscount", Formatter.number(
startRecord + theSearch.getQuery().itemsPerPage > indexcount ? startRecord
+ indexcount
% theSearch.getQuery().itemsPerPage : startRecord + theSearch.getQuery().itemsPerPage,
true));
prop.put("num-results_itemsPerPage", itemsPerPage);
prop.put("num-results_totalcount", indexcount);
prop.put("num-results_globalresults", global && (indexReceiveGranted || clustersearch)
? "1"
: "0");
prop.put(
"num-results_globalresults_localResourceSize",
Formatter.number(theSearch.rankingProcess.rwiAvailableCount(), true));
prop.put(
"num-results_globalresults_localMissCount",
Formatter.number(theSearch.rankingProcess.getMissCount(), true));
prop.put(
"num-results_globalresults_remoteResourceSize",
Formatter.number(theSearch.rankingProcess.getRemoteResourceSize(), true));
prop.put(
"num-results_globalresults_remoteIndexCount",
Formatter.number(theSearch.rankingProcess.getRemoteIndexCount(), true));
prop.put(
"num-results_globalresults_remotePeerCount",
Formatter.number(theSearch.rankingProcess.getRemotePeerCount(), true));
prop.put("num-results_itemscount", Formatter.number(startRecord + theSearch.query.itemsPerPage > theSearch.query.getResultCount() ? startRecord + theSearch.query.getResultCount() % theSearch.query.itemsPerPage : startRecord + theSearch.query.itemsPerPage, true));
prop.put("num-results_itemsPerPage", Formatter.number(itemsPerPage));
prop.put("num-results_totalcount", Formatter.number(theSearch.query.getResultCount()));
prop.put("num-results_globalresults", global && (indexReceiveGranted || clustersearch) ? "1" : "0");
prop.put("num-results_globalresults_localResourceSize", Formatter.number(theSearch.query.local_rwi_available.get() + theSearch.query.local_solr_available.get(), true));
prop.put("num-results_globalresults_localMissCount", Formatter.number(theSearch.query.misses.size(), true));
prop.put("num-results_globalresults_remoteResourceSize", Formatter.number(theSearch.query.remote_available.get(), true));
prop.put("num-results_globalresults_remoteIndexCount", Formatter.number(theSearch.query.remote_stored.get(), true));
prop.put("num-results_globalresults_remotePeerCount", Formatter.number(theSearch.query.remote_peerCount.get(), true));
// compose page navigation
final StringBuilder resnav = new StringBuilder(200);
@ -982,7 +952,7 @@ public class yacysearch {
resnav
.append("\"><img src=\"env/grafics/navdl.gif\" alt=\"arrowleft\" width=\"16\" height=\"16\" /></a>&nbsp;");
}
final int numberofpages = Math.min(10, 1 + ((indexcount - 1) / theQuery.itemsPerPage()));
final int numberofpages = Math.min(10, 1 + ((theSearch.query.getResultCount() - 1) / theQuery.itemsPerPage()));
for ( int i = 0; i < numberofpages; i++ ) {
if ( i == thispage ) {
@ -1020,7 +990,7 @@ public class yacysearch {
}
final String resnavs = resnav.toString();
prop.put("num-results_resnav", resnavs);
prop.put("pageNavBottom", (indexcount - startRecord > 6) ? 1 : 0); // if there are more results than may fit on the page we add a navigation at the bottom
prop.put("pageNavBottom", (theSearch.query.getResultCount() - startRecord > 6) ? 1 : 0); // if there are more results than may fit on the page we add a navigation at the bottom
prop.put("pageNavBottom_resnav", resnavs);
// generate the search result lines; the content will be produced by another servlet

73
htroot/yacysearchitem.java
View File

@ -89,28 +89,25 @@ public class yacysearchitem {
// the event does not exist, show empty page
return prop;
}
final QueryParams theQuery = theSearch.getQuery();
// dynamically update count values
final int totalcount = theSearch.rankingProcess.rwiAvailableCount() - theSearch.rankingProcess.getMissCount() - theSearch.getSortOutCount() + theSearch.rankingProcess.getRemoteIndexCount();
final int offset = theQuery.neededResults() - theQuery.itemsPerPage() + 1;
prop.put("offset", offset);
prop.put("itemscount", Formatter.number(Math.min((item < 0) ? theQuery.neededResults() : item + 1, totalcount)));
prop.put("itemsperpage", Formatter.number(theQuery.itemsPerPage));
prop.put("totalcount", Formatter.number(totalcount, true));
prop.put("localResourceSize", Formatter.number(theSearch.rankingProcess.rwiAvailableCount(), true));
prop.put("localMissCount", Formatter.number(theSearch.rankingProcess.getMissCount(), true));
prop.put("remoteResourceSize", Formatter.number(theSearch.rankingProcess.getRemoteResourceSize(), true));
prop.put("remoteIndexCount", Formatter.number(theSearch.rankingProcess.getRemoteIndexCount(), true));
prop.put("remotePeerCount", Formatter.number(theSearch.rankingProcess.getRemotePeerCount(), true));
prop.put("navurlBase", QueryParams.navurlBase("html", theQuery, null, theQuery.urlMask.toString(), theQuery.navigators).toString());
prop.put("offset", theSearch.query.neededResults() - theSearch.query.itemsPerPage() + 1);
prop.put("itemscount", Formatter.number(Math.min((item < 0) ? theSearch.query.neededResults() : item + 1, theSearch.query.getResultCount())));
prop.put("itemsperpage", Formatter.number(theSearch.query.itemsPerPage));
prop.put("totalcount", Formatter.number(theSearch.query.getResultCount(), true));
prop.put("localResourceSize", Formatter.number(theSearch.query.local_rwi_available.get() + theSearch.query.local_solr_available.get(), true));
prop.put("localMissCount", Formatter.number(theSearch.query.misses.size(), true));
prop.put("remoteResourceSize", Formatter.number(theSearch.query.remote_available.get(), true));
prop.put("remoteIndexCount", Formatter.number(theSearch.query.remote_stored.get(), true));
prop.put("remotePeerCount", Formatter.number(theSearch.query.remote_peerCount.get(), true));
prop.put("navurlBase", QueryParams.navurlBase("html", theSearch.query, null, theSearch.query.urlMask.toString(), theSearch.query.navigators).toString());
final String target_special_pattern = sb.getConfig(SwitchboardConstants.SEARCH_TARGET_SPECIAL_PATTERN, "");
if (theQuery.contentdom == Classification.ContentDomain.TEXT || theQuery.contentdom == Classification.ContentDomain.ALL) {
if (theSearch.query.contentdom == Classification.ContentDomain.TEXT || theSearch.query.contentdom == Classification.ContentDomain.ALL) {
// text search
// generate result object
final ResultEntry result = theSearch.oneResult(item, theQuery.isLocal() ? 1000 : 5000);
final ResultEntry result = theSearch.oneResult(item, theSearch.query.isLocal() ? 1000 : 5000);
if (result == null) return prop; // no content
final String resultUrlstring = result.urlstring();
final DigestURI resultURL = result.url();
@ -124,7 +121,7 @@ public class yacysearchitem {
Log.logException(e1);
faviconURL = null;
}
final String resource = theQuery.domType.toString();
final String resource = theSearch.query.domType.toString();
prop.put("content", 1); // switch on specific content
prop.put("content_showDate", sb.getConfigBool("search.result.show.date", true) ? 1 : 0);
prop.put("content_showSize", sb.getConfigBool("search.result.show.size", true) ? 1 : 0);
@ -137,10 +134,10 @@ public class yacysearchitem {
prop.put("content_authorized", authenticated ? "1" : "0");
final String urlhash = ASCII.String(result.hash());
prop.put("content_authorized_bookmark", sb.tables.bookmarks.hasBookmark("admin", urlhash) ? "0" : "1");
prop.putHTML("content_authorized_bookmark_bookmarklink", "/yacysearch.html?query=" + theQuery.queryString.replace(' ', '+') + "&Enter=Search&count=" + theQuery.itemsPerPage() + "&offset=" + (theQuery.neededResults() - theQuery.itemsPerPage()) + "&order=" + crypt.simpleEncode(theQuery.ranking.toExternalString()) + "&resource=" + resource + "&time=3&bookmarkref=" + urlhash + "&urlmaskfilter=.*");
prop.putHTML("content_authorized_bookmark_bookmarklink", "/yacysearch.html?query=" + theSearch.query.queryString.replace(' ', '+') + "&Enter=Search&count=" + theSearch.query.itemsPerPage() + "&offset=" + (theSearch.query.neededResults() - theSearch.query.itemsPerPage()) + "&order=" + crypt.simpleEncode(theSearch.query.ranking.toExternalString()) + "&resource=" + resource + "&time=3&bookmarkref=" + urlhash + "&urlmaskfilter=.*");
prop.put("content_authorized_recommend", (sb.peers.newsPool.getSpecific(NewsPool.OUTGOING_DB, NewsPool.CATEGORY_SURFTIPP_ADD, "url", resultUrlstring) == null) ? "1" : "0");
prop.putHTML("content_authorized_recommend_deletelink", "/yacysearch.html?query=" + theQuery.queryString.replace(' ', '+') + "&Enter=Search&count=" + theQuery.itemsPerPage() + "&offset=" + (theQuery.neededResults() - theQuery.itemsPerPage()) + "&order=" + crypt.simpleEncode(theQuery.ranking.toExternalString()) + "&resource=" + resource + "&time=3&deleteref=" + urlhash + "&urlmaskfilter=.*");
prop.putHTML("content_authorized_recommend_recommendlink", "/yacysearch.html?query=" + theQuery.queryString.replace(' ', '+') + "&Enter=Search&count=" + theQuery.itemsPerPage() + "&offset=" + (theQuery.neededResults() - theQuery.itemsPerPage()) + "&order=" + crypt.simpleEncode(theQuery.ranking.toExternalString()) + "&resource=" + resource + "&time=3&recommendref=" + urlhash + "&urlmaskfilter=.*");
prop.putHTML("content_authorized_recommend_deletelink", "/yacysearch.html?query=" + theSearch.query.queryString.replace(' ', '+') + "&Enter=Search&count=" + theSearch.query.itemsPerPage() + "&offset=" + (theSearch.query.neededResults() - theSearch.query.itemsPerPage()) + "&order=" + crypt.simpleEncode(theSearch.query.ranking.toExternalString()) + "&resource=" + resource + "&time=3&deleteref=" + urlhash + "&urlmaskfilter=.*");
prop.putHTML("content_authorized_recommend_recommendlink", "/yacysearch.html?query=" + theSearch.query.queryString.replace(' ', '+') + "&Enter=Search&count=" + theSearch.query.itemsPerPage() + "&offset=" + (theSearch.query.neededResults() - theSearch.query.itemsPerPage()) + "&order=" + crypt.simpleEncode(theSearch.query.ranking.toExternalString()) + "&resource=" + resource + "&time=3&recommendref=" + urlhash + "&urlmaskfilter=.*");
prop.put("content_authorized_urlhash", urlhash);
final String resulthashString = urlhash;
prop.putHTML("content_title", result.title());
@ -205,11 +202,11 @@ public class yacysearchitem {
prop.putHTML("content_host", resultURL.getHost() == null ? "" : resultURL.getHost());
prop.putHTML("content_file", resultURL.getFileName());
prop.putHTML("content_path", resultURL.getPath());
prop.put("content_nl", (item == theQuery.offset) ? 0 : 1);
prop.put("content_nl", (item == theSearch.query.offset) ? 0 : 1);
prop.putHTML("content_publisher", result.publisher());
prop.putHTML("content_creator", result.creator());// author
prop.putHTML("content_subject", result.subject());
final Collection<String>[] query = theQuery.queryWords();
final Collection<String>[] query = theSearch.query.queryWords();
final StringBuilder s = new StringBuilder(query[0].size() * 20);
for (final String t: query[0]) {
s.append('+').append(t);
@ -217,10 +214,10 @@ public class yacysearchitem {
final String words = (s.length() > 0) ? s.substring(1) : "";
prop.putHTML("content_words", words);
prop.putHTML("content_showParser_words", words);
prop.putHTML("content_former", theQuery.queryString);
prop.putHTML("content_showPictures_former", theQuery.queryString);
prop.putHTML("content_former", theSearch.query.queryString);
prop.putHTML("content_showPictures_former", theSearch.query.queryString);
final TextSnippet snippet = result.textSnippet();
final String desc = (snippet == null) ? "" : snippet.getLineMarked(theQuery.query_all_hashes);
final String desc = (snippet == null) ? "" : snippet.getLineMarked(theSearch.query.query_all_hashes);
prop.put("content_description", desc);
prop.putXML("content_description-xml", desc);
prop.putJSON("content_description-json", desc);
@ -235,7 +232,7 @@ public class yacysearchitem {
}
prop.put("content_heuristic_name", heuristic.heuristicName);
}
EventTracker.update(EventTracker.EClass.SEARCH, new ProfilingGraph.EventSearch(theQuery.id(true), SearchEventType.FINALIZATION, "" + item, 0, 0), false);
EventTracker.update(EventTracker.EClass.SEARCH, new ProfilingGraph.EventSearch(theSearch.query.id(true), SearchEventType.FINALIZATION, "" + item, 0, 0), false);
final String ext = resultURL.getFileExtension().toLowerCase();
if (ext.equals("png") || ext.equals("jpg") || ext.equals("gif")) {
final String license = sb.licensedURLs.aquireLicense(resultURL);
@ -251,16 +248,16 @@ public class yacysearchitem {
prop.put("content_loc_lon", result.lon());
}
if (sb.getConfigBool("heuristic.searchresults",false)) sb.heuristicSearchResults(resultUrlstring);
theQuery.transmitcount = item + 1;
theSearch.query.transmitcount = item + 1;
return prop;
}
if (theQuery.contentdom == Classification.ContentDomain.IMAGE) {
if (theSearch.query.contentdom == Classification.ContentDomain.IMAGE) {
// image search; shows thumbnails
prop.put("content", theQuery.contentdom.getCode() + 1); // switch on specific content
prop.put("content", theSearch.query.contentdom.getCode() + 1); // switch on specific content
//final MediaSnippet ms = theSearch.result().oneImage(item);
final ResultEntry ms = theSearch.oneResult(item, theQuery.isLocal() ? 1000 : 5000);
final ResultEntry ms = theSearch.oneResult(item, theSearch.query.isLocal() ? 1000 : 5000);
if (ms == null) {
prop.put("content_item", "0");
} else {
@ -283,21 +280,21 @@ public class yacysearchitem {
prop.put("content_item_source", ms.url().toNormalform(true));
prop.putXML("content_item_source-xml", ms.url().toNormalform(true));
prop.put("content_item_sourcedom", ms.url().getHost());
prop.put("content_item_nl", (item == theQuery.offset) ? 0 : 1);
prop.put("content_item_nl", (item == theSearch.query.offset) ? 0 : 1);
prop.put("content_item", 1);
}
theQuery.transmitcount = item + 1;
theSearch.query.transmitcount = item + 1;
return prop;
}
if ((theQuery.contentdom == ContentDomain.AUDIO) ||
(theQuery.contentdom == ContentDomain.VIDEO) ||
(theQuery.contentdom == ContentDomain.APP)) {
if ((theSearch.query.contentdom == ContentDomain.AUDIO) ||
(theSearch.query.contentdom == ContentDomain.VIDEO) ||
(theSearch.query.contentdom == ContentDomain.APP)) {
// any other media content
// generate result object
final ResultEntry ms = theSearch.oneResult(item, theQuery.isLocal() ? 1000 : 5000);
prop.put("content", theQuery.contentdom.getCode() + 1); // switch on specific content
final ResultEntry ms = theSearch.oneResult(item, theSearch.query.isLocal() ? 1000 : 5000);
prop.put("content", theSearch.query.contentdom.getCode() + 1); // switch on specific content
if (ms == null) {
prop.put("content_item", "0");
} else {
@ -308,10 +305,10 @@ public class yacysearchitem {
prop.putHTML("content_item_target", target);
prop.putHTML("content_item_name", shorten(ms.title(), MAX_NAME_LENGTH));
prop.put("content_item_col", (item % 2 == 0) ? "0" : "1");
prop.put("content_item_nl", (item == theQuery.offset) ? 0 : 1);
prop.put("content_item_nl", (item == theSearch.query.offset) ? 0 : 1);
prop.put("content_item", 1);
}
theQuery.transmitcount = item + 1;
theSearch.query.transmitcount = item + 1;
return prop;
}

22
htroot/yacysearchlatestinfo.java
View File

@ -30,21 +30,19 @@ public class yacysearchlatestinfo {
prop.put("remotePeerCount", 0);
return prop;
}
final QueryParams theQuery = theSearch.getQuery();
// dynamically update count values
final int totalcount = theSearch.rankingProcess.rwiAvailableCount() - theSearch.rankingProcess.getMissCount() - theSearch.getSortOutCount() + theSearch.rankingProcess.getRemoteIndexCount();
final int offset = theQuery.neededResults() - theQuery.itemsPerPage() + 1;
final int offset = theSearch.query.neededResults() - theSearch.query.itemsPerPage() + 1;
prop.put("offset", offset);
prop.put("itemscount",Formatter.number(offset + theSearch.getQuery().itemsPerPage >= totalcount ? offset + totalcount % theSearch.getQuery().itemsPerPage - 1 : offset + theSearch.getQuery().itemsPerPage - 1));
prop.put("itemsperpage", theSearch.getQuery().itemsPerPage);
prop.put("totalcount", Formatter.number(totalcount, true));
prop.put("localResourceSize", Formatter.number(theSearch.rankingProcess.rwiAvailableCount(), true));
prop.put("localMissCount", Formatter.number(theSearch.rankingProcess.getMissCount(), true));
prop.put("remoteResourceSize", Formatter.number(theSearch.rankingProcess.getRemoteResourceSize(), true));
prop.put("remoteIndexCount", Formatter.number(theSearch.rankingProcess.getRemoteIndexCount(), true));
prop.put("remotePeerCount", Formatter.number(theSearch.rankingProcess.getRemotePeerCount(), true));
prop.putJSON("navurlBase", QueryParams.navurlBase("html", theQuery, null, theQuery.urlMask.toString(), theQuery.navigators).toString());
prop.put("itemscount",Formatter.number(offset + theSearch.query.itemsPerPage >= theSearch.query.getResultCount() ? offset + theSearch.query.getResultCount() % theSearch.query.itemsPerPage - 1 : offset + theSearch.query.itemsPerPage - 1));
prop.put("itemsperpage", theSearch.query.itemsPerPage);
prop.put("totalcount", Formatter.number(theSearch.query.getResultCount(), true));
prop.put("localResourceSize", Formatter.number(theSearch.query.local_rwi_available.get() + theSearch.query.local_solr_available.get(), true));
prop.put("localMissCount", Formatter.number(theSearch.query.misses.size(), true));
prop.put("remoteResourceSize", Formatter.number(theSearch.query.remote_available.get(), true));
prop.put("remoteIndexCount", Formatter.number(theSearch.query.remote_stored.get(), true));
prop.put("remotePeerCount", Formatter.number(theSearch.query.remote_peerCount.get(), true));
prop.putJSON("navurlBase", QueryParams.navurlBase("html", theSearch.query, null, theSearch.query.urlMask.toString(), theSearch.query.navigators).toString());
return prop;
}

49
htroot/yacysearchtrailer.java
View File

@ -59,7 +59,6 @@ public class yacysearchtrailer {
// the event does not exist, show empty page
return prop;
}
final QueryParams theQuery = theSearch.getQuery();
final RequestHeader.FileType fileType = header.fileType();
// compose search navigation
@ -80,7 +79,7 @@ public class yacysearchtrailer {
name = navigatorIterator.next();
count = namespaceNavigator.get(name);
nav = "inurl%3A" + name;
queryStringForUrl = theQuery.queryStringForUrl();
queryStringForUrl = theSearch.query.queryStringForUrl();
p = queryStringForUrl.indexOf(nav);
if (p < 0) {
pos++;
@ -94,7 +93,7 @@ public class yacysearchtrailer {
queryStringForUrl = (queryStringForUrl.substring(0, p) + queryStringForUrl.substring(p + nav.length())).trim();
}
prop.put(fileType, "nav-namespace_element_" + i + "_name", name);
prop.put(fileType, "nav-namespace_element_" + i + "_url", QueryParams.navurl(fileType.name().toLowerCase(), 0, theQuery, queryStringForUrl, theQuery.urlMask.toString(), theQuery.navigators).toString());
prop.put(fileType, "nav-namespace_element_" + i + "_url", QueryParams.navurl(fileType.name().toLowerCase(), 0, theSearch.query, queryStringForUrl, theSearch.query.urlMask.toString(), theSearch.query.navigators).toString());
prop.put("nav-namespace_element_" + i + "_count", count);
prop.put("nav-namespace_element_" + i + "_nl", 1);
i++;
@ -119,7 +118,7 @@ public class yacysearchtrailer {
name = navigatorIterator.next();
count = hostNavigator.get(name);
nav = "site%3A" + name;
queryStringForUrl = theQuery.queryStringForUrl();
queryStringForUrl = theSearch.query.queryStringForUrl();
p = queryStringForUrl.indexOf(nav);
if (p < 0) {
pos++;
@ -133,7 +132,7 @@ public class yacysearchtrailer {
prop.put(fileType, "nav-authors_element_" + i + "_modifier", "-" + nav);
}
prop.put(fileType, "nav-domains_element_" + i + "_name", name);
prop.put(fileType, "nav-domains_element_" + i + "_url", QueryParams.navurl(fileType.name().toLowerCase(), 0, theQuery, queryStringForUrl, theQuery.urlMask.toString(), theQuery.navigators).toString());
prop.put(fileType, "nav-domains_element_" + i + "_url", QueryParams.navurl(fileType.name().toLowerCase(), 0, theSearch.query, queryStringForUrl, theSearch.query.urlMask.toString(), theSearch.query.navigators).toString());
prop.put("nav-domains_element_" + i + "_count", count);
prop.put("nav-domains_element_" + i + "_nl", 1);
i++;
@ -158,7 +157,7 @@ public class yacysearchtrailer {
name = navigatorIterator.next().trim();
count = authorNavigator.get(name);
nav = (name.indexOf(' ', 0) < 0) ? "author%3A" + name : "author%3A%28" + name.replace(" ", "+") + "%29";
queryStringForUrl = theQuery.queryStringForUrl();
queryStringForUrl = theSearch.query.queryStringForUrl();
p = queryStringForUrl.indexOf(nav);
if (p < 0) {
pos++;
@ -172,7 +171,7 @@ public class yacysearchtrailer {
prop.put(fileType, "nav-authors_element_" + i + "_modifier", "-" + nav);
}
prop.put(fileType, "nav-authors_element_" + i + "_name", name);
prop.put(fileType, "nav-authors_element_" + i + "_url", QueryParams.navurl(fileType.name().toLowerCase(), 0, theQuery, queryStringForUrl, theQuery.urlMask.toString(), theQuery.navigators).toString());
prop.put(fileType, "nav-authors_element_" + i + "_url", QueryParams.navurl(fileType.name().toLowerCase(), 0, theSearch.query, queryStringForUrl, theSearch.query.urlMask.toString(), theSearch.query.navigators).toString());
prop.put("nav-authors_element_" + i + "_count", count);
prop.put("nav-authors_element_" + i + "_nl", 1);
i++;
@ -196,13 +195,13 @@ public class yacysearchtrailer {
while (i < MAX_TOPWORDS && navigatorIterator.hasNext()) {
name = navigatorIterator.next();
count = topicNavigator.get(name);
if (theQuery.queryString == null) break;
if (theSearch.query.queryString == null) break;
if (name != null) {
queryStringForUrl = theQuery.queryStringForUrl();
queryStringForUrl = theSearch.query.queryStringForUrl();
prop.put("nav-topics_element_" + i + "_on", 1);
prop.put(fileType, "nav-topics_element_" + i + "_modifier", name);
prop.put(fileType, "nav-topics_element_" + i + "_name", name);
prop.put(fileType, "nav-topics_element_" + i + "_url", QueryParams.navurl(fileType.name().toLowerCase(), 0, theQuery, queryStringForUrl + "+" + name, theQuery.urlMask.toString(), theQuery.navigators).toString());
prop.put(fileType, "nav-topics_element_" + i + "_url", QueryParams.navurl(fileType.name().toLowerCase(), 0, theSearch.query, queryStringForUrl + "+" + name, theSearch.query.urlMask.toString(), theSearch.query.navigators).toString());
prop.put("nav-topics_element_" + i + "_count", count);
prop.put("nav-topics_element_" + i + "_nl", 1);
i++;
@ -226,7 +225,7 @@ public class yacysearchtrailer {
name = navigatorIterator.next().trim();
count = protocolNavigator.get(name);
nav = "%2F" + name;
queryStringForUrl = theQuery.queryStringForUrl();
queryStringForUrl = theSearch.query.queryStringForUrl();
p = queryStringForUrl.indexOf(nav);
if (p < 0) {
pos++;
@ -240,7 +239,7 @@ public class yacysearchtrailer {
prop.put(fileType, "nav-protocols_element_" + i + "_modifier", "-" + nav);
}
prop.put(fileType, "nav-protocols_element_" + i + "_name", name);
prop.put(fileType, "nav-protocols_element_" + i + "_url", QueryParams.navurl(fileType.name().toLowerCase(), 0, theQuery, queryStringForUrl, (p >= 0 && theQuery.urlMask.toString().startsWith(name)) ? ".*" : theQuery.urlMask.toString(), theQuery.navigators).toString());
prop.put(fileType, "nav-protocols_element_" + i + "_url", QueryParams.navurl(fileType.name().toLowerCase(), 0, theSearch.query, queryStringForUrl, (p >= 0 && theSearch.query.urlMask.toString().startsWith(name)) ? ".*" : theSearch.query.urlMask.toString(), theSearch.query.navigators).toString());
prop.put("nav-protocols_element_" + i + "_count", count);
prop.put("nav-protocols_element_" + i + "_nl", 1);
i++;
@ -265,7 +264,7 @@ public class yacysearchtrailer {
name = navigatorIterator.next().trim();
count = filetypeNavigator.get(name);
nav = "filetype%3A" + name;
queryStringForUrl = theQuery.queryStringForUrl();
queryStringForUrl = theSearch.query.queryStringForUrl();
p = queryStringForUrl.indexOf(nav);
if (p < 0) {
pos++;
@ -279,7 +278,7 @@ public class yacysearchtrailer {
prop.put(fileType, "nav-filetypes_element_" + i + "_modifier", "-" + nav);
}
prop.put(fileType, "nav-filetypes_element_" + i + "_name", name);
prop.put(fileType, "nav-filetypes_element_" + i + "_url", QueryParams.navurl(fileType.name().toLowerCase(), 0, theQuery, queryStringForUrl, (p >= 0 && theQuery.urlMask.toString().endsWith(name)) ? ".*" : theQuery.urlMask.toString(), theQuery.navigators).toString());
prop.put(fileType, "nav-filetypes_element_" + i + "_url", QueryParams.navurl(fileType.name().toLowerCase(), 0, theSearch.query, queryStringForUrl, (p >= 0 && theSearch.query.urlMask.toString().endsWith(name)) ? ".*" : theSearch.query.urlMask.toString(), theSearch.query.navigators).toString());
prop.put("nav-filetypes_element_" + i + "_count", count);
prop.put("nav-filetypes_element_" + i + "_nl", 1);
i++;
@ -308,7 +307,7 @@ public class yacysearchtrailer {
name = navigatorIterator.next();
count = ve.getValue().get(name);
nav = "%2Fvocabulary%2F" + navname + "%2F" + MultiProtocolURI.escape(Tagging.encodePrintname(name)).toString();
queryStringForUrl = theQuery.queryStringForUrl();
queryStringForUrl = theSearch.query.queryStringForUrl();
p = queryStringForUrl.indexOf(nav);
if (p < 0) {
queryStringForUrl += "+" + nav;
@ -320,7 +319,7 @@ public class yacysearchtrailer {
prop.put(fileType, "nav-vocabulary_" + navvoccount + "_element_" + i + "_modifier", "-" + nav);
}
prop.put(fileType, "nav-vocabulary_" + navvoccount + "_element_" + i + "_name", name);
prop.put(fileType, "nav-vocabulary_" + navvoccount + "_element_" + i + "_url", QueryParams.navurl(fileType.name().toLowerCase(), 0, theQuery, queryStringForUrl, theQuery.urlMask.toString(), theQuery.navigators).toString());
prop.put(fileType, "nav-vocabulary_" + navvoccount + "_element_" + i + "_url", QueryParams.navurl(fileType.name().toLowerCase(), 0, theSearch.query, queryStringForUrl, theSearch.query.urlMask.toString(), theSearch.query.navigators).toString());
prop.put("nav-vocabulary_" + navvoccount + "_element_" + i + "_count", count);
prop.put("nav-vocabulary_" + navvoccount + "_element_" + i + "_nl", 1);
i++;
@ -338,8 +337,7 @@ public class yacysearchtrailer {
// about box
final String aboutBody = env.getConfig("about.body", "");
final String aboutHeadline = env.getConfig("about.headline", "");
if ((aboutBody.isEmpty() && aboutHeadline.isEmpty()) ||
theSearch.rankingProcess.rwiAvailableCount() - theSearch.rankingProcess.getMissCount() - theSearch.getSortOutCount() + theSearch.rankingProcess.getRemoteIndexCount() == 0) {
if ((aboutBody.isEmpty() && aboutHeadline.isEmpty()) || theSearch.query.getResultCount() == 0) {
prop.put("nav-about", 0);
} else {
prop.put("nav-about", 1);
@ -349,20 +347,15 @@ public class yacysearchtrailer {
// category: location search
// show only if there is a location database present and if there had been any search results
if (LibraryProvider.geoLoc.isEmpty() ||
theSearch.rankingProcess.rwiAvailableCount() == 0) {
if (LibraryProvider.geoLoc.isEmpty() || theSearch.query.getResultCount() == 0) {
prop.put("cat-location", 0);
} else {
prop.put("cat-location", 1);
prop.put(fileType, "cat-location_query", theQuery.queryString(true));
prop.put(fileType, "cat-location_queryenc", theQuery.queryString(true).replace(' ', '+'));
prop.put(fileType, "cat-location_query", theSearch.query.queryString(true));
prop.put(fileType, "cat-location_queryenc", theSearch.query.queryString(true).replace(' ', '+'));
}
final int indexcount = theSearch.rankingProcess.rwiAvailableCount() - theSearch.rankingProcess.getMissCount() - theSearch.getSortOutCount() + theSearch.rankingProcess.getRemoteIndexCount();
prop.put("num-results_totalcount", indexcount);
EventTracker.update(EventTracker.EClass.SEARCH, new ProfilingGraph.EventSearch(theQuery.id(true), SearchEventType.FINALIZATION, "bottomline", 0, 0), false);
prop.put("num-results_totalcount", theSearch.query.getResultCount());
EventTracker.update(EventTracker.EClass.SEARCH, new ProfilingGraph.EventSearch(theSearch.query.id(true), SearchEventType.FINALIZATION, "bottomline", 0, 0), false);
return prop;
}

2
source/net/yacy/cora/federate/solr/connector/EmbeddedSolrConnector.java
View File

@ -132,7 +132,9 @@ public class EmbeddedSolrConnector extends SolrServerConnector implements SolrCo
@Override
public synchronized void close() {
this.commit();
super.close();
this.defaultCore.close();
this.cores.shutdown();
}

37
source/net/yacy/peers/Protocol.java
View File

@ -117,6 +117,10 @@ import org.apache.solr.client.solrj.util.ClientUtils;
import org.apache.solr.common.SolrDocument;
import org.apache.solr.common.SolrDocumentList;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.SimpleOrderedMap;
import org.apache.solr.response.ResultContext;
import org.apache.solr.search.DocList;
public final class Protocol
@ -590,7 +594,7 @@ public final class Protocol
null);
}
public static int primarySearch(
protected static int primarySearch(
final SearchEvent event,
final String wordhashes,
final String excludehashes,
@ -689,7 +693,7 @@ public final class Protocol
return result.urlcount;
}
public static int secondarySearch(
protected static int secondarySearch(
final SearchEvent event,
final String wordhashes,
final String urlhashes,
@ -810,7 +814,7 @@ public final class Protocol
// passed all checks, store url
try {
event.getQuery().getSegment().fulltext().putMetadata(urlEntry);
event.query.getSegment().fulltext().putMetadata(urlEntry);
ResultURLs.stack(
ASCII.String(urlEntry.url().hash()),
urlEntry.url().getHost(),
@ -853,7 +857,7 @@ public final class Protocol
// insert the containers to the index
for ( final ReferenceContainer<WordReference> c : container ) {
try {
event.getQuery().getSegment().storeRWI(c);
event.query.getSegment().storeRWI(c);
} catch ( final Exception e ) {
Log.logException(e);
}
@ -938,8 +942,8 @@ public final class Protocol
parts.put("exclude", UTF8.StringBody(excludehashes));
parts.put("duetime", UTF8.StringBody("1000"));
parts.put("urls", UTF8.StringBody(urlhashes));
parts.put("prefer", UTF8.StringBody(event.getQuery().prefer.pattern()));
parts.put("filter", UTF8.StringBody(event.getQuery().urlMask.pattern()));
parts.put("prefer", UTF8.StringBody(event.query.prefer.pattern()));
parts.put("filter", UTF8.StringBody(event.query.urlMask.pattern()));
parts.put("modifier", UTF8.StringBody(modifier));
parts.put("language", UTF8.StringBody(language));
parts.put("sitehash", UTF8.StringBody(sitehash));
@ -947,8 +951,8 @@ public final class Protocol
parts.put("contentdom", UTF8.StringBody(contentdom));
parts.put("ttl", UTF8.StringBody("0"));
parts.put("maxdist", UTF8.StringBody(Integer.toString(maxDistance)));
parts.put("profile", UTF8.StringBody(crypt.simpleEncode(event.getQuery().ranking.toExternalString())));
parts.put("constraint", UTF8.StringBody((event.getQuery().constraint == null) ? "" : event.getQuery().constraint.exportB64()));
parts.put("profile", UTF8.StringBody(crypt.simpleEncode(event.query.ranking.toExternalString())));
parts.put("constraint", UTF8.StringBody((event.query.constraint == null) ? "" : event.query.constraint.exportB64()));
if ( secondarySearchSuperviser != null ) {
parts.put("abstracts", UTF8.StringBody("auto"));
// resultMap = FileUtils.table(HTTPConnector.getConnector(MultiProtocolURI.yacybotUserAgent).post(new MultiProtocolURI("http://" + hostaddress + "/yacy/search.html"), 60000, hostname, parts));
@ -1018,26 +1022,27 @@ public final class Protocol
}
}
public static int solrQuery(
protected static int solrQuery(
final SearchEvent event,
final int offset,
final int count,
final Seed target,
final Blacklist blacklist) {
if (event.getQuery().queryString == null || event.getQuery().queryString.length() == 0) {
if (event.query.queryString == null || event.query.queryString.length() == 0) {
return -1; // we cannot query solr only with word hashes, there is no clear text string
}
event.addExpectedRemoteReferences(count);
QueryResponse rsp = null;
SolrDocumentList docList = null;
final SolrQuery solrQuery = event.getQuery().solrQuery();
final SolrQuery solrQuery = event.query.solrQuery();
solrQuery.setStart(offset);
solrQuery.setRows(count);
boolean localsearch = target == null || target.equals(event.peers.mySeed());
if (localsearch) {
// search the local index
try {
QueryResponse rsp = event.rankingProcess.getQuery().getSegment().fulltext().getSolr().query(solrQuery);
rsp = event.rankingProcess.getQuery().getSegment().fulltext().getSolr().query(solrQuery);
docList = rsp.getResults();
} catch (SolrException e) {
Network.log.logInfo("SEARCH failed (solr, 1), localpeer (" + e.getMessage() + ")", e);
@ -1050,7 +1055,7 @@ public final class Protocol
final String solrURL = "http://" + target.getPublicAddress() + "/solr";
try {
SolrConnector solrConnector = new RemoteSolrConnector(solrURL);
QueryResponse rsp = solrConnector.query(solrQuery);
rsp = solrConnector.query(solrQuery);
docList = rsp.getResults();
// no need to close this here because that sends a commit to remote solr which is not wanted here
} catch (IOException e) {
@ -1103,7 +1108,7 @@ public final class Protocol
// passed all checks, store url
if (!localsearch) {
try {
event.getQuery().getSegment().fulltext().putDocument(ClientUtils.toSolrInputDocument(doc));
event.query.getSegment().fulltext().putDocument(ClientUtils.toSolrInputDocument(doc));
ResultURLs.stack(
ASCII.String(urlEntry.url().hash()),
urlEntry.url().getHost(),
@ -1121,12 +1126,12 @@ public final class Protocol
}
if (localsearch) {
event.add(container, true, "localpeer", docList.size());
event.add(container, true, "localpeer", (int) docList.getNumFound());
event.rankingProcess.addFinalize();
event.addExpectedRemoteReferences(-count);
Network.log.logInfo("local search (solr): localpeer sent " + container.get(0).size() + "/" + docList.size() + " references");
} else {
event.add(container, false, target.getName() + "/" + target.hash, docList.size());
event.add(container, false, target.getName() + "/" + target.hash, (int) docList.getNumFound());
event.rankingProcess.addFinalize();
event.addExpectedRemoteReferences(-count);
Network.log.logInfo("remote search (solr): peer " + target.getName() + " sent " + container.get(0).size() + "/" + docList.size() + " references");

18
source/net/yacy/peers/RemoteSearch.java
View File

@ -149,7 +149,7 @@ public class RemoteSearch extends Thread {
(clusterselection == null) ?
DHTSelection.selectSearchTargets(
event.peers,
event.getQuery().query_include_hashes,
event.query.query_include_hashes,
event.peers.redundancy(),
burstRobinsonPercent,
burstMultiwordPercent)
@ -172,16 +172,16 @@ public class RemoteSearch extends Thread {
try {
RemoteSearch rs = new RemoteSearch(
event,
QueryParams.hashSet2hashString(event.getQuery().query_include_hashes),
QueryParams.hashSet2hashString(event.getQuery().query_exclude_hashes),
event.getQuery().modifier,
event.getQuery().targetlang == null ? "" : event.getQuery().targetlang,
event.getQuery().sitehash == null ? "" : event.getQuery().sitehash,
event.getQuery().authorhash == null ? "" : event.getQuery().authorhash,
event.getQuery().contentdom == null ? "all" : event.getQuery().contentdom.toString(),
QueryParams.hashSet2hashString(event.query.query_include_hashes),
QueryParams.hashSet2hashString(event.query.query_exclude_hashes),
event.query.modifier,
event.query.targetlang == null ? "" : event.query.targetlang,
event.query.sitehash == null ? "" : event.query.sitehash,
event.query.authorhash == null ? "" : event.query.authorhash,
event.query.contentdom == null ? "all" : event.query.contentdom.toString(),
count,
time,
event.getQuery().maxDistance,
event.query.maxDistance,
targets,
targetPeers[i],
event.secondarySearchSuperviser,

3
source/net/yacy/peers/graphics/NetworkGraph.java
View File

@ -157,8 +157,7 @@ public class NetworkGraph {
*/
// draw in the search target
final QueryParams query = event.getQuery();
final Iterator<byte[]> i = query.query_include_hashes.iterator();
final Iterator<byte[]> i = event.query.query_include_hashes.iterator();
eventPicture.setColor(RasterPlotter.GREY);
while (i.hasNext()) {
byte[] wordHash = i.next();

5
source/net/yacy/search/Switchboard.java
View File

@ -2671,7 +2671,7 @@ public final class Switchboard extends serverSwitch {
}
// check if some of the links match with the query
final Map<MultiProtocolURI, String> matcher = searchEvent.getQuery().separateMatches(links);
final Map<MultiProtocolURI, String> matcher = searchEvent.query.separateMatches(links);
// take the matcher and load them all
for ( final Map.Entry<MultiProtocolURI, String> entry : matcher.entrySet() ) {
@ -3319,8 +3319,7 @@ public final class Switchboard extends serverSwitch {
new Thread() {
@Override
public void run() {
QueryParams query = searchEvent.getQuery();
String queryString = query.queryString(true);
String queryString = searchEvent.query.queryString(true);
Thread.currentThread().setName("Switchboard.heuristicRSS:" + queryString);
final int meta = queryString.indexOf("heuristic:", 0);
if ( meta >= 0 ) {

2
source/net/yacy/search/index/Segment.java
View File

@ -413,7 +413,7 @@ public class Segment {
// data during search-time. To transfer indexed data directly to the search process
// the following lines push the index data additionally to the search process
// this is done only for searched words
if (searchEvent != null && !searchEvent.getQuery().query_exclude_hashes.has(wordhash) && searchEvent.getQuery().query_include_hashes.has(wordhash)) {
if (searchEvent != null && !searchEvent.query.query_exclude_hashes.has(wordhash) && searchEvent.query.query_include_hashes.has(wordhash)) {
// if the page was added in the context of a heuristic this shall ensure that findings will fire directly into the search result
ReferenceContainer<WordReference> container;
try {

10
source/net/yacy/search/query/AccessTracker.java
View File

@ -44,9 +44,9 @@ public class AccessTracker {
private final static long DUMP_PERIOD = 60000L;
public static final int minSize = 100;
public static final int maxSize = 1000;
public static final int maxAge = 24 * 60 * 60 * 1000;
private static final int minSize = 100;
private static final int maxSize = 1000;
private static final int maxAge = 24 * 60 * 60 * 1000;
public enum Location {local, remote}
@ -67,7 +67,7 @@ public class AccessTracker {
private static void add(final LinkedList<QueryParams> list, final QueryParams query) {
// learn that this word can be a word completion for the DidYouMeanLibrary
if (query.resultcount > 10 && query.queryString != null && query.queryString.length() > 0) {
if (query.getResultCount() > 10 && query.queryString != null && query.queryString.length() > 0) {
final StringBuilder sb = new StringBuilder(query.queryString);
sb.append(query.queryString);
WordCache.learn(sb);
@ -109,7 +109,7 @@ public class AccessTracker {
private static void addToDump(final QueryParams query) {
if (query.queryString == null || query.queryString.isEmpty()) return;
addToDump(query.queryString, Integer.toString(query.resultcount), new Date(query.starttime));
addToDump(query.queryString, Integer.toString(query.getResultCount()), new Date(query.starttime));
}
public static void addToDump(String querystring, String resultcount) {

50
source/net/yacy/search/query/QueryParams.java
View File

@ -30,12 +30,15 @@ import java.io.UnsupportedEncodingException;
import java.net.URLEncoder;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.Map;
import java.util.Set;
import java.util.SortedSet;
import java.util.TreeSet;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.regex.Pattern;
import java.util.regex.PatternSyntaxException;
@ -58,6 +61,7 @@ import net.yacy.document.Condenser;
import net.yacy.document.parser.html.AbstractScraper;
import net.yacy.document.parser.html.CharacterCoding;
import net.yacy.kelondro.data.meta.DigestURI;
import net.yacy.kelondro.data.meta.URIMetadataRow;
import net.yacy.kelondro.data.word.Word;
import net.yacy.kelondro.data.word.WordReferenceRow;
import net.yacy.kelondro.index.RowHandleSet;
@ -103,18 +107,16 @@ public final class QueryParams {
public final String queryString;
public final HandleSet query_include_hashes, query_exclude_hashes, query_all_hashes;
private final Collection<String> query_include_words, query_exclude_words, query_all_words;
public final int itemsPerPage;
public int itemsPerPage;
public int offset;
public final Pattern urlMask, prefer;
final boolean urlMask_isCatchall;
private final boolean prefer_isMatchnothing;
public final Classification.ContentDomain contentdom;
public final String targetlang;
protected final Collection<Tagging.Metatag> metatags;
public final String navigators;
public final Searchdom domType;
private final int zonecode;
private final int domMaxTargets;
public final int maxDistance;
public final Bitfield constraint;
final boolean allofconstraint;
@ -131,14 +133,21 @@ public final class QueryParams {
protected final long maxtime;
protected final long timeout;
// values that are set after a search:
public int resultcount; // number of found results
public int transmitcount; // number of results that had been shown to the user
public long searchtime, urlretrievaltime, snippetcomputationtime; // time to perform the search, to get all the urls, and to compute the snippets
private boolean specialRights; // is true if the user has a special authorization and my use more database-extensive options
public final String userAgent;
public boolean filterfailurls;
protected boolean filterfailurls;
protected double lat, lon, radius;
// the following values are filled during the search process as statistics for the search
public final AtomicInteger local_rwi_available; // the number of hits generated/ranked by the local search in rwi index
public final AtomicInteger local_rwi_stored; // the number of existing hits by the local search in rwi index
public final AtomicInteger local_solr_available; // the number of hits generated/ranked by the local search in solr
public final AtomicInteger local_solr_stored; // the number of existing hits by the local search in solr
public final AtomicInteger remote_available; // the number of hits imported from remote peers (rwi/solr mixed)
public final AtomicInteger remote_stored; // the number of existing hits at remote site
public final AtomicInteger remote_peerCount; // the number of peers which contributed to the remote search result
public final SortedSet<byte[]> misses; // url hashes that had been sorted out because of constraints in postranking
public QueryParams(
final String queryString,
@ -178,7 +187,6 @@ public final class QueryParams {
this.urlMask = catchall_pattern;
this.urlMask_isCatchall = true;
this.prefer = matchnothing_pattern;
this.prefer_isMatchnothing = true;
this.contentdom = ContentDomain.ALL;
this.itemsPerPage = itemsPerPage;
this.offset = 0;
@ -186,7 +194,6 @@ public final class QueryParams {
this.metatags = new ArrayList<Tagging.Metatag>(0);
this.domType = Searchdom.LOCAL;
this.zonecode = DigestURI.TLD_any_zone_filter;
this.domMaxTargets = 0;
this.constraint = constraint;
this.allofconstraint = false;
this.snippetCacheStrategy = null;
@ -198,7 +205,6 @@ public final class QueryParams {
this.starttime = Long.valueOf(System.currentTimeMillis());
this.maxtime = 10000;
this.timeout = this.starttime + this.timeout;
this.specialRights = false;
this.navigators = "all";
this.indexSegment = indexSegment;
this.userAgent = userAgent;
@ -207,6 +213,14 @@ public final class QueryParams {
this.lat = 0.0d;
this.lon = 0.0d;
this.radius = 0.0d;
this.local_rwi_available = new AtomicInteger(0); // the number of results in the local peer after filtering
this.local_rwi_stored = new AtomicInteger(0);
this.local_solr_available= new AtomicInteger(0);
this.local_solr_stored = new AtomicInteger(0);
this.remote_stored = new AtomicInteger(0);
this.remote_available = new AtomicInteger(0); // the number of result contributions from all the remote peers
this.remote_peerCount = new AtomicInteger(0); // the number of remote peers that have contributed
this.misses = Collections.synchronizedSortedSet(new TreeSet<byte[]>(URIMetadataRow.rowdef.objectOrder));
}
public QueryParams(
@ -262,14 +276,13 @@ public final class QueryParams {
} catch (final PatternSyntaxException ex) {
throw new IllegalArgumentException("Not a valid regular expression: " + prefer, ex);
}
this.prefer_isMatchnothing = this.prefer.toString().equals(matchnothing_pattern.toString());
this.prefer.toString().equals(matchnothing_pattern.toString());
assert language != null;
this.targetlang = language;
this.metatags = metatags;
this.navigators = navigators;
this.domType = domType;
this.zonecode = domainzone;
this.domMaxTargets = domMaxTargets;
this.constraint = constraint;
this.allofconstraint = allofconstraint;
this.sitehash = site; assert site == null || site.length() == 6;
@ -281,7 +294,6 @@ public final class QueryParams {
this.starttime = Long.valueOf(System.currentTimeMillis());
this.maxtime = 10000;
this.timeout = this.starttime + this.timeout;
this.specialRights = specialRights;
this.indexSegment = indexSegment;
this.userAgent = userAgent;
this.transmitcount = 0;
@ -291,6 +303,14 @@ public final class QueryParams {
this.lat = Math.floor(lat * this.kmNormal) / this.kmNormal;
this.lon = Math.floor(lon * this.kmNormal) / this.kmNormal;
this.radius = Math.floor(radius * this.kmNormal + 1) / this.kmNormal;
this.local_rwi_available = new AtomicInteger(0); // the number of results in the local peer after filtering
this.local_rwi_stored = new AtomicInteger(0);
this.local_solr_available= new AtomicInteger(0);
this.local_solr_stored = new AtomicInteger(0);
this.remote_stored = new AtomicInteger(0);
this.remote_available = new AtomicInteger(0); // the number of result contributions from all the remote peers
this.remote_peerCount = new AtomicInteger(0); // the number of remote peers that have contributed
this.misses = Collections.synchronizedSortedSet(new TreeSet<byte[]>(URIMetadataRow.rowdef.objectOrder));
}
private double kmNormal = 100.d; // 100 =ca 40000.d / 360.d == 111.11 - if lat/lon is multiplied with this, rounded and diveded by this, the location is normalized to a 1km grid
@ -309,6 +329,10 @@ public final class QueryParams {
return this.itemsPerPage;
}
public int getResultCount() {
return this.local_rwi_available.get() + this.local_solr_stored.get() - this.misses.size();
}
public void setOffset(final int newOffset) {
this.offset = newOffset;
}
@ -502,7 +526,7 @@ public final class QueryParams {
final SolrQuery params = new SolrQuery();
params.setQuery(q.toString());
params.setStart(this.offset);
params.setRows(this.resultcount);
params.setRows(this.itemsPerPage);
params.setFacet(false);
if (this.radius > 0.0d && this.lat != 0.0d && this.lon != 0.0d) {

60
source/net/yacy/search/query/RankingProcess.java
View File

@ -84,23 +84,18 @@ public final class RankingProcess extends Thread {
protected final WeakPriorityBlockingQueue<WordReferenceVars> rwiStack;
protected final ConcurrentHashMap<String, WeakPriorityBlockingQueue<WordReferenceVars>> doubleDomCache; // key = domhash (6 bytes); value = like stack
private final int[] flagcount; // flag counter
private final HandleSet misses; // contains url-hashes that could not been found in the LURL-DB
private final AtomicInteger feedersAlive, feedersTerminated;
private boolean addRunning;
protected final AtomicInteger receivedRemoteReferences;
protected final ReferenceOrder order;
int local_indexCount;
int remote_indexCount;
int remote_resourceSize;
int remote_peerCount;
final HandleSet urlhashes; // map for double-check; String/Long relation, addresses ranking number (backreference for deletion)
final ScoreMap<String> hostNavigator; // a counter for the appearance of the host hash
final Map<String, byte[]> hostResolver; // a mapping from a host hash (6 bytes) to the full url hash of one of these urls that have the host hash
final Map<String, String> taggingPredicates; // a map from tagging vocabulary names to tagging predicate uris
final Map<String, ScoreMap<String>> vocabularyNavigator; // counters for Vocabularies; key is metatag.getVocabularyName()
protected final HandleSet urlhashes; // map for double-check; String/Long relation, addresses ranking number (backreference for deletion)
protected final ScoreMap<String> hostNavigator; // a counter for the appearance of the host hash
protected final Map<String, byte[]> hostResolver; // a mapping from a host hash (6 bytes) to the full url hash of one of these urls that have the host hash
protected final Map<String, String> taggingPredicates; // a map from tagging vocabulary names to tagging predicate uris
protected final Map<String, ScoreMap<String>> vocabularyNavigator; // counters for Vocabularies; key is metatag.getVocabularyName()
private boolean remote;
public RankingProcess(final QueryParams query, boolean remote) {
protected RankingProcess(final QueryParams query, boolean remote) {
// we collect the urlhashes and construct a list with urlEntry objects
// attention: if minEntries is too high, this method will not terminate within the maxTime
// sortorder: 0 = hash, 1 = url, 2 = ranking
@ -112,7 +107,6 @@ public final class RankingProcess extends Thread {
int stackMaxsize = query.snippetCacheStrategy == null || query.snippetCacheStrategy == CacheStrategy.CACHEONLY ? max_results_preparation_special : max_results_preparation;
this.rwiStack = new WeakPriorityBlockingQueue<WordReferenceVars>(stackMaxsize, false);
this.doubleDomCache = new ConcurrentHashMap<String, WeakPriorityBlockingQueue<WordReferenceVars>>();
this.misses = new RowHandleSet(URIMetadataRow.rowdef.primaryKeyLength, URIMetadataRow.rowdef.objectOrder, 100);
this.flagcount = new int[32];
for ( int i = 0; i < 32; i++ ) {
this.flagcount[i] = 0;
@ -122,10 +116,6 @@ public final class RankingProcess extends Thread {
this.addRunning = true;
this.receivedRemoteReferences = new AtomicInteger(0);
this.order = new ReferenceOrder(this.query.ranking, UTF8.getBytes(this.query.targetlang));
this.remote_resourceSize = 0;
this.remote_indexCount = 0;
this.remote_peerCount = 0;
this.local_indexCount = 0;
this.urlhashes = new RowHandleSet(URIMetadataRow.rowdef.primaryKeyLength, URIMetadataRow.rowdef.objectOrder, 100);
this.hostNavigator = new ConcurrentScoreMap<String>();
this.hostResolver = new ConcurrentHashMap<String, byte[]>();
@ -160,11 +150,6 @@ public final class RankingProcess extends Thread {
this.feedersAlive.addAndGet(1);
}
public int rwiAvailableCount() {
// the number of results in the local peer after filtering
return this.rwiStack.sizeAvailable();
}
public QueryParams getQuery() {
return this.query;
}
@ -173,14 +158,6 @@ public final class RankingProcess extends Thread {
return this.flagcount;
}
public Iterator<byte[]> miss() {
return this.misses.iterator();
}
public int getMissCount() {
return this.misses.size();
}
protected void addBegin() {
this.addRunning = true;
}
@ -230,21 +207,6 @@ public final class RankingProcess extends Thread {
return false;
}
public int getRemoteIndexCount() {
// the number of result contributions from all the remote peers
return this.remote_indexCount;
}
public int getRemoteResourceSize() {
// the number of all hits in all the remote peers
return Math.max(this.remote_resourceSize, this.remote_indexCount);
}
public int getRemotePeerCount() {
// the number of remote peers that have contributed
return this.remote_peerCount;
}
@Override
public void run() {
@ -300,10 +262,12 @@ public final class RankingProcess extends Thread {
this.addRunning = true;
assert (index != null);
if (index.isEmpty()) return;
if (!local) {
if (local) {
this.query.local_rwi_stored.addAndGet(fullResource);
} else {
assert fullResource >= 0 : "fullResource = " + fullResource;
this.remote_resourceSize += fullResource;
this.remote_peerCount++;
this.query.remote_stored.addAndGet(fullResource);
this.query.remote_peerCount.incrementAndGet();
}
long timer = System.currentTimeMillis();
@ -433,7 +397,7 @@ public final class RankingProcess extends Thread {
}
}
// increase counter for statistics
if (local) this.local_indexCount++; else this.remote_indexCount++;
if (local) this.query.local_rwi_available.incrementAndGet(); else this.query.remote_available.incrementAndGet();
}
if (System.currentTimeMillis() >= timeout) Log.logWarning("RWIProcess", "rwi normalization ended with timeout = " + maxtime);

55
source/net/yacy/search/query/SearchEvent.java
View File

@ -90,7 +90,7 @@ public final class SearchEvent {
private final static int SNIPPET_WORKER_THREADS = Math.max(4, Runtime.getRuntime().availableProcessors() * 2);
private long eventTime;
protected QueryParams query;
public QueryParams query;
public final SeedDB peers;
final WorkTables workTables;
public final RankingProcess rankingProcess; // ordered search results, grows dynamically as all the query threads enrich this container
@ -104,8 +104,7 @@ public final class SearchEvent {
private final SortedMap<byte[], HeuristicResult> heuristics;
private byte[] IAmaxcounthash, IAneardhthash;
private final Thread localsearch;
private final AtomicInteger expectedRemoteReferences, maxExpectedRemoteReferences;
private int sortout; // counter for referenced that had been sorted out for other reasons
private final AtomicInteger expectedRemoteReferences, maxExpectedRemoteReferences; // counter for referenced that had been sorted out for other reasons
private final ScoreMap<String> authorNavigator; // a counter for the appearances of authors
private final ScoreMap<String> namespaceNavigator; // a counter for name spaces
private final ScoreMap<String> protocolNavigator; // a counter for protocol types
@ -120,7 +119,6 @@ public final class SearchEvent {
protected long snippetComputationAllTime;
private final boolean remote;
private boolean cleanupState;
private int resultCounter = 0;
protected SearchEvent(
final QueryParams query,
@ -144,7 +142,6 @@ public final class SearchEvent {
this.maxExpectedRemoteReferences = new AtomicInteger(0);
this.expectedRemoteReferences = new AtomicInteger(0);
this.sortout = 0;
this.authorNavigator = new ConcurrentScoreMap<String>();
this.namespaceNavigator = new ConcurrentScoreMap<String>();
this.protocolNavigator = new ConcurrentScoreMap<String>();
@ -306,7 +303,7 @@ public final class SearchEvent {
}
SearchEventCache.put(this.query.id(false), this);
}
public long getEventTime() {
return this.eventTime;
}
@ -315,14 +312,6 @@ public final class SearchEvent {
this.eventTime = System.currentTimeMillis();
}
public QueryParams getQuery() {
return this.query;
}
public void setQuery(final QueryParams query) {
this.query = query;
}
protected void cleanup() {
this.cleanupState = true;
@ -486,10 +475,12 @@ public final class SearchEvent {
assert (index != null);
if (index.isEmpty()) return;
if (!local) {
if (local) {
this.query.local_solr_stored.set(fullResource);
} else {
assert fullResource >= 0 : "fullResource = " + fullResource;
this.rankingProcess.remote_resourceSize += fullResource;
this.rankingProcess.remote_peerCount++;
this.query.remote_stored.addAndGet(fullResource);
this.query.remote_peerCount.incrementAndGet();
}
long timer = System.currentTimeMillis();
@ -604,7 +595,7 @@ public final class SearchEvent {
}
}
// increase counter for statistics
if (local) this.rankingProcess.local_indexCount++; else this.rankingProcess.remote_indexCount++;
if (local) this.query.local_solr_available.incrementAndGet(); else this.query.remote_available.incrementAndGet();
}
} catch ( final SpaceExceededException e ) {
}
@ -626,10 +617,6 @@ public final class SearchEvent {
this.expectedRemoteReferences.addAndGet(x);
}
public int getSortOutCount() {
return this.sortout;
}
private URIMetadataNode takeRWI(final boolean skipDoubleDom, final long waitingtime) {
// returns from the current RWI list the best entry and removes this entry from the list
@ -771,27 +758,27 @@ public final class SearchEvent {
if ( !this.query.urlMask_isCatchall ) {
// check url mask
if ( !page.matches(this.query.urlMask) ) {
this.sortout++;
this.query.misses.add(page.hash());
continue;
}
}
// check for more errors
if ( page.url() == null ) {
this.sortout++;
this.query.misses.add(page.hash());
continue; // rare case where the url is corrupted
}
// check content domain
if ((this.query.contentdom.getCode() > 0 && page.url().getContentDomain() != this.query.contentdom) ||
(this.query.contentdom == Classification.ContentDomain.TEXT && page.url().getContentDomain().getCode() > 0)) {
this.sortout++;
this.query.misses.add(page.hash());
continue;
}
// Check for blacklist
if (Switchboard.urlBlacklist.isListed(BlacklistType.SEARCH, page)) {
this.sortout++;
this.query.misses.add(page.hash());
continue;
}
@ -805,7 +792,7 @@ public final class SearchEvent {
FilterEngine f = ContentControlFilterUpdateThread.getNetworkFilter();
if (f != null) {
if (!f.isListed(page.url(), null)) {
this.sortout++;
this.query.misses.add(page.hash());
continue;
}
}
@ -821,7 +808,7 @@ public final class SearchEvent {
((QueryParams.anymatch(pagetitle, this.query.query_exclude_hashes))
|| (QueryParams.anymatch(pageurl.toLowerCase(), this.query.query_exclude_hashes))
|| (QueryParams.anymatch(pageauthor.toLowerCase(), this.query.query_exclude_hashes)))) {
this.sortout++;
this.query.misses.add(page.hash());
continue;
}
@ -831,13 +818,13 @@ public final class SearchEvent {
while ( wi.hasNext() ) {
this.query.getSegment().termIndex().removeDelayed(wi.next(), page.hash());
}
this.sortout++;
this.query.misses.add(page.hash());
continue;
}
// check location constraint
if ((this.query.constraint != null) && (this.query.constraint.get(Condenser.flag_cat_haslocation)) && (page.lat() == 0.0f || page.lon() == 0.0f)) {
this.sortout++;
this.query.misses.add(page.hash());
continue;
}
@ -848,7 +835,7 @@ public final class SearchEvent {
double lonDelta = this.query.lon - lon;
double distance = Math.sqrt(latDelta * latDelta + lonDelta * lonDelta); // pythagoras
if (distance > this.query.radius) {
this.sortout++;
this.query.misses.add(page.hash());
continue;
}
}
@ -861,20 +848,20 @@ public final class SearchEvent {
// check if we already are filtering for authors
if ( this.query.authorhash != null && !this.query.authorhash.equals(authorhash) ) {
this.sortout++;
this.query.misses.add(page.hash());
continue;
}
// add author to the author navigator
this.authorNavigator.inc(pageauthor);
} else if ( this.query.authorhash != null ) {
this.sortout++;
this.query.misses.add(page.hash());
continue;
}
// check Scanner
if ( !Scanner.acceptURL(page.url()) ) {
this.sortout++;
this.query.misses.add(page.hash());
continue;
}

5
source/net/yacy/search/query/SearchEventCache.java
View File

@ -138,8 +138,9 @@ public class SearchEventCache {
if (event != null) {
//re-new the event time for this event, so it is not deleted next time too early
event.resetEventTime();
// replace the query, because this contains the current result offset
event.setQuery(query);
// replace the current result offset
event.query.offset = query.offset;
event.query.itemsPerPage = query.itemsPerPage;
}
}
if (event == null) {