diff --git a/htroot/AccessTracker_p.java b/htroot/AccessTracker_p.java index c9a3b8224..c17ad357f 100644 --- a/htroot/AccessTracker_p.java +++ b/htroot/AccessTracker_p.java @@ -173,18 +173,18 @@ public class AccessTracker_p { } prop.putNum("page_list_" + m + "_querycount", query.itemsPerPage); prop.putNum("page_list_" + m + "_transmitcount", query.transmitcount); - prop.putNum("page_list_" + m + "_resultcount", query.resultcount); + prop.putNum("page_list_" + m + "_resultcount", query.getResultCount()); prop.putNum("page_list_" + m + "_urltime", query.urlretrievaltime); prop.putNum("page_list_" + m + "_snippettime", query.snippetcomputationtime); prop.putNum("page_list_" + m + "_resulttime", query.searchtime); prop.putHTML("page_list_" + m + "_userAgent", query.userAgent); qcountSum += query.itemsPerPage; - rcountSum += query.resultcount; + rcountSum += query.getResultCount(); tcountSum += query.transmitcount; utimeSum += query.urlretrievaltime; stimeSum += query.snippetcomputationtime; rtimeSum += query.searchtime; - if (query.resultcount > 0){ + if (query.getResultCount() > 0){ rcount++; utimeSum1 += query.urlretrievaltime; stimeSum1 += query.snippetcomputationtime; diff --git a/htroot/HostBrowser.java b/htroot/HostBrowser.java index 1dd6ef4f5..18d3e68f9 100644 --- a/htroot/HostBrowser.java +++ b/htroot/HostBrowser.java @@ -189,9 +189,12 @@ public class HostBrowser { prop.putHTML("outbound_host", host); prop.putHTML("inbound_host", host); String hosthash = ASCII.String(uri.hash(), 6, 6); + String[] pathparts = uri.getPaths(); // get all files for a specific host from the index - BlockingQueue docs = fulltext.getSolr().concurrentQuery(YaCySchema.host_s.name() + ":" + host, 0, 100000, 3000, 100); + String query = YaCySchema.host_s.name() + ":" + host; + for (String pe: pathparts) if (pe.length() > 0) query += " AND " + YaCySchema.url_paths_sxt.name() + ":" + pe; + BlockingQueue docs = fulltext.getSolr().concurrentQuery(query, 0, 100000, 3000, 100); SolrDocument doc; Set storedDocs = new HashSet(); Set inboundLinks = new HashSet(); diff --git a/htroot/IndexControlRWIs_p.java b/htroot/IndexControlRWIs_p.java index 295e6353d..fe13c6e47 100644 --- a/htroot/IndexControlRWIs_p.java +++ b/htroot/IndexControlRWIs_p.java @@ -126,7 +126,7 @@ public class IndexControlRWIs_p { if ( post.containsKey("keystringsearch") ) { prop.put("keyhash", keyhash); final SearchEvent theSearch = genSearchresult(prop, sb, segment, keyhash, null); - if ( theSearch.rankingProcess.rwiAvailableCount() == 0 ) { + if (theSearch.query.local_rwi_available.get() == 0) { prop.put("searchresult", 1); prop.putHTML("searchresult_word", keystring); } @@ -137,7 +137,7 @@ public class IndexControlRWIs_p { prop.put("keystring", "<" + errmsg + ">"); } final SearchEvent theSearch = genSearchresult(prop, sb, segment, keyhash, null); - if ( theSearch.rankingProcess.rwiAvailableCount() == 0 ) { + if (theSearch.query.local_rwi_available.get() == 0) { prop.put("searchresult", 2); prop.putHTML("searchresult_wordhash", ASCII.String(keyhash)); } @@ -466,7 +466,7 @@ public class IndexControlRWIs_p { final String keyhashs = ASCII.String(keyhash); prop.put("genUrlList_keyHash", keyhashs); - if ( theSearch.rankingProcess.rwiAvailableCount() == 0 ) { + if (theSearch.query.local_rwi_stored.get() == 0) { prop.put("genUrlList", 1); prop.put("genUrlList_count", 0); prop.put("searchresult", 2); @@ -533,39 +533,17 @@ public class IndexControlRWIs_p { prop .put( "genUrlList_urlList_" + i + "_urlExists_props", - ((entry.word().flags().get(Condenser.flag_cat_indexof)) - ? "appears on index page, " - : "") - + ((entry.word().flags().get(Condenser.flag_cat_hasimage)) - ? "contains images, " - : "") - + ((entry.word().flags().get(Condenser.flag_cat_hasaudio)) - ? "contains audio, " - : "") - + ((entry.word().flags().get(Condenser.flag_cat_hasvideo)) - ? "contains video, " - : "") - + ((entry.word().flags().get(Condenser.flag_cat_hasapp)) - ? "contains applications, " - : "") - + ((entry.word().flags().get(WordReferenceRow.flag_app_dc_identifier)) - ? "appears in url, " - : "") - + ((entry.word().flags().get(WordReferenceRow.flag_app_dc_title)) - ? "appears in title, " - : "") - + ((entry.word().flags().get(WordReferenceRow.flag_app_dc_creator)) - ? "appears in author, " - : "") - + ((entry.word().flags().get(WordReferenceRow.flag_app_dc_subject)) - ? "appears in subject, " - : "") - + ((entry.word().flags().get(WordReferenceRow.flag_app_dc_description)) - ? "appears in description, " - : "") - + ((entry.word().flags().get(WordReferenceRow.flag_app_emphasized)) - ? "appears emphasized, " - : "") + ((entry.word().flags().get(Condenser.flag_cat_indexof)) ? "appears on index page, " : "") + + ((entry.word().flags().get(Condenser.flag_cat_hasimage)) ? "contains images, " : "") + + ((entry.word().flags().get(Condenser.flag_cat_hasaudio)) ? "contains audio, " : "") + + ((entry.word().flags().get(Condenser.flag_cat_hasvideo)) ? "contains video, " : "") + + ((entry.word().flags().get(Condenser.flag_cat_hasapp)) ? "contains applications, " : "") + + ((entry.word().flags().get(WordReferenceRow.flag_app_dc_identifier)) ? "appears in url, " : "") + + ((entry.word().flags().get(WordReferenceRow.flag_app_dc_title)) ? "appears in title, " : "") + + ((entry.word().flags().get(WordReferenceRow.flag_app_dc_creator)) ? "appears in author, " : "") + + ((entry.word().flags().get(WordReferenceRow.flag_app_dc_subject)) ? "appears in subject, " : "") + + ((entry.word().flags().get(WordReferenceRow.flag_app_dc_description)) ? "appears in description, " : "") + + ((entry.word().flags().get(WordReferenceRow.flag_app_emphasized)) ? "appears emphasized, " : "") + ((DigestURI.probablyRootURL(entry.word().urlhash())) ? "probably root url" : "")); if ( Switchboard.urlBlacklist.isListed(BlacklistType.DHT, url) ) { prop.put("genUrlList_urlList_" + i + "_urlExists_urlhxChecked", "1"); @@ -575,7 +553,7 @@ public class IndexControlRWIs_p { break; } } - final Iterator iter = theSearch.rankingProcess.miss(); // iterates url hash strings + final Iterator iter = theSearch.query.misses.iterator(); // iterates url hash strings byte[] b; while ( iter.hasNext() ) { b = iter.next(); @@ -677,12 +655,12 @@ public class IndexControlRWIs_p { final SearchEvent theSearch = SearchEventCache.getEvent(query, sb.peers, sb.tables, null, false, sb.loader, Integer.MAX_VALUE, Long.MAX_VALUE, (int) sb.getConfigLong(SwitchboardConstants.DHT_BURST_ROBINSON, 0), (int) sb.getConfigLong(SwitchboardConstants.DHT_BURST_MULTIWORD, 0)); //theSearch.rankingProcess.run(); RankingProcess ranked = theSearch.rankingProcess; - if ( ranked.rwiAvailableCount() == 0 ) { + if (theSearch.query.local_rwi_available.get() == 0) { prop.put("searchresult", 2); prop.put("searchresult_wordhash", keyhash); } else { prop.put("searchresult", 3); - prop.put("searchresult_allurl", ranked.rwiAvailableCount()); + prop.put("searchresult_allurl", theSearch.query.local_rwi_available.get()); prop .put("searchresult_description", ranked.flagCount()[WordReferenceRow.flag_app_dc_description]); prop.put("searchresult_title", ranked.flagCount()[WordReferenceRow.flag_app_dc_title]); diff --git a/htroot/yacy/search.java b/htroot/yacy/search.java index 4fff73976..3e668383a 100644 --- a/htroot/yacy/search.java +++ b/htroot/yacy/search.java @@ -213,7 +213,6 @@ public final class search { // prepare an abstract result final StringBuilder indexabstract = new StringBuilder(6000); int indexabstractContainercount = 0; - int joincount = 0; QueryParams theQuery = null; SearchEvent theSearch = null; ArrayList> accu = null; @@ -324,12 +323,11 @@ public final class search { theSearch = SearchEventCache.getEvent(theQuery, sb.peers, sb.tables, null, abstracts.length() > 0, sb.loader, count, maxtime, (int) sb.getConfigLong(SwitchboardConstants.DHT_BURST_ROBINSON, 0), (int) sb.getConfigLong(SwitchboardConstants.DHT_BURST_MULTIWORD, 0)); // set statistic details of search result and find best result index set - joincount = theSearch.rankingProcess.rwiAvailableCount() - theSearch.rankingProcess.getMissCount() - theSearch.getSortOutCount(); - prop.put("joincount", Integer.toString(joincount)); - if (joincount != 0) { + prop.put("joincount", Integer.toString(theQuery.getResultCount())); + if (theQuery.getResultCount() > 0) { accu = theSearch.completeResults(maxtime); } - if (joincount <= 0 || abstracts.isEmpty()) { + if (theQuery.getResultCount() <= 0 || abstracts.isEmpty()) { prop.put("indexcount", ""); } else { // attach information about index abstracts @@ -391,7 +389,7 @@ public final class search { prop.put("indexabstract", indexabstract.toString()); // prepare result - if (joincount == 0 || accu == null || accu.isEmpty()) { + if (theQuery.getResultCount() == 0 || accu == null || accu.isEmpty()) { // no results prop.put("links", ""); @@ -419,7 +417,6 @@ public final class search { // prepare search statistics theQuery.remotepeer = client == null ? null : sb.peers.lookupByIP(Domains.dnsResolve(client), -1, true, false, false); - theQuery.resultcount = (theSearch == null) ? 0 : joincount; theQuery.searchtime = System.currentTimeMillis() - timestamp; theQuery.urlretrievaltime = (theSearch == null) ? 0 : theSearch.getURLRetrievalTime(); theQuery.snippetcomputationtime = (theSearch == null) ? 0 : theSearch.getSnippetComputationTime(); @@ -436,7 +433,7 @@ public final class search { // log Network.log.logInfo("EXIT HASH SEARCH: " + - QueryParams.anonymizedQueryHashes(theQuery.query_include_hashes) + " - " + joincount + " links found, " + + QueryParams.anonymizedQueryHashes(theQuery.query_include_hashes) + " - " + theQuery.getResultCount() + " links found, " + prop.get("linkcount", "?") + " links selected, " + indexabstractContainercount + " index abstracts, " + (System.currentTimeMillis() - timestamp) + " milliseconds"); diff --git a/htroot/yacysearch.java b/htroot/yacysearch.java index f574bd212..96d308449 100644 --- a/htroot/yacysearch.java +++ b/htroot/yacysearch.java @@ -831,27 +831,18 @@ public class yacysearch { Log.logInfo("LOCAL_SEARCH", "EXIT WORD SEARCH: " + theQuery.queryString + " - " - + "local-unfiltered(" - + theSearch.rankingProcess.rwiAvailableCount() - + "), " - + "local_miss(" - + theSearch.rankingProcess.getMissCount() - + "), " - + "local_sortout(" - + theSearch.getSortOutCount() - + "), " - + "remote(" - + theSearch.rankingProcess.getRemoteResourceSize() - + ") links found, " + + "local_rwi_available(" + theSearch.query.local_rwi_available.get() + "), " + + "local_rwi_stored(" + theSearch.query.local_rwi_stored.get() + "), " + + "local_solr_available(" + theSearch.query.local_solr_available.get() + "), " + + "local_solr_stored(" + theSearch.query.local_solr_stored.get() + "), " + + "remote_available(" + theSearch.query.remote_available.get() + "), " + + "remote_stored(" + theSearch.query.remote_stored.get() + "), " + + "remote_peerCount(" + theSearch.query.remote_peerCount.get() + "), " + + "local_sortout(" + theSearch.query.misses.size() + "), " + (System.currentTimeMillis() - timestamp) + " ms"); // prepare search statistics - theQuery.resultcount = - theSearch.rankingProcess.rwiAvailableCount() - - theSearch.getSortOutCount() - - theSearch.rankingProcess.getMissCount() - + theSearch.rankingProcess.getRemoteIndexCount(); theQuery.searchtime = System.currentTimeMillis() - timestamp; theQuery.urlretrievaltime = theSearch.getURLRetrievalTime(); theQuery.snippetcomputationtime = theSearch.getSnippetComputationTime(); @@ -932,37 +923,16 @@ public class yacysearch { Log.logException(e); } - final int indexcount = - theSearch.rankingProcess.rwiAvailableCount() - - theSearch.getSortOutCount() - - theSearch.rankingProcess.getMissCount() - + theSearch.rankingProcess.getRemoteIndexCount(); prop.put("num-results_offset", startRecord == 0 ? 0 : startRecord + 1); - prop.put("num-results_itemscount", Formatter.number( - startRecord + theSearch.getQuery().itemsPerPage > indexcount ? startRecord - + indexcount - % theSearch.getQuery().itemsPerPage : startRecord + theSearch.getQuery().itemsPerPage, - true)); - prop.put("num-results_itemsPerPage", itemsPerPage); - prop.put("num-results_totalcount", indexcount); - prop.put("num-results_globalresults", global && (indexReceiveGranted || clustersearch) - ? "1" - : "0"); - prop.put( - "num-results_globalresults_localResourceSize", - Formatter.number(theSearch.rankingProcess.rwiAvailableCount(), true)); - prop.put( - "num-results_globalresults_localMissCount", - Formatter.number(theSearch.rankingProcess.getMissCount(), true)); - prop.put( - "num-results_globalresults_remoteResourceSize", - Formatter.number(theSearch.rankingProcess.getRemoteResourceSize(), true)); - prop.put( - "num-results_globalresults_remoteIndexCount", - Formatter.number(theSearch.rankingProcess.getRemoteIndexCount(), true)); - prop.put( - "num-results_globalresults_remotePeerCount", - Formatter.number(theSearch.rankingProcess.getRemotePeerCount(), true)); + prop.put("num-results_itemscount", Formatter.number(startRecord + theSearch.query.itemsPerPage > theSearch.query.getResultCount() ? startRecord + theSearch.query.getResultCount() % theSearch.query.itemsPerPage : startRecord + theSearch.query.itemsPerPage, true)); + prop.put("num-results_itemsPerPage", Formatter.number(itemsPerPage)); + prop.put("num-results_totalcount", Formatter.number(theSearch.query.getResultCount())); + prop.put("num-results_globalresults", global && (indexReceiveGranted || clustersearch) ? "1" : "0"); + prop.put("num-results_globalresults_localResourceSize", Formatter.number(theSearch.query.local_rwi_available.get() + theSearch.query.local_solr_available.get(), true)); + prop.put("num-results_globalresults_localMissCount", Formatter.number(theSearch.query.misses.size(), true)); + prop.put("num-results_globalresults_remoteResourceSize", Formatter.number(theSearch.query.remote_available.get(), true)); + prop.put("num-results_globalresults_remoteIndexCount", Formatter.number(theSearch.query.remote_stored.get(), true)); + prop.put("num-results_globalresults_remotePeerCount", Formatter.number(theSearch.query.remote_peerCount.get(), true)); // compose page navigation final StringBuilder resnav = new StringBuilder(200); @@ -982,7 +952,7 @@ public class yacysearch { resnav .append("\">\"arrowleft\" "); } - final int numberofpages = Math.min(10, 1 + ((indexcount - 1) / theQuery.itemsPerPage())); + final int numberofpages = Math.min(10, 1 + ((theSearch.query.getResultCount() - 1) / theQuery.itemsPerPage())); for ( int i = 0; i < numberofpages; i++ ) { if ( i == thispage ) { @@ -1020,7 +990,7 @@ public class yacysearch { } final String resnavs = resnav.toString(); prop.put("num-results_resnav", resnavs); - prop.put("pageNavBottom", (indexcount - startRecord > 6) ? 1 : 0); // if there are more results than may fit on the page we add a navigation at the bottom + prop.put("pageNavBottom", (theSearch.query.getResultCount() - startRecord > 6) ? 1 : 0); // if there are more results than may fit on the page we add a navigation at the bottom prop.put("pageNavBottom_resnav", resnavs); // generate the search result lines; the content will be produced by another servlet diff --git a/htroot/yacysearchitem.java b/htroot/yacysearchitem.java index 8df838dc5..827469f04 100644 --- a/htroot/yacysearchitem.java +++ b/htroot/yacysearchitem.java @@ -89,28 +89,25 @@ public class yacysearchitem { // the event does not exist, show empty page return prop; } - final QueryParams theQuery = theSearch.getQuery(); // dynamically update count values - final int totalcount = theSearch.rankingProcess.rwiAvailableCount() - theSearch.rankingProcess.getMissCount() - theSearch.getSortOutCount() + theSearch.rankingProcess.getRemoteIndexCount(); - final int offset = theQuery.neededResults() - theQuery.itemsPerPage() + 1; - prop.put("offset", offset); - prop.put("itemscount", Formatter.number(Math.min((item < 0) ? theQuery.neededResults() : item + 1, totalcount))); - prop.put("itemsperpage", Formatter.number(theQuery.itemsPerPage)); - prop.put("totalcount", Formatter.number(totalcount, true)); - prop.put("localResourceSize", Formatter.number(theSearch.rankingProcess.rwiAvailableCount(), true)); - prop.put("localMissCount", Formatter.number(theSearch.rankingProcess.getMissCount(), true)); - prop.put("remoteResourceSize", Formatter.number(theSearch.rankingProcess.getRemoteResourceSize(), true)); - prop.put("remoteIndexCount", Formatter.number(theSearch.rankingProcess.getRemoteIndexCount(), true)); - prop.put("remotePeerCount", Formatter.number(theSearch.rankingProcess.getRemotePeerCount(), true)); - prop.put("navurlBase", QueryParams.navurlBase("html", theQuery, null, theQuery.urlMask.toString(), theQuery.navigators).toString()); + prop.put("offset", theSearch.query.neededResults() - theSearch.query.itemsPerPage() + 1); + prop.put("itemscount", Formatter.number(Math.min((item < 0) ? theSearch.query.neededResults() : item + 1, theSearch.query.getResultCount()))); + prop.put("itemsperpage", Formatter.number(theSearch.query.itemsPerPage)); + prop.put("totalcount", Formatter.number(theSearch.query.getResultCount(), true)); + prop.put("localResourceSize", Formatter.number(theSearch.query.local_rwi_available.get() + theSearch.query.local_solr_available.get(), true)); + prop.put("localMissCount", Formatter.number(theSearch.query.misses.size(), true)); + prop.put("remoteResourceSize", Formatter.number(theSearch.query.remote_available.get(), true)); + prop.put("remoteIndexCount", Formatter.number(theSearch.query.remote_stored.get(), true)); + prop.put("remotePeerCount", Formatter.number(theSearch.query.remote_peerCount.get(), true)); + prop.put("navurlBase", QueryParams.navurlBase("html", theSearch.query, null, theSearch.query.urlMask.toString(), theSearch.query.navigators).toString()); final String target_special_pattern = sb.getConfig(SwitchboardConstants.SEARCH_TARGET_SPECIAL_PATTERN, ""); - if (theQuery.contentdom == Classification.ContentDomain.TEXT || theQuery.contentdom == Classification.ContentDomain.ALL) { + if (theSearch.query.contentdom == Classification.ContentDomain.TEXT || theSearch.query.contentdom == Classification.ContentDomain.ALL) { // text search // generate result object - final ResultEntry result = theSearch.oneResult(item, theQuery.isLocal() ? 1000 : 5000); + final ResultEntry result = theSearch.oneResult(item, theSearch.query.isLocal() ? 1000 : 5000); if (result == null) return prop; // no content final String resultUrlstring = result.urlstring(); final DigestURI resultURL = result.url(); @@ -124,7 +121,7 @@ public class yacysearchitem { Log.logException(e1); faviconURL = null; } - final String resource = theQuery.domType.toString(); + final String resource = theSearch.query.domType.toString(); prop.put("content", 1); // switch on specific content prop.put("content_showDate", sb.getConfigBool("search.result.show.date", true) ? 1 : 0); prop.put("content_showSize", sb.getConfigBool("search.result.show.size", true) ? 1 : 0); @@ -137,10 +134,10 @@ public class yacysearchitem { prop.put("content_authorized", authenticated ? "1" : "0"); final String urlhash = ASCII.String(result.hash()); prop.put("content_authorized_bookmark", sb.tables.bookmarks.hasBookmark("admin", urlhash) ? "0" : "1"); - prop.putHTML("content_authorized_bookmark_bookmarklink", "/yacysearch.html?query=" + theQuery.queryString.replace(' ', '+') + "&Enter=Search&count=" + theQuery.itemsPerPage() + "&offset=" + (theQuery.neededResults() - theQuery.itemsPerPage()) + "&order=" + crypt.simpleEncode(theQuery.ranking.toExternalString()) + "&resource=" + resource + "&time=3&bookmarkref=" + urlhash + "&urlmaskfilter=.*"); + prop.putHTML("content_authorized_bookmark_bookmarklink", "/yacysearch.html?query=" + theSearch.query.queryString.replace(' ', '+') + "&Enter=Search&count=" + theSearch.query.itemsPerPage() + "&offset=" + (theSearch.query.neededResults() - theSearch.query.itemsPerPage()) + "&order=" + crypt.simpleEncode(theSearch.query.ranking.toExternalString()) + "&resource=" + resource + "&time=3&bookmarkref=" + urlhash + "&urlmaskfilter=.*"); prop.put("content_authorized_recommend", (sb.peers.newsPool.getSpecific(NewsPool.OUTGOING_DB, NewsPool.CATEGORY_SURFTIPP_ADD, "url", resultUrlstring) == null) ? "1" : "0"); - prop.putHTML("content_authorized_recommend_deletelink", "/yacysearch.html?query=" + theQuery.queryString.replace(' ', '+') + "&Enter=Search&count=" + theQuery.itemsPerPage() + "&offset=" + (theQuery.neededResults() - theQuery.itemsPerPage()) + "&order=" + crypt.simpleEncode(theQuery.ranking.toExternalString()) + "&resource=" + resource + "&time=3&deleteref=" + urlhash + "&urlmaskfilter=.*"); - prop.putHTML("content_authorized_recommend_recommendlink", "/yacysearch.html?query=" + theQuery.queryString.replace(' ', '+') + "&Enter=Search&count=" + theQuery.itemsPerPage() + "&offset=" + (theQuery.neededResults() - theQuery.itemsPerPage()) + "&order=" + crypt.simpleEncode(theQuery.ranking.toExternalString()) + "&resource=" + resource + "&time=3&recommendref=" + urlhash + "&urlmaskfilter=.*"); + prop.putHTML("content_authorized_recommend_deletelink", "/yacysearch.html?query=" + theSearch.query.queryString.replace(' ', '+') + "&Enter=Search&count=" + theSearch.query.itemsPerPage() + "&offset=" + (theSearch.query.neededResults() - theSearch.query.itemsPerPage()) + "&order=" + crypt.simpleEncode(theSearch.query.ranking.toExternalString()) + "&resource=" + resource + "&time=3&deleteref=" + urlhash + "&urlmaskfilter=.*"); + prop.putHTML("content_authorized_recommend_recommendlink", "/yacysearch.html?query=" + theSearch.query.queryString.replace(' ', '+') + "&Enter=Search&count=" + theSearch.query.itemsPerPage() + "&offset=" + (theSearch.query.neededResults() - theSearch.query.itemsPerPage()) + "&order=" + crypt.simpleEncode(theSearch.query.ranking.toExternalString()) + "&resource=" + resource + "&time=3&recommendref=" + urlhash + "&urlmaskfilter=.*"); prop.put("content_authorized_urlhash", urlhash); final String resulthashString = urlhash; prop.putHTML("content_title", result.title()); @@ -205,11 +202,11 @@ public class yacysearchitem { prop.putHTML("content_host", resultURL.getHost() == null ? "" : resultURL.getHost()); prop.putHTML("content_file", resultURL.getFileName()); prop.putHTML("content_path", resultURL.getPath()); - prop.put("content_nl", (item == theQuery.offset) ? 0 : 1); + prop.put("content_nl", (item == theSearch.query.offset) ? 0 : 1); prop.putHTML("content_publisher", result.publisher()); prop.putHTML("content_creator", result.creator());// author prop.putHTML("content_subject", result.subject()); - final Collection[] query = theQuery.queryWords(); + final Collection[] query = theSearch.query.queryWords(); final StringBuilder s = new StringBuilder(query[0].size() * 20); for (final String t: query[0]) { s.append('+').append(t); @@ -217,10 +214,10 @@ public class yacysearchitem { final String words = (s.length() > 0) ? s.substring(1) : ""; prop.putHTML("content_words", words); prop.putHTML("content_showParser_words", words); - prop.putHTML("content_former", theQuery.queryString); - prop.putHTML("content_showPictures_former", theQuery.queryString); + prop.putHTML("content_former", theSearch.query.queryString); + prop.putHTML("content_showPictures_former", theSearch.query.queryString); final TextSnippet snippet = result.textSnippet(); - final String desc = (snippet == null) ? "" : snippet.getLineMarked(theQuery.query_all_hashes); + final String desc = (snippet == null) ? "" : snippet.getLineMarked(theSearch.query.query_all_hashes); prop.put("content_description", desc); prop.putXML("content_description-xml", desc); prop.putJSON("content_description-json", desc); @@ -235,7 +232,7 @@ public class yacysearchitem { } prop.put("content_heuristic_name", heuristic.heuristicName); } - EventTracker.update(EventTracker.EClass.SEARCH, new ProfilingGraph.EventSearch(theQuery.id(true), SearchEventType.FINALIZATION, "" + item, 0, 0), false); + EventTracker.update(EventTracker.EClass.SEARCH, new ProfilingGraph.EventSearch(theSearch.query.id(true), SearchEventType.FINALIZATION, "" + item, 0, 0), false); final String ext = resultURL.getFileExtension().toLowerCase(); if (ext.equals("png") || ext.equals("jpg") || ext.equals("gif")) { final String license = sb.licensedURLs.aquireLicense(resultURL); @@ -251,16 +248,16 @@ public class yacysearchitem { prop.put("content_loc_lon", result.lon()); } if (sb.getConfigBool("heuristic.searchresults",false)) sb.heuristicSearchResults(resultUrlstring); - theQuery.transmitcount = item + 1; + theSearch.query.transmitcount = item + 1; return prop; } - if (theQuery.contentdom == Classification.ContentDomain.IMAGE) { + if (theSearch.query.contentdom == Classification.ContentDomain.IMAGE) { // image search; shows thumbnails - prop.put("content", theQuery.contentdom.getCode() + 1); // switch on specific content + prop.put("content", theSearch.query.contentdom.getCode() + 1); // switch on specific content //final MediaSnippet ms = theSearch.result().oneImage(item); - final ResultEntry ms = theSearch.oneResult(item, theQuery.isLocal() ? 1000 : 5000); + final ResultEntry ms = theSearch.oneResult(item, theSearch.query.isLocal() ? 1000 : 5000); if (ms == null) { prop.put("content_item", "0"); } else { @@ -283,21 +280,21 @@ public class yacysearchitem { prop.put("content_item_source", ms.url().toNormalform(true)); prop.putXML("content_item_source-xml", ms.url().toNormalform(true)); prop.put("content_item_sourcedom", ms.url().getHost()); - prop.put("content_item_nl", (item == theQuery.offset) ? 0 : 1); + prop.put("content_item_nl", (item == theSearch.query.offset) ? 0 : 1); prop.put("content_item", 1); } - theQuery.transmitcount = item + 1; + theSearch.query.transmitcount = item + 1; return prop; } - if ((theQuery.contentdom == ContentDomain.AUDIO) || - (theQuery.contentdom == ContentDomain.VIDEO) || - (theQuery.contentdom == ContentDomain.APP)) { + if ((theSearch.query.contentdom == ContentDomain.AUDIO) || + (theSearch.query.contentdom == ContentDomain.VIDEO) || + (theSearch.query.contentdom == ContentDomain.APP)) { // any other media content // generate result object - final ResultEntry ms = theSearch.oneResult(item, theQuery.isLocal() ? 1000 : 5000); - prop.put("content", theQuery.contentdom.getCode() + 1); // switch on specific content + final ResultEntry ms = theSearch.oneResult(item, theSearch.query.isLocal() ? 1000 : 5000); + prop.put("content", theSearch.query.contentdom.getCode() + 1); // switch on specific content if (ms == null) { prop.put("content_item", "0"); } else { @@ -308,10 +305,10 @@ public class yacysearchitem { prop.putHTML("content_item_target", target); prop.putHTML("content_item_name", shorten(ms.title(), MAX_NAME_LENGTH)); prop.put("content_item_col", (item % 2 == 0) ? "0" : "1"); - prop.put("content_item_nl", (item == theQuery.offset) ? 0 : 1); + prop.put("content_item_nl", (item == theSearch.query.offset) ? 0 : 1); prop.put("content_item", 1); } - theQuery.transmitcount = item + 1; + theSearch.query.transmitcount = item + 1; return prop; } diff --git a/htroot/yacysearchlatestinfo.java b/htroot/yacysearchlatestinfo.java index 92d6a3a4d..1ac3e3d61 100644 --- a/htroot/yacysearchlatestinfo.java +++ b/htroot/yacysearchlatestinfo.java @@ -30,21 +30,19 @@ public class yacysearchlatestinfo { prop.put("remotePeerCount", 0); return prop; } - final QueryParams theQuery = theSearch.getQuery(); // dynamically update count values - final int totalcount = theSearch.rankingProcess.rwiAvailableCount() - theSearch.rankingProcess.getMissCount() - theSearch.getSortOutCount() + theSearch.rankingProcess.getRemoteIndexCount(); - final int offset = theQuery.neededResults() - theQuery.itemsPerPage() + 1; + final int offset = theSearch.query.neededResults() - theSearch.query.itemsPerPage() + 1; prop.put("offset", offset); - prop.put("itemscount",Formatter.number(offset + theSearch.getQuery().itemsPerPage >= totalcount ? offset + totalcount % theSearch.getQuery().itemsPerPage - 1 : offset + theSearch.getQuery().itemsPerPage - 1)); - prop.put("itemsperpage", theSearch.getQuery().itemsPerPage); - prop.put("totalcount", Formatter.number(totalcount, true)); - prop.put("localResourceSize", Formatter.number(theSearch.rankingProcess.rwiAvailableCount(), true)); - prop.put("localMissCount", Formatter.number(theSearch.rankingProcess.getMissCount(), true)); - prop.put("remoteResourceSize", Formatter.number(theSearch.rankingProcess.getRemoteResourceSize(), true)); - prop.put("remoteIndexCount", Formatter.number(theSearch.rankingProcess.getRemoteIndexCount(), true)); - prop.put("remotePeerCount", Formatter.number(theSearch.rankingProcess.getRemotePeerCount(), true)); - prop.putJSON("navurlBase", QueryParams.navurlBase("html", theQuery, null, theQuery.urlMask.toString(), theQuery.navigators).toString()); + prop.put("itemscount",Formatter.number(offset + theSearch.query.itemsPerPage >= theSearch.query.getResultCount() ? offset + theSearch.query.getResultCount() % theSearch.query.itemsPerPage - 1 : offset + theSearch.query.itemsPerPage - 1)); + prop.put("itemsperpage", theSearch.query.itemsPerPage); + prop.put("totalcount", Formatter.number(theSearch.query.getResultCount(), true)); + prop.put("localResourceSize", Formatter.number(theSearch.query.local_rwi_available.get() + theSearch.query.local_solr_available.get(), true)); + prop.put("localMissCount", Formatter.number(theSearch.query.misses.size(), true)); + prop.put("remoteResourceSize", Formatter.number(theSearch.query.remote_available.get(), true)); + prop.put("remoteIndexCount", Formatter.number(theSearch.query.remote_stored.get(), true)); + prop.put("remotePeerCount", Formatter.number(theSearch.query.remote_peerCount.get(), true)); + prop.putJSON("navurlBase", QueryParams.navurlBase("html", theSearch.query, null, theSearch.query.urlMask.toString(), theSearch.query.navigators).toString()); return prop; } diff --git a/htroot/yacysearchtrailer.java b/htroot/yacysearchtrailer.java index 25bfc5e7a..7044874f1 100644 --- a/htroot/yacysearchtrailer.java +++ b/htroot/yacysearchtrailer.java @@ -59,7 +59,6 @@ public class yacysearchtrailer { // the event does not exist, show empty page return prop; } - final QueryParams theQuery = theSearch.getQuery(); final RequestHeader.FileType fileType = header.fileType(); // compose search navigation @@ -80,7 +79,7 @@ public class yacysearchtrailer { name = navigatorIterator.next(); count = namespaceNavigator.get(name); nav = "inurl%3A" + name; - queryStringForUrl = theQuery.queryStringForUrl(); + queryStringForUrl = theSearch.query.queryStringForUrl(); p = queryStringForUrl.indexOf(nav); if (p < 0) { pos++; @@ -94,7 +93,7 @@ public class yacysearchtrailer { queryStringForUrl = (queryStringForUrl.substring(0, p) + queryStringForUrl.substring(p + nav.length())).trim(); } prop.put(fileType, "nav-namespace_element_" + i + "_name", name); - prop.put(fileType, "nav-namespace_element_" + i + "_url", QueryParams.navurl(fileType.name().toLowerCase(), 0, theQuery, queryStringForUrl, theQuery.urlMask.toString(), theQuery.navigators).toString()); + prop.put(fileType, "nav-namespace_element_" + i + "_url", QueryParams.navurl(fileType.name().toLowerCase(), 0, theSearch.query, queryStringForUrl, theSearch.query.urlMask.toString(), theSearch.query.navigators).toString()); prop.put("nav-namespace_element_" + i + "_count", count); prop.put("nav-namespace_element_" + i + "_nl", 1); i++; @@ -119,7 +118,7 @@ public class yacysearchtrailer { name = navigatorIterator.next(); count = hostNavigator.get(name); nav = "site%3A" + name; - queryStringForUrl = theQuery.queryStringForUrl(); + queryStringForUrl = theSearch.query.queryStringForUrl(); p = queryStringForUrl.indexOf(nav); if (p < 0) { pos++; @@ -133,7 +132,7 @@ public class yacysearchtrailer { prop.put(fileType, "nav-authors_element_" + i + "_modifier", "-" + nav); } prop.put(fileType, "nav-domains_element_" + i + "_name", name); - prop.put(fileType, "nav-domains_element_" + i + "_url", QueryParams.navurl(fileType.name().toLowerCase(), 0, theQuery, queryStringForUrl, theQuery.urlMask.toString(), theQuery.navigators).toString()); + prop.put(fileType, "nav-domains_element_" + i + "_url", QueryParams.navurl(fileType.name().toLowerCase(), 0, theSearch.query, queryStringForUrl, theSearch.query.urlMask.toString(), theSearch.query.navigators).toString()); prop.put("nav-domains_element_" + i + "_count", count); prop.put("nav-domains_element_" + i + "_nl", 1); i++; @@ -158,7 +157,7 @@ public class yacysearchtrailer { name = navigatorIterator.next().trim(); count = authorNavigator.get(name); nav = (name.indexOf(' ', 0) < 0) ? "author%3A" + name : "author%3A%28" + name.replace(" ", "+") + "%29"; - queryStringForUrl = theQuery.queryStringForUrl(); + queryStringForUrl = theSearch.query.queryStringForUrl(); p = queryStringForUrl.indexOf(nav); if (p < 0) { pos++; @@ -172,7 +171,7 @@ public class yacysearchtrailer { prop.put(fileType, "nav-authors_element_" + i + "_modifier", "-" + nav); } prop.put(fileType, "nav-authors_element_" + i + "_name", name); - prop.put(fileType, "nav-authors_element_" + i + "_url", QueryParams.navurl(fileType.name().toLowerCase(), 0, theQuery, queryStringForUrl, theQuery.urlMask.toString(), theQuery.navigators).toString()); + prop.put(fileType, "nav-authors_element_" + i + "_url", QueryParams.navurl(fileType.name().toLowerCase(), 0, theSearch.query, queryStringForUrl, theSearch.query.urlMask.toString(), theSearch.query.navigators).toString()); prop.put("nav-authors_element_" + i + "_count", count); prop.put("nav-authors_element_" + i + "_nl", 1); i++; @@ -196,13 +195,13 @@ public class yacysearchtrailer { while (i < MAX_TOPWORDS && navigatorIterator.hasNext()) { name = navigatorIterator.next(); count = topicNavigator.get(name); - if (theQuery.queryString == null) break; + if (theSearch.query.queryString == null) break; if (name != null) { - queryStringForUrl = theQuery.queryStringForUrl(); + queryStringForUrl = theSearch.query.queryStringForUrl(); prop.put("nav-topics_element_" + i + "_on", 1); prop.put(fileType, "nav-topics_element_" + i + "_modifier", name); prop.put(fileType, "nav-topics_element_" + i + "_name", name); - prop.put(fileType, "nav-topics_element_" + i + "_url", QueryParams.navurl(fileType.name().toLowerCase(), 0, theQuery, queryStringForUrl + "+" + name, theQuery.urlMask.toString(), theQuery.navigators).toString()); + prop.put(fileType, "nav-topics_element_" + i + "_url", QueryParams.navurl(fileType.name().toLowerCase(), 0, theSearch.query, queryStringForUrl + "+" + name, theSearch.query.urlMask.toString(), theSearch.query.navigators).toString()); prop.put("nav-topics_element_" + i + "_count", count); prop.put("nav-topics_element_" + i + "_nl", 1); i++; @@ -226,7 +225,7 @@ public class yacysearchtrailer { name = navigatorIterator.next().trim(); count = protocolNavigator.get(name); nav = "%2F" + name; - queryStringForUrl = theQuery.queryStringForUrl(); + queryStringForUrl = theSearch.query.queryStringForUrl(); p = queryStringForUrl.indexOf(nav); if (p < 0) { pos++; @@ -240,7 +239,7 @@ public class yacysearchtrailer { prop.put(fileType, "nav-protocols_element_" + i + "_modifier", "-" + nav); } prop.put(fileType, "nav-protocols_element_" + i + "_name", name); - prop.put(fileType, "nav-protocols_element_" + i + "_url", QueryParams.navurl(fileType.name().toLowerCase(), 0, theQuery, queryStringForUrl, (p >= 0 && theQuery.urlMask.toString().startsWith(name)) ? ".*" : theQuery.urlMask.toString(), theQuery.navigators).toString()); + prop.put(fileType, "nav-protocols_element_" + i + "_url", QueryParams.navurl(fileType.name().toLowerCase(), 0, theSearch.query, queryStringForUrl, (p >= 0 && theSearch.query.urlMask.toString().startsWith(name)) ? ".*" : theSearch.query.urlMask.toString(), theSearch.query.navigators).toString()); prop.put("nav-protocols_element_" + i + "_count", count); prop.put("nav-protocols_element_" + i + "_nl", 1); i++; @@ -265,7 +264,7 @@ public class yacysearchtrailer { name = navigatorIterator.next().trim(); count = filetypeNavigator.get(name); nav = "filetype%3A" + name; - queryStringForUrl = theQuery.queryStringForUrl(); + queryStringForUrl = theSearch.query.queryStringForUrl(); p = queryStringForUrl.indexOf(nav); if (p < 0) { pos++; @@ -279,7 +278,7 @@ public class yacysearchtrailer { prop.put(fileType, "nav-filetypes_element_" + i + "_modifier", "-" + nav); } prop.put(fileType, "nav-filetypes_element_" + i + "_name", name); - prop.put(fileType, "nav-filetypes_element_" + i + "_url", QueryParams.navurl(fileType.name().toLowerCase(), 0, theQuery, queryStringForUrl, (p >= 0 && theQuery.urlMask.toString().endsWith(name)) ? ".*" : theQuery.urlMask.toString(), theQuery.navigators).toString()); + prop.put(fileType, "nav-filetypes_element_" + i + "_url", QueryParams.navurl(fileType.name().toLowerCase(), 0, theSearch.query, queryStringForUrl, (p >= 0 && theSearch.query.urlMask.toString().endsWith(name)) ? ".*" : theSearch.query.urlMask.toString(), theSearch.query.navigators).toString()); prop.put("nav-filetypes_element_" + i + "_count", count); prop.put("nav-filetypes_element_" + i + "_nl", 1); i++; @@ -308,7 +307,7 @@ public class yacysearchtrailer { name = navigatorIterator.next(); count = ve.getValue().get(name); nav = "%2Fvocabulary%2F" + navname + "%2F" + MultiProtocolURI.escape(Tagging.encodePrintname(name)).toString(); - queryStringForUrl = theQuery.queryStringForUrl(); + queryStringForUrl = theSearch.query.queryStringForUrl(); p = queryStringForUrl.indexOf(nav); if (p < 0) { queryStringForUrl += "+" + nav; @@ -320,7 +319,7 @@ public class yacysearchtrailer { prop.put(fileType, "nav-vocabulary_" + navvoccount + "_element_" + i + "_modifier", "-" + nav); } prop.put(fileType, "nav-vocabulary_" + navvoccount + "_element_" + i + "_name", name); - prop.put(fileType, "nav-vocabulary_" + navvoccount + "_element_" + i + "_url", QueryParams.navurl(fileType.name().toLowerCase(), 0, theQuery, queryStringForUrl, theQuery.urlMask.toString(), theQuery.navigators).toString()); + prop.put(fileType, "nav-vocabulary_" + navvoccount + "_element_" + i + "_url", QueryParams.navurl(fileType.name().toLowerCase(), 0, theSearch.query, queryStringForUrl, theSearch.query.urlMask.toString(), theSearch.query.navigators).toString()); prop.put("nav-vocabulary_" + navvoccount + "_element_" + i + "_count", count); prop.put("nav-vocabulary_" + navvoccount + "_element_" + i + "_nl", 1); i++; @@ -338,8 +337,7 @@ public class yacysearchtrailer { // about box final String aboutBody = env.getConfig("about.body", ""); final String aboutHeadline = env.getConfig("about.headline", ""); - if ((aboutBody.isEmpty() && aboutHeadline.isEmpty()) || - theSearch.rankingProcess.rwiAvailableCount() - theSearch.rankingProcess.getMissCount() - theSearch.getSortOutCount() + theSearch.rankingProcess.getRemoteIndexCount() == 0) { + if ((aboutBody.isEmpty() && aboutHeadline.isEmpty()) || theSearch.query.getResultCount() == 0) { prop.put("nav-about", 0); } else { prop.put("nav-about", 1); @@ -349,20 +347,15 @@ public class yacysearchtrailer { // category: location search // show only if there is a location database present and if there had been any search results - if (LibraryProvider.geoLoc.isEmpty() || - theSearch.rankingProcess.rwiAvailableCount() == 0) { + if (LibraryProvider.geoLoc.isEmpty() || theSearch.query.getResultCount() == 0) { prop.put("cat-location", 0); } else { prop.put("cat-location", 1); - prop.put(fileType, "cat-location_query", theQuery.queryString(true)); - prop.put(fileType, "cat-location_queryenc", theQuery.queryString(true).replace(' ', '+')); + prop.put(fileType, "cat-location_query", theSearch.query.queryString(true)); + prop.put(fileType, "cat-location_queryenc", theSearch.query.queryString(true).replace(' ', '+')); } - - final int indexcount = theSearch.rankingProcess.rwiAvailableCount() - theSearch.rankingProcess.getMissCount() - theSearch.getSortOutCount() + theSearch.rankingProcess.getRemoteIndexCount(); - prop.put("num-results_totalcount", indexcount); - - EventTracker.update(EventTracker.EClass.SEARCH, new ProfilingGraph.EventSearch(theQuery.id(true), SearchEventType.FINALIZATION, "bottomline", 0, 0), false); - + prop.put("num-results_totalcount", theSearch.query.getResultCount()); + EventTracker.update(EventTracker.EClass.SEARCH, new ProfilingGraph.EventSearch(theSearch.query.id(true), SearchEventType.FINALIZATION, "bottomline", 0, 0), false); return prop; } diff --git a/source/net/yacy/cora/federate/solr/connector/EmbeddedSolrConnector.java b/source/net/yacy/cora/federate/solr/connector/EmbeddedSolrConnector.java index 897fd9bf1..d7325dfea 100644 --- a/source/net/yacy/cora/federate/solr/connector/EmbeddedSolrConnector.java +++ b/source/net/yacy/cora/federate/solr/connector/EmbeddedSolrConnector.java @@ -132,7 +132,9 @@ public class EmbeddedSolrConnector extends SolrServerConnector implements SolrCo @Override public synchronized void close() { + this.commit(); super.close(); + this.defaultCore.close(); this.cores.shutdown(); } diff --git a/source/net/yacy/peers/Protocol.java b/source/net/yacy/peers/Protocol.java index 36fe29735..d8470c00e 100644 --- a/source/net/yacy/peers/Protocol.java +++ b/source/net/yacy/peers/Protocol.java @@ -117,6 +117,10 @@ import org.apache.solr.client.solrj.util.ClientUtils; import org.apache.solr.common.SolrDocument; import org.apache.solr.common.SolrDocumentList; import org.apache.solr.common.SolrException; +import org.apache.solr.common.util.NamedList; +import org.apache.solr.common.util.SimpleOrderedMap; +import org.apache.solr.response.ResultContext; +import org.apache.solr.search.DocList; public final class Protocol @@ -590,7 +594,7 @@ public final class Protocol null); } - public static int primarySearch( + protected static int primarySearch( final SearchEvent event, final String wordhashes, final String excludehashes, @@ -689,7 +693,7 @@ public final class Protocol return result.urlcount; } - public static int secondarySearch( + protected static int secondarySearch( final SearchEvent event, final String wordhashes, final String urlhashes, @@ -810,7 +814,7 @@ public final class Protocol // passed all checks, store url try { - event.getQuery().getSegment().fulltext().putMetadata(urlEntry); + event.query.getSegment().fulltext().putMetadata(urlEntry); ResultURLs.stack( ASCII.String(urlEntry.url().hash()), urlEntry.url().getHost(), @@ -853,7 +857,7 @@ public final class Protocol // insert the containers to the index for ( final ReferenceContainer c : container ) { try { - event.getQuery().getSegment().storeRWI(c); + event.query.getSegment().storeRWI(c); } catch ( final Exception e ) { Log.logException(e); } @@ -938,8 +942,8 @@ public final class Protocol parts.put("exclude", UTF8.StringBody(excludehashes)); parts.put("duetime", UTF8.StringBody("1000")); parts.put("urls", UTF8.StringBody(urlhashes)); - parts.put("prefer", UTF8.StringBody(event.getQuery().prefer.pattern())); - parts.put("filter", UTF8.StringBody(event.getQuery().urlMask.pattern())); + parts.put("prefer", UTF8.StringBody(event.query.prefer.pattern())); + parts.put("filter", UTF8.StringBody(event.query.urlMask.pattern())); parts.put("modifier", UTF8.StringBody(modifier)); parts.put("language", UTF8.StringBody(language)); parts.put("sitehash", UTF8.StringBody(sitehash)); @@ -947,8 +951,8 @@ public final class Protocol parts.put("contentdom", UTF8.StringBody(contentdom)); parts.put("ttl", UTF8.StringBody("0")); parts.put("maxdist", UTF8.StringBody(Integer.toString(maxDistance))); - parts.put("profile", UTF8.StringBody(crypt.simpleEncode(event.getQuery().ranking.toExternalString()))); - parts.put("constraint", UTF8.StringBody((event.getQuery().constraint == null) ? "" : event.getQuery().constraint.exportB64())); + parts.put("profile", UTF8.StringBody(crypt.simpleEncode(event.query.ranking.toExternalString()))); + parts.put("constraint", UTF8.StringBody((event.query.constraint == null) ? "" : event.query.constraint.exportB64())); if ( secondarySearchSuperviser != null ) { parts.put("abstracts", UTF8.StringBody("auto")); // resultMap = FileUtils.table(HTTPConnector.getConnector(MultiProtocolURI.yacybotUserAgent).post(new MultiProtocolURI("http://" + hostaddress + "/yacy/search.html"), 60000, hostname, parts)); @@ -1018,26 +1022,27 @@ public final class Protocol } } - public static int solrQuery( + protected static int solrQuery( final SearchEvent event, final int offset, final int count, final Seed target, final Blacklist blacklist) { - if (event.getQuery().queryString == null || event.getQuery().queryString.length() == 0) { + if (event.query.queryString == null || event.query.queryString.length() == 0) { return -1; // we cannot query solr only with word hashes, there is no clear text string } event.addExpectedRemoteReferences(count); + QueryResponse rsp = null; SolrDocumentList docList = null; - final SolrQuery solrQuery = event.getQuery().solrQuery(); + final SolrQuery solrQuery = event.query.solrQuery(); solrQuery.setStart(offset); solrQuery.setRows(count); boolean localsearch = target == null || target.equals(event.peers.mySeed()); if (localsearch) { // search the local index try { - QueryResponse rsp = event.rankingProcess.getQuery().getSegment().fulltext().getSolr().query(solrQuery); + rsp = event.rankingProcess.getQuery().getSegment().fulltext().getSolr().query(solrQuery); docList = rsp.getResults(); } catch (SolrException e) { Network.log.logInfo("SEARCH failed (solr, 1), localpeer (" + e.getMessage() + ")", e); @@ -1050,7 +1055,7 @@ public final class Protocol final String solrURL = "http://" + target.getPublicAddress() + "/solr"; try { SolrConnector solrConnector = new RemoteSolrConnector(solrURL); - QueryResponse rsp = solrConnector.query(solrQuery); + rsp = solrConnector.query(solrQuery); docList = rsp.getResults(); // no need to close this here because that sends a commit to remote solr which is not wanted here } catch (IOException e) { @@ -1103,7 +1108,7 @@ public final class Protocol // passed all checks, store url if (!localsearch) { try { - event.getQuery().getSegment().fulltext().putDocument(ClientUtils.toSolrInputDocument(doc)); + event.query.getSegment().fulltext().putDocument(ClientUtils.toSolrInputDocument(doc)); ResultURLs.stack( ASCII.String(urlEntry.url().hash()), urlEntry.url().getHost(), @@ -1121,12 +1126,12 @@ public final class Protocol } if (localsearch) { - event.add(container, true, "localpeer", docList.size()); + event.add(container, true, "localpeer", (int) docList.getNumFound()); event.rankingProcess.addFinalize(); event.addExpectedRemoteReferences(-count); Network.log.logInfo("local search (solr): localpeer sent " + container.get(0).size() + "/" + docList.size() + " references"); } else { - event.add(container, false, target.getName() + "/" + target.hash, docList.size()); + event.add(container, false, target.getName() + "/" + target.hash, (int) docList.getNumFound()); event.rankingProcess.addFinalize(); event.addExpectedRemoteReferences(-count); Network.log.logInfo("remote search (solr): peer " + target.getName() + " sent " + container.get(0).size() + "/" + docList.size() + " references"); diff --git a/source/net/yacy/peers/RemoteSearch.java b/source/net/yacy/peers/RemoteSearch.java index e9a94e0bc..182ed377e 100644 --- a/source/net/yacy/peers/RemoteSearch.java +++ b/source/net/yacy/peers/RemoteSearch.java @@ -149,7 +149,7 @@ public class RemoteSearch extends Thread { (clusterselection == null) ? DHTSelection.selectSearchTargets( event.peers, - event.getQuery().query_include_hashes, + event.query.query_include_hashes, event.peers.redundancy(), burstRobinsonPercent, burstMultiwordPercent) @@ -172,16 +172,16 @@ public class RemoteSearch extends Thread { try { RemoteSearch rs = new RemoteSearch( event, - QueryParams.hashSet2hashString(event.getQuery().query_include_hashes), - QueryParams.hashSet2hashString(event.getQuery().query_exclude_hashes), - event.getQuery().modifier, - event.getQuery().targetlang == null ? "" : event.getQuery().targetlang, - event.getQuery().sitehash == null ? "" : event.getQuery().sitehash, - event.getQuery().authorhash == null ? "" : event.getQuery().authorhash, - event.getQuery().contentdom == null ? "all" : event.getQuery().contentdom.toString(), + QueryParams.hashSet2hashString(event.query.query_include_hashes), + QueryParams.hashSet2hashString(event.query.query_exclude_hashes), + event.query.modifier, + event.query.targetlang == null ? "" : event.query.targetlang, + event.query.sitehash == null ? "" : event.query.sitehash, + event.query.authorhash == null ? "" : event.query.authorhash, + event.query.contentdom == null ? "all" : event.query.contentdom.toString(), count, time, - event.getQuery().maxDistance, + event.query.maxDistance, targets, targetPeers[i], event.secondarySearchSuperviser, diff --git a/source/net/yacy/peers/graphics/NetworkGraph.java b/source/net/yacy/peers/graphics/NetworkGraph.java index 369439ced..87eff83f0 100644 --- a/source/net/yacy/peers/graphics/NetworkGraph.java +++ b/source/net/yacy/peers/graphics/NetworkGraph.java @@ -157,8 +157,7 @@ public class NetworkGraph { */ // draw in the search target - final QueryParams query = event.getQuery(); - final Iterator i = query.query_include_hashes.iterator(); + final Iterator i = event.query.query_include_hashes.iterator(); eventPicture.setColor(RasterPlotter.GREY); while (i.hasNext()) { byte[] wordHash = i.next(); diff --git a/source/net/yacy/search/Switchboard.java b/source/net/yacy/search/Switchboard.java index 6f5dbc489..8c6ab6dcf 100644 --- a/source/net/yacy/search/Switchboard.java +++ b/source/net/yacy/search/Switchboard.java @@ -2671,7 +2671,7 @@ public final class Switchboard extends serverSwitch { } // check if some of the links match with the query - final Map matcher = searchEvent.getQuery().separateMatches(links); + final Map matcher = searchEvent.query.separateMatches(links); // take the matcher and load them all for ( final Map.Entry entry : matcher.entrySet() ) { @@ -3319,8 +3319,7 @@ public final class Switchboard extends serverSwitch { new Thread() { @Override public void run() { - QueryParams query = searchEvent.getQuery(); - String queryString = query.queryString(true); + String queryString = searchEvent.query.queryString(true); Thread.currentThread().setName("Switchboard.heuristicRSS:" + queryString); final int meta = queryString.indexOf("heuristic:", 0); if ( meta >= 0 ) { diff --git a/source/net/yacy/search/index/Segment.java b/source/net/yacy/search/index/Segment.java index 7835df1fd..2ba66f3e3 100644 --- a/source/net/yacy/search/index/Segment.java +++ b/source/net/yacy/search/index/Segment.java @@ -413,7 +413,7 @@ public class Segment { // data during search-time. To transfer indexed data directly to the search process // the following lines push the index data additionally to the search process // this is done only for searched words - if (searchEvent != null && !searchEvent.getQuery().query_exclude_hashes.has(wordhash) && searchEvent.getQuery().query_include_hashes.has(wordhash)) { + if (searchEvent != null && !searchEvent.query.query_exclude_hashes.has(wordhash) && searchEvent.query.query_include_hashes.has(wordhash)) { // if the page was added in the context of a heuristic this shall ensure that findings will fire directly into the search result ReferenceContainer container; try { diff --git a/source/net/yacy/search/query/AccessTracker.java b/source/net/yacy/search/query/AccessTracker.java index 2799aed29..def62139e 100644 --- a/source/net/yacy/search/query/AccessTracker.java +++ b/source/net/yacy/search/query/AccessTracker.java @@ -44,9 +44,9 @@ public class AccessTracker { private final static long DUMP_PERIOD = 60000L; - public static final int minSize = 100; - public static final int maxSize = 1000; - public static final int maxAge = 24 * 60 * 60 * 1000; + private static final int minSize = 100; + private static final int maxSize = 1000; + private static final int maxAge = 24 * 60 * 60 * 1000; public enum Location {local, remote} @@ -67,7 +67,7 @@ public class AccessTracker { private static void add(final LinkedList list, final QueryParams query) { // learn that this word can be a word completion for the DidYouMeanLibrary - if (query.resultcount > 10 && query.queryString != null && query.queryString.length() > 0) { + if (query.getResultCount() > 10 && query.queryString != null && query.queryString.length() > 0) { final StringBuilder sb = new StringBuilder(query.queryString); sb.append(query.queryString); WordCache.learn(sb); @@ -109,7 +109,7 @@ public class AccessTracker { private static void addToDump(final QueryParams query) { if (query.queryString == null || query.queryString.isEmpty()) return; - addToDump(query.queryString, Integer.toString(query.resultcount), new Date(query.starttime)); + addToDump(query.queryString, Integer.toString(query.getResultCount()), new Date(query.starttime)); } public static void addToDump(String querystring, String resultcount) { diff --git a/source/net/yacy/search/query/QueryParams.java b/source/net/yacy/search/query/QueryParams.java index 36602262e..45737bb15 100644 --- a/source/net/yacy/search/query/QueryParams.java +++ b/source/net/yacy/search/query/QueryParams.java @@ -30,12 +30,15 @@ import java.io.UnsupportedEncodingException; import java.net.URLEncoder; import java.util.ArrayList; import java.util.Collection; +import java.util.Collections; import java.util.HashMap; import java.util.Iterator; import java.util.LinkedHashMap; import java.util.Map; import java.util.Set; import java.util.SortedSet; +import java.util.TreeSet; +import java.util.concurrent.atomic.AtomicInteger; import java.util.regex.Pattern; import java.util.regex.PatternSyntaxException; @@ -58,6 +61,7 @@ import net.yacy.document.Condenser; import net.yacy.document.parser.html.AbstractScraper; import net.yacy.document.parser.html.CharacterCoding; import net.yacy.kelondro.data.meta.DigestURI; +import net.yacy.kelondro.data.meta.URIMetadataRow; import net.yacy.kelondro.data.word.Word; import net.yacy.kelondro.data.word.WordReferenceRow; import net.yacy.kelondro.index.RowHandleSet; @@ -103,18 +107,16 @@ public final class QueryParams { public final String queryString; public final HandleSet query_include_hashes, query_exclude_hashes, query_all_hashes; private final Collection query_include_words, query_exclude_words, query_all_words; - public final int itemsPerPage; + public int itemsPerPage; public int offset; public final Pattern urlMask, prefer; final boolean urlMask_isCatchall; - private final boolean prefer_isMatchnothing; public final Classification.ContentDomain contentdom; public final String targetlang; protected final Collection metatags; public final String navigators; public final Searchdom domType; private final int zonecode; - private final int domMaxTargets; public final int maxDistance; public final Bitfield constraint; final boolean allofconstraint; @@ -131,14 +133,21 @@ public final class QueryParams { protected final long maxtime; protected final long timeout; // values that are set after a search: - public int resultcount; // number of found results public int transmitcount; // number of results that had been shown to the user public long searchtime, urlretrievaltime, snippetcomputationtime; // time to perform the search, to get all the urls, and to compute the snippets - private boolean specialRights; // is true if the user has a special authorization and my use more database-extensive options public final String userAgent; - public boolean filterfailurls; + protected boolean filterfailurls; protected double lat, lon, radius; + // the following values are filled during the search process as statistics for the search + public final AtomicInteger local_rwi_available; // the number of hits generated/ranked by the local search in rwi index + public final AtomicInteger local_rwi_stored; // the number of existing hits by the local search in rwi index + public final AtomicInteger local_solr_available; // the number of hits generated/ranked by the local search in solr + public final AtomicInteger local_solr_stored; // the number of existing hits by the local search in solr + public final AtomicInteger remote_available; // the number of hits imported from remote peers (rwi/solr mixed) + public final AtomicInteger remote_stored; // the number of existing hits at remote site + public final AtomicInteger remote_peerCount; // the number of peers which contributed to the remote search result + public final SortedSet misses; // url hashes that had been sorted out because of constraints in postranking public QueryParams( final String queryString, @@ -178,7 +187,6 @@ public final class QueryParams { this.urlMask = catchall_pattern; this.urlMask_isCatchall = true; this.prefer = matchnothing_pattern; - this.prefer_isMatchnothing = true; this.contentdom = ContentDomain.ALL; this.itemsPerPage = itemsPerPage; this.offset = 0; @@ -186,7 +194,6 @@ public final class QueryParams { this.metatags = new ArrayList(0); this.domType = Searchdom.LOCAL; this.zonecode = DigestURI.TLD_any_zone_filter; - this.domMaxTargets = 0; this.constraint = constraint; this.allofconstraint = false; this.snippetCacheStrategy = null; @@ -198,7 +205,6 @@ public final class QueryParams { this.starttime = Long.valueOf(System.currentTimeMillis()); this.maxtime = 10000; this.timeout = this.starttime + this.timeout; - this.specialRights = false; this.navigators = "all"; this.indexSegment = indexSegment; this.userAgent = userAgent; @@ -207,6 +213,14 @@ public final class QueryParams { this.lat = 0.0d; this.lon = 0.0d; this.radius = 0.0d; + this.local_rwi_available = new AtomicInteger(0); // the number of results in the local peer after filtering + this.local_rwi_stored = new AtomicInteger(0); + this.local_solr_available= new AtomicInteger(0); + this.local_solr_stored = new AtomicInteger(0); + this.remote_stored = new AtomicInteger(0); + this.remote_available = new AtomicInteger(0); // the number of result contributions from all the remote peers + this.remote_peerCount = new AtomicInteger(0); // the number of remote peers that have contributed + this.misses = Collections.synchronizedSortedSet(new TreeSet(URIMetadataRow.rowdef.objectOrder)); } public QueryParams( @@ -262,14 +276,13 @@ public final class QueryParams { } catch (final PatternSyntaxException ex) { throw new IllegalArgumentException("Not a valid regular expression: " + prefer, ex); } - this.prefer_isMatchnothing = this.prefer.toString().equals(matchnothing_pattern.toString()); + this.prefer.toString().equals(matchnothing_pattern.toString()); assert language != null; this.targetlang = language; this.metatags = metatags; this.navigators = navigators; this.domType = domType; this.zonecode = domainzone; - this.domMaxTargets = domMaxTargets; this.constraint = constraint; this.allofconstraint = allofconstraint; this.sitehash = site; assert site == null || site.length() == 6; @@ -281,7 +294,6 @@ public final class QueryParams { this.starttime = Long.valueOf(System.currentTimeMillis()); this.maxtime = 10000; this.timeout = this.starttime + this.timeout; - this.specialRights = specialRights; this.indexSegment = indexSegment; this.userAgent = userAgent; this.transmitcount = 0; @@ -291,6 +303,14 @@ public final class QueryParams { this.lat = Math.floor(lat * this.kmNormal) / this.kmNormal; this.lon = Math.floor(lon * this.kmNormal) / this.kmNormal; this.radius = Math.floor(radius * this.kmNormal + 1) / this.kmNormal; + this.local_rwi_available = new AtomicInteger(0); // the number of results in the local peer after filtering + this.local_rwi_stored = new AtomicInteger(0); + this.local_solr_available= new AtomicInteger(0); + this.local_solr_stored = new AtomicInteger(0); + this.remote_stored = new AtomicInteger(0); + this.remote_available = new AtomicInteger(0); // the number of result contributions from all the remote peers + this.remote_peerCount = new AtomicInteger(0); // the number of remote peers that have contributed + this.misses = Collections.synchronizedSortedSet(new TreeSet(URIMetadataRow.rowdef.objectOrder)); } private double kmNormal = 100.d; // 100 =ca 40000.d / 360.d == 111.11 - if lat/lon is multiplied with this, rounded and diveded by this, the location is normalized to a 1km grid @@ -309,6 +329,10 @@ public final class QueryParams { return this.itemsPerPage; } + public int getResultCount() { + return this.local_rwi_available.get() + this.local_solr_stored.get() - this.misses.size(); + } + public void setOffset(final int newOffset) { this.offset = newOffset; } @@ -502,7 +526,7 @@ public final class QueryParams { final SolrQuery params = new SolrQuery(); params.setQuery(q.toString()); params.setStart(this.offset); - params.setRows(this.resultcount); + params.setRows(this.itemsPerPage); params.setFacet(false); if (this.radius > 0.0d && this.lat != 0.0d && this.lon != 0.0d) { diff --git a/source/net/yacy/search/query/RankingProcess.java b/source/net/yacy/search/query/RankingProcess.java index c46f74c63..06a0b09f9 100644 --- a/source/net/yacy/search/query/RankingProcess.java +++ b/source/net/yacy/search/query/RankingProcess.java @@ -84,23 +84,18 @@ public final class RankingProcess extends Thread { protected final WeakPriorityBlockingQueue rwiStack; protected final ConcurrentHashMap> doubleDomCache; // key = domhash (6 bytes); value = like stack private final int[] flagcount; // flag counter - private final HandleSet misses; // contains url-hashes that could not been found in the LURL-DB private final AtomicInteger feedersAlive, feedersTerminated; private boolean addRunning; protected final AtomicInteger receivedRemoteReferences; protected final ReferenceOrder order; - int local_indexCount; - int remote_indexCount; - int remote_resourceSize; - int remote_peerCount; - final HandleSet urlhashes; // map for double-check; String/Long relation, addresses ranking number (backreference for deletion) - final ScoreMap hostNavigator; // a counter for the appearance of the host hash - final Map hostResolver; // a mapping from a host hash (6 bytes) to the full url hash of one of these urls that have the host hash - final Map taggingPredicates; // a map from tagging vocabulary names to tagging predicate uris - final Map> vocabularyNavigator; // counters for Vocabularies; key is metatag.getVocabularyName() + protected final HandleSet urlhashes; // map for double-check; String/Long relation, addresses ranking number (backreference for deletion) + protected final ScoreMap hostNavigator; // a counter for the appearance of the host hash + protected final Map hostResolver; // a mapping from a host hash (6 bytes) to the full url hash of one of these urls that have the host hash + protected final Map taggingPredicates; // a map from tagging vocabulary names to tagging predicate uris + protected final Map> vocabularyNavigator; // counters for Vocabularies; key is metatag.getVocabularyName() private boolean remote; - public RankingProcess(final QueryParams query, boolean remote) { + protected RankingProcess(final QueryParams query, boolean remote) { // we collect the urlhashes and construct a list with urlEntry objects // attention: if minEntries is too high, this method will not terminate within the maxTime // sortorder: 0 = hash, 1 = url, 2 = ranking @@ -112,7 +107,6 @@ public final class RankingProcess extends Thread { int stackMaxsize = query.snippetCacheStrategy == null || query.snippetCacheStrategy == CacheStrategy.CACHEONLY ? max_results_preparation_special : max_results_preparation; this.rwiStack = new WeakPriorityBlockingQueue(stackMaxsize, false); this.doubleDomCache = new ConcurrentHashMap>(); - this.misses = new RowHandleSet(URIMetadataRow.rowdef.primaryKeyLength, URIMetadataRow.rowdef.objectOrder, 100); this.flagcount = new int[32]; for ( int i = 0; i < 32; i++ ) { this.flagcount[i] = 0; @@ -122,10 +116,6 @@ public final class RankingProcess extends Thread { this.addRunning = true; this.receivedRemoteReferences = new AtomicInteger(0); this.order = new ReferenceOrder(this.query.ranking, UTF8.getBytes(this.query.targetlang)); - this.remote_resourceSize = 0; - this.remote_indexCount = 0; - this.remote_peerCount = 0; - this.local_indexCount = 0; this.urlhashes = new RowHandleSet(URIMetadataRow.rowdef.primaryKeyLength, URIMetadataRow.rowdef.objectOrder, 100); this.hostNavigator = new ConcurrentScoreMap(); this.hostResolver = new ConcurrentHashMap(); @@ -160,11 +150,6 @@ public final class RankingProcess extends Thread { this.feedersAlive.addAndGet(1); } - public int rwiAvailableCount() { - // the number of results in the local peer after filtering - return this.rwiStack.sizeAvailable(); - } - public QueryParams getQuery() { return this.query; } @@ -173,14 +158,6 @@ public final class RankingProcess extends Thread { return this.flagcount; } - public Iterator miss() { - return this.misses.iterator(); - } - - public int getMissCount() { - return this.misses.size(); - } - protected void addBegin() { this.addRunning = true; } @@ -230,21 +207,6 @@ public final class RankingProcess extends Thread { return false; } - public int getRemoteIndexCount() { - // the number of result contributions from all the remote peers - return this.remote_indexCount; - } - - public int getRemoteResourceSize() { - // the number of all hits in all the remote peers - return Math.max(this.remote_resourceSize, this.remote_indexCount); - } - - public int getRemotePeerCount() { - // the number of remote peers that have contributed - return this.remote_peerCount; - } - @Override public void run() { @@ -300,10 +262,12 @@ public final class RankingProcess extends Thread { this.addRunning = true; assert (index != null); if (index.isEmpty()) return; - if (!local) { + if (local) { + this.query.local_rwi_stored.addAndGet(fullResource); + } else { assert fullResource >= 0 : "fullResource = " + fullResource; - this.remote_resourceSize += fullResource; - this.remote_peerCount++; + this.query.remote_stored.addAndGet(fullResource); + this.query.remote_peerCount.incrementAndGet(); } long timer = System.currentTimeMillis(); @@ -433,7 +397,7 @@ public final class RankingProcess extends Thread { } } // increase counter for statistics - if (local) this.local_indexCount++; else this.remote_indexCount++; + if (local) this.query.local_rwi_available.incrementAndGet(); else this.query.remote_available.incrementAndGet(); } if (System.currentTimeMillis() >= timeout) Log.logWarning("RWIProcess", "rwi normalization ended with timeout = " + maxtime); diff --git a/source/net/yacy/search/query/SearchEvent.java b/source/net/yacy/search/query/SearchEvent.java index 23b82da75..e7985fd34 100644 --- a/source/net/yacy/search/query/SearchEvent.java +++ b/source/net/yacy/search/query/SearchEvent.java @@ -90,7 +90,7 @@ public final class SearchEvent { private final static int SNIPPET_WORKER_THREADS = Math.max(4, Runtime.getRuntime().availableProcessors() * 2); private long eventTime; - protected QueryParams query; + public QueryParams query; public final SeedDB peers; final WorkTables workTables; public final RankingProcess rankingProcess; // ordered search results, grows dynamically as all the query threads enrich this container @@ -104,8 +104,7 @@ public final class SearchEvent { private final SortedMap heuristics; private byte[] IAmaxcounthash, IAneardhthash; private final Thread localsearch; - private final AtomicInteger expectedRemoteReferences, maxExpectedRemoteReferences; - private int sortout; // counter for referenced that had been sorted out for other reasons + private final AtomicInteger expectedRemoteReferences, maxExpectedRemoteReferences; // counter for referenced that had been sorted out for other reasons private final ScoreMap authorNavigator; // a counter for the appearances of authors private final ScoreMap namespaceNavigator; // a counter for name spaces private final ScoreMap protocolNavigator; // a counter for protocol types @@ -120,7 +119,6 @@ public final class SearchEvent { protected long snippetComputationAllTime; private final boolean remote; private boolean cleanupState; - private int resultCounter = 0; protected SearchEvent( final QueryParams query, @@ -144,7 +142,6 @@ public final class SearchEvent { this.maxExpectedRemoteReferences = new AtomicInteger(0); this.expectedRemoteReferences = new AtomicInteger(0); - this.sortout = 0; this.authorNavigator = new ConcurrentScoreMap(); this.namespaceNavigator = new ConcurrentScoreMap(); this.protocolNavigator = new ConcurrentScoreMap(); @@ -306,7 +303,7 @@ public final class SearchEvent { } SearchEventCache.put(this.query.id(false), this); } - + public long getEventTime() { return this.eventTime; } @@ -315,14 +312,6 @@ public final class SearchEvent { this.eventTime = System.currentTimeMillis(); } - public QueryParams getQuery() { - return this.query; - } - - public void setQuery(final QueryParams query) { - this.query = query; - } - protected void cleanup() { this.cleanupState = true; @@ -486,10 +475,12 @@ public final class SearchEvent { assert (index != null); if (index.isEmpty()) return; - if (!local) { + if (local) { + this.query.local_solr_stored.set(fullResource); + } else { assert fullResource >= 0 : "fullResource = " + fullResource; - this.rankingProcess.remote_resourceSize += fullResource; - this.rankingProcess.remote_peerCount++; + this.query.remote_stored.addAndGet(fullResource); + this.query.remote_peerCount.incrementAndGet(); } long timer = System.currentTimeMillis(); @@ -604,7 +595,7 @@ public final class SearchEvent { } } // increase counter for statistics - if (local) this.rankingProcess.local_indexCount++; else this.rankingProcess.remote_indexCount++; + if (local) this.query.local_solr_available.incrementAndGet(); else this.query.remote_available.incrementAndGet(); } } catch ( final SpaceExceededException e ) { } @@ -626,10 +617,6 @@ public final class SearchEvent { this.expectedRemoteReferences.addAndGet(x); } - public int getSortOutCount() { - return this.sortout; - } - private URIMetadataNode takeRWI(final boolean skipDoubleDom, final long waitingtime) { // returns from the current RWI list the best entry and removes this entry from the list @@ -771,27 +758,27 @@ public final class SearchEvent { if ( !this.query.urlMask_isCatchall ) { // check url mask if ( !page.matches(this.query.urlMask) ) { - this.sortout++; + this.query.misses.add(page.hash()); continue; } } // check for more errors if ( page.url() == null ) { - this.sortout++; + this.query.misses.add(page.hash()); continue; // rare case where the url is corrupted } // check content domain if ((this.query.contentdom.getCode() > 0 && page.url().getContentDomain() != this.query.contentdom) || (this.query.contentdom == Classification.ContentDomain.TEXT && page.url().getContentDomain().getCode() > 0)) { - this.sortout++; + this.query.misses.add(page.hash()); continue; } // Check for blacklist if (Switchboard.urlBlacklist.isListed(BlacklistType.SEARCH, page)) { - this.sortout++; + this.query.misses.add(page.hash()); continue; } @@ -805,7 +792,7 @@ public final class SearchEvent { FilterEngine f = ContentControlFilterUpdateThread.getNetworkFilter(); if (f != null) { if (!f.isListed(page.url(), null)) { - this.sortout++; + this.query.misses.add(page.hash()); continue; } } @@ -821,7 +808,7 @@ public final class SearchEvent { ((QueryParams.anymatch(pagetitle, this.query.query_exclude_hashes)) || (QueryParams.anymatch(pageurl.toLowerCase(), this.query.query_exclude_hashes)) || (QueryParams.anymatch(pageauthor.toLowerCase(), this.query.query_exclude_hashes)))) { - this.sortout++; + this.query.misses.add(page.hash()); continue; } @@ -831,13 +818,13 @@ public final class SearchEvent { while ( wi.hasNext() ) { this.query.getSegment().termIndex().removeDelayed(wi.next(), page.hash()); } - this.sortout++; + this.query.misses.add(page.hash()); continue; } // check location constraint if ((this.query.constraint != null) && (this.query.constraint.get(Condenser.flag_cat_haslocation)) && (page.lat() == 0.0f || page.lon() == 0.0f)) { - this.sortout++; + this.query.misses.add(page.hash()); continue; } @@ -848,7 +835,7 @@ public final class SearchEvent { double lonDelta = this.query.lon - lon; double distance = Math.sqrt(latDelta * latDelta + lonDelta * lonDelta); // pythagoras if (distance > this.query.radius) { - this.sortout++; + this.query.misses.add(page.hash()); continue; } } @@ -861,20 +848,20 @@ public final class SearchEvent { // check if we already are filtering for authors if ( this.query.authorhash != null && !this.query.authorhash.equals(authorhash) ) { - this.sortout++; + this.query.misses.add(page.hash()); continue; } // add author to the author navigator this.authorNavigator.inc(pageauthor); } else if ( this.query.authorhash != null ) { - this.sortout++; + this.query.misses.add(page.hash()); continue; } // check Scanner if ( !Scanner.acceptURL(page.url()) ) { - this.sortout++; + this.query.misses.add(page.hash()); continue; } diff --git a/source/net/yacy/search/query/SearchEventCache.java b/source/net/yacy/search/query/SearchEventCache.java index 56805e4eb..e3daec4da 100644 --- a/source/net/yacy/search/query/SearchEventCache.java +++ b/source/net/yacy/search/query/SearchEventCache.java @@ -138,8 +138,9 @@ public class SearchEventCache { if (event != null) { //re-new the event time for this event, so it is not deleted next time too early event.resetEventTime(); - // replace the query, because this contains the current result offset - event.setQuery(query); + // replace the current result offset + event.query.offset = query.offset; + event.query.itemsPerPage = query.itemsPerPage; } } if (event == null) {