mirror of
https://github.com/yacy/yacy_search_server.git
synced 2024-09-22 00:00:59 +02:00
d2ba1fd2ab
This change is inspired by the need to see a network connected to the index it creates in a indexing team. It is not possible to divide the network and the index. Therefore all control files for the network was moved to the network within the INDEX/<network-name> subfolder. The remaining YACYDB is superfluous and can be deleted. The yacyDB and yacyNews data structures are now part of plasmaWordIndex. Therefore all methods, using static access to yacySeedDB had to be rewritten. A special problem had been all the port forwarding methods which had been tightly mixed with seed construction. It was not possible to move the port forwarding functions to the place, meaning and usage of plasmaWordIndex. Therefore the port forwarding had been deleted (I guess nobody used it and it can be simulated by methods outside of YaCy). The mySeed.txt is automatically moved to the current network position. A new effect causes that every network will create a different local seed file, which is ok, since the seed identifies the peer only against the network (it is the purpose of the seed hash to give a peer a location within the DHT). No other functional change has been made. The next steps to enable network switcing are: - shift of crawler tables from PLASMADB into the network (crawls are also network-specific) - possibly shift of plasmaWordIndex code into yacy package (index management is network-specific) - servlet to switch networks git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@4765 6c8d7289-2bf4-0310-a012-ef5d649a1542
329 lines
18 KiB
Java
329 lines
18 KiB
Java
// ysearchitem.java
|
|
// (C) 2004-2008 by Michael Peter Christen; mc@yacy.net, Frankfurt a. M., Germany
|
|
// first published 2004 on http://yacy.net
|
|
//
|
|
// This is a part of YaCy, a peer-to-peer based web search engine
|
|
//
|
|
// $LastChangedDate: 2006-04-02 22:40:07 +0200 (So, 02 Apr 2006) $
|
|
// $LastChangedRevision: 1986 $
|
|
// $LastChangedBy: orbiter $
|
|
//
|
|
// LICENSE
|
|
//
|
|
// This program is free software; you can redistribute it and/or modify
|
|
// it under the terms of the GNU General Public License as published by
|
|
// the Free Software Foundation; either version 2 of the License, or
|
|
// (at your option) any later version.
|
|
//
|
|
// This program is distributed in the hope that it will be useful,
|
|
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
// GNU General Public License for more details.
|
|
//
|
|
// You should have received a copy of the GNU General Public License
|
|
// along with this program; if not, write to the Free Software
|
|
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
|
|
|
import java.util.TreeSet;
|
|
|
|
import de.anomic.http.httpHeader;
|
|
import de.anomic.index.indexWord;
|
|
import de.anomic.kelondro.kelondroBitfield;
|
|
import de.anomic.kelondro.kelondroMSetTools;
|
|
import de.anomic.plasma.plasmaCondenser;
|
|
import de.anomic.plasma.plasmaSearchEvent;
|
|
import de.anomic.plasma.plasmaSearchQuery;
|
|
import de.anomic.plasma.plasmaSearchRankingProfile;
|
|
import de.anomic.plasma.plasmaSwitchboard;
|
|
import de.anomic.server.serverCore;
|
|
import de.anomic.server.serverObjects;
|
|
import de.anomic.server.serverSwitch;
|
|
import de.anomic.server.logging.serverLog;
|
|
import de.anomic.tools.yFormatter;
|
|
import de.anomic.yacy.yacyURL;
|
|
|
|
public class result {
|
|
|
|
public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch<?> env) {
|
|
final plasmaSwitchboard sb = (plasmaSwitchboard) env;
|
|
sb.localSearchLastAccess = System.currentTimeMillis();
|
|
|
|
boolean searchAllowed = sb.getConfigBool("publicSearchpage", true) || sb.verifyAuthentication(header, false);
|
|
boolean authenticated = sb.adminAuthenticated(header) >= 2;
|
|
int display = (post == null) ? 0 : post.getInt("display", 0);
|
|
if ((display == 1) && (!authenticated)) display = 0;
|
|
int input = (post == null) ? 2 : post.getInt("input", 2);
|
|
String promoteSearchPageGreeting = env.getConfig("promoteSearchPageGreeting", "");
|
|
if (env.getConfigBool("promoteSearchPageGreeting.useNetworkName", false)) promoteSearchPageGreeting = env.getConfig("network.unit.description", "");
|
|
if (promoteSearchPageGreeting.length() == 0) promoteSearchPageGreeting = "P2P WEB SEARCH";
|
|
|
|
// get query
|
|
String querystring = (post == null) ? "" : post.get("search", "").trim();
|
|
final serverObjects prop = new serverObjects();
|
|
|
|
boolean rss = (post == null) ? false : post.get("rss", "false").equals("true");
|
|
if ((post == null) || (env == null) || (querystring.length() == 0) || (!searchAllowed)) {
|
|
// we create empty entries for template strings
|
|
prop.put("searchagain", "0");
|
|
prop.put("input", input);
|
|
prop.put("display", display);
|
|
prop.put("input_input", input);
|
|
prop.put("input_display", display);
|
|
prop.put("promoteSearchPageGreeting", promoteSearchPageGreeting);
|
|
prop.put("input_former", "");
|
|
prop.put("former", "");
|
|
prop.put("input_count", "10");
|
|
prop.put("input_offset", "0");
|
|
prop.put("input_resource", "global");
|
|
prop.put("input_urlmaskfilter", ".*");
|
|
prop.put("input_prefermaskfilter", "");
|
|
prop.put("input_indexof", "off");
|
|
prop.put("input_constraint", "");
|
|
prop.put("input_cat", "href");
|
|
prop.put("input_depth", "0");
|
|
prop.put("input_contentdom", "text");
|
|
prop.put("input_contentdomCheckText", "1");
|
|
prop.put("input_contentdomCheckAudio", "0");
|
|
prop.put("input_contentdomCheckVideo", "0");
|
|
prop.put("input_contentdomCheckImage", "0");
|
|
prop.put("input_contentdomCheckApp", "0");
|
|
prop.put("excluded", "0");
|
|
prop.put("results", "");
|
|
prop.put("resultTable", "0");
|
|
prop.put("num-results", searchAllowed ? "0" : "4");
|
|
|
|
return prop;
|
|
}
|
|
|
|
// collect search attributes
|
|
int maxDistance = Integer.MAX_VALUE;
|
|
|
|
if ((querystring.length() > 2) && (querystring.charAt(0) == '"') && (querystring.charAt(querystring.length() - 1) == '"')) {
|
|
querystring = querystring.substring(1, querystring.length() - 1).trim();
|
|
maxDistance = 1;
|
|
}
|
|
if (sb.facilityDB != null) try { sb.facilityDB.update("zeitgeist", querystring, post); } catch (Exception e) {}
|
|
|
|
int itemsPerPage = post.getInt("count", 10);
|
|
int offset = post.getInt("offset", 0);
|
|
boolean global = (post == null) ? true : post.get("resource", "global").equals("global");
|
|
final boolean indexof = post.get("indexof","").equals("on");
|
|
String urlmask = "";
|
|
if (post.containsKey("urlmask") && post.get("urlmask").equals("no")) {
|
|
urlmask = ".*";
|
|
} else {
|
|
urlmask = (post.containsKey("urlmaskfilter")) ? (String) post.get("urlmaskfilter") : ".*";
|
|
}
|
|
String prefermask = post.get("prefermaskfilter", "");
|
|
if ((prefermask.length() > 0) && (prefermask.indexOf(".*") < 0)) prefermask = ".*" + prefermask + ".*";
|
|
|
|
kelondroBitfield constraint = ((post.containsKey("constraint")) && (post.get("constraint", "").length() > 0)) ? new kelondroBitfield(4, post.get("constraint", "______")) : null;
|
|
if (indexof) {
|
|
constraint = new kelondroBitfield(4);
|
|
constraint.set(plasmaCondenser.flag_cat_indexof, true);
|
|
}
|
|
|
|
int domainzone = post.getInt("zone", yacyURL.TLD_any_zone_filter);
|
|
|
|
// SEARCH
|
|
//final boolean indexDistributeGranted = sb.getConfig(plasmaSwitchboard.INDEX_DIST_ALLOW, "true").equals("true");
|
|
//final boolean indexReceiveGranted = sb.getConfig("allowReceiveIndex", "true").equals("true");
|
|
//final boolean offline = yacyCore.seedDB.mySeed().isVirgin();
|
|
final boolean clustersearch = sb.isRobinsonMode() &&
|
|
(sb.getConfig("cluster.mode", "").equals("privatecluster") ||
|
|
sb.getConfig("cluster.mode", "").equals("publiccluster"));
|
|
//if (offline || !indexDistributeGranted || !indexReceiveGranted) { global = false; }
|
|
if (clustersearch) global = true; // switches search on, but search target is limited to cluster nodes
|
|
|
|
// find search domain
|
|
int contentdomCode = plasmaSearchQuery.contentdomParser(post.get("contentdom", "text"));
|
|
|
|
// patch until better search profiles are available
|
|
if ((contentdomCode != plasmaSearchQuery.CONTENTDOM_TEXT) && (itemsPerPage <= 32)) itemsPerPage = 32;
|
|
|
|
if (post.get("cat", "href").equals("href")) {
|
|
|
|
final TreeSet<String>[] query = plasmaSearchQuery.cleanQuery(querystring); // converts also umlaute
|
|
boolean near = (query[0].contains("near")) && (querystring.indexOf("NEAR") >= 0);
|
|
if (near) {
|
|
query[0].remove("near");
|
|
}
|
|
plasmaSearchRankingProfile ranking = sb.getRanking();
|
|
if (near) {
|
|
ranking.coeff_worddistance = plasmaSearchRankingProfile.COEFF_MAX;
|
|
}
|
|
|
|
// filter out stopwords
|
|
final TreeSet<String> filtered = kelondroMSetTools.joinConstructive(query[0], plasmaSwitchboard.stopwords);
|
|
if (filtered.size() > 0) {
|
|
kelondroMSetTools.excludeDestructive(query[0], plasmaSwitchboard.stopwords);
|
|
}
|
|
|
|
// prepare search properties
|
|
final boolean yacyonline = ((sb.wordIndex.seedDB != null) && (sb.wordIndex.seedDB.mySeed() != null) && (sb.wordIndex.seedDB.mySeed().getPublicAddress() != null));
|
|
final boolean globalsearch = (global) && (yacyonline) && (sb.getConfigBool(plasmaSwitchboard.INDEX_RECEIVE_ALLOW, false));
|
|
|
|
// do the search
|
|
String client = (String) header.get(httpHeader.CONNECTION_PROP_CLIENTIP); // the search client who initiated the search
|
|
TreeSet<String> queryHashes = indexWord.words2hashes(query[0]);
|
|
plasmaSearchQuery theQuery = new plasmaSearchQuery(
|
|
querystring,
|
|
queryHashes,
|
|
indexWord.words2hashes(query[1]),
|
|
ranking,
|
|
maxDistance,
|
|
prefermask,
|
|
contentdomCode,
|
|
true,
|
|
itemsPerPage,
|
|
offset,
|
|
urlmask,
|
|
(clustersearch && globalsearch) ? plasmaSearchQuery.SEARCHDOM_CLUSTERALL :
|
|
((globalsearch) ? plasmaSearchQuery.SEARCHDOM_GLOBALDHT : plasmaSearchQuery.SEARCHDOM_LOCAL),
|
|
"",
|
|
20,
|
|
constraint,
|
|
true,
|
|
domainzone,
|
|
client);
|
|
|
|
|
|
// tell all threads to do nothing for a specific time
|
|
sb.intermissionAllThreads(3000);
|
|
|
|
// filter out words that appear in bluelist
|
|
theQuery.filterOut(plasmaSwitchboard.blueList);
|
|
|
|
// log
|
|
serverLog.logInfo("LOCAL_SEARCH", "INIT WORD SEARCH: " + theQuery.queryString + ":" + theQuery.queryHashes + " - " + theQuery.neededResults() + " links to be computed, " + theQuery.displayResults() + " lines to be displayed");
|
|
long timestamp = System.currentTimeMillis();
|
|
|
|
// create a new search event
|
|
if (plasmaSearchEvent.getEvent(theQuery.id(false)) == null) {
|
|
theQuery.setOffset(0); // in case that this is a new search, always start without a offset
|
|
offset = 0;
|
|
}
|
|
plasmaSearchEvent theSearch = plasmaSearchEvent.getEvent(theQuery, ranking, sb.wordIndex, sb.crawlResults, (sb.isRobinsonMode()) ? sb.clusterhashes : null, false);
|
|
|
|
// generate result object
|
|
serverLog.logFine("LOCAL_SEARCH", "SEARCH TIME AFTER ORDERING OF SEARCH RESULTS: " + ((System.currentTimeMillis() - timestamp) / 1000) + " seconds");
|
|
serverLog.logFine("LOCAL_SEARCH", "SEARCH TIME AFTER RESULT PREPARATION: " + ((System.currentTimeMillis() - timestamp) / 1000) + " seconds");
|
|
|
|
// calc some more cross-reference
|
|
serverLog.logFine("LOCAL_SEARCH", "SEARCH TIME AFTER XREF PREPARATION: " + ((System.currentTimeMillis() - timestamp) / 1000) + " seconds");
|
|
|
|
// log
|
|
serverLog.logInfo("LOCAL_SEARCH", "EXIT WORD SEARCH: " + theQuery.queryString + " - " +
|
|
(theSearch.getRankingResult().getLocalResourceSize() + theSearch.getRankingResult().getRemoteResourceSize()) + " links found, " +
|
|
((System.currentTimeMillis() - timestamp) / 1000) + " seconds");
|
|
|
|
// prepare search statistics
|
|
theQuery.resultcount = theSearch.getRankingResult().getLocalResourceSize() + theSearch.getRankingResult().getRemoteResourceSize();
|
|
theQuery.searchtime = System.currentTimeMillis() - timestamp;
|
|
theQuery.urlretrievaltime = theSearch.getURLRetrievalTime();
|
|
theQuery.snippetcomputationtime = theSearch.getSnippetComputationTime();
|
|
sb.localSearches.add(theQuery);
|
|
TreeSet<Long> handles = sb.localSearchTracker.get(client);
|
|
if (handles == null) handles = new TreeSet<Long>();
|
|
handles.add(theQuery.handle);
|
|
sb.localSearchTracker.put(client, handles);
|
|
|
|
int totalcount = theSearch.getRankingResult().getLocalResourceSize() + theSearch.getRankingResult().getRemoteResourceSize();
|
|
prop.put("num-results_offset", offset);
|
|
prop.put("num-results_itemscount", "0");
|
|
prop.put("num-results_itemsPerPage", itemsPerPage);
|
|
prop.put("num-results_totalcount", yFormatter.number(totalcount, !rss));
|
|
prop.put("num-results_globalresults", (globalsearch) ? "1" : "0");
|
|
prop.put("num-results_globalresults_localResourceSize", yFormatter.number(theSearch.getRankingResult().getLocalResourceSize(), !rss));
|
|
prop.put("num-results_globalresults_remoteResourceSize", yFormatter.number(theSearch.getRankingResult().getRemoteResourceSize(), !rss));
|
|
prop.put("num-results_globalresults_remoteIndexCount", yFormatter.number(theSearch.getRankingResult().getRemoteIndexCount(), !rss));
|
|
prop.put("num-results_globalresults_remotePeerCount", yFormatter.number(theSearch.getRankingResult().getRemotePeerCount(), !rss));
|
|
|
|
// generate the search result lines; they will be produced by another servlet
|
|
for (int i = 0; i < theQuery.displayResults(); i++) {
|
|
prop.put("results_" + i + "_item", offset + i);
|
|
prop.put("results_" + i + "_eventID", theQuery.id(false));
|
|
prop.put("results_" + i + "_ID", theQuery.id(false).substring(1,6));
|
|
}
|
|
prop.put("results", theQuery.displayResults());
|
|
prop.put("resultTable", (contentdomCode <= 1) ? "0" : "1");
|
|
prop.put("eventID", theQuery.id(false)); // for bottomline
|
|
prop.put("ID", theQuery.id(false).substring(1,6)); // for bottomline
|
|
|
|
// process result of search
|
|
if (filtered.size() > 0) {
|
|
prop.put("excluded", "1");
|
|
prop.putHTML("excluded_stopwords", filtered.toString());
|
|
} else {
|
|
prop.put("excluded", "0");
|
|
}
|
|
|
|
if (prop == null || prop.isEmpty()) {
|
|
if (post.get("search", "").length() < 3) {
|
|
prop.put("num-results", "2"); // no results - at least 3 chars
|
|
} else {
|
|
prop.put("num-results", "1"); // no results
|
|
}
|
|
} else {
|
|
prop.put("num-results", "3");
|
|
}
|
|
|
|
prop.put("input_cat", "href");
|
|
prop.put("input_depth", "0");
|
|
|
|
// adding some additional properties needed for the rss feed
|
|
String hostName = (String) header.get("Host", "localhost");
|
|
if (hostName.indexOf(":") == -1) hostName += ":" + serverCore.getPortNr(env.getConfig("port", "8080"));
|
|
prop.put("searchBaseURL", "http://" + hostName + "/yacysearch.html");
|
|
prop.put("rssYacyImageURL", "http://" + hostName + "/env/grafics/yacy.gif");
|
|
}
|
|
|
|
prop.put("searchagain", global ? "1" : "0");
|
|
prop.put("input", input);
|
|
prop.put("display", display);
|
|
prop.put("input_input", input);
|
|
prop.put("input_display", display);
|
|
prop.put("promoteSearchPageGreeting", promoteSearchPageGreeting);
|
|
prop.putHTML("input_former", querystring);
|
|
//prop.put("former", post.get("search", ""));
|
|
prop.put("input_count", itemsPerPage);
|
|
prop.put("input_offset", offset);
|
|
prop.put("input_resource", global ? "global" : "local");
|
|
prop.putHTML("input_urlmaskfilter", urlmask);
|
|
prop.putHTML("input_prefermaskfilter", prefermask);
|
|
prop.put("input_indexof", (indexof) ? "on" : "off");
|
|
prop.put("input_constraint", (constraint == null) ? "" : constraint.exportB64());
|
|
prop.put("input_contentdom", post.get("contentdom", "text"));
|
|
prop.put("input_contentdomCheckText", (contentdomCode == plasmaSearchQuery.CONTENTDOM_TEXT) ? "1" : "0");
|
|
prop.put("input_contentdomCheckAudio", (contentdomCode == plasmaSearchQuery.CONTENTDOM_AUDIO) ? "1" : "0");
|
|
prop.put("input_contentdomCheckVideo", (contentdomCode == plasmaSearchQuery.CONTENTDOM_VIDEO) ? "1" : "0");
|
|
prop.put("input_contentdomCheckImage", (contentdomCode == plasmaSearchQuery.CONTENTDOM_IMAGE) ? "1" : "0");
|
|
prop.put("input_contentdomCheckApp", (contentdomCode == plasmaSearchQuery.CONTENTDOM_APP) ? "1" : "0");
|
|
|
|
// for RSS: don't HTML encode some elements
|
|
prop.putHTML("rss_query", querystring, true);
|
|
prop.put("rss_queryenc", yacyURL.escape(querystring.replace(' ', '+')));
|
|
|
|
sb.localSearchLastAccess = System.currentTimeMillis();
|
|
|
|
// return rewrite properties
|
|
return prop;
|
|
}
|
|
/*
|
|
private static String navhidden(int page, int display, plasmaSearchQuery theQuery) {
|
|
return
|
|
"<input type=\"hidden\" name=\"display\" value=\"" + display + "\" />" +
|
|
"<input type=\"hidden\" name=\"search\" value=\"" + theQuery.queryString() + "\" />" +
|
|
"<input type=\"hidden\" name=\"count\" value=\"" + theQuery.displayResults() + "\" />" +
|
|
"<input type=\"hidden\" name=\"offset\" value=\"" + (page * theQuery.displayResults()) + "\" />" +
|
|
"<input type=\"hidden\" name=\"resource\" value=\"" + theQuery.searchdom() + "\" />" +
|
|
"<input type=\"hidden\" name=\"urlmaskfilter\" value=\"" + theQuery.urlMask + "\" />" +
|
|
"<input type=\"hidden\" name=\"prefermaskfilter\" value=\"" + theQuery.prefer + "\" />" +
|
|
"<input type=\"hidden\" name=\"cat\" value=\"href\" />" +
|
|
"<input type=\"hidden\" name=\"constraint\" value=\"" + ((theQuery.constraint == null) ? "" : theQuery.constraint.exportB64()) + "\" />" +
|
|
"<input type=\"hidden\" name=\"contentdom\" value=\"" + theQuery.contentdom() + "\" />" +
|
|
"<input type=\"hidden\" name=\"former\" value=\"" + theQuery.queryString() + "\" />";
|
|
}
|
|
*/
|
|
}
|