mirror of
https://github.com/yacy/yacy_search_server.git
synced 2024-09-19 00:01:41 +02:00
code simplifications
This commit is contained in:
Michael Peter Christen
parent
755f5e76cf
commit
f00733186b
|
|
@ -64,7 +64,7 @@ public final class Connections_p {
|
|||
doNameLookup = post.getBoolean("nameLookup");
|
||||
if (post.containsKey("closeServerSession")) {
|
||||
final String sessionName = post.get("closeServerSession", null);
|
||||
sb.closeSessions("10_httpd", sessionName);
|
||||
sb.closeSessions(sessionName);
|
||||
prop.put("LOCATION","");
|
||||
return prop;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -31,10 +31,8 @@ import java.io.IOException;
|
|||
import java.io.InputStreamReader;
|
||||
import java.io.Reader;
|
||||
import java.net.InetAddress;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collection;
|
||||
import java.util.Iterator;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.NavigableMap;
|
||||
import java.util.Random;
|
||||
|
|
@ -462,26 +460,10 @@ public class serverSwitch
|
|||
}
|
||||
}
|
||||
|
||||
public String[] sessionsOlderThan(String threadName, long timeout) {
|
||||
final List<String> list = new ArrayList<String>();
|
||||
final WorkflowThread st = getThread(threadName);
|
||||
|
||||
for ( final Session s : serverCore.getJobList() ) {
|
||||
if ( !s.isAlive() ) {
|
||||
continue;
|
||||
}
|
||||
if ( s.getTime() > timeout ) {
|
||||
list.add(s.getName());
|
||||
}
|
||||
}
|
||||
return (String[]) list.toArray();
|
||||
}
|
||||
|
||||
public void closeSessions(String threadName, String sessionName) {
|
||||
public void closeSessions(String sessionName) {
|
||||
if ( sessionName == null ) {
|
||||
return;
|
||||
}
|
||||
final WorkflowThread st = getThread(threadName);
|
||||
|
||||
for ( final Session s : serverCore.getJobList() ) {
|
||||
if ( (s.isAlive()) && (s.getName().equals(sessionName)) ) {
|
||||
|
|
|
|||
|
|
@ -3,20 +3,15 @@ package net.yacy.interaction;
|
|||
import java.io.BufferedReader;
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.File;
|
||||
import java.io.FileReader;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.io.StringReader;
|
||||
import java.net.URLEncoder;
|
||||
|
||||
import net.yacy.yacy;
|
||||
import net.yacy.cora.document.ASCII;
|
||||
import net.yacy.cora.protocol.Domains;
|
||||
import net.yacy.cora.protocol.RequestHeader;
|
||||
import net.yacy.cora.protocol.http.HTTPClient;
|
||||
import net.yacy.document.Document;
|
||||
import net.yacy.document.TextParser;
|
||||
import net.yacy.kelondro.data.meta.DigestURI;
|
||||
import net.yacy.kelondro.logging.Log;
|
||||
import net.yacy.search.Switchboard;
|
||||
|
|
@ -89,28 +84,6 @@ public class AugmentHtmlStream {
|
|||
|
||||
}
|
||||
|
||||
/**
|
||||
* load snippet from resource text file
|
||||
*
|
||||
* @return text from resource text file
|
||||
*/
|
||||
private static String loadPart(String part) {
|
||||
String result = "";
|
||||
try {
|
||||
BufferedReader in = new BufferedReader(new FileReader(yacy.homedir + File.separatorChar + "htroot"
|
||||
+ File.separatorChar + "interaction" + File.separatorChar
|
||||
+ "parts" + File.separatorChar + part));
|
||||
String str;
|
||||
while ((str = in.readLine()) != null) {
|
||||
result += str;
|
||||
}
|
||||
in.close();
|
||||
} catch (IOException e) {
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
public static StringBuffer process(StringBuffer data, DigestURI url, RequestHeader requestHeader) {
|
||||
|
||||
String action = requestHeader.get("YACYACTION");
|
||||
|
|
@ -150,28 +123,28 @@ public class AugmentHtmlStream {
|
|||
Doc = processAddDoctype(Doc);
|
||||
augmented = true;
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
if (sb.getConfigBool("augmentation.reparse", false) == true) {
|
||||
|
||||
|
||||
org.jsoup.nodes.Document d = Jsoup.parse(Doc);
|
||||
|
||||
|
||||
d.title ("yacy - "+d.title());
|
||||
|
||||
|
||||
if (sb.getConfigBool("interaction.overlayinteraction.enabled", false) == true) {
|
||||
|
||||
|
||||
d.head().append (loadInternal("env/templates/jqueryheader.template", requestHeader));
|
||||
d.head().append ("<script type='text/javascript'>"+loadInternal("interaction_elements/interaction.js", requestHeader)+"</script>");
|
||||
d.head().append ("<script type='text/javascript'>"+loadInternal("interaction_elements/interaction_metadata.js", requestHeader)+"</script>");
|
||||
|
||||
|
||||
d.head().append ("<script type='text/javascript'>"+loadInternal("interaction_elements/interaction_metadata.js", requestHeader)+"</script>");
|
||||
|
||||
|
||||
d.body().append (loadInternal("interaction_elements/OverlayInteraction.html?action="+action+"&urlhash="+ ASCII.String(url.hash()) +"&url="+url.toNormalform(false, true), requestHeader));
|
||||
d.body().append (loadInternal("interaction_elements/Footer.html?action="+action+"&urlhash="+ ASCII.String(url.hash()) +"&url="+url.toNormalform(false, true), requestHeader));
|
||||
|
||||
d.body().append (loadInternal("interaction_elements/Footer.html?action="+action+"&urlhash="+ ASCII.String(url.hash()) +"&url="+url.toNormalform(false, true), requestHeader));
|
||||
|
||||
}
|
||||
|
||||
|
||||
Doc = d.html();
|
||||
|
||||
|
||||
augmented = true;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -45,11 +45,9 @@ package net.yacy.peers;
|
|||
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStreamReader;
|
||||
import java.net.InetAddress;
|
||||
import java.net.MalformedURLException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.Iterator;
|
||||
|
|
@ -58,12 +56,10 @@ import java.util.List;
|
|||
import java.util.Map;
|
||||
import java.util.SortedMap;
|
||||
import java.util.TreeMap;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
import net.yacy.migration;
|
||||
import net.yacy.cora.date.GenericFormatter;
|
||||
import net.yacy.cora.document.ASCII;
|
||||
import net.yacy.cora.document.Classification;
|
||||
import net.yacy.cora.document.JSONArray;
|
||||
import net.yacy.cora.document.JSONException;
|
||||
import net.yacy.cora.document.JSONObject;
|
||||
|
|
@ -88,7 +84,6 @@ import net.yacy.kelondro.data.word.WordReference;
|
|||
import net.yacy.kelondro.data.word.WordReferenceFactory;
|
||||
import net.yacy.kelondro.logging.Log;
|
||||
import net.yacy.kelondro.order.Base64Order;
|
||||
import net.yacy.kelondro.order.Bitfield;
|
||||
import net.yacy.kelondro.order.Digest;
|
||||
import net.yacy.kelondro.rwi.Reference;
|
||||
import net.yacy.kelondro.rwi.ReferenceContainer;
|
||||
|
|
@ -105,10 +100,8 @@ import net.yacy.search.EventTracker;
|
|||
import net.yacy.search.Switchboard;
|
||||
import net.yacy.search.SwitchboardConstants;
|
||||
import net.yacy.search.index.Segment;
|
||||
import net.yacy.search.query.QueryParams;
|
||||
import net.yacy.search.query.RWIProcess;
|
||||
import net.yacy.search.query.SearchEvent;
|
||||
import net.yacy.search.ranking.RankingProfile;
|
||||
import net.yacy.search.snippet.TextSnippet;
|
||||
|
||||
import org.apache.http.entity.mime.content.ContentBody;
|
||||
|
|
@ -593,12 +586,10 @@ public final class Protocol
|
|||
}
|
||||
|
||||
public static int search(
|
||||
final Seed mySeed,
|
||||
final SearchEvent event,
|
||||
final String wordhashes,
|
||||
final String excludehashes,
|
||||
final String urlhashes,
|
||||
final Pattern prefer,
|
||||
final Pattern filter,
|
||||
final String modifier,
|
||||
final String language,
|
||||
final String sitehash,
|
||||
|
|
@ -610,12 +601,8 @@ public final class Protocol
|
|||
final boolean global,
|
||||
final int partitions,
|
||||
final Seed target,
|
||||
final Segment indexSegment,
|
||||
final RWIProcess containerCache,
|
||||
final SearchEvent.SecondarySearchSuperviser secondarySearchSuperviser,
|
||||
final Blacklist blacklist,
|
||||
final RankingProfile rankingProfile,
|
||||
final Bitfield constraint) {
|
||||
final Blacklist blacklist) {
|
||||
// send a search request to peer with remote Hash
|
||||
|
||||
// INPUT:
|
||||
|
|
@ -632,18 +619,16 @@ public final class Protocol
|
|||
// duetime : maximum time that a peer should spent to create a result
|
||||
|
||||
final long timestamp = System.currentTimeMillis();
|
||||
containerCache.addExpectedRemoteReferences(count);
|
||||
event.rankingProcess.addExpectedRemoteReferences(count);
|
||||
SearchResult result;
|
||||
try {
|
||||
result =
|
||||
new SearchResult(
|
||||
event,
|
||||
basicRequestParts(Switchboard.getSwitchboard(), target.hash, crypt.randomSalt()),
|
||||
mySeed,
|
||||
wordhashes,
|
||||
excludehashes,
|
||||
urlhashes,
|
||||
prefer,
|
||||
filter,
|
||||
modifier,
|
||||
language,
|
||||
sitehash,
|
||||
|
|
@ -656,9 +641,8 @@ public final class Protocol
|
|||
partitions,
|
||||
target.getHexHash() + ".yacyh",
|
||||
target.getClusterAddress(),
|
||||
secondarySearchSuperviser,
|
||||
rankingProfile,
|
||||
constraint);
|
||||
secondarySearchSuperviser
|
||||
);
|
||||
} catch ( final IOException e ) {
|
||||
Network.log.logInfo("SEARCH failed, Peer: "
|
||||
+ target.hash
|
||||
|
|
@ -757,10 +741,10 @@ public final class Protocol
|
|||
|
||||
// passed all checks, store url
|
||||
try {
|
||||
indexSegment.fulltext().putMetadata(urlEntry);
|
||||
event.getQuery().getSegment().fulltext().putMetadata(urlEntry);
|
||||
ResultURLs.stack(
|
||||
urlEntry,
|
||||
mySeed.hash.getBytes(),
|
||||
event.peers.mySeed().hash.getBytes(),
|
||||
UTF8.getBytes(target.hash),
|
||||
EventOrigin.QUERIES);
|
||||
} catch ( final IOException e ) {
|
||||
|
|
@ -792,14 +776,14 @@ public final class Protocol
|
|||
// store remote result to local result container
|
||||
// insert one container into the search result buffer
|
||||
// one is enough, only the references are used, not the word
|
||||
containerCache.add(container.get(0), false, target.getName() + "/" + target.hash, result.joincount, time);
|
||||
containerCache.addFinalize();
|
||||
containerCache.addExpectedRemoteReferences(-count);
|
||||
event.rankingProcess.add(container.get(0), false, target.getName() + "/" + target.hash, result.joincount, time);
|
||||
event.rankingProcess.addFinalize();
|
||||
event.rankingProcess.addExpectedRemoteReferences(-count);
|
||||
|
||||
// insert the containers to the index
|
||||
for ( final ReferenceContainer<WordReference> c : container ) {
|
||||
try {
|
||||
indexSegment.termIndex().add(c);
|
||||
event.getQuery().getSegment().termIndex().add(c);
|
||||
} catch ( final Exception e ) {
|
||||
Log.logException(e);
|
||||
}
|
||||
|
|
@ -822,9 +806,9 @@ public final class Protocol
|
|||
+ result.references.length
|
||||
+ " topics");
|
||||
// add references twice, so they can be counted (must have at least 2 entries)
|
||||
synchronized ( containerCache ) {
|
||||
containerCache.addTopic(result.references);
|
||||
containerCache.addTopic(result.references);
|
||||
synchronized ( event.rankingProcess ) {
|
||||
event.rankingProcess.addTopic(result.references);
|
||||
event.rankingProcess.addTopic(result.references);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -895,13 +879,11 @@ public final class Protocol
|
|||
public Map<byte[], String> indexabstract; // index abstracts, a collection of url-hashes per word
|
||||
|
||||
public SearchResult(
|
||||
final SearchEvent event,
|
||||
final Map<String, ContentBody> parts,
|
||||
final Seed mySeed,
|
||||
final String wordhashes,
|
||||
final String excludehashes,
|
||||
final String urlhashes,
|
||||
final Pattern prefer,
|
||||
final Pattern filter,
|
||||
final String modifier,
|
||||
final String language,
|
||||
final String sitehash,
|
||||
|
|
@ -914,9 +896,8 @@ public final class Protocol
|
|||
final int partitions,
|
||||
final String hostname,
|
||||
final String hostaddress,
|
||||
final SearchEvent.SecondarySearchSuperviser secondarySearchSuperviser,
|
||||
final RankingProfile rankingProfile,
|
||||
final Bitfield constraint) throws IOException {
|
||||
final SearchEvent.SecondarySearchSuperviser secondarySearchSuperviser
|
||||
) throws IOException {
|
||||
// send a search request to peer with remote Hash
|
||||
|
||||
//if (hostaddress.equals(mySeed.getClusterAddress())) hostaddress = "127.0.0.1:" + mySeed.getPort(); // for debugging
|
||||
|
|
@ -943,7 +924,7 @@ public final class Protocol
|
|||
keyBody.writeTo(baos);
|
||||
key = baos.toString();
|
||||
}
|
||||
parts.put("myseed", UTF8.StringBody((mySeed == null) ? "" : mySeed.genSeedStr(key)));
|
||||
parts.put("myseed", UTF8.StringBody((event.peers.mySeed() == null) ? "" : event.peers.mySeed().genSeedStr(key)));
|
||||
parts.put("count", UTF8.StringBody(Integer.toString(Math.max(10, count))));
|
||||
parts.put("time", UTF8.StringBody(Long.toString(Math.max(3000, time))));
|
||||
parts.put("resource", UTF8.StringBody(((global) ? "global" : "local")));
|
||||
|
|
@ -952,8 +933,8 @@ public final class Protocol
|
|||
parts.put("exclude", UTF8.StringBody(excludehashes));
|
||||
parts.put("duetime", UTF8.StringBody("1000"));
|
||||
parts.put("urls", UTF8.StringBody(urlhashes));
|
||||
parts.put("prefer", UTF8.StringBody(prefer.pattern()));
|
||||
parts.put("filter", UTF8.StringBody(filter.pattern()));
|
||||
parts.put("prefer", UTF8.StringBody(event.getQuery().prefer.pattern()));
|
||||
parts.put("filter", UTF8.StringBody(event.getQuery().urlMask.pattern()));
|
||||
parts.put("modifier", UTF8.StringBody(modifier));
|
||||
parts.put("language", UTF8.StringBody(language));
|
||||
parts.put("sitehash", UTF8.StringBody(sitehash));
|
||||
|
|
@ -961,8 +942,8 @@ public final class Protocol
|
|||
parts.put("contentdom", UTF8.StringBody(contentdom));
|
||||
parts.put("ttl", UTF8.StringBody("0"));
|
||||
parts.put("maxdist", UTF8.StringBody(Integer.toString(maxDistance)));
|
||||
parts.put("profile", UTF8.StringBody(crypt.simpleEncode(rankingProfile.toExternalString())));
|
||||
parts.put("constraint", UTF8.StringBody((constraint == null) ? "" : constraint.exportB64()));
|
||||
parts.put("profile", UTF8.StringBody(crypt.simpleEncode(event.getQuery().ranking.toExternalString())));
|
||||
parts.put("constraint", UTF8.StringBody((event.getQuery().constraint == null) ? "" : event.getQuery().constraint.exportB64()));
|
||||
if ( secondarySearchSuperviser != null ) {
|
||||
parts.put("abstracts", UTF8.StringBody("auto"));
|
||||
// resultMap = FileUtils.table(HTTPConnector.getConnector(MultiProtocolURI.yacybotUserAgent).post(new MultiProtocolURI("http://" + hostaddress + "/yacy/search.html"), 60000, hostname, parts));
|
||||
|
|
@ -1040,14 +1021,14 @@ public final class Protocol
|
|||
try {
|
||||
solrConnector = new RemoteSolrConnector(solrURL);
|
||||
SolrDocumentList docList = solrConnector.query(querystring, offset, count);
|
||||
|
||||
|
||||
} catch (IOException e) {
|
||||
Log.logException(e);
|
||||
} finally {
|
||||
containerCache.oneFeederTerminated();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
public static Map<String, String> permissionMessage(final SeedDB seedDB, final String targetHash) {
|
||||
// ask for allowed message size and attachement size
|
||||
// if this replies null, the peer does not answer
|
||||
|
|
@ -1503,107 +1484,6 @@ public final class Protocol
|
|||
}
|
||||
}
|
||||
|
||||
public static void main(final String[] args) {
|
||||
if ( args.length > 2 ) {
|
||||
// search a remote peer. arguments:
|
||||
// first arg: path to application home
|
||||
// second arg: address of target peer
|
||||
// third arg: search word or file name with list of search words
|
||||
// i.e. /Data/workspace1/yacy/ localhost:8090 /Data/workspace1/yacy/test/words/searchtest.words
|
||||
System.out.println("yacyClient Test");
|
||||
final File searchwordfile = new File(args[2]);
|
||||
final List<String> searchlines = new ArrayList<String>();
|
||||
if ( searchwordfile.exists() ) {
|
||||
Iterator<String> i;
|
||||
try {
|
||||
i = FileUtils.strings(FileUtils.read(searchwordfile));
|
||||
while ( i.hasNext() ) {
|
||||
searchlines.add(i.next());
|
||||
}
|
||||
} catch ( final IOException e ) {
|
||||
e.printStackTrace();
|
||||
System.exit(-1);
|
||||
}
|
||||
} else {
|
||||
searchlines.add(args[2]);
|
||||
}
|
||||
for ( final String line : searchlines ) {
|
||||
final byte[] wordhashe =
|
||||
ASCII.getBytes(QueryParams.hashSet2hashString(Word.words2hashesHandles(QueryParams
|
||||
.cleanQuery(line)[0])));
|
||||
final long time = System.currentTimeMillis();
|
||||
SearchResult result;
|
||||
try {
|
||||
result =
|
||||
new SearchResult(basicRequestParts((String) null, (String) null, "freeworld"), null, // sb.peers.mySeed(),
|
||||
ASCII.String(wordhashe),
|
||||
"", // excludehashes,
|
||||
"", // urlhashes,
|
||||
QueryParams.matchnothing_pattern, // prefer,
|
||||
QueryParams.catchall_pattern, // filter,
|
||||
"", // modifier
|
||||
"", // language,
|
||||
"", // sitehash,
|
||||
"", // authorhash,
|
||||
"all", // contentdom,
|
||||
10, // count,
|
||||
3000, // time,
|
||||
1000, // maxDistance,
|
||||
true, //global,
|
||||
16, // partitions,
|
||||
"",
|
||||
args[1],
|
||||
null, //secondarySearchSuperviser,
|
||||
new RankingProfile(Classification.ContentDomain.TEXT), // rankingProfile,
|
||||
null // constraint);
|
||||
);
|
||||
for ( final URIMetadata link : result.links ) {
|
||||
System.out.println(link.url().toNormalform(true, false));
|
||||
System.out.println(link.snippet());
|
||||
}
|
||||
} catch ( final IOException e ) {
|
||||
// TODO Auto-generated catch block
|
||||
e.printStackTrace();
|
||||
}
|
||||
System.out.println("Search Time: " + (System.currentTimeMillis() - time));
|
||||
}
|
||||
System.exit(0);
|
||||
} else if ( args.length == 1 ) {
|
||||
System.out.println("wput Test");
|
||||
// connection params
|
||||
MultiProtocolURI url = null;
|
||||
try {
|
||||
url = new MultiProtocolURI(args[0]);
|
||||
} catch ( final MalformedURLException e ) {
|
||||
Log.logException(e);
|
||||
}
|
||||
if ( url == null ) {
|
||||
System.exit(1);
|
||||
return;
|
||||
}
|
||||
final String vhost = url.getHost();
|
||||
final int timeout = 10000;
|
||||
// new data
|
||||
final Map<String, ContentBody> newpost = new LinkedHashMap<String, ContentBody>();
|
||||
newpost.put("process", UTF8.StringBody("permission"));
|
||||
newpost.put("purpose", UTF8.StringBody("crcon"));
|
||||
byte[] res;
|
||||
try {
|
||||
// res = HTTPConnector.getConnector(MultiProtocolURI.yacybotUserAgent).post(url, timeout, vhost, newpost, true);
|
||||
final HTTPClient httpClient = new HTTPClient(ClientIdentification.getUserAgent(), timeout);
|
||||
res = httpClient.POSTbytes(url, vhost, newpost, true);
|
||||
System.out.println(UTF8.String(res));
|
||||
} catch ( final IOException e1 ) {
|
||||
Log.logException(e1);
|
||||
}
|
||||
}
|
||||
try {
|
||||
net.yacy.cora.protocol.http.HTTPClient.closeConnectionManager();
|
||||
} catch ( final InterruptedException e ) {
|
||||
Log.logException(e);
|
||||
}
|
||||
}
|
||||
|
||||
public static final boolean authentifyRequest(final serverObjects post, final serverSwitch env) {
|
||||
if ( post == null || env == null ) {
|
||||
return false;
|
||||
|
|
|
|||
|
|
@ -27,19 +27,14 @@ package net.yacy.peers;
|
|||
import java.util.Iterator;
|
||||
import java.util.Set;
|
||||
import java.util.SortedMap;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
import net.yacy.cora.document.ASCII;
|
||||
import net.yacy.cora.storage.HandleSet;
|
||||
import net.yacy.kelondro.logging.Log;
|
||||
import net.yacy.kelondro.order.Bitfield;
|
||||
import net.yacy.peers.dht.PeerSelection;
|
||||
import net.yacy.repository.Blacklist;
|
||||
import net.yacy.search.index.Segment;
|
||||
import net.yacy.search.query.QueryParams;
|
||||
import net.yacy.search.query.RWIProcess;
|
||||
import net.yacy.search.query.SearchEvent;
|
||||
import net.yacy.search.ranking.RankingProfile;
|
||||
|
||||
|
||||
public class RemoteSearch extends Thread {
|
||||
|
|
@ -50,49 +45,37 @@ public class RemoteSearch extends Thread {
|
|||
final private String wordhashes, excludehashes, urlhashes, sitehash, authorhash, contentdom;
|
||||
final private boolean global;
|
||||
final private int partitions;
|
||||
final private Segment indexSegment;
|
||||
final private RWIProcess containerCache;
|
||||
final private SearchEvent.SecondarySearchSuperviser secondarySearchSuperviser;
|
||||
final private Blacklist blacklist;
|
||||
final private Seed targetPeer;
|
||||
private int urls;
|
||||
private final int count, maxDistance;
|
||||
private final long time;
|
||||
final private RankingProfile rankingProfile;
|
||||
final private Pattern prefer, filter;
|
||||
final private QueryParams.Modifier modifier;
|
||||
final private String language;
|
||||
final private Bitfield constraint;
|
||||
final private SeedDB peers;
|
||||
|
||||
public RemoteSearch(
|
||||
final SearchEvent event,
|
||||
final String wordhashes, final String excludehashes,
|
||||
final String urlhashes, // this is the field that is filled during a secondary search to restrict to specific urls that are to be retrieved
|
||||
final Pattern prefer,
|
||||
final Pattern filter,
|
||||
final QueryParams.Modifier modifier,
|
||||
final String language,
|
||||
final String sitehash, final String authorhash, final String contentdom,
|
||||
final int count, final long time, final int maxDistance,
|
||||
final boolean global, final int partitions,
|
||||
final String sitehash,
|
||||
final String authorhash,
|
||||
final String contentdom,
|
||||
final int count,
|
||||
final long time,
|
||||
final int maxDistance,
|
||||
final boolean global,
|
||||
final int partitions,
|
||||
final Seed targetPeer,
|
||||
final Segment indexSegment,
|
||||
final SeedDB peers,
|
||||
final RWIProcess containerCache,
|
||||
final SearchEvent.SecondarySearchSuperviser secondarySearchSuperviser,
|
||||
final Blacklist blacklist,
|
||||
final RankingProfile rankingProfile,
|
||||
final Bitfield constraint) {
|
||||
final Blacklist blacklist) {
|
||||
super(ysThreadGroup, "yacySearch_" + targetPeer.getName());
|
||||
//System.out.println("DEBUG - yacySearch thread " + this.getName() + " initialized " + ((urlhashes.isEmpty()) ? "(primary)" : "(secondary)"));
|
||||
assert wordhashes.length() >= 12;
|
||||
this.event = event;
|
||||
this.wordhashes = wordhashes;
|
||||
this.excludehashes = excludehashes;
|
||||
this.urlhashes = urlhashes;
|
||||
this.prefer = prefer;
|
||||
this.filter = filter;
|
||||
this.modifier = modifier;
|
||||
this.language = language;
|
||||
this.sitehash = sitehash;
|
||||
|
|
@ -100,9 +83,6 @@ public class RemoteSearch extends Thread {
|
|||
this.contentdom = contentdom;
|
||||
this.global = global;
|
||||
this.partitions = partitions;
|
||||
this.indexSegment = indexSegment;
|
||||
this.peers = peers;
|
||||
this.containerCache = containerCache;
|
||||
this.secondarySearchSuperviser = secondarySearchSuperviser;
|
||||
this.blacklist = blacklist;
|
||||
this.targetPeer = targetPeer;
|
||||
|
|
@ -110,34 +90,42 @@ public class RemoteSearch extends Thread {
|
|||
this.count = count;
|
||||
this.time = time;
|
||||
this.maxDistance = maxDistance;
|
||||
this.rankingProfile = rankingProfile;
|
||||
this.constraint = constraint;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void run() {
|
||||
this.containerCache.oneFeederStarted();
|
||||
this.event.rankingProcess.oneFeederStarted();
|
||||
try {
|
||||
this.urls = Protocol.search(
|
||||
this.peers.mySeed(),
|
||||
this.wordhashes, this.excludehashes, this.urlhashes,
|
||||
this.prefer, this.filter, this.modifier.getModifier(),
|
||||
this.language, this.sitehash, this.authorhash, this.contentdom,
|
||||
this.count, this.time, this.maxDistance, this.global, this.partitions,
|
||||
this.targetPeer, this.indexSegment, this.containerCache, this.secondarySearchSuperviser,
|
||||
this.blacklist, this.rankingProfile, this.constraint);
|
||||
this.event,
|
||||
this.wordhashes,
|
||||
this.excludehashes,
|
||||
this.urlhashes,
|
||||
this.modifier.getModifier(),
|
||||
this.language,
|
||||
this.sitehash,
|
||||
this.authorhash,
|
||||
this.contentdom,
|
||||
this.count,
|
||||
this.time,
|
||||
this.maxDistance,
|
||||
this.global,
|
||||
this.partitions,
|
||||
this.targetPeer,
|
||||
this.secondarySearchSuperviser,
|
||||
this.blacklist);
|
||||
if (this.urls >= 0) {
|
||||
// urls is an array of url hashes. this is only used for log output
|
||||
if (this.urlhashes != null && this.urlhashes.length() > 0) Network.log.logInfo("SECONDARY REMOTE SEARCH - remote peer " + this.targetPeer.hash + ":" + this.targetPeer.getName() + " contributed " + this.urls + " links for word hash " + this.wordhashes);
|
||||
this.peers.mySeed().incRI(this.urls);
|
||||
this.peers.mySeed().incRU(this.urls);
|
||||
this.event.peers.mySeed().incRI(this.urls);
|
||||
this.event.peers.mySeed().incRU(this.urls);
|
||||
} else {
|
||||
Network.log.logInfo("REMOTE SEARCH - no answer from remote peer " + this.targetPeer.hash + ":" + this.targetPeer.getName());
|
||||
}
|
||||
} catch (final Exception e) {
|
||||
Log.logException(e);
|
||||
} finally {
|
||||
this.containerCache.oneFeederTerminated();
|
||||
this.event.rankingProcess.oneFeederTerminated();
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -162,21 +150,8 @@ public class RemoteSearch extends Thread {
|
|||
|
||||
public static void primaryRemoteSearches(
|
||||
final SearchEvent event,
|
||||
final String wordhashes, final String excludehashes,
|
||||
final Pattern prefer, final Pattern filter,
|
||||
final QueryParams.Modifier modifier,
|
||||
final String language,
|
||||
final String sitehash,
|
||||
final String authorhash,
|
||||
final String contentdom,
|
||||
final int count, final long time, final int maxDist,
|
||||
final Segment indexSegment,
|
||||
final SeedDB peers,
|
||||
final RWIProcess containerCache,
|
||||
final SearchEvent.SecondarySearchSuperviser secondarySearchSuperviser,
|
||||
final int count, final long time,
|
||||
final Blacklist blacklist,
|
||||
final RankingProfile rankingProfile,
|
||||
final Bitfield constraint,
|
||||
final SortedMap<byte[], String> clusterselection,
|
||||
final int burstRobinsonPercent,
|
||||
final int burstMultiwordPercent) {
|
||||
|
|
@ -184,17 +159,15 @@ public class RemoteSearch extends Thread {
|
|||
//if (wordIndex.seedDB.mySeed() == null || wordIndex.seedDB.mySeed().getPublicAddress() == null) { return null; }
|
||||
|
||||
// prepare seed targets and threads
|
||||
assert language != null;
|
||||
assert wordhashes.length() >= 12 : "wordhashes = " + wordhashes;
|
||||
final Seed[] targetPeers =
|
||||
(clusterselection == null) ?
|
||||
PeerSelection.selectSearchTargets(
|
||||
peers,
|
||||
QueryParams.hashes2Set(wordhashes),
|
||||
peers.redundancy(),
|
||||
event.peers,
|
||||
event.getQuery().query_include_hashes,
|
||||
event.peers.redundancy(),
|
||||
burstRobinsonPercent,
|
||||
burstMultiwordPercent)
|
||||
: PeerSelection.selectClusterPeers(peers, clusterselection);
|
||||
: PeerSelection.selectClusterPeers(event.peers, clusterselection);
|
||||
if (targetPeers == null) return;
|
||||
final int targets = targetPeers.length;
|
||||
if (targets == 0) return;
|
||||
|
|
@ -203,10 +176,22 @@ public class RemoteSearch extends Thread {
|
|||
try {
|
||||
RemoteSearch rs = new RemoteSearch(
|
||||
event,
|
||||
wordhashes, excludehashes, "", prefer, filter, modifier,
|
||||
language, sitehash, authorhash, contentdom,
|
||||
count, time, maxDist, true, targets, targetPeers[i],
|
||||
indexSegment, peers, containerCache, secondarySearchSuperviser, blacklist, rankingProfile, constraint);
|
||||
QueryParams.hashSet2hashString(event.getQuery().query_include_hashes),
|
||||
QueryParams.hashSet2hashString(event.getQuery().query_exclude_hashes),
|
||||
"",
|
||||
event.getQuery().modifier,
|
||||
event.getQuery().targetlang == null ? "" : event.getQuery().targetlang,
|
||||
event.getQuery().sitehash == null ? "" : event.getQuery().sitehash,
|
||||
event.getQuery().authorhash == null ? "" : event.getQuery().authorhash,
|
||||
event.getQuery().contentdom == null ? "all" : event.getQuery().contentdom.toString(),
|
||||
count,
|
||||
time,
|
||||
event.getQuery().maxDistance,
|
||||
true,
|
||||
targets,
|
||||
targetPeers[i],
|
||||
event.secondarySearchSuperviser,
|
||||
blacklist);
|
||||
rs.start();
|
||||
event.primarySearchThreadsL.add(rs);
|
||||
} catch (final OutOfMemoryError e) {
|
||||
|
|
@ -215,33 +200,43 @@ public class RemoteSearch extends Thread {
|
|||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
public static RemoteSearch secondaryRemoteSearch(
|
||||
final SearchEvent event,
|
||||
final Set<String> wordhashes, final String urlhashes,
|
||||
final Set<String> wordhashes,
|
||||
final String urlhashes,
|
||||
final long time,
|
||||
final Segment indexSegment,
|
||||
final SeedDB peers,
|
||||
final RWIProcess containerCache,
|
||||
final String targethash, final Blacklist blacklist,
|
||||
final RankingProfile rankingProfile,
|
||||
final Bitfield constraint, final SortedMap<byte[], String> clusterselection) {
|
||||
final String targethash,
|
||||
final Blacklist blacklist) {
|
||||
|
||||
// check own peer status
|
||||
if (peers.mySeed() == null || peers.mySeed().getPublicAddress() == null) { return null; }
|
||||
if (event.peers.mySeed() == null || event.peers.mySeed().getPublicAddress() == null) { return null; }
|
||||
assert urlhashes != null;
|
||||
assert urlhashes.length() > 0;
|
||||
|
||||
// prepare seed targets and threads
|
||||
final Seed targetPeer = peers.getConnected(targethash);
|
||||
final Seed targetPeer = event.peers.getConnected(targethash);
|
||||
if (targetPeer == null || targetPeer.hash == null) return null;
|
||||
if (clusterselection != null) targetPeer.setAlternativeAddress(clusterselection.get(ASCII.getBytes(targetPeer.hash)));
|
||||
StringBuilder whs = new StringBuilder(24);
|
||||
for (String s: wordhashes) whs.append(s);
|
||||
if (event.preselectedPeerHashes != null) targetPeer.setAlternativeAddress(event.preselectedPeerHashes.get(ASCII.getBytes(targetPeer.hash)));
|
||||
|
||||
final RemoteSearch searchThread = new RemoteSearch(
|
||||
event,
|
||||
whs.toString(), "", urlhashes, QueryParams.matchnothing_pattern, QueryParams.catchall_pattern, new QueryParams.Modifier(""), "", "", "", "all", 20, time, 9999, true, 0, targetPeer,
|
||||
indexSegment, peers, containerCache, null, blacklist, rankingProfile, constraint);
|
||||
QueryParams.hashSet2hashString(wordhashes),
|
||||
"",
|
||||
urlhashes,
|
||||
new QueryParams.Modifier(""),
|
||||
"",
|
||||
"",
|
||||
"",
|
||||
"all",
|
||||
20,
|
||||
time,
|
||||
9999,
|
||||
true,
|
||||
0,
|
||||
targetPeer,
|
||||
null,
|
||||
blacklist);
|
||||
searchThread.start();
|
||||
return searchThread;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -367,6 +367,17 @@ public final class QueryParams {
|
|||
return ASCII.String(bb);
|
||||
}
|
||||
|
||||
public static String hashSet2hashString(final Set<String> hashes) {
|
||||
final byte[] bb = new byte[hashes.size() * Word.commonHashLength];
|
||||
int p = 0;
|
||||
for (final String s : hashes) {
|
||||
assert s.length() == Word.commonHashLength : "hash = " + s;
|
||||
System.arraycopy(ASCII.getBytes(s), 0, bb, p, Word.commonHashLength);
|
||||
p += Word.commonHashLength;
|
||||
}
|
||||
return ASCII.String(bb);
|
||||
}
|
||||
|
||||
public static String anonymizedQueryHashes(final HandleSet hashes) {
|
||||
// create a more anonymized representation of a query hashes for logging
|
||||
final Iterator<byte[]> i = hashes.iterator();
|
||||
|
|
|
|||
|
|
@ -86,17 +86,17 @@ public final class SearchEvent {
|
|||
// class variables that may be implemented with an abstract class
|
||||
private long eventTime;
|
||||
private QueryParams query;
|
||||
private final SeedDB peers;
|
||||
public final SeedDB peers;
|
||||
private final WorkTables workTables;
|
||||
private final RWIProcess rankingProcess; // ordered search results, grows dynamically as all the query threads enrich this container
|
||||
public final RWIProcess rankingProcess; // ordered search results, grows dynamically as all the query threads enrich this container
|
||||
private final SnippetProcess resultFetcher;
|
||||
|
||||
private final SecondarySearchSuperviser secondarySearchSuperviser;
|
||||
public final SecondarySearchSuperviser secondarySearchSuperviser;
|
||||
|
||||
// class variables for remote searches
|
||||
public final List<RemoteSearch> primarySearchThreadsL;
|
||||
private RemoteSearch[] secondarySearchThreads;
|
||||
private final SortedMap<byte[], String> preselectedPeerHashes;
|
||||
public final SortedMap<byte[], String> preselectedPeerHashes;
|
||||
private final Thread localSearchThread;
|
||||
private final SortedMap<byte[], Integer> IACount;
|
||||
private final SortedMap<byte[], String> IAResults;
|
||||
|
|
@ -167,25 +167,9 @@ public final class SearchEvent {
|
|||
Thread.currentThread().setName("SearchEvent.primaryRemoteSearches");
|
||||
RemoteSearch.primaryRemoteSearches(
|
||||
SearchEvent.this,
|
||||
QueryParams.hashSet2hashString(SearchEvent.this.query.query_include_hashes),
|
||||
QueryParams.hashSet2hashString(SearchEvent.this.query.query_exclude_hashes),
|
||||
SearchEvent.this.query.prefer,
|
||||
SearchEvent.this.query.urlMask,
|
||||
SearchEvent.this.query.modifier,
|
||||
SearchEvent.this.query.targetlang == null ? "" : SearchEvent.this.query.targetlang,
|
||||
SearchEvent.this.query.sitehash == null ? "" : SearchEvent.this.query.sitehash,
|
||||
SearchEvent.this.query.authorhash == null ? "" : SearchEvent.this.query.authorhash,
|
||||
SearchEvent.this.query.contentdom == null ? "all" : SearchEvent.this.query.contentdom.toString(),
|
||||
remote_maxcount,
|
||||
remote_maxtime,
|
||||
SearchEvent.this.query.maxDistance,
|
||||
SearchEvent.this.query.getSegment(),
|
||||
peers,
|
||||
SearchEvent.this.rankingProcess,
|
||||
SearchEvent.this.secondarySearchSuperviser,
|
||||
Switchboard.urlBlacklist,
|
||||
SearchEvent.this.query.ranking,
|
||||
SearchEvent.this.query.constraint,
|
||||
(SearchEvent.this.query.domType == QueryParams.Searchdom.GLOBAL) ? null : preselectedPeerHashes,
|
||||
burstRobinsonPercent,
|
||||
burstMultiwordPercent);
|
||||
|
|
@ -303,7 +287,7 @@ public final class SearchEvent {
|
|||
}
|
||||
SearchEventCache.put(this.query.id(false), this);
|
||||
}
|
||||
|
||||
|
||||
public ReferenceOrder getOrder() {
|
||||
return this.order;
|
||||
}
|
||||
|
|
@ -694,14 +678,8 @@ public final class SearchEvent {
|
|||
words,
|
||||
urls.toString(),
|
||||
6000,
|
||||
SearchEvent.this.query.getSegment(),
|
||||
SearchEvent.this.peers,
|
||||
SearchEvent.this.rankingProcess,
|
||||
peer,
|
||||
Switchboard.urlBlacklist,
|
||||
SearchEvent.this.query.ranking,
|
||||
SearchEvent.this.query.constraint,
|
||||
SearchEvent.this.preselectedPeerHashes);
|
||||
Switchboard.urlBlacklist);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue
Block a user