Marown/yacy_search_server
1
0
Fork 0
mirror of https://github.com/yacy/yacy_search_server.git synced 2024-09-19 00:01:41 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity

enhancement during search process: fast fail of search in case that all index feeder have terminated.

Browse Source 
This change should affect filtering and navigators and should cause that search navigation gets faster

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@7614 6c8d7289-2bf4-0310-a012-ef5d649a1542
This commit is contained in:
orbiter 2011-03-21 13:05:51 +00:00
parent ba03ca8620
commit 78d4c45d09
7 changed files with 40 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
source/de/anomic/search/RankingProcess.java
View File

@ -91,11 +91,13 @@ public final class RankingProcess extends Thread {
private final ScoreMap<String> namespaceNavigator;
private final ReferenceOrder order;
private final long startTime;
private boolean addRunning;
public RankingProcess(final QueryParams query, final ReferenceOrder order, final int maxentries) {
// we collect the urlhashes and construct a list with urlEntry objects
// attention: if minEntries is too high, this method will not terminate within the maxTime
// sortorder: 0 = hash, 1 = url, 2 = ranking
this.addRunning = true;
this.localSearchInclusion = null;
this.stack = new WeakPriorityBlockingQueue<WordReferenceVars>(maxentries);
this.doubleDomCache = new ConcurrentHashMap<String, WeakPriorityBlockingQueue<WordReferenceVars>>();
@ -150,7 +152,7 @@ public final class RankingProcess extends Thread {
return;
}
add(index, true, "local index: " + this.query.getSegment().getLocation(), -1);
add(index, true, "local index: " + this.query.getSegment().getLocation(), -1, true);
} catch (final Exception e) {
Log.logException(e);
} finally {
@ -158,10 +160,17 @@ public final class RankingProcess extends Thread {
}
}
public void add(final ReferenceContainer<WordReference> index, final boolean local, String resourceName, final int fullResource) {
public void add(
final ReferenceContainer<WordReference> index,
final boolean local,
String resourceName,
final int fullResource,
final boolean finalizeAddAtEnd) {
// we collect the urlhashes and construct a list with urlEntry objects
// attention: if minEntries is too high, this method will not terminate within the maxTime
this.addRunning = true;
assert (index != null);
if (index.isEmpty()) return;
@ -245,7 +254,9 @@ public final class RankingProcess extends Thread {
}
}
} catch (InterruptedException e) {}
} catch (InterruptedException e) {} finally {
if (finalizeAddAtEnd) this.addRunning = false;
}
//if ((query.neededResults() > 0) && (container.size() > query.neededResults())) remove(true, true);
EventTracker.update(EventTracker.EClass.SEARCH, new ProfilingGraph.searchEvent(query.id(true), SearchEvent.Type.PRESORT, resourceName, index.size(), System.currentTimeMillis() - timer), false);
@ -297,11 +308,12 @@ public final class RankingProcess extends Thread {
WeakPriorityBlockingQueue.Element<WordReferenceVars> rwi = null;
// take one entry from the stack if there are entries on that stack or the feeding is not yet finished
if (!feedingIsFinished() || stack.sizeQueue() > 0) try {
try {
//System.out.println("stack.poll: feeders = " + this.feeders + ", stack.sizeQueue = " + stack.sizeQueue());
int loops = 0; // a loop counter to terminate the reading if all the results are from the same domain
long timeout = System.currentTimeMillis() + waitingtime;
while (this.query.itemsPerPage < 1 || loops++ < this.query.itemsPerPage) {
while (((!feedingIsFinished() && this.addRunning) || stack.sizeQueue() > 0) &&
(this.query.itemsPerPage < 1 || loops++ < this.query.itemsPerPage)) {
if (waitingtime <= 0) {
rwi = stack.poll();
} else timeoutloop:while (System.currentTimeMillis() < timeout) {

4
source/de/anomic/search/ResultFetcher.java
View File

@ -319,13 +319,13 @@ public class ResultFetcher {
// check if we have enough
if (result.sizeAvailable() >= this.neededResults) {
System.out.println(result.sizeAvailable() + " = result.sizeAvailable() >= this.neededResults = " + this.neededResults);
Log.logWarning("ResultFetcher", result.sizeAvailable() + " = result.sizeAvailable() >= this.neededResults = " + this.neededResults);
break;
}
// check if we can succeed if we try to take another url
if (rankingProcess.feedingIsFinished() && rankingProcess.sizeQueue() == 0) {
System.out.println("rankingProcess.feedingIsFinished() && rankingProcess.sizeQueue() == 0");
Log.logWarning("ResultFetcher", "rankingProcess.feedingIsFinished() && rankingProcess.sizeQueue() == 0");
break;
}

2
source/de/anomic/search/Segment.java
View File

@ -247,7 +247,7 @@ public class Segment {
try {
container = ReferenceContainer.emptyContainer(Segment.wordReferenceFactory, wordhash, 1);
container.add(ientry);
rankingProcess.add(container, true, sourceName, -1);
rankingProcess.add(container, true, sourceName, -1, !i.hasNext());
} catch (RowSpaceExceededException e) {
continue;
}

2
source/de/anomic/yacy/yacyClient.java
View File

@ -506,7 +506,7 @@ public final class yacyClient {
// store remote result to local result container
// insert one container into the search result buffer
// one is enough, only the references are used, not the word
containerCache.add(container[0], false, target.getName() + "/" + target.hash, result.joincount);
containerCache.add(container[0], false, target.getName() + "/" + target.hash, result.joincount, true);
// insert the containers to the index
for (ReferenceContainer<WordReference> c: container) try {

9
source/net/yacy/cora/protocol/Domains.java
View File

@ -813,10 +813,15 @@ public class Domains {
System.err.println(e);
}
*/
InetAddress a;
a = dnsResolve("yacy.net"); System.out.println(a);
a = dnsResolve("kaskelix.de"); System.out.println(a);
a = dnsResolve("yacy.net"); System.out.println(a);
try { Thread.sleep(1000);} catch (InterruptedException e) {} // get time for class init
System.out.println("myPublicLocalIP: " + myPublicLocalIP());
for (final InetAddress a : myIntranetIPs()) {
System.out.println("Intranet IP: " + a);
for (final InetAddress b : myIntranetIPs()) {
System.out.println("Intranet IP: " + b);
}
}
}

10
source/net/yacy/document/parser/html/ContentScraper.java
View File

@ -39,6 +39,7 @@ import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.Set;
import java.util.regex.Pattern;
import javax.swing.event.EventListenerList;
@ -425,6 +426,9 @@ public class ContentScraper extends AbstractScraper implements Scraper {
if (s == null) return "";
return s;
}
private final static Pattern commaSepPattern = Pattern.compile(" |,");
private final static Pattern semicSepPattern = Pattern.compile(" |;");
public Set<String> getContentLanguages() {
// i.e. <meta name="DC.language" content="en" scheme="DCTERMS.RFC3066">
@ -433,7 +437,7 @@ public class ContentScraper extends AbstractScraper implements Scraper {
if (s == null) s = metas.get("dc.language");
if (s == null) return null;
Set<String> hs = new HashSet<String>();
String[] cl = s.split(" |,");
String[] cl = commaSepPattern.split(s);
int p;
for (int i = 0; i < cl.length; i++) {
cl[i] = cl[i].toLowerCase();
@ -452,8 +456,8 @@ public class ContentScraper extends AbstractScraper implements Scraper {
if (s.length() == 0) {
return MultiProtocolURI.splitpattern.split(getTitle().toLowerCase());
}
if (s.contains(",")) return s.split(" |,");
if (s.contains(";")) return s.split(" |;");
if (s.contains(",")) return commaSepPattern.split(s);
if (s.contains(";")) return semicSepPattern.split(s);
return s.split("\\s");
}

7
source/net/yacy/kelondro/index/Row.java
View File

@ -34,6 +34,7 @@ import java.util.Map;
import java.util.StringTokenizer;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.ConcurrentHashMap;
import java.util.regex.Pattern;
import net.yacy.cora.document.UTF8;
import net.yacy.kelondro.logging.Log;
@ -48,7 +49,9 @@ import net.yacy.kelondro.util.kelondroException;
public final class Row {
private final static Pattern commaPattern = Pattern.compile(",");
protected final Column[] row;
public final int[] colstart;
public final ByteOrder objectOrder;
@ -281,7 +284,7 @@ public final class Row {
public Entry(String external, final boolean decimalCardinal) {
// parse external form
if (external.length() > 0 && external.charAt(0) == '{') external = external.substring(1, external.length() - 1);
final String[] elts = external.split(",");
final String[] elts = commaPattern.split(external);
if (nickref == null) genNickRef();
String nick;
int p;