diff --git a/source/de/anomic/search/RankingProcess.java b/source/de/anomic/search/RankingProcess.java index 367c83425..90ac93475 100644 --- a/source/de/anomic/search/RankingProcess.java +++ b/source/de/anomic/search/RankingProcess.java @@ -91,11 +91,13 @@ public final class RankingProcess extends Thread { private final ScoreMap namespaceNavigator; private final ReferenceOrder order; private final long startTime; + private boolean addRunning; public RankingProcess(final QueryParams query, final ReferenceOrder order, final int maxentries) { // we collect the urlhashes and construct a list with urlEntry objects // attention: if minEntries is too high, this method will not terminate within the maxTime // sortorder: 0 = hash, 1 = url, 2 = ranking + this.addRunning = true; this.localSearchInclusion = null; this.stack = new WeakPriorityBlockingQueue(maxentries); this.doubleDomCache = new ConcurrentHashMap>(); @@ -150,7 +152,7 @@ public final class RankingProcess extends Thread { return; } - add(index, true, "local index: " + this.query.getSegment().getLocation(), -1); + add(index, true, "local index: " + this.query.getSegment().getLocation(), -1, true); } catch (final Exception e) { Log.logException(e); } finally { @@ -158,10 +160,17 @@ public final class RankingProcess extends Thread { } } - public void add(final ReferenceContainer index, final boolean local, String resourceName, final int fullResource) { + public void add( + final ReferenceContainer index, + final boolean local, + String resourceName, + final int fullResource, + final boolean finalizeAddAtEnd) { // we collect the urlhashes and construct a list with urlEntry objects // attention: if minEntries is too high, this method will not terminate within the maxTime + this.addRunning = true; + assert (index != null); if (index.isEmpty()) return; @@ -245,7 +254,9 @@ public final class RankingProcess extends Thread { } } - } catch (InterruptedException e) {} + } catch (InterruptedException e) {} finally { + if (finalizeAddAtEnd) this.addRunning = false; + } //if ((query.neededResults() > 0) && (container.size() > query.neededResults())) remove(true, true); EventTracker.update(EventTracker.EClass.SEARCH, new ProfilingGraph.searchEvent(query.id(true), SearchEvent.Type.PRESORT, resourceName, index.size(), System.currentTimeMillis() - timer), false); @@ -297,11 +308,12 @@ public final class RankingProcess extends Thread { WeakPriorityBlockingQueue.Element rwi = null; // take one entry from the stack if there are entries on that stack or the feeding is not yet finished - if (!feedingIsFinished() || stack.sizeQueue() > 0) try { + try { //System.out.println("stack.poll: feeders = " + this.feeders + ", stack.sizeQueue = " + stack.sizeQueue()); int loops = 0; // a loop counter to terminate the reading if all the results are from the same domain long timeout = System.currentTimeMillis() + waitingtime; - while (this.query.itemsPerPage < 1 || loops++ < this.query.itemsPerPage) { + while (((!feedingIsFinished() && this.addRunning) || stack.sizeQueue() > 0) && + (this.query.itemsPerPage < 1 || loops++ < this.query.itemsPerPage)) { if (waitingtime <= 0) { rwi = stack.poll(); } else timeoutloop:while (System.currentTimeMillis() < timeout) { diff --git a/source/de/anomic/search/ResultFetcher.java b/source/de/anomic/search/ResultFetcher.java index 201fbb26a..7fb8910fe 100644 --- a/source/de/anomic/search/ResultFetcher.java +++ b/source/de/anomic/search/ResultFetcher.java @@ -319,13 +319,13 @@ public class ResultFetcher { // check if we have enough if (result.sizeAvailable() >= this.neededResults) { - System.out.println(result.sizeAvailable() + " = result.sizeAvailable() >= this.neededResults = " + this.neededResults); + Log.logWarning("ResultFetcher", result.sizeAvailable() + " = result.sizeAvailable() >= this.neededResults = " + this.neededResults); break; } // check if we can succeed if we try to take another url if (rankingProcess.feedingIsFinished() && rankingProcess.sizeQueue() == 0) { - System.out.println("rankingProcess.feedingIsFinished() && rankingProcess.sizeQueue() == 0"); + Log.logWarning("ResultFetcher", "rankingProcess.feedingIsFinished() && rankingProcess.sizeQueue() == 0"); break; } diff --git a/source/de/anomic/search/Segment.java b/source/de/anomic/search/Segment.java index 915d9496d..35aff4bc7 100644 --- a/source/de/anomic/search/Segment.java +++ b/source/de/anomic/search/Segment.java @@ -247,7 +247,7 @@ public class Segment { try { container = ReferenceContainer.emptyContainer(Segment.wordReferenceFactory, wordhash, 1); container.add(ientry); - rankingProcess.add(container, true, sourceName, -1); + rankingProcess.add(container, true, sourceName, -1, !i.hasNext()); } catch (RowSpaceExceededException e) { continue; } diff --git a/source/de/anomic/yacy/yacyClient.java b/source/de/anomic/yacy/yacyClient.java index 8d0e57b37..1d65b2d86 100644 --- a/source/de/anomic/yacy/yacyClient.java +++ b/source/de/anomic/yacy/yacyClient.java @@ -506,7 +506,7 @@ public final class yacyClient { // store remote result to local result container // insert one container into the search result buffer // one is enough, only the references are used, not the word - containerCache.add(container[0], false, target.getName() + "/" + target.hash, result.joincount); + containerCache.add(container[0], false, target.getName() + "/" + target.hash, result.joincount, true); // insert the containers to the index for (ReferenceContainer c: container) try { diff --git a/source/net/yacy/cora/protocol/Domains.java b/source/net/yacy/cora/protocol/Domains.java index 1734a61ad..0b752bf8d 100644 --- a/source/net/yacy/cora/protocol/Domains.java +++ b/source/net/yacy/cora/protocol/Domains.java @@ -813,10 +813,15 @@ public class Domains { System.err.println(e); } */ + InetAddress a; + a = dnsResolve("yacy.net"); System.out.println(a); + a = dnsResolve("kaskelix.de"); System.out.println(a); + a = dnsResolve("yacy.net"); System.out.println(a); + try { Thread.sleep(1000);} catch (InterruptedException e) {} // get time for class init System.out.println("myPublicLocalIP: " + myPublicLocalIP()); - for (final InetAddress a : myIntranetIPs()) { - System.out.println("Intranet IP: " + a); + for (final InetAddress b : myIntranetIPs()) { + System.out.println("Intranet IP: " + b); } } } diff --git a/source/net/yacy/document/parser/html/ContentScraper.java b/source/net/yacy/document/parser/html/ContentScraper.java index ea4543221..5a5ff8044 100644 --- a/source/net/yacy/document/parser/html/ContentScraper.java +++ b/source/net/yacy/document/parser/html/ContentScraper.java @@ -39,6 +39,7 @@ import java.util.List; import java.util.Map; import java.util.Properties; import java.util.Set; +import java.util.regex.Pattern; import javax.swing.event.EventListenerList; @@ -425,6 +426,9 @@ public class ContentScraper extends AbstractScraper implements Scraper { if (s == null) return ""; return s; } + + private final static Pattern commaSepPattern = Pattern.compile(" |,"); + private final static Pattern semicSepPattern = Pattern.compile(" |;"); public Set getContentLanguages() { // i.e. @@ -433,7 +437,7 @@ public class ContentScraper extends AbstractScraper implements Scraper { if (s == null) s = metas.get("dc.language"); if (s == null) return null; Set hs = new HashSet(); - String[] cl = s.split(" |,"); + String[] cl = commaSepPattern.split(s); int p; for (int i = 0; i < cl.length; i++) { cl[i] = cl[i].toLowerCase(); @@ -452,8 +456,8 @@ public class ContentScraper extends AbstractScraper implements Scraper { if (s.length() == 0) { return MultiProtocolURI.splitpattern.split(getTitle().toLowerCase()); } - if (s.contains(",")) return s.split(" |,"); - if (s.contains(";")) return s.split(" |;"); + if (s.contains(",")) return commaSepPattern.split(s); + if (s.contains(";")) return semicSepPattern.split(s); return s.split("\\s"); } diff --git a/source/net/yacy/kelondro/index/Row.java b/source/net/yacy/kelondro/index/Row.java index d7001ffd1..0091a6600 100644 --- a/source/net/yacy/kelondro/index/Row.java +++ b/source/net/yacy/kelondro/index/Row.java @@ -34,6 +34,7 @@ import java.util.Map; import java.util.StringTokenizer; import java.util.concurrent.ArrayBlockingQueue; import java.util.concurrent.ConcurrentHashMap; +import java.util.regex.Pattern; import net.yacy.cora.document.UTF8; import net.yacy.kelondro.logging.Log; @@ -48,7 +49,9 @@ import net.yacy.kelondro.util.kelondroException; public final class Row { - + + private final static Pattern commaPattern = Pattern.compile(","); + protected final Column[] row; public final int[] colstart; public final ByteOrder objectOrder; @@ -281,7 +284,7 @@ public final class Row { public Entry(String external, final boolean decimalCardinal) { // parse external form if (external.length() > 0 && external.charAt(0) == '{') external = external.substring(1, external.length() - 1); - final String[] elts = external.split(","); + final String[] elts = commaPattern.split(external); if (nickref == null) genNickRef(); String nick; int p;