diff --git a/htroot/ConfigPortal.java b/htroot/ConfigPortal.java index 2c3bffdb9..38a8595d7 100644 --- a/htroot/ConfigPortal.java +++ b/htroot/ConfigPortal.java @@ -1,4 +1,4 @@ -// ConfigPortal.java +// ConfigPortal.java // ----------------------- // part of YaCy // (C) by Michael Peter Christen; mc@yacy.net @@ -10,7 +10,7 @@ //$LastChangedBy$ // // LICENSE -// +// // This program is free software; you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by // the Free Software Foundation; either version 2 of the License, or @@ -38,14 +38,14 @@ public class ConfigPortal { public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) { final serverObjects prop = new serverObjects(); final Switchboard sb = (Switchboard) env; - + if (post != null) { // AUTHENTICATE if (!header.containsKey(RequestHeader.AUTHORIZATION)) { prop.putHTML("AUTHENTICATE","log-in"); return prop; } - + if (post.containsKey("popup")) { final String popup = post.get("popup", "status"); if ("front".equals(popup)) { @@ -62,7 +62,7 @@ public class ConfigPortal { final String newGreeting = post.get(SwitchboardConstants.GREETING, ""); // store this call as api call sb.tables.recordAPICall(post, "ConfigPortal.html", WorkTables.TABLE_API_TYPE_CONFIGURATION, "new portal design. greeting: " + newGreeting); - + sb.setConfig(SwitchboardConstants.GREETING, newGreeting); sb.setConfig(SwitchboardConstants.GREETING_HOMEPAGE, post.get(SwitchboardConstants.GREETING_HOMEPAGE, "")); sb.setConfig(SwitchboardConstants.GREETING_LARGE_IMAGE, post.get(SwitchboardConstants.GREETING_LARGE_IMAGE, "")); @@ -80,7 +80,7 @@ public class ConfigPortal { sb.setConfig("search.result.show.parser", post.getBoolean("search.result.show.parser", false)); sb.setConfig("search.result.show.pictures", post.getBoolean("search.result.show.pictures", false)); sb.setConfig("search.verify", post.get("search.verify", "ifexist")); - sb.setConfig("search.verify.delete", post.getBoolean("search.verify.delete", false)); + sb.setConfig(SwitchboardConstants.SEARCH_VERIFY_DELETE, post.getBoolean("search.verify.delete", false)); // construct navigation String String nav = ""; if (post.getBoolean("search.navigation.hosts", false)) nav += "hosts,"; @@ -109,8 +109,8 @@ public class ConfigPortal { sb.setConfig("search.result.show.parser", true); sb.setConfig("search.result.show.pictures", true); sb.setConfig("search.verify", "iffresh"); - sb.setConfig("search.verify.delete", "true"); - } + sb.setConfig(SwitchboardConstants.SEARCH_VERIFY_DELETE, "true"); + } } prop.putHTML(SwitchboardConstants.GREETING, sb.getConfig(SwitchboardConstants.GREETING, "")); @@ -121,7 +121,7 @@ public class ConfigPortal { prop.put("publicTopmenu", sb.getConfigBool("publicTopmenu", false) ? 1 : 0); prop.put("publicSearchpage", sb.getConfigBool("publicSearchpage", false) ? 1 : 0); prop.put("search.options", sb.getConfigBool("search.options", false) ? 1 : 0); - + prop.put("search.result.show.date", sb.getConfigBool("search.result.show.date", false) ? 1 : 0); prop.put("search.result.show.size", sb.getConfigBool("search.result.show.size", false) ? 1 : 0); prop.put("search.result.show.metadata", sb.getConfigBool("search.result.show.metadata", false) ? 1 : 0); @@ -138,7 +138,7 @@ public class ConfigPortal { prop.put("search.verify.ifexist", sb.getConfig("search.verify", "").equals("ifexist") ? 1 : 0); prop.put("search.verify.cacheonly", sb.getConfig("search.verify", "").equals("cacheonly") ? 1 : 0); prop.put("search.verify.false", sb.getConfig("search.verify", "").equals("false") ? 1 : 0); - prop.put("search.verify.delete", sb.getConfigBool("search.verify.delete", true) ? 1 : 0); + prop.put("search.verify.delete", sb.getConfigBool(SwitchboardConstants.SEARCH_VERIFY_DELETE, true) ? 1 : 0); final String browserPopUpPage = sb.getConfig(SwitchboardConstants.BROWSER_POP_UP_PAGE, "ConfigBasic.html"); prop.put("popupFront", 0); @@ -156,14 +156,14 @@ public class ConfigPortal { } prop.put("maximumRecords", sb.getConfigInt(SwitchboardConstants.SEARCH_ITEMS, 10)); - + final String target = sb.getConfig(SwitchboardConstants.SEARCH_TARGET, "_self"); prop.put("selected_blank", "_blank".equals(target) ? 1 : 0); prop.put("selected_self", "_self".equals(target) ? 1 : 0); prop.put("selected_parent", "_parent".equals(target) ? 1 : 0); prop.put("selected_top", "_top".equals(target) ? 1 : 0); prop.put("selected_searchresult", "searchresult".equals(target) ? 1 : 0); - + String myaddress = (sb.peers == null) ? null : sb.peers.mySeed() == null ? null : sb.peers.mySeed().getPublicAddress(); if (myaddress == null) { myaddress = "localhost:" + sb.getConfig("port", "8090"); diff --git a/htroot/yacysearch.java b/htroot/yacysearch.java index 056b3b1f6..f31f52e66 100644 --- a/htroot/yacysearch.java +++ b/htroot/yacysearch.java @@ -597,7 +597,7 @@ public class yacysearch { indexSegment, ranking, header.get(RequestHeader.USER_AGENT, ""), - sb.getConfigBool(SwitchboardConstants.NETWORK_SEARCHVERIFY, false) && sb.peers.mySeed().getFlagAcceptRemoteIndex()); + sb.getConfigBool(SwitchboardConstants.SEARCH_VERIFY_DELETE, false) && sb.getConfigBool(SwitchboardConstants.NETWORK_SEARCHVERIFY, false) && sb.peers.mySeed().getFlagAcceptRemoteIndex()); EventTracker.delete(EventTracker.EClass.SEARCH); EventTracker.update(EventTracker.EClass.SEARCH, new ProfilingGraph.EventSearch(theQuery.id(true), SearchEvent.Type.INITIALIZATION, "", 0, 0), false); diff --git a/source/net/yacy/cora/date/GenericFormatter.java b/source/net/yacy/cora/date/GenericFormatter.java index d79a10337..1c5c3e432 100644 --- a/source/net/yacy/cora/date/GenericFormatter.java +++ b/source/net/yacy/cora/date/GenericFormatter.java @@ -133,10 +133,7 @@ public class GenericFormatter extends AbstractFormatter implements DateFormatter if (UTCOffset == null || UTCOffset.length() == 0) { return new Date(); } try { return new Date(this.dateFormat.parse(timeString).getTime() - UTCDiff() + UTCDiff(UTCOffset)); - } catch (final java.text.ParseException e) { - //serverLog.logFinest("parseUniversalDate", e.getMessage() + ", remoteTimeString=[" + remoteTimeString + "]"); - return new Date(); - } catch (final java.lang.NumberFormatException e) { + } catch (final Throwable e) { //serverLog.logFinest("parseUniversalDate", e.getMessage() + ", remoteTimeString=[" + remoteTimeString + "]"); return new Date(); } diff --git a/source/net/yacy/document/parser/images/genericImageParser.java b/source/net/yacy/document/parser/images/genericImageParser.java index 244d9db7c..08ae5d827 100644 --- a/source/net/yacy/document/parser/images/genericImageParser.java +++ b/source/net/yacy/document/parser/images/genericImageParser.java @@ -28,7 +28,6 @@ package net.yacy.document.parser.images; import java.awt.image.BufferedImage; import java.io.ByteArrayInputStream; -import java.io.EOFException; import java.io.File; import java.io.FileInputStream; import java.io.FileNotFoundException; @@ -227,14 +226,8 @@ public class genericImageParser extends AbstractParser implements Parser { try { ImageIO.setUseCache(false); // do not write a cache to disc; keep in RAM image = ImageIO.read(sourceStream); - } catch (final EOFException e) { - Log.logException(e); - throw new Parser.Failure(e.getMessage(), location); - } catch (final IOException e) { - Log.logException(e); - throw new Parser.Failure(e.getMessage(), location); - } catch (final Throwable e) { // may appear in case of an OutOfMemoryError - Log.logException(e); + } catch (final Throwable e) { + //Log.logException(e); throw new Parser.Failure(e.getMessage(), location); } if (image == null) throw new Parser.Failure("ImageIO returned NULL", location); diff --git a/source/net/yacy/search/SwitchboardConstants.java b/source/net/yacy/search/SwitchboardConstants.java index 44dcd8cd6..73425d0c9 100644 --- a/source/net/yacy/search/SwitchboardConstants.java +++ b/source/net/yacy/search/SwitchboardConstants.java @@ -437,6 +437,7 @@ public final class SwitchboardConstants { public static final String SEARCH_ITEMS = "search.items"; public static final String SEARCH_TARGET = "search.target"; + public static final String SEARCH_VERIFY_DELETE = "search.verify.delete"; /** * system tray diff --git a/source/net/yacy/search/query/RWIProcess.java b/source/net/yacy/search/query/RWIProcess.java index 7aea35a1f..84b1c3d8e 100644 --- a/source/net/yacy/search/query/RWIProcess.java +++ b/source/net/yacy/search/query/RWIProcess.java @@ -35,6 +35,7 @@ import java.util.SortedMap; import java.util.concurrent.BlockingQueue; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; import java.util.regex.Pattern; import net.yacy.cora.document.ASCII; @@ -79,7 +80,7 @@ public final class RWIProcess extends Thread { private int remote_resourceSize, remote_indexCount, remote_peerCount; private int local_indexCount; private final WeakPriorityBlockingQueue stack; - private int feeders; + private final AtomicInteger feeders; private final ConcurrentHashMap> doubleDomCache; // key = domhash (6 bytes); value = like stack //private final HandleSet handover; // key = urlhash; used for double-check of urls that had been handed over to search process @@ -123,7 +124,7 @@ public final class RWIProcess extends Thread { this.protocolNavigator = new ConcurrentScoreMap(); this.filetypeNavigator = new ConcurrentScoreMap(); this.ref = new ConcurrentScoreMap(); - this.feeders = 1; + this.feeders = new AtomicInteger(1); this.startTime = System.currentTimeMillis(); } @@ -288,16 +289,16 @@ public final class RWIProcess extends Thread { * method to signal the incoming stack that one feeder has terminated */ public void oneFeederTerminated() { - this.feeders--; - assert this.feeders >= 0 : "feeders = " + this.feeders; + final int c = this.feeders.decrementAndGet(); + assert c >= 0 : "feeders = " + c; } public void moreFeeders(final int countMoreFeeders) { - this.feeders += countMoreFeeders; + this.feeders.addAndGet(countMoreFeeders); } public boolean feedingIsFinished() { - return System.currentTimeMillis() - this.startTime > 50 && this.feeders == 0; + return this.feeders.get() == 0; } private boolean testFlags(final WordReference ientry) { diff --git a/source/net/yacy/search/query/SearchEvent.java b/source/net/yacy/search/query/SearchEvent.java index 8638ab4f5..895b962b9 100644 --- a/source/net/yacy/search/query/SearchEvent.java +++ b/source/net/yacy/search/query/SearchEvent.java @@ -371,14 +371,13 @@ public final class SearchEvent { } public ResultEntry oneResult(final int item, final long timeout) { - if ((this.query.domType == QueryParams.Searchdom.GLOBAL) || - (this.query.domType == QueryParams.Searchdom.CLUSTER)) { + if ((this.query.domType == QueryParams.Searchdom.GLOBAL) || (this.query.domType == QueryParams.Searchdom.CLUSTER)) { // this is a search using remote search threads. Also the local // search thread is started as background process if ((this.localSearchThread != null) && (this.localSearchThread.isAlive())) { // in case that the local search takes longer than some other // remote search requests, wait that the local process terminates first - try {this.localSearchThread.join();} catch (final InterruptedException e) {} + try {this.localSearchThread.join(300);} catch (final InterruptedException e) {} } } return this.resultFetcher.oneResult(item, timeout); diff --git a/source/net/yacy/search/query/SearchEventCache.java b/source/net/yacy/search/query/SearchEventCache.java index 40afda8ef..24e706c7b 100644 --- a/source/net/yacy/search/query/SearchEventCache.java +++ b/source/net/yacy/search/query/SearchEventCache.java @@ -37,6 +37,7 @@ import net.yacy.kelondro.util.MemoryControl; import net.yacy.peers.SeedDB; import net.yacy.repository.LoaderDispatcher; import net.yacy.search.Switchboard; +import net.yacy.search.SwitchboardConstants; import net.yacy.search.index.Segment; import net.yacy.search.ranking.RankingProfile; import net.yacy.search.snippet.ContentDomain; @@ -178,7 +179,7 @@ public class SearchEventCache { Log.logInfo("SearchEventCache", "getEvent: " + SearchEventCache.lastEvents.size() + " in cache; " + countAliveThreads() + " alive"); // start a new event - final boolean delete = Switchboard.getSwitchboard() == null | Switchboard.getSwitchboard().getConfigBool("search.verify.delete", true); + final boolean delete = Switchboard.getSwitchboard() == null || Switchboard.getSwitchboard().getConfigBool(SwitchboardConstants.SEARCH_VERIFY_DELETE, true); event = new SearchEvent(query, peers, workTables, preselectedPeerHashes, generateAbstracts, loader, remote_maxcount, remote_maxtime, burstRobinsonPercent, burstMultiwordPercent, delete); MemoryControl.request(100 * 1024 * 1024, false); // this may trigger a short memory status which causes a reducing of cache space of other threads } diff --git a/source/net/yacy/search/query/SnippetProcess.java b/source/net/yacy/search/query/SnippetProcess.java index 913924b12..c545eeb76 100644 --- a/source/net/yacy/search/query/SnippetProcess.java +++ b/source/net/yacy/search/query/SnippetProcess.java @@ -147,7 +147,8 @@ public class SnippetProcess { final long waittimeout = System.currentTimeMillis() + 300; while ( item == 0 && - this.result.sizeAvailable() < this.query.neededResults() + this.query.itemsPerPage && + (!this.rankingProcess.feedingIsFinished() || this.rankingProcess.sizeQueue() > 0) && + this.result.sizeAvailable() < this.query.neededResults() /* + this.query.itemsPerPage */ && System.currentTimeMillis() < waittimeout && anyWorkerAlive() ) { @@ -324,7 +325,7 @@ public class SnippetProcess { } private boolean anyWorkerAlive() { - if (this.workerThreads == null) return false; + if (this.workerThreads == null || this.workerThreads.length == 0) return false; synchronized(this.workerThreads) { for (final Worker workerThread : this.workerThreads) { if ((workerThread != null) && @@ -377,7 +378,7 @@ public class SnippetProcess { // check if we have enough if (SnippetProcess.this.result.sizeAvailable() >= this.neededResults) { - //Log.logWarning("ResultFetcher", ResultFetcher.this.result.sizeAvailable() + " = result.sizeAvailable() >= this.neededResults = " + this.neededResults); + //Log.logWarning("ResultFetcher", SnippetProcess.this.result.sizeAvailable() + " = result.sizeAvailable() >= this.neededResults = " + this.neededResults); break; } @@ -391,7 +392,7 @@ public class SnippetProcess { page = SnippetProcess.this.rankingProcess.takeURL(true, Math.min(100, this.timeout - System.currentTimeMillis())); //if (page == null) page = rankedCache.takeURL(false, this.timeout - System.currentTimeMillis()); if (page == null) { - //System.out.println("page == null"); + //Log.logWarning("ResultFetcher", "page == null"); break; // no more available } if (SnippetProcess.this.query.filterfailurls && SnippetProcess.this.workTables.failURLsContains(page.hash())) continue; @@ -405,7 +406,6 @@ public class SnippetProcess { if (sd != null) solrContent = this.solr.getScheme().solrGetText(sd); } - loops++; resultEntry = fetchSnippet(page, solrContent, this.cacheStrategy); // does not fetch snippets if snippetMode == 0 if (resultEntry == null) continue; // the entry had some problems, cannot be used