Merge branch 'master' of https://git.gitorious.org/yacy/rc1.git

htroot/ConfigPortal.java

@@ -1,4 +1,4 @@
-// ConfigPortal.java 
+// ConfigPortal.java
 // -----------------------
 // part of YaCy
 // (C) by Michael Peter Christen; mc@yacy.net
@@ -10,7 +10,7 @@
 //$LastChangedBy$
 //
 // LICENSE
-// 
+//
 // This program is free software; you can redistribute it and/or modify
 // it under the terms of the GNU General Public License as published by
 // the Free Software Foundation; either version 2 of the License, or
@@ -38,14 +38,14 @@ public class ConfigPortal {
     public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) {
         final serverObjects prop = new serverObjects();
         final Switchboard sb = (Switchboard) env;
-        
+
         if (post != null) {
             // AUTHENTICATE
             if (!header.containsKey(RequestHeader.AUTHORIZATION)) {
                 prop.putHTML("AUTHENTICATE","log-in");
                 return prop;
             }
-            
+
             if (post.containsKey("popup")) {
                 final String popup = post.get("popup", "status");
                 if ("front".equals(popup)) {
@@ -62,7 +62,7 @@ public class ConfigPortal {
                 final String newGreeting = post.get(SwitchboardConstants.GREETING, "");
                 // store this call as api call
                 sb.tables.recordAPICall(post, "ConfigPortal.html", WorkTables.TABLE_API_TYPE_CONFIGURATION, "new portal design. greeting: " + newGreeting);
-                
+
                 sb.setConfig(SwitchboardConstants.GREETING, newGreeting);
                 sb.setConfig(SwitchboardConstants.GREETING_HOMEPAGE, post.get(SwitchboardConstants.GREETING_HOMEPAGE, ""));
                 sb.setConfig(SwitchboardConstants.GREETING_LARGE_IMAGE, post.get(SwitchboardConstants.GREETING_LARGE_IMAGE, ""));
@@ -80,7 +80,7 @@ public class ConfigPortal {
                 sb.setConfig("search.result.show.parser", post.getBoolean("search.result.show.parser", false));
                 sb.setConfig("search.result.show.pictures", post.getBoolean("search.result.show.pictures", false));
                 sb.setConfig("search.verify", post.get("search.verify", "ifexist"));
-                sb.setConfig("search.verify.delete", post.getBoolean("search.verify.delete", false));
+                sb.setConfig(SwitchboardConstants.SEARCH_VERIFY_DELETE, post.getBoolean("search.verify.delete", false));
                 // construct navigation String
                 String nav = "";
                 if (post.getBoolean("search.navigation.hosts", false)) nav += "hosts,";
@@ -109,8 +109,8 @@ public class ConfigPortal {
                 sb.setConfig("search.result.show.parser", true);
                 sb.setConfig("search.result.show.pictures", true);
                 sb.setConfig("search.verify", "iffresh");
-                sb.setConfig("search.verify.delete", "true");
+                sb.setConfig(SwitchboardConstants.SEARCH_VERIFY_DELETE, "true");
-            }            
+            }
         }
 
         prop.putHTML(SwitchboardConstants.GREETING, sb.getConfig(SwitchboardConstants.GREETING, ""));
@@ -121,7 +121,7 @@ public class ConfigPortal {
         prop.put("publicTopmenu", sb.getConfigBool("publicTopmenu", false) ? 1 : 0);
         prop.put("publicSearchpage", sb.getConfigBool("publicSearchpage", false) ? 1 : 0);
         prop.put("search.options", sb.getConfigBool("search.options", false) ? 1 : 0);
-        
+
         prop.put("search.result.show.date", sb.getConfigBool("search.result.show.date", false) ? 1 : 0);
         prop.put("search.result.show.size", sb.getConfigBool("search.result.show.size", false) ? 1 : 0);
         prop.put("search.result.show.metadata", sb.getConfigBool("search.result.show.metadata", false) ? 1 : 0);
@@ -138,7 +138,7 @@ public class ConfigPortal {
         prop.put("search.verify.ifexist", sb.getConfig("search.verify", "").equals("ifexist") ? 1 : 0);
         prop.put("search.verify.cacheonly", sb.getConfig("search.verify", "").equals("cacheonly") ? 1 : 0);
         prop.put("search.verify.false", sb.getConfig("search.verify", "").equals("false") ? 1 : 0);
-        prop.put("search.verify.delete", sb.getConfigBool("search.verify.delete", true) ? 1 : 0);
+        prop.put("search.verify.delete", sb.getConfigBool(SwitchboardConstants.SEARCH_VERIFY_DELETE, true) ? 1 : 0);
 
         final String  browserPopUpPage = sb.getConfig(SwitchboardConstants.BROWSER_POP_UP_PAGE, "ConfigBasic.html");
         prop.put("popupFront", 0);
@@ -156,14 +156,14 @@ public class ConfigPortal {
         }
 
         prop.put("maximumRecords", sb.getConfigInt(SwitchboardConstants.SEARCH_ITEMS, 10));
-        
+
         final String target = sb.getConfig(SwitchboardConstants.SEARCH_TARGET, "_self");
         prop.put("selected_blank", "_blank".equals(target) ? 1 : 0);
         prop.put("selected_self", "_self".equals(target) ? 1 : 0);
         prop.put("selected_parent", "_parent".equals(target) ? 1 : 0);
         prop.put("selected_top", "_top".equals(target) ? 1 : 0);
         prop.put("selected_searchresult", "searchresult".equals(target) ? 1 : 0);
-                
+
         String myaddress = (sb.peers == null) ? null : sb.peers.mySeed() == null ? null : sb.peers.mySeed().getPublicAddress();
         if (myaddress == null) {
             myaddress = "localhost:" + sb.getConfig("port", "8090");

htroot/yacysearch.java

@@ -597,7 +597,7 @@ public class yacysearch {
                     indexSegment,
                     ranking,
                     header.get(RequestHeader.USER_AGENT, ""),
-                    sb.getConfigBool(SwitchboardConstants.NETWORK_SEARCHVERIFY, false) && sb.peers.mySeed().getFlagAcceptRemoteIndex());
+                    sb.getConfigBool(SwitchboardConstants.SEARCH_VERIFY_DELETE, false) && sb.getConfigBool(SwitchboardConstants.NETWORK_SEARCHVERIFY, false) && sb.peers.mySeed().getFlagAcceptRemoteIndex());
             EventTracker.delete(EventTracker.EClass.SEARCH);
             EventTracker.update(EventTracker.EClass.SEARCH, new ProfilingGraph.EventSearch(theQuery.id(true), SearchEvent.Type.INITIALIZATION, "", 0, 0), false);
 

source/net/yacy/cora/date/GenericFormatter.java

@@ -133,10 +133,7 @@ public class GenericFormatter extends AbstractFormatter implements DateFormatter
         if (UTCOffset == null || UTCOffset.length() == 0) { return new Date(); }
         try {
             return new Date(this.dateFormat.parse(timeString).getTime() - UTCDiff() + UTCDiff(UTCOffset));
-        } catch (final java.text.ParseException e) {
+        } catch (final Throwable e) {
-            //serverLog.logFinest("parseUniversalDate", e.getMessage() + ", remoteTimeString=[" + remoteTimeString + "]");
-            return new Date();
-        } catch (final java.lang.NumberFormatException e) {
             //serverLog.logFinest("parseUniversalDate", e.getMessage() + ", remoteTimeString=[" + remoteTimeString + "]");
             return new Date();
         }

source/net/yacy/document/parser/images/genericImageParser.java

@@ -28,7 +28,6 @@ package net.yacy.document.parser.images;
 
 import java.awt.image.BufferedImage;
 import java.io.ByteArrayInputStream;
-import java.io.EOFException;
 import java.io.File;
 import java.io.FileInputStream;
 import java.io.FileNotFoundException;
@@ -227,14 +226,8 @@ public class genericImageParser extends AbstractParser implements Parser {
         try {
             ImageIO.setUseCache(false); // do not write a cache to disc; keep in RAM
             image = ImageIO.read(sourceStream);
-        } catch (final EOFException e) {
+        } catch (final Throwable e) {
-            Log.logException(e);
+            //Log.logException(e);
-            throw new Parser.Failure(e.getMessage(), location);
-        } catch (final IOException e) {
-            Log.logException(e);
-            throw new Parser.Failure(e.getMessage(), location);
-        } catch (final Throwable e) { // may appear in case of an OutOfMemoryError
-            Log.logException(e);
             throw new Parser.Failure(e.getMessage(), location);
         }
         if (image == null) throw new Parser.Failure("ImageIO returned NULL", location);

source/net/yacy/search/SwitchboardConstants.java

@@ -437,6 +437,7 @@ public final class SwitchboardConstants {
 
     public static final String SEARCH_ITEMS   = "search.items";
     public static final String SEARCH_TARGET  = "search.target";
+    public static final String SEARCH_VERIFY_DELETE = "search.verify.delete";
 
     /**
      * system tray

source/net/yacy/search/query/RWIProcess.java

@@ -35,6 +35,7 @@ import java.util.SortedMap;
 import java.util.concurrent.BlockingQueue;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
 import java.util.regex.Pattern;
 
 import net.yacy.cora.document.ASCII;
@@ -79,7 +80,7 @@ public final class RWIProcess extends Thread {
     private int remote_resourceSize, remote_indexCount, remote_peerCount;
     private int local_indexCount;
     private final WeakPriorityBlockingQueue<WordReferenceVars> stack;
-    private int feeders;
+    private final AtomicInteger feeders;
     private final ConcurrentHashMap<String, WeakPriorityBlockingQueue<WordReferenceVars>> doubleDomCache; // key = domhash (6 bytes); value = like stack
     //private final HandleSet handover; // key = urlhash; used for double-check of urls that had been handed over to search process
 
@@ -123,7 +124,7 @@ public final class RWIProcess extends Thread {
         this.protocolNavigator = new ConcurrentScoreMap<String>();
         this.filetypeNavigator = new ConcurrentScoreMap<String>();
         this.ref = new ConcurrentScoreMap<String>();
-        this.feeders = 1;
+        this.feeders = new AtomicInteger(1);
         this.startTime = System.currentTimeMillis();
     }
 
@@ -288,16 +289,16 @@ public final class RWIProcess extends Thread {
      * method to signal the incoming stack that one feeder has terminated
      */
     public void oneFeederTerminated() {
-    	this.feeders--;
+    	final int c = this.feeders.decrementAndGet();
-    	assert this.feeders >= 0 : "feeders = " + this.feeders;
+    	assert c >= 0 : "feeders = " + c;
     }
 
     public void moreFeeders(final int countMoreFeeders) {
-    	this.feeders += countMoreFeeders;
+    	this.feeders.addAndGet(countMoreFeeders);
     }
 
     public boolean feedingIsFinished() {
-    	return System.currentTimeMillis() - this.startTime > 50 && this.feeders == 0;
+    	return this.feeders.get() == 0;
     }
 
     private boolean testFlags(final WordReference ientry) {

source/net/yacy/search/query/SearchEvent.java

@@ -371,14 +371,13 @@ public final class SearchEvent {
     }
 
     public ResultEntry oneResult(final int item, final long timeout) {
-        if ((this.query.domType == QueryParams.Searchdom.GLOBAL) ||
+        if ((this.query.domType == QueryParams.Searchdom.GLOBAL) || (this.query.domType == QueryParams.Searchdom.CLUSTER)) {
-             (this.query.domType == QueryParams.Searchdom.CLUSTER)) {
             // this is a search using remote search threads. Also the local
             // search thread is started as background process
             if ((this.localSearchThread != null) && (this.localSearchThread.isAlive())) {
                 // in case that the local search takes longer than some other
                 // remote search requests, wait that the local process terminates first
-            	try {this.localSearchThread.join();} catch (final InterruptedException e) {}
+            	try {this.localSearchThread.join(300);} catch (final InterruptedException e) {}
             }
         }
         return this.resultFetcher.oneResult(item, timeout);

source/net/yacy/search/query/SearchEventCache.java

@@ -37,6 +37,7 @@ import net.yacy.kelondro.util.MemoryControl;
 import net.yacy.peers.SeedDB;
 import net.yacy.repository.LoaderDispatcher;
 import net.yacy.search.Switchboard;
+import net.yacy.search.SwitchboardConstants;
 import net.yacy.search.index.Segment;
 import net.yacy.search.ranking.RankingProfile;
 import net.yacy.search.snippet.ContentDomain;
@@ -178,7 +179,7 @@ public class SearchEventCache {
             Log.logInfo("SearchEventCache", "getEvent: " + SearchEventCache.lastEvents.size() + " in cache; " + countAliveThreads() + " alive");
 
             // start a new event
-            final boolean delete = Switchboard.getSwitchboard() == null | Switchboard.getSwitchboard().getConfigBool("search.verify.delete", true);
+            final boolean delete = Switchboard.getSwitchboard() == null || Switchboard.getSwitchboard().getConfigBool(SwitchboardConstants.SEARCH_VERIFY_DELETE, true);
             event = new SearchEvent(query, peers, workTables, preselectedPeerHashes, generateAbstracts, loader, remote_maxcount, remote_maxtime, burstRobinsonPercent, burstMultiwordPercent, delete);
             MemoryControl.request(100 * 1024 * 1024, false); // this may trigger a short memory status which causes a reducing of cache space of other threads
         }

source/net/yacy/search/query/SnippetProcess.java

@@ -147,7 +147,8 @@ public class SnippetProcess {
         final long waittimeout = System.currentTimeMillis() + 300;
         while (
           item == 0 &&
-          this.result.sizeAvailable() < this.query.neededResults() + this.query.itemsPerPage &&
+          (!this.rankingProcess.feedingIsFinished() || this.rankingProcess.sizeQueue() > 0) &&
+          this.result.sizeAvailable() < this.query.neededResults() /* + this.query.itemsPerPage */ &&
           System.currentTimeMillis() < waittimeout &&
           anyWorkerAlive()
           ) {
@@ -324,7 +325,7 @@ public class SnippetProcess {
     }
 
    private boolean anyWorkerAlive() {
-        if (this.workerThreads == null) return false;
+        if (this.workerThreads == null || this.workerThreads.length == 0) return false;
         synchronized(this.workerThreads) {
             for (final Worker workerThread : this.workerThreads) {
                if ((workerThread != null) &&
@@ -377,7 +378,7 @@ public class SnippetProcess {
 
                     // check if we have enough
                     if (SnippetProcess.this.result.sizeAvailable() >= this.neededResults) {
-                        //Log.logWarning("ResultFetcher", ResultFetcher.this.result.sizeAvailable() + " = result.sizeAvailable() >= this.neededResults = " + this.neededResults);
+                        //Log.logWarning("ResultFetcher", SnippetProcess.this.result.sizeAvailable() + " = result.sizeAvailable() >= this.neededResults = " + this.neededResults);
                         break;
                     }
 
@@ -391,7 +392,7 @@ public class SnippetProcess {
                     page = SnippetProcess.this.rankingProcess.takeURL(true, Math.min(100, this.timeout - System.currentTimeMillis()));
                     //if (page == null) page = rankedCache.takeURL(false, this.timeout - System.currentTimeMillis());
                     if (page == null) {
-                        //System.out.println("page == null");
+                        //Log.logWarning("ResultFetcher", "page == null");
                         break; // no more available
                     }
                     if (SnippetProcess.this.query.filterfailurls && SnippetProcess.this.workTables.failURLsContains(page.hash())) continue;
@@ -405,7 +406,6 @@ public class SnippetProcess {
                         if (sd != null) solrContent = this.solr.getScheme().solrGetText(sd);
                     }
 
-
                     loops++;
                     resultEntry = fetchSnippet(page, solrContent, this.cacheStrategy); // does not fetch snippets if snippetMode == 0
                     if (resultEntry == null) continue; // the entry had some problems, cannot be used