Merge branch 'master' of https://git.gitorious.org/yacy/rc1.git

defaults/yacy.init

@@ -648,7 +648,7 @@ javastart_priority=10
 # wordCacheMaxLow/High is the number of word indexes that shall be held in the
 # ram cache during indexing. If you want to increase indexing speed, increase this
 # value i.e. up to one million, but increase also the memory limit to a minimum of 2GB
-wordCacheMaxCount = 100000
+wordCacheMaxCount = 50000
 
 # Specifies if yacy can be used as transparent http proxy.
 # 
@@ -724,9 +724,9 @@ search.options = true
 # search domains. If set to false then that search is not available
 search.text = true
 search.image = true
-search.audio = true
+search.audio = false
-search.video = true
+search.video = false
-search.app = true
+search.app = false
 
 # number of search results displayed by default
 search.items = 10
@@ -741,8 +741,8 @@ search.target = _self
 # these information pieces may be switched on or off
 search.result.show.date = true
 search.result.show.size = false
-search.result.show.metadata = false
+search.result.show.metadata = true
-search.result.show.parser = false
+search.result.show.parser = true
 search.result.show.pictures = false
 
 # search navigators: comma-separated list of default values for search navigation.
@@ -762,7 +762,7 @@ search.navigation=hosts,authors,namespace,topics,filetype,protocol
 #            consider content nevertheless as available and show result without snippet
 # false:     no link verification and not snippet generation:
              all search results are valid without verification
-search.verify = cacheonly
+search.verify = iffresh
 
 # in case that a link verification fails then the corresponding index reference can be
 # deleted to clean up the index. If this property is set then failed index verification in

htroot/ConfigBasic.java

@@ -1,4 +1,4 @@
-// ConfigBasic.java 
+// ConfigBasic.java
 // -----------------------
 // part of YaCy
 // (C) by Michael Peter Christen; mc@yacy.net
@@ -40,9 +40,8 @@ import net.yacy.kelondro.workflow.InstantBusyThread;
 import net.yacy.peers.Seed;
 import net.yacy.search.Switchboard;
 import net.yacy.search.SwitchboardConstants;
-
-import de.anomic.data.WorkTables;
 import de.anomic.data.Translator;
+import de.anomic.data.WorkTables;
 import de.anomic.http.server.HTTPDFileHandler;
 import de.anomic.server.serverCore;
 import de.anomic.server.serverObjects;
@@ -50,48 +49,48 @@ import de.anomic.server.serverSwitch;
 import de.anomic.tools.UPnP;
 
 public class ConfigBasic {
-    
+
     private static final int NEXTSTEP_FINISHED  = 0;
     private static final int NEXTSTEP_PWD       = 1;
     private static final int NEXTSTEP_PEERNAME  = 2;
     private static final int NEXTSTEP_PEERPORT  = 3;
     private static final int NEXTSTEP_RECONNECT = 4;
-    
+
     public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) throws FileNotFoundException, IOException {
-        
+
         // return variable that accumulates replacements
         final Switchboard sb = (Switchboard) env;
         final serverObjects prop = new serverObjects();
         final String langPath = env.getDataPath("locale.work", "DATA/LOCALE/locales").getAbsolutePath();
         String lang = env.getConfig("locale.language", "default");
-        
+
         final int authentication = sb.adminAuthenticated(header);
         if (authentication < 2) {
             // must authenticate
-            prop.put("AUTHENTICATE", "admin log-in"); 
+            prop.put("AUTHENTICATE", "admin log-in");
             return prop;
         }
-        
+
         // store this call as api call
         if (post != null && post.containsKey("set")) {
             sb.tables.recordAPICall(post, "ConfigBasic.html", WorkTables.TABLE_API_TYPE_CONFIGURATION, "basic settings");
         }
-        
+
         //boolean doPeerPing = false;
         if ((sb.peers.mySeed().isVirgin()) || (sb.peers.mySeed().isJunior())) {
             InstantBusyThread.oneTimeJob(sb.yc, "peerPing", null, 0);
             //doPeerPing = true;
         }
-        
+
         // language settings
         if (post != null && post.containsKey("language")  && !lang.equals(post.get("language", "default")) &&
                 (Translator.changeLang(env, langPath, post.get("language", "default") + ".lng"))) {
             prop.put("changedLanguage", "1");
         }
-        
+
         // peer name settings
         final String peerName = (post == null) ? sb.peers.mySeed().getName() : post.get("peername", "");
-        
+
         // port settings
         final long port;
         if (post != null && post.getInt("port", 0) > 1023) {
@@ -108,7 +107,7 @@ public class ConfigBasic {
             sb.peers.setMyName(peerName);
             sb.peers.saveMySeed();
         }
-        
+
         // UPnP config
         final boolean upnp;
         if (post != null && post.containsKey("port")) { // hack to allow checkbox
@@ -123,22 +122,22 @@ public class ConfigBasic {
         } else {
             upnp = false;
         }
-        
+
         // check port
         final boolean reconnect;
         if (!(env.getConfigLong("port", port) == port)) {
             // validate port
             final serverCore theServerCore = (serverCore) env.getThread("10_httpd");
             env.setConfig("port", port);
-            
+
             // redirect the browser to the new port
             reconnect = true;
-            
+
             // renew upnp port mapping
             if (upnp) {
                 UPnP.addPortMapping();
             }
-            
+
             String host = null;
             if (header.containsKey(HeaderFramework.HOST)) {
                 host = header.get(HeaderFramework.HOST);
@@ -147,7 +146,7 @@ public class ConfigBasic {
             } else {
                 host = Domains.myPublicLocalIP().getHostAddress();
             }
-            
+
             prop.put("reconnect", "1");
             prop.put("reconnect_host", host);
             prop.put("nextStep_host", host);
@@ -155,11 +154,11 @@ public class ConfigBasic {
             prop.put("nextStep_port", port);
             prop.put("reconnect_sslSupport", theServerCore.withSSL() ? "1" : "0");
             prop.put("nextStep_sslSupport", theServerCore.withSSL() ? "1" : "0");
-            
+
             // generate new shortcut (used for Windows)
             //yacyAccessible.setNewPortBat(Integer.parseInt(port));
             //yacyAccessible.setNewPortLink(Integer.parseInt(port));
-            
+
             // force reconnection in 7 seconds
             theServerCore.reconnect(7000);
         } else {
@@ -176,6 +175,9 @@ public class ConfigBasic {
                 // switch to p2p mode
                 sb.setConfig(SwitchboardConstants.INDEX_DIST_ALLOW, true);
                 sb.setConfig(SwitchboardConstants.INDEX_RECEIVE_ALLOW, true);
+                // set default behavior for search verification
+                sb.setConfig(SwitchboardConstants.SEARCH_VERIFY, "iffresh"); // nocache,iffresh,ifexist,cacheonly,false
+                sb.setConfig(SwitchboardConstants.SEARCH_VERIFY_DELETE, "true");
             }
             if ("portal".equals(post.get("usecase", "")) && !"webportal".equals(networkName)) {
                 // switch to webportal network
@@ -183,6 +185,9 @@ public class ConfigBasic {
                 // switch to robinson mode
                 sb.setConfig(SwitchboardConstants.INDEX_DIST_ALLOW, false);
                 sb.setConfig(SwitchboardConstants.INDEX_RECEIVE_ALLOW, false);
+                // set default behavior for search verification
+                sb.setConfig(SwitchboardConstants.SEARCH_VERIFY, "ifexist"); // nocache,iffresh,ifexist,cacheonly,false
+                sb.setConfig(SwitchboardConstants.SEARCH_VERIFY_DELETE, "false");
             }
             if ("intranet".equals(post.get("usecase", "")) && !"intranet".equals(networkName)) {
                 // switch to intranet network
@@ -190,6 +195,9 @@ public class ConfigBasic {
                 // switch to p2p mode: enable ad-hoc networks between intranet users
                 sb.setConfig(SwitchboardConstants.INDEX_DIST_ALLOW, false);
                 sb.setConfig(SwitchboardConstants.INDEX_RECEIVE_ALLOW, false);
+                // set default behavior for search verification
+                sb.setConfig(SwitchboardConstants.SEARCH_VERIFY, "cacheonly"); // nocache,iffresh,ifexist,cacheonly,false
+                sb.setConfig(SwitchboardConstants.SEARCH_VERIFY_DELETE, "false");
             }
             if ("intranet".equals(post.get("usecase", ""))) {
                 final String repositoryPath = post.get("repositoryPath", "/DATA/HTROOT/repository");
@@ -199,7 +207,7 @@ public class ConfigBasic {
                 }
             }
         }
-        
+
         networkName = sb.getConfig(SwitchboardConstants.NETWORK_NAME, "");
         if ("freeworld".equals(networkName)) {
             prop.put("setUseCase", 1);
@@ -215,18 +223,18 @@ public class ConfigBasic {
         }
         prop.put("setUseCase_port", port);
         prop.put("setUseCase_repositoryPath", sb.getConfig("repositoryPath", "/DATA/HTROOT/repository"));
-        
+
         // check if values are proper
         final boolean properPassword = (sb.getConfig(SwitchboardConstants.ADMIN_ACCOUNT_B64MD5, "").length() > 0) || sb.getConfigBool("adminAccountForLocalhost", false);
         final boolean properName = (sb.peers.mySeed().getName().length() >= 3) && (!(Seed.isDefaultPeerName(sb.peers.mySeed().getName())));
         final boolean properPort = (sb.peers.mySeed().isSenior()) || (sb.peers.mySeed().isPrincipal());
-        
+
         if ((env.getConfig("defaultFiles", "").startsWith("ConfigBasic.html,"))) {
             env.setConfig("defaultFiles", env.getConfig("defaultFiles", "").substring(17));
             env.setConfig("browserPopUpPage", "Status.html");
             HTTPDFileHandler.initDefaultPath();
         }
-        
+
         prop.put("statusName", properName ? "1" : "0");
         prop.put("statusPort", properPort ? "1" : "0");
         if (reconnect) {
@@ -240,7 +248,7 @@ public class ConfigBasic {
         } else {
             prop.put("nextStep", NEXTSTEP_FINISHED);
         }
-                
+
         final boolean upnp_enabled = env.getConfigBool(SwitchboardConstants.UPNP_ENABLED, false);
         prop.put("upnp", "1");
         prop.put("upnp_enabled", upnp_enabled ? "1" : "0");
@@ -250,8 +258,8 @@ public class ConfigBasic {
         else {
             prop.put("upnp_success", "0");
         }
-        
+
-        // set default values       
+        // set default values
         prop.putHTML("defaultName", sb.peers.mySeed().getName());
         prop.putHTML("defaultPort", env.getConfig("port", "8090"));
         lang = env.getConfig("locale.language", "default"); // re-assign lang, may have changed

htroot/ConfigPortal.html

@@ -48,6 +48,15 @@
           <input type="radio" name="search.options" value="false" #(search.options)#checked="checked"::#(/search.options)# />do not show Advanced Search
           </dd>
           
+          <dt>Show Media Search Options</dt>
+          <dd>
+          <input type="checkbox" name="search.text" value="true" #(search.text)#::checked="checked"#(/search.text)# />Text&nbsp;
+          <input type="checkbox" name="search.image" value="true" #(search.image)#::checked="checked"#(/search.image)# />Images&nbsp;
+          <input type="checkbox" name="search.audio" value="true" #(search.audio)#::checked="checked"#(/search.audio)# />Audio&nbsp;
+          <input type="checkbox" name="search.video" value="true" #(search.video)#::checked="checked"#(/search.video)# />Video&nbsp;
+          <input type="checkbox" name="search.app" value="true" #(search.app)#::checked="checked"#(/search.app)# />Applications
+          </dd>
+          
           <dt>Snippet Fetch Strategy &amp; Link Verification</dt>
           <dd>
           <input type="radio" name="search.verify" value="nocache" #(search.verify.nocache)#::checked="checked"#(/search.verify.nocache)# onclick="document.getElementById('search_verify_delete').disabled=false;document.getElementById('search_verify_delete').checked=true;"/>NOCACHE: no use of web cache, load all snippets online<br/>

htroot/ConfigPortal.java

@@ -74,12 +74,20 @@ public class ConfigPortal {
                 sb.setConfig("publicTopmenu", post.getBoolean("publicTopmenu", true));
                 sb.setConfig("publicSearchpage", post.getBoolean("publicSearchpage", true));
                 sb.setConfig("search.options", post.getBoolean("search.options", false));
+
+                sb.setConfig("search.text", post.getBoolean("search.text", false));
+                sb.setConfig("search.image", post.getBoolean("search.image", false));
+                sb.setConfig("search.audio", post.getBoolean("search.audio", false));
+                sb.setConfig("search.video", post.getBoolean("search.video", false));
+                sb.setConfig("search.app", post.getBoolean("search.app", false));
+
                 sb.setConfig("search.result.show.date", post.getBoolean("search.result.show.date", false));
                 sb.setConfig("search.result.show.size", post.getBoolean("search.result.show.size", false));
                 sb.setConfig("search.result.show.metadata", post.getBoolean("search.result.show.metadata", false));
                 sb.setConfig("search.result.show.parser", post.getBoolean("search.result.show.parser", false));
                 sb.setConfig("search.result.show.pictures", post.getBoolean("search.result.show.pictures", false));
-                sb.setConfig("search.verify", post.get("search.verify", "ifexist"));
+
+                sb.setConfig(SwitchboardConstants.SEARCH_VERIFY, post.get("search.verify", "ifexist"));
                 sb.setConfig(SwitchboardConstants.SEARCH_VERIFY_DELETE, post.getBoolean("search.verify.delete", false));
                 // construct navigation String
                 String nav = "";
@@ -103,12 +111,17 @@ public class ConfigPortal {
                 sb.setConfig("publicSearchpage", true);
                 sb.setConfig("search.navigation", "hosts,authors,namespace,topics");
                 sb.setConfig("search.options", true);
+                sb.setConfig("search.text", true);
+                sb.setConfig("search.image", true);
+                sb.setConfig("search.audio", false);
+                sb.setConfig("search.video", false);
+                sb.setConfig("search.app", false);
                 sb.setConfig("search.result.show.date", true);
-                sb.setConfig("search.result.show.size", true);
+                sb.setConfig("search.result.show.size", false);
                 sb.setConfig("search.result.show.metadata", true);
                 sb.setConfig("search.result.show.parser", true);
-                sb.setConfig("search.result.show.pictures", true);
+                sb.setConfig("search.result.show.pictures", false);
-                sb.setConfig("search.verify", "iffresh");
+                sb.setConfig(SwitchboardConstants.SEARCH_VERIFY, "iffresh");
                 sb.setConfig(SwitchboardConstants.SEARCH_VERIFY_DELETE, "true");
             }
         }
@@ -122,6 +135,12 @@ public class ConfigPortal {
         prop.put("publicSearchpage", sb.getConfigBool("publicSearchpage", false) ? 1 : 0);
         prop.put("search.options", sb.getConfigBool("search.options", false) ? 1 : 0);
 
+        prop.put("search.text", sb.getConfigBool("search.text", false) ? 1 : 0);
+        prop.put("search.image", sb.getConfigBool("search.image", false) ? 1 : 0);
+        prop.put("search.audio", sb.getConfigBool("search.audio", false) ? 1 : 0);
+        prop.put("search.video", sb.getConfigBool("search.video", false) ? 1 : 0);
+        prop.put("search.app", sb.getConfigBool("search.app", false) ? 1 : 0);
+
         prop.put("search.result.show.date", sb.getConfigBool("search.result.show.date", false) ? 1 : 0);
         prop.put("search.result.show.size", sb.getConfigBool("search.result.show.size", false) ? 1 : 0);
         prop.put("search.result.show.metadata", sb.getConfigBool("search.result.show.metadata", false) ? 1 : 0);

htroot/WatchWebStructure_p.java

@@ -7,7 +7,6 @@
 import net.yacy.cora.protocol.RequestHeader;
 import net.yacy.kelondro.data.meta.DigestURI;
 import net.yacy.search.Switchboard;
-
 import de.anomic.crawler.CrawlProfile;
 import de.anomic.crawler.CrawlSwitchboard;
 import de.anomic.server.serverObjects;
@@ -24,7 +23,7 @@ public class WatchWebStructure_p {
         String color_dot     = "11BB11";
         String color_line    = "222222";
         String color_lineend = "333333";
-        
+
         int width = 1024;
         int height = 576;
         int depth = 3;
@@ -32,7 +31,7 @@ public class WatchWebStructure_p {
         int time = -1;
         String host = "auto";
         String besthost;
-        
+
         if (post != null) {
             width         = post.getInt("width", 1024);
             height        = post.getInt("height", 576);
@@ -46,11 +45,11 @@ public class WatchWebStructure_p {
             color_line    = post.get("colorline",    color_line);
             color_lineend = post.get("colorlineend", color_lineend);
         }
-        
+
         if (host.equals("auto")) {
         	// try to find the host from the crawl profiles
         	CrawlProfile e;
-            for (byte[] handle: sb.crawler.getActive()) {
+            for (final byte[] handle: sb.crawler.getActive()) {
                 e = sb.crawler.getActive(handle);
                 if (e.name().equals(CrawlSwitchboard.CRAWL_PROFILE_PROXY) ||
                     e.name().equals(CrawlSwitchboard.CRAWL_PROFILE_REMOTE) ||
@@ -64,7 +63,7 @@ public class WatchWebStructure_p {
                 break; // take the first one
             }
         }
-        
+
         // find start point
         if (host == null ||
             host.length() == 0 ||
@@ -72,10 +71,11 @@ public class WatchWebStructure_p {
             sb.webStructure.referencesCount(DigestURI.hosthash6(host)) == 0) {
             // find domain with most references
             besthost = sb.webStructure.hostWithMaxReferences();
+            if (besthost == null) besthost = host;
         } else {
             besthost = host;
         }
-        
+
         prop.putHTML("host", host);
         prop.putHTML("besthost", besthost);
         prop.put("depth", depth);

htroot/yacysearch.html

@@ -214,7 +214,7 @@ document.getElementById("search").focus();
 #(resultTable)#::</table>#(/resultTable)#
 <!-- linklist end -->
 <!-- attach the bottomline -->
-#(pageNavBottom)#::<div id="pageNavBottom" style="text-align:center">#[resnav]#</div>#(/pageNavBottom)#
+<!--#(pageNavBottom)#::<div id="pageNavBottom" style="text-align:center">#[resnav]#</div>#(/pageNavBottom)#-->
 </div>
 <!-- div style="width=220px;" -->
 <!--#include virtual="yacysearchtrailer.html?eventID=#[eventID]#" -->

source/net/yacy/kelondro/data/meta/URIMetadataRow.java

@@ -327,7 +327,7 @@ public class URIMetadataRow implements URIMetadata {
             assert (s.toString().indexOf(0) < 0);
             s.append(",flags=").append(flags().exportB64());
             assert (s.toString().indexOf(0) < 0);
-            s.append(",lang=").append(UTF8.String(language()));
+            s.append(",lang=").append(language() == null ? "EN" : UTF8.String(language()));
             assert (s.toString().indexOf(0) < 0);
             s.append(",llocal=").append(llocal());
             assert (s.toString().indexOf(0) < 0);

source/net/yacy/peers/graphics/NetworkGraph.java

@@ -319,7 +319,7 @@ public class NetworkGraph {
         PrintTool.arcPrint(img, centerX, centerY, innerradius + linelength, angle, name);
 
         // draw corona around dot for crawling activity
-        final int ppmx = seed.getPPM() / 30;
+        final int ppmx = seed.getPPM() / 40;
         if (coronaangle >= 0 && ppmx > 0) {
             drawCorona(img, centerX, centerY, innerradius, angle, dotsize, ppmx, coronaangle, true, false, 24, 24, 24); // color = 0..63
         }
@@ -327,7 +327,7 @@ public class NetworkGraph {
         // draw corona around dot for query activity
         final int qphx = ((int) (seed.getQPM() * 4.0));
         if (coronaangle >= 0 && qphx > 0) {
-            drawCorona(img, centerX, centerY, innerradius, angle, dotsize, qphx, coronaangle, false, true, 8, 62, 8); // color = 0..63
+            drawCorona(img, centerX, centerY, innerradius, angle, dotsize, qphx, coronaangle, false, true, 20, 63, 20); // color = 0..63
         }
     }
 

source/net/yacy/search/SwitchboardConstants.java

@@ -437,6 +437,7 @@ public final class SwitchboardConstants {
 
     public static final String SEARCH_ITEMS   = "search.items";
     public static final String SEARCH_TARGET  = "search.target";
+    public static final String SEARCH_VERIFY  = "search.verify";
     public static final String SEARCH_VERIFY_DELETE = "search.verify.delete";
 
     /**