Render a relevant message and status on blocked search requests

When unauthenticated (or with insufficient rights) client is blocked
either because blacklisted or excessive request rate, render an error
message and a relevant HTTP status for API requests, instead of an empty
response that appears broken.

htroot/yacysearch.html

@@ -92,7 +92,10 @@
 <div id="api" style="top:58px;">
 <script type="text/javascript">
 //<![CDATA[
-document.write("\<a href=\"yacysearch.rss?" + window.location.search.substring(1) + "\" id=\"apilink\" target=\"_blank\"\><img src=\"env/grafics/rss.png\" width=\"16\" height=\"16\" alt=\"RSS\" /></a>")
+	var showRSSIcon = #(num-results)#false::true::true::true::false::false#(/num-results)#;
+	if(showRSSIcon) {
+		document.write("\<a href=\"yacysearch.rss?" + window.location.search.substring(1) + "\" id=\"apilink\" target=\"_blank\"\><img src=\"env/grafics/rss.png\" width=\"16\" height=\"16\" alt=\"RSS\" /></a>");
+	}
 //]]>
 </script>
 <span>This search result can also be retrieved as RSS/<a href="http://www.opensearch.org" target="_blank">opensearch</a> output.
@@ -160,7 +163,17 @@ Use the RSS search result format to add static searches to your RSS reader, if y
       </div>
     </div>
 	::
-	<p>Searching the web with this peer is disabled for unauthorized users. Please <a href="Status.html?login=">log in</a> as administrator to use the search function</p>
+	<p class="alert alert-danger" role="alert">Searching the web with this peer is disabled for unauthorized users. Please <a href="Status.html?login=">log in</a> as administrator to use the search function</p>
+	::
+	<p class="alert alert-danger col-sm-10" role="alert">
+		#(blockReason)#::You are not allowed to search the web with this peer.
+		::You have reached the maximum allowed number of accesses to this search page within ten minutes.
+		Please try again later or log in as administrator or as a user with extended search right.
+		::You have reached the maximum allowed number of accesses to this search page within one minute.
+		Please try again later or log in as administrator or as a user with extended search right.
+		::You have reached the maximum allowed number of accesses to this search page within three seconds.
+		Please try again later or log in as administrator or as a user with extended search right.
+	</p>
 #(/num-results)#
 
 #(urlmaskerror)#::

htroot/yacysearch.java

@@ -44,6 +44,8 @@ import java.util.TreeSet;
 import java.util.regex.Pattern;
 import java.util.regex.PatternSyntaxException;
 
+import org.apache.http.HttpStatus;
+
 import net.yacy.cora.document.analysis.Classification;
 import net.yacy.cora.document.analysis.Classification.ContentDomain;
 import net.yacy.cora.document.encoding.UTF8;
@@ -67,6 +69,7 @@ import net.yacy.data.UserDB;
 import net.yacy.data.ymark.YMarkTables;
 import net.yacy.document.LibraryProvider;
 import net.yacy.document.Tokenizer;
+import net.yacy.http.servlets.TemplateProcessingException;
 import net.yacy.http.servlets.YaCyDefaultServlet;
 import net.yacy.kelondro.data.meta.URIMetadataNode;
 import net.yacy.kelondro.util.Bitfield;
@@ -321,9 +324,15 @@ public class yacysearch {
                 snippetFetchStrategy = null;
             }
             block = true;
+            prop.put("num-results_blockReason", 1);
             ConcurrentLog.warn("LOCAL_SEARCH", "ACCESS CONTROL: BLACKLISTED CLIENT FROM "
                 + client
                 + " gets no permission to search");
+			if (!"html".equals(EXT)) {
+				/* API request : return the relevant HTTP status */
+				throw new TemplateProcessingException("You are not allowed to search the web with this peer.",
+						HttpStatus.SC_FORBIDDEN);
+			}
         } else if ( !extendedSearchRights && !localhostAccess && !intranetMode ) {
             // in case that we do a global search or we want to fetch snippets, we check for DoS cases
         	final int accInThreeSeconds;
@@ -405,13 +414,24 @@ public class yacysearch {
                 }
             }
             // general load protection
+            String timePeriodMsg = "";
 			if (accInTenMinutes >= sb.getConfigInt(SearchAccessRateConstants.PUBLIC_MAX_ACCESS_10MN.getKey(),
-					SearchAccessRateConstants.PUBLIC_MAX_ACCESS_10MN.getDefaultValue())
+					SearchAccessRateConstants.PUBLIC_MAX_ACCESS_10MN.getDefaultValue())) {
-					|| accInOneMinute >= sb.getConfigInt(SearchAccessRateConstants.PUBLIC_MAX_ACCESS_1MN.getKey(),
+				block = true;
-							SearchAccessRateConstants.PUBLIC_MAX_ACCESS_1MN.getDefaultValue())
+				timePeriodMsg = "ten minutes";
-					|| accInThreeSeconds >= sb.getConfigInt(SearchAccessRateConstants.PUBLIC_MAX_ACCESS_3S.getKey(),
+				prop.put("num-results_blockReason", 2);
-							SearchAccessRateConstants.PUBLIC_MAX_ACCESS_3S.getDefaultValue())) {
+			} else if (accInOneMinute >= sb.getConfigInt(SearchAccessRateConstants.PUBLIC_MAX_ACCESS_1MN.getKey(),
-                block = true;
+					SearchAccessRateConstants.PUBLIC_MAX_ACCESS_1MN.getDefaultValue())) {
+				block = true;
+				timePeriodMsg = "one minute";
+				prop.put("num-results_blockReason", 3);
+			} else if (accInThreeSeconds >= sb.getConfigInt(SearchAccessRateConstants.PUBLIC_MAX_ACCESS_3S.getKey(),
+					SearchAccessRateConstants.PUBLIC_MAX_ACCESS_3S.getDefaultValue())) {
+				block = true;
+				timePeriodMsg = "three seconds";
+				prop.put("num-results_blockReason", 4);
+			}
+        	if(block) {
                 ConcurrentLog.warn("LOCAL_SEARCH", "ACCESS CONTROL: CLIENT FROM "
                     + client
                     + ": "
@@ -422,10 +442,22 @@ public class yacysearch {
                     + accInTenMinutes
                     + "/600s, "
                     + " requests, disallowed search");
+				if (!"html".equals(EXT)) {
+					/*
+					 * API request : return the relevant HTTP status (429 - Too Many Requests - see
+					 * https://tools.ietf.org/html/rfc6585#section-4)
+					 */
+					throw new TemplateProcessingException(
+							"You have reached the maximum allowed number of accesses to this search service within "
+									+ timePeriodMsg + ". Please try again later or log in as administrator or as a user with extended search right.",
+							429);
+				}
             }
         }
 
-        if ( !block ) {
+        if (block) {
+        	prop.put("num-results", 5);
+        } else {
             String urlmask = (post == null) ? ".*" : post.get("urlmaskfilter", ".*"); // the expression must be a subset of the java Match syntax described in http://lucene.apache.org/core/4_4_0/core/org/apache/lucene/util/automaton/RegExp.html
             String tld = null;
             String inlink = null;

htroot/yacysearchtrailer.html

@@ -56,6 +56,7 @@ $('#resource-switch-form').popover()
 </script>
 #(/resource-switches)#
 
+#(ranking-switches)#::
 <p class="navbutton"></p>
 <form action="yacysearch.html" method="get" accept-charset="UTF-8" name="rankingSwitchForm">
     #(authSearch)#::
@@ -88,6 +89,7 @@ $('#resource-switch-form').popover()
 	  	</div>
 	</div>
 </form>
+#(/ranking-switches)#
 
 #(searchdomswitches)#::<p class="navbutton"></p>
 <form action="yacysearch.html" method="get" accept-charset="UTF-8" name="contentdomSwitchForm"

htroot/yacysearchtrailer.java

@@ -104,9 +104,6 @@ public class yacysearchtrailer {
         boolean global = post == null || (!post.get("resource-switch", post.get("resource", "global")).equals("local") && p2pmode);
         boolean stealthmode = p2pmode && !global;
         
-        /* Add information about the current navigators generation (number of updates since their initialization) */
-        prop.put("nav-generation", theSearch.getNavGeneration());
-
         // compose search navigation
         ContentDomain contentdom = theSearch.getQuery().contentdom;
         prop.put("searchdomswitches",
@@ -130,11 +127,17 @@ public class yacysearchtrailer {
         prop.put("resource-switches_global", adminAuthenticated && global);
         appendSearchFormValues("resource-switches_", post, prop, global, theSearch, former, snippetFetchStrategyName, startRecord, meanMax);
         
-        appendSearchFormValues("", post, prop, global, theSearch, former, snippetFetchStrategyName, startRecord, meanMax);
+        /* The search event has been found : we can render ranking switches */
-        prop.put("contextRanking", !former.contains(" /date"));
+        prop.put("ranking-switches", true);
-        prop.put("contextRanking_formerWithoutDate", former.replace(" /date", ""));
+        appendSearchFormValues("ranking-switches_", post, prop, global, theSearch, former, snippetFetchStrategyName, startRecord, meanMax);
-        prop.put("dateRanking", former.contains(" /date"));
+        
-        prop.put("dateRanking_former", former);
+        /* Add information about the current navigators generation (number of updates since their initialization) */
+        prop.put("ranking-switches_nav-generation", theSearch.getNavGeneration());
+        
+        prop.put("ranking-switches_contextRanking", !former.contains(" /date"));
+        prop.put("ranking-switches_contextRanking_formerWithoutDate", former.replace(" /date", ""));
+        prop.put("ranking-switches_dateRanking", former.contains(" /date"));
+        prop.put("ranking-switches_dateRanking_former", former);
         
         appendSearchFormValues("searchdomswitches_", post, prop, global, theSearch, former, snippetFetchStrategyName, startRecord, meanMax);