Marown/yacy_search_server
1
0
Fork 0
mirror of https://github.com/yacy/yacy_search_server.git synced 2024-09-19 00:01:41 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity

Removed use of deprecated Jetty IPAccessHandler for client filtering.

Browse Source 
Upgraded to InetAccessHandler.
Added InetPathAccessHandler extension to InetAccessHandler to maintain
path patterns capability previously available in IPAccessHandler but
lost in InetAccessHandler.

Filtering on IPv6 addresses is now supported.

Support for deprecated pattern formats such as "192.168." and
"192.168.1.1/path" has been removed, but startup automated migration
should convert such patterns eventually present in serverClient.
This commit is contained in:
luccioman 2017-12-08 15:12:08 +01:00
parent cc7a93e6b6
commit d95b288f19
10 changed files with 759 additions and 41 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
defaults/yacy.init
View File

@ -391,9 +391,9 @@ proxyClient=localhost,127\.0\.0\.1,192\.168\..*,10\..*,0:0:0:0:0:0:0:1.*
# to prevent that the asked peer knows which peer asks.
YaCyHop=true
# serverClient: client-ip's that may connect to the web server,
# serverClient: comma separated client-ip's that may connect to the web server,
# thus are allowed to use the search service
# if you set this to another value, search requst from others
# if you set this to another value, search requests from others
# are blocked, but you will also be blocked from using others
# search services.
serverClient=*

7
htroot/SettingsAck_p.java
View File

@ -35,6 +35,7 @@ import java.util.regex.PatternSyntaxException;
import net.yacy.cora.order.Digest;
import net.yacy.cora.protocol.RequestHeader;
import net.yacy.http.InetPathAccessHandler;
import net.yacy.kelondro.util.Formatter;
import net.yacy.peers.Network;
import net.yacy.peers.Seed;
@ -231,14 +232,14 @@ public class SettingsAck_p {
// testing proxy filter
int patternCount = 0;
String patternStr = null;
final StringTokenizer st = new StringTokenizer(filter,",");
try {
final StringTokenizer st = new StringTokenizer(filter,",");
while (st.hasMoreTokens()) {
patternCount++;
patternStr = st.nextToken();
Pattern.compile(patternStr);
InetPathAccessHandler.checkPattern(patternStr);
}
} catch (final PatternSyntaxException e) {
} catch (final IllegalArgumentException e) {
prop.put("info", "27");
prop.putHTML("info_filter", filter);
prop.put("info_nr", patternCount);

4
htroot/Settings_ServerAccess.inc
View File

@ -10,9 +10,9 @@
from using other peers' indexes for search service.
However, blocking access may be correct in enterprise environments where you only want to index your
company's own web pages.<br />
Filter have to be entered as IP, IP range or first part of allowed IP's separated by comma (e.g. 10.100.0-100.0-100, 127. )
Filter have to be entered as IP, IP range or using CIDR notation separated by comma (e.g. 192.168.1.1,2001:db8::ff00:42:8329,192.168.1.10-192.168.1.20,192.168.1.30-40,192.168.2.0/24)
further details on format see Jetty
<a href="http://download.eclipse.org/jetty/stable-9/apidocs/org/eclipse/jetty/server/handler/IPAccessHandler.html" target="_blank">IPAccessHandler</a> docu.
<a href="http://download.eclipse.org/jetty/stable-9/apidocs/org/eclipse/jetty/util/InetAddressSet.html" target="_blank">InetAddressSet</a> documentation.
</td>
</tr>
<tr valign="top">

5
locales/ru.lng
View File

@ -3117,9 +3117,8 @@ If you block access to your server (setting anything else than '*'), then you wi
from using other peers' indexes for search service.==использование поиска индексов другими узлами.
However, blocking access may be correct in enterprise environments where you only want to index your==Ограничение доступа может быть полезным в корпоративной сети, где требуется только индексирование
company's own web pages.==вэб-страниц компаний.
Filter have to be entered as IP, IP range or first part of allowed IP's separated by comma (e.g. 10.100.0-100.0-100, 127. )== Фильтр возможен по ip-адресу, по диапазону ip-адресов или по первым числам ip-адреса через запятую, (например 10.100.0-100.0-100, 127.)
further details on format see Jetty ==Подробную информацию об IPAccessHandler смотрите
<a href="http://download.eclipse.org/jetty/8.1.14.v20131031/apidocs/org/eclipse/jetty/server/handler/IPAccessHandler.html" target="_blank">IPAccessHandler</a> docu.==<a href="http://download.eclipse.org/jetty/8.1.14.v20131031/apidocs/org/eclipse/jetty/server/handler/IPAccessHandler.html" target="_blank">здесь</a>.
further details on format see Jetty ==Подробную информацию об InetAddressSet смотрите
<a href="http://download.eclipse.org/jetty/stable-9/apidocs/org/eclipse/jetty/util/InetAddressSet.html" target="_blank">InetAddressSet</a> documentation.==<a href="http://download.eclipse.org/jetty/stable-9/apidocs/org/eclipse/jetty/util/InetAddressSet.html" target="_blank">здесь</a>.
fileHost:==Размещение файлов:
staticIP (optional):==Постоянный IP-адрес (необязательно):
<strong>The staticIP can help that your peer can be reached by other peers in case that your==<strong>Использование постоянного IP-адреса может помочь вашему узлу соединиться с другиму узлами

173
source/net/yacy/http/InetPathAccessHandler.java Normal file
View File

@ -0,0 +1,173 @@
// InetPathAccessHandler.java
// Copyright 2017 by luccioman; https://github.com/luccioman
//
// This is a part of YaCy, a peer-to-peer based web search engine
//
// LICENSE
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation; either version 2 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
package net.yacy.http;
import java.io.IOException;
import java.net.InetAddress;
import javax.servlet.http.HttpServletRequest;
import org.eclipse.jetty.http.pathmap.MappedResource;
import org.eclipse.jetty.http.pathmap.PathMappings;
import org.eclipse.jetty.http.pathmap.PathSpec;
import org.eclipse.jetty.server.handler.InetAccessHandler;
import org.eclipse.jetty.util.InetAddressSet;
/**
* InetPathAccessHandler Access Handler
* <p>
* Extends {@link InetAccessHandler} by adding path patterns capabilities as
* previously available in the deprecated IPAccessHandler.
* </p>
*
*/
public class InetPathAccessHandler extends InetAccessHandler {
/** List of white listed paths mapped to adresses sets */
private final PathMappings<InetAddressSet> white = new PathMappings<>();
/** List of black listed paths mapped to adresses sets */
private final PathMappings<InetAddressSet> black = new PathMappings<>();
/**
* @throws IllegalArgumentException when the pattern is malformed
*/
@Override
public void include(final String pattern) throws IllegalArgumentException {
addPattern(pattern, this.white);
}
/**
* @throws IllegalArgumentException when a pattern is malformed
*/
@Override
public void include(final String... patterns) throws IllegalArgumentException {
for (final String pattern : patterns) {
include(pattern);
}
}
/**
* @throws IllegalArgumentException when the pattern is malformed
*/
@Override
public void exclude(final String pattern) throws IllegalArgumentException {
addPattern(pattern, this.black);
}
/**
* @throws IllegalArgumentException when a pattern is malformed
*/
@Override
public void exclude(final String... patterns) throws IllegalArgumentException {
for (final String pattern : patterns) {
exclude(pattern);
}
}
/**
* Helper method to parse the new pattern and add it to the specified mapping.
*
* @param pattern
* a new pattern to process
* @param pathMappings
* target mapping from paths to addresses sets. Must not be null.
* @throws IllegalArgumentException
* when the pattern is malformed
*/
protected void addPattern(final String pattern, final PathMappings<InetAddressSet> pathMappings)
throws IllegalArgumentException {
if (pattern != null && !pattern.isEmpty()) {
final int idx = pattern.indexOf('|');
final String addr = idx > 0 ? pattern.substring(0, idx) : pattern;
final String path = (idx > 0 && (pattern.length() > idx + 1)) ? pattern.substring(idx + 1) : "/*";
if (!addr.isEmpty()) {
final PathSpec pathSpec = PathMappings.asPathSpec(path);
InetAddressSet addresses = pathMappings.get(pathSpec);
if (addresses == null) {
addresses = new InetAddressSet();
pathMappings.put(pathSpec, addresses);
}
addresses.add(addr);
}
}
}
/**
* Helper method to check pattern syntax.
*
* @param pattern pattern to check for syntax errors
* @throws IllegalArgumentException
* when the pattern is malformed
*/
public static void checkPattern(final String pattern) throws IllegalArgumentException {
new InetPathAccessHandler().include(pattern);
}
@Override
protected boolean isAllowed(final InetAddress address, final HttpServletRequest request) {
return isAllowed(address, request.getPathInfo());
}
/**
* Check whether the given address and path are allowed by current rules.
*
* @param address
* the address to check
* @param path
* an eventual path string starting with "/"
* @return true when allowed
*/
protected boolean isAllowed(final InetAddress address, final String path) {
boolean allowed = true;
final String nonNullPath = path != null ? path : "/";
if (this.white.size() > 0) {
/* Non empty white list patterns : MUST match at least one of it */
allowed = false;
for (final MappedResource<InetAddressSet> mapping : this.white.getMatches(nonNullPath)) {
if (mapping.getResource().test(address)) {
allowed = true;
break;
}
}
}
if (allowed) {
/* Finally check against black list patterns even when the first step passed */
for (final MappedResource<InetAddressSet> mapping : this.black.getMatches(nonNullPath)) {
if (mapping.getResource().test(address)) {
allowed = false;
break;
}
}
}
return allowed;
}
@Override
public void dump(final Appendable out, final String indent) throws IOException {
this.dumpBeans(out, indent, this.white.getMappings(), this.black.getMappings());
}
}

55
source/net/yacy/http/Jetty9HttpServerImpl.java
View File

@ -27,15 +27,13 @@ package net.yacy.http;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.net.InetAddress;
import java.security.KeyStore;
import java.util.StringTokenizer;
import javax.net.ssl.KeyManagerFactory;
import javax.net.ssl.SSLContext;
import net.yacy.cora.util.ConcurrentLog;
import net.yacy.http.servlets.YaCyDefaultServlet;
import net.yacy.search.Switchboard;
import net.yacy.search.SwitchboardConstants;
import net.yacy.utils.PKCS12Tool;
import org.eclipse.jetty.http.HttpVersion;
import org.eclipse.jetty.server.Connector;
import org.eclipse.jetty.server.Handler;
@ -49,12 +47,18 @@ import org.eclipse.jetty.server.handler.ContextHandler;
import org.eclipse.jetty.server.handler.ContextHandlerCollection;
import org.eclipse.jetty.server.handler.DefaultHandler;
import org.eclipse.jetty.server.handler.HandlerList;
import org.eclipse.jetty.server.handler.IPAccessHandler;
import org.eclipse.jetty.server.handler.InetAccessHandler;
import org.eclipse.jetty.servlet.ServletHolder;
import org.eclipse.jetty.util.resource.Resource;
import org.eclipse.jetty.util.ssl.SslContextFactory;
import org.eclipse.jetty.webapp.WebAppContext;
import net.yacy.cora.util.ConcurrentLog;
import net.yacy.http.servlets.YaCyDefaultServlet;
import net.yacy.search.Switchboard;
import net.yacy.search.SwitchboardConstants;
import net.yacy.utils.PKCS12Tool;
/**
* class to embedded Jetty 9 http server into YaCy
*/
@ -196,33 +200,44 @@ public class Jetty9HttpServerImpl implements YaCyHttpServer {
// wrap all handlers
Handler crashHandler = new CrashProtectionHandler(server, allrequesthandlers);
// check server access restriction and add IPAccessHandler if restrictions are needed
// check server access restriction and add InetAccessHandler if restrictions are needed
// otherwise don't (to save performance)
String white = sb.getConfig("serverClient", "*");
if (!white.equals("*")) { // full ip (allowed ranges 0-255 or prefix 10.0-255,0,0-100 or 127.)
final String white = sb.getConfig("serverClient", "*");
if (!white.equals("*")) { // full ip (allowed ranges 0-255 or prefix 10.0-255,0,0-100 or CIDR notation 192.168.1.0/24)
final StringTokenizer st = new StringTokenizer(white, ",");
IPAccessHandler iphandler = new IPAccessHandler();
int i=0;
final InetAccessHandler whiteListHandler;
if (white.contains("|")) {
/*
* At least one pattern includes a path definition : we must use the
* InetPathAccessHandler as InetAccessHandler doesn't support path patterns
*/
whiteListHandler = new InetPathAccessHandler();
} else {
whiteListHandler = new InetAccessHandler();
}
int i = 0;
while (st.hasMoreTokens()) {
String ip = st.nextToken();
final String pattern = st.nextToken();
try {
iphandler.addWhite(ip); // accepts only ipv4
} catch (IllegalArgumentException nex) { // catch number format exception on non ipv4 input
whiteListHandler.include(pattern);
} catch (final IllegalArgumentException nex) { // catch format exception on wrong ip address pattern
ConcurrentLog.severe("SERVER", "Server Access Settings - IP filter: " + nex.getMessage());
continue;
}
i++;
}
if (i > 0) {
iphandler.addWhite("127.0.0.1"); // allow localhost (loopback addr)
iphandler.setHandler(crashHandler);
server.setHandler(iphandler);
ConcurrentLog.info("SERVER","activated IP access restriction to: [127.0.0.1," + white +"] (this works only correct with start parameter -Djava.net.preferIPv4Stack=true)");
final String loopbackAddress = InetAddress.getLoopbackAddress().getHostAddress();
whiteListHandler.include(loopbackAddress);
whiteListHandler.setHandler(crashHandler);
this.server.setHandler(whiteListHandler);
ConcurrentLog.info("SERVER","activated IP access restriction to: [" + loopbackAddress + "," + white +"]");
} else {
server.setHandler(crashHandler); // iphandler not needed
server.setHandler(crashHandler); // InetAccessHandler not needed
}
} else {
server.setHandler(crashHandler); // iphandler not needed
server.setHandler(crashHandler); // InetAccessHandler not needed
}
}

106
source/net/yacy/migration.java
View File

@ -30,6 +30,7 @@ import java.util.Iterator;
import java.util.List;
import java.util.Locale;
import java.util.Set;
import java.util.StringTokenizer;
import net.yacy.cora.order.Base64Order;
import net.yacy.cora.order.Digest;
@ -53,6 +54,9 @@ public class migration {
public static final double NEW_OVERLAYS =0.56504422;
public static final double IDX_HOST_VER =0.99007724; // api for index retrieval: host index
public static final double SSLPORT_CFG =1.67009578; // https port in cfg
/** Removal of deprecated IPAccessHandler for white list implementation (serverClient setting) */
public static final double NEW_IPPATTERNS = 1.92109489;
/**
* Migrates older configuratin to current version
@ -68,6 +72,9 @@ public class migration {
if(fromVer < NEW_OVERLAYS){
migrateDefaultFiles(sb);
}
if (fromVer < NEW_IPPATTERNS) {
migrateServerClientSetting(sb);
}
// use String.format to cut-off small rounding errors
ConcurrentLog.info("MIGRATION", "Migrating from "+ String.format(Locale.US, "%.8f",fromVer) + " to " + String.format(Locale.US, "%.8f",toVer));
if (fromVer < 0.47d) {
@ -293,6 +300,105 @@ public class migration {
}
}
/**
* Setting "serverClient" : migrate eventual address patterns using deprecated
* formats previously supported by the IPAccessHandler and IPAddressMap classes.
*/
public static void migrateServerClientSetting(final Switchboard sb) {
final String patternSeparator = ",";
final String white = sb.getConfig("serverClient", "*");
if (!white.equals("*")) {
final StringBuilder migrated = new StringBuilder();
boolean hasDeprecated = migrateIPAddressPatterns(patternSeparator, white, migrated);
if (hasDeprecated) {
sb.setConfig("serverClient", migrated.toString());
ConcurrentLog.info("MIGRATION", "Migrated serverClient setting from " + white + " to " + migrated);
}
}
}
/**
* Convert eventual address patterns using deprecated formats previously
* supported by the IPAccessHandler and IPAddressMap classes. All parameters
* must be not null.
*
* @param patternSeparator
* pattern separator
* @param patterns
* patterns to convert
* @param migrated
* the result of the conversion. Equals the patterns String when it
* contained no pattern using a deprecated format.
* @return true when patterns contained at least one pattern using a deprecated
* format.
*/
protected static boolean migrateIPAddressPatterns(final String patternSeparator, final String patterns,
final StringBuilder migrated) {
final StringTokenizer st = new StringTokenizer(patterns, patternSeparator);
boolean hasDeprecated = false;
while (st.hasMoreTokens()) {
final String pattern = st.nextToken();
int idx;
if (pattern.indexOf('|') > 0) {
idx = pattern.indexOf('|');
} else {
idx = pattern.indexOf('/');
if (idx >= 0) {
/*
* First "/" character of the URI pattern used to separate it from the internet
* address. But it can now be used in CIDR notation
*/
final String intPart = pattern.substring(idx + 1);
try {
int intValue = Integer.parseInt(intPart);
if (intValue >= 0 && intValue <= 128) {
idx = -1;
} else {
/* No a valid CIDR notation : maybe a path with only numbers? */
hasDeprecated = true;
}
} catch (final NumberFormatException e) {
hasDeprecated = true;
}
}
}
String addr = idx > 0 ? pattern.substring(0, idx) : pattern;
String path = idx > 0 ? pattern.substring(idx) : "/*";
if (addr.endsWith(".")) {
/*
* Migrating prefix wildcard specification range format (e.g. "10.10." becomes
* "10.10.0.0-10.10.255.255") .
*/
hasDeprecated = true;
final String[] parts = addr.split("\\.");
final StringBuilder migratedAddr = new StringBuilder(addr.substring(0, addr.length() - 1));
for (int i = parts.length; i < 4; i++) {
migratedAddr.append(".0");
}
migratedAddr.append("-").append(addr.substring(0, addr.length() - 1));
for (int i = parts.length; i < 4; i++) {
migratedAddr.append(".255");
}
addr = migratedAddr.toString();
}
if (path.startsWith("|") || path.startsWith("/*.")) {
path = path.substring(1);
}
if (migrated.length() > 0) {
migrated.append(patternSeparator);
}
migrated.append(addr);
if (!"/*".equals(path)) {
migrated.append("|").append(path);
}
}
return hasDeprecated;
}
/**
* Reindex embedded solr index
* - all documents with inactive fields (according to current schema)

11
source/net/yacy/yacy.java
View File

@ -707,17 +707,6 @@ public final class yacy {
try {
fis = new FileInputStream(configFile);
p.load(fis);
// Test for server access restriction (is implemented using Jetty IPaccessHandler which does not support IPv6
// try to disable IPv6
String teststr = p.getProperty("serverClient", "*");
if (!teststr.equals("*")) {
// testing on Win-8 showed this property has to be set befor Switchboard starts
// and seems to be sensitive (or time critical) if other code had been executed before this (don't know why ... ?)
System.setProperty("java.net.preferIPv6Addresses", "false");
System.setProperty("java.net.preferIPv4Stack", "true"); // DO NOT PREFER IPv6, i.e. freifunk uses ipv6 only and host resolving does not work
teststr = System.getProperty("java.net.preferIPv4Stack");
System.out.println("set system property java.net.preferIP4Stack=" + teststr);
}
// test for yacy already running
if (lockFile.exists()) { // another instance running? VM crash? User will have to care about this

343
test/java/net/yacy/http/InetPathAccessHandlerTest.java Normal file
View File

@ -0,0 +1,343 @@
// InetPathAccessHandlerTest.java
// Copyright 2017 by luccioman; https://github.com/luccioman
//
// This is a part of YaCy, a peer-to-peer based web search engine
//
// LICENSE
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation; either version 2 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
package net.yacy.http;
import java.net.InetAddress;
import java.net.UnknownHostException;
import org.junit.Assert;
import org.junit.Test;
/**
* Unit tests for the {@link InetPathAccessHandler} class.
*/
public class InetPathAccessHandlerTest {
/**
* Check the handler allow the given ip/path pairs.
*
* @param handler
* the handler to test. Must not be null.
* @param ipAndPaths
* array of ip address and path pairs. Must not be null.
* @throws UnknownHostException
* when a test address is incorrect.
*/
private void assertAllowed(final InetPathAccessHandler handler, final String[][] ipAndPaths)
throws UnknownHostException {
for (final String[] ipAndPath : ipAndPaths) {
final String ip = ipAndPath[0];
final String path = ipAndPath[1];
Assert.assertTrue("Should allow " + ip + path, handler.isAllowed(InetAddress.getByName(ip), path));
}
}
/**
* Check the handler dos not allow the given ip/path pairs.
*
* @param handler
* the handler to test. Must not be null.
* @param ipAndPaths
* array of ip address and path pairs. Must not be null.
* @throws UnknownHostException
* when a test address is incorrect.
*/
private void assertRejected(final InetPathAccessHandler handler, final String[][] ipAndPaths)
throws UnknownHostException {
for (final String[] ipAndPath : ipAndPaths) {
final String ip = ipAndPath[0];
final String path = ipAndPath[1];
Assert.assertFalse("Should not allow " + ip + path, handler.isAllowed(InetAddress.getByName(ip), path));
}
}
/**
* Test inclusion with a single white listed IPv4 address.
*
* @throws UnknownHostException
* when a test address is incorrect. Should not happen.
*/
@Test
public void testIncludeSingleIPv4() throws UnknownHostException {
final InetPathAccessHandler handler = new InetPathAccessHandler();
handler.include("10.10.1.2");
final String[][] allowed = { { "10.10.1.2", "/" }, // matching address, root path
{ "10.10.1.2", "/foo/bar" }, // matching address, non root path
{ "10.10.1.2", null } // matching address, no path information provided
};
this.assertAllowed(handler, allowed);
final String[][] rejected = { { "10.10.1.3", "/" }, // non matching address, root path
{ null, null } // no address nor path information provided
};
this.assertRejected(handler, rejected);
}
/**
* Test inclusion with a single white listed IPv6 address.
*
* @throws UnknownHostException
* when a test address is incorrect. Should not happen.
*/
@Test
public void testIncludeSingleIPv6() throws UnknownHostException {
final InetPathAccessHandler handler = new InetPathAccessHandler();
handler.include("2001:db8::ff00:42:8329");
final String[][] allowed = { { "2001:db8::ff00:42:8329", "/" }, // matching address, root path
{ "2001:0db8:0000:0000:0000:ff00:0042:8329", "/" }, // matching address in long representation, root
// path
{ "2001:db8::ff00:42:8329", "/foo/bar" }, // matching address, non root path
{ "2001:db8::ff00:42:8329", null } // matching address, no path information provided
};
this.assertAllowed(handler, allowed);
final String[][] rejected = { { "2001:db8::ff00:42:8539", "/" }, // non matching address, root path
{ null, null } // no address nor path information provided
};
this.assertRejected(handler, rejected);
}
/**
* Test inclusion with a single white listed IPV4 address and path.
*
* @throws UnknownHostException
* when a test address is incorrect. Should not happen.
*/
@Test
public void testIncludeSingleAddressAndPath() throws UnknownHostException {
final InetPathAccessHandler handler = new InetPathAccessHandler();
handler.include("10.10.1.2|/foo/bar");
final String[][] allowed = { { "10.10.1.2", "/foo/bar" } // matching address, matching path
};
this.assertAllowed(handler, allowed);
final String[][] rejected = { { "10.10.1.3", "/" }, // non matching address, non matching path
{ "10.10.1.3", "/foo/bar" }, // non matching address, even if matching path
{ "10.10.1.2", "/" }, // matching address, but non matching root path
{ "10.10.1.2", "/foo" }, // matching address, but non matching parent path
{ "10.10.1.2", "/foo/" }, // matching address, but non matching parent path
{ "10.10.1.2", "/foo/wrong" }, // matching address, but non matching sub path
{ "10.10.1.2", "/foo/bar/file.txt" } // matching address, but non matching sub path with file
};
this.assertRejected(handler, rejected);
}
/**
* Test inclusion with a single white listed IPV4 address and wildcard path.
*
* @throws UnknownHostException
* when a test address is incorrect. Should not happen.
*/
@Test
public void testIncludeSingleAddressAndWildcardPath() throws UnknownHostException {
final InetPathAccessHandler handler = new InetPathAccessHandler();
handler.include("10.10.1.2|/foo/*");
final String[][] allowed = { { "10.10.1.2", "/foo/bar" }, // matching address, matching sub path
{ "10.10.1.2", "/foo/bar/sub" }, // matching address, matching sub path
{ "10.10.1.2", "/foo/file.txt" }, // matching address, matching sub path with file
{ "10.10.1.2", "/foo" }, // matching address, matching path
};
this.assertAllowed(handler, allowed);
final String[][] rejected = { { "10.10.1.3", "/" }, // non matching address, non matching path
{ "10.10.1.3", "/foo/bar" }, // non matching address, event if matching path
{ "10.10.1.2", "/" }, // matching address, but non matching root path
{ "10.10.1.2", null }, // matching address, but no path information provided
{ null, "/foo/bar" } // no address provided, event if matching path
};
this.assertRejected(handler, rejected);
}
/**
* Test inclusion with a single white listed IPV4 address and wildcard path
* suffix.
*
* @throws UnknownHostException
* when a test address is incorrect. Should not happen.
*/
@Test
public void testIncludeSingleAddressAndWildcardSuffix() throws UnknownHostException {
final InetPathAccessHandler handler = new InetPathAccessHandler();
handler.include("10.10.1.2|*.html");
final String[][] allowed = { { "10.10.1.2", "/index.html" }, // matching address, matching file path
{ "10.10.1.2", "/foo/bar/index.html" }, // matching address, matching file with parent path
};
this.assertAllowed(handler, allowed);
final String[][] rejected = { { "10.10.1.3", "/" }, // non matching address, non matching path
{ "10.10.1.3", "/index.html" }, // non matching address, event if matching file path
{ "10.10.1.2", "/" }, // matching address, but non matching root path
{ "10.10.1.2", "/index.txt" }, // matching address, but non matching file path
{ "10.10.1.2", null }, // matching address, but no path information provided
{ null, "/index.html" } // no address provided, event if matching path
};
this.assertRejected(handler, rejected);
}
/**
* Test inclusion with ranges of white listed addresses.
*
* @throws UnknownHostException
* when a test address is incorrect. Should not happen.
*/
@Test
public void testIncludeRanges() throws UnknownHostException {
final InetPathAccessHandler handler = new InetPathAccessHandler();
handler.include("10.10.1.1-255"); // legacy IPv4 range format used by IPAddressMap
handler.include("192.168.128.0-192.168.128.255"); // inclusive range of IPv4 addresses
handler.include("2001:db8::ff00:42:8329-2001:db8::ff00:42:ffff"); // inclusive range of IPv6 addresses
handler.include("192.168.1.0/24"); // CIDR notation on IPv4
handler.include("2001:db8::aaaa:0:0/96"); // CIDR notation on IPv6
final String[][] allowed = { { "10.10.1.1", "/" }, // matching legacy IPv4 range
{ "10.10.1.255", "/" }, // matching legacy IPv4 range
{ "192.168.128.0", "/" }, // matching second range of IPv4 addresses
{ "192.168.128.255", "/" }, // matching second range of IPv4 addresses
{ "2001:db8::ff00:42:8329", "/" }, // matching IPv6 range
{ "2001:db8::ff00:42:99ff", "/" }, // matching IPv6 range
{ "192.168.1.0", "/" }, // matching IPv4 CIDR notation range
{ "192.168.1.255", "/" }, // matching IPv4 CIDR notation range
{ "2001:db8::aaaa:1:1", "/" }, // matching IPv6 CIDR notation range
{ "2001:db8::aaaa:ffff:ffff", "/" } // matching IPv6 CIDR notation range
};
this.assertAllowed(handler, allowed);
final String[][] rejected = { { "10.9.1.1", "/" }, { "10.10.2.1", "/" }, { "192.168.127.1", "/" },
{ "2001:db8::ff00:43:1234", "/" }, { "192.168.2.1", "/" }, { "2001:db8::aabb:ffff:ffff", "/" } };
this.assertRejected(handler, rejected);
}
/**
* Test inclusion with ranges of white listed addresses associated with wildcard
* paths.
*
* @throws UnknownHostException
* when a test address is incorrect. Should not happen.
*/
@Test
public void testIncludeRangesAndWildcardPaths() throws UnknownHostException {
final InetPathAccessHandler handler = new InetPathAccessHandler();
handler.include("10.10.1.1-255|/foo/*"); // legacy IPv4 range format used by IPAddressMap
handler.include("192.168.128.0-192.168.128.255|/path/*"); // inclusive range of IPv4 adresses
handler.include("2001:db8::ff00:42:8329-2001:db8::ff00:42:ffff|/root/*"); // inclusive range of IPv6 adresses
handler.include("192.168.1.0/24|/www/*"); // CIDR notation
final String[][] allowed = { { "10.10.1.1", "/foo/bar" }, // matching legacy IPv4 range and path
{ "10.10.1.255", "/foo/bar" }, // matching legacy IPv4 range and path
{ "192.168.128.0", "/path/index.html" }, // matching second range of IPv4 addresses and path
{ "192.168.128.255", "/path/file.txt" }, // matching second range of IPv4 addresses and path
{ "2001:db8::ff00:42:8329", "/root/index.txt" }, // matching IPv6 range and path
{ "2001:db8::ff00:42:99ff", "/root/image.jpg" }, // matching IPv6 range and path
{ "192.168.1.0", "/www/resource" }, // matching IPv4 CIDR notation range and path
{ "192.168.1.255", "/www/home" } }; // matching IPv4 CIDR notation range and path
this.assertAllowed(handler, allowed);
final String[][] rejected = { { "10.9.1.1", "/" }, { "10.9.1.1", "/foo/bar" }, { "10.10.2.1", "/" },
{ "10.10.2.1", "/foo/bar" }, { "192.168.127.1", "/" }, { "192.168.127.1", "/path/index.html" },
{ "2001:db8::ff00:43:1234", "/" }, { "2001:db8::ff00:43:1234", "/root/index.txt" },
{ "192.168.2.1", "/" }, { "192.168.2.1", "/www/content" } };
this.assertRejected(handler, rejected);
}
/**
* Test inclusion with multiple patterns using the same path
*
* @throws UnknownHostException
* when a test address is incorrect. Should not happen.
*/
@Test
public void testIncludeMultiplePatternsOnSamePath() throws UnknownHostException {
final InetPathAccessHandler handler = new InetPathAccessHandler();
handler.include("10.10.1.1|/foo/bar"); // a single address pattern
handler.include("192.168.128.0-192.168.128.255|/foo/bar"); // inclusive range of IPv4 adresses
final String[][] allowed = { { "10.10.1.1", "/foo/bar" }, // matching single address pattern
{ "192.168.128.0", "/foo/bar" }, { "192.168.128.255", "/foo/bar" } // matching range pattern
};
this.assertAllowed(handler, allowed);
final String[][] rejected = { { "10.10.1.1", "/" }, // matching single address pattern bu root path
{ "127.0.0.1", "/" }, // non matching address
};
this.assertRejected(handler, rejected);
}
/**
* Test exclusion with a single white listed IPV4 address and path.
*
* @throws UnknownHostException
* when a test address is incorrect. Should not happen.
*/
@Test
public void testExcludeSingleAddressAndPath() throws UnknownHostException {
final InetPathAccessHandler handler = new InetPathAccessHandler();
handler.exclude("10.10.1.2|/foo/bar");
final String[][] allowed = { { "10.10.1.3", "/" }, // non matching address, non matching path
{ "10.10.1.3", "/foo/bar" }, // non matching address, even if matching path
{ "10.10.1.2", "/" }, // matching address, but non matching root path
{ "10.10.1.2", "/foo" }, // matching address, but non matching parent path
{ "10.10.1.2", "/foo/" }, // matching address, but non matching parent path
{ "10.10.1.2", "/foo/wrong" }, // matching address, but non matching sub path
{ "10.10.1.2", "/foo/bar/file.txt" } // matching address, but non matching sub path with file
};
this.assertAllowed(handler, allowed);
final String[][] rejected = { { "10.10.1.2", "/foo/bar" } // matching address, matching path
};
this.assertRejected(handler, rejected);
}
/**
* Test inclusion and exclusion rules applied on the same address
*
* @throws UnknownHostException
* when a test address is incorrect. Should not happen.
*/
@Test
public void testIncludeExcludeOnSameAddress() throws UnknownHostException {
final InetPathAccessHandler handler = new InetPathAccessHandler();
handler.include("10.10.1.1-10.10.1.255"); // include a range of addresses without path restrictions
handler.exclude("10.10.1.2|/foo/bar"); // exclude a specific address and path
final String[][] allowed = { { "10.10.1.3", "/" }, // matching included addresses range
{ "10.10.1.2", "/" }, // matching excluded address, but non matching root path
{ "10.10.1.2", "/foo" }, // matching excluded address, but non matching parent path
{ "10.10.1.2", "/foo/wrong" }, // matching excluded address, but non matching sub path
{ "10.10.1.2", "/foo/bar/file.txt" } // matching excluded address, but non matching sub path with file
};
this.assertAllowed(handler, allowed);
final String[][] rejected = { { "10.10.1.2", "/foo/bar" } // matching excluded address and path
};
this.assertRejected(handler, rejected);
}
}

92
test/java/net/yacy/migrationTest.java Normal file
View File

@ -0,0 +1,92 @@
// migrationTest.java
// Copyright 2017 by luccioman; https://github.com/luccioman
//
// This is a part of YaCy, a peer-to-peer based web search engine
//
// LICENSE
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation; either version 2 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
package net.yacy;
import java.util.HashMap;
import java.util.Map;
import java.util.Map.Entry;
import org.junit.Assert;
import org.junit.Test;
/**
* Unit tests for the {@link migration} class.
*/
public class migrationTest {
/**
* Testing the conversion of IP addresses patterns
*/
@Test
public void testMigrateIPAddressPatterns() {
final String patternSeparator = ",";
final String[] nonDeprecatedPatterns = { "*", // match all (default)
"10.10.1.2,2001:db8::ff00:42:8329", // single IPv4 and IPv6 addresses
"10.10.1.2|/foo/bar,2001:db8::ff00:42:8329|/foo/bar", // single IPv4 and IPv6 addresses with path
"192.168.1.1-192.168.1.10,2001:db8::ff00:42:8330-2001:db8::ff00:42:83ff", // IPv4 and IPv6 addresses
// ranges
"192.168.1.1-192.168.1.10|/path,2001:db8::ff00:42:8330-2001:db8::ff00:42:83ff|/path", // IPv4 and IPv6 addresses ranges with path
"127.0.0.1/8,192.168.1.0/24,2001:db8::aaaa:0:0/96,::1/128", // IPv4 and IPv6 addresses ranges defined using CIDR notation
"127.0.0.1/8|*.html,192.168.1.0/24|/foo/bar,2001:db8::aaaa:0:0/96|/foo/bar,::1/128|*.html", // IPv4 and IPv6 addresses ranges defined using CIDR notation with path
"192.168.3.0-255", // legacy IPv4 addresses range format
"192.168.3.0-255|/foo/bar,192.168.1.0-255|*.html", // legacy IPv4 addresses range format with path
};
final StringBuilder migrated = new StringBuilder();
for (final String patterns : nonDeprecatedPatterns) {
migrated.setLength(0);
Assert.assertFalse("Should not be detected as deprecated : " + patterns,
migration.migrateIPAddressPatterns(patternSeparator, patterns, migrated));
Assert.assertEquals(patterns, migrated.toString());
}
final Map<String, String> deprecatedToMigrated = new HashMap<>();
/* old IPv4 wildcard notation */
deprecatedToMigrated.put("127.", "127.0.0.0-127.255.255.255");
/* old IPv4 wildcard notation */
deprecatedToMigrated.put("192.168.", "192.168.0.0-192.168.255.255");
/* old IPv4 wildcard notation */
deprecatedToMigrated.put("192.168.1.", "192.168.1.0-192.168.1.255");
/* IPV4 address and old style path pattern */
deprecatedToMigrated.put("192.168.1.1/foo/bar,127.0.0.1/*.txt", "192.168.1.1|/foo/bar,127.0.0.1|*.txt");
/* old IPv4 wildcard notation and old style path pattern */
deprecatedToMigrated.put("192.168./foo/bar,127./*.txt", "192.168.0.0-192.168.255.255|/foo/bar,127.0.0.0-127.255.255.255|*.txt");
/* old IPv4 wildcard notation and new style path pattern */
deprecatedToMigrated.put("192.168.|/foo/bar,127.|*.txt", "192.168.0.0-192.168.255.255|/foo/bar,127.0.0.0-127.255.255.255|*.txt");
/* mixed deprecated and non deprecated patterns */
deprecatedToMigrated.put("10.10.1.2,2001:db8::ff00:42:8329|/foo/bar,192.168.|/foo/bar,192.168.1.0/24,127.|*.txt",
"10.10.1.2,2001:db8::ff00:42:8329|/foo/bar,192.168.0.0-192.168.255.255|/foo/bar,192.168.1.0/24,127.0.0.0-127.255.255.255|*.txt");
for (final Entry<String, String> entry : deprecatedToMigrated.entrySet()) {
migrated.setLength(0);
Assert.assertTrue("Should be detected as deprecated : " + entry.getKey(),
migration.migrateIPAddressPatterns(patternSeparator, entry.getKey(), migrated));
Assert.assertEquals(entry.getValue(), migrated.toString());
}
}
}