Marown/yacy_search_server
1
0
Fork 0
mirror of https://github.com/yacy/yacy_search_server.git synced 2024-09-19 00:01:41 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity
700218846c
yacy_search_server/htroot/BlacklistCleaner_p.java
orbiter 1d8d51075c refactoring: 
- removed the plasma package. The name of that package came from a very early pre-version of YaCy, even before YaCy was named AnomicHTTPProxy. The Proxy project introduced search for cache contents using class files that had been developed during the plasma project. Information from 2002 about plasma can be found here:
http://web.archive.org/web/20020802110827/http://anomic.de/AnomicPlasma/index.html
We stil have one class that comes mostly unchanged from the plasma project, the Condenser class. But this is now part of the document package and all other classes in the plasma package can be assigned to other packages.
- cleaned up the http package: better structure of that class and clean isolation of server and client classes. The old HTCache becomes part of the client sub-package of http.
- because the plasmaSwitchboard is now part of the search package all servlets had to be touched to declare a different package source.

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@6232 6c8d7289-2bf4-0310-a012-ef5d649a1542
2009-07-19 20:37:44 +00:00

349 lines
14 KiB
Java
Raw Blame History

// BlacklistCleaner_p.java
// -----------------------
// part of YaCy
// (C) by Michael Peter Christen; mc@yacy.net
// first published on http://www.anomic.de
// Frankfurt, Germany, 2004
//
// This File is contributed by Franz Brausze
//
// $LastChangedDate: 2007-01-27 14:07:54 +0000 (Sa, 27 Jan 2007) $
// $LastChangedRevision: 3217 $
// $LastChangedBy: karlchenofhell $
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation; either version 2 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
// You must compile this file with
// javac -classpath .:../classes BlacklistCleaner_p.java
// if the shell's current path is HTROOT
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.io.PrintWriter;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.regex.PatternSyntaxException;
import de.anomic.data.AbstractBlacklist;
import de.anomic.data.Blacklist;
import de.anomic.data.DefaultBlacklist;
import de.anomic.data.listManager;
import de.anomic.http.metadata.RequestHeader;
import de.anomic.search.Switchboard;
import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch;
import de.anomic.yacy.logging.Log;
public class BlacklistCleaner_p {
private static final String RESULTS = "results_";
private static final String DISABLED = "disabled_";
private static final String BLACKLISTS = "blacklists_";
private static final String ENTRIES = "entries_";
private static final int ERR_TWO_WILDCARDS_IN_HOST = 0;
private static final int ERR_SUBDOMAIN_XOR_WILDCARD = 1;
private static final int ERR_PATH_REGEX = 2;
private static final int ERR_WILDCARD_BEGIN_OR_END = 3;
private static final int ERR_HOST_WRONG_CHARS = 4;
private static final int ERR_DOUBLE_OCCURANCE = 5;
private final static String BLACKLIST_FILENAME_FILTER = "^.*\\.black$";
public static final Class<?>[] supportedBLEngines = {
DefaultBlacklist.class
};
public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) {
final serverObjects prop = new serverObjects();
// initialize the list manager
listManager.switchboard = (Switchboard) env;
listManager.listsPath = new File(env.getRootPath(), env.getConfig("listManager.listsPath", "DATA/LISTS"));
String blacklistToUse = null;
// getting the list of supported blacklist types
final String supportedBlacklistTypesStr = AbstractBlacklist.BLACKLIST_TYPES_STRING;
final String[] supportedBlacklistTypes = supportedBlacklistTypesStr.split(",");
if (post == null) {
prop.put("results", "0");
putBlacklists(prop, listManager.getDirListing(listManager.listsPath, BLACKLIST_FILENAME_FILTER), blacklistToUse);
return prop;
}
if (post.containsKey("listNames")) {
blacklistToUse = post.get("listNames");
if (blacklistToUse.length() == 0 || !listManager.listSetContains("listManager.listsPath", blacklistToUse))
prop.put("results", "2");
}
putBlacklists(prop, listManager.getDirListing(listManager.listsPath, BLACKLIST_FILENAME_FILTER), blacklistToUse);
if (blacklistToUse != null) {
prop.put("results", "1");
if (post.containsKey("delete")) {
prop.put(RESULTS + "modified", "1");
prop.put(RESULTS + "modified_delCount", removeEntries(blacklistToUse, supportedBlacklistTypes, getByPrefix(post, "select", true, true)));
} else if (post.containsKey("alter")) {
prop.put(RESULTS + "modified", "2");
prop.put(RESULTS + "modified_alterCount", alterEntries(blacklistToUse, supportedBlacklistTypes, getByPrefix(post, "select", true, false), getByPrefix(post, "entry", false, false)));
}
// list illegal entries
final HashMap<String, Integer> ies = getIllegalEntries(blacklistToUse, Switchboard.urlBlacklist);
prop.put(RESULTS + "blList", blacklistToUse);
prop.put(RESULTS + "entries", ies.size());
prop.putHTML(RESULTS + "blEngine", Switchboard.urlBlacklist.getEngineInfo());
prop.put(RESULTS + "disabled", (ies.size() == 0) ? "1" : "0");
if (ies.size() > 0) {
prop.put(RESULTS + DISABLED + "entries", ies.size());
int i = 0;
String s;
for (Entry<String, Integer> entry: ies.entrySet()) {
s = entry.getKey();
prop.put(RESULTS + DISABLED + ENTRIES + i + "_error", entry.getValue().longValue());
prop.putHTML(RESULTS + DISABLED + ENTRIES + i + "_entry", s);
i++;
}
}
}
return prop;
}
private static void putBlacklists(final serverObjects prop, final List<String> lists, final String selected) {
boolean supported = false;
for (int i=0; i<supportedBLEngines.length && !supported; i++) {
supported |= (Switchboard.urlBlacklist.getClass() == supportedBLEngines[i]);
}
if (supported) {
if (lists.size() > 0) {
prop.put("disabled", "0");
prop.put(DISABLED + "blacklists", lists.size());
int count = 0;
for (String list : lists) {
prop.putHTML(DISABLED + BLACKLISTS + count + "_name", list);
prop.put(DISABLED + BLACKLISTS + count + "_selected", (list.equals(selected)) ? "1" : "0");
count++;
}
} else {
prop.put("disabled", "2");
}
} else {
prop.put("disabled", "1");
for (int i=0; i<supportedBLEngines.length; i++) {
prop.putHTML(DISABLED + "engines_" + i + "_name", supportedBLEngines[i].getName());
}
prop.put(DISABLED + "engines", supportedBLEngines.length);
}
}
private static String[] getByPrefix(final serverObjects post, final String prefix, final boolean useKeys, final boolean useHashSet) {
Collection<String> r;
if (useHashSet) {
r = new HashSet<String>();
} else {
r = new ArrayList<String>();
}
String s;
if (useKeys) {
final Iterator<String> it = post.keySet().iterator();
while (it.hasNext()) {
if ((s = it.next()).indexOf(prefix) == 0) {
r.add(s.substring(prefix.length()));
}
}
} else {
final Iterator<Map.Entry<String, String>> it = post.entrySet().iterator();
Map.Entry<String, String> entry;
while (it.hasNext()) {
entry = it.next();
if (entry.getKey().indexOf(prefix) == 0) {
r.add(entry.getValue());
}
}
}
return r.toArray(new String[r.size()]);
}
private static HashMap<String, Integer>/* entry, error-code */ getIllegalEntries(final String blacklistToUse, final Blacklist blEngine) {
final HashMap<String, Integer> r = new HashMap<String, Integer>();
final HashSet<String> ok = new HashSet<String>();
final ArrayList<String> list = listManager.getListArray(new File(listManager.listsPath, blacklistToUse));
final Iterator<String> it = list.iterator();
String s, host, path;
if (blEngine instanceof DefaultBlacklist) {
int slashPos;
while (it.hasNext()) {
s = (it.next()).trim();
// check for double-occurance
if (ok.contains(s)) {
r.put(s, Integer.valueOf(ERR_DOUBLE_OCCURANCE));
continue;
}
ok.add(s);
if ((slashPos = s.indexOf("/")) == -1) {
host = s;
path = ".*";
} else {
host = s.substring(0, slashPos);
path = s.substring(slashPos + 1);
}
final int i = host.indexOf("*");
// check whether host begins illegally
if (!host.matches("([A-Za-z0-9_-]+|\\*)(\\.([A-Za-z0-9_-]+|\\*))*")) {
if (i == 0 && host.length() > 1 && host.charAt(1) != '.') {
r.put(s, Integer.valueOf(ERR_SUBDOMAIN_XOR_WILDCARD));
continue;
}
r.put(s, Integer.valueOf(ERR_HOST_WRONG_CHARS));
continue;
}
// in host-part only full sub-domains may be wildcards
if (host.length() > 0 && i > -1) {
if (!(i == 0 || i == host.length() - 1)) {
r.put(s, Integer.valueOf(ERR_WILDCARD_BEGIN_OR_END));
continue;
}
if (i == host.length() - 1 && host.length() > 1 && host.charAt(i - 1) != '.') {
r.put(s, Integer.valueOf(ERR_SUBDOMAIN_XOR_WILDCARD));
continue;
}
}
// check for double-occurences of "*" in host
if (host.indexOf("*", i + 1) > -1) {
r.put(s, Integer.valueOf(ERR_TWO_WILDCARDS_IN_HOST));
continue;
}
// check for errors on regex-compiling path
try {
Pattern.compile(path);
} catch (final PatternSyntaxException e) {
r.put(s, Integer.valueOf(ERR_PATH_REGEX));
continue;
}
}
}
return r;
}
private static int removeEntries(final String blacklistToUse, final String[] supportedBlacklistTypes, final String[] entries) {
// load blacklist data from file
final ArrayList<String> list = listManager.getListArray(new File(listManager.listsPath, blacklistToUse));
boolean listChanged = false;
// delete the old entry from file
String s;
for (int i=0; i<entries.length; i++) {
s = entries[i];
if (list != null){
// getting rid of escape characters which make it impossible to
// properly use contains()
if (s.contains("\\\\")) {
s = s.replaceAll(Pattern.quote("\\\\"), Matcher.quoteReplacement("\\"));
}
if (list.contains(s)) {
list.remove(s);
listChanged = true;
}
}
// remove the entry from the running blacklist engine
for (int blTypes=0; blTypes < supportedBlacklistTypes.length; blTypes++) {
if (listManager.listSetContains(supportedBlacklistTypes[blTypes] + ".BlackLists", blacklistToUse)) {
final String host = (s.indexOf("/") == -1) ? s : s.substring(0, s.indexOf("/"));
final String path = (s.indexOf("/") == -1) ? ".*" : s.substring(s.indexOf("/") + 1);
try {
Switchboard.urlBlacklist.remove(supportedBlacklistTypes[blTypes],
host,path);
} catch (final RuntimeException e) {
//System.err.println(e.getMessage() + ": " + host + "/" + path);
Log.logSevere("BLACKLIST-CLEANER", e.getMessage() + ": " + host + "/" + path);
}
}
}
}
if (listChanged){
listManager.writeList(new File(listManager.listsPath, blacklistToUse), list.toArray(new String[list.size()]));
}
return entries.length;
}
private static int alterEntries(
final String blacklistToUse,
final String[] supportedBlacklistTypes,
final String[] oldE,
final String[] newE) {
removeEntries(blacklistToUse, supportedBlacklistTypes, oldE);
PrintWriter pw = null;
try {
pw = new PrintWriter(new FileWriter(new File(listManager.listsPath, blacklistToUse), true));
String host, path;
for (int i=0, pos; i<newE.length; i++) {
pos = newE[i].indexOf("/");
if (pos < 0) {
host = newE[i];
path = ".*";
} else {
host = newE[i].substring(0, pos);
path = newE[i].substring(pos + 1);
}
pw.println(host + "/" + path);
for (int blTypes=0; blTypes < supportedBlacklistTypes.length; blTypes++) {
if (listManager.listSetContains(supportedBlacklistTypes[blTypes] + ".BlackLists",blacklistToUse)) {
Switchboard.urlBlacklist.add(
supportedBlacklistTypes[blTypes],
host,
path);
}
}
}
pw.close();
} catch (final IOException e) {
Log.logSevere("BLACKLIST-CLEANER", "error on writing altered entries to blacklist", e);
}
return newE.length;
}
}
Reference in New Issue View Git Blame Copy Permalink