mirror of
https://github.com/yacy/yacy_search_server.git
synced 2024-09-19 00:01:41 +02:00
ce1adf9955
high-performance search query situations as seen in yacy-metager integration showed deadlock situation caused by synchronization effects inside of sun.java code. It appears that the logger is not completely safe against deadlock situations in concurrent calls of the logger. One possible solution would be a outside-synchronization with 'synchronized' statements, but that would further apply blocking on all high-efficient methods that call the logger. It is much better to do a non-blocking hand-over of logging lines and work off log entries with a concurrent log writer. This also disconnects IO operations from logging, which can also cause IO operation when a log is written to a file. This commit not only moves the logger from kelondro to yacy.logging, it also inserts the concurrency methods to realize non-blocking logging. git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@6078 6c8d7289-2bf4-0310-a012-ef5d649a1542
349 lines
14 KiB
Java
349 lines
14 KiB
Java
// BlacklistCleaner_p.java
|
|
// -----------------------
|
|
// part of YaCy
|
|
// (C) by Michael Peter Christen; mc@yacy.net
|
|
// first published on http://www.anomic.de
|
|
// Frankfurt, Germany, 2004
|
|
//
|
|
// This File is contributed by Franz Brausze
|
|
//
|
|
// $LastChangedDate: 2007-01-27 14:07:54 +0000 (Sa, 27 Jan 2007) $
|
|
// $LastChangedRevision: 3217 $
|
|
// $LastChangedBy: karlchenofhell $
|
|
//
|
|
// This program is free software; you can redistribute it and/or modify
|
|
// it under the terms of the GNU General Public License as published by
|
|
// the Free Software Foundation; either version 2 of the License, or
|
|
// (at your option) any later version.
|
|
//
|
|
// This program is distributed in the hope that it will be useful,
|
|
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
// GNU General Public License for more details.
|
|
//
|
|
// You should have received a copy of the GNU General Public License
|
|
// along with this program; if not, write to the Free Software
|
|
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
|
|
|
// You must compile this file with
|
|
// javac -classpath .:../classes BlacklistCleaner_p.java
|
|
// if the shell's current path is HTROOT
|
|
|
|
import java.io.File;
|
|
import java.io.FileWriter;
|
|
import java.io.IOException;
|
|
import java.io.PrintWriter;
|
|
import java.util.ArrayList;
|
|
import java.util.Collection;
|
|
import java.util.HashMap;
|
|
import java.util.HashSet;
|
|
import java.util.Iterator;
|
|
import java.util.List;
|
|
import java.util.Map;
|
|
import java.util.Map.Entry;
|
|
import java.util.regex.Matcher;
|
|
import java.util.regex.Pattern;
|
|
import java.util.regex.PatternSyntaxException;
|
|
|
|
import de.anomic.data.AbstractBlacklist;
|
|
import de.anomic.data.Blacklist;
|
|
import de.anomic.data.DefaultBlacklist;
|
|
import de.anomic.data.listManager;
|
|
import de.anomic.http.httpRequestHeader;
|
|
import de.anomic.plasma.plasmaSwitchboard;
|
|
import de.anomic.server.serverObjects;
|
|
import de.anomic.server.serverSwitch;
|
|
import de.anomic.yacy.logging.Log;
|
|
|
|
public class BlacklistCleaner_p {
|
|
|
|
private static final String RESULTS = "results_";
|
|
private static final String DISABLED = "disabled_";
|
|
private static final String BLACKLISTS = "blacklists_";
|
|
private static final String ENTRIES = "entries_";
|
|
|
|
private static final int ERR_TWO_WILDCARDS_IN_HOST = 0;
|
|
private static final int ERR_SUBDOMAIN_XOR_WILDCARD = 1;
|
|
private static final int ERR_PATH_REGEX = 2;
|
|
private static final int ERR_WILDCARD_BEGIN_OR_END = 3;
|
|
private static final int ERR_HOST_WRONG_CHARS = 4;
|
|
private static final int ERR_DOUBLE_OCCURANCE = 5;
|
|
|
|
private final static String BLACKLIST_FILENAME_FILTER = "^.*\\.black$";
|
|
|
|
public static final Class<?>[] supportedBLEngines = {
|
|
DefaultBlacklist.class
|
|
};
|
|
|
|
public static serverObjects respond(final httpRequestHeader header, final serverObjects post, final serverSwitch<?> env) {
|
|
final serverObjects prop = new serverObjects();
|
|
|
|
// initialize the list manager
|
|
listManager.switchboard = (plasmaSwitchboard) env;
|
|
listManager.listsPath = new File(env.getRootPath(), env.getConfig("listManager.listsPath", "DATA/LISTS"));
|
|
String blacklistToUse = null;
|
|
|
|
// getting the list of supported blacklist types
|
|
final String supportedBlacklistTypesStr = AbstractBlacklist.BLACKLIST_TYPES_STRING;
|
|
final String[] supportedBlacklistTypes = supportedBlacklistTypesStr.split(",");
|
|
|
|
if (post == null) {
|
|
prop.put("results", "0");
|
|
putBlacklists(prop, listManager.getDirListing(listManager.listsPath, BLACKLIST_FILENAME_FILTER), blacklistToUse);
|
|
return prop;
|
|
}
|
|
|
|
if (post.containsKey("listNames")) {
|
|
blacklistToUse = post.get("listNames");
|
|
if (blacklistToUse.length() == 0 || !listManager.listSetContains("listManager.listsPath", blacklistToUse))
|
|
prop.put("results", "2");
|
|
}
|
|
|
|
putBlacklists(prop, listManager.getDirListing(listManager.listsPath, BLACKLIST_FILENAME_FILTER), blacklistToUse);
|
|
|
|
if (blacklistToUse != null) {
|
|
prop.put("results", "1");
|
|
|
|
if (post.containsKey("delete")) {
|
|
prop.put(RESULTS + "modified", "1");
|
|
prop.put(RESULTS + "modified_delCount", removeEntries(blacklistToUse, supportedBlacklistTypes, getByPrefix(post, "select", true, true)));
|
|
} else if (post.containsKey("alter")) {
|
|
prop.put(RESULTS + "modified", "2");
|
|
prop.put(RESULTS + "modified_alterCount", alterEntries(blacklistToUse, supportedBlacklistTypes, getByPrefix(post, "select", true, false), getByPrefix(post, "entry", false, false)));
|
|
}
|
|
|
|
// list illegal entries
|
|
final HashMap<String, Integer> ies = getIllegalEntries(blacklistToUse, supportedBlacklistTypes, plasmaSwitchboard.urlBlacklist);
|
|
prop.put(RESULTS + "blList", blacklistToUse);
|
|
prop.put(RESULTS + "entries", ies.size());
|
|
prop.putHTML(RESULTS + "blEngine", plasmaSwitchboard.urlBlacklist.getEngineInfo());
|
|
prop.put(RESULTS + "disabled", (ies.size() == 0) ? "1" : "0");
|
|
if (ies.size() > 0) {
|
|
prop.put(RESULTS + DISABLED + "entries", ies.size());
|
|
int i = 0;
|
|
String s;
|
|
for (Entry<String, Integer> entry: ies.entrySet()) {
|
|
s = entry.getKey();
|
|
prop.put(RESULTS + DISABLED + ENTRIES + i + "_error", entry.getValue().longValue());
|
|
prop.putHTML(RESULTS + DISABLED + ENTRIES + i + "_entry", s);
|
|
i++;
|
|
}
|
|
}
|
|
}
|
|
|
|
return prop;
|
|
}
|
|
|
|
private static void putBlacklists(final serverObjects prop, final List<String> lists, final String selected) {
|
|
boolean supported = false;
|
|
for (int i=0; i<supportedBLEngines.length && !supported; i++) {
|
|
supported |= (plasmaSwitchboard.urlBlacklist.getClass() == supportedBLEngines[i]);
|
|
}
|
|
|
|
if (supported) {
|
|
if (lists.size() > 0) {
|
|
prop.put("disabled", "0");
|
|
prop.put(DISABLED + "blacklists", lists.size());
|
|
int count = 0;
|
|
for (String list : lists) {
|
|
prop.putHTML(DISABLED + BLACKLISTS + count + "_name", list);
|
|
prop.put(DISABLED + BLACKLISTS + count + "_selected", (list.equals(selected)) ? "1" : "0");
|
|
count++;
|
|
}
|
|
} else {
|
|
prop.put("disabled", "2");
|
|
}
|
|
} else {
|
|
prop.put("disabled", "1");
|
|
for (int i=0; i<supportedBLEngines.length; i++) {
|
|
prop.putHTML(DISABLED + "engines_" + i + "_name", supportedBLEngines[i].getName());
|
|
}
|
|
prop.put(DISABLED + "engines", supportedBLEngines.length);
|
|
}
|
|
}
|
|
|
|
private static String[] getByPrefix(final serverObjects post, final String prefix, final boolean useKeys, final boolean useHashSet) {
|
|
Collection<String> r;
|
|
if (useHashSet) {
|
|
r = new HashSet<String>();
|
|
} else {
|
|
r = new ArrayList<String>();
|
|
}
|
|
|
|
String s;
|
|
if (useKeys) {
|
|
final Iterator<String> it = post.keySet().iterator();
|
|
while (it.hasNext()) {
|
|
if ((s = it.next()).indexOf(prefix) == 0) {
|
|
r.add(s.substring(prefix.length()));
|
|
}
|
|
}
|
|
} else {
|
|
final Iterator<Map.Entry<String, String>> it = post.entrySet().iterator();
|
|
Map.Entry<String, String> entry;
|
|
while (it.hasNext()) {
|
|
entry = it.next();
|
|
if (entry.getKey().indexOf(prefix) == 0) {
|
|
r.add(entry.getValue());
|
|
}
|
|
}
|
|
}
|
|
|
|
return r.toArray(new String[r.size()]);
|
|
}
|
|
|
|
private static HashMap<String, Integer>/* entry, error-code */ getIllegalEntries(final String blacklistToUse, final String[] supportedBlacklistTypes, final Blacklist blEngine) {
|
|
final HashMap<String, Integer> r = new HashMap<String, Integer>();
|
|
final HashSet<String> ok = new HashSet<String>();
|
|
|
|
final ArrayList<String> list = listManager.getListArray(new File(listManager.listsPath, blacklistToUse));
|
|
final Iterator<String> it = list.iterator();
|
|
String s, host, path;
|
|
|
|
if (blEngine instanceof DefaultBlacklist) {
|
|
int slashPos;
|
|
while (it.hasNext()) {
|
|
s = (it.next()).trim();
|
|
|
|
// check for double-occurance
|
|
if (ok.contains(s)) {
|
|
r.put(s, Integer.valueOf(ERR_DOUBLE_OCCURANCE));
|
|
continue;
|
|
}
|
|
ok.add(s);
|
|
|
|
if ((slashPos = s.indexOf("/")) == -1) {
|
|
host = s;
|
|
path = ".*";
|
|
} else {
|
|
host = s.substring(0, slashPos);
|
|
path = s.substring(slashPos + 1);
|
|
}
|
|
|
|
final int i = host.indexOf("*");
|
|
|
|
// check whether host begins illegally
|
|
if (!host.matches("([A-Za-z0-9_-]+|\\*)(\\.([A-Za-z0-9_-]+|\\*))*")) {
|
|
if (i == 0 && host.length() > 1 && host.charAt(1) != '.') {
|
|
r.put(s, Integer.valueOf(ERR_SUBDOMAIN_XOR_WILDCARD));
|
|
continue;
|
|
}
|
|
r.put(s, Integer.valueOf(ERR_HOST_WRONG_CHARS));
|
|
continue;
|
|
}
|
|
|
|
// in host-part only full sub-domains may be wildcards
|
|
if (host.length() > 0 && i > -1) {
|
|
if (!(i == 0 || i == host.length() - 1)) {
|
|
r.put(s, Integer.valueOf(ERR_WILDCARD_BEGIN_OR_END));
|
|
continue;
|
|
}
|
|
|
|
if (i == host.length() - 1 && host.length() > 1 && host.charAt(i - 1) != '.') {
|
|
r.put(s, Integer.valueOf(ERR_SUBDOMAIN_XOR_WILDCARD));
|
|
continue;
|
|
}
|
|
}
|
|
|
|
// check for double-occurences of "*" in host
|
|
if (host.indexOf("*", i + 1) > -1) {
|
|
r.put(s, Integer.valueOf(ERR_TWO_WILDCARDS_IN_HOST));
|
|
continue;
|
|
}
|
|
|
|
// check for errors on regex-compiling path
|
|
try {
|
|
Pattern.compile(path);
|
|
} catch (final PatternSyntaxException e) {
|
|
r.put(s, Integer.valueOf(ERR_PATH_REGEX));
|
|
continue;
|
|
}
|
|
}
|
|
}
|
|
|
|
return r;
|
|
}
|
|
|
|
private static int removeEntries(final String blacklistToUse, final String[] supportedBlacklistTypes, final String[] entries) {
|
|
// load blacklist data from file
|
|
final ArrayList<String> list = listManager.getListArray(new File(listManager.listsPath, blacklistToUse));
|
|
|
|
boolean listChanged = false;
|
|
|
|
// delete the old entry from file
|
|
String s;
|
|
for (int i=0; i<entries.length; i++) {
|
|
s = entries[i];
|
|
|
|
if (list != null){
|
|
|
|
// getting rid of escape characters which make it impossible to
|
|
// properly use contains()
|
|
if (s.contains("\\\\")) {
|
|
s = s.replaceAll(Pattern.quote("\\\\"), Matcher.quoteReplacement("\\"));
|
|
}
|
|
|
|
if (list.contains(s)) {
|
|
list.remove(s);
|
|
listChanged = true;
|
|
}
|
|
}
|
|
|
|
// remove the entry from the running blacklist engine
|
|
for (int blTypes=0; blTypes < supportedBlacklistTypes.length; blTypes++) {
|
|
if (listManager.listSetContains(supportedBlacklistTypes[blTypes] + ".BlackLists", blacklistToUse)) {
|
|
final String host = (s.indexOf("/") == -1) ? s : s.substring(0, s.indexOf("/"));
|
|
final String path = (s.indexOf("/") == -1) ? ".*" : s.substring(s.indexOf("/") + 1);
|
|
try {
|
|
plasmaSwitchboard.urlBlacklist.remove(supportedBlacklistTypes[blTypes],
|
|
host,path);
|
|
} catch (final RuntimeException e) {
|
|
//System.err.println(e.getMessage() + ": " + host + "/" + path);
|
|
Log.logSevere("BLACKLIST-CLEANER", e.getMessage() + ": " + host + "/" + path);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
if (listChanged){
|
|
listManager.writeList(new File(listManager.listsPath, blacklistToUse), list.toArray(new String[list.size()]));
|
|
}
|
|
return entries.length;
|
|
}
|
|
|
|
private static int alterEntries(
|
|
final String blacklistToUse,
|
|
final String[] supportedBlacklistTypes,
|
|
final String[] oldE,
|
|
final String[] newE) {
|
|
removeEntries(blacklistToUse, supportedBlacklistTypes, oldE);
|
|
PrintWriter pw = null;
|
|
try {
|
|
pw = new PrintWriter(new FileWriter(new File(listManager.listsPath, blacklistToUse), true));
|
|
String host, path;
|
|
for (int i=0, pos; i<newE.length; i++) {
|
|
pos = newE[i].indexOf("/");
|
|
if (pos < 0) {
|
|
host = newE[i];
|
|
path = ".*";
|
|
} else {
|
|
host = newE[i].substring(0, pos);
|
|
path = newE[i].substring(pos + 1);
|
|
}
|
|
pw.println(host + "/" + path);
|
|
for (int blTypes=0; blTypes < supportedBlacklistTypes.length; blTypes++) {
|
|
if (listManager.listSetContains(supportedBlacklistTypes[blTypes] + ".BlackLists",blacklistToUse)) {
|
|
plasmaSwitchboard.urlBlacklist.add(
|
|
supportedBlacklistTypes[blTypes],
|
|
host,
|
|
path);
|
|
}
|
|
}
|
|
}
|
|
pw.close();
|
|
} catch (final IOException e) {
|
|
Log.logSevere("BLACKLIST-CLEANER", "error on writing altered entries to blacklist", e);
|
|
}
|
|
return newE.length;
|
|
}
|
|
}
|