From f88487e964a08e16ae64da4ecb587dad802dcff9 Mon Sep 17 00:00:00 2001 From: karlchenofhell Date: Sat, 27 Jan 2007 23:30:58 +0000 Subject: [PATCH] - added servlet to clean blacklists (find & delete invalid entries) git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@3286 6c8d7289-2bf4-0310-a012-ef5d649a1542 --- htroot/BlacklistCleaner_p.html | 54 ++++ htroot/BlacklistCleaner_p.java | 270 ++++++++++++++++++ .../env/templates/submenuBlacklist.template | 7 + 3 files changed, 331 insertions(+) create mode 100644 htroot/BlacklistCleaner_p.html create mode 100644 htroot/BlacklistCleaner_p.java create mode 100644 htroot/env/templates/submenuBlacklist.template diff --git a/htroot/BlacklistCleaner_p.html b/htroot/BlacklistCleaner_p.html new file mode 100644 index 000000000..bc137e0c1 --- /dev/null +++ b/htroot/BlacklistCleaner_p.html @@ -0,0 +1,54 @@ + + + + YaCy '#[clientname]#': Blacklist Manager + #%env/templates/metas.template%# + + + #%env/templates/header.template%# + #%env/templates/submenuBlacklist.template%# + +

Blacklist Cleaner

+

Remove illegal blacklist-entries

+ +
#(disabled)# +
Check list +
+ + +
::#(/disabled)# +
+ + #(results)# + :: +
#[entries]# Illegal Entries for #[blEngine]# + #(modified)# + ::

Deleted #[delCount]# entries

+ ::

Altered #[alterCount] entries!

+ #(/modified)# + + #(disabled)# +
+ #{entries}# +
+ + +
+
+ #{/entries}# +
+ +
::#(/disabled)# +
+ :: + No Blacklist selected + #(/results)# +
+ + #%env/templates/footer.template%# + + diff --git a/htroot/BlacklistCleaner_p.java b/htroot/BlacklistCleaner_p.java new file mode 100644 index 000000000..941841e92 --- /dev/null +++ b/htroot/BlacklistCleaner_p.java @@ -0,0 +1,270 @@ +// Blacklist_p.java +// ----------------------- +// part of YaCy +// (C) by Michael Peter Christen; mc@anomic.de +// first published on http://www.anomic.de +// Frankfurt, Germany, 2004 +// +// This File is contributed by Franz Brauße +// +// $LastChangedDate: 2007-01-27 14:07:54 +0000 (Sa, 27 Jan 2007) $ +// $LastChangedRevision: 3217 $ +// $LastChangedBy: karlchenofhell $ +// +// This program is free software; you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation; either version 2 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +// +// Using this software in any meaning (reading, learning, copying, compiling, +// running) means that you agree that the Author(s) is (are) not responsible +// for cost, loss of data or any harm that may be caused directly or indirectly +// by usage of this softare or this documentation. The usage of this software +// is on your own risk. The installation and usage (starting/running) of this +// software may allow other people or application to access your computer and +// any attached devices and is highly dependent on the configuration of the +// software which must be done by the user of the software; the author(s) is +// (are) also not responsible for proper configuration and usage of the +// software, even if provoked by documentation provided together with +// the software. +// +// Any changes to this file according to the GPL as documented in the file +// gpl.txt aside this file in the shipment you received can be done to the +// lines that follows this copyright notice here, but changes must not be +// done inside the copyright notive above. A re-distribution must contain +// the intact and unchanged copyright notice. +// Contributions and changes to the program code must be marked as such. + +// You must compile this file with +// javac -classpath .:../classes BlacklistCleaner_p.java +// if the shell's current path is HTROOT + +import java.io.File; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Iterator; +import java.util.regex.Pattern; +import java.util.regex.PatternSyntaxException; + +import de.anomic.data.listManager; +import de.anomic.http.httpHeader; +import de.anomic.plasma.plasmaSwitchboard; +import de.anomic.plasma.urlPattern.defaultURLPattern; +import de.anomic.plasma.urlPattern.plasmaURLPattern; +import de.anomic.server.serverObjects; +import de.anomic.server.serverSwitch; + +public class BlacklistCleaner_p { + + private static final String RESULTS = "results_"; + private static final String DISABLED = "disabled_"; + private static final String BLACKLISTS = "blacklists_"; + private static final String ENTRIES = "entries_"; + + private static final int ERR_TWO_WILDCARDS_IN_HOST = 0; + private static final int ERR_SUBDOMAIN_XOR_WILDCARD = 1; + private static final int ERR_PATH_REGEX = 2; + + public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch env) { + serverObjects prop = new serverObjects(); + + // initialize the list manager + listManager.switchboard = (plasmaSwitchboard) env; + listManager.listsPath = new File(env.getRootPath(), env.getConfig("listManager.listsPath", "DATA/LISTS")); + String blacklistToUse = null; + + // getting the list of supported blacklist types + String supportedBlacklistTypesStr = env.getConfig("BlackLists.types", ""); + String[] supportedBlacklistTypes = supportedBlacklistTypesStr.split(","); + + if (post == null) { + prop.put("results", 0); + putBlacklists(prop, listManager.getDirListing(listManager.listsPath), blacklistToUse); + return prop; + } + + if (post.containsKey("listNames")) { + blacklistToUse = (String)post.get("listNames"); + if (blacklistToUse.length() == 0 || !listManager.ListInListslist("listManager.listsPath", blacklistToUse)) + prop.put("results", 2); + } + + putBlacklists(prop, listManager.getDirListing(listManager.listsPath), blacklistToUse); + + if (blacklistToUse != null) { + prop.put("results", 1); + + if (post.containsKey("delete")) { + prop.put(RESULTS + "modified", 1); + prop.put(RESULTS + "modified_delCount", removeEntries(blacklistToUse, supportedBlacklistTypes, getByPrefix(post, "select"))); + } else if (post.containsKey("alter")) { + prop.put(RESULTS + "modified", 2); + prop.put(RESULTS + "modified_alterCount", alterEntries(blacklistToUse, supportedBlacklistTypes, getByPrefix(post, "select"), getByPrefix(post, "entry"))); + } + + // list illegal entries + HashMap ies = getIllegalEntries(blacklistToUse, supportedBlacklistTypes, plasmaSwitchboard.urlBlacklist); + prop.put(RESULTS + "entries", ies.size()); + prop.putSafeXML(RESULTS + "blEngine", plasmaSwitchboard.urlBlacklist.getEngineInfo()); + prop.put(RESULTS + "disabled", (ies.size() == 0) ? 1 : 0); + if (ies.size() > 0) { + prop.put(RESULTS + DISABLED + "entries", ies.size()); + Iterator it = ies.keySet().iterator(); + int i = 0; + String s; + while (it.hasNext()) { + s = (String)it.next(); + prop.put(RESULTS + DISABLED + ENTRIES + i + "_error", ((Integer)ies.get(s)).intValue()); + prop.putSafeXML(RESULTS + DISABLED + ENTRIES + i + "_entry", s); + i++; + } + } + } + + return prop; + } + + private static void putBlacklists(serverObjects prop, String[] lists, String selected) { + if (lists.length > 0) { + prop.put("disabled", 0); + prop.put(DISABLED + "blacklists", lists.length); + for (int i=0; i 0 && i > -1) { + if (host.length() > i + 1) + if (host.charAt(i + 1) != '.') { + r.put(s, Integer.valueOf(ERR_SUBDOMAIN_XOR_WILDCARD)); + continue; + } + if (i > 0) + if (host.charAt(i - 1) != '.') { + r.put(s, Integer.valueOf(ERR_SUBDOMAIN_XOR_WILDCARD)); + continue; + } + } + + // check for double-occurences of "*" in host + if (host.indexOf("*", i + 1) > -1) { + r.put(s, Integer.valueOf(ERR_TWO_WILDCARDS_IN_HOST)); + continue; + } + + // check for errors on regex-compiling path + try { + Pattern.compile(path); + } catch (PatternSyntaxException e) { + r.put(s, Integer.valueOf(ERR_PATH_REGEX)); + continue; + } + } + } + + return r; + } + + private static int removeEntries(String blacklistToUse, String[] supportedBlacklistTypes, String[] entries) { + // load blacklist data from file + ArrayList list = listManager.getListArray(new File(listManager.listsPath, blacklistToUse)); + + // delete the old entry from file + String s; + for (int i=0; i +

Blacklist Menu

+ + \ No newline at end of file