Marown/yacy_search_server
1
0
Fork 0
mirror of https://github.com/yacy/yacy_search_server.git synced 2024-09-19 00:01:41 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity

various fixes and cleanups for blacklist handling:

Browse Source 
1. avoid adding duplicate file name entries in config properties for lists, 
2. correctly merge all path masks from all list files for the same host masks,
3. rewrite helper methods standard java methods for Collection transformations,
4. merged various methods with identical functionality for different Collection implementations into one,
5. minor refactoring to improve code readability.

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@4087 6c8d7289-2bf4-0310-a012-ef5d649a1542
This commit is contained in:
fuchsi 2007-09-10 06:20:27 +00:00
parent e27aeb7fdc
commit 5b0c1449e1
13 changed files with 285 additions and 189 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
htroot/BlacklistCleaner_p.java
View File

@ -109,7 +109,7 @@ public class BlacklistCleaner_p {
if (post.containsKey("listNames")) {
blacklistToUse = (String)post.get("listNames");
if (blacklistToUse.length() == 0 || !listManager.ListInListslist("listManager.listsPath", blacklistToUse))
if (blacklistToUse.length() == 0 || !listManager.listSetContains("listManager.listsPath", blacklistToUse))
prop.put("results", 2);
}
@ -283,7 +283,7 @@ public class BlacklistCleaner_p {
// remove the entry from the running blacklist engine
for (int blTypes=0; blTypes < supportedBlacklistTypes.length; blTypes++) {
if (listManager.ListInListslist(supportedBlacklistTypes[blTypes] + ".BlackLists", blacklistToUse)) {
if (listManager.listSetContains(supportedBlacklistTypes[blTypes] + ".BlackLists", blacklistToUse)) {
String host = (s.indexOf("/") == -1) ? s : s.substring(0, s.indexOf("/"));
String path = (s.indexOf("/") == -1) ? ".*" : s.substring(s.indexOf("/") + 1);
try {
@ -318,7 +318,7 @@ public class BlacklistCleaner_p {
}
pw.println(host + "/" + path);
for (int blTypes=0; blTypes < supportedBlacklistTypes.length; blTypes++) {
if (listManager.ListInListslist(supportedBlacklistTypes[blTypes] + ".BlackLists",blacklistToUse)) {
if (listManager.listSetContains(supportedBlacklistTypes[blTypes] + ".BlackLists",blacklistToUse)) {
plasmaSwitchboard.urlBlacklist.add(
supportedBlacklistTypes[blTypes],
host,

32
htroot/Blacklist_p.java
View File

@ -138,11 +138,11 @@ public class Blacklist_p {
newFile.createNewFile();
// share the newly created blacklist
listManager.addListToListslist(BLACKLIST_SHARED, blacklistToUse);
listManager.updateListSet(BLACKLIST_SHARED, blacklistToUse);
// activate it for all known blacklist types
for (int blTypes=0; blTypes < supportedBlacklistTypes.length; blTypes++) {
listManager.addListToListslist(supportedBlacklistTypes[blTypes] + ".BlackLists",blacklistToUse);
listManager.updateListSet(supportedBlacklistTypes[blTypes] + ".BlackLists",blacklistToUse);
}
} catch (IOException e) {/* */}
@ -161,11 +161,11 @@ public class Blacklist_p {
BlackListFile.delete();
for (int blTypes=0; blTypes < supportedBlacklistTypes.length; blTypes++) {
listManager.removeListFromListslist(supportedBlacklistTypes[blTypes] + ".BlackLists",blacklistToUse);
listManager.removeFromListSet(supportedBlacklistTypes[blTypes] + ".BlackLists",blacklistToUse);
}
// remove it from the shared list
listManager.removeListFromListslist(BLACKLIST_SHARED, blacklistToUse);
listManager.removeFromListSet(BLACKLIST_SHARED, blacklistToUse);
blacklistToUse = null;
// reload Blacklists
@ -185,9 +185,9 @@ public class Blacklist_p {
for (int blTypes=0; blTypes < supportedBlacklistTypes.length; blTypes++) {
if (post.containsKey("activateList4" + supportedBlacklistTypes[blTypes])) {
listManager.addListToListslist(supportedBlacklistTypes[blTypes] + ".BlackLists",blacklistToUse);
listManager.updateListSet(supportedBlacklistTypes[blTypes] + ".BlackLists",blacklistToUse);
} else {
listManager.removeListFromListslist(supportedBlacklistTypes[blTypes] + ".BlackLists",blacklistToUse);
listManager.removeFromListSet(supportedBlacklistTypes[blTypes] + ".BlackLists",blacklistToUse);
}
}
@ -205,11 +205,11 @@ public class Blacklist_p {
return prop;
}
if (listManager.ListInListslist(BLACKLIST_SHARED, blacklistToUse)) {
if (listManager.listSetContains(BLACKLIST_SHARED, blacklistToUse)) {
// Remove from shared BlackLists
listManager.removeListFromListslist(BLACKLIST_SHARED, blacklistToUse);
listManager.removeFromListSet(BLACKLIST_SHARED, blacklistToUse);
} else { // inactive list -> enable
listManager.addListToListslist(BLACKLIST_SHARED, blacklistToUse);
listManager.updateListSet(BLACKLIST_SHARED, blacklistToUse);
}
} else if (post.containsKey("deleteBlacklistEntry")) {
@ -253,7 +253,7 @@ public class Blacklist_p {
oldEntry = oldEntry + "/.*";
}
for (int blTypes=0; blTypes < supportedBlacklistTypes.length; blTypes++) {
if (listManager.ListInListslist(supportedBlacklistTypes[blTypes] + ".BlackLists",blacklistToUse)) {
if (listManager.listSetContains(supportedBlacklistTypes[blTypes] + ".BlackLists",blacklistToUse)) {
plasmaSwitchboard.urlBlacklist.remove(supportedBlacklistTypes[blTypes],oldEntry.substring(0, pos), oldEntry.substring(pos + 1));
}
}
@ -303,7 +303,7 @@ public class Blacklist_p {
// add to blacklist
for (int blTypes=0; blTypes < supportedBlacklistTypes.length; blTypes++) {
if (listManager.ListInListslist(supportedBlacklistTypes[blTypes] + ".BlackLists",blacklistToUse)) {
if (listManager.listSetContains(supportedBlacklistTypes[blTypes] + ".BlackLists",blacklistToUse)) {
plasmaSwitchboard.urlBlacklist.add(supportedBlacklistTypes[blTypes],newEntry.substring(0, pos), newEntry.substring(pos + 1));
}
}
@ -352,7 +352,7 @@ public class Blacklist_p {
// add to blacklist
for (int blTypes=0; blTypes < supportedBlacklistTypes.length; blTypes++) {
if (listManager.ListInListslist(supportedBlacklistTypes[blTypes] + ".BlackLists",blacklistToUse)) {
if (listManager.listSetContains(supportedBlacklistTypes[blTypes] + ".BlackLists",blacklistToUse)) {
plasmaSwitchboard.urlBlacklist.add(supportedBlacklistTypes[blTypes],entry.substring(0, pos), entry.substring(pos + 1));
}
}
@ -390,7 +390,7 @@ public class Blacklist_p {
entry = entry + "/.*";
}
for (int blTypes=0; blTypes < supportedBlacklistTypes.length; blTypes++) {
if (listManager.ListInListslist(supportedBlacklistTypes[blTypes] + ".BlackLists",blacklistToUse)) {
if (listManager.listSetContains(supportedBlacklistTypes[blTypes] + ".BlackLists",blacklistToUse)) {
plasmaSwitchboard.urlBlacklist.remove(supportedBlacklistTypes[blTypes],entry.substring(0, pos), entry.substring(pos + 1));
}
}
@ -469,13 +469,13 @@ public class Blacklist_p {
for (int blTypes=0; blTypes < supportedBlacklistTypes.length; blTypes++) {
prop.put(DISABLED + "currentActiveFor_" + blTypes + "_blTypeName",supportedBlacklistTypes[blTypes]);
prop.put(DISABLED + "currentActiveFor_" + blTypes + "_checked",
listManager.ListInListslist(supportedBlacklistTypes[blTypes] + ".BlackLists",dirlist[i])?0:1);
listManager.listSetContains(supportedBlacklistTypes[blTypes] + ".BlackLists",dirlist[i])?0:1);
}
prop.put(DISABLED + "currentActiveFor",supportedBlacklistTypes.length);
}
if (listManager.ListInListslist(BLACKLIST_SHARED, dirlist[i])) {
if (listManager.listSetContains(BLACKLIST_SHARED, dirlist[i])) {
prop.put(DISABLED + BLACKLIST + blacklistCount + "_shared", 1);
} else {
prop.put(DISABLED + BLACKLIST + blacklistCount + "_shared", 0);
@ -483,7 +483,7 @@ public class Blacklist_p {
int activeCount = 0;
for (int blTypes=0; blTypes < supportedBlacklistTypes.length; blTypes++) {
if (listManager.ListInListslist(supportedBlacklistTypes[blTypes] + ".BlackLists",dirlist[i])) {
if (listManager.listSetContains(supportedBlacklistTypes[blTypes] + ".BlackLists",dirlist[i])) {
prop.put(DISABLED + BLACKLIST + blacklistCount + "_active_" + activeCount + "_blTypeName",supportedBlacklistTypes[blTypes]);
activeCount++;
}

6
htroot/Bookmarks.java
View File

@ -49,8 +49,8 @@ import java.io.File;
import java.net.MalformedURLException;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.Set;
import de.anomic.data.bookmarksDB;
import de.anomic.data.listManager;
@ -127,7 +127,7 @@ public class Bookmarks {
if(tagsString.equals("")){
tagsString="unsorted"; //defaulttag
}
HashSet tags=listManager.string2hashset(tagsString);
Set tags=listManager.string2set(tagsString);
bookmarksDB.Bookmark bookmark = switchboard.bookmarksDB.createBookmark(url, username);
if(bookmark != null){
@ -264,7 +264,7 @@ public class Bookmarks {
count++;
}
count=0;
HashSet tags;
Set tags;
Iterator tagsIt;
int tagCount;
while(count<max_count && it.hasNext()){

4
htroot/IndexControl_p.java
View File

@ -381,7 +381,7 @@ public class IndexControl_p {
url = e.comp().url();
pw.println(url.getHost() + "/" + url.getFile());
for (int blTypes=0; blTypes < supportedBlacklistTypes.length; blTypes++) {
if (listManager.ListInListslist(supportedBlacklistTypes[blTypes] + ".BlackLists", blacklist)) {
if (listManager.listSetContains(supportedBlacklistTypes[blTypes] + ".BlackLists", blacklist)) {
plasmaSwitchboard.urlBlacklist.add(
supportedBlacklistTypes[blTypes],
url.getHost(),
@ -409,7 +409,7 @@ public class IndexControl_p {
url = e.comp().url();
pw.println(url.getHost() + "/.*");
for (int blTypes=0; blTypes < supportedBlacklistTypes.length; blTypes++) {
if (listManager.ListInListslist(supportedBlacklistTypes[blTypes] + ".BlackLists", blacklist)) {
if (listManager.listSetContains(supportedBlacklistTypes[blTypes] + ".BlackLists", blacklist)) {
plasmaSwitchboard.urlBlacklist.add(
supportedBlacklistTypes[blTypes],
url.getHost(),

2
htroot/sharedBlacklist_p.java
View File

@ -211,7 +211,7 @@ public class sharedBlacklist_p {
String[] supportedBlacklistTypes = supportedBlacklistTypesStr.split(",");
for (int blTypes=0; blTypes < supportedBlacklistTypes.length; blTypes++) {
if (listManager.ListInListslist(supportedBlacklistTypes[blTypes] + ".BlackLists",selectedBlacklistName)) {
if (listManager.listSetContains(supportedBlacklistTypes[blTypes] + ".BlackLists",selectedBlacklistName)) {
plasmaSwitchboard.urlBlacklist.add(supportedBlacklistTypes[blTypes],newItem.substring(0, pos), newItem.substring(pos + 1));
}
}

4
htroot/xml/blacklists_p.java
View File

@ -44,7 +44,7 @@ public class blacklists_p {
for (int i = 0; i <= dirlist.length - 1; i++) {
prop.put("lists_" + blacklistCount + "_name", dirlist[i]);
if (listManager.ListInListslist("BlackLists.Shared", dirlist[i])) {
if (listManager.listSetContains("BlackLists.Shared", dirlist[i])) {
prop.put("lists_" + blacklistCount + "_shared", 1);
} else {
prop.put("lists_" + blacklistCount + "_shared", 0);
@ -54,7 +54,7 @@ public class blacklists_p {
for (int j=0; j<types.length; j++) {
prop.put("lists_" + blacklistCount + "_types_" + j + "_name", types[j]);
prop.put("lists_" + blacklistCount + "_types_" + j + "_value",
listManager.ListInListslist(types[j] + ".Blacklist", dirlist[i]) ? 1 : 0);
listManager.listSetContains(types[j] + ".Blacklist", dirlist[i]) ? 1 : 0);
}
prop.put("lists_" + blacklistCount + "_types", types.length);

10
source/de/anomic/data/blogBoard.java
View File

@ -132,8 +132,8 @@ public class blogBoard {
record.put("ip", ip);
if (page == null) record.put("page", "");
else record.put("page", kelondroBase64Order.enhancedCoder.encode(page));
if (comments == null) record.put("comments", listManager.arraylist2string(new ArrayList()));
else record.put("comments", listManager.arraylist2string(comments));
if (comments == null) record.put("comments", listManager.collection2string(new ArrayList()));
else record.put("comments", listManager.collection2string(comments));
if (commentMode == null) record.put("commentMode", "1");
else record.put("commentMode", commentMode);
@ -144,7 +144,7 @@ public class blogBoard {
private entry(String key, Map record) {
this.key = key;
this.record = record;
if (this.record.get("comments")==null) this.record.put("comments", listManager.arraylist2string(new ArrayList()));
if (this.record.get("comments")==null) this.record.put("comments", listManager.collection2string(new ArrayList()));
if (this.record.get("commentMode")==null || this.record.get("commentMode").equals("")) this.record.put("commentMode", "1");
}
@ -221,13 +221,13 @@ public class blogBoard {
public void addComment(String commentID) {
ArrayList comments = listManager.string2arraylist((String) record.get("comments"));
comments.add(commentID);
record.put("comments", listManager.arraylist2string(comments));
record.put("comments", listManager.collection2string(comments));
}
public boolean removeComment(String commentID) {
ArrayList comments = listManager.string2arraylist((String) record.get("comments"));
boolean success = comments.remove(commentID);
record.put("comments", listManager.arraylist2string(comments));
record.put("comments", listManager.collection2string(comments));
return success;
}

2
source/de/anomic/data/blogBoardComments.java
View File

@ -150,7 +150,7 @@ public class blogBoardComments {
private CommentEntry(String key, Map record) {
this.key = key;
this.record = record;
if (this.record.get("comments")==null) this.record.put("comments", listManager.arraylist2string(new ArrayList()));
if (this.record.get("comments")==null) this.record.put("comments", listManager.collection2string(new ArrayList()));
}
public String key() {

49
source/de/anomic/data/bookmarksDB.java
View File

@ -58,6 +58,7 @@ import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.Map;
import java.util.Set;
import java.util.TreeSet;
import javax.xml.parsers.DocumentBuilder;
@ -248,7 +249,7 @@ public class bookmarksDB {
String tagHash=tagHash(oldName);
Tag tag=getTag(tagHash);
if (tag != null) {
HashSet urlHashes = tag.getUrlHashes();
Set urlHashes = tag.getUrlHashes();
try {
if(tagCache.containsKey(tagHash(oldName))){
tagCache.remove(tagHash(oldName));
@ -260,7 +261,7 @@ public class bookmarksDB {
saveTag(tag);
Iterator it = urlHashes.iterator();
Bookmark bookmark;
HashSet tags;
Set tags;
while (it.hasNext()) {
bookmark = getBookmark((String) it.next());
tags = bookmark.getTags();
@ -313,7 +314,7 @@ public class bookmarksDB {
TreeSet set=new TreeSet(new bookmarkComparator(true));
String tagHash=tagHash(tagName);
Tag tag=getTag(tagHash);
HashSet hashes=new HashSet();
Set hashes=new HashSet();
if(tag != null){
hashes=getTag(tagHash).getUrlHashes();
}
@ -346,7 +347,7 @@ public class bookmarksDB {
public boolean removeBookmark(String urlHash){
Bookmark bookmark = getBookmark(urlHash);
if(bookmark == null) return false; //does not exist
HashSet tags = bookmark.getTags();
Set tags = bookmark.getTags();
bookmarksDB.Tag tag=null;
Iterator it=tags.iterator();
while(it.hasNext()){
@ -409,7 +410,7 @@ public class bookmarksDB {
Iterator it;
String url,title;
Bookmark bm;
HashSet tags=listManager.string2hashset(tag); //this allow multiple default tags
Set tags=listManager.string2set(tag); //this allow multiple default tags
try {
//load the links
htmlFilterContentScraper scraper = new htmlFilterContentScraper(baseURL);
@ -493,13 +494,13 @@ public class bookmarksDB {
if(attributes.getNamedItem("time")!=null){
time=attributes.getNamedItem("time").getNodeValue();
}
HashSet tags=new HashSet();
Set tags=new HashSet();
if(title != null){
bm.setProperty(Bookmark.BOOKMARK_TITLE, title);
}
if(tagsString!=null){
tags = listManager.string2hashset(tagsString.replace(' ', ','));
tags = listManager.string2set(tagsString.replace(' ', ','));
}
bm.setTags(tags, true);
if(time != null){
@ -539,13 +540,13 @@ public class bookmarksDB {
public static final String TAG_NAME="tagName";
private String tagHash;
private Map mem;
private HashSet urlHashes;
private Set urlHashes;
public Tag(String hash, Map map){
tagHash=hash;
mem=map;
if(mem.containsKey(URL_HASHES))
urlHashes=listManager.string2hashset((String) mem.get(URL_HASHES));
urlHashes=listManager.string2set((String) mem.get(URL_HASHES));
else
urlHashes=new HashSet();
}
@ -564,7 +565,7 @@ public class bookmarksDB {
mem.put(TAG_NAME, name);
}
public Map getMap(){
mem.put(URL_HASHES, listManager.hashset2string(this.urlHashes));
mem.put(URL_HASHES, listManager.collection2string(this.urlHashes));
return mem;
}
/**
@ -593,7 +594,7 @@ public class bookmarksDB {
}
return "notagname";
}
public HashSet getUrlHashes(){
public Set getUrlHashes(){
return urlHashes;
}
public boolean hasPublicItems(){
@ -634,7 +635,7 @@ public class bookmarksDB {
//round to seconds, but store as milliseconds (java timestamp)
date=String.valueOf((Long.parseLong(mydate)/1000)*1000);
mem=new HashMap();
mem.put(URL_HASHES, listManager.arraylist2string(entries));
mem.put(URL_HASHES, listManager.collection2string(entries));
}
public void add(String urlHash){
String urlHashes = (String)mem.get(URL_HASHES);
@ -647,7 +648,7 @@ public class bookmarksDB {
if(!list.contains(urlHash) && urlHash != null && !urlHash.equals("")){
list.add(urlHash);
}
this.mem.put(URL_HASHES, listManager.arraylist2string(list));
this.mem.put(URL_HASHES, listManager.collection2string(list));
/*if(urlHashes!=null && !urlHashes.equals("") ){
if(urlHashes.indexOf(urlHash) <0){
this.mem.put(URL_HASHES, urlHashes+","+urlHash);
@ -661,7 +662,7 @@ public class bookmarksDB {
if(list.contains(urlHash)){
list.remove(urlHash);
}
this.mem.put(URL_HASHES, listManager.arraylist2string(list));
this.mem.put(URL_HASHES, listManager.collection2string(list));
}
public void setDatesTable(){
try {
@ -696,13 +697,13 @@ public class bookmarksDB {
public static final String BOOKMARK_OWNER="bookmarkOwner";
public static final String BOOKMARK_IS_FEED="bookmarkIsFeed";
private String urlHash;
private HashSet tags;
private Set tags;
private long timestamp;
public Bookmark(String urlHash, Map map){
super(map);
this.urlHash=urlHash;
if(map.containsKey(BOOKMARK_TAGS))
tags=listManager.string2hashset((String) map.get(BOOKMARK_TAGS));
tags=listManager.string2set((String) map.get(BOOKMARK_TAGS));
else
tags=new HashSet();
loadTimestamp();
@ -752,7 +753,7 @@ public class bookmarksDB {
}
private Map toMap(){
entry.put(BOOKMARK_TAGS, listManager.hashset2string(tags));
entry.put(BOOKMARK_TAGS, listManager.collection2string(tags));
entry.put(BOOKMARK_TIMESTAMP, String.valueOf(this.timestamp));
return entry;
}
@ -766,11 +767,11 @@ public class bookmarksDB {
public String getUrl(){
return (String) entry.get(BOOKMARK_URL);
}
public HashSet getTags(){
public Set getTags(){
return tags;
}
public String getTagsString(){
return listManager.hashset2string(getTags());
return listManager.collection2string(getTags());
}
public String getDescription(){
if(entry.containsKey(BOOKMARK_DESCRIPTION)){
@ -828,18 +829,18 @@ public class bookmarksDB {
}
/**
* set the Tags of the bookmark, and write them into the tags table.
* @param tags a ArrayList with the tags
* @param tags2 a ArrayList with the tags
*/
public void setTags(HashSet tags){
setTags(tags, true);
public void setTags(Set tags2){
setTags(tags2, true);
}
/**
* set the Tags of the bookmark
* @param tags ArrayList with the tagnames
* @param local sets, whether the updated tags should be stored to tagsDB
*/
public void setTags(HashSet mytags, boolean local){
tags.addAll(mytags);
public void setTags(Set tags2, boolean local){
tags.addAll(tags2);
Iterator it=tags.iterator();
while(it.hasNext()){
String tagName=(String) it.next();

290
source/de/anomic/data/listManager.java
View File

@ -3,6 +3,7 @@
// part of YACY
//
// (C) 2005, 2006 by Alexander Schier
// (C) 2007 by Bjoern 'Fuchs' Krombholz; fox.box@gmail.com
//
// last change: $LastChangedDate$ by $LastChangedBy$
// $LastChangedRevision$
@ -51,8 +52,11 @@ import java.io.IOException;
import java.io.InputStreamReader;
import java.io.PrintWriter;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.HashSet;
import java.util.Iterator;
import java.util.Set;
import java.util.Vector;
import de.anomic.plasma.plasmaSwitchboard;
@ -66,59 +70,75 @@ public class listManager {
public static plasmaSwitchboard switchboard;
public static File listsPath;
//===============Listslists=====================
// get an array of all Lists from a Config Property
public static String[] getListslistArray(String Listname) {
return switchboard.getConfig(Listname, "").split(",");
/**
* Get ListSet from configuration file and return it as a unified Set.
*
* <b>Meaning of ListSet</b>: There are various "lists" in YaCy which are
* actually disjunct (pairwise unequal) sets which themselves can be seperated
* into different subsets. E.g., there can be more than one blacklist of a type.
* A ListSet is the set of all those "lists" (subsets) of an equal type.
*
* @param setName name of the ListSet
* @return a ListSet from configuration file
*/
public static Set getListSet(String setName) {
return string2set(switchboard.getConfig(setName, ""));
}
/**
* Removes an element from a ListSet and updates the configuration file
* accordingly. If the element doesn't exist, then nothing will be changed.
*
* @param setName name of the ListSet.
* @param listName name of the element to remove from the ListSet.
*/
public static void removeFromListSet(String setName, String listName) {
Set listSet = getListSet(setName);
if (listSet.size() > 0) {
listSet.remove(listName);
switchboard.setConfig(setName, collection2string(listSet));
}
}
// removes a List from a Lists-List
public static void removeListFromListslist(String ListName, String BlackList) {
String[] Lists = getListslistArray(ListName);
String temp = "";
/**
* Adds an element to an existing ListSet. If the ListSet doesn't exist yet,
* a new one will be added. If the ListSet already contains an identical element,
* then nothing happens.
*
* The new list will be written to the configuartion file.
*
* @param setName
* @param newListName
*/
public static void updateListSet(String setName, String newListName) {
Set listSet = getListSet(setName);
listSet.add(newListName);
for (int i=0; i <= Lists.length -1; i++) {
if (!Lists[i].equals(BlackList) && !Lists[i].equals("")) {
temp += Lists[i] + ",";
}
}
if (temp.endsWith(",")) { //remove "," at end...
temp = temp.substring(0, temp.length() -1);
}
if (temp.startsWith(",") ) { //remove "," at end...
temp = temp.substring(1, temp.length());
}
switchboard.setConfig(setName, collection2string(listSet));
}
switchboard.setConfig(ListName, temp);
}
/**
* @param setName ListSet in which to search for an element.
* @param listName the element to search for.
* @return <code>true</code> if the ListSet "setName" contains an element
* "listName", <code>false</code> otherwise.
*/
public static boolean listSetContains(String setName, String listName) {
Set Lists = getListSet(setName);
// add a new List to a List-List
public static void addListToListslist(String ListName, String newList) {
String[] Lists = getListslistArray(ListName);
String temp = "";
return Lists.contains(listName);
}
for (int i = 0; i <= (Lists.length -1); i++) {
temp += Lists[i] + ",";
}
temp += newList;
switchboard.setConfig(ListName, temp);
}
// returns true, if the Lists-List contains the Listname
public static boolean ListInListslist(String Listname, String BlackList) {
String[] Lists = getListslistArray(Listname);
//================general Lists==================
for (int u=0; u <= Lists.length -1; u++) {
if (BlackList.equals(Lists[u])) {
return true;
}
}
return false;
}
//================generel Lists==================
// Gets a Array of all lines(Items) of a (list)file
/**
* Read lines of a file into an ArrayList.
*
* @param listFile the file
* @return the resulting array as an ArrayList
*/
public static ArrayList getListArray(File listFile){
String line;
ArrayList list = new ArrayList();
@ -140,7 +160,13 @@ public class listManager {
return list;
}
// Writes the Liststring to a file
/**
* Write a String to a file (used for string representation of lists).
*
* @param listFile the file to write to
* @param out the String to write
* @return returns <code>true</code> if successful, <code>false</code> otherwise
*/
public static boolean writeList(File listFile, String out) {
BufferedWriter bw = null;
try {
@ -155,7 +181,13 @@ public class listManager {
}
}
// overloaded function to write an array
/**
* Write elements of an Array of Strings to a file (one element per line).
*
* @param listFile the file to write to
* @param list the Array to write
* @return returns <code>true</code> if successful, <code>false</code> otherwise
*/
public static boolean writeList(File listFile, String[] list){
StringBuffer out = new StringBuffer();
for(int i=0;i < list.length; i++){
@ -166,11 +198,19 @@ public class listManager {
return writeList(listFile, new String(out)); //(File, String)
}
// same as below
public static String getListString(String filename, boolean withcomments) {
File listFile = new File(listsPath ,filename);
return getListString(listFile, withcomments);
}
/**
* Read lines of a text file into a String, optionally ignoring comments.
*
* @param listFile the File to read from.
* @param withcomments If <code>false</code> ignore lines starting with '#'.
* @return String representation of the file content.
*/
public static String getListString(File listFile, boolean withcomments){
StringBuffer temp = new StringBuffer();
@ -203,6 +243,13 @@ public class listManager {
return getDirListing(dir);
}
/**
* Read content of a directory into a String array of file names.
*
* @param dir The directory to get the file listing from. If it doesn't exist yet,
* it will be created.
* @return array of file names
*/
public static String[] getDirListing(File dir){
String[] fileListString;
File[] fileList;
@ -221,9 +268,11 @@ public class listManager {
return null;
}
// same as below
public static ArrayList getDirsRecursive(File dir, String notdir){
return getDirsRecursive(dir, notdir, true);
}
/**
* Returns a List of all dirs and subdirs as File Objects
*
@ -246,92 +295,89 @@ public class listManager {
}
return resultList;
}
public static String arraylist2string(ArrayList list){
Iterator it=list.iterator();
String ret="";
if(it.hasNext()){
ret=(String) it.next();
while(it.hasNext()){
ret+=","+(String)it.next();
//================Helper functions for collection conversion==================
/**
* Simple conversion of a Collection of Strings to a comma separated String.
* If the implementing Collection subclass guaranties an order of its elements,
* the substrings of the result will have the same order.
*
* @param col a Collection of Strings.
* @return String with elements from set separated by comma.
*/
public static String collection2string(Collection col){
StringBuffer str = new StringBuffer();
if (col != null && (col.size() > 0)) {
Iterator it = col.iterator();
str.append((String) it.next());
while(it.hasNext()) {
str.append(",").append((String) it.next());
}
}
return ret;
return str.toString();
}
/**
* @see listManager#string2vector(String)
*/
public static ArrayList string2arraylist(String string){
ArrayList ret=new ArrayList();
String[] hashes=string.split(",");
if(string.indexOf(",") > -1){
for(int i=0;i<hashes.length;i++){
ret.add(hashes[i]);
}
}else{
ret = new ArrayList();
if(!string.equals("")){
ret.add(string);
}
ArrayList l;
if (string != null) {
l = new ArrayList(Arrays.asList(string.split(",")));
} else {
l = new ArrayList();
}
return ret;
}
public static String vector2string(Vector vector){
Iterator it=vector.iterator();
String ret="";
if(it.hasNext()){
ret=(String) it.next();
while(it.hasNext()){
ret+=","+(String)it.next();
}
}
return ret;
return l;
}
/**
* Simple conversion of a comma separated list to a unified Set.
*
* @param string list of comma separated Strings
* @return resulting Set or empty Set if string is <code>null</code>
*/
public static Set string2set(String string){
HashSet set;
if (string != null) {
set = new HashSet(Arrays.asList(string.split(",")));
} else {
set = new HashSet();
}
return set;
}
/**
* Simple conversion of a comma separated list to a Vector containing
* the order of the substrings.
*
* @param string list of comma separated Strings
* @return resulting Vector or empty Vector if string is <code>null</code>
*/
public static Vector string2vector(String string){
Vector ret=new Vector();
String[] hashes=string.split(",");
if(string.indexOf(",") > -1){
for(int i=0;i<hashes.length;i++){
ret.add(hashes[i]);
}
}else{
ret = new Vector();
if(!string.equals("")){
ret.add(string);
}
}
return ret;
}
public static String hashset2string(HashSet hashset){
StringBuffer ret=new StringBuffer();
if(hashset!=null){
Iterator it=hashset.iterator();
if(it.hasNext()){
ret.append((String)it.next());
while(it.hasNext()){
ret.append(",").append((String)it.next());
}
}
}
return new String(ret);
}
public static HashSet string2hashset(String string){
HashSet ret=new HashSet();
String[] hashes=string.split(",");
if(string.indexOf(",") > -1){
for(int i=0;i<hashes.length;i++){
ret.add(hashes[i]);
}
}else{
ret = new HashSet();
if(!string.equals("")){
ret.add(string);
}
}
return ret;
}
Vector v;
if (string != null) {
v = new Vector(Arrays.asList(string.split(",")));
} else {
v = new Vector();
}
return v;
}
//=============Blacklist specific================
// load all active Blacklists in the Proxy
/**
* Load or reload all active Blacklists
*/
public static void reloadBlacklists(){
String supportedBlacklistTypesStr = abstractURLPattern.BLACKLIST_TYPES_STRING;
String[] supportedBlacklistTypes = supportedBlacklistTypesStr.split(",");

6
source/de/anomic/plasma/plasmaSwitchboard.java
View File

@ -993,14 +993,14 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser
}
// load the black-list / inspired by [AS]
File ulrBlackListFile = new File(getRootPath(), getConfig(LISTS_PATH, LISTS_PATH_DEFAULT));
File blacklistsPath = new File(getRootPath(), getConfig(LISTS_PATH, LISTS_PATH_DEFAULT));
String blacklistClassName = getConfig(BLACKLIST_CLASS, BLACKLIST_CLASS_DEFAULT);
this.log.logConfig("Starting blacklist engine ...");
try {
Class blacklistClass = Class.forName(blacklistClassName);
Constructor blacklistClassConstr = blacklistClass.getConstructor( new Class[] { File.class } );
urlBlacklist = (plasmaURLPattern) blacklistClassConstr.newInstance(new Object[] { ulrBlackListFile });
urlBlacklist = (plasmaURLPattern) blacklistClassConstr.newInstance(new Object[] { blacklistsPath });
this.log.logFine("Used blacklist engine class: " + blacklistClassName);
this.log.logConfig("Using blacklist engine: " + urlBlacklist.getEngineInfo());
} catch (Exception e) {
@ -1013,7 +1013,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser
this.log.logConfig("Loading backlist data ...");
listManager.switchboard = this;
listManager.listsPath = ulrBlackListFile;
listManager.listsPath = blacklistsPath;
listManager.reloadBlacklists();
// load badwords (to filter the topwords)

47
source/de/anomic/plasma/urlPattern/abstractURLPattern.java
View File

@ -4,6 +4,7 @@
// (C) by Michael Peter Christen; mc@anomic.de
// first published on http://www.anomic.de
// Frankfurt, Germany, 2005
// (C) 2007 by Bjoern Krombholz
// last major change: 12. August 2006 (theli) ?
//
// $LastChangedDate$
@ -46,14 +47,15 @@
package de.anomic.plasma.urlPattern;
import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.Map;
import java.util.Set;
import de.anomic.kelondro.kelondroMSetTools;
import de.anomic.yacy.yacyURL;
@ -141,17 +143,48 @@ public abstract class abstractURLPattern implements plasmaURLPattern {
}
}
public void loadList(String blacklistType, String filenames, String sep) {
HashMap blacklistMap = getBlacklistMap(blacklistType);
String[] filenamesarray = filenames.split(",");
public void loadList(blacklistFile blFile, String sep) {
HashMap blacklistMap = getBlacklistMap(blFile.getType());
Set loadedBlacklist;
Map.Entry loadedEntry;
ArrayList paths, loadedPaths;
if (filenamesarray.length > 0) {
for (int i = 0; i < filenamesarray.length; i++) {
blacklistMap.putAll(kelondroMSetTools.loadMapMultiValsPerKey(new File(this.blacklistRootPath, filenamesarray[i]).toString(), sep));
String[] fileNames = blFile.getFileNamesUnified();
if (fileNames.length > 0) {
for (int i = 0; i < fileNames.length; i++) {
// make sure all requested blacklist files exist
File file = new File(this.blacklistRootPath, fileNames[i]);
try {
file.createNewFile();
} catch (IOException e) { /* */ }
// join all blacklists from files into one internal blacklist map
loadedBlacklist = kelondroMSetTools.loadMapMultiValsPerKey(file.toString(), sep).entrySet();
for (Iterator mi = loadedBlacklist.iterator(); mi.hasNext(); ) {
loadedEntry = (Map.Entry) mi.next();
loadedPaths = (ArrayList) loadedEntry.getValue();
// create new entry if host mask unknown, otherwise merge
// existing one with path patterns from blacklist file
paths = (ArrayList) blacklistMap.get(loadedEntry.getKey());
if (paths == null) {
blacklistMap.put(loadedEntry.getKey(), loadedPaths);
} else {
// TODO check for duplicates? (refactor List -> Set)
paths.addAll(loadedPaths);
}
}
}
}
}
public void loadList(String blacklistType, String fileNames, String sep) {
// method for not breaking older plasmaURLPattern interface
blacklistFile blFile = new blacklistFile(fileNames, blacklistType);
loadList(blFile, sep);
}
public void removeAll(String blacklistType, String host) {
HashMap blacklistMap = getBlacklistMap(blacklistType);
blacklistMap.remove(host);

16
source/de/anomic/plasma/urlPattern/plasmaURLPattern.java
View File

@ -1,6 +1,8 @@
package de.anomic.plasma.urlPattern;
import java.io.File;
import java.util.Arrays;
import java.util.HashSet;
import de.anomic.yacy.yacyURL;
@ -24,6 +26,20 @@ public interface plasmaURLPattern {
}
public String getFileName() { return this.filename; }
/**
* Construct a unified array of file names from comma seperated file name
* list.
*
* @return unified String array of file names
*/
public String[] getFileNamesUnified() {
HashSet hs = new HashSet(Arrays.asList(this.filename.split(",")));
return (String[]) hs.toArray(new String[hs.size()]);
}
public String getType() { return this.type; }
}