Marown/yacy_search_server
1
0
Fork 0
mirror of https://github.com/yacy/yacy_search_server.git synced 2024-09-19 00:01:41 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity

*) fixed bug which made it possible to write files outside of the DATA/LIST directory when creating a new blacklist

Browse Source 
*) a blacklist will only be created if no blacklist with same name exists (some refactoring has been necessary for this)
*) further minor fixes
*) to be continued...

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@5301 6c8d7289-2bf4-0310-a012-ef5d649a1542
This commit is contained in:
low012 2008-10-25 00:11:03 +00:00
parent 0f9c0bd0d5
commit 00e27e5050
10 changed files with 133 additions and 86 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
htroot/BlacklistCleaner_p.java
View File

@ -38,6 +38,7 @@ import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.regex.Pattern;
@ -129,19 +130,21 @@ public class BlacklistCleaner_p {
return prop;
}
private static void putBlacklists(final serverObjects prop, final String[] lists, final String selected) {
private static void putBlacklists(final serverObjects prop, final List<String> lists, final String selected) {
boolean supported = false;
for (int i=0; i<supportedBLEngines.length && !supported; i++) {
supported |= (plasmaSwitchboard.urlBlacklist.getClass() == supportedBLEngines[i]);
}
if (supported) {
if (lists.length > 0) {
if (lists.size() > 0) {
prop.put("disabled", "0");
prop.put(DISABLED + "blacklists", lists.length);
for (int i=0; i<lists.length; i++) {
prop.putHTML(DISABLED + BLACKLISTS + i + "_name", lists[i]);
prop.put(DISABLED + BLACKLISTS + i + "_selected", (lists[i].equals(selected)) ? "1" : "0");
prop.put(DISABLED + "blacklists", lists.size());
int count = 0;
for (String list : lists) {
prop.putHTML(DISABLED + BLACKLISTS + count + "_name", list);
prop.put(DISABLED + BLACKLISTS + count + "_selected", (list.equals(selected)) ? "1" : "0");
count++;
}
} else {
prop.put("disabled", "2");

28
htroot/Blacklist_p.html
View File

@ -40,24 +40,38 @@
#(/testlist)#
</form>
</fieldset>
<fieldset class="selectList">
<legend>Select list:</legend>
<form action="Blacklist_p.html" method="post" enctype="multipart/form-data">#(disabled)#
<div style="display:inline;">
#(disabled)#
<form action="Blacklist_p.html" method="post" enctype="multipart/form-data">
<select name="selectedListName" size="1">
#{blackLists}#
<option value="#[name]#" #(selected)#::selected="selected"#(/selected)#>#[name]# [#(shared)#not shared::shared#(/shared)#] #{active}# #[blTypeName]##{/active}#</option>
#{/blackLists}#
</select>
<input type="submit" name="selectList" value="select" />
</div>::#(/disabled)#
<div style="display:inline;">
<label for="newListName">New list</label>:
<input type="text" id="newListName" name="newListName" />
</form>
::
#(/disabled)#
</fieldset>
<fieldset class=" createList">
<legend>Create new list:</legend>
#(error)#
::
<p>Unable to create list <strong>#[name]#</strong> since it contains illegal characters.
A legal name is made up from a letter, digit, minus, plus or underscore as the first character
followed by letters, digits, minus, plus, underscores or dots.</p>
::
<p>Unable to create list <strong>#[name]#</strong> since a list of this name exists already.</p>
#(/error)#
<form action="Blacklist_p.html" method="post" enctype="multipart/form-data">
<input type="text" id="newListName" size ="50" name="newListName" />
<input type="submit" name="createNewList" value="create" />
</div>
</form>
</fieldset>
#(disabled)#
<fieldset class="listSettings">
<legend>Settings for this list</legend>

47
htroot/Blacklist_p.java
View File

@ -49,6 +49,7 @@ import de.anomic.server.serverSwitch;
import de.anomic.server.logging.serverLog;
import de.anomic.yacy.yacySeed;
import de.anomic.yacy.yacyURL;
import java.util.List;
public class Blacklist_p {
private final static String DISABLED = "disabled_";
@ -66,10 +67,13 @@ public class Blacklist_p {
final String supportedBlacklistTypesStr = indexAbstractReferenceBlacklist.BLACKLIST_TYPES_STRING;
final String[] supportedBlacklistTypes = supportedBlacklistTypesStr.split(",");
// loading all blacklist files located in the directory
final List<String> dirlist = listManager.getDirListing(listManager.listsPath);
String blacklistToUse = null;
final serverObjects prop = new serverObjects();
prop.putHTML("blacklistEngine", plasmaSwitchboard.urlBlacklist.getEngineInfo());
prop.putHTML("asd", "0");
// do all post operations
if (post != null) {
@ -113,8 +117,18 @@ prop.putHTML("asd", "0");
return prop;
}
// Check if blacklist name only consists of "legal" characters.
// This is mainly done to prevent files from being written to other directories
// than the LISTS directory.
if (!blacklistToUse.matches("^[\\p{L}\\d\\+\\-_]+[\\p{L}\\d\\+\\-_.]*(\\.black){0,1}$")) {
prop.put("error", 1);
prop.putHTML("error_name", blacklistToUse);
blacklistToUse = null;
} else {
if (!blacklistToUse.endsWith(".black")) blacklistToUse += ".black";
if (!dirlist.contains(blacklistToUse)) {
try {
final File newFile = new File(listManager.listsPath, blacklistToUse);
newFile.createNewFile();
@ -127,6 +141,12 @@ prop.putHTML("asd", "0");
listManager.updateListSet(supportedBlacklistTypes[blTypes] + ".BlackLists", blacklistToUse);
}
} catch (final IOException e) {/* */}
} else {
prop.put("error", 2);
prop.putHTML("error_name", blacklistToUse);
blacklistToUse = null;
}
}
} else if (post.containsKey("deleteList")) {
/* ===========================================================
@ -250,7 +270,7 @@ prop.putHTML("asd", "0");
}
/* ===========================================================
* Thent add new entry to blacklist
* Then add new entry to blacklist
* =========================================================== */
temp = addBlacklistEntry(post.get("currentBlacklist"),
post.get("editedBlacklistEntry"), header, supportedBlacklistTypes);
@ -273,19 +293,15 @@ prop.putHTML("asd", "0");
}
// loading all blacklist files located in the directory
final String[] dirlist = listManager.getDirListing(listManager.listsPath);
// if we have not chosen a blacklist until yet we use the first file
if (blacklistToUse == null && dirlist != null && dirlist.length > 0) {
blacklistToUse = dirlist[0];
if (blacklistToUse == null && dirlist != null && dirlist.size() > 0) {
blacklistToUse = dirlist.get(0);
}
// Read the blacklist items from file
if (blacklistToUse != null) {
int entryCount = 0;
final ArrayList<String> list = listManager.getListArray(new File(listManager.listsPath, blacklistToUse));
final List<String> list = listManager.getListArray(new File(listManager.listsPath, blacklistToUse));
// sort them
final String[] sortedlist = new String[list.size()];
@ -331,23 +347,24 @@ prop.putHTML("asd", "0");
// List BlackLists
int blacklistCount = 0;
if (dirlist != null) {
for (int i = 0; i <= dirlist.length - 1; i++) {
prop.putXML(DISABLED + BLACKLIST + blacklistCount + "_name", dirlist[i]);
for (String element : dirlist) {
prop.putXML(DISABLED + BLACKLIST + blacklistCount + "_name", element);
prop.put(DISABLED + BLACKLIST + blacklistCount + "_selected", "0");
if (dirlist[i].equals(blacklistToUse)) { //current List
if (element.equals(blacklistToUse)) { //current List
prop.put(DISABLED + BLACKLIST + blacklistCount + "_selected", "1");
for (int blTypes=0; blTypes < supportedBlacklistTypes.length; blTypes++) {
prop.putXML(DISABLED + "currentActiveFor_" + blTypes + "_blTypeName",supportedBlacklistTypes[blTypes]);
prop.put(DISABLED + "currentActiveFor_" + blTypes + "_checked",
listManager.listSetContains(supportedBlacklistTypes[blTypes] + ".BlackLists",dirlist[i]) ? "0" : "1");
listManager.listSetContains(supportedBlacklistTypes[blTypes] + ".BlackLists", element) ? "0" : "1");
}
prop.put(DISABLED + "currentActiveFor", supportedBlacklistTypes.length);
}
if (listManager.listSetContains(BLACKLIST_SHARED, dirlist[i])) {
if (listManager.listSetContains(BLACKLIST_SHARED, element)) {
prop.put(DISABLED + BLACKLIST + blacklistCount + "_shared", "1");
} else {
prop.put(DISABLED + BLACKLIST + blacklistCount + "_shared", "0");
@ -355,7 +372,7 @@ prop.putHTML("asd", "0");
int activeCount = 0;
for (int blTypes=0; blTypes < supportedBlacklistTypes.length; blTypes++) {
if (listManager.listSetContains(supportedBlacklistTypes[blTypes] + ".BlackLists",dirlist[i])) {
if (listManager.listSetContains(supportedBlacklistTypes[blTypes] + ".BlackLists", element)) {
prop.putHTML(DISABLED + BLACKLIST + blacklistCount + "_active_" + activeCount + "_blTypeName", supportedBlacklistTypes[blTypes]);
activeCount++;
}

18
htroot/ConfigAppearance_p.java
View File

@ -32,6 +32,7 @@ import java.io.IOException;
import java.io.PrintWriter;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import de.anomic.crawler.HTTPLoader;
import de.anomic.data.listManager;
@ -55,14 +56,14 @@ public class ConfigAppearance_p {
prop.put("currentskin", "");
prop.put("status", "0"); // nothing
String[] skinFiles = listManager.getDirListing(skinPath);
List<String> skinFiles = listManager.getDirListing(skinPath);
if (skinFiles == null) {
return prop;
}
// if there are no skins, use the current style as default
// normally only invoked at first start of YaCy
if (skinFiles.length == 0) {
if (skinFiles.size() == 0) {
try {
serverFileUtils.copy(new File(env.getRootPath(), "htroot/env/style.css"), new File(skinPath, "default.css"));
env.setConfig("currentSkin", "default");
@ -131,14 +132,15 @@ public class ConfigAppearance_p {
// reread skins
skinFiles = listManager.getDirListing(skinPath);
int i;
for (i = 0; i <= skinFiles.length - 1; i++) {
if (skinFiles[i].endsWith(".css")) {
prop.put("skinlist_" + i + "_file", skinFiles[i]);
prop.put("skinlist_" + i + "_name", skinFiles[i].substring(0, skinFiles[i].length() - 4));
int count = 0;
for (String skinFile : skinFiles) {
if (skinFile.endsWith(".css")) {
prop.put("skinlist_" + count + "_file", skinFile);
prop.put("skinlist_" + count + "_name", skinFile.substring(0, skinFile.length() - 4));
count++;
}
}
prop.put("skinlist", i);
prop.put("skinlist", count);
prop.putHTML("currentskin", env.getConfig("currentSkin", "default"));

23
htroot/ConfigLanguage_p.java
View File

@ -34,6 +34,7 @@ import java.io.PrintWriter;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import de.anomic.crawler.HTTPLoader;
import de.anomic.data.listManager;
@ -57,7 +58,7 @@ public class ConfigLanguage_p {
//prop.put("currentlang", ""); //is done by Translationtemplate
prop.put("status", "0");//nothing
String[] langFiles = listManager.getDirListing(langPath);
List<String> langFiles = listManager.getDirListing(langPath);
if(langFiles == null){
return prop;
}
@ -116,22 +117,24 @@ public class ConfigLanguage_p {
prop.put("langlist_0_name", ((langNames.get("default") == null) ? "default" : (String) langNames.get("default")));
prop.put("langlist_0_selected", "selected=\"selected\"");
for(i=0;i<= langFiles.length-1 ;i++){
if(langFiles[i].endsWith(".lng")){
int count = 0;
for(String langFile : langFiles){
if(langFile.endsWith(".lng")){
//+1 because of the virtual entry "default" at top
langKey = langFiles[i].substring(0, langFiles[i].length() -4);
langKey = langFile.substring(0, langFile.length() -4);
langName = langNames.get(langKey);
prop.put("langlist_"+(i+1)+"_file", langFiles[i]);
prop.put("langlist_"+(i+1)+"_name", ((langName == null) ? langKey : langName));
prop.put("langlist_" + (count + 1) + "_file", langFile);
prop.put("langlist_" + (count + 1) + "_name", ((langName == null) ? langKey : langName));
if(env.getConfig("locale.language", "default").equals(langKey)) {
prop.put("langlist_"+(i+1)+"_selected", "selected=\"selected\"");
prop.put("langlist_" + (count + 1) + "_selected", "selected=\"selected\"");
prop.put("langlist_0_selected", " "); // reset Default
} else {
prop.put("langlist_"+(i+1)+"_selected", " ");
prop.put("langlist_" + (count + 1) + "_selected", " ");
}
count++;
}
}
}
prop.put("langlist", (i+1));
prop.put("langlist", (count + 1));
//is done by Translationtemplate
//langName = (String) langNames.get(env.getConfig("locale.language", "default"));

3
htroot/sharedBlacklist_p.java
View File

@ -97,10 +97,12 @@ public class sharedBlacklist_p {
downloadURL = "http://" + IP + ":" + Port + "/yacy/list.html?col=black";
} else {
prop.put("status", STATUS_PEER_UNKNOWN);//YaCy-Peer not found
prop.putHTML("status_name", Hash);
prop.put("page", "1");
}
} else {
prop.put("status", STATUS_PEER_UNKNOWN);//YaCy-Peer not found
prop.putHTML("status_name", Hash);
prop.put("page", "1");
}
@ -117,6 +119,7 @@ public class sharedBlacklist_p {
otherBlacklist = nxTools.strings(HttpClient.wget(u.toString(), reqHeader, 1000), "UTF-8");
} catch (final Exception e) {
prop.put("status", STATUS_PEER_UNKNOWN);
prop.putHTML("status_name", Hash);
prop.put("page", "1");
}
}

13
htroot/xml/blacklists_p.java
View File

@ -21,6 +21,7 @@
package xml;
import java.io.File;
import java.util.ArrayList;
import java.util.List;
import de.anomic.data.listManager;
import de.anomic.http.httpRequestHeader;
@ -35,16 +36,16 @@ public class blacklists_p {
final serverObjects prop = new serverObjects();
listManager.listsPath = new File(listManager.switchboard.getRootPath(),listManager.switchboard.getConfig("listManager.listsPath", "DATA/LISTS"));
final String[] dirlist = listManager.getDirListing(listManager.listsPath);
final List<String> dirlist = listManager.getDirListing(listManager.listsPath);
int blacklistCount=0;
ArrayList<String> list;
int count;
if (dirlist != null) {
for (int i = 0; i <= dirlist.length - 1; i++) {
prop.putHTML("lists_" + blacklistCount + "_name", dirlist[i]);
for (String element : dirlist) {
prop.putHTML("lists_" + blacklistCount + "_name", element);
if (listManager.listSetContains("BlackLists.Shared", dirlist[i])) {
if (listManager.listSetContains("BlackLists.Shared", element)) {
prop.put("lists_" + blacklistCount + "_shared", "1");
} else {
prop.put("lists_" + blacklistCount + "_shared", "0");
@ -54,11 +55,11 @@ public class blacklists_p {
for (int j=0; j<types.length; j++) {
prop.put("lists_" + blacklistCount + "_types_" + j + "_name", types[j]);
prop.put("lists_" + blacklistCount + "_types_" + j + "_value",
listManager.listSetContains(types[j] + ".Blacklist", dirlist[i]) ? 1 : 0);
listManager.listSetContains(types[j] + ".Blacklist", element) ? 1 : 0);
}
prop.put("lists_" + blacklistCount + "_types", types.length);
list = listManager.getListArray(new File(listManager.listsPath, dirlist[i]));
list = listManager.getListArray(new File(listManager.listsPath, element));
count=0;
for (int j=0;j<list.size();++j){

13
source/de/anomic/data/listManager.java
View File

@ -37,6 +37,8 @@ import java.util.Arrays;
import java.util.Collection;
import java.util.HashSet;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.Set;
import java.util.Vector;
@ -219,7 +221,7 @@ public class listManager {
}
// get a Directory Listing as a String Array
public static String[] getDirListing(final String dirname){
public static List<String> getDirListing(final String dirname){
final File dir = new File(dirname);
return getDirListing(dir);
}
@ -231,8 +233,8 @@ public class listManager {
* it will be created.
* @return array of file names
*/
public static String[] getDirListing(final File dir){
String[] fileListString;
public static List<String> getDirListing(final File dir){
List<String> ret = new LinkedList();
File[] fileList;
if (dir != null ) {
@ -240,11 +242,10 @@ public class listManager {
dir.mkdir();
}
fileList = dir.listFiles();
fileListString = new String[fileList.length];
for (int i=0; i<= fileList.length-1; i++) {
fileListString[i]=fileList[i].getName();
ret.add(fileList[i].getName());
}
return fileListString;
return ret;
}
return null;
}

10
source/de/anomic/plasma/plasmaSearchAPI.java
View File

@ -30,6 +30,7 @@ package de.anomic.plasma;
import java.util.Date;
import java.util.Iterator;
import java.util.List;
import de.anomic.data.listManager;
import de.anomic.index.indexRWIEntry;
@ -194,9 +195,10 @@ public class plasmaSearchAPI {
}
}
public static void putBlacklists(final serverObjects prop, final String[] lists) {
prop.put("genUrlList_blacklists", lists.length);
for (int i=0; i<lists.length; i++)
prop.put("genUrlList_blacklists_" + i + "_name", lists[i]);
public static void putBlacklists(final serverObjects prop, final List<String> lists) {
prop.put("genUrlList_blacklists", lists.size());
int i = 0;
for (String list : lists)
prop.put("genUrlList_blacklists_" + i++ + "_name", list);
}
}

9
source/migration.java
View File

@ -20,6 +20,7 @@
import java.io.File;
import java.io.IOException;
import java.util.List;
import de.anomic.data.listManager;
import de.anomic.http.httpd;
@ -80,12 +81,12 @@ public class migration {
final File skinsPath = sb.getConfigPath("skinPath", "DATA/SKINS");
final File defaultSkinsPath = new File(sb.getRootPath(), "skins");
if(defaultSkinsPath.exists()){
final String[] skinFiles = listManager.getDirListing(defaultSkinsPath.getAbsolutePath());
final List<String> skinFiles = listManager.getDirListing(defaultSkinsPath.getAbsolutePath());
mkdirs(skinsPath);
for(int i=0;i<skinFiles.length;i++){
if(skinFiles[i].endsWith(".css")){
for(String skinFile : skinFiles){
if(skinFile.endsWith(".css")){
try{
serverFileUtils.copy(new File(defaultSkinsPath, skinFiles[i]), new File(skinsPath, skinFiles[i]));
serverFileUtils.copy(new File(defaultSkinsPath, skinFile), new File(skinsPath, skinFile));
}catch(final IOException e){}
}
}