Marown/yacy_search_server
1
0
Fork 0
mirror of https://github.com/yacy/yacy_search_server.git synced 2024-09-19 00:01:41 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity

major step forward to network switching (target is easy switch to intranet or other networks .. and back)

Browse Source 
This change is inspired by the need to see a network connected to the index it creates in a indexing team.
It is not possible to divide the network and the index. Therefore all control files for the network was moved to the network within the INDEX/<network-name> subfolder.
The remaining YACYDB is superfluous and can be deleted.
The yacyDB and yacyNews data structures are now part of plasmaWordIndex. Therefore all methods, using static access to yacySeedDB had to be rewritten. A special problem had been all the port forwarding methods which had been tightly mixed with seed construction. It was not possible to move the port forwarding functions to the place, meaning and usage of plasmaWordIndex. Therefore the port forwarding had been deleted (I guess nobody used it and it can be simulated by methods outside of YaCy).
The mySeed.txt is automatically moved to the current network position. A new effect causes that every network will create a different local seed file, which is ok, since the seed identifies the peer only against the network (it is the purpose of the seed hash to give a peer a location within the DHT).
No other functional change has been made. The next steps to enable network switcing are:
- shift of crawler tables from PLASMADB into the network (crawls are also network-specific)
- possibly shift of plasmaWordIndex code into yacy package (index management is network-specific)
- servlet to switch networks 

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@4765 6c8d7289-2bf4-0310-a012-ef5d649a1542
This commit is contained in:
orbiter 2008-05-05 23:13:47 +00:00
parent d70a472460
commit d2ba1fd2ab
110 changed files with 1100 additions and 2114 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.classpath
View File

@ -39,5 +39,6 @@
<classpathentry kind="lib" path="libx/bcprov-jdk14-132.jar"/>
<classpathentry kind="lib" path="libx/commons-codec-1.3.jar"/>
<classpathentry kind="lib" path="lib/commons-httpclient-3.1.jar"/>
<classpathentry kind="lib" path="/Users/admin/Documents/workspace/yacy/lib/commons-httpclient-3.1.jar"/>
<classpathentry kind="output" path="gen"/>
</classpath>

16
defaults/yacy.init
View File

@ -624,22 +624,6 @@ proxy.sendViaHeader=true
# Specifies if the proxy should send the X-Forwarded-For header
proxy.sendXForwardedForHeader=true
# Configuration options needed to configure server port forwarding
portForwarding.Enabled=false
portForwarding.Type=none
# port forwarding via sch
portForwarding.sch=de.anomic.server.portForwarding.sch.serverPortForwardingSch
portForwarding.sch.UseProxy=false
portForwarding.sch.Port=
portForwarding.sch.Host=
portForwarding.sch.HostPort=22
portForwarding.sch.HostUser=
portForwarding.sch.HostPwd=
# port forwarding via upnp
portForwarding.upnp=de.anomic.server.portForwarding.upnp.serverPortForwardingUpnp
# msgForwarding: Specifies if yacy should forward received messages via
# email to the configured email address
msgForwardingEnabled=false

25
htroot/AccessTracker_p.java
View File

@ -40,7 +40,6 @@ import de.anomic.plasma.plasmaSwitchboard;
import de.anomic.server.serverDate;
import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch;
import de.anomic.yacy.yacyCore;
import de.anomic.yacy.yacySeed;
public class AccessTracker_p {
@ -53,8 +52,8 @@ public class AccessTracker_p {
return accessClone;
}
public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch<?> sb) {
plasmaSwitchboard switchboard = (plasmaSwitchboard) sb;
public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch<?> env) {
plasmaSwitchboard sb = (plasmaSwitchboard) env;
// return variable that accumulates replacements
serverObjects prop = new serverObjects();
@ -66,14 +65,14 @@ public class AccessTracker_p {
int maxCount = 1000;
boolean dark = true;
if (page == 0) {
Iterator<String> i = switchboard.accessHosts();
Iterator<String> i = sb.accessHosts();
String host;
TreeMap<Long, String> access;
int entCount = 0;
try {
while ((entCount < maxCount) && (i.hasNext())) {
host = (String) i.next();
access = switchboard.accessTrack(host);
access = sb.accessTrack(host);
prop.putHTML("page_list_" + entCount + "_host", host);
prop.putNum("page_list_" + entCount + "_countSecond", access.tailMap(new Long(System.currentTimeMillis() - 1000)).size());
prop.putNum("page_list_" + entCount + "_countMinute", access.tailMap(new Long(System.currentTimeMillis() - 1000 * 60)).size());
@ -91,7 +90,7 @@ public class AccessTracker_p {
TreeMap<Long, String> access;
Map.Entry<Long, String> entry;
if (host.length() > 0) {
access = switchboard.accessTrack(host);
access = sb.accessTrack(host);
if (access != null) {
try {
Iterator<Map.Entry<Long, String>> ii = treemapclone(access).entrySet().iterator();
@ -106,10 +105,10 @@ public class AccessTracker_p {
}
} else {
try {
Iterator<String> i = switchboard.accessHosts();
Iterator<String> i = sb.accessHosts();
while ((entCount < maxCount) && (i.hasNext())) {
host = (String) i.next();
access = switchboard.accessTrack(host);
access = sb.accessTrack(host);
Iterator<Map.Entry<Long, String>> ii = treemapclone(access).entrySet().iterator();
while (ii.hasNext()) {
entry = ii.next();
@ -125,7 +124,7 @@ public class AccessTracker_p {
prop.put("page_num", entCount);
}
if ((page == 2) || (page == 4)) {
ArrayList<plasmaSearchQuery> array = (page == 2) ? switchboard.localSearches : switchboard.remoteSearches;
ArrayList<plasmaSearchQuery> array = (page == 2) ? sb.localSearches : sb.remoteSearches;
plasmaSearchQuery searchProfile;
int m = Math.min(maxCount, array.size());
long qcountSum = 0;
@ -173,10 +172,10 @@ public class AccessTracker_p {
prop.putNum("page_urltime_avg", (double) utimeSum / m);
prop.putNum("page_snippettime_avg", (double) stimeSum / m);
prop.putNum("page_resulttime_avg", (double) rtimeSum / m);
prop.putNum("page_total", (page == 2) ? switchboard.localSearches.size() : switchboard.remoteSearches.size());
prop.putNum("page_total", (page == 2) ? sb.localSearches.size() : sb.remoteSearches.size());
}
if ((page == 3) || (page == 5)) {
Iterator<Entry<String, TreeSet<Long>>> i = (page == 3) ? switchboard.localSearchTracker.entrySet().iterator() : switchboard.remoteSearchTracker.entrySet().iterator();
Iterator<Entry<String, TreeSet<Long>>> i = (page == 3) ? sb.localSearchTracker.entrySet().iterator() : sb.remoteSearchTracker.entrySet().iterator();
String host;
TreeSet<Long> handles;
int entCount = 0;
@ -204,7 +203,7 @@ public class AccessTracker_p {
prop.put("page_list_" + entCount + "_dark", ((dark) ? 1 : 0) ); dark =! dark;
prop.putHTML("page_list_" + entCount + "_host", host);
if (page == 5) {
yacySeed remotepeer = yacyCore.seedDB.lookupByIP(natLib.getInetAddress(host), true, true, true);
yacySeed remotepeer = sb.wordIndex.seedDB.lookupByIP(natLib.getInetAddress(host), true, true, true);
prop.putHTML("page_list_" + entCount + "_peername", (remotepeer == null) ? "UNKNOWN" : remotepeer.getName());
}
prop.putNum("page_list_" + entCount + "_count", handles.size());
@ -215,7 +214,7 @@ public class AccessTracker_p {
} catch (ConcurrentModificationException e) {} // we dont want to synchronize this
prop.put("page_list", entCount);
prop.putNum("page_num", entCount);
prop.putNum("page_total", (page == 3) ? switchboard.localSearches.size() : switchboard.remoteSearches.size());
prop.putNum("page_total", (page == 3) ? sb.localSearches.size() : sb.remoteSearches.size());
prop.putNum("page_qph_sum", qphSum);
}
// return rewrite properties

24
htroot/Banner.java
View File

@ -46,10 +46,10 @@
import de.anomic.http.httpHeader;
import de.anomic.plasma.plasmaGrafics;
import de.anomic.plasma.plasmaSwitchboard;
import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch;
import de.anomic.ymage.ymageMatrix;
import de.anomic.yacy.yacyCore;
import de.anomic.yacy.yacySeed;
import java.awt.image.BufferedImage;
@ -61,7 +61,7 @@ import javax.imageio.ImageIO;
public class Banner {
public static ymageMatrix respond(httpHeader header, serverObjects post, serverSwitch<?> env) throws IOException {
plasmaSwitchboard sb = (plasmaSwitchboard) env;
final String IMAGE = "htroot/env/grafics/yacy.gif";
int width = 468;
int height = 60;
@ -84,14 +84,14 @@ public class Banner {
double myqph = 0;
String type = "";
String network = env.getConfig("network.unit.name", "unspecified").toUpperCase();
int peers = yacyCore.seedDB.sizeConnected() + 1; // the '+ 1': the own peer is not included in sizeConnected()
long nlinks = yacyCore.seedDB.countActiveURL();
long nwords = yacyCore.seedDB.countActiveRWI();
double nqpm = yacyCore.seedDB.countActiveQPM();
long nppm = yacyCore.seedDB.countActivePPM();
int peers = sb.wordIndex.seedDB.sizeConnected() + 1; // the '+ 1': the own peer is not included in sizeConnected()
long nlinks = sb.wordIndex.seedDB.countActiveURL();
long nwords = sb.wordIndex.seedDB.countActiveRWI();
double nqpm = sb.wordIndex.seedDB.countActiveQPM();
long nppm = sb.wordIndex.seedDB.countActivePPM();
double nqph = 0;
yacySeed seed = yacyCore.seedDB.mySeed();
yacySeed seed = sb.wordIndex.seedDB.mySeed();
if (seed != null){
name = seed.get(yacySeed.NAME, "-").toUpperCase();
links = Long.parseLong(seed.get(yacySeed.LCOUNT, "0"));
@ -99,19 +99,19 @@ public class Banner {
myppm = seed.getPPM();
myqph = 60d * seed.getQPM();
if (yacyCore.seedDB.mySeed().isVirgin()) {
if (sb.wordIndex.seedDB.mySeed().isVirgin()) {
type = "VIRGIN";
nqph = Math.round(6000d * nqpm) / 100d;
} else if(yacyCore.seedDB.mySeed().isJunior()) {
} else if(sb.wordIndex.seedDB.mySeed().isJunior()) {
type = "JUNIOR";
nqph = Math.round(6000d * nqpm) / 100d;
} else if(yacyCore.seedDB.mySeed().isSenior()) {
} else if(sb.wordIndex.seedDB.mySeed().isSenior()) {
type = "SENIOR";
nlinks = nlinks + links;
nwords = nwords + words;
nqph = Math.round(6000d * nqpm + 100d * myqph) / 100d;
nppm = nppm + myppm;
} else if(yacyCore.seedDB.mySeed().isPrincipal()) {
} else if(sb.wordIndex.seedDB.mySeed().isPrincipal()) {
type = "PRINCIPAL";
nlinks = nlinks + links;
nwords = nwords + words;

8
htroot/Blacklist_p.java
View File

@ -65,7 +65,6 @@ import de.anomic.index.indexReferenceBlacklist;
import de.anomic.plasma.plasmaSwitchboard;
import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch;
import de.anomic.yacy.yacyCore;
import de.anomic.yacy.yacySeed;
import de.anomic.yacy.yacyURL;
@ -75,7 +74,8 @@ public class Blacklist_p {
private final static String BLACKLIST_SHARED = "BlackLists.Shared";
public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch<?> env) {
plasmaSwitchboard sb = (plasmaSwitchboard) env;
// initialize the list manager
listManager.switchboard = (plasmaSwitchboard) env;
listManager.listsPath = new File(listManager.switchboard.getRootPath(),listManager.switchboard.getConfig("listManager.listsPath", "DATA/LISTS"));
@ -321,11 +321,11 @@ prop.putHTML("asd", "0");
// List known hosts for BlackList retrieval
if (yacyCore.seedDB != null && yacyCore.seedDB.sizeConnected() > 0) { // no nullpointer error
if (sb.wordIndex.seedDB != null && sb.wordIndex.seedDB.sizeConnected() > 0) { // no nullpointer error
int peerCount = 0;
try {
TreeMap<String, String> hostList = new TreeMap<String, String>();
final Iterator<yacySeed> e = yacyCore.seedDB.seedsConnected(true, false, null, (float) 0.0);
final Iterator<yacySeed> e = sb.wordIndex.seedDB.seedsConnected(true, false, null, (float) 0.0);
while (e.hasNext()) {
yacySeed seed = (yacySeed) e.next();
if (seed != null) hostList.put(seed.get(yacySeed.NAME, "nameless"),seed.hash);

39
htroot/Blog.java
View File

@ -60,7 +60,6 @@ import de.anomic.http.httpHeader;
import de.anomic.plasma.plasmaSwitchboard;
import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch;
import de.anomic.yacy.yacyCore;
import de.anomic.yacy.yacyNewsPool;
import de.anomic.yacy.yacyNewsRecord;
@ -76,11 +75,11 @@ public class Blog {
}
public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch<?> env) {
final plasmaSwitchboard switchboard = (plasmaSwitchboard) env;
final plasmaSwitchboard sb = (plasmaSwitchboard) env;
final serverObjects prop = new serverObjects();
blogBoard.BlogEntry page = null;
boolean hasRights = switchboard.verifyAuthentication(header, true);
boolean hasRights = sb.verifyAuthentication(header, true);
//final int display = (hasRights || post == null) ? 1 : post.getInt("display", 0);
//prop.put("display", display);
@ -88,7 +87,7 @@ public class Blog {
final boolean xml = ((String)header.get(httpHeader.CONNECTION_PROP_PATH)).endsWith(".xml");
final String address = yacyCore.seedDB.mySeed().getPublicAddress();
final String address = sb.wordIndex.seedDB.mySeed().getPublicAddress();
if(hasRights) {
prop.put("mode_admin", "1");
@ -97,16 +96,16 @@ public class Blog {
}
if (post == null) {
prop.putHTML("peername", yacyCore.seedDB.mySeed().getName());
prop.putHTML("peername", sb.wordIndex.seedDB.mySeed().getName());
prop.put("address", address);
return putBlogDefault(prop, switchboard, address, 0, 10, hasRights, xml);
return putBlogDefault(prop, sb, address, 0, 10, hasRights, xml);
}
final int start = post.getInt("start",0); //indicates from where entries should be shown
final int num = post.getInt("num",10); //indicates how many entries should be shown
if(!hasRights){
final userDB.Entry userentry = switchboard.userDB.proxyAuth((String)header.get("Authorization", "xxxxxx"));
final userDB.Entry userentry = sb.userDB.proxyAuth((String)header.get("Authorization", "xxxxxx"));
if(userentry != null && userentry.hasRight(userDB.Entry.BLOG_RIGHT)){
hasRights=true;
} else if(post.containsKey("login")) {
@ -121,13 +120,13 @@ public class Blog {
String StrAuthor = post.get("author", "");
if (StrAuthor.equals("anonymous")) {
StrAuthor = switchboard.blogDB.guessAuthor(ip);
StrAuthor = sb.blogDB.guessAuthor(ip);
if (StrAuthor == null || StrAuthor.length() == 0) {
if (de.anomic.yacy.yacyCore.seedDB.mySeed() == null) {
if (sb.wordIndex.seedDB.mySeed() == null) {
StrAuthor = "anonymous";
} else {
StrAuthor = de.anomic.yacy.yacyCore.seedDB.mySeed().get("Name", "anonymous");
StrAuthor = sb.wordIndex.seedDB.mySeed().get("Name", "anonymous");
}
}
}
@ -140,12 +139,12 @@ public class Blog {
}
if(hasRights && post.containsKey("delete") && post.get("delete").equals("sure")) {
page = switchboard.blogDB.readBlogEntry(pagename);
page = sb.blogDB.readBlogEntry(pagename);
final Iterator<String> i = page.getComments().iterator();
while(i.hasNext()) {
switchboard.blogCommentDB.delete(i.next());
sb.blogCommentDB.delete(i.next());
}
switchboard.blogDB.deleteBlogEntry(pagename);
sb.blogDB.deleteBlogEntry(pagename);
pagename = DEFAULT_PAGE;
}
@ -169,7 +168,7 @@ public class Blog {
if(pagename.equals(DEFAULT_PAGE)) {
pagename = String.valueOf(System.currentTimeMillis());
} else {
page = switchboard.blogDB.readBlogEntry(pagename);
page = sb.blogDB.readBlogEntry(pagename);
comments = page.getComments();
date = page.getDate();
}
@ -182,17 +181,17 @@ public class Blog {
subject = StrSubject.getBytes();
}
switchboard.blogDB.writeBlogEntry(switchboard.blogDB.newEntry(pagename, subject, author, ip, date, content, comments, commentMode));
sb.blogDB.writeBlogEntry(sb.blogDB.newEntry(pagename, subject, author, ip, date, content, comments, commentMode));
// create a news message
final HashMap<String, String> map = new HashMap<String, String>();
map.put("page", pagename);
map.put("subject", StrSubject.replace(',', ' '));
map.put("author", StrAuthor.replace(',', ' '));
yacyCore.newsPool.publishMyNews(yacyNewsRecord.newRecord(yacyNewsPool.CATEGORY_BLOG_ADD, map));
sb.wordIndex.newsPool.publishMyNews(yacyNewsRecord.newRecord(sb.wordIndex.seedDB.mySeed(), yacyNewsPool.CATEGORY_BLOG_ADD, map));
}
page = switchboard.blogDB.readBlogEntry(pagename); //maybe "if(page == null)"
page = sb.blogDB.readBlogEntry(pagename); //maybe "if(page == null)"
if (post.containsKey("edit")) {
//edit an entry
@ -253,7 +252,7 @@ public class Blog {
}
else if (post.containsKey("xmlfile")) {
prop.put("mode", "5");
if(switchboard.blogDB.importXML(post.get("xmlfile$file"))) {
if(sb.blogDB.importXML(post.get("xmlfile$file"))) {
prop.put("mode_state", "1");
}
else {
@ -266,10 +265,10 @@ public class Blog {
if(pagename.equals(DEFAULT_PAGE)) {
// XXX: where are "peername" and "address" used in the template?
// XXX: "clientname" is already set to the peername, no need for a new setting
prop.putHTML("peername", yacyCore.seedDB.mySeed().getName(), xml);
prop.putHTML("peername", sb.wordIndex.seedDB.mySeed().getName(), xml);
prop.put("address", address);
//index all entries
putBlogDefault(prop, switchboard, address, start, num, hasRights, xml);
putBlogDefault(prop, sb, address, start, num, hasRights, xml);
}
else {
//only show 1 entry

56
htroot/BlogComments.java
View File

@ -77,10 +77,10 @@ public class BlogComments {
}
public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch<?> env) {
plasmaSwitchboard switchboard = (plasmaSwitchboard) env;
plasmaSwitchboard sb = (plasmaSwitchboard) env;
serverObjects prop = new serverObjects();
blogBoard.BlogEntry page = null;
boolean hasRights = switchboard.verifyAuthentication(header, true);
boolean hasRights = sb.verifyAuthentication(header, true);
if (hasRights) prop.put("mode_admin", "1");
else prop.put("mode_admin", "0");
@ -91,7 +91,7 @@ public class BlogComments {
}
if(!hasRights){
userDB.Entry userentry = switchboard.userDB.proxyAuth((String)header.get("Authorization", "xxxxxx"));
userDB.Entry userentry = sb.userDB.proxyAuth((String)header.get("Authorization", "xxxxxx"));
if(userentry != null && userentry.hasRight(userDB.Entry.BLOG_RIGHT)){
hasRights=true;
}
@ -107,14 +107,14 @@ public class BlogComments {
String StrAuthor = post.get("author", "anonymous");
if (StrAuthor.equals("anonymous")) {
StrAuthor = switchboard.blogDB.guessAuthor(ip);
StrAuthor = sb.blogDB.guessAuthor(ip);
if (StrAuthor == null || StrAuthor.length() == 0) {
if (de.anomic.yacy.yacyCore.seedDB.mySeed() == null) {
if (sb.wordIndex.seedDB.mySeed() == null) {
StrAuthor = "anonymous";
}
else {
StrAuthor = de.anomic.yacy.yacyCore.seedDB.mySeed().get("Name", "anonymous");
StrAuthor = sb.wordIndex.seedDB.mySeed().get("Name", "anonymous");
}
}
}
@ -126,7 +126,7 @@ public class BlogComments {
author = StrAuthor.getBytes();
}
page = switchboard.blogDB.readBlogEntry(pagename); //maybe "if(page == null)"
page = sb.blogDB.readBlogEntry(pagename); //maybe "if(page == null)"
// comments not allowed
if (page.getCommentMode() == 0) {
@ -158,34 +158,34 @@ public class BlogComments {
subject = StrSubject.getBytes();
}
String commentID = String.valueOf(System.currentTimeMillis());
BlogEntry blogEntry = switchboard.blogDB.readBlogEntry(pagename);
BlogEntry blogEntry = sb.blogDB.readBlogEntry(pagename);
blogEntry.addComment(commentID);
switchboard.blogDB.writeBlogEntry(blogEntry);
switchboard.blogCommentDB.write(switchboard.blogCommentDB.newEntry(commentID, subject, author, ip, date, content));
sb.blogDB.writeBlogEntry(blogEntry);
sb.blogCommentDB.write(sb.blogCommentDB.newEntry(commentID, subject, author, ip, date, content));
prop.put("LOCATION","BlogComments.html?page=" + pagename);
messageBoard.entry msgEntry = null;
try {
switchboard.messageDB.write(msgEntry = switchboard.messageDB.newEntry(
sb.messageDB.write(msgEntry = sb.messageDB.newEntry(
"blogComment",
StrAuthor,
yacyCore.seedDB.mySeed().hash,
yacyCore.seedDB.mySeed().getName(), yacyCore.seedDB.mySeed().hash,
sb.wordIndex.seedDB.mySeed().hash,
sb.wordIndex.seedDB.mySeed().getName(), sb.wordIndex.seedDB.mySeed().hash,
"new blog comment: " + new String(blogEntry.getSubject(),"UTF-8"), content));
} catch (UnsupportedEncodingException e1) {
switchboard.messageDB.write(msgEntry = switchboard.messageDB.newEntry(
sb.messageDB.write(msgEntry = sb.messageDB.newEntry(
"blogComment",
StrAuthor,
yacyCore.seedDB.mySeed().hash,
yacyCore.seedDB.mySeed().getName(), yacyCore.seedDB.mySeed().hash,
sb.wordIndex.seedDB.mySeed().hash,
sb.wordIndex.seedDB.mySeed().getName(), sb.wordIndex.seedDB.mySeed().hash,
"new blog comment: " + new String(blogEntry.getSubject()), content));
}
messageForwardingViaEmail(env, msgEntry);
messageForwardingViaEmail(sb, msgEntry);
// finally write notification
File notifierSource = new File(switchboard.getRootPath(), switchboard.getConfig("htRootPath","htroot") + "/env/grafics/message.gif");
File notifierDest = new File(switchboard.getConfigPath("htDocsPath", "DATA/HTDOCS"), "notifier.gif");
File notifierSource = new File(sb.getRootPath(), sb.getConfig("htRootPath","htroot") + "/env/grafics/message.gif");
File notifierDest = new File(sb.getConfigPath("htDocsPath", "DATA/HTDOCS"), "notifier.gif");
try {
serverFileUtils.copy(notifierSource, notifierDest);
} catch (IOException e) {
@ -197,14 +197,14 @@ public class BlogComments {
if(hasRights && post.containsKey("delete") && post.containsKey("page") && post.containsKey("comment")) {
if(page.removeComment((String) post.get("comment"))) {
switchboard.blogCommentDB.delete((String) post.get("comment"));
sb.blogCommentDB.delete((String) post.get("comment"));
}
}
if(hasRights && post.containsKey("allow") && post.containsKey("page") && post.containsKey("comment")) {
blogBoardComments.CommentEntry entry = switchboard.blogCommentDB.read((String) post.get("comment"));
blogBoardComments.CommentEntry entry = sb.blogCommentDB.read((String) post.get("comment"));
entry.allow();
switchboard.blogCommentDB.write(entry);
sb.blogCommentDB.write(entry);
}
if(post.containsKey("preview") && page.getCommentMode() != 0) {
@ -279,7 +279,7 @@ public class BlogComments {
continue;
}
entry = switchboard.blogCommentDB.read(pageid);
entry = sb.blogCommentDB.read(pageid);
if (commentMode == 2 && !hasRights && !entry.isAllowed())
continue;
@ -338,15 +338,15 @@ public class BlogComments {
return prop;
}
private static void messageForwardingViaEmail(serverSwitch<?> env, messageBoard.entry msgEntry) {
private static void messageForwardingViaEmail(plasmaSwitchboard sb, messageBoard.entry msgEntry) {
try {
if (!Boolean.valueOf(env.getConfig("msgForwardingEnabled","false")).booleanValue()) return;
if (!Boolean.valueOf(sb.getConfig("msgForwardingEnabled","false")).booleanValue()) return;
// getting the recipient address
String sendMailTo = env.getConfig("msgForwardingTo","root@localhost").trim();
String sendMailTo = sb.getConfig("msgForwardingTo","root@localhost").trim();
// getting the sendmail configuration
String sendMailStr = env.getConfig("msgForwardingCmd","/usr/bin/sendmail")+" "+sendMailTo;
String sendMailStr = sb.getConfig("msgForwardingCmd","/usr/bin/sendmail")+" "+sendMailTo;
String[] sendMail = sendMailStr.trim().split(" ");
// building the message text
@ -355,7 +355,7 @@ public class BlogComments {
.append(sendMailTo)
.append("\nFrom: ")
.append("yacy@")
.append(yacyCore.seedDB.mySeed().getName())
.append(sb.wordIndex.seedDB.mySeed().getName())
.append("\nSubject: [YaCy] ")
.append(msgEntry.subject().replace('\n', ' '))
.append("\nDate: ")

45
htroot/Bookmarks.java
View File

@ -69,7 +69,6 @@ import de.anomic.server.serverDate;
import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch;
import de.anomic.server.logging.serverLog;
import de.anomic.yacy.yacyCore;
import de.anomic.yacy.yacyNewsPool;
import de.anomic.yacy.yacyNewsRecord;
import de.anomic.yacy.yacyURL;
@ -78,7 +77,7 @@ import de.anomic.yacy.yacyURL;
public class Bookmarks {
private static final serverObjects prop = new serverObjects();
private static plasmaSwitchboard switchboard = null;
private static plasmaSwitchboard sb = null;
private static userDB.Entry user = null;
private static boolean isAdmin = false;
@ -96,9 +95,9 @@ public class Bookmarks {
String username="";
prop.clear();
switchboard = (plasmaSwitchboard) env;
user = switchboard.userDB.getUser(header);
isAdmin=(switchboard.verifyAuthentication(header, true) || user!= null && user.hasRight(userDB.Entry.BOOKMARK_RIGHT));
sb = (plasmaSwitchboard) env;
user = sb.userDB.getUser(header);
isAdmin=(sb.verifyAuthentication(header, true) || user!= null && user.hasRight(userDB.Entry.BOOKMARK_RIGHT));
// set user name
if(user != null) username=user.getUserName();
@ -112,7 +111,7 @@ public class Bookmarks {
*/
// set peer address
final String address = yacyCore.seedDB.mySeed().getPublicAddress();
final String address = sb.wordIndex.seedDB.mySeed().getPublicAddress();
prop.put("address", address);
//defaultvalues
@ -157,7 +156,7 @@ public class Bookmarks {
tagsString="unsorted"; //default tag
}
Set<String> tags=listManager.string2set(bookmarksDB.cleanTagsString(tagsString));
bookmarksDB.Bookmark bookmark = switchboard.bookmarksDB.createBookmark(url, username);
bookmarksDB.Bookmark bookmark = sb.bookmarksDB.createBookmark(url, username);
if(bookmark != null){
bookmark.setProperty(bookmarksDB.Bookmark.BOOKMARK_TITLE, title);
bookmark.setProperty(bookmarksDB.Bookmark.BOOKMARK_DESCRIPTION, description);
@ -176,7 +175,7 @@ public class Bookmarks {
bookmark.setFeed(false);
}
bookmark.setTags(tags, true);
switchboard.bookmarksDB.saveBookmark(bookmark);
sb.bookmarksDB.saveBookmark(bookmark);
//}else{
//ERROR
}
@ -193,10 +192,10 @@ public class Bookmarks {
prop.put("mode_public", "0");
prop.put("mode_feed", "0");
} else {
bookmarksDB.Bookmark bookmark = switchboard.bookmarksDB.getBookmark(urlHash);
bookmarksDB.Bookmark bookmark = sb.bookmarksDB.getBookmark(urlHash);
if (bookmark == null) {
// try to get the bookmark from the LURL database
indexURLReference urlentry = switchboard.wordIndex.getURL(urlHash, null, 0);
indexURLReference urlentry = sb.wordIndex.getURL(urlHash, null, 0);
plasmaParserDocument document = null;
if (urlentry != null) {
indexURLReference.Components comp = urlentry.comp();
@ -244,7 +243,7 @@ public class Bookmarks {
serverLog.logInfo("BOOKMARKS", "I try to import bookmarks from HTML-file");
try {
File file=new File((String)post.get("htmlfile"));
switchboard.bookmarksDB.importFromBookmarks(new yacyURL(file) , post.get("htmlfile$file"), tags, isPublic);
sb.bookmarksDB.importFromBookmarks(new yacyURL(file) , post.get("htmlfile$file"), tags, isPublic);
} catch (MalformedURLException e) {}
serverLog.logInfo("BOOKMARKS", "success!!");
}else if(post.containsKey("xmlfile")){
@ -252,10 +251,10 @@ public class Bookmarks {
if(((String) post.get("public")).equals("public")){
isPublic=true;
}
switchboard.bookmarksDB.importFromXML(post.get("xmlfile$file"), isPublic);
sb.bookmarksDB.importFromXML(post.get("xmlfile$file"), isPublic);
}else if(post.containsKey("delete")){
String urlHash=(String) post.get("delete");
switchboard.bookmarksDB.removeBookmark(urlHash);
sb.bookmarksDB.removeBookmark(urlHash);
}
if(post.containsKey("tag")){
tagName=(String) post.get("tag");
@ -285,15 +284,15 @@ public class Bookmarks {
Iterator<String> tagsIt;
int tagCount;
prop.put("num-bookmarks", switchboard.bookmarksDB.bookmarksSize());
prop.put("num-bookmarks", sb.bookmarksDB.bookmarksSize());
count=0;
if(!tagName.equals("")){
prop.put("selected", "");
it=switchboard.bookmarksDB.getBookmarksIterator(tagName, isAdmin);
it=sb.bookmarksDB.getBookmarksIterator(tagName, isAdmin);
}else{
prop.put("selected", " selected=\"selected\"");
it=switchboard.bookmarksDB.getBookmarksIterator(isAdmin);
it=sb.bookmarksDB.getBookmarksIterator(isAdmin);
}
//skip the first entries (display next page)
@ -305,7 +304,7 @@ public class Bookmarks {
count=0;
while(count<max_count && it.hasNext()){
bookmark=switchboard.bookmarksDB.getBookmark((String)it.next());
bookmark=sb.bookmarksDB.getBookmark((String)it.next());
if(bookmark!=null){
if(bookmark.getFeed() && isAdmin)
prop.put("bookmarks_"+count+"_link", "/FeedReader_p.html?url="+bookmark.getUrl());
@ -359,7 +358,7 @@ public class Bookmarks {
//-----------------------
count = 0;
count = recurseFolders(switchboard.bookmarksDB.getFolderList(isAdmin),"/",0,true,"");
count = recurseFolders(sb.bookmarksDB.getFolderList(isAdmin),"/",0,true,"");
prop.put("folderlist", count);
@ -372,9 +371,9 @@ public class Bookmarks {
Iterator<Tag> it = null;
if (tagName.equals("")) {
it = switchboard.bookmarksDB.getTagIterator(isAdmin, comp, max);
it = sb.bookmarksDB.getTagIterator(isAdmin, comp, max);
} else {
it = switchboard.bookmarksDB.getTagIterator(tagName, isAdmin, comp, max);
it = sb.bookmarksDB.getTagIterator(tagName, isAdmin, comp, max);
}
while(it.hasNext()){
tag=(Tag) it.next();
@ -418,9 +417,9 @@ public class Bookmarks {
if(fn.startsWith(root)){
prop.put("folderlist_"+count+"_folder", "<li>"+fn.replaceFirst(root+"/*","")+"<ul class=\"folder\">");
count++;
Iterator<String> bit=switchboard.bookmarksDB.getBookmarksIterator(fn, isAdmin);
Iterator<String> bit=sb.bookmarksDB.getBookmarksIterator(fn, isAdmin);
while(bit.hasNext()){
bookmark=switchboard.bookmarksDB.getBookmark((String)bit.next());
bookmark=sb.bookmarksDB.getBookmark((String)bit.next());
prop.put("folderlist_"+count+"_folder", "<li><a href=\""+bookmark.getUrl()+"\" title=\""+bookmark.getDescription()+"\">"+ bookmark.getTitle()+"</a></li>");
count++;
}
@ -444,7 +443,7 @@ public class Bookmarks {
map.put("title", title.replace(',', ' '));
map.put("description", description.replace(',', ' '));
map.put("tags", tagsString.replace(',', ' '));
yacyCore.newsPool.publishMyNews(yacyNewsRecord.newRecord(yacyNewsPool.CATEGORY_BOOKMARK_ADD, map));
sb.wordIndex.newsPool.publishMyNews(yacyNewsRecord.newRecord(sb.wordIndex.seedDB.mySeed(), yacyNewsPool.CATEGORY_BOOKMARK_ADD, map));
}
}

115
htroot/ConfigBasic.java
View File

@ -62,7 +62,6 @@ import de.anomic.server.serverDomains;
import de.anomic.server.serverInstantBusyThread;
import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch;
import de.anomic.yacy.yacyCore;
import de.anomic.yacy.yacySeed;
public class ConfigBasic {
@ -76,7 +75,6 @@ public class ConfigBasic {
public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch<?> env) {
// return variable that accumulates replacements
ConfigBasic config = new ConfigBasic();
plasmaSwitchboard sb = (plasmaSwitchboard) env;
serverObjects prop = new serverObjects();
String langPath = env.getConfigPath("locale.work", "DATA/LOCALE/locales").getAbsolutePath();
@ -89,45 +87,14 @@ public class ConfigBasic {
return prop;
}
// reconfigure port forwarding
if ((post != null)) config.reinitPortForwarding(post, env);
// starting a peer ping
//boolean doPeerPing = false;
if ((yacyCore.seedDB.mySeed().isVirgin()) || (yacyCore.seedDB.mySeed().isJunior())) {
if ((sb.wordIndex.seedDB.mySeed().isVirgin()) || (sb.wordIndex.seedDB.mySeed().isJunior())) {
serverInstantBusyThread.oneTimeJob(sb.yc, "peerPing", null, 0);
//doPeerPing = true;
}
// scan for Upnp routers
/*
long begin = System.currentTimeMillis();
boolean upnpRouterFound = false;
if (yacyCore.seedDB.mySeed.isVirgin() || yacyCore.seedDB.mySeed.isJunior()) {
upnpRouterFound = config.findUPnPRouter(3000);
}
long end = System.currentTimeMillis();
// if the upnp router scan has taken less than 3 sec, we need to wait
// a little bit for success of peer ping
if ((doPeerPing) && ((end - begin) < 3000 )) {
try {Thread.sleep(3000-(end - begin));} catch (InterruptedException e) {/* *///}
/*}
// if a UPnP router is available
String currentForwarder = env.getConfig("portForwarding.Type", "none");
boolean forwardingEnabled = env.getConfigBool("portForwarding.Enabled",false);
boolean otherForwarderEnabled = serverCore.portForwardingEnabled && serverCore.portForwarding != null && !currentForwarder.equalsIgnoreCase("upnp");
if (otherForwarderEnabled) {
prop.put("upnp",0);
} else {
prop.put("upnp", upnpRouterFound ? 1 : 0);
}
// if UPnp is already enabled
prop.put("upnp_enabled", currentForwarder.equalsIgnoreCase("upnp") && forwardingEnabled ? 1 : 0);
*/
// language settings
if ((post != null) && (!(post.get("language", "default").equals(lang)))) {
translator.changeLang(env, langPath, post.get("language", "default") + ".lng");
@ -158,7 +125,7 @@ public class ConfigBasic {
}
// check if peer name already exists
yacySeed oldSeed = yacyCore.seedDB.lookupByName(peerName);
yacySeed oldSeed = sb.wordIndex.seedDB.lookupByName(peerName);
if ((oldSeed == null) && (!(env.getConfig("peerName", "").equals(peerName)))) {
// the name is new
boolean nameOK = Pattern.compile("[A-Za-z0-9\\-_]{3,80}").matcher(peerName).matches();
@ -201,7 +168,7 @@ public class ConfigBasic {
// check if values are proper
boolean properPW = (env.getConfig("adminAccount", "").length() == 0) && (env.getConfig(httpd.ADMIN_ACCOUNT_B64MD5, "").length() > 0);
boolean properName = (env.getConfig("peerName","").length() >= 3) && (!(yacySeed.isDefaultPeerName(env.getConfig("peerName",""))));
boolean properPort = (yacyCore.seedDB.mySeed().isSenior()) || (yacyCore.seedDB.mySeed().isPrincipal());
boolean properPort = (sb.wordIndex.seedDB.mySeed().isSenior()) || (sb.wordIndex.seedDB.mySeed().isPrincipal());
if ((properPW) && (env.getConfig("defaultFiles", "").startsWith("ConfigBasic.html,"))) {
env.setConfig("defaultFiles", env.getConfig("defaultFiles", "").substring(17));
@ -240,80 +207,4 @@ public class ConfigBasic {
}
return prop;
}
/*
private boolean findUPnPRouter(int timeout) {
// determine if the upnp port forwarding class is available and load it dynamically
Object[] UpnpForwarder = this.getUpnpForwarderClasses();
serverPortForwarding upnp = (serverPortForwarding) UpnpForwarder[0];
Method scanForRouter = (Method) UpnpForwarder[1];
if ((upnp == null) || (scanForRouter == null)) return false;
// trying to find a upnp router
try {
Object result = scanForRouter.invoke(upnp, new Object[]{new Integer(timeout)});
if ((result != null)&&(result instanceof Boolean)) {
return ((Boolean)result).booleanValue();
}
} catch (Exception e) { // ignore this error
} catch (Error e) {} // ignore this error
return false;
}
private Object[] getUpnpForwarderClasses() {
serverPortForwarding upnp = null;
Method scanForRouter = null;
try {
// trying to load the upnp forwarder class
Class forwarderClass = Class.forName("de.anomic.server.portForwarding.upnp.serverPortForwardingUpnp");
// create a new instance
upnp = (serverPortForwarding) forwarderClass.newInstance();
// trying to get the proper method for router scanning
scanForRouter = upnp.getClass().getMethod("routerAvailable", new Class[] {int.class});
} catch (Exception e) { // ignore this error
} catch (Error e) {} // ignore this error
return new Object[]{upnp,scanForRouter};
}
*/
private void reinitPortForwarding(serverObjects post, serverSwitch<?> env) {
if ((post != null)) {
try {
boolean reinitPortForwarding = false;
if (post.containsKey("enableUpnp")) {
// upnp should be enabled
env.setConfig("portForwarding.Enabled","true");
env.setConfig("portForwarding.Type", "upnp");
reinitPortForwarding = true;
} else {
String currentForwarder = env.getConfig("portForwarding.Type", "none");
boolean otherForwarderEnabled = serverCore.portForwardingEnabled && serverCore.portForwarding != null && !currentForwarder.equalsIgnoreCase("upnp");
// if no other forwarder is running we deactivate forwarding
// and try to stop an eventually running upnp forwarder
if (!otherForwarderEnabled) {
env.setConfig("portForwarding.Enabled","false");
env.setConfig("portForwarding.Type", "none");
reinitPortForwarding = true;
}
}
if (reinitPortForwarding) {
if ((serverCore.portForwardingEnabled) && (serverCore.portForwarding != null)) {
// trying to shutdown the current port forwarding channel
serverCore.portForwarding.disconnect();
}
// trying to reinitialize the port forwarding
serverCore httpd = (serverCore) env.getThread("10_httpd");
httpd.initPortForwarding();
}
} catch (Exception e) { /* */ }
}
}
}

15
htroot/ConfigNetwork_p.java
View File

@ -31,7 +31,6 @@ import de.anomic.server.serverBusyThread;
import de.anomic.server.serverCodings;
import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch;
import de.anomic.yacy.yacyCore;
public class ConfigNetwork_p {
@ -103,10 +102,10 @@ public class ConfigNetwork_p {
if (indexReceive) {
sb.setConfig("allowReceiveIndex", "true");
yacyCore.seedDB.mySeed().setFlagAcceptRemoteIndex(true);
sb.wordIndex.seedDB.mySeed().setFlagAcceptRemoteIndex(true);
} else {
sb.setConfig("allowReceiveIndex", "false");
yacyCore.seedDB.mySeed().setFlagAcceptRemoteIndex(false);
sb.wordIndex.seedDB.mySeed().setFlagAcceptRemoteIndex(false);
}
if (post.get("indexReceiveBlockBlacklist", "").equals("on")) {
@ -116,7 +115,7 @@ public class ConfigNetwork_p {
}
if (post.containsKey("peertags")) {
yacyCore.seedDB.mySeed().setPeerTags(serverCodings.string2set(normalizedList((String) post.get("peertags")), ","));
sb.wordIndex.seedDB.mySeed().setPeerTags(serverCodings.string2set(normalizedList((String) post.get("peertags")), ","));
}
sb.setConfig("cluster.mode", post.get("cluster.mode", "publicpeer"));
@ -136,7 +135,7 @@ public class ConfigNetwork_p {
sb.setConfig("cluster.peers.yacydomain", checkYaCyDomainList(post.get("cluster.peers.yacydomain", "")));
// update the cluster hash set
sb.clusterhashes = yacyCore.seedDB.clusterHashes(sb.getConfig("cluster.peers.yacydomain", ""));
sb.clusterhashes = sb.wordIndex.seedDB.clusterHashes(sb.getConfig("cluster.peers.yacydomain", ""));
}
@ -162,11 +161,11 @@ public class ConfigNetwork_p {
prop.put("indexReceiveChecked", (indexReceive) ? "1" : "0");
prop.put("indexReceiveBlockBlacklistChecked.on", (sb.getConfig("indexReceiveBlockBlacklist", "true").equals("true")) ? "1" : "0");
prop.put("indexReceiveBlockBlacklistChecked.off", (sb.getConfig("indexReceiveBlockBlacklist", "true").equals("true")) ? "0" : "1");
prop.putHTML("peertags", serverCodings.set2string(yacyCore.seedDB.mySeed().getPeerTags(), ",", false));
prop.putHTML("peertags", serverCodings.set2string(sb.wordIndex.seedDB.mySeed().getPeerTags(), ",", false));
// set seed information directly
yacyCore.seedDB.mySeed().setFlagAcceptRemoteCrawl(sb.getConfigBool("crawlResponse", false));
yacyCore.seedDB.mySeed().setFlagAcceptRemoteIndex(indexReceive);
sb.wordIndex.seedDB.mySeed().setFlagAcceptRemoteCrawl(sb.getConfigBool("crawlResponse", false));
sb.wordIndex.seedDB.mySeed().setFlagAcceptRemoteIndex(indexReceive);
// set p2p/robinson mode flags and values
prop.put("p2p.checked", (indexDistribute || indexReceive) ? "1" : "0");

6
htroot/ConfigProfile_p.java
View File

@ -55,16 +55,16 @@ import java.io.IOException;
import java.util.Properties;
import de.anomic.http.httpHeader;
import de.anomic.plasma.plasmaSwitchboard;
import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch;
import de.anomic.yacy.yacyCore;
import de.anomic.yacy.yacyNewsPool;
import de.anomic.yacy.yacyNewsRecord;
public class ConfigProfile_p {
public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch<?> env) {
//listManager.switchboard = (plasmaSwitchboard) env;
plasmaSwitchboard sb = (plasmaSwitchboard) env;
final serverObjects prop = new serverObjects();
final Properties profile = new Properties();
FileInputStream fileIn = null;
@ -113,7 +113,7 @@ public class ConfigProfile_p {
// generate a news message
Properties news = profile;
news.remove("comment");
yacyCore.newsPool.publishMyNews(yacyNewsRecord.newRecord(yacyNewsPool.CATEGORY_PROFILE_UPDATE, news));
sb.wordIndex.newsPool.publishMyNews(yacyNewsRecord.newRecord(sb.wordIndex.seedDB.mySeed(), yacyNewsPool.CATEGORY_PROFILE_UPDATE, news));
//yacyCore.newsPool.publishMyNews(new yacyNewsRecord(yacyNewsRecord.CATEGORY_PROFILE_UPDATE, profile));
} catch(IOException e) {
} finally {

4
htroot/ConfigRobotsTxt_p.java
View File

@ -54,15 +54,15 @@ import de.anomic.plasma.plasmaSwitchboard;
import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch;
import de.anomic.server.servletProperties;
import de.anomic.yacy.yacyCore;
public class ConfigRobotsTxt_p {
public static servletProperties respond(httpHeader header, serverObjects post, serverSwitch<?> env) {
plasmaSwitchboard sb = (plasmaSwitchboard) env;
final servletProperties prop = new servletProperties();
httpdRobotsTxtConfig rbc = ((plasmaSwitchboard)env).robotstxtConfig;
prop.put("clientname", yacyCore.seedDB.mySeed().getPublicAddress());
prop.put("clientname", sb.wordIndex.seedDB.mySeed().getPublicAddress());
if (post != null) {
if (post.containsKey("save")) {

15
htroot/Connections_p.java
View File

@ -65,22 +65,21 @@ import de.anomic.server.serverSwitch;
import de.anomic.server.serverThread;
import de.anomic.server.serverCore.Session;
import de.anomic.urlRedirector.urlRedirectord;
import de.anomic.yacy.yacyCore;
import de.anomic.yacy.yacySeed;
public final class Connections_p {
public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch<?> sb) {
public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch<?> env) {
// return variable that accumulates replacements
plasmaSwitchboard switchboard = (plasmaSwitchboard) sb;
plasmaSwitchboard sb = (plasmaSwitchboard) env;
serverObjects prop = new serverObjects();
// get the virtualHost string
String virtualHost = switchboard.getConfig("fileHost","localhost");
String virtualHost = sb.getConfig("fileHost","localhost");
// get the serverCore thread
serverThread httpd = switchboard.getThread("10_httpd");
serverThread httpd = sb.getThread("10_httpd");
/* waiting for all threads to finish */
int threadCount = serverCore.sessionThreadGroup.activeCount();
@ -177,14 +176,14 @@ public final class Connections_p {
commandLine = urlRedir.getURL();
}
if ((dest != null) && (dest.equals(virtualHost))) dest = yacyCore.seedDB.mySeed().getName() + ".yacy";
if ((dest != null) && (dest.equals(virtualHost))) dest = sb.wordIndex.seedDB.mySeed().getName() + ".yacy";
// determining if the source is a yacy host
yacySeed seed = null;
if (doNameLookup) {
seed = yacyCore.seedDB.lookupByIP(userAddress,true,false,false);
seed = sb.wordIndex.seedDB.lookupByIP(userAddress,true,false,false);
if (seed != null) {
if ((seed.hash.equals(yacyCore.seedDB.mySeed().hash)) &&
if ((seed.hash.equals(sb.wordIndex.seedDB.mySeed().hash)) &&
(!seed.get(yacySeed.PORT,"").equals(Integer.toString(userPort)))) {
seed = null;
}

5
htroot/CrawlResults.java
View File

@ -36,7 +36,6 @@ import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch;
import de.anomic.server.logging.serverLog;
import de.anomic.tools.nxTools;
import de.anomic.yacy.yacyCore;
import de.anomic.yacy.yacySeed;
import de.anomic.yacy.yacyURL;
@ -170,8 +169,8 @@ public class CrawlResults {
urle = sb.wordIndex.getURL(urlHash, null, 0);
indexURLReference.Components comp = urle.comp();
// serverLog.logFinest("PLASMA", "plasmaCrawlLURL/genTableProps urle=" + urle.toString());
initiatorSeed = yacyCore.seedDB.getConnected(initiatorHash);
executorSeed = yacyCore.seedDB.getConnected(executorHash);
initiatorSeed = sb.wordIndex.seedDB.getConnected(initiatorHash);
executorSeed = sb.wordIndex.seedDB.getConnected(executorHash);
urlstr = comp.url().toNormalform(false, true);
urltxt = nxTools.shortenURLString(urlstr, 72); // shorten the string text like a URL

16
htroot/CrawlStartSimple_p.java
View File

@ -34,12 +34,12 @@ import de.anomic.yacy.yacyCore;
import de.anomic.yacy.yacyNewsPool;
import de.anomic.yacy.yacyNewsRecord;
import de.anomic.yacy.yacySeed;
import de.anomic.yacy.yacyURL;
public class CrawlStartSimple_p {
public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch<?> env) {
// return variable that accumulates replacements
plasmaSwitchboard sb = (plasmaSwitchboard) env;
serverObjects prop = new serverObjects();
// define visible variables
@ -100,7 +100,7 @@ public class CrawlStartSimple_p {
boolean dark = true;
// create other peer crawl table using YaCyNews
Iterator<yacyNewsRecord> recordIterator = yacyCore.newsPool.recordIterator(yacyNewsPool.INCOMING_DB, true);
Iterator<yacyNewsRecord> recordIterator = sb.wordIndex.newsPool.recordIterator(yacyNewsPool.INCOMING_DB, true);
int showedCrawl = 0;
yacyNewsRecord record;
yacySeed peer;
@ -109,7 +109,7 @@ public class CrawlStartSimple_p {
record = recordIterator.next();
if (record == null) continue;
if (record.category().equals(yacyNewsPool.CATEGORY_CRAWL_START)) {
peer = yacyCore.seedDB.get(record.originator());
peer = sb.wordIndex.seedDB.get(record.originator());
if (peer == null) peername = record.originator(); else peername = peer.getName();
prop.put("otherCrawlStartInProgress_" + showedCrawl + "_dark", dark ? "1" : "0");
prop.put("otherCrawlStartInProgress_" + showedCrawl + "_cre", record.created().toString());
@ -125,13 +125,13 @@ public class CrawlStartSimple_p {
prop.put("otherCrawlStartInProgress", showedCrawl);
// finished remote crawls
recordIterator = yacyCore.newsPool.recordIterator(yacyNewsPool.PROCESSED_DB, true);
recordIterator = sb.wordIndex.newsPool.recordIterator(yacyNewsPool.PROCESSED_DB, true);
showedCrawl = 0;
while (recordIterator.hasNext()) {
record = (yacyNewsRecord) recordIterator.next();
if (record == null) continue;
if (record.category().equals(yacyNewsPool.CATEGORY_CRAWL_START)) {
peer = yacyCore.seedDB.get(record.originator());
peer = sb.wordIndex.seedDB.get(record.originator());
if (peer == null) peername = record.originator(); else peername = peer.getName();
prop.put("otherCrawlStartFinished_" + showedCrawl + "_dark", dark ? "1" : "0");
prop.put("otherCrawlStartFinished_" + showedCrawl + "_cre", record.created().toString());
@ -148,11 +148,11 @@ public class CrawlStartSimple_p {
// remote crawl peers
if ((yacyCore.seedDB == null) || (yacyCore.seedDB.mySeed().isVirgin()) || (yacyCore.seedDB.mySeed().isJunior())) {
if ((sb.wordIndex.seedDB == null) || (sb.wordIndex.seedDB.mySeed().isVirgin()) || (sb.wordIndex.seedDB.mySeed().isJunior())) {
prop.put("remoteCrawlPeers", "0");
} else {
Iterator<yacySeed> crawlavail = yacyCore.dhtAgent.getAcceptRemoteCrawlSeeds(yacyURL.dummyHash, true);
Iterator<yacySeed> crawlpendi = yacyCore.dhtAgent.getAcceptRemoteCrawlSeeds(yacyURL.dummyHash, false);
Iterator<yacySeed> crawlavail = yacyCore.dhtAgent.getAcceptRemoteCrawlSeeds(null, true);
Iterator<yacySeed> crawlpendi = yacyCore.dhtAgent.getAcceptRemoteCrawlSeeds(null, false);
if ((!(crawlavail.hasNext())) && (!(crawlpendi.hasNext()))) {
prop.put("remoteCrawlPeers", "0"); //no peers availible
} else {

8
htroot/IndexControlRWIs_p.java
View File

@ -50,7 +50,6 @@ import de.anomic.plasma.plasmaSwitchboard;
import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch;
import de.anomic.yacy.yacyClient;
import de.anomic.yacy.yacyCore;
import de.anomic.yacy.yacySeed;
import de.anomic.yacy.yacyURL;
@ -174,14 +173,14 @@ public class IndexControlRWIs_p {
if (host.length() != 0) {
if (host.length() == 12) {
// the host string is a peer hash
seed = yacyCore.seedDB.getConnected(host);
seed = sb.wordIndex.seedDB.getConnected(host);
} else {
// the host string can be a host name
seed = yacyCore.seedDB.lookupByName(host);
seed = sb.wordIndex.seedDB.lookupByName(host);
}
} else {
host = post.get("hostHash", ""); // if input field is empty, get from select box
seed = yacyCore.seedDB.getConnected(host);
seed = sb.wordIndex.seedDB.getConnected(host);
}
// prepare index
@ -210,6 +209,7 @@ public class IndexControlRWIs_p {
String gzipBody = sb.getConfig("indexControl.gzipBody","false");
int timeout = (int) sb.getConfigLong("indexControl.timeout",60000);
HashMap<String, Object> resultObj = yacyClient.transferIndex(
sb.wordIndex.seedDB,
seed,
new indexContainer[]{index},
knownURLs,

39
htroot/IndexCreateIndexingQueue_p.java
View File

@ -54,7 +54,6 @@ import de.anomic.plasma.plasmaSwitchboardQueue;
import de.anomic.server.serverMemory;
import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch;
import de.anomic.yacy.yacyCore;
import de.anomic.yacy.yacySeed;
import de.anomic.yacy.yacyURL;
@ -62,7 +61,7 @@ public class IndexCreateIndexingQueue_p {
public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch<?> env) {
// return variable that accumulates replacements
plasmaSwitchboard switchboard = (plasmaSwitchboard) env;
plasmaSwitchboard sb = (plasmaSwitchboard) env;
serverObjects prop = new serverObjects();
prop.put("rejected", "0");
int showRejectedCount = 100;
@ -76,27 +75,27 @@ public class IndexCreateIndexingQueue_p {
}
if (post.containsKey("clearRejected")) {
switchboard.crawlQueues.errorURL.clearStack();
sb.crawlQueues.errorURL.clearStack();
}
if (post.containsKey("moreRejected")) {
showRejectedCount = post.getInt("showRejected", 10);
}
if (post.containsKey("clearIndexingQueue")) {
try {
synchronized (switchboard.sbQueue) {
synchronized (sb.sbQueue) {
plasmaSwitchboardQueue.QueueEntry entry = null;
while ((entry = switchboard.sbQueue.pop()) != null) {
while ((entry = sb.sbQueue.pop()) != null) {
if ((entry != null) && (entry.profile() != null) && (!(entry.profile().storeHTCache()))) {
plasmaHTCache.deleteURLfromCache(entry.url());
}
}
switchboard.sbQueue.clear(); // reset file to clean up content completely
sb.sbQueue.clear(); // reset file to clean up content completely
}
} catch (Exception e) {}
} else if (post.containsKey("deleteEntry")) {
String urlHash = (String) post.get("deleteEntry");
try {
switchboard.sbQueue.remove(urlHash);
sb.sbQueue.remove(urlHash);
} catch (Exception e) {}
prop.put("LOCATION","");
return prop;
@ -106,7 +105,7 @@ public class IndexCreateIndexingQueue_p {
yacySeed initiator;
boolean dark;
if ((switchboard.sbQueue.size() == 0) && (switchboard.sbQueue.getActiveQueueSize() == 0)) {
if ((sb.sbQueue.size() == 0) && (sb.sbQueue.getActiveQueueSize() == 0)) {
prop.put("indexing-queue", "0"); //is empty
} else {
prop.put("indexing-queue", "1"); // there are entries in the queue or in process
@ -118,12 +117,12 @@ public class IndexCreateIndexingQueue_p {
// getting all entries that are currently in process
ArrayList<plasmaSwitchboardQueue.QueueEntry> entryList = new ArrayList<plasmaSwitchboardQueue.QueueEntry>();
entryList.addAll(switchboard.sbQueue.getActiveQueueEntries());
entryList.addAll(sb.sbQueue.getActiveQueueEntries());
int inProcessCount = entryList.size();
// getting all enqueued entries
if ((switchboard.sbQueue.size() > 0)) {
Iterator<plasmaSwitchboardQueue.QueueEntry> i = switchboard.sbQueue.entryIterator(false);
if ((sb.sbQueue.size() > 0)) {
Iterator<plasmaSwitchboardQueue.QueueEntry> i = sb.sbQueue.entryIterator(false);
while (i.hasNext()) entryList.add(i.next());
}
@ -136,7 +135,7 @@ public class IndexCreateIndexingQueue_p {
if ((pcentry != null)&&(pcentry.url() != null)) {
long entrySize = pcentry.size();
totalSize += entrySize;
initiator = yacyCore.seedDB.getConnected(pcentry.initiator());
initiator = sb.wordIndex.seedDB.getConnected(pcentry.initiator());
prop.put("indexing-queue_list_"+entryCount+"_dark", inProcess ? "2" : (dark ? "1" : "0"));
prop.putHTML("indexing-queue_list_"+entryCount+"_initiator", ((initiator == null) ? "proxy" : initiator.getName()));
prop.put("indexing-queue_list_"+entryCount+"_depth", pcentry.depth());
@ -158,11 +157,11 @@ public class IndexCreateIndexingQueue_p {
}
// failure cases
if (switchboard.crawlQueues.errorURL.stackSize() != 0) {
if (showRejectedCount > switchboard.crawlQueues.errorURL.stackSize()) showRejectedCount = switchboard.crawlQueues.errorURL.stackSize();
if (sb.crawlQueues.errorURL.stackSize() != 0) {
if (showRejectedCount > sb.crawlQueues.errorURL.stackSize()) showRejectedCount = sb.crawlQueues.errorURL.stackSize();
prop.put("rejected", "1");
prop.putNum("rejected_num", switchboard.crawlQueues.errorURL.stackSize());
if (showRejectedCount != switchboard.crawlQueues.errorURL.stackSize()) {
prop.putNum("rejected_num", sb.crawlQueues.errorURL.stackSize());
if (showRejectedCount != sb.crawlQueues.errorURL.stackSize()) {
prop.put("rejected_only-latest", "1");
prop.putNum("rejected_only-latest_num", showRejectedCount);
prop.put("rejected_only-latest_newnum", ((int) (showRejectedCount * 1.5)));
@ -175,16 +174,16 @@ public class IndexCreateIndexingQueue_p {
plasmaCrawlZURL.Entry entry;
yacySeed initiatorSeed, executorSeed;
int j=0;
for (int i = switchboard.crawlQueues.errorURL.stackSize() - 1; i >= (switchboard.crawlQueues.errorURL.stackSize() - showRejectedCount); i--) {
entry = switchboard.crawlQueues.errorURL.top(i);
for (int i = sb.crawlQueues.errorURL.stackSize() - 1; i >= (sb.crawlQueues.errorURL.stackSize() - showRejectedCount); i--) {
entry = sb.crawlQueues.errorURL.top(i);
if (entry == null) continue;
url = entry.url();
if (url == null) continue;
initiatorHash = entry.initiator();
executorHash = entry.executor();
initiatorSeed = yacyCore.seedDB.getConnected(initiatorHash);
executorSeed = yacyCore.seedDB.getConnected(executorHash);
initiatorSeed = sb.wordIndex.seedDB.getConnected(initiatorHash);
executorSeed = sb.wordIndex.seedDB.getConnected(executorHash);
prop.putHTML("rejected_list_"+j+"_initiator", ((initiatorSeed == null) ? "proxy" : initiatorSeed.getName()));
prop.putHTML("rejected_list_"+j+"_executor", ((executorSeed == null) ? "proxy" : executorSeed.getName()));
prop.putHTML("rejected_list_"+j+"_url", url.toNormalform(false, true));

9
htroot/IndexCreateLoaderQueue_p.java
View File

@ -48,29 +48,28 @@ import de.anomic.plasma.plasmaCrawlEntry;
import de.anomic.plasma.plasmaSwitchboard;
import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch;
import de.anomic.yacy.yacyCore;
import de.anomic.yacy.yacySeed;
public class IndexCreateLoaderQueue_p {
public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch<?> env) {
// return variable that accumulates replacements
plasmaSwitchboard switchboard = (plasmaSwitchboard) env;
plasmaSwitchboard sb = (plasmaSwitchboard) env;
serverObjects prop = new serverObjects();
if (switchboard.crawlQueues.size() == 0) {
if (sb.crawlQueues.size() == 0) {
prop.put("loader-set", "0");
} else {
prop.put("loader-set", "1");
boolean dark = true;
plasmaCrawlEntry[] w = switchboard.crawlQueues.activeWorkerEntries();
plasmaCrawlEntry[] w = sb.crawlQueues.activeWorkerEntries();
yacySeed initiator;
int count = 0;
for (int i = 0; i < w.length; i++) {
if (w[i] == null) continue;
initiator = yacyCore.seedDB.getConnected(w[i].initiator());
initiator = sb.wordIndex.seedDB.getConnected(w[i].initiator());
prop.put("loader-set_list_"+count+"_dark", dark ? "1" : "0");
prop.putHTML("loader-set_list_"+count+"_initiator", ((initiator == null) ? "proxy" : initiator.getName()));
prop.put("loader-set_list_"+count+"_depth", w[i].depth());

19
htroot/IndexCreateWWWGlobalQueue_p.java
View File

@ -54,7 +54,6 @@ import de.anomic.plasma.plasmaCrawlProfile;
import de.anomic.plasma.plasmaSwitchboard;
import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch;
import de.anomic.yacy.yacyCore;
import de.anomic.yacy.yacySeed;
public class IndexCreateWWWGlobalQueue_p {
@ -67,7 +66,7 @@ public class IndexCreateWWWGlobalQueue_p {
public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch<?> env) {
// return variable that accumulates replacements
plasmaSwitchboard switchboard = (plasmaSwitchboard) env;
plasmaSwitchboard sb = (plasmaSwitchboard) env;
serverObjects prop = new serverObjects();
int showLimit = 100;
@ -79,9 +78,9 @@ public class IndexCreateWWWGlobalQueue_p {
}
if (post.containsKey("clearcrawlqueue")) {
int c = switchboard.crawlQueues.noticeURL.stackSize(plasmaCrawlNURL.STACK_TYPE_LIMIT);
switchboard.crawlQueues.noticeURL.clear(plasmaCrawlNURL.STACK_TYPE_LIMIT);
try { switchboard.cleanProfiles(); } catch (InterruptedException e) { /* Ignore this */}
int c = sb.crawlQueues.noticeURL.stackSize(plasmaCrawlNURL.STACK_TYPE_LIMIT);
sb.crawlQueues.noticeURL.clear(plasmaCrawlNURL.STACK_TYPE_LIMIT);
try { sb.cleanProfiles(); } catch (InterruptedException e) { /* Ignore this */}
/*
int c = 0;
while (switchboard.urlPool.noticeURL.stackSize(plasmaCrawlNURL.STACK_TYPE_LIMIT) > 0) {
@ -93,18 +92,18 @@ public class IndexCreateWWWGlobalQueue_p {
prop.putNum("info_numEntries", c);
} else if (post.containsKey("deleteEntry")) {
String urlHash = (String) post.get("deleteEntry");
switchboard.crawlQueues.noticeURL.removeByURLHash(urlHash);
sb.crawlQueues.noticeURL.removeByURLHash(urlHash);
prop.put("LOCATION","");
return prop;
}
}
int stackSize = switchboard.crawlQueues.noticeURL.stackSize(plasmaCrawlNURL.STACK_TYPE_LIMIT);
int stackSize = sb.crawlQueues.noticeURL.stackSize(plasmaCrawlNURL.STACK_TYPE_LIMIT);
if (stackSize == 0) {
prop.put("crawler-queue", "0");
} else {
prop.put("crawler-queue", "1");
plasmaCrawlEntry[] crawlerList = switchboard.crawlQueues.noticeURL.top(plasmaCrawlNURL.STACK_TYPE_LIMIT, showLimit);
plasmaCrawlEntry[] crawlerList = sb.crawlQueues.noticeURL.top(plasmaCrawlNURL.STACK_TYPE_LIMIT, showLimit);
plasmaCrawlEntry urle;
boolean dark = true;
@ -115,9 +114,9 @@ public class IndexCreateWWWGlobalQueue_p {
for (i = 0; (i < crawlerList.length) && (showNum < showLimit); i++) {
urle = crawlerList[i];
if ((urle != null)&&(urle.url()!=null)) {
initiator = yacyCore.seedDB.getConnected(urle.initiator());
initiator = sb.wordIndex.seedDB.getConnected(urle.initiator());
profileHandle = urle.profileHandle();
profileEntry = (profileHandle == null) ? null : switchboard.profilesActiveCrawls.getEntry(profileHandle);
profileEntry = (profileHandle == null) ? null : sb.profilesActiveCrawls.getEntry(profileHandle);
prop.put("crawler-queue_list_"+showNum+"_dark", dark ? "1" : "0");
prop.putHTML("crawler-queue_list_"+showNum+"_initiator", ((initiator == null) ? "proxy" : initiator.getName()) );
prop.put("crawler-queue_list_"+showNum+"_profile", ((profileEntry == null) ? "unknown" : profileEntry.name()));

3
htroot/IndexCreateWWWLocalQueue_p.java
View File

@ -58,7 +58,6 @@ import de.anomic.plasma.plasmaCrawlProfile;
import de.anomic.plasma.plasmaSwitchboard;
import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch;
import de.anomic.yacy.yacyCore;
import de.anomic.yacy.yacySeed;
public class IndexCreateWWWLocalQueue_p {
@ -182,7 +181,7 @@ public class IndexCreateWWWLocalQueue_p {
for (i = 0; (i < crawlerList.length) && (showNum < showLimit); i++) {
urle = crawlerList[i];
if ((urle != null)&&(urle.url()!=null)) {
initiator = yacyCore.seedDB.getConnected(urle.initiator());
initiator = sb.wordIndex.seedDB.getConnected(urle.initiator());
profileHandle = urle.profileHandle();
profileEntry = (profileHandle == null) ? null : sb.profilesActiveCrawls.getEntry(profileHandle);
prop.put("crawler-queue_list_"+showNum+"_dark", dark ? "1" : "0");

3
htroot/IndexCreateWWWRemoteQueue_p.java
View File

@ -55,7 +55,6 @@ import de.anomic.plasma.plasmaSwitchboard;
import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch;
import de.anomic.server.servletProperties;
import de.anomic.yacy.yacyCore;
import de.anomic.yacy.yacySeed;
public class IndexCreateWWWRemoteQueue_p {
@ -115,7 +114,7 @@ public class IndexCreateWWWRemoteQueue_p {
for (i = 0; (i < crawlerList.length) && (showNum < showLimit); i++) {
urle = crawlerList[i];
if (urle != null && urle.url() != null) {
initiator = yacyCore.seedDB.getConnected(urle.initiator());
initiator = sb.wordIndex.seedDB.getConnected(urle.initiator());
profileHandle = urle.profileHandle();
profileEntry = (profileHandle == null) ? null : sb.profilesActiveCrawls.getEntry(profileHandle);
prop.put("crawler-queue_list_" + showNum + "_dark", dark ? "1" : "0");

42
htroot/IndexTransfer_p.java
View File

@ -65,12 +65,12 @@ public final class IndexTransfer_p {
public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch<?> env) {
// return variable that accumulates replacements
plasmaSwitchboard switchboard = (plasmaSwitchboard) env;
plasmaSwitchboard sb = (plasmaSwitchboard) env;
serverObjects prop = new serverObjects();
if (post != null) {
if (post.containsKey("startIndexTransfer")) {
yacySeed seed = yacyCore.seedDB.getConnected(post.get("hostHash", ""));
yacySeed seed = sb.wordIndex.seedDB.getConnected(post.get("hostHash", ""));
if (seed == null) {
prop.put("running_status","Disconnected peer");
} else {
@ -78,30 +78,30 @@ public final class IndexTransfer_p {
if(prop.containsKey("overwriteIP") && ! ((String)prop.get("overwriteIP")).equals("")){
seed.setIP((String) prop.get("overwriteIP"));
}
switchboard.startTransferWholeIndex(seed,deleteIndex);
sb.startTransferWholeIndex(seed,deleteIndex);
prop.put("LOCATION","");
return prop;
}
} else if (post.containsKey("stopIndexTransfer")) {
switchboard.stopTransferWholeIndex(true);
sb.stopTransferWholeIndex(true);
prop.put("LOCATION","");
return prop;
} else if (post.containsKey("newIndexTransfer")) {
switchboard.abortTransferWholeIndex(true);
sb.abortTransferWholeIndex(true);
prop.put("LOCATION","");
return prop;
}
}
// insert constants
prop.putNum("wcount", switchboard.wordIndex.size());
prop.putNum("ucount", switchboard.wordIndex.countURL());
prop.put("running",(switchboard.transferIdxThread==null) ? "0" : "1");
if (switchboard.transferIdxThread != null) {
String[] status = switchboard.transferIdxThread.getStatus();
String[] range = switchboard.transferIdxThread.getRange();
int[] chunk = switchboard.transferIdxThread.getIndexCount();
prop.putNum("wcount", sb.wordIndex.size());
prop.putNum("ucount", sb.wordIndex.countURL());
prop.put("running",(sb.transferIdxThread==null) ? "0" : "1");
if (sb.transferIdxThread != null) {
String[] status = sb.transferIdxThread.getStatus();
String[] range = sb.transferIdxThread.getRange();
int[] chunk = sb.transferIdxThread.getIndexCount();
prop.put("running_selection.status",status[0]);
prop.put("running_selection.twrange", range[0]);
@ -112,15 +112,15 @@ public final class IndexTransfer_p {
prop.put("running_transfer.twchunk", chunk[1]);
prop.putNum("running_twEntityCount", switchboard.transferIdxThread.getTransferedContainerCount());
prop.putNum("running_twEntryCount", switchboard.transferIdxThread.getTransferedEntryCount());
prop.put("running_twPayloadSize", serverMemory.bytesToString(switchboard.transferIdxThread.getTransferedBytes()));
prop.putNum("running_twEntityPercent", switchboard.transferIdxThread.getTransferedContainerPercent());
prop.putNum("running_twEntrySpeed", switchboard.transferIdxThread.getTransferedEntrySpeed());
prop.putNum("running_twEntityCount", sb.transferIdxThread.getTransferedContainerCount());
prop.putNum("running_twEntryCount", sb.transferIdxThread.getTransferedEntryCount());
prop.put("running_twPayloadSize", serverMemory.bytesToString(sb.transferIdxThread.getTransferedBytes()));
prop.putNum("running_twEntityPercent", sb.transferIdxThread.getTransferedContainerPercent());
prop.putNum("running_twEntrySpeed", sb.transferIdxThread.getTransferedEntrySpeed());
prop.put("running_deleteIndex", switchboard.transferIdxThread.deleteIndex() ? "1" : "0");
prop.put("running_peerName",switchboard.transferIdxThread.getSeed().getName());
prop.put("running_stopped",(switchboard.transferIdxThread.isFinished()) || (!switchboard.transferIdxThread.isAlive()) ? "1" : "0");
prop.put("running_deleteIndex", sb.transferIdxThread.deleteIndex() ? "1" : "0");
prop.put("running_peerName",sb.transferIdxThread.getSeed().getName());
prop.put("running_stopped",(sb.transferIdxThread.isFinished()) || (!sb.transferIdxThread.isAlive()) ? "1" : "0");
} else {
if (!prop.containsKey("running_status")) prop.put("running_status","Not running");
}
@ -130,7 +130,7 @@ public final class IndexTransfer_p {
//List known hosts
yacySeed seed;
int hc = 0;
if ((yacyCore.seedDB != null) && (yacyCore.seedDB.sizeConnected() > 0)) {
if ((sb.wordIndex.seedDB != null) && (sb.wordIndex.seedDB.sizeConnected() > 0)) {
Iterator<yacySeed> e = yacyCore.dhtAgent.getAcceptRemoteIndexSeeds("------------");
TreeMap<String, String> hostList = new TreeMap<String, String>();
while (e.hasNext()) {

12
htroot/MessageSend_p.java
View File

@ -50,6 +50,7 @@ import java.util.Date;
import java.util.HashMap;
import de.anomic.http.httpHeader;
import de.anomic.plasma.plasmaSwitchboard;
import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch;
import de.anomic.yacy.yacyClient;
@ -65,6 +66,7 @@ public class MessageSend_p {
public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch<?> env) {
plasmaSwitchboard sb = (plasmaSwitchboard) env;
serverObjects prop = new serverObjects();
if ((post == null) || (post.get("hash","").length() == 0)) {
@ -85,14 +87,14 @@ public class MessageSend_p {
// open an editor page for the message
// first ask if the other peer is online, and also what kind of document it accepts
HashMap<String, String> result = yacyClient.permissionMessage(hash);
HashMap<String, String> result = yacyClient.permissionMessage(sb.wordIndex.seedDB, hash);
//System.out.println("DEBUG: permission request result = " + result.toString());
String peerName;
yacySeed targetPeer = null;
if (hash.equals(yacyCore.seedDB.mySeed().hash)) {
peerName = yacyCore.seedDB.mySeed().get(yacySeed.NAME,"nameless");
if (hash.equals(sb.wordIndex.seedDB.mySeed().hash)) {
peerName = sb.wordIndex.seedDB.mySeed().get(yacySeed.NAME,"nameless");
} else {
targetPeer = yacyCore.seedDB.getConnected(hash);
targetPeer = sb.wordIndex.seedDB.getConnected(hash);
if (targetPeer == null)
peerName = "nameless";
else
@ -149,7 +151,7 @@ public class MessageSend_p {
} catch (UnsupportedEncodingException e) {
mb = message.getBytes();
}
HashMap<String, String> result = yacyClient.postMessage(hash, subject, mb);
HashMap<String, String> result = yacyClient.postMessage(sb.wordIndex.seedDB, hash, subject, mb);
//message has been sent
prop.put("mode_status_response", (String) result.get("response"));

23
htroot/Messages_p.java
View File

@ -58,7 +58,6 @@ import de.anomic.plasma.plasmaSwitchboard;
import de.anomic.server.serverFileUtils;
import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch;
import de.anomic.yacy.yacyCore;
import de.anomic.yacy.yacySeed;
public class Messages_p {
@ -71,22 +70,22 @@ public class Messages_p {
}
public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch<?> env) {
plasmaSwitchboard switchboard = (plasmaSwitchboard) env;
plasmaSwitchboard sb = (plasmaSwitchboard) env;
serverObjects prop = new serverObjects();
// set peer address / name
final String peerAddress = yacyCore.seedDB.mySeed().getPublicAddress();
final String peerName = yacyCore.seedDB.mySeed().getName();
final String peerAddress = sb.wordIndex.seedDB.mySeed().getPublicAddress();
final String peerName = sb.wordIndex.seedDB.mySeed().getName();
prop.put("peerAddress", peerAddress);
prop.putHTML("peerName", peerName, true);
// List known hosts for message sending (from Blacklist_p.java)
if (yacyCore.seedDB != null && yacyCore.seedDB.sizeConnected() > 0) {
if (sb.wordIndex.seedDB != null && sb.wordIndex.seedDB.sizeConnected() > 0) {
prop.put("peersKnown", "1");
int peerCount = 0;
try {
TreeMap<String, String> hostList = new TreeMap<String, String>();
final Iterator<yacySeed> e = yacyCore.seedDB.seedsConnected(true, false, null, (float) 0.0);
final Iterator<yacySeed> e = sb.wordIndex.seedDB.seedsConnected(true, false, null, (float) 0.0);
while (e.hasNext()) {
yacySeed seed = e.next();
if (seed != null) hostList.put(seed.get(yacySeed.NAME, "nameless"),seed.hash);
@ -113,8 +112,8 @@ public class Messages_p {
messageBoard.entry message;
// first reset notification
File notifierSource = new File(switchboard.getRootPath(), switchboard.getConfig("htRootPath", "htroot") + "/env/grafics/empty.gif");
File notifierDest = new File(switchboard.getConfigPath("htDocsPath", "DATA/HTDOCS"), "notifier.gif");
File notifierSource = new File(sb.getRootPath(), sb.getConfig("htRootPath", "htroot") + "/env/grafics/empty.gif");
File notifierDest = new File(sb.getConfigPath("htDocsPath", "DATA/HTDOCS"), "notifier.gif");
try {
serverFileUtils.copy(notifierSource, notifierDest);
} catch (IOException e) {
@ -122,21 +121,21 @@ public class Messages_p {
if (action.equals("delete")) {
String key = post.get("object", "");
switchboard.messageDB.remove(key);
sb.messageDB.remove(key);
action = "list";
}
if (action.equals("list")) {
prop.put("mode", "0"); //list
try {
Iterator<String> i = switchboard.messageDB.keys(null, true);
Iterator<String> i = sb.messageDB.keys(null, true);
String key;
boolean dark = true;
int count=0;
while (i.hasNext()) {
key = i.next();
message = switchboard.messageDB.read(key);
message = sb.messageDB.read(key);
prop.put("mode_messages_"+count+"_dark", ((dark) ? "1" : "0") );
prop.put("mode_messages_"+count+"_date", dateString(message.date()));
prop.putHTML("mode_messages_"+count+"_from", message.author(), true);
@ -174,7 +173,7 @@ public class Messages_p {
if (action.equals("view")) {
prop.put("mode", "1"); //view
String key = post.get("object", "");
message = switchboard.messageDB.read(key);
message = sb.messageDB.read(key);
if (message == null) throw new NullPointerException("Message with ID " + key + " does not exist");
prop.putHTML("mode_from", message.author(), true);

64
htroot/Network.java
View File

@ -84,30 +84,30 @@ public class Network {
prop.putHTML("page_networkName", sb.getConfig("network.unit.name", "unspecified"));
final boolean overview = (post == null) || (post.get("page", "0").equals("0"));
final String mySeedType = yacyCore.seedDB.mySeed().get(yacySeed.PEERTYPE, yacySeed.PEERTYPE_VIRGIN);
final String mySeedType = sb.wordIndex.seedDB.mySeed().get(yacySeed.PEERTYPE, yacySeed.PEERTYPE_VIRGIN);
final boolean iAmActive = (mySeedType.equals(yacySeed.PEERTYPE_SENIOR) || mySeedType.equals(yacySeed.PEERTYPE_PRINCIPAL));
if (overview) {
long accActLinks = yacyCore.seedDB.countActiveURL();
long accActWords = yacyCore.seedDB.countActiveRWI();
final long accPassLinks = yacyCore.seedDB.countPassiveURL();
final long accPassWords = yacyCore.seedDB.countPassiveRWI();
long accPotLinks = yacyCore.seedDB.countPotentialURL();
long accPotWords = yacyCore.seedDB.countPotentialRWI();
long accActLinks = sb.wordIndex.seedDB.countActiveURL();
long accActWords = sb.wordIndex.seedDB.countActiveRWI();
final long accPassLinks = sb.wordIndex.seedDB.countPassiveURL();
final long accPassWords = sb.wordIndex.seedDB.countPassiveRWI();
long accPotLinks = sb.wordIndex.seedDB.countPotentialURL();
long accPotWords = sb.wordIndex.seedDB.countPotentialRWI();
int conCount = yacyCore.seedDB.sizeConnected();
final int disconCount = yacyCore.seedDB.sizeDisconnected();
int potCount = yacyCore.seedDB.sizePotential();
int conCount = sb.wordIndex.seedDB.sizeConnected();
final int disconCount = sb.wordIndex.seedDB.sizeDisconnected();
int potCount = sb.wordIndex.seedDB.sizePotential();
// final boolean complete = ((post == null) ? false : post.get("links", "false").equals("true"));
final long otherppm = yacyCore.seedDB.countActivePPM();
final double otherqpm = yacyCore.seedDB.countActiveQPM();
final long otherppm = sb.wordIndex.seedDB.countActivePPM();
final double otherqpm = sb.wordIndex.seedDB.countActiveQPM();
long myppm = 0;
double myqph = 0d;
// create own peer info
yacySeed seed = yacyCore.seedDB.mySeed();
if (yacyCore.seedDB.mySeed() != null){ //our Peer
yacySeed seed = sb.wordIndex.seedDB.mySeed();
if (sb.wordIndex.seedDB.mySeed() != null){ //our Peer
// update seed info
yacyCore.peerActions.updateMySeed();
@ -123,17 +123,17 @@ public class Network {
// my-info
prop.putHTML("table_my-name", seed.get(yacySeed.NAME, "-") );
prop.put("table_my-hash", seed.hash );
if (yacyCore.seedDB.mySeed().isVirgin()) {
if (sb.wordIndex.seedDB.mySeed().isVirgin()) {
prop.put("table_my-info", 0);
} else if(yacyCore.seedDB.mySeed().isJunior()) {
} else if(sb.wordIndex.seedDB.mySeed().isJunior()) {
prop.put("table_my-info", 1);
accPotLinks += LCount;
accPotWords += ICount;
} else if(yacyCore.seedDB.mySeed().isSenior()) {
} else if(sb.wordIndex.seedDB.mySeed().isSenior()) {
prop.put("table_my-info", 2);
accActLinks += LCount;
accActWords += ICount;
} else if(yacyCore.seedDB.mySeed().isPrincipal()) {
} else if(sb.wordIndex.seedDB.mySeed().isPrincipal()) {
prop.put("table_my-info", 3);
accActLinks += LCount;
accActWords += ICount;
@ -214,13 +214,13 @@ public class Network {
yacySeed peer = new yacySeed(post.get("peerHash"),map);
yacyCore.peerActions.updateMySeed();
final int added = yacyClient.publishMySeed(peer.getPublicAddress(), peer.hash);
final int added = yacyClient.publishMySeed(sb.wordIndex.seedDB.mySeed(), peer.getPublicAddress(), peer.hash);
if (added <= 0) {
prop.put("table_comment",1);
prop.putHTML("table_comment_status","publish: disconnected peer '" + peer.getName() + "/" + post.get("peerHash") + "' from " + peer.getPublicAddress());
} else {
peer = yacyCore.seedDB.getConnected(peer.hash);
peer = sb.wordIndex.seedDB.getConnected(peer.hash);
if (peer == null) {
prop.put("table_comment",1);
prop.put("table_comment_status","publish: disconnected peer 'UNKNOWN/" + post.get("peerHash") + "' from UNKNOWN");
@ -246,14 +246,14 @@ public class Network {
final int page = Integer.parseInt(post.get("page", "1"));
final int maxCount = Integer.parseInt(post.get("maxCount", "300"));
int conCount = 0;
if (yacyCore.seedDB == null) {
if (sb.wordIndex.seedDB == null) {
prop.put("table", 0);//no remote senior/principal proxies known"
} else {
int size = 0;
switch (page) {
case 1 : size = yacyCore.seedDB.sizeConnected(); break;
case 2 : size = yacyCore.seedDB.sizeDisconnected(); break;
case 3 : size = yacyCore.seedDB.sizePotential(); break;
case 1 : size = sb.wordIndex.seedDB.sizeConnected(); break;
case 2 : size = sb.wordIndex.seedDB.sizeDisconnected(); break;
case 3 : size = sb.wordIndex.seedDB.sizePotential(); break;
default: break;
}
if (size == 0) {
@ -262,7 +262,7 @@ public class Network {
// add temporary the own seed to the database
if (iAmActive) {
yacyCore.peerActions.updateMySeed();
yacyCore.seedDB.addConnected(yacyCore.seedDB.mySeed());
sb.wordIndex.seedDB.addConnected(sb.wordIndex.seedDB.mySeed());
}
// find updated Information using YaCyNews
@ -271,7 +271,7 @@ public class Network {
final HashMap<String, Map<String, String>> updatedBlog = new HashMap<String, Map<String, String>>();
final HashMap<String, String> isCrawling = new HashMap<String, String>();
yacyNewsRecord record;
Iterator<yacyNewsRecord> recordIterator = yacyCore.newsPool.recordIterator(yacyNewsPool.INCOMING_DB, true);
Iterator<yacyNewsRecord> recordIterator = sb.wordIndex.newsPool.recordIterator(yacyNewsPool.INCOMING_DB, true);
while (recordIterator.hasNext()) {
record = recordIterator.next();
if (record == null) {
@ -292,9 +292,9 @@ public class Network {
final boolean complete = post.containsKey("ip");
Iterator<yacySeed> e = null;
switch (page) {
case 1 : e = yacyCore.seedDB.seedsSortedConnected(post.get("order", "down").equals("up"), post.get("sort", yacySeed.LCOUNT)); break;
case 2 : e = yacyCore.seedDB.seedsSortedDisconnected(post.get("order", "down").equals("up"), post.get("sort", yacySeed.LASTSEEN)); break;
case 3 : e = yacyCore.seedDB.seedsSortedPotential(post.get("order", "down").equals("up"), post.get("sort", yacySeed.LASTSEEN)); break;
case 1 : e = sb.wordIndex.seedDB.seedsSortedConnected(post.get("order", "down").equals("up"), post.get("sort", yacySeed.LCOUNT)); break;
case 2 : e = sb.wordIndex.seedDB.seedsSortedDisconnected(post.get("order", "down").equals("up"), post.get("sort", yacySeed.LASTSEEN)); break;
case 3 : e = sb.wordIndex.seedDB.seedsSortedPotential(post.get("order", "down").equals("up"), post.get("sort", yacySeed.LASTSEEN)); break;
default: break;
}
String startURL;
@ -336,7 +336,7 @@ public class Network {
prop.put(STR_TABLE_LIST + conCount + "_updatedBlog", 0);
prop.put(STR_TABLE_LIST + conCount + "_isCrawling", 0);
if (conCount >= maxCount) { break; }
if (seed.hash.equals(yacyCore.seedDB.mySeed().hash)) {
if (seed.hash.equals(sb.wordIndex.seedDB.mySeed().hash)) {
prop.put(STR_TABLE_LIST + conCount + "_dark", 2);
} else {
prop.put(STR_TABLE_LIST + conCount + "_dark", ((dark) ? 1 : 0) ); dark=!dark;
@ -374,7 +374,7 @@ public class Network {
prop.putHTML(STR_TABLE_LIST + conCount + "_shortname", shortname);
prop.putHTML(STR_TABLE_LIST + conCount + "_fullname", seed.get(yacySeed.NAME, "deadlink"));
userAgent = null;
if (seed.hash.equals(yacyCore.seedDB.mySeed().hash)) {
if (seed.hash.equals(sb.wordIndex.seedDB.mySeed().hash)) {
final JakartaCommonsHttpClient httpClient = new JakartaCommonsHttpClient(10000, null, null);
userAgent = httpClient.getUserAgent();
location = HttpClient.generateLocation();
@ -463,7 +463,7 @@ public class Network {
conCount++;
} // seed != null
} // while
if (iAmActive) { yacyCore.seedDB.removeMySeed(); }
if (iAmActive) { sb.wordIndex.seedDB.removeMySeed(); }
prop.putNum("table_list", conCount);
prop.put("table", 1);
prop.putNum("table_num", conCount);

6
htroot/NetworkPicture.java
View File

@ -45,6 +45,7 @@
import de.anomic.http.httpHeader;
import de.anomic.plasma.plasmaGrafics;
import de.anomic.plasma.plasmaSwitchboard;
import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch;
import de.anomic.ymage.ymageMatrix;
@ -53,7 +54,8 @@ import de.anomic.ymage.ymageMatrix;
public class NetworkPicture {
public static ymageMatrix respond(httpHeader header, serverObjects post, serverSwitch<?> env) {
plasmaSwitchboard sb = (plasmaSwitchboard) env;
int width = 768;
int height = 576;
int passiveLimit = 300;
@ -80,7 +82,7 @@ public class NetworkPicture {
if (passiveLimit > 1000000) passiveLimit = 1000000;
if (potentialLimit > 1000000) potentialLimit = 1000000;
if (maxCount > 1000) maxCount = 1000;
return plasmaGrafics.getNetworkPicture(10000, width, height, passiveLimit, potentialLimit, maxCount, corona, env.getConfig("network.unit.name", "unspecified"), env.getConfig("network.unit.description", "unspecified"), bgcolor);
return plasmaGrafics.getNetworkPicture(sb.wordIndex.seedDB, 10000, width, height, passiveLimit, potentialLimit, maxCount, corona, env.getConfig("network.unit.name", "unspecified"), env.getConfig("network.unit.description", "unspecified"), bgcolor);
}
}

33
htroot/News.java
View File

@ -53,7 +53,6 @@ import de.anomic.plasma.plasmaSwitchboard;
import de.anomic.server.serverDate;
import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch;
import de.anomic.yacy.yacyCore;
import de.anomic.yacy.yacyNewsPool;
import de.anomic.yacy.yacyNewsRecord;
import de.anomic.yacy.yacySeed;
@ -61,7 +60,7 @@ import de.anomic.yacy.yacySeed;
public class News {
public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch<?> env) {
plasmaSwitchboard switchboard = (plasmaSwitchboard) env;
plasmaSwitchboard sb = (plasmaSwitchboard) env;
serverObjects prop = new serverObjects();
boolean overview = (post == null) || (post.get("page", "0").equals("0"));
int tableID = (overview) ? -1 : Integer.parseInt(post.get("page", "0")) - 1;
@ -70,7 +69,7 @@ public class News {
if (post != null) {
if ((post.containsKey("deletespecific")) && (tableID >= 0)) {
if (switchboard.adminAuthenticated(header) < 2) {
if (sb.adminAuthenticated(header) < 2) {
prop.put("AUTHENTICATE", "admin log-in");
return prop; // this button needs authentication, force log-in
}
@ -82,22 +81,22 @@ public class News {
if ((check.startsWith("del_")) && (post.get(check, "off").equals("on"))) {
id = check.substring(4);
try {
yacyCore.newsPool.moveOff(tableID, id);
sb.wordIndex.newsPool.moveOff(tableID, id);
} catch (IOException ee) {ee.printStackTrace();}
}
}
}
if ((post.containsKey("deleteall")) && (tableID >= 0)) {
if (switchboard.adminAuthenticated(header) < 2) {
if (sb.adminAuthenticated(header) < 2) {
prop.put("AUTHENTICATE", "admin log-in");
return prop; // this button needs authentication, force log-in
}
try {
if ((tableID == yacyNewsPool.PROCESSED_DB) || (tableID == yacyNewsPool.PUBLISHED_DB)) {
yacyCore.newsPool.clear(tableID);
sb.wordIndex.newsPool.clear(tableID);
} else {
yacyCore.newsPool.moveOffAll(tableID);
sb.wordIndex.newsPool.moveOffAll(tableID);
}
} catch (IOException e) {
e.printStackTrace();
@ -110,19 +109,19 @@ public class News {
// show overview
prop.put("table", "0");
prop.put("page", "0");
prop.putNum("table_insize", yacyCore.newsPool.size(yacyNewsPool.INCOMING_DB));
prop.putNum("table_prsize", yacyCore.newsPool.size(yacyNewsPool.PROCESSED_DB));
prop.putNum("table_ousize", yacyCore.newsPool.size(yacyNewsPool.OUTGOING_DB));
prop.putNum("table_pusize", yacyCore.newsPool.size(yacyNewsPool.PUBLISHED_DB));
prop.putNum("table_insize", sb.wordIndex.newsPool.size(yacyNewsPool.INCOMING_DB));
prop.putNum("table_prsize", sb.wordIndex.newsPool.size(yacyNewsPool.PROCESSED_DB));
prop.putNum("table_ousize", sb.wordIndex.newsPool.size(yacyNewsPool.OUTGOING_DB));
prop.putNum("table_pusize", sb.wordIndex.newsPool.size(yacyNewsPool.PUBLISHED_DB));
} else {
// generate table
prop.put("table", "1");
prop.put("page", tableID + 1);
prop.put("table_page", tableID + 1);
if (yacyCore.seedDB != null) {
int maxCount = Math.min(1000, yacyCore.newsPool.size(tableID));
Iterator<yacyNewsRecord> recordIterator = yacyCore.newsPool.recordIterator(tableID, false);
if (sb.wordIndex.seedDB != null) {
int maxCount = Math.min(1000, sb.wordIndex.newsPool.size(tableID));
Iterator<yacyNewsRecord> recordIterator = sb.wordIndex.newsPool.recordIterator(tableID, false);
yacyNewsRecord record;
yacySeed seed;
int i = 0;
@ -130,8 +129,8 @@ public class News {
record = recordIterator.next();
if (record == null) continue;
seed = yacyCore.seedDB.getConnected(record.originator());
if (seed == null) seed = yacyCore.seedDB.getDisconnected(record.originator());
seed = sb.wordIndex.seedDB.getConnected(record.originator());
if (seed == null) seed = sb.wordIndex.seedDB.getDisconnected(record.originator());
String category = record.category();
prop.put("table_list_" + i + "_id", record.id());
prop.putHTML("table_list_" + i + "_ori", (seed == null) ? record.originator() : seed.getName());
@ -202,7 +201,7 @@ public class News {
}
// adding the peer address
prop.put("address",yacyCore.seedDB.mySeed().getPublicAddress());
prop.put("address", sb.wordIndex.seedDB.mySeed().getPublicAddress());
// return rewrite properties
return prop;

15
htroot/QuickCrawlLink_p.java
View File

@ -58,7 +58,6 @@ import de.anomic.plasma.plasmaCrawlProfile;
import de.anomic.plasma.plasmaSwitchboard;
import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch;
import de.anomic.yacy.yacyCore;
import de.anomic.yacy.yacyURL;
public class QuickCrawlLink_p {
@ -74,7 +73,7 @@ public class QuickCrawlLink_p {
public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch<?> env) {
serverObjects prop = new serverObjects();
plasmaSwitchboard switchboard = (plasmaSwitchboard)env;
plasmaSwitchboard sb = (plasmaSwitchboard) env;
if (post == null) {
// send back usage example
@ -139,14 +138,14 @@ public class QuickCrawlLink_p {
}
String urlhash = crawlingStartURL.hash();
switchboard.wordIndex.removeURL(urlhash);
switchboard.crawlQueues.noticeURL.removeByURLHash(urlhash);
switchboard.crawlQueues.errorURL.remove(urlhash);
sb.wordIndex.removeURL(urlhash);
sb.crawlQueues.noticeURL.removeByURLHash(urlhash);
sb.crawlQueues.errorURL.remove(urlhash);
// create crawling profile
plasmaCrawlProfile.entry pe = null;
try {
pe = switchboard.profilesActiveCrawls.newEntry(
pe = sb.profilesActiveCrawls.newEntry(
crawlingStartURL.getHost(),
crawlingStartURL,
crawlingFilter,
@ -176,10 +175,10 @@ public class QuickCrawlLink_p {
// stack URL
String reasonString = null;
reasonString = switchboard.crawlStacker.stackCrawl(
reasonString = sb.crawlStacker.stackCrawl(
crawlingStartURL,
null,
yacyCore.seedDB.mySeed().hash,
sb.wordIndex.seedDB.mySeed().hash,
(title==null)?"CRAWLING-ROOT":title,
new Date(),
0,

5
htroot/SearchEventPicture.java
View File

@ -47,6 +47,7 @@
import de.anomic.http.httpHeader;
import de.anomic.plasma.plasmaGrafics;
import de.anomic.plasma.plasmaSearchEvent;
import de.anomic.plasma.plasmaSwitchboard;
import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch;
import de.anomic.ymage.ymageMatrix;
@ -56,10 +57,10 @@ import de.anomic.ymage.ymageMatrix;
public class SearchEventPicture {
public static ymageMatrix respond(httpHeader header, serverObjects post, serverSwitch<?> env) {
plasmaSwitchboard sb = (plasmaSwitchboard) env;
String eventID = (String) header.get("event", plasmaSearchEvent.lastEventID);
if (eventID == null) return null;
ymageMatrix yp = plasmaGrafics.getSearchEventPicture(eventID);
ymageMatrix yp = plasmaGrafics.getSearchEventPicture(sb.wordIndex.seedDB, eventID);
if (yp == null) return new ymageMatrix(1, 1, ymageMatrix.MODE_SUB, "000000"); // empty image
return yp;

31
htroot/SettingsAck_p.html
View File

@ -145,38 +145,7 @@
</tr>
</table>
::<!-- 22 -->
<p><strong>Your port forwarding settings have been changed.</strong></p>
<table>
<tr>
<td>Port Forwarding Support is:</td>
<td class="settingsValue">#[portForwarding.Enabled]#</td>
</tr>
<tr>
<td>Port Forwarding Method:</td>
<td class="settingsValue">#[portForwarding.Type]#</td>
</tr>
<tr><td colspan="2"><hr /></td></tr>
<tr><td colspan="2"><strong>Secure channel specific settings:</strong></td></tr>
<tr>
<td>Port Forwarding Port:</td>
<td class="settingsValue">#[portForwarding.sch.Port]#</td>
</tr>
<tr>
<td>Port Forwarding Host:</td>
<td class="settingsValue">#[portForwarding.sch.HostUser]#@#[portForwarding.sch.Host]#:#[portForwarding.sch.HostPort]#</td>
</tr>
<tr>
<td>Port Forwarding uses proxy:</td>
<td class="settingsValue">#[portForwarding.sch.UseProxy]#</td>
</tr>
</table>
::<!-- 23 : Port forwarding - error -->
<p><strong>Port Forwarding Settings changed, but something is wrong.</strong></p>
<p class="error">#[errormsg]#</p>
<p>
Port Forwarding was deactivated automatically.<br />
Please return to the settings page and modify the data.<br />
</p>
::<!-- 24 -->
<p>You are now <strong>event-based online</strong>. After a short while you should see the effect on the <a href="Status.html">status</a> page.</p>
::<!-- 25 -->

56
htroot/SettingsAck_p.java
View File

@ -64,7 +64,6 @@ import de.anomic.http.httpdProxyHandler;
import de.anomic.kelondro.kelondroBase64Order;
import de.anomic.plasma.plasmaParser;
import de.anomic.plasma.plasmaSwitchboard;
import de.anomic.server.serverBusyThread;
import de.anomic.server.serverCodings;
import de.anomic.server.serverCore;
import de.anomic.server.serverDate;
@ -82,6 +81,7 @@ public class SettingsAck_p {
public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch<?> env) {
// return variable that accumulates replacements
serverObjects prop = new serverObjects();
plasmaSwitchboard sb = (plasmaSwitchboard) env;
// get referer for backlink
String referer = (String) header.get(httpHeader.REFERER);
@ -234,51 +234,6 @@ public class SettingsAck_p {
return prop;
}
// port forwarding configuration
if (post.containsKey("portForwarding")) {
env.setConfig("portForwarding.Enabled", post.containsKey("portForwarding.Enabled")?"true":"false");
env.setConfig("portForwarding.Type", (String)post.get("portForwarding.Type"));
env.setConfig("portForwarding.sch.UseProxy",post.containsKey("portForwarding.sch.UseProxy")?"true":"false");
env.setConfig("portForwarding.sch.Port", (String)post.get("portForwarding.sch.Port"));
env.setConfig("portForwarding.sch.Host", (String)post.get("portForwarding.sch.Host"));
env.setConfig("portForwarding.sch.HostPort",(String)post.get("portForwarding.sch.HostPort"));
env.setConfig("portForwarding.sch.HostUser",(String)post.get("portForwarding.sch.HostUser"));
env.setConfig("portForwarding.sch.HostPwd", (String)post.get("portForwarding.sch.HostPwd"));
// trying to reconnect the port forwarding channel
try {
serverCore httpd = (serverCore) env.getThread("10_httpd");
if ((serverCore.portForwardingEnabled) && (serverCore.portForwarding != null)) {
// trying to shutdown the current port forwarding channel
serverCore.portForwarding.disconnect();
}
// trying to reinitialize the port forwarding
httpd.initPortForwarding();
// notifying publishSeed Thread
serverBusyThread peerPing = env.getThread("30_peerping");
peerPing.notifyThread();
} catch (Exception e) {
prop.put("info", "23");
prop.putHTML("info_errormsg",(e.getMessage() == null) ? "unknown" : e.getMessage().replaceAll("\n","<br>"));
return prop;
}
prop.put("info", "22");
prop.put("info_portForwarding.Enabled", post.containsKey("portForwarding.Enabled")?"on":"off");
prop.put("info_portForwarding.Type", (String)post.get("portForwarding.Type"));
prop.put("info_portForwarding.sch.UseProxy",post.containsKey("portForwarding.sch.UseProxy")?"on":"off");
prop.put("info_portForwarding.sch.Port", (String)post.get("portForwarding.sch.Port"));
prop.put("info_portForwarding.sch.Host", (String)post.get("portForwarding.sch.Host"));
prop.put("info_portForwarding.sch.HostPort",(String)post.get("portForwarding.sch.HostPort"));
prop.put("info_portForwarding.sch.HostUser",(String)post.get("portForwarding.sch.HostUser"));
prop.put("info_portForwarding.sch.HostPwd", (String)post.get("portForwarding.sch.HostPwd"));
return prop;
}
// server access
if (post.containsKey("serveraccount")) {
@ -297,7 +252,7 @@ public class SettingsAck_p {
} else {
serverCore.useStaticIP = true;
}
yacyCore.seedDB.mySeed().put(yacySeed.IP, staticIP);
sb.wordIndex.seedDB.mySeed().put(yacySeed.IP, staticIP);
env.setConfig("staticIP", staticIP);
// server access data
@ -413,7 +368,6 @@ public class SettingsAck_p {
/* ====================================================================
* Enabling settings
* ==================================================================== */
plasmaSwitchboard sb = (plasmaSwitchboard)env;
httpdProxyHandler.setRemoteProxyConfig(httpRemoteProxyConfig.init(sb));
// httpdProxyHandler.remoteProxyUse = post.get("remoteProxyUse", "").equals("on");
@ -437,7 +391,7 @@ public class SettingsAck_p {
if (post.containsKey("seedUploadRetry")) {
String error;
if ((error = ((plasmaSwitchboard)env).yc.saveSeedList(env)) == null) {
if ((error = yacyCore.saveSeedList(sb)) == null) {
// trying to upload the seed-list file
prop.put("info", "13");
prop.put("info_success", "1");
@ -468,7 +422,7 @@ public class SettingsAck_p {
// try an upload
String error;
if ((error = ((plasmaSwitchboard)env).yc.saveSeedList(env)) == null) {
if ((error = yacyCore.saveSeedList(sb)) == null) {
// we have successfully uploaded the seed-list file
prop.put("info_seedUploadMethod", newSeedUploadMethod);
prop.putHTML("info_seedURL",newSeedURLStr);
@ -514,7 +468,7 @@ public class SettingsAck_p {
// were changed, we now try to upload the seed list with the new settings
if (env.getConfig("seedUploadMethod","none").equalsIgnoreCase(uploaderName)) {
String error;
if ((error = ((plasmaSwitchboard)env).yc.saveSeedList(env)) == null) {
if ((error = yacyCore.saveSeedList(sb)) == null) {
// we have successfully uploaded the seed file
prop.put("info", "13");

66
htroot/Settings_PortForwarding.inc
View File

@ -1,66 +0,0 @@
<form action="SettingsAck_p.html" method="post" enctype="multipart/form-data">
<fieldset><legend id="portForwarding">Port Forwarding (optional)</legend>
<p>
You can use a remote server running a ssh demon to forward your server/proxy port.
This is useful if you want to tunnel throug a NAT/router.
Alternatively, you can simply set a virtual server port on your NAT/Server to enable connections from outside.
</p>
#(portForwarding.Available)#<p><b><font color="red">You need to install libx to use this feature</font></b></p>::#(/portForwarding.Available)#
<table border="0" cellspacing="5">
<tr valign="top">
<td><label for="pfwd">Enable port forwarding</label>:</td>
<td><input type="checkbox" name="portForwarding.Enabled" id="pfwd" #(portForwarding.Enabled)#::checked="checked" #(/portForwarding.Enabled)##(portForwarding.Available)#disabled="disabled" ::#(/portForwarding.Available)#/></td>
<td>Enabling / disabling port forwarding</td>
</tr>
<tr valign="top">
<td><label for="sel_pfwd">Forwarding method</label>:</td>
<td><select name="portForwarding.Type" id="sel_pfwd" size="1">
#{forwardingMethods}#
<option value="#[name]#" #(selected)#::selected="selected"#(/selected)#>#[name]#</option>
#{/forwardingMethods}#
</select>
</td>
<td>Select the port forwarding method to use.</td>
</tr>
<tr><td colspan="3"><hr /></td></tr>
<tr><td colspan="3"><b>Special settings for port forwarding using secure channel</b></td></tr>
<tr valign="top">
<td><label for="pfwd_proxy">Forwarding via proxy</label>:</td>
<td><input type="checkbox" name="portForwarding.sch.UseProxy" id="pfwd_proxy" #(portForwarding.sch.UseProxy)#::checked="checked" #(/portForwarding.sch.UseProxy)#disabled="disabled" /></td>
<td>
<!-- Specifies if the connection should be established through the <a href="#proxy">remote proxy</a>.-->
<i>Function not available at the moment.</i>
</td>
</tr>
<tr valign="top">
<td><label for="pfwd_port">Forwarding port</label>:</td>
<td><input name="portForwarding.sch.Port" id="pfwd_port" type="text" size="5" maxlength="5" value="#[portForwarding.sch.Port]#" /></td>
<td>The tcp/ip port on the remote server that should be forwarded via the secure channel to the local host.</td>
</tr>
<tr valign="top">
<td><label for="pfwd_host">Forwarding host</label>:</td>
<td><input name="portForwarding.sch.Host" id="pfwd_host" type="text" size="32" maxlength="128" value="#[portForwarding.sch.Host]#" /></td>
<td>IP address or name of the remote server.</td>
</tr>
<tr valign="top">
<td><label for="pfwd_host_port">Forwarding host port</label>:</td>
<td><input name="portForwarding.sch.HostPort" id="pfwd_host_port" type="text" size="5" maxlength="5" value="#[portForwarding.sch.HostPort]#" /></td>
<td>SSH port of the remote server.</td>
</tr>
<tr valign="top">
<td><label for="pfwd_host_user">Forwarding host user</label>:</td>
<td><input name="portForwarding.sch.HostUser" id="pfwd_host_user" type="text" size="32" maxlength="128" value="#[portForwarding.sch.HostUser]#" /></td>
<td>Username to login on the remote server.</td>
</tr>
<tr valign="top">
<td><label for="pfwd_host_pwd">Forwarding host password</label>:</td>
<td><input name="portForwarding.sch.HostPwd" id="pfwd_host_pwd" type="password" size="32" maxlength="128" value="#[portForwarding.sch.HostPwd]#" /></td>
<td>Password to login on the remote server.</td>
</tr>
<tr><td colspan="3"><hr /></td></tr>
<tr valign="top">
<td colspan="3"><input type="submit" name="portForwarding" value="Submit"#(portForwarding.Available)# disabled="disabled"::#(/portForwarding.Available)# /> <i>Changes will take effect immediately.</i></td>
</tr>
</table>
</fieldset>
</form><br />

1
htroot/Settings_p.html
View File

@ -35,7 +35,6 @@
<li><a href="?page=crawler">Crawler Settings</a></li>
<li><a href="?page=http">HTTP Networking</a></li>
<li><a href="?page=proxy">Remote Proxy (optional)</a></li>
<li><a href="?page=portForwarding">Port Forwarding (optional)</a></li>
<li><a href="?page=seed">Seed Upload Settings</a></li>
<li><a href="?page=messageForwarding">Message Forwarding (optional)</a></li>
</ul>

35
htroot/Settings_p.java
View File

@ -89,9 +89,6 @@ public final class Settings_p {
else if (page.equals("messageForwarding")) {
prop.put("settingsTables", "Settings_MessageForwarding.inc");
}
else if (page.equals("portForwarding")) {
prop.put("settingsTables", "Settings_PortForwarding.inc");
}
else if (page.equals("parser")) {
prop.put("settingsTables", "Settings_Parser.inc");
}
@ -114,37 +111,7 @@ public final class Settings_p {
prop.put("connectionKeepAliveSupport", env.getConfig("connectionKeepAliveSupport", "false").equals("true") ? "1" : "0");
prop.put("proxy.sendViaHeader", env.getConfig("proxy.sendViaHeader", "false").equals("true") ? "1" : "0");
prop.put("proxy.sendXForwardedForHeader", env.getConfig("proxy.sendXForwardedForHeader", "true").equals("true") ? "1" : "0");
// remote port forwarding settings
String[] forwardingMethods = new String[]{"sch","upnp"};
String currentForwarder = env.getConfig("portForwarding.Type", "none");
boolean portForwardingAvailable = false;
int methodCount = 0;
for (int i=0; i < forwardingMethods.length; i++) {
try {
//Class forwarder = Class.forName(env.getConfig("portForwarding." + forwardingMethods[i],"")); // FIXME: why is forwarder declared, but never used?
prop.put("forwardingMethods_" + methodCount + "_name",forwardingMethods[i]);
prop.put("forwardingMethods_" + methodCount + "_selected", forwardingMethods[i].equals(currentForwarder)?1:0);
methodCount++;
} catch (Exception e) {
} catch (Error e) {}
}
prop.put("forwardingMethods", methodCount);
if (methodCount > 0) portForwardingAvailable = true;
prop.put("portForwarding.Type",currentForwarder);
prop.put("portForwarding.Available",portForwardingAvailable? "1" : "0");
prop.put("portForwarding.Enabled",env.getConfig("portForwarding.Enabled","false").equals("true")? "1" : "0");
prop.put("portForwarding.sch.UseProxy",env.getConfig("portForwarding.sch.UseProxy", "false").equals("true") ? "1" : "0");
prop.put("portForwarding.sch.Port",env.getConfig("portForwarding.sch.Port", ""));
prop.put("portForwarding.sch.Host",env.getConfig("portForwarding.sch.Host", ""));
prop.put("portForwarding.sch.HostPort",env.getConfig("portForwarding.sch.HostPort", ""));
prop.put("portForwarding.sch.HostUser",env.getConfig("portForwarding.sch.HostUser", ""));
prop.put("portForwarding.sch.HostPwd",env.getConfig("portForwarding.sch.HostPwd", ""));
// set values
String s;
int pos;

42
htroot/Status.java
View File

@ -170,16 +170,6 @@ public class Status {
// ssl support
prop.put("sslSupport",sb.getConfig("keyStore", "").length() == 0 ? "0" : "1");
// port forwarding: hostname and port
if ((serverCore.portForwardingEnabled) && (serverCore.portForwarding != null)) {
prop.put("portForwarding", "1");
prop.put("portForwarding_host", serverCore.portForwarding.getHost());
prop.put("portForwarding_port", Integer.toString(serverCore.portForwarding.getPort()));
prop.put("portForwarding_status", serverCore.portForwarding.isConnected() ? "1" : "0");
} else {
prop.put("portForwarding", "0");
}
if (sb.getConfig("remoteProxyUse", "false").equals("true")) {
prop.put("remoteProxy", "1");
prop.putHTML("remoteProxy_host", sb.getConfig("remoteProxyHost", "<unknown>"), true);
@ -192,35 +182,35 @@ public class Status {
// peer information
String thisHash = "";
final String thisName = sb.getConfig("peerName", "<nameless>");
if (yacyCore.seedDB.mySeed() == null) {
if (sb.wordIndex.seedDB.mySeed() == null) {
thisHash = "not assigned";
prop.put("peerAddress", "0"); // not assigned
prop.put("peerStatistics", "0"); // unknown
} else {
final long uptime = 60000 * Long.parseLong(yacyCore.seedDB.mySeed().get(yacySeed.UPTIME, "0"));
final long uptime = 60000 * Long.parseLong(sb.wordIndex.seedDB.mySeed().get(yacySeed.UPTIME, "0"));
prop.put("peerStatistics", "1");
prop.put("peerStatistics_uptime", serverDate.formatInterval(uptime));
prop.putNum("peerStatistics_pagesperminute", yacyCore.seedDB.mySeed().getPPM());
prop.putNum("peerStatistics_queriesperhour", Math.round(6000d * yacyCore.seedDB.mySeed().getQPM()) / 100d);
prop.putNum("peerStatistics_links", yacyCore.seedDB.mySeed().getLinkCount());
prop.put("peerStatistics_words", yFormatter.number(yacyCore.seedDB.mySeed().get(yacySeed.ICOUNT, "0")));
prop.putNum("peerStatistics_pagesperminute", sb.wordIndex.seedDB.mySeed().getPPM());
prop.putNum("peerStatistics_queriesperhour", Math.round(6000d * sb.wordIndex.seedDB.mySeed().getQPM()) / 100d);
prop.putNum("peerStatistics_links", sb.wordIndex.seedDB.mySeed().getLinkCount());
prop.put("peerStatistics_words", yFormatter.number(sb.wordIndex.seedDB.mySeed().get(yacySeed.ICOUNT, "0")));
prop.putNum("peerStatistics_juniorConnects", yacyCore.peerActions.juniorConnects);
prop.putNum("peerStatistics_seniorConnects", yacyCore.peerActions.seniorConnects);
prop.putNum("peerStatistics_principalConnects", yacyCore.peerActions.principalConnects);
prop.putNum("peerStatistics_disconnects", yacyCore.peerActions.disconnects);
prop.put("peerStatistics_connects", yFormatter.number(yacyCore.seedDB.mySeed().get(yacySeed.CCOUNT, "0")));
if (yacyCore.seedDB.mySeed().getPublicAddress() == null) {
thisHash = yacyCore.seedDB.mySeed().hash;
prop.put("peerStatistics_connects", yFormatter.number(sb.wordIndex.seedDB.mySeed().get(yacySeed.CCOUNT, "0")));
if (sb.wordIndex.seedDB.mySeed().getPublicAddress() == null) {
thisHash = sb.wordIndex.seedDB.mySeed().hash;
prop.put("peerAddress", "0"); // not assigned + instructions
prop.put("warningGoOnline", "1");
} else {
thisHash = yacyCore.seedDB.mySeed().hash;
thisHash = sb.wordIndex.seedDB.mySeed().hash;
prop.put("peerAddress", "1"); // Address
prop.put("peerAddress_address", yacyCore.seedDB.mySeed().getPublicAddress());
prop.put("peerAddress_address", sb.wordIndex.seedDB.mySeed().getPublicAddress());
prop.putHTML("peerAddress_peername", sb.getConfig("peerName", "<nameless>").toLowerCase(), true);
}
}
final String peerStatus = ((yacyCore.seedDB.mySeed() == null) ? yacySeed.PEERTYPE_VIRGIN : yacyCore.seedDB.mySeed().get(yacySeed.PEERTYPE, yacySeed.PEERTYPE_VIRGIN));
final String peerStatus = ((sb.wordIndex.seedDB.mySeed() == null) ? yacySeed.PEERTYPE_VIRGIN : sb.wordIndex.seedDB.mySeed().get(yacySeed.PEERTYPE, yacySeed.PEERTYPE_VIRGIN));
if (peerStatus.equals(yacySeed.PEERTYPE_VIRGIN)) {
prop.put(PEERSTATUS, "0");
prop.put("urgentStatusVirgin", "1");
@ -233,7 +223,7 @@ public class Status {
} else if (peerStatus.equals(yacySeed.PEERTYPE_PRINCIPAL)) {
prop.put(PEERSTATUS, "3");
prop.put("hintStatusPrincipal", "1");
prop.put("hintStatusPrincipal_seedURL", yacyCore.seedDB.mySeed().get("seedURL", "?"));
prop.put("hintStatusPrincipal_seedURL", sb.wordIndex.seedDB.mySeed().get("seedURL", "?"));
}
prop.putHTML("peerName", thisName);
prop.put("hash", thisHash);
@ -262,14 +252,14 @@ public class Status {
prop.put("seedServer_seedFile", sb.getConfig("seedFilePath", ""));
}
prop.put("seedServer_lastUpload",
serverDate.formatInterval(System.currentTimeMillis() - sb.yc.lastSeedUpload_timeStamp));
serverDate.formatInterval(System.currentTimeMillis() - sb.wordIndex.seedDB.lastSeedUpload_timeStamp));
} else {
prop.put(SEEDSERVER, "0"); // disabled
}
if (yacyCore.seedDB != null && yacyCore.seedDB.sizeConnected() > 0){
if (sb.wordIndex.seedDB != null && sb.wordIndex.seedDB.sizeConnected() > 0){
prop.put("otherPeers", "1");
prop.putNum("otherPeers_num", yacyCore.seedDB.sizeConnected());
prop.putNum("otherPeers_num", sb.wordIndex.seedDB.sizeConnected());
}else{
prop.put("otherPeers", "0"); // not online
}

28
htroot/Supporter.java
View File

@ -43,7 +43,6 @@ import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch;
import de.anomic.tools.crypt;
import de.anomic.tools.nxTools;
import de.anomic.yacy.yacyCore;
import de.anomic.yacy.yacyNewsPool;
import de.anomic.yacy.yacyNewsRecord;
import de.anomic.yacy.yacySeed;
@ -79,7 +78,7 @@ public class Supporter {
map.put("urlhash", hash);
map.put("vote", "negative");
map.put("refid", post.get("refid", ""));
yacyCore.newsPool.publishMyNews(yacyNewsRecord.newRecord(yacyNewsPool.CATEGORY_SURFTIPP_VOTE_ADD, map));
sb.wordIndex.newsPool.publishMyNews(yacyNewsRecord.newRecord(sb.wordIndex.seedDB.mySeed(), yacyNewsPool.CATEGORY_SURFTIPP_VOTE_ADD, map));
}
if ((post != null) && ((hash = post.get("votePositive", null)) != null)) {
if (!sb.verifyAuthentication(header, false)) {
@ -95,19 +94,19 @@ public class Supporter {
map.put("vote", "positive");
map.put("refid", post.get("refid", ""));
map.put("comment", post.get("comment", ""));
yacyCore.newsPool.publishMyNews(new yacyNewsRecord(yacyNewsPool.CATEGORY_SURFTIPP_VOTE_ADD, map));
sb.wordIndex.newsPool.publishMyNews(new yacyNewsRecord(sb.wordIndex.seedDB.mySeed(), yacyNewsPool.CATEGORY_SURFTIPP_VOTE_ADD, map));
}
// create Supporter
HashMap<String, Integer> negativeHashes = new HashMap<String, Integer>(); // a mapping from an url hash to Integer (count of votes)
HashMap<String, Integer> positiveHashes = new HashMap<String, Integer>(); // a mapping from an url hash to Integer (count of votes)
accumulateVotes(negativeHashes, positiveHashes, yacyNewsPool.INCOMING_DB);
accumulateVotes(sb, negativeHashes, positiveHashes, yacyNewsPool.INCOMING_DB);
//accumulateVotes(negativeHashes, positiveHashes, yacyNewsPool.OUTGOING_DB);
//accumulateVotes(negativeHashes, positiveHashes, yacyNewsPool.PUBLISHED_DB);
kelondroMScoreCluster<String> ranking = new kelondroMScoreCluster<String>(); // score cluster for url hashes
kelondroRow rowdef = new kelondroRow("String url-255, String title-120, String description-120, String refid-" + (serverDate.PATTERN_SHORT_SECOND.length() + 12), kelondroNaturalOrder.naturalOrder, 0);
HashMap<String, Entry> Supporter = new HashMap<String, Entry>(); // a mapping from an url hash to a kelondroRow.Entry with display properties
accumulateSupporter(Supporter, ranking, rowdef, negativeHashes, positiveHashes, yacyNewsPool.INCOMING_DB);
accumulateSupporter(sb, Supporter, ranking, rowdef, negativeHashes, positiveHashes, yacyNewsPool.INCOMING_DB);
//accumulateSupporter(Supporter, ranking, rowdef, negativeHashes, positiveHashes, yacyNewsPool.OUTGOING_DB);
//accumulateSupporter(Supporter, ranking, rowdef, negativeHashes, positiveHashes, yacyNewsPool.PUBLISHED_DB);
@ -132,8 +131,8 @@ public class Supporter {
description = row.getColString(2,"UTF-8");
if ((url == null) || (title == null) || (description == null)) continue;
refid = row.getColString(3, null);
voted = (yacyCore.newsPool.getSpecific(yacyNewsPool.OUTGOING_DB, yacyNewsPool.CATEGORY_SURFTIPP_VOTE_ADD, "refid", refid) != null) ||
(yacyCore.newsPool.getSpecific(yacyNewsPool.PUBLISHED_DB, yacyNewsPool.CATEGORY_SURFTIPP_VOTE_ADD, "refid", refid) != null);
voted = (sb.wordIndex.newsPool.getSpecific(yacyNewsPool.OUTGOING_DB, yacyNewsPool.CATEGORY_SURFTIPP_VOTE_ADD, "refid", refid) != null) ||
(sb.wordIndex.newsPool.getSpecific(yacyNewsPool.PUBLISHED_DB, yacyNewsPool.CATEGORY_SURFTIPP_VOTE_ADD, "refid", refid) != null);
prop.put("supporter_results_" + i + "_authorized", authenticated ? "1" : "0");
prop.put("supporter_results_" + i + "_authorized_recommend", voted ? "0" : "1");
@ -168,10 +167,10 @@ public class Supporter {
return (int) Math.max(0, 10 - ((System.currentTimeMillis() - created.getTime()) / 24 / 60 / 60 / 1000));
}
private static void accumulateVotes(HashMap<String, Integer> negativeHashes, HashMap<String, Integer> positiveHashes, int dbtype) {
int maxCount = Math.min(1000, yacyCore.newsPool.size(dbtype));
private static void accumulateVotes(plasmaSwitchboard sb, HashMap<String, Integer> negativeHashes, HashMap<String, Integer> positiveHashes, int dbtype) {
int maxCount = Math.min(1000, sb.wordIndex.newsPool.size(dbtype));
yacyNewsRecord record;
Iterator<yacyNewsRecord> recordIterator = yacyCore.newsPool.recordIterator(dbtype, true);
Iterator<yacyNewsRecord> recordIterator = sb.wordIndex.newsPool.recordIterator(dbtype, true);
int j = 0;
while ((recordIterator.hasNext()) && (j++ < maxCount)) {
record = recordIterator.next();
@ -196,11 +195,12 @@ public class Supporter {
}
private static void accumulateSupporter(
plasmaSwitchboard sb,
HashMap<String, Entry> Supporter, kelondroMScoreCluster<String> ranking, kelondroRow rowdef,
HashMap<String, Integer> negativeHashes, HashMap<String, Integer> positiveHashes, int dbtype) {
int maxCount = Math.min(1000, yacyCore.newsPool.size(dbtype));
int maxCount = Math.min(1000, sb.wordIndex.newsPool.size(dbtype));
yacyNewsRecord record;
Iterator<yacyNewsRecord> recordIterator = yacyCore.newsPool.recordIterator(dbtype, true);
Iterator<yacyNewsRecord> recordIterator = sb.wordIndex.newsPool.recordIterator(dbtype, true);
int j = 0;
String url = "", urlhash;
kelondroRow.Entry entry;
@ -213,7 +213,7 @@ public class Supporter {
entry = null;
if ((record.category().equals(yacyNewsPool.CATEGORY_PROFILE_UPDATE)) &&
((seed = yacyCore.seedDB.getConnected(record.originator())) != null)) try {
((seed = sb.wordIndex.seedDB.getConnected(record.originator())) != null)) try {
url = record.attribute("homepage", "");
if (url.length() < 12) continue;
entry = rowdef.newEntry(new byte[][]{
@ -226,7 +226,7 @@ public class Supporter {
} catch (IOException e) {}
if ((record.category().equals(yacyNewsPool.CATEGORY_PROFILE_BROADCAST)) &&
((seed = yacyCore.seedDB.getConnected(record.originator())) != null)) try {
((seed = sb.wordIndex.seedDB.getConnected(record.originator())) != null)) try {
url = record.attribute("homepage", "");
if (url.length() < 12) continue;
entry = rowdef.newEntry(new byte[][]{

32
htroot/Surftips.java
View File

@ -43,7 +43,6 @@ import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch;
import de.anomic.tools.crypt;
import de.anomic.tools.nxTools;
import de.anomic.yacy.yacyCore;
import de.anomic.yacy.yacyNewsPool;
import de.anomic.yacy.yacyNewsRecord;
import de.anomic.yacy.yacySeed;
@ -87,7 +86,7 @@ public class Surftips {
map.put("urlhash", hash);
map.put("vote", "negative");
map.put("refid", post.get("refid", ""));
yacyCore.newsPool.publishMyNews(yacyNewsRecord.newRecord(yacyNewsPool.CATEGORY_SURFTIPP_VOTE_ADD, map));
sb.wordIndex.newsPool.publishMyNews(yacyNewsRecord.newRecord(sb.wordIndex.seedDB.mySeed(), yacyNewsPool.CATEGORY_SURFTIPP_VOTE_ADD, map));
}
if ((post != null) && ((hash = post.get("votePositive", null)) != null)) {
if (!sb.verifyAuthentication(header, false)) {
@ -103,19 +102,19 @@ public class Surftips {
map.put("vote", "positive");
map.put("refid", post.get("refid", ""));
map.put("comment", post.get("comment", ""));
yacyCore.newsPool.publishMyNews(new yacyNewsRecord(yacyNewsPool.CATEGORY_SURFTIPP_VOTE_ADD, map));
sb.wordIndex.newsPool.publishMyNews(new yacyNewsRecord(sb.wordIndex.seedDB.mySeed(), yacyNewsPool.CATEGORY_SURFTIPP_VOTE_ADD, map));
}
// create surftips
HashMap<String, Integer> negativeHashes = new HashMap<String, Integer>(); // a mapping from an url hash to Integer (count of votes)
HashMap<String, Integer> positiveHashes = new HashMap<String, Integer>(); // a mapping from an url hash to Integer (count of votes)
accumulateVotes(negativeHashes, positiveHashes, yacyNewsPool.INCOMING_DB);
accumulateVotes(sb , negativeHashes, positiveHashes, yacyNewsPool.INCOMING_DB);
//accumulateVotes(negativeHashes, positiveHashes, yacyNewsPool.OUTGOING_DB);
//accumulateVotes(negativeHashes, positiveHashes, yacyNewsPool.PUBLISHED_DB);
kelondroMScoreCluster<String> ranking = new kelondroMScoreCluster<String>(); // score cluster for url hashes
kelondroRow rowdef = new kelondroRow("String url-255, String title-120, String description-120, String refid-" + (serverDate.PATTERN_SHORT_SECOND.length() + 12), kelondroNaturalOrder.naturalOrder, 0);
HashMap<String, Entry> surftips = new HashMap<String, Entry>(); // a mapping from an url hash to a kelondroRow.Entry with display properties
accumulateSurftips(surftips, ranking, rowdef, negativeHashes, positiveHashes, yacyNewsPool.INCOMING_DB);
accumulateSurftips(sb, surftips, ranking, rowdef, negativeHashes, positiveHashes, yacyNewsPool.INCOMING_DB);
//accumulateSurftips(surftips, ranking, rowdef, negativeHashes, positiveHashes, yacyNewsPool.OUTGOING_DB);
//accumulateSurftips(surftips, ranking, rowdef, negativeHashes, positiveHashes, yacyNewsPool.PUBLISHED_DB);
@ -141,8 +140,8 @@ public class Surftips {
description = row.getColString(2,"UTF-8");
if ((url == null) || (title == null) || (description == null)) continue;
refid = row.getColString(3, null);
voted = (yacyCore.newsPool.getSpecific(yacyNewsPool.OUTGOING_DB, yacyNewsPool.CATEGORY_SURFTIPP_VOTE_ADD, "refid", refid) != null) ||
(yacyCore.newsPool.getSpecific(yacyNewsPool.PUBLISHED_DB, yacyNewsPool.CATEGORY_SURFTIPP_VOTE_ADD, "refid", refid) != null);
voted = (sb.wordIndex.newsPool.getSpecific(yacyNewsPool.OUTGOING_DB, yacyNewsPool.CATEGORY_SURFTIPP_VOTE_ADD, "refid", refid) != null) ||
(sb.wordIndex.newsPool.getSpecific(yacyNewsPool.PUBLISHED_DB, yacyNewsPool.CATEGORY_SURFTIPP_VOTE_ADD, "refid", refid) != null);
prop.put("surftips_results_" + i + "_authorized", (authenticated) ? "1" : "0");
prop.put("surftips_results_" + i + "_authorized_recommend", (voted) ? "0" : "1");
@ -177,10 +176,10 @@ public class Surftips {
return (int) Math.max(0, 10 - ((System.currentTimeMillis() - created.getTime()) / 24 / 60 / 60 / 1000));
}
private static void accumulateVotes(HashMap<String, Integer> negativeHashes, HashMap<String, Integer> positiveHashes, int dbtype) {
int maxCount = Math.min(1000, yacyCore.newsPool.size(dbtype));
private static void accumulateVotes(plasmaSwitchboard sb, HashMap<String, Integer> negativeHashes, HashMap<String, Integer> positiveHashes, int dbtype) {
int maxCount = Math.min(1000, sb.wordIndex.newsPool.size(dbtype));
yacyNewsRecord record;
Iterator<yacyNewsRecord> recordIterator = yacyCore.newsPool.recordIterator(dbtype, true);
Iterator<yacyNewsRecord> recordIterator = sb.wordIndex.newsPool.recordIterator(dbtype, true);
int j = 0;
while ((recordIterator.hasNext()) && (j++ < maxCount)) {
record = recordIterator.next();
@ -205,11 +204,12 @@ public class Surftips {
}
private static void accumulateSurftips(
plasmaSwitchboard sb,
HashMap<String, Entry> surftips, kelondroMScoreCluster<String> ranking, kelondroRow rowdef,
HashMap<String, Integer> negativeHashes, HashMap<String, Integer> positiveHashes, int dbtype) {
int maxCount = Math.min(1000, yacyCore.newsPool.size(dbtype));
int maxCount = Math.min(1000, sb.wordIndex.newsPool.size(dbtype));
yacyNewsRecord record;
Iterator<yacyNewsRecord> recordIterator = yacyCore.newsPool.recordIterator(dbtype, true);
Iterator<yacyNewsRecord> recordIterator = sb.wordIndex.newsPool.recordIterator(dbtype, true);
int j = 0;
String url = "", urlhash;
kelondroRow.Entry entry;
@ -271,8 +271,8 @@ public class Surftips {
} catch (IOException e) {}
if (record.category().equals(yacyNewsPool.CATEGORY_WIKI_UPDATE)) try {
yacySeed seed = yacyCore.seedDB.getConnected(record.originator());
if (seed == null) seed = yacyCore.seedDB.getDisconnected(record.originator());
yacySeed seed = sb.wordIndex.seedDB.getConnected(record.originator());
if (seed == null) seed = sb.wordIndex.seedDB.getDisconnected(record.originator());
if (seed != null) {
url = "http://" + seed.getPublicAddress() + "/Wiki.html?page=" + record.attribute("page", "");
entry = rowdef.newEntry(new byte[][]{
@ -286,8 +286,8 @@ public class Surftips {
} catch (IOException e) {}
if (record.category().equals(yacyNewsPool.CATEGORY_BLOG_ADD)) try {
yacySeed seed = yacyCore.seedDB.getConnected(record.originator());
if (seed == null) seed = yacyCore.seedDB.getDisconnected(record.originator());
yacySeed seed = sb.wordIndex.seedDB.getConnected(record.originator());
if (seed == null) seed = sb.wordIndex.seedDB.getDisconnected(record.originator());
if (seed != null) {
url = "http://" + seed.getPublicAddress() + "/Blog.html?page=" + record.attribute("page", "");
entry = rowdef.newEntry(new byte[][]{

23
htroot/ViewProfile.java
View File

@ -70,15 +70,14 @@ public class ViewProfile {
@SuppressWarnings("unchecked")
public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch env) {
// listManager.switchboard = (plasmaSwitchboard) env;
serverObjects prop = new serverObjects();
plasmaSwitchboard switchboard = (plasmaSwitchboard) env;
boolean authenticated = switchboard.adminAuthenticated(header) >= 2;
plasmaSwitchboard sb = (plasmaSwitchboard) env;
boolean authenticated = sb.adminAuthenticated(header) >= 2;
int display = ((post == null) || (!authenticated)) ? 0 : post.getInt("display", 0);
prop.put("display", display);
String hash = (post == null) ? null : (String) post.get("hash");
if ((hash == null) || (yacyCore.seedDB == null)) {
if ((hash == null) || (sb.wordIndex.seedDB == null)) {
// wrong access
prop.put("success", "0");
return prop;
@ -102,24 +101,24 @@ public class ViewProfile {
profile.putAll(p);
prop.put("success", "3"); // everything ok
prop.put("localremotepeer", "0");
prop.putHTML("success_peername", yacyCore.seedDB.mySeed().getName());
prop.put("success_peerhash", yacyCore.seedDB.mySeed().hash);
address = yacyCore.seedDB.mySeed().getPublicAddress();
prop.putHTML("success_peername", sb.wordIndex.seedDB.mySeed().getName());
prop.put("success_peerhash", sb.wordIndex.seedDB.mySeed().hash);
address = sb.wordIndex.seedDB.mySeed().getPublicAddress();
} else {
// read the profile from remote peer
yacySeed seed = yacyCore.seedDB.getConnected(hash);
if (seed == null) seed = yacyCore.seedDB.getDisconnected(hash);
yacySeed seed = sb.wordIndex.seedDB.getConnected(hash);
if (seed == null) seed = sb.wordIndex.seedDB.getDisconnected(hash);
if (seed == null) {
prop.put("success", "1"); // peer unknown
} else {
// process news if existent
try {
yacyNewsRecord record = yacyCore.newsPool.getByOriginator(yacyNewsPool.INCOMING_DB, yacyNewsPool.CATEGORY_PROFILE_UPDATE, seed.hash);
if (record != null) yacyCore.newsPool.moveOff(yacyNewsPool.INCOMING_DB, record.id());
yacyNewsRecord record = sb.wordIndex.newsPool.getByOriginator(yacyNewsPool.INCOMING_DB, yacyNewsPool.CATEGORY_PROFILE_UPDATE, seed.hash);
if (record != null) sb.wordIndex.newsPool.moveOff(yacyNewsPool.INCOMING_DB, record.id());
} catch (IOException e) {}
// try to get the profile from remote peer
if (switchboard.clusterhashes != null) seed.setAlternativeAddress((String) switchboard.clusterhashes.get(seed.hash));
if (sb.clusterhashes != null) seed.setAlternativeAddress((String) sb.clusterhashes.get(seed.hash));
profile = yacyClient.getProfile(seed);
// if profile did not arrive, say that peer is disconnected

71
htroot/WatchCrawler_p.java
View File

@ -38,6 +38,7 @@ import java.util.regex.PatternSyntaxException;
import de.anomic.htmlFilter.htmlFilterContentScraper;
import de.anomic.htmlFilter.htmlFilterWriter;
import de.anomic.http.httpHeader;
import de.anomic.plasma.plasmaCrawlEntry;
import de.anomic.plasma.plasmaCrawlProfile;
import de.anomic.plasma.plasmaCrawlZURL;
import de.anomic.plasma.plasmaSwitchboard;
@ -45,7 +46,6 @@ import de.anomic.plasma.dbImport.SitemapImporter;
import de.anomic.server.serverFileUtils;
import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch;
import de.anomic.yacy.yacyCore;
import de.anomic.yacy.yacyNewsPool;
import de.anomic.yacy.yacyNewsRecord;
import de.anomic.yacy.yacyURL;
@ -61,7 +61,7 @@ public class WatchCrawler_p {
public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch<?> env) {
// return variable that accumulates replacements
plasmaSwitchboard switchboard = (plasmaSwitchboard) env;
plasmaSwitchboard sb = (plasmaSwitchboard) env;
serverObjects prop = new serverObjects();
prop.put("forwardToCrawlStart", "0");
@ -72,9 +72,9 @@ public class WatchCrawler_p {
prop.put("info", "0");
if ((post.containsKey("autoforward")) &&
(switchboard.crawlQueues.coreCrawlJobSize() == 0) &&
(switchboard.crawlQueues.remoteTriggeredCrawlJobSize() == 0) &&
(switchboard.queueSize() < 30)) {
(sb.crawlQueues.coreCrawlJobSize() == 0) &&
(sb.crawlQueues.remoteTriggeredCrawlJobSize() == 0) &&
(sb.queueSize() < 30)) {
prop.put("forwardToCrawlStart", "1");
}
@ -82,9 +82,9 @@ public class WatchCrawler_p {
// continue queue
String queue = post.get("continue", "");
if (queue.equals("localcrawler")) {
switchboard.continueCrawlJob(plasmaSwitchboard.CRAWLJOB_LOCAL_CRAWL);
sb.continueCrawlJob(plasmaSwitchboard.CRAWLJOB_LOCAL_CRAWL);
} else if (queue.equals("remotecrawler")) {
switchboard.continueCrawlJob(plasmaSwitchboard.CRAWLJOB_REMOTE_TRIGGERED_CRAWL);
sb.continueCrawlJob(plasmaSwitchboard.CRAWLJOB_REMOTE_TRIGGERED_CRAWL);
}
}
@ -92,15 +92,15 @@ public class WatchCrawler_p {
// pause queue
String queue = post.get("pause", "");
if (queue.equals("localcrawler")) {
switchboard.pauseCrawlJob(plasmaSwitchboard.CRAWLJOB_LOCAL_CRAWL);
sb.pauseCrawlJob(plasmaSwitchboard.CRAWLJOB_LOCAL_CRAWL);
} else if (queue.equals("remotecrawler")) {
switchboard.pauseCrawlJob(plasmaSwitchboard.CRAWLJOB_REMOTE_TRIGGERED_CRAWL);
sb.pauseCrawlJob(plasmaSwitchboard.CRAWLJOB_REMOTE_TRIGGERED_CRAWL);
}
}
if (post.containsKey("crawlingstart")) {
// init crawl
if (yacyCore.seedDB == null) {
if (sb.wordIndex.seedDB == null) {
prop.put("info", "3");
} else {
// set new properties
@ -185,20 +185,20 @@ public class WatchCrawler_p {
// first delete old entry, if exists
yacyURL url = new yacyURL(crawlingStart, null);
String urlhash = url.hash();
switchboard.wordIndex.removeURL(urlhash);
switchboard.crawlQueues.noticeURL.removeByURLHash(urlhash);
switchboard.crawlQueues.errorURL.remove(urlhash);
sb.wordIndex.removeURL(urlhash);
sb.crawlQueues.noticeURL.removeByURLHash(urlhash);
sb.crawlQueues.errorURL.remove(urlhash);
// stack url
switchboard.profilesPassiveCrawls.removeEntry(crawlingStartURL.hash()); // if there is an old entry, delete it
plasmaCrawlProfile.entry pe = switchboard.profilesActiveCrawls.newEntry(
sb.profilesPassiveCrawls.removeEntry(crawlingStartURL.hash()); // if there is an old entry, delete it
plasmaCrawlProfile.entry pe = sb.profilesActiveCrawls.newEntry(
crawlingStartURL.getHost(), crawlingStartURL, newcrawlingfilter, newcrawlingfilter,
newcrawlingdepth, newcrawlingdepth,
crawlingIfOlder, crawlingDomFilterDepth, crawlingDomMaxPages,
crawlingQ,
indexText, indexMedia,
storeHTCache, true, crawlOrder, xsstopw, xdstopw, xpstopw);
String reasonString = switchboard.crawlStacker.stackCrawl(url, null, yacyCore.seedDB.mySeed().hash, "CRAWLING-ROOT", new Date(), 0, pe);
String reasonString = sb.crawlStacker.stackCrawl(url, null, sb.wordIndex.seedDB.mySeed().hash, "CRAWLING-ROOT", new Date(), 0, pe);
if (reasonString == null) {
// liftoff!
@ -220,7 +220,7 @@ public class WatchCrawler_p {
m.remove("generalFilter");
m.remove("specificFilter");
m.put("intention", post.get("intention", "").replace(',', '/'));
yacyCore.newsPool.publishMyNews(yacyNewsRecord.newRecord(yacyNewsPool.CATEGORY_CRAWL_START, m));
sb.wordIndex.newsPool.publishMyNews(yacyNewsRecord.newRecord(sb.wordIndex.seedDB.mySeed(), yacyNewsPool.CATEGORY_CRAWL_START, m));
}
} else {
@ -228,9 +228,24 @@ public class WatchCrawler_p {
prop.putHTML("info_crawlingURL", ((String) post.get("crawlingURL")));
prop.putHTML("info_reasonString", reasonString);
plasmaCrawlZURL.Entry ee = switchboard.crawlQueues.errorURL.newEntry(crawlingStartURL, reasonString);
plasmaCrawlZURL.Entry ee = sb.crawlQueues.errorURL.newEntry(
new plasmaCrawlEntry(
sb.wordIndex.seedDB.mySeed().hash,
crawlingStartURL,
"",
"",
new Date(),
pe.handle(),
0,
0,
0),
sb.wordIndex.seedDB.mySeed().hash,
new Date(),
1,
reasonString);
ee.store();
switchboard.crawlQueues.errorURL.push(ee);
sb.crawlQueues.errorURL.push(ee);
}
} catch (PatternSyntaxException e) {
prop.put("info", "4"); //crawlfilter does not match url
@ -270,10 +285,10 @@ public class WatchCrawler_p {
// creating a crawler profile
yacyURL crawlURL = new yacyURL("file://" + file.toString(), null);
plasmaCrawlProfile.entry profile = switchboard.profilesActiveCrawls.newEntry(fileName, crawlURL, newcrawlingfilter, newcrawlingfilter, newcrawlingdepth, newcrawlingdepth, crawlingIfOlder, crawlingDomFilterDepth, crawlingDomMaxPages, crawlingQ, indexText, indexMedia, storeHTCache, true, crawlOrder, xsstopw, xdstopw, xpstopw);
plasmaCrawlProfile.entry profile = sb.profilesActiveCrawls.newEntry(fileName, crawlURL, newcrawlingfilter, newcrawlingfilter, newcrawlingdepth, newcrawlingdepth, crawlingIfOlder, crawlingDomFilterDepth, crawlingDomMaxPages, crawlingQ, indexText, indexMedia, storeHTCache, true, crawlOrder, xsstopw, xdstopw, xpstopw);
// pause local crawl here
switchboard.pauseCrawlJob(plasmaSwitchboard.CRAWLJOB_LOCAL_CRAWL);
sb.pauseCrawlJob(plasmaSwitchboard.CRAWLJOB_LOCAL_CRAWL);
// loop through the contained links
Iterator<Map.Entry<yacyURL, String>> linkiterator = hyperlinks.entrySet().iterator();
@ -284,10 +299,10 @@ public class WatchCrawler_p {
if (nexturl == null) continue;
// enqueuing the url for crawling
switchboard.crawlStacker.enqueueEntry(
sb.crawlStacker.enqueueEntry(
nexturl,
null,
yacyCore.seedDB.mySeed().hash,
sb.wordIndex.seedDB.mySeed().hash,
(String) e.getValue(),
new Date(),
0,
@ -306,7 +321,7 @@ public class WatchCrawler_p {
prop.putHTML("info_error", e.getMessage());
e.printStackTrace();
}
switchboard.continueCrawlJob(plasmaSwitchboard.CRAWLJOB_LOCAL_CRAWL);
sb.continueCrawlJob(plasmaSwitchboard.CRAWLJOB_LOCAL_CRAWL);
}
} else if (crawlingMode.equals(CRAWLING_MODE_SITEMAP)) {
String sitemapURLStr = null;
@ -316,7 +331,7 @@ public class WatchCrawler_p {
yacyURL sitemapURL = new yacyURL(sitemapURLStr, null);
// create a new profile
plasmaCrawlProfile.entry pe = switchboard.profilesActiveCrawls.newEntry(
plasmaCrawlProfile.entry pe = sb.profilesActiveCrawls.newEntry(
sitemapURLStr, sitemapURL, newcrawlingfilter, newcrawlingfilter,
newcrawlingdepth, newcrawlingdepth,
crawlingIfOlder, crawlingDomFilterDepth, crawlingDomMaxPages,
@ -325,9 +340,9 @@ public class WatchCrawler_p {
storeHTCache, true, crawlOrder, xsstopw, xdstopw, xpstopw);
// create a new sitemap importer
SitemapImporter importerThread = (SitemapImporter) switchboard.dbImportManager.getNewImporter("sitemap");
SitemapImporter importerThread = (SitemapImporter) sb.dbImportManager.getNewImporter("sitemap");
if (importerThread != null) {
importerThread.init(switchboard, 0);
importerThread.init(sb, 0);
importerThread.initSitemap(new yacyURL(sitemapURLStr, null), pe);
importerThread.startIt();
}
@ -343,7 +358,7 @@ public class WatchCrawler_p {
}
if (post.containsKey("crawlingPerformance")) {
setPerformance(switchboard, post);
setPerformance(sb, post);
}
}

37
htroot/Wiki.java
View File

@ -59,7 +59,6 @@ import de.anomic.http.httpHeader;
import de.anomic.plasma.plasmaSwitchboard;
import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch;
import de.anomic.yacy.yacyCore;
import de.anomic.yacy.yacyNewsPool;
import de.anomic.yacy.yacyNewsRecord;
@ -75,48 +74,48 @@ public class Wiki {
public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch<?> env) throws IOException {
plasmaSwitchboard switchboard = (plasmaSwitchboard) env;
plasmaSwitchboard sb = (plasmaSwitchboard) env;
serverObjects prop = new serverObjects();
if (post == null) {
post = new serverObjects();
post.put("page", "start");
}
final boolean authenticated = switchboard.adminAuthenticated(header) >= 2;
final boolean authenticated = sb.adminAuthenticated(header) >= 2;
final int display = ((post == null) || (!authenticated)) ? 0 : post.getInt("display", 0);
prop.put("display", display);
String access = switchboard.getConfig("WikiAccess", "admin");
String access = sb.getConfig("WikiAccess", "admin");
String pagename = post.get("page", "start");
String ip = post.get(httpHeader.CONNECTION_PROP_CLIENTIP, "127.0.0.1");
String author = post.get("author", "anonymous");
if (author.equals("anonymous")) {
author = wikiBoard.guessAuthor(ip);
if (author == null) {
if (de.anomic.yacy.yacyCore.seedDB.mySeed() == null) author = "anonymous";
else author = de.anomic.yacy.yacyCore.seedDB.mySeed().get("Name", "anonymous");
if (sb.wordIndex.seedDB.mySeed() == null) author = "anonymous";
else author = sb.wordIndex.seedDB.mySeed().get("Name", "anonymous");
}
}
if (post.containsKey("access")) {
// only the administrator may change the access right
if (!switchboard.verifyAuthentication(header, true)) {
if (!sb.verifyAuthentication(header, true)) {
// check access right for admin
prop.put("AUTHENTICATE", "admin log-in"); // force log-in
return prop;
}
access = post.get("access", "admin");
switchboard.setConfig("WikiAccess", access);
sb.setConfig("WikiAccess", access);
}
if (access.equals("admin")) prop.put("mode_access", "0");
if (access.equals("all")) prop.put("mode_access", "1");
wikiBoard.entry page = switchboard.wikiDB.read(pagename);
wikiBoard.entry page = sb.wikiDB.read(pagename);
if (post.containsKey("submit")) {
if ((access.equals("admin") && (!switchboard.verifyAuthentication(header, true)))) {
if ((access.equals("admin") && (!sb.verifyAuthentication(header, true)))) {
// check access right for admin
prop.put("AUTHENTICATE", "admin log-in"); // force log-in
return prop;
@ -129,20 +128,20 @@ public class Wiki {
} catch (UnsupportedEncodingException e) {
content = post.get("content", "").getBytes();
}
wikiBoard.entry newEntry = switchboard.wikiDB.newEntry(pagename, author, ip, post.get("reason", "edit"), content);
switchboard.wikiDB.write(newEntry);
wikiBoard.entry newEntry = sb.wikiDB.newEntry(pagename, author, ip, post.get("reason", "edit"), content);
sb.wikiDB.write(newEntry);
// create a news message
HashMap<String, String> map = new HashMap<String, String>();
map.put("page", pagename);
map.put("author", author.replace(',', ' '));
if (post.get("content", "").trim().length() > 0 && !page.page().equals(content))
yacyCore.newsPool.publishMyNews(yacyNewsRecord.newRecord(yacyNewsPool.CATEGORY_WIKI_UPDATE, map));
sb.wordIndex.newsPool.publishMyNews(yacyNewsRecord.newRecord(sb.wordIndex.seedDB.mySeed(), yacyNewsPool.CATEGORY_WIKI_UPDATE, map));
page = newEntry;
prop.put("LOCATION", "/Wiki.html?page=" + pagename);
}
if (post.containsKey("edit")) {
if ((access.equals("admin") && (!switchboard.verifyAuthentication(header, true)))) {
if ((access.equals("admin") && (!sb.verifyAuthentication(header, true)))) {
// check access right for admin
prop.put("AUTHENTICATE", "admin log-in"); // force log-in
return prop;
@ -174,12 +173,12 @@ public class Wiki {
prop.put("mode", "3"); //Index
String subject;
try {
Iterator<String> i = switchboard.wikiDB.keys(true);
Iterator<String> i = sb.wikiDB.keys(true);
wikiBoard.entry entry;
int count=0;
while (i.hasNext()) {
subject = i.next();
entry = switchboard.wikiDB.read(subject);
entry = sb.wikiDB.read(subject);
prop.putHTML("mode_pages_"+count+"_name",wikiBoard.webalize(subject));
prop.putHTML("mode_pages_"+count+"_subject", subject);
prop.put("mode_pages_"+count+"_date", dateString(entry.date()));
@ -201,14 +200,14 @@ public class Wiki {
prop.putHTML("mode_error_page", pagename);
try {
Iterator<String> it = switchboard.wikiDB.keysBkp(true);
Iterator<String> it = sb.wikiDB.keysBkp(true);
wikiBoard.entry entry;
wikiBoard.entry oentry = null;
wikiBoard.entry nentry = null;
int count = 0;
boolean oldselected = false, newselected = false;
while (it.hasNext()) {
entry = switchboard.wikiDB.readBkp(it.next());
entry = sb.wikiDB.readBkp(it.next());
prop.put("mode_error_versions_" + count + "_date", wikiBoard.dateString(entry.date()));
prop.put("mode_error_versions_" + count + "_fdate", dateString(entry.date()));
if (wikiBoard.dateString(entry.date()).equals(post.get("old", null))) {
@ -235,7 +234,7 @@ public class Wiki {
prop.put("mode_error_versions", count);
}
entry = switchboard.wikiDB.read(pagename);
entry = sb.wikiDB.read(pagename);
if (entry != null) {
prop.put("mode_error_curdate", wikiBoard.dateString(entry.date()));
prop.put("mode_error_curfdate", dateString(entry.date()));

6
htroot/YaCySearchPluginFF.java
View File

@ -44,9 +44,9 @@
//if the shell's current path is HTROOT
import de.anomic.http.httpHeader;
import de.anomic.plasma.plasmaSwitchboard;
import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch;
import de.anomic.yacy.yacyCore;
public class YaCySearchPluginFF {
@ -57,7 +57,7 @@ public class YaCySearchPluginFF {
* @return the rewrite-properties for the template
*/
public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch<?> env) {
plasmaSwitchboard sb = (plasmaSwitchboard) env;
serverObjects prop = new serverObjects();
// getting the http host header
@ -72,7 +72,7 @@ public class YaCySearchPluginFF {
prop.put("host", host);
prop.put("port", port);
prop.putHTML("name", yacyCore.seedDB.mySeed().getName());
prop.putHTML("name", sb.wordIndex.seedDB.mySeed().getName());
return prop;
}

3
htroot/index.java
View File

@ -34,7 +34,6 @@ import de.anomic.plasma.plasmaSearchQuery;
import de.anomic.plasma.plasmaSwitchboard;
import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch;
import de.anomic.yacy.yacyCore;
public class index {
@ -80,7 +79,7 @@ public class index {
if (cds.equals("app")) contentdom = plasmaSearchQuery.CONTENTDOM_APP;
//long mylinks = 0;
prop.putNum("links", yacyCore.seedDB.mySeed().getLinkCount());
prop.putNum("links", sb.wordIndex.seedDB.mySeed().getLinkCount());
// we create empty entries for template strings
String promoteSearchPageGreeting = env.getConfig("promoteSearchPageGreeting", "");

6
htroot/opensearchdescription.java
View File

@ -25,15 +25,15 @@
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
import de.anomic.http.httpHeader;
import de.anomic.plasma.plasmaSwitchboard;
import de.anomic.server.serverCore;
import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch;
import de.anomic.yacy.yacyCore;
public class opensearchdescription {
public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch<?> env) {
plasmaSwitchboard sb = (plasmaSwitchboard) env;
// generate message content for open search description
String promoteSearchPageGreeting = env.getConfig("promoteSearchPageGreeting", "");
if (env.getConfigBool("promoteSearchPageGreeting.useNetworkName", false)) promoteSearchPageGreeting = env.getConfig("network.unit.description", "");
@ -45,7 +45,7 @@ public class opensearchdescription {
final serverObjects prop = new serverObjects();
prop.putHTML("thisaddress", thisaddress, true);
prop.putHTML("SearchPageGreeting", promoteSearchPageGreeting, true);
prop.putHTML("clientname", yacyCore.seedDB.mySeed().getName(), true);
prop.putHTML("clientname", sb.wordIndex.seedDB.mySeed().getName(), true);
// return rewrite properties
return prop;

10
htroot/rct_p.java
View File

@ -52,8 +52,8 @@ public class rct_p {
if (post != null) {
if (post.containsKey("retrieve")) {
String peerhash = post.get("peer", null);
yacySeed seed = (peerhash == null) ? null : yacyCore.seedDB.getConnected(peerhash);
RSSFeed feed = (seed == null) ? null : yacyClient.queryRemoteCrawlURLs(seed, 10);
yacySeed seed = (peerhash == null) ? null : sb.wordIndex.seedDB.getConnected(peerhash);
RSSFeed feed = (seed == null) ? null : yacyClient.queryRemoteCrawlURLs(sb.wordIndex.seedDB, seed, 10);
if (feed != null) {
for (RSSMessage item: feed) {
//System.out.println("URL=" + item.getLink() + ", desc=" + item.getDescription() + ", pubDate=" + item.getPubDate());
@ -95,17 +95,17 @@ public class rct_p {
}
}
listHosts(prop);
listHosts(sb, prop);
// return rewrite properties
return prop;
}
private static void listHosts(serverObjects prop) {
private static void listHosts(plasmaSwitchboard sb, serverObjects prop) {
// list known hosts
yacySeed seed;
int hc = 0;
if (yacyCore.seedDB != null && yacyCore.seedDB.sizeConnected() > 0) {
if (sb.wordIndex.seedDB != null && sb.wordIndex.seedDB.sizeConnected() > 0) {
Iterator<yacySeed> e = yacyCore.dhtAgent.getProvidesRemoteCrawlURLs();
while (e.hasNext()) {
seed = e.next();

6
htroot/sharedBlacklist_p.java
View File

@ -63,7 +63,6 @@ import de.anomic.plasma.plasmaSwitchboard;
import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch;
import de.anomic.tools.nxTools;
import de.anomic.yacy.yacyCore;
import de.anomic.yacy.yacySeed;
import de.anomic.yacy.yacyURL;
@ -77,6 +76,7 @@ public class sharedBlacklist_p {
public static final int STATUS_WRONG_INVOCATION = 5;
public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch<?> env) {
plasmaSwitchboard sb = (plasmaSwitchboard) env;
// return variable that accumulates replacements
serverObjects prop = new serverObjects();
@ -104,8 +104,8 @@ public class sharedBlacklist_p {
// generate the download URL
String downloadURL = null;
if( yacyCore.seedDB != null ){ //no nullpointer error..
yacySeed seed = yacyCore.seedDB.getConnected(Hash);
if( sb.wordIndex.seedDB != null ){ //no nullpointer error..
yacySeed seed = sb.wordIndex.seedDB.getConnected(Hash);
if (seed != null) {
String IP = seed.get(yacySeed.IP, "127.0.0.1");
String Port = seed.get(yacySeed.PORT, "8080");

7
htroot/www/welcome.java
View File

@ -52,6 +52,7 @@ import java.net.InetAddress;
import java.net.UnknownHostException;
import de.anomic.http.httpHeader;
import de.anomic.plasma.plasmaSwitchboard;
import de.anomic.server.serverCore;
import de.anomic.server.serverDomains;
import de.anomic.server.serverObjects;
@ -62,6 +63,8 @@ import de.anomic.yacy.yacySeed;
public class welcome {
public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch<?> env) {
plasmaSwitchboard sb = (plasmaSwitchboard) env;
// return variable that accumulates replacements
final serverObjects prop = new serverObjects();
@ -70,7 +73,7 @@ public class welcome {
prop.putHTML("peername", env.getConfig("peerName", "<nameless>"));
prop.putHTML("peerdomain", env.getConfig("peerName", "<nameless>").toLowerCase());
prop.putHTML("peeraddress", yacyCore.seedDB.mySeed().getPublicAddress());
prop.putHTML("peeraddress", sb.wordIndex.seedDB.mySeed().getPublicAddress());
prop.put("hostname", serverDomains.myPublicIP());
try{
prop.put("hostip", InetAddress.getByName(serverDomains.myPublicIP()).getHostAddress());
@ -80,7 +83,7 @@ public class welcome {
prop.put("port", serverCore.getPortNr(env.getConfig("port","8080")));
prop.put("clientip", (String) header.get(httpHeader.CONNECTION_PROP_CLIENTIP, ""));
final String peertype = (yacyCore.seedDB.mySeed() == null) ? yacySeed.PEERTYPE_JUNIOR : yacyCore.seedDB.mySeed().get(yacySeed.PEERTYPE, yacySeed.PEERTYPE_VIRGIN);
final String peertype = (sb.wordIndex.seedDB.mySeed() == null) ? yacySeed.PEERTYPE_JUNIOR : sb.wordIndex.seedDB.mySeed().get(yacySeed.PEERTYPE, yacySeed.PEERTYPE_VIRGIN);
final boolean senior = (peertype.equals(yacySeed.PEERTYPE_SENIOR)) || (peertype.equals(yacySeed.PEERTYPE_PRINCIPAL));
if (senior) { prop.put("couldcan", "can"); } else { prop.put("couldcan", "could"); }
if (senior) { prop.put("seniorinfo", "This peer runs in senior mode which means that your peer can be accessed using the addresses shown above."); } else { prop.putHTML("seniorinfo", "<b>Nobody can access your peer from the outside of your intranet. You must open your firewall and/or set a 'virtual server' in the settings of your router to enable access to the addresses as shown below.</b>"); }

13
htroot/xml/queues_p.java
View File

@ -60,7 +60,6 @@ import de.anomic.plasma.plasmaSwitchboard;
import de.anomic.plasma.plasmaSwitchboardQueue;
import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch;
import de.anomic.yacy.yacyCore;
import de.anomic.yacy.yacySeed;
public class queues_p {
@ -119,7 +118,7 @@ public class queues_p {
if ((pcentry != null) && (pcentry.url() != null)) {
long entrySize = pcentry.size();
totalSize += entrySize;
initiator = yacyCore.seedDB.getConnected(pcentry.initiator());
initiator = sb.wordIndex.seedDB.getConnected(pcentry.initiator());
prop.put("list-indexing_"+i+"_profile", (pcentry.profile() != null) ? pcentry.profile().name() : "deleted");
prop.put("list-indexing_"+i+"_initiator", ((initiator == null) ? "proxy" : initiator.getName()));
prop.put("list-indexing_"+i+"_depth", pcentry.depth());
@ -146,7 +145,7 @@ public class queues_p {
for (int i = 0; i < w.length; i++) {
if (w[i] == null) continue;
prop.put("list-loader_"+count+"_profile", w[i].profileHandle());
initiator = yacyCore.seedDB.getConnected(w[i].initiator());
initiator = sb.wordIndex.seedDB.getConnected(w[i].initiator());
prop.put("list-loader_"+count+"_initiator", ((initiator == null) ? "proxy" : initiator.getName()));
prop.put("list-loader_"+count+"_depth", w[i].depth());
prop.putHTML("list-loader_"+count+"_url", w[i].url().toString(), true);
@ -159,7 +158,7 @@ public class queues_p {
prop.putNum("localCrawlSize", Integer.toString(sb.getThread(plasmaSwitchboard.CRAWLJOB_LOCAL_CRAWL).getJobCount()));
prop.put("localCrawlState", sb.crawlJobIsPaused(plasmaSwitchboard.CRAWLJOB_LOCAL_CRAWL) ? STATE_PAUSED : STATE_RUNNING);
int stackSize = sb.crawlQueues.noticeURL.stackSize(plasmaCrawlNURL.STACK_TYPE_CORE);
addNTable(prop, "list-local", sb.crawlQueues.noticeURL.top(plasmaCrawlNURL.STACK_TYPE_CORE, Math.min(10, stackSize)));
addNTable(sb, prop, "list-local", sb.crawlQueues.noticeURL.top(plasmaCrawlNURL.STACK_TYPE_CORE, Math.min(10, stackSize)));
//global crawl queue
prop.putNum("limitCrawlSize", Integer.toString(sb.crawlQueues.limitCrawlJobSize()));
@ -174,7 +173,7 @@ public class queues_p {
if (stackSize == 0) {
prop.put("list-remote", "0");
} else {
addNTable(prop, "list-remote", sb.crawlQueues.noticeURL.top(plasmaCrawlNURL.STACK_TYPE_LIMIT, Math.min(10, stackSize)));
addNTable(sb, prop, "list-remote", sb.crawlQueues.noticeURL.top(plasmaCrawlNURL.STACK_TYPE_LIMIT, Math.min(10, stackSize)));
}
// return rewrite properties
@ -182,7 +181,7 @@ public class queues_p {
}
public static final void addNTable(serverObjects prop, String tableName, plasmaCrawlEntry[] crawlerList) {
public static final void addNTable(plasmaSwitchboard sb, serverObjects prop, String tableName, plasmaCrawlEntry[] crawlerList) {
int showNum = 0;
plasmaCrawlEntry urle;
@ -190,7 +189,7 @@ public class queues_p {
for (int i = 0; i < crawlerList.length; i++) {
urle = crawlerList[i];
if ((urle != null) && (urle.url() != null)) {
initiator = yacyCore.seedDB.getConnected(urle.initiator());
initiator = sb.wordIndex.seedDB.getConnected(urle.initiator());
prop.put(tableName + "_" + showNum + "_profile", urle.profileHandle());
prop.put(tableName + "_" + showNum + "_initiator", ((initiator == null) ? "proxy" : initiator.getName()));
prop.put(tableName + "_" + showNum + "_depth", urle.depth());

14
htroot/xml/status_p.java
View File

@ -54,19 +54,19 @@ public class status_p {
public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch<?> env) {
// return variable that accumulates replacements
plasmaSwitchboard switchboard = (plasmaSwitchboard) env;
plasmaSwitchboard sb = (plasmaSwitchboard) env;
serverObjects prop = new serverObjects();
if (post == null || !post.containsKey("html"))
prop.setLocalized(false);
prop.put("rejected", "0");
yacyCore.peerActions.updateMySeed();
final int cacheOutSize = switchboard.wordIndex.dhtOutCacheSize();
final long cacheMaxSize = switchboard.getConfigLong(plasmaSwitchboard.WORDCACHE_MAX_COUNT, 10000);
prop.putNum("ppm", yacyCore.seedDB.mySeed().getPPM());
prop.putNum("qpm", yacyCore.seedDB.mySeed().getQPM());
prop.putNum("wordCacheSize", switchboard.wordIndex.dhtOutCacheSize() + switchboard.wordIndex.dhtInCacheSize());
final int cacheOutSize = sb.wordIndex.dhtOutCacheSize();
final long cacheMaxSize = sb.getConfigLong(plasmaSwitchboard.WORDCACHE_MAX_COUNT, 10000);
prop.putNum("ppm", sb.wordIndex.seedDB.mySeed().getPPM());
prop.putNum("qpm", sb.wordIndex.seedDB.mySeed().getQPM());
prop.putNum("wordCacheSize", sb.wordIndex.dhtOutCacheSize() + sb.wordIndex.dhtInCacheSize());
prop.putNum("wordCacheWSize", cacheOutSize);
prop.putNum("wordCacheKSize", switchboard.wordIndex.dhtInCacheSize());
prop.putNum("wordCacheKSize", sb.wordIndex.dhtInCacheSize());
prop.putNum("wordCacheMaxSize", cacheMaxSize);
prop.put("wordCacheWCount", cacheOutSize);
prop.put("wordCacheMaxCount", cacheMaxSize);

30
htroot/yacy/crawlReceipt.java
View File

@ -56,7 +56,6 @@ import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch;
import de.anomic.server.logging.serverLog;
import de.anomic.tools.crypt;
import de.anomic.yacy.yacyCore;
import de.anomic.yacy.yacyNetwork;
import de.anomic.yacy.yacySeed;
@ -69,12 +68,12 @@ public final class crawlReceipt {
public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch<?> env) {
// return variable that accumulates replacements
plasmaSwitchboard switchboard = (plasmaSwitchboard) env;
plasmaSwitchboard sb = (plasmaSwitchboard) env;
serverObjects prop = new serverObjects();
if ((post == null) || (env == null)) return prop;
if (!yacyNetwork.authentifyRequest(post, env)) return prop;
serverLog log = switchboard.getLog();
serverLog log = sb.getLog();
//int proxyPrefetchDepth = Integer.parseInt(env.getConfig("proxyPrefetchDepth", "0"));
//int crawlingDepth = Integer.parseInt(env.getConfig("crawlingDepth", "0"));
@ -111,10 +110,10 @@ public final class crawlReceipt {
*/
final yacySeed otherPeer = yacyCore.seedDB.get(iam);
final yacySeed otherPeer = sb.wordIndex.seedDB.get(iam);
final String otherPeerName = iam + ":" + ((otherPeer == null) ? "NULL" : (otherPeer.getName() + "/" + otherPeer.getVersion()));
if ((yacyCore.seedDB.mySeed() == null) || (!(yacyCore.seedDB.mySeed().hash.equals(youare)))) {
if ((sb.wordIndex.seedDB.mySeed() == null) || (!(sb.wordIndex.seedDB.mySeed().hash.equals(youare)))) {
// no yacy connection / unknown peers
prop.put("delay", "3600");
return prop;
@ -126,7 +125,7 @@ public final class crawlReceipt {
return prop;
}
if ((switchboard.isRobinsonMode()) && (!switchboard.isInMyCluster(otherPeer))) {
if ((sb.isRobinsonMode()) && (!sb.isInMyCluster(otherPeer))) {
// we reject urls that are from outside our cluster
prop.put("delay", "9999");
}
@ -147,7 +146,7 @@ public final class crawlReceipt {
}
// check if the entry is in our network domain
String urlRejectReason = switchboard.acceptURL(comp.url());
String urlRejectReason = sb.acceptURL(comp.url());
if (urlRejectReason != null) {
log.logWarning("crawlReceipt: RECEIVED wrong RECEIPT (" + urlRejectReason + ") for hash " + entry.hash() + " from peer " + iam + "\n\tURL properties: "+ propStr);
prop.put("delay", "9999");
@ -156,9 +155,9 @@ public final class crawlReceipt {
if (result.equals("fill")) try {
// put new entry into database
switchboard.wordIndex.putURL(entry);
switchboard.crawlResults.stack(entry, youare, iam, 1);
switchboard.crawlQueues.delegatedURL.remove(entry.hash()); // the delegated work has been done
sb.wordIndex.putURL(entry);
sb.crawlResults.stack(entry, youare, iam, 1);
sb.crawlQueues.delegatedURL.remove(entry.hash()); // the delegated work has been done
log.logInfo("crawlReceipt: RECEIVED RECEIPT from " + otherPeerName + " for URL " + entry.hash() + ":" + comp.url().toNormalform(false, true));
// ready for more
@ -170,10 +169,15 @@ public final class crawlReceipt {
return prop;
}
switchboard.crawlQueues.delegatedURL.remove(entry.hash()); // the delegated work is transformed into an error case
plasmaCrawlZURL.Entry ee = switchboard.crawlQueues.errorURL.newEntry(entry.toBalancerEntry(), youare, null, 0, result + ":" + reason);
sb.crawlQueues.delegatedURL.remove(entry.hash()); // the delegated work is transformed into an error case
plasmaCrawlZURL.Entry ee = sb.crawlQueues.errorURL.newEntry(
entry.toBalancerEntry(iam),
youare,
null,
0,
result + ":" + reason);
ee.store();
switchboard.crawlQueues.errorURL.push(ee);
sb.crawlQueues.errorURL.push(ee);
//switchboard.noticeURL.remove(receivedUrlhash);
prop.put("delay", "3600");
return prop;

12
htroot/yacy/hello.java
View File

@ -144,7 +144,7 @@ public final class hello {
boolean isNotLocal = true;
// we are only allowed to connect to the client IP address if it's not our own address
if (serverCore.portForwardingEnabled || serverCore.useStaticIP) {
if (serverCore.useStaticIP) {
isNotLocal = !ias.isSiteLocalAddress();
}
if (isNotLocal) {
@ -194,15 +194,15 @@ public final class hello {
serverCore.checkInterruption();
final StringBuffer seeds = new StringBuffer(768);
// attach some more seeds, as requested
if ((yacyCore.seedDB != null) && (yacyCore.seedDB.sizeConnected() > 0)) {
if (count > yacyCore.seedDB.sizeConnected()) { count = yacyCore.seedDB.sizeConnected(); }
if ((sb.wordIndex.seedDB != null) && (sb.wordIndex.seedDB.sizeConnected() > 0)) {
if (count > sb.wordIndex.seedDB.sizeConnected()) { count = sb.wordIndex.seedDB.sizeConnected(); }
if (count > 100) { count = 100; }
// latest seeds
final Map<String, yacySeed> ySeeds = yacyCore.seedDB.seedsByAge(true, count); // peerhash/yacySeed relation
final Map<String, yacySeed> ySeeds = sb.wordIndex.seedDB.seedsByAge(true, count); // peerhash/yacySeed relation
// attach also my own seed
seeds.append("seed0=").append(yacyCore.seedDB.mySeed().genSeedStr(key)).append(serverCore.CRLF_STRING);
seeds.append("seed0=").append(sb.wordIndex.seedDB.mySeed().genSeedStr(key)).append(serverCore.CRLF_STRING);
count = 1;
// attach other seeds
@ -222,7 +222,7 @@ public final class hello {
}
} else {
// attach also my own seed
seeds.append("seed0=").append(yacyCore.seedDB.mySeed().genSeedStr(key)).append(serverCore.CRLF_STRING);
seeds.append("seed0=").append(sb.wordIndex.seedDB.mySeed().genSeedStr(key)).append(serverCore.CRLF_STRING);
}
prop.put("seedlist", seeds.toString());

3
htroot/yacy/list.java
View File

@ -34,7 +34,6 @@ import de.anomic.plasma.plasmaSwitchboard;
import de.anomic.server.serverCore;
import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch;
import de.anomic.yacy.yacyCore;
import de.anomic.yacy.yacyNetwork;
import de.anomic.yacy.yacySeed;
@ -55,7 +54,7 @@ public final class list {
String otherPeerName = null;
if (post.containsKey("iam")) {
yacySeed bla = yacyCore.seedDB.get(post.get("iam", ""));
yacySeed bla = sb.wordIndex.seedDB.get(post.get("iam", ""));
if (bla != null) otherPeerName = bla.getName();
}
if (otherPeerName == null) otherPeerName = (String)header.get(httpHeader.CONNECTION_PROP_CLIENTIP);

16
htroot/yacy/message.java
View File

@ -92,7 +92,7 @@ public final class message {
String youare = post.get("youare", ""); // seed hash of the target peer, needed for network stability
// check if we are the right target and requester has correct information about this peer
if ((yacyCore.seedDB.mySeed() == null) || (!(yacyCore.seedDB.mySeed().hash.equals(youare)))) {
if ((sb.wordIndex.seedDB.mySeed() == null) || (!(sb.wordIndex.seedDB.mySeed().hash.equals(youare)))) {
// this request has a wrong target
prop.put("response", "-1"); // request rejected
return prop;
@ -152,10 +152,10 @@ public final class message {
sb.messageDB.write(msgEntry = sb.messageDB.newEntry(
"remote",
otherSeed.get(yacySeed.NAME, "anonymous"), otherSeed.hash,
yacyCore.seedDB.mySeed().getName(), yacyCore.seedDB.mySeed().hash,
sb.wordIndex.seedDB.mySeed().getName(), sb.wordIndex.seedDB.mySeed().hash,
subject, mb));
messageForwardingViaEmail(env, msgEntry);
messageForwardingViaEmail(sb, msgEntry);
// finally write notification
File notifierSource = new File(sb.getRootPath(), sb.getConfig("htRootPath","htroot") + "/env/grafics/message.gif");
@ -181,15 +181,15 @@ public final class message {
#[message]#
*/
private static void messageForwardingViaEmail(serverSwitch<?> env, messageBoard.entry msgEntry) {
private static void messageForwardingViaEmail(plasmaSwitchboard sb, messageBoard.entry msgEntry) {
try {
if (!Boolean.valueOf(env.getConfig("msgForwardingEnabled","false")).booleanValue()) return;
if (!Boolean.valueOf(sb.getConfig("msgForwardingEnabled","false")).booleanValue()) return;
// getting the recipient address
String sendMailTo = env.getConfig("msgForwardingTo","root@localhost").trim();
String sendMailTo = sb.getConfig("msgForwardingTo","root@localhost").trim();
// getting the sendmail configuration
String sendMailStr = env.getConfig("msgForwardingCmd","/usr/bin/sendmail")+" "+sendMailTo;
String sendMailStr = sb.getConfig("msgForwardingCmd","/usr/bin/sendmail")+" "+sendMailTo;
String[] sendMail = sendMailStr.trim().split(" ");
// building the message text
@ -198,7 +198,7 @@ public final class message {
.append(sendMailTo)
.append("\nFrom: ")
.append("yacy@")
.append(yacyCore.seedDB.mySeed().getName())
.append(sb.wordIndex.seedDB.mySeed().getName())
.append("\nSubject: [YaCy] ")
.append(msgEntry.subject().replace('\n', ' '))
.append("\nDate: ")

3
htroot/yacy/query.java
View File

@ -51,7 +51,6 @@ import de.anomic.plasma.plasmaSwitchboard;
import de.anomic.server.serverDate;
import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch;
import de.anomic.yacy.yacyCore;
import de.anomic.yacy.yacyNetwork;
public final class query {
@ -88,7 +87,7 @@ public final class query {
prop.put("mytime", serverDate.formatShortSecond());
// check if we are the right target and requester has correct information about this peer
if (yacyCore.seedDB.mySeed() == null || !yacyCore.seedDB.mySeed().hash.equals(youare)) {
if (sb.wordIndex.seedDB.mySeed() == null || !sb.wordIndex.seedDB.mySeed().hash.equals(youare)) {
// this request has a wrong target
prop.put("response", "-1"); // request rejected
return prop;

8
htroot/yacy/search.java
View File

@ -128,7 +128,7 @@ public final class search {
// store accessing peer
yacySeed remoteSeed = yacySeed.genRemoteSeed(oseed, key, true);
if (yacyCore.seedDB == null) {
if (sb.wordIndex.seedDB == null) {
yacyCore.log.logSevere("yacy.search: seed cache not initialized");
} else {
yacyCore.peerActions.peerArrival(remoteSeed, true);
@ -285,7 +285,7 @@ public final class search {
prop.put("fwrec", ""); // peers that would have helped to construct this result (recommendations)
// prepare search statistics
theQuery.remotepeer = yacyCore.seedDB.lookupByIP(natLib.getInetAddress(client), true, false, false);
theQuery.remotepeer = sb.wordIndex.seedDB.lookupByIP(natLib.getInetAddress(client), true, false, false);
theQuery.resultcount = (theSearch == null) ? 0 : theSearch.getRankingResult().getLocalResourceSize() + theSearch.getRankingResult().getRemoteResourceSize();
theQuery.searchtime = System.currentTimeMillis() - timestamp;
theQuery.urlretrievaltime = (theSearch == null) ? 0 : theSearch.getURLRetrievalTime();
@ -306,8 +306,8 @@ public final class search {
prop.put("searchtime", System.currentTimeMillis() - timestamp);
final int links = Integer.parseInt(prop.get("linkcount","0"));
yacyCore.seedDB.mySeed().incSI(links);
yacyCore.seedDB.mySeed().incSU(links);
sb.wordIndex.seedDB.mySeed().incSI(links);
sb.wordIndex.seedDB.mySeed().incSU(links);
return prop;
}

5
htroot/yacy/transfer.java
View File

@ -54,7 +54,6 @@ import de.anomic.server.serverCodings;
import de.anomic.server.serverFileUtils;
import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch;
import de.anomic.yacy.yacyCore;
import de.anomic.yacy.yacyNetwork;
import de.anomic.yacy.yacySeed;
@ -86,7 +85,7 @@ public final class transfer {
return prop;
}
yacySeed otherseed = yacyCore.seedDB.get(otherpeer);
yacySeed otherseed = sb.wordIndex.seedDB.get(otherpeer);
if ((otherseed == null) || (filename.indexOf("..") >= 0)) {
// reject unknown peers: this does not appear fair, but anonymous senders are dangerous
// reject paths that contain '..' because they are dangerous
@ -106,7 +105,7 @@ public final class transfer {
String access = kelondroBase64Order.enhancedCoder.encode(serverCodings.encodeMD5Raw(otherpeer + ":" + filename)) + ":" + kelondroBase64Order.enhancedCoder.encode(serverCodings.encodeMD5Raw("" + System.currentTimeMillis()));
prop.put("response", "ok");
prop.put("process_access", access);
prop.put("process_address", yacyCore.seedDB.mySeed().getPublicAddress());
prop.put("process_address", sb.wordIndex.seedDB.mySeed().getPublicAddress());
prop.put("process_protocol", "http");
prop.put("process_path", ""); // currently empty; the store process will find a path
prop.put("process_maxsize", "-1"); // if response is too big we return the size of the file

10
htroot/yacy/transferRWI.java
View File

@ -86,7 +86,7 @@ public final class transferRWI {
boolean blockBlacklist = sb.getConfig("indexReceiveBlockBlacklist", "false").equals("true");
boolean checkLimit = sb.getConfigBool("indexDistribution.transferRWIReceiptLimitEnabled", true);
final long cachelimit = sb.getConfigLong("indexDistribution.dhtReceiptLimit", 10000);
final yacySeed otherPeer = yacyCore.seedDB.get(iam);
final yacySeed otherPeer = sb.wordIndex.seedDB.get(iam);
final String otherPeerName = iam + ":" + ((otherPeer == null) ? "NULL" : (otherPeer.getName() + "/" + otherPeer.getVersion()));
// response values
@ -94,8 +94,8 @@ public final class transferRWI {
StringBuffer unknownURLs = new StringBuffer();
int pause = 10000;
if ((youare == null) || (!youare.equals(yacyCore.seedDB.mySeed().hash))) {
sb.getLog().logInfo("Rejecting RWIs from peer " + otherPeerName + ". Wrong target. Wanted peer=" + youare + ", iam=" + yacyCore.seedDB.mySeed().hash);
if ((youare == null) || (!youare.equals(sb.wordIndex.seedDB.mySeed().hash))) {
sb.getLog().logInfo("Rejecting RWIs from peer " + otherPeerName + ". Wrong target. Wanted peer=" + youare + ", iam=" + sb.wordIndex.seedDB.mySeed().hash);
result = "wrong_target";
pause = 0;
} else if ((!granted) || (sb.isRobinsonMode())) {
@ -184,7 +184,7 @@ public final class transferRWI {
}
received++;
}
yacyCore.seedDB.mySeed().incRI(received);
sb.wordIndex.seedDB.mySeed().incRI(received);
// finally compose the unknownURL hash list
final Iterator<String> it = unknownURL.iterator();
@ -196,7 +196,7 @@ public final class transferRWI {
if ((wordhashes.length == 0) || (received == 0)) {
sb.getLog().logInfo("Received 0 RWIs from " + otherPeerName + ", processed in " + (System.currentTimeMillis() - startProcess) + " milliseconds, requesting " + unknownURL.size() + " URLs, blocked " + blocked + " RWIs");
} else {
final double avdist = (yacyDHTAction.dhtDistance(yacyCore.seedDB.mySeed().hash, wordhashes[0]) + yacyDHTAction.dhtDistance(yacyCore.seedDB.mySeed().hash, wordhashes[received - 1])) / 2.0;
final double avdist = (yacyDHTAction.dhtDistance(sb.wordIndex.seedDB.mySeed().hash, wordhashes[0]) + yacyDHTAction.dhtDistance(sb.wordIndex.seedDB.mySeed().hash, wordhashes[received - 1])) / 2.0;
sb.getLog().logInfo("Received " + received + " Entries " + wordc + " Words [" + wordhashes[0] + " .. " + wordhashes[received - 1] + "]/" + avdist + " from " + otherPeerName + ", processed in " + (System.currentTimeMillis() - startProcess) + " milliseconds, requesting " + unknownURL.size() + "/" + receivedURL + " URLs, blocked " + blocked + " RWIs");
}
result = "ok";

8
htroot/yacy/transferURL.java
View File

@ -86,11 +86,11 @@ public final class transferURL {
String result = "";
String doublevalues = "0";
final yacySeed otherPeer = yacyCore.seedDB.get(iam);
final yacySeed otherPeer = sb.wordIndex.seedDB.get(iam);
final String otherPeerName = iam + ":" + ((otherPeer == null) ? "NULL" : (otherPeer.getName() + "/" + otherPeer.getVersion()));
if ((youare == null) || (!youare.equals(yacyCore.seedDB.mySeed().hash))) {
sb.getLog().logInfo("Rejecting URLs from peer " + otherPeerName + ". Wrong target. Wanted peer=" + youare + ", iam=" + yacyCore.seedDB.mySeed().hash);
if ((youare == null) || (!youare.equals(sb.wordIndex.seedDB.mySeed().hash))) {
sb.getLog().logInfo("Rejecting URLs from peer " + otherPeerName + ". Wrong target. Wanted peer=" + youare + ", iam=" + sb.wordIndex.seedDB.mySeed().hash);
result = "wrong_target";
} else if ((!granted) || (sb.isRobinsonMode())) {
sb.getLog().logInfo("Rejecting URLs from peer " + otherPeerName + ". Not granted.");
@ -164,7 +164,7 @@ public final class transferURL {
}
}
yacyCore.seedDB.mySeed().incRU(received);
sb.wordIndex.seedDB.mySeed().incRU(received);
// return rewrite properties
final int more = sb.wordIndex.countURL() - sizeBefore;

3
htroot/yacy/ui/result.java
View File

@ -40,7 +40,6 @@ import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch;
import de.anomic.server.logging.serverLog;
import de.anomic.tools.yFormatter;
import de.anomic.yacy.yacyCore;
import de.anomic.yacy.yacyURL;
public class result {
@ -161,7 +160,7 @@ public class result {
}
// prepare search properties
final boolean yacyonline = ((yacyCore.seedDB != null) && (yacyCore.seedDB.mySeed() != null) && (yacyCore.seedDB.mySeed().getPublicAddress() != null));
final boolean yacyonline = ((sb.wordIndex.seedDB != null) && (sb.wordIndex.seedDB.mySeed() != null) && (sb.wordIndex.seedDB.mySeed().getPublicAddress() != null));
final boolean globalsearch = (global) && (yacyonline) && (sb.getConfigBool(plasmaSwitchboard.INDEX_RECEIVE_ALLOW, false));
// do the search

60
htroot/yacy/ui/ymarks.java
View File

@ -69,7 +69,6 @@ import de.anomic.server.serverDate;
import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch;
import de.anomic.server.logging.serverLog;
import de.anomic.yacy.yacyCore;
import de.anomic.yacy.yacyNewsPool;
import de.anomic.yacy.yacyNewsRecord;
import de.anomic.yacy.yacyURL;
@ -78,7 +77,6 @@ import de.anomic.yacy.yacyURL;
public class ymarks {
private static final serverObjects prop = new serverObjects();
private static plasmaSwitchboard switchboard = null;
private static userDB.Entry user = null;
private static boolean isAdmin = false;
@ -96,9 +94,9 @@ public class ymarks {
String username="";
prop.clear();
switchboard = (plasmaSwitchboard) env;
user = switchboard.userDB.getUser(header);
isAdmin=(switchboard.verifyAuthentication(header, true) || user!= null && user.hasRight(userDB.Entry.BOOKMARK_RIGHT));
plasmaSwitchboard sb = (plasmaSwitchboard) env;
user = sb.userDB.getUser(header);
isAdmin=(sb.verifyAuthentication(header, true) || user!= null && user.hasRight(userDB.Entry.BOOKMARK_RIGHT));
// set user name
if(user != null) username=user.getUserName();
@ -112,7 +110,7 @@ public class ymarks {
*/
// set peer address
final String address = yacyCore.seedDB.mySeed().getPublicAddress();
final String address = sb.wordIndex.seedDB.mySeed().getPublicAddress();
prop.put("address", address);
//defaultvalues
@ -157,7 +155,7 @@ public class ymarks {
tagsString="unsorted"; //default tag
}
Set<String> tags=listManager.string2set(bookmarksDB.cleanTagsString(tagsString));
bookmarksDB.Bookmark bookmark = switchboard.bookmarksDB.createBookmark(url, username);
bookmarksDB.Bookmark bookmark = sb.bookmarksDB.createBookmark(url, username);
if(bookmark != null){
bookmark.setProperty(bookmarksDB.Bookmark.BOOKMARK_TITLE, title);
bookmark.setProperty(bookmarksDB.Bookmark.BOOKMARK_DESCRIPTION, description);
@ -166,7 +164,7 @@ public class ymarks {
}
if(((String) post.get("public")).equals("public")){
bookmark.setPublic(true);
publishNews(url, title, description, tagsString);
publishNews(sb, url, title, description, tagsString);
}else{
bookmark.setPublic(false);
}
@ -176,7 +174,7 @@ public class ymarks {
bookmark.setFeed(false);
}
bookmark.setTags(tags, true);
switchboard.bookmarksDB.saveBookmark(bookmark);
sb.bookmarksDB.saveBookmark(bookmark);
//}else{
//ERROR
}
@ -193,10 +191,10 @@ public class ymarks {
prop.put("mode_public", "0");
prop.put("mode_feed", "0");
} else {
bookmarksDB.Bookmark bookmark = switchboard.bookmarksDB.getBookmark(urlHash);
bookmarksDB.Bookmark bookmark = sb.bookmarksDB.getBookmark(urlHash);
if (bookmark == null) {
// try to get the bookmark from the LURL database
indexURLReference urlentry = switchboard.wordIndex.getURL(urlHash, null, 0);
indexURLReference urlentry = sb.wordIndex.getURL(urlHash, null, 0);
plasmaParserDocument document = null;
if (urlentry != null) {
indexURLReference.Components comp = urlentry.comp();
@ -244,7 +242,7 @@ public class ymarks {
serverLog.logInfo("BOOKMARKS", "I try to import bookmarks from HTML-file");
try {
File file=new File((String)post.get("htmlfile"));
switchboard.bookmarksDB.importFromBookmarks(new yacyURL(file) , post.get("htmlfile$file"), tags, isPublic);
sb.bookmarksDB.importFromBookmarks(new yacyURL(file) , post.get("htmlfile$file"), tags, isPublic);
} catch (MalformedURLException e) {}
serverLog.logInfo("BOOKMARKS", "success!!");
}else if(post.containsKey("xmlfile")){
@ -252,10 +250,10 @@ public class ymarks {
if(((String) post.get("public")).equals("public")){
isPublic=true;
}
switchboard.bookmarksDB.importFromXML(post.get("xmlfile$file"), isPublic);
sb.bookmarksDB.importFromXML(post.get("xmlfile$file"), isPublic);
}else if(post.containsKey("delete")){
String urlHash=(String) post.get("delete");
switchboard.bookmarksDB.removeBookmark(urlHash);
sb.bookmarksDB.removeBookmark(urlHash);
}
if(post.containsKey("tag")){
tagName=(String) post.get("tag");
@ -272,8 +270,8 @@ public class ymarks {
//-----------------------
// create tag list
//-----------------------
printTagList("taglist", tagName, SORT_SIZE, 25, false);
printTagList("optlist", tagName, SORT_ALPHA, SHOW_ALL, true);
printTagList(sb, "taglist", tagName, SORT_SIZE, 25, false);
printTagList(sb, "optlist", tagName, SORT_ALPHA, SHOW_ALL, true);
//-----------------------
// create bookmark list
@ -285,15 +283,15 @@ public class ymarks {
Iterator<String> tagsIt;
int tagCount;
prop.put("num-bookmarks", switchboard.bookmarksDB.bookmarksSize());
prop.put("num-bookmarks", sb.bookmarksDB.bookmarksSize());
count=0;
if(!tagName.equals("")){
prop.put("selected", "");
it=switchboard.bookmarksDB.getBookmarksIterator(tagName, isAdmin);
it=sb.bookmarksDB.getBookmarksIterator(tagName, isAdmin);
}else{
prop.put("selected", " selected=\"selected\"");
it=switchboard.bookmarksDB.getBookmarksIterator(isAdmin);
it=sb.bookmarksDB.getBookmarksIterator(isAdmin);
}
//skip the first entries (display next page)
@ -305,7 +303,7 @@ public class ymarks {
count=0;
while(count<max_count && it.hasNext()){
bookmark=switchboard.bookmarksDB.getBookmark((String)it.next());
bookmark=sb.bookmarksDB.getBookmark((String)it.next());
if(bookmark!=null){
if(bookmark.getFeed() && isAdmin)
prop.put("bookmarks_"+count+"_link", "/FeedReader_p.html?url="+bookmark.getUrl());
@ -359,22 +357,22 @@ public class ymarks {
//-----------------------
count = 0;
count = recurseFolders(switchboard.bookmarksDB.getFolderList(isAdmin),"/",0,true,"");
count = recurseFolders(sb, sb.bookmarksDB.getFolderList(isAdmin),"/",0,true,"");
prop.put("folderlist", count);
return prop; // return from serverObjects respond()
}
private static void printTagList(String id, String tagName, int comp, int max, boolean opt){
private static void printTagList(plasmaSwitchboard sb, String id, String tagName, int comp, int max, boolean opt){
int count=0;
bookmarksDB.Tag tag;
Iterator<Tag> it = null;
if (tagName.equals("")) {
it = switchboard.bookmarksDB.getTagIterator(isAdmin, comp, max);
it = sb.bookmarksDB.getTagIterator(isAdmin, comp, max);
} else {
it = switchboard.bookmarksDB.getTagIterator(tagName, isAdmin, comp, max);
it = sb.bookmarksDB.getTagIterator(tagName, isAdmin, comp, max);
}
while(it.hasNext()){
tag=(Tag) it.next();
@ -398,7 +396,7 @@ public class ymarks {
prop.put(id, count);
}
private static int recurseFolders(Iterator<String> it, String root, int count, boolean next, String prev){
private static int recurseFolders(plasmaSwitchboard sb, Iterator<String> it, String root, int count, boolean next, String prev){
String fn="";
bookmarksDB.Bookmark bookmark;
@ -418,33 +416,33 @@ public class ymarks {
if(fn.startsWith(root)){
prop.put("folderlist_"+count+"_folder", "<li>"+fn.replaceFirst(root+"/*","")+"<ul class=\"folder\">");
count++;
Iterator<String> bit=switchboard.bookmarksDB.getBookmarksIterator(fn, isAdmin);
Iterator<String> bit=sb.bookmarksDB.getBookmarksIterator(fn, isAdmin);
while(bit.hasNext()){
bookmark=switchboard.bookmarksDB.getBookmark((String)bit.next());
bookmark=sb.bookmarksDB.getBookmark((String)bit.next());
prop.put("folderlist_"+count+"_folder", "<li><a href=\""+bookmark.getUrl()+"\" title=\""+bookmark.getDescription()+"\">"+ bookmark.getTitle()+"</a></li>");
count++;
}
if(it.hasNext()){
count = recurseFolders(it, fn, count, true, fn);
count = recurseFolders(sb, it, fn, count, true, fn);
}
} else {
prop.put("folderlist_"+count+"_folder", "</ul></li>");
count++;
root = root.replaceAll("(/.[^/]*$)", "");
if(root.equals("")) root = "/";
count = recurseFolders(it, root, count, false, fn);
count = recurseFolders(sb, it, root, count, false, fn);
}
return count;
}
private static void publishNews(String url, String title, String description, String tagsString) {
private static void publishNews(plasmaSwitchboard sb, String url, String title, String description, String tagsString) {
// create a news message
HashMap<String, String> map = new HashMap<String, String>();
map.put("url", url.replace(',', '|'));
map.put("title", title.replace(',', ' '));
map.put("description", description.replace(',', ' '));
map.put("tags", tagsString.replace(',', ' '));
yacyCore.newsPool.publishMyNews(yacyNewsRecord.newRecord(yacyNewsPool.CATEGORY_BOOKMARK_ADD, map));
sb.wordIndex.newsPool.publishMyNews(yacyNewsRecord.newRecord(sb.wordIndex.seedDB.mySeed(), yacyNewsPool.CATEGORY_BOOKMARK_ADD, map));
}
}

16
htroot/yacy/urls.java
View File

@ -25,6 +25,7 @@
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
import java.io.IOException;
import java.util.Date;
import de.anomic.http.httpHeader;
import de.anomic.index.indexURLReference;
@ -34,7 +35,6 @@ import de.anomic.plasma.plasmaSwitchboard;
import de.anomic.server.serverDate;
import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch;
import de.anomic.yacy.yacyCore;
import de.anomic.yacy.yacyNetwork;
import de.anomic.yacy.yacyURL;
@ -45,7 +45,7 @@ public class urls {
// insert default values
serverObjects prop = new serverObjects();
prop.put("iam", yacyCore.seedDB.mySeed().hash);
prop.put("iam", sb.wordIndex.seedDB.mySeed().hash);
prop.put("response", "rejected - insufficient call parameters");
prop.put("channel_title", "");
prop.put("channel_description", "");
@ -69,10 +69,20 @@ public class urls {
break;
}
if (entry == null) break;
// find referrer, if there is one
referrer = sb.getURL(entry.referrerhash());
// place url to notice-url db
sb.crawlQueues.delegatedURL.push(sb.crawlQueues.delegatedURL.newEntry(entry.url(), "client=____________"));
sb.crawlQueues.delegatedURL.push(
sb.crawlQueues.delegatedURL.newEntry(
entry,
sb.wordIndex.seedDB.mySeed().hash,
new Date(),
0,
"client=____________")
);
// create RSS entry
prop.put("item_" + c + "_title", "");
prop.putHTML("item_" + c + "_link", entry.url().toNormalform(true, false));

3
htroot/yacy/user/ysearch.java
View File

@ -40,7 +40,6 @@ import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch;
import de.anomic.server.logging.serverLog;
import de.anomic.tools.yFormatter;
import de.anomic.yacy.yacyCore;
import de.anomic.yacy.yacyURL;
public class ysearch {
@ -161,7 +160,7 @@ public class ysearch {
}
// prepare search properties
final boolean yacyonline = ((yacyCore.seedDB != null) && (yacyCore.seedDB.mySeed() != null) && (yacyCore.seedDB.mySeed().getPublicAddress() != null));
final boolean yacyonline = ((sb.wordIndex.seedDB != null) && (sb.wordIndex.seedDB.mySeed() != null) && (sb.wordIndex.seedDB.mySeed().getPublicAddress() != null));
final boolean globalsearch = (global) && (yacyonline) && (sb.getConfigBool(plasmaSwitchboard.INDEX_RECEIVE_ALLOW, false));
// do the search

7
htroot/yacysearch.java
View File

@ -68,7 +68,6 @@ import de.anomic.server.serverProfiling;
import de.anomic.server.serverSwitch;
import de.anomic.server.logging.serverLog;
import de.anomic.tools.yFormatter;
import de.anomic.yacy.yacyCore;
import de.anomic.yacy.yacyNewsPool;
import de.anomic.yacy.yacyNewsRecord;
import de.anomic.yacy.yacyURL;
@ -208,7 +207,7 @@ public class yacysearch {
map.put("urlhash", delHash);
map.put("vote", "negative");
map.put("refid", "");
yacyCore.newsPool.publishMyNews(yacyNewsRecord.newRecord(yacyNewsPool.CATEGORY_SURFTIPP_VOTE_ADD, map));
sb.wordIndex.newsPool.publishMyNews(yacyNewsRecord.newRecord(sb.wordIndex.seedDB.mySeed(), yacyNewsPool.CATEGORY_SURFTIPP_VOTE_ADD, map));
}
// if a plus-button was hit, create new voting message
@ -231,14 +230,14 @@ public class yacysearch {
map.put("description", ((document == null) ? comp.dc_title() : document.dc_title()).replace(',', ' '));
map.put("author", ((document == null) ? "" : document.dc_creator()));
map.put("tags", ((document == null) ? "" : document.dc_subject(' ')));
yacyCore.newsPool.publishMyNews(yacyNewsRecord.newRecord(yacyNewsPool.CATEGORY_SURFTIPP_ADD, map));
sb.wordIndex.newsPool.publishMyNews(yacyNewsRecord.newRecord(sb.wordIndex.seedDB.mySeed(), yacyNewsPool.CATEGORY_SURFTIPP_ADD, map));
document.close();
}
}
}
// prepare search properties
final boolean yacyonline = ((yacyCore.seedDB != null) && (yacyCore.seedDB.mySeed() != null) && (yacyCore.seedDB.mySeed().getPublicAddress() != null));
final boolean yacyonline = ((sb.wordIndex.seedDB != null) && (sb.wordIndex.seedDB.mySeed() != null) && (sb.wordIndex.seedDB.mySeed().getPublicAddress() != null));
final boolean globalsearch = (global) && (yacyonline) && (sb.getConfigBool(plasmaSwitchboard.INDEX_RECEIVE_ALLOW, false));
// do the search

3
htroot/yacysearchitem.java
View File

@ -47,7 +47,6 @@ import de.anomic.server.serverSwitch;
import de.anomic.tools.crypt;
import de.anomic.tools.nxTools;
import de.anomic.tools.yFormatter;
import de.anomic.yacy.yacyCore;
import de.anomic.yacy.yacyNewsPool;
import de.anomic.yacy.yacySeed;
import de.anomic.yacy.yacyURL;
@ -192,7 +191,7 @@ public class yacysearchitem {
prop.put("content", theQuery.contentdom + 1); // switch on specific content
prop.put("content_authorized", authenticated ? "1" : "0");
prop.put("content_authorized_recommend", (yacyCore.newsPool.getSpecific(yacyNewsPool.OUTGOING_DB, yacyNewsPool.CATEGORY_SURFTIPP_ADD, "url", result.urlstring()) == null) ? "1" : "0");
prop.put("content_authorized_recommend", (sb.wordIndex.newsPool.getSpecific(yacyNewsPool.OUTGOING_DB, yacyNewsPool.CATEGORY_SURFTIPP_ADD, "url", result.urlstring()) == null) ? "1" : "0");
prop.put("content_authorized_recommend_deletelink", "/yacysearch.html?search=" + theQuery.queryString + "&Enter=Search&count=" + theQuery.displayResults() + "&offset=" + (theQuery.neededResults() - theQuery.displayResults()) + "&order=" + crypt.simpleEncode(theQuery.ranking.toExternalString()) + "&resource=local&time=3&deleteref=" + result.hash() + "&urlmaskfilter=.*");
prop.put("content_authorized_recommend_recommendlink", "/yacysearch.html?search=" + theQuery.queryString + "&Enter=Search&count=" + theQuery.displayResults() + "&offset=" + (theQuery.neededResults() - theQuery.displayResults()) + "&order=" + crypt.simpleEncode(theQuery.ranking.toExternalString()) + "&resource=local&time=3&recommendref=" + result.hash() + "&urlmaskfilter=.*");
prop.put("content_authorized_urlhash", result.hash());

21
source/de/anomic/data/SitemapParser.java
View File

@ -62,12 +62,12 @@ import de.anomic.http.JakartaCommonsHttpClient;
import de.anomic.http.JakartaCommonsHttpResponse;
import de.anomic.http.httpdByteCountInputStream;
import de.anomic.index.indexURLReference;
import de.anomic.plasma.plasmaCrawlEntry;
import de.anomic.plasma.plasmaCrawlProfile;
import de.anomic.plasma.plasmaCrawlZURL;
import de.anomic.plasma.plasmaSwitchboard;
import de.anomic.server.serverDate;
import de.anomic.server.logging.serverLog;
import de.anomic.yacy.yacyCore;
import de.anomic.yacy.yacyURL;
/**
@ -298,7 +298,7 @@ public class SitemapParser extends DefaultHandler {
String error = null;
error = this.switchboard.crawlStacker.stackCrawl(url,
null, // this.siteMapURL.toString(),
yacyCore.seedDB.mySeed().hash, this.nextURL, new Date(),
this.switchboard.wordIndex.seedDB.mySeed().hash, this.nextURL, new Date(),
0, this.crawlingProfile);
if (error != null) {
@ -306,8 +306,21 @@ public class SitemapParser extends DefaultHandler {
this.logger.logInfo("The URL '" + this.nextURL + "' can not be crawled. Reason: " + error);
// insert URL into the error DB
plasmaCrawlZURL.Entry ee = this.switchboard.crawlQueues.errorURL.newEntry(new yacyURL(this.nextURL,
null), error);
plasmaCrawlZURL.Entry ee = this.switchboard.crawlQueues.errorURL.newEntry(
new plasmaCrawlEntry(
switchboard.wordIndex.seedDB.mySeed().hash,
new yacyURL(this.nextURL, null),
"",
"",
new Date(),
null,
0,
0,
0),
this.switchboard.wordIndex.seedDB.mySeed().hash,
new Date(),
1,
error);
ee.store();
this.switchboard.crawlQueues.errorURL.push(ee);
} catch (MalformedURLException e) {/* ignore this */

5
source/de/anomic/data/wiki/abstractWikiParser.java
View File

@ -8,7 +8,6 @@ import java.io.StringReader;
import java.io.UnsupportedEncodingException;
import de.anomic.plasma.plasmaSwitchboard;
import de.anomic.yacy.yacyCore;
public abstract class abstractWikiParser implements wikiParser {
@ -29,7 +28,7 @@ public abstract class abstractWikiParser implements wikiParser {
return transform(
new BufferedReader(new StringReader(content)),
content.length(),
yacyCore.seedDB.mySeed().getPublicAddress(),
sb.wordIndex.seedDB.mySeed().getPublicAddress(),
sb);
} catch (IOException e) {
return "internal error: " + e.getMessage();
@ -75,7 +74,7 @@ public abstract class abstractWikiParser implements wikiParser {
return transform(
new BufferedReader(new InputStreamReader(bais, encoding)),
content.length,
yacyCore.seedDB.mySeed().getPublicAddress(),
sb.wordIndex.seedDB.mySeed().getPublicAddress(),
switchboard);
} catch (IOException e) {
return "internal error: " + e.getMessage();

3
source/de/anomic/data/wiki/knwikiParser.java
View File

@ -58,7 +58,6 @@ import de.anomic.data.wiki.tokens.SimpleToken;
import de.anomic.data.wiki.tokens.TableToken;
import de.anomic.data.wiki.tokens.Token;
import de.anomic.plasma.plasmaSwitchboard;
import de.anomic.yacy.yacyCore;
public class knwikiParser implements wikiParser {
@ -146,7 +145,7 @@ public class knwikiParser implements wikiParser {
tokens = new Token[] {
new SimpleToken('=', '=', new String[][] { null, { "h2" }, { "h3" }, { "h4" } }, true),
new SimpleToken('\'', '\'', new String[][] { null, { "i" }, { "b" }, null, { "b", "i" } }, false),
new LinkToken((publicAddress == null) ? yacyCore.seedDB.mySeed().getPublicAddress() : publicAddress, "Wiki.html?page=", sb),
new LinkToken((publicAddress == null) ? sb.wordIndex.seedDB.mySeed().getPublicAddress() : publicAddress, "Wiki.html?page=", sb),
new ListToken('*', "ul"),
new ListToken('#', "ol"),
new ListToken(':', "blockquote", null),

51
source/de/anomic/http/httpd.java
View File

@ -79,8 +79,6 @@ import de.anomic.server.serverHandler;
import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch;
import de.anomic.server.logging.serverLog;
import de.anomic.yacy.yacyCore;
import de.anomic.yacy.yacySeed;
import de.anomic.yacy.yacyURL;
@ -102,6 +100,7 @@ public final class httpd implements serverHandler {
public static final int ERRORCASE_MESSAGE = 4;
public static final int ERRORCASE_FILE = 5;
public static httpdAlternativeDomainNames alternativeResolver = null;
/**
* A hashset containing extensions that indicate content that should not be transported
@ -363,8 +362,10 @@ public final class httpd implements serverHandler {
// user = addressed peer name
// pw = addressed peer hash (b64-hash)
String auth = (String) header.get(httpHeader.PROXY_AUTHORIZATION,"xxxxxx");
String test = kelondroBase64Order.standardCoder.encodeString(yacyCore.seedDB.mySeed().getName() + ":" + yacyCore.seedDB.mySeed().hash);
if (!test.equals(auth.trim().substring(6))) return false;
if (alternativeResolver != null) {
String test = kelondroBase64Order.standardCoder.encodeString(alternativeResolver.myName() + ":" + alternativeResolver.myID());
if (!test.equals(auth.trim().substring(6))) return false;
}
// the accessing client must use a yacy user-agent
if (!(((String) header.get(httpHeader.USER_AGENT,"")).startsWith("yacy"))) return false;
@ -1184,21 +1185,19 @@ public final class httpd implements serverHandler {
tp.put("port", serverCore.getPortNr(switchboard.getConfig("port", "8080")));
} else {
tp.put("host", serverDomains.myPublicIP());
tp.put("port", (serverCore.portForwardingEnabled && (serverCore.portForwarding != null))
? Integer.toString(serverCore.portForwarding.getPort())
: Integer.toString(serverCore.getPortNr(switchboard.getConfig("port", "8080"))));
tp.put("port", Integer.toString(serverCore.getPortNr(switchboard.getConfig("port", "8080"))));
}
// if peer has public address it will be used
if (yacyCore.seedDB.mySeed().getPublicAddress() != null) {
tp.put("extAddress", yacyCore.seedDB.mySeed().getPublicAddress());
if (alternativeResolver != null) {
tp.put("extAddress", alternativeResolver.myIP() + ":" + alternativeResolver.myPort());
}
// otherwise the local ip address will be used
else {
tp.put("extAddress", tp.get("host", "127.0.0.1") + ":" + tp.get("port", "8080"));
}
tp.put("peerName", yacyCore.seedDB.mySeed().getName());
tp.put("peerName", (alternativeResolver == null) ? "" : alternativeResolver.myName());
tp.put("errorMessageType", errorcase);
tp.put("httpStatus", Integer.toString(httpStatusCode) + " " + httpStatusText);
tp.put("requestMethod", conProp.getProperty(httpHeader.CONNECTION_PROP_METHOD));
@ -1496,34 +1495,14 @@ public final class httpd implements serverHandler {
// httpHeader.CONNECTION_PROP_PROXY_RESPOND_STATUS
}
public static boolean isThisHostPortForwardingIP(String hostName) {
if ((hostName == null) || (hostName.length() == 0)) return false;
if ((!serverCore.portForwardingEnabled) || (serverCore.portForwarding == null)) return false;
boolean isThisHostIP = false;
try {
//InetAddress hostAddress = InetAddress.getByName(hostName);
InetAddress hostAddress = serverDomains.dnsResolve(hostName);
//InetAddress forwardingAddress = InetAddress.getByName(serverCore.portForwarding.getHost());
InetAddress forwardingAddress = serverDomains.dnsResolve(serverCore.portForwarding.getHost());
if ((hostAddress==null)||(forwardingAddress==null)) return false;
if (hostAddress.equals(forwardingAddress)) return true;
} catch (Exception e) {}
return isThisHostIP;
}
public static boolean isThisSeedIP(String hostName) {
if ((hostName == null) || (hostName.length() == 0)) return false;
// getting ip address and port of this seed
yacySeed thisSeed = yacyCore.seedDB.mySeed();
String thisSeedIP = thisSeed.get(yacySeed.IP,null);
String thisSeedPort = thisSeed.get(yacySeed.PORT,null);
if (alternativeResolver == null) return false;
// resolve ip addresses
if (thisSeedIP == null || thisSeedPort == null) return false;
InetAddress seedInetAddress = serverDomains.dnsResolve(thisSeedIP);
InetAddress seedInetAddress = serverDomains.dnsResolve(alternativeResolver.myIP());
InetAddress hostInetAddress = serverDomains.dnsResolve(hostName);
if (seedInetAddress == null || hostInetAddress == null) return false;
@ -1580,7 +1559,8 @@ public final class httpd implements serverHandler {
final Integer dstPort = (idx != -1) ? Integer.valueOf(hostName.substring(idx+1).trim()) : new Integer(80);
// if the hostname endswith thisPeerName.yacy ...
if (dstHost.endsWith(yacyCore.seedDB.mySeed().getName() + ".yacy")) {
String alternativeAddress = (alternativeResolver == null) ? null : alternativeResolver.myAlternativeAddress();
if ((alternativeAddress != null) && (dstHost.endsWith(alternativeAddress))) {
return true;
/*
* If the port number is equal to the yacy port and the IP address is an address of this host ...
@ -1597,11 +1577,6 @@ public final class httpd implements serverHandler {
)
) {
return true;
} else if ((serverCore.portForwardingEnabled) &&
(serverCore.portForwarding != null) &&
(dstPort.intValue() == serverCore.portForwarding.getPort()) &&
isThisHostPortForwardingIP(dstHost)) {
return true;
}
} catch (Exception e) {}
return false;

68
source/de/anomic/http/httpdAlternativeDomainNames.java Normal file
View File

@ -0,0 +1,68 @@
// httpdAlternativeDomainNames.java
// (C) 2008 by Michael Peter Christen; mc@yacy.net, Frankfurt a. M., Germany
// first published 05.05.2008 on http://yacy.net
//
// $LastChangedDate: 2006-04-02 22:40:07 +0200 (So, 02 Apr 2006) $
// $LastChangedRevision: 1986 $
// $LastChangedBy: orbiter $
//
// LICENSE
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation; either version 2 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
package de.anomic.http;
public interface httpdAlternativeDomainNames {
/**
* for a given domain name, return a new address
* the new address may also be a combination of a standard domain name or an IP with ':' and a port number
* @param name: a domain name
* @return an alternative name
*/
public String resolve(String name);
/**
* while other servers may have alternative addresses, this server may also have an alternative
* address
* @return the alternative address of this server which other servers may can resolve
*/
public String myAlternativeAddress();
/**
* return the IP as string of my server address
* @return IP as string of this server
*/
public String myIP();
/**
* return the port of my server address
* @return port number of this server
*/
public int myPort();
/**
* return a name of this server. this may be any string and there is no need that it must be unique
* @return
*/
public String myName();
/**
* return an unique ID of this server
* @return
*/
public String myID();
}

1
source/de/anomic/http/httpdFileHandler.java
View File

@ -83,7 +83,6 @@ import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.PushbackInputStream;
import java.io.UnsupportedEncodingException;
import java.lang.ref.SoftReference;
import java.lang.reflect.InvocationTargetException;

69
source/de/anomic/http/httpdProxyHandler.java
View File

@ -1,14 +1,13 @@
// httpdProxyHandler.java
// -----------------------
// part of YACY
// (C) by Michael Peter Christen; mc@anomic.de
// first published on http://www.anomic.de
// Frankfurt, Germany, 2004
// (C) 2004 by Michael Peter Christen; mc@yacy.net, Frankfurt a. M., Germany
// first published 2004 on http://yacy.net
//
// $LastChangedDate$
// $LastChangedRevision$
// $LastChangedBy$
//
// LICENSE
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation; either version 2 of the License, or
@ -22,25 +21,6 @@
// You should have received a copy of the GNU General Public License
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
//
// Using this software in any meaning (reading, learning, copying, compiling,
// running) means that you agree that the Author(s) is (are) not responsible
// for cost, loss of data or any harm that may be caused directly or indirectly
// by usage of this softare or this documentation. The usage of this software
// is on your own risk. The installation and usage (starting/running) of this
// software may allow other people or application to access your computer and
// any attached devices and is highly dependent on the configuration of the
// software which must be done by the user of the software; the author(s) is
// (are) also not responsible for proper configuration and usage of the
// software, even if provoked by documentation provided together with
// the software.
//
// Any changes to this file according to the GPL as documented in the file
// gpl.txt aside this file in the shipment you received can be done to the
// lines that follows this copyright notice here, but changes must not be
// done inside the copyright notive above. A re-distribution must contain
// the intact and unchanged copyright notice.
// Contributions and changes to the program code must be marked as such.
// Contributions:
// [AS] Alexander Schier: Blacklist (404 response for AGIS hosts)
@ -53,7 +33,7 @@
or a host is given in the header host field of an HTTP/1.0 / HTTP/1.1
command.
Transparency is maintained, whenever appropriate. We change header
atributes if necessary for the indexing mechanism; i.e. we do not
attributes if necessary for the indexing mechanism; i.e. we do not
support gzip-ed encoding. We also do not support unrealistic
'expires' values that would force a cache to be flushed immediately
pragma non-cache attributes are supported
@ -507,7 +487,7 @@ public final class httpdProxyHandler {
}
// resolve yacy and yacyh domains
String yAddress = yacyCore.seedDB.resolveYacyAddress(host);
String yAddress = (httpd.alternativeResolver == null) ? null : httpd.alternativeResolver.resolve(host);
// re-calc the url path
String remotePath = (args == null) ? path : (path + "?" + args); // with leading '/'
@ -975,7 +955,7 @@ public final class httpdProxyHandler {
return;
}
// set another userAgent, if not yellowlisted
// set another userAgent, if not yellow-listed
if (!(yellowList.contains(domain(hostlow)))) {
// change the User-Agent
requestHeader.put(httpHeader.USER_AGENT, generateUserAgent(requestHeader));
@ -984,7 +964,7 @@ public final class httpdProxyHandler {
addXForwardedForHeader(conProp, requestHeader);
// resolve yacy and yacyh domains
String yAddress = yacyCore.seedDB.resolveYacyAddress(host);
String yAddress = (httpd.alternativeResolver == null) ? null : httpd.alternativeResolver.resolve(host);
// attach possible yacy-sublevel-domain
if ((yAddress != null) && ((pos = yAddress.indexOf("/")) >= 0)) remotePath = yAddress.substring(pos) + remotePath;
@ -1084,7 +1064,7 @@ public final class httpdProxyHandler {
addXForwardedForHeader(conProp, requestHeader);
// resolve yacy and yacyh domains
String yAddress = yacyCore.seedDB.resolveYacyAddress(host);
String yAddress = (httpd.alternativeResolver == null) ? null : httpd.alternativeResolver.resolve(host);
// re-calc the url path
String remotePath = (args == null) ? path : (path + "?" + args);
@ -1658,20 +1638,23 @@ public final class httpdProxyHandler {
private static void setViaHeader(httpHeader header, String httpVer) {
if (!switchboard.getConfigBool("proxy.sendViaHeader", true)) return;
// getting header set by other proxies in the chain
StringBuffer viaValue = new StringBuffer();
if (header.containsKey(httpHeader.VIA)) viaValue.append((String)header.get(httpHeader.VIA));
if (viaValue.length() > 0) viaValue.append(", ");
// appending info about this peer
viaValue
.append(httpVer).append(" ")
.append(yacyCore.seedDB.mySeed().getName()).append(".yacy ")
.append("(YaCy ").append(switchboard.getConfig("vString", "0.0")).append(")");
// storing header back
header.put(httpHeader.VIA, new String(viaValue));
String myAddress = (httpd.alternativeResolver == null) ? null : httpd.alternativeResolver.myAlternativeAddress();
if (myAddress != null) {
// getting header set by other proxies in the chain
StringBuffer viaValue = new StringBuffer();
if (header.containsKey(httpHeader.VIA)) viaValue.append((String)header.get(httpHeader.VIA));
if (viaValue.length() > 0) viaValue.append(", ");
// appending info about this peer
viaValue
.append(httpVer).append(" ")
.append(myAddress).append(" ")
.append("(YaCy ").append(switchboard.getConfig("vString", "0.0")).append(")");
// storing header back
header.put(httpHeader.VIA, new String(viaValue));
}
}
/**

8
source/de/anomic/index/indexURLReference.java
View File

@ -191,7 +191,7 @@ public class indexURLReference {
this.ranking = ranking;
}
public indexURLReference(Properties prop){
public indexURLReference(Properties prop) {
// generates an plasmaLURLEntry using the properties from the argument
// the property names must correspond to the one from toString
//System.out.println("DEBUG-ENTRY: prop=" + prop.toString());
@ -224,7 +224,7 @@ public class indexURLReference {
} catch (ParseException e) {
encodeDate(col_fresh, new Date());
}
this.entry.setCol(col_referrer, prop.getProperty("referrer", yacyURL.dummyHash).getBytes());
this.entry.setCol(col_referrer, prop.getProperty("referrer", "").getBytes());
this.entry.setCol(col_md5, serverCodings.decodeHex(prop.getProperty("md5", "")));
this.entry.setCol(col_size, Integer.parseInt(prop.getProperty("size", "0")));
this.entry.setCol(col_wc, Integer.parseInt(prop.getProperty("wc", "0")));
@ -434,9 +434,9 @@ public class indexURLReference {
//return "{" + core + ",snippet=" + crypt.simpleEncode(snippet) + "}";
}
public plasmaCrawlEntry toBalancerEntry() {
public plasmaCrawlEntry toBalancerEntry(String initiatorHash) {
return new plasmaCrawlEntry(
null,
initiatorHash,
comp().url(),
referrerHash(),
comp().dc_title(),

41
source/de/anomic/plasma/crawler/plasmaCrawlQueues.java
View File

@ -106,7 +106,7 @@ public class plasmaCrawlQueues {
}
public yacyURL getURL(String urlhash) {
if (urlhash.equals(yacyURL.dummyHash)) return null;
assert urlhash != null;
plasmaCrawlEntry ne = noticeURL.get(urlhash);
if (ne != null) return ne.url();
plasmaCrawlZURL.Entry ee = delegatedURL.getEntry(urlhash);
@ -220,7 +220,7 @@ public class plasmaCrawlQueues {
}
if (this.log.isFine()) log.logFine("LOCALCRAWL: URL=" + urlEntry.url() + ", initiator=" + urlEntry.initiator() + ", crawlOrder=" + ((profile.remoteIndexing()) ? "true" : "false") + ", depth=" + urlEntry.depth() + ", crawlDepth=" + profile.generalDepth() + ", filter=" + profile.generalFilter()
+ ", permission=" + ((yacyCore.seedDB == null) ? "undefined" : (((yacyCore.seedDB.mySeed().isSenior()) || (yacyCore.seedDB.mySeed().isPrincipal())) ? "true" : "false")));
+ ", permission=" + ((sb.wordIndex.seedDB == null) ? "undefined" : (((sb.wordIndex.seedDB.mySeed().isSenior()) || (sb.wordIndex.seedDB.mySeed().isPrincipal())) ? "true" : "false")));
processLocalCrawling(urlEntry, stats);
return true;
@ -234,13 +234,13 @@ public class plasmaCrawlQueues {
public boolean remoteCrawlLoaderJob() {
// check if we are allowed to crawl urls provided by other peers
if (!yacyCore.seedDB.mySeed().getFlagAcceptRemoteCrawl()) {
if (!sb.wordIndex.seedDB.mySeed().getFlagAcceptRemoteCrawl()) {
//this.log.logInfo("remoteCrawlLoaderJob: not done, we are not allowed to do that");
return false;
}
// check if we are a senior peer
if (!yacyCore.seedDB.mySeed().isActive()) {
if (!sb.wordIndex.seedDB.mySeed().isActive()) {
//this.log.logInfo("remoteCrawlLoaderJob: not done, this should be a senior or principal peer");
return false;
}
@ -266,7 +266,7 @@ public class plasmaCrawlQueues {
(coreCrawlJobSize() == 0) &&
(remoteTriggeredCrawlJobSize() == 0) &&
(sb.queueSize() < 10)) {
if (yacyCore.seedDB != null && yacyCore.seedDB.sizeConnected() > 0) {
if (sb.wordIndex.seedDB != null && sb.wordIndex.seedDB.sizeConnected() > 0) {
Iterator<yacySeed> e = yacyCore.dhtAgent.getProvidesRemoteCrawlURLs();
while (e.hasNext()) {
seed = e.next();
@ -285,7 +285,7 @@ public class plasmaCrawlQueues {
while ((seed == null) && (remoteCrawlProviderHashes.size() > 0)) {
hash = (String) remoteCrawlProviderHashes.remove(remoteCrawlProviderHashes.size() - 1);
if (hash == null) continue;
seed = yacyCore.seedDB.get(hash);
seed = sb.wordIndex.seedDB.get(hash);
if (seed == null) continue;
// check if the peer is inside our cluster
if ((sb.isRobinsonMode()) && (!sb.isInMyCluster(seed))) {
@ -296,7 +296,7 @@ public class plasmaCrawlQueues {
if (seed == null) return false;
// we know a peer which should provide remote crawl entries. load them now.
RSSFeed feed = (seed == null) ? null : yacyClient.queryRemoteCrawlURLs(seed, 20);
RSSFeed feed = (seed == null) ? null : yacyClient.queryRemoteCrawlURLs(sb.wordIndex.seedDB, seed, 20);
if (feed == null) return true;
// parse the rss
yacyURL url, referrer;
@ -408,7 +408,7 @@ public class plasmaCrawlQueues {
}
if (this.log.isFine()) log.logFine("plasmaSwitchboard.remoteTriggeredCrawlJob: url=" + urlEntry.url() + ", initiator=" + urlEntry.initiator() + ", crawlOrder=" + ((profile.remoteIndexing()) ? "true" : "false") + ", depth=" + urlEntry.depth() + ", crawlDepth=" + profile.generalDepth() + ", filter="
+ profile.generalFilter() + ", permission=" + ((yacyCore.seedDB == null) ? "undefined" : (((yacyCore.seedDB.mySeed().isSenior()) || (yacyCore.seedDB.mySeed().isPrincipal())) ? "true" : "false")));
+ profile.generalFilter() + ", permission=" + ((sb.wordIndex.seedDB == null) ? "undefined" : (((sb.wordIndex.seedDB.mySeed().isSenior()) || (sb.wordIndex.seedDB.mySeed().isPrincipal())) ? "true" : "false")));
processLocalCrawling(urlEntry, stats);
return true;
@ -440,9 +440,9 @@ public class plasmaCrawlQueues {
) {
plasmaCrawlEntry centry = new plasmaCrawlEntry(
yacyCore.seedDB.mySeed().hash,
sb.wordIndex.seedDB.mySeed().hash,
url,
null,
"",
"",
new Date(),
(forText) ?
@ -485,7 +485,12 @@ public class plasmaCrawlQueues {
this.entry.setStatus("worker-checkingrobots");
if ((entry.url().getProtocol().equals("http") || entry.url().getProtocol().equals("https")) && robotsParser.isDisallowed(entry.url())) {
if (log.isFine()) log.logFine("Crawling of URL '" + entry.url().toString() + "' disallowed by robots.txt.");
plasmaCrawlZURL.Entry eentry = errorURL.newEntry(this.entry.url(), "denied by robots.txt");
plasmaCrawlZURL.Entry eentry = errorURL.newEntry(
this.entry,
sb.wordIndex.seedDB.mySeed().hash,
new Date(),
1,
"denied by robots.txt");
eentry.store();
errorURL.push(eentry);
} else {
@ -493,7 +498,12 @@ public class plasmaCrawlQueues {
this.entry.setStatus("worker-loading");
String result = loader.process(this.entry, plasmaParser.PARSER_MODE_CRAWLER);
if (result != null) {
plasmaCrawlZURL.Entry eentry = errorURL.newEntry(this.entry.url(), "cannot load: " + result);
plasmaCrawlZURL.Entry eentry = errorURL.newEntry(
this.entry,
sb.wordIndex.seedDB.mySeed().hash,
new Date(),
1,
"cannot load: " + result);
eentry.store();
errorURL.push(eentry);
} else {
@ -501,7 +511,12 @@ public class plasmaCrawlQueues {
}
}
} catch (Exception e) {
plasmaCrawlZURL.Entry eentry = errorURL.newEntry(this.entry.url(), e.getMessage() + " - in worker");
plasmaCrawlZURL.Entry eentry = errorURL.newEntry(
this.entry,
sb.wordIndex.seedDB.mySeed().hash,
new Date(),
1,
e.getMessage() + " - in worker");
eentry.store();
errorURL.push(eentry);
e.printStackTrace();

4
source/de/anomic/plasma/crawler/plasmaFTPLoader.java
View File

@ -157,7 +157,7 @@ public class plasmaFTPLoader {
// some error logging
final String detail = (berr.size() > 0) ? "\n Errorlog: " + berr.toString() : "";
log.logWarning("Unable to download URL " + entry.url().toString() + detail);
sb.crawlQueues.errorURL.newEntry(entry, null, new Date(), 1, plasmaCrawlEURL.DENIED_SERVER_DOWNLOAD_ERROR);
sb.crawlQueues.errorURL.newEntry(entry, sb.wordIndex.seedDB.mySeed().hash, new Date(), 1, plasmaCrawlEURL.DENIED_SERVER_DOWNLOAD_ERROR);
// an error has occured. cleanup
if (cacheFile.exists()) {
@ -281,7 +281,7 @@ public class plasmaFTPLoader {
ftpClient.exec("get \"" + path + "\" \"" + cacheFile.getAbsolutePath() + "\"", false);
} else {
log.logInfo("REJECTED TOO BIG FILE with size " + size + " Bytes for URL " + entry.url().toString());
sb.crawlQueues.errorURL.newEntry(entry, null, new Date(), 1,
sb.crawlQueues.errorURL.newEntry(entry, this.sb.wordIndex.seedDB.mySeed().hash, new Date(), 1,
plasmaCrawlEURL.DENIED_FILESIZE_LIMIT_EXCEEDED);
throw new Exception("file size exceeds limit");
}

24
source/de/anomic/plasma/crawler/plasmaHTTPLoader.java
View File

@ -139,7 +139,7 @@ public final class plasmaHTTPLoader {
if (retryCount < 0) {
this.log.logInfo("Redirection counter exceeded for URL " + entry.url().toString() + ". Processing aborted.");
sb.crawlQueues.errorURL.newEntry(entry, null, new Date(), 1, plasmaCrawlEURL.DENIED_REDIRECTION_COUNTER_EXCEEDED).store();
sb.crawlQueues.errorURL.newEntry(entry, sb.wordIndex.seedDB.mySeed().hash, new Date(), 1, plasmaCrawlEURL.DENIED_REDIRECTION_COUNTER_EXCEEDED).store();
return null;
}
@ -154,7 +154,7 @@ public final class plasmaHTTPLoader {
String hostlow = host.toLowerCase();
if (plasmaSwitchboard.urlBlacklist.isListed(indexReferenceBlacklist.BLACKLIST_CRAWLER, hostlow, path)) {
this.log.logInfo("CRAWLER Rejecting URL '" + entry.url().toString() + "'. URL is in blacklist.");
sb.crawlQueues.errorURL.newEntry(entry, null, new Date(), 1, plasmaCrawlEURL.DENIED_URL_IN_BLACKLIST).store();
sb.crawlQueues.errorURL.newEntry(entry, sb.wordIndex.seedDB.mySeed().hash, new Date(), 1, plasmaCrawlEURL.DENIED_URL_IN_BLACKLIST).store();
return null;
}
@ -192,7 +192,7 @@ public final class plasmaHTTPLoader {
// aborting download if content is to long ...
if (htCache.cacheFile().getAbsolutePath().length() > serverSystem.maxPathLength) {
this.log.logInfo("REJECTED URL " + entry.url().toString() + " because path too long '" + plasmaHTCache.cachePath.getAbsolutePath() + "'");
sb.crawlQueues.errorURL.newEntry(entry, null, new Date(), 1, plasmaCrawlEURL.DENIED_CACHEFILE_PATH_TOO_LONG);
sb.crawlQueues.errorURL.newEntry(entry, sb.wordIndex.seedDB.mySeed().hash, new Date(), 1, plasmaCrawlEURL.DENIED_CACHEFILE_PATH_TOO_LONG);
return (htCache = null);
}
@ -202,7 +202,7 @@ public final class plasmaHTTPLoader {
this.log.logInfo("REJECTED URL " + entry.url().toString() + " because of an invalid file path ('" +
htCache.cacheFile().getCanonicalPath() + "' does not start with '" +
plasmaHTCache.cachePath.getAbsolutePath() + "').");
sb.crawlQueues.errorURL.newEntry(entry, null, new Date(), 1, plasmaCrawlEURL.DENIED_INVALID_CACHEFILE_PATH);
sb.crawlQueues.errorURL.newEntry(entry, sb.wordIndex.seedDB.mySeed().hash, new Date(), 1, plasmaCrawlEURL.DENIED_INVALID_CACHEFILE_PATH);
return (htCache = null);
}
@ -232,7 +232,7 @@ public final class plasmaHTTPLoader {
fos = new httpdBoundedSizeOutputStream(fos,this.maxFileSize);
} else if (contentLength > this.maxFileSize) {
this.log.logInfo("REJECTED URL " + entry.url() + " because file size '" + contentLength + "' exceeds max filesize limit of " + this.maxFileSize + " bytes.");
sb.crawlQueues.errorURL.newEntry(entry, null, new Date(), 1, plasmaCrawlEURL.DENIED_FILESIZE_LIMIT_EXCEEDED);
sb.crawlQueues.errorURL.newEntry(entry, sb.wordIndex.seedDB.mySeed().hash, new Date(), 1, plasmaCrawlEURL.DENIED_FILESIZE_LIMIT_EXCEEDED);
return null;
}
}
@ -251,7 +251,7 @@ public final class plasmaHTTPLoader {
} else {
// if the response has not the right file type then reject file
this.log.logInfo("REJECTED WRONG MIME/EXT TYPE " + res.getResponseHeader().mime() + " for URL " + entry.url().toString());
sb.crawlQueues.errorURL.newEntry(entry, null, new Date(), 1, plasmaCrawlEURL.DENIED_WRONG_MIMETYPE_OR_EXT);
sb.crawlQueues.errorURL.newEntry(entry, sb.wordIndex.seedDB.mySeed().hash, new Date(), 1, plasmaCrawlEURL.DENIED_WRONG_MIMETYPE_OR_EXT);
return null;
}
} catch (SocketException e) {
@ -262,7 +262,7 @@ public final class plasmaHTTPLoader {
// and most possible corrupted
if (cacheFile.exists()) cacheFile.delete();
this.log.logSevere("CRAWLER LOADER ERROR1: with URL=" + entry.url().toString() + ": " + e.toString());
sb.crawlQueues.errorURL.newEntry(entry, null, new Date(), 1, plasmaCrawlEURL.DENIED_CONNECTION_ERROR);
sb.crawlQueues.errorURL.newEntry(entry, sb.wordIndex.seedDB.mySeed().hash, new Date(), 1, plasmaCrawlEURL.DENIED_CONNECTION_ERROR);
htCache = null;
}
} else if (res.getStatusLine().startsWith("30")) {
@ -273,7 +273,7 @@ public final class plasmaHTTPLoader {
if (redirectionUrlString.length() == 0) {
this.log.logWarning("CRAWLER Redirection of URL=" + entry.url().toString() + " aborted. Location header is empty.");
sb.crawlQueues.errorURL.newEntry(entry, null, new Date(), 1, plasmaCrawlEURL.DENIED_REDIRECTION_HEADER_EMPTY);
sb.crawlQueues.errorURL.newEntry(entry, sb.wordIndex.seedDB.mySeed().hash, new Date(), 1, plasmaCrawlEURL.DENIED_REDIRECTION_HEADER_EMPTY);
return null;
}
@ -287,7 +287,7 @@ public final class plasmaHTTPLoader {
// if we are already doing a shutdown we don't need to retry crawling
if (Thread.currentThread().isInterrupted()) {
this.log.logSevere("CRAWLER Retry of URL=" + entry.url().toString() + " aborted because of server shutdown.");
sb.crawlQueues.errorURL.newEntry(entry, null, new Date(), 1, plasmaCrawlEURL.DENIED_SERVER_SHUTDOWN);
sb.crawlQueues.errorURL.newEntry(entry, sb.wordIndex.seedDB.mySeed().hash, new Date(), 1, plasmaCrawlEURL.DENIED_SERVER_SHUTDOWN);
return null;
}
@ -298,7 +298,7 @@ public final class plasmaHTTPLoader {
String dbname = sb.urlExists(urlhash);
if (dbname != null) {
this.log.logWarning("CRAWLER Redirection of URL=" + entry.url().toString() + " ignored. The url appears already in db " + dbname);
sb.crawlQueues.errorURL.newEntry(entry, null, new Date(), 1, plasmaCrawlEURL.DENIED_REDIRECTION_TO_DOUBLE_CONTENT);
sb.crawlQueues.errorURL.newEntry(entry, sb.wordIndex.seedDB.mySeed().hash, new Date(), 1, plasmaCrawlEURL.DENIED_REDIRECTION_TO_DOUBLE_CONTENT);
return null;
}
@ -312,7 +312,7 @@ public final class plasmaHTTPLoader {
this.log.logInfo("REJECTED WRONG STATUS TYPE '" + res.getStatusLine() + "' for URL " + entry.url().toString());
// not processed any further
sb.crawlQueues.errorURL.newEntry(entry, null, new Date(), 1, plasmaCrawlEURL.DENIED_WRONG_HTTP_STATUSCODE + res.getStatusCode() + ")");
sb.crawlQueues.errorURL.newEntry(entry, sb.wordIndex.seedDB.mySeed().hash, new Date(), 1, plasmaCrawlEURL.DENIED_WRONG_HTTP_STATUSCODE + res.getStatusCode() + ")");
}
} finally {
@ -402,7 +402,7 @@ public final class plasmaHTTPLoader {
if (failreason != null) {
// add url into error db
sb.crawlQueues.errorURL.newEntry(entry, null, new Date(), 1, failreason);
sb.crawlQueues.errorURL.newEntry(entry, sb.wordIndex.seedDB.mySeed().hash, new Date(), 1, failreason);
}
return null;
}

2
source/de/anomic/plasma/dbImport/plasmaDbImporter.java
View File

@ -99,7 +99,7 @@ public class plasmaDbImporter extends AbstractImporter implements dbImporter {
}
this.log.logFine("Initializing source word index db.");
this.importWordIndex = new plasmaWordIndex(this.importPrimaryPath, this.importSecondaryPath, sb.getConfig("network.unit.name", ""), this.log);
this.importWordIndex = new plasmaWordIndex(sb.getConfig("network.unit.name", ""), this.log, this.importPrimaryPath, this.importSecondaryPath);
this.importStartSize = this.importWordIndex.size();
}

13
source/de/anomic/plasma/plasmaCrawlEntry.java
View File

@ -34,7 +34,6 @@ import de.anomic.kelondro.kelondroBase64Order;
import de.anomic.kelondro.kelondroBitfield;
import de.anomic.kelondro.kelondroNaturalOrder;
import de.anomic.kelondro.kelondroRow;
import de.anomic.yacy.yacyCore;
import de.anomic.yacy.yacySeedDB;
import de.anomic.yacy.yacyURL;
@ -79,10 +78,6 @@ public class plasmaCrawlEntry {
private String status;
private int initialHash; // to provide a object hash that does not change even if the url changes because of redirection
public plasmaCrawlEntry(yacyURL url) {
this(yacyCore.seedDB.mySeed().hash, url, null, null, new Date(), null, 0, 0, 0);
}
/**
* @param initiator the hash of the initiator peer
* @param url the {@link URL} to crawl
@ -108,10 +103,12 @@ public class plasmaCrawlEntry {
// create new entry and store it into database
assert appdate != null;
assert url != null;
if ((initiator == null) || (initiator.length() == 0)) initiator = yacyURL.dummyHash;
assert initiator != null;
assert initiator.length() > 0;
assert referrerhash != null;
this.initiator = initiator;
this.url = url;
this.refhash = (referrerhash == null) ? yacyURL.dummyHash : referrerhash;
this.refhash = referrerhash;
this.name = (name == null) ? "" : name;
this.appdate = (appdate == null) ? 0 : appdate.getTime();
this.profileHandle = profileHandle; // must not be null
@ -137,7 +134,7 @@ public class plasmaCrawlEntry {
if (urlstring == null) throw new IOException ("url string is null");
this.initiator = entry.getColString(1, null);
this.url = new yacyURL(urlstring, entry.getColString(0, null));
this.refhash = (entry.empty(3)) ? yacyURL.dummyHash : entry.getColString(3, null);
this.refhash = (entry.empty(3)) ? "" : entry.getColString(3, null);
this.name = (entry.empty(4)) ? "" : entry.getColString(4, "UTF-8").trim();
this.appdate = entry.getColLong(5);
this.profileHandle = (entry.empty(6)) ? null : entry.getColString(6, null).trim();

5
source/de/anomic/plasma/plasmaCrawlResultURLs.java
View File

@ -56,7 +56,6 @@ import java.util.LinkedList;
import de.anomic.index.indexURLReference;
import de.anomic.yacy.yacySeedDB;
import de.anomic.yacy.yacyURL;
public final class plasmaCrawlResultURLs {
@ -81,10 +80,10 @@ public final class plasmaCrawlResultURLs {
}
public synchronized void stack(indexURLReference e, String initiatorHash, String executorHash, int stackType) {
assert initiatorHash != null;
assert executorHash != null;
if (e == null) { return; }
try {
if (initiatorHash == null) { initiatorHash = yacyURL.dummyHash; }
if (executorHash == null) { executorHash = yacyURL.dummyHash; }
switch (stackType) {
case 0: break;
case 1: externResultStack.add(e.hash() + initiatorHash + executorHash); break;

11
source/de/anomic/plasma/plasmaCrawlStacker.java
View File

@ -66,7 +66,6 @@ import de.anomic.kelondro.kelondroRowSet;
import de.anomic.kelondro.kelondroTree;
import de.anomic.server.serverDomains;
import de.anomic.server.logging.serverLog;
import de.anomic.yacy.yacyCore;
import de.anomic.yacy.yacyURL;
public final class plasmaCrawlStacker extends Thread {
@ -213,7 +212,7 @@ public final class plasmaCrawlStacker extends Thread {
// if the url was rejected we store it into the error URL db
if (rejectReason != null) {
plasmaCrawlZURL.Entry ee = sb.crawlQueues.errorURL.newEntry(entry, yacyCore.seedDB.mySeed().hash, null, 0, rejectReason);
plasmaCrawlZURL.Entry ee = sb.crawlQueues.errorURL.newEntry(entry, sb.wordIndex.seedDB.mySeed().hash, new Date(), 1, rejectReason);
ee.store();
sb.crawlQueues.errorURL.push(ee);
}
@ -357,7 +356,7 @@ public final class plasmaCrawlStacker extends Thread {
plasmaCrawlEntry entry = new plasmaCrawlEntry(
initiatorHash, // initiator, needed for p2p-feedback
url, // url clear text string
(referrer == null) ? null : referrer.hash(), // last url in crawling queue
(referrer == null) ? "" : referrer.hash(), // last url in crawling queue
name, // load date
loadDate, // the anchor name
(profile == null) ? null : profile.handle(), // profile must not be null!
@ -481,15 +480,15 @@ public final class plasmaCrawlStacker extends Thread {
}
// store information
boolean local = ((entry.initiator().equals(yacyURL.dummyHash)) || (entry.initiator().equals(yacyCore.seedDB.mySeed().hash)));
boolean local = entry.initiator().equals(sb.wordIndex.seedDB.mySeed().hash);
boolean global =
(profile != null) &&
(profile.remoteIndexing()) /* granted */ &&
(entry.depth() == profile.generalDepth()) /* leaf node */ &&
//(initiatorHash.equals(yacyCore.seedDB.mySeed.hash)) /* not proxy */ &&
(
(yacyCore.seedDB.mySeed().isSenior()) ||
(yacyCore.seedDB.mySeed().isPrincipal())
(sb.wordIndex.seedDB.mySeed().isSenior()) ||
(sb.wordIndex.seedDB.mySeed().isPrincipal())
) /* qualified */;
if ((!local)&&(!global)&&(!profile.handle().equals(this.sb.defaultRemoteProfile.handle()))) {

37
source/de/anomic/plasma/plasmaCrawlZURL.java
View File

@ -38,7 +38,6 @@ import de.anomic.kelondro.kelondroFlexWidthArray;
import de.anomic.kelondro.kelondroIndex;
import de.anomic.kelondro.kelondroRow;
import de.anomic.kelondro.kelondroRowSet;
import de.anomic.yacy.yacyCore;
import de.anomic.yacy.yacySeedDB;
import de.anomic.yacy.yacyURL;
@ -90,17 +89,17 @@ public class plasmaCrawlZURL {
}
public synchronized Entry newEntry(
plasmaCrawlEntry bentry, String executor, Date workdate,
int workcount, String anycause) {
if ((executor == null) || (executor.length() < yacySeedDB.commonHashLength)) executor = yacyURL.dummyHash;
plasmaCrawlEntry bentry,
String executor,
Date workdate,
int workcount,
String anycause) {
assert executor != null;
assert executor.length() > 0;
if (anycause == null) anycause = "unknown";
return new Entry(bentry, executor, workdate, workcount, anycause);
}
public synchronized Entry newEntry(yacyURL url, String anycause) {
return new Entry(url, anycause);
}
public boolean remove(String hash) {
if (hash == null) return false;
try {
@ -155,23 +154,23 @@ public class plasmaCrawlZURL {
public class Entry {
plasmaCrawlEntry bentry; // the balancer entry
private String executor; // the crawling initiator
private String executor; // the crawling executor
private Date workdate; // the time when the url was last time tried to load
private int workcount; // number of tryings
private String anycause; // string describing reason for load fail
private boolean stored;
public Entry(yacyURL url, String reason) {
this(new plasmaCrawlEntry(url), null, new Date(), 0, reason);
}
public Entry(
plasmaCrawlEntry bentry, String executor, Date workdate,
int workcount, String anycause) {
plasmaCrawlEntry bentry,
String executor,
Date workdate,
int workcount,
String anycause) {
// create new entry
assert bentry != null;
assert executor != null;
this.bentry = bentry;
this.executor = (executor == null) ? yacyCore.seedDB.mySeed().hash : executor;
this.executor = executor;
this.workdate = (workdate == null) ? new Date() : workdate;
this.workcount = workcount;
this.anycause = (anycause == null) ? "" : anycause;
@ -179,11 +178,6 @@ public class plasmaCrawlZURL {
}
public Entry(kelondroRow.Entry entry) throws IOException {
insertEntry(entry);
this.stored = true;
}
private void insertEntry(kelondroRow.Entry entry) throws IOException {
assert (entry != null);
this.executor = entry.getColString(1, "UTF-8");
this.workdate = new Date(entry.getColLong(2));
@ -191,6 +185,7 @@ public class plasmaCrawlZURL {
this.anycause = entry.getColString(4, "UTF-8");
this.bentry = new plasmaCrawlEntry(plasmaCrawlEntry.rowdef.newEntry(entry.getColBytes(5)));
assert ((new String(entry.getColBytes(0))).equals(bentry.url().hash()));
this.stored = true;
return;
}

15
source/de/anomic/plasma/plasmaDHTChunk.java
View File

@ -58,7 +58,6 @@ import de.anomic.kelondro.kelondroBase64Order;
import de.anomic.kelondro.kelondroException;
import de.anomic.server.serverCodings;
import de.anomic.server.logging.serverLog;
import de.anomic.yacy.yacyCore;
import de.anomic.yacy.yacyDHTAction;
import de.anomic.yacy.yacySeedDB;
@ -132,7 +131,7 @@ public class plasmaDHTChunk {
this.log = log;
this.wordIndex = wordIndex;
this.startPointHash = selectTransferStart();
if (this.log.isFine()) log.logFine("Selected hash " + this.startPointHash + " as start point for index distribution, distance = " + yacyDHTAction.dhtDistance(yacyCore.seedDB.mySeed().hash, this.startPointHash));
if (this.log.isFine()) log.logFine("Selected hash " + this.startPointHash + " as start point for index distribution, distance = " + yacyDHTAction.dhtDistance(wordIndex.seedDB.mySeed().hash, this.startPointHash));
selectTransferContainers(this.startPointHash, minCount, maxCount, maxtime);
// count the indexes, can be smaller as expected
@ -150,7 +149,7 @@ public class plasmaDHTChunk {
try {
this.log = log;
this.wordIndex = wordIndex;
if (this.log.isFine()) log.logFine("Demanded hash " + startHash + " as start point for index distribution, distance = " + yacyDHTAction.dhtDistance(yacyCore.seedDB.mySeed().hash, this.startPointHash));
if (this.log.isFine()) log.logFine("Demanded hash " + startHash + " as start point for index distribution, distance = " + yacyDHTAction.dhtDistance(wordIndex.seedDB.mySeed().hash, this.startPointHash));
selectTransferContainers(startHash, minCount, maxCount, maxtime);
// count the indexes, can be smaller as expected
@ -167,12 +166,12 @@ public class plasmaDHTChunk {
private String selectTransferStart() {
String startPointHash;
// first try to select with increasing probality a good start point
double minimumDistance = ((double) peerRedundancy) / ((double) yacyCore.seedDB.sizeConnected());
double minimumDistance = ((double) peerRedundancy) / ((double) wordIndex.seedDB.sizeConnected());
double d, bestDistance = 0.0;
String bestHash = null;
for (int i = yacyCore.seedDB.sizeConnected() / 8; i > 0; i--) {
for (int i = wordIndex.seedDB.sizeConnected() / 8; i > 0; i--) {
startPointHash = kelondroBase64Order.enhancedCoder.encode(serverCodings.encodeMD5Raw(Long.toString(i + System.currentTimeMillis()))).substring(2, 2 + yacySeedDB.commonHashLength);
d = yacyDHTAction.dhtDistance(yacyCore.seedDB.mySeed().hash, startPointHash);
d = yacyDHTAction.dhtDistance(wordIndex.seedDB.mySeed().hash, startPointHash);
if (d > (minimumDistance + ((double) i / (double) 10))) {
return startPointHash;
}
@ -183,7 +182,7 @@ public class plasmaDHTChunk {
}
// if that fails, take simply the best start point
if (bestHash == null) {
return yacyCore.seedDB.mySeed().hash.substring(0, 11) + "z";
return wordIndex.seedDB.mySeed().hash.substring(0, 11) + "z";
} else {
return bestHash;
}
@ -219,7 +218,7 @@ public class plasmaDHTChunk {
int wholesize;
urlCache = new HashMap<String, indexURLReference>();
final double maximumDistance = ((double) peerRedundancy * 2) / ((double) yacyCore.seedDB.sizeConnected());
final double maximumDistance = ((double) peerRedundancy * 2) / ((double) wordIndex.seedDB.sizeConnected());
final long timeout = (maxtime < 0) ? Long.MAX_VALUE : System.currentTimeMillis() + maxtime;
while (
(maxcount > refcount) &&

2
source/de/anomic/plasma/plasmaDHTFlush.java
View File

@ -230,7 +230,7 @@ public class plasmaDHTFlush extends Thread {
// handover chunk to transfer worker
if ((newDHTChunk.containerSize() > 0) || (newDHTChunk.getStatus() == plasmaDHTChunk.chunkStatus_FILLED)) {
this.worker = new plasmaDHTTransfer(this.log, this.seed, newDHTChunk, this.gzipBody4Transfer, this.timeout4Transfer, 5);
this.worker = new plasmaDHTTransfer(this.log, this.wordIndex.seedDB, this.seed, newDHTChunk, this.gzipBody4Transfer, this.timeout4Transfer, 5);
this.worker.setTransferMode(plasmaDHTTransfer.TRANSFER_MODE_FLUSH);
this.worker.start();
}

12
source/de/anomic/plasma/plasmaDHTTransfer.java
View File

@ -48,6 +48,7 @@ import de.anomic.server.logging.serverLog;
import de.anomic.yacy.yacyClient;
import de.anomic.yacy.yacyCore;
import de.anomic.yacy.yacySeed;
import de.anomic.yacy.yacySeedDB;
public class plasmaDHTTransfer extends Thread {
@ -72,12 +73,14 @@ public class plasmaDHTTransfer extends Thread {
plasmaDHTChunk dhtChunk;
// other fields
private yacySeedDB seedDB;
private int maxRetry;
private int transferMode = TRANSFER_MODE_DISTRIBUTION;
serverLog log;
public plasmaDHTTransfer(
serverLog log,
serverLog log,
yacySeedDB seedDB,
yacySeed destSeed,
plasmaDHTChunk dhtChunk,
boolean gzipBody,
@ -86,6 +89,7 @@ public class plasmaDHTTransfer extends Thread {
) {
super(new ThreadGroup("TransferIndexThreadGroup"), "TransferIndexWorker_" + destSeed.getName());
this.log = log;
this.seedDB = seedDB;
this.gzipBody4Transfer = gzipBody;
this.timeout4Transfer = timeout;
this.dhtChunk = dhtChunk;
@ -152,7 +156,7 @@ public class plasmaDHTTransfer extends Thread {
// transfering seleted words to remote peer
this.transferStatusMessage = "Running: Transfering chunk to target " + this.seed.hash + "/" + this.seed.getName();
HashMap<String, Object> result = yacyClient.transferIndex(this.seed, this.dhtChunk.containers(), this.dhtChunk.urlCacheMap(), this.gzipBody4Transfer, this.timeout4Transfer);
HashMap<String, Object> result = yacyClient.transferIndex(this.seedDB, this.seed, this.dhtChunk.containers(), this.dhtChunk.urlCacheMap(), this.gzipBody4Transfer, this.timeout4Transfer);
String error = (String) result.get("result");
if (error == null) {
// words successfully transfered
@ -236,7 +240,7 @@ public class plasmaDHTTransfer extends Thread {
return;
// doing a peer ping to the remote seed
int added = yacyClient.publishMySeed(this.seed.getPublicAddress(), this.seed.hash);
int added = yacyClient.publishMySeed(this.seedDB.mySeed(), this.seed.getPublicAddress(), this.seed.hash);
if (added < 0) {
// inc. retry counter
retryCount++;
@ -246,7 +250,7 @@ public class plasmaDHTTransfer extends Thread {
continue;
}
yacyCore.seedDB.getConnected(this.seed.hash);
this.seedDB.getConnected(this.seed.hash);
this.transferStatusMessage = "running";
break;
}

28
source/de/anomic/plasma/plasmaGrafics.java
View File

@ -54,9 +54,9 @@ import java.awt.image.BufferedImage;
import java.util.Date;
import java.util.Iterator;
import de.anomic.yacy.yacyCore;
import de.anomic.yacy.yacySearch;
import de.anomic.yacy.yacySeed;
import de.anomic.yacy.yacySeedDB;
import de.anomic.ymage.ymageMatrix;
import de.anomic.ymage.ymageToolPrint;
@ -125,7 +125,7 @@ public class plasmaGrafics {
private static BufferedImage logo = null; // [MN]
private static long bannerPictureDate = 0; // [MN]
public static ymageMatrix getSearchEventPicture(String eventID) {
public static ymageMatrix getSearchEventPicture(yacySeedDB seedDB, String eventID) {
plasmaSearchEvent event = plasmaSearchEvent.getEvent(eventID);
if (event == null) return null;
yacySearch[] primarySearches = event.getPrimarySearchThreads();
@ -133,7 +133,7 @@ public class plasmaGrafics {
if (primarySearches == null) return null; // this was a local search and there are no threads
// get a copy of a recent network picture
ymageMatrix eventPicture = getNetworkPicture(120000, plasmaSwitchboard.getSwitchboard().getConfig("network.unit.name", "unspecified"), plasmaSwitchboard.getSwitchboard().getConfig("network.unit.description", "unspecified"), COL_BACKGROUND);
ymageMatrix eventPicture = getNetworkPicture(seedDB, 120000, plasmaSwitchboard.getSwitchboard().getConfig("network.unit.name", "unspecified"), plasmaSwitchboard.getSwitchboard().getConfig("network.unit.description", "unspecified"), COL_BACKGROUND);
//if (eventPicture instanceof ymageMatrix) eventPicture = (ymageMatrix) eventPicture; //new ymageMatrix((ymageMatrix) eventPicture);
// TODO: fix cloning of ymageMatrix pictures
@ -177,24 +177,24 @@ public class plasmaGrafics {
return eventPicture;
}
public static ymageMatrix getNetworkPicture(long maxAge, String networkName, String networkTitle, String bgcolor) {
return getNetworkPicture(maxAge, 640, 480, 300, 300, 1000, true, networkName, networkTitle, bgcolor);
public static ymageMatrix getNetworkPicture(yacySeedDB seedDB, long maxAge, String networkName, String networkTitle, String bgcolor) {
return getNetworkPicture(seedDB, maxAge, 640, 480, 300, 300, 1000, true, networkName, networkTitle, bgcolor);
}
public static ymageMatrix getNetworkPicture(long maxAge, int width, int height, int passiveLimit, int potentialLimit, int maxCount, boolean corona, String networkName, String networkTitle, String bgcolor) {
public static ymageMatrix getNetworkPicture(yacySeedDB seedDB, long maxAge, int width, int height, int passiveLimit, int potentialLimit, int maxCount, boolean corona, String networkName, String networkTitle, String bgcolor) {
if ((networkPicture == null) || ((System.currentTimeMillis() - networkPictureDate) > maxAge)) {
drawNetworkPicture(width, height, passiveLimit, potentialLimit, maxCount, corona, networkName, networkTitle, bgcolor);
drawNetworkPicture(seedDB, width, height, passiveLimit, potentialLimit, maxCount, corona, networkName, networkTitle, bgcolor);
}
return networkPicture;
}
private static void drawNetworkPicture(int width, int height, int passiveLimit, int potentialLimit, int maxCount, boolean corona, String networkName, String networkTitle, String bgcolor) {
private static void drawNetworkPicture(yacySeedDB seedDB, int width, int height, int passiveLimit, int potentialLimit, int maxCount, boolean corona, String networkName, String networkTitle, String bgcolor) {
int innerradius = Math.min(width, height) / 5;
int outerradius = innerradius + innerradius * yacyCore.seedDB.sizeConnected() / 100;
int outerradius = innerradius + innerradius * seedDB.sizeConnected() / 100;
if (outerradius > innerradius * 2) outerradius = innerradius * 2;
if (yacyCore.seedDB == null) return; // no other peers known
if (seedDB == null) return; // no other peers known
networkPicture = new ymageMatrix(width, height, ymageMatrix.MODE_SUB, bgcolor);
@ -211,7 +211,7 @@ public class plasmaGrafics {
// draw connected senior and principals
int count = 0;
int totalCount = 0;
Iterator<yacySeed> e = yacyCore.seedDB.seedsConnected(true, false, null, (float) 0.0);
Iterator<yacySeed> e = seedDB.seedsConnected(true, false, null, (float) 0.0);
while (e.hasNext() && count < maxCount) {
seed = (yacySeed) e.next();
@ -224,7 +224,7 @@ public class plasmaGrafics {
// draw disconnected senior and principals that have been seen lately
count = 0;
e = yacyCore.seedDB.seedsSortedDisconnected(false, yacySeed.LASTSEEN);
e = seedDB.seedsSortedDisconnected(false, yacySeed.LASTSEEN);
while (e.hasNext() && count < maxCount) {
seed = (yacySeed) e.next();
if (seed != null) {
@ -238,7 +238,7 @@ public class plasmaGrafics {
// draw juniors that have been seen lately
count = 0;
e = yacyCore.seedDB.seedsSortedPotential(false, yacySeed.LASTSEEN);
e = seedDB.seedsSortedPotential(false, yacySeed.LASTSEEN);
while (e.hasNext() && count < maxCount) {
seed = (yacySeed) e.next();
if (seed != null) {
@ -251,7 +251,7 @@ public class plasmaGrafics {
totalCount += count;
// draw my own peer
drawNetworkPicturePeer(networkPicture, width / 2, height / 2 + 20, innerradius, outerradius, yacyCore.seedDB.mySeed(), COL_WE_DOT, COL_WE_LINE, COL_WE_TEXT, corona);
drawNetworkPicturePeer(networkPicture, width / 2, height / 2 + 20, innerradius, outerradius, seedDB.mySeed(), COL_WE_DOT, COL_WE_LINE, COL_WE_TEXT, corona);
// draw description
networkPicture.setColor(COL_HEADLINE);

5
source/de/anomic/plasma/plasmaProfiling.java
View File

@ -32,6 +32,7 @@ import java.util.Iterator;
import de.anomic.server.serverProfiling;
import de.anomic.server.serverProfiling.Event;
import de.anomic.yacy.yacyCore;
import de.anomic.yacy.yacySeed;
import de.anomic.ymage.ymageChart;
import de.anomic.ymage.ymageMatrix;
@ -41,11 +42,11 @@ public class plasmaProfiling {
public static long lastPPMUpdate = System.currentTimeMillis()- 30000;
public static void updateIndexedPage(plasmaSwitchboardQueue.QueueEntry entry) {
public static void updateIndexedPage(yacySeed mySeed, plasmaSwitchboardQueue.QueueEntry entry) {
if (System.currentTimeMillis() - lastPPMUpdate > 30000) {
// we don't want to do this too often
yacyCore.peerActions.updateMySeed();
serverProfiling.update("ppm", new Long(yacyCore.seedDB.mySeed().getPPM()));
serverProfiling.update("ppm", new Long(mySeed.getPPM()));
lastPPMUpdate = System.currentTimeMillis();
}
serverProfiling.update("indexed", entry.url().toNormalform(true, false));

14
source/de/anomic/plasma/plasmaRankingDistribution.java
View File

@ -52,8 +52,8 @@ import java.util.StringTokenizer;
import de.anomic.server.serverFileUtils;
import de.anomic.server.logging.serverLog;
import de.anomic.yacy.yacyClient;
import de.anomic.yacy.yacyCore;
import de.anomic.yacy.yacySeed;
import de.anomic.yacy.yacySeedDB;
import de.anomic.yacy.yacyVersion;
public final class plasmaRankingDistribution {
@ -73,10 +73,12 @@ public final class plasmaRankingDistribution {
private int method; // of peer selection
private int percentage; // to select any other peer
private String address[]; // of fixed other peer
private yacySeedDB seedDB;
private static Random random = new Random(System.currentTimeMillis());
public plasmaRankingDistribution(serverLog log, File sourcePath, int method, int percentage, String addresses) {
public plasmaRankingDistribution(serverLog log, yacySeedDB seedDB, File sourcePath, int method, int percentage, String addresses) {
this.log = log;
this.seedDB = seedDB;
this.sourcePath = sourcePath;
this.method = method;
this.percentage = percentage;
@ -104,15 +106,15 @@ public final class plasmaRankingDistribution {
log.logFine("no ranking distribution: no transfer method given");
return false;
}
if (yacyCore.seedDB == null) {
if (seedDB == null) {
log.logFine("no ranking distribution: seedDB == null");
return false;
}
if (yacyCore.seedDB.mySeed() == null) {
if (seedDB.mySeed() == null) {
log.logFine("no ranking distribution: mySeed == null");
return false;
}
if (yacyCore.seedDB.mySeed().isVirgin()) {
if (seedDB.mySeed().isVirgin()) {
log.logFine("no ranking distribution: status is virgin");
return false;
}
@ -164,7 +166,7 @@ public final class plasmaRankingDistribution {
if (Thread.currentThread().isInterrupted()) throw new InterruptedException("Shutdown in progress");
// get next target
target = yacyCore.seedDB.anySeedVersion(yacyVersion.YACY_ACCEPTS_RANKING_TRANSMISSION);
target = seedDB.anySeedVersion(yacyVersion.YACY_ACCEPTS_RANKING_TRANSMISSION);
if (target == null) continue;
String targetaddress = target.getPublicAddress();

20
source/de/anomic/plasma/plasmaSearchAPI.java
View File

@ -72,20 +72,16 @@ public class plasmaSearchAPI {
yacySeed seed;
int hc = 0;
prop.put("searchresult_keyhash", startHash);
if (yacyCore.seedDB != null && yacyCore.seedDB.sizeConnected() > 0) {
Iterator<yacySeed> e = yacyCore.dhtAgent.getAcceptRemoteIndexSeeds(startHash);
while (e.hasNext()) {
seed = (yacySeed) e.next();
if (seed != null) {
prop.put("searchresult_hosts_" + hc + "_hosthash", seed.hash);
prop.putHTML("searchresult_hosts_" + hc + "_hostname", seed.hash + " " + seed.get(yacySeed.NAME, "nameless"));
hc++;
}
Iterator<yacySeed> e = yacyCore.dhtAgent.getAcceptRemoteIndexSeeds(startHash);
while (e.hasNext()) {
seed = (yacySeed) e.next();
if (seed != null) {
prop.put("searchresult_hosts_" + hc + "_hosthash", seed.hash);
prop.putHTML("searchresult_hosts_" + hc + "_hostname", seed.hash + " " + seed.get(yacySeed.NAME, "nameless"));
hc++;
}
prop.put("searchresult_hosts", hc);
} else {
prop.put("searchresult_hosts", "0");
}
prop.put("searchresult_hosts", hc);
}
public static plasmaSearchRankingProcess genSearchresult(serverObjects prop, plasmaSwitchboard sb, String keyhash, kelondroBitfield filter) {

9
source/de/anomic/plasma/plasmaSearchEvent.java
View File

@ -48,7 +48,6 @@ import de.anomic.kelondro.kelondroSortStore;
import de.anomic.plasma.plasmaSnippetCache.MediaSnippet;
import de.anomic.server.serverProfiling;
import de.anomic.server.logging.serverLog;
import de.anomic.yacy.yacyCore;
import de.anomic.yacy.yacyDHTAction;
import de.anomic.yacy.yacySearch;
import de.anomic.yacy.yacySeed;
@ -184,7 +183,7 @@ public final class plasmaSearchEvent {
IAmaxcounthash = wordhash;
maxcount = container.size();
}
d = yacyDHTAction.dhtDistance(yacyCore.seedDB.mySeed().hash, wordhash);
d = yacyDHTAction.dhtDistance(wordIndex.seedDB.mySeed().hash, wordhash);
if (d < mindhtdistance) {
// calculate the word hash that is closest to our dht position
mindhtdistance = d;
@ -364,7 +363,7 @@ public final class plasmaSearchEvent {
} else {
// problems with snippet fetch
registerFailure(page.hash(), "no text snippet for URL " + comp.url());
plasmaSnippetCache.failConsequences(snippet, query.id(false));
if (!wordIndex.seedDB.mySeed().isVirgin()) plasmaSnippetCache.failConsequences(snippet, query.id(false));
return null;
}
} else {
@ -677,7 +676,7 @@ public final class plasmaSearchEvent {
Iterator<Map.Entry<String, String>> i1 = abstractJoin.entrySet().iterator();
Map.Entry<String, String> entry1;
String url, urls, peer, peers;
String mypeerhash = yacyCore.seedDB.mySeed().hash;
String mypeerhash = wordIndex.seedDB.mySeed().hash;
boolean mypeerinvolved = false;
int mypeercount;
while (i1.hasNext()) {
@ -788,7 +787,7 @@ public final class plasmaSearchEvent {
// translate host into current IP
int p = host.indexOf(".");
String hash = yacySeed.hexHash2b64Hash(host.substring(p + 1, host.length() - 6));
yacySeed seed = yacyCore.seedDB.getConnected(hash);
yacySeed seed = wordIndex.seedDB.getConnected(hash);
String filename = urlcomps.url().getFile();
String address = null;
if ((seed == null) || ((address = seed.getPublicAddress()) == null)) {

2
source/de/anomic/plasma/plasmaSnippetCache.java
View File

@ -66,7 +66,6 @@ import de.anomic.kelondro.kelondroMSetTools;
import de.anomic.plasma.cache.IResourceInfo;
import de.anomic.plasma.parser.ParserException;
import de.anomic.server.logging.serverLog;
import de.anomic.yacy.yacyCore;
import de.anomic.yacy.yacySearch;
import de.anomic.yacy.yacyURL;
@ -897,7 +896,6 @@ public class plasmaSnippetCache {
public static String failConsequences(TextSnippet snippet, String eventID) {
// problems with snippet fetch
if (yacyCore.seedDB.mySeed().isVirgin()) return snippet.getError() + " (no consequences, no network connection)"; // no consequences if we do not have a network connection
String urlHash = snippet.getUrl().hash();
String querystring = kelondroMSetTools.setToString(snippet.getRemainingHashes(), ' ');
if ((snippet.getErrorCode() == ERROR_SOURCE_LOADING) ||

60
source/de/anomic/plasma/plasmaSwitchboard.java
View File

@ -932,7 +932,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch<plasmaSwitchbo
// start indexing management
log.logConfig("Starting Indexing Management");
String networkName = getConfig("network.unit.name", "");
wordIndex = new plasmaWordIndex(indexPrimaryPath, indexSecondaryPath, networkName, log);
wordIndex = new plasmaWordIndex(networkName, log, indexPrimaryPath, indexSecondaryPath);
crawlResults = new plasmaCrawlResultURLs();
// start yacy core
@ -1166,8 +1166,8 @@ public final class plasmaSwitchboard extends serverAbstractSwitch<plasmaSwitchbo
CRDist1Target = kaskelix.de:8080,yacy.dyndns.org:8000,suma-lab.de:8080
**/
rankingOn = getConfig(RANKING_DIST_ON, "true").equals("true") && networkName.equals("freeworld");
rankingOwnDistribution = new plasmaRankingDistribution(log, new File(rankingPath, getConfig(RANKING_DIST_0_PATH, plasmaRankingDistribution.CR_OWN)), (int) getConfigLong(RANKING_DIST_0_METHOD, plasmaRankingDistribution.METHOD_ANYSENIOR), (int) getConfigLong(RANKING_DIST_0_METHOD, 0), getConfig(RANKING_DIST_0_TARGET, ""));
rankingOtherDistribution = new plasmaRankingDistribution(log, new File(rankingPath, getConfig(RANKING_DIST_1_PATH, plasmaRankingDistribution.CR_OTHER)), (int) getConfigLong(RANKING_DIST_1_METHOD, plasmaRankingDistribution.METHOD_MIXEDSENIOR), (int) getConfigLong(RANKING_DIST_1_METHOD, 30), getConfig(RANKING_DIST_1_TARGET, "kaskelix.de:8080,yacy.dyndns.org:8000,suma-lab.de:8080"));
rankingOwnDistribution = new plasmaRankingDistribution(log, wordIndex.seedDB, new File(rankingPath, getConfig(RANKING_DIST_0_PATH, plasmaRankingDistribution.CR_OWN)), (int) getConfigLong(RANKING_DIST_0_METHOD, plasmaRankingDistribution.METHOD_ANYSENIOR), (int) getConfigLong(RANKING_DIST_0_METHOD, 0), getConfig(RANKING_DIST_0_TARGET, ""));
rankingOtherDistribution = new plasmaRankingDistribution(log, wordIndex.seedDB, new File(rankingPath, getConfig(RANKING_DIST_1_PATH, plasmaRankingDistribution.CR_OTHER)), (int) getConfigLong(RANKING_DIST_1_METHOD, plasmaRankingDistribution.METHOD_MIXEDSENIOR), (int) getConfigLong(RANKING_DIST_1_METHOD, 30), getConfig(RANKING_DIST_1_TARGET, "kaskelix.de:8080,yacy.dyndns.org:8000,suma-lab.de:8080"));
// init facility DB
/*
@ -1218,9 +1218,9 @@ public final class plasmaSwitchboard extends serverAbstractSwitch<plasmaSwitchbo
// init robinson cluster
// before we do that, we wait some time until the seed list is loaded.
while (((System.currentTimeMillis() - startedSeedListAquisition) < 8000) && (yacyCore.seedDB.sizeConnected() == 0)) try {Thread.sleep(1000);} catch (InterruptedException e) {}
while (((System.currentTimeMillis() - startedSeedListAquisition) < 8000) && (this.wordIndex.seedDB.sizeConnected() == 0)) try {Thread.sleep(1000);} catch (InterruptedException e) {}
try {Thread.sleep(1000);} catch (InterruptedException e) {}
this.clusterhashes = yacyCore.seedDB.clusterHashes(getConfig("cluster.peers.yacydomain", ""));
this.clusterhashes = this.wordIndex.seedDB.clusterHashes(getConfig("cluster.peers.yacydomain", ""));
// deploy blocking threads
indexingStorageProcessor = new serverProcessor<indexingQueueEntry>(this, "storeDocumentIndex", 1, null);
@ -1419,7 +1419,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch<plasmaSwitchbo
public yacyURL getURL(String urlhash) {
if (urlhash == null) return null;
if (urlhash.equals(yacyURL.dummyHash)) return null;
if (urlhash.length() == 0) return null;
yacyURL ne = crawlQueues.getURL(urlhash);
if (ne != null) return ne;
indexURLReference le = wordIndex.getURL(urlhash, null, 0);
@ -1707,7 +1707,6 @@ public final class plasmaSwitchboard extends serverAbstractSwitch<plasmaSwitchbo
crawlQueues.close();
log.logConfig("SWITCHBOARD SHUTDOWN STEP 3: sending termination signal to database manager (stand by...)");
wordIndex.close();
yc.close();
log.logConfig("SWITCHBOARD SHUTDOWN TERMINATED");
}
@ -1939,20 +1938,20 @@ public final class plasmaSwitchboard extends serverAbstractSwitch<plasmaSwitchbo
// clean up news
checkInterruption();
try {
if (this.log.isFine()) log.logFine("Cleaning Incoming News, " + yacyCore.newsPool.size(yacyNewsPool.INCOMING_DB) + " entries on stack");
if (yacyCore.newsPool.automaticProcess() > 0) hasDoneSomething = true;
if (this.log.isFine()) log.logFine("Cleaning Incoming News, " + this.wordIndex.newsPool.size(yacyNewsPool.INCOMING_DB) + " entries on stack");
if (this.wordIndex.newsPool.automaticProcess(wordIndex.seedDB) > 0) hasDoneSomething = true;
} catch (IOException e) {}
if (getConfigBool("cleanup.deletionProcessedNews", true)) {
yacyCore.newsPool.clear(yacyNewsPool.PROCESSED_DB);
this.wordIndex.newsPool.clear(yacyNewsPool.PROCESSED_DB);
}
if (getConfigBool("cleanup.deletionPublishedNews", true)) {
yacyCore.newsPool.clear(yacyNewsPool.PUBLISHED_DB);
this.wordIndex.newsPool.clear(yacyNewsPool.PUBLISHED_DB);
}
// clean up seed-dbs
if(getConfigBool("routing.deleteOldSeeds.permission",true)) {
final long deleteOldSeedsTime = getConfigLong("routing.deleteOldSeeds.time",7)*24*3600000;
Iterator<yacySeed> e = yacyCore.seedDB.seedsSortedDisconnected(true,yacySeed.LASTSEEN);
Iterator<yacySeed> e = this.wordIndex.seedDB.seedsSortedDisconnected(true,yacySeed.LASTSEEN);
yacySeed seed = null;
ArrayList<String> deleteQueue = new ArrayList<String>();
checkInterruption();
@ -1966,9 +1965,9 @@ public final class plasmaSwitchboard extends serverAbstractSwitch<plasmaSwitchbo
deleteQueue.add(seed.hash);
}
}
for(int i=0;i<deleteQueue.size();++i) yacyCore.seedDB.removeDisconnected((String)deleteQueue.get(i));
for(int i=0;i<deleteQueue.size();++i) this.wordIndex.seedDB.removeDisconnected((String)deleteQueue.get(i));
deleteQueue.clear();
e = yacyCore.seedDB.seedsSortedPotential(true,yacySeed.LASTSEEN);
e = this.wordIndex.seedDB.seedsSortedPotential(true,yacySeed.LASTSEEN);
checkInterruption();
//clean potential seeds
while(e.hasNext()) {
@ -1980,7 +1979,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch<plasmaSwitchbo
deleteQueue.add(seed.hash);
}
}
for (int i = 0; i < deleteQueue.size(); ++i) yacyCore.seedDB.removePotential((String)deleteQueue.get(i));
for (int i = 0; i < deleteQueue.size(); ++i) this.wordIndex.seedDB.removePotential((String)deleteQueue.get(i));
}
// check if update is available and
@ -2003,7 +2002,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch<plasmaSwitchbo
}
// initiate broadcast about peer startup to spread supporter url
if (yacyCore.newsPool.size(yacyNewsPool.OUTGOING_DB) == 0) {
if (this.wordIndex.newsPool.size(yacyNewsPool.OUTGOING_DB) == 0) {
// read profile
final Properties profile = new Properties();
FileInputStream fileIn = null;
@ -2018,7 +2017,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch<plasmaSwitchbo
if ((homepage != null) && (homepage.length() > 10)) {
Properties news = new Properties();
news.put("homepage", profile.get("homepage"));
yacyCore.newsPool.publishMyNews(yacyNewsRecord.newRecord(yacyNewsPool.CATEGORY_PROFILE_BROADCAST, news));
this.wordIndex.newsPool.publishMyNews(yacyNewsRecord.newRecord(wordIndex.seedDB.mySeed(), yacyNewsPool.CATEGORY_PROFILE_BROADCAST, news));
}
}
/*
@ -2030,7 +2029,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch<plasmaSwitchbo
kelondroCache.setCacheGrowStati(40 * 1024 * 1024, 20 * 1024 * 1024);
*/
// update the cluster set
this.clusterhashes = yacyCore.seedDB.clusterHashes(getConfig("cluster.peers.yacydomain", ""));
this.clusterhashes = this.wordIndex.seedDB.clusterHashes(getConfig("cluster.peers.yacydomain", ""));
return hasDoneSomething;
} catch (InterruptedException e) {
@ -2206,7 +2205,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch<plasmaSwitchbo
checkInterruption();
log.logFine("Not indexed any word in URL " + entry.url() + "; cause: " + noIndexReason);
addURLtoErrorDB(entry.url(), (referrerURL == null) ? null : referrerURL.hash(), entry.initiator(), dc_title, noIndexReason, new kelondroBitfield());
addURLtoErrorDB(entry.url(), (referrerURL == null) ? "" : referrerURL.hash(), entry.initiator(), dc_title, noIndexReason, new kelondroBitfield());
/*
if ((processCase == PROCESSCASE_6_GLOBAL_CRAWLING) && (initiatorPeer != null)) {
if (clusterhashes != null) initiatorPeer.setAlternativeAddress((String) clusterhashes.get(initiatorPeer.hash));
@ -2267,7 +2266,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch<plasmaSwitchbo
crawlResults.stack(
newEntry, // loaded url db entry
queueEntry.initiator(), // initiator peer hash
yacyCore.seedDB.mySeed().hash, // executor peer hash
this.wordIndex.seedDB.mySeed().hash, // executor peer hash
processCase // process case
);
@ -2282,7 +2281,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch<plasmaSwitchbo
indexedPages++;
// update profiling info
plasmaProfiling.updateIndexedPage(queueEntry);
plasmaProfiling.updateIndexedPage(wordIndex.seedDB.mySeed(), queueEntry);
// if this was performed for a remote crawl request, notify requester
yacySeed initiatorPeer = queueEntry.initiatorPeer();
@ -2303,7 +2302,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch<plasmaSwitchbo
this.reference = reference;
}
public void run() {
yacyClient.crawlReceipt(initiatorPeer, "crawl", "fill", "indexed", reference, "");
yacyClient.crawlReceipt(wordIndex.seedDB.mySeed(), initiatorPeer, "crawl", "fill", "indexed", reference, "");
}
}
@ -2486,16 +2485,16 @@ public final class plasmaSwitchboard extends serverAbstractSwitch<plasmaSwitchbo
}
public String dhtShallTransfer() {
if (yacyCore.seedDB == null) {
if (this.wordIndex.seedDB == null) {
return "no DHT distribution: seedDB == null";
}
if (yacyCore.seedDB.mySeed() == null) {
if (this.wordIndex.seedDB.mySeed() == null) {
return "no DHT distribution: mySeed == null";
}
if (yacyCore.seedDB.mySeed().isVirgin()) {
if (this.wordIndex.seedDB.mySeed().isVirgin()) {
return "no DHT distribution: status is virgin";
}
if (yacyCore.seedDB.noDHTActivity()) {
if (this.wordIndex.seedDB.noDHTActivity()) {
return "no DHT distribution: network too small";
}
if (!this.getConfigBool("network.unit.dht", true)) {
@ -2535,7 +2534,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch<plasmaSwitchbo
}
// do the transfer
int peerCount = Math.max(1, (yacyCore.seedDB.mySeed().isJunior()) ?
int peerCount = Math.max(1, (this.wordIndex.seedDB.mySeed().isJunior()) ?
(int) getConfigLong("network.unit.dhtredundancy.junior", 1) :
(int) getConfigLong("network.unit.dhtredundancy.senior", 1)); // set redundancy factor
long starttime = System.currentTimeMillis();
@ -2575,12 +2574,12 @@ public final class plasmaSwitchboard extends serverAbstractSwitch<plasmaSwitchbo
}
public boolean dhtTransferProcess(plasmaDHTChunk dhtChunk, int peerCount) {
if ((yacyCore.seedDB == null) || (yacyCore.seedDB.sizeConnected() == 0)) return false;
if ((this.wordIndex.seedDB == null) || (this.wordIndex.seedDB.sizeConnected() == 0)) return false;
try {
// find a list of DHT-peers
double maxDist = 0.2;
ArrayList<yacySeed> seeds = yacyCore.dhtAgent.getDHTTargets(log, peerCount, Math.min(8, (int) (yacyCore.seedDB.sizeConnected() * maxDist)), dhtChunk.firstContainer().getWordHash(), dhtChunk.lastContainer().getWordHash(), maxDist);
ArrayList<yacySeed> seeds = yacyCore.dhtAgent.getDHTTargets(wordIndex.seedDB, log, peerCount, Math.min(8, (int) (this.wordIndex.seedDB.sizeConnected() * maxDist)), dhtChunk.firstContainer().getWordHash(), dhtChunk.lastContainer().getWordHash(), maxDist);
if (seeds.size() < peerCount) {
log.logWarning("found not enough (" + seeds.size() + ") peers for distribution for dhtchunk [" + dhtChunk.firstContainer().getWordHash() + " .. " + dhtChunk.lastContainer().getWordHash() + "]");
return false;
@ -2606,7 +2605,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch<plasmaSwitchbo
checkInterruption();
if (seedIter.hasNext()) {
plasmaDHTTransfer t = new plasmaDHTTransfer(log, (yacySeed)seedIter.next(), dhtChunk,gzipBody,timeout,retries);
plasmaDHTTransfer t = new plasmaDHTTransfer(log, wordIndex.seedDB, (yacySeed)seedIter.next(), dhtChunk,gzipBody,timeout,retries);
t.start();
transfer.add(t);
} else {
@ -2657,6 +2656,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch<plasmaSwitchbo
String failreason,
kelondroBitfield flags
) {
assert initiator != null;
// create a new errorURL DB entry
plasmaCrawlEntry bentry = new plasmaCrawlEntry(
initiator,

29
source/de/anomic/plasma/plasmaSwitchboardQueue.java
View File

@ -59,7 +59,6 @@ import de.anomic.kelondro.kelondroRow;
import de.anomic.kelondro.kelondroStack;
import de.anomic.plasma.cache.IResourceInfo;
import de.anomic.server.logging.serverLog;
import de.anomic.yacy.yacyCore;
import de.anomic.yacy.yacySeed;
import de.anomic.yacy.yacySeedDB;
import de.anomic.yacy.yacyURL;
@ -68,14 +67,14 @@ public class plasmaSwitchboardQueue {
kelondroStack sbQueueStack;
plasmaCrawlProfile profiles;
plasmaWordIndex index;
plasmaWordIndex wordIndex;
private File sbQueueStackPath;
ConcurrentHashMap<String, QueueEntry> queueInProcess;
public plasmaSwitchboardQueue(plasmaWordIndex index, File sbQueueStackPath, plasmaCrawlProfile profiles) {
public plasmaSwitchboardQueue(plasmaWordIndex wordIndex, File sbQueueStackPath, plasmaCrawlProfile profiles) {
this.sbQueueStackPath = sbQueueStackPath;
this.profiles = profiles;
this.index = index;
this.wordIndex = wordIndex;
this.queueInProcess = new ConcurrentHashMap<String, QueueEntry>();
initQueueStack();
@ -112,12 +111,12 @@ public class plasmaSwitchboardQueue {
if (entry == null) return;
sbQueueStack.push(sbQueueStack.row().newEntry(new byte[][]{
entry.url.toString().getBytes(),
(entry.referrerHash == null) ? yacyURL.dummyHash.getBytes() : entry.referrerHash.getBytes(),
(entry.referrerHash == null) ? "".getBytes() : entry.referrerHash.getBytes(),
kelondroBase64Order.enhancedCoder.encodeLong((entry.ifModifiedSince == null) ? 0 : entry.ifModifiedSince.getTime(), 11).getBytes(),
new byte[]{entry.flags},
(entry.initiator == null) ? yacyURL.dummyHash.getBytes() : entry.initiator.getBytes(),
(entry.initiator == null) ? "".getBytes() : entry.initiator.getBytes(),
kelondroBase64Order.enhancedCoder.encodeLong((long) entry.depth, rowdef.width(5)).getBytes(),
(entry.profileHandle == null) ? yacyURL.dummyHash.getBytes() : entry.profileHandle.getBytes(),
(entry.profileHandle == null) ? "".getBytes() : entry.profileHandle.getBytes(),
(entry.anchorName == null) ? "-".getBytes("UTF-8") : entry.anchorName.getBytes("UTF-8")
}));
}
@ -326,7 +325,7 @@ public class plasmaSwitchboardQueue {
}
public boolean proxy() {
return (initiator == null) || (initiator.equals(yacyURL.dummyHash));
return (initiator == null) || (initiator.equals(initiator.length() == 0));
}
public String initiator() {
@ -334,13 +333,13 @@ public class plasmaSwitchboardQueue {
}
public yacySeed initiatorPeer() {
if ((initiator == null) || (initiator.equals(yacyURL.dummyHash))) return null;
if (initiator.equals(yacyCore.seedDB.mySeed().hash)) {
if ((initiator == null) || (initiator.length() == 0)) return null;
if (initiator.equals(wordIndex.seedDB.mySeed().hash)) {
// normal crawling
return null;
} else {
// this was done for remote peer (a global crawl)
return yacyCore.seedDB.getConnected(initiator);
return wordIndex.seedDB.getConnected(initiator);
}
}
@ -384,8 +383,8 @@ public class plasmaSwitchboardQueue {
public yacyURL referrerURL() {
if (referrerURL == null) {
if ((referrerHash == null) || (referrerHash.equals(yacyURL.dummyHash))) return null;
indexURLReference entry = index.getURL(referrerHash, null, 0);
if ((referrerHash == null) || (referrerHash.equals(initiator.length() == 0))) return null;
indexURLReference entry = wordIndex.getURL(referrerHash, null, 0);
if (entry == null) referrerURL = null; else referrerURL = entry.comp().url();
}
return referrerURL;
@ -408,10 +407,10 @@ public class plasmaSwitchboardQueue {
// 5) local prefetch/crawling (initiator is own seedHash)
// 6) local fetching for global crawling (other known or unknwon initiator)
int processCase = plasmaSwitchboard.PROCESSCASE_0_UNKNOWN;
if ((initiator == null) || (initiator.equals(yacyURL.dummyHash))) {
if ((initiator == null) || (initiator.equals(initiator.length() == 0))) {
// proxy-load
processCase = plasmaSwitchboard.PROCESSCASE_4_PROXY_LOAD;
} else if ((initiator != null) && (initiator.equals(yacyCore.seedDB.mySeed().hash))) {
} else if ((initiator != null) && (initiator.equals(wordIndex.seedDB.mySeed().hash))) {
// normal crawling
processCase = plasmaSwitchboard.PROCESSCASE_5_LOCAL_CRAWLING;
} else {

31
source/de/anomic/plasma/plasmaWordIndex.java
View File

@ -63,6 +63,7 @@ import de.anomic.xml.RSSFeed;
import de.anomic.xml.RSSMessage;
import de.anomic.yacy.yacyCore;
import de.anomic.yacy.yacyDHTAction;
import de.anomic.yacy.yacyNewsPool;
import de.anomic.yacy.yacySeedDB;
import de.anomic.yacy.yacyURL;
@ -79,8 +80,12 @@ public final class plasmaWordIndex implements indexRI {
private final indexCollectionRI collections; // new database structure to replace AssortmentCluster and FileCluster
private serverLog log;
final indexRepositoryReference referenceURL;
public yacySeedDB seedDB;
public yacyNewsPool newsPool;
public plasmaWordIndex(File indexPrimaryRoot, File indexSecondaryRoot, String networkName, serverLog log) {
public plasmaWordIndex(String networkName, serverLog log, File indexPrimaryRoot, File indexSecondaryRoot) {
this.log = log;
File indexPrimaryPath = new File(indexPrimaryRoot, networkName);
File indexPrimaryTextLocation = new File(indexPrimaryPath, "TEXT");
@ -113,6 +118,22 @@ public final class plasmaWordIndex implements indexRI {
// create LURL-db
referenceURL = new indexRepositoryReference(indexSecondaryRoot, networkName);
// create or init seed cache
File networkRoot = new File(indexPrimaryPath, "NETWORK");
networkRoot.mkdirs();
File mySeedFile = new File(networkRoot, "mySeed.txt");
File oldSeedFile = new File(new File(indexPrimaryRoot.getParentFile(), "YACYDB"), "mySeed.txt");
if (oldSeedFile.exists()) oldSeedFile.renameTo(mySeedFile);
seedDB = new yacySeedDB(
new File(networkRoot, "seed.new.db"),
new File(networkRoot, "seed.old.db"),
new File(networkRoot, "seed.pot.db"),
mySeedFile
);
// create or init news database
newsPool = new yacyNewsPool(networkRoot);
}
public void putURL(indexURLReference entry) throws IOException {
@ -236,7 +257,7 @@ public final class plasmaWordIndex implements indexRI {
public void addEntry(String wordHash, indexRWIRowEntry entry, long updateTime, boolean dhtInCase) {
// set dhtInCase depending on wordHash
if ((!dhtInCase) && (yacyDHTAction.shallBeOwnWord(wordHash))) dhtInCase = true;
if ((!dhtInCase) && (yacyDHTAction.shallBeOwnWord(seedDB, wordHash))) dhtInCase = true;
// add the entry
if (dhtInCase) {
@ -256,7 +277,7 @@ public final class plasmaWordIndex implements indexRI {
assert (entries.row().objectsize == indexRWIRowEntry.urlEntryRow.objectsize);
// set dhtInCase depending on wordHash
if ((!dhtInCase) && (yacyDHTAction.shallBeOwnWord(entries.getWordHash()))) dhtInCase = true;
if ((!dhtInCase) && (yacyDHTAction.shallBeOwnWord(seedDB, entries.getWordHash()))) dhtInCase = true;
// add the entry
if (dhtInCase) {
@ -508,8 +529,10 @@ public final class plasmaWordIndex implements indexRI {
dhtOutCache.close();
collections.close();
referenceURL.close();
seedDB.close();
newsPool.close();
}
public indexContainer deleteContainer(String wordHash) {
indexContainer c = new indexContainer(
wordHash,

321
source/de/anomic/server/portForwarding/sch/serverPortForwardingSch.java
View File

@ -1,321 +0,0 @@
// serverPortForwardingSch.java
// -------------------------------------
// part of YACY
// (C) by Michael Peter Christen; mc@anomic.de
// first published on http://www.anomic.de
// Frankfurt, Germany, 2004
//
// This file ist contributed by Martin Thelian
//
// $LastChangedDate: 2006-02-20 23:57:42 +0100 (Mo, 20 Feb 2006) $
// $LastChangedRevision: 1715 $
// $LastChangedBy: borg-0300 $
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation; either version 2 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
//
// Using this software in any meaning (reading, learning, copying, compiling,
// running) means that you agree that the Author(s) is (are) not responsible
// for cost, loss of data or any harm that may be caused directly or indirectly
// by usage of this softare or this documentation. The usage of this software
// is on your own risk. The installation and usage (starting/running) of this
// software may allow other people or application to access your computer and
// any attached devices and is highly dependent on the configuration of the
// software which must be done by the user of the software; the author(s) is
// (are) also not responsible for proper configuration and usage of the
// software, even if provoked by documentation provided together with
// the software.
//
// Any changes to this file according to the GPL as documented in the file
// gpl.txt aside this file in the shipment you received can be done to the
// lines that follows this copyright notice here, but changes must not be
// done inside the copyright notive above. A re-distribution must contain
// the intact and unchanged copyright notice.
// Contributions and changes to the program code must be marked as such.
package de.anomic.server.portForwarding.sch;
import java.io.IOException;
import com.jcraft.jsch.JSch;
import com.jcraft.jsch.ProxyHTTP;
import com.jcraft.jsch.Session;
import com.jcraft.jsch.UIKeyboardInteractive;
import com.jcraft.jsch.UserInfo;
import de.anomic.server.serverInstantBusyThread;
import de.anomic.server.serverSwitch;
import de.anomic.server.logging.serverLog;
import de.anomic.server.portForwarding.serverPortForwarding;
import de.anomic.yacy.yacyClient;
import de.anomic.yacy.yacyCore;
public class serverPortForwardingSch implements serverPortForwarding{
/* ========================================================================
* Constants needed to read properties from the configuration file
* ======================================================================== */
public static final String FORWARDING_HOST = "portForwarding.sch.Host";
public static final String FORWARDING_HOST_PORT = "portForwarding.sch.HostPort";
public static final String FORWARDING_HOST_USER = "portForwarding.sch.HostUser";
public static final String FORWARDING_HOST_PWD = "portForwarding.sch.HostPwd";
public static final String FORWARDING_PORT = "portForwarding.sch.Port";
public static final String FORWARDING_USE_PROXY = "portForwarding.sch.UseProxy";
/* ========================================================================
* Other object fields
* ======================================================================== */
private serverSwitch<?> switchboard;
private String forwardingHost;
private int forwardingHostPort;
private String forwardingHostUser;
private String forwardingHostPwd;
private int forwardingPort;
private boolean useProxy;
private String remoteProxyHost;
private int remoteProxyPort;
private String localHost;
private int localHostPort;
private static Session session = null;
private static serverInstantBusyThread sessionWatcher = null;
private final serverLog log;
public serverPortForwardingSch() {
super();
this.log = new serverLog("PORT_FORWARDING_SCH");
}
public void init(
serverSwitch<?> switchboard,
String localHost,
int localPort
) throws Exception {
try {
if (this.log.isFine()) this.log.logFine("Initializing port forwarding via sch ...");
this.switchboard = switchboard;
this.forwardingHost = switchboard.getConfig(FORWARDING_HOST,"localhost");
this.forwardingHostPort = Integer.valueOf(switchboard.getConfig(FORWARDING_HOST_PORT,"8080")).intValue();
this.forwardingHostUser = switchboard.getConfig(FORWARDING_HOST_USER,"xxx");
this.forwardingHostPwd = switchboard.getConfig(FORWARDING_HOST_PWD,"xxx");
this.forwardingPort = Integer.valueOf(switchboard.getConfig(FORWARDING_PORT,"8080")).intValue();
this.useProxy = Boolean.valueOf(switchboard.getConfig(FORWARDING_USE_PROXY,"false")).booleanValue();
this.localHost = localHost;
this.localHostPort = localPort;
// load remote proxy data
this.remoteProxyHost = switchboard.getConfig("remoteProxyHost","");
try {
this.remoteProxyPort = Integer.parseInt(switchboard.getConfig("remoteProxyPort","3128"));
} catch (NumberFormatException e) {
remoteProxyPort = 3128;
}
// checking if all needed libs are availalbe
String javaClassPath = System.getProperty("java.class.path");
if (javaClassPath.indexOf("jsch") == -1) {
throw new IllegalStateException("Missing library.");
}
} catch (Exception e) {
this.log.logSevere("Unable to initialize port forwarding.",e);
throw e;
}
}
public String getHost() {
return this.forwardingHost;
}
public int getPort() {
return this.forwardingPort;
}
public synchronized void connect() throws IOException {
try{
if ((session != null) && (session.isConnected()))
throw new IOException("Session already connected");
this.log.logInfo("Trying to connect to remote port forwarding host " + this.forwardingHostUser + "@" + this.forwardingHost + ":" + this.forwardingHostPort);
JSch jsch=new JSch();
session=jsch.getSession(this.forwardingHostUser, this.forwardingHost, this.forwardingHostPort);
session.setPassword(this.forwardingHostPwd);
/*
* Setting the StrictHostKeyChecking to ignore unknown
* hosts because of a missing known_hosts file ...
*/
java.util.Properties config = new java.util.Properties();
config.put("StrictHostKeyChecking","no");
session.setConfig(config);
// setting the proxy that should be used
if (this.useProxy) {
session.setProxy(new ProxyHTTP(this.remoteProxyHost, this.remoteProxyPort));
}
// username and password will be given via UserInfo interface.
UserInfo ui= new MyUserInfo(this.forwardingHostPwd);
session.setUserInfo(ui);
// trying to connect ...
session.connect();
// activating remote port forwarding
session.setPortForwardingR(this.forwardingPort, this.localHost, this.localHostPort);
// using a timer task to control if the session remains open
if (sessionWatcher == null) {
this.log.logFine("Deploying port forwarding session watcher thread.");
this.switchboard.deployThread("portForwardingWatcher", "Remote Port Forwarding Watcher", "this thread is used to detect broken connections and to re-establish it if necessary.", null,
sessionWatcher = new serverInstantBusyThread(this, "reconnect", null, null), 30000,30000,30000,1000);
}
this.log.logInfo("Remote port forwarding connection established: " +
this.forwardingHost+ ":" + this.forwardingPort + " -> " +
this.localHost + ":" + this.localHostPort);
}
catch(Exception e){
this.log.logSevere("Unable to connect to remote port forwarding host.",e);
throw new IOException(e.getMessage());
}
}
public synchronized boolean reconnect() throws IOException {
if ((!this.isConnected()) && (!Thread.currentThread().isInterrupted())) {
if (this.log.isFine()) this.log.logFine("Trying to reconnect to port forwarding host.");
this.disconnect();
this.connect();
return this.isConnected();
}
return false;
}
public synchronized void disconnect() throws IOException {
if (session == null) throw new IOException("No connection established.");
// terminating port watcher thread
this.log.logFine("Terminating port forwarding session watcher thread.");
this.switchboard.terminateThread("portForwardingWatcher",true);
sessionWatcher = null;
// disconnection the session
try {
session.disconnect();
this.log.logFine("Successfully disconnected from port forwarding host.");
} catch (Exception e) {
this.log.logSevere("Error while trying to disconnect from port forwarding host.",e);
throw new IOException(e.getMessage());
}
}
public synchronized boolean isConnected() {
if (session == null) return false;
if (!session.isConnected()) return false;
int urls = yacyClient.queryUrlCount(yacyCore.seedDB.mySeed());
return !(urls < 0);
}
class MyUserInfo
implements UserInfo, UIKeyboardInteractive {
String passwd;
public MyUserInfo(String password) {
this.passwd = password;
}
public String getPassword() {
return this.passwd;
}
public boolean promptYesNo(String str){
System.err.println("User was prompted from: " + str);
return true;
}
public String getPassphrase() {
return null;
}
public boolean promptPassphrase(String message) {
System.out.println("promptPassphrase : " + message);
return false;
}
public boolean promptPassword(String message) {
System.out.println("promptPassword : " + message);
return true;
}
/**
* @see com.jcraft.jsch.UserInfo#showMessage(java.lang.String)
*/
public void showMessage(String message) {
System.out.println("Sch has tried to show the following message to the user: " + message);
}
public String[] promptKeyboardInteractive(String destination,
String name,
String instruction,
String[] prompt,
boolean[] echo) {
System.out.println("User was prompted using interactive-keyboard: " +
"\n\tDestination: " + destination +
"\n\tName: " + name +
"\n\tInstruction: " + instruction +
"\n\tPrompt: " + arrayToString2(prompt,"|") +
"\n\techo: " + arrayToString2(echo,"|"));
if ((prompt.length >= 1) && (prompt[0].startsWith("Password")))
return new String[]{this.passwd};
return new String[]{};
}
String arrayToString2(String[] a, String separator) {
StringBuffer result = new StringBuffer(); // start with first element
if (a.length > 0) {
result.append(a[0]);
for (int i=1; i<a.length; i++) {
result.append(separator);
result.append(a[i]);
}
}
return result.toString();
}
String arrayToString2(boolean[] a, String separator) {
StringBuffer result = new StringBuffer(); // start with first element
if (a.length > 0) {
result.append(a[0]);
for (int i=1; i<a.length; i++) {
result.append(separator);
result.append(a[i]);
}
}
return result.toString();
}
}
}

63
source/de/anomic/server/portForwarding/serverPortForwarding.java
View File

@ -1,63 +0,0 @@
// serverPortForwarding.java
// -------------------------------------
// part of YACY
// (C) by Michael Peter Christen; mc@anomic.de
// first published on http://www.anomic.de
// Frankfurt, Germany, 2004
//
// This file ist contributed by Martin Thelian
//
// $LastChangedDate: 2006-02-20 23:57:42 +0100 (Mo, 20 Feb 2006) $
// $LastChangedRevision: 1715 $
// $LastChangedBy: borg-0300 $
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation; either version 2 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
//
// Using this software in any meaning (reading, learning, copying, compiling,
// running) means that you agree that the Author(s) is (are) not responsible
// for cost, loss of data or any harm that may be caused directly or indirectly
// by usage of this softare or this documentation. The usage of this software
// is on your own risk. The installation and usage (starting/running) of this
// software may allow other people or application to access your computer and
// any attached devices and is highly dependent on the configuration of the
// software which must be done by the user of the software; the author(s) is
// (are) also not responsible for proper configuration and usage of the
// software, even if provoked by documentation provided together with
// the software.
//
// Any changes to this file according to the GPL as documented in the file
// gpl.txt aside this file in the shipment you received can be done to the
// lines that follows this copyright notice here, but changes must not be
// done inside the copyright notive above. A re-distribution must contain
// the intact and unchanged copyright notice.
// Contributions and changes to the program code must be marked as such.
package de.anomic.server.portForwarding;
import java.io.IOException;
import de.anomic.server.serverSwitch;
public interface serverPortForwarding {
public void init(serverSwitch<?> switchboard, String localHost, int localPort) throws Exception;
public String getHost();
public int getPort();
public void connect() throws IOException;
public void disconnect() throws IOException;
public boolean reconnect() throws IOException;
public boolean isConnected();
}

220
source/de/anomic/server/portForwarding/upnp/serverPortForwardingUpnp.java
View File

@ -1,220 +0,0 @@
// serverPortForwardingUpnp.java
// -------------------------------------
// part of YACY
// (C) by Michael Peter Christen; mc@anomic.de
// first published on http://www.anomic.de
// Frankfurt, Germany, 2004
//
// This file ist contributed by Martin Thelian
//
// $LastChangedDate: 2006-02-20 23:57:42 +0100 (Mo, 20 Feb 2006) $
// $LastChangedRevision: 1715 $
// $LastChangedBy: theli $
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation; either version 2 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
//
// Using this software in any meaning (reading, learning, copying, compiling,
// running) means that you agree that the Author(s) is (are) not responsible
// for cost, loss of data or any harm that may be caused directly or indirectly
// by usage of this softare or this documentation. The usage of this software
// is on your own risk. The installation and usage (starting/running) of this
// software may allow other people or application to access your computer and
// any attached devices and is highly dependent on the configuration of the
// software which must be done by the user of the software; the author(s) is
// (are) also not responsible for proper configuration and usage of the
// software, even if provoked by documentation provided together with
// the software.
//
// Any changes to this file according to the GPL as documented in the file
// gpl.txt aside this file in the shipment you received can be done to the
// lines that follows this copyright notice here, but changes must not be
// done inside the copyright notive above. A re-distribution must contain
// the intact and unchanged copyright notice.
// Contributions and changes to the program code must be marked as such.
package de.anomic.server.portForwarding.upnp;
import java.io.IOException;
import java.net.InetAddress;
import net.sbbi.upnp.impls.InternetGatewayDevice;
import net.sbbi.upnp.messages.ActionResponse;
import net.sbbi.upnp.messages.UPNPResponseException;
import de.anomic.server.serverSwitch;
import de.anomic.server.logging.serverLog;
import de.anomic.server.portForwarding.serverPortForwarding;
public class serverPortForwardingUpnp implements serverPortForwarding {
private InternetGatewayDevice gateway;
private String localHost;
private int localHostPort;
private String externalAddress = null;
private serverLog log;
public serverPortForwardingUpnp() {
super();
this.log = new serverLog("PORT_FORWARDING_UPNP");
}
public void connect() throws IOException {
try {
if (this.gateway != null)
throw new IOException("Session already connected");
int timeout = 8000;
try {
// trying to get all internet gateways on the local network
this.log.logInfo("Trying to find all available internet gateways.");
InternetGatewayDevice[] IGDs = InternetGatewayDevice.getDevices(timeout);
boolean mapped = false;
if ( IGDs != null ) {
for ( int i = 0; i < IGDs.length; i++ ) {
this.gateway = IGDs[i];
this.log.logInfo("Found device " + this.gateway.getIGDRootDevice().getModelName() );
this.log.logInfo("NAT table size is " + this.gateway.getNatTableSize() );
// now let's open the port
this.log.logInfo("Adding port mapping ...");
mapped = this.gateway.addPortMapping(
"YaCy port forwarding",
null,
this.localHostPort,
this.localHostPort,
this.localHost,
0,
"TCP"
);
if ( mapped ) {
this.log.logInfo("Gateway port " + this.localHostPort + " mapped to " + this.localHost );
this.log.logInfo("Current mappings count is " + this.gateway.getNatMappingsCount() );
mapped = isConnected();
}
if (mapped) break;
}
if (!mapped) {
throw new IOException("Unable to configure the port mapping.");
}
} else {
throw new IOException("No internet gateway device found.");
}
} catch ( IOException ex ) {
throw new Exception("IOException occured during discovery or ports mapping. " + ex.getMessage());
} catch( UPNPResponseException respEx ) {
throw new Exception("UPNP device unhappy " + respEx.getDetailErrorCode() + " " + respEx.getDetailErrorDescription() );
}
} catch (Exception e) {
this.gateway = null;
this.log.logSevere("Unable to connect to remote port forwarding host. ",e);
throw new IOException(e.getMessage());
}
}
public boolean routerAvailable(int timeout) {
try {
InternetGatewayDevice[] IGDs = InternetGatewayDevice.getDevices(timeout);
return (IGDs != null) && (IGDs.length > 0);
} catch (Exception e) {
this.log.logSevere("Unable to determine available routers: " + e.getMessage());
return false;
}
}
public void disconnect() throws IOException {
if (this.gateway == null) throw new IOException("No connection established.");
boolean unmapped;
try {
this.log.logInfo("Trying to disable port mapping ...");
unmapped = this.gateway.deletePortMapping( null, this.localHostPort, "TCP" );
if ( unmapped ) {
this.log.logInfo("Port mapping disabled");
}
} catch (UPNPResponseException e) {
new IOException("Unable to disable port forwarding. " + e.getMessage());
}
}
public String getHost() {
if (this.gateway == null) return "";
if (this.externalAddress == null) {
try {
this.externalAddress = this.gateway.getExternalIPAddress();
} catch (Exception e) {
this.log.logWarning("Unable to get the external address of the gateway");
}
}
return this.externalAddress;
}
public int getPort() {
return this.localHostPort;
}
public void init(serverSwitch<?> switchboard, String localHost, int localPort)
throws Exception {
try {
this.log.logFine("Initializing port forwarding via UPnP ...");
if (localHost.equals("0.0.0.0")) {
this.localHost = InetAddress.getLocalHost().getHostAddress();
} else {
this.localHost = localHost;
}
this.localHostPort = localPort;
// checking if all needed libs are availalbe
String javaClassPath = System.getProperty("java.class.path");
if (javaClassPath.indexOf("sbbi-upnplib") == -1) {
throw new IllegalStateException("Missing library.");
}
// setting the proper xml parser
// if (System.getProperty("javax.xml.parsers.DocumentBuilderFactory", "").equals("")) {
// System.setProperty("javax.xml.parsers.DocumentBuilderFactory", "");
// }
} catch (Exception e) {
this.log.logSevere("Unable to initialize port forwarding.",e);
throw e;
}
}
public boolean isConnected() {
if (this.gateway == null) return false;
try {
ActionResponse resp = this.gateway.getSpecificPortMappingEntry( null, this.localHostPort, "TCP" );
return ( resp != null );
} catch (Exception e) {
this.log.logSevere("Unable to determine the connection status");
return false;
}
}
public boolean reconnect() throws IOException {
if (!this.isConnected()) {
this.log.logFine("Trying to reconnect to port forwarding host.");
this.disconnect();
this.connect();
return this.isConnected();
}
return false;
}
}

82
source/de/anomic/server/serverCore.java
View File

@ -77,11 +77,8 @@ import javax.net.ssl.SSLSocketFactory;
import de.anomic.icap.icapd;
import de.anomic.server.logging.serverLog;
import de.anomic.server.portForwarding.serverPortForwarding;
import de.anomic.tools.PKCS12Tool;
import de.anomic.urlRedirector.urlRedirectord;
import de.anomic.yacy.yacyCore;
import de.anomic.yacy.yacySeed;
public final class serverCore extends serverAbstractBusyThread implements serverBusyThread {
@ -142,9 +139,7 @@ public final class serverCore extends serverAbstractBusyThread implements server
*/
public boolean forceRestart = false;
public static boolean portForwardingEnabled = false;
public static boolean useStaticIP = false;
public static serverPortForwarding portForwarding = null;
private SSLSocketFactory sslSocketFactory = null;
private ServerSocket socket; // listener
@ -261,25 +256,13 @@ public final class serverCore extends serverAbstractBusyThread implements server
// updating the port information
//yacyCore.seedDB.mySeed.put(yacySeed.PORT,Integer.toString(bindAddress.getPort()));
yacyCore.seedDB.mySeed().put(yacySeed.PORT, extendedPort);
//yacyCore.seedDB.mySeed().put(yacySeed.PORT, extendedPort);
} catch (Exception e) {
String errorMsg = "FATAL ERROR: " + e.getMessage() + " - probably root access rights needed. check port number";
this.log.logSevere(errorMsg);
System.out.println(errorMsg);
System.exit(0);
}
// init port forwarding
try {
this.initPortForwarding();
} catch (Exception e) {
this.log.logSevere("Unable to initialize server port forwarding.",e);
this.switchboard.setConfig("portForwardingEnabled","false");
} catch (Error e) {
this.log.logSevere("Unable to initialize server port forwarding.",e);
this.switchboard.setConfig("portForwardingEnabled","false");
}
}
public static int getPortNr(String extendedPortString) {
@ -340,57 +323,6 @@ public final class serverCore extends serverAbstractBusyThread implements server
: new InetSocketAddress(bindIP, bindPort);
}
public void initPortForwarding() throws Exception {
// doing the port forwarding stuff
if (this.switchboard.getConfigBool("portForwarding.Enabled",false)) {
this.log.logInfo("Initializing port forwarding ...");
try {
// getting the port forwarding type to use
String forwardingType = this.switchboard.getConfig("portForwarding.Type","none");
// loading port forwarding class
this.log.logInfo("Trying to load port forwarding class for forwarding type '" + forwardingType + "'.");
String forwardingClass = this.switchboard.getConfig("portForwarding." + forwardingType ,"");
Class<?> forwarderClass = Class.forName(forwardingClass);
serverCore.portForwarding = (serverPortForwarding) forwarderClass.newInstance();
// initializing port forwarding
String localHost = this.socket.getInetAddress().getHostName();
Integer localPort = new Integer(this.socket.getLocalPort());
serverCore.portForwarding.init(
this.switchboard,
localHost,
localPort.intValue());
// connection to port forwarding host
serverCore.portForwarding.connect();
serverCore.portForwardingEnabled = true;
yacyCore.seedDB.mySeed().put(yacySeed.IP, serverDomains.myPublicIP());
yacyCore.seedDB.mySeed().put(yacySeed.PORT,Integer.toString(serverCore.portForwarding.getPort()));
} catch (Exception e) {
serverCore.portForwardingEnabled = false;
this.switchboard.setConfig("portForwarding.Enabled", "false");
throw e;
} catch (Error e) {
serverCore.portForwardingEnabled = false;
this.switchboard.setConfig("portForwarding.Enabled", "false");
throw e;
}
} else {
serverCore.portForwardingEnabled = false;
serverCore.portForwarding = null;
yacyCore.seedDB.mySeed().put(yacySeed.IP, serverDomains.myPublicIP());
yacyCore.seedDB.mySeed().put(yacySeed.PORT,Integer.toString(serverCore.getPortNr(this.switchboard.getConfig("port", "8080"))));
}
if(! this.switchboard.getConfig("staticIP", "").equals(""))
serverCore.useStaticIP=true;
}
public void open() {
this.log.logConfig("* server started on " + serverDomains.myPublicLocalIP() + ":" + this.extendedPort);
}
@ -494,18 +426,6 @@ public final class serverCore extends serverAbstractBusyThread implements server
}
}
// closing the port forwarding channel
if ((portForwardingEnabled) && (portForwarding != null) ) {
try {
this.log.logInfo("Shutdown port forwarding ...");
portForwarding.disconnect();
portForwardingEnabled = false;
portForwarding = null;
} catch (Exception e) {
this.log.logWarning("Unable to shutdown the port forwarding channel.");
}
}
// close the serverchannel and socket
try {
this.log.logInfo("Closing server socket ...");

37
source/de/anomic/server/serverDomains.java
View File

@ -596,34 +596,19 @@ public class serverDomains {
}
public static String myPublicIP() {
try {
// if a static IP was configured, we have to return it here ...
plasmaSwitchboard sb = plasmaSwitchboard.getSwitchboard();
if (sb != null) {
String staticIP = sb.getConfig("staticIP", "");
if ((!staticIP.equals(""))) {
return staticIP;
}
// if a static IP was configured, we have to return it here ...
plasmaSwitchboard sb = plasmaSwitchboard.getSwitchboard();
if (sb != null) {
String staticIP = sb.getConfig("staticIP", "");
if ((!staticIP.equals(""))) {
return staticIP;
}
// If port forwarding was enabled we need to return the remote IP
// Address
if ((serverCore.portForwardingEnabled) && (serverCore.portForwarding != null)) {
// does not return serverCore.portForwarding.getHost(), because
// hostnames are not valid, except in DebugMode
return InetAddress.getByName(
serverCore.portForwarding.getHost()).getHostAddress();
}
// otherwise we return the real IP address of this host
InetAddress pLIP = myPublicLocalIP();
if (pLIP != null) return pLIP.getHostAddress();
return null;
} catch (java.net.UnknownHostException e) {
System.err.println("ERROR: (internal) " + e.getMessage());
return null;
}
// otherwise we return the real IP address of this host
InetAddress pLIP = myPublicLocalIP();
if (pLIP != null) return pLIP.getHostAddress();
return null;
}
public static InetAddress myPublicLocalIP() {

3
source/de/anomic/urlRedirector/urlRedirectord.java
View File

@ -17,7 +17,6 @@ import de.anomic.server.serverCore;
import de.anomic.server.serverHandler;
import de.anomic.server.logging.serverLog;
import de.anomic.server.serverCore.Session;
import de.anomic.yacy.yacyCore;
import de.anomic.yacy.yacyURL;
public class urlRedirectord implements serverHandler {
@ -199,7 +198,7 @@ public class urlRedirectord implements serverHandler {
reasonString = switchboard.crawlStacker.stackCrawl(
reqURL,
null,
yacyCore.seedDB.mySeed().hash,
switchboard.wordIndex.seedDB.mySeed().hash,
"URL Redirector",
new Date(),
0,

161
source/de/anomic/yacy/yacyClient.java
View File

@ -91,7 +91,7 @@ import de.anomic.xml.RSSReader;
public final class yacyClient {
public static int publishMySeed(String address, String otherHash) {
public static int publishMySeed(yacySeed mySeed, String address, String otherHash) {
// this is called to enrich the seed information by
// - own address (if peer is behind a nat/router)
// - check peer type (virgin/junior/senior/principal)
@ -116,7 +116,7 @@ public final class yacyClient {
for (int retry = 0; retry < 4; retry++) try {
// generate request
post.add(new StringPart("count", "20"));
post.add(new StringPart("seed", yacyCore.seedDB.mySeed().genSeedStr(salt)));
post.add(new StringPart("seed", mySeed.genSeedStr(salt)));
yacyCore.log.logFine("yacyClient.publishMySeed thread '" + Thread.currentThread().getName() + "' contacting peer at " + address);
// send request
byte[] content = wput("http://" + address + "/yacy/hello.html", yacySeed.b64Hash2hexHash(otherHash) + ".yacyh", post);
@ -141,7 +141,6 @@ public final class yacyClient {
// check consistency with expectation
yacySeed otherPeer = null;
float otherPeerVersion = 0;
String seed;
if ((otherHash != null) &&
(otherHash.length() > 0) &&
@ -154,72 +153,50 @@ public final class yacyClient {
yacyCore.log.logFine("yacyClient.publishMySeed: consistency error: other peer '" + ((otherPeer==null)?"unknown":otherPeer.getName()) + "' wrong");
return -1; // no success
}
otherPeerVersion = otherPeer.getVersion();
}
}
// set my own seed according to new information
// we overwrite our own IP number only, if we do not portForwarding
if (serverCore.portForwardingEnabled || serverCore.useStaticIP) {
yacyCore.seedDB.mySeed().put(yacySeed.IP, serverDomains.myPublicIP());
// we overwrite our own IP number only
if (serverCore.useStaticIP) {
mySeed.put(yacySeed.IP, serverDomains.myPublicIP());
} else {
yacyCore.seedDB.mySeed().put(yacySeed.IP, (String) result.get("yourip"));
mySeed.put(yacySeed.IP, (String) result.get("yourip"));
}
/* If we have port forwarding enabled but the other peer uses a too old yacy version
* we can ignore the seed-type that was reported by the peer.
*
* Otherwise we have to change our seed-type
*
* @see serverCore#portForwardingEnabled
// change our seed-type
String mytype = (String) result.get(yacySeed.YOURTYPE);
if (mytype == null) { mytype = ""; }
yacyAccessible accessible = new yacyAccessible();
if (mytype.equals(yacySeed.PEERTYPE_SENIOR)||mytype.equals(yacySeed.PEERTYPE_PRINCIPAL)) {
accessible.IWasAccessed = true;
if (mySeed.isPrincipal()) {
mytype = yacySeed.PEERTYPE_PRINCIPAL;
}
} else {
accessible.IWasAccessed = false;
}
accessible.lastUpdated = System.currentTimeMillis();
yacyCore.amIAccessibleDB.put(otherHash, accessible);
/*
* If we were reported as junior we have to check if your port forwarding channel is broken
* If this is true we try to reconnect the sch channel to the remote server now.
*/
if (!serverCore.portForwardingEnabled || otherPeerVersion >= yacyVersion.YACY_SUPPORTS_PORT_FORWARDING) {
String mytype = (String) result.get(yacySeed.YOURTYPE);
if (mytype == null) { mytype = ""; }
yacyAccessible accessible = new yacyAccessible();
if (mytype.equals(yacySeed.PEERTYPE_SENIOR)||mytype.equals(yacySeed.PEERTYPE_PRINCIPAL)) {
accessible.IWasAccessed = true;
if (yacyCore.seedDB.mySeed().isPrincipal()) {
mytype = yacySeed.PEERTYPE_PRINCIPAL;
}
} else {
accessible.IWasAccessed = false;
}
accessible.lastUpdated = System.currentTimeMillis();
yacyCore.amIAccessibleDB.put(otherHash, accessible);
/*
* If we were reported as junior we have to check if your port forwarding channel is broken
* If this is true we try to reconnect the sch channel to the remote server now.
*/
if (mytype.equalsIgnoreCase(yacySeed.PEERTYPE_JUNIOR)) {
yacyCore.log.logInfo("yacyClient.publishMySeed: Peer '" + ((otherPeer==null)?"unknown":otherPeer.getName()) + "' reported us as junior.");
if (serverCore.portForwardingEnabled) {
if (!Thread.currentThread().isInterrupted() &&
serverCore.portForwarding != null &&
!serverCore.portForwarding.isConnected()
) {
yacyCore.log.logWarning("yacyClient.publishMySeed: Broken portForwarding channel detected. Trying to reconnect ...");
try {
serverCore.portForwarding.reconnect();
} catch (IOException e) {
yacyCore.log.logWarning("yacyClient.publishMySeed: Unable to reconnect to port forwarding host.");
}
}
}
} else if ((mytype.equalsIgnoreCase(yacySeed.PEERTYPE_SENIOR)) ||
(mytype.equalsIgnoreCase(yacySeed.PEERTYPE_PRINCIPAL))) {
yacyCore.log.logFine("yacyClient.publishMySeed: Peer '" + ((otherPeer==null)?"unknown":otherPeer.getName()) + "' reported us as " + mytype + ", accepted other peer.");
} else {
// wrong type report
yacyCore.log.logFine("yacyClient.publishMySeed: Peer '" + ((otherPeer==null)?"unknown":otherPeer.getName()) + "' reported us as " + mytype + ", rejecting other peer.");
return -1;
}
if (yacyCore.seedDB.mySeed().orVirgin().equals(yacySeed.PEERTYPE_VIRGIN))
yacyCore.seedDB.mySeed().put(yacySeed.PEERTYPE, mytype);
if (mytype.equalsIgnoreCase(yacySeed.PEERTYPE_JUNIOR)) {
yacyCore.log.logInfo("yacyClient.publishMySeed: Peer '" + ((otherPeer==null)?"unknown":otherPeer.getName()) + "' reported us as junior.");
} else if ((mytype.equalsIgnoreCase(yacySeed.PEERTYPE_SENIOR)) ||
(mytype.equalsIgnoreCase(yacySeed.PEERTYPE_PRINCIPAL))) {
yacyCore.log.logFine("yacyClient.publishMySeed: Peer '" + ((otherPeer==null)?"unknown":otherPeer.getName()) + "' reported us as " + mytype + ", accepted other peer.");
} else {
// wrong type report
yacyCore.log.logFine("yacyClient.publishMySeed: Peer '" + ((otherPeer==null)?"unknown":otherPeer.getName()) + "' reported us as " + mytype + ", rejecting other peer.");
return -1;
}
if (mySeed.orVirgin().equals(yacySeed.PEERTYPE_VIRGIN))
mySeed.put(yacySeed.PEERTYPE, mytype);
final String error = yacyCore.seedDB.mySeed().isProper();
final String error = mySeed.isProper();
if (error != null) {
yacyCore.log.logSevere("yacyClient.publishMySeed mySeed error - not proper: " + error);
return -1;
@ -328,8 +305,8 @@ public final class yacyClient {
private static byte[] postToFile(final yacySeed target, final String filename, final List<Part> post) throws IOException {
return wput("http://" + target.getClusterAddress() + "/yacy/" + filename, target.getHexHash() + ".yacyh", post);
}
private static byte[] postToFile(final String targetHash, final String filename, final List<Part> post) throws IOException {
return wput("http://" + targetAddress(targetHash) + "/yacy/" + filename, yacySeed.b64Hash2hexHash(targetHash)+ ".yacyh", post);
private static byte[] postToFile(yacySeedDB seedDB, final String targetHash, final String filename, final List<Part> post) throws IOException {
return wput("http://" + targetAddress(seedDB, targetHash) + "/yacy/" + filename, yacySeed.b64Hash2hexHash(targetHash)+ ".yacyh", post);
}
public static yacySeed querySeed(yacySeed target, String seedHash) {
@ -376,7 +353,6 @@ public final class yacyClient {
public static int queryUrlCount(yacySeed target) {
if (target == null) { return -1; }
if (yacyCore.seedDB.mySeed() == null) return -1;
// prepare request
final String salt = crypt.randomSalt();
@ -405,10 +381,9 @@ public final class yacyClient {
}
}
public static RSSFeed queryRemoteCrawlURLs(yacySeed target, int count) {
public static RSSFeed queryRemoteCrawlURLs(yacySeedDB seedDB, yacySeed target, int count) {
// returns a list of
if (target == null) { return null; }
if (yacyCore.seedDB.mySeed() == null) return null;
// prepare request
final String salt = crypt.randomSalt();
@ -427,7 +402,7 @@ public final class yacyClient {
yacyCore.log.logWarning("yacyClient.queryRemoteCrawlURLs failed asking peer '" + target.getName() + "': probably bad response from remote peer");
System.out.println("***DEBUG*** rss input = " + new String(result));
target.put(yacySeed.RCOUNT, "0");
yacyCore.seedDB.update(target.hash, target); // overwrite number of remote-available number to avoid that this peer is called again (until update is done by peer ping)
seedDB.update(target.hash, target); // overwrite number of remote-available number to avoid that this peer is called again (until update is done by peer ping)
//e.printStackTrace();
return null;
}
@ -439,6 +414,7 @@ public final class yacyClient {
}
public static String[] search(
yacySeed mySeed,
String wordhashes,
String excludehashes,
String urlhashes,
@ -476,7 +452,7 @@ public final class yacyClient {
// prepare request
final String salt = crypt.randomSalt();
final List<Part> post = yacyNetwork.basicRequestPost(plasmaSwitchboard.getSwitchboard(), target.hash, salt);
post.add(new StringPart("myseed", yacyCore.seedDB.mySeed().genSeedStr(salt)));
post.add(new StringPart("myseed", mySeed.genSeedStr(salt)));
post.add(new StringPart("count", Integer.toString(Math.max(10, count))));
post.add(new StringPart("resource", ((global) ? "global" : "local")));
post.add(new StringPart("partitions", Integer.toString(partitions)));
@ -590,7 +566,7 @@ public final class yacyClient {
// passed all checks, store url
try {
wordIndex.putURL(urlEntry);
crawlResults.stack(urlEntry, yacyCore.seedDB.mySeed().hash, target.hash, 2);
crawlResults.stack(urlEntry, mySeed.hash, target.hash, 2);
} catch (IOException e) {
yacyCore.log.logSevere("could not store search result", e);
continue; // db-error
@ -681,11 +657,10 @@ public final class yacyClient {
return urls;
}
public static HashMap<String, String> permissionMessage(String targetHash) {
public static HashMap<String, String> permissionMessage(yacySeedDB seedDB, String targetHash) {
// ask for allowed message size and attachement size
// if this replies null, the peer does not answer
if (yacyCore.seedDB == null || yacyCore.seedDB.mySeed() == null) { return null; }
// prepare request
final String salt = crypt.randomSalt();
final List<Part> post = yacyNetwork.basicRequestPost(plasmaSwitchboard.getSwitchboard(), targetHash, salt);
@ -693,7 +668,7 @@ public final class yacyClient {
// send request
try {
final byte[] content = postToFile(targetHash, "message.html", post);
final byte[] content = postToFile(seedDB, targetHash, "message.html", post);
final HashMap<String, String> result = nxTools.table(content, "UTF-8");
return result;
} catch (Exception e) {
@ -703,14 +678,14 @@ public final class yacyClient {
}
}
public static HashMap<String, String> postMessage(String targetHash, String subject, byte[] message) {
public static HashMap<String, String> postMessage(yacySeedDB seedDB, String targetHash, String subject, byte[] message) {
// this post a message to the remote message board
// prepare request
final String salt = crypt.randomSalt();
final List<Part> post = yacyNetwork.basicRequestPost(plasmaSwitchboard.getSwitchboard(), targetHash, salt);
post.add(new StringPart("process", "post"));
post.add(new StringPart("myseed", yacyCore.seedDB.mySeed().genSeedStr(salt)));
post.add(new StringPart("myseed", seedDB.mySeed().genSeedStr(salt)));
post.add(new StringPart("subject", subject));
try {
post.add(new StringPart("message", new String(message, "UTF-8")));
@ -720,7 +695,7 @@ public final class yacyClient {
// send request
try {
final byte[] content = postToFile(targetHash, "message.html", post);
final byte[] content = postToFile(seedDB, targetHash, "message.html", post);
final HashMap<String, String> result = nxTools.table(content, "UTF-8");
return result;
} catch (Exception e) {
@ -729,13 +704,13 @@ public final class yacyClient {
}
}
public static String targetAddress(String targetHash) {
public static String targetAddress(yacySeedDB seedDB, String targetHash) {
// find target address
String address;
if (targetHash.equals(yacyCore.seedDB.mySeed().hash)) {
address = yacyCore.seedDB.mySeed().getClusterAddress();
if (targetHash.equals(seedDB.mySeed().hash)) {
address = seedDB.mySeed().getClusterAddress();
} else {
final yacySeed targetSeed = yacyCore.seedDB.getConnected(targetHash);
final yacySeed targetSeed = seedDB.getConnected(targetHash);
if (targetSeed == null) { return null; }
address = targetSeed.getClusterAddress();
}
@ -818,10 +793,10 @@ public final class yacyClient {
return "wrong protocol: " + protocol;
}
public static HashMap<String, String> crawlReceipt(yacySeed target, String process, String result, String reason, indexURLReference entry, String wordhashes) {
public static HashMap<String, String> crawlReceipt(yacySeed mySeed, yacySeed target, String process, String result, String reason, indexURLReference entry, String wordhashes) {
assert (target != null);
assert (yacyCore.seedDB.mySeed() != null);
assert (yacyCore.seedDB.mySeed() != target);
assert (mySeed != null);
assert (mySeed != target);
/*
the result can have one of the following values:
@ -868,7 +843,7 @@ public final class yacyClient {
}
}
public static HashMap<String, Object> transferIndex(yacySeed targetSeed, indexContainer[] indexes, HashMap<String, indexURLReference> urlCache, boolean gzipBody, int timeout) {
public static HashMap<String, Object> transferIndex(yacySeedDB seedDB, yacySeed targetSeed, indexContainer[] indexes, HashMap<String, indexURLReference> urlCache, boolean gzipBody, int timeout) {
HashMap<String, Object> resultObj = new HashMap<String, Object>();
int payloadSize = 0;
@ -888,7 +863,7 @@ public final class yacyClient {
}
// transfer the RWI without the URLs
HashMap<String, String> in = transferRWI(targetSeed, indexes, gzipBody, timeout);
HashMap<String, String> in = transferRWI(seedDB, targetSeed, indexes, gzipBody, timeout);
resultObj.put("resultTransferRWI", in);
if (in == null) {
@ -904,7 +879,7 @@ public final class yacyClient {
}
if (!(result.equals("ok"))) {
targetSeed.setFlagAcceptRemoteIndex(false);
yacyCore.seedDB.update(targetSeed.hash, targetSeed);
seedDB.update(targetSeed.hash, targetSeed);
resultObj.put("result", result);
return resultObj;
}
@ -929,7 +904,7 @@ public final class yacyClient {
}
}
in = transferURL(targetSeed, urls, gzipBody, timeout);
in = transferURL(seedDB, targetSeed, urls, gzipBody, timeout);
resultObj.put("resultTransferURL", in);
if (in == null) {
@ -945,7 +920,7 @@ public final class yacyClient {
}
if (!(result.equals("ok"))) {
targetSeed.setFlagAcceptRemoteIndex(false);
yacyCore.seedDB.update(targetSeed.hash, targetSeed);
seedDB.update(targetSeed.hash, targetSeed);
resultObj.put("result",result);
return resultObj;
}
@ -958,7 +933,7 @@ public final class yacyClient {
}
}
private static HashMap<String, String> transferRWI(yacySeed targetSeed, indexContainer[] indexes, boolean gzipBody, int timeout) {
private static HashMap<String, String> transferRWI(yacySeedDB seedDB, yacySeed targetSeed, indexContainer[] indexes, boolean gzipBody, int timeout) {
final String address = targetSeed.getPublicAddress();
if (address == null) { return null; }
@ -1002,7 +977,7 @@ public final class yacyClient {
final ArrayList<String> v = nxTools.strings(content, "UTF-8");
// this should return a list of urlhashes that are unknown
if ((v != null) && (v.size() > 0)) {
yacyCore.seedDB.mySeed().incSI(indexcount);
seedDB.mySeed().incSI(indexcount);
}
final HashMap<String, String> result = nxTools.table(v);
@ -1015,7 +990,7 @@ public final class yacyClient {
}
}
private static HashMap<String, String> transferURL(yacySeed targetSeed, indexURLReference[] urls, boolean gzipBody, int timeout) {
private static HashMap<String, String> transferURL(yacySeedDB seedDB, yacySeed targetSeed, indexURLReference[] urls, boolean gzipBody, int timeout) {
// this post a message to the remote message board
final String address = targetSeed.getPublicAddress();
if (address == null) { return null; }
@ -1048,7 +1023,7 @@ public final class yacyClient {
final ArrayList<String> v = nxTools.strings(content, "UTF-8");
if ((v != null) && (v.size() > 0)) {
yacyCore.seedDB.mySeed().incSU(urlc);
seedDB.mySeed().incSU(urlc);
}
HashMap<String, String> result = nxTools.table(v);
@ -1094,15 +1069,15 @@ public final class yacyClient {
final plasmaSwitchboard sb = new plasmaSwitchboard(new File(args[0]), "httpProxy.init", "DATA/SETTINGS/yacy.conf", false);
/*final yacyCore core =*/ new yacyCore(sb);
yacyCore.peerActions.loadSeedLists();
final yacySeed target = yacyCore.seedDB.getConnected(args[1]);
final yacySeed target = sb.wordIndex.seedDB.getConnected(args[1]);
final String wordhashe = indexWord.word2hash("test");
//System.out.println("permission=" + permissionMessage(args[1]));
final byte[] content = HttpClient.wget(
"http://" + target.getPublicAddress() + "/yacy/search.html" +
"?myseed=" + yacyCore.seedDB.mySeed().genSeedStr(null) +
"?myseed=" + sb.wordIndex.seedDB.mySeed().genSeedStr(null) +
"&youare=" + target.hash + "&key=" +
"&myseed=" + yacyCore.seedDB.mySeed() .genSeedStr(null) +
"&myseed=" + sb.wordIndex.seedDB.mySeed() .genSeedStr(null) +
"&count=10" +
"&resource=global" +
"&query=" + wordhashe +

179
source/de/anomic/yacy/yacyCore.java
View File

@ -74,7 +74,6 @@ import de.anomic.plasma.plasmaSwitchboard;
import de.anomic.server.serverCore;
import de.anomic.server.serverDate;
import de.anomic.server.serverSemaphore;
import de.anomic.server.serverSwitch;
import de.anomic.server.logging.serverLog;
import de.anomic.xml.RSSFeed;
import de.anomic.xml.RSSMessage;
@ -83,8 +82,6 @@ public class yacyCore {
// statics
public static final ThreadGroup publishThreadGroup = new ThreadGroup("publishThreadGroup");
public static yacySeedDB seedDB = null;
public static yacyNewsPool newsPool = null;
public static final HashMap<String, String> seedUploadMethods = new HashMap<String, String>();
public static yacyPeerActions peerActions = null;
public static yacyDHTAction dhtAgent = null;
@ -92,7 +89,6 @@ public class yacyCore {
public static long lastOnlineTime = 0;
/** pseudo-random key derived from a time-interval while YaCy startup*/
public static long speedKey = 0;
public static File yacyDBPath = null;
public static final Map<String, yacyAccessible> amIAccessibleDB = Collections.synchronizedMap(new HashMap<String, yacyAccessible>()); // Holds PeerHash / yacyAccessible Relations
// constants for PeerPing behaviour
private static final int PING_INITIAL = 10;
@ -107,13 +103,8 @@ public class yacyCore {
// public static boolean terminate = false;
// class variables
private int lastSeedUpload_seedDBSize = 0;
public long lastSeedUpload_timeStamp = System.currentTimeMillis();
// private String lastSeedUpload_myPeerType = "";
private String lastSeedUpload_myIP = "";
private static int onlineMode = 1;
private plasmaSwitchboard switchboard;
private plasmaSwitchboard sb;
public static int yacyTime() {
// the time since startup of yacy in seconds
@ -123,37 +114,21 @@ public class yacyCore {
public yacyCore(plasmaSwitchboard sb) {
long time = System.currentTimeMillis();
this.switchboard = sb;
switchboard.setConfig("yacyStatus", "");
this.sb = sb;
sb.setConfig("yacyStatus", "");
// create a peer news channel
RSSFeed peernews = RSSFeed.channels(channelName);
peernews.setMaxsize(1000);
peernews.addMessage(new RSSMessage("YaCy started", ""));
// create a yacy db
yacyDBPath = sb.getConfigPath("yacyDB", "DATA/YACYDB");
if (!yacyDBPath.exists()) { yacyDBPath.mkdir(); }
// create or init seed cache
seedDB = new yacySeedDB(
sb,
new File(yacyDBPath, "seed2.new.db"),
new File(yacyDBPath, "seed2.old.db"),
new File(yacyDBPath, "seed2.pot.db"));
// create or init news database
newsPool = new yacyNewsPool(yacyDBPath);
loadSeedUploadMethods();
// deploy peer actions
peerActions = new yacyPeerActions(seedDB, switchboard);
dhtAgent = new yacyDHTAction(seedDB);
peerActions = new yacyPeerActions(sb.wordIndex.seedDB, sb);
dhtAgent = new yacyDHTAction(sb.wordIndex.seedDB);
peerActions.deploy(dhtAgent);
peerActions.deploy(new yacyNewsAction(newsPool));
lastSeedUpload_seedDBSize = seedDB.sizeConnected();
peerActions.deploy(new yacyNewsAction(sb.wordIndex.newsPool));
log.logConfig("CORE INITIALIZED");
// ATTENTION, VERY IMPORTANT: before starting the thread, the httpd yacy server must be running!
@ -161,7 +136,7 @@ public class yacyCore {
speedKey = System.currentTimeMillis() - time;
// start with a seedList update to propagate out peer, if possible
onlineMode = Integer.parseInt(switchboard.getConfig("onlineMode", "1"));
onlineMode = Integer.parseInt(sb.getConfig("onlineMode", "1"));
//lastSeedUpdate = universalTime();
lastOnlineTime = 0;
@ -173,21 +148,16 @@ public class yacyCore {
log.logConfig("YOU ARE OFFLINE! ---");
log.logConfig("--- TO START BOOTSTRAPING, YOU MUST USE THE PROXY,");
log.logConfig("--- OR HIT THE BUTTON 'go online'");
log.logConfig("--- ON THE STATUS PAGE http://localhost:" + serverCore.getPortNr(switchboard.getConfig("port", "8080")) + "/Status.html");
log.logConfig("--- ON THE STATUS PAGE http://localhost:" + serverCore.getPortNr(sb.getConfig("port", "8080")) + "/Status.html");
}
}
public synchronized void close() {
seedDB.close();
newsPool.close();
}
synchronized static public void triggerOnlineAction() {
lastOnlineTime = System.currentTimeMillis();
}
public final boolean online() {
onlineMode = Integer.parseInt(switchboard.getConfig("onlineMode", "1"));
onlineMode = Integer.parseInt(sb.getConfig("onlineMode", "1"));
return ((onlineMode == 2) || ((System.currentTimeMillis() - lastOnlineTime) < 10000));
}
@ -212,37 +182,37 @@ public class yacyCore {
yacyCore.log.logDebug("***DEBUG publishSeedList: I can reach myself");
*/
if ((this.lastSeedUpload_myIP.equals(seedDB.mySeed().get(yacySeed.IP, "127.0.0.1"))) &&
(this.lastSeedUpload_seedDBSize == seedDB.sizeConnected()) &&
if ((sb.wordIndex.seedDB.lastSeedUpload_myIP.equals(sb.wordIndex.seedDB.mySeed().get(yacySeed.IP, "127.0.0.1"))) &&
(sb.wordIndex.seedDB.lastSeedUpload_seedDBSize == sb.wordIndex.seedDB.sizeConnected()) &&
(canReachMyself()) &&
(System.currentTimeMillis() - this.lastSeedUpload_timeStamp < 1000 * 60 * 60 * 24) &&
(seedDB.mySeed().isPrincipal())
(System.currentTimeMillis() - sb.wordIndex.seedDB.lastSeedUpload_timeStamp < 1000 * 60 * 60 * 24) &&
(sb.wordIndex.seedDB.mySeed().isPrincipal())
) {
log.logFine("yacyCore.publishSeedList: not necessary to publish: oldIP is equal, sizeConnected is equal and I can reach myself under the old IP.");
return;
}
// getting the seed upload method that should be used ...
final String seedUploadMethod = this.switchboard.getConfig("seedUploadMethod", "");
final String seedUploadMethod = this.sb.getConfig("seedUploadMethod", "");
if (
(!seedUploadMethod.equalsIgnoreCase("none")) ||
((seedUploadMethod.equals("")) && (this.switchboard.getConfig("seedFTPPassword", "").length() > 0)) ||
((seedUploadMethod.equals("")) && (this.switchboard.getConfig("seedFilePath", "").length() > 0))
((seedUploadMethod.equals("")) && (this.sb.getConfig("seedFTPPassword", "").length() > 0)) ||
((seedUploadMethod.equals("")) && (this.sb.getConfig("seedFilePath", "").length() > 0))
) {
if (seedUploadMethod.equals("")) {
if (this.switchboard.getConfig("seedFTPPassword", "").length() > 0) {
this.switchboard.setConfig("seedUploadMethod", "Ftp");
if (this.sb.getConfig("seedFTPPassword", "").length() > 0) {
this.sb.setConfig("seedUploadMethod", "Ftp");
}
if (this.switchboard.getConfig("seedFilePath", "").length() > 0) {
this.switchboard.setConfig("seedUploadMethod", "File");
if (this.sb.getConfig("seedFilePath", "").length() > 0) {
this.sb.setConfig("seedUploadMethod", "File");
}
}
// we want to be a principal...
saveSeedList();
saveSeedList(sb);
} else {
if (seedUploadMethod.equals("")) {
this.switchboard.setConfig("seedUploadMethod", "none");
this.sb.setConfig("seedUploadMethod", "none");
}
log.logFine("yacyCore.publishSeedList: No uploading method configured");
return;
@ -251,7 +221,7 @@ public class yacyCore {
public final void peerPing() {
if (!online()) { return; }
if ((switchboard.isRobinsonMode()) && (switchboard.getConfig("cluster.mode", "").equals("privatepeer"))) {
if ((sb.isRobinsonMode()) && (sb.getConfig("cluster.mode", "").equals("privatepeer"))) {
// in case this peer is a privat peer we omit the peer ping
// all other robinson peer types do a peer ping:
// the privatecluster does the ping to the other cluster members
@ -264,29 +234,29 @@ public class yacyCore {
peerActions.updateMySeed();
// publish own seed to other peer, this can every peer, but makes only sense for senior peers
if (seedDB.sizeConnected() == 0) {
if (sb.wordIndex.seedDB.sizeConnected() == 0) {
// reload the seed lists
peerActions.loadSeedLists();
log.logInfo("re-initialized seed list. received " + seedDB.sizeConnected() + " new peer(s)");
log.logInfo("re-initialized seed list. received " + sb.wordIndex.seedDB.sizeConnected() + " new peer(s)");
}
final int newSeeds = publishMySeed(false);
if (newSeeds > 0) {
log.logInfo("received " + newSeeds + " new peer(s), know a total of " + seedDB.sizeConnected() + " different peers");
log.logInfo("received " + newSeeds + " new peer(s), know a total of " + sb.wordIndex.seedDB.sizeConnected() + " different peers");
}
}
private boolean canReachMyself() { // TODO: check if this method is necessary - depending on the used router it will not work
// returns true if we can reach ourself under our known peer address
// if we cannot reach ourself, we call a forced publishMySeed and return false
final int urlc = yacyClient.queryUrlCount(seedDB.mySeed());
final int urlc = yacyClient.queryUrlCount(sb.wordIndex.seedDB.mySeed());
if (urlc >= 0) {
seedDB.mySeed().setLastSeenUTC();
sb.wordIndex.seedDB.mySeed().setLastSeenUTC();
return true;
}
log.logInfo("re-connect own seed");
final String oldAddress = seedDB.mySeed().getPublicAddress();
final String oldAddress = sb.wordIndex.seedDB.mySeed().getPublicAddress();
/*final int newSeeds =*/ publishMySeed(true);
return (oldAddress != null && oldAddress.equals(seedDB.mySeed().getPublicAddress()));
return (oldAddress != null && oldAddress.equals(sb.wordIndex.seedDB.mySeed().getPublicAddress()));
}
protected class publishThread extends Thread {
@ -309,7 +279,7 @@ public class yacyCore {
public final void run() {
try {
this.added = yacyClient.publishMySeed(seed.getClusterAddress(), seed.hash);
this.added = yacyClient.publishMySeed(seed, seed.getClusterAddress(), seed.hash);
if (this.added < 0) {
// no or wrong response, delete that address
String cause = "peer ping to peer resulted in error response (added < 0)";
@ -320,7 +290,7 @@ public class yacyCore {
// update latest news from the other peer
log.logInfo("publish: handshaked " + this.seed.get(yacySeed.PEERTYPE, yacySeed.PEERTYPE_SENIOR) + " peer '" + this.seed.getName() + "' at " + this.seed.getPublicAddress());
// check if seed's lastSeen has been updated
yacySeed newSeed = seedDB.getConnected(this.seed.hash);
yacySeed newSeed = sb.wordIndex.seedDB.getConnected(this.seed.hash);
if (newSeed != null) {
if (newSeed.getLastSeenUTC() < (System.currentTimeMillis() - 10000)) {
// update last seed date
@ -373,13 +343,13 @@ public class yacyCore {
// init yacyHello-process
Map<String, yacySeed> seeds; // hash/yacySeed relation
int attempts = seedDB.sizeConnected();
int attempts = sb.wordIndex.seedDB.sizeConnected();
// getting a list of peers to contact
if (seedDB.mySeed().get(yacySeed.PEERTYPE, yacySeed.PEERTYPE_VIRGIN).equals(yacySeed.PEERTYPE_VIRGIN)) {
if (sb.wordIndex.seedDB.mySeed().get(yacySeed.PEERTYPE, yacySeed.PEERTYPE_VIRGIN).equals(yacySeed.PEERTYPE_VIRGIN)) {
if (attempts > PING_INITIAL) { attempts = PING_INITIAL; }
Map<String, String> ch = plasmaSwitchboard.getSwitchboard().clusterhashes;
seeds = seedDB.seedsByAge(true, attempts - ((ch == null) ? 0 : ch.size())); // best for fast connection
seeds = sb.wordIndex.seedDB.seedsByAge(true, attempts - ((ch == null) ? 0 : ch.size())); // best for fast connection
// add also all peers from cluster if this is a public robinson cluster
if (plasmaSwitchboard.getSwitchboard().clusterhashes != null) {
Iterator<Map.Entry<String, String>> i = ch.entrySet().iterator();
@ -391,7 +361,7 @@ public class yacyCore {
hash = entry.getKey();
seed = (yacySeed) seeds.get(hash);
if (seed == null) {
seed = seedDB.get(hash);
seed = sb.wordIndex.seedDB.get(hash);
if (seed == null) continue;
}
seed.setAlternativeAddress((String) entry.getValue());
@ -406,7 +376,7 @@ public class yacyCore {
} else {
if (attempts > PING_MIN_RUNNING) { attempts = PING_MIN_RUNNING; }
}
seeds = seedDB.seedsByAge(false, attempts); // best for seed list maintenance/cleaning
seeds = sb.wordIndex.seedDB.seedsByAge(false, attempts); // best for seed list maintenance/cleaning
}
if ((seeds == null) || seeds.size() == 0) { return 0; }
@ -418,20 +388,20 @@ public class yacyCore {
// include a YaCyNews record to my seed
try {
final yacyNewsRecord record = newsPool.myPublication();
final yacyNewsRecord record = sb.wordIndex.newsPool.myPublication();
if (record == null) {
seedDB.mySeed().put("news", "");
sb.wordIndex.seedDB.mySeed().put("news", "");
} else {
seedDB.mySeed().put("news", de.anomic.tools.crypt.simpleEncode(record.toString()));
sb.wordIndex.seedDB.mySeed().put("news", de.anomic.tools.crypt.simpleEncode(record.toString()));
}
} catch (IOException e) {
log.logSevere("publishMySeed: problem with news encoding", e);
}
seedDB.mySeed().setUnusedFlags();
sb.wordIndex.seedDB.mySeed().setUnusedFlags();
// include current citation-rank file count
seedDB.mySeed().put(yacySeed.CRWCNT, Integer.toString(switchboard.rankingOwnDistribution.size()));
seedDB.mySeed().put(yacySeed.CRTCNT, Integer.toString(switchboard.rankingOtherDistribution.size()));
sb.wordIndex.seedDB.mySeed().put(yacySeed.CRWCNT, Integer.toString(sb.rankingOwnDistribution.size()));
sb.wordIndex.seedDB.mySeed().put(yacySeed.CRTCNT, Integer.toString(sb.rankingOtherDistribution.size()));
int newSeeds = -1;
//if (seeds.length > 1) {
// holding a reference to all started threads
@ -519,7 +489,7 @@ public class yacyCore {
if ((accessible >= PING_MIN_PEERSEEN) ||
(accessible >= notaccessible)) {
// We can be reached from a majority of other Peers
if (yacyCore.seedDB.mySeed().isPrincipal()) {
if (sb.wordIndex.seedDB.mySeed().isPrincipal()) {
newPeerType = yacySeed.PEERTYPE_PRINCIPAL;
} else {
newPeerType = yacySeed.PEERTYPE_SENIOR;
@ -528,14 +498,14 @@ public class yacyCore {
// We cannot be reached from the outside
newPeerType = yacySeed.PEERTYPE_JUNIOR;
}
if (yacyCore.seedDB.mySeed().orVirgin().equals(newPeerType)) {
log.logInfo("PeerPing: myType is " + yacyCore.seedDB.mySeed().orVirgin());
if (sb.wordIndex.seedDB.mySeed().orVirgin().equals(newPeerType)) {
log.logInfo("PeerPing: myType is " + sb.wordIndex.seedDB.mySeed().orVirgin());
} else {
log.logInfo("PeerPing: changing myType from '" + yacyCore.seedDB.mySeed().orVirgin() + "' to '" + newPeerType + "'");
yacyCore.seedDB.mySeed().put(yacySeed.PEERTYPE, newPeerType);
log.logInfo("PeerPing: changing myType from '" + sb.wordIndex.seedDB.mySeed().orVirgin() + "' to '" + newPeerType + "'");
sb.wordIndex.seedDB.mySeed().put(yacySeed.PEERTYPE, newPeerType);
}
} else {
log.logInfo("PeerPing: No data, staying at myType: " + yacyCore.seedDB.mySeed().orVirgin());
log.logInfo("PeerPing: No data, staying at myType: " + sb.wordIndex.seedDB.mySeed().orVirgin());
}
if (newSeeds >= 0) {
@ -547,20 +517,20 @@ public class yacyCore {
}
// if we have an address, we do nothing
if (seedDB.mySeed().isProper() == null && !force) { return 0; }
if (sb.wordIndex.seedDB.mySeed().isProper() == null && !force) { return 0; }
// still no success: ask own NAT or internet responder
//final boolean DI604use = switchboard.getConfig("DI604use", "false").equals("true");
//final String DI604pw = switchboard.getConfig("DI604pw", "");
String ip = switchboard.getConfig("staticIP", "");
String ip = sb.getConfig("staticIP", "");
//if (ip.equals("")) ip = natLib.retrieveIP(DI604use, DI604pw);
// yacyCore.log.logDebug("DEBUG: new IP=" + ip);
seedDB.mySeed().put(yacySeed.IP, ip);
if (seedDB.mySeed().get(yacySeed.PEERTYPE, yacySeed.PEERTYPE_JUNIOR).equals(yacySeed.PEERTYPE_JUNIOR)) // ???????????????
seedDB.mySeed().put(yacySeed.PEERTYPE, yacySeed.PEERTYPE_SENIOR); // to start bootstraping, we need to be recognised as PEERTYPE_SENIOR peer
sb.wordIndex.seedDB.mySeed().put(yacySeed.IP, ip);
if (sb.wordIndex.seedDB.mySeed().get(yacySeed.PEERTYPE, yacySeed.PEERTYPE_JUNIOR).equals(yacySeed.PEERTYPE_JUNIOR)) // ???????????????
sb.wordIndex.seedDB.mySeed().put(yacySeed.PEERTYPE, yacySeed.PEERTYPE_SENIOR); // to start bootstraping, we need to be recognised as PEERTYPE_SENIOR peer
log.logInfo("publish: no recipient found, our address is " +
((seedDB.mySeed().getPublicAddress() == null) ? "unknown" : seedDB.mySeed().getPublicAddress()));
((sb.wordIndex.seedDB.mySeed().getPublicAddress() == null) ? "unknown" : sb.wordIndex.seedDB.mySeed().getPublicAddress()));
peerActions.saveMySeed();
return 0;
} catch (InterruptedException e) {
@ -582,16 +552,6 @@ public class yacyCore {
final Thread[] threadList = new Thread[threadCount];
threadCount = yacyCore.publishThreadGroup.enumerate(threadList);
// we need to use a timeout here because of missing interruptable session threads ...
log.logFine("publish: Trying to abort " + yacyCore.publishThreadGroup.activeCount() + " remaining publishing threads ...");
for (int currentThreadIdx = 0; currentThreadIdx < threadCount; currentThreadIdx++) {
Thread currentThread = threadList[currentThreadIdx];
/*if (currentThread.isAlive()) {
// TODO: this object should care of all open clien connections within this class and close them here
}*/
}
// we need to use a timeout here because of missing interruptable session threads ...
log.logFine("publish: Waiting for " + yacyCore.publishThreadGroup.activeCount() + " remaining publishing threads to finish shutdown ...");
for (int currentThreadIdx = 0; currentThreadIdx < threadCount; currentThreadIdx++) {
@ -699,18 +659,13 @@ public class yacyCore {
}
}
public final String saveSeedList() {
// return an error if this is not successful, and NULL if everything is fine
return saveSeedList(this.switchboard);
}
public final String saveSeedList(serverSwitch<?> sb) {
public static final String saveSeedList(plasmaSwitchboard sb) {
try {
// return an error if this is not successful, and NULL if everything is fine
String logt;
// be shure that we have something to say
if (seedDB.mySeed().getPublicAddress() == null) {
if (sb.wordIndex.seedDB.mySeed().getPublicAddress() == null) {
final String errorMsg = "We have no valid IP address until now";
log.logWarning("SaveSeedList: " + errorMsg);
return errorMsg;
@ -764,20 +719,20 @@ public class yacyCore {
}
// upload the seed-list using the configured uploader class
String prevStatus = seedDB.mySeed().get(yacySeed.PEERTYPE, yacySeed.PEERTYPE_JUNIOR);
String prevStatus = sb.wordIndex.seedDB.mySeed().get(yacySeed.PEERTYPE, yacySeed.PEERTYPE_JUNIOR);
if (prevStatus.equals(yacySeed.PEERTYPE_PRINCIPAL)) { prevStatus = yacySeed.PEERTYPE_SENIOR; }
try {
seedDB.mySeed().put(yacySeed.PEERTYPE, yacySeed.PEERTYPE_PRINCIPAL); // this information shall also be uploaded
sb.wordIndex.seedDB.mySeed().put(yacySeed.PEERTYPE, yacySeed.PEERTYPE_PRINCIPAL); // this information shall also be uploaded
log.logFine("SaveSeedList: Using seed uploading method '" + seedUploadMethod + "' for seed-list uploading." +
"\n\tPrevious peerType is '" + seedDB.mySeed().get(yacySeed.PEERTYPE, yacySeed.PEERTYPE_JUNIOR) + "'.");
"\n\tPrevious peerType is '" + sb.wordIndex.seedDB.mySeed().get(yacySeed.PEERTYPE, yacySeed.PEERTYPE_JUNIOR) + "'.");
// logt = seedDB.uploadCache(seedFTPServer, seedFTPAccount, seedFTPPassword, seedFTPPath, seedURL);
logt = seedDB.uploadCache(uploader, sb, seedDB, seedURL);
logt = sb.wordIndex.seedDB.uploadCache(uploader, sb, sb.wordIndex.seedDB, seedURL);
if (logt != null) {
if (logt.indexOf("Error") >= 0) {
seedDB.mySeed().put(yacySeed.PEERTYPE, prevStatus);
sb.wordIndex.seedDB.mySeed().put(yacySeed.PEERTYPE, prevStatus);
final String errorMsg = "SaveSeedList: seed upload failed using " + uploader.getClass().getName() + " (error): " + logt.substring(logt.indexOf("Error") + 6);
log.logSevere(errorMsg);
return errorMsg;
@ -789,18 +744,16 @@ public class yacyCore {
sb.setConfig("yacyStatus", yacySeed.PEERTYPE_PRINCIPAL);
return null;
} catch (Exception e) {
seedDB.mySeed().put(yacySeed.PEERTYPE, prevStatus);
sb.wordIndex.seedDB.mySeed().put(yacySeed.PEERTYPE, prevStatus);
sb.setConfig("yacyStatus", prevStatus);
final String errorMsg = "SaveSeedList: Seed upload failed (IO error): " + e.getMessage();
log.logInfo(errorMsg, e);
return errorMsg;
}
} finally {
this.lastSeedUpload_seedDBSize = seedDB.sizeConnected();
this.lastSeedUpload_timeStamp = System.currentTimeMillis();
this.lastSeedUpload_myIP = seedDB.mySeed().get(yacySeed.IP, "127.0.0.1");
//this.lastSeedUpload_myPeerType = seedDB.mySeed.get(yacySeed.PEERTYPE, yacySeed.PEERTYPE_JUNIOR);
sb.wordIndex.seedDB.lastSeedUpload_seedDBSize = sb.wordIndex.seedDB.sizeConnected();
sb.wordIndex.seedDB.lastSeedUpload_timeStamp = System.currentTimeMillis();
sb.wordIndex.seedDB.lastSeedUpload_myIP = sb.wordIndex.seedDB.mySeed().get(yacySeed.IP, "127.0.0.1");
}
}

20
source/de/anomic/yacy/yacyDHTAction.java
View File

@ -315,11 +315,11 @@ public class yacyDHTAction implements yacyPeerAction {
public void processPeerPing(yacySeed peer) {
}
public static boolean shallBeOwnWord(String wordhash) {
if (yacyCore.seedDB == null) return false;
if (yacyCore.seedDB.mySeed().isPotential()) return false;
final double distance = dhtDistance(yacyCore.seedDB.mySeed().hash, wordhash);
final double max = 1.2 / yacyCore.seedDB.sizeConnected();
public static boolean shallBeOwnWord(yacySeedDB seedDB, String wordhash) {
if (seedDB == null) return false;
if (seedDB.mySeed().isPotential()) return false;
final double distance = dhtDistance(seedDB.mySeed().hash, wordhash);
final double max = 1.2 / seedDB.sizeConnected();
//System.out.println("Distance for " + wordhash + ": " + distance + "; max is " + max);
return (distance > 0) && (distance <= max);
}
@ -347,13 +347,13 @@ public class yacyDHTAction implements yacyPeerAction {
return ((double) (kelondroBase64Order.enhancedCoder.cardinal(from.getBytes()) - kelondroBase64Order.enhancedCoder.cardinal(to.getBytes()))) / ((double) Long.MAX_VALUE);
}
public synchronized ArrayList<yacySeed> getDHTTargets(serverLog log, int primaryPeerCount, int reservePeerCount, String firstKey, String lastKey, double maxDist) {
public synchronized ArrayList<yacySeed> getDHTTargets(yacySeedDB seedDB, serverLog log, int primaryPeerCount, int reservePeerCount, String firstKey, String lastKey, double maxDist) {
// find a list of DHT-peers
assert firstKey != null;
assert lastKey != null;
assert yacyCore.seedDB != null;
assert yacyCore.seedDB.mySeed() != null;
assert yacyCore.seedDB.mySeed().hash != null;
assert seedDB != null;
assert seedDB.mySeed() != null;
assert seedDB.mySeed().hash != null;
/*
assert
!(kelondroBase64Order.enhancedCoder.cardinal(firstKey.getBytes()) < kelondroBase64Order.enhancedCoder.cardinal(yacyCore.seedDB.mySeed.hash.getBytes()) &&
@ -367,7 +367,7 @@ public class yacyDHTAction implements yacyPeerAction {
double firstdist, lastdist;
Iterator<yacySeed> e = this.getAcceptRemoteIndexSeeds(lastKey);
TreeSet<String> doublecheck = new TreeSet<String>(kelondroBase64Order.enhancedComparator);
int maxloop = Math.min(100, yacyCore.seedDB.sizeConnected()); // to ensure termination
int maxloop = Math.min(100, seedDB.sizeConnected()); // to ensure termination
if (log != null) log.logInfo("Collecting DHT target peers for first_hash = " + firstKey + ", last_hash = " + lastKey);
while ((e.hasNext()) && (seeds.size() < (primaryPeerCount + reservePeerCount)) && (maxloop-- > 0)) {
seed = (yacySeed) e.next();

Some files were not shown because too many files have changed in this diff Show More