+
#(/table)# #[footer]# diff --git a/htroot/News.java b/htroot/News.java index 771b682d2..4b3c6e50c 100644 --- a/htroot/News.java +++ b/htroot/News.java @@ -54,23 +54,50 @@ import de.anomic.server.serverDate; import de.anomic.yacy.yacyCore; import de.anomic.yacy.yacySeed; import de.anomic.yacy.yacyNewsRecord; +import de.anomic.plasma.plasmaSwitchboard; public class News { - public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch sb) { - // return variable that accumulates replacements + public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch env) { + plasmaSwitchboard switchboard = (plasmaSwitchboard) env; serverObjects prop = new serverObjects(); boolean overview = (post == null) || (((String) post.get("page", "0")).equals("0")); + int tableID = (overview) ? -1 : Integer.parseInt((String) post.get("page", "0")) - 1; + + // execute commands + if (post != null) { + if (switchboard.adminAuthenticated(header) < 2) { + // not authenticated, force log-in + prop.put("AUTHENTICATE", "admin log-in"); + return prop; + } + + if ((post.containsKey("delete")) && (tableID >= 0)) { + Enumeration e = post.keys(); + String check; + String id; + while (e.hasMoreElements()) { + check = (String) e.nextElement(); + if ((check.startsWith("del_")) && (post.get(check, "off").equals("on"))) { + id = check.substring(4); + try { + yacyCore.newsPool.moveOff(tableID, id); + } catch (IOException ee) {ee.printStackTrace();} + } + } + } + } + // generate properties for output if (overview) { // show overview prop.put("table", 0); prop.put("page", 0); } else { // generate table - int tableID = Integer.parseInt((String) post.get("page", "1")) - 1; prop.put("table", 1); prop.put("page", tableID + 1); + prop.put("table_page", tableID + 1); if (yacyCore.seedDB == null) { @@ -84,6 +111,7 @@ public class News { record = yacyCore.newsPool.get(tableID, i); seed = yacyCore.seedDB.getConnected(record.originator()); if (seed == null) seed = yacyCore.seedDB.getDisconnected(record.originator()); + prop.put("table_list_" + i + "_id", record.id()); prop.put("table_list_" + i + "_ori", (seed == null) ? record.originator() : seed.getName()); prop.put("table_list_" + i + "_cre", yacyCore.universalDateShortString(record.created())); prop.put("table_list_" + i + "_cat", record.category()); diff --git a/source/de/anomic/http/httpdFileHandler.java b/source/de/anomic/http/httpdFileHandler.java index 6ebf41c7d..ceb349c1d 100644 --- a/source/de/anomic/http/httpdFileHandler.java +++ b/source/de/anomic/http/httpdFileHandler.java @@ -113,10 +113,7 @@ public final class httpdFileHandler extends httpdAbstractHandler implements http private String[] defaultFiles = null; private File htDefaultPath = null; private File htLocalePath = null; - private serverSwitch switchboard; - private String adminAccountBase64MD5; - private MessageDigest md5Digest = null; public httpdFileHandler(serverSwitch switchboard) { @@ -179,7 +176,6 @@ public final class httpdFileHandler extends httpdAbstractHandler implements http for (int i = 0; i < ps.length; i++) System.out.println("PACKAGE IN PROVIDER: " + ps[i].toString()); */ } - adminAccountBase64MD5 = null; // initialise an message digest for Content-MD5 support ... try { @@ -247,17 +243,17 @@ public final class httpdFileHandler extends httpdAbstractHandler implements http } // check permission/granted access - if ((path.endsWith("_p.html")) && - ((adminAccountBase64MD5 = switchboard.getConfig("adminAccountBase64MD5", "")).length() != 0)) { + String authorization = (String) requestHeader.get(httpHeader.AUTHORIZATION); + String adminAccountBase64MD5 = switchboard.getConfig("adminAccountBase64MD5", ""); + if ((path.endsWith("_p.html")) && (adminAccountBase64MD5.length() != 0)) { // authentication required - String auth = (String) requestHeader.get(httpHeader.AUTHORIZATION); - if (auth == null) { + if (authorization == null) { // no authorization given in response. Ask for that httpHeader headers = getDefaultHeaders(); headers.put(httpHeader.WWW_AUTHENTICATE,"Basic realm=\"admin log-in\""); httpd.sendRespondHeader(conProp,out,httpVersion,401,headers); return; - } else if (adminAccountBase64MD5.equals(serverCodings.standardCoder.encodeMD5Hex(auth.trim().substring(6)))) { + } else if (adminAccountBase64MD5.equals(serverCodings.standardCoder.encodeMD5Hex(authorization.trim().substring(6)))) { // remove brute-force flag serverCore.bfHost.remove(conProp.getProperty("CLIENTIP")); } else { @@ -277,6 +273,14 @@ public final class httpdFileHandler extends httpdAbstractHandler implements http } } + // handle bfHost in case we have authentified correctly + if ((authorization != null) && + (adminAccountBase64MD5.length() != 0) && + (adminAccountBase64MD5.equals(serverCodings.standardCoder.encodeMD5Hex(authorization.trim().substring(6))))) { + // remove brute-force flag + serverCore.bfHost.remove(conProp.getProperty("CLIENTIP")); + } + // parse arguments serverObjects args = new serverObjects(); int argc; @@ -409,6 +413,17 @@ public final class httpdFileHandler extends httpdAbstractHandler implements http if (tp == null) tp = new serverObjects(); // check if the servlets requests authentification if (tp.containsKey("AUTHENTICATE")) { + // handle brute-force protection + if (authorization != null) { + String clientIP = conProp.getProperty("CLIENTIP", "unknown-host"); + serverLog.logInfo("HTTPD", "dynamic log-in for account 'admin' in http file handler for path '" + path + "' from host '" + clientIP + "'"); + Integer attempts = (Integer) serverCore.bfHost.get(clientIP); + if (attempts == null) + serverCore.bfHost.put(clientIP, new Integer(1)); + else + serverCore.bfHost.put(clientIP, new Integer(attempts.intValue() + 1)); + } + // send authentication request to browser httpHeader headers = getDefaultHeaders(); headers.put(httpHeader.WWW_AUTHENTICATE,"Basic realm=\"" + tp.get("AUTHENTICATE", "") + "\""); httpd.sendRespondHeader(conProp,out,httpVersion,401,headers); diff --git a/source/de/anomic/yacy/yacyCore.java b/source/de/anomic/yacy/yacyCore.java index 79c5e22d5..02f46ccc7 100644 --- a/source/de/anomic/yacy/yacyCore.java +++ b/source/de/anomic/yacy/yacyCore.java @@ -389,8 +389,11 @@ public class yacyCore { // include a YaCyNews record to my seed try { - yacyNewsRecord record = newsPool.dequeueMyNews(); - if (record != null) seedDB.mySeed.put("news", de.anomic.tools.crypt.simpleEncode(record.toString())); + yacyNewsRecord record = newsPool.myPublication(); + if (record == null) + seedDB.mySeed.put("news", ""); + else + seedDB.mySeed.put("news", de.anomic.tools.crypt.simpleEncode(record.toString())); } catch (IOException e) { e.printStackTrace(); } diff --git a/source/de/anomic/yacy/yacyNewsAction.java b/source/de/anomic/yacy/yacyNewsAction.java index eba34c41b..c0ce12442 100644 --- a/source/de/anomic/yacy/yacyNewsAction.java +++ b/source/de/anomic/yacy/yacyNewsAction.java @@ -53,12 +53,12 @@ public class yacyNewsAction implements yacyPeerAction { public void processPeerArrival(yacySeed peer, boolean direct) { String recordString = peer.get("news", null); - System.out.println("### triggered news arrival from peer " + peer.getName() + ", news " + ((recordString == null) ? "empty" : "attached")); - if (recordString == null) return; + //System.out.println("### triggered news arrival from peer " + peer.getName() + ", news " + ((recordString == null) ? "empty" : "attached")); + if ((recordString == null) || (recordString.length() == 0)) return; yacyNewsRecord record = new yacyNewsRecord(de.anomic.tools.crypt.simpleDecode(recordString, "")); - System.out.println("### news arrival from peer " + peer.getName() + ", news=" + record.toString()); + //System.out.println("### news arrival from peer " + peer.getName() + ", news=" + record.toString()); try { - this.pool.enqueueGlobalNews(record); + this.pool.enqueueIncomingNews(record); } catch (IOException e) {e.printStackTrace();} } diff --git a/source/de/anomic/yacy/yacyNewsDB.java b/source/de/anomic/yacy/yacyNewsDB.java index 780a5b118..e63bed826 100644 --- a/source/de/anomic/yacy/yacyNewsDB.java +++ b/source/de/anomic/yacy/yacyNewsDB.java @@ -101,6 +101,10 @@ public class yacyNewsDB { public int size() { return news.size(); } + + public void remove(String id) throws IOException { + news.remove(id.getBytes()); + } public synchronized yacyNewsRecord put(yacyNewsRecord record) throws IOException { try { diff --git a/source/de/anomic/yacy/yacyNewsPool.java b/source/de/anomic/yacy/yacyNewsPool.java index ab16a66b3..c61a3a469 100644 --- a/source/de/anomic/yacy/yacyNewsPool.java +++ b/source/de/anomic/yacy/yacyNewsPool.java @@ -64,11 +64,12 @@ public class yacyNewsPool { maxDistribution = 30; } - public void enqueueMyNews(yacyNewsRecord record) throws IOException { + public void publishMyNews(yacyNewsRecord record) throws IOException { + // this shall be called if our peer generated a new news record and wants to publish it if (newsDB.get(record.id()) == null) outgoingNews.push(record); } - public yacyNewsRecord dequeueMyNews() throws IOException { + public yacyNewsRecord myPublication() throws IOException { // generate a record for next peer-ping if (outgoingNews.size() == 0) return null; yacyNewsRecord record = outgoingNews.topInc(); @@ -79,15 +80,13 @@ public class yacyNewsPool { return record; } - public void enqueueGlobalNews(yacyNewsRecord record) throws IOException { + public void enqueueIncomingNews(yacyNewsRecord record) throws IOException { + // called if a news is attached to a seed if (newsDB.get(record.id()) == null) incomingNews.push(record); } - public yacyNewsRecord getGlobalNews(int job) throws IOException { - return incomingNews.top(job); - } - - public synchronized boolean removeGlobalNews(String id) throws IOException { + public synchronized boolean commitIncomingNews(String id) throws IOException { + // called if a incoming news was processed yacyNewsRecord record; for (int i = 0; i < incomingNews.size(); i++) { record = incomingNews.top(i); @@ -100,24 +99,79 @@ public class yacyNewsPool { return false; } + public synchronized boolean deleteProcessedNews(String id) throws IOException { + // called if a processed news shall be removed + // the news stays in the news database to prevent + // that it is loaded again from the net + yacyNewsRecord record; + for (int i = 0; i < processedNews.size(); i++) { + record = processedNews.top(i); + if (record.id().equals(id)) { + processedNews.pop(i); + return true; + } + } + return false; + } + + public synchronized boolean interruptPublication(String id) throws IOException { + // called if a outgoing news shall not be published any more + yacyNewsRecord record; + for (int i = 0; i < outgoingNews.size(); i++) { + record = outgoingNews.top(i); + if (record.id().equals(id)) { + outgoingNews.pop(i); + publishedNews.push(record); + return true; + } + } + return false; + } + + public synchronized boolean deletePublishedNews(String id) throws IOException { + // called if a published news shall be removed + // the news is also removed from the news database + yacyNewsRecord record; + for (int i = 0; i < publishedNews.size(); i++) { + record = publishedNews.top(i); + if (record.id().equals(id)) { + publishedNews.pop(i); + newsDB.remove(id); + return true; + } + } + return false; + } + public int size(int dbKey) { switch (dbKey) { - case OUTGOING_DB: return outgoingNews.size(); - case PUBLISHED_DB: return publishedNews.size(); case INCOMING_DB: return incomingNews.size(); case PROCESSED_DB: return processedNews.size(); + case OUTGOING_DB: return outgoingNews.size(); + case PUBLISHED_DB: return publishedNews.size(); default: return -1; } } public yacyNewsRecord get(int dbKey, int element) throws IOException { switch (dbKey) { - case OUTGOING_DB: return outgoingNews.top(element); - case PUBLISHED_DB: return publishedNews.top(element); case INCOMING_DB: return incomingNews.top(element); case PROCESSED_DB: return processedNews.top(element); + case OUTGOING_DB: return outgoingNews.top(element); + case PUBLISHED_DB: return publishedNews.top(element); default: return null; } } + public void moveOff(int dbKey, String id) throws IOException { + // this is called if a queue element shall be moved to another queue or off the queue + // it depends on the dbKey how the record is handled + switch (dbKey) { + case INCOMING_DB: commitIncomingNews(id); break; + case PROCESSED_DB: deleteProcessedNews(id); break; + case OUTGOING_DB: interruptPublication(id); break; + case PUBLISHED_DB: deletePublishedNews(id); break; + } + } + } diff --git a/source/de/anomic/yacy/yacyNewsRecord.java b/source/de/anomic/yacy/yacyNewsRecord.java index e5b8b30bb..b4b225a73 100644 --- a/source/de/anomic/yacy/yacyNewsRecord.java +++ b/source/de/anomic/yacy/yacyNewsRecord.java @@ -58,9 +58,8 @@ public class yacyNewsRecord { private int distributed; // counter that counts number of distributions of this news record private Map attributes; // elemets of the news for a special category - public yacyNewsRecord(String encodedNewsString) { - String decodedString = serverCodings.enhancedCoder.decodeBase64String(encodedNewsString); - this.attributes = serverCodings.string2map(decodedString); + public yacyNewsRecord(String newsString) { + this.attributes = serverCodings.string2map(newsString); this.received = (attributes.containsKey("rec")) ? yacyCore.parseUniversalDate((String) attributes.get("rec")) : new Date(); this.created = (attributes.containsKey("cre")) ? yacyCore.parseUniversalDate((String) attributes.get("cre")) : new Date(); this.category = (attributes.containsKey("cat")) ? (String) attributes.get("cat") : null; @@ -101,11 +100,11 @@ public class yacyNewsRecord { public String toString() { // this creates the string that shall be distributed // attention: this has no additional encoding - if (originator != null) attributes.put("ori", originator); - if (category != null) attributes.put("cat", category); - attributes.put("cre", yacyCore.universalDateShortString(created)); - attributes.put("rec", yacyCore.universalDateShortString(received)); - attributes.put("dis", "" + distributed); + if (this.originator != null) attributes.put("ori", this.originator); + if (this.category != null) attributes.put("cat", this.category); + attributes.put("cre", yacyCore.universalDateShortString(this.created)); + attributes.put("rec", yacyCore.universalDateShortString(this.received)); + attributes.put("dis", "" +this. distributed); String theString = attributes.toString(); removeStandards(); return theString; diff --git a/source/de/anomic/yacy/yacyPeerActions.java b/source/de/anomic/yacy/yacyPeerActions.java index 9f793bd3a..8def13ce1 100644 --- a/source/de/anomic/yacy/yacyPeerActions.java +++ b/source/de/anomic/yacy/yacyPeerActions.java @@ -323,7 +323,7 @@ public class yacyPeerActions { // this is a return of a lost peer yacyCore.log.logDebug("connect: returned KNOWN " + peerType + " peer '" + seed.getName() + "' from " + seed.getAddress()); seedDB.addConnected(seed); - return false; + return true; } else { yacySeed connectedSeed = seedDB.getConnected(seed.hash); if (connectedSeed != null) { @@ -337,7 +337,7 @@ public class yacyPeerActions { } catch (java.text.ParseException e) {} yacyCore.log.logDebug("connect: updated KNOWN " + ((direct) ? "direct " : "") + peerType + " peer '" + seed.getName() + "' from " + seed.getAddress()); seedDB.addConnected(seed); - return false; + return true; } else { // the seed is new if (((String) seed.get("IP", "127.0.0.1")).equals((String) seedDB.mySeed.get("IP", "127.0.0.1"))) { diff --git a/yacy.init b/yacy.init index f56f75948..4e695fde5 100644 --- a/yacy.init +++ b/yacy.init @@ -482,7 +482,7 @@ maxWaitingWordFlush = 180 isTransparentProxy=false # Specifies if yacy should use the http connection keep-alive feature -connectionKeepAliveSupport=false +connectionKeepAliveSupport=true # Configuration options needed to configure server port forwarding portForwardingEnabled=false