yacy_search_server/htroot/Surftips.java

337 lines
18 KiB
Java
Raw Normal View History

// Surftips.java
// (C) 2006 by Michael Peter Christen; mc@yacy.net, Frankfurt a. M., Germany
// first published 2006 on http://www.anomic.de
//
// This is a part of YaCy, a peer-to-peer based web search engine
//
// $LastChangedDate: 2006-04-02 22:40:07 +0200 (So, 02 Apr 2006) $
// $LastChangedRevision: 1986 $
// $LastChangedBy: orbiter $
//
// LICENSE
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation; either version 2 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
import java.io.IOException;
import java.net.MalformedURLException;
import java.util.Date;
import java.util.HashMap;
import java.util.Iterator;
import de.anomic.http.httpHeader;
import de.anomic.index.indexReferenceBlacklist;
import de.anomic.kelondro.kelondroMScoreCluster;
import de.anomic.kelondro.kelondroNaturalOrder;
import de.anomic.kelondro.kelondroRow;
import de.anomic.kelondro.kelondroRow.Entry;
import de.anomic.plasma.plasmaSwitchboard;
import de.anomic.server.serverDate;
import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch;
import de.anomic.tools.crypt;
import de.anomic.tools.nxTools;
import de.anomic.yacy.yacyNewsPool;
import de.anomic.yacy.yacyNewsRecord;
import de.anomic.yacy.yacySeed;
import de.anomic.yacy.yacyURL;
public class Surftips {
public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch<?> env) {
final plasmaSwitchboard sb = (plasmaSwitchboard) env;
final serverObjects prop = new serverObjects();
boolean authenticated = sb.adminAuthenticated(header) >= 2;
int display = ((post == null) || (!authenticated)) ? 0 : post.getInt("display", 0);
prop.put("display", display);
boolean showScore = ((post != null) && (post.containsKey("score")));
// access control
boolean publicPage = sb.getConfigBool("publicSurftips", true);
boolean authorizedAccess = sb.verifyAuthentication(header, false);
if ((post != null) && (post.containsKey("publicPage"))) {
if (!authorizedAccess) {
prop.put("AUTHENTICATE", "admin log-in"); // force log-in
return prop;
}
publicPage = post.get("publicPage", "0").equals("1");
sb.setConfig("publicSurftips", publicPage);
}
if ((publicPage) || (authorizedAccess)) {
// read voting
String hash;
if ((post != null) && ((hash = post.get("voteNegative", null)) != null)) {
if (!sb.verifyAuthentication(header, false)) {
prop.put("AUTHENTICATE", "admin log-in"); // force log-in
return prop;
}
// make new news message with voting
HashMap<String, String> map = new HashMap<String, String>();
map.put("urlhash", hash);
map.put("vote", "negative");
map.put("refid", post.get("refid", ""));
sb.webIndex.newsPool.publishMyNews(yacyNewsRecord.newRecord(sb.webIndex.seedDB.mySeed(), yacyNewsPool.CATEGORY_SURFTIPP_VOTE_ADD, map));
}
if ((post != null) && ((hash = post.get("votePositive", null)) != null)) {
if (!sb.verifyAuthentication(header, false)) {
prop.put("AUTHENTICATE", "admin log-in"); // force log-in
return prop;
}
// make new news message with voting
HashMap<String, String> map = new HashMap<String, String>();
map.put("urlhash", hash);
map.put("url", crypt.simpleDecode(post.get("url", ""), null));
map.put("title", crypt.simpleDecode(post.get("title", ""), null));
map.put("description", crypt.simpleDecode(post.get("description", ""), null));
map.put("vote", "positive");
map.put("refid", post.get("refid", ""));
map.put("comment", post.get("comment", ""));
sb.webIndex.newsPool.publishMyNews(new yacyNewsRecord(sb.webIndex.seedDB.mySeed(), yacyNewsPool.CATEGORY_SURFTIPP_VOTE_ADD, map));
}
// create surftips
HashMap<String, Integer> negativeHashes = new HashMap<String, Integer>(); // a mapping from an url hash to Integer (count of votes)
HashMap<String, Integer> positiveHashes = new HashMap<String, Integer>(); // a mapping from an url hash to Integer (count of votes)
major step forward to network switching (target is easy switch to intranet or other networks .. and back) This change is inspired by the need to see a network connected to the index it creates in a indexing team. It is not possible to divide the network and the index. Therefore all control files for the network was moved to the network within the INDEX/<network-name> subfolder. The remaining YACYDB is superfluous and can be deleted. The yacyDB and yacyNews data structures are now part of plasmaWordIndex. Therefore all methods, using static access to yacySeedDB had to be rewritten. A special problem had been all the port forwarding methods which had been tightly mixed with seed construction. It was not possible to move the port forwarding functions to the place, meaning and usage of plasmaWordIndex. Therefore the port forwarding had been deleted (I guess nobody used it and it can be simulated by methods outside of YaCy). The mySeed.txt is automatically moved to the current network position. A new effect causes that every network will create a different local seed file, which is ok, since the seed identifies the peer only against the network (it is the purpose of the seed hash to give a peer a location within the DHT). No other functional change has been made. The next steps to enable network switcing are: - shift of crawler tables from PLASMADB into the network (crawls are also network-specific) - possibly shift of plasmaWordIndex code into yacy package (index management is network-specific) - servlet to switch networks git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@4765 6c8d7289-2bf4-0310-a012-ef5d649a1542
2008-05-06 01:13:47 +02:00
accumulateVotes(sb , negativeHashes, positiveHashes, yacyNewsPool.INCOMING_DB);
//accumulateVotes(negativeHashes, positiveHashes, yacyNewsPool.OUTGOING_DB);
//accumulateVotes(negativeHashes, positiveHashes, yacyNewsPool.PUBLISHED_DB);
kelondroMScoreCluster<String> ranking = new kelondroMScoreCluster<String>(); // score cluster for url hashes
kelondroRow rowdef = new kelondroRow("String url-255, String title-120, String description-120, String refid-" + (serverDate.PATTERN_SHORT_SECOND.length() + 12), kelondroNaturalOrder.naturalOrder, 0);
HashMap<String, Entry> surftips = new HashMap<String, Entry>(); // a mapping from an url hash to a kelondroRow.Entry with display properties
major step forward to network switching (target is easy switch to intranet or other networks .. and back) This change is inspired by the need to see a network connected to the index it creates in a indexing team. It is not possible to divide the network and the index. Therefore all control files for the network was moved to the network within the INDEX/<network-name> subfolder. The remaining YACYDB is superfluous and can be deleted. The yacyDB and yacyNews data structures are now part of plasmaWordIndex. Therefore all methods, using static access to yacySeedDB had to be rewritten. A special problem had been all the port forwarding methods which had been tightly mixed with seed construction. It was not possible to move the port forwarding functions to the place, meaning and usage of plasmaWordIndex. Therefore the port forwarding had been deleted (I guess nobody used it and it can be simulated by methods outside of YaCy). The mySeed.txt is automatically moved to the current network position. A new effect causes that every network will create a different local seed file, which is ok, since the seed identifies the peer only against the network (it is the purpose of the seed hash to give a peer a location within the DHT). No other functional change has been made. The next steps to enable network switcing are: - shift of crawler tables from PLASMADB into the network (crawls are also network-specific) - possibly shift of plasmaWordIndex code into yacy package (index management is network-specific) - servlet to switch networks git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@4765 6c8d7289-2bf4-0310-a012-ef5d649a1542
2008-05-06 01:13:47 +02:00
accumulateSurftips(sb, surftips, ranking, rowdef, negativeHashes, positiveHashes, yacyNewsPool.INCOMING_DB);
//accumulateSurftips(surftips, ranking, rowdef, negativeHashes, positiveHashes, yacyNewsPool.OUTGOING_DB);
//accumulateSurftips(surftips, ranking, rowdef, negativeHashes, positiveHashes, yacyNewsPool.PUBLISHED_DB);
// read out surftipp array and create property entries
Iterator<String> k = ranking.scores(false);
int i = 0;
kelondroRow.Entry row;
String url, urlhash, refid, title, description;
boolean voted;
while (k.hasNext()) {
urlhash = k.next();
if (urlhash == null) continue;
row = surftips.get(urlhash);
if (row == null) continue;
url = row.getColString(0, null);
try{
if(plasmaSwitchboard.urlBlacklist.isListed(indexReferenceBlacklist.BLACKLIST_SURFTIPS ,new yacyURL(url, null)))
continue;
}catch(MalformedURLException e){continue;};
title = row.getColString(1,"UTF-8");
description = row.getColString(2,"UTF-8");
if ((url == null) || (title == null) || (description == null)) continue;
refid = row.getColString(3, null);
voted = (sb.webIndex.newsPool.getSpecific(yacyNewsPool.OUTGOING_DB, yacyNewsPool.CATEGORY_SURFTIPP_VOTE_ADD, "refid", refid) != null) ||
(sb.webIndex.newsPool.getSpecific(yacyNewsPool.PUBLISHED_DB, yacyNewsPool.CATEGORY_SURFTIPP_VOTE_ADD, "refid", refid) != null);
* Complete number localization and provide a more reasonable interface to serverObjects: - put(key, value) methods are now used if a value added to the map should be kept as it is. Numbers are transformed (but not formatted) to an equivalent String representation. - putASIS(...) have been removed, now done with simple put(...) (see above). - puNum(...) can be used for number values which should be stored in a formatted way, either depending on the current locale setting for yacy (default) or in a "none" locale (see javadocs and setLocalize()). - putHTML(...) escapes special characters into corresponding HTML enities ('<' => '&lt;') which was done with put(...) before and so was called too often, becauses it is necessary only for very few cases. Additionally there is a "forXML" mode which only replaces < > & ". In short: Use put(...) for almost everything, use putXY(...) if you need some special transformation of the value. A few bugs have been fixed as well, and there should be a small performance improvement for complex pages with a lot of values. * added additional Sum/Avg rows to access tracker pages, see http://forum.yacy-websuche.de/viewtopic.php?f=5&t=456 * removed duplicate code (mostly related to the big changes above). TODO: - make sure, number formats work as expected _everywhere_, report overseen stuff http://forum.yacy-websuche.de/viewtopic.php?f=5&t=437 - probably a good idea to add special putDate() methods as they are used in many pages and create duplicated formatting code + maybe some centralized handling for memory value formatting. - further improve the speed of page creation for the WatchCrawler. git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@4178 6c8d7289-2bf4-0310-a012-ef5d649a1542
2007-10-24 23:38:19 +02:00
prop.put("surftips_results_" + i + "_authorized", (authenticated) ? "1" : "0");
prop.put("surftips_results_" + i + "_authorized_recommend", (voted) ? "0" : "1");
prop.put("surftips_results_" + i + "_authorized_recommend_urlhash", urlhash);
prop.put("surftips_results_" + i + "_authorized_recommend_refid", refid);
* Complete number localization and provide a more reasonable interface to serverObjects: - put(key, value) methods are now used if a value added to the map should be kept as it is. Numbers are transformed (but not formatted) to an equivalent String representation. - putASIS(...) have been removed, now done with simple put(...) (see above). - puNum(...) can be used for number values which should be stored in a formatted way, either depending on the current locale setting for yacy (default) or in a "none" locale (see javadocs and setLocalize()). - putHTML(...) escapes special characters into corresponding HTML enities ('<' => '&lt;') which was done with put(...) before and so was called too often, becauses it is necessary only for very few cases. Additionally there is a "forXML" mode which only replaces < > & ". In short: Use put(...) for almost everything, use putXY(...) if you need some special transformation of the value. A few bugs have been fixed as well, and there should be a small performance improvement for complex pages with a lot of values. * added additional Sum/Avg rows to access tracker pages, see http://forum.yacy-websuche.de/viewtopic.php?f=5&t=456 * removed duplicate code (mostly related to the big changes above). TODO: - make sure, number formats work as expected _everywhere_, report overseen stuff http://forum.yacy-websuche.de/viewtopic.php?f=5&t=437 - probably a good idea to add special putDate() methods as they are used in many pages and create duplicated formatting code + maybe some centralized handling for memory value formatting. - further improve the speed of page creation for the WatchCrawler. git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@4178 6c8d7289-2bf4-0310-a012-ef5d649a1542
2007-10-24 23:38:19 +02:00
prop.put("surftips_results_" + i + "_authorized_recommend_url", crypt.simpleEncode(url, null, 'b'));
prop.put("surftips_results_" + i + "_authorized_recommend_title", crypt.simpleEncode(title, null, 'b'));
prop.put("surftips_results_" + i + "_authorized_recommend_description", crypt.simpleEncode(description, null, 'b'));
prop.put("surftips_results_" + i + "_authorized_recommend_display", display);
* Complete number localization and provide a more reasonable interface to serverObjects: - put(key, value) methods are now used if a value added to the map should be kept as it is. Numbers are transformed (but not formatted) to an equivalent String representation. - putASIS(...) have been removed, now done with simple put(...) (see above). - puNum(...) can be used for number values which should be stored in a formatted way, either depending on the current locale setting for yacy (default) or in a "none" locale (see javadocs and setLocalize()). - putHTML(...) escapes special characters into corresponding HTML enities ('<' => '&lt;') which was done with put(...) before and so was called too often, becauses it is necessary only for very few cases. Additionally there is a "forXML" mode which only replaces < > & ". In short: Use put(...) for almost everything, use putXY(...) if you need some special transformation of the value. A few bugs have been fixed as well, and there should be a small performance improvement for complex pages with a lot of values. * added additional Sum/Avg rows to access tracker pages, see http://forum.yacy-websuche.de/viewtopic.php?f=5&t=456 * removed duplicate code (mostly related to the big changes above). TODO: - make sure, number formats work as expected _everywhere_, report overseen stuff http://forum.yacy-websuche.de/viewtopic.php?f=5&t=437 - probably a good idea to add special putDate() methods as they are used in many pages and create duplicated formatting code + maybe some centralized handling for memory value formatting. - further improve the speed of page creation for the WatchCrawler. git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@4178 6c8d7289-2bf4-0310-a012-ef5d649a1542
2007-10-24 23:38:19 +02:00
prop.put("surftips_results_" + i + "_authorized_recommend_showScore", (showScore ? "1" : "0"));
prop.putHTML("surftips_results_" + i + "_authorized_urlhash", urlhash, true);
prop.putHTML("surftips_results_" + i + "_url", url, true);
prop.putHTML("surftips_results_" + i + "_urlname", nxTools.shortenURLString(url, 60), true);
prop.putHTML("surftips_results_" + i + "_urlhash", urlhash, true);
prop.putHTML("surftips_results_" + i + "_title", (showScore) ? ("(" + ranking.getScore(urlhash) + ") " + title) : title, true);
* Complete number localization and provide a more reasonable interface to serverObjects: - put(key, value) methods are now used if a value added to the map should be kept as it is. Numbers are transformed (but not formatted) to an equivalent String representation. - putASIS(...) have been removed, now done with simple put(...) (see above). - puNum(...) can be used for number values which should be stored in a formatted way, either depending on the current locale setting for yacy (default) or in a "none" locale (see javadocs and setLocalize()). - putHTML(...) escapes special characters into corresponding HTML enities ('<' => '&lt;') which was done with put(...) before and so was called too often, becauses it is necessary only for very few cases. Additionally there is a "forXML" mode which only replaces < > & ". In short: Use put(...) for almost everything, use putXY(...) if you need some special transformation of the value. A few bugs have been fixed as well, and there should be a small performance improvement for complex pages with a lot of values. * added additional Sum/Avg rows to access tracker pages, see http://forum.yacy-websuche.de/viewtopic.php?f=5&t=456 * removed duplicate code (mostly related to the big changes above). TODO: - make sure, number formats work as expected _everywhere_, report overseen stuff http://forum.yacy-websuche.de/viewtopic.php?f=5&t=437 - probably a good idea to add special putDate() methods as they are used in many pages and create duplicated formatting code + maybe some centralized handling for memory value formatting. - further improve the speed of page creation for the WatchCrawler. git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@4178 6c8d7289-2bf4-0310-a012-ef5d649a1542
2007-10-24 23:38:19 +02:00
prop.putHTML("surftips_results_" + i + "_description", description);
i++;
if (i >= 50) break;
}
prop.put("surftips_results", i);
* Complete number localization and provide a more reasonable interface to serverObjects: - put(key, value) methods are now used if a value added to the map should be kept as it is. Numbers are transformed (but not formatted) to an equivalent String representation. - putASIS(...) have been removed, now done with simple put(...) (see above). - puNum(...) can be used for number values which should be stored in a formatted way, either depending on the current locale setting for yacy (default) or in a "none" locale (see javadocs and setLocalize()). - putHTML(...) escapes special characters into corresponding HTML enities ('<' => '&lt;') which was done with put(...) before and so was called too often, becauses it is necessary only for very few cases. Additionally there is a "forXML" mode which only replaces < > & ". In short: Use put(...) for almost everything, use putXY(...) if you need some special transformation of the value. A few bugs have been fixed as well, and there should be a small performance improvement for complex pages with a lot of values. * added additional Sum/Avg rows to access tracker pages, see http://forum.yacy-websuche.de/viewtopic.php?f=5&t=456 * removed duplicate code (mostly related to the big changes above). TODO: - make sure, number formats work as expected _everywhere_, report overseen stuff http://forum.yacy-websuche.de/viewtopic.php?f=5&t=437 - probably a good idea to add special putDate() methods as they are used in many pages and create duplicated formatting code + maybe some centralized handling for memory value formatting. - further improve the speed of page creation for the WatchCrawler. git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@4178 6c8d7289-2bf4-0310-a012-ef5d649a1542
2007-10-24 23:38:19 +02:00
prop.put("surftips", "1");
} else {
* Complete number localization and provide a more reasonable interface to serverObjects: - put(key, value) methods are now used if a value added to the map should be kept as it is. Numbers are transformed (but not formatted) to an equivalent String representation. - putASIS(...) have been removed, now done with simple put(...) (see above). - puNum(...) can be used for number values which should be stored in a formatted way, either depending on the current locale setting for yacy (default) or in a "none" locale (see javadocs and setLocalize()). - putHTML(...) escapes special characters into corresponding HTML enities ('<' => '&lt;') which was done with put(...) before and so was called too often, becauses it is necessary only for very few cases. Additionally there is a "forXML" mode which only replaces < > & ". In short: Use put(...) for almost everything, use putXY(...) if you need some special transformation of the value. A few bugs have been fixed as well, and there should be a small performance improvement for complex pages with a lot of values. * added additional Sum/Avg rows to access tracker pages, see http://forum.yacy-websuche.de/viewtopic.php?f=5&t=456 * removed duplicate code (mostly related to the big changes above). TODO: - make sure, number formats work as expected _everywhere_, report overseen stuff http://forum.yacy-websuche.de/viewtopic.php?f=5&t=437 - probably a good idea to add special putDate() methods as they are used in many pages and create duplicated formatting code + maybe some centralized handling for memory value formatting. - further improve the speed of page creation for the WatchCrawler. git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@4178 6c8d7289-2bf4-0310-a012-ef5d649a1542
2007-10-24 23:38:19 +02:00
prop.put("surftips", "0");
}
return prop;
}
private static int timeFactor(Date created) {
return (int) Math.max(0, 10 - ((System.currentTimeMillis() - created.getTime()) / 24 / 60 / 60 / 1000));
}
major step forward to network switching (target is easy switch to intranet or other networks .. and back) This change is inspired by the need to see a network connected to the index it creates in a indexing team. It is not possible to divide the network and the index. Therefore all control files for the network was moved to the network within the INDEX/<network-name> subfolder. The remaining YACYDB is superfluous and can be deleted. The yacyDB and yacyNews data structures are now part of plasmaWordIndex. Therefore all methods, using static access to yacySeedDB had to be rewritten. A special problem had been all the port forwarding methods which had been tightly mixed with seed construction. It was not possible to move the port forwarding functions to the place, meaning and usage of plasmaWordIndex. Therefore the port forwarding had been deleted (I guess nobody used it and it can be simulated by methods outside of YaCy). The mySeed.txt is automatically moved to the current network position. A new effect causes that every network will create a different local seed file, which is ok, since the seed identifies the peer only against the network (it is the purpose of the seed hash to give a peer a location within the DHT). No other functional change has been made. The next steps to enable network switcing are: - shift of crawler tables from PLASMADB into the network (crawls are also network-specific) - possibly shift of plasmaWordIndex code into yacy package (index management is network-specific) - servlet to switch networks git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@4765 6c8d7289-2bf4-0310-a012-ef5d649a1542
2008-05-06 01:13:47 +02:00
private static void accumulateVotes(plasmaSwitchboard sb, HashMap<String, Integer> negativeHashes, HashMap<String, Integer> positiveHashes, int dbtype) {
int maxCount = Math.min(1000, sb.webIndex.newsPool.size(dbtype));
yacyNewsRecord record;
Iterator<yacyNewsRecord> recordIterator = sb.webIndex.newsPool.recordIterator(dbtype, true);
int j = 0;
while ((recordIterator.hasNext()) && (j++ < maxCount)) {
record = recordIterator.next();
if (record == null) continue;
if (record.category().equals(yacyNewsPool.CATEGORY_SURFTIPP_VOTE_ADD)) {
String urlhash = record.attribute("urlhash", "");
String vote = record.attribute("vote", "");
int factor = ((dbtype == yacyNewsPool.OUTGOING_DB) || (dbtype == yacyNewsPool.PUBLISHED_DB)) ? 2 : 1;
if (vote.equals("negative")) {
Integer i = negativeHashes.get(urlhash);
if (i == null) negativeHashes.put(urlhash, new Integer(factor));
else negativeHashes.put(urlhash, new Integer(i.intValue() + factor));
}
if (vote.equals("positive")) {
Integer i = positiveHashes.get(urlhash);
if (i == null) positiveHashes.put(urlhash, new Integer(factor));
else positiveHashes.put(urlhash, new Integer(i.intValue() + factor));
}
}
}
}
private static void accumulateSurftips(
major step forward to network switching (target is easy switch to intranet or other networks .. and back) This change is inspired by the need to see a network connected to the index it creates in a indexing team. It is not possible to divide the network and the index. Therefore all control files for the network was moved to the network within the INDEX/<network-name> subfolder. The remaining YACYDB is superfluous and can be deleted. The yacyDB and yacyNews data structures are now part of plasmaWordIndex. Therefore all methods, using static access to yacySeedDB had to be rewritten. A special problem had been all the port forwarding methods which had been tightly mixed with seed construction. It was not possible to move the port forwarding functions to the place, meaning and usage of plasmaWordIndex. Therefore the port forwarding had been deleted (I guess nobody used it and it can be simulated by methods outside of YaCy). The mySeed.txt is automatically moved to the current network position. A new effect causes that every network will create a different local seed file, which is ok, since the seed identifies the peer only against the network (it is the purpose of the seed hash to give a peer a location within the DHT). No other functional change has been made. The next steps to enable network switcing are: - shift of crawler tables from PLASMADB into the network (crawls are also network-specific) - possibly shift of plasmaWordIndex code into yacy package (index management is network-specific) - servlet to switch networks git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@4765 6c8d7289-2bf4-0310-a012-ef5d649a1542
2008-05-06 01:13:47 +02:00
plasmaSwitchboard sb,
HashMap<String, Entry> surftips, kelondroMScoreCluster<String> ranking, kelondroRow rowdef,
HashMap<String, Integer> negativeHashes, HashMap<String, Integer> positiveHashes, int dbtype) {
int maxCount = Math.min(1000, sb.webIndex.newsPool.size(dbtype));
yacyNewsRecord record;
Iterator<yacyNewsRecord> recordIterator = sb.webIndex.newsPool.recordIterator(dbtype, true);
int j = 0;
String url = "", urlhash;
kelondroRow.Entry entry;
int score = 0;
Integer vote;
while ((recordIterator.hasNext()) && (j++ < maxCount)) {
record = recordIterator.next();
if (record == null) continue;
entry = null;
if (record.category().equals(yacyNewsPool.CATEGORY_CRAWL_START)) try {
String intention = record.attribute("intention", "");
url = record.attribute("startURL", "");
if (url.length() < 12) continue;
entry = rowdef.newEntry(new byte[][]{
url.getBytes(),
(((intention.length() == 0) || (intention.equals("simple web crawl"))) ? record.attribute("startURL", "") : intention).getBytes(),
("Crawl Start Point").getBytes("UTF-8"),
record.id().getBytes()
});
score = 2 + Math.min(10, intention.length() / 4) + timeFactor(record.created());
} catch (IOException e) {}
if (record.category().equals(yacyNewsPool.CATEGORY_BOOKMARK_ADD)) try {
url = record.attribute("url", "");
if (url.length() < 12) continue;
entry = rowdef.newEntry(new byte[][]{
url.getBytes(),
(record.attribute("title", "")).getBytes("UTF-8"),
("Bookmark: " + record.attribute("description", "")).getBytes("UTF-8"),
record.id().getBytes()
});
score = 8 + timeFactor(record.created());
} catch (IOException e) {}
if (record.category().equals(yacyNewsPool.CATEGORY_SURFTIPP_ADD)) try {
url = record.attribute("url", "");
if (url.length() < 12) continue;
entry = rowdef.newEntry(new byte[][]{
url.getBytes(),
(record.attribute("title", "")).getBytes("UTF-8"),
("Surf Tipp: " + record.attribute("description", "")).getBytes("UTF-8"),
record.id().getBytes()
});
score = 5 + timeFactor(record.created());
} catch (IOException e) {}
if (record.category().equals(yacyNewsPool.CATEGORY_SURFTIPP_VOTE_ADD)) try {
if (!(record.attribute("vote", "negative").equals("positive"))) continue;
url = record.attribute("url", "");
if (url.length() < 12) continue;
entry = rowdef.newEntry(new byte[][]{
url.getBytes(),
record.attribute("title", "").getBytes("UTF-8"),
record.attribute("description", "").getBytes("UTF-8"),
record.attribute("refid", "").getBytes()
});
score = 5 + timeFactor(record.created());
} catch (IOException e) {}
if (record.category().equals(yacyNewsPool.CATEGORY_WIKI_UPDATE)) try {
yacySeed seed = sb.webIndex.seedDB.getConnected(record.originator());
if (seed == null) seed = sb.webIndex.seedDB.getDisconnected(record.originator());
if (seed != null) {
url = "http://" + seed.getPublicAddress() + "/Wiki.html?page=" + record.attribute("page", "");
entry = rowdef.newEntry(new byte[][]{
url.getBytes(),
(record.attribute("author", "Anonymous") + ": " + record.attribute("page", "")).getBytes("UTF-8"),
("Wiki Update: " + record.attribute("description", "")).getBytes("UTF-8"),
record.id().getBytes()
});
score = 4 + timeFactor(record.created());
}
} catch (IOException e) {}
if (record.category().equals(yacyNewsPool.CATEGORY_BLOG_ADD)) try {
yacySeed seed = sb.webIndex.seedDB.getConnected(record.originator());
if (seed == null) seed = sb.webIndex.seedDB.getDisconnected(record.originator());
if (seed != null) {
url = "http://" + seed.getPublicAddress() + "/Blog.html?page=" + record.attribute("page", "");
entry = rowdef.newEntry(new byte[][]{
url.getBytes(),
(record.attribute("author", "Anonymous") + ": " + record.attribute("page", "")).getBytes("UTF-8"),
("Blog Entry: " + record.attribute("subject", "")).getBytes("UTF-8"),
record.id().getBytes()
});
score = 4 + timeFactor(record.created());
}
} catch (IOException e) {}
// add/subtract votes and write record
if (entry != null) {
try {
urlhash = (new yacyURL(url, null)).hash();
} catch (MalformedURLException e) {
urlhash = null;
}
if (urlhash == null)
try {
urlhash = (new yacyURL("http://"+url, null)).hash();
} catch (MalformedURLException e) {
urlhash = null;
}
if (urlhash == null) {
System.out.println("Surftips: bad url '" + url + "' from news record " + record.toString());
continue;
}
if ((vote = negativeHashes.get(urlhash)) != null) {
score = Math.max(0, score - vote.intValue()); // do not go below zero
}
if ((vote = positiveHashes.get(urlhash)) != null) {
score += 2 * vote.intValue();
}
// consider double-entries
if (surftips.containsKey(urlhash)) {
ranking.addScore(urlhash, score);
} else {
ranking.setScore(urlhash, score);
surftips.put(urlhash, entry);
}
}
}
}
}