mirror of
https://github.com/yacy/yacy_search_server.git
synced 2024-09-19 00:01:41 +02:00
moved more servlets to new location
This commit is contained in:
Michael Peter Christen
parent
6d388bb7bf
commit
33889b4501
11
.classpath
11
.classpath
|
|
@ -1,18 +1,7 @@
|
|||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<classpath>
|
||||
<classpathentry excluding="api/|env/|processing/domaingraph/applet/|yacy/|api/bookmarks/|api/bookmarks/posts/|api/bookmarks/tags/|api/bookmarks/xbel/|solr/|gsa/|solr/collection1/|api/blacklists/|proxymsg/|p2p/" kind="src" path="htroot"/>
|
||||
<classpathentry excluding="bookmarks/|bookmarks/posts/|bookmarks/tags/|bookmarks/xbel/|blacklists/" kind="src" path="htroot/api"/>
|
||||
<classpathentry excluding="posts/|tags/|xbel/" kind="src" path="htroot/api/bookmarks"/>
|
||||
<classpathentry kind="src" path="htroot/api/blacklists"/>
|
||||
<classpathentry kind="src" path="htroot/api/bookmarks/posts"/>
|
||||
<classpathentry kind="src" path="htroot/api/bookmarks/tags"/>
|
||||
<classpathentry kind="src" path="htroot/api/bookmarks/xbel"/>
|
||||
<classpathentry kind="src" path="htroot/env"/>
|
||||
<classpathentry kind="src" path="htroot/p2p"/>
|
||||
<classpathentry kind="src" path="htroot/proxymsg"/>
|
||||
<classpathentry kind="src" path="htroot/yacy"/>
|
||||
<classpathentry kind="src" path="source"/>
|
||||
<classpathentry kind="src" path="test/java"/>
|
||||
<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER">
|
||||
<attributes>
|
||||
<attribute name="module" value="true"/>
|
||||
|
|
|
|||
|
|
@ -1,3 +1,5 @@
|
|||
package net.yacy.htroot.api.blacklists;
|
||||
|
||||
import java.util.Locale;
|
||||
|
||||
import net.yacy.cora.protocol.RequestHeader;
|
||||
|
|
@ -30,7 +32,7 @@ public class add_entry_p {
|
|||
"add_entry_p." + header.fileType().toString().toLowerCase(Locale.ROOT),
|
||||
WorkTables.TABLE_API_TYPE_CONFIGURATION,
|
||||
"add to blacklist '" + blacklistToUse + "': " + entry);
|
||||
|
||||
|
||||
if (BlacklistHelper.addBlacklistEntry(blacklistToUse, entry)) {
|
||||
prop.put(XML_ITEM_STATUS, RESULT_SUCCESS);
|
||||
|
||||
|
|
@ -39,7 +41,7 @@ public class add_entry_p {
|
|||
} else {
|
||||
prop.put(XML_ITEM_STATUS, RESULT_FAILURE);
|
||||
}
|
||||
|
||||
|
||||
} else {
|
||||
prop.put(XML_ITEM_STATUS, RESULT_FAILURE);
|
||||
}
|
||||
|
|
@ -1,3 +1,5 @@
|
|||
package net.yacy.htroot.api.blacklists;
|
||||
|
||||
import java.util.Locale;
|
||||
|
||||
import net.yacy.cora.protocol.RequestHeader;
|
||||
|
|
@ -40,7 +42,7 @@ public class delete_entry_p {
|
|||
} else {
|
||||
prop.put(XML_ITEM_STATUS, RESULT_FAILURE);
|
||||
}
|
||||
|
||||
|
||||
} else {
|
||||
prop.put(XML_ITEM_STATUS, RESULT_FAILURE);
|
||||
}
|
||||
|
|
@ -1,3 +1,5 @@
|
|||
package net.yacy.htroot.api.blacklists;
|
||||
|
||||
import java.io.File;
|
||||
import java.util.Collection;
|
||||
|
||||
|
|
@ -30,7 +32,7 @@ public class get_list_p {
|
|||
private static final int lastTypeIndex = BLACKLIST_TYPE_VALUES.length - 1;
|
||||
|
||||
public static serverObjects respond(@SuppressWarnings("unused") final RequestHeader header, final serverObjects post, @SuppressWarnings("unused") final serverSwitch env) {
|
||||
|
||||
|
||||
final serverObjects prop = new serverObjects();
|
||||
|
||||
final Collection<String> dirlist = FileUtils.getDirListing(ListManager.listsPath, Blacklist.BLACKLIST_FILENAME_FILTER);
|
||||
|
|
@ -40,23 +42,23 @@ public class get_list_p {
|
|||
if (dirlist != null) {
|
||||
for (final String element : dirlist) {
|
||||
if (element.equals(blackListName)) {
|
||||
|
||||
|
||||
prop.putXML(NAME, element);
|
||||
|
||||
prop.put(SHARED, ListManager.listSetContains(BLACK_LISTS_SHARED, element));
|
||||
|
||||
|
||||
int j = 0;
|
||||
for (final BlacklistType type : BLACKLIST_TYPE_VALUES) {
|
||||
prop.putXML(PREFIX_TYPES + j + POSTFIX_NAME, type.toString());
|
||||
prop.put(PREFIX_TYPES + j + POSTFIX_VALUE,
|
||||
ListManager.listSetContains(type + TYPES_EXT, element));
|
||||
|
||||
|
||||
prop.put(PREFIX_TYPES + j + POSTFIX_COMMA, j < lastTypeIndex);
|
||||
|
||||
|
||||
j++;
|
||||
}
|
||||
prop.put(TYPES, BlacklistType.values().length);
|
||||
|
||||
|
||||
prop.putXML(NAME, element);
|
||||
|
||||
final Collection<String> list = FileUtils.getListArray(new File(ListManager.listsPath, element));
|
||||
|
|
@ -68,18 +70,18 @@ public class get_list_p {
|
|||
if (entry.charAt(0) == '#') continue;
|
||||
|
||||
prop.putXML(PREFIX_ITEMS + count + POSTFIX_ITEM, entry);
|
||||
|
||||
|
||||
prop.put(PREFIX_ITEMS + count + POSTFIX_COMMA, count < lastItemCount);
|
||||
|
||||
|
||||
count++;
|
||||
}
|
||||
prop.put(ITEMS, count);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
return prop;
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
|
@ -1,3 +1,5 @@
|
|||
package net.yacy.htroot.api.blacklists;
|
||||
|
||||
import java.util.Collection;
|
||||
|
||||
import net.yacy.cora.protocol.RequestHeader;
|
||||
|
|
@ -46,13 +48,13 @@ public class get_metadata_p {
|
|||
ListManager.listSetContains(type + TYPES_EXT, element));
|
||||
|
||||
prop.put(PREFIX_LISTS + blacklistCount + INFIX_TYPES + j + POSTFIX_COMMA, j < lastTypeIndex);
|
||||
|
||||
|
||||
j++;
|
||||
}
|
||||
prop.put(PREFIX_LISTS + blacklistCount + POSTFIX_TYPES, BLACKLIST_TYPE_VALUES.length);
|
||||
|
||||
prop.put(PREFIX_LISTS + blacklistCount + POSTFIX_COMMA, blacklistCount < lastBlacklistCount);
|
||||
|
||||
|
||||
blacklistCount++;
|
||||
}
|
||||
}
|
||||
|
|
@ -1,4 +1,6 @@
|
|||
|
||||
package net.yacy.htroot.api;
|
||||
|
||||
import java.io.File;
|
||||
import java.util.List;
|
||||
|
||||
|
|
@ -8,7 +10,7 @@ import net.yacy.kelondro.util.FileUtils;
|
|||
import net.yacy.server.serverObjects;
|
||||
import net.yacy.server.serverSwitch;
|
||||
|
||||
public class blacklists {
|
||||
public class blacklists_ {
|
||||
|
||||
public static serverObjects respond(@SuppressWarnings("unused") final RequestHeader header, final serverObjects post, @SuppressWarnings("unused") final serverSwitch env) {
|
||||
final serverObjects prop = new serverObjects();
|
||||
|
|
@ -1,4 +1,6 @@
|
|||
|
||||
package net.yacy.htroot.api;
|
||||
|
||||
import java.io.File;
|
||||
import java.util.List;
|
||||
|
||||
|
|
@ -1,5 +1,7 @@
|
|||
|
||||
|
||||
package net.yacy.htroot.api.bookmarks;
|
||||
|
||||
import java.util.Date;
|
||||
import java.util.Iterator;
|
||||
|
||||
|
|
@ -155,8 +157,8 @@ public class get_bookmarks {
|
|||
}
|
||||
count = 0;
|
||||
while (count < itemsPerPage && it.hasNext()) {
|
||||
|
||||
BookmarksDB.Bookmark bookmark = sb.bookmarksDB.getBookmark(it.next());
|
||||
|
||||
final BookmarksDB.Bookmark bookmark = sb.bookmarksDB.getBookmark(it.next());
|
||||
if (bookmark != null) {
|
||||
prop.put("display_bookmarks_"+count+"_id",count);
|
||||
prop.put("display_bookmarks_"+count+"_link",bookmark.getUrl());
|
||||
|
|
@ -250,7 +252,7 @@ public class get_bookmarks {
|
|||
private static int print_XBEL(final Iterator<String> bit, int count) {
|
||||
Date date;
|
||||
while(bit.hasNext()){
|
||||
BookmarksDB.Bookmark bookmark = sb.bookmarksDB.getBookmark(bit.next());
|
||||
final BookmarksDB.Bookmark bookmark = sb.bookmarksDB.getBookmark(bit.next());
|
||||
if (bookmark != null) {
|
||||
date = new Date(bookmark.getTimeStamp());
|
||||
prop.put("display_xbel_"+count+"_elements", "<bookmark id=\"" + bookmark.getUrlHash()
|
||||
|
|
@ -1,6 +1,9 @@
|
|||
|
||||
|
||||
package net.yacy.htroot.api.bookmarks;
|
||||
|
||||
import java.util.Iterator;
|
||||
|
||||
import net.yacy.cora.protocol.RequestHeader;
|
||||
import net.yacy.cora.util.CommonPattern;
|
||||
import net.yacy.data.BookmarkHelper;
|
||||
|
|
@ -73,7 +76,7 @@ public class get_folders {
|
|||
// loop through bookmarkList
|
||||
it = sb.bookmarksDB.getBookmarksIterator(root, isAdmin);
|
||||
while (it.hasNext()) {
|
||||
BookmarksDB.Bookmark bm = sb.bookmarksDB.getBookmark(it.next());
|
||||
final BookmarksDB.Bookmark bm = sb.bookmarksDB.getBookmark(it.next());
|
||||
if (bm != null) {
|
||||
// TODO: get rid of bmtype
|
||||
if (post.containsKey("bmtype")) {
|
||||
|
|
@ -1,4 +1,6 @@
|
|||
|
||||
package net.yacy.htroot.api.bookmarks.posts;
|
||||
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
|
|
@ -1,4 +1,6 @@
|
|||
|
||||
package net.yacy.htroot.api.bookmarks.posts;
|
||||
|
||||
import java.util.Date;
|
||||
import java.util.Iterator;
|
||||
|
||||
|
|
@ -30,7 +32,7 @@ public class all {
|
|||
int count=0;
|
||||
Date date;
|
||||
while(it.hasNext()){
|
||||
BookmarksDB.Bookmark bookmark=switchboard.bookmarksDB.getBookmark(it.next());
|
||||
final BookmarksDB.Bookmark bookmark=switchboard.bookmarksDB.getBookmark(it.next());
|
||||
if (bookmark != null) {
|
||||
prop.putXML("posts_"+count+"_url", bookmark.getUrl());
|
||||
prop.putXML("posts_"+count+"_title", bookmark.getTitle());
|
||||
|
|
@ -1,4 +1,6 @@
|
|||
|
||||
package net.yacy.htroot.api.bookmarks.posts;
|
||||
|
||||
import java.net.MalformedURLException;
|
||||
|
||||
import net.yacy.cora.document.encoding.ASCII;
|
||||
|
|
@ -1,5 +1,7 @@
|
|||
|
||||
|
||||
package net.yacy.htroot.api.bookmarks.posts;
|
||||
|
||||
import java.text.ParseException;
|
||||
import java.util.Date;
|
||||
import java.util.List;
|
||||
|
|
@ -45,7 +47,7 @@ public class get {
|
|||
|
||||
final List<String> bookmark_hashes = switchboard.bookmarksDB.getDate(Long.toString(parsedDate.getTime())).getBookmarkList();
|
||||
for (final String bookmark_hash : bookmark_hashes){
|
||||
Bookmark bookmark = switchboard.bookmarksDB.getBookmark(bookmark_hash);
|
||||
final Bookmark bookmark = switchboard.bookmarksDB.getBookmark(bookmark_hash);
|
||||
if (bookmark != null) {
|
||||
if (ISO8601Formatter.FORMATTER.format(new Date(bookmark.getTimeStamp())).equals(date) &&
|
||||
tag==null || bookmark.getTags().contains(tag) &&
|
||||
|
|
@ -1,4 +1,6 @@
|
|||
|
||||
package net.yacy.htroot.api.bookmarks.tags;
|
||||
|
||||
import net.yacy.cora.protocol.RequestHeader;
|
||||
import net.yacy.data.BookmarksDB.Bookmark;
|
||||
import net.yacy.search.Switchboard;
|
||||
|
|
@ -21,7 +23,7 @@ public class addTag_p {
|
|||
switchboard.bookmarksDB.addTag(post.get("selectTag"), post.get("addTag"));
|
||||
prop.put("result", "1");//success
|
||||
} else if (post.containsKey("urlhash") && post.containsKey("addTag")) {
|
||||
Bookmark bm = switchboard.bookmarksDB.getBookmark(post.get("urlhash"));
|
||||
final Bookmark bm = switchboard.bookmarksDB.getBookmark(post.get("urlhash"));
|
||||
if (bm != null) {
|
||||
bm.addTag(post.get("addTag"));
|
||||
prop.put("result", "1");//success
|
||||
|
|
@ -1,4 +1,6 @@
|
|||
|
||||
package net.yacy.htroot.api.bookmarks.tags;
|
||||
|
||||
import net.yacy.cora.protocol.RequestHeader;
|
||||
import net.yacy.search.Switchboard;
|
||||
import net.yacy.server.serverObjects;
|
||||
|
|
@ -1,4 +1,6 @@
|
|||
|
||||
package net.yacy.htroot.api.bookmarks.tags;
|
||||
|
||||
import java.util.Iterator;
|
||||
|
||||
import net.yacy.cora.protocol.RequestHeader;
|
||||
|
|
@ -1,5 +1,7 @@
|
|||
|
||||
|
||||
package net.yacy.htroot.api.bookmarks.xbel;
|
||||
|
||||
import java.util.Date;
|
||||
import java.util.Iterator;
|
||||
|
||||
|
|
@ -107,7 +109,7 @@ public class xbel {
|
|||
private static int print_XBEL(final Iterator<String> bit, int count) {
|
||||
Date date;
|
||||
while(bit.hasNext()){
|
||||
BookmarksDB.Bookmark bookmark=switchboard.bookmarksDB.getBookmark(bit.next());
|
||||
final BookmarksDB.Bookmark bookmark=switchboard.bookmarksDB.getBookmark(bit.next());
|
||||
if (bookmark != null) {
|
||||
date=new Date(bookmark.getTimeStamp());
|
||||
prop.put("xbel_"+count+"_elements", "<bookmark id=\"" + bookmark.getUrlHash()
|
||||
|
|
@ -18,6 +18,7 @@
|
|||
* If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
package net.yacy.htroot.api;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.net.MalformedURLException;
|
||||
|
|
@ -96,7 +97,7 @@ public class citation {
|
|||
u = sb.getURL(ASCII.getBytes(hash));
|
||||
}
|
||||
if (u != null) uri = new DigestURL(u);
|
||||
} catch (IOException e) {
|
||||
} catch (final IOException e) {
|
||||
ConcurrentLog.logException(e);
|
||||
}
|
||||
}
|
||||
|
|
@ -112,13 +113,14 @@ public class citation {
|
|||
return prop;
|
||||
}
|
||||
@SuppressWarnings("unchecked")
|
||||
final
|
||||
ArrayList<String> title = (ArrayList<String>) doc.getFieldValue(CollectionSchema.title.getSolrFieldName());
|
||||
String text = (String) doc.getFieldValue(CollectionSchema.text_t.getSolrFieldName());
|
||||
final String text = (String) doc.getFieldValue(CollectionSchema.text_t.getSolrFieldName());
|
||||
|
||||
ArrayList<String> sentences = new ArrayList<String>();
|
||||
if (title != null) for (String s: title) if (s.length() > 0) sentences.add(s);
|
||||
final ArrayList<String> sentences = new ArrayList<String>();
|
||||
if (title != null) for (final String s: title) if (s.length() > 0) sentences.add(s);
|
||||
if (text != null && !text.isEmpty()) {
|
||||
SentenceReader sr = new SentenceReader(text);
|
||||
final SentenceReader sr = new SentenceReader(text);
|
||||
StringBuilder line;
|
||||
while (sr.hasNext()) {
|
||||
line = sr.next();
|
||||
|
|
@ -127,8 +129,8 @@ public class citation {
|
|||
}
|
||||
|
||||
// for each line make a statistic about the number of occurrences somewhere else
|
||||
OrderedScoreMap<String> scores = new OrderedScoreMap<String>(null); // accumulates scores for citating urls
|
||||
LinkedHashMap<String, Set<DigestURL>> sentenceOcc = new LinkedHashMap<String, Set<DigestURL>>();
|
||||
final OrderedScoreMap<String> scores = new OrderedScoreMap<String>(null); // accumulates scores for citating urls
|
||||
final LinkedHashMap<String, Set<DigestURL>> sentenceOcc = new LinkedHashMap<String, Set<DigestURL>>();
|
||||
for (String sentence: sentences) {
|
||||
if (sentence == null || sentence.length() < 40) {
|
||||
// do not count the very short sentences
|
||||
|
|
@ -137,12 +139,12 @@ public class citation {
|
|||
}
|
||||
try {
|
||||
sentence = sentence.replace('"', '\'');
|
||||
SolrDocumentList doclist = connector.getDocumentListByQuery("text_t:\"" + sentence + "\"", CollectionSchema.url_chars_i.getSolrFieldName() + " asc", 0, 100, CollectionSchema.sku.getSolrFieldName());
|
||||
int count = (int) doclist.getNumFound();
|
||||
final SolrDocumentList doclist = connector.getDocumentListByQuery("text_t:\"" + sentence + "\"", CollectionSchema.url_chars_i.getSolrFieldName() + " asc", 0, 100, CollectionSchema.sku.getSolrFieldName());
|
||||
final int count = (int) doclist.getNumFound();
|
||||
if (count > 0) {
|
||||
Set<DigestURL> list = new TreeSet<DigestURL>();
|
||||
for (SolrDocument d: doclist) {
|
||||
String u = (String) d.getFieldValue(CollectionSchema.sku.getSolrFieldName());
|
||||
final Set<DigestURL> list = new TreeSet<DigestURL>();
|
||||
for (final SolrDocument d: doclist) {
|
||||
final String u = (String) d.getFieldValue(CollectionSchema.sku.getSolrFieldName());
|
||||
if (u == null || u.equals(url)) continue;
|
||||
scores.inc(u);
|
||||
try {list.add(new DigestURL(u, null));} catch (final MalformedURLException e) {}
|
||||
|
|
@ -158,14 +160,14 @@ public class citation {
|
|||
// iterate the sentences
|
||||
int i = 0;
|
||||
int sentenceNr = 0;
|
||||
for (Map.Entry<String, Set<DigestURL>> se: sentenceOcc.entrySet()) {
|
||||
Set<DigestURL> app = se.getValue();
|
||||
for (final Map.Entry<String, Set<DigestURL>> se: sentenceOcc.entrySet()) {
|
||||
final Set<DigestURL> app = se.getValue();
|
||||
if (filter) { // prepare list, only include sentence with citation
|
||||
if (app != null && app.size() > 0) {
|
||||
StringBuilder dd = new StringBuilder(se.getKey());
|
||||
final StringBuilder dd = new StringBuilder(se.getKey());
|
||||
prop.put("sentences_" + i + "_dt", sentenceNr);
|
||||
dd.append("<br/>appears in:");
|
||||
for (DigestURL u : app) {
|
||||
for (final DigestURL u : app) {
|
||||
if (u != null) {
|
||||
dd.append(" <a href=\"").append(u.toNormalform(false)).append("\">").append(u.getHost()).append("</a>");
|
||||
}
|
||||
|
|
@ -174,11 +176,11 @@ public class citation {
|
|||
i++;
|
||||
}
|
||||
} else { // prepare list, include all sentences
|
||||
StringBuilder dd = new StringBuilder(se.getKey());
|
||||
final StringBuilder dd = new StringBuilder(se.getKey());
|
||||
prop.put("sentences_" + i + "_dt", sentenceNr);
|
||||
if (app != null && app.size() > 0) {
|
||||
dd.append("<br/>appears in:");
|
||||
for (DigestURL u : app) {
|
||||
for (final DigestURL u : app) {
|
||||
if (u != null) {
|
||||
dd.append(" <a href=\"").append(u.toNormalform(false)).append("\">").append(u.getHost()).append("</a>");
|
||||
}
|
||||
|
|
@ -193,14 +195,14 @@ public class citation {
|
|||
|
||||
// iterate the citations in order of number of citations
|
||||
i = 0;
|
||||
for (String u: scores.keyList(false)) {
|
||||
for (final String u: scores.keyList(false)) {
|
||||
try {
|
||||
DigestURL uu = new DigestURL(u, null);
|
||||
final DigestURL uu = new DigestURL(u, null);
|
||||
prop.put("citations_" + i + "_dt", "<a href=\"" + u + "\">" + u + "</a>");
|
||||
StringBuilder dd = new StringBuilder();
|
||||
final StringBuilder dd = new StringBuilder();
|
||||
dd.append("makes ").append(Integer.toString(scores.get(u))).append(" citations: of ").append(url);
|
||||
for (Map.Entry<String, Set<DigestURL>> se: sentenceOcc.entrySet()) {
|
||||
Set<DigestURL> occurls = se.getValue();
|
||||
for (final Map.Entry<String, Set<DigestURL>> se: sentenceOcc.entrySet()) {
|
||||
final Set<DigestURL> occurls = se.getValue();
|
||||
if (occurls != null && occurls.contains(uu)) dd.append("<br/><a href=\"../solr/select?q=text_t:%22").append(se.getKey().replace('"', '\'')).append("%22&rows=100&grep=&wt=grephtml\">").append(se.getKey()).append("</a>");
|
||||
}
|
||||
prop.put("citations_" + i + "_dd", dd.toString());
|
||||
|
|
@ -211,10 +213,10 @@ public class citation {
|
|||
|
||||
// find similar documents from different hosts
|
||||
i = 0;
|
||||
for (String u: scores.keyList(false)) {
|
||||
for (final String u: scores.keyList(false)) {
|
||||
if (scores.get(u) < ch) continue;
|
||||
try {
|
||||
DigestURL uu = new DigestURL(u, null);
|
||||
final DigestURL uu = new DigestURL(u, null);
|
||||
if (uu.getOrganization().equals(uri.getOrganization())) continue;
|
||||
prop.put("similar_links_" + i + "_url", u);
|
||||
i++;
|
||||
|
|
@ -1,4 +1,6 @@
|
|||
|
||||
package net.yacy.htroot.api;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.Iterator;
|
||||
|
|
@ -1,5 +1,7 @@
|
|||
|
||||
|
||||
package net.yacy.htroot.api;
|
||||
|
||||
import java.util.Date;
|
||||
import java.util.List;
|
||||
|
||||
|
|
@ -24,6 +24,8 @@
|
|||
// along with this program; if not, write to the Free Software
|
||||
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
||||
|
||||
package net.yacy.htroot.api;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Map.Entry;
|
||||
|
||||
|
|
@ -40,7 +42,7 @@ public class getpageinfo {
|
|||
@SuppressWarnings("unused")
|
||||
public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) {
|
||||
final serverObjects prop = new serverObjects();
|
||||
|
||||
|
||||
/* Redirect to getpageinfo_p */
|
||||
StringBuilder redirectedLocation;
|
||||
if(header != null && header.getPathInfo() != null && header.getPathInfo().endsWith(".json")) {
|
||||
|
|
@ -48,20 +50,20 @@ public class getpageinfo {
|
|||
} else {
|
||||
redirectedLocation = new StringBuilder("getpageinfo_p.xml");
|
||||
}
|
||||
|
||||
|
||||
/* Append eventual request parameters to the redirected location */
|
||||
if (post != null) {
|
||||
List<Entry<String, String>> parameters = post.entrySet();
|
||||
final List<Entry<String, String>> parameters = post.entrySet();
|
||||
if (parameters != null && !parameters.isEmpty()) {
|
||||
redirectedLocation.append("?");
|
||||
for (Entry<String, String> entry : parameters) {
|
||||
for (final Entry<String, String> entry : parameters) {
|
||||
redirectedLocation.append(entry.getKey()).append("=").append(entry.getValue()).append("&");
|
||||
}
|
||||
/* Remove trailing "&" */
|
||||
redirectedLocation.setLength(redirectedLocation.length() - 1);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
prop.put(serverObjects.ACTION_LOCATION, redirectedLocation.toString());
|
||||
return prop;
|
||||
}
|
||||
|
|
@ -24,6 +24,8 @@
|
|||
// along with this program; if not, write to the Free Software
|
||||
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
||||
|
||||
package net.yacy.htroot.api;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.net.MalformedURLException;
|
||||
import java.util.ArrayList;
|
||||
|
|
@ -61,7 +63,7 @@ public class getpageinfo_p {
|
|||
|
||||
/**
|
||||
* <p>Scrape and parse a resource at a specified URL to provide some information, depending on the requested actions.</p>
|
||||
*
|
||||
*
|
||||
* <p>
|
||||
* Example API calls :
|
||||
* <ul>
|
||||
|
|
@ -70,8 +72,8 @@ public class getpageinfo_p {
|
|||
* <li>Only check for an OAI Repository at CiteSeerX : http://localhost:8090/api/getpageinfo_p.xml?url=http://citeseerx.ist.psu.edu/oai2&actions=oai</li>
|
||||
* </ul>
|
||||
* </p>
|
||||
*
|
||||
*
|
||||
*
|
||||
*
|
||||
* @param header
|
||||
* servlet request header
|
||||
* @param post
|
||||
|
|
@ -82,7 +84,7 @@ public class getpageinfo_p {
|
|||
* <ul>
|
||||
* <li>title : look for the resource title, description, language, icons, keywords, and links</li>
|
||||
* <li>robots : check if crawling the resource is allowed by the eventual robots.txt policy file, and also if this file exposes sitemap(s) URLs.</li>
|
||||
* <li>oai : send an "Identify" OAI-PMH request (http://www.openarchives.org/OAI/openarchivesprotocol.html#Identify)
|
||||
* <li>oai : send an "Identify" OAI-PMH request (http://www.openarchives.org/OAI/openarchivesprotocol.html#Identify)
|
||||
* at the URL to check for a OAI-PMH response from an Open Archive Initiative Repository</li>
|
||||
* </ul>
|
||||
* </li>
|
||||
|
|
@ -139,14 +141,14 @@ public class getpageinfo_p {
|
|||
}
|
||||
net.yacy.document.Document scraper = null;
|
||||
if (u != null) try {
|
||||
ClientIdentification.Agent agent = ClientIdentification.getAgent(post.get("agentName", ClientIdentification.yacyInternetCrawlerAgentName));
|
||||
|
||||
final ClientIdentification.Agent agent = ClientIdentification.getAgent(post.get("agentName", ClientIdentification.yacyInternetCrawlerAgentName));
|
||||
|
||||
if(post.containsKey("maxBytes")) {
|
||||
/* A maxBytes limit is specified : let's try to parse only the amount of bytes given */
|
||||
final long maxBytes = post.getLong("maxBytes", sb.loader.protocolMaxFileSize(u));
|
||||
scraper = sb.loader.loadDocumentAsLimitedStream(u, CacheStrategy.IFEXIST, BlacklistType.CRAWLER, agent, maxLinks, maxBytes);
|
||||
} else {
|
||||
/* No maxBytes limit : apply regular parsing with default crawler limits.
|
||||
/* No maxBytes limit : apply regular parsing with default crawler limits.
|
||||
* Eventual maxLinks limit will apply after loading and parsing the document. */
|
||||
scraper = sb.loader.loadDocumentAsStream(u, CacheStrategy.IFEXIST, BlacklistType.CRAWLER, agent);
|
||||
}
|
||||
|
|
@ -161,9 +163,9 @@ public class getpageinfo_p {
|
|||
prop.putXML("title", scraper.dc_title());
|
||||
|
||||
// put the icons that belong to the document
|
||||
Set<DigestURL> iconURLs = scraper.getIcons().keySet();
|
||||
final Set<DigestURL> iconURLs = scraper.getIcons().keySet();
|
||||
long count = 0;
|
||||
for (DigestURL iconURL : iconURLs) {
|
||||
for (final DigestURL iconURL : iconURLs) {
|
||||
if(count >= maxLinks) {
|
||||
break;
|
||||
}
|
||||
|
|
@ -200,7 +202,7 @@ public class getpageinfo_p {
|
|||
count = 0;
|
||||
final Iterator<AnchorURL> urisIt = uris.iterator();
|
||||
while (urisIt.hasNext()) {
|
||||
AnchorURL uri = urisIt.next();
|
||||
final AnchorURL uri = urisIt.next();
|
||||
if (uri == null) continue;
|
||||
if(count >= maxLinks) {
|
||||
break;
|
||||
|
|
@ -221,15 +223,15 @@ public class getpageinfo_p {
|
|||
final DigestURL theURL = new DigestURL(url);
|
||||
|
||||
// determine if crawling of the current URL is allowed
|
||||
ClientIdentification.Agent agent = ClientIdentification.getAgent(post.get("agentName", ClientIdentification.yacyInternetCrawlerAgentName));
|
||||
RobotsTxtEntry robotsEntry = sb.robots.getEntry(theURL, agent);
|
||||
final ClientIdentification.Agent agent = ClientIdentification.getAgent(post.get("agentName", ClientIdentification.yacyInternetCrawlerAgentName));
|
||||
final RobotsTxtEntry robotsEntry = sb.robots.getEntry(theURL, agent);
|
||||
prop.put("robots-allowed", robotsEntry == null ? 1 : robotsEntry.isDisallowed(theURL) ? 0 : 1);
|
||||
prop.putHTML("robotsInfo", robotsEntry == null ? "" : robotsEntry.getInfo());
|
||||
|
||||
// get the sitemap URL(s) of the domain
|
||||
final List<String> sitemaps = robotsEntry == null ? new ArrayList<String>(0) : robotsEntry.getSitemaps();
|
||||
int count = 0;
|
||||
for (String sitemap : sitemaps) {
|
||||
for (final String sitemap : sitemaps) {
|
||||
if(count >= maxLinks) {
|
||||
break;
|
||||
}
|
||||
|
|
@ -21,6 +21,8 @@
|
|||
// along with this program; if not, write to the Free Software
|
||||
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
||||
|
||||
package net.yacy.htroot.api;
|
||||
|
||||
import java.util.Date;
|
||||
import java.util.Iterator;
|
||||
import java.util.Map;
|
||||
|
|
@ -43,7 +45,7 @@ public class latency_p {
|
|||
Map.Entry<String, Host> e;
|
||||
int c = 0;
|
||||
Latency.Host host;
|
||||
ClientIdentification.Agent agent = post == null ? ClientIdentification.yacyInternetCrawlerAgent : ClientIdentification.getAgent(post.get("agentName", ClientIdentification.yacyInternetCrawlerAgentName));
|
||||
final ClientIdentification.Agent agent = post == null ? ClientIdentification.yacyInternetCrawlerAgent : ClientIdentification.getAgent(post.get("agentName", ClientIdentification.yacyInternetCrawlerAgentName));
|
||||
while (i.hasNext()) {
|
||||
e = i.next();
|
||||
host = e.getValue();
|
||||
|
|
@ -17,6 +17,8 @@
|
|||
// along with this program; if not, write to the Free Software
|
||||
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
||||
|
||||
package net.yacy.htroot.api;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.net.MalformedURLException;
|
||||
|
||||
|
|
@ -46,25 +48,25 @@ public class linkstructure {
|
|||
final boolean xml = ext.equals("xml");
|
||||
|
||||
final Switchboard sb = (Switchboard) env;
|
||||
Fulltext fulltext = sb.index.fulltext();
|
||||
final Fulltext fulltext = sb.index.fulltext();
|
||||
if (post == null) return prop;
|
||||
boolean authenticated = sb.adminAuthenticated(header) >= 2;
|
||||
int maxtime = Math.min(post.getInt("maxtime", 60000), authenticated ? 300000 : 1000);
|
||||
int maxnodes = Math.min(post.getInt("maxnodes", 10000), authenticated ? 10000000 : 100);
|
||||
HyperlinkGraph hlg = new HyperlinkGraph();
|
||||
final boolean authenticated = sb.adminAuthenticated(header) >= 2;
|
||||
final int maxtime = Math.min(post.getInt("maxtime", 60000), authenticated ? 300000 : 1000);
|
||||
final int maxnodes = Math.min(post.getInt("maxnodes", 10000), authenticated ? 10000000 : 100);
|
||||
final HyperlinkGraph hlg = new HyperlinkGraph();
|
||||
int maxdepth = 0;
|
||||
|
||||
if (post.get("about", null) != null) try {
|
||||
// get link structure within a host
|
||||
String about = post.get("about", null); // may be a URL, a URL hash or a domain hash
|
||||
final String about = post.get("about", null); // may be a URL, a URL hash or a domain hash
|
||||
DigestURL url = null;
|
||||
String hostname = null;
|
||||
if (about.length() == 12 && Base64Order.enhancedCoder.wellformed(ASCII.getBytes(about))) {
|
||||
byte[] urlhash = ASCII.getBytes(about);
|
||||
final byte[] urlhash = ASCII.getBytes(about);
|
||||
try {
|
||||
String u = authenticated ? sb.getURL(urlhash) : null;
|
||||
final String u = authenticated ? sb.getURL(urlhash) : null;
|
||||
url = u == null ? null : new DigestURL(u);
|
||||
} catch (IOException e) {
|
||||
} catch (final IOException e) {
|
||||
ConcurrentLog.logException(e);
|
||||
}
|
||||
} else if (url == null && about.length() > 0) { // consider "about" as url or hostname
|
||||
|
|
@ -79,8 +81,8 @@ public class linkstructure {
|
|||
} catch (final MalformedURLException e) {}
|
||||
else if (post.get("to", null) != null) try {
|
||||
// get link structure between two links
|
||||
DigestURL to = new DigestURL(post.get("to", null), null); // must be an url
|
||||
DigestURL from = post.get("from", null) == null ? null : new DigestURL(post.get("from", null)); // can be null or must be an url
|
||||
final DigestURL to = new DigestURL(post.get("to", null), null); // must be an url
|
||||
final DigestURL from = post.get("from", null) == null ? null : new DigestURL(post.get("from", null)); // can be null or must be an url
|
||||
hlg.path(sb.index, from, to, maxtime, maxnodes);
|
||||
} catch (final MalformedURLException e) {}
|
||||
|
||||
|
|
@ -100,12 +102,12 @@ public class linkstructure {
|
|||
|
||||
private static void writeGraph(final servletProperties prop, final HyperlinkGraph hlg, final int maxdepth) {
|
||||
int c = 0;
|
||||
for (HyperlinkEdge e: hlg) {
|
||||
for (final HyperlinkEdge e: hlg) {
|
||||
prop.putJSON("edges_" + c + "_source", e.source.getPath());
|
||||
prop.putJSON("edges_" + c + "_target", e.target.type.equals(HyperlinkType.Outbound) ? e.target.toNormalform(true) : e.target.getPath());
|
||||
prop.putJSON("edges_" + c + "_type", e.target.type.name());
|
||||
Integer depth_source = hlg.getDepth(e.source);
|
||||
Integer depth_target = hlg.getDepth(e.target);
|
||||
final Integer depth_source = hlg.getDepth(e.source);
|
||||
final Integer depth_target = hlg.getDepth(e.target);
|
||||
prop.put("edges_" + c + "_depthSource", depth_source == null ? -1 : depth_source.intValue());
|
||||
prop.put("edges_" + c + "_depthTarget", depth_target == null ? -1 : depth_target.intValue());
|
||||
c++;
|
||||
|
|
@ -7,17 +7,19 @@
|
|||
* modify it under the terms of the GNU Lesser General Public
|
||||
* License as published by the Free Software Foundation; either
|
||||
* version 2.1 of the License, or (at your option) any later version.
|
||||
*
|
||||
*
|
||||
* This library is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
* Lesser General Public License for more details.
|
||||
*
|
||||
*
|
||||
* You should have received a copy of the GNU Lesser General Public License
|
||||
* along with this program in the file lgpl21.txt
|
||||
* If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
package net.yacy.htroot.api;
|
||||
|
||||
import java.net.MalformedURLException;
|
||||
import java.util.Date;
|
||||
|
||||
|
|
@ -38,15 +40,15 @@ import net.yacy.server.serverObjects;
|
|||
import net.yacy.server.serverSwitch;
|
||||
|
||||
public class push_p {
|
||||
|
||||
|
||||
// test: http://localhost:8090/api/push_p.json?count=1&synchronous=false&commit=false&url-0=http://nowhere.cc/example.txt&data-0=%22hello%20world%22&lastModified-0=Tue,%2015%20Nov%201994%2012:45:26%20GMT&contentType-0=text/plain&collection-0=testpush&responseHeader-0=
|
||||
|
||||
|
||||
public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) {
|
||||
final Switchboard sb = (Switchboard) env;
|
||||
final serverObjects prop = new serverObjects();
|
||||
|
||||
// display mode: this only helps to display a nice input form for test cases
|
||||
int c = post == null ? 1 : post.getInt("c", 0);
|
||||
final int c = post == null ? 1 : post.getInt("c", 0);
|
||||
if (c > 0) {
|
||||
prop.put("mode", 0);
|
||||
for (int i = 0; i < c; i++) prop.put("mode_input_" + i + "_count", i);
|
||||
|
|
@ -54,45 +56,45 @@ public class push_p {
|
|||
prop.put("mode_count", c);
|
||||
return prop;
|
||||
}
|
||||
|
||||
|
||||
// push mode: this does a document upload
|
||||
prop.put("mode", 1);
|
||||
if (post == null) return prop;
|
||||
boolean commit = post.getBoolean("commit");
|
||||
boolean synchronous = commit || post.getBoolean("synchronous");
|
||||
int count = post.getInt("count", 0);
|
||||
final boolean commit = post.getBoolean("commit");
|
||||
final boolean synchronous = commit || post.getBoolean("synchronous");
|
||||
final int count = post.getInt("count", 0);
|
||||
boolean successall = true;
|
||||
int countsuccess = 0;
|
||||
int countfail = 0;
|
||||
for (int i = 0; i < count; i++) {
|
||||
try {
|
||||
prop.put("mode_results_" + i + "_item", i);
|
||||
String u = post.get("url-" + i, "");
|
||||
final String u = post.get("url-" + i, "");
|
||||
prop.put("mode_results_" + i + "_url", u);
|
||||
DigestURL url = new DigestURL(u);
|
||||
String collection = post.get("collection-" + i, "");
|
||||
String lastModified = post.get("lastModified-" + i, ""); // must be in RFC1123 format
|
||||
String contentType = post.get("contentType-" + i, "");
|
||||
String data64 = post.get("data-" + i + "$file", ""); // multi-file uploads are all base64-encoded in YaCyDefaultServlet.parseMultipart
|
||||
final DigestURL url = new DigestURL(u);
|
||||
final String collection = post.get("collection-" + i, "");
|
||||
final String lastModified = post.get("lastModified-" + i, ""); // must be in RFC1123 format
|
||||
final String contentType = post.get("contentType-" + i, "");
|
||||
final String data64 = post.get("data-" + i + "$file", ""); // multi-file uploads are all base64-encoded in YaCyDefaultServlet.parseMultipart
|
||||
byte[] data = Base64Order.standardCoder.decode(data64);
|
||||
if ((data == null || data.length == 0) && data64.length() > 0) data = UTF8.getBytes(data64); // for test cases
|
||||
|
||||
|
||||
// create response header
|
||||
final ResponseHeader responseHeader = new ResponseHeader(200);
|
||||
responseHeader.put(HeaderFramework.LAST_MODIFIED, lastModified);
|
||||
responseHeader.put(HeaderFramework.CONTENT_TYPE, contentType);
|
||||
responseHeader.put(HeaderFramework.CONTENT_LENGTH, Long.toString(data.length));
|
||||
// add generic fields
|
||||
String[] responseHeaderMap = post.getParams("responseHeader-" + i); // strings with key-value pairs; separated by ':'
|
||||
for (String kv: responseHeaderMap) {
|
||||
int p = kv.indexOf(':');
|
||||
final String[] responseHeaderMap = post.getParams("responseHeader-" + i); // strings with key-value pairs; separated by ':'
|
||||
for (final String kv: responseHeaderMap) {
|
||||
final int p = kv.indexOf(':');
|
||||
if (p < 0) continue;
|
||||
String key = kv.substring(0, p).trim();
|
||||
String value = kv.substring(p + 1).trim();
|
||||
final String key = kv.substring(0, p).trim();
|
||||
final String value = kv.substring(p + 1).trim();
|
||||
responseHeader.put(key, value);
|
||||
}
|
||||
CrawlProfile profile = sb.crawler.getPushCrawlProfile(collection);
|
||||
|
||||
final CrawlProfile profile = sb.crawler.getPushCrawlProfile(collection);
|
||||
|
||||
// create requests and artificial response
|
||||
final Request request = new Request(
|
||||
ASCII.getBytes(sb.peers.mySeed().hash),
|
||||
|
|
@ -103,15 +105,15 @@ public class push_p {
|
|||
profile.handle(), // the name of the prefetch profile. This must not be null!
|
||||
0, // forkfactor sum of anchors of all ancestors
|
||||
profile.timezoneOffset());
|
||||
Response response = new Response(
|
||||
final Response response = new Response(
|
||||
request,
|
||||
null,
|
||||
responseHeader,
|
||||
profile,
|
||||
false, // from cache?
|
||||
data); // content
|
||||
IndexingQueueEntry in = new IndexingQueueEntry(response, null, null);
|
||||
|
||||
final IndexingQueueEntry in = new IndexingQueueEntry(response, null, null);
|
||||
|
||||
if (synchronous) {
|
||||
// synchronously process the content
|
||||
sb.storeDocumentIndex(sb.webStructureAnalysis(sb.condenseDocument(sb.parseDocument(in))));
|
||||
|
|
@ -123,7 +125,7 @@ public class push_p {
|
|||
|
||||
prop.put("mode_results_" + i + "_success_message", YaCyDefaultServlet.getContext(header, sb) + "/solr/select?q=sku:%22" + u + "%22");
|
||||
countsuccess++;
|
||||
} catch (MalformedURLException e) {
|
||||
} catch (final MalformedURLException e) {
|
||||
e.printStackTrace();
|
||||
prop.put("mode_results_" + i + "_success", "0");
|
||||
prop.put("mode_results_" + i + "_success_message", e.getMessage());
|
||||
|
|
@ -136,10 +138,10 @@ public class push_p {
|
|||
prop.put("mode_count", count);
|
||||
prop.put("mode_countsuccess", countsuccess);
|
||||
prop.put("mode_countfail", countfail);
|
||||
|
||||
|
||||
if (synchronous && commit) sb.index.fulltext().commit(true);
|
||||
|
||||
|
||||
return prop;
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
|
@ -22,6 +22,8 @@
|
|||
* If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
package net.yacy.htroot.api;
|
||||
|
||||
import net.yacy.cora.federate.solr.SchemaConfiguration;
|
||||
import net.yacy.cora.federate.solr.SchemaDeclaration;
|
||||
import net.yacy.cora.protocol.HeaderFramework;
|
||||
|
|
@ -42,13 +44,13 @@ public class schema {
|
|||
final Switchboard sb = (Switchboard) env;
|
||||
|
||||
String schemaName = CollectionSchema.CORE_NAME;
|
||||
if (post != null) schemaName = post.get("core", schemaName);
|
||||
|
||||
if (post != null) schemaName = post.get("core", schemaName);
|
||||
|
||||
// write schema
|
||||
int c = 0;
|
||||
SchemaConfiguration solrSchema = schemaName.equals(CollectionSchema.CORE_NAME) ? sb.index.fulltext().getDefaultConfiguration() : sb.index.fulltext().getWebgraphConfiguration();
|
||||
SchemaDeclaration[] cc = schemaName.equals(CollectionSchema.CORE_NAME) ? CollectionSchema.values() : WebgraphSchema.values();
|
||||
for (SchemaDeclaration field : cc) {
|
||||
final SchemaConfiguration solrSchema = schemaName.equals(CollectionSchema.CORE_NAME) ? sb.index.fulltext().getDefaultConfiguration() : sb.index.fulltext().getWebgraphConfiguration();
|
||||
final SchemaDeclaration[] cc = schemaName.equals(CollectionSchema.CORE_NAME) ? CollectionSchema.values() : WebgraphSchema.values();
|
||||
for (final SchemaDeclaration field : cc) {
|
||||
if (solrSchema.contains(field.name())) {
|
||||
addField(prop, c, field);
|
||||
c++;
|
||||
|
|
@ -63,13 +65,13 @@ public class schema {
|
|||
c++;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
//if (solrScheme.contains(YaCySchema.author)) {addField(prop, c, YaCySchema.author_sxt);}
|
||||
prop.put("fields", c);
|
||||
|
||||
if (schemaName.equals(CollectionSchema.CORE_NAME)) {
|
||||
prop.put("copyFieldAuthor", solrSchema.contains(CollectionSchema.author) ? 1 : 0);
|
||||
|
||||
|
||||
prop.put("solruniquekey", CollectionSchema.id.getSolrFieldName());
|
||||
prop.put("solrdefaultsearchfield",
|
||||
solrSchema.contains(CollectionSchema.text_t) ? CollectionSchema.text_t.getSolrFieldName() :
|
||||
|
|
@ -79,7 +81,7 @@ public class schema {
|
|||
);
|
||||
} else {
|
||||
prop.put("copyFieldAuthor", 0);
|
||||
|
||||
|
||||
prop.put("solruniquekey", WebgraphSchema.id.getSolrFieldName());
|
||||
prop.put("solrdefaultsearchfield",
|
||||
solrSchema.contains(WebgraphSchema.target_linktext_s) ? WebgraphSchema.target_linktext_s.getSolrFieldName() :
|
||||
|
|
@ -87,18 +89,18 @@ public class schema {
|
|||
solrSchema.contains(WebgraphSchema.target_alt_s) ? WebgraphSchema.target_alt_s.getSolrFieldName() :
|
||||
WebgraphSchema.id.getSolrFieldName()
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// add CORS Access header
|
||||
final ResponseHeader outgoingHeader = new ResponseHeader(200);
|
||||
outgoingHeader.put(HeaderFramework.CORS_ALLOW_ORIGIN, "*");
|
||||
prop.setOutgoingHeader(outgoingHeader);
|
||||
|
||||
prop.setOutgoingHeader(outgoingHeader);
|
||||
|
||||
// return rewrite properties
|
||||
return prop;
|
||||
}
|
||||
|
||||
private static void addField(servletProperties prop, int c, SchemaDeclaration field) {
|
||||
|
||||
private static void addField(final servletProperties prop, final int c, final SchemaDeclaration field) {
|
||||
prop.put("fields_" + c + "_solrname", field.getSolrFieldName());
|
||||
prop.put("fields_" + c + "_type", field.getType().printName());
|
||||
prop.put("fields_" + c + "_comment", field.getComment());
|
||||
|
|
@ -7,19 +7,21 @@
|
|||
* modify it under the terms of the GNU Lesser General Public
|
||||
* License as published by the Free Software Foundation; either
|
||||
* version 2.1 of the License, or (at your option) any later version.
|
||||
*
|
||||
*
|
||||
* This library is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
* Lesser General Public License for more details.
|
||||
*
|
||||
*
|
||||
* You should have received a copy of the GNU Lesser General Public License
|
||||
* along with this program in the file lgpl21.txt
|
||||
* If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
import java.io.File;
|
||||
package net.yacy.htroot.api;
|
||||
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.StandardCopyOption;
|
||||
|
|
@ -34,7 +36,7 @@ import net.yacy.server.serverObjects;
|
|||
import net.yacy.server.serverSwitch;
|
||||
|
||||
public class share {
|
||||
|
||||
|
||||
/**
|
||||
* Servlet to share any kind of binary to this peer.
|
||||
* That mean you can upload 'things'. While this is the generic view,
|
||||
|
|
@ -53,12 +55,12 @@ public class share {
|
|||
final serverObjects prop = new serverObjects();
|
||||
|
||||
// display mode: this only helps to display a nice input form for test cases
|
||||
int c = post == null ? 1 : post.getInt("c", 0);
|
||||
final int c = post == null ? 1 : post.getInt("c", 0);
|
||||
if (c > 0) {
|
||||
prop.put("mode", 0);
|
||||
return prop;
|
||||
}
|
||||
|
||||
|
||||
// push mode: this does a document upload
|
||||
prop.put("mode", 1);
|
||||
prop.put("mode_success", 0);
|
||||
|
|
@ -66,9 +68,9 @@ public class share {
|
|||
prop.put("mode_countsuccess", 0);
|
||||
prop.put("mode_countfail", 0);
|
||||
prop.put("mode_item", "");
|
||||
|
||||
|
||||
if (post == null) return prop;
|
||||
|
||||
|
||||
// check file name
|
||||
String filename = post.get("data", "");
|
||||
if (filename.isEmpty()) {
|
||||
|
|
@ -79,9 +81,9 @@ public class share {
|
|||
prop.put("mode_success_message", "no index dump file (" + Fulltext.yacy_dump_prefix + "*.xml.gz)");
|
||||
return prop;
|
||||
}
|
||||
|
||||
|
||||
// check data
|
||||
String dataString = post.get("data$file", "");
|
||||
final String dataString = post.get("data$file", "");
|
||||
if (dataString.length() == 0) return prop;
|
||||
byte[] data;
|
||||
if (filename.endsWith(".base64")) {
|
||||
|
|
@ -91,20 +93,20 @@ public class share {
|
|||
data = UTF8.getBytes(dataString);
|
||||
}
|
||||
if (data == null || data.length == 0) return prop;
|
||||
|
||||
|
||||
// modify the file name; ignore and replace the used transaction token
|
||||
int ttp = filename.indexOf("_t");
|
||||
final int ttp = filename.indexOf("_t");
|
||||
if (ttp < 0) return prop;
|
||||
if (filename.charAt(ttp + 3) != '.') return prop;
|
||||
filename = filename.substring(0, ttp) + "_ts" + filename.substring(ttp + 3); // transaction token: 's' as 'shared'.
|
||||
|
||||
|
||||
// process the data
|
||||
File tmpFile = new File(yacy.shareDumpDefaultPath, filename + ".tmp");
|
||||
File finalFile = new File(yacy.shareDumpDefaultPath, filename);
|
||||
final File tmpFile = new File(yacy.shareDumpDefaultPath, filename + ".tmp");
|
||||
final File finalFile = new File(yacy.shareDumpDefaultPath, filename);
|
||||
try {
|
||||
Files.copy(new ByteArrayInputStream(data), tmpFile.toPath(), StandardCopyOption.REPLACE_EXISTING);
|
||||
tmpFile.renameTo(finalFile);
|
||||
} catch (IOException e) {
|
||||
} catch (final IOException e) {
|
||||
ConcurrentLog.logException(e);
|
||||
return prop;
|
||||
}
|
||||
|
|
@ -112,5 +114,5 @@ public class share {
|
|||
prop.put("mode_success", 1);
|
||||
return prop;
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
|
@ -18,6 +18,8 @@
|
|||
* If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
package net.yacy.htroot.api;
|
||||
|
||||
import java.awt.Container;
|
||||
import java.awt.Image;
|
||||
import java.awt.MediaTracker;
|
||||
|
|
@ -89,30 +91,30 @@ public class snapshot {
|
|||
defaultResponse.authenticationRequired();
|
||||
return defaultResponse;
|
||||
}
|
||||
int maxcount = post == null ? 10 : post.getInt("maxcount", 10);
|
||||
int depthx = post == null ? -1 : post.getInt("depth", -1);
|
||||
Integer depth = depthx == -1 ? null : depthx;
|
||||
String orderx = post == null ? "ANY" : post.get("order", "ANY");
|
||||
Snapshots.Order order = Snapshots.Order.valueOf(orderx);
|
||||
String statex = post == null ? Transactions.State.INVENTORY.name() : post.get("state", Transactions.State.INVENTORY.name());
|
||||
Transactions.State state = Transactions.State.valueOf(statex);
|
||||
String host = post == null ? null : post.get("host");
|
||||
Map<String, Revisions> iddate = Transactions.select(host, depth, order, maxcount, state);
|
||||
final int maxcount = post == null ? 10 : post.getInt("maxcount", 10);
|
||||
final int depthx = post == null ? -1 : post.getInt("depth", -1);
|
||||
final Integer depth = depthx == -1 ? null : depthx;
|
||||
final String orderx = post == null ? "ANY" : post.get("order", "ANY");
|
||||
final Snapshots.Order order = Snapshots.Order.valueOf(orderx);
|
||||
final String statex = post == null ? Transactions.State.INVENTORY.name() : post.get("state", Transactions.State.INVENTORY.name());
|
||||
final Transactions.State state = Transactions.State.valueOf(statex);
|
||||
final String host = post == null ? null : post.get("host");
|
||||
final Map<String, Revisions> iddate = Transactions.select(host, depth, order, maxcount, state);
|
||||
// now select the URL from the index for these ids in iddate and make an RSS feed
|
||||
RSSFeed rssfeed = new RSSFeed(Integer.MAX_VALUE);
|
||||
final RSSFeed rssfeed = new RSSFeed(Integer.MAX_VALUE);
|
||||
rssfeed.setChannel(new RSSMessage("Snapshot list for host = " + host + ", depth = " + depth + ", order = " + order + ", maxcount = " + maxcount, "", ""));
|
||||
for (Map.Entry<String, Revisions> e: iddate.entrySet()) {
|
||||
for (final Map.Entry<String, Revisions> e: iddate.entrySet()) {
|
||||
try {
|
||||
String u = e.getValue().url == null ? sb.index.fulltext().getURL(e.getKey()) : e.getValue().url;
|
||||
final String u = e.getValue().url == null ? sb.index.fulltext().getURL(e.getKey()) : e.getValue().url;
|
||||
if (u == null) continue;
|
||||
RSSMessage message = new RSSMessage(u, "", new DigestURL(u), e.getKey());
|
||||
final RSSMessage message = new RSSMessage(u, "", new DigestURL(u), e.getKey());
|
||||
message.setPubDate(e.getValue().dates[0]);
|
||||
rssfeed.addMessage(message);
|
||||
} catch (IOException ee) {
|
||||
} catch (final IOException ee) {
|
||||
ConcurrentLog.logException(ee);
|
||||
}
|
||||
}
|
||||
byte[] rssBinary = UTF8.getBytes(rssfeed.toString());
|
||||
final byte[] rssBinary = UTF8.getBytes(rssfeed.toString());
|
||||
return new ByteArrayInputStream(rssBinary);
|
||||
}
|
||||
|
||||
|
|
@ -126,34 +128,34 @@ public class snapshot {
|
|||
}
|
||||
final boolean pngjpg = ext.equals("png") || ext.equals(DEFAULT_EXT);
|
||||
String urlhash = post.get("urlhash", "");
|
||||
String url = post.get("url", "");
|
||||
final String url = post.get("url", "");
|
||||
DigestURL durl = null;
|
||||
if (urlhash.length() == 0 && url.length() > 0) {
|
||||
try {
|
||||
durl = new DigestURL(url);
|
||||
urlhash = ASCII.String(durl.hash());
|
||||
} catch (MalformedURLException e) {
|
||||
} catch (final MalformedURLException e) {
|
||||
}
|
||||
}
|
||||
if (durl == null && urlhash.length() > 0) {
|
||||
try {
|
||||
String u = sb.index.fulltext().getURL(urlhash);
|
||||
final String u = sb.index.fulltext().getURL(urlhash);
|
||||
durl = u == null ? null : new DigestURL(u);
|
||||
} catch (IOException e) {
|
||||
} catch (final IOException e) {
|
||||
ConcurrentLog.logException(e);
|
||||
}
|
||||
}
|
||||
|
||||
if (ext.equals("json")) {
|
||||
// command interface: view and change a transaction state, get metadata about transactions in the past
|
||||
String command = post.get("command", "metadata");
|
||||
String statename = post.get("state");
|
||||
JSONObject result = new JSONObject();
|
||||
final String command = post.get("command", "metadata");
|
||||
final String statename = post.get("state");
|
||||
final JSONObject result = new JSONObject();
|
||||
try {
|
||||
if (command.equals("status")) {
|
||||
// return a status of the transaction archive
|
||||
JSONObject sizes = new JSONObject();
|
||||
for (Map.Entry<String, Integer> state: Transactions.sizes().entrySet()) sizes.put(state.getKey(), state.getValue());
|
||||
final JSONObject sizes = new JSONObject();
|
||||
for (final Map.Entry<String, Integer> state: Transactions.sizes().entrySet()) sizes.put(state.getKey(), state.getValue());
|
||||
result.put("size", sizes);
|
||||
} else if (command.equals("list")) {
|
||||
if (!authenticated) {
|
||||
|
|
@ -161,36 +163,36 @@ public class snapshot {
|
|||
return defaultResponse;
|
||||
}
|
||||
// return a status of the transaction archive
|
||||
String host = post.get("host");
|
||||
String depth = post.get("depth");
|
||||
int depthi = depth == null ? -1 : Integer.parseInt(depth);
|
||||
for (Transactions.State state: statename == null ?
|
||||
final String host = post.get("host");
|
||||
final String depth = post.get("depth");
|
||||
final int depthi = depth == null ? -1 : Integer.parseInt(depth);
|
||||
for (final Transactions.State state: statename == null ?
|
||||
new Transactions.State[]{Transactions.State.INVENTORY, Transactions.State.ARCHIVE} :
|
||||
new Transactions.State[]{Transactions.State.valueOf(statename)}) {
|
||||
if (host == null) {
|
||||
JSONObject hostCountInventory = new JSONObject();
|
||||
for (String h: Transactions.listHosts(state)) {
|
||||
int size = Transactions.listIDsSize(h, depthi, state);
|
||||
final JSONObject hostCountInventory = new JSONObject();
|
||||
for (final String h: Transactions.listHosts(state)) {
|
||||
final int size = Transactions.listIDsSize(h, depthi, state);
|
||||
if (size > 0) hostCountInventory.put(h, size);
|
||||
}
|
||||
result.put("count." + state.name(), hostCountInventory);
|
||||
} else {
|
||||
TreeMap<Integer, Collection<Revisions>> ids = Transactions.listIDs(host, depthi, state);
|
||||
final TreeMap<Integer, Collection<Revisions>> ids = Transactions.listIDs(host, depthi, state);
|
||||
if (ids == null) {
|
||||
result.put("result", "fail");
|
||||
result.put("comment", "no entries for host " + host + " found");
|
||||
} else {
|
||||
for (Map.Entry<Integer, Collection<Revisions>> entry: ids.entrySet()) {
|
||||
for (Revisions r: entry.getValue()) {
|
||||
for (final Map.Entry<Integer, Collection<Revisions>> entry: ids.entrySet()) {
|
||||
for (final Revisions r: entry.getValue()) {
|
||||
try {
|
||||
JSONObject metadata = new JSONObject();
|
||||
String u = r.url != null ? r.url : sb.index.fulltext().getURL(r.urlhash);
|
||||
final JSONObject metadata = new JSONObject();
|
||||
final String u = r.url != null ? r.url : sb.index.fulltext().getURL(r.urlhash);
|
||||
metadata.put("url", u == null ? "unknown" : u);
|
||||
metadata.put("dates", r.dates);
|
||||
assert r.depth == entry.getKey().intValue();
|
||||
metadata.put("depth", entry.getKey().intValue());
|
||||
result.put(r.urlhash, metadata);
|
||||
} catch (IOException e) {}
|
||||
} catch (final IOException e) {}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -201,7 +203,7 @@ public class snapshot {
|
|||
defaultResponse.authenticationRequired();
|
||||
return defaultResponse;
|
||||
}
|
||||
Revisions r = Transactions.commit(urlhash);
|
||||
final Revisions r = Transactions.commit(urlhash);
|
||||
if (r != null) {
|
||||
result.put("result", "success");
|
||||
result.put("depth", r.depth);
|
||||
|
|
@ -216,7 +218,7 @@ public class snapshot {
|
|||
defaultResponse.authenticationRequired();
|
||||
return defaultResponse;
|
||||
}
|
||||
Revisions r = Transactions.rollback(urlhash);
|
||||
final Revisions r = Transactions.rollback(urlhash);
|
||||
if (r != null) {
|
||||
result.put("result", "success");
|
||||
result.put("depth", r.depth);
|
||||
|
|
@ -239,8 +241,8 @@ public class snapshot {
|
|||
r = Transactions.getRevisions(state, urlhash);
|
||||
}
|
||||
if (r != null) {
|
||||
JSONObject metadata = new JSONObject();
|
||||
String u = r.url != null ? r.url : sb.index.fulltext().getURL(r.urlhash);
|
||||
final JSONObject metadata = new JSONObject();
|
||||
final String u = r.url != null ? r.url : sb.index.fulltext().getURL(r.urlhash);
|
||||
metadata.put("url", u == null ? "unknown" : u);
|
||||
metadata.put("dates", r.dates);
|
||||
metadata.put("depth", r.depth);
|
||||
|
|
@ -249,7 +251,7 @@ public class snapshot {
|
|||
}
|
||||
} catch (IOException |IllegalArgumentException e) {}
|
||||
}
|
||||
} catch (JSONException e) {
|
||||
} catch (final JSONException e) {
|
||||
ConcurrentLog.logException(e);
|
||||
}
|
||||
String json = result.toString();
|
||||
|
|
@ -263,14 +265,14 @@ public class snapshot {
|
|||
}
|
||||
|
||||
if (xml) {
|
||||
Collection<File> xmlSnapshots = Transactions.findPaths(durl, "xml", Transactions.State.ANY);
|
||||
final Collection<File> xmlSnapshots = Transactions.findPaths(durl, "xml", Transactions.State.ANY);
|
||||
File xmlFile = null;
|
||||
if (xmlSnapshots.isEmpty()) {
|
||||
throw new TemplateProcessingException("Could not find the xml snapshot file.", HttpStatus.SC_NOT_FOUND);
|
||||
}
|
||||
xmlFile = xmlSnapshots.iterator().next();
|
||||
try {
|
||||
byte[] xmlBinary = FileUtils.read(xmlFile);
|
||||
final byte[] xmlBinary = FileUtils.read(xmlFile);
|
||||
return new ByteArrayInputStream(xmlBinary);
|
||||
} catch (final IOException e) {
|
||||
ConcurrentLog.logException(e);
|
||||
|
|
@ -288,12 +290,12 @@ public class snapshot {
|
|||
"Could not find the pdf snapshot file. You must be authenticated to generate one on the fly.",
|
||||
HttpStatus.SC_NOT_FOUND);
|
||||
}
|
||||
SolrDocument sd = sb.index.fulltext().getMetadata(durl.hash());
|
||||
final SolrDocument sd = sb.index.fulltext().getMetadata(durl.hash());
|
||||
boolean success = false;
|
||||
if (sd == null) {
|
||||
success = Transactions.store(durl, new Date(), 99, false, true, sb.getConfigBool(SwitchboardConstants.PROXY_TRANSPARENT_PROXY, false) ? "http://127.0.0.1:" + sb.getConfigInt(SwitchboardConstants.SERVER_PORT, 8090) : null, sb.getConfig("crawler.http.acceptLanguage", null));
|
||||
} else {
|
||||
SolrInputDocument sid = sb.index.fulltext().getDefaultConfiguration().toSolrInputDocument(sd);
|
||||
final SolrInputDocument sid = sb.index.fulltext().getDefaultConfiguration().toSolrInputDocument(sd);
|
||||
success = Transactions.store(sid, false, true, true, sb.getConfigBool(SwitchboardConstants.PROXY_TRANSPARENT_PROXY, false) ? "http://127.0.0.1:" + sb.getConfigInt(SwitchboardConstants.SERVER_PORT, 8090) : null, sb.getConfig("crawler.http.acceptLanguage", null));
|
||||
}
|
||||
if (success) {
|
||||
|
|
@ -312,7 +314,7 @@ public class snapshot {
|
|||
}
|
||||
if (pdf) {
|
||||
try {
|
||||
byte[] pdfBinary = FileUtils.read(pdfFile);
|
||||
final byte[] pdfBinary = FileUtils.read(pdfFile);
|
||||
return new ByteArrayInputStream(pdfBinary);
|
||||
} catch (final IOException e) {
|
||||
ConcurrentLog.logException(e);
|
||||
|
|
@ -321,10 +323,10 @@ public class snapshot {
|
|||
}
|
||||
|
||||
if (pngjpg) {
|
||||
int width = Math.min(post.getInt("width", DEFAULT_WIDTH), DEFAULT_WIDTH);
|
||||
int height = Math.min(post.getInt("height", DEFAULT_HEIGHT), DEFAULT_HEIGHT);
|
||||
final int width = Math.min(post.getInt("width", DEFAULT_WIDTH), DEFAULT_WIDTH);
|
||||
final int height = Math.min(post.getInt("height", DEFAULT_HEIGHT), DEFAULT_HEIGHT);
|
||||
String imageFileStub = pdfFile.getAbsolutePath(); imageFileStub = imageFileStub.substring(0, imageFileStub.length() - 3); // cut off extension
|
||||
File imageFile = new File(imageFileStub + DEFAULT_WIDTH + "." + DEFAULT_HEIGHT + "." + ext);
|
||||
final File imageFile = new File(imageFileStub + DEFAULT_WIDTH + "." + DEFAULT_HEIGHT + "." + ext);
|
||||
if (!imageFile.exists() && authenticated) {
|
||||
if(!Html2Image.pdf2image(pdfFile, imageFile, DEFAULT_WIDTH, DEFAULT_HEIGHT, DEFAULT_DENSITY, DEFAULT_QUALITY)) {
|
||||
throw new TemplateProcessingException(
|
||||
|
|
@ -339,7 +341,7 @@ public class snapshot {
|
|||
}
|
||||
if (width == DEFAULT_WIDTH && height == DEFAULT_HEIGHT) {
|
||||
try {
|
||||
byte[] imageBinary = FileUtils.read(imageFile);
|
||||
final byte[] imageBinary = FileUtils.read(imageFile);
|
||||
return new ByteArrayInputStream(imageBinary);
|
||||
} catch (final IOException e) {
|
||||
ConcurrentLog.logException(e);
|
||||
|
|
@ -362,7 +364,7 @@ public class snapshot {
|
|||
* Ensure there is no alpha component on the ouput image, as it is pointless
|
||||
* here and it is not well supported by the JPEGImageWriter from OpenJDK
|
||||
*/
|
||||
BufferedImage scaledBufferedImg = new BufferedImage(width, height, BufferedImage.TYPE_INT_RGB);
|
||||
final BufferedImage scaledBufferedImg = new BufferedImage(width, height, BufferedImage.TYPE_INT_RGB);
|
||||
scaledBufferedImg.createGraphics().drawImage(scaled, 0, 0, width, height, null);
|
||||
return new EncodedImage(scaledBufferedImg, ext, true);
|
||||
} catch (final IOException e) {
|
||||
|
|
@ -25,6 +25,8 @@
|
|||
// along with this program; if not, write to the Free Software
|
||||
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
||||
|
||||
package net.yacy.htroot.api;
|
||||
|
||||
import java.io.IOException;
|
||||
|
||||
import net.yacy.cora.protocol.RequestHeader;
|
||||
|
|
@ -57,8 +59,8 @@ public class status_p {
|
|||
final serverObjects prop = new serverObjects();
|
||||
final boolean html = post != null && post.containsKey("html");
|
||||
prop.setLocalized(html);
|
||||
Segment segment = sb.index;
|
||||
Fulltext fulltext = segment.fulltext();
|
||||
final Segment segment = sb.index;
|
||||
final Fulltext fulltext = segment.fulltext();
|
||||
|
||||
prop.put("rejected", "0");
|
||||
sb.updateMySeed();
|
||||
|
|
@ -98,7 +100,7 @@ public class status_p {
|
|||
prop.putNum("loaderMax", sb.getConfigLong(SwitchboardConstants.CRAWLER_THREADS_ACTIVE_MAX, 10));
|
||||
|
||||
//local crawl queue
|
||||
BusyThread localCrawl = sb.getThread(SwitchboardConstants.CRAWLJOB_LOCAL_CRAWL);
|
||||
final BusyThread localCrawl = sb.getThread(SwitchboardConstants.CRAWLJOB_LOCAL_CRAWL);
|
||||
prop.putNum("localCrawlSize", localCrawl == null ? 0 : localCrawl.getJobCount());
|
||||
prop.put("localCrawlState", sb.crawlJobIsPaused(SwitchboardConstants.CRAWLJOB_LOCAL_CRAWL) ? STATE_PAUSED : STATE_RUNNING);
|
||||
|
||||
|
|
@ -107,7 +109,7 @@ public class status_p {
|
|||
prop.put("limitCrawlState", STATE_RUNNING);
|
||||
|
||||
//remote crawl queue
|
||||
BusyThread remoteCrawl = sb.getThread(SwitchboardConstants.CRAWLJOB_REMOTE_TRIGGERED_CRAWL);
|
||||
final BusyThread remoteCrawl = sb.getThread(SwitchboardConstants.CRAWLJOB_REMOTE_TRIGGERED_CRAWL);
|
||||
prop.putNum("remoteCrawlSize", remoteCrawl == null ? 0 : remoteCrawl.getJobCount());
|
||||
prop.put("remoteCrawlState", sb.crawlJobIsPaused(SwitchboardConstants.CRAWLJOB_REMOTE_TRIGGERED_CRAWL) ? STATE_PAUSED : STATE_RUNNING);
|
||||
|
||||
|
|
@ -125,7 +127,7 @@ public class status_p {
|
|||
profile = sb.crawler.getActive(h);
|
||||
if (CrawlSwitchboard.DEFAULT_PROFILES.contains(profile.name())) continue;
|
||||
profile.putProfileEntry("crawlProfiles_list_", prop, true, false, count, domlistlength);
|
||||
RowHandleSet urlhashes = sb.crawler.getURLHashes(h);
|
||||
final RowHandleSet urlhashes = sb.crawler.getURLHashes(h);
|
||||
prop.put("crawlProfiles_list_" + count + "_count", urlhashes == null ? "unknown" : Integer.toString(urlhashes.size()));
|
||||
if (profile.urlMustMatchPattern() == CrawlProfile.MATCH_ALL_PATTERN) {
|
||||
hosts = hosts + "," + profile.name();
|
||||
|
|
@ -138,20 +140,20 @@ public class status_p {
|
|||
|
||||
prop.put("postprocessingRunning", CollectionConfiguration.postprocessingRunning ? 1 : 0);
|
||||
|
||||
boolean processCollection = sb.index.fulltext().getDefaultConfiguration().contains(CollectionSchema.process_sxt) && (sb.index.connectedCitation() || sb.index.fulltext().useWebgraph());
|
||||
boolean processWebgraph = sb.index.fulltext().getWebgraphConfiguration().contains(WebgraphSchema.process_sxt) && sb.index.fulltext().useWebgraph();
|
||||
final boolean processCollection = sb.index.fulltext().getDefaultConfiguration().contains(CollectionSchema.process_sxt) && (sb.index.connectedCitation() || sb.index.fulltext().useWebgraph());
|
||||
final boolean processWebgraph = sb.index.fulltext().getWebgraphConfiguration().contains(WebgraphSchema.process_sxt) && sb.index.fulltext().useWebgraph();
|
||||
|
||||
long timeSinceStart = (processCollection || processWebgraph) && CollectionConfiguration.postprocessingRunning ? System.currentTimeMillis() - CollectionConfiguration.postprocessingStartTime : 0;
|
||||
final long timeSinceStart = (processCollection || processWebgraph) && CollectionConfiguration.postprocessingRunning ? System.currentTimeMillis() - CollectionConfiguration.postprocessingStartTime : 0;
|
||||
//postprocessingCollection1Count = 0;
|
||||
//postprocessingsWebgraphCount = 0;
|
||||
long collectionRemainingCount = 0, webgraphRemainingCount = 0;
|
||||
if (processCollection) try {collectionRemainingCount = sb.index.fulltext().getDefaultConnector().getCountByQuery("{!cache=false}" + CollectionConfiguration.collection1query(sb.index, null));} catch (IOException e) {}
|
||||
if (processWebgraph) try {webgraphRemainingCount = sb.index.fulltext().getWebgraphConnector().getCountByQuery(CollectionConfiguration.webgraphquery(sb.index, null));} catch (IOException e) {}
|
||||
long countSinceStart = CollectionConfiguration.postprocessingRunning ? CollectionConfiguration.postprocessingCollection1Count + CollectionConfiguration.postprocessingWebgraphCount - collectionRemainingCount - webgraphRemainingCount : 0;
|
||||
int speed = timeSinceStart == 0 ? 0 : (int) (60000 * countSinceStart / timeSinceStart); // pages per minute
|
||||
long remainingTime = speed == 0 ? 0 : 60000 * collectionRemainingCount / speed; // millis
|
||||
int remainingTimeMinutes = (int) (remainingTime / 60000);
|
||||
int remainingTimeSeconds = (int) ((remainingTime - (remainingTimeMinutes * 60000)) / 1000);
|
||||
if (processCollection) try {collectionRemainingCount = sb.index.fulltext().getDefaultConnector().getCountByQuery("{!cache=false}" + CollectionConfiguration.collection1query(sb.index, null));} catch (final IOException e) {}
|
||||
if (processWebgraph) try {webgraphRemainingCount = sb.index.fulltext().getWebgraphConnector().getCountByQuery(CollectionConfiguration.webgraphquery(sb.index, null));} catch (final IOException e) {}
|
||||
final long countSinceStart = CollectionConfiguration.postprocessingRunning ? CollectionConfiguration.postprocessingCollection1Count + CollectionConfiguration.postprocessingWebgraphCount - collectionRemainingCount - webgraphRemainingCount : 0;
|
||||
final int speed = timeSinceStart == 0 ? 0 : (int) (60000 * countSinceStart / timeSinceStart); // pages per minute
|
||||
final long remainingTime = speed == 0 ? 0 : 60000 * collectionRemainingCount / speed; // millis
|
||||
final int remainingTimeMinutes = (int) (remainingTime / 60000);
|
||||
final int remainingTimeSeconds = (int) ((remainingTime - (remainingTimeMinutes * 60000)) / 1000);
|
||||
|
||||
prop.put("postprocessingCollectionRemainingCount", collectionRemainingCount);
|
||||
prop.put("postprocessingWebgraphRemainingCount", webgraphRemainingCount);
|
||||
|
|
@ -17,6 +17,8 @@
|
|||
// along with this program; if not, write to the Free Software
|
||||
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
||||
|
||||
package net.yacy.htroot.api;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
|
|
@ -21,6 +21,8 @@
|
|||
// along with this program; if not, write to the Free Software
|
||||
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
||||
|
||||
package net.yacy.htroot.api;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Iterator;
|
||||
|
|
@ -39,12 +41,12 @@ import net.yacy.server.serverSwitch;
|
|||
public class termlist_p {
|
||||
|
||||
private final static ConcurrentLog log = new ConcurrentLog("TERMLIST");
|
||||
|
||||
|
||||
public static serverObjects respond(@SuppressWarnings("unused") final RequestHeader header, final serverObjects post, final serverSwitch env) {
|
||||
|
||||
final serverObjects prop = new serverObjects();
|
||||
final Switchboard sb = (Switchboard) env;
|
||||
Segment segment = sb.index;
|
||||
final Segment segment = sb.index;
|
||||
final boolean delete = post != null && post.containsKey("delete");
|
||||
final long mincount = post == null ? 10000 : post.getLong("mincount", 10000);
|
||||
Rating<byte[]> e;
|
||||
|
|
@ -54,7 +56,7 @@ public class termlist_p {
|
|||
String hstring;
|
||||
final ArrayList<byte[]> deleteterms = new ArrayList<byte[]>();
|
||||
long over1000 = 0, over10000 = 0, over100000 = 0, over1000000 = 0, over10000000 = 0, over100000000 = 0;
|
||||
|
||||
|
||||
final IndexCell<WordReference> termIndex = segment.termIndex();
|
||||
int rowsize = 0;
|
||||
if(termIndex != null) {
|
||||
|
|
@ -24,6 +24,8 @@
|
|||
// along with this program; if not, write to the Free Software
|
||||
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
||||
|
||||
package net.yacy.htroot.api;
|
||||
|
||||
import java.text.ParseException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Date;
|
||||
|
|
@ -38,8 +40,8 @@ import net.yacy.cora.date.GenericFormatter;
|
|||
import net.yacy.cora.protocol.RequestHeader;
|
||||
import net.yacy.cora.sorting.OrderedScoreMap;
|
||||
import net.yacy.cora.util.CommonPattern;
|
||||
import net.yacy.search.EventTracker.Event;
|
||||
import net.yacy.search.EventTracker;
|
||||
import net.yacy.search.EventTracker.Event;
|
||||
import net.yacy.search.query.AccessTracker;
|
||||
import net.yacy.server.serverObjects;
|
||||
import net.yacy.server.serverSwitch;
|
||||
|
|
@ -48,18 +50,18 @@ public final class timeline_p {
|
|||
|
||||
// example:
|
||||
// http://localhost:8090/api/timeline_p.xml?from=20140601000000&to=20140629000000&data=queries&head=2&period=6h
|
||||
|
||||
|
||||
public static serverObjects respond(@SuppressWarnings("unused") final RequestHeader header, final serverObjects post, final serverSwitch env) {
|
||||
final serverObjects prop = new serverObjects();
|
||||
if ((post == null) || (env == null)) return prop;
|
||||
|
||||
// get type of data to be listed in the timeline
|
||||
int maxeventsperperiod = post.getInt("head", 1); // the maximum number of events per period
|
||||
String period = post.get("period", ""); // must be an integer with a character c at the end, c = Y|M|d|h|m|s
|
||||
final int maxeventsperperiod = post.getInt("head", 1); // the maximum number of events per period
|
||||
final String period = post.get("period", ""); // must be an integer with a character c at the end, c = Y|M|d|h|m|s
|
||||
long periodlength = 0;
|
||||
if (period.length() > 0) {
|
||||
char c = period.charAt(period.length() - 1);
|
||||
long p = Long.parseLong(period.substring(0, period.length() - 1));
|
||||
final char c = period.charAt(period.length() - 1);
|
||||
final long p = Long.parseLong(period.substring(0, period.length() - 1));
|
||||
if (c == 's') periodlength = p * AbstractFormatter.secondMillis;
|
||||
else if (c == 'm') periodlength = p * AbstractFormatter.minuteMillis;
|
||||
else if (c == 'h') periodlength = p * AbstractFormatter.hourMillis;
|
||||
|
|
@ -69,42 +71,42 @@ public final class timeline_p {
|
|||
else periodlength = 0;
|
||||
}
|
||||
final String[] data = CommonPattern.COMMA.split(post.get("data", "")); // a string of word hashes that shall be searched and combined
|
||||
Map<String, List<EventTracker.Event>> proc = new HashMap<>();
|
||||
for (String s: data) if (s.length() > 0) proc.put(s, null);
|
||||
|
||||
final Map<String, List<EventTracker.Event>> proc = new HashMap<>();
|
||||
for (final String s: data) if (s.length() > 0) proc.put(s, null);
|
||||
|
||||
// get a time period
|
||||
Date fromDate = new Date(0);
|
||||
Date toDate = new Date();
|
||||
try {fromDate = GenericFormatter.SHORT_SECOND_FORMATTER.parse(post.get("from", GenericFormatter.SHORT_SECOND_FORMATTER.format(fromDate)), 0).getTime();} catch (ParseException e) {}
|
||||
try {toDate = GenericFormatter.SHORT_SECOND_FORMATTER.parse(post.get("to", GenericFormatter.SHORT_SECOND_FORMATTER.format(toDate)), 0).getTime();} catch (ParseException e) {}
|
||||
|
||||
try {fromDate = GenericFormatter.SHORT_SECOND_FORMATTER.parse(post.get("from", GenericFormatter.SHORT_SECOND_FORMATTER.format(fromDate)), 0).getTime();} catch (final ParseException e) {}
|
||||
try {toDate = GenericFormatter.SHORT_SECOND_FORMATTER.parse(post.get("to", GenericFormatter.SHORT_SECOND_FORMATTER.format(toDate)), 0).getTime();} catch (final ParseException e) {}
|
||||
|
||||
// get latest dump;
|
||||
AccessTracker.dumpLog();
|
||||
|
||||
|
||||
// fill proc with events from the given data and time period
|
||||
if (proc.containsKey("queries")) {
|
||||
List<EventTracker.Event> events = AccessTracker.readLog(AccessTracker.getDumpFile(), fromDate, toDate);
|
||||
final List<EventTracker.Event> events = AccessTracker.readLog(AccessTracker.getDumpFile(), fromDate, toDate);
|
||||
proc.put("queries", events);
|
||||
}
|
||||
|
||||
|
||||
// mix all events into one event list
|
||||
TreeMap<String, EventTracker.Event> eax = new TreeMap<>();
|
||||
for (List<EventTracker.Event> events: proc.values()) if (events != null) {
|
||||
for (EventTracker.Event event: events) eax.put(event.getFormattedDate(), event);
|
||||
final TreeMap<String, EventTracker.Event> eax = new TreeMap<>();
|
||||
for (final List<EventTracker.Event> events: proc.values()) if (events != null) {
|
||||
for (final EventTracker.Event event: events) eax.put(event.getFormattedDate(), event);
|
||||
}
|
||||
proc.clear(); // we don't need that here any more
|
||||
List<EventTracker.Event> ea = new ArrayList<>();
|
||||
for (Event event: eax.values()) ea.add(event);
|
||||
|
||||
for (final Event event: eax.values()) ea.add(event);
|
||||
|
||||
if (periodlength > 0 && ea.size() > 0) {
|
||||
// create a statistical analysis; step by chunks of periodlength entries
|
||||
Event firstEvent = ea.iterator().next();
|
||||
long startDate = fromDate.getTime();
|
||||
//TreeMap<Date, EventTracker.Event>
|
||||
OrderedScoreMap<String> accumulation = new OrderedScoreMap<>(null);
|
||||
List<EventTracker.Event> eap = new ArrayList<>();
|
||||
final OrderedScoreMap<String> accumulation = new OrderedScoreMap<>(null);
|
||||
final List<EventTracker.Event> eap = new ArrayList<>();
|
||||
String limit = GenericFormatter.SHORT_SECOND_FORMATTER.format(new Date(startDate + periodlength));
|
||||
for (Event event: ea) {
|
||||
for (final Event event: ea) {
|
||||
if (event.getFormattedDate().compareTo(limit) >= 0) {
|
||||
// write accumulation of the score map into eap
|
||||
stats(accumulation, eap, startDate, periodlength, maxeventsperperiod, firstEvent.type);
|
||||
|
|
@ -115,14 +117,14 @@ public final class timeline_p {
|
|||
accumulation.inc(event.payload.toString());
|
||||
}
|
||||
stats(accumulation, eap, startDate, periodlength, maxeventsperperiod, firstEvent.type);
|
||||
|
||||
|
||||
// overwrite the old table for out
|
||||
ea = eap;
|
||||
}
|
||||
|
||||
|
||||
// create a list of these events
|
||||
int count = 0;
|
||||
for (Event event: ea) {
|
||||
for (final Event event: ea) {
|
||||
prop.put("event_" + count + "_time", event.getFormattedDate());
|
||||
prop.put("event_" + count + "_isPeriod", event.duration == 0 ? 0 : 1);
|
||||
prop.put("event_" + count + "_isPeriod_duration", event.duration);
|
||||
|
|
@ -136,15 +138,15 @@ public final class timeline_p {
|
|||
return prop;
|
||||
}
|
||||
|
||||
private static void stats(OrderedScoreMap<String> accumulation, List<EventTracker.Event> eap, long startDate, long periodlength, int head, String type) {
|
||||
private static void stats(final OrderedScoreMap<String> accumulation, final List<EventTracker.Event> eap, final long startDate, final long periodlength, final int head, final String type) {
|
||||
// write accumulation of the score map into eap
|
||||
Iterator<String> si = accumulation.keys(false);
|
||||
final Iterator<String> si = accumulation.keys(false);
|
||||
int c = 0;
|
||||
while (si.hasNext() && c++ < head) {
|
||||
String key = si.next();
|
||||
final String key = si.next();
|
||||
eap.add(new Event(startDate, periodlength, type, key, accumulation.get(key)));
|
||||
}
|
||||
accumulation.clear();
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
|
@ -1,5 +1,7 @@
|
|||
|
||||
|
||||
package net.yacy.htroot.api;
|
||||
|
||||
import net.yacy.cora.protocol.RequestHeader;
|
||||
import net.yacy.peers.operation.yacyBuildProperties;
|
||||
import net.yacy.server.serverObjects;
|
||||
|
|
@ -21,6 +21,7 @@
|
|||
// along with this program; if not, write to the Free Software
|
||||
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
||||
|
||||
package net.yacy.htroot.api;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.net.MalformedURLException;
|
||||
|
|
@ -113,25 +114,25 @@ public class webstructure {
|
|||
public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) {
|
||||
final serverObjects prop = new serverObjects();
|
||||
final Switchboard sb = (Switchboard) env;
|
||||
String about = post == null ? null : post.get("about", null); // may be a URL, a URL hash or a domain hash
|
||||
final String about = post == null ? null : post.get("about", null); // may be a URL, a URL hash or a domain hash
|
||||
prop.put("out", 0);
|
||||
prop.put("in", 0);
|
||||
prop.put("references", 0);
|
||||
prop.put("citations", 0);
|
||||
boolean authenticated = sb.adminAuthenticated(header) >= 2;
|
||||
final boolean authenticated = sb.adminAuthenticated(header) >= 2;
|
||||
if (about != null) {
|
||||
DigestURL url = null;
|
||||
byte[] urlhash = null;
|
||||
Set<String> hostHashes = new HashSet<>();
|
||||
final Set<String> hostHashes = new HashSet<>();
|
||||
if (about.length() == 6 && Base64Order.enhancedCoder.wellformed(ASCII.getBytes(about))) {
|
||||
hostHashes.add(about);
|
||||
} else if (about.length() == 12 && Base64Order.enhancedCoder.wellformed(ASCII.getBytes(about))) {
|
||||
urlhash = ASCII.getBytes(about);
|
||||
hostHashes.add(about.substring(6));
|
||||
if (authenticated) try {
|
||||
String u = sb.getURL(urlhash);
|
||||
final String u = sb.getURL(urlhash);
|
||||
url = u == null ? null : new DigestURL(u);
|
||||
} catch (IOException e) {
|
||||
} catch (final IOException e) {
|
||||
url = null;
|
||||
ConcurrentLog.logException(e);
|
||||
}
|
||||
|
|
@ -187,7 +188,7 @@ public class webstructure {
|
|||
prop.put("references", 1);
|
||||
net.yacy.document.Document scraper = null;
|
||||
if (url != null) try {
|
||||
ClientIdentification.Agent agent = ClientIdentification.getAgent(post.get("agentName", ClientIdentification.yacyInternetCrawlerAgentName));
|
||||
final ClientIdentification.Agent agent = ClientIdentification.getAgent(post.get("agentName", ClientIdentification.yacyInternetCrawlerAgentName));
|
||||
scraper = sb.loader.loadDocument(url, CacheStrategy.IFEXIST, null, agent);
|
||||
} catch (final IOException e) {
|
||||
ConcurrentLog.logException(e);
|
||||
|
|
@ -203,8 +204,8 @@ public class webstructure {
|
|||
int d = 0;
|
||||
Iterator<DigestURL> i = scraper.inboundLinks().keySet().iterator();
|
||||
while (i.hasNext()) {
|
||||
DigestURL refurl = i.next();
|
||||
byte[] refhash = refurl.hash();
|
||||
final DigestURL refurl = i.next();
|
||||
final byte[] refhash = refurl.hash();
|
||||
prop.putXML("references_documents_0_anchors_" + d + "_url", refurl.toNormalform(true));
|
||||
prop.put("references_documents_0_anchors_" + d + "_hash", refhash);
|
||||
prop.put("references_documents_0_anchors_" + d + "_outbound", 0);
|
||||
|
|
@ -212,8 +213,8 @@ public class webstructure {
|
|||
}
|
||||
i = scraper.outboundLinks().keySet().iterator();
|
||||
while (i.hasNext()) {
|
||||
DigestURL refurl = i.next();
|
||||
byte[] refhash = refurl.hash();
|
||||
final DigestURL refurl = i.next();
|
||||
final byte[] refhash = refurl.hash();
|
||||
prop.putXML("references_documents_0_anchors_" + d + "_url", refurl.toNormalform(true));
|
||||
prop.put("references_documents_0_anchors_" + d + "_hash", refhash);
|
||||
prop.put("references_documents_0_anchors_" + d + "_outbound", 1);
|
||||
|
|
@ -228,9 +229,9 @@ public class webstructure {
|
|||
|
||||
// citations
|
||||
prop.put("citations", 1);
|
||||
ReferenceReportCache rrc = sb.index.getReferenceReportCache();
|
||||
final ReferenceReportCache rrc = sb.index.getReferenceReportCache();
|
||||
ReferenceReport rr = null;
|
||||
try {rr = rrc.getReferenceReport(ASCII.String(urlhash), true);} catch (IOException e) {}
|
||||
try {rr = rrc.getReferenceReport(ASCII.String(urlhash), true);} catch (final IOException e) {}
|
||||
if (rr != null && rr.getInternalCount() > 0 && rr.getExternalCount() > 0) {
|
||||
prop.put("citations_count", 1);
|
||||
prop.put("citations_documents", 1);
|
||||
|
|
@ -240,11 +241,11 @@ public class webstructure {
|
|||
prop.put("citations_documents_0_urle", url == null ? 0 : 1);
|
||||
if (url != null) prop.putXML("citations_documents_0_urle_url", url.toNormalform(true));
|
||||
int d = 0;
|
||||
HandleSet ids = rr.getInternallIDs();
|
||||
try {ids.putAll(rr.getExternalIDs());} catch (SpaceExceededException e) {}
|
||||
Iterator<byte[]> i = ids.iterator();
|
||||
final HandleSet ids = rr.getInternallIDs();
|
||||
try {ids.putAll(rr.getExternalIDs());} catch (final SpaceExceededException e) {}
|
||||
final Iterator<byte[]> i = ids.iterator();
|
||||
while (i.hasNext()) {
|
||||
byte[] refhash = i.next();
|
||||
final byte[] refhash = i.next();
|
||||
String refurl;
|
||||
try {
|
||||
refurl = authenticated ? sb.getURL(refhash) : null;
|
||||
|
|
@ -253,7 +254,7 @@ public class webstructure {
|
|||
prop.put("citations_documents_0_anchors_" + d + "_urle_hash", refhash);
|
||||
prop.put("citations_documents_0_anchors_" + d + "_urle_date", GenericFormatter.SHORT_DAY_FORMATTER.format(new Date())); // superfluous?
|
||||
d++;
|
||||
} catch (IOException e) {
|
||||
} catch (final IOException e) {
|
||||
ConcurrentLog.logException(e);
|
||||
}
|
||||
}
|
||||
|
|
@ -290,11 +291,11 @@ public class webstructure {
|
|||
return prop;
|
||||
}
|
||||
|
||||
public static void reference(serverObjects prop, String prefix, int c, WebStructureGraph.StructureEntry sentry, WebStructureGraph ws) {
|
||||
public static void reference(final serverObjects prop, final String prefix, final int c, final WebStructureGraph.StructureEntry sentry, final WebStructureGraph ws) {
|
||||
prop.put(prefix + "_domains_" + c + "_hash", sentry.hosthash);
|
||||
prop.putXML(prefix + "_domains_" + c + "_domain", sentry.hostname);
|
||||
prop.put(prefix + "_domains_" + c + "_date", sentry.date);
|
||||
Iterator<Map.Entry<String, Integer>> k = sentry.references.entrySet().iterator();
|
||||
final Iterator<Map.Entry<String, Integer>> k = sentry.references.entrySet().iterator();
|
||||
Map.Entry<String, Integer> refentry;
|
||||
String refdom, refhash;
|
||||
Integer refcount;
|
||||
|
|
@ -25,6 +25,8 @@
|
|||
// along with this program; if not, write to the Free Software
|
||||
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
||||
|
||||
package net.yacy.htroot.api;
|
||||
|
||||
import java.net.MalformedURLException;
|
||||
import java.util.Arrays;
|
||||
|
||||
|
|
@ -1,4 +1,5 @@
|
|||
|
||||
package net.yacy.htroot.api;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
|
|
@ -15,19 +16,19 @@ import net.yacy.server.serverSwitch;
|
|||
|
||||
public class ynetSearch {
|
||||
|
||||
public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) {
|
||||
public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) {
|
||||
final Switchboard sb = (Switchboard) env;
|
||||
final boolean isAdmin=sb.verifyAuthentication(header);
|
||||
final serverObjects prop = new serverObjects();
|
||||
|
||||
if(post != null){
|
||||
if(!isAdmin){
|
||||
// force authentication if desired
|
||||
if(post.containsKey("login")){
|
||||
prop.authenticationRequired();
|
||||
}
|
||||
return prop;
|
||||
}
|
||||
if(post != null){
|
||||
if(!isAdmin){
|
||||
// force authentication if desired
|
||||
if(post.containsKey("login")){
|
||||
prop.authenticationRequired();
|
||||
}
|
||||
return prop;
|
||||
}
|
||||
InputStream is = null;
|
||||
try {
|
||||
String searchaddress = post.get("url");
|
||||
|
|
@ -41,21 +42,21 @@ public class ynetSearch {
|
|||
String s = searchaddress;
|
||||
Map.Entry<String, String> k;
|
||||
while(it.hasNext()) {
|
||||
k = it.next();
|
||||
s = s + "&" + k.getKey() + "=" + k.getValue();
|
||||
k = it.next();
|
||||
s = s + "&" + k.getKey() + "=" + k.getValue();
|
||||
}
|
||||
// final String s = searchaddress+"&query="+post.get("search")+"&maximumRecords="+post.get("maximumRecords")+"&startRecord="+post.get("startRecord");
|
||||
final URL url = new URL(s);
|
||||
is = url.openStream();
|
||||
final String httpout = new Scanner(is).useDelimiter( "\\Z" ).next();
|
||||
prop.put("http", httpout);
|
||||
// final String s = searchaddress+"&query="+post.get("search")+"&maximumRecords="+post.get("maximumRecords")+"&startRecord="+post.get("startRecord");
|
||||
final URL url = new URL(s);
|
||||
is = url.openStream();
|
||||
final String httpout = new Scanner(is).useDelimiter( "\\Z" ).next();
|
||||
prop.put("http", httpout);
|
||||
} catch (final Exception e ) {
|
||||
prop.put("url", "error!");
|
||||
prop.put("url", "error!");
|
||||
} finally {
|
||||
if ( is != null )
|
||||
try { is.close(); } catch (final IOException e ) { }
|
||||
if ( is != null )
|
||||
try { is.close(); } catch (final IOException e ) { }
|
||||
}
|
||||
}
|
||||
return prop;
|
||||
}
|
||||
}
|
||||
return prop;
|
||||
}
|
||||
}
|
||||
|
|
@ -17,6 +17,8 @@
|
|||
// along with this program; if not, write to the Free Software
|
||||
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
||||
|
||||
package net.yacy.htroot.env;
|
||||
|
||||
import java.util.Iterator;
|
||||
|
||||
import net.yacy.cora.protocol.RequestHeader;
|
||||
|
|
@ -27,7 +29,7 @@ public class style {
|
|||
|
||||
public static serverObjects respond(@SuppressWarnings("unused") final RequestHeader header, @SuppressWarnings("unused") final serverObjects post, final serverSwitch env) {
|
||||
final serverObjects prop = new serverObjects();
|
||||
Iterator<String> i = env.configKeys();
|
||||
final Iterator<String> i = env.configKeys();
|
||||
String key;
|
||||
while (i.hasNext()) {
|
||||
key = i.next();
|
||||
|
|
@ -1,4 +1,4 @@
|
|||
|
||||
package net.yacy.htroot.p2p;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Map;
|
||||
|
|
@ -24,38 +24,38 @@ import net.yacy.server.serverSwitch;
|
|||
public final class seeds {
|
||||
|
||||
private static final int LISTMAX = 1000;
|
||||
|
||||
|
||||
public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) {
|
||||
// return variable that accumulates replacements
|
||||
final Switchboard sb = (Switchboard) env;
|
||||
int maxcount = Math.min(LISTMAX, post == null ? Integer.MAX_VALUE : post.getInt("maxcount", Integer.MAX_VALUE));
|
||||
float minversion = Math.min(LISTMAX, post == null ? 0.0f : post.getFloat("minversion", 0.0f));
|
||||
boolean nodeonly = post == null || !post.containsKey("node") ? false : post.getBoolean("node");
|
||||
boolean includeme = post == null || !post.containsKey("me") ? true : post.getBoolean("me");
|
||||
boolean addressonly = post == null || !post.containsKey("address") ? false : post.getBoolean("address");
|
||||
String peername = post == null ? null : post.containsKey("my") ? sb.peers.myName() : post.get("peername");
|
||||
final int maxcount = Math.min(LISTMAX, post == null ? Integer.MAX_VALUE : post.getInt("maxcount", Integer.MAX_VALUE));
|
||||
final float minversion = Math.min(LISTMAX, post == null ? 0.0f : post.getFloat("minversion", 0.0f));
|
||||
final boolean nodeonly = post == null || !post.containsKey("node") ? false : post.getBoolean("node");
|
||||
final boolean includeme = post == null || !post.containsKey("me") ? true : post.getBoolean("me");
|
||||
final boolean addressonly = post == null || !post.containsKey("address") ? false : post.getBoolean("address");
|
||||
final String peername = post == null ? null : post.containsKey("my") ? sb.peers.myName() : post.get("peername");
|
||||
final ArrayList<Seed> v;
|
||||
if (post != null && post.containsKey("my")) {
|
||||
v = new ArrayList<Seed>(1);
|
||||
v.add(sb.peers.mySeed());
|
||||
} else if (post != null && post.containsKey("id")) {
|
||||
v = new ArrayList<Seed>(1);
|
||||
Seed s = sb.peers.get(post.get("id"));
|
||||
final Seed s = sb.peers.get(post.get("id"));
|
||||
if (s != null) v.add(s);
|
||||
} else if (post != null && post.containsKey("name")) {
|
||||
v = new ArrayList<Seed>(1);
|
||||
Seed s = sb.peers.lookupByName(post.get("name"));
|
||||
final Seed s = sb.peers.lookupByName(post.get("name"));
|
||||
if (s != null) v.add(s);
|
||||
} else {
|
||||
v= sb.peers.getSeedlist(maxcount, includeme, nodeonly, minversion);
|
||||
}
|
||||
final serverObjects prop = new serverObjects();
|
||||
|
||||
|
||||
// write simple-encoded seed lines or json
|
||||
String EXT = header.get(HeaderFramework.CONNECTION_PROP_EXT);
|
||||
boolean json = EXT != null && EXT.equals("json");
|
||||
boolean xml = EXT != null && EXT.equals("xml");
|
||||
|
||||
final String EXT = header.get(HeaderFramework.CONNECTION_PROP_EXT);
|
||||
final boolean json = EXT != null && EXT.equals("json");
|
||||
final boolean xml = EXT != null && EXT.equals("xml");
|
||||
|
||||
if (json) {
|
||||
// check for JSONP
|
||||
if ( post != null && post.containsKey("callback") ) {
|
||||
|
|
@ -68,17 +68,17 @@ public final class seeds {
|
|||
// construct json property lists
|
||||
int count = 0;
|
||||
for (int i = 0; i < v.size(); i++) {
|
||||
Seed seed = v.get(i);
|
||||
final Seed seed = v.get(i);
|
||||
if (peername != null && !peername.equals(seed.getName())) continue;
|
||||
Set<String> ips = seed.getIPs();
|
||||
final Set<String> ips = seed.getIPs();
|
||||
if (ips == null || ips.size() == 0) continue;
|
||||
prop.putJSON("peers_" + count + "_map_0_k", Seed.HASH);
|
||||
prop.put("peers_" + count + "_map_0_v", JSONObject.quote(seed.hash));
|
||||
prop.put("peers_" + count + "_map_0_c", 1);
|
||||
Map<String, String> map = seed.getMap();
|
||||
final Map<String, String> map = seed.getMap();
|
||||
int c = 1;
|
||||
if (!addressonly) {
|
||||
for (Map.Entry<String, String> m: map.entrySet()) {
|
||||
for (final Map.Entry<String, String> m: map.entrySet()) {
|
||||
prop.putJSON("peers_" + count + "_map_" + c + "_k", m.getKey());
|
||||
prop.put("peers_" + count + "_map_" + c + "_v", JSONObject.quote(m.getValue()));
|
||||
prop.put("peers_" + count + "_map_" + c + "_c", 1);
|
||||
|
|
@ -86,9 +86,9 @@ public final class seeds {
|
|||
}
|
||||
}
|
||||
// construct a list of ips
|
||||
StringBuilder a = new StringBuilder();
|
||||
final StringBuilder a = new StringBuilder();
|
||||
a.append('[');
|
||||
for (String ip: ips) a.append(JSONObject.quote(seed.getPublicAddress(ip))).append(',');
|
||||
for (final String ip: ips) a.append(JSONObject.quote(seed.getPublicAddress(ip))).append(',');
|
||||
a.setCharAt(a.length()-1, ']');
|
||||
prop.putJSON("peers_" + count + "_map_" + c + "_k", "Address");
|
||||
prop.put("peers_" + count + "_map_" + c + "_v", a.toString());
|
||||
|
|
@ -103,22 +103,22 @@ public final class seeds {
|
|||
} else if (xml) {
|
||||
int count = 0;
|
||||
for (int i = 0; i < v.size(); i++) {
|
||||
Seed seed = v.get(i);
|
||||
final Seed seed = v.get(i);
|
||||
if (peername != null && !peername.equals(seed.getName())) continue;
|
||||
Set<String> ips = seed.getIPs();
|
||||
final Set<String> ips = seed.getIPs();
|
||||
if (ips == null || ips.size() == 0) continue;
|
||||
prop.putXML("peers_" + count + "_map_0_k", Seed.HASH);
|
||||
prop.putXML("peers_" + count + "_map_0_v", seed.hash);
|
||||
Map<String, String> map = seed.getMap();
|
||||
final Map<String, String> map = seed.getMap();
|
||||
int c = 1;
|
||||
if (!addressonly) {
|
||||
for (Map.Entry<String, String> m: map.entrySet()) {
|
||||
for (final Map.Entry<String, String> m: map.entrySet()) {
|
||||
prop.putXML("peers_" + count + "_map_" + c + "_k", m.getKey());
|
||||
prop.putXML("peers_" + count + "_map_" + c + "_v", m.getValue());
|
||||
c++;
|
||||
}
|
||||
}
|
||||
for (String ip: ips) {
|
||||
for (final String ip: ips) {
|
||||
prop.putXML("peers_" + count + "_map_" + c + "_k", "Address");
|
||||
prop.putXML("peers_" + count + "_map_" + c + "_v", seed.getPublicAddress(ip));
|
||||
c++;
|
||||
|
|
@ -126,17 +126,17 @@ public final class seeds {
|
|||
prop.put("peers_" + count + "_map", c);
|
||||
count++;
|
||||
}
|
||||
|
||||
|
||||
prop.put("peers_" + (count - 1) + "_c", 0);
|
||||
prop.put("peers", count);
|
||||
} else {
|
||||
final StringBuilder encoded = new StringBuilder(1024);
|
||||
for (Seed seed: v) {
|
||||
for (final Seed seed: v) {
|
||||
encoded.append(seed.genSeedStr(null)).append(serverCore.CRLF_STRING);
|
||||
}
|
||||
}
|
||||
prop.put("encoded", encoded.toString());
|
||||
}
|
||||
|
||||
|
||||
// return rewrite properties
|
||||
return prop;
|
||||
}
|
||||
|
|
@ -1,3 +1,5 @@
|
|||
package net.yacy.htroot.proxymsg;
|
||||
|
||||
import net.yacy.cora.protocol.RequestHeader;
|
||||
import net.yacy.data.BookmarksDB.Bookmark;
|
||||
import net.yacy.search.Switchboard;
|
||||
|
|
@ -15,19 +17,19 @@ public class urlproxyheader {
|
|||
final Switchboard sb = (Switchboard) env;
|
||||
|
||||
String proxyurlstr = post.get("url",""); // the url of remote page currently viewed
|
||||
boolean hasRights = sb.verifyAuthentication(requestHeader);
|
||||
final boolean hasRights = sb.verifyAuthentication(requestHeader);
|
||||
prop.put("allowbookmark", hasRights);
|
||||
|
||||
if (post.containsKey("addbookmark")) {
|
||||
proxyurlstr = post.get("bookmark");
|
||||
Bookmark bmk = sb.bookmarksDB.createorgetBookmark(proxyurlstr, null);
|
||||
final Bookmark bmk = sb.bookmarksDB.createorgetBookmark(proxyurlstr, null);
|
||||
if (bmk != null) {
|
||||
bmk.setPublic(false);
|
||||
bmk.addTag("/proxy"); // add to bookmark folder
|
||||
sb.bookmarksDB.saveBookmark(bmk);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
prop.put("proxyurl", proxyurlstr);
|
||||
prop.put("allowbookmark_proxyurl", proxyurlstr);
|
||||
|
||||
|
|
@ -36,7 +38,7 @@ public class urlproxyheader {
|
|||
} else {
|
||||
prop.put("httpsAlertMsg", "0");
|
||||
}
|
||||
|
||||
|
||||
// TODO: get some index data to display
|
||||
/*
|
||||
if (post.containsKey("hash")) {
|
||||
|
|
@ -773,18 +773,21 @@ public class YaCyDefaultServlet extends HttpServlet {
|
|||
|
||||
protected Class<?> rewriteClass(String target) {
|
||||
assert target.charAt(0) == '/';
|
||||
final int p = target.lastIndexOf('.');
|
||||
if (p < 0) {
|
||||
return null;
|
||||
}
|
||||
target = "net.yacy.htroot" + target.substring(0, p).replace('/', '.');
|
||||
try {
|
||||
final int p = target.lastIndexOf('.');
|
||||
if (p < 0) {
|
||||
return null;
|
||||
}
|
||||
target = "net.yacy.htroot" + target.substring(0, p).replace('/', '.');
|
||||
|
||||
final Class<?> servletClass = Class.forName(target);
|
||||
|
||||
return servletClass;
|
||||
} catch (final ClassNotFoundException e) {
|
||||
return null;
|
||||
try {
|
||||
final Class<?> servletClass = Class.forName(target + "_"); // for some targets we need alternative names
|
||||
return servletClass;
|
||||
} catch (final ClassNotFoundException ee) {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue
Block a user