mirror of
https://github.com/yacy/yacy_search_server.git
synced 2024-09-19 00:01:41 +02:00
added a new 'Citations' function: each search result item can now be
explored for citations within other documents. A click on the 'Citations' link shows an analysis with all text lines in the document each with a complete list of documents which contain the same line. A second section shows the linking documents in ascending order of number of citations from the original document. Because documents from different hosts are most interesting here, they are listed at the top of the page as possible 'copypasta' source.
This commit is contained in:
Michael Peter Christen
parent
fc3ff92c69
commit
fd1776a3b0
|
|
@ -769,6 +769,7 @@ search.result.show.date = true
|
|||
search.result.show.size = false
|
||||
search.result.show.metadata = false
|
||||
search.result.show.parser = false
|
||||
search.result.show.citation = true
|
||||
search.result.show.pictures = false
|
||||
search.result.show.cache = true
|
||||
search.result.show.proxy = false
|
||||
|
|
|
|||
|
|
@ -79,6 +79,7 @@
|
|||
<input type="checkbox" name="search.result.show.size" value="true" #(search.result.show.size)#::checked="checked"#(/search.result.show.size)# />Size
|
||||
<input type="checkbox" name="search.result.show.metadata" value="true" #(search.result.show.metadata)#::checked="checked"#(/search.result.show.metadata)# />Metadata
|
||||
<input type="checkbox" name="search.result.show.parser" value="true" #(search.result.show.parser)#::checked="checked"#(/search.result.show.parser)# />Parser
|
||||
<input type="checkbox" name="search.result.show.citation" value="true" #(search.result.show.citation)#::checked="checked"#(/search.result.show.citation)# />Citations
|
||||
<input type="checkbox" name="search.result.show.pictures" value="true" #(search.result.show.pictures)#::checked="checked"#(/search.result.show.pictures)# />Pictures
|
||||
<input type="checkbox" name="search.result.show.cache" value="true" #(search.result.show.cache)#::checked="checked"#(/search.result.show.cache)# />Cache
|
||||
<input type="checkbox" name="search.result.show.proxy" value="true" #(search.result.show.proxy)#::checked="checked"#(/search.result.show.proxy)# />Augmented Browsing
|
||||
|
|
|
|||
|
|
@ -99,6 +99,7 @@ public class ConfigPortal {
|
|||
sb.setConfig("search.result.show.size", post.getBoolean("search.result.show.size"));
|
||||
sb.setConfig("search.result.show.metadata", post.getBoolean("search.result.show.metadata"));
|
||||
sb.setConfig("search.result.show.parser", post.getBoolean("search.result.show.parser"));
|
||||
sb.setConfig("search.result.show.citation", post.getBoolean("search.result.show.citation"));
|
||||
sb.setConfig("search.result.show.pictures", post.getBoolean("search.result.show.pictures"));
|
||||
sb.setConfig("search.result.show.cache", post.getBoolean("search.result.show.cache"));
|
||||
sb.setConfig("search.result.show.proxy", post.getBoolean("search.result.show.proxy"));
|
||||
|
|
@ -170,6 +171,7 @@ public class ConfigPortal {
|
|||
sb.setConfig("search.result.show.size", config.getProperty("search.result.show.size","false"));
|
||||
sb.setConfig("search.result.show.metadata", config.getProperty("search.result.show.metadata","false"));
|
||||
sb.setConfig("search.result.show.parser", config.getProperty("search.result.show.parser","false"));
|
||||
sb.setConfig("search.result.show.citation", config.getProperty("search.result.show.citation","false"));
|
||||
sb.setConfig("search.result.show.pictures", config.getProperty("search.result.show.pictures","false"));
|
||||
sb.setConfig("search.result.show.cache", config.getProperty("search.result.show.cache","true"));
|
||||
sb.setConfig("search.result.show.proxy", config.getProperty("search.result.show.proxy","false"));
|
||||
|
|
@ -205,6 +207,7 @@ public class ConfigPortal {
|
|||
prop.put("search.result.show.size", sb.getConfigBool("search.result.show.size", false) ? 1 : 0);
|
||||
prop.put("search.result.show.metadata", sb.getConfigBool("search.result.show.metadata", false) ? 1 : 0);
|
||||
prop.put("search.result.show.parser", sb.getConfigBool("search.result.show.parser", false) ? 1 : 0);
|
||||
prop.put("search.result.show.citation", sb.getConfigBool("search.result.show.citation", false) ? 1 : 0);
|
||||
prop.put("search.result.show.pictures", sb.getConfigBool("search.result.show.pictures", false) ? 1 : 0);
|
||||
prop.put("search.result.show.cache", sb.getConfigBool("search.result.show.cache", false) ? 1 : 0);
|
||||
prop.put("search.result.show.proxy", sb.getConfigBool("search.result.show.proxy", false) ? 1 : 0);
|
||||
|
|
|
|||
|
|
@ -179,6 +179,7 @@ $(function() {
|
|||
<td> | 42 kbyte</td>
|
||||
<td> | <a href="/solr/select?q=*:*&defType=edismax&start=0&rows=3&core=collection1&wt=html" target="LayouTest" onclick="return hs.htmlExpand(this, { objectType: 'ajax'} )">Metadata</a></td>
|
||||
<td> | <a href="ViewFile.html" target="LayouTest">Parser</a></td>
|
||||
<td> | <a href="/api/citation.html?url=yacy.net" target="LayouTest">Citation</a></td>
|
||||
<td> | <a href="yacysearch.html" target="LayouTest">Pictures</a></td>
|
||||
<td> | <a href="CacheResource_p.html" target="LayouTest">Cache</a></td>
|
||||
<td> | <a href="proxy.html" target="LayouTest">Augmented Browsing</a></td>
|
||||
|
|
@ -190,6 +191,7 @@ $(function() {
|
|||
<td align="center"><input type="checkbox" name="search.result.show.size" value="true" #(search.result.show.size)#::checked="checked" #(/search.result.show.size)# /></td>
|
||||
<td align="center"><input type="checkbox" name="search.result.show.metadata" value="true" #(search.result.show.metadata)#::checked="checked" #(/search.result.show.metadata)# /></td>
|
||||
<td align="center"><input type="checkbox" name="search.result.show.parser" value="true" #(search.result.show.parser)#::checked="checked" #(/search.result.show.parser)# /></td>
|
||||
<td align="center"><input type="checkbox" name="search.result.show.citation" value="true" #(search.result.show.citation)#::checked="checked" #(/search.result.show.citation)# /></td>
|
||||
<td align="center"><input type="checkbox" name="search.result.show.pictures" value="true" #(search.result.show.pictures)#::checked="checked" #(/search.result.show.pictures)# /></td>
|
||||
<td align="center"><input type="checkbox" name="search.result.show.cache" value="true" #(search.result.show.cache)#::checked="checked" #(/search.result.show.cache)# /></td>
|
||||
<td align="center"><input type="checkbox" name="search.result.show.proxy" value="true" #(search.result.show.proxy)#::checked="checked" #(/search.result.show.proxy)# /></td>
|
||||
|
|
|
|||
|
|
@ -72,6 +72,7 @@ public class ConfigSearchPage_p {
|
|||
sb.setConfig("search.result.show.size", post.getBoolean("search.result.show.size"));
|
||||
sb.setConfig("search.result.show.metadata", post.getBoolean("search.result.show.metadata"));
|
||||
sb.setConfig("search.result.show.parser", post.getBoolean("search.result.show.parser"));
|
||||
sb.setConfig("search.result.show.citation", post.getBoolean("search.result.show.citation"));
|
||||
sb.setConfig("search.result.show.pictures", post.getBoolean("search.result.show.pictures"));
|
||||
sb.setConfig("search.result.show.cache", post.getBoolean("search.result.show.cache"));
|
||||
sb.setConfig("search.result.show.proxy", post.getBoolean("search.result.show.proxy"));
|
||||
|
|
@ -124,6 +125,7 @@ public class ConfigSearchPage_p {
|
|||
sb.setConfig("search.result.show.size", config.getProperty("search.result.show.size","false"));
|
||||
sb.setConfig("search.result.show.metadata", config.getProperty("search.result.show.metadata","false"));
|
||||
sb.setConfig("search.result.show.parser", config.getProperty("search.result.show.parser","false"));
|
||||
sb.setConfig("search.result.show.citation", config.getProperty("search.result.show.citation","false"));
|
||||
sb.setConfig("search.result.show.pictures", config.getProperty("search.result.show.pictures","false"));
|
||||
sb.setConfig("search.result.show.cache", config.getProperty("search.result.show.cache","true"));
|
||||
sb.setConfig("search.result.show.proxy", config.getProperty("search.result.show.proxy","false"));
|
||||
|
|
@ -150,6 +152,7 @@ public class ConfigSearchPage_p {
|
|||
prop.put("search.result.show.size", sb.getConfigBool("search.result.show.size", false) ? 1 : 0);
|
||||
prop.put("search.result.show.metadata", sb.getConfigBool("search.result.show.metadata", false) ? 1 : 0);
|
||||
prop.put("search.result.show.parser", sb.getConfigBool("search.result.show.parser", false) ? 1 : 0);
|
||||
prop.put("search.result.show.citation", sb.getConfigBool("search.result.show.citation", false) ? 1 : 0);
|
||||
prop.put("search.result.show.pictures", sb.getConfigBool("search.result.show.pictures", false) ? 1 : 0);
|
||||
prop.put("search.result.show.cache", sb.getConfigBool("search.result.show.cache", false) ? 1 : 0);
|
||||
prop.put("search.result.show.proxy", sb.getConfigBool("search.result.show.proxy", false) ? 1 : 0);
|
||||
|
|
|
|||
34
htroot/api/citation.html
Normal file
34
htroot/api/citation.html
Normal file
|
|
@ -0,0 +1,34 @@
|
|||
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
|
||||
<html xmlns="http://www.w3.org/1999/xhtml">
|
||||
<head>
|
||||
<title>YaCy '#[clientname]#': Document Citations for url #[url]#</title>
|
||||
#%env/templates/metas.template%#
|
||||
</head>
|
||||
<body>
|
||||
#%env/templates/embeddedheader.template%#
|
||||
<h1>Document Citations for<br><a href="#[url]#">#[url]#</a></h1>
|
||||
#(similar)#::
|
||||
<h3>Similar documents from different hosts:</h3>
|
||||
<ul>#{links}#
|
||||
<li><a href="#[url]#">#[url]#</a></li>
|
||||
#{/links}#</ul>
|
||||
#(/similar)#
|
||||
<form action="#" name="" method="post" enctype="multipart/form-data" accept-charset="UTF-8">
|
||||
<fieldset>
|
||||
<legend><label>List of Sentences in #[url]#</label></legend>
|
||||
<dl>#{sentences}#
|
||||
<dt>#[dt]#</dt><dd>#[dd]#</dd>
|
||||
#{/sentences}#</dl>
|
||||
</fieldset>
|
||||
</form>
|
||||
<form action="#" name="" method="post" enctype="multipart/form-data" accept-charset="UTF-8">
|
||||
<fieldset>
|
||||
<legend><label>List of other web pages with citations</label></legend>
|
||||
<dl>#{citations}#
|
||||
<dt>#[dt]#</dt><dd>#[dd]#</dd>
|
||||
#{/citations}#</dl>
|
||||
</fieldset>
|
||||
</form>
|
||||
#%env/templates/embeddedfooter.template%#
|
||||
</body>
|
||||
</html>
|
||||
199
htroot/api/citation.java
Normal file
199
htroot/api/citation.java
Normal file
|
|
@ -0,0 +1,199 @@
|
|||
/**
|
||||
* citation
|
||||
* Copyright 2013 by Michael Peter Christen
|
||||
* First released 12.6.2013 at http://yacy.net
|
||||
*
|
||||
* This library is free software; you can redistribute it and/or
|
||||
* modify it under the terms of the GNU Lesser General Public
|
||||
* License as published by the Free Software Foundation; either
|
||||
* version 2.1 of the License, or (at your option) any later version.
|
||||
*
|
||||
* This library is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
* Lesser General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Lesser General Public License
|
||||
* along with this program in the file lgpl21.txt
|
||||
* If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
|
||||
import java.io.IOException;
|
||||
import java.net.MalformedURLException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.TreeSet;
|
||||
|
||||
import org.apache.solr.common.SolrDocument;
|
||||
import org.apache.solr.common.SolrDocumentList;
|
||||
|
||||
import net.yacy.cora.document.ASCII;
|
||||
import net.yacy.cora.federate.solr.connector.SolrConnector;
|
||||
import net.yacy.cora.protocol.RequestHeader;
|
||||
import net.yacy.cora.sorting.OrderedScoreMap;
|
||||
import net.yacy.document.SentenceReader;
|
||||
import net.yacy.kelondro.data.meta.DigestURI;
|
||||
import net.yacy.search.Switchboard;
|
||||
import net.yacy.search.index.Segment;
|
||||
import net.yacy.search.schema.CollectionSchema;
|
||||
import net.yacy.server.serverObjects;
|
||||
import net.yacy.server.serverSwitch;
|
||||
|
||||
public class citation {
|
||||
|
||||
public static serverObjects respond(@SuppressWarnings("unused") final RequestHeader header, final serverObjects post, final serverSwitch env) {
|
||||
// return variable that accumulates replacements
|
||||
final Switchboard sb = (Switchboard) env;
|
||||
final serverObjects prop = new serverObjects();
|
||||
final Segment segment = sb.index;
|
||||
final SolrConnector connector = segment.fulltext().getDefaultConnector();
|
||||
|
||||
// avoid UNRESOLVED PATTERN
|
||||
prop.put("url", "");
|
||||
prop.put("citations", 0);
|
||||
prop.put("sentences", 0);
|
||||
|
||||
DigestURI uri = null;
|
||||
String url = "";
|
||||
String hash = "";
|
||||
int ch = 10;
|
||||
if (post != null) {
|
||||
if (post.containsKey("url")) {
|
||||
url = post.get("url");
|
||||
if (!url.startsWith("http://") &&
|
||||
!url.startsWith("https://") &&
|
||||
!url.startsWith("ftp://") &&
|
||||
!url.startsWith("smb://") &&
|
||||
!url.startsWith("file://")) {
|
||||
url = "http://" + url;
|
||||
}
|
||||
}
|
||||
if (post.containsKey("hash")) {
|
||||
hash = post.get("hash");
|
||||
}
|
||||
if (post.containsKey("ch")) {
|
||||
ch = post.getInt("ch", ch);
|
||||
}
|
||||
}
|
||||
|
||||
if (url.length() > 0) {
|
||||
try {
|
||||
uri = new DigestURI(url, null);
|
||||
hash = ASCII.String(uri.hash());
|
||||
} catch (MalformedURLException e) {}
|
||||
}
|
||||
if (uri == null && hash.length() > 0) {
|
||||
uri = sb.getURL(ASCII.getBytes(hash));
|
||||
}
|
||||
if (uri == null) return prop; // no proper url addressed
|
||||
url = uri.toNormalform(true);
|
||||
prop.put("url", url);
|
||||
|
||||
// get the document from the index
|
||||
SolrDocument doc;
|
||||
try {
|
||||
doc = segment.fulltext().getDefaultConnector().getDocumentById(hash, CollectionSchema.title.getSolrFieldName(), CollectionSchema.text_t.getSolrFieldName());
|
||||
} catch (IOException e1) {
|
||||
return prop;
|
||||
}
|
||||
@SuppressWarnings("unchecked")
|
||||
ArrayList<String> title = (ArrayList<String>) doc.getFieldValue(CollectionSchema.title.getSolrFieldName());
|
||||
String text = (String) doc.getFieldValue(CollectionSchema.text_t.getSolrFieldName());
|
||||
|
||||
ArrayList<String> sentences = new ArrayList<String>();
|
||||
if (title != null) for (String s: title) if (s.length() > 0) sentences.add(s);
|
||||
SentenceReader sr = new SentenceReader(text);
|
||||
StringBuilder line;
|
||||
while (sr.hasNext()) {
|
||||
line = sr.next();
|
||||
if (line.length() > 0) sentences.add(line.toString());
|
||||
}
|
||||
|
||||
// for each line make a statistic about the number of occurrences somewhere else
|
||||
OrderedScoreMap<String> scores = new OrderedScoreMap<String>(null); // accumulates scores for citating urls
|
||||
LinkedHashMap<String, Set<DigestURI>> sentenceOcc = new LinkedHashMap<String, Set<DigestURI>>();
|
||||
for (String sentence: sentences) {
|
||||
if (sentence == null || sentence.length() < 40) {
|
||||
// do not count the very short sentences
|
||||
sentenceOcc.put(sentence, null);
|
||||
continue;
|
||||
}
|
||||
try {
|
||||
sentence = sentence.replace('"', '\'');
|
||||
SolrDocumentList doclist = connector.getDocumentListByQuery("text_t:\"" + sentence + "\"", 0, 100, CollectionSchema.sku.getSolrFieldName());
|
||||
int count = (int) doclist.getNumFound();
|
||||
if (count > 0) {
|
||||
Set<DigestURI> list = new TreeSet<DigestURI>();
|
||||
for (SolrDocument d: doclist) {
|
||||
String u = (String) d.getFieldValue(CollectionSchema.sku.getSolrFieldName());
|
||||
if (u == null || u.equals(url)) continue;
|
||||
scores.inc(u);
|
||||
try {list.add(new DigestURI(u, null));} catch (MalformedURLException e) {}
|
||||
}
|
||||
sentenceOcc.put(sentence, list);
|
||||
}
|
||||
} catch (Throwable ee) {
|
||||
|
||||
}
|
||||
}
|
||||
sentences.clear(); // we do not need this again
|
||||
|
||||
// iterate the sentences
|
||||
int i = 0;
|
||||
for (Map.Entry<String, Set<DigestURI>> se: sentenceOcc.entrySet()) {
|
||||
prop.put("sentences_" + i + "_dt", i);
|
||||
StringBuilder dd = new StringBuilder(se.getKey());
|
||||
Set<DigestURI> app = se.getValue();
|
||||
if (app != null && app.size() > 0) {
|
||||
dd.append("<br/>appears in:");
|
||||
for (DigestURI u: app) {
|
||||
if (u != null) {
|
||||
dd.append(" <a href=\"").append(u.toNormalform(false)).append("\">").append(u.getHost()).append("</a>");
|
||||
}
|
||||
}
|
||||
}
|
||||
prop.put("sentences_" + i + "_dd", dd.toString());
|
||||
i++;
|
||||
}
|
||||
prop.put("sentences", i);
|
||||
|
||||
// iterate the citations in order of number of citations
|
||||
i = 0;
|
||||
for (String u: scores.keyList(false)) {
|
||||
try {
|
||||
DigestURI uu = new DigestURI(u, null);
|
||||
prop.put("citations_" + i + "_dt", "<a href=\"" + u + "\">" + u + "</a>");
|
||||
StringBuilder dd = new StringBuilder();
|
||||
dd.append("makes ").append(Integer.toString(scores.get(u))).append(" citations: of ").append(url);
|
||||
for (Map.Entry<String, Set<DigestURI>> se: sentenceOcc.entrySet()) {
|
||||
Set<DigestURI> occurls = se.getValue();
|
||||
if (occurls != null && occurls.contains(uu)) dd.append("<br/><a href=\"/solr/select?q=text_t:%22").append(se.getKey().replace('"', '\'')).append("%22&rows=100&grep=&wt=grephtml\">").append(se.getKey()).append("</a>");
|
||||
}
|
||||
prop.put("citations_" + i + "_dd", dd.toString());
|
||||
i++;
|
||||
} catch (MalformedURLException e) {}
|
||||
}
|
||||
prop.put("citations", i);
|
||||
|
||||
// find similar documents from different hosts
|
||||
i = 0;
|
||||
for (String u: scores.keyList(false)) {
|
||||
if (scores.get(u) < ch) continue;
|
||||
try {
|
||||
DigestURI uu = new DigestURI(u, null);
|
||||
if (uu.getOrganization().equals(uri.getOrganization())) continue;
|
||||
prop.put("similar_links_" + i + "_url", u);
|
||||
i++;
|
||||
} catch (MalformedURLException e) {}
|
||||
}
|
||||
prop.put("similar_links", i);
|
||||
prop.put("similar", i > 0 ? 1 : 0);
|
||||
|
||||
// return rewrite properties
|
||||
return prop;
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -28,6 +28,7 @@
|
|||
#(showSize)#:: | #[sizename]##(/showSize)#
|
||||
#(showMetadata)#:: | <a href="/solr/select?q=id:%22#[urlhash]#%22&defType=edismax&start=0&rows=1&core=collection1&wt=html" target="_blank" onclick="return hs.htmlExpand(this, { objectType: 'ajax'} )">Metadata</a>#(/showMetadata)#
|
||||
#(showParser)#:: | <a href="ViewFile.html?urlHash=#[urlhash]#&words=#[words]#" target="_blank">Parser</a>#(/showParser)#
|
||||
#(showCitation)#:: | <a href="/api/citation.html?hash=#[urlhash]#" target="_blank">Citations</a>#(/showCitation)#
|
||||
#(showPictures)#:: | <a href="yacysearch.html?cat=image&url=#[link]#&query=#[former]#">Pictures</a>#(/showPictures)#
|
||||
#(showCache)#:: | <a href="CacheResource_p.html?url=#[link]#" target="_blank">Cache</a>#(/showCache)#
|
||||
#(showProxy)#:: | <a href="proxy.html?url=#[link]#" target="_blank">Augmented Browsing</a>#(/showProxy)#
|
||||
|
|
|
|||
|
|
@ -133,6 +133,7 @@ public class yacysearchitem {
|
|||
prop.put("content_showSize", sb.getConfigBool("search.result.show.size", true) ? 1 : 0);
|
||||
prop.put("content_showMetadata", sb.getConfigBool("search.result.show.metadata", true) ? 1 : 0);
|
||||
prop.put("content_showParser", sb.getConfigBool("search.result.show.parser", true) ? 1 : 0);
|
||||
prop.put("content_showCitation", sb.getConfigBool("search.result.show.citation", true) ? 1 : 0);
|
||||
prop.put("content_showPictures", sb.getConfigBool("search.result.show.pictures", true) ? 1 : 0);
|
||||
prop.put("content_showCache", sb.getConfigBool("search.result.show.cache", true) && Cache.has(resultURL.hash()) ? 1 : 0);
|
||||
prop.put("content_showProxy", sb.getConfigBool("search.result.show.proxy", true) ? 1 : 0);
|
||||
|
|
@ -198,6 +199,7 @@ public class yacysearchitem {
|
|||
prop.put("content_showProxy_link", resultUrlstring);
|
||||
prop.put("content_showHostBrowser_link", resultUrlstring);
|
||||
prop.put("content_showParser_urlhash", resulthashString);
|
||||
prop.put("content_showCitation_urlhash", resulthashString);
|
||||
prop.put("content_showTags_urlhash", resulthashString);
|
||||
prop.put("content_urlhexhash", Seed.b64Hash2hexHash(resulthashString));
|
||||
prop.putHTML("content_urlname", nxTools.shortenURLString(result.urlname(), MAX_URL_LENGTH));
|
||||
|
|
|
|||
|
|
@ -755,6 +755,14 @@ public class MultiProtocolURI implements Serializable, Comparable<MultiProtocolU
|
|||
}
|
||||
return this.host;
|
||||
}
|
||||
|
||||
public String getOrganization() {
|
||||
String dnc = Domains.getDNC(host);
|
||||
String subdomOrga = host.length() - dnc.length() <= 0 ? "" : host.substring(0, host.length() - dnc.length() - 1);
|
||||
int p = subdomOrga.lastIndexOf('.');
|
||||
String orga = (p < 0) ? subdomOrga : subdomOrga.substring(p + 1);
|
||||
return orga;
|
||||
}
|
||||
|
||||
public String getTLD() {
|
||||
if (this.host == null) return "";
|
||||
|
|
|
|||
|
|
@ -23,17 +23,12 @@ package net.yacy.cora.federate.solr.responsewriter;
|
|||
import java.io.IOException;
|
||||
import java.io.Writer;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collection;
|
||||
import java.util.HashSet;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
import net.yacy.cora.federate.solr.connector.SolrConnector;
|
||||
import net.yacy.cora.sorting.OrderedScoreMap;
|
||||
import net.yacy.document.SentenceReader;
|
||||
import net.yacy.search.Switchboard;
|
||||
import net.yacy.search.schema.CollectionSchema;
|
||||
|
||||
import org.apache.lucene.document.Document;
|
||||
|
|
@ -86,11 +81,10 @@ public class GrepHTMLResponseWriter implements QueryResponseWriter {
|
|||
assert values.get("responseHeader") != null;
|
||||
assert values.get("response") != null;
|
||||
|
||||
writer.write("<!DOCTYPE html PUBLIC \"-//W3C//DTD XHTML 1.0 Transitional//EN\" \"http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd\">\n");
|
||||
writer.write("<!DOCTYPE html PUBLIC \"-//W3C//DTD XHTML 1.0 Transitional//EN\" \"http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd\">\n<html xmlns=\"http://www.w3.org/1999/xhtml\">\n<head>\n");
|
||||
writer.write("<link rel=\"stylesheet\" type=\"text/css\" media=\"all\" href=\"/env/base.css\" />\n");
|
||||
writer.write("<link rel=\"stylesheet\" type=\"text/css\" media=\"screen\" href=\"/env/style.css\" />\n");
|
||||
SolrParams params = request.getOriginalParams();
|
||||
boolean discover = params.getBool("discover", false);
|
||||
String grep = params.get("grep");
|
||||
String query = "";
|
||||
String q = params.get("q"); if (q == null) q = "";
|
||||
|
|
@ -112,8 +106,6 @@ public class GrepHTMLResponseWriter implements QueryResponseWriter {
|
|||
NamedList<Object> paramsList = params.toNamedList();
|
||||
paramsList.remove("wt");
|
||||
String xmlquery = dqp.matcher("/solr/select?" + SolrParams.toSolrParams(paramsList).toString()).replaceAll("%22");
|
||||
writer.write("<div id=\"api\"><a href=\"" + xmlquery + "\"><img src=\"../env/grafics/api.png\" width=\"60\" height=\"40\" alt=\"API\" /></a>\n");
|
||||
writer.write("<span>This search result can also be retrieved as XML. Click the API icon to see an example call to the search rss API.</div>\n");
|
||||
|
||||
DocList response = ((ResultContext) values.get("response")).docs;
|
||||
final int sz = response.size();
|
||||
|
|
@ -121,10 +113,10 @@ public class GrepHTMLResponseWriter implements QueryResponseWriter {
|
|||
SolrIndexSearcher searcher = request.getSearcher();
|
||||
DocIterator iterator = response.iterator();
|
||||
IndexSchema schema = request.getSchema();
|
||||
writer.write("<title>Document Grep for query \"" + query + "\" and grep phrase \"" + grep + "\"</title>\n</head><body>\n");
|
||||
|
||||
LinkedHashMap<String, ArrayList<String>> sentenceCache = new LinkedHashMap<String, ArrayList<String>>();
|
||||
|
||||
String h1 = "Document Grep for query \"" + query + "\" and grep phrase \"" + grep + "\"";
|
||||
writer.write("<title>" + h1 + "</title>\n</head><body>\n<h1>" + h1 + "</h1>\n");
|
||||
writer.write("<div id=\"api\"><a href=\"" + xmlquery + "\"><img src=\"../env/grafics/api.png\" width=\"60\" height=\"40\" alt=\"API\" /></a>\n");
|
||||
writer.write("<span>This search result can also be retrieved as XML. Click the API icon to see an example call to the search rss API.</span>\n");
|
||||
for (int i = 0; i < sz; i++) {
|
||||
int id = iterator.nextDoc();
|
||||
Document doc = searcher.doc(id, DEFAULT_FIELD_LIST);
|
||||
|
|
@ -141,24 +133,7 @@ public class GrepHTMLResponseWriter implements QueryResponseWriter {
|
|||
line = sr.next();
|
||||
if (line.length() > 0) sentences.add(line.toString());
|
||||
}
|
||||
sentenceCache.put(sku, sentences);
|
||||
}
|
||||
|
||||
OrderedScoreMap<String> scores = null;
|
||||
if (discover) {
|
||||
// for each line make a statistic about the number of occurrences somewhere else
|
||||
SolrConnector connector = Switchboard.getSwitchboard().index.fulltext().getDefaultConnector();
|
||||
scores = new OrderedScoreMap<String>(null);
|
||||
for (Map.Entry<String, ArrayList<String>> entry: sentenceCache.entrySet()) {
|
||||
for (String line: entry.getValue()) {
|
||||
long count = connector.getCountByQuery("text_t:\"" + line + "\"");
|
||||
if (count > 0) scores.inc(entry.getKey());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for (Map.Entry<String, ArrayList<String>> entry: sentenceCache.entrySet()) {
|
||||
writeDoc(writer, entry.getKey(), entry.getValue(), grep, scores);
|
||||
writeDoc(writer, sku, sentences, grep);
|
||||
}
|
||||
} else {
|
||||
writer.write("<title>No Document Found</title>\n</head><body>\n");
|
||||
|
|
@ -167,7 +142,7 @@ public class GrepHTMLResponseWriter implements QueryResponseWriter {
|
|||
writer.write("</body></html>\n");
|
||||
}
|
||||
|
||||
private static final void writeDoc(Writer writer, String url, ArrayList<String> sentences, String grep, OrderedScoreMap<String> scores) throws IOException {
|
||||
private static final void writeDoc(Writer writer, String url, ArrayList<String> sentences, String grep) throws IOException {
|
||||
writer.write("<form name=\"yacydoc" + url + "\" method=\"post\" action=\"#\" enctype=\"multipart/form-data\" accept-charset=\"UTF-8\">\n");
|
||||
writer.write("<fieldset>\n");
|
||||
writer.write("<h1><a href=\"" + url + "\">" + url + "</a></h1>\n");
|
||||
|
|
@ -180,29 +155,19 @@ public class GrepHTMLResponseWriter implements QueryResponseWriter {
|
|||
if (grep == null || grep.length() == 0) writer.write("all lines in document"); else {writer.write("matches for grep phrase \"");writer.write(grep);writer.write("\"");}
|
||||
}
|
||||
writer.write("</dt>");
|
||||
writedd(writer, line, scores);
|
||||
}
|
||||
if (scores != null) {
|
||||
Collection<String> discoveries = scores.keyList(false);
|
||||
writer.write("<dt>Citations:</dt><dd></dd>");
|
||||
for (String u: discoveries) {
|
||||
writer.write("<dt>");
|
||||
writer.write(Integer.toString(scores.get(u)));
|
||||
writer.write(" citations</dt><dd>");
|
||||
writedd(writer, u, scores);
|
||||
}
|
||||
writedd(writer, line, grep);
|
||||
}
|
||||
writer.write("</dl>\n");
|
||||
writer.write("</fieldset>\n");
|
||||
writer.write("</form>\n");
|
||||
}
|
||||
|
||||
private static void writedd(Writer writer, String line, OrderedScoreMap<String> scores) throws IOException {
|
||||
private static void writedd(Writer writer, String line, String grep) throws IOException {
|
||||
writer.write("<dd><a href=\"/solr/select?q=text_t:%22");
|
||||
XML.escapeAttributeValue(line, writer);
|
||||
writer.write("%22&rows=100&discover=");
|
||||
writer.write(scores != null ? "true" : "false");
|
||||
writer.write("&wt=grephtml\">");
|
||||
writer.write("%22&rows=100&grep=%22");
|
||||
XML.escapeAttributeValue(grep, writer);
|
||||
writer.write("%22&wt=grephtml\">");
|
||||
XML.escapeAttributeValue(line, writer);
|
||||
writer.write("</a></dd>\n");
|
||||
}
|
||||
|
|
|
|||
|
|
@ -90,7 +90,7 @@ public class HTMLResponseWriter implements QueryResponseWriter {
|
|||
paramsList.remove("wt");
|
||||
String xmlquery = dqp.matcher("/solr/select?" + SolrParams.toSolrParams(paramsList).toString()).replaceAll("%22");
|
||||
writer.write("<div id=\"api\"><a href=\"" + xmlquery + "\"><img src=\"../env/grafics/api.png\" width=\"60\" height=\"40\" alt=\"API\" /></a>\n");
|
||||
writer.write("<span>This search result can also be retrieved as XML. Click the API icon to see an example call to the search rss API.</div>\n");
|
||||
writer.write("<span>This search result can also be retrieved as XML. Click the API icon to see this page as XML.</div>\n");
|
||||
|
||||
DocList response = ((ResultContext) values.get("response")).docs;
|
||||
final int sz = response.size();
|
||||
|
|
|
|||
Loading…
Reference in New Issue
Block a user