mirror of
https://github.com/yacy/yacy_search_server.git
synced 2024-09-19 00:01:41 +02:00
ehanced location search:
- search request are now made using a map boundary - search results are only computed for the map boundary - the number of results is adopted to the results in the visible range - added a double-buffering for the search result markers - added a search query option for the search results: /radius/<lat>/<lon>/<radius>
This commit is contained in:
Michael Peter Christen
parent
434af404c1
commit
9b4c699526
|
|
@ -255,7 +255,7 @@ public final class search {
|
|||
indexSegment,
|
||||
rankingProfile,
|
||||
header.get(RequestHeader.USER_AGENT, ""),
|
||||
false
|
||||
false, 0.0d, 0.0d, 0.0d
|
||||
);
|
||||
Network.log.logInfo("INIT HASH SEARCH (abstracts only): " + QueryParams.anonymizedQueryHashes(theQuery.queryHashes) + " - " + theQuery.itemsPerPage() + " links");
|
||||
|
||||
|
|
@ -318,7 +318,7 @@ public final class search {
|
|||
sb.indexSegments.segment(Segments.Process.PUBLIC),
|
||||
rankingProfile,
|
||||
header.get(RequestHeader.USER_AGENT, ""),
|
||||
false
|
||||
false, 0.0d, 0.0d, 0.0d
|
||||
);
|
||||
Network.log.logInfo("INIT HASH SEARCH (query-" + abstracts + "): " + QueryParams.anonymizedQueryHashes(theQuery.queryHashes) + " - " + theQuery.itemsPerPage() + " links");
|
||||
EventChannel.channels(EventChannel.REMOTESEARCH).addMessage(new RSSMessage("Remote Search Request from " + ((remoteSeed == null) ? "unknown" : remoteSeed.getName()), QueryParams.anonymizedQueryHashes(theQuery.queryHashes), ""));
|
||||
|
|
|
|||
|
|
@ -485,6 +485,29 @@ public class yacysearch {
|
|||
}
|
||||
}
|
||||
|
||||
int radius = 0;
|
||||
double lon = 0.0d, lat = 0.0d, rad = 0.0d;
|
||||
if ((radius = querystring.indexOf("/radius/")) >= 0) {
|
||||
int ve = querystring.indexOf(' ', radius + 8);
|
||||
String geo = "";
|
||||
if (ve < 0) {
|
||||
geo = querystring.substring(radius);
|
||||
querystring = querystring.substring(0, radius).trim();
|
||||
} else {
|
||||
geo = querystring.substring(radius, ve);
|
||||
querystring = querystring.substring(0, radius) + querystring.substring(ve);
|
||||
}
|
||||
geo = geo.substring(8);
|
||||
String[] sp = geo.split("/");
|
||||
if (sp.length == 3) try {
|
||||
lat = Double.parseDouble(sp[0]);
|
||||
lon = Double.parseDouble(sp[1]);
|
||||
rad = Double.parseDouble(sp[2]);
|
||||
} catch (NumberFormatException e) {
|
||||
lon = 0.0d; lat = 0.0d; rad = 0.0d;
|
||||
}
|
||||
}
|
||||
|
||||
String tenant = null;
|
||||
if ( post.containsKey("tenant") ) {
|
||||
tenant = post.get("tenant");
|
||||
|
|
@ -757,7 +780,8 @@ public class yacysearch {
|
|||
header.get(RequestHeader.USER_AGENT, ""),
|
||||
sb.getConfigBool(SwitchboardConstants.SEARCH_VERIFY_DELETE, false)
|
||||
&& sb.getConfigBool(SwitchboardConstants.NETWORK_SEARCHVERIFY, false)
|
||||
&& sb.peers.mySeed().getFlagAcceptRemoteIndex());
|
||||
&& sb.peers.mySeed().getFlagAcceptRemoteIndex(),
|
||||
lat, lon, rad);
|
||||
EventTracker.delete(EventTracker.EClass.SEARCH);
|
||||
EventTracker.update(EventTracker.EClass.SEARCH, new ProfilingGraph.EventSearch(
|
||||
theQuery.id(true),
|
||||
|
|
|
|||
|
|
@ -20,6 +20,8 @@
|
|||
var shallZoom = true; // flag if the map shall be zoomed to the search result
|
||||
var shallReload = false; // flag if the search shall be repeated upon next resize of the window
|
||||
var poisfound = 0;
|
||||
var kmNormal = 100; // ca. 1km grid for radius
|
||||
var meterNormal = 100000; // ca. 1m grid for location
|
||||
|
||||
// automatically adapt height of map upon window resize
|
||||
function adaptHeight() {
|
||||
|
|
@ -66,6 +68,7 @@
|
|||
|
||||
function zoomChanged () {
|
||||
shallZoom = false; // no automatic zoom if user hs zoomed
|
||||
refreshsearch();
|
||||
}
|
||||
|
||||
// called if a search is submitted
|
||||
|
|
@ -81,8 +84,11 @@
|
|||
var center = map.getCenter().transform(new OpenLayers.Projection("EPSG:900913"), new OpenLayers.Projection("EPSG:4326"));
|
||||
var extend = map.getExtent().transform(new OpenLayers.Projection("EPSG:900913"), new OpenLayers.Projection("EPSG:4326"));
|
||||
var radius = Math.sqrt(extend.getWidth() * extend.getWidth() + extend.getHeight() * extend.getHeight()) / 2;
|
||||
searchLayer_co0 = new OpenLayers.Layer.GeoRSS('Communities', path_losearch + query + '&lon=' + center.lon + '&lat=' + center.lat + '&r=' + radius + '&z=' + map.getZoom(), {'icon':marker_co});
|
||||
searchLayer_md0 = new OpenLayers.Layer.GeoRSS('Point Of Interest', path_mdsearch + query + '&lon=' + center.lon + '&lat=' + center.lat + '&r=' + radius + '&z=' + map.getZoom(), {'icon':marker_md});
|
||||
radius = Math.floor(radius * kmNormal + 1) / kmNormal;
|
||||
var lon = Math.floor(center.lon * meterNormal) / meterNormal;
|
||||
var lat = Math.floor(center.lat * meterNormal) / meterNormal;
|
||||
searchLayer_co0 = new OpenLayers.Layer.GeoRSS('Communities', path_losearch + query + '&lon=' + lon + '&lat=' + lat + '&r=' + radius + '&z=' + map.getZoom(), {'icon':marker_co});
|
||||
searchLayer_md0 = new OpenLayers.Layer.GeoRSS('Point Of Interest', path_mdsearch + query + '&lon=' + lon + '&lat=' + lat + '&r=' + radius + '&z=' + map.getZoom(), {'icon':marker_md});
|
||||
map.addLayer(searchLayer_co0);
|
||||
map.addLayer(searchLayer_md0);
|
||||
//document.getElementById('apilink').setAttribute('href', 'yacysearch_location.rss?query=' + query);
|
||||
|
|
@ -92,7 +98,7 @@
|
|||
|
||||
function refresh() {
|
||||
// check double-buffer state
|
||||
if (searchLayer_md0 != null && searchLayer_md0.markers.length > 0) {
|
||||
if (searchLayer_md0 != null) {
|
||||
// switch the double-buffer
|
||||
if (searchLayer_md1 != null) searchLayer_md1.destroy();
|
||||
if (searchLayer_co1 != null) searchLayer_co1.destroy();
|
||||
|
|
@ -108,13 +114,13 @@
|
|||
}
|
||||
var cocount = searchLayer_co1.markers.length;
|
||||
var mdcount = searchLayer_md1.markers.length;
|
||||
if (mdcount + cocount <= poisfound) {
|
||||
return;
|
||||
}
|
||||
//if (mdcount + cocount <= poisfound) {
|
||||
// return;
|
||||
//}
|
||||
|
||||
// update map and result line
|
||||
poisfound = mdcount + cocount;
|
||||
document.getElementById('resultline').innerHTML = poisfound + " POIs found";
|
||||
document.getElementById('resultline').innerHTML = poisfound + " POIs found in visible map";
|
||||
|
||||
// check if any of the markers is visible on the screen.
|
||||
// this is only necessary if shallZoom = false since this would set shallZoom = true
|
||||
|
|
|
|||
|
|
@ -60,6 +60,9 @@ public class yacysearch_location {
|
|||
final boolean search_subject = alltext || post.get("dom", "").indexOf("subject",0) >= 0;
|
||||
final long maximumTime = post.getLong("maximumTime", 5000);
|
||||
final int maximumRecords = post.getInt("maximumRecords", 3000);
|
||||
final double lon = post.getDouble("lon", 0.0d);
|
||||
final double lat = post.getDouble("lat", 0.0d);
|
||||
final double radius = post.getDouble("r", 0.0d);
|
||||
//i.e. http://localhost:8090/yacysearch_location.kml?query=berlin&maximumTime=2000&maximumRecords=100
|
||||
|
||||
int placemarkCounter = 0;
|
||||
|
|
@ -89,7 +92,7 @@ public class yacysearch_location {
|
|||
// get a queue of search results
|
||||
final String rssSearchServiceURL = "http://127.0.0.1:" + sb.getConfig("port", "8090") + "/yacysearch.rss";
|
||||
final BlockingQueue<RSSMessage> results = new LinkedBlockingQueue<RSSMessage>();
|
||||
SRURSSConnector.searchSRURSS(results, rssSearchServiceURL, query, maximumTime, Integer.MAX_VALUE, null, false, null);
|
||||
SRURSSConnector.searchSRURSS(results, rssSearchServiceURL, lon == 0.0d && lat == 0.0d ? query : query + " /radius/" + lat + "/" + lon + "/" + radius, maximumTime, Integer.MAX_VALUE, null, false, null);
|
||||
|
||||
// take the results and compute some locations
|
||||
RSSMessage message;
|
||||
|
|
|
|||
|
|
@ -222,7 +222,7 @@ public final class ResultURLs {
|
|||
public static void main(final String[] args) {
|
||||
try {
|
||||
final DigestURI url = new DigestURI("http", "www.yacy.net", 80, "/");
|
||||
final URIMetadataRow urlRef = new URIMetadataRow(url, "YaCy Homepage", "", "", "", 0.0f, 0.0f, new Date(), new Date(), new Date(), "", new byte[] {}, 123, 42, '?', new Bitfield(), UTF8.getBytes("de"), 0, 0, 0, 0, 0, 0);
|
||||
final URIMetadataRow urlRef = new URIMetadataRow(url, "YaCy Homepage", "", "", "", 0.0d, 0.0d, new Date(), new Date(), new Date(), "", new byte[] {}, 123, 42, '?', new Bitfield(), UTF8.getBytes("de"), 0, 0, 0, 0, 0, 0);
|
||||
final EventOrigin stackNo = EventOrigin.LOCAL_CRAWLING;
|
||||
System.out.println("valid test:\n=======");
|
||||
// add
|
||||
|
|
|
|||
|
|
@ -183,6 +183,10 @@ public class serverObjects extends HashMap<String, String> implements Cloneable
|
|||
return (null == this.put(key, Float.toString(value))) ? Float.NaN : value;
|
||||
}
|
||||
|
||||
public double put(final String key, final double value) {
|
||||
return (null == this.put(key, Double.toString(value))) ? Double.NaN : value;
|
||||
}
|
||||
|
||||
/**
|
||||
* same as {@link #put(String, double)} but for integer types
|
||||
* @return Returns 0 for the error case.
|
||||
|
|
@ -359,6 +363,16 @@ public class serverObjects extends HashMap<String, String> implements Cloneable
|
|||
}
|
||||
}
|
||||
|
||||
public double getDouble(final String key, final double dflt) {
|
||||
final String s = removeByteOrderMark(super.get(key));
|
||||
if (s == null) return dflt;
|
||||
try {
|
||||
return Double.parseDouble(s);
|
||||
} catch (final NumberFormatException e) {
|
||||
return dflt;
|
||||
}
|
||||
}
|
||||
|
||||
public boolean getBoolean(final String key, final boolean dflt) {
|
||||
String s = removeByteOrderMark(super.get(key));
|
||||
if (s == null) return dflt;
|
||||
|
|
|
|||
|
|
@ -64,6 +64,10 @@ public class SolrDoc extends SolrInputDocument {
|
|||
this.setField(key.getSolrFieldName(), value);
|
||||
}
|
||||
|
||||
public final void addSolr(final SolrField key, final double value) {
|
||||
this.setField(key.getSolrFieldName(), value);
|
||||
}
|
||||
|
||||
public final void addSolr(final SolrField key, final boolean value) {
|
||||
this.setField(key.getSolrFieldName(), value);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -87,7 +87,7 @@ public class Document {
|
|||
private boolean resorted;
|
||||
private final Set<String> languages;
|
||||
private final boolean indexingDenied;
|
||||
private final float lon, lat;
|
||||
private final double lon, lat;
|
||||
private final Object parserObject; // the source object that was used to create the Document
|
||||
|
||||
public Document(final MultiProtocolURI location, final String mimeType, final String charset,
|
||||
|
|
@ -95,7 +95,7 @@ public class Document {
|
|||
final Set<String> languages,
|
||||
final String[] keywords, final String title, final String author, final String publisher,
|
||||
final String[] sections, final String abstrct,
|
||||
final float lon, final float lat,
|
||||
final double lon, final double lat,
|
||||
final Object text,
|
||||
final Map<MultiProtocolURI, Properties> anchors,
|
||||
final Map<MultiProtocolURI, String> rss,
|
||||
|
|
@ -400,11 +400,11 @@ dc_rights
|
|||
return this.emaillinks;
|
||||
}
|
||||
|
||||
public float lon() {
|
||||
public double lon() {
|
||||
return this.lon;
|
||||
}
|
||||
|
||||
public float lat() {
|
||||
public double lat() {
|
||||
return this.lat;
|
||||
}
|
||||
|
||||
|
|
@ -743,7 +743,7 @@ dc_rights
|
|||
final Map<MultiProtocolURI, Properties> anchors = new HashMap<MultiProtocolURI, Properties>();
|
||||
final Map<MultiProtocolURI, String> rss = new HashMap<MultiProtocolURI, String>();
|
||||
final Map<MultiProtocolURI, ImageEntry> images = new HashMap<MultiProtocolURI, ImageEntry>();
|
||||
float lon = 0.0f, lat = 0.0f;
|
||||
double lon = 0.0d, lat = 0.0d;
|
||||
|
||||
for (final Document doc: docs) {
|
||||
|
||||
|
|
@ -784,7 +784,7 @@ dc_rights
|
|||
anchors.putAll(doc.getAnchors());
|
||||
rss.putAll(doc.getRSS());
|
||||
ContentScraper.addAllImages(images, doc.getImages());
|
||||
if (doc.lon() != 0.0f && doc.lat() != 0.0f) { lon = doc.lon(); lat = doc.lat(); }
|
||||
if (doc.lon() != 0.0d && doc.lat() != 0.0d) { lon = doc.lon(); lat = doc.lat(); }
|
||||
}
|
||||
return new Document(
|
||||
location,
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@
|
|||
// $LastChangedBy$
|
||||
//
|
||||
// LICENSE
|
||||
//
|
||||
//
|
||||
// This program is free software; you can redistribute it and/or modify
|
||||
// it under the terms of the GNU General Public License as published by
|
||||
// the Free Software Foundation; either version 2 of the License, or
|
||||
|
|
@ -43,9 +43,9 @@ import net.yacy.kelondro.logging.Log;
|
|||
|
||||
|
||||
public class DCEntry extends TreeMap<String, String> {
|
||||
|
||||
|
||||
private static final long serialVersionUID = -2050291583515701559L;
|
||||
|
||||
|
||||
// use a collator to relax when distinguishing between lowercase und uppercase letters
|
||||
private static final Collator insensitiveCollator = Collator.getInstance(Locale.US);
|
||||
static {
|
||||
|
|
@ -53,19 +53,19 @@ public class DCEntry extends TreeMap<String, String> {
|
|||
insensitiveCollator.setDecomposition(Collator.NO_DECOMPOSITION);
|
||||
}
|
||||
public static final DCEntry poison = new DCEntry();
|
||||
|
||||
|
||||
public DCEntry() {
|
||||
super((Collator) insensitiveCollator.clone());
|
||||
}
|
||||
|
||||
|
||||
public DCEntry(
|
||||
DigestURI url,
|
||||
Date date,
|
||||
String title,
|
||||
String author,
|
||||
String body,
|
||||
float lat,
|
||||
float lon
|
||||
double lat,
|
||||
double lon
|
||||
) {
|
||||
super((Collator) insensitiveCollator.clone());
|
||||
this.put("dc:identifier", url.toNormalform(true, false));
|
||||
|
|
@ -73,10 +73,10 @@ public class DCEntry extends TreeMap<String, String> {
|
|||
this.put("dc:title", title);
|
||||
this.put("dc:creator", author);
|
||||
this.put("dc:description", body);
|
||||
this.put("geo:lat", Float.toString(lat));
|
||||
this.put("geo:long", Float.toString(lon));
|
||||
this.put("geo:lat", Double.toString(lat));
|
||||
this.put("geo:long", Double.toString(lon));
|
||||
}
|
||||
|
||||
|
||||
/*
|
||||
DC according to rfc 5013
|
||||
|
||||
|
|
@ -108,7 +108,7 @@ public class DCEntry extends TreeMap<String, String> {
|
|||
return new Date();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
public DigestURI getIdentifier(boolean useRelationAsAlternative) {
|
||||
String u = this.get("url");
|
||||
if (u == null) u = this.get("dc:identifier");
|
||||
|
|
@ -130,7 +130,7 @@ public class DCEntry extends TreeMap<String, String> {
|
|||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
public DigestURI getRelation() {
|
||||
String u = this.get("dc:relation");
|
||||
if (u == null) return null;
|
||||
|
|
@ -146,7 +146,7 @@ public class DCEntry extends TreeMap<String, String> {
|
|||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
private String bestU(String[] urls) {
|
||||
for (String uu: urls) {
|
||||
if (uu.startsWith("http://") && (uu.endsWith(".html") || uu.endsWith(".htm") || uu.endsWith(".pdf") || uu.endsWith(".doc") || uu.endsWith(".rss") || uu.endsWith(".xml"))) return uu;
|
||||
|
|
@ -164,7 +164,7 @@ public class DCEntry extends TreeMap<String, String> {
|
|||
}
|
||||
return urls[0];
|
||||
}
|
||||
|
||||
|
||||
//modified by copperdust; Ukraine, 2012
|
||||
public String getLanguage() {//final language computation
|
||||
String l = this.get("dc:language");//from document metainfo
|
||||
|
|
@ -172,31 +172,31 @@ public class DCEntry extends TreeMap<String, String> {
|
|||
if (l == null) return this.get("language");//from TLD
|
||||
return l;
|
||||
}
|
||||
|
||||
|
||||
public String getType() {
|
||||
String t = this.get("dc:type");
|
||||
if (t == null) return "";
|
||||
return t;
|
||||
}
|
||||
|
||||
|
||||
public String getFormat() {
|
||||
String t = this.get("dc:format");
|
||||
if (t == null) return "";
|
||||
return t;
|
||||
}
|
||||
|
||||
|
||||
public String getSource() {
|
||||
String t = this.get("dc:source");
|
||||
if (t == null) return "";
|
||||
return t;
|
||||
}
|
||||
|
||||
|
||||
public String getRights() {
|
||||
String t = this.get("dc:rights");
|
||||
if (t == null) return "";
|
||||
return t;
|
||||
}
|
||||
|
||||
|
||||
public String getTitle() {
|
||||
String t = this.get("title");
|
||||
if (t == null) t = this.get("dc:title");
|
||||
|
|
@ -204,14 +204,14 @@ public class DCEntry extends TreeMap<String, String> {
|
|||
if (t == null) return "";
|
||||
return t;
|
||||
}
|
||||
|
||||
|
||||
public String getPublisher() {
|
||||
String t = this.get("dc:publisher");
|
||||
t = stripCDATA(t);
|
||||
if (t == null) return "";
|
||||
return t;
|
||||
}
|
||||
|
||||
|
||||
public String getCreator() {
|
||||
String t = this.get("author");
|
||||
if (t == null) t = this.get("dc:creator");
|
||||
|
|
@ -219,7 +219,7 @@ public class DCEntry extends TreeMap<String, String> {
|
|||
if (t == null) return "";
|
||||
return t;
|
||||
}
|
||||
|
||||
|
||||
public String getDescription() {
|
||||
String t = this.get("body");
|
||||
if (t == null) t = this.get("dc:description");
|
||||
|
|
@ -227,7 +227,7 @@ public class DCEntry extends TreeMap<String, String> {
|
|||
if (t == null) return "";
|
||||
return t;
|
||||
}
|
||||
|
||||
|
||||
public String[] getSubject() {
|
||||
String t = this.get("categories");
|
||||
if (t == null) this.get("dc:subject");
|
||||
|
|
@ -235,23 +235,23 @@ public class DCEntry extends TreeMap<String, String> {
|
|||
if (t == null) return new String[]{};
|
||||
return t.split(";");
|
||||
}
|
||||
|
||||
public float getLon() {
|
||||
|
||||
public double getLon() {
|
||||
String t = this.get("geo:long");
|
||||
if (t == null) this.get("geo:lon");
|
||||
t = stripCDATA(t);
|
||||
if (t == null) return 0.0f;
|
||||
return Float.parseFloat(t);
|
||||
if (t == null) return 0.0d;
|
||||
return Double.parseDouble(t);
|
||||
}
|
||||
|
||||
public float getLat() {
|
||||
|
||||
public double getLat() {
|
||||
String t = this.get("geo:lat");
|
||||
if (t == null) this.get("geo:lat");
|
||||
t = stripCDATA(t);
|
||||
if (t == null) return 0.0f;
|
||||
return Float.parseFloat(t);
|
||||
if (t == null) return 0.0d;
|
||||
return Double.parseDouble(t);
|
||||
}
|
||||
|
||||
|
||||
private String stripCDATA(String s) {
|
||||
if (s == null) return null;
|
||||
s = s.trim();
|
||||
|
|
@ -259,11 +259,11 @@ public class DCEntry extends TreeMap<String, String> {
|
|||
if (s.endsWith("]]")) s = s.substring(0, s.length() - 2);
|
||||
return s;
|
||||
}
|
||||
|
||||
|
||||
public Document document() {
|
||||
HashSet<String> languages = new HashSet<String>();
|
||||
languages.add(getLanguage());
|
||||
|
||||
|
||||
return new Document(
|
||||
getIdentifier(true),
|
||||
"text/html",
|
||||
|
|
@ -283,7 +283,7 @@ public class DCEntry extends TreeMap<String, String> {
|
|||
null,
|
||||
false);
|
||||
}
|
||||
|
||||
|
||||
public void writeXML(OutputStreamWriter os) throws IOException {
|
||||
Document doc = document();
|
||||
if (doc != null) {
|
||||
|
|
|
|||
|
|
@ -214,7 +214,7 @@ public class PhpBB3Dao implements Dao {
|
|||
String text = xmlCleaner(rs.getString("post_text"));
|
||||
String user = getUser(rs.getInt("poster_id"));
|
||||
Date date = new Date(rs.getLong("post_time") * 1000L);
|
||||
return new DCEntry(url, date, subject, user, text, 0.0f, 0.0f);
|
||||
return new DCEntry(url, date, subject, user, text, 0.0d, 0.0d);
|
||||
}
|
||||
|
||||
public static String xmlCleaner(String s) {
|
||||
|
|
|
|||
|
|
@ -130,7 +130,7 @@ public class ContentScraper extends AbstractScraper implements Scraper {
|
|||
private final List<String> li;
|
||||
private final CharBuffer content;
|
||||
private final EventListenerList htmlFilterEventListeners;
|
||||
private float lon, lat;
|
||||
private double lon, lat;
|
||||
private MultiProtocolURI canonical;
|
||||
|
||||
|
||||
|
|
@ -803,7 +803,7 @@ public class ContentScraper extends AbstractScraper implements Scraper {
|
|||
// <meta NAME="ICBM" CONTENT="38.90551492, 1.454004505" />
|
||||
// <meta NAME="geo.position" CONTENT="38.90551492;1.454004505" />
|
||||
|
||||
public float getLon() {
|
||||
public double getLon() {
|
||||
if (this.lon != 0.0f) return this.lon;
|
||||
String s = this.metas.get("ICBM"); // InterContinental Ballistic Missile (abbrev. supposed to be a joke: http://www.jargon.net/jargonfile/i/ICBMaddress.html), see http://geourl.org/add.html#icbm
|
||||
if (s != null) {
|
||||
|
|
@ -811,8 +811,8 @@ public class ContentScraper extends AbstractScraper implements Scraper {
|
|||
if (p < 0) p = s.indexOf(',');
|
||||
if (p < 0) p = s.indexOf(' ');
|
||||
if (p > 0) {
|
||||
this.lat = Float.parseFloat(s.substring(0, p).trim());
|
||||
this.lon = Float.parseFloat(s.substring(p + 1).trim());
|
||||
this.lat = Double.parseDouble(s.substring(0, p).trim());
|
||||
this.lon = Double.parseDouble(s.substring(p + 1).trim());
|
||||
}
|
||||
}
|
||||
if (this.lon != 0.0f) return this.lon;
|
||||
|
|
@ -822,15 +822,15 @@ public class ContentScraper extends AbstractScraper implements Scraper {
|
|||
if (p < 0) p = s.indexOf(',');
|
||||
if (p < 0) p = s.indexOf(' ');
|
||||
if (p > 0) {
|
||||
this.lat = Float.parseFloat(s.substring(0, p).trim());
|
||||
this.lon = Float.parseFloat(s.substring(p + 1).trim());
|
||||
this.lat = Double.parseDouble(s.substring(0, p).trim());
|
||||
this.lon = Double.parseDouble(s.substring(p + 1).trim());
|
||||
}
|
||||
}
|
||||
return this.lon;
|
||||
}
|
||||
|
||||
public float getLat() {
|
||||
if (this.lat != 0.0f) return this.lat;
|
||||
public double getLat() {
|
||||
if (this.lat != 0.0d) return this.lat;
|
||||
getLon(); // parse with getLon() method which creates also the lat value
|
||||
return this.lat;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -9,12 +9,12 @@
|
|||
* modify it under the terms of the GNU Lesser General Public
|
||||
* License as published by the Free Software Foundation; either
|
||||
* version 2.1 of the License, or (at your option) any later version.
|
||||
*
|
||||
*
|
||||
* This library is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
* Lesser General Public License for more details.
|
||||
*
|
||||
*
|
||||
* You should have received a copy of the GNU Lesser General Public License
|
||||
* along with this program in the file lgpl21.txt
|
||||
* If not, see <http://www.gnu.org/licenses/>.
|
||||
|
|
@ -31,17 +31,17 @@ import net.yacy.kelondro.rwi.Reference;
|
|||
public interface URIMetadata extends URIReference {
|
||||
|
||||
public String dc_title();
|
||||
|
||||
|
||||
public String dc_creator();
|
||||
|
||||
|
||||
public String dc_publisher();
|
||||
|
||||
|
||||
public String dc_subject();
|
||||
|
||||
public float lat();
|
||||
|
||||
public float lon();
|
||||
|
||||
|
||||
public double lat();
|
||||
|
||||
public double lon();
|
||||
|
||||
public long ranking();
|
||||
|
||||
public Date loaddate();
|
||||
|
|
@ -71,13 +71,13 @@ public interface URIMetadata extends URIReference {
|
|||
public int lvideo();
|
||||
|
||||
public int lapp();
|
||||
|
||||
|
||||
public String snippet();
|
||||
|
||||
public Reference word();
|
||||
|
||||
public boolean isOlder(final URIMetadata other);
|
||||
|
||||
|
||||
public String toString(final String snippet);
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -123,7 +123,7 @@ public class URIMetadataRow implements URIMetadata {
|
|||
final String dc_creator,
|
||||
final String dc_subject,
|
||||
final String dc_publisher,
|
||||
final float lon, final float lat, // decimal degrees as in WGS84; if unknown both values may be 0.0f;
|
||||
final double lon, final double lat, // decimal degrees as in WGS84; if unknown both values may be 0.0d;
|
||||
final Date mod,
|
||||
final Date load,
|
||||
final Date fresh,
|
||||
|
|
@ -194,8 +194,8 @@ public class URIMetadataRow implements URIMetadata {
|
|||
final String dc_creator,
|
||||
final String dc_subject,
|
||||
final String dc_publisher,
|
||||
final float lat,
|
||||
final float lon) {
|
||||
final double lat,
|
||||
final double lon) {
|
||||
final CharBuffer s = new CharBuffer(3600, 360);
|
||||
s.append(url.toNormalform(false, true)).appendLF();
|
||||
s.append(dc_title).appendLF();
|
||||
|
|
@ -205,7 +205,7 @@ public class URIMetadataRow implements URIMetadata {
|
|||
s.appendLF();
|
||||
if (dc_publisher.length() > 80) s.append(dc_publisher, 0, 80); else s.append(dc_publisher);
|
||||
s.appendLF();
|
||||
if (lon == 0.0f && lat == 0.0f) s.appendLF(); else s.append(Float.toString(lat)).append(',').append(Float.toString(lon)).appendLF();
|
||||
if (lon == 0.0f && lat == 0.0f) s.appendLF(); else s.append(Double.toString(lat)).append(',').append(Double.toString(lon)).appendLF();
|
||||
return UTF8.getBytes(s.toString());
|
||||
}
|
||||
|
||||
|
|
@ -418,11 +418,11 @@ public class URIMetadataRow implements URIMetadata {
|
|||
return this.metadata().dc_subject();
|
||||
}
|
||||
|
||||
public float lat() {
|
||||
public double lat() {
|
||||
return this.metadata().lat();
|
||||
}
|
||||
|
||||
public float lon() {
|
||||
public double lon() {
|
||||
return this.metadata().lon();
|
||||
}
|
||||
|
||||
|
|
@ -638,15 +638,15 @@ public class URIMetadataRow implements URIMetadata {
|
|||
public String dc_creator() { return this.dc_creator; }
|
||||
public String dc_publisher() { return this.dc_publisher; }
|
||||
public String dc_subject() { return this.dc_subject; }
|
||||
public float lat() {
|
||||
if (this.latlon == null || this.latlon.length() == 0) return 0.0f;
|
||||
public double lat() {
|
||||
if (this.latlon == null || this.latlon.length() == 0) return 0.0d;
|
||||
final int p = this.latlon.indexOf(',');
|
||||
return p < 0 ? 0.0f : Float.parseFloat(this.latlon.substring(0, p));
|
||||
return p < 0 ? 0.0f : Double.parseDouble(this.latlon.substring(0, p));
|
||||
}
|
||||
public float lon() {
|
||||
if (this.latlon == null || this.latlon.length() == 0) return 0.0f;
|
||||
public double lon() {
|
||||
if (this.latlon == null || this.latlon.length() == 0) return 0.0d;
|
||||
final int p = this.latlon.indexOf(',');
|
||||
return p < 0 ? 0.0f : Float.parseFloat(this.latlon.substring(p + 1));
|
||||
return p < 0 ? 0.0f : Double.parseDouble(this.latlon.substring(p + 1));
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -377,7 +377,7 @@ public class Segment {
|
|||
document.dc_subject(' '), // tags
|
||||
document.dc_publisher(), // publisher (may be important to get location data)
|
||||
document.lon(), // decimal degrees as in WGS84;
|
||||
document.lat(), // if unknown both values may be 0.0f;
|
||||
document.lat(), // if unknown both values may be 0.0d;
|
||||
modDate, // modification date
|
||||
loadDate, // loaded date
|
||||
new Date(loadDate.getTime() + Math.max(0, loadDate.getTime() - modDate.getTime()) / 2), // freshdate, computed with Proxy-TTL formula
|
||||
|
|
|
|||
|
|
@ -108,6 +108,10 @@ public class SolrConfiguration extends ConfigurationSet implements Serializable
|
|||
if (isEmpty() || contains(key.name())) solrdoc.addSolr(key, value);
|
||||
}
|
||||
|
||||
protected void addSolr(final SolrDoc solrdoc, final SolrField key, final double value) {
|
||||
if (isEmpty() || contains(key.name())) solrdoc.addSolr(key, value);
|
||||
}
|
||||
|
||||
protected void addSolr(final SolrDoc solrdoc, final SolrField key, final boolean value) {
|
||||
if (isEmpty() || contains(key.name())) solrdoc.addSolr(key, value);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -141,6 +141,7 @@ public final class QueryParams {
|
|||
public boolean specialRights; // is true if the user has a special authorization and my use more database-extensive options
|
||||
public final String userAgent;
|
||||
public boolean filterfailurls;
|
||||
public double lat, lon, radius;
|
||||
|
||||
public QueryParams(
|
||||
final String queryString,
|
||||
|
|
@ -198,6 +199,9 @@ public final class QueryParams {
|
|||
this.userAgent = userAgent;
|
||||
this.transmitcount = 0;
|
||||
this.filterfailurls = false;
|
||||
this.lat = 0.0d;
|
||||
this.lon = 0.0d;
|
||||
this.radius = 0.0d;
|
||||
}
|
||||
|
||||
public QueryParams(
|
||||
|
|
@ -224,7 +228,8 @@ public final class QueryParams {
|
|||
final Segment indexSegment,
|
||||
final RankingProfile ranking,
|
||||
final String userAgent,
|
||||
final boolean filterfailurls) {
|
||||
final boolean filterfailurls,
|
||||
final double lat, final double lon, final double radius) {
|
||||
|
||||
this.queryString = queryString;
|
||||
this.queryHashes = queryHashes;
|
||||
|
|
@ -271,8 +276,15 @@ public final class QueryParams {
|
|||
this.userAgent = userAgent;
|
||||
this.transmitcount = 0;
|
||||
this.filterfailurls = filterfailurls;
|
||||
// we normalize here the location and radius because that should cause a better caching
|
||||
// and as surplus it will increase privacy
|
||||
this.lat = Math.floor(lat * this.kmNormal) / this.kmNormal;
|
||||
this.lon = Math.floor(lon * this.kmNormal) / this.kmNormal;
|
||||
this.radius = Math.floor(radius * this.kmNormal + 1) / this.kmNormal;
|
||||
}
|
||||
|
||||
double kmNormal = 100.d; // 100 =ca 40000.d / 360.d == 111.11 - if lat/lon is multiplied with this, rounded and diveded by this, the location is normalized to a 1km grid
|
||||
|
||||
public Segment getSegment() {
|
||||
return this.indexSegment;
|
||||
}
|
||||
|
|
@ -511,6 +523,8 @@ public final class QueryParams {
|
|||
context.append(asterisk);
|
||||
context.append(this.modifier.s);
|
||||
context.append(asterisk);
|
||||
context.append(this.lat).append(asterisk).append(this.lon).append(asterisk).append(this.radius);
|
||||
context.append(asterisk);
|
||||
context.append(this.snippetCacheStrategy == null ? "null" : this.snippetCacheStrategy.name());
|
||||
String result = context.toString();
|
||||
if (anonymized) {
|
||||
|
|
|
|||
|
|
@ -629,6 +629,18 @@ public final class RWIProcess extends Thread
|
|||
continue;
|
||||
}
|
||||
|
||||
// check geo coordinates
|
||||
double lat, lon;
|
||||
if (this.query.radius > 0.0d && this.query.lat != 0.0d && this.query.lon != 0.0d && (lat = page.lat()) > 0.0d && (lon = page.lon()) > 0.0d) {
|
||||
double latDelta = this.query.lat - lat;
|
||||
double lonDelta = this.query.lon - lon;
|
||||
double distance = Math.sqrt(latDelta * latDelta + lonDelta * lonDelta); // pythagoras
|
||||
if (distance > this.query.radius) {
|
||||
this.sortout++;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
// check vocabulary constraint
|
||||
final String tags = page.dc_subject();
|
||||
final String[] taglist = tags == null || tags.length() == 0 ? new String[0] : SPACE_PATTERN.split(page.dc_subject());
|
||||
|
|
|
|||
|
|
@ -360,7 +360,7 @@ public class SnippetProcess {
|
|||
Worker worker;
|
||||
if (this.workerThreads == null) {
|
||||
this.workerThreads = new Worker[deployCount];
|
||||
synchronized(this.workerThreads) {
|
||||
synchronized(this.workerThreads) {try {
|
||||
for (int i = 0; i < this.workerThreads.length; i++) {
|
||||
if (this.result.sizeAvailable() >= neededResults ||
|
||||
(this.rankingProcess.feedingIsFinished() && this.rankingProcess.sizeQueue() == 0)) {
|
||||
|
|
@ -376,7 +376,7 @@ public class SnippetProcess {
|
|||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (OutOfMemoryError e) {}}
|
||||
} else {
|
||||
// there are still worker threads running, but some may be dead.
|
||||
// if we find dead workers, reanimate them
|
||||
|
|
|
|||
|
|
@ -182,10 +182,10 @@ public class ResultEntry implements Comparable<ResultEntry>, Comparator<ResultEn
|
|||
public int lapp() {
|
||||
return this.urlentry.lapp();
|
||||
}
|
||||
public float lat() {
|
||||
public double lat() {
|
||||
return this.urlentry.lat();
|
||||
}
|
||||
public float lon() {
|
||||
public double lon() {
|
||||
return this.urlentry.lon();
|
||||
}
|
||||
public WordReferenceVars word() {
|
||||
|
|
|
|||
Loading…
Reference in New Issue
Block a user