added a location search service: using the following servlet/example:

http://localhost:8080/yacysearch_location.kml?query=berlin&maximumTime=2000&maximumRecords=100

This will open any application that can consume kml data (which will probably be google earth) on your computer and displays the search result as positions on a map


git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@6865 6c8d7289-2bf4-0310-a012-ef5d649a1542

htroot/yacysearch.java

@@ -560,7 +560,7 @@ public class yacysearch {
             }
             
             // find geographic info
-            Set<Location> coordinates = LibraryProvider.geoDB.find(originalquerystring, false);
+            Set<Location> coordinates = LibraryProvider.geoDB.find(originalquerystring, true, false, true, true, true);
             if (coordinates == null || coordinates.isEmpty() || offset > 0) {
                 prop.put("geoinfo", "0");
             } else {

htroot/yacysearch_location.java

@@ -0,0 +1,98 @@
+// yacysearch_location.java
+// -----------------------
+// (C) 2010 by Michael Peter Christen; mc@yacy.net
+// first published 09.05.2010 in Frankfurt, Germany on http://yacy.net
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program; if not, write to the Free Software
+// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+import java.util.HashSet;
+import java.util.Set;
+import java.util.concurrent.BlockingQueue;
+
+import net.yacy.document.content.RSSMessage;
+import net.yacy.document.geolocalization.Location;
+import de.anomic.data.LibraryProvider;
+import de.anomic.http.server.HeaderFramework;
+import de.anomic.http.server.RequestHeader;
+import de.anomic.server.serverObjects;
+import de.anomic.server.serverSwitch;
+import de.anomic.yacy.yacyClient;
+
+public class yacysearch_location {
+    
+    public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) {
+        //final Switchboard sb = (Switchboard) env;
+        final serverObjects prop = new serverObjects();
+        
+        prop.put("kml", 0);
+        if (post == null) return prop;
+        
+        if (header.get(HeaderFramework.CONNECTION_PROP_EXT, "").equals("kml") || header.get(HeaderFramework.CONNECTION_PROP_EXT, "").equals("xml")) {
+            // generate a kml output page
+            prop.put("kml", 1);
+            String query = post.get("query", "");
+            long maximumTime = post.getLong("maximumTime", 1000);
+            int maximumRecords = post.getInt("maximumRecords", 100);
+            //i.e. http://localhost:8080/yacysearch_location.kml?query=berlin&maximumTime=2000&maximumRecords=100
+            
+            // get a queue of search results
+            BlockingQueue<RSSMessage> results = yacyClient.search(null, query, false, false, maximumTime, Integer.MAX_VALUE);
+            
+            // take the results and compute some locations
+            RSSMessage message;
+            int placemarkCounter = 0;
+            try {
+                loop: while ((message = results.take()) != RSSMessage.POISON) {
+                    // find all associated locations
+                    Set<Location> locations = new HashSet<Location>();
+                    String words = message.getTitle() + " " + message.getCopyright() + " " + message.getAuthor();
+                    String subject = "";
+                    for (String s: message.getSubject()) subject += " " + s;
+                    words += subject;
+                    for (String word: words.split(" ")) if (word.length() >= 3) locations.addAll(LibraryProvider.geoDB.find(word, true, true, false, false, false));
+                    
+                    if (locations.size() > 0) {
+                        String locnames = "";
+                        for (Location location: locations) locnames += ", " + location.getName();
+                        locnames = locnames.substring(2);
+                        // write for all locations a point to this message
+                        prop.put("kml_placemark_" + placemarkCounter + "_location", locnames);
+                        prop.put("kml_placemark_" + placemarkCounter + "_name", message.getTitle());
+                        prop.put("kml_placemark_" + placemarkCounter + "_author", message.getAuthor());
+                        prop.put("kml_placemark_" + placemarkCounter + "_copyright", message.getCopyright());
+                        prop.put("kml_placemark_" + placemarkCounter + "_subject", subject.trim());
+                        prop.put("kml_placemark_" + placemarkCounter + "_description", message.getDescription());
+                        prop.put("kml_placemark_" + placemarkCounter + "_date", message.getPubDate());
+                        prop.put("kml_placemark_" + placemarkCounter + "_url", message.getLink());
+                        int pc = 0;
+                        for (Location location: locations) {
+                            prop.put("kml_placemark_" + placemarkCounter + "_point_" + pc + "_name", location.getName());
+                            prop.put("kml_placemark_" + placemarkCounter + "_point_" + pc + "_lon", location.lon());
+                            prop.put("kml_placemark_" + placemarkCounter + "_point_" + pc + "_lat", location.lat());
+                            pc++;
+                        }
+                        prop.put("kml_placemark_" + placemarkCounter + "_point", pc);
+                        placemarkCounter++;
+                        if (placemarkCounter >= maximumRecords) break loop;
+                    }
+                }
+                prop.put("kml_placemark", placemarkCounter);
+            } catch (InterruptedException e) {}
+        }
+        // return rewrite properties
+        return prop;
+    }
+    
+}

htroot/yacysearch_location.kml

@@ -0,0 +1,20 @@
+#(kml)#::<?xml version="1.0" encoding="UTF-8"?>
+<kml xmlns="http://www.opengis.net/kml/2.2">
+<Document>
+  #{placemark}#
+  <Placemark>
+    <name>#[name]#</name>
+    <description><![CDATA[<p>Location: #[location]#</p><p>Author: #[author]#</p><p>Publisher: #[copyright]#</p><p>Subject: #[subject]#</p><p>Abstract: #[description]#</p><p>Source: <a href="#[url]#">#[url]#</a></p>]]></description>
+    #{point}#
+    <Point>
+      <name>#[name]#</name>
+      <coordinates>#[lon]#,#[lat]#</coordinates>
+    </Point>
+    #{/point}#
+    <TimeStamp>
+      <when>#[date]#</when>
+    </TimeStamp>
+  </Placemark>
+  #{/placemark}#
+</Document>
+</kml>#(/kml)#

htroot/yacysearch_location.xml

@@ -0,0 +1,20 @@
+#(kml)#::<?xml version="1.0" encoding="UTF-8"?>
+<kml xmlns="http://www.opengis.net/kml/2.2">
+<Document>
+  #{placemark}#
+  <Placemark>
+    <name>#[name]#</name>
+    <description><![CDATA[<p>Location: #[location]#</p><p>Author: #[author]#</p><p>Publisher: #[copyright]#</p><p>Subject: #[subject]#</p><p>Abstract: #[description]#</p><p>Source: <a href="#[url]#">#[url]#</a></p>]]></description>
+    #{point}#
+    <Point>
+      <name>#[name]#</name>
+      <coordinates>#[lon]#,#[lat]#</coordinates>
+    </Point>
+    #{/point}#
+    <TimeStamp>
+      <when>#[date]#</when>
+    </TimeStamp>
+  </Placemark>
+  #{/placemark}#
+</Document>
+</kml>#(/kml)#

htroot/yacysearchitem.java

@@ -31,6 +31,7 @@ import java.util.ArrayList;
 import java.util.TreeSet;
 
 import net.yacy.kelondro.data.meta.DigestURI;
+import net.yacy.kelondro.util.DateFormatter;
 import net.yacy.kelondro.util.EventTracker;
 import net.yacy.kelondro.util.Formatter;
 
@@ -128,8 +129,8 @@ public class yacysearchitem {
             prop.put("content_urlhash", resulthashString);
             prop.put("content_urlhexhash", yacySeed.b64Hash2hexHash(resulthashString));
             prop.putHTML("content_urlname", nxTools.shortenURLString(result.urlname(), urllength));
-            prop.put("content_date", Switchboard.dateString(result.modified()));
-            prop.put("content_date822", Switchboard.dateString822(result.modified()));
+            prop.put("content_date", DateFormatter.formatRFC1123_short(result.modified()));
+            prop.put("content_date822", DateFormatter.formatRFC1123(result.modified()));
             //prop.put("content_ybr", RankingProcess.ybr(result.hash()));
             prop.putHTML("content_size", Integer.toString(result.filesize())); // we don't use putNUM here because that number shall be usable as sorting key. To print the size, use 'sizename'
             prop.putHTML("content_sizename", sizename(result.filesize()));

htroot/yacysearchitem.xml

@@ -3,9 +3,9 @@
 <link>#[link]#</link>
 <description>#[description-xml]#</description>
 <pubDate>#[date822]#</pubDate>
-<dc:publisher>#[publisher]#</dc:publisher>
-<dc:creator>#[creator]#</dc:creator>
-<dc:subject>#[subject]#</dc:subject>
+<dc:publisher><![CDATA[#[publisher]#]]></dc:publisher>
+<dc:creator><![CDATA[#[creator]#]]></dc:creator>
+<dc:subject><![CDATA[#[subject]#]]></dc:subject>
 <yacy:size>#[size]#</yacy:size>
 <yacy:sizename>#[sizename]#</yacy:sizename>
 <yacy:host>#[host]#</yacy:host>

source/de/anomic/search/Switchboard.java

@@ -52,14 +52,12 @@ import java.net.MalformedURLException;
 import java.security.NoSuchAlgorithmException;
 import java.security.PublicKey;
 import java.security.spec.InvalidKeySpecException;
-import java.text.SimpleDateFormat;
 import java.util.ArrayList;
 import java.util.Date;
 import java.util.HashMap;
 import java.util.Hashtable;
 import java.util.Iterator;
 import java.util.List;
-import java.util.Locale;
 import java.util.Map;
 import java.util.Properties;
 import java.util.TreeMap;
@@ -1911,24 +1909,6 @@ public final class Switchboard extends serverSwitch {
         }
     }
     
-    private static SimpleDateFormat DateFormat1 = new SimpleDateFormat("EEE, dd MMM yyyy", Locale.US);
-    public static String dateString(final Date date) {
-        if (date == null) return "";
-        return DateFormat1.format(date);
-    }
-    
-    // we need locale independent RFC-822 dates at some places
-    private static SimpleDateFormat DateFormatter822 = new SimpleDateFormat("EEE, dd MMM yyyy HH:mm:ss Z", Locale.US);
-    public static String dateString822(final Date date) {
-        if (date == null) return "";
-        try {
-        	return DateFormatter822.format(date);
-        } catch (Exception e) {
-            Log.logException(e);
-        	return DateFormatter822.format(new Date());
-        }
-    }
-    
     public int adminAuthenticated(final RequestHeader requestHeader) {
         
         // authorization for localhost, only if flag is set to grant localhost access as admin

source/de/anomic/yacy/yacyClient.java

@@ -517,7 +517,8 @@ public final class yacyClient {
         
         // send request
         try {
-            final byte[] result = wput(urlBase, uri.getHost(), post, (int) timeout); 
+            final byte[] result = wput(urlBase, uri.getHost(), post, (int) timeout);
+            //String debug = new String(result); System.out.println("*** DEBUG: " + debug);
             final RSSReader reader = RSSReader.parse(result);
             if (reader == null) {
                 yacyCore.log.logWarning("yacyClient.search failed asking peer '" + uri.getHost() + "': probably bad response from remote peer (1), reader == null");

source/net/yacy/document/content/RSSMessage.java

@@ -161,4 +161,8 @@ public class RSSMessage {
         for (String s: map.values()) sb.append(s).append(" ");
         return sb.toString();
     }
+    
+    public String toString() {
+        return this.map.toString();
+    }
 }

source/net/yacy/document/geolocalization/OpenGeoDB.java

@@ -173,20 +173,22 @@ public class OpenGeoDB {
      * @param anyname
      * @return
      */
-    public HashSet<Location> find(String anyname, boolean exact) {
+    public HashSet<Location> find(String anyname, boolean location, boolean locationexact, boolean kfz, boolean predial, boolean zip) {
         HashSet<Integer> r = new HashSet<Integer>();
         List<Integer> c;
-        if (exact) {
-            c = this.locationName2ids.get(anyname); if (c != null) r.addAll(c);
-        } else {
-            SortedMap<String, List<Integer>> cities = this.locationName2ids.tailMap(anyname);
-            for (Map.Entry<String, List<Integer>> e: cities.entrySet()) {
-            	if (e.getKey().toLowerCase().startsWith(anyname.toLowerCase())) r.addAll(e.getValue()); else break;
+        if (location) {
+            if (locationexact) {
+                c = this.locationName2ids.get(anyname); if (c != null) r.addAll(c);
+            } else {
+                SortedMap<String, List<Integer>> cities = this.locationName2ids.tailMap(anyname);
+                for (Map.Entry<String, List<Integer>> e: cities.entrySet()) {
+                	if (e.getKey().toLowerCase().startsWith(anyname.toLowerCase())) r.addAll(e.getValue()); else break;
+                }
             }
         }
-        c = this.kfz2ids.get(anyname); if (c != null) r.addAll(c);
-        c = this.predial2ids.get(anyname); if (c != null) r.addAll(c);
-        Integer i = this.zip2id.get(anyname); if (i != null) r.add(i);
+        if (kfz) {c = this.kfz2ids.get(anyname); if (c != null) r.addAll(c);}
+        if (predial) {c = this.predial2ids.get(anyname); if (c != null) r.addAll(c);}
+        if (zip) {Integer i = this.zip2id.get(anyname); if (i != null) r.add(i);}
         HashSet<Location> a = new HashSet<Location>();
         for (Integer e: r) {
             Location w = this.id2loc.get(e);

source/net/yacy/kelondro/util/DateFormatter.java

@@ -45,6 +45,7 @@ public final class DateFormatter {
     
     /** default HTTP 1.1 header date format pattern */
     public static final String PATTERN_RFC1123 = "EEE, dd MMM yyyy HH:mm:ss Z"; // with numeric time zone indicator as defined in RFC5322
+    public static final String PATTERN_RFC1123_SHORT = "EEE, dd MMM yyyy";
     
     /** date pattern used in older HTTP implementations */
     public static final String PATTERN_ANSIC   = "EEE MMM d HH:mm:ss yyyy";
@@ -74,7 +75,9 @@ public final class DateFormatter {
     private static final SimpleDateFormat FORMAT_RFC1123      = new SimpleDateFormat(PATTERN_RFC1123, Locale.US);
     private static final SimpleDateFormat FORMAT_RFC1036      = new SimpleDateFormat(PATTERN_RFC1036, Locale.US); 
     private static final SimpleDateFormat FORMAT_ANSIC        = new SimpleDateFormat(PATTERN_ANSIC, Locale.US); 
-
+    
+    private static final SimpleDateFormat FORMAT_RFC1123_SHORT = new SimpleDateFormat(PATTERN_RFC1123_SHORT, Locale.US);
+    
     /**
      * RFC 2616 requires that HTTP clients are able to parse all 3 different
      * formats. All times MUST be in GMT/UTC, but ...
@@ -161,7 +164,13 @@ public final class DateFormatter {
         lastRFC1123string = s;
         return s;
     }
-
+    
+    public static String formatRFC1123_short(final Date date) {
+        if (date == null) return "";
+        return FORMAT_RFC1123_SHORT.format(date);
+    }
+    
+    
     /**
      * Parse dates as defined in {@linkplain http://www.w3.org/TR/NOTE-datetime}.
      * This format (also specified in ISO8601) allows different "precisions".