refactoring of tables data organisation

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@6644 6c8d7289-2bf4-0310-a012-ef5d649a1542

htroot/Blacklist_p.java

@@ -43,7 +43,7 @@ import net.yacy.kelondro.logging.Log;
 import net.yacy.kelondro.util.FileUtils;
 import net.yacy.repository.Blacklist;
 
-import de.anomic.data.Tables;
+import de.anomic.data.WorkTables;
 import de.anomic.data.listManager;
 import de.anomic.http.server.RequestHeader;
 import de.anomic.search.SearchEventCache;
@@ -254,7 +254,7 @@ public class Blacklist_p {
                 String blentry = post.get("newEntry", "").trim();
                 
                 // store this call as api call
-                listManager.switchboard.tables.recordAPICall(post, "Blacklist_p.html", Tables.API_TYPE_CONFIGURATION, "add to blacklist: " + blentry);
+                listManager.switchboard.tables.recordAPICall(post, "Blacklist_p.html", WorkTables.TABLE_API_TYPE_CONFIGURATION, "add to blacklist: " + blentry);
                 
                 final String temp = addBlacklistEntry(blacklistToUse, blentry, header, supportedBlacklistTypes);
                 if (temp != null) {

htroot/ConfigBasic.java

@@ -34,7 +34,7 @@ import java.util.regex.Pattern;
 import net.yacy.kelondro.util.Domains;
 import net.yacy.kelondro.workflow.InstantBusyThread;
 
-import de.anomic.data.Tables;
+import de.anomic.data.WorkTables;
 import de.anomic.data.translator;
 import de.anomic.http.server.HTTPDemon;
 import de.anomic.http.server.HTTPDFileHandler;
@@ -73,7 +73,7 @@ public class ConfigBasic {
         
         // store this call as api call
         if (post != null && post.containsKey("set")) {
-            sb.tables.recordAPICall(post, "ConfigBasic.html", Tables.API_TYPE_CONFIGURATION, "basic settings");
+            sb.tables.recordAPICall(post, "ConfigBasic.html", WorkTables.TABLE_API_TYPE_CONFIGURATION, "basic settings");
         }
         
         //boolean doPeerPing = false;

htroot/ConfigLanguage_p.java

@@ -42,7 +42,7 @@ import net.yacy.kelondro.data.meta.DigestURI;
 import net.yacy.kelondro.util.FileUtils;
 
 import de.anomic.crawler.retrieval.HTTPLoader;
-import de.anomic.data.Tables;
+import de.anomic.data.WorkTables;
 import de.anomic.data.translator;
 import de.anomic.http.client.Client;
 import de.anomic.http.server.HeaderFramework;
@@ -75,7 +75,7 @@ public class ConfigLanguage_p {
             String selectedLanguage = post.get("language");
 
             // store this call as api call
-            ((Switchboard) env).tables.recordAPICall(post, "ConfigLanguage.html", Tables.API_TYPE_CONFIGURATION, "language settings: " + selectedLanguage);
+            ((Switchboard) env).tables.recordAPICall(post, "ConfigLanguage.html", WorkTables.TABLE_API_TYPE_CONFIGURATION, "language settings: " + selectedLanguage);
             
             //change language
             if(post.containsKey("use_button") && selectedLanguage != null){

htroot/ConfigNetwork_p.java

@@ -32,7 +32,7 @@ import net.yacy.kelondro.util.FileUtils;
 import net.yacy.kelondro.util.MapTools;
 import net.yacy.kelondro.workflow.BusyThread;
 
-import de.anomic.data.Tables;
+import de.anomic.data.WorkTables;
 import de.anomic.http.server.HTTPDemon;
 import de.anomic.http.server.RequestHeader;
 import de.anomic.search.Switchboard;
@@ -56,7 +56,7 @@ public class ConfigNetwork_p {
         if (post != null) {
             
             // store this call as api call
-            sb.tables.recordAPICall(post, "ConfigNetwork.html", Tables.API_TYPE_CONFIGURATION, "network settings");
+            sb.tables.recordAPICall(post, "ConfigNetwork.html", WorkTables.TABLE_API_TYPE_CONFIGURATION, "network settings");
             
             if (post.containsKey("changeNetwork")) {
                 final String networkDefinition = post.get("networkDefinition", "defaults/yacy.network.freeworld.unit");

htroot/ConfigPortal.java

@@ -25,7 +25,7 @@
 // along with this program; if not, write to the Free Software
 // Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 
-import de.anomic.data.Tables;
+import de.anomic.data.WorkTables;
 import de.anomic.http.server.RequestHeader;
 import de.anomic.search.Switchboard;
 import de.anomic.search.SwitchboardConstants;
@@ -60,7 +60,7 @@ public class ConfigPortal {
             if (post.containsKey("searchpage_set")) {
                 String newGreeting = post.get(SwitchboardConstants.GREETING, "");
                 // store this call as api call
-                sb.tables.recordAPICall(post, "ConfigPortal.html", Tables.API_TYPE_CONFIGURATION, "new portal design. greeting: " + newGreeting);
+                sb.tables.recordAPICall(post, "ConfigPortal.html", WorkTables.TABLE_API_TYPE_CONFIGURATION, "new portal design. greeting: " + newGreeting);
                 
                 sb.setConfig(SwitchboardConstants.GREETING, newGreeting);
                 sb.setConfig(SwitchboardConstants.GREETING_HOMEPAGE, post.get(SwitchboardConstants.GREETING_HOMEPAGE, ""));

htroot/CrawlMonitorRemoteStart.java

@@ -28,7 +28,6 @@ import java.util.Iterator;
 
 import de.anomic.http.server.RequestHeader;
 import de.anomic.search.Switchboard;
-import de.anomic.search.SwitchboardConstants;
 import de.anomic.server.serverObjects;
 import de.anomic.server.serverSwitch;
 import de.anomic.yacy.yacyNewsPool;

htroot/Crawler_p.java

@@ -46,7 +46,7 @@ import de.anomic.crawler.CrawlProfile;
 import de.anomic.crawler.SitemapImporter;
 import de.anomic.crawler.retrieval.Request;
 import de.anomic.data.BookmarkHelper;
-import de.anomic.data.Tables;
+import de.anomic.data.WorkTables;
 import de.anomic.data.bookmarksDB;
 import de.anomic.data.listManager;
 import de.anomic.http.server.RequestHeader;
@@ -143,7 +143,7 @@ public class Crawler_p {
                     crawlingStart = (crawlingStartURL == null) ? null : crawlingStartURL.toNormalform(true, true);
                    
                     // store this call as api call
-                    sb.tables.recordAPICall(post, "Crawler_p.html", Tables.API_TYPE_CRAWLER, "crawl start for " + crawlingStart);
+                    sb.tables.recordAPICall(post, "Crawler_p.html", WorkTables.TABLE_API_TYPE_CRAWLER, "crawl start for " + crawlingStart);
                     
                     // set new properties
                     final boolean fullDomain = post.get("range", "wide").equals("domain"); // special property in simple crawl start

htroot/IndexCreateDomainCrawl_p.java

@@ -24,22 +24,16 @@
 // along with this program; if not, write to the Free Software
 // Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 
-import java.util.Iterator;
-
 import de.anomic.http.server.RequestHeader;
-import de.anomic.search.Switchboard;
 import de.anomic.search.SwitchboardConstants;
 import de.anomic.server.serverObjects;
 import de.anomic.server.serverSwitch;
-import de.anomic.yacy.yacyNewsPool;
-import de.anomic.yacy.yacyNewsRecord;
-import de.anomic.yacy.yacySeed;
 
 public class IndexCreateDomainCrawl_p {
     
     public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) {
         // return variable that accumulates replacements
-        Switchboard sb = (Switchboard) env;
+        //Switchboard sb = (Switchboard) env;
         serverObjects prop = new serverObjects();
         
         // define visible variables

htroot/Table_API_p.java

@@ -22,9 +22,10 @@ import java.util.Iterator;
 import java.util.LinkedHashMap;
 import java.util.Map;
 
+import net.yacy.kelondro.blob.Tables;
 import net.yacy.kelondro.logging.Log;
 
-import de.anomic.data.Tables;
+import de.anomic.data.WorkTables;
 import de.anomic.http.client.Client;
 import de.anomic.http.server.RequestHeader;
 import de.anomic.http.server.ResponseContainer;
@@ -41,7 +42,11 @@ public class Table_API_p {
         if (post != null && post.get("deleterows", "").length() > 0) {
             for (Map.Entry<String, String> entry: post.entrySet()) {
                 if (entry.getKey().startsWith("mark_") && entry.getValue().equals("on")) {
-                    sb.tables.delete(Tables.API_TABLENAME, entry.getKey().substring(5).getBytes());
+                    try {
+                        sb.tables.delete(WorkTables.TABLE_API_NAME, entry.getKey().substring(5).getBytes());
+                    } catch (IOException e) {
+                        Log.logException(e);
+                    }
                 }
             }
         }
@@ -56,9 +61,9 @@ public class Table_API_p {
             LinkedHashMap<String, Integer> l = new LinkedHashMap<String, Integer>();
             for (Map.Entry<String, String> entry: post.entrySet()) {
                 if (entry.getKey().startsWith("mark_") && entry.getValue().equals("on")) {
-                    Map<String, byte[]> map = sb.tables.select(Tables.API_TABLENAME, entry.getKey().substring(5).getBytes());
-                    String url = "http://localhost:" + sb.getConfig("port", "8080") + new String(map.get(Tables.API_COL_URL));
                     try {
+                        Tables.Row row = sb.tables.select(WorkTables.TABLE_API_NAME, entry.getKey().substring(5).getBytes());
+                        String url = "http://localhost:" + sb.getConfig("port", "8080") + new String(row.from(WorkTables.TABLE_API_COL_URL));
                         result = client.GET(url);
                         l.put(url, result.getStatusCode());
                     } catch (IOException e) {
@@ -88,23 +93,31 @@ public class Table_API_p {
         prop.put("showtable", 1);
         
         // insert rows
-        final int maxCount = Math.min(1000, sb.tables.size(Tables.API_TABLENAME));
+        int maxCount;
-        final Iterator<Map.Entry<byte[], Map<String, byte[]>>> mapIterator = sb.tables.iterator(Tables.API_TABLENAME);
+        try {
-        Map.Entry<byte[], Map<String, byte[]>> record;
+            maxCount = Math.min(1000, sb.tables.size(WorkTables.TABLE_API_NAME));
-        Map<String, byte[]> map;
+        } catch (IOException e) {
+            Log.logException(e);
+            maxCount = 0;
+        }
         int count = 0;
-        boolean dark = true;
+        try {
-        while ((mapIterator.hasNext()) && (count < maxCount)) {
+            final Iterator<Tables.Row> mapIterator = sb.tables.iterator(WorkTables.TABLE_API_NAME);
-            record = mapIterator.next();
+            Tables.Row row;
-            if (record == null) continue;
+            boolean dark = true;
-            map = record.getValue();
+            while ((mapIterator.hasNext()) && (count < maxCount)) {
-            prop.put("showtable_list_" + count + "_dark", ((dark) ? 1 : 0) ); dark=!dark;
+                row = mapIterator.next();
-            prop.put("showtable_list_" + count + "_pk", new String(record.getKey()));
+                if (row == null) continue;
-            prop.put("showtable_list_" + count + "_date", map.get(Tables.API_COL_DATE));
+                prop.put("showtable_list_" + count + "_dark", ((dark) ? 1 : 0) ); dark=!dark;
-            prop.put("showtable_list_" + count + "_type", map.get(Tables.API_COL_TYPE));
+                prop.put("showtable_list_" + count + "_pk", new String(row.getPK()));
-            prop.put("showtable_list_" + count + "_comment", map.get(Tables.API_COL_COMMENT));
+                prop.put("showtable_list_" + count + "_date", row.from(WorkTables.TABLE_API_COL_DATE));
-            prop.put("showtable_list_" + count + "_url", "http://" + sb.myPublicIP() + ":" + sb.getConfig("port", "8080") + new String(map.get(Tables.API_COL_URL)));
+                prop.put("showtable_list_" + count + "_type", row.from(WorkTables.TABLE_API_COL_TYPE));
-            count++;
+                prop.put("showtable_list_" + count + "_comment", row.from(WorkTables.TABLE_API_COL_COMMENT));
+                prop.put("showtable_list_" + count + "_url", "http://" + sb.myPublicIP() + ":" + sb.getConfig("port", "8080") + new String(row.from(WorkTables.TABLE_API_COL_URL)));
+                count++;
+            }
+        } catch (IOException e) {
+            Log.logException(e);
         }
         prop.put("showtable_list", count);
         

htroot/Tables_p.java

@@ -17,11 +17,16 @@
 // along with this program; if not, write to the Free Software
 // Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 
+import java.io.IOException;
+import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 
+import net.yacy.kelondro.blob.Tables;
+import net.yacy.kelondro.logging.Log;
+
 import de.anomic.http.server.RequestHeader;
 import de.anomic.search.Switchboard;
 import de.anomic.server.serverObjects;
@@ -33,7 +38,7 @@ public class Tables_p {
         final Switchboard sb = (Switchboard) env;
         final serverObjects prop = new serverObjects();
         String table = (post == null) ? null : post.get("table", null);
-        if (table != null && !sb.tables.has(table)) table = null;
+        if (table != null && !sb.tables.hasHeap(table)) table = null;
 
         // show table selection
         int count = 0;
@@ -48,19 +53,26 @@ public class Tables_p {
         prop.put("tables", count);
         
         List<String> columns = null;
-        if (table != null) {
+        if (table != null) try {
             columns = sb.tables.columns(table);
+        } catch (IOException e) {
+            Log.logException(e);
+            columns = new ArrayList<String>();
         }
         
         // apply deletion requests
-        if (post != null && post.get("deletetable", "").length() > 0) {
+        if (post != null && post.get("deletetable", "").length() > 0) try {
             sb.tables.clear(table);
+        } catch (IOException e) {
+            Log.logException(e);
         }
         
         if (post != null && post.get("deleterows", "").length() > 0) {
             for (Map.Entry<String, String> entry: post.entrySet()) {
-                if (entry.getKey().startsWith("mark_") && entry.getValue().equals("on")) {
+                if (entry.getKey().startsWith("mark_") && entry.getValue().equals("on")) try {
                     sb.tables.delete(table, entry.getKey().substring(5).getBytes());
+                } catch (IOException e) {
+                    Log.logException(e);
                 }
             }
         }
@@ -73,7 +85,11 @@ public class Tables_p {
                     map.put(entry.getKey().substring(4), entry.getValue().getBytes());
                 }
             }
-            sb.tables.insert(table, pk.getBytes(), map);
+            try {
+                sb.tables.insert(table, pk.getBytes(), map);
+            } catch (IOException e) {
+                Log.logException(e);
+            }
         }
         
         // generate table
@@ -92,28 +108,34 @@ public class Tables_p {
             prop.put("showtable_columns", columns.size());
             
             // insert all rows
-            final int maxCount = Math.min(1000, sb.tables.size(table));
+            int maxCount;
-            final Iterator<Map.Entry<byte[], Map<String, byte[]>>> mapIterator = sb.tables.iterator(table);
+            try {
-            Map.Entry<byte[], Map<String, byte[]>> record;
+                maxCount = Math.min(1000, sb.tables.size(table));
-            Map<String, byte[]> map;
+            } catch (IOException e) {
-            byte[] pk;
+                Log.logException(e);
+                maxCount = 0;
+            }
             count = 0;
-            boolean dark = true;
+            try {
-            byte[] cell;
+                final Iterator<Tables.Row> mapIterator = sb.tables.iterator(table);
-            while ((mapIterator.hasNext()) && (count < maxCount)) {
+                Tables.Row row;
-                record = mapIterator.next();
+                boolean dark = true;
-                if (record == null) continue;
+                byte[] cell;
-                pk = record.getKey();
+                while ((mapIterator.hasNext()) && (count < maxCount)) {
-                map = record.getValue();
+                    row = mapIterator.next();
-                prop.put("showtable_list_" + count + "_dark", ((dark) ? 1 : 0) ); dark=!dark;
+                    if (row == null) continue;
-                prop.put("showtable_list_" + count + "_pk",  new String(pk));
+                    prop.put("showtable_list_" + count + "_dark", ((dark) ? 1 : 0) ); dark=!dark;
-                for (int i = 0; i < columns.size(); i++) {
+                    prop.put("showtable_list_" + count + "_pk",  new String(row.getPK()));
-                    cell = map.get(columns.get(i));
+                    for (int i = 0; i < columns.size(); i++) {
-                    prop.putHTML("showtable_list_" + count + "_columns_" + i + "_cell", cell == null ? "" : new String(cell));
+                        cell = row.from(columns.get(i));
+                        prop.putHTML("showtable_list_" + count + "_columns_" + i + "_cell", cell == null ? "" : new String(cell));
+                    }
+                    prop.put("showtable_list_" + count + "_columns", columns.size());
+                    
+                    count++;
                 }
-                prop.put("showtable_list_" + count + "_columns", columns.size());
+            } catch (IOException e) {
-                
+                Log.logException(e);
-                count++;
             }
             prop.put("showtable_list", count);
         }
@@ -127,15 +149,21 @@ public class Tables_p {
                     break;
                 }
             }
-            if (pk != null && sb.tables.has(table, pk.getBytes())) {
+            try {
-                setEdit(sb, prop, table, pk, columns);
+                if (pk != null && sb.tables.has(table, pk.getBytes())) {
+                    setEdit(sb, prop, table, pk, columns);
+                }
+            } catch (IOException e) {
+                Log.logException(e);
             }
         }
         
-        if (post != null && table != null && post.containsKey("addrow")) {
+        if (post != null && table != null && post.containsKey("addrow")) try {
             // get a new key
-            String pk = sb.tables.createRow(table);
+            String pk = new String(sb.tables.createRow(table));
             setEdit(sb, prop, table, pk, columns);
+        } catch (IOException e) {
+            Log.logException(e);
         }
         
         // adding the peer address
@@ -145,15 +173,15 @@ public class Tables_p {
         return prop;
     }
     
-    private static void setEdit(final Switchboard sb, final serverObjects prop, final String table, final String pk, List<String> columns) {
+    private static void setEdit(final Switchboard sb, final serverObjects prop, final String table, final String pk, List<String> columns) throws IOException {
         prop.put("showedit", 1);
         prop.put("showedit_table", table);
         prop.put("showedit_pk", pk);
-        Map<String, byte[]> map = sb.tables.select(table, pk.getBytes());
+        Tables.Row row = sb.tables.select(table, pk.getBytes());
         int count = 0;
         byte[] cell;
         for (String col: columns) {
-            cell = map.get(col);
+            cell = row.from(col);
             prop.put("showedit_list_" + count + "_key", col);
             prop.put("showedit_list_" + count + "_value", cell == null ? "" : new String(cell));
             count++;

source/de/anomic/data/Tables.java

@@ -1,151 +0,0 @@
-package de.anomic.data;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Date;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.TreeMap;
-
-import net.yacy.kelondro.blob.BEncodedHeapArray;
-import net.yacy.kelondro.index.RowSpaceExceededException;
-import net.yacy.kelondro.logging.Log;
-import net.yacy.kelondro.util.DateFormatter;
-import de.anomic.server.serverObjects;
-
-public class Tables {
-
-    public final static String API_TABLENAME = "api";
-    public final static String API_TYPE_STEERING = "steering";
-    public final static String API_TYPE_CONFIGURATION = "configuration";
-    public final static String API_TYPE_CRAWLER = "crawler";
-    
-    public final static String API_COL_TYPE = "type";
-    public final static String API_COL_COMMENT = "comment";
-    public final static String API_COL_DATE = "date";
-    public final static String API_COL_URL = "url";
-    
-    private BEncodedHeapArray tables;
-    
-    public Tables(File workPath) {
-        this.tables = new BEncodedHeapArray(workPath, 12);
-    }
-    
-    public boolean has(String table) {
-        return tables.hasHeap(table);
-    }
-    
-    public boolean has(String table, byte[] pk) {
-        try {
-            return tables.has(table, pk);
-        } catch (IOException e) {
-            Log.logException(e);
-            return false;
-        }
-    }
-    
-    public Iterator<String> tables() {
-        return this.tables.tables();
-    }
-    
-    public List<String> columns(String table) {
-        try {
-            return this.tables.columns(table);
-        } catch (IOException e) {
-            Log.logException(e);
-            return new ArrayList<String>(0);
-        }
-    }
-    
-    public void clear(String table) {
-        try {
-            this.tables.clear(table);
-        } catch (IOException e) {
-            Log.logException(e);
-        }
-    }
-    
-    public void delete(String table, byte[] pk) {
-        try {
-            this.tables.delete(table, pk);
-        } catch (IOException e) {
-            Log.logException(e);
-        }
-    }
-    
-    public int size(String table) {
-        try {
-            return this.tables.size(table);
-        } catch (IOException e) {
-            Log.logException(e);
-            return 0;
-        }
-    }
-    
-    public Iterator<Map.Entry<byte[], Map<String, byte[]>>> iterator(String table) {
-        try {
-            return this.tables.iterator(table);
-        } catch (IOException e) {
-            Log.logException(e);
-            return new TreeMap<byte[], Map<String, byte[]>>().entrySet().iterator();
-        }
-    }
-    
-    public void close() {
-        this.tables.close();
-    }
-    
-    public Map<String, byte[]> select(String table, byte[] pk) {
-        try {
-            return tables.select(table, pk);
-        } catch (IOException e) {
-            Log.logException(e);
-            return new TreeMap<String, byte[]>();
-        }
-    }
-    
-    public void insert(String table, byte[] pk, Map<String, byte[]> map) {
-        try {
-            this.tables.insert(table, pk, map);
-        } catch (RowSpaceExceededException e) {
-            Log.logException(e);
-        } catch (IOException e) {
-            Log.logException(e);
-        }
-    }
-    
-    public String createRow(String table) {
-        try {
-            return new String(this.tables.insert(table, new HashMap<String, byte[]>()));
-        } catch (RowSpaceExceededException e) {
-            Log.logException(e);
-            return null;
-        } catch (IOException e) {
-            Log.logException(e);
-            return null;
-        }
-    }
-    
-    public void recordAPICall(final serverObjects post, final String servletName, String type, String comment) {
-        String apiurl = /*"http://localhost:" + getConfig("port", "8080") +*/ "/" + servletName + "?" + post.toString();
-        try {
-            this.tables.insert(
-                    API_TABLENAME,
-                    API_COL_TYPE, type.getBytes(),
-                    API_COL_COMMENT, comment.getBytes(),
-                    API_COL_DATE, DateFormatter.formatShortMilliSecond(new Date()).getBytes(),
-                    API_COL_URL, apiurl.getBytes()
-                    );
-        } catch (RowSpaceExceededException e2) {
-            Log.logException(e2);
-        } catch (IOException e2) {
-            Log.logException(e2);
-        }
-        Log.logInfo("APICALL", apiurl);
-    }
-    
-    
-}

source/de/anomic/data/WorkTables.java

@@ -0,0 +1,71 @@
+// Work.java
+// (C) 2010 by Michael Peter Christen; mc@yacy.net, Frankfurt a. M., Germany
+// first published 04.02.2010 on http://yacy.net
+//
+// This is a part of YaCy, a peer-to-peer based web search engine
+//
+// $LastChangedDate: 2006-04-02 22:40:07 +0200 (So, 02 Apr 2006) $
+// $LastChangedRevision: 6539 $
+// $LastChangedBy: low012 $
+//
+// LICENSE
+// 
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program; if not, write to the Free Software
+// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+package de.anomic.data;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.Date;
+
+import net.yacy.kelondro.blob.Tables;
+import net.yacy.kelondro.logging.Log;
+import net.yacy.kelondro.util.DateFormatter;
+import de.anomic.server.serverObjects;
+
+public class WorkTables extends Tables {
+
+    public final static String TABLE_API_NAME = "api";
+    public final static String TABLE_API_TYPE_STEERING = "steering";
+    public final static String TABLE_API_TYPE_CONFIGURATION = "configuration";
+    public final static String TABLE_API_TYPE_CRAWLER = "crawler";
+    
+    public final static String TABLE_API_COL_TYPE = "type";
+    public final static String TABLE_API_COL_COMMENT = "comment";
+    public final static String TABLE_API_COL_DATE = "date";
+    public final static String TABLE_API_COL_URL = "url";
+    
+    
+    public WorkTables(File workPath) {
+        super(workPath, 12);
+    }
+    
+    public void recordAPICall(final serverObjects post, final String servletName, String type, String comment) {
+        String apiurl = /*"http://localhost:" + getConfig("port", "8080") +*/ "/" + servletName + "?" + post.toString();
+        try {
+            super.insert(
+                    TABLE_API_NAME,
+                    TABLE_API_COL_TYPE, type.getBytes(),
+                    TABLE_API_COL_COMMENT, comment.getBytes(),
+                    TABLE_API_COL_DATE, DateFormatter.formatShortMilliSecond(new Date()).getBytes(),
+                    TABLE_API_COL_URL, apiurl.getBytes()
+                    );
+        } catch (IOException e) {
+            Log.logException(e);
+        }
+        Log.logInfo("APICALL", apiurl);
+    }
+    
+}

source/net/yacy/kelondro/blob/HeapReader.java

@@ -41,6 +41,7 @@ import net.yacy.kelondro.io.Writer;
 import net.yacy.kelondro.logging.Log;
 import net.yacy.kelondro.order.ByteOrder;
 import net.yacy.kelondro.order.CloneableIterator;
+import net.yacy.kelondro.order.NaturalOrder;
 import net.yacy.kelondro.order.RotateIterator;
 import net.yacy.kelondro.util.FileUtils;
 import net.yacy.kelondro.util.MemoryControl;
@@ -112,10 +113,12 @@ public class HeapReader {
     }
 
     protected byte[] normalizeKey(byte[] key) {
+        // check size of key: zero-filled keys are only possible of the ordering is
+        // an instance of the natural ordering. Base64-orderings cannot use zeros in keys.
+        assert key.length >= this.keylength || this.ordering instanceof NaturalOrder;
         return normalizeKey(key, this.keylength);
     }
     
-    
     private static final byte zero = 0;
     
     protected static byte[] normalizeKey(byte[] key, int keylength) {

source/net/yacy/kelondro/blob/Tables.java

@@ -1,4 +1,4 @@
-// BEncodedHeapArray.java
+// Tables.java
 // (C) 2010 by Michael Peter Christen; mc@yacy.net, Frankfurt a. M., Germany
 // first published 14.01.2010 on http://yacy.net
 //
@@ -37,10 +37,12 @@ import java.util.concurrent.ConcurrentHashMap;
 
 import net.yacy.kelondro.index.RowSpaceExceededException;
 import net.yacy.kelondro.logging.Log;
+import net.yacy.kelondro.util.ByteBuffer;
 import net.yacy.kelondro.util.FileUtils;
+import net.yacy.kelondro.util.LookAheadIterator;
 
 
-public class BEncodedHeapArray {
+public class Tables {
  
     private static final String suffix = ".bheap";
     private static final String system_table_pkcounter = "pkcounter";
@@ -50,7 +52,7 @@ public class BEncodedHeapArray {
     private ConcurrentHashMap<String, BEncodedHeap> tables;
     private int keymaxlen;
     
-    public BEncodedHeapArray(final File location, final int keymaxlen) {
+    public Tables(final File location, final int keymaxlen) {
         this.location = new File(location.getAbsolutePath());
         if (!this.location.exists()) this.location.mkdirs();
         this.keymaxlen = keymaxlen;
@@ -126,7 +128,7 @@ public class BEncodedHeapArray {
     }
     
     private byte[] ukey(String tablename) throws IOException {
-        byte[] pk = select(system_table_pkcounter, tablename.getBytes(), system_table_pkcounter_counterName);
+        byte[] pk = select(system_table_pkcounter, tablename.getBytes()).from(system_table_pkcounter_counterName);
         int pki;
         if (pk == null) {
             pki = size(tablename);
@@ -156,14 +158,14 @@ public class BEncodedHeapArray {
      * @throws RowSpaceExceededException
      * @throws IOException
      */
-    public byte[] insert(final String tablename, Map<String, byte[]> map) throws RowSpaceExceededException, IOException {
+    public byte[] insert(final String tablename, Map<String, byte[]> map) throws IOException {
         byte[] uk = ukey(tablename);
         insert(tablename, uk, map);
         insert(system_table_pkcounter, tablename.getBytes(), system_table_pkcounter_counterName, uk);
         return uk;
     }
 
-    public byte[] insert(final String tablename, String key, byte[] value) throws RowSpaceExceededException, IOException {
+    public byte[] insert(final String tablename, String key, byte[] value) throws IOException {
         byte[] uk = ukey(tablename);
         insert(tablename, uk, key, value);
         insert(system_table_pkcounter, tablename.getBytes(), system_table_pkcounter_counterName, uk);
@@ -173,12 +175,12 @@ public class BEncodedHeapArray {
     public byte[] insert(final String tablename,
             String key0, byte[] value0,
             String key1, byte[] value1
-            ) throws RowSpaceExceededException, IOException {
+            ) throws IOException {
         byte[] uk = ukey(tablename);
         insert(tablename, uk,
-                key0, value0,
+            key0, value0,
-                key1, value1
+            key1, value1
-                );
+            );
         insert(system_table_pkcounter, tablename.getBytes(), system_table_pkcounter_counterName, uk);
         return uk;
     }
@@ -187,13 +189,13 @@ public class BEncodedHeapArray {
             String key0, byte[] value0,
             String key1, byte[] value1,
             String key2, byte[] value2
-            ) throws RowSpaceExceededException, IOException {
+            ) throws IOException {
         byte[] uk = ukey(tablename);
         insert(tablename, uk,
-                key0, value0,
+            key0, value0,
-                key1, value1,
+            key1, value1,
-                key2, value2
+            key2, value2
-                );
+            );
         insert(system_table_pkcounter, tablename.getBytes(), system_table_pkcounter_counterName, uk);
         return uk;
     }
@@ -203,47 +205,59 @@ public class BEncodedHeapArray {
             String key1, byte[] value1,
             String key2, byte[] value2,
             String key3, byte[] value3
-            ) throws RowSpaceExceededException, IOException {
+            ) throws IOException {
         byte[] uk = ukey(tablename);
         insert(tablename, uk,
-                key0, value0,
+            key0, value0,
-                key1, value1,
+            key1, value1,
-                key2, value2,
+            key2, value2,
-                key3, value3
+            key3, value3
-                );
+            );
         insert(system_table_pkcounter, tablename.getBytes(), system_table_pkcounter_counterName, uk);
         return uk;
     }
 
     public void insert(final String table, byte[] pk,
             String key, byte[] value
-            ) throws RowSpaceExceededException, IOException {
+            ) throws IOException {
         BEncodedHeap heap = getHeap(table);
-        heap.put(pk, key, value);
+        try {
+            heap.put(pk, key, value);
+        } catch (RowSpaceExceededException e) {
+            throw new IOException(e.getMessage());
+        }
     }
 
     public void insert(final String table, byte[] pk,
             String key0, byte[] value0,
             String key1, byte[] value1
-            ) throws RowSpaceExceededException, IOException {
+            ) throws IOException {
         BEncodedHeap heap = getHeap(table);
-        heap.put(pk,
+        try {
+            heap.put(pk,
                 key0, value0,
                 key1, value1
                 );
+        } catch (RowSpaceExceededException e) {
+            throw new IOException(e.getMessage());
+        }
     }
     
     public void insert(final String table, byte[] pk,
             String key0, byte[] value0,
             String key1, byte[] value1,
             String key2, byte[] value2
-            ) throws RowSpaceExceededException, IOException {
+            ) throws IOException {
         BEncodedHeap heap = getHeap(table);
-        heap.put(pk,
+        try {
+            heap.put(pk,
                 key0, value0,
                 key1, value1,
                 key2, value2
                 );
+        } catch (RowSpaceExceededException e) {
+            throw new IOException(e.getMessage());
+        }
     }
     
     public void insert(final String table, byte[] pk,
@@ -251,29 +265,45 @@ public class BEncodedHeapArray {
             String key1, byte[] value1,
             String key2, byte[] value2,
             String key3, byte[] value3
-            ) throws RowSpaceExceededException, IOException {
+            ) throws IOException {
         BEncodedHeap heap = getHeap(table);
-        heap.put(pk,
+        try {
+            heap.put(pk,
                 key0, value0,
                 key1, value1,
                 key2, value2,
                 key3, value3
                 );
+        } catch (RowSpaceExceededException e) {
+            throw new IOException(e.getMessage());
+        }
     }
 
-    public void insert(final String table, byte[] pk, Map<String, byte[]> map) throws RowSpaceExceededException, IOException {
+    public void insert(final String table, byte[] pk, Map<String, byte[]> map) throws IOException {
         BEncodedHeap heap = getHeap(table);
-        heap.put(pk, map);
+        try {
+            heap.put(pk, map);
+        } catch (RowSpaceExceededException e) {
+            throw new IOException(e.getMessage());
+        }
     }
 
-    public Map<String, byte[]> select(final String table, byte[] pk) throws IOException {
+    public void insert(final String table, Row row) throws IOException {
         BEncodedHeap heap = getHeap(table);
-        return heap.get(pk);
+        try {
+            heap.put(row.pk, row.map);
+        } catch (RowSpaceExceededException e) {
+            throw new IOException(e.getMessage());
+        }
     }
 
-    public byte[] select(final String table, byte[] pk, String key) throws IOException {
+    public byte[] createRow(String table) throws IOException {
+        return this.insert(table, new HashMap<String, byte[]>());
+    }
+    
+    public Row select(final String table, byte[] pk) throws IOException {
         BEncodedHeap heap = getHeap(table);
-        return heap.getProp(pk, key);
+        return new Row(pk, heap.get(pk));
     }
 
     public void delete(final String table, byte[] pk) throws IOException {
@@ -291,15 +321,79 @@ public class BEncodedHeapArray {
         return heap.keys();
     }
 
-    public Iterator<Map.Entry<byte[], Map<String, byte[]>>> iterator(String table) throws IOException {
+    public Iterator<Row> iterator(String table) throws IOException {
-        BEncodedHeap heap = getHeap(table);
+        return new RowIterator(table);
-        return heap.iterator();
+    }
+    
+    public Iterator<Row> iterator(String table, String whereKey, byte[] whereValue) throws IOException {
+        return new RowIterator(table, whereKey, whereValue);
     }
     
     public List<String> columns(String table) throws IOException {
         BEncodedHeap heap = getHeap(table);
         return heap.columns();
     }
+
+    public class RowIterator extends LookAheadIterator<Row> implements Iterator<Row> {
+
+        private final String whereKey;
+        private final byte[] whereValue;
+        private final Iterator<Map.Entry<byte[], Map<String, byte[]>>> i;
+        
+        public RowIterator(String table) throws IOException {
+            this.whereKey = null;
+            this.whereValue = null;
+            BEncodedHeap heap = getHeap(table);
+            i = heap.iterator();
+        }
+        
+        public RowIterator(String table, String whereKey, byte[] whereValue) throws IOException {
+            this.whereKey = whereKey;
+            this.whereValue = whereValue;
+            BEncodedHeap heap = getHeap(table);
+            i = heap.iterator();
+        }
+        
+        protected Row next0() {
+            while (i.hasNext()) {
+                Row r = new Row(i.next());
+                if (this.whereKey == null) return r;
+                if (ByteBuffer.equals(r.from(this.whereKey), this.whereValue)) return r;
+            }
+            return null;
+        }
+        
+    }
+    
+    public class Row {
+        
+        private final byte[] pk;
+        private final Map<String, byte[]> map;
+        
+        public Row(final Map.Entry<byte[], Map<String, byte[]>> entry) {
+            this.pk = entry.getKey();
+            this.map = entry.getValue();
+        }
+        
+        public Row(final byte[] pk, final Map<String, byte[]> map) {
+            this.pk = pk;
+            this.map = map;
+        }
+        
+        public byte[] getPK() {
+            return this.pk;
+        }
+        
+        public byte[] from(String colname) {
+            return this.map.get(colname);
+        }
+        
+        public String toString() {
+            StringBuilder sb = new StringBuilder(keymaxlen + 20 * map.size());
+            sb.append(new String(pk)).append(":").append(map.toString());
+            return sb.toString();
+        }
+    }
     
     public static void main(String[] args) {
         // test the class
@@ -307,7 +401,7 @@ public class BEncodedHeapArray {
         // System.out.println(f.getAbsolutePath());
         // System.out.println(f.getParent());
         try {
-            BEncodedHeapArray map = new BEncodedHeapArray(f.getParentFile(), 4);
+            Tables map = new Tables(f.getParentFile(), 4);
             // put some values into the map
             Map<String, byte[]> m = new HashMap<String, byte[]>();
             m.put("k", "000".getBytes());
@@ -317,18 +411,14 @@ public class BEncodedHeapArray {
             m.put("k", "222".getBytes());
             map.insert("testdao", "789".getBytes(), m);
             // iterate over keys
-            Iterator<Map.Entry<byte[], Map<String, byte[]>>> i = map.iterator("testdao");
+            Iterator<Row> i = map.iterator("testdao");
             while (i.hasNext()) {
-                Map.Entry<byte[], Map<String, byte[]>> entry = i.next();
+                System.out.println(i.next().toString());
-                System.out.println(new String(entry.getKey(), "UTF-8") + ": "
-                        + entry.getValue());
             }
             // clean up
             map.close();
         } catch (IOException e) {
             Log.logException(e);
-        } catch (RowSpaceExceededException e) {
-            Log.logException(e);
         }
     }
 }

source/net/yacy/kelondro/util/LookAheadIterator.java

@@ -0,0 +1,78 @@
+// LookAheadIterator.java
+// (C) 2010 by Michael Peter Christen; mc@yacy.net, Frankfurt a. M., Germany
+// first published 04.02.2010 on http://yacy.net
+//
+// This is a part of YaCy, a peer-to-peer based web search engine
+//
+// $LastChangedDate: 2006-04-02 22:40:07 +0200 (So, 02 Apr 2006) $
+// $LastChangedRevision: 6539 $
+// $LastChangedBy: low012 $
+//
+// LICENSE
+// 
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program; if not, write to the Free Software
+// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+package net.yacy.kelondro.util;
+
+import java.util.Iterator;
+
+/**
+ * convenience class for iterator implementations that naturally terminate
+ * when the next() method would return null. This is usually implemented using
+ * a next0() method and a wrapper for next() and hasNext() that evaluates the
+ * latest return value of next0()
+ * To use this class just implement the next0() method
+ */
+public abstract class LookAheadIterator<A> implements Iterator<A> {
+
+    private boolean fresh = true;
+    private A next = null;
+    
+    public LookAheadIterator() {
+    }
+    
+    /**
+     * the internal next-method
+     * @return a value of type A if available or null if no more value are available
+     */
+    protected abstract A next0() ;
+    
+    private final void checkInit() {
+        if (fresh) {
+            next = next0();
+            fresh = false;
+        }
+    }
+    
+    public final boolean hasNext() {
+        checkInit();
+        return next != null;
+    }
+
+    public final A next() {
+        checkInit();
+        A n = next;
+        next = next0();
+        return n;
+    }
+    
+    /**
+     * a remove is not possible with this implementation
+     */
+    public final void remove() {
+        throw new UnsupportedOperationException();
+    }
+    
+}