mirror of
https://github.com/yacy/yacy_search_server.git
synced 2024-09-19 00:01:41 +02:00
refactoring of tables data organisation
git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@6644 6c8d7289-2bf4-0310-a012-ef5d649a1542
This commit is contained in:
parent
85ca96227f
commit
308a973503
|
@ -43,7 +43,7 @@ import net.yacy.kelondro.logging.Log;
|
|||
import net.yacy.kelondro.util.FileUtils;
|
||||
import net.yacy.repository.Blacklist;
|
||||
|
||||
import de.anomic.data.Tables;
|
||||
import de.anomic.data.WorkTables;
|
||||
import de.anomic.data.listManager;
|
||||
import de.anomic.http.server.RequestHeader;
|
||||
import de.anomic.search.SearchEventCache;
|
||||
|
@ -254,7 +254,7 @@ public class Blacklist_p {
|
|||
String blentry = post.get("newEntry", "").trim();
|
||||
|
||||
// store this call as api call
|
||||
listManager.switchboard.tables.recordAPICall(post, "Blacklist_p.html", Tables.API_TYPE_CONFIGURATION, "add to blacklist: " + blentry);
|
||||
listManager.switchboard.tables.recordAPICall(post, "Blacklist_p.html", WorkTables.TABLE_API_TYPE_CONFIGURATION, "add to blacklist: " + blentry);
|
||||
|
||||
final String temp = addBlacklistEntry(blacklistToUse, blentry, header, supportedBlacklistTypes);
|
||||
if (temp != null) {
|
||||
|
|
|
@ -34,7 +34,7 @@ import java.util.regex.Pattern;
|
|||
import net.yacy.kelondro.util.Domains;
|
||||
import net.yacy.kelondro.workflow.InstantBusyThread;
|
||||
|
||||
import de.anomic.data.Tables;
|
||||
import de.anomic.data.WorkTables;
|
||||
import de.anomic.data.translator;
|
||||
import de.anomic.http.server.HTTPDemon;
|
||||
import de.anomic.http.server.HTTPDFileHandler;
|
||||
|
@ -73,7 +73,7 @@ public class ConfigBasic {
|
|||
|
||||
// store this call as api call
|
||||
if (post != null && post.containsKey("set")) {
|
||||
sb.tables.recordAPICall(post, "ConfigBasic.html", Tables.API_TYPE_CONFIGURATION, "basic settings");
|
||||
sb.tables.recordAPICall(post, "ConfigBasic.html", WorkTables.TABLE_API_TYPE_CONFIGURATION, "basic settings");
|
||||
}
|
||||
|
||||
//boolean doPeerPing = false;
|
||||
|
|
|
@ -42,7 +42,7 @@ import net.yacy.kelondro.data.meta.DigestURI;
|
|||
import net.yacy.kelondro.util.FileUtils;
|
||||
|
||||
import de.anomic.crawler.retrieval.HTTPLoader;
|
||||
import de.anomic.data.Tables;
|
||||
import de.anomic.data.WorkTables;
|
||||
import de.anomic.data.translator;
|
||||
import de.anomic.http.client.Client;
|
||||
import de.anomic.http.server.HeaderFramework;
|
||||
|
@ -75,7 +75,7 @@ public class ConfigLanguage_p {
|
|||
String selectedLanguage = post.get("language");
|
||||
|
||||
// store this call as api call
|
||||
((Switchboard) env).tables.recordAPICall(post, "ConfigLanguage.html", Tables.API_TYPE_CONFIGURATION, "language settings: " + selectedLanguage);
|
||||
((Switchboard) env).tables.recordAPICall(post, "ConfigLanguage.html", WorkTables.TABLE_API_TYPE_CONFIGURATION, "language settings: " + selectedLanguage);
|
||||
|
||||
//change language
|
||||
if(post.containsKey("use_button") && selectedLanguage != null){
|
||||
|
|
|
@ -32,7 +32,7 @@ import net.yacy.kelondro.util.FileUtils;
|
|||
import net.yacy.kelondro.util.MapTools;
|
||||
import net.yacy.kelondro.workflow.BusyThread;
|
||||
|
||||
import de.anomic.data.Tables;
|
||||
import de.anomic.data.WorkTables;
|
||||
import de.anomic.http.server.HTTPDemon;
|
||||
import de.anomic.http.server.RequestHeader;
|
||||
import de.anomic.search.Switchboard;
|
||||
|
@ -56,7 +56,7 @@ public class ConfigNetwork_p {
|
|||
if (post != null) {
|
||||
|
||||
// store this call as api call
|
||||
sb.tables.recordAPICall(post, "ConfigNetwork.html", Tables.API_TYPE_CONFIGURATION, "network settings");
|
||||
sb.tables.recordAPICall(post, "ConfigNetwork.html", WorkTables.TABLE_API_TYPE_CONFIGURATION, "network settings");
|
||||
|
||||
if (post.containsKey("changeNetwork")) {
|
||||
final String networkDefinition = post.get("networkDefinition", "defaults/yacy.network.freeworld.unit");
|
||||
|
|
|
@ -25,7 +25,7 @@
|
|||
// along with this program; if not, write to the Free Software
|
||||
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
||||
|
||||
import de.anomic.data.Tables;
|
||||
import de.anomic.data.WorkTables;
|
||||
import de.anomic.http.server.RequestHeader;
|
||||
import de.anomic.search.Switchboard;
|
||||
import de.anomic.search.SwitchboardConstants;
|
||||
|
@ -60,7 +60,7 @@ public class ConfigPortal {
|
|||
if (post.containsKey("searchpage_set")) {
|
||||
String newGreeting = post.get(SwitchboardConstants.GREETING, "");
|
||||
// store this call as api call
|
||||
sb.tables.recordAPICall(post, "ConfigPortal.html", Tables.API_TYPE_CONFIGURATION, "new portal design. greeting: " + newGreeting);
|
||||
sb.tables.recordAPICall(post, "ConfigPortal.html", WorkTables.TABLE_API_TYPE_CONFIGURATION, "new portal design. greeting: " + newGreeting);
|
||||
|
||||
sb.setConfig(SwitchboardConstants.GREETING, newGreeting);
|
||||
sb.setConfig(SwitchboardConstants.GREETING_HOMEPAGE, post.get(SwitchboardConstants.GREETING_HOMEPAGE, ""));
|
||||
|
|
|
@ -28,7 +28,6 @@ import java.util.Iterator;
|
|||
|
||||
import de.anomic.http.server.RequestHeader;
|
||||
import de.anomic.search.Switchboard;
|
||||
import de.anomic.search.SwitchboardConstants;
|
||||
import de.anomic.server.serverObjects;
|
||||
import de.anomic.server.serverSwitch;
|
||||
import de.anomic.yacy.yacyNewsPool;
|
||||
|
|
|
@ -46,7 +46,7 @@ import de.anomic.crawler.CrawlProfile;
|
|||
import de.anomic.crawler.SitemapImporter;
|
||||
import de.anomic.crawler.retrieval.Request;
|
||||
import de.anomic.data.BookmarkHelper;
|
||||
import de.anomic.data.Tables;
|
||||
import de.anomic.data.WorkTables;
|
||||
import de.anomic.data.bookmarksDB;
|
||||
import de.anomic.data.listManager;
|
||||
import de.anomic.http.server.RequestHeader;
|
||||
|
@ -143,7 +143,7 @@ public class Crawler_p {
|
|||
crawlingStart = (crawlingStartURL == null) ? null : crawlingStartURL.toNormalform(true, true);
|
||||
|
||||
// store this call as api call
|
||||
sb.tables.recordAPICall(post, "Crawler_p.html", Tables.API_TYPE_CRAWLER, "crawl start for " + crawlingStart);
|
||||
sb.tables.recordAPICall(post, "Crawler_p.html", WorkTables.TABLE_API_TYPE_CRAWLER, "crawl start for " + crawlingStart);
|
||||
|
||||
// set new properties
|
||||
final boolean fullDomain = post.get("range", "wide").equals("domain"); // special property in simple crawl start
|
||||
|
|
|
@ -24,22 +24,16 @@
|
|||
// along with this program; if not, write to the Free Software
|
||||
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
||||
|
||||
import java.util.Iterator;
|
||||
|
||||
import de.anomic.http.server.RequestHeader;
|
||||
import de.anomic.search.Switchboard;
|
||||
import de.anomic.search.SwitchboardConstants;
|
||||
import de.anomic.server.serverObjects;
|
||||
import de.anomic.server.serverSwitch;
|
||||
import de.anomic.yacy.yacyNewsPool;
|
||||
import de.anomic.yacy.yacyNewsRecord;
|
||||
import de.anomic.yacy.yacySeed;
|
||||
|
||||
public class IndexCreateDomainCrawl_p {
|
||||
|
||||
public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) {
|
||||
// return variable that accumulates replacements
|
||||
Switchboard sb = (Switchboard) env;
|
||||
//Switchboard sb = (Switchboard) env;
|
||||
serverObjects prop = new serverObjects();
|
||||
|
||||
// define visible variables
|
||||
|
|
|
@ -22,9 +22,10 @@ import java.util.Iterator;
|
|||
import java.util.LinkedHashMap;
|
||||
import java.util.Map;
|
||||
|
||||
import net.yacy.kelondro.blob.Tables;
|
||||
import net.yacy.kelondro.logging.Log;
|
||||
|
||||
import de.anomic.data.Tables;
|
||||
import de.anomic.data.WorkTables;
|
||||
import de.anomic.http.client.Client;
|
||||
import de.anomic.http.server.RequestHeader;
|
||||
import de.anomic.http.server.ResponseContainer;
|
||||
|
@ -41,7 +42,11 @@ public class Table_API_p {
|
|||
if (post != null && post.get("deleterows", "").length() > 0) {
|
||||
for (Map.Entry<String, String> entry: post.entrySet()) {
|
||||
if (entry.getKey().startsWith("mark_") && entry.getValue().equals("on")) {
|
||||
sb.tables.delete(Tables.API_TABLENAME, entry.getKey().substring(5).getBytes());
|
||||
try {
|
||||
sb.tables.delete(WorkTables.TABLE_API_NAME, entry.getKey().substring(5).getBytes());
|
||||
} catch (IOException e) {
|
||||
Log.logException(e);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
@ -56,9 +61,9 @@ public class Table_API_p {
|
|||
LinkedHashMap<String, Integer> l = new LinkedHashMap<String, Integer>();
|
||||
for (Map.Entry<String, String> entry: post.entrySet()) {
|
||||
if (entry.getKey().startsWith("mark_") && entry.getValue().equals("on")) {
|
||||
Map<String, byte[]> map = sb.tables.select(Tables.API_TABLENAME, entry.getKey().substring(5).getBytes());
|
||||
String url = "http://localhost:" + sb.getConfig("port", "8080") + new String(map.get(Tables.API_COL_URL));
|
||||
try {
|
||||
Tables.Row row = sb.tables.select(WorkTables.TABLE_API_NAME, entry.getKey().substring(5).getBytes());
|
||||
String url = "http://localhost:" + sb.getConfig("port", "8080") + new String(row.from(WorkTables.TABLE_API_COL_URL));
|
||||
result = client.GET(url);
|
||||
l.put(url, result.getStatusCode());
|
||||
} catch (IOException e) {
|
||||
|
@ -88,24 +93,32 @@ public class Table_API_p {
|
|||
prop.put("showtable", 1);
|
||||
|
||||
// insert rows
|
||||
final int maxCount = Math.min(1000, sb.tables.size(Tables.API_TABLENAME));
|
||||
final Iterator<Map.Entry<byte[], Map<String, byte[]>>> mapIterator = sb.tables.iterator(Tables.API_TABLENAME);
|
||||
Map.Entry<byte[], Map<String, byte[]>> record;
|
||||
Map<String, byte[]> map;
|
||||
int maxCount;
|
||||
try {
|
||||
maxCount = Math.min(1000, sb.tables.size(WorkTables.TABLE_API_NAME));
|
||||
} catch (IOException e) {
|
||||
Log.logException(e);
|
||||
maxCount = 0;
|
||||
}
|
||||
int count = 0;
|
||||
try {
|
||||
final Iterator<Tables.Row> mapIterator = sb.tables.iterator(WorkTables.TABLE_API_NAME);
|
||||
Tables.Row row;
|
||||
boolean dark = true;
|
||||
while ((mapIterator.hasNext()) && (count < maxCount)) {
|
||||
record = mapIterator.next();
|
||||
if (record == null) continue;
|
||||
map = record.getValue();
|
||||
row = mapIterator.next();
|
||||
if (row == null) continue;
|
||||
prop.put("showtable_list_" + count + "_dark", ((dark) ? 1 : 0) ); dark=!dark;
|
||||
prop.put("showtable_list_" + count + "_pk", new String(record.getKey()));
|
||||
prop.put("showtable_list_" + count + "_date", map.get(Tables.API_COL_DATE));
|
||||
prop.put("showtable_list_" + count + "_type", map.get(Tables.API_COL_TYPE));
|
||||
prop.put("showtable_list_" + count + "_comment", map.get(Tables.API_COL_COMMENT));
|
||||
prop.put("showtable_list_" + count + "_url", "http://" + sb.myPublicIP() + ":" + sb.getConfig("port", "8080") + new String(map.get(Tables.API_COL_URL)));
|
||||
prop.put("showtable_list_" + count + "_pk", new String(row.getPK()));
|
||||
prop.put("showtable_list_" + count + "_date", row.from(WorkTables.TABLE_API_COL_DATE));
|
||||
prop.put("showtable_list_" + count + "_type", row.from(WorkTables.TABLE_API_COL_TYPE));
|
||||
prop.put("showtable_list_" + count + "_comment", row.from(WorkTables.TABLE_API_COL_COMMENT));
|
||||
prop.put("showtable_list_" + count + "_url", "http://" + sb.myPublicIP() + ":" + sb.getConfig("port", "8080") + new String(row.from(WorkTables.TABLE_API_COL_URL)));
|
||||
count++;
|
||||
}
|
||||
} catch (IOException e) {
|
||||
Log.logException(e);
|
||||
}
|
||||
prop.put("showtable_list", count);
|
||||
|
||||
// return rewrite properties
|
||||
|
|
|
@ -17,11 +17,16 @@
|
|||
// along with this program; if not, write to the Free Software
|
||||
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.Iterator;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import net.yacy.kelondro.blob.Tables;
|
||||
import net.yacy.kelondro.logging.Log;
|
||||
|
||||
import de.anomic.http.server.RequestHeader;
|
||||
import de.anomic.search.Switchboard;
|
||||
import de.anomic.server.serverObjects;
|
||||
|
@ -33,7 +38,7 @@ public class Tables_p {
|
|||
final Switchboard sb = (Switchboard) env;
|
||||
final serverObjects prop = new serverObjects();
|
||||
String table = (post == null) ? null : post.get("table", null);
|
||||
if (table != null && !sb.tables.has(table)) table = null;
|
||||
if (table != null && !sb.tables.hasHeap(table)) table = null;
|
||||
|
||||
// show table selection
|
||||
int count = 0;
|
||||
|
@ -48,19 +53,26 @@ public class Tables_p {
|
|||
prop.put("tables", count);
|
||||
|
||||
List<String> columns = null;
|
||||
if (table != null) {
|
||||
if (table != null) try {
|
||||
columns = sb.tables.columns(table);
|
||||
} catch (IOException e) {
|
||||
Log.logException(e);
|
||||
columns = new ArrayList<String>();
|
||||
}
|
||||
|
||||
// apply deletion requests
|
||||
if (post != null && post.get("deletetable", "").length() > 0) {
|
||||
if (post != null && post.get("deletetable", "").length() > 0) try {
|
||||
sb.tables.clear(table);
|
||||
} catch (IOException e) {
|
||||
Log.logException(e);
|
||||
}
|
||||
|
||||
if (post != null && post.get("deleterows", "").length() > 0) {
|
||||
for (Map.Entry<String, String> entry: post.entrySet()) {
|
||||
if (entry.getKey().startsWith("mark_") && entry.getValue().equals("on")) {
|
||||
if (entry.getKey().startsWith("mark_") && entry.getValue().equals("on")) try {
|
||||
sb.tables.delete(table, entry.getKey().substring(5).getBytes());
|
||||
} catch (IOException e) {
|
||||
Log.logException(e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
@ -73,7 +85,11 @@ public class Tables_p {
|
|||
map.put(entry.getKey().substring(4), entry.getValue().getBytes());
|
||||
}
|
||||
}
|
||||
try {
|
||||
sb.tables.insert(table, pk.getBytes(), map);
|
||||
} catch (IOException e) {
|
||||
Log.logException(e);
|
||||
}
|
||||
}
|
||||
|
||||
// generate table
|
||||
|
@ -92,29 +108,35 @@ public class Tables_p {
|
|||
prop.put("showtable_columns", columns.size());
|
||||
|
||||
// insert all rows
|
||||
final int maxCount = Math.min(1000, sb.tables.size(table));
|
||||
final Iterator<Map.Entry<byte[], Map<String, byte[]>>> mapIterator = sb.tables.iterator(table);
|
||||
Map.Entry<byte[], Map<String, byte[]>> record;
|
||||
Map<String, byte[]> map;
|
||||
byte[] pk;
|
||||
int maxCount;
|
||||
try {
|
||||
maxCount = Math.min(1000, sb.tables.size(table));
|
||||
} catch (IOException e) {
|
||||
Log.logException(e);
|
||||
maxCount = 0;
|
||||
}
|
||||
count = 0;
|
||||
try {
|
||||
final Iterator<Tables.Row> mapIterator = sb.tables.iterator(table);
|
||||
Tables.Row row;
|
||||
boolean dark = true;
|
||||
byte[] cell;
|
||||
while ((mapIterator.hasNext()) && (count < maxCount)) {
|
||||
record = mapIterator.next();
|
||||
if (record == null) continue;
|
||||
pk = record.getKey();
|
||||
map = record.getValue();
|
||||
row = mapIterator.next();
|
||||
if (row == null) continue;
|
||||
prop.put("showtable_list_" + count + "_dark", ((dark) ? 1 : 0) ); dark=!dark;
|
||||
prop.put("showtable_list_" + count + "_pk", new String(pk));
|
||||
prop.put("showtable_list_" + count + "_pk", new String(row.getPK()));
|
||||
for (int i = 0; i < columns.size(); i++) {
|
||||
cell = map.get(columns.get(i));
|
||||
cell = row.from(columns.get(i));
|
||||
prop.putHTML("showtable_list_" + count + "_columns_" + i + "_cell", cell == null ? "" : new String(cell));
|
||||
}
|
||||
prop.put("showtable_list_" + count + "_columns", columns.size());
|
||||
|
||||
count++;
|
||||
}
|
||||
} catch (IOException e) {
|
||||
Log.logException(e);
|
||||
}
|
||||
prop.put("showtable_list", count);
|
||||
}
|
||||
|
||||
|
@ -127,15 +149,21 @@ public class Tables_p {
|
|||
break;
|
||||
}
|
||||
}
|
||||
try {
|
||||
if (pk != null && sb.tables.has(table, pk.getBytes())) {
|
||||
setEdit(sb, prop, table, pk, columns);
|
||||
}
|
||||
} catch (IOException e) {
|
||||
Log.logException(e);
|
||||
}
|
||||
}
|
||||
|
||||
if (post != null && table != null && post.containsKey("addrow")) {
|
||||
if (post != null && table != null && post.containsKey("addrow")) try {
|
||||
// get a new key
|
||||
String pk = sb.tables.createRow(table);
|
||||
String pk = new String(sb.tables.createRow(table));
|
||||
setEdit(sb, prop, table, pk, columns);
|
||||
} catch (IOException e) {
|
||||
Log.logException(e);
|
||||
}
|
||||
|
||||
// adding the peer address
|
||||
|
@ -145,15 +173,15 @@ public class Tables_p {
|
|||
return prop;
|
||||
}
|
||||
|
||||
private static void setEdit(final Switchboard sb, final serverObjects prop, final String table, final String pk, List<String> columns) {
|
||||
private static void setEdit(final Switchboard sb, final serverObjects prop, final String table, final String pk, List<String> columns) throws IOException {
|
||||
prop.put("showedit", 1);
|
||||
prop.put("showedit_table", table);
|
||||
prop.put("showedit_pk", pk);
|
||||
Map<String, byte[]> map = sb.tables.select(table, pk.getBytes());
|
||||
Tables.Row row = sb.tables.select(table, pk.getBytes());
|
||||
int count = 0;
|
||||
byte[] cell;
|
||||
for (String col: columns) {
|
||||
cell = map.get(col);
|
||||
cell = row.from(col);
|
||||
prop.put("showedit_list_" + count + "_key", col);
|
||||
prop.put("showedit_list_" + count + "_value", cell == null ? "" : new String(cell));
|
||||
count++;
|
||||
|
|
|
@ -1,151 +0,0 @@
|
|||
package de.anomic.data;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Date;
|
||||
import java.util.HashMap;
|
||||
import java.util.Iterator;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.TreeMap;
|
||||
|
||||
import net.yacy.kelondro.blob.BEncodedHeapArray;
|
||||
import net.yacy.kelondro.index.RowSpaceExceededException;
|
||||
import net.yacy.kelondro.logging.Log;
|
||||
import net.yacy.kelondro.util.DateFormatter;
|
||||
import de.anomic.server.serverObjects;
|
||||
|
||||
public class Tables {
|
||||
|
||||
public final static String API_TABLENAME = "api";
|
||||
public final static String API_TYPE_STEERING = "steering";
|
||||
public final static String API_TYPE_CONFIGURATION = "configuration";
|
||||
public final static String API_TYPE_CRAWLER = "crawler";
|
||||
|
||||
public final static String API_COL_TYPE = "type";
|
||||
public final static String API_COL_COMMENT = "comment";
|
||||
public final static String API_COL_DATE = "date";
|
||||
public final static String API_COL_URL = "url";
|
||||
|
||||
private BEncodedHeapArray tables;
|
||||
|
||||
public Tables(File workPath) {
|
||||
this.tables = new BEncodedHeapArray(workPath, 12);
|
||||
}
|
||||
|
||||
public boolean has(String table) {
|
||||
return tables.hasHeap(table);
|
||||
}
|
||||
|
||||
public boolean has(String table, byte[] pk) {
|
||||
try {
|
||||
return tables.has(table, pk);
|
||||
} catch (IOException e) {
|
||||
Log.logException(e);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
public Iterator<String> tables() {
|
||||
return this.tables.tables();
|
||||
}
|
||||
|
||||
public List<String> columns(String table) {
|
||||
try {
|
||||
return this.tables.columns(table);
|
||||
} catch (IOException e) {
|
||||
Log.logException(e);
|
||||
return new ArrayList<String>(0);
|
||||
}
|
||||
}
|
||||
|
||||
public void clear(String table) {
|
||||
try {
|
||||
this.tables.clear(table);
|
||||
} catch (IOException e) {
|
||||
Log.logException(e);
|
||||
}
|
||||
}
|
||||
|
||||
public void delete(String table, byte[] pk) {
|
||||
try {
|
||||
this.tables.delete(table, pk);
|
||||
} catch (IOException e) {
|
||||
Log.logException(e);
|
||||
}
|
||||
}
|
||||
|
||||
public int size(String table) {
|
||||
try {
|
||||
return this.tables.size(table);
|
||||
} catch (IOException e) {
|
||||
Log.logException(e);
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
public Iterator<Map.Entry<byte[], Map<String, byte[]>>> iterator(String table) {
|
||||
try {
|
||||
return this.tables.iterator(table);
|
||||
} catch (IOException e) {
|
||||
Log.logException(e);
|
||||
return new TreeMap<byte[], Map<String, byte[]>>().entrySet().iterator();
|
||||
}
|
||||
}
|
||||
|
||||
public void close() {
|
||||
this.tables.close();
|
||||
}
|
||||
|
||||
public Map<String, byte[]> select(String table, byte[] pk) {
|
||||
try {
|
||||
return tables.select(table, pk);
|
||||
} catch (IOException e) {
|
||||
Log.logException(e);
|
||||
return new TreeMap<String, byte[]>();
|
||||
}
|
||||
}
|
||||
|
||||
public void insert(String table, byte[] pk, Map<String, byte[]> map) {
|
||||
try {
|
||||
this.tables.insert(table, pk, map);
|
||||
} catch (RowSpaceExceededException e) {
|
||||
Log.logException(e);
|
||||
} catch (IOException e) {
|
||||
Log.logException(e);
|
||||
}
|
||||
}
|
||||
|
||||
public String createRow(String table) {
|
||||
try {
|
||||
return new String(this.tables.insert(table, new HashMap<String, byte[]>()));
|
||||
} catch (RowSpaceExceededException e) {
|
||||
Log.logException(e);
|
||||
return null;
|
||||
} catch (IOException e) {
|
||||
Log.logException(e);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
public void recordAPICall(final serverObjects post, final String servletName, String type, String comment) {
|
||||
String apiurl = /*"http://localhost:" + getConfig("port", "8080") +*/ "/" + servletName + "?" + post.toString();
|
||||
try {
|
||||
this.tables.insert(
|
||||
API_TABLENAME,
|
||||
API_COL_TYPE, type.getBytes(),
|
||||
API_COL_COMMENT, comment.getBytes(),
|
||||
API_COL_DATE, DateFormatter.formatShortMilliSecond(new Date()).getBytes(),
|
||||
API_COL_URL, apiurl.getBytes()
|
||||
);
|
||||
} catch (RowSpaceExceededException e2) {
|
||||
Log.logException(e2);
|
||||
} catch (IOException e2) {
|
||||
Log.logException(e2);
|
||||
}
|
||||
Log.logInfo("APICALL", apiurl);
|
||||
}
|
||||
|
||||
|
||||
}
|
71
source/de/anomic/data/WorkTables.java
Normal file
71
source/de/anomic/data/WorkTables.java
Normal file
|
@ -0,0 +1,71 @@
|
|||
// Work.java
|
||||
// (C) 2010 by Michael Peter Christen; mc@yacy.net, Frankfurt a. M., Germany
|
||||
// first published 04.02.2010 on http://yacy.net
|
||||
//
|
||||
// This is a part of YaCy, a peer-to-peer based web search engine
|
||||
//
|
||||
// $LastChangedDate: 2006-04-02 22:40:07 +0200 (So, 02 Apr 2006) $
|
||||
// $LastChangedRevision: 6539 $
|
||||
// $LastChangedBy: low012 $
|
||||
//
|
||||
// LICENSE
|
||||
//
|
||||
// This program is free software; you can redistribute it and/or modify
|
||||
// it under the terms of the GNU General Public License as published by
|
||||
// the Free Software Foundation; either version 2 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU General Public License
|
||||
// along with this program; if not, write to the Free Software
|
||||
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
||||
|
||||
package de.anomic.data;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.util.Date;
|
||||
|
||||
import net.yacy.kelondro.blob.Tables;
|
||||
import net.yacy.kelondro.logging.Log;
|
||||
import net.yacy.kelondro.util.DateFormatter;
|
||||
import de.anomic.server.serverObjects;
|
||||
|
||||
public class WorkTables extends Tables {
|
||||
|
||||
public final static String TABLE_API_NAME = "api";
|
||||
public final static String TABLE_API_TYPE_STEERING = "steering";
|
||||
public final static String TABLE_API_TYPE_CONFIGURATION = "configuration";
|
||||
public final static String TABLE_API_TYPE_CRAWLER = "crawler";
|
||||
|
||||
public final static String TABLE_API_COL_TYPE = "type";
|
||||
public final static String TABLE_API_COL_COMMENT = "comment";
|
||||
public final static String TABLE_API_COL_DATE = "date";
|
||||
public final static String TABLE_API_COL_URL = "url";
|
||||
|
||||
|
||||
public WorkTables(File workPath) {
|
||||
super(workPath, 12);
|
||||
}
|
||||
|
||||
public void recordAPICall(final serverObjects post, final String servletName, String type, String comment) {
|
||||
String apiurl = /*"http://localhost:" + getConfig("port", "8080") +*/ "/" + servletName + "?" + post.toString();
|
||||
try {
|
||||
super.insert(
|
||||
TABLE_API_NAME,
|
||||
TABLE_API_COL_TYPE, type.getBytes(),
|
||||
TABLE_API_COL_COMMENT, comment.getBytes(),
|
||||
TABLE_API_COL_DATE, DateFormatter.formatShortMilliSecond(new Date()).getBytes(),
|
||||
TABLE_API_COL_URL, apiurl.getBytes()
|
||||
);
|
||||
} catch (IOException e) {
|
||||
Log.logException(e);
|
||||
}
|
||||
Log.logInfo("APICALL", apiurl);
|
||||
}
|
||||
|
||||
}
|
|
@ -41,6 +41,7 @@ import net.yacy.kelondro.io.Writer;
|
|||
import net.yacy.kelondro.logging.Log;
|
||||
import net.yacy.kelondro.order.ByteOrder;
|
||||
import net.yacy.kelondro.order.CloneableIterator;
|
||||
import net.yacy.kelondro.order.NaturalOrder;
|
||||
import net.yacy.kelondro.order.RotateIterator;
|
||||
import net.yacy.kelondro.util.FileUtils;
|
||||
import net.yacy.kelondro.util.MemoryControl;
|
||||
|
@ -112,10 +113,12 @@ public class HeapReader {
|
|||
}
|
||||
|
||||
protected byte[] normalizeKey(byte[] key) {
|
||||
// check size of key: zero-filled keys are only possible of the ordering is
|
||||
// an instance of the natural ordering. Base64-orderings cannot use zeros in keys.
|
||||
assert key.length >= this.keylength || this.ordering instanceof NaturalOrder;
|
||||
return normalizeKey(key, this.keylength);
|
||||
}
|
||||
|
||||
|
||||
private static final byte zero = 0;
|
||||
|
||||
protected static byte[] normalizeKey(byte[] key, int keylength) {
|
||||
|
|
|
@ -1,4 +1,4 @@
|
|||
// BEncodedHeapArray.java
|
||||
// Tables.java
|
||||
// (C) 2010 by Michael Peter Christen; mc@yacy.net, Frankfurt a. M., Germany
|
||||
// first published 14.01.2010 on http://yacy.net
|
||||
//
|
||||
|
@ -37,10 +37,12 @@ import java.util.concurrent.ConcurrentHashMap;
|
|||
|
||||
import net.yacy.kelondro.index.RowSpaceExceededException;
|
||||
import net.yacy.kelondro.logging.Log;
|
||||
import net.yacy.kelondro.util.ByteBuffer;
|
||||
import net.yacy.kelondro.util.FileUtils;
|
||||
import net.yacy.kelondro.util.LookAheadIterator;
|
||||
|
||||
|
||||
public class BEncodedHeapArray {
|
||||
public class Tables {
|
||||
|
||||
private static final String suffix = ".bheap";
|
||||
private static final String system_table_pkcounter = "pkcounter";
|
||||
|
@ -50,7 +52,7 @@ public class BEncodedHeapArray {
|
|||
private ConcurrentHashMap<String, BEncodedHeap> tables;
|
||||
private int keymaxlen;
|
||||
|
||||
public BEncodedHeapArray(final File location, final int keymaxlen) {
|
||||
public Tables(final File location, final int keymaxlen) {
|
||||
this.location = new File(location.getAbsolutePath());
|
||||
if (!this.location.exists()) this.location.mkdirs();
|
||||
this.keymaxlen = keymaxlen;
|
||||
|
@ -126,7 +128,7 @@ public class BEncodedHeapArray {
|
|||
}
|
||||
|
||||
private byte[] ukey(String tablename) throws IOException {
|
||||
byte[] pk = select(system_table_pkcounter, tablename.getBytes(), system_table_pkcounter_counterName);
|
||||
byte[] pk = select(system_table_pkcounter, tablename.getBytes()).from(system_table_pkcounter_counterName);
|
||||
int pki;
|
||||
if (pk == null) {
|
||||
pki = size(tablename);
|
||||
|
@ -156,14 +158,14 @@ public class BEncodedHeapArray {
|
|||
* @throws RowSpaceExceededException
|
||||
* @throws IOException
|
||||
*/
|
||||
public byte[] insert(final String tablename, Map<String, byte[]> map) throws RowSpaceExceededException, IOException {
|
||||
public byte[] insert(final String tablename, Map<String, byte[]> map) throws IOException {
|
||||
byte[] uk = ukey(tablename);
|
||||
insert(tablename, uk, map);
|
||||
insert(system_table_pkcounter, tablename.getBytes(), system_table_pkcounter_counterName, uk);
|
||||
return uk;
|
||||
}
|
||||
|
||||
public byte[] insert(final String tablename, String key, byte[] value) throws RowSpaceExceededException, IOException {
|
||||
public byte[] insert(final String tablename, String key, byte[] value) throws IOException {
|
||||
byte[] uk = ukey(tablename);
|
||||
insert(tablename, uk, key, value);
|
||||
insert(system_table_pkcounter, tablename.getBytes(), system_table_pkcounter_counterName, uk);
|
||||
|
@ -173,7 +175,7 @@ public class BEncodedHeapArray {
|
|||
public byte[] insert(final String tablename,
|
||||
String key0, byte[] value0,
|
||||
String key1, byte[] value1
|
||||
) throws RowSpaceExceededException, IOException {
|
||||
) throws IOException {
|
||||
byte[] uk = ukey(tablename);
|
||||
insert(tablename, uk,
|
||||
key0, value0,
|
||||
|
@ -187,7 +189,7 @@ public class BEncodedHeapArray {
|
|||
String key0, byte[] value0,
|
||||
String key1, byte[] value1,
|
||||
String key2, byte[] value2
|
||||
) throws RowSpaceExceededException, IOException {
|
||||
) throws IOException {
|
||||
byte[] uk = ukey(tablename);
|
||||
insert(tablename, uk,
|
||||
key0, value0,
|
||||
|
@ -203,7 +205,7 @@ public class BEncodedHeapArray {
|
|||
String key1, byte[] value1,
|
||||
String key2, byte[] value2,
|
||||
String key3, byte[] value3
|
||||
) throws RowSpaceExceededException, IOException {
|
||||
) throws IOException {
|
||||
byte[] uk = ukey(tablename);
|
||||
insert(tablename, uk,
|
||||
key0, value0,
|
||||
|
@ -217,33 +219,45 @@ public class BEncodedHeapArray {
|
|||
|
||||
public void insert(final String table, byte[] pk,
|
||||
String key, byte[] value
|
||||
) throws RowSpaceExceededException, IOException {
|
||||
) throws IOException {
|
||||
BEncodedHeap heap = getHeap(table);
|
||||
try {
|
||||
heap.put(pk, key, value);
|
||||
} catch (RowSpaceExceededException e) {
|
||||
throw new IOException(e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
public void insert(final String table, byte[] pk,
|
||||
String key0, byte[] value0,
|
||||
String key1, byte[] value1
|
||||
) throws RowSpaceExceededException, IOException {
|
||||
) throws IOException {
|
||||
BEncodedHeap heap = getHeap(table);
|
||||
try {
|
||||
heap.put(pk,
|
||||
key0, value0,
|
||||
key1, value1
|
||||
);
|
||||
} catch (RowSpaceExceededException e) {
|
||||
throw new IOException(e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
public void insert(final String table, byte[] pk,
|
||||
String key0, byte[] value0,
|
||||
String key1, byte[] value1,
|
||||
String key2, byte[] value2
|
||||
) throws RowSpaceExceededException, IOException {
|
||||
) throws IOException {
|
||||
BEncodedHeap heap = getHeap(table);
|
||||
try {
|
||||
heap.put(pk,
|
||||
key0, value0,
|
||||
key1, value1,
|
||||
key2, value2
|
||||
);
|
||||
} catch (RowSpaceExceededException e) {
|
||||
throw new IOException(e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
public void insert(final String table, byte[] pk,
|
||||
|
@ -251,29 +265,45 @@ public class BEncodedHeapArray {
|
|||
String key1, byte[] value1,
|
||||
String key2, byte[] value2,
|
||||
String key3, byte[] value3
|
||||
) throws RowSpaceExceededException, IOException {
|
||||
) throws IOException {
|
||||
BEncodedHeap heap = getHeap(table);
|
||||
try {
|
||||
heap.put(pk,
|
||||
key0, value0,
|
||||
key1, value1,
|
||||
key2, value2,
|
||||
key3, value3
|
||||
);
|
||||
} catch (RowSpaceExceededException e) {
|
||||
throw new IOException(e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
public void insert(final String table, byte[] pk, Map<String, byte[]> map) throws RowSpaceExceededException, IOException {
|
||||
public void insert(final String table, byte[] pk, Map<String, byte[]> map) throws IOException {
|
||||
BEncodedHeap heap = getHeap(table);
|
||||
try {
|
||||
heap.put(pk, map);
|
||||
} catch (RowSpaceExceededException e) {
|
||||
throw new IOException(e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
public Map<String, byte[]> select(final String table, byte[] pk) throws IOException {
|
||||
public void insert(final String table, Row row) throws IOException {
|
||||
BEncodedHeap heap = getHeap(table);
|
||||
return heap.get(pk);
|
||||
try {
|
||||
heap.put(row.pk, row.map);
|
||||
} catch (RowSpaceExceededException e) {
|
||||
throw new IOException(e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
public byte[] select(final String table, byte[] pk, String key) throws IOException {
|
||||
public byte[] createRow(String table) throws IOException {
|
||||
return this.insert(table, new HashMap<String, byte[]>());
|
||||
}
|
||||
|
||||
public Row select(final String table, byte[] pk) throws IOException {
|
||||
BEncodedHeap heap = getHeap(table);
|
||||
return heap.getProp(pk, key);
|
||||
return new Row(pk, heap.get(pk));
|
||||
}
|
||||
|
||||
public void delete(final String table, byte[] pk) throws IOException {
|
||||
|
@ -291,9 +321,12 @@ public class BEncodedHeapArray {
|
|||
return heap.keys();
|
||||
}
|
||||
|
||||
public Iterator<Map.Entry<byte[], Map<String, byte[]>>> iterator(String table) throws IOException {
|
||||
BEncodedHeap heap = getHeap(table);
|
||||
return heap.iterator();
|
||||
public Iterator<Row> iterator(String table) throws IOException {
|
||||
return new RowIterator(table);
|
||||
}
|
||||
|
||||
public Iterator<Row> iterator(String table, String whereKey, byte[] whereValue) throws IOException {
|
||||
return new RowIterator(table, whereKey, whereValue);
|
||||
}
|
||||
|
||||
public List<String> columns(String table) throws IOException {
|
||||
|
@ -301,13 +334,74 @@ public class BEncodedHeapArray {
|
|||
return heap.columns();
|
||||
}
|
||||
|
||||
public class RowIterator extends LookAheadIterator<Row> implements Iterator<Row> {
|
||||
|
||||
private final String whereKey;
|
||||
private final byte[] whereValue;
|
||||
private final Iterator<Map.Entry<byte[], Map<String, byte[]>>> i;
|
||||
|
||||
public RowIterator(String table) throws IOException {
|
||||
this.whereKey = null;
|
||||
this.whereValue = null;
|
||||
BEncodedHeap heap = getHeap(table);
|
||||
i = heap.iterator();
|
||||
}
|
||||
|
||||
public RowIterator(String table, String whereKey, byte[] whereValue) throws IOException {
|
||||
this.whereKey = whereKey;
|
||||
this.whereValue = whereValue;
|
||||
BEncodedHeap heap = getHeap(table);
|
||||
i = heap.iterator();
|
||||
}
|
||||
|
||||
protected Row next0() {
|
||||
while (i.hasNext()) {
|
||||
Row r = new Row(i.next());
|
||||
if (this.whereKey == null) return r;
|
||||
if (ByteBuffer.equals(r.from(this.whereKey), this.whereValue)) return r;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
public class Row {
|
||||
|
||||
private final byte[] pk;
|
||||
private final Map<String, byte[]> map;
|
||||
|
||||
public Row(final Map.Entry<byte[], Map<String, byte[]>> entry) {
|
||||
this.pk = entry.getKey();
|
||||
this.map = entry.getValue();
|
||||
}
|
||||
|
||||
public Row(final byte[] pk, final Map<String, byte[]> map) {
|
||||
this.pk = pk;
|
||||
this.map = map;
|
||||
}
|
||||
|
||||
public byte[] getPK() {
|
||||
return this.pk;
|
||||
}
|
||||
|
||||
public byte[] from(String colname) {
|
||||
return this.map.get(colname);
|
||||
}
|
||||
|
||||
public String toString() {
|
||||
StringBuilder sb = new StringBuilder(keymaxlen + 20 * map.size());
|
||||
sb.append(new String(pk)).append(":").append(map.toString());
|
||||
return sb.toString();
|
||||
}
|
||||
}
|
||||
|
||||
public static void main(String[] args) {
|
||||
// test the class
|
||||
File f = new File(new File("maptest").getAbsolutePath());
|
||||
// System.out.println(f.getAbsolutePath());
|
||||
// System.out.println(f.getParent());
|
||||
try {
|
||||
BEncodedHeapArray map = new BEncodedHeapArray(f.getParentFile(), 4);
|
||||
Tables map = new Tables(f.getParentFile(), 4);
|
||||
// put some values into the map
|
||||
Map<String, byte[]> m = new HashMap<String, byte[]>();
|
||||
m.put("k", "000".getBytes());
|
||||
|
@ -317,18 +411,14 @@ public class BEncodedHeapArray {
|
|||
m.put("k", "222".getBytes());
|
||||
map.insert("testdao", "789".getBytes(), m);
|
||||
// iterate over keys
|
||||
Iterator<Map.Entry<byte[], Map<String, byte[]>>> i = map.iterator("testdao");
|
||||
Iterator<Row> i = map.iterator("testdao");
|
||||
while (i.hasNext()) {
|
||||
Map.Entry<byte[], Map<String, byte[]>> entry = i.next();
|
||||
System.out.println(new String(entry.getKey(), "UTF-8") + ": "
|
||||
+ entry.getValue());
|
||||
System.out.println(i.next().toString());
|
||||
}
|
||||
// clean up
|
||||
map.close();
|
||||
} catch (IOException e) {
|
||||
Log.logException(e);
|
||||
} catch (RowSpaceExceededException e) {
|
||||
Log.logException(e);
|
||||
}
|
||||
}
|
||||
}
|
78
source/net/yacy/kelondro/util/LookAheadIterator.java
Normal file
78
source/net/yacy/kelondro/util/LookAheadIterator.java
Normal file
|
@ -0,0 +1,78 @@
|
|||
// LookAheadIterator.java
|
||||
// (C) 2010 by Michael Peter Christen; mc@yacy.net, Frankfurt a. M., Germany
|
||||
// first published 04.02.2010 on http://yacy.net
|
||||
//
|
||||
// This is a part of YaCy, a peer-to-peer based web search engine
|
||||
//
|
||||
// $LastChangedDate: 2006-04-02 22:40:07 +0200 (So, 02 Apr 2006) $
|
||||
// $LastChangedRevision: 6539 $
|
||||
// $LastChangedBy: low012 $
|
||||
//
|
||||
// LICENSE
|
||||
//
|
||||
// This program is free software; you can redistribute it and/or modify
|
||||
// it under the terms of the GNU General Public License as published by
|
||||
// the Free Software Foundation; either version 2 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU General Public License
|
||||
// along with this program; if not, write to the Free Software
|
||||
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
||||
|
||||
package net.yacy.kelondro.util;
|
||||
|
||||
import java.util.Iterator;
|
||||
|
||||
/**
|
||||
* convenience class for iterator implementations that naturally terminate
|
||||
* when the next() method would return null. This is usually implemented using
|
||||
* a next0() method and a wrapper for next() and hasNext() that evaluates the
|
||||
* latest return value of next0()
|
||||
* To use this class just implement the next0() method
|
||||
*/
|
||||
public abstract class LookAheadIterator<A> implements Iterator<A> {
|
||||
|
||||
private boolean fresh = true;
|
||||
private A next = null;
|
||||
|
||||
public LookAheadIterator() {
|
||||
}
|
||||
|
||||
/**
|
||||
* the internal next-method
|
||||
* @return a value of type A if available or null if no more value are available
|
||||
*/
|
||||
protected abstract A next0() ;
|
||||
|
||||
private final void checkInit() {
|
||||
if (fresh) {
|
||||
next = next0();
|
||||
fresh = false;
|
||||
}
|
||||
}
|
||||
|
||||
public final boolean hasNext() {
|
||||
checkInit();
|
||||
return next != null;
|
||||
}
|
||||
|
||||
public final A next() {
|
||||
checkInit();
|
||||
A n = next;
|
||||
next = next0();
|
||||
return n;
|
||||
}
|
||||
|
||||
/**
|
||||
* a remove is not possible with this implementation
|
||||
*/
|
||||
public final void remove() {
|
||||
throw new UnsupportedOperationException();
|
||||
}
|
||||
|
||||
}
|
Loading…
Reference in New Issue
Block a user