Marown/yacy_search_server
1
0
Fork 0
mirror of https://github.com/yacy/yacy_search_server.git synced 2024-09-19 00:01:41 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity

added a full, complete, database insert, update and delete API for the tables.

Browse Source 
Please see this example:

list all database tables:
http://localhost:8090/api/table_p.xml

now create a new table and insert some values into 'mytable'
http://localhost:8090/api/table_p.xml?table=mytable&pk=&commitrow=&col_termin=Release%20Machen&col_datum=24.11.2011&col_status=ongoing

list the table content:
http://localhost:8090/api/table_p.xml?table=mytable&pk=

update the table and change a single value inside. You must refer to the row using a primary key 'pk'
http://localhost:8090/api/table_p.xml?table=mytable&pk=000000000001&commitrow=&col_datum=29.11.2011

you can also select rows using a search operator
http://localhost:8090/api/table_p.xml?table=mytable&pk=&count=10&search=

now lets delete the row:
http://localhost:8090/api/table_p.xml?table=mytable&pk=&deleterows=pk_000000000001

and we can also delete the complete table:
http://localhost:8090/api/table_p.xml?table=mytable&deletetable=

You can use this to administrate the robots, bookmarks and API steering using an outside application!

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@8071 6c8d7289-2bf4-0310-a012-ef5d649a1542
This commit is contained in:
orbiter 2011-11-22 12:31:07 +00:00
parent 3cc93325f0
commit 05f34a3fa7
7 changed files with 260 additions and 174 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
htroot/Table_YMark_p.java
View File

@ -40,7 +40,7 @@ public class Table_YMark_p {
// show table selection // show table selection
int count = 0; int count = 0;
final Iterator<String> ti = sb.tables.tables(); final Iterator<String> ti = sb.tables.iterator();
String tablename; String tablename;
prop.put("showselection", 1); prop.put("showselection", 1);
while (ti.hasNext()) { while (ti.hasNext()) {

2
htroot/Tables_p.html
View File

@ -84,7 +84,7 @@ document.write("<div id=\"api\">\<a href=\"/api/table_p.xml" + window.location.s
</tr> </tr>
#{list}# #{list}#
<tr class="TableCell#(dark)#Light::Dark::Summary#(/dark)#"> <tr class="TableCell#(dark)#Light::Dark::Summary#(/dark)#">
<td align="left"><input type="checkbox" name="item_#[count]#" value="mark_#[pk]#" /></td> <td align="left"><input type="checkbox" name="item_#[count]#" value="pk_#[pk]#" /></td>
<td>#[pk]#</td> <td>#[pk]#</td>
#{columns}# #{columns}#
<td align="left">#[cell]#</td> <td align="left">#[cell]#</td>

21
htroot/Tables_p.java
View File

@ -49,7 +49,7 @@ public class Tables_p {
// show table selection // show table selection
int count = 0; int count = 0;
final Iterator<String> ti = sb.tables.tables(); final Iterator<String> ti = sb.tables.iterator();
String tablename; String tablename;
prop.put("showselection", 1); prop.put("showselection", 1);
while (ti.hasNext()) { while (ti.hasNext()) {
@ -69,21 +69,13 @@ public class Tables_p {
final Pattern matcher = (pattern.length() == 0 || pattern.equals(".*")) ? null : Pattern.compile(".*" + pattern + ".*"); final Pattern matcher = (pattern.length() == 0 || pattern.equals(".*")) ? null : Pattern.compile(".*" + pattern + ".*");
prop.put("pattern", pattern); prop.put("pattern", pattern);
List<String> columns = null;
if (table != null) try {
columns = sb.tables.columns(table);
} catch (final IOException e) {
Log.logException(e);
columns = new ArrayList<String>();
}
// apply deletion requests // apply deletion requests
if (post.get("deletetable", "").length() > 0) if (post.get("deletetable", "").length() > 0)
sb.tables.clear(table); sb.tables.clear(table);
if (post.get("deleterows", "").length() > 0) { if (post.get("deleterows", "").length() > 0) {
for (final Map.Entry<String, String> entry: post.entrySet()) { for (final Map.Entry<String, String> entry: post.entrySet()) {
if (entry.getValue().startsWith("mark_")) try { if (entry.getValue().startsWith("pk_")) try {
sb.tables.delete(table, entry.getValue().substring(5).getBytes()); sb.tables.delete(table, entry.getValue().substring(5).getBytes());
} catch (final IOException e) { } catch (final IOException e) {
Log.logException(e); Log.logException(e);
@ -110,8 +102,17 @@ public class Tables_p {
prop.put("showtable", 0); prop.put("showtable", 0);
prop.put("showedit", 0); prop.put("showedit", 0);
if (table != null) { if (table != null) {
List<String> columns = null;
try {
columns = sb.tables.columns(table);
} catch (final IOException e) {
Log.logException(e);
columns = new ArrayList<String>();
}
if (post.containsKey("editrow")) { if (post.containsKey("editrow")) {
// check if we can find a key // check if we can find a key
String pk = null; String pk = null;

95
htroot/api/table_p.java
View File

@ -19,11 +19,15 @@
import java.io.IOException; import java.io.IOException;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator; import java.util.Iterator;
import java.util.Map;
import java.util.regex.Pattern;
import net.yacy.cora.document.UTF8; import net.yacy.cora.document.UTF8;
import net.yacy.cora.protocol.RequestHeader; import net.yacy.cora.protocol.RequestHeader;
import net.yacy.kelondro.blob.Tables; import net.yacy.kelondro.blob.Tables;
import net.yacy.kelondro.index.RowSpaceExceededException;
import net.yacy.kelondro.logging.Log; import net.yacy.kelondro.logging.Log;
import net.yacy.search.Switchboard; import net.yacy.search.Switchboard;
import de.anomic.server.serverObjects; import de.anomic.server.serverObjects;
@ -39,17 +43,85 @@ public class table_p {
final boolean html = EXT.equals("html"); final boolean html = EXT.equals("html");
final boolean xml = EXT.equals("xml"); final boolean xml = EXT.equals("xml");
String table = (post == null) ? null : post.get("table", null); String table = (post == null) ? null : post.get("table");
if (table != null && !sb.tables.hasHeap(table)) table = null; if (post == null || (!post.containsKey("commitrow") && table != null && !sb.tables.hasHeap(table))) table = null;
prop.put("showtable", 0); prop.put("showtable", 0);
prop.put("tablecount", sb.tables.size());
if (table == null) return prop; // apply deletion requests
if (table != null && post != null && post.containsKey("deletetable")) {
sb.tables.clear(table);
table = null;
}
if (table == null) {
// list all tables that we know
int c = 0;
for (final String name: sb.tables) {
try {
if (html) {
prop.putHTML("showtable_tables_" + c + "_table", name);
}
if (xml) {
prop.putXML("showtable_tables_" + c + "_table", name);
}
prop.put("showtable_tables_" + c + "_num", sb.tables.size(name));
c++;
} catch (final IOException e) {
}
}
prop.put("showtable_tables", c);
prop.put("tablecount", c);
return prop;
}
final boolean showpk = post.containsKey("pk"); final boolean showpk = post.containsKey("pk");
final String selectKey = post.containsKey("selectKey") ? post.get("selectKey") : null; final String selectKey = post.containsKey("selectKey") ? post.get("selectKey") : null;
final String selectValue = (selectKey != null && post.containsKey("selectValue")) ? post.get("selectValue") : null; final String selectValue = (selectKey != null && post.containsKey("selectValue")) ? post.get("selectValue") : null;
final String counts = post.get("count", null);
int maxcount = (counts == null || counts.equals("all")) ? Integer.MAX_VALUE : post.getInt("count", 10);
final String pattern = post.get("search", "");
final Pattern matcher = (pattern.length() == 0 || pattern.equals(".*")) ? null : Pattern.compile(".*" + pattern + ".*");
if (post.containsKey("deleterows")) {
for (final Map.Entry<String, String> entry: post.entrySet()) {
if (entry.getValue().startsWith("pk_")) try {
sb.tables.delete(table, entry.getValue().substring(3).getBytes());
} catch (final IOException e) {
Log.logException(e);
}
}
}
if (post.containsKey("commitrow")) {
final String pk = post.get("pk");
final Map<String, byte[]> map = new HashMap<String, byte[]>();
for (final Map.Entry<String, String> entry: post.entrySet()) {
if (entry.getKey().startsWith("col_")) {
map.put(entry.getKey().substring(4), entry.getValue().getBytes());
}
}
try {
if (pk == null || pk.length() == 0) {
sb.tables.insert(table, map);
} else {
sb.tables.update(table, pk.getBytes(), map);
}
} catch (final IOException e) {
Log.logException(e);
} catch (final RowSpaceExceededException e) {
Log.logException(e);
}
}
// generate table
prop.put("showtable", 1);
prop.put("showtable_table", table);
// insert the columns
ArrayList<String> columns = null; ArrayList<String> columns = null;
try { try {
columns = sb.tables.columns(table); columns = sb.tables.columns(table);
@ -67,12 +139,6 @@ public class table_p {
if (i < columns.size()) columns.add(i, row[i]); if (i < columns.size()) columns.add(i, row[i]);
} }
} }
// generate table
prop.put("showtable", 1);
prop.put("showtable_table", table);
// insert the columns
prop.put("showtable_showpk", showpk ? 1 : 0); prop.put("showtable_showpk", showpk ? 1 : 0);
for (int i = 0; i < columns.size(); i++) { for (int i = 0; i < columns.size(); i++) {
prop.putHTML("showtable_columns_" + i + "_header", columns.get(i)); prop.putHTML("showtable_columns_" + i + "_header", columns.get(i));
@ -80,21 +146,20 @@ public class table_p {
prop.put("showtable_columns", columns.size()); prop.put("showtable_columns", columns.size());
// insert all rows // insert all rows
int maxCount;
try { try {
maxCount = Math.min(1000, sb.tables.size(table)); maxcount = Math.min(maxcount, sb.tables.size(table));
} catch (final IOException e) { } catch (final IOException e) {
Log.logException(e); Log.logException(e);
maxCount = 0; maxcount = 0;
} }
int count = 0; int count = 0;
try { try {
final Iterator<Tables.Row> plainIterator = sb.tables.iterator(table); final Iterator<Tables.Row> plainIterator = sb.tables.iterator(table, matcher);
final Iterator<Tables.Row> mapIterator = sb.tables.orderByPK(plainIterator, maxCount).iterator(); final Iterator<Tables.Row> mapIterator = sb.tables.orderByPK(plainIterator, maxcount).iterator();
Tables.Row trow; Tables.Row trow;
boolean dark = true; boolean dark = true;
String cellName, cellValue; String cellName, cellValue;
rowloop: while ((mapIterator.hasNext()) && (count < maxCount)) { rowloop: while ((mapIterator.hasNext()) && (count < maxcount)) {
trow = mapIterator.next(); trow = mapIterator.next();
if (row == null) continue; if (row == null) continue;
prop.put("showtable_list_" + count + "_dark", ((dark) ? 1 : 0) ); dark=!dark; prop.put("showtable_list_" + count + "_dark", ((dark) ? 1 : 0) ); dark=!dark;

38
htroot/api/table_p.xml
View File

@ -1,18 +1,24 @@
#(showtable)#:: <tables count="#[tablecount]#">
<table name="#[table]#" count="#[num]#"> #(showtable)#
<columns> #{tables}#
#{columns}# <table name="#[table]#" count="#[num]#"/>
<column name="#[header]#" /> #{/tables}#
#{/columns}# ::
</columns> <table name="#[table]#" count="#[num]#">
<rows> <columns>
#{list}#
<row#(showpk)#:: key="#[pk]#"#(/showpk)#>
#{columns}# #{columns}#
<column name="#[column]#">#[cell]#</column> <column name="#[header]#" />
#{/columns}# #{/columns}#
</row> </columns>
#{/list}# <rows>
</rows> #{list}#
</table> <row#(showpk)#:: key="#[pk]#"#(/showpk)#>
#(/showtable)# #{columns}#
<column name="#[column]#">#[cell]#</column>
#{/columns}#
</row>
#{/list}#
</rows>
</table>
#(/showtable)#
</tables>

239
source/net/yacy/kelondro/blob/BEncodedHeap.java
View File

@ -7,7 +7,7 @@
// $LastChangedBy$ // $LastChangedBy$
// //
// LICENSE // LICENSE
// //
// This program is free software; you can redistribute it and/or modify // This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by // it under the terms of the GNU General Public License as published by
// the Free Software Foundation; either version 2 of the License, or // the Free Software Foundation; either version 2 of the License, or
@ -45,9 +45,9 @@ import net.yacy.kelondro.order.ByteOrder;
import net.yacy.kelondro.order.Digest; import net.yacy.kelondro.order.Digest;
import net.yacy.kelondro.order.NaturalOrder; import net.yacy.kelondro.order.NaturalOrder;
import net.yacy.kelondro.util.BDecoder; import net.yacy.kelondro.util.BDecoder;
import net.yacy.kelondro.util.BDecoder.BObject;
import net.yacy.kelondro.util.BEncoder; import net.yacy.kelondro.util.BEncoder;
import net.yacy.kelondro.util.FileUtils; import net.yacy.kelondro.util.FileUtils;
import net.yacy.kelondro.util.BDecoder.BObject;
/** /**
* store a table of properties (instead of fixed-field entries) * store a table of properties (instead of fixed-field entries)
@ -56,8 +56,8 @@ import net.yacy.kelondro.util.BDecoder.BObject;
public class BEncodedHeap implements Map<byte[], Map<String, byte[]>>, Iterable<Map.Entry<byte[], Map<String, byte[]>>> { public class BEncodedHeap implements Map<byte[], Map<String, byte[]>>, Iterable<Map.Entry<byte[], Map<String, byte[]>>> {
private Heap table; private Heap table;
private LinkedHashSet<String> columnames; private final LinkedHashSet<String> columnames;
/** /**
* produce or open a properties table * produce or open a properties table
* @param location the file * @param location the file
@ -70,11 +70,11 @@ public class BEncodedHeap implements Map<byte[], Map<String, byte[]>>, Iterable<
final File location, final File location,
final int keylength, final int keylength,
final ByteOrder ordering, final ByteOrder ordering,
int buffermax) throws IOException { final int buffermax) throws IOException {
this.table = new Heap(location, keylength, ordering, buffermax); this.table = new Heap(location, keylength, ordering, buffermax);
this.columnames = new LinkedHashSet<String>(); this.columnames = new LinkedHashSet<String>();
} }
/** /**
* convenience method to open a properies table * convenience method to open a properies table
* @param location the file * @param location the file
@ -86,72 +86,72 @@ public class BEncodedHeap implements Map<byte[], Map<String, byte[]>>, Iterable<
this.table = new Heap(location, keylength, NaturalOrder.naturalOrder, 100); this.table = new Heap(location, keylength, NaturalOrder.naturalOrder, 100);
this.columnames = new LinkedHashSet<String>(); this.columnames = new LinkedHashSet<String>();
} }
public byte[] encodedKey(String key) { public byte[] encodedKey(final String key) {
return Base64Order.enhancedCoder.encodeSubstring(Digest.encodeMD5Raw(key), this.table.keylength); return Base64Order.enhancedCoder.encodeSubstring(Digest.encodeMD5Raw(key), this.table.keylength);
} }
private static class EntryIter implements Iterator<Map.Entry<byte[], Map<String, byte[]>>> { private static class EntryIter implements Iterator<Map.Entry<byte[], Map<String, byte[]>>> {
HeapReader.entries iter; HeapReader.entries iter;
public EntryIter(File location, int keylen) throws IOException { public EntryIter(final File location, final int keylen) throws IOException {
iter = new HeapReader.entries(location, keylen); this.iter = new HeapReader.entries(location, keylen);
} }
public boolean hasNext() { public boolean hasNext() {
return iter.hasNext(); return this.iter.hasNext();
} }
public Entry<byte[], Map<String, byte[]>> next() { public Entry<byte[], Map<String, byte[]>> next() {
Map.Entry<byte[], byte[]> entry = iter.next(); final Map.Entry<byte[], byte[]> entry = this.iter.next();
Map<String, byte[]> map = b2m(entry.getValue()); final Map<String, byte[]> map = b2m(entry.getValue());
return new b2mEntry(entry.getKey(), map); return new b2mEntry(entry.getKey(), map);
} }
public void remove() { public void remove() {
throw new UnsupportedOperationException(); throw new UnsupportedOperationException();
} }
} }
public static class b2mEntry implements Map.Entry<byte[], Map<String, byte[]>> { public static class b2mEntry implements Map.Entry<byte[], Map<String, byte[]>> {
private final byte[] s; private final byte[] s;
private Map<String,byte[]> b; private Map<String,byte[]> b;
public b2mEntry(final byte[] s, final Map<String, byte[]> b) { public b2mEntry(final byte[] s, final Map<String, byte[]> b) {
this.s = s; this.s = s;
this.b = b; this.b = b;
} }
public byte[] getKey() { public byte[] getKey() {
return s; return this.s;
} }
public Map<String, byte[]> getValue() { public Map<String, byte[]> getValue() {
return b; return this.b;
} }
public Map<String, byte[]> setValue(Map<String, byte[]> value) { public Map<String, byte[]> setValue(final Map<String, byte[]> value) {
Map<String, byte[]> b1 = b; final Map<String, byte[]> b1 = this.b;
b = value; this.b = value;
return b1; return b1;
} }
} }
private static Map<String, byte[]> b2m(byte[] b) { private static Map<String, byte[]> b2m(final byte[] b) {
if (b == null) return null; if (b == null) return null;
//System.out.println("b = " + UTF8.String(b)); //System.out.println("b = " + UTF8.String(b));
BDecoder decoder = new BDecoder(b); final BDecoder decoder = new BDecoder(b);
BObject bobj = decoder.parse(); final BObject bobj = decoder.parse();
if (bobj.getType() != BDecoder.BType.dictionary) return null; if (bobj.getType() != BDecoder.BType.dictionary) return null;
Map<String, BDecoder.BObject> map = bobj.getMap(); final Map<String, BDecoder.BObject> map = bobj.getMap();
Map<String, byte[]> m = new HashMap<String, byte[]>(); final Map<String, byte[]> m = new HashMap<String, byte[]>();
for (final Map.Entry<String, BDecoder.BObject> entry: map.entrySet()) { for (final Map.Entry<String, BDecoder.BObject> entry: map.entrySet()) {
if (entry.getValue().getType() != BDecoder.BType.string) continue; if (entry.getValue().getType() != BDecoder.BType.string) continue;
m.put(entry.getKey(), entry.getValue().getString()); m.put(entry.getKey(), entry.getValue().getString());
} }
return m; return m;
} }
/** /**
* the map is stored inside a file; this method may return the file * the map is stored inside a file; this method may return the file
* @return the file where the map is stored * @return the file where the map is stored
@ -167,76 +167,76 @@ public class BEncodedHeap implements Map<byte[], Map<String, byte[]>>, Iterable<
public int size() { public int size() {
return this.table.size(); return this.table.size();
} }
/** /**
* return true if the table is empty * return true if the table is empty
*/ */
public boolean isEmpty() { public boolean isEmpty() {
return this.table.size() == 0; return this.table.size() == 0;
} }
/** /**
* check if a row with given key exists in the table * check if a row with given key exists in the table
* @param name * @param name
* @return true if the row exists * @return true if the row exists
*/ */
public boolean containsKey(byte[] pk) { public boolean containsKey(final byte[] pk) {
return this.table.containsKey(pk); return this.table.containsKey(pk);
} }
/** /**
* check if a row with given key exists in the table * check if a row with given key exists in the table
* This method is here to implement the Map interface * This method is here to implement the Map interface
* @param name * @param name
* @return true if the row exists * @return true if the row exists
*/ */
public boolean containsKey(Object key) { public boolean containsKey(final Object key) {
if (key instanceof byte[]) return containsKey((byte[]) key); if (key instanceof byte[]) return containsKey((byte[]) key);
return false; return false;
} }
/** /**
* the containsValue method cannot be used in this method * the containsValue method cannot be used in this method
* and is only here to implement the Map interface * and is only here to implement the Map interface
*/ */
public boolean containsValue(Object value) { public boolean containsValue(final Object value) {
// this method shall not be used because it is not appropriate for this kind of data // this method shall not be used because it is not appropriate for this kind of data
throw new UnsupportedOperationException(); throw new UnsupportedOperationException();
} }
/** /**
* get a map from the table * get a map from the table
* @param name * @param name
* @return the map if one found or NULL if no entry exists or the entry is corrupt * @return the map if one found or NULL if no entry exists or the entry is corrupt
* @throws RowSpaceExceededException * @throws RowSpaceExceededException
* @throws IOException * @throws IOException
*/ */
public Map<String, byte[]> get(byte[] pk) throws IOException, RowSpaceExceededException { public Map<String, byte[]> get(final byte[] pk) throws IOException, RowSpaceExceededException {
byte[] b = this.table.get(pk); final byte[] b = this.table.get(pk);
if (b == null) return null; if (b == null) return null;
return b2m(b); return b2m(b);
} }
/** /**
* get a map from the table * get a map from the table
* this method is here to implement the Map interface * this method is here to implement the Map interface
* @param name * @param name
* @return the map if one found or NULL if no entry exists or the entry is corrupt * @return the map if one found or NULL if no entry exists or the entry is corrupt
*/ */
public Map<String, byte[]> get(Object key) { public Map<String, byte[]> get(final Object key) {
if (key instanceof byte[]) if (key instanceof byte[])
try { try {
return get((byte[]) key); return get((byte[]) key);
} catch (IOException e) { } catch (final IOException e) {
Log.logException(e); Log.logException(e);
return null; return null;
} catch (RowSpaceExceededException e) { } catch (final RowSpaceExceededException e) {
Log.logException(e); Log.logException(e);
return null; return null;
} }
return null; return null;
} }
/** /**
* convenience method to get a value from a map * convenience method to get a value from a map
* @param pk * @param pk
@ -245,10 +245,10 @@ public class BEncodedHeap implements Map<byte[], Map<String, byte[]>>, Iterable<
* @throws IOException * @throws IOException
* @throws RowSpaceExceededException * @throws RowSpaceExceededException
*/ */
public byte[] getProp(byte[] pk, String key) throws IOException, RowSpaceExceededException { public byte[] getProp(final byte[] pk, final String key) throws IOException, RowSpaceExceededException {
byte[] b = this.table.get(pk); final byte[] b = this.table.get(pk);
if (b == null) return null; if (b == null) return null;
Map<String, byte[]> map = b2m(b); final Map<String, byte[]> map = b2m(b);
return map.get(key); return map.get(key);
} }
@ -259,12 +259,12 @@ public class BEncodedHeap implements Map<byte[], Map<String, byte[]>>, Iterable<
* @param columnMatcher the matcher for the elements of the column * @param columnMatcher the matcher for the elements of the column
* @return a set of primary keys where the matcher matched * @return a set of primary keys where the matcher matched
*/ */
public Set<byte[]> select(String columnName, Pattern columnMatcher) { public Set<byte[]> select(final String columnName, final Pattern columnMatcher) {
Iterator<Map.Entry<byte[], Map<String, byte[]>>> i = iterator(); final Iterator<Map.Entry<byte[], Map<String, byte[]>>> i = iterator();
Map.Entry<byte[], Map<String, byte[]>> row; Map.Entry<byte[], Map<String, byte[]>> row;
Map<String, byte[]> prop; Map<String, byte[]> prop;
byte[] val; byte[] val;
Set<byte[]> pks = new TreeSet<byte[]>(this.table.ordering); final Set<byte[]> pks = new TreeSet<byte[]>(this.table.ordering);
while (i.hasNext()) { while (i.hasNext()) {
row = i.next(); row = i.next();
prop = row.getValue(); prop = row.getValue();
@ -283,8 +283,8 @@ public class BEncodedHeap implements Map<byte[], Map<String, byte[]>>, Iterable<
* @param columnMatcher the matcher for the elements of the column * @param columnMatcher the matcher for the elements of the column
* @return the row where the matcher matched the given column * @return the row where the matcher matched the given column
*/ */
public Map.Entry<byte[], Map<String, byte[]>> selectOne(String columnName, Pattern columnMatcher) { public Map.Entry<byte[], Map<String, byte[]>> selectOne(final String columnName, final Pattern columnMatcher) {
Iterator<Map.Entry<byte[], Map<String, byte[]>>> i = iterator(); final Iterator<Map.Entry<byte[], Map<String, byte[]>>> i = iterator();
Map.Entry<byte[], Map<String, byte[]>> row; Map.Entry<byte[], Map<String, byte[]>> row;
Map<String, byte[]> prop; Map<String, byte[]> prop;
byte[] val; byte[] val;
@ -298,7 +298,7 @@ public class BEncodedHeap implements Map<byte[], Map<String, byte[]>>, Iterable<
} }
return null; return null;
} }
/** /**
* insert a map into the table * insert a map into the table
* this method shall be used in exchange of the get method if the * this method shall be used in exchange of the get method if the
@ -308,20 +308,20 @@ public class BEncodedHeap implements Map<byte[], Map<String, byte[]>>, Iterable<
* @throws RowSpaceExceededException * @throws RowSpaceExceededException
* @throws IOException * @throws IOException
*/ */
public void insert(byte[] pk, Map<String, byte[]> map) throws RowSpaceExceededException, IOException { public void insert(final byte[] pk, final Map<String, byte[]> map) throws RowSpaceExceededException, IOException {
byte[] b = BEncoder.encode(BEncoder.transcode(map)); final byte[] b = BEncoder.encode(BEncoder.transcode(map));
this.table.insert(pk, b); this.table.insert(pk, b);
this.columnames.addAll(map.keySet()); this.columnames.addAll(map.keySet());
} }
public void insert(byte[] pk, String key, byte[] value) throws IOException { public void insert(final byte[] pk, final String key, final byte[] value) throws IOException {
byte[] b = BEncoder.encodeMap(key, value); final byte[] b = BEncoder.encodeMap(key, value);
this.table.insert(pk, b); this.table.insert(pk, b);
this.columnames.add(key); this.columnames.add(key);
} }
public void update(byte[] pk, Map<String, byte[]> map) throws RowSpaceExceededException, IOException { public void update(final byte[] pk, final Map<String, byte[]> map) throws RowSpaceExceededException, IOException {
Map<String, byte[]> entry = this.get(pk); final Map<String, byte[]> entry = this.get(pk);
if (entry == null) { if (entry == null) {
insert(pk, map); insert(pk, map);
} else { } else {
@ -329,8 +329,8 @@ public class BEncodedHeap implements Map<byte[], Map<String, byte[]>>, Iterable<
insert(pk, entry); insert(pk, entry);
} }
} }
public void update(byte[] pk, String key, byte[] value) throws RowSpaceExceededException, IOException { public void update(final byte[] pk, final String key, final byte[] value) throws RowSpaceExceededException, IOException {
Map<String, byte[]> entry = this.get(pk); Map<String, byte[]> entry = this.get(pk);
if (entry == null) { if (entry == null) {
entry = new HashMap<String, byte[]>(); entry = new HashMap<String, byte[]>();
@ -347,74 +347,74 @@ public class BEncodedHeap implements Map<byte[], Map<String, byte[]>>, Iterable<
* @param name * @param name
* @param map * @param map
*/ */
public Map<String, byte[]> put(byte[] pk, Map<String, byte[]> map) { public Map<String, byte[]> put(final byte[] pk, final Map<String, byte[]> map) {
try { try {
Map<String, byte[]> entry = this.get(pk); final Map<String, byte[]> entry = this.get(pk);
byte[] b = BEncoder.encode(BEncoder.transcode(map)); final byte[] b = BEncoder.encode(BEncoder.transcode(map));
this.table.insert(pk, b); this.table.insert(pk, b);
this.columnames.addAll(map.keySet()); this.columnames.addAll(map.keySet());
return entry; return entry;
} catch (IOException e) { } catch (final IOException e) {
Log.logException(e); Log.logException(e);
return null; return null;
} catch (RowSpaceExceededException e) { } catch (final RowSpaceExceededException e) {
Log.logException(e); Log.logException(e);
return null; return null;
} }
} }
/** /**
* delete a map from the table * delete a map from the table
* @param name * @param name
* @throws IOException * @throws IOException
*/ */
public void delete(byte[] pk) throws IOException { public void delete(final byte[] pk) throws IOException {
this.table.delete(pk); this.table.delete(pk);
} }
/** /**
* delete a map from the table * delete a map from the table
* @param name * @param name
* @throws RowSpaceExceededException * @throws RowSpaceExceededException
* @throws IOException * @throws IOException
*/ */
public Map<String, byte[]> remove(byte[] key) throws IOException, RowSpaceExceededException { public Map<String, byte[]> remove(final byte[] key) throws IOException, RowSpaceExceededException {
Map<String, byte[]> value = get(key); final Map<String, byte[]> value = get(key);
this.delete(key); delete(key);
return value; return value;
} }
public Map<String, byte[]> remove(Object key) { public Map<String, byte[]> remove(final Object key) {
if (key instanceof byte[]) if (key instanceof byte[])
try { try {
return remove((byte[]) key); return remove((byte[]) key);
} catch (IOException e) { } catch (final IOException e) {
Log.logException(e); Log.logException(e);
return null; return null;
} catch (RowSpaceExceededException e) { } catch (final RowSpaceExceededException e) {
Log.logException(e); Log.logException(e);
return null; return null;
} }
return null; return null;
} }
/** /**
* Copy all the mappings from the specified map to this map. * Copy all the mappings from the specified map to this map.
* *
* @param m mappings to be stored in this map * @param m mappings to be stored in this map
*/ */
public void putAll(Map<? extends byte[], ? extends Map<String, byte[]>> map) { public void putAll(final Map<? extends byte[], ? extends Map<String, byte[]>> map) {
for (final Map.Entry<? extends byte[], ? extends Map<String, byte[]>> me: map.entrySet()) { for (final Map.Entry<? extends byte[], ? extends Map<String, byte[]>> me: map.entrySet()) {
try { try {
this.insert(me.getKey(), me.getValue()); this.insert(me.getKey(), me.getValue());
} catch (RowSpaceExceededException e) { } catch (final RowSpaceExceededException e) {
Log.logException(e); Log.logException(e);
} catch (IOException e) { } catch (final IOException e) {
Log.logException(e); Log.logException(e);
} }
} }
} }
/** /**
* remove all entries from the map; * remove all entries from the map;
* possibly removes the backend-file * possibly removes the backend-file
@ -422,11 +422,12 @@ public class BEncodedHeap implements Map<byte[], Map<String, byte[]>>, Iterable<
public void clear() { public void clear() {
try { try {
this.table.clear(); this.table.clear();
} catch (IOException e) { this.columnames.clear();
} catch (final IOException e) {
Log.logException(e); Log.logException(e);
} }
} }
/** /**
* close the backen-file. * close the backen-file.
* Should be called explicitely to ensure that all data * Should be called explicitely to ensure that all data
@ -435,20 +436,20 @@ public class BEncodedHeap implements Map<byte[], Map<String, byte[]>>, Iterable<
public void close() { public void close() {
this.table.close(); this.table.close();
} }
/** /**
* Return a Set of the keys contained in this map. * Return a Set of the keys contained in this map.
* This may not be a useful method, if possible use the keys() * This may not be a useful method, if possible use the keys()
* method instead to iterate all keys from the backend-file * method instead to iterate all keys from the backend-file
* *
* @return a set view of the keys contained in this map * @return a set view of the keys contained in this map
*/ */
public Set<byte[]> keySet() { public Set<byte[]> keySet() {
TreeSet<byte[]> set = new TreeSet<byte[]>(this.table.ordering); final TreeSet<byte[]> set = new TreeSet<byte[]>(this.table.ordering);
try { try {
Iterator<byte[]> i = this.table.keys(true, false); final Iterator<byte[]> i = this.table.keys(true, false);
while (i.hasNext()) set.add(i.next()); while (i.hasNext()) set.add(i.next());
} catch (IOException e) {} } catch (final IOException e) {}
return set; return set;
} }
@ -473,7 +474,7 @@ public class BEncodedHeap implements Map<byte[], Map<String, byte[]>>, Iterable<
// this method shall not be used because it is not appropriate for this kind of data // this method shall not be used because it is not appropriate for this kind of data
throw new UnsupportedOperationException(); throw new UnsupportedOperationException();
} }
/** /**
* The abstract method entrySet() from AbstractMap must be implemented, * The abstract method entrySet() from AbstractMap must be implemented,
* but never used because that is not useful for this file-based storage class. * but never used because that is not useful for this file-based storage class.
@ -482,14 +483,14 @@ public class BEncodedHeap implements Map<byte[], Map<String, byte[]>>, Iterable<
* methods must be overriden in this class. These methods are: * methods must be overriden in this class. These methods are:
* size, containsValue, containsKey, get, remove, putAll, clear, * size, containsValue, containsKey, get, remove, putAll, clear,
* keySet, values, equals, hashCode and toString * keySet, values, equals, hashCode and toString
* *
* Instead of using this method, use the iterator() method to iterate * Instead of using this method, use the iterator() method to iterate
* all elements in the back-end blob file * all elements in the back-end blob file
*/ */
public Set<Map.Entry<byte[], Map<String, byte[]>>> entrySet() { public Set<Map.Entry<byte[], Map<String, byte[]>>> entrySet() {
throw new UnsupportedOperationException(); throw new UnsupportedOperationException();
} }
/** /**
* iterate all rows of the table. * iterate all rows of the table.
* This method implements the * This method implements the
@ -497,20 +498,20 @@ public class BEncodedHeap implements Map<byte[], Map<String, byte[]>>, Iterable<
* interface * interface
*/ */
public Iterator<Map.Entry<byte[], Map<String, byte[]>>> iterator() { public Iterator<Map.Entry<byte[], Map<String, byte[]>>> iterator() {
File location = this.table.location(); final File location = this.table.location();
int keylen = this.table.keylength(); final int keylen = this.table.keylength();
try { try {
this.table.flushBuffer(); this.table.flushBuffer();
return new EntryIter(location, keylen); return new EntryIter(location, keylen);
} catch (IOException e1) { } catch (final IOException e1) {
ByteOrder order = this.table.ordering(); final ByteOrder order = this.table.ordering();
int buffermax = this.table.getBuffermax(); final int buffermax = this.table.getBuffermax();
this.table.close(); this.table.close();
try { try {
Iterator<Map.Entry<byte[], Map<String, byte[]>>> iter = new EntryIter(location, keylen); final Iterator<Map.Entry<byte[], Map<String, byte[]>>> iter = new EntryIter(location, keylen);
this.table = new Heap(location, keylen, order, buffermax); this.table = new Heap(location, keylen, order, buffermax);
return iter; return iter;
} catch (IOException e) { } catch (final IOException e) {
Log.logSevere("PropertiesTable", e.getMessage(), e); Log.logSevere("PropertiesTable", e.getMessage(), e);
return null; return null;
} }
@ -525,11 +526,11 @@ public class BEncodedHeap implements Map<byte[], Map<String, byte[]>>, Iterable<
* @return * @return
* @throws IOException * @throws IOException
*/ */
public static Iterator<Map.Entry<byte[], Map<String, byte[]>>> iterator(File location, int keylen) throws IOException { public static Iterator<Map.Entry<byte[], Map<String, byte[]>>> iterator(final File location, final int keylen) throws IOException {
return new EntryIter(location, keylen); return new EntryIter(location, keylen);
} }
/** /**
* a hashcode for the object * a hashcode for the object
*/ */
@ -552,51 +553,51 @@ public class BEncodedHeap implements Map<byte[], Map<String, byte[]>>, Iterable<
this.columnames.addAll(row.getValue().keySet()); this.columnames.addAll(row.getValue().keySet());
} }
} }
ArrayList<String> l = new ArrayList<String>(); final ArrayList<String> l = new ArrayList<String>();
l.addAll(this.columnames); l.addAll(this.columnames);
return l; return l;
} }
public static void main(String[] args) { public static void main(final String[] args) {
if (args.length == 0) { if (args.length == 0) {
// test the class // test the class
File f = new File(new File("maptest").getAbsolutePath()); final File f = new File(new File("maptest").getAbsolutePath());
//System.out.println(f.getAbsolutePath()); //System.out.println(f.getAbsolutePath());
//System.out.println(f.getParent()); //System.out.println(f.getParent());
if (f.exists()) FileUtils.deletedelete(f); if (f.exists()) FileUtils.deletedelete(f);
try { try {
BEncodedHeap map = new BEncodedHeap(f, 4); final BEncodedHeap map = new BEncodedHeap(f, 4);
// put some values into the map // put some values into the map
Map<String, byte[]> m = new HashMap<String, byte[]>(); final Map<String, byte[]> m = new HashMap<String, byte[]>();
m.put("k", "000".getBytes()); map.insert("123".getBytes(), m); m.put("k", "000".getBytes()); map.insert("123".getBytes(), m);
m.put("k", "111".getBytes()); map.insert("456".getBytes(), m); m.put("k", "111".getBytes()); map.insert("456".getBytes(), m);
m.put("k", "222".getBytes()); map.insert("789".getBytes(), m); m.put("k", "222".getBytes()); map.insert("789".getBytes(), m);
// iterate over keys // iterate over keys
Map.Entry<byte[], Map<String, byte[]>> entry; Map.Entry<byte[], Map<String, byte[]>> entry;
Iterator<Map.Entry<byte[], Map<String, byte[]>>> i = map.iterator(); final Iterator<Map.Entry<byte[], Map<String, byte[]>>> i = map.iterator();
while (i.hasNext()) { while (i.hasNext()) {
entry = i.next(); entry = i.next();
System.out.println(ASCII.String(entry.getKey()) + ": " + entry.getValue()); System.out.println(ASCII.String(entry.getKey()) + ": " + entry.getValue());
} }
// clean up // clean up
map.close(); map.close();
} catch (IOException e) { } catch (final IOException e) {
Log.logException(e); Log.logException(e);
} catch (RowSpaceExceededException e) { } catch (final RowSpaceExceededException e) {
Log.logException(e); Log.logException(e);
} }
} else { } else {
File f = new File(args[0]); final File f = new File(args[0]);
try { try {
Map.Entry<byte[], Map<String, byte[]>> entry; Map.Entry<byte[], Map<String, byte[]>> entry;
BEncodedHeap map = new BEncodedHeap(f, 12); final BEncodedHeap map = new BEncodedHeap(f, 12);
Iterator<Map.Entry<byte[], Map<String, byte[]>>> i = map.iterator(); final Iterator<Map.Entry<byte[], Map<String, byte[]>>> i = map.iterator();
while (i.hasNext()) { while (i.hasNext()) {
entry = i.next(); entry = i.next();
System.out.println(ASCII.String(entry.getKey()) + ": " + entry.getValue()); System.out.println(ASCII.String(entry.getKey()) + ": " + entry.getValue());
} }
map.close(); map.close();
} catch (IOException e) { } catch (final IOException e) {
Log.logException(e); Log.logException(e);
} }
} }

37
source/net/yacy/kelondro/blob/Tables.java
View File

@ -51,7 +51,7 @@ import net.yacy.kelondro.util.FileUtils;
import net.yacy.kelondro.util.LookAheadIterator; import net.yacy.kelondro.util.LookAheadIterator;
public class Tables { public class Tables implements Iterable<String> {
private static final String suffix = ".bheap"; private static final String suffix = ".bheap";
private static final String system_table_pkcounter = "pkcounter"; private static final String system_table_pkcounter = "pkcounter";
@ -88,7 +88,8 @@ public class Tables {
} }
} }
public Iterator<String> tables() { @Override
public Iterator<String> iterator() {
return this.tables.keySet().iterator(); return this.tables.keySet().iterator();
} }
@ -104,17 +105,20 @@ public class Tables {
} }
public void clear(final String tablename) { public void clear(final String tablename) {
BEncodedHeap heap = null;
try { try {
heap = getHeap(tablename); BEncodedHeap heap = getHeap(tablename);
if (heap != null) heap.clear(); if (heap != null) {
heap.close(); final File f = heap.getFile();
heap.clear();
heap.close();
FileUtils.deletedelete(f);
heap = null;
}
} catch (final IOException e) { } catch (final IOException e) {
Log.logException(e);
} finally {
this.tables.remove(tablename);
} }
heap = null;
final File f = new File(this.location, tablename + suffix);
this.tables.remove(tablename);
FileUtils.deletedelete(f);
} }
public boolean hasHeap(final String tablename) { public boolean hasHeap(final String tablename) {
@ -137,6 +141,14 @@ public class Tables {
return heap; return heap;
} }
/**
* get the total number of known tables
* @return
*/
public int size() {
return this.tables.size();
}
public int size(final String table) throws IOException { public int size(final String table) throws IOException {
final BEncodedHeap heap = getHeap(table); final BEncodedHeap heap = getHeap(table);
return heap.size(); return heap.size();
@ -344,7 +356,7 @@ public class Tables {
public RowIterator(final String table, final String whereColumn, final Pattern wherePattern) throws IOException { public RowIterator(final String table, final String whereColumn, final Pattern wherePattern) throws IOException {
this.whereColumn = whereColumn; this.whereColumn = whereColumn;
this.whereValue = null; this.whereValue = null;
this.wherePattern = wherePattern; this.wherePattern = wherePattern == null || wherePattern.toString().length() == 0 ? null : wherePattern;
final BEncodedHeap heap = getHeap(table); final BEncodedHeap heap = getHeap(table);
this.i = heap.iterator(); this.i = heap.iterator();
} }
@ -359,7 +371,7 @@ public class Tables {
public RowIterator(final String table, final Pattern pattern) throws IOException { public RowIterator(final String table, final Pattern pattern) throws IOException {
this.whereColumn = null; this.whereColumn = null;
this.whereValue = null; this.whereValue = null;
this.wherePattern = pattern; this.wherePattern = pattern == null || pattern.toString().length() == 0 ? null : pattern;
final BEncodedHeap heap = getHeap(table); final BEncodedHeap heap = getHeap(table);
this.i = heap.iterator(); this.i = heap.iterator();
} }
@ -541,4 +553,5 @@ public class Tables {
Log.logException(e); Log.logException(e);
} }
} }
} }