Marown/yacy_search_server
1
0
Fork 0
mirror of https://github.com/yacy/yacy_search_server.git synced 2024-09-19 00:01:41 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity

- MapHeap now implements Map<byte[], Map<String, String>>

Browse Source 
- refactoring of method names to comply with Map method names

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@7072 6c8d7289-2bf4-0310-a012-ef5d649a1542
This commit is contained in:
orbiter 2010-08-24 12:36:56 +00:00
parent cf07b34c2d
commit 0f276dd63f
21 changed files with 271 additions and 156 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
source/de/anomic/crawler/CrawlProfile.java
View File

@ -136,7 +136,7 @@ public class CrawlProfile {
public void removeEntry(final byte[] handle) {
try {
profileTable.remove(handle);
profileTable.delete(handle);
} catch (final IOException e) {
Log.logException(e);
}
@ -145,11 +145,11 @@ public class CrawlProfile {
public entry newEntry(final Map<String, String> mem) {
final entry ne = new entry(mem);
try {
profileTable.put(ne.handle().getBytes(), ne.map());
profileTable.insert(ne.handle().getBytes(), ne.map());
} catch (final Exception e) {
clear();
try {
profileTable.put(ne.handle().getBytes(), ne.map());
profileTable.insert(ne.handle().getBytes(), ne.map());
} catch (final Exception ee) {
Log.logException(e);
System.exit(0);
@ -182,11 +182,11 @@ public class CrawlProfile {
xsstopw, xdstopw, xpstopw,
cacheStrategy);
try {
profileTable.put(ne.handle().getBytes(), ne.map());
profileTable.insert(ne.handle().getBytes(), ne.map());
} catch (final Exception e) {
clear();
try {
profileTable.put(ne.handle().getBytes(), ne.map());
profileTable.insert(ne.handle().getBytes(), ne.map());
} catch (final Exception ee) {
Log.logException(e);
System.exit(0);
@ -196,7 +196,7 @@ public class CrawlProfile {
}
public boolean hasEntry(final String handle) {
return profileTable.has(handle.getBytes());
return profileTable.containsKey(handle.getBytes());
}
public entry getEntry(final String handle) {
@ -218,7 +218,7 @@ public class CrawlProfile {
public void changeEntry(final entry e, final String propName, final String newValue) throws IOException, RowSpaceExceededException {
e.mem.put(propName, newValue);
assert e.handle() != null;
profileTable.put(e.handle().getBytes(), e.mem);
profileTable.insert(e.handle().getBytes(), e.mem);
}
public long getRecrawlDate(final long oldTimeMinutes) {

4
source/de/anomic/data/BookmarkDate.java
View File

@ -144,13 +144,13 @@ public class BookmarkDate {
public void setDatesTable() {
if (this.size() >0) {
try {
datesTable.put(getDateString().getBytes(), mem);
datesTable.insert(getDateString().getBytes(), mem);
} catch (Exception e) {
Log.logException(e);
}
} else {
try {
datesTable.remove(getDateString().getBytes());
datesTable.delete(getDateString().getBytes());
} catch (IOException e) {
Log.logException(e);
}

6
source/de/anomic/data/blogBoard.java
View File

@ -81,7 +81,7 @@ public class blogBoard {
* @return true if the database contains the element, else false
*/
public boolean contains(final String key) {
return database.has(key.getBytes());
return database.containsKey(key.getBytes());
}
public void close() {
@ -122,7 +122,7 @@ public class blogBoard {
public String writeBlogEntry(final BlogEntry page) {
String ret = null;
try {
database.put(page.key.getBytes(), page.record);
database.insert(page.key.getBytes(), page.record);
ret = page.key;
} catch (IOException ex) {
Log.logException(ex);
@ -240,7 +240,7 @@ public class blogBoard {
public void deleteBlogEntry(final String key) {
try {
database.remove(normalize(key).getBytes());
database.delete(normalize(key).getBytes());
} catch (final IOException e) { }
}

4
source/de/anomic/data/blogBoardComments.java
View File

@ -112,7 +112,7 @@ public class blogBoardComments {
public String write(final CommentEntry page) {
// writes a new page and returns key
try {
database.put(page.key.getBytes(), page.record);
database.insert(page.key.getBytes(), page.record);
return page.key;
} catch (final Exception e) {
Log.logException(e);
@ -231,7 +231,7 @@ public class blogBoardComments {
public void delete(final String key) {
try {
database.remove(normalize(key).getBytes());
database.delete(normalize(key).getBytes());
}
catch (final IOException e) { }
}

4
source/de/anomic/data/bookmarksDB.java
View File

@ -131,7 +131,7 @@ public class bookmarksDB {
// adding a bookmark to the bookmarksDB
public void saveBookmark(final Bookmark bookmark){
try {
bookmarks.put(bookmark.getUrlHash().getBytes(), bookmark.entry);
bookmarks.insert(bookmark.getUrlHash().getBytes(), bookmark.entry);
} catch (final Exception e) {
Log.logException(e);
}
@ -171,7 +171,7 @@ public class bookmarksDB {
Bookmark b;
try {
b = getBookmark(urlHash);
bookmarks.remove(urlHash.getBytes());
bookmarks.delete(urlHash.getBytes());
} catch (final IOException e) {
b = null;
}

4
source/de/anomic/data/messageBoard.java
View File

@ -183,7 +183,7 @@ public class messageBoard {
public String write(final entry message) {
// writes a message and returns key
try {
database.put(message.key.getBytes(), message.record);
database.insert(message.key.getBytes(), message.record);
return message.key;
} catch (final Exception e) {
Log.logException(e);
@ -207,7 +207,7 @@ public class messageBoard {
public void remove(final String key) {
try {
database.remove(key.getBytes());
database.delete(key.getBytes());
} catch (final IOException e) {
}
}

8
source/de/anomic/data/userDB.java
View File

@ -85,7 +85,7 @@ public final class userDB {
public void removeEntry(final String hostName) {
try {
userTable.remove(hostName.toLowerCase().getBytes());
userTable.delete(hostName.toLowerCase().getBytes());
} catch (final IOException e) {}
}
@ -114,7 +114,7 @@ public final class userDB {
public String addEntry(final Entry entry) {
try {
userTable.put(entry.userName.getBytes(), entry.mem);
userTable.insert(entry.userName.getBytes(), entry.mem);
return entry.userName;
} catch (final Exception e) {
Log.logException(e);
@ -488,7 +488,7 @@ public final class userDB {
}
try {
userDB.this.userTable.put(getUserName().getBytes(), this.mem);
userDB.this.userTable.insert(getUserName().getBytes(), this.mem);
} catch(final Exception e){
Log.logException(e);
}
@ -505,7 +505,7 @@ public final class userDB {
public void setProperty(final String propName, final String newValue) throws IOException, RowSpaceExceededException {
this.mem.put(propName, newValue);
userDB.this.userTable.put(getUserName().getBytes(), this.mem);
userDB.this.userTable.insert(getUserName().getBytes(), this.mem);
}
public String getProperty(final String propName, final String defaultValue) {

4
source/de/anomic/data/wiki/wikiBoard.java
View File

@ -266,9 +266,9 @@ public class wikiBoard {
//System.out.println("key = " + page.key);
//System.out.println("oldDate = " + oldDate);
//System.out.println("record = " + oldEntry.record.toString());
bkpbase.put((page.key + dateString(oldDate)).getBytes(), oldEntry.record);
bkpbase.insert((page.key + dateString(oldDate)).getBytes(), oldEntry.record);
// write the new page
datbase.put(page.key.getBytes(), page.record);
datbase.insert(page.key.getBytes(), page.record);
return page.key;
} catch (final Exception e) {
Log.logException(e);

44
source/de/anomic/http/client/Cache.java
View File

@ -56,7 +56,7 @@ public final class Cache {
private static final String RESPONSE_HEADER_DB_NAME = "responseHeader.heap";
private static final String FILE_DB_NAME = "file.array";
private static MapHeap responseHeaderDB = null;
private static Map<byte[], Map<String, String>> responseHeaderDB = null;
private static Compressor fileDB = null;
private static ArrayStack fileDBunbuffered = null;
@ -105,7 +105,9 @@ public final class Cache {
* close the databases
*/
public static void close() {
responseHeaderDB.close();
if (responseHeaderDB instanceof MapHeap) {
((MapHeap) responseHeaderDB).close();
}
fileDB.close(true);
}
@ -119,14 +121,18 @@ public final class Cache {
hm.putAll(responseHeader);
hm.put("@@URL", url.toNormalform(true, false));
try {
responseHeaderDB.put(url.hash(), hm);
if (responseHeaderDB instanceof MapHeap) {
((MapHeap) responseHeaderDB).insert(url.hash(), hm);
} else {
responseHeaderDB.put(url.hash(), hm);
}
} catch (Exception e) {
throw new IOException("Cache.store: cannot write to headerDB: " + e.getMessage());
}
// store the file
try {
fileDB.put(url.hash(), file);
fileDB.insert(url.hash(), file);
} catch (UnsupportedEncodingException e) {
throw new IOException("Cache.store: cannot write to fileDB (1): " + e.getMessage());
} catch (IOException e) {
@ -142,18 +148,22 @@ public final class Cache {
*/
public static boolean has(final DigestURI url) {
boolean headerExists;
headerExists = responseHeaderDB.has(url.hash());
boolean fileExists = fileDB.has(url.hash());
headerExists = responseHeaderDB.containsKey(url.hash());
boolean fileExists = fileDB.containsKey(url.hash());
if (headerExists && fileExists) return true;
if (!headerExists && !fileExists) return false;
// if not both is there then we do a clean-up
if (headerExists) try {
log.logWarning("header but not content of url " + url.toString() + " in cache; cleaned up");
responseHeaderDB.remove(url.hash());
if (responseHeaderDB instanceof MapHeap) {
((MapHeap) responseHeaderDB).delete(url.hash());
} else {
responseHeaderDB.remove(url.hash());
}
} catch (IOException e) {}
if (fileExists) try {
log.logWarning("content but not header of url " + url.toString() + " in cache; cleaned up");
fileDB.remove(url.hash());
fileDB.delete(url.hash());
} catch (IOException e) {}
return false;
}
@ -171,15 +181,7 @@ public final class Cache {
// loading data from database
Map<String, String> hdb;
try {
hdb = responseHeaderDB.get(url.hash());
} catch (final IOException e) {
Log.logException(e);
return null;
} catch (RowSpaceExceededException e) {
Log.logException(e);
return null;
}
hdb = responseHeaderDB.get(url.hash());
if (hdb == null) return null;
return new ResponseHeader(null, hdb);
@ -219,7 +221,11 @@ public final class Cache {
* @throws IOException
*/
public static void delete(final DigestURI url) throws IOException {
responseHeaderDB.remove(url.hash());
fileDB.remove(url.hash());
if (responseHeaderDB instanceof MapHeap) {
((MapHeap) responseHeaderDB).delete(url.hash());
} else {
responseHeaderDB.remove(url.hash());
}
fileDB.delete(url.hash());
}
}

56
source/de/anomic/yacy/yacySeedDB.java
View File

@ -269,9 +269,9 @@ public final class yacySeedDB implements AlternativeDomainNames {
if (seedActiveDB.isEmpty() && seedPassiveDB.isEmpty() && seedPotentialDB.isEmpty()) return; // avoid that the own seed is initialized too early
if (this.mySeed == null) initMySeed();
try {
seedActiveDB.remove(mySeed.hash.getBytes());
seedPassiveDB.remove(mySeed.hash.getBytes());
seedPotentialDB.remove(mySeed.hash.getBytes());
seedActiveDB.delete(mySeed.hash.getBytes());
seedPassiveDB.delete(mySeed.hash.getBytes());
seedPotentialDB.delete(mySeed.hash.getBytes());
} catch (final IOException e) { Log.logWarning("yacySeedDB", "could not remove hash ("+ e.getClass() +"): "+ e.getMessage()); }
}
@ -470,10 +470,10 @@ public final class yacySeedDB implements AlternativeDomainNames {
nameLookupCache.put(seed.getName(), seed.hash);
final Map<String, String> seedPropMap = seed.getMap();
synchronized (seedPropMap) {
seedActiveDB.put(seed.hash.getBytes(), seedPropMap);
seedActiveDB.insert(seed.hash.getBytes(), seedPropMap);
}
seedPassiveDB.remove(seed.hash.getBytes());
seedPotentialDB.remove(seed.hash.getBytes());
seedPassiveDB.delete(seed.hash.getBytes());
seedPotentialDB.delete(seed.hash.getBytes());
} catch (final Exception e) {
yacyCore.log.logSevere("ERROR add: seed.db corrupt (" + e.getMessage() + "); resetting seed.db", e);
resetActiveTable();
@ -484,14 +484,14 @@ public final class yacySeedDB implements AlternativeDomainNames {
if (seed.isProper(false) != null) return;
try {
nameLookupCache.remove(seed.getName());
seedActiveDB.remove(seed.hash.getBytes());
seedPotentialDB.remove(seed.hash.getBytes());
seedActiveDB.delete(seed.hash.getBytes());
seedPotentialDB.delete(seed.hash.getBytes());
} catch (final Exception e) { Log.logWarning("yacySeedDB", "could not remove hash ("+ e.getClass() +"): "+ e.getMessage()); }
//seed.put(yacySeed.LASTSEEN, yacyCore.shortFormatter.format(new Date(yacyCore.universalTime())));
try {
final Map<String, String> seedPropMap = seed.getMap();
synchronized (seedPropMap) {
seedPassiveDB.put(seed.hash.getBytes(), seedPropMap);
seedPassiveDB.insert(seed.hash.getBytes(), seedPropMap);
}
} catch (final Exception e) {
yacyCore.log.logSevere("ERROR add: seed.db corrupt (" + e.getMessage() + "); resetting seed.db", e);
@ -503,14 +503,14 @@ public final class yacySeedDB implements AlternativeDomainNames {
if (seed.isProper(false) != null) return;
try {
nameLookupCache.remove(seed.getName());
seedActiveDB.remove(seed.hash.getBytes());
seedPassiveDB.remove(seed.hash.getBytes());
seedActiveDB.delete(seed.hash.getBytes());
seedPassiveDB.delete(seed.hash.getBytes());
} catch (final Exception e) { Log.logWarning("yacySeedDB", "could not remove hash ("+ e.getClass() +"): "+ e.getMessage()); }
//seed.put(yacySeed.LASTSEEN, yacyCore.shortFormatter.format(new Date(yacyCore.universalTime())));
try {
final Map<String, String> seedPropMap = seed.getMap();
synchronized (seedPropMap) {
seedPotentialDB.put(seed.hash.getBytes(), seedPropMap);
seedPotentialDB.insert(seed.hash.getBytes(), seedPropMap);
}
} catch (final Exception e) {
yacyCore.log.logSevere("ERROR add: seed.db corrupt (" + e.getMessage() + "); resetting seed.db", e);
@ -521,27 +521,27 @@ public final class yacySeedDB implements AlternativeDomainNames {
public synchronized void removeDisconnected(final String peerHash) {
if(peerHash == null) return;
try {
seedPassiveDB.remove(peerHash.getBytes());
seedPassiveDB.delete(peerHash.getBytes());
} catch (final IOException e) { Log.logWarning("yacySeedDB", "could not remove hash ("+ e.getClass() +"): "+ e.getMessage()); }
}
public synchronized void removePotential(final String peerHash) {
if(peerHash == null) return;
try {
seedPotentialDB.remove(peerHash.getBytes());
seedPotentialDB.delete(peerHash.getBytes());
} catch (final IOException e) { Log.logWarning("yacySeedDB", "could not remove hash ("+ e.getClass() +"): "+ e.getMessage()); }
}
public boolean hasConnected(final byte[] hash) {
return seedActiveDB.has(hash);
return seedActiveDB.containsKey(hash);
}
public boolean hasDisconnected(final byte[] hash) {
return seedPassiveDB.has(hash);
return seedPassiveDB.containsKey(hash);
}
public boolean hasPotential(final byte[] hash) {
return seedPotentialDB.has(hash);
return seedPotentialDB.containsKey(hash);
}
private yacySeed get(final String hash, final MapDataMining database) {
@ -589,13 +589,13 @@ public final class yacySeedDB implements AlternativeDomainNames {
}
yacySeed s = get(hash, seedActiveDB);
if (s != null) try { seedActiveDB.put(hash.getBytes(), seed.getMap()); return;} catch (final Exception e) {Log.logException(e);}
if (s != null) try { seedActiveDB.insert(hash.getBytes(), seed.getMap()); return;} catch (final Exception e) {Log.logException(e);}
s = get(hash, seedPassiveDB);
if (s != null) try { seedPassiveDB.put(hash.getBytes(), seed.getMap()); return;} catch (final Exception e) {Log.logException(e);}
if (s != null) try { seedPassiveDB.insert(hash.getBytes(), seed.getMap()); return;} catch (final Exception e) {Log.logException(e);}
s = get(hash, seedPotentialDB);
if (s != null) try { seedPotentialDB.put(hash.getBytes(), seed.getMap()); return;} catch (final Exception e) {Log.logException(e);}
if (s != null) try { seedPotentialDB.insert(hash.getBytes(), seed.getMap()); return;} catch (final Exception e) {Log.logException(e);}
}
public yacySeed lookupByName(String peerName) {
@ -695,7 +695,7 @@ public final class yacySeedDB implements AlternativeDomainNames {
}
// delete bad peers
final Iterator<byte[]> i = badPeerHashes.iterator();
while (i.hasNext()) try {seedActiveDB.remove(i.next());} catch (final IOException e1) {Log.logException(e1);}
while (i.hasNext()) try {seedActiveDB.delete(i.next());} catch (final IOException e1) {Log.logException(e1);}
badPeerHashes.clear();
}
@ -729,7 +729,7 @@ public final class yacySeedDB implements AlternativeDomainNames {
}
// delete bad peers
final Iterator<byte[]> i = badPeerHashes.iterator();
while (i.hasNext()) try {seedActiveDB.remove(i.next());} catch (final IOException e1) {Log.logException(e1);}
while (i.hasNext()) try {seedActiveDB.delete(i.next());} catch (final IOException e1) {Log.logException(e1);}
badPeerHashes.clear();
}
@ -1061,15 +1061,9 @@ public final class yacySeedDB implements AlternativeDomainNames {
Log.logException(e);
// emergency reset
yacyCore.log.logSevere("seed-db emergency reset", e);
try {
database.clear();
nextSeed = null;
return null;
} catch (final IOException e1) {
// no recovery possible
Log.logException(e1);
System.exit(-1);
}
database.clear();
nextSeed = null;
return null;
}
return seed;
}

40
source/net/yacy/kelondro/blob/ArrayStack.java
View File

@ -521,7 +521,7 @@ public class ArrayStack implements BLOB {
* @return
* @throws IOException
*/
public synchronized boolean has(byte[] key) {
public synchronized boolean containsKey(byte[] key) {
blobItem bi = keeperOf(key);
return bi != null;
//for (blobItem bi: blobs) if (bi.blob.has(key)) return true;
@ -541,7 +541,7 @@ public class ArrayStack implements BLOB {
try {
cs.submit(new Callable<blobItem>() {
public blobItem call() {
if (bi.blob.has(key)) return bi;
if (bi.blob.containsKey(key)) return bi;
return null;
}
});
@ -549,7 +549,7 @@ public class ArrayStack implements BLOB {
} catch (final RejectedExecutionException e) {
// the executor is either shutting down or the blocking queue is full
// execute the search direct here without concurrency
if (bi.blob.has(key)) return bi;
if (bi.blob.containsKey(key)) return bi;
}
}
@ -596,6 +596,18 @@ public class ArrayStack implements BLOB {
}
public byte[] get(Object key) {
if (!(key instanceof byte[])) return null;
try {
return get((byte[]) key);
} catch (IOException e) {
Log.logException(e);
} catch (RowSpaceExceededException e) {
Log.logException(e);
}
return null;
}
/**
* get all BLOBs in the array.
* this is useful when it is not clear if an entry is unique in all BLOBs in this array.
@ -671,7 +683,7 @@ public class ArrayStack implements BLOB {
* @throws IOException
* @throws RowSpaceExceededException
*/
public synchronized void put(byte[] key, byte[] b) throws IOException {
public synchronized void insert(byte[] key, byte[] b) throws IOException {
blobItem bi = (blobs.isEmpty()) ? null : blobs.get(blobs.size() - 1);
/*
if (bi == null)
@ -687,7 +699,7 @@ public class ArrayStack implements BLOB {
blobs.add(bi);
}
assert bi.blob instanceof Heap;
bi.blob.put(key, b);
bi.blob.insert(key, b);
executeLimits();
}
@ -724,9 +736,9 @@ public class ArrayStack implements BLOB {
* @param key the primary key
* @throws IOException
*/
public synchronized void remove(byte[] key) throws IOException {
public synchronized void delete(byte[] key) throws IOException {
long m = this.mem();
for (blobItem bi: blobs) bi.blob.remove(key);
for (blobItem bi: blobs) bi.blob.delete(key);
assert this.mem() <= m : "m = " + m + ", mem() = " + mem();
}
@ -996,18 +1008,18 @@ public class ArrayStack implements BLOB {
try {
//f.delete();
final ArrayStack heap = new ArrayStack(f, "test", 12, NaturalOrder.naturalOrder, 512 * 1024, false);
heap.put("aaaaaaaaaaaa".getBytes(), "eins zwei drei".getBytes());
heap.put("aaaaaaaaaaab".getBytes(), "vier fuenf sechs".getBytes());
heap.put("aaaaaaaaaaac".getBytes(), "sieben acht neun".getBytes());
heap.put("aaaaaaaaaaad".getBytes(), "zehn elf zwoelf".getBytes());
heap.insert("aaaaaaaaaaaa".getBytes(), "eins zwei drei".getBytes());
heap.insert("aaaaaaaaaaab".getBytes(), "vier fuenf sechs".getBytes());
heap.insert("aaaaaaaaaaac".getBytes(), "sieben acht neun".getBytes());
heap.insert("aaaaaaaaaaad".getBytes(), "zehn elf zwoelf".getBytes());
// iterate over keys
Iterator<byte[]> i = heap.keys(true, false);
while (i.hasNext()) {
System.out.println("key_b: " + new String(i.next()));
}
heap.remove("aaaaaaaaaaab".getBytes());
heap.remove("aaaaaaaaaaac".getBytes());
heap.put("aaaaaaaaaaaX".getBytes(), "WXYZ".getBytes());
heap.delete("aaaaaaaaaaab".getBytes());
heap.delete("aaaaaaaaaaac".getBytes());
heap.insert("aaaaaaaaaaaX".getBytes(), "WXYZ".getBytes());
heap.close(true);
} catch (final IOException e) {
Log.logException(e);

11
source/net/yacy/kelondro/blob/BEncodedHeap.java
View File

@ -178,7 +178,7 @@ public class BEncodedHeap implements Map<byte[], Map<String, byte[]>>, Iterable<
* @return true if the row exists
*/
public boolean containsKey(byte[] pk) {
return this.table.has(pk);
return this.table.containsKey(pk);
}
/**
@ -260,13 +260,13 @@ public class BEncodedHeap implements Map<byte[], Map<String, byte[]>>, Iterable<
*/
public void insert(byte[] pk, Map<String, byte[]> map) throws RowSpaceExceededException, IOException {
byte[] b = BEncoder.encode(BEncoder.transcode(map));
this.table.put(pk, b);
this.table.insert(pk, b);
this.columnames.addAll(map.keySet());
}
public void insert(byte[] pk, String key, byte[] value) throws IOException {
byte[] b = BEncoder.encodeMap(key, value);
this.table.put(pk, b);
this.table.insert(pk, b);
this.columnames.add(key);
}
@ -279,7 +279,7 @@ public class BEncodedHeap implements Map<byte[], Map<String, byte[]>>, Iterable<
try {
Map<String, byte[]> entry = this.get(pk);
byte[] b = BEncoder.encode(BEncoder.transcode(map));
this.table.put(pk, b);
this.table.insert(pk, b);
this.columnames.addAll(map.keySet());
return entry;
} catch (IOException e) {
@ -297,7 +297,7 @@ public class BEncodedHeap implements Map<byte[], Map<String, byte[]>>, Iterable<
* @throws IOException
*/
public void delete(byte[] pk) throws IOException {
this.table.remove(pk);
this.table.delete(pk);
}
/**
@ -339,7 +339,6 @@ public class BEncodedHeap implements Map<byte[], Map<String, byte[]>>, Iterable<
Log.logException(e);
} catch (IOException e) {
Log.logException(e);
e.printStackTrace();
}
}
}

7
source/net/yacy/kelondro/blob/BLOB.java
View File

@ -107,7 +107,7 @@ public interface BLOB {
* @return
* @throws IOException
*/
public boolean has(byte[] key);
public boolean containsKey(byte[] key);
/**
* retrieve the whole BLOB from the table
@ -116,6 +116,7 @@ public interface BLOB {
* @throws IOException
*/
public byte[] get(byte[] key) throws IOException, RowSpaceExceededException;
public byte[] get(Object key);
/**
* retrieve the size of the database
@ -140,7 +141,7 @@ public interface BLOB {
* @throws IOException
* @throws RowSpaceExceededException
*/
public void put(byte[] key, byte[] b) throws IOException;
public void insert(byte[] key, byte[] b) throws IOException;
/**
* replace an existing entry in the BLOB with a new entry
@ -173,7 +174,7 @@ public interface BLOB {
* @param key the primary key
* @throws IOException
*/
public void remove(byte[] key) throws IOException;
public void delete(byte[] key) throws IOException;
/**
* close the BLOB table

30
source/net/yacy/kelondro/blob/Compressor.java
View File

@ -110,7 +110,7 @@ public class Compressor implements BLOB {
try {
while ((entry = writeQueue.take()) != poisonWorkerEntry) {
try {
Compressor.this.backend.put(entry.getKey().getBytes(), compress(entry.getValue()));
Compressor.this.backend.insert(entry.getKey().getBytes(), compress(entry.getValue()));
} catch (IOException e) {
Log.logException(e);
buffer.put(entry.getKey(), entry.getValue());
@ -250,9 +250,21 @@ public class Compressor implements BLOB {
return decompress(b);
}
public synchronized boolean has(byte[] key) {
public byte[] get(Object key) {
if (!(key instanceof byte[])) return null;
try {
return get((byte[]) key);
} catch (IOException e) {
Log.logException(e);
} catch (RowSpaceExceededException e) {
Log.logException(e);
}
return null;
}
public synchronized boolean containsKey(byte[] key) {
return
this.buffer.containsKey(new String(key)) || this.backend.has(key);
this.buffer.containsKey(new String(key)) || this.backend.containsKey(key);
}
public int keylength() {
@ -287,10 +299,10 @@ public class Compressor implements BLOB {
return 0;
}
public void put(byte[] key, byte[] b) throws IOException {
public void insert(byte[] key, byte[] b) throws IOException {
// first ensure that the files do not exist anywhere
remove(key);
delete(key);
// check if the buffer is full or could be full after this write
if (this.bufferlength + b.length * 2 > this.maxbufferlength) synchronized (this) {
@ -312,8 +324,8 @@ public class Compressor implements BLOB {
}
}
public synchronized void remove(byte[] key) throws IOException {
this.backend.remove(key);
public synchronized void delete(byte[] key) throws IOException {
this.backend.delete(key);
long rx = removeFromQueues(key);
if (rx > 0) this.bufferlength -= rx;
}
@ -366,7 +378,7 @@ public class Compressor implements BLOB {
int reduction = c.length - b.length;
assert reduction >= 0;
if (reduction == 0) return 0;
this.put(key, c);
this.insert(key, c);
return reduction;
}
@ -377,7 +389,7 @@ public class Compressor implements BLOB {
int reduction = c.length - b.length;
assert reduction >= 0;
if (reduction == 0) return 0;
this.put(key, c);
this.insert(key, c);
return reduction;
}

40
source/net/yacy/kelondro/blob/Heap.java
View File

@ -121,7 +121,7 @@ public final class Heap extends HeapModifier implements BLOB {
* @return true if the key exists, false otherwise
*/
@Override
public boolean has(byte[] key) {
public boolean containsKey(byte[] key) {
assert index != null;
key = normalizeKey(key);
synchronized (this) {
@ -130,7 +130,7 @@ public final class Heap extends HeapModifier implements BLOB {
if (buffer == null) {
if (this.buffer.containsKey(key)) return true;
}
return super.has(key);
return super.containsKey(key);
}
}
@ -322,7 +322,7 @@ public final class Heap extends HeapModifier implements BLOB {
* @throws RowSpaceExceededException
*/
@Override
public void put(byte[] key, final byte[] b) throws IOException {
public void insert(byte[] key, final byte[] b) throws IOException {
key = normalizeKey(key);
// we do not write records of length 0 into the BLOB
@ -331,7 +331,7 @@ public final class Heap extends HeapModifier implements BLOB {
synchronized (this) {
// first remove the old entry (removes from buffer and file)
// TODO: this can be enhanced!
this.remove(key);
this.delete(key);
// then look if we can use a free entry
try {
@ -456,7 +456,7 @@ public final class Heap extends HeapModifier implements BLOB {
* @throws IOException
*/
@Override
public void remove(byte[] key) throws IOException {
public void delete(byte[] key) throws IOException {
key = normalizeKey(key);
synchronized (this) {
@ -470,7 +470,7 @@ public final class Heap extends HeapModifier implements BLOB {
}
}
super.remove(key);
super.delete(key);
}
}
@ -510,10 +510,10 @@ public final class Heap extends HeapModifier implements BLOB {
try {
//f.delete();
final Heap heap = new Heap(f, 12, NaturalOrder.naturalOrder, 1024 * 512);
heap.put("aaaaaaaaaaaa".getBytes(), "eins zwei drei".getBytes());
heap.put("aaaaaaaaaaab".getBytes(), "vier fuenf sechs".getBytes());
heap.put("aaaaaaaaaaac".getBytes(), "sieben acht neun".getBytes());
heap.put("aaaaaaaaaaad".getBytes(), "zehn elf zwoelf".getBytes());
heap.insert("aaaaaaaaaaaa".getBytes(), "eins zwei drei".getBytes());
heap.insert("aaaaaaaaaaab".getBytes(), "vier fuenf sechs".getBytes());
heap.insert("aaaaaaaaaaac".getBytes(), "sieben acht neun".getBytes());
heap.insert("aaaaaaaaaaad".getBytes(), "zehn elf zwoelf".getBytes());
// iterate over keys
Iterator<byte[]> i = heap.index.keys(true, null);
while (i.hasNext()) {
@ -523,9 +523,9 @@ public final class Heap extends HeapModifier implements BLOB {
while (i.hasNext()) {
System.out.println("key_b: " + new String(i.next()));
}
heap.remove("aaaaaaaaaaab".getBytes());
heap.remove("aaaaaaaaaaac".getBytes());
heap.put("aaaaaaaaaaaX".getBytes(), "WXYZ".getBytes());
heap.delete("aaaaaaaaaaab".getBytes());
heap.delete("aaaaaaaaaaac".getBytes());
heap.insert("aaaaaaaaaaaX".getBytes(), "WXYZ".getBytes());
heap.close(true);
} catch (final IOException e) {
Log.logException(e);
@ -543,13 +543,13 @@ public final class Heap extends HeapModifier implements BLOB {
try {
//f.delete();
final MapHeap heap = new MapHeap(f, 12, NaturalOrder.naturalOrder, 1024 * 512, 500, '_');
heap.put("aaaaaaaaaaaa".getBytes(), map("aaaaaaaaaaaa", "eins zwei drei"));
heap.put("aaaaaaaaaaab".getBytes(), map("aaaaaaaaaaab", "vier fuenf sechs"));
heap.put("aaaaaaaaaaac".getBytes(), map("aaaaaaaaaaac", "sieben acht neun"));
heap.put("aaaaaaaaaaad".getBytes(), map("aaaaaaaaaaad", "zehn elf zwoelf"));
heap.remove("aaaaaaaaaaab".getBytes());
heap.remove("aaaaaaaaaaac".getBytes());
heap.put("aaaaaaaaaaaX".getBytes(), map("aaaaaaaaaaad", "WXYZ"));
heap.insert("aaaaaaaaaaaa".getBytes(), map("aaaaaaaaaaaa", "eins zwei drei"));
heap.insert("aaaaaaaaaaab".getBytes(), map("aaaaaaaaaaab", "vier fuenf sechs"));
heap.insert("aaaaaaaaaaac".getBytes(), map("aaaaaaaaaaac", "sieben acht neun"));
heap.insert("aaaaaaaaaaad".getBytes(), map("aaaaaaaaaaad", "zehn elf zwoelf"));
heap.delete("aaaaaaaaaaab".getBytes());
heap.delete("aaaaaaaaaaac".getBytes());
heap.insert("aaaaaaaaaaaX".getBytes(), map("aaaaaaaaaaad", "WXYZ"));
heap.close();
} catch (final IOException e) {
Log.logException(e);

4
source/net/yacy/kelondro/blob/HeapModifier.java
View File

@ -93,7 +93,7 @@ public class HeapModifier extends HeapReader implements BLOB {
* @param key the primary key
* @throws IOException
*/
public void remove(byte[] key) throws IOException {
public void delete(byte[] key) throws IOException {
key = normalizeKey(key);
// pre-check before synchronization
@ -234,7 +234,7 @@ public class HeapModifier extends HeapReader implements BLOB {
}
}
public void put(byte[] key, byte[] b) throws IOException {
public void insert(byte[] key, byte[] b) throws IOException {
throw new UnsupportedOperationException("put is not supported in BLOBHeapModifier");
}

14
source/net/yacy/kelondro/blob/HeapReader.java
View File

@ -317,7 +317,7 @@ public class HeapReader {
* @param key
* @return true if the key exists, false otherwise
*/
public boolean has(byte[] key) {
public boolean containsKey(byte[] key) {
assert index != null;
key = normalizeKey(key);
@ -427,6 +427,18 @@ public class HeapReader {
return blob;
}
}
public byte[] get(Object key) {
if (!(key instanceof byte[])) return null;
try {
return get((byte[]) key);
} catch (IOException e) {
Log.logException(e);
} catch (RowSpaceExceededException e) {
Log.logException(e);
}
return null;
}
protected boolean checkKey(byte[] key, final long pos) throws IOException {
key = normalizeKey(key);

10
source/net/yacy/kelondro/blob/MapDataMining.java
View File

@ -150,7 +150,7 @@ public class MapDataMining extends MapHeap {
}
@Override
public synchronized void clear() throws IOException {
public synchronized void clear() {
super.clear();
if (sortfields == null) sortClusterMap = null; else {
sortClusterMap = new HashMap<String, ScoreCluster<String>>();
@ -178,7 +178,7 @@ public class MapDataMining extends MapHeap {
}
@Override
public synchronized void put(final byte[] key, final Map<String, String> newMap) throws IOException, RowSpaceExceededException {
public synchronized void insert(final byte[] key, final Map<String, String> newMap) throws IOException, RowSpaceExceededException {
assert (key != null);
assert (key.length > 0);
assert (newMap != null);
@ -195,7 +195,7 @@ public class MapDataMining extends MapHeap {
updateAcc(newMap, true);
}
super.put(key, newMap);
super.insert(key, newMap);
// update sortCluster
if (sortClusterMap != null) updateSortCluster(new String(key), newMap);
@ -251,7 +251,7 @@ public class MapDataMining extends MapHeap {
}
@Override
public synchronized void remove(final byte[] key) throws IOException {
public synchronized void delete(final byte[] key) throws IOException {
if (key == null) return;
// update elementCount
@ -272,7 +272,7 @@ public class MapDataMining extends MapHeap {
Log.logException(e);
}
}
super.remove(key);
super.delete(key);
}
private void deleteSortCluster(final String key) {

111
source/net/yacy/kelondro/blob/MapHeap.java
View File

@ -32,9 +32,12 @@ import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.IOException;
import java.io.InputStreamReader;
import java.util.Collection;
import java.util.HashMap;
import java.util.Iterator;
import java.util.Map;
import java.util.Set;
import java.util.TreeSet;
import java.util.concurrent.ConcurrentHashMap;
import net.yacy.cora.storage.ARC;
@ -51,7 +54,7 @@ import net.yacy.kelondro.util.kelondroException;
public class MapHeap {
public class MapHeap implements Map<byte[], Map<String, String>> {
private BLOB blob;
private ARC<String, Map<String, String>> cache;
@ -82,8 +85,12 @@ public class MapHeap {
* clears the content of the database
* @throws IOException
*/
public synchronized void clear() throws IOException {
this.blob.clear();
public synchronized void clear() {
try {
this.blob.clear();
} catch (IOException e) {
Log.logException(e);
}
this.cache.clear();
}
@ -129,7 +136,7 @@ public class MapHeap {
* @throws IOException
* @throws RowSpaceExceededException
*/
public void put(byte[] key, final Map<String, String> newMap) throws IOException, RowSpaceExceededException {
public void insert(byte[] key, final Map<String, String> newMap) throws IOException, RowSpaceExceededException {
assert key != null;
assert key.length > 0;
assert newMap != null;
@ -139,19 +146,32 @@ public class MapHeap {
byte[] sb = s.getBytes();
synchronized (this) {
// write entry
if (blob != null) blob.put(key, sb);
if (blob != null) blob.insert(key, sb);
// write map to cache
if (cache != null) cache.put(new String(key), newMap);
}
}
public Map<String, String> put(byte[] key, final Map<String, String> newMap) {
Map<String, String> v = null;
try {
v = this.get(key);
insert(key, newMap);
} catch (IOException e) {
Log.logException(e);
} catch (RowSpaceExceededException e) {
Log.logException(e);
}
return v;
}
/**
* remove a Map
* @param key the primary key
* @throws IOException
*/
public void remove(byte[] key) throws IOException {
public void delete(byte[] key) throws IOException {
// update elementCount
if (key == null) return;
key = normalizeKey(key);
@ -161,10 +181,21 @@ public class MapHeap {
cache.remove(new String(key));
// remove from file
blob.remove(key);
blob.delete(key);
}
}
public Map<String, String> remove(Object key) {
Map<String, String> v = null;
try {
v = this.get(key);
this.delete((byte[]) key);
} catch (IOException e) {
Log.logException(e);
}
return v;
}
/**
* check if a specific key is in the database
* @param key the primary key
@ -172,13 +203,14 @@ public class MapHeap {
* @throws IOException
*/
public boolean has(byte[] key) {
assert key != null;
public boolean containsKey(Object k) {
if (!(k instanceof byte[])) return false;
assert k != null;
if (cache == null) return false; // case may appear during shutdown
key = normalizeKey(key);
byte[] key = normalizeKey((byte[]) k);
boolean h;
synchronized (this) {
h = this.cache.containsKey(new String(key)) || this.blob.has(key);
h = this.cache.containsKey(new String(key)) || this.blob.containsKey(key);
}
return h;
}
@ -193,6 +225,18 @@ public class MapHeap {
if (key == null) return null;
return get(key, true);
}
public Map<String, String> get(final Object key) {
if (key == null) return null;
try {
return get((byte[]) key);
} catch (IOException e) {
Log.logException(e);
} catch (RowSpaceExceededException e) {
Log.logException(e);
}
return null;
}
private byte[] normalizeKey(byte[] key) {
if (blob == null || key == null) return key;
@ -360,7 +404,43 @@ public class MapHeap {
throw new UnsupportedOperationException();
}
} // class mapIterator
public void putAll(Map<? extends byte[], ? extends Map<String, String>> map) {
for (Map.Entry<? extends byte[], ? extends Map<String, String>> me: map.entrySet()) {
try {
this.insert(me.getKey(), me.getValue());
} catch (RowSpaceExceededException e) {
Log.logException(e);
} catch (IOException e) {
Log.logException(e);
}
}
}
public Set<byte[]> keySet() {
TreeSet<byte[]> set = new TreeSet<byte[]>(this.blob.ordering());
try {
Iterator<byte[]> i = this.blob.keys(true, false);
while (i.hasNext()) set.add(i.next());
} catch (IOException e) {}
return set;
}
public Collection<Map<String, String>> values() {
// this method shall not be used because it is not appropriate for this kind of data
throw new UnsupportedOperationException();
}
public Set<java.util.Map.Entry<byte[], Map<String, String>>> entrySet() {
// this method shall not be used because it is not appropriate for this kind of data
throw new UnsupportedOperationException();
}
public boolean containsValue(Object value) {
// this method shall not be used because it is not appropriate for this kind of data
throw new UnsupportedOperationException();
}
public static void main(String[] args) {
// test the class
File f = new File("maptest");
@ -370,9 +450,9 @@ public class MapHeap {
MapHeap map = new MapHeap(f, 12, NaturalOrder.naturalOrder, 1024 * 1024, 1024, '_');
// put some values into the map
Map<String, String> m = new HashMap<String, String>();
m.put("k", "000"); map.put("123".getBytes(), m);
m.put("k", "111"); map.put("456".getBytes(), m);
m.put("k", "222"); map.put("789".getBytes(), m);
m.put("k", "000"); map.insert("123".getBytes(), m);
m.put("k", "111"); map.insert("456".getBytes(), m);
m.put("k", "222"); map.insert("789".getBytes(), m);
// iterate over keys
Iterator<byte[]> i = map.keys(true, false);
while (i.hasNext()) {
@ -386,5 +466,4 @@ public class MapHeap {
Log.logException(e);
}
}
}

8
source/net/yacy/kelondro/blob/Stack.java
View File

@ -94,7 +94,7 @@ public class Stack {
*/
public synchronized long push(final byte[] b) throws IOException, RowSpaceExceededException {
long handle = nextHandle();
this.stack.put(NaturalOrder.encodeLong(handle, 8), b);
this.stack.insert(NaturalOrder.encodeLong(handle, 8), b);
return handle;
}
@ -108,7 +108,7 @@ public class Stack {
* @throws RowSpaceExceededException
*/
protected synchronized void push(final Entry e) throws IOException, RowSpaceExceededException {
this.stack.put(NaturalOrder.encodeLong(e.h, 8), e.b);
this.stack.insert(NaturalOrder.encodeLong(e.h, 8), e.b);
}
/**
@ -137,7 +137,7 @@ public class Stack {
byte[] k = NaturalOrder.encodeLong(handle, 8);
byte[] b = this.stack.get(k);
if (b == null) return null;
this.stack.remove(k);
this.stack.delete(k);
return b;
}
@ -193,7 +193,7 @@ public class Stack {
}
assert b != null;
if (b == null) return null;
if (remove) this.stack.remove(k);
if (remove) this.stack.delete(k);
return new Entry(k, b);
}

4
source/net/yacy/kelondro/rwi/ReferenceContainerArray.java
View File

@ -197,7 +197,7 @@ public final class ReferenceContainerArray<ReferenceType extends Reference> {
* @throws IOException
*/
public boolean has(final byte[] termHash) {
return this.array.has(termHash);
return this.array.containsKey(termHash);
}
/**
@ -251,7 +251,7 @@ public final class ReferenceContainerArray<ReferenceType extends Reference> {
*/
public void delete(final byte[] termHash) throws IOException {
// returns the index that had been deleted
array.remove(termHash);
array.delete(termHash);
}
public int reduce(final byte[] termHash, ContainerReducer<ReferenceType> reducer) throws IOException, RowSpaceExceededException {