Marown/yacy_search_server
1
0
Fork 0
mirror of https://github.com/yacy/yacy_search_server.git synced 2024-09-19 00:01:41 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity

- fixed a bug in fast digest computation

Browse Source 
- added a open-on-demand hack to heap files: when a heap file is
  opened the first time, it is first scanned to get a key index
  and then it is closed again. This will free up file pointers
  in cases where a really large number of blob files are opened
  upon initialization of ArrayStack objects. This should solve
  also a problem reported in
  http://forum.yacy-websuche.de/viewtopic.php?p=17191#p17191

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@6267 6c8d7289-2bf4-0310-a012-ef5d649a1542
This commit is contained in:
orbiter 2009-08-27 11:03:21 +00:00
parent 93b2622503
commit 0e471ba33b
4 changed files with 65 additions and 39 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

32
source/de/anomic/kelondro/blob/HeapModifier.java
View File

@ -26,8 +26,6 @@ package de.anomic.kelondro.blob;
import java.io.File;
import java.io.IOException;
import java.util.Iterator;
import java.util.Map;
import java.util.SortedMap;
import de.anomic.kelondro.io.CachedRandomAccess;
@ -55,37 +53,9 @@ public class HeapModifier extends HeapReader implements BLOB {
*/
public HeapModifier(final File heapFile, final int keylength, final ByteOrder ordering) throws IOException {
super(heapFile, keylength, ordering);
mergeFreeEntries();
}
private void mergeFreeEntries() throws IOException {
// try to merge free entries
if (super.free.size() > 1) {
int merged = 0;
Map.Entry<Long, Integer> lastFree, nextFree;
final Iterator<Map.Entry<Long, Integer>> i = this.free.entrySet().iterator();
lastFree = i.next();
while (i.hasNext()) {
nextFree = i.next();
//System.out.println("*** DEBUG BLOB: free-seek = " + nextFree.seek + ", size = " + nextFree.size);
// check if they follow directly
if (lastFree.getKey() + lastFree.getValue() + 4 == nextFree.getKey()) {
// merge those records
this.file.seek(lastFree.getKey());
lastFree.setValue(lastFree.getValue() + nextFree.getValue() + 4); // this updates also the free map
this.file.writeInt(lastFree.getValue());
this.file.seek(nextFree.getKey());
this.file.writeInt(0);
i.remove();
merged++;
} else {
lastFree = nextFree;
}
}
Log.logInfo("kelondroBLOBHeap", "BLOB " + heapFile.getName() + ": merged " + merged + " free records");
}
}
/**
* clears the content of the database

40
source/de/anomic/kelondro/blob/HeapReader.java
View File

@ -63,7 +63,7 @@ public class HeapReader {
this.keylength = keylength;
this.index = null; // will be created as result of initialization process
this.free = null; // will be initialized later depending on existing idx/gap file
this.file = new CachedRandomAccess(heapFile);
this.file = new CachedRandomAccess(this.heapFile);
// read or initialize the index
if (initIndexReadDump()) {
@ -94,6 +94,15 @@ public class HeapReader {
// if we did not have a dump, create a new index
initIndexReadFromHeap();
}
// merge gaps that follow directly
mergeFreeEntries();
// after the initial initialization of the heap, we close the file again
// to make more room to file pointers which may run out if the number
// of file descriptors is too low and the number of files is too high
this.file.close();
// the file will be opened again automatically when the next access to it comes.
}
private boolean initIndexReadDump() {
@ -205,6 +214,35 @@ public class HeapReader {
}
private void mergeFreeEntries() throws IOException {
// try to merge free entries
if (free.size() > 1) {
int merged = 0;
Map.Entry<Long, Integer> lastFree, nextFree;
final Iterator<Map.Entry<Long, Integer>> i = this.free.entrySet().iterator();
lastFree = i.next();
while (i.hasNext()) {
nextFree = i.next();
//System.out.println("*** DEBUG BLOB: free-seek = " + nextFree.seek + ", size = " + nextFree.size);
// check if they follow directly
if (lastFree.getKey() + lastFree.getValue() + 4 == nextFree.getKey()) {
// merge those records
this.file.seek(lastFree.getKey());
lastFree.setValue(lastFree.getValue() + nextFree.getValue() + 4); // this updates also the free map
this.file.writeInt(lastFree.getValue());
this.file.seek(nextFree.getKey());
this.file.writeInt(0);
i.remove();
merged++;
} else {
lastFree = nextFree;
}
}
Log.logInfo("kelondroBLOBHeap", "BLOB " + heapFile.getName() + ": merged " + merged + " free records");
}
}
public String name() {
return this.heapFile.getName();
}

28
source/de/anomic/kelondro/io/CachedRandomAccess.java
View File

@ -38,26 +38,30 @@ public final class CachedRandomAccess extends AbstractRandomAccess implements Ra
public CachedRandomAccess(final File file) throws IOException, FileNotFoundException {
this.name = file.getName();
this.file = file;
RAFile = new RandomAccessFile(file, "rw");
cache = new byte[8192];
cachestart = 0;
cachelen = 0;
this.RAFile = new RandomAccessFile(this.file, "rw");
this.cache = new byte[8192];
this.cachestart = 0;
this.cachelen = 0;
}
public synchronized long length() throws IOException {
checkReopen();
return this.RAFile.length();
}
public synchronized void setLength(long length) throws IOException {
checkReopen();
cachelen = 0;
RAFile.setLength(length);
}
public synchronized long available() throws IOException {
checkReopen();
return this.length() - RAFile.getFilePointer();
}
public synchronized final void readFully(final byte[] b, final int off, int len) throws IOException {
checkReopen();
long seek = RAFile.getFilePointer();
if (cache != null && cachestart <= seek && cachelen - seek + cachestart >= len) {
// read from cache
@ -93,6 +97,7 @@ public final class CachedRandomAccess extends AbstractRandomAccess implements Ra
}
public synchronized void write(final byte[] b, final int off, final int len) throws IOException {
checkReopen();
//assert len > 0;
// write to file
if (this.cache.length > 512) {
@ -122,6 +127,7 @@ public final class CachedRandomAccess extends AbstractRandomAccess implements Ra
}
public synchronized void seek(final long pos) throws IOException {
checkReopen();
RAFile.seek(pos);
}
@ -134,10 +140,22 @@ public final class CachedRandomAccess extends AbstractRandomAccess implements Ra
} catch (IOException e) {
e.printStackTrace();
}
this.file = null;
this.cache = null;
this.RAFile = null;
}
private void checkReopen() {
if (this.RAFile != null) return;
// re-open the file
try {
this.RAFile = new RandomAccessFile(this.file, "rw");
} catch (FileNotFoundException e) {
e.printStackTrace();
}
this.cache = new byte[8192];
this.cachestart = 0;
this.cachelen = 0;
}
protected void finalize() throws Throwable {
this.close();

4
source/de/anomic/kelondro/order/Digest.java
View File

@ -305,10 +305,10 @@ public class Digest {
byte[] a = new byte[mb];
try {
raf.seek(0);
raf.readFully(a, 0, mb - 1);
raf.readFully(a, 0, mb);
digest.update(a, 0, mb);
raf.seek(fl - mb);
raf.readFully(a, 0, mb - 1);
raf.readFully(a, 0, mb);
digest.update(a, 0, mb);
digest.update(NaturalOrder.encodeLong(fl, 8), 0, 8);
if (includeDate) digest.update(NaturalOrder.encodeLong(file.lastModified(), 8), 0, 8);