Marown/yacy_search_server
1
0
Fork 0
mirror of https://github.com/yacy/yacy_search_server.git synced 2024-09-19 00:01:41 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity

added another filed information for BLOBHeap dumps: the gaps

Browse Source 
git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@5420 6c8d7289-2bf4-0310-a012-ef5d649a1542
This commit is contained in:
orbiter 2008-12-30 10:49:43 +00:00
parent abdd4aa414
commit 3567c58b18
3 changed files with 155 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

98
source/de/anomic/kelondro/kelondroBLOBGap.java Normal file
View File

@ -0,0 +1,98 @@
// kelondroBLOBGap.java
// (C) 2008 by Michael Peter Christen; mc@yacy.net, Frankfurt a. M., Germany
// first published 30.12.2008 on http:// yacy.net
//
// This is a part of YaCy, a peer-to-peer based web search engine
//
// $LastChangedDate: 2008-03-14 01:16:04 +0100 (Fr, 14 Mrz 2008) $
// $LastChangedRevision: 4558 $
// $LastChangedBy: orbiter $
//
// LICENSE
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation; either version 2 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
package de.anomic.kelondro;
import java.io.BufferedInputStream;
import java.io.BufferedOutputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.util.Iterator;
import java.util.Map;
import java.util.TreeMap;
/**
* this is an extension of a set of {seek, size} pairs
* it is used to denote the position and space of free records
* The class provides methods to dump such a set to a file and read it again
*/
public class kelondroBLOBGap extends TreeMap<Long, Integer> {
private static final long serialVersionUID = 1L;
public kelondroBLOBGap() {
super();
}
/**
* initialize a kelondroBLOBGap with the content of a dump
* @param file
* @throws IOException
*/
public kelondroBLOBGap(final File file) throws IOException {
super();
// read the index dump and fill the index
InputStream is = new BufferedInputStream(new FileInputStream(file), 1024 * 1024);
byte[] k = new byte[8];
byte[] v = new byte[4];
int c;
while (true) {
c = is.read(k);
if (c <= 0) break;
c = is.read(v);
if (c <= 0) break;
this.put(new Long(kelondroNaturalOrder.decodeLong(k)), new Integer((int) kelondroNaturalOrder.decodeLong(v)));
}
}
/**
* dump the set to a file
* @param file
* @return
* @throws IOException
*/
public int dump(File file) throws IOException {
Iterator<Map.Entry<Long, Integer>> i = this.entrySet().iterator();
OutputStream os = new BufferedOutputStream(new FileOutputStream(file), 1024 * 1024);
int c = 0;
Map.Entry<Long, Integer> e;
while (i.hasNext()) {
e = i.next();
os.write(kelondroNaturalOrder.encodeLong(e.getKey().longValue(), 8));
os.write(kelondroNaturalOrder.encodeLong(e.getValue().longValue(), 4));
c++;
}
os.flush();
os.close();
return c;
}
}

82
source/de/anomic/kelondro/kelondroBLOBHeap.java
View File

@ -32,7 +32,6 @@ import java.util.HashMap;
import java.util.Iterator;
import java.util.Map;
import java.util.SortedMap;
import java.util.TreeMap;
import java.util.concurrent.ExecutionException;
import de.anomic.server.serverMemory;
@ -42,7 +41,7 @@ public final class kelondroBLOBHeap implements kelondroBLOB {
private int keylength; // the length of the primary key
private kelondroBytesLongMap index; // key/seek relation for used records
private TreeMap<Long, Integer> free; // list of {size, seek} pairs denoting space and position of free records
private kelondroBLOBGap free; // set of {seek, size} pairs denoting space and position of free records
private final File heapFile; // the file of the heap
private final kelondroByteOrder ordering; // the ordering on keys
private kelondroCachedFileRA file; // a random access to the file
@ -86,7 +85,7 @@ public final class kelondroBLOBHeap implements kelondroBLOB {
this.buffermax = buffermax;
this.keylength = keylength;
this.index = null; // will be created as result of initialization process
this.free = new TreeMap<Long, Integer>();
this.free = null; // will be initialized later depending on existing idx/gap file
this.buffer = new HashMap<String, byte[]>();
this.buffersize = 0;
this.file = new kelondroCachedFileRA(heapFile);
@ -142,43 +141,63 @@ public final class kelondroBLOBHeap implements kelondroBLOB {
private boolean initIndexReadDump(File f) {
// look for an index dump and read it if it exist
// if this is successfull, return true; otherwise false
File ff = fingerprintFile(f);
if (!ff.exists()) {
File fif = fingerprintIndexFile(f);
File fgf = fingerprintGapFile(f);
if (!fif.exists() || !fgf.exists()) {
deleteAllFingerprints(f);
return false;
}
// there is a file: read it:
// there is an index and a gap file:
// read the index file:
try {
this.index = new kelondroBytesLongMap(this.keylength, this.ordering, ff);
this.index = new kelondroBytesLongMap(this.keylength, this.ordering, fif);
} catch (IOException e) {
e.printStackTrace();
return false;
}
// an index file is a one-time throw-away object, so just delete it now
ff.delete();
fif.delete();
// read the gap file:
try {
this.free = new kelondroBLOBGap(fgf);
} catch (IOException e) {
e.printStackTrace();
return false;
}
// same with gap file
fgf.delete();
// everything is fine now
return this.index.size() > 0;
}
private File fingerprintFile(File f) {
String fingerprint = kelondroDigest.fastFingerprintB64(f, false).substring(0, 12);
return new File(f.getParentFile(), f.getName() + "." + fingerprint + ".idx");
private static File fingerprintIndexFile(File f) {
return new File(f.getParentFile(), f.getName() + "." + fingerprintFileHash(f) + ".idx");
}
private void deleteAllFingerprints(File f) {
private static File fingerprintGapFile(File f) {
return new File(f.getParentFile(), f.getName() + "." + fingerprintFileHash(f) + ".gap");
}
private static String fingerprintFileHash(File f) {
return kelondroDigest.fastFingerprintB64(f, false).substring(0, 12);
}
private static void deleteAllFingerprints(File f) {
File d = f.getParentFile();
String n = f.getName();
String[] l = d.list();
for (int i = 0; i < l.length; i++) {
if (l[i].startsWith(n) && l[i].endsWith(".idx")) new File(d, l[i]).delete();
if (l[i].startsWith(n) && (l[i].endsWith(".idx") || l[i].endsWith(".gap"))) new File(d, l[i]).delete();
}
}
private void initIndexReadFromHeap() throws IOException {
// this initializes the this.index object by reading positions from the heap file
this.free = new kelondroBLOBGap();
kelondroBytesLongMap.initDataConsumer indexready = kelondroBytesLongMap.asynchronusInitializer(keylength, this.ordering, 0, Math.max(10, (int) (Runtime.getRuntime().freeMemory() / (10 * 1024 * 1024))));
byte[] key = new byte[keylength];
int reclen;
@ -463,20 +482,31 @@ public final class kelondroBLOBHeap implements kelondroBLOB {
e.printStackTrace();
}
file = null;
// now we can create a dump of the index, to speed up the next start
try {
long start = System.currentTimeMillis();
index.dump(fingerprintFile(this.heapFile));
serverLog.logInfo("kelondroBLOBHeap", "wrote a dump for the " + this.index.size() + " index entries of " + heapFile.getName()+ " in " + (System.currentTimeMillis() - start) + " milliseconds.");
} catch (IOException e) {
e.printStackTrace();
if (index.size() > 3 || free.size() > 3) {
// now we can create a dump of the index and the gap information
// to speed up the next start
try {
long start = System.currentTimeMillis();
free.dump(fingerprintGapFile(this.heapFile));
free.clear();
free = null;
index.dump(fingerprintIndexFile(this.heapFile));
serverLog.logInfo("kelondroBLOBHeap", "wrote a dump for the " + this.index.size() + " index entries of " + heapFile.getName()+ " in " + (System.currentTimeMillis() - start) + " milliseconds.");
index.close();
index = null;
} catch (IOException e) {
e.printStackTrace();
}
} else {
// this is small.. just free resources, do not write index
free.clear();
free = null;
index.close();
index = null;
}
index.close();
free.clear();
index = null;
free = null;
}
/**
* ask for the length of the primary key
* @return the length of the key

2
source/de/anomic/kelondro/kelondroBytesLongMap.java
View File

@ -103,7 +103,7 @@ public class kelondroBytesLongMap {
os.close();
return c;
}
public kelondroRow row() {
return index.row();
}