Marown/yacy_search_server
1
0
Fork 0
mirror of https://github.com/yacy/yacy_search_server.git synced 2024-09-19 00:01:41 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity

a different naming scheme for BLOBArray files. This may be necessary if blobs are written more often than once in a second.

Browse Source 
git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@5771 6c8d7289-2bf4-0310-a012-ef5d649a1542
This commit is contained in:
orbiter 2009-04-02 15:08:56 +00:00
parent 7ba078daa1
commit f21a8c9e9c
9 changed files with 50 additions and 52 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
htroot/PerformanceGraph.java
View File

@ -41,7 +41,7 @@ public class PerformanceGraph {
final int width = post.getInt("width", 660);
final int height = post.getInt("height", 240);
return plasmaProfiling.performanceGraph(width, height, sb.webIndex.metadata().size() + " URLS / " + sb.webIndex.index().getBackendSize() + " WORDS IN COLLECTIONS / " + sb.webIndex.index().getBufferSize() + " WORDS IN CACHE");
return plasmaProfiling.performanceGraph(width, height, sb.webIndex.metadata().size() + " URLS / " + sb.webIndex.index().getBackendSize() + " WORDS IN INDEX / " + sb.webIndex.index().getBufferSize() + " WORDS IN CACHE");
}
}

42
source/de/anomic/kelondro/blob/BLOBArray.java
View File

@ -74,17 +74,17 @@ public class BLOBArray implements BLOB {
private long repositoryAgeMax;
private long repositorySizeMax;
private List<blobItem> blobs;
private String blobSalt;
private String prefix;
private int buffersize;
public BLOBArray(
final File heapLocation,
final String blobSalt,
final String prefix,
final int keylength,
final ByteOrder ordering,
final int buffersize) throws IOException {
this.keylength = keylength;
this.blobSalt = blobSalt;
this.prefix = prefix;
this.ordering = ordering;
this.buffersize = buffersize;
this.heapLocation = heapLocation;
@ -120,26 +120,39 @@ public class BLOBArray implements BLOB {
}
}
if (deletions) files = heapLocation.list(); // make a fresh list
// find maximum time: the file with this time will be given a write buffer
// migrate old file names
Date d;
TreeMap<Long, blobItem> sortedItems = new TreeMap<Long, blobItem>();
BLOB oneBlob;
File f;
long time, maxtime = 0;
long time;
deletions = false;
for (int i = 0; i < files.length; i++) {
if (files[i].length() >= 19 && files[i].endsWith(".blob")) {
try {
d = DateFormatter.parseShortSecond(files[i].substring(0, 14));
new File(heapLocation, files[i]).renameTo(newBLOB(d));
deletions = true;
} catch (ParseException e) {continue;}
}
}
if (deletions) files = heapLocation.list(); // make a fresh list
// find maximum time: the file with this time will be given a write buffer
TreeMap<Long, blobItem> sortedItems = new TreeMap<Long, blobItem>();
BLOB oneBlob;
File f;
long maxtime = 0;
for (int i = 0; i < files.length; i++) {
if (files[i].length() >= 22 && files[i].startsWith(prefix) && files[i].endsWith(".blob")) {
try {
d = DateFormatter.parseShortMilliSecond(files[i].substring(prefix.length() + 1, prefix.length() + 18));
time = d.getTime();
if (time > maxtime) maxtime = time;
} catch (ParseException e) {continue;}
}
}
for (int i = 0; i < files.length; i++) {
if (files[i].length() >= 19 && files[i].endsWith(".blob")) {
try {
d = DateFormatter.parseShortSecond(files[i].substring(0, 14));
if (files[i].length() >= 22 && files[i].startsWith(prefix) && files[i].endsWith(".blob")) {
try {
d = DateFormatter.parseShortMilliSecond(files[i].substring(prefix.length() + 1, prefix.length() + 18));
f = new File(heapLocation, files[i]);
time = d.getTime();
oneBlob = (time == maxtime && buffersize > 0) ? new BLOBHeap(f, keylength, ordering, buffersize) : new BLOBHeapModifier(f, keylength, ordering);
@ -164,7 +177,7 @@ public class BLOBArray implements BLOB {
public synchronized void mountBLOB(File location) throws IOException {
Date d;
try {
d = DateFormatter.parseShortSecond(location.getName().substring(0, 14));
d = DateFormatter.parseShortMilliSecond(location.getName().substring(prefix.length() + 1, prefix.length() + 18));
} catch (ParseException e) {
throw new IOException("date parse problem with file " + location.toString() + ": " + e.getMessage());
}
@ -291,7 +304,8 @@ public class BLOBArray implements BLOB {
* @return
*/
public synchronized File newBLOB(Date creation) {
return new File(heapLocation, DateFormatter.formatShortSecond(creation) + "." + blobSalt + ".blob");
//return new File(heapLocation, DateFormatter.formatShortSecond(creation) + "." + blobSalt + ".blob");
return new File(heapLocation, prefix + "." + DateFormatter.formatShortMilliSecond(creation) + ".blob");
}
public String name() {

2
source/de/anomic/kelondro/index/ObjectIndex.java
View File

@ -48,8 +48,6 @@ public interface ObjectIndex {
public void put(Row.Entry row) throws IOException;
public void put(List<Row.Entry> rows) throws IOException; // for R/W head path optimization
public void addUnique(Row.Entry row) throws IOException; // no double-check
public void addUnique(List<Row.Entry> rows) throws IOException; // no double-check
//public long inc(final byte[] key, int col, long add, Row.Entry initrow); // replace a column with a recomputed value
public ArrayList<RowCollection> removeDoubles() throws IOException; // removes all elements that are double (to be used after all addUnique)
public Row.Entry remove(byte[] key) throws IOException;
public Row.Entry removeOne() throws IOException;

28
source/de/anomic/kelondro/table/FlexTable.java
View File

@ -223,7 +223,18 @@ public class FlexTable extends FlexWidthArray implements ObjectIndex {
super.setMultiple(old_rows_ordered);
// write new entries to index
addUnique(new_rows_sequential);
// add a list of entries in a ordered way.
// this should save R/W head positioning time
final TreeMap<Integer, byte[]> indexed_result = super.addMultiple(new_rows_sequential);
// indexed_result is a Integer/byte[] relation
// that is used here to store the index
final Iterator<Map.Entry<Integer, byte[]>> j = indexed_result.entrySet().iterator();
Map.Entry<Integer, byte[]> entry;
while (j.hasNext()) {
entry = j.next();
index.put(entry.getValue(), entry.getKey().intValue());
}
assert this.size() == index.size() : "content.size() = " + this.size() + ", index.size() = " + index.size();
}
@ -291,21 +302,6 @@ public class FlexTable extends FlexWidthArray implements ObjectIndex {
index.putUnique(row.getColBytes(0), super.add(row));
}
public synchronized void addUnique(final List<Row.Entry> rows) throws IOException {
// add a list of entries in a ordered way.
// this should save R/W head positioning time
final TreeMap<Integer, byte[]> indexed_result = super.addMultiple(rows);
// indexed_result is a Integer/byte[] relation
// that is used here to store the index
final Iterator<Map.Entry<Integer, byte[]>> i = indexed_result.entrySet().iterator();
Map.Entry<Integer, byte[]> entry;
while (i.hasNext()) {
entry = i.next();
index.put(entry.getValue(), entry.getKey().intValue());
}
assert this.size() == index.size() : "content.size() = " + this.size() + ", index.size() = " + index.size();
}
public synchronized ArrayList<RowCollection> removeDoubles() throws IOException {
final ArrayList<RowCollection> report = new ArrayList<RowCollection>();
RowSet rows;

16
source/de/anomic/kelondro/table/SplitTable.java
View File

@ -328,12 +328,8 @@ public class SplitTable implements ObjectIndex {
}
public synchronized void addUnique(final Row.Entry row) throws IOException {
addUnique(row, null);
}
public synchronized void addUnique(final Row.Entry row, Date entryDate) throws IOException {
assert row.objectsize() <= this.rowdef.objectsize;
if ((entryDate == null) || (entryDate.after(new Date()))) entryDate = new Date(); // fix date
Date entryDate = new Date();
final String suffix = dateSuffix(entryDate);
if (suffix == null) return;
ObjectIndex table = tables.get(suffix);
@ -345,16 +341,6 @@ public class SplitTable implements ObjectIndex {
table.addUnique(row);
}
public synchronized void addUnique(final List<Row.Entry> rows) throws IOException {
final Iterator<Row.Entry> i = rows.iterator();
while (i.hasNext()) addUnique(i.next());
}
public synchronized void addUniqueMultiple(final List<Row.Entry> rows, final Date entryDate) throws IOException {
final Iterator<Row.Entry> i = rows.iterator();
while (i.hasNext()) addUnique(i.next(), entryDate);
}
public ArrayList<RowCollection> removeDoubles() throws IOException {
final Iterator<ObjectIndex> i = tables.values().iterator();
final ArrayList<RowCollection> report = new ArrayList<RowCollection>();

1
source/de/anomic/kelondro/text/ReferenceContainerArray.java
View File

@ -143,6 +143,7 @@ public final class ReferenceContainerArray {
}
public boolean hasNext() {
if (this.iterator == null) return false;
if (rot) return true;
return iterator.hasNext();
}

3
source/de/anomic/kelondro/util/DateFormatter.java
View File

@ -349,6 +349,9 @@ public final class DateFormatter {
public static Date parseShortSecond(final String timeString) throws ParseException {
return parse(FORMAT_SHORT_SECOND, timeString);
}
public static Date parseShortMilliSecond(final String timeString) throws ParseException {
return parse(FORMAT_SHORT_MILSEC, timeString);
}
/**
* Like {@link #parseShortSecond(String)} using additional timezone information provided in an

6
source/de/anomic/plasma/plasmaHTCache.java
View File

@ -67,7 +67,7 @@ public final class plasmaHTCache {
public static long maxCacheSize = 0l;
public static File cachePath = null;
public static String salt;
public static String prefix;
public static final Log log = new Log("HTCACHE");
@ -85,7 +85,7 @@ public final class plasmaHTCache {
cachePath = htCachePath;
maxCacheSize = CacheSizeMax;
salt = peerSalt;
prefix = peerSalt;
// reset old HTCache ?
String[] list = cachePath.list();
@ -136,7 +136,7 @@ public final class plasmaHTCache {
}
responseHeaderDB = new MapView(blob, 500);
try {
fileDBunbuffered = new BLOBArray(new File(cachePath, FILE_DB_NAME), salt, 12, Base64Order.enhancedCoder, 1024 * 1024 * 2);
fileDBunbuffered = new BLOBArray(new File(cachePath, FILE_DB_NAME), prefix, 12, Base64Order.enhancedCoder, 1024 * 1024 * 2);
fileDBunbuffered.setMaxSize(maxCacheSize);
fileDB = new BLOBCompressor(fileDBunbuffered, 2 * 1024 * 1024);
} catch (IOException e) {

2
source/de/anomic/plasma/plasmaWordIndex.java
View File

@ -109,7 +109,7 @@ public final class plasmaWordIndex {
final File indexPrimaryRoot,
final File indexSecondaryRoot,
final int entityCacheMaxSize,
final boolean useCommons,
final boolean useCommons,
final int redundancy,
final int partitionExponent,
final boolean useCell) throws IOException {