mirror of
https://github.com/yacy/yacy_search_server.git
synced 2024-09-19 00:01:41 +02:00
- smaller caches to save memory
- close cloneable iterators to free memory
This commit is contained in:
parent
7249d9c9de
commit
8a82609360
|
@ -28,8 +28,17 @@ import java.util.Iterator;
|
|||
|
||||
public interface CloneableIterator<E> extends Iterator<E> {
|
||||
|
||||
// clone the iterator using a modifier
|
||||
// the modifier can be i.e. a re-start position
|
||||
/**
|
||||
* clone the iterator using a modifier
|
||||
* the modifier can be i.e. a re-start position
|
||||
* @param modifier
|
||||
* @return
|
||||
*/
|
||||
public CloneableIterator<E> clone(Object modifier);
|
||||
|
||||
/**
|
||||
* a CloneableIterator should be closed after usage to free resources
|
||||
*/
|
||||
public void close();
|
||||
|
||||
}
|
||||
|
|
|
@ -81,4 +81,8 @@ public class CloneableMapIterator<E> implements CloneableIterator<E> {
|
|||
this.map.remove(this.last);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void close() {
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
@ -1010,6 +1010,7 @@ public class ArrayStack implements BLOB {
|
|||
final HeapWriter writer = new HeapWriter(tmpFile, newFile, keylength, order, writeBuffer);
|
||||
rewrite(i, order, writer);
|
||||
writer.close(true);
|
||||
i.close();
|
||||
} catch (final IOException e) {
|
||||
Log.logSevere("ArrayStack", "cannot writing or close writing rewrite, newFile = " + newFile.toString() + ", tmpFile = " + tmpFile.toString() + ": " + e.getMessage(), e);
|
||||
FileUtils.deletedelete(tmpFile);
|
||||
|
|
|
@ -121,6 +121,10 @@ public class BEncodedHeap implements MapStore {
|
|||
return this;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void close() {
|
||||
}
|
||||
|
||||
};
|
||||
}
|
||||
}
|
||||
|
|
|
@ -385,14 +385,15 @@ public class MapHeap implements Map<byte[], Map<String, String>> {
|
|||
final boolean up, rotating;
|
||||
final byte[] firstKey, secondKey;
|
||||
Iterator<byte[]> iterator;
|
||||
final private CloneableIterator<byte[]> blobkeys;
|
||||
|
||||
public KeyIterator(final boolean up, final boolean rotating, final byte[] firstKey, final byte[] secondKey) throws IOException {
|
||||
this.up = up;
|
||||
this.rotating = rotating;
|
||||
this.firstKey = firstKey;
|
||||
this.secondKey = secondKey;
|
||||
final CloneableIterator<byte[]> i = MapHeap.this.blob.keys(up, firstKey);
|
||||
this.iterator = rotating ? new RotateIterator<byte[]>(i, secondKey, MapHeap.this.blob.size()) : i;
|
||||
this.blobkeys = MapHeap.this.blob.keys(up, firstKey);
|
||||
this.iterator = rotating ? new RotateIterator<byte[]>(this.blobkeys, secondKey, MapHeap.this.blob.size()) : this.blobkeys;
|
||||
}
|
||||
|
||||
@Override
|
||||
|
@ -421,6 +422,10 @@ public class MapHeap implements Map<byte[], Map<String, String>> {
|
|||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void close() {
|
||||
this.blobkeys.close();
|
||||
}
|
||||
}
|
||||
|
||||
public synchronized Iterator<Map.Entry<byte[], Map<String, String>>> entries(final boolean up, final boolean rotating) throws IOException {
|
||||
|
|
|
@ -56,7 +56,7 @@ public class Word {
|
|||
*/
|
||||
public static final int commonHashLength = 12;
|
||||
|
||||
private static final int hashCacheSize = Math.max(200000, Math.min(10000000, (int) (MemoryControl.available() / 20000L)));
|
||||
private static final int hashCacheSize = Math.max(100000, Math.min(1000000, (int) (MemoryControl.available() / 40000L)));
|
||||
private static ARC<String, byte[]> hashCache = null;
|
||||
static {
|
||||
try {
|
||||
|
|
|
@ -411,6 +411,10 @@ public class RowSet extends RowCollection implements Index, Iterable<Row.Entry>,
|
|||
public final void remove() {
|
||||
throw new UnsupportedOperationException();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void close() {
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
|
@ -478,6 +482,10 @@ public class RowSet extends RowCollection implements Index, Iterable<Row.Entry>,
|
|||
public final void remove() {
|
||||
throw new UnsupportedOperationException();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void close() {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
|
@ -56,7 +56,7 @@ public class Digest {
|
|||
|
||||
public static BlockingQueue<MessageDigest> digestPool = new LinkedBlockingDeque<MessageDigest>();
|
||||
|
||||
private static final int md5CacheSize = Math.max(200000, Math.min(10000000, (int) (MemoryControl.available() / 20000L)));
|
||||
private static final int md5CacheSize = Math.max(100000, Math.min(1000000, (int) (MemoryControl.available() / 40000L)));
|
||||
private static ARC<String, byte[]> md5Cache = null;
|
||||
static {
|
||||
try {
|
||||
|
|
|
@ -61,6 +61,12 @@ public class MergeIterator<E> implements CloneableIterator<E> {
|
|||
nextb();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void close() {
|
||||
this.a.close();
|
||||
this.b.close();
|
||||
}
|
||||
|
||||
@Override
|
||||
public MergeIterator<E> clone(final Object modifier) {
|
||||
assert this.a != null;
|
||||
|
@ -159,6 +165,9 @@ public class MergeIterator<E> implements CloneableIterator<E> {
|
|||
public CloneableIterator<A> clone(Object modifier) {
|
||||
return this;
|
||||
}
|
||||
@Override
|
||||
public void close() {
|
||||
}
|
||||
};
|
||||
return cascade(iterators.iterator(), c, merger, up);
|
||||
}
|
||||
|
@ -180,6 +189,9 @@ public class MergeIterator<E> implements CloneableIterator<E> {
|
|||
public CloneableIterator<A> clone(Object modifier) {
|
||||
return this;
|
||||
}
|
||||
@Override
|
||||
public void close() {
|
||||
}
|
||||
};
|
||||
final CloneableIterator<A> one = iiterators.next();
|
||||
if (!(iiterators.hasNext())) return one;
|
||||
|
|
|
@ -45,28 +45,38 @@ public class RotateIterator<E> implements CloneableIterator<E> {
|
|||
this.nempty = this.clone.hasNext();
|
||||
}
|
||||
|
||||
@Override
|
||||
public RotateIterator<E> clone(final Object modifier) {
|
||||
return new RotateIterator<E>(a, modifier, terminationCount - 1);
|
||||
return new RotateIterator<E>(this.a, modifier, this.terminationCount - 1);
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean hasNext() {
|
||||
return (terminationCount > 0) && (this.nempty);
|
||||
return (this.terminationCount > 0) && (this.nempty);
|
||||
}
|
||||
|
||||
@Override
|
||||
public E next() {
|
||||
// attention: this iterator has no termination - on purpose.
|
||||
// it must be taken care that a calling method has a termination predicate different
|
||||
// from the hasNext() method
|
||||
if (!(a.hasNext())) {
|
||||
a = clone.clone(modifier);
|
||||
assert a.hasNext();
|
||||
if (!(this.a.hasNext())) {
|
||||
this.a = this.clone.clone(this.modifier);
|
||||
assert this.a.hasNext();
|
||||
}
|
||||
terminationCount--;
|
||||
return a.next();
|
||||
this.terminationCount--;
|
||||
return this.a.next();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void remove() {
|
||||
a.remove();
|
||||
this.a.remove();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void close() {
|
||||
this.a.close();
|
||||
this.clone.close();
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
@ -111,6 +111,9 @@ public class StackIterator<E> implements CloneableIterator<E> {
|
|||
public CloneableIterator<A> clone(Object modifier) {
|
||||
return null;
|
||||
}
|
||||
@Override
|
||||
public void close() {
|
||||
}
|
||||
};
|
||||
if (iterators.length == 1) {
|
||||
return iterators[0];
|
||||
|
@ -126,4 +129,10 @@ public class StackIterator<E> implements CloneableIterator<E> {
|
|||
if (a == null) return stack(iterators0);
|
||||
return new StackIterator<A>(a, stack(iterators0));
|
||||
}
|
||||
|
||||
@Override
|
||||
public void close() {
|
||||
this.a.close();
|
||||
this.b.close();
|
||||
}
|
||||
}
|
||||
|
|
|
@ -194,6 +194,11 @@ public final class ReferenceContainerArray<ReferenceType extends Reference> {
|
|||
return this;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void close() {
|
||||
this.iterator.close();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
/**
|
||||
|
@ -276,6 +281,11 @@ public final class ReferenceContainerArray<ReferenceType extends Reference> {
|
|||
return this;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void close() {
|
||||
this.iterator.close();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
/**
|
||||
|
@ -472,6 +482,7 @@ public final class ReferenceContainerArray<ReferenceType extends Reference> {
|
|||
lastlog = System.currentTimeMillis();
|
||||
}
|
||||
}
|
||||
ei.close();
|
||||
}
|
||||
references.trim();
|
||||
System.out.println("CELL REFERENCE COLLECTION finished");
|
||||
|
|
|
@ -319,6 +319,9 @@ public final class ReferenceContainerCache<ReferenceType extends Reference> exte
|
|||
return this;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void close() {
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
|
@ -401,6 +404,9 @@ public final class ReferenceContainerCache<ReferenceType extends Reference> exte
|
|||
return this;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void close() {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
|
@ -809,6 +809,7 @@ public class Table implements Index, Iterable<Row.Entry> {
|
|||
return this.rowdef.newEntry(b);
|
||||
}
|
||||
|
||||
@Override
|
||||
public synchronized Entry removeOne() throws IOException {
|
||||
assert this.file.size() == this.index.size() : "file.size() = " + this.file.size() + ", index.size() = " + this.index.size();
|
||||
assert this.table == null || this.table.size() == this.index.size() : "table.size() = " + this.table.size() + ", index.size() = " + this.index.size();
|
||||
|
@ -835,6 +836,7 @@ public class Table implements Index, Iterable<Row.Entry> {
|
|||
return lr;
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<Row.Entry> top(int count) throws IOException {
|
||||
final ArrayList<Row.Entry> list = new ArrayList<Row.Entry>();
|
||||
if ((this.file == null) || (this.index == null)) return list;
|
||||
|
@ -849,6 +851,7 @@ public class Table implements Index, Iterable<Row.Entry> {
|
|||
return list;
|
||||
}
|
||||
|
||||
@Override
|
||||
public synchronized void clear() throws IOException {
|
||||
final File f = this.file.filename();
|
||||
this.file.close();
|
||||
|
@ -873,19 +876,23 @@ public class Table implements Index, Iterable<Row.Entry> {
|
|||
this.index.clear();
|
||||
}
|
||||
|
||||
@Override
|
||||
public Row row() {
|
||||
return this.rowdef;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int size() {
|
||||
if (this.index == null) return 0;
|
||||
return this.index.size();
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean isEmpty() {
|
||||
return this.index.isEmpty();
|
||||
}
|
||||
|
||||
@Override
|
||||
public Iterator<Entry> iterator() {
|
||||
try {
|
||||
return rows();
|
||||
|
@ -894,6 +901,7 @@ public class Table implements Index, Iterable<Row.Entry> {
|
|||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public synchronized CloneableIterator<Entry> rows() throws IOException {
|
||||
this.file.flushBuffer();
|
||||
return new rowIteratorNoOrder();
|
||||
|
@ -910,14 +918,17 @@ public class Table implements Index, Iterable<Row.Entry> {
|
|||
this.i = Table.this.index.iterator();
|
||||
}
|
||||
|
||||
@Override
|
||||
public CloneableIterator<Entry> clone(final Object modifier) {
|
||||
return new rowIteratorNoOrder();
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean hasNext() {
|
||||
return this.i != null && this.i.hasNext();
|
||||
}
|
||||
|
||||
@Override
|
||||
public Entry next() {
|
||||
final Row.Entry entry = this.i.next();
|
||||
if (entry == null) return null;
|
||||
|
@ -931,6 +942,7 @@ public class Table implements Index, Iterable<Row.Entry> {
|
|||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void remove() {
|
||||
if (this.key != null) {
|
||||
try {
|
||||
|
@ -942,14 +954,22 @@ public class Table implements Index, Iterable<Row.Entry> {
|
|||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void close() {
|
||||
if (this.i instanceof CloneableIterator) {
|
||||
((CloneableIterator<Entry>) this.i).close();
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@Override
|
||||
public synchronized CloneableIterator<Entry> rows(final boolean up, final byte[] firstKey) throws IOException {
|
||||
return new rowIterator(up, firstKey);
|
||||
}
|
||||
|
||||
private class rowIterator implements CloneableIterator<Entry> {
|
||||
private final Iterator<byte[]> i;
|
||||
private final CloneableIterator<byte[]> i;
|
||||
private final boolean up;
|
||||
private final byte[] fk;
|
||||
private int c;
|
||||
|
@ -961,14 +981,17 @@ public class Table implements Index, Iterable<Row.Entry> {
|
|||
this.c = -1;
|
||||
}
|
||||
|
||||
@Override
|
||||
public CloneableIterator<Entry> clone(final Object modifier) {
|
||||
return new rowIterator(this.up, this.fk);
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean hasNext() {
|
||||
return this.i.hasNext();
|
||||
}
|
||||
|
||||
@Override
|
||||
public Entry next() {
|
||||
final byte[] k = this.i.next();
|
||||
assert k != null;
|
||||
|
@ -995,10 +1018,15 @@ public class Table implements Index, Iterable<Row.Entry> {
|
|||
return Table.this.rowdef.newEntry(b);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void remove() {
|
||||
throw new UnsupportedOperationException("no remove in Table.rowIterator");
|
||||
}
|
||||
|
||||
@Override
|
||||
public void close() {
|
||||
this.i.close();
|
||||
}
|
||||
}
|
||||
|
||||
private static byte[] testWord(final char c) {
|
||||
|
@ -1125,6 +1153,7 @@ public class Table implements Index, Iterable<Row.Entry> {
|
|||
*/
|
||||
}
|
||||
|
||||
@Override
|
||||
public void deleteOnExit() {
|
||||
this.file.deleteOnExit();
|
||||
}
|
||||
|
|
|
@ -281,7 +281,7 @@ public final class MetadataRepository implements /*Metadata,*/ Iterable<byte[]>
|
|||
|
||||
public class kiter implements CloneableIterator<URIMetadataRow> {
|
||||
// enumerates entry elements
|
||||
private final Iterator<Row.Entry> iter;
|
||||
private final CloneableIterator<Row.Entry> iter;
|
||||
private final boolean error;
|
||||
boolean up;
|
||||
|
||||
|
@ -326,6 +326,11 @@ public final class MetadataRepository implements /*Metadata,*/ Iterable<byte[]>
|
|||
public final void remove() {
|
||||
this.iter.remove();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void close() {
|
||||
this.iter.close();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
|
|
@ -144,6 +144,7 @@ public class BlockRank {
|
|||
final ReferenceContainer<HostReference> references = ri.next();
|
||||
index.add(references);
|
||||
}
|
||||
ri.close();
|
||||
} catch (final IOException e) {
|
||||
Log.logException(e);
|
||||
} catch (final RowSpaceExceededException e) {
|
||||
|
|
Loading…
Reference in New Issue
Block a user