Marown/yacy_search_server
1
0
Fork 0
mirror of https://github.com/yacy/yacy_search_server.git synced 2024-09-19 00:01:41 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity

- smaller caches to save memory

Browse Source 
- close cloneable iterators to free memory
This commit is contained in:
Michael Peter Christen 2012-07-02 15:40:40 +02:00
parent 7249d9c9de
commit 8a82609360
16 changed files with 143 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
source/net/yacy/cora/order/CloneableIterator.java
View File

@ -28,8 +28,17 @@ import java.util.Iterator;
public interface CloneableIterator<E> extends Iterator<E> {
// clone the iterator using a modifier
// the modifier can be i.e. a re-start position
/**
* clone the iterator using a modifier
* the modifier can be i.e. a re-start position
* @param modifier
* @return
*/
public CloneableIterator<E> clone(Object modifier);
/**
* a CloneableIterator should be closed after usage to free resources
*/
public void close();
}

8
source/net/yacy/cora/order/CloneableMapIterator.java
View File

@ -35,7 +35,7 @@ public class CloneableMapIterator<E> implements CloneableIterator<E> {
E next, last;
Object start;
Iterator<E> iter;
public CloneableMapIterator(final TreeMap<E, ?> map, final E start) {
// map must contain eiter a byte[]/Object or a String/Object mapping.
@ -52,7 +52,7 @@ public class CloneableMapIterator<E> implements CloneableIterator<E> {
}
this.last = null;
}
@Override
@SuppressWarnings("unchecked")
public CloneableMapIterator<E> clone(final Object modifier) {
@ -81,4 +81,8 @@ public class CloneableMapIterator<E> implements CloneableIterator<E> {
this.map.remove(this.last);
}
@Override
public void close() {
}
}

1
source/net/yacy/kelondro/blob/ArrayStack.java
View File

@ -1010,6 +1010,7 @@ public class ArrayStack implements BLOB {
final HeapWriter writer = new HeapWriter(tmpFile, newFile, keylength, order, writeBuffer);
rewrite(i, order, writer);
writer.close(true);
i.close();
} catch (final IOException e) {
Log.logSevere("ArrayStack", "cannot writing or close writing rewrite, newFile = " + newFile.toString() + ", tmpFile = " + tmpFile.toString() + ": " + e.getMessage(), e);
FileUtils.deletedelete(tmpFile);

4
source/net/yacy/kelondro/blob/BEncodedHeap.java
View File

@ -121,6 +121,10 @@ public class BEncodedHeap implements MapStore {
return this;
}
@Override
public void close() {
}
};
}
}

9
source/net/yacy/kelondro/blob/MapHeap.java
View File

@ -385,14 +385,15 @@ public class MapHeap implements Map<byte[], Map<String, String>> {
final boolean up, rotating;
final byte[] firstKey, secondKey;
Iterator<byte[]> iterator;
final private CloneableIterator<byte[]> blobkeys;
public KeyIterator(final boolean up, final boolean rotating, final byte[] firstKey, final byte[] secondKey) throws IOException {
this.up = up;
this.rotating = rotating;
this.firstKey = firstKey;
this.secondKey = secondKey;
final CloneableIterator<byte[]> i = MapHeap.this.blob.keys(up, firstKey);
this.iterator = rotating ? new RotateIterator<byte[]>(i, secondKey, MapHeap.this.blob.size()) : i;
this.blobkeys = MapHeap.this.blob.keys(up, firstKey);
this.iterator = rotating ? new RotateIterator<byte[]>(this.blobkeys, secondKey, MapHeap.this.blob.size()) : this.blobkeys;
}
@Override
@ -421,6 +422,10 @@ public class MapHeap implements Map<byte[], Map<String, String>> {
}
}
@Override
public void close() {
this.blobkeys.close();
}
}
public synchronized Iterator<Map.Entry<byte[], Map<String, String>>> entries(final boolean up, final boolean rotating) throws IOException {

2
source/net/yacy/kelondro/data/word/Word.java
View File

@ -56,7 +56,7 @@ public class Word {
*/
public static final int commonHashLength = 12;
private static final int hashCacheSize = Math.max(200000, Math.min(10000000, (int) (MemoryControl.available() / 20000L)));
private static final int hashCacheSize = Math.max(100000, Math.min(1000000, (int) (MemoryControl.available() / 40000L)));
private static ARC<String, byte[]> hashCache = null;
static {
try {

8
source/net/yacy/kelondro/index/RowSet.java
View File

@ -411,6 +411,10 @@ public class RowSet extends RowCollection implements Index, Iterable<Row.Entry>,
public final void remove() {
throw new UnsupportedOperationException();
}
@Override
public void close() {
}
}
@Override
@ -478,6 +482,10 @@ public class RowSet extends RowCollection implements Index, Iterable<Row.Entry>,
public final void remove() {
throw new UnsupportedOperationException();
}
@Override
public void close() {
}
}
/**

2
source/net/yacy/kelondro/order/Digest.java
View File

@ -56,7 +56,7 @@ public class Digest {
public static BlockingQueue<MessageDigest> digestPool = new LinkedBlockingDeque<MessageDigest>();
private static final int md5CacheSize = Math.max(200000, Math.min(10000000, (int) (MemoryControl.available() / 20000L)));
private static final int md5CacheSize = Math.max(100000, Math.min(1000000, (int) (MemoryControl.available() / 40000L)));
private static ARC<String, byte[]> md5Cache = null;
static {
try {

12
source/net/yacy/kelondro/order/MergeIterator.java
View File

@ -61,6 +61,12 @@ public class MergeIterator<E> implements CloneableIterator<E> {
nextb();
}
@Override
public void close() {
this.a.close();
this.b.close();
}
@Override
public MergeIterator<E> clone(final Object modifier) {
assert this.a != null;
@ -159,6 +165,9 @@ public class MergeIterator<E> implements CloneableIterator<E> {
public CloneableIterator<A> clone(Object modifier) {
return this;
}
@Override
public void close() {
}
};
return cascade(iterators.iterator(), c, merger, up);
}
@ -180,6 +189,9 @@ public class MergeIterator<E> implements CloneableIterator<E> {
public CloneableIterator<A> clone(Object modifier) {
return this;
}
@Override
public void close() {
}
};
final CloneableIterator<A> one = iiterators.next();
if (!(iiterators.hasNext())) return one;

44
source/net/yacy/kelondro/order/RotateIterator.java
View File

@ -9,7 +9,7 @@
// $LastChangedBy$
//
// LICENSE
//
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation; either version 2 of the License, or
@ -30,12 +30,12 @@ import net.yacy.cora.order.CloneableIterator;
public class RotateIterator<E> implements CloneableIterator<E> {
CloneableIterator<E> a, clone;
Object modifier;
boolean nempty;
int terminationCount;
public RotateIterator(final CloneableIterator<E> a, final Object modifier, final int terminationCount) {
// this works currently only for String-type key iterations
this.a = a;
@ -44,29 +44,39 @@ public class RotateIterator<E> implements CloneableIterator<E> {
this.clone = a.clone(modifier);
this.nempty = this.clone.hasNext();
}
public RotateIterator<E> clone(final Object modifier) {
return new RotateIterator<E>(a, modifier, terminationCount - 1);
@Override
public RotateIterator<E> clone(final Object modifier) {
return new RotateIterator<E>(this.a, modifier, this.terminationCount - 1);
}
@Override
public boolean hasNext() {
return (terminationCount > 0) && (this.nempty);
return (this.terminationCount > 0) && (this.nempty);
}
@Override
public E next() {
// attention: this iterator has no termination - on purpose.
// it must be taken care that a calling method has a termination predicate different
// from the hasNext() method
if (!(a.hasNext())) {
a = clone.clone(modifier);
assert a.hasNext();
if (!(this.a.hasNext())) {
this.a = this.clone.clone(this.modifier);
assert this.a.hasNext();
}
terminationCount--;
return a.next();
this.terminationCount--;
return this.a.next();
}
@Override
public void remove() {
a.remove();
this.a.remove();
}
@Override
public void close() {
this.a.close();
this.clone.close();
}
}

9
source/net/yacy/kelondro/order/StackIterator.java
View File

@ -111,6 +111,9 @@ public class StackIterator<E> implements CloneableIterator<E> {
public CloneableIterator<A> clone(Object modifier) {
return null;
}
@Override
public void close() {
}
};
if (iterators.length == 1) {
return iterators[0];
@ -126,4 +129,10 @@ public class StackIterator<E> implements CloneableIterator<E> {
if (a == null) return stack(iterators0);
return new StackIterator<A>(a, stack(iterators0));
}
@Override
public void close() {
this.a.close();
this.b.close();
}
}

13
source/net/yacy/kelondro/rwi/ReferenceContainerArray.java
View File

@ -194,6 +194,11 @@ public final class ReferenceContainerArray<ReferenceType extends Reference> {
return this;
}
@Override
public void close() {
this.iterator.close();
}
}
/**
@ -276,6 +281,11 @@ public final class ReferenceContainerArray<ReferenceType extends Reference> {
return this;
}
@Override
public void close() {
this.iterator.close();
}
}
/**
@ -446,7 +456,7 @@ public final class ReferenceContainerArray<ReferenceType extends Reference> {
if (f.length() < 22 || !f.startsWith("text.index") || !f.endsWith(".blob")) continue;
final File fl = new File(heapLocation, f);
System.out.println("CELL REFERENCE COLLECTION opening blob " + fl);
final CloneableIterator<ReferenceContainer<ReferenceType>> ei = new ReferenceIterator<ReferenceType>(fl, factory);
final CloneableIterator<ReferenceContainer<ReferenceType>> ei = new ReferenceIterator<ReferenceType>(fl, factory);
ReferenceContainer<ReferenceType> container;
final long start = System.currentTimeMillis();
@ -472,6 +482,7 @@ public final class ReferenceContainerArray<ReferenceType extends Reference> {
lastlog = System.currentTimeMillis();
}
}
ei.close();
}
references.trim();
System.out.println("CELL REFERENCE COLLECTION finished");

6
source/net/yacy/kelondro/rwi/ReferenceContainerCache.java
View File

@ -319,6 +319,9 @@ public final class ReferenceContainerCache<ReferenceType extends Reference> exte
return this;
}
@Override
public void close() {
}
}
@Override
@ -401,6 +404,9 @@ public final class ReferenceContainerCache<ReferenceType extends Reference> exte
return this;
}
@Override
public void close() {
}
}
/**

31
source/net/yacy/kelondro/table/Table.java
View File

@ -809,6 +809,7 @@ public class Table implements Index, Iterable<Row.Entry> {
return this.rowdef.newEntry(b);
}
@Override
public synchronized Entry removeOne() throws IOException {
assert this.file.size() == this.index.size() : "file.size() = " + this.file.size() + ", index.size() = " + this.index.size();
assert this.table == null || this.table.size() == this.index.size() : "table.size() = " + this.table.size() + ", index.size() = " + this.index.size();
@ -835,6 +836,7 @@ public class Table implements Index, Iterable<Row.Entry> {
return lr;
}
@Override
public List<Row.Entry> top(int count) throws IOException {
final ArrayList<Row.Entry> list = new ArrayList<Row.Entry>();
if ((this.file == null) || (this.index == null)) return list;
@ -849,6 +851,7 @@ public class Table implements Index, Iterable<Row.Entry> {
return list;
}
@Override
public synchronized void clear() throws IOException {
final File f = this.file.filename();
this.file.close();
@ -873,19 +876,23 @@ public class Table implements Index, Iterable<Row.Entry> {
this.index.clear();
}
@Override
public Row row() {
return this.rowdef;
}
@Override
public int size() {
if (this.index == null) return 0;
return this.index.size();
}
@Override
public boolean isEmpty() {
return this.index.isEmpty();
}
@Override
public Iterator<Entry> iterator() {
try {
return rows();
@ -894,6 +901,7 @@ public class Table implements Index, Iterable<Row.Entry> {
}
}
@Override
public synchronized CloneableIterator<Entry> rows() throws IOException {
this.file.flushBuffer();
return new rowIteratorNoOrder();
@ -910,14 +918,17 @@ public class Table implements Index, Iterable<Row.Entry> {
this.i = Table.this.index.iterator();
}
@Override
public CloneableIterator<Entry> clone(final Object modifier) {
return new rowIteratorNoOrder();
}
@Override
public boolean hasNext() {
return this.i != null && this.i.hasNext();
}
@Override
public Entry next() {
final Row.Entry entry = this.i.next();
if (entry == null) return null;
@ -931,6 +942,7 @@ public class Table implements Index, Iterable<Row.Entry> {
}
}
@Override
public void remove() {
if (this.key != null) {
try {
@ -942,14 +954,22 @@ public class Table implements Index, Iterable<Row.Entry> {
}
}
@Override
public void close() {
if (this.i instanceof CloneableIterator) {
((CloneableIterator<Entry>) this.i).close();
}
}
}
@Override
public synchronized CloneableIterator<Entry> rows(final boolean up, final byte[] firstKey) throws IOException {
return new rowIterator(up, firstKey);
}
private class rowIterator implements CloneableIterator<Entry> {
private final Iterator<byte[]> i;
private final CloneableIterator<byte[]> i;
private final boolean up;
private final byte[] fk;
private int c;
@ -961,14 +981,17 @@ public class Table implements Index, Iterable<Row.Entry> {
this.c = -1;
}
@Override
public CloneableIterator<Entry> clone(final Object modifier) {
return new rowIterator(this.up, this.fk);
}
@Override
public boolean hasNext() {
return this.i.hasNext();
}
@Override
public Entry next() {
final byte[] k = this.i.next();
assert k != null;
@ -995,10 +1018,15 @@ public class Table implements Index, Iterable<Row.Entry> {
return Table.this.rowdef.newEntry(b);
}
@Override
public void remove() {
throw new UnsupportedOperationException("no remove in Table.rowIterator");
}
@Override
public void close() {
this.i.close();
}
}
private static byte[] testWord(final char c) {
@ -1125,6 +1153,7 @@ public class Table implements Index, Iterable<Row.Entry> {
*/
}
@Override
public void deleteOnExit() {
this.file.deleteOnExit();
}

7
source/net/yacy/search/index/MetadataRepository.java
View File

@ -281,7 +281,7 @@ public final class MetadataRepository implements /*Metadata,*/ Iterable<byte[]>
public class kiter implements CloneableIterator<URIMetadataRow> {
// enumerates entry elements
private final Iterator<Row.Entry> iter;
private final CloneableIterator<Row.Entry> iter;
private final boolean error;
boolean up;
@ -326,6 +326,11 @@ public final class MetadataRepository implements /*Metadata,*/ Iterable<byte[]>
public final void remove() {
this.iter.remove();
}
@Override
public void close() {
this.iter.close();
}
}

1
source/net/yacy/search/ranking/BlockRank.java
View File

@ -144,6 +144,7 @@ public class BlockRank {
final ReferenceContainer<HostReference> references = ri.next();
index.add(references);
}
ri.close();
} catch (final IOException e) {
Log.logException(e);
} catch (final RowSpaceExceededException e) {