mirror of
https://github.com/yacy/yacy_search_server.git
synced 2024-09-19 00:01:41 +02:00
some enhancements to caching and kelondroRA-methods
git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@2081 6c8d7289-2bf4-0310-a012-ef5d649a1542
This commit is contained in:
parent
2e89477f22
commit
35995cf8c7
|
@ -44,7 +44,11 @@
|
||||||
|
|
||||||
package de.anomic.kelondro;
|
package de.anomic.kelondro;
|
||||||
|
|
||||||
|
import java.io.BufferedReader;
|
||||||
|
import java.io.ByteArrayInputStream;
|
||||||
|
import java.io.ByteArrayOutputStream;
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
|
import java.io.InputStreamReader;
|
||||||
import java.util.Enumeration;
|
import java.util.Enumeration;
|
||||||
import java.util.Iterator;
|
import java.util.Iterator;
|
||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
|
@ -80,7 +84,21 @@ abstract class kelondroAbstractRA implements kelondroRA {
|
||||||
len -= r;
|
len -= r;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
/*
|
||||||
|
public byte[] readFully() throws IOException {
|
||||||
|
ByteArrayOutputStream dest = new ByteArrayOutputStream();
|
||||||
|
byte[] buffer = new byte[1024];
|
||||||
|
|
||||||
|
int c, total = 0;
|
||||||
|
while ((c = read(buffer, 0, 1024)) > 0) {
|
||||||
|
dest.write(buffer, 0, c);
|
||||||
|
total += c;
|
||||||
|
}
|
||||||
|
dest.flush();
|
||||||
|
dest.close();
|
||||||
|
return dest.toByteArray();
|
||||||
|
}
|
||||||
|
*/
|
||||||
public byte readByte() throws IOException {
|
public byte readByte() throws IOException {
|
||||||
final int ch = this.read();
|
final int ch = this.read();
|
||||||
if (ch < 0) throw new IOException();
|
if (ch < 0) throw new IOException();
|
||||||
|
@ -135,9 +153,19 @@ abstract class kelondroAbstractRA implements kelondroRA {
|
||||||
private static final byte lf = 10;
|
private static final byte lf = 10;
|
||||||
|
|
||||||
public void writeLine(final String line) throws IOException {
|
public void writeLine(final String line) throws IOException {
|
||||||
this.write(line.getBytes());
|
byte[] b = new byte[line.length() + 2];
|
||||||
this.write(cr);
|
System.arraycopy(line.getBytes(), 0, b, 0, line.length());
|
||||||
this.write(lf);
|
b[b.length - 2] = cr;
|
||||||
|
b[b.length - 1] = lf;
|
||||||
|
this.write(b);
|
||||||
|
}
|
||||||
|
|
||||||
|
public void writeLine(final byte[] line) throws IOException {
|
||||||
|
byte[] b = new byte[line.length + 2];
|
||||||
|
System.arraycopy(line, 0, b, 0, line.length);
|
||||||
|
b[b.length - 2] = cr;
|
||||||
|
b[b.length - 1] = lf;
|
||||||
|
this.write(b);
|
||||||
}
|
}
|
||||||
|
|
||||||
public String readLine() throws IOException {
|
public String readLine() throws IOException {
|
||||||
|
@ -166,6 +194,30 @@ abstract class kelondroAbstractRA implements kelondroRA {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/*
|
||||||
|
public void writeProperties(final Properties props, final String comment) throws IOException {
|
||||||
|
this.seek(0);
|
||||||
|
final Enumeration e = props.propertyNames();
|
||||||
|
String key, value;
|
||||||
|
StringBuffer sb = new StringBuffer(props.size() * 40);
|
||||||
|
sb.append("# " + comment);
|
||||||
|
sb.append(cr);
|
||||||
|
sb.append(lf);
|
||||||
|
while (e.hasMoreElements()) {
|
||||||
|
key = (String) e.nextElement();
|
||||||
|
value = props.getProperty(key, "");
|
||||||
|
sb.append(key);
|
||||||
|
sb.append('=');
|
||||||
|
sb.append(value);
|
||||||
|
sb.append(cr);
|
||||||
|
sb.append(lf);
|
||||||
|
}
|
||||||
|
sb.append("# EOF");
|
||||||
|
sb.append(cr);
|
||||||
|
sb.append(lf);
|
||||||
|
write(new String(sb).getBytes());
|
||||||
|
}
|
||||||
|
*/
|
||||||
public void writeProperties(final Properties props, final String comment) throws IOException {
|
public void writeProperties(final Properties props, final String comment) throws IOException {
|
||||||
this.seek(0);
|
this.seek(0);
|
||||||
writeLine("# " + comment);
|
writeLine("# " + comment);
|
||||||
|
@ -181,6 +233,26 @@ abstract class kelondroAbstractRA implements kelondroRA {
|
||||||
writeLine("# EOF");
|
writeLine("# EOF");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/*
|
||||||
|
public Properties readProperties() throws IOException {
|
||||||
|
this.seek(0);
|
||||||
|
byte[] b = readFully();
|
||||||
|
//System.out.println("DEBUG-Properties:" + new String(b));
|
||||||
|
BufferedReader br = new BufferedReader(new InputStreamReader(new ByteArrayInputStream(b)));
|
||||||
|
final Properties props = new Properties();
|
||||||
|
String line;
|
||||||
|
int pos;
|
||||||
|
while ((line = br.readLine()) != null) {
|
||||||
|
line = line.trim();
|
||||||
|
if (line.equals("# EOF")) return props;
|
||||||
|
if ((line.length() == 0) || (line.charAt(0) == '#')) continue;
|
||||||
|
pos = line.indexOf("=");
|
||||||
|
if (pos < 0) continue;
|
||||||
|
props.setProperty(line.substring(0, pos).trim(), line.substring(pos + 1).trim());
|
||||||
|
}
|
||||||
|
return props;
|
||||||
|
}
|
||||||
|
*/
|
||||||
public Properties readProperties() throws IOException {
|
public Properties readProperties() throws IOException {
|
||||||
this.seek(0);
|
this.seek(0);
|
||||||
final Properties props = new Properties();
|
final Properties props = new Properties();
|
||||||
|
@ -196,7 +268,30 @@ abstract class kelondroAbstractRA implements kelondroRA {
|
||||||
}
|
}
|
||||||
return props;
|
return props;
|
||||||
}
|
}
|
||||||
|
/*
|
||||||
|
public void writeMap(final Map map, final String comment) throws IOException {
|
||||||
|
this.seek(0);
|
||||||
|
final Iterator iter = map.entrySet().iterator();
|
||||||
|
Map.Entry entry;
|
||||||
|
StringBuffer sb = new StringBuffer(map.size() * 40);
|
||||||
|
sb.append("# " + comment);
|
||||||
|
sb.append(cr);
|
||||||
|
sb.append(lf);
|
||||||
|
while (iter.hasNext()) {
|
||||||
|
entry = (Map.Entry) iter.next();
|
||||||
|
sb.append((String) entry.getKey());
|
||||||
|
sb.append('=');
|
||||||
|
sb.append((String) entry.getValue());
|
||||||
|
sb.append(cr);
|
||||||
|
sb.append(lf);
|
||||||
|
}
|
||||||
|
sb.append("# EOF");
|
||||||
|
sb.append(cr);
|
||||||
|
sb.append(lf);
|
||||||
|
//System.out.println("DEBUG-WRITE-MAP:" + new String(sb));
|
||||||
|
write(new String(sb).getBytes());
|
||||||
|
}
|
||||||
|
*/
|
||||||
public void writeMap(final Map map, final String comment) throws IOException {
|
public void writeMap(final Map map, final String comment) throws IOException {
|
||||||
this.seek(0);
|
this.seek(0);
|
||||||
writeLine("# " + comment);
|
writeLine("# " + comment);
|
||||||
|
@ -211,6 +306,26 @@ abstract class kelondroAbstractRA implements kelondroRA {
|
||||||
writeLine("# EOF");
|
writeLine("# EOF");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/*
|
||||||
|
public Map readMap() throws IOException {
|
||||||
|
this.seek(0);
|
||||||
|
byte[] b = readFully();
|
||||||
|
//System.out.println("DEBUG-READ-MAP:" + new String(b));
|
||||||
|
BufferedReader br = new BufferedReader(new InputStreamReader(new ByteArrayInputStream(b)));
|
||||||
|
final TreeMap map = new TreeMap();
|
||||||
|
String line;
|
||||||
|
int pos;
|
||||||
|
while ((line = br.readLine()) != null) { // very slow readLine????
|
||||||
|
line = line.trim();
|
||||||
|
if (line.equals("# EOF")) return map;
|
||||||
|
if ((line.length() == 0) || (line.charAt(0) == '#')) continue;
|
||||||
|
pos = line.indexOf("=");
|
||||||
|
if (pos < 0) continue;
|
||||||
|
map.put(line.substring(0, pos), line.substring(pos + 1));
|
||||||
|
}
|
||||||
|
return map;
|
||||||
|
}
|
||||||
|
*/
|
||||||
public Map readMap() throws IOException {
|
public Map readMap() throws IOException {
|
||||||
this.seek(0);
|
this.seek(0);
|
||||||
final TreeMap map = new TreeMap();
|
final TreeMap map = new TreeMap();
|
||||||
|
|
|
@ -339,8 +339,7 @@ public class kelondroDyn extends kelondroTree {
|
||||||
|
|
||||||
public int read(byte[] b, int off, int len) throws IOException {
|
public int read(byte[] b, int off, int len) throws IOException {
|
||||||
byte[] buf = getDyn(filekey, seekpos, len);
|
byte[] buf = getDyn(filekey, seekpos, len);
|
||||||
if (buf == null)
|
if (buf == null) return 0;
|
||||||
return 0;
|
|
||||||
System.arraycopy(buf, 0, b, off, len);
|
System.arraycopy(buf, 0, b, off, len);
|
||||||
seekpos += len;
|
seekpos += len;
|
||||||
return len;
|
return len;
|
||||||
|
|
|
@ -215,7 +215,7 @@ public class kelondroObjectCache {
|
||||||
) {
|
) {
|
||||||
cache.remove(k);
|
cache.remove(k);
|
||||||
ages.deleteScore(k);
|
ages.deleteScore(k);
|
||||||
if (Runtime.getRuntime().freeMemory() < minMem) System.gc(); // prevent unnecessary loops
|
//if (Runtime.getRuntime().freeMemory() < minMem) System.gc(); // prevent unnecessary loops
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -71,6 +71,7 @@ public interface kelondroRA {
|
||||||
|
|
||||||
// derived methods:
|
// derived methods:
|
||||||
public void readFully(byte[] b, int off, int len) throws IOException;
|
public void readFully(byte[] b, int off, int len) throws IOException;
|
||||||
|
//public byte[] readFully() throws IOException;
|
||||||
public byte readByte() throws IOException;
|
public byte readByte() throws IOException;
|
||||||
public void writeByte(int v) throws IOException;
|
public void writeByte(int v) throws IOException;
|
||||||
|
|
||||||
|
|
|
@ -410,12 +410,14 @@ public class yacyCore {
|
||||||
LinkedList seedList = new LinkedList();
|
LinkedList seedList = new LinkedList();
|
||||||
LinkedList tmpSeedList = new LinkedList();
|
LinkedList tmpSeedList = new LinkedList();
|
||||||
for(int i = 0; i < seeds.length; i++) {
|
for(int i = 0; i < seeds.length; i++) {
|
||||||
|
if (seeds[i] != null) {
|
||||||
if (amIAccessibleDB.containsKey(seeds[i].hash)) {
|
if (amIAccessibleDB.containsKey(seeds[i].hash)) {
|
||||||
tmpSeedList.add(seeds[i]);
|
tmpSeedList.add(seeds[i]);
|
||||||
} else {
|
} else {
|
||||||
seedList.add(seeds[i]);
|
seedList.add(seeds[i]);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
}
|
||||||
while (!tmpSeedList.isEmpty()) { seedList.add(tmpSeedList.remove(0)); }
|
while (!tmpSeedList.isEmpty()) { seedList.add(tmpSeedList.remove(0)); }
|
||||||
if (seedList.size() < attempts) { attempts = seedList.size(); }
|
if (seedList.size() < attempts) { attempts = seedList.size(); }
|
||||||
|
|
||||||
|
|
Loading…
Reference in New Issue
Block a user