Marown/yacy_search_server
1
0
Fork 0
mirror of https://github.com/yacy/yacy_search_server.git synced 2024-09-19 00:01:41 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity

more performance hacks: less overhead in word hash computation

Browse Source 
git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@5825 6c8d7289-2bf4-0310-a012-ef5d649a1542
This commit is contained in:
orbiter 2009-04-17 13:47:06 +00:00
parent 29e96c1a60
commit d7cbf4cdd4
3 changed files with 40 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
source/de/anomic/http/httpClient.java
View File

@ -554,9 +554,8 @@ public class httpClient {
Log.logWarning("HTTPC", "Proxy authentication contains invalid characters, trying anyway");
}
final String remoteProxyPwd = hostProxyConfig.getProxyPwd();
final String credentials = Base64Order.standardCoder.encodeString(remoteProxyUser.replace(":",
"") +
":" + remoteProxyPwd);
final String credentials = Base64Order.standardCoder.encodeString(
remoteProxyUser.replace(":", "") + ":" + remoteProxyPwd);
method.setRequestHeader(httpRequestHeader.PROXY_AUTHORIZATION, "Basic " + credentials);
}
}

37
source/de/anomic/kelondro/order/Base64Order.java
View File

@ -154,6 +154,15 @@ public class Base64Order extends AbstractOrder<byte[]> implements ByteOrder, Cod
return new String(s);
}
public final byte[] encodeLongSubstr(long c, int length) {
final byte[] s = new byte[length];
while (length > 0) {
s[--length] = (byte) alpha[(byte) (c & 0x3F)];
c >>= 6;
}
return s;
}
public final void encodeLong(long c, final byte[] b, final int offset, int length) {
assert offset + length <= b.length;
while (length > 0) {
@ -214,6 +223,34 @@ public class Base64Order extends AbstractOrder<byte[]> implements ByteOrder, Cod
return new String(out);
}
public final byte[] encodeSubstring(final byte[] in, int sublen) {
if (in.length == 0) return null;
byte[] out = new byte[sublen];
int writepos = 0;
int pos = 0;
long l;
while (in.length - pos >= 3 && writepos < sublen) {
l = ((((0XffL & in[pos]) << 8) + (0XffL & in[pos + 1])) << 8) + (0XffL & in[pos + 2]);
pos += 3;
System.arraycopy(encodeLongSubstr(l, 4), 0, out, writepos, 4);
writepos += 4;
}
// now there may be remaining bytes
if (in.length % 3 != 0 && writepos < sublen) {
if (in.length % 3 == 2) {
System.arraycopy(encodeLong((((0XffL & in[pos]) << 8) + (0XffL & in[pos + 1])) << 8, 4), 0, out, writepos, 3);
writepos += 3;
} else {
System.arraycopy(encodeLong((((0XffL & in[pos])) << 8) << 8, 4).substring(0, 2), 0, out, writepos, 2);
writepos += 2;
}
}
if (rfc1113compliant) while (writepos % 4 > 0 && writepos < sublen) out[writepos] = '=';
//assert encode(in).substring(0, sublen).equals(new String(out));
return out;
}
public final String decodeString(final String in, final String info) {
try {
//return new String(decode(in), "ISO-8859-1");

2
source/de/anomic/plasma/parser/Word.java
View File

@ -84,7 +84,7 @@ public class Word {
public static final byte[] word2hash(final String word) {
byte[] h = hashCache.get(word);
if (h != null) return h;
h = Base64Order.enhancedCoder.encode(Digest.encodeMD5Raw(word.toLowerCase(Locale.ENGLISH))).substring(0, yacySeedDB.commonHashLength).getBytes();
h = Base64Order.enhancedCoder.encodeSubstring(Digest.encodeMD5Raw(word.toLowerCase(Locale.ENGLISH)), yacySeedDB.commonHashLength);
hashCache.put(word, h); // prevent expensive MD5 computation and encoding
return h;
}