mirror of
https://github.com/yacy/yacy_search_server.git
synced 2024-09-21 00:00:13 +02:00
more performance hacks: less overhead in word hash computation
git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@5825 6c8d7289-2bf4-0310-a012-ef5d649a1542
This commit is contained in:
parent
29e96c1a60
commit
d7cbf4cdd4
|
@ -554,9 +554,8 @@ public class httpClient {
|
||||||
Log.logWarning("HTTPC", "Proxy authentication contains invalid characters, trying anyway");
|
Log.logWarning("HTTPC", "Proxy authentication contains invalid characters, trying anyway");
|
||||||
}
|
}
|
||||||
final String remoteProxyPwd = hostProxyConfig.getProxyPwd();
|
final String remoteProxyPwd = hostProxyConfig.getProxyPwd();
|
||||||
final String credentials = Base64Order.standardCoder.encodeString(remoteProxyUser.replace(":",
|
final String credentials = Base64Order.standardCoder.encodeString(
|
||||||
"") +
|
remoteProxyUser.replace(":", "") + ":" + remoteProxyPwd);
|
||||||
":" + remoteProxyPwd);
|
|
||||||
method.setRequestHeader(httpRequestHeader.PROXY_AUTHORIZATION, "Basic " + credentials);
|
method.setRequestHeader(httpRequestHeader.PROXY_AUTHORIZATION, "Basic " + credentials);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -154,6 +154,15 @@ public class Base64Order extends AbstractOrder<byte[]> implements ByteOrder, Cod
|
||||||
return new String(s);
|
return new String(s);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public final byte[] encodeLongSubstr(long c, int length) {
|
||||||
|
final byte[] s = new byte[length];
|
||||||
|
while (length > 0) {
|
||||||
|
s[--length] = (byte) alpha[(byte) (c & 0x3F)];
|
||||||
|
c >>= 6;
|
||||||
|
}
|
||||||
|
return s;
|
||||||
|
}
|
||||||
|
|
||||||
public final void encodeLong(long c, final byte[] b, final int offset, int length) {
|
public final void encodeLong(long c, final byte[] b, final int offset, int length) {
|
||||||
assert offset + length <= b.length;
|
assert offset + length <= b.length;
|
||||||
while (length > 0) {
|
while (length > 0) {
|
||||||
|
@ -214,6 +223,34 @@ public class Base64Order extends AbstractOrder<byte[]> implements ByteOrder, Cod
|
||||||
return new String(out);
|
return new String(out);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public final byte[] encodeSubstring(final byte[] in, int sublen) {
|
||||||
|
if (in.length == 0) return null;
|
||||||
|
byte[] out = new byte[sublen];
|
||||||
|
int writepos = 0;
|
||||||
|
int pos = 0;
|
||||||
|
long l;
|
||||||
|
while (in.length - pos >= 3 && writepos < sublen) {
|
||||||
|
l = ((((0XffL & in[pos]) << 8) + (0XffL & in[pos + 1])) << 8) + (0XffL & in[pos + 2]);
|
||||||
|
pos += 3;
|
||||||
|
System.arraycopy(encodeLongSubstr(l, 4), 0, out, writepos, 4);
|
||||||
|
writepos += 4;
|
||||||
|
}
|
||||||
|
// now there may be remaining bytes
|
||||||
|
if (in.length % 3 != 0 && writepos < sublen) {
|
||||||
|
if (in.length % 3 == 2) {
|
||||||
|
System.arraycopy(encodeLong((((0XffL & in[pos]) << 8) + (0XffL & in[pos + 1])) << 8, 4), 0, out, writepos, 3);
|
||||||
|
writepos += 3;
|
||||||
|
} else {
|
||||||
|
System.arraycopy(encodeLong((((0XffL & in[pos])) << 8) << 8, 4).substring(0, 2), 0, out, writepos, 2);
|
||||||
|
writepos += 2;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (rfc1113compliant) while (writepos % 4 > 0 && writepos < sublen) out[writepos] = '=';
|
||||||
|
//assert encode(in).substring(0, sublen).equals(new String(out));
|
||||||
|
return out;
|
||||||
|
}
|
||||||
|
|
||||||
public final String decodeString(final String in, final String info) {
|
public final String decodeString(final String in, final String info) {
|
||||||
try {
|
try {
|
||||||
//return new String(decode(in), "ISO-8859-1");
|
//return new String(decode(in), "ISO-8859-1");
|
||||||
|
|
|
@ -84,7 +84,7 @@ public class Word {
|
||||||
public static final byte[] word2hash(final String word) {
|
public static final byte[] word2hash(final String word) {
|
||||||
byte[] h = hashCache.get(word);
|
byte[] h = hashCache.get(word);
|
||||||
if (h != null) return h;
|
if (h != null) return h;
|
||||||
h = Base64Order.enhancedCoder.encode(Digest.encodeMD5Raw(word.toLowerCase(Locale.ENGLISH))).substring(0, yacySeedDB.commonHashLength).getBytes();
|
h = Base64Order.enhancedCoder.encodeSubstring(Digest.encodeMD5Raw(word.toLowerCase(Locale.ENGLISH)), yacySeedDB.commonHashLength);
|
||||||
hashCache.put(word, h); // prevent expensive MD5 computation and encoding
|
hashCache.put(word, h); // prevent expensive MD5 computation and encoding
|
||||||
return h;
|
return h;
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in New Issue
Block a user