mirror of
https://github.com/yacy/yacy_search_server.git
synced 2024-09-19 00:01:41 +02:00
many bug-fixes
git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@73 6c8d7289-2bf4-0310-a012-ef5d649a1542
This commit is contained in:
orbiter
parent
48650c082c
commit
c7c6aaf06e
|
|
@ -39,19 +39,21 @@ globalheader();
|
|||
|
||||
<br><p>v0.37
|
||||
<ul>
|
||||
<li>YaCy's source code is now hosted in a subversion version control system on berlios: <a href="http://developer.berlios.de/projects/yacy/">yacy@berlios.de</a></li>
|
||||
<li>YaCy's source code is now hosted in a Subversion/svn version control system on developer.berlios.de: <a href="http://developer.berlios.de/projects/yacy/">yacy@berlios.de</a></li>
|
||||
<li>overall speed enhancements:</li>
|
||||
<ul>
|
||||
<li>Check on new peer names: must not occur already and may only contain letters, numbers and '_' or '-'.</li>
|
||||
<li>New ThreadPool and performance enhancements from Martin Thelian</li>
|
||||
<li>new Thread-Pools and performance enhancements from Martin Thelian: much faster http-server and more responsive web interface</li>
|
||||
<li>fixed bug in database caching that prevented from caching at all; now database much faster. This also speeded up proxy mode (must read http-header from database)</li>
|
||||
<li>modified thread control for non-blocking dequeueing</li>
|
||||
<li>increased cache memory settings</li>
|
||||
</ul>
|
||||
<li></li>
|
||||
<li></li>
|
||||
<li></li>
|
||||
<li></li>
|
||||
<li>added a concept for external parsers; pdf an doc parser are integrated but not active yet.</li>
|
||||
<li>fixed several bugs that caused thread-locks and 100% CPU load</li>
|
||||
<li>fixed bug with cookie storage; changed handling of multiple cookies</li>
|
||||
<li>check on new peer names: must not occur already and may only contain letters, numbers and '_' or '-'.</li>
|
||||
<li>many minor bug fixes and spell corrections in interface</li>
|
||||
</ul>
|
||||
|
||||
|
||||
|
||||
<br><p>v0.36_build20050326
|
||||
<ul>
|
||||
<li>Enhanced thread control and added performance menu: this can be used to steer scheduling tasks and for profiling.</li>
|
||||
|
|
|
|||
|
|
@ -63,7 +63,9 @@ public class CookieMonitorIncoming_p {
|
|||
boolean dark = true;
|
||||
Iterator i = switchboard.incomingCookies.entrySet().iterator();
|
||||
Map.Entry entry;
|
||||
String host, client, cookie;
|
||||
String host, client;
|
||||
Object[] cookies;
|
||||
String ucl;
|
||||
Date date;
|
||||
Object[] oa;
|
||||
while ((entCount < maxCount) && (i.hasNext())) {
|
||||
|
|
@ -73,14 +75,17 @@ public class CookieMonitorIncoming_p {
|
|||
oa = (Object[]) entry.getValue();
|
||||
date = (Date) oa[0];
|
||||
client = (String) oa[1];
|
||||
cookie = (String) oa[2];
|
||||
cookies = (Object[]) oa[2];
|
||||
ucl = "<ul>";
|
||||
for (int j = 0; j < cookies.length; j++) ucl = ucl + "<li>" + ((String) cookies[j]) + "</li>";
|
||||
ucl = ucl + "</ul>";
|
||||
|
||||
// put values in template
|
||||
prop.put("list_" + entCount + "_dark", ((dark) ? 1 : 0) ); dark =! dark;
|
||||
prop.put("list_" + entCount + "_host", host);
|
||||
prop.put("list_" + entCount + "_date", httpc.dateString(date));
|
||||
prop.put("list_" + entCount + "_client", client);
|
||||
prop.put("list_" + entCount + "_cookie", cookie);
|
||||
prop.put("list_" + entCount + "_cookie", ucl);
|
||||
|
||||
// next
|
||||
entCount++;
|
||||
|
|
|
|||
|
|
@ -63,7 +63,9 @@ public class CookieMonitorOutgoing_p {
|
|||
boolean dark = true;
|
||||
Iterator i = switchboard.outgoingCookies.entrySet().iterator();
|
||||
Map.Entry entry;
|
||||
String host, client, cookie;
|
||||
String host, client;
|
||||
Object[] cookies;
|
||||
String ucl;
|
||||
Date date;
|
||||
Object[] oa;
|
||||
while ((entCount < maxCount) && (i.hasNext())) {
|
||||
|
|
@ -73,14 +75,17 @@ public class CookieMonitorOutgoing_p {
|
|||
oa = (Object[]) entry.getValue();
|
||||
date = (Date) oa[0];
|
||||
client = (String) oa[1];
|
||||
cookie = (String) oa[2];
|
||||
cookies = (Object[]) oa[2];
|
||||
ucl = "<ul>";
|
||||
for (int j = 0; j < cookies.length; j++) ucl = ucl + "<li>" + ((String) cookies[j]) + "</li>";
|
||||
ucl = ucl + "</ul>";
|
||||
|
||||
// put values in template
|
||||
prop.put("list_" + entCount + "_dark", ((dark) ? 1 : 0) ); dark =! dark;
|
||||
prop.put("list_" + entCount + "_host", host);
|
||||
prop.put("list_" + entCount + "_date", httpc.dateString(date));
|
||||
prop.put("list_" + entCount + "_client", client);
|
||||
prop.put("list_" + entCount + "_cookie", cookie);
|
||||
prop.put("list_" + entCount + "_cookie", ucl);
|
||||
|
||||
// next
|
||||
entCount++;
|
||||
|
|
|
|||
1
htroot/env/templates/metas.template
vendored
1
htroot/env/templates/metas.template
vendored
|
|
@ -1,4 +1,5 @@
|
|||
<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">
|
||||
<link rel="shortcut icon" href="favicon.ico">
|
||||
<meta name="Content-Language" content="English, Englisch">
|
||||
<meta name="keywords" content="Anomic HTTP Proxy search engine spider indexer java network open free download Mac Windwos Software development">
|
||||
<meta name="description" content="Anomic Software HTTP Proxy Freeware Home Page">
|
||||
|
|
|
|||
BIN
htroot/favicon.ico
Normal file
BIN
htroot/favicon.ico
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 1.3 KiB |
|
|
@ -45,7 +45,7 @@
|
|||
# Contributions and changes to the program code must be marked as such.
|
||||
|
||||
# define variables
|
||||
version='0.3681'
|
||||
version='0.369'
|
||||
datestr=`date +%Y%m%d`
|
||||
#release='yacy_v'$version'_'$datestr
|
||||
release='yacy_dev_v'$version'_'$datestr
|
||||
|
|
|
|||
|
|
@ -127,12 +127,38 @@ public final class httpHeader extends TreeMap implements Map {
|
|||
}
|
||||
}
|
||||
|
||||
// a convenience method to access the map with fail-over deafults
|
||||
// to make the occurrence of multiple keys possible, we add them using a counter
|
||||
public Object add(Object key, Object value) {
|
||||
int c = keyCount((String) key);
|
||||
if (c == 0) return put(key, value); else return put("*" + key + "-" + c, value);
|
||||
}
|
||||
|
||||
public int keyCount(String key) {
|
||||
if (!(containsKey(key))) return 0;
|
||||
int c = 1;
|
||||
while (containsKey("*" + key + "-" + c)) c++;
|
||||
return c;
|
||||
}
|
||||
|
||||
// a convenience method to access the map with fail-over defaults
|
||||
public Object get(Object key, Object dflt) {
|
||||
Object result = get(key);
|
||||
if (result == null) return dflt; else return result;
|
||||
}
|
||||
|
||||
// return multiple results
|
||||
public Object getSingle(Object key, int count) {
|
||||
if (count == 0) return get(key, null);
|
||||
return get("*" + key + "-" + count, null);
|
||||
}
|
||||
|
||||
public Object[] getMultiple(String key) {
|
||||
int count = keyCount(key);
|
||||
Object[] result = new Object[count];
|
||||
for (int i = 0; i < count; i++) result[i] = getSingle(key, i);
|
||||
return result;
|
||||
}
|
||||
|
||||
// convenience methods for storing and loading to a file system
|
||||
public void store(File f) throws IOException {
|
||||
FileOutputStream fos = new FileOutputStream(f);
|
||||
|
|
|
|||
|
|
@ -391,7 +391,6 @@ public final class httpc {
|
|||
|
||||
// at this point we should have a valid response. read in the header properties
|
||||
String key = "";
|
||||
String value = "";
|
||||
while ((b = serverCore.receive(clientInput, readLineBuffer, timeout, terminalMaxLength, false)) != null) {
|
||||
if (b.length == 0) break;
|
||||
buffer = new String(b);
|
||||
|
|
@ -406,16 +405,7 @@ public final class httpc {
|
|||
// create new entry
|
||||
p = buffer.indexOf(":");
|
||||
if (p > 0) {
|
||||
key = buffer.substring(0, p).trim();
|
||||
value = (String) responseHeader.get(key);
|
||||
// check if the header occurred already
|
||||
if (value == null) {
|
||||
// create new entry
|
||||
responseHeader.put(key, buffer.substring(p + 1).trim());
|
||||
} else {
|
||||
// attach to old entry
|
||||
responseHeader.put(key, value + "#" + buffer.substring(p + 1).trim());
|
||||
}
|
||||
responseHeader.add(buffer.substring(0, p).trim(), buffer.substring(p + 1).trim());
|
||||
} else {
|
||||
serverLog.logError("HTTPC", "RESPONSE PARSE ERROR: HOST='" + host + "', PATH='" + requestPath + "', STATUS='" + status + "'");
|
||||
serverLog.logError("HTTPC", "..............BUFFER: " + buffer);
|
||||
|
|
@ -640,19 +630,18 @@ public final class httpc {
|
|||
Iterator i = header.keySet().iterator();
|
||||
String key;
|
||||
String value;
|
||||
int pos;
|
||||
int count;
|
||||
char tag;
|
||||
while (i.hasNext()) {
|
||||
key = (String) i.next();
|
||||
value = (String) header.get(key);
|
||||
while ((pos = value.lastIndexOf("#")) >= 0) {
|
||||
// special handling is needed if a key appeared several times, which is valid.
|
||||
// all lines with same key are combined in one value, separated by a "#"
|
||||
serverCore.send(clientOutput, key + ": " + value.substring(pos + 1).trim());
|
||||
//System.out.println("**+" + key + ": " + value.substring(pos + 1).trim()); // debug
|
||||
value = value.substring(0, pos).trim();
|
||||
tag = key.charAt(0);
|
||||
if ((tag != '*') && (tag != '#')) {
|
||||
count = header.keyCount(key);
|
||||
for (int j = 0; j < count; j++) {
|
||||
serverCore.send(clientOutput, key + ": " + ((String) header.getSingle(key, j)).trim());
|
||||
}
|
||||
//System.out.println("#" + key + ": " + value);
|
||||
}
|
||||
serverCore.send(clientOutput, key + ": " + value);
|
||||
//System.out.println("***" + key + ": " + value); // debug
|
||||
}
|
||||
|
||||
// send terminating line
|
||||
|
|
|
|||
|
|
@ -181,8 +181,6 @@ public final class httpd implements serverHandler {
|
|||
httpHeader header = new httpHeader(reverseMappingCache);
|
||||
int p;
|
||||
String line;
|
||||
String key;
|
||||
String value;
|
||||
while ((line = readLine()) != null) {
|
||||
if (line.length() == 0) break; // this seperates the header of the HTTP request from the body
|
||||
//System.out.println("***" + line); // debug
|
||||
|
|
@ -190,16 +188,7 @@ public final class httpd implements serverHandler {
|
|||
p = line.indexOf(":");
|
||||
if (p >= 0) {
|
||||
// store a property
|
||||
key = line.substring(0, p).trim();
|
||||
value = (String) header.get(key);
|
||||
// check if the header occurred already
|
||||
if (value == null) {
|
||||
// create new entry
|
||||
header.put(key, line.substring(p + 1).trim());
|
||||
} else {
|
||||
// value can occur double times, attach with '#' - separator
|
||||
header.put(key, value + "#" + line.substring(p + 1).trim());
|
||||
}
|
||||
header.add(line.substring(0, p).trim(), line.substring(p + 1).trim());
|
||||
}
|
||||
}
|
||||
return header;
|
||||
|
|
|
|||
|
|
@ -223,8 +223,6 @@ public final class httpdProxyHandler extends httpdAbstractHandler implements htt
|
|||
}
|
||||
|
||||
public void handleOutgoingCookies(httpHeader requestHeader, String targethost, String clienthost) {
|
||||
// request header may have double-entries: they are accumulated in one entry
|
||||
// by the httpd and separated by a "#" in the value field
|
||||
/*
|
||||
The syntax for the header is:
|
||||
|
||||
|
|
@ -238,14 +236,12 @@ public final class httpdProxyHandler extends httpdAbstractHandler implements htt
|
|||
domain = "$Domain" "=" value
|
||||
*/
|
||||
if (requestHeader.containsKey("Cookie")) {
|
||||
Object[] entry = new Object[]{new Date(), clienthost, requestHeader.get("Cookie")};
|
||||
Object[] entry = new Object[]{new Date(), clienthost, requestHeader.getMultiple("Cookie")};
|
||||
switchboard.outgoingCookies.put(targethost, entry);
|
||||
}
|
||||
}
|
||||
|
||||
public void handleIncomingCookies(httpHeader respondHeader, String serverhost, String targetclient) {
|
||||
// respond header may have double-entries: they are accumulated in one entry
|
||||
// by the httpc and separated by a "#" in the value field
|
||||
/*
|
||||
The syntax for the Set-Cookie response header is
|
||||
|
||||
|
|
@ -262,7 +258,7 @@ public final class httpdProxyHandler extends httpdAbstractHandler implements htt
|
|||
| "Version" "=" 1*DIGIT
|
||||
*/
|
||||
if (respondHeader.containsKey("Set-Cookie")) {
|
||||
Object[] entry = new Object[]{new Date(), targetclient, respondHeader.get("Set-Cookie")};
|
||||
Object[] entry = new Object[]{new Date(), targetclient, respondHeader.getMultiple("Set-Cookie")};
|
||||
switchboard.incomingCookies.put(serverhost, entry);
|
||||
}
|
||||
}
|
||||
|
|
@ -934,9 +930,7 @@ public final class httpdProxyHandler extends httpdAbstractHandler implements htt
|
|||
StringBuffer headerStringBuffer = new StringBuffer(200);
|
||||
|
||||
// write status line
|
||||
headerStringBuffer.append("HTTP/1.1 ")
|
||||
.append(status)
|
||||
.append("\r\n");
|
||||
headerStringBuffer.append("HTTP/1.1 ").append(status).append("\r\n");
|
||||
|
||||
//System.out.println("HEADER: PROXY TO CLIENT = " + header.toString()); // DEBUG
|
||||
|
||||
|
|
@ -944,32 +938,20 @@ public final class httpdProxyHandler extends httpdAbstractHandler implements htt
|
|||
Iterator i = header.keySet().iterator();
|
||||
String key;
|
||||
String value;
|
||||
int pos;
|
||||
char tag;
|
||||
int count;
|
||||
//System.out.println("vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv");
|
||||
while (i.hasNext()) {
|
||||
key = (String) i.next();
|
||||
if (!(key.startsWith("#"))) { // '#' in key is reserved for proxy attributes as artificial header values
|
||||
value = (String) header.get(key);
|
||||
if (!(key.equals("Location"))) while ((pos = value.lastIndexOf("#")) >= 0) {
|
||||
// special handling is needed if a key appeared several times, which is valid.
|
||||
// all lines with same key are combined in one value, separated by a "#"
|
||||
headerStringBuffer
|
||||
.append(key)
|
||||
.append(": ")
|
||||
.append(value.substring(pos + 1).trim())
|
||||
.append("\r\n");
|
||||
//System.out.println("#" + key + ": " + value.substring(pos + 1).trim());
|
||||
value = value.substring(0, pos).trim();
|
||||
tag = key.charAt(0);
|
||||
if ((tag != '*') && (tag != '#')) { // '#' in key is reserved for proxy attributes as artificial header values
|
||||
count = header.keyCount(key);
|
||||
for (int j = 0; j < count; j++) {
|
||||
headerStringBuffer.append(key).append(": ").append((String) header.getSingle(key, j)).append("\r\n");
|
||||
}
|
||||
headerStringBuffer
|
||||
.append(key)
|
||||
.append(": ")
|
||||
.append(value)
|
||||
.append("\r\n");
|
||||
//System.out.println("#" + key + ": " + value);
|
||||
}
|
||||
}
|
||||
|
||||
headerStringBuffer.append("\r\n");
|
||||
|
||||
// end header
|
||||
|
|
|
|||
|
|
@ -166,8 +166,6 @@ public class kelondroMScoreCluster {
|
|||
c = cs.longValue();
|
||||
gcount -= (c & 0xFFFFFFFF00000000L) >> 32;
|
||||
en = (int) (c & 0xFFFFFFFFL);
|
||||
// decrease overall counter
|
||||
gcount -= c;
|
||||
}
|
||||
|
||||
// set new value
|
||||
|
|
@ -190,8 +188,9 @@ public class kelondroMScoreCluster {
|
|||
keyrefDB.remove(cs);
|
||||
refkeyDB.remove(obj);
|
||||
// decrease overall counter
|
||||
gcount -= cs.longValue();
|
||||
return (int) ((cs.longValue() & 0xFFFFFFFF00000000L) >> 32);
|
||||
long oldScore = (cs.longValue() & 0xFFFFFFFF00000000L) >> 32;
|
||||
gcount -= oldScore;
|
||||
return (int) oldScore;
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -183,7 +183,7 @@ public final class plasmaCrawlWorker extends Thread {
|
|||
this.setName(this.threadBaseName + "_" + this.url);
|
||||
load(this.url, this.referer, this.initiator, this.depth, this.profile);
|
||||
} catch (IOException e) {
|
||||
throw e;
|
||||
//throw e;
|
||||
}
|
||||
finally {
|
||||
this.done = true;
|
||||
|
|
|
|||
|
|
@ -74,7 +74,8 @@ public class yacyClient {
|
|||
HashMap result = null;
|
||||
try {
|
||||
/*
|
||||
URL url = new URL("http://" + address + "/yacy/hello.html?iam=" + yacyCore.seedCache.mySeed.hash +
|
||||
URL url = new URL("http://" + address + "/yacy/hello.html?iam=" +
|
||||
yacyCore.seedCache.mySeed.hash +
|
||||
"&pattern=&count=20" +
|
||||
"&key=" + key + "&seed=" + yacyCore.seedCache.mySeed.genSeedStr(key));
|
||||
yacyCore.log.logDebug("HELLO to URL " + url.toString());
|
||||
|
|
@ -196,7 +197,7 @@ public class yacyClient {
|
|||
String resp = (String) result.get("response");
|
||||
if (resp == null) return -1; else return Integer.parseInt(resp);
|
||||
} catch (Exception e) {
|
||||
//yacyCore.log.logError("yacyClient.queryUrlCount error asking peer '" + target.getName() + "':" + e.toString());
|
||||
yacyCore.log.logError("yacyClient.queryUrlCount error asking peer '" + target.getName() + "':" + e.toString());
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue
Block a user