mirror of
https://github.com/yacy/yacy_search_server.git
synced 2024-09-19 00:01:41 +02:00
less synchronization, better thread dump tool
git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@7556 6c8d7289-2bf4-0310-a012-ef5d649a1542
This commit is contained in:
orbiter
parent
8d14916c74
commit
7138f4036b
|
|
@ -32,7 +32,7 @@ import java.util.Map;
|
|||
import java.util.ArrayList;
|
||||
|
||||
import net.yacy.cora.protocol.RequestHeader;
|
||||
import net.yacy.kelondro.logging.ThreadDumpGenerator;
|
||||
import net.yacy.kelondro.logging.ThreadDump;
|
||||
import net.yacy.kelondro.util.MemoryControl;
|
||||
|
||||
import de.anomic.search.Switchboard;
|
||||
|
|
@ -58,14 +58,14 @@ public class Threaddump_p {
|
|||
final String versionstring = yacyBuildProperties.getVersion() + "/" + yacyBuildProperties.getSVNRevision();
|
||||
Runtime runtime = Runtime.getRuntime();
|
||||
|
||||
ThreadDumpGenerator.bufferappend(buffer, plain, "************* Start Thread Dump " + dt + " *******************");
|
||||
ThreadDumpGenerator.bufferappend(buffer, plain, "");
|
||||
ThreadDumpGenerator.bufferappend(buffer, plain, "YaCy Version: " + versionstring);
|
||||
ThreadDumpGenerator.bufferappend(buffer, plain, "Assigned Memory = " + (runtime.maxMemory()));
|
||||
ThreadDumpGenerator.bufferappend(buffer, plain, "Used Memory = " + (runtime.totalMemory() - runtime.freeMemory()));
|
||||
ThreadDumpGenerator.bufferappend(buffer, plain, "Available Memory = " + (runtime.maxMemory() - runtime.totalMemory() + runtime.freeMemory()));
|
||||
ThreadDumpGenerator.bufferappend(buffer, plain, "");
|
||||
ThreadDumpGenerator.bufferappend(buffer, plain, "");
|
||||
ThreadDump.bufferappend(buffer, plain, "************* Start Thread Dump " + dt + " *******************");
|
||||
ThreadDump.bufferappend(buffer, plain, "");
|
||||
ThreadDump.bufferappend(buffer, plain, "YaCy Version: " + versionstring);
|
||||
ThreadDump.bufferappend(buffer, plain, "Assigned Memory = " + (runtime.maxMemory()));
|
||||
ThreadDump.bufferappend(buffer, plain, "Used Memory = " + (runtime.totalMemory() - runtime.freeMemory()));
|
||||
ThreadDump.bufferappend(buffer, plain, "Available Memory = " + (runtime.maxMemory() - runtime.totalMemory() + runtime.freeMemory()));
|
||||
ThreadDump.bufferappend(buffer, plain, "");
|
||||
ThreadDump.bufferappend(buffer, plain, "");
|
||||
|
||||
int multipleCount = 100;
|
||||
File appPath = sb.getAppPath();
|
||||
|
|
@ -76,7 +76,7 @@ public class Threaddump_p {
|
|||
traces.add(Thread.getAllStackTraces());
|
||||
if (MemoryControl.available() < 20 * 1024 * 1024) break;
|
||||
}
|
||||
ThreadDumpGenerator.appendStackTraceStats(appPath, buffer, traces, plain, null);
|
||||
ThreadDump.appendStackTraceStats(appPath, buffer, traces, plain, null);
|
||||
/*
|
||||
ThreadDumpGenerator.appendStackTraceStats(appPath, buffer, traces, plain, Thread.State.BLOCKED);
|
||||
ThreadDumpGenerator.appendStackTraceStats(appPath, buffer, traces, plain, Thread.State.RUNNABLE);
|
||||
|
|
@ -88,15 +88,15 @@ public class Threaddump_p {
|
|||
} else {
|
||||
// generate a single thread dump
|
||||
final Map<Thread,StackTraceElement[]> stackTraces = Thread.getAllStackTraces();
|
||||
ThreadDumpGenerator.appendStackTraces(appPath, buffer, stackTraces, plain, Thread.State.BLOCKED);
|
||||
ThreadDumpGenerator.appendStackTraces(appPath, buffer, stackTraces, plain, Thread.State.RUNNABLE);
|
||||
ThreadDumpGenerator.appendStackTraces(appPath, buffer, stackTraces, plain, Thread.State.TIMED_WAITING);
|
||||
ThreadDumpGenerator.appendStackTraces(appPath, buffer, stackTraces, plain, Thread.State.WAITING);
|
||||
ThreadDumpGenerator.appendStackTraces(appPath, buffer, stackTraces, plain, Thread.State.NEW);
|
||||
ThreadDumpGenerator.appendStackTraces(appPath, buffer, stackTraces, plain, Thread.State.TERMINATED);
|
||||
ThreadDump.appendStackTraces(appPath, buffer, stackTraces, plain, Thread.State.BLOCKED);
|
||||
ThreadDump.appendStackTraces(appPath, buffer, stackTraces, plain, Thread.State.RUNNABLE);
|
||||
ThreadDump.appendStackTraces(appPath, buffer, stackTraces, plain, Thread.State.TIMED_WAITING);
|
||||
ThreadDump.appendStackTraces(appPath, buffer, stackTraces, plain, Thread.State.WAITING);
|
||||
ThreadDump.appendStackTraces(appPath, buffer, stackTraces, plain, Thread.State.NEW);
|
||||
ThreadDump.appendStackTraces(appPath, buffer, stackTraces, plain, Thread.State.TERMINATED);
|
||||
}
|
||||
|
||||
ThreadDumpGenerator.bufferappend(buffer, plain, "************* End Thread Dump " + dt + " *******************");
|
||||
ThreadDump.bufferappend(buffer, plain, "************* End Thread Dump " + dt + " *******************");
|
||||
|
||||
prop.put("plain_count", multipleCount);
|
||||
prop.put("plain_content", buffer.toString());
|
||||
|
|
|
|||
|
|
@ -33,6 +33,7 @@ import java.util.concurrent.BlockingQueue;
|
|||
import java.util.concurrent.LinkedBlockingQueue;
|
||||
import java.util.concurrent.Semaphore;
|
||||
|
||||
import net.yacy.cora.document.UTF8;
|
||||
import net.yacy.cora.storage.DynamicScore;
|
||||
import net.yacy.cora.storage.ScoreCluster;
|
||||
import net.yacy.document.Condenser;
|
||||
|
|
@ -152,7 +153,7 @@ public class ReferenceOrder {
|
|||
if (max == null) max = iEntry.clone(); else max.max(iEntry);
|
||||
out.put(iEntry); // must be after the min/max check to prevent that min/max is null in cardinal()
|
||||
// update domcount
|
||||
dom = new String(iEntry.metadataHash(), 6, 6);
|
||||
dom = UTF8.String(iEntry.metadataHash(), 6, 6);
|
||||
count = doms0.get(dom);
|
||||
if (count == null) {
|
||||
doms0.put(dom, int1);
|
||||
|
|
|
|||
|
|
@ -234,7 +234,7 @@ public class yacySeed implements Cloneable, Comparable<yacySeed>, Comparator<yac
|
|||
|
||||
/**
|
||||
* check the peer name: protect against usage as XSS hack
|
||||
* @param name
|
||||
* @param id
|
||||
* @return a checked name without "<" and ">"
|
||||
*/
|
||||
final static Pattern ltp = Pattern.compile("<");
|
||||
|
|
|
|||
|
|
@ -48,4 +48,18 @@ public class UTF8 {
|
|||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* using the string method with the default charset given as argument should prevent using the charset cache
|
||||
* in FastCharsetProvider.java:118 which locks all concurrent threads using a new String() method
|
||||
* @param bytes
|
||||
* @return
|
||||
*/
|
||||
public static String String(byte[] bytes) {
|
||||
return new String(bytes, charset);
|
||||
}
|
||||
|
||||
public static String String(byte[] bytes, int offset, int length) {
|
||||
return new String(bytes, charset);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -106,8 +106,6 @@ public class URIMetadataRow implements URIMetadata {
|
|||
private final long ranking; // during generation of a search result this value is set
|
||||
private Components comp;
|
||||
|
||||
private static final GenericFormatter mySHORT_DAY_FORMATTER = new GenericFormatter(GenericFormatter.FORMAT_SHORT_DAY, GenericFormatter.time_minute);
|
||||
|
||||
public URIMetadataRow() {
|
||||
// create a dummy entry, good to produce poison objects
|
||||
this.entry = rowdef.newEntry();
|
||||
|
|
@ -229,18 +227,22 @@ public class URIMetadataRow implements URIMetadata {
|
|||
this.entry = rowdef.newEntry();
|
||||
this.entry.setCol(col_hash, url.hash()); // FIXME potential null pointer access
|
||||
this.entry.setCol(col_comp, encodeComp(url, descr, dc_creator, tags, dc_publisher));
|
||||
|
||||
// create new formatters to make concurrency possible
|
||||
GenericFormatter formatter = new GenericFormatter(GenericFormatter.FORMAT_SHORT_DAY, GenericFormatter.time_minute);
|
||||
|
||||
try {
|
||||
encodeDate(col_mod, mySHORT_DAY_FORMATTER.parse(prop.getProperty("mod", "20000101")));
|
||||
encodeDate(col_mod, formatter.parse(prop.getProperty("mod", "20000101")));
|
||||
} catch (final ParseException e) {
|
||||
encodeDate(col_mod, new Date());
|
||||
}
|
||||
try {
|
||||
encodeDate(col_load, mySHORT_DAY_FORMATTER.parse(prop.getProperty("load", "20000101")));
|
||||
encodeDate(col_load, formatter.parse(prop.getProperty("load", "20000101")));
|
||||
} catch (final ParseException e) {
|
||||
encodeDate(col_load, new Date());
|
||||
}
|
||||
try {
|
||||
encodeDate(col_fresh, mySHORT_DAY_FORMATTER.parse(prop.getProperty("fresh", "20000101")));
|
||||
encodeDate(col_fresh, formatter.parse(prop.getProperty("fresh", "20000101")));
|
||||
} catch (final ParseException e) {
|
||||
encodeDate(col_fresh, new Date());
|
||||
}
|
||||
|
|
@ -295,6 +297,10 @@ public class URIMetadataRow implements URIMetadata {
|
|||
final StringBuilder s = new StringBuilder(300);
|
||||
if (metadata == null) return null;
|
||||
//System.out.println("author=" + comp.author());
|
||||
|
||||
// create new formatters to make concurrency possible
|
||||
GenericFormatter formatter = new GenericFormatter(GenericFormatter.FORMAT_SHORT_DAY, GenericFormatter.time_minute);
|
||||
|
||||
try {
|
||||
s.append("hash=").append(new String(hash()));
|
||||
assert (s.toString().indexOf(0) < 0);
|
||||
|
|
@ -308,11 +314,11 @@ public class URIMetadataRow implements URIMetadata {
|
|||
assert (s.toString().indexOf(0) < 0);
|
||||
s.append(",publisher=").append(crypt.simpleEncode(metadata.dc_publisher()));
|
||||
assert (s.toString().indexOf(0) < 0);
|
||||
s.append(",mod=").append(mySHORT_DAY_FORMATTER.format(moddate()));
|
||||
s.append(",mod=").append(formatter.format(moddate()));
|
||||
assert (s.toString().indexOf(0) < 0);
|
||||
s.append(",load=").append(mySHORT_DAY_FORMATTER.format(loaddate()));
|
||||
s.append(",load=").append(formatter.format(loaddate()));
|
||||
assert (s.toString().indexOf(0) < 0);
|
||||
s.append(",fresh=").append(mySHORT_DAY_FORMATTER.format(freshdate()));
|
||||
s.append(",fresh=").append(formatter.format(freshdate()));
|
||||
assert (s.toString().indexOf(0) < 0);
|
||||
s.append(",referrer=").append(referrerHash() == null ? "" : new String(referrerHash()));
|
||||
assert (s.toString().indexOf(0) < 0);
|
||||
|
|
|
|||
|
|
@ -345,11 +345,9 @@ public class RowCollection implements Iterable<Row.Entry>, Cloneable {
|
|||
if ((chunkcache == null) || (rowdef == null)) return null; // case may appear during shutdown
|
||||
Row.Entry entry;
|
||||
final int addr = index * rowdef.objectsize;
|
||||
synchronized (this) {
|
||||
if (index >= chunkcount) return null;
|
||||
if (addr + rowdef.objectsize > chunkcache.length) return null; // the whole chunk does not fit into the chunkcache
|
||||
entry = rowdef.newEntry(chunkcache, addr, clone);
|
||||
}
|
||||
if (index >= chunkcount) return null;
|
||||
if (addr + rowdef.objectsize > chunkcache.length) return null; // the whole chunk does not fit into the chunkcache
|
||||
entry = rowdef.newEntry(chunkcache, addr, clone);
|
||||
return entry;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -25,22 +25,105 @@
|
|||
|
||||
package net.yacy.kelondro.logging;
|
||||
|
||||
import java.io.BufferedReader;
|
||||
import java.io.File;
|
||||
import java.io.FileInputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.io.InputStreamReader;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.SortedSet;
|
||||
import java.util.TreeSet;
|
||||
import java.util.Map.Entry;
|
||||
|
||||
import net.yacy.document.parser.html.CharacterCoding;
|
||||
import net.yacy.kelondro.util.FileUtils;
|
||||
import de.anomic.tools.nxTools;
|
||||
|
||||
public class ThreadDumpGenerator {
|
||||
public class ThreadDump extends HashMap<ThreadDump.Thread, List<String>> implements Map<ThreadDump.Thread, List<String>> {
|
||||
|
||||
public static void appendStackTraces(final File rootPath, final StringBuilder buffer, final Map<Thread,StackTraceElement[]> stackTraces, final boolean plain, final Thread.State stateIn) {
|
||||
private static final long serialVersionUID = -5587850671040354397L;
|
||||
|
||||
public class Thread {
|
||||
public String name;
|
||||
public Thread(String name) {
|
||||
this.name = name;
|
||||
}
|
||||
public boolean equals(Object a) {
|
||||
if (a == null) return false;
|
||||
return this.name.equals(((Thread) a).name);
|
||||
}
|
||||
public boolean equals(Thread a) {
|
||||
if (a == null) return false;
|
||||
return this.name.equals(a.name);
|
||||
}
|
||||
public int hashCode() {
|
||||
return this.name.hashCode();
|
||||
}
|
||||
public String toString() {
|
||||
return this.name;
|
||||
}
|
||||
}
|
||||
|
||||
public class Lock {
|
||||
public String id;
|
||||
public Lock(String name) {
|
||||
this.id = name;
|
||||
}
|
||||
public boolean equals(Object a) {
|
||||
if (a == null) return false;
|
||||
return this.id.equals(((Lock) a).id);
|
||||
}
|
||||
public boolean equals(Lock a) {
|
||||
if (a == null) return false;
|
||||
return this.id.equals(a.id);
|
||||
}
|
||||
public int hashCode() {
|
||||
return this.id.hashCode();
|
||||
}
|
||||
public String toString() {
|
||||
return this.id;
|
||||
}
|
||||
}
|
||||
|
||||
public static Map<java.lang.Thread, StackTraceElement[]> getAllStackTraces() {
|
||||
return java.lang.Thread.getAllStackTraces();
|
||||
}
|
||||
|
||||
public ThreadDump(File f) throws IOException {
|
||||
this(new FileInputStream(f));
|
||||
}
|
||||
|
||||
public ThreadDump(InputStream is) throws IOException {
|
||||
super();
|
||||
BufferedReader br = new BufferedReader(new InputStreamReader(is));
|
||||
String line;
|
||||
String thread = null;
|
||||
int p;
|
||||
List<String> list = new ArrayList<String>();
|
||||
while ((line = br.readLine()) != null) {
|
||||
if (line.isEmpty()) {
|
||||
if (thread != null) {
|
||||
this.put(new ThreadDump.Thread(thread), list);
|
||||
}
|
||||
list = new ArrayList<String>();
|
||||
thread = null;
|
||||
continue;
|
||||
}
|
||||
if (line.charAt(0) == '"' && (p = line.indexOf("\" prio=")) > 0) {
|
||||
// start a new thread
|
||||
thread = line.substring(1, p);
|
||||
continue;
|
||||
}
|
||||
if (thread != null) {
|
||||
list.add(line);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public static void appendStackTraces(final File rootPath, final StringBuilder buffer, final Map<java.lang.Thread, StackTraceElement[]> stackTraces, final boolean plain, final java.lang.Thread.State stateIn) {
|
||||
bufferappend(buffer, plain, "THREADS WITH STATES: " + stateIn.toString());
|
||||
bufferappend(buffer, plain, "");
|
||||
// collect single dumps
|
||||
|
|
@ -56,7 +139,7 @@ public class ThreadDumpGenerator {
|
|||
bufferappend(buffer, plain, "");
|
||||
}
|
||||
|
||||
public static void appendStackTraceStats(final File rootPath, final StringBuilder buffer, final ArrayList<Map<Thread,StackTraceElement[]>> traces, final boolean plain, final Thread.State stateIn) {
|
||||
public static void appendStackTraceStats(final File rootPath, final StringBuilder buffer, final ArrayList<Map<java.lang.Thread, StackTraceElement[]>> traces, final boolean plain, final java.lang.Thread.State stateIn) {
|
||||
if (stateIn != null) {
|
||||
bufferappend(buffer, plain, "THREADS WITH STATES: " + stateIn.toString());
|
||||
bufferappend(buffer, plain, "");
|
||||
|
|
@ -85,13 +168,11 @@ public class ThreadDumpGenerator {
|
|||
return max;
|
||||
}
|
||||
|
||||
private static HashMap<String, Integer> dumpStatistic(final File rootPath, final ArrayList<Map<Thread,StackTraceElement[]>> stackTraces, final boolean plain, final Thread.State stateIn) {
|
||||
Map<Thread,StackTraceElement[]> trace;
|
||||
private static HashMap<String, Integer> dumpStatistic(final File rootPath, final ArrayList<Map<java.lang.Thread, StackTraceElement[]>> stackTraces, final boolean plain, final java.lang.Thread.State stateIn) {
|
||||
HashMap<String, Integer> result = new HashMap<String, Integer>();
|
||||
HashMap<String, SortedSet<String>> x;
|
||||
int count;
|
||||
for (int i = 0; i < stackTraces.size(); i++) {
|
||||
trace = stackTraces.get(i);
|
||||
for (Map<java.lang.Thread,StackTraceElement[]> trace: stackTraces) {
|
||||
x = dumpCollection(rootPath, trace, plain, stateIn);
|
||||
for (final Entry<String, SortedSet<String>> e: x.entrySet()) {
|
||||
Integer c = result.get(e.getKey());
|
||||
|
|
@ -106,13 +187,13 @@ public class ThreadDumpGenerator {
|
|||
return result;
|
||||
}
|
||||
|
||||
private static HashMap<String, SortedSet<String>> dumpCollection(final File appPath, final Map<Thread,StackTraceElement[]> stackTraces, final boolean plain, final Thread.State stateIn) {
|
||||
private static HashMap<String, SortedSet<String>> dumpCollection(final File appPath, final Map<java.lang.Thread,StackTraceElement[]> stackTraces, final boolean plain, final java.lang.Thread.State stateIn) {
|
||||
final File classPath = new File(appPath, "source");
|
||||
|
||||
Thread thread;
|
||||
java.lang.Thread thread;
|
||||
// collect single dumps
|
||||
HashMap<String, SortedSet<String>> dumps = new HashMap<String, SortedSet<String>>();
|
||||
for (final Entry<Thread, StackTraceElement[]> entry: stackTraces.entrySet()) {
|
||||
for (final Entry<java.lang.Thread, StackTraceElement[]> entry: stackTraces.entrySet()) {
|
||||
thread = entry.getKey();
|
||||
final StackTraceElement[] stackTraceElements = entry.getValue();
|
||||
StackTraceElement ste;
|
||||
|
|
@ -189,4 +270,84 @@ public class ThreadDumpGenerator {
|
|||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* find all locks in this dump
|
||||
* @return a map from lock ids to the name of the thread where the lock occurs
|
||||
*/
|
||||
public Map<Lock, Thread> locks() {
|
||||
int p;
|
||||
Map<Lock, Thread> locks = new HashMap<Lock, Thread>();
|
||||
for (Map.Entry<Thread, List<String>> entry: this.entrySet()) {
|
||||
for (String s: entry.getValue()) {
|
||||
if ((p = s.indexOf("locked <")) > 0) {
|
||||
locks.put(new Lock(s.substring(p + 8, s.indexOf('>'))), entry.getKey());
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
return locks;
|
||||
}
|
||||
|
||||
/**
|
||||
* check if a thread is locked by another thread
|
||||
* @param threadName
|
||||
* @return the thread id if there is a lock or null if there is none
|
||||
*/
|
||||
public Lock lockedBy(Thread threadName) {
|
||||
int p;
|
||||
List<String> list = this.get(threadName);
|
||||
if (list == null) return null;
|
||||
for (String s: list) {
|
||||
if ((p = s.indexOf("<")) > 0 && s.indexOf("locked <") < 0) {
|
||||
return new Lock(s.substring(p + 1, s.indexOf('>')));
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
public Map<Thread, Integer> countLocks() {
|
||||
Map<Lock, Thread> locks = locks();
|
||||
Map<Thread, Integer> count = new HashMap<Thread, Integer>();
|
||||
for (Map.Entry<Lock, Thread> entry: locks.entrySet()) {
|
||||
// look where the lock has an effect
|
||||
int c = 0;
|
||||
for (Thread thread: this.keySet()) if (entry.getKey().equals(lockedBy(thread))) c++;
|
||||
if (c > 0) count.put(entry.getValue(), c);
|
||||
}
|
||||
return count;
|
||||
}
|
||||
|
||||
public void print() {
|
||||
for (Thread thread: this.keySet()) print(thread);
|
||||
}
|
||||
|
||||
public void print(Thread thread) {
|
||||
List<String> list = this.get(thread);
|
||||
if (list == null) return;
|
||||
System.out.println("Thread: " + thread);
|
||||
for (String s: list) System.out.println(" " + s);
|
||||
System.out.println("");
|
||||
}
|
||||
|
||||
public static void main(String[] args) {
|
||||
if (args.length == 2 && args[0].equals("-f")) {
|
||||
File dumpfile = new File(args[1]);
|
||||
ThreadDump dump = null;
|
||||
try {
|
||||
dump = new ThreadDump(dumpfile);
|
||||
} catch (IOException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
//dump.print();
|
||||
Map<Thread, Integer> locks = dump.countLocks();
|
||||
for (int i = 0; i < dump.size() + 10; i++) {
|
||||
for (Map.Entry<Thread, Integer> entry: locks.entrySet()) {
|
||||
if (entry.getValue().intValue() == i) {
|
||||
System.out.println("holds lock for " + i + " threads:");
|
||||
dump.print(entry.getKey());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Loading…
Reference in New Issue
Block a user