From 22841ffbf11dc9631a7d1236d0ae3506a2ef8b35 Mon Sep 17 00:00:00 2001 From: Michael Peter Christen Date: Tue, 1 Dec 2020 03:00:24 +0100 Subject: [PATCH] creating a threaddump during every cleanup process to be able to find out what a peer did (not) last time before a crash --- htroot/Threaddump_p.java | 100 +++--------------- .../net/yacy/kelondro/logging/ThreadDump.java | 75 ++++++++++++- source/net/yacy/search/Switchboard.java | 17 ++- 3 files changed, 104 insertions(+), 88 deletions(-) diff --git a/htroot/Threaddump_p.java b/htroot/Threaddump_p.java index 32bd879f0..de52b3fd9 100644 --- a/htroot/Threaddump_p.java +++ b/htroot/Threaddump_p.java @@ -25,20 +25,8 @@ // along with this program; if not, write to the Free Software // Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - -import java.io.File; -import java.io.IOException; -import java.lang.management.ManagementFactory; -import java.lang.management.ThreadInfo; -import java.lang.management.ThreadMXBean; -import java.util.ArrayList; -import java.util.Date; -import java.util.Map; - import net.yacy.cora.protocol.RequestHeader; import net.yacy.kelondro.logging.ThreadDump; -import net.yacy.kelondro.util.OS; -import net.yacy.peers.operation.yacyBuildProperties; import net.yacy.search.Switchboard; import net.yacy.server.serverObjects; import net.yacy.server.serverSwitch; @@ -47,84 +35,26 @@ public class Threaddump_p { public static serverObjects respond(@SuppressWarnings("unused") final RequestHeader header, final serverObjects post, final serverSwitch env) { - serverObjects prop = new serverObjects(); - Switchboard sb = (Switchboard) env; + serverObjects prop = new serverObjects(); + Switchboard sb = (Switchboard) env; - final StringBuilder buffer = new StringBuilder(1000); + final boolean plain = post != null && post.getBoolean("plain"); + final int sleep = (post == null) ? 0 : post.getInt("sleep", 0); // a sleep before creation of a thread dump can be used for profiling + if (sleep > 0) try {Thread.sleep(sleep);} catch (final InterruptedException e) {} + prop.put("dump", "1"); - final boolean plain = post != null && post.getBoolean("plain"); - final int sleep = (post == null) ? 0 : post.getInt("sleep", 0); // a sleep before creation of a thread dump can be used for profiling - if (sleep > 0) try {Thread.sleep(sleep);} catch (final InterruptedException e) {} - prop.put("dump", "1"); - // Thread dump - final Date dt = new Date(); - final String versionstring = yacyBuildProperties.getVersion() + "/" + yacyBuildProperties.getSVNRevision(); - Runtime runtime = Runtime.getRuntime(); - - ThreadDump.bufferappend(buffer, plain, "************* Start Thread Dump " + dt + " *******************"); - ThreadDump.bufferappend(buffer, plain, " "); - ThreadDump.bufferappend(buffer, plain, "YaCy Version: " + versionstring); - ThreadDump.bufferappend(buffer, plain, "Assigned   Memory = " + (runtime.maxMemory())); - ThreadDump.bufferappend(buffer, plain, "Used       Memory = " + (runtime.totalMemory() - runtime.freeMemory())); - ThreadDump.bufferappend(buffer, plain, "Available  Memory = " + (runtime.maxMemory() - runtime.totalMemory() + runtime.freeMemory())); - ThreadDump.bufferappend(buffer, plain, " "); - ThreadDump.bufferappend(buffer, plain, " "); - - int multipleCount = 100; - File appPath = sb.getAppPath(); - if (post != null && post.containsKey("multipleThreaddump")) { - multipleCount = post.getInt("count", multipleCount); - final ArrayList> traces = new ArrayList>(); - for (int i = 0; i < multipleCount; i++) { - try { - traces.add(ThreadDump.getAllStackTraces()); - } catch (final OutOfMemoryError e) { - break; - } - } - ThreadDump.appendStackTraceStats(appPath, buffer, traces, plain); - } else { - // write a thread dump to standard error output - File logFile = new File("yacy.log"); - if (ThreadDump.canProduceLockedBy(logFile)) { - try { - new ThreadDump(logFile).appendBlockTraces(buffer, plain); - } catch (final IOException e) { - e.printStackTrace(); - } - } else if (OS.canExecUnix) { - ThreadDump.bufferappend(buffer, plain, "this thread dump function can find threads that lock others, to enable this function start YaCy with 'startYACY.sh -l'"); - ThreadDump.bufferappend(buffer, plain, " "); - } - - // generate a single thread dump - final Map stackTraces = ThreadDump.getAllStackTraces(); - new ThreadDump(appPath, stackTraces, plain, Thread.State.BLOCKED).appendStackTraces(buffer, plain, Thread.State.BLOCKED); - new ThreadDump(appPath, stackTraces, plain, Thread.State.RUNNABLE).appendStackTraces(buffer, plain, Thread.State.RUNNABLE); - new ThreadDump(appPath, stackTraces, plain, Thread.State.TIMED_WAITING).appendStackTraces(buffer, plain, Thread.State.TIMED_WAITING); - new ThreadDump(appPath, stackTraces, plain, Thread.State.WAITING).appendStackTraces(buffer, plain, Thread.State.WAITING); - new ThreadDump(appPath, stackTraces, plain, Thread.State.NEW).appendStackTraces(buffer, plain, Thread.State.NEW); - new ThreadDump(appPath, stackTraces, plain, Thread.State.TERMINATED).appendStackTraces(buffer, plain, Thread.State.TERMINATED); + int multipleCount = 100; + boolean multiple = post != null && post.containsKey("multipleThreaddump"); + if (multiple) { + multipleCount = post.getInt("count", multipleCount); } - ThreadDump.bufferappend(buffer, plain, "************* End Thread Dump " + dt + " *******************"); + String threaddump = ThreadDump.threaddump(sb, plain, sleep, multiple, multipleCount); + prop.put("plain_count", multipleCount); + prop.put("plain_content", threaddump); + prop.put("plain", (plain) ? 1 : 0); - - ThreadDump.bufferappend(buffer, plain, ""); - ThreadMXBean threadbean = ManagementFactory.getThreadMXBean(); - ThreadDump.bufferappend(buffer, plain, "Thread list from ThreadMXBean, " + threadbean.getThreadCount() + " threads:"); - ThreadInfo[] threadinfo = threadbean.dumpAllThreads(true, true); - for (ThreadInfo ti: threadinfo) { - ThreadDump.bufferappend(buffer, plain, ti.getThreadName()); - } - - prop.put("plain_count", multipleCount); - prop.put("plain_content", buffer.toString()); - prop.put("plain", (plain) ? 1 : 0); - - return prop; // return from serverObjects respond() + return prop; } - - } diff --git a/source/net/yacy/kelondro/logging/ThreadDump.java b/source/net/yacy/kelondro/logging/ThreadDump.java index c554d304c..9f1f24eaa 100644 --- a/source/net/yacy/kelondro/logging/ThreadDump.java +++ b/source/net/yacy/kelondro/logging/ThreadDump.java @@ -32,7 +32,11 @@ import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; import java.io.RandomAccessFile; +import java.lang.management.ManagementFactory; +import java.lang.management.ThreadInfo; +import java.lang.management.ThreadMXBean; import java.util.ArrayList; +import java.util.Date; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -41,6 +45,8 @@ import java.util.regex.Pattern; import net.yacy.document.parser.html.CharacterCoding; import net.yacy.kelondro.util.FileUtils; import net.yacy.kelondro.util.OS; +import net.yacy.peers.operation.yacyBuildProperties; +import net.yacy.search.Switchboard; import net.yacy.utils.nxTools; public class ThreadDump extends HashMap> implements Map> { @@ -372,7 +378,7 @@ public class ThreadDump extends HashMap> imp } public static void bufferappend(final StringBuilder buffer, final boolean plain, final String a) { - buffer.append(a); + buffer.append(plain ? a.replaceAll(" ", "") : a); buffer.append(plain ? "\n" : "
"); } @@ -449,6 +455,73 @@ public class ThreadDump extends HashMap> imp System.out.println(""); } + public static String threaddump(Switchboard sb, boolean plain, int sleep, boolean multiple, int multipleCount) { + final StringBuilder buffer = new StringBuilder(1000); + + if (sleep > 0) try {Thread.sleep(sleep);} catch (final InterruptedException e) {} + // Thread dump + final Date dt = new Date(); + final String versionstring = yacyBuildProperties.getVersion() + "/" + yacyBuildProperties.getSVNRevision(); + Runtime runtime = Runtime.getRuntime(); + + ThreadDump.bufferappend(buffer, plain, "************* Start Thread Dump " + dt + " *******************"); + ThreadDump.bufferappend(buffer, plain, " "); + ThreadDump.bufferappend(buffer, plain, "YaCy Version: " + versionstring); + ThreadDump.bufferappend(buffer, plain, "Assigned   Memory = " + (runtime.maxMemory())); + ThreadDump.bufferappend(buffer, plain, "Used       Memory = " + (runtime.totalMemory() - runtime.freeMemory())); + ThreadDump.bufferappend(buffer, plain, "Available  Memory = " + (runtime.maxMemory() - runtime.totalMemory() + runtime.freeMemory())); + ThreadDump.bufferappend(buffer, plain, " "); + ThreadDump.bufferappend(buffer, plain, " "); + + File appPath = sb.getAppPath(); + if (multiple) { + final ArrayList> traces = new ArrayList>(); + for (int i = 0; i < multipleCount; i++) { + try { + traces.add(ThreadDump.getAllStackTraces()); + } catch (final OutOfMemoryError e) { + break; + } + } + ThreadDump.appendStackTraceStats(appPath, buffer, traces, plain); + } else { + // write a thread dump to standard error output + File logFile = new File("yacy.log"); + if (ThreadDump.canProduceLockedBy(logFile)) { + try { + new ThreadDump(logFile).appendBlockTraces(buffer, plain); + } catch (final IOException e) { + e.printStackTrace(); + } + } else if (OS.canExecUnix) { + ThreadDump.bufferappend(buffer, plain, "this thread dump function can find threads that lock others, to enable this function start YaCy with 'startYACY.sh -l'"); + ThreadDump.bufferappend(buffer, plain, " "); + } + + // generate a single thread dump + final Map stackTraces = ThreadDump.getAllStackTraces(); + new ThreadDump(appPath, stackTraces, plain, Thread.State.BLOCKED).appendStackTraces(buffer, plain, Thread.State.BLOCKED); + new ThreadDump(appPath, stackTraces, plain, Thread.State.RUNNABLE).appendStackTraces(buffer, plain, Thread.State.RUNNABLE); + new ThreadDump(appPath, stackTraces, plain, Thread.State.TIMED_WAITING).appendStackTraces(buffer, plain, Thread.State.TIMED_WAITING); + new ThreadDump(appPath, stackTraces, plain, Thread.State.WAITING).appendStackTraces(buffer, plain, Thread.State.WAITING); + new ThreadDump(appPath, stackTraces, plain, Thread.State.NEW).appendStackTraces(buffer, plain, Thread.State.NEW); + new ThreadDump(appPath, stackTraces, plain, Thread.State.TERMINATED).appendStackTraces(buffer, plain, Thread.State.TERMINATED); + } + + ThreadDump.bufferappend(buffer, plain, "************* End Thread Dump " + dt + " *******************"); + + ThreadDump.bufferappend(buffer, plain, ""); + ThreadMXBean threadbean = ManagementFactory.getThreadMXBean(); + ThreadDump.bufferappend(buffer, plain, "Thread list from ThreadMXBean, " + threadbean.getThreadCount() + " threads:"); + ThreadInfo[] threadinfo = threadbean.dumpAllThreads(true, true); + for (ThreadInfo ti: threadinfo) { + ThreadDump.bufferappend(buffer, plain, ti.getThreadName()); + } + + return buffer.toString(); + } + + public static void main(final String[] args) { ThreadDump dump = null; if (args.length == 2 && args[0].equals("-f")) { diff --git a/source/net/yacy/search/Switchboard.java b/source/net/yacy/search/Switchboard.java index 6d823344b..fb6178ab0 100644 --- a/source/net/yacy/search/Switchboard.java +++ b/source/net/yacy/search/Switchboard.java @@ -49,6 +49,7 @@ import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; import java.io.OutputStream; +import java.io.PrintWriter; import java.io.Reader; import java.net.InetAddress; import java.net.MalformedURLException; @@ -193,6 +194,7 @@ import net.yacy.kelondro.blob.Tables.SortDirection; import net.yacy.kelondro.data.meta.URIMetadataNode; import net.yacy.kelondro.data.word.Word; import net.yacy.kelondro.logging.GuiHandler; +import net.yacy.kelondro.logging.ThreadDump; import net.yacy.kelondro.rwi.ReferenceContainer; import net.yacy.kelondro.util.FileUtils; import net.yacy.kelondro.util.MemoryControl; @@ -2593,11 +2595,22 @@ public final class Switchboard extends serverSwitch { } public boolean cleanupJob() { - + ConcurrentLog.ensureWorkerIsRunning(); try { clearCaches(); + // write a thread dump to log path + try { + File tdlog = new File(dataPath, "DATA/LOG/threaddump.txt"); + PrintWriter out = new PrintWriter(tdlog); + String threaddump = ThreadDump.threaddump(this, true, 0, false, 0); + out.println(threaddump); + out.close(); + } catch (IOException e) { + log.info("cannot write threaddump", e); + } + // clear caches if necessary if ( !MemoryControl.request(128000000L, false) ) { this.index.clearCaches(); @@ -2621,7 +2634,7 @@ public final class Switchboard extends serverSwitch { log.info("finishing greedy learning phase, size=" +cs); } } - + // refresh recrawl dates try { CrawlProfile selentry;