From 86388ac0cadaffdcf05bc08da4c48917fc38bbe3 Mon Sep 17 00:00:00 2001 From: xingsuo-zbz Date: Wed, 15 Jul 2026 11:00:42 +0800 Subject: [PATCH] [FLINK-39984][runtime][webUI] Support LITE/FULL thread dump modes ThreadMXBean.dumpAllThreads(true, true) enters a single JVM-wide safepoint to collect monitor/synchronizer state; on busy JVMs the pause can exceed heartbeat.timeout and cause unnecessary TaskManager failover. - Introduce ThreadDumpMode {LITE, FULL}: LITE = dumpAllThreads(false, false), FULL preserves today's (true, true) behavior. Exposed via an optional query parameter `?mode=lite|full` on the JM/TM thread-dump endpoints. - Add config cluster.thread-dump.default-mode (default FULL to preserve upgrade behavior; LITE recommended for large clusters). - Add a Lite/Full toggle to both Web UI thread-dump pages; selecting a mode does not auto-fetch, the download link tracks the selection. Co-Authored-By: Claude Opus 4.7 --- .../generated/cluster_configuration.html | 6 + .../generated/expert_cluster_section.html | 6 + .../generated/rest_v1_dispatcher.html | 20 ++++ docs/static/generated/rest_v1_dispatcher.yml | 22 ++++ .../flink/configuration/ClusterOptions.java | 25 +++++ .../src/test/resources/rest_api_v1.snapshot | 10 +- .../job-manager-thread-dump.component.html | 36 +++++- .../job-manager-thread-dump.component.less | 22 ++++ .../job-manager-thread-dump.component.ts | 40 ++++++- .../task-manager-thread-dump.component.html | 36 +++++- .../task-manager-thread-dump.component.less | 25 +++++ .../task-manager-thread-dump.component.ts | 52 ++++++++- .../src/app/services/job-manager.service.ts | 8 +- .../src/app/services/task-manager.service.ts | 18 +-- .../flink/runtime/dispatcher/Dispatcher.java | 13 ++- .../resourcemanager/ResourceManager.java | 5 +- .../ResourceManagerGateway.java | 7 +- .../cluster/JobManagerThreadDumpHandler.java | 17 ++- .../TaskManagerThreadDumpHandler.java | 17 ++- .../runtime/rest/messages/ThreadDumpInfo.java | 6 +- .../runtime/rest/messages/ThreadDumpMode.java | 83 ++++++++++++++ .../ThreadDumpModeQueryParameter.java | 53 +++++++++ .../cluster/JobManagerThreadDumpHeaders.java | 8 +- ...JobManagerThreadDumpMessageParameters.java | 44 ++++++++ .../TaskManagerThreadDumpHeaders.java | 6 +- ...askManagerThreadDumpMessageParameters.java | 37 ++++++ .../runtime/taskexecutor/TaskExecutor.java | 16 ++- .../taskexecutor/TaskExecutorGateway.java | 5 +- .../TaskExecutorGatewayDecoratorBase.java | 6 +- .../apache/flink/runtime/util/JvmUtils.java | 12 +- .../NonLeaderRetrievalRestfulGateway.java | 4 +- .../runtime/webmonitor/RestfulGateway.java | 5 +- .../utils/TestingResourceManagerGateway.java | 3 +- .../ThreadDumpModeQueryParameterTest.java | 66 +++++++++++ .../TaskExecutorThreadDumpTest.java | 105 ++++++++++++++++++ .../TestingTaskExecutorGateway.java | 4 +- .../webmonitor/TestingRestfulGateway.java | 4 +- 37 files changed, 780 insertions(+), 72 deletions(-) create mode 100644 flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/ThreadDumpMode.java create mode 100644 flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/ThreadDumpModeQueryParameter.java create mode 100644 flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/cluster/JobManagerThreadDumpMessageParameters.java create mode 100644 flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/taskmanager/TaskManagerThreadDumpMessageParameters.java create mode 100644 flink-runtime/src/test/java/org/apache/flink/runtime/rest/messages/ThreadDumpModeQueryParameterTest.java create mode 100644 flink-runtime/src/test/java/org/apache/flink/runtime/taskexecutor/TaskExecutorThreadDumpTest.java diff --git a/docs/layouts/shortcodes/generated/cluster_configuration.html b/docs/layouts/shortcodes/generated/cluster_configuration.html index 1d1dbd87d9cf1d..ada54985eadf2f 100644 --- a/docs/layouts/shortcodes/generated/cluster_configuration.html +++ b/docs/layouts/shortcodes/generated/cluster_configuration.html @@ -62,6 +62,12 @@ Duration The shutdown timeout for cluster services like executors. + +
cluster.thread-dump.default-mode
+ "FULL" + String + Default granularity of the JobManager/TaskManager thread-dump REST endpoint when no explicit mode query parameter is supplied. Allowed values (case-insensitive): LITE (stack traces only, negligible JVM pause) and FULL (adds locked monitors and j.u.c. synchronizers, equivalent to jstack -l; pauses the JVM in a safepoint for a duration that scales with heap size and thread count -- seconds on large TaskManagers). The default is FULL to preserve historical behavior; operators of large clusters are strongly encouraged to switch to LITE to avoid heartbeat timeouts caused by long safepoint pauses. +
cluster.thread-dump.stacktrace-max-depth
50 diff --git a/docs/layouts/shortcodes/generated/expert_cluster_section.html b/docs/layouts/shortcodes/generated/expert_cluster_section.html index f32e45d44d0093..82b6b9364a403c 100644 --- a/docs/layouts/shortcodes/generated/expert_cluster_section.html +++ b/docs/layouts/shortcodes/generated/expert_cluster_section.html @@ -26,6 +26,12 @@ Boolean Whether processes should halt on fatal errors instead of performing a graceful shutdown. In some environments (e.g. Java 8 with the G1 garbage collector), a regular graceful shutdown can lead to a JVM deadlock. See FLINK-16510 for details. + +
cluster.thread-dump.default-mode
+ "FULL" + String + Default granularity of the JobManager/TaskManager thread-dump REST endpoint when no explicit mode query parameter is supplied. Allowed values (case-insensitive): LITE (stack traces only, negligible JVM pause) and FULL (adds locked monitors and j.u.c. synchronizers, equivalent to jstack -l; pauses the JVM in a safepoint for a duration that scales with heap size and thread count -- seconds on large TaskManagers). The default is FULL to preserve historical behavior; operators of large clusters are strongly encouraged to switch to LITE to avoid heartbeat timeouts caused by long safepoint pauses. +
cluster.thread-dump.stacktrace-max-depth
50 diff --git a/docs/layouts/shortcodes/generated/rest_v1_dispatcher.html b/docs/layouts/shortcodes/generated/rest_v1_dispatcher.html index 98c02d7a418fc5..26749b00f691b4 100644 --- a/docs/layouts/shortcodes/generated/rest_v1_dispatcher.html +++ b/docs/layouts/shortcodes/generated/rest_v1_dispatcher.html @@ -1383,6 +1383,16 @@ Returns the thread dump of the JobManager. + + Query parameters + + + + + +