From: Andrew Morton <akpm@linux-foundation.org>
To: akpm@linux-foundation.org, gpiccoli@canonical.com,
keescook@chromium.org, linux-mm@kvack.org,
mm-commits@vger.kernel.org, penguin-kernel@I-love.SAKURA.ne.jp,
torvalds@linux-foundation.org
Subject: [patch 13/54] kernel/hung_task.c: introduce sysctl to print all traces when a hung task is detected
Date: Sun, 07 Jun 2020 21:40:45 -0700 [thread overview]
Message-ID: <20200608044045.zb88z803t%akpm@linux-foundation.org> (raw)
In-Reply-To: <20200607212615.b050e41fac139a1e16fe00bd@linux-foundation.org>
From: "Guilherme G. Piccoli" <gpiccoli@canonical.com>
Subject: kernel/hung_task.c: introduce sysctl to print all traces when a hung task is detected
Commit 401c636a0eeb ("kernel/hung_task.c: show all hung tasks before
panic") introduced a change in that we started to show all CPUs backtraces
when a hung task is detected _and_ the sysctl/kernel parameter
"hung_task_panic" is set. The idea is good, because usually when
observing deadlocks (that may lead to hung tasks), the culprit is another
task holding a lock and not necessarily the task detected as hung.
The problem with this approach is that dumping backtraces is a slightly
expensive task, specially printing that on console (and specially in many
CPU machines, as servers commonly found nowadays). So, users that plan to
collect a kdump to investigate the hung tasks and narrow down the deadlock
definitely don't need the CPUs backtrace on dmesg/console, which will
delay the panic and pollute the log (crash tool would easily grab all CPUs
traces with 'bt -a' command).
Also, there's the reciprocal scenario: some users may be interested in
seeing the CPUs backtraces but not have the system panic when a hung task
is detected. The current approach hence is almost as embedding a policy
in the kernel, by forcing the CPUs backtraces' dump (only) on
hung_task_panic.
This patch decouples the panic event on hung task from the CPUs backtraces
dump, by creating (and documenting) a new sysctl called
"hung_task_all_cpu_backtrace", analog to the approach taken on soft/hard
lockups, that have both a panic and an "all_cpu_backtrace" sysctl to allow
individual control. The new mechanism for dumping the CPUs backtraces on
hung task detection respects "hung_task_warnings" by not dumping the
traces in case there's no warnings left.
Link: http://lkml.kernel.org/r/20200327223646.20779-1-gpiccoli@canonical.com
Signed-off-by: Guilherme G. Piccoli <gpiccoli@canonical.com>
Reviewed-by: Kees Cook <keescook@chromium.org>
Cc: Tetsuo Handa <penguin-kernel@I-love.SAKURA.ne.jp>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
---
Documentation/admin-guide/sysctl/kernel.rst | 14 ++++++++++++
include/linux/sched/sysctl.h | 7 ++++++
kernel/hung_task.c | 20 ++++++++++++++++--
kernel/sysctl.c | 11 +++++++++
4 files changed, 50 insertions(+), 2 deletions(-)
--- a/Documentation/admin-guide/sysctl/kernel.rst~kernel-hung_taskc-introduce-sysctl-to-print-all-traces-when-a-hung-task-is-detected
+++ a/Documentation/admin-guide/sysctl/kernel.rst
@@ -335,6 +335,20 @@ Path for the hotplug policy agent.
Default value is "``/sbin/hotplug``".
+hung_task_all_cpu_backtrace:
+================
+
+If this option is set, the kernel will send an NMI to all CPUs to dump
+their backtraces when a hung task is detected. This file shows up if
+CONFIG_DETECT_HUNG_TASK and CONFIG_SMP are enabled.
+
+0: Won't show all CPUs backtraces when a hung task is detected.
+This is the default behavior.
+
+1: Will non-maskably interrupt all CPUs and dump their backtraces when
+a hung task is detected.
+
+
hung_task_panic
===============
--- a/include/linux/sched/sysctl.h~kernel-hung_taskc-introduce-sysctl-to-print-all-traces-when-a-hung-task-is-detected
+++ a/include/linux/sched/sysctl.h
@@ -7,6 +7,13 @@
struct ctl_table;
#ifdef CONFIG_DETECT_HUNG_TASK
+
+#ifdef CONFIG_SMP
+extern unsigned int sysctl_hung_task_all_cpu_backtrace;
+#else
+#define sysctl_hung_task_all_cpu_backtrace 0
+#endif /* CONFIG_SMP */
+
extern int sysctl_hung_task_check_count;
extern unsigned int sysctl_hung_task_panic;
extern unsigned long sysctl_hung_task_timeout_secs;
--- a/kernel/hung_task.c~kernel-hung_taskc-introduce-sysctl-to-print-all-traces-when-a-hung-task-is-detected
+++ a/kernel/hung_task.c
@@ -53,9 +53,18 @@ int __read_mostly sysctl_hung_task_warni
static int __read_mostly did_panic;
static bool hung_task_show_lock;
static bool hung_task_call_panic;
+static bool hung_task_show_all_bt;
static struct task_struct *watchdog_task;
+#ifdef CONFIG_SMP
+/*
+ * Should we dump all CPUs backtraces in a hung task event?
+ * Defaults to 0, can be changed via sysctl.
+ */
+unsigned int __read_mostly sysctl_hung_task_all_cpu_backtrace;
+#endif /* CONFIG_SMP */
+
/*
* Should we panic (and reboot, if panic_timeout= is set) when a
* hung task is detected:
@@ -127,6 +136,9 @@ static void check_hung_task(struct task_
" disables this message.\n");
sched_show_task(t);
hung_task_show_lock = true;
+
+ if (sysctl_hung_task_all_cpu_backtrace)
+ hung_task_show_all_bt = true;
}
touch_nmi_watchdog();
@@ -191,10 +203,14 @@ static void check_hung_uninterruptible_t
rcu_read_unlock();
if (hung_task_show_lock)
debug_show_all_locks();
- if (hung_task_call_panic) {
+
+ if (hung_task_show_all_bt) {
+ hung_task_show_all_bt = false;
trigger_all_cpu_backtrace();
- panic("hung_task: blocked tasks");
}
+
+ if (hung_task_call_panic)
+ panic("hung_task: blocked tasks");
}
static long hung_timeout_jiffies(unsigned long last_checked,
--- a/kernel/sysctl.c~kernel-hung_taskc-introduce-sysctl-to-print-all-traces-when-a-hung-task-is-detected
+++ a/kernel/sysctl.c
@@ -2437,6 +2437,17 @@ static struct ctl_table kern_table[] = {
},
#endif
#ifdef CONFIG_DETECT_HUNG_TASK
+#ifdef CONFIG_SMP
+ {
+ .procname = "hung_task_all_cpu_backtrace",
+ .data = &sysctl_hung_task_all_cpu_backtrace,
+ .maxlen = sizeof(int),
+ .mode = 0644,
+ .proc_handler = proc_dointvec_minmax,
+ .extra1 = SYSCTL_ZERO,
+ .extra2 = SYSCTL_ONE,
+ },
+#endif /* CONFIG_SMP */
{
.procname = "hung_task_panic",
.data = &sysctl_hung_task_panic,
_
next prev parent reply other threads:[~2020-06-08 4:40 UTC|newest]
Thread overview: 57+ messages / expand[flat|nested] mbox.gz Atom feed top
[not found] <20200607212615.b050e41fac139a1e16fe00bd@linux-foundation.org>
2020-06-08 4:40 ` [patch 01/54] mm/page_idle.c: skip offline pages Andrew Morton
2020-06-08 4:40 ` [patch 02/54] ipc/msg: add missing annotation for freeque() Andrew Morton
2020-06-08 4:40 ` [patch 03/54] ipc/namespace.c: use a work queue to free_ipc Andrew Morton
2020-06-08 4:40 ` [patch 04/54] dynamic_debug: add an option to enable dynamic debug for modules only Andrew Morton
2020-06-08 4:58 ` 答复: " 翟京 (Orson Zhai)
2020-06-08 4:40 ` [patch 05/54] kernel: add panic_on_taint Andrew Morton
2020-06-08 4:40 ` [patch 06/54] xarray.h: correct return code documentation for xa_store_{bh,irq}() Andrew Morton
2020-06-08 4:40 ` [patch 07/54] kernel/sysctl: support setting sysctl parameters from kernel command line Andrew Morton
2020-06-08 4:40 ` [patch 08/54] kernel/sysctl: support handling command line aliases Andrew Morton
2020-06-08 4:40 ` [patch 09/54] kernel/hung_task convert hung_task_panic boot parameter to sysctl Andrew Morton
2020-06-08 4:40 ` [patch 10/54] tools/testing/selftests/sysctl/sysctl.sh: support CONFIG_TEST_SYSCTL=y Andrew Morton
2020-06-08 4:40 ` [patch 11/54] lib/test_sysctl: support testing of sysctl. boot parameter Andrew Morton
2020-06-08 4:40 ` [patch 12/54] kernel/watchdog.c: convert {soft/hard}lockup boot parameters to sysctl aliases Andrew Morton
2020-06-08 4:40 ` Andrew Morton [this message]
2020-06-08 4:40 ` [patch 14/54] panic: add sysctl to dump all CPUs backtraces on oops event Andrew Morton
2020-06-08 4:40 ` [patch 15/54] kernel/sysctl.c: ignore out-of-range taint bits introduced via kernel.tainted Andrew Morton
2020-06-08 4:40 ` [patch 16/54] mm/gup.c: convert to use get_user_{page|pages}_fast_only() Andrew Morton
2020-06-08 4:40 ` [patch 17/54] mm/gup: update pin_user_pages.rst for "case 3" (mmu notifiers) Andrew Morton
2020-06-08 4:41 ` [patch 18/54] mm/gup: introduce pin_user_pages_locked() Andrew Morton
2020-06-08 4:41 ` [patch 19/54] mm/gup: frame_vector: convert get_user_pages() --> pin_user_pages() Andrew Morton
2020-06-08 4:41 ` [patch 20/54] mm/gup: documentation fix for pin_user_pages*() APIs Andrew Morton
2020-06-08 4:41 ` [patch 21/54] docs: mm/gup: pin_user_pages.rst: add a "case 5" Andrew Morton
2020-06-08 4:41 ` [patch 22/54] vhost: convert get_user_pages() --> pin_user_pages() Andrew Morton
2020-06-08 4:41 ` [patch 23/54] mm/mmap.c: add more sanity checks to get_unmapped_area() Andrew Morton
2020-06-08 4:41 ` [patch 24/54] mm/mmap.c: do not allow mappings outside of allowed limits Andrew Morton
2020-06-08 17:50 ` Linus Torvalds
2020-06-08 17:55 ` Linus Torvalds
2020-06-08 4:41 ` [patch 25/54] arm: fix the flush_icache_range arguments in set_fiq_handler Andrew Morton
2020-06-08 4:41 ` [patch 26/54] nds32: unexport flush_icache_page Andrew Morton
2020-06-08 4:41 ` [patch 27/54] powerpc: unexport flush_icache_user_range Andrew Morton
2020-06-08 4:41 ` [patch 28/54] unicore32: remove flush_cache_user_range Andrew Morton
2020-06-08 4:41 ` [patch 29/54] asm-generic: fix the inclusion guards for cacheflush.h Andrew Morton
2020-06-08 4:41 ` [patch 30/54] asm-generic: don't include <linux/mm.h> in cacheflush.h Andrew Morton
2020-06-08 4:41 ` [patch 31/54] asm-generic: improve the flush_dcache_page stub Andrew Morton
2020-06-08 4:41 ` [patch 32/54] alpha: use asm-generic/cacheflush.h Andrew Morton
2020-06-08 4:41 ` [patch 33/54] arm64: " Andrew Morton
2020-06-08 4:41 ` [patch 34/54] c6x: " Andrew Morton
2020-06-08 4:41 ` [patch 35/54] hexagon: " Andrew Morton
2020-06-08 4:42 ` [patch 36/54] ia64: " Andrew Morton
2020-06-08 4:42 ` [patch 37/54] microblaze: " Andrew Morton
2020-06-08 4:42 ` [patch 38/54] m68knommu: " Andrew Morton
2020-06-08 4:42 ` [patch 39/54] openrisc: " Andrew Morton
2020-06-08 4:42 ` [patch 40/54] powerpc: " Andrew Morton
2020-06-08 4:42 ` [patch 41/54] riscv: " Andrew Morton
2020-06-08 4:42 ` [patch 42/54] arm,sparc,unicore32: remove flush_icache_user_range Andrew Morton
2020-06-08 4:42 ` [patch 43/54] mm: rename flush_icache_user_range to flush_icache_user_page Andrew Morton
2020-06-08 4:42 ` [patch 44/54] asm-generic: add a flush_icache_user_range stub Andrew Morton
2020-06-08 4:42 ` [patch 45/54] sh: implement flush_icache_user_range Andrew Morton
2020-06-08 4:42 ` [patch 46/54] xtensa: " Andrew Morton
2020-06-08 4:42 ` [patch 47/54] arm: rename flush_cache_user_range to flush_icache_user_range Andrew Morton
2020-06-08 4:42 ` [patch 48/54] m68k: implement flush_icache_user_range Andrew Morton
2020-06-08 4:42 ` [patch 49/54] exec: only build read_code when needed Andrew Morton
2020-06-08 4:42 ` [patch 50/54] exec: use flush_icache_user_range in read_code Andrew Morton
2020-06-08 4:42 ` [patch 51/54] binfmt_flat: use flush_icache_user_range Andrew Morton
2020-06-08 4:42 ` [patch 52/54] nommu: use flush_icache_user_range in brk and mmap Andrew Morton
2020-06-08 4:42 ` [patch 53/54] module: move the set_fs hack for flush_icache_range to m68k Andrew Morton
2020-06-08 4:42 ` [patch 54/54] doc: cgroup: update note about conditions when oom killer is invoked Andrew Morton
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20200608044045.zb88z803t%akpm@linux-foundation.org \
--to=akpm@linux-foundation.org \
--cc=gpiccoli@canonical.com \
--cc=keescook@chromium.org \
--cc=linux-mm@kvack.org \
--cc=mm-commits@vger.kernel.org \
--cc=penguin-kernel@I-love.SAKURA.ne.jp \
--cc=torvalds@linux-foundation.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).