From: Andy Lutomirski <luto@kernel.org>
To: x86@kernel.org
Cc: LKML <linux-kernel@vger.kernel.org>,
Mark Rutland <mark.rutland@arm.com>,
Brian Gerst <brgerst@gmail.com>,
Andy Lutomirski <luto@kernel.org>
Subject: [PATCH v4 7/9] kentry: Add debugging checks for proper kentry API usage
Date: Wed, 17 Mar 2021 11:12:46 -0700 [thread overview]
Message-ID: <df38bc3aa82dc122c88bf902f6922ab7c22bf783.1616004689.git.luto@kernel.org> (raw)
In-Reply-To: <cover.1616004689.git.luto@kernel.org>
It's quite easy to mess up kentry calls. Add debgging checks that kentry
transitions to and from user mode match up and that kentry_nmi_enter() and
kentry_nmi_exit() match up.
Checking full matching of kentry_enter() with kentry_exit() needs
per-task state.
Signed-off-by: Andy Lutomirski <luto@kernel.org>
---
kernel/entry/common.c | 81 ++++++++++++++++++++++++++++++++++++++++++-
1 file changed, 80 insertions(+), 1 deletion(-)
diff --git a/kernel/entry/common.c b/kernel/entry/common.c
index 6fbe1c109877..8ba774184e00 100644
--- a/kernel/entry/common.c
+++ b/kernel/entry/common.c
@@ -11,9 +11,65 @@
#define CREATE_TRACE_POINTS
#include <trace/events/syscalls.h>
+/*
+ * kentry_cpu_depth is 0 in user mode, 1 in normal kernel mode, and
+ * 1 + n * KENTRY_DEPTH_NMI in kentry_nmi_enter() mode. We can't
+ * use a percpu variable to match up kentry_enter() from kernel mode
+ * with the corresponding kentry_exit() because tasks may schedule inside
+ * nested kentry_enter() regions.
+ */
+#define KENTRY_CPU_DEPTH_NMI 1024UL
+
+#ifdef CONFIG_DEBUG_ENTRY
+
+/*
+ * Extra safe WARN_ONCE. Per-arch optimized WARN_ONCE() implementations
+ * might go through the low-level entry and kentry code even before noticing
+ * that the warning already fired, which could result in recursive warnings.
+ * This carefully avoids any funny business once a given warning has fired.
+ */
+#define DEBUG_ENTRY_WARN_ONCE(condition, format...) ({ \
+ static bool __section(".data.once") __warned; \
+ int __ret_warn_once = !!(condition); \
+ \
+ if (unlikely(__ret_warn_once && !READ_ONCE(__warned))) {\
+ WRITE_ONCE(__warned, true); \
+ WARN(1, format); \
+ } \
+ unlikely(__ret_warn_once); \
+})
+
+
+static DEFINE_PER_CPU(unsigned int, kentry_cpu_depth) = 1UL;
+
+static __always_inline void kentry_cpu_depth_add(unsigned int n)
+{
+ this_cpu_add(kentry_cpu_depth, n);
+}
+
+static void kentry_cpu_depth_check(unsigned int n)
+{
+ DEBUG_ENTRY_WARN_ONCE(this_cpu_read(kentry_cpu_depth) < n, "kentry depth underflow");
+}
+
+static __always_inline void kentry_cpu_depth_sub(unsigned int n)
+{
+ this_cpu_sub(kentry_cpu_depth, n);
+}
+#else
+
+#define DEBUG_ENTRY_WARN_ONCE(condition, format...) do {} while (0)
+
+static __always_inline void kentry_cpu_depth_add(unsigned int n) {}
+static void kentry_cpu_depth_check(unsigned int n) {}
+static __always_inline void kentry_cpu_depth_sub(unsigned int n) {}
+
+#endif
+
/* See comment for enter_from_user_mode() in entry-common.h */
static __always_inline void __enter_from_user_mode(struct pt_regs *regs)
{
+ kentry_cpu_depth_add(1);
arch_check_user_regs(regs);
lockdep_hardirqs_off(CALLER_ADDR0);
@@ -22,6 +78,14 @@ static __always_inline void __enter_from_user_mode(struct pt_regs *regs)
instrumentation_begin();
trace_hardirqs_off_finish();
+
+#ifdef CONFIG_DEBUG_ENTRY
+ DEBUG_ENTRY_WARN_ONCE(
+ this_cpu_read(kentry_cpu_depth) != 1,
+ "kentry: __enter_from_user_mode() called while kentry thought the CPU was in the kernel (%u)",
+ this_cpu_read(kentry_cpu_depth));
+#endif
+
instrumentation_end();
}
@@ -92,6 +156,11 @@ long kentry_syscall_begin(struct pt_regs *regs, long syscall)
static __always_inline void __exit_to_user_mode(void)
{
instrumentation_begin();
+#ifdef CONFIG_DEBUG_ENTRY
+ DEBUG_ENTRY_WARN_ONCE(this_cpu_read(kentry_cpu_depth) != 1,
+ "__exit_to_user_mode called at wrong kentry cpu depth (%u)",
+ this_cpu_read(kentry_cpu_depth));
+#endif
trace_hardirqs_on_prepare();
lockdep_hardirqs_on_prepare(CALLER_ADDR0);
instrumentation_end();
@@ -99,6 +168,7 @@ static __always_inline void __exit_to_user_mode(void)
user_enter_irqoff();
arch_exit_to_user_mode();
lockdep_hardirqs_on(CALLER_ADDR0);
+ kentry_cpu_depth_sub(1);
}
/* Workaround to allow gradual conversion of architecture code */
@@ -346,7 +416,12 @@ noinstr void kentry_exit(struct pt_regs *regs, kentry_state_t state)
/* Check whether this returns to user mode */
if (user_mode(regs)) {
kentry_exit_to_user_mode(regs);
- } else if (!regs_irqs_disabled(regs)) {
+ return;
+ }
+
+ kentry_cpu_depth_check(1);
+
+ if (!regs_irqs_disabled(regs)) {
/*
* If RCU was not watching on entry this needs to be done
* carefully and needs the same ordering of lockdep/tracing
@@ -385,6 +460,8 @@ kentry_state_t noinstr kentry_nmi_enter(struct pt_regs *regs)
irq_state.lockdep = lockdep_hardirqs_enabled();
+ kentry_cpu_depth_add(KENTRY_CPU_DEPTH_NMI);
+
__nmi_enter();
lockdep_hardirqs_off(CALLER_ADDR0);
lockdep_hardirq_enter();
@@ -401,6 +478,7 @@ kentry_state_t noinstr kentry_nmi_enter(struct pt_regs *regs)
void noinstr kentry_nmi_exit(struct pt_regs *regs, kentry_state_t irq_state)
{
instrumentation_begin();
+ kentry_cpu_depth_check(KENTRY_CPU_DEPTH_NMI);
ftrace_nmi_exit();
if (irq_state.lockdep) {
trace_hardirqs_on_prepare();
@@ -413,4 +491,5 @@ void noinstr kentry_nmi_exit(struct pt_regs *regs, kentry_state_t irq_state)
if (irq_state.lockdep)
lockdep_hardirqs_on(CALLER_ADDR0);
__nmi_exit();
+ kentry_cpu_depth_sub(KENTRY_CPU_DEPTH_NMI);
}
--
2.30.2
next prev parent reply other threads:[~2021-03-17 18:13 UTC|newest]
Thread overview: 18+ messages / expand[flat|nested] mbox.gz Atom feed top
2021-03-17 18:12 [PATCH v4 0/9] kentry: A stable bugfix and a bunch of improvements Andy Lutomirski
2021-03-17 18:12 ` [PATCH v4 1/9] x86/dumpstack: Remove unnecessary range check fetching opcode bytes Andy Lutomirski
2021-03-17 18:12 ` [PATCH v4 2/9] x86/kthread,dumpstack: Set task_pt_regs->cs.RPL=3 for kernel threads Andy Lutomirski
2021-03-18 0:36 ` Josh Poimboeuf
2021-03-17 18:12 ` [PATCH v4 3/9] x86/entry: Convert ret_from_fork to C Andy Lutomirski
2021-03-18 0:38 ` Josh Poimboeuf
2021-03-19 16:05 ` Thomas Gleixner
2021-03-17 18:12 ` [PATCH v4 4/9] kentry: Simplify the common syscall API Andy Lutomirski
2021-03-19 16:09 ` Thomas Gleixner
2021-03-19 18:07 ` Thomas Gleixner
2021-03-17 18:12 ` [PATCH v4 5/9] kentry: Remove enter_from/exit_to_user_mode() Andy Lutomirski
2021-03-19 18:08 ` Thomas Gleixner
2021-03-17 18:12 ` [PATCH v4 6/9] entry: Make CONFIG_DEBUG_ENTRY available outside x86 Andy Lutomirski
2021-03-17 18:12 ` Andy Lutomirski [this message]
2021-03-19 16:17 ` [PATCH v4 7/9] kentry: Add debugging checks for proper kentry API usage Thomas Gleixner
2021-03-19 18:16 ` Thomas Gleixner
2021-03-17 18:12 ` [PATCH v4 8/9] kentry: Check that syscall entries and syscall exits match Andy Lutomirski
2021-03-17 18:12 ` [PATCH v4 9/9] kentry: Verify kentry state in instrumentation_begin/end() Andy Lutomirski
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=df38bc3aa82dc122c88bf902f6922ab7c22bf783.1616004689.git.luto@kernel.org \
--to=luto@kernel.org \
--cc=brgerst@gmail.com \
--cc=linux-kernel@vger.kernel.org \
--cc=mark.rutland@arm.com \
--cc=x86@kernel.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).