From: Thomas Gleixner <tglx@linutronix.de>
To: LKML <linux-kernel@vger.kernel.org>
Cc: x86@kernel.org, "Paul E. McKenney" <paulmck@kernel.org>,
Andy Lutomirski <luto@kernel.org>,
Alexandre Chartre <alexandre.chartre@oracle.com>,
Frederic Weisbecker <frederic@kernel.org>,
Paolo Bonzini <pbonzini@redhat.com>,
Sean Christopherson <sean.j.christopherson@intel.com>,
Masami Hiramatsu <mhiramat@kernel.org>,
Petr Mladek <pmladek@suse.com>,
Steven Rostedt <rostedt@goodmis.org>,
Joel Fernandes <joel@joelfernandes.org>,
Boris Ostrovsky <boris.ostrovsky@oracle.com>,
Juergen Gross <jgross@suse.com>, Brian Gerst <brgerst@gmail.com>,
Mathieu Desnoyers <mathieu.desnoyers@efficios.com>,
Josh Poimboeuf <jpoimboe@redhat.com>,
Will Deacon <will@kernel.org>
Subject: [patch V4 part 4 14/24] x86/nmi: Protect NMI entry against instrumentation
Date: Tue, 05 May 2020 15:49:40 +0200 [thread overview]
Message-ID: <20200505135314.716186134@linutronix.de> (raw)
In-Reply-To: 20200505134926.578885807@linutronix.de
Mark all functions in the fragile code parts noinstr or force inlining so
they can't be instrumented.
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
---
arch/x86/include/asm/desc.h | 8 ++++----
arch/x86/kernel/cpu/common.c | 6 ++----
arch/x86/kernel/nmi.c | 10 ++++++----
3 files changed, 12 insertions(+), 12 deletions(-)
--- a/arch/x86/include/asm/desc.h
+++ b/arch/x86/include/asm/desc.h
@@ -214,7 +214,7 @@ static inline void native_load_gdt(const
asm volatile("lgdt %0"::"m" (*dtr));
}
-static inline void native_load_idt(const struct desc_ptr *dtr)
+static __always_inline void native_load_idt(const struct desc_ptr *dtr)
{
asm volatile("lidt %0"::"m" (*dtr));
}
@@ -393,7 +393,7 @@ extern unsigned long system_vectors[];
#ifdef CONFIG_X86_64
DECLARE_PER_CPU(u32, debug_idt_ctr);
-static inline bool is_debug_idt_enabled(void)
+static __always_inline bool is_debug_idt_enabled(void)
{
if (this_cpu_read(debug_idt_ctr))
return true;
@@ -401,7 +401,7 @@ static inline bool is_debug_idt_enabled(
return false;
}
-static inline void load_debug_idt(void)
+static __always_inline void load_debug_idt(void)
{
load_idt((const struct desc_ptr *)&debug_idt_descr);
}
@@ -423,7 +423,7 @@ static inline void load_debug_idt(void)
* that doesn't need to disable interrupts, as nothing should be
* bothering the CPU then.
*/
-static inline void load_current_idt(void)
+static __always_inline void load_current_idt(void)
{
if (is_debug_idt_enabled())
load_debug_idt();
--- a/arch/x86/kernel/cpu/common.c
+++ b/arch/x86/kernel/cpu/common.c
@@ -1692,21 +1692,19 @@ void syscall_init(void)
DEFINE_PER_CPU(int, debug_stack_usage);
DEFINE_PER_CPU(u32, debug_idt_ctr);
-void debug_stack_set_zero(void)
+noinstr void debug_stack_set_zero(void)
{
this_cpu_inc(debug_idt_ctr);
load_current_idt();
}
-NOKPROBE_SYMBOL(debug_stack_set_zero);
-void debug_stack_reset(void)
+noinstr void debug_stack_reset(void)
{
if (WARN_ON(!this_cpu_read(debug_idt_ctr)))
return;
if (this_cpu_dec_return(debug_idt_ctr) == 0)
load_current_idt();
}
-NOKPROBE_SYMBOL(debug_stack_reset);
#else /* CONFIG_X86_64 */
--- a/arch/x86/kernel/nmi.c
+++ b/arch/x86/kernel/nmi.c
@@ -307,7 +307,7 @@ NOKPROBE_SYMBOL(unknown_nmi_error);
static DEFINE_PER_CPU(bool, swallow_nmi);
static DEFINE_PER_CPU(unsigned long, last_nmi_rip);
-static void default_do_nmi(struct pt_regs *regs)
+static noinstr void default_do_nmi(struct pt_regs *regs)
{
unsigned char reason = 0;
int handled;
@@ -333,6 +333,7 @@ static void default_do_nmi(struct pt_reg
__this_cpu_write(last_nmi_rip, regs->ip);
+ instr_begin();
handled = nmi_handle(NMI_LOCAL, regs);
__this_cpu_add(nmi_stats.normal, handled);
if (handled) {
@@ -346,6 +347,7 @@ static void default_do_nmi(struct pt_reg
*/
if (handled > 1)
__this_cpu_write(swallow_nmi, true);
+ instr_end();
return;
}
@@ -378,6 +380,7 @@ static void default_do_nmi(struct pt_reg
#endif
__this_cpu_add(nmi_stats.external, 1);
raw_spin_unlock(&nmi_reason_lock);
+ instr_end();
return;
}
raw_spin_unlock(&nmi_reason_lock);
@@ -416,8 +419,8 @@ static void default_do_nmi(struct pt_reg
__this_cpu_add(nmi_stats.swallow, 1);
else
unknown_nmi_error(reason, regs);
+ instr_end();
}
-NOKPROBE_SYMBOL(default_do_nmi);
/*
* NMIs can page fault or hit breakpoints which will cause it to lose
@@ -489,7 +492,7 @@ static DEFINE_PER_CPU(unsigned long, nmi
*/
static DEFINE_PER_CPU(int, update_debug_stack);
-static bool notrace is_debug_stack(unsigned long addr)
+static noinstr bool is_debug_stack(unsigned long addr)
{
struct cea_exception_stacks *cs = __this_cpu_read(cea_exception_stacks);
unsigned long top = CEA_ESTACK_TOP(cs, DB);
@@ -504,7 +507,6 @@ static bool notrace is_debug_stack(unsig
*/
return addr >= bot && addr < top;
}
-NOKPROBE_SYMBOL(is_debug_stack);
#endif
DEFINE_IDTENTRY_NMI(exc_nmi)
next prev parent reply other threads:[~2020-05-05 14:16 UTC|newest]
Thread overview: 94+ messages / expand[flat|nested] mbox.gz Atom feed top
2020-05-05 13:49 [patch V4 part 4 00/24] x86/entry: Entry/exception code rework, nasty exceptions Thomas Gleixner
2020-05-05 13:49 ` [patch V4 part 4 01/24] x86/int3: Ensure that poke_int3_handler() is not traced Thomas Gleixner
2020-05-14 4:57 ` Andy Lutomirski
2020-05-19 19:58 ` [tip: x86/entry] " tip-bot2 for Thomas Gleixner
2020-05-05 13:49 ` [patch V4 part 4 02/24] x86/int3: Avoid atomic instrumentation Thomas Gleixner
2020-05-08 13:27 ` Masami Hiramatsu
2020-05-14 4:57 ` Andy Lutomirski
2020-05-14 9:32 ` Peter Zijlstra
2020-05-14 12:51 ` Thomas Gleixner
2020-05-14 13:15 ` Peter Zijlstra
2020-05-14 14:55 ` Andy Lutomirski
2020-05-14 15:06 ` Thomas Gleixner
2020-05-14 15:08 ` Andy Lutomirski
2020-05-14 15:10 ` Peter Zijlstra
2020-05-19 19:58 ` [tip: x86/entry] " tip-bot2 for Peter Zijlstra
2020-05-05 13:49 ` [patch V4 part 4 03/24] lib/bsearch: Provide __always_inline variant Thomas Gleixner
2020-05-14 4:58 ` Andy Lutomirski
2020-05-19 19:58 ` [tip: x86/entry] " tip-bot2 for Peter Zijlstra
2020-05-05 13:49 ` [patch V4 part 4 04/24] x86/int3: Inline bsearch() Thomas Gleixner
2020-05-14 4:58 ` Andy Lutomirski
2020-05-19 19:58 ` [tip: x86/entry] " tip-bot2 for Peter Zijlstra
2020-05-05 13:49 ` [patch V4 part 4 05/24] x86/entry: Provide IDTENTRY_RAW Thomas Gleixner
2020-05-14 4:59 ` Andy Lutomirski
2020-05-19 19:58 ` [tip: x86/entry] x86/idtentry: " tip-bot2 for Thomas Gleixner
2020-05-05 13:49 ` [patch V4 part 4 06/24] x86/entry: Convert INT3 exception to IDTENTRY_RAW Thomas Gleixner
2020-05-14 5:01 ` Andy Lutomirski
2020-05-19 19:58 ` [tip: x86/entry] " tip-bot2 for Thomas Gleixner
2020-05-05 13:49 ` [patch V4 part 4 07/24] x86/traps: Split int3 handler up Thomas Gleixner
2020-05-14 5:03 ` Andy Lutomirski
2020-05-14 9:39 ` Peter Zijlstra
2020-05-19 19:58 ` [tip: x86/entry] " tip-bot2 for Peter Zijlstra
2020-05-05 13:49 ` [patch V4 part 4 08/24] x86/entry: Provide IDTENTRY_IST Thomas Gleixner
2020-05-14 16:39 ` Andy Lutomirski
2020-05-14 18:44 ` Thomas Gleixner
2020-05-19 19:58 ` [tip: x86/entry] x86/idtentry: " tip-bot2 for Thomas Gleixner
2020-05-05 13:49 ` [patch V4 part 4 09/24] x86/mce: Move nmi_enter/exit() into the entry point Thomas Gleixner
2020-05-15 5:23 ` Andy Lutomirski
2020-05-19 19:58 ` [tip: x86/entry] " tip-bot2 for Thomas Gleixner
2020-05-05 13:49 ` [patch V4 part 4 10/24] x86/entry: Convert Machine Check to IDTENTRY_IST Thomas Gleixner
2020-05-15 5:24 ` Andy Lutomirski
2020-05-19 19:58 ` [tip: x86/entry] " tip-bot2 for Thomas Gleixner
2020-05-05 13:49 ` [patch V4 part 4 11/24] x86/mce: Use untraced rd/wrmsr in the MCE offline/crash check Thomas Gleixner
2020-05-15 5:24 ` Andy Lutomirski
2020-05-19 19:58 ` [tip: x86/entry] " tip-bot2 for Thomas Gleixner
2020-05-05 13:49 ` [patch V4 part 4 12/24] x86/idtentry: Provide IDTENTRY_XEN for XEN/PV Thomas Gleixner
2020-05-15 5:25 ` Andy Lutomirski
2020-05-19 19:58 ` [tip: x86/entry] " tip-bot2 for Thomas Gleixner
2020-05-05 13:49 ` [patch V4 part 4 13/24] x86/entry: Convert NMI to IDTENTRY_NMI Thomas Gleixner
2020-05-15 5:26 ` Andy Lutomirski
2020-05-19 19:58 ` [tip: x86/entry] " tip-bot2 for Thomas Gleixner
2020-05-05 13:49 ` Thomas Gleixner [this message]
2020-05-15 5:26 ` [patch V4 part 4 14/24] x86/nmi: Protect NMI entry against instrumentation Andy Lutomirski
2020-05-19 19:58 ` [tip: x86/entry] " tip-bot2 for Thomas Gleixner
2020-05-05 13:49 ` [patch V4 part 4 15/24] x86/db: Split out dr6/7 handling Thomas Gleixner
2020-05-07 17:18 ` Alexandre Chartre
2020-05-08 8:59 ` Peter Zijlstra
2020-05-08 11:58 ` Thomas Gleixner
2020-05-08 12:45 ` Peter Zijlstra
2020-05-14 2:24 ` Mathieu Desnoyers
2020-05-14 17:28 ` Thomas Gleixner
2020-05-14 17:46 ` Mathieu Desnoyers
2020-05-15 14:32 ` Thomas Gleixner
2020-05-14 18:06 ` Steven Rostedt
2020-05-15 5:37 ` Andy Lutomirski
2020-05-19 19:58 ` [tip: x86/entry] " tip-bot2 for Peter Zijlstra
2020-05-05 13:49 ` [patch V4 part 4 16/24] x86/entry: Convert Debug exception to IDTENTRY_DB Thomas Gleixner
2020-05-15 5:27 ` Andy Lutomirski
2020-05-19 19:58 ` [tip: x86/entry] " tip-bot2 for Thomas Gleixner
2020-05-05 13:49 ` [patch V4 part 4 17/24] x86/entry/64: Remove error code clearing from #DB and #MCE ASM stub Thomas Gleixner
2020-05-15 5:27 ` Andy Lutomirski
2020-05-19 19:58 ` [tip: x86/entry] " tip-bot2 for Thomas Gleixner
2020-05-05 13:49 ` [patch V4 part 4 18/24] x86/entry: Provide IDTRENTRY_NOIST variants for #DB and #MC Thomas Gleixner
2020-05-15 5:29 ` Andy Lutomirski
2020-05-19 19:58 ` [tip: x86/entry] x86/idtentry: " tip-bot2 for Thomas Gleixner
2020-05-05 13:49 ` [patch V4 part 4 19/24] x86/entry: Implement user mode C entry points for #DB and #MCE Thomas Gleixner
2020-05-15 5:32 ` Andy Lutomirski
2020-05-19 19:58 ` [tip: x86/entry] " tip-bot2 for Thomas Gleixner
2020-05-05 13:49 ` [patch V4 part 4 20/24] x86/traps: Restructure #DB handling Thomas Gleixner
2020-05-15 5:39 ` Andy Lutomirski
2020-05-19 19:58 ` [tip: x86/entry] " tip-bot2 for Thomas Gleixner
2020-05-05 13:49 ` [patch V4 part 4 21/24] x86/traps: Address objtool noinstr complaints in #DB Thomas Gleixner
2020-05-15 5:39 ` Andy Lutomirski
2020-05-19 19:58 ` [tip: x86/entry] " tip-bot2 for Thomas Gleixner
2020-05-05 13:49 ` [patch V4 part 4 22/24] x86/mce: Address objtools noinstr complaints Thomas Gleixner
2020-05-15 5:40 ` Andy Lutomirski
2020-05-19 19:58 ` [tip: x86/entry] " tip-bot2 for Thomas Gleixner
2020-05-05 13:49 ` [patch V4 part 4 23/24] x86/entry: Provide IDTENTRY_DF Thomas Gleixner
2020-05-15 5:41 ` Andy Lutomirski
2020-05-15 15:01 ` Thomas Gleixner
2020-05-19 19:58 ` [tip: x86/entry] x86/idtentry: " tip-bot2 for Thomas Gleixner
2020-05-19 19:58 ` [tip: x86/entry] x86/entry: Convert double fault exception to IDTENTRY_DF tip-bot2 for Thomas Gleixner
2020-05-05 13:49 ` [patch V4 part 4 24/24] " Thomas Gleixner
2020-05-07 19:55 ` Alexandre Chartre
2020-05-15 5:42 ` Andy Lutomirski
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20200505135314.716186134@linutronix.de \
--to=tglx@linutronix.de \
--cc=alexandre.chartre@oracle.com \
--cc=boris.ostrovsky@oracle.com \
--cc=brgerst@gmail.com \
--cc=frederic@kernel.org \
--cc=jgross@suse.com \
--cc=joel@joelfernandes.org \
--cc=jpoimboe@redhat.com \
--cc=linux-kernel@vger.kernel.org \
--cc=luto@kernel.org \
--cc=mathieu.desnoyers@efficios.com \
--cc=mhiramat@kernel.org \
--cc=paulmck@kernel.org \
--cc=pbonzini@redhat.com \
--cc=pmladek@suse.com \
--cc=rostedt@goodmis.org \
--cc=sean.j.christopherson@intel.com \
--cc=will@kernel.org \
--cc=x86@kernel.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).