From: Peter Zijlstra <peterz@infradead.org>
To: tglx@linutronix.de, bp@alien8.de, mingo@kernel.org,
rostedt@goodmis.org, luto@kernel.org,
torvalds@linux-foundation.org
Cc: hpa@zytor.com, dave.hansen@linux.intel.com, jgross@suse.com,
linux-kernel@vger.kernel.org, zhe.he@windriver.com,
joel@joelfernandes.org, devel@etsukata.com, peterz@infradead.org
Subject: [RFC][PATCH v2 7/7] x86/entry/64: Pull bits into C
Date: Thu, 04 Jul 2019 21:56:02 +0200 [thread overview]
Message-ID: <20190704200050.648944551@infradead.org> (raw)
In-Reply-To: 20190704195555.580363209@infradead.org
TODO: 32bit, Xen
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
---
arch/x86/entry/entry_64.S | 13 +++-------
arch/x86/include/asm/idtentry.h | 49 ++++++++++++++++++++++++++++++++++++++++
arch/x86/include/asm/traps.h | 1
arch/x86/kernel/cpu/mce/core.c | 2 -
arch/x86/kernel/kvm.c | 4 ---
arch/x86/kernel/traps.c | 30 ++++++++----------------
arch/x86/mm/fault.c | 4 ---
7 files changed, 68 insertions(+), 35 deletions(-)
--- a/arch/x86/entry/entry_64.S
+++ b/arch/x86/entry/entry_64.S
@@ -841,15 +841,6 @@ apicinterrupt IRQ_WORK_VECTOR irq_work
GET_CR2_INTO(%rdx); /* can clobber %rax */
.endif
- TRACE_IRQS_OFF
-
- .if \paranoid == 0
- testb $3, CS(%rsp)
- jz .Lfrom_kernel_no_context_tracking_\@
- CALL_enter_from_user_mode
-.Lfrom_kernel_no_context_tracking_\@:
- .endif
-
movq %rsp, %rdi /* pt_regs pointer */
.if \has_error_code
@@ -863,7 +854,11 @@ apicinterrupt IRQ_WORK_VECTOR irq_work
subq $\ist_offset, CPU_TSS_IST(\shift_ist)
.endif
+ .if \paranoid
+ call \do_sym\()_paranoid
+ .else
call \do_sym
+ .endif
.if \shift_ist != -1
addq $\ist_offset, CPU_TSS_IST(\shift_ist)
--- /dev/null
+++ b/arch/x86/include/asm/idtentry.h
@@ -0,0 +1,49 @@
+#ifndef __ASM_IDTENTRY_H
+#define __ASM_IDTENTRY_H
+
+/* shamelessly stolen from linux/syscalls.h; XXX share */
+
+#define __IDT_MAP0(m,...)
+#define __IDT_MAP1(m,t,a,...) m(t,a)
+#define __IDT_MAP2(m,t,a,...) m(t,a), __IDT_MAP1(m,__VA_ARGS__)
+#define __IDT_MAP3(m,t,a,...) m(t,a), __IDT_MAP2(m,__VA_ARGS__)
+
+#define __IDT_MAP(n,...) __IDT_MAP##n(__VA_ARGS__)
+
+#define __IDT_DECL(t, a) t a
+#define __IDT_ARGS(t, a) a
+#define __IDT_TEST(t, a) (void)BUILD_BUG_ON_ZERO(sizeof(t) != sizeof(long))
+
+#ifdef CONFIG_CONTEXT_TRACKING
+#define CALL_enter_from_user_mode(_regs) \
+ if (static_branch_unlikely(&context_tracking_enabled) && user_mode(_regs)) \
+ enter_from_user_mode()
+#else
+#define CALL_enter_from_user_mode(_regs)
+#endif
+
+#define IDTENTRYx(n, name, ...) \
+ static notrace void __idt_##name(__IDT_MAP(n, __IDT_DECL, __VA_ARGS__)); \
+ NOKPROBE_SYMBOL(__idt_##name); \
+ dotraplinkage notrace void name(__IDT_MAP(n, __IDT_DECL, __VA_ARGS__)) \
+ { \
+ __IDT_MAP(n, __IDT_TEST, __VA_ARGS__); \
+ trace_hardirqs_off(); \
+ CALL_enter_from_user_mode(regs); \
+ __idt_##name(__IDT_MAP(n, __IDT_ARGS, __VA_ARGS__)); \
+ } \
+ NOKPROBE_SYMBOL(name); \
+ dotraplinkage notrace void name##_paranoid(__IDT_MAP(n, __IDT_DECL, __VA_ARGS__)) \
+ { \
+ __IDT_MAP(n, __IDT_TEST, __VA_ARGS__); \
+ trace_hardirqs_off(); \
+ __idt_##name(__IDT_MAP(n, __IDT_ARGS, __VA_ARGS__)); \
+ } \
+ NOKPROBE_SYMBOL(name##_paranoid); \
+ static notrace void __idt_##name(__IDT_MAP(n, __IDT_DECL, __VA_ARGS__))
+
+#define IDTENTRY1(name,...) IDTENTRYx(1, name, __VA_ARGS__)
+#define IDTENTRY2(name,...) IDTENTRYx(2, name, __VA_ARGS__)
+#define IDTENTRY3(name,...) IDTENTRYx(3, name, __VA_ARGS__)
+
+#endif /* __ASM_IDTENTRY_H */
--- a/arch/x86/include/asm/traps.h
+++ b/arch/x86/include/asm/traps.h
@@ -7,6 +7,7 @@
#include <asm/debugreg.h>
#include <asm/siginfo.h> /* TRAP_TRACE, ... */
+#include <asm/idtentry.h>
#define dotraplinkage __visible
--- a/arch/x86/kernel/cpu/mce/core.c
+++ b/arch/x86/kernel/cpu/mce/core.c
@@ -1842,7 +1842,7 @@ static void unexpected_machine_check(str
void (*machine_check_vector)(struct pt_regs *, long error_code) =
unexpected_machine_check;
-dotraplinkage void do_mce(struct pt_regs *regs, long error_code)
+IDTENTRY2(do_mce, struct pt_regs *, regs, long, error_code)
{
machine_check_vector(regs, error_code);
}
--- a/arch/x86/kernel/kvm.c
+++ b/arch/x86/kernel/kvm.c
@@ -241,8 +241,7 @@ u32 kvm_read_and_reset_pf_reason(void)
EXPORT_SYMBOL_GPL(kvm_read_and_reset_pf_reason);
NOKPROBE_SYMBOL(kvm_read_and_reset_pf_reason);
-dotraplinkage void
-do_async_page_fault(struct pt_regs *regs, unsigned long error_code, unsigned long address)
+IDTENTRY3(do_async_page_fault, struct pt_regs *, regs, unsigned long, error_code, unsigned long, address)
{
enum ctx_state prev_state;
@@ -263,7 +262,6 @@ do_async_page_fault(struct pt_regs *regs
break;
}
}
-NOKPROBE_SYMBOL(do_async_page_fault);
static void __init paravirt_ops_setup(void)
{
--- a/arch/x86/kernel/traps.c
+++ b/arch/x86/kernel/traps.c
@@ -281,7 +281,7 @@ static void do_error_trap(struct pt_regs
#define IP ((void __user *)uprobe_get_trap_addr(regs))
#define DO_ERROR(trapnr, signr, sicode, addr, str, name) \
-dotraplinkage void do_##name(struct pt_regs *regs, long error_code) \
+IDTENTRY2(do_##name, struct pt_regs *, regs, long, error_code) \
{ \
do_error_trap(regs, error_code, str, trapnr, signr, sicode, addr); \
}
@@ -313,7 +313,7 @@ __visible void __noreturn handle_stack_o
#ifdef CONFIG_X86_64
/* Runs on IST stack */
-dotraplinkage void do_double_fault(struct pt_regs *regs, long error_code, unsigned long cr2)
+IDTENTRY3(do_double_fault, struct pt_regs *, regs, long, error_code, unsigned long, cr2)
{
static const char str[] = "double fault";
struct task_struct *tsk = current;
@@ -428,7 +428,7 @@ dotraplinkage void do_double_fault(struc
}
#endif
-dotraplinkage void do_bounds(struct pt_regs *regs, long error_code)
+IDTENTRY2(do_bounds, struct pt_regs *, regs, long, error_code)
{
const struct mpx_bndcsr *bndcsr;
@@ -514,8 +514,7 @@ dotraplinkage void do_bounds(struct pt_r
do_trap(X86_TRAP_BR, SIGSEGV, "bounds", regs, error_code, 0, NULL);
}
-dotraplinkage void
-do_general_protection(struct pt_regs *regs, long error_code)
+IDTENTRY2(do_general_protection, struct pt_regs *, regs, long, error_code)
{
const char *desc = "general protection fault";
struct task_struct *tsk;
@@ -564,9 +563,8 @@ do_general_protection(struct pt_regs *re
force_sig(SIGSEGV, tsk);
}
-NOKPROBE_SYMBOL(do_general_protection);
-dotraplinkage void notrace do_int3(struct pt_regs *regs, long error_code)
+IDTENTRY2(do_int3, struct pt_regs *, regs, long, error_code)
{
#ifdef CONFIG_DYNAMIC_FTRACE
/*
@@ -611,7 +609,6 @@ dotraplinkage void notrace do_int3(struc
exit:
ist_exit(regs);
}
-NOKPROBE_SYMBOL(do_int3);
#ifdef CONFIG_X86_64
/*
@@ -706,7 +703,7 @@ static bool is_sysenter_singlestep(struc
*
* May run on IST stack.
*/
-dotraplinkage void do_debug(struct pt_regs *regs, long error_code)
+IDTENTRY2(do_debug, struct pt_regs *, regs, long, error_code)
{
struct task_struct *tsk = current;
int user_icebp = 0;
@@ -808,7 +805,6 @@ dotraplinkage void do_debug(struct pt_re
exit:
ist_exit(regs);
}
-NOKPROBE_SYMBOL(do_debug);
/*
* Note that we play around with the 'TS' bit in an attempt to get
@@ -855,27 +851,24 @@ static void math_error(struct pt_regs *r
(void __user *)uprobe_get_trap_addr(regs), task);
}
-dotraplinkage void do_coprocessor_error(struct pt_regs *regs, long error_code)
+IDTENTRY2(do_coprocessor_error, struct pt_regs *, regs, long, error_code)
{
RCU_LOCKDEP_WARN(!rcu_is_watching(), "entry code didn't wake RCU");
math_error(regs, error_code, X86_TRAP_MF);
}
-dotraplinkage void
-do_simd_coprocessor_error(struct pt_regs *regs, long error_code)
+IDTENTRY2(do_simd_coprocessor_error, struct pt_regs *, regs, long, error_code)
{
RCU_LOCKDEP_WARN(!rcu_is_watching(), "entry code didn't wake RCU");
math_error(regs, error_code, X86_TRAP_XF);
}
-dotraplinkage void
-do_spurious_interrupt_bug(struct pt_regs *regs, long error_code)
+IDTENTRY2(do_spurious_interrupt_bug, struct pt_regs *, regs, long, error_code)
{
cond_local_irq_enable(regs);
}
-dotraplinkage void
-do_device_not_available(struct pt_regs *regs, long error_code)
+IDTENTRY2(do_device_not_available, struct pt_regs *, regs, long, error_code)
{
unsigned long cr0 = read_cr0();
@@ -906,10 +899,9 @@ do_device_not_available(struct pt_regs *
die("unexpected #NM exception", regs, error_code);
}
}
-NOKPROBE_SYMBOL(do_device_not_available);
#ifdef CONFIG_X86_32
-dotraplinkage void do_iret_error(struct pt_regs *regs, long error_code)
+IDTENTRY2(do_iret_error, struct pt_regs *, regs, long, error_code)
{
RCU_LOCKDEP_WARN(!rcu_is_watching(), "entry code didn't wake RCU");
local_irq_enable();
--- a/arch/x86/mm/fault.c
+++ b/arch/x86/mm/fault.c
@@ -1542,8 +1542,7 @@ trace_page_fault_entries(struct pt_regs
trace_page_fault_kernel(address, regs, error_code);
}
-dotraplinkage void
-do_page_fault(struct pt_regs *regs, unsigned long error_code, unsigned long address)
+IDTENTRY3(do_page_fault, struct pt_regs *, regs, unsigned long, error_code, unsigned long, address)
{
enum ctx_state prev_state;
@@ -1552,4 +1551,3 @@ do_page_fault(struct pt_regs *regs, unsi
__do_page_fault(regs, error_code, address);
exception_exit(prev_state);
}
-NOKPROBE_SYMBOL(do_page_fault);
prev parent reply other threads:[~2019-07-04 20:03 UTC|newest]
Thread overview: 42+ messages / expand[flat|nested] mbox.gz Atom feed top
2019-07-04 19:55 [PATCH v2 0/7] Tracing vs CR2 (and cleanups) Peter Zijlstra
2019-07-04 19:55 ` [PATCH v2 1/7] x86/paravirt: Make read_cr2() CALLEE_SAVE Peter Zijlstra
2019-07-04 21:49 ` Andy Lutomirski
2019-07-10 19:53 ` Steven Rostedt
2019-07-04 19:55 ` [PATCH v2 2/7] x86/entry/32: Simplify common_exception Peter Zijlstra
2019-07-04 21:51 ` Andy Lutomirski
2019-07-10 20:11 ` Steven Rostedt
2019-07-10 20:14 ` Peter Zijlstra
2019-07-04 19:55 ` [PATCH v2 3/7] x86/entry/64: Simplify idtentry a little Peter Zijlstra
2019-07-04 21:54 ` Andy Lutomirski
2019-07-10 20:23 ` Steven Rostedt
2019-07-04 19:55 ` [PATCH v2 4/7] x86/entry/64: Update comments and sanity tests for create_gap Peter Zijlstra
2019-07-04 21:55 ` Andy Lutomirski
2019-07-10 20:24 ` Steven Rostedt
2019-07-04 19:56 ` [PATCH v2 5/7] x86/mm, tracing: Fix CR2 corruption Peter Zijlstra
2019-07-05 2:18 ` Linus Torvalds
2019-07-05 3:16 ` Andy Lutomirski
2019-07-05 3:27 ` Linus Torvalds
2019-07-05 13:49 ` Peter Zijlstra
2019-07-06 21:41 ` Linus Torvalds
2019-07-06 22:27 ` Steven Rostedt
2019-07-06 22:41 ` Linus Torvalds
2019-07-07 0:08 ` Linus Torvalds
2019-07-07 0:36 ` Andy Lutomirski
2019-07-06 23:50 ` Andy Lutomirski
2019-07-07 3:44 ` Eiichi Tsukata
2019-07-06 11:07 ` Eiichi Tsukata
2019-07-08 7:48 ` Peter Zijlstra
2019-07-08 8:58 ` Eiichi Tsukata
2019-07-08 9:42 ` Eiichi Tsukata
2019-07-09 5:17 ` Eiichi Tsukata
2019-07-07 15:10 ` Andy Lutomirski
2019-07-07 15:11 ` Andy Lutomirski
2019-07-07 18:17 ` Linus Torvalds
2019-07-10 20:27 ` Steven Rostedt
2019-07-11 6:45 ` Greg Kroah-Hartman
2019-07-11 12:12 ` Sasha Levin
2019-07-11 12:21 ` Peter Zijlstra
2019-07-04 19:56 ` [PATCH v2 6/7] x86/entry/64: Remove TRACE_IRQS_*_DEBUG Peter Zijlstra
2019-07-11 3:24 ` Steven Rostedt
2019-07-11 8:05 ` Peter Zijlstra
2019-07-04 19:56 ` Peter Zijlstra [this message]
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20190704200050.648944551@infradead.org \
--to=peterz@infradead.org \
--cc=bp@alien8.de \
--cc=dave.hansen@linux.intel.com \
--cc=devel@etsukata.com \
--cc=hpa@zytor.com \
--cc=jgross@suse.com \
--cc=joel@joelfernandes.org \
--cc=linux-kernel@vger.kernel.org \
--cc=luto@kernel.org \
--cc=mingo@kernel.org \
--cc=rostedt@goodmis.org \
--cc=tglx@linutronix.de \
--cc=torvalds@linux-foundation.org \
--cc=zhe.he@windriver.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).