From: Peter Zijlstra <peterz@infradead.org>
To: x86@kernel.org, joao@overdrivepizza.com, hjl.tools@gmail.com,
jpoimboe@redhat.com, andrew.cooper3@citrix.com
Cc: linux-kernel@vger.kernel.org, peterz@infradead.org,
ndesaulniers@google.com, keescook@chromium.org,
samitolvanen@google.com, mark.rutland@arm.com,
alyssa.milburn@intel.com, mbenes@suse.cz, rostedt@goodmis.org,
mhiramat@kernel.org, alexei.starovoitov@gmail.com,
Andrew Cooper <Andrew.Cooper3@citrix.com>
Subject: [PATCH v2 27/39] x86/entry,xen: Early rewrite of restore_regs_and_return_to_kernel()
Date: Thu, 24 Feb 2022 15:52:05 +0100 [thread overview]
Message-ID: <20220224151323.601401399@infradead.org> (raw)
In-Reply-To: 20220224145138.952963315@infradead.org
By doing an early rewrite of 'jmp native_iret` in
restore_regs_and_return_to_kernel() we can get rid of the last
INTERRUPT_RETURN user and paravirt_iret.
Suggested-by: Andrew Cooper <Andrew.Cooper3@citrix.com>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
---
arch/x86/entry/entry_64.S | 12 ++++++++----
arch/x86/include/asm/irqflags.h | 5 -----
arch/x86/include/asm/paravirt_types.h | 1 -
arch/x86/kernel/head_64.S | 3 ++-
arch/x86/kernel/paravirt.c | 4 ----
arch/x86/xen/enlighten_pv.c | 7 ++++++-
arch/x86/xen/xen-asm.S | 1 +
7 files changed, 17 insertions(+), 16 deletions(-)
--- a/arch/x86/entry/entry_64.S
+++ b/arch/x86/entry/entry_64.S
@@ -620,7 +620,7 @@ SYM_INNER_LABEL(swapgs_restore_regs_and_
/* Restore RDI. */
popq %rdi
swapgs
- jmp native_iret
+ jmp .Lnative_iret
SYM_INNER_LABEL(restore_regs_and_return_to_kernel, SYM_L_GLOBAL)
@@ -637,11 +637,15 @@ SYM_INNER_LABEL(restore_regs_and_return_
* ARCH_HAS_MEMBARRIER_SYNC_CORE rely on IRET core serialization
* when returning from IPI handler.
*/
- INTERRUPT_RETURN
+#ifdef CONFIG_XEN_PV
+SYM_INNER_LABEL(early_xen_iret_patch, SYM_L_GLOBAL)
+ ANNOTATE_NOENDBR
+ .byte 0xe9
+ .long .Lnative_iret - (. + 4)
+#endif
-SYM_INNER_LABEL_ALIGN(native_iret, SYM_L_GLOBAL)
+.Lnative_iret:
UNWIND_HINT_IRET_REGS
- ENDBR // paravirt_iret
/*
* Are we returning to a stack segment from the LDT? Note: in
* 64-bit mode SS:RSP on the exception stack is always valid.
--- a/arch/x86/include/asm/irqflags.h
+++ b/arch/x86/include/asm/irqflags.h
@@ -141,13 +141,8 @@ static __always_inline void arch_local_i
#ifdef CONFIG_X86_64
#ifdef CONFIG_XEN_PV
#define SWAPGS ALTERNATIVE "swapgs", "", X86_FEATURE_XENPV
-#define INTERRUPT_RETURN \
- ANNOTATE_RETPOLINE_SAFE; \
- ALTERNATIVE_TERNARY("jmp *paravirt_iret(%rip);", \
- X86_FEATURE_XENPV, "jmp xen_iret;", "jmp native_iret;")
#else
#define SWAPGS swapgs
-#define INTERRUPT_RETURN jmp native_iret
#endif
#endif
#endif /* !__ASSEMBLY__ */
--- a/arch/x86/include/asm/paravirt_types.h
+++ b/arch/x86/include/asm/paravirt_types.h
@@ -272,7 +272,6 @@ struct paravirt_patch_template {
extern struct pv_info pv_info;
extern struct paravirt_patch_template pv_ops;
-extern void (*paravirt_iret)(void);
#define PARAVIRT_PATCH(x) \
(offsetof(struct paravirt_patch_template, x) / sizeof(void *))
--- a/arch/x86/kernel/head_64.S
+++ b/arch/x86/kernel/head_64.S
@@ -350,7 +350,6 @@ SYM_CODE_START_NOALIGN(vc_boot_ghcb)
/* Remove Error Code */
addq $8, %rsp
- /* Pure iret required here - don't use INTERRUPT_RETURN */
iretq
SYM_CODE_END(vc_boot_ghcb)
#endif
@@ -434,6 +433,8 @@ SYM_CODE_END(early_idt_handler_common)
* early_idt_handler_array can't be used because it returns via the
* paravirtualized INTERRUPT_RETURN and pv-ops don't work that early.
*
+ * XXX it does, fix this.
+ *
* This handler will end up in the .init.text section and not be
* available to boot secondary CPUs.
*/
--- a/arch/x86/kernel/paravirt.c
+++ b/arch/x86/kernel/paravirt.c
@@ -134,8 +134,6 @@ void paravirt_set_sched_clock(u64 (*func
}
/* These are in entry.S */
-extern void native_iret(void);
-
static struct resource reserve_ioports = {
.start = 0,
.end = IO_SPACE_LIMIT,
@@ -399,8 +397,6 @@ struct paravirt_patch_template pv_ops =
#ifdef CONFIG_PARAVIRT_XXL
NOKPROBE_SYMBOL(native_load_idt);
-
-void (*paravirt_iret)(void) = native_iret;
#endif
EXPORT_SYMBOL(pv_ops);
--- a/arch/x86/xen/enlighten_pv.c
+++ b/arch/x86/xen/enlighten_pv.c
@@ -1177,6 +1177,8 @@ static void __init xen_domu_set_legacy_f
x86_platform.legacy.rtc = 0;
}
+extern void early_xen_iret_patch(void);
+
/* First C function to be called on Xen boot */
asmlinkage __visible void __init xen_start_kernel(void)
{
@@ -1187,6 +1189,10 @@ asmlinkage __visible void __init xen_sta
if (!xen_start_info)
return;
+ __text_gen_insn(&early_xen_iret_patch,
+ JMP32_INSN_OPCODE, &early_xen_iret_patch, &xen_iret,
+ JMP32_INSN_SIZE);
+
xen_domain_type = XEN_PV_DOMAIN;
xen_start_flags = xen_start_info->flags;
@@ -1195,7 +1201,6 @@ asmlinkage __visible void __init xen_sta
/* Install Xen paravirt ops */
pv_info = xen_info;
pv_ops.cpu = xen_cpu_ops.cpu;
- paravirt_iret = xen_iret;
xen_init_irq_ops();
/*
--- a/arch/x86/xen/xen-asm.S
+++ b/arch/x86/xen/xen-asm.S
@@ -189,6 +189,7 @@ hypercall_iret = hypercall_page + __HYPE
*/
SYM_CODE_START(xen_iret)
UNWIND_HINT_EMPTY
+ ANNOTATE_NOENDBR
pushq $0
jmp hypercall_iret
SYM_CODE_END(xen_iret)
next prev parent reply other threads:[~2022-02-24 15:19 UTC|newest]
Thread overview: 183+ messages / expand[flat|nested] mbox.gz Atom feed top
2022-02-24 14:51 [PATCH v2 00/39] x86: Kernel IBT Peter Zijlstra
2022-02-24 14:51 ` [PATCH v2 01/39] kbuild: Fix clang build Peter Zijlstra
2022-02-25 0:11 ` Kees Cook
2022-03-01 21:16 ` Nick Desaulniers
2022-03-02 0:47 ` Kees Cook
2022-03-02 0:53 ` Fangrui Song
2022-03-02 16:37 ` Nathan Chancellor
2022-03-02 18:40 ` Kees Cook
2022-03-02 19:18 ` Nick Desaulniers
2022-03-02 21:15 ` Nathan Chancellor
2022-03-02 22:07 ` Nick Desaulniers
2022-03-02 23:00 ` Kees Cook
2022-03-02 23:10 ` Peter Zijlstra
2022-02-24 14:51 ` [PATCH v2 02/39] static_call: Avoid building empty .static_call_sites Peter Zijlstra
2022-02-24 14:51 ` [PATCH v2 03/39] x86/module: Fix the paravirt vs alternative order Peter Zijlstra
2022-03-01 14:37 ` Miroslav Benes
2022-02-24 14:51 ` [PATCH v2 04/39] objtool: Add --dry-run Peter Zijlstra
2022-02-25 0:27 ` Kees Cook
2022-03-01 14:37 ` Miroslav Benes
2022-02-24 14:51 ` [PATCH v2 05/39] x86: Base IBT bits Peter Zijlstra
2022-02-25 0:35 ` Kees Cook
2022-02-25 0:46 ` Nathan Chancellor
2022-02-25 22:08 ` Nathan Chancellor
2022-02-26 0:29 ` Joao Moreira
2022-02-26 4:58 ` Kees Cook
2022-02-26 4:59 ` Fāng-ruì Sòng
2022-02-26 5:04 ` Kees Cook
2022-02-25 13:41 ` Peter Zijlstra
2022-02-24 14:51 ` [PATCH v2 06/39] x86/ibt: Add ANNOTATE_NOENDBR Peter Zijlstra
2022-02-25 0:36 ` Kees Cook
2022-02-24 14:51 ` [PATCH v2 07/39] x86/entry: Sprinkle ENDBR dust Peter Zijlstra
2022-02-24 22:37 ` Josh Poimboeuf
2022-02-25 0:42 ` Kees Cook
2022-02-25 9:22 ` Andrew Cooper
2022-02-24 14:51 ` [PATCH v2 08/39] x86/linkage: Add ENDBR to SYM_FUNC_START*() Peter Zijlstra
2022-02-25 0:45 ` Kees Cook
2022-02-24 14:51 ` [PATCH v2 09/39] x86/ibt,paravirt: Sprinkle ENDBR Peter Zijlstra
2022-02-25 0:47 ` Kees Cook
2022-02-24 14:51 ` [PATCH v2 10/39] x86/ibt,crypto: Add ENDBR for the jump-table entries Peter Zijlstra
2022-02-24 22:41 ` Josh Poimboeuf
2022-02-25 0:50 ` Kees Cook
2022-02-25 10:22 ` Peter Zijlstra
2022-02-24 14:51 ` [PATCH v2 11/39] x86/ibt,kvm: Add ENDBR to fastops Peter Zijlstra
2022-02-25 0:54 ` Kees Cook
2022-02-25 10:24 ` Peter Zijlstra
2022-02-25 13:09 ` David Laight
2022-02-24 14:51 ` [PATCH v2 12/39] x86/ibt,ftrace: Search for __fentry__ location Peter Zijlstra
2022-02-24 15:55 ` Masami Hiramatsu
2022-02-24 15:58 ` Steven Rostedt
2022-02-24 15:59 ` Steven Rostedt
2022-02-24 16:01 ` Steven Rostedt
2022-02-24 22:46 ` Josh Poimboeuf
2022-02-24 22:51 ` Steven Rostedt
2022-02-25 1:34 ` Masami Hiramatsu
2022-02-25 2:19 ` Steven Rostedt
2022-02-25 10:20 ` Masami Hiramatsu
2022-02-25 13:36 ` Steven Rostedt
2022-03-01 18:57 ` Naveen N. Rao
2022-03-01 19:20 ` Steven Rostedt
2022-03-02 13:20 ` Peter Zijlstra
2022-03-02 16:01 ` Steven Rostedt
2022-03-02 19:47 ` Steven Rostedt
2022-03-02 20:48 ` Steven Rostedt
2022-03-02 20:51 ` Peter Zijlstra
2022-03-03 9:45 ` Naveen N. Rao
2022-03-03 13:04 ` Peter Zijlstra
2022-03-03 14:34 ` Steven Rostedt
2022-03-03 15:59 ` Peter Zijlstra
2022-03-06 3:48 ` Masami Hiramatsu
2022-03-09 11:47 ` Naveen N. Rao
2022-03-03 14:39 ` Naveen N. Rao
2022-02-25 0:55 ` Kees Cook
2022-03-02 16:25 ` Naveen N. Rao
2022-02-24 14:51 ` [PATCH v2 13/39] x86/livepatch: Validate " Peter Zijlstra
2022-02-24 23:02 ` Josh Poimboeuf
2022-02-24 14:51 ` [PATCH v2 14/39] x86/ibt,ftrace: Make function-graph play nice Peter Zijlstra
2022-02-24 15:36 ` Peter Zijlstra
2022-02-24 15:42 ` Steven Rostedt
2022-02-24 23:09 ` Peter Zijlstra
2022-02-24 14:51 ` [PATCH v2 15/39] x86/ibt,kprobes: Fix more +0 assumptions Peter Zijlstra
2022-02-25 0:58 ` Kees Cook
2022-02-25 1:32 ` Masami Hiramatsu
2022-02-25 10:46 ` Peter Zijlstra
2022-02-25 13:42 ` Masami Hiramatsu
2022-02-25 15:41 ` Peter Zijlstra
2022-02-26 2:10 ` Masami Hiramatsu
2022-02-26 11:48 ` Peter Zijlstra
2022-02-25 14:14 ` Steven Rostedt
2022-02-26 7:09 ` Masami Hiramatsu
2022-02-28 6:07 ` Masami Hiramatsu
2022-02-28 23:25 ` Peter Zijlstra
2022-03-01 2:49 ` Masami Hiramatsu
2022-03-01 8:28 ` Peter Zijlstra
2022-03-01 17:19 ` Naveen N. Rao
2022-03-01 19:12 ` Peter Zijlstra
2022-03-01 20:05 ` Peter Zijlstra
2022-03-02 15:59 ` Naveen N. Rao
2022-03-02 16:38 ` Peter Zijlstra
2022-03-02 16:17 ` Naveen N. Rao
2022-03-02 19:32 ` Peter Zijlstra
2022-03-02 19:39 ` Peter Zijlstra
2022-03-03 12:11 ` Naveen N. Rao
2022-03-03 1:54 ` Masami Hiramatsu
2022-03-02 0:11 ` Masami Hiramatsu
2022-03-02 10:25 ` Peter Zijlstra
2022-03-01 17:03 ` Naveen N. Rao
2022-02-24 14:51 ` [PATCH v2 16/39] x86/bpf: Add ENDBR instructions to prologue and trampoline Peter Zijlstra
2022-02-24 23:37 ` Josh Poimboeuf
2022-02-25 0:59 ` Kees Cook
2022-02-25 11:20 ` Peter Zijlstra
2022-02-25 12:24 ` Peter Zijlstra
2022-02-25 22:46 ` Josh Poimboeuf
2022-02-24 14:51 ` [PATCH v2 17/39] x86/ibt,ftrace: Add ENDBR to samples/ftrace Peter Zijlstra
2022-02-24 14:51 ` [PATCH v2 18/39] x86/ibt: Add IBT feature, MSR and #CP handling Peter Zijlstra
2022-02-24 23:55 ` Josh Poimboeuf
2022-02-25 10:51 ` Peter Zijlstra
2022-02-25 11:10 ` Peter Zijlstra
2022-02-25 23:51 ` Josh Poimboeuf
2022-02-26 11:55 ` Peter Zijlstra
2022-02-25 1:09 ` Kees Cook
2022-02-25 19:59 ` Edgecombe, Rick P
2022-03-01 15:14 ` Peter Zijlstra
2022-03-01 21:02 ` Peter Zijlstra
2022-03-01 23:13 ` Josh Poimboeuf
2022-03-02 1:59 ` Edgecombe, Rick P
2022-03-02 13:49 ` Peter Zijlstra
2022-03-02 18:38 ` Kees Cook
2022-02-24 14:51 ` [PATCH v2 19/39] x86: Disable IBT around firmware Peter Zijlstra
2022-02-25 1:10 ` Kees Cook
2022-02-24 14:51 ` [PATCH v2 20/39] x86/bugs: Disable Retpoline when IBT Peter Zijlstra
2022-02-25 1:11 ` Kees Cook
2022-02-25 2:22 ` Josh Poimboeuf
2022-02-25 10:55 ` Peter Zijlstra
2022-02-24 14:51 ` [PATCH v2 21/39] x86/ibt: Annotate text references Peter Zijlstra
2022-02-25 0:47 ` Josh Poimboeuf
2022-02-25 12:57 ` Peter Zijlstra
2022-02-25 13:04 ` Peter Zijlstra
2022-02-24 14:52 ` [PATCH v2 22/39] x86/ibt,ftrace: Annotate ftrace code patching Peter Zijlstra
2022-02-24 14:52 ` [PATCH v2 23/39] x86/ibt,sev: Annotations Peter Zijlstra
2022-02-24 14:52 ` [PATCH v2 24/39] x86/text-patching: Make text_gen_insn() IBT aware Peter Zijlstra
2022-02-25 0:49 ` Josh Poimboeuf
2022-02-24 14:52 ` [PATCH v2 25/39] x86/ibt,paravirt: Use text_gen_insn() for paravirt_patch() Peter Zijlstra
2022-02-24 14:52 ` [PATCH v2 26/39] x86/entry: Cleanup PARAVIRT Peter Zijlstra
2022-02-24 14:52 ` Peter Zijlstra [this message]
2022-02-24 17:51 ` [PATCH v2 27/39] x86/entry,xen: Early rewrite of restore_regs_and_return_to_kernel() Andrew Cooper
2022-02-24 14:52 ` [PATCH v2 28/39] x86/ibt,xen: Sprinkle the ENDBR Peter Zijlstra
2022-02-25 0:54 ` Josh Poimboeuf
2022-02-25 13:16 ` Peter Zijlstra
2022-02-24 14:52 ` [PATCH v2 29/39] objtool: Rename --duplicate to --lto Peter Zijlstra
2022-02-24 14:52 ` [PATCH v2 30/39] Kbuild: Allow whole module objtool runs Peter Zijlstra
2022-02-24 14:52 ` [PATCH v2 31/39] objtool: Read the NOENDBR annotation Peter Zijlstra
2022-02-24 14:52 ` [PATCH v2 32/39] x86/ibt: Dont generate ENDBR in .discard.text Peter Zijlstra
2022-02-24 14:52 ` [PATCH v2 33/39] objtool: Add IBT/ENDBR decoding Peter Zijlstra
2022-03-03 10:53 ` Miroslav Benes
2022-03-03 11:06 ` Andrew Cooper
2022-03-03 12:33 ` Miroslav Benes
2022-03-03 14:13 ` Peter Zijlstra
2022-02-24 14:52 ` [PATCH v2 34/39] objtool: Validate IBT assumptions Peter Zijlstra
2022-02-27 3:13 ` Josh Poimboeuf
2022-02-27 17:00 ` Peter Zijlstra
2022-02-27 22:20 ` Josh Poimboeuf
2022-02-28 9:47 ` Peter Zijlstra
2022-02-28 18:36 ` Josh Poimboeuf
2022-02-28 20:10 ` Peter Zijlstra
2022-02-28 9:26 ` Peter Zijlstra
2022-02-28 18:39 ` Josh Poimboeuf
2022-02-24 14:52 ` [PATCH v2 35/39] objtool: IBT fix direct JMP/CALL Peter Zijlstra
2022-02-24 14:52 ` [PATCH v2 36/39] objtool: Find unused ENDBR instructions Peter Zijlstra
2022-02-27 3:46 ` Josh Poimboeuf
2022-02-28 12:41 ` Peter Zijlstra
2022-02-28 17:36 ` Josh Poimboeuf
2022-02-24 14:52 ` [PATCH v2 37/39] x86/ibt: Finish --ibt-fix-direct on module loading Peter Zijlstra
2022-02-24 14:52 ` [PATCH v2 38/39] x86/ibt: Ensure module init/exit points have references Peter Zijlstra
2022-02-24 14:52 ` [PATCH v2 39/39] x86/alternative: Use .ibt_endbr_sites to seal indirect calls Peter Zijlstra
2022-02-24 20:26 ` [PATCH v2 00/39] x86: Kernel IBT Josh Poimboeuf
2022-02-25 15:28 ` Peter Zijlstra
2022-02-25 15:43 ` Peter Zijlstra
2022-02-25 17:26 ` Josh Poimboeuf
2022-02-25 17:32 ` Steven Rostedt
2022-02-25 19:53 ` Peter Zijlstra
2022-02-25 20:15 ` Josh Poimboeuf
2022-03-01 23:10 ` Josh Poimboeuf
2022-03-02 10:20 ` Peter Zijlstra
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20220224151323.601401399@infradead.org \
--to=peterz@infradead.org \
--cc=alexei.starovoitov@gmail.com \
--cc=alyssa.milburn@intel.com \
--cc=andrew.cooper3@citrix.com \
--cc=hjl.tools@gmail.com \
--cc=joao@overdrivepizza.com \
--cc=jpoimboe@redhat.com \
--cc=keescook@chromium.org \
--cc=linux-kernel@vger.kernel.org \
--cc=mark.rutland@arm.com \
--cc=mbenes@suse.cz \
--cc=mhiramat@kernel.org \
--cc=ndesaulniers@google.com \
--cc=rostedt@goodmis.org \
--cc=samitolvanen@google.com \
--cc=x86@kernel.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.