From: Thomas Gleixner <tglx@linutronix.de> To: LKML <linux-kernel@vger.kernel.org> Cc: x86@kernel.org, Peter Zijlstra <peterz@infradead.org>, Andy Lutomirski <luto@kernel.org>, Linus Torvalds <torvalds@linux-foundation.org>, Jiri Kosina <jkosina@suse.cz>, Tom Lendacky <thomas.lendacky@amd.com>, Josh Poimboeuf <jpoimboe@redhat.com>, Andrea Arcangeli <aarcange@redhat.com>, David Woodhouse <dwmw@amazon.co.uk>, Tim Chen <tim.c.chen@linux.intel.com>, Andi Kleen <ak@linux.intel.com>, Dave Hansen <dave.hansen@intel.com>, Casey Schaufler <casey.schaufler@intel.com>, Asit Mallick <asit.k.mallick@intel.com>, Arjan van de Ven <arjan@linux.intel.com>, Jon Masters <jcm@redhat.com>, Waiman Long <longman9394@gmail.com>, Greg KH <gregkh@linuxfoundation.org>, Dave Stewart <david.c.stewart@intel.com>, Kees Cook <keescook@chromium.org> Subject: [patch V2 19/28] x86/process: Consolidate and simplify switch_to_xtra() code Date: Sun, 25 Nov 2018 19:33:47 +0100 Message-ID: <20181125185005.280855518@linutronix.de> (raw) In-Reply-To: <20181125183328.318175777@linutronix.de> [-- Attachment #0: x86-process--Consolidate-and-simplify-switch_to_xtra---code.patch --] [-- Type: text/plain, Size: 5635 bytes --] Move the conditional invocation of __switch_to_xtra() into an inline function so the logic can be shared between 32 and 64 bit. Remove the handthrough of the TSS pointer and retrieve the pointer directly in the bitmap handling function. Use this_cpu_ptr() instead of the per_cpu() indirection. This is a preparatory change so integration of conditional indirect branch speculation optimization happens only in one place. Signed-off-by: Thomas Gleixner <tglx@linutronix.de> --- arch/x86/include/asm/switch_to.h | 3 --- arch/x86/kernel/process.c | 12 +++++++----- arch/x86/kernel/process.h | 24 ++++++++++++++++++++++++ arch/x86/kernel/process_32.c | 10 +++------- arch/x86/kernel/process_64.c | 10 +++------- 5 files changed, 37 insertions(+), 22 deletions(-) --- a/arch/x86/include/asm/switch_to.h +++ b/arch/x86/include/asm/switch_to.h @@ -11,9 +11,6 @@ struct task_struct *__switch_to_asm(stru __visible struct task_struct *__switch_to(struct task_struct *prev, struct task_struct *next); -struct tss_struct; -void __switch_to_xtra(struct task_struct *prev_p, struct task_struct *next_p, - struct tss_struct *tss); /* This runs runs on the previous thread's stack. */ static inline void prepare_switch_to(struct task_struct *next) --- a/arch/x86/kernel/process.c +++ b/arch/x86/kernel/process.c @@ -40,6 +40,8 @@ #include <asm/prctl.h> #include <asm/spec-ctrl.h> +#include "process.h" + /* * per-CPU TSS segments. Threads are completely 'soft' on Linux, * no more per-task TSS's. The TSS size is kept cacheline-aligned @@ -252,11 +254,12 @@ void arch_setup_new_exec(void) enable_cpuid(); } -static inline void switch_to_bitmap(struct tss_struct *tss, - struct thread_struct *prev, +static inline void switch_to_bitmap(struct thread_struct *prev, struct thread_struct *next, unsigned long tifp, unsigned long tifn) { + struct tss_struct *tss = this_cpu_ptr(&cpu_tss_rw); + if (tifn & _TIF_IO_BITMAP) { /* * Copy the relevant range of the IO bitmap. @@ -461,8 +464,7 @@ void speculation_ctrl_update(unsigned lo preempt_enable(); } -void __switch_to_xtra(struct task_struct *prev_p, struct task_struct *next_p, - struct tss_struct *tss) +void __switch_to_xtra(struct task_struct *prev_p, struct task_struct *next_p) { struct thread_struct *prev, *next; unsigned long tifp, tifn; @@ -472,7 +474,7 @@ void __switch_to_xtra(struct task_struct tifn = READ_ONCE(task_thread_info(next_p)->flags); tifp = READ_ONCE(task_thread_info(prev_p)->flags); - switch_to_bitmap(tss, prev, next, tifp, tifn); + switch_to_bitmap(prev, next, tifp, tifn); propagate_user_return_notify(prev_p, next_p); --- /dev/null +++ b/arch/x86/kernel/process.h @@ -0,0 +1,24 @@ +// SPDX-License-Identifier: GPL-2.0 +// +// Code shared between 32 and 64 bit + +void __switch_to_xtra(struct task_struct *prev_p, struct task_struct *next_p); + +/* + * This needs to be inline to optimize for the common case where no extra + * work needs to be done. + */ +static inline void switch_to_extra(struct task_struct *prev, + struct task_struct *next) +{ + unsigned long next_tif = task_thread_info(next)->flags; + unsigned long prev_tif = task_thread_info(prev)->flags; + + /* + * __switch_to_xtra() handles debug registers, i/o bitmaps, + * speculation mitigations etc. + */ + if (unlikely(next_tif & _TIF_WORK_CTXSW_NEXT || + prev_tif & _TIF_WORK_CTXSW_PREV)) + __switch_to_xtra(prev, next); +} --- a/arch/x86/kernel/process_32.c +++ b/arch/x86/kernel/process_32.c @@ -59,6 +59,8 @@ #include <asm/intel_rdt_sched.h> #include <asm/proto.h> +#include "process.h" + void __show_regs(struct pt_regs *regs, enum show_regs_mode mode) { unsigned long cr0 = 0L, cr2 = 0L, cr3 = 0L, cr4 = 0L; @@ -232,7 +234,6 @@ EXPORT_SYMBOL_GPL(start_thread); struct fpu *prev_fpu = &prev->fpu; struct fpu *next_fpu = &next->fpu; int cpu = smp_processor_id(); - struct tss_struct *tss = &per_cpu(cpu_tss_rw, cpu); /* never put a printk in __switch_to... printk() calls wake_up*() indirectly */ @@ -264,12 +265,7 @@ EXPORT_SYMBOL_GPL(start_thread); if (get_kernel_rpl() && unlikely(prev->iopl != next->iopl)) set_iopl_mask(next->iopl); - /* - * Now maybe handle debug registers and/or IO bitmaps - */ - if (unlikely(task_thread_info(prev_p)->flags & _TIF_WORK_CTXSW_PREV || - task_thread_info(next_p)->flags & _TIF_WORK_CTXSW_NEXT)) - __switch_to_xtra(prev_p, next_p, tss); + switch_to_extra(prev_p, next_p); /* * Leave lazy mode, flushing any hypercalls made here. --- a/arch/x86/kernel/process_64.c +++ b/arch/x86/kernel/process_64.c @@ -60,6 +60,8 @@ #include <asm/unistd_32_ia32.h> #endif +#include "process.h" + /* Prints also some state that isn't saved in the pt_regs */ void __show_regs(struct pt_regs *regs, enum show_regs_mode mode) { @@ -553,7 +555,6 @@ void compat_start_thread(struct pt_regs struct fpu *prev_fpu = &prev->fpu; struct fpu *next_fpu = &next->fpu; int cpu = smp_processor_id(); - struct tss_struct *tss = &per_cpu(cpu_tss_rw, cpu); WARN_ON_ONCE(IS_ENABLED(CONFIG_DEBUG_ENTRY) && this_cpu_read(irq_count) != -1); @@ -617,12 +618,7 @@ void compat_start_thread(struct pt_regs /* Reload sp0. */ update_task_stack(next_p); - /* - * Now maybe reload the debug registers and handle I/O bitmaps - */ - if (unlikely(task_thread_info(next_p)->flags & _TIF_WORK_CTXSW_NEXT || - task_thread_info(prev_p)->flags & _TIF_WORK_CTXSW_PREV)) - __switch_to_xtra(prev_p, next_p, tss); + switch_to_extra(prev_p, next_p); #ifdef CONFIG_XEN_PV /*
next prev parent reply index Thread overview: 112+ messages / expand[flat|nested] mbox.gz Atom feed top 2018-11-25 18:33 [patch V2 00/28] x86/speculation: Remedy the STIBP/IBPB overhead Thomas Gleixner 2018-11-25 18:33 ` [patch V2 01/28] x86/speculation: Update the TIF_SSBD comment Thomas Gleixner 2018-11-28 14:20 ` [tip:x86/pti] " tip-bot for Tim Chen 2018-11-29 14:27 ` [patch V2 01/28] " Konrad Rzeszutek Wilk 2018-11-25 18:33 ` [patch V2 02/28] x86/speculation: Clean up spectre_v2_parse_cmdline() Thomas Gleixner 2018-11-28 14:20 ` [tip:x86/pti] " tip-bot for Tim Chen 2018-11-29 14:28 ` [patch V2 02/28] " Konrad Rzeszutek Wilk 2018-11-25 18:33 ` [patch V2 03/28] x86/speculation: Remove unnecessary ret variable in cpu_show_common() Thomas Gleixner 2018-11-28 14:21 ` [tip:x86/pti] " tip-bot for Tim Chen 2018-11-29 14:28 ` [patch V2 03/28] " Konrad Rzeszutek Wilk 2018-11-25 18:33 ` [patch V2 04/28] x86/speculation: Reorganize cpu_show_common() Thomas Gleixner 2018-11-26 15:08 ` Borislav Petkov 2018-11-28 14:22 ` [tip:x86/pti] x86/speculation: Move STIPB/IBPB string conditionals out of cpu_show_common() tip-bot for Tim Chen 2018-11-29 14:29 ` [patch V2 04/28] x86/speculation: Reorganize cpu_show_common() Konrad Rzeszutek Wilk 2018-11-25 18:33 ` [patch V2 05/28] x86/speculation: Disable STIBP when enhanced IBRS is in use Thomas Gleixner 2018-11-28 14:22 ` [tip:x86/pti] " tip-bot for Tim Chen 2018-11-29 14:35 ` [patch V2 05/28] " Konrad Rzeszutek Wilk 2018-11-25 18:33 ` [patch V2 06/28] x86/speculation: Rename SSBD update functions Thomas Gleixner 2018-11-26 15:24 ` Borislav Petkov 2018-11-28 14:23 ` [tip:x86/pti] " tip-bot for Thomas Gleixner 2018-11-29 14:37 ` [patch V2 06/28] " Konrad Rzeszutek Wilk 2018-11-25 18:33 ` [patch V2 07/28] x86/speculation: Reorganize speculation control MSRs update Thomas Gleixner 2018-11-26 15:47 ` Borislav Petkov 2018-11-28 14:23 ` [tip:x86/pti] " tip-bot for Tim Chen 2018-11-29 14:41 ` [patch V2 07/28] " Konrad Rzeszutek Wilk 2018-11-25 18:33 ` [patch V2 08/28] sched/smt: Make sched_smt_present track topology Thomas Gleixner 2018-11-28 14:24 ` [tip:x86/pti] " tip-bot for Peter Zijlstra (Intel) 2018-11-29 14:42 ` [patch V2 08/28] " Konrad Rzeszutek Wilk 2018-11-29 14:50 ` Konrad Rzeszutek Wilk 2018-11-29 15:48 ` Peter Zijlstra 2018-11-25 18:33 ` [patch V2 09/28] x86/Kconfig: Select SCHED_SMT if SMP enabled Thomas Gleixner 2018-11-28 14:24 ` [tip:x86/pti] " tip-bot for Thomas Gleixner 2018-11-29 14:44 ` [patch V2 09/28] " Konrad Rzeszutek Wilk 2018-11-25 18:33 ` [patch V2 10/28] sched/smt: Expose sched_smt_present static key Thomas Gleixner 2018-11-28 14:25 ` [tip:x86/pti] " tip-bot for Thomas Gleixner 2018-11-29 14:44 ` [patch V2 10/28] " Konrad Rzeszutek Wilk 2018-11-25 18:33 ` [patch V2 11/28] x86/speculation: Rework SMT state change Thomas Gleixner 2018-11-28 14:26 ` [tip:x86/pti] " tip-bot for Thomas Gleixner 2018-11-25 18:33 ` [patch V2 12/28] x86/l1tf: Show actual SMT state Thomas Gleixner 2018-11-28 14:26 ` [tip:x86/pti] " tip-bot for Thomas Gleixner 2018-11-25 18:33 ` [patch V2 13/28] x86/speculation: Reorder the spec_v2 code Thomas Gleixner 2018-11-26 22:21 ` Borislav Petkov 2018-11-28 14:27 ` [tip:x86/pti] " tip-bot for Thomas Gleixner 2018-11-25 18:33 ` [patch V2 14/28] x86/speculation: Mark string arrays const correctly Thomas Gleixner 2018-11-28 14:27 ` [tip:x86/pti] " tip-bot for Thomas Gleixner 2018-11-25 18:33 ` [patch V2 15/28] x86/speculataion: Mark command line parser data __initdata Thomas Gleixner 2018-11-28 14:28 ` [tip:x86/pti] " tip-bot for Thomas Gleixner 2018-11-25 18:33 ` [patch V2 16/28] x86/speculation: Unify conditional spectre v2 print functions Thomas Gleixner 2018-11-28 14:29 ` [tip:x86/pti] " tip-bot for Thomas Gleixner 2018-11-25 18:33 ` [patch V2 17/28] x86/speculation: Add command line control for indirect branch speculation Thomas Gleixner 2018-11-28 14:29 ` [tip:x86/pti] " tip-bot for Thomas Gleixner 2018-11-25 18:33 ` [patch V2 18/28] x86/speculation: Prepare for per task indirect branch speculation control Thomas Gleixner 2018-11-27 17:25 ` Lendacky, Thomas 2018-11-27 19:51 ` Tim Chen 2018-11-28 9:39 ` Thomas Gleixner 2018-11-27 20:39 ` Thomas Gleixner 2018-11-27 20:42 ` Thomas Gleixner 2018-11-27 21:52 ` Lendacky, Thomas 2018-11-28 14:30 ` [tip:x86/pti] " tip-bot for Tim Chen 2018-11-25 18:33 ` Thomas Gleixner [this message] 2018-11-26 18:30 ` [patch V2 19/28] x86/process: Consolidate and simplify switch_to_xtra() code Borislav Petkov 2018-11-28 14:30 ` [tip:x86/pti] " tip-bot for Thomas Gleixner 2018-11-25 18:33 ` [patch V2 20/28] x86/speculation: Avoid __switch_to_xtra() calls Thomas Gleixner 2018-11-28 14:31 ` [tip:x86/pti] " tip-bot for Thomas Gleixner 2018-11-25 18:33 ` [patch V2 21/28] x86/speculation: Prepare for conditional IBPB in switch_mm() Thomas Gleixner 2018-11-25 19:11 ` Thomas Gleixner 2018-11-25 20:53 ` Andi Kleen 2018-11-25 22:20 ` Thomas Gleixner 2018-11-25 23:04 ` Andy Lutomirski 2018-11-26 7:10 ` Thomas Gleixner 2018-11-26 13:36 ` Ingo Molnar 2018-11-26 3:07 ` Andi Kleen 2018-11-26 6:50 ` Thomas Gleixner 2018-11-28 14:31 ` [tip:x86/pti] " tip-bot for Thomas Gleixner 2018-11-25 18:33 ` [patch V2 22/28] ptrace: Remove unused ptrace_may_access_sched() and MODE_IBRS Thomas Gleixner 2018-11-28 14:32 ` [tip:x86/pti] " tip-bot for Thomas Gleixner 2018-11-25 18:33 ` [patch V2 23/28] x86/speculation: Split out TIF update Thomas Gleixner 2018-11-28 14:33 ` [tip:x86/pti] " tip-bot for Thomas Gleixner 2018-11-25 18:33 ` [patch V2 24/28] x86/speculation: Prepare arch_smt_update() for PRCTL mode Thomas Gleixner 2018-11-27 20:18 ` Lendacky, Thomas 2018-11-27 20:30 ` Thomas Gleixner 2018-11-27 21:20 ` Lendacky, Thomas 2018-11-28 14:34 ` [tip:x86/pti] " tip-bot for Thomas Gleixner 2018-11-25 18:33 ` [patch V2 25/28] x86/speculation: Add prctl() control for indirect branch speculation Thomas Gleixner 2018-11-28 14:34 ` [tip:x86/pti] " tip-bot for Thomas Gleixner 2018-11-25 18:33 ` [patch V2 26/28] x86/speculation: Enable prctl mode for spectre_v2_user Thomas Gleixner 2018-11-26 7:56 ` Dominik Brodowski 2018-11-28 14:35 ` [tip:x86/pti] " tip-bot for Thomas Gleixner 2018-11-25 18:33 ` [patch V2 27/28] x86/speculation: Add seccomp Spectre v2 user space protection mode Thomas Gleixner 2018-11-25 19:35 ` Randy Dunlap 2018-11-25 20:40 ` Linus Torvalds 2018-11-25 20:52 ` Jiri Kosina 2018-11-25 22:28 ` Thomas Gleixner 2018-11-26 13:30 ` Ingo Molnar 2018-11-26 20:48 ` Andrea Arcangeli 2018-11-26 20:58 ` Thomas Gleixner 2018-11-26 21:52 ` Lendacky, Thomas 2018-11-27 0:37 ` Tim Chen 2018-12-04 1:38 ` Tim Chen 2018-12-04 8:39 ` Jiri Kosina 2018-12-04 9:43 ` Arjan van de Ven 2018-12-04 9:46 ` Arjan van de Ven 2018-12-04 17:20 ` Linus Torvalds 2018-12-04 18:58 ` Tim Chen 2018-11-28 14:35 ` [tip:x86/pti] " tip-bot for Thomas Gleixner 2018-12-04 18:45 ` [patch V2 27/28] " Dave Hansen 2018-11-25 18:33 ` [patch V2 28/28] x86/speculation: Provide IBPB always command line options Thomas Gleixner 2018-11-28 14:36 ` [tip:x86/pti] " tip-bot for Thomas Gleixner 2018-11-26 13:37 ` [patch V2 00/28] x86/speculation: Remedy the STIBP/IBPB overhead Ingo Molnar 2018-11-28 14:24 ` Thomas Gleixner 2018-11-29 19:02 ` Tim Chen 2018-12-10 23:43 ` Pavel Machek
Reply instructions: You may reply publicly to this message via plain-text email using any one of the following methods: * Save the following mbox file, import it into your mail client, and reply-to-all from there: mbox Avoid top-posting and favor interleaved quoting: https://en.wikipedia.org/wiki/Posting_style#Interleaved_style * Reply using the --to, --cc, and --in-reply-to switches of git-send-email(1): git send-email \ --in-reply-to=20181125185005.280855518@linutronix.de \ --to=tglx@linutronix.de \ --cc=aarcange@redhat.com \ --cc=ak@linux.intel.com \ --cc=arjan@linux.intel.com \ --cc=asit.k.mallick@intel.com \ --cc=casey.schaufler@intel.com \ --cc=dave.hansen@intel.com \ --cc=david.c.stewart@intel.com \ --cc=dwmw@amazon.co.uk \ --cc=gregkh@linuxfoundation.org \ --cc=jcm@redhat.com \ --cc=jkosina@suse.cz \ --cc=jpoimboe@redhat.com \ --cc=keescook@chromium.org \ --cc=linux-kernel@vger.kernel.org \ --cc=longman9394@gmail.com \ --cc=luto@kernel.org \ --cc=peterz@infradead.org \ --cc=thomas.lendacky@amd.com \ --cc=tim.c.chen@linux.intel.com \ --cc=torvalds@linux-foundation.org \ --cc=x86@kernel.org \ /path/to/YOUR_REPLY https://kernel.org/pub/software/scm/git/docs/git-send-email.html * If your mail client supports setting the In-Reply-To header via mailto: links, try the mailto: link
LKML Archive on lore.kernel.org Archives are clonable: git clone --mirror https://lore.kernel.org/lkml/0 lkml/git/0.git git clone --mirror https://lore.kernel.org/lkml/1 lkml/git/1.git git clone --mirror https://lore.kernel.org/lkml/2 lkml/git/2.git git clone --mirror https://lore.kernel.org/lkml/3 lkml/git/3.git git clone --mirror https://lore.kernel.org/lkml/4 lkml/git/4.git git clone --mirror https://lore.kernel.org/lkml/5 lkml/git/5.git git clone --mirror https://lore.kernel.org/lkml/6 lkml/git/6.git git clone --mirror https://lore.kernel.org/lkml/7 lkml/git/7.git git clone --mirror https://lore.kernel.org/lkml/8 lkml/git/8.git git clone --mirror https://lore.kernel.org/lkml/9 lkml/git/9.git # If you have public-inbox 1.1+ installed, you may # initialize and index your mirror using the following commands: public-inbox-init -V2 lkml lkml/ https://lore.kernel.org/lkml \ linux-kernel@vger.kernel.org public-inbox-index lkml Example config snippet for mirrors Newsgroup available over NNTP: nntp://nntp.lore.kernel.org/org.kernel.vger.linux-kernel AGPL code for this site: git clone https://public-inbox.org/public-inbox.git