From: Thomas Gleixner <tglx@linutronix.de>
To: LKML <linux-kernel@vger.kernel.org>
Cc: x86@kernel.org, Christoph Hellwig <hch@lst.de>,
Kees Cook <keescook@chromium.org>,
Alexandre Chartre <alexandre.chartre@oracle.com>,
"Peter Zijlstra (Intel)" <peterz@infradead.org>,
Juergen Gross <jgross@suse.com>,
Thomas Lendacky <Thomas.Lendacky@amd.com>,
Boris Ostrovsky <boris.ostrovsky@oracle.com>
Subject: [patch V2 07/16] x86/tlb: Move __flush_tlb_one_user() out of line
Date: Tue, 21 Apr 2020 11:20:34 +0200 [thread overview]
Message-ID: <20200421092559.428213098@linutronix.de> (raw)
In-Reply-To: 20200421092027.591582014@linutronix.de
cpu_tlbstate is exported because various TLB related functions need access
to it, but cpu_tlbstate is sensitive information which should only be
accessed by well contained kernel functions and not be directly exposed to
modules.
The various TLB flush functions need access to cpu_tlbstate. As third step
move _flush_tlb_one_user() out of line and hide the native function. The
latter can be static when CONFIG_PARAVIRT is disabled.
Consolidate the name space while at it and remove the pointless extra
wrapper in the paravirt code.
No functional change.
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Reviewed-by: Alexandre Chartre <alexandre.chartre@oracle.com>
Acked-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Cc: Juergen Gross <jgross@suse.com>
---
V2: Fixup TLB internal and UV callsite
---
arch/x86/include/asm/paravirt.h | 1
arch/x86/include/asm/tlbflush.h | 53 +------------------------------------
arch/x86/kernel/paravirt.c | 5 ---
arch/x86/mm/tlb.c | 56 +++++++++++++++++++++++++++++++++++++++-
arch/x86/platform/uv/tlb_uv.c | 2 -
5 files changed, 59 insertions(+), 58 deletions(-)
--- a/arch/x86/include/asm/paravirt.h
+++ b/arch/x86/include/asm/paravirt.h
@@ -49,6 +49,7 @@ static inline void slow_down_io(void)
void native_flush_tlb_local(void);
void native_flush_tlb_global(void);
+void native_flush_tlb_one_user(unsigned long addr);
static inline void __flush_tlb_local(void)
{
--- a/arch/x86/include/asm/tlbflush.h
+++ b/arch/x86/include/asm/tlbflush.h
@@ -142,11 +142,10 @@ static inline unsigned long build_cr3_no
void flush_tlb_local(void);
void flush_tlb_global(void);
+void flush_tlb_one_user(unsigned long addr);
#ifdef CONFIG_PARAVIRT
#include <asm/paravirt.h>
-#else
-#define __flush_tlb_one_user(addr) __native_flush_tlb_one_user(addr)
#endif
struct tlb_context {
@@ -346,54 +345,6 @@ static inline void cr4_set_bits_and_upda
extern void initialize_tlbstate_and_flush(void);
/*
- * Given an ASID, flush the corresponding user ASID. We can delay this
- * until the next time we switch to it.
- *
- * See SWITCH_TO_USER_CR3.
- */
-static inline void invalidate_user_asid(u16 asid)
-{
- /* There is no user ASID if address space separation is off */
- if (!IS_ENABLED(CONFIG_PAGE_TABLE_ISOLATION))
- return;
-
- /*
- * We only have a single ASID if PCID is off and the CR3
- * write will have flushed it.
- */
- if (!cpu_feature_enabled(X86_FEATURE_PCID))
- return;
-
- if (!static_cpu_has(X86_FEATURE_PTI))
- return;
-
- __set_bit(kern_pcid(asid),
- (unsigned long *)this_cpu_ptr(&cpu_tlbstate.user_pcid_flush_mask));
-}
-
-/*
- * flush one page in the user mapping
- */
-static inline void __native_flush_tlb_one_user(unsigned long addr)
-{
- u32 loaded_mm_asid = this_cpu_read(cpu_tlbstate.loaded_mm_asid);
-
- asm volatile("invlpg (%0)" ::"r" (addr) : "memory");
-
- if (!static_cpu_has(X86_FEATURE_PTI))
- return;
-
- /*
- * Some platforms #GP if we call invpcid(type=1/2) before CR4.PCIDE=1.
- * Just use invalidate_user_asid() in case we are called early.
- */
- if (!this_cpu_has(X86_FEATURE_INVPCID_SINGLE))
- invalidate_user_asid(loaded_mm_asid);
- else
- invpcid_flush_one(user_pcid(loaded_mm_asid), addr);
-}
-
-/*
* flush everything
*/
static inline void __flush_tlb_all(void)
@@ -432,7 +383,7 @@ static inline void __flush_tlb_one_kerne
* kernel address space and for its usermode counterpart, but it does
* not flush it for other address spaces.
*/
- __flush_tlb_one_user(addr);
+ flush_tlb_one_user(addr);
if (!static_cpu_has(X86_FEATURE_PTI))
return;
--- a/arch/x86/kernel/paravirt.c
+++ b/arch/x86/kernel/paravirt.c
@@ -160,11 +160,6 @@ unsigned paravirt_patch_insns(void *insn
return insn_len;
}
-static void native_flush_tlb_one_user(unsigned long addr)
-{
- __native_flush_tlb_one_user(addr);
-}
-
struct static_key paravirt_steal_enabled;
struct static_key paravirt_steal_rq_enabled;
--- a/arch/x86/mm/tlb.c
+++ b/arch/x86/mm/tlb.c
@@ -24,6 +24,7 @@
# define STATIC_NOPV static
# define __flush_tlb_local native_flush_tlb_local
# define __flush_tlb_global native_flush_tlb_global
+# define __flush_tlb_one_user(addr) native_flush_tlb_one_user(addr)
#endif
/*
@@ -118,6 +119,32 @@ static void choose_new_asid(struct mm_st
*need_flush = true;
}
+/*
+ * Given an ASID, flush the corresponding user ASID. We can delay this
+ * until the next time we switch to it.
+ *
+ * See SWITCH_TO_USER_CR3.
+ */
+static inline void invalidate_user_asid(u16 asid)
+{
+ /* There is no user ASID if address space separation is off */
+ if (!IS_ENABLED(CONFIG_PAGE_TABLE_ISOLATION))
+ return;
+
+ /*
+ * We only have a single ASID if PCID is off and the CR3
+ * write will have flushed it.
+ */
+ if (!cpu_feature_enabled(X86_FEATURE_PCID))
+ return;
+
+ if (!static_cpu_has(X86_FEATURE_PTI))
+ return;
+
+ __set_bit(kern_pcid(asid),
+ (unsigned long *)this_cpu_ptr(&cpu_tlbstate.user_pcid_flush_mask));
+}
+
static void load_new_mm_cr3(pgd_t *pgdir, u16 new_asid, bool need_flush)
{
unsigned long new_mm_cr3;
@@ -645,7 +672,7 @@ static void flush_tlb_func_common(const
unsigned long addr = f->start;
while (addr < f->end) {
- __flush_tlb_one_user(addr);
+ flush_tlb_one_user(addr);
addr += 1UL << f->stride_shift;
}
if (local)
@@ -892,6 +919,33 @@ unsigned long __get_current_cr3_fast(voi
EXPORT_SYMBOL_GPL(__get_current_cr3_fast);
/*
+ * Flush one page in the user mapping
+ */
+STATIC_NOPV void native_flush_tlb_one_user(unsigned long addr)
+{
+ u32 loaded_mm_asid = this_cpu_read(cpu_tlbstate.loaded_mm_asid);
+
+ asm volatile("invlpg (%0)" ::"r" (addr) : "memory");
+
+ if (!static_cpu_has(X86_FEATURE_PTI))
+ return;
+
+ /*
+ * Some platforms #GP if we call invpcid(type=1/2) before CR4.PCIDE=1.
+ * Just use invalidate_user_asid() in case we are called early.
+ */
+ if (!this_cpu_has(X86_FEATURE_INVPCID_SINGLE))
+ invalidate_user_asid(loaded_mm_asid);
+ else
+ invpcid_flush_one(user_pcid(loaded_mm_asid), addr);
+}
+
+void flush_tlb_one_user(unsigned long addr)
+{
+ __flush_tlb_one_user(addr);
+}
+
+/*
* Flush everything
*/
STATIC_NOPV void native_flush_tlb_global(void)
--- a/arch/x86/platform/uv/tlb_uv.c
+++ b/arch/x86/platform/uv/tlb_uv.c
@@ -296,7 +296,7 @@ static void bau_process_message(struct m
flush_tlb_local();
stat->d_alltlb++;
} else {
- __flush_tlb_one_user(msg->address);
+ flush_tlb_one_user(msg->address);
stat->d_onetlb++;
}
stat->d_requestee++;
next prev parent reply other threads:[~2020-04-21 9:27 UTC|newest]
Thread overview: 35+ messages / expand[flat|nested] mbox.gz Atom feed top
2020-04-21 9:20 [patch V2 00/16] x86/tlb: Unexport per-CPU tlbstate Thomas Gleixner
2020-04-21 9:20 ` [patch V2 01/16] x86/tlb: Uninline __get_current_cr3_fast() Thomas Gleixner
2020-04-21 17:04 ` Andy Lutomirski
2020-04-26 18:42 ` [tip: x86/mm] " tip-bot2 for Thomas Gleixner
2020-04-21 9:20 ` [patch V2 02/16] x86/cpu: Uninline CR4 accessors Thomas Gleixner
2020-04-26 18:42 ` [tip: x86/mm] x86/cpu: Export native_write_cr4() only when CONFIG_LKTDM=m tip-bot2 for Thomas Gleixner
2020-04-26 18:42 ` [tip: x86/mm] x86/cpu: Uninline CR4 accessors tip-bot2 for Thomas Gleixner
2020-04-21 9:20 ` [patch V2 03/16] x86/cr4: Sanitize CR4.PCE update Thomas Gleixner
2020-04-26 18:42 ` [tip: x86/mm] " tip-bot2 for Thomas Gleixner
2020-04-21 9:20 ` [patch V2 04/16] x86/alternatives: Move temporary_mm helpers into C Thomas Gleixner
2020-04-26 18:42 ` [tip: x86/mm] " tip-bot2 for Thomas Gleixner
2020-04-21 9:20 ` [patch V2 05/16] x86/tlb: Move __flush_tlb() out of line Thomas Gleixner
2020-04-26 18:42 ` [tip: x86/mm] " tip-bot2 for Thomas Gleixner
2020-04-21 9:20 ` [patch V2 06/16] x86/tlb: Move __flush_tlb_global() " Thomas Gleixner
2020-04-26 18:42 ` [tip: x86/mm] " tip-bot2 for Thomas Gleixner
2020-04-21 9:20 ` Thomas Gleixner [this message]
2020-04-26 18:42 ` [tip: x86/mm] x86/tlb: Move __flush_tlb_one_user() " tip-bot2 for Thomas Gleixner
2020-04-21 9:20 ` [patch V2 08/16] x86/tlb: Move __flush_tlb_one_kernel() " Thomas Gleixner
2020-04-26 18:42 ` [tip: x86/mm] " tip-bot2 for Thomas Gleixner
2020-04-21 9:20 ` [patch V2 09/16] x86/tlb: Move flush_tlb_others() " Thomas Gleixner
2020-04-26 18:42 ` [tip: x86/mm] " tip-bot2 for Thomas Gleixner
2020-04-21 9:20 ` [patch V2 10/16] x86/tlb: Move __flush_tlb_all() " Thomas Gleixner
2020-04-26 18:42 ` [tip: x86/mm] " tip-bot2 for Thomas Gleixner
2020-04-21 9:20 ` [patch V2 11/16] x86/tlb: Move paravirt_tlb_remove_table() to the usage site Thomas Gleixner
2020-04-26 18:42 ` [tip: x86/mm] " tip-bot2 for Thomas Gleixner
2020-04-21 9:20 ` [patch V2 12/16] x86/tlb: Move cr4_set_bits_and_update_boot() " Thomas Gleixner
2020-04-26 18:42 ` [tip: x86/mm] " tip-bot2 for Thomas Gleixner
2020-04-21 9:20 ` [patch V2 13/16] x86/tlb: Uninline nmi_uaccess_okay() Thomas Gleixner
2020-04-26 18:42 ` [tip: x86/mm] " tip-bot2 for Thomas Gleixner
2020-04-21 9:20 ` [patch V2 14/16] x86/tlb: Move PCID helpers where they are used Thomas Gleixner
2020-04-26 18:42 ` [tip: x86/mm] " tip-bot2 for Thomas Gleixner
2020-04-21 9:20 ` [patch V2 15/16] xen/privcmd: Remove unneeded asm/tlb.h include Thomas Gleixner
2020-04-26 18:42 ` [tip: x86/mm] " tip-bot2 for Thomas Gleixner
2020-04-21 9:20 ` [patch V2 16/16] x86/tlb: Restrict access to tlbstate Thomas Gleixner
2020-04-26 18:42 ` [tip: x86/mm] " tip-bot2 for Thomas Gleixner
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20200421092559.428213098@linutronix.de \
--to=tglx@linutronix.de \
--cc=Thomas.Lendacky@amd.com \
--cc=alexandre.chartre@oracle.com \
--cc=boris.ostrovsky@oracle.com \
--cc=hch@lst.de \
--cc=jgross@suse.com \
--cc=keescook@chromium.org \
--cc=linux-kernel@vger.kernel.org \
--cc=peterz@infradead.org \
--cc=x86@kernel.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).