From: Thomas Gleixner <tglx@linutronix.de>
To: LKML <linux-kernel@vger.kernel.org>
Cc: x86@kernel.org, Kees Cook <keescook@chromium.org>,
Paolo Bonzini <pbonzini@redhat.com>,
Thomas Lendacky <Thomas.Lendacky@amd.com>,
Juergen Gross <jgross@suse.com>,
Boris Ostrovsky <boris.ostrovsky@oracle.com>
Subject: [patch 08/15] x86/tlb: Move __flush_tlb_one_kernel() out of line
Date: Sun, 19 Apr 2020 22:31:45 +0200 [thread overview]
Message-ID: <20200419203336.425338564@linutronix.de> (raw)
In-Reply-To: 20200419203137.214111265@linutronix.de
cpu_tlbstate is exported because various TLB related functions need access
to it, but cpu_tlbstate is sensitive information which should only be
accessed by well contained kernel functions and not be directly exposed to
modules.
The various TLB flush functions need access to cpu_tlbstate. As forth step
move __flush_tlb_one_kernel() out of line and hide the native function. The
latter can be static when CONFIG_PARAVIRT is disabled.
Consolidate the name space while at it and remove the pointless extra
wrapper in the paravirt code.
No functional change.
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
---
arch/x86/include/asm/pgtable_32.h | 2 -
arch/x86/include/asm/tlbflush.h | 41 --------------------------------------
arch/x86/mm/init_64.c | 2 -
arch/x86/mm/ioremap.c | 2 -
arch/x86/mm/kmmio.c | 2 -
arch/x86/mm/pat/set_memory.c | 2 -
arch/x86/mm/pgtable_32.c | 2 -
arch/x86/mm/tlb.c | 34 ++++++++++++++++++++++++++++++-
8 files changed, 40 insertions(+), 47 deletions(-)
--- a/arch/x86/include/asm/pgtable_32.h
+++ b/arch/x86/include/asm/pgtable_32.h
@@ -60,7 +60,7 @@ void sync_initial_page_table(void);
#define kpte_clear_flush(ptep, vaddr) \
do { \
pte_clear(&init_mm, (vaddr), (ptep)); \
- __flush_tlb_one_kernel((vaddr)); \
+ flush_tlb_one_kernel((vaddr)); \
} while (0)
#endif /* !__ASSEMBLY__ */
--- a/arch/x86/include/asm/tlbflush.h
+++ b/arch/x86/include/asm/tlbflush.h
@@ -143,6 +143,7 @@ static inline unsigned long build_cr3_no
void flush_tlb_local(void);
void flush_tlb_global(void);
void flush_tlb_one_user(unsigned long addr);
+void flush_tlb_one_kernel(unsigned long addr);
#ifdef CONFIG_PARAVIRT
#include <asm/paravirt.h>
@@ -318,14 +319,6 @@ static inline void cr4_clear_bits(unsign
}
/*
- * Mark all other ASIDs as invalid, preserves the current.
- */
-static inline void invalidate_other_asid(void)
-{
- this_cpu_write(cpu_tlbstate.invalidate_other, true);
-}
-
-/*
* Save some of cr4 feature set we're using (e.g. Pentium 4MB
* enable and PPro Global page enable), so that any CPU's that boot
* up after us can get the correct flags. This should only be used
@@ -365,38 +358,6 @@ static inline void __flush_tlb_all(void)
}
}
-/*
- * flush one page in the kernel mapping
- */
-static inline void __flush_tlb_one_kernel(unsigned long addr)
-{
- count_vm_tlb_event(NR_TLB_LOCAL_FLUSH_ONE);
-
- /*
- * If PTI is off, then __flush_tlb_one_user() is just INVLPG or its
- * paravirt equivalent. Even with PCID, this is sufficient: we only
- * use PCID if we also use global PTEs for the kernel mapping, and
- * INVLPG flushes global translations across all address spaces.
- *
- * If PTI is on, then the kernel is mapped with non-global PTEs, and
- * __flush_tlb_one_user() will flush the given address for the current
- * kernel address space and for its usermode counterpart, but it does
- * not flush it for other address spaces.
- */
- flush_tlb_one_user(addr);
-
- if (!static_cpu_has(X86_FEATURE_PTI))
- return;
-
- /*
- * See above. We need to propagate the flush to all other address
- * spaces. In principle, we only need to propagate it to kernelmode
- * address spaces, but the extra bookkeeping we would need is not
- * worth it.
- */
- invalidate_other_asid();
-}
-
#define TLB_FLUSH_ALL -1UL
/*
--- a/arch/x86/mm/init_64.c
+++ b/arch/x86/mm/init_64.c
@@ -298,7 +298,7 @@ static void __set_pte_vaddr(pud_t *pud,
* It's enough to flush this one mapping.
* (PGE mappings get flushed as well)
*/
- __flush_tlb_one_kernel(vaddr);
+ flush_tlb_one_kernel(vaddr);
}
void set_pte_vaddr_p4d(p4d_t *p4d_page, unsigned long vaddr, pte_t new_pte)
--- a/arch/x86/mm/ioremap.c
+++ b/arch/x86/mm/ioremap.c
@@ -889,5 +889,5 @@ void __init __early_set_fixmap(enum fixe
set_pte(pte, pfn_pte(phys >> PAGE_SHIFT, flags));
else
pte_clear(&init_mm, addr, pte);
- __flush_tlb_one_kernel(addr);
+ flush_tlb_one_kernel(addr);
}
--- a/arch/x86/mm/kmmio.c
+++ b/arch/x86/mm/kmmio.c
@@ -173,7 +173,7 @@ static int clear_page_presence(struct km
return -1;
}
- __flush_tlb_one_kernel(f->addr);
+ flush_tlb_one_kernel(f->addr);
return 0;
}
--- a/arch/x86/mm/pat/set_memory.c
+++ b/arch/x86/mm/pat/set_memory.c
@@ -340,7 +340,7 @@ static void __cpa_flush_tlb(void *data)
unsigned int i;
for (i = 0; i < cpa->numpages; i++)
- __flush_tlb_one_kernel(fix_addr(__cpa_addr(cpa, i)));
+ flush_tlb_one_kernel(fix_addr(__cpa_addr(cpa, i)));
}
static void cpa_flush(struct cpa_data *data, int cache)
--- a/arch/x86/mm/pgtable_32.c
+++ b/arch/x86/mm/pgtable_32.c
@@ -64,7 +64,7 @@ void set_pte_vaddr(unsigned long vaddr,
* It's enough to flush this one mapping.
* (PGE mappings get flushed as well)
*/
- __flush_tlb_one_kernel(vaddr);
+ flush_tlb_one_kernel(vaddr);
}
unsigned long __FIXADDR_TOP = 0xfffff000;
--- a/arch/x86/mm/tlb.c
+++ b/arch/x86/mm/tlb.c
@@ -876,7 +876,7 @@ static void do_kernel_range_flush(void *
/* flush range by one by one 'invlpg' */
for (addr = f->start; addr < f->end; addr += PAGE_SIZE)
- __flush_tlb_one_kernel(addr);
+ flush_tlb_one_kernel(addr);
}
void flush_tlb_kernel_range(unsigned long start, unsigned long end)
@@ -919,6 +919,38 @@ unsigned long __get_current_cr3_fast(voi
EXPORT_SYMBOL_GPL(__get_current_cr3_fast);
/*
+ * Flush one page in the kernel mapping
+ */
+void flush_tlb_one_kernel(unsigned long addr)
+{
+ count_vm_tlb_event(NR_TLB_LOCAL_FLUSH_ONE);
+
+ /*
+ * If PTI is off, then __flush_tlb_one_user() is just INVLPG or its
+ * paravirt equivalent. Even with PCID, this is sufficient: we only
+ * use PCID if we also use global PTEs for the kernel mapping, and
+ * INVLPG flushes global translations across all address spaces.
+ *
+ * If PTI is on, then the kernel is mapped with non-global PTEs, and
+ * __flush_tlb_one_user() will flush the given address for the current
+ * kernel address space and for its usermode counterpart, but it does
+ * not flush it for other address spaces.
+ */
+ flush_tlb_one_user(addr);
+
+ if (!static_cpu_has(X86_FEATURE_PTI))
+ return;
+
+ /*
+ * See above. We need to propagate the flush to all other address
+ * spaces. In principle, we only need to propagate it to kernelmode
+ * address spaces, but the extra bookkeeping we would need is not
+ * worth it.
+ */
+ this_cpu_write(cpu_tlbstate.invalidate_other, true);
+}
+
+/*
* Flush one page in the user mapping
*/
STATIC_NOPV void native_flush_tlb_one_user(unsigned long addr)
next prev parent reply other threads:[~2020-04-19 20:36 UTC|newest]
Thread overview: 34+ messages / expand[flat|nested] mbox.gz Atom feed top
2020-04-19 20:31 [patch 00/15] x86/tlb: Unexport per-CPU tlbstate Thomas Gleixner
2020-04-19 20:31 ` [patch 01/15] x86/tlb: Uninline __get_current_cr3_fast() Thomas Gleixner
2020-04-19 20:31 ` [patch 02/15] x86/cpu: Uninline CR4 accessors Thomas Gleixner
2020-04-20 9:01 ` Christoph Hellwig
2020-04-20 9:34 ` Borislav Petkov
2020-04-20 17:25 ` Thomas Gleixner
2020-04-19 20:31 ` [patch 03/15] x86/cr4: Sanitize CR4.PCE update Thomas Gleixner
2020-04-19 20:31 ` [patch 04/15] x86/alternatives: Move temporary_mm helpers into C Thomas Gleixner
2020-04-19 20:31 ` [patch 05/15] x86/tlb: Move __flush_tlb() out of line Thomas Gleixner
2020-04-20 13:48 ` Tom Lendacky
2020-04-20 14:03 ` Jürgen Groß
2020-04-20 14:26 ` Tom Lendacky
2020-04-20 14:38 ` Jürgen Groß
2020-04-20 18:30 ` Tom Lendacky
2020-04-19 20:31 ` [patch 06/15] x86/tlb: Move __flush_tlb_global() " Thomas Gleixner
2020-04-19 20:31 ` [patch 07/15] x86/tlb: Move __flush_tlb_one_user() " Thomas Gleixner
2020-04-19 20:31 ` Thomas Gleixner [this message]
2020-04-19 20:31 ` [patch 09/15] x86/tlb: Move flush_tlb_others() " Thomas Gleixner
2020-04-19 20:31 ` [patch 10/15] x86/tlb: Move paravirt_tlb_remove_table() to the usage site Thomas Gleixner
2020-04-19 20:31 ` [patch 11/15] x86/tlb: Move cr4_set_bits_and_update_boot() " Thomas Gleixner
2020-04-19 20:31 ` [patch 12/15] x86/tlb: Uninline nmi_uaccess_okay() Thomas Gleixner
2020-04-19 20:31 ` [patch 13/15] x86/tlb: Move PCID helpers where they are used Thomas Gleixner
2020-04-19 20:31 ` [patch 14/15] xen/privcmd: Remove unneeded asm/tlb.h include Thomas Gleixner
2020-04-19 20:31 ` [patch 15/15] x86/tlb: Restrict access to tlbstate Thomas Gleixner
2020-04-20 9:20 ` [patch 00/15] x86/tlb: Unexport per-CPU tlbstate Christoph Hellwig
2020-04-20 16:58 ` Alexandre Chartre
2020-04-20 20:08 ` Thomas Gleixner
2020-04-20 17:27 ` Thomas Gleixner
2020-04-21 8:09 ` Sean Christopherson
2020-04-21 9:09 ` Thomas Gleixner
2020-04-22 0:42 ` Sean Christopherson
2020-04-20 10:25 ` Peter Zijlstra
2020-04-20 16:33 ` Alexandre Chartre
2020-04-21 17:10 ` Andy Lutomirski
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20200419203336.425338564@linutronix.de \
--to=tglx@linutronix.de \
--cc=Thomas.Lendacky@amd.com \
--cc=boris.ostrovsky@oracle.com \
--cc=jgross@suse.com \
--cc=keescook@chromium.org \
--cc=linux-kernel@vger.kernel.org \
--cc=pbonzini@redhat.com \
--cc=x86@kernel.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).