[PATCH 0/6] more KAISER bits

[PATCH 0/6] more KAISER bits

[Peter Zijlstra]

Here's more patches, includes the TLB invalidate rework.

Has not actually been tested on a INVPCID machine yet, but does seem to
work fine on my IVB-EP (which is PCID only).

[PATCH 1/6] x86/mm/kaiser: Add some static

[Peter Zijlstra]

[-- Attachment #1: peterz-kaiser-moar-static.patch --]
[-- Type: text/plain, Size: 1138 bytes --]

These fuctions are only used in this TU, make em static.

Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
---
 arch/x86/mm/kaiser.c |   10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

--- a/arch/x86/mm/kaiser.c
+++ b/arch/x86/mm/kaiser.c
@@ -260,8 +260,8 @@ static pte_t *kaiser_shadow_pagetable_wa
  * the user (shadow) page tables.  This may need to allocate page
  * table pages.
  */
-int kaiser_add_user_map(const void *__start_addr, unsigned long size,
-			unsigned long flags)
+static int kaiser_add_user_map(const void *__start_addr, unsigned long size,
+			       unsigned long flags)
 {
 	unsigned long start_addr = (unsigned long)__start_addr;
 	unsigned long address = start_addr & PAGE_MASK;
@@ -310,9 +310,9 @@ int kaiser_add_user_map(const void *__st
 	return 0;
 }
 
-int kaiser_add_user_map_ptrs(const void *__start_addr,
-			     const void *__end_addr,
-			     unsigned long flags)
+static int kaiser_add_user_map_ptrs(const void *__start_addr,
+				    const void *__end_addr,
+				    unsigned long flags)
 {
 	return kaiser_add_user_map(__start_addr,
 				   __end_addr - __start_addr,

[PATCH 2/6] x86/mm/kaiser: Fix inconsistency in SAVE_AND_SWITCH_TO_KERNEL_CR3

[Peter Zijlstra]

[-- Attachment #1: peterz-kaiser-clean-up-scratch_reg.patch --]
[-- Type: text/plain, Size: 1452 bytes --]

For some obscure reason \scratch_reg is not including the %r while
\save_reg is.

Also-Reported-by: Borislav Petkov <bp@alien8.de>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
---
 arch/x86/entry/calling.h  |   10 +++++-----
 arch/x86/entry/entry_64.S |    2 +-
 2 files changed, 6 insertions(+), 6 deletions(-)

--- a/arch/x86/entry/calling.h
+++ b/arch/x86/entry/calling.h
@@ -227,8 +227,8 @@ For 32-bit we have the following convent
 
 .macro SAVE_AND_SWITCH_TO_KERNEL_CR3 scratch_reg:req save_reg:req
 	STATIC_JUMP_IF_FALSE .Ldone_\@, kaiser_enabled_key, def=1
-	movq	%cr3, %r\scratch_reg
-	movq	%r\scratch_reg, \save_reg
+	movq	%cr3, \scratch_reg
+	movq	\scratch_reg, \save_reg
 	/*
 	 * Is the "switch mask" all zero?  That means that both of
 	 * these are zero:
@@ -239,11 +239,11 @@ For 32-bit we have the following convent
 	 *
 	 * That indicates a kernel CR3 value, not user/shadow.
 	 */
-	testq	$(KAISER_SWITCH_MASK), %r\scratch_reg
+	testq	$(KAISER_SWITCH_MASK), \scratch_reg
 	jz	.Ldone_\@
 
-	ADJUST_KERNEL_CR3 %r\scratch_reg
-	movq	%r\scratch_reg, %cr3
+	ADJUST_KERNEL_CR3 \scratch_reg
+	movq	\scratch_reg, %cr3
 
 .Ldone_\@:
 .endm
--- a/arch/x86/entry/entry_64.S
+++ b/arch/x86/entry/entry_64.S
@@ -1254,7 +1254,7 @@ ENTRY(paranoid_entry)
 	xorl	%ebx, %ebx
 
 1:
-	SAVE_AND_SWITCH_TO_KERNEL_CR3 scratch_reg=ax save_reg=%r14
+	SAVE_AND_SWITCH_TO_KERNEL_CR3 scratch_reg=%rax save_reg=%r14
 
 	ret
 END(paranoid_entry)

[PATCH 3/6] x86/mm/kaiser: Allow PCID with nokaiser

[Peter Zijlstra]

[-- Attachment #1: peterz-nokaiser-dont-kill-pcid.patch --]
[-- Type: text/plain, Size: 2367 bytes --]

Currently KAISER kills PCID on platforms that lack INVPCID, even when
nokaiser.

Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
---
 arch/x86/include/asm/tlbflush.h |    9 +++++----
 arch/x86/mm/init.c              |    2 +-
 arch/x86/mm/tlb.c               |    7 ++++++-
 3 files changed, 12 insertions(+), 6 deletions(-)

--- a/arch/x86/include/asm/tlbflush.h
+++ b/arch/x86/include/asm/tlbflush.h
@@ -9,6 +9,7 @@
 #include <asm/cpufeature.h>
 #include <asm/special_insns.h>
 #include <asm/smp.h>
+#include <asm/kaiser.h>
 
 static inline void __invpcid(unsigned long pcid, unsigned long addr,
 			     unsigned long type)
@@ -355,12 +356,12 @@ static inline void __native_flush_tlb(vo
 		 * CR4 has X86_CR4_PCIDE set.  In other words, this does
 		 * not fully flush the TLB if PCIDs are in use.
 		 *
-		 * With KAISER and PCIDs, the means that we did not
+		 * With KAISER and PCIDs, that means that we did not
 		 * flush the user PCID.  Warn if it gets called.
 		 */
-		if (IS_ENABLED(CONFIG_KAISER))
-			WARN_ON_ONCE(this_cpu_read(cpu_tlbstate.cr4) &
-				     X86_CR4_PCIDE);
+		if (IS_ENABLED(CONFIG_KAISER) && kaiser_enabled)
+			WARN_ON_ONCE(this_cpu_read(cpu_tlbstate.cr4) & X86_CR4_PCIDE);
+
 		/*
 		 * If current->mm == NULL then we borrow a mm
 		 * which may change during a task switch and
--- a/arch/x86/mm/init.c
+++ b/arch/x86/mm/init.c
@@ -219,7 +219,7 @@ static void setup_pcid(void)
 		 * have KAISER and do not have INVPCID.
 		 */
 		if (!IS_ENABLED(CONFIG_X86_GLOBAL_PAGES) &&
-		    !boot_cpu_has(X86_FEATURE_INVPCID)) {
+		    kaiser_enabled && !boot_cpu_has(X86_FEATURE_INVPCID)) {
 			setup_clear_cpu_cap(X86_FEATURE_PCID);
 			return;
 		}
--- a/arch/x86/mm/tlb.c
+++ b/arch/x86/mm/tlb.c
@@ -6,13 +6,14 @@
 #include <linux/interrupt.h>
 #include <linux/export.h>
 #include <linux/cpu.h>
+#include <linux/debugfs.h>
 
 #include <asm/tlbflush.h>
 #include <asm/mmu_context.h>
 #include <asm/cache.h>
 #include <asm/apic.h>
 #include <asm/uv/uv.h>
-#include <linux/debugfs.h>
+#include <asm/kaiser.h>
 
 /*
  *	TLB flushing, formerly SMP-only
@@ -115,6 +116,10 @@ static void flush_user_asid(pgd_t *pgd,
 	 */
 	if (!cpu_feature_enabled(X86_FEATURE_PCID))
 		return;
+
+	if (!kaiser_enabled)
+		return;
+
 	/*
 	 * With PCIDs enabled, write_cr3() only flushes TLB
 	 * entries for the current (kernel) ASID.  This leaves

[PATCH 4/6] x86/mm/kaiser: Support PCID without INVPCID

[Peter Zijlstra]

[-- Attachment #1: peterz-kaiser-support-pcid-noinvpcid.patch --]
[-- Type: text/plain, Size: 6560 bytes --]

Instead of relying on INVPCID to shoot down user PCID, delay the
invalidate until we switch to the user page-tables.

This gets rid of the INVPCID dependence for KAISER PCID.

XXX we could do a much larger ALTERNATIVE, there is no point in
testing the mask if we don't have PCID support.

Suggested-by: Andy Lutomirski <luto@amacapital.net>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
---
 arch/x86/entry/calling.h        |   29 +++++++++++++++++------
 arch/x86/include/asm/tlbflush.h |   39 +++++++++++++++++++++++--------
 arch/x86/mm/init.c              |   13 ----------
 arch/x86/mm/tlb.c               |   49 +---------------------------------------
 4 files changed, 53 insertions(+), 77 deletions(-)

--- a/arch/x86/entry/calling.h
+++ b/arch/x86/entry/calling.h
@@ -4,6 +4,7 @@
 #include <asm/cpufeatures.h>
 #include <asm/page_types.h>
 #include <asm/pgtable_types.h>
+#include <asm/percpu.h>
 
 /*
 
@@ -203,12 +204,6 @@ For 32-bit we have the following convent
 	andq    $(~KAISER_SWITCH_MASK), \reg
 .endm
 
-.macro ADJUST_USER_CR3 reg:req
-	ALTERNATIVE "", "bts $63, \reg", X86_FEATURE_PCID
-	/* Set user PCID bit, and move CR3 up a page to the user page tables: */
-	orq     $(KAISER_SWITCH_MASK), \reg
-.endm
-
 .macro SWITCH_TO_KERNEL_CR3 scratch_reg:req
 	STATIC_JUMP_IF_FALSE .Lend_\@, kaiser_enabled_key, def=1
 	mov	%cr3, \scratch_reg
@@ -220,7 +215,27 @@ For 32-bit we have the following convent
 .macro SWITCH_TO_USER_CR3 scratch_reg:req
 	STATIC_JUMP_IF_FALSE .Lend_\@, kaiser_enabled_key, def=1
 	mov	%cr3, \scratch_reg
-	ADJUST_USER_CR3 \scratch_reg
+
+	/*
+	 * Test if the ASID needs a flush.
+	 */
+	push	\scratch_reg			/* preserve CR3 */
+	andq	$(0x7FF), \scratch_reg		/* mask ASID */
+	bt	\scratch_reg, PER_CPU_VAR(user_asid_flush_mask)
+	jnc	.Lnoflush_\@
+
+	/* Flush needed, clear the bit */
+	btr	\scratch_reg, PER_CPU_VAR(user_asid_flush_mask)
+	pop	\scratch_reg			/* original CR3 */
+	jmp	.Ldo_\@
+
+.Lnoflush_\@:
+	pop	\scratch_reg			/* original CR3 */
+	ALTERNATIVE "", "bts $63, \scratch_reg", X86_FEATURE_PCID
+
+.Ldo_\@:
+	/* Flip the PGD and ASID to the user version */
+	orq     $(KAISER_SWITCH_MASK), \scratch_reg
 	mov	\scratch_reg, %cr3
 .Lend_\@:
 .endm
--- a/arch/x86/include/asm/tlbflush.h
+++ b/arch/x86/include/asm/tlbflush.h
@@ -348,19 +348,37 @@ static inline void cr4_set_bits_and_upda
 
 extern void initialize_tlbstate_and_flush(void);
 
+DECLARE_PER_CPU(unsigned long, user_asid_flush_mask);
+
+/*
+ * Given an ASID, flush the corresponding user ASID.
+ * We can delay this until the next time we switch to it.
+ *
+ * See SWITCH_TO_USER_CR3.
+ */
+static inline void flush_user_asid(u16 asid)
+{
+	/* There is no user ASID if KAISER is off */
+	if (!IS_ENABLED(CONFIG_KAISER))
+		return;
+
+	/*
+	 * We only have a single ASID if PCID is off and the CR3
+	 * write will have flushed it.
+	 */
+	if (!cpu_feature_enabled(X86_FEATURE_PCID))
+		return;
+
+	if (!kaiser_enabled)
+		return;
+
+	__set_bit(kern_asid(asid), this_cpu_ptr(&user_asid_flush_mask));
+}
+
 static inline void __native_flush_tlb(void)
 {
 	if (!cpu_feature_enabled(X86_FEATURE_INVPCID)) {
-		/*
-		 * native_write_cr3() only clears the current PCID if
-		 * CR4 has X86_CR4_PCIDE set.  In other words, this does
-		 * not fully flush the TLB if PCIDs are in use.
-		 *
-		 * With KAISER and PCIDs, that means that we did not
-		 * flush the user PCID.  Warn if it gets called.
-		 */
-		if (IS_ENABLED(CONFIG_KAISER) && kaiser_enabled)
-			WARN_ON_ONCE(this_cpu_read(cpu_tlbstate.cr4) & X86_CR4_PCIDE);
+		flush_user_asid(this_cpu_read(cpu_tlbstate.loaded_mm_asid));
 
 		/*
 		 * If current->mm == NULL then we borrow a mm
@@ -436,6 +454,7 @@ static inline void __native_flush_tlb_si
 	 * early.
 	 */
 	if (!this_cpu_has(X86_FEATURE_INVPCID_SINGLE)) {
+		flush_user_asid(loaded_mm_asid);
 		asm volatile("invlpg (%0)" ::"r" (addr) : "memory");
 		return;
 	}
--- a/arch/x86/mm/init.c
+++ b/arch/x86/mm/init.c
@@ -211,19 +211,6 @@ static void setup_pcid(void)
 
 	if (boot_cpu_has(X86_FEATURE_PGE)) {
 		/*
-		 * KAISER uses a PCID for the kernel and another
-		 * for userspace.  Both PCIDs need to be flushed
-		 * when the TLB flush functions are called.  But,
-		 * flushing *another* PCID is insane without
-		 * INVPCID.  Just avoid using PCIDs at all if we
-		 * have KAISER and do not have INVPCID.
-		 */
-		if (!IS_ENABLED(CONFIG_X86_GLOBAL_PAGES) &&
-		    kaiser_enabled && !boot_cpu_has(X86_FEATURE_INVPCID)) {
-			setup_clear_cpu_cap(X86_FEATURE_PCID);
-			return;
-		}
-		/*
 		 * This can't be cr4_set_bits_and_update_boot() --
 		 * the trampoline code can't handle CR4.PCIDE and
 		 * it wouldn't do any good anyway.  Despite the name,
--- a/arch/x86/mm/tlb.c
+++ b/arch/x86/mm/tlb.c
@@ -101,59 +101,14 @@ static void choose_new_asid(struct mm_st
 	*need_flush = true;
 }
 
-/*
- * Given a kernel asid, flush the corresponding KAISER
- * user ASID.
- */
-static void flush_user_asid(pgd_t *pgd, u16 kern_asid)
-{
-	/* There is no user ASID if KAISER is off */
-	if (!IS_ENABLED(CONFIG_KAISER))
-		return;
-	/*
-	 * We only have a single ASID if PCID is off and the CR3
-	 * write will have flushed it.
-	 */
-	if (!cpu_feature_enabled(X86_FEATURE_PCID))
-		return;
-
-	if (!kaiser_enabled)
-		return;
-
-	/*
-	 * With PCIDs enabled, write_cr3() only flushes TLB
-	 * entries for the current (kernel) ASID.  This leaves
-	 * old TLB entries for the user ASID in place and we must
-	 * flush that context separately.  We can theoretically
-	 * delay doing this until we actually load up the
-	 * userspace CR3, but do it here for simplicity.
-	 */
-	if (cpu_feature_enabled(X86_FEATURE_INVPCID)) {
-		invpcid_flush_single_context(user_asid(kern_asid));
-	} else {
-		/*
-		 * On systems with PCIDs, but no INVPCID, the only
-		 * way to flush a PCID is a CR3 write.  Note that
-		 * we use the kernel page tables with the *user*
-		 * ASID here.
-		 */
-		unsigned long user_asid_flush_cr3;
-		user_asid_flush_cr3 = build_cr3(pgd, user_asid(kern_asid));
-		write_cr3(user_asid_flush_cr3);
-		/*
-		 * We do not use PCIDs with KAISER unless we also
-		 * have INVPCID.  Getting here is unexpected.
-		 */
-		WARN_ON_ONCE(1);
-	}
-}
+__visible DEFINE_PER_CPU(unsigned long, user_asid_flush_mask);
 
 static void load_new_mm_cr3(pgd_t *pgdir, u16 new_asid, bool need_flush)
 {
 	unsigned long new_mm_cr3;
 
 	if (need_flush) {
-		flush_user_asid(pgdir, new_asid);
+		flush_user_asid(new_asid);
 		new_mm_cr3 = build_cr3(pgdir, new_asid);
 	} else {
 		new_mm_cr3 = build_cr3_noflush(pgdir, new_asid);

[PATCH 5/6] x86/mm/kaiser: Optimize RESTORE_CR3

[Peter Zijlstra]

[-- Attachment #1: peterz-kaiser-optimize-restore_cr3.patch --]
[-- Type: text/plain, Size: 2289 bytes --]

Currently RESTORE_CR3 does an unconditional flush
(SAVE_AND_SWITCH_TO_KERNEL_CR3 does not set bit 63 on \save_reg).

When restoring to a user ASID, check the user_asid_flush_mask to see
if we can avoid the flush.

For kernel ASIDs we can unconditionaly avoid the flush, since we do
explicit flushes for them.

Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
---
 arch/x86/entry/calling.h  |   29 +++++++++++++++++++++++++++--
 arch/x86/entry/entry_64.S |    4 ++--
 2 files changed, 29 insertions(+), 4 deletions(-)

--- a/arch/x86/entry/calling.h
+++ b/arch/x86/entry/calling.h
@@ -263,8 +263,33 @@ For 32-bit we have the following convent
 .Ldone_\@:
 .endm
 
-.macro RESTORE_CR3 save_reg:req
+.macro RESTORE_CR3 scratch_reg:req save_reg:req
 	STATIC_JUMP_IF_FALSE .Lend_\@, kaiser_enabled_key, def=1
+
+	/* ASID bit 11 is for user */
+	bt	$11, \save_reg
+	/*
+	 * KERNEL pages can always resume with NOFLUSH as we do
+	 * explicit flushes.
+	 */
+	jnc	.Lnoflush_\@
+
+	/*
+	 * Check if there's a pending flush for the user ASID we're
+	 * about to set.
+	 */
+	movq	\save_reg, \scratch_reg
+	andq	$(0x7FF), \scratch_reg
+	bt	\scratch_reg, PER_CPU_VAR(user_asid_flush_mask)
+	jnc	.Lnoflush_\@
+
+	btr	\scratch_reg, PER_CPU_VAR(user_asid_flush_mask)
+	jmp	.Ldo_\@
+
+.Lnoflush_\@:
+	ALTERNATIVE "", "bts $63, \save_reg", X86_FEATURE_PCID
+
+.Ldo_\@:
 	/*
 	 * The CR3 write could be avoided when not changing its value,
 	 * but would require a CR3 read *and* a scratch register.
@@ -281,7 +306,7 @@ For 32-bit we have the following convent
 .endm
 .macro SAVE_AND_SWITCH_TO_KERNEL_CR3 scratch_reg:req save_reg:req
 .endm
-.macro RESTORE_CR3 save_reg:req
+.macro RESTORE_CR3 scratch_reg:req save_reg:req
 .endm
 
 #endif
--- a/arch/x86/entry/entry_64.S
+++ b/arch/x86/entry/entry_64.S
@@ -1278,7 +1278,7 @@ ENTRY(paranoid_exit)
 	testl	%ebx, %ebx			/* swapgs needed? */
 	jnz	.Lparanoid_exit_no_swapgs
 	TRACE_IRQS_IRETQ
-	RESTORE_CR3	save_reg=%r14
+	RESTORE_CR3	scratch_reg=%rbx save_reg=%r14
 	SWAPGS_UNSAFE_STACK
 	jmp	.Lparanoid_exit_restore
 .Lparanoid_exit_no_swapgs:
@@ -1720,7 +1720,7 @@ ENTRY(nmi)
 	movq	$-1, %rsi
 	call	do_nmi
 
-	RESTORE_CR3 save_reg=%r14
+	RESTORE_CR3 scratch_reg=%r15 save_reg=%r14
 
 	testl	%ebx, %ebx			/* swapgs needed? */
 	jnz	nmi_restore

[PATCH 6/6] x86/mm/kaiser: Optimize __native_flush_tlb

[Peter Zijlstra]

[-- Attachment #1: peterz-kaiser-opt-tlb.patch --]
[-- Type: text/plain, Size: 1842 bytes --]

Now that we have lazy user asid flushing, use that even if we have
INVPCID. Even if INVPCID would not be slower than a flushing CR3 write
(it is) this allows folding multiple user flushes.

Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
---
 arch/x86/include/asm/tlbflush.h |   38 ++++++++++++++------------------------
 1 file changed, 14 insertions(+), 24 deletions(-)

--- a/arch/x86/include/asm/tlbflush.h
+++ b/arch/x86/include/asm/tlbflush.h
@@ -377,33 +377,23 @@ static inline void flush_user_asid(u16 a
 
 static inline void __native_flush_tlb(void)
 {
-	if (!cpu_feature_enabled(X86_FEATURE_INVPCID)) {
-		flush_user_asid(this_cpu_read(cpu_tlbstate.loaded_mm_asid));
+	flush_user_asid(this_cpu_read(cpu_tlbstate.loaded_mm_asid));
 
-		/*
-		 * If current->mm == NULL then we borrow a mm
-		 * which may change during a task switch and
-		 * therefore we must not be preempted while we
-		 * write CR3 back:
-		 */
-		preempt_disable();
-		native_write_cr3(__native_read_cr3());
-		preempt_enable();
-		/*
-		 * Does not need tlb_flush_shared_nonglobals()
-		 * since the CR3 write without PCIDs flushes all
-		 * non-globals.
-		 */
-		return;
-	}
 	/*
-	 * We are no longer using globals with KAISER, so a
-	 * "nonglobals" flush would work too. But, this is more
-	 * conservative.
-	 *
-	 * Note, this works with CR4.PCIDE=0 or 1.
+	 * If current->mm == NULL then we borrow a mm
+	 * which may change during a task switch and
+	 * therefore we must not be preempted while we
+	 * write CR3 back:
 	 */
-	invpcid_flush_all();
+	preempt_disable();
+	native_write_cr3(__native_read_cr3());
+	preempt_enable();
+	/*
+	 * Does not need tlb_flush_shared_nonglobals()
+	 * since the CR3 write without PCIDs flushes all
+	 * non-globals.
+	 */
+	return;
 }
 
 static inline void __native_flush_tlb_global_irq_disabled(void)

Re: [PATCH 4/6] x86/mm/kaiser: Support PCID without INVPCID

[Peter Zijlstra]

On Wed, Nov 29, 2017 at 11:33:05AM +0100, Peter Zijlstra wrote:
> +DECLARE_PER_CPU(unsigned long, user_asid_flush_mask);

Ah, I meant to make that: DECLARE_BITMAP(TLB_NR_DYN_ASIDS)

Re: [PATCH 4/6] x86/mm/kaiser: Support PCID without INVPCID

[Peter Zijlstra]

On Wed, Nov 29, 2017 at 11:33:05AM +0100, Peter Zijlstra wrote:

> XXX we could do a much larger ALTERNATIVE, there is no point in
> testing the mask if we don't have PCID support.

This.

> @@ -220,7 +215,27 @@ For 32-bit we have the following convent
>  .macro SWITCH_TO_USER_CR3 scratch_reg:req
>  	STATIC_JUMP_IF_FALSE .Lend_\@, kaiser_enabled_key, def=1
>  	mov	%cr3, \scratch_reg
> -	ADJUST_USER_CR3 \scratch_reg
> +
> +	/*
> +	 * Test if the ASID needs a flush.
> +	 */
> +	push	\scratch_reg			/* preserve CR3 */
> +	andq	$(0x7FF), \scratch_reg		/* mask ASID */
> +	bt	\scratch_reg, PER_CPU_VAR(user_asid_flush_mask)
> +	jnc	.Lnoflush_\@
> +
> +	/* Flush needed, clear the bit */
> +	btr	\scratch_reg, PER_CPU_VAR(user_asid_flush_mask)
> +	pop	\scratch_reg			/* original CR3 */
> +	jmp	.Ldo_\@
> +
> +.Lnoflush_\@:
> +	pop	\scratch_reg			/* original CR3 */
> +	ALTERNATIVE "", "bts $63, \scratch_reg", X86_FEATURE_PCID
> +
> +.Ldo_\@:
> +	/* Flip the PGD and ASID to the user version */
> +	orq     $(KAISER_SWITCH_MASK), \scratch_reg
>  	mov	\scratch_reg, %cr3
>  .Lend_\@:
>  .endm

Something like so seems to actually compile and generate sensible code,
not tested it though.


--- a/arch/x86/entry/calling.h
+++ b/arch/x86/entry/calling.h
@@ -216,6 +216,8 @@ For 32-bit we have the following convent
 	STATIC_JUMP_IF_FALSE .Lend_\@, kaiser_enabled_key, def=1
 	mov	%cr3, \scratch_reg
 
+	ALTERNATIVE "jmp .Ldo_\@", "", X86_FEATURE_PCID
+
 	/*
 	 * Test if the ASID needs a flush.
 	 */
@@ -231,7 +233,7 @@ For 32-bit we have the following convent
 
 .Lnoflush_\@:
 	pop	\scratch_reg			/* original CR3 */
-	ALTERNATIVE "", "bts $63, \scratch_reg", X86_FEATURE_PCID
+	bts	$63, \scratch_reg
 
 .Ldo_\@:
 	/* Flip the PGD and ASID to the user version */
@@ -266,6 +268,8 @@ For 32-bit we have the following convent
 .macro RESTORE_CR3 scratch_reg:req save_reg:req
 	STATIC_JUMP_IF_FALSE .Lend_\@, kaiser_enabled_key, def=1
 
+	ALTERNATIVE "jmp .Ldo_\@", "", X86_FEATURE_PCID
+
 	/* ASID bit 11 is for user */
 	bt	$11, \save_reg
 	/*
@@ -287,7 +291,7 @@ For 32-bit we have the following convent
 	jmp	.Ldo_\@
 
 .Lnoflush_\@:
-	ALTERNATIVE "", "bts $63, \save_reg", X86_FEATURE_PCID
+	bts	$63, \save_reg
 
 .Ldo_\@:
 	/*

Re: [PATCH 4/6] x86/mm/kaiser: Support PCID without INVPCID

[Peter Zijlstra]

On Wed, Nov 29, 2017 at 11:33:05AM +0100, Peter Zijlstra wrote:
> @@ -220,7 +215,27 @@ For 32-bit we have the following convent
>  .macro SWITCH_TO_USER_CR3 scratch_reg:req
>  	STATIC_JUMP_IF_FALSE .Lend_\@, kaiser_enabled_key, def=1
>  	mov	%cr3, \scratch_reg
> -	ADJUST_USER_CR3 \scratch_reg
> +
> +	/*
> +	 * Test if the ASID needs a flush.
> +	 */
> +	push	\scratch_reg			/* preserve CR3 */

So I was just staring at disasm of a few functions and I noticed this
one reads like push, while others read like pushq.

So does the stupid assembler thing really do a 32bit push if you provide
it with a 64bit register?

> +	andq	$(0x7FF), \scratch_reg		/* mask ASID */
> +	bt	\scratch_reg, PER_CPU_VAR(user_asid_flush_mask)
> +	jnc	.Lnoflush_\@
> +
> +	/* Flush needed, clear the bit */
> +	btr	\scratch_reg, PER_CPU_VAR(user_asid_flush_mask)
> +	pop	\scratch_reg			/* original CR3 */
> +	jmp	.Ldo_\@
> +
> +.Lnoflush_\@:
> +	pop	\scratch_reg			/* original CR3 */
> +	ALTERNATIVE "", "bts $63, \scratch_reg", X86_FEATURE_PCID
> +
> +.Ldo_\@:
> +	/* Flip the PGD and ASID to the user version */
> +	orq     $(KAISER_SWITCH_MASK), \scratch_reg
>  	mov	\scratch_reg, %cr3
>  .Lend_\@:
>  .endm

Re: [PATCH 4/6] x86/mm/kaiser: Support PCID without INVPCID

[Peter Zijlstra]

On Wed, Nov 29, 2017 at 01:31:58PM +0100, Peter Zijlstra wrote:
> On Wed, Nov 29, 2017 at 11:33:05AM +0100, Peter Zijlstra wrote:
> > @@ -220,7 +215,27 @@ For 32-bit we have the following convent
> >  .macro SWITCH_TO_USER_CR3 scratch_reg:req
> >  	STATIC_JUMP_IF_FALSE .Lend_\@, kaiser_enabled_key, def=1
> >  	mov	%cr3, \scratch_reg
> > -	ADJUST_USER_CR3 \scratch_reg
> > +
> > +	/*
> > +	 * Test if the ASID needs a flush.
> > +	 */
> > +	push	\scratch_reg			/* preserve CR3 */
> 
> So I was just staring at disasm of a few functions and I noticed this
> one reads like push, while others read like pushq.
> 
> So does the stupid assembler thing really do a 32bit push if you provide
> it with a 64bit register?

N/m, I just really cannot read straight today. The pushq's were a mem64,
not a r64 argument to push.

Re: [PATCH 0/6] more KAISER bits

[Thomas Gleixner]

On Wed, 29 Nov 2017, Peter Zijlstra wrote:

> Here's more patches, includes the TLB invalidate rework.
> 
> Has not actually been tested on a INVPCID machine yet, but does seem to
> work fine on my IVB-EP (which is PCID only).

I've picked all of that up including other bits and pieces which have been
circulated in various threads.

Note that the next series will have

 - a rename as lots of people complained about KAISER

 - stuff folded back where ever it makes sense
 
 - reordering of the queue to put fixes and preparatory changes first

Will take a bit, but this needs to be done before I completely drown in
conflicting patches and patch snippets of all sorts.

Thanks,

	tglx

Re: [PATCH 0/6] more KAISER bits

[Thomas Gleixner]

On Wed, 29 Nov 2017, Thomas Gleixner wrote:

> On Wed, 29 Nov 2017, Peter Zijlstra wrote:
> 
> > Here's more patches, includes the TLB invalidate rework.
> > 
> > Has not actually been tested on a INVPCID machine yet, but does seem to
> > work fine on my IVB-EP (which is PCID only).
> 
> I've picked all of that up including other bits and pieces which have been
> circulated in various threads.
> 
> Note that the next series will have
> 
>  - a rename as lots of people complained about KAISER
> 
>  - stuff folded back where ever it makes sense
>  
>  - reordering of the queue to put fixes and preparatory changes first
> 
> Will take a bit, but this needs to be done before I completely drown in
> conflicting patches and patch snippets of all sorts.

Current pile at:

	https://tglx.de/~tglx/patches.tar

I need to walk the dogs and run some errands. Will finish and post later
tonight.

Thanks,

	tglx

Re: [PATCH 0/6] more KAISER bits

[Dave Hansen]

On 11/29/2017 08:02 AM, Thomas Gleixner wrote:
> Current pile at:
> 
> 	https://tglx.de/~tglx/patches.tar

I don't see any show stoppers in there.  The biggest change is Peter's
rework of the user asid flushing, but that all looks OK to me.

Re: [PATCH 5/6] x86/mm/kaiser: Optimize RESTORE_CR3

[Borislav Petkov]

On Wed, Nov 29, 2017 at 11:33:06AM +0100, Peter Zijlstra wrote:
> Currently RESTORE_CR3 does an unconditional flush
> (SAVE_AND_SWITCH_TO_KERNEL_CR3 does not set bit 63 on \save_reg).
> 
> When restoring to a user ASID, check the user_asid_flush_mask to see
> if we can avoid the flush.
> 
> For kernel ASIDs we can unconditionaly avoid the flush, since we do
> explicit flushes for them.
> 
> Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
> ---
>  arch/x86/entry/calling.h  |   29 +++++++++++++++++++++++++++--
>  arch/x86/entry/entry_64.S |    4 ++--
>  2 files changed, 29 insertions(+), 4 deletions(-)
> 
> --- a/arch/x86/entry/calling.h
> +++ b/arch/x86/entry/calling.h
> @@ -263,8 +263,33 @@ For 32-bit we have the following convent
>  .Ldone_\@:
>  .endm
>  
> -.macro RESTORE_CR3 save_reg:req
> +.macro RESTORE_CR3 scratch_reg:req save_reg:req
>  	STATIC_JUMP_IF_FALSE .Lend_\@, kaiser_enabled_key, def=1
> +
> +	/* ASID bit 11 is for user */
> +	bt	$11, \save_reg

<---- newline here.

> +	/*
> +	 * KERNEL pages can always resume with NOFLUSH as we do
> +	 * explicit flushes.
> +	 */
> +	jnc	.Lnoflush_\@
> +
> +	/*
> +	 * Check if there's a pending flush for the user ASID we're
> +	 * about to set.
> +	 */
> +	movq	\save_reg, \scratch_reg
> +	andq	$(0x7FF), \scratch_reg
> +	bt	\scratch_reg, PER_CPU_VAR(user_asid_flush_mask)
> +	jnc	.Lnoflush_\@
> +
> +	btr	\scratch_reg, PER_CPU_VAR(user_asid_flush_mask)
> +	jmp	.Ldo_\@

Can you save yourself one of the BT-insns?

	...
	andq	$(0x7FF), \scratch_reg
	btr     \scratch_reg, PER_CPU_VAR(user_asid_flush_mask)
	jnc	.Lnoflush_\@
	jmp     .Ldo_\@
	...

or am I missing a case?

-- 
Regards/Gruss,
    Boris.

Good mailing practices for 400: avoid top-posting and trim the reply.

Re: [PATCH 5/6] x86/mm/kaiser: Optimize RESTORE_CR3

[Peter Zijlstra]

On Wed, Nov 29, 2017 at 09:02:12PM +0100, Borislav Petkov wrote:
> On Wed, Nov 29, 2017 at 11:33:06AM +0100, Peter Zijlstra wrote:
> > +.macro RESTORE_CR3 scratch_reg:req save_reg:req
> >  	STATIC_JUMP_IF_FALSE .Lend_\@, kaiser_enabled_key, def=1
> > +
> > +	/* ASID bit 11 is for user */
> > +	bt	$11, \save_reg
> 
> <---- newline here.

Seems weird to me, the bt and jnc are a pair.

> > +	/*
> > +	 * KERNEL pages can always resume with NOFLUSH as we do
> > +	 * explicit flushes.
> > +	 */
> > +	jnc	.Lnoflush_\@
> > +
> > +	/*
> > +	 * Check if there's a pending flush for the user ASID we're
> > +	 * about to set.
> > +	 */
> > +	movq	\save_reg, \scratch_reg
> > +	andq	$(0x7FF), \scratch_reg
> > +	bt	\scratch_reg, PER_CPU_VAR(user_asid_flush_mask)
> > +	jnc	.Lnoflush_\@
> > +
> > +	btr	\scratch_reg, PER_CPU_VAR(user_asid_flush_mask)
> > +	jmp	.Ldo_\@
> 
> Can you save yourself one of the BT-insns?
> 
> 	...
> 	andq	$(0x7FF), \scratch_reg
> 	btr     \scratch_reg, PER_CPU_VAR(user_asid_flush_mask)
> 	jnc	.Lnoflush_\@
> 	jmp     .Ldo_\@
> 	...
> 
> or am I missing a case?

BTR is an unconditional write and will modify the line and cause a
write-back later. The common case is the bit not set, so BT, which is a
pure read, avoids all that overhead.

Re: [PATCH 6/6] x86/mm/kaiser: Optimize __native_flush_tlb

[Peter Zijlstra]

On Wed, Nov 29, 2017 at 11:33:07AM +0100, Peter Zijlstra wrote:

>  static inline void __native_flush_tlb(void)
>  {
> +	flush_user_asid(this_cpu_read(cpu_tlbstate.loaded_mm_asid));
>  
>  	/*
> +	 * If current->mm == NULL then we borrow a mm
> +	 * which may change during a task switch and
> +	 * therefore we must not be preempted while we
> +	 * write CR3 back:
>  	 */
> +	preempt_disable();
> +	native_write_cr3(__native_read_cr3());
> +	preempt_enable();



> +	/*
> +	 * Does not need tlb_flush_shared_nonglobals()
> +	 * since the CR3 write without PCIDs flushes all
> +	 * non-globals.
> +	 */
> +	return;

OK, so seeing that comment today made me realize I had so far failed to
audit the whole flush user vs flush kernel thing.

In short the above comment is complete crap.

>  }


The longer story is that:

  flush_tlb_all()
  flush_tlb_kernel_range()

need to flush kernel pages and thus flush _all_ the (kernel) ASIDs.

Whereas:

  flush_tlb_mm()
  flush_tlb_range()
  flush_tlb_page()

Only flush user pages, and thus only need to flush the respective user
and kernel ASID.

The last 3 all map to flush_tlb_mm_range() which, through
flush_tlb_func_{local,remote} ends up in flush_tlb_func_common(), which
then uses either __flush_tlb() or __flush_tlb_single().

Both __flush_tlb() (the above function) and __flush_tlb_single() only
(need to) flush the 2 ASIDs that contain the user mapping.

Now the problem is that flush_tlb_kernel_range() is implemented using
either __flush_tlb_all() or __flush_tlb_single(), and it is that last
use that is buggered.

So at the very least we need the below to cure things, but there is
another inconsistency; do_flush_tlb_all() is used by both
flush_tlb_all() and flush_tlb_kernel_range() and increments NR_TLB_*,
do_kernel_range_flush() OTOH does not increment NR_TLB_*. I'm not fixing
that, but I'll leave a comment around or something, so we can later try
and figure out what exact statistics we want.

---

diff --git a/arch/x86/include/asm/tlbflush.h b/arch/x86/include/asm/tlbflush.h
index 9587722162ee..ccaf6e126582 100644
--- a/arch/x86/include/asm/tlbflush.h
+++ b/arch/x86/include/asm/tlbflush.h
@@ -388,12 +388,6 @@ static inline void __native_flush_tlb(void)
 	preempt_disable();
 	native_write_cr3(__native_read_cr3());
 	preempt_enable();
-	/*
-	 * Does not need tlb_flush_shared_nonglobals()
-	 * since the CR3 write without PCIDs flushes all
-	 * non-globals.
-	 */
-	return;
 }
 
 static inline void __native_flush_tlb_global_irq_disabled(void)
diff --git a/arch/x86/mm/tlb.c b/arch/x86/mm/tlb.c
index 122c48fa6012..24bd86118b46 100644
--- a/arch/x86/mm/tlb.c
+++ b/arch/x86/mm/tlb.c
@@ -609,6 +609,8 @@ static void do_kernel_range_flush(void *info)
 	/* flush range by one by one 'invlpg' */
 	for (addr = f->start; addr < f->end; addr += PAGE_SIZE)
 		__flush_tlb_single(addr);
+
+	tlb_flush_shared_nonglobals();
 }
 
 void flush_tlb_kernel_range(unsigned long start, unsigned long end)

Re: [PATCH 6/6] x86/mm/kaiser: Optimize __native_flush_tlb

[Peter Zijlstra]

On Thu, Nov 30, 2017 at 01:43:19PM +0100, Peter Zijlstra wrote:
> Now the problem is that flush_tlb_kernel_range() is implemented using
> either __flush_tlb_all() or __flush_tlb_single(), and it is that last
> use that is buggered.
> 
> So at the very least we need the below to cure things, but there is
> another inconsistency; do_flush_tlb_all() is used by both
> flush_tlb_all() and flush_tlb_kernel_range() and increments NR_TLB_*,
> do_kernel_range_flush() OTOH does not increment NR_TLB_*. I'm not fixing
> that, but I'll leave a comment around or something, so we can later try
> and figure out what exact statistics we want.

Alternatively, we'd simply kill the entire invlpg path for
flush_tlb_kernel_range() and simply do __flush_tlb_all().