Re: [patch 02/14] tmpfs: fix regressions from wider use of ZERO_PAGE

[Borislav Petkov <bp@alien8.de>]

On Sat, Apr 16, 2022 at 11:15:16PM +0200, Borislav Petkov wrote:
> > but wouldn't it be lovely if we could start moving towards a model
> > where we can just inline 'memset' and 'memcpy' like this?
> 
> Yeah, inlined insns without even a CALL insn would be the most optimal
> thing to do.

Here's a diff ontop of Mark's patch, it inlines clear_user() everywhere
and it seems to patch properly

vmlinux has:

ffffffff812ba53e:       0f 87 f8 fd ff ff       ja     ffffffff812ba33c <load_elf_binary+0x60c>
ffffffff812ba544:       90                      nop
ffffffff812ba545:       90                      nop
ffffffff812ba546:       90                      nop
ffffffff812ba547:       4c 89 e7                mov    %r12,%rdi
ffffffff812ba54a:       f3 aa                   rep stos %al,%es:(%rdi)
ffffffff812ba54c:       90                      nop
ffffffff812ba54d:       90                      nop
ffffffff812ba54e:       90                      nop
ffffffff812ba54f:       90                      nop
ffffffff812ba550:       90                      nop
ffffffff812ba551:       90                      nop

which is the rep; stosb and when I boot my guest, it has been patched to:

0xffffffff812ba544 <+2068>:  0f 01 cb        stac   
0xffffffff812ba547 <+2071>:  4c 89 e7        mov    %r12,%rdi
0xffffffff812ba54a <+2074>:  e8 71 9a 15 00  call   0xffffffff81413fc0 <clear_user_rep_good>
0xffffffff812ba54f <+2079>:  0f 01 ca        clac

(stac and clac are also alternative-patched). And there's the call to
rep_good because the guest doesn't advertize FRSM nor ERMS.

Anyway, more playing with this later to make sure it really does what it
should.

---

diff --git a/arch/x86/include/asm/uaccess.h b/arch/x86/include/asm/uaccess.h
index f78e2b3501a1..335d571e8a79 100644
--- a/arch/x86/include/asm/uaccess.h
+++ b/arch/x86/include/asm/uaccess.h
@@ -405,9 +405,6 @@ strncpy_from_user(char *dst, const char __user *src, long count);
 
 extern __must_check long strnlen_user(const char __user *str, long n);
 
-unsigned long __must_check clear_user(void __user *mem, unsigned long len);
-unsigned long __must_check __clear_user(void __user *mem, unsigned long len);
-
 #ifdef CONFIG_ARCH_HAS_COPY_MC
 unsigned long __must_check
 copy_mc_to_kernel(void *to, const void *from, unsigned len);
@@ -429,6 +426,8 @@ extern struct movsl_mask {
 #define ARCH_HAS_NOCACHE_UACCESS 1
 
 #ifdef CONFIG_X86_32
+unsigned long __must_check clear_user(void __user *mem, unsigned long len);
+unsigned long __must_check __clear_user(void __user *mem, unsigned long len);
 # include <asm/uaccess_32.h>
 #else
 # include <asm/uaccess_64.h>
diff --git a/arch/x86/include/asm/uaccess_64.h b/arch/x86/include/asm/uaccess_64.h
index 6a4995e4cfae..dc0f2bfc80a4 100644
--- a/arch/x86/include/asm/uaccess_64.h
+++ b/arch/x86/include/asm/uaccess_64.h
@@ -91,28 +91,34 @@ clear_user_rep_good(void __user *addr, unsigned long len);
 __must_check unsigned long
 clear_user_erms(void __user *addr, unsigned long len);
 
-static __always_inline __must_check unsigned long
-___clear_user(void __user *addr, unsigned long len)
+static __always_inline __must_check unsigned long __clear_user(void __user *addr, unsigned long size)
 {
-	unsigned long	ret;
-
 	/*
 	 * No memory constraint because it doesn't change any memory gcc
 	 * knows about.
 	 */
 
 	might_fault();
-	alternative_call_2(
-		clear_user_original,
-		clear_user_rep_good,
-		X86_FEATURE_REP_GOOD,
-		clear_user_erms,
-		X86_FEATURE_ERMS,
-		ASM_OUTPUT2("=a" (ret), "=D" (addr), "=c" (len)),
-		"1" (addr), "2" (len)
-		: "%rdx", "cc");
-	return ret;
+	stac();
+	asm_inline volatile(
+		"1:"
+		ALTERNATIVE_3("rep stosb",
+			      "call clear_user_erms",	  ALT_NOT(X86_FEATURE_FSRM),
+			      "call clear_user_rep_good", ALT_NOT(X86_FEATURE_ERMS),
+			      "call clear_user_original", ALT_NOT(X86_FEATURE_REP_GOOD))
+		"2:"
+	       _ASM_EXTABLE_UA(1b, 2b)
+	       : "+&c" (size), "+&D" (addr), ASM_CALL_CONSTRAINT
+	       : "a" (0));
+
+	clac();
+	return size;
 }
 
-#define __clear_user(d, n)	___clear_user(d, n)
+static __always_inline unsigned long clear_user(void __user *to, unsigned long n)
+{
+	if (access_ok(to, n))
+		return __clear_user(to, n);
+	return n;
+}
 #endif /* _ASM_X86_UACCESS_64_H */
diff --git a/arch/x86/lib/clear_page_64.S b/arch/x86/lib/clear_page_64.S
index abe1f44ea422..b80f710a7f30 100644
--- a/arch/x86/lib/clear_page_64.S
+++ b/arch/x86/lib/clear_page_64.S
@@ -62,9 +62,7 @@ EXPORT_SYMBOL_GPL(clear_page_erms)
  * Output:
  * rax uncopied bytes or 0 if successful.
  */
-
 SYM_FUNC_START(clear_user_original)
-	ASM_STAC
 	movq %rcx,%rax
 	shrq $3,%rcx
 	andq $7,%rax
@@ -86,7 +84,7 @@ SYM_FUNC_START(clear_user_original)
 	decl %ecx
 	jnz  2b
 
-3:	ASM_CLAC
+3:
 	movq %rcx,%rax
 	RET
 
@@ -105,11 +103,8 @@ EXPORT_SYMBOL(clear_user_original)
  * Output:
  * rax uncopied bytes or 0 if successful.
  */
-
 SYM_FUNC_START(clear_user_rep_good)
-	ASM_STAC
 	movq %rcx,%rdx
-	xorq %rax,%rax
 	shrq $3,%rcx
 	andq $7,%rdx
 
@@ -118,7 +113,7 @@ SYM_FUNC_START(clear_user_rep_good)
 
 1:	rep stosb
 
-3:	ASM_CLAC
+3:
 	movq %rcx,%rax
 	RET
 
@@ -135,15 +130,13 @@ EXPORT_SYMBOL(clear_user_rep_good)
  *
  * Output:
  * rax uncopied bytes or 0 if successful.
+ *
+ * XXX: check for small sizes and call the original version.
+ * Benchmark it first though.
  */
-
 SYM_FUNC_START(clear_user_erms)
-	xorq %rax,%rax
-	ASM_STAC
-
 0:	rep stosb
-
-3:	ASM_CLAC
+3:
 	movq %rcx,%rax
 	RET
 
diff --git a/arch/x86/lib/usercopy_64.c b/arch/x86/lib/usercopy_64.c
index 3a2872c9c4a9..8bf04d95dc04 100644
--- a/arch/x86/lib/usercopy_64.c
+++ b/arch/x86/lib/usercopy_64.c
@@ -14,14 +14,6 @@
  * Zero Userspace
  */
 
-unsigned long clear_user(void __user *to, unsigned long n)
-{
-	if (access_ok(to, n))
-		return __clear_user(to, n);
-	return n;
-}
-EXPORT_SYMBOL(clear_user);
-
 #ifdef CONFIG_ARCH_HAS_UACCESS_FLUSHCACHE
 /**
  * clean_cache_range - write back a cache range with CLWB
diff --git a/tools/objtool/check.c b/tools/objtool/check.c
index bd0c2c828940..ea48ee11521f 100644
--- a/tools/objtool/check.c
+++ b/tools/objtool/check.c
@@ -1019,6 +1019,9 @@ static const char *uaccess_safe_builtin[] = {
 	"copy_mc_fragile_handle_tail",
 	"copy_mc_enhanced_fast_string",
 	"ftrace_likely_update", /* CONFIG_TRACE_BRANCH_PROFILING */
+	"clear_user_erms",
+	"clear_user_rep_good",
+	"clear_user_original",
 	NULL
 };
 

-- 
Regards/Gruss,
    Boris.

https://people.kernel.org/tglx/notes-about-netiquette