From f9ce989826f283efc429be41caded7c885b8950c Mon Sep 17 00:00:00 2001 From: Sedat Dilek Date: Sat, 28 May 2022 17:54:40 +0200 Subject: [PATCH] x86/usercopy: speed up 64-bit __clear_user() with stos{b,q} From: Samuel Neves Link: https://lore.kernel.org/lkml/20210523180423.108087-1-sneves@dei.uc.pt/ --- arch/x86/lib/usercopy_64.c | 61 +++++++++++++++++++++++++------------- 1 file changed, 41 insertions(+), 20 deletions(-) diff --git a/arch/x86/lib/usercopy_64.c b/arch/x86/lib/usercopy_64.c index 0ae6cf804197..799db5536d25 100644 --- a/arch/x86/lib/usercopy_64.c +++ b/arch/x86/lib/usercopy_64.c @@ -9,6 +9,7 @@ #include #include #include +#include /* * Zero Userspace @@ -16,31 +17,51 @@ unsigned long __clear_user(void __user *addr, unsigned long size) { - long __d0; + long __d0, __d1; might_fault(); /* no memory constraint because it doesn't change any memory gcc knows about */ stac(); asm volatile( - " testq %[size8],%[size8]\n" - " jz 4f\n" - " .align 16\n" - "0: movq $0,(%[dst])\n" - " addq $8,%[dst]\n" - " decl %%ecx ; jnz 0b\n" - "4: movq %[size1],%%rcx\n" - " testl %%ecx,%%ecx\n" - " jz 2f\n" - "1: movb $0,(%[dst])\n" - " incq %[dst]\n" - " decl %%ecx ; jnz 1b\n" - "2:\n" - - _ASM_EXTABLE_TYPE_REG(0b, 2b, EX_TYPE_UCOPY_LEN8, %[size1]) - _ASM_EXTABLE_UA(1b, 2b) - - : [size8] "=&c"(size), [dst] "=&D" (__d0) - : [size1] "r"(size & 7), "[size8]" (size / 8), "[dst]"(addr)); + " cmp $256, %[size]\n" + " jae 3f\n" /* size >= 256 */ + " mov %k[size], %k[aux]\n" + " and $7, %k[aux]\n" + " shr $3, %[size]\n" + " jz 1f\n" /* size < 8 */ + ".align 16\n" + "0: movq %%rax,(%[dst])\n" + " add $8,%[dst]\n" + " dec %[size]; jnz 0b\n" + "1: mov %k[aux],%k[size]\n" + " test %k[aux], %k[aux]\n" + " jz 6f\n" + "2: movb %%al,(%[dst])\n" + " inc %[dst]\n" + " dec %k[size]; jnz 2b\n" + " jmp 6f\n" + "3: \n" + ALTERNATIVE( + "mov %k[size], %k[aux]\n" + "shr $3, %[size]\n" + "and $7, %k[aux]\n" + "4: rep stosq\n" + "mov %k[aux], %k[size]\n", + "", + X86_FEATURE_ERMS + ) + "5: rep stosb\n" + "6: \n" + ".section .fixup,\"ax\"\n" + "7: lea 0(%[aux],%[size],8),%[size]\n" + " jmp 6b\n" + ".previous\n" + _ASM_EXTABLE_UA(0b, 7b) + _ASM_EXTABLE_UA(2b, 6b) + _ASM_EXTABLE_UA(4b, 7b) + _ASM_EXTABLE_UA(5b, 6b) + : [size] "=&c"(size), [dst] "=&D" (__d0), [aux] "=&r"(__d1) + : "[size]" (size), "[dst]"(addr), "a"(0)); clac(); return size; } -- 2.36.1