From: Peter Zijlstra <peterz@infradead.org>
To: x86@kernel.org
Cc: linux-kernel@vger.kernel.org, peterz@infradead.org,
jpoimboe@redhat.com, mark.rutland@arm.com, dvyukov@google.com,
seanjc@google.com, pbonzini@redhat.com, mbenes@suse.cz
Subject: [RFC][PATCH 20/22] x86,usercopy: Remove .fixup usage
Date: Thu, 04 Nov 2021 17:47:49 +0100 [thread overview]
Message-ID: <20211104165525.708941275@infradead.org> (raw)
In-Reply-To: 20211104164729.226550532@infradead.org
Typically usercopy does whole word copies followed by a number of byte
copies to finish the tail. This means that on exception it needs to
compute the remaining length as: words*sizeof(long) + bytes.
Create a new extable handler to do just this.
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
---
arch/x86/include/asm/extable_fixup_types.h | 5 +++++
arch/x86/lib/usercopy_32.c | 28 +++++-----------------------
arch/x86/lib/usercopy_64.c | 8 +++-----
arch/x86/mm/extable.c | 9 +++++++++
4 files changed, 22 insertions(+), 28 deletions(-)
--- a/arch/x86/include/asm/extable_fixup_types.h
+++ b/arch/x86/include/asm/extable_fixup_types.h
@@ -42,4 +42,9 @@
#define EX_TYPE_FAULT_SGX 18
#define EX_TYPE_KVM_FASTOP 19
+#define EX_TYPE_UACCESS_LEN 20 /* cx := reg + imm*cx */
+#define EX_TYPE_UACCESS_LEN1 (EX_TYPE_UACCESS_LEN | EX_TYPE_IMM(1))
+#define EX_TYPE_UACCESS_LEN4 (EX_TYPE_UACCESS_LEN | EX_TYPE_IMM(4))
+#define EX_TYPE_UACCESS_LEN8 (EX_TYPE_UACCESS_LEN | EX_TYPE_IMM(8))
+
#endif
--- a/arch/x86/lib/usercopy_32.c
+++ b/arch/x86/lib/usercopy_32.c
@@ -43,11 +43,7 @@ do { \
" movl %2,%0\n" \
"1: rep; stosb\n" \
"2: " ASM_CLAC "\n" \
- ".section .fixup,\"ax\"\n" \
- "3: lea 0(%2,%0,4),%0\n" \
- " jmp 2b\n" \
- ".previous\n" \
- _ASM_EXTABLE_UA(0b, 3b) \
+ _ASM_EXTABLE_TYPE_REG(0b, 2b, EX_TYPE_UACCESS_LEN4, %2) \
_ASM_EXTABLE_UA(1b, 2b) \
: "=&c"(size), "=&D" (__d0) \
: "r"(size & 3), "0"(size / 4), "1"(addr), "a"(0)); \
@@ -149,10 +145,6 @@ __copy_user_intel(void __user *to, const
"36: movl %%eax, %0\n"
"37: rep; movsb\n"
"100:\n"
- ".section .fixup,\"ax\"\n"
- "101: lea 0(%%eax,%0,4),%0\n"
- " jmp 100b\n"
- ".previous\n"
_ASM_EXTABLE_UA(1b, 100b)
_ASM_EXTABLE_UA(2b, 100b)
_ASM_EXTABLE_UA(3b, 100b)
@@ -190,7 +182,7 @@ __copy_user_intel(void __user *to, const
_ASM_EXTABLE_UA(35b, 100b)
_ASM_EXTABLE_UA(36b, 100b)
_ASM_EXTABLE_UA(37b, 100b)
- _ASM_EXTABLE_UA(99b, 101b)
+ _ASM_EXTABLE_TYPE_REG(99b, 100b, EX_TYPE_UACCESS_LEN4, %eax)
: "=&c"(size), "=&D" (d0), "=&S" (d1)
: "1"(to), "2"(from), "0"(size)
: "eax", "edx", "memory");
@@ -255,10 +247,6 @@ static unsigned long __copy_user_intel_n
" movl %%eax,%0\n"
"7: rep; movsb\n"
"8:\n"
- ".section .fixup,\"ax\"\n"
- "9: lea 0(%%eax,%0,4),%0\n"
- " jmp 8b\n"
- ".previous\n"
_ASM_EXTABLE_UA(0b, 8b)
_ASM_EXTABLE_UA(1b, 8b)
_ASM_EXTABLE_UA(2b, 8b)
@@ -277,7 +265,7 @@ static unsigned long __copy_user_intel_n
_ASM_EXTABLE_UA(81b, 8b)
_ASM_EXTABLE_UA(14b, 8b)
_ASM_EXTABLE_UA(91b, 8b)
- _ASM_EXTABLE_UA(6b, 9b)
+ _ASM_EXTABLE_TYPE_REG(6b, 8b, EX_TYPE_UACCESS_LEN4, %eax)
_ASM_EXTABLE_UA(7b, 8b)
: "=&c"(size), "=&D" (d0), "=&S" (d1)
: "1"(to), "2"(from), "0"(size)
@@ -315,14 +303,8 @@ do { \
" movl %3,%0\n" \
"1: rep; movsb\n" \
"2:\n" \
- ".section .fixup,\"ax\"\n" \
- "5: addl %3,%0\n" \
- " jmp 2b\n" \
- "3: lea 0(%3,%0,4),%0\n" \
- " jmp 2b\n" \
- ".previous\n" \
- _ASM_EXTABLE_UA(4b, 5b) \
- _ASM_EXTABLE_UA(0b, 3b) \
+ _ASM_EXTABLE_TYPE_REG(4b, 2b, EX_TYPE_UACCESS_LEN1, %3) \
+ _ASM_EXTABLE_TYPE_REG(0b, 2b, EX_TYPE_UACCESS_LEN4, %3) \
_ASM_EXTABLE_UA(1b, 2b) \
: "=&c"(size), "=&D" (__d0), "=&S" (__d1), "=r"(__d2) \
: "3"(size), "0"(size), "1"(to), "2"(from) \
--- a/arch/x86/lib/usercopy_64.c
+++ b/arch/x86/lib/usercopy_64.c
@@ -35,12 +35,10 @@ unsigned long __clear_user(void __user *
" incq %[dst]\n"
" decl %%ecx ; jnz 1b\n"
"2:\n"
- ".section .fixup,\"ax\"\n"
- "3: lea 0(%[size1],%[size8],8),%[size8]\n"
- " jmp 2b\n"
- ".previous\n"
- _ASM_EXTABLE_UA(0b, 3b)
+
+ _ASM_EXTABLE_TYPE_REG(0b, 2b, EX_TYPE_UACCESS_LEN8, %[size1])
_ASM_EXTABLE_UA(1b, 2b)
+
: [size8] "=&c"(size), [dst] "=&D" (__d0)
: [size1] "r"(size & 7), "[size8]" (size / 8), "[dst]"(addr));
clac();
--- a/arch/x86/mm/extable.c
+++ b/arch/x86/mm/extable.c
@@ -163,6 +163,13 @@ static bool ex_handler_kvm_fastop(const
return true;
}
+static bool ex_handler_uaccess_len(const struct exception_table_entry *fixup,
+ struct pt_regs *regs, int trapnr, int reg, int imm)
+{
+ regs->cx = imm * regs->cx + *pt_regs_nr(regs, reg);
+ return ex_handler_uaccess(fixup, regs, trapnr);
+}
+
#define EX_TYPE_MASK 0x000000FF
#define EX_REG_MASK 0x00000F00
#define EX_FLAG_MASK 0x0000F000
@@ -242,6 +249,8 @@ int fixup_exception(struct pt_regs *regs
return ex_handler_sgx(e, regs, trapnr);
case EX_TYPE_KVM_FASTOP:
return ex_handler_kvm_fastop(e, regs);
+ case EX_TYPE_UACCESS_LEN:
+ return ex_handler_uaccess_len(e, regs, trapnr, reg, imm);
}
BUG();
}
next prev parent reply other threads:[~2021-11-04 16:57 UTC|newest]
Thread overview: 59+ messages / expand[flat|nested] mbox.gz Atom feed top
2021-11-04 16:47 [RFC][PATCH 00/22] x86: Remove anonymous out-of-line fixups Peter Zijlstra
2021-11-04 16:47 ` [RFC][PATCH 01/22] bitfield.h: Fix "type of reg too small for mask" test Peter Zijlstra
2021-11-04 16:47 ` [RFC][PATCH 02/22] x86,mmx_32: Remove .fixup usage Peter Zijlstra
2021-11-04 18:00 ` Borislav Petkov
2021-11-05 11:20 ` David Laight
2021-11-04 20:22 ` Josh Poimboeuf
2021-11-05 8:05 ` Peter Zijlstra
2021-11-04 16:47 ` [RFC][PATCH 03/22] x86,copy_user_64: " Peter Zijlstra
2021-11-04 16:47 ` [RFC][PATCH 04/22] x86,copy_mc_64: " Peter Zijlstra
2021-11-04 16:47 ` [RFC][PATCH 05/22] x86,entry_64: " Peter Zijlstra
2021-11-04 16:47 ` [RFC][PATCH 06/22] x86,entry_32: " Peter Zijlstra
2021-11-04 20:39 ` Josh Poimboeuf
2021-11-05 7:43 ` Peter Zijlstra
2021-11-04 16:47 ` [RFC][PATCH 07/22] x86,extable: Extend extable functionality Peter Zijlstra
2021-11-04 21:49 ` Josh Poimboeuf
2021-11-05 7:54 ` Peter Zijlstra
2021-11-05 10:16 ` Mark Rutland
2021-11-05 17:32 ` Sean Christopherson
2021-11-05 18:45 ` Peter Zijlstra
2021-11-05 19:17 ` Sean Christopherson
2021-11-05 19:32 ` Peter Zijlstra
2021-11-05 19:47 ` Sean Christopherson
2021-11-05 20:15 ` Peter Zijlstra
2021-11-05 20:26 ` Peter Zijlstra
2021-11-05 22:30 ` Sean Christopherson
2021-11-04 16:47 ` [RFC][PATCH 08/22] x86,msr: Remove .fixup usage Peter Zijlstra
2021-11-04 16:47 ` [RFC][PATCH 09/22] x86,futex: " Peter Zijlstra
2021-11-04 16:47 ` [RFC][PATCH 10/22] x86,uaccess: " Peter Zijlstra
2021-11-04 22:28 ` Josh Poimboeuf
2021-11-04 16:47 ` [RFC][PATCH 11/22] x86,xen: " Peter Zijlstra
2021-11-04 22:31 ` Josh Poimboeuf
2021-11-05 7:56 ` Peter Zijlstra
2021-11-04 16:47 ` [RFC][PATCH 12/22] x86,fpu: " Peter Zijlstra
2021-11-04 16:47 ` [RFC][PATCH 13/22] x86,segment: " Peter Zijlstra
2021-11-04 16:47 ` [RFC][PATCH 14/22] x86,ftrace: " Peter Zijlstra
2021-11-04 22:35 ` Josh Poimboeuf
2021-11-05 7:57 ` Peter Zijlstra
2021-11-04 16:47 ` [RFC][PATCH 15/22] x86,vmx: " Peter Zijlstra
2021-11-04 18:50 ` Paolo Bonzini
2021-11-05 18:17 ` Sean Christopherson
2021-11-05 18:52 ` Peter Zijlstra
2021-11-05 20:58 ` Peter Zijlstra
2021-11-05 22:29 ` Sean Christopherson
2021-11-06 7:05 ` Paolo Bonzini
2021-11-06 8:36 ` Peter Zijlstra
2021-11-07 19:13 ` Paolo Bonzini
2021-11-06 8:28 ` Peter Zijlstra
2021-11-04 16:47 ` [RFC][PATCH 16/22] x86,checksum_32: " Peter Zijlstra
2021-11-04 16:47 ` [RFC][PATCH 17/22] x86,sgx: " Peter Zijlstra
2021-11-04 16:47 ` [RFC][PATCH 18/22] x86,kvm: " Peter Zijlstra
2021-11-04 18:50 ` Paolo Bonzini
2021-11-05 7:58 ` Peter Zijlstra
2021-11-04 16:47 ` [RFC][PATCH 19/22] x86,usercopy_32: Simplify Peter Zijlstra
2021-11-04 16:47 ` Peter Zijlstra [this message]
2021-11-04 16:47 ` [RFC][PATCH 21/22] x86,word-at-a-time: Remove .fixup usage Peter Zijlstra
2021-11-04 23:33 ` Josh Poimboeuf
2021-11-05 8:04 ` Peter Zijlstra
2021-11-04 16:47 ` [RFC][PATCH 22/22] x86: Remove .fixup section Peter Zijlstra
2021-11-04 23:00 ` Josh Poimboeuf
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20211104165525.708941275@infradead.org \
--to=peterz@infradead.org \
--cc=dvyukov@google.com \
--cc=jpoimboe@redhat.com \
--cc=linux-kernel@vger.kernel.org \
--cc=mark.rutland@arm.com \
--cc=mbenes@suse.cz \
--cc=pbonzini@redhat.com \
--cc=seanjc@google.com \
--cc=x86@kernel.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).