From: Peter Zijlstra <peterz@infradead.org>
To: x86@kernel.org
Cc: linux-kernel@vger.kernel.org, peterz@infradead.org,
jpoimboe@redhat.com, mark.rutland@arm.com, dvyukov@google.com,
seanjc@google.com, pbonzini@redhat.com, mbenes@suse.cz
Subject: [PATCH v2 20/23] x86,usercopy: Remove .fixup usage
Date: Wed, 10 Nov 2021 11:01:22 +0100 [thread overview]
Message-ID: <20211110101326.081701085@infradead.org> (raw)
In-Reply-To: 20211110100102.250793167@infradead.org
Typically usercopy does whole word copies followed by a number of byte
copies to finish the tail. This means that on exception it needs to
compute the remaining length as: words*sizeof(long) + bytes.
Create a new extable handler to do just this.
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
---
arch/x86/include/asm/extable_fixup_types.h | 5 +++++
arch/x86/lib/usercopy_32.c | 28 +++++-----------------------
arch/x86/lib/usercopy_64.c | 8 +++-----
arch/x86/mm/extable.c | 9 +++++++++
4 files changed, 22 insertions(+), 28 deletions(-)
--- a/arch/x86/include/asm/extable_fixup_types.h
+++ b/arch/x86/include/asm/extable_fixup_types.h
@@ -46,4 +46,9 @@
#define EX_TYPE_FAULT_SGX 18
+#define EX_TYPE_UCOPY_LEN 19 /* cx := reg + imm*cx */
+#define EX_TYPE_UCOPY_LEN1 (EX_TYPE_UCOPY_LEN | EX_DATA_IMM(1))
+#define EX_TYPE_UCOPY_LEN4 (EX_TYPE_UCOPY_LEN | EX_DATA_IMM(4))
+#define EX_TYPE_UCOPY_LEN8 (EX_TYPE_UCOPY_LEN | EX_DATA_IMM(8))
+
#endif
--- a/arch/x86/lib/usercopy_32.c
+++ b/arch/x86/lib/usercopy_32.c
@@ -43,11 +43,7 @@ do { \
" movl %2,%0\n" \
"1: rep; stosb\n" \
"2: " ASM_CLAC "\n" \
- ".section .fixup,\"ax\"\n" \
- "3: lea 0(%2,%0,4),%0\n" \
- " jmp 2b\n" \
- ".previous\n" \
- _ASM_EXTABLE_UA(0b, 3b) \
+ _ASM_EXTABLE_TYPE_REG(0b, 2b, EX_TYPE_UCOPY_LEN4, %2) \
_ASM_EXTABLE_UA(1b, 2b) \
: "=&c"(size), "=&D" (__d0) \
: "r"(size & 3), "0"(size / 4), "1"(addr), "a"(0)); \
@@ -149,10 +145,6 @@ __copy_user_intel(void __user *to, const
"36: movl %%eax, %0\n"
"37: rep; movsb\n"
"100:\n"
- ".section .fixup,\"ax\"\n"
- "101: lea 0(%%eax,%0,4),%0\n"
- " jmp 100b\n"
- ".previous\n"
_ASM_EXTABLE_UA(1b, 100b)
_ASM_EXTABLE_UA(2b, 100b)
_ASM_EXTABLE_UA(3b, 100b)
@@ -190,7 +182,7 @@ __copy_user_intel(void __user *to, const
_ASM_EXTABLE_UA(35b, 100b)
_ASM_EXTABLE_UA(36b, 100b)
_ASM_EXTABLE_UA(37b, 100b)
- _ASM_EXTABLE_UA(99b, 101b)
+ _ASM_EXTABLE_TYPE_REG(99b, 100b, EX_TYPE_UCOPY_LEN4, %%eax)
: "=&c"(size), "=&D" (d0), "=&S" (d1)
: "1"(to), "2"(from), "0"(size)
: "eax", "edx", "memory");
@@ -255,10 +247,6 @@ static unsigned long __copy_user_intel_n
" movl %%eax,%0\n"
"7: rep; movsb\n"
"8:\n"
- ".section .fixup,\"ax\"\n"
- "9: lea 0(%%eax,%0,4),%0\n"
- " jmp 8b\n"
- ".previous\n"
_ASM_EXTABLE_UA(0b, 8b)
_ASM_EXTABLE_UA(1b, 8b)
_ASM_EXTABLE_UA(2b, 8b)
@@ -277,7 +265,7 @@ static unsigned long __copy_user_intel_n
_ASM_EXTABLE_UA(81b, 8b)
_ASM_EXTABLE_UA(14b, 8b)
_ASM_EXTABLE_UA(91b, 8b)
- _ASM_EXTABLE_UA(6b, 9b)
+ _ASM_EXTABLE_TYPE_REG(6b, 8b, EX_TYPE_UCOPY_LEN4, %%eax)
_ASM_EXTABLE_UA(7b, 8b)
: "=&c"(size), "=&D" (d0), "=&S" (d1)
: "1"(to), "2"(from), "0"(size)
@@ -315,14 +303,8 @@ do { \
" movl %3,%0\n" \
"1: rep; movsb\n" \
"2:\n" \
- ".section .fixup,\"ax\"\n" \
- "5: addl %3,%0\n" \
- " jmp 2b\n" \
- "3: lea 0(%3,%0,4),%0\n" \
- " jmp 2b\n" \
- ".previous\n" \
- _ASM_EXTABLE_UA(4b, 5b) \
- _ASM_EXTABLE_UA(0b, 3b) \
+ _ASM_EXTABLE_TYPE_REG(4b, 2b, EX_TYPE_UCOPY_LEN1, %3) \
+ _ASM_EXTABLE_TYPE_REG(0b, 2b, EX_TYPE_UCOPY_LEN4, %3) \
_ASM_EXTABLE_UA(1b, 2b) \
: "=&c"(size), "=&D" (__d0), "=&S" (__d1), "=r"(__d2) \
: "3"(size), "0"(size), "1"(to), "2"(from) \
--- a/arch/x86/lib/usercopy_64.c
+++ b/arch/x86/lib/usercopy_64.c
@@ -35,12 +35,10 @@ unsigned long __clear_user(void __user *
" incq %[dst]\n"
" decl %%ecx ; jnz 1b\n"
"2:\n"
- ".section .fixup,\"ax\"\n"
- "3: lea 0(%[size1],%[size8],8),%[size8]\n"
- " jmp 2b\n"
- ".previous\n"
- _ASM_EXTABLE_UA(0b, 3b)
+
+ _ASM_EXTABLE_TYPE_REG(0b, 2b, EX_TYPE_UCOPY_LEN8, %[size1])
_ASM_EXTABLE_UA(1b, 2b)
+
: [size8] "=&c"(size), [dst] "=&D" (__d0)
: [size1] "r"(size & 7), "[size8]" (size / 8), "[dst]"(addr));
clac();
--- a/arch/x86/mm/extable.c
+++ b/arch/x86/mm/extable.c
@@ -145,6 +145,13 @@ static bool ex_handler_imm_reg(const str
return ex_handler_default(fixup, regs);
}
+static bool ex_handler_ucopy_leng(const struct exception_table_entry *fixup,
+ struct pt_regs *regs, int trapnr, int reg, int imm)
+{
+ regs->cx = imm * regs->cx + *pt_regs_nr(regs, reg);
+ return ex_handler_uaccess(fixup, regs, trapnr);
+}
+
int ex_get_fixup_type(unsigned long ip)
{
const struct exception_table_entry *e = search_exception_tables(ip);
@@ -217,6 +224,8 @@ int fixup_exception(struct pt_regs *regs
return ex_handler_imm_reg(e, regs, reg, imm);
case EX_TYPE_FAULT_SGX:
return ex_handler_sgx(e, regs, trapnr);
+ case EX_TYPE_UCOPY_LEN:
+ return ex_handler_ucopy_leng(e, regs, trapnr, reg, imm);
}
BUG();
}
next prev parent reply other threads:[~2021-11-10 10:16 UTC|newest]
Thread overview: 66+ messages / expand[flat|nested] mbox.gz Atom feed top
2021-11-10 10:01 [PATCH v2 00/23] x86: Remove anonymous out-of-line fixups Peter Zijlstra
2021-11-10 10:01 ` [PATCH v2 01/23] bitfield.h: Fix "type of reg too small for mask" test Peter Zijlstra
2021-12-13 9:50 ` [tip: x86/core] " tip-bot2 for Peter Zijlstra
2021-11-10 10:01 ` [PATCH v2 02/23] x86,mmx_32: Remove .fixup usage Peter Zijlstra
2021-11-15 16:54 ` [PATCH v2.1 02/23] x86/mmx_32: Remove X86_USE_3DNOW Peter Zijlstra
2021-11-16 18:25 ` Borislav Petkov
2021-12-13 9:50 ` [tip: x86/core] " tip-bot2 for Peter Zijlstra
2021-11-10 10:01 ` [PATCH v2 03/23] x86,copy_user_64: Remove .fixup usage Peter Zijlstra
2021-11-16 19:04 ` Borislav Petkov
2021-12-13 9:50 ` [tip: x86/core] x86/copy_user_64: " tip-bot2 for Peter Zijlstra
2021-11-10 10:01 ` [PATCH v2 04/23] x86,copy_mc_64: " Peter Zijlstra
2021-11-17 16:28 ` Borislav Petkov
2021-12-13 9:50 ` [tip: x86/core] x86/copy_mc_64: " tip-bot2 for Peter Zijlstra
2021-11-10 10:01 ` [PATCH v2 05/23] x86,entry_64: " Peter Zijlstra
2021-11-17 16:29 ` Borislav Petkov
2021-11-19 8:00 ` Lai Jiangshan
2021-12-13 9:50 ` [tip: x86/core] x86/entry_64: " tip-bot2 for Peter Zijlstra
2021-11-10 10:01 ` [PATCH v2 06/23] x86,entry_32: " Peter Zijlstra
2021-12-13 9:50 ` [tip: x86/core] x86/entry_32: " tip-bot2 for Peter Zijlstra
2021-11-10 10:01 ` [PATCH v2 07/23] x86,extable: Extend extable functionality Peter Zijlstra
2021-12-13 9:50 ` [tip: x86/core] x86/extable: " tip-bot2 for Peter Zijlstra
2021-11-10 10:01 ` [PATCH v2 08/23] x86,msr: Remove .fixup usage Peter Zijlstra
2021-12-13 9:50 ` [tip: x86/core] x86/msr: " tip-bot2 for Peter Zijlstra
2021-11-10 10:01 ` [PATCH v2 09/23] x86,futex: " Peter Zijlstra
2021-12-13 9:50 ` [tip: x86/core] x86/futex: " tip-bot2 for Peter Zijlstra
2021-11-10 10:01 ` [PATCH v2 10/23] x86,uaccess: " Peter Zijlstra
2021-12-13 9:50 ` [tip: x86/core] x86/uaccess: " tip-bot2 for Peter Zijlstra
2021-11-10 10:01 ` [PATCH v2 11/23] x86,xen: " Peter Zijlstra
2021-11-10 10:35 ` Juergen Gross
2021-11-10 11:00 ` Peter Zijlstra
2021-11-10 12:46 ` Peter Zijlstra
2021-11-10 12:52 ` Juergen Gross
2021-11-10 16:17 ` Peter Zijlstra
2021-11-11 7:20 ` Juergen Gross
2021-11-10 12:45 ` [PATCH v2.1 " Peter Zijlstra
2021-12-13 9:50 ` [tip: x86/core] x86/xen: " tip-bot2 for Peter Zijlstra
2021-11-10 10:01 ` [PATCH v2 12/23] x86,fpu: " Peter Zijlstra
2021-12-13 9:50 ` [tip: x86/core] x86/fpu: " tip-bot2 for Peter Zijlstra
2021-11-10 10:01 ` [PATCH v2 13/23] x86,segment: " Peter Zijlstra
2021-12-13 9:50 ` [tip: x86/core] x86/segment: " tip-bot2 for Peter Zijlstra
2021-11-10 10:01 ` [PATCH v2 14/23] x86,kvm: " Peter Zijlstra
2021-12-13 9:50 ` [tip: x86/core] x86/kvm: " tip-bot2 for Peter Zijlstra
2021-11-10 10:01 ` [PATCH v2 15/23] x86,vmx: " Peter Zijlstra
2021-12-13 9:50 ` [tip: x86/core] x86/vmx: " tip-bot2 for Peter Zijlstra
2021-11-10 10:01 ` [PATCH v2 16/23] x86,vmx: Provide asm-goto-output vmread Peter Zijlstra
2021-12-13 10:07 ` Peter Zijlstra
2021-12-14 18:44 ` Sean Christopherson
2021-12-14 22:46 ` Peter Zijlstra
2021-12-14 23:07 ` Sean Christopherson
2021-11-10 10:01 ` [PATCH v2 17/23] x86,checksum_32: Remove .fixup usage Peter Zijlstra
2021-12-13 9:50 ` [tip: x86/core] x86/checksum_32: " tip-bot2 for Peter Zijlstra
2021-11-10 10:01 ` [PATCH v2 18/23] x86,sgx: " Peter Zijlstra
2021-12-13 9:50 ` [tip: x86/core] x86/sgx: " tip-bot2 for Peter Zijlstra
2021-11-10 10:01 ` [PATCH v2 19/23] x86,usercopy_32: Simplify __copy_user_intel_nocache() Peter Zijlstra
2021-12-13 9:50 ` [tip: x86/core] x86/usercopy_32: " tip-bot2 for Peter Zijlstra
2021-11-10 10:01 ` Peter Zijlstra [this message]
2021-11-11 7:51 ` [PATCH v2 20/23] x86,usercopy: Remove .fixup usage Josh Poimboeuf
2021-11-11 8:28 ` Peter Zijlstra
2021-12-13 9:50 ` [tip: x86/core] x86/usercopy: " tip-bot2 for Peter Zijlstra
2021-11-10 10:01 ` [PATCH v2 21/23] x86,word-at-a-time: " Peter Zijlstra
2021-12-13 9:50 ` [tip: x86/core] x86/word-at-a-time: " tip-bot2 for Peter Zijlstra
2021-11-10 10:01 ` [PATCH v2 22/23] x86: Remove .fixup section Peter Zijlstra
2021-12-13 9:50 ` [tip: x86/core] " tip-bot2 for Peter Zijlstra
2021-11-10 10:01 ` [PATCH v2 23/23] objtool: Remove .fixup handling Peter Zijlstra
2021-12-13 9:50 ` [tip: x86/core] " tip-bot2 for Peter Zijlstra
2021-11-11 7:55 ` [PATCH v2 00/23] x86: Remove anonymous out-of-line fixups Josh Poimboeuf
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20211110101326.081701085@infradead.org \
--to=peterz@infradead.org \
--cc=dvyukov@google.com \
--cc=jpoimboe@redhat.com \
--cc=linux-kernel@vger.kernel.org \
--cc=mark.rutland@arm.com \
--cc=mbenes@suse.cz \
--cc=pbonzini@redhat.com \
--cc=seanjc@google.com \
--cc=x86@kernel.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.