From: Robin Murphy <robin.murphy@arm.com>
To: will@kernel.org, catalin.marinas@arm.com
Cc: linux-arm-kernel@lists.infradead.org, mark.rutland@arm.com
Subject: [PATCH v2 8/8] arm64: Rewrite __arch_clear_user()
Date: Thu, 27 May 2021 16:34:48 +0100 [thread overview]
Message-ID: <1cbd78b12c076a8ad4656a345811cfb9425df0b3.1622128527.git.robin.murphy@arm.com> (raw)
In-Reply-To: <cover.1622128527.git.robin.murphy@arm.com>
Now that we're always using STTR variants rather than abstracting two
different addressing modes, the user_ldst macro here is frankly more
obfuscating than helpful. Rewrite __arch_clear_user() with regular
USER() annotations so that it's clearer what's going on, and take the
opportunity to minimise the branchiness in the most common paths, while
also allowing the exception fixup to return an accurate result.
Apparently some folks examine large reads from /dev/zero closely enough
to notice the loop being hot, so align it per the other critical loops
(presumably around a typical instruction fetch granularity).
Reviewed-by: Mark Rutland <mark.rutland@arm.com>
Signed-off-by: Robin Murphy <robin.murphy@arm.com>
---
arch/arm64/lib/clear_user.S | 47 +++++++++++++++++++++----------------
1 file changed, 27 insertions(+), 20 deletions(-)
diff --git a/arch/arm64/lib/clear_user.S b/arch/arm64/lib/clear_user.S
index af9afcbec92c..a7efb2ad2a1c 100644
--- a/arch/arm64/lib/clear_user.S
+++ b/arch/arm64/lib/clear_user.S
@@ -1,12 +1,9 @@
/* SPDX-License-Identifier: GPL-2.0-only */
/*
- * Based on arch/arm/lib/clear_user.S
- *
- * Copyright (C) 2012 ARM Ltd.
+ * Copyright (C) 2021 Arm Ltd.
*/
-#include <linux/linkage.h>
-#include <asm/asm-uaccess.h>
+#include <linux/linkage.h>
#include <asm/assembler.h>
.text
@@ -19,25 +16,33 @@
*
* Alignment fixed up by hardware.
*/
+
+ .p2align 4
+ // Alignment is for the loop, but since the prologue (including BTI)
+ // is also 16 bytes we can keep any padding outside the function
SYM_FUNC_START(__arch_clear_user)
- mov x2, x1 // save the size for fixup return
+ add x2, x0, x1
subs x1, x1, #8
b.mi 2f
1:
-user_ldst 9f, sttr, xzr, x0, 8
+USER(9f, sttr xzr, [x0])
+ add x0, x0, #8
subs x1, x1, #8
- b.pl 1b
-2: adds x1, x1, #4
- b.mi 3f
-user_ldst 9f, sttr, wzr, x0, 4
- sub x1, x1, #4
-3: adds x1, x1, #2
- b.mi 4f
-user_ldst 9f, sttrh, wzr, x0, 2
- sub x1, x1, #2
-4: adds x1, x1, #1
- b.mi 5f
-user_ldst 9f, sttrb, wzr, x0, 0
+ b.hi 1b
+USER(9f, sttr xzr, [x2, #-8])
+ mov x0, #0
+ ret
+
+2: tbz x1, #2, 3f
+USER(9f, sttr wzr, [x0])
+USER(8f, sttr wzr, [x2, #-4])
+ mov x0, #0
+ ret
+
+3: tbz x1, #1, 4f
+USER(9f, sttrh wzr, [x0])
+4: tbz x1, #0, 5f
+USER(7f, sttrb wzr, [x2, #-1])
5: mov x0, #0
ret
SYM_FUNC_END(__arch_clear_user)
@@ -45,6 +50,8 @@ EXPORT_SYMBOL(__arch_clear_user)
.section .fixup,"ax"
.align 2
-9: mov x0, x2 // return the original size
+7: sub x0, x2, #5 // Adjust for faulting on the final byte...
+8: add x0, x0, #4 // ...or the second word of the 4-7 byte case
+9: sub x0, x2, x0
ret
.previous
--
2.21.0.dirty
_______________________________________________
linux-arm-kernel mailing list
linux-arm-kernel@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel
next prev parent reply other threads:[~2021-05-27 16:57 UTC|newest]
Thread overview: 31+ messages / expand[flat|nested] mbox.gz Atom feed top
2021-05-27 15:34 [PATCH v2 0/8] arm64: String function updates Robin Murphy
2021-05-27 15:34 ` [PATCH v2 1/8] arm64: Import latest version of Cortex Strings' memcmp Robin Murphy
2021-05-27 16:52 ` Mark Rutland
2021-06-01 18:26 ` Will Deacon
2021-05-27 15:34 ` [PATCH v2 2/8] arm64: Import latest version of Cortex Strings' strcmp Robin Murphy
2021-05-27 15:34 ` [PATCH v2 3/8] arm64: Import updated version of Cortex Strings' strlen Robin Murphy
2021-05-27 15:34 ` [PATCH v2 4/8] arm64: Import latest version of Cortex Strings' strncmp Robin Murphy
2021-05-27 15:34 ` [PATCH v2 5/8] arm64: Add assembly annotations for weak-PI-alias madness Robin Murphy
2021-05-27 15:34 ` [PATCH v2 6/8] arm64: Import latest memcpy()/memmove() implementation Robin Murphy
[not found] ` <CGME20210608111534eucas1p2964e360336878b9e7a791c0fbeb12940@eucas1p2.samsung.com>
2021-06-08 11:15 ` Marek Szyprowski
2021-06-08 11:15 ` Marek Szyprowski
2021-06-08 11:37 ` Robin Murphy
2021-06-08 11:37 ` Robin Murphy
2021-06-08 12:21 ` Marek Szyprowski
2021-06-08 12:21 ` Marek Szyprowski
2021-06-08 12:36 ` Neil Armstrong
2021-06-08 12:36 ` Neil Armstrong
2021-06-08 12:42 ` Mark Rutland
2021-06-08 12:42 ` Mark Rutland
2022-05-20 23:30 ` dann frazier
2022-05-20 23:30 ` dann frazier
2022-05-21 7:56 ` Robin Murphy
2022-05-21 7:56 ` Robin Murphy
2022-05-23 17:27 ` dann frazier
2022-05-23 17:27 ` dann frazier
[not found] ` <CAMn1gO7rJzUg53cet8ocN0aMrEgQ2iqUN2pB-iQ=nBT7dafdtA@mail.gmail.com>
2021-09-10 11:36 ` Catalin Marinas
2021-09-10 11:42 ` Robin Murphy
2021-09-10 20:32 ` Peter Collingbourne
2021-05-27 15:34 ` [PATCH v2 7/8] arm64: Better optimised memchr() Robin Murphy
2021-05-27 15:34 ` Robin Murphy [this message]
2021-06-01 18:21 ` [PATCH v2 0/8] arm64: String function updates Will Deacon
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=1cbd78b12c076a8ad4656a345811cfb9425df0b3.1622128527.git.robin.murphy@arm.com \
--to=robin.murphy@arm.com \
--cc=catalin.marinas@arm.com \
--cc=linux-arm-kernel@lists.infradead.org \
--cc=mark.rutland@arm.com \
--cc=will@kernel.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.