From: "Aneesh Kumar K.V" <aneesh.kumar@linux.ibm.com>
To: linuxppc-dev@lists.ozlabs.org, mpe@ellerman.id.au
Cc: "Aneesh Kumar K.V" <aneesh.kumar@linux.ibm.com>
Subject: [PATCH v7 22/22] powerpc/book3s64/pkeys: Optimize KUAP and KUEP feature disabled case
Date: Fri, 27 Nov 2020 10:14:24 +0530 [thread overview]
Message-ID: <20201127044424.40686-23-aneesh.kumar@linux.ibm.com> (raw)
In-Reply-To: <20201127044424.40686-1-aneesh.kumar@linux.ibm.com>
If FTR_BOOK3S_KUAP is disabled, kernel will continue to run with the same AMR
value with which it was entered. Hence there is a high chance that
we can return without restoring the AMR value. This also helps the case
when applications are not using the pkey feature. In this case, different
applications will have the same AMR values and hence we can avoid restoring
AMR in this case too.
Also avoid isync() if not really needed.
Do the same for IAMR.
null-syscall benchmark results:
With smap/smep disabled:
Without patch:
957.95 ns 2778.17 cycles
With patch:
858.38 ns 2489.30 cycles
With smap/smep enabled:
Without patch:
1017.26 ns 2950.36 cycles
With patch:
1021.51 ns 2962.44 cycles
Signed-off-by: Aneesh Kumar K.V <aneesh.kumar@linux.ibm.com>
---
arch/powerpc/include/asm/book3s/64/kup.h | 63 +++++++++++++++++++++---
arch/powerpc/kernel/entry_64.S | 2 +-
arch/powerpc/kernel/syscall_64.c | 12 +++--
3 files changed, 67 insertions(+), 10 deletions(-)
diff --git a/arch/powerpc/include/asm/book3s/64/kup.h b/arch/powerpc/include/asm/book3s/64/kup.h
index 9a475a6bf0a1..685e5ad14628 100644
--- a/arch/powerpc/include/asm/book3s/64/kup.h
+++ b/arch/powerpc/include/asm/book3s/64/kup.h
@@ -12,28 +12,54 @@
#ifdef __ASSEMBLY__
-.macro kuap_user_restore gpr1
+.macro kuap_user_restore gpr1, gpr2
#if defined(CONFIG_PPC_PKEY)
BEGIN_MMU_FTR_SECTION_NESTED(67)
+ b 100f // skip_restore_amr
+ END_MMU_FTR_SECTION_NESTED_IFCLR(MMU_FTR_PKEY, 67)
/*
* AMR and IAMR are going to be different when
* returning to userspace.
*/
ld \gpr1, STACK_REGS_AMR(r1)
+
+ /*
+ * If kuap feature is not enabled, do the mtspr
+ * only if AMR value is different.
+ */
+ BEGIN_MMU_FTR_SECTION_NESTED(68)
+ mfspr \gpr2, SPRN_AMR
+ cmpd \gpr1, \gpr2
+ beq 99f
+ END_MMU_FTR_SECTION_NESTED_IFCLR(MMU_FTR_BOOK3S_KUAP, 68)
+
isync
mtspr SPRN_AMR, \gpr1
+99:
/*
* Restore IAMR only when returning to userspace
*/
ld \gpr1, STACK_REGS_IAMR(r1)
+
+ /*
+ * If kuep feature is not enabled, do the mtspr
+ * only if IAMR value is different.
+ */
+ BEGIN_MMU_FTR_SECTION_NESTED(69)
+ mfspr \gpr2, SPRN_IAMR
+ cmpd \gpr1, \gpr2
+ beq 100f
+ END_MMU_FTR_SECTION_NESTED_IFCLR(MMU_FTR_BOOK3S_KUEP, 69)
+
+ isync
mtspr SPRN_IAMR, \gpr1
+100: //skip_restore_amr
/* No isync required, see kuap_user_restore() */
- END_MMU_FTR_SECTION_NESTED_IFSET(MMU_FTR_PKEY, 67)
#endif
.endm
-.macro kuap_kernel_restore gpr1, gpr2
+.macro kuap_kernel_restore gpr1, gpr2
#if defined(CONFIG_PPC_PKEY)
BEGIN_MMU_FTR_SECTION_NESTED(67)
@@ -197,18 +223,43 @@ static inline u64 current_thread_iamr(void)
static inline void kuap_user_restore(struct pt_regs *regs)
{
+ bool restore_amr = false, restore_iamr = false;
+ unsigned long amr, iamr;
+
if (!mmu_has_feature(MMU_FTR_PKEY))
return;
- isync();
- mtspr(SPRN_AMR, regs->amr);
- mtspr(SPRN_IAMR, regs->iamr);
+ if (!mmu_has_feature(MMU_FTR_BOOK3S_KUAP)) {
+ amr = mfspr(SPRN_AMR);
+ if (amr != regs->amr)
+ restore_amr = true;
+ } else {
+ restore_amr = true;
+ }
+
+ if (!mmu_has_feature(MMU_FTR_BOOK3S_KUEP)) {
+ iamr = mfspr(SPRN_IAMR);
+ if (iamr != regs->iamr)
+ restore_iamr = true;
+ } else {
+ restore_iamr = true;
+ }
+
+
+ if (restore_amr || restore_iamr) {
+ isync();
+ if (restore_amr)
+ mtspr(SPRN_AMR, regs->amr);
+ if (restore_iamr)
+ mtspr(SPRN_IAMR, regs->iamr);
+ }
/*
* No isync required here because we are about to rfi
* back to previous context before any user accesses
* would be made, which is a CSI.
*/
}
+
static inline void kuap_kernel_restore(struct pt_regs *regs,
unsigned long amr)
{
diff --git a/arch/powerpc/kernel/entry_64.S b/arch/powerpc/kernel/entry_64.S
index da23c397ceb2..c9d59450fba0 100644
--- a/arch/powerpc/kernel/entry_64.S
+++ b/arch/powerpc/kernel/entry_64.S
@@ -675,7 +675,7 @@ _ASM_NOKPROBE_SYMBOL(interrupt_return)
bne- .Lrestore_nvgprs
.Lfast_user_interrupt_return_amr:
- kuap_user_restore r3
+ kuap_user_restore r3, r4
.Lfast_user_interrupt_return:
ld r11,_NIP(r1)
ld r12,_MSR(r1)
diff --git a/arch/powerpc/kernel/syscall_64.c b/arch/powerpc/kernel/syscall_64.c
index 11f1c6360291..7c85ed04a164 100644
--- a/arch/powerpc/kernel/syscall_64.c
+++ b/arch/powerpc/kernel/syscall_64.c
@@ -38,6 +38,7 @@ notrace long system_call_exception(long r3, long r4, long r5,
#ifdef CONFIG_PPC_PKEY
if (mmu_has_feature(MMU_FTR_PKEY)) {
unsigned long amr, iamr;
+ bool flush_needed = false;
/*
* When entering from userspace we mostly have the AMR/IAMR
* different from kernel default values. Hence don't compare.
@@ -46,11 +47,16 @@ notrace long system_call_exception(long r3, long r4, long r5,
iamr = mfspr(SPRN_IAMR);
regs->amr = amr;
regs->iamr = iamr;
- if (mmu_has_feature(MMU_FTR_BOOK3S_KUAP))
+ if (mmu_has_feature(MMU_FTR_BOOK3S_KUAP)) {
mtspr(SPRN_AMR, AMR_KUAP_BLOCKED);
- if (mmu_has_feature(MMU_FTR_BOOK3S_KUEP))
+ flush_needed = true;
+ }
+ if (mmu_has_feature(MMU_FTR_BOOK3S_KUEP)) {
mtspr(SPRN_IAMR, AMR_KUEP_BLOCKED);
- isync();
+ flush_needed = true;
+ }
+ if (flush_needed)
+ isync();
} else
#endif
kuap_check_amr();
--
2.28.0
next prev parent reply other threads:[~2020-11-27 5:23 UTC|newest]
Thread overview: 27+ messages / expand[flat|nested] mbox.gz Atom feed top
2020-11-27 4:44 [PATCH v7 00/22] Kernel userspace access/execution prevention with hash translation Aneesh Kumar K.V
2020-11-27 4:44 ` [PATCH v7 01/22] powerpc: Add new macro to handle NESTED_IFCLR Aneesh Kumar K.V
2020-11-27 4:44 ` [PATCH v7 02/22] KVM: PPC: BOOK3S: PR: Ignore UAMOR SPR Aneesh Kumar K.V
2020-11-27 4:44 ` [PATCH v7 03/22] powerpc/book3s64/kuap/kuep: Add PPC_PKEY config on book3s64 Aneesh Kumar K.V
2020-11-27 4:44 ` [PATCH v7 04/22] powerpc/book3s64/kuap/kuep: Move uamor setup to pkey init Aneesh Kumar K.V
2020-11-27 4:44 ` [PATCH v7 05/22] powerpc/book3s64/kuap: Move KUAP related function outside radix Aneesh Kumar K.V
2020-11-27 4:44 ` [PATCH v7 06/22] powerpc/book3s64/kuep: Move KUEP " Aneesh Kumar K.V
2020-11-27 4:44 ` [PATCH v7 07/22] powerpc/book3s64/kuap: Rename MMU_FTR_RADIX_KUAP and MMU_FTR_KUEP Aneesh Kumar K.V
2020-11-27 4:44 ` [PATCH v7 08/22] powerpc/book3s64/kuap: Use Key 3 for kernel mapping with hash translation Aneesh Kumar K.V
2020-11-27 4:44 ` [PATCH v7 09/22] powerpc/exec: Set thread.regs early during exec Aneesh Kumar K.V
2020-11-27 4:44 ` [PATCH v7 10/22] powerpc/book3s64/pkeys: Store/restore userspace AMR/IAMR correctly on entry and exit from kernel Aneesh Kumar K.V
2020-11-27 4:44 ` [PATCH v7 11/22] powerpc/book3s64/pkeys: Inherit correctly on fork Aneesh Kumar K.V
2020-11-27 4:44 ` [PATCH v7 12/22] powerpc/book3s64/pkeys: Reset userspace AMR correctly on exec Aneesh Kumar K.V
2020-11-27 4:44 ` [PATCH v7 13/22] powerpc/ptrace-view: Use pt_regs values instead of thread_struct based one Aneesh Kumar K.V
2020-11-27 4:44 ` [PATCH v7 14/22] powerpc/book3s64/pkeys: Don't update SPRN_AMR when in kernel mode Aneesh Kumar K.V
2020-11-27 4:44 ` [PATCH v7 15/22] powerpc/book3s64/kuap: Restrict access to userspace based on userspace AMR Aneesh Kumar K.V
2020-11-27 4:44 ` [PATCH v7 16/22] powerpc/book3s64/kuap: Improve error reporting with KUAP Aneesh Kumar K.V
2020-11-27 4:44 ` [PATCH v7 17/22] powerpc/book3s64/kuap: Use Key 3 to implement KUAP with hash translation Aneesh Kumar K.V
2020-11-27 4:44 ` [PATCH v7 18/22] powerpc/book3s64/kuep: Use Key 3 to implement KUEP " Aneesh Kumar K.V
2020-11-27 4:44 ` [PATCH v7 19/22] powerpc/book3s64/hash/kuap: Enable kuap on hash Aneesh Kumar K.V
2020-11-27 4:44 ` [PATCH v7 20/22] powerpc/book3s64/hash/kuep: Enable KUEP " Aneesh Kumar K.V
2020-11-27 4:44 ` [PATCH v7 21/22] powerpc/book3s64/hash/kup: Don't hardcode kup key Aneesh Kumar K.V
2020-12-02 4:38 ` [PATCH v7 updated 21/22 ] powerpc/book3s64/kup: Check max key supported before enabling kup Aneesh Kumar K.V
2020-12-15 11:19 ` Michael Ellerman
2020-11-27 4:44 ` Aneesh Kumar K.V [this message]
2020-12-01 3:41 ` [PATCH v7 00/22] Kernel userspace access/execution prevention with hash translation Aneesh Kumar K.V
2020-12-10 11:29 ` Michael Ellerman
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20201127044424.40686-23-aneesh.kumar@linux.ibm.com \
--to=aneesh.kumar@linux.ibm.com \
--cc=linuxppc-dev@lists.ozlabs.org \
--cc=mpe@ellerman.id.au \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.