linux-kernel.vger.kernel.org archive mirror
 help / color / mirror / Atom feed
From: Christophe Leroy <christophe.leroy@csgroup.eu>
To: Benjamin Herrenschmidt <benh@kernel.crashing.org>,
	Paul Mackerras <paulus@samba.org>,
	Michael Ellerman <mpe@ellerman.id.au>,
	npiggin@gmail.com
Cc: linux-kernel@vger.kernel.org, linuxppc-dev@lists.ozlabs.org
Subject: [PATCH v3 37/41] powerpc/32s: Move KUEP locking/unlocking in C
Date: Fri, 12 Mar 2021 12:50:47 +0000 (UTC)	[thread overview]
Message-ID: <4eadd873927e9a73c3d1dfe2f9497353465514cf.1615552867.git.christophe.leroy@csgroup.eu> (raw)
In-Reply-To: <cover.1615552866.git.christophe.leroy@csgroup.eu>

This can be done in C, do it.

Unrolling the loop gains approx. 15% performance.

From now on, prepare_transfer_to_handler() is only for
interrupts from kernel.

Signed-off-by: Christophe Leroy <christophe.leroy@csgroup.eu>
---
 arch/powerpc/include/asm/book3s/32/kup.h | 31 -------------------
 arch/powerpc/include/asm/interrupt.h     |  6 +++-
 arch/powerpc/include/asm/kup.h           |  8 +++++
 arch/powerpc/kernel/entry_32.S           | 16 ----------
 arch/powerpc/kernel/head_32.h            |  3 ++
 arch/powerpc/kernel/head_booke.h         |  3 ++
 arch/powerpc/kernel/interrupt.c          |  4 +++
 arch/powerpc/mm/book3s32/Makefile        |  1 +
 arch/powerpc/mm/book3s32/kuep.c          | 38 ++++++++++++++++++++++++
 9 files changed, 62 insertions(+), 48 deletions(-)
 create mode 100644 arch/powerpc/mm/book3s32/kuep.c

diff --git a/arch/powerpc/include/asm/book3s/32/kup.h b/arch/powerpc/include/asm/book3s/32/kup.h
index 73bc5d2c431d..b97ea60f6fa3 100644
--- a/arch/powerpc/include/asm/book3s/32/kup.h
+++ b/arch/powerpc/include/asm/book3s/32/kup.h
@@ -7,37 +7,6 @@
 
 #ifdef __ASSEMBLY__
 
-.macro kuep_update_sr	gpr1, gpr2		/* NEVER use r0 as gpr2 due to addis */
-101:	mtsrin	\gpr1, \gpr2
-	addi	\gpr1, \gpr1, 0x111		/* next VSID */
-	rlwinm	\gpr1, \gpr1, 0, 0xf0ffffff	/* clear VSID overflow */
-	addis	\gpr2, \gpr2, 0x1000		/* address of next segment */
-	bdnz	101b
-	isync
-.endm
-
-.macro kuep_lock	gpr1, gpr2
-#ifdef CONFIG_PPC_KUEP
-	li	\gpr1, NUM_USER_SEGMENTS
-	li	\gpr2, 0
-	mtctr	\gpr1
-	mfsrin	\gpr1, \gpr2
-	oris	\gpr1, \gpr1, SR_NX@h		/* set Nx */
-	kuep_update_sr \gpr1, \gpr2
-#endif
-.endm
-
-.macro kuep_unlock	gpr1, gpr2
-#ifdef CONFIG_PPC_KUEP
-	li	\gpr1, NUM_USER_SEGMENTS
-	li	\gpr2, 0
-	mtctr	\gpr1
-	mfsrin	\gpr1, \gpr2
-	rlwinm	\gpr1, \gpr1, 0, ~SR_NX		/* Clear Nx */
-	kuep_update_sr \gpr1, \gpr2
-#endif
-.endm
-
 #ifdef CONFIG_PPC_KUAP
 
 .macro kuap_update_sr	gpr1, gpr2, gpr3	/* NEVER use r0 as gpr2 due to addis */
diff --git a/arch/powerpc/include/asm/interrupt.h b/arch/powerpc/include/asm/interrupt.h
index 861e6eadc98c..857375309255 100644
--- a/arch/powerpc/include/asm/interrupt.h
+++ b/arch/powerpc/include/asm/interrupt.h
@@ -33,8 +33,10 @@ static inline void interrupt_enter_prepare(struct pt_regs *regs, struct interrup
 	if (!arch_irq_disabled_regs(regs))
 		trace_hardirqs_off();
 
-	if (user_mode(regs))
+	if (user_mode(regs)) {
+		kuep_lock();
 		account_cpu_user_entry();
+	}
 #endif
 	/*
 	 * Book3E reconciles irq soft mask in asm
@@ -89,6 +91,8 @@ static inline void interrupt_exit_prepare(struct pt_regs *regs, struct interrupt
 	exception_exit(state->ctx_state);
 #endif
 
+	if (user_mode(regs))
+		kuep_unlock();
 	/*
 	 * Book3S exits to user via interrupt_exit_user_prepare(), which does
 	 * context tracking, which is a cleaner way to handle PREEMPT=y
diff --git a/arch/powerpc/include/asm/kup.h b/arch/powerpc/include/asm/kup.h
index 7ec21af49a45..25671f711ec2 100644
--- a/arch/powerpc/include/asm/kup.h
+++ b/arch/powerpc/include/asm/kup.h
@@ -55,6 +55,14 @@ void setup_kuep(bool disabled);
 static inline void setup_kuep(bool disabled) { }
 #endif /* CONFIG_PPC_KUEP */
 
+#if defined(CONFIG_PPC_KUEP) && defined(CONFIG_PPC_BOOK3S_32)
+void kuep_lock(void);
+void kuep_unlock(void);
+#else
+static inline void kuep_lock(void) { }
+static inline void kuep_unlock(void) { }
+#endif
+
 #ifdef CONFIG_PPC_KUAP
 void setup_kuap(bool disabled);
 #else
diff --git a/arch/powerpc/kernel/entry_32.S b/arch/powerpc/kernel/entry_32.S
index 44d0eddf8738..112d6247c391 100644
--- a/arch/powerpc/kernel/entry_32.S
+++ b/arch/powerpc/kernel/entry_32.S
@@ -51,16 +51,9 @@
 #if defined(CONFIG_PPC_BOOK3S_32) || defined(CONFIG_E500)
 	.globl	prepare_transfer_to_handler
 prepare_transfer_to_handler:
-	andi.	r12,r9,MSR_PR
 	addi	r12,r2,THREAD
-	beq	2f
-#ifdef CONFIG_PPC_BOOK3S_32
-	kuep_lock r11, r12
-#endif
-	blr
 
 	/* if from kernel, check interrupted DOZE/NAP mode */
-2:
 	kuap_save_and_lock r11, r12, r9, r5, r6
 	lwz	r12,TI_LOCAL_FLAGS(r2)
 	mtcrf	0x01,r12
@@ -86,9 +79,6 @@ _ASM_NOKPROBE_SYMBOL(prepare_transfer_to_handler)
 	.globl	transfer_to_syscall
 transfer_to_syscall:
 	SAVE_NVGPRS(r1)
-#ifdef CONFIG_PPC_BOOK3S_32
-	kuep_lock r11, r12
-#endif
 
 	/* Calling convention has r9 = orig r0, r10 = regs */
 	addi	r10,r1,STACK_FRAME_OVERHEAD
@@ -105,9 +95,6 @@ ret_from_syscall:
 	cmplwi	cr0,r5,0
 	bne-	2f
 #endif /* CONFIG_PPC_47x */
-#ifdef CONFIG_PPC_BOOK3S_32
-	kuep_unlock r5, r7
-#endif
 	kuap_check r2, r4
 	lwz	r4,_LINK(r1)
 	lwz	r5,_CCR(r1)
@@ -311,9 +298,6 @@ interrupt_return:
 	bne-	.Lrestore_nvgprs
 
 .Lfast_user_interrupt_return:
-#ifdef CONFIG_PPC_BOOK3S_32
-	kuep_unlock	r10, r11
-#endif
 	kuap_check r2, r4
 	lwz	r11,_NIP(r1)
 	lwz	r12,_MSR(r1)
diff --git a/arch/powerpc/kernel/head_32.h b/arch/powerpc/kernel/head_32.h
index c018fcdf9157..a8221ddcbd66 100644
--- a/arch/powerpc/kernel/head_32.h
+++ b/arch/powerpc/kernel/head_32.h
@@ -133,7 +133,10 @@ _ASM_NOKPROBE_SYMBOL(\name\()_virt)
 
 .macro prepare_transfer_to_handler
 #ifdef CONFIG_PPC_BOOK3S_32
+	andi.	r12,r9,MSR_PR
+	bne	777f
 	bl	prepare_transfer_to_handler
+777:
 #endif
 .endm
 
diff --git a/arch/powerpc/kernel/head_booke.h b/arch/powerpc/kernel/head_booke.h
index cb96ae002af6..f82470091697 100644
--- a/arch/powerpc/kernel/head_booke.h
+++ b/arch/powerpc/kernel/head_booke.h
@@ -105,7 +105,10 @@ END_BTB_FLUSH_SECTION
 
 .macro prepare_transfer_to_handler
 #ifdef CONFIG_E500
+	andi.	r12,r9,MSR_PR
+	bne	777f
 	bl	prepare_transfer_to_handler
+777:
 #endif
 .endm
 
diff --git a/arch/powerpc/kernel/interrupt.c b/arch/powerpc/kernel/interrupt.c
index 6875b82f613a..20ace874cd98 100644
--- a/arch/powerpc/kernel/interrupt.c
+++ b/arch/powerpc/kernel/interrupt.c
@@ -33,6 +33,8 @@ notrace long system_call_exception(long r3, long r4, long r5,
 {
 	syscall_fn f;
 
+	kuep_lock();
+
 	regs->orig_gpr3 = r3;
 
 	if (IS_ENABLED(CONFIG_PPC_IRQ_SOFT_MASK_DEBUG))
@@ -354,6 +356,8 @@ notrace unsigned long syscall_exit_prepare(unsigned long r3,
 	 */
 	kuap_user_restore(regs);
 #endif
+	kuep_unlock();
+
 	return ret;
 }
 
diff --git a/arch/powerpc/mm/book3s32/Makefile b/arch/powerpc/mm/book3s32/Makefile
index 446d9de88ce4..7f0c8a78ba0c 100644
--- a/arch/powerpc/mm/book3s32/Makefile
+++ b/arch/powerpc/mm/book3s32/Makefile
@@ -9,3 +9,4 @@ endif
 obj-y += mmu.o mmu_context.o
 obj-$(CONFIG_PPC_BOOK3S_603) += nohash_low.o
 obj-$(CONFIG_PPC_BOOK3S_604) += hash_low.o tlb.o
+obj-$(CONFIG_PPC_KUEP) += kuep.o
diff --git a/arch/powerpc/mm/book3s32/kuep.c b/arch/powerpc/mm/book3s32/kuep.c
new file mode 100644
index 000000000000..c70532568a28
--- /dev/null
+++ b/arch/powerpc/mm/book3s32/kuep.c
@@ -0,0 +1,38 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+#include <asm/reg.h>
+#include <asm/task_size_32.h>
+#include <asm/mmu.h>
+
+#define KUEP_UPDATE_TWO_USER_SEGMENTS(n) do {		\
+	if (TASK_SIZE > ((n) << 28))			\
+		mtsr(val1, (n) << 28);			\
+	if (TASK_SIZE > (((n) + 1) << 28))		\
+		mtsr(val2, ((n) + 1) << 28);		\
+	val1 = (val1 + 0x222) & 0xf0ffffff;		\
+	val2 = (val2 + 0x222) & 0xf0ffffff;		\
+} while (0)
+
+static __always_inline void kuep_update(u32 val)
+{
+	int val1 = val;
+	int val2 = (val + 0x111) & 0xf0ffffff;
+
+	KUEP_UPDATE_TWO_USER_SEGMENTS(0);
+	KUEP_UPDATE_TWO_USER_SEGMENTS(2);
+	KUEP_UPDATE_TWO_USER_SEGMENTS(4);
+	KUEP_UPDATE_TWO_USER_SEGMENTS(6);
+	KUEP_UPDATE_TWO_USER_SEGMENTS(8);
+	KUEP_UPDATE_TWO_USER_SEGMENTS(10);
+	KUEP_UPDATE_TWO_USER_SEGMENTS(12);
+	KUEP_UPDATE_TWO_USER_SEGMENTS(14);
+}
+
+void kuep_lock(void)
+{
+	kuep_update(mfsr(0) | SR_NX);
+}
+
+void kuep_unlock(void)
+{
+	kuep_update(mfsr(0) & ~SR_NX);
+}
-- 
2.25.0


  parent reply	other threads:[~2021-03-12 12:52 UTC|newest]

Thread overview: 53+ messages / expand[flat|nested]  mbox.gz  Atom feed  top
2021-03-12 12:50 [PATCH v3 00/41] powerpc/32: Switch to interrupt entry/exit in C Christophe Leroy
2021-03-12 12:50 ` [PATCH v3 01/41] powerpc/traps: Declare unrecoverable_exception() as __noreturn Christophe Leroy
2021-03-12 12:50 ` [PATCH v3 02/41] powerpc/40x: Don't use SPRN_SPRG_SCRATCH0/1 in TLB miss handlers Christophe Leroy
2021-03-12 12:50 ` [PATCH v3 03/41] powerpc/40x: Change CRITICAL_EXCEPTION_PROLOG macro to a gas macro Christophe Leroy
2021-03-12 12:50 ` [PATCH v3 04/41] powerpc/40x: Save SRR0/SRR1 and r10/r11 earlier in critical exception Christophe Leroy
2021-03-12 12:50 ` [PATCH v3 05/41] powerpc/40x: Reorder a few instructions in critical exception prolog Christophe Leroy
2021-03-12 12:50 ` [PATCH v3 06/41] powerpc/40x: Prepare for enabling MMU " Christophe Leroy
2021-03-12 12:50 ` [PATCH v3 07/41] powerpc/40x: Prepare normal exception handler for enabling MMU early Christophe Leroy
2021-03-12 12:50 ` [PATCH v3 08/41] powerpc/32: Reconcile interrupts in C Christophe Leroy
2021-03-12 12:50 ` [PATCH v3 09/41] powerpc/32: Entry cpu time accounting " Christophe Leroy
2021-03-12 12:50 ` [PATCH v3 10/41] powerpc/32: Handle bookE debugging in C in exception entry Christophe Leroy
2021-03-12 12:50 ` [PATCH v3 11/41] powerpc/32: Use fast instruction to set MSR RI in exception prolog on 8xx Christophe Leroy
2021-03-12 12:50 ` [PATCH v3 12/41] powerpc/32: Remove ksp_limit Christophe Leroy
2021-03-12 12:50 ` [PATCH v3 13/41] powerpc/32: Always enable data translation in exception prolog Christophe Leroy
2021-03-12 12:50 ` [PATCH v3 14/41] powerpc/32: Tag DAR in EXCEPTION_PROLOG_2 for the 8xx Christophe Leroy
2021-03-12 12:50 ` [PATCH v3 15/41] powerpc/32: Enable instruction translation at the same time as data translation Christophe Leroy
2021-03-12 12:50 ` [PATCH v3 16/41] powerpc/32: Statically initialise first emergency context Christophe Leroy
2021-03-12 12:50 ` [PATCH v3 17/41] powerpc/32: Add vmap_stack_overflow label inside the macro Christophe Leroy
2021-03-12 12:50 ` [PATCH v3 18/41] powerpc/32: Use START_EXCEPTION() as much as possible Christophe Leroy
2021-03-12 12:50 ` [PATCH v3 19/41] powerpc/32: Move exception prolog code into .text once MMU is back on Christophe Leroy
2021-03-12 12:50 ` [PATCH v3 20/41] powerpc/32: Provide a name to exception prolog continuation in virtual mode Christophe Leroy
2021-03-12 12:50 ` [PATCH v3 21/41] powerpc/32: Refactor booke critical registers saving Christophe Leroy
2021-03-12 12:50 ` [PATCH v3 22/41] powerpc/32: Perform normal function call in exception entry Christophe Leroy
2021-03-12 12:50 ` [PATCH v3 23/41] powerpc/32: Always save non volatile registers on " Christophe Leroy
2021-03-12 12:50 ` [PATCH v3 24/41] powerpc/32: Replace ASM exception exit by C exception exit from ppc64 Christophe Leroy
2021-03-12 12:50 ` [PATCH v3 25/41] powerpc/32: Don't save thread.regs on interrupt entry Christophe Leroy
2021-03-12 12:50 ` [PATCH v3 26/41] powerpc/32: Set regs parameter in r3 in transfer_to_handler Christophe Leroy
2021-03-12 12:50 ` [PATCH v3 27/41] powerpc/32: Call bad_page_fault() from do_page_fault() Christophe Leroy
2021-03-12 12:50 ` [PATCH v3 28/41] powerpc/32: Save trap number on stack in exception prolog Christophe Leroy
2021-03-12 12:50 ` [PATCH v3 29/41] powerpc/32: Add a prepare_transfer_to_handler macro for exception prologs Christophe Leroy
2021-03-12 12:50 ` [PATCH v3 30/41] powerpc/32: Only restore non volatile registers when required Christophe Leroy
2021-03-12 12:50 ` [PATCH v3 31/41] powerpc/32: Dismantle EXC_XFER_STD/LITE/TEMPLATE Christophe Leroy
2021-07-31 10:12   ` Finn Thain
2021-07-31 15:39     ` Christophe Leroy
2021-08-01  1:21       ` Finn Thain
     [not found]         ` <34e52975-4ab9-4eb4-3501-02376e38a27c@yahoo.com>
2021-08-01 23:48           ` Finn Thain
2021-08-02 15:19         ` LEROY Christophe
2021-08-03  0:43           ` Finn Thain
     [not found]           ` <62d36dbb-4b39-f5bd-7962-4dcfbea729b3@linux-m68k.org>
     [not found]             ` <683c8156-97b0-5ba7-ce31-2e8613089836@yahoo.com>
2021-08-04  4:57               ` Finn Thain
2021-07-31 16:40     ` Christophe Leroy
2021-08-01  1:39       ` Finn Thain
2021-03-12 12:50 ` [PATCH v3 32/41] powerpc/32: Remove the xfer parameter in EXCEPTION() macro Christophe Leroy
2021-03-12 12:50 ` [PATCH v3 33/41] powerpc/32: Refactor saving of volatile registers in exception prologs Christophe Leroy
2021-03-12 12:50 ` [PATCH v3 34/41] powerpc/32: Save remaining registers in exception prolog Christophe Leroy
2021-03-12 12:50 ` [PATCH v3 35/41] powerpc/32: Return directly from power_save_ppc32_restore() Christophe Leroy
2021-03-12 12:50 ` [PATCH v3 36/41] powerpc/32: Only use prepare_transfer_to_handler function on book3s/32 and e500 Christophe Leroy
2021-03-12 12:50 ` Christophe Leroy [this message]
2021-03-27 17:24   ` [PATCH v3 37/41] powerpc/32s: Move KUEP locking/unlocking in C Christophe Leroy
2021-03-12 12:50 ` [PATCH v3 38/41] powerpc/64s: Make kuap_check_amr() and kuap_get_and_check_amr() generic Christophe Leroy
2021-03-12 12:50 ` [PATCH v3 39/41] powerpc/32s: Create C version of kuap save/restore/check helpers Christophe Leroy
2021-03-12 12:50 ` [PATCH v3 40/41] powerpc/8xx: " Christophe Leroy
2021-03-12 12:50 ` [PATCH v3 41/41] powerpc/32: Manage KUAP in C Christophe Leroy
2021-03-31  1:09 ` [PATCH v3 00/41] powerpc/32: Switch to interrupt entry/exit " Michael Ellerman

Reply instructions:

You may reply publicly to this message via plain-text email
using any one of the following methods:

* Save the following mbox file, import it into your mail client,
  and reply-to-all from there: mbox

  Avoid top-posting and favor interleaved quoting:
  https://en.wikipedia.org/wiki/Posting_style#Interleaved_style

* Reply using the --to, --cc, and --in-reply-to
  switches of git-send-email(1):

  git send-email \
    --in-reply-to=4eadd873927e9a73c3d1dfe2f9497353465514cf.1615552867.git.christophe.leroy@csgroup.eu \
    --to=christophe.leroy@csgroup.eu \
    --cc=benh@kernel.crashing.org \
    --cc=linux-kernel@vger.kernel.org \
    --cc=linuxppc-dev@lists.ozlabs.org \
    --cc=mpe@ellerman.id.au \
    --cc=npiggin@gmail.com \
    --cc=paulus@samba.org \
    /path/to/YOUR_REPLY

  https://kernel.org/pub/software/scm/git/docs/git-send-email.html

* If your mail client supports setting the In-Reply-To header
  via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line before the message body. 
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).