From: Mark Rutland <mark.rutland@arm.com>
To: linux-arm-kernel@lists.infradead.org
Cc: catalin.marinas@arm.com, james.morse@arm.com, joey.gouly@arm.com,
mark.rutland@arm.com, maz@kernel.org, will@kernel.org
Subject: [PATCH 3/4] arm64: entry: move bulk of ret_to_user to C
Date: Mon, 2 Aug 2021 15:07:32 +0100 [thread overview]
Message-ID: <20210802140733.52716-4-mark.rutland@arm.com> (raw)
In-Reply-To: <20210802140733.52716-1-mark.rutland@arm.com>
In `ret_to_user` we perform some conditional work depending on the
thread flags, then perform some IRQ/context tracking which is intended
to balance with the IRQ/context tracking performed in the entry C code.
For simplicity and consistency, it would be preferable to move this all
to C. As a step towards that, this patch moves the conditional work and
IRQ/context tracking into a C helper function. To aid bisectability,
this is called from the `ret_to_user` assembly, and a subsequent patch
will move the call to C code.
As local_daif_mask() handles all necessary tracing and PMR manipulation,
we no longer need to handle this explicitly. As we call
exit_to_user_mode() directly, the `user_enter_irqoff` macro is no longer
used, and can be removed. As enter_from_user_mode() and
exit_to_user_mode() are no longer called from assembly, these can be
made static, and as these are typically very small, they are marked
__always_inline to avoid the overhead of a function call.
For now, enablement of single-step is left in entry.S, and for this we
still need to read the flags in ret_to_user(). It is safe to read this
separately as TIF_SINGLESTEP is not part of _TIF_WORK_MASK.
There should be no functional change as a result of this patch.
Signed-off-by: Mark Rutland <mark.rutland@arm.com>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: James Morse <james.morse@arm.com>
Cc: Joey Gouly <joey.gouly@arm.com>
Cc: Marc Zyngier <maz@kernel.org>
Cc: Will Deacon <will@kernel.org>
---
arch/arm64/include/asm/exception.h | 5 +++--
arch/arm64/kernel/entry-common.c | 21 +++++++++++++++++++--
arch/arm64/kernel/entry.S | 36 +++---------------------------------
arch/arm64/kernel/signal.c | 3 +--
4 files changed, 26 insertions(+), 39 deletions(-)
diff --git a/arch/arm64/include/asm/exception.h b/arch/arm64/include/asm/exception.h
index 4afbc45b8bb0..339477dca551 100644
--- a/arch/arm64/include/asm/exception.h
+++ b/arch/arm64/include/asm/exception.h
@@ -55,8 +55,8 @@ asmlinkage void el0t_32_error_handler(struct pt_regs *regs);
asmlinkage void call_on_irq_stack(struct pt_regs *regs,
void (*func)(struct pt_regs *));
-asmlinkage void enter_from_user_mode(void);
-asmlinkage void exit_to_user_mode(void);
+asmlinkage void asm_exit_to_user_mode(struct pt_regs *regs);
+
void do_mem_abort(unsigned long far, unsigned int esr, struct pt_regs *regs);
void do_undefinstr(struct pt_regs *regs);
void do_bti(struct pt_regs *regs);
@@ -73,6 +73,7 @@ void do_el0_svc(struct pt_regs *regs);
void do_el0_svc_compat(struct pt_regs *regs);
void do_ptrauth_fault(struct pt_regs *regs, unsigned int esr);
void do_serror(struct pt_regs *regs, unsigned int esr);
+void do_notify_resume(struct pt_regs *regs, unsigned long thread_flags);
void panic_bad_stack(struct pt_regs *regs, unsigned int esr, unsigned long far);
#endif /* __ASM_EXCEPTION_H */
diff --git a/arch/arm64/kernel/entry-common.c b/arch/arm64/kernel/entry-common.c
index 6dc64f99f185..8ab4084d0f00 100644
--- a/arch/arm64/kernel/entry-common.c
+++ b/arch/arm64/kernel/entry-common.c
@@ -104,7 +104,7 @@ static __always_inline void __enter_from_user_mode(void)
trace_hardirqs_off_finish();
}
-asmlinkage void noinstr enter_from_user_mode(void)
+static __always_inline void enter_from_user_mode(void)
{
__enter_from_user_mode();
}
@@ -123,12 +123,29 @@ static __always_inline void __exit_to_user_mode(void)
lockdep_hardirqs_on(CALLER_ADDR0);
}
-asmlinkage void noinstr exit_to_user_mode(void)
+static __always_inline void exit_to_user_mode(void)
{
mte_check_tfsr_exit();
__exit_to_user_mode();
}
+static __always_inline void prepare_exit_to_user_mode(struct pt_regs *regs)
+{
+ unsigned long flags;
+
+ local_daif_mask();
+
+ flags = READ_ONCE(current_thread_info()->flags);
+ if (unlikely(flags & _TIF_WORK_MASK))
+ do_notify_resume(regs, flags);
+}
+
+asmlinkage void noinstr asm_exit_to_user_mode(struct pt_regs *regs)
+{
+ prepare_exit_to_user_mode(regs);
+ exit_to_user_mode();
+}
+
/*
* Handle IRQ/context state management when entering an NMI from user/kernel
* mode. Before this function is called it is not safe to call regular kernel
diff --git a/arch/arm64/kernel/entry.S b/arch/arm64/kernel/entry.S
index 863d44f73028..fe0a9bcc3e1f 100644
--- a/arch/arm64/kernel/entry.S
+++ b/arch/arm64/kernel/entry.S
@@ -29,16 +29,6 @@
#include <asm/asm-uaccess.h>
#include <asm/unistd.h>
-/*
- * Context tracking and irqflag tracing need to instrument transitions between
- * user and kernel mode.
- */
- .macro user_enter_irqoff
-#if defined(CONFIG_CONTEXT_TRACKING) || defined(CONFIG_TRACE_IRQFLAGS)
- bl exit_to_user_mode
-#endif
- .endm
-
.macro clear_gp_regs
.irp n,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29
mov x\n, xzr
@@ -585,37 +575,17 @@ SYM_CODE_START_LOCAL(ret_to_kernel)
kernel_exit 1
SYM_CODE_END(ret_to_kernel)
-/*
- * "slow" syscall return path.
- */
SYM_CODE_START_LOCAL(ret_to_user)
- disable_daif
- gic_prio_kentry_setup tmp=x3
-#ifdef CONFIG_TRACE_IRQFLAGS
- bl trace_hardirqs_off
-#endif
- ldr x19, [tsk, #TSK_TI_FLAGS]
- and x2, x19, #_TIF_WORK_MASK
- cbnz x2, work_pending
-finish_ret_to_user:
- user_enter_irqoff
+ mov x0, sp
+ bl asm_exit_to_user_mode
/* Ignore asynchronous tag check faults in the uaccess routines */
clear_mte_async_tcf
+ ldr x19, [tsk, #TSK_TI_FLAGS] // re-check for single-step
enable_step_tsk x19, x2
#ifdef CONFIG_GCC_PLUGIN_STACKLEAK
bl stackleak_erase
#endif
kernel_exit 0
-
-/*
- * Ok, we need to do extra processing, enter the slow path.
- */
-work_pending:
- mov x0, sp // 'regs'
- mov x1, x19
- bl do_notify_resume
- ldr x19, [tsk, #TSK_TI_FLAGS] // re-check for single-step
- b finish_ret_to_user
SYM_CODE_END(ret_to_user)
.popsection // .entry.text
diff --git a/arch/arm64/kernel/signal.c b/arch/arm64/kernel/signal.c
index f8192f4ae0b8..53c2c85efb34 100644
--- a/arch/arm64/kernel/signal.c
+++ b/arch/arm64/kernel/signal.c
@@ -924,8 +924,7 @@ static bool cpu_affinity_invalid(struct pt_regs *regs)
system_32bit_el0_cpumask());
}
-asmlinkage void do_notify_resume(struct pt_regs *regs,
- unsigned long thread_flags)
+void do_notify_resume(struct pt_regs *regs, unsigned long thread_flags)
{
do {
if (thread_flags & _TIF_NEED_RESCHED) {
--
2.11.0
_______________________________________________
linux-arm-kernel mailing list
linux-arm-kernel@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel
next prev parent reply other threads:[~2021-08-02 14:09 UTC|newest]
Thread overview: 10+ messages / expand[flat|nested] mbox.gz Atom feed top
2021-08-02 14:07 [PATCH 0/4] arm64: entry: rework user return handling Mark Rutland
2021-08-02 14:07 ` [PATCH 1/4] arm64: entry: consolidate entry/exit helpers Mark Rutland
2021-08-03 12:07 ` Joey Gouly
2021-08-02 14:07 ` [PATCH 2/4] arm64: entry: clarify " Mark Rutland
2021-08-03 11:53 ` Joey Gouly
2021-08-02 14:07 ` Mark Rutland [this message]
2021-08-03 12:06 ` [PATCH 3/4] arm64: entry: move bulk of ret_to_user to C Joey Gouly
2021-08-02 14:07 ` [PATCH 4/4] arm64: entry: call exit_to_user_mode() from C Mark Rutland
2021-08-03 12:17 ` Joey Gouly
2021-08-05 13:38 ` [PATCH 0/4] arm64: entry: rework user return handling Catalin Marinas
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20210802140733.52716-4-mark.rutland@arm.com \
--to=mark.rutland@arm.com \
--cc=catalin.marinas@arm.com \
--cc=james.morse@arm.com \
--cc=joey.gouly@arm.com \
--cc=linux-arm-kernel@lists.infradead.org \
--cc=maz@kernel.org \
--cc=will@kernel.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).