[PATCH v2 0/2] arm64: entry: Simplify KPTI trampoline exit code

[PATCH v2 0/2] arm64: entry: Simplify KPTI trampoline exit code

[Ard Biesheuvel]

Changes since v1:
- add a separate patch that merges the native and compat code paths
- improve commit log on patch #2

Ard Biesheuvel (2):
  arm64: entry: Preserve/restore X29 even for compat tasks
  arm64: entry: Simplify tramp_alias macro and tramp_exit routine

 arch/arm64/kernel/entry.S | 57 +++++++-------------
 1 file changed, 20 insertions(+), 37 deletions(-)

-- 
2.39.2


_______________________________________________
linux-arm-kernel mailing list
linux-arm-kernel@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel

[PATCH v2 1/2] arm64: entry: Preserve/restore X29 even for compat tasks

[Ard Biesheuvel]

Currently, the KPTI trampoline code for returning to user space takes
care to only preserve X29 into FAR_EL1 for native tasks, as compat tasks
don't have access to this register anyway, and so preserving it is not
necessary. It also means it does not need to be restored, and so we have
two code paths for returning back to user space: the native one that
restores X29 from FAR_EL1, and the compat one that leaves X29 clobbered,
containing the value of TTBR1_EL1, which carries a physical address
pointing somewhere into the kernel image.

This is needlessly complex, and given that FAR_EL1 becomes UNKNOWN after
an exception return anway, the only benefit of avoiding the preserve and
restore is that we can skip the system register write and read.

So let's simplify this, and collapse the two code paths into one that
always preserves X29 into FAR_EL1, and always restores it again after
the TTBR switch.

Signed-off-by: Ard Biesheuvel <ardb@kernel.org>
---
 arch/arm64/kernel/entry.S | 10 ----------
 1 file changed, 10 deletions(-)

diff --git a/arch/arm64/kernel/entry.S b/arch/arm64/kernel/entry.S
index ab2a6e33c0528d82..16fbd0d9790dd436 100644
--- a/arch/arm64/kernel/entry.S
+++ b/arch/arm64/kernel/entry.S
@@ -435,13 +435,9 @@ alternative_if_not ARM64_UNMAP_KERNEL_AT_EL0
 	eret
 alternative_else_nop_endif
 #ifdef CONFIG_UNMAP_KERNEL_AT_EL0
-	bne	4f
 	msr	far_el1, x29
 	tramp_alias	x30, tramp_exit_native, x29
 	br	x30
-4:
-	tramp_alias	x30, tramp_exit_compat, x29
-	br	x30
 #endif
 	.else
 	ldr	lr, [sp, #S_LR]
@@ -740,9 +736,7 @@ alternative_else_nop_endif
 	msr	vbar_el1, x30
 	ldr	lr, [sp, #S_LR]
 	tramp_unmap_kernel	x29
-	.if	\regsize == 64
 	mrs	x29, far_el1
-	.endif
 	add	sp, sp, #PT_REGS_SIZE		// restore sp
 	eret
 	sb
@@ -780,10 +774,6 @@ SYM_CODE_END(tramp_vectors)
 SYM_CODE_START(tramp_exit_native)
 	tramp_exit
 SYM_CODE_END(tramp_exit_native)
-
-SYM_CODE_START(tramp_exit_compat)
-	tramp_exit	32
-SYM_CODE_END(tramp_exit_compat)
 	.popsection				// .entry.tramp.text
 #endif /* CONFIG_UNMAP_KERNEL_AT_EL0 */
 
-- 
2.39.2


_______________________________________________
linux-arm-kernel mailing list
linux-arm-kernel@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel

[PATCH v2 2/2] arm64: entry: Simplify tramp_alias macro and tramp_exit routine

[Ard Biesheuvel]

The tramp_alias macro constructs the virtual alias of a symbol in the
trampoline text mapping, based on its kernel text address, and does so
in a way that is more convoluted than necessary. So let's simplify that.

Also, now that the address of the vector table is kept in a per-CPU
variable, there is no need to defer the load and the assignment of
VBAR_EL1 to tramp_exit(). This means we can use a PC-relative reference
to the per-CPU variable instead of storing its absolute address in a
global variable in the trampoline rodata.

And given that tramp_alias no longer needs a temp register, this means
we can restore X30 earlier as well, and only leave X29 for tramp_exit()
to restore.

While at it, give some related symbols static linkage, considering that
they are only referenced from the object file that defines them.

Signed-off-by: Ard Biesheuvel <ardb@kernel.org>
---
 arch/arm64/kernel/entry.S | 47 +++++++++-----------
 1 file changed, 20 insertions(+), 27 deletions(-)

diff --git a/arch/arm64/kernel/entry.S b/arch/arm64/kernel/entry.S
index 16fbd0d9790dd436..a40e5e50fa55232e 100644
--- a/arch/arm64/kernel/entry.S
+++ b/arch/arm64/kernel/entry.S
@@ -101,12 +101,11 @@
 .org .Lventry_start\@ + 128	// Did we overflow the ventry slot?
 	.endm
 
-	.macro tramp_alias, dst, sym, tmp
-	mov_q	\dst, TRAMP_VALIAS
-	adr_l	\tmp, \sym
-	add	\dst, \dst, \tmp
-	adr_l	\tmp, .entry.tramp.text
-	sub	\dst, \dst, \tmp
+	.macro	tramp_alias, dst, sym
+	.set	.Lalias\@, TRAMP_VALIAS + \sym - .entry.tramp.text
+	movz	\dst, :abs_g2_s:.Lalias\@
+	movk	\dst, :abs_g1_nc:.Lalias\@
+	movk	\dst, :abs_g0_nc:.Lalias\@
 	.endm
 
 	/*
@@ -436,8 +435,13 @@ alternative_if_not ARM64_UNMAP_KERNEL_AT_EL0
 alternative_else_nop_endif
 #ifdef CONFIG_UNMAP_KERNEL_AT_EL0
 	msr	far_el1, x29
-	tramp_alias	x30, tramp_exit_native, x29
-	br	x30
+
+	ldr_this_cpu	x30, this_cpu_vector, x29
+	tramp_alias	x29, tramp_exit
+	msr		vbar_el1, x30		// install vector table
+	ldr		lr, [sp, #S_LR]		// restore x30
+	add		sp, sp, #PT_REGS_SIZE	// restore sp
+	br		x29
 #endif
 	.else
 	ldr	lr, [sp, #S_LR]
@@ -728,20 +732,6 @@ alternative_else_nop_endif
 .org 1b + 128	// Did we overflow the ventry slot?
 	.endm
 
-	.macro tramp_exit, regsize = 64
-	tramp_data_read_var	x30, this_cpu_vector
-	get_this_cpu_offset x29
-	ldr	x30, [x30, x29]
-
-	msr	vbar_el1, x30
-	ldr	lr, [sp, #S_LR]
-	tramp_unmap_kernel	x29
-	mrs	x29, far_el1
-	add	sp, sp, #PT_REGS_SIZE		// restore sp
-	eret
-	sb
-	.endm
-
 	.macro	generate_tramp_vector,	kpti, bhb
 .Lvector_start\@:
 	.space	0x400
@@ -762,7 +752,7 @@ alternative_else_nop_endif
  */
 	.pushsection ".entry.tramp.text", "ax"
 	.align	11
-SYM_CODE_START_NOALIGN(tramp_vectors)
+SYM_CODE_START_LOCAL_NOALIGN(tramp_vectors)
 #ifdef CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY
 	generate_tramp_vector	kpti=1, bhb=BHB_MITIGATION_LOOP
 	generate_tramp_vector	kpti=1, bhb=BHB_MITIGATION_FW
@@ -771,9 +761,12 @@ SYM_CODE_START_NOALIGN(tramp_vectors)
 	generate_tramp_vector	kpti=1, bhb=BHB_MITIGATION_NONE
 SYM_CODE_END(tramp_vectors)
 
-SYM_CODE_START(tramp_exit_native)
-	tramp_exit
-SYM_CODE_END(tramp_exit_native)
+SYM_CODE_START_LOCAL(tramp_exit)
+	tramp_unmap_kernel	x29
+	mrs		x29, far_el1		// restore x29
+	eret
+	sb
+SYM_CODE_END(tramp_exit)
 	.popsection				// .entry.tramp.text
 #endif /* CONFIG_UNMAP_KERNEL_AT_EL0 */
 
@@ -1067,7 +1060,7 @@ alternative_if_not ARM64_UNMAP_KERNEL_AT_EL0
 alternative_else_nop_endif
 
 #ifdef CONFIG_UNMAP_KERNEL_AT_EL0
-	tramp_alias	dst=x5, sym=__sdei_asm_exit_trampoline, tmp=x3
+	tramp_alias	dst=x5, sym=__sdei_asm_exit_trampoline
 	br	x5
 #endif
 SYM_CODE_END(__sdei_asm_handler)
-- 
2.39.2


_______________________________________________
linux-arm-kernel mailing list
linux-arm-kernel@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel

Re: [PATCH v2 0/2] arm64: entry: Simplify KPTI trampoline exit code

[Will Deacon]

On Tue, Apr 18, 2023 at 04:36:02PM +0200, Ard Biesheuvel wrote:
> Changes since v1:
> - add a separate patch that merges the native and compat code paths
> - improve commit log on patch #2
> 
> Ard Biesheuvel (2):
>   arm64: entry: Preserve/restore X29 even for compat tasks
>   arm64: entry: Simplify tramp_alias macro and tramp_exit routine
> 
>  arch/arm64/kernel/entry.S | 57 +++++++-------------
>  1 file changed, 20 insertions(+), 37 deletions(-)

For both patches:

Reviewed-by: Will Deacon <will@kernel.org>

Will

_______________________________________________
linux-arm-kernel mailing list
linux-arm-kernel@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel

Re: [PATCH v2 0/2] arm64: entry: Simplify KPTI trampoline exit code

[Catalin Marinas]

On Tue, 18 Apr 2023 16:36:02 +0200, Ard Biesheuvel wrote:
> Changes since v1:
> - add a separate patch that merges the native and compat code paths
> - improve commit log on patch #2
> 
> Ard Biesheuvel (2):
>   arm64: entry: Preserve/restore X29 even for compat tasks
>   arm64: entry: Simplify tramp_alias macro and tramp_exit routine
> 
> [...]

Applied to arm64 (for-next/kpti), thanks!

[1/2] arm64: entry: Preserve/restore X29 even for compat tasks
      https://git.kernel.org/arm64/c/0936243cabf0
[2/2] arm64: entry: Simplify tramp_alias macro and tramp_exit routine
      https://git.kernel.org/arm64/c/211ceca377f4

-- 
Catalin


_______________________________________________
linux-arm-kernel mailing list
linux-arm-kernel@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel