From: Pavel Tatashin <pasha.tatashin@soleen.com>
To: pasha.tatashin@soleen.com, jmorris@namei.org, sashal@kernel.org,
ebiederm@xmission.com, kexec@lists.infradead.org,
linux-kernel@vger.kernel.org, corbet@lwn.net,
catalin.marinas@arm.com, will@kernel.org,
linux-arm-kernel@lists.infradead.org, marc.zyngier@arm.com,
james.morse@arm.com, vladimir.murzin@arm.com,
matthias.bgg@gmail.com, bhsharma@redhat.com, linux-mm@kvack.org,
mark.rutland@arm.com, steve.capper@arm.com, rfontana@redhat.com,
tglx@linutronix.de
Subject: [PATCH v8 24/25] arm64: kexec: enable MMU during kexec relocation
Date: Wed, 4 Dec 2019 10:59:37 -0500 [thread overview]
Message-ID: <20191204155938.2279686-25-pasha.tatashin@soleen.com> (raw)
In-Reply-To: <20191204155938.2279686-1-pasha.tatashin@soleen.com>
Now, that we have transitional page tables configured, temporarily enable
MMU to allow faster relocation of segments to final destination.
The performance data: for a moderate size kernel + initramfs: 25M the
relocation was taking 0.382s, with enabled MMU it now takes
0.019s only or x20 improvement.
The time is proportional to the size of relocation, therefore if initramfs
is larger, 100M it could take over a second.
Signed-off-by: Pavel Tatashin <pasha.tatashin@soleen.com>
---
arch/arm64/kernel/relocate_kernel.S | 135 +++++++++++++++++-----------
1 file changed, 83 insertions(+), 52 deletions(-)
diff --git a/arch/arm64/kernel/relocate_kernel.S b/arch/arm64/kernel/relocate_kernel.S
index 67efa42575a5..e623b02f72a3 100644
--- a/arch/arm64/kernel/relocate_kernel.S
+++ b/arch/arm64/kernel/relocate_kernel.S
@@ -4,6 +4,8 @@
*
* Copyright (C) Linaro.
* Copyright (C) Huawei Futurewei Technologies.
+ * Copyright (c) 2019, Microsoft Corporation.
+ * Pavel Tatashin <patatash@linux.microsoft.com>
*/
#include <linux/kexec.h>
@@ -15,6 +17,48 @@
#include <asm/sysreg.h>
GLOBAL(kexec_relocate_code_start)
+/* Invalidae TLB */
+.macro tlb_invalidate
+ dsb sy
+ dsb ish
+ tlbi vmalle1
+ dsb ish
+ isb
+.endm
+
+/* Turn-off mmu at level specified by sctlr */
+.macro turn_off_mmu sctlr, tmp1, tmp2
+ mrs \tmp1, \sctlr
+ ldr \tmp2, =SCTLR_ELx_FLAGS
+ bic \tmp1, \tmp1, \tmp2
+ pre_disable_mmu_workaround
+ msr \sctlr, \tmp1
+ isb
+.endm
+
+/* Turn-on mmu at level specified by sctlr */
+.macro turn_on_mmu sctlr, tmp1, tmp2
+ mrs \tmp1, \sctlr
+ ldr \tmp2, =SCTLR_ELx_FLAGS
+ orr \tmp1, \tmp1, \tmp2
+ msr \sctlr, \tmp1
+ ic iallu
+ dsb nsh
+ isb
+.endm
+
+/*
+ * Set ttbr0 and ttbr1, called while MMU is disabled, so no need to temporarily
+ * set zero_page table. Invalidate TLB after new tables are set.
+ */
+.macro set_ttbr arg, tmp
+ ldr \tmp, [\arg, #KEXEC_KRELOC_TRANS_TTBR0]
+ msr ttbr0_el1, \tmp
+ ldr \tmp, [\arg, #KEXEC_KRELOC_TRANS_TTBR1]
+ offset_ttbr1 \tmp
+ msr ttbr1_el1, \tmp
+ isb
+.endm
/*
* arm64_relocate_new_kernel - Put a 2nd stage image in place and boot it.
@@ -26,65 +70,52 @@ GLOBAL(kexec_relocate_code_start)
* symbols arm64_relocate_new_kernel and arm64_relocate_new_kernel_end. The
* machine_kexec() routine will copy arm64_relocate_new_kernel to the kexec
* safe memory that has been set up to be preserved during the copy operation.
+ *
+ * This function temporarily enables MMU if kernel relocation is needed.
+ * Also, if we enter this function at EL2 on non-VHE kernel, we temporarily go
+ * to EL1 to enable MMU, and escalate back to EL2 at the end to do the jump to
+ * the new kernel. This is determined by presence of el2_vector.
*/
ENTRY(arm64_relocate_new_kernel)
- /* Clear the sctlr_el2 flags. */
- mrs x2, CurrentEL
- cmp x2, #CurrentEL_EL2
+ mrs x1, CurrentEL
+ cmp x1, #CurrentEL_EL2
b.ne 1f
- mrs x2, sctlr_el2
- ldr x1, =SCTLR_ELx_FLAGS
- bic x2, x2, x1
- pre_disable_mmu_workaround
- msr sctlr_el2, x2
- isb
-1: /* Check if the new image needs relocation. */
- ldr x16, [x0, #KEXEC_KRELOC_HEAD] /* x16 = kimage_head */
- tbnz x16, IND_DONE_BIT, .Ldone
- raw_dcache_line_size x15, x1 /* x15 = dcache line size */
-.Lloop:
- and x12, x16, PAGE_MASK /* x12 = addr */
- /* Test the entry flags. */
-.Ltest_source:
- tbz x16, IND_SOURCE_BIT, .Ltest_indirection
-
- /* Invalidate dest page to PoC. */
- mov x2, x13
- add x20, x2, #PAGE_SIZE
- sub x1, x15, #1
- bic x2, x2, x1
-2: dc ivac, x2
- add x2, x2, x15
- cmp x2, x20
- b.lo 2b
- dsb sy
-
- copy_page x13, x12, x1, x2, x3, x4, x5, x6, x7, x8
- b .Lnext
-.Ltest_indirection:
- tbz x16, IND_INDIRECTION_BIT, .Ltest_destination
- mov x14, x12 /* ptr = addr */
- b .Lnext
-.Ltest_destination:
- tbz x16, IND_DESTINATION_BIT, .Lnext
- mov x13, x12 /* dest = addr */
-.Lnext:
- ldr x16, [x14], #8 /* entry = *ptr++ */
- tbz x16, IND_DONE_BIT, .Lloop /* while (!(entry & DONE)) */
-.Ldone:
- /* wait for writes from copy_page to finish */
- dsb nsh
- ic iallu
- dsb nsh
- isb
-
- /* Start new image. */
- ldr x4, [x0, #KEXEC_KRELOC_ENTRY_ADDR] /* x4 = kimage_start */
+ turn_off_mmu sctlr_el2, x1, x2 /* Turn off MMU at EL2 */
+1: mov x20, xzr /* x20 will hold vector value */
+ ldr x11, [x0, #KEXEC_KRELOC_COPY_LEN]
+ cbz x11, 5f /* Check if need to relocate */
+ ldr x20, [x0, #KEXEC_KRELOC_EL2_VECTOR]
+ cbz x20, 2f /* need to reduce to EL1? */
+ msr vbar_el2, x20 /* el2_vector present, means */
+ adr x1, 2f /* we will do copy in el1 but */
+ msr elr_el2, x1 /* do final jump from el2 */
+ eret /* Reduce to EL1 */
+2: set_ttbr x0, x1 /* Set our page tables */
+ tlb_invalidate
+ turn_on_mmu sctlr_el1, x1, x2 /* Turn MMU back on */
+ ldr x1, [x0, #KEXEC_KRELOC_DST_ADDR];
+ ldr x2, [x0, #KEXEC_KRELOC_SRC_ADDR];
+ mov x12, x1 /* x12 dst backup */
+3: copy_page x1, x2, x3, x4, x5, x6, x7, x8, x9, x10
+ sub x11, x11, #PAGE_SIZE
+ cbnz x11, 3b /* page copy loop */
+ raw_dcache_line_size x2, x3 /* x2 = dcache line size */
+ sub x3, x2, #1 /* x3 = dcache_size - 1 */
+ bic x12, x12, x3
+4: dc cvau, x12 /* Flush D-cache */
+ add x12, x12, x2
+ cmp x12, x1 /* Compare to dst + len */
+ b.ne 4b /* D-cache flush loop */
+ turn_off_mmu sctlr_el1, x1, x2 /* Turn off MMU */
+ tlb_invalidate /* Invalidate TLB */
+5: ldr x4, [x0, #KEXEC_KRELOC_ENTRY_ADDR] /* x4 = kimage_start */
ldr x3, [x0, #KEXEC_KRELOC_KERN_ARG3]
ldr x2, [x0, #KEXEC_KRELOC_KERN_ARG2]
ldr x1, [x0, #KEXEC_KRELOC_KERN_ARG1]
ldr x0, [x0, #KEXEC_KRELOC_KERN_ARG0] /* x0 = dtb address */
- br x4
+ cbnz x20, 6f /* need to escalate to el2? */
+ br x4 /* Jump to new world */
+6: hvc #0 /* enters kexec_el1_sync */
.ltorg
END(arm64_relocate_new_kernel)
--
2.24.0
next prev parent reply other threads:[~2019-12-04 16:00 UTC|newest]
Thread overview: 29+ messages / expand[flat|nested] mbox.gz Atom feed top
2019-12-04 15:59 [PATCH v8 00/25] arm64: MMU enabled kexec relocation Pavel Tatashin
2019-12-04 15:59 ` [PATCH v8 01/25] kexec: quiet down kexec reboot Pavel Tatashin
2019-12-04 15:59 ` [PATCH v8 02/25] kexec: add machine_kexec_post_load() Pavel Tatashin
2019-12-04 15:59 ` [PATCH v8 03/25] arm64: kexec: remove unnecessary debug prints Pavel Tatashin
2019-12-04 15:59 ` [PATCH v8 04/25] arm64: kexec: make dtb_mem always enabled Pavel Tatashin
2019-12-04 15:59 ` [PATCH v8 05/25] arm64: hibernate: pass the allocated pgdp to ttbr0 Pavel Tatashin
2019-12-04 15:59 ` [PATCH v8 06/25] arm64: hibernate: use get_safe_page directly Pavel Tatashin
2019-12-04 15:59 ` [PATCH v8 07/25] arm64: hibernate: remove gotos as they are not needed Pavel Tatashin
2019-12-04 15:59 ` [PATCH v8 08/25] arm64: hibernate: rename dst to page in create_safe_exec_page Pavel Tatashin
2019-12-04 15:59 ` [PATCH v8 09/25] arm64: hibernate: add PUD_SECT_RDONLY Pavel Tatashin
2019-12-04 15:59 ` [PATCH v8 10/25] arm64: hibernate: add trans_pgd public functions Pavel Tatashin
2019-12-04 15:59 ` [PATCH v8 11/25] arm64: hibernate: move page handling function to new trans_pgd.c Pavel Tatashin
2019-12-04 15:59 ` [PATCH v8 12/25] arm64: trans_pgd: make trans_pgd_map_page generic Pavel Tatashin
2019-12-04 15:59 ` [PATCH v8 13/25] arm64: trans_pgd: pass allocator trans_pgd_create_copy Pavel Tatashin
2019-12-04 15:59 ` [PATCH v8 14/25] arm64: trans_pgd: pass NULL instead of init_mm to *_populate functions Pavel Tatashin
2019-12-04 15:59 ` [PATCH v8 15/25] arm64: kexec: move relocation function setup Pavel Tatashin
2019-12-04 15:59 ` [PATCH v8 16/25] arm64: kexec: call kexec_image_info only once Pavel Tatashin
2019-12-04 15:59 ` [PATCH v8 17/25] arm64: kexec: cpu_soft_restart change argument types Pavel Tatashin
2019-12-04 15:59 ` [PATCH v8 18/25] arm64: kexec: arm64_relocate_new_kernel clean-ups Pavel Tatashin
2019-12-04 15:59 ` [PATCH v8 19/25] arm64: kexec: arm64_relocate_new_kernel don't use x0 as temp Pavel Tatashin
2019-12-04 15:59 ` [PATCH v8 20/25] arm64: kexec: add expandable argument to relocation function Pavel Tatashin
2019-12-04 15:59 ` [PATCH v8 21/25] arm64: kexec: offset for " Pavel Tatashin
2019-12-04 15:59 ` [PATCH v8 22/25] arm64: kexec: kexec EL2 vectors Pavel Tatashin
2019-12-04 15:59 ` [PATCH v8 23/25] arm64: kexec: configure trans_pgd page table for kexec Pavel Tatashin
2019-12-04 15:59 ` Pavel Tatashin [this message]
2019-12-04 15:59 ` [PATCH v8 25/25] arm64: kexec: remove head from relocation argument Pavel Tatashin
2020-01-08 17:32 ` [PATCH v8 00/25] arm64: MMU enabled kexec relocation Will Deacon
2020-01-08 17:59 ` Pavel Tatashin
2020-01-15 14:36 ` James Morse
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20191204155938.2279686-25-pasha.tatashin@soleen.com \
--to=pasha.tatashin@soleen.com \
--cc=bhsharma@redhat.com \
--cc=catalin.marinas@arm.com \
--cc=corbet@lwn.net \
--cc=ebiederm@xmission.com \
--cc=james.morse@arm.com \
--cc=jmorris@namei.org \
--cc=kexec@lists.infradead.org \
--cc=linux-arm-kernel@lists.infradead.org \
--cc=linux-kernel@vger.kernel.org \
--cc=linux-mm@kvack.org \
--cc=marc.zyngier@arm.com \
--cc=mark.rutland@arm.com \
--cc=matthias.bgg@gmail.com \
--cc=rfontana@redhat.com \
--cc=sashal@kernel.org \
--cc=steve.capper@arm.com \
--cc=tglx@linutronix.de \
--cc=vladimir.murzin@arm.com \
--cc=will@kernel.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).