From: Pavel Tatashin <pasha.tatashin@soleen.com>
To: pasha.tatashin@soleen.com, jmorris@namei.org, sashal@kernel.org,
ebiederm@xmission.com, kexec@lists.infradead.org,
linux-kernel@vger.kernel.org, corbet@lwn.net,
catalin.marinas@arm.com, will@kernel.org,
linux-arm-kernel@lists.infradead.org, maz@kernel.org,
james.morse@arm.com, vladimir.murzin@arm.com,
matthias.bgg@gmail.com, linux-mm@kvack.org, mark.rutland@arm.com,
steve.capper@arm.com, rfontana@redhat.com, tglx@linutronix.de,
selindag@gmail.com, tyhicks@linux.microsoft.com,
kernelfans@gmail.com, akpm@linux-foundation.org,
madvenka@linux.microsoft.com
Subject: [PATCH v15 12/15] arm64: kexec: keep MMU enabled during kexec relocation
Date: Tue, 8 Jun 2021 20:44:16 -0400 [thread overview]
Message-ID: <20210609004419.936873-13-pasha.tatashin@soleen.com> (raw)
In-Reply-To: <20210609004419.936873-1-pasha.tatashin@soleen.com>
Now, that we have linear map page tables configured, keep MMU enabled
to allow faster relocation of segments to final destination.
Cavium ThunderX2:
Kernel Image size: 38M Iniramfs size: 46M Total relocation size: 84M
MMU-disabled:
relocation 7.489539915s
MMU-enabled:
relocation 0.03946095s
Broadcom Stingray:
The performance data: for a moderate size kernel + initramfs: 25M the
relocation was taking 0.382s, with enabled MMU it now takes
0.019s only or x20 improvement.
The time is proportional to the size of relocation, therefore if initramfs
is larger, 100M it could take over a second.
Signed-off-by: Pavel Tatashin <pasha.tatashin@soleen.com>
---
arch/arm64/include/asm/kexec.h | 3 +++
arch/arm64/kernel/asm-offsets.c | 1 +
arch/arm64/kernel/machine_kexec.c | 16 +++++++++++----
arch/arm64/kernel/relocate_kernel.S | 31 +++++++++++++++++++----------
4 files changed, 36 insertions(+), 15 deletions(-)
diff --git a/arch/arm64/include/asm/kexec.h b/arch/arm64/include/asm/kexec.h
index d678f0ceb7ee..dca6dedc3b25 100644
--- a/arch/arm64/include/asm/kexec.h
+++ b/arch/arm64/include/asm/kexec.h
@@ -97,8 +97,11 @@ struct kimage_arch {
phys_addr_t dtb_mem;
phys_addr_t kern_reloc;
phys_addr_t el2_vectors;
+ phys_addr_t ttbr0;
phys_addr_t ttbr1;
phys_addr_t zero_page;
+ unsigned long phys_offset;
+ unsigned long t0sz;
};
#ifdef CONFIG_KEXEC_FILE
diff --git a/arch/arm64/kernel/asm-offsets.c b/arch/arm64/kernel/asm-offsets.c
index f9794cd95eef..bbdf20930491 100644
--- a/arch/arm64/kernel/asm-offsets.c
+++ b/arch/arm64/kernel/asm-offsets.c
@@ -161,6 +161,7 @@ int main(void)
DEFINE(KIMAGE_ARCH_DTB_MEM, offsetof(struct kimage, arch.dtb_mem));
DEFINE(KIMAGE_ARCH_EL2_VECTORS, offsetof(struct kimage, arch.el2_vectors));
DEFINE(KIMAGE_ARCH_ZERO_PAGE, offsetof(struct kimage, arch.zero_page));
+ DEFINE(KIMAGE_ARCH_PHYS_OFFSET, offsetof(struct kimage, arch.phys_offset));
DEFINE(KIMAGE_ARCH_TTBR1, offsetof(struct kimage, arch.ttbr1));
DEFINE(KIMAGE_HEAD, offsetof(struct kimage, head));
DEFINE(KIMAGE_START, offsetof(struct kimage, start));
diff --git a/arch/arm64/kernel/machine_kexec.c b/arch/arm64/kernel/machine_kexec.c
index 4e796e46baa3..a2163ea24fc1 100644
--- a/arch/arm64/kernel/machine_kexec.c
+++ b/arch/arm64/kernel/machine_kexec.c
@@ -190,6 +190,11 @@ int machine_kexec_post_load(struct kimage *kimage)
reloc_size = __relocate_new_kernel_end - __relocate_new_kernel_start;
memcpy(reloc_code, __relocate_new_kernel_start, reloc_size);
kimage->arch.kern_reloc = __pa(reloc_code);
+ rc = trans_pgd_idmap_page(&info, &kimage->arch.ttbr0,
+ &kimage->arch.t0sz, reloc_code);
+ if (rc)
+ return rc;
+ kimage->arch.phys_offset = virt_to_phys(kimage) - (long)kimage;
/* Flush the reloc_code in preparation for its execution. */
__flush_dcache_area(reloc_code, reloc_size);
@@ -223,9 +228,9 @@ void machine_kexec(struct kimage *kimage)
local_daif_mask();
/*
- * Both restart and cpu_soft_restart will shutdown the MMU, disable data
+ * Both restart and kernel_reloc will shutdown the MMU, disable data
* caches. However, restart will start new kernel or purgatory directly,
- * cpu_soft_restart will transfer control to arm64_relocate_new_kernel
+ * kernel_reloc contains the body of arm64_relocate_new_kernel
* In kexec case, kimage->start points to purgatory assuming that
* kernel entry and dtb address are embedded in purgatory by
* userspace (kexec-tools).
@@ -239,10 +244,13 @@ void machine_kexec(struct kimage *kimage)
restart(is_hyp_nvhe(), kimage->start, kimage->arch.dtb_mem,
0, 0);
} else {
+ void (*kernel_reloc)(struct kimage *kimage);
+
if (is_hyp_nvhe())
__hyp_set_vectors(kimage->arch.el2_vectors);
- cpu_soft_restart(kimage->arch.kern_reloc,
- virt_to_phys(kimage), 0, 0);
+ cpu_install_ttbr0(kimage->arch.ttbr0, kimage->arch.t0sz);
+ kernel_reloc = (void *)kimage->arch.kern_reloc;
+ kernel_reloc(kimage);
}
BUG(); /* Should never get here. */
diff --git a/arch/arm64/kernel/relocate_kernel.S b/arch/arm64/kernel/relocate_kernel.S
index e83b6380907d..fc2348584a54 100644
--- a/arch/arm64/kernel/relocate_kernel.S
+++ b/arch/arm64/kernel/relocate_kernel.S
@@ -4,6 +4,8 @@
*
* Copyright (C) Linaro.
* Copyright (C) Huawei Futurewei Technologies.
+ * Copyright (C) 2020, Microsoft Corporation.
+ * Pavel Tatashin <pasha.tatashin@soleen.com>
*/
#include <linux/kexec.h>
@@ -15,6 +17,13 @@
#include <asm/sysreg.h>
#include <asm/virt.h>
+.macro turn_off_mmu tmp1, tmp2
+ mov_q \tmp1, INIT_SCTLR_EL1_MMU_OFF
+ pre_disable_mmu_workaround
+ msr sctlr_el1, \tmp1
+ isb
+.endm
+
.pushsection ".kexec_relocate.text", "ax"
/*
* arm64_relocate_new_kernel - Put a 2nd stage image in place and boot it.
@@ -32,22 +41,21 @@ SYM_CODE_START(arm64_relocate_new_kernel)
ldr x18, [x0, #KIMAGE_ARCH_ZERO_PAGE] /* x18 = zero page for BBM */
ldr x17, [x0, #KIMAGE_ARCH_TTBR1] /* x17 = linear map copy */
ldr x16, [x0, #KIMAGE_HEAD] /* x16 = kimage_head */
- mov x14, xzr /* x14 = entry ptr */
- mov x13, xzr /* x13 = copy dest */
+ ldr x22, [x0, #KIMAGE_ARCH_PHYS_OFFSET] /* x22 phys_offset */
raw_dcache_line_size x15, x1 /* x15 = dcache line size */
break_before_make_ttbr_switch x18, x17, x1, x2 /* set linear map */
.Lloop:
and x12, x16, PAGE_MASK /* x12 = addr */
-
+ sub x12, x12, x22 /* Convert x12 to virt */
/* Test the entry flags. */
.Ltest_source:
tbz x16, IND_SOURCE_BIT, .Ltest_indirection
/* Invalidate dest page to PoC. */
- mov x2, x13
- mov x1, #PAGE_SIZE
- dcache_by_myline_op ivac, sy, x2, x1, x15, x20
+ mov x19, x13
copy_page x13, x12, x1, x2, x3, x4, x5, x6, x7, x8
+ mov x1, #PAGE_SIZE
+ dcache_by_myline_op civac, sy, x19, x1, x15, x20
b .Lnext
.Ltest_indirection:
tbz x16, IND_INDIRECTION_BIT, .Ltest_destination
@@ -64,19 +72,20 @@ SYM_CODE_START(arm64_relocate_new_kernel)
ic iallu
dsb nsh
isb
+ ldr x4, [x0, #KIMAGE_START] /* relocation start */
+ ldr x1, [x0, #KIMAGE_ARCH_EL2_VECTORS] /* relocation start */
+ ldr x0, [x0, #KIMAGE_ARCH_DTB_MEM] /* dtb address */
+ turn_off_mmu x12, x13
/* Start new image. */
- ldr x1, [x0, #KIMAGE_ARCH_EL2_VECTORS] /* relocation start */
cbz x1, .Lel1
- ldr x1, [x0, #KIMAGE_START] /* relocation start */
- ldr x2, [x0, #KIMAGE_ARCH_DTB_MEM] /* dtb address */
+ mov x1, x4 /* relocation start */
+ mov x2, x0 /* dtb address */
mov x3, xzr
mov x4, xzr
mov x0, #HVC_SOFT_RESTART
hvc #0 /* Jumps from el2 */
.Lel1:
- ldr x4, [x0, #KIMAGE_START] /* relocation start */
- ldr x0, [x0, #KIMAGE_ARCH_DTB_MEM] /* dtb address */
mov x2, xzr
mov x3, xzr
br x4 /* Jumps from el1 */
--
2.25.1
WARNING: multiple messages have this Message-ID (diff)
From: Pavel Tatashin <pasha.tatashin@soleen.com>
To: pasha.tatashin@soleen.com, jmorris@namei.org, sashal@kernel.org,
ebiederm@xmission.com, kexec@lists.infradead.org,
linux-kernel@vger.kernel.org, corbet@lwn.net,
catalin.marinas@arm.com, will@kernel.org,
linux-arm-kernel@lists.infradead.org, maz@kernel.org,
james.morse@arm.com, vladimir.murzin@arm.com,
matthias.bgg@gmail.com, linux-mm@kvack.org, mark.rutland@arm.com,
steve.capper@arm.com, rfontana@redhat.com, tglx@linutronix.de,
selindag@gmail.com, tyhicks@linux.microsoft.com,
kernelfans@gmail.com, akpm@linux-foundation.org,
madvenka@linux.microsoft.com
Subject: [PATCH v15 12/15] arm64: kexec: keep MMU enabled during kexec relocation
Date: Tue, 8 Jun 2021 20:44:16 -0400 [thread overview]
Message-ID: <20210609004419.936873-13-pasha.tatashin@soleen.com> (raw)
In-Reply-To: <20210609004419.936873-1-pasha.tatashin@soleen.com>
Now, that we have linear map page tables configured, keep MMU enabled
to allow faster relocation of segments to final destination.
Cavium ThunderX2:
Kernel Image size: 38M Iniramfs size: 46M Total relocation size: 84M
MMU-disabled:
relocation 7.489539915s
MMU-enabled:
relocation 0.03946095s
Broadcom Stingray:
The performance data: for a moderate size kernel + initramfs: 25M the
relocation was taking 0.382s, with enabled MMU it now takes
0.019s only or x20 improvement.
The time is proportional to the size of relocation, therefore if initramfs
is larger, 100M it could take over a second.
Signed-off-by: Pavel Tatashin <pasha.tatashin@soleen.com>
---
arch/arm64/include/asm/kexec.h | 3 +++
arch/arm64/kernel/asm-offsets.c | 1 +
arch/arm64/kernel/machine_kexec.c | 16 +++++++++++----
arch/arm64/kernel/relocate_kernel.S | 31 +++++++++++++++++++----------
4 files changed, 36 insertions(+), 15 deletions(-)
diff --git a/arch/arm64/include/asm/kexec.h b/arch/arm64/include/asm/kexec.h
index d678f0ceb7ee..dca6dedc3b25 100644
--- a/arch/arm64/include/asm/kexec.h
+++ b/arch/arm64/include/asm/kexec.h
@@ -97,8 +97,11 @@ struct kimage_arch {
phys_addr_t dtb_mem;
phys_addr_t kern_reloc;
phys_addr_t el2_vectors;
+ phys_addr_t ttbr0;
phys_addr_t ttbr1;
phys_addr_t zero_page;
+ unsigned long phys_offset;
+ unsigned long t0sz;
};
#ifdef CONFIG_KEXEC_FILE
diff --git a/arch/arm64/kernel/asm-offsets.c b/arch/arm64/kernel/asm-offsets.c
index f9794cd95eef..bbdf20930491 100644
--- a/arch/arm64/kernel/asm-offsets.c
+++ b/arch/arm64/kernel/asm-offsets.c
@@ -161,6 +161,7 @@ int main(void)
DEFINE(KIMAGE_ARCH_DTB_MEM, offsetof(struct kimage, arch.dtb_mem));
DEFINE(KIMAGE_ARCH_EL2_VECTORS, offsetof(struct kimage, arch.el2_vectors));
DEFINE(KIMAGE_ARCH_ZERO_PAGE, offsetof(struct kimage, arch.zero_page));
+ DEFINE(KIMAGE_ARCH_PHYS_OFFSET, offsetof(struct kimage, arch.phys_offset));
DEFINE(KIMAGE_ARCH_TTBR1, offsetof(struct kimage, arch.ttbr1));
DEFINE(KIMAGE_HEAD, offsetof(struct kimage, head));
DEFINE(KIMAGE_START, offsetof(struct kimage, start));
diff --git a/arch/arm64/kernel/machine_kexec.c b/arch/arm64/kernel/machine_kexec.c
index 4e796e46baa3..a2163ea24fc1 100644
--- a/arch/arm64/kernel/machine_kexec.c
+++ b/arch/arm64/kernel/machine_kexec.c
@@ -190,6 +190,11 @@ int machine_kexec_post_load(struct kimage *kimage)
reloc_size = __relocate_new_kernel_end - __relocate_new_kernel_start;
memcpy(reloc_code, __relocate_new_kernel_start, reloc_size);
kimage->arch.kern_reloc = __pa(reloc_code);
+ rc = trans_pgd_idmap_page(&info, &kimage->arch.ttbr0,
+ &kimage->arch.t0sz, reloc_code);
+ if (rc)
+ return rc;
+ kimage->arch.phys_offset = virt_to_phys(kimage) - (long)kimage;
/* Flush the reloc_code in preparation for its execution. */
__flush_dcache_area(reloc_code, reloc_size);
@@ -223,9 +228,9 @@ void machine_kexec(struct kimage *kimage)
local_daif_mask();
/*
- * Both restart and cpu_soft_restart will shutdown the MMU, disable data
+ * Both restart and kernel_reloc will shutdown the MMU, disable data
* caches. However, restart will start new kernel or purgatory directly,
- * cpu_soft_restart will transfer control to arm64_relocate_new_kernel
+ * kernel_reloc contains the body of arm64_relocate_new_kernel
* In kexec case, kimage->start points to purgatory assuming that
* kernel entry and dtb address are embedded in purgatory by
* userspace (kexec-tools).
@@ -239,10 +244,13 @@ void machine_kexec(struct kimage *kimage)
restart(is_hyp_nvhe(), kimage->start, kimage->arch.dtb_mem,
0, 0);
} else {
+ void (*kernel_reloc)(struct kimage *kimage);
+
if (is_hyp_nvhe())
__hyp_set_vectors(kimage->arch.el2_vectors);
- cpu_soft_restart(kimage->arch.kern_reloc,
- virt_to_phys(kimage), 0, 0);
+ cpu_install_ttbr0(kimage->arch.ttbr0, kimage->arch.t0sz);
+ kernel_reloc = (void *)kimage->arch.kern_reloc;
+ kernel_reloc(kimage);
}
BUG(); /* Should never get here. */
diff --git a/arch/arm64/kernel/relocate_kernel.S b/arch/arm64/kernel/relocate_kernel.S
index e83b6380907d..fc2348584a54 100644
--- a/arch/arm64/kernel/relocate_kernel.S
+++ b/arch/arm64/kernel/relocate_kernel.S
@@ -4,6 +4,8 @@
*
* Copyright (C) Linaro.
* Copyright (C) Huawei Futurewei Technologies.
+ * Copyright (C) 2020, Microsoft Corporation.
+ * Pavel Tatashin <pasha.tatashin@soleen.com>
*/
#include <linux/kexec.h>
@@ -15,6 +17,13 @@
#include <asm/sysreg.h>
#include <asm/virt.h>
+.macro turn_off_mmu tmp1, tmp2
+ mov_q \tmp1, INIT_SCTLR_EL1_MMU_OFF
+ pre_disable_mmu_workaround
+ msr sctlr_el1, \tmp1
+ isb
+.endm
+
.pushsection ".kexec_relocate.text", "ax"
/*
* arm64_relocate_new_kernel - Put a 2nd stage image in place and boot it.
@@ -32,22 +41,21 @@ SYM_CODE_START(arm64_relocate_new_kernel)
ldr x18, [x0, #KIMAGE_ARCH_ZERO_PAGE] /* x18 = zero page for BBM */
ldr x17, [x0, #KIMAGE_ARCH_TTBR1] /* x17 = linear map copy */
ldr x16, [x0, #KIMAGE_HEAD] /* x16 = kimage_head */
- mov x14, xzr /* x14 = entry ptr */
- mov x13, xzr /* x13 = copy dest */
+ ldr x22, [x0, #KIMAGE_ARCH_PHYS_OFFSET] /* x22 phys_offset */
raw_dcache_line_size x15, x1 /* x15 = dcache line size */
break_before_make_ttbr_switch x18, x17, x1, x2 /* set linear map */
.Lloop:
and x12, x16, PAGE_MASK /* x12 = addr */
-
+ sub x12, x12, x22 /* Convert x12 to virt */
/* Test the entry flags. */
.Ltest_source:
tbz x16, IND_SOURCE_BIT, .Ltest_indirection
/* Invalidate dest page to PoC. */
- mov x2, x13
- mov x1, #PAGE_SIZE
- dcache_by_myline_op ivac, sy, x2, x1, x15, x20
+ mov x19, x13
copy_page x13, x12, x1, x2, x3, x4, x5, x6, x7, x8
+ mov x1, #PAGE_SIZE
+ dcache_by_myline_op civac, sy, x19, x1, x15, x20
b .Lnext
.Ltest_indirection:
tbz x16, IND_INDIRECTION_BIT, .Ltest_destination
@@ -64,19 +72,20 @@ SYM_CODE_START(arm64_relocate_new_kernel)
ic iallu
dsb nsh
isb
+ ldr x4, [x0, #KIMAGE_START] /* relocation start */
+ ldr x1, [x0, #KIMAGE_ARCH_EL2_VECTORS] /* relocation start */
+ ldr x0, [x0, #KIMAGE_ARCH_DTB_MEM] /* dtb address */
+ turn_off_mmu x12, x13
/* Start new image. */
- ldr x1, [x0, #KIMAGE_ARCH_EL2_VECTORS] /* relocation start */
cbz x1, .Lel1
- ldr x1, [x0, #KIMAGE_START] /* relocation start */
- ldr x2, [x0, #KIMAGE_ARCH_DTB_MEM] /* dtb address */
+ mov x1, x4 /* relocation start */
+ mov x2, x0 /* dtb address */
mov x3, xzr
mov x4, xzr
mov x0, #HVC_SOFT_RESTART
hvc #0 /* Jumps from el2 */
.Lel1:
- ldr x4, [x0, #KIMAGE_START] /* relocation start */
- ldr x0, [x0, #KIMAGE_ARCH_DTB_MEM] /* dtb address */
mov x2, xzr
mov x3, xzr
br x4 /* Jumps from el1 */
--
2.25.1
_______________________________________________
linux-arm-kernel mailing list
linux-arm-kernel@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel
WARNING: multiple messages have this Message-ID (diff)
From: Pavel Tatashin <pasha.tatashin@soleen.com>
To: pasha.tatashin@soleen.com, jmorris@namei.org, sashal@kernel.org,
ebiederm@xmission.com, kexec@lists.infradead.org,
linux-kernel@vger.kernel.org, corbet@lwn.net,
catalin.marinas@arm.com, will@kernel.org,
linux-arm-kernel@lists.infradead.org, maz@kernel.org,
james.morse@arm.com, vladimir.murzin@arm.com,
matthias.bgg@gmail.com, linux-mm@kvack.org, mark.rutland@arm.com,
steve.capper@arm.com, rfontana@redhat.com, tglx@linutronix.de,
selindag@gmail.com, tyhicks@linux.microsoft.com,
kernelfans@gmail.com, akpm@linux-foundation.org,
madvenka@linux.microsoft.com
Subject: [PATCH v15 12/15] arm64: kexec: keep MMU enabled during kexec relocation
Date: Tue, 8 Jun 2021 20:44:16 -0400 [thread overview]
Message-ID: <20210609004419.936873-13-pasha.tatashin@soleen.com> (raw)
In-Reply-To: <20210609004419.936873-1-pasha.tatashin@soleen.com>
Now, that we have linear map page tables configured, keep MMU enabled
to allow faster relocation of segments to final destination.
Cavium ThunderX2:
Kernel Image size: 38M Iniramfs size: 46M Total relocation size: 84M
MMU-disabled:
relocation 7.489539915s
MMU-enabled:
relocation 0.03946095s
Broadcom Stingray:
The performance data: for a moderate size kernel + initramfs: 25M the
relocation was taking 0.382s, with enabled MMU it now takes
0.019s only or x20 improvement.
The time is proportional to the size of relocation, therefore if initramfs
is larger, 100M it could take over a second.
Signed-off-by: Pavel Tatashin <pasha.tatashin@soleen.com>
---
arch/arm64/include/asm/kexec.h | 3 +++
arch/arm64/kernel/asm-offsets.c | 1 +
arch/arm64/kernel/machine_kexec.c | 16 +++++++++++----
arch/arm64/kernel/relocate_kernel.S | 31 +++++++++++++++++++----------
4 files changed, 36 insertions(+), 15 deletions(-)
diff --git a/arch/arm64/include/asm/kexec.h b/arch/arm64/include/asm/kexec.h
index d678f0ceb7ee..dca6dedc3b25 100644
--- a/arch/arm64/include/asm/kexec.h
+++ b/arch/arm64/include/asm/kexec.h
@@ -97,8 +97,11 @@ struct kimage_arch {
phys_addr_t dtb_mem;
phys_addr_t kern_reloc;
phys_addr_t el2_vectors;
+ phys_addr_t ttbr0;
phys_addr_t ttbr1;
phys_addr_t zero_page;
+ unsigned long phys_offset;
+ unsigned long t0sz;
};
#ifdef CONFIG_KEXEC_FILE
diff --git a/arch/arm64/kernel/asm-offsets.c b/arch/arm64/kernel/asm-offsets.c
index f9794cd95eef..bbdf20930491 100644
--- a/arch/arm64/kernel/asm-offsets.c
+++ b/arch/arm64/kernel/asm-offsets.c
@@ -161,6 +161,7 @@ int main(void)
DEFINE(KIMAGE_ARCH_DTB_MEM, offsetof(struct kimage, arch.dtb_mem));
DEFINE(KIMAGE_ARCH_EL2_VECTORS, offsetof(struct kimage, arch.el2_vectors));
DEFINE(KIMAGE_ARCH_ZERO_PAGE, offsetof(struct kimage, arch.zero_page));
+ DEFINE(KIMAGE_ARCH_PHYS_OFFSET, offsetof(struct kimage, arch.phys_offset));
DEFINE(KIMAGE_ARCH_TTBR1, offsetof(struct kimage, arch.ttbr1));
DEFINE(KIMAGE_HEAD, offsetof(struct kimage, head));
DEFINE(KIMAGE_START, offsetof(struct kimage, start));
diff --git a/arch/arm64/kernel/machine_kexec.c b/arch/arm64/kernel/machine_kexec.c
index 4e796e46baa3..a2163ea24fc1 100644
--- a/arch/arm64/kernel/machine_kexec.c
+++ b/arch/arm64/kernel/machine_kexec.c
@@ -190,6 +190,11 @@ int machine_kexec_post_load(struct kimage *kimage)
reloc_size = __relocate_new_kernel_end - __relocate_new_kernel_start;
memcpy(reloc_code, __relocate_new_kernel_start, reloc_size);
kimage->arch.kern_reloc = __pa(reloc_code);
+ rc = trans_pgd_idmap_page(&info, &kimage->arch.ttbr0,
+ &kimage->arch.t0sz, reloc_code);
+ if (rc)
+ return rc;
+ kimage->arch.phys_offset = virt_to_phys(kimage) - (long)kimage;
/* Flush the reloc_code in preparation for its execution. */
__flush_dcache_area(reloc_code, reloc_size);
@@ -223,9 +228,9 @@ void machine_kexec(struct kimage *kimage)
local_daif_mask();
/*
- * Both restart and cpu_soft_restart will shutdown the MMU, disable data
+ * Both restart and kernel_reloc will shutdown the MMU, disable data
* caches. However, restart will start new kernel or purgatory directly,
- * cpu_soft_restart will transfer control to arm64_relocate_new_kernel
+ * kernel_reloc contains the body of arm64_relocate_new_kernel
* In kexec case, kimage->start points to purgatory assuming that
* kernel entry and dtb address are embedded in purgatory by
* userspace (kexec-tools).
@@ -239,10 +244,13 @@ void machine_kexec(struct kimage *kimage)
restart(is_hyp_nvhe(), kimage->start, kimage->arch.dtb_mem,
0, 0);
} else {
+ void (*kernel_reloc)(struct kimage *kimage);
+
if (is_hyp_nvhe())
__hyp_set_vectors(kimage->arch.el2_vectors);
- cpu_soft_restart(kimage->arch.kern_reloc,
- virt_to_phys(kimage), 0, 0);
+ cpu_install_ttbr0(kimage->arch.ttbr0, kimage->arch.t0sz);
+ kernel_reloc = (void *)kimage->arch.kern_reloc;
+ kernel_reloc(kimage);
}
BUG(); /* Should never get here. */
diff --git a/arch/arm64/kernel/relocate_kernel.S b/arch/arm64/kernel/relocate_kernel.S
index e83b6380907d..fc2348584a54 100644
--- a/arch/arm64/kernel/relocate_kernel.S
+++ b/arch/arm64/kernel/relocate_kernel.S
@@ -4,6 +4,8 @@
*
* Copyright (C) Linaro.
* Copyright (C) Huawei Futurewei Technologies.
+ * Copyright (C) 2020, Microsoft Corporation.
+ * Pavel Tatashin <pasha.tatashin@soleen.com>
*/
#include <linux/kexec.h>
@@ -15,6 +17,13 @@
#include <asm/sysreg.h>
#include <asm/virt.h>
+.macro turn_off_mmu tmp1, tmp2
+ mov_q \tmp1, INIT_SCTLR_EL1_MMU_OFF
+ pre_disable_mmu_workaround
+ msr sctlr_el1, \tmp1
+ isb
+.endm
+
.pushsection ".kexec_relocate.text", "ax"
/*
* arm64_relocate_new_kernel - Put a 2nd stage image in place and boot it.
@@ -32,22 +41,21 @@ SYM_CODE_START(arm64_relocate_new_kernel)
ldr x18, [x0, #KIMAGE_ARCH_ZERO_PAGE] /* x18 = zero page for BBM */
ldr x17, [x0, #KIMAGE_ARCH_TTBR1] /* x17 = linear map copy */
ldr x16, [x0, #KIMAGE_HEAD] /* x16 = kimage_head */
- mov x14, xzr /* x14 = entry ptr */
- mov x13, xzr /* x13 = copy dest */
+ ldr x22, [x0, #KIMAGE_ARCH_PHYS_OFFSET] /* x22 phys_offset */
raw_dcache_line_size x15, x1 /* x15 = dcache line size */
break_before_make_ttbr_switch x18, x17, x1, x2 /* set linear map */
.Lloop:
and x12, x16, PAGE_MASK /* x12 = addr */
-
+ sub x12, x12, x22 /* Convert x12 to virt */
/* Test the entry flags. */
.Ltest_source:
tbz x16, IND_SOURCE_BIT, .Ltest_indirection
/* Invalidate dest page to PoC. */
- mov x2, x13
- mov x1, #PAGE_SIZE
- dcache_by_myline_op ivac, sy, x2, x1, x15, x20
+ mov x19, x13
copy_page x13, x12, x1, x2, x3, x4, x5, x6, x7, x8
+ mov x1, #PAGE_SIZE
+ dcache_by_myline_op civac, sy, x19, x1, x15, x20
b .Lnext
.Ltest_indirection:
tbz x16, IND_INDIRECTION_BIT, .Ltest_destination
@@ -64,19 +72,20 @@ SYM_CODE_START(arm64_relocate_new_kernel)
ic iallu
dsb nsh
isb
+ ldr x4, [x0, #KIMAGE_START] /* relocation start */
+ ldr x1, [x0, #KIMAGE_ARCH_EL2_VECTORS] /* relocation start */
+ ldr x0, [x0, #KIMAGE_ARCH_DTB_MEM] /* dtb address */
+ turn_off_mmu x12, x13
/* Start new image. */
- ldr x1, [x0, #KIMAGE_ARCH_EL2_VECTORS] /* relocation start */
cbz x1, .Lel1
- ldr x1, [x0, #KIMAGE_START] /* relocation start */
- ldr x2, [x0, #KIMAGE_ARCH_DTB_MEM] /* dtb address */
+ mov x1, x4 /* relocation start */
+ mov x2, x0 /* dtb address */
mov x3, xzr
mov x4, xzr
mov x0, #HVC_SOFT_RESTART
hvc #0 /* Jumps from el2 */
.Lel1:
- ldr x4, [x0, #KIMAGE_START] /* relocation start */
- ldr x0, [x0, #KIMAGE_ARCH_DTB_MEM] /* dtb address */
mov x2, xzr
mov x3, xzr
br x4 /* Jumps from el1 */
--
2.25.1
_______________________________________________
kexec mailing list
kexec@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/kexec
next prev parent reply other threads:[~2021-06-09 0:45 UTC|newest]
Thread overview: 52+ messages / expand[flat|nested] mbox.gz Atom feed top
2021-06-09 0:44 [PATCH v15 00/15] arm64: MMU enabled kexec relocation Pavel Tatashin
2021-06-09 0:44 ` Pavel Tatashin
2021-06-09 0:44 ` Pavel Tatashin
2021-06-09 0:44 ` [PATCH v15 01/15] arm64: kernel: add helper for booted at EL2 and not VHE Pavel Tatashin
2021-06-09 0:44 ` Pavel Tatashin
2021-06-09 0:44 ` Pavel Tatashin
2021-06-09 0:44 ` [PATCH v15 02/15] arm64: trans_pgd: hibernate: Add trans_pgd_copy_el2_vectors Pavel Tatashin
2021-06-09 0:44 ` Pavel Tatashin
2021-06-09 0:44 ` Pavel Tatashin
2021-06-09 0:44 ` [PATCH v15 03/15] arm64: hibernate: abstract ttrb0 setup function Pavel Tatashin
2021-06-09 0:44 ` Pavel Tatashin
2021-06-09 0:44 ` Pavel Tatashin
2021-06-09 0:44 ` [PATCH v15 04/15] arm64: kexec: flush image and lists during kexec load time Pavel Tatashin
2021-06-09 0:44 ` Pavel Tatashin
2021-06-09 0:44 ` Pavel Tatashin
2021-06-09 0:44 ` [PATCH v15 05/15] arm64: kexec: skip relocation code for inplace kexec Pavel Tatashin
2021-06-09 0:44 ` Pavel Tatashin
2021-06-09 0:44 ` Pavel Tatashin
2021-06-09 0:44 ` [PATCH v15 06/15] arm64: kexec: Use dcache ops macros instead of open-coding Pavel Tatashin
2021-06-09 0:44 ` Pavel Tatashin
2021-06-09 0:44 ` Pavel Tatashin
2021-06-09 0:44 ` [PATCH v15 07/15] arm64: kexec: pass kimage as the only argument to relocation function Pavel Tatashin
2021-06-09 0:44 ` Pavel Tatashin
2021-06-09 0:44 ` Pavel Tatashin
2021-06-09 0:44 ` [PATCH v15 08/15] arm64: kexec: configure EL2 vectors for kexec Pavel Tatashin
2021-06-09 0:44 ` Pavel Tatashin
2021-06-09 0:44 ` Pavel Tatashin
2021-06-09 0:44 ` [PATCH v15 09/15] arm64: kexec: relocate in EL1 mode Pavel Tatashin
2021-06-09 0:44 ` Pavel Tatashin
2021-06-09 0:44 ` Pavel Tatashin
2021-06-09 0:44 ` [PATCH v15 10/15] arm64: kexec: use ld script for relocation function Pavel Tatashin
2021-06-09 0:44 ` Pavel Tatashin
2021-06-09 0:44 ` Pavel Tatashin
2021-06-09 0:44 ` [PATCH v15 11/15] arm64: kexec: install a copy of the linear-map Pavel Tatashin
2021-06-09 0:44 ` Pavel Tatashin
2021-06-09 0:44 ` Pavel Tatashin
2021-06-09 0:44 ` Pavel Tatashin [this message]
2021-06-09 0:44 ` [PATCH v15 12/15] arm64: kexec: keep MMU enabled during kexec relocation Pavel Tatashin
2021-06-09 0:44 ` Pavel Tatashin
2021-06-09 0:44 ` [PATCH v15 13/15] arm64: kexec: remove the pre-kexec PoC maintenance Pavel Tatashin
2021-06-09 0:44 ` Pavel Tatashin
2021-06-09 0:44 ` Pavel Tatashin
2021-06-09 0:44 ` [PATCH v15 14/15] arm64: kexec: remove cpu-reset.h Pavel Tatashin
2021-06-09 0:44 ` Pavel Tatashin
2021-06-09 0:44 ` Pavel Tatashin
2021-06-09 0:44 ` [PATCH v15 15/15] arm64: trans_pgd: remove trans_pgd_map_page() Pavel Tatashin
2021-06-09 0:44 ` Pavel Tatashin
2021-06-09 0:44 ` Pavel Tatashin
2021-06-16 12:17 ` [PATCH v15 00/15] arm64: MMU enabled kexec relocation Pavel Tatashin
2021-06-16 12:17 ` Pavel Tatashin
2021-06-16 12:17 ` Pavel Tatashin
2021-06-16 12:17 ` Pavel Tatashin
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20210609004419.936873-13-pasha.tatashin@soleen.com \
--to=pasha.tatashin@soleen.com \
--cc=akpm@linux-foundation.org \
--cc=catalin.marinas@arm.com \
--cc=corbet@lwn.net \
--cc=ebiederm@xmission.com \
--cc=james.morse@arm.com \
--cc=jmorris@namei.org \
--cc=kernelfans@gmail.com \
--cc=kexec@lists.infradead.org \
--cc=linux-arm-kernel@lists.infradead.org \
--cc=linux-kernel@vger.kernel.org \
--cc=linux-mm@kvack.org \
--cc=madvenka@linux.microsoft.com \
--cc=mark.rutland@arm.com \
--cc=matthias.bgg@gmail.com \
--cc=maz@kernel.org \
--cc=rfontana@redhat.com \
--cc=sashal@kernel.org \
--cc=selindag@gmail.com \
--cc=steve.capper@arm.com \
--cc=tglx@linutronix.de \
--cc=tyhicks@linux.microsoft.com \
--cc=vladimir.murzin@arm.com \
--cc=will@kernel.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.