From: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
To: linux-kernel@vger.kernel.org
Cc: Greg Kroah-Hartman <gregkh@linuxfoundation.org>,
stable@vger.kernel.org, Catalin Marinas <catalin.marinas@arm.com>,
James Morse <james.morse@arm.com>
Subject: [PATCH 4.9 38/43] arm64: Add percpu vectors for EL1
Date: Wed, 6 Apr 2022 20:26:47 +0200 [thread overview]
Message-ID: <20220406182437.786062691@linuxfoundation.org> (raw)
In-Reply-To: <20220406182436.675069715@linuxfoundation.org>
From: James Morse <james.morse@arm.com>
commit bd09128d16fac3c34b80bd6a29088ac632e8ce09 upstream.
The Spectre-BHB workaround adds a firmware call to the vectors. This
is needed on some CPUs, but not others. To avoid the unaffected CPU in
a big/little pair from making the firmware call, create per cpu vectors.
The per-cpu vectors only apply when returning from EL0.
Systems using KPTI can use the canonical 'full-fat' vectors directly at
EL1, the trampoline exit code will switch to this_cpu_vector on exit to
EL0. Systems not using KPTI should always use this_cpu_vector.
this_cpu_vector will point at a vector in tramp_vecs or
__bp_harden_el1_vectors, depending on whether KPTI is in use.
Reviewed-by: Catalin Marinas <catalin.marinas@arm.com>
Signed-off-by: James Morse <james.morse@arm.com>
Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
---
arch/arm64/include/asm/mmu.h | 2 +-
arch/arm64/include/asm/vectors.h | 27 +++++++++++++++++++++++++++
arch/arm64/kernel/cpufeature.c | 11 +++++++++++
arch/arm64/kernel/entry.S | 16 ++++++++++------
arch/arm64/kvm/hyp/switch.c | 9 ++++++---
5 files changed, 55 insertions(+), 10 deletions(-)
--- a/arch/arm64/include/asm/mmu.h
+++ b/arch/arm64/include/asm/mmu.h
@@ -34,7 +34,7 @@ typedef struct {
*/
#define ASID(mm) ((mm)->context.id.counter & 0xffff)
-static inline bool arm64_kernel_unmapped_at_el0(void)
+static __always_inline bool arm64_kernel_unmapped_at_el0(void)
{
return IS_ENABLED(CONFIG_UNMAP_KERNEL_AT_EL0) &&
cpus_have_const_cap(ARM64_UNMAP_KERNEL_AT_EL0);
--- a/arch/arm64/include/asm/vectors.h
+++ b/arch/arm64/include/asm/vectors.h
@@ -5,6 +5,15 @@
#ifndef __ASM_VECTORS_H
#define __ASM_VECTORS_H
+#include <linux/bug.h>
+#include <linux/percpu.h>
+
+#include <asm/fixmap.h>
+
+extern char vectors[];
+extern char tramp_vectors[];
+extern char __bp_harden_el1_vectors[];
+
/*
* Note: the order of this enum corresponds to two arrays in entry.S:
* tramp_vecs and __bp_harden_el1_vectors. By default the canonical
@@ -31,4 +40,22 @@ enum arm64_bp_harden_el1_vectors {
EL1_VECTOR_KPTI,
};
+/* The vectors to use on return from EL0. e.g. to remap the kernel */
+DECLARE_PER_CPU_READ_MOSTLY(const char *, this_cpu_vector);
+
+#ifndef CONFIG_UNMAP_KERNEL_AT_EL0
+#define TRAMP_VALIAS 0
+#endif
+
+static inline const char *
+arm64_get_bp_hardening_vector(enum arm64_bp_harden_el1_vectors slot)
+{
+ if (arm64_kernel_unmapped_at_el0())
+ return (char *)TRAMP_VALIAS + SZ_2K * slot;
+
+ WARN_ON_ONCE(slot == EL1_VECTOR_KPTI);
+
+ return __bp_harden_el1_vectors + SZ_2K * slot;
+}
+
#endif /* __ASM_VECTORS_H */
--- a/arch/arm64/kernel/cpufeature.c
+++ b/arch/arm64/kernel/cpufeature.c
@@ -20,15 +20,18 @@
#include <linux/bsearch.h>
#include <linux/cpumask.h>
+#include <linux/percpu.h>
#include <linux/sort.h>
#include <linux/stop_machine.h>
#include <linux/types.h>
+
#include <asm/cpu.h>
#include <asm/cpufeature.h>
#include <asm/cpu_ops.h>
#include <asm/mmu_context.h>
#include <asm/processor.h>
#include <asm/sysreg.h>
+#include <asm/vectors.h>
#include <asm/virt.h>
unsigned long elf_hwcap __read_mostly;
@@ -49,6 +52,8 @@ unsigned int compat_elf_hwcap2 __read_mo
DECLARE_BITMAP(cpu_hwcaps, ARM64_NCAPS);
EXPORT_SYMBOL(cpu_hwcaps);
+DEFINE_PER_CPU_READ_MOSTLY(const char *, this_cpu_vector) = vectors;
+
DEFINE_STATIC_KEY_ARRAY_FALSE(cpu_hwcap_keys, ARM64_NCAPS);
EXPORT_SYMBOL(cpu_hwcap_keys);
@@ -821,6 +826,12 @@ kpti_install_ng_mappings(const struct ar
static bool kpti_applied = false;
int cpu = smp_processor_id();
+ if (__this_cpu_read(this_cpu_vector) == vectors) {
+ const char *v = arm64_get_bp_hardening_vector(EL1_VECTOR_KPTI);
+
+ __this_cpu_write(this_cpu_vector, v);
+ }
+
if (kpti_applied)
return;
--- a/arch/arm64/kernel/entry.S
+++ b/arch/arm64/kernel/entry.S
@@ -75,7 +75,6 @@
.macro kernel_ventry, el, label, regsize = 64
.align 7
.Lventry_start\@:
-#ifdef CONFIG_UNMAP_KERNEL_AT_EL0
.if \el == 0
/*
* This must be the first instruction of the EL0 vector entries. It is
@@ -90,7 +89,6 @@
.endif
.Lskip_tramp_vectors_cleanup\@:
.endif
-#endif
sub sp, sp, #S_FRAME_SIZE
b el\()\el\()_\label
@@ -983,10 +981,14 @@ __ni_sys_trace:
.endm
.macro tramp_exit, regsize = 64
- adr x30, tramp_vectors
-#ifdef CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY
- add x30, x30, SZ_4K
-#endif
+ tramp_data_read_var x30, this_cpu_vector
+alternative_if_not ARM64_HAS_VIRT_HOST_EXTN
+ mrs x29, tpidr_el1
+alternative_else
+ mrs x29, tpidr_el2
+alternative_endif
+ ldr x30, [x30, x29]
+
msr vbar_el1, x30
ldr lr, [sp, #S_LR]
tramp_unmap_kernel x29
@@ -1046,6 +1048,8 @@ __entry_tramp_data_vectors:
__entry_tramp_data___sdei_asm_trampoline_next_handler:
.quad __sdei_asm_handler
#endif /* CONFIG_ARM_SDE_INTERFACE */
+__entry_tramp_data_this_cpu_vector:
+ .quad this_cpu_vector
.popsection // .rodata
#endif /* CONFIG_RANDOMIZE_BASE */
#endif /* CONFIG_UNMAP_KERNEL_AT_EL0 */
--- a/arch/arm64/kvm/hyp/switch.c
+++ b/arch/arm64/kvm/hyp/switch.c
@@ -26,7 +26,7 @@
#include <asm/kvm_emulate.h>
#include <asm/kvm_hyp.h>
#include <asm/uaccess.h>
-
+#include <asm/vectors.h>
extern struct exception_table_entry __start___kvm_ex_table;
extern struct exception_table_entry __stop___kvm_ex_table;
@@ -107,11 +107,14 @@ static void __hyp_text __activate_traps(
static void __hyp_text __deactivate_traps_vhe(void)
{
- extern char vectors[]; /* kernel exception vectors */
+ const char *host_vectors = vectors;
write_sysreg(HCR_HOST_VHE_FLAGS, hcr_el2);
write_sysreg(CPACR_EL1_FPEN, cpacr_el1);
- write_sysreg(vectors, vbar_el1);
+
+ if (!arm64_kernel_unmapped_at_el0())
+ host_vectors = __this_cpu_read(this_cpu_vector);
+ write_sysreg(host_vectors, vbar_el1);
}
static void __hyp_text __deactivate_traps_nvhe(void)
next prev parent reply other threads:[~2022-04-06 19:13 UTC|newest]
Thread overview: 53+ messages / expand[flat|nested] mbox.gz Atom feed top
2022-04-06 18:26 [PATCH 4.9 00/43] 4.9.310-rc1 review Greg Kroah-Hartman
2022-04-06 18:26 ` [PATCH 4.9 01/43] arm64: errata: Provide macro for major and minor cpu revisions Greg Kroah-Hartman
2022-04-06 18:26 ` [PATCH 4.9 02/43] arm64: Remove useless UAO IPI and describe how this gets enabled Greg Kroah-Hartman
2022-04-06 18:26 ` [PATCH 4.9 03/43] arm64: Add MIDR encoding for Arm Cortex-A55 and Cortex-A35 Greg Kroah-Hartman
2022-04-06 18:26 ` [PATCH 4.9 04/43] arm64: capabilities: Update prototype for enable call back Greg Kroah-Hartman
2022-04-06 18:26 ` [PATCH 4.9 05/43] arm64: capabilities: Move errata work around check on boot CPU Greg Kroah-Hartman
2022-04-06 18:26 ` [PATCH 4.9 06/43] arm64: capabilities: Move errata processing code Greg Kroah-Hartman
2022-04-06 18:26 ` [PATCH 4.9 07/43] arm64: capabilities: Prepare for fine grained capabilities Greg Kroah-Hartman
2022-04-06 18:26 ` [PATCH 4.9 08/43] arm64: capabilities: Add flags to handle the conflicts on late CPU Greg Kroah-Hartman
2022-04-06 18:26 ` [PATCH 4.9 09/43] arm64: capabilities: Clean up midr range helpers Greg Kroah-Hartman
2022-04-06 18:26 ` [PATCH 4.9 10/43] arm64: Add helpers for checking CPU MIDR against a range Greg Kroah-Hartman
2022-04-06 18:26 ` [PATCH 4.9 11/43] arm64: capabilities: Add support for checks based on a list of MIDRs Greg Kroah-Hartman
2022-04-06 18:26 ` [PATCH 4.9 12/43] clocksource/drivers/arm_arch_timer: Remove fsl-a008585 parameter Greg Kroah-Hartman
2022-04-06 18:26 ` [PATCH 4.9 13/43] clocksource/drivers/arm_arch_timer: Introduce generic errata handling infrastructure Greg Kroah-Hartman
2022-04-06 18:26 ` [PATCH 4.9 14/43] arm64: arch_timer: Add infrastructure for multiple erratum detection methods Greg Kroah-Hartman
2022-04-06 18:26 ` [PATCH 4.9 15/43] arm64: arch_timer: Add erratum handler for CPU-specific capability Greg Kroah-Hartman
2022-04-06 18:26 ` [PATCH 4.9 16/43] arm64: arch_timer: Add workaround for ARM erratum 1188873 Greg Kroah-Hartman
2022-04-06 18:26 ` [PATCH 4.9 17/43] arm64: arch_timer: avoid unused function warning Greg Kroah-Hartman
2022-04-06 18:26 ` [PATCH 4.9 18/43] arm64: Add silicon-errata.txt entry for ARM erratum 1188873 Greg Kroah-Hartman
2022-04-06 18:26 ` [PATCH 4.9 19/43] arm64: Make ARM64_ERRATUM_1188873 depend on COMPAT Greg Kroah-Hartman
2022-04-06 18:26 ` [PATCH 4.9 20/43] arm64: Add part number for Neoverse N1 Greg Kroah-Hartman
2022-04-06 18:26 ` [PATCH 4.9 21/43] arm64: Add part number for Arm Cortex-A77 Greg Kroah-Hartman
2022-04-06 18:26 ` [PATCH 4.9 22/43] arm64: Add Neoverse-N2, Cortex-A710 CPU part definition Greg Kroah-Hartman
2022-04-06 18:26 ` [PATCH 4.9 23/43] arm64: Add Cortex-X2 " Greg Kroah-Hartman
2022-04-06 18:26 ` [PATCH 4.9 24/43] arm64: Add helper to decode register from instruction Greg Kroah-Hartman
2022-04-06 18:26 ` [PATCH 4.9 25/43] arm64: entry.S: Add ventry overflow sanity checks Greg Kroah-Hartman
2022-04-06 18:26 ` [PATCH 4.9 26/43] arm64: entry: Make the trampoline cleanup optional Greg Kroah-Hartman
2022-04-06 18:26 ` [PATCH 4.9 27/43] arm64: entry: Free up another register on kptis tramp_exit path Greg Kroah-Hartman
2022-04-06 18:26 ` [PATCH 4.9 28/43] arm64: entry: Move the trampoline data page before the text page Greg Kroah-Hartman
2022-04-06 18:26 ` [PATCH 4.9 29/43] arm64: entry: Allow tramp_alias to access symbols after the 4K boundary Greg Kroah-Hartman
2022-04-06 18:26 ` [PATCH 4.9 30/43] arm64: entry: Dont assume tramp_vectors is the start of the vectors Greg Kroah-Hartman
2022-04-06 18:26 ` [PATCH 4.9 31/43] arm64: entry: Move trampoline macros out of ifdefd section Greg Kroah-Hartman
2022-04-06 18:26 ` [PATCH 4.9 32/43] arm64: entry: Make the kpti trampolines kpti sequence optional Greg Kroah-Hartman
2022-04-06 18:26 ` [PATCH 4.9 33/43] arm64: entry: Allow the trampoline text to occupy multiple pages Greg Kroah-Hartman
2022-04-06 18:26 ` [PATCH 4.9 34/43] arm64: entry: Add non-kpti __bp_harden_el1_vectors for mitigations Greg Kroah-Hartman
2022-04-06 18:26 ` [PATCH 4.9 35/43] arm64: Move arm64_update_smccc_conduit() out of SSBD ifdef Greg Kroah-Hartman
2022-04-06 18:26 ` [PATCH 4.9 36/43] arm64: entry: Add vectors that have the bhb mitigation sequences Greg Kroah-Hartman
2022-04-06 18:26 ` [PATCH 4.9 37/43] arm64: entry: Add macro for reading symbol addresses from the trampoline Greg Kroah-Hartman
2022-04-06 18:26 ` Greg Kroah-Hartman [this message]
2022-04-06 18:26 ` [PATCH 4.9 39/43] KVM: arm64: Add templates for BHB mitigation sequences Greg Kroah-Hartman
2022-04-06 18:26 ` [PATCH 4.9 40/43] arm64: Mitigate spectre style branch history side channels Greg Kroah-Hartman
2022-04-06 18:26 ` [PATCH 4.9 41/43] KVM: arm64: Allow SMCCC_ARCH_WORKAROUND_3 to be discovered and migrated Greg Kroah-Hartman
2022-04-06 18:26 ` [PATCH 4.9 42/43] arm64: add ID_AA64ISAR2_EL1 sys register Greg Kroah-Hartman
2022-04-06 18:26 ` [PATCH 4.9 43/43] arm64: Use the clearbhb instruction in mitigations Greg Kroah-Hartman
2022-04-06 20:57 ` [PATCH 4.9 00/43] 4.9.310-rc1 review Florian Fainelli
2022-04-06 21:55 ` Pavel Machek
2022-04-06 22:51 ` Shuah Khan
2022-04-07 9:32 ` Guenter Roeck
2022-04-07 10:23 ` Greg Kroah-Hartman
2022-04-07 17:20 ` James Morse
2022-04-12 5:51 ` Greg Kroah-Hartman
2022-04-07 11:20 ` Naresh Kamboju
2022-04-07 11:28 ` Naresh Kamboju
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20220406182437.786062691@linuxfoundation.org \
--to=gregkh@linuxfoundation.org \
--cc=catalin.marinas@arm.com \
--cc=james.morse@arm.com \
--cc=linux-kernel@vger.kernel.org \
--cc=stable@vger.kernel.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).