From: Mark Rutland <mark.rutland@arm.com>
To: linux-arm-kernel@lists.infradead.org
Cc: ardb@kernel.org, catalin.marinas@arm.com, james.morse@arm.com,
joey.gouly@arm.com, mark.rutland@arm.com, maz@kernel.org,
will@kernel.org
Subject: [PATCH 6/9] arm64: alternatives: have callbacks take a cap
Date: Thu, 1 Sep 2022 16:14:00 +0100 [thread overview]
Message-ID: <20220901151403.1735836-7-mark.rutland@arm.com> (raw)
In-Reply-To: <20220901151403.1735836-1-mark.rutland@arm.com>
Today, callback alternatives are special-cased within
__apply_alternatives(), and are applied alongside patching for system
capabilities as ARM64_NCAPS is not part of the boot_capabilities feature
mask.
This special-casing is less than ideal. Giving special meaning to
ARM64_NCAPS for this requires some structures and loops to use
ARM64_NCAPS + 1 (AKA ARM64_NPATCHABLE), while others use ARM64_NCAPS.
It's also not immediately clear callback alternatives are only applied
when applying alternatives for system-wide features.
To make this a bit clearer, changes the way that callback alternatives
are identified to remove the special-casing of ARM64_NCAPS, and to allow
callback alternatives to be associated with a cpucap as with all other
alternatives.
New cpucaps, ARM64_ALWAYS_BOOT and ARM64_ALWAYS_SYSTEM are added which
are always detected alongside boot cpu capabilities and system
capabilities respectively. All existing callback alternatives are made
to use ARM64_ALWAYS_SYSTEM, and so will be patched at the same point
during the boot flow as before.
Subsequent patches will make more use of these new cpucaps.
There should be no functional change as a result of this patch.
Signed-off-by: Mark Rutland <mark.rutland@arm.com>
Cc: Ard Biesheuvel <ardb@kernel.org>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: James Morse <james.morse@arm.com>
Cc: Joey Gouly <joey.gouly@arm.com>
Cc: Marc Zyngier <maz@kernel.org>
Cc: Will Deacon <will@kernel.org>
---
arch/arm64/include/asm/alternative-macros.h | 18 ++++++++-----
arch/arm64/include/asm/assembler.h | 10 ++++----
arch/arm64/include/asm/cpufeature.h | 4 +--
arch/arm64/include/asm/kvm_mmu.h | 5 ++--
arch/arm64/kernel/alternative.c | 28 ++++++++++++---------
arch/arm64/kernel/cpufeature.c | 19 ++++++++++++--
arch/arm64/kernel/entry.S | 8 +++---
arch/arm64/kvm/hyp/hyp-entry.S | 4 +--
arch/arm64/tools/cpucaps | 2 ++
9 files changed, 62 insertions(+), 36 deletions(-)
diff --git a/arch/arm64/include/asm/alternative-macros.h b/arch/arm64/include/asm/alternative-macros.h
index 7e157ab6cd505..189c31be163ce 100644
--- a/arch/arm64/include/asm/alternative-macros.h
+++ b/arch/arm64/include/asm/alternative-macros.h
@@ -2,10 +2,16 @@
#ifndef __ASM_ALTERNATIVE_MACROS_H
#define __ASM_ALTERNATIVE_MACROS_H
+#include <linux/const.h>
+
#include <asm/cpucaps.h>
#include <asm/insn-def.h>
-#define ARM64_CB_PATCH ARM64_NCAPS
+#define ARM64_CB_BIT (UL(1) << 15)
+
+#if ARM64_NCAPS >= ARM64_CB_BIT
+#error "cpucaps have overflown ARM64_CB_BIT"
+#endif
#ifndef __ASSEMBLY__
@@ -73,8 +79,8 @@
#define _ALTERNATIVE_CFG(oldinstr, newinstr, feature, cfg, ...) \
__ALTERNATIVE_CFG(oldinstr, newinstr, feature, IS_ENABLED(cfg))
-#define ALTERNATIVE_CB(oldinstr, cb) \
- __ALTERNATIVE_CFG_CB(oldinstr, ARM64_CB_PATCH, 1, cb)
+#define ALTERNATIVE_CB(oldinstr, feature, cb) \
+ __ALTERNATIVE_CFG_CB(oldinstr, ARM64_CB_BIT | (feature), 1, cb)
#else
#include <asm/assembler.h>
@@ -82,7 +88,7 @@
.macro altinstruction_entry orig_offset alt_offset feature orig_len alt_len
.word \orig_offset - .
.word \alt_offset - .
- .hword \feature
+ .hword (\feature)
.byte \orig_len
.byte \alt_len
.endm
@@ -141,10 +147,10 @@
661:
.endm
-.macro alternative_cb cb
+.macro alternative_cb cap, cb
.set .Lasm_alt_mode, 0
.pushsection .altinstructions, "a"
- altinstruction_entry 661f, \cb, ARM64_CB_PATCH, 662f-661f, 0
+ altinstruction_entry 661f, \cb, ARM64_CB_BIT | \cap, 662f-661f, 0
.popsection
661:
.endm
diff --git a/arch/arm64/include/asm/assembler.h b/arch/arm64/include/asm/assembler.h
index 5846145be523c..d851e27334395 100644
--- a/arch/arm64/include/asm/assembler.h
+++ b/arch/arm64/include/asm/assembler.h
@@ -293,7 +293,7 @@ alternative_endif
alternative_if_not ARM64_KVM_PROTECTED_MODE
ASM_BUG()
alternative_else_nop_endif
-alternative_cb kvm_compute_final_ctr_el0
+alternative_cb ARM64_ALWAYS_SYSTEM, kvm_compute_final_ctr_el0
movz \reg, #0
movk \reg, #0, lsl #16
movk \reg, #0, lsl #32
@@ -877,7 +877,7 @@ alternative_endif
.macro __mitigate_spectre_bhb_loop tmp
#ifdef CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY
-alternative_cb spectre_bhb_patch_loop_iter
+alternative_cb ARM64_ALWAYS_SYSTEM, spectre_bhb_patch_loop_iter
mov \tmp, #32 // Patched to correct the immediate
alternative_cb_end
.Lspectre_bhb_loop\@:
@@ -890,7 +890,7 @@ alternative_cb_end
.macro mitigate_spectre_bhb_loop tmp
#ifdef CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY
-alternative_cb spectre_bhb_patch_loop_mitigation_enable
+alternative_cb ARM64_ALWAYS_SYSTEM, spectre_bhb_patch_loop_mitigation_enable
b .L_spectre_bhb_loop_done\@ // Patched to NOP
alternative_cb_end
__mitigate_spectre_bhb_loop \tmp
@@ -904,7 +904,7 @@ alternative_cb_end
stp x0, x1, [sp, #-16]!
stp x2, x3, [sp, #-16]!
mov w0, #ARM_SMCCC_ARCH_WORKAROUND_3
-alternative_cb smccc_patch_fw_mitigation_conduit
+alternative_cb ARM64_ALWAYS_SYSTEM, smccc_patch_fw_mitigation_conduit
nop // Patched to SMC/HVC #0
alternative_cb_end
ldp x2, x3, [sp], #16
@@ -914,7 +914,7 @@ alternative_cb_end
.macro mitigate_spectre_bhb_clear_insn
#ifdef CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY
-alternative_cb spectre_bhb_patch_clearbhb
+alternative_cb ARM64_ALWAYS_SYSTEM, spectre_bhb_patch_clearbhb
/* Patched to NOP when not supported */
clearbhb
isb
diff --git a/arch/arm64/include/asm/cpufeature.h b/arch/arm64/include/asm/cpufeature.h
index 630c337c77746..f5852ad6e8845 100644
--- a/arch/arm64/include/asm/cpufeature.h
+++ b/arch/arm64/include/asm/cpufeature.h
@@ -422,9 +422,7 @@ extern DECLARE_BITMAP(cpu_hwcaps, ARM64_NCAPS);
extern struct static_key_false cpu_hwcap_keys[ARM64_NCAPS];
extern struct static_key_false arm64_const_caps_ready;
-/* ARM64 CAPS + alternative_cb */
-#define ARM64_NPATCHABLE (ARM64_NCAPS + 1)
-extern DECLARE_BITMAP(boot_capabilities, ARM64_NPATCHABLE);
+extern DECLARE_BITMAP(boot_capabilities, ARM64_NCAPS);
#define for_each_available_cap(cap) \
for_each_set_bit(cap, cpu_hwcaps, ARM64_NCAPS)
diff --git a/arch/arm64/include/asm/kvm_mmu.h b/arch/arm64/include/asm/kvm_mmu.h
index b208da3bebec8..7784081088e78 100644
--- a/arch/arm64/include/asm/kvm_mmu.h
+++ b/arch/arm64/include/asm/kvm_mmu.h
@@ -63,7 +63,7 @@
* specific registers encoded in the instructions).
*/
.macro kern_hyp_va reg
-alternative_cb kvm_update_va_mask
+alternative_cb ARM64_ALWAYS_SYSTEM, kvm_update_va_mask
and \reg, \reg, #1 /* mask with va_mask */
ror \reg, \reg, #1 /* rotate to the first tag bit */
add \reg, \reg, #0 /* insert the low 12 bits of the tag */
@@ -97,7 +97,7 @@ alternative_cb_end
hyp_pa \reg, \tmp
/* Load kimage_voffset. */
-alternative_cb kvm_get_kimage_voffset
+alternative_cb ARM64_ALWAYS_SYSTEM, kvm_get_kimage_voffset
movz \tmp, #0
movk \tmp, #0, lsl #16
movk \tmp, #0, lsl #32
@@ -131,6 +131,7 @@ static __always_inline unsigned long __kern_hyp_va(unsigned long v)
"add %0, %0, #0\n"
"add %0, %0, #0, lsl 12\n"
"ror %0, %0, #63\n",
+ ARM64_ALWAYS_SYSTEM,
kvm_update_va_mask)
: "+r" (v));
return v;
diff --git a/arch/arm64/kernel/alternative.c b/arch/arm64/kernel/alternative.c
index 2e18c9c0f612b..da706c9f9a9a5 100644
--- a/arch/arm64/kernel/alternative.c
+++ b/arch/arm64/kernel/alternative.c
@@ -21,6 +21,11 @@
#define ALT_ORIG_PTR(a) __ALT_PTR(a, orig_offset)
#define ALT_REPL_PTR(a) __ALT_PTR(a, alt_offset)
+#define ALT_CAP(a) ((a)->cpufeature & ~ARM64_CB_BIT)
+#define ALT_HAS_CB(a) ((a)->cpufeature & ARM64_CB_BIT)
+
+#define ALT_NR_INST(a) ((a)->orig_len / AARCH64_INSN_SIZE)
+
/* Volatile, as we may be patching the guts of READ_ONCE() */
static volatile int all_alternatives_applied;
@@ -143,16 +148,15 @@ static void __nocfi __apply_alternatives(const struct alt_region *region,
for (alt = region->begin; alt < region->end; alt++) {
int nr_inst;
+ int cap = ALT_CAP(alt);
- if (!test_bit(alt->cpufeature, feature_mask))
+ if (!test_bit(cap, feature_mask))
continue;
- /* Use ARM64_CB_PATCH as an unconditional patch */
- if (alt->cpufeature < ARM64_CB_PATCH &&
- !cpus_have_cap(alt->cpufeature))
+ if (!cpus_have_cap(cap))
continue;
- if (alt->cpufeature == ARM64_CB_PATCH)
+ if (ALT_HAS_CB(alt))
BUG_ON(alt->alt_len != 0);
else
BUG_ON(alt->alt_len != alt->orig_len);
@@ -161,10 +165,10 @@ static void __nocfi __apply_alternatives(const struct alt_region *region,
updptr = is_module ? origptr : lm_alias(origptr);
nr_inst = alt->orig_len / AARCH64_INSN_SIZE;
- if (alt->cpufeature < ARM64_CB_PATCH)
- alt_cb = patch_alternative;
- else
+ if (ALT_HAS_CB(alt))
alt_cb = ALT_REPL_PTR(alt);
+ else
+ alt_cb = patch_alternative;
alt_cb(alt, origptr, updptr, nr_inst);
@@ -208,10 +212,10 @@ static int __apply_alternatives_multi_stop(void *unused)
cpu_relax();
isb();
} else {
- DECLARE_BITMAP(remaining_capabilities, ARM64_NPATCHABLE);
+ DECLARE_BITMAP(remaining_capabilities, ARM64_NCAPS);
bitmap_complement(remaining_capabilities, boot_capabilities,
- ARM64_NPATCHABLE);
+ ARM64_NCAPS);
BUG_ON(all_alternatives_applied);
__apply_alternatives(&kernel_alternatives, false,
@@ -254,9 +258,9 @@ void apply_alternatives_module(void *start, size_t length)
.begin = start,
.end = start + length,
};
- DECLARE_BITMAP(all_capabilities, ARM64_NPATCHABLE);
+ DECLARE_BITMAP(all_capabilities, ARM64_NCAPS);
- bitmap_fill(all_capabilities, ARM64_NPATCHABLE);
+ bitmap_fill(all_capabilities, ARM64_NCAPS);
__apply_alternatives(®ion, true, &all_capabilities[0]);
}
diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c
index af4de817d7123..68a0545285a1d 100644
--- a/arch/arm64/kernel/cpufeature.c
+++ b/arch/arm64/kernel/cpufeature.c
@@ -108,8 +108,7 @@ DECLARE_BITMAP(cpu_hwcaps, ARM64_NCAPS);
EXPORT_SYMBOL(cpu_hwcaps);
static struct arm64_cpu_capabilities const __ro_after_init *cpu_hwcaps_ptrs[ARM64_NCAPS];
-/* Need also bit for ARM64_CB_PATCH */
-DECLARE_BITMAP(boot_capabilities, ARM64_NPATCHABLE);
+DECLARE_BITMAP(boot_capabilities, ARM64_NCAPS);
bool arm64_use_ng_mappings = false;
EXPORT_SYMBOL(arm64_use_ng_mappings);
@@ -1391,6 +1390,12 @@ u64 __read_sysreg_by_encoding(u32 sys_id)
#include <linux/irqchip/arm-gic-v3.h>
+static bool
+has_always(const struct arm64_cpu_capabilities *entry, int scope)
+{
+ return true;
+}
+
static bool
feature_matches(u64 reg, const struct arm64_cpu_capabilities *entry)
{
@@ -2087,6 +2092,16 @@ cpucap_panic_on_conflict(const struct arm64_cpu_capabilities *cap)
}
static const struct arm64_cpu_capabilities arm64_features[] = {
+ {
+ .capability = ARM64_ALWAYS_BOOT,
+ .type = ARM64_CPUCAP_BOOT_CPU_FEATURE,
+ .matches = has_always,
+ },
+ {
+ .capability = ARM64_ALWAYS_SYSTEM,
+ .type = ARM64_CPUCAP_SYSTEM_FEATURE,
+ .matches = has_always,
+ },
{
.desc = "GIC system register CPU interface",
.capability = ARM64_HAS_SYSREG_GIC_CPUIF,
diff --git a/arch/arm64/kernel/entry.S b/arch/arm64/kernel/entry.S
index 2d73b3e793b2b..e28137d64b768 100644
--- a/arch/arm64/kernel/entry.S
+++ b/arch/arm64/kernel/entry.S
@@ -114,7 +114,7 @@
* them if required.
*/
.macro apply_ssbd, state, tmp1, tmp2
-alternative_cb spectre_v4_patch_fw_mitigation_enable
+alternative_cb ARM64_ALWAYS_SYSTEM, spectre_v4_patch_fw_mitigation_enable
b .L__asm_ssbd_skip\@ // Patched to NOP
alternative_cb_end
ldr_this_cpu \tmp2, arm64_ssbd_callback_required, \tmp1
@@ -123,7 +123,7 @@ alternative_cb_end
tbnz \tmp2, #TIF_SSBD, .L__asm_ssbd_skip\@
mov w0, #ARM_SMCCC_ARCH_WORKAROUND_2
mov w1, #\state
-alternative_cb smccc_patch_fw_mitigation_conduit
+alternative_cb ARM64_ALWAYS_SYSTEM, smccc_patch_fw_mitigation_conduit
nop // Patched to SMC/HVC #0
alternative_cb_end
.L__asm_ssbd_skip\@:
@@ -175,7 +175,7 @@ alternative_else_nop_endif
.macro mte_set_kernel_gcr, tmp, tmp2
#ifdef CONFIG_KASAN_HW_TAGS
-alternative_cb kasan_hw_tags_enable
+alternative_cb ARM64_ALWAYS_SYSTEM, kasan_hw_tags_enable
b 1f
alternative_cb_end
mov \tmp, KERNEL_GCR_EL1
@@ -186,7 +186,7 @@ alternative_cb_end
.macro mte_set_user_gcr, tsk, tmp, tmp2
#ifdef CONFIG_KASAN_HW_TAGS
-alternative_cb kasan_hw_tags_enable
+alternative_cb ARM64_ALWAYS_SYSTEM, kasan_hw_tags_enable
b 1f
alternative_cb_end
ldr \tmp, [\tsk, #THREAD_MTE_CTRL]
diff --git a/arch/arm64/kvm/hyp/hyp-entry.S b/arch/arm64/kvm/hyp/hyp-entry.S
index 7839d075729b1..8f3f93fa119ed 100644
--- a/arch/arm64/kvm/hyp/hyp-entry.S
+++ b/arch/arm64/kvm/hyp/hyp-entry.S
@@ -196,7 +196,7 @@ SYM_CODE_END(__kvm_hyp_vector)
sub sp, sp, #(8 * 4)
stp x2, x3, [sp, #(8 * 0)]
stp x0, x1, [sp, #(8 * 2)]
- alternative_cb spectre_bhb_patch_wa3
+ alternative_cb ARM64_ALWAYS_SYSTEM, spectre_bhb_patch_wa3
/* Patched to mov WA3 when supported */
mov w0, #ARM_SMCCC_ARCH_WORKAROUND_1
alternative_cb_end
@@ -216,7 +216,7 @@ SYM_CODE_END(__kvm_hyp_vector)
mitigate_spectre_bhb_clear_insn
.endif
.if \indirect != 0
- alternative_cb kvm_patch_vector_branch
+ alternative_cb ARM64_ALWAYS_SYSTEM, kvm_patch_vector_branch
/*
* For ARM64_SPECTRE_V3A configurations, these NOPs get replaced with:
*
diff --git a/arch/arm64/tools/cpucaps b/arch/arm64/tools/cpucaps
index 63b2484ce6c3d..ff882ec175e73 100644
--- a/arch/arm64/tools/cpucaps
+++ b/arch/arm64/tools/cpucaps
@@ -2,6 +2,8 @@
#
# Internal CPU capabilities constants, keep this list sorted
+ALWAYS_BOOT
+ALWAYS_SYSTEM
BTI
# Unreliable: use system_supports_32bit_el0() instead.
HAS_32BIT_EL0_DO_NOT_USE
--
2.30.2
_______________________________________________
linux-arm-kernel mailing list
linux-arm-kernel@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel
next prev parent reply other threads:[~2022-09-01 15:16 UTC|newest]
Thread overview: 20+ messages / expand[flat|nested] mbox.gz Atom feed top
2022-09-01 15:13 [PATCH 0/9] arm64: alternatives: improvements Mark Rutland
2022-09-01 15:13 ` [PATCH 1/9] arm64: cpufeature: make cpus_have_cap() noinstr-safe Mark Rutland
2022-09-01 15:13 ` [PATCH 2/9] arm64: alternatives: kvm: prepare for cap changes Mark Rutland
2022-09-01 15:13 ` [PATCH 3/9] arm64: alternatives: proton-pack: " Mark Rutland
2022-09-02 16:19 ` Joey Gouly
2022-09-05 8:46 ` Mark Rutland
2022-09-01 15:13 ` [PATCH 4/9] arm64: alternatives: hoist print out of __apply_alternatives() Mark Rutland
2022-09-01 15:13 ` [PATCH 5/9] arm64: alternatives: make alt_region const Mark Rutland
2022-09-06 15:18 ` Ard Biesheuvel
2022-09-12 9:31 ` Mark Rutland
2022-09-12 10:13 ` Ard Biesheuvel
2022-09-12 12:13 ` Mark Rutland
2022-09-01 15:14 ` Mark Rutland [this message]
2022-09-02 15:54 ` [PATCH 6/9] arm64: alternatives: have callbacks take a cap Joey Gouly
2022-09-05 8:48 ` Mark Rutland
2022-09-01 15:14 ` [PATCH 7/9] arm64: alternatives: add alternative_has_feature_*() Mark Rutland
2022-09-01 15:14 ` [PATCH 8/9] arm64: alternatives: add shared NOP callback Mark Rutland
2022-09-01 15:14 ` [PATCH 9/9] HACK: arm64: alternatives: dump summary of alternatives Mark Rutland
2022-09-12 12:36 ` Mark Brown
2022-09-12 16:14 ` Mark Rutland
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20220901151403.1735836-7-mark.rutland@arm.com \
--to=mark.rutland@arm.com \
--cc=ardb@kernel.org \
--cc=catalin.marinas@arm.com \
--cc=james.morse@arm.com \
--cc=joey.gouly@arm.com \
--cc=linux-arm-kernel@lists.infradead.org \
--cc=maz@kernel.org \
--cc=will@kernel.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).