From: Russell King <rmk+kernel@armlinux.org.uk> To: linux-arm-kernel@lists.infradead.org Cc: Marc Zyngier <marc.zyngier@arm.com>, Florian Fainelli <f.fainelli@gmail.com>, kvmarm@lists.cs.columbia.edu Subject: [PATCH v3 08/15] ARM: spectre-v2: harden user aborts in kernel space Date: Fri, 25 May 2018 15:01:03 +0100 [thread overview] Message-ID: <E1fMDHD-00073Q-Ob@rmk-PC.armlinux.org.uk> (raw) In-Reply-To: <20180525135938.GE17671@n2100.armlinux.org.uk> In order to prevent aliasing attacks on the branch predictor, invalidate the BTB or instruction cache on CPUs that are known to be affected when taking an abort on a address that is outside of a user task limit: Cortex A8, A9, A12, A17, A73, A75: flush BTB. Cortex A15, Brahma B15: invalidate icache. If the IBE bit is not set, then there is little point to enabling the workaround. Signed-off-by: Russell King <rmk+kernel@armlinux.org.uk> --- arch/arm/include/asm/cp15.h | 3 ++ arch/arm/include/asm/system_misc.h | 15 ++++++++ arch/arm/mm/fault.c | 3 ++ arch/arm/mm/proc-v7-bugs.c | 76 +++++++++++++++++++++++++++++++++++--- arch/arm/mm/proc-v7.S | 8 ++-- 5 files changed, 96 insertions(+), 9 deletions(-) diff --git a/arch/arm/include/asm/cp15.h b/arch/arm/include/asm/cp15.h index 4c9fa72b59f5..07e27f212dc7 100644 --- a/arch/arm/include/asm/cp15.h +++ b/arch/arm/include/asm/cp15.h @@ -65,6 +65,9 @@ #define __write_sysreg(v, r, w, c, t) asm volatile(w " " c : : "r" ((t)(v))) #define write_sysreg(v, ...) __write_sysreg(v, __VA_ARGS__) +#define BPIALL __ACCESS_CP15(c7, 0, c5, 6) +#define ICIALLU __ACCESS_CP15(c7, 0, c5, 0) + extern unsigned long cr_alignment; /* defined in entry-armv.S */ static inline unsigned long get_cr(void) diff --git a/arch/arm/include/asm/system_misc.h b/arch/arm/include/asm/system_misc.h index 78f6db114faf..8e76db83c498 100644 --- a/arch/arm/include/asm/system_misc.h +++ b/arch/arm/include/asm/system_misc.h @@ -8,6 +8,7 @@ #include <linux/linkage.h> #include <linux/irqflags.h> #include <linux/reboot.h> +#include <linux/percpu.h> extern void cpu_init(void); @@ -15,6 +16,20 @@ void soft_restart(unsigned long); extern void (*arm_pm_restart)(enum reboot_mode reboot_mode, const char *cmd); extern void (*arm_pm_idle)(void); +#ifdef CONFIG_HARDEN_BRANCH_PREDICTOR +typedef void (*harden_branch_predictor_fn_t)(void); +DECLARE_PER_CPU(harden_branch_predictor_fn_t, harden_branch_predictor_fn); +static inline void harden_branch_predictor(void) +{ + harden_branch_predictor_fn_t fn = per_cpu(harden_branch_predictor_fn, + smp_processor_id()); + if (fn) + fn(); +} +#else +#define harden_branch_predictor() do { } while (0) +#endif + #define UDBG_UNDEFINED (1 << 0) #define UDBG_SYSCALL (1 << 1) #define UDBG_BADABORT (1 << 2) diff --git a/arch/arm/mm/fault.c b/arch/arm/mm/fault.c index b75eada23d0a..3b1ba003c4f9 100644 --- a/arch/arm/mm/fault.c +++ b/arch/arm/mm/fault.c @@ -163,6 +163,9 @@ __do_user_fault(struct task_struct *tsk, unsigned long addr, { struct siginfo si; + if (addr > TASK_SIZE) + harden_branch_predictor(); + #ifdef CONFIG_DEBUG_USER if (((user_debug & UDBG_SEGV) && (sig == SIGSEGV)) || ((user_debug & UDBG_BUS) && (sig == SIGBUS))) { diff --git a/arch/arm/mm/proc-v7-bugs.c b/arch/arm/mm/proc-v7-bugs.c index a32ce13479d9..585b2b61d7d3 100644 --- a/arch/arm/mm/proc-v7-bugs.c +++ b/arch/arm/mm/proc-v7-bugs.c @@ -2,28 +2,92 @@ #include <linux/kernel.h> #include <linux/smp.h> -static __maybe_unused void cpu_v7_check_auxcr_set(u32 mask, const char *msg) +#include <asm/cp15.h> +#include <asm/cputype.h> +#include <asm/system_misc.h> + +#ifdef CONFIG_HARDEN_BRANCH_PREDICTOR +DEFINE_PER_CPU(harden_branch_predictor_fn_t, harden_branch_predictor_fn); + +static void harden_branch_predictor_bpiall(void) +{ + write_sysreg(0, BPIALL); +} + +static void harden_branch_predictor_iciallu(void) +{ + write_sysreg(0, ICIALLU); +} + +static void cpu_v7_spectre_init(void) +{ + const char *spectre_v2_method = NULL; + int cpu = smp_processor_id(); + + if (per_cpu(harden_branch_predictor_fn, cpu)) + return; + + switch (read_cpuid_part()) { + case ARM_CPU_PART_CORTEX_A8: + case ARM_CPU_PART_CORTEX_A9: + case ARM_CPU_PART_CORTEX_A12: + case ARM_CPU_PART_CORTEX_A17: + case ARM_CPU_PART_CORTEX_A73: + case ARM_CPU_PART_CORTEX_A75: + per_cpu(harden_branch_predictor_fn, cpu) = + harden_branch_predictor_bpiall; + spectre_v2_method = "BPIALL"; + break; + + case ARM_CPU_PART_CORTEX_A15: + case ARM_CPU_PART_BRAHMA_B15: + per_cpu(harden_branch_predictor_fn, cpu) = + harden_branch_predictor_iciallu; + spectre_v2_method = "ICIALLU"; + break; + } + if (spectre_v2_method) + pr_info("CPU%u: Spectre v2: using %s workaround\n", + smp_processor_id(), spectre_v2_method); +} +#else +static void cpu_v7_spectre_init(void) +{ +} +#endif + +static __maybe_unused bool cpu_v7_check_auxcr_set(u32 mask, const char *msg) { u32 aux_cr; asm("mrc p15, 0, %0, c1, c0, 1" : "=r" (aux_cr)); - if ((aux_cr & mask) != mask) + if ((aux_cr & mask) != mask) { pr_err("CPU%u: %s", smp_processor_id(), msg); + return false; + } + return true; } -static void check_spectre_auxcr(u32 bit) +static bool check_spectre_auxcr(u32 bit) { - if (IS_ENABLED(CONFIG_HARDEN_BRANCH_PREDICTOR)) + return IS_ENABLED(CONFIG_HARDEN_BRANCH_PREDICTOR) && cpu_v7_check_auxcr_set(bit, "Spectre v2: firmware did not set auxiliary control register IBE bit, system vulnerable\n"); } void cpu_v7_ca8_ibe(void) { - check_spectre_auxcr(BIT(6)); + if (check_spectre_auxcr(BIT(6))) + cpu_v7_spectre_init(); } void cpu_v7_ca15_ibe(void) { - check_spectre_auxcr(BIT(0)); + if (check_spectre_auxcr(BIT(0))) + cpu_v7_spectre_init(); +} + +void cpu_v7_bugs_init(void) +{ + cpu_v7_spectre_init(); } diff --git a/arch/arm/mm/proc-v7.S b/arch/arm/mm/proc-v7.S index fa9214036fb3..79510011e7eb 100644 --- a/arch/arm/mm/proc-v7.S +++ b/arch/arm/mm/proc-v7.S @@ -532,8 +532,10 @@ ENDPROC(__v7_setup) __INITDATA + .weak cpu_v7_bugs_init + @ define struct processor (see <asm/proc-fns.h> and proc-macros.S) - define_processor_functions v7, dabort=v7_early_abort, pabort=v7_pabort, suspend=1 + define_processor_functions v7, dabort=v7_early_abort, pabort=v7_pabort, suspend=1, bugs=cpu_v7_bugs_init #ifdef CONFIG_HARDEN_BRANCH_PREDICTOR @ generic v7 bpiall on context switch @@ -548,7 +550,7 @@ ENDPROC(__v7_setup) globl_equ cpu_v7_bpiall_do_suspend, cpu_v7_do_suspend globl_equ cpu_v7_bpiall_do_resume, cpu_v7_do_resume #endif - define_processor_functions v7_bpiall, dabort=v7_early_abort, pabort=v7_pabort, suspend=1 + define_processor_functions v7_bpiall, dabort=v7_early_abort, pabort=v7_pabort, suspend=1, bugs=cpu_v7_bugs_init #define HARDENED_BPIALL_PROCESSOR_FUNCTIONS v7_bpiall_processor_functions #else @@ -584,7 +586,7 @@ ENDPROC(__v7_setup) globl_equ cpu_ca9mp_switch_mm, cpu_v7_switch_mm #endif globl_equ cpu_ca9mp_set_pte_ext, cpu_v7_set_pte_ext - define_processor_functions ca9mp, dabort=v7_early_abort, pabort=v7_pabort, suspend=1 + define_processor_functions ca9mp, dabort=v7_early_abort, pabort=v7_pabort, suspend=1, bugs=cpu_v7_bugs_init #endif @ Cortex-A15 - needs iciallu switch_mm for hardening -- 2.7.4
WARNING: multiple messages have this Message-ID (diff)
From: rmk+kernel@armlinux.org.uk (Russell King) To: linux-arm-kernel@lists.infradead.org Subject: [PATCH v3 08/15] ARM: spectre-v2: harden user aborts in kernel space Date: Fri, 25 May 2018 15:01:03 +0100 [thread overview] Message-ID: <E1fMDHD-00073Q-Ob@rmk-PC.armlinux.org.uk> (raw) In-Reply-To: <20180525135938.GE17671@n2100.armlinux.org.uk> In order to prevent aliasing attacks on the branch predictor, invalidate the BTB or instruction cache on CPUs that are known to be affected when taking an abort on a address that is outside of a user task limit: Cortex A8, A9, A12, A17, A73, A75: flush BTB. Cortex A15, Brahma B15: invalidate icache. If the IBE bit is not set, then there is little point to enabling the workaround. Signed-off-by: Russell King <rmk+kernel@armlinux.org.uk> --- arch/arm/include/asm/cp15.h | 3 ++ arch/arm/include/asm/system_misc.h | 15 ++++++++ arch/arm/mm/fault.c | 3 ++ arch/arm/mm/proc-v7-bugs.c | 76 +++++++++++++++++++++++++++++++++++--- arch/arm/mm/proc-v7.S | 8 ++-- 5 files changed, 96 insertions(+), 9 deletions(-) diff --git a/arch/arm/include/asm/cp15.h b/arch/arm/include/asm/cp15.h index 4c9fa72b59f5..07e27f212dc7 100644 --- a/arch/arm/include/asm/cp15.h +++ b/arch/arm/include/asm/cp15.h @@ -65,6 +65,9 @@ #define __write_sysreg(v, r, w, c, t) asm volatile(w " " c : : "r" ((t)(v))) #define write_sysreg(v, ...) __write_sysreg(v, __VA_ARGS__) +#define BPIALL __ACCESS_CP15(c7, 0, c5, 6) +#define ICIALLU __ACCESS_CP15(c7, 0, c5, 0) + extern unsigned long cr_alignment; /* defined in entry-armv.S */ static inline unsigned long get_cr(void) diff --git a/arch/arm/include/asm/system_misc.h b/arch/arm/include/asm/system_misc.h index 78f6db114faf..8e76db83c498 100644 --- a/arch/arm/include/asm/system_misc.h +++ b/arch/arm/include/asm/system_misc.h @@ -8,6 +8,7 @@ #include <linux/linkage.h> #include <linux/irqflags.h> #include <linux/reboot.h> +#include <linux/percpu.h> extern void cpu_init(void); @@ -15,6 +16,20 @@ void soft_restart(unsigned long); extern void (*arm_pm_restart)(enum reboot_mode reboot_mode, const char *cmd); extern void (*arm_pm_idle)(void); +#ifdef CONFIG_HARDEN_BRANCH_PREDICTOR +typedef void (*harden_branch_predictor_fn_t)(void); +DECLARE_PER_CPU(harden_branch_predictor_fn_t, harden_branch_predictor_fn); +static inline void harden_branch_predictor(void) +{ + harden_branch_predictor_fn_t fn = per_cpu(harden_branch_predictor_fn, + smp_processor_id()); + if (fn) + fn(); +} +#else +#define harden_branch_predictor() do { } while (0) +#endif + #define UDBG_UNDEFINED (1 << 0) #define UDBG_SYSCALL (1 << 1) #define UDBG_BADABORT (1 << 2) diff --git a/arch/arm/mm/fault.c b/arch/arm/mm/fault.c index b75eada23d0a..3b1ba003c4f9 100644 --- a/arch/arm/mm/fault.c +++ b/arch/arm/mm/fault.c @@ -163,6 +163,9 @@ __do_user_fault(struct task_struct *tsk, unsigned long addr, { struct siginfo si; + if (addr > TASK_SIZE) + harden_branch_predictor(); + #ifdef CONFIG_DEBUG_USER if (((user_debug & UDBG_SEGV) && (sig == SIGSEGV)) || ((user_debug & UDBG_BUS) && (sig == SIGBUS))) { diff --git a/arch/arm/mm/proc-v7-bugs.c b/arch/arm/mm/proc-v7-bugs.c index a32ce13479d9..585b2b61d7d3 100644 --- a/arch/arm/mm/proc-v7-bugs.c +++ b/arch/arm/mm/proc-v7-bugs.c @@ -2,28 +2,92 @@ #include <linux/kernel.h> #include <linux/smp.h> -static __maybe_unused void cpu_v7_check_auxcr_set(u32 mask, const char *msg) +#include <asm/cp15.h> +#include <asm/cputype.h> +#include <asm/system_misc.h> + +#ifdef CONFIG_HARDEN_BRANCH_PREDICTOR +DEFINE_PER_CPU(harden_branch_predictor_fn_t, harden_branch_predictor_fn); + +static void harden_branch_predictor_bpiall(void) +{ + write_sysreg(0, BPIALL); +} + +static void harden_branch_predictor_iciallu(void) +{ + write_sysreg(0, ICIALLU); +} + +static void cpu_v7_spectre_init(void) +{ + const char *spectre_v2_method = NULL; + int cpu = smp_processor_id(); + + if (per_cpu(harden_branch_predictor_fn, cpu)) + return; + + switch (read_cpuid_part()) { + case ARM_CPU_PART_CORTEX_A8: + case ARM_CPU_PART_CORTEX_A9: + case ARM_CPU_PART_CORTEX_A12: + case ARM_CPU_PART_CORTEX_A17: + case ARM_CPU_PART_CORTEX_A73: + case ARM_CPU_PART_CORTEX_A75: + per_cpu(harden_branch_predictor_fn, cpu) = + harden_branch_predictor_bpiall; + spectre_v2_method = "BPIALL"; + break; + + case ARM_CPU_PART_CORTEX_A15: + case ARM_CPU_PART_BRAHMA_B15: + per_cpu(harden_branch_predictor_fn, cpu) = + harden_branch_predictor_iciallu; + spectre_v2_method = "ICIALLU"; + break; + } + if (spectre_v2_method) + pr_info("CPU%u: Spectre v2: using %s workaround\n", + smp_processor_id(), spectre_v2_method); +} +#else +static void cpu_v7_spectre_init(void) +{ +} +#endif + +static __maybe_unused bool cpu_v7_check_auxcr_set(u32 mask, const char *msg) { u32 aux_cr; asm("mrc p15, 0, %0, c1, c0, 1" : "=r" (aux_cr)); - if ((aux_cr & mask) != mask) + if ((aux_cr & mask) != mask) { pr_err("CPU%u: %s", smp_processor_id(), msg); + return false; + } + return true; } -static void check_spectre_auxcr(u32 bit) +static bool check_spectre_auxcr(u32 bit) { - if (IS_ENABLED(CONFIG_HARDEN_BRANCH_PREDICTOR)) + return IS_ENABLED(CONFIG_HARDEN_BRANCH_PREDICTOR) && cpu_v7_check_auxcr_set(bit, "Spectre v2: firmware did not set auxiliary control register IBE bit, system vulnerable\n"); } void cpu_v7_ca8_ibe(void) { - check_spectre_auxcr(BIT(6)); + if (check_spectre_auxcr(BIT(6))) + cpu_v7_spectre_init(); } void cpu_v7_ca15_ibe(void) { - check_spectre_auxcr(BIT(0)); + if (check_spectre_auxcr(BIT(0))) + cpu_v7_spectre_init(); +} + +void cpu_v7_bugs_init(void) +{ + cpu_v7_spectre_init(); } diff --git a/arch/arm/mm/proc-v7.S b/arch/arm/mm/proc-v7.S index fa9214036fb3..79510011e7eb 100644 --- a/arch/arm/mm/proc-v7.S +++ b/arch/arm/mm/proc-v7.S @@ -532,8 +532,10 @@ ENDPROC(__v7_setup) __INITDATA + .weak cpu_v7_bugs_init + @ define struct processor (see <asm/proc-fns.h> and proc-macros.S) - define_processor_functions v7, dabort=v7_early_abort, pabort=v7_pabort, suspend=1 + define_processor_functions v7, dabort=v7_early_abort, pabort=v7_pabort, suspend=1, bugs=cpu_v7_bugs_init #ifdef CONFIG_HARDEN_BRANCH_PREDICTOR @ generic v7 bpiall on context switch @@ -548,7 +550,7 @@ ENDPROC(__v7_setup) globl_equ cpu_v7_bpiall_do_suspend, cpu_v7_do_suspend globl_equ cpu_v7_bpiall_do_resume, cpu_v7_do_resume #endif - define_processor_functions v7_bpiall, dabort=v7_early_abort, pabort=v7_pabort, suspend=1 + define_processor_functions v7_bpiall, dabort=v7_early_abort, pabort=v7_pabort, suspend=1, bugs=cpu_v7_bugs_init #define HARDENED_BPIALL_PROCESSOR_FUNCTIONS v7_bpiall_processor_functions #else @@ -584,7 +586,7 @@ ENDPROC(__v7_setup) globl_equ cpu_ca9mp_switch_mm, cpu_v7_switch_mm #endif globl_equ cpu_ca9mp_set_pte_ext, cpu_v7_set_pte_ext - define_processor_functions ca9mp, dabort=v7_early_abort, pabort=v7_pabort, suspend=1 + define_processor_functions ca9mp, dabort=v7_early_abort, pabort=v7_pabort, suspend=1, bugs=cpu_v7_bugs_init #endif @ Cortex-A15 - needs iciallu switch_mm for hardening -- 2.7.4
next prev parent reply other threads:[~2018-05-25 13:52 UTC|newest] Thread overview: 42+ messages / expand[flat|nested] mbox.gz Atom feed top 2018-05-25 13:59 [PATCH v3 00/15] ARM Spectre variant 2 fixes Russell King - ARM Linux 2018-05-25 13:59 ` Russell King - ARM Linux 2018-05-25 14:00 ` [PATCH v3 01/15] ARM: add more CPU part numbers for Cortex and Brahma B15 CPUs Russell King 2018-05-25 14:00 ` Russell King 2018-05-25 14:00 ` [PATCH v3 02/15] ARM: bugs: prepare processor bug infrastructure Russell King 2018-05-25 14:00 ` Russell King 2018-05-25 14:00 ` [PATCH v3 03/15] ARM: bugs: hook processor bug checking into SMP and suspend paths Russell King 2018-05-25 14:00 ` Russell King 2018-05-25 14:00 ` [PATCH v3 04/15] ARM: bugs: add support for per-processor bug checking Russell King 2018-05-25 14:00 ` Russell King 2018-05-25 14:00 ` [PATCH v3 05/15] ARM: spectre: add Kconfig symbol for CPUs vulnerable to Spectre Russell King 2018-05-25 14:00 ` Russell King 2018-05-25 14:00 ` [PATCH v3 06/15] ARM: spectre-v2: harden branch predictor on context switches Russell King 2018-05-25 14:00 ` Russell King 2018-05-25 14:00 ` [PATCH v3 07/15] ARM: spectre-v2: add Cortex A8 and A15 validation of the IBE bit Russell King 2018-05-25 14:00 ` Russell King 2018-05-25 14:01 ` Russell King [this message] 2018-05-25 14:01 ` [PATCH v3 08/15] ARM: spectre-v2: harden user aborts in kernel space Russell King 2018-05-25 15:47 ` Tony Lindgren 2018-05-25 15:47 ` Tony Lindgren 2018-05-25 15:52 ` Russell King - ARM Linux 2018-05-25 15:52 ` Russell King - ARM Linux 2018-05-25 16:01 ` Tony Lindgren 2018-05-25 16:01 ` Tony Lindgren 2018-05-25 16:15 ` Tony Lindgren 2018-05-25 16:15 ` Tony Lindgren 2018-05-25 14:01 ` [PATCH v3 09/15] ARM: spectre-v2: add firmware based hardening Russell King 2018-05-25 14:01 ` Russell King 2018-05-25 14:01 ` [PATCH v3 10/15] ARM: spectre-v2: warn about incorrect context switching functions Russell King 2018-05-25 14:01 ` Russell King 2018-05-25 14:01 ` [PATCH v3 11/15] ARM: KVM: invalidate BTB on guest exit for Cortex-A12/A17 Russell King 2018-05-25 14:01 ` Russell King 2018-05-25 14:01 ` [PATCH v3 12/15] ARM: KVM: invalidate icache on guest exit for Cortex-A15 Russell King 2018-05-25 14:01 ` Russell King 2018-05-25 14:01 ` [PATCH v3 13/15] ARM: spectre-v2: KVM: invalidate icache on guest exit for Brahma B15 Russell King 2018-05-25 14:01 ` Russell King 2018-05-25 14:01 ` [PATCH v3 14/15] ARM: KVM: Add SMCCC_ARCH_WORKAROUND_1 fast handling Russell King 2018-05-25 14:01 ` Russell King 2018-05-25 14:01 ` [PATCH v3 15/15] ARM: KVM: report support for SMCCC_ARCH_WORKAROUND_1 Russell King 2018-05-25 14:01 ` Russell King 2018-05-25 16:25 ` [PATCH v3 00/15] ARM Spectre variant 2 fixes Tony Lindgren 2018-05-25 16:25 ` Tony Lindgren
Reply instructions: You may reply publicly to this message via plain-text email using any one of the following methods: * Save the following mbox file, import it into your mail client, and reply-to-all from there: mbox Avoid top-posting and favor interleaved quoting: https://en.wikipedia.org/wiki/Posting_style#Interleaved_style * Reply using the --to, --cc, and --in-reply-to switches of git-send-email(1): git send-email \ --in-reply-to=E1fMDHD-00073Q-Ob@rmk-PC.armlinux.org.uk \ --to=rmk+kernel@armlinux.org.uk \ --cc=f.fainelli@gmail.com \ --cc=kvmarm@lists.cs.columbia.edu \ --cc=linux-arm-kernel@lists.infradead.org \ --cc=marc.zyngier@arm.com \ /path/to/YOUR_REPLY https://kernel.org/pub/software/scm/git/docs/git-send-email.html * If your mail client supports setting the In-Reply-To header via mailto: links, try the mailto: linkBe sure your reply has a Subject: header at the top and a blank line before the message body.
This is an external index of several public inboxes, see mirroring instructions on how to clone and mirror all data and code used by this external index.