* [PATCH 1/7] powerpc/security: re-name count cache flush to branch cache flush
2020-06-09 6:16 [PATCH 0/7] powerpc: branch cache flush changes Nicholas Piggin
@ 2020-06-09 6:16 ` Nicholas Piggin
2020-06-09 6:16 ` [PATCH 2/7] powerpc/security: change link stack flush state to the flush type enum Nicholas Piggin
` (5 subsequent siblings)
6 siblings, 0 replies; 10+ messages in thread
From: Nicholas Piggin @ 2020-06-09 6:16 UTC (permalink / raw)
Cc: linuxppc-dev, Nicholas Piggin
The count cache flush mostly refers to both count cache and link stack
flushing. As a first step to untangling these a bit, re-name the bits
that apply to both.
Signed-off-by: Nicholas Piggin <npiggin@gmail.com>
---
arch/powerpc/include/asm/asm-prototypes.h | 4 +--
arch/powerpc/kernel/entry_64.S | 7 ++---
arch/powerpc/kernel/security.c | 36 +++++++++++------------
3 files changed, 23 insertions(+), 24 deletions(-)
diff --git a/arch/powerpc/include/asm/asm-prototypes.h b/arch/powerpc/include/asm/asm-prototypes.h
index 7d81e86a1e5d..fa9057360e88 100644
--- a/arch/powerpc/include/asm/asm-prototypes.h
+++ b/arch/powerpc/include/asm/asm-prototypes.h
@@ -144,13 +144,13 @@ void _kvmppc_restore_tm_pr(struct kvm_vcpu *vcpu, u64 guest_msr);
void _kvmppc_save_tm_pr(struct kvm_vcpu *vcpu, u64 guest_msr);
/* Patch sites */
-extern s32 patch__call_flush_count_cache;
+extern s32 patch__call_flush_branch_caches;
extern s32 patch__flush_count_cache_return;
extern s32 patch__flush_link_stack_return;
extern s32 patch__call_kvm_flush_link_stack;
extern s32 patch__memset_nocache, patch__memcpy_nocache;
-extern long flush_count_cache;
+extern long flush_branch_caches;
extern long kvm_flush_link_stack;
#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
diff --git a/arch/powerpc/kernel/entry_64.S b/arch/powerpc/kernel/entry_64.S
index 9d49338e0c85..2ba25b3b701e 100644
--- a/arch/powerpc/kernel/entry_64.S
+++ b/arch/powerpc/kernel/entry_64.S
@@ -259,8 +259,7 @@ _ASM_NOKPROBE_SYMBOL(save_nvgprs);
#define FLUSH_COUNT_CACHE \
1: nop; \
- patch_site 1b, patch__call_flush_count_cache
-
+ patch_site 1b, patch__call_flush_branch_caches
#define BCCTR_FLUSH .long 0x4c400420
@@ -271,8 +270,8 @@ _ASM_NOKPROBE_SYMBOL(save_nvgprs);
.endm
.balign 32
-.global flush_count_cache
-flush_count_cache:
+.global flush_branch_caches
+flush_branch_caches:
/* Save LR into r9 */
mflr r9
diff --git a/arch/powerpc/kernel/security.c b/arch/powerpc/kernel/security.c
index d86701ce116b..df2a3eff950b 100644
--- a/arch/powerpc/kernel/security.c
+++ b/arch/powerpc/kernel/security.c
@@ -21,12 +21,12 @@
u64 powerpc_security_features __read_mostly = SEC_FTR_DEFAULT;
-enum count_cache_flush_type {
- COUNT_CACHE_FLUSH_NONE = 0x1,
- COUNT_CACHE_FLUSH_SW = 0x2,
- COUNT_CACHE_FLUSH_HW = 0x4,
+enum branch_cache_flush_type {
+ BRANCH_CACHE_FLUSH_NONE = 0x1,
+ BRANCH_CACHE_FLUSH_SW = 0x2,
+ BRANCH_CACHE_FLUSH_HW = 0x4,
};
-static enum count_cache_flush_type count_cache_flush_type = COUNT_CACHE_FLUSH_NONE;
+static enum branch_cache_flush_type count_cache_flush_type = BRANCH_CACHE_FLUSH_NONE;
static bool link_stack_flush_enabled;
bool barrier_nospec_enabled;
@@ -222,10 +222,10 @@ ssize_t cpu_show_spectre_v2(struct device *dev, struct device_attribute *attr, c
if (link_stack_flush_enabled)
seq_buf_printf(&s, ", Software link stack flush");
- } else if (count_cache_flush_type != COUNT_CACHE_FLUSH_NONE) {
+ } else if (count_cache_flush_type != BRANCH_CACHE_FLUSH_NONE) {
seq_buf_printf(&s, "Mitigation: Software count cache flush");
- if (count_cache_flush_type == COUNT_CACHE_FLUSH_HW)
+ if (count_cache_flush_type == BRANCH_CACHE_FLUSH_HW)
seq_buf_printf(&s, " (hardware accelerated)");
if (link_stack_flush_enabled)
@@ -429,18 +429,18 @@ device_initcall(stf_barrier_debugfs_init);
static void no_count_cache_flush(void)
{
- count_cache_flush_type = COUNT_CACHE_FLUSH_NONE;
+ count_cache_flush_type = BRANCH_CACHE_FLUSH_NONE;
pr_info("count-cache-flush: software flush disabled.\n");
}
-static void toggle_count_cache_flush(bool enable)
+static void toggle_branch_cache_flush(bool enable)
{
if (!security_ftr_enabled(SEC_FTR_FLUSH_COUNT_CACHE) &&
!security_ftr_enabled(SEC_FTR_FLUSH_LINK_STACK))
enable = false;
if (!enable) {
- patch_instruction_site(&patch__call_flush_count_cache,
+ patch_instruction_site(&patch__call_flush_branch_caches,
ppc_inst(PPC_INST_NOP));
#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE
patch_instruction_site(&patch__call_kvm_flush_link_stack,
@@ -452,9 +452,9 @@ static void toggle_count_cache_flush(bool enable)
return;
}
- // This enables the branch from _switch to flush_count_cache
- patch_branch_site(&patch__call_flush_count_cache,
- (u64)&flush_count_cache, BRANCH_SET_LINK);
+ // This enables the branch from _switch to flush_branch_caches
+ patch_branch_site(&patch__call_flush_branch_caches,
+ (u64)&flush_branch_caches, BRANCH_SET_LINK);
#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE
// This enables the branch from guest_exit_cont to kvm_flush_link_stack
@@ -474,13 +474,13 @@ static void toggle_count_cache_flush(bool enable)
}
if (!security_ftr_enabled(SEC_FTR_BCCTR_FLUSH_ASSIST)) {
- count_cache_flush_type = COUNT_CACHE_FLUSH_SW;
+ count_cache_flush_type = BRANCH_CACHE_FLUSH_SW;
pr_info("count-cache-flush: full software flush sequence enabled.\n");
return;
}
patch_instruction_site(&patch__flush_count_cache_return, ppc_inst(PPC_INST_BLR));
- count_cache_flush_type = COUNT_CACHE_FLUSH_HW;
+ count_cache_flush_type = BRANCH_CACHE_FLUSH_HW;
pr_info("count-cache-flush: hardware assisted flush sequence enabled\n");
}
@@ -505,7 +505,7 @@ void setup_count_cache_flush(void)
security_ftr_enabled(SEC_FTR_FLUSH_COUNT_CACHE))
security_ftr_set(SEC_FTR_FLUSH_LINK_STACK);
- toggle_count_cache_flush(enable);
+ toggle_branch_cache_flush(enable);
}
#ifdef CONFIG_DEBUG_FS
@@ -520,14 +520,14 @@ static int count_cache_flush_set(void *data, u64 val)
else
return -EINVAL;
- toggle_count_cache_flush(enable);
+ toggle_branch_cache_flush(enable);
return 0;
}
static int count_cache_flush_get(void *data, u64 *val)
{
- if (count_cache_flush_type == COUNT_CACHE_FLUSH_NONE)
+ if (count_cache_flush_type == BRANCH_CACHE_FLUSH_NONE)
*val = 0;
else
*val = 1;
--
2.23.0
^ permalink raw reply related [flat|nested] 10+ messages in thread* [PATCH 2/7] powerpc/security: change link stack flush state to the flush type enum
2020-06-09 6:16 [PATCH 0/7] powerpc: branch cache flush changes Nicholas Piggin
2020-06-09 6:16 ` [PATCH 1/7] powerpc/security: re-name count cache flush to branch cache flush Nicholas Piggin
@ 2020-06-09 6:16 ` Nicholas Piggin
2020-06-09 6:16 ` [PATCH 3/7] powerpc/security: make display of branch cache flush more consistent Nicholas Piggin
` (4 subsequent siblings)
6 siblings, 0 replies; 10+ messages in thread
From: Nicholas Piggin @ 2020-06-09 6:16 UTC (permalink / raw)
Cc: linuxppc-dev, Nicholas Piggin
Prepare to allow for hardware link stack flushing by using the
none/sw/hw type, same as the count cache state.
Signed-off-by: Nicholas Piggin <npiggin@gmail.com>
---
arch/powerpc/kernel/security.c | 10 +++++-----
1 file changed, 5 insertions(+), 5 deletions(-)
diff --git a/arch/powerpc/kernel/security.c b/arch/powerpc/kernel/security.c
index df2a3eff950b..28f4cb062f69 100644
--- a/arch/powerpc/kernel/security.c
+++ b/arch/powerpc/kernel/security.c
@@ -27,7 +27,7 @@ enum branch_cache_flush_type {
BRANCH_CACHE_FLUSH_HW = 0x4,
};
static enum branch_cache_flush_type count_cache_flush_type = BRANCH_CACHE_FLUSH_NONE;
-static bool link_stack_flush_enabled;
+static enum branch_cache_flush_type link_stack_flush_type = BRANCH_CACHE_FLUSH_NONE;
bool barrier_nospec_enabled;
static bool no_nospec;
@@ -219,7 +219,7 @@ ssize_t cpu_show_spectre_v2(struct device *dev, struct device_attribute *attr, c
if (ccd)
seq_buf_printf(&s, "Indirect branch cache disabled");
- if (link_stack_flush_enabled)
+ if (link_stack_flush_type == BRANCH_CACHE_FLUSH_SW)
seq_buf_printf(&s, ", Software link stack flush");
} else if (count_cache_flush_type != BRANCH_CACHE_FLUSH_NONE) {
@@ -228,7 +228,7 @@ ssize_t cpu_show_spectre_v2(struct device *dev, struct device_attribute *attr, c
if (count_cache_flush_type == BRANCH_CACHE_FLUSH_HW)
seq_buf_printf(&s, " (hardware accelerated)");
- if (link_stack_flush_enabled)
+ if (link_stack_flush_type == BRANCH_CACHE_FLUSH_SW)
seq_buf_printf(&s, ", Software link stack flush");
} else if (btb_flush_enabled) {
@@ -447,7 +447,7 @@ static void toggle_branch_cache_flush(bool enable)
ppc_inst(PPC_INST_NOP));
#endif
pr_info("link-stack-flush: software flush disabled.\n");
- link_stack_flush_enabled = false;
+ link_stack_flush_type = BRANCH_CACHE_FLUSH_NONE;
no_count_cache_flush();
return;
}
@@ -463,7 +463,7 @@ static void toggle_branch_cache_flush(bool enable)
#endif
pr_info("link-stack-flush: software flush enabled.\n");
- link_stack_flush_enabled = true;
+ link_stack_flush_type = BRANCH_CACHE_FLUSH_SW;
// If we just need to flush the link stack, patch an early return
if (!security_ftr_enabled(SEC_FTR_FLUSH_COUNT_CACHE)) {
--
2.23.0
^ permalink raw reply related [flat|nested] 10+ messages in thread* [PATCH 3/7] powerpc/security: make display of branch cache flush more consistent
2020-06-09 6:16 [PATCH 0/7] powerpc: branch cache flush changes Nicholas Piggin
2020-06-09 6:16 ` [PATCH 1/7] powerpc/security: re-name count cache flush to branch cache flush Nicholas Piggin
2020-06-09 6:16 ` [PATCH 2/7] powerpc/security: change link stack flush state to the flush type enum Nicholas Piggin
@ 2020-06-09 6:16 ` Nicholas Piggin
2020-06-09 6:16 ` [PATCH 4/7] powerpc/security: split branch cache flush toggle from code patching Nicholas Piggin
` (3 subsequent siblings)
6 siblings, 0 replies; 10+ messages in thread
From: Nicholas Piggin @ 2020-06-09 6:16 UTC (permalink / raw)
Cc: linuxppc-dev, Nicholas Piggin
Make the count-cache and link-stack messages look the same
Signed-off-by: Nicholas Piggin <npiggin@gmail.com>
---
arch/powerpc/kernel/security.c | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
diff --git a/arch/powerpc/kernel/security.c b/arch/powerpc/kernel/security.c
index 28f4cb062f69..659ef6a92bb9 100644
--- a/arch/powerpc/kernel/security.c
+++ b/arch/powerpc/kernel/security.c
@@ -430,7 +430,7 @@ device_initcall(stf_barrier_debugfs_init);
static void no_count_cache_flush(void)
{
count_cache_flush_type = BRANCH_CACHE_FLUSH_NONE;
- pr_info("count-cache-flush: software flush disabled.\n");
+ pr_info("count-cache-flush: flush disabled.\n");
}
static void toggle_branch_cache_flush(bool enable)
@@ -446,7 +446,7 @@ static void toggle_branch_cache_flush(bool enable)
patch_instruction_site(&patch__call_kvm_flush_link_stack,
ppc_inst(PPC_INST_NOP));
#endif
- pr_info("link-stack-flush: software flush disabled.\n");
+ pr_info("link-stack-flush: flush disabled.\n");
link_stack_flush_type = BRANCH_CACHE_FLUSH_NONE;
no_count_cache_flush();
return;
@@ -475,13 +475,13 @@ static void toggle_branch_cache_flush(bool enable)
if (!security_ftr_enabled(SEC_FTR_BCCTR_FLUSH_ASSIST)) {
count_cache_flush_type = BRANCH_CACHE_FLUSH_SW;
- pr_info("count-cache-flush: full software flush sequence enabled.\n");
+ pr_info("count-cache-flush: software flush enabled.\n");
return;
}
patch_instruction_site(&patch__flush_count_cache_return, ppc_inst(PPC_INST_BLR));
count_cache_flush_type = BRANCH_CACHE_FLUSH_HW;
- pr_info("count-cache-flush: hardware assisted flush sequence enabled\n");
+ pr_info("count-cache-flush: hardware flush enabled.\n");
}
void setup_count_cache_flush(void)
--
2.23.0
^ permalink raw reply related [flat|nested] 10+ messages in thread* [PATCH 4/7] powerpc/security: split branch cache flush toggle from code patching
2020-06-09 6:16 [PATCH 0/7] powerpc: branch cache flush changes Nicholas Piggin
` (2 preceding siblings ...)
2020-06-09 6:16 ` [PATCH 3/7] powerpc/security: make display of branch cache flush more consistent Nicholas Piggin
@ 2020-06-09 6:16 ` Nicholas Piggin
2020-06-09 6:16 ` [PATCH 5/7] powerpc/64s: Move branch cache flushing bcctr variant to ppc-ops.h Nicholas Piggin
` (2 subsequent siblings)
6 siblings, 0 replies; 10+ messages in thread
From: Nicholas Piggin @ 2020-06-09 6:16 UTC (permalink / raw)
Cc: linuxppc-dev, Nicholas Piggin
Branch cache flushing code patching has inter-dependencies on both the
link stack and the count cache flushing state.
To make the code clearer and to separate the link stack and count
cache handling, split the "toggle" (setting up variables and printing
enable/disable) from the code patching.
Signed-off-by: Nicholas Piggin <npiggin@gmail.com>
---
arch/powerpc/kernel/security.c | 94 ++++++++++++++++++----------------
1 file changed, 51 insertions(+), 43 deletions(-)
diff --git a/arch/powerpc/kernel/security.c b/arch/powerpc/kernel/security.c
index 659ef6a92bb9..2a413af21124 100644
--- a/arch/powerpc/kernel/security.c
+++ b/arch/powerpc/kernel/security.c
@@ -427,61 +427,69 @@ static __init int stf_barrier_debugfs_init(void)
device_initcall(stf_barrier_debugfs_init);
#endif /* CONFIG_DEBUG_FS */
-static void no_count_cache_flush(void)
+static void update_branch_cache_flush(void)
{
- count_cache_flush_type = BRANCH_CACHE_FLUSH_NONE;
- pr_info("count-cache-flush: flush disabled.\n");
-}
-
-static void toggle_branch_cache_flush(bool enable)
-{
- if (!security_ftr_enabled(SEC_FTR_FLUSH_COUNT_CACHE) &&
- !security_ftr_enabled(SEC_FTR_FLUSH_LINK_STACK))
- enable = false;
-
- if (!enable) {
- patch_instruction_site(&patch__call_flush_branch_caches,
- ppc_inst(PPC_INST_NOP));
#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE
+ // This controls the branch from guest_exit_cont to kvm_flush_link_stack
+ if (link_stack_flush_type == BRANCH_CACHE_FLUSH_NONE) {
patch_instruction_site(&patch__call_kvm_flush_link_stack,
ppc_inst(PPC_INST_NOP));
-#endif
- pr_info("link-stack-flush: flush disabled.\n");
- link_stack_flush_type = BRANCH_CACHE_FLUSH_NONE;
- no_count_cache_flush();
- return;
+ } else {
+ patch_branch_site(&patch__call_kvm_flush_link_stack,
+ (u64)&kvm_flush_link_stack, BRANCH_SET_LINK);
}
-
- // This enables the branch from _switch to flush_branch_caches
- patch_branch_site(&patch__call_flush_branch_caches,
- (u64)&flush_branch_caches, BRANCH_SET_LINK);
-
-#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE
- // This enables the branch from guest_exit_cont to kvm_flush_link_stack
- patch_branch_site(&patch__call_kvm_flush_link_stack,
- (u64)&kvm_flush_link_stack, BRANCH_SET_LINK);
#endif
- pr_info("link-stack-flush: software flush enabled.\n");
- link_stack_flush_type = BRANCH_CACHE_FLUSH_SW;
+ // This controls the branch from _switch to flush_branch_caches
+ if (count_cache_flush_type == BRANCH_CACHE_FLUSH_NONE &&
+ link_stack_flush_type == BRANCH_CACHE_FLUSH_NONE) {
+ patch_instruction_site(&patch__call_flush_branch_caches,
+ ppc_inst(PPC_INST_NOP));
+ } else {
+ patch_branch_site(&patch__call_flush_branch_caches,
+ (u64)&flush_branch_caches, BRANCH_SET_LINK);
+
+ // If we just need to flush the link stack, early return
+ if (count_cache_flush_type == BRANCH_CACHE_FLUSH_NONE) {
+ patch_instruction_site(&patch__flush_link_stack_return,
+ ppc_inst(PPC_INST_BLR));
+
+ // If we have flush instruction, early return
+ } else if (count_cache_flush_type == BRANCH_CACHE_FLUSH_HW) {
+ patch_instruction_site(&patch__flush_count_cache_return,
+ ppc_inst(PPC_INST_BLR));
+ }
+ }
+}
- // If we just need to flush the link stack, patch an early return
- if (!security_ftr_enabled(SEC_FTR_FLUSH_COUNT_CACHE)) {
- patch_instruction_site(&patch__flush_link_stack_return,
- ppc_inst(PPC_INST_BLR));
- no_count_cache_flush();
- return;
+static void toggle_branch_cache_flush(bool enable)
+{
+ if (!enable || !security_ftr_enabled(SEC_FTR_FLUSH_COUNT_CACHE)) {
+ if (count_cache_flush_type != BRANCH_CACHE_FLUSH_NONE) {
+ count_cache_flush_type = BRANCH_CACHE_FLUSH_NONE;
+ pr_info("count-cache-flush: flush disabled.\n");
+ }
+ } else {
+ if (security_ftr_enabled(SEC_FTR_BCCTR_FLUSH_ASSIST)) {
+ count_cache_flush_type = BRANCH_CACHE_FLUSH_HW;
+ pr_info("count-cache-flush: hardware flush enabled.\n");
+ } else {
+ count_cache_flush_type = BRANCH_CACHE_FLUSH_SW;
+ pr_info("count-cache-flush: software flush enabled.\n");
+ }
}
- if (!security_ftr_enabled(SEC_FTR_BCCTR_FLUSH_ASSIST)) {
- count_cache_flush_type = BRANCH_CACHE_FLUSH_SW;
- pr_info("count-cache-flush: software flush enabled.\n");
- return;
+ if (!enable || !security_ftr_enabled(SEC_FTR_FLUSH_LINK_STACK)) {
+ if (link_stack_flush_type != BRANCH_CACHE_FLUSH_NONE) {
+ link_stack_flush_type = BRANCH_CACHE_FLUSH_NONE;
+ pr_info("link-stack-flush: flush disabled.\n");
+ }
+ } else {
+ link_stack_flush_type = BRANCH_CACHE_FLUSH_SW;
+ pr_info("link-stack-flush: software flush enabled.\n");
}
- patch_instruction_site(&patch__flush_count_cache_return, ppc_inst(PPC_INST_BLR));
- count_cache_flush_type = BRANCH_CACHE_FLUSH_HW;
- pr_info("count-cache-flush: hardware flush enabled.\n");
+ update_branch_cache_flush();
}
void setup_count_cache_flush(void)
--
2.23.0
^ permalink raw reply related [flat|nested] 10+ messages in thread* [PATCH 5/7] powerpc/64s: Move branch cache flushing bcctr variant to ppc-ops.h
2020-06-09 6:16 [PATCH 0/7] powerpc: branch cache flush changes Nicholas Piggin
` (3 preceding siblings ...)
2020-06-09 6:16 ` [PATCH 4/7] powerpc/security: split branch cache flush toggle from code patching Nicholas Piggin
@ 2020-06-09 6:16 ` Nicholas Piggin
2020-06-09 6:16 ` [PATCH 6/7] powerpc/security: Allow for processors that flush the link stack using the special bcctr Nicholas Piggin
2020-06-09 6:16 ` [PATCH 7/7] powerpc/64s: advertise hardware link stack flush Nicholas Piggin
6 siblings, 0 replies; 10+ messages in thread
From: Nicholas Piggin @ 2020-06-09 6:16 UTC (permalink / raw)
Cc: linuxppc-dev, Nicholas Piggin
Signed-off-by: Nicholas Piggin <npiggin@gmail.com>
---
arch/powerpc/include/asm/ppc-opcode.h | 2 ++
arch/powerpc/kernel/entry_64.S | 6 ++----
2 files changed, 4 insertions(+), 4 deletions(-)
diff --git a/arch/powerpc/include/asm/ppc-opcode.h b/arch/powerpc/include/asm/ppc-opcode.h
index 2a39c716c343..79d511a38bbb 100644
--- a/arch/powerpc/include/asm/ppc-opcode.h
+++ b/arch/powerpc/include/asm/ppc-opcode.h
@@ -195,6 +195,7 @@
#define OP_LQ 56
/* sorted alphabetically */
+#define PPC_INST_BCCTR_FLUSH 0x4c400420
#define PPC_INST_BHRBE 0x7c00025c
#define PPC_INST_CLRBHRB 0x7c00035c
#define PPC_INST_COPY 0x7c20060c
@@ -432,6 +433,7 @@
#endif
/* Deal with instructions that older assemblers aren't aware of */
+#define PPC_BCCTR_FLUSH stringify_in_c(.long PPC_INST_BCCTR_FLUSH)
#define PPC_CP_ABORT stringify_in_c(.long PPC_INST_CP_ABORT)
#define PPC_COPY(a, b) stringify_in_c(.long PPC_INST_COPY | \
___PPC_RA(a) | ___PPC_RB(b))
diff --git a/arch/powerpc/kernel/entry_64.S b/arch/powerpc/kernel/entry_64.S
index 2ba25b3b701e..a115aeb2983a 100644
--- a/arch/powerpc/kernel/entry_64.S
+++ b/arch/powerpc/kernel/entry_64.S
@@ -261,8 +261,6 @@ _ASM_NOKPROBE_SYMBOL(save_nvgprs);
1: nop; \
patch_site 1b, patch__call_flush_branch_caches
-#define BCCTR_FLUSH .long 0x4c400420
-
.macro nops number
.rept \number
nop
@@ -293,7 +291,7 @@ flush_branch_caches:
li r9,0x7fff
mtctr r9
- BCCTR_FLUSH
+ PPC_BCCTR_FLUSH
2: nop
patch_site 2b patch__flush_count_cache_return
@@ -302,7 +300,7 @@ flush_branch_caches:
.rept 278
.balign 32
- BCCTR_FLUSH
+ PPC_BCCTR_FLUSH
nops 7
.endr
--
2.23.0
^ permalink raw reply related [flat|nested] 10+ messages in thread* [PATCH 6/7] powerpc/security: Allow for processors that flush the link stack using the special bcctr
2020-06-09 6:16 [PATCH 0/7] powerpc: branch cache flush changes Nicholas Piggin
` (4 preceding siblings ...)
2020-06-09 6:16 ` [PATCH 5/7] powerpc/64s: Move branch cache flushing bcctr variant to ppc-ops.h Nicholas Piggin
@ 2020-06-09 6:16 ` Nicholas Piggin
2020-06-09 6:16 ` [PATCH 7/7] powerpc/64s: advertise hardware link stack flush Nicholas Piggin
6 siblings, 0 replies; 10+ messages in thread
From: Nicholas Piggin @ 2020-06-09 6:16 UTC (permalink / raw)
Cc: linuxppc-dev, Nicholas Piggin
If both count cache and link stack are to be flushed, and can be flushed
with the special bcctr, patch that in directly to the flush/branch nop
site.
Signed-off-by: Nicholas Piggin <npiggin@gmail.com>
---
arch/powerpc/include/asm/security_features.h | 2 ++
arch/powerpc/kernel/security.c | 27 ++++++++++++++------
2 files changed, 21 insertions(+), 8 deletions(-)
diff --git a/arch/powerpc/include/asm/security_features.h b/arch/powerpc/include/asm/security_features.h
index 7c05e95a5c44..fbb8fa32150f 100644
--- a/arch/powerpc/include/asm/security_features.h
+++ b/arch/powerpc/include/asm/security_features.h
@@ -63,6 +63,8 @@ static inline bool security_ftr_enabled(u64 feature)
// bcctr 2,0,0 triggers a hardware assisted count cache flush
#define SEC_FTR_BCCTR_FLUSH_ASSIST 0x0000000000000800ull
+// bcctr 2,0,0 triggers a hardware assisted link stack flush
+#define SEC_FTR_BCCTR_LINK_FLUSH_ASSIST 0x0000000000002000ull
// Features indicating need for Spectre/Meltdown mitigations
diff --git a/arch/powerpc/kernel/security.c b/arch/powerpc/kernel/security.c
index 2a413af21124..6ad5c753d47c 100644
--- a/arch/powerpc/kernel/security.c
+++ b/arch/powerpc/kernel/security.c
@@ -219,24 +219,25 @@ ssize_t cpu_show_spectre_v2(struct device *dev, struct device_attribute *attr, c
if (ccd)
seq_buf_printf(&s, "Indirect branch cache disabled");
- if (link_stack_flush_type == BRANCH_CACHE_FLUSH_SW)
- seq_buf_printf(&s, ", Software link stack flush");
-
} else if (count_cache_flush_type != BRANCH_CACHE_FLUSH_NONE) {
seq_buf_printf(&s, "Mitigation: Software count cache flush");
if (count_cache_flush_type == BRANCH_CACHE_FLUSH_HW)
seq_buf_printf(&s, " (hardware accelerated)");
- if (link_stack_flush_type == BRANCH_CACHE_FLUSH_SW)
- seq_buf_printf(&s, ", Software link stack flush");
-
} else if (btb_flush_enabled) {
seq_buf_printf(&s, "Mitigation: Branch predictor state flush");
} else {
seq_buf_printf(&s, "Vulnerable");
}
+ if (bcs || ccd || count_cache_flush_type != BRANCH_CACHE_FLUSH_NONE) {
+ if (link_stack_flush_type != BRANCH_CACHE_FLUSH_NONE)
+ seq_buf_printf(&s, ", Software link stack flush");
+ if (link_stack_flush_type == BRANCH_CACHE_FLUSH_HW)
+ seq_buf_printf(&s, " (hardware accelerated)");
+ }
+
seq_buf_printf(&s, "\n");
return s.len;
@@ -435,6 +436,7 @@ static void update_branch_cache_flush(void)
patch_instruction_site(&patch__call_kvm_flush_link_stack,
ppc_inst(PPC_INST_NOP));
} else {
+ // Could use HW flush, but that could also flush count cache
patch_branch_site(&patch__call_kvm_flush_link_stack,
(u64)&kvm_flush_link_stack, BRANCH_SET_LINK);
}
@@ -445,6 +447,10 @@ static void update_branch_cache_flush(void)
link_stack_flush_type == BRANCH_CACHE_FLUSH_NONE) {
patch_instruction_site(&patch__call_flush_branch_caches,
ppc_inst(PPC_INST_NOP));
+ } else if (count_cache_flush_type == BRANCH_CACHE_FLUSH_HW &&
+ link_stack_flush_type == BRANCH_CACHE_FLUSH_HW) {
+ patch_instruction_site(&patch__call_flush_branch_caches,
+ ppc_inst(PPC_INST_BCCTR_FLUSH));
} else {
patch_branch_site(&patch__call_flush_branch_caches,
(u64)&flush_branch_caches, BRANCH_SET_LINK);
@@ -485,8 +491,13 @@ static void toggle_branch_cache_flush(bool enable)
pr_info("link-stack-flush: flush disabled.\n");
}
} else {
- link_stack_flush_type = BRANCH_CACHE_FLUSH_SW;
- pr_info("link-stack-flush: software flush enabled.\n");
+ if (security_ftr_enabled(SEC_FTR_BCCTR_LINK_FLUSH_ASSIST)) {
+ link_stack_flush_type = BRANCH_CACHE_FLUSH_HW;
+ pr_info("link-stack-flush: hardware flush enabled.\n");
+ } else {
+ link_stack_flush_type = BRANCH_CACHE_FLUSH_SW;
+ pr_info("link-stack-flush: software flush enabled.\n");
+ }
}
update_branch_cache_flush();
--
2.23.0
^ permalink raw reply related [flat|nested] 10+ messages in thread* [PATCH 7/7] powerpc/64s: advertise hardware link stack flush
2020-06-09 6:16 [PATCH 0/7] powerpc: branch cache flush changes Nicholas Piggin
` (5 preceding siblings ...)
2020-06-09 6:16 ` [PATCH 6/7] powerpc/security: Allow for processors that flush the link stack using the special bcctr Nicholas Piggin
@ 2020-06-09 6:16 ` Nicholas Piggin
6 siblings, 0 replies; 10+ messages in thread
From: Nicholas Piggin @ 2020-06-09 6:16 UTC (permalink / raw)
Cc: linuxppc-dev, Nicholas Piggin
For testing only at the moment, firmware does not define these bits.
---
arch/powerpc/include/asm/hvcall.h | 1 +
arch/powerpc/include/uapi/asm/kvm.h | 1 +
arch/powerpc/kvm/powerpc.c | 9 +++++++--
arch/powerpc/platforms/powernv/setup.c | 3 +++
arch/powerpc/platforms/pseries/setup.c | 3 +++
tools/arch/powerpc/include/uapi/asm/kvm.h | 1 +
6 files changed, 16 insertions(+), 2 deletions(-)
diff --git a/arch/powerpc/include/asm/hvcall.h b/arch/powerpc/include/asm/hvcall.h
index e90c073e437e..a92a07c89b6f 100644
--- a/arch/powerpc/include/asm/hvcall.h
+++ b/arch/powerpc/include/asm/hvcall.h
@@ -373,6 +373,7 @@
#define H_CPU_CHAR_THREAD_RECONFIG_CTRL (1ull << 57) // IBM bit 6
#define H_CPU_CHAR_COUNT_CACHE_DISABLED (1ull << 56) // IBM bit 7
#define H_CPU_CHAR_BCCTR_FLUSH_ASSIST (1ull << 54) // IBM bit 9
+#define H_CPU_CHAR_BCCTR_LINK_FLUSH_ASSIST (1ull << 53) // IBM bit 10
#define H_CPU_BEHAV_FAVOUR_SECURITY (1ull << 63) // IBM bit 0
#define H_CPU_BEHAV_L1D_FLUSH_PR (1ull << 62) // IBM bit 1
diff --git a/arch/powerpc/include/uapi/asm/kvm.h b/arch/powerpc/include/uapi/asm/kvm.h
index 264e266a85bf..dd229d5f46ee 100644
--- a/arch/powerpc/include/uapi/asm/kvm.h
+++ b/arch/powerpc/include/uapi/asm/kvm.h
@@ -464,6 +464,7 @@ struct kvm_ppc_cpu_char {
#define KVM_PPC_CPU_CHAR_MTTRIG_THR_RECONF (1ULL << 57)
#define KVM_PPC_CPU_CHAR_COUNT_CACHE_DIS (1ULL << 56)
#define KVM_PPC_CPU_CHAR_BCCTR_FLUSH_ASSIST (1ull << 54)
+#define KVM_PPC_CPU_CHAR_BCCTR_LINK_FLUSH_ASSIST (1ull << 53)
#define KVM_PPC_CPU_BEHAV_FAVOUR_SECURITY (1ULL << 63)
#define KVM_PPC_CPU_BEHAV_L1D_FLUSH_PR (1ULL << 62)
diff --git a/arch/powerpc/kvm/powerpc.c b/arch/powerpc/kvm/powerpc.c
index 27ccff612903..fa981ee09dec 100644
--- a/arch/powerpc/kvm/powerpc.c
+++ b/arch/powerpc/kvm/powerpc.c
@@ -2221,7 +2221,8 @@ static int pseries_get_cpu_char(struct kvm_ppc_cpu_char *cp)
KVM_PPC_CPU_CHAR_BR_HINT_HONOURED |
KVM_PPC_CPU_CHAR_MTTRIG_THR_RECONF |
KVM_PPC_CPU_CHAR_COUNT_CACHE_DIS |
- KVM_PPC_CPU_CHAR_BCCTR_FLUSH_ASSIST;
+ KVM_PPC_CPU_CHAR_BCCTR_FLUSH_ASSIST |
+ KVM_PPC_CPU_CHAR_BCCTR_LINK_FLUSH_ASSIST;
cp->behaviour_mask = KVM_PPC_CPU_BEHAV_FAVOUR_SECURITY |
KVM_PPC_CPU_BEHAV_L1D_FLUSH_PR |
KVM_PPC_CPU_BEHAV_BNDS_CHK_SPEC_BAR |
@@ -2287,13 +2288,17 @@ static int kvmppc_get_cpu_char(struct kvm_ppc_cpu_char *cp)
if (have_fw_feat(fw_features, "enabled",
"fw-count-cache-flush-bcctr2,0,0"))
cp->character |= KVM_PPC_CPU_CHAR_BCCTR_FLUSH_ASSIST;
+ if (have_fw_feat(fw_features, "enabled",
+ "fw-link-stack-flush-bcctr2,0,0"))
+ cp->character |= KVM_PPC_CPU_CHAR_BCCTR_LINK_FLUSH_ASSIST;
cp->character_mask = KVM_PPC_CPU_CHAR_SPEC_BAR_ORI31 |
KVM_PPC_CPU_CHAR_BCCTRL_SERIALISED |
KVM_PPC_CPU_CHAR_L1D_FLUSH_ORI30 |
KVM_PPC_CPU_CHAR_L1D_FLUSH_TRIG2 |
KVM_PPC_CPU_CHAR_L1D_THREAD_PRIV |
KVM_PPC_CPU_CHAR_COUNT_CACHE_DIS |
- KVM_PPC_CPU_CHAR_BCCTR_FLUSH_ASSIST;
+ KVM_PPC_CPU_CHAR_BCCTR_FLUSH_ASSIST |
+ KVM_PPC_CPU_CHAR_BCCTR_LINK_FLUSH_ASSIST;
if (have_fw_feat(fw_features, "enabled",
"speculation-policy-favor-security"))
diff --git a/arch/powerpc/platforms/powernv/setup.c b/arch/powerpc/platforms/powernv/setup.c
index 3bc188da82ba..1a06d3b4c0a9 100644
--- a/arch/powerpc/platforms/powernv/setup.c
+++ b/arch/powerpc/platforms/powernv/setup.c
@@ -78,6 +78,9 @@ static void init_fw_feat_flags(struct device_node *np)
if (fw_feature_is("enabled", "fw-count-cache-flush-bcctr2,0,0", np))
security_ftr_set(SEC_FTR_BCCTR_FLUSH_ASSIST);
+ if (fw_feature_is("enabled", "fw-link-stack-flush-bcctr2,0,0", np))
+ security_ftr_set(SEC_FTR_BCCTR_LINK_FLUSH_ASSIST);
+
if (fw_feature_is("enabled", "needs-count-cache-flush-on-context-switch", np))
security_ftr_set(SEC_FTR_FLUSH_COUNT_CACHE);
diff --git a/arch/powerpc/platforms/pseries/setup.c b/arch/powerpc/platforms/pseries/setup.c
index 64d18f4bf093..70c9264f23c5 100644
--- a/arch/powerpc/platforms/pseries/setup.c
+++ b/arch/powerpc/platforms/pseries/setup.c
@@ -517,6 +517,9 @@ static void init_cpu_char_feature_flags(struct h_cpu_char_result *result)
if (result->character & H_CPU_CHAR_BCCTR_FLUSH_ASSIST)
security_ftr_set(SEC_FTR_BCCTR_FLUSH_ASSIST);
+ if (result->character & H_CPU_CHAR_BCCTR_LINK_FLUSH_ASSIST)
+ security_ftr_set(SEC_FTR_BCCTR_LINK_FLUSH_ASSIST);
+
if (result->behaviour & H_CPU_BEHAV_FLUSH_COUNT_CACHE)
security_ftr_set(SEC_FTR_FLUSH_COUNT_CACHE);
diff --git a/tools/arch/powerpc/include/uapi/asm/kvm.h b/tools/arch/powerpc/include/uapi/asm/kvm.h
index 264e266a85bf..dd229d5f46ee 100644
--- a/tools/arch/powerpc/include/uapi/asm/kvm.h
+++ b/tools/arch/powerpc/include/uapi/asm/kvm.h
@@ -464,6 +464,7 @@ struct kvm_ppc_cpu_char {
#define KVM_PPC_CPU_CHAR_MTTRIG_THR_RECONF (1ULL << 57)
#define KVM_PPC_CPU_CHAR_COUNT_CACHE_DIS (1ULL << 56)
#define KVM_PPC_CPU_CHAR_BCCTR_FLUSH_ASSIST (1ull << 54)
+#define KVM_PPC_CPU_CHAR_BCCTR_LINK_FLUSH_ASSIST (1ull << 53)
#define KVM_PPC_CPU_BEHAV_FAVOUR_SECURITY (1ULL << 63)
#define KVM_PPC_CPU_BEHAV_L1D_FLUSH_PR (1ULL << 62)
--
2.23.0
^ permalink raw reply related [flat|nested] 10+ messages in thread