From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: X-Google-Smtp-Source: AH8x2268bdJZHR4OB/Af+pYNZupMZPtZNtYPzUUuKRGWysfHisrzJsZf9ZJsFmEjNtgSlV1k84dN ARC-Seal: i=1; a=rsa-sha256; t=1516907260; cv=none; d=google.com; s=arc-20160816; b=Ob4WQLpNaKfSkSP5s2QpRonnlpUtLw7k5MceNtcXhiYaViXNi+D2z21zwSBgHf4Qow nVCQZT7yGz9l1Y34eRZrVNwS8ncisol5HLQMgFKEjupcs7pcrWXHksbLe1WyWNsxyodu Izl2s1OZMrkkQ0N3LawpnePdfKW2N3hyCAM+GeXWGq7dJnbQQHBEksxqoU7roWCtICJL P3AxeWt5NPB60vk15va011QveHW+s83F4euVEAhcrqOjzgWYv+Q9wYYdG3dB1l/14l2a 0fytQYofIqcqJscpyNvLilB732qZpTjLBv1Dc+UsP+CHWoaoPhMEK0jH4ByWjjVlxvUC lbIQ== ARC-Message-Signature: i=1; a=rsa-sha256; c=relaxed/relaxed; d=google.com; s=arc-20160816; h=user-agent:in-reply-to:content-disposition:mime-version:references :message-id:subject:cc:to:from:date:arc-authentication-results; bh=2ezcIPAYhE/mQevkSi9saeJVM1ZBV8btIvICJC4oy4c=; b=XWu6AV8GEZlp573xO/urh7aySTpOmwLq40Aq6tBBOyQNnXDTLgvvWMulnsC3XLrY9u 8j2b7vpRrmbfS8uZxZWHM2y8YXyFcsAosghGkMN112vGYzC4zgNDeNw6uEX5pGHX2KpL WjnaSniTEOOd8G7Ub9YdcYVcNiG7IGX+tFSlqOFty3qRdzZ2Wrb8kInjDYc1Bjue0Z+T INXXoGuT5mPdaIMzg+lZQ9eQF0IbXlgPAWjIJ/VYvt7b+4dYOLgDZgKckWJ2llmggoTq l0MlwBVDJKmL89OGr/mGhgUx/CTRn+Fv1WkyK0doZSuJ0p0MKT/V/6Edw/jy90WYi4xT jecw== ARC-Authentication-Results: i=1; mx.google.com; spf=pass (google.com: domain of bp@alien8.de designates 5.9.137.197 as permitted sender) smtp.mailfrom=bp@alien8.de Authentication-Results: mx.google.com; spf=pass (google.com: domain of bp@alien8.de designates 5.9.137.197 as permitted sender) smtp.mailfrom=bp@alien8.de Date: Thu, 25 Jan 2018 20:07:29 +0100 From: Borislav Petkov To: David Woodhouse Cc: Josh Poimboeuf , tim.c.chen@linux.intel.com, pjt@google.com, jikos@kernel.org, gregkh@linux-foundation.org, dave.hansen@intel.com, mingo@kernel.org, riel@redhat.com, luto@amacapital.net, torvalds@linux-foundation.org, ak@linux.intel.com, keescook@google.com, peterz@infradead.org, tglx@linutronix.de, hpa@zytor.com, linux-kernel@vger.kernel.org, linux-tip-commits@vger.kernel.org Subject: Re: [tip:x86/pti] x86/retpoline: Fill return stack buffer on vmexit Message-ID: <20180125190729.jxey2k44sptuk42t@pd.tnic> References: <20180125120743.ey32gvl5mjam4r2s@pd.tnic> <1516882849.30244.94.camel@infradead.org> <20180125124554.vdx7rrnfrxrzl2ng@pd.tnic> <20180125151024.bidjr26r667vs7h5@treble> <20180125155110.mw655b7fwgm7qqc7@pd.tnic> <1516896198.30244.133.camel@infradead.org> <20180125165646.ytc4upthpaqtwi45@treble> <1516899639.30244.145.camel@infradead.org> <20180125175334.7wlin4ftnqkq2fil@pd.tnic> <1516903463.30244.148.camel@infradead.org> MIME-Version: 1.0 Content-Type: text/plain; charset=utf-8 Content-Disposition: inline In-Reply-To: <1516903463.30244.148.camel@infradead.org> User-Agent: NeoMutt/20170609 (1.8.3) X-getmail-retrieved-from-mailbox: INBOX X-GMAIL-THRID: =?utf-8?q?1589334225298273984?= X-GMAIL-MSGID: =?utf-8?q?1590592547786270901?= X-Mailing-List: linux-kernel@vger.kernel.org List-ID: On Thu, Jan 25, 2018 at 06:04:23PM +0000, David Woodhouse wrote: > Yep, I'll buy that. But first we need Josh to work out what he's having > for lunch. > > Although just another marker to tell objtool "ignore this whole > function" might be sufficient to allow us to have an out-of-line RSB- > stuffing function. Ok, I think we solved it all on IRC. I'm adding a conglomerate patch at the end. It builds and boots here in a vm. Here's what I did: The barrier: static inline void indirect_branch_prediction_barrier(void) { alternative_input("", "call __ibp_barrier", X86_FEATURE_IBPB, ASM_NO_INPUT_CLOBBER("eax", "ecx", "edx", "memory")); } with void __ibp_barrier(void) { wrmsr(MSR_IA32_PRED_CMD, PRED_CMD_IBPB, 0); } Alternatives applies correctly: [ 1.040965] apply_alternatives: feat: 7*32+16, old: (ffffffff8259d806 len: 5), repl: (ffffffff826aa88b, len: 5), pad: 5 [ 1.042798] ffffffff8259d806: old_insn: 90 90 90 90 90 [ 1.044001] ffffffff826aa88b: rpl_insn: e8 80 c3 9c fe [ 1.044938] apply_alternatives: Fix CALL offset: 0xfead9405, CALL 0xffffffff81076c10 [ 1.046343] ffffffff8259d806: final_insn: e8 05 94 ad fe with final_insn becoming: ffffffff8259d806: e8 80 c3 9c fe callq ffffffff81076c10 <__ibp_barrier> When we look at the __ibp_barrier: ffffffff81076c10 <__ibp_barrier>: ffffffff81076c10: e8 bb ac 98 00 callq ffffffff81a018d0 <__fentry__> ffffffff81076c15: b9 49 00 00 00 mov $0x49,%ecx ffffffff81076c1a: b8 01 00 00 00 mov $0x1,%eax ffffffff81076c1f: 31 d2 xor %edx,%edx ffffffff81076c21: 0f 30 wrmsr ffffffff81076c23: 0f 1f 44 00 00 nopl 0x0(%rax,%rax,1) ffffffff81076c28: c3 retq ffffffff81076c29: 31 d2 xor %edx,%edx ffffffff81076c2b: be 01 00 00 00 mov $0x1,%esi ffffffff81076c30: bf 49 00 00 00 mov $0x49,%edi ffffffff81076c35: e9 76 64 3f 00 jmpq ffffffff8146d0b0 ffffffff81076c3a: 90 nop ffffffff81076c3b: 90 nop ffffffff81076c3c: 90 nop ffffffff81076c3d: 90 nop ffffffff81076c3e: 90 nop It has the tracing shit too. We could replace that with __wrmsr() which does exception handling. And I think we should do that instead. /me does it. The RSB filler became: static inline void vmexit_fill_RSB(void) { #ifdef CONFIG_RETPOLINE alternative_input("", "call __fill_rsb_clobber_ax", X86_FEATURE_RETPOLINE, ASM_NO_INPUT_CLOBBER("memory")); #endif } and __fill_rsb_clobber_ax looks like: ENTRY(__fill_rsb_clobber_ax) ___FILL_RETURN_BUFFER %_ASM_AX, RSB_CLEAR_LOOPS, %_ASM_SP END(__fill_rsb_clobber_ax) where: .macro ___FILL_RETURN_BUFFER reg:req nr:req sp:req mov (\nr / 2), \reg .align 16 771: call 772f 773: /* speculation trap */ pause lfence jmp 773b .align 16 772: call 774f 775: /* speculation trap */ pause lfence jmp 775b .align 16 774: dec \reg jnz 771b add (BITS_PER_LONG /8) * \nr, \sp .endm Please check the alignment directives are at the right places. Application looks correct too: [ 0.428974] apply_alternatives: feat: 7*32+12, old: (ffffffff8105fe50 len: 5), repl: (ffffffff826aa69f, len: 5), pad: 5 [ 0.432003] ffffffff8105fe50: old_insn: 90 90 90 90 90 [ 0.432977] ffffffff826aa69f: rpl_insn: e8 5c 8b 55 ff [ 0.433943] apply_alternatives: Fix CALL offset: 0xba33ab, CALL 0xffffffff81c03200 [ 0.435368] ffffffff8105fe50: final_insn: e8 ab 33 ba 00 final_insn becomes ffffffff8105fe50: e8 5c 8b 55 ff callq ffffffff81c03200 <__fill_rsb_clobber_ax> and the __fill_rsb_clobber_ax asm look good to me too: ffffffff81c03200 <__fill_rsb_clobber_ax>: ffffffff81c03200: 48 8b 04 25 10 00 00 mov 0x10,%rax ffffffff81c03207: 00 ffffffff81c03208: 0f 1f 84 00 00 00 00 nopl 0x0(%rax,%rax,1) ffffffff81c0320f: 00 ffffffff81c03210: e8 0b 00 00 00 callq ffffffff81c03220 <__fill_rsb_clobber_ax+0x20> ffffffff81c03215: f3 90 pause ffffffff81c03217: 0f ae e8 lfence ffffffff81c0321a: eb f9 jmp ffffffff81c03215 <__fill_rsb_clobber_ax+0x15> ffffffff81c0321c: 0f 1f 40 00 nopl 0x0(%rax) ffffffff81c03220: e8 0b 00 00 00 callq ffffffff81c03230 <__fill_rsb_clobber_ax+0x30> ffffffff81c03225: f3 90 pause ffffffff81c03227: 0f ae e8 lfence ffffffff81c0322a: eb f9 jmp ffffffff81c03225 <__fill_rsb_clobber_ax+0x25> ffffffff81c0322c: 0f 1f 40 00 nopl 0x0(%rax) ffffffff81c03230: 48 ff c8 dec %rax ffffffff81c03233: 75 db jne ffffffff81c03210 <__fill_rsb_clobber_ax+0x10> ffffffff81c03235: 48 03 24 25 00 01 00 add 0x100,%rsp ffffffff81c0323c: 00 with the respective alignment. Btw, I've forced setup_force_cpu_cap(X86_FEATURE_IBPB); so that I can see it applies only - nothing else. Here's the full diff I have so far: diff --git a/arch/x86/include/asm/cpufeatures.h b/arch/x86/include/asm/cpufeatures.h index 67bbfaa1448b..a7c5602847b1 100644 --- a/arch/x86/include/asm/cpufeatures.h +++ b/arch/x86/include/asm/cpufeatures.h @@ -210,6 +210,7 @@ #define X86_FEATURE_AVX512_4VNNIW ( 7*32+16) /* AVX-512 Neural Network Instructions */ #define X86_FEATURE_AVX512_4FMAPS ( 7*32+17) /* AVX-512 Multiply Accumulation Single precision */ +#define X86_FEATURE_IBPB ( 7*32+16) /* Using Indirect Branch Prediction Barrier */ #define X86_FEATURE_MBA ( 7*32+18) /* Memory Bandwidth Allocation */ #define X86_FEATURE_RSB_CTXSW ( 7*32+19) /* Fill RSB on context switches */ diff --git a/arch/x86/include/asm/msr-index.h b/arch/x86/include/asm/msr-index.h index e7b983a35506..1c4e87b73789 100644 --- a/arch/x86/include/asm/msr-index.h +++ b/arch/x86/include/asm/msr-index.h @@ -39,6 +39,9 @@ /* Intel MSRs. Some also available on other CPUs */ +#define MSR_IA32_PRED_CMD 0x00000049 /* Prediction Command */ +#define PRED_CMD_IBPB (1 << 0) /* Indirect Branch Prediction Barrier */ + #define MSR_PPIN_CTL 0x0000004e #define MSR_PPIN 0x0000004f diff --git a/arch/x86/include/asm/nospec-branch.h b/arch/x86/include/asm/nospec-branch.h index 4ad41087ce0e..3e4d4996374e 100644 --- a/arch/x86/include/asm/nospec-branch.h +++ b/arch/x86/include/asm/nospec-branch.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 */ -#ifndef __NOSPEC_BRANCH_H__ -#define __NOSPEC_BRANCH_H__ +#ifndef __X86_ASM_NOSPEC_BRANCH_H__ +#define __X86_ASM_NOSPEC_BRANCH_H__ #include #include @@ -53,6 +53,9 @@ #ifdef __ASSEMBLY__ +#include + + /* * This should be used immediately before a retpoline alternative. It tells * objtool where the retpolines are so that it can make sense of the control @@ -121,6 +124,29 @@ #endif .endm +.macro ___FILL_RETURN_BUFFER reg:req nr:req sp:req + mov (\nr / 2), \reg + .align 16 +771: + call 772f +773: /* speculation trap */ + pause + lfence + jmp 773b + .align 16 +772: + call 774f +775: /* speculation trap */ + pause + lfence + jmp 775b + .align 16 +774: + dec \reg + jnz 771b + add (BITS_PER_LONG /8) * \nr, \sp +.endm + /* * A simpler FILL_RETURN_BUFFER macro. Don't make people use the CPP * monstrosity above, manually. @@ -206,17 +232,14 @@ extern char __indirect_thunk_end[]; static inline void vmexit_fill_RSB(void) { #ifdef CONFIG_RETPOLINE - unsigned long loops; - - asm volatile (ANNOTATE_NOSPEC_ALTERNATIVE - ALTERNATIVE("jmp 910f", - __stringify(__FILL_RETURN_BUFFER(%0, RSB_CLEAR_LOOPS, %1)), - X86_FEATURE_RETPOLINE) - "910:" - : "=r" (loops), ASM_CALL_CONSTRAINT - : : "memory" ); + alternative_input("", "call __fill_rsb_clobber_ax", X86_FEATURE_RETPOLINE, ASM_NO_INPUT_CLOBBER("memory")); #endif } +static inline void indirect_branch_prediction_barrier(void) +{ + alternative_input("", "call __ibp_barrier", X86_FEATURE_IBPB, ASM_NO_INPUT_CLOBBER("eax", "ecx", "edx", "memory")); +} + #endif /* __ASSEMBLY__ */ -#endif /* __NOSPEC_BRANCH_H__ */ +#endif /* __X86_ASM_NOSPEC_BRANCH_H__ */ diff --git a/arch/x86/include/asm/processor.h b/arch/x86/include/asm/processor.h index d3a67fba200a..624bc37dc696 100644 --- a/arch/x86/include/asm/processor.h +++ b/arch/x86/include/asm/processor.h @@ -971,4 +971,9 @@ bool xen_set_default_idle(void); void stop_this_cpu(void *dummy); void df_debug(struct pt_regs *regs, long error_code); + +#ifdef CONFIG_RETPOLINE +asmlinkage void __fill_rsb_clobber_ax(void); +void __ibp_barrier(void); +#endif #endif /* _ASM_X86_PROCESSOR_H */ diff --git a/arch/x86/kernel/cpu/bugs.c b/arch/x86/kernel/cpu/bugs.c index 390b3dc3d438..35f60e5b93bf 100644 --- a/arch/x86/kernel/cpu/bugs.c +++ b/arch/x86/kernel/cpu/bugs.c @@ -249,6 +249,10 @@ static void __init spectre_v2_select_mitigation(void) setup_force_cpu_cap(X86_FEATURE_RSB_CTXSW); pr_info("Filling RSB on context switch\n"); } + + setup_force_cpu_cap(X86_FEATURE_IBPB); + + indirect_branch_prediction_barrier(); } #undef pr_fmt @@ -281,3 +285,10 @@ ssize_t cpu_show_spectre_v2(struct device *dev, return sprintf(buf, "%s\n", spectre_v2_strings[spectre_v2_enabled]); } #endif + +#ifdef CONFIG_RETPOLINE +void __ibp_barrier(void) +{ + __wrmsr(MSR_IA32_PRED_CMD, PRED_CMD_IBPB, 0); +} +#endif diff --git a/arch/x86/lib/Makefile b/arch/x86/lib/Makefile index f23934bbaf4e..69a473919260 100644 --- a/arch/x86/lib/Makefile +++ b/arch/x86/lib/Makefile @@ -27,6 +27,7 @@ lib-$(CONFIG_RWSEM_XCHGADD_ALGORITHM) += rwsem.o lib-$(CONFIG_INSTRUCTION_DECODER) += insn.o inat.o insn-eval.o lib-$(CONFIG_RANDOMIZE_BASE) += kaslr.o lib-$(CONFIG_RETPOLINE) += retpoline.o +OBJECT_FILES_NON_STANDARD_retpoline.o :=y obj-y += msr.o msr-reg.o msr-reg-export.o hweight.o diff --git a/arch/x86/lib/retpoline.S b/arch/x86/lib/retpoline.S index dfb2ba91b670..62eaa8f80d30 100644 --- a/arch/x86/lib/retpoline.S +++ b/arch/x86/lib/retpoline.S @@ -47,3 +47,7 @@ GENERATE_THUNK(r13) GENERATE_THUNK(r14) GENERATE_THUNK(r15) #endif + +ENTRY(__fill_rsb_clobber_ax) + ___FILL_RETURN_BUFFER %_ASM_AX, RSB_CLEAR_LOOPS, %_ASM_SP +END(__fill_rsb_clobber_ax) -- 2.13.0 -- Regards/Gruss, Boris. Good mailing practices for 400: avoid top-posting and trim the reply.