From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: X-Google-Smtp-Source: AH8x224VSMJu5QTBAwKYXZ0LdgeDp8JvMBF6Exvz5MT0sq/9K8CyiQ6sWM44+jf/oSzd1ecCcG42 ARC-Seal: i=1; a=rsa-sha256; t=1516997255; cv=none; d=google.com; s=arc-20160816; b=loEqpYWWpTI3QOBNwN01k3zWjndk3i75f5v4irqJqKDDDugSM1bQ9MfSIZsQE2p6Pk nilNN/g2J6CEJax9563Whc5ujG6Iv7eec1rzzd2f7W5TUEeABdSEdYZKIL2O+Id62Qln g8MMP3ZFvqFqRVNxILLJUV4RoQ1XRjX35CDd+g1bbdFDsxjpcbZzRECFbzABfRxBLqAD uEL0QOvidg1tckp6t1tu13grI6LHP8WmAVICKxaBqXZvGZFFpf+Cw9cuk98gAxCFTLg3 yVaV35TN7WCOluv4rYenjs36TZsX+ERcSvsqvnSnIhotUMXWCMePtKml33Y5VLi2anVi SAsg== ARC-Message-Signature: i=1; a=rsa-sha256; c=relaxed/relaxed; d=google.com; s=arc-20160816; h=user-agent:in-reply-to:content-transfer-encoding :content-disposition:mime-version:references:message-id:subject:cc :to:from:date:arc-authentication-results; bh=lhdkIMybaKboa5jobd02QUWbtHK9tJY5HT+AYM975BA=; b=SVHNZvvCDnL6cySmxm+q9mGtNpLmnfFwGbctfgvSBCfXEB+c/eV4CoWl71Rirl146G P73R7ugkE4kmS8MQkcNMWGviFLNtQgiyVtfRY6Q4xYU+2vC1CHSXbPXAP7gnhufNACUF R20CAPWGe9SiSy3fkIkKhFEiR/v5dONM1cM6oB1SVTqkmPkEZni9tFHg6i/xfMBdNsf9 4Yn8hoYYwC1O0C+2CsFT/BJSNgHAeK+FUQYw0QUOhjN3PzFC2BOpnlLZ/W0cxA/QInTJ Q8yP/+d9KUn+yTzWmTsojKflTzOiaZrgy9tkNSD5NiAd0lI7seDbMVt1VJK3xbKPP5gE ADBA== ARC-Authentication-Results: i=1; mx.google.com; spf=pass (google.com: domain of bp@alien8.de designates 5.9.137.197 as permitted sender) smtp.mailfrom=bp@alien8.de Authentication-Results: mx.google.com; spf=pass (google.com: domain of bp@alien8.de designates 5.9.137.197 as permitted sender) smtp.mailfrom=bp@alien8.de Date: Fri, 26 Jan 2018 21:07:25 +0100 From: Borislav Petkov To: David Woodhouse Cc: X86 ML , LKML , Josh Poimboeuf , tim.c.chen@linux.intel.com, pjt@google.com, jikos@kernel.org, gregkh@linux-foundation.org, dave.hansen@intel.com, riel@redhat.com, luto@amacapital.net, torvalds@linux-foundation.org, ak@linux.intel.com, keescook@google.com, peterz@infradead.org Subject: [PATCH v2 1/2] x86/retpoline: Simplify vmexit_fill_RSB() Message-ID: <20180126200725.rueqb2m6b4w47eu3@pd.tnic> References: <20180126121139.31959-1-bp@alien8.de> <20180126121139.31959-4-bp@alien8.de> <1516970011.30244.223.camel@infradead.org> <20180126132431.fsbd3c3g2yreazy6@pd.tnic> <1516983879.30244.236.camel@infradead.org> <20180126164746.dpo7dswid5tjk2tz@pd.tnic> <20180126200616.5xfn244uzeu7ptyo@pd.tnic> MIME-Version: 1.0 Content-Type: text/plain; charset=utf-8 Content-Disposition: inline Content-Transfer-Encoding: 8bit In-Reply-To: <20180126200616.5xfn244uzeu7ptyo@pd.tnic> User-Agent: NeoMutt/20170609 (1.8.3) X-getmail-retrieved-from-mailbox: INBOX X-GMAIL-THRID: =?utf-8?q?1590656989548468968?= X-GMAIL-MSGID: =?utf-8?q?1590686914239603356?= X-Mailing-List: linux-kernel@vger.kernel.org List-ID: Simplify it to call an asm-function instead of pasting 41 insn bytes at every call site. Also, add alignment to the macro as suggested here: https://support.google.com/faqs/answer/7625886 Signed-off-by: Borislav Petkov Cc: David Woodhouse --- arch/x86/entry/entry_32.S | 2 +- arch/x86/entry/entry_64.S | 2 +- arch/x86/include/asm/asm-prototypes.h | 3 +++ arch/x86/include/asm/nospec-branch.h | 49 +++++------------------------------ arch/x86/lib/Makefile | 1 + arch/x86/lib/retpoline.S | 44 +++++++++++++++++++++++++++++++ 6 files changed, 56 insertions(+), 45 deletions(-) diff --git a/arch/x86/entry/entry_32.S b/arch/x86/entry/entry_32.S index 60c4c342316c..f7823a5a8714 100644 --- a/arch/x86/entry/entry_32.S +++ b/arch/x86/entry/entry_32.S @@ -252,7 +252,7 @@ ENTRY(__switch_to_asm) * exist, overwrite the RSB with entries which capture * speculative execution to prevent attack. */ - FILL_RETURN_BUFFER %ebx, RSB_CLEAR_LOOPS, X86_FEATURE_RSB_CTXSW + FILL_RETURN_BUFFER RSB_CLEAR_LOOPS, X86_FEATURE_RSB_CTXSW #endif /* restore callee-saved registers */ diff --git a/arch/x86/entry/entry_64.S b/arch/x86/entry/entry_64.S index ff6f8022612c..7a190ff524e2 100644 --- a/arch/x86/entry/entry_64.S +++ b/arch/x86/entry/entry_64.S @@ -499,7 +499,7 @@ ENTRY(__switch_to_asm) * exist, overwrite the RSB with entries which capture * speculative execution to prevent attack. */ - FILL_RETURN_BUFFER %r12, RSB_CLEAR_LOOPS, X86_FEATURE_RSB_CTXSW + FILL_RETURN_BUFFER RSB_CLEAR_LOOPS, X86_FEATURE_RSB_CTXSW #endif /* restore callee-saved registers */ diff --git a/arch/x86/include/asm/asm-prototypes.h b/arch/x86/include/asm/asm-prototypes.h index 1908214b9125..4d111616524b 100644 --- a/arch/x86/include/asm/asm-prototypes.h +++ b/arch/x86/include/asm/asm-prototypes.h @@ -38,4 +38,7 @@ INDIRECT_THUNK(dx) INDIRECT_THUNK(si) INDIRECT_THUNK(di) INDIRECT_THUNK(bp) +asmlinkage void __fill_rsb(void); +asmlinkage void __clear_rsb(void); + #endif /* CONFIG_RETPOLINE */ diff --git a/arch/x86/include/asm/nospec-branch.h b/arch/x86/include/asm/nospec-branch.h index 865192a2cc31..4f88e1b2599f 100644 --- a/arch/x86/include/asm/nospec-branch.h +++ b/arch/x86/include/asm/nospec-branch.h @@ -27,30 +27,6 @@ #define RSB_CLEAR_LOOPS 32 /* To forcibly overwrite all entries */ #define RSB_FILL_LOOPS 16 /* To avoid underflow */ -/* - * Google experimented with loop-unrolling and this turned out to be - * the optimal version — two calls, each with their own speculation - * trap should their return address end up getting used, in a loop. - */ -#define __FILL_RETURN_BUFFER(reg, nr, sp) \ - mov $(nr/2), reg; \ -771: \ - call 772f; \ -773: /* speculation trap */ \ - pause; \ - lfence; \ - jmp 773b; \ -772: \ - call 774f; \ -775: /* speculation trap */ \ - pause; \ - lfence; \ - jmp 775b; \ -774: \ - dec reg; \ - jnz 771b; \ - add $(BITS_PER_LONG/8) * nr, sp; - #ifdef __ASSEMBLY__ /* @@ -121,17 +97,9 @@ #endif .endm - /* - * A simpler FILL_RETURN_BUFFER macro. Don't make people use the CPP - * monstrosity above, manually. - */ -.macro FILL_RETURN_BUFFER reg:req nr:req ftr:req +.macro FILL_RETURN_BUFFER nr:req ftr:req #ifdef CONFIG_RETPOLINE - ANNOTATE_NOSPEC_ALTERNATIVE - ALTERNATIVE "jmp .Lskip_rsb_\@", \ - __stringify(__FILL_RETURN_BUFFER(\reg,\nr,%_ASM_SP)) \ - \ftr -.Lskip_rsb_\@: + ALTERNATIVE "", "call __clear_rsb", \ftr #endif .endm @@ -206,15 +174,10 @@ extern char __indirect_thunk_end[]; static inline void vmexit_fill_RSB(void) { #ifdef CONFIG_RETPOLINE - unsigned long loops; - - asm volatile (ANNOTATE_NOSPEC_ALTERNATIVE - ALTERNATIVE("jmp 910f", - __stringify(__FILL_RETURN_BUFFER(%0, RSB_CLEAR_LOOPS, %1)), - X86_FEATURE_RETPOLINE) - "910:" - : "=r" (loops), ASM_CALL_CONSTRAINT - : : "memory" ); + alternative_input("", + "call __fill_rsb", + X86_FEATURE_RETPOLINE, + ASM_NO_INPUT_CLOBBER("memory")); #endif } diff --git a/arch/x86/lib/Makefile b/arch/x86/lib/Makefile index f23934bbaf4e..69a473919260 100644 --- a/arch/x86/lib/Makefile +++ b/arch/x86/lib/Makefile @@ -27,6 +27,7 @@ lib-$(CONFIG_RWSEM_XCHGADD_ALGORITHM) += rwsem.o lib-$(CONFIG_INSTRUCTION_DECODER) += insn.o inat.o insn-eval.o lib-$(CONFIG_RANDOMIZE_BASE) += kaslr.o lib-$(CONFIG_RETPOLINE) += retpoline.o +OBJECT_FILES_NON_STANDARD_retpoline.o :=y obj-y += msr.o msr-reg.o msr-reg-export.o hweight.o diff --git a/arch/x86/lib/retpoline.S b/arch/x86/lib/retpoline.S index c909961e678a..3dcabe2ea2d6 100644 --- a/arch/x86/lib/retpoline.S +++ b/arch/x86/lib/retpoline.S @@ -7,6 +7,7 @@ #include #include #include +#include .macro THUNK reg .section .text.__x86.indirect_thunk @@ -19,6 +20,37 @@ ENDPROC(__x86_indirect_thunk_\reg) .endm /* + * Google experimented with loop-unrolling and this turned out to be + * the optimal version — two calls, each with their own speculation + * trap should their return address end up getting used, in a loop. + */ +.macro BOINK_RSB nr:req sp:req + push %_ASM_AX + mov $(\nr / 2), %_ASM_AX + .align 16 +771: + call 772f +773: /* speculation trap */ + pause + lfence + jmp 773b + .align 16 +772: + call 774f +775: /* speculation trap */ + pause + lfence + jmp 775b + .align 16 +774: + dec %_ASM_AX + jnz 771b + add $((BITS_PER_LONG/8) * \nr), \sp + pop %_ASM_AX +.endm + + +/* * Despite being an assembler file we can't just use .irp here * because __KSYM_DEPS__ only uses the C preprocessor and would * only see one instance of "__x86_indirect_thunk_\reg" rather @@ -46,3 +78,15 @@ GENERATE_THUNK(r13) GENERATE_THUNK(r14) GENERATE_THUNK(r15) #endif + +ENTRY(__fill_rsb) + BOINK_RSB RSB_FILL_LOOPS, %_ASM_SP + ret +END(__fill_rsb) +EXPORT_SYMBOL_GPL(__fill_rsb) + +ENTRY(__clear_rsb) + BOINK_RSB RSB_CLEAR_LOOPS, %_ASM_SP + ret +END(__clear_rsb) +EXPORT_SYMBOL_GPL(__clear_rsb) -- 2.13.0 -- Regards/Gruss, Boris. Good mailing practices for 400: avoid top-posting and trim the reply.