linux-kernel.vger.kernel.org archive mirror
 help / color / mirror / Atom feed
From: Nadav Amit <namit@vmware.com>
To: Ingo Molnar <mingo@redhat.com>
Cc: <linux-kernel@vger.kernel.org>, <x86@kernel.org>,
	Nadav Amit <namit@vmware.com>,
	Andy Lutomirski <luto@amacapital.net>,
	Borislav Petkov <bp@alien8.de>, Brian Gerst <brgerst@gmail.com>,
	Denys Vlasenko <dvlasenk@redhat.com>,
	"H . Peter Anvin" <hpa@zytor.com>,
	Linus Torvalds <torvalds@linux-foundation.org>,
	Peter Zijlstra <peterz@infradead.org>,
	Thomas Gleixner <tglx@linutronix.de>,
	Ingo Molnar <mingo@kernel.org>
Subject: [PATCH 2/3] x86/cpufeature: Macrofy inline assembly code to work around GCC inlining bugs
Date: Fri, 5 Oct 2018 13:27:17 -0700	[thread overview]
Message-ID: <20181005202718.229565-3-namit@vmware.com> (raw)
In-Reply-To: <20181005202718.229565-1-namit@vmware.com>

As described in:

77b0bf55bc67: ("kbuild/Makefile: Prepare for using macros in inline assembly code to work around asm() related GCC inlining bugs")

GCC's inlining heuristics are broken with common asm() patterns used in
kernel code, resulting in the effective disabling of inlining.

The workaround is to set an assembly macro and call it from the inline
assembly block - which is pretty pointless indirection in the static_cpu_has()
case, but is worth it to improve overall inlining quality.

The patch slightly increases the kernel size:

 text     data     bss      dec     hex  filename
18162879 10226256 2957312 31346447 1de4f0f  ./vmlinux before
18163528 10226300 2957312 31347140 1de51c4  ./vmlinux after (+693)

And enables the inlining of function such as free_ldt_pgtables().

Tested-by: Kees Cook <keescook@chromium.org>
Signed-off-by: Nadav Amit <namit@vmware.com>
Acked-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Cc: Andy Lutomirski <luto@amacapital.net>
Cc: Borislav Petkov <bp@alien8.de>
Cc: Brian Gerst <brgerst@gmail.com>
Cc: Denys Vlasenko <dvlasenk@redhat.com>
Cc: H. Peter Anvin <hpa@zytor.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Link: https://lore.kernel.org/lkml/20181003213100.189959-10-namit@vmware.com/T/#u
Signed-off-by: Ingo Molnar <mingo@kernel.org>
---
 arch/x86/include/asm/cpufeature.h | 82 ++++++++++++++++++-------------
 arch/x86/kernel/macros.S          |  1 +
 2 files changed, 48 insertions(+), 35 deletions(-)

diff --git a/arch/x86/include/asm/cpufeature.h b/arch/x86/include/asm/cpufeature.h
index aced6c9290d6..7d442722ef24 100644
--- a/arch/x86/include/asm/cpufeature.h
+++ b/arch/x86/include/asm/cpufeature.h
@@ -2,10 +2,10 @@
 #ifndef _ASM_X86_CPUFEATURE_H
 #define _ASM_X86_CPUFEATURE_H
 
-#include <asm/processor.h>
-
-#if defined(__KERNEL__) && !defined(__ASSEMBLY__)
+#ifdef __KERNEL__
+#ifndef __ASSEMBLY__
 
+#include <asm/processor.h>
 #include <asm/asm.h>
 #include <linux/bitops.h>
 
@@ -161,37 +161,10 @@ extern void clear_cpu_cap(struct cpuinfo_x86 *c, unsigned int bit);
  */
 static __always_inline __pure bool _static_cpu_has(u16 bit)
 {
-	asm_volatile_goto("1: jmp 6f\n"
-		 "2:\n"
-		 ".skip -(((5f-4f) - (2b-1b)) > 0) * "
-			 "((5f-4f) - (2b-1b)),0x90\n"
-		 "3:\n"
-		 ".section .altinstructions,\"a\"\n"
-		 " .long 1b - .\n"		/* src offset */
-		 " .long 4f - .\n"		/* repl offset */
-		 " .word %P[always]\n"		/* always replace */
-		 " .byte 3b - 1b\n"		/* src len */
-		 " .byte 5f - 4f\n"		/* repl len */
-		 " .byte 3b - 2b\n"		/* pad len */
-		 ".previous\n"
-		 ".section .altinstr_replacement,\"ax\"\n"
-		 "4: jmp %l[t_no]\n"
-		 "5:\n"
-		 ".previous\n"
-		 ".section .altinstructions,\"a\"\n"
-		 " .long 1b - .\n"		/* src offset */
-		 " .long 0\n"			/* no replacement */
-		 " .word %P[feature]\n"		/* feature bit */
-		 " .byte 3b - 1b\n"		/* src len */
-		 " .byte 0\n"			/* repl len */
-		 " .byte 0\n"			/* pad len */
-		 ".previous\n"
-		 ".section .altinstr_aux,\"ax\"\n"
-		 "6:\n"
-		 " testb %[bitnum],%[cap_byte]\n"
-		 " jnz %l[t_yes]\n"
-		 " jmp %l[t_no]\n"
-		 ".previous\n"
+	asm_volatile_goto("STATIC_CPU_HAS bitnum=%[bitnum] "
+			  "cap_byte=\"%[cap_byte]\" "
+			  "feature=%P[feature] t_yes=%l[t_yes] "
+			  "t_no=%l[t_no] always=%P[always]"
 		 : : [feature]  "i" (bit),
 		     [always]   "i" (X86_FEATURE_ALWAYS),
 		     [bitnum]   "i" (1 << (bit & 7)),
@@ -226,5 +199,44 @@ static __always_inline __pure bool _static_cpu_has(u16 bit)
 #define CPU_FEATURE_TYPEVAL		boot_cpu_data.x86_vendor, boot_cpu_data.x86, \
 					boot_cpu_data.x86_model
 
-#endif /* defined(__KERNEL__) && !defined(__ASSEMBLY__) */
+#else /* __ASSEMBLY__ */
+
+.macro STATIC_CPU_HAS bitnum:req cap_byte:req feature:req t_yes:req t_no:req always:req
+1:
+	jmp 6f
+2:
+	.skip -(((5f-4f) - (2b-1b)) > 0) * ((5f-4f) - (2b-1b)),0x90
+3:
+	.section .altinstructions,"a"
+	.long 1b - .		/* src offset */
+	.long 4f - .		/* repl offset */
+	.word \always		/* always replace */
+	.byte 3b - 1b		/* src len */
+	.byte 5f - 4f		/* repl len */
+	.byte 3b - 2b		/* pad len */
+	.previous
+	.section .altinstr_replacement,"ax"
+4:
+	jmp \t_no
+5:
+	.previous
+	.section .altinstructions,"a"
+	.long 1b - .		/* src offset */
+	.long 0			/* no replacement */
+	.word \feature		/* feature bit */
+	.byte 3b - 1b		/* src len */
+	.byte 0			/* repl len */
+	.byte 0			/* pad len */
+	.previous
+	.section .altinstr_aux,"ax"
+6:
+	testb \bitnum,\cap_byte
+	jnz \t_yes
+	jmp \t_no
+	.previous
+.endm
+
+#endif /* __ASSEMBLY__ */
+
+#endif /* __KERNEL__ */
 #endif /* _ASM_X86_CPUFEATURE_H */
diff --git a/arch/x86/kernel/macros.S b/arch/x86/kernel/macros.S
index 7baa40d5bf16..bf8b9c93e255 100644
--- a/arch/x86/kernel/macros.S
+++ b/arch/x86/kernel/macros.S
@@ -12,3 +12,4 @@
 #include <asm/bug.h>
 #include <asm/paravirt.h>
 #include <asm/asm.h>
+#include <asm/cpufeature.h>
-- 
2.17.1


  parent reply	other threads:[~2018-10-05 20:29 UTC|newest]

Thread overview: 116+ messages / expand[flat|nested]  mbox.gz  Atom feed  top
2018-10-03 21:30 [PATCH v9 00/10] x86: macrofying inline asm Nadav Amit
2018-10-03 21:30 ` [PATCH v9 01/10] xtensa: defining LINKER_SCRIPT for the linker script Nadav Amit
2018-10-04 10:00   ` [tip:x86/build] kbuild/arch/xtensa: Define " tip-bot for Nadav Amit
2018-10-03 21:30 ` [PATCH v9 02/10] Makefile: Prepare for using macros for inline asm Nadav Amit
2018-10-04 10:01   ` [tip:x86/build] kbuild/Makefile: Prepare for using macros in inline assembly code to work around asm() related GCC inlining bugs tip-bot for Nadav Amit
2018-11-06 18:57   ` [PATCH v9 02/10] Makefile: Prepare for using macros for inline asm Logan Gunthorpe
2018-11-06 19:18     ` Nadav Amit
2018-11-06 20:01       ` Logan Gunthorpe
2018-11-07 18:01         ` Nadav Amit
2018-11-07 18:53           ` Logan Gunthorpe
2018-11-07 18:56             ` Nadav Amit
2018-11-07 21:43               ` Logan Gunthorpe
2018-11-07 21:50                 ` hpa
2018-11-08  6:18                   ` Nadav Amit
2018-11-08 17:14                     ` Logan Gunthorpe
2018-11-08 19:54                       ` Nadav Amit
2018-11-08 20:00                         ` Logan Gunthorpe
2018-11-08 20:18                           ` Nadav Amit
2018-11-10 22:04                             ` Nadav Amit
2018-11-13  4:56                               ` Logan Gunthorpe
2018-10-03 21:30 ` [PATCH v9 03/10] x86: objtool: use asm macro for better compiler decisions Nadav Amit
2018-10-04 10:02   ` [tip:x86/build] x86/objtool: Use asm macros to work around GCC inlining bugs tip-bot for Nadav Amit
2018-10-03 21:30 ` [PATCH v9 04/10] x86: refcount: prevent gcc distortions Nadav Amit
2018-10-04  7:57   ` Ingo Molnar
2018-10-04  8:33     ` Ingo Molnar
2018-10-04  8:40       ` hpa
2018-10-04  8:56         ` Ingo Molnar
2018-10-04  8:56         ` Nadav Amit
2018-10-04  9:02           ` hpa
2018-10-04  9:16             ` Ingo Molnar
2018-10-04 19:33               ` H. Peter Anvin
2018-10-04 20:05                 ` Nadav Amit
2018-10-04 20:08                   ` H. Peter Anvin
2018-10-04 20:29                 ` Andy Lutomirski
2018-10-04 23:11                   ` H. Peter Anvin
2018-10-06  1:40                 ` Rasmus Villemoes
2018-10-04  9:12           ` Ingo Molnar
2018-10-04  9:17             ` hpa
2018-10-04  9:30             ` Nadav Amit
2018-10-04  9:45               ` Ingo Molnar
2018-10-04 10:23                 ` Nadav Amit
2018-10-05  9:31                   ` Ingo Molnar
2018-10-05 11:20                     ` Borislav Petkov
2018-10-05 12:52                       ` Ingo Molnar
2018-10-05 20:27                     ` [PATCH 0/3] Macrofying inline asm rebased Nadav Amit
2018-10-05 20:27                       ` [PATCH 1/3] x86/extable: Macrofy inline assembly code to work around GCC inlining bugs Nadav Amit
2018-10-06 14:42                         ` [tip:x86/build] " tip-bot for Nadav Amit
2018-10-05 20:27                       ` Nadav Amit [this message]
2018-10-06 14:43                         ` [tip:x86/build] x86/cpufeature: " tip-bot for Nadav Amit
2018-10-05 20:27                       ` [PATCH 3/3] x86/jump-labels: " Nadav Amit
2018-10-06 14:44                         ` [tip:x86/build] " tip-bot for Nadav Amit
2018-10-08  2:17                     ` [PATCH v9 04/10] x86: refcount: prevent gcc distortions Nadav Amit
2018-10-04  8:40     ` Nadav Amit
2018-10-04  9:01       ` Ingo Molnar
2018-10-04 10:02   ` [tip:x86/build] x86/refcount: Work around GCC inlining bug tip-bot for Nadav Amit
2018-10-03 21:30 ` [PATCH v9 05/10] x86: alternatives: macrofy locks for better inlining Nadav Amit
2018-10-04 10:03   ` [tip:x86/build] x86/alternatives: Macrofy lock prefixes to work around GCC inlining bugs tip-bot for Nadav Amit
2018-10-03 21:30 ` [PATCH v9 06/10] x86: bug: prevent gcc distortions Nadav Amit
2018-10-04 10:03   ` [tip:x86/build] x86/bug: Macrofy the BUG table section handling, to work around GCC inlining bugs tip-bot for Nadav Amit
2018-10-03 21:30 ` [PATCH v9 07/10] x86: prevent inline distortion by paravirt ops Nadav Amit
2018-10-04 10:04   ` [tip:x86/build] x86/paravirt: Work around GCC inlining bugs when compiling " tip-bot for Nadav Amit
2018-10-03 21:30 ` [PATCH v9 08/10] x86: extable: use macros instead of inline assembly Nadav Amit
2018-10-03 21:30 ` [PATCH v9 09/10] x86: cpufeature: " Nadav Amit
2018-10-03 21:31 ` [PATCH v9 10/10] x86: jump-labels: " Nadav Amit
2018-10-07  9:18 ` PROPOSAL: Extend inline asm syntax with size spec Borislav Petkov
     [not found]   ` <20181007132228.GJ29268@gate.crashing.org>
2018-10-07 14:13     ` Borislav Petkov
2018-10-07 15:14       ` Segher Boessenkool
2018-10-08  5:58         ` Ingo Molnar
2018-10-08  7:53           ` Segher Boessenkool
2018-10-07 15:53     ` Michael Matz
2018-10-08  6:13       ` Ingo Molnar
2018-10-08  8:18         ` Segher Boessenkool
2018-10-08  7:31       ` Segher Boessenkool
2018-10-08  9:07         ` Richard Biener
2018-10-08 10:02           ` Segher Boessenkool
2018-10-09 14:53           ` Segher Boessenkool
2018-10-10  6:35             ` Ingo Molnar
2018-10-10  7:12             ` Richard Biener
2018-10-10  7:22               ` Ingo Molnar
2018-10-10  8:03                 ` Segher Boessenkool
2018-10-10  8:19                   ` Borislav Petkov
2018-10-10  8:35                     ` Richard Biener
2018-10-10 18:54                     ` Segher Boessenkool
2018-10-10 19:14                       ` Borislav Petkov
2018-10-13 19:33                         ` Borislav Petkov
2018-10-13 21:14                           ` Alexander Monakov
2018-10-13 21:30                             ` Borislav Petkov
2018-10-25 10:24                           ` Borislav Petkov
2018-10-31 12:55                           ` Peter Zijlstra
2018-10-31 13:11                             ` Peter Zijlstra
2018-10-31 16:31                             ` Segher Boessenkool
2018-11-01  5:20                             ` Joe Perches
2018-11-01  9:01                               ` Peter Zijlstra
2018-11-01  9:20                                 ` Joe Perches
2018-11-01 11:15                                   ` Peter Zijlstra
2018-12-27  4:47                             ` Masahiro Yamada
2018-10-10 10:29                   ` Richard Biener
2018-10-10  7:53               ` Segher Boessenkool
2018-10-10 16:31             ` Nadav Amit
2018-10-10 19:21               ` Segher Boessenkool
2018-10-11  7:04               ` Richard Biener
2018-11-29 11:46             ` Masahiro Yamada
2018-11-29 12:25               ` Segher Boessenkool
2018-11-30  9:06                 ` Boris Petkov
2018-11-30 13:16                   ` Segher Boessenkool
2018-12-10  8:16                     ` Masahiro Yamada
2018-11-29 13:07               ` Borislav Petkov
2018-11-29 13:09                 ` Richard Biener
2018-11-29 13:16                   ` Borislav Petkov
2018-11-29 13:24                     ` Richard Biener
2018-10-08 16:24       ` David Laight
2018-10-07 16:09   ` Nadav Amit
2018-10-07 16:46     ` Richard Biener
2018-10-07 19:06       ` Nadav Amit
2018-10-07 19:52         ` Jeff Law
2018-10-08  7:46         ` Richard Biener

Reply instructions:

You may reply publicly to this message via plain-text email
using any one of the following methods:

* Save the following mbox file, import it into your mail client,
  and reply-to-all from there: mbox

  Avoid top-posting and favor interleaved quoting:
  https://en.wikipedia.org/wiki/Posting_style#Interleaved_style

* Reply using the --to, --cc, and --in-reply-to
  switches of git-send-email(1):

  git send-email \
    --in-reply-to=20181005202718.229565-3-namit@vmware.com \
    --to=namit@vmware.com \
    --cc=bp@alien8.de \
    --cc=brgerst@gmail.com \
    --cc=dvlasenk@redhat.com \
    --cc=hpa@zytor.com \
    --cc=linux-kernel@vger.kernel.org \
    --cc=luto@amacapital.net \
    --cc=mingo@kernel.org \
    --cc=mingo@redhat.com \
    --cc=peterz@infradead.org \
    --cc=tglx@linutronix.de \
    --cc=torvalds@linux-foundation.org \
    --cc=x86@kernel.org \
    /path/to/YOUR_REPLY

  https://kernel.org/pub/software/scm/git/docs/git-send-email.html

* If your mail client supports setting the In-Reply-To header
  via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).