[13/13] jump_label,x86: Allow short NOPs
diff mbox series

Message ID 20210506194158.216763632@infradead.org
State New, archived
Headers show
Series
  • jump_label: Yet another attempt at variable sized jump_labels
Related show

Commit Message

Peter Zijlstra May 6, 2021, 7:34 p.m. UTC
Now that objtool is able to rewrite jump_label instructions, have the
compiler emit a JMP, such that it can decide on the optimal encoding,
and set jump_entry::key bit1 to indicate that objtool should rewrite
the instruction to a matching NOP.

For x86_64-allyesconfig this gives:

jl\     NOP     JMP
short:  22997   124
long:   30874   90

IOW, we save (22997+124) * 3 bytes of kernel text in hotpaths.

Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
---
 arch/x86/include/asm/jump_label.h |   18 ++++++++++++++++++
 1 file changed, 18 insertions(+)

Comments

Peter Zijlstra May 6, 2021, 7:49 p.m. UTC | #1
On Thu, May 06, 2021 at 09:34:05PM +0200, Peter Zijlstra wrote:
> +#ifdef CONFIG_STACK_VALIDATION

Do we want something like this?

---
--- a/Documentation/x86/orc-unwinder.rst
+++ b/Documentation/x86/orc-unwinder.rst
@@ -15,7 +15,7 @@ the ORC unwinder to be much simpler and
 The ORC data consists of unwind tables which are generated by objtool.
 They contain out-of-band data which is used by the in-kernel ORC
 unwinder.  Objtool generates the ORC data by first doing compile-time
-stack metadata validation (CONFIG_STACK_VALIDATION).  After analyzing
+stack metadata validation (CONFIG_OBJTOOL).  After analyzing
 all the code paths of a .o file, it determines information about the
 stack state at each instruction address in the file and outputs that
 information to the .orc_unwind and .orc_unwind_ip sections.
--- a/Makefile
+++ b/Makefile
@@ -1091,12 +1091,12 @@ HOST_LIBELF_LIBS = $(shell pkg-config li
 has_libelf = $(call try-run,\
                echo "int main() {}" | $(HOSTCC) $(KBUILD_HOSTLDFLAGS) -xc -o /dev/null $(HOST_LIBELF_LIBS) -,1,0)
 
-ifdef CONFIG_STACK_VALIDATION
+ifdef CONFIG_OBJTOOL
   ifeq ($(has_libelf),1)
     objtool_target := tools/objtool FORCE
   else
-    SKIP_STACK_VALIDATION := 1
-    export SKIP_STACK_VALIDATION
+    SKIP_OBJTOOL := 1
+    export SKIP_OBJTOOL
   endif
 endif
 
@@ -1247,7 +1247,7 @@ asm-generic: uapi-asm-generic
 
 PHONY += prepare-objtool prepare-resolve_btfids
 prepare-objtool: $(objtool_target)
-ifeq ($(SKIP_STACK_VALIDATION),1)
+ifeq ($(SKIP_OBJTOOL),1)
 ifdef CONFIG_FTRACE_MCOUNT_USE_OBJTOOL
 	@echo "error: Cannot generate __mcount_loc for CONFIG_DYNAMIC_FTRACE=y, please install libelf-dev, libelf-devel or elfutils-libelf-devel" >&2
 	@false
@@ -1256,7 +1256,7 @@ ifdef CONFIG_UNWINDER_ORC
 	@echo "error: Cannot generate ORC metadata for CONFIG_UNWINDER_ORC=y, please install libelf-dev, libelf-devel or elfutils-libelf-devel" >&2
 	@false
 else
-	@echo "warning: Cannot use CONFIG_STACK_VALIDATION=y, please install libelf-dev, libelf-devel or elfutils-libelf-devel" >&2
+	@echo "warning: Cannot use CONFIG_OBJTOOL=y, please install libelf-dev, libelf-devel or elfutils-libelf-devel" >&2
 endif
 endif
 
--- a/arch/Kconfig
+++ b/arch/Kconfig
@@ -984,7 +984,7 @@ config ARCH_WANT_DEFAULT_TOPDOWN_MMAP_LA
 	depends on MMU
 	select ARCH_HAS_ELF_RANDOMIZE
 
-config HAVE_STACK_VALIDATION
+config HAVE_OBJTOOL
 	bool
 	help
 	  Architecture supports the 'objtool check' host tool command, which
--- a/arch/x86/Kconfig
+++ b/arch/x86/Kconfig
@@ -76,7 +76,7 @@ config X86
 	select ARCH_HAS_FILTER_PGPROT
 	select ARCH_HAS_FORTIFY_SOURCE
 	select ARCH_HAS_GCOV_PROFILE_ALL
-	select ARCH_HAS_KCOV			if X86_64 && STACK_VALIDATION
+	select ARCH_HAS_KCOV			if X86_64 && OBJTOOL
 	select ARCH_HAS_MEM_ENCRYPT
 	select ARCH_HAS_MEMBARRIER_SYNC_CORE
 	select ARCH_HAS_NON_OVERLAPPING_ADDRESS_SPACE
@@ -181,7 +181,7 @@ config X86
 	select HAVE_CONTEXT_TRACKING		if X86_64
 	select HAVE_CONTEXT_TRACKING_OFFSTACK	if HAVE_CONTEXT_TRACKING
 	select HAVE_C_RECORDMCOUNT
-	select HAVE_OBJTOOL_MCOUNT		if STACK_VALIDATION
+	select HAVE_OBJTOOL_MCOUNT		if OBJTOOL
 	select HAVE_DEBUG_KMEMLEAK
 	select HAVE_DMA_CONTIGUOUS
 	select HAVE_DYNAMIC_FTRACE
@@ -232,13 +232,13 @@ config X86
 	select MMU_GATHER_RCU_TABLE_FREE		if PARAVIRT
 	select HAVE_POSIX_CPU_TIMERS_TASK_WORK
 	select HAVE_REGS_AND_STACK_ACCESS_API
-	select HAVE_RELIABLE_STACKTRACE		if X86_64 && (UNWINDER_FRAME_POINTER || UNWINDER_ORC) && STACK_VALIDATION
+	select HAVE_RELIABLE_STACKTRACE		if X86_64 && (UNWINDER_FRAME_POINTER || UNWINDER_ORC) && OBJTOOL
 	select HAVE_FUNCTION_ARG_ACCESS_API
 	select HAVE_SOFTIRQ_ON_OWN_STACK
 	select HAVE_STACKPROTECTOR		if CC_HAS_SANE_STACKPROTECTOR
-	select HAVE_STACK_VALIDATION		if X86_64
+	select HAVE_OBJTOOL		if X86_64
 	select HAVE_STATIC_CALL
-	select HAVE_STATIC_CALL_INLINE		if HAVE_STACK_VALIDATION
+	select HAVE_STATIC_CALL_INLINE		if HAVE_OBJTOOL
 	select HAVE_PREEMPT_DYNAMIC
 	select HAVE_RSEQ
 	select HAVE_SYSCALL_TRACEPOINTS
@@ -255,7 +255,7 @@ config X86
 	select RTC_MC146818_LIB
 	select SPARSE_IRQ
 	select SRCU
-	select STACK_VALIDATION			if HAVE_STACK_VALIDATION && (HAVE_STATIC_CALL_INLINE || RETPOLINE)
+	select OBJTOOL			if HAVE_OBJTOOL && (HAVE_STATIC_CALL_INLINE || RETPOLINE)
 	select SYSCTL_EXCEPTION_TRACE
 	select THREAD_INFO_IN_TASK
 	select USER_STACKTRACE_SUPPORT
--- a/arch/x86/Kconfig.debug
+++ b/arch/x86/Kconfig.debug
@@ -240,7 +240,7 @@ choice
 config UNWINDER_ORC
 	bool "ORC unwinder"
 	depends on X86_64
-	select STACK_VALIDATION
+	select OBJTOOL
 	help
 	  This option enables the ORC (Oops Rewind Capability) unwinder for
 	  unwinding kernel stack traces.  It uses a custom data format which is
--- a/arch/x86/include/asm/jump_label.h
+++ b/arch/x86/include/asm/jump_label.h
@@ -20,7 +20,7 @@
 	_ASM_PTR "%c0 + %c1 - .\n\t"			\
 	".popsection \n\t"
 
-#ifdef CONFIG_STACK_VALIDATION
+#ifdef CONFIG_OBJTOOL
 
 static __always_inline bool arch_static_branch(struct static_key *key, bool branch)
 {
@@ -48,7 +48,7 @@ static __always_inline bool arch_static_
 	return true;
 }
 
-#endif /* STACK_VALIDATION */
+#endif /* OBJTOOL */
 
 static __always_inline bool arch_static_branch_jump(struct static_key * const key, const bool branch)
 {
--- a/include/linux/compiler.h
+++ b/include/linux/compiler.h
@@ -109,7 +109,7 @@ void ftrace_likely_update(struct ftrace_
 #endif
 
 /* Unreachable code */
-#ifdef CONFIG_STACK_VALIDATION
+#ifdef CONFIG_OBJTOOL
 /*
  * These macros help objtool understand GCC code flow for unreachable code.
  * The __COUNTER__ based labels are a hack to make each instance of the macros
--- a/include/linux/instrumentation.h
+++ b/include/linux/instrumentation.h
@@ -2,7 +2,7 @@
 #ifndef __LINUX_INSTRUMENTATION_H
 #define __LINUX_INSTRUMENTATION_H
 
-#if defined(CONFIG_DEBUG_ENTRY) && defined(CONFIG_STACK_VALIDATION)
+#if defined(CONFIG_DEBUG_ENTRY) && defined(CONFIG_OBJTOOL)
 
 /* Begin/end of an instrumentation safe region */
 #define instrumentation_begin() ({					\
--- a/include/linux/objtool.h
+++ b/include/linux/objtool.h
@@ -38,7 +38,7 @@ struct unwind_hint {
 #define UNWIND_HINT_TYPE_REGS_PARTIAL	2
 #define UNWIND_HINT_TYPE_FUNC		3
 
-#ifdef CONFIG_STACK_VALIDATION
+#ifdef CONFIG_OBJTOOL
 
 #ifndef __ASSEMBLY__
 
@@ -120,7 +120,7 @@ struct unwind_hint {
 
 #endif /* __ASSEMBLY__ */
 
-#else /* !CONFIG_STACK_VALIDATION */
+#else /* !CONFIG_OBJTOOL */
 
 #ifndef __ASSEMBLY__
 
@@ -135,6 +135,6 @@ struct unwind_hint {
 .endm
 #endif
 
-#endif /* CONFIG_STACK_VALIDATION */
+#endif /* CONFIG_OBJTOOL */
 
 #endif /* _LINUX_OBJTOOL_H */
--- a/lib/Kconfig.debug
+++ b/lib/Kconfig.debug
@@ -428,9 +428,9 @@ config FRAME_POINTER
 	  larger and slower, but it gives very useful debugging information
 	  in case of kernel bugs. (precise oopses/stacktraces/warnings)
 
-config STACK_VALIDATION
+config OBJTOOL
 	bool "Compile-time stack metadata validation"
-	depends on HAVE_STACK_VALIDATION
+	depends on HAVE_OBJTOOL
 	default n
 	help
 	  Add compile-time checks to validate stack metadata, including frame
@@ -445,7 +445,7 @@ config STACK_VALIDATION
 
 config VMLINUX_VALIDATION
 	bool
-	depends on STACK_VALIDATION && DEBUG_ENTRY && !PARAVIRT
+	depends on OBJTOOL && DEBUG_ENTRY && !PARAVIRT
 	default y
 
 config VMLINUX_MAP
--- a/lib/Kconfig.ubsan
+++ b/lib/Kconfig.ubsan
@@ -106,7 +106,7 @@ config UBSAN_UNREACHABLE
 	bool "Perform checking for unreachable code"
 	# objtool already handles unreachable checking and gets angry about
 	# seeing UBSan instrumentation located in unreachable places.
-	depends on !STACK_VALIDATION
+	depends on !OBJTOOL
 	depends on $(cc-option,-fsanitize=unreachable)
 	help
 	  This option enables -fsanitize=unreachable which checks for control
--- a/scripts/Makefile.build
+++ b/scripts/Makefile.build
@@ -217,9 +217,9 @@ cmd_record_mcount = $(if $(findstring $(
 	$(sub_cmd_record_mcount))
 endif # CONFIG_FTRACE_MCOUNT_USE_RECORDMCOUNT
 
-ifdef CONFIG_STACK_VALIDATION
+ifdef CONFIG_OBJTOOL
 ifndef CONFIG_LTO_CLANG
-ifneq ($(SKIP_STACK_VALIDATION),1)
+ifneq ($(SKIP_OBJTOOL),1)
 
 __objtool_obj := $(objtree)/tools/objtool/objtool
 
@@ -233,14 +233,14 @@ objtool_obj = $(if $(patsubst y%,, \
 	$(OBJECT_FILES_NON_STANDARD_$(basetarget).o)$(OBJECT_FILES_NON_STANDARD)n), \
 	$(__objtool_obj))
 
-endif # SKIP_STACK_VALIDATION
+endif # SKIP_OBJTOOL
 endif # CONFIG_LTO_CLANG
-endif # CONFIG_STACK_VALIDATION
+endif # CONFIG_OBJTOOL
 
 # Rebuild all objects when objtool changes, or is enabled/disabled.
 objtool_dep = $(objtool_obj)					\
 	      $(wildcard include/config/ORC_UNWINDER		\
-			 include/config/STACK_VALIDATION)
+			 include/config/OBJTOOL)
 
 ifdef CONFIG_TRIM_UNUSED_KSYMS
 cmd_gen_ksymdeps = \
--- a/scripts/Makefile.modfinal
+++ b/scripts/Makefile.modfinal
@@ -38,14 +38,14 @@ prelink-ext := .lto
 # ELF processing was skipped earlier because we didn't have native code,
 # so let's now process the prelinked binary before we link the module.
 
-ifdef CONFIG_STACK_VALIDATION
-ifneq ($(SKIP_STACK_VALIDATION),1)
+ifdef CONFIG_OBJTOOL
+ifneq ($(SKIP_OBJTOOL),1)
 cmd_ld_ko_o +=								\
 	$(objtree)/tools/objtool/objtool $(objtool_args)		\
 		$(@:.ko=$(prelink-ext).o);
 
-endif # SKIP_STACK_VALIDATION
-endif # CONFIG_STACK_VALIDATION
+endif # SKIP_OBJTOOL
+endif # CONFIG_OBJTOOL
 
 endif # CONFIG_LTO_CLANG
 
--- a/scripts/link-vmlinux.sh
+++ b/scripts/link-vmlinux.sh
@@ -106,7 +106,7 @@ objtool_link()
 	local objtoolcmd;
 	local objtoolopt;
 
-	if [ "${CONFIG_LTO_CLANG} ${CONFIG_STACK_VALIDATION}" = "y y" ]; then
+	if [ "${CONFIG_LTO_CLANG} ${CONFIG_OBJTOOL}" = "y y" ]; then
 		# Don't perform vmlinux validation unless explicitly requested,
 		# but run objtool on vmlinux.o now that we have an object file.
 		if [ -n "${CONFIG_UNWINDER_ORC}" ]; then
--- a/scripts/package/builddeb
+++ b/scripts/package/builddeb
@@ -67,7 +67,7 @@ deploy_kernel_headers () {
 	) > debian/hdrsrcfiles
 
 	{
-		if is_enabled CONFIG_STACK_VALIDATION; then
+		if is_enabled CONFIG_OBJTOOL; then
 			echo tools/objtool/objtool
 		fi
 
--- a/tools/include/linux/objtool.h
+++ b/tools/include/linux/objtool.h
@@ -38,7 +38,7 @@ struct unwind_hint {
 #define UNWIND_HINT_TYPE_REGS_PARTIAL	2
 #define UNWIND_HINT_TYPE_FUNC		3
 
-#ifdef CONFIG_STACK_VALIDATION
+#ifdef CONFIG_OBJTOOL
 
 #ifndef __ASSEMBLY__
 
@@ -120,7 +120,7 @@ struct unwind_hint {
 
 #endif /* __ASSEMBLY__ */
 
-#else /* !CONFIG_STACK_VALIDATION */
+#else /* !CONFIG_OBJTOOL */
 
 #ifndef __ASSEMBLY__
 
@@ -135,6 +135,6 @@ struct unwind_hint {
 .endm
 #endif
 
-#endif /* CONFIG_STACK_VALIDATION */
+#endif /* CONFIG_OBJTOOL */
 
 #endif /* _LINUX_OBJTOOL_H */
--- a/tools/objtool/Documentation/stack-validation.txt
+++ b/tools/objtool/Documentation/stack-validation.txt
@@ -5,7 +5,7 @@ Compile-time stack metadata validation
 Overview
 --------
 
-The kernel CONFIG_STACK_VALIDATION option enables a host tool named
+The kernel CONFIG_OBJTOOL option enables a host tool named
 objtool which runs at compile time.  It has a "check" subcommand which
 analyzes every .o file and ensures the validity of its stack metadata.
 It enforces a set of rules on asm code and C inline assembly code so
--- a/tools/testing/selftests/wireguard/qemu/debug.config
+++ b/tools/testing/selftests/wireguard/qemu/debug.config
@@ -1,6 +1,6 @@
 CONFIG_LOCALVERSION="-debug"
 CONFIG_FRAME_POINTER=y
-CONFIG_STACK_VALIDATION=y
+CONFIG_OBJTOOL=y
 CONFIG_DEBUG_KERNEL=y
 CONFIG_DEBUG_INFO=y
 CONFIG_DEBUG_INFO_DWARF4=y

Patch
diff mbox series

--- a/arch/x86/include/asm/jump_label.h
+++ b/arch/x86/include/asm/jump_label.h
@@ -28,6 +28,22 @@ 
 	_ASM_PTR "%c0 + %c1 - .\n\t"			\
 	".popsection \n\t"
 
+#ifdef CONFIG_STACK_VALIDATION
+
+static __always_inline bool arch_static_branch(struct static_key *key, bool branch)
+{
+	asm_volatile_goto("1:"
+		"jmp %l[l_yes] # objtool NOPs this \n\t"
+		JUMP_TABLE_ENTRY
+		: :  "i" (key), "i" (2 | branch) : : l_yes);
+
+	return false;
+l_yes:
+	return true;
+}
+
+#else
+
 static __always_inline bool arch_static_branch(struct static_key * const key, const bool branch)
 {
 	asm_volatile_goto("1:"
@@ -40,6 +56,8 @@  static __always_inline bool arch_static_
 	return true;
 }
 
+#endif /* STACK_VALIDATION */
+
 static __always_inline bool arch_static_branch_jump(struct static_key * const key, const bool branch)
 {
 	asm_volatile_goto("1:"