[PATCH 1/5] x86: change THREAD_INFO definition to not depend on KERNEL_STACK_OFFSET

[PATCH 1/5] x86: change THREAD_INFO definition to not depend on KERNEL_STACK_OFFSET

[Denys Vlasenko]

This changes THREAD_INFO definition and all its callsites
so that they do not count stack position from
(top of stack - KERNEL_STACK_OFFSET), but from top of stack.

Semi-mysterious expressions THREAD_INFO(%rsp,RIP) - "why RIP??"
are now replaced by more logical THREAD_INFO(%rsp,SIZEOF_PTREGS) -
"calculate thread_info's address using information that
rsp is SIZEOF_PTREGS bytes below top of stack".

While at it, replace "(off)-THREAD_SIZE(reg)" with equivalent
"((off)-THREAD_SIZE)(reg)". The form without parentheses
falsely looks like we invoke THREAD_SIZE() macro.

Improve comment atop THREAD_INFO macro definition.

This patch does not change generated code (verified by objdump).

Signed-off-by: Denys Vlasenko <dvlasenk@redhat.com>
CC: Linus Torvalds <torvalds@linux-foundation.org>
CC: Steven Rostedt <rostedt@goodmis.org>
CC: Ingo Molnar <mingo@kernel.org>
CC: Borislav Petkov <bp@alien8.de>
CC: "H. Peter Anvin" <hpa@zytor.com>
CC: Andy Lutomirski <luto@amacapital.net>
CC: Oleg Nesterov <oleg@redhat.com>
CC: Frederic Weisbecker <fweisbec@gmail.com>
CC: Alexei Starovoitov <ast@plumgrid.com>
CC: Will Drewry <wad@chromium.org>
CC: Kees Cook <keescook@chromium.org>
CC: x86@kernel.org
CC: linux-kernel@vger.kernel.org
---

Changes since last version:
* split this change into a separate patch
* on Borislav's request, improve comment atop THREAD_INFO definition.

 arch/x86/ia32/ia32entry.S          | 30 +++++++++++++++---------------
 arch/x86/include/asm/thread_info.h |  8 +++++---
 arch/x86/kernel/entry_64.S         |  4 ++--
 3 files changed, 22 insertions(+), 20 deletions(-)

diff --git a/arch/x86/ia32/ia32entry.S b/arch/x86/ia32/ia32entry.S
index ad9efef..50190e1 100644
--- a/arch/x86/ia32/ia32entry.S
+++ b/arch/x86/ia32/ia32entry.S
@@ -127,7 +127,7 @@ ENTRY(ia32_sysenter_target)
 	CFI_REL_OFFSET rsp,0
 	pushfq_cfi
 	/*CFI_REL_OFFSET rflags,0*/
-	movl	TI_sysenter_return+THREAD_INFO(%rsp,3*8-KERNEL_STACK_OFFSET),%r10d
+	movl	TI_sysenter_return+THREAD_INFO(%rsp,3*8),%r10d
 	CFI_REGISTER rip,r10
 	pushq_cfi $__USER32_CS
 	/*CFI_REL_OFFSET cs,0*/
@@ -159,8 +159,8 @@ ENTRY(ia32_sysenter_target)
 	jnz sysenter_fix_flags
 sysenter_flags_fixed:
 
-	orl     $TS_COMPAT,TI_status+THREAD_INFO(%rsp,RIP)
-	testl   $_TIF_WORK_SYSCALL_ENTRY,TI_flags+THREAD_INFO(%rsp,RIP)
+	orl     $TS_COMPAT,TI_status+THREAD_INFO(%rsp,SIZEOF_PTREGS)
+	testl   $_TIF_WORK_SYSCALL_ENTRY,TI_flags+THREAD_INFO(%rsp,SIZEOF_PTREGS)
 	CFI_REMEMBER_STATE
 	jnz  sysenter_tracesys
 	cmpq	$(IA32_NR_syscalls-1),%rax
@@ -177,10 +177,10 @@ sysenter_dispatch:
 	movq	%rax,RAX(%rsp)
 	DISABLE_INTERRUPTS(CLBR_NONE)
 	TRACE_IRQS_OFF
-	testl	$_TIF_ALLWORK_MASK,TI_flags+THREAD_INFO(%rsp,RIP)
+	testl	$_TIF_ALLWORK_MASK,TI_flags+THREAD_INFO(%rsp,SIZEOF_PTREGS)
 	jnz	sysexit_audit
 sysexit_from_sys_call:
-	andl    $~TS_COMPAT,TI_status+THREAD_INFO(%rsp,RIP)
+	andl    $~TS_COMPAT,TI_status+THREAD_INFO(%rsp,SIZEOF_PTREGS)
 	/* clear IF, that popfq doesn't enable interrupts early */
 	andl	$~0x200,EFLAGS(%rsp)
 	movl	RIP(%rsp),%edx		/* User %eip */
@@ -225,7 +225,7 @@ sysexit_from_sys_call:
 	.endm
 
 	.macro auditsys_exit exit
-	testl $(_TIF_ALLWORK_MASK & ~_TIF_SYSCALL_AUDIT),TI_flags+THREAD_INFO(%rsp,RIP)
+	testl $(_TIF_ALLWORK_MASK & ~_TIF_SYSCALL_AUDIT),TI_flags+THREAD_INFO(%rsp,SIZEOF_PTREGS)
 	jnz ia32_ret_from_sys_call
 	TRACE_IRQS_ON
 	ENABLE_INTERRUPTS(CLBR_NONE)
@@ -240,7 +240,7 @@ sysexit_from_sys_call:
 	movl $(_TIF_ALLWORK_MASK & ~_TIF_SYSCALL_AUDIT),%edi
 	DISABLE_INTERRUPTS(CLBR_NONE)
 	TRACE_IRQS_OFF
-	testl %edi,TI_flags+THREAD_INFO(%rsp,RIP)
+	testl %edi,TI_flags+THREAD_INFO(%rsp,SIZEOF_PTREGS)
 	jz \exit
 	CLEAR_RREGS
 	jmp int_with_check
@@ -262,7 +262,7 @@ sysenter_fix_flags:
 
 sysenter_tracesys:
 #ifdef CONFIG_AUDITSYSCALL
-	testl	$(_TIF_WORK_SYSCALL_ENTRY & ~_TIF_SYSCALL_AUDIT),TI_flags+THREAD_INFO(%rsp,RIP)
+	testl	$(_TIF_WORK_SYSCALL_ENTRY & ~_TIF_SYSCALL_AUDIT),TI_flags+THREAD_INFO(%rsp,SIZEOF_PTREGS)
 	jz	sysenter_auditsys
 #endif
 	SAVE_EXTRA_REGS
@@ -346,8 +346,8 @@ ENTRY(ia32_cstar_target)
 1:	movl	(%r8),%r9d
 	_ASM_EXTABLE(1b,ia32_badarg)
 	ASM_CLAC
-	orl     $TS_COMPAT,TI_status+THREAD_INFO(%rsp,RIP)
-	testl   $_TIF_WORK_SYSCALL_ENTRY,TI_flags+THREAD_INFO(%rsp,RIP)
+	orl     $TS_COMPAT,TI_status+THREAD_INFO(%rsp,SIZEOF_PTREGS)
+	testl   $_TIF_WORK_SYSCALL_ENTRY,TI_flags+THREAD_INFO(%rsp,SIZEOF_PTREGS)
 	CFI_REMEMBER_STATE
 	jnz   cstar_tracesys
 	cmpq $IA32_NR_syscalls-1,%rax
@@ -364,10 +364,10 @@ cstar_dispatch:
 	movq %rax,RAX(%rsp)
 	DISABLE_INTERRUPTS(CLBR_NONE)
 	TRACE_IRQS_OFF
-	testl $_TIF_ALLWORK_MASK,TI_flags+THREAD_INFO(%rsp,RIP)
+	testl $_TIF_ALLWORK_MASK,TI_flags+THREAD_INFO(%rsp,SIZEOF_PTREGS)
 	jnz sysretl_audit
 sysretl_from_sys_call:
-	andl $~TS_COMPAT,TI_status+THREAD_INFO(%rsp,RIP)
+	andl $~TS_COMPAT,TI_status+THREAD_INFO(%rsp,SIZEOF_PTREGS)
 	RESTORE_RSI_RDI_RDX
 	movl RIP(%rsp),%ecx
 	CFI_REGISTER rip,rcx
@@ -402,7 +402,7 @@ sysretl_audit:
 
 cstar_tracesys:
 #ifdef CONFIG_AUDITSYSCALL
-	testl $(_TIF_WORK_SYSCALL_ENTRY & ~_TIF_SYSCALL_AUDIT),TI_flags+THREAD_INFO(%rsp,RIP)
+	testl $(_TIF_WORK_SYSCALL_ENTRY & ~_TIF_SYSCALL_AUDIT),TI_flags+THREAD_INFO(%rsp,SIZEOF_PTREGS)
 	jz cstar_auditsys
 #endif
 	xchgl %r9d,%ebp
@@ -469,8 +469,8 @@ ENTRY(ia32_syscall)
 	   this could be a problem. */
 	ALLOC_PT_GPREGS_ON_STACK
 	SAVE_C_REGS_EXCEPT_R891011
-	orl $TS_COMPAT,TI_status+THREAD_INFO(%rsp,RIP)
-	testl $_TIF_WORK_SYSCALL_ENTRY,TI_flags+THREAD_INFO(%rsp,RIP)
+	orl $TS_COMPAT,TI_status+THREAD_INFO(%rsp,SIZEOF_PTREGS)
+	testl $_TIF_WORK_SYSCALL_ENTRY,TI_flags+THREAD_INFO(%rsp,SIZEOF_PTREGS)
 	jnz ia32_tracesys
 	cmpq $(IA32_NR_syscalls-1),%rax
 	ja ia32_badsys
diff --git a/arch/x86/include/asm/thread_info.h b/arch/x86/include/asm/thread_info.h
index ba115eb..0519945 100644
--- a/arch/x86/include/asm/thread_info.h
+++ b/arch/x86/include/asm/thread_info.h
@@ -207,10 +207,12 @@ static inline unsigned long current_stack_pointer(void)
 	_ASM_SUB $(THREAD_SIZE-KERNEL_STACK_OFFSET),reg ;
 
 /*
- * Same if PER_CPU_VAR(kernel_stack) is, perhaps with some offset, already in
- * a certain register (to be used in assembler memory operands).
+ * ASM operand which evaluates to thread_info address
+ * if it is known that "reg" is exactly "off" bytes below stack top.
+ * Example (fetch thread_info->fieldname):
+ *  mov TI_fieldname+THREAD_INFO(reg, off),%eax
  */
-#define THREAD_INFO(reg, off) KERNEL_STACK_OFFSET+(off)-THREAD_SIZE(reg)
+#define THREAD_INFO(reg, off) ((off)-THREAD_SIZE)(reg)
 
 #endif
 
diff --git a/arch/x86/kernel/entry_64.S b/arch/x86/kernel/entry_64.S
index 0c91256..d785a7d 100644
--- a/arch/x86/kernel/entry_64.S
+++ b/arch/x86/kernel/entry_64.S
@@ -258,7 +258,7 @@ GLOBAL(system_call_after_swapgs)
 	SAVE_C_REGS_EXCEPT_RAX_RCX_R11
 	movq	$-ENOSYS,RAX(%rsp)
 	CFI_REL_OFFSET rip,RIP
-	testl $_TIF_WORK_SYSCALL_ENTRY,TI_flags+THREAD_INFO(%rsp,RIP)
+	testl $_TIF_WORK_SYSCALL_ENTRY,TI_flags+THREAD_INFO(%rsp,SIZEOF_PTREGS)
 	jnz tracesys
 system_call_fastpath:
 #if __SYSCALL_MASK == ~0
@@ -276,7 +276,7 @@ system_call_fastpath:
  * Has incompletely filled pt_regs, iret frame is also incomplete.
  */
 ret_from_sys_call:
-	testl $_TIF_ALLWORK_MASK,TI_flags+THREAD_INFO(%rsp,RIP)
+	testl $_TIF_ALLWORK_MASK,TI_flags+THREAD_INFO(%rsp,SIZEOF_PTREGS)
 	jnz int_ret_from_sys_call_fixup	/* Go the the slow path */
 
 	LOCKDEP_SYS_EXIT
-- 
1.8.1.4

[PATCH 2/5] x86: get rid of KERNEL_STACK_OFFSET

[Denys Vlasenko]

PER_CPU_VAR(kernel_stack) was set up in a way where it points
five stack slots below the top of stack.

Presumably, it was done to avoid one "sub $5*8,%rsp"
in syscall/sysenter code paths, where iret frame needs to be
created by hand.

Ironically, none of them benefits from this optimization,
since all of them need to allocate additional data on stack
(struct pt_regs), so they still have to perform subtraction.

This patch eliminates KERNEL_STACK_OFFSET.

PER_CPU_VAR(kernel_stack) now points directly to top of stack.
pt_regs allocations are adjusted to allocate iret frame as well.
Hopefully we can merge it later with 32-bit specific
PER_CPU_VAR(cpu_current_top_of_stack) variable...

Net result in generated code is that constants in several insns
are changed.

This change is necessary for changing struct pt_regs creation
in SYSCALL64 code path from MOV to PUSH instructions.

Signed-off-by: Denys Vlasenko <dvlasenk@redhat.com>
CC: Linus Torvalds <torvalds@linux-foundation.org>
CC: Steven Rostedt <rostedt@goodmis.org>
CC: Ingo Molnar <mingo@kernel.org>
CC: Borislav Petkov <bp@alien8.de>
CC: "H. Peter Anvin" <hpa@zytor.com>
CC: Andy Lutomirski <luto@amacapital.net>
CC: Oleg Nesterov <oleg@redhat.com>
CC: Frederic Weisbecker <fweisbec@gmail.com>
CC: Alexei Starovoitov <ast@plumgrid.com>
CC: Will Drewry <wad@chromium.org>
CC: Kees Cook <keescook@chromium.org>
CC: x86@kernel.org
CC: linux-kernel@vger.kernel.org
---
 arch/x86/ia32/ia32entry.S          | 4 ++--
 arch/x86/include/asm/thread_info.h | 5 ++---
 arch/x86/kernel/cpu/common.c       | 2 +-
 arch/x86/kernel/entry_64.S         | 5 ++---
 arch/x86/kernel/process_32.c       | 2 +-
 arch/x86/kernel/process_64.c       | 3 +--
 arch/x86/kernel/smpboot.c          | 3 +--
 arch/x86/xen/smp.c                 | 3 +--
 8 files changed, 11 insertions(+), 16 deletions(-)

diff --git a/arch/x86/ia32/ia32entry.S b/arch/x86/ia32/ia32entry.S
index 50190e1..acbff3f 100644
--- a/arch/x86/ia32/ia32entry.S
+++ b/arch/x86/ia32/ia32entry.S
@@ -311,7 +311,7 @@ ENDPROC(ia32_sysenter_target)
 ENTRY(ia32_cstar_target)
 	CFI_STARTPROC32	simple
 	CFI_SIGNAL_FRAME
-	CFI_DEF_CFA	rsp,KERNEL_STACK_OFFSET
+	CFI_DEF_CFA	rsp,0
 	CFI_REGISTER	rip,rcx
 	/*CFI_REGISTER	rflags,r11*/
 	SWAPGS_UNSAFE_STACK
@@ -323,7 +323,7 @@ ENTRY(ia32_cstar_target)
 	 * disabled irqs and here we enable it straight after entry:
 	 */
 	ENABLE_INTERRUPTS(CLBR_NONE)
-	ALLOC_PT_GPREGS_ON_STACK 8	/* +8: space for orig_ax */
+	ALLOC_PT_GPREGS_ON_STACK 6*8 /* 6*8: space for orig_ax and iret frame */
 	SAVE_C_REGS_EXCEPT_RCX_R891011
 	movl 	%eax,%eax	/* zero extension */
 	movq	%rax,ORIG_RAX(%rsp)
diff --git a/arch/x86/include/asm/thread_info.h b/arch/x86/include/asm/thread_info.h
index 0519945..89b814e 100644
--- a/arch/x86/include/asm/thread_info.h
+++ b/arch/x86/include/asm/thread_info.h
@@ -172,7 +172,6 @@ struct thread_info {
 #define _TIF_WORK_CTXSW_NEXT (_TIF_WORK_CTXSW)
 
 #define STACK_WARN		(THREAD_SIZE/8)
-#define KERNEL_STACK_OFFSET	(5*(BITS_PER_LONG/8))
 
 /*
  * macros/functions for gaining access to the thread information structure
@@ -201,10 +200,10 @@ static inline unsigned long current_stack_pointer(void)
 
 #else /* !__ASSEMBLY__ */
 
-/* how to get the thread information struct from ASM */
+/* Load thread_info address into "reg" */
 #define GET_THREAD_INFO(reg) \
 	_ASM_MOV PER_CPU_VAR(kernel_stack),reg ; \
-	_ASM_SUB $(THREAD_SIZE-KERNEL_STACK_OFFSET),reg ;
+	_ASM_SUB $(THREAD_SIZE),reg ;
 
 /*
  * ASM operand which evaluates to thread_info address
diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c
index c5b5ccb..9461c83 100644
--- a/arch/x86/kernel/cpu/common.c
+++ b/arch/x86/kernel/cpu/common.c
@@ -1136,7 +1136,7 @@ static __init int setup_disablecpuid(char *arg)
 __setup("clearcpuid=", setup_disablecpuid);
 
 DEFINE_PER_CPU(unsigned long, kernel_stack) =
-	(unsigned long)&init_thread_union - KERNEL_STACK_OFFSET + THREAD_SIZE;
+	(unsigned long)&init_thread_union + THREAD_SIZE;
 EXPORT_PER_CPU_SYMBOL(kernel_stack);
 
 #ifdef CONFIG_X86_64
diff --git a/arch/x86/kernel/entry_64.S b/arch/x86/kernel/entry_64.S
index d785a7d..ecb68d8 100644
--- a/arch/x86/kernel/entry_64.S
+++ b/arch/x86/kernel/entry_64.S
@@ -225,7 +225,7 @@ ENDPROC(native_usergs_sysret64)
 ENTRY(system_call)
 	CFI_STARTPROC	simple
 	CFI_SIGNAL_FRAME
-	CFI_DEF_CFA	rsp,KERNEL_STACK_OFFSET
+	CFI_DEF_CFA	rsp,0
 	CFI_REGISTER	rip,rcx
 	/*CFI_REGISTER	rflags,r11*/
 	SWAPGS_UNSAFE_STACK
@@ -242,9 +242,8 @@ GLOBAL(system_call_after_swapgs)
 	 * so we can enable interrupts only after we're done with using rsp_scratch:
 	 */
 	movq	%rsp,PER_CPU_VAR(rsp_scratch)
-	/* kernel_stack is set so that 5 slots (iret frame) are preallocated */
 	movq	PER_CPU_VAR(kernel_stack),%rsp
-	ALLOC_PT_GPREGS_ON_STACK 8		/* +8: space for orig_ax */
+	ALLOC_PT_GPREGS_ON_STACK 6*8 /* 6*8: space for orig_ax and iret frame */
 	movq	%rcx,RIP(%rsp)
 	movq	PER_CPU_VAR(rsp_scratch),%rcx
 	movq	%r11,EFLAGS(%rsp)
diff --git a/arch/x86/kernel/process_32.c b/arch/x86/kernel/process_32.c
index 1b9963f..5a4c2f8 100644
--- a/arch/x86/kernel/process_32.c
+++ b/arch/x86/kernel/process_32.c
@@ -312,7 +312,7 @@ __switch_to(struct task_struct *prev_p, struct task_struct *next_p)
 	load_sp0(tss, next);
 	this_cpu_write(kernel_stack,
 		       (unsigned long)task_stack_page(next_p) +
-		       THREAD_SIZE - KERNEL_STACK_OFFSET);
+		       THREAD_SIZE);
 	this_cpu_write(cpu_current_top_of_stack,
 		       (unsigned long)task_stack_page(next_p) +
 		       THREAD_SIZE);
diff --git a/arch/x86/kernel/process_64.c b/arch/x86/kernel/process_64.c
index 97f5658..db49063 100644
--- a/arch/x86/kernel/process_64.c
+++ b/arch/x86/kernel/process_64.c
@@ -409,8 +409,7 @@ __switch_to(struct task_struct *prev_p, struct task_struct *next_p)
 	load_sp0(tss, next);
 
 	this_cpu_write(kernel_stack,
-		  (unsigned long)task_stack_page(next_p) +
-		  THREAD_SIZE - KERNEL_STACK_OFFSET);
+		(unsigned long)task_stack_page(next_p) + THREAD_SIZE);
 
 	/*
 	 * Now maybe reload the debug registers and handle I/O bitmaps
diff --git a/arch/x86/kernel/smpboot.c b/arch/x86/kernel/smpboot.c
index 759388c..7b20ffd 100644
--- a/arch/x86/kernel/smpboot.c
+++ b/arch/x86/kernel/smpboot.c
@@ -813,8 +813,7 @@ static int do_boot_cpu(int apicid, int cpu, struct task_struct *idle)
 	initial_gs = per_cpu_offset(cpu);
 #endif
 	per_cpu(kernel_stack, cpu) =
-		(unsigned long)task_stack_page(idle) -
-		KERNEL_STACK_OFFSET + THREAD_SIZE;
+		(unsigned long)task_stack_page(idle) + THREAD_SIZE;
 	early_gdt_descr.address = (unsigned long)get_cpu_gdt_table(cpu);
 	initial_code = (unsigned long)start_secondary;
 	stack_start  = idle->thread.sp;
diff --git a/arch/x86/xen/smp.c b/arch/x86/xen/smp.c
index 08e8489..765b768 100644
--- a/arch/x86/xen/smp.c
+++ b/arch/x86/xen/smp.c
@@ -452,8 +452,7 @@ static int xen_cpu_up(unsigned int cpu, struct task_struct *idle)
 	clear_tsk_thread_flag(idle, TIF_FORK);
 #endif
 	per_cpu(kernel_stack, cpu) =
-		(unsigned long)task_stack_page(idle) -
-		KERNEL_STACK_OFFSET + THREAD_SIZE;
+		(unsigned long)task_stack_page(idle) + THREAD_SIZE;
 
 	xen_setup_runstate_info(cpu);
 	xen_setup_timer(cpu);
-- 
1.8.1.4

[PATCH 3/5] x86/entry_64.S: use PUSH insns to build pt_regs on stack

[Denys Vlasenko]

With this change, on SYSCALL64 code path we are now populating
pt_regs->cs, pt_regs->ss and pt_regs->rcx unconditionally and
therefore don't need to do that in FIXUP_TOP_OF_STACK.

We lose a number of large insns there:

    text    data     bss     dec     hex filename
   13298       0       0   13298    33f2 entry_64_before.o
   12978       0       0   12978    32b2 entry_64.o

What's more important, we convert two "MOVQ $imm,off(%rsp)" to "PUSH $imm"
(the ones which fill pt_regs->cs,ss).

Before this patch, placing them on fast path was slowing it down by two cycles:
this form of MOV is very large, 12 bytes, and this probably reduces decode bandwidth
to one insn per cycle when CPU sees them.

Therefore they were living in FIXUP_TOP_OF_STACK instead (away from fast path).

"PUSH $imm" is a small 2-byte insn. Moving it to fast path does not slow it down
in my measurements.

Signed-off-by: Denys Vlasenko <dvlasenk@redhat.com>
CC: Linus Torvalds <torvalds@linux-foundation.org>
CC: Steven Rostedt <rostedt@goodmis.org>
CC: Ingo Molnar <mingo@kernel.org>
CC: Borislav Petkov <bp@alien8.de>
CC: "H. Peter Anvin" <hpa@zytor.com>
CC: Andy Lutomirski <luto@amacapital.net>
CC: Oleg Nesterov <oleg@redhat.com>
CC: Frederic Weisbecker <fweisbec@gmail.com>
CC: Alexei Starovoitov <ast@plumgrid.com>
CC: Will Drewry <wad@chromium.org>
CC: Kees Cook <keescook@chromium.org>
CC: x86@kernel.org
CC: linux-kernel@vger.kernel.org
---

Changes since last version: reformulate old comment which was
mostly failing to explain why we don't have TRACE_IRQS_OFF/ONs
around a irq-off section in SYSCALL64 code path.

 arch/x86/kernel/entry_64.S | 54 +++++++++++++++++++++++++++-------------------
 1 file changed, 32 insertions(+), 22 deletions(-)

diff --git a/arch/x86/kernel/entry_64.S b/arch/x86/kernel/entry_64.S
index ecb68d8..e9c1882 100644
--- a/arch/x86/kernel/entry_64.S
+++ b/arch/x86/kernel/entry_64.S
@@ -126,11 +126,8 @@ ENDPROC(native_usergs_sysret64)
  * manipulation.
  */
 	.macro FIXUP_TOP_OF_STACK tmp offset=0
-	movq $__USER_DS,SS+\offset(%rsp)
-	movq $__USER_CS,CS+\offset(%rsp)
-	movq RIP+\offset(%rsp),\tmp  /* get rip */
-	movq \tmp,RCX+\offset(%rsp)  /* copy it to rcx as sysret would do */
-	movq EFLAGS+\offset(%rsp),\tmp /* ditto for rflags->r11 */
+	/* copy flags to r11 as sysret would do */
+	movq EFLAGS+\offset(%rsp),\tmp
 	movq \tmp,R11+\offset(%rsp)
 	.endm
 
@@ -214,7 +211,6 @@ ENDPROC(native_usergs_sysret64)
  * r9   arg5
  * (note: r12-r15,rbp,rbx are callee-preserved in C ABI)
  *
- * Interrupts are off on entry.
  * Only called from user space.
  *
  * When user can change pt_regs->foo always force IRET. That is because
@@ -228,6 +224,12 @@ ENTRY(system_call)
 	CFI_DEF_CFA	rsp,0
 	CFI_REGISTER	rip,rcx
 	/*CFI_REGISTER	rflags,r11*/
+
+	/*
+	 * Interrupts are off on entry.
+	 * We do not frame this tiny irq-off block with TRACE_IRQS_OFF/ON,
+	 * it is too small to ever cause noticeable irq latency.
+	 */
 	SWAPGS_UNSAFE_STACK
 	/*
 	 * A hypervisor implementation might want to use a label
@@ -236,27 +238,35 @@ ENTRY(system_call)
 	 */
 GLOBAL(system_call_after_swapgs)
 
-	/*
-	 * We use 'rsp_scratch' as a scratch register, hence this block must execute
-	 * atomically in the face of possible interrupt-driven task preemption,
-	 * so we can enable interrupts only after we're done with using rsp_scratch:
-	 */
 	movq	%rsp,PER_CPU_VAR(rsp_scratch)
 	movq	PER_CPU_VAR(kernel_stack),%rsp
-	ALLOC_PT_GPREGS_ON_STACK 6*8 /* 6*8: space for orig_ax and iret frame */
-	movq	%rcx,RIP(%rsp)
-	movq	PER_CPU_VAR(rsp_scratch),%rcx
-	movq	%r11,EFLAGS(%rsp)
-	movq	%rcx,RSP(%rsp)
+
+	/* Construct struct pt_regs on stack */
+	pushq_cfi $__USER_DS			/* pt_regs->ss */
+	pushq_cfi PER_CPU_VAR(rsp_scratch)	/* pt_regs->sp */
 	/*
-	 * No need to follow this irqs off/on section - it's straight
-	 * and short:
+	 * Re-enable interrupts.
+	 * We use 'rsp_scratch' as a scratch space, hence irq-off block above
+	 * must execute atomically in the face of possible interrupt-driven
+	 * task preemption. We must enable interrupts only after we're done
+	 * with using rsp_scratch:
 	 */
 	ENABLE_INTERRUPTS(CLBR_NONE)
-	movq_cfi rax,ORIG_RAX
-	SAVE_C_REGS_EXCEPT_RAX_RCX_R11
-	movq	$-ENOSYS,RAX(%rsp)
-	CFI_REL_OFFSET rip,RIP
+	pushq_cfi	%r11			/* pt_regs->flags */
+	pushq_cfi	$__USER_CS		/* pt_regs->cs */
+	pushq_cfi	%rcx			/* pt_regs->ip */
+	CFI_REL_OFFSET rip,0
+	pushq_cfi_reg	rax			/* pt_regs->orig_ax */
+	pushq_cfi_reg	rdi			/* pt_regs->di */
+	pushq_cfi_reg	rsi			/* pt_regs->si */
+	pushq_cfi_reg	rdx			/* pt_regs->dx */
+	pushq_cfi_reg	rcx			/* pt_regs->cx */
+	pushq_cfi	$-ENOSYS		/* pt_regs->ax */
+	pushq_cfi_reg	r8			/* pt_regs->r8 */
+	pushq_cfi_reg	r9			/* pt_regs->r9 */
+	pushq_cfi_reg	r10			/* pt_regs->r10 */
+	sub	$(7*8),%rsp /* pt_regs->r11,bp,bx,r12-15 not saved */
+
 	testl $_TIF_WORK_SYSCALL_ENTRY,TI_flags+THREAD_INFO(%rsp,SIZEOF_PTREGS)
 	jnz tracesys
 system_call_fastpath:
-- 
1.8.1.4

[PATCH 4/5] x86/entry_64.S: get rid of FIXUP_TOP_OF_STACK/RESTORE_TOP_OF_STACK

[Denys Vlasenko]

FIXUP_TOP_OF_STACK is only necessary because we don't save %r11
to pt_regs->r11 on SYSCALL64 fast path, but we want ptrace to see
it populated.

Bite the bullet, add a single additional PUSH insn, and remove
FIXUP_TOP_OF_STACK.

RESTORE_TOP_OF_STACK is already a nop. Remove it too.

On SandyBridge CPU, it does not get slower:
measured 54.22 ns per getpid syscall before and after last two changes
on defconfig kernel.

Signed-off-by: Denys Vlasenko <dvlasenk@redhat.com>
CC: Linus Torvalds <torvalds@linux-foundation.org>
CC: Steven Rostedt <rostedt@goodmis.org>
CC: Ingo Molnar <mingo@kernel.org>
CC: Borislav Petkov <bp@alien8.de>
CC: "H. Peter Anvin" <hpa@zytor.com>
CC: Andy Lutomirski <luto@amacapital.net>
CC: Oleg Nesterov <oleg@redhat.com>
CC: Frederic Weisbecker <fweisbec@gmail.com>
CC: Alexei Starovoitov <ast@plumgrid.com>
CC: Will Drewry <wad@chromium.org>
CC: Kees Cook <keescook@chromium.org>
CC: x86@kernel.org
CC: linux-kernel@vger.kernel.org
---
 arch/x86/kernel/entry_64.S | 35 ++---------------------------------
 1 file changed, 2 insertions(+), 33 deletions(-)

diff --git a/arch/x86/kernel/entry_64.S b/arch/x86/kernel/entry_64.S
index e9c1882..829459e 100644
--- a/arch/x86/kernel/entry_64.S
+++ b/arch/x86/kernel/entry_64.S
@@ -22,8 +22,6 @@
  * - CFI macros are used to generate dwarf2 unwind information for better
  * backtraces. They don't change any code.
  * - ENTRY/END Define functions in the symbol table.
- * - FIXUP_TOP_OF_STACK/RESTORE_TOP_OF_STACK - Fix up the hardware stack
- * frame that is otherwise undefined after a SYSCALL
  * - TRACE_IRQ_* - Trace hard interrupt state for lock debugging.
  * - idtentry - Define exception entry points.
  */
@@ -119,23 +117,6 @@ ENDPROC(native_usergs_sysret64)
 #endif
 
 /*
- * C code is not supposed to know that the iret frame is not populated.
- * Every time a C function with an pt_regs argument is called from
- * the SYSCALL based fast path FIXUP_TOP_OF_STACK is needed.
- * RESTORE_TOP_OF_STACK syncs the syscall state after any possible ptregs
- * manipulation.
- */
-	.macro FIXUP_TOP_OF_STACK tmp offset=0
-	/* copy flags to r11 as sysret would do */
-	movq EFLAGS+\offset(%rsp),\tmp
-	movq \tmp,R11+\offset(%rsp)
-	.endm
-
-	.macro RESTORE_TOP_OF_STACK tmp offset=0
-	/* nothing to do */
-	.endm
-
-/*
  * empty frame
  */
 	.macro EMPTY_FRAME start=1 offset=0
@@ -265,7 +246,8 @@ GLOBAL(system_call_after_swapgs)
 	pushq_cfi_reg	r8			/* pt_regs->r8 */
 	pushq_cfi_reg	r9			/* pt_regs->r9 */
 	pushq_cfi_reg	r10			/* pt_regs->r10 */
-	sub	$(7*8),%rsp /* pt_regs->r11,bp,bx,r12-15 not saved */
+	pushq_cfi_reg	r11			/* pt_regs->r11 */
+	sub	$(6*8),%rsp /* pt_regs->bp,bx,r12-15 not saved */
 
 	testl $_TIF_WORK_SYSCALL_ENTRY,TI_flags+THREAD_INFO(%rsp,SIZEOF_PTREGS)
 	jnz tracesys
@@ -312,7 +294,6 @@ ret_from_sys_call:
 	CFI_RESTORE_STATE
 
 int_ret_from_sys_call_fixup:
-	FIXUP_TOP_OF_STACK %r11
 	jmp int_ret_from_sys_call
 
 	/* Do syscall entry tracing */
@@ -328,7 +309,6 @@ tracesys:
 
 tracesys_phase2:
 	SAVE_EXTRA_REGS
-	FIXUP_TOP_OF_STACK %rdi
 	movq %rsp, %rdi
 	movq $AUDIT_ARCH_X86_64, %rsi
 	movq %rax,%rdx
@@ -421,9 +401,7 @@ ENTRY(stub_\func)
 	CFI_STARTPROC
 	DEFAULT_FRAME 0, 8		/* offset 8: return address */
 	SAVE_EXTRA_REGS 8
-	FIXUP_TOP_OF_STACK %r11, 8
 	call sys_\func
-	RESTORE_TOP_OF_STACK %r11, 8
 	ret
 	CFI_ENDPROC
 END(stub_\func)
@@ -438,7 +416,6 @@ ENTRY(stub_execve)
 	addq $8, %rsp
 	DEFAULT_FRAME 0
 	SAVE_EXTRA_REGS
-	FIXUP_TOP_OF_STACK %r11
 	call sys_execve
 	movq %rax,RAX(%rsp)
 	RESTORE_EXTRA_REGS
@@ -451,9 +428,7 @@ ENTRY(stub_execveat)
 	addq $8, %rsp
 	DEFAULT_FRAME 0
 	SAVE_EXTRA_REGS
-	FIXUP_TOP_OF_STACK %r11
 	call sys_execveat
-	RESTORE_TOP_OF_STACK %r11
 	movq %rax,RAX(%rsp)
 	RESTORE_EXTRA_REGS
 	jmp int_ret_from_sys_call
@@ -469,7 +444,6 @@ ENTRY(stub_rt_sigreturn)
 	addq $8, %rsp
 	DEFAULT_FRAME 0
 	SAVE_EXTRA_REGS
-	FIXUP_TOP_OF_STACK %r11
 	call sys_rt_sigreturn
 	movq %rax,RAX(%rsp) # fixme, this could be done at the higher layer
 	RESTORE_EXTRA_REGS
@@ -483,7 +457,6 @@ ENTRY(stub_x32_rt_sigreturn)
 	addq $8, %rsp
 	DEFAULT_FRAME 0
 	SAVE_EXTRA_REGS
-	FIXUP_TOP_OF_STACK %r11
 	call sys32_x32_rt_sigreturn
 	movq %rax,RAX(%rsp) # fixme, this could be done at the higher layer
 	RESTORE_EXTRA_REGS
@@ -496,9 +469,7 @@ ENTRY(stub_x32_execve)
 	addq $8, %rsp
 	DEFAULT_FRAME 0
 	SAVE_EXTRA_REGS
-	FIXUP_TOP_OF_STACK %r11
 	call compat_sys_execve
-	RESTORE_TOP_OF_STACK %r11
 	movq %rax,RAX(%rsp)
 	RESTORE_EXTRA_REGS
 	jmp int_ret_from_sys_call
@@ -510,9 +481,7 @@ ENTRY(stub_x32_execveat)
 	addq $8, %rsp
 	DEFAULT_FRAME 0
 	SAVE_EXTRA_REGS
-	FIXUP_TOP_OF_STACK %r11
 	call compat_sys_execveat
-	RESTORE_TOP_OF_STACK %r11
 	movq %rax,RAX(%rsp)
 	RESTORE_EXTRA_REGS
 	jmp int_ret_from_sys_call
-- 
1.8.1.4

[PATCH 5/5] x86/entry_64.S: get rid of int_ret_from_sys_call_fixup

[Denys Vlasenko]

With FIXUP_TOP_OF_STACK removed, this intermediate jump
is unnecessary.

Signed-off-by: Denys Vlasenko <dvlasenk@redhat.com>
CC: Linus Torvalds <torvalds@linux-foundation.org>
CC: Steven Rostedt <rostedt@goodmis.org>
CC: Ingo Molnar <mingo@kernel.org>
CC: Borislav Petkov <bp@alien8.de>
CC: "H. Peter Anvin" <hpa@zytor.com>
CC: Andy Lutomirski <luto@amacapital.net>
CC: Oleg Nesterov <oleg@redhat.com>
CC: Frederic Weisbecker <fweisbec@gmail.com>
CC: Alexei Starovoitov <ast@plumgrid.com>
CC: Will Drewry <wad@chromium.org>
CC: Kees Cook <keescook@chromium.org>
CC: x86@kernel.org
CC: linux-kernel@vger.kernel.org
---
 arch/x86/kernel/entry_64.S | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/arch/x86/kernel/entry_64.S b/arch/x86/kernel/entry_64.S
index 829459e..8d7f905 100644
--- a/arch/x86/kernel/entry_64.S
+++ b/arch/x86/kernel/entry_64.S
@@ -268,7 +268,7 @@ system_call_fastpath:
  */
 ret_from_sys_call:
 	testl $_TIF_ALLWORK_MASK,TI_flags+THREAD_INFO(%rsp,SIZEOF_PTREGS)
-	jnz int_ret_from_sys_call_fixup	/* Go the the slow path */
+	jnz int_ret_from_sys_call	/* Go the slow path */
 
 	LOCKDEP_SYS_EXIT
 	DISABLE_INTERRUPTS(CLBR_NONE)
@@ -293,9 +293,6 @@ ret_from_sys_call:
 
 	CFI_RESTORE_STATE
 
-int_ret_from_sys_call_fixup:
-	jmp int_ret_from_sys_call
-
 	/* Do syscall entry tracing */
 tracesys:
 	movq %rsp, %rdi
-- 
1.8.1.4

Re: [PATCH 1/5] x86: change THREAD_INFO definition to not depend on KERNEL_STACK_OFFSET

[Borislav Petkov]

On Thu, Mar 19, 2015 at 06:17:45PM +0100, Denys Vlasenko wrote:
> This changes THREAD_INFO definition and all its callsites
> so that they do not count stack position from
> (top of stack - KERNEL_STACK_OFFSET), but from top of stack.
> 
> Semi-mysterious expressions THREAD_INFO(%rsp,RIP) - "why RIP??"
> are now replaced by more logical THREAD_INFO(%rsp,SIZEOF_PTREGS) -
> "calculate thread_info's address using information that
> rsp is SIZEOF_PTREGS bytes below top of stack".
> 
> While at it, replace "(off)-THREAD_SIZE(reg)" with equivalent
> "((off)-THREAD_SIZE)(reg)". The form without parentheses
> falsely looks like we invoke THREAD_SIZE() macro.
> 
> Improve comment atop THREAD_INFO macro definition.
> 
> This patch does not change generated code (verified by objdump).
> 
> Signed-off-by: Denys Vlasenko <dvlasenk@redhat.com>
> CC: Linus Torvalds <torvalds@linux-foundation.org>
> CC: Steven Rostedt <rostedt@goodmis.org>
> CC: Ingo Molnar <mingo@kernel.org>
> CC: Borislav Petkov <bp@alien8.de>
> CC: "H. Peter Anvin" <hpa@zytor.com>
> CC: Andy Lutomirski <luto@amacapital.net>
> CC: Oleg Nesterov <oleg@redhat.com>
> CC: Frederic Weisbecker <fweisbec@gmail.com>
> CC: Alexei Starovoitov <ast@plumgrid.com>
> CC: Will Drewry <wad@chromium.org>
> CC: Kees Cook <keescook@chromium.org>
> CC: x86@kernel.org
> CC: linux-kernel@vger.kernel.org

Acked-by: Borislav Petkov <bp@suse.de>

-- 
Regards/Gruss,
    Boris.

ECO tip #101: Trim your mails when you reply.
--

Re: [PATCH 2/5] x86: get rid of KERNEL_STACK_OFFSET

[Borislav Petkov]

On Thu, Mar 19, 2015 at 06:17:46PM +0100, Denys Vlasenko wrote:
> PER_CPU_VAR(kernel_stack) was set up in a way where it points
> five stack slots below the top of stack.
> 
> Presumably, it was done to avoid one "sub $5*8,%rsp"
> in syscall/sysenter code paths, where iret frame needs to be
> created by hand.
> 
> Ironically, none of them benefits from this optimization,
> since all of them need to allocate additional data on stack
> (struct pt_regs), so they still have to perform subtraction.
> 
> This patch eliminates KERNEL_STACK_OFFSET.
> 
> PER_CPU_VAR(kernel_stack) now points directly to top of stack.
> pt_regs allocations are adjusted to allocate iret frame as well.
> Hopefully we can merge it later with 32-bit specific
> PER_CPU_VAR(cpu_current_top_of_stack) variable...
> 
> Net result in generated code is that constants in several insns
> are changed.
> 
> This change is necessary for changing struct pt_regs creation
> in SYSCALL64 code path from MOV to PUSH instructions.
> 
> Signed-off-by: Denys Vlasenko <dvlasenk@redhat.com>
> CC: Linus Torvalds <torvalds@linux-foundation.org>
> CC: Steven Rostedt <rostedt@goodmis.org>
> CC: Ingo Molnar <mingo@kernel.org>
> CC: Borislav Petkov <bp@alien8.de>
> CC: "H. Peter Anvin" <hpa@zytor.com>
> CC: Andy Lutomirski <luto@amacapital.net>
> CC: Oleg Nesterov <oleg@redhat.com>
> CC: Frederic Weisbecker <fweisbec@gmail.com>
> CC: Alexei Starovoitov <ast@plumgrid.com>
> CC: Will Drewry <wad@chromium.org>
> CC: Kees Cook <keescook@chromium.org>
> CC: x86@kernel.org
> CC: linux-kernel@vger.kernel.org

Acked-by: Borislav Petkov <bp@suse.de>

-- 
Regards/Gruss,
    Boris.

ECO tip #101: Trim your mails when you reply.
--

Re: [PATCH 3/5] x86/entry_64.S: use PUSH insns to build pt_regs on stack

[Borislav Petkov]

On Thu, Mar 19, 2015 at 06:17:47PM +0100, Denys Vlasenko wrote:
> With this change, on SYSCALL64 code path we are now populating
> pt_regs->cs, pt_regs->ss and pt_regs->rcx unconditionally and
> therefore don't need to do that in FIXUP_TOP_OF_STACK.
> 
> We lose a number of large insns there:
> 
>     text    data     bss     dec     hex filename
>    13298       0       0   13298    33f2 entry_64_before.o
>    12978       0       0   12978    32b2 entry_64.o
> 
> What's more important, we convert two "MOVQ $imm,off(%rsp)" to "PUSH $imm"
> (the ones which fill pt_regs->cs,ss).
> 
> Before this patch, placing them on fast path was slowing it down by two cycles:
> this form of MOV is very large, 12 bytes, and this probably reduces decode bandwidth
> to one insn per cycle when CPU sees them.
> 
> Therefore they were living in FIXUP_TOP_OF_STACK instead (away from fast path).
> 
> "PUSH $imm" is a small 2-byte insn. Moving it to fast path does not slow it down
> in my measurements.
> 
> Signed-off-by: Denys Vlasenko <dvlasenk@redhat.com>
> CC: Linus Torvalds <torvalds@linux-foundation.org>
> CC: Steven Rostedt <rostedt@goodmis.org>
> CC: Ingo Molnar <mingo@kernel.org>
> CC: Borislav Petkov <bp@alien8.de>
> CC: "H. Peter Anvin" <hpa@zytor.com>
> CC: Andy Lutomirski <luto@amacapital.net>
> CC: Oleg Nesterov <oleg@redhat.com>
> CC: Frederic Weisbecker <fweisbec@gmail.com>
> CC: Alexei Starovoitov <ast@plumgrid.com>
> CC: Will Drewry <wad@chromium.org>
> CC: Kees Cook <keescook@chromium.org>
> CC: x86@kernel.org
> CC: linux-kernel@vger.kernel.org
> ---
> 
> Changes since last version: reformulate old comment which was
> mostly failing to explain why we don't have TRACE_IRQS_OFF/ONs
> around a irq-off section in SYSCALL64 code path.
> 
>  arch/x86/kernel/entry_64.S | 54 +++++++++++++++++++++++++++-------------------
>  1 file changed, 32 insertions(+), 22 deletions(-)

Acked-by: Borislav Petkov <bp@suse.de>

-- 
Regards/Gruss,
    Boris.

ECO tip #101: Trim your mails when you reply.
--

Re: [PATCH 4/5] x86/entry_64.S: get rid of FIXUP_TOP_OF_STACK/RESTORE_TOP_OF_STACK

[Borislav Petkov]

On Thu, Mar 19, 2015 at 06:17:48PM +0100, Denys Vlasenko wrote:
> FIXUP_TOP_OF_STACK is only necessary because we don't save %r11
> to pt_regs->r11 on SYSCALL64 fast path, but we want ptrace to see
> it populated.
> 
> Bite the bullet, add a single additional PUSH insn, and remove
> FIXUP_TOP_OF_STACK.
> 
> RESTORE_TOP_OF_STACK is already a nop. Remove it too.
> 
> On SandyBridge CPU, it does not get slower:
> measured 54.22 ns per getpid syscall before and after last two changes
> on defconfig kernel.
> 
> Signed-off-by: Denys Vlasenko <dvlasenk@redhat.com>
> CC: Linus Torvalds <torvalds@linux-foundation.org>
> CC: Steven Rostedt <rostedt@goodmis.org>
> CC: Ingo Molnar <mingo@kernel.org>
> CC: Borislav Petkov <bp@alien8.de>
> CC: "H. Peter Anvin" <hpa@zytor.com>
> CC: Andy Lutomirski <luto@amacapital.net>
> CC: Oleg Nesterov <oleg@redhat.com>
> CC: Frederic Weisbecker <fweisbec@gmail.com>
> CC: Alexei Starovoitov <ast@plumgrid.com>
> CC: Will Drewry <wad@chromium.org>
> CC: Kees Cook <keescook@chromium.org>
> CC: x86@kernel.org
> CC: linux-kernel@vger.kernel.org
> ---
>  arch/x86/kernel/entry_64.S | 35 ++---------------------------------
>  1 file changed, 2 insertions(+), 33 deletions(-)

Nice diffstat.

Acked-by: Borislav Petkov <bp@suse.de>

-- 
Regards/Gruss,
    Boris.

ECO tip #101: Trim your mails when you reply.
--

Re: [PATCH 5/5] x86/entry_64.S: get rid of int_ret_from_sys_call_fixup

[Borislav Petkov]

On Thu, Mar 19, 2015 at 06:17:49PM +0100, Denys Vlasenko wrote:
> With FIXUP_TOP_OF_STACK removed, this intermediate jump
> is unnecessary.
> 
> Signed-off-by: Denys Vlasenko <dvlasenk@redhat.com>
> CC: Linus Torvalds <torvalds@linux-foundation.org>
> CC: Steven Rostedt <rostedt@goodmis.org>
> CC: Ingo Molnar <mingo@kernel.org>
> CC: Borislav Petkov <bp@alien8.de>
> CC: "H. Peter Anvin" <hpa@zytor.com>
> CC: Andy Lutomirski <luto@amacapital.net>
> CC: Oleg Nesterov <oleg@redhat.com>
> CC: Frederic Weisbecker <fweisbec@gmail.com>
> CC: Alexei Starovoitov <ast@plumgrid.com>
> CC: Will Drewry <wad@chromium.org>
> CC: Kees Cook <keescook@chromium.org>
> CC: x86@kernel.org
> CC: linux-kernel@vger.kernel.org
> ---
>  arch/x86/kernel/entry_64.S | 5 +----
>  1 file changed, 1 insertion(+), 4 deletions(-)

Acked-by: Borislav Petkov <bp@suse.de>

-- 
Regards/Gruss,
    Boris.

ECO tip #101: Trim your mails when you reply.
--

Re: [PATCH 1/5] x86: change THREAD_INFO definition to not depend on KERNEL_STACK_OFFSET

[Andy Lutomirski]

On Thu, Mar 19, 2015 at 10:17 AM, Denys Vlasenko <dvlasenk@redhat.com> wrote:
> This changes THREAD_INFO definition and all its callsites
> so that they do not count stack position from
> (top of stack - KERNEL_STACK_OFFSET), but from top of stack.
>

This whole series is:

Acked-by: Andy Lutomirski <luto@kernel.org>

Thanks!

Re: [PATCH 1/5] x86: change THREAD_INFO definition to not depend on KERNEL_STACK_OFFSET

[Ingo Molnar]

* Denys Vlasenko <dvlasenk@redhat.com> wrote:

> This changes THREAD_INFO definition and all its callsites
> so that they do not count stack position from
> (top of stack - KERNEL_STACK_OFFSET), but from top of stack.
> 
> Semi-mysterious expressions THREAD_INFO(%rsp,RIP) - "why RIP??"
> are now replaced by more logical THREAD_INFO(%rsp,SIZEOF_PTREGS) -
> "calculate thread_info's address using information that
> rsp is SIZEOF_PTREGS bytes below top of stack".
> 
> While at it, replace "(off)-THREAD_SIZE(reg)" with equivalent
> "((off)-THREAD_SIZE)(reg)". The form without parentheses
> falsely looks like we invoke THREAD_SIZE() macro.
> 
> Improve comment atop THREAD_INFO macro definition.
> 
> This patch does not change generated code (verified by objdump).

> --- a/arch/x86/include/asm/thread_info.h
> +++ b/arch/x86/include/asm/thread_info.h
> @@ -207,10 +207,12 @@ static inline unsigned long current_stack_pointer(void)
>  	_ASM_SUB $(THREAD_SIZE-KERNEL_STACK_OFFSET),reg ;
>  
>  /*
> - * Same if PER_CPU_VAR(kernel_stack) is, perhaps with some offset, already in
> - * a certain register (to be used in assembler memory operands).
> + * ASM operand which evaluates to thread_info address
> + * if it is known that "reg" is exactly "off" bytes below stack top.
> + * Example (fetch thread_info->fieldname):
> + *  mov TI_fieldname+THREAD_INFO(reg, off),%eax
>   */
> -#define THREAD_INFO(reg, off) KERNEL_STACK_OFFSET+(off)-THREAD_SIZE(reg)
> +#define THREAD_INFO(reg, off) ((off)-THREAD_SIZE)(reg)

We need more assembly hackers, so I have improved this still somewhat 
cryptic comment to:

/*
 * ASM operand which evaluates to a 'thread_info' address of
 * the current task, if it is known that "reg" is exactly "off"
 * bytes below the top of the stack currently.
 *
 * ( The kernel stack's size is known at build time, it is usually
 *   2 or 4 pages, and the bottom  of the kernel stack contains
 *   the thread_info structure. So to access the thread_info very
 *   quickly from assembly code we can calculate down from the
 *   top of the kernel stack to the bottom, using constant,
 *   build-time calculations only. )
 *
 * For example, to fetch the current thread_info->flags value into %eax
 * on x86-64 defconfig kernels:
 *
 *      mov TI_flags+THREAD_INFO(%rsp, SIZEOF_PTREGS), %eax
 *
 * will translate to:
 *
 *      8b 84 24 b8 c0 ff ff      mov    -0x3f48(%rsp), %eax
 *
 * which is below the current RSP by almost 16K.
 */
#define THREAD_INFO(reg, off) ((off)-THREAD_SIZE)(reg)

Agreed?

Thanks,

	Ingo

[PATCH] x86/asm/entry/64: Improve the THREAD_INFO() macro explanation

[Ingo Molnar]

>From 0229a184997a7d4ad4398ee3ac2f5ae78c1c1a03 Mon Sep 17 00:00:00 2001
From: Ingo Molnar <mingo@kernel.org>
Date: Tue, 24 Mar 2015 18:57:13 +0100
Subject: [PATCH] x86/asm/entry/64: Improve the THREAD_INFO() macro explanation

Explain the background, and add a real example.

Cc: Alexei Starovoitov <ast@plumgrid.com>
Cc: Andy Lutomirski <luto@amacapital.net>
Cc: Andy Lutomirski <luto@kernel.org>
Cc: Borislav Petkov <bp@alien8.de>
Cc: Borislav Petkov <bp@suse.de>
Cc: Denys Vlasenko <dvlasenk@redhat.com>
Cc: Frederic Weisbecker <fweisbec@gmail.com>
Cc: H. Peter Anvin <hpa@zytor.com>
Cc: Kees Cook <keescook@chromium.org>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Oleg Nesterov <oleg@redhat.com>
Cc: Steven Rostedt <rostedt@goodmis.org>
Cc: Will Drewry <wad@chromium.org>
Signed-off-by: Ingo Molnar <mingo@kernel.org>
---
 arch/x86/include/asm/thread_info.h | 27 +++++++++++++++++++++++----
 1 file changed, 23 insertions(+), 4 deletions(-)

diff --git a/arch/x86/include/asm/thread_info.h b/arch/x86/include/asm/thread_info.h
index ad0ee3423da5..813dfbb867a7 100644
--- a/arch/x86/include/asm/thread_info.h
+++ b/arch/x86/include/asm/thread_info.h
@@ -206,10 +206,29 @@ static inline unsigned long current_stack_pointer(void)
 	_ASM_SUB $(THREAD_SIZE),reg ;
 
 /*
- * ASM operand which evaluates to thread_info address
- * if it is known that "reg" is exactly "off" bytes below stack top.
- * Example (fetch thread_info->fieldname):
- *  mov TI_fieldname+THREAD_INFO(reg, off),%eax
+ * ASM operand which evaluates to a 'thread_info' address of
+ * the current task, if it is known that "reg" is exactly "off"
+ * bytes below the top of the stack currently.
+ *
+ * ( The kernel stack's size is known at build time, it is usually
+ *   2 or 4 pages, and the bottom  of the kernel stack contains
+ *   the thread_info structure. So to access the thread_info very
+ *   quickly from assembly code we can calculate down from the
+ *   top of the kernel stack to the bottom, using constant,
+ *   build-time calculations only. )
+ *
+ * For example, to fetch the current thread_info->flags value into %eax
+ * on x86-64 defconfig kernels, in syscall entry code where RSP is
+ * currently at exactly SIZEOF_PTREGS bytes away from the top of the
+ * stack:
+ *
+ *      mov TI_flags+THREAD_INFO(%rsp, SIZEOF_PTREGS), %eax
+ *
+ * will translate to:
+ *
+ *      8b 84 24 b8 c0 ff ff      mov    -0x3f48(%rsp), %eax
+ *
+ * which is below the current RSP by almost 16K.
  */
 #define THREAD_INFO(reg, off) ((off)-THREAD_SIZE)(reg)
 

[PATCH] x86/asm/entry/64: Merge the field offset into the THREAD_INFO() macro

[Ingo Molnar]

>From 11e2761ba0969466299b7109eba749d2292e8796 Mon Sep 17 00:00:00 2001
From: Ingo Molnar <mingo@kernel.org>
Date: Tue, 24 Mar 2015 19:18:41 +0100
Subject: [PATCH] x86/asm/entry/64: Merge the field offset into the THREAD_INFO() macro

Before:

   TI_sysenter_return+THREAD_INFO(%rsp,3*8),%r10d

After:

   movl    THREAD_INFO(TI_sysenter_return, %rsp, 3*8), %r10d

to turn it into a clear thread_info accessor.

No code changed:

 md5:
   fb4cb2b3ce05d89940ca304efc8ff183  ia32entry.o.before.asm
   fb4cb2b3ce05d89940ca304efc8ff183  ia32entry.o.after.asm

   e39f2958a5d1300158e276e4f7663263  entry_64.o.before.asm
   e39f2958a5d1300158e276e4f7663263  entry_64.o.after.asm

Cc: Alexei Starovoitov <ast@plumgrid.com>
Cc: Andy Lutomirski <luto@amacapital.net>
Cc: Andy Lutomirski <luto@kernel.org>
Cc: Borislav Petkov <bp@alien8.de>
Cc: Borislav Petkov <bp@suse.de>
Cc: Denys Vlasenko <dvlasenk@redhat.com>
Cc: Frederic Weisbecker <fweisbec@gmail.com>
Cc: H. Peter Anvin <hpa@zytor.com>
Cc: Kees Cook <keescook@chromium.org>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Oleg Nesterov <oleg@redhat.com>
Cc: Steven Rostedt <rostedt@goodmis.org>
Cc: Will Drewry <wad@chromium.org>
Signed-off-by: Ingo Molnar <mingo@kernel.org>
---
 arch/x86/ia32/ia32entry.S          | 30 +++++++++++++++---------------
 arch/x86/include/asm/thread_info.h |  4 ++--
 arch/x86/kernel/entry_64.S         |  4 ++--
 3 files changed, 19 insertions(+), 19 deletions(-)

diff --git a/arch/x86/ia32/ia32entry.S b/arch/x86/ia32/ia32entry.S
index acbff3fb96a1..32e94aec6073 100644
--- a/arch/x86/ia32/ia32entry.S
+++ b/arch/x86/ia32/ia32entry.S
@@ -127,7 +127,7 @@ ENTRY(ia32_sysenter_target)
 	CFI_REL_OFFSET rsp,0
 	pushfq_cfi
 	/*CFI_REL_OFFSET rflags,0*/
-	movl	TI_sysenter_return+THREAD_INFO(%rsp,3*8),%r10d
+	movl	THREAD_INFO(TI_sysenter_return, %rsp, 3*8), %r10d
 	CFI_REGISTER rip,r10
 	pushq_cfi $__USER32_CS
 	/*CFI_REL_OFFSET cs,0*/
@@ -159,8 +159,8 @@ ENTRY(ia32_sysenter_target)
 	jnz sysenter_fix_flags
 sysenter_flags_fixed:
 
-	orl     $TS_COMPAT,TI_status+THREAD_INFO(%rsp,SIZEOF_PTREGS)
-	testl   $_TIF_WORK_SYSCALL_ENTRY,TI_flags+THREAD_INFO(%rsp,SIZEOF_PTREGS)
+	orl     $TS_COMPAT, THREAD_INFO(TI_status, %rsp, SIZEOF_PTREGS)
+	testl   $_TIF_WORK_SYSCALL_ENTRY, THREAD_INFO(TI_flags, %rsp, SIZEOF_PTREGS)
 	CFI_REMEMBER_STATE
 	jnz  sysenter_tracesys
 	cmpq	$(IA32_NR_syscalls-1),%rax
@@ -177,10 +177,10 @@ ENTRY(ia32_sysenter_target)
 	movq	%rax,RAX(%rsp)
 	DISABLE_INTERRUPTS(CLBR_NONE)
 	TRACE_IRQS_OFF
-	testl	$_TIF_ALLWORK_MASK,TI_flags+THREAD_INFO(%rsp,SIZEOF_PTREGS)
+	testl	$_TIF_ALLWORK_MASK, THREAD_INFO(TI_flags, %rsp, SIZEOF_PTREGS)
 	jnz	sysexit_audit
 sysexit_from_sys_call:
-	andl    $~TS_COMPAT,TI_status+THREAD_INFO(%rsp,SIZEOF_PTREGS)
+	andl    $~TS_COMPAT,THREAD_INFO(TI_status, %rsp, SIZEOF_PTREGS)
 	/* clear IF, that popfq doesn't enable interrupts early */
 	andl	$~0x200,EFLAGS(%rsp)
 	movl	RIP(%rsp),%edx		/* User %eip */
@@ -225,7 +225,7 @@ ENTRY(ia32_sysenter_target)
 	.endm
 
 	.macro auditsys_exit exit
-	testl $(_TIF_ALLWORK_MASK & ~_TIF_SYSCALL_AUDIT),TI_flags+THREAD_INFO(%rsp,SIZEOF_PTREGS)
+	testl $(_TIF_ALLWORK_MASK & ~_TIF_SYSCALL_AUDIT), THREAD_INFO(TI_flags, %rsp, SIZEOF_PTREGS)
 	jnz ia32_ret_from_sys_call
 	TRACE_IRQS_ON
 	ENABLE_INTERRUPTS(CLBR_NONE)
@@ -240,7 +240,7 @@ ENTRY(ia32_sysenter_target)
 	movl $(_TIF_ALLWORK_MASK & ~_TIF_SYSCALL_AUDIT),%edi
 	DISABLE_INTERRUPTS(CLBR_NONE)
 	TRACE_IRQS_OFF
-	testl %edi,TI_flags+THREAD_INFO(%rsp,SIZEOF_PTREGS)
+	testl %edi, THREAD_INFO(TI_flags, %rsp, SIZEOF_PTREGS)
 	jz \exit
 	CLEAR_RREGS
 	jmp int_with_check
@@ -262,7 +262,7 @@ ENTRY(ia32_sysenter_target)
 
 sysenter_tracesys:
 #ifdef CONFIG_AUDITSYSCALL
-	testl	$(_TIF_WORK_SYSCALL_ENTRY & ~_TIF_SYSCALL_AUDIT),TI_flags+THREAD_INFO(%rsp,SIZEOF_PTREGS)
+	testl	$(_TIF_WORK_SYSCALL_ENTRY & ~_TIF_SYSCALL_AUDIT), THREAD_INFO(TI_flags, %rsp, SIZEOF_PTREGS)
 	jz	sysenter_auditsys
 #endif
 	SAVE_EXTRA_REGS
@@ -346,8 +346,8 @@ ENTRY(ia32_cstar_target)
 1:	movl	(%r8),%r9d
 	_ASM_EXTABLE(1b,ia32_badarg)
 	ASM_CLAC
-	orl     $TS_COMPAT,TI_status+THREAD_INFO(%rsp,SIZEOF_PTREGS)
-	testl   $_TIF_WORK_SYSCALL_ENTRY,TI_flags+THREAD_INFO(%rsp,SIZEOF_PTREGS)
+	orl     $TS_COMPAT, THREAD_INFO(TI_status, %rsp, SIZEOF_PTREGS)
+	testl   $_TIF_WORK_SYSCALL_ENTRY, THREAD_INFO(TI_flags, %rsp, SIZEOF_PTREGS)
 	CFI_REMEMBER_STATE
 	jnz   cstar_tracesys
 	cmpq $IA32_NR_syscalls-1,%rax
@@ -364,10 +364,10 @@ ENTRY(ia32_cstar_target)
 	movq %rax,RAX(%rsp)
 	DISABLE_INTERRUPTS(CLBR_NONE)
 	TRACE_IRQS_OFF
-	testl $_TIF_ALLWORK_MASK,TI_flags+THREAD_INFO(%rsp,SIZEOF_PTREGS)
+	testl $_TIF_ALLWORK_MASK, THREAD_INFO(TI_flags, %rsp, SIZEOF_PTREGS)
 	jnz sysretl_audit
 sysretl_from_sys_call:
-	andl $~TS_COMPAT,TI_status+THREAD_INFO(%rsp,SIZEOF_PTREGS)
+	andl $~TS_COMPAT, THREAD_INFO(TI_status, %rsp, SIZEOF_PTREGS)
 	RESTORE_RSI_RDI_RDX
 	movl RIP(%rsp),%ecx
 	CFI_REGISTER rip,rcx
@@ -402,7 +402,7 @@ ENTRY(ia32_cstar_target)
 
 cstar_tracesys:
 #ifdef CONFIG_AUDITSYSCALL
-	testl $(_TIF_WORK_SYSCALL_ENTRY & ~_TIF_SYSCALL_AUDIT),TI_flags+THREAD_INFO(%rsp,SIZEOF_PTREGS)
+	testl $(_TIF_WORK_SYSCALL_ENTRY & ~_TIF_SYSCALL_AUDIT), THREAD_INFO(TI_flags, %rsp, SIZEOF_PTREGS)
 	jz cstar_auditsys
 #endif
 	xchgl %r9d,%ebp
@@ -469,8 +469,8 @@ ENTRY(ia32_syscall)
 	   this could be a problem. */
 	ALLOC_PT_GPREGS_ON_STACK
 	SAVE_C_REGS_EXCEPT_R891011
-	orl $TS_COMPAT,TI_status+THREAD_INFO(%rsp,SIZEOF_PTREGS)
-	testl $_TIF_WORK_SYSCALL_ENTRY,TI_flags+THREAD_INFO(%rsp,SIZEOF_PTREGS)
+	orl $TS_COMPAT, THREAD_INFO(TI_status, %rsp, SIZEOF_PTREGS)
+	testl $_TIF_WORK_SYSCALL_ENTRY, THREAD_INFO(TI_flags, %rsp, SIZEOF_PTREGS)
 	jnz ia32_tracesys
 	cmpq $(IA32_NR_syscalls-1),%rax
 	ja ia32_badsys
diff --git a/arch/x86/include/asm/thread_info.h b/arch/x86/include/asm/thread_info.h
index 813dfbb867a7..224285b674ca 100644
--- a/arch/x86/include/asm/thread_info.h
+++ b/arch/x86/include/asm/thread_info.h
@@ -222,7 +222,7 @@ static inline unsigned long current_stack_pointer(void)
  * currently at exactly SIZEOF_PTREGS bytes away from the top of the
  * stack:
  *
- *      mov TI_flags+THREAD_INFO(%rsp, SIZEOF_PTREGS), %eax
+ *      mov THREAD_INFO(TI_flags, %rsp, SIZEOF_PTREGS), %eax
  *
  * will translate to:
  *
@@ -230,7 +230,7 @@ static inline unsigned long current_stack_pointer(void)
  *
  * which is below the current RSP by almost 16K.
  */
-#define THREAD_INFO(reg, off) ((off)-THREAD_SIZE)(reg)
+#define THREAD_INFO(field, reg, off) ((field)+(off)-THREAD_SIZE)(reg)
 
 #endif
 
diff --git a/arch/x86/kernel/entry_64.S b/arch/x86/kernel/entry_64.S
index df04ee069b1f..8f01a4f1cf9e 100644
--- a/arch/x86/kernel/entry_64.S
+++ b/arch/x86/kernel/entry_64.S
@@ -249,7 +249,7 @@ GLOBAL(system_call_after_swapgs)
 	pushq_cfi_reg	r11			/* pt_regs->r11 */
 	sub	$(6*8),%rsp /* pt_regs->bp,bx,r12-15 not saved */
 
-	testl $_TIF_WORK_SYSCALL_ENTRY,TI_flags+THREAD_INFO(%rsp,SIZEOF_PTREGS)
+	testl $_TIF_WORK_SYSCALL_ENTRY, THREAD_INFO(TI_flags, %rsp, SIZEOF_PTREGS)
 	jnz tracesys
 system_call_fastpath:
 #if __SYSCALL_MASK == ~0
@@ -267,7 +267,7 @@ GLOBAL(system_call_after_swapgs)
  * Has incompletely filled pt_regs, iret frame is also incomplete.
  */
 ret_from_sys_call:
-	testl $_TIF_ALLWORK_MASK,TI_flags+THREAD_INFO(%rsp,SIZEOF_PTREGS)
+	testl $_TIF_ALLWORK_MASK, THREAD_INFO(TI_flags, %rsp, SIZEOF_PTREGS)
 	jnz int_ret_from_sys_call	/* Go the slow path */
 
 	LOCKDEP_SYS_EXIT

[PATCH] x86/asm/entry/64: Rename THREAD_INFO() to ASM_ASM_THREAD_INFO_MEMOP()

[Ingo Molnar]

>From 234ed7356ecfa60f4e209d513d89fa9de41aa4c5 Mon Sep 17 00:00:00 2001
From: Ingo Molnar <mingo@kernel.org>
Date: Tue, 24 Mar 2015 19:29:16 +0100
Subject: [PATCH] x86/asm/entry/64: Rename THREAD_INFO() to ASM_ASM_THREAD_INFO_MEMOP()

The THREAD_INFO() macro has a somewhat confusingly generic name, defined in
a generic .h C header file. It also does not make it clear that it constructs
a memory operand for use in assembly code.

Rename it to ASM_THREAD_INFO_MEMOP() to make it all glaringly obvious on
first glance.

Cc: Alexei Starovoitov <ast@plumgrid.com>
Cc: Andy Lutomirski <luto@amacapital.net>
Cc: Andy Lutomirski <luto@kernel.org>
Cc: Borislav Petkov <bp@alien8.de>
Cc: Borislav Petkov <bp@suse.de>
Cc: Denys Vlasenko <dvlasenk@redhat.com>
Cc: Frederic Weisbecker <fweisbec@gmail.com>
Cc: H. Peter Anvin <hpa@zytor.com>
Cc: Kees Cook <keescook@chromium.org>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Oleg Nesterov <oleg@redhat.com>
Cc: Steven Rostedt <rostedt@goodmis.org>
Cc: Will Drewry <wad@chromium.org>
Signed-off-by: Ingo Molnar <mingo@kernel.org>
---
 arch/x86/ia32/ia32entry.S          | 30 +++++++++++++++---------------
 arch/x86/include/asm/thread_info.h |  4 ++--
 arch/x86/kernel/entry_64.S         |  4 ++--
 3 files changed, 19 insertions(+), 19 deletions(-)

diff --git a/arch/x86/ia32/ia32entry.S b/arch/x86/ia32/ia32entry.S
index 32e94aec6073..b77ea8242b17 100644
--- a/arch/x86/ia32/ia32entry.S
+++ b/arch/x86/ia32/ia32entry.S
@@ -127,7 +127,7 @@ ENTRY(ia32_sysenter_target)
 	CFI_REL_OFFSET rsp,0
 	pushfq_cfi
 	/*CFI_REL_OFFSET rflags,0*/
-	movl	THREAD_INFO(TI_sysenter_return, %rsp, 3*8), %r10d
+	movl	ASM_THREAD_INFO_MEMOP(TI_sysenter_return, %rsp, 3*8), %r10d
 	CFI_REGISTER rip,r10
 	pushq_cfi $__USER32_CS
 	/*CFI_REL_OFFSET cs,0*/
@@ -159,8 +159,8 @@ ENTRY(ia32_sysenter_target)
 	jnz sysenter_fix_flags
 sysenter_flags_fixed:
 
-	orl     $TS_COMPAT, THREAD_INFO(TI_status, %rsp, SIZEOF_PTREGS)
-	testl   $_TIF_WORK_SYSCALL_ENTRY, THREAD_INFO(TI_flags, %rsp, SIZEOF_PTREGS)
+	orl     $TS_COMPAT, ASM_THREAD_INFO_MEMOP(TI_status, %rsp, SIZEOF_PTREGS)
+	testl   $_TIF_WORK_SYSCALL_ENTRY, ASM_THREAD_INFO_MEMOP(TI_flags, %rsp, SIZEOF_PTREGS)
 	CFI_REMEMBER_STATE
 	jnz  sysenter_tracesys
 	cmpq	$(IA32_NR_syscalls-1),%rax
@@ -177,10 +177,10 @@ ENTRY(ia32_sysenter_target)
 	movq	%rax,RAX(%rsp)
 	DISABLE_INTERRUPTS(CLBR_NONE)
 	TRACE_IRQS_OFF
-	testl	$_TIF_ALLWORK_MASK, THREAD_INFO(TI_flags, %rsp, SIZEOF_PTREGS)
+	testl	$_TIF_ALLWORK_MASK, ASM_THREAD_INFO_MEMOP(TI_flags, %rsp, SIZEOF_PTREGS)
 	jnz	sysexit_audit
 sysexit_from_sys_call:
-	andl    $~TS_COMPAT,THREAD_INFO(TI_status, %rsp, SIZEOF_PTREGS)
+	andl    $~TS_COMPAT,ASM_THREAD_INFO_MEMOP(TI_status, %rsp, SIZEOF_PTREGS)
 	/* clear IF, that popfq doesn't enable interrupts early */
 	andl	$~0x200,EFLAGS(%rsp)
 	movl	RIP(%rsp),%edx		/* User %eip */
@@ -225,7 +225,7 @@ ENTRY(ia32_sysenter_target)
 	.endm
 
 	.macro auditsys_exit exit
-	testl $(_TIF_ALLWORK_MASK & ~_TIF_SYSCALL_AUDIT), THREAD_INFO(TI_flags, %rsp, SIZEOF_PTREGS)
+	testl $(_TIF_ALLWORK_MASK & ~_TIF_SYSCALL_AUDIT), ASM_THREAD_INFO_MEMOP(TI_flags, %rsp, SIZEOF_PTREGS)
 	jnz ia32_ret_from_sys_call
 	TRACE_IRQS_ON
 	ENABLE_INTERRUPTS(CLBR_NONE)
@@ -240,7 +240,7 @@ ENTRY(ia32_sysenter_target)
 	movl $(_TIF_ALLWORK_MASK & ~_TIF_SYSCALL_AUDIT),%edi
 	DISABLE_INTERRUPTS(CLBR_NONE)
 	TRACE_IRQS_OFF
-	testl %edi, THREAD_INFO(TI_flags, %rsp, SIZEOF_PTREGS)
+	testl %edi, ASM_THREAD_INFO_MEMOP(TI_flags, %rsp, SIZEOF_PTREGS)
 	jz \exit
 	CLEAR_RREGS
 	jmp int_with_check
@@ -262,7 +262,7 @@ ENTRY(ia32_sysenter_target)
 
 sysenter_tracesys:
 #ifdef CONFIG_AUDITSYSCALL
-	testl	$(_TIF_WORK_SYSCALL_ENTRY & ~_TIF_SYSCALL_AUDIT), THREAD_INFO(TI_flags, %rsp, SIZEOF_PTREGS)
+	testl	$(_TIF_WORK_SYSCALL_ENTRY & ~_TIF_SYSCALL_AUDIT), ASM_THREAD_INFO_MEMOP(TI_flags, %rsp, SIZEOF_PTREGS)
 	jz	sysenter_auditsys
 #endif
 	SAVE_EXTRA_REGS
@@ -346,8 +346,8 @@ ENTRY(ia32_cstar_target)
 1:	movl	(%r8),%r9d
 	_ASM_EXTABLE(1b,ia32_badarg)
 	ASM_CLAC
-	orl     $TS_COMPAT, THREAD_INFO(TI_status, %rsp, SIZEOF_PTREGS)
-	testl   $_TIF_WORK_SYSCALL_ENTRY, THREAD_INFO(TI_flags, %rsp, SIZEOF_PTREGS)
+	orl     $TS_COMPAT, ASM_THREAD_INFO_MEMOP(TI_status, %rsp, SIZEOF_PTREGS)
+	testl   $_TIF_WORK_SYSCALL_ENTRY, ASM_THREAD_INFO_MEMOP(TI_flags, %rsp, SIZEOF_PTREGS)
 	CFI_REMEMBER_STATE
 	jnz   cstar_tracesys
 	cmpq $IA32_NR_syscalls-1,%rax
@@ -364,10 +364,10 @@ ENTRY(ia32_cstar_target)
 	movq %rax,RAX(%rsp)
 	DISABLE_INTERRUPTS(CLBR_NONE)
 	TRACE_IRQS_OFF
-	testl $_TIF_ALLWORK_MASK, THREAD_INFO(TI_flags, %rsp, SIZEOF_PTREGS)
+	testl $_TIF_ALLWORK_MASK, ASM_THREAD_INFO_MEMOP(TI_flags, %rsp, SIZEOF_PTREGS)
 	jnz sysretl_audit
 sysretl_from_sys_call:
-	andl $~TS_COMPAT, THREAD_INFO(TI_status, %rsp, SIZEOF_PTREGS)
+	andl $~TS_COMPAT, ASM_THREAD_INFO_MEMOP(TI_status, %rsp, SIZEOF_PTREGS)
 	RESTORE_RSI_RDI_RDX
 	movl RIP(%rsp),%ecx
 	CFI_REGISTER rip,rcx
@@ -402,7 +402,7 @@ ENTRY(ia32_cstar_target)
 
 cstar_tracesys:
 #ifdef CONFIG_AUDITSYSCALL
-	testl $(_TIF_WORK_SYSCALL_ENTRY & ~_TIF_SYSCALL_AUDIT), THREAD_INFO(TI_flags, %rsp, SIZEOF_PTREGS)
+	testl $(_TIF_WORK_SYSCALL_ENTRY & ~_TIF_SYSCALL_AUDIT), ASM_THREAD_INFO_MEMOP(TI_flags, %rsp, SIZEOF_PTREGS)
 	jz cstar_auditsys
 #endif
 	xchgl %r9d,%ebp
@@ -469,8 +469,8 @@ ENTRY(ia32_syscall)
 	   this could be a problem. */
 	ALLOC_PT_GPREGS_ON_STACK
 	SAVE_C_REGS_EXCEPT_R891011
-	orl $TS_COMPAT, THREAD_INFO(TI_status, %rsp, SIZEOF_PTREGS)
-	testl $_TIF_WORK_SYSCALL_ENTRY, THREAD_INFO(TI_flags, %rsp, SIZEOF_PTREGS)
+	orl $TS_COMPAT, ASM_THREAD_INFO_MEMOP(TI_status, %rsp, SIZEOF_PTREGS)
+	testl $_TIF_WORK_SYSCALL_ENTRY, ASM_THREAD_INFO_MEMOP(TI_flags, %rsp, SIZEOF_PTREGS)
 	jnz ia32_tracesys
 	cmpq $(IA32_NR_syscalls-1),%rax
 	ja ia32_badsys
diff --git a/arch/x86/include/asm/thread_info.h b/arch/x86/include/asm/thread_info.h
index 224285b674ca..fca7c56a6daa 100644
--- a/arch/x86/include/asm/thread_info.h
+++ b/arch/x86/include/asm/thread_info.h
@@ -222,7 +222,7 @@ static inline unsigned long current_stack_pointer(void)
  * currently at exactly SIZEOF_PTREGS bytes away from the top of the
  * stack:
  *
- *      mov THREAD_INFO(TI_flags, %rsp, SIZEOF_PTREGS), %eax
+ *      mov ASM_THREAD_INFO_MEMOP(TI_flags, %rsp, SIZEOF_PTREGS), %eax
  *
  * will translate to:
  *
@@ -230,7 +230,7 @@ static inline unsigned long current_stack_pointer(void)
  *
  * which is below the current RSP by almost 16K.
  */
-#define THREAD_INFO(field, reg, off) ((field)+(off)-THREAD_SIZE)(reg)
+#define ASM_THREAD_INFO_MEMOP(field, reg, off) ((field)+(off)-THREAD_SIZE)(reg)
 
 #endif
 
diff --git a/arch/x86/kernel/entry_64.S b/arch/x86/kernel/entry_64.S
index 8f01a4f1cf9e..40421175e44c 100644
--- a/arch/x86/kernel/entry_64.S
+++ b/arch/x86/kernel/entry_64.S
@@ -249,7 +249,7 @@ GLOBAL(system_call_after_swapgs)
 	pushq_cfi_reg	r11			/* pt_regs->r11 */
 	sub	$(6*8),%rsp /* pt_regs->bp,bx,r12-15 not saved */
 
-	testl $_TIF_WORK_SYSCALL_ENTRY, THREAD_INFO(TI_flags, %rsp, SIZEOF_PTREGS)
+	testl $_TIF_WORK_SYSCALL_ENTRY, ASM_THREAD_INFO_MEMOP(TI_flags, %rsp, SIZEOF_PTREGS)
 	jnz tracesys
 system_call_fastpath:
 #if __SYSCALL_MASK == ~0
@@ -267,7 +267,7 @@ GLOBAL(system_call_after_swapgs)
  * Has incompletely filled pt_regs, iret frame is also incomplete.
  */
 ret_from_sys_call:
-	testl $_TIF_ALLWORK_MASK, THREAD_INFO(TI_flags, %rsp, SIZEOF_PTREGS)
+	testl $_TIF_ALLWORK_MASK, ASM_THREAD_INFO_MEMOP(TI_flags, %rsp, SIZEOF_PTREGS)
 	jnz int_ret_from_sys_call	/* Go the slow path */
 
 	LOCKDEP_SYS_EXIT

Re: [PATCH] x86/asm/entry/64: Merge the field offset into the THREAD_INFO() macro

[Denys Vlasenko]

On 03/24/2015 07:44 PM, Ingo Molnar wrote:
> From 11e2761ba0969466299b7109eba749d2292e8796 Mon Sep 17 00:00:00 2001
> From: Ingo Molnar <mingo@kernel.org>
> Date: Tue, 24 Mar 2015 19:18:41 +0100
> Subject: [PATCH] x86/asm/entry/64: Merge the field offset into the THREAD_INFO() macro
> 
> Before:
> 
>    TI_sysenter_return+THREAD_INFO(%rsp,3*8),%r10d
> 
> After:
> 
>    movl    THREAD_INFO(TI_sysenter_return, %rsp, 3*8), %r10d
> 
> to turn it into a clear thread_info accessor.

Good idea, I also wanted to do this.
I propose a more C-like order of arguments instead.
In C, field names are on the right: obj.field, ptr->field.

    THREAD_INFO(%rsp, 3*8, TI_field_name)

would suggest to the reader a pseudo-C construct:

    THREAD_INFO(sp, offset)->field_name

Re: [PATCH] x86/asm/entry/64: Improve the THREAD_INFO() macro explanation

[Denys Vlasenko]

On 03/24/2015 07:43 PM, Ingo Molnar wrote:
> From 0229a184997a7d4ad4398ee3ac2f5ae78c1c1a03 Mon Sep 17 00:00:00 2001
> From: Ingo Molnar <mingo@kernel.org>
> Date: Tue, 24 Mar 2015 18:57:13 +0100
> Subject: [PATCH] x86/asm/entry/64: Improve the THREAD_INFO() macro explanation
> 
> Explain the background, and add a real example.
> 
> Cc: Alexei Starovoitov <ast@plumgrid.com>
> Cc: Andy Lutomirski <luto@amacapital.net>
> Cc: Andy Lutomirski <luto@kernel.org>
> Cc: Borislav Petkov <bp@alien8.de>
> Cc: Borislav Petkov <bp@suse.de>
> Cc: Denys Vlasenko <dvlasenk@redhat.com>
> Cc: Frederic Weisbecker <fweisbec@gmail.com>
> Cc: H. Peter Anvin <hpa@zytor.com>
> Cc: Kees Cook <keescook@chromium.org>
> Cc: Linus Torvalds <torvalds@linux-foundation.org>
> Cc: Oleg Nesterov <oleg@redhat.com>
> Cc: Steven Rostedt <rostedt@goodmis.org>
> Cc: Will Drewry <wad@chromium.org>
> Signed-off-by: Ingo Molnar <mingo@kernel.org>
> ---
>  arch/x86/include/asm/thread_info.h | 27 +++++++++++++++++++++++----
>  1 file changed, 23 insertions(+), 4 deletions(-)
> 
> diff --git a/arch/x86/include/asm/thread_info.h b/arch/x86/include/asm/thread_info.h
> index ad0ee3423da5..813dfbb867a7 100644
> --- a/arch/x86/include/asm/thread_info.h
> +++ b/arch/x86/include/asm/thread_info.h
> @@ -206,10 +206,29 @@ static inline unsigned long current_stack_pointer(void)
>  	_ASM_SUB $(THREAD_SIZE),reg ;
>  
>  /*
> - * ASM operand which evaluates to thread_info address
> - * if it is known that "reg" is exactly "off" bytes below stack top.
> - * Example (fetch thread_info->fieldname):
> - *  mov TI_fieldname+THREAD_INFO(reg, off),%eax
> + * ASM operand which evaluates to a 'thread_info' address of
> + * the current task, if it is known that "reg" is exactly "off"
> + * bytes below the top of the stack currently.
> + *
> + * ( The kernel stack's size is known at build time, it is usually
> + *   2 or 4 pages, and the bottom  of the kernel stack contains
> + *   the thread_info structure. So to access the thread_info very
> + *   quickly from assembly code we can calculate down from the
> + *   top of the kernel stack to the bottom, using constant,
> + *   build-time calculations only. )
> + *
> + * For example, to fetch the current thread_info->flags value into %eax
> + * on x86-64 defconfig kernels, in syscall entry code where RSP is
> + * currently at exactly SIZEOF_PTREGS bytes away from the top of the
> + * stack:
> + *
> + *      mov TI_flags+THREAD_INFO(%rsp, SIZEOF_PTREGS), %eax
> + *
> + * will translate to:
> + *
> + *      8b 84 24 b8 c0 ff ff      mov    -0x3f48(%rsp), %eax
> + *
> + * which is below the current RSP by almost 16K.
>   */
>  #define THREAD_INFO(reg, off) ((off)-THREAD_SIZE)(reg)

Acked-by: Denys Vlasenko <dvlasenk@redhat.com>

Re: [PATCH] x86/asm/entry/64: Improve the THREAD_INFO() macro explanation

[Andy Lutomirski]

Acked-by: Andy Lutomirski <luto@kernel.org>

On Tue, Mar 24, 2015 at 11:50 AM, Denys Vlasenko <dvlasenk@redhat.com> wrote:
> On 03/24/2015 07:43 PM, Ingo Molnar wrote:
>> From 0229a184997a7d4ad4398ee3ac2f5ae78c1c1a03 Mon Sep 17 00:00:00 2001
>> From: Ingo Molnar <mingo@kernel.org>
>> Date: Tue, 24 Mar 2015 18:57:13 +0100
>> Subject: [PATCH] x86/asm/entry/64: Improve the THREAD_INFO() macro explanation
>>
>> Explain the background, and add a real example.
>>
>> Cc: Alexei Starovoitov <ast@plumgrid.com>
>> Cc: Andy Lutomirski <luto@amacapital.net>
>> Cc: Andy Lutomirski <luto@kernel.org>
>> Cc: Borislav Petkov <bp@alien8.de>
>> Cc: Borislav Petkov <bp@suse.de>
>> Cc: Denys Vlasenko <dvlasenk@redhat.com>
>> Cc: Frederic Weisbecker <fweisbec@gmail.com>
>> Cc: H. Peter Anvin <hpa@zytor.com>
>> Cc: Kees Cook <keescook@chromium.org>
>> Cc: Linus Torvalds <torvalds@linux-foundation.org>
>> Cc: Oleg Nesterov <oleg@redhat.com>
>> Cc: Steven Rostedt <rostedt@goodmis.org>
>> Cc: Will Drewry <wad@chromium.org>
>> Signed-off-by: Ingo Molnar <mingo@kernel.org>
>> ---
>>  arch/x86/include/asm/thread_info.h | 27 +++++++++++++++++++++++----
>>  1 file changed, 23 insertions(+), 4 deletions(-)
>>
>> diff --git a/arch/x86/include/asm/thread_info.h b/arch/x86/include/asm/thread_info.h
>> index ad0ee3423da5..813dfbb867a7 100644
>> --- a/arch/x86/include/asm/thread_info.h
>> +++ b/arch/x86/include/asm/thread_info.h
>> @@ -206,10 +206,29 @@ static inline unsigned long current_stack_pointer(void)
>>       _ASM_SUB $(THREAD_SIZE),reg ;
>>
>>  /*
>> - * ASM operand which evaluates to thread_info address
>> - * if it is known that "reg" is exactly "off" bytes below stack top.
>> - * Example (fetch thread_info->fieldname):
>> - *  mov TI_fieldname+THREAD_INFO(reg, off),%eax
>> + * ASM operand which evaluates to a 'thread_info' address of
>> + * the current task, if it is known that "reg" is exactly "off"
>> + * bytes below the top of the stack currently.
>> + *
>> + * ( The kernel stack's size is known at build time, it is usually
>> + *   2 or 4 pages, and the bottom  of the kernel stack contains
>> + *   the thread_info structure. So to access the thread_info very
>> + *   quickly from assembly code we can calculate down from the
>> + *   top of the kernel stack to the bottom, using constant,
>> + *   build-time calculations only. )
>> + *
>> + * For example, to fetch the current thread_info->flags value into %eax
>> + * on x86-64 defconfig kernels, in syscall entry code where RSP is
>> + * currently at exactly SIZEOF_PTREGS bytes away from the top of the
>> + * stack:
>> + *
>> + *      mov TI_flags+THREAD_INFO(%rsp, SIZEOF_PTREGS), %eax
>> + *
>> + * will translate to:
>> + *
>> + *      8b 84 24 b8 c0 ff ff      mov    -0x3f48(%rsp), %eax
>> + *
>> + * which is below the current RSP by almost 16K.
>>   */
>>  #define THREAD_INFO(reg, off) ((off)-THREAD_SIZE)(reg)
>
> Acked-by: Denys Vlasenko <dvlasenk@redhat.com>
>



-- 
Andy Lutomirski
AMA Capital Management, LLC

Re: [PATCH] x86/asm/entry/64: Merge the field offset into the THREAD_INFO() macro

[Andy Lutomirski]

On Tue, Mar 24, 2015 at 11:44 AM, Ingo Molnar <mingo@kernel.org> wrote:
> From 11e2761ba0969466299b7109eba749d2292e8796 Mon Sep 17 00:00:00 2001
> From: Ingo Molnar <mingo@kernel.org>
> Date: Tue, 24 Mar 2015 19:18:41 +0100
> Subject: [PATCH] x86/asm/entry/64: Merge the field offset into the THREAD_INFO() macro
>
> Before:
>
>    TI_sysenter_return+THREAD_INFO(%rsp,3*8),%r10d
>
> After:
>
>    movl    THREAD_INFO(TI_sysenter_return, %rsp, 3*8), %r10d
>
> to turn it into a clear thread_info accessor.

LGTM (in either order), but I'd fix the nice docs that you just introduced, too.

Acked-by: Andy Lutomirski <luto@kernel.org>

>
> No code changed:
>
>  md5:
>    fb4cb2b3ce05d89940ca304efc8ff183  ia32entry.o.before.asm
>    fb4cb2b3ce05d89940ca304efc8ff183  ia32entry.o.after.asm
>
>    e39f2958a5d1300158e276e4f7663263  entry_64.o.before.asm
>    e39f2958a5d1300158e276e4f7663263  entry_64.o.after.asm
>
> Cc: Alexei Starovoitov <ast@plumgrid.com>
> Cc: Andy Lutomirski <luto@amacapital.net>
> Cc: Andy Lutomirski <luto@kernel.org>
> Cc: Borislav Petkov <bp@alien8.de>
> Cc: Borislav Petkov <bp@suse.de>
> Cc: Denys Vlasenko <dvlasenk@redhat.com>
> Cc: Frederic Weisbecker <fweisbec@gmail.com>
> Cc: H. Peter Anvin <hpa@zytor.com>
> Cc: Kees Cook <keescook@chromium.org>
> Cc: Linus Torvalds <torvalds@linux-foundation.org>
> Cc: Oleg Nesterov <oleg@redhat.com>
> Cc: Steven Rostedt <rostedt@goodmis.org>
> Cc: Will Drewry <wad@chromium.org>
> Signed-off-by: Ingo Molnar <mingo@kernel.org>
> ---
>  arch/x86/ia32/ia32entry.S          | 30 +++++++++++++++---------------
>  arch/x86/include/asm/thread_info.h |  4 ++--
>  arch/x86/kernel/entry_64.S         |  4 ++--
>  3 files changed, 19 insertions(+), 19 deletions(-)
>
> diff --git a/arch/x86/ia32/ia32entry.S b/arch/x86/ia32/ia32entry.S
> index acbff3fb96a1..32e94aec6073 100644
> --- a/arch/x86/ia32/ia32entry.S
> +++ b/arch/x86/ia32/ia32entry.S
> @@ -127,7 +127,7 @@ ENTRY(ia32_sysenter_target)
>         CFI_REL_OFFSET rsp,0
>         pushfq_cfi
>         /*CFI_REL_OFFSET rflags,0*/
> -       movl    TI_sysenter_return+THREAD_INFO(%rsp,3*8),%r10d
> +       movl    THREAD_INFO(TI_sysenter_return, %rsp, 3*8), %r10d
>         CFI_REGISTER rip,r10
>         pushq_cfi $__USER32_CS
>         /*CFI_REL_OFFSET cs,0*/
> @@ -159,8 +159,8 @@ ENTRY(ia32_sysenter_target)
>         jnz sysenter_fix_flags
>  sysenter_flags_fixed:
>
> -       orl     $TS_COMPAT,TI_status+THREAD_INFO(%rsp,SIZEOF_PTREGS)
> -       testl   $_TIF_WORK_SYSCALL_ENTRY,TI_flags+THREAD_INFO(%rsp,SIZEOF_PTREGS)
> +       orl     $TS_COMPAT, THREAD_INFO(TI_status, %rsp, SIZEOF_PTREGS)
> +       testl   $_TIF_WORK_SYSCALL_ENTRY, THREAD_INFO(TI_flags, %rsp, SIZEOF_PTREGS)
>         CFI_REMEMBER_STATE
>         jnz  sysenter_tracesys
>         cmpq    $(IA32_NR_syscalls-1),%rax
> @@ -177,10 +177,10 @@ ENTRY(ia32_sysenter_target)
>         movq    %rax,RAX(%rsp)
>         DISABLE_INTERRUPTS(CLBR_NONE)
>         TRACE_IRQS_OFF
> -       testl   $_TIF_ALLWORK_MASK,TI_flags+THREAD_INFO(%rsp,SIZEOF_PTREGS)
> +       testl   $_TIF_ALLWORK_MASK, THREAD_INFO(TI_flags, %rsp, SIZEOF_PTREGS)
>         jnz     sysexit_audit
>  sysexit_from_sys_call:
> -       andl    $~TS_COMPAT,TI_status+THREAD_INFO(%rsp,SIZEOF_PTREGS)
> +       andl    $~TS_COMPAT,THREAD_INFO(TI_status, %rsp, SIZEOF_PTREGS)
>         /* clear IF, that popfq doesn't enable interrupts early */
>         andl    $~0x200,EFLAGS(%rsp)
>         movl    RIP(%rsp),%edx          /* User %eip */
> @@ -225,7 +225,7 @@ ENTRY(ia32_sysenter_target)
>         .endm
>
>         .macro auditsys_exit exit
> -       testl $(_TIF_ALLWORK_MASK & ~_TIF_SYSCALL_AUDIT),TI_flags+THREAD_INFO(%rsp,SIZEOF_PTREGS)
> +       testl $(_TIF_ALLWORK_MASK & ~_TIF_SYSCALL_AUDIT), THREAD_INFO(TI_flags, %rsp, SIZEOF_PTREGS)
>         jnz ia32_ret_from_sys_call
>         TRACE_IRQS_ON
>         ENABLE_INTERRUPTS(CLBR_NONE)
> @@ -240,7 +240,7 @@ ENTRY(ia32_sysenter_target)
>         movl $(_TIF_ALLWORK_MASK & ~_TIF_SYSCALL_AUDIT),%edi
>         DISABLE_INTERRUPTS(CLBR_NONE)
>         TRACE_IRQS_OFF
> -       testl %edi,TI_flags+THREAD_INFO(%rsp,SIZEOF_PTREGS)
> +       testl %edi, THREAD_INFO(TI_flags, %rsp, SIZEOF_PTREGS)
>         jz \exit
>         CLEAR_RREGS
>         jmp int_with_check
> @@ -262,7 +262,7 @@ ENTRY(ia32_sysenter_target)
>
>  sysenter_tracesys:
>  #ifdef CONFIG_AUDITSYSCALL
> -       testl   $(_TIF_WORK_SYSCALL_ENTRY & ~_TIF_SYSCALL_AUDIT),TI_flags+THREAD_INFO(%rsp,SIZEOF_PTREGS)
> +       testl   $(_TIF_WORK_SYSCALL_ENTRY & ~_TIF_SYSCALL_AUDIT), THREAD_INFO(TI_flags, %rsp, SIZEOF_PTREGS)
>         jz      sysenter_auditsys
>  #endif
>         SAVE_EXTRA_REGS
> @@ -346,8 +346,8 @@ ENTRY(ia32_cstar_target)
>  1:     movl    (%r8),%r9d
>         _ASM_EXTABLE(1b,ia32_badarg)
>         ASM_CLAC
> -       orl     $TS_COMPAT,TI_status+THREAD_INFO(%rsp,SIZEOF_PTREGS)
> -       testl   $_TIF_WORK_SYSCALL_ENTRY,TI_flags+THREAD_INFO(%rsp,SIZEOF_PTREGS)
> +       orl     $TS_COMPAT, THREAD_INFO(TI_status, %rsp, SIZEOF_PTREGS)
> +       testl   $_TIF_WORK_SYSCALL_ENTRY, THREAD_INFO(TI_flags, %rsp, SIZEOF_PTREGS)
>         CFI_REMEMBER_STATE
>         jnz   cstar_tracesys
>         cmpq $IA32_NR_syscalls-1,%rax
> @@ -364,10 +364,10 @@ ENTRY(ia32_cstar_target)
>         movq %rax,RAX(%rsp)
>         DISABLE_INTERRUPTS(CLBR_NONE)
>         TRACE_IRQS_OFF
> -       testl $_TIF_ALLWORK_MASK,TI_flags+THREAD_INFO(%rsp,SIZEOF_PTREGS)
> +       testl $_TIF_ALLWORK_MASK, THREAD_INFO(TI_flags, %rsp, SIZEOF_PTREGS)
>         jnz sysretl_audit
>  sysretl_from_sys_call:
> -       andl $~TS_COMPAT,TI_status+THREAD_INFO(%rsp,SIZEOF_PTREGS)
> +       andl $~TS_COMPAT, THREAD_INFO(TI_status, %rsp, SIZEOF_PTREGS)
>         RESTORE_RSI_RDI_RDX
>         movl RIP(%rsp),%ecx
>         CFI_REGISTER rip,rcx
> @@ -402,7 +402,7 @@ ENTRY(ia32_cstar_target)
>
>  cstar_tracesys:
>  #ifdef CONFIG_AUDITSYSCALL
> -       testl $(_TIF_WORK_SYSCALL_ENTRY & ~_TIF_SYSCALL_AUDIT),TI_flags+THREAD_INFO(%rsp,SIZEOF_PTREGS)
> +       testl $(_TIF_WORK_SYSCALL_ENTRY & ~_TIF_SYSCALL_AUDIT), THREAD_INFO(TI_flags, %rsp, SIZEOF_PTREGS)
>         jz cstar_auditsys
>  #endif
>         xchgl %r9d,%ebp
> @@ -469,8 +469,8 @@ ENTRY(ia32_syscall)
>            this could be a problem. */
>         ALLOC_PT_GPREGS_ON_STACK
>         SAVE_C_REGS_EXCEPT_R891011
> -       orl $TS_COMPAT,TI_status+THREAD_INFO(%rsp,SIZEOF_PTREGS)
> -       testl $_TIF_WORK_SYSCALL_ENTRY,TI_flags+THREAD_INFO(%rsp,SIZEOF_PTREGS)
> +       orl $TS_COMPAT, THREAD_INFO(TI_status, %rsp, SIZEOF_PTREGS)
> +       testl $_TIF_WORK_SYSCALL_ENTRY, THREAD_INFO(TI_flags, %rsp, SIZEOF_PTREGS)
>         jnz ia32_tracesys
>         cmpq $(IA32_NR_syscalls-1),%rax
>         ja ia32_badsys
> diff --git a/arch/x86/include/asm/thread_info.h b/arch/x86/include/asm/thread_info.h
> index 813dfbb867a7..224285b674ca 100644
> --- a/arch/x86/include/asm/thread_info.h
> +++ b/arch/x86/include/asm/thread_info.h
> @@ -222,7 +222,7 @@ static inline unsigned long current_stack_pointer(void)
>   * currently at exactly SIZEOF_PTREGS bytes away from the top of the
>   * stack:
>   *
> - *      mov TI_flags+THREAD_INFO(%rsp, SIZEOF_PTREGS), %eax
> + *      mov THREAD_INFO(TI_flags, %rsp, SIZEOF_PTREGS), %eax
>   *
>   * will translate to:
>   *
> @@ -230,7 +230,7 @@ static inline unsigned long current_stack_pointer(void)
>   *
>   * which is below the current RSP by almost 16K.
>   */
> -#define THREAD_INFO(reg, off) ((off)-THREAD_SIZE)(reg)
> +#define THREAD_INFO(field, reg, off) ((field)+(off)-THREAD_SIZE)(reg)
>
>  #endif
>
> diff --git a/arch/x86/kernel/entry_64.S b/arch/x86/kernel/entry_64.S
> index df04ee069b1f..8f01a4f1cf9e 100644
> --- a/arch/x86/kernel/entry_64.S
> +++ b/arch/x86/kernel/entry_64.S
> @@ -249,7 +249,7 @@ GLOBAL(system_call_after_swapgs)
>         pushq_cfi_reg   r11                     /* pt_regs->r11 */
>         sub     $(6*8),%rsp /* pt_regs->bp,bx,r12-15 not saved */
>
> -       testl $_TIF_WORK_SYSCALL_ENTRY,TI_flags+THREAD_INFO(%rsp,SIZEOF_PTREGS)
> +       testl $_TIF_WORK_SYSCALL_ENTRY, THREAD_INFO(TI_flags, %rsp, SIZEOF_PTREGS)
>         jnz tracesys
>  system_call_fastpath:
>  #if __SYSCALL_MASK == ~0
> @@ -267,7 +267,7 @@ GLOBAL(system_call_after_swapgs)
>   * Has incompletely filled pt_regs, iret frame is also incomplete.
>   */
>  ret_from_sys_call:
> -       testl $_TIF_ALLWORK_MASK,TI_flags+THREAD_INFO(%rsp,SIZEOF_PTREGS)
> +       testl $_TIF_ALLWORK_MASK, THREAD_INFO(TI_flags, %rsp, SIZEOF_PTREGS)
>         jnz int_ret_from_sys_call       /* Go the slow path */
>
>         LOCKDEP_SYS_EXIT



-- 
Andy Lutomirski
AMA Capital Management, LLC

Re: [PATCH] x86/asm/entry/64: Improve the THREAD_INFO() macro explanation

[Borislav Petkov]

On Tue, Mar 24, 2015 at 07:43:11PM +0100, Ingo Molnar wrote:
> From 0229a184997a7d4ad4398ee3ac2f5ae78c1c1a03 Mon Sep 17 00:00:00 2001
> From: Ingo Molnar <mingo@kernel.org>
> Date: Tue, 24 Mar 2015 18:57:13 +0100
> Subject: [PATCH] x86/asm/entry/64: Improve the THREAD_INFO() macro explanation
> 
> Explain the background, and add a real example.
> 
> Cc: Alexei Starovoitov <ast@plumgrid.com>
> Cc: Andy Lutomirski <luto@amacapital.net>
> Cc: Andy Lutomirski <luto@kernel.org>
> Cc: Borislav Petkov <bp@alien8.de>
> Cc: Borislav Petkov <bp@suse.de>
> Cc: Denys Vlasenko <dvlasenk@redhat.com>
> Cc: Frederic Weisbecker <fweisbec@gmail.com>
> Cc: H. Peter Anvin <hpa@zytor.com>
> Cc: Kees Cook <keescook@chromium.org>
> Cc: Linus Torvalds <torvalds@linux-foundation.org>
> Cc: Oleg Nesterov <oleg@redhat.com>
> Cc: Steven Rostedt <rostedt@goodmis.org>
> Cc: Will Drewry <wad@chromium.org>
> Signed-off-by: Ingo Molnar <mingo@kernel.org>
> ---
>  arch/x86/include/asm/thread_info.h | 27 +++++++++++++++++++++++----
>  1 file changed, 23 insertions(+), 4 deletions(-)
> 
> diff --git a/arch/x86/include/asm/thread_info.h b/arch/x86/include/asm/thread_info.h
> index ad0ee3423da5..813dfbb867a7 100644
> --- a/arch/x86/include/asm/thread_info.h
> +++ b/arch/x86/include/asm/thread_info.h
> @@ -206,10 +206,29 @@ static inline unsigned long current_stack_pointer(void)
>  	_ASM_SUB $(THREAD_SIZE),reg ;
>  
>  /*
> - * ASM operand which evaluates to thread_info address
> - * if it is known that "reg" is exactly "off" bytes below stack top.
> - * Example (fetch thread_info->fieldname):
> - *  mov TI_fieldname+THREAD_INFO(reg, off),%eax
> + * ASM operand which evaluates to a 'thread_info' address of
> + * the current task, if it is known that "reg" is exactly "off"
> + * bytes below the top of the stack currently.
> + *
> + * ( The kernel stack's size is known at build time, it is usually
> + *   2 or 4 pages, and the bottom  of the kernel stack contains
> + *   the thread_info structure. So to access the thread_info very
> + *   quickly from assembly code we can calculate down from the
> + *   top of the kernel stack to the bottom, using constant,
> + *   build-time calculations only. )
> + *
> + * For example, to fetch the current thread_info->flags value into %eax
> + * on x86-64 defconfig kernels, in syscall entry code where RSP is
> + * currently at exactly SIZEOF_PTREGS bytes away from the top of the
> + * stack:
> + *
> + *      mov TI_flags+THREAD_INFO(%rsp, SIZEOF_PTREGS), %eax
> + *
> + * will translate to:
> + *
> + *      8b 84 24 b8 c0 ff ff      mov    -0x3f48(%rsp), %eax
> + *
> + * which is below the current RSP by almost 16K.
>   */
>  #define THREAD_INFO(reg, off) ((off)-THREAD_SIZE)(reg)

Vehemently-with-both-thumbs-up-acked-by: Borislav Petkov <bp@suse.de>

-- 
Regards/Gruss,
    Boris.

ECO tip #101: Trim your mails when you reply.
--

Re: [PATCH] x86/asm/entry/64: Rename THREAD_INFO() to ASM_ASM_THREAD_INFO_MEMOP()

[Borislav Petkov]

On Tue, Mar 24, 2015 at 07:44:42PM +0100, Ingo Molnar wrote:
> From 234ed7356ecfa60f4e209d513d89fa9de41aa4c5 Mon Sep 17 00:00:00 2001
> From: Ingo Molnar <mingo@kernel.org>
> Date: Tue, 24 Mar 2015 19:29:16 +0100
> Subject: [PATCH] x86/asm/entry/64: Rename THREAD_INFO() to ASM_ASM_THREAD_INFO_MEMOP()
> 
> The THREAD_INFO() macro has a somewhat confusingly generic name, defined in
> a generic .h C header file. It also does not make it clear that it constructs
> a memory operand for use in assembly code.
> 
> Rename it to ASM_THREAD_INFO_MEMOP() to make it all glaringly obvious on
> first glance.

Why the MEMOP?

ASM_THREAD_INFO() looks pretty enough to me. :)

-- 
Regards/Gruss,
    Boris.

ECO tip #101: Trim your mails when you reply.
--

Re: [PATCH] x86/asm/entry/64: Merge the field offset into the THREAD_INFO() macro

[Ingo Molnar]

* Denys Vlasenko <dvlasenk@redhat.com> wrote:

> On 03/24/2015 07:44 PM, Ingo Molnar wrote:
> > From 11e2761ba0969466299b7109eba749d2292e8796 Mon Sep 17 00:00:00 2001
> > From: Ingo Molnar <mingo@kernel.org>
> > Date: Tue, 24 Mar 2015 19:18:41 +0100
> > Subject: [PATCH] x86/asm/entry/64: Merge the field offset into the THREAD_INFO() macro
> > 
> > Before:
> > 
> >    TI_sysenter_return+THREAD_INFO(%rsp,3*8),%r10d
> > 
> > After:
> > 
> >    movl    THREAD_INFO(TI_sysenter_return, %rsp, 3*8), %r10d
> > 
> > to turn it into a clear thread_info accessor.
> 
> Good idea, I also wanted to do this.
> I propose a more C-like order of arguments instead.
> In C, field names are on the right: obj.field, ptr->field.
> 
>     THREAD_INFO(%rsp, 3*8, TI_field_name)
> 
> would suggest to the reader a pseudo-C construct:
> 
>     THREAD_INFO(sp, offset)->field_name

So I picked that order, because the C code we want to emulate here 
visually is:

	thread_info->field_name

and visually this order represents just that:

	THREAD_INFO(TI_field_name, ...)

" ,%reg, offset" in that sense is just a 'detail' to how to access 
thread_info.

That order also resembles the assembly format more, which is usually 
in field(reg) order, i.e.:

	THREAD_INFO(field, %reg, ...)

Hm?

Thanks,

	Ingo

Re: [PATCH] x86/asm/entry/64: Merge the field offset into the THREAD_INFO() macro

[Denys Vlasenko]

On 03/24/2015 08:29 PM, Ingo Molnar wrote:
>>> Before:
>>>
>>>    TI_sysenter_return+THREAD_INFO(%rsp,3*8),%r10d
>>>
>>> After:
>>>
>>>    movl    THREAD_INFO(TI_sysenter_return, %rsp, 3*8), %r10d
>>>
>>> to turn it into a clear thread_info accessor.
>>
>> Good idea, I also wanted to do this.
>> I propose a more C-like order of arguments instead.
>> In C, field names are on the right: obj.field, ptr->field.
>>
>>     THREAD_INFO(%rsp, 3*8, TI_field_name)
>>
>> would suggest to the reader a pseudo-C construct:
>>
>>     THREAD_INFO(sp, offset)->field_name
> 
> So I picked that order, because the C code we want to emulate here 
> visually is:
> 
> 	thread_info->field_name
> 
> and visually this order represents just that:
> 
> 	THREAD_INFO(TI_field_name, ...)
> 
> " ,%reg, offset" in that sense is just a 'detail' to how to access 
> thread_info.
> 
> That order also resembles the assembly format more, which is usually 
> in field(reg) order, i.e.:
> 
> 	THREAD_INFO(field, %reg, ...)
> 
> Hm?

Okay.

Re: [PATCH] x86/asm/entry/64: Rename THREAD_INFO() to ASM_ASM_THREAD_INFO_MEMOP()

[Ingo Molnar]

* Borislav Petkov <bp@alien8.de> wrote:

> On Tue, Mar 24, 2015 at 07:44:42PM +0100, Ingo Molnar wrote:
> > From 234ed7356ecfa60f4e209d513d89fa9de41aa4c5 Mon Sep 17 00:00:00 2001
> > From: Ingo Molnar <mingo@kernel.org>
> > Date: Tue, 24 Mar 2015 19:29:16 +0100
> > Subject: [PATCH] x86/asm/entry/64: Rename THREAD_INFO() to ASM_ASM_THREAD_INFO_MEMOP()
> > 
> > The THREAD_INFO() macro has a somewhat confusingly generic name, defined in
> > a generic .h C header file. It also does not make it clear that it constructs
> > a memory operand for use in assembly code.
> > 
> > Rename it to ASM_THREAD_INFO_MEMOP() to make it all glaringly obvious on
> > first glance.
> 
> Why the MEMOP?
> 
> ASM_THREAD_INFO() looks pretty enough to me. :)

Ok.

Thanks,

	Ingo

[tip:x86/asm] x86/asm/entry/64: Change the THREAD_INFO() definition to not depend on KERNEL_STACK_OFFSET

[tip-bot for Denys Vlasenko]

Commit-ID:  b3fe8ba320ace38cee6859b4c015d81627254ddb
Gitweb:     http://git.kernel.org/tip/b3fe8ba320ace38cee6859b4c015d81627254ddb
Author:     Denys Vlasenko <dvlasenk@redhat.com>
AuthorDate: Thu, 19 Mar 2015 18:17:45 +0100
Committer:  Ingo Molnar <mingo@kernel.org>
CommitDate: Tue, 24 Mar 2015 19:42:37 +0100

x86/asm/entry/64: Change the THREAD_INFO() definition to not depend on KERNEL_STACK_OFFSET

This changes the THREAD_INFO() definition and all its callsites
so that they do not count stack position from
(top of stack - KERNEL_STACK_OFFSET), but from top of stack.

Semi-mysterious expressions THREAD_INFO(%rsp,RIP) - "why RIP??"
are now replaced by more logical THREAD_INFO(%rsp,SIZEOF_PTREGS)
- "calculate thread_info's address using information that
rsp is SIZEOF_PTREGS bytes below top of stack".

While at it, replace "(off)-THREAD_SIZE(reg)" with equivalent
"((off)-THREAD_SIZE)(reg)". The form without parentheses
falsely looks like we invoke THREAD_SIZE() macro.

Improve comment atop THREAD_INFO macro definition.

This patch does not change generated code (verified by objdump).

Signed-off-by: Denys Vlasenko <dvlasenk@redhat.com>
Acked-by: Borislav Petkov <bp@suse.de>
Acked-by: Andy Lutomirski <luto@kernel.org>
Cc: Alexei Starovoitov <ast@plumgrid.com>
Cc: Andy Lutomirski <luto@amacapital.net>
Cc: Borislav Petkov <bp@alien8.de>
Cc: Frederic Weisbecker <fweisbec@gmail.com>
Cc: H. Peter Anvin <hpa@zytor.com>
Cc: Kees Cook <keescook@chromium.org>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Oleg Nesterov <oleg@redhat.com>
Cc: Steven Rostedt <rostedt@goodmis.org>
Cc: Will Drewry <wad@chromium.org>
Link: http://lkml.kernel.org/r/1426785469-15125-1-git-send-email-dvlasenk@redhat.com
Signed-off-by: Ingo Molnar <mingo@kernel.org>
---
 arch/x86/ia32/ia32entry.S          | 30 +++++++++++++++---------------
 arch/x86/include/asm/thread_info.h |  8 +++++---
 arch/x86/kernel/entry_64.S         |  4 ++--
 3 files changed, 22 insertions(+), 20 deletions(-)

diff --git a/arch/x86/ia32/ia32entry.S b/arch/x86/ia32/ia32entry.S
index ad9efef..50190e1 100644
--- a/arch/x86/ia32/ia32entry.S
+++ b/arch/x86/ia32/ia32entry.S
@@ -127,7 +127,7 @@ ENTRY(ia32_sysenter_target)
 	CFI_REL_OFFSET rsp,0
 	pushfq_cfi
 	/*CFI_REL_OFFSET rflags,0*/
-	movl	TI_sysenter_return+THREAD_INFO(%rsp,3*8-KERNEL_STACK_OFFSET),%r10d
+	movl	TI_sysenter_return+THREAD_INFO(%rsp,3*8),%r10d
 	CFI_REGISTER rip,r10
 	pushq_cfi $__USER32_CS
 	/*CFI_REL_OFFSET cs,0*/
@@ -159,8 +159,8 @@ ENTRY(ia32_sysenter_target)
 	jnz sysenter_fix_flags
 sysenter_flags_fixed:
 
-	orl     $TS_COMPAT,TI_status+THREAD_INFO(%rsp,RIP)
-	testl   $_TIF_WORK_SYSCALL_ENTRY,TI_flags+THREAD_INFO(%rsp,RIP)
+	orl     $TS_COMPAT,TI_status+THREAD_INFO(%rsp,SIZEOF_PTREGS)
+	testl   $_TIF_WORK_SYSCALL_ENTRY,TI_flags+THREAD_INFO(%rsp,SIZEOF_PTREGS)
 	CFI_REMEMBER_STATE
 	jnz  sysenter_tracesys
 	cmpq	$(IA32_NR_syscalls-1),%rax
@@ -177,10 +177,10 @@ sysenter_dispatch:
 	movq	%rax,RAX(%rsp)
 	DISABLE_INTERRUPTS(CLBR_NONE)
 	TRACE_IRQS_OFF
-	testl	$_TIF_ALLWORK_MASK,TI_flags+THREAD_INFO(%rsp,RIP)
+	testl	$_TIF_ALLWORK_MASK,TI_flags+THREAD_INFO(%rsp,SIZEOF_PTREGS)
 	jnz	sysexit_audit
 sysexit_from_sys_call:
-	andl    $~TS_COMPAT,TI_status+THREAD_INFO(%rsp,RIP)
+	andl    $~TS_COMPAT,TI_status+THREAD_INFO(%rsp,SIZEOF_PTREGS)
 	/* clear IF, that popfq doesn't enable interrupts early */
 	andl	$~0x200,EFLAGS(%rsp)
 	movl	RIP(%rsp),%edx		/* User %eip */
@@ -225,7 +225,7 @@ sysexit_from_sys_call:
 	.endm
 
 	.macro auditsys_exit exit
-	testl $(_TIF_ALLWORK_MASK & ~_TIF_SYSCALL_AUDIT),TI_flags+THREAD_INFO(%rsp,RIP)
+	testl $(_TIF_ALLWORK_MASK & ~_TIF_SYSCALL_AUDIT),TI_flags+THREAD_INFO(%rsp,SIZEOF_PTREGS)
 	jnz ia32_ret_from_sys_call
 	TRACE_IRQS_ON
 	ENABLE_INTERRUPTS(CLBR_NONE)
@@ -240,7 +240,7 @@ sysexit_from_sys_call:
 	movl $(_TIF_ALLWORK_MASK & ~_TIF_SYSCALL_AUDIT),%edi
 	DISABLE_INTERRUPTS(CLBR_NONE)
 	TRACE_IRQS_OFF
-	testl %edi,TI_flags+THREAD_INFO(%rsp,RIP)
+	testl %edi,TI_flags+THREAD_INFO(%rsp,SIZEOF_PTREGS)
 	jz \exit
 	CLEAR_RREGS
 	jmp int_with_check
@@ -262,7 +262,7 @@ sysenter_fix_flags:
 
 sysenter_tracesys:
 #ifdef CONFIG_AUDITSYSCALL
-	testl	$(_TIF_WORK_SYSCALL_ENTRY & ~_TIF_SYSCALL_AUDIT),TI_flags+THREAD_INFO(%rsp,RIP)
+	testl	$(_TIF_WORK_SYSCALL_ENTRY & ~_TIF_SYSCALL_AUDIT),TI_flags+THREAD_INFO(%rsp,SIZEOF_PTREGS)
 	jz	sysenter_auditsys
 #endif
 	SAVE_EXTRA_REGS
@@ -346,8 +346,8 @@ ENTRY(ia32_cstar_target)
 1:	movl	(%r8),%r9d
 	_ASM_EXTABLE(1b,ia32_badarg)
 	ASM_CLAC
-	orl     $TS_COMPAT,TI_status+THREAD_INFO(%rsp,RIP)
-	testl   $_TIF_WORK_SYSCALL_ENTRY,TI_flags+THREAD_INFO(%rsp,RIP)
+	orl     $TS_COMPAT,TI_status+THREAD_INFO(%rsp,SIZEOF_PTREGS)
+	testl   $_TIF_WORK_SYSCALL_ENTRY,TI_flags+THREAD_INFO(%rsp,SIZEOF_PTREGS)
 	CFI_REMEMBER_STATE
 	jnz   cstar_tracesys
 	cmpq $IA32_NR_syscalls-1,%rax
@@ -364,10 +364,10 @@ cstar_dispatch:
 	movq %rax,RAX(%rsp)
 	DISABLE_INTERRUPTS(CLBR_NONE)
 	TRACE_IRQS_OFF
-	testl $_TIF_ALLWORK_MASK,TI_flags+THREAD_INFO(%rsp,RIP)
+	testl $_TIF_ALLWORK_MASK,TI_flags+THREAD_INFO(%rsp,SIZEOF_PTREGS)
 	jnz sysretl_audit
 sysretl_from_sys_call:
-	andl $~TS_COMPAT,TI_status+THREAD_INFO(%rsp,RIP)
+	andl $~TS_COMPAT,TI_status+THREAD_INFO(%rsp,SIZEOF_PTREGS)
 	RESTORE_RSI_RDI_RDX
 	movl RIP(%rsp),%ecx
 	CFI_REGISTER rip,rcx
@@ -402,7 +402,7 @@ sysretl_audit:
 
 cstar_tracesys:
 #ifdef CONFIG_AUDITSYSCALL
-	testl $(_TIF_WORK_SYSCALL_ENTRY & ~_TIF_SYSCALL_AUDIT),TI_flags+THREAD_INFO(%rsp,RIP)
+	testl $(_TIF_WORK_SYSCALL_ENTRY & ~_TIF_SYSCALL_AUDIT),TI_flags+THREAD_INFO(%rsp,SIZEOF_PTREGS)
 	jz cstar_auditsys
 #endif
 	xchgl %r9d,%ebp
@@ -469,8 +469,8 @@ ENTRY(ia32_syscall)
 	   this could be a problem. */
 	ALLOC_PT_GPREGS_ON_STACK
 	SAVE_C_REGS_EXCEPT_R891011
-	orl $TS_COMPAT,TI_status+THREAD_INFO(%rsp,RIP)
-	testl $_TIF_WORK_SYSCALL_ENTRY,TI_flags+THREAD_INFO(%rsp,RIP)
+	orl $TS_COMPAT,TI_status+THREAD_INFO(%rsp,SIZEOF_PTREGS)
+	testl $_TIF_WORK_SYSCALL_ENTRY,TI_flags+THREAD_INFO(%rsp,SIZEOF_PTREGS)
 	jnz ia32_tracesys
 	cmpq $(IA32_NR_syscalls-1),%rax
 	ja ia32_badsys
diff --git a/arch/x86/include/asm/thread_info.h b/arch/x86/include/asm/thread_info.h
index 0abf7ab..ae9c2f1 100644
--- a/arch/x86/include/asm/thread_info.h
+++ b/arch/x86/include/asm/thread_info.h
@@ -207,10 +207,12 @@ static inline unsigned long current_stack_pointer(void)
 	_ASM_SUB $(THREAD_SIZE-KERNEL_STACK_OFFSET),reg ;
 
 /*
- * Same if PER_CPU_VAR(kernel_stack) is, perhaps with some offset, already in
- * a certain register (to be used in assembler memory operands).
+ * ASM operand which evaluates to thread_info address
+ * if it is known that "reg" is exactly "off" bytes below stack top.
+ * Example (fetch thread_info->fieldname):
+ *  mov TI_fieldname+THREAD_INFO(reg, off),%eax
  */
-#define THREAD_INFO(reg, off) KERNEL_STACK_OFFSET+(off)-THREAD_SIZE(reg)
+#define THREAD_INFO(reg, off) ((off)-THREAD_SIZE)(reg)
 
 #endif
 
diff --git a/arch/x86/kernel/entry_64.S b/arch/x86/kernel/entry_64.S
index 9184392..8076df9 100644
--- a/arch/x86/kernel/entry_64.S
+++ b/arch/x86/kernel/entry_64.S
@@ -258,7 +258,7 @@ GLOBAL(system_call_after_swapgs)
 	SAVE_C_REGS_EXCEPT_RAX_RCX_R11
 	movq	$-ENOSYS,RAX(%rsp)
 	CFI_REL_OFFSET rip,RIP
-	testl $_TIF_WORK_SYSCALL_ENTRY,TI_flags+THREAD_INFO(%rsp,RIP)
+	testl $_TIF_WORK_SYSCALL_ENTRY,TI_flags+THREAD_INFO(%rsp,SIZEOF_PTREGS)
 	jnz tracesys
 system_call_fastpath:
 #if __SYSCALL_MASK == ~0
@@ -276,7 +276,7 @@ system_call_fastpath:
  * Has incompletely filled pt_regs, iret frame is also incomplete.
  */
 ret_from_sys_call:
-	testl $_TIF_ALLWORK_MASK,TI_flags+THREAD_INFO(%rsp,RIP)
+	testl $_TIF_ALLWORK_MASK,TI_flags+THREAD_INFO(%rsp,SIZEOF_PTREGS)
 	jnz int_ret_from_sys_call_fixup	/* Go the the slow path */
 
 	LOCKDEP_SYS_EXIT

[tip:x86/asm] x86/asm/entry: Get rid of KERNEL_STACK_OFFSET

[tip-bot for Denys Vlasenko]

Commit-ID:  ef593260f0cae2699874f098fb5b19fb46502cb3
Gitweb:     http://git.kernel.org/tip/ef593260f0cae2699874f098fb5b19fb46502cb3
Author:     Denys Vlasenko <dvlasenk@redhat.com>
AuthorDate: Thu, 19 Mar 2015 18:17:46 +0100
Committer:  Ingo Molnar <mingo@kernel.org>
CommitDate: Tue, 24 Mar 2015 19:42:38 +0100

x86/asm/entry: Get rid of KERNEL_STACK_OFFSET

PER_CPU_VAR(kernel_stack) was set up in a way where it points
five stack slots below the top of stack.

Presumably, it was done to avoid one "sub $5*8,%rsp"
in syscall/sysenter code paths, where iret frame needs to be
created by hand.

Ironically, none of them benefits from this optimization,
since all of them need to allocate additional data on stack
(struct pt_regs), so they still have to perform subtraction.

This patch eliminates KERNEL_STACK_OFFSET.

PER_CPU_VAR(kernel_stack) now points directly to top of stack.
pt_regs allocations are adjusted to allocate iret frame as well.
Hopefully we can merge it later with 32-bit specific
PER_CPU_VAR(cpu_current_top_of_stack) variable...

Net result in generated code is that constants in several insns
are changed.

This change is necessary for changing struct pt_regs creation
in SYSCALL64 code path from MOV to PUSH instructions.

Signed-off-by: Denys Vlasenko <dvlasenk@redhat.com>
Acked-by: Borislav Petkov <bp@suse.de>
Acked-by: Andy Lutomirski <luto@kernel.org>
Cc: Alexei Starovoitov <ast@plumgrid.com>
Cc: Andy Lutomirski <luto@amacapital.net>
Cc: Borislav Petkov <bp@alien8.de>
Cc: Frederic Weisbecker <fweisbec@gmail.com>
Cc: H. Peter Anvin <hpa@zytor.com>
Cc: Kees Cook <keescook@chromium.org>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Oleg Nesterov <oleg@redhat.com>
Cc: Steven Rostedt <rostedt@goodmis.org>
Cc: Will Drewry <wad@chromium.org>
Link: http://lkml.kernel.org/r/1426785469-15125-2-git-send-email-dvlasenk@redhat.com
Signed-off-by: Ingo Molnar <mingo@kernel.org>
---
 arch/x86/ia32/ia32entry.S          | 4 ++--
 arch/x86/include/asm/thread_info.h | 5 ++---
 arch/x86/kernel/cpu/common.c       | 2 +-
 arch/x86/kernel/entry_64.S         | 5 ++---
 arch/x86/kernel/process_32.c       | 2 +-
 arch/x86/kernel/process_64.c       | 3 +--
 arch/x86/kernel/smpboot.c          | 3 +--
 arch/x86/xen/smp.c                 | 3 +--
 8 files changed, 11 insertions(+), 16 deletions(-)

diff --git a/arch/x86/ia32/ia32entry.S b/arch/x86/ia32/ia32entry.S
index 50190e1..acbff3f 100644
--- a/arch/x86/ia32/ia32entry.S
+++ b/arch/x86/ia32/ia32entry.S
@@ -311,7 +311,7 @@ ENDPROC(ia32_sysenter_target)
 ENTRY(ia32_cstar_target)
 	CFI_STARTPROC32	simple
 	CFI_SIGNAL_FRAME
-	CFI_DEF_CFA	rsp,KERNEL_STACK_OFFSET
+	CFI_DEF_CFA	rsp,0
 	CFI_REGISTER	rip,rcx
 	/*CFI_REGISTER	rflags,r11*/
 	SWAPGS_UNSAFE_STACK
@@ -323,7 +323,7 @@ ENTRY(ia32_cstar_target)
 	 * disabled irqs and here we enable it straight after entry:
 	 */
 	ENABLE_INTERRUPTS(CLBR_NONE)
-	ALLOC_PT_GPREGS_ON_STACK 8	/* +8: space for orig_ax */
+	ALLOC_PT_GPREGS_ON_STACK 6*8 /* 6*8: space for orig_ax and iret frame */
 	SAVE_C_REGS_EXCEPT_RCX_R891011
 	movl 	%eax,%eax	/* zero extension */
 	movq	%rax,ORIG_RAX(%rsp)
diff --git a/arch/x86/include/asm/thread_info.h b/arch/x86/include/asm/thread_info.h
index ae9c2f1..ad0ee34 100644
--- a/arch/x86/include/asm/thread_info.h
+++ b/arch/x86/include/asm/thread_info.h
@@ -172,7 +172,6 @@ struct thread_info {
 #define _TIF_WORK_CTXSW_NEXT (_TIF_WORK_CTXSW)
 
 #define STACK_WARN		(THREAD_SIZE/8)
-#define KERNEL_STACK_OFFSET	(5*(BITS_PER_LONG/8))
 
 /*
  * macros/functions for gaining access to the thread information structure
@@ -201,10 +200,10 @@ static inline unsigned long current_stack_pointer(void)
 
 #else /* !__ASSEMBLY__ */
 
-/* how to get the thread information struct from ASM */
+/* Load thread_info address into "reg" */
 #define GET_THREAD_INFO(reg) \
 	_ASM_MOV PER_CPU_VAR(kernel_stack),reg ; \
-	_ASM_SUB $(THREAD_SIZE-KERNEL_STACK_OFFSET),reg ;
+	_ASM_SUB $(THREAD_SIZE),reg ;
 
 /*
  * ASM operand which evaluates to thread_info address
diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c
index 66d62ae..002216ab 100644
--- a/arch/x86/kernel/cpu/common.c
+++ b/arch/x86/kernel/cpu/common.c
@@ -1116,7 +1116,7 @@ static __init int setup_disablecpuid(char *arg)
 __setup("clearcpuid=", setup_disablecpuid);
 
 DEFINE_PER_CPU(unsigned long, kernel_stack) =
-	(unsigned long)&init_thread_union - KERNEL_STACK_OFFSET + THREAD_SIZE;
+	(unsigned long)&init_thread_union + THREAD_SIZE;
 EXPORT_PER_CPU_SYMBOL(kernel_stack);
 
 #ifdef CONFIG_X86_64
diff --git a/arch/x86/kernel/entry_64.S b/arch/x86/kernel/entry_64.S
index 8076df9..eae69bb 100644
--- a/arch/x86/kernel/entry_64.S
+++ b/arch/x86/kernel/entry_64.S
@@ -225,7 +225,7 @@ ENDPROC(native_usergs_sysret64)
 ENTRY(system_call)
 	CFI_STARTPROC	simple
 	CFI_SIGNAL_FRAME
-	CFI_DEF_CFA	rsp,KERNEL_STACK_OFFSET
+	CFI_DEF_CFA	rsp,0
 	CFI_REGISTER	rip,rcx
 	/*CFI_REGISTER	rflags,r11*/
 	SWAPGS_UNSAFE_STACK
@@ -242,9 +242,8 @@ GLOBAL(system_call_after_swapgs)
 	 * so we can enable interrupts only after we're done with using rsp_scratch:
 	 */
 	movq	%rsp,PER_CPU_VAR(rsp_scratch)
-	/* kernel_stack is set so that 5 slots (iret frame) are preallocated */
 	movq	PER_CPU_VAR(kernel_stack),%rsp
-	ALLOC_PT_GPREGS_ON_STACK 8		/* +8: space for orig_ax */
+	ALLOC_PT_GPREGS_ON_STACK 6*8 /* 6*8: space for orig_ax and iret frame */
 	movq	%rcx,RIP(%rsp)
 	movq	PER_CPU_VAR(rsp_scratch),%rcx
 	movq	%r11,EFLAGS(%rsp)
diff --git a/arch/x86/kernel/process_32.c b/arch/x86/kernel/process_32.c
index c5e9870..8ed2106 100644
--- a/arch/x86/kernel/process_32.c
+++ b/arch/x86/kernel/process_32.c
@@ -308,7 +308,7 @@ __switch_to(struct task_struct *prev_p, struct task_struct *next_p)
 	load_sp0(tss, next);
 	this_cpu_write(kernel_stack,
 		       (unsigned long)task_stack_page(next_p) +
-		       THREAD_SIZE - KERNEL_STACK_OFFSET);
+		       THREAD_SIZE);
 	this_cpu_write(cpu_current_top_of_stack,
 		       (unsigned long)task_stack_page(next_p) +
 		       THREAD_SIZE);
diff --git a/arch/x86/kernel/process_64.c b/arch/x86/kernel/process_64.c
index da8b745..4baaa97 100644
--- a/arch/x86/kernel/process_64.c
+++ b/arch/x86/kernel/process_64.c
@@ -410,8 +410,7 @@ __switch_to(struct task_struct *prev_p, struct task_struct *next_p)
 	load_sp0(tss, next);
 
 	this_cpu_write(kernel_stack,
-		  (unsigned long)task_stack_page(next_p) +
-		  THREAD_SIZE - KERNEL_STACK_OFFSET);
+		(unsigned long)task_stack_page(next_p) + THREAD_SIZE);
 
 	/*
 	 * Now maybe reload the debug registers and handle I/O bitmaps
diff --git a/arch/x86/kernel/smpboot.c b/arch/x86/kernel/smpboot.c
index 759388c..7b20ffd 100644
--- a/arch/x86/kernel/smpboot.c
+++ b/arch/x86/kernel/smpboot.c
@@ -813,8 +813,7 @@ static int do_boot_cpu(int apicid, int cpu, struct task_struct *idle)
 	initial_gs = per_cpu_offset(cpu);
 #endif
 	per_cpu(kernel_stack, cpu) =
-		(unsigned long)task_stack_page(idle) -
-		KERNEL_STACK_OFFSET + THREAD_SIZE;
+		(unsigned long)task_stack_page(idle) + THREAD_SIZE;
 	early_gdt_descr.address = (unsigned long)get_cpu_gdt_table(cpu);
 	initial_code = (unsigned long)start_secondary;
 	stack_start  = idle->thread.sp;
diff --git a/arch/x86/xen/smp.c b/arch/x86/xen/smp.c
index 08e8489..765b768 100644
--- a/arch/x86/xen/smp.c
+++ b/arch/x86/xen/smp.c
@@ -452,8 +452,7 @@ static int xen_cpu_up(unsigned int cpu, struct task_struct *idle)
 	clear_tsk_thread_flag(idle, TIF_FORK);
 #endif
 	per_cpu(kernel_stack, cpu) =
-		(unsigned long)task_stack_page(idle) -
-		KERNEL_STACK_OFFSET + THREAD_SIZE;
+		(unsigned long)task_stack_page(idle) + THREAD_SIZE;
 
 	xen_setup_runstate_info(cpu);
 	xen_setup_timer(cpu);

[tip:x86/asm] x86/asm/entry/64: Use PUSH instructions to build pt_regs on stack

[tip-bot for Denys Vlasenko]

Commit-ID:  9ed8e7d86061e7c3fb3855358d51ba4abb19ceb1
Gitweb:     http://git.kernel.org/tip/9ed8e7d86061e7c3fb3855358d51ba4abb19ceb1
Author:     Denys Vlasenko <dvlasenk@redhat.com>
AuthorDate: Thu, 19 Mar 2015 18:17:47 +0100
Committer:  Ingo Molnar <mingo@kernel.org>
CommitDate: Tue, 24 Mar 2015 19:42:38 +0100

x86/asm/entry/64: Use PUSH instructions to build pt_regs on stack

With this change, on SYSCALL64 code path we are now populating
pt_regs->cs, pt_regs->ss and pt_regs->rcx unconditionally and
therefore don't need to do that in FIXUP_TOP_OF_STACK.

We lose a number of large instructions there:

    text    data     bss     dec     hex filename
   13298       0       0   13298    33f2 entry_64_before.o
   12978       0       0   12978    32b2 entry_64.o

What's more important, we convert two "MOVQ $imm,off(%rsp)" to
"PUSH $imm" (the ones which fill pt_regs->cs,ss).

Before this patch, placing them on fast path was slowing it down
by two cycles: this form of MOV is very large, 12 bytes, and
this probably reduces decode bandwidth to one instruction per cycle
when CPU sees them.

Therefore they were living in FIXUP_TOP_OF_STACK instead (away
from fast path).

"PUSH $imm" is a small 2-byte instruction. Moving it to fast path does
not slow it down in my measurements.

Signed-off-by: Denys Vlasenko <dvlasenk@redhat.com>
Acked-by: Borislav Petkov <bp@suse.de>
Acked-by: Andy Lutomirski <luto@kernel.org>
Cc: Alexei Starovoitov <ast@plumgrid.com>
Cc: Andy Lutomirski <luto@amacapital.net>
Cc: Borislav Petkov <bp@alien8.de>
Cc: Frederic Weisbecker <fweisbec@gmail.com>
Cc: H. Peter Anvin <hpa@zytor.com>
Cc: Kees Cook <keescook@chromium.org>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Oleg Nesterov <oleg@redhat.com>
Cc: Steven Rostedt <rostedt@goodmis.org>
Cc: Will Drewry <wad@chromium.org>
Link: http://lkml.kernel.org/r/1426785469-15125-3-git-send-email-dvlasenk@redhat.com
Signed-off-by: Ingo Molnar <mingo@kernel.org>
---
 arch/x86/kernel/entry_64.S | 54 +++++++++++++++++++++++++++-------------------
 1 file changed, 32 insertions(+), 22 deletions(-)

diff --git a/arch/x86/kernel/entry_64.S b/arch/x86/kernel/entry_64.S
index eae69bb..3ea4f6d 100644
--- a/arch/x86/kernel/entry_64.S
+++ b/arch/x86/kernel/entry_64.S
@@ -126,11 +126,8 @@ ENDPROC(native_usergs_sysret64)
  * manipulation.
  */
 	.macro FIXUP_TOP_OF_STACK tmp offset=0
-	movq $__USER_DS,SS+\offset(%rsp)
-	movq $__USER_CS,CS+\offset(%rsp)
-	movq RIP+\offset(%rsp),\tmp  /* get rip */
-	movq \tmp,RCX+\offset(%rsp)  /* copy it to rcx as sysret would do */
-	movq EFLAGS+\offset(%rsp),\tmp /* ditto for rflags->r11 */
+	/* copy flags to r11 as sysret would do */
+	movq EFLAGS+\offset(%rsp),\tmp
 	movq \tmp,R11+\offset(%rsp)
 	.endm
 
@@ -214,7 +211,6 @@ ENDPROC(native_usergs_sysret64)
  * r9   arg5
  * (note: r12-r15,rbp,rbx are callee-preserved in C ABI)
  *
- * Interrupts are off on entry.
  * Only called from user space.
  *
  * When user can change pt_regs->foo always force IRET. That is because
@@ -228,6 +224,12 @@ ENTRY(system_call)
 	CFI_DEF_CFA	rsp,0
 	CFI_REGISTER	rip,rcx
 	/*CFI_REGISTER	rflags,r11*/
+
+	/*
+	 * Interrupts are off on entry.
+	 * We do not frame this tiny irq-off block with TRACE_IRQS_OFF/ON,
+	 * it is too small to ever cause noticeable irq latency.
+	 */
 	SWAPGS_UNSAFE_STACK
 	/*
 	 * A hypervisor implementation might want to use a label
@@ -236,27 +238,35 @@ ENTRY(system_call)
 	 */
 GLOBAL(system_call_after_swapgs)
 
-	/*
-	 * We use 'rsp_scratch' as a scratch register, hence this block must execute
-	 * atomically in the face of possible interrupt-driven task preemption,
-	 * so we can enable interrupts only after we're done with using rsp_scratch:
-	 */
 	movq	%rsp,PER_CPU_VAR(rsp_scratch)
 	movq	PER_CPU_VAR(kernel_stack),%rsp
-	ALLOC_PT_GPREGS_ON_STACK 6*8 /* 6*8: space for orig_ax and iret frame */
-	movq	%rcx,RIP(%rsp)
-	movq	PER_CPU_VAR(rsp_scratch),%rcx
-	movq	%r11,EFLAGS(%rsp)
-	movq	%rcx,RSP(%rsp)
+
+	/* Construct struct pt_regs on stack */
+	pushq_cfi $__USER_DS			/* pt_regs->ss */
+	pushq_cfi PER_CPU_VAR(rsp_scratch)	/* pt_regs->sp */
 	/*
-	 * No need to follow this irqs off/on section - it's straight
-	 * and short:
+	 * Re-enable interrupts.
+	 * We use 'rsp_scratch' as a scratch space, hence irq-off block above
+	 * must execute atomically in the face of possible interrupt-driven
+	 * task preemption. We must enable interrupts only after we're done
+	 * with using rsp_scratch:
 	 */
 	ENABLE_INTERRUPTS(CLBR_NONE)
-	movq_cfi rax,ORIG_RAX
-	SAVE_C_REGS_EXCEPT_RAX_RCX_R11
-	movq	$-ENOSYS,RAX(%rsp)
-	CFI_REL_OFFSET rip,RIP
+	pushq_cfi	%r11			/* pt_regs->flags */
+	pushq_cfi	$__USER_CS		/* pt_regs->cs */
+	pushq_cfi	%rcx			/* pt_regs->ip */
+	CFI_REL_OFFSET rip,0
+	pushq_cfi_reg	rax			/* pt_regs->orig_ax */
+	pushq_cfi_reg	rdi			/* pt_regs->di */
+	pushq_cfi_reg	rsi			/* pt_regs->si */
+	pushq_cfi_reg	rdx			/* pt_regs->dx */
+	pushq_cfi_reg	rcx			/* pt_regs->cx */
+	pushq_cfi	$-ENOSYS		/* pt_regs->ax */
+	pushq_cfi_reg	r8			/* pt_regs->r8 */
+	pushq_cfi_reg	r9			/* pt_regs->r9 */
+	pushq_cfi_reg	r10			/* pt_regs->r10 */
+	sub	$(7*8),%rsp /* pt_regs->r11,bp,bx,r12-15 not saved */
+
 	testl $_TIF_WORK_SYSCALL_ENTRY,TI_flags+THREAD_INFO(%rsp,SIZEOF_PTREGS)
 	jnz tracesys
 system_call_fastpath:

[tip:x86/asm] x86/asm/entry/64: Get rid of the FIXUP_TOP_OF_STACK /RESTORE_TOP_OF_STACK macros

[tip-bot for Denys Vlasenko]

Commit-ID:  a71ffdd780760dc62c3d4cffb98eaaedaf5068b8
Gitweb:     http://git.kernel.org/tip/a71ffdd780760dc62c3d4cffb98eaaedaf5068b8
Author:     Denys Vlasenko <dvlasenk@redhat.com>
AuthorDate: Thu, 19 Mar 2015 18:17:48 +0100
Committer:  Ingo Molnar <mingo@kernel.org>
CommitDate: Tue, 24 Mar 2015 19:42:38 +0100

x86/asm/entry/64: Get rid of the FIXUP_TOP_OF_STACK/RESTORE_TOP_OF_STACK macros

The FIXUP_TOP_OF_STACK macro is only necessary because we don't save %r11
to pt_regs->r11 on SYSCALL64 fast path, but we want ptrace to see it populated.

Bite the bullet, add a single additional PUSH instruction, and remove
the FIXUP_TOP_OF_STACK macro.

The RESTORE_TOP_OF_STACK macro is already a nop. Remove it too.

On SandyBridge CPU, it does not get slower:
measured 54.22 ns per getpid syscall before and after last two
changes on defconfig kernel.

Signed-off-by: Denys Vlasenko <dvlasenk@redhat.com>
Acked-by: Borislav Petkov <bp@suse.de>
Acked-by: Andy Lutomirski <luto@kernel.org>
Cc: Alexei Starovoitov <ast@plumgrid.com>
Cc: Andy Lutomirski <luto@amacapital.net>
Cc: Borislav Petkov <bp@alien8.de>
Cc: Frederic Weisbecker <fweisbec@gmail.com>
Cc: H. Peter Anvin <hpa@zytor.com>
Cc: Kees Cook <keescook@chromium.org>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Oleg Nesterov <oleg@redhat.com>
Cc: Steven Rostedt <rostedt@goodmis.org>
Cc: Will Drewry <wad@chromium.org>
Link: http://lkml.kernel.org/r/1426785469-15125-4-git-send-email-dvlasenk@redhat.com
Signed-off-by: Ingo Molnar <mingo@kernel.org>
---
 arch/x86/kernel/entry_64.S | 35 ++---------------------------------
 1 file changed, 2 insertions(+), 33 deletions(-)

diff --git a/arch/x86/kernel/entry_64.S b/arch/x86/kernel/entry_64.S
index 3ea4f6d..3f8daba 100644
--- a/arch/x86/kernel/entry_64.S
+++ b/arch/x86/kernel/entry_64.S
@@ -22,8 +22,6 @@
  * - CFI macros are used to generate dwarf2 unwind information for better
  * backtraces. They don't change any code.
  * - ENTRY/END Define functions in the symbol table.
- * - FIXUP_TOP_OF_STACK/RESTORE_TOP_OF_STACK - Fix up the hardware stack
- * frame that is otherwise undefined after a SYSCALL
  * - TRACE_IRQ_* - Trace hard interrupt state for lock debugging.
  * - idtentry - Define exception entry points.
  */
@@ -119,23 +117,6 @@ ENDPROC(native_usergs_sysret64)
 #endif
 
 /*
- * C code is not supposed to know that the iret frame is not populated.
- * Every time a C function with an pt_regs argument is called from
- * the SYSCALL based fast path FIXUP_TOP_OF_STACK is needed.
- * RESTORE_TOP_OF_STACK syncs the syscall state after any possible ptregs
- * manipulation.
- */
-	.macro FIXUP_TOP_OF_STACK tmp offset=0
-	/* copy flags to r11 as sysret would do */
-	movq EFLAGS+\offset(%rsp),\tmp
-	movq \tmp,R11+\offset(%rsp)
-	.endm
-
-	.macro RESTORE_TOP_OF_STACK tmp offset=0
-	/* nothing to do */
-	.endm
-
-/*
  * empty frame
  */
 	.macro EMPTY_FRAME start=1 offset=0
@@ -265,7 +246,8 @@ GLOBAL(system_call_after_swapgs)
 	pushq_cfi_reg	r8			/* pt_regs->r8 */
 	pushq_cfi_reg	r9			/* pt_regs->r9 */
 	pushq_cfi_reg	r10			/* pt_regs->r10 */
-	sub	$(7*8),%rsp /* pt_regs->r11,bp,bx,r12-15 not saved */
+	pushq_cfi_reg	r11			/* pt_regs->r11 */
+	sub	$(6*8),%rsp /* pt_regs->bp,bx,r12-15 not saved */
 
 	testl $_TIF_WORK_SYSCALL_ENTRY,TI_flags+THREAD_INFO(%rsp,SIZEOF_PTREGS)
 	jnz tracesys
@@ -312,7 +294,6 @@ ret_from_sys_call:
 	CFI_RESTORE_STATE
 
 int_ret_from_sys_call_fixup:
-	FIXUP_TOP_OF_STACK %r11
 	jmp int_ret_from_sys_call
 
 	/* Do syscall entry tracing */
@@ -328,7 +309,6 @@ tracesys:
 
 tracesys_phase2:
 	SAVE_EXTRA_REGS
-	FIXUP_TOP_OF_STACK %rdi
 	movq %rsp, %rdi
 	movq $AUDIT_ARCH_X86_64, %rsi
 	movq %rax,%rdx
@@ -421,9 +401,7 @@ ENTRY(stub_\func)
 	CFI_STARTPROC
 	DEFAULT_FRAME 0, 8		/* offset 8: return address */
 	SAVE_EXTRA_REGS 8
-	FIXUP_TOP_OF_STACK %r11, 8
 	call sys_\func
-	RESTORE_TOP_OF_STACK %r11, 8
 	ret
 	CFI_ENDPROC
 END(stub_\func)
@@ -438,7 +416,6 @@ ENTRY(stub_execve)
 	addq $8, %rsp
 	DEFAULT_FRAME 0
 	SAVE_EXTRA_REGS
-	FIXUP_TOP_OF_STACK %r11
 	call sys_execve
 	movq %rax,RAX(%rsp)
 	RESTORE_EXTRA_REGS
@@ -451,9 +428,7 @@ ENTRY(stub_execveat)
 	addq $8, %rsp
 	DEFAULT_FRAME 0
 	SAVE_EXTRA_REGS
-	FIXUP_TOP_OF_STACK %r11
 	call sys_execveat
-	RESTORE_TOP_OF_STACK %r11
 	movq %rax,RAX(%rsp)
 	RESTORE_EXTRA_REGS
 	jmp int_ret_from_sys_call
@@ -469,7 +444,6 @@ ENTRY(stub_rt_sigreturn)
 	addq $8, %rsp
 	DEFAULT_FRAME 0
 	SAVE_EXTRA_REGS
-	FIXUP_TOP_OF_STACK %r11
 	call sys_rt_sigreturn
 	movq %rax,RAX(%rsp) # fixme, this could be done at the higher layer
 	RESTORE_EXTRA_REGS
@@ -483,7 +457,6 @@ ENTRY(stub_x32_rt_sigreturn)
 	addq $8, %rsp
 	DEFAULT_FRAME 0
 	SAVE_EXTRA_REGS
-	FIXUP_TOP_OF_STACK %r11
 	call sys32_x32_rt_sigreturn
 	movq %rax,RAX(%rsp) # fixme, this could be done at the higher layer
 	RESTORE_EXTRA_REGS
@@ -496,9 +469,7 @@ ENTRY(stub_x32_execve)
 	addq $8, %rsp
 	DEFAULT_FRAME 0
 	SAVE_EXTRA_REGS
-	FIXUP_TOP_OF_STACK %r11
 	call compat_sys_execve
-	RESTORE_TOP_OF_STACK %r11
 	movq %rax,RAX(%rsp)
 	RESTORE_EXTRA_REGS
 	jmp int_ret_from_sys_call
@@ -510,9 +481,7 @@ ENTRY(stub_x32_execveat)
 	addq $8, %rsp
 	DEFAULT_FRAME 0
 	SAVE_EXTRA_REGS
-	FIXUP_TOP_OF_STACK %r11
 	call compat_sys_execveat
-	RESTORE_TOP_OF_STACK %r11
 	movq %rax,RAX(%rsp)
 	RESTORE_EXTRA_REGS
 	jmp int_ret_from_sys_call

[tip:x86/asm] x86/asm/entry/64: Get rid of int_ret_from_sys_call_fixup

[tip-bot for Denys Vlasenko]

Commit-ID:  65c2377486c0b68f149f7d8770499a86b15786b6
Gitweb:     http://git.kernel.org/tip/65c2377486c0b68f149f7d8770499a86b15786b6
Author:     Denys Vlasenko <dvlasenk@redhat.com>
AuthorDate: Thu, 19 Mar 2015 18:17:49 +0100
Committer:  Ingo Molnar <mingo@kernel.org>
CommitDate: Tue, 24 Mar 2015 19:42:38 +0100

x86/asm/entry/64: Get rid of int_ret_from_sys_call_fixup

With the FIXUP_TOP_OF_STACK macro removed, this intermediate jump
is unnecessary.

Signed-off-by: Denys Vlasenko <dvlasenk@redhat.com>
Acked-by: Borislav Petkov <bp@suse.de>
Acked-by: Andy Lutomirski <luto@kernel.org>
Cc: Alexei Starovoitov <ast@plumgrid.com>
Cc: Andy Lutomirski <luto@amacapital.net>
Cc: Borislav Petkov <bp@alien8.de>
Cc: Frederic Weisbecker <fweisbec@gmail.com>
Cc: H. Peter Anvin <hpa@zytor.com>
Cc: Kees Cook <keescook@chromium.org>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Oleg Nesterov <oleg@redhat.com>
Cc: Steven Rostedt <rostedt@goodmis.org>
Cc: Will Drewry <wad@chromium.org>
Link: http://lkml.kernel.org/r/1426785469-15125-5-git-send-email-dvlasenk@redhat.com
Signed-off-by: Ingo Molnar <mingo@kernel.org>
---
 arch/x86/kernel/entry_64.S | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/arch/x86/kernel/entry_64.S b/arch/x86/kernel/entry_64.S
index 3f8daba..df04ee06 100644
--- a/arch/x86/kernel/entry_64.S
+++ b/arch/x86/kernel/entry_64.S
@@ -268,7 +268,7 @@ system_call_fastpath:
  */
 ret_from_sys_call:
 	testl $_TIF_ALLWORK_MASK,TI_flags+THREAD_INFO(%rsp,SIZEOF_PTREGS)
-	jnz int_ret_from_sys_call_fixup	/* Go the the slow path */
+	jnz int_ret_from_sys_call	/* Go the slow path */
 
 	LOCKDEP_SYS_EXIT
 	DISABLE_INTERRUPTS(CLBR_NONE)
@@ -293,9 +293,6 @@ ret_from_sys_call:
 
 	CFI_RESTORE_STATE
 
-int_ret_from_sys_call_fixup:
-	jmp int_ret_from_sys_call
-
 	/* Do syscall entry tracing */
 tracesys:
 	movq %rsp, %rdi

[tip:x86/asm] x86/asm/entry/64: Improve the THREAD_INFO() macro explanation

[tip-bot for Ingo Molnar]

Commit-ID:  1ddc6f3c60d75a7577dd33bc441e309febe2fc76
Gitweb:     http://git.kernel.org/tip/1ddc6f3c60d75a7577dd33bc441e309febe2fc76
Author:     Ingo Molnar <mingo@kernel.org>
AuthorDate: Tue, 24 Mar 2015 19:43:11 +0100
Committer:  Ingo Molnar <mingo@kernel.org>
CommitDate: Tue, 24 Mar 2015 20:57:30 +0100

x86/asm/entry/64: Improve the THREAD_INFO() macro explanation

Explain the background, and add a real example.

Acked-by: Denys Vlasenko <dvlasenk@redhat.com>
Acked-by: Andy Lutomirski <luto@kernel.org>
Acked-by: Borislav Petkov <bp@suse.de>
Cc: Alexei Starovoitov <ast@plumgrid.com>
Cc: Andy Lutomirski <luto@amacapital.net>
Cc: Borislav Petkov <bp@alien8.de>
Cc: Frederic Weisbecker <fweisbec@gmail.com>
Cc: H. Peter Anvin <hpa@zytor.com>
Cc: Kees Cook <keescook@chromium.org>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Oleg Nesterov <oleg@redhat.com>
Cc: Steven Rostedt <rostedt@goodmis.org>
Cc: Will Drewry <wad@chromium.org>
Link: http://lkml.kernel.org/r/20150324184311.GA14760@gmail.com
Signed-off-by: Ingo Molnar <mingo@kernel.org>
---
 arch/x86/include/asm/thread_info.h | 27 +++++++++++++++++++++++----
 1 file changed, 23 insertions(+), 4 deletions(-)

diff --git a/arch/x86/include/asm/thread_info.h b/arch/x86/include/asm/thread_info.h
index ad0ee34..813dfbb 100644
--- a/arch/x86/include/asm/thread_info.h
+++ b/arch/x86/include/asm/thread_info.h
@@ -206,10 +206,29 @@ static inline unsigned long current_stack_pointer(void)
 	_ASM_SUB $(THREAD_SIZE),reg ;
 
 /*
- * ASM operand which evaluates to thread_info address
- * if it is known that "reg" is exactly "off" bytes below stack top.
- * Example (fetch thread_info->fieldname):
- *  mov TI_fieldname+THREAD_INFO(reg, off),%eax
+ * ASM operand which evaluates to a 'thread_info' address of
+ * the current task, if it is known that "reg" is exactly "off"
+ * bytes below the top of the stack currently.
+ *
+ * ( The kernel stack's size is known at build time, it is usually
+ *   2 or 4 pages, and the bottom  of the kernel stack contains
+ *   the thread_info structure. So to access the thread_info very
+ *   quickly from assembly code we can calculate down from the
+ *   top of the kernel stack to the bottom, using constant,
+ *   build-time calculations only. )
+ *
+ * For example, to fetch the current thread_info->flags value into %eax
+ * on x86-64 defconfig kernels, in syscall entry code where RSP is
+ * currently at exactly SIZEOF_PTREGS bytes away from the top of the
+ * stack:
+ *
+ *      mov TI_flags+THREAD_INFO(%rsp, SIZEOF_PTREGS), %eax
+ *
+ * will translate to:
+ *
+ *      8b 84 24 b8 c0 ff ff      mov    -0x3f48(%rsp), %eax
+ *
+ * which is below the current RSP by almost 16K.
  */
 #define THREAD_INFO(reg, off) ((off)-THREAD_SIZE)(reg)
 

[tip:x86/asm] x86/asm/entry/64: Merge the field offset into the THREAD_INFO() macro

[tip-bot for Ingo Molnar]

Commit-ID:  f9d71854b4fe9b22ca199c4676da5a6ece1e5c17
Gitweb:     http://git.kernel.org/tip/f9d71854b4fe9b22ca199c4676da5a6ece1e5c17
Author:     Ingo Molnar <mingo@kernel.org>
AuthorDate: Tue, 24 Mar 2015 19:44:11 +0100
Committer:  Ingo Molnar <mingo@kernel.org>
CommitDate: Tue, 24 Mar 2015 20:57:31 +0100

x86/asm/entry/64: Merge the field offset into the THREAD_INFO() macro

Before:

   TI_sysenter_return+THREAD_INFO(%rsp,3*8),%r10d

After:

   movl    THREAD_INFO(TI_sysenter_return, %rsp, 3*8), %r10d

to turn it into a clear thread_info accessor.

No code changed:

 md5:
   fb4cb2b3ce05d89940ca304efc8ff183  ia32entry.o.before.asm
   fb4cb2b3ce05d89940ca304efc8ff183  ia32entry.o.after.asm

   e39f2958a5d1300158e276e4f7663263  entry_64.o.before.asm
   e39f2958a5d1300158e276e4f7663263  entry_64.o.after.asm

Acked-by: Andy Lutomirski <luto@kernel.org>
Acked-by: Denys Vlasenko <dvlasenk@redhat.com>
Cc: Alexei Starovoitov <ast@plumgrid.com>
Cc: Andy Lutomirski <luto@amacapital.net>
Cc: Borislav Petkov <bp@alien8.de>
Cc: Borislav Petkov <bp@suse.de>
Cc: Frederic Weisbecker <fweisbec@gmail.com>
Cc: H. Peter Anvin <hpa@zytor.com>
Cc: Kees Cook <keescook@chromium.org>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Oleg Nesterov <oleg@redhat.com>
Cc: Steven Rostedt <rostedt@goodmis.org>
Cc: Will Drewry <wad@chromium.org>
Link: http://lkml.kernel.org/r/20150324184411.GB14760@gmail.com
Signed-off-by: Ingo Molnar <mingo@kernel.org>
---
 arch/x86/ia32/ia32entry.S          | 30 +++++++++++++++---------------
 arch/x86/include/asm/thread_info.h |  4 ++--
 arch/x86/kernel/entry_64.S         |  4 ++--
 3 files changed, 19 insertions(+), 19 deletions(-)

diff --git a/arch/x86/ia32/ia32entry.S b/arch/x86/ia32/ia32entry.S
index acbff3f..32e94ae 100644
--- a/arch/x86/ia32/ia32entry.S
+++ b/arch/x86/ia32/ia32entry.S
@@ -127,7 +127,7 @@ ENTRY(ia32_sysenter_target)
 	CFI_REL_OFFSET rsp,0
 	pushfq_cfi
 	/*CFI_REL_OFFSET rflags,0*/
-	movl	TI_sysenter_return+THREAD_INFO(%rsp,3*8),%r10d
+	movl	THREAD_INFO(TI_sysenter_return, %rsp, 3*8), %r10d
 	CFI_REGISTER rip,r10
 	pushq_cfi $__USER32_CS
 	/*CFI_REL_OFFSET cs,0*/
@@ -159,8 +159,8 @@ ENTRY(ia32_sysenter_target)
 	jnz sysenter_fix_flags
 sysenter_flags_fixed:
 
-	orl     $TS_COMPAT,TI_status+THREAD_INFO(%rsp,SIZEOF_PTREGS)
-	testl   $_TIF_WORK_SYSCALL_ENTRY,TI_flags+THREAD_INFO(%rsp,SIZEOF_PTREGS)
+	orl     $TS_COMPAT, THREAD_INFO(TI_status, %rsp, SIZEOF_PTREGS)
+	testl   $_TIF_WORK_SYSCALL_ENTRY, THREAD_INFO(TI_flags, %rsp, SIZEOF_PTREGS)
 	CFI_REMEMBER_STATE
 	jnz  sysenter_tracesys
 	cmpq	$(IA32_NR_syscalls-1),%rax
@@ -177,10 +177,10 @@ sysenter_dispatch:
 	movq	%rax,RAX(%rsp)
 	DISABLE_INTERRUPTS(CLBR_NONE)
 	TRACE_IRQS_OFF
-	testl	$_TIF_ALLWORK_MASK,TI_flags+THREAD_INFO(%rsp,SIZEOF_PTREGS)
+	testl	$_TIF_ALLWORK_MASK, THREAD_INFO(TI_flags, %rsp, SIZEOF_PTREGS)
 	jnz	sysexit_audit
 sysexit_from_sys_call:
-	andl    $~TS_COMPAT,TI_status+THREAD_INFO(%rsp,SIZEOF_PTREGS)
+	andl    $~TS_COMPAT,THREAD_INFO(TI_status, %rsp, SIZEOF_PTREGS)
 	/* clear IF, that popfq doesn't enable interrupts early */
 	andl	$~0x200,EFLAGS(%rsp)
 	movl	RIP(%rsp),%edx		/* User %eip */
@@ -225,7 +225,7 @@ sysexit_from_sys_call:
 	.endm
 
 	.macro auditsys_exit exit
-	testl $(_TIF_ALLWORK_MASK & ~_TIF_SYSCALL_AUDIT),TI_flags+THREAD_INFO(%rsp,SIZEOF_PTREGS)
+	testl $(_TIF_ALLWORK_MASK & ~_TIF_SYSCALL_AUDIT), THREAD_INFO(TI_flags, %rsp, SIZEOF_PTREGS)
 	jnz ia32_ret_from_sys_call
 	TRACE_IRQS_ON
 	ENABLE_INTERRUPTS(CLBR_NONE)
@@ -240,7 +240,7 @@ sysexit_from_sys_call:
 	movl $(_TIF_ALLWORK_MASK & ~_TIF_SYSCALL_AUDIT),%edi
 	DISABLE_INTERRUPTS(CLBR_NONE)
 	TRACE_IRQS_OFF
-	testl %edi,TI_flags+THREAD_INFO(%rsp,SIZEOF_PTREGS)
+	testl %edi, THREAD_INFO(TI_flags, %rsp, SIZEOF_PTREGS)
 	jz \exit
 	CLEAR_RREGS
 	jmp int_with_check
@@ -262,7 +262,7 @@ sysenter_fix_flags:
 
 sysenter_tracesys:
 #ifdef CONFIG_AUDITSYSCALL
-	testl	$(_TIF_WORK_SYSCALL_ENTRY & ~_TIF_SYSCALL_AUDIT),TI_flags+THREAD_INFO(%rsp,SIZEOF_PTREGS)
+	testl	$(_TIF_WORK_SYSCALL_ENTRY & ~_TIF_SYSCALL_AUDIT), THREAD_INFO(TI_flags, %rsp, SIZEOF_PTREGS)
 	jz	sysenter_auditsys
 #endif
 	SAVE_EXTRA_REGS
@@ -346,8 +346,8 @@ ENTRY(ia32_cstar_target)
 1:	movl	(%r8),%r9d
 	_ASM_EXTABLE(1b,ia32_badarg)
 	ASM_CLAC
-	orl     $TS_COMPAT,TI_status+THREAD_INFO(%rsp,SIZEOF_PTREGS)
-	testl   $_TIF_WORK_SYSCALL_ENTRY,TI_flags+THREAD_INFO(%rsp,SIZEOF_PTREGS)
+	orl     $TS_COMPAT, THREAD_INFO(TI_status, %rsp, SIZEOF_PTREGS)
+	testl   $_TIF_WORK_SYSCALL_ENTRY, THREAD_INFO(TI_flags, %rsp, SIZEOF_PTREGS)
 	CFI_REMEMBER_STATE
 	jnz   cstar_tracesys
 	cmpq $IA32_NR_syscalls-1,%rax
@@ -364,10 +364,10 @@ cstar_dispatch:
 	movq %rax,RAX(%rsp)
 	DISABLE_INTERRUPTS(CLBR_NONE)
 	TRACE_IRQS_OFF
-	testl $_TIF_ALLWORK_MASK,TI_flags+THREAD_INFO(%rsp,SIZEOF_PTREGS)
+	testl $_TIF_ALLWORK_MASK, THREAD_INFO(TI_flags, %rsp, SIZEOF_PTREGS)
 	jnz sysretl_audit
 sysretl_from_sys_call:
-	andl $~TS_COMPAT,TI_status+THREAD_INFO(%rsp,SIZEOF_PTREGS)
+	andl $~TS_COMPAT, THREAD_INFO(TI_status, %rsp, SIZEOF_PTREGS)
 	RESTORE_RSI_RDI_RDX
 	movl RIP(%rsp),%ecx
 	CFI_REGISTER rip,rcx
@@ -402,7 +402,7 @@ sysretl_audit:
 
 cstar_tracesys:
 #ifdef CONFIG_AUDITSYSCALL
-	testl $(_TIF_WORK_SYSCALL_ENTRY & ~_TIF_SYSCALL_AUDIT),TI_flags+THREAD_INFO(%rsp,SIZEOF_PTREGS)
+	testl $(_TIF_WORK_SYSCALL_ENTRY & ~_TIF_SYSCALL_AUDIT), THREAD_INFO(TI_flags, %rsp, SIZEOF_PTREGS)
 	jz cstar_auditsys
 #endif
 	xchgl %r9d,%ebp
@@ -469,8 +469,8 @@ ENTRY(ia32_syscall)
 	   this could be a problem. */
 	ALLOC_PT_GPREGS_ON_STACK
 	SAVE_C_REGS_EXCEPT_R891011
-	orl $TS_COMPAT,TI_status+THREAD_INFO(%rsp,SIZEOF_PTREGS)
-	testl $_TIF_WORK_SYSCALL_ENTRY,TI_flags+THREAD_INFO(%rsp,SIZEOF_PTREGS)
+	orl $TS_COMPAT, THREAD_INFO(TI_status, %rsp, SIZEOF_PTREGS)
+	testl $_TIF_WORK_SYSCALL_ENTRY, THREAD_INFO(TI_flags, %rsp, SIZEOF_PTREGS)
 	jnz ia32_tracesys
 	cmpq $(IA32_NR_syscalls-1),%rax
 	ja ia32_badsys
diff --git a/arch/x86/include/asm/thread_info.h b/arch/x86/include/asm/thread_info.h
index 813dfbb..224285b 100644
--- a/arch/x86/include/asm/thread_info.h
+++ b/arch/x86/include/asm/thread_info.h
@@ -222,7 +222,7 @@ static inline unsigned long current_stack_pointer(void)
  * currently at exactly SIZEOF_PTREGS bytes away from the top of the
  * stack:
  *
- *      mov TI_flags+THREAD_INFO(%rsp, SIZEOF_PTREGS), %eax
+ *      mov THREAD_INFO(TI_flags, %rsp, SIZEOF_PTREGS), %eax
  *
  * will translate to:
  *
@@ -230,7 +230,7 @@ static inline unsigned long current_stack_pointer(void)
  *
  * which is below the current RSP by almost 16K.
  */
-#define THREAD_INFO(reg, off) ((off)-THREAD_SIZE)(reg)
+#define THREAD_INFO(field, reg, off) ((field)+(off)-THREAD_SIZE)(reg)
 
 #endif
 
diff --git a/arch/x86/kernel/entry_64.S b/arch/x86/kernel/entry_64.S
index df04ee06..8f01a4f 100644
--- a/arch/x86/kernel/entry_64.S
+++ b/arch/x86/kernel/entry_64.S
@@ -249,7 +249,7 @@ GLOBAL(system_call_after_swapgs)
 	pushq_cfi_reg	r11			/* pt_regs->r11 */
 	sub	$(6*8),%rsp /* pt_regs->bp,bx,r12-15 not saved */
 
-	testl $_TIF_WORK_SYSCALL_ENTRY,TI_flags+THREAD_INFO(%rsp,SIZEOF_PTREGS)
+	testl $_TIF_WORK_SYSCALL_ENTRY, THREAD_INFO(TI_flags, %rsp, SIZEOF_PTREGS)
 	jnz tracesys
 system_call_fastpath:
 #if __SYSCALL_MASK == ~0
@@ -267,7 +267,7 @@ system_call_fastpath:
  * Has incompletely filled pt_regs, iret frame is also incomplete.
  */
 ret_from_sys_call:
-	testl $_TIF_ALLWORK_MASK,TI_flags+THREAD_INFO(%rsp,SIZEOF_PTREGS)
+	testl $_TIF_ALLWORK_MASK, THREAD_INFO(TI_flags, %rsp, SIZEOF_PTREGS)
 	jnz int_ret_from_sys_call	/* Go the slow path */
 
 	LOCKDEP_SYS_EXIT

[tip:x86/asm] x86/asm/entry/64: Rename THREAD_INFO() to ASM_THREAD_INFO()

[tip-bot for Ingo Molnar]

Commit-ID:  dca5b52ad76b10c3adc29e2a006d4b1721c44a8d
Gitweb:     http://git.kernel.org/tip/dca5b52ad76b10c3adc29e2a006d4b1721c44a8d
Author:     Ingo Molnar <mingo@kernel.org>
AuthorDate: Tue, 24 Mar 2015 19:44:42 +0100
Committer:  Ingo Molnar <mingo@kernel.org>
CommitDate: Tue, 24 Mar 2015 20:57:31 +0100

x86/asm/entry/64: Rename THREAD_INFO() to ASM_THREAD_INFO()

The THREAD_INFO() macro has a somewhat confusingly generic name,
defined in a generic .h C header file. It also does not make it
clear that it constructs a memory operand for use in assembly
code.

Rename it to ASM_THREAD_INFO() to make it all glaringly
obvious on first glance.

Acked-by: Borislav Petkov <bp@suse.de>
Cc: Alexei Starovoitov <ast@plumgrid.com>
Cc: Andy Lutomirski <luto@amacapital.net>
Cc: Andy Lutomirski <luto@kernel.org>
Cc: Borislav Petkov <bp@alien8.de>
Cc: Denys Vlasenko <dvlasenk@redhat.com>
Cc: Frederic Weisbecker <fweisbec@gmail.com>
Cc: H. Peter Anvin <hpa@zytor.com>
Cc: Kees Cook <keescook@chromium.org>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Oleg Nesterov <oleg@redhat.com>
Cc: Steven Rostedt <rostedt@goodmis.org>
Cc: Will Drewry <wad@chromium.org>
Link: http://lkml.kernel.org/r/20150324184442.GC14760@gmail.com
Signed-off-by: Ingo Molnar <mingo@kernel.org>
---
 arch/x86/ia32/ia32entry.S          | 30 +++++++++++++++---------------
 arch/x86/include/asm/thread_info.h |  4 ++--
 arch/x86/kernel/entry_64.S         |  4 ++--
 3 files changed, 19 insertions(+), 19 deletions(-)

diff --git a/arch/x86/ia32/ia32entry.S b/arch/x86/ia32/ia32entry.S
index 32e94ae..5d2641c 100644
--- a/arch/x86/ia32/ia32entry.S
+++ b/arch/x86/ia32/ia32entry.S
@@ -127,7 +127,7 @@ ENTRY(ia32_sysenter_target)
 	CFI_REL_OFFSET rsp,0
 	pushfq_cfi
 	/*CFI_REL_OFFSET rflags,0*/
-	movl	THREAD_INFO(TI_sysenter_return, %rsp, 3*8), %r10d
+	movl	ASM_THREAD_INFO(TI_sysenter_return, %rsp, 3*8), %r10d
 	CFI_REGISTER rip,r10
 	pushq_cfi $__USER32_CS
 	/*CFI_REL_OFFSET cs,0*/
@@ -159,8 +159,8 @@ ENTRY(ia32_sysenter_target)
 	jnz sysenter_fix_flags
 sysenter_flags_fixed:
 
-	orl     $TS_COMPAT, THREAD_INFO(TI_status, %rsp, SIZEOF_PTREGS)
-	testl   $_TIF_WORK_SYSCALL_ENTRY, THREAD_INFO(TI_flags, %rsp, SIZEOF_PTREGS)
+	orl     $TS_COMPAT, ASM_THREAD_INFO(TI_status, %rsp, SIZEOF_PTREGS)
+	testl   $_TIF_WORK_SYSCALL_ENTRY, ASM_THREAD_INFO(TI_flags, %rsp, SIZEOF_PTREGS)
 	CFI_REMEMBER_STATE
 	jnz  sysenter_tracesys
 	cmpq	$(IA32_NR_syscalls-1),%rax
@@ -177,10 +177,10 @@ sysenter_dispatch:
 	movq	%rax,RAX(%rsp)
 	DISABLE_INTERRUPTS(CLBR_NONE)
 	TRACE_IRQS_OFF
-	testl	$_TIF_ALLWORK_MASK, THREAD_INFO(TI_flags, %rsp, SIZEOF_PTREGS)
+	testl	$_TIF_ALLWORK_MASK, ASM_THREAD_INFO(TI_flags, %rsp, SIZEOF_PTREGS)
 	jnz	sysexit_audit
 sysexit_from_sys_call:
-	andl    $~TS_COMPAT,THREAD_INFO(TI_status, %rsp, SIZEOF_PTREGS)
+	andl    $~TS_COMPAT,ASM_THREAD_INFO(TI_status, %rsp, SIZEOF_PTREGS)
 	/* clear IF, that popfq doesn't enable interrupts early */
 	andl	$~0x200,EFLAGS(%rsp)
 	movl	RIP(%rsp),%edx		/* User %eip */
@@ -225,7 +225,7 @@ sysexit_from_sys_call:
 	.endm
 
 	.macro auditsys_exit exit
-	testl $(_TIF_ALLWORK_MASK & ~_TIF_SYSCALL_AUDIT), THREAD_INFO(TI_flags, %rsp, SIZEOF_PTREGS)
+	testl $(_TIF_ALLWORK_MASK & ~_TIF_SYSCALL_AUDIT), ASM_THREAD_INFO(TI_flags, %rsp, SIZEOF_PTREGS)
 	jnz ia32_ret_from_sys_call
 	TRACE_IRQS_ON
 	ENABLE_INTERRUPTS(CLBR_NONE)
@@ -240,7 +240,7 @@ sysexit_from_sys_call:
 	movl $(_TIF_ALLWORK_MASK & ~_TIF_SYSCALL_AUDIT),%edi
 	DISABLE_INTERRUPTS(CLBR_NONE)
 	TRACE_IRQS_OFF
-	testl %edi, THREAD_INFO(TI_flags, %rsp, SIZEOF_PTREGS)
+	testl %edi, ASM_THREAD_INFO(TI_flags, %rsp, SIZEOF_PTREGS)
 	jz \exit
 	CLEAR_RREGS
 	jmp int_with_check
@@ -262,7 +262,7 @@ sysenter_fix_flags:
 
 sysenter_tracesys:
 #ifdef CONFIG_AUDITSYSCALL
-	testl	$(_TIF_WORK_SYSCALL_ENTRY & ~_TIF_SYSCALL_AUDIT), THREAD_INFO(TI_flags, %rsp, SIZEOF_PTREGS)
+	testl	$(_TIF_WORK_SYSCALL_ENTRY & ~_TIF_SYSCALL_AUDIT), ASM_THREAD_INFO(TI_flags, %rsp, SIZEOF_PTREGS)
 	jz	sysenter_auditsys
 #endif
 	SAVE_EXTRA_REGS
@@ -346,8 +346,8 @@ ENTRY(ia32_cstar_target)
 1:	movl	(%r8),%r9d
 	_ASM_EXTABLE(1b,ia32_badarg)
 	ASM_CLAC
-	orl     $TS_COMPAT, THREAD_INFO(TI_status, %rsp, SIZEOF_PTREGS)
-	testl   $_TIF_WORK_SYSCALL_ENTRY, THREAD_INFO(TI_flags, %rsp, SIZEOF_PTREGS)
+	orl     $TS_COMPAT, ASM_THREAD_INFO(TI_status, %rsp, SIZEOF_PTREGS)
+	testl   $_TIF_WORK_SYSCALL_ENTRY, ASM_THREAD_INFO(TI_flags, %rsp, SIZEOF_PTREGS)
 	CFI_REMEMBER_STATE
 	jnz   cstar_tracesys
 	cmpq $IA32_NR_syscalls-1,%rax
@@ -364,10 +364,10 @@ cstar_dispatch:
 	movq %rax,RAX(%rsp)
 	DISABLE_INTERRUPTS(CLBR_NONE)
 	TRACE_IRQS_OFF
-	testl $_TIF_ALLWORK_MASK, THREAD_INFO(TI_flags, %rsp, SIZEOF_PTREGS)
+	testl $_TIF_ALLWORK_MASK, ASM_THREAD_INFO(TI_flags, %rsp, SIZEOF_PTREGS)
 	jnz sysretl_audit
 sysretl_from_sys_call:
-	andl $~TS_COMPAT, THREAD_INFO(TI_status, %rsp, SIZEOF_PTREGS)
+	andl $~TS_COMPAT, ASM_THREAD_INFO(TI_status, %rsp, SIZEOF_PTREGS)
 	RESTORE_RSI_RDI_RDX
 	movl RIP(%rsp),%ecx
 	CFI_REGISTER rip,rcx
@@ -402,7 +402,7 @@ sysretl_audit:
 
 cstar_tracesys:
 #ifdef CONFIG_AUDITSYSCALL
-	testl $(_TIF_WORK_SYSCALL_ENTRY & ~_TIF_SYSCALL_AUDIT), THREAD_INFO(TI_flags, %rsp, SIZEOF_PTREGS)
+	testl $(_TIF_WORK_SYSCALL_ENTRY & ~_TIF_SYSCALL_AUDIT), ASM_THREAD_INFO(TI_flags, %rsp, SIZEOF_PTREGS)
 	jz cstar_auditsys
 #endif
 	xchgl %r9d,%ebp
@@ -469,8 +469,8 @@ ENTRY(ia32_syscall)
 	   this could be a problem. */
 	ALLOC_PT_GPREGS_ON_STACK
 	SAVE_C_REGS_EXCEPT_R891011
-	orl $TS_COMPAT, THREAD_INFO(TI_status, %rsp, SIZEOF_PTREGS)
-	testl $_TIF_WORK_SYSCALL_ENTRY, THREAD_INFO(TI_flags, %rsp, SIZEOF_PTREGS)
+	orl $TS_COMPAT, ASM_THREAD_INFO(TI_status, %rsp, SIZEOF_PTREGS)
+	testl $_TIF_WORK_SYSCALL_ENTRY, ASM_THREAD_INFO(TI_flags, %rsp, SIZEOF_PTREGS)
 	jnz ia32_tracesys
 	cmpq $(IA32_NR_syscalls-1),%rax
 	ja ia32_badsys
diff --git a/arch/x86/include/asm/thread_info.h b/arch/x86/include/asm/thread_info.h
index 224285b..ea2dbe8 100644
--- a/arch/x86/include/asm/thread_info.h
+++ b/arch/x86/include/asm/thread_info.h
@@ -222,7 +222,7 @@ static inline unsigned long current_stack_pointer(void)
  * currently at exactly SIZEOF_PTREGS bytes away from the top of the
  * stack:
  *
- *      mov THREAD_INFO(TI_flags, %rsp, SIZEOF_PTREGS), %eax
+ *      mov ASM_THREAD_INFO(TI_flags, %rsp, SIZEOF_PTREGS), %eax
  *
  * will translate to:
  *
@@ -230,7 +230,7 @@ static inline unsigned long current_stack_pointer(void)
  *
  * which is below the current RSP by almost 16K.
  */
-#define THREAD_INFO(field, reg, off) ((field)+(off)-THREAD_SIZE)(reg)
+#define ASM_THREAD_INFO(field, reg, off) ((field)+(off)-THREAD_SIZE)(reg)
 
 #endif
 
diff --git a/arch/x86/kernel/entry_64.S b/arch/x86/kernel/entry_64.S
index 8f01a4f..daf5d94 100644
--- a/arch/x86/kernel/entry_64.S
+++ b/arch/x86/kernel/entry_64.S
@@ -249,7 +249,7 @@ GLOBAL(system_call_after_swapgs)
 	pushq_cfi_reg	r11			/* pt_regs->r11 */
 	sub	$(6*8),%rsp /* pt_regs->bp,bx,r12-15 not saved */
 
-	testl $_TIF_WORK_SYSCALL_ENTRY, THREAD_INFO(TI_flags, %rsp, SIZEOF_PTREGS)
+	testl $_TIF_WORK_SYSCALL_ENTRY, ASM_THREAD_INFO(TI_flags, %rsp, SIZEOF_PTREGS)
 	jnz tracesys
 system_call_fastpath:
 #if __SYSCALL_MASK == ~0
@@ -267,7 +267,7 @@ system_call_fastpath:
  * Has incompletely filled pt_regs, iret frame is also incomplete.
  */
 ret_from_sys_call:
-	testl $_TIF_ALLWORK_MASK, THREAD_INFO(TI_flags, %rsp, SIZEOF_PTREGS)
+	testl $_TIF_ALLWORK_MASK, ASM_THREAD_INFO(TI_flags, %rsp, SIZEOF_PTREGS)
 	jnz int_ret_from_sys_call	/* Go the slow path */
 
 	LOCKDEP_SYS_EXIT