[PATCH 1/9] x86/asm/entry/64: do not TRACE_IRQS fast SYSRET64 path

[PATCH 1/9] x86/asm/entry/64: do not TRACE_IRQS fast SYSRET64 path

[Denys Vlasenko]

SYSRET code path has a small irq-off block.
On this code path, TRACE_IRQS_ON can't be called right before interrupts
are enabled for real, we can't clobber registers there.
So current code does it earlier, in a safe place.

But with this, TRACE_IRQS_OFF/ON frames just two fast instructions,
which is ridiculous: now most of irq-off block is _outside_ of the framing.

Do the same thing that we do on SYSCALL entry: do not track this irq-off block,
it is very small to ever cause noticeable irq latency.

Be careful: make sure that "jnz int_ret_from_sys_call_irqs_off" now does
invoke TRACE_IRQS_OFF - move int_ret_from_sys_call_irqs_off label before
TRACE_IRQS_OFF.

Signed-off-by: Denys Vlasenko <dvlasenk@redhat.com>
CC: Linus Torvalds <torvalds@linux-foundation.org>
CC: Steven Rostedt <rostedt@goodmis.org>
CC: Ingo Molnar <mingo@kernel.org>
CC: Borislav Petkov <bp@alien8.de>
CC: "H. Peter Anvin" <hpa@zytor.com>
CC: Andy Lutomirski <luto@amacapital.net>
CC: Oleg Nesterov <oleg@redhat.com>
CC: Frederic Weisbecker <fweisbec@gmail.com>
CC: Alexei Starovoitov <ast@plumgrid.com>
CC: Will Drewry <wad@chromium.org>
CC: Kees Cook <keescook@chromium.org>
CC: x86@kernel.org
CC: linux-kernel@vger.kernel.org
---
 arch/x86/kernel/entry_64.S | 13 +++++++------
 1 file changed, 7 insertions(+), 6 deletions(-)

diff --git a/arch/x86/kernel/entry_64.S b/arch/x86/kernel/entry_64.S
index 6f251a5..f6e37de 100644
--- a/arch/x86/kernel/entry_64.S
+++ b/arch/x86/kernel/entry_64.S
@@ -269,8 +269,11 @@ system_call_fastpath:
  * Has incompletely filled pt_regs.
  */
 	LOCKDEP_SYS_EXIT
+	/*
+	 * We do not frame this tiny irq-off block with TRACE_IRQS_OFF/ON,
+	 * it is too small to ever cause noticeable irq latency.
+	 */
 	DISABLE_INTERRUPTS(CLBR_NONE)
-	TRACE_IRQS_OFF
 
 	/*
 	 * We must check ti flags with interrupts (or at least preemption)
@@ -284,10 +287,7 @@ system_call_fastpath:
 	jnz int_ret_from_sys_call_irqs_off	/* Go to the slow path */
 
 	CFI_REMEMBER_STATE
-	/*
-	 * sysretq will re-enable interrupts:
-	 */
-	TRACE_IRQS_ON
+
 	RESTORE_C_REGS_EXCEPT_RCX_R11
 	movq	RIP(%rsp),%rcx
 	CFI_REGISTER	rip,rcx
@@ -298,6 +298,7 @@ system_call_fastpath:
 	 * 64bit SYSRET restores rip from rcx,
 	 * rflags from r11 (but RF and VM bits are forced to 0),
 	 * cs and ss are loaded from MSRs.
+	 * Restoration of rflags re-enables interrupts.
 	 */
 	USERGS_SYSRET64
 
@@ -346,8 +347,8 @@ tracesys_phase2:
  */
 GLOBAL(int_ret_from_sys_call)
 	DISABLE_INTERRUPTS(CLBR_NONE)
+int_ret_from_sys_call_irqs_off: /* jumps come here from the irqs-off SYSRET path */
 	TRACE_IRQS_OFF
-int_ret_from_sys_call_irqs_off:
 	movl $_TIF_ALLWORK_MASK,%edi
 	/* edi:	mask to check */
 GLOBAL(int_with_check)
-- 
1.8.1.4

[PATCH 2/9] x86/asm/entry/32: Use PUSH instructions to build pt_regs on stack

[Denys Vlasenko]

This mimics the recent similar 64-bit change.
Saves ~110 bytes of code.

Signed-off-by: Denys Vlasenko <dvlasenk@redhat.com>
CC: Linus Torvalds <torvalds@linux-foundation.org>
CC: Steven Rostedt <rostedt@goodmis.org>
CC: Ingo Molnar <mingo@kernel.org>
CC: Borislav Petkov <bp@alien8.de>
CC: "H. Peter Anvin" <hpa@zytor.com>
CC: Andy Lutomirski <luto@amacapital.net>
CC: Oleg Nesterov <oleg@redhat.com>
CC: Frederic Weisbecker <fweisbec@gmail.com>
CC: Alexei Starovoitov <ast@plumgrid.com>
CC: Will Drewry <wad@chromium.org>
CC: Kees Cook <keescook@chromium.org>
CC: x86@kernel.org
CC: linux-kernel@vger.kernel.org
---

Patch was run-tested on 32 and 64 bits, Intel and AMD CPU.
I also looked at the diff of entry_64.o disassembly, to have
a different view of the changes.

 arch/x86/ia32/ia32entry.S | 82 ++++++++++++++++++++++++++---------------------
 1 file changed, 46 insertions(+), 36 deletions(-)

diff --git a/arch/x86/ia32/ia32entry.S b/arch/x86/ia32/ia32entry.S
index dec8c1d..8d01cce 100644
--- a/arch/x86/ia32/ia32entry.S
+++ b/arch/x86/ia32/ia32entry.S
@@ -126,26 +126,27 @@ ENTRY(ia32_sysenter_target)
 	movl	%ebp, %ebp
 	movl	%eax, %eax
 
-	/* Construct iret frame (ss,rsp,rflags,cs,rip) */
-	pushq_cfi $__USER32_DS
-	/*CFI_REL_OFFSET ss,0*/
-	pushq_cfi %rbp
-	CFI_REL_OFFSET rsp,0
-	pushfq_cfi
-	/*CFI_REL_OFFSET rflags,0*/
-	movl	ASM_THREAD_INFO(TI_sysenter_return, %rsp, 3*8), %r10d
+	movl	ASM_THREAD_INFO(TI_sysenter_return, %rsp, 0), %r10d
 	CFI_REGISTER rip,r10
-	pushq_cfi $__USER32_CS
-	/*CFI_REL_OFFSET cs,0*/
-	/* Store thread_info->sysenter_return in rip stack slot */
-	pushq_cfi %r10
-	CFI_REL_OFFSET rip,0
-	/* Store orig_ax */
-	pushq_cfi %rax
-	/* Construct the rest of "struct pt_regs" */
+
+	/* Construct struct pt_regs on stack */
+	pushq_cfi	$__USER32_DS		/* pt_regs->ss */
+	pushq_cfi	%rbp			/* pt_regs->sp */
+	CFI_REL_OFFSET	rsp,0
+	pushfq_cfi				/* pt_regs->flags */
+	pushq_cfi	$__USER32_CS		/* pt_regs->cs */
+	pushq_cfi	%r10 /* pt_regs->ip = thread_info->sysenter_return */
+	CFI_REL_OFFSET	rip,0
+	pushq_cfi_reg	rax			/* pt_regs->orig_ax */
+	pushq_cfi_reg	rdi			/* pt_regs->di */
+	pushq_cfi_reg	rsi			/* pt_regs->si */
+	pushq_cfi_reg	rdx			/* pt_regs->dx */
+	pushq_cfi_reg	rcx			/* pt_regs->cx */
+	pushq_cfi_reg	rax			/* pt_regs->ax */
 	cld
-	ALLOC_PT_GPREGS_ON_STACK
-	SAVE_C_REGS_EXCEPT_R891011
+	sub	$(10*8),%rsp /* pt_regs->r8-11,bp,bx,r12-15 not saved */
+	CFI_ADJUST_CFA_OFFSET 10*8
+
 	/*
 	 * no need to do an access_ok check here because rbp has been
 	 * 32bit zero extended
@@ -334,20 +335,24 @@ ENTRY(ia32_cstar_target)
 	/* Zero-extending 32-bit regs, do not remove */
 	movl	%eax,%eax
 
-	ALLOC_PT_GPREGS_ON_STACK 6*8 /* 6*8: space for orig_ax and iret frame */
-	SAVE_C_REGS_EXCEPT_RCX_R891011
-	movq	%rax,ORIG_RAX(%rsp)
-	movq	%rcx,RIP(%rsp)
-	CFI_REL_OFFSET rip,RIP
-	movq	%rbp,RCX(%rsp) /* this lies slightly to ptrace */
+	/* Construct struct pt_regs on stack */
+	pushq_cfi	$__USER32_DS		/* pt_regs->ss */
+	pushq_cfi	%r8			/* pt_regs->sp */
+	CFI_REL_OFFSET rsp,0
+	pushq_cfi	%r11			/* pt_regs->flags */
+	pushq_cfi	$__USER32_CS		/* pt_regs->cs */
+	pushq_cfi	%rcx			/* pt_regs->ip */
+	CFI_REL_OFFSET rip,0
+	pushq_cfi_reg	rax			/* pt_regs->orig_ax */
+	pushq_cfi_reg	rdi			/* pt_regs->di */
+	pushq_cfi_reg	rsi			/* pt_regs->si */
+	pushq_cfi_reg	rdx			/* pt_regs->dx */
+	pushq_cfi_reg	rbp			/* pt_regs->cx */
 	movl	%ebp,%ecx
-	movq	$__USER32_CS,CS(%rsp)
-	movq	$__USER32_DS,SS(%rsp)
-	movq	%r11,EFLAGS(%rsp)
-	/*CFI_REL_OFFSET rflags,EFLAGS*/
-	movq	%r8,RSP(%rsp)
-	CFI_REL_OFFSET rsp,RSP
-	/* iret stack frame is complete now */
+	pushq_cfi_reg	rax			/* pt_regs->ax */
+	sub	$(10*8),%rsp /* pt_regs->r8-11,bp,bx,r12-15 not saved */
+	CFI_ADJUST_CFA_OFFSET 10*8
+
 	/*
 	 * no need to do an access_ok check here because r8 has been
 	 * 32bit zero extended
@@ -478,12 +483,17 @@ ENTRY(ia32_syscall)
 	/* Zero-extending 32-bit regs, do not remove */
 	movl	%eax,%eax
 
-	pushq_cfi %rax		/* store orig_ax */
+	/* Construct struct pt_regs on stack (iret frame is already on stack) */
+	pushq_cfi_reg	rax			/* pt_regs->orig_ax */
+	pushq_cfi_reg	rdi			/* pt_regs->di */
+	pushq_cfi_reg	rsi			/* pt_regs->si */
+	pushq_cfi_reg	rdx			/* pt_regs->dx */
+	pushq_cfi_reg	rcx			/* pt_regs->cx */
+	pushq_cfi_reg	rax			/* pt_regs->ax */
 	cld
-	/* note the registers are not zero extended to the sf.
-	   this could be a problem. */
-	ALLOC_PT_GPREGS_ON_STACK
-	SAVE_C_REGS_EXCEPT_R891011
+	sub	$(10*8),%rsp /* pt_regs->r8-11,bp,bx,r12-15 not saved */
+	CFI_ADJUST_CFA_OFFSET 10*8
+
 	orl $TS_COMPAT, ASM_THREAD_INFO(TI_status, %rsp, SIZEOF_PTREGS)
 	testl $_TIF_WORK_SYSCALL_ENTRY, ASM_THREAD_INFO(TI_flags, %rsp, SIZEOF_PTREGS)
 	jnz ia32_tracesys
-- 
1.8.1.4

Re: [PATCH 2/9] x86/asm/entry/32: Use PUSH instructions to build pt_regs on stack

[Ingo Molnar]

* Denys Vlasenko <dvlasenk@redhat.com> wrote:

> This mimics the recent similar 64-bit change.
> Saves ~110 bytes of code.
> 
> Patch was run-tested on 32 and 64 bits, Intel and AMD CPU.
> I also looked at the diff of entry_64.o disassembly, to have
> a different view of the changes.

The other important question would be: what performance difference (if 
any) did you observe before/after the change?

Thanks,

	Ingo

Re: [PATCH 2/9] x86/asm/entry/32: Use PUSH instructions to build pt_regs on stack

[Denys Vlasenko]

On 04/01/2015 10:51 AM, Ingo Molnar wrote:
> 
> * Denys Vlasenko <dvlasenk@redhat.com> wrote:
> 
>> This mimics the recent similar 64-bit change.
>> Saves ~110 bytes of code.
>>
>> Patch was run-tested on 32 and 64 bits, Intel and AMD CPU.
>> I also looked at the diff of entry_64.o disassembly, to have
>> a different view of the changes.
> 
> The other important question would be: what performance difference (if 
> any) did you observe before/after the change?

I did not measure it then.

At the moment I don't have AMD CPUs here, cant benchmark
32-bit syscall-based codepath.

On a Sandy Bridge CPU (IOW: sysenter codepath) -

Before: 78.57 ns per getpid
After:  76.90 ns per getpid

It's better than I thought it would be.
Probably because this load:

movl	ASM_THREAD_INFO(TI_sysenter_return, %rsp, 0), %r10d

has been moved up by the patch (happens sooner).

Re: [PATCH 2/9] x86/asm/entry/32: Use PUSH instructions to build pt_regs on stack

[Ingo Molnar]

* Denys Vlasenko <dvlasenk@redhat.com> wrote:

> On 04/01/2015 10:51 AM, Ingo Molnar wrote:
> > 
> > * Denys Vlasenko <dvlasenk@redhat.com> wrote:
> > 
> >> This mimics the recent similar 64-bit change.
> >> Saves ~110 bytes of code.
> >>
> >> Patch was run-tested on 32 and 64 bits, Intel and AMD CPU.
> >> I also looked at the diff of entry_64.o disassembly, to have
> >> a different view of the changes.
> > 
> > The other important question would be: what performance difference (if 
> > any) did you observe before/after the change?
> 
> I did not measure it then.
> 
> At the moment I don't have AMD CPUs here, cant benchmark
> 32-bit syscall-based codepath.
> 
> On a Sandy Bridge CPU (IOW: sysenter codepath) -
> 
> Before: 78.57 ns per getpid
> After:  76.90 ns per getpid
> 
> It's better than I thought it would be.
> Probably because this load:
> 
> movl	ASM_THREAD_INFO(TI_sysenter_return, %rsp, 0), %r10d
> 
> has been moved up by the patch (happens sooner).

There's also less I$ used, and in straight, continuous spots, which 
should result in less cache misses in the very common "the kernel's 
code is cache cold" situation that syscall entry operates under - and 
that's not captured by your benchmark.

So it's a good change.

Thanks,

	Ingo

Re: [PATCH 2/9] x86/asm/entry/32: Use PUSH instructions to build pt_regs on stack

[Borislav Petkov]

On Wed, Apr 01, 2015 at 03:12:50PM +0200, Denys Vlasenko wrote:
> At the moment I don't have AMD CPUs here, cant benchmark
> 32-bit syscall-based codepath.

You could send me your measuring tool - I'll run it on AMD.

-- 
Regards/Gruss,
    Boris.

ECO tip #101: Trim your mails when you reply.
--

[tip:x86/asm] x86/asm/entry/32: Use smaller PUSH instructions instead of MOV, to build 'pt_regs' on stack

[tip-bot for Denys Vlasenko]

Commit-ID:  4c9c0e919fef05b3bc6a8aff1db7a31b2ba4f4b6
Gitweb:     http://git.kernel.org/tip/4c9c0e919fef05b3bc6a8aff1db7a31b2ba4f4b6
Author:     Denys Vlasenko <dvlasenk@redhat.com>
AuthorDate: Tue, 31 Mar 2015 19:00:04 +0200
Committer:  Ingo Molnar <mingo@kernel.org>
CommitDate: Wed, 1 Apr 2015 13:17:38 +0200

x86/asm/entry/32: Use smaller PUSH instructions instead of MOV, to build 'pt_regs' on stack

This mimics the recent similar 64-bit change.
Saves ~110 bytes of code.

Patch was run-tested on 32 and 64 bits, Intel and AMD CPU.
I also looked at the diff of entry_64.o disassembly, to have
a different view of the changes.

Signed-off-by: Denys Vlasenko <dvlasenk@redhat.com>
Cc: Alexei Starovoitov <ast@plumgrid.com>
Cc: Andy Lutomirski <luto@amacapital.net>
Cc: Borislav Petkov <bp@alien8.de>
Cc: Frederic Weisbecker <fweisbec@gmail.com>
Cc: H. Peter Anvin <hpa@zytor.com>
Cc: Kees Cook <keescook@chromium.org>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Oleg Nesterov <oleg@redhat.com>
Cc: Steven Rostedt <rostedt@goodmis.org>
Cc: Will Drewry <wad@chromium.org>
Link: http://lkml.kernel.org/r/1427821211-25099-2-git-send-email-dvlasenk@redhat.com
Signed-off-by: Ingo Molnar <mingo@kernel.org>
---
 arch/x86/ia32/ia32entry.S | 82 ++++++++++++++++++++++++++---------------------
 1 file changed, 46 insertions(+), 36 deletions(-)

diff --git a/arch/x86/ia32/ia32entry.S b/arch/x86/ia32/ia32entry.S
index dec8c1d..8d01cce 100644
--- a/arch/x86/ia32/ia32entry.S
+++ b/arch/x86/ia32/ia32entry.S
@@ -126,26 +126,27 @@ ENTRY(ia32_sysenter_target)
 	movl	%ebp, %ebp
 	movl	%eax, %eax
 
-	/* Construct iret frame (ss,rsp,rflags,cs,rip) */
-	pushq_cfi $__USER32_DS
-	/*CFI_REL_OFFSET ss,0*/
-	pushq_cfi %rbp
-	CFI_REL_OFFSET rsp,0
-	pushfq_cfi
-	/*CFI_REL_OFFSET rflags,0*/
-	movl	ASM_THREAD_INFO(TI_sysenter_return, %rsp, 3*8), %r10d
+	movl	ASM_THREAD_INFO(TI_sysenter_return, %rsp, 0), %r10d
 	CFI_REGISTER rip,r10
-	pushq_cfi $__USER32_CS
-	/*CFI_REL_OFFSET cs,0*/
-	/* Store thread_info->sysenter_return in rip stack slot */
-	pushq_cfi %r10
-	CFI_REL_OFFSET rip,0
-	/* Store orig_ax */
-	pushq_cfi %rax
-	/* Construct the rest of "struct pt_regs" */
+
+	/* Construct struct pt_regs on stack */
+	pushq_cfi	$__USER32_DS		/* pt_regs->ss */
+	pushq_cfi	%rbp			/* pt_regs->sp */
+	CFI_REL_OFFSET	rsp,0
+	pushfq_cfi				/* pt_regs->flags */
+	pushq_cfi	$__USER32_CS		/* pt_regs->cs */
+	pushq_cfi	%r10 /* pt_regs->ip = thread_info->sysenter_return */
+	CFI_REL_OFFSET	rip,0
+	pushq_cfi_reg	rax			/* pt_regs->orig_ax */
+	pushq_cfi_reg	rdi			/* pt_regs->di */
+	pushq_cfi_reg	rsi			/* pt_regs->si */
+	pushq_cfi_reg	rdx			/* pt_regs->dx */
+	pushq_cfi_reg	rcx			/* pt_regs->cx */
+	pushq_cfi_reg	rax			/* pt_regs->ax */
 	cld
-	ALLOC_PT_GPREGS_ON_STACK
-	SAVE_C_REGS_EXCEPT_R891011
+	sub	$(10*8),%rsp /* pt_regs->r8-11,bp,bx,r12-15 not saved */
+	CFI_ADJUST_CFA_OFFSET 10*8
+
 	/*
 	 * no need to do an access_ok check here because rbp has been
 	 * 32bit zero extended
@@ -334,20 +335,24 @@ ENTRY(ia32_cstar_target)
 	/* Zero-extending 32-bit regs, do not remove */
 	movl	%eax,%eax
 
-	ALLOC_PT_GPREGS_ON_STACK 6*8 /* 6*8: space for orig_ax and iret frame */
-	SAVE_C_REGS_EXCEPT_RCX_R891011
-	movq	%rax,ORIG_RAX(%rsp)
-	movq	%rcx,RIP(%rsp)
-	CFI_REL_OFFSET rip,RIP
-	movq	%rbp,RCX(%rsp) /* this lies slightly to ptrace */
+	/* Construct struct pt_regs on stack */
+	pushq_cfi	$__USER32_DS		/* pt_regs->ss */
+	pushq_cfi	%r8			/* pt_regs->sp */
+	CFI_REL_OFFSET rsp,0
+	pushq_cfi	%r11			/* pt_regs->flags */
+	pushq_cfi	$__USER32_CS		/* pt_regs->cs */
+	pushq_cfi	%rcx			/* pt_regs->ip */
+	CFI_REL_OFFSET rip,0
+	pushq_cfi_reg	rax			/* pt_regs->orig_ax */
+	pushq_cfi_reg	rdi			/* pt_regs->di */
+	pushq_cfi_reg	rsi			/* pt_regs->si */
+	pushq_cfi_reg	rdx			/* pt_regs->dx */
+	pushq_cfi_reg	rbp			/* pt_regs->cx */
 	movl	%ebp,%ecx
-	movq	$__USER32_CS,CS(%rsp)
-	movq	$__USER32_DS,SS(%rsp)
-	movq	%r11,EFLAGS(%rsp)
-	/*CFI_REL_OFFSET rflags,EFLAGS*/
-	movq	%r8,RSP(%rsp)
-	CFI_REL_OFFSET rsp,RSP
-	/* iret stack frame is complete now */
+	pushq_cfi_reg	rax			/* pt_regs->ax */
+	sub	$(10*8),%rsp /* pt_regs->r8-11,bp,bx,r12-15 not saved */
+	CFI_ADJUST_CFA_OFFSET 10*8
+
 	/*
 	 * no need to do an access_ok check here because r8 has been
 	 * 32bit zero extended
@@ -478,12 +483,17 @@ ENTRY(ia32_syscall)
 	/* Zero-extending 32-bit regs, do not remove */
 	movl	%eax,%eax
 
-	pushq_cfi %rax		/* store orig_ax */
+	/* Construct struct pt_regs on stack (iret frame is already on stack) */
+	pushq_cfi_reg	rax			/* pt_regs->orig_ax */
+	pushq_cfi_reg	rdi			/* pt_regs->di */
+	pushq_cfi_reg	rsi			/* pt_regs->si */
+	pushq_cfi_reg	rdx			/* pt_regs->dx */
+	pushq_cfi_reg	rcx			/* pt_regs->cx */
+	pushq_cfi_reg	rax			/* pt_regs->ax */
 	cld
-	/* note the registers are not zero extended to the sf.
-	   this could be a problem. */
-	ALLOC_PT_GPREGS_ON_STACK
-	SAVE_C_REGS_EXCEPT_R891011
+	sub	$(10*8),%rsp /* pt_regs->r8-11,bp,bx,r12-15 not saved */
+	CFI_ADJUST_CFA_OFFSET 10*8
+
 	orl $TS_COMPAT, ASM_THREAD_INFO(TI_status, %rsp, SIZEOF_PTREGS)
 	testl $_TIF_WORK_SYSCALL_ENTRY, ASM_THREAD_INFO(TI_flags, %rsp, SIZEOF_PTREGS)
 	jnz ia32_tracesys

[PATCH 3/9] x86/asm/entry/64: simplify retint_kernel label usage, make retint_restore_args label local

[Denys Vlasenko]

Get rid of #define obfuscation of retint_kernel in CONFIG_PREEMPT case
by defining retint_kernel label always, not only for CONFIG_PREEMPT.

Strip retint_kernel of .global-ness (ENTRY macro) - it has no users
outside of this file.

This looks like cosmetics, but it is not:
"je LABEL" can be optimized to short jump by assember
only if LABEL is not global, for global labels jump is always
a near one with relocation.

Convert retint_restore_args to a local numeric label, making it clearer
that it is not used elsewhere in the file.

Signed-off-by: Denys Vlasenko <dvlasenk@redhat.com>
CC: Linus Torvalds <torvalds@linux-foundation.org>
CC: Steven Rostedt <rostedt@goodmis.org>
CC: Ingo Molnar <mingo@kernel.org>
CC: Borislav Petkov <bp@alien8.de>
CC: "H. Peter Anvin" <hpa@zytor.com>
CC: Andy Lutomirski <luto@amacapital.net>
CC: Oleg Nesterov <oleg@redhat.com>
CC: Frederic Weisbecker <fweisbec@gmail.com>
CC: Alexei Starovoitov <ast@plumgrid.com>
CC: Will Drewry <wad@chromium.org>
CC: Kees Cook <keescook@chromium.org>
CC: x86@kernel.org
CC: linux-kernel@vger.kernel.org
---
 arch/x86/kernel/entry_64.S | 12 ++++--------
 1 file changed, 4 insertions(+), 8 deletions(-)

diff --git a/arch/x86/kernel/entry_64.S b/arch/x86/kernel/entry_64.S
index f6e37de..1879c55 100644
--- a/arch/x86/kernel/entry_64.S
+++ b/arch/x86/kernel/entry_64.S
@@ -57,10 +57,6 @@
 	.section .entry.text, "ax"
 
 
-#ifndef CONFIG_PREEMPT
-#define retint_kernel retint_restore_args
-#endif
-
 #ifdef CONFIG_PARAVIRT
 ENTRY(native_usergs_sysret64)
 	swapgs
@@ -741,18 +737,18 @@ opportunistic_sysret_failed:
 	jmp restore_args
 
 /* Returning to kernel space */
+retint_kernel:
 #ifdef CONFIG_PREEMPT
 	/* Interrupts are off */
 	/* Check if we need preemption */
-ENTRY(retint_kernel)
 	cmpl	$0,PER_CPU_VAR(__preempt_count)
-	jnz	retint_restore_args
+	jnz	1f
 	bt	$9,EFLAGS(%rsp)	/* interrupts were off? */
-	jnc	retint_restore_args
+	jnc	1f
 	call	preempt_schedule_irq
 	jmp	exit_intr
+1:
 #endif
-retint_restore_args:
 	DISABLE_INTERRUPTS(CLBR_ANY)
 	/*
 	 * The iretq could re-enable interrupts:
-- 
1.8.1.4

[tip:x86/asm] x86/asm/entry/64: Simplify retint_kernel label usage, make retint_restore_args label local

[tip-bot for Denys Vlasenko]

Commit-ID:  6ba71b7617f1fa65f19bd34f4484a0694ef9a520
Gitweb:     http://git.kernel.org/tip/6ba71b7617f1fa65f19bd34f4484a0694ef9a520
Author:     Denys Vlasenko <dvlasenk@redhat.com>
AuthorDate: Tue, 31 Mar 2015 19:00:05 +0200
Committer:  Ingo Molnar <mingo@kernel.org>
CommitDate: Wed, 1 Apr 2015 13:17:38 +0200

x86/asm/entry/64: Simplify retint_kernel label usage, make retint_restore_args label local

Get rid of #define obfuscation of retint_kernel in
CONFIG_PREEMPT case by defining retint_kernel label always, not
only for CONFIG_PREEMPT.

Strip retint_kernel of .global-ness (ENTRY macro) - it has no
users outside of this file.

This looks like cosmetics, but it is not:
"je LABEL" can be optimized to short jump by assember
only if LABEL is not global, for global labels jump is always
a near one with relocation.

Convert retint_restore_args to a local numeric label, making it
clearer that it is not used elsewhere in the file.

Signed-off-by: Denys Vlasenko <dvlasenk@redhat.com>
Cc: Alexei Starovoitov <ast@plumgrid.com>
Cc: Andy Lutomirski <luto@amacapital.net>
Cc: Borislav Petkov <bp@alien8.de>
Cc: Frederic Weisbecker <fweisbec@gmail.com>
Cc: H. Peter Anvin <hpa@zytor.com>
Cc: Kees Cook <keescook@chromium.org>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Oleg Nesterov <oleg@redhat.com>
Cc: Steven Rostedt <rostedt@goodmis.org>
Cc: Will Drewry <wad@chromium.org>
Link: http://lkml.kernel.org/r/1427821211-25099-3-git-send-email-dvlasenk@redhat.com
Signed-off-by: Ingo Molnar <mingo@kernel.org>
---
 arch/x86/kernel/entry_64.S | 12 ++++--------
 1 file changed, 4 insertions(+), 8 deletions(-)

diff --git a/arch/x86/kernel/entry_64.S b/arch/x86/kernel/entry_64.S
index f6e37de..1879c55 100644
--- a/arch/x86/kernel/entry_64.S
+++ b/arch/x86/kernel/entry_64.S
@@ -57,10 +57,6 @@
 	.section .entry.text, "ax"
 
 
-#ifndef CONFIG_PREEMPT
-#define retint_kernel retint_restore_args
-#endif
-
 #ifdef CONFIG_PARAVIRT
 ENTRY(native_usergs_sysret64)
 	swapgs
@@ -741,18 +737,18 @@ opportunistic_sysret_failed:
 	jmp restore_args
 
 /* Returning to kernel space */
+retint_kernel:
 #ifdef CONFIG_PREEMPT
 	/* Interrupts are off */
 	/* Check if we need preemption */
-ENTRY(retint_kernel)
 	cmpl	$0,PER_CPU_VAR(__preempt_count)
-	jnz	retint_restore_args
+	jnz	1f
 	bt	$9,EFLAGS(%rsp)	/* interrupts were off? */
-	jnc	retint_restore_args
+	jnc	1f
 	call	preempt_schedule_irq
 	jmp	exit_intr
+1:
 #endif
-retint_restore_args:
 	DISABLE_INTERRUPTS(CLBR_ANY)
 	/*
 	 * The iretq could re-enable interrupts:

[PATCH 4/9] x86/asm/entry/64: remove redundant DISABLE_INTERRUPTS

[Denys Vlasenko]

At this location, we already have interrupts off, always.
To be more specific, we already disabled them here:

    ret_from_intr:
	    DISABLE_INTERRUPTS(CLBR_NONE)

Signed-off-by: Denys Vlasenko <dvlasenk@redhat.com>
CC: Linus Torvalds <torvalds@linux-foundation.org>
CC: Steven Rostedt <rostedt@goodmis.org>
CC: Ingo Molnar <mingo@kernel.org>
CC: Borislav Petkov <bp@alien8.de>
CC: "H. Peter Anvin" <hpa@zytor.com>
CC: Andy Lutomirski <luto@amacapital.net>
CC: Oleg Nesterov <oleg@redhat.com>
CC: Frederic Weisbecker <fweisbec@gmail.com>
CC: Alexei Starovoitov <ast@plumgrid.com>
CC: Will Drewry <wad@chromium.org>
CC: Kees Cook <keescook@chromium.org>
CC: x86@kernel.org
CC: linux-kernel@vger.kernel.org
---
 arch/x86/kernel/entry_64.S | 1 -
 1 file changed, 1 deletion(-)

diff --git a/arch/x86/kernel/entry_64.S b/arch/x86/kernel/entry_64.S
index 1879c55..9f8d01f 100644
--- a/arch/x86/kernel/entry_64.S
+++ b/arch/x86/kernel/entry_64.S
@@ -749,7 +749,6 @@ retint_kernel:
 	jmp	exit_intr
 1:
 #endif
-	DISABLE_INTERRUPTS(CLBR_ANY)
 	/*
 	 * The iretq could re-enable interrupts:
 	 */
-- 
1.8.1.4

[tip:x86/asm] x86/asm/entry/64: Remove redundant DISABLE_INTERRUPTS()

[tip-bot for Denys Vlasenko]

Commit-ID:  32a04077fe401842424a4b555572fa459c01e0a3
Gitweb:     http://git.kernel.org/tip/32a04077fe401842424a4b555572fa459c01e0a3
Author:     Denys Vlasenko <dvlasenk@redhat.com>
AuthorDate: Tue, 31 Mar 2015 19:00:06 +0200
Committer:  Ingo Molnar <mingo@kernel.org>
CommitDate: Wed, 1 Apr 2015 13:17:38 +0200

x86/asm/entry/64: Remove redundant DISABLE_INTERRUPTS()

At this location, we already have interrupts off, always.
To be more specific, we already disabled them here:

    ret_from_intr:
	    DISABLE_INTERRUPTS(CLBR_NONE)

Signed-off-by: Denys Vlasenko <dvlasenk@redhat.com>
Cc: Alexei Starovoitov <ast@plumgrid.com>
Cc: Andy Lutomirski <luto@amacapital.net>
Cc: Borislav Petkov <bp@alien8.de>
Cc: Frederic Weisbecker <fweisbec@gmail.com>
Cc: H. Peter Anvin <hpa@zytor.com>
Cc: Kees Cook <keescook@chromium.org>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Oleg Nesterov <oleg@redhat.com>
Cc: Steven Rostedt <rostedt@goodmis.org>
Cc: Will Drewry <wad@chromium.org>
Link: http://lkml.kernel.org/r/1427821211-25099-4-git-send-email-dvlasenk@redhat.com
Signed-off-by: Ingo Molnar <mingo@kernel.org>
---
 arch/x86/kernel/entry_64.S | 1 -
 1 file changed, 1 deletion(-)

diff --git a/arch/x86/kernel/entry_64.S b/arch/x86/kernel/entry_64.S
index 1879c55..9f8d01f 100644
--- a/arch/x86/kernel/entry_64.S
+++ b/arch/x86/kernel/entry_64.S
@@ -749,7 +749,6 @@ retint_kernel:
 	jmp	exit_intr
 1:
 #endif
-	DISABLE_INTERRUPTS(CLBR_ANY)
 	/*
 	 * The iretq could re-enable interrupts:
 	 */

[PATCH 5/9] x86/asm/entry/64: simplify looping around preempt_schedule_irq

[Denys Vlasenko]

At exit_intr label, we test whether interrupt/exception was in kernel.
If it did, we jump to preemption check. If preemption does happen
(IOW if we call preempt_schedule_irq), we go back to exit_intr.

But it's pointless, we already know that test succeeded last time,
preemption doesn't change the fact that interrupt/exception
was in kernel. We can go back directly to checking
PER_CPU_VAR(__preempt_count) instead.

This makes exit_intr label unused. Dropping it.

Signed-off-by: Denys Vlasenko <dvlasenk@redhat.com>
CC: Linus Torvalds <torvalds@linux-foundation.org>
CC: Steven Rostedt <rostedt@goodmis.org>
CC: Ingo Molnar <mingo@kernel.org>
CC: Borislav Petkov <bp@alien8.de>
CC: "H. Peter Anvin" <hpa@zytor.com>
CC: Andy Lutomirski <luto@amacapital.net>
CC: Oleg Nesterov <oleg@redhat.com>
CC: Frederic Weisbecker <fweisbec@gmail.com>
CC: Alexei Starovoitov <ast@plumgrid.com>
CC: Will Drewry <wad@chromium.org>
CC: Kees Cook <keescook@chromium.org>
CC: x86@kernel.org
CC: linux-kernel@vger.kernel.org
---
 arch/x86/kernel/entry_64.S | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/arch/x86/kernel/entry_64.S b/arch/x86/kernel/entry_64.S
index 9f8d01f..bad285d 100644
--- a/arch/x86/kernel/entry_64.S
+++ b/arch/x86/kernel/entry_64.S
@@ -654,7 +654,6 @@ ret_from_intr:
 	CFI_DEF_CFA_REGISTER	rsp
 	CFI_ADJUST_CFA_OFFSET	RBP
 
-exit_intr:
 	testl $3,CS(%rsp)
 	je retint_kernel
 	/* Interrupt came from user space */
@@ -741,12 +740,12 @@ retint_kernel:
 #ifdef CONFIG_PREEMPT
 	/* Interrupts are off */
 	/* Check if we need preemption */
-	cmpl	$0,PER_CPU_VAR(__preempt_count)
-	jnz	1f
 	bt	$9,EFLAGS(%rsp)	/* interrupts were off? */
 	jnc	1f
+0:	cmpl	$0,PER_CPU_VAR(__preempt_count)
+	jnz	1f
 	call	preempt_schedule_irq
-	jmp	exit_intr
+	jmp	0b
 1:
 #endif
 	/*
-- 
1.8.1.4

[tip:x86/asm] x86/asm/entry/64: Simplify looping around preempt_schedule_irq()

[tip-bot for Denys Vlasenko]

Commit-ID:  36acef2510853e2831047ca9e22d333ba7a1047b
Gitweb:     http://git.kernel.org/tip/36acef2510853e2831047ca9e22d333ba7a1047b
Author:     Denys Vlasenko <dvlasenk@redhat.com>
AuthorDate: Tue, 31 Mar 2015 19:00:07 +0200
Committer:  Ingo Molnar <mingo@kernel.org>
CommitDate: Wed, 1 Apr 2015 13:17:39 +0200

x86/asm/entry/64: Simplify looping around preempt_schedule_irq()

At the 'exit_intr' label we test whether interrupt/exception was in
kernel. If it did, we jump to the preemption check. If preemption
does happen (IOW if we call preempt_schedule_irq()), we go back to
'exit_intr'.

But it's pointless, we already know that the test succeeded last
time, preemption doesn't change the fact that interrupt/exception
was in the kernel.

We can go back directly to checking PER_CPU_VAR(__preempt_count) instead.

This makes the 'exit_intr' label unused, drop it.

Signed-off-by: Denys Vlasenko <dvlasenk@redhat.com>
Cc: Alexei Starovoitov <ast@plumgrid.com>
Cc: Andy Lutomirski <luto@amacapital.net>
Cc: Borislav Petkov <bp@alien8.de>
Cc: Frederic Weisbecker <fweisbec@gmail.com>
Cc: H. Peter Anvin <hpa@zytor.com>
Cc: Kees Cook <keescook@chromium.org>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Oleg Nesterov <oleg@redhat.com>
Cc: Steven Rostedt <rostedt@goodmis.org>
Cc: Will Drewry <wad@chromium.org>
Link: http://lkml.kernel.org/r/1427821211-25099-5-git-send-email-dvlasenk@redhat.com
Signed-off-by: Ingo Molnar <mingo@kernel.org>
---
 arch/x86/kernel/entry_64.S | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/arch/x86/kernel/entry_64.S b/arch/x86/kernel/entry_64.S
index 9f8d01f..bad285d 100644
--- a/arch/x86/kernel/entry_64.S
+++ b/arch/x86/kernel/entry_64.S
@@ -654,7 +654,6 @@ ret_from_intr:
 	CFI_DEF_CFA_REGISTER	rsp
 	CFI_ADJUST_CFA_OFFSET	RBP
 
-exit_intr:
 	testl $3,CS(%rsp)
 	je retint_kernel
 	/* Interrupt came from user space */
@@ -741,12 +740,12 @@ retint_kernel:
 #ifdef CONFIG_PREEMPT
 	/* Interrupts are off */
 	/* Check if we need preemption */
-	cmpl	$0,PER_CPU_VAR(__preempt_count)
-	jnz	1f
 	bt	$9,EFLAGS(%rsp)	/* interrupts were off? */
 	jnc	1f
+0:	cmpl	$0,PER_CPU_VAR(__preempt_count)
+	jnz	1f
 	call	preempt_schedule_irq
-	jmp	exit_intr
+	jmp	0b
 1:
 #endif
 	/*

[PATCH 6/9] x86/asm/entry/64: tidy up some instructions

[Denys Vlasenko]

Tidy up TEST insns width to use shorter insn form,
Use logically correct JZ mnemonic instead of JE (this doesn't change code).
Replace several BT insns with equivalent, but shorter TEST insns.

Signed-off-by: Denys Vlasenko <dvlasenk@redhat.com>
CC: Linus Torvalds <torvalds@linux-foundation.org>
CC: Steven Rostedt <rostedt@goodmis.org>
CC: Ingo Molnar <mingo@kernel.org>
CC: Borislav Petkov <bp@alien8.de>
CC: "H. Peter Anvin" <hpa@zytor.com>
CC: Andy Lutomirski <luto@amacapital.net>
CC: Oleg Nesterov <oleg@redhat.com>
CC: Frederic Weisbecker <fweisbec@gmail.com>
CC: Alexei Starovoitov <ast@plumgrid.com>
CC: Will Drewry <wad@chromium.org>
CC: Kees Cook <keescook@chromium.org>
CC: x86@kernel.org
CC: linux-kernel@vger.kernel.org
---
 arch/x86/kernel/entry_64.S | 34 +++++++++++++++++-----------------
 1 file changed, 17 insertions(+), 17 deletions(-)

diff --git a/arch/x86/kernel/entry_64.S b/arch/x86/kernel/entry_64.S
index bad285d..f49f973 100644
--- a/arch/x86/kernel/entry_64.S
+++ b/arch/x86/kernel/entry_64.S
@@ -67,8 +67,8 @@ ENDPROC(native_usergs_sysret64)
 
 .macro TRACE_IRQS_IRETQ
 #ifdef CONFIG_TRACE_IRQFLAGS
-	bt   $9,EFLAGS(%rsp)	/* interrupts off? */
-	jnc  1f
+	testb	$2, EFLAGS+1(%rsp)	/* interrupts off? */
+	jz	1f
 	TRACE_IRQS_ON
 1:
 #endif
@@ -100,8 +100,8 @@ ENDPROC(native_usergs_sysret64)
 .endm
 
 .macro TRACE_IRQS_IRETQ_DEBUG
-	bt   $9,EFLAGS(%rsp)	/* interrupts off? */
-	jnc  1f
+	testb	$2, EFLAGS+1(%rsp)	/* interrupts off? */
+	jz	1f
 	TRACE_IRQS_ON_DEBUG
 1:
 .endm
@@ -514,8 +514,8 @@ ENTRY(ret_from_fork)
 
 	RESTORE_EXTRA_REGS
 
-	testl $3,CS(%rsp)			# from kernel_thread?
-	jz   1f
+	testb	$3, CS(%rsp)			# from kernel_thread?
+	jz	1f
 
 	/*
 	 * By the time we get here, we have no idea whether our pt_regs,
@@ -599,8 +599,8 @@ END(interrupt)
 
 	leaq -RBP(%rsp),%rdi	/* arg1 for \func (pointer to pt_regs) */
 
-	testl $3, CS-RBP(%rsp)
-	je 1f
+	testb	$3, CS-RBP(%rsp)
+	jz	1f
 	SWAPGS
 1:
 	/*
@@ -654,8 +654,8 @@ ret_from_intr:
 	CFI_DEF_CFA_REGISTER	rsp
 	CFI_ADJUST_CFA_OFFSET	RBP
 
-	testl $3,CS(%rsp)
-	je retint_kernel
+	testb	$3, CS(%rsp)
+	jz	retint_kernel
 	/* Interrupt came from user space */
 
 	GET_THREAD_INFO(%rcx)
@@ -740,8 +740,8 @@ retint_kernel:
 #ifdef CONFIG_PREEMPT
 	/* Interrupts are off */
 	/* Check if we need preemption */
-	bt	$9,EFLAGS(%rsp)	/* interrupts were off? */
-	jnc	1f
+	testb	$2,EFLAGS+1(%rsp)	/* interrupts were off? */
+	jz	1f
 0:	cmpl	$0,PER_CPU_VAR(__preempt_count)
 	jnz	1f
 	call	preempt_schedule_irq
@@ -951,8 +951,8 @@ ENTRY(\sym)
 	.if \paranoid
 	.if \paranoid == 1
 	CFI_REMEMBER_STATE
-	testl $3, CS(%rsp)		/* If coming from userspace, switch */
-	jnz 1f				/* stacks. */
+	testb	$3, CS(%rsp)		/* If coming from userspace, switch */
+	jnz	1f			/* stacks. */
 	.endif
 	call paranoid_entry
 	.else
@@ -1292,8 +1292,8 @@ ENTRY(error_entry)
 	SAVE_C_REGS 8
 	SAVE_EXTRA_REGS 8
 	xorl %ebx,%ebx
-	testl $3,CS+8(%rsp)
-	je error_kernelspace
+	testb $3,CS+8(%rsp)
+	jz error_kernelspace
 error_swapgs:
 	SWAPGS
 error_sti:
@@ -1344,7 +1344,7 @@ ENTRY(error_exit)
 	TRACE_IRQS_OFF
 	GET_THREAD_INFO(%rcx)
 	testl %eax,%eax
-	jne retint_kernel
+	jnz retint_kernel
 	LOCKDEP_SYS_EXIT_IRQ
 	movl TI_flags(%rcx),%edx
 	movl $_TIF_WORK_MASK,%edi
-- 
1.8.1.4

[PATCH 7/9] x86/asm/entry/32: tidy up some instructions

[Denys Vlasenko]

After TESTs, use logically correct JZ mnemonic instead of JE
(this doesn't change code).

Tidy up CMPW insns:

Modern CPUs are not good with 16-bit operations.
The instructions with 16-bit immediates are especially bad,
on many CPUs they cause length changing prefix stall
in the decoders, costing ~6 cycles to recover.

Replace CMPWs with CMPLs.
Of these, for form with 8-bit sign-extended immediates
it is a win because they are smaller now
(no 0x66 prefix anymore);
ones with 16-bit immediates are faster.

Signed-off-by: Denys Vlasenko <dvlasenk@redhat.com>
CC: Linus Torvalds <torvalds@linux-foundation.org>
CC: Steven Rostedt <rostedt@goodmis.org>
CC: Ingo Molnar <mingo@kernel.org>
CC: Borislav Petkov <bp@alien8.de>
CC: "H. Peter Anvin" <hpa@zytor.com>
CC: Andy Lutomirski <luto@amacapital.net>
CC: Oleg Nesterov <oleg@redhat.com>
CC: Frederic Weisbecker <fweisbec@gmail.com>
CC: Alexei Starovoitov <ast@plumgrid.com>
CC: Will Drewry <wad@chromium.org>
CC: Kees Cook <keescook@chromium.org>
CC: x86@kernel.org
CC: linux-kernel@vger.kernel.org
---
 arch/x86/kernel/entry_32.S | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/arch/x86/kernel/entry_32.S b/arch/x86/kernel/entry_32.S
index 4c8cc34..9a31d5e 100644
--- a/arch/x86/kernel/entry_32.S
+++ b/arch/x86/kernel/entry_32.S
@@ -432,7 +432,7 @@ sysenter_after_call:
 	TRACE_IRQS_OFF
 	movl TI_flags(%ebp), %ecx
 	testl $_TIF_ALLWORK_MASK, %ecx
-	jne sysexit_audit
+	jnz sysexit_audit
 sysenter_exit:
 /* if something modifies registers it must also disable sysexit */
 	movl PT_EIP(%esp), %edx
@@ -460,7 +460,7 @@ sysenter_audit:
 
 sysexit_audit:
 	testl $(_TIF_ALLWORK_MASK & ~_TIF_SYSCALL_AUDIT), %ecx
-	jne syscall_exit_work
+	jnz syscall_exit_work
 	TRACE_IRQS_ON
 	ENABLE_INTERRUPTS(CLBR_ANY)
 	movl %eax,%edx		/* second arg, syscall return value */
@@ -472,7 +472,7 @@ sysexit_audit:
 	TRACE_IRQS_OFF
 	movl TI_flags(%ebp), %ecx
 	testl $(_TIF_ALLWORK_MASK & ~_TIF_SYSCALL_AUDIT), %ecx
-	jne syscall_exit_work
+	jnz syscall_exit_work
 	movl PT_EAX(%esp),%eax	/* reload syscall return value */
 	jmp sysenter_exit
 #endif
@@ -510,7 +510,7 @@ syscall_exit:
 	TRACE_IRQS_OFF
 	movl TI_flags(%ebp), %ecx
 	testl $_TIF_ALLWORK_MASK, %ecx	# current->work
-	jne syscall_exit_work
+	jnz syscall_exit_work
 
 restore_all:
 	TRACE_IRQS_IRET
@@ -612,7 +612,7 @@ work_notifysig:				# deal with pending signals and
 #ifdef CONFIG_VM86
 	testl $X86_EFLAGS_VM, PT_EFLAGS(%esp)
 	movl %esp, %eax
-	jne work_notifysig_v86		# returning to kernel-space or
+	jnz work_notifysig_v86		# returning to kernel-space or
 					# vm86-space
 1:
 #else
@@ -708,7 +708,7 @@ END(sysenter_badsys)
 #ifdef CONFIG_X86_ESPFIX32
 	movl %ss, %eax
 	/* see if on espfix stack */
-	cmpw $__ESPFIX_SS, %ax
+	cmpl $__ESPFIX_SS, %eax
 	jne 27f
 	movl $__KERNEL_DS, %eax
 	movl %eax, %ds
@@ -1275,7 +1275,7 @@ END(page_fault)
  * the instruction that would have done it for sysenter.
  */
 .macro FIX_STACK offset ok label
-	cmpw $__KERNEL_CS, 4(%esp)
+	cmpl $__KERNEL_CS, 4(%esp)
 	jne \ok
 \label:
 	movl TSS_sysenter_sp0 + \offset(%esp), %esp
@@ -1318,7 +1318,7 @@ ENTRY(nmi)
 #ifdef CONFIG_X86_ESPFIX32
 	pushl_cfi %eax
 	movl %ss, %eax
-	cmpw $__ESPFIX_SS, %ax
+	cmpl $__ESPFIX_SS, %eax
 	popl_cfi %eax
 	je nmi_espfix_stack
 #endif
@@ -1352,7 +1352,7 @@ nmi_stack_fixup:
 
 nmi_debug_stack_check:
 	/* We have a RING0_INT_FRAME here */
-	cmpw $__KERNEL_CS,16(%esp)
+	cmpl $__KERNEL_CS,16(%esp)
 	jne nmi_stack_correct
 	cmpl $debug,(%esp)
 	jb nmi_stack_correct
-- 
1.8.1.4

Re: [PATCH 7/9] x86/asm/entry/32: tidy up some instructions

[Brian Gerst]

On Tue, Mar 31, 2015 at 1:00 PM, Denys Vlasenko <dvlasenk@redhat.com> wrote:
> After TESTs, use logically correct JZ mnemonic instead of JE
> (this doesn't change code).
>
> Tidy up CMPW insns:
>
> Modern CPUs are not good with 16-bit operations.
> The instructions with 16-bit immediates are especially bad,
> on many CPUs they cause length changing prefix stall
> in the decoders, costing ~6 cycles to recover.
>
> Replace CMPWs with CMPLs.
> Of these, for form with 8-bit sign-extended immediates
> it is a win because they are smaller now
> (no 0x66 prefix anymore);
> ones with 16-bit immediates are faster.
>
> @@ -708,7 +708,7 @@ END(sysenter_badsys)
>  #ifdef CONFIG_X86_ESPFIX32
>         movl %ss, %eax
>         /* see if on espfix stack */
> -       cmpw $__ESPFIX_SS, %ax
> +       cmpl $__ESPFIX_SS, %eax
>         jne 27f
>         movl $__KERNEL_DS, %eax
>         movl %eax, %ds

This is incorrect.  32-bit reads from a segment register are not
zero-extended.  The upper 16 bits are implementation-defined.  Most
processors will clear them but it's not guaranteed.

--
Brian Gerst

Re: [PATCH 7/9] x86/asm/entry/32: tidy up some instructions

[Linus Torvalds]

On Tue, Mar 31, 2015 at 3:21 PM, Brian Gerst <brgerst@gmail.com> wrote:
>>
>> @@ -708,7 +708,7 @@ END(sysenter_badsys)
>>  #ifdef CONFIG_X86_ESPFIX32
>>         movl %ss, %eax
>>         /* see if on espfix stack */
>> -       cmpw $__ESPFIX_SS, %ax
>> +       cmpl $__ESPFIX_SS, %eax
>>         jne 27f
>>         movl $__KERNEL_DS, %eax
>>         movl %eax, %ds
>
> This is incorrect.  32-bit reads from a segment register are not
> zero-extended.  The upper 16 bits are implementation-defined.  Most
> processors will clear them but it's not guaranteed.

Indeed. Brian is right. That cmpw needs to stay as a 16-bit compare.

                      Linus

Re: [PATCH 7/9] x86/asm/entry/32: tidy up some instructions

[Denys Vlasenko]

On 04/01/2015 12:21 AM, Brian Gerst wrote:
> On Tue, Mar 31, 2015 at 1:00 PM, Denys Vlasenko <dvlasenk@redhat.com> wrote:
>> After TESTs, use logically correct JZ mnemonic instead of JE
>> (this doesn't change code).
>>
>> Tidy up CMPW insns:
>>
>> Modern CPUs are not good with 16-bit operations.
>> The instructions with 16-bit immediates are especially bad,
>> on many CPUs they cause length changing prefix stall
>> in the decoders, costing ~6 cycles to recover.
>>
>> Replace CMPWs with CMPLs.
>> Of these, for form with 8-bit sign-extended immediates
>> it is a win because they are smaller now
>> (no 0x66 prefix anymore);
>> ones with 16-bit immediates are faster.
>>
>> @@ -708,7 +708,7 @@ END(sysenter_badsys)
>>  #ifdef CONFIG_X86_ESPFIX32
>>         movl %ss, %eax
>>         /* see if on espfix stack */
>> -       cmpw $__ESPFIX_SS, %ax
>> +       cmpl $__ESPFIX_SS, %eax
>>         jne 27f
>>         movl $__KERNEL_DS, %eax
>>         movl %eax, %ds
> 
> This is incorrect.  32-bit reads from a segment register are not
> zero-extended.  The upper 16 bits are implementation-defined.  Most
> processors will clear them but it's not guaranteed.

I did not know that. I was sure they are always zero extended.

Re: [PATCH 7/9] x86/asm/entry/32: tidy up some instructions

[Linus Torvalds]

On Wed, Apr 1, 2015 at 4:10 AM, Denys Vlasenko <dvlasenk@redhat.com> wrote:
>
> I did not know that. I was sure they are always zero extended.

On all half-way modern cpu's they are. But on some older cpu's
(possibly just the original 386) the segment move instructions
basically are always 16-bit, and the operand size is ignored (so the
32-bit version is just smaller and faster to decode, because it
doesn't have a 16-bit operand size prefix)

Iirc, the same is true for the values pushed to memory on exceptions,
so the 'cs/ss' values on the exception stack may not be reliable in
the upper 16 bits.

I don't remember if the same might be true of "pushl %Sseg". The intel
architecture manual says segment registers are zero-extended on push.

                         Linus

Re: [PATCH 7/9] x86/asm/entry/32: tidy up some instructions

[Denys Vlasenko]

On 04/01/2015 05:50 PM, Linus Torvalds wrote:
> On Wed, Apr 1, 2015 at 4:10 AM, Denys Vlasenko <dvlasenk@redhat.com> wrote:
>>
>> I did not know that. I was sure they are always zero extended.
> 
> On all half-way modern cpu's they are. But on some older cpu's
> (possibly just the original 386) the segment move instructions
> basically are always 16-bit, and the operand size is ignored (so the
> 32-bit version is just smaller and faster to decode, because it
> doesn't have a 16-bit operand size prefix)
> 
> Iirc, the same is true for the values pushed to memory on exceptions,
> so the 'cs/ss' values on the exception stack may not be reliable in
> the upper 16 bits.
> 
> I don't remember if the same might be true of "pushl %Sseg". The intel
> architecture manual says segment registers are zero-extended on push.

BTW, AMD64 docs do explicitly say that MOVs from segment registers
to gpregs are zero-extending.

Re: [PATCH 7/9] x86/asm/entry/32: tidy up some instructions

[H. Peter Anvin]

On 04/01/2015 01:52 PM, Denys Vlasenko wrote:
> On 04/01/2015 05:50 PM, Linus Torvalds wrote:
>> On Wed, Apr 1, 2015 at 4:10 AM, Denys Vlasenko <dvlasenk@redhat.com> wrote:
>>>
>>> I did not know that. I was sure they are always zero extended.
>>
>> On all half-way modern cpu's they are. But on some older cpu's
>> (possibly just the original 386) the segment move instructions
>> basically are always 16-bit, and the operand size is ignored (so the
>> 32-bit version is just smaller and faster to decode, because it
>> doesn't have a 16-bit operand size prefix)
>>
>> Iirc, the same is true for the values pushed to memory on exceptions,
>> so the 'cs/ss' values on the exception stack may not be reliable in
>> the upper 16 bits.
>>
>> I don't remember if the same might be true of "pushl %Sseg". The intel
>> architecture manual says segment registers are zero-extended on push.
> 
> BTW, AMD64 docs do explicitly say that MOVs from segment registers
> to gpregs are zero-extending.
> 

For Intel processors it is true for Pentium Pro and later processors, as
far as I know.

	-hpa

Re: [PATCH 7/9] x86/asm/entry/32: tidy up some instructions

[Linus Torvalds]

On Wed, Apr 1, 2015 at 1:52 PM, Denys Vlasenko <dvlasenk@redhat.com> wrote:
>
> BTW, AMD64 docs do explicitly say that MOVs from segment registers
> to gpregs are zero-extending.

Yeah, I think anything even *remotely* recent enough to do 64-bit does
zero-extending.

Even on the 32-bit side, anything that does register renaming is much
better off with zero-extension than with partial register writes.

And I found the "push" thing. It's actually documented:

  "When pushing a segment selector onto the stack, the Pentium 4,
Intel Xeon, P6 family, and Intel486 processors
decrement the ESP register by the operand size and then write 2 bytes.
If the operand size is 32-bits, the upper
two bytes of the write are not modified"

but I can't find any similar documentation for the "mov
Sreg->register" thing. So now I'm starting to doubt my own memory.

                          Linus

Re: [PATCH 7/9] x86/asm/entry/32: tidy up some instructions

[Brian Gerst]

On Wed, Apr 1, 2015 at 6:14 PM, Linus Torvalds
<torvalds@linux-foundation.org> wrote:
> On Wed, Apr 1, 2015 at 1:52 PM, Denys Vlasenko <dvlasenk@redhat.com> wrote:
>>
>> BTW, AMD64 docs do explicitly say that MOVs from segment registers
>> to gpregs are zero-extending.
>
> Yeah, I think anything even *remotely* recent enough to do 64-bit does
> zero-extending.
>
> Even on the 32-bit side, anything that does register renaming is much
> better off with zero-extension than with partial register writes.
>
> And I found the "push" thing. It's actually documented:
>
>   "When pushing a segment selector onto the stack, the Pentium 4,
> Intel Xeon, P6 family, and Intel486 processors
> decrement the ESP register by the operand size and then write 2 bytes.
> If the operand size is 32-bits, the upper
> two bytes of the write are not modified"
>
> but I can't find any similar documentation for the "mov
> Sreg->register" thing. So now I'm starting to doubt my own memory.
>
>                           Linus

It's in the description of MOV:

"When the processor executes the instruction with a 32-bit
general-purpose register, it assumes that the 16 least-significant
bits of the general-purpose register are the destination or source
operand.  If the register is a destination operand, the resulting
value in the two high-order bytes of the register is implementation
dependent. For the Pentium 4, Intel Xeon, and P6 family processors,
the two high-order bytes are filled with zeros; for earlier 32-bit
IA-32 processors, the two high order bytes are undefined."

AMD will always zero-extend, although this applies specifically to
64-bit processors:

"When reading segment-registers with a 32-bit operand size, the
processor zero-extends the 16-bit selector results to 32 bits. When
reading segment-registers with a 64-bit operand size, the processor
zero-extends the 16-bit selector to 64 bits."

So I think it's safe to assume zero-extension on 64-bit, but not 32-bit.

--
Brian Gerst

Re: [PATCH 7/9] x86/asm/entry/32: tidy up some instructions

[Ingo Molnar]

* Denys Vlasenko <dvlasenk@redhat.com> wrote:

> After TESTs, use logically correct JZ mnemonic instead of JE
> (this doesn't change code).
> 
> Tidy up CMPW insns:
> 
> Modern CPUs are not good with 16-bit operations.
> The instructions with 16-bit immediates are especially bad,
> on many CPUs they cause length changing prefix stall
> in the decoders, costing ~6 cycles to recover.
> 
> Replace CMPWs with CMPLs.
> Of these, for form with 8-bit sign-extended immediates
> it is a win because they are smaller now
> (no 0x66 prefix anymore);
> ones with 16-bit immediates are faster.

This patch does JE->JZ transitions, but it also does CMPW instruction 
tweaking - which was buggy as Brian (miraculously!) noticed.

This isn't the first such incident, and I made this point about three 
times already in the past, but it appears I've not made it loud 
enough: which part of 'do not put two unrelated changes into the same 
patch' did you not understand??

We _DO NOT PUT_ multiple, unrelated changes to assembly files into a 
single patch! And we _especially_ don't mix them up under a 
meaningless, repetitive, misleading 'tidy up instructions' title!

Full stop.

The titles of the two patches should have been something like:

 x86/asm/entry/32: Convert JNE to JNZ mnemonics, to improve readability
 x86/asm/entry/32: Optimize CMPW to CMPL instructions, to make use of automatic zero-extend

We were lucky that Brian was alert enough to have read through a 
misleadingly titled, seemingly harmless patch and noticed the bug in 
your patch, but heck you made it hard!!!

And no, it's not a problem if you create a dozen trivial looking 
patches and have to wait a bit more for them to trickle into the 
maintainer tree: asm patches are seldom trivial, and even if they are 
trivial, both reviewability and bisectability will improve from the 
process.

You are doing a nice job improving the x86/asm/entry code, but if you 
cannot create suitably conservative, maximally reviewable and 
maximally bisectable patches to x86/asm then I won't be able to apply 
assembly patches from you!

</rant>

Thanks,

	Ingo

[PATCH 8/9] x86/asm: replace MOVQ $imm,%reg with MOVL

[Denys Vlasenko]

There is no reason to use MOVQ to load a nonnegative immediate
into a 64-bit register. MOVL does the same, since upper 32 bits
are zero extended.

Signed-off-by: Denys Vlasenko <dvlasenk@redhat.com>
CC: Linus Torvalds <torvalds@linux-foundation.org>
CC: Steven Rostedt <rostedt@goodmis.org>
CC: Ingo Molnar <mingo@kernel.org>
CC: Borislav Petkov <bp@alien8.de>
CC: "H. Peter Anvin" <hpa@zytor.com>
CC: Andy Lutomirski <luto@amacapital.net>
CC: Oleg Nesterov <oleg@redhat.com>
CC: Frederic Weisbecker <fweisbec@gmail.com>
CC: Alexei Starovoitov <ast@plumgrid.com>
CC: Will Drewry <wad@chromium.org>
CC: Kees Cook <keescook@chromium.org>
CC: x86@kernel.org
CC: linux-kernel@vger.kernel.org
---
 arch/x86/crypto/crc32c-pcl-intel-asm_64.S | 2 +-
 arch/x86/crypto/twofish-x86_64-asm_64.S   | 4 ++--
 arch/x86/kernel/relocate_kernel_64.S      | 8 ++++----
 3 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/arch/x86/crypto/crc32c-pcl-intel-asm_64.S b/arch/x86/crypto/crc32c-pcl-intel-asm_64.S
index 26d49eb..225be06 100644
--- a/arch/x86/crypto/crc32c-pcl-intel-asm_64.S
+++ b/arch/x86/crypto/crc32c-pcl-intel-asm_64.S
@@ -178,7 +178,7 @@ continue_block:
 	## 2a) PROCESS FULL BLOCKS:
 	################################################################
 full_block:
-	movq    $128,%rax
+	movl    $128,%eax
 	lea     128*8*2(block_0), block_1
 	lea     128*8*3(block_0), block_2
 	add     $128*8*1, block_0
diff --git a/arch/x86/crypto/twofish-x86_64-asm_64.S b/arch/x86/crypto/twofish-x86_64-asm_64.S
index a039d21..a350c99 100644
--- a/arch/x86/crypto/twofish-x86_64-asm_64.S
+++ b/arch/x86/crypto/twofish-x86_64-asm_64.S
@@ -264,7 +264,7 @@ ENTRY(twofish_enc_blk)
 	movq	R1,	8(%rsi)
 
 	popq	R1
-	movq	$1,%rax
+	movl	$1,%eax
 	ret
 ENDPROC(twofish_enc_blk)
 
@@ -316,6 +316,6 @@ ENTRY(twofish_dec_blk)
 	movq	R1,	8(%rsi)
 
 	popq	R1
-	movq	$1,%rax
+	movl	$1,%eax
 	ret
 ENDPROC(twofish_dec_blk)
diff --git a/arch/x86/kernel/relocate_kernel_64.S b/arch/x86/kernel/relocate_kernel_64.S
index 04cb179..98111b3 100644
--- a/arch/x86/kernel/relocate_kernel_64.S
+++ b/arch/x86/kernel/relocate_kernel_64.S
@@ -123,7 +123,7 @@ identity_mapped:
 	 * Set cr4 to a known state:
 	 *  - physical address extension enabled
 	 */
-	movq	$X86_CR4_PAE, %rax
+	movl	$X86_CR4_PAE, %eax
 	movq	%rax, %cr4
 
 	jmp 1f
@@ -246,17 +246,17 @@ swap_pages:
 	movq	%rsi, %rax
 
 	movq	%r10, %rdi
-	movq	$512,   %rcx
+	movl	$512, %ecx
 	rep ; movsq
 
 	movq	%rax, %rdi
 	movq	%rdx, %rsi
-	movq	$512,   %rcx
+	movl	$512, %ecx
 	rep ; movsq
 
 	movq	%rdx, %rdi
 	movq	%r10, %rsi
-	movq	$512,   %rcx
+	movl	$512, %ecx
 	rep ; movsq
 
 	lea	PAGE_SIZE(%rax), %rsi
-- 
1.8.1.4

[tip:x86/asm] x86/asm: Replace "MOVQ $imm, %reg" with MOVL

[tip-bot for Denys Vlasenko]

Commit-ID:  a734b4a23e4b5a5bba577d11b6e2ff21f6ca4fce
Gitweb:     http://git.kernel.org/tip/a734b4a23e4b5a5bba577d11b6e2ff21f6ca4fce
Author:     Denys Vlasenko <dvlasenk@redhat.com>
AuthorDate: Tue, 31 Mar 2015 19:00:10 +0200
Committer:  Ingo Molnar <mingo@kernel.org>
CommitDate: Wed, 1 Apr 2015 13:17:39 +0200

x86/asm: Replace "MOVQ $imm, %reg" with MOVL

There is no reason to use MOVQ to load a non-negative immediate
constant value into a 64-bit register. MOVL does the same, since
the upper 32 bits are zero-extended by the CPU.

This makes the code a bit smaller, while leaving functionality
unchanged.

Signed-off-by: Denys Vlasenko <dvlasenk@redhat.com>
Cc: Alexei Starovoitov <ast@plumgrid.com>
Cc: Andy Lutomirski <luto@amacapital.net>
Cc: Borislav Petkov <bp@alien8.de>
Cc: Frederic Weisbecker <fweisbec@gmail.com>
Cc: H. Peter Anvin <hpa@zytor.com>
Cc: Kees Cook <keescook@chromium.org>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Oleg Nesterov <oleg@redhat.com>
Cc: Steven Rostedt <rostedt@goodmis.org>
Cc: Will Drewry <wad@chromium.org>
Link: http://lkml.kernel.org/r/1427821211-25099-8-git-send-email-dvlasenk@redhat.com
Signed-off-by: Ingo Molnar <mingo@kernel.org>
---
 arch/x86/crypto/crc32c-pcl-intel-asm_64.S | 2 +-
 arch/x86/crypto/twofish-x86_64-asm_64.S   | 4 ++--
 arch/x86/kernel/relocate_kernel_64.S      | 8 ++++----
 3 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/arch/x86/crypto/crc32c-pcl-intel-asm_64.S b/arch/x86/crypto/crc32c-pcl-intel-asm_64.S
index 26d49eb..225be06 100644
--- a/arch/x86/crypto/crc32c-pcl-intel-asm_64.S
+++ b/arch/x86/crypto/crc32c-pcl-intel-asm_64.S
@@ -178,7 +178,7 @@ continue_block:
 	## 2a) PROCESS FULL BLOCKS:
 	################################################################
 full_block:
-	movq    $128,%rax
+	movl    $128,%eax
 	lea     128*8*2(block_0), block_1
 	lea     128*8*3(block_0), block_2
 	add     $128*8*1, block_0
diff --git a/arch/x86/crypto/twofish-x86_64-asm_64.S b/arch/x86/crypto/twofish-x86_64-asm_64.S
index a039d21..a350c99 100644
--- a/arch/x86/crypto/twofish-x86_64-asm_64.S
+++ b/arch/x86/crypto/twofish-x86_64-asm_64.S
@@ -264,7 +264,7 @@ ENTRY(twofish_enc_blk)
 	movq	R1,	8(%rsi)
 
 	popq	R1
-	movq	$1,%rax
+	movl	$1,%eax
 	ret
 ENDPROC(twofish_enc_blk)
 
@@ -316,6 +316,6 @@ ENTRY(twofish_dec_blk)
 	movq	R1,	8(%rsi)
 
 	popq	R1
-	movq	$1,%rax
+	movl	$1,%eax
 	ret
 ENDPROC(twofish_dec_blk)
diff --git a/arch/x86/kernel/relocate_kernel_64.S b/arch/x86/kernel/relocate_kernel_64.S
index 04cb179..98111b3 100644
--- a/arch/x86/kernel/relocate_kernel_64.S
+++ b/arch/x86/kernel/relocate_kernel_64.S
@@ -123,7 +123,7 @@ identity_mapped:
 	 * Set cr4 to a known state:
 	 *  - physical address extension enabled
 	 */
-	movq	$X86_CR4_PAE, %rax
+	movl	$X86_CR4_PAE, %eax
 	movq	%rax, %cr4
 
 	jmp 1f
@@ -246,17 +246,17 @@ swap_pages:
 	movq	%rsi, %rax
 
 	movq	%r10, %rdi
-	movq	$512,   %rcx
+	movl	$512, %ecx
 	rep ; movsq
 
 	movq	%rax, %rdi
 	movq	%rdx, %rsi
-	movq	$512,   %rcx
+	movl	$512, %ecx
 	rep ; movsq
 
 	movq	%rdx, %rdi
 	movq	%r10, %rsi
-	movq	$512,   %rcx
+	movl	$512, %ecx
 	rep ; movsq
 
 	lea	PAGE_SIZE(%rax), %rsi

[PATCH 9/9] x86/asm/entry/64: use local label to skip around sycall dispatch

[Denys Vlasenko]

Logically, we just want to jump around the "call *sys_call_table(,%rax,8)"
if syscall number is too big, we do not specifically target
"int_ret_from_sys_call" label.

Use a local numeric label for this jump.

This also makes code smaller because jumps to global labels
are never translated to short jump insns by as.

Signed-off-by: Denys Vlasenko <dvlasenk@redhat.com>
CC: Linus Torvalds <torvalds@linux-foundation.org>
CC: Steven Rostedt <rostedt@goodmis.org>
CC: Ingo Molnar <mingo@kernel.org>
CC: Borislav Petkov <bp@alien8.de>
CC: "H. Peter Anvin" <hpa@zytor.com>
CC: Andy Lutomirski <luto@amacapital.net>
CC: Oleg Nesterov <oleg@redhat.com>
CC: Frederic Weisbecker <fweisbec@gmail.com>
CC: Alexei Starovoitov <ast@plumgrid.com>
CC: Will Drewry <wad@chromium.org>
CC: Kees Cook <keescook@chromium.org>
CC: x86@kernel.org
CC: linux-kernel@vger.kernel.org
---
 arch/x86/kernel/entry_64.S | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/arch/x86/kernel/entry_64.S b/arch/x86/kernel/entry_64.S
index f49f973..e498542 100644
--- a/arch/x86/kernel/entry_64.S
+++ b/arch/x86/kernel/entry_64.S
@@ -331,10 +331,11 @@ tracesys_phase2:
 	andl $__SYSCALL_MASK,%eax
 	cmpl $__NR_syscall_max,%eax
 #endif
-	ja   int_ret_from_sys_call	/* RAX(%rsp) is already set */
+	ja	1f	/* return -ENOSYS (already in pt_regs->ax) */
 	movq %r10,%rcx	/* fixup for C */
 	call *sys_call_table(,%rax,8)
 	movq %rax,RAX(%rsp)
+1:
 	/* Use IRET because user could have changed pt_regs->foo */
 
 /*
-- 
1.8.1.4

[tip:x86/asm] x86/asm/entry/64: Use local label to skip around sycall dispatch

[tip-bot for Denys Vlasenko]

Commit-ID:  a6de5a21fb25cdbbdf3c3e9afd8481581c4f2464
Gitweb:     http://git.kernel.org/tip/a6de5a21fb25cdbbdf3c3e9afd8481581c4f2464
Author:     Denys Vlasenko <dvlasenk@redhat.com>
AuthorDate: Tue, 31 Mar 2015 19:00:11 +0200
Committer:  Ingo Molnar <mingo@kernel.org>
CommitDate: Wed, 1 Apr 2015 13:17:39 +0200

x86/asm/entry/64: Use local label to skip around sycall dispatch

Logically, we just want to jump around the following instruction
and its prologue/epilogue:

  call *sys_call_table(,%rax,8)

if the syscall number is too big - we do not specifically target
the "int_ret_from_sys_call" label.

Use a local, numerical label for this jump, for more clarity.

This also makes the code smaller:

 -ffffffff8187756b:      0f 87 0f 00 00 00       ja     ffffffff81877580 <int_ret_from_sys_call>
 +ffffffff8187756b:      77 0f                   ja     ffffffff8187757c <int_ret_from_sys_call>

because jumps to global labels are never translated to short jump
instructions by GAS.

Signed-off-by: Denys Vlasenko <dvlasenk@redhat.com>
Cc: Alexei Starovoitov <ast@plumgrid.com>
Cc: Andy Lutomirski <luto@amacapital.net>
Cc: Borislav Petkov <bp@alien8.de>
Cc: Frederic Weisbecker <fweisbec@gmail.com>
Cc: H. Peter Anvin <hpa@zytor.com>
Cc: Kees Cook <keescook@chromium.org>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Oleg Nesterov <oleg@redhat.com>
Cc: Steven Rostedt <rostedt@goodmis.org>
Cc: Will Drewry <wad@chromium.org>
Link: http://lkml.kernel.org/r/1427821211-25099-9-git-send-email-dvlasenk@redhat.com
[ Improved the changelog. ]
Signed-off-by: Ingo Molnar <mingo@kernel.org>
---
 arch/x86/kernel/entry_64.S | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/arch/x86/kernel/entry_64.S b/arch/x86/kernel/entry_64.S
index bad285d..03c52e2 100644
--- a/arch/x86/kernel/entry_64.S
+++ b/arch/x86/kernel/entry_64.S
@@ -331,10 +331,11 @@ tracesys_phase2:
 	andl $__SYSCALL_MASK,%eax
 	cmpl $__NR_syscall_max,%eax
 #endif
-	ja   int_ret_from_sys_call	/* RAX(%rsp) is already set */
+	ja	1f	/* return -ENOSYS (already in pt_regs->ax) */
 	movq %r10,%rcx	/* fixup for C */
 	call *sys_call_table(,%rax,8)
 	movq %rax,RAX(%rsp)
+1:
 	/* Use IRET because user could have changed pt_regs->foo */
 
 /*

[tip:x86/asm] x86/asm/entry/64: Do not TRACE_IRQS fast SYSRET64 path

[tip-bot for Denys Vlasenko]

Commit-ID:  4416c5a6dacdddd55378e7011f9c8720d2a7470f
Gitweb:     http://git.kernel.org/tip/4416c5a6dacdddd55378e7011f9c8720d2a7470f
Author:     Denys Vlasenko <dvlasenk@redhat.com>
AuthorDate: Tue, 31 Mar 2015 19:00:03 +0200
Committer:  Ingo Molnar <mingo@kernel.org>
CommitDate: Wed, 1 Apr 2015 13:17:38 +0200

x86/asm/entry/64: Do not TRACE_IRQS fast SYSRET64 path

SYSRET code path has a small irq-off block.
On this code path, TRACE_IRQS_ON can't be called right before
interrupts are enabled for real, we can't clobber registers
there. So current code does it earlier, in a safe place.

But with this, TRACE_IRQS_OFF/ON frames just two fast
instructions, which is ridiculous: now most of irq-off block is
_outside_ of the framing.

Do the same thing that we do on SYSCALL entry: do not track this
irq-off block, it is very small to ever cause noticeable irq
latency.

Be careful: make sure that "jnz int_ret_from_sys_call_irqs_off"
now does invoke TRACE_IRQS_OFF - move
int_ret_from_sys_call_irqs_off label before TRACE_IRQS_OFF.

Signed-off-by: Denys Vlasenko <dvlasenk@redhat.com>
Cc: Alexei Starovoitov <ast@plumgrid.com>
Cc: Andy Lutomirski <luto@amacapital.net>
Cc: Borislav Petkov <bp@alien8.de>
Cc: Frederic Weisbecker <fweisbec@gmail.com>
Cc: H. Peter Anvin <hpa@zytor.com>
Cc: Kees Cook <keescook@chromium.org>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Oleg Nesterov <oleg@redhat.com>
Cc: Steven Rostedt <rostedt@goodmis.org>
Cc: Will Drewry <wad@chromium.org>
Link: http://lkml.kernel.org/r/1427821211-25099-1-git-send-email-dvlasenk@redhat.com
Signed-off-by: Ingo Molnar <mingo@kernel.org>
---
 arch/x86/kernel/entry_64.S | 13 +++++++------
 1 file changed, 7 insertions(+), 6 deletions(-)

diff --git a/arch/x86/kernel/entry_64.S b/arch/x86/kernel/entry_64.S
index 6f251a5..f6e37de 100644
--- a/arch/x86/kernel/entry_64.S
+++ b/arch/x86/kernel/entry_64.S
@@ -269,8 +269,11 @@ system_call_fastpath:
  * Has incompletely filled pt_regs.
  */
 	LOCKDEP_SYS_EXIT
+	/*
+	 * We do not frame this tiny irq-off block with TRACE_IRQS_OFF/ON,
+	 * it is too small to ever cause noticeable irq latency.
+	 */
 	DISABLE_INTERRUPTS(CLBR_NONE)
-	TRACE_IRQS_OFF
 
 	/*
 	 * We must check ti flags with interrupts (or at least preemption)
@@ -284,10 +287,7 @@ system_call_fastpath:
 	jnz int_ret_from_sys_call_irqs_off	/* Go to the slow path */
 
 	CFI_REMEMBER_STATE
-	/*
-	 * sysretq will re-enable interrupts:
-	 */
-	TRACE_IRQS_ON
+
 	RESTORE_C_REGS_EXCEPT_RCX_R11
 	movq	RIP(%rsp),%rcx
 	CFI_REGISTER	rip,rcx
@@ -298,6 +298,7 @@ system_call_fastpath:
 	 * 64bit SYSRET restores rip from rcx,
 	 * rflags from r11 (but RF and VM bits are forced to 0),
 	 * cs and ss are loaded from MSRs.
+	 * Restoration of rflags re-enables interrupts.
 	 */
 	USERGS_SYSRET64
 
@@ -346,8 +347,8 @@ tracesys_phase2:
  */
 GLOBAL(int_ret_from_sys_call)
 	DISABLE_INTERRUPTS(CLBR_NONE)
+int_ret_from_sys_call_irqs_off: /* jumps come here from the irqs-off SYSRET path */
 	TRACE_IRQS_OFF
-int_ret_from_sys_call_irqs_off:
 	movl $_TIF_ALLWORK_MASK,%edi
 	/* edi:	mask to check */
 GLOBAL(int_with_check)