From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1753311AbbCPMFn (ORCPT ); Mon, 16 Mar 2015 08:05:43 -0400 Received: from terminus.zytor.com ([198.137.202.10]:35851 "EHLO terminus.zytor.com" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S1750936AbbCPMFk (ORCPT ); Mon, 16 Mar 2015 08:05:40 -0400 Date: Mon, 16 Mar 2015 05:04:53 -0700 From: tip-bot for Denys Vlasenko Message-ID: Cc: dvlasenk@redhat.com, torvalds@linux-foundation.org, keescook@chromium.org, fweisbec@gmail.com, oleg@redhat.com, rostedt@goodmis.org, hpa@zytor.com, mingo@kernel.org, linux-kernel@vger.kernel.org, ast@plumgrid.com, bp@alien8.de, luto@amacapital.net, tglx@linutronix.de, wad@chromium.org Reply-To: luto@amacapital.net, tglx@linutronix.de, wad@chromium.org, ast@plumgrid.com, bp@alien8.de, keescook@chromium.org, fweisbec@gmail.com, oleg@redhat.com, hpa@zytor.com, mingo@kernel.org, rostedt@goodmis.org, linux-kernel@vger.kernel.org, dvlasenk@redhat.com, torvalds@linux-foundation.org In-Reply-To: <1425926364-9526-2-git-send-email-dvlasenk@redhat.com> References: <1425926364-9526-2-git-send-email-dvlasenk@redhat.com> To: linux-tip-commits@vger.kernel.org Subject: [tip:x86/asm] x86/asm/entry/64: Save R11 into pt_regs-> flags on SYSCALL64 fastpath Git-Commit-ID: 29722cd4ef666705b2eda1c3ba44435488e509eb X-Mailer: tip-git-log-daemon Robot-ID: Robot-Unsubscribe: Contact to get blacklisted from these emails MIME-Version: 1.0 Content-Transfer-Encoding: 8bit Content-Type: text/plain; charset=UTF-8 Content-Disposition: inline Sender: linux-kernel-owner@vger.kernel.org List-ID: X-Mailing-List: linux-kernel@vger.kernel.org Commit-ID: 29722cd4ef666705b2eda1c3ba44435488e509eb Gitweb: http://git.kernel.org/tip/29722cd4ef666705b2eda1c3ba44435488e509eb Author: Denys Vlasenko AuthorDate: Mon, 9 Mar 2015 19:39:21 +0100 Committer: Ingo Molnar CommitDate: Tue, 10 Mar 2015 13:56:10 +0100 x86/asm/entry/64: Save R11 into pt_regs->flags on SYSCALL64 fastpath Before this patch, R11 was saved in pt_regs->r11. Which looks natural, but requires messy shuffling to/from iret frame whenever ptrace or e.g. sys_iopl() wants to modify flags - because that's how this register is used by SYSCALL/SYSRET. This patch saves R11 in pt_regs->flags, and uses that value for the SYSRET64 instruction. Shuffling is eliminated. FIXUP/RESTORE_TOP_OF_STACK are simplified. stub_iopl is no longer needed: pt_regs->flags needs no fixing up. Testing shows that syscall fast path is ~54.3 ns before and after the patch (on 2.7 GHz Sandy Bridge CPU). Signed-off-by: Denys Vlasenko Cc: Alexei Starovoitov Cc: Andy Lutomirski Cc: Borislav Petkov Cc: Frederic Weisbecker Cc: H. Peter Anvin Cc: Kees Cook Cc: Linus Torvalds Cc: Oleg Nesterov Cc: Steven Rostedt Cc: Will Drewry Link: http://lkml.kernel.org/r/1425926364-9526-2-git-send-email-dvlasenk@redhat.com Signed-off-by: Ingo Molnar --- arch/x86/include/asm/calling.h | 20 ++++++++++++++------ arch/x86/kernel/entry_64.S | 24 +++++++++++------------- 2 files changed, 25 insertions(+), 19 deletions(-) diff --git a/arch/x86/include/asm/calling.h b/arch/x86/include/asm/calling.h index f1a962f..4b5f7bf 100644 --- a/arch/x86/include/asm/calling.h +++ b/arch/x86/include/asm/calling.h @@ -95,9 +95,11 @@ For 32-bit we have the following conventions - kernel is built with CFI_ADJUST_CFA_OFFSET 15*8+\addskip .endm - .macro SAVE_C_REGS_HELPER offset=0 rax=1 rcx=1 r8plus=1 - .if \r8plus + .macro SAVE_C_REGS_HELPER offset=0 rax=1 rcx=1 r8910=1 r11=1 + .if \r11 movq_cfi r11, 6*8+\offset + .endif + .if \r8910 movq_cfi r10, 7*8+\offset movq_cfi r9, 8*8+\offset movq_cfi r8, 9*8+\offset @@ -113,16 +115,19 @@ For 32-bit we have the following conventions - kernel is built with movq_cfi rdi, 14*8+\offset .endm .macro SAVE_C_REGS offset=0 - SAVE_C_REGS_HELPER \offset, 1, 1, 1 + SAVE_C_REGS_HELPER \offset, 1, 1, 1, 1 .endm .macro SAVE_C_REGS_EXCEPT_RAX_RCX offset=0 - SAVE_C_REGS_HELPER \offset, 0, 0, 1 + SAVE_C_REGS_HELPER \offset, 0, 0, 1, 1 .endm .macro SAVE_C_REGS_EXCEPT_R891011 - SAVE_C_REGS_HELPER 0, 1, 1, 0 + SAVE_C_REGS_HELPER 0, 1, 1, 0, 0 .endm .macro SAVE_C_REGS_EXCEPT_RCX_R891011 - SAVE_C_REGS_HELPER 0, 1, 0, 0 + SAVE_C_REGS_HELPER 0, 1, 0, 0, 0 + .endm + .macro SAVE_C_REGS_EXCEPT_RAX_RCX_R11 + SAVE_C_REGS_HELPER 0, 0, 0, 1, 0 .endm .macro SAVE_EXTRA_REGS offset=0 @@ -179,6 +184,9 @@ For 32-bit we have the following conventions - kernel is built with .macro RESTORE_C_REGS_EXCEPT_R11 RESTORE_C_REGS_HELPER 1,1,0,1,1 .endm + .macro RESTORE_C_REGS_EXCEPT_RCX_R11 + RESTORE_C_REGS_HELPER 1,0,0,1,1 + .endm .macro RESTORE_RSI_RDI RESTORE_C_REGS_HELPER 0,0,0,0,0 .endm diff --git a/arch/x86/kernel/entry_64.S b/arch/x86/kernel/entry_64.S index 5117a2b..324200a 100644 --- a/arch/x86/kernel/entry_64.S +++ b/arch/x86/kernel/entry_64.S @@ -121,14 +121,12 @@ ENDPROC(native_usergs_sysret64) #endif /* - * C code is not supposed to know about undefined top of stack. Every time - * a C function with an pt_regs argument is called from the SYSCALL based - * fast path FIXUP_TOP_OF_STACK is needed. + * C code is not supposed to know that the iret frame is not populated. + * Every time a C function with an pt_regs argument is called from + * the SYSCALL based fast path FIXUP_TOP_OF_STACK is needed. * RESTORE_TOP_OF_STACK syncs the syscall state after any possible ptregs * manipulation. */ - - /* %rsp:at FRAMEEND */ .macro FIXUP_TOP_OF_STACK tmp offset=0 movq PER_CPU_VAR(old_rsp),\tmp movq \tmp,RSP+\offset(%rsp) @@ -136,15 +134,13 @@ ENDPROC(native_usergs_sysret64) movq $__USER_CS,CS+\offset(%rsp) movq RIP+\offset(%rsp),\tmp /* get rip */ movq \tmp,RCX+\offset(%rsp) /* copy it to rcx as sysret would do */ - movq R11+\offset(%rsp),\tmp /* get eflags */ - movq \tmp,EFLAGS+\offset(%rsp) + movq EFLAGS+\offset(%rsp),\tmp /* ditto for rflags->r11 */ + movq \tmp,R11+\offset(%rsp) .endm .macro RESTORE_TOP_OF_STACK tmp offset=0 movq RSP+\offset(%rsp),\tmp movq \tmp,PER_CPU_VAR(old_rsp) - movq EFLAGS+\offset(%rsp),\tmp - movq \tmp,R11+\offset(%rsp) .endm /* @@ -257,9 +253,10 @@ GLOBAL(system_call_after_swapgs) */ ENABLE_INTERRUPTS(CLBR_NONE) ALLOC_PT_GPREGS_ON_STACK 8 /* +8: space for orig_ax */ - SAVE_C_REGS_EXCEPT_RAX_RCX + SAVE_C_REGS_EXCEPT_RAX_RCX_R11 movq $-ENOSYS,RAX(%rsp) movq_cfi rax,ORIG_RAX + movq %r11,EFLAGS(%rsp) movq %rcx,RIP(%rsp) CFI_REL_OFFSET rip,RIP testl $_TIF_WORK_SYSCALL_ENTRY,TI_flags+THREAD_INFO(%rsp,RIP) @@ -277,7 +274,7 @@ system_call_fastpath: movq %rax,RAX(%rsp) /* * Syscall return path ending with SYSRET (fast path) - * Has incomplete stack frame and undefined top of stack. + * Has incompletely filled pt_regs, iret frame is also incomplete. */ ret_from_sys_call: testl $_TIF_ALLWORK_MASK,TI_flags+THREAD_INFO(%rsp,RIP) @@ -291,9 +288,10 @@ ret_from_sys_call: * sysretq will re-enable interrupts: */ TRACE_IRQS_ON - RESTORE_C_REGS_EXCEPT_RCX - movq RIP(%rsp),%rcx + RESTORE_C_REGS_EXCEPT_RCX_R11 + movq RIP(%rsp),%rcx CFI_REGISTER rip,rcx + movq EFLAGS(%rsp),%r11 /*CFI_REGISTER rflags,r11*/ movq PER_CPU_VAR(old_rsp), %rsp /*