From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1753608AbbCIUCf (ORCPT ); Mon, 9 Mar 2015 16:02:35 -0400 Received: from mail-lb0-f178.google.com ([209.85.217.178]:36976 "EHLO mail-lb0-f178.google.com" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S1751666AbbCIUCd (ORCPT ); Mon, 9 Mar 2015 16:02:33 -0400 MIME-Version: 1.0 In-Reply-To: <1425926364-9526-2-git-send-email-dvlasenk@redhat.com> References: <1425926364-9526-1-git-send-email-dvlasenk@redhat.com> <1425926364-9526-2-git-send-email-dvlasenk@redhat.com> From: Andy Lutomirski Date: Mon, 9 Mar 2015 13:02:11 -0700 Message-ID: Subject: Re: [PATCH 1/4] x86: save r11 into pt_regs->flags on SYSCALL64 fastpath To: Denys Vlasenko Cc: Linus Torvalds , Steven Rostedt , Ingo Molnar , Borislav Petkov , "H. Peter Anvin" , Oleg Nesterov , Frederic Weisbecker , Alexei Starovoitov , Will Drewry , Kees Cook , X86 ML , "linux-kernel@vger.kernel.org" Content-Type: text/plain; charset=UTF-8 Sender: linux-kernel-owner@vger.kernel.org List-ID: X-Mailing-List: linux-kernel@vger.kernel.org On Mon, Mar 9, 2015 at 11:39 AM, Denys Vlasenko wrote: > Before this patch, r11 was saved in pt_regs->r11. > Which looks natural, but requires messy shuffling to/from iret frame > whenever ptrace or e.g. sys_iopl wants to modify flags - because > that's how this register is used by SYSCALL/SYSRET. > > This patch saves r11 in pt_regs->flags, > and uses that value for SYSRET64 insn. Shuffling is eliminated. > > FIXUP/RESTORE_TOP_OF_STACK are simplified. > > stub_iopl is no longer needed: pt_regs->flags needs no fixing up. > > Testing shows that syscall fast path is ~54.3 ns before > and after the patch (on 2.7 GHz Sandy Bridge CPU). Acked-by: Andy Lutomirski > > Signed-off-by: Denys Vlasenko > CC: Linus Torvalds > CC: Steven Rostedt > CC: Ingo Molnar > CC: Borislav Petkov > CC: "H. Peter Anvin" > CC: Andy Lutomirski > CC: Oleg Nesterov > CC: Frederic Weisbecker > CC: Alexei Starovoitov > CC: Will Drewry > CC: Kees Cook > CC: x86@kernel.org > CC: linux-kernel@vger.kernel.org > --- > arch/x86/include/asm/calling.h | 20 ++++++++++++++------ > arch/x86/kernel/entry_64.S | 24 +++++++++++------------- > 2 files changed, 25 insertions(+), 19 deletions(-) > > diff --git a/arch/x86/include/asm/calling.h b/arch/x86/include/asm/calling.h > index f1a962f..4b5f7bf 100644 > --- a/arch/x86/include/asm/calling.h > +++ b/arch/x86/include/asm/calling.h > @@ -95,9 +95,11 @@ For 32-bit we have the following conventions - kernel is built with > CFI_ADJUST_CFA_OFFSET 15*8+\addskip > .endm > > - .macro SAVE_C_REGS_HELPER offset=0 rax=1 rcx=1 r8plus=1 > - .if \r8plus > + .macro SAVE_C_REGS_HELPER offset=0 rax=1 rcx=1 r8910=1 r11=1 > + .if \r11 > movq_cfi r11, 6*8+\offset > + .endif > + .if \r8910 > movq_cfi r10, 7*8+\offset > movq_cfi r9, 8*8+\offset > movq_cfi r8, 9*8+\offset > @@ -113,16 +115,19 @@ For 32-bit we have the following conventions - kernel is built with > movq_cfi rdi, 14*8+\offset > .endm > .macro SAVE_C_REGS offset=0 > - SAVE_C_REGS_HELPER \offset, 1, 1, 1 > + SAVE_C_REGS_HELPER \offset, 1, 1, 1, 1 > .endm > .macro SAVE_C_REGS_EXCEPT_RAX_RCX offset=0 > - SAVE_C_REGS_HELPER \offset, 0, 0, 1 > + SAVE_C_REGS_HELPER \offset, 0, 0, 1, 1 > .endm > .macro SAVE_C_REGS_EXCEPT_R891011 > - SAVE_C_REGS_HELPER 0, 1, 1, 0 > + SAVE_C_REGS_HELPER 0, 1, 1, 0, 0 > .endm > .macro SAVE_C_REGS_EXCEPT_RCX_R891011 > - SAVE_C_REGS_HELPER 0, 1, 0, 0 > + SAVE_C_REGS_HELPER 0, 1, 0, 0, 0 > + .endm > + .macro SAVE_C_REGS_EXCEPT_RAX_RCX_R11 > + SAVE_C_REGS_HELPER 0, 0, 0, 1, 0 > .endm > > .macro SAVE_EXTRA_REGS offset=0 > @@ -179,6 +184,9 @@ For 32-bit we have the following conventions - kernel is built with > .macro RESTORE_C_REGS_EXCEPT_R11 > RESTORE_C_REGS_HELPER 1,1,0,1,1 > .endm > + .macro RESTORE_C_REGS_EXCEPT_RCX_R11 > + RESTORE_C_REGS_HELPER 1,0,0,1,1 > + .endm > .macro RESTORE_RSI_RDI > RESTORE_C_REGS_HELPER 0,0,0,0,0 > .endm > diff --git a/arch/x86/kernel/entry_64.S b/arch/x86/kernel/entry_64.S > index 5117a2b..324200a 100644 > --- a/arch/x86/kernel/entry_64.S > +++ b/arch/x86/kernel/entry_64.S > @@ -121,14 +121,12 @@ ENDPROC(native_usergs_sysret64) > #endif > > /* > - * C code is not supposed to know about undefined top of stack. Every time > - * a C function with an pt_regs argument is called from the SYSCALL based > - * fast path FIXUP_TOP_OF_STACK is needed. > + * C code is not supposed to know that the iret frame is not populated. > + * Every time a C function with an pt_regs argument is called from > + * the SYSCALL based fast path FIXUP_TOP_OF_STACK is needed. > * RESTORE_TOP_OF_STACK syncs the syscall state after any possible ptregs > * manipulation. > */ > - > - /* %rsp:at FRAMEEND */ > .macro FIXUP_TOP_OF_STACK tmp offset=0 > movq PER_CPU_VAR(old_rsp),\tmp > movq \tmp,RSP+\offset(%rsp) > @@ -136,15 +134,13 @@ ENDPROC(native_usergs_sysret64) > movq $__USER_CS,CS+\offset(%rsp) > movq RIP+\offset(%rsp),\tmp /* get rip */ > movq \tmp,RCX+\offset(%rsp) /* copy it to rcx as sysret would do */ > - movq R11+\offset(%rsp),\tmp /* get eflags */ > - movq \tmp,EFLAGS+\offset(%rsp) > + movq EFLAGS+\offset(%rsp),\tmp /* ditto for rflags->r11 */ > + movq \tmp,R11+\offset(%rsp) > .endm > > .macro RESTORE_TOP_OF_STACK tmp offset=0 > movq RSP+\offset(%rsp),\tmp > movq \tmp,PER_CPU_VAR(old_rsp) > - movq EFLAGS+\offset(%rsp),\tmp > - movq \tmp,R11+\offset(%rsp) > .endm > > /* > @@ -257,9 +253,10 @@ GLOBAL(system_call_after_swapgs) > */ > ENABLE_INTERRUPTS(CLBR_NONE) > ALLOC_PT_GPREGS_ON_STACK 8 /* +8: space for orig_ax */ > - SAVE_C_REGS_EXCEPT_RAX_RCX > + SAVE_C_REGS_EXCEPT_RAX_RCX_R11 > movq $-ENOSYS,RAX(%rsp) > movq_cfi rax,ORIG_RAX > + movq %r11,EFLAGS(%rsp) > movq %rcx,RIP(%rsp) > CFI_REL_OFFSET rip,RIP > testl $_TIF_WORK_SYSCALL_ENTRY,TI_flags+THREAD_INFO(%rsp,RIP) > @@ -277,7 +274,7 @@ system_call_fastpath: > movq %rax,RAX(%rsp) > /* > * Syscall return path ending with SYSRET (fast path) > - * Has incomplete stack frame and undefined top of stack. > + * Has incompletely filled pt_regs, iret frame is also incomplete. > */ > ret_from_sys_call: > testl $_TIF_ALLWORK_MASK,TI_flags+THREAD_INFO(%rsp,RIP) > @@ -291,9 +288,10 @@ ret_from_sys_call: > * sysretq will re-enable interrupts: > */ > TRACE_IRQS_ON > - RESTORE_C_REGS_EXCEPT_RCX > - movq RIP(%rsp),%rcx > + RESTORE_C_REGS_EXCEPT_RCX_R11 > + movq RIP(%rsp),%rcx > CFI_REGISTER rip,rcx > + movq EFLAGS(%rsp),%r11 > /*CFI_REGISTER rflags,r11*/ > movq PER_CPU_VAR(old_rsp), %rsp > /* > -- > 1.8.1.4 > -- Andy Lutomirski AMA Capital Management, LLC