[PATCH v2 0/2] x86/entry: speed up context-tracking system calls by 150 clock cycles

[PATCH v2 0/2] x86/entry: speed up context-tracking system calls by 150 clock cycles

[Paolo Bonzini]

The first patches are the two optimizations I posted on May 30th
for the system call entry/exit code.  The only change is in the
function names, which use the user_{enter,exit}_irqoff favored
by Andy and Ingo.  The first patch matches what commit d0e536d8939
("context_tracking: avoid irq_save/irq_restore on guest entry and exit",
2015-10-28) did for guest entry and exit.  The second simply adds
an inline annotation; the compiler doesn't figure it out because the
function is not static.

The second two patches move guest_{enter,exit} to the same naming
convention, removing the KVM wrappers kvm_guest_{enter,exit} and
__kvm_guest_{enter,exit} in the process.  I would like these two to
go through the KVM tree because I have other optimizations for 4.8
on top of these patches.

Thanks,

Paolo

Paolo Bonzini (4):
  x86/entry: Avoid interrupt flag save and restore
  x86/entry: Inline enter_from_user_mode
  context_tracking: move rcu_virt_note_context_switch out of kvm_host.h
  KVM: remove kvm_guest_enter/exit wrappers

 arch/arm/kvm/arm.c               |  8 +++---
 arch/mips/kvm/mips.c             |  4 +--
 arch/powerpc/kvm/book3s_hv.c     |  4 +--
 arch/powerpc/kvm/book3s_pr.c     |  4 +--
 arch/powerpc/kvm/booke.c         |  4 +--
 arch/powerpc/kvm/powerpc.c       |  2 +-
 arch/s390/kvm/kvm-s390.c         |  4 +--
 arch/x86/entry/common.c          |  6 ++---
 arch/x86/kvm/x86.c               |  4 +--
 include/linux/context_tracking.h | 53 +++++++++++++++++++++++++++++++++++++---
 include/linux/kvm_host.h         | 39 -----------------------------
 11 files changed, 69 insertions(+), 63 deletions(-)

-- 
1.8.3.1

[PATCH 1/2] x86/entry: Avoid interrupt flag save and restore

[Paolo Bonzini]

Thanks to all the work that was done by Andy Lutomirski and others,
enter_from_user_mode and prepare_exit_to_usermode are now called only with
interrupts disabled.  Let's provide them a version of user_enter/user_exit
that skips saving and restoring the interrupt flag.

On an AMD-based machine I tested this patch on, with force-enabled
context tracking, the speed-up in system calls was 90 clock cycles or 6%,
measured with the following simple benchmark:

    #include <sys/signal.h>
    #include <time.h>
    #include <unistd.h>
    #include <stdio.h>

    unsigned long rdtsc()
    {
        unsigned long result;
        asm volatile("rdtsc; shl $32, %%rdx; mov %%eax, %%eax\n"
                     "or %%rdx, %%rax" : "=a" (result) : : "rdx");
        return result;
    }

    int main()
    {
        unsigned long tsc1, tsc2;
        int pid = getpid();
        int i;

        tsc1 = rdtsc();
        for (i = 0; i < 100000000; i++)
            kill(pid, SIGWINCH);
        tsc2 = rdtsc();

        printf("%ld\n", tsc2 - tsc1);
    }

Cc: Andy Lutomirski <luto@kernel.org>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Rik van Riel <riel@redhat.com>
Cc: H. Peter Anvin <hpa@zytor.com>
Cc: Ingo Molnar <mingo@kernel.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>
---
 arch/x86/entry/common.c          |  4 ++--
 include/linux/context_tracking.h | 15 +++++++++++++++
 2 files changed, 17 insertions(+), 2 deletions(-)

diff --git a/arch/x86/entry/common.c b/arch/x86/entry/common.c
index ec138e538c44..618bc61d35b7 100644
--- a/arch/x86/entry/common.c
+++ b/arch/x86/entry/common.c
@@ -43,7 +43,7 @@ static struct thread_info *pt_regs_to_thread_info(struct pt_regs *regs)
 __visible void enter_from_user_mode(void)
 {
 	CT_WARN_ON(ct_state() != CONTEXT_USER);
-	user_exit();
+	user_exit_irqoff();
 }
 #else
 static inline void enter_from_user_mode(void) {}
@@ -274,7 +274,7 @@ __visible inline void prepare_exit_to_usermode(struct pt_regs *regs)
 	ti->status &= ~TS_COMPAT;
 #endif
 
-	user_enter();
+	user_enter_irqoff();
 }
 
 #define SYSCALL_EXIT_WORK_FLAGS				\
diff --git a/include/linux/context_tracking.h b/include/linux/context_tracking.h
index d259274238db..d9aef2a0ec8e 100644
--- a/include/linux/context_tracking.h
+++ b/include/linux/context_tracking.h
@@ -31,6 +31,19 @@ static inline void user_exit(void)
 		context_tracking_exit(CONTEXT_USER);
 }
 
+/* Called with interrupts disabled.  */
+static inline void user_enter_irqoff(void)
+{
+	if (context_tracking_is_enabled())
+		__context_tracking_enter(CONTEXT_USER);
+
+}
+static inline void user_exit_irqoff(void)
+{
+	if (context_tracking_is_enabled())
+		__context_tracking_exit(CONTEXT_USER);
+}
+
 static inline enum ctx_state exception_enter(void)
 {
 	enum ctx_state prev_ctx;
@@ -69,6 +82,8 @@ static inline enum ctx_state ct_state(void)
 #else
 static inline void user_enter(void) { }
 static inline void user_exit(void) { }
+static inline void user_enter_irqoff(void) { }
+static inline void user_exit_irqoff(void) { }
 static inline enum ctx_state exception_enter(void) { return 0; }
 static inline void exception_exit(enum ctx_state prev_ctx) { }
 static inline enum ctx_state ct_state(void) { return CONTEXT_DISABLED; }
-- 
1.8.3.1

[PATCH 2/2] x86/entry: Inline enter_from_user_mode

[Paolo Bonzini]

This matches what is already done for prepare_exit_to_usermode,
and saves about 60 clock cycles (4% speedup) with the benchmark
in the previous commit message.

Cc: Andy Lutomirski <luto@kernel.org>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Rik van Riel <riel@redhat.com>
Cc: H. Peter Anvin <hpa@zytor.com>
Cc: Ingo Molnar <mingo@kernel.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>
---
 arch/x86/entry/common.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/arch/x86/entry/common.c b/arch/x86/entry/common.c
index 618bc61d35b7..9e1e27d31c6d 100644
--- a/arch/x86/entry/common.c
+++ b/arch/x86/entry/common.c
@@ -40,7 +40,7 @@ static struct thread_info *pt_regs_to_thread_info(struct pt_regs *regs)
 
 #ifdef CONFIG_CONTEXT_TRACKING
 /* Called on entry from user mode with IRQs off. */
-__visible void enter_from_user_mode(void)
+__visible inline void enter_from_user_mode(void)
 {
 	CT_WARN_ON(ct_state() != CONTEXT_USER);
 	user_exit_irqoff();
-- 
1.8.3.1

[PATCH 3/2] context_tracking: move rcu_virt_note_context_switch out of kvm_host.h

[Paolo Bonzini]

Make kvm_guest_{enter,exit} and __kvm_guest_{enter,exit} trivial wrappers
around the code in context_tracking.h.  Name the context_tracking.h functions
consistently with what those for kernel<->user switch.

Cc: Andy Lutomirski <luto@kernel.org>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Rik van Riel <riel@redhat.com>
Cc: H. Peter Anvin <hpa@zytor.com>
Cc: Ingo Molnar <mingo@kernel.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>
---
 include/linux/context_tracking.h | 38 ++++++++++++++++++++++++++++++++++----
 include/linux/kvm_host.h         | 25 ++++---------------------
 2 files changed, 38 insertions(+), 25 deletions(-)

diff --git a/include/linux/context_tracking.h b/include/linux/context_tracking.h
index d9aef2a0ec8e..c78fc27418f2 100644
--- a/include/linux/context_tracking.h
+++ b/include/linux/context_tracking.h
@@ -99,7 +99,8 @@ static inline void context_tracking_init(void) { }
 
 
 #ifdef CONFIG_VIRT_CPU_ACCOUNTING_GEN
-static inline void guest_enter(void)
+/* must be called with irqs disabled */
+static inline void guest_enter_irqoff(void)
 {
 	if (vtime_accounting_cpu_enabled())
 		vtime_guest_enter(current);
@@ -108,9 +109,19 @@ static inline void guest_enter(void)
 
 	if (context_tracking_is_enabled())
 		__context_tracking_enter(CONTEXT_GUEST);
+
+	/* KVM does not hold any references to rcu protected data when it
+	 * switches CPU into a guest mode. In fact switching to a guest mode
+	 * is very similar to exiting to userspace from rcu point of view. In
+	 * addition CPU may stay in a guest mode for quite a long time (up to
+	 * one time slice). Lets treat guest mode as quiescent state, just like
+	 * we do with user-mode execution.
+	 */
+	if (!context_tracking_cpu_is_enabled())
+		rcu_virt_note_context_switch(smp_processor_id());
 }
 
-static inline void guest_exit(void)
+static inline void guest_exit_irqoff(void)
 {
 	if (context_tracking_is_enabled())
 		__context_tracking_exit(CONTEXT_GUEST);
@@ -122,7 +133,7 @@ static inline void guest_exit(void)
 }
 
 #else
-static inline void guest_enter(void)
+static inline void guest_enter_irqoff(void)
 {
 	/*
 	 * This is running in ioctl context so its safe
@@ -131,9 +142,10 @@ static inline void guest_enter(void)
 	 */
 	vtime_account_system(current);
 	current->flags |= PF_VCPU;
+	rcu_virt_note_context_switch(smp_processor_id());
 }
 
-static inline void guest_exit(void)
+static inline void guest_exit_irqoff(void)
 {
 	/* Flush the guest cputime we spent on the guest */
 	vtime_account_system(current);
@@ -141,4 +153,22 @@ static inline void guest_exit(void)
 }
 #endif /* CONFIG_VIRT_CPU_ACCOUNTING_GEN */
 
+static inline void guest_enter(void)
+{
+	unsigned long flags;
+
+	local_irq_save(flags);
+	guest_enter_irqoff();
+	local_irq_restore(flags);
+}
+
+static inline void guest_exit(void)
+{
+	unsigned long flags;
+
+	local_irq_save(flags);
+	guest_exit_irqoff();
+	local_irq_restore(flags);
+}
+
 #endif
diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h
index 1c9c973a7dd9..30c9224545b1 100644
--- a/include/linux/kvm_host.h
+++ b/include/linux/kvm_host.h
@@ -870,40 +870,23 @@ static inline void kvm_iommu_unmap_pages(struct kvm *kvm,
 /* must be called with irqs disabled */
 static inline void __kvm_guest_enter(void)
 {
-	guest_enter();
-	/* KVM does not hold any references to rcu protected data when it
-	 * switches CPU into a guest mode. In fact switching to a guest mode
-	 * is very similar to exiting to userspace from rcu point of view. In
-	 * addition CPU may stay in a guest mode for quite a long time (up to
-	 * one time slice). Lets treat guest mode as quiescent state, just like
-	 * we do with user-mode execution.
-	 */
-	if (!context_tracking_cpu_is_enabled())
-		rcu_virt_note_context_switch(smp_processor_id());
+	guest_enter_irqoff();
 }
 
 /* must be called with irqs disabled */
 static inline void __kvm_guest_exit(void)
 {
-	guest_exit();
+	guest_exit_irqoff();
 }
 
 static inline void kvm_guest_enter(void)
 {
-	unsigned long flags;
-
-	local_irq_save(flags);
-	__kvm_guest_enter();
-	local_irq_restore(flags);
+	guest_enter();
 }
 
 static inline void kvm_guest_exit(void)
 {
-	unsigned long flags;
-
-	local_irq_save(flags);
-	__kvm_guest_exit();
-	local_irq_restore(flags);
+	guest_exit();
 }
 
 /*
-- 
1.8.3.1

[PATCH 4/2] KVM: remove kvm_guest_enter/exit wrappers

[Paolo Bonzini]

Use the functions from context_tracking.h directly.

Cc: Andy Lutomirski <luto@kernel.org>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Rik van Riel <riel@redhat.com>
Cc: H. Peter Anvin <hpa@zytor.com>
Cc: Ingo Molnar <mingo@kernel.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>
---
 arch/arm/kvm/arm.c           |  8 ++++----
 arch/mips/kvm/mips.c         |  4 ++--
 arch/powerpc/kvm/book3s_hv.c |  4 ++--
 arch/powerpc/kvm/book3s_pr.c |  4 ++--
 arch/powerpc/kvm/booke.c     |  4 ++--
 arch/powerpc/kvm/powerpc.c   |  2 +-
 arch/s390/kvm/kvm-s390.c     |  4 ++--
 arch/x86/kvm/x86.c           |  4 ++--
 include/linux/kvm_host.h     | 22 ----------------------
 9 files changed, 17 insertions(+), 39 deletions(-)

diff --git a/arch/arm/kvm/arm.c b/arch/arm/kvm/arm.c
index 893941ec98dc..96b473c745a6 100644
--- a/arch/arm/kvm/arm.c
+++ b/arch/arm/kvm/arm.c
@@ -615,7 +615,7 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *run)
 		 * Enter the guest
 		 */
 		trace_kvm_entry(*vcpu_pc(vcpu));
-		__kvm_guest_enter();
+		guest_enter_irqoff();
 		vcpu->mode = IN_GUEST_MODE;
 
 		ret = kvm_call_hyp(__kvm_vcpu_run, vcpu);
@@ -641,14 +641,14 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *run)
 		local_irq_enable();
 
 		/*
-		 * We do local_irq_enable() before calling kvm_guest_exit() so
+		 * We do local_irq_enable() before calling guest_exit() so
 		 * that if a timer interrupt hits while running the guest we
 		 * account that tick as being spent in the guest.  We enable
-		 * preemption after calling kvm_guest_exit() so that if we get
+		 * preemption after calling guest_exit() so that if we get
 		 * preempted we make sure ticks after that is not counted as
 		 * guest time.
 		 */
-		kvm_guest_exit();
+		guest_exit();
 		trace_kvm_exit(ret, kvm_vcpu_trap_get_class(vcpu), *vcpu_pc(vcpu));
 
 		/*
diff --git a/arch/mips/kvm/mips.c b/arch/mips/kvm/mips.c
index dc052fb5c7a2..7b3f438802db 100644
--- a/arch/mips/kvm/mips.c
+++ b/arch/mips/kvm/mips.c
@@ -399,7 +399,7 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *run)
 	kvm_mips_deliver_interrupts(vcpu,
 				    kvm_read_c0_guest_cause(vcpu->arch.cop0));
 
-	__kvm_guest_enter();
+	guest_enter_irqoff();
 
 	/* Disable hardware page table walking while in guest */
 	htw_stop();
@@ -409,7 +409,7 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *run)
 	/* Re-enable HTW before enabling interrupts */
 	htw_start();
 
-	__kvm_guest_exit();
+	guest_exit_irqoff();
 	local_irq_enable();
 
 	if (vcpu->sigset_active)
diff --git a/arch/powerpc/kvm/book3s_hv.c b/arch/powerpc/kvm/book3s_hv.c
index e20beae5ca7a..6b2859c12ae8 100644
--- a/arch/powerpc/kvm/book3s_hv.c
+++ b/arch/powerpc/kvm/book3s_hv.c
@@ -2522,7 +2522,7 @@ static noinline void kvmppc_run_core(struct kvmppc_vcore *vc)
 		list_for_each_entry(pvc, &core_info.vcs[sub], preempt_list)
 			spin_unlock(&pvc->lock);
 
-	kvm_guest_enter();
+	guest_enter();
 
 	srcu_idx = srcu_read_lock(&vc->kvm->srcu);
 
@@ -2570,7 +2570,7 @@ static noinline void kvmppc_run_core(struct kvmppc_vcore *vc)
 
 	/* make sure updates to secondary vcpu structs are visible now */
 	smp_mb();
-	kvm_guest_exit();
+	guest_exit();
 
 	for (sub = 0; sub < core_info.n_subcores; ++sub)
 		list_for_each_entry_safe(pvc, vcnext, &core_info.vcs[sub],
diff --git a/arch/powerpc/kvm/book3s_pr.c b/arch/powerpc/kvm/book3s_pr.c
index 8e4f64f0b774..6a66c5ff0827 100644
--- a/arch/powerpc/kvm/book3s_pr.c
+++ b/arch/powerpc/kvm/book3s_pr.c
@@ -914,7 +914,7 @@ int kvmppc_handle_exit_pr(struct kvm_run *run, struct kvm_vcpu *vcpu,
 	/* We get here with MSR.EE=1 */
 
 	trace_kvm_exit(exit_nr, vcpu);
-	kvm_guest_exit();
+	guest_exit();
 
 	switch (exit_nr) {
 	case BOOK3S_INTERRUPT_INST_STORAGE:
@@ -1531,7 +1531,7 @@ static int kvmppc_vcpu_run_pr(struct kvm_run *kvm_run, struct kvm_vcpu *vcpu)
 
 	kvmppc_clear_debug(vcpu);
 
-	/* No need for kvm_guest_exit. It's done in handle_exit.
+	/* No need for guest_exit. It's done in handle_exit.
 	   We also get here with interrupts enabled. */
 
 	/* Make sure we save the guest FPU/Altivec/VSX state */
diff --git a/arch/powerpc/kvm/booke.c b/arch/powerpc/kvm/booke.c
index 4afae695899a..02b4672f7347 100644
--- a/arch/powerpc/kvm/booke.c
+++ b/arch/powerpc/kvm/booke.c
@@ -776,7 +776,7 @@ int kvmppc_vcpu_run(struct kvm_run *kvm_run, struct kvm_vcpu *vcpu)
 
 	ret = __kvmppc_vcpu_run(kvm_run, vcpu);
 
-	/* No need for kvm_guest_exit. It's done in handle_exit.
+	/* No need for guest_exit. It's done in handle_exit.
 	   We also get here with interrupts enabled. */
 
 	/* Switch back to user space debug context */
@@ -1012,7 +1012,7 @@ int kvmppc_handle_exit(struct kvm_run *run, struct kvm_vcpu *vcpu,
 	}
 
 	trace_kvm_exit(exit_nr, vcpu);
-	__kvm_guest_exit();
+	guest_exit_irqoff();
 
 	local_irq_enable();
 
diff --git a/arch/powerpc/kvm/powerpc.c b/arch/powerpc/kvm/powerpc.c
index 02416fea7653..1ac036e45ed4 100644
--- a/arch/powerpc/kvm/powerpc.c
+++ b/arch/powerpc/kvm/powerpc.c
@@ -119,7 +119,7 @@ int kvmppc_prepare_to_enter(struct kvm_vcpu *vcpu)
 			continue;
 		}
 
-		__kvm_guest_enter();
+		guest_enter_irqoff();
 		return 1;
 	}
 
diff --git a/arch/s390/kvm/kvm-s390.c b/arch/s390/kvm/kvm-s390.c
index 6d8ec3ac9dd8..6d054561f3a6 100644
--- a/arch/s390/kvm/kvm-s390.c
+++ b/arch/s390/kvm/kvm-s390.c
@@ -2361,14 +2361,14 @@ static int __vcpu_run(struct kvm_vcpu *vcpu)
 		 * guest_enter and guest_exit should be no uaccess.
 		 */
 		local_irq_disable();
-		__kvm_guest_enter();
+		guest_enter_irqoff();
 		__disable_cpu_timer_accounting(vcpu);
 		local_irq_enable();
 		exit_reason = sie64a(vcpu->arch.sie_block,
 				     vcpu->run->s.regs.gprs);
 		local_irq_disable();
 		__enable_cpu_timer_accounting(vcpu);
-		__kvm_guest_exit();
+		guest_exit_irqoff();
 		local_irq_enable();
 		vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
 
diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c
index 902d9da12392..12d4f1fb568c 100644
--- a/arch/x86/kvm/x86.c
+++ b/arch/x86/kvm/x86.c
@@ -6636,7 +6636,7 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu)
 
 	trace_kvm_entry(vcpu->vcpu_id);
 	wait_lapic_expire(vcpu);
-	__kvm_guest_enter();
+	guest_enter_irqoff();
 
 	if (unlikely(vcpu->arch.switch_db_regs)) {
 		set_debugreg(0, 7);
@@ -6695,7 +6695,7 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu)
 	 */
 	barrier();
 
-	kvm_guest_exit();
+	guest_exit();
 
 	preempt_enable();
 
diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h
index 30c9224545b1..6fe786901df1 100644
--- a/include/linux/kvm_host.h
+++ b/include/linux/kvm_host.h
@@ -867,28 +867,6 @@ static inline void kvm_iommu_unmap_pages(struct kvm *kvm,
 }
 #endif
 
-/* must be called with irqs disabled */
-static inline void __kvm_guest_enter(void)
-{
-	guest_enter_irqoff();
-}
-
-/* must be called with irqs disabled */
-static inline void __kvm_guest_exit(void)
-{
-	guest_exit_irqoff();
-}
-
-static inline void kvm_guest_enter(void)
-{
-	guest_enter();
-}
-
-static inline void kvm_guest_exit(void)
-{
-	guest_exit();
-}
-
 /*
  * search_memslots() and __gfn_to_memslot() are here because they are
  * used in non-modular code in arch/powerpc/kvm/book3s_hv_rm_mmu.c.
-- 
1.8.3.1

Re: [PATCH 1/2] x86/entry: Avoid interrupt flag save and restore

[Rik van Riel]

[-- Attachment #1: Type: text/plain, Size: 1701 bytes --]

On Mon, 2016-06-20 at 16:58 +0200, Paolo Bonzini wrote:
> Thanks to all the work that was done by Andy Lutomirski and others,
> enter_from_user_mode and prepare_exit_to_usermode are now called only
> with
> interrupts disabled.  Let's provide them a version of
> user_enter/user_exit
> that skips saving and restoring the interrupt flag.
> 
> On an AMD-based machine I tested this patch on, with force-enabled
> context tracking, the speed-up in system calls was 90 clock cycles or
> 6%,
> measured with the following simple benchmark:
> 
>     #include <sys/signal.h>
>     #include <time.h>
>     #include <unistd.h>
>     #include <stdio.h>
> 
>     unsigned long rdtsc()
>     {
>         unsigned long result;
>         asm volatile("rdtsc; shl $32, %%rdx; mov %%eax, %%eax\n"
>                      "or %%rdx, %%rax" : "=a" (result) : : "rdx");
>         return result;
>     }
> 
>     int main()
>     {
>         unsigned long tsc1, tsc2;
>         int pid = getpid();
>         int i;
> 
>         tsc1 = rdtsc();
>         for (i = 0; i < 100000000; i++)
>             kill(pid, SIGWINCH);
>         tsc2 = rdtsc();
> 
>         printf("%ld\n", tsc2 - tsc1);
>     }
> 
> Cc: Andy Lutomirski <luto@kernel.org>
> Cc: Peter Zijlstra <peterz@infradead.org>
> Cc: Rik van Riel <riel@redhat.com>
> Cc: H. Peter Anvin <hpa@zytor.com>
> Cc: Ingo Molnar <mingo@kernel.org>
> Cc: Thomas Gleixner <tglx@linutronix.de>
> Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>
> 
Reviewed-by: Rik van Riel <riel@redhat.com>

-- 
All Rights Reversed.


[-- Attachment #2: This is a digitally signed message part --]
[-- Type: application/pgp-signature, Size: 473 bytes --]

Re: [PATCH 2/2] x86/entry: Inline enter_from_user_mode

[Rik van Riel]

[-- Attachment #1: Type: text/plain, Size: 610 bytes --]

On Mon, 2016-06-20 at 16:58 +0200, Paolo Bonzini wrote:
> This matches what is already done for prepare_exit_to_usermode,
> and saves about 60 clock cycles (4% speedup) with the benchmark
> in the previous commit message.
> 
> Cc: Andy Lutomirski <luto@kernel.org>
> Cc: Peter Zijlstra <peterz@infradead.org>
> Cc: Rik van Riel <riel@redhat.com>
> Cc: H. Peter Anvin <hpa@zytor.com>
> Cc: Ingo Molnar <mingo@kernel.org>
> Cc: Thomas Gleixner <tglx@linutronix.de>
> Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>
> 

Reviewed-by: Rik van Riel <riel@redhat.com>

-- 
All Rights Reversed.


[-- Attachment #2: This is a digitally signed message part --]
[-- Type: application/pgp-signature, Size: 473 bytes --]

Re: [PATCH 3/2] context_tracking: move rcu_virt_note_context_switch out of kvm_host.h

[Rik van Riel]

[-- Attachment #1: Type: text/plain, Size: 764 bytes --]

On Mon, 2016-06-20 at 16:58 +0200, Paolo Bonzini wrote:
> Make kvm_guest_{enter,exit} and __kvm_guest_{enter,exit} trivial
> wrappers
> around the code in context_tracking.h.  Name the context_tracking.h
> functions
> consistently with what those for kernel<->user switch.
> 
> Cc: Andy Lutomirski <luto@kernel.org>
> Cc: Peter Zijlstra <peterz@infradead.org>
> Cc: Rik van Riel <riel@redhat.com>
> Cc: H. Peter Anvin <hpa@zytor.com>
> Cc: Ingo Molnar <mingo@kernel.org>
> Cc: Thomas Gleixner <tglx@linutronix.de>
> Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>
> 

Yay. I remember being confused by this when going
through the context tracking code last year.

Reviewed-by: Rik van Riel <riel@redhat.com>

-- 
All Rights Reversed.


[-- Attachment #2: This is a digitally signed message part --]
[-- Type: application/pgp-signature, Size: 473 bytes --]

Re: [PATCH 4/2] KVM: remove kvm_guest_enter/exit wrappers

[Rik van Riel]

[-- Attachment #1: Type: text/plain, Size: 494 bytes --]

On Mon, 2016-06-20 at 16:58 +0200, Paolo Bonzini wrote:
> Use the functions from context_tracking.h directly.
> 
> Cc: Andy Lutomirski <luto@kernel.org>
> Cc: Peter Zijlstra <peterz@infradead.org>
> Cc: Rik van Riel <riel@redhat.com>
> Cc: H. Peter Anvin <hpa@zytor.com>
> Cc: Ingo Molnar <mingo@kernel.org>
> Cc: Thomas Gleixner <tglx@linutronix.de>
> Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>
> 
Reviewed-by: Rik van Riel <riel@redhat.com>

-- 
All Rights Reversed.


[-- Attachment #2: This is a digitally signed message part --]
[-- Type: application/pgp-signature, Size: 473 bytes --]

Re: [PATCH 1/2] x86/entry: Avoid interrupt flag save and restore

[Andy Lutomirski]

On Mon, Jun 20, 2016 at 7:58 AM, Paolo Bonzini <pbonzini@redhat.com> wrote:
> Thanks to all the work that was done by Andy Lutomirski and others,
> enter_from_user_mode and prepare_exit_to_usermode are now called only with
> interrupts disabled.  Let's provide them a version of user_enter/user_exit
> that skips saving and restoring the interrupt flag.

You're also skipping the in_interrupt() check, but that appears to be fine.

Reviewed-by: Andy Lutomirski <luto@kernel.org>

Re: [PATCH v2 0/2] x86/entry: speed up context-tracking system calls by 150 clock cycles

[Christian Borntraeger]

On 06/20/2016 04:58 PM, Paolo Bonzini wrote:
> The first patches are the two optimizations I posted on May 30th
> for the system call entry/exit code.  The only change is in the
> function names, which use the user_{enter,exit}_irqoff favored
> by Andy and Ingo.  The first patch matches what commit d0e536d8939
> ("context_tracking: avoid irq_save/irq_restore on guest entry and exit",
> 2015-10-28) did for guest entry and exit.  The second simply adds
> an inline annotation; the compiler doesn't figure it out because the
> function is not static.
> 
> The second two patches move guest_{enter,exit} to the same naming
> convention, removing the KVM wrappers kvm_guest_{enter,exit} and
> __kvm_guest_{enter,exit} in the process.  I would like these two to
> go through the KVM tree because I have other optimizations for 4.8
> on top of these patches.
> 
> Thanks,
> 
> Paolo
> 
> Paolo Bonzini (4):
>   x86/entry: Avoid interrupt flag save and restore
>   x86/entry: Inline enter_from_user_mode
>   context_tracking: move rcu_virt_note_context_switch out of kvm_host.h
>   KVM: remove kvm_guest_enter/exit wrappers
> 
>  arch/arm/kvm/arm.c               |  8 +++---
>  arch/mips/kvm/mips.c             |  4 +--
>  arch/powerpc/kvm/book3s_hv.c     |  4 +--
>  arch/powerpc/kvm/book3s_pr.c     |  4 +--
>  arch/powerpc/kvm/booke.c         |  4 +--
>  arch/powerpc/kvm/powerpc.c       |  2 +-
>  arch/s390/kvm/kvm-s390.c         |  4 +--
>  arch/x86/entry/common.c          |  6 ++---
>  arch/x86/kvm/x86.c               |  4 +--
>  include/linux/context_tracking.h | 53 +++++++++++++++++++++++++++++++++++++---
>  include/linux/kvm_host.h         | 39 -----------------------------
>  11 files changed, 69 insertions(+), 63 deletions(-)
> 
Series looks sane and does work on s390.
It has a minor conflict with my vsie pull request (so either add vsie.c
to this patch set or fixup my pull request in the merge commit to replace
kvm_guest_exit/enter with the new functions.

Christian

Re: [PATCH v2 0/2] x86/entry: speed up context-tracking system calls by 150 clock cycles

[Paolo Bonzini]

On 21/06/2016 15:24, Christian Borntraeger wrote:
> Series looks sane and does work on s390.
> It has a minor conflict with my vsie pull request (so either add vsie.c
> to this patch set or fixup my pull request in the merge commit to replace
> kvm_guest_exit/enter with the new functions.

It should not be an issue if I take the last two patches as proposed.

Paolo

Re: [PATCH v2 0/2] x86/entry: speed up context-tracking system calls by 150 clock cycles

[Paolo Bonzini]

On 20/06/2016 16:58, Paolo Bonzini wrote:
> The first patches are the two optimizations I posted on May 30th
> for the system call entry/exit code.  The only change is in the
> function names, which use the user_{enter,exit}_irqoff favored
> by Andy and Ingo.  The first patch matches what commit d0e536d8939
> ("context_tracking: avoid irq_save/irq_restore on guest entry and exit",
> 2015-10-28) did for guest entry and exit.  The second simply adds
> an inline annotation; the compiler doesn't figure it out because the
> function is not static.
> 
> The second two patches move guest_{enter,exit} to the same naming
> convention, removing the KVM wrappers kvm_guest_{enter,exit} and
> __kvm_guest_{enter,exit} in the process.  I would like these two to
> go through the KVM tree because I have other optimizations for 4.8
> on top of these patches.

I'm applying patches 3 and 4 to the KVM tree.

Paolo

Re: [PATCH v2 0/2] x86/entry: speed up context-tracking system calls by 150 clock cycles

[Paolo Bonzini]

On 20/06/2016 16:58, Paolo Bonzini wrote:
> The first patches are the two optimizations I posted on May 30th
> for the system call entry/exit code.  The only change is in the
> function names, which use the user_{enter,exit}_irqoff favored
> by Andy and Ingo.  The first patch matches what commit d0e536d8939
> ("context_tracking: avoid irq_save/irq_restore on guest entry and exit",
> 2015-10-28) did for guest entry and exit.  The second simply adds
> an inline annotation; the compiler doesn't figure it out because the
> function is not static.
> 
> The second two patches move guest_{enter,exit} to the same naming
> convention, removing the KVM wrappers kvm_guest_{enter,exit} and
> __kvm_guest_{enter,exit} in the process.  I would like these two to
> go through the KVM tree because I have other optimizations for 4.8
> on top of these patches.
> 
> Thanks,

Ingo,

ping for these two patches:

http://article.gmane.org/gmane.linux.kernel/2248541/raw
http://article.gmane.org/gmane.comp.emulators.kvm.devel/153909/raw

Thanks,

Paolo

[tip:x86/asm] x86/entry: Avoid interrupt flag save and restore

[tip-bot for Paolo Bonzini]

Commit-ID:  0b95364f977c180e1f336e00273fda5d3eca54b4
Gitweb:     http://git.kernel.org/tip/0b95364f977c180e1f336e00273fda5d3eca54b4
Author:     Paolo Bonzini <pbonzini@redhat.com>
AuthorDate: Mon, 20 Jun 2016 16:58:29 +0200
Committer:  Ingo Molnar <mingo@kernel.org>
CommitDate: Sat, 9 Jul 2016 10:44:01 +0200

x86/entry: Avoid interrupt flag save and restore

Thanks to all the work that was done by Andy Lutomirski and others,
enter_from_user_mode() and prepare_exit_to_usermode() are now called only with
interrupts disabled.  Let's provide them a version of user_enter()/user_exit()
that skips saving and restoring the interrupt flag.

On an AMD-based machine I tested this patch on, with force-enabled
context tracking, the speed-up in system calls was 90 clock cycles or 6%,
measured with the following simple benchmark:

    #include <sys/signal.h>
    #include <time.h>
    #include <unistd.h>
    #include <stdio.h>

    unsigned long rdtsc()
    {
        unsigned long result;
        asm volatile("rdtsc; shl $32, %%rdx; mov %%eax, %%eax\n"
                     "or %%rdx, %%rax" : "=a" (result) : : "rdx");
        return result;
    }

    int main()
    {
        unsigned long tsc1, tsc2;
        int pid = getpid();
        int i;

        tsc1 = rdtsc();
        for (i = 0; i < 100000000; i++)
            kill(pid, SIGWINCH);
        tsc2 = rdtsc();

        printf("%ld\n", tsc2 - tsc1);
    }

Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>
Reviewed-by: Rik van Riel <riel@redhat.com>
Reviewed-by: Andy Lutomirski <luto@kernel.org>
Reviewed-by: Rik van Riel <riel@redhat.com>
Reviewed-by: Andy Lutomirski <luto@kernel.org>
Reviewed-by: Rik van Riel <riel@redhat.com>
Reviewed-by: Andy Lutomirski <luto@kernel.org>
Reviewed-by: Rik van Riel <riel@redhat.com>
Reviewed-by: Andy Lutomirski <luto@kernel.org>
Acked-by: Paolo Bonzini <pbonzini@redhat.com>
Cc: Borislav Petkov <bp@alien8.de>
Cc: Brian Gerst <brgerst@gmail.com>
Cc: Denys Vlasenko <dvlasenk@redhat.com>
Cc: H. Peter Anvin <hpa@zytor.com>
Cc: Josh Poimboeuf <jpoimboe@redhat.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: kvm@vger.kernel.org
Link: http://lkml.kernel.org/r/1466434712-31440-2-git-send-email-pbonzini@redhat.com
Signed-off-by: Ingo Molnar <mingo@kernel.org>
---
 arch/x86/entry/common.c          |  4 ++--
 include/linux/context_tracking.h | 15 +++++++++++++++
 2 files changed, 17 insertions(+), 2 deletions(-)

diff --git a/arch/x86/entry/common.c b/arch/x86/entry/common.c
index ec138e5..618bc61 100644
--- a/arch/x86/entry/common.c
+++ b/arch/x86/entry/common.c
@@ -43,7 +43,7 @@ static struct thread_info *pt_regs_to_thread_info(struct pt_regs *regs)
 __visible void enter_from_user_mode(void)
 {
 	CT_WARN_ON(ct_state() != CONTEXT_USER);
-	user_exit();
+	user_exit_irqoff();
 }
 #else
 static inline void enter_from_user_mode(void) {}
@@ -274,7 +274,7 @@ __visible inline void prepare_exit_to_usermode(struct pt_regs *regs)
 	ti->status &= ~TS_COMPAT;
 #endif
 
-	user_enter();
+	user_enter_irqoff();
 }
 
 #define SYSCALL_EXIT_WORK_FLAGS				\
diff --git a/include/linux/context_tracking.h b/include/linux/context_tracking.h
index d259274..d9aef2a 100644
--- a/include/linux/context_tracking.h
+++ b/include/linux/context_tracking.h
@@ -31,6 +31,19 @@ static inline void user_exit(void)
 		context_tracking_exit(CONTEXT_USER);
 }
 
+/* Called with interrupts disabled.  */
+static inline void user_enter_irqoff(void)
+{
+	if (context_tracking_is_enabled())
+		__context_tracking_enter(CONTEXT_USER);
+
+}
+static inline void user_exit_irqoff(void)
+{
+	if (context_tracking_is_enabled())
+		__context_tracking_exit(CONTEXT_USER);
+}
+
 static inline enum ctx_state exception_enter(void)
 {
 	enum ctx_state prev_ctx;
@@ -69,6 +82,8 @@ static inline enum ctx_state ct_state(void)
 #else
 static inline void user_enter(void) { }
 static inline void user_exit(void) { }
+static inline void user_enter_irqoff(void) { }
+static inline void user_exit_irqoff(void) { }
 static inline enum ctx_state exception_enter(void) { return 0; }
 static inline void exception_exit(enum ctx_state prev_ctx) { }
 static inline enum ctx_state ct_state(void) { return CONTEXT_DISABLED; }

[tip:x86/asm] x86/entry: Inline enter_from_user_mode()

[tip-bot for Paolo Bonzini]

Commit-ID:  eec4b1227db153ca16f8f5f285d01fefdce05438
Gitweb:     http://git.kernel.org/tip/eec4b1227db153ca16f8f5f285d01fefdce05438
Author:     Paolo Bonzini <pbonzini@redhat.com>
AuthorDate: Mon, 20 Jun 2016 16:58:30 +0200
Committer:  Ingo Molnar <mingo@kernel.org>
CommitDate: Sat, 9 Jul 2016 10:44:02 +0200

x86/entry: Inline enter_from_user_mode()

This matches what is already done for prepare_exit_to_usermode(),
and saves about 60 clock cycles (4% speedup) with the benchmark
in the previous commit message.

Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>
Reviewed-by: Rik van Riel <riel@redhat.com>
Reviewed-by: Andy Lutomirski <luto@kernel.org>
Reviewed-by: Rik van Riel <riel@redhat.com>
Reviewed-by: Andy Lutomirski <luto@kernel.org>
Reviewed-by: Rik van Riel <riel@redhat.com>
Reviewed-by: Andy Lutomirski <luto@kernel.org>
Reviewed-by: Rik van Riel <riel@redhat.com>
Reviewed-by: Andy Lutomirski <luto@kernel.org>
Acked-by: Paolo Bonzini <pbonzini@redhat.com>
Cc: Borislav Petkov <bp@alien8.de>
Cc: Brian Gerst <brgerst@gmail.com>
Cc: Denys Vlasenko <dvlasenk@redhat.com>
Cc: H. Peter Anvin <hpa@zytor.com>
Cc: Josh Poimboeuf <jpoimboe@redhat.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: kvm@vger.kernel.org
Link: http://lkml.kernel.org/r/1466434712-31440-3-git-send-email-pbonzini@redhat.com
Signed-off-by: Ingo Molnar <mingo@kernel.org>
---
 arch/x86/entry/common.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/arch/x86/entry/common.c b/arch/x86/entry/common.c
index 618bc61..9e1e27d 100644
--- a/arch/x86/entry/common.c
+++ b/arch/x86/entry/common.c
@@ -40,7 +40,7 @@ static struct thread_info *pt_regs_to_thread_info(struct pt_regs *regs)
 
 #ifdef CONFIG_CONTEXT_TRACKING
 /* Called on entry from user mode with IRQs off. */
-__visible void enter_from_user_mode(void)
+__visible inline void enter_from_user_mode(void)
 {
 	CT_WARN_ON(ct_state() != CONTEXT_USER);
 	user_exit_irqoff();

Re: [tip:x86/asm] x86/entry: Inline enter_from_user_mode()

[Borislav Petkov]

tip-bot for Paolo Bonzini <tipbot@zytor.com> wrote:

>Commit-ID:  eec4b1227db153ca16f8f5f285d01fefdce05438
>Gitweb:    
>http://git.kernel.org/tip/eec4b1227db153ca16f8f5f285d01fefdce05438
>Author:     Paolo Bonzini <pbonzini@redhat.com>
>AuthorDate: Mon, 20 Jun 2016 16:58:30 +0200
>Committer:  Ingo Molnar <mingo@kernel.org>
>CommitDate: Sat, 9 Jul 2016 10:44:02 +0200
>
>x86/entry: Inline enter_from_user_mode()
>
>This matches what is already done for prepare_exit_to_usermode(),
>and saves about 60 clock cycles (4% speedup) with the benchmark
>in the previous commit message.
>
>Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>
>Reviewed-by: Rik van Riel <riel@redhat.com>
>Reviewed-by: Andy Lutomirski <luto@kernel.org>
>Reviewed-by: Rik van Riel <riel@redhat.com>
>Reviewed-by: Andy Lutomirski <luto@kernel.org>
>Reviewed-by: Rik van Riel <riel@redhat.com>
>Reviewed-by: Andy Lutomirski <luto@kernel.org>
>Reviewed-by: Rik van Riel <riel@redhat.com>
>Reviewed-by: Andy Lutomirski <luto@kernel.org>
>Acked-by: Paolo Bonzini <pbonzini@redhat.com>

Woohaa, if that amount of review doesn't get this patch upstream I don't know what will ;-)))))

-- 
Sent from a small device: formatting sucks and brevity is inevitable. 

Re: [tip:x86/asm] x86/entry: Inline enter_from_user_mode()

[Ingo Molnar]

* Borislav Petkov <bp@alien8.de> wrote:

> tip-bot for Paolo Bonzini <tipbot@zytor.com> wrote:
> 
> >Commit-ID:  eec4b1227db153ca16f8f5f285d01fefdce05438
> >Gitweb:    
> >http://git.kernel.org/tip/eec4b1227db153ca16f8f5f285d01fefdce05438
> >Author:     Paolo Bonzini <pbonzini@redhat.com>
> >AuthorDate: Mon, 20 Jun 2016 16:58:30 +0200
> >Committer:  Ingo Molnar <mingo@kernel.org>
> >CommitDate: Sat, 9 Jul 2016 10:44:02 +0200
> >
> >x86/entry: Inline enter_from_user_mode()
> >
> >This matches what is already done for prepare_exit_to_usermode(),
> >and saves about 60 clock cycles (4% speedup) with the benchmark
> >in the previous commit message.
> >
> >Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>
> >Reviewed-by: Rik van Riel <riel@redhat.com>
> >Reviewed-by: Andy Lutomirski <luto@kernel.org>
> >Reviewed-by: Rik van Riel <riel@redhat.com>
> >Reviewed-by: Andy Lutomirski <luto@kernel.org>
> >Reviewed-by: Rik van Riel <riel@redhat.com>
> >Reviewed-by: Andy Lutomirski <luto@kernel.org>
> >Reviewed-by: Rik van Riel <riel@redhat.com>
> >Reviewed-by: Andy Lutomirski <luto@kernel.org>
> >Acked-by: Paolo Bonzini <pbonzini@redhat.com>
> 
> Woohaa, if that amount of review doesn't get this patch upstream I don't know what will ;-)))))

Gah, that's a script gone bad - fixed!

Thanks,

	Ingo

[tip:x86/asm] x86/entry: Avoid interrupt flag save and restore

[tip-bot for Paolo Bonzini]

Commit-ID:  2e9d1e150abf88cb63e5d34ca286edbb95b4c53d
Gitweb:     http://git.kernel.org/tip/2e9d1e150abf88cb63e5d34ca286edbb95b4c53d
Author:     Paolo Bonzini <pbonzini@redhat.com>
AuthorDate: Mon, 20 Jun 2016 16:58:29 +0200
Committer:  Ingo Molnar <mingo@kernel.org>
CommitDate: Sun, 10 Jul 2016 13:33:02 +0200

x86/entry: Avoid interrupt flag save and restore

Thanks to all the work that was done by Andy Lutomirski and others,
enter_from_user_mode() and prepare_exit_to_usermode() are now called only with
interrupts disabled.  Let's provide them a version of user_enter()/user_exit()
that skips saving and restoring the interrupt flag.

On an AMD-based machine I tested this patch on, with force-enabled
context tracking, the speed-up in system calls was 90 clock cycles or 6%,
measured with the following simple benchmark:

    #include <sys/signal.h>
    #include <time.h>
    #include <unistd.h>
    #include <stdio.h>

    unsigned long rdtsc()
    {
        unsigned long result;
        asm volatile("rdtsc; shl $32, %%rdx; mov %%eax, %%eax\n"
                     "or %%rdx, %%rax" : "=a" (result) : : "rdx");
        return result;
    }

    int main()
    {
        unsigned long tsc1, tsc2;
        int pid = getpid();
        int i;

        tsc1 = rdtsc();
        for (i = 0; i < 100000000; i++)
            kill(pid, SIGWINCH);
        tsc2 = rdtsc();

        printf("%ld\n", tsc2 - tsc1);
    }

Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>
Reviewed-by: Rik van Riel <riel@redhat.com>
Reviewed-by: Andy Lutomirski <luto@kernel.org>
Acked-by: Paolo Bonzini <pbonzini@redhat.com>
Cc: Borislav Petkov <bp@alien8.de>
Cc: Brian Gerst <brgerst@gmail.com>
Cc: Denys Vlasenko <dvlasenk@redhat.com>
Cc: H. Peter Anvin <hpa@zytor.com>
Cc: Josh Poimboeuf <jpoimboe@redhat.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: kvm@vger.kernel.org
Link: http://lkml.kernel.org/r/1466434712-31440-2-git-send-email-pbonzini@redhat.com
Signed-off-by: Ingo Molnar <mingo@kernel.org>
---
 arch/x86/entry/common.c          |  4 ++--
 include/linux/context_tracking.h | 15 +++++++++++++++
 2 files changed, 17 insertions(+), 2 deletions(-)

diff --git a/arch/x86/entry/common.c b/arch/x86/entry/common.c
index ec138e5..618bc61 100644
--- a/arch/x86/entry/common.c
+++ b/arch/x86/entry/common.c
@@ -43,7 +43,7 @@ static struct thread_info *pt_regs_to_thread_info(struct pt_regs *regs)
 __visible void enter_from_user_mode(void)
 {
 	CT_WARN_ON(ct_state() != CONTEXT_USER);
-	user_exit();
+	user_exit_irqoff();
 }
 #else
 static inline void enter_from_user_mode(void) {}
@@ -274,7 +274,7 @@ __visible inline void prepare_exit_to_usermode(struct pt_regs *regs)
 	ti->status &= ~TS_COMPAT;
 #endif
 
-	user_enter();
+	user_enter_irqoff();
 }
 
 #define SYSCALL_EXIT_WORK_FLAGS				\
diff --git a/include/linux/context_tracking.h b/include/linux/context_tracking.h
index d259274..d9aef2a 100644
--- a/include/linux/context_tracking.h
+++ b/include/linux/context_tracking.h
@@ -31,6 +31,19 @@ static inline void user_exit(void)
 		context_tracking_exit(CONTEXT_USER);
 }
 
+/* Called with interrupts disabled.  */
+static inline void user_enter_irqoff(void)
+{
+	if (context_tracking_is_enabled())
+		__context_tracking_enter(CONTEXT_USER);
+
+}
+static inline void user_exit_irqoff(void)
+{
+	if (context_tracking_is_enabled())
+		__context_tracking_exit(CONTEXT_USER);
+}
+
 static inline enum ctx_state exception_enter(void)
 {
 	enum ctx_state prev_ctx;
@@ -69,6 +82,8 @@ static inline enum ctx_state ct_state(void)
 #else
 static inline void user_enter(void) { }
 static inline void user_exit(void) { }
+static inline void user_enter_irqoff(void) { }
+static inline void user_exit_irqoff(void) { }
 static inline enum ctx_state exception_enter(void) { return 0; }
 static inline void exception_exit(enum ctx_state prev_ctx) { }
 static inline enum ctx_state ct_state(void) { return CONTEXT_DISABLED; }

[tip:x86/asm] x86/entry: Inline enter_from_user_mode()

[tip-bot for Paolo Bonzini]

Commit-ID:  be8a18e2e98e04a5def5887d913b267865562448
Gitweb:     http://git.kernel.org/tip/be8a18e2e98e04a5def5887d913b267865562448
Author:     Paolo Bonzini <pbonzini@redhat.com>
AuthorDate: Mon, 20 Jun 2016 16:58:30 +0200
Committer:  Ingo Molnar <mingo@kernel.org>
CommitDate: Sun, 10 Jul 2016 13:33:02 +0200

x86/entry: Inline enter_from_user_mode()

This matches what is already done for prepare_exit_to_usermode(),
and saves about 60 clock cycles (4% speedup) with the benchmark
in the previous commit message.

Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>
Reviewed-by: Rik van Riel <riel@redhat.com>
Reviewed-by: Andy Lutomirski <luto@kernel.org>
Acked-by: Paolo Bonzini <pbonzini@redhat.com>
Cc: Borislav Petkov <bp@alien8.de>
Cc: Brian Gerst <brgerst@gmail.com>
Cc: Denys Vlasenko <dvlasenk@redhat.com>
Cc: H. Peter Anvin <hpa@zytor.com>
Cc: Josh Poimboeuf <jpoimboe@redhat.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: kvm@vger.kernel.org
Link: http://lkml.kernel.org/r/1466434712-31440-3-git-send-email-pbonzini@redhat.com
Signed-off-by: Ingo Molnar <mingo@kernel.org>
---
 arch/x86/entry/common.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/arch/x86/entry/common.c b/arch/x86/entry/common.c
index 618bc61..9e1e27d 100644
--- a/arch/x86/entry/common.c
+++ b/arch/x86/entry/common.c
@@ -40,7 +40,7 @@ static struct thread_info *pt_regs_to_thread_info(struct pt_regs *regs)
 
 #ifdef CONFIG_CONTEXT_TRACKING
 /* Called on entry from user mode with IRQs off. */
-__visible void enter_from_user_mode(void)
+__visible inline void enter_from_user_mode(void)
 {
 	CT_WARN_ON(ct_state() != CONTEXT_USER);
 	user_exit_irqoff();