All of lore.kernel.org
 help / color / mirror / Atom feed
From: tip-bot for Peter Zijlstra <a.p.zijlstra@chello.nl>
To: linux-tip-commits@vger.kernel.org
Cc: linux-kernel@vger.kernel.org, hpa@zytor.com, mingo@redhat.com,
	torvalds@linux-foundation.org, a.p.zijlstra@chello.nl,
	efault@gmx.de, npiggin@kernel.dk, akpm@linux-foundation.org,
	tglx@linutronix.de, frank.rowand@am.sony.com, mingo@elte.hu
Subject: [tip:sched/locking] sched: Always provide p->on_cpu
Date: Thu, 14 Apr 2011 08:31:38 GMT	[thread overview]
Message-ID: <tip-3ca7a440da394808571dad32d33d3bc0389982e6@git.kernel.org> (raw)
In-Reply-To: <20110405152728.785452014@chello.nl>

Commit-ID:  3ca7a440da394808571dad32d33d3bc0389982e6
Gitweb:     http://git.kernel.org/tip/3ca7a440da394808571dad32d33d3bc0389982e6
Author:     Peter Zijlstra <a.p.zijlstra@chello.nl>
AuthorDate: Tue, 5 Apr 2011 17:23:40 +0200
Committer:  Ingo Molnar <mingo@elte.hu>
CommitDate: Thu, 14 Apr 2011 08:52:32 +0200

sched: Always provide p->on_cpu

Always provide p->on_cpu so that we can determine if its on a cpu
without having to lock the rq.

Reviewed-by: Frank Rowand <frank.rowand@am.sony.com>
Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Mike Galbraith <efault@gmx.de>
Cc: Nick Piggin <npiggin@kernel.dk>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Andrew Morton <akpm@linux-foundation.org>
Link: http://lkml.kernel.org/r/20110405152728.785452014@chello.nl
Signed-off-by: Ingo Molnar <mingo@elte.hu>
---
 include/linux/sched.h |    4 +---
 kernel/sched.c        |   46 +++++++++++++++++++++++++++++-----------------
 2 files changed, 30 insertions(+), 20 deletions(-)

diff --git a/include/linux/sched.h b/include/linux/sched.h
index 758e27a..3435837 100644
--- a/include/linux/sched.h
+++ b/include/linux/sched.h
@@ -1200,9 +1200,7 @@ struct task_struct {
 	int lock_depth;		/* BKL lock depth */
 
 #ifdef CONFIG_SMP
-#ifdef __ARCH_WANT_UNLOCKED_CTXSW
-	int oncpu;
-#endif
+	int on_cpu;
 #endif
 
 	int prio, static_prio, normal_prio;
diff --git a/kernel/sched.c b/kernel/sched.c
index a187c3f..cd2593e 100644
--- a/kernel/sched.c
+++ b/kernel/sched.c
@@ -838,18 +838,39 @@ static inline int task_current(struct rq *rq, struct task_struct *p)
 	return rq->curr == p;
 }
 
-#ifndef __ARCH_WANT_UNLOCKED_CTXSW
 static inline int task_running(struct rq *rq, struct task_struct *p)
 {
+#ifdef CONFIG_SMP
+	return p->on_cpu;
+#else
 	return task_current(rq, p);
+#endif
 }
 
+#ifndef __ARCH_WANT_UNLOCKED_CTXSW
 static inline void prepare_lock_switch(struct rq *rq, struct task_struct *next)
 {
+#ifdef CONFIG_SMP
+	/*
+	 * We can optimise this out completely for !SMP, because the
+	 * SMP rebalancing from interrupt is the only thing that cares
+	 * here.
+	 */
+	next->on_cpu = 1;
+#endif
 }
 
 static inline void finish_lock_switch(struct rq *rq, struct task_struct *prev)
 {
+#ifdef CONFIG_SMP
+	/*
+	 * After ->on_cpu is cleared, the task can be moved to a different CPU.
+	 * We must ensure this doesn't happen until the switch is completely
+	 * finished.
+	 */
+	smp_wmb();
+	prev->on_cpu = 0;
+#endif
 #ifdef CONFIG_DEBUG_SPINLOCK
 	/* this is a valid case when another task releases the spinlock */
 	rq->lock.owner = current;
@@ -865,15 +886,6 @@ static inline void finish_lock_switch(struct rq *rq, struct task_struct *prev)
 }
 
 #else /* __ARCH_WANT_UNLOCKED_CTXSW */
-static inline int task_running(struct rq *rq, struct task_struct *p)
-{
-#ifdef CONFIG_SMP
-	return p->oncpu;
-#else
-	return task_current(rq, p);
-#endif
-}
-
 static inline void prepare_lock_switch(struct rq *rq, struct task_struct *next)
 {
 #ifdef CONFIG_SMP
@@ -882,7 +894,7 @@ static inline void prepare_lock_switch(struct rq *rq, struct task_struct *next)
 	 * SMP rebalancing from interrupt is the only thing that cares
 	 * here.
 	 */
-	next->oncpu = 1;
+	next->on_cpu = 1;
 #endif
 #ifdef __ARCH_WANT_INTERRUPTS_ON_CTXSW
 	raw_spin_unlock_irq(&rq->lock);
@@ -895,12 +907,12 @@ static inline void finish_lock_switch(struct rq *rq, struct task_struct *prev)
 {
 #ifdef CONFIG_SMP
 	/*
-	 * After ->oncpu is cleared, the task can be moved to a different CPU.
+	 * After ->on_cpu is cleared, the task can be moved to a different CPU.
 	 * We must ensure this doesn't happen until the switch is completely
 	 * finished.
 	 */
 	smp_wmb();
-	prev->oncpu = 0;
+	prev->on_cpu = 0;
 #endif
 #ifndef __ARCH_WANT_INTERRUPTS_ON_CTXSW
 	local_irq_enable();
@@ -2686,8 +2698,8 @@ void sched_fork(struct task_struct *p, int clone_flags)
 	if (likely(sched_info_on()))
 		memset(&p->sched_info, 0, sizeof(p->sched_info));
 #endif
-#if defined(CONFIG_SMP) && defined(__ARCH_WANT_UNLOCKED_CTXSW)
-	p->oncpu = 0;
+#if defined(CONFIG_SMP)
+	p->on_cpu = 0;
 #endif
 #ifdef CONFIG_PREEMPT
 	/* Want to start with kernel preemption disabled. */
@@ -5776,8 +5788,8 @@ void __cpuinit init_idle(struct task_struct *idle, int cpu)
 	rcu_read_unlock();
 
 	rq->curr = rq->idle = idle;
-#if defined(CONFIG_SMP) && defined(__ARCH_WANT_UNLOCKED_CTXSW)
-	idle->oncpu = 1;
+#if defined(CONFIG_SMP)
+	idle->on_cpu = 1;
 #endif
 	raw_spin_unlock_irqrestore(&rq->lock, flags);
 

  reply	other threads:[~2011-04-14  8:33 UTC|newest]

Thread overview: 152+ messages / expand[flat|nested]  mbox.gz  Atom feed  top
2011-04-05 15:23 [PATCH 00/21] sched: Reduce runqueue lock contention -v6 Peter Zijlstra
2011-04-05 15:23 ` [PATCH 01/21] sched: Provide scheduler_ipi() callback in response to smp_send_reschedule() Peter Zijlstra
2011-04-13 21:15   ` Tony Luck
2011-04-13 21:38     ` Peter Zijlstra
2011-04-14  8:31   ` [tip:sched/locking] " tip-bot for Peter Zijlstra
2011-04-05 15:23 ` [PATCH 02/21] sched: Always provide p->on_cpu Peter Zijlstra
2011-04-14  8:31   ` tip-bot for Peter Zijlstra [this message]
2011-04-05 15:23 ` [PATCH 03/21] mutex: Use p->on_cpu for the adaptive spin Peter Zijlstra
2011-04-14  8:32   ` [tip:sched/locking] " tip-bot for Peter Zijlstra
2011-04-05 15:23 ` [PATCH 04/21] sched: Change the ttwu success details Peter Zijlstra
2011-04-13  9:23   ` Peter Zijlstra
2011-04-13 10:48     ` Peter Zijlstra
2011-04-13 11:06       ` Peter Zijlstra
2011-04-13 18:39         ` Tejun Heo
2011-04-13 19:11           ` Peter Zijlstra
2011-04-14  8:32   ` [tip:sched/locking] sched: Change the ttwu() " tip-bot for Peter Zijlstra
2011-04-05 15:23 ` [PATCH 05/21] sched: Clean up ttwu stats Peter Zijlstra
2011-04-14  8:33   ` [tip:sched/locking] sched: Clean up ttwu() stats tip-bot for Peter Zijlstra
2011-04-05 15:23 ` [PATCH 06/21] sched: Provide p->on_rq Peter Zijlstra
2011-04-14  8:33   ` [tip:sched/locking] " tip-bot for Peter Zijlstra
2011-04-05 15:23 ` [PATCH 07/21] sched: Serialize p->cpus_allowed and ttwu() using p->pi_lock Peter Zijlstra
2011-04-14  8:34   ` [tip:sched/locking] " tip-bot for Peter Zijlstra
2011-04-05 15:23 ` [PATCH 08/21] sched: Drop the rq argument to sched_class::select_task_rq() Peter Zijlstra
2011-04-14  8:34   ` [tip:sched/locking] " tip-bot for Peter Zijlstra
2011-04-05 15:23 ` [PATCH 09/21] sched: Remove rq argument to sched_class::task_waking() Peter Zijlstra
2011-04-14  8:35   ` [tip:sched/locking] " tip-bot for Peter Zijlstra
2011-04-05 15:23 ` [PATCH 10/21] sched: Deal with non-atomic min_vruntime reads on 32bits Peter Zijlstra
2011-04-14  8:35   ` [tip:sched/locking] " tip-bot for Peter Zijlstra
2011-04-05 15:23 ` [PATCH 11/21] sched: Delay task_contributes_to_load() Peter Zijlstra
2011-04-14  8:35   ` [tip:sched/locking] " tip-bot for Peter Zijlstra
2011-04-05 15:23 ` [PATCH 12/21] sched: Also serialize ttwu_local() with p->pi_lock Peter Zijlstra
2011-04-14  8:36   ` [tip:sched/locking] " tip-bot for Peter Zijlstra
2011-04-05 15:23 ` [PATCH 13/21] sched: Add p->pi_lock to task_rq_lock() Peter Zijlstra
2011-04-14  8:36   ` [tip:sched/locking] " tip-bot for Peter Zijlstra
2011-06-01 13:58     ` Arne Jansen
2011-06-01 16:35       ` Peter Zijlstra
2011-06-01 17:20         ` Arne Jansen
2011-06-01 18:09           ` Peter Zijlstra
2011-06-01 18:44             ` Peter Zijlstra
2011-06-01 19:30               ` Arne Jansen
2011-06-01 21:09                 ` Linus Torvalds
2011-06-03  9:15                   ` Peter Zijlstra
2011-06-03 10:02                     ` Arne Jansen
2011-06-03 10:30                       ` Peter Zijlstra
2011-06-03 11:52                         ` Arne Jansen
2011-06-05  8:17                         ` Ingo Molnar
2011-06-05  8:53                           ` Arne Jansen
2011-06-05  9:41                             ` Ingo Molnar
2011-06-05  9:45                               ` Ingo Molnar
2011-06-05  9:43                           ` Arne Jansen
2011-06-05  9:55                             ` Ingo Molnar
2011-06-05 10:22                               ` Arne Jansen
2011-06-05 11:01                                 ` Ingo Molnar
2011-06-05 11:19                                   ` [debug patch] printk: Add a printk killswitch to robustify NMI watchdog messages Ingo Molnar
2011-06-05 11:36                                     ` Ingo Molnar
2011-06-05 11:57                                       ` Arne Jansen
2011-06-05 13:39                                         ` Ingo Molnar
2011-06-05 13:54                                           ` Arne Jansen
2011-06-05 14:06                                             ` Ingo Molnar
2011-06-05 14:45                                               ` Arne Jansen
2011-06-05 14:10                                             ` Ingo Molnar
2011-06-05 14:31                                               ` Arne Jansen
2011-06-05 15:13                                                 ` Ingo Molnar
2011-06-05 15:26                                                   ` Ingo Molnar
2011-06-05 15:32                                                     ` Ingo Molnar
2011-06-05 16:07                                                       ` Arne Jansen
2011-06-05 16:35                                                         ` Arne Jansen
2011-06-05 16:50                                                           ` Arne Jansen
2011-06-05 17:20                                                             ` Ingo Molnar
2011-06-05 17:42                                                               ` Arne Jansen
2011-06-05 18:59                                                                 ` Ingo Molnar
2011-06-05 19:30                                                                   ` Arne Jansen
2011-06-05 19:44                                                                     ` Ingo Molnar
2011-06-05 20:15                                                                       ` Arne Jansen
2011-06-06  6:56                                                                         ` Arne Jansen
2011-06-06  9:01                                                                         ` Peter Zijlstra
2011-06-06  9:18                                                                           ` Arne Jansen
2011-06-06  9:24                                                                             ` Peter Zijlstra
2011-06-06  9:52                                                                               ` Peter Zijlstra
2011-06-06 10:00                                                                           ` Arne Jansen
2011-06-06 10:26                                                                             ` Peter Zijlstra
2011-06-06 13:25                                                                               ` Peter Zijlstra
2011-06-06 15:04                                                                           ` Ingo Molnar
2011-06-06 15:08                                                                             ` Ingo Molnar
2011-06-06 17:44                                                                               ` Mike Galbraith
2011-06-07  5:20                                                                           ` Mike Galbraith
2011-06-06 13:10                                                                   ` Ingo Molnar
2011-06-06 13:12                                                                     ` Peter Zijlstra
2011-06-06 13:21                                                                       ` Ingo Molnar
2011-06-06 13:31                                                                         ` Peter Zijlstra
2011-06-06  7:34                                                     ` Arne Jansen
2011-06-05 15:34                                                   ` Arne Jansen
2011-06-06  8:38                                                   ` Peter Zijlstra
2011-06-06 14:58                                                     ` Ingo Molnar
2011-06-06 15:09                                                       ` Peter Zijlstra
2011-06-06 15:47                                                         ` Peter Zijlstra
2011-06-06 15:52                                                           ` Ingo Molnar
2011-06-06 16:00                                                             ` Peter Zijlstra
2011-06-06 16:08                                                               ` Ingo Molnar
2011-06-06 16:12                                                                 ` Peter Zijlstra
2011-06-06 16:17                                                                   ` Ingo Molnar
2011-06-06 16:38                                                                     ` Arne Jansen
2011-06-06 16:45                                                                       ` Arne Jansen
2011-06-06 16:53                                                                         ` Peter Zijlstra
2011-06-06 17:07                                                                         ` Ingo Molnar
2011-06-06 17:11                                                                           ` Peter Zijlstra
2011-06-08 15:50                                                                             ` Peter Zijlstra
2011-06-08 19:17                                                                               ` Ingo Molnar
2011-06-08 19:27                                                                                 ` Linus Torvalds
2011-06-08 20:32                                                                                   ` Peter Zijlstra
2011-06-08 20:53                                                                                     ` Linus Torvalds
2011-06-08 20:54                                                                                     ` Thomas Gleixner
2011-06-08 19:45                                                                                 ` Peter Zijlstra
2011-06-08 20:52                                                                                   ` Ingo Molnar
2011-06-08 21:49                                                                                     ` Peter Zijlstra
2011-06-08 21:57                                                                                       ` Thomas Gleixner
2011-06-06 16:44                                                                     ` Peter Zijlstra
2011-06-06 16:50                                                                       ` Peter Zijlstra
2011-06-06 17:13                                                                         ` Ingo Molnar
2011-06-06 17:04                                                                       ` Peter Zijlstra
2011-06-06 17:11                                                                       ` Ingo Molnar
2011-06-06 17:57                                                                         ` Arne Jansen
2011-06-06 18:07                                                                           ` Ingo Molnar
2011-06-06 18:14                                                                             ` Arne Jansen
2011-06-06 18:19                                                                             ` Peter Zijlstra
2011-06-06 22:08                                                                               ` Ingo Molnar
2011-06-03 12:44                       ` [tip:sched/locking] sched: Add p->pi_lock to task_rq_lock() Linus Torvalds
2011-06-03 13:05                         ` Arne Jansen
2011-06-04 21:29                           ` Linus Torvalds
2011-06-04 22:08                             ` Peter Zijlstra
2011-06-04 22:50                               ` Linus Torvalds
2011-06-05  6:01                               ` Arne Jansen
2011-06-05  7:57                                 ` Mike Galbraith
2011-04-05 15:23 ` [PATCH 14/21] sched: Drop rq->lock from first part of wake_up_new_task() Peter Zijlstra
2011-04-14  8:37   ` [tip:sched/locking] " tip-bot for Peter Zijlstra
2011-04-05 15:23 ` [PATCH 15/21] sched: Drop rq->lock from sched_exec() Peter Zijlstra
2011-04-14  8:37   ` [tip:sched/locking] " tip-bot for Peter Zijlstra
2011-04-05 15:23 ` [PATCH 16/21] sched: Remove rq->lock from the first half of ttwu() Peter Zijlstra
2011-04-14  8:38   ` [tip:sched/locking] " tip-bot for Peter Zijlstra
2011-04-05 15:23 ` [PATCH 17/21] sched: Remove rq argument from ttwu_stat() Peter Zijlstra
2011-04-14  8:38   ` [tip:sched/locking] " tip-bot for Peter Zijlstra
2011-04-05 15:23 ` [PATCH 18/21] sched: Rename ttwu_post_activation Peter Zijlstra
2011-04-14  8:39   ` [tip:sched/locking] sched: Rename ttwu_post_activation() to ttwu_do_wakeup() tip-bot for Peter Zijlstra
2011-04-05 15:23 ` [PATCH 19/21] sched: Restructure ttwu some more Peter Zijlstra
2011-04-14  8:39   ` [tip:sched/locking] sched: Restructure ttwu() " tip-bot for Peter Zijlstra
2011-04-05 15:23 ` [PATCH 20/21] sched: Move the second half of ttwu() to the remote cpu Peter Zijlstra
2011-04-14  8:39   ` [tip:sched/locking] " tip-bot for Peter Zijlstra
2011-04-05 15:23 ` [PATCH 21/21] sched: Remove need_migrate_task() Peter Zijlstra
2011-04-14  8:40   ` [tip:sched/locking] " tip-bot for Peter Zijlstra
2011-04-05 15:59 ` [PATCH 00/21] sched: Reduce runqueue lock contention -v6 Peter Zijlstra
2011-04-06 11:00 ` Peter Zijlstra
2011-04-27 16:54 ` Dave Kleikamp

Reply instructions:

You may reply publicly to this message via plain-text email
using any one of the following methods:

* Save the following mbox file, import it into your mail client,
  and reply-to-all from there: mbox

  Avoid top-posting and favor interleaved quoting:
  https://en.wikipedia.org/wiki/Posting_style#Interleaved_style

* Reply using the --to, --cc, and --in-reply-to
  switches of git-send-email(1):

  git send-email \
    --in-reply-to=tip-3ca7a440da394808571dad32d33d3bc0389982e6@git.kernel.org \
    --to=a.p.zijlstra@chello.nl \
    --cc=akpm@linux-foundation.org \
    --cc=efault@gmx.de \
    --cc=frank.rowand@am.sony.com \
    --cc=hpa@zytor.com \
    --cc=linux-kernel@vger.kernel.org \
    --cc=linux-tip-commits@vger.kernel.org \
    --cc=mingo@elte.hu \
    --cc=mingo@redhat.com \
    --cc=npiggin@kernel.dk \
    --cc=tglx@linutronix.de \
    --cc=torvalds@linux-foundation.org \
    /path/to/YOUR_REPLY

  https://kernel.org/pub/software/scm/git/docs/git-send-email.html

* If your mail client supports setting the In-Reply-To header
  via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line before the message body.
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.