From: Peter Zijlstra <a.p.zijlstra@chello.nl> To: Chris Mason <chris.mason@oracle.com>, Frank Rowand <frank.rowand@am.sony.com>, Ingo Molnar <mingo@elte.hu>, Thomas Gleixner <tglx@linutronix.de>, Mike Galbraith <efault@gmx.de>, Oleg Nesterov <oleg@redhat.com>, Paul Turner <pjt@google.com>, Jens Axboe <axboe@kernel.dk>, Yong Zhang <yong.zhang0@gmail.com> Cc: linux-kernel@vger.kernel.org, Peter Zijlstra <a.p.zijlstra@chello.nl> Subject: [PATCH 03/21] mutex: Use p->on_cpu for the adaptive spin Date: Tue, 05 Apr 2011 17:23:41 +0200 Message-ID: <20110405152728.826338173@chello.nl> (raw) In-Reply-To: <20110405152338.692966333@chello.nl> [-- Attachment #0: sched-on_cpu-use.patch --] [-- Type: text/plain, Size: 5922 bytes --] Since we now have p->on_cpu unconditionally available, use it to re-implement mutex_spin_on_owner. Requested-by: Thomas Gleixner <tglx@linutronix.de> Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl> Reviewed-by: Frank Rowand <frank.rowand@am.sony.com> --- include/linux/mutex.h | 2 - include/linux/sched.h | 2 - kernel/mutex-debug.c | 2 - kernel/mutex-debug.h | 2 - kernel/mutex.c | 2 - kernel/mutex.h | 2 - kernel/sched.c | 83 +++++++++++++++++++------------------------------- 7 files changed, 39 insertions(+), 56 deletions(-) Index: linux-2.6/include/linux/mutex.h =================================================================== --- linux-2.6.orig/include/linux/mutex.h +++ linux-2.6/include/linux/mutex.h @@ -51,7 +51,7 @@ struct mutex { spinlock_t wait_lock; struct list_head wait_list; #if defined(CONFIG_DEBUG_MUTEXES) || defined(CONFIG_SMP) - struct thread_info *owner; + struct task_struct *owner; #endif #ifdef CONFIG_DEBUG_MUTEXES const char *name; Index: linux-2.6/kernel/mutex-debug.c =================================================================== --- linux-2.6.orig/kernel/mutex-debug.c +++ linux-2.6/kernel/mutex-debug.c @@ -75,7 +75,7 @@ void debug_mutex_unlock(struct mutex *lo return; DEBUG_LOCKS_WARN_ON(lock->magic != lock); - DEBUG_LOCKS_WARN_ON(lock->owner != current_thread_info()); + DEBUG_LOCKS_WARN_ON(lock->owner != current); DEBUG_LOCKS_WARN_ON(!lock->wait_list.prev && !lock->wait_list.next); mutex_clear_owner(lock); } Index: linux-2.6/kernel/mutex-debug.h =================================================================== --- linux-2.6.orig/kernel/mutex-debug.h +++ linux-2.6/kernel/mutex-debug.h @@ -29,7 +29,7 @@ extern void debug_mutex_init(struct mute static inline void mutex_set_owner(struct mutex *lock) { - lock->owner = current_thread_info(); + lock->owner = current; } static inline void mutex_clear_owner(struct mutex *lock) Index: linux-2.6/kernel/mutex.c =================================================================== --- linux-2.6.orig/kernel/mutex.c +++ linux-2.6/kernel/mutex.c @@ -160,7 +160,7 @@ __mutex_lock_common(struct mutex *lock, */ for (;;) { - struct thread_info *owner; + struct task_struct *owner; /* * If we own the BKL, then don't spin. The owner of Index: linux-2.6/kernel/mutex.h =================================================================== --- linux-2.6.orig/kernel/mutex.h +++ linux-2.6/kernel/mutex.h @@ -19,7 +19,7 @@ #ifdef CONFIG_SMP static inline void mutex_set_owner(struct mutex *lock) { - lock->owner = current_thread_info(); + lock->owner = current; } static inline void mutex_clear_owner(struct mutex *lock) Index: linux-2.6/kernel/sched.c =================================================================== --- linux-2.6.orig/kernel/sched.c +++ linux-2.6/kernel/sched.c @@ -4034,70 +4034,53 @@ asmlinkage void __sched schedule(void) EXPORT_SYMBOL(schedule); #ifdef CONFIG_MUTEX_SPIN_ON_OWNER -/* - * Look out! "owner" is an entirely speculative pointer - * access and not reliable. - */ -int mutex_spin_on_owner(struct mutex *lock, struct thread_info *owner) -{ - unsigned int cpu; - struct rq *rq; - if (!sched_feat(OWNER_SPIN)) - return 0; +static inline bool owner_running(struct mutex *lock, struct task_struct *owner) +{ + bool ret = false; -#ifdef CONFIG_DEBUG_PAGEALLOC - /* - * Need to access the cpu field knowing that - * DEBUG_PAGEALLOC could have unmapped it if - * the mutex owner just released it and exited. - */ - if (probe_kernel_address(&owner->cpu, cpu)) - return 0; -#else - cpu = owner->cpu; -#endif + rcu_read_lock(); + if (lock->owner != owner) + goto fail; /* - * Even if the access succeeded (likely case), - * the cpu field may no longer be valid. + * Ensure we emit the owner->on_cpu, dereference _after_ checking + * lock->owner still matches owner, if that fails, owner might + * point to free()d memory, if it still matches, the rcu_read_lock() + * ensures the memory stays valid. */ - if (cpu >= nr_cpumask_bits) - return 0; + barrier(); - /* - * We need to validate that we can do a - * get_cpu() and that we have the percpu area. - */ - if (!cpu_online(cpu)) - return 0; + ret = owner->on_cpu; +fail: + rcu_read_unlock(); - rq = cpu_rq(cpu); + return ret; +} - for (;;) { - /* - * Owner changed, break to re-assess state. - */ - if (lock->owner != owner) { - /* - * If the lock has switched to a different owner, - * we likely have heavy contention. Return 0 to quit - * optimistic spinning and not contend further: - */ - if (lock->owner) - return 0; - break; - } +/* + * Look out! "owner" is an entirely speculative pointer + * access and not reliable. + */ +int mutex_spin_on_owner(struct mutex *lock, struct task_struct *owner) +{ + if (!sched_feat(OWNER_SPIN)) + return 0; - /* - * Is that owner really running on that cpu? - */ - if (task_thread_info(rq->curr) != owner || need_resched()) + while (owner_running(lock, owner)) { + if (need_resched()) return 0; arch_mutex_cpu_relax(); } + /* + * If the owner changed to another task there is likely + * heavy contention, stop spinning. + */ + if (lock->owner) + return 0; + return 1; } #endif Index: linux-2.6/include/linux/sched.h =================================================================== --- linux-2.6.orig/include/linux/sched.h +++ linux-2.6/include/linux/sched.h @@ -360,7 +360,7 @@ extern signed long schedule_timeout_inte extern signed long schedule_timeout_killable(signed long timeout); extern signed long schedule_timeout_uninterruptible(signed long timeout); asmlinkage void schedule(void); -extern int mutex_spin_on_owner(struct mutex *lock, struct thread_info *owner); +extern int mutex_spin_on_owner(struct mutex *lock, struct task_struct *owner); struct nsproxy; struct user_namespace;
next prev parent reply index Thread overview: 152+ messages / expand[flat|nested] mbox.gz Atom feed top 2011-04-05 15:23 [PATCH 00/21] sched: Reduce runqueue lock contention -v6 Peter Zijlstra 2011-04-05 15:23 ` [PATCH 01/21] sched: Provide scheduler_ipi() callback in response to smp_send_reschedule() Peter Zijlstra 2011-04-13 21:15 ` Tony Luck 2011-04-13 21:38 ` Peter Zijlstra 2011-04-14 8:31 ` [tip:sched/locking] " tip-bot for Peter Zijlstra 2011-04-05 15:23 ` [PATCH 02/21] sched: Always provide p->on_cpu Peter Zijlstra 2011-04-14 8:31 ` [tip:sched/locking] " tip-bot for Peter Zijlstra 2011-04-05 15:23 ` Peter Zijlstra [this message] 2011-04-14 8:32 ` [tip:sched/locking] mutex: Use p->on_cpu for the adaptive spin tip-bot for Peter Zijlstra 2011-04-05 15:23 ` [PATCH 04/21] sched: Change the ttwu success details Peter Zijlstra 2011-04-13 9:23 ` Peter Zijlstra 2011-04-13 10:48 ` Peter Zijlstra 2011-04-13 11:06 ` Peter Zijlstra 2011-04-13 18:39 ` Tejun Heo 2011-04-13 19:11 ` Peter Zijlstra 2011-04-14 8:32 ` [tip:sched/locking] sched: Change the ttwu() " tip-bot for Peter Zijlstra 2011-04-05 15:23 ` [PATCH 05/21] sched: Clean up ttwu stats Peter Zijlstra 2011-04-14 8:33 ` [tip:sched/locking] sched: Clean up ttwu() stats tip-bot for Peter Zijlstra 2011-04-05 15:23 ` [PATCH 06/21] sched: Provide p->on_rq Peter Zijlstra 2011-04-14 8:33 ` [tip:sched/locking] " tip-bot for Peter Zijlstra 2011-04-05 15:23 ` [PATCH 07/21] sched: Serialize p->cpus_allowed and ttwu() using p->pi_lock Peter Zijlstra 2011-04-14 8:34 ` [tip:sched/locking] " tip-bot for Peter Zijlstra 2011-04-05 15:23 ` [PATCH 08/21] sched: Drop the rq argument to sched_class::select_task_rq() Peter Zijlstra 2011-04-14 8:34 ` [tip:sched/locking] " tip-bot for Peter Zijlstra 2011-04-05 15:23 ` [PATCH 09/21] sched: Remove rq argument to sched_class::task_waking() Peter Zijlstra 2011-04-14 8:35 ` [tip:sched/locking] " tip-bot for Peter Zijlstra 2011-04-05 15:23 ` [PATCH 10/21] sched: Deal with non-atomic min_vruntime reads on 32bits Peter Zijlstra 2011-04-14 8:35 ` [tip:sched/locking] " tip-bot for Peter Zijlstra 2011-04-05 15:23 ` [PATCH 11/21] sched: Delay task_contributes_to_load() Peter Zijlstra 2011-04-14 8:35 ` [tip:sched/locking] " tip-bot for Peter Zijlstra 2011-04-05 15:23 ` [PATCH 12/21] sched: Also serialize ttwu_local() with p->pi_lock Peter Zijlstra 2011-04-14 8:36 ` [tip:sched/locking] " tip-bot for Peter Zijlstra 2011-04-05 15:23 ` [PATCH 13/21] sched: Add p->pi_lock to task_rq_lock() Peter Zijlstra 2011-04-14 8:36 ` [tip:sched/locking] " tip-bot for Peter Zijlstra 2011-06-01 13:58 ` Arne Jansen 2011-06-01 16:35 ` Peter Zijlstra 2011-06-01 17:20 ` Arne Jansen 2011-06-01 18:09 ` Peter Zijlstra 2011-06-01 18:44 ` Peter Zijlstra 2011-06-01 19:30 ` Arne Jansen 2011-06-01 21:09 ` Linus Torvalds 2011-06-03 9:15 ` Peter Zijlstra 2011-06-03 10:02 ` Arne Jansen 2011-06-03 10:30 ` Peter Zijlstra 2011-06-03 11:52 ` Arne Jansen 2011-06-05 8:17 ` Ingo Molnar 2011-06-05 8:53 ` Arne Jansen 2011-06-05 9:41 ` Ingo Molnar 2011-06-05 9:45 ` Ingo Molnar 2011-06-05 9:43 ` Arne Jansen 2011-06-05 9:55 ` Ingo Molnar 2011-06-05 10:22 ` Arne Jansen 2011-06-05 11:01 ` Ingo Molnar 2011-06-05 11:19 ` [debug patch] printk: Add a printk killswitch to robustify NMI watchdog messages Ingo Molnar 2011-06-05 11:36 ` Ingo Molnar 2011-06-05 11:57 ` Arne Jansen 2011-06-05 13:39 ` Ingo Molnar 2011-06-05 13:54 ` Arne Jansen 2011-06-05 14:06 ` Ingo Molnar 2011-06-05 14:45 ` Arne Jansen 2011-06-05 14:10 ` Ingo Molnar 2011-06-05 14:31 ` Arne Jansen 2011-06-05 15:13 ` Ingo Molnar 2011-06-05 15:26 ` Ingo Molnar 2011-06-05 15:32 ` Ingo Molnar 2011-06-05 16:07 ` Arne Jansen 2011-06-05 16:35 ` Arne Jansen 2011-06-05 16:50 ` Arne Jansen 2011-06-05 17:20 ` Ingo Molnar 2011-06-05 17:42 ` Arne Jansen 2011-06-05 18:59 ` Ingo Molnar 2011-06-05 19:30 ` Arne Jansen 2011-06-05 19:44 ` Ingo Molnar 2011-06-05 20:15 ` Arne Jansen 2011-06-06 6:56 ` Arne Jansen 2011-06-06 9:01 ` Peter Zijlstra 2011-06-06 9:18 ` Arne Jansen 2011-06-06 9:24 ` Peter Zijlstra 2011-06-06 9:52 ` Peter Zijlstra 2011-06-06 10:00 ` Arne Jansen 2011-06-06 10:26 ` Peter Zijlstra 2011-06-06 13:25 ` Peter Zijlstra 2011-06-06 15:04 ` Ingo Molnar 2011-06-06 15:08 ` Ingo Molnar 2011-06-06 17:44 ` Mike Galbraith 2011-06-07 5:20 ` Mike Galbraith 2011-06-06 13:10 ` Ingo Molnar 2011-06-06 13:12 ` Peter Zijlstra 2011-06-06 13:21 ` Ingo Molnar 2011-06-06 13:31 ` Peter Zijlstra 2011-06-06 7:34 ` Arne Jansen 2011-06-05 15:34 ` Arne Jansen 2011-06-06 8:38 ` Peter Zijlstra 2011-06-06 14:58 ` Ingo Molnar 2011-06-06 15:09 ` Peter Zijlstra 2011-06-06 15:47 ` Peter Zijlstra 2011-06-06 15:52 ` Ingo Molnar 2011-06-06 16:00 ` Peter Zijlstra 2011-06-06 16:08 ` Ingo Molnar 2011-06-06 16:12 ` Peter Zijlstra 2011-06-06 16:17 ` Ingo Molnar 2011-06-06 16:38 ` Arne Jansen 2011-06-06 16:45 ` Arne Jansen 2011-06-06 16:53 ` Peter Zijlstra 2011-06-06 17:07 ` Ingo Molnar 2011-06-06 17:11 ` Peter Zijlstra 2011-06-08 15:50 ` Peter Zijlstra 2011-06-08 19:17 ` Ingo Molnar 2011-06-08 19:27 ` Linus Torvalds 2011-06-08 20:32 ` Peter Zijlstra 2011-06-08 20:53 ` Linus Torvalds 2011-06-08 20:54 ` Thomas Gleixner 2011-06-08 19:45 ` Peter Zijlstra 2011-06-08 20:52 ` Ingo Molnar 2011-06-08 21:49 ` Peter Zijlstra 2011-06-08 21:57 ` Thomas Gleixner 2011-06-06 16:44 ` Peter Zijlstra 2011-06-06 16:50 ` Peter Zijlstra 2011-06-06 17:13 ` Ingo Molnar 2011-06-06 17:04 ` Peter Zijlstra 2011-06-06 17:11 ` Ingo Molnar 2011-06-06 17:57 ` Arne Jansen 2011-06-06 18:07 ` Ingo Molnar 2011-06-06 18:14 ` Arne Jansen 2011-06-06 18:19 ` Peter Zijlstra 2011-06-06 22:08 ` Ingo Molnar 2011-06-03 12:44 ` [tip:sched/locking] sched: Add p->pi_lock to task_rq_lock() Linus Torvalds 2011-06-03 13:05 ` Arne Jansen 2011-06-04 21:29 ` Linus Torvalds 2011-06-04 22:08 ` Peter Zijlstra 2011-06-04 22:50 ` Linus Torvalds 2011-06-05 6:01 ` Arne Jansen 2011-06-05 7:57 ` Mike Galbraith 2011-04-05 15:23 ` [PATCH 14/21] sched: Drop rq->lock from first part of wake_up_new_task() Peter Zijlstra 2011-04-14 8:37 ` [tip:sched/locking] " tip-bot for Peter Zijlstra 2011-04-05 15:23 ` [PATCH 15/21] sched: Drop rq->lock from sched_exec() Peter Zijlstra 2011-04-14 8:37 ` [tip:sched/locking] " tip-bot for Peter Zijlstra 2011-04-05 15:23 ` [PATCH 16/21] sched: Remove rq->lock from the first half of ttwu() Peter Zijlstra 2011-04-14 8:38 ` [tip:sched/locking] " tip-bot for Peter Zijlstra 2011-04-05 15:23 ` [PATCH 17/21] sched: Remove rq argument from ttwu_stat() Peter Zijlstra 2011-04-14 8:38 ` [tip:sched/locking] " tip-bot for Peter Zijlstra 2011-04-05 15:23 ` [PATCH 18/21] sched: Rename ttwu_post_activation Peter Zijlstra 2011-04-14 8:39 ` [tip:sched/locking] sched: Rename ttwu_post_activation() to ttwu_do_wakeup() tip-bot for Peter Zijlstra 2011-04-05 15:23 ` [PATCH 19/21] sched: Restructure ttwu some more Peter Zijlstra 2011-04-14 8:39 ` [tip:sched/locking] sched: Restructure ttwu() " tip-bot for Peter Zijlstra 2011-04-05 15:23 ` [PATCH 20/21] sched: Move the second half of ttwu() to the remote cpu Peter Zijlstra 2011-04-14 8:39 ` [tip:sched/locking] " tip-bot for Peter Zijlstra 2011-04-05 15:23 ` [PATCH 21/21] sched: Remove need_migrate_task() Peter Zijlstra 2011-04-14 8:40 ` [tip:sched/locking] " tip-bot for Peter Zijlstra 2011-04-05 15:59 ` [PATCH 00/21] sched: Reduce runqueue lock contention -v6 Peter Zijlstra 2011-04-06 11:00 ` Peter Zijlstra 2011-04-27 16:54 ` Dave Kleikamp
Reply instructions: You may reply publicly to this message via plain-text email using any one of the following methods: * Save the following mbox file, import it into your mail client, and reply-to-all from there: mbox Avoid top-posting and favor interleaved quoting: https://en.wikipedia.org/wiki/Posting_style#Interleaved_style * Reply using the --to, --cc, and --in-reply-to switches of git-send-email(1): git send-email \ --in-reply-to=20110405152728.826338173@chello.nl \ --to=a.p.zijlstra@chello.nl \ --cc=axboe@kernel.dk \ --cc=chris.mason@oracle.com \ --cc=efault@gmx.de \ --cc=frank.rowand@am.sony.com \ --cc=linux-kernel@vger.kernel.org \ --cc=mingo@elte.hu \ --cc=oleg@redhat.com \ --cc=pjt@google.com \ --cc=tglx@linutronix.de \ --cc=yong.zhang0@gmail.com \ /path/to/YOUR_REPLY https://kernel.org/pub/software/scm/git/docs/git-send-email.html * If your mail client supports setting the In-Reply-To header via mailto: links, try the mailto: link
LKML Archive on lore.kernel.org Archives are clonable: git clone --mirror https://lore.kernel.org/lkml/0 lkml/git/0.git git clone --mirror https://lore.kernel.org/lkml/1 lkml/git/1.git git clone --mirror https://lore.kernel.org/lkml/2 lkml/git/2.git git clone --mirror https://lore.kernel.org/lkml/3 lkml/git/3.git git clone --mirror https://lore.kernel.org/lkml/4 lkml/git/4.git git clone --mirror https://lore.kernel.org/lkml/5 lkml/git/5.git git clone --mirror https://lore.kernel.org/lkml/6 lkml/git/6.git git clone --mirror https://lore.kernel.org/lkml/7 lkml/git/7.git git clone --mirror https://lore.kernel.org/lkml/8 lkml/git/8.git git clone --mirror https://lore.kernel.org/lkml/9 lkml/git/9.git # If you have public-inbox 1.1+ installed, you may # initialize and index your mirror using the following commands: public-inbox-init -V2 lkml lkml/ https://lore.kernel.org/lkml \ linux-kernel@vger.kernel.org public-inbox-index lkml Example config snippet for mirrors Newsgroup available over NNTP: nntp://nntp.lore.kernel.org/org.kernel.vger.linux-kernel AGPL code for this site: git clone https://public-inbox.org/public-inbox.git