From: Thomas Gleixner <tglx@linutronix.de>
To: LKML <linux-kernel@vger.kernel.org>
Cc: Peter Zijlstra <peterz@infradead.org>,
Ingo Molnar <mingo@kernel.org>,
Juri Lelli <juri.lelli@redhat.com>,
Steven Rostedt <rostedt@goodmis.org>,
Daniel Bristot de Oliveira <bristot@redhat.com>,
Will Deacon <will@kernel.org>, Waiman Long <longman@redhat.com>,
Boqun Feng <boqun.feng@gmail.com>,
Sebastian Andrzej Siewior <bigeasy@linutronix.de>,
Davidlohr Bueso <dave@stgolabs.net>,
Mike Galbraith <efault@gmx.de>
Subject: [patch V3 52/64] locking/rtmutex: Add mutex variant for RT
Date: Thu, 05 Aug 2021 17:13:52 +0200 [thread overview]
Message-ID: <20210805153955.826492729@linutronix.de> (raw)
In-Reply-To: 20210805151300.330412127@linutronix.de
From: Thomas Gleixner <tglx@linutronix.de>
Add the necessary defines, helpers and API functions for replacing mutex on
a PREEMPT_RT enabled kernel with a rtmutex based variant.
No functional change when CONFIG_PREEMPT_RT=n
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
---
include/linux/mutex.h | 67 ++++++++++++++++++++---
kernel/locking/mutex.c | 4 +
kernel/locking/rtmutex_api.c | 122 +++++++++++++++++++++++++++++++++++++++++++
lib/Kconfig.debug | 11 ++-
4 files changed, 188 insertions(+), 16 deletions(-)
---
--- a/include/linux/mutex.h
+++ b/include/linux/mutex.h
@@ -20,6 +20,18 @@
#include <linux/osq_lock.h>
#include <linux/debug_locks.h>
+#ifdef CONFIG_DEBUG_LOCK_ALLOC
+# define __DEP_MAP_MUTEX_INITIALIZER(lockname) \
+ , .dep_map = { \
+ .name = #lockname, \
+ .wait_type_inner = LD_WAIT_SLEEP, \
+ }
+#else
+# define __DEP_MAP_MUTEX_INITIALIZER(lockname)
+#endif
+
+#ifndef CONFIG_PREEMPT_RT
+
/*
* Simple, straightforward mutexes with strict semantics:
*
@@ -93,16 +105,6 @@ do { \
__mutex_init((mutex), #mutex, &__key); \
} while (0)
-#ifdef CONFIG_DEBUG_LOCK_ALLOC
-# define __DEP_MAP_MUTEX_INITIALIZER(lockname) \
- , .dep_map = { \
- .name = #lockname, \
- .wait_type_inner = LD_WAIT_SLEEP, \
- }
-#else
-# define __DEP_MAP_MUTEX_INITIALIZER(lockname)
-#endif
-
#define __MUTEX_INITIALIZER(lockname) \
{ .owner = ATOMIC_LONG_INIT(0) \
, .wait_lock = __RAW_SPIN_LOCK_UNLOCKED(lockname.wait_lock) \
@@ -124,6 +126,51 @@ extern void __mutex_init(struct mutex *l
*/
extern bool mutex_is_locked(struct mutex *lock);
+#else /* !CONFIG_PREEMPT_RT */
+/*
+ * Preempt-RT variant based on rtmutexes.
+ */
+#include <linux/rtmutex.h>
+
+struct mutex {
+ struct rt_mutex_base rtmutex;
+#ifdef CONFIG_DEBUG_LOCK_ALLOC
+ struct lockdep_map dep_map;
+#endif
+};
+
+#define __MUTEX_INITIALIZER(mutexname) \
+{ \
+ .rtmutex = __RT_MUTEX_BASE_INITIALIZER(mutexname.rtmutex) \
+ __DEP_MAP_MUTEX_INITIALIZER(mutexname) \
+}
+
+#define DEFINE_MUTEX(mutexname) \
+ struct mutex mutexname = __MUTEX_INITIALIZER(mutexname)
+
+extern void __mutex_rt_init(struct mutex *lock, const char *name,
+ struct lock_class_key *key);
+extern int mutex_trylock(struct mutex *lock);
+
+static inline void mutex_destroy(struct mutex *lock) { }
+
+#define mutex_is_locked(l) rt_mutex_base_is_locked(&(l)->rtmutex)
+
+#define mutex_init(mutex) \
+do { \
+ static struct lock_class_key __key; \
+ \
+ rt_mutex_base_init(&(mutex)->rtmutex); \
+ __mutex_rt_init((mutex), #mutex, &__key); \
+} while (0)
+
+#define __mutex_init(mutex, name, key) \
+do { \
+ rt_mutex_base_init(&(mutex)->rtmutex); \
+ __mutex_rt_init((mutex), name, key); \
+} while (0)
+#endif /* CONFIG_PREEMPT_RT */
+
/*
* See kernel/locking/mutex.c for detailed documentation of these APIs.
* Also see Documentation/locking/mutex-design.rst.
--- a/kernel/locking/mutex.c
+++ b/kernel/locking/mutex.c
@@ -30,6 +30,7 @@
#include <linux/debug_locks.h>
#include <linux/osq_lock.h>
+#ifndef CONFIG_PREEMPT_RT
#include "mutex.h"
void
@@ -1078,7 +1079,8 @@ ww_mutex_lock_interruptible(struct ww_mu
}
EXPORT_SYMBOL(ww_mutex_lock_interruptible);
-#endif
+#endif /* !CONFIG_DEBUG_LOCK_ALLOC */
+#endif /* !CONFIG_PREEMPT_RT */
/**
* atomic_dec_and_mutex_lock - return holding mutex if we dec to 0
--- a/kernel/locking/rtmutex_api.c
+++ b/kernel/locking/rtmutex_api.c
@@ -454,3 +454,125 @@ void rt_mutex_debug_task_free(struct tas
DEBUG_LOCKS_WARN_ON(task->pi_blocked_on);
}
#endif
+
+#ifdef CONFIG_PREEMPT_RT
+/* Mutexes */
+void __mutex_rt_init(struct mutex *mutex, const char *name,
+ struct lock_class_key *key)
+{
+ debug_check_no_locks_freed((void *)mutex, sizeof(*mutex));
+ lockdep_init_map(&mutex->dep_map, name, key, 0);
+}
+EXPORT_SYMBOL(__mutex_rt_init);
+
+static __always_inline int __mutex_lock_common(struct mutex *lock,
+ unsigned int state,
+ unsigned int subclass,
+ struct lockdep_map *nest_lock,
+ unsigned long ip)
+{
+ int ret;
+
+ might_sleep();
+ mutex_acquire_nest(&lock->dep_map, subclass, 0, nest_lock, ip);
+ ret = __rt_mutex_lock(&lock->rtmutex, state);
+ if (ret)
+ mutex_release(&lock->dep_map, ip);
+ else
+ lock_acquired(&lock->dep_map, ip);
+ return ret;
+}
+
+#ifdef CONFIG_DEBUG_LOCK_ALLOC
+void __sched mutex_lock_nested(struct mutex *lock, unsigned int subclass)
+{
+ __mutex_lock_common(lock, TASK_UNINTERRUPTIBLE, subclass, NULL, _RET_IP_);
+}
+EXPORT_SYMBOL_GPL(mutex_lock_nested);
+
+void __sched _mutex_lock_nest_lock(struct mutex *lock,
+ struct lockdep_map *nest_lock)
+{
+ __mutex_lock_common(lock, TASK_UNINTERRUPTIBLE, 0, nest_lock, _RET_IP_);
+}
+EXPORT_SYMBOL_GPL(_mutex_lock_nest_lock);
+
+int __sched mutex_lock_interruptible_nested(struct mutex *lock,
+ unsigned int subclass)
+{
+ return __mutex_lock_common(lock, TASK_INTERRUPTIBLE, subclass, NULL, _RET_IP_);
+}
+EXPORT_SYMBOL_GPL(mutex_lock_interruptible_nested);
+
+int __sched mutex_lock_killable_nested(struct mutex *lock,
+ unsigned int subclass)
+{
+ return __mutex_lock_common(lock, TASK_KILLABLE, subclass, NULL, _RET_IP_);
+}
+EXPORT_SYMBOL_GPL(mutex_lock_killable_nested);
+
+void __sched mutex_lock_io_nested(struct mutex *lock, unsigned int subclass)
+{
+ int token;
+
+ might_sleep();
+
+ token = io_schedule_prepare();
+ __mutex_lock_common(lock, TASK_UNINTERRUPTIBLE, subclass, NULL, _RET_IP_);
+ io_schedule_finish(token);
+}
+EXPORT_SYMBOL_GPL(mutex_lock_io_nested);
+
+#else /* CONFIG_DEBUG_LOCK_ALLOC */
+
+void __sched mutex_lock(struct mutex *lock)
+{
+ __mutex_lock_common(lock, TASK_UNINTERRUPTIBLE, 0, NULL, _RET_IP_);
+}
+EXPORT_SYMBOL(mutex_lock);
+
+int __sched mutex_lock_interruptible(struct mutex *lock)
+{
+ return __mutex_lock_common(lock, TASK_INTERRUPTIBLE, 0, NULL, _RET_IP_);
+}
+EXPORT_SYMBOL(mutex_lock_interruptible);
+
+int __sched mutex_lock_killable(struct mutex *lock)
+{
+ return __mutex_lock_common(lock, TASK_KILLABLE, 0, NULL, _RET_IP_);
+}
+EXPORT_SYMBOL(mutex_lock_killable);
+
+void __sched mutex_lock_io(struct mutex *lock)
+{
+ int token = io_schedule_prepare();
+
+ __mutex_lock_common(lock, TASK_UNINTERRUPTIBLE, 0, NULL, _RET_IP_);
+ io_schedule_finish(token);
+}
+EXPORT_SYMBOL(mutex_lock_io);
+#endif /* !CONFIG_DEBUG_LOCK_ALLOC */
+
+int __sched mutex_trylock(struct mutex *lock)
+{
+ int ret;
+
+ if (IS_ENABLED(CONFIG_DEBUG_RT_MUTEXES) && WARN_ON_ONCE(!in_task()))
+ return 0;
+
+ ret = __rt_mutex_trylock(&lock->rtmutex);
+ if (ret)
+ mutex_acquire(&lock->dep_map, 0, 1, _RET_IP_);
+
+ return ret;
+}
+EXPORT_SYMBOL(mutex_trylock);
+
+void __sched mutex_unlock(struct mutex *lock)
+{
+ mutex_release(&lock->dep_map, _RET_IP_);
+ __rt_mutex_unlock(&lock->rtmutex);
+}
+EXPORT_SYMBOL(mutex_unlock);
+
+#endif /* CONFIG_PREEMPT_RT */
--- a/lib/Kconfig.debug
+++ b/lib/Kconfig.debug
@@ -1235,7 +1235,7 @@ config PROVE_LOCKING
depends on DEBUG_KERNEL && LOCK_DEBUGGING_SUPPORT
select LOCKDEP
select DEBUG_SPINLOCK
- select DEBUG_MUTEXES
+ select DEBUG_MUTEXES if !PREEMPT_RT
select DEBUG_RT_MUTEXES if RT_MUTEXES
select DEBUG_RWSEMS
select DEBUG_WW_MUTEX_SLOWPATH
@@ -1299,7 +1299,7 @@ config LOCK_STAT
depends on DEBUG_KERNEL && LOCK_DEBUGGING_SUPPORT
select LOCKDEP
select DEBUG_SPINLOCK
- select DEBUG_MUTEXES
+ select DEBUG_MUTEXES if !PREEMPT_RT
select DEBUG_RT_MUTEXES if RT_MUTEXES
select DEBUG_LOCK_ALLOC
default n
@@ -1335,7 +1335,7 @@ config DEBUG_SPINLOCK
config DEBUG_MUTEXES
bool "Mutex debugging: basic checks"
- depends on DEBUG_KERNEL
+ depends on DEBUG_KERNEL && !PREEMPT_RT
help
This feature allows mutex semantics violations to be detected and
reported.
@@ -1345,7 +1345,8 @@ config DEBUG_WW_MUTEX_SLOWPATH
depends on DEBUG_KERNEL && LOCK_DEBUGGING_SUPPORT
select DEBUG_LOCK_ALLOC
select DEBUG_SPINLOCK
- select DEBUG_MUTEXES
+ select DEBUG_MUTEXES if !PREEMPT_RT
+ select DEBUG_RT_MUTEXES if PREEMPT_RT
help
This feature enables slowpath testing for w/w mutex users by
injecting additional -EDEADLK wound/backoff cases. Together with
@@ -1368,7 +1369,7 @@ config DEBUG_LOCK_ALLOC
bool "Lock debugging: detect incorrect freeing of live locks"
depends on DEBUG_KERNEL && LOCK_DEBUGGING_SUPPORT
select DEBUG_SPINLOCK
- select DEBUG_MUTEXES
+ select DEBUG_MUTEXES if !PREEMPT_RT
select DEBUG_RT_MUTEXES if RT_MUTEXES
select LOCKDEP
help
next prev parent reply other threads:[~2021-08-05 15:45 UTC|newest]
Thread overview: 79+ messages / expand[flat|nested] mbox.gz Atom feed top
2021-08-05 15:13 [patch V3 00/64] locking, sched: The PREEMPT-RT locking infrastructure Thomas Gleixner
2021-08-05 15:13 ` [patch V3 01/64] sched: Split out the wakeup state check Thomas Gleixner
2021-08-05 15:13 ` [patch V3 02/64] sched: Introduce TASK_RTLOCK_WAIT Thomas Gleixner
2021-08-05 15:13 ` [patch V3 03/64] sched: Reorganize current::__state helpers Thomas Gleixner
2021-08-05 15:13 ` [patch V3 04/64] sched: Prepare for RT sleeping spin/rwlocks Thomas Gleixner
2021-08-05 15:13 ` [patch V3 05/64] sched: Rework the __schedule() preempt argument Thomas Gleixner
2021-08-05 15:13 ` [patch V3 06/64] sched: Provide schedule point for RT locks Thomas Gleixner
2021-08-05 15:13 ` [patch V3 07/64] sched/wake_q: Provide WAKE_Q_HEAD_INITIALIZER Thomas Gleixner
2021-08-05 15:13 ` [patch V3 08/64] media/atomisp: Use lockdep instead of *mutex_is_locked() Thomas Gleixner
2021-08-05 15:13 ` [patch V3 09/64] rtmutex: Remove rt_mutex_is_locked() Thomas Gleixner
2021-08-05 15:13 ` [patch V3 10/64] rtmutex: Convert macros to inlines Thomas Gleixner
2021-08-05 15:13 ` [patch V3 11/64] rtmutex: Switch to try_cmpxchg() Thomas Gleixner
2021-08-05 15:13 ` [patch V3 12/64] rtmutex: Split API and implementation Thomas Gleixner
2021-08-05 15:13 ` [patch V3 13/64] rtmutex: Split out the inner parts of struct rtmutex Thomas Gleixner
2021-08-05 15:13 ` [patch V3 14/64] locking/rtmutex: Provide rt_mutex_slowlock_locked() Thomas Gleixner
2021-08-05 15:13 ` [patch V3 15/64] rtmutex: Provide rt_mutex_base_is_locked() Thomas Gleixner
2021-08-08 20:41 ` Davidlohr Bueso
2021-08-09 10:18 ` Thomas Gleixner
2021-08-05 15:13 ` [patch V3 16/64] locking: Add base code for RT rw_semaphore and rwlock Thomas Gleixner
2021-08-05 15:13 ` [patch V3 17/64] locking/rwsem: Add rtmutex based R/W semaphore implementation Thomas Gleixner
2021-08-05 15:13 ` [patch V3 18/64] locking/rtmutex: Add wake_state to rt_mutex_waiter Thomas Gleixner
2021-08-05 15:13 ` [patch V3 19/64] locking/rtmutex: Provide rt_wake_q and helpers Thomas Gleixner
2021-08-05 15:13 ` [patch V3 20/64] locking/rtmutex: Use rt_mutex_wake_q_head Thomas Gleixner
2021-08-05 15:13 ` [patch V3 21/64] locking/rtmutex: Prepare RT rt_mutex_wake_q for RT locks Thomas Gleixner
2021-08-05 15:13 ` [patch V3 22/64] locking/rtmutex: Guard regular sleeping locks specific functions Thomas Gleixner
2021-08-05 15:13 ` [patch V3 23/64] locking/spinlock: Split the lock types header Thomas Gleixner
2021-08-05 15:13 ` [patch V3 24/64] locking/rtmutex: Prevent future include recursion hell Thomas Gleixner
2021-08-05 15:13 ` [patch V3 25/64] locking/lockdep: Reduce includes in debug_locks.h Thomas Gleixner
2021-08-05 15:13 ` [patch V3 26/64] rbtree: Split out the rbtree type definitions Thomas Gleixner
2021-08-05 15:13 ` [patch V3 27/64] locking/rtmutex: Include only rbtree types Thomas Gleixner
2021-08-05 15:13 ` [patch V3 28/64] locking/spinlock: Provide RT specific spinlock type Thomas Gleixner
2021-08-05 15:13 ` [patch V3 29/64] locking/spinlock: Provide RT variant header Thomas Gleixner
2021-08-05 15:13 ` [patch V3 30/64] locking/rtmutex: Provide the spin/rwlock core lock function Thomas Gleixner
2021-08-05 15:13 ` [patch V3 31/64] locking/spinlock: Provide RT variant Thomas Gleixner
2021-08-05 15:13 ` [patch V3 32/64] locking/rwlock: " Thomas Gleixner
2021-08-05 15:13 ` [patch V3 33/64] locking/mutex: Consolidate core headers Thomas Gleixner
2021-08-05 15:13 ` [patch V3 34/64] locking/mutex: Move waiter to core header Thomas Gleixner
2021-08-05 15:13 ` [patch V3 35/64] locking/ww_mutex: Move ww_mutex declarations into ww_mutex.h Thomas Gleixner
2021-08-05 15:13 ` [patch V3 36/64] locking/mutex: Make mutex::wait_lock raw Thomas Gleixner
2021-08-05 15:13 ` [patch V3 37/64] locking/ww_mutex: Simplify lockdep annotation Thomas Gleixner
2021-08-05 15:13 ` [patch V3 38/64] locking/ww_mutex: Gather mutex_waiter initialization Thomas Gleixner
2021-08-05 15:13 ` [patch V3 39/64] locking/ww_mutex: Split up ww_mutex_unlock() Thomas Gleixner
2021-08-05 15:13 ` [patch V3 40/64] locking/ww_mutex: Split W/W implementation logic Thomas Gleixner
2021-08-05 15:13 ` [patch V3 41/64] locking/ww_mutex: Remove __sched annotation Thomas Gleixner
2021-08-05 15:13 ` [patch V3 42/64] locking/ww_mutex: Abstract waiter iteration Thomas Gleixner
2021-08-05 15:13 ` [patch V3 43/64] locking/ww_mutex: Abstract waiter enqueueing Thomas Gleixner
2021-08-05 15:13 ` [patch V3 44/64] locking/ww_mutex: Abstract mutex accessors Thomas Gleixner
2021-08-05 15:13 ` [patch V3 45/64] locking/ww_mutex: Abstract mutex types Thomas Gleixner
2021-08-05 15:13 ` [patch V3 46/64] locking/ww_mutex: Abstract internal lock access Thomas Gleixner
2021-08-05 15:13 ` [patch V3 47/64] locking/ww_mutex: Implement rt_mutex accessors Thomas Gleixner
2021-08-05 15:13 ` [patch V3 48/64] locking/ww_mutex: Add RT priority to W/W order Thomas Gleixner
2021-08-06 10:48 ` Peter Zijlstra
2021-08-06 11:50 ` Thomas Gleixner
2021-08-05 15:13 ` [patch V3 49/64] locking/ww_mutex: Add rt_mutex based lock type and accessors Thomas Gleixner
2021-08-05 15:13 ` [patch V3 50/64] locking/rtmutex: Extend the rtmutex core to support ww_mutex Thomas Gleixner
2021-08-06 11:00 ` Peter Zijlstra
2021-08-06 11:19 ` Thomas Gleixner
2021-08-06 13:48 ` Peter Zijlstra
2021-08-07 20:07 ` Thomas Gleixner
2021-08-05 15:13 ` [patch V3 51/64] locking/ww_mutex: Implement rtmutex based ww_mutex API functions Thomas Gleixner
2021-08-05 15:13 ` Thomas Gleixner [this message]
2021-08-05 15:13 ` [patch V3 53/64] lib/test_lockup: Adapt to changed variables Thomas Gleixner
2021-08-05 15:13 ` [patch V3 54/64] futex: Validate waiter correctly in futex_proxy_trylock_atomic() Thomas Gleixner
2021-08-05 15:13 ` [patch V3 55/64] futex: Cleanup stale comments Thomas Gleixner
2021-08-05 15:13 ` [patch V3 56/64] futex: Correct the number of requeued waiters for PI Thomas Gleixner
2021-08-08 17:05 ` Davidlohr Bueso
2021-08-09 8:18 ` Thomas Gleixner
2021-08-09 10:52 ` Thomas Gleixner
2021-08-05 15:13 ` [patch V3 57/64] futex: Restructure futex_requeue() Thomas Gleixner
2021-08-05 15:13 ` [patch V3 58/64] futex: Clarify comment in futex_requeue() Thomas Gleixner
2021-08-08 18:43 ` Davidlohr Bueso
2021-08-09 8:18 ` Thomas Gleixner
2021-08-05 15:13 ` [patch V3 59/64] futex: Simplify handle_early_requeue_pi_wakeup() Thomas Gleixner
2021-08-05 15:14 ` [patch V3 60/64] futex: Prevent requeue_pi() lock nesting issue on RT Thomas Gleixner
2021-08-05 15:14 ` [patch V3 61/64] rtmutex: Prevent lockdep false positive with PI futexes Thomas Gleixner
2021-08-05 15:14 ` [patch V3 62/64] preempt: Adjust PREEMPT_LOCK_OFFSET for RT Thomas Gleixner
2021-08-05 15:14 ` [patch V3 63/64] locking/rtmutex: Implement equal priority lock stealing Thomas Gleixner
2021-08-05 15:14 ` [patch V3 64/64] locking/rtmutex: Add adaptive spinwait mechanism Thomas Gleixner
2021-08-09 20:28 ` Davidlohr Bueso
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20210805153955.826492729@linutronix.de \
--to=tglx@linutronix.de \
--cc=bigeasy@linutronix.de \
--cc=boqun.feng@gmail.com \
--cc=bristot@redhat.com \
--cc=dave@stgolabs.net \
--cc=efault@gmx.de \
--cc=juri.lelli@redhat.com \
--cc=linux-kernel@vger.kernel.org \
--cc=longman@redhat.com \
--cc=mingo@kernel.org \
--cc=peterz@infradead.org \
--cc=rostedt@goodmis.org \
--cc=will@kernel.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).