From: Alex Kogan <alex.kogan@oracle.com>
To: linux@armlinux.org.uk, peterz@infradead.org, mingo@redhat.com,
will.deacon@arm.com, arnd@arndb.de, longman@redhat.com,
linux-arch@vger.kernel.org, linux-arm-kernel@lists.infradead.org,
linux-kernel@vger.kernel.org, tglx@linutronix.de, bp@alien8.de,
hpa@zytor.com, x86@kernel.org, guohanjun@huawei.com,
jglauber@marvell.com
Cc: steven.sistare@oracle.com, daniel.m.jordan@oracle.com,
alex.kogan@oracle.com, dave.dice@oracle.com
Subject: [PATCH v9 2/5] locking/qspinlock: Refactor the qspinlock slow path
Date: Tue, 14 Jan 2020 22:59:17 -0500 [thread overview]
Message-ID: <20200115035920.54451-3-alex.kogan@oracle.com> (raw)
In-Reply-To: <20200115035920.54451-1-alex.kogan@oracle.com>
Move some of the code manipulating the spin lock into separate functions.
This would allow easier integration of alternative ways to manipulate
that lock.
Signed-off-by: Alex Kogan <alex.kogan@oracle.com>
Reviewed-by: Steve Sistare <steven.sistare@oracle.com>
Reviewed-by: Waiman Long <longman@redhat.com>
---
kernel/locking/qspinlock.c | 38 ++++++++++++++++++++++++++++++++++++--
1 file changed, 36 insertions(+), 2 deletions(-)
diff --git a/kernel/locking/qspinlock.c b/kernel/locking/qspinlock.c
index 804c0fbd6328..c06d1e8075d9 100644
--- a/kernel/locking/qspinlock.c
+++ b/kernel/locking/qspinlock.c
@@ -288,6 +288,34 @@ static __always_inline u32 __pv_wait_head_or_lock(struct qspinlock *lock,
#define queued_spin_lock_slowpath native_queued_spin_lock_slowpath
#endif
+/*
+ * __try_clear_tail - try to clear tail by setting the lock value to
+ * _Q_LOCKED_VAL.
+ * @lock: Pointer to the queued spinlock structure
+ * @val: Current value of the lock
+ * @node: Pointer to the MCS node of the lock holder
+ */
+static __always_inline bool __try_clear_tail(struct qspinlock *lock,
+ u32 val,
+ struct mcs_spinlock *node)
+{
+ return atomic_try_cmpxchg_relaxed(&lock->val, &val, _Q_LOCKED_VAL);
+}
+
+/*
+ * __mcs_pass_lock - pass the MCS lock to the next waiter
+ * @node: Pointer to the MCS node of the lock holder
+ * @next: Pointer to the MCS node of the first waiter in the MCS queue
+ */
+static __always_inline void __mcs_pass_lock(struct mcs_spinlock *node,
+ struct mcs_spinlock *next)
+{
+ arch_mcs_pass_lock(&next->locked, 1);
+}
+
+#define try_clear_tail __try_clear_tail
+#define mcs_pass_lock __mcs_pass_lock
+
#endif /* _GEN_PV_LOCK_SLOWPATH */
/**
@@ -532,7 +560,7 @@ void queued_spin_lock_slowpath(struct qspinlock *lock, u32 val)
* PENDING will make the uncontended transition fail.
*/
if ((val & _Q_TAIL_MASK) == tail) {
- if (atomic_try_cmpxchg_relaxed(&lock->val, &val, _Q_LOCKED_VAL))
+ if (try_clear_tail(lock, val, node))
goto release; /* No contention */
}
@@ -549,7 +577,7 @@ void queued_spin_lock_slowpath(struct qspinlock *lock, u32 val)
if (!next)
next = smp_cond_load_relaxed(&node->next, (VAL));
- arch_mcs_pass_lock(&next->locked, 1);
+ mcs_pass_lock(node, next);
pv_kick_node(lock, next);
release:
@@ -574,6 +602,12 @@ EXPORT_SYMBOL(queued_spin_lock_slowpath);
#undef pv_kick_node
#undef pv_wait_head_or_lock
+#undef try_clear_tail
+#define try_clear_tail __try_clear_tail
+
+#undef mcs_pass_lock
+#define mcs_pass_lock __mcs_pass_lock
+
#undef queued_spin_lock_slowpath
#define queued_spin_lock_slowpath __pv_queued_spin_lock_slowpath
--
2.21.0 (Apple Git-122.2)
WARNING: multiple messages have this Message-ID (diff)
From: Alex Kogan <alex.kogan@oracle.com>
To: linux@armlinux.org.uk, peterz@infradead.org, mingo@redhat.com,
will.deacon@arm.com, arnd@arndb.de, longman@redhat.com,
linux-arch@vger.kernel.org, linux-arm-kernel@lists.infradead.org,
linux-kernel@vger.kernel.org, tglx@linutronix.de, bp@alien8.de,
hpa@zytor.com, x86@kernel.org, guohanjun@huawei.com,
jglauber@marvell.com
Cc: alex.kogan@oracle.com, dave.dice@oracle.com,
steven.sistare@oracle.com, daniel.m.jordan@oracle.com
Subject: [PATCH v9 2/5] locking/qspinlock: Refactor the qspinlock slow path
Date: Tue, 14 Jan 2020 22:59:17 -0500 [thread overview]
Message-ID: <20200115035920.54451-3-alex.kogan@oracle.com> (raw)
In-Reply-To: <20200115035920.54451-1-alex.kogan@oracle.com>
Move some of the code manipulating the spin lock into separate functions.
This would allow easier integration of alternative ways to manipulate
that lock.
Signed-off-by: Alex Kogan <alex.kogan@oracle.com>
Reviewed-by: Steve Sistare <steven.sistare@oracle.com>
Reviewed-by: Waiman Long <longman@redhat.com>
---
kernel/locking/qspinlock.c | 38 ++++++++++++++++++++++++++++++++++++--
1 file changed, 36 insertions(+), 2 deletions(-)
diff --git a/kernel/locking/qspinlock.c b/kernel/locking/qspinlock.c
index 804c0fbd6328..c06d1e8075d9 100644
--- a/kernel/locking/qspinlock.c
+++ b/kernel/locking/qspinlock.c
@@ -288,6 +288,34 @@ static __always_inline u32 __pv_wait_head_or_lock(struct qspinlock *lock,
#define queued_spin_lock_slowpath native_queued_spin_lock_slowpath
#endif
+/*
+ * __try_clear_tail - try to clear tail by setting the lock value to
+ * _Q_LOCKED_VAL.
+ * @lock: Pointer to the queued spinlock structure
+ * @val: Current value of the lock
+ * @node: Pointer to the MCS node of the lock holder
+ */
+static __always_inline bool __try_clear_tail(struct qspinlock *lock,
+ u32 val,
+ struct mcs_spinlock *node)
+{
+ return atomic_try_cmpxchg_relaxed(&lock->val, &val, _Q_LOCKED_VAL);
+}
+
+/*
+ * __mcs_pass_lock - pass the MCS lock to the next waiter
+ * @node: Pointer to the MCS node of the lock holder
+ * @next: Pointer to the MCS node of the first waiter in the MCS queue
+ */
+static __always_inline void __mcs_pass_lock(struct mcs_spinlock *node,
+ struct mcs_spinlock *next)
+{
+ arch_mcs_pass_lock(&next->locked, 1);
+}
+
+#define try_clear_tail __try_clear_tail
+#define mcs_pass_lock __mcs_pass_lock
+
#endif /* _GEN_PV_LOCK_SLOWPATH */
/**
@@ -532,7 +560,7 @@ void queued_spin_lock_slowpath(struct qspinlock *lock, u32 val)
* PENDING will make the uncontended transition fail.
*/
if ((val & _Q_TAIL_MASK) == tail) {
- if (atomic_try_cmpxchg_relaxed(&lock->val, &val, _Q_LOCKED_VAL))
+ if (try_clear_tail(lock, val, node))
goto release; /* No contention */
}
@@ -549,7 +577,7 @@ void queued_spin_lock_slowpath(struct qspinlock *lock, u32 val)
if (!next)
next = smp_cond_load_relaxed(&node->next, (VAL));
- arch_mcs_pass_lock(&next->locked, 1);
+ mcs_pass_lock(node, next);
pv_kick_node(lock, next);
release:
@@ -574,6 +602,12 @@ EXPORT_SYMBOL(queued_spin_lock_slowpath);
#undef pv_kick_node
#undef pv_wait_head_or_lock
+#undef try_clear_tail
+#define try_clear_tail __try_clear_tail
+
+#undef mcs_pass_lock
+#define mcs_pass_lock __mcs_pass_lock
+
#undef queued_spin_lock_slowpath
#define queued_spin_lock_slowpath __pv_queued_spin_lock_slowpath
--
2.21.0 (Apple Git-122.2)
WARNING: multiple messages have this Message-ID (diff)
From: Alex Kogan <alex.kogan@oracle.com>
To: linux@armlinux.org.uk, peterz@infradead.org, mingo@redhat.com,
will.deacon@arm.com, arnd@arndb.de, longman@redhat.com,
linux-arch@vger.kernel.org, linux-arm-kernel@lists.infradead.org,
linux-kernel@vger.kernel.org, tglx@linutronix.de, bp@alien8.de,
hpa@zytor.com, x86@kernel.org, guohanjun@huawei.com,
jglauber@marvell.com
Cc: alex.kogan@oracle.com, dave.dice@oracle.com,
steven.sistare@oracle.com, daniel.m.jordan@oracle.com
Subject: [PATCH v9 2/5] locking/qspinlock: Refactor the qspinlock slow path
Date: Tue, 14 Jan 2020 22:59:17 -0500 [thread overview]
Message-ID: <20200115035920.54451-3-alex.kogan@oracle.com> (raw)
In-Reply-To: <20200115035920.54451-1-alex.kogan@oracle.com>
Move some of the code manipulating the spin lock into separate functions.
This would allow easier integration of alternative ways to manipulate
that lock.
Signed-off-by: Alex Kogan <alex.kogan@oracle.com>
Reviewed-by: Steve Sistare <steven.sistare@oracle.com>
Reviewed-by: Waiman Long <longman@redhat.com>
---
kernel/locking/qspinlock.c | 38 ++++++++++++++++++++++++++++++++++++--
1 file changed, 36 insertions(+), 2 deletions(-)
diff --git a/kernel/locking/qspinlock.c b/kernel/locking/qspinlock.c
index 804c0fbd6328..c06d1e8075d9 100644
--- a/kernel/locking/qspinlock.c
+++ b/kernel/locking/qspinlock.c
@@ -288,6 +288,34 @@ static __always_inline u32 __pv_wait_head_or_lock(struct qspinlock *lock,
#define queued_spin_lock_slowpath native_queued_spin_lock_slowpath
#endif
+/*
+ * __try_clear_tail - try to clear tail by setting the lock value to
+ * _Q_LOCKED_VAL.
+ * @lock: Pointer to the queued spinlock structure
+ * @val: Current value of the lock
+ * @node: Pointer to the MCS node of the lock holder
+ */
+static __always_inline bool __try_clear_tail(struct qspinlock *lock,
+ u32 val,
+ struct mcs_spinlock *node)
+{
+ return atomic_try_cmpxchg_relaxed(&lock->val, &val, _Q_LOCKED_VAL);
+}
+
+/*
+ * __mcs_pass_lock - pass the MCS lock to the next waiter
+ * @node: Pointer to the MCS node of the lock holder
+ * @next: Pointer to the MCS node of the first waiter in the MCS queue
+ */
+static __always_inline void __mcs_pass_lock(struct mcs_spinlock *node,
+ struct mcs_spinlock *next)
+{
+ arch_mcs_pass_lock(&next->locked, 1);
+}
+
+#define try_clear_tail __try_clear_tail
+#define mcs_pass_lock __mcs_pass_lock
+
#endif /* _GEN_PV_LOCK_SLOWPATH */
/**
@@ -532,7 +560,7 @@ void queued_spin_lock_slowpath(struct qspinlock *lock, u32 val)
* PENDING will make the uncontended transition fail.
*/
if ((val & _Q_TAIL_MASK) == tail) {
- if (atomic_try_cmpxchg_relaxed(&lock->val, &val, _Q_LOCKED_VAL))
+ if (try_clear_tail(lock, val, node))
goto release; /* No contention */
}
@@ -549,7 +577,7 @@ void queued_spin_lock_slowpath(struct qspinlock *lock, u32 val)
if (!next)
next = smp_cond_load_relaxed(&node->next, (VAL));
- arch_mcs_pass_lock(&next->locked, 1);
+ mcs_pass_lock(node, next);
pv_kick_node(lock, next);
release:
@@ -574,6 +602,12 @@ EXPORT_SYMBOL(queued_spin_lock_slowpath);
#undef pv_kick_node
#undef pv_wait_head_or_lock
+#undef try_clear_tail
+#define try_clear_tail __try_clear_tail
+
+#undef mcs_pass_lock
+#define mcs_pass_lock __mcs_pass_lock
+
#undef queued_spin_lock_slowpath
#define queued_spin_lock_slowpath __pv_queued_spin_lock_slowpath
--
2.21.0 (Apple Git-122.2)
_______________________________________________
linux-arm-kernel mailing list
linux-arm-kernel@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel
next prev parent reply other threads:[~2020-01-15 4:15 UTC|newest]
Thread overview: 89+ messages / expand[flat|nested] mbox.gz Atom feed top
2020-01-15 3:59 [PATCH v9 0/5] Add NUMA-awareness to qspinlock Alex Kogan
2020-01-15 3:59 ` Alex Kogan
2020-01-15 3:59 ` [PATCH v9 1/5] locking/qspinlock: Rename mcs lock/unlock macros and make them more generic Alex Kogan
2020-01-15 3:59 ` Alex Kogan
2020-01-15 3:59 ` Alex Kogan [this message]
2020-01-15 3:59 ` [PATCH v9 2/5] locking/qspinlock: Refactor the qspinlock slow path Alex Kogan
2020-01-15 3:59 ` Alex Kogan
2020-01-15 3:59 ` [PATCH v9 3/5] locking/qspinlock: Introduce CNA into the slow path of qspinlock Alex Kogan
2020-01-15 3:59 ` Alex Kogan
2020-01-15 3:59 ` Alex Kogan
2020-01-23 9:26 ` Peter Zijlstra
2020-01-23 9:26 ` Peter Zijlstra
2020-01-23 9:26 ` Peter Zijlstra
2020-01-23 10:06 ` Peter Zijlstra
2020-01-23 10:06 ` Peter Zijlstra
2020-01-23 10:06 ` Peter Zijlstra
2020-01-23 10:16 ` Peter Zijlstra
2020-01-23 10:16 ` Peter Zijlstra
2020-01-23 10:16 ` Peter Zijlstra
2020-01-23 11:22 ` Will Deacon
2020-01-23 11:22 ` Will Deacon
2020-01-23 13:17 ` Peter Zijlstra
2020-01-23 13:17 ` Peter Zijlstra
2020-01-23 13:17 ` Peter Zijlstra
2020-01-23 14:15 ` Waiman Long
2020-01-23 14:15 ` Waiman Long
2020-01-23 15:29 ` Peter Zijlstra
2020-01-23 15:29 ` Peter Zijlstra
2020-01-23 15:29 ` Peter Zijlstra
2020-01-15 3:59 ` [PATCH v9 4/5] locking/qspinlock: Introduce starvation avoidance into CNA Alex Kogan
2020-01-15 3:59 ` Alex Kogan
2020-01-23 19:55 ` Waiman Long
2020-01-23 19:55 ` Waiman Long
2020-01-23 20:39 ` Waiman Long
2020-01-23 20:39 ` Waiman Long
2020-01-23 23:39 ` Alex Kogan
2020-01-23 23:39 ` Alex Kogan
2020-01-15 3:59 ` [PATCH v9 5/5] locking/qspinlock: Introduce the shuffle reduction optimization " Alex Kogan
2020-01-15 3:59 ` Alex Kogan
2020-03-02 1:14 ` [locking/qspinlock] 7b6da71157: unixbench.score 8.4% improvement kernel test robot
2020-03-02 1:14 ` kernel test robot
2020-03-02 1:14 ` kernel test robot
2020-01-22 11:45 ` [PATCH v9 0/5] Add NUMA-awareness to qspinlock Lihao Liang
2020-01-22 11:45 ` Lihao Liang
2020-01-22 17:24 ` Waiman Long
2020-01-22 17:24 ` Waiman Long
2020-01-23 11:35 ` Will Deacon
2020-01-23 11:35 ` Will Deacon
2020-01-23 15:25 ` Waiman Long
2020-01-23 15:25 ` Waiman Long
2020-01-23 19:08 ` Waiman Long
2020-01-23 19:08 ` Waiman Long
2020-01-22 19:29 ` Alex Kogan
2020-01-22 19:29 ` Alex Kogan
2020-01-26 0:32 ` Lihao Liang
2020-01-26 0:32 ` Lihao Liang
2020-01-26 1:58 ` Lihao Liang
2020-01-26 1:58 ` Lihao Liang
2020-01-26 1:58 ` Lihao Liang
2020-01-27 16:01 ` Alex Kogan
2020-01-27 16:01 ` Alex Kogan
2020-01-29 1:39 ` Lihao Liang
2020-01-29 1:39 ` Lihao Liang
2020-01-27 6:16 ` Alex Kogan
2020-01-27 6:16 ` Alex Kogan
2020-01-24 22:24 ` Paul E. McKenney
2020-01-24 22:24 ` Paul E. McKenney
[not found] ` <6AAE7FC6-F5DE-4067-8BC4-77F27948CD09@oracle.com>
2020-01-25 0:57 ` Paul E. McKenney
2020-01-25 0:57 ` Paul E. McKenney
2020-01-25 1:59 ` Waiman Long
2020-01-25 1:59 ` Waiman Long
[not found] ` <adb4fb09-f374-4d64-096b-ba9ad8b35fd5@redhat.com>
2020-01-25 4:58 ` Paul E. McKenney
2020-01-25 4:58 ` Paul E. McKenney
2020-01-25 19:41 ` Waiman Long
2020-01-25 19:41 ` Waiman Long
2020-01-26 15:35 ` Paul E. McKenney
2020-01-26 15:35 ` Paul E. McKenney
2020-01-26 22:42 ` Paul E. McKenney
2020-01-26 22:42 ` Paul E. McKenney
2020-01-26 23:32 ` Paul E. McKenney
2020-01-26 23:32 ` Paul E. McKenney
2020-01-27 6:04 ` Alex Kogan
2020-01-27 6:04 ` Alex Kogan
2020-01-27 14:11 ` Waiman Long
2020-01-27 14:11 ` Waiman Long
2020-01-27 15:09 ` Paul E. McKenney
2020-01-27 15:09 ` Paul E. McKenney
[not found] ` <9b3a3f16-5405-b6d1-d023-b85f4aab46dd@redhat.com>
2020-01-27 17:17 ` Waiman Long
2020-01-27 17:17 ` Waiman Long
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20200115035920.54451-3-alex.kogan@oracle.com \
--to=alex.kogan@oracle.com \
--cc=arnd@arndb.de \
--cc=bp@alien8.de \
--cc=daniel.m.jordan@oracle.com \
--cc=dave.dice@oracle.com \
--cc=guohanjun@huawei.com \
--cc=hpa@zytor.com \
--cc=jglauber@marvell.com \
--cc=linux-arch@vger.kernel.org \
--cc=linux-arm-kernel@lists.infradead.org \
--cc=linux-kernel@vger.kernel.org \
--cc=linux@armlinux.org.uk \
--cc=longman@redhat.com \
--cc=mingo@redhat.com \
--cc=peterz@infradead.org \
--cc=steven.sistare@oracle.com \
--cc=tglx@linutronix.de \
--cc=will.deacon@arm.com \
--cc=x86@kernel.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.