* [PATCH v11 01/16] qspinlock: A simple generic 4-byte queue spinlock
2014-05-30 15:43 [PATCH v11 00/16] qspinlock: a 4-byte queue spinlock with PV support Waiman Long
2014-05-30 15:43 ` Waiman Long
2014-05-30 15:43 ` Waiman Long
@ 2014-05-30 15:43 ` Waiman Long
2014-05-30 15:43 ` Waiman Long
` (34 subsequent siblings)
37 siblings, 0 replies; 102+ messages in thread
From: Waiman Long @ 2014-05-30 15:43 UTC (permalink / raw)
To: Thomas Gleixner, Ingo Molnar, H. Peter Anvin, Peter Zijlstra
Cc: linux-arch, x86, linux-kernel, virtualization, xen-devel, kvm,
Paolo Bonzini, Konrad Rzeszutek Wilk, Boris Ostrovsky,
Paul E. McKenney, Rik van Riel, Linus Torvalds, Raghavendra K T,
David Vrabel, Oleg Nesterov, Gleb Natapov, Scott J Norton,
Chegu Vinod, Waiman Long, Peter Zijlstra
This patch introduces a new generic queue spinlock implementation that
can serve as an alternative to the default ticket spinlock. Compared
with the ticket spinlock, this queue spinlock should be almost as fair
as the ticket spinlock. It has about the same speed in single-thread
and it can be much faster in high contention situations especially when
the spinlock is embedded within the data structure to be protected.
Only in light to moderate contention where the average queue depth
is around 1-3 will this queue spinlock be potentially a bit slower
due to the higher slowpath overhead.
This queue spinlock is especially suit to NUMA machines with a large
number of cores as the chance of spinlock contention is much higher
in those machines. The cost of contention is also higher because of
slower inter-node memory traffic.
Due to the fact that spinlocks are acquired with preemption disabled,
the process will not be migrated to another CPU while it is trying
to get a spinlock. Ignoring interrupt handling, a CPU can only be
contending in one spinlock at any one time. Counting soft IRQ, hard
IRQ and NMI, a CPU can only have a maximum of 4 concurrent lock waiting
activities. By allocating a set of per-cpu queue nodes and used them
to form a waiting queue, we can encode the queue node address into a
much smaller 24-bit size (including CPU number and queue node index)
leaving one byte for the lock.
Please note that the queue node is only needed when waiting for the
lock. Once the lock is acquired, the queue node can be released to
be used later.
Signed-off-by: Waiman Long <Waiman.Long@hp.com>
Signed-off-by: Peter Zijlstra <peterz@infradead.org>
---
include/asm-generic/qspinlock.h | 118 ++++++++++++++++++++
include/asm-generic/qspinlock_types.h | 61 ++++++++++
kernel/Kconfig.locks | 7 +
kernel/locking/Makefile | 1 +
kernel/locking/mcs_spinlock.h | 1 +
kernel/locking/qspinlock.c | 197 +++++++++++++++++++++++++++++++++
6 files changed, 385 insertions(+), 0 deletions(-)
create mode 100644 include/asm-generic/qspinlock.h
create mode 100644 include/asm-generic/qspinlock_types.h
create mode 100644 kernel/locking/qspinlock.c
diff --git a/include/asm-generic/qspinlock.h b/include/asm-generic/qspinlock.h
new file mode 100644
index 0000000..e8a7ae8
--- /dev/null
+++ b/include/asm-generic/qspinlock.h
@@ -0,0 +1,118 @@
+/*
+ * Queue spinlock
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * (C) Copyright 2013-2014 Hewlett-Packard Development Company, L.P.
+ *
+ * Authors: Waiman Long <waiman.long@hp.com>
+ */
+#ifndef __ASM_GENERIC_QSPINLOCK_H
+#define __ASM_GENERIC_QSPINLOCK_H
+
+#include <asm-generic/qspinlock_types.h>
+
+/**
+ * queue_spin_is_locked - is the spinlock locked?
+ * @lock: Pointer to queue spinlock structure
+ * Return: 1 if it is locked, 0 otherwise
+ */
+static __always_inline int queue_spin_is_locked(struct qspinlock *lock)
+{
+ return atomic_read(&lock->val);
+}
+
+/**
+ * queue_spin_value_unlocked - is the spinlock structure unlocked?
+ * @lock: queue spinlock structure
+ * Return: 1 if it is unlocked, 0 otherwise
+ *
+ * N.B. Whenever there are tasks waiting for the lock, it is considered
+ * locked wrt the lockref code to avoid lock stealing by the lockref
+ * code and change things underneath the lock. This also allows some
+ * optimizations to be applied without conflict with lockref.
+ */
+static __always_inline int queue_spin_value_unlocked(struct qspinlock lock)
+{
+ return !atomic_read(&lock.val);
+}
+
+/**
+ * queue_spin_is_contended - check if the lock is contended
+ * @lock : Pointer to queue spinlock structure
+ * Return: 1 if lock contended, 0 otherwise
+ */
+static __always_inline int queue_spin_is_contended(struct qspinlock *lock)
+{
+ return atomic_read(&lock->val) & ~_Q_LOCKED_MASK;
+}
+/**
+ * queue_spin_trylock - try to acquire the queue spinlock
+ * @lock : Pointer to queue spinlock structure
+ * Return: 1 if lock acquired, 0 if failed
+ */
+static __always_inline int queue_spin_trylock(struct qspinlock *lock)
+{
+ if (!atomic_read(&lock->val) &&
+ (atomic_cmpxchg(&lock->val, 0, _Q_LOCKED_VAL) == 0))
+ return 1;
+ return 0;
+}
+
+extern void queue_spin_lock_slowpath(struct qspinlock *lock, u32 val);
+
+/**
+ * queue_spin_lock - acquire a queue spinlock
+ * @lock: Pointer to queue spinlock structure
+ */
+static __always_inline void queue_spin_lock(struct qspinlock *lock)
+{
+ u32 val;
+
+ val = atomic_cmpxchg(&lock->val, 0, _Q_LOCKED_VAL);
+ if (likely(val == 0))
+ return;
+ queue_spin_lock_slowpath(lock, val);
+}
+
+#ifndef queue_spin_unlock
+/**
+ * queue_spin_unlock - release a queue spinlock
+ * @lock : Pointer to queue spinlock structure
+ */
+static __always_inline void queue_spin_unlock(struct qspinlock *lock)
+{
+ /*
+ * smp_mb__before_atomic() in order to guarantee release semantics
+ */
+ smp_mb__before_atomic_dec();
+ atomic_sub(_Q_LOCKED_VAL, &lock->val);
+}
+#endif
+
+/*
+ * Initializier
+ */
+#define __ARCH_SPIN_LOCK_UNLOCKED { ATOMIC_INIT(0) }
+
+/*
+ * Remapping spinlock architecture specific functions to the corresponding
+ * queue spinlock functions.
+ */
+#define arch_spin_is_locked(l) queue_spin_is_locked(l)
+#define arch_spin_is_contended(l) queue_spin_is_contended(l)
+#define arch_spin_value_unlocked(l) queue_spin_value_unlocked(l)
+#define arch_spin_lock(l) queue_spin_lock(l)
+#define arch_spin_trylock(l) queue_spin_trylock(l)
+#define arch_spin_unlock(l) queue_spin_unlock(l)
+#define arch_spin_lock_flags(l, f) queue_spin_lock(l)
+
+#endif /* __ASM_GENERIC_QSPINLOCK_H */
diff --git a/include/asm-generic/qspinlock_types.h b/include/asm-generic/qspinlock_types.h
new file mode 100644
index 0000000..f66f845
--- /dev/null
+++ b/include/asm-generic/qspinlock_types.h
@@ -0,0 +1,61 @@
+/*
+ * Queue spinlock
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * (C) Copyright 2013-2014 Hewlett-Packard Development Company, L.P.
+ *
+ * Authors: Waiman Long <waiman.long@hp.com>
+ */
+#ifndef __ASM_GENERIC_QSPINLOCK_TYPES_H
+#define __ASM_GENERIC_QSPINLOCK_TYPES_H
+
+/*
+ * Including atomic.h with PARAVIRT on will cause compilation errors because
+ * of recursive header file incluson via paravirt_types.h. A workaround is
+ * to include paravirt_types.h here.
+ */
+#ifdef CONFIG_PARAVIRT
+#include <asm/paravirt_types.h>
+#else
+#include <linux/types.h>
+#include <linux/atomic.h>
+#include <linux/bug.h>
+#endif
+
+typedef struct qspinlock {
+ atomic_t val;
+} arch_spinlock_t;
+
+/*
+ * Bitfields in the atomic value:
+ *
+ * 0- 7: locked byte
+ * 8- 9: tail index
+ * 10-31: tail cpu (+1)
+ */
+#define _Q_SET_MASK(type) (((1U << _Q_ ## type ## _BITS) - 1)\
+ << _Q_ ## type ## _OFFSET)
+#define _Q_LOCKED_OFFSET 0
+#define _Q_LOCKED_BITS 8
+#define _Q_LOCKED_MASK _Q_SET_MASK(LOCKED)
+
+#define _Q_TAIL_IDX_OFFSET (_Q_LOCKED_OFFSET + _Q_LOCKED_BITS)
+#define _Q_TAIL_IDX_BITS 2
+#define _Q_TAIL_IDX_MASK _Q_SET_MASK(TAIL_IDX)
+
+#define _Q_TAIL_CPU_OFFSET (_Q_TAIL_IDX_OFFSET + _Q_TAIL_IDX_BITS)
+#define _Q_TAIL_CPU_BITS (32 - _Q_TAIL_CPU_OFFSET)
+#define _Q_TAIL_CPU_MASK _Q_SET_MASK(TAIL_CPU)
+
+#define _Q_LOCKED_VAL (1U << _Q_LOCKED_OFFSET)
+
+#endif /* __ASM_GENERIC_QSPINLOCK_TYPES_H */
diff --git a/kernel/Kconfig.locks b/kernel/Kconfig.locks
index d2b32ac..f185584 100644
--- a/kernel/Kconfig.locks
+++ b/kernel/Kconfig.locks
@@ -223,3 +223,10 @@ endif
config MUTEX_SPIN_ON_OWNER
def_bool y
depends on SMP && !DEBUG_MUTEXES
+
+config ARCH_USE_QUEUE_SPINLOCK
+ bool
+
+config QUEUE_SPINLOCK
+ def_bool y if ARCH_USE_QUEUE_SPINLOCK
+ depends on SMP && !PARAVIRT_SPINLOCKS
diff --git a/kernel/locking/Makefile b/kernel/locking/Makefile
index b8bdcd4..e6741ac 100644
--- a/kernel/locking/Makefile
+++ b/kernel/locking/Makefile
@@ -16,6 +16,7 @@ endif
obj-$(CONFIG_SMP) += spinlock.o
obj-$(CONFIG_SMP) += lglock.o
obj-$(CONFIG_PROVE_LOCKING) += spinlock.o
+obj-$(CONFIG_QUEUE_SPINLOCK) += qspinlock.o
obj-$(CONFIG_RT_MUTEXES) += rtmutex.o
obj-$(CONFIG_DEBUG_RT_MUTEXES) += rtmutex-debug.o
obj-$(CONFIG_RT_MUTEX_TESTER) += rtmutex-tester.o
diff --git a/kernel/locking/mcs_spinlock.h b/kernel/locking/mcs_spinlock.h
index a2dbac4..a59b677 100644
--- a/kernel/locking/mcs_spinlock.h
+++ b/kernel/locking/mcs_spinlock.h
@@ -17,6 +17,7 @@
struct mcs_spinlock {
struct mcs_spinlock *next;
int locked; /* 1 if lock acquired */
+ int count;
};
#ifndef arch_mcs_spin_lock_contended
diff --git a/kernel/locking/qspinlock.c b/kernel/locking/qspinlock.c
new file mode 100644
index 0000000..b97a1ad
--- /dev/null
+++ b/kernel/locking/qspinlock.c
@@ -0,0 +1,197 @@
+/*
+ * Queue spinlock
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * (C) Copyright 2013-2014 Hewlett-Packard Development Company, L.P.
+ *
+ * Authors: Waiman Long <waiman.long@hp.com>
+ * Peter Zijlstra <pzijlstr@redhat.com>
+ */
+#include <linux/smp.h>
+#include <linux/bug.h>
+#include <linux/cpumask.h>
+#include <linux/percpu.h>
+#include <linux/hardirq.h>
+#include <linux/mutex.h>
+#include <asm/qspinlock.h>
+
+/*
+ * The basic principle of a queue-based spinlock can best be understood
+ * by studying a classic queue-based spinlock implementation called the
+ * MCS lock. The paper below provides a good description for this kind
+ * of lock.
+ *
+ * http://www.cise.ufl.edu/tr/DOC/REP-1992-71.pdf
+ *
+ * This queue spinlock implementation is based on the MCS lock, however to make
+ * it fit the 4 bytes we assume spinlock_t to be, and preserve its existing
+ * API, we must modify it some.
+ *
+ * In particular; where the traditional MCS lock consists of a tail pointer
+ * (8 bytes) and needs the next pointer (another 8 bytes) of its own node to
+ * unlock the next pending (next->locked), we compress both these: {tail,
+ * next->locked} into a single u32 value.
+ *
+ * Since a spinlock disables recursion of its own context and there is a limit
+ * to the contexts that can nest; namely: task, softirq, hardirq, nmi, we can
+ * encode the tail as and index indicating this context and a cpu number.
+ *
+ * We can further change the first spinner to spin on a bit in the lock word
+ * instead of its node; whereby avoiding the need to carry a node from lock to
+ * unlock, and preserving API.
+ */
+
+#include "mcs_spinlock.h"
+
+/*
+ * Per-CPU queue node structures; we can never have more than 4 nested
+ * contexts: task, softirq, hardirq, nmi.
+ *
+ * Exactly fits one cacheline.
+ */
+static DEFINE_PER_CPU_ALIGNED(struct mcs_spinlock, mcs_nodes[4]);
+
+/*
+ * We must be able to distinguish between no-tail and the tail at 0:0,
+ * therefore increment the cpu number by one.
+ */
+
+static inline u32 encode_tail(int cpu, int idx)
+{
+ u32 tail;
+
+ tail = (cpu + 1) << _Q_TAIL_CPU_OFFSET;
+ tail |= idx << _Q_TAIL_IDX_OFFSET; /* assume < 4 */
+
+ return tail;
+}
+
+static inline struct mcs_spinlock *decode_tail(u32 tail)
+{
+ int cpu = (tail >> _Q_TAIL_CPU_OFFSET) - 1;
+ int idx = (tail & _Q_TAIL_IDX_MASK) >> _Q_TAIL_IDX_OFFSET;
+
+ return per_cpu_ptr(&mcs_nodes[idx], cpu);
+}
+
+/**
+ * queue_spin_lock_slowpath - acquire the queue spinlock
+ * @lock: Pointer to queue spinlock structure
+ * @val: Current value of the queue spinlock 32-bit word
+ *
+ * (queue tail, lock bit)
+ *
+ * fast : slow : unlock
+ * : :
+ * uncontended (0,0) --:--> (0,1) --------------------------------:--> (*,0)
+ * : | ^--------. / :
+ * : v \ | :
+ * uncontended : (n,x) --+--> (n,0) | :
+ * queue : | ^--' | :
+ * : v | :
+ * contended : (*,x) --+--> (*,0) -----> (*,1) ---' :
+ * queue : ^--' :
+ *
+ */
+void queue_spin_lock_slowpath(struct qspinlock *lock, u32 val)
+{
+ struct mcs_spinlock *prev, *next, *node;
+ u32 new, old, tail;
+ int idx;
+
+ BUILD_BUG_ON(CONFIG_NR_CPUS >= (1U << _Q_TAIL_CPU_BITS));
+
+ node = this_cpu_ptr(&mcs_nodes[0]);
+ idx = node->count++;
+ tail = encode_tail(smp_processor_id(), idx);
+
+ node += idx;
+ node->locked = 0;
+ node->next = NULL;
+
+ /*
+ * trylock || xchg(lock, node)
+ *
+ * 0,0 -> 0,1 ; trylock
+ * p,x -> n,x ; prev = xchg(lock, node)
+ */
+ for (;;) {
+ new = _Q_LOCKED_VAL;
+ if (val)
+ new = tail | (val & _Q_LOCKED_MASK);
+
+ old = atomic_cmpxchg(&lock->val, val, new);
+ if (old == val)
+ break;
+
+ val = old;
+ }
+
+ /*
+ * we won the trylock; forget about queueing.
+ */
+ if (new == _Q_LOCKED_VAL)
+ goto release;
+
+ /*
+ * if there was a previous node; link it and wait.
+ */
+ if (old & ~_Q_LOCKED_MASK) {
+ prev = decode_tail(old);
+ ACCESS_ONCE(prev->next) = node;
+
+ arch_mcs_spin_lock_contended(&node->locked);
+ }
+
+ /*
+ * we're at the head of the waitqueue, wait for the owner to go away.
+ *
+ * *,x -> *,0
+ */
+ while ((val = atomic_read(&lock->val)) & _Q_LOCKED_MASK)
+ arch_mutex_cpu_relax();
+
+ /*
+ * claim the lock:
+ *
+ * n,0 -> 0,1 : lock, uncontended
+ * *,0 -> *,1 : lock, contended
+ */
+ for (;;) {
+ new = _Q_LOCKED_VAL;
+ if (val != tail)
+ new |= val;
+
+ old = atomic_cmpxchg(&lock->val, val, new);
+ if (old == val)
+ break;
+
+ val = old;
+ }
+
+ /*
+ * contended path; wait for next, release.
+ */
+ if (new != _Q_LOCKED_VAL) {
+ while (!(next = ACCESS_ONCE(node->next)))
+ arch_mutex_cpu_relax();
+
+ arch_mcs_spin_unlock_contended(&next->locked);
+ }
+
+release:
+ /*
+ * release the node
+ */
+ this_cpu_dec(mcs_nodes[0].count);
+}
+EXPORT_SYMBOL(queue_spin_lock_slowpath);
--
1.7.1
^ permalink raw reply related [flat|nested] 102+ messages in thread* [PATCH v11 01/16] qspinlock: A simple generic 4-byte queue spinlock
@ 2014-05-30 15:43 ` Waiman Long
0 siblings, 0 replies; 102+ messages in thread
From: Waiman Long @ 2014-05-30 15:43 UTC (permalink / raw)
To: Thomas Gleixner, Ingo Molnar, H. Peter Anvin, Peter Zijlstra
Cc: linux-arch, Waiman Long, Rik van Riel, Raghavendra K T,
Gleb Natapov, kvm, Konrad Rzeszutek Wilk, Peter Zijlstra,
Scott J Norton, x86, Paolo Bonzini, linux-kernel, virtualization,
Chegu Vinod, David Vrabel, Oleg Nesterov, xen-devel,
Boris Ostrovsky, Paul E. McKenney, Linus Torvalds
This patch introduces a new generic queue spinlock implementation that
can serve as an alternative to the default ticket spinlock. Compared
with the ticket spinlock, this queue spinlock should be almost as fair
as the ticket spinlock. It has about the same speed in single-thread
and it can be much faster in high contention situations especially when
the spinlock is embedded within the data structure to be protected.
Only in light to moderate contention where the average queue depth
is around 1-3 will this queue spinlock be potentially a bit slower
due to the higher slowpath overhead.
This queue spinlock is especially suit to NUMA machines with a large
number of cores as the chance of spinlock contention is much higher
in those machines. The cost of contention is also higher because of
slower inter-node memory traffic.
Due to the fact that spinlocks are acquired with preemption disabled,
the process will not be migrated to another CPU while it is trying
to get a spinlock. Ignoring interrupt handling, a CPU can only be
contending in one spinlock at any one time. Counting soft IRQ, hard
IRQ and NMI, a CPU can only have a maximum of 4 concurrent lock waiting
activities. By allocating a set of per-cpu queue nodes and used them
to form a waiting queue, we can encode the queue node address into a
much smaller 24-bit size (including CPU number and queue node index)
leaving one byte for the lock.
Please note that the queue node is only needed when waiting for the
lock. Once the lock is acquired, the queue node can be released to
be used later.
Signed-off-by: Waiman Long <Waiman.Long@hp.com>
Signed-off-by: Peter Zijlstra <peterz@infradead.org>
---
include/asm-generic/qspinlock.h | 118 ++++++++++++++++++++
include/asm-generic/qspinlock_types.h | 61 ++++++++++
kernel/Kconfig.locks | 7 +
kernel/locking/Makefile | 1 +
kernel/locking/mcs_spinlock.h | 1 +
kernel/locking/qspinlock.c | 197 +++++++++++++++++++++++++++++++++
6 files changed, 385 insertions(+), 0 deletions(-)
create mode 100644 include/asm-generic/qspinlock.h
create mode 100644 include/asm-generic/qspinlock_types.h
create mode 100644 kernel/locking/qspinlock.c
diff --git a/include/asm-generic/qspinlock.h b/include/asm-generic/qspinlock.h
new file mode 100644
index 0000000..e8a7ae8
--- /dev/null
+++ b/include/asm-generic/qspinlock.h
@@ -0,0 +1,118 @@
+/*
+ * Queue spinlock
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * (C) Copyright 2013-2014 Hewlett-Packard Development Company, L.P.
+ *
+ * Authors: Waiman Long <waiman.long@hp.com>
+ */
+#ifndef __ASM_GENERIC_QSPINLOCK_H
+#define __ASM_GENERIC_QSPINLOCK_H
+
+#include <asm-generic/qspinlock_types.h>
+
+/**
+ * queue_spin_is_locked - is the spinlock locked?
+ * @lock: Pointer to queue spinlock structure
+ * Return: 1 if it is locked, 0 otherwise
+ */
+static __always_inline int queue_spin_is_locked(struct qspinlock *lock)
+{
+ return atomic_read(&lock->val);
+}
+
+/**
+ * queue_spin_value_unlocked - is the spinlock structure unlocked?
+ * @lock: queue spinlock structure
+ * Return: 1 if it is unlocked, 0 otherwise
+ *
+ * N.B. Whenever there are tasks waiting for the lock, it is considered
+ * locked wrt the lockref code to avoid lock stealing by the lockref
+ * code and change things underneath the lock. This also allows some
+ * optimizations to be applied without conflict with lockref.
+ */
+static __always_inline int queue_spin_value_unlocked(struct qspinlock lock)
+{
+ return !atomic_read(&lock.val);
+}
+
+/**
+ * queue_spin_is_contended - check if the lock is contended
+ * @lock : Pointer to queue spinlock structure
+ * Return: 1 if lock contended, 0 otherwise
+ */
+static __always_inline int queue_spin_is_contended(struct qspinlock *lock)
+{
+ return atomic_read(&lock->val) & ~_Q_LOCKED_MASK;
+}
+/**
+ * queue_spin_trylock - try to acquire the queue spinlock
+ * @lock : Pointer to queue spinlock structure
+ * Return: 1 if lock acquired, 0 if failed
+ */
+static __always_inline int queue_spin_trylock(struct qspinlock *lock)
+{
+ if (!atomic_read(&lock->val) &&
+ (atomic_cmpxchg(&lock->val, 0, _Q_LOCKED_VAL) == 0))
+ return 1;
+ return 0;
+}
+
+extern void queue_spin_lock_slowpath(struct qspinlock *lock, u32 val);
+
+/**
+ * queue_spin_lock - acquire a queue spinlock
+ * @lock: Pointer to queue spinlock structure
+ */
+static __always_inline void queue_spin_lock(struct qspinlock *lock)
+{
+ u32 val;
+
+ val = atomic_cmpxchg(&lock->val, 0, _Q_LOCKED_VAL);
+ if (likely(val == 0))
+ return;
+ queue_spin_lock_slowpath(lock, val);
+}
+
+#ifndef queue_spin_unlock
+/**
+ * queue_spin_unlock - release a queue spinlock
+ * @lock : Pointer to queue spinlock structure
+ */
+static __always_inline void queue_spin_unlock(struct qspinlock *lock)
+{
+ /*
+ * smp_mb__before_atomic() in order to guarantee release semantics
+ */
+ smp_mb__before_atomic_dec();
+ atomic_sub(_Q_LOCKED_VAL, &lock->val);
+}
+#endif
+
+/*
+ * Initializier
+ */
+#define __ARCH_SPIN_LOCK_UNLOCKED { ATOMIC_INIT(0) }
+
+/*
+ * Remapping spinlock architecture specific functions to the corresponding
+ * queue spinlock functions.
+ */
+#define arch_spin_is_locked(l) queue_spin_is_locked(l)
+#define arch_spin_is_contended(l) queue_spin_is_contended(l)
+#define arch_spin_value_unlocked(l) queue_spin_value_unlocked(l)
+#define arch_spin_lock(l) queue_spin_lock(l)
+#define arch_spin_trylock(l) queue_spin_trylock(l)
+#define arch_spin_unlock(l) queue_spin_unlock(l)
+#define arch_spin_lock_flags(l, f) queue_spin_lock(l)
+
+#endif /* __ASM_GENERIC_QSPINLOCK_H */
diff --git a/include/asm-generic/qspinlock_types.h b/include/asm-generic/qspinlock_types.h
new file mode 100644
index 0000000..f66f845
--- /dev/null
+++ b/include/asm-generic/qspinlock_types.h
@@ -0,0 +1,61 @@
+/*
+ * Queue spinlock
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * (C) Copyright 2013-2014 Hewlett-Packard Development Company, L.P.
+ *
+ * Authors: Waiman Long <waiman.long@hp.com>
+ */
+#ifndef __ASM_GENERIC_QSPINLOCK_TYPES_H
+#define __ASM_GENERIC_QSPINLOCK_TYPES_H
+
+/*
+ * Including atomic.h with PARAVIRT on will cause compilation errors because
+ * of recursive header file incluson via paravirt_types.h. A workaround is
+ * to include paravirt_types.h here.
+ */
+#ifdef CONFIG_PARAVIRT
+#include <asm/paravirt_types.h>
+#else
+#include <linux/types.h>
+#include <linux/atomic.h>
+#include <linux/bug.h>
+#endif
+
+typedef struct qspinlock {
+ atomic_t val;
+} arch_spinlock_t;
+
+/*
+ * Bitfields in the atomic value:
+ *
+ * 0- 7: locked byte
+ * 8- 9: tail index
+ * 10-31: tail cpu (+1)
+ */
+#define _Q_SET_MASK(type) (((1U << _Q_ ## type ## _BITS) - 1)\
+ << _Q_ ## type ## _OFFSET)
+#define _Q_LOCKED_OFFSET 0
+#define _Q_LOCKED_BITS 8
+#define _Q_LOCKED_MASK _Q_SET_MASK(LOCKED)
+
+#define _Q_TAIL_IDX_OFFSET (_Q_LOCKED_OFFSET + _Q_LOCKED_BITS)
+#define _Q_TAIL_IDX_BITS 2
+#define _Q_TAIL_IDX_MASK _Q_SET_MASK(TAIL_IDX)
+
+#define _Q_TAIL_CPU_OFFSET (_Q_TAIL_IDX_OFFSET + _Q_TAIL_IDX_BITS)
+#define _Q_TAIL_CPU_BITS (32 - _Q_TAIL_CPU_OFFSET)
+#define _Q_TAIL_CPU_MASK _Q_SET_MASK(TAIL_CPU)
+
+#define _Q_LOCKED_VAL (1U << _Q_LOCKED_OFFSET)
+
+#endif /* __ASM_GENERIC_QSPINLOCK_TYPES_H */
diff --git a/kernel/Kconfig.locks b/kernel/Kconfig.locks
index d2b32ac..f185584 100644
--- a/kernel/Kconfig.locks
+++ b/kernel/Kconfig.locks
@@ -223,3 +223,10 @@ endif
config MUTEX_SPIN_ON_OWNER
def_bool y
depends on SMP && !DEBUG_MUTEXES
+
+config ARCH_USE_QUEUE_SPINLOCK
+ bool
+
+config QUEUE_SPINLOCK
+ def_bool y if ARCH_USE_QUEUE_SPINLOCK
+ depends on SMP && !PARAVIRT_SPINLOCKS
diff --git a/kernel/locking/Makefile b/kernel/locking/Makefile
index b8bdcd4..e6741ac 100644
--- a/kernel/locking/Makefile
+++ b/kernel/locking/Makefile
@@ -16,6 +16,7 @@ endif
obj-$(CONFIG_SMP) += spinlock.o
obj-$(CONFIG_SMP) += lglock.o
obj-$(CONFIG_PROVE_LOCKING) += spinlock.o
+obj-$(CONFIG_QUEUE_SPINLOCK) += qspinlock.o
obj-$(CONFIG_RT_MUTEXES) += rtmutex.o
obj-$(CONFIG_DEBUG_RT_MUTEXES) += rtmutex-debug.o
obj-$(CONFIG_RT_MUTEX_TESTER) += rtmutex-tester.o
diff --git a/kernel/locking/mcs_spinlock.h b/kernel/locking/mcs_spinlock.h
index a2dbac4..a59b677 100644
--- a/kernel/locking/mcs_spinlock.h
+++ b/kernel/locking/mcs_spinlock.h
@@ -17,6 +17,7 @@
struct mcs_spinlock {
struct mcs_spinlock *next;
int locked; /* 1 if lock acquired */
+ int count;
};
#ifndef arch_mcs_spin_lock_contended
diff --git a/kernel/locking/qspinlock.c b/kernel/locking/qspinlock.c
new file mode 100644
index 0000000..b97a1ad
--- /dev/null
+++ b/kernel/locking/qspinlock.c
@@ -0,0 +1,197 @@
+/*
+ * Queue spinlock
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * (C) Copyright 2013-2014 Hewlett-Packard Development Company, L.P.
+ *
+ * Authors: Waiman Long <waiman.long@hp.com>
+ * Peter Zijlstra <pzijlstr@redhat.com>
+ */
+#include <linux/smp.h>
+#include <linux/bug.h>
+#include <linux/cpumask.h>
+#include <linux/percpu.h>
+#include <linux/hardirq.h>
+#include <linux/mutex.h>
+#include <asm/qspinlock.h>
+
+/*
+ * The basic principle of a queue-based spinlock can best be understood
+ * by studying a classic queue-based spinlock implementation called the
+ * MCS lock. The paper below provides a good description for this kind
+ * of lock.
+ *
+ * http://www.cise.ufl.edu/tr/DOC/REP-1992-71.pdf
+ *
+ * This queue spinlock implementation is based on the MCS lock, however to make
+ * it fit the 4 bytes we assume spinlock_t to be, and preserve its existing
+ * API, we must modify it some.
+ *
+ * In particular; where the traditional MCS lock consists of a tail pointer
+ * (8 bytes) and needs the next pointer (another 8 bytes) of its own node to
+ * unlock the next pending (next->locked), we compress both these: {tail,
+ * next->locked} into a single u32 value.
+ *
+ * Since a spinlock disables recursion of its own context and there is a limit
+ * to the contexts that can nest; namely: task, softirq, hardirq, nmi, we can
+ * encode the tail as and index indicating this context and a cpu number.
+ *
+ * We can further change the first spinner to spin on a bit in the lock word
+ * instead of its node; whereby avoiding the need to carry a node from lock to
+ * unlock, and preserving API.
+ */
+
+#include "mcs_spinlock.h"
+
+/*
+ * Per-CPU queue node structures; we can never have more than 4 nested
+ * contexts: task, softirq, hardirq, nmi.
+ *
+ * Exactly fits one cacheline.
+ */
+static DEFINE_PER_CPU_ALIGNED(struct mcs_spinlock, mcs_nodes[4]);
+
+/*
+ * We must be able to distinguish between no-tail and the tail at 0:0,
+ * therefore increment the cpu number by one.
+ */
+
+static inline u32 encode_tail(int cpu, int idx)
+{
+ u32 tail;
+
+ tail = (cpu + 1) << _Q_TAIL_CPU_OFFSET;
+ tail |= idx << _Q_TAIL_IDX_OFFSET; /* assume < 4 */
+
+ return tail;
+}
+
+static inline struct mcs_spinlock *decode_tail(u32 tail)
+{
+ int cpu = (tail >> _Q_TAIL_CPU_OFFSET) - 1;
+ int idx = (tail & _Q_TAIL_IDX_MASK) >> _Q_TAIL_IDX_OFFSET;
+
+ return per_cpu_ptr(&mcs_nodes[idx], cpu);
+}
+
+/**
+ * queue_spin_lock_slowpath - acquire the queue spinlock
+ * @lock: Pointer to queue spinlock structure
+ * @val: Current value of the queue spinlock 32-bit word
+ *
+ * (queue tail, lock bit)
+ *
+ * fast : slow : unlock
+ * : :
+ * uncontended (0,0) --:--> (0,1) --------------------------------:--> (*,0)
+ * : | ^--------. / :
+ * : v \ | :
+ * uncontended : (n,x) --+--> (n,0) | :
+ * queue : | ^--' | :
+ * : v | :
+ * contended : (*,x) --+--> (*,0) -----> (*,1) ---' :
+ * queue : ^--' :
+ *
+ */
+void queue_spin_lock_slowpath(struct qspinlock *lock, u32 val)
+{
+ struct mcs_spinlock *prev, *next, *node;
+ u32 new, old, tail;
+ int idx;
+
+ BUILD_BUG_ON(CONFIG_NR_CPUS >= (1U << _Q_TAIL_CPU_BITS));
+
+ node = this_cpu_ptr(&mcs_nodes[0]);
+ idx = node->count++;
+ tail = encode_tail(smp_processor_id(), idx);
+
+ node += idx;
+ node->locked = 0;
+ node->next = NULL;
+
+ /*
+ * trylock || xchg(lock, node)
+ *
+ * 0,0 -> 0,1 ; trylock
+ * p,x -> n,x ; prev = xchg(lock, node)
+ */
+ for (;;) {
+ new = _Q_LOCKED_VAL;
+ if (val)
+ new = tail | (val & _Q_LOCKED_MASK);
+
+ old = atomic_cmpxchg(&lock->val, val, new);
+ if (old == val)
+ break;
+
+ val = old;
+ }
+
+ /*
+ * we won the trylock; forget about queueing.
+ */
+ if (new == _Q_LOCKED_VAL)
+ goto release;
+
+ /*
+ * if there was a previous node; link it and wait.
+ */
+ if (old & ~_Q_LOCKED_MASK) {
+ prev = decode_tail(old);
+ ACCESS_ONCE(prev->next) = node;
+
+ arch_mcs_spin_lock_contended(&node->locked);
+ }
+
+ /*
+ * we're at the head of the waitqueue, wait for the owner to go away.
+ *
+ * *,x -> *,0
+ */
+ while ((val = atomic_read(&lock->val)) & _Q_LOCKED_MASK)
+ arch_mutex_cpu_relax();
+
+ /*
+ * claim the lock:
+ *
+ * n,0 -> 0,1 : lock, uncontended
+ * *,0 -> *,1 : lock, contended
+ */
+ for (;;) {
+ new = _Q_LOCKED_VAL;
+ if (val != tail)
+ new |= val;
+
+ old = atomic_cmpxchg(&lock->val, val, new);
+ if (old == val)
+ break;
+
+ val = old;
+ }
+
+ /*
+ * contended path; wait for next, release.
+ */
+ if (new != _Q_LOCKED_VAL) {
+ while (!(next = ACCESS_ONCE(node->next)))
+ arch_mutex_cpu_relax();
+
+ arch_mcs_spin_unlock_contended(&next->locked);
+ }
+
+release:
+ /*
+ * release the node
+ */
+ this_cpu_dec(mcs_nodes[0].count);
+}
+EXPORT_SYMBOL(queue_spin_lock_slowpath);
--
1.7.1
^ permalink raw reply related [flat|nested] 102+ messages in thread* [PATCH v11 01/16] qspinlock: A simple generic 4-byte queue spinlock
@ 2014-05-30 15:43 ` Waiman Long
0 siblings, 0 replies; 102+ messages in thread
From: Waiman Long @ 2014-05-30 15:43 UTC (permalink / raw)
To: Thomas Gleixner, Ingo Molnar, H. Peter Anvin, Peter Zijlstra
Cc: linux-arch, x86, linux-kernel, virtualization, xen-devel, kvm,
Paolo Bonzini, Konrad Rzeszutek Wilk, Boris Ostrovsky,
Paul E. McKenney, Rik van Riel, Linus Torvalds, Raghavendra K T,
David Vrabel, Oleg Nesterov, Gleb Natapov, Scott J Norton,
Chegu Vinod, Waiman Long
This patch introduces a new generic queue spinlock implementation that
can serve as an alternative to the default ticket spinlock. Compared
with the ticket spinlock, this queue spinlock should be almost as fair
as the ticket spinlock. It has about the same speed in single-thread
and it can be much faster in high contention situations especially when
the spinlock is embedded within the data structure to be protected.
Only in light to moderate contention where the average queue depth
is around 1-3 will this queue spinlock be potentially a bit slower
due to the higher slowpath overhead.
This queue spinlock is especially suit to NUMA machines with a large
number of cores as the chance of spinlock contention is much higher
in those machines. The cost of contention is also higher because of
slower inter-node memory traffic.
Due to the fact that spinlocks are acquired with preemption disabled,
the process will not be migrated to another CPU while it is trying
to get a spinlock. Ignoring interrupt handling, a CPU can only be
contending in one spinlock at any one time. Counting soft IRQ, hard
IRQ and NMI, a CPU can only have a maximum of 4 concurrent lock waiting
activities. By allocating a set of per-cpu queue nodes and used them
to form a waiting queue, we can encode the queue node address into a
much smaller 24-bit size (including CPU number and queue node index)
leaving one byte for the lock.
Please note that the queue node is only needed when waiting for the
lock. Once the lock is acquired, the queue node can be released to
be used later.
Signed-off-by: Waiman Long <Waiman.Long@hp.com>
Signed-off-by: Peter Zijlstra <peterz@infradead.org>
---
include/asm-generic/qspinlock.h | 118 ++++++++++++++++++++
include/asm-generic/qspinlock_types.h | 61 ++++++++++
kernel/Kconfig.locks | 7 +
kernel/locking/Makefile | 1 +
kernel/locking/mcs_spinlock.h | 1 +
kernel/locking/qspinlock.c | 197 +++++++++++++++++++++++++++++++++
6 files changed, 385 insertions(+), 0 deletions(-)
create mode 100644 include/asm-generic/qspinlock.h
create mode 100644 include/asm-generic/qspinlock_types.h
create mode 100644 kernel/locking/qspinlock.c
diff --git a/include/asm-generic/qspinlock.h b/include/asm-generic/qspinlock.h
new file mode 100644
index 0000000..e8a7ae8
--- /dev/null
+++ b/include/asm-generic/qspinlock.h
@@ -0,0 +1,118 @@
+/*
+ * Queue spinlock
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * (C) Copyright 2013-2014 Hewlett-Packard Development Company, L.P.
+ *
+ * Authors: Waiman Long <waiman.long@hp.com>
+ */
+#ifndef __ASM_GENERIC_QSPINLOCK_H
+#define __ASM_GENERIC_QSPINLOCK_H
+
+#include <asm-generic/qspinlock_types.h>
+
+/**
+ * queue_spin_is_locked - is the spinlock locked?
+ * @lock: Pointer to queue spinlock structure
+ * Return: 1 if it is locked, 0 otherwise
+ */
+static __always_inline int queue_spin_is_locked(struct qspinlock *lock)
+{
+ return atomic_read(&lock->val);
+}
+
+/**
+ * queue_spin_value_unlocked - is the spinlock structure unlocked?
+ * @lock: queue spinlock structure
+ * Return: 1 if it is unlocked, 0 otherwise
+ *
+ * N.B. Whenever there are tasks waiting for the lock, it is considered
+ * locked wrt the lockref code to avoid lock stealing by the lockref
+ * code and change things underneath the lock. This also allows some
+ * optimizations to be applied without conflict with lockref.
+ */
+static __always_inline int queue_spin_value_unlocked(struct qspinlock lock)
+{
+ return !atomic_read(&lock.val);
+}
+
+/**
+ * queue_spin_is_contended - check if the lock is contended
+ * @lock : Pointer to queue spinlock structure
+ * Return: 1 if lock contended, 0 otherwise
+ */
+static __always_inline int queue_spin_is_contended(struct qspinlock *lock)
+{
+ return atomic_read(&lock->val) & ~_Q_LOCKED_MASK;
+}
+/**
+ * queue_spin_trylock - try to acquire the queue spinlock
+ * @lock : Pointer to queue spinlock structure
+ * Return: 1 if lock acquired, 0 if failed
+ */
+static __always_inline int queue_spin_trylock(struct qspinlock *lock)
+{
+ if (!atomic_read(&lock->val) &&
+ (atomic_cmpxchg(&lock->val, 0, _Q_LOCKED_VAL) == 0))
+ return 1;
+ return 0;
+}
+
+extern void queue_spin_lock_slowpath(struct qspinlock *lock, u32 val);
+
+/**
+ * queue_spin_lock - acquire a queue spinlock
+ * @lock: Pointer to queue spinlock structure
+ */
+static __always_inline void queue_spin_lock(struct qspinlock *lock)
+{
+ u32 val;
+
+ val = atomic_cmpxchg(&lock->val, 0, _Q_LOCKED_VAL);
+ if (likely(val == 0))
+ return;
+ queue_spin_lock_slowpath(lock, val);
+}
+
+#ifndef queue_spin_unlock
+/**
+ * queue_spin_unlock - release a queue spinlock
+ * @lock : Pointer to queue spinlock structure
+ */
+static __always_inline void queue_spin_unlock(struct qspinlock *lock)
+{
+ /*
+ * smp_mb__before_atomic() in order to guarantee release semantics
+ */
+ smp_mb__before_atomic_dec();
+ atomic_sub(_Q_LOCKED_VAL, &lock->val);
+}
+#endif
+
+/*
+ * Initializier
+ */
+#define __ARCH_SPIN_LOCK_UNLOCKED { ATOMIC_INIT(0) }
+
+/*
+ * Remapping spinlock architecture specific functions to the corresponding
+ * queue spinlock functions.
+ */
+#define arch_spin_is_locked(l) queue_spin_is_locked(l)
+#define arch_spin_is_contended(l) queue_spin_is_contended(l)
+#define arch_spin_value_unlocked(l) queue_spin_value_unlocked(l)
+#define arch_spin_lock(l) queue_spin_lock(l)
+#define arch_spin_trylock(l) queue_spin_trylock(l)
+#define arch_spin_unlock(l) queue_spin_unlock(l)
+#define arch_spin_lock_flags(l, f) queue_spin_lock(l)
+
+#endif /* __ASM_GENERIC_QSPINLOCK_H */
diff --git a/include/asm-generic/qspinlock_types.h b/include/asm-generic/qspinlock_types.h
new file mode 100644
index 0000000..f66f845
--- /dev/null
+++ b/include/asm-generic/qspinlock_types.h
@@ -0,0 +1,61 @@
+/*
+ * Queue spinlock
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * (C) Copyright 2013-2014 Hewlett-Packard Development Company, L.P.
+ *
+ * Authors: Waiman Long <waiman.long@hp.com>
+ */
+#ifndef __ASM_GENERIC_QSPINLOCK_TYPES_H
+#define __ASM_GENERIC_QSPINLOCK_TYPES_H
+
+/*
+ * Including atomic.h with PARAVIRT on will cause compilation errors because
+ * of recursive header file incluson via paravirt_types.h. A workaround is
+ * to include paravirt_types.h here.
+ */
+#ifdef CONFIG_PARAVIRT
+#include <asm/paravirt_types.h>
+#else
+#include <linux/types.h>
+#include <linux/atomic.h>
+#include <linux/bug.h>
+#endif
+
+typedef struct qspinlock {
+ atomic_t val;
+} arch_spinlock_t;
+
+/*
+ * Bitfields in the atomic value:
+ *
+ * 0- 7: locked byte
+ * 8- 9: tail index
+ * 10-31: tail cpu (+1)
+ */
+#define _Q_SET_MASK(type) (((1U << _Q_ ## type ## _BITS) - 1)\
+ << _Q_ ## type ## _OFFSET)
+#define _Q_LOCKED_OFFSET 0
+#define _Q_LOCKED_BITS 8
+#define _Q_LOCKED_MASK _Q_SET_MASK(LOCKED)
+
+#define _Q_TAIL_IDX_OFFSET (_Q_LOCKED_OFFSET + _Q_LOCKED_BITS)
+#define _Q_TAIL_IDX_BITS 2
+#define _Q_TAIL_IDX_MASK _Q_SET_MASK(TAIL_IDX)
+
+#define _Q_TAIL_CPU_OFFSET (_Q_TAIL_IDX_OFFSET + _Q_TAIL_IDX_BITS)
+#define _Q_TAIL_CPU_BITS (32 - _Q_TAIL_CPU_OFFSET)
+#define _Q_TAIL_CPU_MASK _Q_SET_MASK(TAIL_CPU)
+
+#define _Q_LOCKED_VAL (1U << _Q_LOCKED_OFFSET)
+
+#endif /* __ASM_GENERIC_QSPINLOCK_TYPES_H */
diff --git a/kernel/Kconfig.locks b/kernel/Kconfig.locks
index d2b32ac..f185584 100644
--- a/kernel/Kconfig.locks
+++ b/kernel/Kconfig.locks
@@ -223,3 +223,10 @@ endif
config MUTEX_SPIN_ON_OWNER
def_bool y
depends on SMP && !DEBUG_MUTEXES
+
+config ARCH_USE_QUEUE_SPINLOCK
+ bool
+
+config QUEUE_SPINLOCK
+ def_bool y if ARCH_USE_QUEUE_SPINLOCK
+ depends on SMP && !PARAVIRT_SPINLOCKS
diff --git a/kernel/locking/Makefile b/kernel/locking/Makefile
index b8bdcd4..e6741ac 100644
--- a/kernel/locking/Makefile
+++ b/kernel/locking/Makefile
@@ -16,6 +16,7 @@ endif
obj-$(CONFIG_SMP) += spinlock.o
obj-$(CONFIG_SMP) += lglock.o
obj-$(CONFIG_PROVE_LOCKING) += spinlock.o
+obj-$(CONFIG_QUEUE_SPINLOCK) += qspinlock.o
obj-$(CONFIG_RT_MUTEXES) += rtmutex.o
obj-$(CONFIG_DEBUG_RT_MUTEXES) += rtmutex-debug.o
obj-$(CONFIG_RT_MUTEX_TESTER) += rtmutex-tester.o
diff --git a/kernel/locking/mcs_spinlock.h b/kernel/locking/mcs_spinlock.h
index a2dbac4..a59b677 100644
--- a/kernel/locking/mcs_spinlock.h
+++ b/kernel/locking/mcs_spinlock.h
@@ -17,6 +17,7 @@
struct mcs_spinlock {
struct mcs_spinlock *next;
int locked; /* 1 if lock acquired */
+ int count;
};
#ifndef arch_mcs_spin_lock_contended
diff --git a/kernel/locking/qspinlock.c b/kernel/locking/qspinlock.c
new file mode 100644
index 0000000..b97a1ad
--- /dev/null
+++ b/kernel/locking/qspinlock.c
@@ -0,0 +1,197 @@
+/*
+ * Queue spinlock
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * (C) Copyright 2013-2014 Hewlett-Packard Development Company, L.P.
+ *
+ * Authors: Waiman Long <waiman.long@hp.com>
+ * Peter Zijlstra <pzijlstr@redhat.com>
+ */
+#include <linux/smp.h>
+#include <linux/bug.h>
+#include <linux/cpumask.h>
+#include <linux/percpu.h>
+#include <linux/hardirq.h>
+#include <linux/mutex.h>
+#include <asm/qspinlock.h>
+
+/*
+ * The basic principle of a queue-based spinlock can best be understood
+ * by studying a classic queue-based spinlock implementation called the
+ * MCS lock. The paper below provides a good description for this kind
+ * of lock.
+ *
+ * http://www.cise.ufl.edu/tr/DOC/REP-1992-71.pdf
+ *
+ * This queue spinlock implementation is based on the MCS lock, however to make
+ * it fit the 4 bytes we assume spinlock_t to be, and preserve its existing
+ * API, we must modify it some.
+ *
+ * In particular; where the traditional MCS lock consists of a tail pointer
+ * (8 bytes) and needs the next pointer (another 8 bytes) of its own node to
+ * unlock the next pending (next->locked), we compress both these: {tail,
+ * next->locked} into a single u32 value.
+ *
+ * Since a spinlock disables recursion of its own context and there is a limit
+ * to the contexts that can nest; namely: task, softirq, hardirq, nmi, we can
+ * encode the tail as and index indicating this context and a cpu number.
+ *
+ * We can further change the first spinner to spin on a bit in the lock word
+ * instead of its node; whereby avoiding the need to carry a node from lock to
+ * unlock, and preserving API.
+ */
+
+#include "mcs_spinlock.h"
+
+/*
+ * Per-CPU queue node structures; we can never have more than 4 nested
+ * contexts: task, softirq, hardirq, nmi.
+ *
+ * Exactly fits one cacheline.
+ */
+static DEFINE_PER_CPU_ALIGNED(struct mcs_spinlock, mcs_nodes[4]);
+
+/*
+ * We must be able to distinguish between no-tail and the tail at 0:0,
+ * therefore increment the cpu number by one.
+ */
+
+static inline u32 encode_tail(int cpu, int idx)
+{
+ u32 tail;
+
+ tail = (cpu + 1) << _Q_TAIL_CPU_OFFSET;
+ tail |= idx << _Q_TAIL_IDX_OFFSET; /* assume < 4 */
+
+ return tail;
+}
+
+static inline struct mcs_spinlock *decode_tail(u32 tail)
+{
+ int cpu = (tail >> _Q_TAIL_CPU_OFFSET) - 1;
+ int idx = (tail & _Q_TAIL_IDX_MASK) >> _Q_TAIL_IDX_OFFSET;
+
+ return per_cpu_ptr(&mcs_nodes[idx], cpu);
+}
+
+/**
+ * queue_spin_lock_slowpath - acquire the queue spinlock
+ * @lock: Pointer to queue spinlock structure
+ * @val: Current value of the queue spinlock 32-bit word
+ *
+ * (queue tail, lock bit)
+ *
+ * fast : slow : unlock
+ * : :
+ * uncontended (0,0) --:--> (0,1) --------------------------------:--> (*,0)
+ * : | ^--------. / :
+ * : v \ | :
+ * uncontended : (n,x) --+--> (n,0) | :
+ * queue : | ^--' | :
+ * : v | :
+ * contended : (*,x) --+--> (*,0) -----> (*,1) ---' :
+ * queue : ^--' :
+ *
+ */
+void queue_spin_lock_slowpath(struct qspinlock *lock, u32 val)
+{
+ struct mcs_spinlock *prev, *next, *node;
+ u32 new, old, tail;
+ int idx;
+
+ BUILD_BUG_ON(CONFIG_NR_CPUS >= (1U << _Q_TAIL_CPU_BITS));
+
+ node = this_cpu_ptr(&mcs_nodes[0]);
+ idx = node->count++;
+ tail = encode_tail(smp_processor_id(), idx);
+
+ node += idx;
+ node->locked = 0;
+ node->next = NULL;
+
+ /*
+ * trylock || xchg(lock, node)
+ *
+ * 0,0 -> 0,1 ; trylock
+ * p,x -> n,x ; prev = xchg(lock, node)
+ */
+ for (;;) {
+ new = _Q_LOCKED_VAL;
+ if (val)
+ new = tail | (val & _Q_LOCKED_MASK);
+
+ old = atomic_cmpxchg(&lock->val, val, new);
+ if (old == val)
+ break;
+
+ val = old;
+ }
+
+ /*
+ * we won the trylock; forget about queueing.
+ */
+ if (new == _Q_LOCKED_VAL)
+ goto release;
+
+ /*
+ * if there was a previous node; link it and wait.
+ */
+ if (old & ~_Q_LOCKED_MASK) {
+ prev = decode_tail(old);
+ ACCESS_ONCE(prev->next) = node;
+
+ arch_mcs_spin_lock_contended(&node->locked);
+ }
+
+ /*
+ * we're at the head of the waitqueue, wait for the owner to go away.
+ *
+ * *,x -> *,0
+ */
+ while ((val = atomic_read(&lock->val)) & _Q_LOCKED_MASK)
+ arch_mutex_cpu_relax();
+
+ /*
+ * claim the lock:
+ *
+ * n,0 -> 0,1 : lock, uncontended
+ * *,0 -> *,1 : lock, contended
+ */
+ for (;;) {
+ new = _Q_LOCKED_VAL;
+ if (val != tail)
+ new |= val;
+
+ old = atomic_cmpxchg(&lock->val, val, new);
+ if (old == val)
+ break;
+
+ val = old;
+ }
+
+ /*
+ * contended path; wait for next, release.
+ */
+ if (new != _Q_LOCKED_VAL) {
+ while (!(next = ACCESS_ONCE(node->next)))
+ arch_mutex_cpu_relax();
+
+ arch_mcs_spin_unlock_contended(&next->locked);
+ }
+
+release:
+ /*
+ * release the node
+ */
+ this_cpu_dec(mcs_nodes[0].count);
+}
+EXPORT_SYMBOL(queue_spin_lock_slowpath);
--
1.7.1
^ permalink raw reply related [flat|nested] 102+ messages in thread* [PATCH v11 01/16] qspinlock: A simple generic 4-byte queue spinlock
@ 2014-05-30 15:43 ` Waiman Long
0 siblings, 0 replies; 102+ messages in thread
From: Waiman Long @ 2014-05-30 15:43 UTC (permalink / raw)
To: Thomas Gleixner, Ingo Molnar, H. Peter Anvin
Cc: linux-arch, Waiman Long, Rik van Riel, Raghavendra K T,
Gleb Natapov, kvm, Konrad Rzeszutek Wilk, Peter Zijlstra,
Scott J Norton, x86, Paolo Bonzini, linux-kernel, virtualization,
Chegu Vinod, David Vrabel, Oleg Nesterov, xen-devel,
Boris Ostrovsky, Paul E. McKenney, Linus Torvalds
This patch introduces a new generic queue spinlock implementation that
can serve as an alternative to the default ticket spinlock. Compared
with the ticket spinlock, this queue spinlock should be almost as fair
as the ticket spinlock. It has about the same speed in single-thread
and it can be much faster in high contention situations especially when
the spinlock is embedded within the data structure to be protected.
Only in light to moderate contention where the average queue depth
is around 1-3 will this queue spinlock be potentially a bit slower
due to the higher slowpath overhead.
This queue spinlock is especially suit to NUMA machines with a large
number of cores as the chance of spinlock contention is much higher
in those machines. The cost of contention is also higher because of
slower inter-node memory traffic.
Due to the fact that spinlocks are acquired with preemption disabled,
the process will not be migrated to another CPU while it is trying
to get a spinlock. Ignoring interrupt handling, a CPU can only be
contending in one spinlock at any one time. Counting soft IRQ, hard
IRQ and NMI, a CPU can only have a maximum of 4 concurrent lock waiting
activities. By allocating a set of per-cpu queue nodes and used them
to form a waiting queue, we can encode the queue node address into a
much smaller 24-bit size (including CPU number and queue node index)
leaving one byte for the lock.
Please note that the queue node is only needed when waiting for the
lock. Once the lock is acquired, the queue node can be released to
be used later.
Signed-off-by: Waiman Long <Waiman.Long@hp.com>
Signed-off-by: Peter Zijlstra <peterz@infradead.org>
---
include/asm-generic/qspinlock.h | 118 ++++++++++++++++++++
include/asm-generic/qspinlock_types.h | 61 ++++++++++
kernel/Kconfig.locks | 7 +
kernel/locking/Makefile | 1 +
kernel/locking/mcs_spinlock.h | 1 +
kernel/locking/qspinlock.c | 197 +++++++++++++++++++++++++++++++++
6 files changed, 385 insertions(+), 0 deletions(-)
create mode 100644 include/asm-generic/qspinlock.h
create mode 100644 include/asm-generic/qspinlock_types.h
create mode 100644 kernel/locking/qspinlock.c
diff --git a/include/asm-generic/qspinlock.h b/include/asm-generic/qspinlock.h
new file mode 100644
index 0000000..e8a7ae8
--- /dev/null
+++ b/include/asm-generic/qspinlock.h
@@ -0,0 +1,118 @@
+/*
+ * Queue spinlock
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * (C) Copyright 2013-2014 Hewlett-Packard Development Company, L.P.
+ *
+ * Authors: Waiman Long <waiman.long@hp.com>
+ */
+#ifndef __ASM_GENERIC_QSPINLOCK_H
+#define __ASM_GENERIC_QSPINLOCK_H
+
+#include <asm-generic/qspinlock_types.h>
+
+/**
+ * queue_spin_is_locked - is the spinlock locked?
+ * @lock: Pointer to queue spinlock structure
+ * Return: 1 if it is locked, 0 otherwise
+ */
+static __always_inline int queue_spin_is_locked(struct qspinlock *lock)
+{
+ return atomic_read(&lock->val);
+}
+
+/**
+ * queue_spin_value_unlocked - is the spinlock structure unlocked?
+ * @lock: queue spinlock structure
+ * Return: 1 if it is unlocked, 0 otherwise
+ *
+ * N.B. Whenever there are tasks waiting for the lock, it is considered
+ * locked wrt the lockref code to avoid lock stealing by the lockref
+ * code and change things underneath the lock. This also allows some
+ * optimizations to be applied without conflict with lockref.
+ */
+static __always_inline int queue_spin_value_unlocked(struct qspinlock lock)
+{
+ return !atomic_read(&lock.val);
+}
+
+/**
+ * queue_spin_is_contended - check if the lock is contended
+ * @lock : Pointer to queue spinlock structure
+ * Return: 1 if lock contended, 0 otherwise
+ */
+static __always_inline int queue_spin_is_contended(struct qspinlock *lock)
+{
+ return atomic_read(&lock->val) & ~_Q_LOCKED_MASK;
+}
+/**
+ * queue_spin_trylock - try to acquire the queue spinlock
+ * @lock : Pointer to queue spinlock structure
+ * Return: 1 if lock acquired, 0 if failed
+ */
+static __always_inline int queue_spin_trylock(struct qspinlock *lock)
+{
+ if (!atomic_read(&lock->val) &&
+ (atomic_cmpxchg(&lock->val, 0, _Q_LOCKED_VAL) == 0))
+ return 1;
+ return 0;
+}
+
+extern void queue_spin_lock_slowpath(struct qspinlock *lock, u32 val);
+
+/**
+ * queue_spin_lock - acquire a queue spinlock
+ * @lock: Pointer to queue spinlock structure
+ */
+static __always_inline void queue_spin_lock(struct qspinlock *lock)
+{
+ u32 val;
+
+ val = atomic_cmpxchg(&lock->val, 0, _Q_LOCKED_VAL);
+ if (likely(val == 0))
+ return;
+ queue_spin_lock_slowpath(lock, val);
+}
+
+#ifndef queue_spin_unlock
+/**
+ * queue_spin_unlock - release a queue spinlock
+ * @lock : Pointer to queue spinlock structure
+ */
+static __always_inline void queue_spin_unlock(struct qspinlock *lock)
+{
+ /*
+ * smp_mb__before_atomic() in order to guarantee release semantics
+ */
+ smp_mb__before_atomic_dec();
+ atomic_sub(_Q_LOCKED_VAL, &lock->val);
+}
+#endif
+
+/*
+ * Initializier
+ */
+#define __ARCH_SPIN_LOCK_UNLOCKED { ATOMIC_INIT(0) }
+
+/*
+ * Remapping spinlock architecture specific functions to the corresponding
+ * queue spinlock functions.
+ */
+#define arch_spin_is_locked(l) queue_spin_is_locked(l)
+#define arch_spin_is_contended(l) queue_spin_is_contended(l)
+#define arch_spin_value_unlocked(l) queue_spin_value_unlocked(l)
+#define arch_spin_lock(l) queue_spin_lock(l)
+#define arch_spin_trylock(l) queue_spin_trylock(l)
+#define arch_spin_unlock(l) queue_spin_unlock(l)
+#define arch_spin_lock_flags(l, f) queue_spin_lock(l)
+
+#endif /* __ASM_GENERIC_QSPINLOCK_H */
diff --git a/include/asm-generic/qspinlock_types.h b/include/asm-generic/qspinlock_types.h
new file mode 100644
index 0000000..f66f845
--- /dev/null
+++ b/include/asm-generic/qspinlock_types.h
@@ -0,0 +1,61 @@
+/*
+ * Queue spinlock
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * (C) Copyright 2013-2014 Hewlett-Packard Development Company, L.P.
+ *
+ * Authors: Waiman Long <waiman.long@hp.com>
+ */
+#ifndef __ASM_GENERIC_QSPINLOCK_TYPES_H
+#define __ASM_GENERIC_QSPINLOCK_TYPES_H
+
+/*
+ * Including atomic.h with PARAVIRT on will cause compilation errors because
+ * of recursive header file incluson via paravirt_types.h. A workaround is
+ * to include paravirt_types.h here.
+ */
+#ifdef CONFIG_PARAVIRT
+#include <asm/paravirt_types.h>
+#else
+#include <linux/types.h>
+#include <linux/atomic.h>
+#include <linux/bug.h>
+#endif
+
+typedef struct qspinlock {
+ atomic_t val;
+} arch_spinlock_t;
+
+/*
+ * Bitfields in the atomic value:
+ *
+ * 0- 7: locked byte
+ * 8- 9: tail index
+ * 10-31: tail cpu (+1)
+ */
+#define _Q_SET_MASK(type) (((1U << _Q_ ## type ## _BITS) - 1)\
+ << _Q_ ## type ## _OFFSET)
+#define _Q_LOCKED_OFFSET 0
+#define _Q_LOCKED_BITS 8
+#define _Q_LOCKED_MASK _Q_SET_MASK(LOCKED)
+
+#define _Q_TAIL_IDX_OFFSET (_Q_LOCKED_OFFSET + _Q_LOCKED_BITS)
+#define _Q_TAIL_IDX_BITS 2
+#define _Q_TAIL_IDX_MASK _Q_SET_MASK(TAIL_IDX)
+
+#define _Q_TAIL_CPU_OFFSET (_Q_TAIL_IDX_OFFSET + _Q_TAIL_IDX_BITS)
+#define _Q_TAIL_CPU_BITS (32 - _Q_TAIL_CPU_OFFSET)
+#define _Q_TAIL_CPU_MASK _Q_SET_MASK(TAIL_CPU)
+
+#define _Q_LOCKED_VAL (1U << _Q_LOCKED_OFFSET)
+
+#endif /* __ASM_GENERIC_QSPINLOCK_TYPES_H */
diff --git a/kernel/Kconfig.locks b/kernel/Kconfig.locks
index d2b32ac..f185584 100644
--- a/kernel/Kconfig.locks
+++ b/kernel/Kconfig.locks
@@ -223,3 +223,10 @@ endif
config MUTEX_SPIN_ON_OWNER
def_bool y
depends on SMP && !DEBUG_MUTEXES
+
+config ARCH_USE_QUEUE_SPINLOCK
+ bool
+
+config QUEUE_SPINLOCK
+ def_bool y if ARCH_USE_QUEUE_SPINLOCK
+ depends on SMP && !PARAVIRT_SPINLOCKS
diff --git a/kernel/locking/Makefile b/kernel/locking/Makefile
index b8bdcd4..e6741ac 100644
--- a/kernel/locking/Makefile
+++ b/kernel/locking/Makefile
@@ -16,6 +16,7 @@ endif
obj-$(CONFIG_SMP) += spinlock.o
obj-$(CONFIG_SMP) += lglock.o
obj-$(CONFIG_PROVE_LOCKING) += spinlock.o
+obj-$(CONFIG_QUEUE_SPINLOCK) += qspinlock.o
obj-$(CONFIG_RT_MUTEXES) += rtmutex.o
obj-$(CONFIG_DEBUG_RT_MUTEXES) += rtmutex-debug.o
obj-$(CONFIG_RT_MUTEX_TESTER) += rtmutex-tester.o
diff --git a/kernel/locking/mcs_spinlock.h b/kernel/locking/mcs_spinlock.h
index a2dbac4..a59b677 100644
--- a/kernel/locking/mcs_spinlock.h
+++ b/kernel/locking/mcs_spinlock.h
@@ -17,6 +17,7 @@
struct mcs_spinlock {
struct mcs_spinlock *next;
int locked; /* 1 if lock acquired */
+ int count;
};
#ifndef arch_mcs_spin_lock_contended
diff --git a/kernel/locking/qspinlock.c b/kernel/locking/qspinlock.c
new file mode 100644
index 0000000..b97a1ad
--- /dev/null
+++ b/kernel/locking/qspinlock.c
@@ -0,0 +1,197 @@
+/*
+ * Queue spinlock
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * (C) Copyright 2013-2014 Hewlett-Packard Development Company, L.P.
+ *
+ * Authors: Waiman Long <waiman.long@hp.com>
+ * Peter Zijlstra <pzijlstr@redhat.com>
+ */
+#include <linux/smp.h>
+#include <linux/bug.h>
+#include <linux/cpumask.h>
+#include <linux/percpu.h>
+#include <linux/hardirq.h>
+#include <linux/mutex.h>
+#include <asm/qspinlock.h>
+
+/*
+ * The basic principle of a queue-based spinlock can best be understood
+ * by studying a classic queue-based spinlock implementation called the
+ * MCS lock. The paper below provides a good description for this kind
+ * of lock.
+ *
+ * http://www.cise.ufl.edu/tr/DOC/REP-1992-71.pdf
+ *
+ * This queue spinlock implementation is based on the MCS lock, however to make
+ * it fit the 4 bytes we assume spinlock_t to be, and preserve its existing
+ * API, we must modify it some.
+ *
+ * In particular; where the traditional MCS lock consists of a tail pointer
+ * (8 bytes) and needs the next pointer (another 8 bytes) of its own node to
+ * unlock the next pending (next->locked), we compress both these: {tail,
+ * next->locked} into a single u32 value.
+ *
+ * Since a spinlock disables recursion of its own context and there is a limit
+ * to the contexts that can nest; namely: task, softirq, hardirq, nmi, we can
+ * encode the tail as and index indicating this context and a cpu number.
+ *
+ * We can further change the first spinner to spin on a bit in the lock word
+ * instead of its node; whereby avoiding the need to carry a node from lock to
+ * unlock, and preserving API.
+ */
+
+#include "mcs_spinlock.h"
+
+/*
+ * Per-CPU queue node structures; we can never have more than 4 nested
+ * contexts: task, softirq, hardirq, nmi.
+ *
+ * Exactly fits one cacheline.
+ */
+static DEFINE_PER_CPU_ALIGNED(struct mcs_spinlock, mcs_nodes[4]);
+
+/*
+ * We must be able to distinguish between no-tail and the tail at 0:0,
+ * therefore increment the cpu number by one.
+ */
+
+static inline u32 encode_tail(int cpu, int idx)
+{
+ u32 tail;
+
+ tail = (cpu + 1) << _Q_TAIL_CPU_OFFSET;
+ tail |= idx << _Q_TAIL_IDX_OFFSET; /* assume < 4 */
+
+ return tail;
+}
+
+static inline struct mcs_spinlock *decode_tail(u32 tail)
+{
+ int cpu = (tail >> _Q_TAIL_CPU_OFFSET) - 1;
+ int idx = (tail & _Q_TAIL_IDX_MASK) >> _Q_TAIL_IDX_OFFSET;
+
+ return per_cpu_ptr(&mcs_nodes[idx], cpu);
+}
+
+/**
+ * queue_spin_lock_slowpath - acquire the queue spinlock
+ * @lock: Pointer to queue spinlock structure
+ * @val: Current value of the queue spinlock 32-bit word
+ *
+ * (queue tail, lock bit)
+ *
+ * fast : slow : unlock
+ * : :
+ * uncontended (0,0) --:--> (0,1) --------------------------------:--> (*,0)
+ * : | ^--------. / :
+ * : v \ | :
+ * uncontended : (n,x) --+--> (n,0) | :
+ * queue : | ^--' | :
+ * : v | :
+ * contended : (*,x) --+--> (*,0) -----> (*,1) ---' :
+ * queue : ^--' :
+ *
+ */
+void queue_spin_lock_slowpath(struct qspinlock *lock, u32 val)
+{
+ struct mcs_spinlock *prev, *next, *node;
+ u32 new, old, tail;
+ int idx;
+
+ BUILD_BUG_ON(CONFIG_NR_CPUS >= (1U << _Q_TAIL_CPU_BITS));
+
+ node = this_cpu_ptr(&mcs_nodes[0]);
+ idx = node->count++;
+ tail = encode_tail(smp_processor_id(), idx);
+
+ node += idx;
+ node->locked = 0;
+ node->next = NULL;
+
+ /*
+ * trylock || xchg(lock, node)
+ *
+ * 0,0 -> 0,1 ; trylock
+ * p,x -> n,x ; prev = xchg(lock, node)
+ */
+ for (;;) {
+ new = _Q_LOCKED_VAL;
+ if (val)
+ new = tail | (val & _Q_LOCKED_MASK);
+
+ old = atomic_cmpxchg(&lock->val, val, new);
+ if (old == val)
+ break;
+
+ val = old;
+ }
+
+ /*
+ * we won the trylock; forget about queueing.
+ */
+ if (new == _Q_LOCKED_VAL)
+ goto release;
+
+ /*
+ * if there was a previous node; link it and wait.
+ */
+ if (old & ~_Q_LOCKED_MASK) {
+ prev = decode_tail(old);
+ ACCESS_ONCE(prev->next) = node;
+
+ arch_mcs_spin_lock_contended(&node->locked);
+ }
+
+ /*
+ * we're at the head of the waitqueue, wait for the owner to go away.
+ *
+ * *,x -> *,0
+ */
+ while ((val = atomic_read(&lock->val)) & _Q_LOCKED_MASK)
+ arch_mutex_cpu_relax();
+
+ /*
+ * claim the lock:
+ *
+ * n,0 -> 0,1 : lock, uncontended
+ * *,0 -> *,1 : lock, contended
+ */
+ for (;;) {
+ new = _Q_LOCKED_VAL;
+ if (val != tail)
+ new |= val;
+
+ old = atomic_cmpxchg(&lock->val, val, new);
+ if (old == val)
+ break;
+
+ val = old;
+ }
+
+ /*
+ * contended path; wait for next, release.
+ */
+ if (new != _Q_LOCKED_VAL) {
+ while (!(next = ACCESS_ONCE(node->next)))
+ arch_mutex_cpu_relax();
+
+ arch_mcs_spin_unlock_contended(&next->locked);
+ }
+
+release:
+ /*
+ * release the node
+ */
+ this_cpu_dec(mcs_nodes[0].count);
+}
+EXPORT_SYMBOL(queue_spin_lock_slowpath);
--
1.7.1
^ permalink raw reply related [flat|nested] 102+ messages in thread
* [PATCH v11 01/16] qspinlock: A simple generic 4-byte queue spinlock
2014-05-30 15:43 [PATCH v11 00/16] qspinlock: a 4-byte queue spinlock with PV support Waiman Long
2014-05-30 15:43 ` Waiman Long
@ 2014-05-30 15:43 ` Waiman Long
2014-05-30 15:43 ` [PATCH v11 02/16] qspinlock, x86: Enable x86-64 to use " Waiman Long
` (35 subsequent siblings)
37 siblings, 0 replies; 102+ messages in thread
From: Waiman Long @ 2014-05-30 15:43 UTC (permalink / raw)
To: Thomas Gleixner, Ingo Molnar, H. Peter Anvin
Cc: linux-arch, Waiman Long, Raghavendra K T, Gleb Natapov, kvm,
Peter Zijlstra, Scott J Norton, x86, Paolo Bonzini, linux-kernel,
virtualization, Chegu Vinod, David Vrabel, Oleg Nesterov,
xen-devel, Boris Ostrovsky, Paul E. McKenney, Linus Torvalds
This patch introduces a new generic queue spinlock implementation that
can serve as an alternative to the default ticket spinlock. Compared
with the ticket spinlock, this queue spinlock should be almost as fair
as the ticket spinlock. It has about the same speed in single-thread
and it can be much faster in high contention situations especially when
the spinlock is embedded within the data structure to be protected.
Only in light to moderate contention where the average queue depth
is around 1-3 will this queue spinlock be potentially a bit slower
due to the higher slowpath overhead.
This queue spinlock is especially suit to NUMA machines with a large
number of cores as the chance of spinlock contention is much higher
in those machines. The cost of contention is also higher because of
slower inter-node memory traffic.
Due to the fact that spinlocks are acquired with preemption disabled,
the process will not be migrated to another CPU while it is trying
to get a spinlock. Ignoring interrupt handling, a CPU can only be
contending in one spinlock at any one time. Counting soft IRQ, hard
IRQ and NMI, a CPU can only have a maximum of 4 concurrent lock waiting
activities. By allocating a set of per-cpu queue nodes and used them
to form a waiting queue, we can encode the queue node address into a
much smaller 24-bit size (including CPU number and queue node index)
leaving one byte for the lock.
Please note that the queue node is only needed when waiting for the
lock. Once the lock is acquired, the queue node can be released to
be used later.
Signed-off-by: Waiman Long <Waiman.Long@hp.com>
Signed-off-by: Peter Zijlstra <peterz@infradead.org>
---
include/asm-generic/qspinlock.h | 118 ++++++++++++++++++++
include/asm-generic/qspinlock_types.h | 61 ++++++++++
kernel/Kconfig.locks | 7 +
kernel/locking/Makefile | 1 +
kernel/locking/mcs_spinlock.h | 1 +
kernel/locking/qspinlock.c | 197 +++++++++++++++++++++++++++++++++
6 files changed, 385 insertions(+), 0 deletions(-)
create mode 100644 include/asm-generic/qspinlock.h
create mode 100644 include/asm-generic/qspinlock_types.h
create mode 100644 kernel/locking/qspinlock.c
diff --git a/include/asm-generic/qspinlock.h b/include/asm-generic/qspinlock.h
new file mode 100644
index 0000000..e8a7ae8
--- /dev/null
+++ b/include/asm-generic/qspinlock.h
@@ -0,0 +1,118 @@
+/*
+ * Queue spinlock
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * (C) Copyright 2013-2014 Hewlett-Packard Development Company, L.P.
+ *
+ * Authors: Waiman Long <waiman.long@hp.com>
+ */
+#ifndef __ASM_GENERIC_QSPINLOCK_H
+#define __ASM_GENERIC_QSPINLOCK_H
+
+#include <asm-generic/qspinlock_types.h>
+
+/**
+ * queue_spin_is_locked - is the spinlock locked?
+ * @lock: Pointer to queue spinlock structure
+ * Return: 1 if it is locked, 0 otherwise
+ */
+static __always_inline int queue_spin_is_locked(struct qspinlock *lock)
+{
+ return atomic_read(&lock->val);
+}
+
+/**
+ * queue_spin_value_unlocked - is the spinlock structure unlocked?
+ * @lock: queue spinlock structure
+ * Return: 1 if it is unlocked, 0 otherwise
+ *
+ * N.B. Whenever there are tasks waiting for the lock, it is considered
+ * locked wrt the lockref code to avoid lock stealing by the lockref
+ * code and change things underneath the lock. This also allows some
+ * optimizations to be applied without conflict with lockref.
+ */
+static __always_inline int queue_spin_value_unlocked(struct qspinlock lock)
+{
+ return !atomic_read(&lock.val);
+}
+
+/**
+ * queue_spin_is_contended - check if the lock is contended
+ * @lock : Pointer to queue spinlock structure
+ * Return: 1 if lock contended, 0 otherwise
+ */
+static __always_inline int queue_spin_is_contended(struct qspinlock *lock)
+{
+ return atomic_read(&lock->val) & ~_Q_LOCKED_MASK;
+}
+/**
+ * queue_spin_trylock - try to acquire the queue spinlock
+ * @lock : Pointer to queue spinlock structure
+ * Return: 1 if lock acquired, 0 if failed
+ */
+static __always_inline int queue_spin_trylock(struct qspinlock *lock)
+{
+ if (!atomic_read(&lock->val) &&
+ (atomic_cmpxchg(&lock->val, 0, _Q_LOCKED_VAL) == 0))
+ return 1;
+ return 0;
+}
+
+extern void queue_spin_lock_slowpath(struct qspinlock *lock, u32 val);
+
+/**
+ * queue_spin_lock - acquire a queue spinlock
+ * @lock: Pointer to queue spinlock structure
+ */
+static __always_inline void queue_spin_lock(struct qspinlock *lock)
+{
+ u32 val;
+
+ val = atomic_cmpxchg(&lock->val, 0, _Q_LOCKED_VAL);
+ if (likely(val == 0))
+ return;
+ queue_spin_lock_slowpath(lock, val);
+}
+
+#ifndef queue_spin_unlock
+/**
+ * queue_spin_unlock - release a queue spinlock
+ * @lock : Pointer to queue spinlock structure
+ */
+static __always_inline void queue_spin_unlock(struct qspinlock *lock)
+{
+ /*
+ * smp_mb__before_atomic() in order to guarantee release semantics
+ */
+ smp_mb__before_atomic_dec();
+ atomic_sub(_Q_LOCKED_VAL, &lock->val);
+}
+#endif
+
+/*
+ * Initializier
+ */
+#define __ARCH_SPIN_LOCK_UNLOCKED { ATOMIC_INIT(0) }
+
+/*
+ * Remapping spinlock architecture specific functions to the corresponding
+ * queue spinlock functions.
+ */
+#define arch_spin_is_locked(l) queue_spin_is_locked(l)
+#define arch_spin_is_contended(l) queue_spin_is_contended(l)
+#define arch_spin_value_unlocked(l) queue_spin_value_unlocked(l)
+#define arch_spin_lock(l) queue_spin_lock(l)
+#define arch_spin_trylock(l) queue_spin_trylock(l)
+#define arch_spin_unlock(l) queue_spin_unlock(l)
+#define arch_spin_lock_flags(l, f) queue_spin_lock(l)
+
+#endif /* __ASM_GENERIC_QSPINLOCK_H */
diff --git a/include/asm-generic/qspinlock_types.h b/include/asm-generic/qspinlock_types.h
new file mode 100644
index 0000000..f66f845
--- /dev/null
+++ b/include/asm-generic/qspinlock_types.h
@@ -0,0 +1,61 @@
+/*
+ * Queue spinlock
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * (C) Copyright 2013-2014 Hewlett-Packard Development Company, L.P.
+ *
+ * Authors: Waiman Long <waiman.long@hp.com>
+ */
+#ifndef __ASM_GENERIC_QSPINLOCK_TYPES_H
+#define __ASM_GENERIC_QSPINLOCK_TYPES_H
+
+/*
+ * Including atomic.h with PARAVIRT on will cause compilation errors because
+ * of recursive header file incluson via paravirt_types.h. A workaround is
+ * to include paravirt_types.h here.
+ */
+#ifdef CONFIG_PARAVIRT
+#include <asm/paravirt_types.h>
+#else
+#include <linux/types.h>
+#include <linux/atomic.h>
+#include <linux/bug.h>
+#endif
+
+typedef struct qspinlock {
+ atomic_t val;
+} arch_spinlock_t;
+
+/*
+ * Bitfields in the atomic value:
+ *
+ * 0- 7: locked byte
+ * 8- 9: tail index
+ * 10-31: tail cpu (+1)
+ */
+#define _Q_SET_MASK(type) (((1U << _Q_ ## type ## _BITS) - 1)\
+ << _Q_ ## type ## _OFFSET)
+#define _Q_LOCKED_OFFSET 0
+#define _Q_LOCKED_BITS 8
+#define _Q_LOCKED_MASK _Q_SET_MASK(LOCKED)
+
+#define _Q_TAIL_IDX_OFFSET (_Q_LOCKED_OFFSET + _Q_LOCKED_BITS)
+#define _Q_TAIL_IDX_BITS 2
+#define _Q_TAIL_IDX_MASK _Q_SET_MASK(TAIL_IDX)
+
+#define _Q_TAIL_CPU_OFFSET (_Q_TAIL_IDX_OFFSET + _Q_TAIL_IDX_BITS)
+#define _Q_TAIL_CPU_BITS (32 - _Q_TAIL_CPU_OFFSET)
+#define _Q_TAIL_CPU_MASK _Q_SET_MASK(TAIL_CPU)
+
+#define _Q_LOCKED_VAL (1U << _Q_LOCKED_OFFSET)
+
+#endif /* __ASM_GENERIC_QSPINLOCK_TYPES_H */
diff --git a/kernel/Kconfig.locks b/kernel/Kconfig.locks
index d2b32ac..f185584 100644
--- a/kernel/Kconfig.locks
+++ b/kernel/Kconfig.locks
@@ -223,3 +223,10 @@ endif
config MUTEX_SPIN_ON_OWNER
def_bool y
depends on SMP && !DEBUG_MUTEXES
+
+config ARCH_USE_QUEUE_SPINLOCK
+ bool
+
+config QUEUE_SPINLOCK
+ def_bool y if ARCH_USE_QUEUE_SPINLOCK
+ depends on SMP && !PARAVIRT_SPINLOCKS
diff --git a/kernel/locking/Makefile b/kernel/locking/Makefile
index b8bdcd4..e6741ac 100644
--- a/kernel/locking/Makefile
+++ b/kernel/locking/Makefile
@@ -16,6 +16,7 @@ endif
obj-$(CONFIG_SMP) += spinlock.o
obj-$(CONFIG_SMP) += lglock.o
obj-$(CONFIG_PROVE_LOCKING) += spinlock.o
+obj-$(CONFIG_QUEUE_SPINLOCK) += qspinlock.o
obj-$(CONFIG_RT_MUTEXES) += rtmutex.o
obj-$(CONFIG_DEBUG_RT_MUTEXES) += rtmutex-debug.o
obj-$(CONFIG_RT_MUTEX_TESTER) += rtmutex-tester.o
diff --git a/kernel/locking/mcs_spinlock.h b/kernel/locking/mcs_spinlock.h
index a2dbac4..a59b677 100644
--- a/kernel/locking/mcs_spinlock.h
+++ b/kernel/locking/mcs_spinlock.h
@@ -17,6 +17,7 @@
struct mcs_spinlock {
struct mcs_spinlock *next;
int locked; /* 1 if lock acquired */
+ int count;
};
#ifndef arch_mcs_spin_lock_contended
diff --git a/kernel/locking/qspinlock.c b/kernel/locking/qspinlock.c
new file mode 100644
index 0000000..b97a1ad
--- /dev/null
+++ b/kernel/locking/qspinlock.c
@@ -0,0 +1,197 @@
+/*
+ * Queue spinlock
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * (C) Copyright 2013-2014 Hewlett-Packard Development Company, L.P.
+ *
+ * Authors: Waiman Long <waiman.long@hp.com>
+ * Peter Zijlstra <pzijlstr@redhat.com>
+ */
+#include <linux/smp.h>
+#include <linux/bug.h>
+#include <linux/cpumask.h>
+#include <linux/percpu.h>
+#include <linux/hardirq.h>
+#include <linux/mutex.h>
+#include <asm/qspinlock.h>
+
+/*
+ * The basic principle of a queue-based spinlock can best be understood
+ * by studying a classic queue-based spinlock implementation called the
+ * MCS lock. The paper below provides a good description for this kind
+ * of lock.
+ *
+ * http://www.cise.ufl.edu/tr/DOC/REP-1992-71.pdf
+ *
+ * This queue spinlock implementation is based on the MCS lock, however to make
+ * it fit the 4 bytes we assume spinlock_t to be, and preserve its existing
+ * API, we must modify it some.
+ *
+ * In particular; where the traditional MCS lock consists of a tail pointer
+ * (8 bytes) and needs the next pointer (another 8 bytes) of its own node to
+ * unlock the next pending (next->locked), we compress both these: {tail,
+ * next->locked} into a single u32 value.
+ *
+ * Since a spinlock disables recursion of its own context and there is a limit
+ * to the contexts that can nest; namely: task, softirq, hardirq, nmi, we can
+ * encode the tail as and index indicating this context and a cpu number.
+ *
+ * We can further change the first spinner to spin on a bit in the lock word
+ * instead of its node; whereby avoiding the need to carry a node from lock to
+ * unlock, and preserving API.
+ */
+
+#include "mcs_spinlock.h"
+
+/*
+ * Per-CPU queue node structures; we can never have more than 4 nested
+ * contexts: task, softirq, hardirq, nmi.
+ *
+ * Exactly fits one cacheline.
+ */
+static DEFINE_PER_CPU_ALIGNED(struct mcs_spinlock, mcs_nodes[4]);
+
+/*
+ * We must be able to distinguish between no-tail and the tail at 0:0,
+ * therefore increment the cpu number by one.
+ */
+
+static inline u32 encode_tail(int cpu, int idx)
+{
+ u32 tail;
+
+ tail = (cpu + 1) << _Q_TAIL_CPU_OFFSET;
+ tail |= idx << _Q_TAIL_IDX_OFFSET; /* assume < 4 */
+
+ return tail;
+}
+
+static inline struct mcs_spinlock *decode_tail(u32 tail)
+{
+ int cpu = (tail >> _Q_TAIL_CPU_OFFSET) - 1;
+ int idx = (tail & _Q_TAIL_IDX_MASK) >> _Q_TAIL_IDX_OFFSET;
+
+ return per_cpu_ptr(&mcs_nodes[idx], cpu);
+}
+
+/**
+ * queue_spin_lock_slowpath - acquire the queue spinlock
+ * @lock: Pointer to queue spinlock structure
+ * @val: Current value of the queue spinlock 32-bit word
+ *
+ * (queue tail, lock bit)
+ *
+ * fast : slow : unlock
+ * : :
+ * uncontended (0,0) --:--> (0,1) --------------------------------:--> (*,0)
+ * : | ^--------. / :
+ * : v \ | :
+ * uncontended : (n,x) --+--> (n,0) | :
+ * queue : | ^--' | :
+ * : v | :
+ * contended : (*,x) --+--> (*,0) -----> (*,1) ---' :
+ * queue : ^--' :
+ *
+ */
+void queue_spin_lock_slowpath(struct qspinlock *lock, u32 val)
+{
+ struct mcs_spinlock *prev, *next, *node;
+ u32 new, old, tail;
+ int idx;
+
+ BUILD_BUG_ON(CONFIG_NR_CPUS >= (1U << _Q_TAIL_CPU_BITS));
+
+ node = this_cpu_ptr(&mcs_nodes[0]);
+ idx = node->count++;
+ tail = encode_tail(smp_processor_id(), idx);
+
+ node += idx;
+ node->locked = 0;
+ node->next = NULL;
+
+ /*
+ * trylock || xchg(lock, node)
+ *
+ * 0,0 -> 0,1 ; trylock
+ * p,x -> n,x ; prev = xchg(lock, node)
+ */
+ for (;;) {
+ new = _Q_LOCKED_VAL;
+ if (val)
+ new = tail | (val & _Q_LOCKED_MASK);
+
+ old = atomic_cmpxchg(&lock->val, val, new);
+ if (old == val)
+ break;
+
+ val = old;
+ }
+
+ /*
+ * we won the trylock; forget about queueing.
+ */
+ if (new == _Q_LOCKED_VAL)
+ goto release;
+
+ /*
+ * if there was a previous node; link it and wait.
+ */
+ if (old & ~_Q_LOCKED_MASK) {
+ prev = decode_tail(old);
+ ACCESS_ONCE(prev->next) = node;
+
+ arch_mcs_spin_lock_contended(&node->locked);
+ }
+
+ /*
+ * we're at the head of the waitqueue, wait for the owner to go away.
+ *
+ * *,x -> *,0
+ */
+ while ((val = atomic_read(&lock->val)) & _Q_LOCKED_MASK)
+ arch_mutex_cpu_relax();
+
+ /*
+ * claim the lock:
+ *
+ * n,0 -> 0,1 : lock, uncontended
+ * *,0 -> *,1 : lock, contended
+ */
+ for (;;) {
+ new = _Q_LOCKED_VAL;
+ if (val != tail)
+ new |= val;
+
+ old = atomic_cmpxchg(&lock->val, val, new);
+ if (old == val)
+ break;
+
+ val = old;
+ }
+
+ /*
+ * contended path; wait for next, release.
+ */
+ if (new != _Q_LOCKED_VAL) {
+ while (!(next = ACCESS_ONCE(node->next)))
+ arch_mutex_cpu_relax();
+
+ arch_mcs_spin_unlock_contended(&next->locked);
+ }
+
+release:
+ /*
+ * release the node
+ */
+ this_cpu_dec(mcs_nodes[0].count);
+}
+EXPORT_SYMBOL(queue_spin_lock_slowpath);
--
1.7.1
^ permalink raw reply related [flat|nested] 102+ messages in thread* [PATCH v11 02/16] qspinlock, x86: Enable x86-64 to use queue spinlock
2014-05-30 15:43 [PATCH v11 00/16] qspinlock: a 4-byte queue spinlock with PV support Waiman Long
2014-05-30 15:43 ` Waiman Long
2014-05-30 15:43 ` Waiman Long
@ 2014-05-30 15:43 ` Waiman Long
2014-05-30 15:43 ` Waiman Long
` (34 subsequent siblings)
37 siblings, 0 replies; 102+ messages in thread
From: Waiman Long @ 2014-05-30 15:43 UTC (permalink / raw)
To: Thomas Gleixner, Ingo Molnar, H. Peter Anvin
Cc: linux-arch, Waiman Long, Raghavendra K T, Gleb Natapov, kvm,
Peter Zijlstra, Scott J Norton, x86, Paolo Bonzini, linux-kernel,
virtualization, Chegu Vinod, David Vrabel, Oleg Nesterov,
xen-devel, Boris Ostrovsky, Paul E. McKenney, Linus Torvalds
This patch makes the necessary changes at the x86 architecture
specific layer to enable the use of queue spinlock for x86-64. As
x86-32 machines are typically not multi-socket. The benefit of queue
spinlock may not be apparent. So queue spinlock is not enabled.
Currently, there is some incompatibilities between the para-virtualized
spinlock code (which hard-codes the use of ticket spinlock) and the
queue spinlock. Therefore, the use of queue spinlock is disabled when
the para-virtualized spinlock is enabled.
The arch/x86/include/asm/qspinlock.h header file includes some x86
specific optimization which will make the queue spinlock code perform
better than the generic implementation.
Signed-off-by: Waiman Long <Waiman.Long@hp.com>
Signed-off-by: Peter Zijlstra <peterz@infradead.org>
---
arch/x86/Kconfig | 1 +
arch/x86/include/asm/qspinlock.h | 29 +++++++++++++++++++++++++++++
arch/x86/include/asm/spinlock.h | 5 +++++
arch/x86/include/asm/spinlock_types.h | 4 ++++
4 files changed, 39 insertions(+), 0 deletions(-)
create mode 100644 arch/x86/include/asm/qspinlock.h
diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig
index 25d2c6f..95c9c4e 100644
--- a/arch/x86/Kconfig
+++ b/arch/x86/Kconfig
@@ -29,6 +29,7 @@ config X86
select ARCH_SUPPORTS_NUMA_BALANCING
select ARCH_SUPPORTS_INT128 if X86_64
select ARCH_WANTS_PROT_NUMA_PROT_NONE
+ select ARCH_USE_QUEUE_SPINLOCK
select HAVE_IDE
select HAVE_OPROFILE
select HAVE_PCSPKR_PLATFORM
diff --git a/arch/x86/include/asm/qspinlock.h b/arch/x86/include/asm/qspinlock.h
new file mode 100644
index 0000000..e4a4f5d
--- /dev/null
+++ b/arch/x86/include/asm/qspinlock.h
@@ -0,0 +1,29 @@
+#ifndef _ASM_X86_QSPINLOCK_H
+#define _ASM_X86_QSPINLOCK_H
+
+#include <asm-generic/qspinlock_types.h>
+
+#if !defined(CONFIG_X86_OOSTORE) && !defined(CONFIG_X86_PPRO_FENCE)
+
+#define queue_spin_unlock queue_spin_unlock
+/**
+ * queue_spin_unlock - release a queue spinlock
+ * @lock : Pointer to queue spinlock structure
+ *
+ * No special memory barrier other than a compiler one is needed for the
+ * x86 architecture. A compiler barrier is added at the end to make sure
+ * that the clearing the lock bit is done ASAP without artificial delay
+ * due to compiler optimization.
+ */
+static inline void queue_spin_unlock(struct qspinlock *lock)
+{
+ barrier();
+ ACCESS_ONCE(*(u8 *)lock) = 0;
+ barrier();
+}
+
+#endif /* !CONFIG_X86_OOSTORE && !CONFIG_X86_PPRO_FENCE */
+
+#include <asm-generic/qspinlock.h>
+
+#endif /* _ASM_X86_QSPINLOCK_H */
diff --git a/arch/x86/include/asm/spinlock.h b/arch/x86/include/asm/spinlock.h
index 0f62f54..958d20f 100644
--- a/arch/x86/include/asm/spinlock.h
+++ b/arch/x86/include/asm/spinlock.h
@@ -42,6 +42,10 @@
extern struct static_key paravirt_ticketlocks_enabled;
static __always_inline bool static_key_false(struct static_key *key);
+#ifdef CONFIG_QUEUE_SPINLOCK
+#include <asm/qspinlock.h>
+#else
+
#ifdef CONFIG_PARAVIRT_SPINLOCKS
static inline void __ticket_enter_slowpath(arch_spinlock_t *lock)
@@ -180,6 +184,7 @@ static __always_inline void arch_spin_lock_flags(arch_spinlock_t *lock,
{
arch_spin_lock(lock);
}
+#endif /* CONFIG_QUEUE_SPINLOCK */
static inline void arch_spin_unlock_wait(arch_spinlock_t *lock)
{
diff --git a/arch/x86/include/asm/spinlock_types.h b/arch/x86/include/asm/spinlock_types.h
index 4f1bea1..7960268 100644
--- a/arch/x86/include/asm/spinlock_types.h
+++ b/arch/x86/include/asm/spinlock_types.h
@@ -23,6 +23,9 @@ typedef u32 __ticketpair_t;
#define TICKET_SHIFT (sizeof(__ticket_t) * 8)
+#ifdef CONFIG_QUEUE_SPINLOCK
+#include <asm-generic/qspinlock_types.h>
+#else
typedef struct arch_spinlock {
union {
__ticketpair_t head_tail;
@@ -33,6 +36,7 @@ typedef struct arch_spinlock {
} arch_spinlock_t;
#define __ARCH_SPIN_LOCK_UNLOCKED { { 0 } }
+#endif /* CONFIG_QUEUE_SPINLOCK */
#include <asm/rwlock.h>
--
1.7.1
^ permalink raw reply related [flat|nested] 102+ messages in thread* [PATCH v11 02/16] qspinlock, x86: Enable x86-64 to use queue spinlock
2014-05-30 15:43 [PATCH v11 00/16] qspinlock: a 4-byte queue spinlock with PV support Waiman Long
2014-05-30 15:43 ` Waiman Long
2014-05-30 15:43 ` Waiman Long
@ 2014-05-30 15:43 ` Waiman Long
2014-05-30 15:43 ` Waiman Long
` (34 subsequent siblings)
37 siblings, 0 replies; 102+ messages in thread
From: Waiman Long @ 2014-05-30 15:43 UTC (permalink / raw)
To: Thomas Gleixner, Ingo Molnar, H. Peter Anvin, Peter Zijlstra
Cc: linux-arch, x86, linux-kernel, virtualization, xen-devel, kvm,
Paolo Bonzini, Konrad Rzeszutek Wilk, Boris Ostrovsky,
Paul E. McKenney, Rik van Riel, Linus Torvalds, Raghavendra K T,
David Vrabel, Oleg Nesterov, Gleb Natapov, Scott J Norton,
Chegu Vinod, Waiman Long, Peter Zijlstra
This patch makes the necessary changes at the x86 architecture
specific layer to enable the use of queue spinlock for x86-64. As
x86-32 machines are typically not multi-socket. The benefit of queue
spinlock may not be apparent. So queue spinlock is not enabled.
Currently, there is some incompatibilities between the para-virtualized
spinlock code (which hard-codes the use of ticket spinlock) and the
queue spinlock. Therefore, the use of queue spinlock is disabled when
the para-virtualized spinlock is enabled.
The arch/x86/include/asm/qspinlock.h header file includes some x86
specific optimization which will make the queue spinlock code perform
better than the generic implementation.
Signed-off-by: Waiman Long <Waiman.Long@hp.com>
Signed-off-by: Peter Zijlstra <peterz@infradead.org>
---
arch/x86/Kconfig | 1 +
arch/x86/include/asm/qspinlock.h | 29 +++++++++++++++++++++++++++++
arch/x86/include/asm/spinlock.h | 5 +++++
arch/x86/include/asm/spinlock_types.h | 4 ++++
4 files changed, 39 insertions(+), 0 deletions(-)
create mode 100644 arch/x86/include/asm/qspinlock.h
diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig
index 25d2c6f..95c9c4e 100644
--- a/arch/x86/Kconfig
+++ b/arch/x86/Kconfig
@@ -29,6 +29,7 @@ config X86
select ARCH_SUPPORTS_NUMA_BALANCING
select ARCH_SUPPORTS_INT128 if X86_64
select ARCH_WANTS_PROT_NUMA_PROT_NONE
+ select ARCH_USE_QUEUE_SPINLOCK
select HAVE_IDE
select HAVE_OPROFILE
select HAVE_PCSPKR_PLATFORM
diff --git a/arch/x86/include/asm/qspinlock.h b/arch/x86/include/asm/qspinlock.h
new file mode 100644
index 0000000..e4a4f5d
--- /dev/null
+++ b/arch/x86/include/asm/qspinlock.h
@@ -0,0 +1,29 @@
+#ifndef _ASM_X86_QSPINLOCK_H
+#define _ASM_X86_QSPINLOCK_H
+
+#include <asm-generic/qspinlock_types.h>
+
+#if !defined(CONFIG_X86_OOSTORE) && !defined(CONFIG_X86_PPRO_FENCE)
+
+#define queue_spin_unlock queue_spin_unlock
+/**
+ * queue_spin_unlock - release a queue spinlock
+ * @lock : Pointer to queue spinlock structure
+ *
+ * No special memory barrier other than a compiler one is needed for the
+ * x86 architecture. A compiler barrier is added at the end to make sure
+ * that the clearing the lock bit is done ASAP without artificial delay
+ * due to compiler optimization.
+ */
+static inline void queue_spin_unlock(struct qspinlock *lock)
+{
+ barrier();
+ ACCESS_ONCE(*(u8 *)lock) = 0;
+ barrier();
+}
+
+#endif /* !CONFIG_X86_OOSTORE && !CONFIG_X86_PPRO_FENCE */
+
+#include <asm-generic/qspinlock.h>
+
+#endif /* _ASM_X86_QSPINLOCK_H */
diff --git a/arch/x86/include/asm/spinlock.h b/arch/x86/include/asm/spinlock.h
index 0f62f54..958d20f 100644
--- a/arch/x86/include/asm/spinlock.h
+++ b/arch/x86/include/asm/spinlock.h
@@ -42,6 +42,10 @@
extern struct static_key paravirt_ticketlocks_enabled;
static __always_inline bool static_key_false(struct static_key *key);
+#ifdef CONFIG_QUEUE_SPINLOCK
+#include <asm/qspinlock.h>
+#else
+
#ifdef CONFIG_PARAVIRT_SPINLOCKS
static inline void __ticket_enter_slowpath(arch_spinlock_t *lock)
@@ -180,6 +184,7 @@ static __always_inline void arch_spin_lock_flags(arch_spinlock_t *lock,
{
arch_spin_lock(lock);
}
+#endif /* CONFIG_QUEUE_SPINLOCK */
static inline void arch_spin_unlock_wait(arch_spinlock_t *lock)
{
diff --git a/arch/x86/include/asm/spinlock_types.h b/arch/x86/include/asm/spinlock_types.h
index 4f1bea1..7960268 100644
--- a/arch/x86/include/asm/spinlock_types.h
+++ b/arch/x86/include/asm/spinlock_types.h
@@ -23,6 +23,9 @@ typedef u32 __ticketpair_t;
#define TICKET_SHIFT (sizeof(__ticket_t) * 8)
+#ifdef CONFIG_QUEUE_SPINLOCK
+#include <asm-generic/qspinlock_types.h>
+#else
typedef struct arch_spinlock {
union {
__ticketpair_t head_tail;
@@ -33,6 +36,7 @@ typedef struct arch_spinlock {
} arch_spinlock_t;
#define __ARCH_SPIN_LOCK_UNLOCKED { { 0 } }
+#endif /* CONFIG_QUEUE_SPINLOCK */
#include <asm/rwlock.h>
--
1.7.1
^ permalink raw reply related [flat|nested] 102+ messages in thread* [PATCH v11 02/16] qspinlock, x86: Enable x86-64 to use queue spinlock
@ 2014-05-30 15:43 ` Waiman Long
0 siblings, 0 replies; 102+ messages in thread
From: Waiman Long @ 2014-05-30 15:43 UTC (permalink / raw)
To: Thomas Gleixner, Ingo Molnar, H. Peter Anvin, Peter Zijlstra
Cc: linux-arch, Waiman Long, Rik van Riel, Raghavendra K T,
Gleb Natapov, kvm, Konrad Rzeszutek Wilk, Peter Zijlstra,
Scott J Norton, x86, Paolo Bonzini, linux-kernel, virtualization,
Chegu Vinod, David Vrabel, Oleg Nesterov, xen-devel,
Boris Ostrovsky, Paul E. McKenney, Linus Torvalds
This patch makes the necessary changes at the x86 architecture
specific layer to enable the use of queue spinlock for x86-64. As
x86-32 machines are typically not multi-socket. The benefit of queue
spinlock may not be apparent. So queue spinlock is not enabled.
Currently, there is some incompatibilities between the para-virtualized
spinlock code (which hard-codes the use of ticket spinlock) and the
queue spinlock. Therefore, the use of queue spinlock is disabled when
the para-virtualized spinlock is enabled.
The arch/x86/include/asm/qspinlock.h header file includes some x86
specific optimization which will make the queue spinlock code perform
better than the generic implementation.
Signed-off-by: Waiman Long <Waiman.Long@hp.com>
Signed-off-by: Peter Zijlstra <peterz@infradead.org>
---
arch/x86/Kconfig | 1 +
arch/x86/include/asm/qspinlock.h | 29 +++++++++++++++++++++++++++++
arch/x86/include/asm/spinlock.h | 5 +++++
arch/x86/include/asm/spinlock_types.h | 4 ++++
4 files changed, 39 insertions(+), 0 deletions(-)
create mode 100644 arch/x86/include/asm/qspinlock.h
diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig
index 25d2c6f..95c9c4e 100644
--- a/arch/x86/Kconfig
+++ b/arch/x86/Kconfig
@@ -29,6 +29,7 @@ config X86
select ARCH_SUPPORTS_NUMA_BALANCING
select ARCH_SUPPORTS_INT128 if X86_64
select ARCH_WANTS_PROT_NUMA_PROT_NONE
+ select ARCH_USE_QUEUE_SPINLOCK
select HAVE_IDE
select HAVE_OPROFILE
select HAVE_PCSPKR_PLATFORM
diff --git a/arch/x86/include/asm/qspinlock.h b/arch/x86/include/asm/qspinlock.h
new file mode 100644
index 0000000..e4a4f5d
--- /dev/null
+++ b/arch/x86/include/asm/qspinlock.h
@@ -0,0 +1,29 @@
+#ifndef _ASM_X86_QSPINLOCK_H
+#define _ASM_X86_QSPINLOCK_H
+
+#include <asm-generic/qspinlock_types.h>
+
+#if !defined(CONFIG_X86_OOSTORE) && !defined(CONFIG_X86_PPRO_FENCE)
+
+#define queue_spin_unlock queue_spin_unlock
+/**
+ * queue_spin_unlock - release a queue spinlock
+ * @lock : Pointer to queue spinlock structure
+ *
+ * No special memory barrier other than a compiler one is needed for the
+ * x86 architecture. A compiler barrier is added at the end to make sure
+ * that the clearing the lock bit is done ASAP without artificial delay
+ * due to compiler optimization.
+ */
+static inline void queue_spin_unlock(struct qspinlock *lock)
+{
+ barrier();
+ ACCESS_ONCE(*(u8 *)lock) = 0;
+ barrier();
+}
+
+#endif /* !CONFIG_X86_OOSTORE && !CONFIG_X86_PPRO_FENCE */
+
+#include <asm-generic/qspinlock.h>
+
+#endif /* _ASM_X86_QSPINLOCK_H */
diff --git a/arch/x86/include/asm/spinlock.h b/arch/x86/include/asm/spinlock.h
index 0f62f54..958d20f 100644
--- a/arch/x86/include/asm/spinlock.h
+++ b/arch/x86/include/asm/spinlock.h
@@ -42,6 +42,10 @@
extern struct static_key paravirt_ticketlocks_enabled;
static __always_inline bool static_key_false(struct static_key *key);
+#ifdef CONFIG_QUEUE_SPINLOCK
+#include <asm/qspinlock.h>
+#else
+
#ifdef CONFIG_PARAVIRT_SPINLOCKS
static inline void __ticket_enter_slowpath(arch_spinlock_t *lock)
@@ -180,6 +184,7 @@ static __always_inline void arch_spin_lock_flags(arch_spinlock_t *lock,
{
arch_spin_lock(lock);
}
+#endif /* CONFIG_QUEUE_SPINLOCK */
static inline void arch_spin_unlock_wait(arch_spinlock_t *lock)
{
diff --git a/arch/x86/include/asm/spinlock_types.h b/arch/x86/include/asm/spinlock_types.h
index 4f1bea1..7960268 100644
--- a/arch/x86/include/asm/spinlock_types.h
+++ b/arch/x86/include/asm/spinlock_types.h
@@ -23,6 +23,9 @@ typedef u32 __ticketpair_t;
#define TICKET_SHIFT (sizeof(__ticket_t) * 8)
+#ifdef CONFIG_QUEUE_SPINLOCK
+#include <asm-generic/qspinlock_types.h>
+#else
typedef struct arch_spinlock {
union {
__ticketpair_t head_tail;
@@ -33,6 +36,7 @@ typedef struct arch_spinlock {
} arch_spinlock_t;
#define __ARCH_SPIN_LOCK_UNLOCKED { { 0 } }
+#endif /* CONFIG_QUEUE_SPINLOCK */
#include <asm/rwlock.h>
--
1.7.1
^ permalink raw reply related [flat|nested] 102+ messages in thread* [PATCH v11 02/16] qspinlock, x86: Enable x86-64 to use queue spinlock
@ 2014-05-30 15:43 ` Waiman Long
0 siblings, 0 replies; 102+ messages in thread
From: Waiman Long @ 2014-05-30 15:43 UTC (permalink / raw)
To: Thomas Gleixner, Ingo Molnar, H. Peter Anvin, Peter Zijlstra
Cc: linux-arch, x86, linux-kernel, virtualization, xen-devel, kvm,
Paolo Bonzini, Konrad Rzeszutek Wilk, Boris Ostrovsky,
Paul E. McKenney, Rik van Riel, Linus Torvalds, Raghavendra K T,
David Vrabel, Oleg Nesterov, Gleb Natapov, Scott J Norton,
Chegu Vinod, Waiman Long
This patch makes the necessary changes at the x86 architecture
specific layer to enable the use of queue spinlock for x86-64. As
x86-32 machines are typically not multi-socket. The benefit of queue
spinlock may not be apparent. So queue spinlock is not enabled.
Currently, there is some incompatibilities between the para-virtualized
spinlock code (which hard-codes the use of ticket spinlock) and the
queue spinlock. Therefore, the use of queue spinlock is disabled when
the para-virtualized spinlock is enabled.
The arch/x86/include/asm/qspinlock.h header file includes some x86
specific optimization which will make the queue spinlock code perform
better than the generic implementation.
Signed-off-by: Waiman Long <Waiman.Long@hp.com>
Signed-off-by: Peter Zijlstra <peterz@infradead.org>
---
arch/x86/Kconfig | 1 +
arch/x86/include/asm/qspinlock.h | 29 +++++++++++++++++++++++++++++
arch/x86/include/asm/spinlock.h | 5 +++++
arch/x86/include/asm/spinlock_types.h | 4 ++++
4 files changed, 39 insertions(+), 0 deletions(-)
create mode 100644 arch/x86/include/asm/qspinlock.h
diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig
index 25d2c6f..95c9c4e 100644
--- a/arch/x86/Kconfig
+++ b/arch/x86/Kconfig
@@ -29,6 +29,7 @@ config X86
select ARCH_SUPPORTS_NUMA_BALANCING
select ARCH_SUPPORTS_INT128 if X86_64
select ARCH_WANTS_PROT_NUMA_PROT_NONE
+ select ARCH_USE_QUEUE_SPINLOCK
select HAVE_IDE
select HAVE_OPROFILE
select HAVE_PCSPKR_PLATFORM
diff --git a/arch/x86/include/asm/qspinlock.h b/arch/x86/include/asm/qspinlock.h
new file mode 100644
index 0000000..e4a4f5d
--- /dev/null
+++ b/arch/x86/include/asm/qspinlock.h
@@ -0,0 +1,29 @@
+#ifndef _ASM_X86_QSPINLOCK_H
+#define _ASM_X86_QSPINLOCK_H
+
+#include <asm-generic/qspinlock_types.h>
+
+#if !defined(CONFIG_X86_OOSTORE) && !defined(CONFIG_X86_PPRO_FENCE)
+
+#define queue_spin_unlock queue_spin_unlock
+/**
+ * queue_spin_unlock - release a queue spinlock
+ * @lock : Pointer to queue spinlock structure
+ *
+ * No special memory barrier other than a compiler one is needed for the
+ * x86 architecture. A compiler barrier is added at the end to make sure
+ * that the clearing the lock bit is done ASAP without artificial delay
+ * due to compiler optimization.
+ */
+static inline void queue_spin_unlock(struct qspinlock *lock)
+{
+ barrier();
+ ACCESS_ONCE(*(u8 *)lock) = 0;
+ barrier();
+}
+
+#endif /* !CONFIG_X86_OOSTORE && !CONFIG_X86_PPRO_FENCE */
+
+#include <asm-generic/qspinlock.h>
+
+#endif /* _ASM_X86_QSPINLOCK_H */
diff --git a/arch/x86/include/asm/spinlock.h b/arch/x86/include/asm/spinlock.h
index 0f62f54..958d20f 100644
--- a/arch/x86/include/asm/spinlock.h
+++ b/arch/x86/include/asm/spinlock.h
@@ -42,6 +42,10 @@
extern struct static_key paravirt_ticketlocks_enabled;
static __always_inline bool static_key_false(struct static_key *key);
+#ifdef CONFIG_QUEUE_SPINLOCK
+#include <asm/qspinlock.h>
+#else
+
#ifdef CONFIG_PARAVIRT_SPINLOCKS
static inline void __ticket_enter_slowpath(arch_spinlock_t *lock)
@@ -180,6 +184,7 @@ static __always_inline void arch_spin_lock_flags(arch_spinlock_t *lock,
{
arch_spin_lock(lock);
}
+#endif /* CONFIG_QUEUE_SPINLOCK */
static inline void arch_spin_unlock_wait(arch_spinlock_t *lock)
{
diff --git a/arch/x86/include/asm/spinlock_types.h b/arch/x86/include/asm/spinlock_types.h
index 4f1bea1..7960268 100644
--- a/arch/x86/include/asm/spinlock_types.h
+++ b/arch/x86/include/asm/spinlock_types.h
@@ -23,6 +23,9 @@ typedef u32 __ticketpair_t;
#define TICKET_SHIFT (sizeof(__ticket_t) * 8)
+#ifdef CONFIG_QUEUE_SPINLOCK
+#include <asm-generic/qspinlock_types.h>
+#else
typedef struct arch_spinlock {
union {
__ticketpair_t head_tail;
@@ -33,6 +36,7 @@ typedef struct arch_spinlock {
} arch_spinlock_t;
#define __ARCH_SPIN_LOCK_UNLOCKED { { 0 } }
+#endif /* CONFIG_QUEUE_SPINLOCK */
#include <asm/rwlock.h>
--
1.7.1
^ permalink raw reply related [flat|nested] 102+ messages in thread* [PATCH v11 02/16] qspinlock, x86: Enable x86-64 to use queue spinlock
@ 2014-05-30 15:43 ` Waiman Long
0 siblings, 0 replies; 102+ messages in thread
From: Waiman Long @ 2014-05-30 15:43 UTC (permalink / raw)
To: Thomas Gleixner, Ingo Molnar, H. Peter Anvin
Cc: linux-arch, Waiman Long, Rik van Riel, Raghavendra K T,
Gleb Natapov, kvm, Konrad Rzeszutek Wilk, Peter Zijlstra,
Scott J Norton, x86, Paolo Bonzini, linux-kernel, virtualization,
Chegu Vinod, David Vrabel, Oleg Nesterov, xen-devel,
Boris Ostrovsky, Paul E. McKenney, Linus Torvalds
This patch makes the necessary changes at the x86 architecture
specific layer to enable the use of queue spinlock for x86-64. As
x86-32 machines are typically not multi-socket. The benefit of queue
spinlock may not be apparent. So queue spinlock is not enabled.
Currently, there is some incompatibilities between the para-virtualized
spinlock code (which hard-codes the use of ticket spinlock) and the
queue spinlock. Therefore, the use of queue spinlock is disabled when
the para-virtualized spinlock is enabled.
The arch/x86/include/asm/qspinlock.h header file includes some x86
specific optimization which will make the queue spinlock code perform
better than the generic implementation.
Signed-off-by: Waiman Long <Waiman.Long@hp.com>
Signed-off-by: Peter Zijlstra <peterz@infradead.org>
---
arch/x86/Kconfig | 1 +
arch/x86/include/asm/qspinlock.h | 29 +++++++++++++++++++++++++++++
arch/x86/include/asm/spinlock.h | 5 +++++
arch/x86/include/asm/spinlock_types.h | 4 ++++
4 files changed, 39 insertions(+), 0 deletions(-)
create mode 100644 arch/x86/include/asm/qspinlock.h
diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig
index 25d2c6f..95c9c4e 100644
--- a/arch/x86/Kconfig
+++ b/arch/x86/Kconfig
@@ -29,6 +29,7 @@ config X86
select ARCH_SUPPORTS_NUMA_BALANCING
select ARCH_SUPPORTS_INT128 if X86_64
select ARCH_WANTS_PROT_NUMA_PROT_NONE
+ select ARCH_USE_QUEUE_SPINLOCK
select HAVE_IDE
select HAVE_OPROFILE
select HAVE_PCSPKR_PLATFORM
diff --git a/arch/x86/include/asm/qspinlock.h b/arch/x86/include/asm/qspinlock.h
new file mode 100644
index 0000000..e4a4f5d
--- /dev/null
+++ b/arch/x86/include/asm/qspinlock.h
@@ -0,0 +1,29 @@
+#ifndef _ASM_X86_QSPINLOCK_H
+#define _ASM_X86_QSPINLOCK_H
+
+#include <asm-generic/qspinlock_types.h>
+
+#if !defined(CONFIG_X86_OOSTORE) && !defined(CONFIG_X86_PPRO_FENCE)
+
+#define queue_spin_unlock queue_spin_unlock
+/**
+ * queue_spin_unlock - release a queue spinlock
+ * @lock : Pointer to queue spinlock structure
+ *
+ * No special memory barrier other than a compiler one is needed for the
+ * x86 architecture. A compiler barrier is added at the end to make sure
+ * that the clearing the lock bit is done ASAP without artificial delay
+ * due to compiler optimization.
+ */
+static inline void queue_spin_unlock(struct qspinlock *lock)
+{
+ barrier();
+ ACCESS_ONCE(*(u8 *)lock) = 0;
+ barrier();
+}
+
+#endif /* !CONFIG_X86_OOSTORE && !CONFIG_X86_PPRO_FENCE */
+
+#include <asm-generic/qspinlock.h>
+
+#endif /* _ASM_X86_QSPINLOCK_H */
diff --git a/arch/x86/include/asm/spinlock.h b/arch/x86/include/asm/spinlock.h
index 0f62f54..958d20f 100644
--- a/arch/x86/include/asm/spinlock.h
+++ b/arch/x86/include/asm/spinlock.h
@@ -42,6 +42,10 @@
extern struct static_key paravirt_ticketlocks_enabled;
static __always_inline bool static_key_false(struct static_key *key);
+#ifdef CONFIG_QUEUE_SPINLOCK
+#include <asm/qspinlock.h>
+#else
+
#ifdef CONFIG_PARAVIRT_SPINLOCKS
static inline void __ticket_enter_slowpath(arch_spinlock_t *lock)
@@ -180,6 +184,7 @@ static __always_inline void arch_spin_lock_flags(arch_spinlock_t *lock,
{
arch_spin_lock(lock);
}
+#endif /* CONFIG_QUEUE_SPINLOCK */
static inline void arch_spin_unlock_wait(arch_spinlock_t *lock)
{
diff --git a/arch/x86/include/asm/spinlock_types.h b/arch/x86/include/asm/spinlock_types.h
index 4f1bea1..7960268 100644
--- a/arch/x86/include/asm/spinlock_types.h
+++ b/arch/x86/include/asm/spinlock_types.h
@@ -23,6 +23,9 @@ typedef u32 __ticketpair_t;
#define TICKET_SHIFT (sizeof(__ticket_t) * 8)
+#ifdef CONFIG_QUEUE_SPINLOCK
+#include <asm-generic/qspinlock_types.h>
+#else
typedef struct arch_spinlock {
union {
__ticketpair_t head_tail;
@@ -33,6 +36,7 @@ typedef struct arch_spinlock {
} arch_spinlock_t;
#define __ARCH_SPIN_LOCK_UNLOCKED { { 0 } }
+#endif /* CONFIG_QUEUE_SPINLOCK */
#include <asm/rwlock.h>
--
1.7.1
^ permalink raw reply related [flat|nested] 102+ messages in thread
* [PATCH v11 03/16] qspinlock: Add pending bit
2014-05-30 15:43 [PATCH v11 00/16] qspinlock: a 4-byte queue spinlock with PV support Waiman Long
2014-05-30 15:43 ` Waiman Long
2014-05-30 15:43 ` Waiman Long
@ 2014-05-30 15:43 ` Waiman Long
2014-05-30 15:43 ` Waiman Long
` (34 subsequent siblings)
37 siblings, 0 replies; 102+ messages in thread
From: Waiman Long @ 2014-05-30 15:43 UTC (permalink / raw)
To: Thomas Gleixner, Ingo Molnar, H. Peter Anvin, Peter Zijlstra
Cc: linux-arch, x86, linux-kernel, virtualization, xen-devel, kvm,
Paolo Bonzini, Konrad Rzeszutek Wilk, Boris Ostrovsky,
Paul E. McKenney, Rik van Riel, Linus Torvalds, Raghavendra K T,
David Vrabel, Oleg Nesterov, Gleb Natapov, Scott J Norton,
Chegu Vinod, Peter Zijlstra, Waiman Long
From: Peter Zijlstra <peterz@infradead.org>
Because the qspinlock needs to touch a second cacheline; add a pending
bit and allow a single in-word spinner before we punt to the second
cacheline.
Signed-off-by: Peter Zijlstra <peterz@infradead.org>
Signed-off-by: Waiman Long <Waiman.Long@hp.com>
---
include/asm-generic/qspinlock_types.h | 12 +++-
kernel/locking/qspinlock.c | 109 ++++++++++++++++++++++++++------
2 files changed, 97 insertions(+), 24 deletions(-)
diff --git a/include/asm-generic/qspinlock_types.h b/include/asm-generic/qspinlock_types.h
index f66f845..bd25081 100644
--- a/include/asm-generic/qspinlock_types.h
+++ b/include/asm-generic/qspinlock_types.h
@@ -39,8 +39,9 @@ typedef struct qspinlock {
* Bitfields in the atomic value:
*
* 0- 7: locked byte
- * 8- 9: tail index
- * 10-31: tail cpu (+1)
+ * 8: pending
+ * 9-10: tail index
+ * 11-31: tail cpu (+1)
*/
#define _Q_SET_MASK(type) (((1U << _Q_ ## type ## _BITS) - 1)\
<< _Q_ ## type ## _OFFSET)
@@ -48,7 +49,11 @@ typedef struct qspinlock {
#define _Q_LOCKED_BITS 8
#define _Q_LOCKED_MASK _Q_SET_MASK(LOCKED)
-#define _Q_TAIL_IDX_OFFSET (_Q_LOCKED_OFFSET + _Q_LOCKED_BITS)
+#define _Q_PENDING_OFFSET (_Q_LOCKED_OFFSET + _Q_LOCKED_BITS)
+#define _Q_PENDING_BITS 1
+#define _Q_PENDING_MASK _Q_SET_MASK(PENDING)
+
+#define _Q_TAIL_IDX_OFFSET (_Q_PENDING_OFFSET + _Q_PENDING_BITS)
#define _Q_TAIL_IDX_BITS 2
#define _Q_TAIL_IDX_MASK _Q_SET_MASK(TAIL_IDX)
@@ -57,5 +62,6 @@ typedef struct qspinlock {
#define _Q_TAIL_CPU_MASK _Q_SET_MASK(TAIL_CPU)
#define _Q_LOCKED_VAL (1U << _Q_LOCKED_OFFSET)
+#define _Q_PENDING_VAL (1U << _Q_PENDING_OFFSET)
#endif /* __ASM_GENERIC_QSPINLOCK_TYPES_H */
diff --git a/kernel/locking/qspinlock.c b/kernel/locking/qspinlock.c
index b97a1ad..1e93c6a 100644
--- a/kernel/locking/qspinlock.c
+++ b/kernel/locking/qspinlock.c
@@ -83,24 +83,28 @@ static inline struct mcs_spinlock *decode_tail(u32 tail)
return per_cpu_ptr(&mcs_nodes[idx], cpu);
}
+#define _Q_LOCKED_PENDING_MASK (_Q_LOCKED_MASK | _Q_PENDING_MASK)
+
/**
* queue_spin_lock_slowpath - acquire the queue spinlock
* @lock: Pointer to queue spinlock structure
* @val: Current value of the queue spinlock 32-bit word
*
- * (queue tail, lock bit)
- *
- * fast : slow : unlock
- * : :
- * uncontended (0,0) --:--> (0,1) --------------------------------:--> (*,0)
- * : | ^--------. / :
- * : v \ | :
- * uncontended : (n,x) --+--> (n,0) | :
- * queue : | ^--' | :
- * : v | :
- * contended : (*,x) --+--> (*,0) -----> (*,1) ---' :
- * queue : ^--' :
+ * (queue tail, pending bit, lock bit)
*
+ * fast : slow : unlock
+ * : :
+ * uncontended (0,0,0) -:--> (0,0,1) ------------------------------:--> (*,*,0)
+ * : | ^--------.------. / :
+ * : v \ \ | :
+ * pending : (0,1,1) +--> (0,1,0) \ | :
+ * : | ^--' | | :
+ * : v | | :
+ * uncontended : (n,x,y) +--> (n,0,0) --' | :
+ * queue : | ^--' | :
+ * : v | :
+ * contended : (*,x,y) +--> (*,0,0) ---> (*,0,1) -' :
+ * queue : ^--' :
*/
void queue_spin_lock_slowpath(struct qspinlock *lock, u32 val)
{
@@ -110,6 +114,65 @@ void queue_spin_lock_slowpath(struct qspinlock *lock, u32 val)
BUILD_BUG_ON(CONFIG_NR_CPUS >= (1U << _Q_TAIL_CPU_BITS));
+ /*
+ * trylock || pending
+ *
+ * 0,0,0 -> 0,0,1 ; trylock
+ * 0,0,1 -> 0,1,1 ; pending
+ */
+ for (;;) {
+ /*
+ * If we observe any contention; queue.
+ */
+ if (val & ~_Q_LOCKED_MASK)
+ goto queue;
+
+ new = _Q_LOCKED_VAL;
+ if (val == new)
+ new |= _Q_PENDING_VAL;
+
+ old = atomic_cmpxchg(&lock->val, val, new);
+ if (old == val)
+ break;
+
+ val = old;
+ }
+
+ /*
+ * we won the trylock
+ */
+ if (new == _Q_LOCKED_VAL)
+ return;
+
+ /*
+ * we're pending, wait for the owner to go away.
+ *
+ * *,1,1 -> *,1,0
+ */
+ while ((val = atomic_read(&lock->val)) & _Q_LOCKED_MASK)
+ arch_mutex_cpu_relax();
+
+ /*
+ * take ownership and clear the pending bit.
+ *
+ * *,1,0 -> *,0,1
+ */
+ for (;;) {
+ new = (val & ~_Q_PENDING_MASK) | _Q_LOCKED_VAL;
+
+ old = atomic_cmpxchg(&lock->val, val, new);
+ if (old == val)
+ break;
+
+ val = old;
+ }
+ return;
+
+ /*
+ * End of pending bit optimistic spinning and beginning of MCS
+ * queuing.
+ */
+queue:
node = this_cpu_ptr(&mcs_nodes[0]);
idx = node->count++;
tail = encode_tail(smp_processor_id(), idx);
@@ -119,15 +182,18 @@ void queue_spin_lock_slowpath(struct qspinlock *lock, u32 val)
node->next = NULL;
/*
+ * we already touched the queueing cacheline; don't bother with pending
+ * stuff.
+ *
* trylock || xchg(lock, node)
*
- * 0,0 -> 0,1 ; trylock
- * p,x -> n,x ; prev = xchg(lock, node)
+ * 0,0,0 -> 0,0,1 ; trylock
+ * p,y,x -> n,y,x ; prev = xchg(lock, node)
*/
for (;;) {
new = _Q_LOCKED_VAL;
if (val)
- new = tail | (val & _Q_LOCKED_MASK);
+ new = tail | (val & _Q_LOCKED_PENDING_MASK);
old = atomic_cmpxchg(&lock->val, val, new);
if (old == val)
@@ -145,7 +211,7 @@ void queue_spin_lock_slowpath(struct qspinlock *lock, u32 val)
/*
* if there was a previous node; link it and wait.
*/
- if (old & ~_Q_LOCKED_MASK) {
+ if (old & ~_Q_LOCKED_PENDING_MASK) {
prev = decode_tail(old);
ACCESS_ONCE(prev->next) = node;
@@ -153,18 +219,19 @@ void queue_spin_lock_slowpath(struct qspinlock *lock, u32 val)
}
/*
- * we're at the head of the waitqueue, wait for the owner to go away.
+ * we're at the head of the waitqueue, wait for the owner & pending to
+ * go away.
*
- * *,x -> *,0
+ * *,x,y -> *,0,0
*/
- while ((val = atomic_read(&lock->val)) & _Q_LOCKED_MASK)
+ while ((val = atomic_read(&lock->val)) & _Q_LOCKED_PENDING_MASK)
arch_mutex_cpu_relax();
/*
* claim the lock:
*
- * n,0 -> 0,1 : lock, uncontended
- * *,0 -> *,1 : lock, contended
+ * n,0,0 -> 0,0,1 : lock, uncontended
+ * *,0,0 -> *,0,1 : lock, contended
*/
for (;;) {
new = _Q_LOCKED_VAL;
--
1.7.1
^ permalink raw reply related [flat|nested] 102+ messages in thread* [PATCH v11 03/16] qspinlock: Add pending bit
@ 2014-05-30 15:43 ` Waiman Long
0 siblings, 0 replies; 102+ messages in thread
From: Waiman Long @ 2014-05-30 15:43 UTC (permalink / raw)
To: Thomas Gleixner, Ingo Molnar, H. Peter Anvin, Peter Zijlstra
Cc: linux-arch, Waiman Long, Rik van Riel, Raghavendra K T,
Gleb Natapov, kvm, Konrad Rzeszutek Wilk, Peter Zijlstra,
Scott J Norton, x86, Paolo Bonzini, linux-kernel, virtualization,
Chegu Vinod, David Vrabel, Oleg Nesterov, xen-devel,
Boris Ostrovsky, Paul E. McKenney, Linus Torvalds
From: Peter Zijlstra <peterz@infradead.org>
Because the qspinlock needs to touch a second cacheline; add a pending
bit and allow a single in-word spinner before we punt to the second
cacheline.
Signed-off-by: Peter Zijlstra <peterz@infradead.org>
Signed-off-by: Waiman Long <Waiman.Long@hp.com>
---
include/asm-generic/qspinlock_types.h | 12 +++-
kernel/locking/qspinlock.c | 109 ++++++++++++++++++++++++++------
2 files changed, 97 insertions(+), 24 deletions(-)
diff --git a/include/asm-generic/qspinlock_types.h b/include/asm-generic/qspinlock_types.h
index f66f845..bd25081 100644
--- a/include/asm-generic/qspinlock_types.h
+++ b/include/asm-generic/qspinlock_types.h
@@ -39,8 +39,9 @@ typedef struct qspinlock {
* Bitfields in the atomic value:
*
* 0- 7: locked byte
- * 8- 9: tail index
- * 10-31: tail cpu (+1)
+ * 8: pending
+ * 9-10: tail index
+ * 11-31: tail cpu (+1)
*/
#define _Q_SET_MASK(type) (((1U << _Q_ ## type ## _BITS) - 1)\
<< _Q_ ## type ## _OFFSET)
@@ -48,7 +49,11 @@ typedef struct qspinlock {
#define _Q_LOCKED_BITS 8
#define _Q_LOCKED_MASK _Q_SET_MASK(LOCKED)
-#define _Q_TAIL_IDX_OFFSET (_Q_LOCKED_OFFSET + _Q_LOCKED_BITS)
+#define _Q_PENDING_OFFSET (_Q_LOCKED_OFFSET + _Q_LOCKED_BITS)
+#define _Q_PENDING_BITS 1
+#define _Q_PENDING_MASK _Q_SET_MASK(PENDING)
+
+#define _Q_TAIL_IDX_OFFSET (_Q_PENDING_OFFSET + _Q_PENDING_BITS)
#define _Q_TAIL_IDX_BITS 2
#define _Q_TAIL_IDX_MASK _Q_SET_MASK(TAIL_IDX)
@@ -57,5 +62,6 @@ typedef struct qspinlock {
#define _Q_TAIL_CPU_MASK _Q_SET_MASK(TAIL_CPU)
#define _Q_LOCKED_VAL (1U << _Q_LOCKED_OFFSET)
+#define _Q_PENDING_VAL (1U << _Q_PENDING_OFFSET)
#endif /* __ASM_GENERIC_QSPINLOCK_TYPES_H */
diff --git a/kernel/locking/qspinlock.c b/kernel/locking/qspinlock.c
index b97a1ad..1e93c6a 100644
--- a/kernel/locking/qspinlock.c
+++ b/kernel/locking/qspinlock.c
@@ -83,24 +83,28 @@ static inline struct mcs_spinlock *decode_tail(u32 tail)
return per_cpu_ptr(&mcs_nodes[idx], cpu);
}
+#define _Q_LOCKED_PENDING_MASK (_Q_LOCKED_MASK | _Q_PENDING_MASK)
+
/**
* queue_spin_lock_slowpath - acquire the queue spinlock
* @lock: Pointer to queue spinlock structure
* @val: Current value of the queue spinlock 32-bit word
*
- * (queue tail, lock bit)
- *
- * fast : slow : unlock
- * : :
- * uncontended (0,0) --:--> (0,1) --------------------------------:--> (*,0)
- * : | ^--------. / :
- * : v \ | :
- * uncontended : (n,x) --+--> (n,0) | :
- * queue : | ^--' | :
- * : v | :
- * contended : (*,x) --+--> (*,0) -----> (*,1) ---' :
- * queue : ^--' :
+ * (queue tail, pending bit, lock bit)
*
+ * fast : slow : unlock
+ * : :
+ * uncontended (0,0,0) -:--> (0,0,1) ------------------------------:--> (*,*,0)
+ * : | ^--------.------. / :
+ * : v \ \ | :
+ * pending : (0,1,1) +--> (0,1,0) \ | :
+ * : | ^--' | | :
+ * : v | | :
+ * uncontended : (n,x,y) +--> (n,0,0) --' | :
+ * queue : | ^--' | :
+ * : v | :
+ * contended : (*,x,y) +--> (*,0,0) ---> (*,0,1) -' :
+ * queue : ^--' :
*/
void queue_spin_lock_slowpath(struct qspinlock *lock, u32 val)
{
@@ -110,6 +114,65 @@ void queue_spin_lock_slowpath(struct qspinlock *lock, u32 val)
BUILD_BUG_ON(CONFIG_NR_CPUS >= (1U << _Q_TAIL_CPU_BITS));
+ /*
+ * trylock || pending
+ *
+ * 0,0,0 -> 0,0,1 ; trylock
+ * 0,0,1 -> 0,1,1 ; pending
+ */
+ for (;;) {
+ /*
+ * If we observe any contention; queue.
+ */
+ if (val & ~_Q_LOCKED_MASK)
+ goto queue;
+
+ new = _Q_LOCKED_VAL;
+ if (val == new)
+ new |= _Q_PENDING_VAL;
+
+ old = atomic_cmpxchg(&lock->val, val, new);
+ if (old == val)
+ break;
+
+ val = old;
+ }
+
+ /*
+ * we won the trylock
+ */
+ if (new == _Q_LOCKED_VAL)
+ return;
+
+ /*
+ * we're pending, wait for the owner to go away.
+ *
+ * *,1,1 -> *,1,0
+ */
+ while ((val = atomic_read(&lock->val)) & _Q_LOCKED_MASK)
+ arch_mutex_cpu_relax();
+
+ /*
+ * take ownership and clear the pending bit.
+ *
+ * *,1,0 -> *,0,1
+ */
+ for (;;) {
+ new = (val & ~_Q_PENDING_MASK) | _Q_LOCKED_VAL;
+
+ old = atomic_cmpxchg(&lock->val, val, new);
+ if (old == val)
+ break;
+
+ val = old;
+ }
+ return;
+
+ /*
+ * End of pending bit optimistic spinning and beginning of MCS
+ * queuing.
+ */
+queue:
node = this_cpu_ptr(&mcs_nodes[0]);
idx = node->count++;
tail = encode_tail(smp_processor_id(), idx);
@@ -119,15 +182,18 @@ void queue_spin_lock_slowpath(struct qspinlock *lock, u32 val)
node->next = NULL;
/*
+ * we already touched the queueing cacheline; don't bother with pending
+ * stuff.
+ *
* trylock || xchg(lock, node)
*
- * 0,0 -> 0,1 ; trylock
- * p,x -> n,x ; prev = xchg(lock, node)
+ * 0,0,0 -> 0,0,1 ; trylock
+ * p,y,x -> n,y,x ; prev = xchg(lock, node)
*/
for (;;) {
new = _Q_LOCKED_VAL;
if (val)
- new = tail | (val & _Q_LOCKED_MASK);
+ new = tail | (val & _Q_LOCKED_PENDING_MASK);
old = atomic_cmpxchg(&lock->val, val, new);
if (old == val)
@@ -145,7 +211,7 @@ void queue_spin_lock_slowpath(struct qspinlock *lock, u32 val)
/*
* if there was a previous node; link it and wait.
*/
- if (old & ~_Q_LOCKED_MASK) {
+ if (old & ~_Q_LOCKED_PENDING_MASK) {
prev = decode_tail(old);
ACCESS_ONCE(prev->next) = node;
@@ -153,18 +219,19 @@ void queue_spin_lock_slowpath(struct qspinlock *lock, u32 val)
}
/*
- * we're at the head of the waitqueue, wait for the owner to go away.
+ * we're at the head of the waitqueue, wait for the owner & pending to
+ * go away.
*
- * *,x -> *,0
+ * *,x,y -> *,0,0
*/
- while ((val = atomic_read(&lock->val)) & _Q_LOCKED_MASK)
+ while ((val = atomic_read(&lock->val)) & _Q_LOCKED_PENDING_MASK)
arch_mutex_cpu_relax();
/*
* claim the lock:
*
- * n,0 -> 0,1 : lock, uncontended
- * *,0 -> *,1 : lock, contended
+ * n,0,0 -> 0,0,1 : lock, uncontended
+ * *,0,0 -> *,0,1 : lock, contended
*/
for (;;) {
new = _Q_LOCKED_VAL;
--
1.7.1
^ permalink raw reply related [flat|nested] 102+ messages in thread* [PATCH v11 03/16] qspinlock: Add pending bit
@ 2014-05-30 15:43 ` Waiman Long
0 siblings, 0 replies; 102+ messages in thread
From: Waiman Long @ 2014-05-30 15:43 UTC (permalink / raw)
To: Thomas Gleixner, Ingo Molnar, H. Peter Anvin, Peter Zijlstra
Cc: linux-arch, x86, linux-kernel, virtualization, xen-devel, kvm,
Paolo Bonzini, Konrad Rzeszutek Wilk, Boris Ostrovsky,
Paul E. McKenney, Rik van Riel, Linus Torvalds, Raghavendra K T,
David Vrabel, Oleg Nesterov, Gleb Natapov, Scott J Norton,
Chegu Vinod, Waiman Long
From: Peter Zijlstra <peterz@infradead.org>
Because the qspinlock needs to touch a second cacheline; add a pending
bit and allow a single in-word spinner before we punt to the second
cacheline.
Signed-off-by: Peter Zijlstra <peterz@infradead.org>
Signed-off-by: Waiman Long <Waiman.Long@hp.com>
---
include/asm-generic/qspinlock_types.h | 12 +++-
kernel/locking/qspinlock.c | 109 ++++++++++++++++++++++++++------
2 files changed, 97 insertions(+), 24 deletions(-)
diff --git a/include/asm-generic/qspinlock_types.h b/include/asm-generic/qspinlock_types.h
index f66f845..bd25081 100644
--- a/include/asm-generic/qspinlock_types.h
+++ b/include/asm-generic/qspinlock_types.h
@@ -39,8 +39,9 @@ typedef struct qspinlock {
* Bitfields in the atomic value:
*
* 0- 7: locked byte
- * 8- 9: tail index
- * 10-31: tail cpu (+1)
+ * 8: pending
+ * 9-10: tail index
+ * 11-31: tail cpu (+1)
*/
#define _Q_SET_MASK(type) (((1U << _Q_ ## type ## _BITS) - 1)\
<< _Q_ ## type ## _OFFSET)
@@ -48,7 +49,11 @@ typedef struct qspinlock {
#define _Q_LOCKED_BITS 8
#define _Q_LOCKED_MASK _Q_SET_MASK(LOCKED)
-#define _Q_TAIL_IDX_OFFSET (_Q_LOCKED_OFFSET + _Q_LOCKED_BITS)
+#define _Q_PENDING_OFFSET (_Q_LOCKED_OFFSET + _Q_LOCKED_BITS)
+#define _Q_PENDING_BITS 1
+#define _Q_PENDING_MASK _Q_SET_MASK(PENDING)
+
+#define _Q_TAIL_IDX_OFFSET (_Q_PENDING_OFFSET + _Q_PENDING_BITS)
#define _Q_TAIL_IDX_BITS 2
#define _Q_TAIL_IDX_MASK _Q_SET_MASK(TAIL_IDX)
@@ -57,5 +62,6 @@ typedef struct qspinlock {
#define _Q_TAIL_CPU_MASK _Q_SET_MASK(TAIL_CPU)
#define _Q_LOCKED_VAL (1U << _Q_LOCKED_OFFSET)
+#define _Q_PENDING_VAL (1U << _Q_PENDING_OFFSET)
#endif /* __ASM_GENERIC_QSPINLOCK_TYPES_H */
diff --git a/kernel/locking/qspinlock.c b/kernel/locking/qspinlock.c
index b97a1ad..1e93c6a 100644
--- a/kernel/locking/qspinlock.c
+++ b/kernel/locking/qspinlock.c
@@ -83,24 +83,28 @@ static inline struct mcs_spinlock *decode_tail(u32 tail)
return per_cpu_ptr(&mcs_nodes[idx], cpu);
}
+#define _Q_LOCKED_PENDING_MASK (_Q_LOCKED_MASK | _Q_PENDING_MASK)
+
/**
* queue_spin_lock_slowpath - acquire the queue spinlock
* @lock: Pointer to queue spinlock structure
* @val: Current value of the queue spinlock 32-bit word
*
- * (queue tail, lock bit)
- *
- * fast : slow : unlock
- * : :
- * uncontended (0,0) --:--> (0,1) --------------------------------:--> (*,0)
- * : | ^--------. / :
- * : v \ | :
- * uncontended : (n,x) --+--> (n,0) | :
- * queue : | ^--' | :
- * : v | :
- * contended : (*,x) --+--> (*,0) -----> (*,1) ---' :
- * queue : ^--' :
+ * (queue tail, pending bit, lock bit)
*
+ * fast : slow : unlock
+ * : :
+ * uncontended (0,0,0) -:--> (0,0,1) ------------------------------:--> (*,*,0)
+ * : | ^--------.------. / :
+ * : v \ \ | :
+ * pending : (0,1,1) +--> (0,1,0) \ | :
+ * : | ^--' | | :
+ * : v | | :
+ * uncontended : (n,x,y) +--> (n,0,0) --' | :
+ * queue : | ^--' | :
+ * : v | :
+ * contended : (*,x,y) +--> (*,0,0) ---> (*,0,1) -' :
+ * queue : ^--' :
*/
void queue_spin_lock_slowpath(struct qspinlock *lock, u32 val)
{
@@ -110,6 +114,65 @@ void queue_spin_lock_slowpath(struct qspinlock *lock, u32 val)
BUILD_BUG_ON(CONFIG_NR_CPUS >= (1U << _Q_TAIL_CPU_BITS));
+ /*
+ * trylock || pending
+ *
+ * 0,0,0 -> 0,0,1 ; trylock
+ * 0,0,1 -> 0,1,1 ; pending
+ */
+ for (;;) {
+ /*
+ * If we observe any contention; queue.
+ */
+ if (val & ~_Q_LOCKED_MASK)
+ goto queue;
+
+ new = _Q_LOCKED_VAL;
+ if (val == new)
+ new |= _Q_PENDING_VAL;
+
+ old = atomic_cmpxchg(&lock->val, val, new);
+ if (old == val)
+ break;
+
+ val = old;
+ }
+
+ /*
+ * we won the trylock
+ */
+ if (new == _Q_LOCKED_VAL)
+ return;
+
+ /*
+ * we're pending, wait for the owner to go away.
+ *
+ * *,1,1 -> *,1,0
+ */
+ while ((val = atomic_read(&lock->val)) & _Q_LOCKED_MASK)
+ arch_mutex_cpu_relax();
+
+ /*
+ * take ownership and clear the pending bit.
+ *
+ * *,1,0 -> *,0,1
+ */
+ for (;;) {
+ new = (val & ~_Q_PENDING_MASK) | _Q_LOCKED_VAL;
+
+ old = atomic_cmpxchg(&lock->val, val, new);
+ if (old == val)
+ break;
+
+ val = old;
+ }
+ return;
+
+ /*
+ * End of pending bit optimistic spinning and beginning of MCS
+ * queuing.
+ */
+queue:
node = this_cpu_ptr(&mcs_nodes[0]);
idx = node->count++;
tail = encode_tail(smp_processor_id(), idx);
@@ -119,15 +182,18 @@ void queue_spin_lock_slowpath(struct qspinlock *lock, u32 val)
node->next = NULL;
/*
+ * we already touched the queueing cacheline; don't bother with pending
+ * stuff.
+ *
* trylock || xchg(lock, node)
*
- * 0,0 -> 0,1 ; trylock
- * p,x -> n,x ; prev = xchg(lock, node)
+ * 0,0,0 -> 0,0,1 ; trylock
+ * p,y,x -> n,y,x ; prev = xchg(lock, node)
*/
for (;;) {
new = _Q_LOCKED_VAL;
if (val)
- new = tail | (val & _Q_LOCKED_MASK);
+ new = tail | (val & _Q_LOCKED_PENDING_MASK);
old = atomic_cmpxchg(&lock->val, val, new);
if (old == val)
@@ -145,7 +211,7 @@ void queue_spin_lock_slowpath(struct qspinlock *lock, u32 val)
/*
* if there was a previous node; link it and wait.
*/
- if (old & ~_Q_LOCKED_MASK) {
+ if (old & ~_Q_LOCKED_PENDING_MASK) {
prev = decode_tail(old);
ACCESS_ONCE(prev->next) = node;
@@ -153,18 +219,19 @@ void queue_spin_lock_slowpath(struct qspinlock *lock, u32 val)
}
/*
- * we're at the head of the waitqueue, wait for the owner to go away.
+ * we're at the head of the waitqueue, wait for the owner & pending to
+ * go away.
*
- * *,x -> *,0
+ * *,x,y -> *,0,0
*/
- while ((val = atomic_read(&lock->val)) & _Q_LOCKED_MASK)
+ while ((val = atomic_read(&lock->val)) & _Q_LOCKED_PENDING_MASK)
arch_mutex_cpu_relax();
/*
* claim the lock:
*
- * n,0 -> 0,1 : lock, uncontended
- * *,0 -> *,1 : lock, contended
+ * n,0,0 -> 0,0,1 : lock, uncontended
+ * *,0,0 -> *,0,1 : lock, contended
*/
for (;;) {
new = _Q_LOCKED_VAL;
--
1.7.1
^ permalink raw reply related [flat|nested] 102+ messages in thread* [PATCH v11 03/16] qspinlock: Add pending bit
@ 2014-05-30 15:43 ` Waiman Long
0 siblings, 0 replies; 102+ messages in thread
From: Waiman Long @ 2014-05-30 15:43 UTC (permalink / raw)
To: Thomas Gleixner, Ingo Molnar, H. Peter Anvin
Cc: linux-arch, Waiman Long, Rik van Riel, Raghavendra K T,
Gleb Natapov, kvm, Konrad Rzeszutek Wilk, Peter Zijlstra,
Scott J Norton, x86, Paolo Bonzini, linux-kernel, virtualization,
Chegu Vinod, David Vrabel, Oleg Nesterov, xen-devel,
Boris Ostrovsky, Paul E. McKenney, Linus Torvalds
From: Peter Zijlstra <peterz@infradead.org>
Because the qspinlock needs to touch a second cacheline; add a pending
bit and allow a single in-word spinner before we punt to the second
cacheline.
Signed-off-by: Peter Zijlstra <peterz@infradead.org>
Signed-off-by: Waiman Long <Waiman.Long@hp.com>
---
include/asm-generic/qspinlock_types.h | 12 +++-
kernel/locking/qspinlock.c | 109 ++++++++++++++++++++++++++------
2 files changed, 97 insertions(+), 24 deletions(-)
diff --git a/include/asm-generic/qspinlock_types.h b/include/asm-generic/qspinlock_types.h
index f66f845..bd25081 100644
--- a/include/asm-generic/qspinlock_types.h
+++ b/include/asm-generic/qspinlock_types.h
@@ -39,8 +39,9 @@ typedef struct qspinlock {
* Bitfields in the atomic value:
*
* 0- 7: locked byte
- * 8- 9: tail index
- * 10-31: tail cpu (+1)
+ * 8: pending
+ * 9-10: tail index
+ * 11-31: tail cpu (+1)
*/
#define _Q_SET_MASK(type) (((1U << _Q_ ## type ## _BITS) - 1)\
<< _Q_ ## type ## _OFFSET)
@@ -48,7 +49,11 @@ typedef struct qspinlock {
#define _Q_LOCKED_BITS 8
#define _Q_LOCKED_MASK _Q_SET_MASK(LOCKED)
-#define _Q_TAIL_IDX_OFFSET (_Q_LOCKED_OFFSET + _Q_LOCKED_BITS)
+#define _Q_PENDING_OFFSET (_Q_LOCKED_OFFSET + _Q_LOCKED_BITS)
+#define _Q_PENDING_BITS 1
+#define _Q_PENDING_MASK _Q_SET_MASK(PENDING)
+
+#define _Q_TAIL_IDX_OFFSET (_Q_PENDING_OFFSET + _Q_PENDING_BITS)
#define _Q_TAIL_IDX_BITS 2
#define _Q_TAIL_IDX_MASK _Q_SET_MASK(TAIL_IDX)
@@ -57,5 +62,6 @@ typedef struct qspinlock {
#define _Q_TAIL_CPU_MASK _Q_SET_MASK(TAIL_CPU)
#define _Q_LOCKED_VAL (1U << _Q_LOCKED_OFFSET)
+#define _Q_PENDING_VAL (1U << _Q_PENDING_OFFSET)
#endif /* __ASM_GENERIC_QSPINLOCK_TYPES_H */
diff --git a/kernel/locking/qspinlock.c b/kernel/locking/qspinlock.c
index b97a1ad..1e93c6a 100644
--- a/kernel/locking/qspinlock.c
+++ b/kernel/locking/qspinlock.c
@@ -83,24 +83,28 @@ static inline struct mcs_spinlock *decode_tail(u32 tail)
return per_cpu_ptr(&mcs_nodes[idx], cpu);
}
+#define _Q_LOCKED_PENDING_MASK (_Q_LOCKED_MASK | _Q_PENDING_MASK)
+
/**
* queue_spin_lock_slowpath - acquire the queue spinlock
* @lock: Pointer to queue spinlock structure
* @val: Current value of the queue spinlock 32-bit word
*
- * (queue tail, lock bit)
- *
- * fast : slow : unlock
- * : :
- * uncontended (0,0) --:--> (0,1) --------------------------------:--> (*,0)
- * : | ^--------. / :
- * : v \ | :
- * uncontended : (n,x) --+--> (n,0) | :
- * queue : | ^--' | :
- * : v | :
- * contended : (*,x) --+--> (*,0) -----> (*,1) ---' :
- * queue : ^--' :
+ * (queue tail, pending bit, lock bit)
*
+ * fast : slow : unlock
+ * : :
+ * uncontended (0,0,0) -:--> (0,0,1) ------------------------------:--> (*,*,0)
+ * : | ^--------.------. / :
+ * : v \ \ | :
+ * pending : (0,1,1) +--> (0,1,0) \ | :
+ * : | ^--' | | :
+ * : v | | :
+ * uncontended : (n,x,y) +--> (n,0,0) --' | :
+ * queue : | ^--' | :
+ * : v | :
+ * contended : (*,x,y) +--> (*,0,0) ---> (*,0,1) -' :
+ * queue : ^--' :
*/
void queue_spin_lock_slowpath(struct qspinlock *lock, u32 val)
{
@@ -110,6 +114,65 @@ void queue_spin_lock_slowpath(struct qspinlock *lock, u32 val)
BUILD_BUG_ON(CONFIG_NR_CPUS >= (1U << _Q_TAIL_CPU_BITS));
+ /*
+ * trylock || pending
+ *
+ * 0,0,0 -> 0,0,1 ; trylock
+ * 0,0,1 -> 0,1,1 ; pending
+ */
+ for (;;) {
+ /*
+ * If we observe any contention; queue.
+ */
+ if (val & ~_Q_LOCKED_MASK)
+ goto queue;
+
+ new = _Q_LOCKED_VAL;
+ if (val == new)
+ new |= _Q_PENDING_VAL;
+
+ old = atomic_cmpxchg(&lock->val, val, new);
+ if (old == val)
+ break;
+
+ val = old;
+ }
+
+ /*
+ * we won the trylock
+ */
+ if (new == _Q_LOCKED_VAL)
+ return;
+
+ /*
+ * we're pending, wait for the owner to go away.
+ *
+ * *,1,1 -> *,1,0
+ */
+ while ((val = atomic_read(&lock->val)) & _Q_LOCKED_MASK)
+ arch_mutex_cpu_relax();
+
+ /*
+ * take ownership and clear the pending bit.
+ *
+ * *,1,0 -> *,0,1
+ */
+ for (;;) {
+ new = (val & ~_Q_PENDING_MASK) | _Q_LOCKED_VAL;
+
+ old = atomic_cmpxchg(&lock->val, val, new);
+ if (old == val)
+ break;
+
+ val = old;
+ }
+ return;
+
+ /*
+ * End of pending bit optimistic spinning and beginning of MCS
+ * queuing.
+ */
+queue:
node = this_cpu_ptr(&mcs_nodes[0]);
idx = node->count++;
tail = encode_tail(smp_processor_id(), idx);
@@ -119,15 +182,18 @@ void queue_spin_lock_slowpath(struct qspinlock *lock, u32 val)
node->next = NULL;
/*
+ * we already touched the queueing cacheline; don't bother with pending
+ * stuff.
+ *
* trylock || xchg(lock, node)
*
- * 0,0 -> 0,1 ; trylock
- * p,x -> n,x ; prev = xchg(lock, node)
+ * 0,0,0 -> 0,0,1 ; trylock
+ * p,y,x -> n,y,x ; prev = xchg(lock, node)
*/
for (;;) {
new = _Q_LOCKED_VAL;
if (val)
- new = tail | (val & _Q_LOCKED_MASK);
+ new = tail | (val & _Q_LOCKED_PENDING_MASK);
old = atomic_cmpxchg(&lock->val, val, new);
if (old == val)
@@ -145,7 +211,7 @@ void queue_spin_lock_slowpath(struct qspinlock *lock, u32 val)
/*
* if there was a previous node; link it and wait.
*/
- if (old & ~_Q_LOCKED_MASK) {
+ if (old & ~_Q_LOCKED_PENDING_MASK) {
prev = decode_tail(old);
ACCESS_ONCE(prev->next) = node;
@@ -153,18 +219,19 @@ void queue_spin_lock_slowpath(struct qspinlock *lock, u32 val)
}
/*
- * we're at the head of the waitqueue, wait for the owner to go away.
+ * we're at the head of the waitqueue, wait for the owner & pending to
+ * go away.
*
- * *,x -> *,0
+ * *,x,y -> *,0,0
*/
- while ((val = atomic_read(&lock->val)) & _Q_LOCKED_MASK)
+ while ((val = atomic_read(&lock->val)) & _Q_LOCKED_PENDING_MASK)
arch_mutex_cpu_relax();
/*
* claim the lock:
*
- * n,0 -> 0,1 : lock, uncontended
- * *,0 -> *,1 : lock, contended
+ * n,0,0 -> 0,0,1 : lock, uncontended
+ * *,0,0 -> *,0,1 : lock, contended
*/
for (;;) {
new = _Q_LOCKED_VAL;
--
1.7.1
^ permalink raw reply related [flat|nested] 102+ messages in thread
* [PATCH v11 03/16] qspinlock: Add pending bit
2014-05-30 15:43 [PATCH v11 00/16] qspinlock: a 4-byte queue spinlock with PV support Waiman Long
` (4 preceding siblings ...)
2014-05-30 15:43 ` Waiman Long
@ 2014-05-30 15:43 ` Waiman Long
2014-05-30 15:43 ` [PATCH v11 04/16] qspinlock: Extract out the exchange of tail code word Waiman Long
` (31 subsequent siblings)
37 siblings, 0 replies; 102+ messages in thread
From: Waiman Long @ 2014-05-30 15:43 UTC (permalink / raw)
To: Thomas Gleixner, Ingo Molnar, H. Peter Anvin
Cc: linux-arch, Waiman Long, Raghavendra K T, Gleb Natapov, kvm,
Peter Zijlstra, Scott J Norton, x86, Paolo Bonzini, linux-kernel,
virtualization, Chegu Vinod, David Vrabel, Oleg Nesterov,
xen-devel, Boris Ostrovsky, Paul E. McKenney, Linus Torvalds
From: Peter Zijlstra <peterz@infradead.org>
Because the qspinlock needs to touch a second cacheline; add a pending
bit and allow a single in-word spinner before we punt to the second
cacheline.
Signed-off-by: Peter Zijlstra <peterz@infradead.org>
Signed-off-by: Waiman Long <Waiman.Long@hp.com>
---
include/asm-generic/qspinlock_types.h | 12 +++-
kernel/locking/qspinlock.c | 109 ++++++++++++++++++++++++++------
2 files changed, 97 insertions(+), 24 deletions(-)
diff --git a/include/asm-generic/qspinlock_types.h b/include/asm-generic/qspinlock_types.h
index f66f845..bd25081 100644
--- a/include/asm-generic/qspinlock_types.h
+++ b/include/asm-generic/qspinlock_types.h
@@ -39,8 +39,9 @@ typedef struct qspinlock {
* Bitfields in the atomic value:
*
* 0- 7: locked byte
- * 8- 9: tail index
- * 10-31: tail cpu (+1)
+ * 8: pending
+ * 9-10: tail index
+ * 11-31: tail cpu (+1)
*/
#define _Q_SET_MASK(type) (((1U << _Q_ ## type ## _BITS) - 1)\
<< _Q_ ## type ## _OFFSET)
@@ -48,7 +49,11 @@ typedef struct qspinlock {
#define _Q_LOCKED_BITS 8
#define _Q_LOCKED_MASK _Q_SET_MASK(LOCKED)
-#define _Q_TAIL_IDX_OFFSET (_Q_LOCKED_OFFSET + _Q_LOCKED_BITS)
+#define _Q_PENDING_OFFSET (_Q_LOCKED_OFFSET + _Q_LOCKED_BITS)
+#define _Q_PENDING_BITS 1
+#define _Q_PENDING_MASK _Q_SET_MASK(PENDING)
+
+#define _Q_TAIL_IDX_OFFSET (_Q_PENDING_OFFSET + _Q_PENDING_BITS)
#define _Q_TAIL_IDX_BITS 2
#define _Q_TAIL_IDX_MASK _Q_SET_MASK(TAIL_IDX)
@@ -57,5 +62,6 @@ typedef struct qspinlock {
#define _Q_TAIL_CPU_MASK _Q_SET_MASK(TAIL_CPU)
#define _Q_LOCKED_VAL (1U << _Q_LOCKED_OFFSET)
+#define _Q_PENDING_VAL (1U << _Q_PENDING_OFFSET)
#endif /* __ASM_GENERIC_QSPINLOCK_TYPES_H */
diff --git a/kernel/locking/qspinlock.c b/kernel/locking/qspinlock.c
index b97a1ad..1e93c6a 100644
--- a/kernel/locking/qspinlock.c
+++ b/kernel/locking/qspinlock.c
@@ -83,24 +83,28 @@ static inline struct mcs_spinlock *decode_tail(u32 tail)
return per_cpu_ptr(&mcs_nodes[idx], cpu);
}
+#define _Q_LOCKED_PENDING_MASK (_Q_LOCKED_MASK | _Q_PENDING_MASK)
+
/**
* queue_spin_lock_slowpath - acquire the queue spinlock
* @lock: Pointer to queue spinlock structure
* @val: Current value of the queue spinlock 32-bit word
*
- * (queue tail, lock bit)
- *
- * fast : slow : unlock
- * : :
- * uncontended (0,0) --:--> (0,1) --------------------------------:--> (*,0)
- * : | ^--------. / :
- * : v \ | :
- * uncontended : (n,x) --+--> (n,0) | :
- * queue : | ^--' | :
- * : v | :
- * contended : (*,x) --+--> (*,0) -----> (*,1) ---' :
- * queue : ^--' :
+ * (queue tail, pending bit, lock bit)
*
+ * fast : slow : unlock
+ * : :
+ * uncontended (0,0,0) -:--> (0,0,1) ------------------------------:--> (*,*,0)
+ * : | ^--------.------. / :
+ * : v \ \ | :
+ * pending : (0,1,1) +--> (0,1,0) \ | :
+ * : | ^--' | | :
+ * : v | | :
+ * uncontended : (n,x,y) +--> (n,0,0) --' | :
+ * queue : | ^--' | :
+ * : v | :
+ * contended : (*,x,y) +--> (*,0,0) ---> (*,0,1) -' :
+ * queue : ^--' :
*/
void queue_spin_lock_slowpath(struct qspinlock *lock, u32 val)
{
@@ -110,6 +114,65 @@ void queue_spin_lock_slowpath(struct qspinlock *lock, u32 val)
BUILD_BUG_ON(CONFIG_NR_CPUS >= (1U << _Q_TAIL_CPU_BITS));
+ /*
+ * trylock || pending
+ *
+ * 0,0,0 -> 0,0,1 ; trylock
+ * 0,0,1 -> 0,1,1 ; pending
+ */
+ for (;;) {
+ /*
+ * If we observe any contention; queue.
+ */
+ if (val & ~_Q_LOCKED_MASK)
+ goto queue;
+
+ new = _Q_LOCKED_VAL;
+ if (val == new)
+ new |= _Q_PENDING_VAL;
+
+ old = atomic_cmpxchg(&lock->val, val, new);
+ if (old == val)
+ break;
+
+ val = old;
+ }
+
+ /*
+ * we won the trylock
+ */
+ if (new == _Q_LOCKED_VAL)
+ return;
+
+ /*
+ * we're pending, wait for the owner to go away.
+ *
+ * *,1,1 -> *,1,0
+ */
+ while ((val = atomic_read(&lock->val)) & _Q_LOCKED_MASK)
+ arch_mutex_cpu_relax();
+
+ /*
+ * take ownership and clear the pending bit.
+ *
+ * *,1,0 -> *,0,1
+ */
+ for (;;) {
+ new = (val & ~_Q_PENDING_MASK) | _Q_LOCKED_VAL;
+
+ old = atomic_cmpxchg(&lock->val, val, new);
+ if (old == val)
+ break;
+
+ val = old;
+ }
+ return;
+
+ /*
+ * End of pending bit optimistic spinning and beginning of MCS
+ * queuing.
+ */
+queue:
node = this_cpu_ptr(&mcs_nodes[0]);
idx = node->count++;
tail = encode_tail(smp_processor_id(), idx);
@@ -119,15 +182,18 @@ void queue_spin_lock_slowpath(struct qspinlock *lock, u32 val)
node->next = NULL;
/*
+ * we already touched the queueing cacheline; don't bother with pending
+ * stuff.
+ *
* trylock || xchg(lock, node)
*
- * 0,0 -> 0,1 ; trylock
- * p,x -> n,x ; prev = xchg(lock, node)
+ * 0,0,0 -> 0,0,1 ; trylock
+ * p,y,x -> n,y,x ; prev = xchg(lock, node)
*/
for (;;) {
new = _Q_LOCKED_VAL;
if (val)
- new = tail | (val & _Q_LOCKED_MASK);
+ new = tail | (val & _Q_LOCKED_PENDING_MASK);
old = atomic_cmpxchg(&lock->val, val, new);
if (old == val)
@@ -145,7 +211,7 @@ void queue_spin_lock_slowpath(struct qspinlock *lock, u32 val)
/*
* if there was a previous node; link it and wait.
*/
- if (old & ~_Q_LOCKED_MASK) {
+ if (old & ~_Q_LOCKED_PENDING_MASK) {
prev = decode_tail(old);
ACCESS_ONCE(prev->next) = node;
@@ -153,18 +219,19 @@ void queue_spin_lock_slowpath(struct qspinlock *lock, u32 val)
}
/*
- * we're at the head of the waitqueue, wait for the owner to go away.
+ * we're at the head of the waitqueue, wait for the owner & pending to
+ * go away.
*
- * *,x -> *,0
+ * *,x,y -> *,0,0
*/
- while ((val = atomic_read(&lock->val)) & _Q_LOCKED_MASK)
+ while ((val = atomic_read(&lock->val)) & _Q_LOCKED_PENDING_MASK)
arch_mutex_cpu_relax();
/*
* claim the lock:
*
- * n,0 -> 0,1 : lock, uncontended
- * *,0 -> *,1 : lock, contended
+ * n,0,0 -> 0,0,1 : lock, uncontended
+ * *,0,0 -> *,0,1 : lock, contended
*/
for (;;) {
new = _Q_LOCKED_VAL;
--
1.7.1
^ permalink raw reply related [flat|nested] 102+ messages in thread* [PATCH v11 04/16] qspinlock: Extract out the exchange of tail code word
2014-05-30 15:43 [PATCH v11 00/16] qspinlock: a 4-byte queue spinlock with PV support Waiman Long
` (5 preceding siblings ...)
2014-05-30 15:43 ` Waiman Long
@ 2014-05-30 15:43 ` Waiman Long
2014-05-30 15:43 ` Waiman Long
` (30 subsequent siblings)
37 siblings, 0 replies; 102+ messages in thread
From: Waiman Long @ 2014-05-30 15:43 UTC (permalink / raw)
To: Thomas Gleixner, Ingo Molnar, H. Peter Anvin, Peter Zijlstra
Cc: linux-arch, x86, linux-kernel, virtualization, xen-devel, kvm,
Paolo Bonzini, Konrad Rzeszutek Wilk, Boris Ostrovsky,
Paul E. McKenney, Rik van Riel, Linus Torvalds, Raghavendra K T,
David Vrabel, Oleg Nesterov, Gleb Natapov, Scott J Norton,
Chegu Vinod, Waiman Long
This patch extracts the logic for the exchange of new and previous tail
code words into a new xchg_tail() function which can be optimized in a
later patch.
Signed-off-by: Waiman Long <Waiman.Long@hp.com>
---
include/asm-generic/qspinlock_types.h | 2 +
kernel/locking/qspinlock.c | 58 ++++++++++++++++++++------------
2 files changed, 38 insertions(+), 22 deletions(-)
diff --git a/include/asm-generic/qspinlock_types.h b/include/asm-generic/qspinlock_types.h
index bd25081..ed5d89a 100644
--- a/include/asm-generic/qspinlock_types.h
+++ b/include/asm-generic/qspinlock_types.h
@@ -61,6 +61,8 @@ typedef struct qspinlock {
#define _Q_TAIL_CPU_BITS (32 - _Q_TAIL_CPU_OFFSET)
#define _Q_TAIL_CPU_MASK _Q_SET_MASK(TAIL_CPU)
+#define _Q_TAIL_MASK (_Q_TAIL_IDX_MASK | _Q_TAIL_CPU_MASK)
+
#define _Q_LOCKED_VAL (1U << _Q_LOCKED_OFFSET)
#define _Q_PENDING_VAL (1U << _Q_PENDING_OFFSET)
diff --git a/kernel/locking/qspinlock.c b/kernel/locking/qspinlock.c
index 1e93c6a..41594a1 100644
--- a/kernel/locking/qspinlock.c
+++ b/kernel/locking/qspinlock.c
@@ -86,6 +86,31 @@ static inline struct mcs_spinlock *decode_tail(u32 tail)
#define _Q_LOCKED_PENDING_MASK (_Q_LOCKED_MASK | _Q_PENDING_MASK)
/**
+ * xchg_tail - Put in the new queue tail code word & retrieve previous one
+ * @lock : Pointer to queue spinlock structure
+ * @tail : The new queue tail code word
+ * Return: The previous queue tail code word
+ *
+ * xchg(lock, tail)
+ *
+ * p,*,* -> n,*,* ; prev = xchg(lock, node)
+ */
+static __always_inline u32 xchg_tail(struct qspinlock *lock, u32 tail)
+{
+ u32 old, new, val = atomic_read(&lock->val);
+
+ for (;;) {
+ new = (val & _Q_LOCKED_PENDING_MASK) | tail;
+ old = atomic_cmpxchg(&lock->val, val, new);
+ if (old == val)
+ break;
+
+ val = old;
+ }
+ return old;
+}
+
+/**
* queue_spin_lock_slowpath - acquire the queue spinlock
* @lock: Pointer to queue spinlock structure
* @val: Current value of the queue spinlock 32-bit word
@@ -182,36 +207,25 @@ queue:
node->next = NULL;
/*
- * we already touched the queueing cacheline; don't bother with pending
- * stuff.
- *
- * trylock || xchg(lock, node)
- *
- * 0,0,0 -> 0,0,1 ; trylock
- * p,y,x -> n,y,x ; prev = xchg(lock, node)
+ * We touched a (possibly) cold cacheline in the per-cpu queue node;
+ * attempt the trylock once more in the hope someone let go while we
+ * weren't watching.
*/
- for (;;) {
- new = _Q_LOCKED_VAL;
- if (val)
- new = tail | (val & _Q_LOCKED_PENDING_MASK);
-
- old = atomic_cmpxchg(&lock->val, val, new);
- if (old == val)
- break;
-
- val = old;
- }
+ if (queue_spin_trylock(lock))
+ goto release;
/*
- * we won the trylock; forget about queueing.
+ * we already touched the queueing cacheline; don't bother with pending
+ * stuff.
+ *
+ * p,*,* -> n,*,*
*/
- if (new == _Q_LOCKED_VAL)
- goto release;
+ old = xchg_tail(lock, tail);
/*
* if there was a previous node; link it and wait.
*/
- if (old & ~_Q_LOCKED_PENDING_MASK) {
+ if (old & _Q_TAIL_MASK) {
prev = decode_tail(old);
ACCESS_ONCE(prev->next) = node;
--
1.7.1
^ permalink raw reply related [flat|nested] 102+ messages in thread* [PATCH v11 04/16] qspinlock: Extract out the exchange of tail code word
2014-05-30 15:43 [PATCH v11 00/16] qspinlock: a 4-byte queue spinlock with PV support Waiman Long
` (6 preceding siblings ...)
2014-05-30 15:43 ` [PATCH v11 04/16] qspinlock: Extract out the exchange of tail code word Waiman Long
@ 2014-05-30 15:43 ` Waiman Long
2014-05-30 15:43 ` Waiman Long
` (29 subsequent siblings)
37 siblings, 0 replies; 102+ messages in thread
From: Waiman Long @ 2014-05-30 15:43 UTC (permalink / raw)
To: Thomas Gleixner, Ingo Molnar, H. Peter Anvin, Peter Zijlstra
Cc: linux-arch, Waiman Long, Raghavendra K T, Gleb Natapov, kvm,
Scott J Norton, x86, Paolo Bonzini, linux-kernel, virtualization,
Chegu Vinod, David Vrabel, Oleg Nesterov, xen-devel,
Boris Ostrovsky, Paul E. McKenney, Linus Torvalds
This patch extracts the logic for the exchange of new and previous tail
code words into a new xchg_tail() function which can be optimized in a
later patch.
Signed-off-by: Waiman Long <Waiman.Long@hp.com>
---
include/asm-generic/qspinlock_types.h | 2 +
kernel/locking/qspinlock.c | 58 ++++++++++++++++++++------------
2 files changed, 38 insertions(+), 22 deletions(-)
diff --git a/include/asm-generic/qspinlock_types.h b/include/asm-generic/qspinlock_types.h
index bd25081..ed5d89a 100644
--- a/include/asm-generic/qspinlock_types.h
+++ b/include/asm-generic/qspinlock_types.h
@@ -61,6 +61,8 @@ typedef struct qspinlock {
#define _Q_TAIL_CPU_BITS (32 - _Q_TAIL_CPU_OFFSET)
#define _Q_TAIL_CPU_MASK _Q_SET_MASK(TAIL_CPU)
+#define _Q_TAIL_MASK (_Q_TAIL_IDX_MASK | _Q_TAIL_CPU_MASK)
+
#define _Q_LOCKED_VAL (1U << _Q_LOCKED_OFFSET)
#define _Q_PENDING_VAL (1U << _Q_PENDING_OFFSET)
diff --git a/kernel/locking/qspinlock.c b/kernel/locking/qspinlock.c
index 1e93c6a..41594a1 100644
--- a/kernel/locking/qspinlock.c
+++ b/kernel/locking/qspinlock.c
@@ -86,6 +86,31 @@ static inline struct mcs_spinlock *decode_tail(u32 tail)
#define _Q_LOCKED_PENDING_MASK (_Q_LOCKED_MASK | _Q_PENDING_MASK)
/**
+ * xchg_tail - Put in the new queue tail code word & retrieve previous one
+ * @lock : Pointer to queue spinlock structure
+ * @tail : The new queue tail code word
+ * Return: The previous queue tail code word
+ *
+ * xchg(lock, tail)
+ *
+ * p,*,* -> n,*,* ; prev = xchg(lock, node)
+ */
+static __always_inline u32 xchg_tail(struct qspinlock *lock, u32 tail)
+{
+ u32 old, new, val = atomic_read(&lock->val);
+
+ for (;;) {
+ new = (val & _Q_LOCKED_PENDING_MASK) | tail;
+ old = atomic_cmpxchg(&lock->val, val, new);
+ if (old == val)
+ break;
+
+ val = old;
+ }
+ return old;
+}
+
+/**
* queue_spin_lock_slowpath - acquire the queue spinlock
* @lock: Pointer to queue spinlock structure
* @val: Current value of the queue spinlock 32-bit word
@@ -182,36 +207,25 @@ queue:
node->next = NULL;
/*
- * we already touched the queueing cacheline; don't bother with pending
- * stuff.
- *
- * trylock || xchg(lock, node)
- *
- * 0,0,0 -> 0,0,1 ; trylock
- * p,y,x -> n,y,x ; prev = xchg(lock, node)
+ * We touched a (possibly) cold cacheline in the per-cpu queue node;
+ * attempt the trylock once more in the hope someone let go while we
+ * weren't watching.
*/
- for (;;) {
- new = _Q_LOCKED_VAL;
- if (val)
- new = tail | (val & _Q_LOCKED_PENDING_MASK);
-
- old = atomic_cmpxchg(&lock->val, val, new);
- if (old == val)
- break;
-
- val = old;
- }
+ if (queue_spin_trylock(lock))
+ goto release;
/*
- * we won the trylock; forget about queueing.
+ * we already touched the queueing cacheline; don't bother with pending
+ * stuff.
+ *
+ * p,*,* -> n,*,*
*/
- if (new == _Q_LOCKED_VAL)
- goto release;
+ old = xchg_tail(lock, tail);
/*
* if there was a previous node; link it and wait.
*/
- if (old & ~_Q_LOCKED_PENDING_MASK) {
+ if (old & _Q_TAIL_MASK) {
prev = decode_tail(old);
ACCESS_ONCE(prev->next) = node;
--
1.7.1
^ permalink raw reply related [flat|nested] 102+ messages in thread* [PATCH v11 04/16] qspinlock: Extract out the exchange of tail code word
2014-05-30 15:43 [PATCH v11 00/16] qspinlock: a 4-byte queue spinlock with PV support Waiman Long
` (7 preceding siblings ...)
2014-05-30 15:43 ` Waiman Long
@ 2014-05-30 15:43 ` Waiman Long
2014-05-30 15:43 ` Waiman Long
` (28 subsequent siblings)
37 siblings, 0 replies; 102+ messages in thread
From: Waiman Long @ 2014-05-30 15:43 UTC (permalink / raw)
To: Thomas Gleixner, Ingo Molnar, H. Peter Anvin, Peter Zijlstra
Cc: linux-arch, Waiman Long, Rik van Riel, Raghavendra K T,
Gleb Natapov, kvm, Konrad Rzeszutek Wilk, Scott J Norton, x86,
Paolo Bonzini, linux-kernel, virtualization, Chegu Vinod,
David Vrabel, Oleg Nesterov, xen-devel, Boris Ostrovsky,
Paul E. McKenney, Linus Torvalds
This patch extracts the logic for the exchange of new and previous tail
code words into a new xchg_tail() function which can be optimized in a
later patch.
Signed-off-by: Waiman Long <Waiman.Long@hp.com>
---
include/asm-generic/qspinlock_types.h | 2 +
kernel/locking/qspinlock.c | 58 ++++++++++++++++++++------------
2 files changed, 38 insertions(+), 22 deletions(-)
diff --git a/include/asm-generic/qspinlock_types.h b/include/asm-generic/qspinlock_types.h
index bd25081..ed5d89a 100644
--- a/include/asm-generic/qspinlock_types.h
+++ b/include/asm-generic/qspinlock_types.h
@@ -61,6 +61,8 @@ typedef struct qspinlock {
#define _Q_TAIL_CPU_BITS (32 - _Q_TAIL_CPU_OFFSET)
#define _Q_TAIL_CPU_MASK _Q_SET_MASK(TAIL_CPU)
+#define _Q_TAIL_MASK (_Q_TAIL_IDX_MASK | _Q_TAIL_CPU_MASK)
+
#define _Q_LOCKED_VAL (1U << _Q_LOCKED_OFFSET)
#define _Q_PENDING_VAL (1U << _Q_PENDING_OFFSET)
diff --git a/kernel/locking/qspinlock.c b/kernel/locking/qspinlock.c
index 1e93c6a..41594a1 100644
--- a/kernel/locking/qspinlock.c
+++ b/kernel/locking/qspinlock.c
@@ -86,6 +86,31 @@ static inline struct mcs_spinlock *decode_tail(u32 tail)
#define _Q_LOCKED_PENDING_MASK (_Q_LOCKED_MASK | _Q_PENDING_MASK)
/**
+ * xchg_tail - Put in the new queue tail code word & retrieve previous one
+ * @lock : Pointer to queue spinlock structure
+ * @tail : The new queue tail code word
+ * Return: The previous queue tail code word
+ *
+ * xchg(lock, tail)
+ *
+ * p,*,* -> n,*,* ; prev = xchg(lock, node)
+ */
+static __always_inline u32 xchg_tail(struct qspinlock *lock, u32 tail)
+{
+ u32 old, new, val = atomic_read(&lock->val);
+
+ for (;;) {
+ new = (val & _Q_LOCKED_PENDING_MASK) | tail;
+ old = atomic_cmpxchg(&lock->val, val, new);
+ if (old == val)
+ break;
+
+ val = old;
+ }
+ return old;
+}
+
+/**
* queue_spin_lock_slowpath - acquire the queue spinlock
* @lock: Pointer to queue spinlock structure
* @val: Current value of the queue spinlock 32-bit word
@@ -182,36 +207,25 @@ queue:
node->next = NULL;
/*
- * we already touched the queueing cacheline; don't bother with pending
- * stuff.
- *
- * trylock || xchg(lock, node)
- *
- * 0,0,0 -> 0,0,1 ; trylock
- * p,y,x -> n,y,x ; prev = xchg(lock, node)
+ * We touched a (possibly) cold cacheline in the per-cpu queue node;
+ * attempt the trylock once more in the hope someone let go while we
+ * weren't watching.
*/
- for (;;) {
- new = _Q_LOCKED_VAL;
- if (val)
- new = tail | (val & _Q_LOCKED_PENDING_MASK);
-
- old = atomic_cmpxchg(&lock->val, val, new);
- if (old == val)
- break;
-
- val = old;
- }
+ if (queue_spin_trylock(lock))
+ goto release;
/*
- * we won the trylock; forget about queueing.
+ * we already touched the queueing cacheline; don't bother with pending
+ * stuff.
+ *
+ * p,*,* -> n,*,*
*/
- if (new == _Q_LOCKED_VAL)
- goto release;
+ old = xchg_tail(lock, tail);
/*
* if there was a previous node; link it and wait.
*/
- if (old & ~_Q_LOCKED_PENDING_MASK) {
+ if (old & _Q_TAIL_MASK) {
prev = decode_tail(old);
ACCESS_ONCE(prev->next) = node;
--
1.7.1
^ permalink raw reply related [flat|nested] 102+ messages in thread* [PATCH v11 05/16] qspinlock: Optimize for smaller NR_CPUS
2014-05-30 15:43 [PATCH v11 00/16] qspinlock: a 4-byte queue spinlock with PV support Waiman Long
2014-05-30 15:43 ` Waiman Long
2014-05-30 15:43 ` Waiman Long
@ 2014-05-30 15:43 ` Waiman Long
2014-05-30 15:43 ` Waiman Long
` (34 subsequent siblings)
37 siblings, 0 replies; 102+ messages in thread
From: Waiman Long @ 2014-05-30 15:43 UTC (permalink / raw)
To: Thomas Gleixner, Ingo Molnar, H. Peter Anvin, Peter Zijlstra
Cc: linux-arch, x86, linux-kernel, virtualization, xen-devel, kvm,
Paolo Bonzini, Konrad Rzeszutek Wilk, Boris Ostrovsky,
Paul E. McKenney, Rik van Riel, Linus Torvalds, Raghavendra K T,
David Vrabel, Oleg Nesterov, Gleb Natapov, Scott J Norton,
Chegu Vinod, Peter Zijlstra, Waiman Long
From: Peter Zijlstra <peterz@infradead.org>
When we allow for a max NR_CPUS < 2^14 we can optimize the pending
wait-acquire and the xchg_tail() operations.
By growing the pending bit to a byte, we reduce the tail to 16bit.
This means we can use xchg16 for the tail part and do away with all
the repeated compxchg() operations.
This in turn allows us to unconditionally acquire; the locked state
as observed by the wait loops cannot change. And because both locked
and pending are now a full byte we can use simple stores for the
state transition, obviating one atomic operation entirely.
All this is horribly broken on Alpha pre EV56 (and any other arch that
cannot do single-copy atomic byte stores).
Signed-off-by: Peter Zijlstra <peterz@infradead.org>
Signed-off-by: Waiman Long <Waiman.Long@hp.com>
---
include/asm-generic/qspinlock_types.h | 13 ++++
kernel/locking/qspinlock.c | 103 +++++++++++++++++++++++++++++---
2 files changed, 106 insertions(+), 10 deletions(-)
diff --git a/include/asm-generic/qspinlock_types.h b/include/asm-generic/qspinlock_types.h
index ed5d89a..4914abe 100644
--- a/include/asm-generic/qspinlock_types.h
+++ b/include/asm-generic/qspinlock_types.h
@@ -38,6 +38,14 @@ typedef struct qspinlock {
/*
* Bitfields in the atomic value:
*
+ * When NR_CPUS < 16K
+ * 0- 7: locked byte
+ * 8: pending
+ * 9-15: not used
+ * 16-17: tail index
+ * 18-31: tail cpu (+1)
+ *
+ * When NR_CPUS >= 16K
* 0- 7: locked byte
* 8: pending
* 9-10: tail index
@@ -50,7 +58,11 @@ typedef struct qspinlock {
#define _Q_LOCKED_MASK _Q_SET_MASK(LOCKED)
#define _Q_PENDING_OFFSET (_Q_LOCKED_OFFSET + _Q_LOCKED_BITS)
+#if CONFIG_NR_CPUS < (1U << 14)
+#define _Q_PENDING_BITS 8
+#else
#define _Q_PENDING_BITS 1
+#endif
#define _Q_PENDING_MASK _Q_SET_MASK(PENDING)
#define _Q_TAIL_IDX_OFFSET (_Q_PENDING_OFFSET + _Q_PENDING_BITS)
@@ -61,6 +73,7 @@ typedef struct qspinlock {
#define _Q_TAIL_CPU_BITS (32 - _Q_TAIL_CPU_OFFSET)
#define _Q_TAIL_CPU_MASK _Q_SET_MASK(TAIL_CPU)
+#define _Q_TAIL_OFFSET _Q_TAIL_IDX_OFFSET
#define _Q_TAIL_MASK (_Q_TAIL_IDX_MASK | _Q_TAIL_CPU_MASK)
#define _Q_LOCKED_VAL (1U << _Q_LOCKED_OFFSET)
diff --git a/kernel/locking/qspinlock.c b/kernel/locking/qspinlock.c
index 41594a1..fc7fd8c 100644
--- a/kernel/locking/qspinlock.c
+++ b/kernel/locking/qspinlock.c
@@ -22,6 +22,7 @@
#include <linux/percpu.h>
#include <linux/hardirq.h>
#include <linux/mutex.h>
+#include <asm/byteorder.h>
#include <asm/qspinlock.h>
/*
@@ -48,6 +49,9 @@
* We can further change the first spinner to spin on a bit in the lock word
* instead of its node; whereby avoiding the need to carry a node from lock to
* unlock, and preserving API.
+ *
+ * N.B. The current implementation only supports architectures that allow
+ * atomic operations on smaller 8-bit and 16-bit data types.
*/
#include "mcs_spinlock.h"
@@ -85,6 +89,87 @@ static inline struct mcs_spinlock *decode_tail(u32 tail)
#define _Q_LOCKED_PENDING_MASK (_Q_LOCKED_MASK | _Q_PENDING_MASK)
+/*
+ * By using the whole 2nd least significant byte for the pending bit, we
+ * can allow better optimization of the lock acquisition for the pending
+ * bit holder.
+ */
+#if _Q_PENDING_BITS == 8
+
+struct __qspinlock {
+ union {
+ atomic_t val;
+ struct {
+#ifdef __LITTLE_ENDIAN
+ u16 locked_pending;
+ u16 tail;
+#else
+ u16 tail;
+ u16 locked_pending;
+#endif
+ };
+ };
+};
+
+/**
+ * clear_pending_set_locked - take ownership and clear the pending bit.
+ * @lock: Pointer to queue spinlock structure
+ * @val : Current value of the queue spinlock 32-bit word
+ *
+ * *,1,0 -> *,0,1
+ *
+ * Lock stealing is not allowed if this function is used.
+ */
+static __always_inline void
+clear_pending_set_locked(struct qspinlock *lock, u32 val)
+{
+ struct __qspinlock *l = (void *)lock;
+
+ ACCESS_ONCE(l->locked_pending) = _Q_LOCKED_VAL;
+}
+
+/*
+ * xchg_tail - Put in the new queue tail code word & retrieve previous one
+ * @lock : Pointer to queue spinlock structure
+ * @tail : The new queue tail code word
+ * Return: The previous queue tail code word
+ *
+ * xchg(lock, tail)
+ *
+ * p,*,* -> n,*,* ; prev = xchg(lock, node)
+ */
+static __always_inline u32 xchg_tail(struct qspinlock *lock, u32 tail)
+{
+ struct __qspinlock *l = (void *)lock;
+
+ return (u32)xchg(&l->tail, tail >> _Q_TAIL_OFFSET) << _Q_TAIL_OFFSET;
+}
+
+#else /* _Q_PENDING_BITS == 8 */
+
+/**
+ * clear_pending_set_locked - take ownership and clear the pending bit.
+ * @lock: Pointer to queue spinlock structure
+ * @val : Current value of the queue spinlock 32-bit word
+ *
+ * *,1,0 -> *,0,1
+ */
+static __always_inline void
+clear_pending_set_locked(struct qspinlock *lock, u32 val)
+{
+ u32 new, old;
+
+ for (;;) {
+ new = (val & ~_Q_PENDING_MASK) | _Q_LOCKED_VAL;
+
+ old = atomic_cmpxchg(&lock->val, val, new);
+ if (old == val)
+ break;
+
+ val = old;
+ }
+}
+
/**
* xchg_tail - Put in the new queue tail code word & retrieve previous one
* @lock : Pointer to queue spinlock structure
@@ -109,6 +194,7 @@ static __always_inline u32 xchg_tail(struct qspinlock *lock, u32 tail)
}
return old;
}
+#endif /* _Q_PENDING_BITS == 8 */
/**
* queue_spin_lock_slowpath - acquire the queue spinlock
@@ -173,8 +259,13 @@ void queue_spin_lock_slowpath(struct qspinlock *lock, u32 val)
* we're pending, wait for the owner to go away.
*
* *,1,1 -> *,1,0
+ *
+ * this wait loop must be a load-acquire such that we match the
+ * store-release that clears the locked bit and create lock
+ * sequentiality; this because not all clear_pending_set_locked()
+ * implementations imply full barriers.
*/
- while ((val = atomic_read(&lock->val)) & _Q_LOCKED_MASK)
+ while ((val = smp_load_acquire(&lock->val.counter)) & _Q_LOCKED_MASK)
arch_mutex_cpu_relax();
/*
@@ -182,15 +273,7 @@ void queue_spin_lock_slowpath(struct qspinlock *lock, u32 val)
*
* *,1,0 -> *,0,1
*/
- for (;;) {
- new = (val & ~_Q_PENDING_MASK) | _Q_LOCKED_VAL;
-
- old = atomic_cmpxchg(&lock->val, val, new);
- if (old == val)
- break;
-
- val = old;
- }
+ clear_pending_set_locked(lock, val);
return;
/*
--
1.7.1
^ permalink raw reply related [flat|nested] 102+ messages in thread* [PATCH v11 05/16] qspinlock: Optimize for smaller NR_CPUS
@ 2014-05-30 15:43 ` Waiman Long
0 siblings, 0 replies; 102+ messages in thread
From: Waiman Long @ 2014-05-30 15:43 UTC (permalink / raw)
To: Thomas Gleixner, Ingo Molnar, H. Peter Anvin, Peter Zijlstra
Cc: linux-arch, Waiman Long, Rik van Riel, Raghavendra K T,
Gleb Natapov, kvm, Konrad Rzeszutek Wilk, Peter Zijlstra,
Scott J Norton, x86, Paolo Bonzini, linux-kernel, virtualization,
Chegu Vinod, David Vrabel, Oleg Nesterov, xen-devel,
Boris Ostrovsky, Paul E. McKenney, Linus Torvalds
From: Peter Zijlstra <peterz@infradead.org>
When we allow for a max NR_CPUS < 2^14 we can optimize the pending
wait-acquire and the xchg_tail() operations.
By growing the pending bit to a byte, we reduce the tail to 16bit.
This means we can use xchg16 for the tail part and do away with all
the repeated compxchg() operations.
This in turn allows us to unconditionally acquire; the locked state
as observed by the wait loops cannot change. And because both locked
and pending are now a full byte we can use simple stores for the
state transition, obviating one atomic operation entirely.
All this is horribly broken on Alpha pre EV56 (and any other arch that
cannot do single-copy atomic byte stores).
Signed-off-by: Peter Zijlstra <peterz@infradead.org>
Signed-off-by: Waiman Long <Waiman.Long@hp.com>
---
include/asm-generic/qspinlock_types.h | 13 ++++
kernel/locking/qspinlock.c | 103 +++++++++++++++++++++++++++++---
2 files changed, 106 insertions(+), 10 deletions(-)
diff --git a/include/asm-generic/qspinlock_types.h b/include/asm-generic/qspinlock_types.h
index ed5d89a..4914abe 100644
--- a/include/asm-generic/qspinlock_types.h
+++ b/include/asm-generic/qspinlock_types.h
@@ -38,6 +38,14 @@ typedef struct qspinlock {
/*
* Bitfields in the atomic value:
*
+ * When NR_CPUS < 16K
+ * 0- 7: locked byte
+ * 8: pending
+ * 9-15: not used
+ * 16-17: tail index
+ * 18-31: tail cpu (+1)
+ *
+ * When NR_CPUS >= 16K
* 0- 7: locked byte
* 8: pending
* 9-10: tail index
@@ -50,7 +58,11 @@ typedef struct qspinlock {
#define _Q_LOCKED_MASK _Q_SET_MASK(LOCKED)
#define _Q_PENDING_OFFSET (_Q_LOCKED_OFFSET + _Q_LOCKED_BITS)
+#if CONFIG_NR_CPUS < (1U << 14)
+#define _Q_PENDING_BITS 8
+#else
#define _Q_PENDING_BITS 1
+#endif
#define _Q_PENDING_MASK _Q_SET_MASK(PENDING)
#define _Q_TAIL_IDX_OFFSET (_Q_PENDING_OFFSET + _Q_PENDING_BITS)
@@ -61,6 +73,7 @@ typedef struct qspinlock {
#define _Q_TAIL_CPU_BITS (32 - _Q_TAIL_CPU_OFFSET)
#define _Q_TAIL_CPU_MASK _Q_SET_MASK(TAIL_CPU)
+#define _Q_TAIL_OFFSET _Q_TAIL_IDX_OFFSET
#define _Q_TAIL_MASK (_Q_TAIL_IDX_MASK | _Q_TAIL_CPU_MASK)
#define _Q_LOCKED_VAL (1U << _Q_LOCKED_OFFSET)
diff --git a/kernel/locking/qspinlock.c b/kernel/locking/qspinlock.c
index 41594a1..fc7fd8c 100644
--- a/kernel/locking/qspinlock.c
+++ b/kernel/locking/qspinlock.c
@@ -22,6 +22,7 @@
#include <linux/percpu.h>
#include <linux/hardirq.h>
#include <linux/mutex.h>
+#include <asm/byteorder.h>
#include <asm/qspinlock.h>
/*
@@ -48,6 +49,9 @@
* We can further change the first spinner to spin on a bit in the lock word
* instead of its node; whereby avoiding the need to carry a node from lock to
* unlock, and preserving API.
+ *
+ * N.B. The current implementation only supports architectures that allow
+ * atomic operations on smaller 8-bit and 16-bit data types.
*/
#include "mcs_spinlock.h"
@@ -85,6 +89,87 @@ static inline struct mcs_spinlock *decode_tail(u32 tail)
#define _Q_LOCKED_PENDING_MASK (_Q_LOCKED_MASK | _Q_PENDING_MASK)
+/*
+ * By using the whole 2nd least significant byte for the pending bit, we
+ * can allow better optimization of the lock acquisition for the pending
+ * bit holder.
+ */
+#if _Q_PENDING_BITS == 8
+
+struct __qspinlock {
+ union {
+ atomic_t val;
+ struct {
+#ifdef __LITTLE_ENDIAN
+ u16 locked_pending;
+ u16 tail;
+#else
+ u16 tail;
+ u16 locked_pending;
+#endif
+ };
+ };
+};
+
+/**
+ * clear_pending_set_locked - take ownership and clear the pending bit.
+ * @lock: Pointer to queue spinlock structure
+ * @val : Current value of the queue spinlock 32-bit word
+ *
+ * *,1,0 -> *,0,1
+ *
+ * Lock stealing is not allowed if this function is used.
+ */
+static __always_inline void
+clear_pending_set_locked(struct qspinlock *lock, u32 val)
+{
+ struct __qspinlock *l = (void *)lock;
+
+ ACCESS_ONCE(l->locked_pending) = _Q_LOCKED_VAL;
+}
+
+/*
+ * xchg_tail - Put in the new queue tail code word & retrieve previous one
+ * @lock : Pointer to queue spinlock structure
+ * @tail : The new queue tail code word
+ * Return: The previous queue tail code word
+ *
+ * xchg(lock, tail)
+ *
+ * p,*,* -> n,*,* ; prev = xchg(lock, node)
+ */
+static __always_inline u32 xchg_tail(struct qspinlock *lock, u32 tail)
+{
+ struct __qspinlock *l = (void *)lock;
+
+ return (u32)xchg(&l->tail, tail >> _Q_TAIL_OFFSET) << _Q_TAIL_OFFSET;
+}
+
+#else /* _Q_PENDING_BITS == 8 */
+
+/**
+ * clear_pending_set_locked - take ownership and clear the pending bit.
+ * @lock: Pointer to queue spinlock structure
+ * @val : Current value of the queue spinlock 32-bit word
+ *
+ * *,1,0 -> *,0,1
+ */
+static __always_inline void
+clear_pending_set_locked(struct qspinlock *lock, u32 val)
+{
+ u32 new, old;
+
+ for (;;) {
+ new = (val & ~_Q_PENDING_MASK) | _Q_LOCKED_VAL;
+
+ old = atomic_cmpxchg(&lock->val, val, new);
+ if (old == val)
+ break;
+
+ val = old;
+ }
+}
+
/**
* xchg_tail - Put in the new queue tail code word & retrieve previous one
* @lock : Pointer to queue spinlock structure
@@ -109,6 +194,7 @@ static __always_inline u32 xchg_tail(struct qspinlock *lock, u32 tail)
}
return old;
}
+#endif /* _Q_PENDING_BITS == 8 */
/**
* queue_spin_lock_slowpath - acquire the queue spinlock
@@ -173,8 +259,13 @@ void queue_spin_lock_slowpath(struct qspinlock *lock, u32 val)
* we're pending, wait for the owner to go away.
*
* *,1,1 -> *,1,0
+ *
+ * this wait loop must be a load-acquire such that we match the
+ * store-release that clears the locked bit and create lock
+ * sequentiality; this because not all clear_pending_set_locked()
+ * implementations imply full barriers.
*/
- while ((val = atomic_read(&lock->val)) & _Q_LOCKED_MASK)
+ while ((val = smp_load_acquire(&lock->val.counter)) & _Q_LOCKED_MASK)
arch_mutex_cpu_relax();
/*
@@ -182,15 +273,7 @@ void queue_spin_lock_slowpath(struct qspinlock *lock, u32 val)
*
* *,1,0 -> *,0,1
*/
- for (;;) {
- new = (val & ~_Q_PENDING_MASK) | _Q_LOCKED_VAL;
-
- old = atomic_cmpxchg(&lock->val, val, new);
- if (old == val)
- break;
-
- val = old;
- }
+ clear_pending_set_locked(lock, val);
return;
/*
--
1.7.1
^ permalink raw reply related [flat|nested] 102+ messages in thread* [PATCH v11 05/16] qspinlock: Optimize for smaller NR_CPUS
@ 2014-05-30 15:43 ` Waiman Long
0 siblings, 0 replies; 102+ messages in thread
From: Waiman Long @ 2014-05-30 15:43 UTC (permalink / raw)
To: Thomas Gleixner, Ingo Molnar, H. Peter Anvin, Peter Zijlstra
Cc: linux-arch, x86, linux-kernel, virtualization, xen-devel, kvm,
Paolo Bonzini, Konrad Rzeszutek Wilk, Boris Ostrovsky,
Paul E. McKenney, Rik van Riel, Linus Torvalds, Raghavendra K T,
David Vrabel, Oleg Nesterov, Gleb Natapov, Scott J Norton,
Chegu Vinod, Waiman Long
From: Peter Zijlstra <peterz@infradead.org>
When we allow for a max NR_CPUS < 2^14 we can optimize the pending
wait-acquire and the xchg_tail() operations.
By growing the pending bit to a byte, we reduce the tail to 16bit.
This means we can use xchg16 for the tail part and do away with all
the repeated compxchg() operations.
This in turn allows us to unconditionally acquire; the locked state
as observed by the wait loops cannot change. And because both locked
and pending are now a full byte we can use simple stores for the
state transition, obviating one atomic operation entirely.
All this is horribly broken on Alpha pre EV56 (and any other arch that
cannot do single-copy atomic byte stores).
Signed-off-by: Peter Zijlstra <peterz@infradead.org>
Signed-off-by: Waiman Long <Waiman.Long@hp.com>
---
include/asm-generic/qspinlock_types.h | 13 ++++
kernel/locking/qspinlock.c | 103 +++++++++++++++++++++++++++++---
2 files changed, 106 insertions(+), 10 deletions(-)
diff --git a/include/asm-generic/qspinlock_types.h b/include/asm-generic/qspinlock_types.h
index ed5d89a..4914abe 100644
--- a/include/asm-generic/qspinlock_types.h
+++ b/include/asm-generic/qspinlock_types.h
@@ -38,6 +38,14 @@ typedef struct qspinlock {
/*
* Bitfields in the atomic value:
*
+ * When NR_CPUS < 16K
+ * 0- 7: locked byte
+ * 8: pending
+ * 9-15: not used
+ * 16-17: tail index
+ * 18-31: tail cpu (+1)
+ *
+ * When NR_CPUS >= 16K
* 0- 7: locked byte
* 8: pending
* 9-10: tail index
@@ -50,7 +58,11 @@ typedef struct qspinlock {
#define _Q_LOCKED_MASK _Q_SET_MASK(LOCKED)
#define _Q_PENDING_OFFSET (_Q_LOCKED_OFFSET + _Q_LOCKED_BITS)
+#if CONFIG_NR_CPUS < (1U << 14)
+#define _Q_PENDING_BITS 8
+#else
#define _Q_PENDING_BITS 1
+#endif
#define _Q_PENDING_MASK _Q_SET_MASK(PENDING)
#define _Q_TAIL_IDX_OFFSET (_Q_PENDING_OFFSET + _Q_PENDING_BITS)
@@ -61,6 +73,7 @@ typedef struct qspinlock {
#define _Q_TAIL_CPU_BITS (32 - _Q_TAIL_CPU_OFFSET)
#define _Q_TAIL_CPU_MASK _Q_SET_MASK(TAIL_CPU)
+#define _Q_TAIL_OFFSET _Q_TAIL_IDX_OFFSET
#define _Q_TAIL_MASK (_Q_TAIL_IDX_MASK | _Q_TAIL_CPU_MASK)
#define _Q_LOCKED_VAL (1U << _Q_LOCKED_OFFSET)
diff --git a/kernel/locking/qspinlock.c b/kernel/locking/qspinlock.c
index 41594a1..fc7fd8c 100644
--- a/kernel/locking/qspinlock.c
+++ b/kernel/locking/qspinlock.c
@@ -22,6 +22,7 @@
#include <linux/percpu.h>
#include <linux/hardirq.h>
#include <linux/mutex.h>
+#include <asm/byteorder.h>
#include <asm/qspinlock.h>
/*
@@ -48,6 +49,9 @@
* We can further change the first spinner to spin on a bit in the lock word
* instead of its node; whereby avoiding the need to carry a node from lock to
* unlock, and preserving API.
+ *
+ * N.B. The current implementation only supports architectures that allow
+ * atomic operations on smaller 8-bit and 16-bit data types.
*/
#include "mcs_spinlock.h"
@@ -85,6 +89,87 @@ static inline struct mcs_spinlock *decode_tail(u32 tail)
#define _Q_LOCKED_PENDING_MASK (_Q_LOCKED_MASK | _Q_PENDING_MASK)
+/*
+ * By using the whole 2nd least significant byte for the pending bit, we
+ * can allow better optimization of the lock acquisition for the pending
+ * bit holder.
+ */
+#if _Q_PENDING_BITS == 8
+
+struct __qspinlock {
+ union {
+ atomic_t val;
+ struct {
+#ifdef __LITTLE_ENDIAN
+ u16 locked_pending;
+ u16 tail;
+#else
+ u16 tail;
+ u16 locked_pending;
+#endif
+ };
+ };
+};
+
+/**
+ * clear_pending_set_locked - take ownership and clear the pending bit.
+ * @lock: Pointer to queue spinlock structure
+ * @val : Current value of the queue spinlock 32-bit word
+ *
+ * *,1,0 -> *,0,1
+ *
+ * Lock stealing is not allowed if this function is used.
+ */
+static __always_inline void
+clear_pending_set_locked(struct qspinlock *lock, u32 val)
+{
+ struct __qspinlock *l = (void *)lock;
+
+ ACCESS_ONCE(l->locked_pending) = _Q_LOCKED_VAL;
+}
+
+/*
+ * xchg_tail - Put in the new queue tail code word & retrieve previous one
+ * @lock : Pointer to queue spinlock structure
+ * @tail : The new queue tail code word
+ * Return: The previous queue tail code word
+ *
+ * xchg(lock, tail)
+ *
+ * p,*,* -> n,*,* ; prev = xchg(lock, node)
+ */
+static __always_inline u32 xchg_tail(struct qspinlock *lock, u32 tail)
+{
+ struct __qspinlock *l = (void *)lock;
+
+ return (u32)xchg(&l->tail, tail >> _Q_TAIL_OFFSET) << _Q_TAIL_OFFSET;
+}
+
+#else /* _Q_PENDING_BITS == 8 */
+
+/**
+ * clear_pending_set_locked - take ownership and clear the pending bit.
+ * @lock: Pointer to queue spinlock structure
+ * @val : Current value of the queue spinlock 32-bit word
+ *
+ * *,1,0 -> *,0,1
+ */
+static __always_inline void
+clear_pending_set_locked(struct qspinlock *lock, u32 val)
+{
+ u32 new, old;
+
+ for (;;) {
+ new = (val & ~_Q_PENDING_MASK) | _Q_LOCKED_VAL;
+
+ old = atomic_cmpxchg(&lock->val, val, new);
+ if (old == val)
+ break;
+
+ val = old;
+ }
+}
+
/**
* xchg_tail - Put in the new queue tail code word & retrieve previous one
* @lock : Pointer to queue spinlock structure
@@ -109,6 +194,7 @@ static __always_inline u32 xchg_tail(struct qspinlock *lock, u32 tail)
}
return old;
}
+#endif /* _Q_PENDING_BITS == 8 */
/**
* queue_spin_lock_slowpath - acquire the queue spinlock
@@ -173,8 +259,13 @@ void queue_spin_lock_slowpath(struct qspinlock *lock, u32 val)
* we're pending, wait for the owner to go away.
*
* *,1,1 -> *,1,0
+ *
+ * this wait loop must be a load-acquire such that we match the
+ * store-release that clears the locked bit and create lock
+ * sequentiality; this because not all clear_pending_set_locked()
+ * implementations imply full barriers.
*/
- while ((val = atomic_read(&lock->val)) & _Q_LOCKED_MASK)
+ while ((val = smp_load_acquire(&lock->val.counter)) & _Q_LOCKED_MASK)
arch_mutex_cpu_relax();
/*
@@ -182,15 +273,7 @@ void queue_spin_lock_slowpath(struct qspinlock *lock, u32 val)
*
* *,1,0 -> *,0,1
*/
- for (;;) {
- new = (val & ~_Q_PENDING_MASK) | _Q_LOCKED_VAL;
-
- old = atomic_cmpxchg(&lock->val, val, new);
- if (old == val)
- break;
-
- val = old;
- }
+ clear_pending_set_locked(lock, val);
return;
/*
--
1.7.1
^ permalink raw reply related [flat|nested] 102+ messages in thread* [PATCH v11 05/16] qspinlock: Optimize for smaller NR_CPUS
@ 2014-05-30 15:43 ` Waiman Long
0 siblings, 0 replies; 102+ messages in thread
From: Waiman Long @ 2014-05-30 15:43 UTC (permalink / raw)
To: Thomas Gleixner, Ingo Molnar, H. Peter Anvin
Cc: linux-arch, Waiman Long, Rik van Riel, Raghavendra K T,
Gleb Natapov, kvm, Konrad Rzeszutek Wilk, Peter Zijlstra,
Scott J Norton, x86, Paolo Bonzini, linux-kernel, virtualization,
Chegu Vinod, David Vrabel, Oleg Nesterov, xen-devel,
Boris Ostrovsky, Paul E. McKenney, Linus Torvalds
From: Peter Zijlstra <peterz@infradead.org>
When we allow for a max NR_CPUS < 2^14 we can optimize the pending
wait-acquire and the xchg_tail() operations.
By growing the pending bit to a byte, we reduce the tail to 16bit.
This means we can use xchg16 for the tail part and do away with all
the repeated compxchg() operations.
This in turn allows us to unconditionally acquire; the locked state
as observed by the wait loops cannot change. And because both locked
and pending are now a full byte we can use simple stores for the
state transition, obviating one atomic operation entirely.
All this is horribly broken on Alpha pre EV56 (and any other arch that
cannot do single-copy atomic byte stores).
Signed-off-by: Peter Zijlstra <peterz@infradead.org>
Signed-off-by: Waiman Long <Waiman.Long@hp.com>
---
include/asm-generic/qspinlock_types.h | 13 ++++
kernel/locking/qspinlock.c | 103 +++++++++++++++++++++++++++++---
2 files changed, 106 insertions(+), 10 deletions(-)
diff --git a/include/asm-generic/qspinlock_types.h b/include/asm-generic/qspinlock_types.h
index ed5d89a..4914abe 100644
--- a/include/asm-generic/qspinlock_types.h
+++ b/include/asm-generic/qspinlock_types.h
@@ -38,6 +38,14 @@ typedef struct qspinlock {
/*
* Bitfields in the atomic value:
*
+ * When NR_CPUS < 16K
+ * 0- 7: locked byte
+ * 8: pending
+ * 9-15: not used
+ * 16-17: tail index
+ * 18-31: tail cpu (+1)
+ *
+ * When NR_CPUS >= 16K
* 0- 7: locked byte
* 8: pending
* 9-10: tail index
@@ -50,7 +58,11 @@ typedef struct qspinlock {
#define _Q_LOCKED_MASK _Q_SET_MASK(LOCKED)
#define _Q_PENDING_OFFSET (_Q_LOCKED_OFFSET + _Q_LOCKED_BITS)
+#if CONFIG_NR_CPUS < (1U << 14)
+#define _Q_PENDING_BITS 8
+#else
#define _Q_PENDING_BITS 1
+#endif
#define _Q_PENDING_MASK _Q_SET_MASK(PENDING)
#define _Q_TAIL_IDX_OFFSET (_Q_PENDING_OFFSET + _Q_PENDING_BITS)
@@ -61,6 +73,7 @@ typedef struct qspinlock {
#define _Q_TAIL_CPU_BITS (32 - _Q_TAIL_CPU_OFFSET)
#define _Q_TAIL_CPU_MASK _Q_SET_MASK(TAIL_CPU)
+#define _Q_TAIL_OFFSET _Q_TAIL_IDX_OFFSET
#define _Q_TAIL_MASK (_Q_TAIL_IDX_MASK | _Q_TAIL_CPU_MASK)
#define _Q_LOCKED_VAL (1U << _Q_LOCKED_OFFSET)
diff --git a/kernel/locking/qspinlock.c b/kernel/locking/qspinlock.c
index 41594a1..fc7fd8c 100644
--- a/kernel/locking/qspinlock.c
+++ b/kernel/locking/qspinlock.c
@@ -22,6 +22,7 @@
#include <linux/percpu.h>
#include <linux/hardirq.h>
#include <linux/mutex.h>
+#include <asm/byteorder.h>
#include <asm/qspinlock.h>
/*
@@ -48,6 +49,9 @@
* We can further change the first spinner to spin on a bit in the lock word
* instead of its node; whereby avoiding the need to carry a node from lock to
* unlock, and preserving API.
+ *
+ * N.B. The current implementation only supports architectures that allow
+ * atomic operations on smaller 8-bit and 16-bit data types.
*/
#include "mcs_spinlock.h"
@@ -85,6 +89,87 @@ static inline struct mcs_spinlock *decode_tail(u32 tail)
#define _Q_LOCKED_PENDING_MASK (_Q_LOCKED_MASK | _Q_PENDING_MASK)
+/*
+ * By using the whole 2nd least significant byte for the pending bit, we
+ * can allow better optimization of the lock acquisition for the pending
+ * bit holder.
+ */
+#if _Q_PENDING_BITS == 8
+
+struct __qspinlock {
+ union {
+ atomic_t val;
+ struct {
+#ifdef __LITTLE_ENDIAN
+ u16 locked_pending;
+ u16 tail;
+#else
+ u16 tail;
+ u16 locked_pending;
+#endif
+ };
+ };
+};
+
+/**
+ * clear_pending_set_locked - take ownership and clear the pending bit.
+ * @lock: Pointer to queue spinlock structure
+ * @val : Current value of the queue spinlock 32-bit word
+ *
+ * *,1,0 -> *,0,1
+ *
+ * Lock stealing is not allowed if this function is used.
+ */
+static __always_inline void
+clear_pending_set_locked(struct qspinlock *lock, u32 val)
+{
+ struct __qspinlock *l = (void *)lock;
+
+ ACCESS_ONCE(l->locked_pending) = _Q_LOCKED_VAL;
+}
+
+/*
+ * xchg_tail - Put in the new queue tail code word & retrieve previous one
+ * @lock : Pointer to queue spinlock structure
+ * @tail : The new queue tail code word
+ * Return: The previous queue tail code word
+ *
+ * xchg(lock, tail)
+ *
+ * p,*,* -> n,*,* ; prev = xchg(lock, node)
+ */
+static __always_inline u32 xchg_tail(struct qspinlock *lock, u32 tail)
+{
+ struct __qspinlock *l = (void *)lock;
+
+ return (u32)xchg(&l->tail, tail >> _Q_TAIL_OFFSET) << _Q_TAIL_OFFSET;
+}
+
+#else /* _Q_PENDING_BITS == 8 */
+
+/**
+ * clear_pending_set_locked - take ownership and clear the pending bit.
+ * @lock: Pointer to queue spinlock structure
+ * @val : Current value of the queue spinlock 32-bit word
+ *
+ * *,1,0 -> *,0,1
+ */
+static __always_inline void
+clear_pending_set_locked(struct qspinlock *lock, u32 val)
+{
+ u32 new, old;
+
+ for (;;) {
+ new = (val & ~_Q_PENDING_MASK) | _Q_LOCKED_VAL;
+
+ old = atomic_cmpxchg(&lock->val, val, new);
+ if (old == val)
+ break;
+
+ val = old;
+ }
+}
+
/**
* xchg_tail - Put in the new queue tail code word & retrieve previous one
* @lock : Pointer to queue spinlock structure
@@ -109,6 +194,7 @@ static __always_inline u32 xchg_tail(struct qspinlock *lock, u32 tail)
}
return old;
}
+#endif /* _Q_PENDING_BITS == 8 */
/**
* queue_spin_lock_slowpath - acquire the queue spinlock
@@ -173,8 +259,13 @@ void queue_spin_lock_slowpath(struct qspinlock *lock, u32 val)
* we're pending, wait for the owner to go away.
*
* *,1,1 -> *,1,0
+ *
+ * this wait loop must be a load-acquire such that we match the
+ * store-release that clears the locked bit and create lock
+ * sequentiality; this because not all clear_pending_set_locked()
+ * implementations imply full barriers.
*/
- while ((val = atomic_read(&lock->val)) & _Q_LOCKED_MASK)
+ while ((val = smp_load_acquire(&lock->val.counter)) & _Q_LOCKED_MASK)
arch_mutex_cpu_relax();
/*
@@ -182,15 +273,7 @@ void queue_spin_lock_slowpath(struct qspinlock *lock, u32 val)
*
* *,1,0 -> *,0,1
*/
- for (;;) {
- new = (val & ~_Q_PENDING_MASK) | _Q_LOCKED_VAL;
-
- old = atomic_cmpxchg(&lock->val, val, new);
- if (old == val)
- break;
-
- val = old;
- }
+ clear_pending_set_locked(lock, val);
return;
/*
--
1.7.1
^ permalink raw reply related [flat|nested] 102+ messages in thread
* [PATCH v11 05/16] qspinlock: Optimize for smaller NR_CPUS
2014-05-30 15:43 [PATCH v11 00/16] qspinlock: a 4-byte queue spinlock with PV support Waiman Long
` (9 preceding siblings ...)
2014-05-30 15:43 ` Waiman Long
@ 2014-05-30 15:43 ` Waiman Long
2014-05-30 15:43 ` [PATCH v11 06/16] qspinlock: prolong the stay in the pending bit path Waiman Long
` (26 subsequent siblings)
37 siblings, 0 replies; 102+ messages in thread
From: Waiman Long @ 2014-05-30 15:43 UTC (permalink / raw)
To: Thomas Gleixner, Ingo Molnar, H. Peter Anvin
Cc: linux-arch, Waiman Long, Raghavendra K T, Gleb Natapov, kvm,
Peter Zijlstra, Scott J Norton, x86, Paolo Bonzini, linux-kernel,
virtualization, Chegu Vinod, David Vrabel, Oleg Nesterov,
xen-devel, Boris Ostrovsky, Paul E. McKenney, Linus Torvalds
From: Peter Zijlstra <peterz@infradead.org>
When we allow for a max NR_CPUS < 2^14 we can optimize the pending
wait-acquire and the xchg_tail() operations.
By growing the pending bit to a byte, we reduce the tail to 16bit.
This means we can use xchg16 for the tail part and do away with all
the repeated compxchg() operations.
This in turn allows us to unconditionally acquire; the locked state
as observed by the wait loops cannot change. And because both locked
and pending are now a full byte we can use simple stores for the
state transition, obviating one atomic operation entirely.
All this is horribly broken on Alpha pre EV56 (and any other arch that
cannot do single-copy atomic byte stores).
Signed-off-by: Peter Zijlstra <peterz@infradead.org>
Signed-off-by: Waiman Long <Waiman.Long@hp.com>
---
include/asm-generic/qspinlock_types.h | 13 ++++
kernel/locking/qspinlock.c | 103 +++++++++++++++++++++++++++++---
2 files changed, 106 insertions(+), 10 deletions(-)
diff --git a/include/asm-generic/qspinlock_types.h b/include/asm-generic/qspinlock_types.h
index ed5d89a..4914abe 100644
--- a/include/asm-generic/qspinlock_types.h
+++ b/include/asm-generic/qspinlock_types.h
@@ -38,6 +38,14 @@ typedef struct qspinlock {
/*
* Bitfields in the atomic value:
*
+ * When NR_CPUS < 16K
+ * 0- 7: locked byte
+ * 8: pending
+ * 9-15: not used
+ * 16-17: tail index
+ * 18-31: tail cpu (+1)
+ *
+ * When NR_CPUS >= 16K
* 0- 7: locked byte
* 8: pending
* 9-10: tail index
@@ -50,7 +58,11 @@ typedef struct qspinlock {
#define _Q_LOCKED_MASK _Q_SET_MASK(LOCKED)
#define _Q_PENDING_OFFSET (_Q_LOCKED_OFFSET + _Q_LOCKED_BITS)
+#if CONFIG_NR_CPUS < (1U << 14)
+#define _Q_PENDING_BITS 8
+#else
#define _Q_PENDING_BITS 1
+#endif
#define _Q_PENDING_MASK _Q_SET_MASK(PENDING)
#define _Q_TAIL_IDX_OFFSET (_Q_PENDING_OFFSET + _Q_PENDING_BITS)
@@ -61,6 +73,7 @@ typedef struct qspinlock {
#define _Q_TAIL_CPU_BITS (32 - _Q_TAIL_CPU_OFFSET)
#define _Q_TAIL_CPU_MASK _Q_SET_MASK(TAIL_CPU)
+#define _Q_TAIL_OFFSET _Q_TAIL_IDX_OFFSET
#define _Q_TAIL_MASK (_Q_TAIL_IDX_MASK | _Q_TAIL_CPU_MASK)
#define _Q_LOCKED_VAL (1U << _Q_LOCKED_OFFSET)
diff --git a/kernel/locking/qspinlock.c b/kernel/locking/qspinlock.c
index 41594a1..fc7fd8c 100644
--- a/kernel/locking/qspinlock.c
+++ b/kernel/locking/qspinlock.c
@@ -22,6 +22,7 @@
#include <linux/percpu.h>
#include <linux/hardirq.h>
#include <linux/mutex.h>
+#include <asm/byteorder.h>
#include <asm/qspinlock.h>
/*
@@ -48,6 +49,9 @@
* We can further change the first spinner to spin on a bit in the lock word
* instead of its node; whereby avoiding the need to carry a node from lock to
* unlock, and preserving API.
+ *
+ * N.B. The current implementation only supports architectures that allow
+ * atomic operations on smaller 8-bit and 16-bit data types.
*/
#include "mcs_spinlock.h"
@@ -85,6 +89,87 @@ static inline struct mcs_spinlock *decode_tail(u32 tail)
#define _Q_LOCKED_PENDING_MASK (_Q_LOCKED_MASK | _Q_PENDING_MASK)
+/*
+ * By using the whole 2nd least significant byte for the pending bit, we
+ * can allow better optimization of the lock acquisition for the pending
+ * bit holder.
+ */
+#if _Q_PENDING_BITS == 8
+
+struct __qspinlock {
+ union {
+ atomic_t val;
+ struct {
+#ifdef __LITTLE_ENDIAN
+ u16 locked_pending;
+ u16 tail;
+#else
+ u16 tail;
+ u16 locked_pending;
+#endif
+ };
+ };
+};
+
+/**
+ * clear_pending_set_locked - take ownership and clear the pending bit.
+ * @lock: Pointer to queue spinlock structure
+ * @val : Current value of the queue spinlock 32-bit word
+ *
+ * *,1,0 -> *,0,1
+ *
+ * Lock stealing is not allowed if this function is used.
+ */
+static __always_inline void
+clear_pending_set_locked(struct qspinlock *lock, u32 val)
+{
+ struct __qspinlock *l = (void *)lock;
+
+ ACCESS_ONCE(l->locked_pending) = _Q_LOCKED_VAL;
+}
+
+/*
+ * xchg_tail - Put in the new queue tail code word & retrieve previous one
+ * @lock : Pointer to queue spinlock structure
+ * @tail : The new queue tail code word
+ * Return: The previous queue tail code word
+ *
+ * xchg(lock, tail)
+ *
+ * p,*,* -> n,*,* ; prev = xchg(lock, node)
+ */
+static __always_inline u32 xchg_tail(struct qspinlock *lock, u32 tail)
+{
+ struct __qspinlock *l = (void *)lock;
+
+ return (u32)xchg(&l->tail, tail >> _Q_TAIL_OFFSET) << _Q_TAIL_OFFSET;
+}
+
+#else /* _Q_PENDING_BITS == 8 */
+
+/**
+ * clear_pending_set_locked - take ownership and clear the pending bit.
+ * @lock: Pointer to queue spinlock structure
+ * @val : Current value of the queue spinlock 32-bit word
+ *
+ * *,1,0 -> *,0,1
+ */
+static __always_inline void
+clear_pending_set_locked(struct qspinlock *lock, u32 val)
+{
+ u32 new, old;
+
+ for (;;) {
+ new = (val & ~_Q_PENDING_MASK) | _Q_LOCKED_VAL;
+
+ old = atomic_cmpxchg(&lock->val, val, new);
+ if (old == val)
+ break;
+
+ val = old;
+ }
+}
+
/**
* xchg_tail - Put in the new queue tail code word & retrieve previous one
* @lock : Pointer to queue spinlock structure
@@ -109,6 +194,7 @@ static __always_inline u32 xchg_tail(struct qspinlock *lock, u32 tail)
}
return old;
}
+#endif /* _Q_PENDING_BITS == 8 */
/**
* queue_spin_lock_slowpath - acquire the queue spinlock
@@ -173,8 +259,13 @@ void queue_spin_lock_slowpath(struct qspinlock *lock, u32 val)
* we're pending, wait for the owner to go away.
*
* *,1,1 -> *,1,0
+ *
+ * this wait loop must be a load-acquire such that we match the
+ * store-release that clears the locked bit and create lock
+ * sequentiality; this because not all clear_pending_set_locked()
+ * implementations imply full barriers.
*/
- while ((val = atomic_read(&lock->val)) & _Q_LOCKED_MASK)
+ while ((val = smp_load_acquire(&lock->val.counter)) & _Q_LOCKED_MASK)
arch_mutex_cpu_relax();
/*
@@ -182,15 +273,7 @@ void queue_spin_lock_slowpath(struct qspinlock *lock, u32 val)
*
* *,1,0 -> *,0,1
*/
- for (;;) {
- new = (val & ~_Q_PENDING_MASK) | _Q_LOCKED_VAL;
-
- old = atomic_cmpxchg(&lock->val, val, new);
- if (old == val)
- break;
-
- val = old;
- }
+ clear_pending_set_locked(lock, val);
return;
/*
--
1.7.1
^ permalink raw reply related [flat|nested] 102+ messages in thread* [PATCH v11 06/16] qspinlock: prolong the stay in the pending bit path
2014-05-30 15:43 [PATCH v11 00/16] qspinlock: a 4-byte queue spinlock with PV support Waiman Long
` (10 preceding siblings ...)
2014-05-30 15:43 ` Waiman Long
@ 2014-05-30 15:43 ` Waiman Long
2014-06-11 10:26 ` Peter Zijlstra
2014-06-11 10:26 ` Peter Zijlstra
2014-05-30 15:43 ` Waiman Long
` (25 subsequent siblings)
37 siblings, 2 replies; 102+ messages in thread
From: Waiman Long @ 2014-05-30 15:43 UTC (permalink / raw)
To: Thomas Gleixner, Ingo Molnar, H. Peter Anvin, Peter Zijlstra
Cc: linux-arch, x86, linux-kernel, virtualization, xen-devel, kvm,
Paolo Bonzini, Konrad Rzeszutek Wilk, Boris Ostrovsky,
Paul E. McKenney, Rik van Riel, Linus Torvalds, Raghavendra K T,
David Vrabel, Oleg Nesterov, Gleb Natapov, Scott J Norton,
Chegu Vinod, Waiman Long
There is a problem in the current pending bit spinning code. When the
lock is free, but the pending bit holder hasn't grabbed the lock &
cleared the pending bit yet, the spinning code will not be run.
As a result, the regular queuing code path might be used most of
the time even when there is only 2 tasks contending for the lock.
Assuming that the pending bit holder is going to get the lock and
clear the pending bit soon, it is actually better to wait than to be
queued up which has a higher overhead.
The following tables show the before-patch execution time (in ms)
of a micro-benchmark where 5M iterations of the lock/unlock cycles
were run on a 10-core Westere-EX x86-64 CPU with 2 different types of
loads - standalone (lock and protected data in different cachelines)
and embedded (lock and protected data in the same cacheline).
[Standalone/Embedded - same node]
# of tasks Ticket lock Queue lock %Change
---------- ----------- ---------- -------
1 135/ 111 135/ 101 0%/ -9%
2 890/ 779 1885/1990 +112%/+156%
3 1932/1859 2333/2341 +21%/ +26%
4 2829/2726 2900/2923 +3%/ +7%
5 3834/3761 3655/3648 -5%/ -3%
6 4963/4976 4336/4326 -13%/ -13%
7 6299/6269 5057/5064 -20%/ -19%
8 7691/7569 5786/5798 -25%/ -23%
Of course, the results will varies depending on what kind of test
machine is used.
With 1 task per NUMA node, the execution times are:
[Standalone - different nodes]
# of nodes Ticket lock Queue lock %Change
---------- ----------- ---------- -------
1 135 135 0%
2 4604 5087 +10%
3 10940 12224 +12%
4 21555 10555 -51%
It can be seen that the queue spinlock is slower than the ticket
spinlock when there are 2 or 3 contending tasks. In all the other case,
the queue spinlock is either equal or faster than the ticket spinlock.
With this patch, the performance data for 2 contending tasks are:
[Standalone/Embedded]
# of tasks Ticket lock Queue lock %Change
---------- ----------- ---------- -------
2 890/779 984/871 +11%/+12%
[Standalone - different nodes]
# of nodes Ticket lock Queue lock %Change
---------- ----------- ---------- -------
2 4604 1364 -70%
It can be seen that the queue spinlock performance for 2 contending
tasks is now comparable to ticket spinlock on the same node, but much
faster when in different nodes. With 3 contending tasks, however,
the ticket spinlock is still quite a bit faster.
Signed-off-by: Waiman Long <Waiman.Long@hp.com>
---
kernel/locking/qspinlock.c | 18 ++++++++++++++++--
1 files changed, 16 insertions(+), 2 deletions(-)
diff --git a/kernel/locking/qspinlock.c b/kernel/locking/qspinlock.c
index fc7fd8c..7f10758 100644
--- a/kernel/locking/qspinlock.c
+++ b/kernel/locking/qspinlock.c
@@ -233,11 +233,25 @@ void queue_spin_lock_slowpath(struct qspinlock *lock, u32 val)
*/
for (;;) {
/*
- * If we observe any contention; queue.
+ * If we observe that the queue is not empty or both
+ * the pending and lock bits are set, queue
*/
- if (val & ~_Q_LOCKED_MASK)
+ if ((val & _Q_TAIL_MASK) ||
+ (val == (_Q_LOCKED_VAL|_Q_PENDING_VAL)))
goto queue;
+ if (val == _Q_PENDING_VAL) {
+ /*
+ * Pending bit is set, but not the lock bit.
+ * Assuming that the pending bit holder is going to
+ * set the lock bit and clear the pending bit soon,
+ * it is better to wait than to exit at this point.
+ */
+ cpu_relax();
+ val = atomic_read(&lock->val);
+ continue;
+ }
+
new = _Q_LOCKED_VAL;
if (val == new)
new |= _Q_PENDING_VAL;
--
1.7.1
^ permalink raw reply related [flat|nested] 102+ messages in thread* Re: [PATCH v11 06/16] qspinlock: prolong the stay in the pending bit path
2014-05-30 15:43 ` [PATCH v11 06/16] qspinlock: prolong the stay in the pending bit path Waiman Long
@ 2014-06-11 10:26 ` Peter Zijlstra
2014-06-11 10:26 ` Peter Zijlstra
1 sibling, 0 replies; 102+ messages in thread
From: Peter Zijlstra @ 2014-06-11 10:26 UTC (permalink / raw)
To: Waiman Long
Cc: Thomas Gleixner, Ingo Molnar, H. Peter Anvin, linux-arch, x86,
linux-kernel, virtualization, xen-devel, kvm, Paolo Bonzini,
Konrad Rzeszutek Wilk, Boris Ostrovsky, Paul E. McKenney,
Rik van Riel, Linus Torvalds, Raghavendra K T, David Vrabel,
Oleg Nesterov, Gleb Natapov, Scott J Norton, Chegu Vinod
[-- Attachment #1: Type: text/plain, Size: 1475 bytes --]
On Fri, May 30, 2014 at 11:43:52AM -0400, Waiman Long wrote:
> ---
> kernel/locking/qspinlock.c | 18 ++++++++++++++++--
> 1 files changed, 16 insertions(+), 2 deletions(-)
>
> diff --git a/kernel/locking/qspinlock.c b/kernel/locking/qspinlock.c
> index fc7fd8c..7f10758 100644
> --- a/kernel/locking/qspinlock.c
> +++ b/kernel/locking/qspinlock.c
> @@ -233,11 +233,25 @@ void queue_spin_lock_slowpath(struct qspinlock *lock, u32 val)
> */
> for (;;) {
> /*
> - * If we observe any contention; queue.
> + * If we observe that the queue is not empty or both
> + * the pending and lock bits are set, queue
> */
> - if (val & ~_Q_LOCKED_MASK)
> + if ((val & _Q_TAIL_MASK) ||
> + (val == (_Q_LOCKED_VAL|_Q_PENDING_VAL)))
> goto queue;
>
> + if (val == _Q_PENDING_VAL) {
> + /*
> + * Pending bit is set, but not the lock bit.
> + * Assuming that the pending bit holder is going to
> + * set the lock bit and clear the pending bit soon,
> + * it is better to wait than to exit at this point.
> + */
> + cpu_relax();
> + val = atomic_read(&lock->val);
> + continue;
> + }
> +
> new = _Q_LOCKED_VAL;
> if (val == new)
> new |= _Q_PENDING_VAL;
So, again, you just posted a new version without replying to the
previous discussion; so let me try again, what's wrong with the proposal
here:
lkml.kernel.org/r/20140417163640.GT11096@twins.programming.kicks-ass.net
[-- Attachment #2: Type: application/pgp-signature, Size: 836 bytes --]
^ permalink raw reply [flat|nested] 102+ messages in thread* Re: [PATCH v11 06/16] qspinlock: prolong the stay in the pending bit path
@ 2014-06-11 10:26 ` Peter Zijlstra
0 siblings, 0 replies; 102+ messages in thread
From: Peter Zijlstra @ 2014-06-11 10:26 UTC (permalink / raw)
To: Waiman Long
Cc: linux-arch, Rik van Riel, Raghavendra K T, Oleg Nesterov,
Gleb Natapov, kvm, Konrad Rzeszutek Wilk, Scott J Norton, x86,
Paolo Bonzini, linux-kernel, virtualization, Ingo Molnar,
Chegu Vinod, David Vrabel, H. Peter Anvin, xen-devel,
Thomas Gleixner, Paul E. McKenney, Linus Torvalds,
Boris Ostrovsky
[-- Attachment #1.1: Type: text/plain, Size: 1475 bytes --]
On Fri, May 30, 2014 at 11:43:52AM -0400, Waiman Long wrote:
> ---
> kernel/locking/qspinlock.c | 18 ++++++++++++++++--
> 1 files changed, 16 insertions(+), 2 deletions(-)
>
> diff --git a/kernel/locking/qspinlock.c b/kernel/locking/qspinlock.c
> index fc7fd8c..7f10758 100644
> --- a/kernel/locking/qspinlock.c
> +++ b/kernel/locking/qspinlock.c
> @@ -233,11 +233,25 @@ void queue_spin_lock_slowpath(struct qspinlock *lock, u32 val)
> */
> for (;;) {
> /*
> - * If we observe any contention; queue.
> + * If we observe that the queue is not empty or both
> + * the pending and lock bits are set, queue
> */
> - if (val & ~_Q_LOCKED_MASK)
> + if ((val & _Q_TAIL_MASK) ||
> + (val == (_Q_LOCKED_VAL|_Q_PENDING_VAL)))
> goto queue;
>
> + if (val == _Q_PENDING_VAL) {
> + /*
> + * Pending bit is set, but not the lock bit.
> + * Assuming that the pending bit holder is going to
> + * set the lock bit and clear the pending bit soon,
> + * it is better to wait than to exit at this point.
> + */
> + cpu_relax();
> + val = atomic_read(&lock->val);
> + continue;
> + }
> +
> new = _Q_LOCKED_VAL;
> if (val == new)
> new |= _Q_PENDING_VAL;
So, again, you just posted a new version without replying to the
previous discussion; so let me try again, what's wrong with the proposal
here:
lkml.kernel.org/r/20140417163640.GT11096@twins.programming.kicks-ass.net
[-- Attachment #1.2: Type: application/pgp-signature, Size: 836 bytes --]
[-- Attachment #2: Type: text/plain, Size: 183 bytes --]
_______________________________________________
Virtualization mailing list
Virtualization@lists.linux-foundation.org
https://lists.linuxfoundation.org/mailman/listinfo/virtualization
^ permalink raw reply [flat|nested] 102+ messages in thread* Re: [PATCH v11 06/16] qspinlock: prolong the stay in the pending bit path
2014-06-11 10:26 ` Peter Zijlstra
(?)
@ 2014-06-11 21:22 ` Long, Wai Man
-1 siblings, 0 replies; 102+ messages in thread
From: Long, Wai Man @ 2014-06-11 21:22 UTC (permalink / raw)
To: Peter Zijlstra
Cc: linux-arch, Raghavendra K T, Oleg Nesterov, Gleb Natapov, kvm,
Scott J Norton, x86, Paolo Bonzini, linux-kernel, virtualization,
Ingo Molnar, Chegu Vinod, David Vrabel, H. Peter Anvin,
xen-devel, Thomas Gleixner, Paul E. McKenney, Linus Torvalds,
Boris Ostrovsky
On 6/11/2014 6:26 AM, Peter Zijlstra wrote:
> On Fri, May 30, 2014 at 11:43:52AM -0400, Waiman Long wrote:
>> ---
>> kernel/locking/qspinlock.c | 18 ++++++++++++++++--
>> 1 files changed, 16 insertions(+), 2 deletions(-)
>>
>> diff --git a/kernel/locking/qspinlock.c b/kernel/locking/qspinlock.c
>> index fc7fd8c..7f10758 100644
>> --- a/kernel/locking/qspinlock.c
>> +++ b/kernel/locking/qspinlock.c
>> @@ -233,11 +233,25 @@ void queue_spin_lock_slowpath(struct qspinlock *lock, u32 val)
>> */
>> for (;;) {
>> /*
>> - * If we observe any contention; queue.
>> + * If we observe that the queue is not empty or both
>> + * the pending and lock bits are set, queue
>> */
>> - if (val & ~_Q_LOCKED_MASK)
>> + if ((val & _Q_TAIL_MASK) ||
>> + (val == (_Q_LOCKED_VAL|_Q_PENDING_VAL)))
>> goto queue;
>>
>> + if (val == _Q_PENDING_VAL) {
>> + /*
>> + * Pending bit is set, but not the lock bit.
>> + * Assuming that the pending bit holder is going to
>> + * set the lock bit and clear the pending bit soon,
>> + * it is better to wait than to exit at this point.
>> + */
>> + cpu_relax();
>> + val = atomic_read(&lock->val);
>> + continue;
>> + }
>> +
>> new = _Q_LOCKED_VAL;
>> if (val == new)
>> new |= _Q_PENDING_VAL;
>
> So, again, you just posted a new version without replying to the
> previous discussion; so let me try again, what's wrong with the proposal
> here:
>
> lkml.kernel.org/r/20140417163640.GT11096@twins.programming.kicks-ass.net
>
>
I thought I had answered you before, maybe the message was lost or the
answer was not complete. Anyway, I will try to response to your question
again here.
> Wouldn't something like:
>
> while (atomic_read(&lock->val) == _Q_PENDING_VAL)
> cpu_relax();
>
> before the cmpxchg loop have gotten you all this?
That is not exactly the same. The loop will exit if other bits are set or the pending
bit cleared. In the case, we will need to do the same check at the beginning of the
for loop in order to avoid doing an extra cmpxchg that is not necessary.
> I just tried this on my code and I cannot see a difference.
As I said before, I did see a difference with that change. I think it
depends on the CPU chip that we used for testing. I ran my test on a
10-core Westmere-EX chip. I run my microbench on different pairs of core
within the same chip. It produces different results that varies from
779.5ms to up to 1192ms. Without that patch, the lowest value I can get
is still close to 800ms, but the highest can be up to 1800ms or so. So I
believe it is just a matter of timing that you did not observed in your
test machine.
-Longman
^ permalink raw reply [flat|nested] 102+ messages in thread* Re: [PATCH v11 06/16] qspinlock: prolong the stay in the pending bit path
2014-06-11 10:26 ` Peter Zijlstra
@ 2014-06-11 21:22 ` Long, Wai Man
-1 siblings, 0 replies; 102+ messages in thread
From: Long, Wai Man @ 2014-06-11 21:22 UTC (permalink / raw)
To: Peter Zijlstra
Cc: Thomas Gleixner, Ingo Molnar, H. Peter Anvin, linux-arch, x86,
linux-kernel, virtualization, xen-devel, kvm, Paolo Bonzini,
Konrad Rzeszutek Wilk, Boris Ostrovsky, Paul E. McKenney,
Rik van Riel, Linus Torvalds, Raghavendra K T, David Vrabel,
Oleg Nesterov, Gleb Natapov, Scott J Norton, Chegu Vinod
On 6/11/2014 6:26 AM, Peter Zijlstra wrote:
> On Fri, May 30, 2014 at 11:43:52AM -0400, Waiman Long wrote:
>> ---
>> kernel/locking/qspinlock.c | 18 ++++++++++++++++--
>> 1 files changed, 16 insertions(+), 2 deletions(-)
>>
>> diff --git a/kernel/locking/qspinlock.c b/kernel/locking/qspinlock.c
>> index fc7fd8c..7f10758 100644
>> --- a/kernel/locking/qspinlock.c
>> +++ b/kernel/locking/qspinlock.c
>> @@ -233,11 +233,25 @@ void queue_spin_lock_slowpath(struct qspinlock *lock, u32 val)
>> */
>> for (;;) {
>> /*
>> - * If we observe any contention; queue.
>> + * If we observe that the queue is not empty or both
>> + * the pending and lock bits are set, queue
>> */
>> - if (val & ~_Q_LOCKED_MASK)
>> + if ((val & _Q_TAIL_MASK) ||
>> + (val == (_Q_LOCKED_VAL|_Q_PENDING_VAL)))
>> goto queue;
>>
>> + if (val == _Q_PENDING_VAL) {
>> + /*
>> + * Pending bit is set, but not the lock bit.
>> + * Assuming that the pending bit holder is going to
>> + * set the lock bit and clear the pending bit soon,
>> + * it is better to wait than to exit at this point.
>> + */
>> + cpu_relax();
>> + val = atomic_read(&lock->val);
>> + continue;
>> + }
>> +
>> new = _Q_LOCKED_VAL;
>> if (val == new)
>> new |= _Q_PENDING_VAL;
>
> So, again, you just posted a new version without replying to the
> previous discussion; so let me try again, what's wrong with the proposal
> here:
>
> lkml.kernel.org/r/20140417163640.GT11096@twins.programming.kicks-ass.net
>
>
I thought I had answered you before, maybe the message was lost or the
answer was not complete. Anyway, I will try to response to your question
again here.
> Wouldn't something like:
>
> while (atomic_read(&lock->val) == _Q_PENDING_VAL)
> cpu_relax();
>
> before the cmpxchg loop have gotten you all this?
That is not exactly the same. The loop will exit if other bits are set or the pending
bit cleared. In the case, we will need to do the same check at the beginning of the
for loop in order to avoid doing an extra cmpxchg that is not necessary.
> I just tried this on my code and I cannot see a difference.
As I said before, I did see a difference with that change. I think it
depends on the CPU chip that we used for testing. I ran my test on a
10-core Westmere-EX chip. I run my microbench on different pairs of core
within the same chip. It produces different results that varies from
779.5ms to up to 1192ms. Without that patch, the lowest value I can get
is still close to 800ms, but the highest can be up to 1800ms or so. So I
believe it is just a matter of timing that you did not observed in your
test machine.
-Longman
^ permalink raw reply [flat|nested] 102+ messages in thread* Re: [PATCH v11 06/16] qspinlock: prolong the stay in the pending bit path
@ 2014-06-11 21:22 ` Long, Wai Man
0 siblings, 0 replies; 102+ messages in thread
From: Long, Wai Man @ 2014-06-11 21:22 UTC (permalink / raw)
To: Peter Zijlstra
Cc: linux-arch, Rik van Riel, Raghavendra K T, Oleg Nesterov,
Gleb Natapov, kvm, Konrad Rzeszutek Wilk, Scott J Norton, x86,
Paolo Bonzini, linux-kernel, virtualization, Ingo Molnar,
Chegu Vinod, David Vrabel, H. Peter Anvin, xen-devel,
Thomas Gleixner, Paul E. McKenney, Linus Torvalds,
Boris Ostrovsky
On 6/11/2014 6:26 AM, Peter Zijlstra wrote:
> On Fri, May 30, 2014 at 11:43:52AM -0400, Waiman Long wrote:
>> ---
>> kernel/locking/qspinlock.c | 18 ++++++++++++++++--
>> 1 files changed, 16 insertions(+), 2 deletions(-)
>>
>> diff --git a/kernel/locking/qspinlock.c b/kernel/locking/qspinlock.c
>> index fc7fd8c..7f10758 100644
>> --- a/kernel/locking/qspinlock.c
>> +++ b/kernel/locking/qspinlock.c
>> @@ -233,11 +233,25 @@ void queue_spin_lock_slowpath(struct qspinlock *lock, u32 val)
>> */
>> for (;;) {
>> /*
>> - * If we observe any contention; queue.
>> + * If we observe that the queue is not empty or both
>> + * the pending and lock bits are set, queue
>> */
>> - if (val & ~_Q_LOCKED_MASK)
>> + if ((val & _Q_TAIL_MASK) ||
>> + (val == (_Q_LOCKED_VAL|_Q_PENDING_VAL)))
>> goto queue;
>>
>> + if (val == _Q_PENDING_VAL) {
>> + /*
>> + * Pending bit is set, but not the lock bit.
>> + * Assuming that the pending bit holder is going to
>> + * set the lock bit and clear the pending bit soon,
>> + * it is better to wait than to exit at this point.
>> + */
>> + cpu_relax();
>> + val = atomic_read(&lock->val);
>> + continue;
>> + }
>> +
>> new = _Q_LOCKED_VAL;
>> if (val == new)
>> new |= _Q_PENDING_VAL;
>
> So, again, you just posted a new version without replying to the
> previous discussion; so let me try again, what's wrong with the proposal
> here:
>
> lkml.kernel.org/r/20140417163640.GT11096@twins.programming.kicks-ass.net
>
>
I thought I had answered you before, maybe the message was lost or the
answer was not complete. Anyway, I will try to response to your question
again here.
> Wouldn't something like:
>
> while (atomic_read(&lock->val) == _Q_PENDING_VAL)
> cpu_relax();
>
> before the cmpxchg loop have gotten you all this?
That is not exactly the same. The loop will exit if other bits are set or the pending
bit cleared. In the case, we will need to do the same check at the beginning of the
for loop in order to avoid doing an extra cmpxchg that is not necessary.
> I just tried this on my code and I cannot see a difference.
As I said before, I did see a difference with that change. I think it
depends on the CPU chip that we used for testing. I ran my test on a
10-core Westmere-EX chip. I run my microbench on different pairs of core
within the same chip. It produces different results that varies from
779.5ms to up to 1192ms. Without that patch, the lowest value I can get
is still close to 800ms, but the highest can be up to 1800ms or so. So I
believe it is just a matter of timing that you did not observed in your
test machine.
-Longman
^ permalink raw reply [flat|nested] 102+ messages in thread* Re: [PATCH v11 06/16] qspinlock: prolong the stay in the pending bit path
2014-06-11 21:22 ` Long, Wai Man
(?)
@ 2014-06-12 6:00 ` Peter Zijlstra
2014-06-12 20:54 ` Waiman Long
2014-06-12 20:54 ` Waiman Long
-1 siblings, 2 replies; 102+ messages in thread
From: Peter Zijlstra @ 2014-06-12 6:00 UTC (permalink / raw)
To: Long, Wai Man
Cc: Thomas Gleixner, Ingo Molnar, H. Peter Anvin, linux-arch, x86,
linux-kernel, virtualization, xen-devel, kvm, Paolo Bonzini,
Konrad Rzeszutek Wilk, Boris Ostrovsky, Paul E. McKenney,
Rik van Riel, Linus Torvalds, Raghavendra K T, David Vrabel,
Oleg Nesterov, Gleb Natapov, Scott J Norton, Chegu Vinod
[-- Attachment #1: Type: text/plain, Size: 1821 bytes --]
On Wed, Jun 11, 2014 at 05:22:28PM -0400, Long, Wai Man wrote:
> >>@@ -233,11 +233,25 @@ void queue_spin_lock_slowpath(struct qspinlock *lock, u32 val)
> >> */
> >> for (;;) {
> >> /*
> >>- * If we observe any contention; queue.
> >>+ * If we observe that the queue is not empty or both
> >>+ * the pending and lock bits are set, queue
> >> */
> >>- if (val & ~_Q_LOCKED_MASK)
> >>+ if ((val & _Q_TAIL_MASK) ||
> >>+ (val == (_Q_LOCKED_VAL|_Q_PENDING_VAL)))
> >> goto queue;
> >>+ if (val == _Q_PENDING_VAL) {
> >>+ /*
> >>+ * Pending bit is set, but not the lock bit.
> >>+ * Assuming that the pending bit holder is going to
> >>+ * set the lock bit and clear the pending bit soon,
> >>+ * it is better to wait than to exit at this point.
> >>+ */
> >>+ cpu_relax();
> >>+ val = atomic_read(&lock->val);
> >>+ continue;
> >>+ }
> >>+
> >> new = _Q_LOCKED_VAL;
> >> if (val == new)
> >> new |= _Q_PENDING_VAL;
> >Wouldn't something like:
> >
> > while (atomic_read(&lock->val) == _Q_PENDING_VAL)
> > cpu_relax();
> >
> >before the cmpxchg loop have gotten you all this?
>
> That is not exactly the same. The loop will exit if other bits are set or the pending
> bit cleared. In the case, we will need to do the same check at the beginning of the
> for loop in order to avoid doing an extra cmpxchg that is not necessary.
If other bits get set we should stop poking at the pending bit and get
queued. The only transition we want to wait for is: 0,1,0 -> 0,0,1.
What extra unneeded cmpxchg() is there? If we have two cpus waiting in
this loop for the pending bit to go away then both will attempt to grab
the now free pending bit, one will loose and get queued?
There's no avoiding that contention.
[-- Attachment #2: Type: application/pgp-signature, Size: 836 bytes --]
^ permalink raw reply [flat|nested] 102+ messages in thread* Re: [PATCH v11 06/16] qspinlock: prolong the stay in the pending bit path
2014-06-12 6:00 ` Peter Zijlstra
@ 2014-06-12 20:54 ` Waiman Long
2014-06-12 20:54 ` Waiman Long
1 sibling, 0 replies; 102+ messages in thread
From: Waiman Long @ 2014-06-12 20:54 UTC (permalink / raw)
To: Peter Zijlstra
Cc: Thomas Gleixner, Ingo Molnar, H. Peter Anvin, linux-arch, x86,
linux-kernel, virtualization, xen-devel, kvm, Paolo Bonzini,
Konrad Rzeszutek Wilk, Boris Ostrovsky, Paul E. McKenney,
Rik van Riel, Linus Torvalds, Raghavendra K T, David Vrabel,
Oleg Nesterov, Gleb Natapov, Scott J Norton, Chegu Vinod
On 06/12/2014 02:00 AM, Peter Zijlstra wrote:
> On Wed, Jun 11, 2014 at 05:22:28PM -0400, Long, Wai Man wrote:
>
>>>> @@ -233,11 +233,25 @@ void queue_spin_lock_slowpath(struct qspinlock *lock, u32 val)
>>>> */
>>>> for (;;) {
>>>> /*
>>>> - * If we observe any contention; queue.
>>>> + * If we observe that the queue is not empty or both
>>>> + * the pending and lock bits are set, queue
>>>> */
>>>> - if (val& ~_Q_LOCKED_MASK)
>>>> + if ((val& _Q_TAIL_MASK) ||
>>>> + (val == (_Q_LOCKED_VAL|_Q_PENDING_VAL)))
>>>> goto queue;
>>>> + if (val == _Q_PENDING_VAL) {
>>>> + /*
>>>> + * Pending bit is set, but not the lock bit.
>>>> + * Assuming that the pending bit holder is going to
>>>> + * set the lock bit and clear the pending bit soon,
>>>> + * it is better to wait than to exit at this point.
>>>> + */
>>>> + cpu_relax();
>>>> + val = atomic_read(&lock->val);
>>>> + continue;
>>>> + }
>>>> +
>>>> new = _Q_LOCKED_VAL;
>>>> if (val == new)
>>>> new |= _Q_PENDING_VAL;
>>> Wouldn't something like:
>>>
>>> while (atomic_read(&lock->val) == _Q_PENDING_VAL)
>>> cpu_relax();
>>>
>>> before the cmpxchg loop have gotten you all this?
>> That is not exactly the same. The loop will exit if other bits are set or the pending
>> bit cleared. In the case, we will need to do the same check at the beginning of the
>> for loop in order to avoid doing an extra cmpxchg that is not necessary.
> If other bits get set we should stop poking at the pending bit and get
> queued. The only transition we want to wait for is: 0,1,0 -> 0,0,1.
>
> What extra unneeded cmpxchg() is there? If we have two cpus waiting in
> this loop for the pending bit to go away then both will attempt to grab
> the now free pending bit, one will loose and get queued?
>
> There's no avoiding that contention.
If two tasks see the pending bit goes away and try to grab it with
cmpxchg, there is no way we can avoid the contention. However, if some
how the pending bit holder get the lock and another task set the pending
bit before the current task, the spinlock value will become
_Q_PENDING_VAL|_Q_LOCKED_VAL. The while loop will end and the code will
blindly try to do a cmpxchg unless we check for this case before hand.
This is what my code does by going back to the beginning of the for loop.
-Longman
^ permalink raw reply [flat|nested] 102+ messages in thread* Re: [PATCH v11 06/16] qspinlock: prolong the stay in the pending bit path
@ 2014-06-12 20:54 ` Waiman Long
0 siblings, 0 replies; 102+ messages in thread
From: Waiman Long @ 2014-06-12 20:54 UTC (permalink / raw)
To: Peter Zijlstra
Cc: linux-arch, Rik van Riel, Raghavendra K T, Oleg Nesterov,
Gleb Natapov, kvm, Konrad Rzeszutek Wilk, Scott J Norton, x86,
Paolo Bonzini, linux-kernel, virtualization, Ingo Molnar,
Chegu Vinod, David Vrabel, H. Peter Anvin, xen-devel,
Thomas Gleixner, Paul E. McKenney, Linus Torvalds,
Boris Ostrovsky
On 06/12/2014 02:00 AM, Peter Zijlstra wrote:
> On Wed, Jun 11, 2014 at 05:22:28PM -0400, Long, Wai Man wrote:
>
>>>> @@ -233,11 +233,25 @@ void queue_spin_lock_slowpath(struct qspinlock *lock, u32 val)
>>>> */
>>>> for (;;) {
>>>> /*
>>>> - * If we observe any contention; queue.
>>>> + * If we observe that the queue is not empty or both
>>>> + * the pending and lock bits are set, queue
>>>> */
>>>> - if (val& ~_Q_LOCKED_MASK)
>>>> + if ((val& _Q_TAIL_MASK) ||
>>>> + (val == (_Q_LOCKED_VAL|_Q_PENDING_VAL)))
>>>> goto queue;
>>>> + if (val == _Q_PENDING_VAL) {
>>>> + /*
>>>> + * Pending bit is set, but not the lock bit.
>>>> + * Assuming that the pending bit holder is going to
>>>> + * set the lock bit and clear the pending bit soon,
>>>> + * it is better to wait than to exit at this point.
>>>> + */
>>>> + cpu_relax();
>>>> + val = atomic_read(&lock->val);
>>>> + continue;
>>>> + }
>>>> +
>>>> new = _Q_LOCKED_VAL;
>>>> if (val == new)
>>>> new |= _Q_PENDING_VAL;
>>> Wouldn't something like:
>>>
>>> while (atomic_read(&lock->val) == _Q_PENDING_VAL)
>>> cpu_relax();
>>>
>>> before the cmpxchg loop have gotten you all this?
>> That is not exactly the same. The loop will exit if other bits are set or the pending
>> bit cleared. In the case, we will need to do the same check at the beginning of the
>> for loop in order to avoid doing an extra cmpxchg that is not necessary.
> If other bits get set we should stop poking at the pending bit and get
> queued. The only transition we want to wait for is: 0,1,0 -> 0,0,1.
>
> What extra unneeded cmpxchg() is there? If we have two cpus waiting in
> this loop for the pending bit to go away then both will attempt to grab
> the now free pending bit, one will loose and get queued?
>
> There's no avoiding that contention.
If two tasks see the pending bit goes away and try to grab it with
cmpxchg, there is no way we can avoid the contention. However, if some
how the pending bit holder get the lock and another task set the pending
bit before the current task, the spinlock value will become
_Q_PENDING_VAL|_Q_LOCKED_VAL. The while loop will end and the code will
blindly try to do a cmpxchg unless we check for this case before hand.
This is what my code does by going back to the beginning of the for loop.
-Longman
^ permalink raw reply [flat|nested] 102+ messages in thread* Re: [PATCH v11 06/16] qspinlock: prolong the stay in the pending bit path
2014-06-12 20:54 ` Waiman Long
(?)
@ 2014-06-15 13:12 ` Peter Zijlstra
-1 siblings, 0 replies; 102+ messages in thread
From: Peter Zijlstra @ 2014-06-15 13:12 UTC (permalink / raw)
To: Waiman Long
Cc: linux-arch, Rik van Riel, Raghavendra K T, Oleg Nesterov,
Gleb Natapov, kvm, Konrad Rzeszutek Wilk, Scott J Norton, x86,
Paolo Bonzini, linux-kernel, virtualization, Ingo Molnar,
Chegu Vinod, David Vrabel, H. Peter Anvin, xen-devel,
Thomas Gleixner, Paul E. McKenney, Linus Torvalds,
Boris Ostrovsky
On Thu, Jun 12, 2014 at 04:54:52PM -0400, Waiman Long wrote:
> If two tasks see the pending bit goes away and try to grab it with cmpxchg,
> there is no way we can avoid the contention. However, if some how the
> pending bit holder get the lock and another task set the pending bit before
> the current task, the spinlock value will become
> _Q_PENDING_VAL|_Q_LOCKED_VAL. The while loop will end and the code will
> blindly try to do a cmpxchg unless we check for this case before hand. This
> is what my code does by going back to the beginning of the for loop.
There is already a test for that; see the goto queue;
---
/*
* wait for in-progress pending->locked hand-overs
*
* 0,1,0 -> 0,0,1
*/
if (val == _Q_PENDING_VAL) {
while ((val = atomic_read(&lock->val)) == _Q_PENDING_VAL)
cpu_relax();
}
/*
* trylock || pending
*
* 0,0,0 -> 0,0,1 ; trylock
* 0,0,1 -> 0,1,1 ; pending
*/
for (;;) {
/*
* If we observe any contention; queue.
*/
if (val & ~_Q_LOCKED_MASK)
goto queue;
new = _Q_LOCKED_VAL;
if (val == new)
new |= _Q_PENDING_VAL;
old = atomic_cmpxchg(&lock->val, val, new);
if (old == val)
break;
val = old;
}
^ permalink raw reply [flat|nested] 102+ messages in thread* Re: [PATCH v11 06/16] qspinlock: prolong the stay in the pending bit path
2014-06-12 20:54 ` Waiman Long
(?)
(?)
@ 2014-06-15 13:12 ` Peter Zijlstra
-1 siblings, 0 replies; 102+ messages in thread
From: Peter Zijlstra @ 2014-06-15 13:12 UTC (permalink / raw)
To: Waiman Long
Cc: linux-arch, Raghavendra K T, Oleg Nesterov, Gleb Natapov, kvm,
Scott J Norton, x86, Paolo Bonzini, linux-kernel, virtualization,
Ingo Molnar, Chegu Vinod, David Vrabel, H. Peter Anvin,
xen-devel, Thomas Gleixner, Paul E. McKenney, Linus Torvalds,
Boris Ostrovsky
On Thu, Jun 12, 2014 at 04:54:52PM -0400, Waiman Long wrote:
> If two tasks see the pending bit goes away and try to grab it with cmpxchg,
> there is no way we can avoid the contention. However, if some how the
> pending bit holder get the lock and another task set the pending bit before
> the current task, the spinlock value will become
> _Q_PENDING_VAL|_Q_LOCKED_VAL. The while loop will end and the code will
> blindly try to do a cmpxchg unless we check for this case before hand. This
> is what my code does by going back to the beginning of the for loop.
There is already a test for that; see the goto queue;
---
/*
* wait for in-progress pending->locked hand-overs
*
* 0,1,0 -> 0,0,1
*/
if (val == _Q_PENDING_VAL) {
while ((val = atomic_read(&lock->val)) == _Q_PENDING_VAL)
cpu_relax();
}
/*
* trylock || pending
*
* 0,0,0 -> 0,0,1 ; trylock
* 0,0,1 -> 0,1,1 ; pending
*/
for (;;) {
/*
* If we observe any contention; queue.
*/
if (val & ~_Q_LOCKED_MASK)
goto queue;
new = _Q_LOCKED_VAL;
if (val == new)
new |= _Q_PENDING_VAL;
old = atomic_cmpxchg(&lock->val, val, new);
if (old == val)
break;
val = old;
}
^ permalink raw reply [flat|nested] 102+ messages in thread* Re: [PATCH v11 06/16] qspinlock: prolong the stay in the pending bit path
2014-06-12 20:54 ` Waiman Long
` (2 preceding siblings ...)
(?)
@ 2014-06-15 13:12 ` Peter Zijlstra
-1 siblings, 0 replies; 102+ messages in thread
From: Peter Zijlstra @ 2014-06-15 13:12 UTC (permalink / raw)
To: Waiman Long
Cc: Thomas Gleixner, Ingo Molnar, H. Peter Anvin, linux-arch, x86,
linux-kernel, virtualization, xen-devel, kvm, Paolo Bonzini,
Konrad Rzeszutek Wilk, Boris Ostrovsky, Paul E. McKenney,
Rik van Riel, Linus Torvalds, Raghavendra K T, David Vrabel,
Oleg Nesterov, Gleb Natapov, Scott J Norton, Chegu Vinod
On Thu, Jun 12, 2014 at 04:54:52PM -0400, Waiman Long wrote:
> If two tasks see the pending bit goes away and try to grab it with cmpxchg,
> there is no way we can avoid the contention. However, if some how the
> pending bit holder get the lock and another task set the pending bit before
> the current task, the spinlock value will become
> _Q_PENDING_VAL|_Q_LOCKED_VAL. The while loop will end and the code will
> blindly try to do a cmpxchg unless we check for this case before hand. This
> is what my code does by going back to the beginning of the for loop.
There is already a test for that; see the goto queue;
---
/*
* wait for in-progress pending->locked hand-overs
*
* 0,1,0 -> 0,0,1
*/
if (val == _Q_PENDING_VAL) {
while ((val = atomic_read(&lock->val)) == _Q_PENDING_VAL)
cpu_relax();
}
/*
* trylock || pending
*
* 0,0,0 -> 0,0,1 ; trylock
* 0,0,1 -> 0,1,1 ; pending
*/
for (;;) {
/*
* If we observe any contention; queue.
*/
if (val & ~_Q_LOCKED_MASK)
goto queue;
new = _Q_LOCKED_VAL;
if (val == new)
new |= _Q_PENDING_VAL;
old = atomic_cmpxchg(&lock->val, val, new);
if (old == val)
break;
val = old;
}
^ permalink raw reply [flat|nested] 102+ messages in thread
* Re: [PATCH v11 06/16] qspinlock: prolong the stay in the pending bit path
2014-06-12 6:00 ` Peter Zijlstra
2014-06-12 20:54 ` Waiman Long
@ 2014-06-12 20:54 ` Waiman Long
1 sibling, 0 replies; 102+ messages in thread
From: Waiman Long @ 2014-06-12 20:54 UTC (permalink / raw)
To: Peter Zijlstra
Cc: linux-arch, Raghavendra K T, Oleg Nesterov, Gleb Natapov, kvm,
Scott J Norton, x86, Paolo Bonzini, linux-kernel, virtualization,
Ingo Molnar, Chegu Vinod, David Vrabel, H. Peter Anvin,
xen-devel, Thomas Gleixner, Paul E. McKenney, Linus Torvalds,
Boris Ostrovsky
On 06/12/2014 02:00 AM, Peter Zijlstra wrote:
> On Wed, Jun 11, 2014 at 05:22:28PM -0400, Long, Wai Man wrote:
>
>>>> @@ -233,11 +233,25 @@ void queue_spin_lock_slowpath(struct qspinlock *lock, u32 val)
>>>> */
>>>> for (;;) {
>>>> /*
>>>> - * If we observe any contention; queue.
>>>> + * If we observe that the queue is not empty or both
>>>> + * the pending and lock bits are set, queue
>>>> */
>>>> - if (val& ~_Q_LOCKED_MASK)
>>>> + if ((val& _Q_TAIL_MASK) ||
>>>> + (val == (_Q_LOCKED_VAL|_Q_PENDING_VAL)))
>>>> goto queue;
>>>> + if (val == _Q_PENDING_VAL) {
>>>> + /*
>>>> + * Pending bit is set, but not the lock bit.
>>>> + * Assuming that the pending bit holder is going to
>>>> + * set the lock bit and clear the pending bit soon,
>>>> + * it is better to wait than to exit at this point.
>>>> + */
>>>> + cpu_relax();
>>>> + val = atomic_read(&lock->val);
>>>> + continue;
>>>> + }
>>>> +
>>>> new = _Q_LOCKED_VAL;
>>>> if (val == new)
>>>> new |= _Q_PENDING_VAL;
>>> Wouldn't something like:
>>>
>>> while (atomic_read(&lock->val) == _Q_PENDING_VAL)
>>> cpu_relax();
>>>
>>> before the cmpxchg loop have gotten you all this?
>> That is not exactly the same. The loop will exit if other bits are set or the pending
>> bit cleared. In the case, we will need to do the same check at the beginning of the
>> for loop in order to avoid doing an extra cmpxchg that is not necessary.
> If other bits get set we should stop poking at the pending bit and get
> queued. The only transition we want to wait for is: 0,1,0 -> 0,0,1.
>
> What extra unneeded cmpxchg() is there? If we have two cpus waiting in
> this loop for the pending bit to go away then both will attempt to grab
> the now free pending bit, one will loose and get queued?
>
> There's no avoiding that contention.
If two tasks see the pending bit goes away and try to grab it with
cmpxchg, there is no way we can avoid the contention. However, if some
how the pending bit holder get the lock and another task set the pending
bit before the current task, the spinlock value will become
_Q_PENDING_VAL|_Q_LOCKED_VAL. The while loop will end and the code will
blindly try to do a cmpxchg unless we check for this case before hand.
This is what my code does by going back to the beginning of the for loop.
-Longman
^ permalink raw reply [flat|nested] 102+ messages in thread
* Re: [PATCH v11 06/16] qspinlock: prolong the stay in the pending bit path
2014-06-11 21:22 ` Long, Wai Man
(?)
(?)
@ 2014-06-12 6:00 ` Peter Zijlstra
-1 siblings, 0 replies; 102+ messages in thread
From: Peter Zijlstra @ 2014-06-12 6:00 UTC (permalink / raw)
To: Long, Wai Man
Cc: linux-arch, Rik van Riel, Raghavendra K T, Oleg Nesterov,
Gleb Natapov, kvm, Konrad Rzeszutek Wilk, Scott J Norton, x86,
Paolo Bonzini, linux-kernel, virtualization, Ingo Molnar,
Chegu Vinod, David Vrabel, H. Peter Anvin, xen-devel,
Thomas Gleixner, Paul E. McKenney, Linus Torvalds,
Boris Ostrovsky
[-- Attachment #1.1: Type: text/plain, Size: 1821 bytes --]
On Wed, Jun 11, 2014 at 05:22:28PM -0400, Long, Wai Man wrote:
> >>@@ -233,11 +233,25 @@ void queue_spin_lock_slowpath(struct qspinlock *lock, u32 val)
> >> */
> >> for (;;) {
> >> /*
> >>- * If we observe any contention; queue.
> >>+ * If we observe that the queue is not empty or both
> >>+ * the pending and lock bits are set, queue
> >> */
> >>- if (val & ~_Q_LOCKED_MASK)
> >>+ if ((val & _Q_TAIL_MASK) ||
> >>+ (val == (_Q_LOCKED_VAL|_Q_PENDING_VAL)))
> >> goto queue;
> >>+ if (val == _Q_PENDING_VAL) {
> >>+ /*
> >>+ * Pending bit is set, but not the lock bit.
> >>+ * Assuming that the pending bit holder is going to
> >>+ * set the lock bit and clear the pending bit soon,
> >>+ * it is better to wait than to exit at this point.
> >>+ */
> >>+ cpu_relax();
> >>+ val = atomic_read(&lock->val);
> >>+ continue;
> >>+ }
> >>+
> >> new = _Q_LOCKED_VAL;
> >> if (val == new)
> >> new |= _Q_PENDING_VAL;
> >Wouldn't something like:
> >
> > while (atomic_read(&lock->val) == _Q_PENDING_VAL)
> > cpu_relax();
> >
> >before the cmpxchg loop have gotten you all this?
>
> That is not exactly the same. The loop will exit if other bits are set or the pending
> bit cleared. In the case, we will need to do the same check at the beginning of the
> for loop in order to avoid doing an extra cmpxchg that is not necessary.
If other bits get set we should stop poking at the pending bit and get
queued. The only transition we want to wait for is: 0,1,0 -> 0,0,1.
What extra unneeded cmpxchg() is there? If we have two cpus waiting in
this loop for the pending bit to go away then both will attempt to grab
the now free pending bit, one will loose and get queued?
There's no avoiding that contention.
[-- Attachment #1.2: Type: application/pgp-signature, Size: 836 bytes --]
[-- Attachment #2: Type: text/plain, Size: 183 bytes --]
_______________________________________________
Virtualization mailing list
Virtualization@lists.linux-foundation.org
https://lists.linuxfoundation.org/mailman/listinfo/virtualization
^ permalink raw reply [flat|nested] 102+ messages in thread* Re: [PATCH v11 06/16] qspinlock: prolong the stay in the pending bit path
2014-06-11 21:22 ` Long, Wai Man
` (2 preceding siblings ...)
(?)
@ 2014-06-12 6:00 ` Peter Zijlstra
-1 siblings, 0 replies; 102+ messages in thread
From: Peter Zijlstra @ 2014-06-12 6:00 UTC (permalink / raw)
To: Long, Wai Man
Cc: linux-arch, Raghavendra K T, Oleg Nesterov, Gleb Natapov, kvm,
Scott J Norton, x86, Paolo Bonzini, linux-kernel, virtualization,
Ingo Molnar, Chegu Vinod, David Vrabel, H. Peter Anvin,
xen-devel, Thomas Gleixner, Paul E. McKenney, Linus Torvalds,
Boris Ostrovsky
[-- Attachment #1.1: Type: text/plain, Size: 1821 bytes --]
On Wed, Jun 11, 2014 at 05:22:28PM -0400, Long, Wai Man wrote:
> >>@@ -233,11 +233,25 @@ void queue_spin_lock_slowpath(struct qspinlock *lock, u32 val)
> >> */
> >> for (;;) {
> >> /*
> >>- * If we observe any contention; queue.
> >>+ * If we observe that the queue is not empty or both
> >>+ * the pending and lock bits are set, queue
> >> */
> >>- if (val & ~_Q_LOCKED_MASK)
> >>+ if ((val & _Q_TAIL_MASK) ||
> >>+ (val == (_Q_LOCKED_VAL|_Q_PENDING_VAL)))
> >> goto queue;
> >>+ if (val == _Q_PENDING_VAL) {
> >>+ /*
> >>+ * Pending bit is set, but not the lock bit.
> >>+ * Assuming that the pending bit holder is going to
> >>+ * set the lock bit and clear the pending bit soon,
> >>+ * it is better to wait than to exit at this point.
> >>+ */
> >>+ cpu_relax();
> >>+ val = atomic_read(&lock->val);
> >>+ continue;
> >>+ }
> >>+
> >> new = _Q_LOCKED_VAL;
> >> if (val == new)
> >> new |= _Q_PENDING_VAL;
> >Wouldn't something like:
> >
> > while (atomic_read(&lock->val) == _Q_PENDING_VAL)
> > cpu_relax();
> >
> >before the cmpxchg loop have gotten you all this?
>
> That is not exactly the same. The loop will exit if other bits are set or the pending
> bit cleared. In the case, we will need to do the same check at the beginning of the
> for loop in order to avoid doing an extra cmpxchg that is not necessary.
If other bits get set we should stop poking at the pending bit and get
queued. The only transition we want to wait for is: 0,1,0 -> 0,0,1.
What extra unneeded cmpxchg() is there? If we have two cpus waiting in
this loop for the pending bit to go away then both will attempt to grab
the now free pending bit, one will loose and get queued?
There's no avoiding that contention.
[-- Attachment #1.2: Type: application/pgp-signature, Size: 836 bytes --]
[-- Attachment #2: Type: text/plain, Size: 126 bytes --]
_______________________________________________
Xen-devel mailing list
Xen-devel@lists.xen.org
http://lists.xen.org/xen-devel
^ permalink raw reply [flat|nested] 102+ messages in thread
* Re: [PATCH v11 06/16] qspinlock: prolong the stay in the pending bit path
2014-05-30 15:43 ` [PATCH v11 06/16] qspinlock: prolong the stay in the pending bit path Waiman Long
2014-06-11 10:26 ` Peter Zijlstra
@ 2014-06-11 10:26 ` Peter Zijlstra
1 sibling, 0 replies; 102+ messages in thread
From: Peter Zijlstra @ 2014-06-11 10:26 UTC (permalink / raw)
To: Waiman Long
Cc: linux-arch, Raghavendra K T, Oleg Nesterov, Gleb Natapov, kvm,
Scott J Norton, x86, Paolo Bonzini, linux-kernel, virtualization,
Ingo Molnar, Chegu Vinod, David Vrabel, H. Peter Anvin,
xen-devel, Thomas Gleixner, Paul E. McKenney, Linus Torvalds,
Boris Ostrovsky
[-- Attachment #1.1: Type: text/plain, Size: 1475 bytes --]
On Fri, May 30, 2014 at 11:43:52AM -0400, Waiman Long wrote:
> ---
> kernel/locking/qspinlock.c | 18 ++++++++++++++++--
> 1 files changed, 16 insertions(+), 2 deletions(-)
>
> diff --git a/kernel/locking/qspinlock.c b/kernel/locking/qspinlock.c
> index fc7fd8c..7f10758 100644
> --- a/kernel/locking/qspinlock.c
> +++ b/kernel/locking/qspinlock.c
> @@ -233,11 +233,25 @@ void queue_spin_lock_slowpath(struct qspinlock *lock, u32 val)
> */
> for (;;) {
> /*
> - * If we observe any contention; queue.
> + * If we observe that the queue is not empty or both
> + * the pending and lock bits are set, queue
> */
> - if (val & ~_Q_LOCKED_MASK)
> + if ((val & _Q_TAIL_MASK) ||
> + (val == (_Q_LOCKED_VAL|_Q_PENDING_VAL)))
> goto queue;
>
> + if (val == _Q_PENDING_VAL) {
> + /*
> + * Pending bit is set, but not the lock bit.
> + * Assuming that the pending bit holder is going to
> + * set the lock bit and clear the pending bit soon,
> + * it is better to wait than to exit at this point.
> + */
> + cpu_relax();
> + val = atomic_read(&lock->val);
> + continue;
> + }
> +
> new = _Q_LOCKED_VAL;
> if (val == new)
> new |= _Q_PENDING_VAL;
So, again, you just posted a new version without replying to the
previous discussion; so let me try again, what's wrong with the proposal
here:
lkml.kernel.org/r/20140417163640.GT11096@twins.programming.kicks-ass.net
[-- Attachment #1.2: Type: application/pgp-signature, Size: 836 bytes --]
[-- Attachment #2: Type: text/plain, Size: 126 bytes --]
_______________________________________________
Xen-devel mailing list
Xen-devel@lists.xen.org
http://lists.xen.org/xen-devel
^ permalink raw reply [flat|nested] 102+ messages in thread
* [PATCH v11 06/16] qspinlock: prolong the stay in the pending bit path
2014-05-30 15:43 [PATCH v11 00/16] qspinlock: a 4-byte queue spinlock with PV support Waiman Long
` (11 preceding siblings ...)
2014-05-30 15:43 ` [PATCH v11 06/16] qspinlock: prolong the stay in the pending bit path Waiman Long
@ 2014-05-30 15:43 ` Waiman Long
2014-05-30 15:43 ` Waiman Long
` (24 subsequent siblings)
37 siblings, 0 replies; 102+ messages in thread
From: Waiman Long @ 2014-05-30 15:43 UTC (permalink / raw)
To: Thomas Gleixner, Ingo Molnar, H. Peter Anvin, Peter Zijlstra
Cc: linux-arch, Waiman Long, Rik van Riel, Raghavendra K T,
Gleb Natapov, kvm, Konrad Rzeszutek Wilk, Scott J Norton, x86,
Paolo Bonzini, linux-kernel, virtualization, Chegu Vinod,
David Vrabel, Oleg Nesterov, xen-devel, Boris Ostrovsky,
Paul E. McKenney, Linus Torvalds
There is a problem in the current pending bit spinning code. When the
lock is free, but the pending bit holder hasn't grabbed the lock &
cleared the pending bit yet, the spinning code will not be run.
As a result, the regular queuing code path might be used most of
the time even when there is only 2 tasks contending for the lock.
Assuming that the pending bit holder is going to get the lock and
clear the pending bit soon, it is actually better to wait than to be
queued up which has a higher overhead.
The following tables show the before-patch execution time (in ms)
of a micro-benchmark where 5M iterations of the lock/unlock cycles
were run on a 10-core Westere-EX x86-64 CPU with 2 different types of
loads - standalone (lock and protected data in different cachelines)
and embedded (lock and protected data in the same cacheline).
[Standalone/Embedded - same node]
# of tasks Ticket lock Queue lock %Change
---------- ----------- ---------- -------
1 135/ 111 135/ 101 0%/ -9%
2 890/ 779 1885/1990 +112%/+156%
3 1932/1859 2333/2341 +21%/ +26%
4 2829/2726 2900/2923 +3%/ +7%
5 3834/3761 3655/3648 -5%/ -3%
6 4963/4976 4336/4326 -13%/ -13%
7 6299/6269 5057/5064 -20%/ -19%
8 7691/7569 5786/5798 -25%/ -23%
Of course, the results will varies depending on what kind of test
machine is used.
With 1 task per NUMA node, the execution times are:
[Standalone - different nodes]
# of nodes Ticket lock Queue lock %Change
---------- ----------- ---------- -------
1 135 135 0%
2 4604 5087 +10%
3 10940 12224 +12%
4 21555 10555 -51%
It can be seen that the queue spinlock is slower than the ticket
spinlock when there are 2 or 3 contending tasks. In all the other case,
the queue spinlock is either equal or faster than the ticket spinlock.
With this patch, the performance data for 2 contending tasks are:
[Standalone/Embedded]
# of tasks Ticket lock Queue lock %Change
---------- ----------- ---------- -------
2 890/779 984/871 +11%/+12%
[Standalone - different nodes]
# of nodes Ticket lock Queue lock %Change
---------- ----------- ---------- -------
2 4604 1364 -70%
It can be seen that the queue spinlock performance for 2 contending
tasks is now comparable to ticket spinlock on the same node, but much
faster when in different nodes. With 3 contending tasks, however,
the ticket spinlock is still quite a bit faster.
Signed-off-by: Waiman Long <Waiman.Long@hp.com>
---
kernel/locking/qspinlock.c | 18 ++++++++++++++++--
1 files changed, 16 insertions(+), 2 deletions(-)
diff --git a/kernel/locking/qspinlock.c b/kernel/locking/qspinlock.c
index fc7fd8c..7f10758 100644
--- a/kernel/locking/qspinlock.c
+++ b/kernel/locking/qspinlock.c
@@ -233,11 +233,25 @@ void queue_spin_lock_slowpath(struct qspinlock *lock, u32 val)
*/
for (;;) {
/*
- * If we observe any contention; queue.
+ * If we observe that the queue is not empty or both
+ * the pending and lock bits are set, queue
*/
- if (val & ~_Q_LOCKED_MASK)
+ if ((val & _Q_TAIL_MASK) ||
+ (val == (_Q_LOCKED_VAL|_Q_PENDING_VAL)))
goto queue;
+ if (val == _Q_PENDING_VAL) {
+ /*
+ * Pending bit is set, but not the lock bit.
+ * Assuming that the pending bit holder is going to
+ * set the lock bit and clear the pending bit soon,
+ * it is better to wait than to exit at this point.
+ */
+ cpu_relax();
+ val = atomic_read(&lock->val);
+ continue;
+ }
+
new = _Q_LOCKED_VAL;
if (val == new)
new |= _Q_PENDING_VAL;
--
1.7.1
^ permalink raw reply related [flat|nested] 102+ messages in thread* [PATCH v11 06/16] qspinlock: prolong the stay in the pending bit path
2014-05-30 15:43 [PATCH v11 00/16] qspinlock: a 4-byte queue spinlock with PV support Waiman Long
` (12 preceding siblings ...)
2014-05-30 15:43 ` Waiman Long
@ 2014-05-30 15:43 ` Waiman Long
2014-05-30 15:43 ` Waiman Long
` (23 subsequent siblings)
37 siblings, 0 replies; 102+ messages in thread
From: Waiman Long @ 2014-05-30 15:43 UTC (permalink / raw)
To: Thomas Gleixner, Ingo Molnar, H. Peter Anvin, Peter Zijlstra
Cc: linux-arch, Waiman Long, Raghavendra K T, Gleb Natapov, kvm,
Scott J Norton, x86, Paolo Bonzini, linux-kernel, virtualization,
Chegu Vinod, David Vrabel, Oleg Nesterov, xen-devel,
Boris Ostrovsky, Paul E. McKenney, Linus Torvalds
There is a problem in the current pending bit spinning code. When the
lock is free, but the pending bit holder hasn't grabbed the lock &
cleared the pending bit yet, the spinning code will not be run.
As a result, the regular queuing code path might be used most of
the time even when there is only 2 tasks contending for the lock.
Assuming that the pending bit holder is going to get the lock and
clear the pending bit soon, it is actually better to wait than to be
queued up which has a higher overhead.
The following tables show the before-patch execution time (in ms)
of a micro-benchmark where 5M iterations of the lock/unlock cycles
were run on a 10-core Westere-EX x86-64 CPU with 2 different types of
loads - standalone (lock and protected data in different cachelines)
and embedded (lock and protected data in the same cacheline).
[Standalone/Embedded - same node]
# of tasks Ticket lock Queue lock %Change
---------- ----------- ---------- -------
1 135/ 111 135/ 101 0%/ -9%
2 890/ 779 1885/1990 +112%/+156%
3 1932/1859 2333/2341 +21%/ +26%
4 2829/2726 2900/2923 +3%/ +7%
5 3834/3761 3655/3648 -5%/ -3%
6 4963/4976 4336/4326 -13%/ -13%
7 6299/6269 5057/5064 -20%/ -19%
8 7691/7569 5786/5798 -25%/ -23%
Of course, the results will varies depending on what kind of test
machine is used.
With 1 task per NUMA node, the execution times are:
[Standalone - different nodes]
# of nodes Ticket lock Queue lock %Change
---------- ----------- ---------- -------
1 135 135 0%
2 4604 5087 +10%
3 10940 12224 +12%
4 21555 10555 -51%
It can be seen that the queue spinlock is slower than the ticket
spinlock when there are 2 or 3 contending tasks. In all the other case,
the queue spinlock is either equal or faster than the ticket spinlock.
With this patch, the performance data for 2 contending tasks are:
[Standalone/Embedded]
# of tasks Ticket lock Queue lock %Change
---------- ----------- ---------- -------
2 890/779 984/871 +11%/+12%
[Standalone - different nodes]
# of nodes Ticket lock Queue lock %Change
---------- ----------- ---------- -------
2 4604 1364 -70%
It can be seen that the queue spinlock performance for 2 contending
tasks is now comparable to ticket spinlock on the same node, but much
faster when in different nodes. With 3 contending tasks, however,
the ticket spinlock is still quite a bit faster.
Signed-off-by: Waiman Long <Waiman.Long@hp.com>
---
kernel/locking/qspinlock.c | 18 ++++++++++++++++--
1 files changed, 16 insertions(+), 2 deletions(-)
diff --git a/kernel/locking/qspinlock.c b/kernel/locking/qspinlock.c
index fc7fd8c..7f10758 100644
--- a/kernel/locking/qspinlock.c
+++ b/kernel/locking/qspinlock.c
@@ -233,11 +233,25 @@ void queue_spin_lock_slowpath(struct qspinlock *lock, u32 val)
*/
for (;;) {
/*
- * If we observe any contention; queue.
+ * If we observe that the queue is not empty or both
+ * the pending and lock bits are set, queue
*/
- if (val & ~_Q_LOCKED_MASK)
+ if ((val & _Q_TAIL_MASK) ||
+ (val == (_Q_LOCKED_VAL|_Q_PENDING_VAL)))
goto queue;
+ if (val == _Q_PENDING_VAL) {
+ /*
+ * Pending bit is set, but not the lock bit.
+ * Assuming that the pending bit holder is going to
+ * set the lock bit and clear the pending bit soon,
+ * it is better to wait than to exit at this point.
+ */
+ cpu_relax();
+ val = atomic_read(&lock->val);
+ continue;
+ }
+
new = _Q_LOCKED_VAL;
if (val == new)
new |= _Q_PENDING_VAL;
--
1.7.1
^ permalink raw reply related [flat|nested] 102+ messages in thread* [PATCH v11 07/16] qspinlock: Use a simple write to grab the lock, if applicable
2014-05-30 15:43 [PATCH v11 00/16] qspinlock: a 4-byte queue spinlock with PV support Waiman Long
@ 2014-05-30 15:43 ` Waiman Long
2014-05-30 15:43 ` Waiman Long
` (36 subsequent siblings)
37 siblings, 0 replies; 102+ messages in thread
From: Waiman Long @ 2014-05-30 15:43 UTC (permalink / raw)
To: Thomas Gleixner, Ingo Molnar, H. Peter Anvin, Peter Zijlstra
Cc: linux-arch, x86, linux-kernel, virtualization, xen-devel, kvm,
Paolo Bonzini, Konrad Rzeszutek Wilk, Boris Ostrovsky,
Paul E. McKenney, Rik van Riel, Linus Torvalds, Raghavendra K T,
David Vrabel, Oleg Nesterov, Gleb Natapov, Scott J Norton,
Chegu Vinod, Waiman Long
Currently, atomic_cmpxchg() is used to get the lock. However, this is
not really necessary if there is more than one task in the queue and
the queue head don't need to reset the queue code word. For that case,
a simple write to set the lock bit is enough as the queue head will
be the only one eligible to get the lock as long as it checks that
both the lock and pending bits are not set. The current pending bit
waiting code will ensure that the bit will not be set as soon as the
queue code word (tail) in the lock is set.
With that change, the are some slight improvement in the performance
of the queue spinlock in the 5M loop micro-benchmark run on a 4-socket
Westere-EX machine as shown in the tables below.
[Standalone/Embedded - same node]
# of tasks Before patch After patch %Change
---------- ----------- ---------- -------
3 2324/2321 2248/2265 -3%/-2%
4 2890/2896 2819/2831 -2%/-2%
5 3611/3595 3522/3512 -2%/-2%
6 4281/4276 4173/4160 -3%/-3%
7 5018/5001 4875/4861 -3%/-3%
8 5759/5750 5563/5568 -3%/-3%
[Standalone/Embedded - different nodes]
# of tasks Before patch After patch %Change
---------- ----------- ---------- -------
3 12242/12237 12087/12093 -1%/-1%
4 10688/10696 10507/10521 -2%/-2%
It was also found that this change produced a much bigger performance
improvement in the newer IvyBridge-EX chip and was essentially to close
the performance gap between the ticket spinlock and queue spinlock.
The disk workload of the AIM7 benchmark was run on a 4-socket
Westmere-EX machine with both ext4 and xfs RAM disks at 3000 users
on a 3.14 based kernel. The results of the test runs were:
AIM7 XFS Disk Test
kernel JPM Real Time Sys Time Usr Time
----- --- --------- -------- --------
ticketlock 5678233 3.17 96.61 5.81
qspinlock 5750799 3.13 94.83 5.97
AIM7 EXT4 Disk Test
kernel JPM Real Time Sys Time Usr Time
----- --- --------- -------- --------
ticketlock 1114551 16.15 509.72 7.11
qspinlock 2184466 8.24 232.99 6.01
The ext4 filesystem run had a much higher spinlock contention than
the xfs filesystem run.
The "ebizzy -m" test was also run with the following results:
kernel records/s Real Time Sys Time Usr Time
----- --------- --------- -------- --------
ticketlock 2075 10.00 216.35 3.49
qspinlock 3023 10.00 198.20 4.80
Signed-off-by: Waiman Long <Waiman.Long@hp.com>
---
kernel/locking/qspinlock.c | 62 ++++++++++++++++++++++++++++++++-----------
1 files changed, 46 insertions(+), 16 deletions(-)
diff --git a/kernel/locking/qspinlock.c b/kernel/locking/qspinlock.c
index 7f10758..2c7abe7 100644
--- a/kernel/locking/qspinlock.c
+++ b/kernel/locking/qspinlock.c
@@ -93,24 +93,33 @@ static inline struct mcs_spinlock *decode_tail(u32 tail)
* By using the whole 2nd least significant byte for the pending bit, we
* can allow better optimization of the lock acquisition for the pending
* bit holder.
+ *
+ * This internal structure is also used by the set_locked function which
+ * is not restricted to _Q_PENDING_BITS == 8.
*/
-#if _Q_PENDING_BITS == 8
-
struct __qspinlock {
union {
atomic_t val;
- struct {
#ifdef __LITTLE_ENDIAN
+ u8 locked;
+ struct {
u16 locked_pending;
u16 tail;
+ };
#else
+ struct {
u16 tail;
u16 locked_pending;
-#endif
};
+ struct {
+ u8 reserved[3];
+ u8 locked;
+ };
+#endif
};
};
+#if _Q_PENDING_BITS == 8
/**
* clear_pending_set_locked - take ownership and clear the pending bit.
* @lock: Pointer to queue spinlock structure
@@ -197,6 +206,22 @@ static __always_inline u32 xchg_tail(struct qspinlock *lock, u32 tail)
#endif /* _Q_PENDING_BITS == 8 */
/**
+ * set_locked - Set the lock bit and own the lock
+ * @lock: Pointer to queue spinlock structure
+ *
+ * This routine should only be called when the caller is the only one
+ * entitled to acquire the lock.
+ */
+static __always_inline void set_locked(struct qspinlock *lock)
+{
+ struct __qspinlock *l = (void *)lock;
+
+ barrier();
+ ACCESS_ONCE(l->locked) = _Q_LOCKED_VAL;
+ barrier();
+}
+
+/**
* queue_spin_lock_slowpath - acquire the queue spinlock
* @lock: Pointer to queue spinlock structure
* @val: Current value of the queue spinlock 32-bit word
@@ -332,10 +357,13 @@ queue:
/*
* we're at the head of the waitqueue, wait for the owner & pending to
* go away.
+ * Load-acquired is used here because the set_locked()
+ * function below may not be a full memory barrier.
*
* *,x,y -> *,0,0
*/
- while ((val = atomic_read(&lock->val)) & _Q_LOCKED_PENDING_MASK)
+ while ((val = smp_load_acquire(&lock->val.counter))
+ & _Q_LOCKED_PENDING_MASK)
arch_mutex_cpu_relax();
/*
@@ -343,15 +371,19 @@ queue:
*
* n,0,0 -> 0,0,1 : lock, uncontended
* *,0,0 -> *,0,1 : lock, contended
+ *
+ * If the queue head is the only one in the queue (lock value == tail),
+ * clear the tail code and grab the lock. Otherwise, we only need
+ * to grab the lock.
*/
for (;;) {
- new = _Q_LOCKED_VAL;
- if (val != tail)
- new |= val;
-
- old = atomic_cmpxchg(&lock->val, val, new);
- if (old == val)
+ if (val != tail) {
+ set_locked(lock);
break;
+ }
+ old = atomic_cmpxchg(&lock->val, val, _Q_LOCKED_VAL);
+ if (old == val)
+ goto release; /* No contention */
val = old;
}
@@ -359,12 +391,10 @@ queue:
/*
* contended path; wait for next, release.
*/
- if (new != _Q_LOCKED_VAL) {
- while (!(next = ACCESS_ONCE(node->next)))
- arch_mutex_cpu_relax();
+ while (!(next = ACCESS_ONCE(node->next)))
+ arch_mutex_cpu_relax();
- arch_mcs_spin_unlock_contended(&next->locked);
- }
+ arch_mcs_spin_unlock_contended(&next->locked);
release:
/*
--
1.7.1
^ permalink raw reply related [flat|nested] 102+ messages in thread* [PATCH v11 07/16] qspinlock: Use a simple write to grab the lock, if applicable
@ 2014-05-30 15:43 ` Waiman Long
0 siblings, 0 replies; 102+ messages in thread
From: Waiman Long @ 2014-05-30 15:43 UTC (permalink / raw)
To: Thomas Gleixner, Ingo Molnar, H. Peter Anvin, Peter Zijlstra
Cc: linux-arch, Waiman Long, Rik van Riel, Raghavendra K T,
Gleb Natapov, kvm, Konrad Rzeszutek Wilk, Scott J Norton, x86,
Paolo Bonzini, linux-kernel, virtualization, Chegu Vinod,
David Vrabel, Oleg Nesterov, xen-devel, Boris Ostrovsky,
Paul E. McKenney, Linus Torvalds
Currently, atomic_cmpxchg() is used to get the lock. However, this is
not really necessary if there is more than one task in the queue and
the queue head don't need to reset the queue code word. For that case,
a simple write to set the lock bit is enough as the queue head will
be the only one eligible to get the lock as long as it checks that
both the lock and pending bits are not set. The current pending bit
waiting code will ensure that the bit will not be set as soon as the
queue code word (tail) in the lock is set.
With that change, the are some slight improvement in the performance
of the queue spinlock in the 5M loop micro-benchmark run on a 4-socket
Westere-EX machine as shown in the tables below.
[Standalone/Embedded - same node]
# of tasks Before patch After patch %Change
---------- ----------- ---------- -------
3 2324/2321 2248/2265 -3%/-2%
4 2890/2896 2819/2831 -2%/-2%
5 3611/3595 3522/3512 -2%/-2%
6 4281/4276 4173/4160 -3%/-3%
7 5018/5001 4875/4861 -3%/-3%
8 5759/5750 5563/5568 -3%/-3%
[Standalone/Embedded - different nodes]
# of tasks Before patch After patch %Change
---------- ----------- ---------- -------
3 12242/12237 12087/12093 -1%/-1%
4 10688/10696 10507/10521 -2%/-2%
It was also found that this change produced a much bigger performance
improvement in the newer IvyBridge-EX chip and was essentially to close
the performance gap between the ticket spinlock and queue spinlock.
The disk workload of the AIM7 benchmark was run on a 4-socket
Westmere-EX machine with both ext4 and xfs RAM disks at 3000 users
on a 3.14 based kernel. The results of the test runs were:
AIM7 XFS Disk Test
kernel JPM Real Time Sys Time Usr Time
----- --- --------- -------- --------
ticketlock 5678233 3.17 96.61 5.81
qspinlock 5750799 3.13 94.83 5.97
AIM7 EXT4 Disk Test
kernel JPM Real Time Sys Time Usr Time
----- --- --------- -------- --------
ticketlock 1114551 16.15 509.72 7.11
qspinlock 2184466 8.24 232.99 6.01
The ext4 filesystem run had a much higher spinlock contention than
the xfs filesystem run.
The "ebizzy -m" test was also run with the following results:
kernel records/s Real Time Sys Time Usr Time
----- --------- --------- -------- --------
ticketlock 2075 10.00 216.35 3.49
qspinlock 3023 10.00 198.20 4.80
Signed-off-by: Waiman Long <Waiman.Long@hp.com>
---
kernel/locking/qspinlock.c | 62 ++++++++++++++++++++++++++++++++-----------
1 files changed, 46 insertions(+), 16 deletions(-)
diff --git a/kernel/locking/qspinlock.c b/kernel/locking/qspinlock.c
index 7f10758..2c7abe7 100644
--- a/kernel/locking/qspinlock.c
+++ b/kernel/locking/qspinlock.c
@@ -93,24 +93,33 @@ static inline struct mcs_spinlock *decode_tail(u32 tail)
* By using the whole 2nd least significant byte for the pending bit, we
* can allow better optimization of the lock acquisition for the pending
* bit holder.
+ *
+ * This internal structure is also used by the set_locked function which
+ * is not restricted to _Q_PENDING_BITS == 8.
*/
-#if _Q_PENDING_BITS == 8
-
struct __qspinlock {
union {
atomic_t val;
- struct {
#ifdef __LITTLE_ENDIAN
+ u8 locked;
+ struct {
u16 locked_pending;
u16 tail;
+ };
#else
+ struct {
u16 tail;
u16 locked_pending;
-#endif
};
+ struct {
+ u8 reserved[3];
+ u8 locked;
+ };
+#endif
};
};
+#if _Q_PENDING_BITS == 8
/**
* clear_pending_set_locked - take ownership and clear the pending bit.
* @lock: Pointer to queue spinlock structure
@@ -197,6 +206,22 @@ static __always_inline u32 xchg_tail(struct qspinlock *lock, u32 tail)
#endif /* _Q_PENDING_BITS == 8 */
/**
+ * set_locked - Set the lock bit and own the lock
+ * @lock: Pointer to queue spinlock structure
+ *
+ * This routine should only be called when the caller is the only one
+ * entitled to acquire the lock.
+ */
+static __always_inline void set_locked(struct qspinlock *lock)
+{
+ struct __qspinlock *l = (void *)lock;
+
+ barrier();
+ ACCESS_ONCE(l->locked) = _Q_LOCKED_VAL;
+ barrier();
+}
+
+/**
* queue_spin_lock_slowpath - acquire the queue spinlock
* @lock: Pointer to queue spinlock structure
* @val: Current value of the queue spinlock 32-bit word
@@ -332,10 +357,13 @@ queue:
/*
* we're at the head of the waitqueue, wait for the owner & pending to
* go away.
+ * Load-acquired is used here because the set_locked()
+ * function below may not be a full memory barrier.
*
* *,x,y -> *,0,0
*/
- while ((val = atomic_read(&lock->val)) & _Q_LOCKED_PENDING_MASK)
+ while ((val = smp_load_acquire(&lock->val.counter))
+ & _Q_LOCKED_PENDING_MASK)
arch_mutex_cpu_relax();
/*
@@ -343,15 +371,19 @@ queue:
*
* n,0,0 -> 0,0,1 : lock, uncontended
* *,0,0 -> *,0,1 : lock, contended
+ *
+ * If the queue head is the only one in the queue (lock value == tail),
+ * clear the tail code and grab the lock. Otherwise, we only need
+ * to grab the lock.
*/
for (;;) {
- new = _Q_LOCKED_VAL;
- if (val != tail)
- new |= val;
-
- old = atomic_cmpxchg(&lock->val, val, new);
- if (old == val)
+ if (val != tail) {
+ set_locked(lock);
break;
+ }
+ old = atomic_cmpxchg(&lock->val, val, _Q_LOCKED_VAL);
+ if (old == val)
+ goto release; /* No contention */
val = old;
}
@@ -359,12 +391,10 @@ queue:
/*
* contended path; wait for next, release.
*/
- if (new != _Q_LOCKED_VAL) {
- while (!(next = ACCESS_ONCE(node->next)))
- arch_mutex_cpu_relax();
+ while (!(next = ACCESS_ONCE(node->next)))
+ arch_mutex_cpu_relax();
- arch_mcs_spin_unlock_contended(&next->locked);
- }
+ arch_mcs_spin_unlock_contended(&next->locked);
release:
/*
--
1.7.1
^ permalink raw reply related [flat|nested] 102+ messages in thread
* [PATCH v11 07/16] qspinlock: Use a simple write to grab the lock, if applicable
2014-05-30 15:43 [PATCH v11 00/16] qspinlock: a 4-byte queue spinlock with PV support Waiman Long
` (14 preceding siblings ...)
2014-05-30 15:43 ` Waiman Long
@ 2014-05-30 15:43 ` Waiman Long
2014-05-30 15:43 ` Waiman Long
` (21 subsequent siblings)
37 siblings, 0 replies; 102+ messages in thread
From: Waiman Long @ 2014-05-30 15:43 UTC (permalink / raw)
To: Thomas Gleixner, Ingo Molnar, H. Peter Anvin, Peter Zijlstra
Cc: linux-arch, Waiman Long, Raghavendra K T, Gleb Natapov, kvm,
Scott J Norton, x86, Paolo Bonzini, linux-kernel, virtualization,
Chegu Vinod, David Vrabel, Oleg Nesterov, xen-devel,
Boris Ostrovsky, Paul E. McKenney, Linus Torvalds
Currently, atomic_cmpxchg() is used to get the lock. However, this is
not really necessary if there is more than one task in the queue and
the queue head don't need to reset the queue code word. For that case,
a simple write to set the lock bit is enough as the queue head will
be the only one eligible to get the lock as long as it checks that
both the lock and pending bits are not set. The current pending bit
waiting code will ensure that the bit will not be set as soon as the
queue code word (tail) in the lock is set.
With that change, the are some slight improvement in the performance
of the queue spinlock in the 5M loop micro-benchmark run on a 4-socket
Westere-EX machine as shown in the tables below.
[Standalone/Embedded - same node]
# of tasks Before patch After patch %Change
---------- ----------- ---------- -------
3 2324/2321 2248/2265 -3%/-2%
4 2890/2896 2819/2831 -2%/-2%
5 3611/3595 3522/3512 -2%/-2%
6 4281/4276 4173/4160 -3%/-3%
7 5018/5001 4875/4861 -3%/-3%
8 5759/5750 5563/5568 -3%/-3%
[Standalone/Embedded - different nodes]
# of tasks Before patch After patch %Change
---------- ----------- ---------- -------
3 12242/12237 12087/12093 -1%/-1%
4 10688/10696 10507/10521 -2%/-2%
It was also found that this change produced a much bigger performance
improvement in the newer IvyBridge-EX chip and was essentially to close
the performance gap between the ticket spinlock and queue spinlock.
The disk workload of the AIM7 benchmark was run on a 4-socket
Westmere-EX machine with both ext4 and xfs RAM disks at 3000 users
on a 3.14 based kernel. The results of the test runs were:
AIM7 XFS Disk Test
kernel JPM Real Time Sys Time Usr Time
----- --- --------- -------- --------
ticketlock 5678233 3.17 96.61 5.81
qspinlock 5750799 3.13 94.83 5.97
AIM7 EXT4 Disk Test
kernel JPM Real Time Sys Time Usr Time
----- --- --------- -------- --------
ticketlock 1114551 16.15 509.72 7.11
qspinlock 2184466 8.24 232.99 6.01
The ext4 filesystem run had a much higher spinlock contention than
the xfs filesystem run.
The "ebizzy -m" test was also run with the following results:
kernel records/s Real Time Sys Time Usr Time
----- --------- --------- -------- --------
ticketlock 2075 10.00 216.35 3.49
qspinlock 3023 10.00 198.20 4.80
Signed-off-by: Waiman Long <Waiman.Long@hp.com>
---
kernel/locking/qspinlock.c | 62 ++++++++++++++++++++++++++++++++-----------
1 files changed, 46 insertions(+), 16 deletions(-)
diff --git a/kernel/locking/qspinlock.c b/kernel/locking/qspinlock.c
index 7f10758..2c7abe7 100644
--- a/kernel/locking/qspinlock.c
+++ b/kernel/locking/qspinlock.c
@@ -93,24 +93,33 @@ static inline struct mcs_spinlock *decode_tail(u32 tail)
* By using the whole 2nd least significant byte for the pending bit, we
* can allow better optimization of the lock acquisition for the pending
* bit holder.
+ *
+ * This internal structure is also used by the set_locked function which
+ * is not restricted to _Q_PENDING_BITS == 8.
*/
-#if _Q_PENDING_BITS == 8
-
struct __qspinlock {
union {
atomic_t val;
- struct {
#ifdef __LITTLE_ENDIAN
+ u8 locked;
+ struct {
u16 locked_pending;
u16 tail;
+ };
#else
+ struct {
u16 tail;
u16 locked_pending;
-#endif
};
+ struct {
+ u8 reserved[3];
+ u8 locked;
+ };
+#endif
};
};
+#if _Q_PENDING_BITS == 8
/**
* clear_pending_set_locked - take ownership and clear the pending bit.
* @lock: Pointer to queue spinlock structure
@@ -197,6 +206,22 @@ static __always_inline u32 xchg_tail(struct qspinlock *lock, u32 tail)
#endif /* _Q_PENDING_BITS == 8 */
/**
+ * set_locked - Set the lock bit and own the lock
+ * @lock: Pointer to queue spinlock structure
+ *
+ * This routine should only be called when the caller is the only one
+ * entitled to acquire the lock.
+ */
+static __always_inline void set_locked(struct qspinlock *lock)
+{
+ struct __qspinlock *l = (void *)lock;
+
+ barrier();
+ ACCESS_ONCE(l->locked) = _Q_LOCKED_VAL;
+ barrier();
+}
+
+/**
* queue_spin_lock_slowpath - acquire the queue spinlock
* @lock: Pointer to queue spinlock structure
* @val: Current value of the queue spinlock 32-bit word
@@ -332,10 +357,13 @@ queue:
/*
* we're at the head of the waitqueue, wait for the owner & pending to
* go away.
+ * Load-acquired is used here because the set_locked()
+ * function below may not be a full memory barrier.
*
* *,x,y -> *,0,0
*/
- while ((val = atomic_read(&lock->val)) & _Q_LOCKED_PENDING_MASK)
+ while ((val = smp_load_acquire(&lock->val.counter))
+ & _Q_LOCKED_PENDING_MASK)
arch_mutex_cpu_relax();
/*
@@ -343,15 +371,19 @@ queue:
*
* n,0,0 -> 0,0,1 : lock, uncontended
* *,0,0 -> *,0,1 : lock, contended
+ *
+ * If the queue head is the only one in the queue (lock value == tail),
+ * clear the tail code and grab the lock. Otherwise, we only need
+ * to grab the lock.
*/
for (;;) {
- new = _Q_LOCKED_VAL;
- if (val != tail)
- new |= val;
-
- old = atomic_cmpxchg(&lock->val, val, new);
- if (old == val)
+ if (val != tail) {
+ set_locked(lock);
break;
+ }
+ old = atomic_cmpxchg(&lock->val, val, _Q_LOCKED_VAL);
+ if (old == val)
+ goto release; /* No contention */
val = old;
}
@@ -359,12 +391,10 @@ queue:
/*
* contended path; wait for next, release.
*/
- if (new != _Q_LOCKED_VAL) {
- while (!(next = ACCESS_ONCE(node->next)))
- arch_mutex_cpu_relax();
+ while (!(next = ACCESS_ONCE(node->next)))
+ arch_mutex_cpu_relax();
- arch_mcs_spin_unlock_contended(&next->locked);
- }
+ arch_mcs_spin_unlock_contended(&next->locked);
release:
/*
--
1.7.1
^ permalink raw reply related [flat|nested] 102+ messages in thread* [PATCH v11 08/16] qspinlock: Prepare for unfair lock support
2014-05-30 15:43 [PATCH v11 00/16] qspinlock: a 4-byte queue spinlock with PV support Waiman Long
@ 2014-05-30 15:43 ` Waiman Long
2014-05-30 15:43 ` Waiman Long
` (36 subsequent siblings)
37 siblings, 0 replies; 102+ messages in thread
From: Waiman Long @ 2014-05-30 15:43 UTC (permalink / raw)
To: Thomas Gleixner, Ingo Molnar, H. Peter Anvin, Peter Zijlstra
Cc: linux-arch, x86, linux-kernel, virtualization, xen-devel, kvm,
Paolo Bonzini, Konrad Rzeszutek Wilk, Boris Ostrovsky,
Paul E. McKenney, Rik van Riel, Linus Torvalds, Raghavendra K T,
David Vrabel, Oleg Nesterov, Gleb Natapov, Scott J Norton,
Chegu Vinod, Waiman Long
If unfair lock is supported, the lock acquisition loop at the end of
the queue_spin_lock_slowpath() function may need to detect the fact
the lock can be stolen. Code are added for the stolen lock detection.
Signed-off-by: Waiman Long <Waiman.Long@hp.com>
---
kernel/locking/qspinlock.c | 26 ++++++++++++++++++--------
1 files changed, 18 insertions(+), 8 deletions(-)
diff --git a/kernel/locking/qspinlock.c b/kernel/locking/qspinlock.c
index 2c7abe7..ae1b19d 100644
--- a/kernel/locking/qspinlock.c
+++ b/kernel/locking/qspinlock.c
@@ -94,7 +94,7 @@ static inline struct mcs_spinlock *decode_tail(u32 tail)
* can allow better optimization of the lock acquisition for the pending
* bit holder.
*
- * This internal structure is also used by the set_locked function which
+ * This internal structure is also used by the try_set_locked function which
* is not restricted to _Q_PENDING_BITS == 8.
*/
struct __qspinlock {
@@ -206,19 +206,21 @@ static __always_inline u32 xchg_tail(struct qspinlock *lock, u32 tail)
#endif /* _Q_PENDING_BITS == 8 */
/**
- * set_locked - Set the lock bit and own the lock
- * @lock: Pointer to queue spinlock structure
+ * try_set_locked - Try to set the lock bit and own the lock
+ * @lock : Pointer to queue spinlock structure
+ * Return: 1 if lock acquired, 0 otherwise
*
* This routine should only be called when the caller is the only one
* entitled to acquire the lock.
*/
-static __always_inline void set_locked(struct qspinlock *lock)
+static __always_inline int try_set_locked(struct qspinlock *lock)
{
struct __qspinlock *l = (void *)lock;
barrier();
ACCESS_ONCE(l->locked) = _Q_LOCKED_VAL;
barrier();
+ return 1;
}
/**
@@ -357,11 +359,12 @@ queue:
/*
* we're at the head of the waitqueue, wait for the owner & pending to
* go away.
- * Load-acquired is used here because the set_locked()
+ * Load-acquired is used here because the try_set_locked()
* function below may not be a full memory barrier.
*
* *,x,y -> *,0,0
*/
+retry_queue_wait:
while ((val = smp_load_acquire(&lock->val.counter))
& _Q_LOCKED_PENDING_MASK)
arch_mutex_cpu_relax();
@@ -378,13 +381,20 @@ queue:
*/
for (;;) {
if (val != tail) {
- set_locked(lock);
- break;
+ /*
+ * The try_set_locked function will only failed if the
+ * lock was stolen.
+ */
+ if (try_set_locked(lock))
+ break;
+ else
+ goto retry_queue_wait;
}
old = atomic_cmpxchg(&lock->val, val, _Q_LOCKED_VAL);
if (old == val)
goto release; /* No contention */
-
+ else if (old & _Q_LOCKED_MASK)
+ goto retry_queue_wait;
val = old;
}
--
1.7.1
^ permalink raw reply related [flat|nested] 102+ messages in thread* [PATCH v11 08/16] qspinlock: Prepare for unfair lock support
@ 2014-05-30 15:43 ` Waiman Long
0 siblings, 0 replies; 102+ messages in thread
From: Waiman Long @ 2014-05-30 15:43 UTC (permalink / raw)
To: Thomas Gleixner, Ingo Molnar, H. Peter Anvin, Peter Zijlstra
Cc: linux-arch, Waiman Long, Rik van Riel, Raghavendra K T,
Gleb Natapov, kvm, Konrad Rzeszutek Wilk, Scott J Norton, x86,
Paolo Bonzini, linux-kernel, virtualization, Chegu Vinod,
David Vrabel, Oleg Nesterov, xen-devel, Boris Ostrovsky,
Paul E. McKenney, Linus Torvalds
If unfair lock is supported, the lock acquisition loop at the end of
the queue_spin_lock_slowpath() function may need to detect the fact
the lock can be stolen. Code are added for the stolen lock detection.
Signed-off-by: Waiman Long <Waiman.Long@hp.com>
---
kernel/locking/qspinlock.c | 26 ++++++++++++++++++--------
1 files changed, 18 insertions(+), 8 deletions(-)
diff --git a/kernel/locking/qspinlock.c b/kernel/locking/qspinlock.c
index 2c7abe7..ae1b19d 100644
--- a/kernel/locking/qspinlock.c
+++ b/kernel/locking/qspinlock.c
@@ -94,7 +94,7 @@ static inline struct mcs_spinlock *decode_tail(u32 tail)
* can allow better optimization of the lock acquisition for the pending
* bit holder.
*
- * This internal structure is also used by the set_locked function which
+ * This internal structure is also used by the try_set_locked function which
* is not restricted to _Q_PENDING_BITS == 8.
*/
struct __qspinlock {
@@ -206,19 +206,21 @@ static __always_inline u32 xchg_tail(struct qspinlock *lock, u32 tail)
#endif /* _Q_PENDING_BITS == 8 */
/**
- * set_locked - Set the lock bit and own the lock
- * @lock: Pointer to queue spinlock structure
+ * try_set_locked - Try to set the lock bit and own the lock
+ * @lock : Pointer to queue spinlock structure
+ * Return: 1 if lock acquired, 0 otherwise
*
* This routine should only be called when the caller is the only one
* entitled to acquire the lock.
*/
-static __always_inline void set_locked(struct qspinlock *lock)
+static __always_inline int try_set_locked(struct qspinlock *lock)
{
struct __qspinlock *l = (void *)lock;
barrier();
ACCESS_ONCE(l->locked) = _Q_LOCKED_VAL;
barrier();
+ return 1;
}
/**
@@ -357,11 +359,12 @@ queue:
/*
* we're at the head of the waitqueue, wait for the owner & pending to
* go away.
- * Load-acquired is used here because the set_locked()
+ * Load-acquired is used here because the try_set_locked()
* function below may not be a full memory barrier.
*
* *,x,y -> *,0,0
*/
+retry_queue_wait:
while ((val = smp_load_acquire(&lock->val.counter))
& _Q_LOCKED_PENDING_MASK)
arch_mutex_cpu_relax();
@@ -378,13 +381,20 @@ queue:
*/
for (;;) {
if (val != tail) {
- set_locked(lock);
- break;
+ /*
+ * The try_set_locked function will only failed if the
+ * lock was stolen.
+ */
+ if (try_set_locked(lock))
+ break;
+ else
+ goto retry_queue_wait;
}
old = atomic_cmpxchg(&lock->val, val, _Q_LOCKED_VAL);
if (old == val)
goto release; /* No contention */
-
+ else if (old & _Q_LOCKED_MASK)
+ goto retry_queue_wait;
val = old;
}
--
1.7.1
^ permalink raw reply related [flat|nested] 102+ messages in thread
* [PATCH v11 08/16] qspinlock: Prepare for unfair lock support
2014-05-30 15:43 [PATCH v11 00/16] qspinlock: a 4-byte queue spinlock with PV support Waiman Long
` (16 preceding siblings ...)
2014-05-30 15:43 ` Waiman Long
@ 2014-05-30 15:43 ` Waiman Long
2014-05-30 15:43 ` Waiman Long
` (19 subsequent siblings)
37 siblings, 0 replies; 102+ messages in thread
From: Waiman Long @ 2014-05-30 15:43 UTC (permalink / raw)
To: Thomas Gleixner, Ingo Molnar, H. Peter Anvin, Peter Zijlstra
Cc: linux-arch, Waiman Long, Raghavendra K T, Gleb Natapov, kvm,
Scott J Norton, x86, Paolo Bonzini, linux-kernel, virtualization,
Chegu Vinod, David Vrabel, Oleg Nesterov, xen-devel,
Boris Ostrovsky, Paul E. McKenney, Linus Torvalds
If unfair lock is supported, the lock acquisition loop at the end of
the queue_spin_lock_slowpath() function may need to detect the fact
the lock can be stolen. Code are added for the stolen lock detection.
Signed-off-by: Waiman Long <Waiman.Long@hp.com>
---
kernel/locking/qspinlock.c | 26 ++++++++++++++++++--------
1 files changed, 18 insertions(+), 8 deletions(-)
diff --git a/kernel/locking/qspinlock.c b/kernel/locking/qspinlock.c
index 2c7abe7..ae1b19d 100644
--- a/kernel/locking/qspinlock.c
+++ b/kernel/locking/qspinlock.c
@@ -94,7 +94,7 @@ static inline struct mcs_spinlock *decode_tail(u32 tail)
* can allow better optimization of the lock acquisition for the pending
* bit holder.
*
- * This internal structure is also used by the set_locked function which
+ * This internal structure is also used by the try_set_locked function which
* is not restricted to _Q_PENDING_BITS == 8.
*/
struct __qspinlock {
@@ -206,19 +206,21 @@ static __always_inline u32 xchg_tail(struct qspinlock *lock, u32 tail)
#endif /* _Q_PENDING_BITS == 8 */
/**
- * set_locked - Set the lock bit and own the lock
- * @lock: Pointer to queue spinlock structure
+ * try_set_locked - Try to set the lock bit and own the lock
+ * @lock : Pointer to queue spinlock structure
+ * Return: 1 if lock acquired, 0 otherwise
*
* This routine should only be called when the caller is the only one
* entitled to acquire the lock.
*/
-static __always_inline void set_locked(struct qspinlock *lock)
+static __always_inline int try_set_locked(struct qspinlock *lock)
{
struct __qspinlock *l = (void *)lock;
barrier();
ACCESS_ONCE(l->locked) = _Q_LOCKED_VAL;
barrier();
+ return 1;
}
/**
@@ -357,11 +359,12 @@ queue:
/*
* we're at the head of the waitqueue, wait for the owner & pending to
* go away.
- * Load-acquired is used here because the set_locked()
+ * Load-acquired is used here because the try_set_locked()
* function below may not be a full memory barrier.
*
* *,x,y -> *,0,0
*/
+retry_queue_wait:
while ((val = smp_load_acquire(&lock->val.counter))
& _Q_LOCKED_PENDING_MASK)
arch_mutex_cpu_relax();
@@ -378,13 +381,20 @@ queue:
*/
for (;;) {
if (val != tail) {
- set_locked(lock);
- break;
+ /*
+ * The try_set_locked function will only failed if the
+ * lock was stolen.
+ */
+ if (try_set_locked(lock))
+ break;
+ else
+ goto retry_queue_wait;
}
old = atomic_cmpxchg(&lock->val, val, _Q_LOCKED_VAL);
if (old == val)
goto release; /* No contention */
-
+ else if (old & _Q_LOCKED_MASK)
+ goto retry_queue_wait;
val = old;
}
--
1.7.1
^ permalink raw reply related [flat|nested] 102+ messages in thread* [PATCH v11 09/16] qspinlock, x86: Allow unfair spinlock in a virtual guest
2014-05-30 15:43 [PATCH v11 00/16] qspinlock: a 4-byte queue spinlock with PV support Waiman Long
@ 2014-05-30 15:43 ` Waiman Long
2014-05-30 15:43 ` Waiman Long
` (36 subsequent siblings)
37 siblings, 0 replies; 102+ messages in thread
From: Waiman Long @ 2014-05-30 15:43 UTC (permalink / raw)
To: Thomas Gleixner, Ingo Molnar, H. Peter Anvin, Peter Zijlstra
Cc: linux-arch, x86, linux-kernel, virtualization, xen-devel, kvm,
Paolo Bonzini, Konrad Rzeszutek Wilk, Boris Ostrovsky,
Paul E. McKenney, Rik van Riel, Linus Torvalds, Raghavendra K T,
David Vrabel, Oleg Nesterov, Gleb Natapov, Scott J Norton,
Chegu Vinod, Waiman Long
Locking is always an issue in a virtualized environment because of 2
different types of problems:
1) Lock holder preemption
2) Lock waiter preemption
One solution to the lock waiter preemption problem is to allow unfair
lock in a virtualized environment. In this case, a new lock acquirer
can come and steal the lock if the next-in-line CPU to get the lock
is scheduled out.
A simple unfair queue spinlock can be implemented by allowing lock
stealing in the fast path. The slowpath will also be modified to run a
simple queue_spin_trylock() loop. A simple test and set lock like that
does have the problem that the The constant spinning on the lock word
put a lot of cacheline contention traffic on the affected cacheline,
thus slowing tasks that need to access the cacheline.
Unfair lock in a native environment is generally not a good idea as
there is a possibility of lock starvation for a heavily contended lock.
This patch adds a new configuration option for the x86 architecture
to enable the use of unfair queue spinlock (AVIRT_UNFAIR_LOCKS)
in a virtual guest. A jump label (virt_unfairlocks_enabled) is
used to switch between a fair and an unfair version of the spinlock
code. This jump label will only be enabled in a virtual guest where
the X86_FEATURE_HYPERVISOR feature bit is set.
Enabling this configuration feature causes a slight decrease the
performance of an uncontended lock-unlock operation by about 1-2%
mainly due to the use of a static key. However, uncontended lock-unlock
operation are really just a tiny percentage of a real workload. So
there should no noticeable change in application performance.
With the unfair locking activated on bare metal 4-socket Westmere-EX
box, the execution times (in ms) of a spinlock micro-benchmark were
as follows:
# of Ticket Fair Unfair
tasks lock queue lock queue lock
------ ------- ---------- ----------
1 135 135 137
2 890 1082 613
3 1932 2248 1211
4 2829 2819 1720
5 3834 3522 2461
6 4963 4173 3715
7 6299 4875 3749
8 7691 5563 4194
Executing one task per node, the performance data were:
# of Ticket Fair Unfair
nodes lock queue lock queue lock
------ ------- ---------- ----------
1 135 135 137
2 4603 1034 1458
3 10940 12087 2562
4 21555 10507 4793
Signed-off-by: Waiman Long <Waiman.Long@hp.com>
---
arch/x86/Kconfig | 11 +++++
arch/x86/include/asm/qspinlock.h | 79 ++++++++++++++++++++++++++++++++++
arch/x86/kernel/Makefile | 1 +
arch/x86/kernel/paravirt-spinlocks.c | 26 +++++++++++
kernel/locking/qspinlock.c | 20 +++++++++
5 files changed, 137 insertions(+), 0 deletions(-)
diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig
index 95c9c4e..961f43a 100644
--- a/arch/x86/Kconfig
+++ b/arch/x86/Kconfig
@@ -585,6 +585,17 @@ config PARAVIRT_SPINLOCKS
If you are unsure how to answer this question, answer Y.
+config VIRT_UNFAIR_LOCKS
+ bool "Enable unfair locks in a virtual guest"
+ depends on SMP && QUEUE_SPINLOCK
+ depends on !CONFIG_X86_OOSTORE && !CONFIG_X86_PPRO_FENCE
+ ---help---
+ This changes the kernel to use unfair locks in a virtual
+ guest. This will help performance in most cases. However,
+ there is a possibility of lock starvation on a heavily
+ contended lock especially in a large guest with many
+ virtual CPUs.
+
source "arch/x86/xen/Kconfig"
config KVM_GUEST
diff --git a/arch/x86/include/asm/qspinlock.h b/arch/x86/include/asm/qspinlock.h
index e4a4f5d..448de8b 100644
--- a/arch/x86/include/asm/qspinlock.h
+++ b/arch/x86/include/asm/qspinlock.h
@@ -5,6 +5,10 @@
#if !defined(CONFIG_X86_OOSTORE) && !defined(CONFIG_X86_PPRO_FENCE)
+#ifdef CONFIG_VIRT_UNFAIR_LOCKS
+extern struct static_key virt_unfairlocks_enabled;
+#endif
+
#define queue_spin_unlock queue_spin_unlock
/**
* queue_spin_unlock - release a queue spinlock
@@ -26,4 +30,79 @@ static inline void queue_spin_unlock(struct qspinlock *lock)
#include <asm-generic/qspinlock.h>
+union arch_qspinlock {
+ atomic_t val;
+ u8 locked;
+};
+
+#ifdef CONFIG_VIRT_UNFAIR_LOCKS
+/**
+ * queue_spin_trylock_unfair - try to acquire the queue spinlock unfairly
+ * @lock : Pointer to queue spinlock structure
+ * Return: 1 if lock acquired, 0 if failed
+ */
+static __always_inline int queue_spin_trylock_unfair(struct qspinlock *lock)
+{
+ union arch_qspinlock *qlock = (union arch_qspinlock *)lock;
+
+ if (!qlock->locked && (cmpxchg(&qlock->locked, 0, _Q_LOCKED_VAL) == 0))
+ return 1;
+ return 0;
+}
+
+/**
+ * queue_spin_lock_unfair - acquire a queue spinlock unfairly
+ * @lock: Pointer to queue spinlock structure
+ */
+static __always_inline void queue_spin_lock_unfair(struct qspinlock *lock)
+{
+ union arch_qspinlock *qlock = (union arch_qspinlock *)lock;
+
+ if (likely(cmpxchg(&qlock->locked, 0, _Q_LOCKED_VAL) == 0))
+ return;
+ /*
+ * Since the lock is now unfair, we should not activate the 2-task
+ * pending bit spinning code path which disallows lock stealing.
+ */
+ queue_spin_lock_slowpath(lock, -1);
+}
+
+/*
+ * Redefine arch_spin_lock and arch_spin_trylock as inline functions that will
+ * jump to the unfair versions if the static key virt_unfairlocks_enabled
+ * is true.
+ */
+#undef arch_spin_lock
+#undef arch_spin_trylock
+#undef arch_spin_lock_flags
+
+/**
+ * arch_spin_lock - acquire a queue spinlock
+ * @lock: Pointer to queue spinlock structure
+ */
+static inline void arch_spin_lock(struct qspinlock *lock)
+{
+ if (static_key_false(&virt_unfairlocks_enabled))
+ queue_spin_lock_unfair(lock);
+ else
+ queue_spin_lock(lock);
+}
+
+/**
+ * arch_spin_trylock - try to acquire the queue spinlock
+ * @lock : Pointer to queue spinlock structure
+ * Return: 1 if lock acquired, 0 if failed
+ */
+static inline int arch_spin_trylock(struct qspinlock *lock)
+{
+ if (static_key_false(&virt_unfairlocks_enabled))
+ return queue_spin_trylock_unfair(lock);
+ else
+ return queue_spin_trylock(lock);
+}
+
+#define arch_spin_lock_flags(l, f) arch_spin_lock(l)
+
+#endif /* CONFIG_VIRT_UNFAIR_LOCKS */
+
#endif /* _ASM_X86_QSPINLOCK_H */
diff --git a/arch/x86/kernel/Makefile b/arch/x86/kernel/Makefile
index f4d9600..cf592f3 100644
--- a/arch/x86/kernel/Makefile
+++ b/arch/x86/kernel/Makefile
@@ -88,6 +88,7 @@ obj-$(CONFIG_DEBUG_NMI_SELFTEST) += nmi_selftest.o
obj-$(CONFIG_KVM_GUEST) += kvm.o kvmclock.o
obj-$(CONFIG_PARAVIRT) += paravirt.o paravirt_patch_$(BITS).o
obj-$(CONFIG_PARAVIRT_SPINLOCKS)+= paravirt-spinlocks.o
+obj-$(CONFIG_VIRT_UNFAIR_LOCKS) += paravirt-spinlocks.o
obj-$(CONFIG_PARAVIRT_CLOCK) += pvclock.o
obj-$(CONFIG_PCSPKR_PLATFORM) += pcspeaker.o
diff --git a/arch/x86/kernel/paravirt-spinlocks.c b/arch/x86/kernel/paravirt-spinlocks.c
index bbb6c73..69ed806 100644
--- a/arch/x86/kernel/paravirt-spinlocks.c
+++ b/arch/x86/kernel/paravirt-spinlocks.c
@@ -8,6 +8,7 @@
#include <asm/paravirt.h>
+#ifdef CONFIG_PARAVIRT_SPINLOCKS
struct pv_lock_ops pv_lock_ops = {
#ifdef CONFIG_SMP
.lock_spinning = __PV_IS_CALLEE_SAVE(paravirt_nop),
@@ -18,3 +19,28 @@ EXPORT_SYMBOL(pv_lock_ops);
struct static_key paravirt_ticketlocks_enabled = STATIC_KEY_INIT_FALSE;
EXPORT_SYMBOL(paravirt_ticketlocks_enabled);
+#endif
+
+#ifdef CONFIG_VIRT_UNFAIR_LOCKS
+struct static_key virt_unfairlocks_enabled = STATIC_KEY_INIT_FALSE;
+EXPORT_SYMBOL(virt_unfairlocks_enabled);
+
+#include <linux/init.h>
+#include <asm/cpufeature.h>
+
+/*
+ * Enable unfair lock only if it is running under a hypervisor
+ */
+static __init int unfair_locks_init_jump(void)
+{
+ if (!boot_cpu_has(X86_FEATURE_HYPERVISOR))
+ return 0;
+
+ static_key_slow_inc(&virt_unfairlocks_enabled);
+ printk(KERN_INFO "Unfair spinlock enabled\n");
+
+ return 0;
+}
+early_initcall(unfair_locks_init_jump);
+
+#endif
diff --git a/kernel/locking/qspinlock.c b/kernel/locking/qspinlock.c
index ae1b19d..3723c83 100644
--- a/kernel/locking/qspinlock.c
+++ b/kernel/locking/qspinlock.c
@@ -217,6 +217,14 @@ static __always_inline int try_set_locked(struct qspinlock *lock)
{
struct __qspinlock *l = (void *)lock;
+#ifdef CONFIG_VIRT_UNFAIR_LOCKS
+ /*
+ * Need to use atomic operation to grab the lock when lock stealing
+ * can happen.
+ */
+ if (static_key_false(&virt_unfairlocks_enabled))
+ return cmpxchg(&l->locked, 0, _Q_LOCKED_VAL) == 0;
+#endif
barrier();
ACCESS_ONCE(l->locked) = _Q_LOCKED_VAL;
barrier();
@@ -252,6 +260,18 @@ void queue_spin_lock_slowpath(struct qspinlock *lock, u32 val)
BUILD_BUG_ON(CONFIG_NR_CPUS >= (1U << _Q_TAIL_CPU_BITS));
+#ifdef CONFIG_VIRT_UNFAIR_LOCKS
+ /*
+ * A simple test and set unfair lock
+ */
+ if (static_key_false(&virt_unfairlocks_enabled)) {
+ cpu_relax(); /* Relax after a failed lock attempt */
+ while (!queue_spin_trylock(lock))
+ cpu_relax();
+ return;
+ }
+#endif /* CONFIG_VIRT_UNFAIR_LOCKS */
+
/*
* trylock || pending
*
--
1.7.1
^ permalink raw reply related [flat|nested] 102+ messages in thread* [PATCH v11 09/16] qspinlock, x86: Allow unfair spinlock in a virtual guest
@ 2014-05-30 15:43 ` Waiman Long
0 siblings, 0 replies; 102+ messages in thread
From: Waiman Long @ 2014-05-30 15:43 UTC (permalink / raw)
To: Thomas Gleixner, Ingo Molnar, H. Peter Anvin, Peter Zijlstra
Cc: linux-arch, Waiman Long, Rik van Riel, Raghavendra K T,
Gleb Natapov, kvm, Konrad Rzeszutek Wilk, Scott J Norton, x86,
Paolo Bonzini, linux-kernel, virtualization, Chegu Vinod,
David Vrabel, Oleg Nesterov, xen-devel, Boris Ostrovsky,
Paul E. McKenney, Linus Torvalds
Locking is always an issue in a virtualized environment because of 2
different types of problems:
1) Lock holder preemption
2) Lock waiter preemption
One solution to the lock waiter preemption problem is to allow unfair
lock in a virtualized environment. In this case, a new lock acquirer
can come and steal the lock if the next-in-line CPU to get the lock
is scheduled out.
A simple unfair queue spinlock can be implemented by allowing lock
stealing in the fast path. The slowpath will also be modified to run a
simple queue_spin_trylock() loop. A simple test and set lock like that
does have the problem that the The constant spinning on the lock word
put a lot of cacheline contention traffic on the affected cacheline,
thus slowing tasks that need to access the cacheline.
Unfair lock in a native environment is generally not a good idea as
there is a possibility of lock starvation for a heavily contended lock.
This patch adds a new configuration option for the x86 architecture
to enable the use of unfair queue spinlock (AVIRT_UNFAIR_LOCKS)
in a virtual guest. A jump label (virt_unfairlocks_enabled) is
used to switch between a fair and an unfair version of the spinlock
code. This jump label will only be enabled in a virtual guest where
the X86_FEATURE_HYPERVISOR feature bit is set.
Enabling this configuration feature causes a slight decrease the
performance of an uncontended lock-unlock operation by about 1-2%
mainly due to the use of a static key. However, uncontended lock-unlock
operation are really just a tiny percentage of a real workload. So
there should no noticeable change in application performance.
With the unfair locking activated on bare metal 4-socket Westmere-EX
box, the execution times (in ms) of a spinlock micro-benchmark were
as follows:
# of Ticket Fair Unfair
tasks lock queue lock queue lock
------ ------- ---------- ----------
1 135 135 137
2 890 1082 613
3 1932 2248 1211
4 2829 2819 1720
5 3834 3522 2461
6 4963 4173 3715
7 6299 4875 3749
8 7691 5563 4194
Executing one task per node, the performance data were:
# of Ticket Fair Unfair
nodes lock queue lock queue lock
------ ------- ---------- ----------
1 135 135 137
2 4603 1034 1458
3 10940 12087 2562
4 21555 10507 4793
Signed-off-by: Waiman Long <Waiman.Long@hp.com>
---
arch/x86/Kconfig | 11 +++++
arch/x86/include/asm/qspinlock.h | 79 ++++++++++++++++++++++++++++++++++
arch/x86/kernel/Makefile | 1 +
arch/x86/kernel/paravirt-spinlocks.c | 26 +++++++++++
kernel/locking/qspinlock.c | 20 +++++++++
5 files changed, 137 insertions(+), 0 deletions(-)
diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig
index 95c9c4e..961f43a 100644
--- a/arch/x86/Kconfig
+++ b/arch/x86/Kconfig
@@ -585,6 +585,17 @@ config PARAVIRT_SPINLOCKS
If you are unsure how to answer this question, answer Y.
+config VIRT_UNFAIR_LOCKS
+ bool "Enable unfair locks in a virtual guest"
+ depends on SMP && QUEUE_SPINLOCK
+ depends on !CONFIG_X86_OOSTORE && !CONFIG_X86_PPRO_FENCE
+ ---help---
+ This changes the kernel to use unfair locks in a virtual
+ guest. This will help performance in most cases. However,
+ there is a possibility of lock starvation on a heavily
+ contended lock especially in a large guest with many
+ virtual CPUs.
+
source "arch/x86/xen/Kconfig"
config KVM_GUEST
diff --git a/arch/x86/include/asm/qspinlock.h b/arch/x86/include/asm/qspinlock.h
index e4a4f5d..448de8b 100644
--- a/arch/x86/include/asm/qspinlock.h
+++ b/arch/x86/include/asm/qspinlock.h
@@ -5,6 +5,10 @@
#if !defined(CONFIG_X86_OOSTORE) && !defined(CONFIG_X86_PPRO_FENCE)
+#ifdef CONFIG_VIRT_UNFAIR_LOCKS
+extern struct static_key virt_unfairlocks_enabled;
+#endif
+
#define queue_spin_unlock queue_spin_unlock
/**
* queue_spin_unlock - release a queue spinlock
@@ -26,4 +30,79 @@ static inline void queue_spin_unlock(struct qspinlock *lock)
#include <asm-generic/qspinlock.h>
+union arch_qspinlock {
+ atomic_t val;
+ u8 locked;
+};
+
+#ifdef CONFIG_VIRT_UNFAIR_LOCKS
+/**
+ * queue_spin_trylock_unfair - try to acquire the queue spinlock unfairly
+ * @lock : Pointer to queue spinlock structure
+ * Return: 1 if lock acquired, 0 if failed
+ */
+static __always_inline int queue_spin_trylock_unfair(struct qspinlock *lock)
+{
+ union arch_qspinlock *qlock = (union arch_qspinlock *)lock;
+
+ if (!qlock->locked && (cmpxchg(&qlock->locked, 0, _Q_LOCKED_VAL) == 0))
+ return 1;
+ return 0;
+}
+
+/**
+ * queue_spin_lock_unfair - acquire a queue spinlock unfairly
+ * @lock: Pointer to queue spinlock structure
+ */
+static __always_inline void queue_spin_lock_unfair(struct qspinlock *lock)
+{
+ union arch_qspinlock *qlock = (union arch_qspinlock *)lock;
+
+ if (likely(cmpxchg(&qlock->locked, 0, _Q_LOCKED_VAL) == 0))
+ return;
+ /*
+ * Since the lock is now unfair, we should not activate the 2-task
+ * pending bit spinning code path which disallows lock stealing.
+ */
+ queue_spin_lock_slowpath(lock, -1);
+}
+
+/*
+ * Redefine arch_spin_lock and arch_spin_trylock as inline functions that will
+ * jump to the unfair versions if the static key virt_unfairlocks_enabled
+ * is true.
+ */
+#undef arch_spin_lock
+#undef arch_spin_trylock
+#undef arch_spin_lock_flags
+
+/**
+ * arch_spin_lock - acquire a queue spinlock
+ * @lock: Pointer to queue spinlock structure
+ */
+static inline void arch_spin_lock(struct qspinlock *lock)
+{
+ if (static_key_false(&virt_unfairlocks_enabled))
+ queue_spin_lock_unfair(lock);
+ else
+ queue_spin_lock(lock);
+}
+
+/**
+ * arch_spin_trylock - try to acquire the queue spinlock
+ * @lock : Pointer to queue spinlock structure
+ * Return: 1 if lock acquired, 0 if failed
+ */
+static inline int arch_spin_trylock(struct qspinlock *lock)
+{
+ if (static_key_false(&virt_unfairlocks_enabled))
+ return queue_spin_trylock_unfair(lock);
+ else
+ return queue_spin_trylock(lock);
+}
+
+#define arch_spin_lock_flags(l, f) arch_spin_lock(l)
+
+#endif /* CONFIG_VIRT_UNFAIR_LOCKS */
+
#endif /* _ASM_X86_QSPINLOCK_H */
diff --git a/arch/x86/kernel/Makefile b/arch/x86/kernel/Makefile
index f4d9600..cf592f3 100644
--- a/arch/x86/kernel/Makefile
+++ b/arch/x86/kernel/Makefile
@@ -88,6 +88,7 @@ obj-$(CONFIG_DEBUG_NMI_SELFTEST) += nmi_selftest.o
obj-$(CONFIG_KVM_GUEST) += kvm.o kvmclock.o
obj-$(CONFIG_PARAVIRT) += paravirt.o paravirt_patch_$(BITS).o
obj-$(CONFIG_PARAVIRT_SPINLOCKS)+= paravirt-spinlocks.o
+obj-$(CONFIG_VIRT_UNFAIR_LOCKS) += paravirt-spinlocks.o
obj-$(CONFIG_PARAVIRT_CLOCK) += pvclock.o
obj-$(CONFIG_PCSPKR_PLATFORM) += pcspeaker.o
diff --git a/arch/x86/kernel/paravirt-spinlocks.c b/arch/x86/kernel/paravirt-spinlocks.c
index bbb6c73..69ed806 100644
--- a/arch/x86/kernel/paravirt-spinlocks.c
+++ b/arch/x86/kernel/paravirt-spinlocks.c
@@ -8,6 +8,7 @@
#include <asm/paravirt.h>
+#ifdef CONFIG_PARAVIRT_SPINLOCKS
struct pv_lock_ops pv_lock_ops = {
#ifdef CONFIG_SMP
.lock_spinning = __PV_IS_CALLEE_SAVE(paravirt_nop),
@@ -18,3 +19,28 @@ EXPORT_SYMBOL(pv_lock_ops);
struct static_key paravirt_ticketlocks_enabled = STATIC_KEY_INIT_FALSE;
EXPORT_SYMBOL(paravirt_ticketlocks_enabled);
+#endif
+
+#ifdef CONFIG_VIRT_UNFAIR_LOCKS
+struct static_key virt_unfairlocks_enabled = STATIC_KEY_INIT_FALSE;
+EXPORT_SYMBOL(virt_unfairlocks_enabled);
+
+#include <linux/init.h>
+#include <asm/cpufeature.h>
+
+/*
+ * Enable unfair lock only if it is running under a hypervisor
+ */
+static __init int unfair_locks_init_jump(void)
+{
+ if (!boot_cpu_has(X86_FEATURE_HYPERVISOR))
+ return 0;
+
+ static_key_slow_inc(&virt_unfairlocks_enabled);
+ printk(KERN_INFO "Unfair spinlock enabled\n");
+
+ return 0;
+}
+early_initcall(unfair_locks_init_jump);
+
+#endif
diff --git a/kernel/locking/qspinlock.c b/kernel/locking/qspinlock.c
index ae1b19d..3723c83 100644
--- a/kernel/locking/qspinlock.c
+++ b/kernel/locking/qspinlock.c
@@ -217,6 +217,14 @@ static __always_inline int try_set_locked(struct qspinlock *lock)
{
struct __qspinlock *l = (void *)lock;
+#ifdef CONFIG_VIRT_UNFAIR_LOCKS
+ /*
+ * Need to use atomic operation to grab the lock when lock stealing
+ * can happen.
+ */
+ if (static_key_false(&virt_unfairlocks_enabled))
+ return cmpxchg(&l->locked, 0, _Q_LOCKED_VAL) == 0;
+#endif
barrier();
ACCESS_ONCE(l->locked) = _Q_LOCKED_VAL;
barrier();
@@ -252,6 +260,18 @@ void queue_spin_lock_slowpath(struct qspinlock *lock, u32 val)
BUILD_BUG_ON(CONFIG_NR_CPUS >= (1U << _Q_TAIL_CPU_BITS));
+#ifdef CONFIG_VIRT_UNFAIR_LOCKS
+ /*
+ * A simple test and set unfair lock
+ */
+ if (static_key_false(&virt_unfairlocks_enabled)) {
+ cpu_relax(); /* Relax after a failed lock attempt */
+ while (!queue_spin_trylock(lock))
+ cpu_relax();
+ return;
+ }
+#endif /* CONFIG_VIRT_UNFAIR_LOCKS */
+
/*
* trylock || pending
*
--
1.7.1
^ permalink raw reply related [flat|nested] 102+ messages in thread* Re: [PATCH v11 09/16] qspinlock, x86: Allow unfair spinlock in a virtual guest
2014-05-30 15:43 ` Waiman Long
@ 2014-06-11 10:54 ` Peter Zijlstra
-1 siblings, 0 replies; 102+ messages in thread
From: Peter Zijlstra @ 2014-06-11 10:54 UTC (permalink / raw)
To: Waiman Long
Cc: Thomas Gleixner, Ingo Molnar, H. Peter Anvin, linux-arch, x86,
linux-kernel, virtualization, xen-devel, kvm, Paolo Bonzini,
Konrad Rzeszutek Wilk, Boris Ostrovsky, Paul E. McKenney,
Rik van Riel, Linus Torvalds, Raghavendra K T, David Vrabel,
Oleg Nesterov, Gleb Natapov, Scott J Norton, Chegu Vinod
[-- Attachment #1: Type: text/plain, Size: 4146 bytes --]
On Fri, May 30, 2014 at 11:43:55AM -0400, Waiman Long wrote:
> Enabling this configuration feature causes a slight decrease the
> performance of an uncontended lock-unlock operation by about 1-2%
> mainly due to the use of a static key. However, uncontended lock-unlock
> operation are really just a tiny percentage of a real workload. So
> there should no noticeable change in application performance.
No, entirely unacceptable.
> +#ifdef CONFIG_VIRT_UNFAIR_LOCKS
> +/**
> + * queue_spin_trylock_unfair - try to acquire the queue spinlock unfairly
> + * @lock : Pointer to queue spinlock structure
> + * Return: 1 if lock acquired, 0 if failed
> + */
> +static __always_inline int queue_spin_trylock_unfair(struct qspinlock *lock)
> +{
> + union arch_qspinlock *qlock = (union arch_qspinlock *)lock;
> +
> + if (!qlock->locked && (cmpxchg(&qlock->locked, 0, _Q_LOCKED_VAL) == 0))
> + return 1;
> + return 0;
> +}
> +
> +/**
> + * queue_spin_lock_unfair - acquire a queue spinlock unfairly
> + * @lock: Pointer to queue spinlock structure
> + */
> +static __always_inline void queue_spin_lock_unfair(struct qspinlock *lock)
> +{
> + union arch_qspinlock *qlock = (union arch_qspinlock *)lock;
> +
> + if (likely(cmpxchg(&qlock->locked, 0, _Q_LOCKED_VAL) == 0))
> + return;
> + /*
> + * Since the lock is now unfair, we should not activate the 2-task
> + * pending bit spinning code path which disallows lock stealing.
> + */
> + queue_spin_lock_slowpath(lock, -1);
> +}
Why is this needed?
> +/*
> + * Redefine arch_spin_lock and arch_spin_trylock as inline functions that will
> + * jump to the unfair versions if the static key virt_unfairlocks_enabled
> + * is true.
> + */
> +#undef arch_spin_lock
> +#undef arch_spin_trylock
> +#undef arch_spin_lock_flags
> +
> +/**
> + * arch_spin_lock - acquire a queue spinlock
> + * @lock: Pointer to queue spinlock structure
> + */
> +static inline void arch_spin_lock(struct qspinlock *lock)
> +{
> + if (static_key_false(&virt_unfairlocks_enabled))
> + queue_spin_lock_unfair(lock);
> + else
> + queue_spin_lock(lock);
> +}
> +
> +/**
> + * arch_spin_trylock - try to acquire the queue spinlock
> + * @lock : Pointer to queue spinlock structure
> + * Return: 1 if lock acquired, 0 if failed
> + */
> +static inline int arch_spin_trylock(struct qspinlock *lock)
> +{
> + if (static_key_false(&virt_unfairlocks_enabled))
> + return queue_spin_trylock_unfair(lock);
> + else
> + return queue_spin_trylock(lock);
> +}
So I really don't see the point of all this? Why do you need special
{try,}lock paths for this case? Are you worried about the upper 24bits?
> diff --git a/kernel/locking/qspinlock.c b/kernel/locking/qspinlock.c
> index ae1b19d..3723c83 100644
> --- a/kernel/locking/qspinlock.c
> +++ b/kernel/locking/qspinlock.c
> @@ -217,6 +217,14 @@ static __always_inline int try_set_locked(struct qspinlock *lock)
> {
> struct __qspinlock *l = (void *)lock;
>
> +#ifdef CONFIG_VIRT_UNFAIR_LOCKS
> + /*
> + * Need to use atomic operation to grab the lock when lock stealing
> + * can happen.
> + */
> + if (static_key_false(&virt_unfairlocks_enabled))
> + return cmpxchg(&l->locked, 0, _Q_LOCKED_VAL) == 0;
> +#endif
> barrier();
> ACCESS_ONCE(l->locked) = _Q_LOCKED_VAL;
> barrier();
Why? If we have a simple test-and-set lock like below, we'll never get
here at all.
> @@ -252,6 +260,18 @@ void queue_spin_lock_slowpath(struct qspinlock *lock, u32 val)
>
> BUILD_BUG_ON(CONFIG_NR_CPUS >= (1U << _Q_TAIL_CPU_BITS));
>
> +#ifdef CONFIG_VIRT_UNFAIR_LOCKS
> + /*
> + * A simple test and set unfair lock
> + */
> + if (static_key_false(&virt_unfairlocks_enabled)) {
> + cpu_relax(); /* Relax after a failed lock attempt */
Meh, I don't think anybody can tell the difference if you put that in or
not, therefore don't.
> + while (!queue_spin_trylock(lock))
> + cpu_relax();
> + return;
> + }
> +#endif /* CONFIG_VIRT_UNFAIR_LOCKS */
If you're really worried about those upper 24bits, you can always clear
them when you get here.
[-- Attachment #2: Type: application/pgp-signature, Size: 836 bytes --]
^ permalink raw reply [flat|nested] 102+ messages in thread* Re: [PATCH v11 09/16] qspinlock, x86: Allow unfair spinlock in a virtual guest
@ 2014-06-11 10:54 ` Peter Zijlstra
0 siblings, 0 replies; 102+ messages in thread
From: Peter Zijlstra @ 2014-06-11 10:54 UTC (permalink / raw)
To: Waiman Long
Cc: linux-arch, Rik van Riel, Raghavendra K T, Oleg Nesterov,
Gleb Natapov, kvm, Konrad Rzeszutek Wilk, Scott J Norton, x86,
Paolo Bonzini, linux-kernel, virtualization, Ingo Molnar,
Chegu Vinod, David Vrabel, H. Peter Anvin, xen-devel,
Thomas Gleixner, Paul E. McKenney, Linus Torvalds,
Boris Ostrovsky
[-- Attachment #1.1: Type: text/plain, Size: 4146 bytes --]
On Fri, May 30, 2014 at 11:43:55AM -0400, Waiman Long wrote:
> Enabling this configuration feature causes a slight decrease the
> performance of an uncontended lock-unlock operation by about 1-2%
> mainly due to the use of a static key. However, uncontended lock-unlock
> operation are really just a tiny percentage of a real workload. So
> there should no noticeable change in application performance.
No, entirely unacceptable.
> +#ifdef CONFIG_VIRT_UNFAIR_LOCKS
> +/**
> + * queue_spin_trylock_unfair - try to acquire the queue spinlock unfairly
> + * @lock : Pointer to queue spinlock structure
> + * Return: 1 if lock acquired, 0 if failed
> + */
> +static __always_inline int queue_spin_trylock_unfair(struct qspinlock *lock)
> +{
> + union arch_qspinlock *qlock = (union arch_qspinlock *)lock;
> +
> + if (!qlock->locked && (cmpxchg(&qlock->locked, 0, _Q_LOCKED_VAL) == 0))
> + return 1;
> + return 0;
> +}
> +
> +/**
> + * queue_spin_lock_unfair - acquire a queue spinlock unfairly
> + * @lock: Pointer to queue spinlock structure
> + */
> +static __always_inline void queue_spin_lock_unfair(struct qspinlock *lock)
> +{
> + union arch_qspinlock *qlock = (union arch_qspinlock *)lock;
> +
> + if (likely(cmpxchg(&qlock->locked, 0, _Q_LOCKED_VAL) == 0))
> + return;
> + /*
> + * Since the lock is now unfair, we should not activate the 2-task
> + * pending bit spinning code path which disallows lock stealing.
> + */
> + queue_spin_lock_slowpath(lock, -1);
> +}
Why is this needed?
> +/*
> + * Redefine arch_spin_lock and arch_spin_trylock as inline functions that will
> + * jump to the unfair versions if the static key virt_unfairlocks_enabled
> + * is true.
> + */
> +#undef arch_spin_lock
> +#undef arch_spin_trylock
> +#undef arch_spin_lock_flags
> +
> +/**
> + * arch_spin_lock - acquire a queue spinlock
> + * @lock: Pointer to queue spinlock structure
> + */
> +static inline void arch_spin_lock(struct qspinlock *lock)
> +{
> + if (static_key_false(&virt_unfairlocks_enabled))
> + queue_spin_lock_unfair(lock);
> + else
> + queue_spin_lock(lock);
> +}
> +
> +/**
> + * arch_spin_trylock - try to acquire the queue spinlock
> + * @lock : Pointer to queue spinlock structure
> + * Return: 1 if lock acquired, 0 if failed
> + */
> +static inline int arch_spin_trylock(struct qspinlock *lock)
> +{
> + if (static_key_false(&virt_unfairlocks_enabled))
> + return queue_spin_trylock_unfair(lock);
> + else
> + return queue_spin_trylock(lock);
> +}
So I really don't see the point of all this? Why do you need special
{try,}lock paths for this case? Are you worried about the upper 24bits?
> diff --git a/kernel/locking/qspinlock.c b/kernel/locking/qspinlock.c
> index ae1b19d..3723c83 100644
> --- a/kernel/locking/qspinlock.c
> +++ b/kernel/locking/qspinlock.c
> @@ -217,6 +217,14 @@ static __always_inline int try_set_locked(struct qspinlock *lock)
> {
> struct __qspinlock *l = (void *)lock;
>
> +#ifdef CONFIG_VIRT_UNFAIR_LOCKS
> + /*
> + * Need to use atomic operation to grab the lock when lock stealing
> + * can happen.
> + */
> + if (static_key_false(&virt_unfairlocks_enabled))
> + return cmpxchg(&l->locked, 0, _Q_LOCKED_VAL) == 0;
> +#endif
> barrier();
> ACCESS_ONCE(l->locked) = _Q_LOCKED_VAL;
> barrier();
Why? If we have a simple test-and-set lock like below, we'll never get
here at all.
> @@ -252,6 +260,18 @@ void queue_spin_lock_slowpath(struct qspinlock *lock, u32 val)
>
> BUILD_BUG_ON(CONFIG_NR_CPUS >= (1U << _Q_TAIL_CPU_BITS));
>
> +#ifdef CONFIG_VIRT_UNFAIR_LOCKS
> + /*
> + * A simple test and set unfair lock
> + */
> + if (static_key_false(&virt_unfairlocks_enabled)) {
> + cpu_relax(); /* Relax after a failed lock attempt */
Meh, I don't think anybody can tell the difference if you put that in or
not, therefore don't.
> + while (!queue_spin_trylock(lock))
> + cpu_relax();
> + return;
> + }
> +#endif /* CONFIG_VIRT_UNFAIR_LOCKS */
If you're really worried about those upper 24bits, you can always clear
them when you get here.
[-- Attachment #1.2: Type: application/pgp-signature, Size: 836 bytes --]
[-- Attachment #2: Type: text/plain, Size: 183 bytes --]
_______________________________________________
Virtualization mailing list
Virtualization@lists.linux-foundation.org
https://lists.linuxfoundation.org/mailman/listinfo/virtualization
^ permalink raw reply [flat|nested] 102+ messages in thread* Re: [PATCH v11 09/16] qspinlock, x86: Allow unfair spinlock in a virtual guest
2014-06-11 10:54 ` Peter Zijlstra
(?)
@ 2014-06-11 11:38 ` Peter Zijlstra
-1 siblings, 0 replies; 102+ messages in thread
From: Peter Zijlstra @ 2014-06-11 11:38 UTC (permalink / raw)
To: Waiman Long
Cc: Thomas Gleixner, Ingo Molnar, H. Peter Anvin, linux-arch, x86,
linux-kernel, virtualization, xen-devel, kvm, Paolo Bonzini,
Konrad Rzeszutek Wilk, Boris Ostrovsky, Paul E. McKenney,
Rik van Riel, Linus Torvalds, Raghavendra K T, David Vrabel,
Oleg Nesterov, Gleb Natapov, Scott J Norton, Chegu Vinod
[-- Attachment #1: Type: text/plain, Size: 1000 bytes --]
On Wed, Jun 11, 2014 at 12:54:02PM +0200, Peter Zijlstra wrote:
> > @@ -252,6 +260,18 @@ void queue_spin_lock_slowpath(struct qspinlock *lock, u32 val)
> >
> > BUILD_BUG_ON(CONFIG_NR_CPUS >= (1U << _Q_TAIL_CPU_BITS));
> >
> > +#ifdef CONFIG_VIRT_UNFAIR_LOCKS
> > + /*
> > + * A simple test and set unfair lock
> > + */
> > + if (static_key_false(&virt_unfairlocks_enabled)) {
> > + cpu_relax(); /* Relax after a failed lock attempt */
>
> Meh, I don't think anybody can tell the difference if you put that in or
> not, therefore don't.
>
> > + while (!queue_spin_trylock(lock))
> > + cpu_relax();
> > + return;
> > + }
> > +#endif /* CONFIG_VIRT_UNFAIR_LOCKS */
>
> If you're really worried about those upper 24bits, you can always clear
> them when you get here.
I don't think its a problem at all; flipping the static_key requires
stop_machine, which guarantees us that there are no spinlocks held. So I
think you can actually BUG_ON() the upper 24bits.
[-- Attachment #2: Type: application/pgp-signature, Size: 836 bytes --]
^ permalink raw reply [flat|nested] 102+ messages in thread* Re: [PATCH v11 09/16] qspinlock, x86: Allow unfair spinlock in a virtual guest
2014-06-11 10:54 ` Peter Zijlstra
(?)
(?)
@ 2014-06-11 11:38 ` Peter Zijlstra
-1 siblings, 0 replies; 102+ messages in thread
From: Peter Zijlstra @ 2014-06-11 11:38 UTC (permalink / raw)
To: Waiman Long
Cc: linux-arch, Raghavendra K T, Oleg Nesterov, Gleb Natapov, kvm,
Scott J Norton, x86, Paolo Bonzini, linux-kernel, virtualization,
Ingo Molnar, Chegu Vinod, David Vrabel, H. Peter Anvin,
xen-devel, Thomas Gleixner, Paul E. McKenney, Linus Torvalds,
Boris Ostrovsky
[-- Attachment #1.1: Type: text/plain, Size: 1000 bytes --]
On Wed, Jun 11, 2014 at 12:54:02PM +0200, Peter Zijlstra wrote:
> > @@ -252,6 +260,18 @@ void queue_spin_lock_slowpath(struct qspinlock *lock, u32 val)
> >
> > BUILD_BUG_ON(CONFIG_NR_CPUS >= (1U << _Q_TAIL_CPU_BITS));
> >
> > +#ifdef CONFIG_VIRT_UNFAIR_LOCKS
> > + /*
> > + * A simple test and set unfair lock
> > + */
> > + if (static_key_false(&virt_unfairlocks_enabled)) {
> > + cpu_relax(); /* Relax after a failed lock attempt */
>
> Meh, I don't think anybody can tell the difference if you put that in or
> not, therefore don't.
>
> > + while (!queue_spin_trylock(lock))
> > + cpu_relax();
> > + return;
> > + }
> > +#endif /* CONFIG_VIRT_UNFAIR_LOCKS */
>
> If you're really worried about those upper 24bits, you can always clear
> them when you get here.
I don't think its a problem at all; flipping the static_key requires
stop_machine, which guarantees us that there are no spinlocks held. So I
think you can actually BUG_ON() the upper 24bits.
[-- Attachment #1.2: Type: application/pgp-signature, Size: 836 bytes --]
[-- Attachment #2: Type: text/plain, Size: 126 bytes --]
_______________________________________________
Xen-devel mailing list
Xen-devel@lists.xen.org
http://lists.xen.org/xen-devel
^ permalink raw reply [flat|nested] 102+ messages in thread* Re: [PATCH v11 09/16] qspinlock, x86: Allow unfair spinlock in a virtual guest
2014-06-11 10:54 ` Peter Zijlstra
` (2 preceding siblings ...)
(?)
@ 2014-06-11 11:38 ` Peter Zijlstra
-1 siblings, 0 replies; 102+ messages in thread
From: Peter Zijlstra @ 2014-06-11 11:38 UTC (permalink / raw)
To: Waiman Long
Cc: linux-arch, Rik van Riel, Raghavendra K T, Oleg Nesterov,
Gleb Natapov, kvm, Konrad Rzeszutek Wilk, Scott J Norton, x86,
Paolo Bonzini, linux-kernel, virtualization, Ingo Molnar,
Chegu Vinod, David Vrabel, H. Peter Anvin, xen-devel,
Thomas Gleixner, Paul E. McKenney, Linus Torvalds,
Boris Ostrovsky
[-- Attachment #1.1: Type: text/plain, Size: 1000 bytes --]
On Wed, Jun 11, 2014 at 12:54:02PM +0200, Peter Zijlstra wrote:
> > @@ -252,6 +260,18 @@ void queue_spin_lock_slowpath(struct qspinlock *lock, u32 val)
> >
> > BUILD_BUG_ON(CONFIG_NR_CPUS >= (1U << _Q_TAIL_CPU_BITS));
> >
> > +#ifdef CONFIG_VIRT_UNFAIR_LOCKS
> > + /*
> > + * A simple test and set unfair lock
> > + */
> > + if (static_key_false(&virt_unfairlocks_enabled)) {
> > + cpu_relax(); /* Relax after a failed lock attempt */
>
> Meh, I don't think anybody can tell the difference if you put that in or
> not, therefore don't.
>
> > + while (!queue_spin_trylock(lock))
> > + cpu_relax();
> > + return;
> > + }
> > +#endif /* CONFIG_VIRT_UNFAIR_LOCKS */
>
> If you're really worried about those upper 24bits, you can always clear
> them when you get here.
I don't think its a problem at all; flipping the static_key requires
stop_machine, which guarantees us that there are no spinlocks held. So I
think you can actually BUG_ON() the upper 24bits.
[-- Attachment #1.2: Type: application/pgp-signature, Size: 836 bytes --]
[-- Attachment #2: Type: text/plain, Size: 183 bytes --]
_______________________________________________
Virtualization mailing list
Virtualization@lists.linux-foundation.org
https://lists.linuxfoundation.org/mailman/listinfo/virtualization
^ permalink raw reply [flat|nested] 102+ messages in thread* Re: [PATCH v11 09/16] qspinlock, x86: Allow unfair spinlock in a virtual guest
2014-06-11 10:54 ` Peter Zijlstra
` (3 preceding siblings ...)
(?)
@ 2014-06-12 1:37 ` Long, Wai Man
-1 siblings, 0 replies; 102+ messages in thread
From: Long, Wai Man @ 2014-06-12 1:37 UTC (permalink / raw)
To: Peter Zijlstra
Cc: linux-arch, Raghavendra K T, Oleg Nesterov, Gleb Natapov, kvm,
Scott J Norton, x86, Paolo Bonzini, linux-kernel, virtualization,
Ingo Molnar, Chegu Vinod, David Vrabel, H. Peter Anvin,
xen-devel, Thomas Gleixner, Paul E. McKenney, Linus Torvalds,
Boris Ostrovsky
On 6/11/2014 6:54 AM, Peter Zijlstra wrote:
> On Fri, May 30, 2014 at 11:43:55AM -0400, Waiman Long wrote:
>> Enabling this configuration feature causes a slight decrease the
>> performance of an uncontended lock-unlock operation by about 1-2%
>> mainly due to the use of a static key. However, uncontended lock-unlock
>> operation are really just a tiny percentage of a real workload. So
>> there should no noticeable change in application performance.
> No, entirely unacceptable.
>
>> +#ifdef CONFIG_VIRT_UNFAIR_LOCKS
>> +/**
>> + * queue_spin_trylock_unfair - try to acquire the queue spinlock unfairly
>> + * @lock : Pointer to queue spinlock structure
>> + * Return: 1 if lock acquired, 0 if failed
>> + */
>> +static __always_inline int queue_spin_trylock_unfair(struct qspinlock *lock)
>> +{
>> + union arch_qspinlock *qlock = (union arch_qspinlock *)lock;
>> +
>> + if (!qlock->locked && (cmpxchg(&qlock->locked, 0, _Q_LOCKED_VAL) == 0))
>> + return 1;
>> + return 0;
>> +}
>> +
>> +/**
>> + * queue_spin_lock_unfair - acquire a queue spinlock unfairly
>> + * @lock: Pointer to queue spinlock structure
>> + */
>> +static __always_inline void queue_spin_lock_unfair(struct qspinlock *lock)
>> +{
>> + union arch_qspinlock *qlock = (union arch_qspinlock *)lock;
>> +
>> + if (likely(cmpxchg(&qlock->locked, 0, _Q_LOCKED_VAL) == 0))
>> + return;
>> + /*
>> + * Since the lock is now unfair, we should not activate the 2-task
>> + * pending bit spinning code path which disallows lock stealing.
>> + */
>> + queue_spin_lock_slowpath(lock, -1);
>> +}
> Why is this needed?
I added the unfair version of lock and trylock as my original version
isn't a simple test-and-set lock. Now I changed the core part to use the
simple test-and-set lock. However, I still think that an unfair version
in the fast path can be helpful to performance when both the unfair lock
and paravirt spinlock are enabled. In this case, paravirt spinlock code
will disable the unfair lock code in the slowpath, but still allow the
unfair version in the fast path to get the best possible performance in
a virtual guest.
Yes, I could take that out to allow either unfair or paravirt spinlock,
but not both. I do think that a little bit of unfairness will help in
the virtual environment.
>> +/*
>> + * Redefine arch_spin_lock and arch_spin_trylock as inline functions that will
>> + * jump to the unfair versions if the static key virt_unfairlocks_enabled
>> + * is true.
>> + */
>> +#undef arch_spin_lock
>> +#undef arch_spin_trylock
>> +#undef arch_spin_lock_flags
>> +
>> +/**
>> + * arch_spin_lock - acquire a queue spinlock
>> + * @lock: Pointer to queue spinlock structure
>> + */
>> +static inline void arch_spin_lock(struct qspinlock *lock)
>> +{
>> + if (static_key_false(&virt_unfairlocks_enabled))
>> + queue_spin_lock_unfair(lock);
>> + else
>> + queue_spin_lock(lock);
>> +}
>> +
>> +/**
>> + * arch_spin_trylock - try to acquire the queue spinlock
>> + * @lock : Pointer to queue spinlock structure
>> + * Return: 1 if lock acquired, 0 if failed
>> + */
>> +static inline int arch_spin_trylock(struct qspinlock *lock)
>> +{
>> + if (static_key_false(&virt_unfairlocks_enabled))
>> + return queue_spin_trylock_unfair(lock);
>> + else
>> + return queue_spin_trylock(lock);
>> +}
> So I really don't see the point of all this? Why do you need special
> {try,}lock paths for this case? Are you worried about the upper 24bits?
No, as I said above. I was planning for the coexistence of unfair lock
in the fast path and paravirt spinlock in the slowpath.
>> diff --git a/kernel/locking/qspinlock.c b/kernel/locking/qspinlock.c
>> index ae1b19d..3723c83 100644
>> --- a/kernel/locking/qspinlock.c
>> +++ b/kernel/locking/qspinlock.c
>> @@ -217,6 +217,14 @@ static __always_inline int try_set_locked(struct qspinlock *lock)
>> {
>> struct __qspinlock *l = (void *)lock;
>>
>> +#ifdef CONFIG_VIRT_UNFAIR_LOCKS
>> + /*
>> + * Need to use atomic operation to grab the lock when lock stealing
>> + * can happen.
>> + */
>> + if (static_key_false(&virt_unfairlocks_enabled))
>> + return cmpxchg(&l->locked, 0, _Q_LOCKED_VAL) == 0;
>> +#endif
>> barrier();
>> ACCESS_ONCE(l->locked) = _Q_LOCKED_VAL;
>> barrier();
> Why? If we have a simple test-and-set lock like below, we'll never get
> here at all.
Again, it is due the coexistence of unfair lock in fast path and
paravirt spinlock in the slowpath.
>> @@ -252,6 +260,18 @@ void queue_spin_lock_slowpath(struct qspinlock *lock, u32 val)
>>
>> BUILD_BUG_ON(CONFIG_NR_CPUS >= (1U << _Q_TAIL_CPU_BITS));
>>
>> +#ifdef CONFIG_VIRT_UNFAIR_LOCKS
>> + /*
>> + * A simple test and set unfair lock
>> + */
>> + if (static_key_false(&virt_unfairlocks_enabled)) {
>> + cpu_relax(); /* Relax after a failed lock attempt */
> Meh, I don't think anybody can tell the difference if you put that in or
> not, therefore don't.
Yes, I can take out the cpu_relax() here.
-Longman
^ permalink raw reply [flat|nested] 102+ messages in thread* Re: [PATCH v11 09/16] qspinlock, x86: Allow unfair spinlock in a virtual guest
2014-06-11 10:54 ` Peter Zijlstra
@ 2014-06-12 1:37 ` Long, Wai Man
-1 siblings, 0 replies; 102+ messages in thread
From: Long, Wai Man @ 2014-06-12 1:37 UTC (permalink / raw)
To: Peter Zijlstra
Cc: Thomas Gleixner, Ingo Molnar, H. Peter Anvin, linux-arch, x86,
linux-kernel, virtualization, xen-devel, kvm, Paolo Bonzini,
Konrad Rzeszutek Wilk, Boris Ostrovsky, Paul E. McKenney,
Rik van Riel, Linus Torvalds, Raghavendra K T, David Vrabel,
Oleg Nesterov, Gleb Natapov, Scott J Norton, Chegu Vinod
On 6/11/2014 6:54 AM, Peter Zijlstra wrote:
> On Fri, May 30, 2014 at 11:43:55AM -0400, Waiman Long wrote:
>> Enabling this configuration feature causes a slight decrease the
>> performance of an uncontended lock-unlock operation by about 1-2%
>> mainly due to the use of a static key. However, uncontended lock-unlock
>> operation are really just a tiny percentage of a real workload. So
>> there should no noticeable change in application performance.
> No, entirely unacceptable.
>
>> +#ifdef CONFIG_VIRT_UNFAIR_LOCKS
>> +/**
>> + * queue_spin_trylock_unfair - try to acquire the queue spinlock unfairly
>> + * @lock : Pointer to queue spinlock structure
>> + * Return: 1 if lock acquired, 0 if failed
>> + */
>> +static __always_inline int queue_spin_trylock_unfair(struct qspinlock *lock)
>> +{
>> + union arch_qspinlock *qlock = (union arch_qspinlock *)lock;
>> +
>> + if (!qlock->locked && (cmpxchg(&qlock->locked, 0, _Q_LOCKED_VAL) == 0))
>> + return 1;
>> + return 0;
>> +}
>> +
>> +/**
>> + * queue_spin_lock_unfair - acquire a queue spinlock unfairly
>> + * @lock: Pointer to queue spinlock structure
>> + */
>> +static __always_inline void queue_spin_lock_unfair(struct qspinlock *lock)
>> +{
>> + union arch_qspinlock *qlock = (union arch_qspinlock *)lock;
>> +
>> + if (likely(cmpxchg(&qlock->locked, 0, _Q_LOCKED_VAL) == 0))
>> + return;
>> + /*
>> + * Since the lock is now unfair, we should not activate the 2-task
>> + * pending bit spinning code path which disallows lock stealing.
>> + */
>> + queue_spin_lock_slowpath(lock, -1);
>> +}
> Why is this needed?
I added the unfair version of lock and trylock as my original version
isn't a simple test-and-set lock. Now I changed the core part to use the
simple test-and-set lock. However, I still think that an unfair version
in the fast path can be helpful to performance when both the unfair lock
and paravirt spinlock are enabled. In this case, paravirt spinlock code
will disable the unfair lock code in the slowpath, but still allow the
unfair version in the fast path to get the best possible performance in
a virtual guest.
Yes, I could take that out to allow either unfair or paravirt spinlock,
but not both. I do think that a little bit of unfairness will help in
the virtual environment.
>> +/*
>> + * Redefine arch_spin_lock and arch_spin_trylock as inline functions that will
>> + * jump to the unfair versions if the static key virt_unfairlocks_enabled
>> + * is true.
>> + */
>> +#undef arch_spin_lock
>> +#undef arch_spin_trylock
>> +#undef arch_spin_lock_flags
>> +
>> +/**
>> + * arch_spin_lock - acquire a queue spinlock
>> + * @lock: Pointer to queue spinlock structure
>> + */
>> +static inline void arch_spin_lock(struct qspinlock *lock)
>> +{
>> + if (static_key_false(&virt_unfairlocks_enabled))
>> + queue_spin_lock_unfair(lock);
>> + else
>> + queue_spin_lock(lock);
>> +}
>> +
>> +/**
>> + * arch_spin_trylock - try to acquire the queue spinlock
>> + * @lock : Pointer to queue spinlock structure
>> + * Return: 1 if lock acquired, 0 if failed
>> + */
>> +static inline int arch_spin_trylock(struct qspinlock *lock)
>> +{
>> + if (static_key_false(&virt_unfairlocks_enabled))
>> + return queue_spin_trylock_unfair(lock);
>> + else
>> + return queue_spin_trylock(lock);
>> +}
> So I really don't see the point of all this? Why do you need special
> {try,}lock paths for this case? Are you worried about the upper 24bits?
No, as I said above. I was planning for the coexistence of unfair lock
in the fast path and paravirt spinlock in the slowpath.
>> diff --git a/kernel/locking/qspinlock.c b/kernel/locking/qspinlock.c
>> index ae1b19d..3723c83 100644
>> --- a/kernel/locking/qspinlock.c
>> +++ b/kernel/locking/qspinlock.c
>> @@ -217,6 +217,14 @@ static __always_inline int try_set_locked(struct qspinlock *lock)
>> {
>> struct __qspinlock *l = (void *)lock;
>>
>> +#ifdef CONFIG_VIRT_UNFAIR_LOCKS
>> + /*
>> + * Need to use atomic operation to grab the lock when lock stealing
>> + * can happen.
>> + */
>> + if (static_key_false(&virt_unfairlocks_enabled))
>> + return cmpxchg(&l->locked, 0, _Q_LOCKED_VAL) == 0;
>> +#endif
>> barrier();
>> ACCESS_ONCE(l->locked) = _Q_LOCKED_VAL;
>> barrier();
> Why? If we have a simple test-and-set lock like below, we'll never get
> here at all.
Again, it is due the coexistence of unfair lock in fast path and
paravirt spinlock in the slowpath.
>> @@ -252,6 +260,18 @@ void queue_spin_lock_slowpath(struct qspinlock *lock, u32 val)
>>
>> BUILD_BUG_ON(CONFIG_NR_CPUS >= (1U << _Q_TAIL_CPU_BITS));
>>
>> +#ifdef CONFIG_VIRT_UNFAIR_LOCKS
>> + /*
>> + * A simple test and set unfair lock
>> + */
>> + if (static_key_false(&virt_unfairlocks_enabled)) {
>> + cpu_relax(); /* Relax after a failed lock attempt */
> Meh, I don't think anybody can tell the difference if you put that in or
> not, therefore don't.
Yes, I can take out the cpu_relax() here.
-Longman
^ permalink raw reply [flat|nested] 102+ messages in thread* Re: [PATCH v11 09/16] qspinlock, x86: Allow unfair spinlock in a virtual guest
@ 2014-06-12 1:37 ` Long, Wai Man
0 siblings, 0 replies; 102+ messages in thread
From: Long, Wai Man @ 2014-06-12 1:37 UTC (permalink / raw)
To: Peter Zijlstra
Cc: linux-arch, Rik van Riel, Raghavendra K T, Oleg Nesterov,
Gleb Natapov, kvm, Konrad Rzeszutek Wilk, Scott J Norton, x86,
Paolo Bonzini, linux-kernel, virtualization, Ingo Molnar,
Chegu Vinod, David Vrabel, H. Peter Anvin, xen-devel,
Thomas Gleixner, Paul E. McKenney, Linus Torvalds,
Boris Ostrovsky
On 6/11/2014 6:54 AM, Peter Zijlstra wrote:
> On Fri, May 30, 2014 at 11:43:55AM -0400, Waiman Long wrote:
>> Enabling this configuration feature causes a slight decrease the
>> performance of an uncontended lock-unlock operation by about 1-2%
>> mainly due to the use of a static key. However, uncontended lock-unlock
>> operation are really just a tiny percentage of a real workload. So
>> there should no noticeable change in application performance.
> No, entirely unacceptable.
>
>> +#ifdef CONFIG_VIRT_UNFAIR_LOCKS
>> +/**
>> + * queue_spin_trylock_unfair - try to acquire the queue spinlock unfairly
>> + * @lock : Pointer to queue spinlock structure
>> + * Return: 1 if lock acquired, 0 if failed
>> + */
>> +static __always_inline int queue_spin_trylock_unfair(struct qspinlock *lock)
>> +{
>> + union arch_qspinlock *qlock = (union arch_qspinlock *)lock;
>> +
>> + if (!qlock->locked && (cmpxchg(&qlock->locked, 0, _Q_LOCKED_VAL) == 0))
>> + return 1;
>> + return 0;
>> +}
>> +
>> +/**
>> + * queue_spin_lock_unfair - acquire a queue spinlock unfairly
>> + * @lock: Pointer to queue spinlock structure
>> + */
>> +static __always_inline void queue_spin_lock_unfair(struct qspinlock *lock)
>> +{
>> + union arch_qspinlock *qlock = (union arch_qspinlock *)lock;
>> +
>> + if (likely(cmpxchg(&qlock->locked, 0, _Q_LOCKED_VAL) == 0))
>> + return;
>> + /*
>> + * Since the lock is now unfair, we should not activate the 2-task
>> + * pending bit spinning code path which disallows lock stealing.
>> + */
>> + queue_spin_lock_slowpath(lock, -1);
>> +}
> Why is this needed?
I added the unfair version of lock and trylock as my original version
isn't a simple test-and-set lock. Now I changed the core part to use the
simple test-and-set lock. However, I still think that an unfair version
in the fast path can be helpful to performance when both the unfair lock
and paravirt spinlock are enabled. In this case, paravirt spinlock code
will disable the unfair lock code in the slowpath, but still allow the
unfair version in the fast path to get the best possible performance in
a virtual guest.
Yes, I could take that out to allow either unfair or paravirt spinlock,
but not both. I do think that a little bit of unfairness will help in
the virtual environment.
>> +/*
>> + * Redefine arch_spin_lock and arch_spin_trylock as inline functions that will
>> + * jump to the unfair versions if the static key virt_unfairlocks_enabled
>> + * is true.
>> + */
>> +#undef arch_spin_lock
>> +#undef arch_spin_trylock
>> +#undef arch_spin_lock_flags
>> +
>> +/**
>> + * arch_spin_lock - acquire a queue spinlock
>> + * @lock: Pointer to queue spinlock structure
>> + */
>> +static inline void arch_spin_lock(struct qspinlock *lock)
>> +{
>> + if (static_key_false(&virt_unfairlocks_enabled))
>> + queue_spin_lock_unfair(lock);
>> + else
>> + queue_spin_lock(lock);
>> +}
>> +
>> +/**
>> + * arch_spin_trylock - try to acquire the queue spinlock
>> + * @lock : Pointer to queue spinlock structure
>> + * Return: 1 if lock acquired, 0 if failed
>> + */
>> +static inline int arch_spin_trylock(struct qspinlock *lock)
>> +{
>> + if (static_key_false(&virt_unfairlocks_enabled))
>> + return queue_spin_trylock_unfair(lock);
>> + else
>> + return queue_spin_trylock(lock);
>> +}
> So I really don't see the point of all this? Why do you need special
> {try,}lock paths for this case? Are you worried about the upper 24bits?
No, as I said above. I was planning for the coexistence of unfair lock
in the fast path and paravirt spinlock in the slowpath.
>> diff --git a/kernel/locking/qspinlock.c b/kernel/locking/qspinlock.c
>> index ae1b19d..3723c83 100644
>> --- a/kernel/locking/qspinlock.c
>> +++ b/kernel/locking/qspinlock.c
>> @@ -217,6 +217,14 @@ static __always_inline int try_set_locked(struct qspinlock *lock)
>> {
>> struct __qspinlock *l = (void *)lock;
>>
>> +#ifdef CONFIG_VIRT_UNFAIR_LOCKS
>> + /*
>> + * Need to use atomic operation to grab the lock when lock stealing
>> + * can happen.
>> + */
>> + if (static_key_false(&virt_unfairlocks_enabled))
>> + return cmpxchg(&l->locked, 0, _Q_LOCKED_VAL) == 0;
>> +#endif
>> barrier();
>> ACCESS_ONCE(l->locked) = _Q_LOCKED_VAL;
>> barrier();
> Why? If we have a simple test-and-set lock like below, we'll never get
> here at all.
Again, it is due the coexistence of unfair lock in fast path and
paravirt spinlock in the slowpath.
>> @@ -252,6 +260,18 @@ void queue_spin_lock_slowpath(struct qspinlock *lock, u32 val)
>>
>> BUILD_BUG_ON(CONFIG_NR_CPUS >= (1U << _Q_TAIL_CPU_BITS));
>>
>> +#ifdef CONFIG_VIRT_UNFAIR_LOCKS
>> + /*
>> + * A simple test and set unfair lock
>> + */
>> + if (static_key_false(&virt_unfairlocks_enabled)) {
>> + cpu_relax(); /* Relax after a failed lock attempt */
> Meh, I don't think anybody can tell the difference if you put that in or
> not, therefore don't.
Yes, I can take out the cpu_relax() here.
-Longman
^ permalink raw reply [flat|nested] 102+ messages in thread* Re: [PATCH v11 09/16] qspinlock, x86: Allow unfair spinlock in a virtual guest
2014-06-12 1:37 ` Long, Wai Man
@ 2014-06-12 5:50 ` Peter Zijlstra
-1 siblings, 0 replies; 102+ messages in thread
From: Peter Zijlstra @ 2014-06-12 5:50 UTC (permalink / raw)
To: Long, Wai Man
Cc: Thomas Gleixner, Ingo Molnar, H. Peter Anvin, linux-arch, x86,
linux-kernel, virtualization, xen-devel, kvm, Paolo Bonzini,
Konrad Rzeszutek Wilk, Boris Ostrovsky, Paul E. McKenney,
Rik van Riel, Linus Torvalds, Raghavendra K T, David Vrabel,
Oleg Nesterov, Gleb Natapov, Scott J Norton, Chegu Vinod
[-- Attachment #1: Type: text/plain, Size: 3137 bytes --]
On Wed, Jun 11, 2014 at 09:37:55PM -0400, Long, Wai Man wrote:
>
> On 6/11/2014 6:54 AM, Peter Zijlstra wrote:
> >On Fri, May 30, 2014 at 11:43:55AM -0400, Waiman Long wrote:
> >>Enabling this configuration feature causes a slight decrease the
> >>performance of an uncontended lock-unlock operation by about 1-2%
> >>mainly due to the use of a static key. However, uncontended lock-unlock
> >>operation are really just a tiny percentage of a real workload. So
> >>there should no noticeable change in application performance.
> >No, entirely unacceptable.
> >
> >>+#ifdef CONFIG_VIRT_UNFAIR_LOCKS
> >>+/**
> >>+ * queue_spin_trylock_unfair - try to acquire the queue spinlock unfairly
> >>+ * @lock : Pointer to queue spinlock structure
> >>+ * Return: 1 if lock acquired, 0 if failed
> >>+ */
> >>+static __always_inline int queue_spin_trylock_unfair(struct qspinlock *lock)
> >>+{
> >>+ union arch_qspinlock *qlock = (union arch_qspinlock *)lock;
> >>+
> >>+ if (!qlock->locked && (cmpxchg(&qlock->locked, 0, _Q_LOCKED_VAL) == 0))
> >>+ return 1;
> >>+ return 0;
> >>+}
> >>+
> >>+/**
> >>+ * queue_spin_lock_unfair - acquire a queue spinlock unfairly
> >>+ * @lock: Pointer to queue spinlock structure
> >>+ */
> >>+static __always_inline void queue_spin_lock_unfair(struct qspinlock *lock)
> >>+{
> >>+ union arch_qspinlock *qlock = (union arch_qspinlock *)lock;
> >>+
> >>+ if (likely(cmpxchg(&qlock->locked, 0, _Q_LOCKED_VAL) == 0))
> >>+ return;
> >>+ /*
> >>+ * Since the lock is now unfair, we should not activate the 2-task
> >>+ * pending bit spinning code path which disallows lock stealing.
> >>+ */
> >>+ queue_spin_lock_slowpath(lock, -1);
> >>+}
> >Why is this needed?
>
> I added the unfair version of lock and trylock as my original version isn't
> a simple test-and-set lock. Now I changed the core part to use the simple
> test-and-set lock. However, I still think that an unfair version in the fast
> path can be helpful to performance when both the unfair lock and paravirt
> spinlock are enabled. In this case, paravirt spinlock code will disable the
> unfair lock code in the slowpath, but still allow the unfair version in the
> fast path to get the best possible performance in a virtual guest.
>
> Yes, I could take that out to allow either unfair or paravirt spinlock, but
> not both. I do think that a little bit of unfairness will help in the
> virtual environment.
When will you learn to like simplicity and stop this massive over
engineering effort?
There's no sane reason to have the test-and-set virt and paravirt locks
enabled at the same bloody time.
There's 3 distinct cases:
- native
- virt
- paravirt
And they do not overlap. Furthermore, if there is any possibility at all
of not polluting the native code, grab it with both hands.
Native performance is king, try your very utmost bestest to preserve
that, paravirt is a distant second and nobody sane should care about the
virt case at all.
If you want extra lock stealing in the paravirt case, put it in the
slowpath code before you start queueing.
[-- Attachment #2: Type: application/pgp-signature, Size: 836 bytes --]
^ permalink raw reply [flat|nested] 102+ messages in thread* Re: [PATCH v11 09/16] qspinlock, x86: Allow unfair spinlock in a virtual guest
@ 2014-06-12 5:50 ` Peter Zijlstra
0 siblings, 0 replies; 102+ messages in thread
From: Peter Zijlstra @ 2014-06-12 5:50 UTC (permalink / raw)
To: Long, Wai Man
Cc: linux-arch, Rik van Riel, Raghavendra K T, Oleg Nesterov,
Gleb Natapov, kvm, Konrad Rzeszutek Wilk, Scott J Norton, x86,
Paolo Bonzini, linux-kernel, virtualization, Ingo Molnar,
Chegu Vinod, David Vrabel, H. Peter Anvin, xen-devel,
Thomas Gleixner, Paul E. McKenney, Linus Torvalds,
Boris Ostrovsky
[-- Attachment #1.1: Type: text/plain, Size: 3137 bytes --]
On Wed, Jun 11, 2014 at 09:37:55PM -0400, Long, Wai Man wrote:
>
> On 6/11/2014 6:54 AM, Peter Zijlstra wrote:
> >On Fri, May 30, 2014 at 11:43:55AM -0400, Waiman Long wrote:
> >>Enabling this configuration feature causes a slight decrease the
> >>performance of an uncontended lock-unlock operation by about 1-2%
> >>mainly due to the use of a static key. However, uncontended lock-unlock
> >>operation are really just a tiny percentage of a real workload. So
> >>there should no noticeable change in application performance.
> >No, entirely unacceptable.
> >
> >>+#ifdef CONFIG_VIRT_UNFAIR_LOCKS
> >>+/**
> >>+ * queue_spin_trylock_unfair - try to acquire the queue spinlock unfairly
> >>+ * @lock : Pointer to queue spinlock structure
> >>+ * Return: 1 if lock acquired, 0 if failed
> >>+ */
> >>+static __always_inline int queue_spin_trylock_unfair(struct qspinlock *lock)
> >>+{
> >>+ union arch_qspinlock *qlock = (union arch_qspinlock *)lock;
> >>+
> >>+ if (!qlock->locked && (cmpxchg(&qlock->locked, 0, _Q_LOCKED_VAL) == 0))
> >>+ return 1;
> >>+ return 0;
> >>+}
> >>+
> >>+/**
> >>+ * queue_spin_lock_unfair - acquire a queue spinlock unfairly
> >>+ * @lock: Pointer to queue spinlock structure
> >>+ */
> >>+static __always_inline void queue_spin_lock_unfair(struct qspinlock *lock)
> >>+{
> >>+ union arch_qspinlock *qlock = (union arch_qspinlock *)lock;
> >>+
> >>+ if (likely(cmpxchg(&qlock->locked, 0, _Q_LOCKED_VAL) == 0))
> >>+ return;
> >>+ /*
> >>+ * Since the lock is now unfair, we should not activate the 2-task
> >>+ * pending bit spinning code path which disallows lock stealing.
> >>+ */
> >>+ queue_spin_lock_slowpath(lock, -1);
> >>+}
> >Why is this needed?
>
> I added the unfair version of lock and trylock as my original version isn't
> a simple test-and-set lock. Now I changed the core part to use the simple
> test-and-set lock. However, I still think that an unfair version in the fast
> path can be helpful to performance when both the unfair lock and paravirt
> spinlock are enabled. In this case, paravirt spinlock code will disable the
> unfair lock code in the slowpath, but still allow the unfair version in the
> fast path to get the best possible performance in a virtual guest.
>
> Yes, I could take that out to allow either unfair or paravirt spinlock, but
> not both. I do think that a little bit of unfairness will help in the
> virtual environment.
When will you learn to like simplicity and stop this massive over
engineering effort?
There's no sane reason to have the test-and-set virt and paravirt locks
enabled at the same bloody time.
There's 3 distinct cases:
- native
- virt
- paravirt
And they do not overlap. Furthermore, if there is any possibility at all
of not polluting the native code, grab it with both hands.
Native performance is king, try your very utmost bestest to preserve
that, paravirt is a distant second and nobody sane should care about the
virt case at all.
If you want extra lock stealing in the paravirt case, put it in the
slowpath code before you start queueing.
[-- Attachment #1.2: Type: application/pgp-signature, Size: 836 bytes --]
[-- Attachment #2: Type: text/plain, Size: 183 bytes --]
_______________________________________________
Virtualization mailing list
Virtualization@lists.linux-foundation.org
https://lists.linuxfoundation.org/mailman/listinfo/virtualization
^ permalink raw reply [flat|nested] 102+ messages in thread* Re: [PATCH v11 09/16] qspinlock, x86: Allow unfair spinlock in a virtual guest
2014-06-12 5:50 ` Peter Zijlstra
@ 2014-06-12 21:08 ` Waiman Long
-1 siblings, 0 replies; 102+ messages in thread
From: Waiman Long @ 2014-06-12 21:08 UTC (permalink / raw)
To: Peter Zijlstra
Cc: Thomas Gleixner, Ingo Molnar, H. Peter Anvin, linux-arch, x86,
linux-kernel, virtualization, xen-devel, kvm, Paolo Bonzini,
Konrad Rzeszutek Wilk, Boris Ostrovsky, Paul E. McKenney,
Rik van Riel, Linus Torvalds, Raghavendra K T, David Vrabel,
Oleg Nesterov, Gleb Natapov, Scott J Norton, Chegu Vinod
On 06/12/2014 01:50 AM, Peter Zijlstra wrote:
> On Wed, Jun 11, 2014 at 09:37:55PM -0400, Long, Wai Man wrote:
>> On 6/11/2014 6:54 AM, Peter Zijlstra wrote:
>>> On Fri, May 30, 2014 at 11:43:55AM -0400, Waiman Long wrote:
>>>> Enabling this configuration feature causes a slight decrease the
>>>> performance of an uncontended lock-unlock operation by about 1-2%
>>>> mainly due to the use of a static key. However, uncontended lock-unlock
>>>> operation are really just a tiny percentage of a real workload. So
>>>> there should no noticeable change in application performance.
>>> No, entirely unacceptable.
>>>
>>>> +#ifdef CONFIG_VIRT_UNFAIR_LOCKS
>>>> +/**
>>>> + * queue_spin_trylock_unfair - try to acquire the queue spinlock unfairly
>>>> + * @lock : Pointer to queue spinlock structure
>>>> + * Return: 1 if lock acquired, 0 if failed
>>>> + */
>>>> +static __always_inline int queue_spin_trylock_unfair(struct qspinlock *lock)
>>>> +{
>>>> + union arch_qspinlock *qlock = (union arch_qspinlock *)lock;
>>>> +
>>>> + if (!qlock->locked&& (cmpxchg(&qlock->locked, 0, _Q_LOCKED_VAL) == 0))
>>>> + return 1;
>>>> + return 0;
>>>> +}
>>>> +
>>>> +/**
>>>> + * queue_spin_lock_unfair - acquire a queue spinlock unfairly
>>>> + * @lock: Pointer to queue spinlock structure
>>>> + */
>>>> +static __always_inline void queue_spin_lock_unfair(struct qspinlock *lock)
>>>> +{
>>>> + union arch_qspinlock *qlock = (union arch_qspinlock *)lock;
>>>> +
>>>> + if (likely(cmpxchg(&qlock->locked, 0, _Q_LOCKED_VAL) == 0))
>>>> + return;
>>>> + /*
>>>> + * Since the lock is now unfair, we should not activate the 2-task
>>>> + * pending bit spinning code path which disallows lock stealing.
>>>> + */
>>>> + queue_spin_lock_slowpath(lock, -1);
>>>> +}
>>> Why is this needed?
>> I added the unfair version of lock and trylock as my original version isn't
>> a simple test-and-set lock. Now I changed the core part to use the simple
>> test-and-set lock. However, I still think that an unfair version in the fast
>> path can be helpful to performance when both the unfair lock and paravirt
>> spinlock are enabled. In this case, paravirt spinlock code will disable the
>> unfair lock code in the slowpath, but still allow the unfair version in the
>> fast path to get the best possible performance in a virtual guest.
>>
>> Yes, I could take that out to allow either unfair or paravirt spinlock, but
>> not both. I do think that a little bit of unfairness will help in the
>> virtual environment.
> When will you learn to like simplicity and stop this massive over
> engineering effort?
>
> There's no sane reason to have the test-and-set virt and paravirt locks
> enabled at the same bloody time.
>
> There's 3 distinct cases:
>
> - native
> - virt
> - paravirt
>
> And they do not overlap. Furthermore, if there is any possibility at all
> of not polluting the native code, grab it with both hands.
>
> Native performance is king, try your very utmost bestest to preserve
> that, paravirt is a distant second and nobody sane should care about the
> virt case at all.
The patch won't affect native performance unless the kernel is built
with VIRT_UNFAIR_LOCKS selected. The same is also true when
PARAVIRT_SPINLOCKS is selected. There is no way around that.
I do agree that I may over-engineer on this patch, but my main purpose
is to achieve the best possible performance and so may sacrifice
simplicity in some cases. Still allowing lock stealing in the fastpath
is already much simpler than what my original patch was doing with lock
stealing in the slowpath. If you still think it is too complex, I am
willing to take that out if it is what I need to get your approval.
Please also review the rests of the pvspinlock patches and let me know
if you have other comments. I would like to have one more version to go
and be done with it.
Thanks,
Longman
^ permalink raw reply [flat|nested] 102+ messages in thread* Re: [PATCH v11 09/16] qspinlock, x86: Allow unfair spinlock in a virtual guest
@ 2014-06-12 21:08 ` Waiman Long
0 siblings, 0 replies; 102+ messages in thread
From: Waiman Long @ 2014-06-12 21:08 UTC (permalink / raw)
To: Peter Zijlstra
Cc: linux-arch, Rik van Riel, Raghavendra K T, Oleg Nesterov,
Gleb Natapov, kvm, Konrad Rzeszutek Wilk, Scott J Norton, x86,
Paolo Bonzini, linux-kernel, virtualization, Ingo Molnar,
Chegu Vinod, David Vrabel, H. Peter Anvin, xen-devel,
Thomas Gleixner, Paul E. McKenney, Linus Torvalds,
Boris Ostrovsky
On 06/12/2014 01:50 AM, Peter Zijlstra wrote:
> On Wed, Jun 11, 2014 at 09:37:55PM -0400, Long, Wai Man wrote:
>> On 6/11/2014 6:54 AM, Peter Zijlstra wrote:
>>> On Fri, May 30, 2014 at 11:43:55AM -0400, Waiman Long wrote:
>>>> Enabling this configuration feature causes a slight decrease the
>>>> performance of an uncontended lock-unlock operation by about 1-2%
>>>> mainly due to the use of a static key. However, uncontended lock-unlock
>>>> operation are really just a tiny percentage of a real workload. So
>>>> there should no noticeable change in application performance.
>>> No, entirely unacceptable.
>>>
>>>> +#ifdef CONFIG_VIRT_UNFAIR_LOCKS
>>>> +/**
>>>> + * queue_spin_trylock_unfair - try to acquire the queue spinlock unfairly
>>>> + * @lock : Pointer to queue spinlock structure
>>>> + * Return: 1 if lock acquired, 0 if failed
>>>> + */
>>>> +static __always_inline int queue_spin_trylock_unfair(struct qspinlock *lock)
>>>> +{
>>>> + union arch_qspinlock *qlock = (union arch_qspinlock *)lock;
>>>> +
>>>> + if (!qlock->locked&& (cmpxchg(&qlock->locked, 0, _Q_LOCKED_VAL) == 0))
>>>> + return 1;
>>>> + return 0;
>>>> +}
>>>> +
>>>> +/**
>>>> + * queue_spin_lock_unfair - acquire a queue spinlock unfairly
>>>> + * @lock: Pointer to queue spinlock structure
>>>> + */
>>>> +static __always_inline void queue_spin_lock_unfair(struct qspinlock *lock)
>>>> +{
>>>> + union arch_qspinlock *qlock = (union arch_qspinlock *)lock;
>>>> +
>>>> + if (likely(cmpxchg(&qlock->locked, 0, _Q_LOCKED_VAL) == 0))
>>>> + return;
>>>> + /*
>>>> + * Since the lock is now unfair, we should not activate the 2-task
>>>> + * pending bit spinning code path which disallows lock stealing.
>>>> + */
>>>> + queue_spin_lock_slowpath(lock, -1);
>>>> +}
>>> Why is this needed?
>> I added the unfair version of lock and trylock as my original version isn't
>> a simple test-and-set lock. Now I changed the core part to use the simple
>> test-and-set lock. However, I still think that an unfair version in the fast
>> path can be helpful to performance when both the unfair lock and paravirt
>> spinlock are enabled. In this case, paravirt spinlock code will disable the
>> unfair lock code in the slowpath, but still allow the unfair version in the
>> fast path to get the best possible performance in a virtual guest.
>>
>> Yes, I could take that out to allow either unfair or paravirt spinlock, but
>> not both. I do think that a little bit of unfairness will help in the
>> virtual environment.
> When will you learn to like simplicity and stop this massive over
> engineering effort?
>
> There's no sane reason to have the test-and-set virt and paravirt locks
> enabled at the same bloody time.
>
> There's 3 distinct cases:
>
> - native
> - virt
> - paravirt
>
> And they do not overlap. Furthermore, if there is any possibility at all
> of not polluting the native code, grab it with both hands.
>
> Native performance is king, try your very utmost bestest to preserve
> that, paravirt is a distant second and nobody sane should care about the
> virt case at all.
The patch won't affect native performance unless the kernel is built
with VIRT_UNFAIR_LOCKS selected. The same is also true when
PARAVIRT_SPINLOCKS is selected. There is no way around that.
I do agree that I may over-engineer on this patch, but my main purpose
is to achieve the best possible performance and so may sacrifice
simplicity in some cases. Still allowing lock stealing in the fastpath
is already much simpler than what my original patch was doing with lock
stealing in the slowpath. If you still think it is too complex, I am
willing to take that out if it is what I need to get your approval.
Please also review the rests of the pvspinlock patches and let me know
if you have other comments. I would like to have one more version to go
and be done with it.
Thanks,
Longman
^ permalink raw reply [flat|nested] 102+ messages in thread* Re: [PATCH v11 09/16] qspinlock, x86: Allow unfair spinlock in a virtual guest
2014-06-12 21:08 ` Waiman Long
(?)
@ 2014-06-15 13:14 ` Peter Zijlstra
-1 siblings, 0 replies; 102+ messages in thread
From: Peter Zijlstra @ 2014-06-15 13:14 UTC (permalink / raw)
To: Waiman Long
Cc: linux-arch, Raghavendra K T, Oleg Nesterov, Gleb Natapov, kvm,
Scott J Norton, x86, Paolo Bonzini, linux-kernel, virtualization,
Ingo Molnar, Chegu Vinod, David Vrabel, H. Peter Anvin,
xen-devel, Thomas Gleixner, Paul E. McKenney, Linus Torvalds,
Boris Ostrovsky
On Thu, Jun 12, 2014 at 05:08:28PM -0400, Waiman Long wrote:
> >Native performance is king, try your very utmost bestest to preserve
> >that, paravirt is a distant second and nobody sane should care about the
> >virt case at all.
>
> The patch won't affect native performance unless the kernel is built with
> VIRT_UNFAIR_LOCKS selected. The same is also true when PARAVIRT_SPINLOCKS is
> selected. There is no way around that.
VIRT_UNFAIR_LOCKS is an impossible switch to have; a distro cannot make
the right choice.
> I do agree that I may over-engineer on this patch,
Simple things first, then add complexity.
^ permalink raw reply [flat|nested] 102+ messages in thread
* Re: [PATCH v11 09/16] qspinlock, x86: Allow unfair spinlock in a virtual guest
2014-06-12 21:08 ` Waiman Long
@ 2014-06-15 13:14 ` Peter Zijlstra
-1 siblings, 0 replies; 102+ messages in thread
From: Peter Zijlstra @ 2014-06-15 13:14 UTC (permalink / raw)
To: Waiman Long
Cc: Thomas Gleixner, Ingo Molnar, H. Peter Anvin, linux-arch, x86,
linux-kernel, virtualization, xen-devel, kvm, Paolo Bonzini,
Konrad Rzeszutek Wilk, Boris Ostrovsky, Paul E. McKenney,
Rik van Riel, Linus Torvalds, Raghavendra K T, David Vrabel,
Oleg Nesterov, Gleb Natapov, Scott J Norton, Chegu Vinod
On Thu, Jun 12, 2014 at 05:08:28PM -0400, Waiman Long wrote:
> >Native performance is king, try your very utmost bestest to preserve
> >that, paravirt is a distant second and nobody sane should care about the
> >virt case at all.
>
> The patch won't affect native performance unless the kernel is built with
> VIRT_UNFAIR_LOCKS selected. The same is also true when PARAVIRT_SPINLOCKS is
> selected. There is no way around that.
VIRT_UNFAIR_LOCKS is an impossible switch to have; a distro cannot make
the right choice.
> I do agree that I may over-engineer on this patch,
Simple things first, then add complexity.
^ permalink raw reply [flat|nested] 102+ messages in thread
* Re: [PATCH v11 09/16] qspinlock, x86: Allow unfair spinlock in a virtual guest
@ 2014-06-15 13:14 ` Peter Zijlstra
0 siblings, 0 replies; 102+ messages in thread
From: Peter Zijlstra @ 2014-06-15 13:14 UTC (permalink / raw)
To: Waiman Long
Cc: linux-arch, Rik van Riel, Raghavendra K T, Oleg Nesterov,
Gleb Natapov, kvm, Konrad Rzeszutek Wilk, Scott J Norton, x86,
Paolo Bonzini, linux-kernel, virtualization, Ingo Molnar,
Chegu Vinod, David Vrabel, H. Peter Anvin, xen-devel,
Thomas Gleixner, Paul E. McKenney, Linus Torvalds,
Boris Ostrovsky
On Thu, Jun 12, 2014 at 05:08:28PM -0400, Waiman Long wrote:
> >Native performance is king, try your very utmost bestest to preserve
> >that, paravirt is a distant second and nobody sane should care about the
> >virt case at all.
>
> The patch won't affect native performance unless the kernel is built with
> VIRT_UNFAIR_LOCKS selected. The same is also true when PARAVIRT_SPINLOCKS is
> selected. There is no way around that.
VIRT_UNFAIR_LOCKS is an impossible switch to have; a distro cannot make
the right choice.
> I do agree that I may over-engineer on this patch,
Simple things first, then add complexity.
^ permalink raw reply [flat|nested] 102+ messages in thread
* Re: [PATCH v11 09/16] qspinlock, x86: Allow unfair spinlock in a virtual guest
2014-06-12 5:50 ` Peter Zijlstra
(?)
(?)
@ 2014-06-12 21:08 ` Waiman Long
-1 siblings, 0 replies; 102+ messages in thread
From: Waiman Long @ 2014-06-12 21:08 UTC (permalink / raw)
To: Peter Zijlstra
Cc: linux-arch, Raghavendra K T, Oleg Nesterov, Gleb Natapov, kvm,
Scott J Norton, x86, Paolo Bonzini, linux-kernel, virtualization,
Ingo Molnar, Chegu Vinod, David Vrabel, H. Peter Anvin,
xen-devel, Thomas Gleixner, Paul E. McKenney, Linus Torvalds,
Boris Ostrovsky
On 06/12/2014 01:50 AM, Peter Zijlstra wrote:
> On Wed, Jun 11, 2014 at 09:37:55PM -0400, Long, Wai Man wrote:
>> On 6/11/2014 6:54 AM, Peter Zijlstra wrote:
>>> On Fri, May 30, 2014 at 11:43:55AM -0400, Waiman Long wrote:
>>>> Enabling this configuration feature causes a slight decrease the
>>>> performance of an uncontended lock-unlock operation by about 1-2%
>>>> mainly due to the use of a static key. However, uncontended lock-unlock
>>>> operation are really just a tiny percentage of a real workload. So
>>>> there should no noticeable change in application performance.
>>> No, entirely unacceptable.
>>>
>>>> +#ifdef CONFIG_VIRT_UNFAIR_LOCKS
>>>> +/**
>>>> + * queue_spin_trylock_unfair - try to acquire the queue spinlock unfairly
>>>> + * @lock : Pointer to queue spinlock structure
>>>> + * Return: 1 if lock acquired, 0 if failed
>>>> + */
>>>> +static __always_inline int queue_spin_trylock_unfair(struct qspinlock *lock)
>>>> +{
>>>> + union arch_qspinlock *qlock = (union arch_qspinlock *)lock;
>>>> +
>>>> + if (!qlock->locked&& (cmpxchg(&qlock->locked, 0, _Q_LOCKED_VAL) == 0))
>>>> + return 1;
>>>> + return 0;
>>>> +}
>>>> +
>>>> +/**
>>>> + * queue_spin_lock_unfair - acquire a queue spinlock unfairly
>>>> + * @lock: Pointer to queue spinlock structure
>>>> + */
>>>> +static __always_inline void queue_spin_lock_unfair(struct qspinlock *lock)
>>>> +{
>>>> + union arch_qspinlock *qlock = (union arch_qspinlock *)lock;
>>>> +
>>>> + if (likely(cmpxchg(&qlock->locked, 0, _Q_LOCKED_VAL) == 0))
>>>> + return;
>>>> + /*
>>>> + * Since the lock is now unfair, we should not activate the 2-task
>>>> + * pending bit spinning code path which disallows lock stealing.
>>>> + */
>>>> + queue_spin_lock_slowpath(lock, -1);
>>>> +}
>>> Why is this needed?
>> I added the unfair version of lock and trylock as my original version isn't
>> a simple test-and-set lock. Now I changed the core part to use the simple
>> test-and-set lock. However, I still think that an unfair version in the fast
>> path can be helpful to performance when both the unfair lock and paravirt
>> spinlock are enabled. In this case, paravirt spinlock code will disable the
>> unfair lock code in the slowpath, but still allow the unfair version in the
>> fast path to get the best possible performance in a virtual guest.
>>
>> Yes, I could take that out to allow either unfair or paravirt spinlock, but
>> not both. I do think that a little bit of unfairness will help in the
>> virtual environment.
> When will you learn to like simplicity and stop this massive over
> engineering effort?
>
> There's no sane reason to have the test-and-set virt and paravirt locks
> enabled at the same bloody time.
>
> There's 3 distinct cases:
>
> - native
> - virt
> - paravirt
>
> And they do not overlap. Furthermore, if there is any possibility at all
> of not polluting the native code, grab it with both hands.
>
> Native performance is king, try your very utmost bestest to preserve
> that, paravirt is a distant second and nobody sane should care about the
> virt case at all.
The patch won't affect native performance unless the kernel is built
with VIRT_UNFAIR_LOCKS selected. The same is also true when
PARAVIRT_SPINLOCKS is selected. There is no way around that.
I do agree that I may over-engineer on this patch, but my main purpose
is to achieve the best possible performance and so may sacrifice
simplicity in some cases. Still allowing lock stealing in the fastpath
is already much simpler than what my original patch was doing with lock
stealing in the slowpath. If you still think it is too complex, I am
willing to take that out if it is what I need to get your approval.
Please also review the rests of the pvspinlock patches and let me know
if you have other comments. I would like to have one more version to go
and be done with it.
Thanks,
Longman
^ permalink raw reply [flat|nested] 102+ messages in thread
* Re: [PATCH v11 09/16] qspinlock, x86: Allow unfair spinlock in a virtual guest
2014-06-12 1:37 ` Long, Wai Man
(?)
(?)
@ 2014-06-12 5:50 ` Peter Zijlstra
-1 siblings, 0 replies; 102+ messages in thread
From: Peter Zijlstra @ 2014-06-12 5:50 UTC (permalink / raw)
To: Long, Wai Man
Cc: linux-arch, Raghavendra K T, Oleg Nesterov, Gleb Natapov, kvm,
Scott J Norton, x86, Paolo Bonzini, linux-kernel, virtualization,
Ingo Molnar, Chegu Vinod, David Vrabel, H. Peter Anvin,
xen-devel, Thomas Gleixner, Paul E. McKenney, Linus Torvalds,
Boris Ostrovsky
[-- Attachment #1.1: Type: text/plain, Size: 3137 bytes --]
On Wed, Jun 11, 2014 at 09:37:55PM -0400, Long, Wai Man wrote:
>
> On 6/11/2014 6:54 AM, Peter Zijlstra wrote:
> >On Fri, May 30, 2014 at 11:43:55AM -0400, Waiman Long wrote:
> >>Enabling this configuration feature causes a slight decrease the
> >>performance of an uncontended lock-unlock operation by about 1-2%
> >>mainly due to the use of a static key. However, uncontended lock-unlock
> >>operation are really just a tiny percentage of a real workload. So
> >>there should no noticeable change in application performance.
> >No, entirely unacceptable.
> >
> >>+#ifdef CONFIG_VIRT_UNFAIR_LOCKS
> >>+/**
> >>+ * queue_spin_trylock_unfair - try to acquire the queue spinlock unfairly
> >>+ * @lock : Pointer to queue spinlock structure
> >>+ * Return: 1 if lock acquired, 0 if failed
> >>+ */
> >>+static __always_inline int queue_spin_trylock_unfair(struct qspinlock *lock)
> >>+{
> >>+ union arch_qspinlock *qlock = (union arch_qspinlock *)lock;
> >>+
> >>+ if (!qlock->locked && (cmpxchg(&qlock->locked, 0, _Q_LOCKED_VAL) == 0))
> >>+ return 1;
> >>+ return 0;
> >>+}
> >>+
> >>+/**
> >>+ * queue_spin_lock_unfair - acquire a queue spinlock unfairly
> >>+ * @lock: Pointer to queue spinlock structure
> >>+ */
> >>+static __always_inline void queue_spin_lock_unfair(struct qspinlock *lock)
> >>+{
> >>+ union arch_qspinlock *qlock = (union arch_qspinlock *)lock;
> >>+
> >>+ if (likely(cmpxchg(&qlock->locked, 0, _Q_LOCKED_VAL) == 0))
> >>+ return;
> >>+ /*
> >>+ * Since the lock is now unfair, we should not activate the 2-task
> >>+ * pending bit spinning code path which disallows lock stealing.
> >>+ */
> >>+ queue_spin_lock_slowpath(lock, -1);
> >>+}
> >Why is this needed?
>
> I added the unfair version of lock and trylock as my original version isn't
> a simple test-and-set lock. Now I changed the core part to use the simple
> test-and-set lock. However, I still think that an unfair version in the fast
> path can be helpful to performance when both the unfair lock and paravirt
> spinlock are enabled. In this case, paravirt spinlock code will disable the
> unfair lock code in the slowpath, but still allow the unfair version in the
> fast path to get the best possible performance in a virtual guest.
>
> Yes, I could take that out to allow either unfair or paravirt spinlock, but
> not both. I do think that a little bit of unfairness will help in the
> virtual environment.
When will you learn to like simplicity and stop this massive over
engineering effort?
There's no sane reason to have the test-and-set virt and paravirt locks
enabled at the same bloody time.
There's 3 distinct cases:
- native
- virt
- paravirt
And they do not overlap. Furthermore, if there is any possibility at all
of not polluting the native code, grab it with both hands.
Native performance is king, try your very utmost bestest to preserve
that, paravirt is a distant second and nobody sane should care about the
virt case at all.
If you want extra lock stealing in the paravirt case, put it in the
slowpath code before you start queueing.
[-- Attachment #1.2: Type: application/pgp-signature, Size: 836 bytes --]
[-- Attachment #2: Type: text/plain, Size: 126 bytes --]
_______________________________________________
Xen-devel mailing list
Xen-devel@lists.xen.org
http://lists.xen.org/xen-devel
^ permalink raw reply [flat|nested] 102+ messages in thread
* Re: [PATCH v11 09/16] qspinlock, x86: Allow unfair spinlock in a virtual guest
2014-05-30 15:43 ` Waiman Long
(?)
(?)
@ 2014-06-11 10:54 ` Peter Zijlstra
-1 siblings, 0 replies; 102+ messages in thread
From: Peter Zijlstra @ 2014-06-11 10:54 UTC (permalink / raw)
To: Waiman Long
Cc: linux-arch, Raghavendra K T, Oleg Nesterov, Gleb Natapov, kvm,
Scott J Norton, x86, Paolo Bonzini, linux-kernel, virtualization,
Ingo Molnar, Chegu Vinod, David Vrabel, H. Peter Anvin,
xen-devel, Thomas Gleixner, Paul E. McKenney, Linus Torvalds,
Boris Ostrovsky
[-- Attachment #1.1: Type: text/plain, Size: 4146 bytes --]
On Fri, May 30, 2014 at 11:43:55AM -0400, Waiman Long wrote:
> Enabling this configuration feature causes a slight decrease the
> performance of an uncontended lock-unlock operation by about 1-2%
> mainly due to the use of a static key. However, uncontended lock-unlock
> operation are really just a tiny percentage of a real workload. So
> there should no noticeable change in application performance.
No, entirely unacceptable.
> +#ifdef CONFIG_VIRT_UNFAIR_LOCKS
> +/**
> + * queue_spin_trylock_unfair - try to acquire the queue spinlock unfairly
> + * @lock : Pointer to queue spinlock structure
> + * Return: 1 if lock acquired, 0 if failed
> + */
> +static __always_inline int queue_spin_trylock_unfair(struct qspinlock *lock)
> +{
> + union arch_qspinlock *qlock = (union arch_qspinlock *)lock;
> +
> + if (!qlock->locked && (cmpxchg(&qlock->locked, 0, _Q_LOCKED_VAL) == 0))
> + return 1;
> + return 0;
> +}
> +
> +/**
> + * queue_spin_lock_unfair - acquire a queue spinlock unfairly
> + * @lock: Pointer to queue spinlock structure
> + */
> +static __always_inline void queue_spin_lock_unfair(struct qspinlock *lock)
> +{
> + union arch_qspinlock *qlock = (union arch_qspinlock *)lock;
> +
> + if (likely(cmpxchg(&qlock->locked, 0, _Q_LOCKED_VAL) == 0))
> + return;
> + /*
> + * Since the lock is now unfair, we should not activate the 2-task
> + * pending bit spinning code path which disallows lock stealing.
> + */
> + queue_spin_lock_slowpath(lock, -1);
> +}
Why is this needed?
> +/*
> + * Redefine arch_spin_lock and arch_spin_trylock as inline functions that will
> + * jump to the unfair versions if the static key virt_unfairlocks_enabled
> + * is true.
> + */
> +#undef arch_spin_lock
> +#undef arch_spin_trylock
> +#undef arch_spin_lock_flags
> +
> +/**
> + * arch_spin_lock - acquire a queue spinlock
> + * @lock: Pointer to queue spinlock structure
> + */
> +static inline void arch_spin_lock(struct qspinlock *lock)
> +{
> + if (static_key_false(&virt_unfairlocks_enabled))
> + queue_spin_lock_unfair(lock);
> + else
> + queue_spin_lock(lock);
> +}
> +
> +/**
> + * arch_spin_trylock - try to acquire the queue spinlock
> + * @lock : Pointer to queue spinlock structure
> + * Return: 1 if lock acquired, 0 if failed
> + */
> +static inline int arch_spin_trylock(struct qspinlock *lock)
> +{
> + if (static_key_false(&virt_unfairlocks_enabled))
> + return queue_spin_trylock_unfair(lock);
> + else
> + return queue_spin_trylock(lock);
> +}
So I really don't see the point of all this? Why do you need special
{try,}lock paths for this case? Are you worried about the upper 24bits?
> diff --git a/kernel/locking/qspinlock.c b/kernel/locking/qspinlock.c
> index ae1b19d..3723c83 100644
> --- a/kernel/locking/qspinlock.c
> +++ b/kernel/locking/qspinlock.c
> @@ -217,6 +217,14 @@ static __always_inline int try_set_locked(struct qspinlock *lock)
> {
> struct __qspinlock *l = (void *)lock;
>
> +#ifdef CONFIG_VIRT_UNFAIR_LOCKS
> + /*
> + * Need to use atomic operation to grab the lock when lock stealing
> + * can happen.
> + */
> + if (static_key_false(&virt_unfairlocks_enabled))
> + return cmpxchg(&l->locked, 0, _Q_LOCKED_VAL) == 0;
> +#endif
> barrier();
> ACCESS_ONCE(l->locked) = _Q_LOCKED_VAL;
> barrier();
Why? If we have a simple test-and-set lock like below, we'll never get
here at all.
> @@ -252,6 +260,18 @@ void queue_spin_lock_slowpath(struct qspinlock *lock, u32 val)
>
> BUILD_BUG_ON(CONFIG_NR_CPUS >= (1U << _Q_TAIL_CPU_BITS));
>
> +#ifdef CONFIG_VIRT_UNFAIR_LOCKS
> + /*
> + * A simple test and set unfair lock
> + */
> + if (static_key_false(&virt_unfairlocks_enabled)) {
> + cpu_relax(); /* Relax after a failed lock attempt */
Meh, I don't think anybody can tell the difference if you put that in or
not, therefore don't.
> + while (!queue_spin_trylock(lock))
> + cpu_relax();
> + return;
> + }
> +#endif /* CONFIG_VIRT_UNFAIR_LOCKS */
If you're really worried about those upper 24bits, you can always clear
them when you get here.
[-- Attachment #1.2: Type: application/pgp-signature, Size: 836 bytes --]
[-- Attachment #2: Type: text/plain, Size: 126 bytes --]
_______________________________________________
Xen-devel mailing list
Xen-devel@lists.xen.org
http://lists.xen.org/xen-devel
^ permalink raw reply [flat|nested] 102+ messages in thread
* [PATCH v11 09/16] qspinlock, x86: Allow unfair spinlock in a virtual guest
2014-05-30 15:43 [PATCH v11 00/16] qspinlock: a 4-byte queue spinlock with PV support Waiman Long
` (18 preceding siblings ...)
2014-05-30 15:43 ` Waiman Long
@ 2014-05-30 15:43 ` Waiman Long
2014-05-30 15:43 ` Waiman Long
` (17 subsequent siblings)
37 siblings, 0 replies; 102+ messages in thread
From: Waiman Long @ 2014-05-30 15:43 UTC (permalink / raw)
To: Thomas Gleixner, Ingo Molnar, H. Peter Anvin, Peter Zijlstra
Cc: linux-arch, Waiman Long, Raghavendra K T, Gleb Natapov, kvm,
Scott J Norton, x86, Paolo Bonzini, linux-kernel, virtualization,
Chegu Vinod, David Vrabel, Oleg Nesterov, xen-devel,
Boris Ostrovsky, Paul E. McKenney, Linus Torvalds
Locking is always an issue in a virtualized environment because of 2
different types of problems:
1) Lock holder preemption
2) Lock waiter preemption
One solution to the lock waiter preemption problem is to allow unfair
lock in a virtualized environment. In this case, a new lock acquirer
can come and steal the lock if the next-in-line CPU to get the lock
is scheduled out.
A simple unfair queue spinlock can be implemented by allowing lock
stealing in the fast path. The slowpath will also be modified to run a
simple queue_spin_trylock() loop. A simple test and set lock like that
does have the problem that the The constant spinning on the lock word
put a lot of cacheline contention traffic on the affected cacheline,
thus slowing tasks that need to access the cacheline.
Unfair lock in a native environment is generally not a good idea as
there is a possibility of lock starvation for a heavily contended lock.
This patch adds a new configuration option for the x86 architecture
to enable the use of unfair queue spinlock (AVIRT_UNFAIR_LOCKS)
in a virtual guest. A jump label (virt_unfairlocks_enabled) is
used to switch between a fair and an unfair version of the spinlock
code. This jump label will only be enabled in a virtual guest where
the X86_FEATURE_HYPERVISOR feature bit is set.
Enabling this configuration feature causes a slight decrease the
performance of an uncontended lock-unlock operation by about 1-2%
mainly due to the use of a static key. However, uncontended lock-unlock
operation are really just a tiny percentage of a real workload. So
there should no noticeable change in application performance.
With the unfair locking activated on bare metal 4-socket Westmere-EX
box, the execution times (in ms) of a spinlock micro-benchmark were
as follows:
# of Ticket Fair Unfair
tasks lock queue lock queue lock
------ ------- ---------- ----------
1 135 135 137
2 890 1082 613
3 1932 2248 1211
4 2829 2819 1720
5 3834 3522 2461
6 4963 4173 3715
7 6299 4875 3749
8 7691 5563 4194
Executing one task per node, the performance data were:
# of Ticket Fair Unfair
nodes lock queue lock queue lock
------ ------- ---------- ----------
1 135 135 137
2 4603 1034 1458
3 10940 12087 2562
4 21555 10507 4793
Signed-off-by: Waiman Long <Waiman.Long@hp.com>
---
arch/x86/Kconfig | 11 +++++
arch/x86/include/asm/qspinlock.h | 79 ++++++++++++++++++++++++++++++++++
arch/x86/kernel/Makefile | 1 +
arch/x86/kernel/paravirt-spinlocks.c | 26 +++++++++++
kernel/locking/qspinlock.c | 20 +++++++++
5 files changed, 137 insertions(+), 0 deletions(-)
diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig
index 95c9c4e..961f43a 100644
--- a/arch/x86/Kconfig
+++ b/arch/x86/Kconfig
@@ -585,6 +585,17 @@ config PARAVIRT_SPINLOCKS
If you are unsure how to answer this question, answer Y.
+config VIRT_UNFAIR_LOCKS
+ bool "Enable unfair locks in a virtual guest"
+ depends on SMP && QUEUE_SPINLOCK
+ depends on !CONFIG_X86_OOSTORE && !CONFIG_X86_PPRO_FENCE
+ ---help---
+ This changes the kernel to use unfair locks in a virtual
+ guest. This will help performance in most cases. However,
+ there is a possibility of lock starvation on a heavily
+ contended lock especially in a large guest with many
+ virtual CPUs.
+
source "arch/x86/xen/Kconfig"
config KVM_GUEST
diff --git a/arch/x86/include/asm/qspinlock.h b/arch/x86/include/asm/qspinlock.h
index e4a4f5d..448de8b 100644
--- a/arch/x86/include/asm/qspinlock.h
+++ b/arch/x86/include/asm/qspinlock.h
@@ -5,6 +5,10 @@
#if !defined(CONFIG_X86_OOSTORE) && !defined(CONFIG_X86_PPRO_FENCE)
+#ifdef CONFIG_VIRT_UNFAIR_LOCKS
+extern struct static_key virt_unfairlocks_enabled;
+#endif
+
#define queue_spin_unlock queue_spin_unlock
/**
* queue_spin_unlock - release a queue spinlock
@@ -26,4 +30,79 @@ static inline void queue_spin_unlock(struct qspinlock *lock)
#include <asm-generic/qspinlock.h>
+union arch_qspinlock {
+ atomic_t val;
+ u8 locked;
+};
+
+#ifdef CONFIG_VIRT_UNFAIR_LOCKS
+/**
+ * queue_spin_trylock_unfair - try to acquire the queue spinlock unfairly
+ * @lock : Pointer to queue spinlock structure
+ * Return: 1 if lock acquired, 0 if failed
+ */
+static __always_inline int queue_spin_trylock_unfair(struct qspinlock *lock)
+{
+ union arch_qspinlock *qlock = (union arch_qspinlock *)lock;
+
+ if (!qlock->locked && (cmpxchg(&qlock->locked, 0, _Q_LOCKED_VAL) == 0))
+ return 1;
+ return 0;
+}
+
+/**
+ * queue_spin_lock_unfair - acquire a queue spinlock unfairly
+ * @lock: Pointer to queue spinlock structure
+ */
+static __always_inline void queue_spin_lock_unfair(struct qspinlock *lock)
+{
+ union arch_qspinlock *qlock = (union arch_qspinlock *)lock;
+
+ if (likely(cmpxchg(&qlock->locked, 0, _Q_LOCKED_VAL) == 0))
+ return;
+ /*
+ * Since the lock is now unfair, we should not activate the 2-task
+ * pending bit spinning code path which disallows lock stealing.
+ */
+ queue_spin_lock_slowpath(lock, -1);
+}
+
+/*
+ * Redefine arch_spin_lock and arch_spin_trylock as inline functions that will
+ * jump to the unfair versions if the static key virt_unfairlocks_enabled
+ * is true.
+ */
+#undef arch_spin_lock
+#undef arch_spin_trylock
+#undef arch_spin_lock_flags
+
+/**
+ * arch_spin_lock - acquire a queue spinlock
+ * @lock: Pointer to queue spinlock structure
+ */
+static inline void arch_spin_lock(struct qspinlock *lock)
+{
+ if (static_key_false(&virt_unfairlocks_enabled))
+ queue_spin_lock_unfair(lock);
+ else
+ queue_spin_lock(lock);
+}
+
+/**
+ * arch_spin_trylock - try to acquire the queue spinlock
+ * @lock : Pointer to queue spinlock structure
+ * Return: 1 if lock acquired, 0 if failed
+ */
+static inline int arch_spin_trylock(struct qspinlock *lock)
+{
+ if (static_key_false(&virt_unfairlocks_enabled))
+ return queue_spin_trylock_unfair(lock);
+ else
+ return queue_spin_trylock(lock);
+}
+
+#define arch_spin_lock_flags(l, f) arch_spin_lock(l)
+
+#endif /* CONFIG_VIRT_UNFAIR_LOCKS */
+
#endif /* _ASM_X86_QSPINLOCK_H */
diff --git a/arch/x86/kernel/Makefile b/arch/x86/kernel/Makefile
index f4d9600..cf592f3 100644
--- a/arch/x86/kernel/Makefile
+++ b/arch/x86/kernel/Makefile
@@ -88,6 +88,7 @@ obj-$(CONFIG_DEBUG_NMI_SELFTEST) += nmi_selftest.o
obj-$(CONFIG_KVM_GUEST) += kvm.o kvmclock.o
obj-$(CONFIG_PARAVIRT) += paravirt.o paravirt_patch_$(BITS).o
obj-$(CONFIG_PARAVIRT_SPINLOCKS)+= paravirt-spinlocks.o
+obj-$(CONFIG_VIRT_UNFAIR_LOCKS) += paravirt-spinlocks.o
obj-$(CONFIG_PARAVIRT_CLOCK) += pvclock.o
obj-$(CONFIG_PCSPKR_PLATFORM) += pcspeaker.o
diff --git a/arch/x86/kernel/paravirt-spinlocks.c b/arch/x86/kernel/paravirt-spinlocks.c
index bbb6c73..69ed806 100644
--- a/arch/x86/kernel/paravirt-spinlocks.c
+++ b/arch/x86/kernel/paravirt-spinlocks.c
@@ -8,6 +8,7 @@
#include <asm/paravirt.h>
+#ifdef CONFIG_PARAVIRT_SPINLOCKS
struct pv_lock_ops pv_lock_ops = {
#ifdef CONFIG_SMP
.lock_spinning = __PV_IS_CALLEE_SAVE(paravirt_nop),
@@ -18,3 +19,28 @@ EXPORT_SYMBOL(pv_lock_ops);
struct static_key paravirt_ticketlocks_enabled = STATIC_KEY_INIT_FALSE;
EXPORT_SYMBOL(paravirt_ticketlocks_enabled);
+#endif
+
+#ifdef CONFIG_VIRT_UNFAIR_LOCKS
+struct static_key virt_unfairlocks_enabled = STATIC_KEY_INIT_FALSE;
+EXPORT_SYMBOL(virt_unfairlocks_enabled);
+
+#include <linux/init.h>
+#include <asm/cpufeature.h>
+
+/*
+ * Enable unfair lock only if it is running under a hypervisor
+ */
+static __init int unfair_locks_init_jump(void)
+{
+ if (!boot_cpu_has(X86_FEATURE_HYPERVISOR))
+ return 0;
+
+ static_key_slow_inc(&virt_unfairlocks_enabled);
+ printk(KERN_INFO "Unfair spinlock enabled\n");
+
+ return 0;
+}
+early_initcall(unfair_locks_init_jump);
+
+#endif
diff --git a/kernel/locking/qspinlock.c b/kernel/locking/qspinlock.c
index ae1b19d..3723c83 100644
--- a/kernel/locking/qspinlock.c
+++ b/kernel/locking/qspinlock.c
@@ -217,6 +217,14 @@ static __always_inline int try_set_locked(struct qspinlock *lock)
{
struct __qspinlock *l = (void *)lock;
+#ifdef CONFIG_VIRT_UNFAIR_LOCKS
+ /*
+ * Need to use atomic operation to grab the lock when lock stealing
+ * can happen.
+ */
+ if (static_key_false(&virt_unfairlocks_enabled))
+ return cmpxchg(&l->locked, 0, _Q_LOCKED_VAL) == 0;
+#endif
barrier();
ACCESS_ONCE(l->locked) = _Q_LOCKED_VAL;
barrier();
@@ -252,6 +260,18 @@ void queue_spin_lock_slowpath(struct qspinlock *lock, u32 val)
BUILD_BUG_ON(CONFIG_NR_CPUS >= (1U << _Q_TAIL_CPU_BITS));
+#ifdef CONFIG_VIRT_UNFAIR_LOCKS
+ /*
+ * A simple test and set unfair lock
+ */
+ if (static_key_false(&virt_unfairlocks_enabled)) {
+ cpu_relax(); /* Relax after a failed lock attempt */
+ while (!queue_spin_trylock(lock))
+ cpu_relax();
+ return;
+ }
+#endif /* CONFIG_VIRT_UNFAIR_LOCKS */
+
/*
* trylock || pending
*
--
1.7.1
^ permalink raw reply related [flat|nested] 102+ messages in thread* [PATCH v11 10/16] qspinlock: Split the MCS queuing code into a separate slowerpath
2014-05-30 15:43 [PATCH v11 00/16] qspinlock: a 4-byte queue spinlock with PV support Waiman Long
@ 2014-05-30 15:43 ` Waiman Long
2014-05-30 15:43 ` Waiman Long
` (36 subsequent siblings)
37 siblings, 0 replies; 102+ messages in thread
From: Waiman Long @ 2014-05-30 15:43 UTC (permalink / raw)
To: Thomas Gleixner, Ingo Molnar, H. Peter Anvin, Peter Zijlstra
Cc: linux-arch, x86, linux-kernel, virtualization, xen-devel, kvm,
Paolo Bonzini, Konrad Rzeszutek Wilk, Boris Ostrovsky,
Paul E. McKenney, Rik van Riel, Linus Torvalds, Raghavendra K T,
David Vrabel, Oleg Nesterov, Gleb Natapov, Scott J Norton,
Chegu Vinod, Waiman Long
With the pending addition of more codes to support PV spinlock, the
complexity of the slowpath function increases to the point that the
number of scratch-pad registers in the x86-64 architecture is not
enough and so those additional non-scratch-pad registers will need
to be used. This has the downside of requiring saving and restoring
of those registers in the prolog and epilog of the slowpath function
slowing down the nominally faster pending bit and trylock code path
at the beginning of the slowpath function.
This patch separates out the actual MCS queuing code into a slowerpath
function. This avoids the slow down of the pending bit and trylock
code path at the expense of a little bit of additional overhead to
the MCS queuing code path.
Signed-off-by: Waiman Long <Waiman.Long@hp.com>
---
kernel/locking/qspinlock.c | 162 ++++++++++++++++++++++++-------------------
1 files changed, 90 insertions(+), 72 deletions(-)
diff --git a/kernel/locking/qspinlock.c b/kernel/locking/qspinlock.c
index 3723c83..93c663a 100644
--- a/kernel/locking/qspinlock.c
+++ b/kernel/locking/qspinlock.c
@@ -232,6 +232,93 @@ static __always_inline int try_set_locked(struct qspinlock *lock)
}
/**
+ * queue_spin_lock_slowerpath - a slower patch for acquiring queue spinlock
+ * @lock: Pointer to queue spinlock structure
+ * @node: Pointer to the queue node
+ * @tail: The tail code
+ *
+ * The reason for splitting a slowerpath from slowpath is to avoid the
+ * unnecessary overhead of non-scratch pad register pushing and popping
+ * due to increased complexity with unfair and PV spinlock from slowing
+ * down the nominally faster pending bit and trylock code path. So this
+ * function is not inlined.
+ */
+static noinline void queue_spin_lock_slowerpath(struct qspinlock *lock,
+ struct mcs_spinlock *node, u32 tail)
+{
+ struct mcs_spinlock *prev, *next;
+ u32 val, old;
+
+ /*
+ * we already touched the queueing cacheline; don't bother with pending
+ * stuff.
+ *
+ * p,*,* -> n,*,*
+ */
+ old = xchg_tail(lock, tail);
+
+ /*
+ * if there was a previous node; link it and wait.
+ */
+ if (old & _Q_TAIL_MASK) {
+ prev = decode_tail(old);
+ ACCESS_ONCE(prev->next) = node;
+
+ arch_mcs_spin_lock_contended(&node->locked);
+ }
+
+ /*
+ * we're at the head of the waitqueue, wait for the owner & pending to
+ * go away.
+ * Load-acquired is used here because the try_set_locked()
+ * function below may not be a full memory barrier.
+ *
+ * *,x,y -> *,0,0
+ */
+retry_queue_wait:
+ while ((val = smp_load_acquire(&lock->val.counter))
+ & _Q_LOCKED_PENDING_MASK)
+ arch_mutex_cpu_relax();
+
+ /*
+ * claim the lock:
+ *
+ * n,0,0 -> 0,0,1 : lock, uncontended
+ * *,0,0 -> *,0,1 : lock, contended
+ *
+ * If the queue head is the only one in the queue (lock value == tail),
+ * clear the tail code and grab the lock. Otherwise, we only need
+ * to grab the lock.
+ */
+ for (;;) {
+ if (val != tail) {
+ /*
+ * The try_set_locked function will only failed if the
+ * lock was stolen.
+ */
+ if (try_set_locked(lock))
+ break;
+ else
+ goto retry_queue_wait;
+ }
+ old = atomic_cmpxchg(&lock->val, val, _Q_LOCKED_VAL);
+ if (old == val)
+ return; /* No contention */
+ else if (old & _Q_LOCKED_MASK)
+ goto retry_queue_wait;
+ val = old;
+ }
+
+ /*
+ * contended path; wait for next
+ */
+ while (!(next = ACCESS_ONCE(node->next)))
+ arch_mutex_cpu_relax();
+
+ arch_mcs_spin_unlock_contended(&next->locked);
+}
+
+/**
* queue_spin_lock_slowpath - acquire the queue spinlock
* @lock: Pointer to queue spinlock structure
* @val: Current value of the queue spinlock 32-bit word
@@ -254,7 +341,7 @@ static __always_inline int try_set_locked(struct qspinlock *lock)
*/
void queue_spin_lock_slowpath(struct qspinlock *lock, u32 val)
{
- struct mcs_spinlock *prev, *next, *node;
+ struct mcs_spinlock *node;
u32 new, old, tail;
int idx;
@@ -355,78 +442,9 @@ queue:
* attempt the trylock once more in the hope someone let go while we
* weren't watching.
*/
- if (queue_spin_trylock(lock))
- goto release;
-
- /*
- * we already touched the queueing cacheline; don't bother with pending
- * stuff.
- *
- * p,*,* -> n,*,*
- */
- old = xchg_tail(lock, tail);
-
- /*
- * if there was a previous node; link it and wait.
- */
- if (old & _Q_TAIL_MASK) {
- prev = decode_tail(old);
- ACCESS_ONCE(prev->next) = node;
-
- arch_mcs_spin_lock_contended(&node->locked);
- }
-
- /*
- * we're at the head of the waitqueue, wait for the owner & pending to
- * go away.
- * Load-acquired is used here because the try_set_locked()
- * function below may not be a full memory barrier.
- *
- * *,x,y -> *,0,0
- */
-retry_queue_wait:
- while ((val = smp_load_acquire(&lock->val.counter))
- & _Q_LOCKED_PENDING_MASK)
- arch_mutex_cpu_relax();
-
- /*
- * claim the lock:
- *
- * n,0,0 -> 0,0,1 : lock, uncontended
- * *,0,0 -> *,0,1 : lock, contended
- *
- * If the queue head is the only one in the queue (lock value == tail),
- * clear the tail code and grab the lock. Otherwise, we only need
- * to grab the lock.
- */
- for (;;) {
- if (val != tail) {
- /*
- * The try_set_locked function will only failed if the
- * lock was stolen.
- */
- if (try_set_locked(lock))
- break;
- else
- goto retry_queue_wait;
- }
- old = atomic_cmpxchg(&lock->val, val, _Q_LOCKED_VAL);
- if (old == val)
- goto release; /* No contention */
- else if (old & _Q_LOCKED_MASK)
- goto retry_queue_wait;
- val = old;
- }
-
- /*
- * contended path; wait for next, release.
- */
- while (!(next = ACCESS_ONCE(node->next)))
- arch_mutex_cpu_relax();
-
- arch_mcs_spin_unlock_contended(&next->locked);
+ if (!queue_spin_trylock(lock))
+ queue_spin_lock_slowerpath(lock, node, tail);
-release:
/*
* release the node
*/
--
1.7.1
^ permalink raw reply related [flat|nested] 102+ messages in thread* [PATCH v11 10/16] qspinlock: Split the MCS queuing code into a separate slowerpath
@ 2014-05-30 15:43 ` Waiman Long
0 siblings, 0 replies; 102+ messages in thread
From: Waiman Long @ 2014-05-30 15:43 UTC (permalink / raw)
To: Thomas Gleixner, Ingo Molnar, H. Peter Anvin, Peter Zijlstra
Cc: linux-arch, Waiman Long, Rik van Riel, Raghavendra K T,
Gleb Natapov, kvm, Konrad Rzeszutek Wilk, Scott J Norton, x86,
Paolo Bonzini, linux-kernel, virtualization, Chegu Vinod,
David Vrabel, Oleg Nesterov, xen-devel, Boris Ostrovsky,
Paul E. McKenney, Linus Torvalds
With the pending addition of more codes to support PV spinlock, the
complexity of the slowpath function increases to the point that the
number of scratch-pad registers in the x86-64 architecture is not
enough and so those additional non-scratch-pad registers will need
to be used. This has the downside of requiring saving and restoring
of those registers in the prolog and epilog of the slowpath function
slowing down the nominally faster pending bit and trylock code path
at the beginning of the slowpath function.
This patch separates out the actual MCS queuing code into a slowerpath
function. This avoids the slow down of the pending bit and trylock
code path at the expense of a little bit of additional overhead to
the MCS queuing code path.
Signed-off-by: Waiman Long <Waiman.Long@hp.com>
---
kernel/locking/qspinlock.c | 162 ++++++++++++++++++++++++-------------------
1 files changed, 90 insertions(+), 72 deletions(-)
diff --git a/kernel/locking/qspinlock.c b/kernel/locking/qspinlock.c
index 3723c83..93c663a 100644
--- a/kernel/locking/qspinlock.c
+++ b/kernel/locking/qspinlock.c
@@ -232,6 +232,93 @@ static __always_inline int try_set_locked(struct qspinlock *lock)
}
/**
+ * queue_spin_lock_slowerpath - a slower patch for acquiring queue spinlock
+ * @lock: Pointer to queue spinlock structure
+ * @node: Pointer to the queue node
+ * @tail: The tail code
+ *
+ * The reason for splitting a slowerpath from slowpath is to avoid the
+ * unnecessary overhead of non-scratch pad register pushing and popping
+ * due to increased complexity with unfair and PV spinlock from slowing
+ * down the nominally faster pending bit and trylock code path. So this
+ * function is not inlined.
+ */
+static noinline void queue_spin_lock_slowerpath(struct qspinlock *lock,
+ struct mcs_spinlock *node, u32 tail)
+{
+ struct mcs_spinlock *prev, *next;
+ u32 val, old;
+
+ /*
+ * we already touched the queueing cacheline; don't bother with pending
+ * stuff.
+ *
+ * p,*,* -> n,*,*
+ */
+ old = xchg_tail(lock, tail);
+
+ /*
+ * if there was a previous node; link it and wait.
+ */
+ if (old & _Q_TAIL_MASK) {
+ prev = decode_tail(old);
+ ACCESS_ONCE(prev->next) = node;
+
+ arch_mcs_spin_lock_contended(&node->locked);
+ }
+
+ /*
+ * we're at the head of the waitqueue, wait for the owner & pending to
+ * go away.
+ * Load-acquired is used here because the try_set_locked()
+ * function below may not be a full memory barrier.
+ *
+ * *,x,y -> *,0,0
+ */
+retry_queue_wait:
+ while ((val = smp_load_acquire(&lock->val.counter))
+ & _Q_LOCKED_PENDING_MASK)
+ arch_mutex_cpu_relax();
+
+ /*
+ * claim the lock:
+ *
+ * n,0,0 -> 0,0,1 : lock, uncontended
+ * *,0,0 -> *,0,1 : lock, contended
+ *
+ * If the queue head is the only one in the queue (lock value == tail),
+ * clear the tail code and grab the lock. Otherwise, we only need
+ * to grab the lock.
+ */
+ for (;;) {
+ if (val != tail) {
+ /*
+ * The try_set_locked function will only failed if the
+ * lock was stolen.
+ */
+ if (try_set_locked(lock))
+ break;
+ else
+ goto retry_queue_wait;
+ }
+ old = atomic_cmpxchg(&lock->val, val, _Q_LOCKED_VAL);
+ if (old == val)
+ return; /* No contention */
+ else if (old & _Q_LOCKED_MASK)
+ goto retry_queue_wait;
+ val = old;
+ }
+
+ /*
+ * contended path; wait for next
+ */
+ while (!(next = ACCESS_ONCE(node->next)))
+ arch_mutex_cpu_relax();
+
+ arch_mcs_spin_unlock_contended(&next->locked);
+}
+
+/**
* queue_spin_lock_slowpath - acquire the queue spinlock
* @lock: Pointer to queue spinlock structure
* @val: Current value of the queue spinlock 32-bit word
@@ -254,7 +341,7 @@ static __always_inline int try_set_locked(struct qspinlock *lock)
*/
void queue_spin_lock_slowpath(struct qspinlock *lock, u32 val)
{
- struct mcs_spinlock *prev, *next, *node;
+ struct mcs_spinlock *node;
u32 new, old, tail;
int idx;
@@ -355,78 +442,9 @@ queue:
* attempt the trylock once more in the hope someone let go while we
* weren't watching.
*/
- if (queue_spin_trylock(lock))
- goto release;
-
- /*
- * we already touched the queueing cacheline; don't bother with pending
- * stuff.
- *
- * p,*,* -> n,*,*
- */
- old = xchg_tail(lock, tail);
-
- /*
- * if there was a previous node; link it and wait.
- */
- if (old & _Q_TAIL_MASK) {
- prev = decode_tail(old);
- ACCESS_ONCE(prev->next) = node;
-
- arch_mcs_spin_lock_contended(&node->locked);
- }
-
- /*
- * we're at the head of the waitqueue, wait for the owner & pending to
- * go away.
- * Load-acquired is used here because the try_set_locked()
- * function below may not be a full memory barrier.
- *
- * *,x,y -> *,0,0
- */
-retry_queue_wait:
- while ((val = smp_load_acquire(&lock->val.counter))
- & _Q_LOCKED_PENDING_MASK)
- arch_mutex_cpu_relax();
-
- /*
- * claim the lock:
- *
- * n,0,0 -> 0,0,1 : lock, uncontended
- * *,0,0 -> *,0,1 : lock, contended
- *
- * If the queue head is the only one in the queue (lock value == tail),
- * clear the tail code and grab the lock. Otherwise, we only need
- * to grab the lock.
- */
- for (;;) {
- if (val != tail) {
- /*
- * The try_set_locked function will only failed if the
- * lock was stolen.
- */
- if (try_set_locked(lock))
- break;
- else
- goto retry_queue_wait;
- }
- old = atomic_cmpxchg(&lock->val, val, _Q_LOCKED_VAL);
- if (old == val)
- goto release; /* No contention */
- else if (old & _Q_LOCKED_MASK)
- goto retry_queue_wait;
- val = old;
- }
-
- /*
- * contended path; wait for next, release.
- */
- while (!(next = ACCESS_ONCE(node->next)))
- arch_mutex_cpu_relax();
-
- arch_mcs_spin_unlock_contended(&next->locked);
+ if (!queue_spin_trylock(lock))
+ queue_spin_lock_slowerpath(lock, node, tail);
-release:
/*
* release the node
*/
--
1.7.1
^ permalink raw reply related [flat|nested] 102+ messages in thread
* [PATCH v11 10/16] qspinlock: Split the MCS queuing code into a separate slowerpath
2014-05-30 15:43 [PATCH v11 00/16] qspinlock: a 4-byte queue spinlock with PV support Waiman Long
` (20 preceding siblings ...)
2014-05-30 15:43 ` Waiman Long
@ 2014-05-30 15:43 ` Waiman Long
2014-05-30 15:43 ` Waiman Long
` (15 subsequent siblings)
37 siblings, 0 replies; 102+ messages in thread
From: Waiman Long @ 2014-05-30 15:43 UTC (permalink / raw)
To: Thomas Gleixner, Ingo Molnar, H. Peter Anvin, Peter Zijlstra
Cc: linux-arch, Waiman Long, Raghavendra K T, Gleb Natapov, kvm,
Scott J Norton, x86, Paolo Bonzini, linux-kernel, virtualization,
Chegu Vinod, David Vrabel, Oleg Nesterov, xen-devel,
Boris Ostrovsky, Paul E. McKenney, Linus Torvalds
With the pending addition of more codes to support PV spinlock, the
complexity of the slowpath function increases to the point that the
number of scratch-pad registers in the x86-64 architecture is not
enough and so those additional non-scratch-pad registers will need
to be used. This has the downside of requiring saving and restoring
of those registers in the prolog and epilog of the slowpath function
slowing down the nominally faster pending bit and trylock code path
at the beginning of the slowpath function.
This patch separates out the actual MCS queuing code into a slowerpath
function. This avoids the slow down of the pending bit and trylock
code path at the expense of a little bit of additional overhead to
the MCS queuing code path.
Signed-off-by: Waiman Long <Waiman.Long@hp.com>
---
kernel/locking/qspinlock.c | 162 ++++++++++++++++++++++++-------------------
1 files changed, 90 insertions(+), 72 deletions(-)
diff --git a/kernel/locking/qspinlock.c b/kernel/locking/qspinlock.c
index 3723c83..93c663a 100644
--- a/kernel/locking/qspinlock.c
+++ b/kernel/locking/qspinlock.c
@@ -232,6 +232,93 @@ static __always_inline int try_set_locked(struct qspinlock *lock)
}
/**
+ * queue_spin_lock_slowerpath - a slower patch for acquiring queue spinlock
+ * @lock: Pointer to queue spinlock structure
+ * @node: Pointer to the queue node
+ * @tail: The tail code
+ *
+ * The reason for splitting a slowerpath from slowpath is to avoid the
+ * unnecessary overhead of non-scratch pad register pushing and popping
+ * due to increased complexity with unfair and PV spinlock from slowing
+ * down the nominally faster pending bit and trylock code path. So this
+ * function is not inlined.
+ */
+static noinline void queue_spin_lock_slowerpath(struct qspinlock *lock,
+ struct mcs_spinlock *node, u32 tail)
+{
+ struct mcs_spinlock *prev, *next;
+ u32 val, old;
+
+ /*
+ * we already touched the queueing cacheline; don't bother with pending
+ * stuff.
+ *
+ * p,*,* -> n,*,*
+ */
+ old = xchg_tail(lock, tail);
+
+ /*
+ * if there was a previous node; link it and wait.
+ */
+ if (old & _Q_TAIL_MASK) {
+ prev = decode_tail(old);
+ ACCESS_ONCE(prev->next) = node;
+
+ arch_mcs_spin_lock_contended(&node->locked);
+ }
+
+ /*
+ * we're at the head of the waitqueue, wait for the owner & pending to
+ * go away.
+ * Load-acquired is used here because the try_set_locked()
+ * function below may not be a full memory barrier.
+ *
+ * *,x,y -> *,0,0
+ */
+retry_queue_wait:
+ while ((val = smp_load_acquire(&lock->val.counter))
+ & _Q_LOCKED_PENDING_MASK)
+ arch_mutex_cpu_relax();
+
+ /*
+ * claim the lock:
+ *
+ * n,0,0 -> 0,0,1 : lock, uncontended
+ * *,0,0 -> *,0,1 : lock, contended
+ *
+ * If the queue head is the only one in the queue (lock value == tail),
+ * clear the tail code and grab the lock. Otherwise, we only need
+ * to grab the lock.
+ */
+ for (;;) {
+ if (val != tail) {
+ /*
+ * The try_set_locked function will only failed if the
+ * lock was stolen.
+ */
+ if (try_set_locked(lock))
+ break;
+ else
+ goto retry_queue_wait;
+ }
+ old = atomic_cmpxchg(&lock->val, val, _Q_LOCKED_VAL);
+ if (old == val)
+ return; /* No contention */
+ else if (old & _Q_LOCKED_MASK)
+ goto retry_queue_wait;
+ val = old;
+ }
+
+ /*
+ * contended path; wait for next
+ */
+ while (!(next = ACCESS_ONCE(node->next)))
+ arch_mutex_cpu_relax();
+
+ arch_mcs_spin_unlock_contended(&next->locked);
+}
+
+/**
* queue_spin_lock_slowpath - acquire the queue spinlock
* @lock: Pointer to queue spinlock structure
* @val: Current value of the queue spinlock 32-bit word
@@ -254,7 +341,7 @@ static __always_inline int try_set_locked(struct qspinlock *lock)
*/
void queue_spin_lock_slowpath(struct qspinlock *lock, u32 val)
{
- struct mcs_spinlock *prev, *next, *node;
+ struct mcs_spinlock *node;
u32 new, old, tail;
int idx;
@@ -355,78 +442,9 @@ queue:
* attempt the trylock once more in the hope someone let go while we
* weren't watching.
*/
- if (queue_spin_trylock(lock))
- goto release;
-
- /*
- * we already touched the queueing cacheline; don't bother with pending
- * stuff.
- *
- * p,*,* -> n,*,*
- */
- old = xchg_tail(lock, tail);
-
- /*
- * if there was a previous node; link it and wait.
- */
- if (old & _Q_TAIL_MASK) {
- prev = decode_tail(old);
- ACCESS_ONCE(prev->next) = node;
-
- arch_mcs_spin_lock_contended(&node->locked);
- }
-
- /*
- * we're at the head of the waitqueue, wait for the owner & pending to
- * go away.
- * Load-acquired is used here because the try_set_locked()
- * function below may not be a full memory barrier.
- *
- * *,x,y -> *,0,0
- */
-retry_queue_wait:
- while ((val = smp_load_acquire(&lock->val.counter))
- & _Q_LOCKED_PENDING_MASK)
- arch_mutex_cpu_relax();
-
- /*
- * claim the lock:
- *
- * n,0,0 -> 0,0,1 : lock, uncontended
- * *,0,0 -> *,0,1 : lock, contended
- *
- * If the queue head is the only one in the queue (lock value == tail),
- * clear the tail code and grab the lock. Otherwise, we only need
- * to grab the lock.
- */
- for (;;) {
- if (val != tail) {
- /*
- * The try_set_locked function will only failed if the
- * lock was stolen.
- */
- if (try_set_locked(lock))
- break;
- else
- goto retry_queue_wait;
- }
- old = atomic_cmpxchg(&lock->val, val, _Q_LOCKED_VAL);
- if (old == val)
- goto release; /* No contention */
- else if (old & _Q_LOCKED_MASK)
- goto retry_queue_wait;
- val = old;
- }
-
- /*
- * contended path; wait for next, release.
- */
- while (!(next = ACCESS_ONCE(node->next)))
- arch_mutex_cpu_relax();
-
- arch_mcs_spin_unlock_contended(&next->locked);
+ if (!queue_spin_trylock(lock))
+ queue_spin_lock_slowerpath(lock, node, tail);
-release:
/*
* release the node
*/
--
1.7.1
^ permalink raw reply related [flat|nested] 102+ messages in thread* [PATCH v11 11/16] pvqspinlock, x86: Rename paravirt_ticketlocks_enabled
2014-05-30 15:43 [PATCH v11 00/16] qspinlock: a 4-byte queue spinlock with PV support Waiman Long
@ 2014-05-30 15:43 ` Waiman Long
2014-05-30 15:43 ` Waiman Long
` (36 subsequent siblings)
37 siblings, 0 replies; 102+ messages in thread
From: Waiman Long @ 2014-05-30 15:43 UTC (permalink / raw)
To: Thomas Gleixner, Ingo Molnar, H. Peter Anvin, Peter Zijlstra
Cc: linux-arch, x86, linux-kernel, virtualization, xen-devel, kvm,
Paolo Bonzini, Konrad Rzeszutek Wilk, Boris Ostrovsky,
Paul E. McKenney, Rik van Riel, Linus Torvalds, Raghavendra K T,
David Vrabel, Oleg Nesterov, Gleb Natapov, Scott J Norton,
Chegu Vinod, Waiman Long
This patch renames the paravirt_ticketlocks_enabled static key to a
more generic paravirt_spinlocks_enabled name.
Signed-off-by: Waiman Long <Waiman.Long@hp.com>
---
arch/x86/include/asm/spinlock.h | 4 ++--
arch/x86/kernel/kvm.c | 2 +-
arch/x86/kernel/paravirt-spinlocks.c | 4 ++--
arch/x86/xen/spinlock.c | 2 +-
4 files changed, 6 insertions(+), 6 deletions(-)
diff --git a/arch/x86/include/asm/spinlock.h b/arch/x86/include/asm/spinlock.h
index 958d20f..428d0d1 100644
--- a/arch/x86/include/asm/spinlock.h
+++ b/arch/x86/include/asm/spinlock.h
@@ -39,7 +39,7 @@
/* How long a lock should spin before we consider blocking */
#define SPIN_THRESHOLD (1 << 15)
-extern struct static_key paravirt_ticketlocks_enabled;
+extern struct static_key paravirt_spinlocks_enabled;
static __always_inline bool static_key_false(struct static_key *key);
#ifdef CONFIG_QUEUE_SPINLOCK
@@ -150,7 +150,7 @@ static inline void __ticket_unlock_slowpath(arch_spinlock_t *lock,
static __always_inline void arch_spin_unlock(arch_spinlock_t *lock)
{
if (TICKET_SLOWPATH_FLAG &&
- static_key_false(¶virt_ticketlocks_enabled)) {
+ static_key_false(¶virt_spinlocks_enabled)) {
arch_spinlock_t prev;
prev = *lock;
diff --git a/arch/x86/kernel/kvm.c b/arch/x86/kernel/kvm.c
index 0331cb3..7ab8ab3 100644
--- a/arch/x86/kernel/kvm.c
+++ b/arch/x86/kernel/kvm.c
@@ -817,7 +817,7 @@ static __init int kvm_spinlock_init_jump(void)
if (!kvm_para_has_feature(KVM_FEATURE_PV_UNHALT))
return 0;
- static_key_slow_inc(¶virt_ticketlocks_enabled);
+ static_key_slow_inc(¶virt_spinlocks_enabled);
printk(KERN_INFO "KVM setup paravirtual spinlock\n");
return 0;
diff --git a/arch/x86/kernel/paravirt-spinlocks.c b/arch/x86/kernel/paravirt-spinlocks.c
index 69ed806..d9041c9 100644
--- a/arch/x86/kernel/paravirt-spinlocks.c
+++ b/arch/x86/kernel/paravirt-spinlocks.c
@@ -17,8 +17,8 @@ struct pv_lock_ops pv_lock_ops = {
};
EXPORT_SYMBOL(pv_lock_ops);
-struct static_key paravirt_ticketlocks_enabled = STATIC_KEY_INIT_FALSE;
-EXPORT_SYMBOL(paravirt_ticketlocks_enabled);
+struct static_key paravirt_spinlocks_enabled = STATIC_KEY_INIT_FALSE;
+EXPORT_SYMBOL(paravirt_spinlocks_enabled);
#endif
#ifdef CONFIG_VIRT_UNFAIR_LOCKS
diff --git a/arch/x86/xen/spinlock.c b/arch/x86/xen/spinlock.c
index 0ba5f3b..d1b6a32 100644
--- a/arch/x86/xen/spinlock.c
+++ b/arch/x86/xen/spinlock.c
@@ -293,7 +293,7 @@ static __init int xen_init_spinlocks_jump(void)
if (!xen_domain())
return 0;
- static_key_slow_inc(¶virt_ticketlocks_enabled);
+ static_key_slow_inc(¶virt_spinlocks_enabled);
return 0;
}
early_initcall(xen_init_spinlocks_jump);
--
1.7.1
^ permalink raw reply related [flat|nested] 102+ messages in thread* [PATCH v11 11/16] pvqspinlock, x86: Rename paravirt_ticketlocks_enabled
@ 2014-05-30 15:43 ` Waiman Long
0 siblings, 0 replies; 102+ messages in thread
From: Waiman Long @ 2014-05-30 15:43 UTC (permalink / raw)
To: Thomas Gleixner, Ingo Molnar, H. Peter Anvin, Peter Zijlstra
Cc: linux-arch, Waiman Long, Rik van Riel, Raghavendra K T,
Gleb Natapov, kvm, Konrad Rzeszutek Wilk, Scott J Norton, x86,
Paolo Bonzini, linux-kernel, virtualization, Chegu Vinod,
David Vrabel, Oleg Nesterov, xen-devel, Boris Ostrovsky,
Paul E. McKenney, Linus Torvalds
This patch renames the paravirt_ticketlocks_enabled static key to a
more generic paravirt_spinlocks_enabled name.
Signed-off-by: Waiman Long <Waiman.Long@hp.com>
---
arch/x86/include/asm/spinlock.h | 4 ++--
arch/x86/kernel/kvm.c | 2 +-
arch/x86/kernel/paravirt-spinlocks.c | 4 ++--
arch/x86/xen/spinlock.c | 2 +-
4 files changed, 6 insertions(+), 6 deletions(-)
diff --git a/arch/x86/include/asm/spinlock.h b/arch/x86/include/asm/spinlock.h
index 958d20f..428d0d1 100644
--- a/arch/x86/include/asm/spinlock.h
+++ b/arch/x86/include/asm/spinlock.h
@@ -39,7 +39,7 @@
/* How long a lock should spin before we consider blocking */
#define SPIN_THRESHOLD (1 << 15)
-extern struct static_key paravirt_ticketlocks_enabled;
+extern struct static_key paravirt_spinlocks_enabled;
static __always_inline bool static_key_false(struct static_key *key);
#ifdef CONFIG_QUEUE_SPINLOCK
@@ -150,7 +150,7 @@ static inline void __ticket_unlock_slowpath(arch_spinlock_t *lock,
static __always_inline void arch_spin_unlock(arch_spinlock_t *lock)
{
if (TICKET_SLOWPATH_FLAG &&
- static_key_false(¶virt_ticketlocks_enabled)) {
+ static_key_false(¶virt_spinlocks_enabled)) {
arch_spinlock_t prev;
prev = *lock;
diff --git a/arch/x86/kernel/kvm.c b/arch/x86/kernel/kvm.c
index 0331cb3..7ab8ab3 100644
--- a/arch/x86/kernel/kvm.c
+++ b/arch/x86/kernel/kvm.c
@@ -817,7 +817,7 @@ static __init int kvm_spinlock_init_jump(void)
if (!kvm_para_has_feature(KVM_FEATURE_PV_UNHALT))
return 0;
- static_key_slow_inc(¶virt_ticketlocks_enabled);
+ static_key_slow_inc(¶virt_spinlocks_enabled);
printk(KERN_INFO "KVM setup paravirtual spinlock\n");
return 0;
diff --git a/arch/x86/kernel/paravirt-spinlocks.c b/arch/x86/kernel/paravirt-spinlocks.c
index 69ed806..d9041c9 100644
--- a/arch/x86/kernel/paravirt-spinlocks.c
+++ b/arch/x86/kernel/paravirt-spinlocks.c
@@ -17,8 +17,8 @@ struct pv_lock_ops pv_lock_ops = {
};
EXPORT_SYMBOL(pv_lock_ops);
-struct static_key paravirt_ticketlocks_enabled = STATIC_KEY_INIT_FALSE;
-EXPORT_SYMBOL(paravirt_ticketlocks_enabled);
+struct static_key paravirt_spinlocks_enabled = STATIC_KEY_INIT_FALSE;
+EXPORT_SYMBOL(paravirt_spinlocks_enabled);
#endif
#ifdef CONFIG_VIRT_UNFAIR_LOCKS
diff --git a/arch/x86/xen/spinlock.c b/arch/x86/xen/spinlock.c
index 0ba5f3b..d1b6a32 100644
--- a/arch/x86/xen/spinlock.c
+++ b/arch/x86/xen/spinlock.c
@@ -293,7 +293,7 @@ static __init int xen_init_spinlocks_jump(void)
if (!xen_domain())
return 0;
- static_key_slow_inc(¶virt_ticketlocks_enabled);
+ static_key_slow_inc(¶virt_spinlocks_enabled);
return 0;
}
early_initcall(xen_init_spinlocks_jump);
--
1.7.1
^ permalink raw reply related [flat|nested] 102+ messages in thread
* [PATCH v11 11/16] pvqspinlock, x86: Rename paravirt_ticketlocks_enabled
2014-05-30 15:43 [PATCH v11 00/16] qspinlock: a 4-byte queue spinlock with PV support Waiman Long
` (22 preceding siblings ...)
2014-05-30 15:43 ` Waiman Long
@ 2014-05-30 15:43 ` Waiman Long
2014-05-30 15:43 ` [PATCH v11 12/16] pvqspinlock, x86: Add PV data structure & methods Waiman Long
` (13 subsequent siblings)
37 siblings, 0 replies; 102+ messages in thread
From: Waiman Long @ 2014-05-30 15:43 UTC (permalink / raw)
To: Thomas Gleixner, Ingo Molnar, H. Peter Anvin, Peter Zijlstra
Cc: linux-arch, Waiman Long, Raghavendra K T, Gleb Natapov, kvm,
Scott J Norton, x86, Paolo Bonzini, linux-kernel, virtualization,
Chegu Vinod, David Vrabel, Oleg Nesterov, xen-devel,
Boris Ostrovsky, Paul E. McKenney, Linus Torvalds
This patch renames the paravirt_ticketlocks_enabled static key to a
more generic paravirt_spinlocks_enabled name.
Signed-off-by: Waiman Long <Waiman.Long@hp.com>
---
arch/x86/include/asm/spinlock.h | 4 ++--
arch/x86/kernel/kvm.c | 2 +-
arch/x86/kernel/paravirt-spinlocks.c | 4 ++--
arch/x86/xen/spinlock.c | 2 +-
4 files changed, 6 insertions(+), 6 deletions(-)
diff --git a/arch/x86/include/asm/spinlock.h b/arch/x86/include/asm/spinlock.h
index 958d20f..428d0d1 100644
--- a/arch/x86/include/asm/spinlock.h
+++ b/arch/x86/include/asm/spinlock.h
@@ -39,7 +39,7 @@
/* How long a lock should spin before we consider blocking */
#define SPIN_THRESHOLD (1 << 15)
-extern struct static_key paravirt_ticketlocks_enabled;
+extern struct static_key paravirt_spinlocks_enabled;
static __always_inline bool static_key_false(struct static_key *key);
#ifdef CONFIG_QUEUE_SPINLOCK
@@ -150,7 +150,7 @@ static inline void __ticket_unlock_slowpath(arch_spinlock_t *lock,
static __always_inline void arch_spin_unlock(arch_spinlock_t *lock)
{
if (TICKET_SLOWPATH_FLAG &&
- static_key_false(¶virt_ticketlocks_enabled)) {
+ static_key_false(¶virt_spinlocks_enabled)) {
arch_spinlock_t prev;
prev = *lock;
diff --git a/arch/x86/kernel/kvm.c b/arch/x86/kernel/kvm.c
index 0331cb3..7ab8ab3 100644
--- a/arch/x86/kernel/kvm.c
+++ b/arch/x86/kernel/kvm.c
@@ -817,7 +817,7 @@ static __init int kvm_spinlock_init_jump(void)
if (!kvm_para_has_feature(KVM_FEATURE_PV_UNHALT))
return 0;
- static_key_slow_inc(¶virt_ticketlocks_enabled);
+ static_key_slow_inc(¶virt_spinlocks_enabled);
printk(KERN_INFO "KVM setup paravirtual spinlock\n");
return 0;
diff --git a/arch/x86/kernel/paravirt-spinlocks.c b/arch/x86/kernel/paravirt-spinlocks.c
index 69ed806..d9041c9 100644
--- a/arch/x86/kernel/paravirt-spinlocks.c
+++ b/arch/x86/kernel/paravirt-spinlocks.c
@@ -17,8 +17,8 @@ struct pv_lock_ops pv_lock_ops = {
};
EXPORT_SYMBOL(pv_lock_ops);
-struct static_key paravirt_ticketlocks_enabled = STATIC_KEY_INIT_FALSE;
-EXPORT_SYMBOL(paravirt_ticketlocks_enabled);
+struct static_key paravirt_spinlocks_enabled = STATIC_KEY_INIT_FALSE;
+EXPORT_SYMBOL(paravirt_spinlocks_enabled);
#endif
#ifdef CONFIG_VIRT_UNFAIR_LOCKS
diff --git a/arch/x86/xen/spinlock.c b/arch/x86/xen/spinlock.c
index 0ba5f3b..d1b6a32 100644
--- a/arch/x86/xen/spinlock.c
+++ b/arch/x86/xen/spinlock.c
@@ -293,7 +293,7 @@ static __init int xen_init_spinlocks_jump(void)
if (!xen_domain())
return 0;
- static_key_slow_inc(¶virt_ticketlocks_enabled);
+ static_key_slow_inc(¶virt_spinlocks_enabled);
return 0;
}
early_initcall(xen_init_spinlocks_jump);
--
1.7.1
^ permalink raw reply related [flat|nested] 102+ messages in thread* [PATCH v11 12/16] pvqspinlock, x86: Add PV data structure & methods
2014-05-30 15:43 [PATCH v11 00/16] qspinlock: a 4-byte queue spinlock with PV support Waiman Long
` (23 preceding siblings ...)
2014-05-30 15:43 ` Waiman Long
@ 2014-05-30 15:43 ` Waiman Long
2014-05-30 15:43 ` Waiman Long
` (12 subsequent siblings)
37 siblings, 0 replies; 102+ messages in thread
From: Waiman Long @ 2014-05-30 15:43 UTC (permalink / raw)
To: Thomas Gleixner, Ingo Molnar, H. Peter Anvin, Peter Zijlstra
Cc: linux-arch, x86, linux-kernel, virtualization, xen-devel, kvm,
Paolo Bonzini, Konrad Rzeszutek Wilk, Boris Ostrovsky,
Paul E. McKenney, Rik van Riel, Linus Torvalds, Raghavendra K T,
David Vrabel, Oleg Nesterov, Gleb Natapov, Scott J Norton,
Chegu Vinod, Waiman Long
This patch modifies the para-virtualization (PV) infrastructure code
of the x86-64 architecture to support the PV queue spinlock. Three
new virtual methods are added to support PV qspinlock:
1) kick_cpu - schedule in a virtual CPU
2) halt_cpu - schedule out a virtual CPU
3) lockstat - update statistical data for debugfs
Signed-off-by: Waiman Long <Waiman.Long@hp.com>
---
arch/x86/include/asm/paravirt.h | 18 +++++++++++++++++-
arch/x86/include/asm/paravirt_types.h | 17 +++++++++++++++++
arch/x86/kernel/paravirt-spinlocks.c | 6 ++++++
3 files changed, 40 insertions(+), 1 deletions(-)
diff --git a/arch/x86/include/asm/paravirt.h b/arch/x86/include/asm/paravirt.h
index cd6e161..d71e123 100644
--- a/arch/x86/include/asm/paravirt.h
+++ b/arch/x86/include/asm/paravirt.h
@@ -711,7 +711,23 @@ static inline void __set_fixmap(unsigned /* enum fixed_addresses */ idx,
}
#if defined(CONFIG_SMP) && defined(CONFIG_PARAVIRT_SPINLOCKS)
+#ifdef CONFIG_QUEUE_SPINLOCK
+static __always_inline void __queue_kick_cpu(int cpu)
+{
+ PVOP_VCALL1(pv_lock_ops.kick_cpu, cpu);
+}
+
+static __always_inline void
+__queue_halt_cpu(enum pv_lock_stats type, s8 *state, s8 sval)
+{
+ PVOP_VCALL3(pv_lock_ops.halt_cpu, type, state, sval);
+}
+static __always_inline void __queue_lockstat(enum pv_lock_stats type)
+{
+ PVOP_VCALL1(pv_lock_ops.lockstat, type);
+}
+#else
static __always_inline void __ticket_lock_spinning(struct arch_spinlock *lock,
__ticket_t ticket)
{
@@ -723,7 +739,7 @@ static __always_inline void __ticket_unlock_kick(struct arch_spinlock *lock,
{
PVOP_VCALL2(pv_lock_ops.unlock_kick, lock, ticket);
}
-
+#endif
#endif
#ifdef CONFIG_X86_32
diff --git a/arch/x86/include/asm/paravirt_types.h b/arch/x86/include/asm/paravirt_types.h
index 7549b8b..549b3a0 100644
--- a/arch/x86/include/asm/paravirt_types.h
+++ b/arch/x86/include/asm/paravirt_types.h
@@ -333,9 +333,26 @@ struct arch_spinlock;
typedef u16 __ticket_t;
#endif
+#ifdef CONFIG_QUEUE_SPINLOCK
+enum pv_lock_stats {
+ PV_HALT_QHEAD, /* Queue head halting */
+ PV_HALT_QNODE, /* Other queue node halting */
+ PV_HALT_ABORT, /* Halting aborted */
+ PV_WAKE_KICKED, /* Wakeup by kicking */
+ PV_WAKE_SPURIOUS, /* Spurious wakeup */
+ PV_KICK_NOHALT /* Kick but CPU not halted */
+};
+#endif
+
struct pv_lock_ops {
+#ifdef CONFIG_QUEUE_SPINLOCK
+ void (*kick_cpu)(int cpu);
+ void (*halt_cpu)(enum pv_lock_stats type, s8 *state, s8 sval);
+ void (*lockstat)(enum pv_lock_stats type);
+#else
struct paravirt_callee_save lock_spinning;
void (*unlock_kick)(struct arch_spinlock *lock, __ticket_t ticket);
+#endif
};
/* This contains all the paravirt structures: we get a convenient
diff --git a/arch/x86/kernel/paravirt-spinlocks.c b/arch/x86/kernel/paravirt-spinlocks.c
index d9041c9..17435b7 100644
--- a/arch/x86/kernel/paravirt-spinlocks.c
+++ b/arch/x86/kernel/paravirt-spinlocks.c
@@ -11,9 +11,15 @@
#ifdef CONFIG_PARAVIRT_SPINLOCKS
struct pv_lock_ops pv_lock_ops = {
#ifdef CONFIG_SMP
+#ifdef CONFIG_QUEUE_SPINLOCK
+ .kick_cpu = paravirt_nop,
+ .halt_cpu = paravirt_nop,
+ .lockstat = paravirt_nop,
+#else
.lock_spinning = __PV_IS_CALLEE_SAVE(paravirt_nop),
.unlock_kick = paravirt_nop,
#endif
+#endif
};
EXPORT_SYMBOL(pv_lock_ops);
--
1.7.1
^ permalink raw reply related [flat|nested] 102+ messages in thread* [PATCH v11 12/16] pvqspinlock, x86: Add PV data structure & methods
2014-05-30 15:43 [PATCH v11 00/16] qspinlock: a 4-byte queue spinlock with PV support Waiman Long
` (24 preceding siblings ...)
2014-05-30 15:43 ` [PATCH v11 12/16] pvqspinlock, x86: Add PV data structure & methods Waiman Long
@ 2014-05-30 15:43 ` Waiman Long
2014-05-30 15:43 ` Waiman Long
` (11 subsequent siblings)
37 siblings, 0 replies; 102+ messages in thread
From: Waiman Long @ 2014-05-30 15:43 UTC (permalink / raw)
To: Thomas Gleixner, Ingo Molnar, H. Peter Anvin, Peter Zijlstra
Cc: linux-arch, Waiman Long, Rik van Riel, Raghavendra K T,
Gleb Natapov, kvm, Konrad Rzeszutek Wilk, Scott J Norton, x86,
Paolo Bonzini, linux-kernel, virtualization, Chegu Vinod,
David Vrabel, Oleg Nesterov, xen-devel, Boris Ostrovsky,
Paul E. McKenney, Linus Torvalds
This patch modifies the para-virtualization (PV) infrastructure code
of the x86-64 architecture to support the PV queue spinlock. Three
new virtual methods are added to support PV qspinlock:
1) kick_cpu - schedule in a virtual CPU
2) halt_cpu - schedule out a virtual CPU
3) lockstat - update statistical data for debugfs
Signed-off-by: Waiman Long <Waiman.Long@hp.com>
---
arch/x86/include/asm/paravirt.h | 18 +++++++++++++++++-
arch/x86/include/asm/paravirt_types.h | 17 +++++++++++++++++
arch/x86/kernel/paravirt-spinlocks.c | 6 ++++++
3 files changed, 40 insertions(+), 1 deletions(-)
diff --git a/arch/x86/include/asm/paravirt.h b/arch/x86/include/asm/paravirt.h
index cd6e161..d71e123 100644
--- a/arch/x86/include/asm/paravirt.h
+++ b/arch/x86/include/asm/paravirt.h
@@ -711,7 +711,23 @@ static inline void __set_fixmap(unsigned /* enum fixed_addresses */ idx,
}
#if defined(CONFIG_SMP) && defined(CONFIG_PARAVIRT_SPINLOCKS)
+#ifdef CONFIG_QUEUE_SPINLOCK
+static __always_inline void __queue_kick_cpu(int cpu)
+{
+ PVOP_VCALL1(pv_lock_ops.kick_cpu, cpu);
+}
+
+static __always_inline void
+__queue_halt_cpu(enum pv_lock_stats type, s8 *state, s8 sval)
+{
+ PVOP_VCALL3(pv_lock_ops.halt_cpu, type, state, sval);
+}
+static __always_inline void __queue_lockstat(enum pv_lock_stats type)
+{
+ PVOP_VCALL1(pv_lock_ops.lockstat, type);
+}
+#else
static __always_inline void __ticket_lock_spinning(struct arch_spinlock *lock,
__ticket_t ticket)
{
@@ -723,7 +739,7 @@ static __always_inline void __ticket_unlock_kick(struct arch_spinlock *lock,
{
PVOP_VCALL2(pv_lock_ops.unlock_kick, lock, ticket);
}
-
+#endif
#endif
#ifdef CONFIG_X86_32
diff --git a/arch/x86/include/asm/paravirt_types.h b/arch/x86/include/asm/paravirt_types.h
index 7549b8b..549b3a0 100644
--- a/arch/x86/include/asm/paravirt_types.h
+++ b/arch/x86/include/asm/paravirt_types.h
@@ -333,9 +333,26 @@ struct arch_spinlock;
typedef u16 __ticket_t;
#endif
+#ifdef CONFIG_QUEUE_SPINLOCK
+enum pv_lock_stats {
+ PV_HALT_QHEAD, /* Queue head halting */
+ PV_HALT_QNODE, /* Other queue node halting */
+ PV_HALT_ABORT, /* Halting aborted */
+ PV_WAKE_KICKED, /* Wakeup by kicking */
+ PV_WAKE_SPURIOUS, /* Spurious wakeup */
+ PV_KICK_NOHALT /* Kick but CPU not halted */
+};
+#endif
+
struct pv_lock_ops {
+#ifdef CONFIG_QUEUE_SPINLOCK
+ void (*kick_cpu)(int cpu);
+ void (*halt_cpu)(enum pv_lock_stats type, s8 *state, s8 sval);
+ void (*lockstat)(enum pv_lock_stats type);
+#else
struct paravirt_callee_save lock_spinning;
void (*unlock_kick)(struct arch_spinlock *lock, __ticket_t ticket);
+#endif
};
/* This contains all the paravirt structures: we get a convenient
diff --git a/arch/x86/kernel/paravirt-spinlocks.c b/arch/x86/kernel/paravirt-spinlocks.c
index d9041c9..17435b7 100644
--- a/arch/x86/kernel/paravirt-spinlocks.c
+++ b/arch/x86/kernel/paravirt-spinlocks.c
@@ -11,9 +11,15 @@
#ifdef CONFIG_PARAVIRT_SPINLOCKS
struct pv_lock_ops pv_lock_ops = {
#ifdef CONFIG_SMP
+#ifdef CONFIG_QUEUE_SPINLOCK
+ .kick_cpu = paravirt_nop,
+ .halt_cpu = paravirt_nop,
+ .lockstat = paravirt_nop,
+#else
.lock_spinning = __PV_IS_CALLEE_SAVE(paravirt_nop),
.unlock_kick = paravirt_nop,
#endif
+#endif
};
EXPORT_SYMBOL(pv_lock_ops);
--
1.7.1
^ permalink raw reply related [flat|nested] 102+ messages in thread* [PATCH v11 12/16] pvqspinlock, x86: Add PV data structure & methods
2014-05-30 15:43 [PATCH v11 00/16] qspinlock: a 4-byte queue spinlock with PV support Waiman Long
` (25 preceding siblings ...)
2014-05-30 15:43 ` Waiman Long
@ 2014-05-30 15:43 ` Waiman Long
2014-05-30 15:43 ` Waiman Long
` (10 subsequent siblings)
37 siblings, 0 replies; 102+ messages in thread
From: Waiman Long @ 2014-05-30 15:43 UTC (permalink / raw)
To: Thomas Gleixner, Ingo Molnar, H. Peter Anvin, Peter Zijlstra
Cc: linux-arch, Waiman Long, Raghavendra K T, Gleb Natapov, kvm,
Scott J Norton, x86, Paolo Bonzini, linux-kernel, virtualization,
Chegu Vinod, David Vrabel, Oleg Nesterov, xen-devel,
Boris Ostrovsky, Paul E. McKenney, Linus Torvalds
This patch modifies the para-virtualization (PV) infrastructure code
of the x86-64 architecture to support the PV queue spinlock. Three
new virtual methods are added to support PV qspinlock:
1) kick_cpu - schedule in a virtual CPU
2) halt_cpu - schedule out a virtual CPU
3) lockstat - update statistical data for debugfs
Signed-off-by: Waiman Long <Waiman.Long@hp.com>
---
arch/x86/include/asm/paravirt.h | 18 +++++++++++++++++-
arch/x86/include/asm/paravirt_types.h | 17 +++++++++++++++++
arch/x86/kernel/paravirt-spinlocks.c | 6 ++++++
3 files changed, 40 insertions(+), 1 deletions(-)
diff --git a/arch/x86/include/asm/paravirt.h b/arch/x86/include/asm/paravirt.h
index cd6e161..d71e123 100644
--- a/arch/x86/include/asm/paravirt.h
+++ b/arch/x86/include/asm/paravirt.h
@@ -711,7 +711,23 @@ static inline void __set_fixmap(unsigned /* enum fixed_addresses */ idx,
}
#if defined(CONFIG_SMP) && defined(CONFIG_PARAVIRT_SPINLOCKS)
+#ifdef CONFIG_QUEUE_SPINLOCK
+static __always_inline void __queue_kick_cpu(int cpu)
+{
+ PVOP_VCALL1(pv_lock_ops.kick_cpu, cpu);
+}
+
+static __always_inline void
+__queue_halt_cpu(enum pv_lock_stats type, s8 *state, s8 sval)
+{
+ PVOP_VCALL3(pv_lock_ops.halt_cpu, type, state, sval);
+}
+static __always_inline void __queue_lockstat(enum pv_lock_stats type)
+{
+ PVOP_VCALL1(pv_lock_ops.lockstat, type);
+}
+#else
static __always_inline void __ticket_lock_spinning(struct arch_spinlock *lock,
__ticket_t ticket)
{
@@ -723,7 +739,7 @@ static __always_inline void __ticket_unlock_kick(struct arch_spinlock *lock,
{
PVOP_VCALL2(pv_lock_ops.unlock_kick, lock, ticket);
}
-
+#endif
#endif
#ifdef CONFIG_X86_32
diff --git a/arch/x86/include/asm/paravirt_types.h b/arch/x86/include/asm/paravirt_types.h
index 7549b8b..549b3a0 100644
--- a/arch/x86/include/asm/paravirt_types.h
+++ b/arch/x86/include/asm/paravirt_types.h
@@ -333,9 +333,26 @@ struct arch_spinlock;
typedef u16 __ticket_t;
#endif
+#ifdef CONFIG_QUEUE_SPINLOCK
+enum pv_lock_stats {
+ PV_HALT_QHEAD, /* Queue head halting */
+ PV_HALT_QNODE, /* Other queue node halting */
+ PV_HALT_ABORT, /* Halting aborted */
+ PV_WAKE_KICKED, /* Wakeup by kicking */
+ PV_WAKE_SPURIOUS, /* Spurious wakeup */
+ PV_KICK_NOHALT /* Kick but CPU not halted */
+};
+#endif
+
struct pv_lock_ops {
+#ifdef CONFIG_QUEUE_SPINLOCK
+ void (*kick_cpu)(int cpu);
+ void (*halt_cpu)(enum pv_lock_stats type, s8 *state, s8 sval);
+ void (*lockstat)(enum pv_lock_stats type);
+#else
struct paravirt_callee_save lock_spinning;
void (*unlock_kick)(struct arch_spinlock *lock, __ticket_t ticket);
+#endif
};
/* This contains all the paravirt structures: we get a convenient
diff --git a/arch/x86/kernel/paravirt-spinlocks.c b/arch/x86/kernel/paravirt-spinlocks.c
index d9041c9..17435b7 100644
--- a/arch/x86/kernel/paravirt-spinlocks.c
+++ b/arch/x86/kernel/paravirt-spinlocks.c
@@ -11,9 +11,15 @@
#ifdef CONFIG_PARAVIRT_SPINLOCKS
struct pv_lock_ops pv_lock_ops = {
#ifdef CONFIG_SMP
+#ifdef CONFIG_QUEUE_SPINLOCK
+ .kick_cpu = paravirt_nop,
+ .halt_cpu = paravirt_nop,
+ .lockstat = paravirt_nop,
+#else
.lock_spinning = __PV_IS_CALLEE_SAVE(paravirt_nop),
.unlock_kick = paravirt_nop,
#endif
+#endif
};
EXPORT_SYMBOL(pv_lock_ops);
--
1.7.1
^ permalink raw reply related [flat|nested] 102+ messages in thread* [PATCH v11 13/16] pvqspinlock: Enable coexistence with the unfair lock
2014-05-30 15:43 [PATCH v11 00/16] qspinlock: a 4-byte queue spinlock with PV support Waiman Long
@ 2014-05-30 15:43 ` Waiman Long
2014-05-30 15:43 ` Waiman Long
` (36 subsequent siblings)
37 siblings, 0 replies; 102+ messages in thread
From: Waiman Long @ 2014-05-30 15:43 UTC (permalink / raw)
To: Thomas Gleixner, Ingo Molnar, H. Peter Anvin, Peter Zijlstra
Cc: linux-arch, x86, linux-kernel, virtualization, xen-devel, kvm,
Paolo Bonzini, Konrad Rzeszutek Wilk, Boris Ostrovsky,
Paul E. McKenney, Rik van Riel, Linus Torvalds, Raghavendra K T,
David Vrabel, Oleg Nesterov, Gleb Natapov, Scott J Norton,
Chegu Vinod, Waiman Long
This patch enables the coexistence of both the PV qspinlock and
unfair lock. When both are enabled, however, only the lock fastpath
will perform lock stealing whereas the slowpath will have that disabled
to get the best of both features.
We also need to transition a CPU spinning too long in the pending
bit code path back to the regular queuing code path so that it can
be properly halted by the PV qspinlock code.
Signed-off-by: Waiman Long <Waiman.Long@hp.com>
---
kernel/locking/qspinlock.c | 47 ++++++++++++++++++++++++++++++++++++++++---
1 files changed, 43 insertions(+), 4 deletions(-)
diff --git a/kernel/locking/qspinlock.c b/kernel/locking/qspinlock.c
index 93c663a..8deedcf 100644
--- a/kernel/locking/qspinlock.c
+++ b/kernel/locking/qspinlock.c
@@ -57,12 +57,24 @@
#include "mcs_spinlock.h"
/*
+ * Check the pending bit spinning threshold only if PV qspinlock is enabled
+ */
+#define PSPIN_THRESHOLD (1 << 10)
+#define MAX_NODES 4
+
+#ifdef CONFIG_PARAVIRT_SPINLOCKS
+#define pv_qspinlock_enabled() static_key_false(¶virt_spinlocks_enabled)
+#else
+#define pv_qspinlock_enabled() false
+#endif
+
+/*
* Per-CPU queue node structures; we can never have more than 4 nested
* contexts: task, softirq, hardirq, nmi.
*
* Exactly fits one cacheline.
*/
-static DEFINE_PER_CPU_ALIGNED(struct mcs_spinlock, mcs_nodes[4]);
+static DEFINE_PER_CPU_ALIGNED(struct mcs_spinlock, mcs_nodes[MAX_NODES]);
/*
* We must be able to distinguish between no-tail and the tail at 0:0,
@@ -265,6 +277,9 @@ static noinline void queue_spin_lock_slowerpath(struct qspinlock *lock,
ACCESS_ONCE(prev->next) = node;
arch_mcs_spin_lock_contended(&node->locked);
+ } else {
+ /* Mark it as the queue head */
+ ACCESS_ONCE(node->locked) = true;
}
/*
@@ -344,14 +359,17 @@ void queue_spin_lock_slowpath(struct qspinlock *lock, u32 val)
struct mcs_spinlock *node;
u32 new, old, tail;
int idx;
+ int retry = INT_MAX; /* Retry count, queue if <= 0 */
BUILD_BUG_ON(CONFIG_NR_CPUS >= (1U << _Q_TAIL_CPU_BITS));
#ifdef CONFIG_VIRT_UNFAIR_LOCKS
/*
* A simple test and set unfair lock
+ * Disable waiter lock stealing if PV spinlock is enabled
*/
- if (static_key_false(&virt_unfairlocks_enabled)) {
+ if (!pv_qspinlock_enabled() &&
+ static_key_false(&virt_unfairlocks_enabled)) {
cpu_relax(); /* Relax after a failed lock attempt */
while (!queue_spin_trylock(lock))
cpu_relax();
@@ -360,6 +378,14 @@ void queue_spin_lock_slowpath(struct qspinlock *lock, u32 val)
#endif /* CONFIG_VIRT_UNFAIR_LOCKS */
/*
+ * When PV qspinlock is enabled, exit the pending bit code path and
+ * go back to the regular queuing path if the lock isn't available
+ * within a certain threshold.
+ */
+ if (pv_qspinlock_enabled())
+ retry = PSPIN_THRESHOLD;
+
+ /*
* trylock || pending
*
* 0,0,0 -> 0,0,1 ; trylock
@@ -370,7 +396,7 @@ void queue_spin_lock_slowpath(struct qspinlock *lock, u32 val)
* If we observe that the queue is not empty or both
* the pending and lock bits are set, queue
*/
- if ((val & _Q_TAIL_MASK) ||
+ if ((val & _Q_TAIL_MASK) || (retry-- <= 0) ||
(val == (_Q_LOCKED_VAL|_Q_PENDING_VAL)))
goto queue;
@@ -413,8 +439,21 @@ void queue_spin_lock_slowpath(struct qspinlock *lock, u32 val)
* sequentiality; this because not all clear_pending_set_locked()
* implementations imply full barriers.
*/
- while ((val = smp_load_acquire(&lock->val.counter)) & _Q_LOCKED_MASK)
+ while ((val = smp_load_acquire(&lock->val.counter)) & _Q_LOCKED_MASK) {
+ if (pv_qspinlock_enabled() && (retry-- <= 0)) {
+ /*
+ * Clear the pending bit and queue
+ */
+ for (;;) {
+ new = val & ~_Q_PENDING_MASK;
+ old = atomic_cmpxchg(&lock->val, val, new);
+ if (old == val)
+ goto queue;
+ val = old;
+ }
+ }
arch_mutex_cpu_relax();
+ }
/*
* take ownership and clear the pending bit.
--
1.7.1
^ permalink raw reply related [flat|nested] 102+ messages in thread* [PATCH v11 13/16] pvqspinlock: Enable coexistence with the unfair lock
@ 2014-05-30 15:43 ` Waiman Long
0 siblings, 0 replies; 102+ messages in thread
From: Waiman Long @ 2014-05-30 15:43 UTC (permalink / raw)
To: Thomas Gleixner, Ingo Molnar, H. Peter Anvin, Peter Zijlstra
Cc: linux-arch, Waiman Long, Rik van Riel, Raghavendra K T,
Gleb Natapov, kvm, Konrad Rzeszutek Wilk, Scott J Norton, x86,
Paolo Bonzini, linux-kernel, virtualization, Chegu Vinod,
David Vrabel, Oleg Nesterov, xen-devel, Boris Ostrovsky,
Paul E. McKenney, Linus Torvalds
This patch enables the coexistence of both the PV qspinlock and
unfair lock. When both are enabled, however, only the lock fastpath
will perform lock stealing whereas the slowpath will have that disabled
to get the best of both features.
We also need to transition a CPU spinning too long in the pending
bit code path back to the regular queuing code path so that it can
be properly halted by the PV qspinlock code.
Signed-off-by: Waiman Long <Waiman.Long@hp.com>
---
kernel/locking/qspinlock.c | 47 ++++++++++++++++++++++++++++++++++++++++---
1 files changed, 43 insertions(+), 4 deletions(-)
diff --git a/kernel/locking/qspinlock.c b/kernel/locking/qspinlock.c
index 93c663a..8deedcf 100644
--- a/kernel/locking/qspinlock.c
+++ b/kernel/locking/qspinlock.c
@@ -57,12 +57,24 @@
#include "mcs_spinlock.h"
/*
+ * Check the pending bit spinning threshold only if PV qspinlock is enabled
+ */
+#define PSPIN_THRESHOLD (1 << 10)
+#define MAX_NODES 4
+
+#ifdef CONFIG_PARAVIRT_SPINLOCKS
+#define pv_qspinlock_enabled() static_key_false(¶virt_spinlocks_enabled)
+#else
+#define pv_qspinlock_enabled() false
+#endif
+
+/*
* Per-CPU queue node structures; we can never have more than 4 nested
* contexts: task, softirq, hardirq, nmi.
*
* Exactly fits one cacheline.
*/
-static DEFINE_PER_CPU_ALIGNED(struct mcs_spinlock, mcs_nodes[4]);
+static DEFINE_PER_CPU_ALIGNED(struct mcs_spinlock, mcs_nodes[MAX_NODES]);
/*
* We must be able to distinguish between no-tail and the tail at 0:0,
@@ -265,6 +277,9 @@ static noinline void queue_spin_lock_slowerpath(struct qspinlock *lock,
ACCESS_ONCE(prev->next) = node;
arch_mcs_spin_lock_contended(&node->locked);
+ } else {
+ /* Mark it as the queue head */
+ ACCESS_ONCE(node->locked) = true;
}
/*
@@ -344,14 +359,17 @@ void queue_spin_lock_slowpath(struct qspinlock *lock, u32 val)
struct mcs_spinlock *node;
u32 new, old, tail;
int idx;
+ int retry = INT_MAX; /* Retry count, queue if <= 0 */
BUILD_BUG_ON(CONFIG_NR_CPUS >= (1U << _Q_TAIL_CPU_BITS));
#ifdef CONFIG_VIRT_UNFAIR_LOCKS
/*
* A simple test and set unfair lock
+ * Disable waiter lock stealing if PV spinlock is enabled
*/
- if (static_key_false(&virt_unfairlocks_enabled)) {
+ if (!pv_qspinlock_enabled() &&
+ static_key_false(&virt_unfairlocks_enabled)) {
cpu_relax(); /* Relax after a failed lock attempt */
while (!queue_spin_trylock(lock))
cpu_relax();
@@ -360,6 +378,14 @@ void queue_spin_lock_slowpath(struct qspinlock *lock, u32 val)
#endif /* CONFIG_VIRT_UNFAIR_LOCKS */
/*
+ * When PV qspinlock is enabled, exit the pending bit code path and
+ * go back to the regular queuing path if the lock isn't available
+ * within a certain threshold.
+ */
+ if (pv_qspinlock_enabled())
+ retry = PSPIN_THRESHOLD;
+
+ /*
* trylock || pending
*
* 0,0,0 -> 0,0,1 ; trylock
@@ -370,7 +396,7 @@ void queue_spin_lock_slowpath(struct qspinlock *lock, u32 val)
* If we observe that the queue is not empty or both
* the pending and lock bits are set, queue
*/
- if ((val & _Q_TAIL_MASK) ||
+ if ((val & _Q_TAIL_MASK) || (retry-- <= 0) ||
(val == (_Q_LOCKED_VAL|_Q_PENDING_VAL)))
goto queue;
@@ -413,8 +439,21 @@ void queue_spin_lock_slowpath(struct qspinlock *lock, u32 val)
* sequentiality; this because not all clear_pending_set_locked()
* implementations imply full barriers.
*/
- while ((val = smp_load_acquire(&lock->val.counter)) & _Q_LOCKED_MASK)
+ while ((val = smp_load_acquire(&lock->val.counter)) & _Q_LOCKED_MASK) {
+ if (pv_qspinlock_enabled() && (retry-- <= 0)) {
+ /*
+ * Clear the pending bit and queue
+ */
+ for (;;) {
+ new = val & ~_Q_PENDING_MASK;
+ old = atomic_cmpxchg(&lock->val, val, new);
+ if (old == val)
+ goto queue;
+ val = old;
+ }
+ }
arch_mutex_cpu_relax();
+ }
/*
* take ownership and clear the pending bit.
--
1.7.1
^ permalink raw reply related [flat|nested] 102+ messages in thread
* [PATCH v11 13/16] pvqspinlock: Enable coexistence with the unfair lock
2014-05-30 15:43 [PATCH v11 00/16] qspinlock: a 4-byte queue spinlock with PV support Waiman Long
` (27 preceding siblings ...)
2014-05-30 15:43 ` Waiman Long
@ 2014-05-30 15:43 ` Waiman Long
2014-05-30 15:44 ` [PATCH v11 14/16] pvqspinlock: Add qspinlock para-virtualization support Waiman Long
` (8 subsequent siblings)
37 siblings, 0 replies; 102+ messages in thread
From: Waiman Long @ 2014-05-30 15:43 UTC (permalink / raw)
To: Thomas Gleixner, Ingo Molnar, H. Peter Anvin, Peter Zijlstra
Cc: linux-arch, Waiman Long, Raghavendra K T, Gleb Natapov, kvm,
Scott J Norton, x86, Paolo Bonzini, linux-kernel, virtualization,
Chegu Vinod, David Vrabel, Oleg Nesterov, xen-devel,
Boris Ostrovsky, Paul E. McKenney, Linus Torvalds
This patch enables the coexistence of both the PV qspinlock and
unfair lock. When both are enabled, however, only the lock fastpath
will perform lock stealing whereas the slowpath will have that disabled
to get the best of both features.
We also need to transition a CPU spinning too long in the pending
bit code path back to the regular queuing code path so that it can
be properly halted by the PV qspinlock code.
Signed-off-by: Waiman Long <Waiman.Long@hp.com>
---
kernel/locking/qspinlock.c | 47 ++++++++++++++++++++++++++++++++++++++++---
1 files changed, 43 insertions(+), 4 deletions(-)
diff --git a/kernel/locking/qspinlock.c b/kernel/locking/qspinlock.c
index 93c663a..8deedcf 100644
--- a/kernel/locking/qspinlock.c
+++ b/kernel/locking/qspinlock.c
@@ -57,12 +57,24 @@
#include "mcs_spinlock.h"
/*
+ * Check the pending bit spinning threshold only if PV qspinlock is enabled
+ */
+#define PSPIN_THRESHOLD (1 << 10)
+#define MAX_NODES 4
+
+#ifdef CONFIG_PARAVIRT_SPINLOCKS
+#define pv_qspinlock_enabled() static_key_false(¶virt_spinlocks_enabled)
+#else
+#define pv_qspinlock_enabled() false
+#endif
+
+/*
* Per-CPU queue node structures; we can never have more than 4 nested
* contexts: task, softirq, hardirq, nmi.
*
* Exactly fits one cacheline.
*/
-static DEFINE_PER_CPU_ALIGNED(struct mcs_spinlock, mcs_nodes[4]);
+static DEFINE_PER_CPU_ALIGNED(struct mcs_spinlock, mcs_nodes[MAX_NODES]);
/*
* We must be able to distinguish between no-tail and the tail at 0:0,
@@ -265,6 +277,9 @@ static noinline void queue_spin_lock_slowerpath(struct qspinlock *lock,
ACCESS_ONCE(prev->next) = node;
arch_mcs_spin_lock_contended(&node->locked);
+ } else {
+ /* Mark it as the queue head */
+ ACCESS_ONCE(node->locked) = true;
}
/*
@@ -344,14 +359,17 @@ void queue_spin_lock_slowpath(struct qspinlock *lock, u32 val)
struct mcs_spinlock *node;
u32 new, old, tail;
int idx;
+ int retry = INT_MAX; /* Retry count, queue if <= 0 */
BUILD_BUG_ON(CONFIG_NR_CPUS >= (1U << _Q_TAIL_CPU_BITS));
#ifdef CONFIG_VIRT_UNFAIR_LOCKS
/*
* A simple test and set unfair lock
+ * Disable waiter lock stealing if PV spinlock is enabled
*/
- if (static_key_false(&virt_unfairlocks_enabled)) {
+ if (!pv_qspinlock_enabled() &&
+ static_key_false(&virt_unfairlocks_enabled)) {
cpu_relax(); /* Relax after a failed lock attempt */
while (!queue_spin_trylock(lock))
cpu_relax();
@@ -360,6 +378,14 @@ void queue_spin_lock_slowpath(struct qspinlock *lock, u32 val)
#endif /* CONFIG_VIRT_UNFAIR_LOCKS */
/*
+ * When PV qspinlock is enabled, exit the pending bit code path and
+ * go back to the regular queuing path if the lock isn't available
+ * within a certain threshold.
+ */
+ if (pv_qspinlock_enabled())
+ retry = PSPIN_THRESHOLD;
+
+ /*
* trylock || pending
*
* 0,0,0 -> 0,0,1 ; trylock
@@ -370,7 +396,7 @@ void queue_spin_lock_slowpath(struct qspinlock *lock, u32 val)
* If we observe that the queue is not empty or both
* the pending and lock bits are set, queue
*/
- if ((val & _Q_TAIL_MASK) ||
+ if ((val & _Q_TAIL_MASK) || (retry-- <= 0) ||
(val == (_Q_LOCKED_VAL|_Q_PENDING_VAL)))
goto queue;
@@ -413,8 +439,21 @@ void queue_spin_lock_slowpath(struct qspinlock *lock, u32 val)
* sequentiality; this because not all clear_pending_set_locked()
* implementations imply full barriers.
*/
- while ((val = smp_load_acquire(&lock->val.counter)) & _Q_LOCKED_MASK)
+ while ((val = smp_load_acquire(&lock->val.counter)) & _Q_LOCKED_MASK) {
+ if (pv_qspinlock_enabled() && (retry-- <= 0)) {
+ /*
+ * Clear the pending bit and queue
+ */
+ for (;;) {
+ new = val & ~_Q_PENDING_MASK;
+ old = atomic_cmpxchg(&lock->val, val, new);
+ if (old == val)
+ goto queue;
+ val = old;
+ }
+ }
arch_mutex_cpu_relax();
+ }
/*
* take ownership and clear the pending bit.
--
1.7.1
^ permalink raw reply related [flat|nested] 102+ messages in thread* [PATCH v11 14/16] pvqspinlock: Add qspinlock para-virtualization support
2014-05-30 15:43 [PATCH v11 00/16] qspinlock: a 4-byte queue spinlock with PV support Waiman Long
` (28 preceding siblings ...)
2014-05-30 15:43 ` Waiman Long
@ 2014-05-30 15:44 ` Waiman Long
2014-06-12 8:17 ` Peter Zijlstra
2014-06-12 8:17 ` Peter Zijlstra
2014-05-30 15:44 ` Waiman Long
` (7 subsequent siblings)
37 siblings, 2 replies; 102+ messages in thread
From: Waiman Long @ 2014-05-30 15:44 UTC (permalink / raw)
To: Thomas Gleixner, Ingo Molnar, H. Peter Anvin, Peter Zijlstra
Cc: linux-arch, x86, linux-kernel, virtualization, xen-devel, kvm,
Paolo Bonzini, Konrad Rzeszutek Wilk, Boris Ostrovsky,
Paul E. McKenney, Rik van Riel, Linus Torvalds, Raghavendra K T,
David Vrabel, Oleg Nesterov, Gleb Natapov, Scott J Norton,
Chegu Vinod, Waiman Long
This patch adds base para-virtualization support to the queue
spinlock in the same way as was done in the PV ticket lock code. In
essence, the lock waiters will spin for a specified number of times
(QSPIN_THRESHOLD = 2^14) and then halted itself. The queue head waiter,
unlike the other waiter, will spins 2*QSPIN_THRESHOLD times before
halting itself. Before being halted, the queue head waiter will set
a flag (_Q_LOCKED_SLOWPATH) in the lock byte to indicate that the
unlock slowpath has to be invoked.
In the unlock slowpath, the current lock holder will find the queue
head by following the previous node pointer links stored in the queue
node structure until it finds one that has the qhead flag turned
on. It then attempt to kick in the CPU of the queue head.
After the queue head acquired the lock, it will also check the status
of the next node and set _Q_LOCKED_SLOWPATH flag if it has been halted.
Enabling the PV code does have a performance impact on spinlock
acquisitions and releases. The following table shows the execution
time (in ms) of a spinlock micro-benchmark that does lock/unlock
operations 5M times for each task versus the number of contending
tasks on a Westmere-EX system.
# of Ticket lock Queue lock
tasks PV off/PV on/%Change PV off/PV on/%Change
------ -------------------- ---------------------
1 135/ 179/+33% 137/ 168/+23%
2 1045/ 1103/ +6% 1161/ 1248/ +7%
3 1827/ 2683/+47% 2357/ 2600/+10%
4 2689/ 4191/+56% 2882/ 3115/ +8%
5 3736/ 5830/+56% 3493/ 3571/ +2%
6 4942/ 7609/+54% 4239/ 4198/ -1%
7 6304/ 9570/+52% 4931/ 4895/ -1%
8 7736/11323/+46% 5632/ 5588/ -1%
It can be seen that the ticket lock PV code has a fairly big decrease
in performance when there are 3 or more contending tasks. The queue
spinlock PV code, on the other hand, only has a relatively minor
drop in performance for with 1-4 contending tasks. With 5 or more
contending tasks, there is practically no difference in performance.
Signed-off-by: Waiman Long <Waiman.Long@hp.com>
---
arch/x86/include/asm/pvqspinlock.h | 359 ++++++++++++++++++++++++++++++++++++
arch/x86/include/asm/qspinlock.h | 33 ++++
kernel/locking/qspinlock.c | 72 +++++++-
3 files changed, 458 insertions(+), 6 deletions(-)
create mode 100644 arch/x86/include/asm/pvqspinlock.h
diff --git a/arch/x86/include/asm/pvqspinlock.h b/arch/x86/include/asm/pvqspinlock.h
new file mode 100644
index 0000000..af00eda
--- /dev/null
+++ b/arch/x86/include/asm/pvqspinlock.h
@@ -0,0 +1,359 @@
+#ifndef _ASM_X86_PVQSPINLOCK_H
+#define _ASM_X86_PVQSPINLOCK_H
+
+/*
+ * Queue Spinlock Para-Virtualization (PV) Support
+ *
+ * +------+ +------+ +------+
+ * pv_qnode |Queue | prev | | prev |Queue |
+ * | Head |<-------| Node | .... <-| Tail |
+ * +------+ +------+ +------+
+ * | | |
+ * V V V
+ * +------+ +------+ +------+
+ * mcs_spinlock |locked| |locked| |locked|
+ * | = 1 |------->| = 0 |-> .... | = 0 |
+ * +------+ next +------+ next +------+
+ *
+ * The PV support code for queue spinlock is roughly the same as that
+ * of the ticket spinlock. Each CPU waiting for the lock will spin until it
+ * reaches a threshold. When that happens, it will put itself to halt so
+ * that the hypervisor can reuse the CPU cycles in some other guests as well
+ * as returning other hold-up CPUs faster.
+ *
+ * Auxillary fields in the pv_qnode structure are used to hold information
+ * relevant to the PV support so that it won't impact on the behavior and
+ * performance of the bare metal code. The structure contains a prev pointer
+ * so that a lock holder can find out the queue head from the queue tail
+ * following the prev pointers.
+ *
+ * A major difference between the two versions of PV spinlock is the fact
+ * that the spin threshold of the queue spinlock is half of that of the
+ * ticket spinlock. However, the queue head will spin twice as long as the
+ * other nodes before it puts itself to halt. The reason for that is to
+ * increase halting chance of heavily contended locks to favor lightly
+ * contended locks (queue depth of 1 or less).
+ *
+ * There are 2 places where races can happen:
+ * 1) Halting of the queue head CPU (in pv_head_spin_check) and the CPU
+ * kicking by the lock holder in the unlock path (in pv_kick_node).
+ * 2) Halting of the queue node CPU (in pv_queue_spin_check) and the
+ * the status check by the previous queue head (in pv_halt_check).
+ * See the comments on those functions to see how the races are being
+ * addressed.
+ */
+
+/*
+ * Spin threshold for queue spinlock
+ */
+#define QSPIN_THRESHOLD (1U<<14)
+#define MAYHALT_THRESHOLD (QSPIN_THRESHOLD - 0x10)
+
+/*
+ * CPU state flags
+ */
+#define PV_CPU_ACTIVE 1 /* This CPU is active */
+#define PV_CPU_KICKED 2 /* This CPU is being kicked */
+#define PV_CPU_HALTED -1 /* This CPU is halted */
+
+/*
+ * Additional fields to be added to the queue node structure
+ *
+ * The size of the mcs_spinlock structure is 16 bytes for x64 and 12 bytes
+ * for i386. Four of those structures are defined per CPU. To add more fields
+ * without increasing the size of the mcs_spinlock structure, we overlay those
+ * additional data fields at an additional mcs_spinlock size bucket at exactly
+ * 3 units away. As a result, we need to double the number of mcs_spinlock
+ * buckets. The mcs_spinlock structure will be casted to the pv_qnode
+ * internally.
+ *
+ * +------------+------------+------------+------------+
+ * | MCS Node 0 | MCS Node 1 | MCS Node 2 | MCS Node 3 |
+ * +------------+------------+------------+------------+
+ * | PV Node 0 | PV Node 1 | PV Node 2 | PV Node 3 |
+ * +------------+------------+------------+------------+
+ */
+struct pv_qnode {
+ struct mcs_spinlock mcs; /* MCS node */
+ struct mcs_spinlock dummy[3]; /* 3 dummy MCS nodes */
+ s8 cpustate; /* CPU status flag */
+ s8 mayhalt; /* May be halted soon */
+ u32 mycpu; /* CPU number of this node */
+ struct pv_qnode *prev; /* Pointer to previous node */
+};
+
+#define qhead mcs.locked /* Queue head flag */
+
+/**
+ * pv_init_vars - initialize fields in struct pv_qnode
+ * @mcs: pointer to struct mcs_spinlock
+ * @cpu: current CPU number
+ */
+static inline void pv_init_vars(struct mcs_spinlock *node, int cpu)
+{
+ struct pv_qnode *pv = (struct pv_qnode *)node;
+
+ BUILD_BUG_ON(sizeof(struct pv_qnode) > 5*sizeof(struct mcs_spinlock));
+
+ if (!static_key_false(¶virt_spinlocks_enabled))
+ return;
+
+ pv->cpustate = PV_CPU_ACTIVE;
+ pv->prev = NULL;
+ pv->mayhalt = false;
+ pv->mycpu = cpu;
+}
+
+/**
+ * pv_head_spin_check - perform para-virtualization checks for queue head
+ * @mcs : pointer to the mcs_spinlock structure
+ * @lock : pointer to the qspinlock structure
+ * @count: loop count pointer
+ *
+ * This function will halt itself if lock is still not available after
+ * QSPIN_THRESHOLD iterations.
+ */
+static inline void
+pv_head_spin_check(struct mcs_spinlock *mcs, struct qspinlock *lock, int *count)
+{
+ struct pv_qnode *pv = (struct pv_qnode *)mcs;
+
+ if (!static_key_false(¶virt_spinlocks_enabled))
+ return;
+
+ (*count)++;
+ if (pv->cpustate == PV_CPU_KICKED) {
+ /*
+ * Reset count and flag
+ */
+ pv->cpustate = PV_CPU_ACTIVE;
+ *count = 0;
+
+ } else if (unlikely(*count >= 2*QSPIN_THRESHOLD)) {
+ u8 lockval;
+ s8 oldstate;
+
+ /*
+ * Set the lock byte to _Q_LOCKED_SLOWPATH before
+ * trying to halt itself. It is possible that the
+ * lock byte had been set to _Q_LOCKED_SLOWPATH
+ * already (spurious wakeup of queue head after a halt
+ * or opportunistic setting in pv_halt_check()).
+ * In this case, just proceeds to sleeping.
+ *
+ * queue head lock holder
+ * ---------- -----------
+ * cpustate = PV_CPU_HALTED
+ * [1] cmpxchg(_Q_LOCKED_VAL [2] cmpxchg(_Q_LOCKED_VAL => 0)
+ * => _Q_LOCKED_SLOWPATH) if (cmpxchg fails &&
+ * if (cmpxchg succeeds) cpustate == PV_CPU_HALTED)
+ * halt() kick()
+ *
+ * Sequence:
+ * 1,2 - slowpath flag set, queue head halted & lock holder
+ * will call slowpath
+ * 2,1 - queue head cmpxchg fails, halt is aborted
+ *
+ * If the queue head CPU is woken up by a spurious interrupt
+ * at the same time as the lock holder check the cpustate,
+ * it is possible that the lock holder will try to kick
+ * the queue head CPU which isn't halted.
+ */
+ oldstate = cmpxchg(&pv->cpustate, PV_CPU_ACTIVE, PV_CPU_HALTED);
+ if (oldstate == PV_CPU_KICKED)
+ goto reset; /* Reset count and state */
+
+ lockval = cmpxchg((u8 *)lock,
+ _Q_LOCKED_VAL, _Q_LOCKED_SLOWPATH);
+ if (lockval != 0) {
+ __queue_halt_cpu(PV_HALT_QHEAD, &pv->cpustate,
+ PV_CPU_HALTED);
+ __queue_lockstat((pv->cpustate == PV_CPU_KICKED)
+ ? PV_WAKE_KICKED : PV_WAKE_SPURIOUS);
+ }
+ /*
+ * Else, the lock is free and no halting is needed
+ */
+reset:
+ ACCESS_ONCE(pv->cpustate) = PV_CPU_ACTIVE;
+ *count = 0; /* Reset count */
+ }
+}
+
+/**
+ * pv_queue_spin_check - perform para-virtualization checks for queue member
+ * @mcs : pointer to the mcs_spinlock structure
+ * @count: loop count pointer
+ */
+static inline void pv_queue_spin_check(struct mcs_spinlock *mcs, int *count)
+{
+ struct pv_qnode *pv = (struct pv_qnode *)mcs;
+
+ if (!static_key_false(¶virt_spinlocks_enabled))
+ return;
+ /*
+ * Attempt to halt oneself after QSPIN_THRESHOLD spins
+ */
+ (*count)++;
+ if (unlikely(*count >= QSPIN_THRESHOLD)) {
+ /*
+ * Time to halt itself
+ */
+ ACCESS_ONCE(pv->cpustate) = PV_CPU_HALTED;
+ /*
+ * One way to avoid the racing between pv_halt_check()
+ * and pv_queue_spin_check() is to use memory barrier or
+ * atomic instruction to synchronize between the two competing
+ * threads. However, that will slow down the queue spinlock
+ * slowpath. One way to eliminate this overhead for normal
+ * cases is to use another flag (mayhalt) to indicate that
+ * racing condition may happen. This flag is set when the
+ * loop count is getting close to the halting threshold.
+ *
+ * When that happens, a 2 variables (cpustate & qhead
+ * [=mcs->locked]) handshake is used to make sure that
+ * pv_halt_check() won't miss setting the _Q_LOCKED_SLOWPATH
+ * when the CPU is about to be halted.
+ *
+ * pv_halt_check pv_queue_spin_check
+ * ------------- -------------------
+ * [1] qhead = true [3] cpustate = PV_CPU_HALTED
+ * smp_mb() smp_mb()
+ * [2] if (cpustate [4] if (qhead)
+ * == PV_CPU_HALTED)
+ *
+ * Sequence:
+ * *,1,*,4,* - halt is aborted as the qhead flag is set,
+ * _Q_LOCKED_SLOWPATH may or may not be set
+ * 3,4,1,2 - the CPU is halt and _Q_LOCKED_SLOWPATH is set
+ */
+ smp_mb();
+ if (!ACCESS_ONCE(pv->qhead)) {
+ /*
+ * Halt the CPU only if it is not the queue head
+ */
+ __queue_halt_cpu(PV_HALT_QNODE, &pv->cpustate,
+ PV_CPU_HALTED);
+ __queue_lockstat((pv->cpustate == PV_CPU_KICKED)
+ ? PV_WAKE_KICKED : PV_WAKE_SPURIOUS);
+ }
+ ACCESS_ONCE(pv->cpustate) = PV_CPU_ACTIVE;
+ *count = 0; /* Reset count & flag */
+ pv->mayhalt = false;
+ } else if (*count == MAYHALT_THRESHOLD) {
+ pv->mayhalt = true;
+ /*
+ * Make sure that the mayhalt flag is visible to others
+ * before proceeding.
+ */
+ smp_mb();
+ }
+}
+
+/**
+ * pv_halt_check - check if the CPU has been halted & set _Q_LOCKED_SLOWPATH
+ * @mcs : pointer to the mcs_spinlock structure
+ * @count: loop count
+ *
+ * The current CPU should have gotten the lock and the queue head flag set
+ * before calling this function.
+ */
+static inline void
+pv_halt_check(struct mcs_spinlock *mcs, struct qspinlock *lock)
+{
+ struct pv_qnode *pv = (struct pv_qnode *)mcs;
+
+ if (!static_key_false(¶virt_spinlocks_enabled))
+ return;
+ /*
+ * Halt state checking will only be done if the mayhalt flag is set
+ * to avoid the overhead of the memory barrier in normal cases.
+ * It is highly unlikely that the actual writing to the qhead flag
+ * will be more than 0x10 iterations later than the reading of the
+ * mayhalt flag so that it misses seeing the PV_CPU_HALTED state
+ * which causes lost wakeup.
+ */
+ if (ACCESS_ONCE(pv->mayhalt)) {
+ /*
+ * A memory barrier is used here to make sure that the setting
+ * of queue head flag prior to this function call is visible
+ * to others before checking the cpustate flag.
+ */
+ smp_mb();
+ if (pv->cpustate == PV_CPU_HALTED)
+ ACCESS_ONCE(*(u8 *)lock) = _Q_LOCKED_SLOWPATH;
+ }
+}
+
+/**
+ * pv_set_prev - set previous queue node pointer
+ * @mcs : pointer to the mcs_spinlock structure
+ * @prev: pointer to the previous pv_qnode
+ */
+static inline void
+pv_set_prev(struct mcs_spinlock *mcs, struct mcs_spinlock *prev)
+{
+ struct pv_qnode *pv = (struct pv_qnode *)mcs;
+
+ ACCESS_ONCE(pv->prev) = (struct pv_qnode *)prev;
+ /*
+ * Make sure the prev field is set up before others
+ */
+ smp_wmb();
+}
+
+/*
+ * The following inlined functions are being used by the
+ * queue_spin_unlock_slowpath() function.
+ */
+
+/**
+ * pv_tail_to_qhead - get queue head pv_qnode from tail code
+ * @tail : pointer to queue tail code
+ * @Return: mcs_spinlock pointer of queue head
+ *
+ * This function should only be called by the current lock holder so that
+ * the queue head won't be changed.
+ */
+static struct mcs_spinlock *pv_tail_to_qhead(u32 tail)
+{
+ struct pv_qnode *prev, *pv = (struct pv_qnode *)decode_tail(tail);
+
+ /*
+ * Locate the queue head node by following the prev pointer from
+ * tail to head. It is assumed that the PV guests won't have that
+ * many CPUs so that it won't take a long time to follow the pointers.
+ */
+ while (!ACCESS_ONCE(pv->qhead)) {
+ prev = ACCESS_ONCE(pv->prev);
+ if (prev)
+ pv = prev;
+ else
+ /*
+ * Delay a bit to allow the prev pointer or the queue
+ * head flag to be set up
+ */
+ cpu_relax();
+ }
+ return (struct mcs_spinlock *)pv;
+}
+
+/**
+ * pv_kick_node - kick up the CPU of the given node
+ * @mcs : pointer to struct mcs_spinlock of the node to be kicked
+ */
+static inline void pv_kick_node(struct mcs_spinlock *mcs)
+{
+ struct pv_qnode *pv = (struct pv_qnode *)mcs;
+ s8 oldstate = xchg(&pv->cpustate, PV_CPU_KICKED);
+
+ /*
+ * Kick up the CPU only if the state was set to PV_CPU_HALTED
+ */
+ if (oldstate != PV_CPU_HALTED)
+ __queue_lockstat(PV_KICK_NOHALT);
+ else
+ __queue_kick_cpu(pv->mycpu);
+}
+
+#endif /* _ASM_X86_PVQSPINLOCK_H */
diff --git a/arch/x86/include/asm/qspinlock.h b/arch/x86/include/asm/qspinlock.h
index 448de8b..5ddc456 100644
--- a/arch/x86/include/asm/qspinlock.h
+++ b/arch/x86/include/asm/qspinlock.h
@@ -19,13 +19,46 @@ extern struct static_key virt_unfairlocks_enabled;
* that the clearing the lock bit is done ASAP without artificial delay
* due to compiler optimization.
*/
+#ifdef CONFIG_PARAVIRT_SPINLOCKS
+static __always_inline void __queue_spin_unlock(struct qspinlock *lock)
+#else
static inline void queue_spin_unlock(struct qspinlock *lock)
+#endif
{
barrier();
ACCESS_ONCE(*(u8 *)lock) = 0;
barrier();
}
+#ifdef CONFIG_PARAVIRT_SPINLOCKS
+/*
+ * The lock byte can have a value of _Q_LOCKED_SLOWPATH to indicate
+ * that it needs to go through the slowpath to do the unlocking.
+ */
+#define _Q_LOCKED_SLOWPATH (_Q_LOCKED_VAL | 2)
+
+extern void queue_spin_unlock_slowpath(struct qspinlock *lock);
+
+static inline void queue_spin_unlock(struct qspinlock *lock)
+{
+ barrier();
+ if (static_key_false(¶virt_spinlocks_enabled)) {
+ /*
+ * Need to atomically clear the lock byte to avoid racing with
+ * queue head waiter trying to set _QLOCK_LOCKED_SLOWPATH.
+ */
+ if (likely(cmpxchg((u8 *)lock, _Q_LOCKED_VAL, 0)
+ == _Q_LOCKED_VAL))
+ return;
+ else
+ queue_spin_unlock_slowpath(lock);
+
+ } else {
+ __queue_spin_unlock(lock);
+ }
+ barrier();
+}
+#endif /* CONFIG_PARAVIRT_SPINLOCKS */
#endif /* !CONFIG_X86_OOSTORE && !CONFIG_X86_PPRO_FENCE */
#include <asm-generic/qspinlock.h>
diff --git a/kernel/locking/qspinlock.c b/kernel/locking/qspinlock.c
index 8deedcf..adedd75 100644
--- a/kernel/locking/qspinlock.c
+++ b/kernel/locking/qspinlock.c
@@ -60,12 +60,17 @@
* Check the pending bit spinning threshold only if PV qspinlock is enabled
*/
#define PSPIN_THRESHOLD (1 << 10)
-#define MAX_NODES 4
+/*
+ * We need to double the number of per-cpu mcs_spinlock structures to hold
+ * additional fields specific to para-virtualization support.
+ */
#ifdef CONFIG_PARAVIRT_SPINLOCKS
-#define pv_qspinlock_enabled() static_key_false(¶virt_spinlocks_enabled)
+# define MAX_NODES 8
+# define pv_qspinlock_enabled() static_key_false(¶virt_spinlocks_enabled)
#else
-#define pv_qspinlock_enabled() false
+# define MAX_NODES 4
+# define pv_qspinlock_enabled() false
#endif
/*
@@ -243,6 +248,22 @@ static __always_inline int try_set_locked(struct qspinlock *lock)
return 1;
}
+/*
+ * Para-virtualization (PV) queue spinlock support
+ */
+#ifdef CONFIG_PARAVIRT_SPINLOCKS
+#include <asm/pvqspinlock.h>
+#else
+static inline void pv_init_vars(struct mcs_spinlock *mcs, int cpu) {}
+static inline void pv_head_spin_check(struct mcs_spinlock *mcs,
+ struct qspinlock *lock, int *count) {}
+static inline void pv_queue_spin_check(struct mcs_spinlock *mcs,
+ int *count) {}
+static inline void pv_halt_check(struct mcs_spinlock *mcs, void *lock) {}
+static inline void pv_set_prev(struct mcs_spinlock *mcs,
+ struct mcs_spinlock *prev) {}
+#endif /* CONFIG_PARAVIRT_SPINLOCKS */
+
/**
* queue_spin_lock_slowerpath - a slower patch for acquiring queue spinlock
* @lock: Pointer to queue spinlock structure
@@ -260,6 +281,7 @@ static noinline void queue_spin_lock_slowerpath(struct qspinlock *lock,
{
struct mcs_spinlock *prev, *next;
u32 val, old;
+ int hcnt = 0; /* Queue head loop counter */
/*
* we already touched the queueing cacheline; don't bother with pending
@@ -273,9 +295,16 @@ static noinline void queue_spin_lock_slowerpath(struct qspinlock *lock,
* if there was a previous node; link it and wait.
*/
if (old & _Q_TAIL_MASK) {
+ int qcnt = 0; /* Queue node loop counter */
+
prev = decode_tail(old);
+ pv_set_prev(node, prev);
ACCESS_ONCE(prev->next) = node;
+ while (!smp_load_acquire(&node->locked)) {
+ pv_queue_spin_check(node, &qcnt);
+ arch_mutex_cpu_relax();
+ }
arch_mcs_spin_lock_contended(&node->locked);
} else {
/* Mark it as the queue head */
@@ -292,8 +321,14 @@ static noinline void queue_spin_lock_slowerpath(struct qspinlock *lock,
*/
retry_queue_wait:
while ((val = smp_load_acquire(&lock->val.counter))
- & _Q_LOCKED_PENDING_MASK)
+ & _Q_LOCKED_PENDING_MASK) {
+ /*
+ * Perform queue head para-virtualization checks
+ */
+ pv_head_spin_check(node, lock, &hcnt);
arch_mutex_cpu_relax();
+ }
+ hcnt = 0; /* Reset loop count */
/*
* claim the lock:
@@ -331,6 +366,7 @@ retry_queue_wait:
arch_mutex_cpu_relax();
arch_mcs_spin_unlock_contended(&next->locked);
+ pv_halt_check(next, lock);
}
/**
@@ -358,7 +394,7 @@ void queue_spin_lock_slowpath(struct qspinlock *lock, u32 val)
{
struct mcs_spinlock *node;
u32 new, old, tail;
- int idx;
+ int idx, cpu_nr;
int retry = INT_MAX; /* Retry count, queue if <= 0 */
BUILD_BUG_ON(CONFIG_NR_CPUS >= (1U << _Q_TAIL_CPU_BITS));
@@ -470,11 +506,12 @@ void queue_spin_lock_slowpath(struct qspinlock *lock, u32 val)
queue:
node = this_cpu_ptr(&mcs_nodes[0]);
idx = node->count++;
- tail = encode_tail(smp_processor_id(), idx);
+ tail = encode_tail(cpu_nr = smp_processor_id(), idx);
node += idx;
node->locked = 0;
node->next = NULL;
+ pv_init_vars(node, cpu_nr);
/*
* We touched a (possibly) cold cacheline in the per-cpu queue node;
@@ -490,3 +527,26 @@ queue:
this_cpu_dec(mcs_nodes[0].count);
}
EXPORT_SYMBOL(queue_spin_lock_slowpath);
+
+#ifdef CONFIG_PARAVIRT_SPINLOCKS
+/**
+ * queue_spin_unlock_slowpath - kick up the CPU of the queue head
+ * @lock : Pointer to queue spinlock structure
+ *
+ * The lock is released after finding the queue head to avoid racing
+ * condition between the queue head and the lock holder.
+ */
+void queue_spin_unlock_slowpath(struct qspinlock *lock)
+{
+ struct mcs_spinlock *node = pv_tail_to_qhead(atomic_read(&lock->val));
+
+ /*
+ * Found the queue head, now release the lock before waking it up
+ * If unfair lock is enabled, this allows other ready tasks to get
+ * lock before the halting CPU is waken up.
+ */
+ __queue_spin_unlock(lock);
+ pv_kick_node(node);
+}
+EXPORT_SYMBOL(queue_spin_unlock_slowpath);
+#endif /* CONFIG_PARAVIRT_SPINLOCKS */
--
1.7.1
^ permalink raw reply related [flat|nested] 102+ messages in thread* Re: [PATCH v11 14/16] pvqspinlock: Add qspinlock para-virtualization support
2014-05-30 15:44 ` [PATCH v11 14/16] pvqspinlock: Add qspinlock para-virtualization support Waiman Long
@ 2014-06-12 8:17 ` Peter Zijlstra
2014-06-12 8:17 ` Peter Zijlstra
1 sibling, 0 replies; 102+ messages in thread
From: Peter Zijlstra @ 2014-06-12 8:17 UTC (permalink / raw)
To: Waiman Long
Cc: Thomas Gleixner, Ingo Molnar, H. Peter Anvin, linux-arch, x86,
linux-kernel, virtualization, xen-devel, kvm, Paolo Bonzini,
Konrad Rzeszutek Wilk, Boris Ostrovsky, Paul E. McKenney,
Rik van Riel, Linus Torvalds, Raghavendra K T, David Vrabel,
Oleg Nesterov, Gleb Natapov, Scott J Norton, Chegu Vinod
[-- Attachment #1: Type: text/plain, Size: 1756 bytes --]
On Fri, May 30, 2014 at 11:44:00AM -0400, Waiman Long wrote:
> @@ -19,13 +19,46 @@ extern struct static_key virt_unfairlocks_enabled;
> * that the clearing the lock bit is done ASAP without artificial delay
> * due to compiler optimization.
> */
> +#ifdef CONFIG_PARAVIRT_SPINLOCKS
> +static __always_inline void __queue_spin_unlock(struct qspinlock *lock)
> +#else
> static inline void queue_spin_unlock(struct qspinlock *lock)
> +#endif
> {
> barrier();
> ACCESS_ONCE(*(u8 *)lock) = 0;
> barrier();
> }
>
> +#ifdef CONFIG_PARAVIRT_SPINLOCKS
> +/*
> + * The lock byte can have a value of _Q_LOCKED_SLOWPATH to indicate
> + * that it needs to go through the slowpath to do the unlocking.
> + */
> +#define _Q_LOCKED_SLOWPATH (_Q_LOCKED_VAL | 2)
> +
> +extern void queue_spin_unlock_slowpath(struct qspinlock *lock);
> +
> +static inline void queue_spin_unlock(struct qspinlock *lock)
> +{
> + barrier();
> + if (static_key_false(¶virt_spinlocks_enabled)) {
> + /*
> + * Need to atomically clear the lock byte to avoid racing with
> + * queue head waiter trying to set _QLOCK_LOCKED_SLOWPATH.
> + */
> + if (likely(cmpxchg((u8 *)lock, _Q_LOCKED_VAL, 0)
> + == _Q_LOCKED_VAL))
> + return;
> + else
> + queue_spin_unlock_slowpath(lock);
> +
> + } else {
> + __queue_spin_unlock(lock);
> + }
> + barrier();
> +}
> +#endif /* CONFIG_PARAVIRT_SPINLOCKS */
Ideally we'd make all this use alternatives or so, such that the actual
function remains short enough to actually inline;
static inline void queue_spin_unlock(struct qspinlock *lock)
{
pv_spinlock_alternative(
ACCESS_ONCE(*(u8 *)lock) = 0,
pv_queue_spin_unlock(lock));
}
Or however that trickery works.
[-- Attachment #2: Type: application/pgp-signature, Size: 836 bytes --]
^ permalink raw reply [flat|nested] 102+ messages in thread* Re: [PATCH v11 14/16] pvqspinlock: Add qspinlock para-virtualization support
@ 2014-06-12 8:17 ` Peter Zijlstra
0 siblings, 0 replies; 102+ messages in thread
From: Peter Zijlstra @ 2014-06-12 8:17 UTC (permalink / raw)
To: Waiman Long
Cc: linux-arch, Rik van Riel, Raghavendra K T, Oleg Nesterov,
Gleb Natapov, kvm, Konrad Rzeszutek Wilk, Scott J Norton, x86,
Paolo Bonzini, linux-kernel, virtualization, Ingo Molnar,
Chegu Vinod, David Vrabel, H. Peter Anvin, xen-devel,
Thomas Gleixner, Paul E. McKenney, Linus Torvalds,
Boris Ostrovsky
[-- Attachment #1.1: Type: text/plain, Size: 1756 bytes --]
On Fri, May 30, 2014 at 11:44:00AM -0400, Waiman Long wrote:
> @@ -19,13 +19,46 @@ extern struct static_key virt_unfairlocks_enabled;
> * that the clearing the lock bit is done ASAP without artificial delay
> * due to compiler optimization.
> */
> +#ifdef CONFIG_PARAVIRT_SPINLOCKS
> +static __always_inline void __queue_spin_unlock(struct qspinlock *lock)
> +#else
> static inline void queue_spin_unlock(struct qspinlock *lock)
> +#endif
> {
> barrier();
> ACCESS_ONCE(*(u8 *)lock) = 0;
> barrier();
> }
>
> +#ifdef CONFIG_PARAVIRT_SPINLOCKS
> +/*
> + * The lock byte can have a value of _Q_LOCKED_SLOWPATH to indicate
> + * that it needs to go through the slowpath to do the unlocking.
> + */
> +#define _Q_LOCKED_SLOWPATH (_Q_LOCKED_VAL | 2)
> +
> +extern void queue_spin_unlock_slowpath(struct qspinlock *lock);
> +
> +static inline void queue_spin_unlock(struct qspinlock *lock)
> +{
> + barrier();
> + if (static_key_false(¶virt_spinlocks_enabled)) {
> + /*
> + * Need to atomically clear the lock byte to avoid racing with
> + * queue head waiter trying to set _QLOCK_LOCKED_SLOWPATH.
> + */
> + if (likely(cmpxchg((u8 *)lock, _Q_LOCKED_VAL, 0)
> + == _Q_LOCKED_VAL))
> + return;
> + else
> + queue_spin_unlock_slowpath(lock);
> +
> + } else {
> + __queue_spin_unlock(lock);
> + }
> + barrier();
> +}
> +#endif /* CONFIG_PARAVIRT_SPINLOCKS */
Ideally we'd make all this use alternatives or so, such that the actual
function remains short enough to actually inline;
static inline void queue_spin_unlock(struct qspinlock *lock)
{
pv_spinlock_alternative(
ACCESS_ONCE(*(u8 *)lock) = 0,
pv_queue_spin_unlock(lock));
}
Or however that trickery works.
[-- Attachment #1.2: Type: application/pgp-signature, Size: 836 bytes --]
[-- Attachment #2: Type: text/plain, Size: 183 bytes --]
_______________________________________________
Virtualization mailing list
Virtualization@lists.linux-foundation.org
https://lists.linuxfoundation.org/mailman/listinfo/virtualization
^ permalink raw reply [flat|nested] 102+ messages in thread* Re: [PATCH v11 14/16] pvqspinlock: Add qspinlock para-virtualization support
2014-06-12 8:17 ` Peter Zijlstra
@ 2014-06-12 20:48 ` Waiman Long
-1 siblings, 0 replies; 102+ messages in thread
From: Waiman Long @ 2014-06-12 20:48 UTC (permalink / raw)
To: Peter Zijlstra
Cc: Thomas Gleixner, Ingo Molnar, H. Peter Anvin, linux-arch, x86,
linux-kernel, virtualization, xen-devel, kvm, Paolo Bonzini,
Konrad Rzeszutek Wilk, Boris Ostrovsky, Paul E. McKenney,
Rik van Riel, Linus Torvalds, Raghavendra K T, David Vrabel,
Oleg Nesterov, Gleb Natapov, Scott J Norton, Chegu Vinod
On 06/12/2014 04:17 AM, Peter Zijlstra wrote:
> On Fri, May 30, 2014 at 11:44:00AM -0400, Waiman Long wrote:
>> @@ -19,13 +19,46 @@ extern struct static_key virt_unfairlocks_enabled;
>> * that the clearing the lock bit is done ASAP without artificial delay
>> * due to compiler optimization.
>> */
>> +#ifdef CONFIG_PARAVIRT_SPINLOCKS
>> +static __always_inline void __queue_spin_unlock(struct qspinlock *lock)
>> +#else
>> static inline void queue_spin_unlock(struct qspinlock *lock)
>> +#endif
>> {
>> barrier();
>> ACCESS_ONCE(*(u8 *)lock) = 0;
>> barrier();
>> }
>>
>> +#ifdef CONFIG_PARAVIRT_SPINLOCKS
>> +/*
>> + * The lock byte can have a value of _Q_LOCKED_SLOWPATH to indicate
>> + * that it needs to go through the slowpath to do the unlocking.
>> + */
>> +#define _Q_LOCKED_SLOWPATH (_Q_LOCKED_VAL | 2)
>> +
>> +extern void queue_spin_unlock_slowpath(struct qspinlock *lock);
>> +
>> +static inline void queue_spin_unlock(struct qspinlock *lock)
>> +{
>> + barrier();
>> + if (static_key_false(¶virt_spinlocks_enabled)) {
>> + /*
>> + * Need to atomically clear the lock byte to avoid racing with
>> + * queue head waiter trying to set _QLOCK_LOCKED_SLOWPATH.
>> + */
>> + if (likely(cmpxchg((u8 *)lock, _Q_LOCKED_VAL, 0)
>> + == _Q_LOCKED_VAL))
>> + return;
>> + else
>> + queue_spin_unlock_slowpath(lock);
>> +
>> + } else {
>> + __queue_spin_unlock(lock);
>> + }
>> + barrier();
>> +}
>> +#endif /* CONFIG_PARAVIRT_SPINLOCKS */
> Ideally we'd make all this use alternatives or so, such that the actual
> function remains short enough to actually inline;
>
> static inline void queue_spin_unlock(struct qspinlock *lock)
> {
> pv_spinlock_alternative(
> ACCESS_ONCE(*(u8 *)lock) = 0,
> pv_queue_spin_unlock(lock));
> }
>
> Or however that trickery works.
I think the paravirt version of the unlock function is already short
enough. In addition, whenever PARAVIRT_SPINLOCKS is enabled, the
inlining of the unlock function is disabled so that the jump label
paravirt_spinlocks_enabled won't be used everywhere in the core kernel
and the kernel modules. This is true for the ticket spinlock and will
also be true for the queue spinlock.
I don't have a good understanding of the kernel alternatives mechanism.
I think it allow boot time modification of the kernel code according to
the CPU type, for example. The jump label mechanism is similar and is
much easier to use than the alternatives. I don't see a need to use the
alternatives unless you saw a big advantage of using it which I am not
aware of.
-Longman
^ permalink raw reply [flat|nested] 102+ messages in thread* Re: [PATCH v11 14/16] pvqspinlock: Add qspinlock para-virtualization support
@ 2014-06-12 20:48 ` Waiman Long
0 siblings, 0 replies; 102+ messages in thread
From: Waiman Long @ 2014-06-12 20:48 UTC (permalink / raw)
To: Peter Zijlstra
Cc: linux-arch, Rik van Riel, Raghavendra K T, Oleg Nesterov,
Gleb Natapov, kvm, Konrad Rzeszutek Wilk, Scott J Norton, x86,
Paolo Bonzini, linux-kernel, virtualization, Ingo Molnar,
Chegu Vinod, David Vrabel, H. Peter Anvin, xen-devel,
Thomas Gleixner, Paul E. McKenney, Linus Torvalds,
Boris Ostrovsky
On 06/12/2014 04:17 AM, Peter Zijlstra wrote:
> On Fri, May 30, 2014 at 11:44:00AM -0400, Waiman Long wrote:
>> @@ -19,13 +19,46 @@ extern struct static_key virt_unfairlocks_enabled;
>> * that the clearing the lock bit is done ASAP without artificial delay
>> * due to compiler optimization.
>> */
>> +#ifdef CONFIG_PARAVIRT_SPINLOCKS
>> +static __always_inline void __queue_spin_unlock(struct qspinlock *lock)
>> +#else
>> static inline void queue_spin_unlock(struct qspinlock *lock)
>> +#endif
>> {
>> barrier();
>> ACCESS_ONCE(*(u8 *)lock) = 0;
>> barrier();
>> }
>>
>> +#ifdef CONFIG_PARAVIRT_SPINLOCKS
>> +/*
>> + * The lock byte can have a value of _Q_LOCKED_SLOWPATH to indicate
>> + * that it needs to go through the slowpath to do the unlocking.
>> + */
>> +#define _Q_LOCKED_SLOWPATH (_Q_LOCKED_VAL | 2)
>> +
>> +extern void queue_spin_unlock_slowpath(struct qspinlock *lock);
>> +
>> +static inline void queue_spin_unlock(struct qspinlock *lock)
>> +{
>> + barrier();
>> + if (static_key_false(¶virt_spinlocks_enabled)) {
>> + /*
>> + * Need to atomically clear the lock byte to avoid racing with
>> + * queue head waiter trying to set _QLOCK_LOCKED_SLOWPATH.
>> + */
>> + if (likely(cmpxchg((u8 *)lock, _Q_LOCKED_VAL, 0)
>> + == _Q_LOCKED_VAL))
>> + return;
>> + else
>> + queue_spin_unlock_slowpath(lock);
>> +
>> + } else {
>> + __queue_spin_unlock(lock);
>> + }
>> + barrier();
>> +}
>> +#endif /* CONFIG_PARAVIRT_SPINLOCKS */
> Ideally we'd make all this use alternatives or so, such that the actual
> function remains short enough to actually inline;
>
> static inline void queue_spin_unlock(struct qspinlock *lock)
> {
> pv_spinlock_alternative(
> ACCESS_ONCE(*(u8 *)lock) = 0,
> pv_queue_spin_unlock(lock));
> }
>
> Or however that trickery works.
I think the paravirt version of the unlock function is already short
enough. In addition, whenever PARAVIRT_SPINLOCKS is enabled, the
inlining of the unlock function is disabled so that the jump label
paravirt_spinlocks_enabled won't be used everywhere in the core kernel
and the kernel modules. This is true for the ticket spinlock and will
also be true for the queue spinlock.
I don't have a good understanding of the kernel alternatives mechanism.
I think it allow boot time modification of the kernel code according to
the CPU type, for example. The jump label mechanism is similar and is
much easier to use than the alternatives. I don't see a need to use the
alternatives unless you saw a big advantage of using it which I am not
aware of.
-Longman
^ permalink raw reply [flat|nested] 102+ messages in thread* Re: [PATCH v11 14/16] pvqspinlock: Add qspinlock para-virtualization support
2014-06-12 20:48 ` Waiman Long
(?)
@ 2014-06-15 13:16 ` Peter Zijlstra
-1 siblings, 0 replies; 102+ messages in thread
From: Peter Zijlstra @ 2014-06-15 13:16 UTC (permalink / raw)
To: Waiman Long
Cc: linux-arch, Raghavendra K T, Oleg Nesterov, Gleb Natapov, kvm,
Scott J Norton, x86, Paolo Bonzini, linux-kernel, virtualization,
Ingo Molnar, Chegu Vinod, David Vrabel, H. Peter Anvin,
xen-devel, Thomas Gleixner, Paul E. McKenney, Linus Torvalds,
Boris Ostrovsky
On Thu, Jun 12, 2014 at 04:48:41PM -0400, Waiman Long wrote:
> I don't have a good understanding of the kernel alternatives mechanism.
I didn't either; I do now, cost me a whole day reading up on
alternative/paravirt code patching.
See the patches I just send out; I got the 'native' case with paravirt
enabled to be one NOP worse than the native case without paravirt -- for
queue_spin_unlock.
The lock slowpath is several nops and some pointless movs more expensive.
^ permalink raw reply [flat|nested] 102+ messages in thread
* Re: [PATCH v11 14/16] pvqspinlock: Add qspinlock para-virtualization support
2014-06-12 20:48 ` Waiman Long
@ 2014-06-15 13:16 ` Peter Zijlstra
-1 siblings, 0 replies; 102+ messages in thread
From: Peter Zijlstra @ 2014-06-15 13:16 UTC (permalink / raw)
To: Waiman Long
Cc: Thomas Gleixner, Ingo Molnar, H. Peter Anvin, linux-arch, x86,
linux-kernel, virtualization, xen-devel, kvm, Paolo Bonzini,
Konrad Rzeszutek Wilk, Boris Ostrovsky, Paul E. McKenney,
Rik van Riel, Linus Torvalds, Raghavendra K T, David Vrabel,
Oleg Nesterov, Gleb Natapov, Scott J Norton, Chegu Vinod
On Thu, Jun 12, 2014 at 04:48:41PM -0400, Waiman Long wrote:
> I don't have a good understanding of the kernel alternatives mechanism.
I didn't either; I do now, cost me a whole day reading up on
alternative/paravirt code patching.
See the patches I just send out; I got the 'native' case with paravirt
enabled to be one NOP worse than the native case without paravirt -- for
queue_spin_unlock.
The lock slowpath is several nops and some pointless movs more expensive.
^ permalink raw reply [flat|nested] 102+ messages in thread
* Re: [PATCH v11 14/16] pvqspinlock: Add qspinlock para-virtualization support
@ 2014-06-15 13:16 ` Peter Zijlstra
0 siblings, 0 replies; 102+ messages in thread
From: Peter Zijlstra @ 2014-06-15 13:16 UTC (permalink / raw)
To: Waiman Long
Cc: linux-arch, Rik van Riel, Raghavendra K T, Oleg Nesterov,
Gleb Natapov, kvm, Konrad Rzeszutek Wilk, Scott J Norton, x86,
Paolo Bonzini, linux-kernel, virtualization, Ingo Molnar,
Chegu Vinod, David Vrabel, H. Peter Anvin, xen-devel,
Thomas Gleixner, Paul E. McKenney, Linus Torvalds,
Boris Ostrovsky
On Thu, Jun 12, 2014 at 04:48:41PM -0400, Waiman Long wrote:
> I don't have a good understanding of the kernel alternatives mechanism.
I didn't either; I do now, cost me a whole day reading up on
alternative/paravirt code patching.
See the patches I just send out; I got the 'native' case with paravirt
enabled to be one NOP worse than the native case without paravirt -- for
queue_spin_unlock.
The lock slowpath is several nops and some pointless movs more expensive.
^ permalink raw reply [flat|nested] 102+ messages in thread
* Re: [PATCH v11 14/16] pvqspinlock: Add qspinlock para-virtualization support
2014-06-15 13:16 ` Peter Zijlstra
(?)
@ 2014-06-17 20:59 ` Konrad Rzeszutek Wilk
-1 siblings, 0 replies; 102+ messages in thread
From: Konrad Rzeszutek Wilk @ 2014-06-17 20:59 UTC (permalink / raw)
To: Peter Zijlstra
Cc: Waiman Long, linux-arch, Raghavendra K T, Gleb Natapov, kvm,
Oleg Nesterov, Scott J Norton, x86, Paolo Bonzini, linux-kernel,
virtualization, Ingo Molnar, Chegu Vinod, David Vrabel,
H. Peter Anvin, xen-devel, Thomas Gleixner, Paul E. McKenney,
Linus Torvalds, Boris Ostrovsky
On Sun, Jun 15, 2014 at 03:16:54PM +0200, Peter Zijlstra wrote:
> On Thu, Jun 12, 2014 at 04:48:41PM -0400, Waiman Long wrote:
> > I don't have a good understanding of the kernel alternatives mechanism.
>
> I didn't either; I do now, cost me a whole day reading up on
> alternative/paravirt code patching.
>
> See the patches I just send out; I got the 'native' case with paravirt
> enabled to be one NOP worse than the native case without paravirt -- for
> queue_spin_unlock.
>
> The lock slowpath is several nops and some pointless movs more expensive.
You could use the asm goto which would optimize the fast path to be the
'native' case. That way you wouldn't have the the nops and movs in the
path.
(And asm goto also uses the alternative_asm macros).
^ permalink raw reply [flat|nested] 102+ messages in thread
* Re: [PATCH v11 14/16] pvqspinlock: Add qspinlock para-virtualization support
2014-06-15 13:16 ` Peter Zijlstra
@ 2014-06-17 20:59 ` Konrad Rzeszutek Wilk
-1 siblings, 0 replies; 102+ messages in thread
From: Konrad Rzeszutek Wilk @ 2014-06-17 20:59 UTC (permalink / raw)
To: Peter Zijlstra
Cc: Waiman Long, Thomas Gleixner, Ingo Molnar, H. Peter Anvin,
linux-arch, x86, linux-kernel, virtualization, xen-devel, kvm,
Paolo Bonzini, Boris Ostrovsky, Paul E. McKenney, Rik van Riel,
Linus Torvalds, Raghavendra K T, David Vrabel, Oleg Nesterov,
Gleb Natapov, Scott J Norton, Chegu Vinod
On Sun, Jun 15, 2014 at 03:16:54PM +0200, Peter Zijlstra wrote:
> On Thu, Jun 12, 2014 at 04:48:41PM -0400, Waiman Long wrote:
> > I don't have a good understanding of the kernel alternatives mechanism.
>
> I didn't either; I do now, cost me a whole day reading up on
> alternative/paravirt code patching.
>
> See the patches I just send out; I got the 'native' case with paravirt
> enabled to be one NOP worse than the native case without paravirt -- for
> queue_spin_unlock.
>
> The lock slowpath is several nops and some pointless movs more expensive.
You could use the asm goto which would optimize the fast path to be the
'native' case. That way you wouldn't have the the nops and movs in the
path.
(And asm goto also uses the alternative_asm macros).
^ permalink raw reply [flat|nested] 102+ messages in thread
* Re: [PATCH v11 14/16] pvqspinlock: Add qspinlock para-virtualization support
@ 2014-06-17 20:59 ` Konrad Rzeszutek Wilk
0 siblings, 0 replies; 102+ messages in thread
From: Konrad Rzeszutek Wilk @ 2014-06-17 20:59 UTC (permalink / raw)
To: Peter Zijlstra
Cc: Waiman Long, linux-arch, Rik van Riel, Raghavendra K T,
Gleb Natapov, kvm, Oleg Nesterov, Scott J Norton, x86,
Paolo Bonzini, linux-kernel, virtualization, Ingo Molnar,
Chegu Vinod, David Vrabel, H. Peter Anvin, xen-devel,
Thomas Gleixner, Paul E. McKenney, Linus Torvalds,
Boris Ostrovsky
On Sun, Jun 15, 2014 at 03:16:54PM +0200, Peter Zijlstra wrote:
> On Thu, Jun 12, 2014 at 04:48:41PM -0400, Waiman Long wrote:
> > I don't have a good understanding of the kernel alternatives mechanism.
>
> I didn't either; I do now, cost me a whole day reading up on
> alternative/paravirt code patching.
>
> See the patches I just send out; I got the 'native' case with paravirt
> enabled to be one NOP worse than the native case without paravirt -- for
> queue_spin_unlock.
>
> The lock slowpath is several nops and some pointless movs more expensive.
You could use the asm goto which would optimize the fast path to be the
'native' case. That way you wouldn't have the the nops and movs in the
path.
(And asm goto also uses the alternative_asm macros).
^ permalink raw reply [flat|nested] 102+ messages in thread
* Re: [PATCH v11 14/16] pvqspinlock: Add qspinlock para-virtualization support
2014-06-12 8:17 ` Peter Zijlstra
(?)
(?)
@ 2014-06-12 20:48 ` Waiman Long
-1 siblings, 0 replies; 102+ messages in thread
From: Waiman Long @ 2014-06-12 20:48 UTC (permalink / raw)
To: Peter Zijlstra
Cc: linux-arch, Raghavendra K T, Oleg Nesterov, Gleb Natapov, kvm,
Scott J Norton, x86, Paolo Bonzini, linux-kernel, virtualization,
Ingo Molnar, Chegu Vinod, David Vrabel, H. Peter Anvin,
xen-devel, Thomas Gleixner, Paul E. McKenney, Linus Torvalds,
Boris Ostrovsky
On 06/12/2014 04:17 AM, Peter Zijlstra wrote:
> On Fri, May 30, 2014 at 11:44:00AM -0400, Waiman Long wrote:
>> @@ -19,13 +19,46 @@ extern struct static_key virt_unfairlocks_enabled;
>> * that the clearing the lock bit is done ASAP without artificial delay
>> * due to compiler optimization.
>> */
>> +#ifdef CONFIG_PARAVIRT_SPINLOCKS
>> +static __always_inline void __queue_spin_unlock(struct qspinlock *lock)
>> +#else
>> static inline void queue_spin_unlock(struct qspinlock *lock)
>> +#endif
>> {
>> barrier();
>> ACCESS_ONCE(*(u8 *)lock) = 0;
>> barrier();
>> }
>>
>> +#ifdef CONFIG_PARAVIRT_SPINLOCKS
>> +/*
>> + * The lock byte can have a value of _Q_LOCKED_SLOWPATH to indicate
>> + * that it needs to go through the slowpath to do the unlocking.
>> + */
>> +#define _Q_LOCKED_SLOWPATH (_Q_LOCKED_VAL | 2)
>> +
>> +extern void queue_spin_unlock_slowpath(struct qspinlock *lock);
>> +
>> +static inline void queue_spin_unlock(struct qspinlock *lock)
>> +{
>> + barrier();
>> + if (static_key_false(¶virt_spinlocks_enabled)) {
>> + /*
>> + * Need to atomically clear the lock byte to avoid racing with
>> + * queue head waiter trying to set _QLOCK_LOCKED_SLOWPATH.
>> + */
>> + if (likely(cmpxchg((u8 *)lock, _Q_LOCKED_VAL, 0)
>> + == _Q_LOCKED_VAL))
>> + return;
>> + else
>> + queue_spin_unlock_slowpath(lock);
>> +
>> + } else {
>> + __queue_spin_unlock(lock);
>> + }
>> + barrier();
>> +}
>> +#endif /* CONFIG_PARAVIRT_SPINLOCKS */
> Ideally we'd make all this use alternatives or so, such that the actual
> function remains short enough to actually inline;
>
> static inline void queue_spin_unlock(struct qspinlock *lock)
> {
> pv_spinlock_alternative(
> ACCESS_ONCE(*(u8 *)lock) = 0,
> pv_queue_spin_unlock(lock));
> }
>
> Or however that trickery works.
I think the paravirt version of the unlock function is already short
enough. In addition, whenever PARAVIRT_SPINLOCKS is enabled, the
inlining of the unlock function is disabled so that the jump label
paravirt_spinlocks_enabled won't be used everywhere in the core kernel
and the kernel modules. This is true for the ticket spinlock and will
also be true for the queue spinlock.
I don't have a good understanding of the kernel alternatives mechanism.
I think it allow boot time modification of the kernel code according to
the CPU type, for example. The jump label mechanism is similar and is
much easier to use than the alternatives. I don't see a need to use the
alternatives unless you saw a big advantage of using it which I am not
aware of.
-Longman
^ permalink raw reply [flat|nested] 102+ messages in thread
* Re: [PATCH v11 14/16] pvqspinlock: Add qspinlock para-virtualization support
2014-05-30 15:44 ` [PATCH v11 14/16] pvqspinlock: Add qspinlock para-virtualization support Waiman Long
2014-06-12 8:17 ` Peter Zijlstra
@ 2014-06-12 8:17 ` Peter Zijlstra
1 sibling, 0 replies; 102+ messages in thread
From: Peter Zijlstra @ 2014-06-12 8:17 UTC (permalink / raw)
To: Waiman Long
Cc: linux-arch, Raghavendra K T, Oleg Nesterov, Gleb Natapov, kvm,
Scott J Norton, x86, Paolo Bonzini, linux-kernel, virtualization,
Ingo Molnar, Chegu Vinod, David Vrabel, H. Peter Anvin,
xen-devel, Thomas Gleixner, Paul E. McKenney, Linus Torvalds,
Boris Ostrovsky
[-- Attachment #1.1: Type: text/plain, Size: 1756 bytes --]
On Fri, May 30, 2014 at 11:44:00AM -0400, Waiman Long wrote:
> @@ -19,13 +19,46 @@ extern struct static_key virt_unfairlocks_enabled;
> * that the clearing the lock bit is done ASAP without artificial delay
> * due to compiler optimization.
> */
> +#ifdef CONFIG_PARAVIRT_SPINLOCKS
> +static __always_inline void __queue_spin_unlock(struct qspinlock *lock)
> +#else
> static inline void queue_spin_unlock(struct qspinlock *lock)
> +#endif
> {
> barrier();
> ACCESS_ONCE(*(u8 *)lock) = 0;
> barrier();
> }
>
> +#ifdef CONFIG_PARAVIRT_SPINLOCKS
> +/*
> + * The lock byte can have a value of _Q_LOCKED_SLOWPATH to indicate
> + * that it needs to go through the slowpath to do the unlocking.
> + */
> +#define _Q_LOCKED_SLOWPATH (_Q_LOCKED_VAL | 2)
> +
> +extern void queue_spin_unlock_slowpath(struct qspinlock *lock);
> +
> +static inline void queue_spin_unlock(struct qspinlock *lock)
> +{
> + barrier();
> + if (static_key_false(¶virt_spinlocks_enabled)) {
> + /*
> + * Need to atomically clear the lock byte to avoid racing with
> + * queue head waiter trying to set _QLOCK_LOCKED_SLOWPATH.
> + */
> + if (likely(cmpxchg((u8 *)lock, _Q_LOCKED_VAL, 0)
> + == _Q_LOCKED_VAL))
> + return;
> + else
> + queue_spin_unlock_slowpath(lock);
> +
> + } else {
> + __queue_spin_unlock(lock);
> + }
> + barrier();
> +}
> +#endif /* CONFIG_PARAVIRT_SPINLOCKS */
Ideally we'd make all this use alternatives or so, such that the actual
function remains short enough to actually inline;
static inline void queue_spin_unlock(struct qspinlock *lock)
{
pv_spinlock_alternative(
ACCESS_ONCE(*(u8 *)lock) = 0,
pv_queue_spin_unlock(lock));
}
Or however that trickery works.
[-- Attachment #1.2: Type: application/pgp-signature, Size: 836 bytes --]
[-- Attachment #2: Type: text/plain, Size: 126 bytes --]
_______________________________________________
Xen-devel mailing list
Xen-devel@lists.xen.org
http://lists.xen.org/xen-devel
^ permalink raw reply [flat|nested] 102+ messages in thread
* [PATCH v11 14/16] pvqspinlock: Add qspinlock para-virtualization support
2014-05-30 15:43 [PATCH v11 00/16] qspinlock: a 4-byte queue spinlock with PV support Waiman Long
` (29 preceding siblings ...)
2014-05-30 15:44 ` [PATCH v11 14/16] pvqspinlock: Add qspinlock para-virtualization support Waiman Long
@ 2014-05-30 15:44 ` Waiman Long
2014-05-30 15:44 ` Waiman Long
` (6 subsequent siblings)
37 siblings, 0 replies; 102+ messages in thread
From: Waiman Long @ 2014-05-30 15:44 UTC (permalink / raw)
To: Thomas Gleixner, Ingo Molnar, H. Peter Anvin, Peter Zijlstra
Cc: linux-arch, Waiman Long, Raghavendra K T, Gleb Natapov, kvm,
Scott J Norton, x86, Paolo Bonzini, linux-kernel, virtualization,
Chegu Vinod, David Vrabel, Oleg Nesterov, xen-devel,
Boris Ostrovsky, Paul E. McKenney, Linus Torvalds
This patch adds base para-virtualization support to the queue
spinlock in the same way as was done in the PV ticket lock code. In
essence, the lock waiters will spin for a specified number of times
(QSPIN_THRESHOLD = 2^14) and then halted itself. The queue head waiter,
unlike the other waiter, will spins 2*QSPIN_THRESHOLD times before
halting itself. Before being halted, the queue head waiter will set
a flag (_Q_LOCKED_SLOWPATH) in the lock byte to indicate that the
unlock slowpath has to be invoked.
In the unlock slowpath, the current lock holder will find the queue
head by following the previous node pointer links stored in the queue
node structure until it finds one that has the qhead flag turned
on. It then attempt to kick in the CPU of the queue head.
After the queue head acquired the lock, it will also check the status
of the next node and set _Q_LOCKED_SLOWPATH flag if it has been halted.
Enabling the PV code does have a performance impact on spinlock
acquisitions and releases. The following table shows the execution
time (in ms) of a spinlock micro-benchmark that does lock/unlock
operations 5M times for each task versus the number of contending
tasks on a Westmere-EX system.
# of Ticket lock Queue lock
tasks PV off/PV on/%Change PV off/PV on/%Change
------ -------------------- ---------------------
1 135/ 179/+33% 137/ 168/+23%
2 1045/ 1103/ +6% 1161/ 1248/ +7%
3 1827/ 2683/+47% 2357/ 2600/+10%
4 2689/ 4191/+56% 2882/ 3115/ +8%
5 3736/ 5830/+56% 3493/ 3571/ +2%
6 4942/ 7609/+54% 4239/ 4198/ -1%
7 6304/ 9570/+52% 4931/ 4895/ -1%
8 7736/11323/+46% 5632/ 5588/ -1%
It can be seen that the ticket lock PV code has a fairly big decrease
in performance when there are 3 or more contending tasks. The queue
spinlock PV code, on the other hand, only has a relatively minor
drop in performance for with 1-4 contending tasks. With 5 or more
contending tasks, there is practically no difference in performance.
Signed-off-by: Waiman Long <Waiman.Long@hp.com>
---
arch/x86/include/asm/pvqspinlock.h | 359 ++++++++++++++++++++++++++++++++++++
arch/x86/include/asm/qspinlock.h | 33 ++++
kernel/locking/qspinlock.c | 72 +++++++-
3 files changed, 458 insertions(+), 6 deletions(-)
create mode 100644 arch/x86/include/asm/pvqspinlock.h
diff --git a/arch/x86/include/asm/pvqspinlock.h b/arch/x86/include/asm/pvqspinlock.h
new file mode 100644
index 0000000..af00eda
--- /dev/null
+++ b/arch/x86/include/asm/pvqspinlock.h
@@ -0,0 +1,359 @@
+#ifndef _ASM_X86_PVQSPINLOCK_H
+#define _ASM_X86_PVQSPINLOCK_H
+
+/*
+ * Queue Spinlock Para-Virtualization (PV) Support
+ *
+ * +------+ +------+ +------+
+ * pv_qnode |Queue | prev | | prev |Queue |
+ * | Head |<-------| Node | .... <-| Tail |
+ * +------+ +------+ +------+
+ * | | |
+ * V V V
+ * +------+ +------+ +------+
+ * mcs_spinlock |locked| |locked| |locked|
+ * | = 1 |------->| = 0 |-> .... | = 0 |
+ * +------+ next +------+ next +------+
+ *
+ * The PV support code for queue spinlock is roughly the same as that
+ * of the ticket spinlock. Each CPU waiting for the lock will spin until it
+ * reaches a threshold. When that happens, it will put itself to halt so
+ * that the hypervisor can reuse the CPU cycles in some other guests as well
+ * as returning other hold-up CPUs faster.
+ *
+ * Auxillary fields in the pv_qnode structure are used to hold information
+ * relevant to the PV support so that it won't impact on the behavior and
+ * performance of the bare metal code. The structure contains a prev pointer
+ * so that a lock holder can find out the queue head from the queue tail
+ * following the prev pointers.
+ *
+ * A major difference between the two versions of PV spinlock is the fact
+ * that the spin threshold of the queue spinlock is half of that of the
+ * ticket spinlock. However, the queue head will spin twice as long as the
+ * other nodes before it puts itself to halt. The reason for that is to
+ * increase halting chance of heavily contended locks to favor lightly
+ * contended locks (queue depth of 1 or less).
+ *
+ * There are 2 places where races can happen:
+ * 1) Halting of the queue head CPU (in pv_head_spin_check) and the CPU
+ * kicking by the lock holder in the unlock path (in pv_kick_node).
+ * 2) Halting of the queue node CPU (in pv_queue_spin_check) and the
+ * the status check by the previous queue head (in pv_halt_check).
+ * See the comments on those functions to see how the races are being
+ * addressed.
+ */
+
+/*
+ * Spin threshold for queue spinlock
+ */
+#define QSPIN_THRESHOLD (1U<<14)
+#define MAYHALT_THRESHOLD (QSPIN_THRESHOLD - 0x10)
+
+/*
+ * CPU state flags
+ */
+#define PV_CPU_ACTIVE 1 /* This CPU is active */
+#define PV_CPU_KICKED 2 /* This CPU is being kicked */
+#define PV_CPU_HALTED -1 /* This CPU is halted */
+
+/*
+ * Additional fields to be added to the queue node structure
+ *
+ * The size of the mcs_spinlock structure is 16 bytes for x64 and 12 bytes
+ * for i386. Four of those structures are defined per CPU. To add more fields
+ * without increasing the size of the mcs_spinlock structure, we overlay those
+ * additional data fields at an additional mcs_spinlock size bucket at exactly
+ * 3 units away. As a result, we need to double the number of mcs_spinlock
+ * buckets. The mcs_spinlock structure will be casted to the pv_qnode
+ * internally.
+ *
+ * +------------+------------+------------+------------+
+ * | MCS Node 0 | MCS Node 1 | MCS Node 2 | MCS Node 3 |
+ * +------------+------------+------------+------------+
+ * | PV Node 0 | PV Node 1 | PV Node 2 | PV Node 3 |
+ * +------------+------------+------------+------------+
+ */
+struct pv_qnode {
+ struct mcs_spinlock mcs; /* MCS node */
+ struct mcs_spinlock dummy[3]; /* 3 dummy MCS nodes */
+ s8 cpustate; /* CPU status flag */
+ s8 mayhalt; /* May be halted soon */
+ u32 mycpu; /* CPU number of this node */
+ struct pv_qnode *prev; /* Pointer to previous node */
+};
+
+#define qhead mcs.locked /* Queue head flag */
+
+/**
+ * pv_init_vars - initialize fields in struct pv_qnode
+ * @mcs: pointer to struct mcs_spinlock
+ * @cpu: current CPU number
+ */
+static inline void pv_init_vars(struct mcs_spinlock *node, int cpu)
+{
+ struct pv_qnode *pv = (struct pv_qnode *)node;
+
+ BUILD_BUG_ON(sizeof(struct pv_qnode) > 5*sizeof(struct mcs_spinlock));
+
+ if (!static_key_false(¶virt_spinlocks_enabled))
+ return;
+
+ pv->cpustate = PV_CPU_ACTIVE;
+ pv->prev = NULL;
+ pv->mayhalt = false;
+ pv->mycpu = cpu;
+}
+
+/**
+ * pv_head_spin_check - perform para-virtualization checks for queue head
+ * @mcs : pointer to the mcs_spinlock structure
+ * @lock : pointer to the qspinlock structure
+ * @count: loop count pointer
+ *
+ * This function will halt itself if lock is still not available after
+ * QSPIN_THRESHOLD iterations.
+ */
+static inline void
+pv_head_spin_check(struct mcs_spinlock *mcs, struct qspinlock *lock, int *count)
+{
+ struct pv_qnode *pv = (struct pv_qnode *)mcs;
+
+ if (!static_key_false(¶virt_spinlocks_enabled))
+ return;
+
+ (*count)++;
+ if (pv->cpustate == PV_CPU_KICKED) {
+ /*
+ * Reset count and flag
+ */
+ pv->cpustate = PV_CPU_ACTIVE;
+ *count = 0;
+
+ } else if (unlikely(*count >= 2*QSPIN_THRESHOLD)) {
+ u8 lockval;
+ s8 oldstate;
+
+ /*
+ * Set the lock byte to _Q_LOCKED_SLOWPATH before
+ * trying to halt itself. It is possible that the
+ * lock byte had been set to _Q_LOCKED_SLOWPATH
+ * already (spurious wakeup of queue head after a halt
+ * or opportunistic setting in pv_halt_check()).
+ * In this case, just proceeds to sleeping.
+ *
+ * queue head lock holder
+ * ---------- -----------
+ * cpustate = PV_CPU_HALTED
+ * [1] cmpxchg(_Q_LOCKED_VAL [2] cmpxchg(_Q_LOCKED_VAL => 0)
+ * => _Q_LOCKED_SLOWPATH) if (cmpxchg fails &&
+ * if (cmpxchg succeeds) cpustate == PV_CPU_HALTED)
+ * halt() kick()
+ *
+ * Sequence:
+ * 1,2 - slowpath flag set, queue head halted & lock holder
+ * will call slowpath
+ * 2,1 - queue head cmpxchg fails, halt is aborted
+ *
+ * If the queue head CPU is woken up by a spurious interrupt
+ * at the same time as the lock holder check the cpustate,
+ * it is possible that the lock holder will try to kick
+ * the queue head CPU which isn't halted.
+ */
+ oldstate = cmpxchg(&pv->cpustate, PV_CPU_ACTIVE, PV_CPU_HALTED);
+ if (oldstate == PV_CPU_KICKED)
+ goto reset; /* Reset count and state */
+
+ lockval = cmpxchg((u8 *)lock,
+ _Q_LOCKED_VAL, _Q_LOCKED_SLOWPATH);
+ if (lockval != 0) {
+ __queue_halt_cpu(PV_HALT_QHEAD, &pv->cpustate,
+ PV_CPU_HALTED);
+ __queue_lockstat((pv->cpustate == PV_CPU_KICKED)
+ ? PV_WAKE_KICKED : PV_WAKE_SPURIOUS);
+ }
+ /*
+ * Else, the lock is free and no halting is needed
+ */
+reset:
+ ACCESS_ONCE(pv->cpustate) = PV_CPU_ACTIVE;
+ *count = 0; /* Reset count */
+ }
+}
+
+/**
+ * pv_queue_spin_check - perform para-virtualization checks for queue member
+ * @mcs : pointer to the mcs_spinlock structure
+ * @count: loop count pointer
+ */
+static inline void pv_queue_spin_check(struct mcs_spinlock *mcs, int *count)
+{
+ struct pv_qnode *pv = (struct pv_qnode *)mcs;
+
+ if (!static_key_false(¶virt_spinlocks_enabled))
+ return;
+ /*
+ * Attempt to halt oneself after QSPIN_THRESHOLD spins
+ */
+ (*count)++;
+ if (unlikely(*count >= QSPIN_THRESHOLD)) {
+ /*
+ * Time to halt itself
+ */
+ ACCESS_ONCE(pv->cpustate) = PV_CPU_HALTED;
+ /*
+ * One way to avoid the racing between pv_halt_check()
+ * and pv_queue_spin_check() is to use memory barrier or
+ * atomic instruction to synchronize between the two competing
+ * threads. However, that will slow down the queue spinlock
+ * slowpath. One way to eliminate this overhead for normal
+ * cases is to use another flag (mayhalt) to indicate that
+ * racing condition may happen. This flag is set when the
+ * loop count is getting close to the halting threshold.
+ *
+ * When that happens, a 2 variables (cpustate & qhead
+ * [=mcs->locked]) handshake is used to make sure that
+ * pv_halt_check() won't miss setting the _Q_LOCKED_SLOWPATH
+ * when the CPU is about to be halted.
+ *
+ * pv_halt_check pv_queue_spin_check
+ * ------------- -------------------
+ * [1] qhead = true [3] cpustate = PV_CPU_HALTED
+ * smp_mb() smp_mb()
+ * [2] if (cpustate [4] if (qhead)
+ * == PV_CPU_HALTED)
+ *
+ * Sequence:
+ * *,1,*,4,* - halt is aborted as the qhead flag is set,
+ * _Q_LOCKED_SLOWPATH may or may not be set
+ * 3,4,1,2 - the CPU is halt and _Q_LOCKED_SLOWPATH is set
+ */
+ smp_mb();
+ if (!ACCESS_ONCE(pv->qhead)) {
+ /*
+ * Halt the CPU only if it is not the queue head
+ */
+ __queue_halt_cpu(PV_HALT_QNODE, &pv->cpustate,
+ PV_CPU_HALTED);
+ __queue_lockstat((pv->cpustate == PV_CPU_KICKED)
+ ? PV_WAKE_KICKED : PV_WAKE_SPURIOUS);
+ }
+ ACCESS_ONCE(pv->cpustate) = PV_CPU_ACTIVE;
+ *count = 0; /* Reset count & flag */
+ pv->mayhalt = false;
+ } else if (*count == MAYHALT_THRESHOLD) {
+ pv->mayhalt = true;
+ /*
+ * Make sure that the mayhalt flag is visible to others
+ * before proceeding.
+ */
+ smp_mb();
+ }
+}
+
+/**
+ * pv_halt_check - check if the CPU has been halted & set _Q_LOCKED_SLOWPATH
+ * @mcs : pointer to the mcs_spinlock structure
+ * @count: loop count
+ *
+ * The current CPU should have gotten the lock and the queue head flag set
+ * before calling this function.
+ */
+static inline void
+pv_halt_check(struct mcs_spinlock *mcs, struct qspinlock *lock)
+{
+ struct pv_qnode *pv = (struct pv_qnode *)mcs;
+
+ if (!static_key_false(¶virt_spinlocks_enabled))
+ return;
+ /*
+ * Halt state checking will only be done if the mayhalt flag is set
+ * to avoid the overhead of the memory barrier in normal cases.
+ * It is highly unlikely that the actual writing to the qhead flag
+ * will be more than 0x10 iterations later than the reading of the
+ * mayhalt flag so that it misses seeing the PV_CPU_HALTED state
+ * which causes lost wakeup.
+ */
+ if (ACCESS_ONCE(pv->mayhalt)) {
+ /*
+ * A memory barrier is used here to make sure that the setting
+ * of queue head flag prior to this function call is visible
+ * to others before checking the cpustate flag.
+ */
+ smp_mb();
+ if (pv->cpustate == PV_CPU_HALTED)
+ ACCESS_ONCE(*(u8 *)lock) = _Q_LOCKED_SLOWPATH;
+ }
+}
+
+/**
+ * pv_set_prev - set previous queue node pointer
+ * @mcs : pointer to the mcs_spinlock structure
+ * @prev: pointer to the previous pv_qnode
+ */
+static inline void
+pv_set_prev(struct mcs_spinlock *mcs, struct mcs_spinlock *prev)
+{
+ struct pv_qnode *pv = (struct pv_qnode *)mcs;
+
+ ACCESS_ONCE(pv->prev) = (struct pv_qnode *)prev;
+ /*
+ * Make sure the prev field is set up before others
+ */
+ smp_wmb();
+}
+
+/*
+ * The following inlined functions are being used by the
+ * queue_spin_unlock_slowpath() function.
+ */
+
+/**
+ * pv_tail_to_qhead - get queue head pv_qnode from tail code
+ * @tail : pointer to queue tail code
+ * @Return: mcs_spinlock pointer of queue head
+ *
+ * This function should only be called by the current lock holder so that
+ * the queue head won't be changed.
+ */
+static struct mcs_spinlock *pv_tail_to_qhead(u32 tail)
+{
+ struct pv_qnode *prev, *pv = (struct pv_qnode *)decode_tail(tail);
+
+ /*
+ * Locate the queue head node by following the prev pointer from
+ * tail to head. It is assumed that the PV guests won't have that
+ * many CPUs so that it won't take a long time to follow the pointers.
+ */
+ while (!ACCESS_ONCE(pv->qhead)) {
+ prev = ACCESS_ONCE(pv->prev);
+ if (prev)
+ pv = prev;
+ else
+ /*
+ * Delay a bit to allow the prev pointer or the queue
+ * head flag to be set up
+ */
+ cpu_relax();
+ }
+ return (struct mcs_spinlock *)pv;
+}
+
+/**
+ * pv_kick_node - kick up the CPU of the given node
+ * @mcs : pointer to struct mcs_spinlock of the node to be kicked
+ */
+static inline void pv_kick_node(struct mcs_spinlock *mcs)
+{
+ struct pv_qnode *pv = (struct pv_qnode *)mcs;
+ s8 oldstate = xchg(&pv->cpustate, PV_CPU_KICKED);
+
+ /*
+ * Kick up the CPU only if the state was set to PV_CPU_HALTED
+ */
+ if (oldstate != PV_CPU_HALTED)
+ __queue_lockstat(PV_KICK_NOHALT);
+ else
+ __queue_kick_cpu(pv->mycpu);
+}
+
+#endif /* _ASM_X86_PVQSPINLOCK_H */
diff --git a/arch/x86/include/asm/qspinlock.h b/arch/x86/include/asm/qspinlock.h
index 448de8b..5ddc456 100644
--- a/arch/x86/include/asm/qspinlock.h
+++ b/arch/x86/include/asm/qspinlock.h
@@ -19,13 +19,46 @@ extern struct static_key virt_unfairlocks_enabled;
* that the clearing the lock bit is done ASAP without artificial delay
* due to compiler optimization.
*/
+#ifdef CONFIG_PARAVIRT_SPINLOCKS
+static __always_inline void __queue_spin_unlock(struct qspinlock *lock)
+#else
static inline void queue_spin_unlock(struct qspinlock *lock)
+#endif
{
barrier();
ACCESS_ONCE(*(u8 *)lock) = 0;
barrier();
}
+#ifdef CONFIG_PARAVIRT_SPINLOCKS
+/*
+ * The lock byte can have a value of _Q_LOCKED_SLOWPATH to indicate
+ * that it needs to go through the slowpath to do the unlocking.
+ */
+#define _Q_LOCKED_SLOWPATH (_Q_LOCKED_VAL | 2)
+
+extern void queue_spin_unlock_slowpath(struct qspinlock *lock);
+
+static inline void queue_spin_unlock(struct qspinlock *lock)
+{
+ barrier();
+ if (static_key_false(¶virt_spinlocks_enabled)) {
+ /*
+ * Need to atomically clear the lock byte to avoid racing with
+ * queue head waiter trying to set _QLOCK_LOCKED_SLOWPATH.
+ */
+ if (likely(cmpxchg((u8 *)lock, _Q_LOCKED_VAL, 0)
+ == _Q_LOCKED_VAL))
+ return;
+ else
+ queue_spin_unlock_slowpath(lock);
+
+ } else {
+ __queue_spin_unlock(lock);
+ }
+ barrier();
+}
+#endif /* CONFIG_PARAVIRT_SPINLOCKS */
#endif /* !CONFIG_X86_OOSTORE && !CONFIG_X86_PPRO_FENCE */
#include <asm-generic/qspinlock.h>
diff --git a/kernel/locking/qspinlock.c b/kernel/locking/qspinlock.c
index 8deedcf..adedd75 100644
--- a/kernel/locking/qspinlock.c
+++ b/kernel/locking/qspinlock.c
@@ -60,12 +60,17 @@
* Check the pending bit spinning threshold only if PV qspinlock is enabled
*/
#define PSPIN_THRESHOLD (1 << 10)
-#define MAX_NODES 4
+/*
+ * We need to double the number of per-cpu mcs_spinlock structures to hold
+ * additional fields specific to para-virtualization support.
+ */
#ifdef CONFIG_PARAVIRT_SPINLOCKS
-#define pv_qspinlock_enabled() static_key_false(¶virt_spinlocks_enabled)
+# define MAX_NODES 8
+# define pv_qspinlock_enabled() static_key_false(¶virt_spinlocks_enabled)
#else
-#define pv_qspinlock_enabled() false
+# define MAX_NODES 4
+# define pv_qspinlock_enabled() false
#endif
/*
@@ -243,6 +248,22 @@ static __always_inline int try_set_locked(struct qspinlock *lock)
return 1;
}
+/*
+ * Para-virtualization (PV) queue spinlock support
+ */
+#ifdef CONFIG_PARAVIRT_SPINLOCKS
+#include <asm/pvqspinlock.h>
+#else
+static inline void pv_init_vars(struct mcs_spinlock *mcs, int cpu) {}
+static inline void pv_head_spin_check(struct mcs_spinlock *mcs,
+ struct qspinlock *lock, int *count) {}
+static inline void pv_queue_spin_check(struct mcs_spinlock *mcs,
+ int *count) {}
+static inline void pv_halt_check(struct mcs_spinlock *mcs, void *lock) {}
+static inline void pv_set_prev(struct mcs_spinlock *mcs,
+ struct mcs_spinlock *prev) {}
+#endif /* CONFIG_PARAVIRT_SPINLOCKS */
+
/**
* queue_spin_lock_slowerpath - a slower patch for acquiring queue spinlock
* @lock: Pointer to queue spinlock structure
@@ -260,6 +281,7 @@ static noinline void queue_spin_lock_slowerpath(struct qspinlock *lock,
{
struct mcs_spinlock *prev, *next;
u32 val, old;
+ int hcnt = 0; /* Queue head loop counter */
/*
* we already touched the queueing cacheline; don't bother with pending
@@ -273,9 +295,16 @@ static noinline void queue_spin_lock_slowerpath(struct qspinlock *lock,
* if there was a previous node; link it and wait.
*/
if (old & _Q_TAIL_MASK) {
+ int qcnt = 0; /* Queue node loop counter */
+
prev = decode_tail(old);
+ pv_set_prev(node, prev);
ACCESS_ONCE(prev->next) = node;
+ while (!smp_load_acquire(&node->locked)) {
+ pv_queue_spin_check(node, &qcnt);
+ arch_mutex_cpu_relax();
+ }
arch_mcs_spin_lock_contended(&node->locked);
} else {
/* Mark it as the queue head */
@@ -292,8 +321,14 @@ static noinline void queue_spin_lock_slowerpath(struct qspinlock *lock,
*/
retry_queue_wait:
while ((val = smp_load_acquire(&lock->val.counter))
- & _Q_LOCKED_PENDING_MASK)
+ & _Q_LOCKED_PENDING_MASK) {
+ /*
+ * Perform queue head para-virtualization checks
+ */
+ pv_head_spin_check(node, lock, &hcnt);
arch_mutex_cpu_relax();
+ }
+ hcnt = 0; /* Reset loop count */
/*
* claim the lock:
@@ -331,6 +366,7 @@ retry_queue_wait:
arch_mutex_cpu_relax();
arch_mcs_spin_unlock_contended(&next->locked);
+ pv_halt_check(next, lock);
}
/**
@@ -358,7 +394,7 @@ void queue_spin_lock_slowpath(struct qspinlock *lock, u32 val)
{
struct mcs_spinlock *node;
u32 new, old, tail;
- int idx;
+ int idx, cpu_nr;
int retry = INT_MAX; /* Retry count, queue if <= 0 */
BUILD_BUG_ON(CONFIG_NR_CPUS >= (1U << _Q_TAIL_CPU_BITS));
@@ -470,11 +506,12 @@ void queue_spin_lock_slowpath(struct qspinlock *lock, u32 val)
queue:
node = this_cpu_ptr(&mcs_nodes[0]);
idx = node->count++;
- tail = encode_tail(smp_processor_id(), idx);
+ tail = encode_tail(cpu_nr = smp_processor_id(), idx);
node += idx;
node->locked = 0;
node->next = NULL;
+ pv_init_vars(node, cpu_nr);
/*
* We touched a (possibly) cold cacheline in the per-cpu queue node;
@@ -490,3 +527,26 @@ queue:
this_cpu_dec(mcs_nodes[0].count);
}
EXPORT_SYMBOL(queue_spin_lock_slowpath);
+
+#ifdef CONFIG_PARAVIRT_SPINLOCKS
+/**
+ * queue_spin_unlock_slowpath - kick up the CPU of the queue head
+ * @lock : Pointer to queue spinlock structure
+ *
+ * The lock is released after finding the queue head to avoid racing
+ * condition between the queue head and the lock holder.
+ */
+void queue_spin_unlock_slowpath(struct qspinlock *lock)
+{
+ struct mcs_spinlock *node = pv_tail_to_qhead(atomic_read(&lock->val));
+
+ /*
+ * Found the queue head, now release the lock before waking it up
+ * If unfair lock is enabled, this allows other ready tasks to get
+ * lock before the halting CPU is waken up.
+ */
+ __queue_spin_unlock(lock);
+ pv_kick_node(node);
+}
+EXPORT_SYMBOL(queue_spin_unlock_slowpath);
+#endif /* CONFIG_PARAVIRT_SPINLOCKS */
--
1.7.1
^ permalink raw reply related [flat|nested] 102+ messages in thread* [PATCH v11 14/16] pvqspinlock: Add qspinlock para-virtualization support
2014-05-30 15:43 [PATCH v11 00/16] qspinlock: a 4-byte queue spinlock with PV support Waiman Long
` (30 preceding siblings ...)
2014-05-30 15:44 ` Waiman Long
@ 2014-05-30 15:44 ` Waiman Long
2014-05-30 15:44 ` [PATCH v11 15/16] pvqspinlock, x86: Enable PV qspinlock PV for KVM Waiman Long
` (5 subsequent siblings)
37 siblings, 0 replies; 102+ messages in thread
From: Waiman Long @ 2014-05-30 15:44 UTC (permalink / raw)
To: Thomas Gleixner, Ingo Molnar, H. Peter Anvin, Peter Zijlstra
Cc: linux-arch, Waiman Long, Rik van Riel, Raghavendra K T,
Gleb Natapov, kvm, Konrad Rzeszutek Wilk, Scott J Norton, x86,
Paolo Bonzini, linux-kernel, virtualization, Chegu Vinod,
David Vrabel, Oleg Nesterov, xen-devel, Boris Ostrovsky,
Paul E. McKenney, Linus Torvalds
This patch adds base para-virtualization support to the queue
spinlock in the same way as was done in the PV ticket lock code. In
essence, the lock waiters will spin for a specified number of times
(QSPIN_THRESHOLD = 2^14) and then halted itself. The queue head waiter,
unlike the other waiter, will spins 2*QSPIN_THRESHOLD times before
halting itself. Before being halted, the queue head waiter will set
a flag (_Q_LOCKED_SLOWPATH) in the lock byte to indicate that the
unlock slowpath has to be invoked.
In the unlock slowpath, the current lock holder will find the queue
head by following the previous node pointer links stored in the queue
node structure until it finds one that has the qhead flag turned
on. It then attempt to kick in the CPU of the queue head.
After the queue head acquired the lock, it will also check the status
of the next node and set _Q_LOCKED_SLOWPATH flag if it has been halted.
Enabling the PV code does have a performance impact on spinlock
acquisitions and releases. The following table shows the execution
time (in ms) of a spinlock micro-benchmark that does lock/unlock
operations 5M times for each task versus the number of contending
tasks on a Westmere-EX system.
# of Ticket lock Queue lock
tasks PV off/PV on/%Change PV off/PV on/%Change
------ -------------------- ---------------------
1 135/ 179/+33% 137/ 168/+23%
2 1045/ 1103/ +6% 1161/ 1248/ +7%
3 1827/ 2683/+47% 2357/ 2600/+10%
4 2689/ 4191/+56% 2882/ 3115/ +8%
5 3736/ 5830/+56% 3493/ 3571/ +2%
6 4942/ 7609/+54% 4239/ 4198/ -1%
7 6304/ 9570/+52% 4931/ 4895/ -1%
8 7736/11323/+46% 5632/ 5588/ -1%
It can be seen that the ticket lock PV code has a fairly big decrease
in performance when there are 3 or more contending tasks. The queue
spinlock PV code, on the other hand, only has a relatively minor
drop in performance for with 1-4 contending tasks. With 5 or more
contending tasks, there is practically no difference in performance.
Signed-off-by: Waiman Long <Waiman.Long@hp.com>
---
arch/x86/include/asm/pvqspinlock.h | 359 ++++++++++++++++++++++++++++++++++++
arch/x86/include/asm/qspinlock.h | 33 ++++
kernel/locking/qspinlock.c | 72 +++++++-
3 files changed, 458 insertions(+), 6 deletions(-)
create mode 100644 arch/x86/include/asm/pvqspinlock.h
diff --git a/arch/x86/include/asm/pvqspinlock.h b/arch/x86/include/asm/pvqspinlock.h
new file mode 100644
index 0000000..af00eda
--- /dev/null
+++ b/arch/x86/include/asm/pvqspinlock.h
@@ -0,0 +1,359 @@
+#ifndef _ASM_X86_PVQSPINLOCK_H
+#define _ASM_X86_PVQSPINLOCK_H
+
+/*
+ * Queue Spinlock Para-Virtualization (PV) Support
+ *
+ * +------+ +------+ +------+
+ * pv_qnode |Queue | prev | | prev |Queue |
+ * | Head |<-------| Node | .... <-| Tail |
+ * +------+ +------+ +------+
+ * | | |
+ * V V V
+ * +------+ +------+ +------+
+ * mcs_spinlock |locked| |locked| |locked|
+ * | = 1 |------->| = 0 |-> .... | = 0 |
+ * +------+ next +------+ next +------+
+ *
+ * The PV support code for queue spinlock is roughly the same as that
+ * of the ticket spinlock. Each CPU waiting for the lock will spin until it
+ * reaches a threshold. When that happens, it will put itself to halt so
+ * that the hypervisor can reuse the CPU cycles in some other guests as well
+ * as returning other hold-up CPUs faster.
+ *
+ * Auxillary fields in the pv_qnode structure are used to hold information
+ * relevant to the PV support so that it won't impact on the behavior and
+ * performance of the bare metal code. The structure contains a prev pointer
+ * so that a lock holder can find out the queue head from the queue tail
+ * following the prev pointers.
+ *
+ * A major difference between the two versions of PV spinlock is the fact
+ * that the spin threshold of the queue spinlock is half of that of the
+ * ticket spinlock. However, the queue head will spin twice as long as the
+ * other nodes before it puts itself to halt. The reason for that is to
+ * increase halting chance of heavily contended locks to favor lightly
+ * contended locks (queue depth of 1 or less).
+ *
+ * There are 2 places where races can happen:
+ * 1) Halting of the queue head CPU (in pv_head_spin_check) and the CPU
+ * kicking by the lock holder in the unlock path (in pv_kick_node).
+ * 2) Halting of the queue node CPU (in pv_queue_spin_check) and the
+ * the status check by the previous queue head (in pv_halt_check).
+ * See the comments on those functions to see how the races are being
+ * addressed.
+ */
+
+/*
+ * Spin threshold for queue spinlock
+ */
+#define QSPIN_THRESHOLD (1U<<14)
+#define MAYHALT_THRESHOLD (QSPIN_THRESHOLD - 0x10)
+
+/*
+ * CPU state flags
+ */
+#define PV_CPU_ACTIVE 1 /* This CPU is active */
+#define PV_CPU_KICKED 2 /* This CPU is being kicked */
+#define PV_CPU_HALTED -1 /* This CPU is halted */
+
+/*
+ * Additional fields to be added to the queue node structure
+ *
+ * The size of the mcs_spinlock structure is 16 bytes for x64 and 12 bytes
+ * for i386. Four of those structures are defined per CPU. To add more fields
+ * without increasing the size of the mcs_spinlock structure, we overlay those
+ * additional data fields at an additional mcs_spinlock size bucket at exactly
+ * 3 units away. As a result, we need to double the number of mcs_spinlock
+ * buckets. The mcs_spinlock structure will be casted to the pv_qnode
+ * internally.
+ *
+ * +------------+------------+------------+------------+
+ * | MCS Node 0 | MCS Node 1 | MCS Node 2 | MCS Node 3 |
+ * +------------+------------+------------+------------+
+ * | PV Node 0 | PV Node 1 | PV Node 2 | PV Node 3 |
+ * +------------+------------+------------+------------+
+ */
+struct pv_qnode {
+ struct mcs_spinlock mcs; /* MCS node */
+ struct mcs_spinlock dummy[3]; /* 3 dummy MCS nodes */
+ s8 cpustate; /* CPU status flag */
+ s8 mayhalt; /* May be halted soon */
+ u32 mycpu; /* CPU number of this node */
+ struct pv_qnode *prev; /* Pointer to previous node */
+};
+
+#define qhead mcs.locked /* Queue head flag */
+
+/**
+ * pv_init_vars - initialize fields in struct pv_qnode
+ * @mcs: pointer to struct mcs_spinlock
+ * @cpu: current CPU number
+ */
+static inline void pv_init_vars(struct mcs_spinlock *node, int cpu)
+{
+ struct pv_qnode *pv = (struct pv_qnode *)node;
+
+ BUILD_BUG_ON(sizeof(struct pv_qnode) > 5*sizeof(struct mcs_spinlock));
+
+ if (!static_key_false(¶virt_spinlocks_enabled))
+ return;
+
+ pv->cpustate = PV_CPU_ACTIVE;
+ pv->prev = NULL;
+ pv->mayhalt = false;
+ pv->mycpu = cpu;
+}
+
+/**
+ * pv_head_spin_check - perform para-virtualization checks for queue head
+ * @mcs : pointer to the mcs_spinlock structure
+ * @lock : pointer to the qspinlock structure
+ * @count: loop count pointer
+ *
+ * This function will halt itself if lock is still not available after
+ * QSPIN_THRESHOLD iterations.
+ */
+static inline void
+pv_head_spin_check(struct mcs_spinlock *mcs, struct qspinlock *lock, int *count)
+{
+ struct pv_qnode *pv = (struct pv_qnode *)mcs;
+
+ if (!static_key_false(¶virt_spinlocks_enabled))
+ return;
+
+ (*count)++;
+ if (pv->cpustate == PV_CPU_KICKED) {
+ /*
+ * Reset count and flag
+ */
+ pv->cpustate = PV_CPU_ACTIVE;
+ *count = 0;
+
+ } else if (unlikely(*count >= 2*QSPIN_THRESHOLD)) {
+ u8 lockval;
+ s8 oldstate;
+
+ /*
+ * Set the lock byte to _Q_LOCKED_SLOWPATH before
+ * trying to halt itself. It is possible that the
+ * lock byte had been set to _Q_LOCKED_SLOWPATH
+ * already (spurious wakeup of queue head after a halt
+ * or opportunistic setting in pv_halt_check()).
+ * In this case, just proceeds to sleeping.
+ *
+ * queue head lock holder
+ * ---------- -----------
+ * cpustate = PV_CPU_HALTED
+ * [1] cmpxchg(_Q_LOCKED_VAL [2] cmpxchg(_Q_LOCKED_VAL => 0)
+ * => _Q_LOCKED_SLOWPATH) if (cmpxchg fails &&
+ * if (cmpxchg succeeds) cpustate == PV_CPU_HALTED)
+ * halt() kick()
+ *
+ * Sequence:
+ * 1,2 - slowpath flag set, queue head halted & lock holder
+ * will call slowpath
+ * 2,1 - queue head cmpxchg fails, halt is aborted
+ *
+ * If the queue head CPU is woken up by a spurious interrupt
+ * at the same time as the lock holder check the cpustate,
+ * it is possible that the lock holder will try to kick
+ * the queue head CPU which isn't halted.
+ */
+ oldstate = cmpxchg(&pv->cpustate, PV_CPU_ACTIVE, PV_CPU_HALTED);
+ if (oldstate == PV_CPU_KICKED)
+ goto reset; /* Reset count and state */
+
+ lockval = cmpxchg((u8 *)lock,
+ _Q_LOCKED_VAL, _Q_LOCKED_SLOWPATH);
+ if (lockval != 0) {
+ __queue_halt_cpu(PV_HALT_QHEAD, &pv->cpustate,
+ PV_CPU_HALTED);
+ __queue_lockstat((pv->cpustate == PV_CPU_KICKED)
+ ? PV_WAKE_KICKED : PV_WAKE_SPURIOUS);
+ }
+ /*
+ * Else, the lock is free and no halting is needed
+ */
+reset:
+ ACCESS_ONCE(pv->cpustate) = PV_CPU_ACTIVE;
+ *count = 0; /* Reset count */
+ }
+}
+
+/**
+ * pv_queue_spin_check - perform para-virtualization checks for queue member
+ * @mcs : pointer to the mcs_spinlock structure
+ * @count: loop count pointer
+ */
+static inline void pv_queue_spin_check(struct mcs_spinlock *mcs, int *count)
+{
+ struct pv_qnode *pv = (struct pv_qnode *)mcs;
+
+ if (!static_key_false(¶virt_spinlocks_enabled))
+ return;
+ /*
+ * Attempt to halt oneself after QSPIN_THRESHOLD spins
+ */
+ (*count)++;
+ if (unlikely(*count >= QSPIN_THRESHOLD)) {
+ /*
+ * Time to halt itself
+ */
+ ACCESS_ONCE(pv->cpustate) = PV_CPU_HALTED;
+ /*
+ * One way to avoid the racing between pv_halt_check()
+ * and pv_queue_spin_check() is to use memory barrier or
+ * atomic instruction to synchronize between the two competing
+ * threads. However, that will slow down the queue spinlock
+ * slowpath. One way to eliminate this overhead for normal
+ * cases is to use another flag (mayhalt) to indicate that
+ * racing condition may happen. This flag is set when the
+ * loop count is getting close to the halting threshold.
+ *
+ * When that happens, a 2 variables (cpustate & qhead
+ * [=mcs->locked]) handshake is used to make sure that
+ * pv_halt_check() won't miss setting the _Q_LOCKED_SLOWPATH
+ * when the CPU is about to be halted.
+ *
+ * pv_halt_check pv_queue_spin_check
+ * ------------- -------------------
+ * [1] qhead = true [3] cpustate = PV_CPU_HALTED
+ * smp_mb() smp_mb()
+ * [2] if (cpustate [4] if (qhead)
+ * == PV_CPU_HALTED)
+ *
+ * Sequence:
+ * *,1,*,4,* - halt is aborted as the qhead flag is set,
+ * _Q_LOCKED_SLOWPATH may or may not be set
+ * 3,4,1,2 - the CPU is halt and _Q_LOCKED_SLOWPATH is set
+ */
+ smp_mb();
+ if (!ACCESS_ONCE(pv->qhead)) {
+ /*
+ * Halt the CPU only if it is not the queue head
+ */
+ __queue_halt_cpu(PV_HALT_QNODE, &pv->cpustate,
+ PV_CPU_HALTED);
+ __queue_lockstat((pv->cpustate == PV_CPU_KICKED)
+ ? PV_WAKE_KICKED : PV_WAKE_SPURIOUS);
+ }
+ ACCESS_ONCE(pv->cpustate) = PV_CPU_ACTIVE;
+ *count = 0; /* Reset count & flag */
+ pv->mayhalt = false;
+ } else if (*count == MAYHALT_THRESHOLD) {
+ pv->mayhalt = true;
+ /*
+ * Make sure that the mayhalt flag is visible to others
+ * before proceeding.
+ */
+ smp_mb();
+ }
+}
+
+/**
+ * pv_halt_check - check if the CPU has been halted & set _Q_LOCKED_SLOWPATH
+ * @mcs : pointer to the mcs_spinlock structure
+ * @count: loop count
+ *
+ * The current CPU should have gotten the lock and the queue head flag set
+ * before calling this function.
+ */
+static inline void
+pv_halt_check(struct mcs_spinlock *mcs, struct qspinlock *lock)
+{
+ struct pv_qnode *pv = (struct pv_qnode *)mcs;
+
+ if (!static_key_false(¶virt_spinlocks_enabled))
+ return;
+ /*
+ * Halt state checking will only be done if the mayhalt flag is set
+ * to avoid the overhead of the memory barrier in normal cases.
+ * It is highly unlikely that the actual writing to the qhead flag
+ * will be more than 0x10 iterations later than the reading of the
+ * mayhalt flag so that it misses seeing the PV_CPU_HALTED state
+ * which causes lost wakeup.
+ */
+ if (ACCESS_ONCE(pv->mayhalt)) {
+ /*
+ * A memory barrier is used here to make sure that the setting
+ * of queue head flag prior to this function call is visible
+ * to others before checking the cpustate flag.
+ */
+ smp_mb();
+ if (pv->cpustate == PV_CPU_HALTED)
+ ACCESS_ONCE(*(u8 *)lock) = _Q_LOCKED_SLOWPATH;
+ }
+}
+
+/**
+ * pv_set_prev - set previous queue node pointer
+ * @mcs : pointer to the mcs_spinlock structure
+ * @prev: pointer to the previous pv_qnode
+ */
+static inline void
+pv_set_prev(struct mcs_spinlock *mcs, struct mcs_spinlock *prev)
+{
+ struct pv_qnode *pv = (struct pv_qnode *)mcs;
+
+ ACCESS_ONCE(pv->prev) = (struct pv_qnode *)prev;
+ /*
+ * Make sure the prev field is set up before others
+ */
+ smp_wmb();
+}
+
+/*
+ * The following inlined functions are being used by the
+ * queue_spin_unlock_slowpath() function.
+ */
+
+/**
+ * pv_tail_to_qhead - get queue head pv_qnode from tail code
+ * @tail : pointer to queue tail code
+ * @Return: mcs_spinlock pointer of queue head
+ *
+ * This function should only be called by the current lock holder so that
+ * the queue head won't be changed.
+ */
+static struct mcs_spinlock *pv_tail_to_qhead(u32 tail)
+{
+ struct pv_qnode *prev, *pv = (struct pv_qnode *)decode_tail(tail);
+
+ /*
+ * Locate the queue head node by following the prev pointer from
+ * tail to head. It is assumed that the PV guests won't have that
+ * many CPUs so that it won't take a long time to follow the pointers.
+ */
+ while (!ACCESS_ONCE(pv->qhead)) {
+ prev = ACCESS_ONCE(pv->prev);
+ if (prev)
+ pv = prev;
+ else
+ /*
+ * Delay a bit to allow the prev pointer or the queue
+ * head flag to be set up
+ */
+ cpu_relax();
+ }
+ return (struct mcs_spinlock *)pv;
+}
+
+/**
+ * pv_kick_node - kick up the CPU of the given node
+ * @mcs : pointer to struct mcs_spinlock of the node to be kicked
+ */
+static inline void pv_kick_node(struct mcs_spinlock *mcs)
+{
+ struct pv_qnode *pv = (struct pv_qnode *)mcs;
+ s8 oldstate = xchg(&pv->cpustate, PV_CPU_KICKED);
+
+ /*
+ * Kick up the CPU only if the state was set to PV_CPU_HALTED
+ */
+ if (oldstate != PV_CPU_HALTED)
+ __queue_lockstat(PV_KICK_NOHALT);
+ else
+ __queue_kick_cpu(pv->mycpu);
+}
+
+#endif /* _ASM_X86_PVQSPINLOCK_H */
diff --git a/arch/x86/include/asm/qspinlock.h b/arch/x86/include/asm/qspinlock.h
index 448de8b..5ddc456 100644
--- a/arch/x86/include/asm/qspinlock.h
+++ b/arch/x86/include/asm/qspinlock.h
@@ -19,13 +19,46 @@ extern struct static_key virt_unfairlocks_enabled;
* that the clearing the lock bit is done ASAP without artificial delay
* due to compiler optimization.
*/
+#ifdef CONFIG_PARAVIRT_SPINLOCKS
+static __always_inline void __queue_spin_unlock(struct qspinlock *lock)
+#else
static inline void queue_spin_unlock(struct qspinlock *lock)
+#endif
{
barrier();
ACCESS_ONCE(*(u8 *)lock) = 0;
barrier();
}
+#ifdef CONFIG_PARAVIRT_SPINLOCKS
+/*
+ * The lock byte can have a value of _Q_LOCKED_SLOWPATH to indicate
+ * that it needs to go through the slowpath to do the unlocking.
+ */
+#define _Q_LOCKED_SLOWPATH (_Q_LOCKED_VAL | 2)
+
+extern void queue_spin_unlock_slowpath(struct qspinlock *lock);
+
+static inline void queue_spin_unlock(struct qspinlock *lock)
+{
+ barrier();
+ if (static_key_false(¶virt_spinlocks_enabled)) {
+ /*
+ * Need to atomically clear the lock byte to avoid racing with
+ * queue head waiter trying to set _QLOCK_LOCKED_SLOWPATH.
+ */
+ if (likely(cmpxchg((u8 *)lock, _Q_LOCKED_VAL, 0)
+ == _Q_LOCKED_VAL))
+ return;
+ else
+ queue_spin_unlock_slowpath(lock);
+
+ } else {
+ __queue_spin_unlock(lock);
+ }
+ barrier();
+}
+#endif /* CONFIG_PARAVIRT_SPINLOCKS */
#endif /* !CONFIG_X86_OOSTORE && !CONFIG_X86_PPRO_FENCE */
#include <asm-generic/qspinlock.h>
diff --git a/kernel/locking/qspinlock.c b/kernel/locking/qspinlock.c
index 8deedcf..adedd75 100644
--- a/kernel/locking/qspinlock.c
+++ b/kernel/locking/qspinlock.c
@@ -60,12 +60,17 @@
* Check the pending bit spinning threshold only if PV qspinlock is enabled
*/
#define PSPIN_THRESHOLD (1 << 10)
-#define MAX_NODES 4
+/*
+ * We need to double the number of per-cpu mcs_spinlock structures to hold
+ * additional fields specific to para-virtualization support.
+ */
#ifdef CONFIG_PARAVIRT_SPINLOCKS
-#define pv_qspinlock_enabled() static_key_false(¶virt_spinlocks_enabled)
+# define MAX_NODES 8
+# define pv_qspinlock_enabled() static_key_false(¶virt_spinlocks_enabled)
#else
-#define pv_qspinlock_enabled() false
+# define MAX_NODES 4
+# define pv_qspinlock_enabled() false
#endif
/*
@@ -243,6 +248,22 @@ static __always_inline int try_set_locked(struct qspinlock *lock)
return 1;
}
+/*
+ * Para-virtualization (PV) queue spinlock support
+ */
+#ifdef CONFIG_PARAVIRT_SPINLOCKS
+#include <asm/pvqspinlock.h>
+#else
+static inline void pv_init_vars(struct mcs_spinlock *mcs, int cpu) {}
+static inline void pv_head_spin_check(struct mcs_spinlock *mcs,
+ struct qspinlock *lock, int *count) {}
+static inline void pv_queue_spin_check(struct mcs_spinlock *mcs,
+ int *count) {}
+static inline void pv_halt_check(struct mcs_spinlock *mcs, void *lock) {}
+static inline void pv_set_prev(struct mcs_spinlock *mcs,
+ struct mcs_spinlock *prev) {}
+#endif /* CONFIG_PARAVIRT_SPINLOCKS */
+
/**
* queue_spin_lock_slowerpath - a slower patch for acquiring queue spinlock
* @lock: Pointer to queue spinlock structure
@@ -260,6 +281,7 @@ static noinline void queue_spin_lock_slowerpath(struct qspinlock *lock,
{
struct mcs_spinlock *prev, *next;
u32 val, old;
+ int hcnt = 0; /* Queue head loop counter */
/*
* we already touched the queueing cacheline; don't bother with pending
@@ -273,9 +295,16 @@ static noinline void queue_spin_lock_slowerpath(struct qspinlock *lock,
* if there was a previous node; link it and wait.
*/
if (old & _Q_TAIL_MASK) {
+ int qcnt = 0; /* Queue node loop counter */
+
prev = decode_tail(old);
+ pv_set_prev(node, prev);
ACCESS_ONCE(prev->next) = node;
+ while (!smp_load_acquire(&node->locked)) {
+ pv_queue_spin_check(node, &qcnt);
+ arch_mutex_cpu_relax();
+ }
arch_mcs_spin_lock_contended(&node->locked);
} else {
/* Mark it as the queue head */
@@ -292,8 +321,14 @@ static noinline void queue_spin_lock_slowerpath(struct qspinlock *lock,
*/
retry_queue_wait:
while ((val = smp_load_acquire(&lock->val.counter))
- & _Q_LOCKED_PENDING_MASK)
+ & _Q_LOCKED_PENDING_MASK) {
+ /*
+ * Perform queue head para-virtualization checks
+ */
+ pv_head_spin_check(node, lock, &hcnt);
arch_mutex_cpu_relax();
+ }
+ hcnt = 0; /* Reset loop count */
/*
* claim the lock:
@@ -331,6 +366,7 @@ retry_queue_wait:
arch_mutex_cpu_relax();
arch_mcs_spin_unlock_contended(&next->locked);
+ pv_halt_check(next, lock);
}
/**
@@ -358,7 +394,7 @@ void queue_spin_lock_slowpath(struct qspinlock *lock, u32 val)
{
struct mcs_spinlock *node;
u32 new, old, tail;
- int idx;
+ int idx, cpu_nr;
int retry = INT_MAX; /* Retry count, queue if <= 0 */
BUILD_BUG_ON(CONFIG_NR_CPUS >= (1U << _Q_TAIL_CPU_BITS));
@@ -470,11 +506,12 @@ void queue_spin_lock_slowpath(struct qspinlock *lock, u32 val)
queue:
node = this_cpu_ptr(&mcs_nodes[0]);
idx = node->count++;
- tail = encode_tail(smp_processor_id(), idx);
+ tail = encode_tail(cpu_nr = smp_processor_id(), idx);
node += idx;
node->locked = 0;
node->next = NULL;
+ pv_init_vars(node, cpu_nr);
/*
* We touched a (possibly) cold cacheline in the per-cpu queue node;
@@ -490,3 +527,26 @@ queue:
this_cpu_dec(mcs_nodes[0].count);
}
EXPORT_SYMBOL(queue_spin_lock_slowpath);
+
+#ifdef CONFIG_PARAVIRT_SPINLOCKS
+/**
+ * queue_spin_unlock_slowpath - kick up the CPU of the queue head
+ * @lock : Pointer to queue spinlock structure
+ *
+ * The lock is released after finding the queue head to avoid racing
+ * condition between the queue head and the lock holder.
+ */
+void queue_spin_unlock_slowpath(struct qspinlock *lock)
+{
+ struct mcs_spinlock *node = pv_tail_to_qhead(atomic_read(&lock->val));
+
+ /*
+ * Found the queue head, now release the lock before waking it up
+ * If unfair lock is enabled, this allows other ready tasks to get
+ * lock before the halting CPU is waken up.
+ */
+ __queue_spin_unlock(lock);
+ pv_kick_node(node);
+}
+EXPORT_SYMBOL(queue_spin_unlock_slowpath);
+#endif /* CONFIG_PARAVIRT_SPINLOCKS */
--
1.7.1
^ permalink raw reply related [flat|nested] 102+ messages in thread* [PATCH v11 15/16] pvqspinlock, x86: Enable PV qspinlock PV for KVM
2014-05-30 15:43 [PATCH v11 00/16] qspinlock: a 4-byte queue spinlock with PV support Waiman Long
` (31 preceding siblings ...)
2014-05-30 15:44 ` Waiman Long
@ 2014-05-30 15:44 ` Waiman Long
2014-05-30 15:44 ` Waiman Long
` (4 subsequent siblings)
37 siblings, 0 replies; 102+ messages in thread
From: Waiman Long @ 2014-05-30 15:44 UTC (permalink / raw)
To: Thomas Gleixner, Ingo Molnar, H. Peter Anvin, Peter Zijlstra
Cc: linux-arch, x86, linux-kernel, virtualization, xen-devel, kvm,
Paolo Bonzini, Konrad Rzeszutek Wilk, Boris Ostrovsky,
Paul E. McKenney, Rik van Riel, Linus Torvalds, Raghavendra K T,
David Vrabel, Oleg Nesterov, Gleb Natapov, Scott J Norton,
Chegu Vinod, Waiman Long
This patch adds the necessary KVM specific code to allow KVM to
support the CPU halting and kicking operations needed by the queue
spinlock PV code.
Two KVM guests of 20 CPU cores (2 nodes) were created for performance
testing in one of the following three configurations:
1) Only 1 VM is active
2) Both VMs are active and they share the same 20 physical CPUs
(200% overcommit)
The tests run included the disk workload of the AIM7 benchmark on both
ext4 and xfs RAM disks at 3000 users on a 3.15-rc7 based kernel. The
"ebizzy -m" test was was also run and its performance data were
recorded. With two VMs running, the "idle=poll" kernel option was
added to simulate a busy guest. The entry "unfair + PV qspinlock"
below means that both the unfair lock and PV spinlock configuration
options were turned on.
AIM7 XFS Disk Test (no overcommit)
kernel JPM Real Time Sys Time Usr Time
----- --- --------- -------- --------
PV ticketlock 2521008 7.24 101.02 5.24
qspinlock 2571429 7.00 99.10 5.49
PV qspinlock 2535211 7.10 100.32 5.45
unfair qspinlock 2571429 7.00 99.25 5.40
unfair + PV qspinlock 2549575 7.06 99.81 5.31
AIM7 XFS Disk Test (200% overcommit)
kernel JPM Real Time Sys Time Usr Time
----- --- --------- -------- --------
PV ticketlock 768902 23.41 341.71 3.07
qspinlock 784656 22.94 346.22 2.90
PV qspinlock 773861 23.26 352.47 2.30
unfair qspinlock 835655 21.54 316.52 1.57
unfair + PV qspinlock 797165 22.58 323.95 3.58
AIM7 EXT4 Disk Test (no overcommit)
kernel JPM Real Time Sys Time Usr Time
----- --- --------- -------- --------
PV ticketlock 1956522 9.20 106.58 5.35
qspinlock 1995565 9.02 103.19 5.37
PV qspinlock 1958651 9.19 106.57 5.30
unfair qspinlock 2022472 8.90 103.58 5.37
unfair + PV qspinlock 1991150 9.04 104.41 5.46
AIM7 EXT4 Disk Test (200% overcommit)
kernel JPM Real Time Sys Time Usr Time
----- --- --------- -------- --------
PV ticketlock 576553 31.22 407.44 1.51
qspinlock 609550 29.53 407.14 1.69
PV qspinlock 592105 30.40 410.51 1.67
unfair qspinlock 672897 26.75 359.78 1.66
unfair + PV qspinlock 670391 26.85 357.09 0.63
EBIZZY-M Test (no overcommit)
kernel Rec/s Real Time Sys Time Usr Time
----- ----- --------- -------- --------
PV ticketlock 1328 10.00 82.82 1.46
qspinlock 1679 10.00 65.37 1.80
PV qspinlock 1470 10.00 75.54 1.54
unfair qspinlock 1518 10.00 70.80 1.71
unfair + PV qspinlock 1585 10.00 69.02 1.76
EBIZZY-M Test (200% overcommit)
kernel Rec/s Real Time Sys Time Usr Time
----- ----- --------- -------- --------
PV ticketlock 453 10.00 77.11 0.00
qspinlock 459 10.00 77.50 0.00
PV qspinlock 402 10.00 91.55 0.00
unfair qspinlock 570 10.00 62.98 0.00
unfair + PV qspinlock 586 10.00 59.68 0.00
Signed-off-by: Waiman Long <Waiman.Long@hp.com>
Tested-by: Raghavendra K T <raghavendra.kt@linux.vnet.ibm.com>
---
arch/x86/kernel/kvm.c | 135 +++++++++++++++++++++++++++++++++++++++++++++++++
kernel/Kconfig.locks | 2 +-
2 files changed, 136 insertions(+), 1 deletions(-)
diff --git a/arch/x86/kernel/kvm.c b/arch/x86/kernel/kvm.c
index 7ab8ab3..eef427b 100644
--- a/arch/x86/kernel/kvm.c
+++ b/arch/x86/kernel/kvm.c
@@ -567,6 +567,7 @@ static void kvm_kick_cpu(int cpu)
kvm_hypercall2(KVM_HC_KICK_CPU, flags, apicid);
}
+#ifndef CONFIG_QUEUE_SPINLOCK
enum kvm_contention_stat {
TAKEN_SLOW,
TAKEN_SLOW_PICKUP,
@@ -794,6 +795,134 @@ static void kvm_unlock_kick(struct arch_spinlock *lock, __ticket_t ticket)
}
}
}
+#else /* !CONFIG_QUEUE_SPINLOCK */
+
+#ifdef CONFIG_KVM_DEBUG_FS
+static struct dentry *d_spin_debug;
+static struct dentry *d_kvm_debug;
+static u32 kick_nohlt_stats; /* Kick but not halt count */
+static u32 halt_qhead_stats; /* Queue head halting count */
+static u32 halt_qnode_stats; /* Queue node halting count */
+static u32 halt_abort_stats; /* Halting abort count */
+static u32 wake_kick_stats; /* Wakeup by kicking count */
+static u32 wake_spur_stats; /* Spurious wakeup count */
+static u64 time_blocked; /* Total blocking time */
+
+static int __init kvm_spinlock_debugfs(void)
+{
+ d_kvm_debug = debugfs_create_dir("kvm-guest", NULL);
+ if (!d_kvm_debug) {
+ printk(KERN_WARNING
+ "Could not create 'kvm' debugfs directory\n");
+ return -ENOMEM;
+ }
+ d_spin_debug = debugfs_create_dir("spinlocks", d_kvm_debug);
+
+ debugfs_create_u32("kick_nohlt_stats",
+ 0644, d_spin_debug, &kick_nohlt_stats);
+ debugfs_create_u32("halt_qhead_stats",
+ 0644, d_spin_debug, &halt_qhead_stats);
+ debugfs_create_u32("halt_qnode_stats",
+ 0644, d_spin_debug, &halt_qnode_stats);
+ debugfs_create_u32("halt_abort_stats",
+ 0644, d_spin_debug, &halt_abort_stats);
+ debugfs_create_u32("wake_kick_stats",
+ 0644, d_spin_debug, &wake_kick_stats);
+ debugfs_create_u32("wake_spur_stats",
+ 0644, d_spin_debug, &wake_spur_stats);
+ debugfs_create_u64("time_blocked",
+ 0644, d_spin_debug, &time_blocked);
+ return 0;
+}
+
+static inline void kvm_halt_stats(enum pv_lock_stats type)
+{
+ if (type == PV_HALT_QHEAD)
+ add_smp(&halt_qhead_stats, 1);
+ else if (type == PV_HALT_QNODE)
+ add_smp(&halt_qnode_stats, 1);
+ else /* type == PV_HALT_ABORT */
+ add_smp(&halt_abort_stats, 1);
+}
+
+static inline void kvm_lock_stats(enum pv_lock_stats type)
+{
+ if (type == PV_WAKE_KICKED)
+ add_smp(&wake_kick_stats, 1);
+ else if (type == PV_WAKE_SPURIOUS)
+ add_smp(&wake_spur_stats, 1);
+ else /* type == PV_KICK_NOHALT */
+ add_smp(&kick_nohlt_stats, 1);
+}
+
+static inline u64 spin_time_start(void)
+{
+ return sched_clock();
+}
+
+static inline void spin_time_accum_blocked(u64 start)
+{
+ u64 delta;
+
+ delta = sched_clock() - start;
+ add_smp(&time_blocked, delta);
+}
+
+fs_initcall(kvm_spinlock_debugfs);
+
+#else /* CONFIG_KVM_DEBUG_FS */
+static inline void kvm_halt_stats(enum pv_lock_stats type)
+{
+}
+
+static inline void kvm_lock_stats(enum pv_lock_stats type)
+{
+}
+
+static inline u64 spin_time_start(void)
+{
+ return 0;
+}
+
+static inline void spin_time_accum_blocked(u64 start)
+{
+}
+#endif /* CONFIG_KVM_DEBUG_FS */
+
+/*
+ * Halt the current CPU & release it back to the host
+ */
+static void kvm_halt_cpu(enum pv_lock_stats type, s8 *state, s8 sval)
+{
+ unsigned long flags;
+ u64 start;
+
+ if (in_nmi())
+ return;
+
+ /*
+ * Make sure an interrupt handler can't upset things in a
+ * partially setup state.
+ */
+ local_irq_save(flags);
+ /*
+ * Don't halt if the CPU state has been changed.
+ */
+ if (ACCESS_ONCE(*state) != sval) {
+ kvm_halt_stats(PV_HALT_ABORT);
+ goto out;
+ }
+ start = spin_time_start();
+ kvm_halt_stats(type);
+ if (arch_irqs_disabled_flags(flags))
+ halt();
+ else
+ safe_halt();
+ spin_time_accum_blocked(start);
+out:
+ local_irq_restore(flags);
+}
+#endif /* !CONFIG_QUEUE_SPINLOCK */
/*
* Setup pv_lock_ops to exploit KVM_FEATURE_PV_UNHALT if present.
@@ -806,8 +935,14 @@ void __init kvm_spinlock_init(void)
if (!kvm_para_has_feature(KVM_FEATURE_PV_UNHALT))
return;
+#ifdef CONFIG_QUEUE_SPINLOCK
+ pv_lock_ops.kick_cpu = kvm_kick_cpu;
+ pv_lock_ops.halt_cpu = kvm_halt_cpu;
+ pv_lock_ops.lockstat = kvm_lock_stats;
+#else
pv_lock_ops.lock_spinning = PV_CALLEE_SAVE(kvm_lock_spinning);
pv_lock_ops.unlock_kick = kvm_unlock_kick;
+#endif
}
static __init int kvm_spinlock_init_jump(void)
diff --git a/kernel/Kconfig.locks b/kernel/Kconfig.locks
index f185584..a70fdeb 100644
--- a/kernel/Kconfig.locks
+++ b/kernel/Kconfig.locks
@@ -229,4 +229,4 @@ config ARCH_USE_QUEUE_SPINLOCK
config QUEUE_SPINLOCK
def_bool y if ARCH_USE_QUEUE_SPINLOCK
- depends on SMP && !PARAVIRT_SPINLOCKS
+ depends on SMP && (!PARAVIRT_SPINLOCKS || !XEN)
--
1.7.1
^ permalink raw reply related [flat|nested] 102+ messages in thread* [PATCH v11 15/16] pvqspinlock, x86: Enable PV qspinlock PV for KVM
2014-05-30 15:43 [PATCH v11 00/16] qspinlock: a 4-byte queue spinlock with PV support Waiman Long
` (32 preceding siblings ...)
2014-05-30 15:44 ` [PATCH v11 15/16] pvqspinlock, x86: Enable PV qspinlock PV for KVM Waiman Long
@ 2014-05-30 15:44 ` Waiman Long
2014-05-30 15:44 ` Waiman Long
` (3 subsequent siblings)
37 siblings, 0 replies; 102+ messages in thread
From: Waiman Long @ 2014-05-30 15:44 UTC (permalink / raw)
To: Thomas Gleixner, Ingo Molnar, H. Peter Anvin, Peter Zijlstra
Cc: linux-arch, Waiman Long, Raghavendra K T, Gleb Natapov, kvm,
Scott J Norton, x86, Paolo Bonzini, linux-kernel, virtualization,
Chegu Vinod, David Vrabel, Oleg Nesterov, xen-devel,
Boris Ostrovsky, Paul E. McKenney, Linus Torvalds
This patch adds the necessary KVM specific code to allow KVM to
support the CPU halting and kicking operations needed by the queue
spinlock PV code.
Two KVM guests of 20 CPU cores (2 nodes) were created for performance
testing in one of the following three configurations:
1) Only 1 VM is active
2) Both VMs are active and they share the same 20 physical CPUs
(200% overcommit)
The tests run included the disk workload of the AIM7 benchmark on both
ext4 and xfs RAM disks at 3000 users on a 3.15-rc7 based kernel. The
"ebizzy -m" test was was also run and its performance data were
recorded. With two VMs running, the "idle=poll" kernel option was
added to simulate a busy guest. The entry "unfair + PV qspinlock"
below means that both the unfair lock and PV spinlock configuration
options were turned on.
AIM7 XFS Disk Test (no overcommit)
kernel JPM Real Time Sys Time Usr Time
----- --- --------- -------- --------
PV ticketlock 2521008 7.24 101.02 5.24
qspinlock 2571429 7.00 99.10 5.49
PV qspinlock 2535211 7.10 100.32 5.45
unfair qspinlock 2571429 7.00 99.25 5.40
unfair + PV qspinlock 2549575 7.06 99.81 5.31
AIM7 XFS Disk Test (200% overcommit)
kernel JPM Real Time Sys Time Usr Time
----- --- --------- -------- --------
PV ticketlock 768902 23.41 341.71 3.07
qspinlock 784656 22.94 346.22 2.90
PV qspinlock 773861 23.26 352.47 2.30
unfair qspinlock 835655 21.54 316.52 1.57
unfair + PV qspinlock 797165 22.58 323.95 3.58
AIM7 EXT4 Disk Test (no overcommit)
kernel JPM Real Time Sys Time Usr Time
----- --- --------- -------- --------
PV ticketlock 1956522 9.20 106.58 5.35
qspinlock 1995565 9.02 103.19 5.37
PV qspinlock 1958651 9.19 106.57 5.30
unfair qspinlock 2022472 8.90 103.58 5.37
unfair + PV qspinlock 1991150 9.04 104.41 5.46
AIM7 EXT4 Disk Test (200% overcommit)
kernel JPM Real Time Sys Time Usr Time
----- --- --------- -------- --------
PV ticketlock 576553 31.22 407.44 1.51
qspinlock 609550 29.53 407.14 1.69
PV qspinlock 592105 30.40 410.51 1.67
unfair qspinlock 672897 26.75 359.78 1.66
unfair + PV qspinlock 670391 26.85 357.09 0.63
EBIZZY-M Test (no overcommit)
kernel Rec/s Real Time Sys Time Usr Time
----- ----- --------- -------- --------
PV ticketlock 1328 10.00 82.82 1.46
qspinlock 1679 10.00 65.37 1.80
PV qspinlock 1470 10.00 75.54 1.54
unfair qspinlock 1518 10.00 70.80 1.71
unfair + PV qspinlock 1585 10.00 69.02 1.76
EBIZZY-M Test (200% overcommit)
kernel Rec/s Real Time Sys Time Usr Time
----- ----- --------- -------- --------
PV ticketlock 453 10.00 77.11 0.00
qspinlock 459 10.00 77.50 0.00
PV qspinlock 402 10.00 91.55 0.00
unfair qspinlock 570 10.00 62.98 0.00
unfair + PV qspinlock 586 10.00 59.68 0.00
Signed-off-by: Waiman Long <Waiman.Long@hp.com>
Tested-by: Raghavendra K T <raghavendra.kt@linux.vnet.ibm.com>
---
arch/x86/kernel/kvm.c | 135 +++++++++++++++++++++++++++++++++++++++++++++++++
kernel/Kconfig.locks | 2 +-
2 files changed, 136 insertions(+), 1 deletions(-)
diff --git a/arch/x86/kernel/kvm.c b/arch/x86/kernel/kvm.c
index 7ab8ab3..eef427b 100644
--- a/arch/x86/kernel/kvm.c
+++ b/arch/x86/kernel/kvm.c
@@ -567,6 +567,7 @@ static void kvm_kick_cpu(int cpu)
kvm_hypercall2(KVM_HC_KICK_CPU, flags, apicid);
}
+#ifndef CONFIG_QUEUE_SPINLOCK
enum kvm_contention_stat {
TAKEN_SLOW,
TAKEN_SLOW_PICKUP,
@@ -794,6 +795,134 @@ static void kvm_unlock_kick(struct arch_spinlock *lock, __ticket_t ticket)
}
}
}
+#else /* !CONFIG_QUEUE_SPINLOCK */
+
+#ifdef CONFIG_KVM_DEBUG_FS
+static struct dentry *d_spin_debug;
+static struct dentry *d_kvm_debug;
+static u32 kick_nohlt_stats; /* Kick but not halt count */
+static u32 halt_qhead_stats; /* Queue head halting count */
+static u32 halt_qnode_stats; /* Queue node halting count */
+static u32 halt_abort_stats; /* Halting abort count */
+static u32 wake_kick_stats; /* Wakeup by kicking count */
+static u32 wake_spur_stats; /* Spurious wakeup count */
+static u64 time_blocked; /* Total blocking time */
+
+static int __init kvm_spinlock_debugfs(void)
+{
+ d_kvm_debug = debugfs_create_dir("kvm-guest", NULL);
+ if (!d_kvm_debug) {
+ printk(KERN_WARNING
+ "Could not create 'kvm' debugfs directory\n");
+ return -ENOMEM;
+ }
+ d_spin_debug = debugfs_create_dir("spinlocks", d_kvm_debug);
+
+ debugfs_create_u32("kick_nohlt_stats",
+ 0644, d_spin_debug, &kick_nohlt_stats);
+ debugfs_create_u32("halt_qhead_stats",
+ 0644, d_spin_debug, &halt_qhead_stats);
+ debugfs_create_u32("halt_qnode_stats",
+ 0644, d_spin_debug, &halt_qnode_stats);
+ debugfs_create_u32("halt_abort_stats",
+ 0644, d_spin_debug, &halt_abort_stats);
+ debugfs_create_u32("wake_kick_stats",
+ 0644, d_spin_debug, &wake_kick_stats);
+ debugfs_create_u32("wake_spur_stats",
+ 0644, d_spin_debug, &wake_spur_stats);
+ debugfs_create_u64("time_blocked",
+ 0644, d_spin_debug, &time_blocked);
+ return 0;
+}
+
+static inline void kvm_halt_stats(enum pv_lock_stats type)
+{
+ if (type == PV_HALT_QHEAD)
+ add_smp(&halt_qhead_stats, 1);
+ else if (type == PV_HALT_QNODE)
+ add_smp(&halt_qnode_stats, 1);
+ else /* type == PV_HALT_ABORT */
+ add_smp(&halt_abort_stats, 1);
+}
+
+static inline void kvm_lock_stats(enum pv_lock_stats type)
+{
+ if (type == PV_WAKE_KICKED)
+ add_smp(&wake_kick_stats, 1);
+ else if (type == PV_WAKE_SPURIOUS)
+ add_smp(&wake_spur_stats, 1);
+ else /* type == PV_KICK_NOHALT */
+ add_smp(&kick_nohlt_stats, 1);
+}
+
+static inline u64 spin_time_start(void)
+{
+ return sched_clock();
+}
+
+static inline void spin_time_accum_blocked(u64 start)
+{
+ u64 delta;
+
+ delta = sched_clock() - start;
+ add_smp(&time_blocked, delta);
+}
+
+fs_initcall(kvm_spinlock_debugfs);
+
+#else /* CONFIG_KVM_DEBUG_FS */
+static inline void kvm_halt_stats(enum pv_lock_stats type)
+{
+}
+
+static inline void kvm_lock_stats(enum pv_lock_stats type)
+{
+}
+
+static inline u64 spin_time_start(void)
+{
+ return 0;
+}
+
+static inline void spin_time_accum_blocked(u64 start)
+{
+}
+#endif /* CONFIG_KVM_DEBUG_FS */
+
+/*
+ * Halt the current CPU & release it back to the host
+ */
+static void kvm_halt_cpu(enum pv_lock_stats type, s8 *state, s8 sval)
+{
+ unsigned long flags;
+ u64 start;
+
+ if (in_nmi())
+ return;
+
+ /*
+ * Make sure an interrupt handler can't upset things in a
+ * partially setup state.
+ */
+ local_irq_save(flags);
+ /*
+ * Don't halt if the CPU state has been changed.
+ */
+ if (ACCESS_ONCE(*state) != sval) {
+ kvm_halt_stats(PV_HALT_ABORT);
+ goto out;
+ }
+ start = spin_time_start();
+ kvm_halt_stats(type);
+ if (arch_irqs_disabled_flags(flags))
+ halt();
+ else
+ safe_halt();
+ spin_time_accum_blocked(start);
+out:
+ local_irq_restore(flags);
+}
+#endif /* !CONFIG_QUEUE_SPINLOCK */
/*
* Setup pv_lock_ops to exploit KVM_FEATURE_PV_UNHALT if present.
@@ -806,8 +935,14 @@ void __init kvm_spinlock_init(void)
if (!kvm_para_has_feature(KVM_FEATURE_PV_UNHALT))
return;
+#ifdef CONFIG_QUEUE_SPINLOCK
+ pv_lock_ops.kick_cpu = kvm_kick_cpu;
+ pv_lock_ops.halt_cpu = kvm_halt_cpu;
+ pv_lock_ops.lockstat = kvm_lock_stats;
+#else
pv_lock_ops.lock_spinning = PV_CALLEE_SAVE(kvm_lock_spinning);
pv_lock_ops.unlock_kick = kvm_unlock_kick;
+#endif
}
static __init int kvm_spinlock_init_jump(void)
diff --git a/kernel/Kconfig.locks b/kernel/Kconfig.locks
index f185584..a70fdeb 100644
--- a/kernel/Kconfig.locks
+++ b/kernel/Kconfig.locks
@@ -229,4 +229,4 @@ config ARCH_USE_QUEUE_SPINLOCK
config QUEUE_SPINLOCK
def_bool y if ARCH_USE_QUEUE_SPINLOCK
- depends on SMP && !PARAVIRT_SPINLOCKS
+ depends on SMP && (!PARAVIRT_SPINLOCKS || !XEN)
--
1.7.1
^ permalink raw reply related [flat|nested] 102+ messages in thread* [PATCH v11 15/16] pvqspinlock, x86: Enable PV qspinlock PV for KVM
2014-05-30 15:43 [PATCH v11 00/16] qspinlock: a 4-byte queue spinlock with PV support Waiman Long
` (33 preceding siblings ...)
2014-05-30 15:44 ` Waiman Long
@ 2014-05-30 15:44 ` Waiman Long
2014-05-30 15:44 ` [PATCH v11 16/16] pvqspinlock, x86: Enable PV qspinlock for XEN Waiman Long
` (2 subsequent siblings)
37 siblings, 0 replies; 102+ messages in thread
From: Waiman Long @ 2014-05-30 15:44 UTC (permalink / raw)
To: Thomas Gleixner, Ingo Molnar, H. Peter Anvin, Peter Zijlstra
Cc: linux-arch, Waiman Long, Rik van Riel, Raghavendra K T,
Gleb Natapov, kvm, Konrad Rzeszutek Wilk, Scott J Norton, x86,
Paolo Bonzini, linux-kernel, virtualization, Chegu Vinod,
David Vrabel, Oleg Nesterov, xen-devel, Boris Ostrovsky,
Paul E. McKenney, Linus Torvalds
This patch adds the necessary KVM specific code to allow KVM to
support the CPU halting and kicking operations needed by the queue
spinlock PV code.
Two KVM guests of 20 CPU cores (2 nodes) were created for performance
testing in one of the following three configurations:
1) Only 1 VM is active
2) Both VMs are active and they share the same 20 physical CPUs
(200% overcommit)
The tests run included the disk workload of the AIM7 benchmark on both
ext4 and xfs RAM disks at 3000 users on a 3.15-rc7 based kernel. The
"ebizzy -m" test was was also run and its performance data were
recorded. With two VMs running, the "idle=poll" kernel option was
added to simulate a busy guest. The entry "unfair + PV qspinlock"
below means that both the unfair lock and PV spinlock configuration
options were turned on.
AIM7 XFS Disk Test (no overcommit)
kernel JPM Real Time Sys Time Usr Time
----- --- --------- -------- --------
PV ticketlock 2521008 7.24 101.02 5.24
qspinlock 2571429 7.00 99.10 5.49
PV qspinlock 2535211 7.10 100.32 5.45
unfair qspinlock 2571429 7.00 99.25 5.40
unfair + PV qspinlock 2549575 7.06 99.81 5.31
AIM7 XFS Disk Test (200% overcommit)
kernel JPM Real Time Sys Time Usr Time
----- --- --------- -------- --------
PV ticketlock 768902 23.41 341.71 3.07
qspinlock 784656 22.94 346.22 2.90
PV qspinlock 773861 23.26 352.47 2.30
unfair qspinlock 835655 21.54 316.52 1.57
unfair + PV qspinlock 797165 22.58 323.95 3.58
AIM7 EXT4 Disk Test (no overcommit)
kernel JPM Real Time Sys Time Usr Time
----- --- --------- -------- --------
PV ticketlock 1956522 9.20 106.58 5.35
qspinlock 1995565 9.02 103.19 5.37
PV qspinlock 1958651 9.19 106.57 5.30
unfair qspinlock 2022472 8.90 103.58 5.37
unfair + PV qspinlock 1991150 9.04 104.41 5.46
AIM7 EXT4 Disk Test (200% overcommit)
kernel JPM Real Time Sys Time Usr Time
----- --- --------- -------- --------
PV ticketlock 576553 31.22 407.44 1.51
qspinlock 609550 29.53 407.14 1.69
PV qspinlock 592105 30.40 410.51 1.67
unfair qspinlock 672897 26.75 359.78 1.66
unfair + PV qspinlock 670391 26.85 357.09 0.63
EBIZZY-M Test (no overcommit)
kernel Rec/s Real Time Sys Time Usr Time
----- ----- --------- -------- --------
PV ticketlock 1328 10.00 82.82 1.46
qspinlock 1679 10.00 65.37 1.80
PV qspinlock 1470 10.00 75.54 1.54
unfair qspinlock 1518 10.00 70.80 1.71
unfair + PV qspinlock 1585 10.00 69.02 1.76
EBIZZY-M Test (200% overcommit)
kernel Rec/s Real Time Sys Time Usr Time
----- ----- --------- -------- --------
PV ticketlock 453 10.00 77.11 0.00
qspinlock 459 10.00 77.50 0.00
PV qspinlock 402 10.00 91.55 0.00
unfair qspinlock 570 10.00 62.98 0.00
unfair + PV qspinlock 586 10.00 59.68 0.00
Signed-off-by: Waiman Long <Waiman.Long@hp.com>
Tested-by: Raghavendra K T <raghavendra.kt@linux.vnet.ibm.com>
---
arch/x86/kernel/kvm.c | 135 +++++++++++++++++++++++++++++++++++++++++++++++++
kernel/Kconfig.locks | 2 +-
2 files changed, 136 insertions(+), 1 deletions(-)
diff --git a/arch/x86/kernel/kvm.c b/arch/x86/kernel/kvm.c
index 7ab8ab3..eef427b 100644
--- a/arch/x86/kernel/kvm.c
+++ b/arch/x86/kernel/kvm.c
@@ -567,6 +567,7 @@ static void kvm_kick_cpu(int cpu)
kvm_hypercall2(KVM_HC_KICK_CPU, flags, apicid);
}
+#ifndef CONFIG_QUEUE_SPINLOCK
enum kvm_contention_stat {
TAKEN_SLOW,
TAKEN_SLOW_PICKUP,
@@ -794,6 +795,134 @@ static void kvm_unlock_kick(struct arch_spinlock *lock, __ticket_t ticket)
}
}
}
+#else /* !CONFIG_QUEUE_SPINLOCK */
+
+#ifdef CONFIG_KVM_DEBUG_FS
+static struct dentry *d_spin_debug;
+static struct dentry *d_kvm_debug;
+static u32 kick_nohlt_stats; /* Kick but not halt count */
+static u32 halt_qhead_stats; /* Queue head halting count */
+static u32 halt_qnode_stats; /* Queue node halting count */
+static u32 halt_abort_stats; /* Halting abort count */
+static u32 wake_kick_stats; /* Wakeup by kicking count */
+static u32 wake_spur_stats; /* Spurious wakeup count */
+static u64 time_blocked; /* Total blocking time */
+
+static int __init kvm_spinlock_debugfs(void)
+{
+ d_kvm_debug = debugfs_create_dir("kvm-guest", NULL);
+ if (!d_kvm_debug) {
+ printk(KERN_WARNING
+ "Could not create 'kvm' debugfs directory\n");
+ return -ENOMEM;
+ }
+ d_spin_debug = debugfs_create_dir("spinlocks", d_kvm_debug);
+
+ debugfs_create_u32("kick_nohlt_stats",
+ 0644, d_spin_debug, &kick_nohlt_stats);
+ debugfs_create_u32("halt_qhead_stats",
+ 0644, d_spin_debug, &halt_qhead_stats);
+ debugfs_create_u32("halt_qnode_stats",
+ 0644, d_spin_debug, &halt_qnode_stats);
+ debugfs_create_u32("halt_abort_stats",
+ 0644, d_spin_debug, &halt_abort_stats);
+ debugfs_create_u32("wake_kick_stats",
+ 0644, d_spin_debug, &wake_kick_stats);
+ debugfs_create_u32("wake_spur_stats",
+ 0644, d_spin_debug, &wake_spur_stats);
+ debugfs_create_u64("time_blocked",
+ 0644, d_spin_debug, &time_blocked);
+ return 0;
+}
+
+static inline void kvm_halt_stats(enum pv_lock_stats type)
+{
+ if (type == PV_HALT_QHEAD)
+ add_smp(&halt_qhead_stats, 1);
+ else if (type == PV_HALT_QNODE)
+ add_smp(&halt_qnode_stats, 1);
+ else /* type == PV_HALT_ABORT */
+ add_smp(&halt_abort_stats, 1);
+}
+
+static inline void kvm_lock_stats(enum pv_lock_stats type)
+{
+ if (type == PV_WAKE_KICKED)
+ add_smp(&wake_kick_stats, 1);
+ else if (type == PV_WAKE_SPURIOUS)
+ add_smp(&wake_spur_stats, 1);
+ else /* type == PV_KICK_NOHALT */
+ add_smp(&kick_nohlt_stats, 1);
+}
+
+static inline u64 spin_time_start(void)
+{
+ return sched_clock();
+}
+
+static inline void spin_time_accum_blocked(u64 start)
+{
+ u64 delta;
+
+ delta = sched_clock() - start;
+ add_smp(&time_blocked, delta);
+}
+
+fs_initcall(kvm_spinlock_debugfs);
+
+#else /* CONFIG_KVM_DEBUG_FS */
+static inline void kvm_halt_stats(enum pv_lock_stats type)
+{
+}
+
+static inline void kvm_lock_stats(enum pv_lock_stats type)
+{
+}
+
+static inline u64 spin_time_start(void)
+{
+ return 0;
+}
+
+static inline void spin_time_accum_blocked(u64 start)
+{
+}
+#endif /* CONFIG_KVM_DEBUG_FS */
+
+/*
+ * Halt the current CPU & release it back to the host
+ */
+static void kvm_halt_cpu(enum pv_lock_stats type, s8 *state, s8 sval)
+{
+ unsigned long flags;
+ u64 start;
+
+ if (in_nmi())
+ return;
+
+ /*
+ * Make sure an interrupt handler can't upset things in a
+ * partially setup state.
+ */
+ local_irq_save(flags);
+ /*
+ * Don't halt if the CPU state has been changed.
+ */
+ if (ACCESS_ONCE(*state) != sval) {
+ kvm_halt_stats(PV_HALT_ABORT);
+ goto out;
+ }
+ start = spin_time_start();
+ kvm_halt_stats(type);
+ if (arch_irqs_disabled_flags(flags))
+ halt();
+ else
+ safe_halt();
+ spin_time_accum_blocked(start);
+out:
+ local_irq_restore(flags);
+}
+#endif /* !CONFIG_QUEUE_SPINLOCK */
/*
* Setup pv_lock_ops to exploit KVM_FEATURE_PV_UNHALT if present.
@@ -806,8 +935,14 @@ void __init kvm_spinlock_init(void)
if (!kvm_para_has_feature(KVM_FEATURE_PV_UNHALT))
return;
+#ifdef CONFIG_QUEUE_SPINLOCK
+ pv_lock_ops.kick_cpu = kvm_kick_cpu;
+ pv_lock_ops.halt_cpu = kvm_halt_cpu;
+ pv_lock_ops.lockstat = kvm_lock_stats;
+#else
pv_lock_ops.lock_spinning = PV_CALLEE_SAVE(kvm_lock_spinning);
pv_lock_ops.unlock_kick = kvm_unlock_kick;
+#endif
}
static __init int kvm_spinlock_init_jump(void)
diff --git a/kernel/Kconfig.locks b/kernel/Kconfig.locks
index f185584..a70fdeb 100644
--- a/kernel/Kconfig.locks
+++ b/kernel/Kconfig.locks
@@ -229,4 +229,4 @@ config ARCH_USE_QUEUE_SPINLOCK
config QUEUE_SPINLOCK
def_bool y if ARCH_USE_QUEUE_SPINLOCK
- depends on SMP && !PARAVIRT_SPINLOCKS
+ depends on SMP && (!PARAVIRT_SPINLOCKS || !XEN)
--
1.7.1
^ permalink raw reply related [flat|nested] 102+ messages in thread* [PATCH v11 16/16] pvqspinlock, x86: Enable PV qspinlock for XEN
2014-05-30 15:43 [PATCH v11 00/16] qspinlock: a 4-byte queue spinlock with PV support Waiman Long
` (34 preceding siblings ...)
2014-05-30 15:44 ` Waiman Long
@ 2014-05-30 15:44 ` Waiman Long
2014-05-30 15:44 ` Waiman Long
2014-05-30 15:44 ` Waiman Long
37 siblings, 0 replies; 102+ messages in thread
From: Waiman Long @ 2014-05-30 15:44 UTC (permalink / raw)
To: Thomas Gleixner, Ingo Molnar, H. Peter Anvin, Peter Zijlstra
Cc: linux-arch, Waiman Long, Raghavendra K T, Gleb Natapov, kvm,
Scott J Norton, x86, Paolo Bonzini, linux-kernel, virtualization,
Chegu Vinod, David Vrabel, Oleg Nesterov, xen-devel,
Boris Ostrovsky, Paul E. McKenney, Linus Torvalds
This patch adds the necessary XEN specific code to allow XEN to
support the CPU halting and kicking operations needed by the queue
spinlock PV code.
Signed-off-by: Waiman Long <Waiman.Long@hp.com>
---
arch/x86/xen/spinlock.c | 147 +++++++++++++++++++++++++++++++++++++++++++++--
kernel/Kconfig.locks | 2 +-
2 files changed, 143 insertions(+), 6 deletions(-)
diff --git a/arch/x86/xen/spinlock.c b/arch/x86/xen/spinlock.c
index d1b6a32..2a259bb 100644
--- a/arch/x86/xen/spinlock.c
+++ b/arch/x86/xen/spinlock.c
@@ -17,6 +17,12 @@
#include "xen-ops.h"
#include "debugfs.h"
+static DEFINE_PER_CPU(int, lock_kicker_irq) = -1;
+static DEFINE_PER_CPU(char *, irq_name);
+static bool xen_pvspin = true;
+
+#ifndef CONFIG_QUEUE_SPINLOCK
+
enum xen_contention_stat {
TAKEN_SLOW,
TAKEN_SLOW_PICKUP,
@@ -100,12 +106,9 @@ struct xen_lock_waiting {
__ticket_t want;
};
-static DEFINE_PER_CPU(int, lock_kicker_irq) = -1;
-static DEFINE_PER_CPU(char *, irq_name);
static DEFINE_PER_CPU(struct xen_lock_waiting, lock_waiting);
static cpumask_t waiting_cpus;
-static bool xen_pvspin = true;
__visible void xen_lock_spinning(struct arch_spinlock *lock, __ticket_t want)
{
int irq = __this_cpu_read(lock_kicker_irq);
@@ -213,6 +216,118 @@ static void xen_unlock_kick(struct arch_spinlock *lock, __ticket_t next)
}
}
+#else /* CONFIG_QUEUE_SPINLOCK */
+
+#ifdef CONFIG_XEN_DEBUG_FS
+static u32 kick_nohlt_stats; /* Kick but not halt count */
+static u32 halt_qhead_stats; /* Queue head halting count */
+static u32 halt_qnode_stats; /* Queue node halting count */
+static u32 halt_abort_stats; /* Halting abort count */
+static u32 wake_kick_stats; /* Wakeup by kicking count */
+static u32 wake_spur_stats; /* Spurious wakeup count */
+static u64 time_blocked; /* Total blocking time */
+
+static inline void xen_halt_stats(enum pv_lock_stats type)
+{
+ if (type == PV_HALT_QHEAD)
+ add_smp(&halt_qhead_stats, 1);
+ else if (type == PV_HALT_QNODE)
+ add_smp(&halt_qnode_stats, 1);
+ else /* type == PV_HALT_ABORT */
+ add_smp(&halt_abort_stats, 1);
+}
+
+static inline void xen_lock_stats(enum pv_lock_stats type)
+{
+ if (type == PV_WAKE_KICKED)
+ add_smp(&wake_kick_stats, 1);
+ else if (type == PV_WAKE_SPURIOUS)
+ add_smp(&wake_spur_stats, 1);
+ else /* type == PV_KICK_NOHALT */
+ add_smp(&kick_nohlt_stats, 1);
+}
+
+static inline u64 spin_time_start(void)
+{
+ return sched_clock();
+}
+
+static inline void spin_time_accum_blocked(u64 start)
+{
+ u64 delta;
+
+ delta = sched_clock() - start;
+ add_smp(&time_blocked, delta);
+}
+#else /* CONFIG_XEN_DEBUG_FS */
+static inline void xen_halt_stats(enum pv_lock_stats type)
+{
+}
+
+static inline void xen_lock_stats(enum pv_lock_stats type)
+{
+}
+
+static inline u64 spin_time_start(void)
+{
+ return 0;
+}
+
+static inline void spin_time_accum_blocked(u64 start)
+{
+}
+#endif /* CONFIG_XEN_DEBUG_FS */
+
+static void xen_kick_cpu(int cpu)
+{
+ xen_send_IPI_one(cpu, XEN_SPIN_UNLOCK_VECTOR);
+}
+
+/*
+ * Halt the current CPU & release it back to the host
+ */
+static void xen_halt_cpu(enum pv_lock_stats type, s8 *state, s8 sval)
+{
+ int irq = __this_cpu_read(lock_kicker_irq);
+ unsigned long flags;
+ u64 start;
+
+ /* If kicker interrupts not initialized yet, just spin */
+ if (irq == -1)
+ return;
+
+ /*
+ * Make sure an interrupt handler can't upset things in a
+ * partially setup state.
+ */
+ local_irq_save(flags);
+ start = spin_time_start();
+
+ xen_halt_stats(type);
+ /* clear pending */
+ xen_clear_irq_pending(irq);
+
+ /* Allow interrupts while blocked */
+ local_irq_restore(flags);
+ /*
+ * Don't halt if the CPU state has been changed.
+ */
+ if (ACCESS_ONCE(*state) != sval) {
+ xen_halt_stats(PV_HALT_ABORT);
+ return;
+ }
+ /*
+ * If an interrupt happens here, it will leave the wakeup irq
+ * pending, which will cause xen_poll_irq() to return
+ * immediately.
+ */
+
+ /* Block until irq becomes pending (or perhaps a spurious wakeup) */
+ xen_poll_irq(irq);
+ spin_time_accum_blocked(start);
+}
+#endif /* CONFIG_QUEUE_SPINLOCK */
+
static irqreturn_t dummy_handler(int irq, void *dev_id)
{
BUG();
@@ -258,7 +373,6 @@ void xen_uninit_lock_cpu(int cpu)
per_cpu(irq_name, cpu) = NULL;
}
-
/*
* Our init of PV spinlocks is split in two init functions due to us
* using paravirt patching and jump labels patching and having to do
@@ -275,8 +389,15 @@ void __init xen_init_spinlocks(void)
return;
}
printk(KERN_DEBUG "xen: PV spinlocks enabled\n");
+
+#ifdef CONFIG_QUEUE_SPINLOCK
+ pv_lock_ops.kick_cpu = xen_kick_cpu;
+ pv_lock_ops.halt_cpu = xen_halt_cpu;
+ pv_lock_ops.lockstat = xen_lock_stats;
+#else
pv_lock_ops.lock_spinning = PV_CALLEE_SAVE(xen_lock_spinning);
pv_lock_ops.unlock_kick = xen_unlock_kick;
+#endif
}
/*
@@ -321,6 +442,7 @@ static int __init xen_spinlock_debugfs(void)
d_spin_debug = debugfs_create_dir("spinlocks", d_xen);
+#ifndef CONFIG_QUEUE_SPINLOCK
debugfs_create_u8("zero_stats", 0644, d_spin_debug, &zero_stats);
debugfs_create_u32("taken_slow", 0444, d_spin_debug,
@@ -340,7 +462,22 @@ static int __init xen_spinlock_debugfs(void)
debugfs_create_u32_array("histo_blocked", 0444, d_spin_debug,
spinlock_stats.histo_spin_blocked, HISTO_BUCKETS + 1);
-
+#else /* CONFIG_QUEUE_SPINLOCK */
+ debugfs_create_u32("kick_nohlt_stats",
+ 0644, d_spin_debug, &kick_nohlt_stats);
+ debugfs_create_u32("halt_qhead_stats",
+ 0644, d_spin_debug, &halt_qhead_stats);
+ debugfs_create_u32("halt_qnode_stats",
+ 0644, d_spin_debug, &halt_qnode_stats);
+ debugfs_create_u32("halt_abort_stats",
+ 0644, d_spin_debug, &halt_abort_stats);
+ debugfs_create_u32("wake_kick_stats",
+ 0644, d_spin_debug, &wake_kick_stats);
+ debugfs_create_u32("wake_spur_stats",
+ 0644, d_spin_debug, &wake_spur_stats);
+ debugfs_create_u64("time_blocked",
+ 0644, d_spin_debug, &time_blocked);
+#endif /* CONFIG_QUEUE_SPINLOCK */
return 0;
}
fs_initcall(xen_spinlock_debugfs);
diff --git a/kernel/Kconfig.locks b/kernel/Kconfig.locks
index a70fdeb..451e392 100644
--- a/kernel/Kconfig.locks
+++ b/kernel/Kconfig.locks
@@ -229,4 +229,4 @@ config ARCH_USE_QUEUE_SPINLOCK
config QUEUE_SPINLOCK
def_bool y if ARCH_USE_QUEUE_SPINLOCK
- depends on SMP && (!PARAVIRT_SPINLOCKS || !XEN)
+ depends on SMP
--
1.7.1
^ permalink raw reply related [flat|nested] 102+ messages in thread* [PATCH v11 16/16] pvqspinlock, x86: Enable PV qspinlock for XEN
2014-05-30 15:43 [PATCH v11 00/16] qspinlock: a 4-byte queue spinlock with PV support Waiman Long
` (35 preceding siblings ...)
2014-05-30 15:44 ` [PATCH v11 16/16] pvqspinlock, x86: Enable PV qspinlock for XEN Waiman Long
@ 2014-05-30 15:44 ` Waiman Long
2014-05-30 15:44 ` Waiman Long
37 siblings, 0 replies; 102+ messages in thread
From: Waiman Long @ 2014-05-30 15:44 UTC (permalink / raw)
To: Thomas Gleixner, Ingo Molnar, H. Peter Anvin, Peter Zijlstra
Cc: linux-arch, x86, linux-kernel, virtualization, xen-devel, kvm,
Paolo Bonzini, Konrad Rzeszutek Wilk, Boris Ostrovsky,
Paul E. McKenney, Rik van Riel, Linus Torvalds, Raghavendra K T,
David Vrabel, Oleg Nesterov, Gleb Natapov, Scott J Norton,
Chegu Vinod, Waiman Long
This patch adds the necessary XEN specific code to allow XEN to
support the CPU halting and kicking operations needed by the queue
spinlock PV code.
Signed-off-by: Waiman Long <Waiman.Long@hp.com>
---
arch/x86/xen/spinlock.c | 147 +++++++++++++++++++++++++++++++++++++++++++++--
kernel/Kconfig.locks | 2 +-
2 files changed, 143 insertions(+), 6 deletions(-)
diff --git a/arch/x86/xen/spinlock.c b/arch/x86/xen/spinlock.c
index d1b6a32..2a259bb 100644
--- a/arch/x86/xen/spinlock.c
+++ b/arch/x86/xen/spinlock.c
@@ -17,6 +17,12 @@
#include "xen-ops.h"
#include "debugfs.h"
+static DEFINE_PER_CPU(int, lock_kicker_irq) = -1;
+static DEFINE_PER_CPU(char *, irq_name);
+static bool xen_pvspin = true;
+
+#ifndef CONFIG_QUEUE_SPINLOCK
+
enum xen_contention_stat {
TAKEN_SLOW,
TAKEN_SLOW_PICKUP,
@@ -100,12 +106,9 @@ struct xen_lock_waiting {
__ticket_t want;
};
-static DEFINE_PER_CPU(int, lock_kicker_irq) = -1;
-static DEFINE_PER_CPU(char *, irq_name);
static DEFINE_PER_CPU(struct xen_lock_waiting, lock_waiting);
static cpumask_t waiting_cpus;
-static bool xen_pvspin = true;
__visible void xen_lock_spinning(struct arch_spinlock *lock, __ticket_t want)
{
int irq = __this_cpu_read(lock_kicker_irq);
@@ -213,6 +216,118 @@ static void xen_unlock_kick(struct arch_spinlock *lock, __ticket_t next)
}
}
+#else /* CONFIG_QUEUE_SPINLOCK */
+
+#ifdef CONFIG_XEN_DEBUG_FS
+static u32 kick_nohlt_stats; /* Kick but not halt count */
+static u32 halt_qhead_stats; /* Queue head halting count */
+static u32 halt_qnode_stats; /* Queue node halting count */
+static u32 halt_abort_stats; /* Halting abort count */
+static u32 wake_kick_stats; /* Wakeup by kicking count */
+static u32 wake_spur_stats; /* Spurious wakeup count */
+static u64 time_blocked; /* Total blocking time */
+
+static inline void xen_halt_stats(enum pv_lock_stats type)
+{
+ if (type == PV_HALT_QHEAD)
+ add_smp(&halt_qhead_stats, 1);
+ else if (type == PV_HALT_QNODE)
+ add_smp(&halt_qnode_stats, 1);
+ else /* type == PV_HALT_ABORT */
+ add_smp(&halt_abort_stats, 1);
+}
+
+static inline void xen_lock_stats(enum pv_lock_stats type)
+{
+ if (type == PV_WAKE_KICKED)
+ add_smp(&wake_kick_stats, 1);
+ else if (type == PV_WAKE_SPURIOUS)
+ add_smp(&wake_spur_stats, 1);
+ else /* type == PV_KICK_NOHALT */
+ add_smp(&kick_nohlt_stats, 1);
+}
+
+static inline u64 spin_time_start(void)
+{
+ return sched_clock();
+}
+
+static inline void spin_time_accum_blocked(u64 start)
+{
+ u64 delta;
+
+ delta = sched_clock() - start;
+ add_smp(&time_blocked, delta);
+}
+#else /* CONFIG_XEN_DEBUG_FS */
+static inline void xen_halt_stats(enum pv_lock_stats type)
+{
+}
+
+static inline void xen_lock_stats(enum pv_lock_stats type)
+{
+}
+
+static inline u64 spin_time_start(void)
+{
+ return 0;
+}
+
+static inline void spin_time_accum_blocked(u64 start)
+{
+}
+#endif /* CONFIG_XEN_DEBUG_FS */
+
+static void xen_kick_cpu(int cpu)
+{
+ xen_send_IPI_one(cpu, XEN_SPIN_UNLOCK_VECTOR);
+}
+
+/*
+ * Halt the current CPU & release it back to the host
+ */
+static void xen_halt_cpu(enum pv_lock_stats type, s8 *state, s8 sval)
+{
+ int irq = __this_cpu_read(lock_kicker_irq);
+ unsigned long flags;
+ u64 start;
+
+ /* If kicker interrupts not initialized yet, just spin */
+ if (irq == -1)
+ return;
+
+ /*
+ * Make sure an interrupt handler can't upset things in a
+ * partially setup state.
+ */
+ local_irq_save(flags);
+ start = spin_time_start();
+
+ xen_halt_stats(type);
+ /* clear pending */
+ xen_clear_irq_pending(irq);
+
+ /* Allow interrupts while blocked */
+ local_irq_restore(flags);
+ /*
+ * Don't halt if the CPU state has been changed.
+ */
+ if (ACCESS_ONCE(*state) != sval) {
+ xen_halt_stats(PV_HALT_ABORT);
+ return;
+ }
+ /*
+ * If an interrupt happens here, it will leave the wakeup irq
+ * pending, which will cause xen_poll_irq() to return
+ * immediately.
+ */
+
+ /* Block until irq becomes pending (or perhaps a spurious wakeup) */
+ xen_poll_irq(irq);
+ spin_time_accum_blocked(start);
+}
+#endif /* CONFIG_QUEUE_SPINLOCK */
+
static irqreturn_t dummy_handler(int irq, void *dev_id)
{
BUG();
@@ -258,7 +373,6 @@ void xen_uninit_lock_cpu(int cpu)
per_cpu(irq_name, cpu) = NULL;
}
-
/*
* Our init of PV spinlocks is split in two init functions due to us
* using paravirt patching and jump labels patching and having to do
@@ -275,8 +389,15 @@ void __init xen_init_spinlocks(void)
return;
}
printk(KERN_DEBUG "xen: PV spinlocks enabled\n");
+
+#ifdef CONFIG_QUEUE_SPINLOCK
+ pv_lock_ops.kick_cpu = xen_kick_cpu;
+ pv_lock_ops.halt_cpu = xen_halt_cpu;
+ pv_lock_ops.lockstat = xen_lock_stats;
+#else
pv_lock_ops.lock_spinning = PV_CALLEE_SAVE(xen_lock_spinning);
pv_lock_ops.unlock_kick = xen_unlock_kick;
+#endif
}
/*
@@ -321,6 +442,7 @@ static int __init xen_spinlock_debugfs(void)
d_spin_debug = debugfs_create_dir("spinlocks", d_xen);
+#ifndef CONFIG_QUEUE_SPINLOCK
debugfs_create_u8("zero_stats", 0644, d_spin_debug, &zero_stats);
debugfs_create_u32("taken_slow", 0444, d_spin_debug,
@@ -340,7 +462,22 @@ static int __init xen_spinlock_debugfs(void)
debugfs_create_u32_array("histo_blocked", 0444, d_spin_debug,
spinlock_stats.histo_spin_blocked, HISTO_BUCKETS + 1);
-
+#else /* CONFIG_QUEUE_SPINLOCK */
+ debugfs_create_u32("kick_nohlt_stats",
+ 0644, d_spin_debug, &kick_nohlt_stats);
+ debugfs_create_u32("halt_qhead_stats",
+ 0644, d_spin_debug, &halt_qhead_stats);
+ debugfs_create_u32("halt_qnode_stats",
+ 0644, d_spin_debug, &halt_qnode_stats);
+ debugfs_create_u32("halt_abort_stats",
+ 0644, d_spin_debug, &halt_abort_stats);
+ debugfs_create_u32("wake_kick_stats",
+ 0644, d_spin_debug, &wake_kick_stats);
+ debugfs_create_u32("wake_spur_stats",
+ 0644, d_spin_debug, &wake_spur_stats);
+ debugfs_create_u64("time_blocked",
+ 0644, d_spin_debug, &time_blocked);
+#endif /* CONFIG_QUEUE_SPINLOCK */
return 0;
}
fs_initcall(xen_spinlock_debugfs);
diff --git a/kernel/Kconfig.locks b/kernel/Kconfig.locks
index a70fdeb..451e392 100644
--- a/kernel/Kconfig.locks
+++ b/kernel/Kconfig.locks
@@ -229,4 +229,4 @@ config ARCH_USE_QUEUE_SPINLOCK
config QUEUE_SPINLOCK
def_bool y if ARCH_USE_QUEUE_SPINLOCK
- depends on SMP && (!PARAVIRT_SPINLOCKS || !XEN)
+ depends on SMP
--
1.7.1
^ permalink raw reply related [flat|nested] 102+ messages in thread* [PATCH v11 16/16] pvqspinlock, x86: Enable PV qspinlock for XEN
2014-05-30 15:43 [PATCH v11 00/16] qspinlock: a 4-byte queue spinlock with PV support Waiman Long
` (36 preceding siblings ...)
2014-05-30 15:44 ` Waiman Long
@ 2014-05-30 15:44 ` Waiman Long
37 siblings, 0 replies; 102+ messages in thread
From: Waiman Long @ 2014-05-30 15:44 UTC (permalink / raw)
To: Thomas Gleixner, Ingo Molnar, H. Peter Anvin, Peter Zijlstra
Cc: linux-arch, Waiman Long, Rik van Riel, Raghavendra K T,
Gleb Natapov, kvm, Konrad Rzeszutek Wilk, Scott J Norton, x86,
Paolo Bonzini, linux-kernel, virtualization, Chegu Vinod,
David Vrabel, Oleg Nesterov, xen-devel, Boris Ostrovsky,
Paul E. McKenney, Linus Torvalds
This patch adds the necessary XEN specific code to allow XEN to
support the CPU halting and kicking operations needed by the queue
spinlock PV code.
Signed-off-by: Waiman Long <Waiman.Long@hp.com>
---
arch/x86/xen/spinlock.c | 147 +++++++++++++++++++++++++++++++++++++++++++++--
kernel/Kconfig.locks | 2 +-
2 files changed, 143 insertions(+), 6 deletions(-)
diff --git a/arch/x86/xen/spinlock.c b/arch/x86/xen/spinlock.c
index d1b6a32..2a259bb 100644
--- a/arch/x86/xen/spinlock.c
+++ b/arch/x86/xen/spinlock.c
@@ -17,6 +17,12 @@
#include "xen-ops.h"
#include "debugfs.h"
+static DEFINE_PER_CPU(int, lock_kicker_irq) = -1;
+static DEFINE_PER_CPU(char *, irq_name);
+static bool xen_pvspin = true;
+
+#ifndef CONFIG_QUEUE_SPINLOCK
+
enum xen_contention_stat {
TAKEN_SLOW,
TAKEN_SLOW_PICKUP,
@@ -100,12 +106,9 @@ struct xen_lock_waiting {
__ticket_t want;
};
-static DEFINE_PER_CPU(int, lock_kicker_irq) = -1;
-static DEFINE_PER_CPU(char *, irq_name);
static DEFINE_PER_CPU(struct xen_lock_waiting, lock_waiting);
static cpumask_t waiting_cpus;
-static bool xen_pvspin = true;
__visible void xen_lock_spinning(struct arch_spinlock *lock, __ticket_t want)
{
int irq = __this_cpu_read(lock_kicker_irq);
@@ -213,6 +216,118 @@ static void xen_unlock_kick(struct arch_spinlock *lock, __ticket_t next)
}
}
+#else /* CONFIG_QUEUE_SPINLOCK */
+
+#ifdef CONFIG_XEN_DEBUG_FS
+static u32 kick_nohlt_stats; /* Kick but not halt count */
+static u32 halt_qhead_stats; /* Queue head halting count */
+static u32 halt_qnode_stats; /* Queue node halting count */
+static u32 halt_abort_stats; /* Halting abort count */
+static u32 wake_kick_stats; /* Wakeup by kicking count */
+static u32 wake_spur_stats; /* Spurious wakeup count */
+static u64 time_blocked; /* Total blocking time */
+
+static inline void xen_halt_stats(enum pv_lock_stats type)
+{
+ if (type == PV_HALT_QHEAD)
+ add_smp(&halt_qhead_stats, 1);
+ else if (type == PV_HALT_QNODE)
+ add_smp(&halt_qnode_stats, 1);
+ else /* type == PV_HALT_ABORT */
+ add_smp(&halt_abort_stats, 1);
+}
+
+static inline void xen_lock_stats(enum pv_lock_stats type)
+{
+ if (type == PV_WAKE_KICKED)
+ add_smp(&wake_kick_stats, 1);
+ else if (type == PV_WAKE_SPURIOUS)
+ add_smp(&wake_spur_stats, 1);
+ else /* type == PV_KICK_NOHALT */
+ add_smp(&kick_nohlt_stats, 1);
+}
+
+static inline u64 spin_time_start(void)
+{
+ return sched_clock();
+}
+
+static inline void spin_time_accum_blocked(u64 start)
+{
+ u64 delta;
+
+ delta = sched_clock() - start;
+ add_smp(&time_blocked, delta);
+}
+#else /* CONFIG_XEN_DEBUG_FS */
+static inline void xen_halt_stats(enum pv_lock_stats type)
+{
+}
+
+static inline void xen_lock_stats(enum pv_lock_stats type)
+{
+}
+
+static inline u64 spin_time_start(void)
+{
+ return 0;
+}
+
+static inline void spin_time_accum_blocked(u64 start)
+{
+}
+#endif /* CONFIG_XEN_DEBUG_FS */
+
+static void xen_kick_cpu(int cpu)
+{
+ xen_send_IPI_one(cpu, XEN_SPIN_UNLOCK_VECTOR);
+}
+
+/*
+ * Halt the current CPU & release it back to the host
+ */
+static void xen_halt_cpu(enum pv_lock_stats type, s8 *state, s8 sval)
+{
+ int irq = __this_cpu_read(lock_kicker_irq);
+ unsigned long flags;
+ u64 start;
+
+ /* If kicker interrupts not initialized yet, just spin */
+ if (irq == -1)
+ return;
+
+ /*
+ * Make sure an interrupt handler can't upset things in a
+ * partially setup state.
+ */
+ local_irq_save(flags);
+ start = spin_time_start();
+
+ xen_halt_stats(type);
+ /* clear pending */
+ xen_clear_irq_pending(irq);
+
+ /* Allow interrupts while blocked */
+ local_irq_restore(flags);
+ /*
+ * Don't halt if the CPU state has been changed.
+ */
+ if (ACCESS_ONCE(*state) != sval) {
+ xen_halt_stats(PV_HALT_ABORT);
+ return;
+ }
+ /*
+ * If an interrupt happens here, it will leave the wakeup irq
+ * pending, which will cause xen_poll_irq() to return
+ * immediately.
+ */
+
+ /* Block until irq becomes pending (or perhaps a spurious wakeup) */
+ xen_poll_irq(irq);
+ spin_time_accum_blocked(start);
+}
+#endif /* CONFIG_QUEUE_SPINLOCK */
+
static irqreturn_t dummy_handler(int irq, void *dev_id)
{
BUG();
@@ -258,7 +373,6 @@ void xen_uninit_lock_cpu(int cpu)
per_cpu(irq_name, cpu) = NULL;
}
-
/*
* Our init of PV spinlocks is split in two init functions due to us
* using paravirt patching and jump labels patching and having to do
@@ -275,8 +389,15 @@ void __init xen_init_spinlocks(void)
return;
}
printk(KERN_DEBUG "xen: PV spinlocks enabled\n");
+
+#ifdef CONFIG_QUEUE_SPINLOCK
+ pv_lock_ops.kick_cpu = xen_kick_cpu;
+ pv_lock_ops.halt_cpu = xen_halt_cpu;
+ pv_lock_ops.lockstat = xen_lock_stats;
+#else
pv_lock_ops.lock_spinning = PV_CALLEE_SAVE(xen_lock_spinning);
pv_lock_ops.unlock_kick = xen_unlock_kick;
+#endif
}
/*
@@ -321,6 +442,7 @@ static int __init xen_spinlock_debugfs(void)
d_spin_debug = debugfs_create_dir("spinlocks", d_xen);
+#ifndef CONFIG_QUEUE_SPINLOCK
debugfs_create_u8("zero_stats", 0644, d_spin_debug, &zero_stats);
debugfs_create_u32("taken_slow", 0444, d_spin_debug,
@@ -340,7 +462,22 @@ static int __init xen_spinlock_debugfs(void)
debugfs_create_u32_array("histo_blocked", 0444, d_spin_debug,
spinlock_stats.histo_spin_blocked, HISTO_BUCKETS + 1);
-
+#else /* CONFIG_QUEUE_SPINLOCK */
+ debugfs_create_u32("kick_nohlt_stats",
+ 0644, d_spin_debug, &kick_nohlt_stats);
+ debugfs_create_u32("halt_qhead_stats",
+ 0644, d_spin_debug, &halt_qhead_stats);
+ debugfs_create_u32("halt_qnode_stats",
+ 0644, d_spin_debug, &halt_qnode_stats);
+ debugfs_create_u32("halt_abort_stats",
+ 0644, d_spin_debug, &halt_abort_stats);
+ debugfs_create_u32("wake_kick_stats",
+ 0644, d_spin_debug, &wake_kick_stats);
+ debugfs_create_u32("wake_spur_stats",
+ 0644, d_spin_debug, &wake_spur_stats);
+ debugfs_create_u64("time_blocked",
+ 0644, d_spin_debug, &time_blocked);
+#endif /* CONFIG_QUEUE_SPINLOCK */
return 0;
}
fs_initcall(xen_spinlock_debugfs);
diff --git a/kernel/Kconfig.locks b/kernel/Kconfig.locks
index a70fdeb..451e392 100644
--- a/kernel/Kconfig.locks
+++ b/kernel/Kconfig.locks
@@ -229,4 +229,4 @@ config ARCH_USE_QUEUE_SPINLOCK
config QUEUE_SPINLOCK
def_bool y if ARCH_USE_QUEUE_SPINLOCK
- depends on SMP && (!PARAVIRT_SPINLOCKS || !XEN)
+ depends on SMP
--
1.7.1
^ permalink raw reply related [flat|nested] 102+ messages in thread