From 34a335b54e382fc0c53c9dc62217d4ce85c07746 Mon Sep 17 00:00:00 2001
From: Alex Kogan <alex.kogan@oracle.com>
Date: Mon, 22 Mar 2021 11:44:08 -0400
Subject: [PATCH v13 7/7] locking/qspinlock: Avoid unnecessary stores into
 queue node structure.

In the common case, cna_wait_head_or_lock() called by a thread T is
expected to return with LOCAL_WAITER_FOUND stored in the
"partial_order" field of T's queue node. This store can cause a cache miss
for the predecessor of T passing the lock to T. This can be avoided
by simply initializing "partial_order" with that value.

Signed-off-by: Alex Kogan <alex.kogan@oracle.com>
---
 kernel/locking/qspinlock_cna.h | 36 +++++++++++++++-------------------
 1 file changed, 16 insertions(+), 20 deletions(-)

diff --git a/kernel/locking/qspinlock_cna.h b/kernel/locking/qspinlock_cna.h
index 621399242735..b2d84c1f6882 100644
--- a/kernel/locking/qspinlock_cna.h
+++ b/kernel/locking/qspinlock_cna.h
@@ -159,6 +159,7 @@ static __always_inline void cna_init_node(struct mcs_spinlock *node)
 
 	cn->numa_node = priority ? CNA_PRIORITY_NODE : cn->real_numa_node;
 	cn->start_time = 0;
+	cn->partial_order = LOCAL_WAITER_FOUND;
 }
 
 /*
@@ -284,13 +285,15 @@ static void cna_splice_next(struct mcs_spinlock *node,
  * the same NUMA node as the lock holder; if not, and it has a waiter behind
  * it in the main queue, move the former onto the secondary queue.
  */
-static u32 cna_order_queue(struct mcs_spinlock *node)
+static void cna_order_queue(struct mcs_spinlock *node)
 {
 	struct mcs_spinlock *next = READ_ONCE(node->next);
 	int numa_node, next_numa_node;
 
-	if (!next)
-		return LOCAL_WAITER_NOT_FOUND;
+	if (!next) {
+		((struct cna_node *)node)->partial_order = LOCAL_WAITER_NOT_FOUND;
+		return;
+	}
 
 	numa_node = ((struct cna_node *)node)->numa_node;
 	next_numa_node = ((struct cna_node *)next)->numa_node;
@@ -310,7 +313,6 @@ static u32 cna_order_queue(struct mcs_spinlock *node)
 		 */
 		((struct cna_node *)next)->numa_node = numa_node;
 	}
-	return LOCAL_WAITER_FOUND;
 }
 
 /* Abuse the pv_wait_head_or_lock() hook to get some work done */
@@ -322,12 +324,14 @@ static __always_inline u32 cna_wait_head_or_lock(struct qspinlock *lock,
 	if (node->locked <= 1 && probably(SHUFFLE_REDUCTION_PROB_ARG)) {
 		/*
 		 * When the secondary queue is empty, skip the call to
-		 * cna_order_queue() with high probability. This optimization
+		 * cna_order_queue() below with high probability. This optimization
 		 * reduces the overhead of unnecessary shuffling of threads
 		 * between waiting queues when the lock is only lightly contended.
 		 */
-		cn->partial_order = LOCAL_WAITER_FOUND;
-	} else if (!cn->start_time || !intra_node_threshold_reached(cn)) {
+		return 0;
+	}
+
+	if (!cn->start_time || !intra_node_threshold_reached(cn)) {
 		/*
 		 * We are at the head of the wait queue, no need to use
 		 * the fake NUMA node ID.
@@ -339,7 +343,7 @@ static __always_inline u32 cna_wait_head_or_lock(struct qspinlock *lock,
 		 * Try and put the time otherwise spent spin waiting on
 		 * _Q_LOCKED_PENDING_MASK to use by sorting our lists.
 		 */
-		cn->partial_order = cna_order_queue(node);
+		cna_order_queue(node);
 	} else {
 		cn->partial_order = FLUSH_SECONDARY_QUEUE;
 	}
@@ -356,19 +360,12 @@ static inline void cna_lock_handoff(struct mcs_spinlock *node,
 	u32 partial_order = cn->partial_order;
 
 	if (partial_order == LOCAL_WAITER_NOT_FOUND)
-		partial_order = cna_order_queue(node);
-
-	/*
-	 * At this point, we must have a successor in the main queue, so if we call
-	 * cna_order_queue() above, we will find a local waiter, either real or
-	 * fake one.
-	 */
-	WARN_ON(partial_order == LOCAL_WAITER_NOT_FOUND);
+		cna_order_queue(node);
 
-	if (partial_order == LOCAL_WAITER_FOUND) {
+	if (partial_order != FLUSH_SECONDARY_QUEUE) {
 		/*
-		 * We found a local waiter; reload @next in case it
-		 * was changed by cna_order_queue().
+		 * We found a local waiter, either real or fake one;
+		 * reload @next in case it was changed by cna_order_queue().
 		 */
 		next = node->next;
 		if (node->locked > 1) {
@@ -376,7 +373,6 @@ static inline void cna_lock_handoff(struct mcs_spinlock *node,
 			((struct cna_node *)next)->start_time = cn->start_time;
 		}
 	} else {
-		WARN_ON(partial_order != FLUSH_SECONDARY_QUEUE);
 		/*
 		 * We decided to flush the secondary queue;
 		 * this can only happen if that queue is not empty.
-- 
2.24.3 (Apple Git-128)

