[PATCH-tip v5 16/21] TP-futex: Enable kernel reader lock stealing

[Waiman Long <longman@redhat.com>]

The TP-futexes prefer writers as the code path is much simpler for
them. This may not be good for overall system performance especially
if there is a fair amount of reader activities.

This patch enables kernel reader to steal the lock when the futex is
currently reader-owned and the lock handoff mechanism hasn't been
enabled yet. A new field locksteal_disabled is added to the futex
state object for controlling reader lock stealing. So the waiting
reader must retrieve the futex state object first before doing it.

Signed-off-by: Waiman Long <longman@redhat.com>
---
 kernel/futex.c | 23 +++++++++++++++++++++--
 1 file changed, 21 insertions(+), 2 deletions(-)

diff --git a/kernel/futex.c b/kernel/futex.c
index 1f97c85..90c8c80 100644
--- a/kernel/futex.c
+++ b/kernel/futex.c
@@ -239,6 +239,7 @@ struct futex_state {
 	atomic_t refcount;
 
 	u32 handoff_pid;			/* For TP futexes only */
+	int locksteal_disabled;			/* For TP futexes only */
 
 	enum futex_type type;
 	union futex_key key;
@@ -3438,6 +3439,7 @@ void exit_robust_list(struct task_struct *curr)
 	state = alloc_futex_state();
 	state->type = TYPE_TP;
 	state->key = *key;
+	state->locksteal_disabled = false;
 	list_add(&state->fs_list, &hb->fs_head);
 	WARN_ON(atomic_read(&state->refcount) != 1);
 
@@ -3783,9 +3785,10 @@ static int futex_spin_on_owner(u32 __user *uaddr, const u32 vpid,
 				WRITE_ONCE(state->handoff_pid, vpid);
 
 				/*
-				 * Disable handoff check.
+				 * Disable handoff check and reader lock
+				 * stealing.
 				 */
-				handoff_set = true;
+				state->locksteal_disabled = handoff_set = true;
 			}
 		}
 
@@ -3888,6 +3891,7 @@ static int futex_spin_on_owner(u32 __user *uaddr, const u32 vpid,
 	 */
 	WRITE_ONCE(state->mutex_owner, NULL);
 	WRITE_ONCE(state->handoff_pid, 0);
+	state->locksteal_disabled = false;
 
 	preempt_enable();
 	return (ret < 0) ? ret : TP_STATUS_SLEEP(ret, nsleep);
@@ -3968,6 +3972,21 @@ static noinline int futex_lock(u32 __user *uaddr, unsigned int flags,
 		goto out_put_state_key;
 	}
 
+	/*
+	 * For reader, we will try to steal the lock here as if it is the
+	 * top waiter without taking the serialization mutex if reader
+	 * lock stealing isn't disabled even though the FUTEX_WAITERS bit
+	 * is set.
+	 */
+	if (shared && !state->locksteal_disabled) {
+		ret = futex_trylock(uaddr, vpid, &uval, true);
+		if (ret) {
+			if (ret > 0)
+				ret = TP_LOCK_STOLEN;
+			goto out_put_state_key;
+		}
+	}
+
 	if (to)
 		hrtimer_start_expires(&to->timer, HRTIMER_MODE_ABS);
 
-- 
1.8.3.1