[RFC, PATCH] 4/5 rcu lock update: Hide locking details

[RFC, PATCH] 4/5 rcu lock update: Hide locking details

[Manfred Spraul]

Hi,

Step four for reducing cacheline trashing within rcupdate.c,
second step for a hierarchical cpumask bitmap: 

Move the locking calls that protect rcu_cpu_mask into cpu_quiet.

What do you think?

--
	Manfred

// $Header$
// Kernel Version:
//  VERSION = 2
//  PATCHLEVEL = 6
//  SUBLEVEL = 6
//  EXTRAVERSION = -mm4
--- 2.6/kernel/rcupdate.c	2004-05-23 11:57:48.000000000 +0200
+++ build-2.6/kernel/rcupdate.c	2004-05-23 11:56:32.000000000 +0200
@@ -153,14 +153,27 @@
  * Clear it from the cpu mask and complete the grace period if it was the last
  * cpu. Start another grace period if someone has further entries pending
  */
-static void cpu_quiet(int cpu)
+static void cpu_quiet(int cpu, int force)
 {
+	spin_lock(&rcu_state.mutex);
+
+	if (unlikely(rcu_ctrlblk.completed == rcu_ctrlblk.cur))
+		goto out_unlock;
+	/*
+	 * RCU_quiescbatch/batch.cur and the cpu bitmap can come out of sync
+	 * during cpu startup. Ignore the quiescent state.
+	 */
+	if (unlikely(RCU_quiescbatch(cpu) != rcu_ctrlblk.cur) && likely(!force))
+       		goto out_unlock;
+
 	cpu_clear(cpu, rcu_state.rcu_cpu_mask);
 	if (cpus_empty(rcu_state.rcu_cpu_mask)) {
 		/* batch completed ! */
 		rcu_ctrlblk.completed = rcu_ctrlblk.cur;
 		rcu_start_batch(0);
 	}
+out_unlock:
+	spin_unlock(&rcu_state.mutex);
 }
 
 /*
@@ -196,15 +209,8 @@
 		return;
 	RCU_qs_pending(cpu) = 0;
 
-	spin_lock(&rcu_state.mutex);
-	/*
-	 * RCU_quiescbatch/batch.cur and the cpu bitmap can come out of sync
-	 * during cpu startup. Ignore the quiescent state.
-	 */
-	if (likely(RCU_quiescbatch(cpu) == rcu_ctrlblk.cur)) 
-		cpu_quiet(cpu);
+	cpu_quiet(cpu, 0);
 
-	spin_unlock(&rcu_state.mutex);
 }
 
 
@@ -234,11 +240,7 @@
 	 * we can block indefinitely waiting for it, so flush
 	 * it here
 	 */
-	spin_lock_bh(&rcu_state.mutex);
-	if (rcu_ctrlblk.cur != rcu_ctrlblk.completed)
-		cpu_quiet(cpu);
-unlock:
-	spin_unlock_bh(&rcu_state.mutex);
+	cpu_quiet(cpu, 1);
 
 	rcu_move_batch(&RCU_curlist(cpu));
 	rcu_move_batch(&RCU_nxtlist(cpu));