[patch 0/2] posix-timers: avoid nohz_full= IPIs via task cpu masks

[patch 0/2] posix-timers: avoid nohz_full= IPIs via task cpu masks

[Marcelo Tosatti]

This patchset avoids IPIs to nohz_full= CPUs when the intersection 
between the set of nohz_full CPUs and task allowed cpus is null.

See individual patches for details.

[patch 1/2] nohz: try to avoid IPI when configuring per-CPU posix timer

[Marcelo Tosatti]

When enabling per-CPU posix timers, an IPI to nohz_full CPUs might be
performed (to re-read the dependencies and possibly not re-enter
nohz_full on a given CPU).

A common case is for applications that run on nohz_full= CPUs 
to not use POSIX timers (eg DPDK). This patch skips the IPI 
in case the task allowed mask does not intersect with nohz_full= CPU mask,
when going through tick_nohz_dep_set_signal.

This reduces interruptions to nohz_full= CPUs.

Signed-off-by: Marcelo Tosatti <mtosatti@redhat.com>

---
 include/linux/tick.h           |   11 +++++++----
 kernel/time/posix-cpu-timers.c |    4 ++--
 kernel/time/tick-sched.c       |   27 +++++++++++++++++++++++++--
 3 files changed, 34 insertions(+), 8 deletions(-)

Index: linux-2.6/include/linux/tick.h
===================================================================
--- linux-2.6.orig/include/linux/tick.h
+++ linux-2.6/include/linux/tick.h
@@ -207,7 +207,8 @@ extern void tick_nohz_dep_set_task(struc
 				   enum tick_dep_bits bit);
 extern void tick_nohz_dep_clear_task(struct task_struct *tsk,
 				     enum tick_dep_bits bit);
-extern void tick_nohz_dep_set_signal(struct signal_struct *signal,
+extern void tick_nohz_dep_set_signal(struct task_struct *tsk,
+				     struct signal_struct *signal,
 				     enum tick_dep_bits bit);
 extern void tick_nohz_dep_clear_signal(struct signal_struct *signal,
 				       enum tick_dep_bits bit);
@@ -252,11 +253,12 @@ static inline void tick_dep_clear_task(s
 	if (tick_nohz_full_enabled())
 		tick_nohz_dep_clear_task(tsk, bit);
 }
-static inline void tick_dep_set_signal(struct signal_struct *signal,
+static inline void tick_dep_set_signal(struct task_struct *tsk,
+				       struct signal_struct *signal,
 				       enum tick_dep_bits bit)
 {
 	if (tick_nohz_full_enabled())
-		tick_nohz_dep_set_signal(signal, bit);
+		tick_nohz_dep_set_signal(tsk, signal, bit);
 }
 static inline void tick_dep_clear_signal(struct signal_struct *signal,
 					 enum tick_dep_bits bit)
@@ -284,7 +286,8 @@ static inline void tick_dep_set_task(str
 				     enum tick_dep_bits bit) { }
 static inline void tick_dep_clear_task(struct task_struct *tsk,
 				       enum tick_dep_bits bit) { }
-static inline void tick_dep_set_signal(struct signal_struct *signal,
+static inline void tick_dep_set_signal(struct task_struct *tsk,
+				       struct signal_struct *signal,
 				       enum tick_dep_bits bit) { }
 static inline void tick_dep_clear_signal(struct signal_struct *signal,
 					 enum tick_dep_bits bit) { }
Index: linux-2.6/kernel/time/posix-cpu-timers.c
===================================================================
--- linux-2.6.orig/kernel/time/posix-cpu-timers.c
+++ linux-2.6/kernel/time/posix-cpu-timers.c
@@ -523,7 +523,7 @@ static void arm_timer(struct k_itimer *t
 	if (CPUCLOCK_PERTHREAD(timer->it_clock))
 		tick_dep_set_task(p, TICK_DEP_BIT_POSIX_TIMER);
 	else
-		tick_dep_set_signal(p->signal, TICK_DEP_BIT_POSIX_TIMER);
+		tick_dep_set_signal(p, p->signal, TICK_DEP_BIT_POSIX_TIMER);
 }
 
 /*
@@ -1358,7 +1358,7 @@ void set_process_cpu_timer(struct task_s
 	if (*newval < *nextevt)
 		*nextevt = *newval;
 
-	tick_dep_set_signal(tsk->signal, TICK_DEP_BIT_POSIX_TIMER);
+	tick_dep_set_signal(tsk, tsk->signal, TICK_DEP_BIT_POSIX_TIMER);
 }
 
 static int do_cpu_nanosleep(const clockid_t which_clock, int flags,
Index: linux-2.6/kernel/time/tick-sched.c
===================================================================
--- linux-2.6.orig/kernel/time/tick-sched.c
+++ linux-2.6/kernel/time/tick-sched.c
@@ -302,6 +302,27 @@ static void tick_nohz_dep_set_all(atomic
 }
 
 /*
+ * Set bit on nohz full dependency, kicking all cpus
+ * only if task can run on nohz full CPUs.
+ */
+static void tick_nohz_dep_set_all_cond(struct task_struct *tsk,
+				       atomic_t *dep,
+				       enum tick_dep_bits bit)
+{
+	int prev;
+	unsigned long flags;
+
+	prev = atomic_fetch_or(BIT(bit), dep);
+	if (prev)
+		return;
+
+	raw_spin_lock_irqsave(&tsk->pi_lock, flags);
+	if (cpumask_intersects(&tsk->cpus_mask, tick_nohz_full_mask))
+		tick_nohz_full_kick_all();
+	raw_spin_unlock_irqrestore(&tsk->pi_lock, flags);
+}
+
+/*
  * Set a global tick dependency. Used by perf events that rely on freq and
  * by unstable clock.
  */
@@ -382,9 +403,11 @@ EXPORT_SYMBOL_GPL(tick_nohz_dep_clear_ta
  * Set a per-taskgroup tick dependency. Posix CPU timers need this in order to elapse
  * per process timers.
  */
-void tick_nohz_dep_set_signal(struct signal_struct *sig, enum tick_dep_bits bit)
+void tick_nohz_dep_set_signal(struct task_struct *tsk,
+			      struct signal_struct *sig,
+			      enum tick_dep_bits bit)
 {
-	tick_nohz_dep_set_all(&sig->tick_dep_mask, bit);
+	tick_nohz_dep_set_all_cond(tsk, &sig->tick_dep_mask, bit);
 }
 
 void tick_nohz_dep_clear_signal(struct signal_struct *sig, enum tick_dep_bits bit)

Re: [patch 1/2] nohz: try to avoid IPI when configuring per-CPU posix timer

[Frederic Weisbecker]

On Tue, Aug 25, 2020 at 03:41:48PM -0300, Marcelo Tosatti wrote:
> When enabling per-CPU posix timers, an IPI to nohz_full CPUs might be
> performed (to re-read the dependencies and possibly not re-enter
> nohz_full on a given CPU).
> 
> A common case is for applications that run on nohz_full= CPUs 
> to not use POSIX timers (eg DPDK). This patch skips the IPI 
> in case the task allowed mask does not intersect with nohz_full= CPU mask,
> when going through tick_nohz_dep_set_signal.
> 
> This reduces interruptions to nohz_full= CPUs.
> 
> Signed-off-by: Marcelo Tosatti <mtosatti@redhat.com>
[...]
>  /*
> + * Set bit on nohz full dependency, kicking all cpus
> + * only if task can run on nohz full CPUs.
> + */
> +static void tick_nohz_dep_set_all_cond(struct task_struct *tsk,
> +				       atomic_t *dep,
> +				       enum tick_dep_bits bit)
> +{
> +	int prev;
> +	unsigned long flags;
> +
> +	prev = atomic_fetch_or(BIT(bit), dep);
> +	if (prev)
> +		return;
> +
> +	raw_spin_lock_irqsave(&tsk->pi_lock, flags);
> +	if (cpumask_intersects(&tsk->cpus_mask, tick_nohz_full_mask))
> +		tick_nohz_full_kick_all();

So that's for one task but what about the other threads in that
process? We are setting the tick dependency on all tasks sharing that
struct signal.

Thanks.

> +	raw_spin_unlock_irqrestore(&tsk->pi_lock, flags);
> +}
> +

Re: [patch 1/2] nohz: try to avoid IPI when configuring per-CPU posix timer

[Marcelo Tosatti]

On Wed, Sep 02, 2020 at 01:38:59AM +0200, Frederic Weisbecker wrote:
> On Tue, Aug 25, 2020 at 03:41:48PM -0300, Marcelo Tosatti wrote:
> > When enabling per-CPU posix timers, an IPI to nohz_full CPUs might be
> > performed (to re-read the dependencies and possibly not re-enter
> > nohz_full on a given CPU).
> > 
> > A common case is for applications that run on nohz_full= CPUs 
> > to not use POSIX timers (eg DPDK). This patch skips the IPI 
> > in case the task allowed mask does not intersect with nohz_full= CPU mask,
> > when going through tick_nohz_dep_set_signal.
> > 
> > This reduces interruptions to nohz_full= CPUs.
> > 
> > Signed-off-by: Marcelo Tosatti <mtosatti@redhat.com>
> [...]
> >  /*
> > + * Set bit on nohz full dependency, kicking all cpus
> > + * only if task can run on nohz full CPUs.
> > + */
> > +static void tick_nohz_dep_set_all_cond(struct task_struct *tsk,
> > +				       atomic_t *dep,
> > +				       enum tick_dep_bits bit)
> > +{
> > +	int prev;
> > +	unsigned long flags;
> > +
> > +	prev = atomic_fetch_or(BIT(bit), dep);
> > +	if (prev)
> > +		return;
> > +
> > +	raw_spin_lock_irqsave(&tsk->pi_lock, flags);
> > +	if (cpumask_intersects(&tsk->cpus_mask, tick_nohz_full_mask))
> > +		tick_nohz_full_kick_all();
> 
> So that's for one task but what about the other threads in that
> process? We are setting the tick dependency on all tasks sharing that
> struct signal.

Hi Frederic,

Yep, fixing in -v2, thanks.

[patch 2/2] nohz: try to avoid IPI when setting tick dependency for task

[Marcelo Tosatti]

When enabling per-CPU posix timers, an IPI to nohz_full CPUs might be
performed (to re-read the dependencies and possibly not re-enter
nohz_full on a given CPU).

A common case is for applications that run on nohz_full= CPUs
to not use POSIX timers (eg DPDK).

This patch optimizes tick_nohz_dep_set_task to avoid kicking
all nohz_full= CPUs in case the task allowed mask does not
intersect with nohz_full= CPU mask,
when going through tick_nohz_dep_set_task.

This reduces interruptions to nohz_full= CPUs.

---
 kernel/time/tick-sched.c |    9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

Index: linux-2.6/kernel/time/tick-sched.c
===================================================================
--- linux-2.6.orig/kernel/time/tick-sched.c
+++ linux-2.6/kernel/time/tick-sched.c
@@ -383,11 +383,16 @@ void tick_nohz_dep_set_task(struct task_
 			tick_nohz_full_kick();
 			preempt_enable();
 		} else {
+			unsigned long flags;
+
 			/*
 			 * Some future tick_nohz_full_kick_task()
-			 * should optimize this.
+			 * should further optimize this.
 			 */
-			tick_nohz_full_kick_all();
+			raw_spin_lock_irqsave(&tsk->pi_lock, flags);
+			if (cpumask_intersects(&tsk->cpus_mask, tick_nohz_full_mask))
+				tick_nohz_full_kick_all();
+			raw_spin_unlock_irqrestore(&tsk->pi_lock, flags);
 		}
 	}
 }

Re: [patch 2/2] nohz: try to avoid IPI when setting tick dependency for task

[Frederic Weisbecker]

On Tue, Aug 25, 2020 at 03:41:49PM -0300, Marcelo Tosatti wrote:
> When enabling per-CPU posix timers, an IPI to nohz_full CPUs might be
> performed (to re-read the dependencies and possibly not re-enter
> nohz_full on a given CPU).
> 
> A common case is for applications that run on nohz_full= CPUs
> to not use POSIX timers (eg DPDK).
> 
> This patch optimizes tick_nohz_dep_set_task to avoid kicking
> all nohz_full= CPUs in case the task allowed mask does not
> intersect with nohz_full= CPU mask,
> when going through tick_nohz_dep_set_task.
> 
> This reduces interruptions to nohz_full= CPUs.
> 
> ---
>  kernel/time/tick-sched.c |    9 +++++++--
>  1 file changed, 7 insertions(+), 2 deletions(-)
> 
> Index: linux-2.6/kernel/time/tick-sched.c
> ===================================================================
> --- linux-2.6.orig/kernel/time/tick-sched.c
> +++ linux-2.6/kernel/time/tick-sched.c
> @@ -383,11 +383,16 @@ void tick_nohz_dep_set_task(struct task_
>  			tick_nohz_full_kick();
>  			preempt_enable();
>  		} else {
> +			unsigned long flags;
> +
>  			/*
>  			 * Some future tick_nohz_full_kick_task()
> -			 * should optimize this.
> +			 * should further optimize this.
>  			 */
> -			tick_nohz_full_kick_all();
> +			raw_spin_lock_irqsave(&tsk->pi_lock, flags);
> +			if (cpumask_intersects(&tsk->cpus_mask, tick_nohz_full_mask))
> +				tick_nohz_full_kick_all();
> +			raw_spin_unlock_irqrestore(&tsk->pi_lock, flags);
>  		}
>  	}
>  }
> 
> 

Not long ago, Peterz suggested that we simply do:

diff --git a/kernel/time/tick-sched.c b/kernel/time/tick-sched.c
index f0199a4ba1ad..42ce8e458013 100644
--- a/kernel/time/tick-sched.c
+++ b/kernel/time/tick-sched.c
@@ -357,17 +357,26 @@ EXPORT_SYMBOL_GPL(tick_nohz_dep_clear_cpu);
 void tick_nohz_dep_set_task(struct task_struct *tsk, enum tick_dep_bits bit)
 {
 	if (!atomic_fetch_or(BIT(bit), &tsk->tick_dep_mask)) {
-		if (tsk == current) {
-			preempt_disable();
-			tick_nohz_full_kick();
-			preempt_enable();
-		} else {
-			/*
-			 * Some future tick_nohz_full_kick_task()
-			 * should optimize this.
-			 */
-			tick_nohz_full_kick_all();
-		}
+		int cpu = task_cpu(tsk);
+
+		/*
+		 * If the task concurrently migrates to another cpu,
+		 * we guarantee it sees the new tick dependency upon
+		 * schedule.
+		 *
+		 * set_task_cpu(p, cpu);
+		 *   STORE p->cpu = @cpu
+		 * __schedule() (switch to task 'p')
+		 *   LOCK rq->lock
+		 *   smp_mb__after_spin_lock()		STORE p->tick_dep_mask
+		 *   tick_nohz_task_switch()		smp_mb() (atomic_fetch_or())
+		 *      LOAD p->tick_dep_mask		LOAD p->cpu
+		 */
+
+		preempt_disable();
+		if (cpu_online(cpu))
+			tick_nohz_full_kick_cpu(cpu);
+		preempt_enable();
 	}
 }
 EXPORT_SYMBOL_GPL(tick_nohz_dep_set_task);

Re: [patch 2/2] nohz: try to avoid IPI when setting tick dependency for task

[Marcelo Tosatti]

On Thu, Sep 03, 2020 at 05:01:53PM +0200, Frederic Weisbecker wrote:
> On Tue, Aug 25, 2020 at 03:41:49PM -0300, Marcelo Tosatti wrote:
> > When enabling per-CPU posix timers, an IPI to nohz_full CPUs might be
> > performed (to re-read the dependencies and possibly not re-enter
> > nohz_full on a given CPU).
> > 
> > A common case is for applications that run on nohz_full= CPUs
> > to not use POSIX timers (eg DPDK).
> > 
> > This patch optimizes tick_nohz_dep_set_task to avoid kicking
> > all nohz_full= CPUs in case the task allowed mask does not
> > intersect with nohz_full= CPU mask,
> > when going through tick_nohz_dep_set_task.
> > 
> > This reduces interruptions to nohz_full= CPUs.
> > 
> > ---
> >  kernel/time/tick-sched.c |    9 +++++++--
> >  1 file changed, 7 insertions(+), 2 deletions(-)
> > 
> > Index: linux-2.6/kernel/time/tick-sched.c
> > ===================================================================
> > --- linux-2.6.orig/kernel/time/tick-sched.c
> > +++ linux-2.6/kernel/time/tick-sched.c
> > @@ -383,11 +383,16 @@ void tick_nohz_dep_set_task(struct task_
> >  			tick_nohz_full_kick();
> >  			preempt_enable();
> >  		} else {
> > +			unsigned long flags;
> > +
> >  			/*
> >  			 * Some future tick_nohz_full_kick_task()
> > -			 * should optimize this.
> > +			 * should further optimize this.
> >  			 */
> > -			tick_nohz_full_kick_all();
> > +			raw_spin_lock_irqsave(&tsk->pi_lock, flags);
> > +			if (cpumask_intersects(&tsk->cpus_mask, tick_nohz_full_mask))
> > +				tick_nohz_full_kick_all();
> > +			raw_spin_unlock_irqrestore(&tsk->pi_lock, flags);
> >  		}
> >  	}
> >  }
> > 
> > 
> 
> Not long ago, Peterz suggested that we simply do:
> 
> diff --git a/kernel/time/tick-sched.c b/kernel/time/tick-sched.c
> index f0199a4ba1ad..42ce8e458013 100644
> --- a/kernel/time/tick-sched.c
> +++ b/kernel/time/tick-sched.c
> @@ -357,17 +357,26 @@ EXPORT_SYMBOL_GPL(tick_nohz_dep_clear_cpu);
>  void tick_nohz_dep_set_task(struct task_struct *tsk, enum tick_dep_bits bit)
>  {
>  	if (!atomic_fetch_or(BIT(bit), &tsk->tick_dep_mask)) {
> -		if (tsk == current) {
> -			preempt_disable();
> -			tick_nohz_full_kick();
> -			preempt_enable();
> -		} else {
> -			/*
> -			 * Some future tick_nohz_full_kick_task()
> -			 * should optimize this.
> -			 */
> -			tick_nohz_full_kick_all();
> -		}
> +		int cpu = task_cpu(tsk);
> +
> +		/*
> +		 * If the task concurrently migrates to another cpu,
> +		 * we guarantee it sees the new tick dependency upon
> +		 * schedule.
> +		 *
> +		 * set_task_cpu(p, cpu);
> +		 *   STORE p->cpu = @cpu
> +		 * __schedule() (switch to task 'p')
> +		 *   LOCK rq->lock
> +		 *   smp_mb__after_spin_lock()		STORE p->tick_dep_mask
> +		 *   tick_nohz_task_switch()		smp_mb() (atomic_fetch_or())
> +		 *      LOAD p->tick_dep_mask		LOAD p->cpu
> +		 */
> +
> +		preempt_disable();
> +		if (cpu_online(cpu))
> +			tick_nohz_full_kick_cpu(cpu);
> +		preempt_enable();
>  	}
>  }
>  EXPORT_SYMBOL_GPL(tick_nohz_dep_set_task);

This can also be used for the signal case... thanks.