[PATCH 0/2] timer: make deferrable cpu unbound timers really not bound to a cpu

[PATCH 0/2] timer: make deferrable cpu unbound timers really not bound to a cpu

[Prasad Sodagudi]

These patches introduce support for global deferrable timers and this feature is very useful
for the mobile chipsets. There was a discussion on global deferrable timer discussion - [1]
and thomas raised cacheline bouncing problem. I am thinking that, that problem is addressed
on later patches problem as much as possible. Global deferrable timers runs on the
tick_do_timer_cpu to avoid cacheline bouncing problem during the execution of timers.  

[1]- https://lore.kernel.org/patchwork/patch/500541/


Joonwoo Park (1):
  timer: make deferrable cpu unbound timers really not bound to a cpu

Prasad Sodagudi (1):
  sched: Add a check for cpu unbound deferrable timers

 include/linux/timer.h    |  3 +++
 kernel/time/tick-sched.c |  6 +++++
 kernel/time/timer.c      | 63 ++++++++++++++++++++++++++++++++++++++++++++----
 3 files changed, 67 insertions(+), 5 deletions(-)

-- 
The Qualcomm Innovation Center, Inc. is a member of the Code Aurora Forum,
a Linux Foundation Collaborative Project

[PATCH 1/2] timer: make deferrable cpu unbound timers really not bound to a cpu

[Prasad Sodagudi]

From: Joonwoo Park <joonwoop@codeaurora.org>

When a deferrable work (INIT_DEFERRABLE_WORK, etc.) is queued via
queue_delayed_work() it's probably intended to run the work item on any
CPU that isn't idle. However, we queue the work to run at a later time
by starting a deferrable timer that binds to whatever CPU the work is
queued on which is same with queue_delayed_work_on(smp_processor_id())
effectively.

As a result WORK_CPU_UNBOUND work items aren't really cpu unbound now.
In fact this is perfectly fine with UP kernel and also won't affect much a
system without dyntick with SMP kernel too as every cpus run timers
periodically.  But on SMP systems with dyntick current implementation leads
deferrable timers not very scalable because the timer's base which has
queued the deferrable timer won't wake up till next non-deferrable timer
expires even though there are possible other non idle cpus are running
which are able to run expired deferrable timers.

The deferrable work is a good example of the current implementation's
victim like below.

INIT_DEFERRABLE_WORK(&dwork, fn);
CPU 0                                 CPU 1
queue_delayed_work(wq, &dwork, HZ);
    queue_delayed_work_on(WORK_CPU_UNBOUND);
        ...
	__mod_timer() -> queues timer to the
			 current cpu's timer
			 base.
	...
tick_nohz_idle_enter() -> cpu enters idle.
A second later
cpu 0 is now in idle.                 cpu 1 exits idle or wasn't in idle so
                                      now it's in active but won't
cpu 0 won't wake up till next         handle cpu unbound deferrable timer
non-deferrable timer expires.         as it's in cpu 0's timer base.

To make all cpu unbound deferrable timers are scalable, introduce a common
timer base which is only for cpu unbound deferrable timers to make those
are indeed cpu unbound so that can be scheduled by any of non idle cpus.
This common timer fixes scalability issue of delayed work and all other cpu
unbound deferrable timer using implementations.

Signed-off-by: Joonwoo Park <joonwoop@codeaurora.org>
Signed-off-by: Prasad Sodagudi <psodagud@codeaurora.org>
---
 kernel/time/timer.c | 36 +++++++++++++++++++++++++++++++-----
 1 file changed, 31 insertions(+), 5 deletions(-)

diff --git a/kernel/time/timer.c b/kernel/time/timer.c
index 4820823..a63ca77 100644
--- a/kernel/time/timer.c
+++ b/kernel/time/timer.c
@@ -220,6 +220,7 @@ static void timer_update_keys(struct work_struct *work);
 static DECLARE_WORK(timer_update_work, timer_update_keys);
 
 #ifdef CONFIG_SMP
+struct timer_base timer_base_deferrable;
 unsigned int sysctl_timer_migration = 1;
 
 DEFINE_STATIC_KEY_FALSE(timers_migration_enabled);
@@ -841,8 +842,14 @@ static inline struct timer_base *get_timer_cpu_base(u32 tflags, u32 cpu)
 	 * If the timer is deferrable and NO_HZ_COMMON is set then we need
 	 * to use the deferrable base.
 	 */
-	if (IS_ENABLED(CONFIG_NO_HZ_COMMON) && (tflags & TIMER_DEFERRABLE))
-		base = per_cpu_ptr(&timer_bases[BASE_DEF], cpu);
+	if (IS_ENABLED(CONFIG_NO_HZ_COMMON) && (tflags & TIMER_DEFERRABLE)) {
+#ifdef CONFIG_SMP
+		base = &timer_base_deferrable;
+#endif
+		if (tflags & TIMER_PINNED)
+			base = per_cpu_ptr(&timer_bases[BASE_DEF], cpu);
+	}
+
 	return base;
 }
 
@@ -854,8 +861,14 @@ static inline struct timer_base *get_timer_this_cpu_base(u32 tflags)
 	 * If the timer is deferrable and NO_HZ_COMMON is set then we need
 	 * to use the deferrable base.
 	 */
-	if (IS_ENABLED(CONFIG_NO_HZ_COMMON) && (tflags & TIMER_DEFERRABLE))
-		base = this_cpu_ptr(&timer_bases[BASE_DEF]);
+	if (IS_ENABLED(CONFIG_NO_HZ_COMMON) && (tflags & TIMER_DEFERRABLE)) {
+#ifdef CONFIG_SMP
+		base = &timer_base_deferrable;
+#endif
+		if (tflags & TIMER_PINNED)
+			base = this_cpu_ptr(&timer_bases[BASE_DEF]);
+	}
+
 	return base;
 }
 
@@ -1784,8 +1797,12 @@ static __latent_entropy void run_timer_softirq(struct softirq_action *h)
 	struct timer_base *base = this_cpu_ptr(&timer_bases[BASE_STD]);
 
 	__run_timers(base);
-	if (IS_ENABLED(CONFIG_NO_HZ_COMMON))
+	if (IS_ENABLED(CONFIG_NO_HZ_COMMON)) {
 		__run_timers(this_cpu_ptr(&timer_bases[BASE_DEF]));
+		if (tick_do_timer_cpu == TICK_DO_TIMER_NONE ||
+				tick_do_timer_cpu == smp_processor_id())
+			__run_timers(&timer_base_deferrable);
+	}
 }
 
 /*
@@ -2022,6 +2039,14 @@ static void __init init_timer_cpu(int cpu)
 	}
 }
 
+static void __init init_timer_deferrable_global(void)
+{
+	timer_base_deferrable.cpu = nr_cpu_ids;
+	raw_spin_lock_init(&timer_base_deferrable.lock);
+	timer_base_deferrable.clk = jiffies;
+	timer_base_init_expiry_lock(&timer_base_deferrable);
+}
+
 static void __init init_timer_cpus(void)
 {
 	int cpu;
@@ -2033,6 +2058,7 @@ static void __init init_timer_cpus(void)
 void __init init_timers(void)
 {
 	init_timer_cpus();
+	init_timer_deferrable_global();
 	open_softirq(TIMER_SOFTIRQ, run_timer_softirq);
 }
 
-- 
The Qualcomm Innovation Center, Inc. is a member of the Code Aurora Forum,
a Linux Foundation Collaborative Project

[PATCH 2/2] sched: Add a check for cpu unbound deferrable timers

[Prasad Sodagudi]

Add a check for cpu unbound deferrable timer expiry and raise
softirq for handling the expired timers so that the CPU can
process the cpu unbound deferrable times as early as possible
when a cpu tries to enter/exit idle loop.

Signed-off-by: Prasad Sodagudi <psodagud@codeaurora.org>
---
 include/linux/timer.h    |  3 +++
 kernel/time/tick-sched.c |  6 ++++++
 kernel/time/timer.c      | 29 ++++++++++++++++++++++++++++-
 3 files changed, 37 insertions(+), 1 deletion(-)

diff --git a/include/linux/timer.h b/include/linux/timer.h
index 1e6650e..25a1837 100644
--- a/include/linux/timer.h
+++ b/include/linux/timer.h
@@ -172,6 +172,9 @@ extern int del_timer(struct timer_list * timer);
 extern int mod_timer(struct timer_list *timer, unsigned long expires);
 extern int mod_timer_pending(struct timer_list *timer, unsigned long expires);
 extern int timer_reduce(struct timer_list *timer, unsigned long expires);
+#ifdef CONFIG_SMP
+extern bool check_pending_deferrable_timers(int cpu);
+#endif
 
 /*
  * The jiffies value which is added to now, when there is no timer
diff --git a/kernel/time/tick-sched.c b/kernel/time/tick-sched.c
index a792d21..fe0836a 100644
--- a/kernel/time/tick-sched.c
+++ b/kernel/time/tick-sched.c
@@ -23,6 +23,7 @@
 #include <linux/module.h>
 #include <linux/irq_work.h>
 #include <linux/posix-timers.h>
+#include <linux/timer.h>
 #include <linux/context_tracking.h>
 #include <linux/mm.h>
 
@@ -948,6 +949,11 @@ static void __tick_nohz_idle_stop_tick(struct tick_sched *ts)
 	ktime_t expires;
 	int cpu = smp_processor_id();
 
+#ifdef CONFIG_SMP
+	if (check_pending_deferrable_timers(cpu))
+		raise_softirq_irqoff(TIMER_SOFTIRQ);
+#endif
+
 	/*
 	 * If tick_nohz_get_sleep_length() ran tick_nohz_next_event(), the
 	 * tick timer expiration time is known already.
diff --git a/kernel/time/timer.c b/kernel/time/timer.c
index a63ca77..3b0aac3 100644
--- a/kernel/time/timer.c
+++ b/kernel/time/timer.c
@@ -221,6 +221,7 @@ static DECLARE_WORK(timer_update_work, timer_update_keys);
 
 #ifdef CONFIG_SMP
 struct timer_base timer_base_deferrable;
+static atomic_t deferrable_pending;
 unsigned int sysctl_timer_migration = 1;
 
 DEFINE_STATIC_KEY_FALSE(timers_migration_enabled);
@@ -1609,6 +1610,31 @@ static u64 cmp_next_hrtimer_event(u64 basem, u64 expires)
 	return DIV_ROUND_UP_ULL(nextevt, TICK_NSEC) * TICK_NSEC;
 }
 
+
+#ifdef CONFIG_SMP
+/*
+ * check_pending_deferrable_timers - Check for unbound deferrable timer expiry
+ * @cpu - Current CPU
+ *
+ * The function checks whether any global deferrable pending timers
+ * are exipired or not. This function does not check cpu bounded
+ * diferrable pending timers expiry.
+ *
+ * The function returns true when a cpu unbounded deferrable timer is expired.
+ */
+bool check_pending_deferrable_timers(int cpu)
+{
+	if (cpu == tick_do_timer_cpu ||
+		tick_do_timer_cpu == TICK_DO_TIMER_NONE) {
+		if (time_after_eq(jiffies, timer_base_deferrable.clk)
+			&& !atomic_cmpxchg(&deferrable_pending, 0, 1)) {
+			return true;
+		}
+	}
+	return false;
+}
+#endif
+
 /**
  * get_next_timer_interrupt - return the time (clock mono) of the next timer
  * @basej:	base time jiffies
@@ -1799,7 +1825,8 @@ static __latent_entropy void run_timer_softirq(struct softirq_action *h)
 	__run_timers(base);
 	if (IS_ENABLED(CONFIG_NO_HZ_COMMON)) {
 		__run_timers(this_cpu_ptr(&timer_bases[BASE_DEF]));
-		if (tick_do_timer_cpu == TICK_DO_TIMER_NONE ||
+		if ((atomic_cmpxchg(&deferrable_pending, 1, 0) &&
+				tick_do_timer_cpu == TICK_DO_TIMER_NONE) ||
 				tick_do_timer_cpu == smp_processor_id())
 			__run_timers(&timer_base_deferrable);
 	}
-- 
The Qualcomm Innovation Center, Inc. is a member of the Code Aurora Forum,
a Linux Foundation Collaborative Project

Re: [PATCH 2/2] sched: Add a check for cpu unbound deferrable timers

[Thomas Gleixner]

Prasad Sodagudi <psodagud@codeaurora.org> writes:
> @@ -948,6 +949,11 @@ static void __tick_nohz_idle_stop_tick(struct tick_sched *ts)
>  	ktime_t expires;
>  	int cpu = smp_processor_id();
>  
> +#ifdef CONFIG_SMP
> +	if (check_pending_deferrable_timers(cpu))
> +		raise_softirq_irqoff(TIMER_SOFTIRQ);
> +#endif

So if that raises the soft interrupt then the warning in 

can_stop_idle_tick()

	if (unlikely(local_softirq_pending())) {
		....
                pr_warn()

will trigger. Try again.

Thanks,

        tglx

Re: [PATCH 2/2] sched: Add a check for cpu unbound deferrable timers

[Pavan Kondeti]

Hi Prasad,

On Mon, Mar 16, 2020 at 8:42 AM Prasad Sodagudi <psodagud@codeaurora.org> wrote:
>
> Add a check for cpu unbound deferrable timer expiry and raise
> softirq for handling the expired timers so that the CPU can
> process the cpu unbound deferrable times as early as possible
> when a cpu tries to enter/exit idle loop.
>
> Signed-off-by: Prasad Sodagudi <psodagud@codeaurora.org>
> ---
>  include/linux/timer.h    |  3 +++
>  kernel/time/tick-sched.c |  6 ++++++
>  kernel/time/timer.c      | 29 ++++++++++++++++++++++++++++-
>  3 files changed, 37 insertions(+), 1 deletion(-)
>
> diff --git a/include/linux/timer.h b/include/linux/timer.h
> index 1e6650e..25a1837 100644
> --- a/include/linux/timer.h
> +++ b/include/linux/timer.h
> @@ -172,6 +172,9 @@ extern int del_timer(struct timer_list * timer);
>  extern int mod_timer(struct timer_list *timer, unsigned long expires);
>  extern int mod_timer_pending(struct timer_list *timer, unsigned long expires);
>  extern int timer_reduce(struct timer_list *timer, unsigned long expires);
> +#ifdef CONFIG_SMP
> +extern bool check_pending_deferrable_timers(int cpu);
> +#endif
>
>  /*
>   * The jiffies value which is added to now, when there is no timer
> diff --git a/kernel/time/tick-sched.c b/kernel/time/tick-sched.c
> index a792d21..fe0836a 100644
> --- a/kernel/time/tick-sched.c
> +++ b/kernel/time/tick-sched.c
> @@ -23,6 +23,7 @@
>  #include <linux/module.h>
>  #include <linux/irq_work.h>
>  #include <linux/posix-timers.h>
> +#include <linux/timer.h>
>  #include <linux/context_tracking.h>
>  #include <linux/mm.h>
>
> @@ -948,6 +949,11 @@ static void __tick_nohz_idle_stop_tick(struct tick_sched *ts)
>         ktime_t expires;
>         int cpu = smp_processor_id();
>
> +#ifdef CONFIG_SMP
> +       if (check_pending_deferrable_timers(cpu))
> +               raise_softirq_irqoff(TIMER_SOFTIRQ);
> +#endif
> +
>         /*
>          * If tick_nohz_get_sleep_length() ran tick_nohz_next_event(), the
>          * tick timer expiration time is known already.

Since this is called outside interrupt context, ksoftirqd task is always woken
up to serve the TIMER softirq. Given that, this CPU exited idle due to an
interrupt, we could probably check this condition and raise the softirq prior
to irq_exit. To make the discussion easier, providing the diff. This will make
sure that TIMER softirq is served after serving the IRQ and avoids waking
ksoftirqd. This should also cover the case, Thoms mentioned in the other
email.

diff --git a/kernel/time/tick-sched.c b/kernel/time/tick-sched.c
index a792d21..d9d1915 100644
--- a/kernel/time/tick-sched.c
+++ b/kernel/time/tick-sched.c
@@ -1268,8 +1268,19 @@ static inline void tick_nohz_irq_enter(void)
        now = ktime_get();
        if (ts->idle_active)
                tick_nohz_stop_idle(ts, now);
-       if (ts->tick_stopped)
+       if (ts->tick_stopped) {
                tick_nohz_update_jiffies(now);
+               /*
+                * TODO: check the possibility of doing this only
+                * when jiffies is updated.
+                *
+                * __raise_softirq_irqoff() marks the softirq pending.
+                * The softirq is served after this IRQ is served from
+                * irq_exit()
+                */
+               if (check_pending_deferrable_timers())
+                       __raise_softirq_irqoff(TIMER_SOFTIRQ);
+       }
 }

 #else


-- 
Qualcomm India Private Limited, on behalf of Qualcomm Innovation Center, Inc.
Qualcomm Innovation Center, Inc. is a member of Code Aurora Forum, a
Linux Foundation Collaborative Project

Re: [PATCH 1/2] timer: make deferrable cpu unbound timers really not bound to a cpu

[kbuild test robot]

[-- Attachment #1: Type: text/plain, Size: 2623 bytes --]

Hi Prasad,

Thank you for the patch! Yet something to improve:

[auto build test ERROR on tip/timers/core]
[also build test ERROR on tip/auto-latest tip/timers/nohz v5.6-rc6 next-20200316]
[if your patch is applied to the wrong git tree, please drop us a note to help
improve the system. BTW, we also suggest to use '--base' option to specify the
base tree in git format-patch, please see https://stackoverflow.com/a/37406982]

url:    https://github.com/0day-ci/linux/commits/Prasad-Sodagudi/timer-make-deferrable-cpu-unbound-timers-really-not-bound-to-a-cpu/20200317-044103
base:   https://git.kernel.org/pub/scm/linux/kernel/git/tip/tip.git d441dceb5dce71150f28add80d36d91bbfccba99
config: nds32-defconfig (attached as .config)
compiler: nds32le-linux-gcc (GCC) 9.2.0
reproduce:
        wget https://raw.githubusercontent.com/intel/lkp-tests/master/sbin/make.cross -O ~/bin/make.cross
        chmod +x ~/bin/make.cross
        # save the attached .config to linux build tree
        GCC_VERSION=9.2.0 make.cross ARCH=nds32 

If you fix the issue, kindly add following tag
Reported-by: kbuild test robot <lkp@intel.com>

All errors (new ones prefixed by >>):

   kernel/time/timer.c: In function 'run_timer_softirq':
>> kernel/time/timer.c:1804:18: error: 'timer_base_deferrable' undeclared (first use in this function)
    1804 |    __run_timers(&timer_base_deferrable);
         |                  ^~~~~~~~~~~~~~~~~~~~~
   kernel/time/timer.c:1804:18: note: each undeclared identifier is reported only once for each function it appears in
   kernel/time/timer.c: In function 'init_timer_deferrable_global':
   kernel/time/timer.c:2046:2: error: 'timer_base_deferrable' undeclared (first use in this function)
    2046 |  timer_base_deferrable.cpu = nr_cpu_ids;
         |  ^~~~~~~~~~~~~~~~~~~~~

vim +/timer_base_deferrable +1804 kernel/time/timer.c

  1791	
  1792	/*
  1793	 * This function runs timers and the timer-tq in bottom half context.
  1794	 */
  1795	static __latent_entropy void run_timer_softirq(struct softirq_action *h)
  1796	{
  1797		struct timer_base *base = this_cpu_ptr(&timer_bases[BASE_STD]);
  1798	
  1799		__run_timers(base);
  1800		if (IS_ENABLED(CONFIG_NO_HZ_COMMON)) {
  1801			__run_timers(this_cpu_ptr(&timer_bases[BASE_DEF]));
  1802			if (tick_do_timer_cpu == TICK_DO_TIMER_NONE ||
  1803					tick_do_timer_cpu == smp_processor_id())
> 1804				__run_timers(&timer_base_deferrable);
  1805		}
  1806	}
  1807	

---
0-DAY CI Kernel Test Service, Intel Corporation
https://lists.01.org/hyperkitty/list/kbuild-all(a)lists.01.org

[-- Attachment #2: config.gz --]
[-- Type: application/gzip, Size: 10815 bytes --]

Re: [PATCH 2/2] sched: Add a check for cpu unbound deferrable timers

[kbuild test robot]

[-- Attachment #1: Type: text/plain, Size: 7968 bytes --]

Hi Prasad,

Thank you for the patch! Yet something to improve:

[auto build test ERROR on tip/timers/core]
[also build test ERROR on tip/auto-latest tip/timers/nohz v5.6-rc6 next-20200312]
[if your patch is applied to the wrong git tree, please drop us a note to help
improve the system. BTW, we also suggest to use '--base' option to specify the
base tree in git format-patch, please see https://stackoverflow.com/a/37406982]

url:    https://github.com/0day-ci/linux/commits/Prasad-Sodagudi/timer-make-deferrable-cpu-unbound-timers-really-not-bound-to-a-cpu/20200317-044103
base:   https://git.kernel.org/pub/scm/linux/kernel/git/tip/tip.git d441dceb5dce71150f28add80d36d91bbfccba99
config: nds32-defconfig (attached as .config)
compiler: nds32le-linux-gcc (GCC) 9.2.0
reproduce:
        wget https://raw.githubusercontent.com/intel/lkp-tests/master/sbin/make.cross -O ~/bin/make.cross
        chmod +x ~/bin/make.cross
        # save the attached .config to linux build tree
        GCC_VERSION=9.2.0 make.cross ARCH=nds32 

If you fix the issue, kindly add following tag
Reported-by: kbuild test robot <lkp@intel.com>

All error/warnings (new ones prefixed by >>):

   In file included from ./arch/nds32/include/generated/asm/cmpxchg.h:1,
                    from include/asm-generic/atomic.h:12,
                    from ./arch/nds32/include/generated/asm/atomic.h:1,
                    from include/linux/atomic.h:7,
                    from include/asm-generic/bitops/lock.h:5,
                    from include/asm-generic/bitops.h:31,
                    from ./arch/nds32/include/generated/asm/bitops.h:1,
                    from include/linux/bitops.h:29,
                    from include/linux/kernel.h:12,
                    from include/linux/list.h:9,
                    from include/linux/smp.h:12,
                    from include/linux/kernel_stat.h:5,
                    from kernel/time/timer.c:21:
   kernel/time/timer.c: In function 'run_timer_softirq':
>> kernel/time/timer.c:1828:24: error: 'deferrable_pending' undeclared (first use in this function)
    1828 |   if ((atomic_cmpxchg(&deferrable_pending, 1, 0) &&
         |                        ^~~~~~~~~~~~~~~~~~
   include/asm-generic/cmpxchg.h:97:17: note: in definition of macro 'cmpxchg_local'
      97 |  ((__typeof__(*(ptr)))__cmpxchg_local_generic((ptr), (unsigned long)(o),\
         |                 ^~~
>> include/asm-generic/atomic.h:196:38: note: in expansion of macro 'cmpxchg'
     196 | #define atomic_cmpxchg(v, old, new) (cmpxchg(&((v)->counter), (old), (new)))
         |                                      ^~~~~~~
>> kernel/time/timer.c:1828:8: note: in expansion of macro 'atomic_cmpxchg'
    1828 |   if ((atomic_cmpxchg(&deferrable_pending, 1, 0) &&
         |        ^~~~~~~~~~~~~~
   kernel/time/timer.c:1828:24: note: each undeclared identifier is reported only once for each function it appears in
    1828 |   if ((atomic_cmpxchg(&deferrable_pending, 1, 0) &&
         |                        ^~~~~~~~~~~~~~~~~~
   include/asm-generic/cmpxchg.h:97:17: note: in definition of macro 'cmpxchg_local'
      97 |  ((__typeof__(*(ptr)))__cmpxchg_local_generic((ptr), (unsigned long)(o),\
         |                 ^~~
>> include/asm-generic/atomic.h:196:38: note: in expansion of macro 'cmpxchg'
     196 | #define atomic_cmpxchg(v, old, new) (cmpxchg(&((v)->counter), (old), (new)))
         |                                      ^~~~~~~
>> kernel/time/timer.c:1828:8: note: in expansion of macro 'atomic_cmpxchg'
    1828 |   if ((atomic_cmpxchg(&deferrable_pending, 1, 0) &&
         |        ^~~~~~~~~~~~~~
   kernel/time/timer.c:1831:18: error: 'timer_base_deferrable' undeclared (first use in this function)
    1831 |    __run_timers(&timer_base_deferrable);
         |                  ^~~~~~~~~~~~~~~~~~~~~
   kernel/time/timer.c: In function 'init_timer_deferrable_global':
   kernel/time/timer.c:2073:2: error: 'timer_base_deferrable' undeclared (first use in this function)
    2073 |  timer_base_deferrable.cpu = nr_cpu_ids;
         |  ^~~~~~~~~~~~~~~~~~~~~
--
   In file included from ./arch/nds32/include/generated/asm/cmpxchg.h:1,
                    from include/asm-generic/atomic.h:12,
                    from ./arch/nds32/include/generated/asm/atomic.h:1,
                    from include/linux/atomic.h:7,
                    from include/asm-generic/bitops/lock.h:5,
                    from include/asm-generic/bitops.h:31,
                    from ./arch/nds32/include/generated/asm/bitops.h:1,
                    from include/linux/bitops.h:29,
                    from include/linux/kernel.h:12,
                    from include/linux/list.h:9,
                    from include/linux/smp.h:12,
                    from include/linux/kernel_stat.h:5,
                    from kernel//time/timer.c:21:
   kernel//time/timer.c: In function 'run_timer_softirq':
   kernel//time/timer.c:1828:24: error: 'deferrable_pending' undeclared (first use in this function)
    1828 |   if ((atomic_cmpxchg(&deferrable_pending, 1, 0) &&
         |                        ^~~~~~~~~~~~~~~~~~
   include/asm-generic/cmpxchg.h:97:17: note: in definition of macro 'cmpxchg_local'
      97 |  ((__typeof__(*(ptr)))__cmpxchg_local_generic((ptr), (unsigned long)(o),\
         |                 ^~~
>> include/asm-generic/atomic.h:196:38: note: in expansion of macro 'cmpxchg'
     196 | #define atomic_cmpxchg(v, old, new) (cmpxchg(&((v)->counter), (old), (new)))
         |                                      ^~~~~~~
   kernel//time/timer.c:1828:8: note: in expansion of macro 'atomic_cmpxchg'
    1828 |   if ((atomic_cmpxchg(&deferrable_pending, 1, 0) &&
         |        ^~~~~~~~~~~~~~
   kernel//time/timer.c:1828:24: note: each undeclared identifier is reported only once for each function it appears in
    1828 |   if ((atomic_cmpxchg(&deferrable_pending, 1, 0) &&
         |                        ^~~~~~~~~~~~~~~~~~
   include/asm-generic/cmpxchg.h:97:17: note: in definition of macro 'cmpxchg_local'
      97 |  ((__typeof__(*(ptr)))__cmpxchg_local_generic((ptr), (unsigned long)(o),\
         |                 ^~~
>> include/asm-generic/atomic.h:196:38: note: in expansion of macro 'cmpxchg'
     196 | #define atomic_cmpxchg(v, old, new) (cmpxchg(&((v)->counter), (old), (new)))
         |                                      ^~~~~~~
   kernel//time/timer.c:1828:8: note: in expansion of macro 'atomic_cmpxchg'
    1828 |   if ((atomic_cmpxchg(&deferrable_pending, 1, 0) &&
         |        ^~~~~~~~~~~~~~
   kernel//time/timer.c:1831:18: error: 'timer_base_deferrable' undeclared (first use in this function)
    1831 |    __run_timers(&timer_base_deferrable);
         |                  ^~~~~~~~~~~~~~~~~~~~~
   kernel//time/timer.c: In function 'init_timer_deferrable_global':
   kernel//time/timer.c:2073:2: error: 'timer_base_deferrable' undeclared (first use in this function)
    2073 |  timer_base_deferrable.cpu = nr_cpu_ids;
         |  ^~~~~~~~~~~~~~~~~~~~~

vim +/deferrable_pending +1828 kernel/time/timer.c

  1817	
  1818	/*
  1819	 * This function runs timers and the timer-tq in bottom half context.
  1820	 */
  1821	static __latent_entropy void run_timer_softirq(struct softirq_action *h)
  1822	{
  1823		struct timer_base *base = this_cpu_ptr(&timer_bases[BASE_STD]);
  1824	
  1825		__run_timers(base);
  1826		if (IS_ENABLED(CONFIG_NO_HZ_COMMON)) {
  1827			__run_timers(this_cpu_ptr(&timer_bases[BASE_DEF]));
> 1828			if ((atomic_cmpxchg(&deferrable_pending, 1, 0) &&
  1829					tick_do_timer_cpu == TICK_DO_TIMER_NONE) ||
  1830					tick_do_timer_cpu == smp_processor_id())
  1831				__run_timers(&timer_base_deferrable);
  1832		}
  1833	}
  1834	

---
0-DAY CI Kernel Test Service, Intel Corporation
https://lists.01.org/hyperkitty/list/kbuild-all(a)lists.01.org

[-- Attachment #2: config.gz --]
[-- Type: application/gzip, Size: 10815 bytes --]

Re: [PATCH 1/2] timer: make deferrable cpu unbound timers really not bound to a cpu

[kbuild test robot]

[-- Attachment #1: Type: text/plain, Size: 3091 bytes --]

Hi Prasad,

Thank you for the patch! Yet something to improve:

[auto build test ERROR on tip/timers/core]
[also build test ERROR on tip/auto-latest tip/timers/nohz v5.6-rc6 next-20200316]
[if your patch is applied to the wrong git tree, please drop us a note to help
improve the system. BTW, we also suggest to use '--base' option to specify the
base tree in git format-patch, please see https://stackoverflow.com/a/37406982]

url:    https://github.com/0day-ci/linux/commits/Prasad-Sodagudi/timer-make-deferrable-cpu-unbound-timers-really-not-bound-to-a-cpu/20200317-044103
base:   https://git.kernel.org/pub/scm/linux/kernel/git/tip/tip.git d441dceb5dce71150f28add80d36d91bbfccba99
config: m68k-m5475evb_defconfig (attached as .config)
compiler: m68k-linux-gcc (GCC) 9.2.0
reproduce:
        wget https://raw.githubusercontent.com/intel/lkp-tests/master/sbin/make.cross -O ~/bin/make.cross
        chmod +x ~/bin/make.cross
        # save the attached .config to linux build tree
        GCC_VERSION=9.2.0 make.cross ARCH=m68k 

If you fix the issue, kindly add following tag
Reported-by: kbuild test robot <lkp@intel.com>

All errors (new ones prefixed by >>):

   kernel/time/timer.c: In function 'run_timer_softirq':
>> kernel/time/timer.c:1802:7: error: 'tick_do_timer_cpu' undeclared (first use in this function); did you mean 'tick_dep_clear_cpu'?
    1802 |   if (tick_do_timer_cpu == TICK_DO_TIMER_NONE ||
         |       ^~~~~~~~~~~~~~~~~
         |       tick_dep_clear_cpu
   kernel/time/timer.c:1802:7: note: each undeclared identifier is reported only once for each function it appears in
>> kernel/time/timer.c:1802:28: error: 'TICK_DO_TIMER_NONE' undeclared (first use in this function)
    1802 |   if (tick_do_timer_cpu == TICK_DO_TIMER_NONE ||
         |                            ^~~~~~~~~~~~~~~~~~
   kernel/time/timer.c:1804:18: error: 'timer_base_deferrable' undeclared (first use in this function)
    1804 |    __run_timers(&timer_base_deferrable);
         |                  ^~~~~~~~~~~~~~~~~~~~~
   kernel/time/timer.c: In function 'init_timer_deferrable_global':
   kernel/time/timer.c:2046:2: error: 'timer_base_deferrable' undeclared (first use in this function)
    2046 |  timer_base_deferrable.cpu = nr_cpu_ids;
         |  ^~~~~~~~~~~~~~~~~~~~~

vim +1802 kernel/time/timer.c

  1791	
  1792	/*
  1793	 * This function runs timers and the timer-tq in bottom half context.
  1794	 */
  1795	static __latent_entropy void run_timer_softirq(struct softirq_action *h)
  1796	{
  1797		struct timer_base *base = this_cpu_ptr(&timer_bases[BASE_STD]);
  1798	
  1799		__run_timers(base);
  1800		if (IS_ENABLED(CONFIG_NO_HZ_COMMON)) {
  1801			__run_timers(this_cpu_ptr(&timer_bases[BASE_DEF]));
> 1802			if (tick_do_timer_cpu == TICK_DO_TIMER_NONE ||
  1803					tick_do_timer_cpu == smp_processor_id())
  1804				__run_timers(&timer_base_deferrable);
  1805		}
  1806	}
  1807	

---
0-DAY CI Kernel Test Service, Intel Corporation
https://lists.01.org/hyperkitty/list/kbuild-all(a)lists.01.org

[-- Attachment #2: config.gz --]
[-- Type: application/gzip, Size: 7289 bytes --]

Re: [PATCH 2/2] sched: Add a check for cpu unbound deferrable timers

[kbuild test robot]

[-- Attachment #1: Type: text/plain, Size: 10327 bytes --]

Hi Prasad,

Thank you for the patch! Perhaps something to improve:

[auto build test WARNING on tip/timers/core]
[also build test WARNING on tip/auto-latest tip/timers/nohz v5.6-rc6 next-20200316]
[if your patch is applied to the wrong git tree, please drop us a note to help
improve the system. BTW, we also suggest to use '--base' option to specify the
base tree in git format-patch, please see https://stackoverflow.com/a/37406982]

url:    https://github.com/0day-ci/linux/commits/Prasad-Sodagudi/timer-make-deferrable-cpu-unbound-timers-really-not-bound-to-a-cpu/20200317-044103
base:   https://git.kernel.org/pub/scm/linux/kernel/git/tip/tip.git d441dceb5dce71150f28add80d36d91bbfccba99
config: ia64-allmodconfig (attached as .config)
compiler: ia64-linux-gcc (GCC) 9.2.0
reproduce:
        wget https://raw.githubusercontent.com/intel/lkp-tests/master/sbin/make.cross -O ~/bin/make.cross
        chmod +x ~/bin/make.cross
        # save the attached .config to linux build tree
        GCC_VERSION=9.2.0 make.cross ARCH=ia64 

If you fix the issue, kindly add following tag
Reported-by: kbuild test robot <lkp@intel.com>

All warnings (new ones prefixed by >>):

   kernel/time/timer.c: In function 'get_timer_cpu_base':
   kernel/time/timer.c:848:11: error: 'timer_base_deferrable' undeclared (first use in this function)
     848 |   base = &timer_base_deferrable;
         |           ^~~~~~~~~~~~~~~~~~~~~
   kernel/time/timer.c:848:11: note: each undeclared identifier is reported only once for each function it appears in
   kernel/time/timer.c: In function 'get_timer_this_cpu_base':
   kernel/time/timer.c:867:11: error: 'timer_base_deferrable' undeclared (first use in this function)
     867 |   base = &timer_base_deferrable;
         |           ^~~~~~~~~~~~~~~~~~~~~
   In file included from arch/ia64/include/uapi/asm/intrinsics.h:22,
                    from arch/ia64/include/asm/intrinsics.h:11,
                    from arch/ia64/include/asm/bitops.h:19,
                    from include/linux/bitops.h:29,
                    from include/linux/kernel.h:12,
                    from include/linux/list.h:9,
                    from include/linux/smp.h:12,
                    from include/linux/kernel_stat.h:5,
                    from kernel/time/timer.c:21:
   kernel/time/timer.c: In function 'run_timer_softirq':
   kernel/time/timer.c:1828:24: error: 'deferrable_pending' undeclared (first use in this function)
    1828 |   if ((atomic_cmpxchg(&deferrable_pending, 1, 0) &&
         |                        ^~~~~~~~~~~~~~~~~~
   arch/ia64/include/uapi/asm/cmpxchg.h:75:10: note: in definition of macro 'ia64_cmpxchg'
      75 |  switch (size) {       \
         |          ^~~~
>> arch/ia64/include/uapi/asm/cmpxchg.h:130:28: note: in expansion of macro 'cmpxchg_acq'
     130 | #define cmpxchg(ptr, o, n) cmpxchg_acq((ptr), (o), (n))
         |                            ^~~~~~~~~~~
>> arch/ia64/include/asm/atomic.h:211:38: note: in expansion of macro 'cmpxchg'
     211 | #define atomic_cmpxchg(v, old, new) (cmpxchg(&((v)->counter), old, new))
         |                                      ^~~~~~~
   kernel/time/timer.c:1828:8: note: in expansion of macro 'atomic_cmpxchg'
    1828 |   if ((atomic_cmpxchg(&deferrable_pending, 1, 0) &&
         |        ^~~~~~~~~~~~~~
   kernel/time/timer.c:1829:5: error: 'tick_do_timer_cpu' undeclared (first use in this function); did you mean 'tick_dep_clear_cpu'?
    1829 |     tick_do_timer_cpu == TICK_DO_TIMER_NONE) ||
         |     ^~~~~~~~~~~~~~~~~
         |     tick_dep_clear_cpu
   kernel/time/timer.c:1829:26: error: 'TICK_DO_TIMER_NONE' undeclared (first use in this function)
    1829 |     tick_do_timer_cpu == TICK_DO_TIMER_NONE) ||
         |                          ^~~~~~~~~~~~~~~~~~
   kernel/time/timer.c:1831:18: error: 'timer_base_deferrable' undeclared (first use in this function)
    1831 |    __run_timers(&timer_base_deferrable);
         |                  ^~~~~~~~~~~~~~~~~~~~~
   kernel/time/timer.c: In function 'init_timer_deferrable_global':
   kernel/time/timer.c:2073:2: error: 'timer_base_deferrable' undeclared (first use in this function)
    2073 |  timer_base_deferrable.cpu = nr_cpu_ids;
         |  ^~~~~~~~~~~~~~~~~~~~~
--
   kernel//time/timer.c: In function 'get_timer_cpu_base':
   kernel//time/timer.c:848:11: error: 'timer_base_deferrable' undeclared (first use in this function)
     848 |   base = &timer_base_deferrable;
         |           ^~~~~~~~~~~~~~~~~~~~~
   kernel//time/timer.c:848:11: note: each undeclared identifier is reported only once for each function it appears in
   kernel//time/timer.c: In function 'get_timer_this_cpu_base':
   kernel//time/timer.c:867:11: error: 'timer_base_deferrable' undeclared (first use in this function)
     867 |   base = &timer_base_deferrable;
         |           ^~~~~~~~~~~~~~~~~~~~~
   In file included from arch/ia64/include/uapi/asm/intrinsics.h:22,
                    from arch/ia64/include/asm/intrinsics.h:11,
                    from arch/ia64/include/asm/bitops.h:19,
                    from include/linux/bitops.h:29,
                    from include/linux/kernel.h:12,
                    from include/linux/list.h:9,
                    from include/linux/smp.h:12,
                    from include/linux/kernel_stat.h:5,
                    from kernel//time/timer.c:21:
   kernel//time/timer.c: In function 'run_timer_softirq':
   kernel//time/timer.c:1828:24: error: 'deferrable_pending' undeclared (first use in this function)
    1828 |   if ((atomic_cmpxchg(&deferrable_pending, 1, 0) &&
         |                        ^~~~~~~~~~~~~~~~~~
   arch/ia64/include/uapi/asm/cmpxchg.h:75:10: note: in definition of macro 'ia64_cmpxchg'
      75 |  switch (size) {       \
         |          ^~~~
>> arch/ia64/include/uapi/asm/cmpxchg.h:130:28: note: in expansion of macro 'cmpxchg_acq'
     130 | #define cmpxchg(ptr, o, n) cmpxchg_acq((ptr), (o), (n))
         |                            ^~~~~~~~~~~
>> arch/ia64/include/asm/atomic.h:211:38: note: in expansion of macro 'cmpxchg'
     211 | #define atomic_cmpxchg(v, old, new) (cmpxchg(&((v)->counter), old, new))
         |                                      ^~~~~~~
   kernel//time/timer.c:1828:8: note: in expansion of macro 'atomic_cmpxchg'
    1828 |   if ((atomic_cmpxchg(&deferrable_pending, 1, 0) &&
         |        ^~~~~~~~~~~~~~
   kernel//time/timer.c:1829:5: error: 'tick_do_timer_cpu' undeclared (first use in this function); did you mean 'tick_dep_clear_cpu'?
    1829 |     tick_do_timer_cpu == TICK_DO_TIMER_NONE) ||
         |     ^~~~~~~~~~~~~~~~~
         |     tick_dep_clear_cpu
   kernel//time/timer.c:1829:26: error: 'TICK_DO_TIMER_NONE' undeclared (first use in this function)
    1829 |     tick_do_timer_cpu == TICK_DO_TIMER_NONE) ||
         |                          ^~~~~~~~~~~~~~~~~~
   kernel//time/timer.c:1831:18: error: 'timer_base_deferrable' undeclared (first use in this function)
    1831 |    __run_timers(&timer_base_deferrable);
         |                  ^~~~~~~~~~~~~~~~~~~~~
   kernel//time/timer.c: In function 'init_timer_deferrable_global':
   kernel//time/timer.c:2073:2: error: 'timer_base_deferrable' undeclared (first use in this function)
    2073 |  timer_base_deferrable.cpu = nr_cpu_ids;
         |  ^~~~~~~~~~~~~~~~~~~~~

vim +/cmpxchg_acq +130 arch/ia64/include/uapi/asm/cmpxchg.h

85f8f7759e418c arch/ia64/include/asm/cmpxchg.h      Paul Gortmaker 2012-04-03  114  
85f8f7759e418c arch/ia64/include/asm/cmpxchg.h      Paul Gortmaker 2012-04-03  115  #define cmpxchg_acq(ptr, o, n)	\
85f8f7759e418c arch/ia64/include/asm/cmpxchg.h      Paul Gortmaker 2012-04-03  116  	ia64_cmpxchg(acq, (ptr), (o), (n), sizeof(*(ptr)))
85f8f7759e418c arch/ia64/include/asm/cmpxchg.h      Paul Gortmaker 2012-04-03  117  #define cmpxchg_rel(ptr, o, n)	\
85f8f7759e418c arch/ia64/include/asm/cmpxchg.h      Paul Gortmaker 2012-04-03  118  	ia64_cmpxchg(rel, (ptr), (o), (n), sizeof(*(ptr)))
85f8f7759e418c arch/ia64/include/asm/cmpxchg.h      Paul Gortmaker 2012-04-03  119  
e4f9bfb3feaeac arch/ia64/include/uapi/asm/cmpxchg.h Peter Zijlstra 2014-02-04  120  /*
e4f9bfb3feaeac arch/ia64/include/uapi/asm/cmpxchg.h Peter Zijlstra 2014-02-04  121   * Worse still - early processor implementations actually just ignored
e4f9bfb3feaeac arch/ia64/include/uapi/asm/cmpxchg.h Peter Zijlstra 2014-02-04  122   * the acquire/release and did a full fence all the time.  Unfortunately
e4f9bfb3feaeac arch/ia64/include/uapi/asm/cmpxchg.h Peter Zijlstra 2014-02-04  123   * this meant a lot of badly written code that used .acq when they really
e4f9bfb3feaeac arch/ia64/include/uapi/asm/cmpxchg.h Peter Zijlstra 2014-02-04  124   * wanted .rel became legacy out in the wild - so when we made a cpu
e4f9bfb3feaeac arch/ia64/include/uapi/asm/cmpxchg.h Peter Zijlstra 2014-02-04  125   * that strictly did the .acq or .rel ... all that code started breaking - so
e4f9bfb3feaeac arch/ia64/include/uapi/asm/cmpxchg.h Peter Zijlstra 2014-02-04  126   * we had to back-pedal and keep the "legacy" behavior of a full fence :-(
e4f9bfb3feaeac arch/ia64/include/uapi/asm/cmpxchg.h Peter Zijlstra 2014-02-04  127   */
e4f9bfb3feaeac arch/ia64/include/uapi/asm/cmpxchg.h Peter Zijlstra 2014-02-04  128  
85f8f7759e418c arch/ia64/include/asm/cmpxchg.h      Paul Gortmaker 2012-04-03  129  /* for compatibility with other platforms: */
85f8f7759e418c arch/ia64/include/asm/cmpxchg.h      Paul Gortmaker 2012-04-03 @130  #define cmpxchg(ptr, o, n)	cmpxchg_acq((ptr), (o), (n))
85f8f7759e418c arch/ia64/include/asm/cmpxchg.h      Paul Gortmaker 2012-04-03  131  #define cmpxchg64(ptr, o, n)	cmpxchg_acq((ptr), (o), (n))
85f8f7759e418c arch/ia64/include/asm/cmpxchg.h      Paul Gortmaker 2012-04-03  132  

:::::: The code at line 130 was first introduced by commit
:::::: 85f8f7759e418c814ee2ceacf73eddb9bed39492 ia64: populate the cmpxchg header with appropriate code

:::::: TO: Paul Gortmaker <paul.gortmaker@windriver.com>
:::::: CC: Paul Gortmaker <paul.gortmaker@windriver.com>

---
0-DAY CI Kernel Test Service, Intel Corporation
https://lists.01.org/hyperkitty/list/kbuild-all(a)lists.01.org

[-- Attachment #2: config.gz --]
[-- Type: application/gzip, Size: 56489 bytes --]