Subject: softirq: Factor loop termination condition From: Peter Zijlstra Date: Fri Sep 11 17:17:20 CEST 2020 Signed-off-by: Peter Zijlstra (Intel) --- kernel/softirq.c | 44 +++++++++++++++++++++++++------------------- 1 file changed, 25 insertions(+), 19 deletions(-) --- a/kernel/softirq.c +++ b/kernel/softirq.c @@ -204,22 +204,6 @@ void __local_bh_enable_ip(unsigned long } EXPORT_SYMBOL(__local_bh_enable_ip); -/* - * We restart softirq processing for at most MAX_SOFTIRQ_RESTART times, - * but break the loop if need_resched() is set or after 2 ms. - * The MAX_SOFTIRQ_TIME provides a nice upper bound in most cases, but in - * certain cases, such as stop_machine(), jiffies may cease to - * increment and so we need the MAX_SOFTIRQ_RESTART limit as - * well to make sure we eventually return from this method. - * - * These limits have been established via experimentation. - * The two things to balance is latency against fairness - - * we want to handle softirqs as soon as possible, but they - * should not be able to lock up the box. - */ -#define MAX_SOFTIRQ_TIME 2*NSEC_PER_MSEC -#define MAX_SOFTIRQ_RESTART 10 - #ifdef CONFIG_TRACE_IRQFLAGS /* * When we run softirqs from irq_exit() and thus on the hardirq stack we need @@ -253,10 +237,33 @@ static inline bool lockdep_softirq_start static inline void lockdep_softirq_end(bool in_hardirq) { } #endif +/* + * We restart softirq processing but break the loop if need_resched() is set or + * after 2 ms. The MAX_SOFTIRQ_RESTART guarantees a loop termination if + * sched_clock() were ever to stall. + * + * These limits have been established via experimentation. The two things to + * balance is latency against fairness - we want to handle softirqs as soon as + * possible, but they should not be able to lock up the box. + */ +#define MAX_SOFTIRQ_TIME 2*NSEC_PER_MSEC +#define MAX_SOFTIRQ_RESTART 10 + +static inline bool __softirq_needs_break(u64 start) +{ + if (need_resched()) + return true; + + if (sched_clock() - start >= MAX_SOFTIRQ_TIME) + return true; + + return false; +} + asmlinkage __visible void __softirq_entry __do_softirq(void) { + unsigned int max_restart = MAX_SOFTIRQ_RESTART; unsigned long old_flags = current->flags; - int max_restart = MAX_SOFTIRQ_RESTART; u64 start = sched_clock(); struct softirq_action *h; unsigned long pending; @@ -310,8 +317,7 @@ asmlinkage __visible void __softirq_entr pending = local_softirq_pending(); if (pending) { - if (sched_clock() - start < MAX_SOFTIRQ_TIME && !need_resched() && - --max_restart) + if (!__softirq_needs_break(start) && --max_restart) goto restart; wakeup_softirqd();