From: Lai Jiangshan <jiangshanlai@gmail.com>
To: linux-kernel@vger.kernel.org
Cc: Lai Jiangshan <laijs@linux.alibaba.com>,
Tejun Heo <tj@kernel.org>, Lai Jiangshan <jiangshanlai@gmail.com>
Subject: [PATCH 05/10] workqueue: introduce wq_online_cpumask
Date: Mon, 14 Dec 2020 23:54:52 +0800 [thread overview]
Message-ID: <20201214155457.3430-6-jiangshanlai@gmail.com> (raw)
In-Reply-To: <20201214155457.3430-1-jiangshanlai@gmail.com>
From: Lai Jiangshan <laijs@linux.alibaba.com>
wq_online_cpumask is the cached result of cpu_online_mask with the
going-down cpu cleared. It is needed for later patches for setting
correct cpumask for workers and break affinity initiatively.
Signed-off-by: Lai Jiangshan <laijs@linux.alibaba.com>
---
kernel/workqueue.c | 34 ++++++++++++++++++----------------
1 file changed, 18 insertions(+), 16 deletions(-)
diff --git a/kernel/workqueue.c b/kernel/workqueue.c
index f679c599a70b..8aca3afc88aa 100644
--- a/kernel/workqueue.c
+++ b/kernel/workqueue.c
@@ -310,6 +310,9 @@ static bool workqueue_freezing; /* PL: have wqs started freezing? */
/* PL: allowable cpus for unbound wqs and work items */
static cpumask_var_t wq_unbound_cpumask;
+/* PL: online cpus (cpu_online_mask with the going-down cpu cleared) */
+static cpumask_var_t wq_online_cpumask;
+
/* CPU where unbound work was last round robin scheduled from this CPU */
static DEFINE_PER_CPU(int, wq_rr_cpu_last);
@@ -3830,12 +3833,10 @@ static struct pool_workqueue *alloc_unbound_pwq(struct workqueue_struct *wq,
* wq_calc_node_cpumask - calculate a wq_attrs' cpumask for the specified node
* @attrs: the wq_attrs of the default pwq of the target workqueue
* @node: the target NUMA node
- * @cpu_going_down: if >= 0, the CPU to consider as offline
* @cpumask: outarg, the resulting cpumask
*
- * Calculate the cpumask a workqueue with @attrs should use on @node. If
- * @cpu_going_down is >= 0, that cpu is considered offline during
- * calculation. The result is stored in @cpumask.
+ * Calculate the cpumask a workqueue with @attrs should use on @node.
+ * The result is stored in @cpumask.
*
* If NUMA affinity is not enabled, @attrs->cpumask is always used. If
* enabled and @node has online CPUs requested by @attrs, the returned
@@ -3849,15 +3850,14 @@ static struct pool_workqueue *alloc_unbound_pwq(struct workqueue_struct *wq,
* %false if equal.
*/
static bool wq_calc_node_cpumask(const struct workqueue_attrs *attrs, int node,
- int cpu_going_down, cpumask_t *cpumask)
+ cpumask_t *cpumask)
{
if (!wq_numa_enabled || attrs->no_numa)
goto use_dfl;
/* does @node have any online CPUs @attrs wants? */
cpumask_and(cpumask, cpumask_of_node(node), attrs->cpumask);
- if (cpu_going_down >= 0)
- cpumask_clear_cpu(cpu_going_down, cpumask);
+ cpumask_and(cpumask, cpumask, wq_online_cpumask);
if (cpumask_empty(cpumask))
goto use_dfl;
@@ -3966,7 +3966,7 @@ apply_wqattrs_prepare(struct workqueue_struct *wq,
goto out_free;
for_each_node(node) {
- if (wq_calc_node_cpumask(new_attrs, node, -1, tmp_attrs->cpumask)) {
+ if (wq_calc_node_cpumask(new_attrs, node, tmp_attrs->cpumask)) {
ctx->pwq_tbl[node] = alloc_unbound_pwq(wq, tmp_attrs);
if (!ctx->pwq_tbl[node])
goto out_free;
@@ -4091,7 +4091,6 @@ int apply_workqueue_attrs(struct workqueue_struct *wq,
* wq_update_unbound_numa - update NUMA affinity of a wq for CPU hot[un]plug
* @wq: the target workqueue
* @cpu: the CPU coming up or going down
- * @online: whether @cpu is coming up or going down
*
* This function is to be called from %CPU_DOWN_PREPARE, %CPU_ONLINE and
* %CPU_DOWN_FAILED. @cpu is being hot[un]plugged, update NUMA affinity of
@@ -4109,11 +4108,9 @@ int apply_workqueue_attrs(struct workqueue_struct *wq,
* affinity, it's the user's responsibility to flush the work item from
* CPU_DOWN_PREPARE.
*/
-static void wq_update_unbound_numa(struct workqueue_struct *wq, int cpu,
- bool online)
+static void wq_update_unbound_numa(struct workqueue_struct *wq, int cpu)
{
int node = cpu_to_node(cpu);
- int cpu_off = online ? -1 : cpu;
struct pool_workqueue *old_pwq = NULL, *pwq;
struct workqueue_attrs *target_attrs;
cpumask_t *cpumask;
@@ -4141,7 +4138,7 @@ static void wq_update_unbound_numa(struct workqueue_struct *wq, int cpu,
* and create a new one if they don't match. If the target cpumask
* equals the default pwq's, the default pwq should be used.
*/
- if (wq_calc_node_cpumask(wq->dfl_pwq->pool->attrs, node, cpu_off, cpumask)) {
+ if (wq_calc_node_cpumask(wq->dfl_pwq->pool->attrs, node, cpumask)) {
if (cpumask_equal(cpumask, pwq->pool->attrs->cpumask))
return;
} else {
@@ -5079,6 +5076,7 @@ int workqueue_online_cpu(unsigned int cpu)
int pi;
mutex_lock(&wq_pool_mutex);
+ cpumask_set_cpu(cpu, wq_online_cpumask);
for_each_pool(pool, pi) {
mutex_lock(&wq_pool_attach_mutex);
@@ -5093,7 +5091,7 @@ int workqueue_online_cpu(unsigned int cpu)
/* update NUMA affinity of unbound workqueues */
list_for_each_entry(wq, &workqueues, list)
- wq_update_unbound_numa(wq, cpu, true);
+ wq_update_unbound_numa(wq, cpu);
mutex_unlock(&wq_pool_mutex);
return 0;
@@ -5111,8 +5109,9 @@ int workqueue_offline_cpu(unsigned int cpu)
/* update NUMA affinity of unbound workqueues */
mutex_lock(&wq_pool_mutex);
+ cpumask_clear_cpu(cpu, wq_online_cpumask);
list_for_each_entry(wq, &workqueues, list)
- wq_update_unbound_numa(wq, cpu, false);
+ wq_update_unbound_numa(wq, cpu);
mutex_unlock(&wq_pool_mutex);
return 0;
@@ -5949,6 +5948,9 @@ void __init workqueue_init_early(void)
BUILD_BUG_ON(__alignof__(struct pool_workqueue) < __alignof__(long long));
+ BUG_ON(!alloc_cpumask_var(&wq_online_cpumask, GFP_KERNEL));
+ cpumask_copy(wq_online_cpumask, cpu_online_mask);
+
BUG_ON(!alloc_cpumask_var(&wq_unbound_cpumask, GFP_KERNEL));
cpumask_copy(wq_unbound_cpumask, housekeeping_cpumask(hk_flags));
@@ -6045,7 +6047,7 @@ void __init workqueue_init(void)
}
list_for_each_entry(wq, &workqueues, list) {
- wq_update_unbound_numa(wq, smp_processor_id(), true);
+ wq_update_unbound_numa(wq, smp_processor_id());
WARN(init_rescuer(wq),
"workqueue: failed to create early rescuer for %s",
wq->name);
--
2.19.1.6.gb485710b
next prev parent reply other threads:[~2020-12-14 14:58 UTC|newest]
Thread overview: 25+ messages / expand[flat|nested] mbox.gz Atom feed top
2020-12-14 15:54 [PATCH 00/10] workqueue: break affinity initiatively Lai Jiangshan
2020-12-14 15:54 ` [PATCH 01/10] workqueue: restore unbound_workers' cpumask correctly Lai Jiangshan
2020-12-14 15:54 ` [PATCH 02/10] workqueue: use cpu_possible_mask instead of cpu_active_mask to break affinity Lai Jiangshan
2020-12-14 17:25 ` Peter Zijlstra
2020-12-15 8:33 ` Lai Jiangshan
2020-12-15 8:40 ` Peter Zijlstra
2020-12-16 14:32 ` Tejun Heo
2020-12-14 15:54 ` [PATCH 03/10] workqueue: Manually break affinity on pool detachment Lai Jiangshan
2020-12-14 15:54 ` [PATCH 04/10] workqueue: don't set the worker's cpumask when kthread_bind_mask() Lai Jiangshan
2020-12-16 14:39 ` Tejun Heo
2020-12-14 15:54 ` Lai Jiangshan [this message]
2020-12-14 15:54 ` [PATCH 06/10] workqueue: use wq_online_cpumask in restore_unbound_workers_cpumask() Lai Jiangshan
2020-12-14 15:54 ` [PATCH 07/10] workqueue: Manually break affinity on hotplug for unbound pool Lai Jiangshan
2020-12-16 14:50 ` Tejun Heo
2020-12-14 15:54 ` [PATCH 08/10] workqueue: reorganize workqueue_online_cpu() Lai Jiangshan
2020-12-14 15:54 ` [PATCH 09/10] workqueue: reorganize workqueue_offline_cpu() unbind_workers() Lai Jiangshan
2020-12-14 15:54 ` [PATCH 10/10] workqueue: Fix affinity of kworkers when attaching into pool Lai Jiangshan
2020-12-15 15:03 ` Valentin Schneider
2020-12-14 17:36 ` [PATCH 00/10] workqueue: break affinity initiatively Peter Zijlstra
2020-12-15 5:44 ` Lai Jiangshan
2020-12-15 7:50 ` Peter Zijlstra
2020-12-15 8:14 ` Lai Jiangshan
2020-12-15 8:49 ` Peter Zijlstra
2020-12-15 9:46 ` Lai Jiangshan
2020-12-16 14:30 ` Tejun Heo
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20201214155457.3430-6-jiangshanlai@gmail.com \
--to=jiangshanlai@gmail.com \
--cc=laijs@linux.alibaba.com \
--cc=linux-kernel@vger.kernel.org \
--cc=tj@kernel.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).