All of lore.kernel.org
 help / color / mirror / Atom feed
* [PATCH v3 0/2] refactor sqthread cpu binding logic
@ 2021-09-01 10:18 Hao Xu
  2021-09-01 10:18 ` [PATCH 1/2] cpuset: add a helper to check if cpu in cpuset of current task Hao Xu
  2021-09-01 10:18 ` [PATCH 2/2] io_uring: consider cgroup setting when binding sqpoll cpu Hao Xu
  0 siblings, 2 replies; 5+ messages in thread
From: Hao Xu @ 2021-09-01 10:18 UTC (permalink / raw)
  To: Jens Axboe, Zefan Li, Tejun Heo, Johannes Weiner, Pavel Begunkov
  Cc: io-uring, cgroups, Joseph Qi

This patchset is to enhance sqthread cpu binding logic, we didn't
consider cgroup setting before. In container environment, theoretically
sqthread is in its container's task group, it shouldn't occupy cpu out
of its container. Otherwise it may cause some problems, for example:
resource balancer may controll cpu resource allocation by a container's
current cpu usage, here sqthread should be counted in.

v1-->v2
 - add helper to do cpu in current-cpuset test

v2-->v3
 - split it to 2 patches, first as prep one, second as comsumer
 - remove warnning, since cpuset may change anytime, we cannot catch
   all cases, so make the behaviour consistent.

Hao Xu (2):
  cpuset: add a helper to check if cpu in cpuset of current task
  io_uring: consider cgroup setting when binding sqpoll cpu

 fs/io_uring.c          | 19 ++++++++++++++-----
 include/linux/cpuset.h |  7 +++++++
 kernel/cgroup/cpuset.c | 11 +++++++++++
 3 files changed, 32 insertions(+), 5 deletions(-)

-- 
2.24.4


^ permalink raw reply	[flat|nested] 5+ messages in thread

* [PATCH 1/2] cpuset: add a helper to check if cpu in cpuset of current task
  2021-09-01 10:18 [PATCH v3 0/2] refactor sqthread cpu binding logic Hao Xu
@ 2021-09-01 10:18 ` Hao Xu
  2021-09-01 10:18 ` [PATCH 2/2] io_uring: consider cgroup setting when binding sqpoll cpu Hao Xu
  1 sibling, 0 replies; 5+ messages in thread
From: Hao Xu @ 2021-09-01 10:18 UTC (permalink / raw)
  To: Jens Axboe, Zefan Li, Tejun Heo, Johannes Weiner, Pavel Begunkov
  Cc: io-uring, cgroups, Joseph Qi

Check if a cpu is in the cpuset of current task, not guaranteed that
cpu is online.

Signed-off-by: Hao Xu <haoxu@linux.alibaba.com>
---
 include/linux/cpuset.h |  7 +++++++
 kernel/cgroup/cpuset.c | 11 +++++++++++
 2 files changed, 18 insertions(+)

diff --git a/include/linux/cpuset.h b/include/linux/cpuset.h
index 04c20de66afc..fad77c91bc1f 100644
--- a/include/linux/cpuset.h
+++ b/include/linux/cpuset.h
@@ -116,6 +116,8 @@ static inline int cpuset_do_slab_mem_spread(void)
 
 extern bool current_cpuset_is_being_rebound(void);
 
+extern bool test_cpu_in_current_cpuset(int cpu);
+
 extern void rebuild_sched_domains(void);
 
 extern void cpuset_print_current_mems_allowed(void);
@@ -257,6 +259,11 @@ static inline bool current_cpuset_is_being_rebound(void)
 	return false;
 }
 
+static inline bool test_cpu_in_current_cpuset(int cpu)
+{
+	return false;
+}
+
 static inline void rebuild_sched_domains(void)
 {
 	partition_sched_domains(1, NULL, NULL);
diff --git a/kernel/cgroup/cpuset.c b/kernel/cgroup/cpuset.c
index adb5190c4429..a63c27e9430e 100644
--- a/kernel/cgroup/cpuset.c
+++ b/kernel/cgroup/cpuset.c
@@ -1849,6 +1849,17 @@ bool current_cpuset_is_being_rebound(void)
 	return ret;
 }
 
+bool test_cpu_in_current_cpuset(int cpu)
+{
+	bool ret;
+
+	rcu_read_lock();
+	ret = cpumask_test_cpu(cpu, task_cs(current)->effective_cpus);
+	rcu_read_unlock();
+
+	return ret;
+}
+
 static int update_relax_domain_level(struct cpuset *cs, s64 val)
 {
 #ifdef CONFIG_SMP
-- 
2.24.4


^ permalink raw reply related	[flat|nested] 5+ messages in thread

* [PATCH 2/2] io_uring: consider cgroup setting when binding sqpoll cpu
  2021-09-01 10:18 [PATCH v3 0/2] refactor sqthread cpu binding logic Hao Xu
  2021-09-01 10:18 ` [PATCH 1/2] cpuset: add a helper to check if cpu in cpuset of current task Hao Xu
@ 2021-09-01 10:18 ` Hao Xu
  2021-09-01 12:31   ` Hao Xu
  1 sibling, 1 reply; 5+ messages in thread
From: Hao Xu @ 2021-09-01 10:18 UTC (permalink / raw)
  To: Jens Axboe, Zefan Li, Tejun Heo, Johannes Weiner, Pavel Begunkov
  Cc: io-uring, cgroups, Joseph Qi

Since sqthread is userspace like thread now, it should respect cgroup
setting, thus we should consider current allowed cpuset when doing
cpu binding for sqthread.

Signed-off-by: Hao Xu <haoxu@linux.alibaba.com>
---
 fs/io_uring.c | 19 ++++++++++++++-----
 1 file changed, 14 insertions(+), 5 deletions(-)

diff --git a/fs/io_uring.c b/fs/io_uring.c
index 7cc458e0b636..414dfedf79a7 100644
--- a/fs/io_uring.c
+++ b/fs/io_uring.c
@@ -79,6 +79,7 @@
 #include <linux/pagemap.h>
 #include <linux/io_uring.h>
 #include <linux/tracehook.h>
+#include <linux/cpuset.h>
 
 #define CREATE_TRACE_POINTS
 #include <trace/events/io_uring.h>
@@ -7102,6 +7103,14 @@ static bool io_sqd_handle_event(struct io_sq_data *sqd)
 	return did_sig || test_bit(IO_SQ_THREAD_SHOULD_STOP, &sqd->state);
 }
 
+static inline int io_sq_bind_cpu(int cpu)
+{
+	if (test_cpu_in_current_cpuset(cpu))
+		set_cpus_allowed_ptr(current, cpumask_of(cpu));
+
+	return 0;
+}
+
 static int io_sq_thread(void *data)
 {
 	struct io_sq_data *sqd = data;
@@ -7112,11 +7121,9 @@ static int io_sq_thread(void *data)
 
 	snprintf(buf, sizeof(buf), "iou-sqp-%d", sqd->task_pid);
 	set_task_comm(current, buf);
-
 	if (sqd->sq_cpu != -1)
-		set_cpus_allowed_ptr(current, cpumask_of(sqd->sq_cpu));
-	else
-		set_cpus_allowed_ptr(current, cpu_online_mask);
+		io_sq_bind_cpu(sqd->sq_cpu);
+
 	current->flags |= PF_NO_SETAFFINITY;
 
 	mutex_lock(&sqd->lock);
@@ -8310,8 +8317,10 @@ static int io_sq_offload_create(struct io_ring_ctx *ctx,
 			int cpu = p->sq_thread_cpu;
 
 			ret = -EINVAL;
-			if (cpu >= nr_cpu_ids || !cpu_online(cpu))
+			if (cpu >= nr_cpu_ids || !cpu_online(cpu) ||
+			    !test_cpu_in_current_cpuset(cpu))
 				goto err_sqpoll;
+
 			sqd->sq_cpu = cpu;
 		} else {
 			sqd->sq_cpu = -1;
-- 
2.24.4


^ permalink raw reply related	[flat|nested] 5+ messages in thread

* Re: [PATCH 2/2] io_uring: consider cgroup setting when binding sqpoll cpu
  2021-09-01 10:18 ` [PATCH 2/2] io_uring: consider cgroup setting when binding sqpoll cpu Hao Xu
@ 2021-09-01 12:31   ` Hao Xu
  0 siblings, 0 replies; 5+ messages in thread
From: Hao Xu @ 2021-09-01 12:31 UTC (permalink / raw)
  To: Jens Axboe, Zefan Li, Tejun Heo, Johannes Weiner, Pavel Begunkov
  Cc: io-uring, cgroups, Joseph Qi

在 2021/9/1 下午6:18, Hao Xu 写道:
> Since sqthread is userspace like thread now, it should respect cgroup
> setting, thus we should consider current allowed cpuset when doing
> cpu binding for sqthread.
> 
> Signed-off-by: Hao Xu <haoxu@linux.alibaba.com>
> ---
>   fs/io_uring.c | 19 ++++++++++++++-----
>   1 file changed, 14 insertions(+), 5 deletions(-)
> 
> diff --git a/fs/io_uring.c b/fs/io_uring.c
> index 7cc458e0b636..414dfedf79a7 100644
> --- a/fs/io_uring.c
> +++ b/fs/io_uring.c
> @@ -79,6 +79,7 @@
>   #include <linux/pagemap.h>
>   #include <linux/io_uring.h>
>   #include <linux/tracehook.h>
> +#include <linux/cpuset.h>
>   
>   #define CREATE_TRACE_POINTS
>   #include <trace/events/io_uring.h>
> @@ -7102,6 +7103,14 @@ static bool io_sqd_handle_event(struct io_sq_data *sqd)
>   	return did_sig || test_bit(IO_SQ_THREAD_SHOULD_STOP, &sqd->state);
>   }
>   
> +static inline int io_sq_bind_cpu(int cpu)
> +{
> +	if (test_cpu_in_current_cpuset(cpu))
> +		set_cpus_allowed_ptr(current, cpumask_of(cpu));
> +
> +	return 0;
Ah, no need to return value anymore, even no need to have this function
here. I'll resend a new version.
> +}
> +
>   static int io_sq_thread(void *data)
>   {
>   	struct io_sq_data *sqd = data;
> @@ -7112,11 +7121,9 @@ static int io_sq_thread(void *data)
>   
>   	snprintf(buf, sizeof(buf), "iou-sqp-%d", sqd->task_pid);
>   	set_task_comm(current, buf);
> -
>   	if (sqd->sq_cpu != -1)
> -		set_cpus_allowed_ptr(current, cpumask_of(sqd->sq_cpu));
> -	else
> -		set_cpus_allowed_ptr(current, cpu_online_mask);
> +		io_sq_bind_cpu(sqd->sq_cpu);
> +
>   	current->flags |= PF_NO_SETAFFINITY;
>   
>   	mutex_lock(&sqd->lock);
> @@ -8310,8 +8317,10 @@ static int io_sq_offload_create(struct io_ring_ctx *ctx,
>   			int cpu = p->sq_thread_cpu;
>   
>   			ret = -EINVAL;
> -			if (cpu >= nr_cpu_ids || !cpu_online(cpu))
> +			if (cpu >= nr_cpu_ids || !cpu_online(cpu) ||
> +			    !test_cpu_in_current_cpuset(cpu))
>   				goto err_sqpoll;
> +
>   			sqd->sq_cpu = cpu;
>   		} else {
>   			sqd->sq_cpu = -1;
> 


^ permalink raw reply	[flat|nested] 5+ messages in thread

* [PATCH 1/2] cpuset: add a helper to check if cpu in cpuset of current task
  2021-09-01 12:43 [PATCH v4 0/2] refactor sqthread cpu binding logic Hao Xu
@ 2021-09-01 12:43 ` Hao Xu
  0 siblings, 0 replies; 5+ messages in thread
From: Hao Xu @ 2021-09-01 12:43 UTC (permalink / raw)
  To: Jens Axboe, Zefan Li, Tejun Heo, Johannes Weiner, Pavel Begunkov
  Cc: io-uring, cgroups, Joseph Qi

Check if a cpu is in the cpuset of current task, not guaranteed that
cpu is online.

Signed-off-by: Hao Xu <haoxu@linux.alibaba.com>
---
 include/linux/cpuset.h |  7 +++++++
 kernel/cgroup/cpuset.c | 11 +++++++++++
 2 files changed, 18 insertions(+)

diff --git a/include/linux/cpuset.h b/include/linux/cpuset.h
index 04c20de66afc..fad77c91bc1f 100644
--- a/include/linux/cpuset.h
+++ b/include/linux/cpuset.h
@@ -116,6 +116,8 @@ static inline int cpuset_do_slab_mem_spread(void)
 
 extern bool current_cpuset_is_being_rebound(void);
 
+extern bool test_cpu_in_current_cpuset(int cpu);
+
 extern void rebuild_sched_domains(void);
 
 extern void cpuset_print_current_mems_allowed(void);
@@ -257,6 +259,11 @@ static inline bool current_cpuset_is_being_rebound(void)
 	return false;
 }
 
+static inline bool test_cpu_in_current_cpuset(int cpu)
+{
+	return false;
+}
+
 static inline void rebuild_sched_domains(void)
 {
 	partition_sched_domains(1, NULL, NULL);
diff --git a/kernel/cgroup/cpuset.c b/kernel/cgroup/cpuset.c
index adb5190c4429..a63c27e9430e 100644
--- a/kernel/cgroup/cpuset.c
+++ b/kernel/cgroup/cpuset.c
@@ -1849,6 +1849,17 @@ bool current_cpuset_is_being_rebound(void)
 	return ret;
 }
 
+bool test_cpu_in_current_cpuset(int cpu)
+{
+	bool ret;
+
+	rcu_read_lock();
+	ret = cpumask_test_cpu(cpu, task_cs(current)->effective_cpus);
+	rcu_read_unlock();
+
+	return ret;
+}
+
 static int update_relax_domain_level(struct cpuset *cs, s64 val)
 {
 #ifdef CONFIG_SMP
-- 
2.24.4


^ permalink raw reply related	[flat|nested] 5+ messages in thread

end of thread, other threads:[~2021-09-01 12:45 UTC | newest]

Thread overview: 5+ messages (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2021-09-01 10:18 [PATCH v3 0/2] refactor sqthread cpu binding logic Hao Xu
2021-09-01 10:18 ` [PATCH 1/2] cpuset: add a helper to check if cpu in cpuset of current task Hao Xu
2021-09-01 10:18 ` [PATCH 2/2] io_uring: consider cgroup setting when binding sqpoll cpu Hao Xu
2021-09-01 12:31   ` Hao Xu
2021-09-01 12:43 [PATCH v4 0/2] refactor sqthread cpu binding logic Hao Xu
2021-09-01 12:43 ` [PATCH 1/2] cpuset: add a helper to check if cpu in cpuset of current task Hao Xu

This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.