[PATCH 1/2] workqueue: don't always set __WQ_ORDERED implicitly

[PATCH 1/2] workqueue: don't always set __WQ_ORDERED implicitly

[Bob Liu]

Current code always set 'Unbound && max_active == 1' workqueues to ordered
implicitly, while this may be not an expected behaviour for some use cases.

E.g some scsi and iscsi workqueues(unbound && max_active = 1) want to be bind
to different cpu so as to get better isolation, but their cpumask can't be
changed because WQ_ORDERED is set implicitly.

This patch adds a flag __WQ_ORDERED_DISABLE and also
create_singlethread_workqueue_noorder() to offer an new option.

Signed-off-by: Bob Liu <bob.liu@oracle.com>
---
 include/linux/workqueue.h | 4 ++++
 kernel/workqueue.c        | 4 +++-
 2 files changed, 7 insertions(+), 1 deletion(-)

diff --git a/include/linux/workqueue.h b/include/linux/workqueue.h
index e48554e..4c86913 100644
--- a/include/linux/workqueue.h
+++ b/include/linux/workqueue.h
@@ -344,6 +344,7 @@ enum {
 	__WQ_ORDERED		= 1 << 17, /* internal: workqueue is ordered */
 	__WQ_LEGACY		= 1 << 18, /* internal: create*_workqueue() */
 	__WQ_ORDERED_EXPLICIT	= 1 << 19, /* internal: alloc_ordered_workqueue() */
+	__WQ_ORDERED_DISABLE	= 1 << 20, /* internal: don't set __WQ_ORDERED implicitly */
 
 	WQ_MAX_ACTIVE		= 512,	  /* I like 512, better ideas? */
 	WQ_MAX_UNBOUND_PER_CPU	= 4,	  /* 4 * #cpus for unbound wq */
@@ -433,6 +434,9 @@ struct workqueue_struct *alloc_workqueue(const char *fmt,
 #define create_singlethread_workqueue(name)				\
 	alloc_ordered_workqueue("%s", __WQ_LEGACY | WQ_MEM_RECLAIM, name)
 
+#define create_singlethread_workqueue_noorder(name)			\
+	alloc_workqueue("%s", WQ_SYSFS | __WQ_LEGACY | WQ_MEM_RECLAIM | \
+			WQ_UNBOUND | __WQ_ORDERED_DISABLE, 1, (name))
 extern void destroy_workqueue(struct workqueue_struct *wq);
 
 struct workqueue_attrs *alloc_workqueue_attrs(void);
diff --git a/kernel/workqueue.c b/kernel/workqueue.c
index 4e01c44..2167013 100644
--- a/kernel/workqueue.c
+++ b/kernel/workqueue.c
@@ -4237,7 +4237,9 @@ struct workqueue_struct *alloc_workqueue(const char *fmt,
 	 * on NUMA.
 	 */
 	if ((flags & WQ_UNBOUND) && max_active == 1)
-		flags |= __WQ_ORDERED;
+		/* the caller may don't want __WQ_ORDERED to be set implicitly. */
+		if (!(flags & __WQ_ORDERED_DISABLE))
+			flags |= __WQ_ORDERED;
 
 	/* see the comment above the definition of WQ_POWER_EFFICIENT */
 	if ((flags & WQ_POWER_EFFICIENT) && wq_power_efficient)
-- 
2.9.5

[PATCH 2/2] scsi: register sysfs for scsi/iscsi workqueues

[Bob Liu]

This patch enable setting cpu affinity through "cpumask" for below
scsi/iscsi workqueues, so as to get better isolation.
- scsi_wq_*
- scsi_tmf_*
- iscsi_q_xx
- iscsi_eh

Signed-off-by: Bob Liu <bob.liu@oracle.com>
---
 drivers/scsi/hosts.c                | 4 ++--
 drivers/scsi/libiscsi.c             | 2 +-
 drivers/scsi/scsi_transport_iscsi.c | 2 +-
 3 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/drivers/scsi/hosts.c b/drivers/scsi/hosts.c
index 1d669e4..4b9f80d 100644
--- a/drivers/scsi/hosts.c
+++ b/drivers/scsi/hosts.c
@@ -272,7 +272,7 @@ int scsi_add_host_with_dma(struct Scsi_Host *shost, struct device *dev,
 	if (shost->transportt->create_work_queue) {
 		snprintf(shost->work_q_name, sizeof(shost->work_q_name),
 			 "scsi_wq_%d", shost->host_no);
-		shost->work_q = create_singlethread_workqueue(
+		shost->work_q = create_singlethread_workqueue_noorder(
 					shost->work_q_name);
 		if (!shost->work_q) {
 			error = -EINVAL;
@@ -487,7 +487,7 @@ struct Scsi_Host *scsi_host_alloc(struct scsi_host_template *sht, int privsize)
 	}
 
 	shost->tmf_work_q = alloc_workqueue("scsi_tmf_%d",
-					    WQ_UNBOUND | WQ_MEM_RECLAIM,
+				WQ_UNBOUND | WQ_MEM_RECLAIM | WQ_SYSFS | __WQ_ORDERED_DISABLE,
 					   1, shost->host_no);
 	if (!shost->tmf_work_q) {
 		shost_printk(KERN_WARNING, shost,
diff --git a/drivers/scsi/libiscsi.c b/drivers/scsi/libiscsi.c
index 70b99c0..6808cf3 100644
--- a/drivers/scsi/libiscsi.c
+++ b/drivers/scsi/libiscsi.c
@@ -2627,7 +2627,7 @@ struct Scsi_Host *iscsi_host_alloc(struct scsi_host_template *sht,
 	if (xmit_can_sleep) {
 		snprintf(ihost->workq_name, sizeof(ihost->workq_name),
 			"iscsi_q_%d", shost->host_no);
-		ihost->workq = create_singlethread_workqueue(ihost->workq_name);
+		ihost->workq = create_singlethread_workqueue_noorder(ihost->workq_name);
 		if (!ihost->workq)
 			goto free_host;
 	}
diff --git a/drivers/scsi/scsi_transport_iscsi.c b/drivers/scsi/scsi_transport_iscsi.c
index dfc726f..d07a0e4 100644
--- a/drivers/scsi/scsi_transport_iscsi.c
+++ b/drivers/scsi/scsi_transport_iscsi.c
@@ -4602,7 +4602,7 @@ static __init int iscsi_transport_init(void)
 		goto unregister_flashnode_bus;
 	}
 
-	iscsi_eh_timer_workq = create_singlethread_workqueue("iscsi_eh");
+	iscsi_eh_timer_workq = create_singlethread_workqueue_noorder("iscsi_eh");
 	if (!iscsi_eh_timer_workq) {
 		err = -ENOMEM;
 		goto release_nls;
-- 
2.9.5

Re: [PATCH 1/2] workqueue: don't always set __WQ_ORDERED implicitly

[Bob Liu]

ping..

On 6/11/20 6:07 PM, Bob Liu wrote:
> Current code always set 'Unbound && max_active == 1' workqueues to ordered
> implicitly, while this may be not an expected behaviour for some use cases.
> 
> E.g some scsi and iscsi workqueues(unbound && max_active = 1) want to be bind
> to different cpu so as to get better isolation, but their cpumask can't be
> changed because WQ_ORDERED is set implicitly.
> 
> This patch adds a flag __WQ_ORDERED_DISABLE and also
> create_singlethread_workqueue_noorder() to offer an new option.
> 
> Signed-off-by: Bob Liu <bob.liu@oracle.com>
> ---
>  include/linux/workqueue.h | 4 ++++
>  kernel/workqueue.c        | 4 +++-
>  2 files changed, 7 insertions(+), 1 deletion(-)
> 
> diff --git a/include/linux/workqueue.h b/include/linux/workqueue.h
> index e48554e..4c86913 100644
> --- a/include/linux/workqueue.h
> +++ b/include/linux/workqueue.h
> @@ -344,6 +344,7 @@ enum {
>  	__WQ_ORDERED		= 1 << 17, /* internal: workqueue is ordered */
>  	__WQ_LEGACY		= 1 << 18, /* internal: create*_workqueue() */
>  	__WQ_ORDERED_EXPLICIT	= 1 << 19, /* internal: alloc_ordered_workqueue() */
> +	__WQ_ORDERED_DISABLE	= 1 << 20, /* internal: don't set __WQ_ORDERED implicitly */
>  
>  	WQ_MAX_ACTIVE		= 512,	  /* I like 512, better ideas? */
>  	WQ_MAX_UNBOUND_PER_CPU	= 4,	  /* 4 * #cpus for unbound wq */
> @@ -433,6 +434,9 @@ struct workqueue_struct *alloc_workqueue(const char *fmt,
>  #define create_singlethread_workqueue(name)				\
>  	alloc_ordered_workqueue("%s", __WQ_LEGACY | WQ_MEM_RECLAIM, name)
>  
> +#define create_singlethread_workqueue_noorder(name)			\
> +	alloc_workqueue("%s", WQ_SYSFS | __WQ_LEGACY | WQ_MEM_RECLAIM | \
> +			WQ_UNBOUND | __WQ_ORDERED_DISABLE, 1, (name))
>  extern void destroy_workqueue(struct workqueue_struct *wq);
>  
>  struct workqueue_attrs *alloc_workqueue_attrs(void);
> diff --git a/kernel/workqueue.c b/kernel/workqueue.c
> index 4e01c44..2167013 100644
> --- a/kernel/workqueue.c
> +++ b/kernel/workqueue.c
> @@ -4237,7 +4237,9 @@ struct workqueue_struct *alloc_workqueue(const char *fmt,
>  	 * on NUMA.
>  	 */
>  	if ((flags & WQ_UNBOUND) && max_active == 1)
> -		flags |= __WQ_ORDERED;
> +		/* the caller may don't want __WQ_ORDERED to be set implicitly. */
> +		if (!(flags & __WQ_ORDERED_DISABLE))
> +			flags |= __WQ_ORDERED;
>  
>  	/* see the comment above the definition of WQ_POWER_EFFICIENT */
>  	if ((flags & WQ_POWER_EFFICIENT) && wq_power_efficient)
> 

Re: [PATCH 2/2] scsi: register sysfs for scsi/iscsi workqueues

[Mike Christie]

On 6/11/20 5:07 AM, Bob Liu wrote:
> This patch enable setting cpu affinity through "cpumask" for below
> scsi/iscsi workqueues, so as to get better isolation.
> - scsi_wq_*
> - scsi_tmf_*
> - iscsi_q_xx
> - iscsi_eh
> 
> Signed-off-by: Bob Liu <bob.liu@oracle.com>
> ---
>   drivers/scsi/hosts.c                | 4 ++--
>   drivers/scsi/libiscsi.c             | 2 +-
>   drivers/scsi/scsi_transport_iscsi.c | 2 +-
>   3 files changed, 4 insertions(+), 4 deletions(-)
> 
> diff --git a/drivers/scsi/hosts.c b/drivers/scsi/hosts.c
> index 1d669e4..4b9f80d 100644
> --- a/drivers/scsi/hosts.c
> +++ b/drivers/scsi/hosts.c
> @@ -272,7 +272,7 @@ int scsi_add_host_with_dma(struct Scsi_Host *shost, struct device *dev,
>   	if (shost->transportt->create_work_queue) {
>   		snprintf(shost->work_q_name, sizeof(shost->work_q_name),
>   			 "scsi_wq_%d", shost->host_no);
> -		shost->work_q = create_singlethread_workqueue(
> +		shost->work_q = create_singlethread_workqueue_noorder(
>   					shost->work_q_name);
>   		if (!shost->work_q) {
>   			error = -EINVAL;

This patch seems ok for the iscsi, fc, tmf, and non transport class scan 
uses. We are either heavy handed with flushes or did not need ordering.

I don't know about the zfcp use though, so I cc'd  the developers listed 
as maintainers. It looks like for zfcp we can do:

zfcp_scsi_rport_register->fc_remote_port_add->fc_remote_port_create->scsi_queue_work 
to scan the scsi target on the rport.

and then zfcp_scsi_rport_register can call zfcp_unit_queue_scsi_scan->
scsi_queue_work which will scan for a specific lun.

It looks ok if those are not ordered, but I would get their review to 
make sure.

Re: [PATCH 2/2] scsi: register sysfs for scsi/iscsi workqueues

[Benjamin Block]

On Mon, Jun 22, 2020 at 10:40:09AM -0500, Mike Christie wrote:
> On 6/11/20 5:07 AM, Bob Liu wrote:
> > This patch enable setting cpu affinity through "cpumask" for below
> > scsi/iscsi workqueues, so as to get better isolation.
> > - scsi_wq_*
> > - scsi_tmf_*
> > - iscsi_q_xx
> > - iscsi_eh
> > 
> > Signed-off-by: Bob Liu <bob.liu@oracle.com>
> > ---
> >   drivers/scsi/hosts.c                | 4 ++--
> >   drivers/scsi/libiscsi.c             | 2 +-
> >   drivers/scsi/scsi_transport_iscsi.c | 2 +-
> >   3 files changed, 4 insertions(+), 4 deletions(-)
> > 
> > diff --git a/drivers/scsi/hosts.c b/drivers/scsi/hosts.c
> > index 1d669e4..4b9f80d 100644
> > --- a/drivers/scsi/hosts.c
> > +++ b/drivers/scsi/hosts.c
> > @@ -272,7 +272,7 @@ int scsi_add_host_with_dma(struct Scsi_Host *shost, struct device *dev,
> >   	if (shost->transportt->create_work_queue) {
> >   		snprintf(shost->work_q_name, sizeof(shost->work_q_name),
> >   			 "scsi_wq_%d", shost->host_no);
> > -		shost->work_q = create_singlethread_workqueue(
> > +		shost->work_q = create_singlethread_workqueue_noorder(
> >   					shost->work_q_name);
> >   		if (!shost->work_q) {
> >   			error = -EINVAL;
> 
> This patch seems ok for the iscsi, fc, tmf, and non transport class scan
> uses. We are either heavy handed with flushes or did not need ordering.
> 
> I don't know about the zfcp use though, so I cc'd  the developers listed as
> maintainers. It looks like for zfcp we can do:

Thx for the notice.

> 
> zfcp_scsi_rport_register->fc_remote_port_add->fc_remote_port_create->scsi_queue_work
> to scan the scsi target on the rport.
> 
> and then zfcp_scsi_rport_register can call zfcp_unit_queue_scsi_scan->
> scsi_queue_work which will scan for a specific lun.
> 
> It looks ok if those are not ordered, but I would get their review to make
> sure.

I am not aware of any temporal requirements of those LUN-scans, so I
think making them not explicitly ordered shouldn't hurt us.

The target scan itself is protected again by `shost->scan_mutex`.. so
all fine I think.

-- 
Best Regards, Benjamin Block  / Linux on IBM Z Kernel Development / IBM Systems
IBM Deutschland Research & Development GmbH    /    https://www.ibm.com/privacy
Vorsitz. AufsR.: Gregor Pillen         /        Geschäftsführung: Dirk Wittkopp
Sitz der Gesellschaft: Böblingen / Registergericht: AmtsG Stuttgart, HRB 243294

Re: [PATCH 1/2] workqueue: don't always set __WQ_ORDERED implicitly

[Lai Jiangshan]

On Thu, Jun 11, 2020 at 6:29 PM Bob Liu <bob.liu@oracle.com> wrote:
>
> Current code always set 'Unbound && max_active == 1' workqueues to ordered
> implicitly, while this may be not an expected behaviour for some use cases.
>
> E.g some scsi and iscsi workqueues(unbound && max_active = 1) want to be bind
> to different cpu so as to get better isolation, but their cpumask can't be
> changed because WQ_ORDERED is set implicitly.

Hello

If I read the code correctly, the reason why their cpumask can't
be changed is because __WQ_ORDERED_EXPLICIT, not __WQ_ORDERED.

>
> This patch adds a flag __WQ_ORDERED_DISABLE and also
> create_singlethread_workqueue_noorder() to offer an new option.
>
> Signed-off-by: Bob Liu <bob.liu@oracle.com>
> ---
>  include/linux/workqueue.h | 4 ++++
>  kernel/workqueue.c        | 4 +++-
>  2 files changed, 7 insertions(+), 1 deletion(-)
>
> diff --git a/include/linux/workqueue.h b/include/linux/workqueue.h
> index e48554e..4c86913 100644
> --- a/include/linux/workqueue.h
> +++ b/include/linux/workqueue.h
> @@ -344,6 +344,7 @@ enum {
>         __WQ_ORDERED            = 1 << 17, /* internal: workqueue is ordered */
>         __WQ_LEGACY             = 1 << 18, /* internal: create*_workqueue() */
>         __WQ_ORDERED_EXPLICIT   = 1 << 19, /* internal: alloc_ordered_workqueue() */
> +       __WQ_ORDERED_DISABLE    = 1 << 20, /* internal: don't set __WQ_ORDERED implicitly */
>
>         WQ_MAX_ACTIVE           = 512,    /* I like 512, better ideas? */
>         WQ_MAX_UNBOUND_PER_CPU  = 4,      /* 4 * #cpus for unbound wq */
> @@ -433,6 +434,9 @@ struct workqueue_struct *alloc_workqueue(const char *fmt,
>  #define create_singlethread_workqueue(name)                            \
>         alloc_ordered_workqueue("%s", __WQ_LEGACY | WQ_MEM_RECLAIM, name)
>
> +#define create_singlethread_workqueue_noorder(name)                    \
> +       alloc_workqueue("%s", WQ_SYSFS | __WQ_LEGACY | WQ_MEM_RECLAIM | \
> +                       WQ_UNBOUND | __WQ_ORDERED_DISABLE, 1, (name))

I think using __WQ_ORDERED without __WQ_ORDERED_EXPLICIT is what you
need, in which case cpumask is allowed to be changed.

Just use alloc_workqueue() with __WQ_ORDERED and max_active=1. It can
be wrapped as a new function or macro, but I don't think
create_singlethread_workqueue_noorder() is a good name for it.

>  extern void destroy_workqueue(struct workqueue_struct *wq);
>
>  struct workqueue_attrs *alloc_workqueue_attrs(void);
> diff --git a/kernel/workqueue.c b/kernel/workqueue.c
> index 4e01c44..2167013 100644
> --- a/kernel/workqueue.c
> +++ b/kernel/workqueue.c
> @@ -4237,7 +4237,9 @@ struct workqueue_struct *alloc_workqueue(const char *fmt,
>          * on NUMA.
>          */
>         if ((flags & WQ_UNBOUND) && max_active == 1)
> -               flags |= __WQ_ORDERED;
> +               /* the caller may don't want __WQ_ORDERED to be set implicitly. */
> +               if (!(flags & __WQ_ORDERED_DISABLE))
> +                       flags |= __WQ_ORDERED;
>
>         /* see the comment above the definition of WQ_POWER_EFFICIENT */
>         if ((flags & WQ_POWER_EFFICIENT) && wq_power_efficient)
> --
> 2.9.5
>

Re: [PATCH 1/2] workqueue: don't always set __WQ_ORDERED implicitly

[Bob Liu]

On 6/28/20 11:54 PM, Lai Jiangshan wrote:
> On Thu, Jun 11, 2020 at 6:29 PM Bob Liu <bob.liu@oracle.com> wrote:
>>
>> Current code always set 'Unbound && max_active == 1' workqueues to ordered
>> implicitly, while this may be not an expected behaviour for some use cases.
>>
>> E.g some scsi and iscsi workqueues(unbound && max_active = 1) want to be bind
>> to different cpu so as to get better isolation, but their cpumask can't be
>> changed because WQ_ORDERED is set implicitly.
> 
> Hello
> 
> If I read the code correctly, the reason why their cpumask can't
> be changed is because __WQ_ORDERED_EXPLICIT, not __WQ_ORDERED.
> 
>>
>> This patch adds a flag __WQ_ORDERED_DISABLE and also
>> create_singlethread_workqueue_noorder() to offer an new option.
>>
>> Signed-off-by: Bob Liu <bob.liu@oracle.com>
>> ---
>>  include/linux/workqueue.h | 4 ++++
>>  kernel/workqueue.c        | 4 +++-
>>  2 files changed, 7 insertions(+), 1 deletion(-)
>>
>> diff --git a/include/linux/workqueue.h b/include/linux/workqueue.h
>> index e48554e..4c86913 100644
>> --- a/include/linux/workqueue.h
>> +++ b/include/linux/workqueue.h
>> @@ -344,6 +344,7 @@ enum {
>>         __WQ_ORDERED            = 1 << 17, /* internal: workqueue is ordered */
>>         __WQ_LEGACY             = 1 << 18, /* internal: create*_workqueue() */
>>         __WQ_ORDERED_EXPLICIT   = 1 << 19, /* internal: alloc_ordered_workqueue() */
>> +       __WQ_ORDERED_DISABLE    = 1 << 20, /* internal: don't set __WQ_ORDERED implicitly */
>>
>>         WQ_MAX_ACTIVE           = 512,    /* I like 512, better ideas? */
>>         WQ_MAX_UNBOUND_PER_CPU  = 4,      /* 4 * #cpus for unbound wq */
>> @@ -433,6 +434,9 @@ struct workqueue_struct *alloc_workqueue(const char *fmt,
>>  #define create_singlethread_workqueue(name)                            \
>>         alloc_ordered_workqueue("%s", __WQ_LEGACY | WQ_MEM_RECLAIM, name)
>>
>> +#define create_singlethread_workqueue_noorder(name)                    \
>> +       alloc_workqueue("%s", WQ_SYSFS | __WQ_LEGACY | WQ_MEM_RECLAIM | \
>> +                       WQ_UNBOUND | __WQ_ORDERED_DISABLE, 1, (name))
> 
> I think using __WQ_ORDERED without __WQ_ORDERED_EXPLICIT is what you
> need, in which case cpumask is allowed to be changed.
> 

I don't think so, see function workqueue_apply_unbound_cpumask():

wq_unbound_cpumask_store()
 > workqueue_set_unbound_cpumask()
   > workqueue_apply_unbound_cpumask() {
     ...
5276                 /* creating multiple pwqs breaks ordering guarantee */
5277                 if (wq->flags & __WQ_ORDERED)
5278                         continue;
                     	  ^^^^
                          Here will skip apply cpumask if only __WQ_ORDERED is set.

5280                 ctx = apply_wqattrs_prepare(wq, wq->unbound_attrs);

     }

Thanks for your review.
Bob

> Just use alloc_workqueue() with __WQ_ORDERED and max_active=1. It can
> be wrapped as a new function or macro, but I don't think> create_singlethread_workqueue_noorder() is a good name for it.
> 
>>  extern void destroy_workqueue(struct workqueue_struct *wq);
>>
>>  struct workqueue_attrs *alloc_workqueue_attrs(void);
>> diff --git a/kernel/workqueue.c b/kernel/workqueue.c
>> index 4e01c44..2167013 100644
>> --- a/kernel/workqueue.c
>> +++ b/kernel/workqueue.c
>> @@ -4237,7 +4237,9 @@ struct workqueue_struct *alloc_workqueue(const char *fmt,
>>          * on NUMA.
>>          */
>>         if ((flags & WQ_UNBOUND) && max_active == 1)
>> -               flags |= __WQ_ORDERED;
>> +               /* the caller may don't want __WQ_ORDERED to be set implicitly. */
>> +               if (!(flags & __WQ_ORDERED_DISABLE))
>> +                       flags |= __WQ_ORDERED;
>>
>>         /* see the comment above the definition of WQ_POWER_EFFICIENT */
>>         if ((flags & WQ_POWER_EFFICIENT) && wq_power_efficient)
>> --
>> 2.9.5
>>

Re: [PATCH 1/2] workqueue: don't always set __WQ_ORDERED implicitly

[Lai Jiangshan]

On Mon, Jun 29, 2020 at 8:13 AM Bob Liu <bob.liu@oracle.com> wrote:
>
> On 6/28/20 11:54 PM, Lai Jiangshan wrote:
> > On Thu, Jun 11, 2020 at 6:29 PM Bob Liu <bob.liu@oracle.com> wrote:
> >>
> >> Current code always set 'Unbound && max_active == 1' workqueues to ordered
> >> implicitly, while this may be not an expected behaviour for some use cases.
> >>
> >> E.g some scsi and iscsi workqueues(unbound && max_active = 1) want to be bind
> >> to different cpu so as to get better isolation, but their cpumask can't be
> >> changed because WQ_ORDERED is set implicitly.
> >
> > Hello
> >
> > If I read the code correctly, the reason why their cpumask can't
> > be changed is because __WQ_ORDERED_EXPLICIT, not __WQ_ORDERED.
> >
> >>
> >> This patch adds a flag __WQ_ORDERED_DISABLE and also
> >> create_singlethread_workqueue_noorder() to offer an new option.
> >>
> >> Signed-off-by: Bob Liu <bob.liu@oracle.com>
> >> ---
> >>  include/linux/workqueue.h | 4 ++++
> >>  kernel/workqueue.c        | 4 +++-
> >>  2 files changed, 7 insertions(+), 1 deletion(-)
> >>
> >> diff --git a/include/linux/workqueue.h b/include/linux/workqueue.h
> >> index e48554e..4c86913 100644
> >> --- a/include/linux/workqueue.h
> >> +++ b/include/linux/workqueue.h
> >> @@ -344,6 +344,7 @@ enum {
> >>         __WQ_ORDERED            = 1 << 17, /* internal: workqueue is ordered */
> >>         __WQ_LEGACY             = 1 << 18, /* internal: create*_workqueue() */
> >>         __WQ_ORDERED_EXPLICIT   = 1 << 19, /* internal: alloc_ordered_workqueue() */
> >> +       __WQ_ORDERED_DISABLE    = 1 << 20, /* internal: don't set __WQ_ORDERED implicitly */
> >>
> >>         WQ_MAX_ACTIVE           = 512,    /* I like 512, better ideas? */
> >>         WQ_MAX_UNBOUND_PER_CPU  = 4,      /* 4 * #cpus for unbound wq */
> >> @@ -433,6 +434,9 @@ struct workqueue_struct *alloc_workqueue(const char *fmt,
> >>  #define create_singlethread_workqueue(name)                            \
> >>         alloc_ordered_workqueue("%s", __WQ_LEGACY | WQ_MEM_RECLAIM, name)
> >>
> >> +#define create_singlethread_workqueue_noorder(name)                    \
> >> +       alloc_workqueue("%s", WQ_SYSFS | __WQ_LEGACY | WQ_MEM_RECLAIM | \
> >> +                       WQ_UNBOUND | __WQ_ORDERED_DISABLE, 1, (name))
> >
> > I think using __WQ_ORDERED without __WQ_ORDERED_EXPLICIT is what you
> > need, in which case cpumask is allowed to be changed.
> >
>
> I don't think so, see function workqueue_apply_unbound_cpumask():
>
> wq_unbound_cpumask_store()
>  > workqueue_set_unbound_cpumask()
>    > workqueue_apply_unbound_cpumask() {
>      ...
> 5276                 /* creating multiple pwqs breaks ordering guarantee */
> 5277                 if (wq->flags & __WQ_ORDERED)
> 5278                         continue;
>                           ^^^^
>                           Here will skip apply cpumask if only __WQ_ORDERED is set.

wq_unbound_cpumask_store() is for changing the cpumask of
*all* workqueues. I don't think it can be used to make
scsi and iscsi workqueues bound to different cpu.

apply_workqueue_attrs() is for changing the cpumask of the specific
workqueue, which can change the cpumask of __WQ_ORDERED workqueue
(but without __WQ_ORDERED_EXPLICIT).

>
> 5280                 ctx = apply_wqattrs_prepare(wq, wq->unbound_attrs);
>
>      }
>
> Thanks for your review.
> Bob
>
> > Just use alloc_workqueue() with __WQ_ORDERED and max_active=1. It can
> > be wrapped as a new function or macro, but I don't think> create_singlethread_workqueue_noorder() is a good name for it.
> >
> >>  extern void destroy_workqueue(struct workqueue_struct *wq);
> >>
> >>  struct workqueue_attrs *alloc_workqueue_attrs(void);
> >> diff --git a/kernel/workqueue.c b/kernel/workqueue.c
> >> index 4e01c44..2167013 100644
> >> --- a/kernel/workqueue.c
> >> +++ b/kernel/workqueue.c
> >> @@ -4237,7 +4237,9 @@ struct workqueue_struct *alloc_workqueue(const char *fmt,
> >>          * on NUMA.
> >>          */
> >>         if ((flags & WQ_UNBOUND) && max_active == 1)
> >> -               flags |= __WQ_ORDERED;
> >> +               /* the caller may don't want __WQ_ORDERED to be set implicitly. */
> >> +               if (!(flags & __WQ_ORDERED_DISABLE))
> >> +                       flags |= __WQ_ORDERED;
> >>
> >>         /* see the comment above the definition of WQ_POWER_EFFICIENT */
> >>         if ((flags & WQ_POWER_EFFICIENT) && wq_power_efficient)
> >> --
> >> 2.9.5
> >>
>

Re: [PATCH 1/2] workqueue: don't always set __WQ_ORDERED implicitly

[Bob Liu]

On 6/29/20 8:37 AM, Lai Jiangshan wrote:
> On Mon, Jun 29, 2020 at 8:13 AM Bob Liu <bob.liu@oracle.com> wrote:
>>
>> On 6/28/20 11:54 PM, Lai Jiangshan wrote:
>>> On Thu, Jun 11, 2020 at 6:29 PM Bob Liu <bob.liu@oracle.com> wrote:
>>>>
>>>> Current code always set 'Unbound && max_active == 1' workqueues to ordered
>>>> implicitly, while this may be not an expected behaviour for some use cases.
>>>>
>>>> E.g some scsi and iscsi workqueues(unbound && max_active = 1) want to be bind
>>>> to different cpu so as to get better isolation, but their cpumask can't be
>>>> changed because WQ_ORDERED is set implicitly.
>>>
>>> Hello
>>>
>>> If I read the code correctly, the reason why their cpumask can't
>>> be changed is because __WQ_ORDERED_EXPLICIT, not __WQ_ORDERED.
>>>
>>>>
>>>> This patch adds a flag __WQ_ORDERED_DISABLE and also
>>>> create_singlethread_workqueue_noorder() to offer an new option.
>>>>
>>>> Signed-off-by: Bob Liu <bob.liu@oracle.com>
>>>> ---
>>>>  include/linux/workqueue.h | 4 ++++
>>>>  kernel/workqueue.c        | 4 +++-
>>>>  2 files changed, 7 insertions(+), 1 deletion(-)
>>>>
>>>> diff --git a/include/linux/workqueue.h b/include/linux/workqueue.h
>>>> index e48554e..4c86913 100644
>>>> --- a/include/linux/workqueue.h
>>>> +++ b/include/linux/workqueue.h
>>>> @@ -344,6 +344,7 @@ enum {
>>>>         __WQ_ORDERED            = 1 << 17, /* internal: workqueue is ordered */
>>>>         __WQ_LEGACY             = 1 << 18, /* internal: create*_workqueue() */
>>>>         __WQ_ORDERED_EXPLICIT   = 1 << 19, /* internal: alloc_ordered_workqueue() */
>>>> +       __WQ_ORDERED_DISABLE    = 1 << 20, /* internal: don't set __WQ_ORDERED implicitly */
>>>>
>>>>         WQ_MAX_ACTIVE           = 512,    /* I like 512, better ideas? */
>>>>         WQ_MAX_UNBOUND_PER_CPU  = 4,      /* 4 * #cpus for unbound wq */
>>>> @@ -433,6 +434,9 @@ struct workqueue_struct *alloc_workqueue(const char *fmt,
>>>>  #define create_singlethread_workqueue(name)                            \
>>>>         alloc_ordered_workqueue("%s", __WQ_LEGACY | WQ_MEM_RECLAIM, name)
>>>>
>>>> +#define create_singlethread_workqueue_noorder(name)                    \
>>>> +       alloc_workqueue("%s", WQ_SYSFS | __WQ_LEGACY | WQ_MEM_RECLAIM | \
>>>> +                       WQ_UNBOUND | __WQ_ORDERED_DISABLE, 1, (name))
>>>
>>> I think using __WQ_ORDERED without __WQ_ORDERED_EXPLICIT is what you
>>> need, in which case cpumask is allowed to be changed.
>>>
>>
>> I don't think so, see function workqueue_apply_unbound_cpumask():
>>
>> wq_unbound_cpumask_store()
>>  > workqueue_set_unbound_cpumask()
>>    > workqueue_apply_unbound_cpumask() {
>>      ...
>> 5276                 /* creating multiple pwqs breaks ordering guarantee */
>> 5277                 if (wq->flags & __WQ_ORDERED)
>> 5278                         continue;
>>                           ^^^^
>>                           Here will skip apply cpumask if only __WQ_ORDERED is set.
> 
> wq_unbound_cpumask_store() is for changing the cpumask of
> *all* workqueues. 

Isn't '/sys/bus/workqueue/devices/xxxx/cpumask' using the same function to change cpumask of 
specific workqueue?
Am I missing something..

> I don't think it can be used to make
> scsi and iscsi workqueues bound to different cpu.
> 

The idea is to register scsi/iscsi workqueues with WQ_SYSFS, and then they can be bounded to different
cpu by writing cpu number to "/sys/bus/workqueue/devices/xxxx/cpumask".

> apply_workqueue_attrs() is for changing the cpumask of the specific
> workqueue, which can change the cpumask of __WQ_ORDERED workqueue
> (but without __WQ_ORDERED_EXPLICIT).
> 
>>
>> 5280                 ctx = apply_wqattrs_prepare(wq, wq->unbound_attrs);
>>
>>      }
>>
>> Thanks for your review.
>> Bob
>>
>>> Just use alloc_workqueue() with __WQ_ORDERED and max_active=1. It can
>>> be wrapped as a new function or macro, but I don't think> create_singlethread_workqueue_noorder() is a good name for it.
>>>
>>>>  extern void destroy_workqueue(struct workqueue_struct *wq);
>>>>
>>>>  struct workqueue_attrs *alloc_workqueue_attrs(void);
>>>> diff --git a/kernel/workqueue.c b/kernel/workqueue.c
>>>> index 4e01c44..2167013 100644
>>>> --- a/kernel/workqueue.c
>>>> +++ b/kernel/workqueue.c
>>>> @@ -4237,7 +4237,9 @@ struct workqueue_struct *alloc_workqueue(const char *fmt,
>>>>          * on NUMA.
>>>>          */
>>>>         if ((flags & WQ_UNBOUND) && max_active == 1)
>>>> -               flags |= __WQ_ORDERED;
>>>> +               /* the caller may don't want __WQ_ORDERED to be set implicitly. */
>>>> +               if (!(flags & __WQ_ORDERED_DISABLE))
>>>> +                       flags |= __WQ_ORDERED;
>>>>
>>>>         /* see the comment above the definition of WQ_POWER_EFFICIENT */
>>>>         if ((flags & WQ_POWER_EFFICIENT) && wq_power_efficient)
>>>> --
>>>> 2.9.5
>>>>
>>

Re: [PATCH 1/2] workqueue: don't always set __WQ_ORDERED implicitly

[Bob Liu]

On 6/29/20 8:37 AM, Lai Jiangshan wrote:
> On Mon, Jun 29, 2020 at 8:13 AM Bob Liu <bob.liu@oracle.com> wrote:
>>
>> On 6/28/20 11:54 PM, Lai Jiangshan wrote:
>>> On Thu, Jun 11, 2020 at 6:29 PM Bob Liu <bob.liu@oracle.com> wrote:
>>>>
>>>> Current code always set 'Unbound && max_active == 1' workqueues to ordered
>>>> implicitly, while this may be not an expected behaviour for some use cases.
>>>>
>>>> E.g some scsi and iscsi workqueues(unbound && max_active = 1) want to be bind
>>>> to different cpu so as to get better isolation, but their cpumask can't be
>>>> changed because WQ_ORDERED is set implicitly.
>>>
>>> Hello
>>>
>>> If I read the code correctly, the reason why their cpumask can't
>>> be changed is because __WQ_ORDERED_EXPLICIT, not __WQ_ORDERED.
>>>
>>>>
>>>> This patch adds a flag __WQ_ORDERED_DISABLE and also
>>>> create_singlethread_workqueue_noorder() to offer an new option.
>>>>
>>>> Signed-off-by: Bob Liu <bob.liu@oracle.com>
>>>> ---
>>>>  include/linux/workqueue.h | 4 ++++
>>>>  kernel/workqueue.c        | 4 +++-
>>>>  2 files changed, 7 insertions(+), 1 deletion(-)
>>>>
>>>> diff --git a/include/linux/workqueue.h b/include/linux/workqueue.h
>>>> index e48554e..4c86913 100644
>>>> --- a/include/linux/workqueue.h
>>>> +++ b/include/linux/workqueue.h
>>>> @@ -344,6 +344,7 @@ enum {
>>>>         __WQ_ORDERED            = 1 << 17, /* internal: workqueue is ordered */
>>>>         __WQ_LEGACY             = 1 << 18, /* internal: create*_workqueue() */
>>>>         __WQ_ORDERED_EXPLICIT   = 1 << 19, /* internal: alloc_ordered_workqueue() */
>>>> +       __WQ_ORDERED_DISABLE    = 1 << 20, /* internal: don't set __WQ_ORDERED implicitly */
>>>>
>>>>         WQ_MAX_ACTIVE           = 512,    /* I like 512, better ideas? */
>>>>         WQ_MAX_UNBOUND_PER_CPU  = 4,      /* 4 * #cpus for unbound wq */
>>>> @@ -433,6 +434,9 @@ struct workqueue_struct *alloc_workqueue(const char *fmt,
>>>>  #define create_singlethread_workqueue(name)                            \
>>>>         alloc_ordered_workqueue("%s", __WQ_LEGACY | WQ_MEM_RECLAIM, name)
>>>>
>>>> +#define create_singlethread_workqueue_noorder(name)                    \
>>>> +       alloc_workqueue("%s", WQ_SYSFS | __WQ_LEGACY | WQ_MEM_RECLAIM | \
>>>> +                       WQ_UNBOUND | __WQ_ORDERED_DISABLE, 1, (name))
>>>
>>> I think using __WQ_ORDERED without __WQ_ORDERED_EXPLICIT is what you
>>> need, in which case cpumask is allowed to be changed.
>>>
>>
>> I don't think so, see function workqueue_apply_unbound_cpumask():
>>
>> wq_unbound_cpumask_store()
>>  > workqueue_set_unbound_cpumask()
>>    > workqueue_apply_unbound_cpumask() {
>>      ...
>> 5276                 /* creating multiple pwqs breaks ordering guarantee */
>> 5277                 if (wq->flags & __WQ_ORDERED)
>> 5278                         continue;
>>                           ^^^^
>>                           Here will skip apply cpumask if only __WQ_ORDERED is set.
> 
> wq_unbound_cpumask_store() is for changing the cpumask of
> *all* workqueues. I don't think it can be used to make
> scsi and iscsi workqueues bound to different cpu.
> 
> apply_workqueue_attrs() is for changing the cpumask of the specific
> workqueue, which can change the cpumask of __WQ_ORDERED workqueue
> (but without __WQ_ORDERED_EXPLICIT).
> 

Yes, you are right. I made a mistake.
Sorry for the noise.

Regards,
Bob

>>
>> 5280                 ctx = apply_wqattrs_prepare(wq, wq->unbound_attrs);
>>
>>      }