[PATCH] PCI/MSI: Only disable affinity settings if pre and post vector count is equal to max_vecs and not min_vecs

[PATCH] PCI/MSI: Only disable affinity settings if pre and post vector count is equal to max_vecs and not min_vecs

[Himanshu Madhani]

min_vecs is the minimum amount of vectors needed to operate in MSI-X mode
which may just include the vectors that don't need affinity.

Disabling affinity settings causes the qla2xxx driver scsi_add_host
to fail when blk_mq is enabled as the blk_mq_pci_map_queues expects
affinity masks on each vector.

Fixes: dfef358bd1be ("PCI/MSI: Don't apply affinity if there aren't enough vectors left")
Signed-off-by: Michael Hernandez <michael.hernandez@cavium.com>
Signed-off-by: Himanshu Madhani <himanshu.madhani@cavium.com>
Cc: <stable@vger.kernel.org> # 4.10
---
This patch has been compile on linux-next tree 

commit f4fa27cbc0b6b9f3f41a8fbd771af3abbc480f13
Author: Stephen Rothwell <sfr@canb.auug.org.au>
Date:   Thu Apr 13 15:35:23 2017 +1000

    Add linux-next specific files for 20170413

    Signed-off-by: Stephen Rothwell <sfr@canb.auug.org.au>

Also has compiled on pci tree next branch with following commit

commit 9c5579a1a10f29c1491b38cfe8a16bf2d0a68aa1
Merge: 22e2495b76db b8e82c1bdd28
Author: Bjorn Helgaas <bhelgaas@google.com>
Date:   Wed Apr 12 12:25:43 2017 -0500

    Merge branch 'pci/host-mvebu' into next

    * pci/host-mvebu:
      PCI: mvebu: Avoid changing the SCC bit in the Link Status register

 drivers/pci/msi.c         | 14 ++------------
 include/linux/interrupt.h |  2 +-
 kernel/irq/affinity.c     | 12 +++++++++++-
 3 files changed, 14 insertions(+), 14 deletions(-)

diff --git a/drivers/pci/msi.c b/drivers/pci/msi.c
index ba44fdfda66b..9e1569107cd6 100644
--- a/drivers/pci/msi.c
+++ b/drivers/pci/msi.c
@@ -1058,7 +1058,7 @@ static int __pci_enable_msi_range(struct pci_dev *dev, int minvec, int maxvec,
 
 	for (;;) {
 		if (affd) {
-			nvec = irq_calc_affinity_vectors(nvec, affd);
+			nvec = irq_calc_affinity_vectors(minvec, nvec, affd);
 			if (nvec < minvec)
 				return -ENOSPC;
 		}
@@ -1097,7 +1097,7 @@ static int __pci_enable_msix_range(struct pci_dev *dev,
 
 	for (;;) {
 		if (affd) {
-			nvec = irq_calc_affinity_vectors(nvec, affd);
+			nvec = irq_calc_affinity_vectors(minvec, nvec, affd);
 			if (nvec < minvec)
 				return -ENOSPC;
 		}
@@ -1165,16 +1165,6 @@ int pci_alloc_irq_vectors_affinity(struct pci_dev *dev, unsigned int min_vecs,
 	if (flags & PCI_IRQ_AFFINITY) {
 		if (!affd)
 			affd = &msi_default_affd;
-
-		if (affd->pre_vectors + affd->post_vectors > min_vecs)
-			return -EINVAL;
-
-		/*
-		 * If there aren't any vectors left after applying the pre/post
-		 * vectors don't bother with assigning affinity.
-		 */
-		if (affd->pre_vectors + affd->post_vectors == min_vecs)
-			affd = NULL;
 	} else {
 		if (WARN_ON(affd))
 			affd = NULL;
diff --git a/include/linux/interrupt.h b/include/linux/interrupt.h
index 53144e78a369..b3c569f7405b 100644
--- a/include/linux/interrupt.h
+++ b/include/linux/interrupt.h
@@ -291,7 +291,7 @@ extern int
 irq_set_affinity_notifier(unsigned int irq, struct irq_affinity_notify *notify);
 
 struct cpumask *irq_create_affinity_masks(int nvec, const struct irq_affinity *affd);
-int irq_calc_affinity_vectors(int maxvec, const struct irq_affinity *affd);
+int irq_calc_affinity_vectors(int minvec, int maxvec, const struct irq_affinity *affd);
 
 #else /* CONFIG_SMP */
 
diff --git a/kernel/irq/affinity.c b/kernel/irq/affinity.c
index dc529116f7e6..e545e7a96270 100644
--- a/kernel/irq/affinity.c
+++ b/kernel/irq/affinity.c
@@ -69,6 +69,13 @@ irq_create_affinity_masks(int nvecs, const struct irq_affinity *affd)
 	if (!zalloc_cpumask_var(&nmsk, GFP_KERNEL))
 		return NULL;
 
+	/*
+	 * If there aren't any vectors left after applying the pre/post
+	 * vectors don't bother with assigning affinity.
+	 */
+	if (!affv)
+		return NULL;
+
 	masks = kcalloc(nvecs, sizeof(*masks), GFP_KERNEL);
 	if (!masks)
 		goto out;
@@ -143,12 +150,15 @@ irq_create_affinity_masks(int nvecs, const struct irq_affinity *affd)
  * @maxvec:	The maximum number of vectors available
  * @affd:	Description of the affinity requirements
  */
-int irq_calc_affinity_vectors(int maxvec, const struct irq_affinity *affd)
+int irq_calc_affinity_vectors(int minvec, int maxvec, const struct irq_affinity *affd)
 {
 	int resv = affd->pre_vectors + affd->post_vectors;
 	int vecs = maxvec - resv;
 	int cpus;
 
+	if (resv > minvec)
+		return 0;
+
 	/* Stabilize the cpumasks */
 	get_online_cpus();
 	cpus = cpumask_weight(cpu_online_mask);
-- 
2.12.0

Re: [PATCH] PCI/MSI: Only disable affinity settings if pre and post vector count is equal to max_vecs and not min_vecs

[kbuild test robot]

[-- Attachment #1: Type: text/plain, Size: 3084 bytes --]

Hi Himanshu,

[auto build test ERROR on linus/master]
[also build test ERROR on v4.11-rc7 next-20170413]
[if your patch is applied to the wrong git tree, please drop us a note to help improve the system]

url:    https://github.com/0day-ci/linux/commits/Himanshu-Madhani/PCI-MSI-Only-disable-affinity-settings-if-pre-and-post-vector-count-is-equal-to-max_vecs-and-not-min_vecs/20170418-060342
config: x86_64-randconfig-x003-201716 (attached as .config)
compiler: gcc-6 (Debian 6.2.0-3) 6.2.0 20160901
reproduce:
        # save the attached .config to linux build tree
        make ARCH=x86_64 

All errors (new ones prefixed by >>):

   drivers/pci/msi.c: In function '__pci_enable_msi_range':
   drivers/pci/msi.c:1075:45: warning: passing argument 2 of 'irq_calc_affinity_vectors' makes pointer from integer without a cast [-Wint-conversion]
       nvec = irq_calc_affinity_vectors(minvec, nvec, affd);
                                                ^~~~
   In file included from drivers/pci/msi.c:13:0:
   include/linux/interrupt.h:334:1: note: expected 'const struct irq_affinity *' but argument is of type 'int'
    irq_calc_affinity_vectors(int maxvec, const struct irq_affinity *affd)
    ^~~~~~~~~~~~~~~~~~~~~~~~~
>> drivers/pci/msi.c:1075:11: error: too many arguments to function 'irq_calc_affinity_vectors'
       nvec = irq_calc_affinity_vectors(minvec, nvec, affd);
              ^~~~~~~~~~~~~~~~~~~~~~~~~
   In file included from drivers/pci/msi.c:13:0:
   include/linux/interrupt.h:334:1: note: declared here
    irq_calc_affinity_vectors(int maxvec, const struct irq_affinity *affd)
    ^~~~~~~~~~~~~~~~~~~~~~~~~
   drivers/pci/msi.c: In function '__pci_enable_msix_range':
   drivers/pci/msi.c:1114:45: warning: passing argument 2 of 'irq_calc_affinity_vectors' makes pointer from integer without a cast [-Wint-conversion]
       nvec = irq_calc_affinity_vectors(minvec, nvec, affd);
                                                ^~~~
   In file included from drivers/pci/msi.c:13:0:
   include/linux/interrupt.h:334:1: note: expected 'const struct irq_affinity *' but argument is of type 'int'
    irq_calc_affinity_vectors(int maxvec, const struct irq_affinity *affd)
    ^~~~~~~~~~~~~~~~~~~~~~~~~
   drivers/pci/msi.c:1114:11: error: too many arguments to function 'irq_calc_affinity_vectors'
       nvec = irq_calc_affinity_vectors(minvec, nvec, affd);
              ^~~~~~~~~~~~~~~~~~~~~~~~~
   In file included from drivers/pci/msi.c:13:0:
   include/linux/interrupt.h:334:1: note: declared here
    irq_calc_affinity_vectors(int maxvec, const struct irq_affinity *affd)
    ^~~~~~~~~~~~~~~~~~~~~~~~~

vim +/irq_calc_affinity_vectors +1075 drivers/pci/msi.c

  1069	
  1070		if (nvec > maxvec)
  1071			nvec = maxvec;
  1072	
  1073		for (;;) {
  1074			if (affd) {
> 1075				nvec = irq_calc_affinity_vectors(minvec, nvec, affd);
  1076				if (nvec < minvec)
  1077					return -ENOSPC;
  1078			}

---
0-DAY kernel test infrastructure                Open Source Technology Center
https://lists.01.org/pipermail/kbuild-all                   Intel Corporation

[-- Attachment #2: .config.gz --]
[-- Type: application/gzip, Size: 24054 bytes --]

[PATCH] PCI/MSI: Only disable affinity settings if pre and post vector count is equal to max_vecs and not min_vecs

[Himanshu Madhani]

min_vecs is the minimum amount of vectors needed to operate in MSI-X mode
which may just include the vectors that don't need affinity.

Disabling affinity settings causes the qla2xxx driver scsi_add_host
to fail when blk_mq is enabled as the blk_mq_pci_map_queues expects
affinity masks on each vector.

Fixes: dfef358bd1be ("PCI/MSI: Don't apply affinity if there aren't enough vectors left")
Signed-off-by: Michael Hernandez <michael.hernandez@cavium.com>
Signed-off-by: Himanshu Madhani <himanshu.madhani@cavium.com>
Cc: <stable@vger.kernel.org> # 4.10
---
v4 --> v5
o Fixed Zero day kernel build failure.

v3 --> v4
o Moved pre/post reserved vector range checks to affinity.c as per
  comments from Bjorn Helgaas.

v2 --> v3
o fixed code as per review comments.

v1 --> v2
o Moved the check from pci_alloc_irq_vectors_affinity() to
  __pci_enable_{msi|msix}_range()


 drivers/pci/msi.c         | 14 ++------------
 include/linux/interrupt.h |  4 ++--
 kernel/irq/affinity.c     | 12 +++++++++++-
 3 files changed, 15 insertions(+), 15 deletions(-)

diff --git a/drivers/pci/msi.c b/drivers/pci/msi.c
index d571bc330686..8adf22f41cf9 100644
--- a/drivers/pci/msi.c
+++ b/drivers/pci/msi.c
@@ -1072,7 +1072,7 @@ static int __pci_enable_msi_range(struct pci_dev *dev, int minvec, int maxvec,
 
 	for (;;) {
 		if (affd) {
-			nvec = irq_calc_affinity_vectors(nvec, affd);
+			nvec = irq_calc_affinity_vectors(minvec, nvec, affd);
 			if (nvec < minvec)
 				return -ENOSPC;
 		}
@@ -1111,7 +1111,7 @@ static int __pci_enable_msix_range(struct pci_dev *dev,
 
 	for (;;) {
 		if (affd) {
-			nvec = irq_calc_affinity_vectors(nvec, affd);
+			nvec = irq_calc_affinity_vectors(minvec, nvec, affd);
 			if (nvec < minvec)
 				return -ENOSPC;
 		}
@@ -1179,16 +1179,6 @@ int pci_alloc_irq_vectors_affinity(struct pci_dev *dev, unsigned int min_vecs,
 	if (flags & PCI_IRQ_AFFINITY) {
 		if (!affd)
 			affd = &msi_default_affd;
-
-		if (affd->pre_vectors + affd->post_vectors > min_vecs)
-			return -EINVAL;
-
-		/*
-		 * If there aren't any vectors left after applying the pre/post
-		 * vectors don't bother with assigning affinity.
-		 */
-		if (affd->pre_vectors + affd->post_vectors == min_vecs)
-			affd = NULL;
 	} else {
 		if (WARN_ON(affd))
 			affd = NULL;
diff --git a/include/linux/interrupt.h b/include/linux/interrupt.h
index 53144e78a369..63b246dd8258 100644
--- a/include/linux/interrupt.h
+++ b/include/linux/interrupt.h
@@ -291,7 +291,7 @@ extern int
 irq_set_affinity_notifier(unsigned int irq, struct irq_affinity_notify *notify);
 
 struct cpumask *irq_create_affinity_masks(int nvec, const struct irq_affinity *affd);
-int irq_calc_affinity_vectors(int maxvec, const struct irq_affinity *affd);
+int irq_calc_affinity_vectors(int minvec, int maxvec, const struct irq_affinity *affd);
 
 #else /* CONFIG_SMP */
 
@@ -331,7 +331,7 @@ irq_create_affinity_masks(int nvec, const struct irq_affinity *affd)
 }
 
 static inline int
-irq_calc_affinity_vectors(int maxvec, const struct irq_affinity *affd)
+irq_calc_affinity_vectors(int minvec, int maxvec, const struct irq_affinity *affd)
 {
 	return maxvec;
 }
diff --git a/kernel/irq/affinity.c b/kernel/irq/affinity.c
index d052947fe785..59645fa66385 100644
--- a/kernel/irq/affinity.c
+++ b/kernel/irq/affinity.c
@@ -69,6 +69,13 @@ irq_create_affinity_masks(int nvecs, const struct irq_affinity *affd)
 	if (!zalloc_cpumask_var(&nmsk, GFP_KERNEL))
 		return NULL;
 
+	/*
+	 * If there aren't any vectors left after applying the pre/post
+	 * vectors don't bother with assigning affinity.
+	 */
+	if (!affv)
+		return NULL;
+
 	masks = kcalloc(nvecs, sizeof(*masks), GFP_KERNEL);
 	if (!masks)
 		goto out;
@@ -143,12 +150,15 @@ irq_create_affinity_masks(int nvecs, const struct irq_affinity *affd)
  * @maxvec:	The maximum number of vectors available
  * @affd:	Description of the affinity requirements
  */
-int irq_calc_affinity_vectors(int maxvec, const struct irq_affinity *affd)
+int irq_calc_affinity_vectors(int minvec, int maxvec, const struct irq_affinity *affd)
 {
 	int resv = affd->pre_vectors + affd->post_vectors;
 	int vecs = maxvec - resv;
 	int cpus;
 
+	if (resv > minvec)
+		return 0;
+
 	/* Stabilize the cpumasks */
 	get_online_cpus();
 	cpus = cpumask_weight(cpu_online_mask);
-- 
2.12.0

Re: [PATCH] PCI/MSI: Only disable affinity settings if pre and post vector count is equal to max_vecs and not min_vecs

[Bjorn Helgaas]

Your summary is "Only disable affinity settings if pre and post vector
count is equal to max_vecs and not min_vecs", but the patch doesn't
mention max_vecs anywhere.  There must be a clearer way to word this.

On Mon, Apr 17, 2017 at 06:19:25PM -0700, Himanshu Madhani wrote:
> min_vecs is the minimum amount of vectors needed to operate in MSI-X mode
> which may just include the vectors that don't need affinity.
> 
> Disabling affinity settings causes the qla2xxx driver scsi_add_host
> to fail when blk_mq is enabled as the blk_mq_pci_map_queues expects
> affinity masks on each vector.
> 
> Fixes: dfef358bd1be ("PCI/MSI: Don't apply affinity if there aren't enough vectors left")
> Signed-off-by: Michael Hernandez <michael.hernandez@cavium.com>
> Signed-off-by: Himanshu Madhani <himanshu.madhani@cavium.com>
> Cc: <stable@vger.kernel.org> # 4.10
> ---
> v4 --> v5
> o Fixed Zero day kernel build failure.
> 
> v3 --> v4
> o Moved pre/post reserved vector range checks to affinity.c as per
>   comments from Bjorn Helgaas.
> 
> v2 --> v3
> o fixed code as per review comments.
> 
> v1 --> v2
> o Moved the check from pci_alloc_irq_vectors_affinity() to
>   __pci_enable_{msi|msix}_range()
> 
> 
>  drivers/pci/msi.c         | 14 ++------------
>  include/linux/interrupt.h |  4 ++--
>  kernel/irq/affinity.c     | 12 +++++++++++-
>  3 files changed, 15 insertions(+), 15 deletions(-)
> 
> diff --git a/drivers/pci/msi.c b/drivers/pci/msi.c
> index d571bc330686..8adf22f41cf9 100644
> --- a/drivers/pci/msi.c
> +++ b/drivers/pci/msi.c
> @@ -1072,7 +1072,7 @@ static int __pci_enable_msi_range(struct pci_dev *dev, int minvec, int maxvec,
>  
>  	for (;;) {
>  		if (affd) {
> -			nvec = irq_calc_affinity_vectors(nvec, affd);
> +			nvec = irq_calc_affinity_vectors(minvec, nvec, affd);
>  			if (nvec < minvec)
>  				return -ENOSPC;
>  		}
> @@ -1111,7 +1111,7 @@ static int __pci_enable_msix_range(struct pci_dev *dev,
>  
>  	for (;;) {
>  		if (affd) {
> -			nvec = irq_calc_affinity_vectors(nvec, affd);
> +			nvec = irq_calc_affinity_vectors(minvec, nvec, affd);
>  			if (nvec < minvec)
>  				return -ENOSPC;
>  		}
> @@ -1179,16 +1179,6 @@ int pci_alloc_irq_vectors_affinity(struct pci_dev *dev, unsigned int min_vecs,
>  	if (flags & PCI_IRQ_AFFINITY) {
>  		if (!affd)
>  			affd = &msi_default_affd;
> -
> -		if (affd->pre_vectors + affd->post_vectors > min_vecs)
> -			return -EINVAL;
> -
> -		/*
> -		 * If there aren't any vectors left after applying the pre/post
> -		 * vectors don't bother with assigning affinity.
> -		 */
> -		if (affd->pre_vectors + affd->post_vectors == min_vecs)
> -			affd = NULL;
>  	} else {
>  		if (WARN_ON(affd))
>  			affd = NULL;
> diff --git a/include/linux/interrupt.h b/include/linux/interrupt.h
> index 53144e78a369..63b246dd8258 100644
> --- a/include/linux/interrupt.h
> +++ b/include/linux/interrupt.h
> @@ -291,7 +291,7 @@ extern int
>  irq_set_affinity_notifier(unsigned int irq, struct irq_affinity_notify *notify);
>  
>  struct cpumask *irq_create_affinity_masks(int nvec, const struct irq_affinity *affd);
> -int irq_calc_affinity_vectors(int maxvec, const struct irq_affinity *affd);
> +int irq_calc_affinity_vectors(int minvec, int maxvec, const struct irq_affinity *affd);
>  
>  #else /* CONFIG_SMP */
>  
> @@ -331,7 +331,7 @@ irq_create_affinity_masks(int nvec, const struct irq_affinity *affd)
>  }
>  
>  static inline int
> -irq_calc_affinity_vectors(int maxvec, const struct irq_affinity *affd)
> +irq_calc_affinity_vectors(int minvec, int maxvec, const struct irq_affinity *affd)
>  {
>  	return maxvec;
>  }
> diff --git a/kernel/irq/affinity.c b/kernel/irq/affinity.c
> index d052947fe785..59645fa66385 100644
> --- a/kernel/irq/affinity.c
> +++ b/kernel/irq/affinity.c
> @@ -69,6 +69,13 @@ irq_create_affinity_masks(int nvecs, const struct irq_affinity *affd)
>  	if (!zalloc_cpumask_var(&nmsk, GFP_KERNEL))
>  		return NULL;
>  
> +	/*
> +	 * If there aren't any vectors left after applying the pre/post
> +	 * vectors don't bother with assigning affinity.
> +	 */
> +	if (!affv)
> +		return NULL;

This looks like it might require cleanup, e.g., branching to "out".

But the test of "affv" doesn't depend on the zalloc_cpu_mask_var(), so
you could just move this test above the call to zalloc_cpu_mask_var(),
and then you wouldn't need any cleanup.

> +
>  	masks = kcalloc(nvecs, sizeof(*masks), GFP_KERNEL);
>  	if (!masks)
>  		goto out;
> @@ -143,12 +150,15 @@ irq_create_affinity_masks(int nvecs, const struct irq_affinity *affd)
>   * @maxvec:	The maximum number of vectors available
>   * @affd:	Description of the affinity requirements
>   */
> -int irq_calc_affinity_vectors(int maxvec, const struct irq_affinity *affd)
> +int irq_calc_affinity_vectors(int minvec, int maxvec, const struct irq_affinity *affd)

This function has a kernel-doc comment above it, and you're adding a
parameter, so please update the kernel-doc to match.

>  {
>  	int resv = affd->pre_vectors + affd->post_vectors;
>  	int vecs = maxvec - resv;
>  	int cpus;
>  
> +	if (resv > minvec)
> +		return 0;
> +
>  	/* Stabilize the cpumasks */
>  	get_online_cpus();
>  	cpus = cpumask_weight(cpu_online_mask);
> -- 
> 2.12.0
> 

Re: [PATCH] PCI/MSI: Only disable affinity settings if pre and post vector count is equal to max_vecs and not min_vecs

[Himanshu Madhani]

On Wed, 17 May 2017, 3:28pm, Bjorn Helgaas wrote:

> Your summary is "Only disable affinity settings if pre and post vector
> count is equal to max_vecs and not min_vecs", but the patch doesn't
> mention max_vecs anywhere.  There must be a clearer way to word this.
>
Agree. will update the patch summary 
 
> On Mon, Apr 17, 2017 at 06:19:25PM -0700, Himanshu Madhani wrote:
> > min_vecs is the minimum amount of vectors needed to operate in MSI-X mode
> > which may just include the vectors that don't need affinity.
> > 
> > Disabling affinity settings causes the qla2xxx driver scsi_add_host
> > to fail when blk_mq is enabled as the blk_mq_pci_map_queues expects
> > affinity masks on each vector.
> > 
> > Fixes: dfef358bd1be ("PCI/MSI: Don't apply affinity if there aren't enough vectors left")
> > Signed-off-by: Michael Hernandez <michael.hernandez@cavium.com>
> > Signed-off-by: Himanshu Madhani <himanshu.madhani@cavium.com>
> > Cc: <stable@vger.kernel.org> # 4.10
> > ---
> > v4 --> v5
> > o Fixed Zero day kernel build failure.
> > 
> > v3 --> v4
> > o Moved pre/post reserved vector range checks to affinity.c as per
> >   comments from Bjorn Helgaas.
> > 
> > v2 --> v3
> > o fixed code as per review comments.
> > 
> > v1 --> v2
> > o Moved the check from pci_alloc_irq_vectors_affinity() to
> >   __pci_enable_{msi|msix}_range()
> > 
> > 
> >  drivers/pci/msi.c         | 14 ++------------
> >  include/linux/interrupt.h |  4 ++--
> >  kernel/irq/affinity.c     | 12 +++++++++++-
> >  3 files changed, 15 insertions(+), 15 deletions(-)
> > 
> > diff --git a/drivers/pci/msi.c b/drivers/pci/msi.c
> > index d571bc330686..8adf22f41cf9 100644
> > --- a/drivers/pci/msi.c
> > +++ b/drivers/pci/msi.c
> > @@ -1072,7 +1072,7 @@ static int __pci_enable_msi_range(struct pci_dev *dev, int minvec, int maxvec,
> >  
> >  	for (;;) {
> >  		if (affd) {
> > -			nvec = irq_calc_affinity_vectors(nvec, affd);
> > +			nvec = irq_calc_affinity_vectors(minvec, nvec, affd);
> >  			if (nvec < minvec)
> >  				return -ENOSPC;
> >  		}
> > @@ -1111,7 +1111,7 @@ static int __pci_enable_msix_range(struct pci_dev *dev,
> >  
> >  	for (;;) {
> >  		if (affd) {
> > -			nvec = irq_calc_affinity_vectors(nvec, affd);
> > +			nvec = irq_calc_affinity_vectors(minvec, nvec, affd);
> >  			if (nvec < minvec)
> >  				return -ENOSPC;
> >  		}
> > @@ -1179,16 +1179,6 @@ int pci_alloc_irq_vectors_affinity(struct pci_dev *dev, unsigned int min_vecs,
> >  	if (flags & PCI_IRQ_AFFINITY) {
> >  		if (!affd)
> >  			affd = &msi_default_affd;
> > -
> > -		if (affd->pre_vectors + affd->post_vectors > min_vecs)
> > -			return -EINVAL;
> > -
> > -		/*
> > -		 * If there aren't any vectors left after applying the pre/post
> > -		 * vectors don't bother with assigning affinity.
> > -		 */
> > -		if (affd->pre_vectors + affd->post_vectors == min_vecs)
> > -			affd = NULL;
> >  	} else {
> >  		if (WARN_ON(affd))
> >  			affd = NULL;
> > diff --git a/include/linux/interrupt.h b/include/linux/interrupt.h
> > index 53144e78a369..63b246dd8258 100644
> > --- a/include/linux/interrupt.h
> > +++ b/include/linux/interrupt.h
> > @@ -291,7 +291,7 @@ extern int
> >  irq_set_affinity_notifier(unsigned int irq, struct irq_affinity_notify *notify);
> >  
> >  struct cpumask *irq_create_affinity_masks(int nvec, const struct irq_affinity *affd);
> > -int irq_calc_affinity_vectors(int maxvec, const struct irq_affinity *affd);
> > +int irq_calc_affinity_vectors(int minvec, int maxvec, const struct irq_affinity *affd);
> >  
> >  #else /* CONFIG_SMP */
> >  
> > @@ -331,7 +331,7 @@ irq_create_affinity_masks(int nvec, const struct irq_affinity *affd)
> >  }
> >  
> >  static inline int
> > -irq_calc_affinity_vectors(int maxvec, const struct irq_affinity *affd)
> > +irq_calc_affinity_vectors(int minvec, int maxvec, const struct irq_affinity *affd)
> >  {
> >  	return maxvec;
> >  }
> > diff --git a/kernel/irq/affinity.c b/kernel/irq/affinity.c
> > index d052947fe785..59645fa66385 100644
> > --- a/kernel/irq/affinity.c
> > +++ b/kernel/irq/affinity.c
> > @@ -69,6 +69,13 @@ irq_create_affinity_masks(int nvecs, const struct irq_affinity *affd)
> >  	if (!zalloc_cpumask_var(&nmsk, GFP_KERNEL))
> >  		return NULL;
> >  
> > +	/*
> > +	 * If there aren't any vectors left after applying the pre/post
> > +	 * vectors don't bother with assigning affinity.
> > +	 */
> > +	if (!affv)
> > +		return NULL;
> 
> This looks like it might require cleanup, e.g., branching to "out".
> 
> But the test of "affv" doesn't depend on the zalloc_cpu_mask_var(), so
> you could just move this test above the call to zalloc_cpu_mask_var(),
> and then you wouldn't need any cleanup.
> 

Make sense. Will update patch.

> > +
> >  	masks = kcalloc(nvecs, sizeof(*masks), GFP_KERNEL);
> >  	if (!masks)
> >  		goto out;
> > @@ -143,12 +150,15 @@ irq_create_affinity_masks(int nvecs, const struct irq_affinity *affd)
> >   * @maxvec:	The maximum number of vectors available
> >   * @affd:	Description of the affinity requirements
> >   */
> > -int irq_calc_affinity_vectors(int maxvec, const struct irq_affinity *affd)
> > +int irq_calc_affinity_vectors(int minvec, int maxvec, const struct irq_affinity *affd)
> 
> This function has a kernel-doc comment above it, and you're adding a
> parameter, so please update the kernel-doc to match.
> 
Sure. will add kernel-doc 

> >  {
> >  	int resv = affd->pre_vectors + affd->post_vectors;
> >  	int vecs = maxvec - resv;
> >  	int cpus;
> >  
> > +	if (resv > minvec)
> > +		return 0;
> > +
> >  	/* Stabilize the cpumasks */
> >  	get_online_cpus();
> >  	cpus = cpumask_weight(cpu_online_mask);
> > -- 
> > 2.12.0
> > 
> 

Will send revised patch after suggested updates.

Thanks,
Himanshu

[PATCH] PCI/MSI: Only disable affinity settings if pre and post vector count is equal to max_vecs and not min_vecs

[Himanshu Madhani]

From: Michael Hernandez <michael.hernandez@cavium.com>

min_vecs is the minimum amount of vectors needed to operate in MSI-X mode
which may just include the vectors that don't need affinity.

Disabling affinity settings causes the qla2xxx driver scsi_add_host
to fail when blk_mq is enabled as the blk_mq_pci_map_queues expects
affinity masks on each vector.

v2 --> v3
o fixed code as per review comments.

v1 --> v2

o Moved the check from pci_alloc_irq_vectors_affinity() to
  __pci_enable_{msi|msix}_range()

Fixes: dfef358 ("PCI/MSI: Don't apply affinity if there aren't enough vectors left")
Signed-off-by: Michael Hernandez <michael.hernandez@cavium.com>
Signed-off-by: Himanshu Madhani <himanshu.madhani@cavium.com>
Cc: Christoph Hellwig <hch@lst.de>
Cc: stable@vger.kernel.org
---
 drivers/pci/msi.c |   25 ++++++++++++++++++-------
 1 files changed, 18 insertions(+), 7 deletions(-)

diff --git a/drivers/pci/msi.c b/drivers/pci/msi.c
index 7f73bac..46c0cdd 100644
--- a/drivers/pci/msi.c
+++ b/drivers/pci/msi.c
@@ -1091,6 +1091,15 @@ static int __pci_enable_msi_range(struct pci_dev *dev, int minvec, int maxvec,
 
 	for (;;) {
 		if (affd) {
+			/*
+			 * If there aren't any vectors left after applying the
+			 * pre/post vectors don't bother with assigning
+			 * affinity.
+			 */
+			if (affd->pre_vectors + affd->post_vectors == nvec)
+				affd = NULL;
+		}
+		if (affd) {
 			nvec = irq_calc_affinity_vectors(nvec, affd);
 			if (nvec < minvec)
 				return -ENOSPC;
@@ -1138,6 +1147,15 @@ static int __pci_enable_msix_range(struct pci_dev *dev,
 
 	for (;;) {
 		if (affd) {
+			/*
+			 * If there aren't any vectors left after applying the
+			 * pre/post vectors don't bother with assigning
+			 * affinity.
+			 */
+			if (affd->pre_vectors + affd->post_vectors == nvec)
+				affd = NULL;
+		}
+		if (affd) {
 			nvec = irq_calc_affinity_vectors(nvec, affd);
 			if (nvec < minvec)
 				return -ENOSPC;
@@ -1209,13 +1227,6 @@ int pci_alloc_irq_vectors_affinity(struct pci_dev *dev, unsigned int min_vecs,
 
 		if (affd->pre_vectors + affd->post_vectors > min_vecs)
 			return -EINVAL;
-
-		/*
-		 * If there aren't any vectors left after applying the pre/post
-		 * vectors don't bother with assigning affinity.
-		 */
-		if (affd->pre_vectors + affd->post_vectors == min_vecs)
-			affd = NULL;
 	} else {
 		if (WARN_ON(affd))
 			affd = NULL;
-- 
1.7.1

Re: [PATCH] PCI/MSI: Only disable affinity settings if pre and post vector count is equal to max_vecs and not min_vecs

[Christoph Hellwig]

Looks good,

Reviewed-by: Christoph Hellwig <hch@lst.de>

Re: [PATCH] PCI/MSI: Only disable affinity settings if pre and post vector count is equal to max_vecs and not min_vecs

[Madhani, Himanshu]

SGkgQmpvcm4sIA0KCQ0KQ2FuIHdlIGdldCB0aGlzIHBhdGNoIGluY2x1ZGVkIGluIG5leHQgYnJh
bmNoLiBUaGlzIHBhdGNoIGhhcyBiZWVuIHJldmlld2VkIGJ5IENocmlzdG9waC4gDQoNClRoYW5r
cywNCkhpbWFuc2h1DQoNCk9uIDMvMS8xNywgMzo0OSBQTSwgIkNocmlzdG9waCBIZWxsd2lnIiA8
aGNoQGxzdC5kZT4gd3JvdGU6DQoNCiAgICBMb29rcyBnb29kLA0KICAgIA0KICAgIFJldmlld2Vk
LWJ5OiBDaHJpc3RvcGggSGVsbHdpZyA8aGNoQGxzdC5kZT4NCiAgICANCg0K

Re: [PATCH] PCI/MSI: Only disable affinity settings if pre and post vector count is equal to max_vecs and not min_vecs

[Bjorn Helgaas]

On Tue, Mar 07, 2017 at 07:00:20PM +0000, Madhani, Himanshu wrote:
> Hi Bjorn, 
> 	
> Can we get this patch included in next branch. This patch has been reviewed by Christoph. 

No worries, it's still in patchwork
(https://patchwork.ozlabs.org/project/linux-pci/list/) so I'll get to it
soon.

> On 3/1/17, 3:49 PM, "Christoph Hellwig" <hch@lst.de> wrote:
> 
>     Looks good,
>     
>     Reviewed-by: Christoph Hellwig <hch@lst.de>
>     
> 

Re: [PATCH] PCI/MSI: Only disable affinity settings if pre and post vector count is equal to max_vecs and not min_vecs

[Bjorn Helgaas]

On Wed, Mar 01, 2017 at 03:31:58PM -0800, Himanshu Madhani wrote:
> From: Michael Hernandez <michael.hernandez@cavium.com>
> 
> min_vecs is the minimum amount of vectors needed to operate in MSI-X mode
> which may just include the vectors that don't need affinity.
> 
> Disabling affinity settings causes the qla2xxx driver scsi_add_host
> to fail when blk_mq is enabled as the blk_mq_pci_map_queues expects
> affinity masks on each vector.
> 
> v2 --> v3
> o fixed code as per review comments.
> 
> v1 --> v2
> 
> o Moved the check from pci_alloc_irq_vectors_affinity() to
>   __pci_enable_{msi|msix}_range()
> 
> Fixes: dfef358 ("PCI/MSI: Don't apply affinity if there aren't enough vectors left")
> Signed-off-by: Michael Hernandez <michael.hernandez@cavium.com>
> Signed-off-by: Himanshu Madhani <himanshu.madhani@cavium.com>
> Cc: Christoph Hellwig <hch@lst.de>
> Cc: stable@vger.kernel.org
> ---
>  drivers/pci/msi.c |   25 ++++++++++++++++++-------
>  1 files changed, 18 insertions(+), 7 deletions(-)
> 
> diff --git a/drivers/pci/msi.c b/drivers/pci/msi.c
> index 7f73bac..46c0cdd 100644
> --- a/drivers/pci/msi.c
> +++ b/drivers/pci/msi.c
> @@ -1091,6 +1091,15 @@ static int __pci_enable_msi_range(struct pci_dev *dev, int minvec, int maxvec,
>  
>  	for (;;) {
>  		if (affd) {
> +			/*
> +			 * If there aren't any vectors left after applying the
> +			 * pre/post vectors don't bother with assigning
> +			 * affinity.
> +			 */
> +			if (affd->pre_vectors + affd->post_vectors == nvec)
> +				affd = NULL;
> +		}

I don't really like this because affd->pre_vectors and
affd->post_vectors are not PCI MSI concepts.  I think they really
belong in irq/affinity.c, e.g., maybe this could be checked in
irq_create_affinity_masks().

If that could be done, we wouldn't have to duplicate the checks here
for both MSI and MSI-X.

I raised a similar question earlier:
http://lkml.kernel.org/r/20170202173659.GD21267@bhelgaas-glaptop.roam.corp.google.com

> +		if (affd) {
>  			nvec = irq_calc_affinity_vectors(nvec, affd);
>  			if (nvec < minvec)
>  				return -ENOSPC;
> @@ -1138,6 +1147,15 @@ static int __pci_enable_msix_range(struct pci_dev *dev,
>  
>  	for (;;) {
>  		if (affd) {
> +			/*
> +			 * If there aren't any vectors left after applying the
> +			 * pre/post vectors don't bother with assigning
> +			 * affinity.
> +			 */
> +			if (affd->pre_vectors + affd->post_vectors == nvec)
> +				affd = NULL;
> +		}
> +		if (affd) {
>  			nvec = irq_calc_affinity_vectors(nvec, affd);
>  			if (nvec < minvec)
>  				return -ENOSPC;
> @@ -1209,13 +1227,6 @@ int pci_alloc_irq_vectors_affinity(struct pci_dev *dev, unsigned int min_vecs,
>  
>  		if (affd->pre_vectors + affd->post_vectors > min_vecs)
>  			return -EINVAL;
> -
> -		/*
> -		 * If there aren't any vectors left after applying the pre/post
> -		 * vectors don't bother with assigning affinity.
> -		 */
> -		if (affd->pre_vectors + affd->post_vectors == min_vecs)
> -			affd = NULL;
>  	} else {
>  		if (WARN_ON(affd))
>  			affd = NULL;
> -- 
> 1.7.1
> 

Re: [PATCH] PCI/MSI: Only disable affinity settings if pre and post vector count is equal to max_vecs and not min_vecs

[Bjorn Helgaas]

On Thu, Mar 23, 2017 at 12:29:28PM -0500, Bjorn Helgaas wrote:
> On Wed, Mar 01, 2017 at 03:31:58PM -0800, Himanshu Madhani wrote:
> > From: Michael Hernandez <michael.hernandez@cavium.com>
> > 
> > min_vecs is the minimum amount of vectors needed to operate in MSI-X mode
> > which may just include the vectors that don't need affinity.
> > 
> > Disabling affinity settings causes the qla2xxx driver scsi_add_host
> > to fail when blk_mq is enabled as the blk_mq_pci_map_queues expects
> > affinity masks on each vector.
> > 
> > v2 --> v3
> > o fixed code as per review comments.
> > 
> > v1 --> v2
> > 
> > o Moved the check from pci_alloc_irq_vectors_affinity() to
> >   __pci_enable_{msi|msix}_range()
> > 
> > Fixes: dfef358 ("PCI/MSI: Don't apply affinity if there aren't enough vectors left")
> > Signed-off-by: Michael Hernandez <michael.hernandez@cavium.com>
> > Signed-off-by: Himanshu Madhani <himanshu.madhani@cavium.com>
> > Cc: Christoph Hellwig <hch@lst.de>
> > Cc: stable@vger.kernel.org
> > ---
> >  drivers/pci/msi.c |   25 ++++++++++++++++++-------
> >  1 files changed, 18 insertions(+), 7 deletions(-)
> > 
> > diff --git a/drivers/pci/msi.c b/drivers/pci/msi.c
> > index 7f73bac..46c0cdd 100644
> > --- a/drivers/pci/msi.c
> > +++ b/drivers/pci/msi.c
> > @@ -1091,6 +1091,15 @@ static int __pci_enable_msi_range(struct pci_dev *dev, int minvec, int maxvec,
> >  
> >  	for (;;) {
> >  		if (affd) {
> > +			/*
> > +			 * If there aren't any vectors left after applying the
> > +			 * pre/post vectors don't bother with assigning
> > +			 * affinity.
> > +			 */
> > +			if (affd->pre_vectors + affd->post_vectors == nvec)
> > +				affd = NULL;
> > +		}
> 
> I don't really like this because affd->pre_vectors and
> affd->post_vectors are not PCI MSI concepts.  I think they really
> belong in irq/affinity.c, e.g., maybe this could be checked in
> irq_create_affinity_masks().
> 
> If that could be done, we wouldn't have to duplicate the checks here
> for both MSI and MSI-X.
> 
> I raised a similar question earlier:
> http://lkml.kernel.org/r/20170202173659.GD21267@bhelgaas-glaptop.roam.corp.google.com

I want to make progress on this, but that means a little conversation
here.  Maybe we want to merge this patch as-is, but I'd like to hear
the reasons why it wouldn't really work to move the checks to
irq/affinity.c.

For now I'll mark it as "changes requested" in patchwork, which means
it will fall off my to-do list until it's reposted.

> > +		if (affd) {
> >  			nvec = irq_calc_affinity_vectors(nvec, affd);
> >  			if (nvec < minvec)
> >  				return -ENOSPC;
> > @@ -1138,6 +1147,15 @@ static int __pci_enable_msix_range(struct pci_dev *dev,
> >  
> >  	for (;;) {
> >  		if (affd) {
> > +			/*
> > +			 * If there aren't any vectors left after applying the
> > +			 * pre/post vectors don't bother with assigning
> > +			 * affinity.
> > +			 */
> > +			if (affd->pre_vectors + affd->post_vectors == nvec)
> > +				affd = NULL;
> > +		}
> > +		if (affd) {
> >  			nvec = irq_calc_affinity_vectors(nvec, affd);
> >  			if (nvec < minvec)
> >  				return -ENOSPC;
> > @@ -1209,13 +1227,6 @@ int pci_alloc_irq_vectors_affinity(struct pci_dev *dev, unsigned int min_vecs,
> >  
> >  		if (affd->pre_vectors + affd->post_vectors > min_vecs)
> >  			return -EINVAL;
> > -
> > -		/*
> > -		 * If there aren't any vectors left after applying the pre/post
> > -		 * vectors don't bother with assigning affinity.
> > -		 */
> > -		if (affd->pre_vectors + affd->post_vectors == min_vecs)
> > -			affd = NULL;
> >  	} else {
> >  		if (WARN_ON(affd))
> >  			affd = NULL;
> > -- 
> > 1.7.1
> > 

Re: [PATCH] PCI/MSI: Only disable affinity settings if pre and post vector count is equal to max_vecs and not min_vecs

[Himanshu Madhani]

Hi Bjorn,

On Thu, 30 Mar 2017, 4:21pm, Bjorn Helgaas wrote:

> On Thu, Mar 23, 2017 at 12:29:28PM -0500, Bjorn Helgaas wrote:
> > On Wed, Mar 01, 2017 at 03:31:58PM -0800, Himanshu Madhani wrote:
> > > From: Michael Hernandez <michael.hernandez@cavium.com>
> > > 
> > > min_vecs is the minimum amount of vectors needed to operate in MSI-X mode
> > > which may just include the vectors that don't need affinity.
> > > 
> > > Disabling affinity settings causes the qla2xxx driver scsi_add_host
> > > to fail when blk_mq is enabled as the blk_mq_pci_map_queues expects
> > > affinity masks on each vector.
> > > 
> > > v2 --> v3
> > > o fixed code as per review comments.
> > > 
> > > v1 --> v2
> > > 
> > > o Moved the check from pci_alloc_irq_vectors_affinity() to
> > >   __pci_enable_{msi|msix}_range()
> > > 
> > > Fixes: dfef358 ("PCI/MSI: Don't apply affinity if there aren't enough vectors left")
> > > Signed-off-by: Michael Hernandez <michael.hernandez@cavium.com>
> > > Signed-off-by: Himanshu Madhani <himanshu.madhani@cavium.com>
> > > Cc: Christoph Hellwig <hch@lst.de>
> > > Cc: stable@vger.kernel.org
> > > ---
> > >  drivers/pci/msi.c |   25 ++++++++++++++++++-------
> > >  1 files changed, 18 insertions(+), 7 deletions(-)
> > > 
> > > diff --git a/drivers/pci/msi.c b/drivers/pci/msi.c
> > > index 7f73bac..46c0cdd 100644
> > > --- a/drivers/pci/msi.c
> > > +++ b/drivers/pci/msi.c
> > > @@ -1091,6 +1091,15 @@ static int __pci_enable_msi_range(struct pci_dev *dev, int minvec, int maxvec,
> > >  
> > >  	for (;;) {
> > >  		if (affd) {
> > > +			/*
> > > +			 * If there aren't any vectors left after applying the
> > > +			 * pre/post vectors don't bother with assigning
> > > +			 * affinity.
> > > +			 */
> > > +			if (affd->pre_vectors + affd->post_vectors == nvec)
> > > +				affd = NULL;
> > > +		}
> > 
> > I don't really like this because affd->pre_vectors and
> > affd->post_vectors are not PCI MSI concepts.  I think they really
> > belong in irq/affinity.c, e.g., maybe this could be checked in
> > irq_create_affinity_masks().
> > 
> > If that could be done, we wouldn't have to duplicate the checks here
> > for both MSI and MSI-X.
> > 
> > I raised a similar question earlier:
> > http://lkml.kernel.org/r/20170202173659.GD21267@bhelgaas-glaptop.roam.corp.google.com
> 
> I want to make progress on this, but that means a little conversation
> here.  Maybe we want to merge this patch as-is, but I'd like to hear
> the reasons why it wouldn't really work to move the checks to
> irq/affinity.c.
> 
> For now I'll mark it as "changes requested" in patchwork, which means
> it will fall off my to-do list until it's reposted.
>

Sorry for going silent on this patch. We've been working internally to 
update patch as per your suggestion. I am planning to repost new patch 
once we have completed our testing.  I should be able to post updated 
patch sometimes next week for your review. 

> > > +		if (affd) {
> > >  			nvec = irq_calc_affinity_vectors(nvec, affd);
> > >  			if (nvec < minvec)
> > >  				return -ENOSPC;
> > > @@ -1138,6 +1147,15 @@ static int __pci_enable_msix_range(struct pci_dev *dev,
> > >  
> > >  	for (;;) {
> > >  		if (affd) {
> > > +			/*
> > > +			 * If there aren't any vectors left after applying the
> > > +			 * pre/post vectors don't bother with assigning
> > > +			 * affinity.
> > > +			 */
> > > +			if (affd->pre_vectors + affd->post_vectors == nvec)
> > > +				affd = NULL;
> > > +		}
> > > +		if (affd) {
> > >  			nvec = irq_calc_affinity_vectors(nvec, affd);
> > >  			if (nvec < minvec)
> > >  				return -ENOSPC;
> > > @@ -1209,13 +1227,6 @@ int pci_alloc_irq_vectors_affinity(struct pci_dev *dev, unsigned int min_vecs,
> > >  
> > >  		if (affd->pre_vectors + affd->post_vectors > min_vecs)
> > >  			return -EINVAL;
> > > -
> > > -		/*
> > > -		 * If there aren't any vectors left after applying the pre/post
> > > -		 * vectors don't bother with assigning affinity.
> > > -		 */
> > > -		if (affd->pre_vectors + affd->post_vectors == min_vecs)
> > > -			affd = NULL;
> > >  	} else {
> > >  		if (WARN_ON(affd))
> > >  			affd = NULL;
> > > -- 
> > > 1.7.1
> > > 
> 

Thanks,
- Himanshu

Re: [PATCH] PCI/MSI: Only disable affinity settings if pre and post vector count is equal to max_vecs and not min_vecs

[Bjorn Helgaas]

On Thu, Mar 30, 2017 at 04:25:54PM -0700, Himanshu Madhani wrote:
> 
> Hi Bjorn,
> 
> On Thu, 30 Mar 2017, 4:21pm, Bjorn Helgaas wrote:
> 
> > On Thu, Mar 23, 2017 at 12:29:28PM -0500, Bjorn Helgaas wrote:
> > > On Wed, Mar 01, 2017 at 03:31:58PM -0800, Himanshu Madhani wrote:
> > > > From: Michael Hernandez <michael.hernandez@cavium.com>
> > > > 
> > > > min_vecs is the minimum amount of vectors needed to operate in MSI-X mode
> > > > which may just include the vectors that don't need affinity.
> > > > 
> > > > Disabling affinity settings causes the qla2xxx driver scsi_add_host
> > > > to fail when blk_mq is enabled as the blk_mq_pci_map_queues expects
> > > > affinity masks on each vector.
> > > > 
> > > > v2 --> v3
> > > > o fixed code as per review comments.
> > > > 
> > > > v1 --> v2
> > > > 
> > > > o Moved the check from pci_alloc_irq_vectors_affinity() to
> > > >   __pci_enable_{msi|msix}_range()
> > > > 
> > > > Fixes: dfef358 ("PCI/MSI: Don't apply affinity if there aren't enough vectors left")
> > > > Signed-off-by: Michael Hernandez <michael.hernandez@cavium.com>
> > > > Signed-off-by: Himanshu Madhani <himanshu.madhani@cavium.com>
> > > > Cc: Christoph Hellwig <hch@lst.de>
> > > > Cc: stable@vger.kernel.org
> > > > ---
> > > >  drivers/pci/msi.c |   25 ++++++++++++++++++-------
> > > >  1 files changed, 18 insertions(+), 7 deletions(-)
> > > > 
> > > > diff --git a/drivers/pci/msi.c b/drivers/pci/msi.c
> > > > index 7f73bac..46c0cdd 100644
> > > > --- a/drivers/pci/msi.c
> > > > +++ b/drivers/pci/msi.c
> > > > @@ -1091,6 +1091,15 @@ static int __pci_enable_msi_range(struct pci_dev *dev, int minvec, int maxvec,
> > > >  
> > > >  	for (;;) {
> > > >  		if (affd) {
> > > > +			/*
> > > > +			 * If there aren't any vectors left after applying the
> > > > +			 * pre/post vectors don't bother with assigning
> > > > +			 * affinity.
> > > > +			 */
> > > > +			if (affd->pre_vectors + affd->post_vectors == nvec)
> > > > +				affd = NULL;
> > > > +		}
> > > 
> > > I don't really like this because affd->pre_vectors and
> > > affd->post_vectors are not PCI MSI concepts.  I think they really
> > > belong in irq/affinity.c, e.g., maybe this could be checked in
> > > irq_create_affinity_masks().
> > > 
> > > If that could be done, we wouldn't have to duplicate the checks here
> > > for both MSI and MSI-X.
> > > 
> > > I raised a similar question earlier:
> > > http://lkml.kernel.org/r/20170202173659.GD21267@bhelgaas-glaptop.roam.corp.google.com
> > 
> > I want to make progress on this, but that means a little conversation
> > here.  Maybe we want to merge this patch as-is, but I'd like to hear
> > the reasons why it wouldn't really work to move the checks to
> > irq/affinity.c.
> > 
> > For now I'll mark it as "changes requested" in patchwork, which means
> > it will fall off my to-do list until it's reposted.
> >
> 
> Sorry for going silent on this patch. We've been working internally to 
> update patch as per your suggestion. I am planning to repost new patch 
> once we have completed our testing.  I should be able to post updated 
> patch sometimes next week for your review. 

Great, thanks!  I just wanted to make sure we weren't each waiting for
the other.  I appreciate your willingness to investigate other
options.

Bjorn

[PATCH] PCI/MSI: Only disable affinity settings if pre and post vector count is equal to max_vecs and not min_vecs

[Himanshu Madhani]

From: Michael Hernandez <michael.hernandez@cavium.com>

min_vecs is the minimum amount of vectors needed to operate in MSI-X mode
which may just include the vectors that don't need affinity.

Disabling affinity settings causes the qla2xxx driver scsi_add_host
to fail when blk_mq is enabled as the blk_mq_pci_map_queues expects
affinity masks on each vector.

v1 --> v2

o Moved the check from pci_alloc_irq_vectors_affinity() to
  __pci_enable_{msi|msix}_range()

Fixes: dfef358 ("PCI/MSI: Don't apply affinity if there aren't enough vectors left")
Signed-off-by: Michael Hernandez <michael.hernandez@cavium.com>
Signed-off-by: Himanshu Madhani <himanshu.madhani@cavium.com>
Cc: Christoph Hellwig <hch@lst.de>
Cc: stable@vger.kernel.org
---
 drivers/pci/msi.c |   32 ++++++++++++++++++++++----------
 1 files changed, 22 insertions(+), 10 deletions(-)

diff --git a/drivers/pci/msi.c b/drivers/pci/msi.c
index 7f73bac..c61039b 100644
--- a/drivers/pci/msi.c
+++ b/drivers/pci/msi.c
@@ -1091,6 +1091,17 @@ static int __pci_enable_msi_range(struct pci_dev *dev, int minvec, int maxvec,
 
 	for (;;) {
 		if (affd) {
+			if (affd->pre_vectors + affd->post_vectors > nvec)
+				return -EINVAL;
+			/*
+			 * If there aren't any vectors left after applying the
+			 * pre/post vectors don't bother with assigning
+			 * affinity.
+			 */
+			if (affd->pre_vectors + affd->post_vectors == nvec)
+				affd = NULL;
+		}
+		if (affd) {
 			nvec = irq_calc_affinity_vectors(nvec, affd);
 			if (nvec < minvec)
 				return -ENOSPC;
@@ -1138,6 +1149,17 @@ static int __pci_enable_msix_range(struct pci_dev *dev,
 
 	for (;;) {
 		if (affd) {
+			if (affd->pre_vectors + affd->post_vectors > nvec)
+				return -EINVAL;
+			/*
+			 * If there aren't any vectors left after applying the
+			 * pre/post vectors don't bother with assigning
+			 * affinity.
+			 */
+			if (affd->pre_vectors + affd->post_vectors == nvec)
+				affd = NULL;
+		}
+		if (affd) {
 			nvec = irq_calc_affinity_vectors(nvec, affd);
 			if (nvec < minvec)
 				return -ENOSPC;
@@ -1206,16 +1228,6 @@ int pci_alloc_irq_vectors_affinity(struct pci_dev *dev, unsigned int min_vecs,
 	if (flags & PCI_IRQ_AFFINITY) {
 		if (!affd)
 			affd = &msi_default_affd;
-
-		if (affd->pre_vectors + affd->post_vectors > min_vecs)
-			return -EINVAL;
-
-		/*
-		 * If there aren't any vectors left after applying the pre/post
-		 * vectors don't bother with assigning affinity.
-		 */
-		if (affd->pre_vectors + affd->post_vectors == min_vecs)
-			affd = NULL;
 	} else {
 		if (WARN_ON(affd))
 			affd = NULL;
-- 
1.7.1

RE: [PATCH] PCI/MSI: Only disable affinity settings if pre and post vector count is equal to max_vecs and not min_vecs

[Madhani, Himanshu]

Hi Christoph, Bjorn,=20

Does this updated patch looks good. Please review.=20

Thanks,
Himanshu


> -----Original Message-----
> From: Himanshu Madhani [mailto:himanshu.madhani@cavium.com]
> Sent: Thursday, February 23, 2017 1:02 PM
> To: bhelgaas@google.com; linux-pci@vger.kernel.org; hch@lst.de
> Cc: Madhani, Himanshu <Himanshu.Madhani@cavium.com>
> Subject: [PATCH] PCI/MSI: Only disable affinity settings if pre and post =
vector
> count is equal to max_vecs and not min_vecs
>=20
> From: Michael Hernandez <michael.hernandez@cavium.com>
>=20
> min_vecs is the minimum amount of vectors needed to operate in MSI-X
> mode which may just include the vectors that don't need affinity.
>=20
> Disabling affinity settings causes the qla2xxx driver scsi_add_host to fa=
il
> when blk_mq is enabled as the blk_mq_pci_map_queues expects affinity
> masks on each vector.
>=20
> v1 --> v2
>=20
> o Moved the check from pci_alloc_irq_vectors_affinity() to
>   __pci_enable_{msi|msix}_range()
>=20
> Fixes: dfef358 ("PCI/MSI: Don't apply affinity if there aren't enough vec=
tors
> left")
> Signed-off-by: Michael Hernandez <michael.hernandez@cavium.com>
> Signed-off-by: Himanshu Madhani <himanshu.madhani@cavium.com>
> Cc: Christoph Hellwig <hch@lst.de>
> Cc: stable@vger.kernel.org
> ---
>  drivers/pci/msi.c |   32 ++++++++++++++++++++++----------
>  1 files changed, 22 insertions(+), 10 deletions(-)
>=20
> diff --git a/drivers/pci/msi.c b/drivers/pci/msi.c index 7f73bac..c61039b
> 100644
> --- a/drivers/pci/msi.c
> +++ b/drivers/pci/msi.c
> @@ -1091,6 +1091,17 @@ static int __pci_enable_msi_range(struct pci_dev
> *dev, int minvec, int maxvec,
>=20
>  	for (;;) {
>  		if (affd) {
> +			if (affd->pre_vectors + affd->post_vectors > nvec)
> +				return -EINVAL;
> +			/*
> +			 * If there aren't any vectors left after applying the
> +			 * pre/post vectors don't bother with assigning
> +			 * affinity.
> +			 */
> +			if (affd->pre_vectors + affd->post_vectors =3D=3D nvec)
> +				affd =3D NULL;
> +		}
> +		if (affd) {
>  			nvec =3D irq_calc_affinity_vectors(nvec, affd);
>  			if (nvec < minvec)
>  				return -ENOSPC;
> @@ -1138,6 +1149,17 @@ static int __pci_enable_msix_range(struct pci_dev
> *dev,
>=20
>  	for (;;) {
>  		if (affd) {
> +			if (affd->pre_vectors + affd->post_vectors > nvec)
> +				return -EINVAL;
> +			/*
> +			 * If there aren't any vectors left after applying the
> +			 * pre/post vectors don't bother with assigning
> +			 * affinity.
> +			 */
> +			if (affd->pre_vectors + affd->post_vectors =3D=3D nvec)
> +				affd =3D NULL;
> +		}
> +		if (affd) {
>  			nvec =3D irq_calc_affinity_vectors(nvec, affd);
>  			if (nvec < minvec)
>  				return -ENOSPC;
> @@ -1206,16 +1228,6 @@ int pci_alloc_irq_vectors_affinity(struct pci_dev
> *dev, unsigned int min_vecs,
>  	if (flags & PCI_IRQ_AFFINITY) {
>  		if (!affd)
>  			affd =3D &msi_default_affd;
> -
> -		if (affd->pre_vectors + affd->post_vectors > min_vecs)
> -			return -EINVAL;
> -
> -		/*
> -		 * If there aren't any vectors left after applying the pre/post
> -		 * vectors don't bother with assigning affinity.
> -		 */
> -		if (affd->pre_vectors + affd->post_vectors =3D=3D min_vecs)
> -			affd =3D NULL;
>  	} else {
>  		if (WARN_ON(affd))
>  			affd =3D NULL;
> --
> 1.7.1

Re: [PATCH] PCI/MSI: Only disable affinity settings if pre and post vector count is equal to max_vecs and not min_vecs

[Christoph Hellwig]

> @@ -1206,16 +1228,6 @@ int pci_alloc_irq_vectors_affinity(struct pci_dev *dev, unsigned int min_vecs,
>  	if (flags & PCI_IRQ_AFFINITY) {
>  		if (!affd)
>  			affd = &msi_default_affd;
> -
> -		if (affd->pre_vectors + affd->post_vectors > min_vecs)
> -			return -EINVAL;

Actually I think we can leave this check here.

> -
> -		/*
> -		 * If there aren't any vectors left after applying the pre/post
> -		 * vectors don't bother with assigning affinity.
> -		 */
> -		if (affd->pre_vectors + affd->post_vectors == min_vecs)
> -			affd = NULL;

And only move this one down into the allocator helpers.

RE: [PATCH] PCI/MSI: Only disable affinity settings if pre and post vector count is equal to max_vecs and not min_vecs

[Madhani, Himanshu]

> -----Original Message-----
> From: Christoph Hellwig [mailto:hch@lst.de]
> Sent: Wednesday, March 1, 2017 7:21 AM
> To: Madhani, Himanshu <Himanshu.Madhani@cavium.com>
> Cc: bhelgaas@google.com; linux-pci@vger.kernel.org; hch@lst.de
> Subject: Re: [PATCH] PCI/MSI: Only disable affinity settings if pre and p=
ost
> vector count is equal to max_vecs and not min_vecs
>=20
> > @@ -1206,16 +1228,6 @@ int pci_alloc_irq_vectors_affinity(struct pci_de=
v
> *dev, unsigned int min_vecs,
> >  	if (flags & PCI_IRQ_AFFINITY) {
> >  		if (!affd)
> >  			affd =3D &msi_default_affd;
> > -
> > -		if (affd->pre_vectors + affd->post_vectors > min_vecs)
> > -			return -EINVAL;
>=20
> Actually I think we can leave this check here.
>=20
> > -
> > -		/*
> > -		 * If there aren't any vectors left after applying the pre/post
> > -		 * vectors don't bother with assigning affinity.
> > -		 */
> > -		if (affd->pre_vectors + affd->post_vectors =3D=3D min_vecs)
> > -			affd =3D NULL;
>=20
> And only move this one down into the allocator helpers.

Will send out update soon.=20

Thanks,
Himanshu

[PATCH] PCI/MSI: Only disable affinity settings if pre and post vector count is equal to max_vecs and not min_vecs

[Himanshu Madhani]

From: Michael Hernandez <michael.hernandez@cavium.com>

min_vecs is the minimum amount of vectors needed to operate in MSI-X mode
which may just include the vectors that don't need affinity.

Disabling affinity settings causes the qla2xxx driver scsi_add_host
to fail when blk_mq is enabled as the blk_mq_pci_map_queues expects
affinity masks on each vector.

Fixes: dfef358 ("PCI/MSI: Don't apply affinity if there aren't enough vectors left")
Signed-off-by: Michael Hernandez <michael.hernandez@cavium.com>
Signed-off-by: Himanshu Madhani <himanshu.madhani@cavium.com>
Cc: Christoph Hellwig <hch@lst.de>
Cc: stable@vger.kernel.org
---
 drivers/pci/msi.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/drivers/pci/msi.c b/drivers/pci/msi.c
index 7f73bac..5823348 100644
--- a/drivers/pci/msi.c
+++ b/drivers/pci/msi.c
@@ -1214,7 +1214,7 @@ int pci_alloc_irq_vectors_affinity(struct pci_dev *dev, unsigned int min_vecs,
 		 * If there aren't any vectors left after applying the pre/post
 		 * vectors don't bother with assigning affinity.
 		 */
-		if (affd->pre_vectors + affd->post_vectors == min_vecs)
+		if (affd->pre_vectors + affd->post_vectors == max_vecs)
 			affd = NULL;
 	} else {
 		if (WARN_ON(affd))
-- 
1.8.3.1

Re: [PATCH] PCI/MSI: Only disable affinity settings if pre and post vector count is equal to max_vecs and not min_vecs

[Christoph Hellwig]

On Wed, Feb 22, 2017 at 10:12:35PM -0800, Himanshu Madhani wrote:
> From: Michael Hernandez <michael.hernandez@cavium.com>
> 
> min_vecs is the minimum amount of vectors needed to operate in MSI-X mode
> which may just include the vectors that don't need affinity.
> 
> Disabling affinity settings causes the qla2xxx driver scsi_add_host
> to fail when blk_mq is enabled as the blk_mq_pci_map_queues expects
> affinity masks on each vector.

I don't think this is correct either.  We'll need to move these checks
into __pci_enable_msix_range instead I think so that they operate
on the actual number of vectors not the min/max ones.

Something like the untested patch below, which will also need a MSI
version of the check, and possibly a bit of cleanup:


diff --git a/drivers/pci/msi.c b/drivers/pci/msi.c
index 7f73bacf13ed..6d11c4f620f3 100644
--- a/drivers/pci/msi.c
+++ b/drivers/pci/msi.c
@@ -1138,6 +1138,18 @@ static int __pci_enable_msix_range(struct pci_dev *dev,
 
 	for (;;) {
 		if (affd) {
+			if (affd->pre_vectors + affd->post_vectors > nvec)
+				return -EINVAL;
+
+			/*
+			 * If there aren't any vectors left after applying the
+			 * pre/post vectors don't bother with assigning
+			 * affinity.
+			 */
+			if (affd->pre_vectors + affd->post_vectors == nvec)
+				affd = NULL;
+		}
+		if (affd) {
 			nvec = irq_calc_affinity_vectors(nvec, affd);
 			if (nvec < minvec)
 				return -ENOSPC;
@@ -1206,16 +1218,6 @@ int pci_alloc_irq_vectors_affinity(struct pci_dev *dev, unsigned int min_vecs,
 	if (flags & PCI_IRQ_AFFINITY) {
 		if (!affd)
 			affd = &msi_default_affd;
-
-		if (affd->pre_vectors + affd->post_vectors > min_vecs)
-			return -EINVAL;
-
-		/*
-		 * If there aren't any vectors left after applying the pre/post
-		 * vectors don't bother with assigning affinity.
-		 */
-		if (affd->pre_vectors + affd->post_vectors == min_vecs)
-			affd = NULL;
 	} else {
 		if (WARN_ON(affd))
 			affd = NULL;