[PATCH 1/2] PCI: introduce locked pci_add/remove_virtfn

[PATCH 1/2] PCI: introduce locked pci_add/remove_virtfn

[Emil Tantilov]

This is to allow moving the mutex lock outside of
pci_iov_add/rem_virtfn() for enabling/disabling SRIOV, while
still making it possible to call the _locked version like it is
the case for PPC's eeh_driver.

CC: Alexander Duyck <alexander.h.duyck@intel.com>
Signed-off-by: Emil Tantilov <emil.s.tantilov@intel.com>
---
 arch/powerpc/kernel/eeh_driver.c |    4 ++--
 drivers/pci/iov.c                |   31 ++++++++++++++++++++++++-------
 include/linux/pci.h              |    4 ++--
 3 files changed, 28 insertions(+), 11 deletions(-)

diff --git a/arch/powerpc/kernel/eeh_driver.c b/arch/powerpc/kernel/eeh_driver.c
index d88573b..81aaea7 100644
--- a/arch/powerpc/kernel/eeh_driver.c
+++ b/arch/powerpc/kernel/eeh_driver.c
@@ -441,7 +441,7 @@ static void *eeh_add_virt_device(void *data, void *userdata)
 	}
 
 #ifdef CONFIG_PPC_POWERNV
-	pci_iov_add_virtfn(edev->physfn, pdn->vf_index, 0);
+	pci_iov_add_virtfn_locked(edev->physfn, pdn->vf_index, 0);
 #endif
 	return NULL;
 }
@@ -499,7 +499,7 @@ static void *eeh_rmv_device(void *data, void *userdata)
 #ifdef CONFIG_PPC_POWERNV
 		struct pci_dn *pdn = eeh_dev_to_pdn(edev);
 
-		pci_iov_remove_virtfn(edev->physfn, pdn->vf_index, 0);
+		pci_iov_remove_virtfn_locked(edev->physfn, pdn->vf_index, 0);
 		edev->pdev = NULL;
 
 		/*
diff --git a/drivers/pci/iov.c b/drivers/pci/iov.c
index 4722782..fea322db 100644
--- a/drivers/pci/iov.c
+++ b/drivers/pci/iov.c
@@ -113,7 +113,7 @@ resource_size_t pci_iov_resource_size(struct pci_dev *dev, int resno)
 	return dev->sriov->barsz[resno - PCI_IOV_RESOURCES];
 }
 
-int pci_iov_add_virtfn(struct pci_dev *dev, int id, int reset)
+static int pci_iov_add_virtfn(struct pci_dev *dev, int id, int reset)
 {
 	int i;
 	int rc = -ENOMEM;
@@ -124,7 +124,6 @@ int pci_iov_add_virtfn(struct pci_dev *dev, int id, int reset)
 	struct pci_sriov *iov = dev->sriov;
 	struct pci_bus *bus;
 
-	mutex_lock(&iov->dev->sriov->lock);
 	bus = virtfn_add_bus(dev->bus, pci_iov_virtfn_bus(dev, id));
 	if (!bus)
 		goto failed;
@@ -162,7 +161,6 @@ int pci_iov_add_virtfn(struct pci_dev *dev, int id, int reset)
 		__pci_reset_function(virtfn);
 
 	pci_device_add(virtfn, virtfn->bus);
-	mutex_unlock(&iov->dev->sriov->lock);
 
 	pci_bus_add_device(virtfn);
 	sprintf(buf, "virtfn%u", id);
@@ -191,11 +189,22 @@ int pci_iov_add_virtfn(struct pci_dev *dev, int id, int reset)
 	return rc;
 }
 
-void pci_iov_remove_virtfn(struct pci_dev *dev, int id, int reset)
+int pci_iov_add_virtfn_locked(struct pci_dev *dev, int id, int reset)
+{
+	struct pci_sriov *iov = dev->sriov;
+	int rc;
+
+	mutex_lock(&iov->dev->sriov->lock);
+	rc = pci_iov_add_virtfn(dev, id, reset);
+	mutex_unlock(&iov->dev->sriov->lock);
+
+	return rc;
+}
+
+static void pci_iov_remove_virtfn(struct pci_dev *dev, int id, int reset)
 {
 	char buf[VIRTFN_ID_LEN];
 	struct pci_dev *virtfn;
-	struct pci_sriov *iov = dev->sriov;
 
 	virtfn = pci_get_domain_bus_and_slot(pci_domain_nr(dev->bus),
 					     pci_iov_virtfn_bus(dev, id),
@@ -218,16 +227,24 @@ void pci_iov_remove_virtfn(struct pci_dev *dev, int id, int reset)
 	if (virtfn->dev.kobj.sd)
 		sysfs_remove_link(&virtfn->dev.kobj, "physfn");
 
-	mutex_lock(&iov->dev->sriov->lock);
 	pci_stop_and_remove_bus_device(virtfn);
 	virtfn_remove_bus(dev->bus, virtfn->bus);
-	mutex_unlock(&iov->dev->sriov->lock);
 
 	/* balance pci_get_domain_bus_and_slot() */
 	pci_dev_put(virtfn);
 	pci_dev_put(dev);
 }
 
+void pci_iov_remove_virtfn_locked(struct pci_dev *dev, int id, int reset)
+{
+	struct pci_sriov *iov = dev->sriov;
+
+	mutex_lock(&iov->dev->sriov->lock);
+	pci_iov_remove_virtfn(dev, id, reset);
+	mutex_unlock(&iov->dev->sriov->lock);
+}
+
+
 int __weak pcibios_sriov_enable(struct pci_dev *pdev, u16 num_vfs)
 {
 	return 0;
diff --git a/include/linux/pci.h b/include/linux/pci.h
index e2d1a12..4351ceb7 100644
--- a/include/linux/pci.h
+++ b/include/linux/pci.h
@@ -1872,8 +1872,8 @@ static inline void pci_mmcfg_late_init(void) { }
 
 int pci_enable_sriov(struct pci_dev *dev, int nr_virtfn);
 void pci_disable_sriov(struct pci_dev *dev);
-int pci_iov_add_virtfn(struct pci_dev *dev, int id, int reset);
-void pci_iov_remove_virtfn(struct pci_dev *dev, int id, int reset);
+int pci_iov_add_virtfn_locked(struct pci_dev *dev, int id, int reset);
+void pci_iov_remove_virtfn_locked(struct pci_dev *dev, int id, int reset);
 int pci_num_vf(struct pci_dev *dev);
 int pci_vfs_assigned(struct pci_dev *dev);
 int pci_sriov_set_totalvfs(struct pci_dev *dev, u16 numvfs);

[PATCH 2/2] PCI: lock each enable/disable num_vfs operation in sysfs

[Emil Tantilov]

Enabling/disabling SRIOV via sysfs by echo-ing multiple values
simultaneously:

echo 63 > /sys/class/net/ethX/device/sriov_numvfs&
echo 63 > /sys/class/net/ethX/device/sriov_numvfs

sleep 5

echo 0 > /sys/class/net/ethX/device/sriov_numvfs&
echo 0 > /sys/class/net/ethX/device/sriov_numvfs

Results in the following bug:

kernel BUG at drivers/pci/iov.c:495!
invalid opcode: 0000 [#1] SMP
CPU: 1 PID: 8050 Comm: bash Tainted: G   W   4.9.0-rc7-net-next #2092
RIP: 0010:[<ffffffff813b1647>]
	  [<ffffffff813b1647>] pci_iov_release+0x57/0x60

Call Trace:
 [<ffffffff81391726>] pci_release_dev+0x26/0x70
 [<ffffffff8155be6e>] device_release+0x3e/0xb0
 [<ffffffff81365ee7>] kobject_cleanup+0x67/0x180
 [<ffffffff81365d9d>] kobject_put+0x2d/0x60
 [<ffffffff8155bc27>] put_device+0x17/0x20
 [<ffffffff8139c08a>] pci_dev_put+0x1a/0x20
 [<ffffffff8139cb6b>] pci_get_dev_by_id+0x5b/0x90
 [<ffffffff8139cca5>] pci_get_subsys+0x35/0x40
 [<ffffffff8139ccc8>] pci_get_device+0x18/0x20
 [<ffffffff8139ccfb>] pci_get_domain_bus_and_slot+0x2b/0x60
 [<ffffffff813b09e7>] pci_iov_remove_virtfn+0x57/0x180
 [<ffffffff813b0b95>] pci_disable_sriov+0x65/0x140
 [<ffffffffa00a1af7>] ixgbe_disable_sriov+0xc7/0x1d0 [ixgbe]
 [<ffffffffa00a1e9d>] ixgbe_pci_sriov_configure+0x3d/0x170 [ixgbe]
 [<ffffffff8139d28c>] sriov_numvfs_store+0xdc/0x130
...
RIP  [<ffffffff813b1647>] pci_iov_release+0x57/0x60

Use the existing mutex lock to protect each enable/disable operation.

CC: Alexander Duyck <alexander.h.duyck@intel.com>
Signed-off-by: Emil Tantilov <emil.s.tantilov@intel.com>
---
 drivers/pci/pci-sysfs.c |   24 +++++++++++++++++-------
 1 file changed, 17 insertions(+), 7 deletions(-)

diff --git a/drivers/pci/pci-sysfs.c b/drivers/pci/pci-sysfs.c
index 0666287..5b54cf5 100644
--- a/drivers/pci/pci-sysfs.c
+++ b/drivers/pci/pci-sysfs.c
@@ -472,7 +472,9 @@ static ssize_t sriov_numvfs_store(struct device *dev,
 				  const char *buf, size_t count)
 {
 	struct pci_dev *pdev = to_pci_dev(dev);
+	struct pci_sriov *iov = pdev->sriov;
 	int ret;
+
 	u16 num_vfs;
 
 	ret = kstrtou16(buf, 0, &num_vfs);
@@ -482,38 +484,46 @@ static ssize_t sriov_numvfs_store(struct device *dev,
 	if (num_vfs > pci_sriov_get_totalvfs(pdev))
 		return -ERANGE;
 
+	mutex_lock(&iov->dev->sriov->lock);
+
 	if (num_vfs == pdev->sriov->num_VFs)
-		return count;		/* no change */
+		goto exit;
 
 	/* is PF driver loaded w/callback */
 	if (!pdev->driver || !pdev->driver->sriov_configure) {
 		dev_info(&pdev->dev, "Driver doesn't support SRIOV configuration via sysfs\n");
-		return -ENOSYS;
+		ret = -EINVAL;
+		goto exit;
 	}
 
 	if (num_vfs == 0) {
 		/* disable VFs */
 		ret = pdev->driver->sriov_configure(pdev, 0);
-		if (ret < 0)
-			return ret;
-		return count;
+		goto exit;
 	}
 
 	/* enable VFs */
 	if (pdev->sriov->num_VFs) {
 		dev_warn(&pdev->dev, "%d VFs already enabled. Disable before enabling %d VFs\n",
 			 pdev->sriov->num_VFs, num_vfs);
-		return -EBUSY;
+		ret = -EBUSY;
+		goto exit;
 	}
 
 	ret = pdev->driver->sriov_configure(pdev, num_vfs);
 	if (ret < 0)
-		return ret;
+		goto exit;
 
 	if (ret != num_vfs)
 		dev_warn(&pdev->dev, "%d VFs requested; only %d enabled\n",
 			 num_vfs, ret);
 
+exit:
+	mutex_unlock(&iov->dev->sriov->lock);
+
+	if (ret < 0)
+		return ret;
+
 	return count;
 }
 

Re: [PATCH 2/2] PCI: lock each enable/disable num_vfs operation in sysfs

[Gavin Shan]

On Tue, Jan 03, 2017 at 04:48:31PM -0800, Emil Tantilov wrote:
>Enabling/disabling SRIOV via sysfs by echo-ing multiple values
>simultaneously:
>
>echo 63 > /sys/class/net/ethX/device/sriov_numvfs&
>echo 63 > /sys/class/net/ethX/device/sriov_numvfs
>
>sleep 5
>
>echo 0 > /sys/class/net/ethX/device/sriov_numvfs&
>echo 0 > /sys/class/net/ethX/device/sriov_numvfs
>
>Results in the following bug:
>
>kernel BUG at drivers/pci/iov.c:495!
>invalid opcode: 0000 [#1] SMP
>CPU: 1 PID: 8050 Comm: bash Tainted: G   W   4.9.0-rc7-net-next #2092
>RIP: 0010:[<ffffffff813b1647>]
>	  [<ffffffff813b1647>] pci_iov_release+0x57/0x60
>
>Call Trace:
> [<ffffffff81391726>] pci_release_dev+0x26/0x70
> [<ffffffff8155be6e>] device_release+0x3e/0xb0
> [<ffffffff81365ee7>] kobject_cleanup+0x67/0x180
> [<ffffffff81365d9d>] kobject_put+0x2d/0x60
> [<ffffffff8155bc27>] put_device+0x17/0x20
> [<ffffffff8139c08a>] pci_dev_put+0x1a/0x20
> [<ffffffff8139cb6b>] pci_get_dev_by_id+0x5b/0x90
> [<ffffffff8139cca5>] pci_get_subsys+0x35/0x40
> [<ffffffff8139ccc8>] pci_get_device+0x18/0x20
> [<ffffffff8139ccfb>] pci_get_domain_bus_and_slot+0x2b/0x60
> [<ffffffff813b09e7>] pci_iov_remove_virtfn+0x57/0x180
> [<ffffffff813b0b95>] pci_disable_sriov+0x65/0x140
> [<ffffffffa00a1af7>] ixgbe_disable_sriov+0xc7/0x1d0 [ixgbe]
> [<ffffffffa00a1e9d>] ixgbe_pci_sriov_configure+0x3d/0x170 [ixgbe]
> [<ffffffff8139d28c>] sriov_numvfs_store+0xdc/0x130
>...
>RIP  [<ffffffff813b1647>] pci_iov_release+0x57/0x60
>
>Use the existing mutex lock to protect each enable/disable operation.
>
>CC: Alexander Duyck <alexander.h.duyck@intel.com>
>Signed-off-by: Emil Tantilov <emil.s.tantilov@intel.com>

Emil, It's going to change semantics of pci_enable_sriov() and pci_disable_sriov().
They can be invoked when writing to the sysfs entry, or loading PF's driver. With
the change applied, the lock (pf->sriov->lock) isn't acquired and released in the
PF's driver loading path.

I think the reasonable way would be adding a flag in "struct sriov", to indicate
someone is accessing the IOV capability through sysfs file. With this, the code
returns with "-EBUSY" immediately for contenders. With it, nothing is going to
be changed in PF's driver loading path.

Also, there are some minor comments as below and I guess most of them won't be
applied if you take my suggestion eventually. However, I'm trying to make the
comments complete.

>---
> drivers/pci/pci-sysfs.c |   24 +++++++++++++++++-------
> 1 file changed, 17 insertions(+), 7 deletions(-)
>
>diff --git a/drivers/pci/pci-sysfs.c b/drivers/pci/pci-sysfs.c
>index 0666287..5b54cf5 100644
>--- a/drivers/pci/pci-sysfs.c
>+++ b/drivers/pci/pci-sysfs.c
>@@ -472,7 +472,9 @@ static ssize_t sriov_numvfs_store(struct device *dev,
> 				  const char *buf, size_t count)
> {
> 	struct pci_dev *pdev = to_pci_dev(dev);
>+	struct pci_sriov *iov = pdev->sriov;
> 	int ret;
>+

Unnecessary change.

> 	u16 num_vfs;
>
> 	ret = kstrtou16(buf, 0, &num_vfs);
>@@ -482,38 +484,46 @@ static ssize_t sriov_numvfs_store(struct device *dev,
> 	if (num_vfs > pci_sriov_get_totalvfs(pdev))
> 		return -ERANGE;
>
>+	mutex_lock(&iov->dev->sriov->lock);
>+
> 	if (num_vfs == pdev->sriov->num_VFs)
>-		return count;		/* no change */
>+		goto exit;
>
> 	/* is PF driver loaded w/callback */
> 	if (!pdev->driver || !pdev->driver->sriov_configure) {
> 		dev_info(&pdev->dev, "Driver doesn't support SRIOV configuration via sysfs\n");
>-		return -ENOSYS;
>+		ret = -EINVAL;
>+		goto exit;

Why we need change the error code here?

> 	}
>
> 	if (num_vfs == 0) {
> 		/* disable VFs */
> 		ret = pdev->driver->sriov_configure(pdev, 0);
>-		if (ret < 0)
>-			return ret;
>-		return count;
>+		goto exit;
> 	}
>
> 	/* enable VFs */
> 	if (pdev->sriov->num_VFs) {
> 		dev_warn(&pdev->dev, "%d VFs already enabled. Disable before enabling %d VFs\n",
> 			 pdev->sriov->num_VFs, num_vfs);
>-		return -EBUSY;
>+		ret = -EBUSY;
>+		goto exit;
> 	}
>
> 	ret = pdev->driver->sriov_configure(pdev, num_vfs);
> 	if (ret < 0)
>-		return ret;
>+		goto exit;
>
> 	if (ret != num_vfs)
> 		dev_warn(&pdev->dev, "%d VFs requested; only %d enabled\n",
> 			 num_vfs, ret);
>
>+exit:
>+	mutex_unlock(&iov->dev->sriov->lock);
>+
>+	if (ret < 0)
>+		return ret;
>+
> 	return count;

The code might be clearer if @ret is returned here. In that case, We need
set it properly in error paths.

> }
>

Thanks,
Gavin

Re: [Intel-wired-lan] [PATCH 1/2] PCI: introduce locked pci_add/remove_virtfn

[kbuild test robot]

[-- Attachment #1: Type: text/plain, Size: 3981 bytes --]

Hi Emil,

[auto build test ERROR on pci/next]
[also build test ERROR on v4.10-rc2 next-20170104]
[if your patch is applied to the wrong git tree, please drop us a note to help improve the system]

url:    https://github.com/0day-ci/linux/commits/Emil-Tantilov/PCI-introduce-locked-pci_add-remove_virtfn/20170104-193518
base:   https://git.kernel.org/pub/scm/linux/kernel/git/helgaas/pci.git next
config: powerpc-defconfig (attached as .config)
compiler: powerpc64-linux-gnu-gcc (Debian 6.1.1-9) 6.1.1 20160705
reproduce:
        wget https://git.kernel.org/cgit/linux/kernel/git/wfg/lkp-tests.git/plain/sbin/make.cross -O ~/bin/make.cross
        chmod +x ~/bin/make.cross
        # save the attached .config to linux build tree
        make.cross ARCH=powerpc 

All errors (new ones prefixed by >>):

   arch/powerpc/kernel/eeh_driver.c: In function 'eeh_add_virt_device':
>> arch/powerpc/kernel/eeh_driver.c:444:2: error: implicit declaration of function 'pci_iov_add_virtfn_locked' [-Werror=implicit-function-declaration]
     pci_iov_add_virtfn_locked(edev->physfn, pdn->vf_index, 0);
     ^~~~~~~~~~~~~~~~~~~~~~~~~
   arch/powerpc/kernel/eeh_driver.c: In function 'eeh_rmv_device':
>> arch/powerpc/kernel/eeh_driver.c:502:3: error: implicit declaration of function 'pci_iov_remove_virtfn_locked' [-Werror=implicit-function-declaration]
      pci_iov_remove_virtfn_locked(edev->physfn, pdn->vf_index, 0);
      ^~~~~~~~~~~~~~~~~~~~~~~~~~~~
   cc1: all warnings being treated as errors

vim +/pci_iov_add_virtfn_locked +444 arch/powerpc/kernel/eeh_driver.c

   438			eeh_pcid_put(dev);
   439			if (driver->err_handler)
   440				return NULL;
   441		}
   442	
   443	#ifdef CONFIG_PPC_POWERNV
 > 444		pci_iov_add_virtfn_locked(edev->physfn, pdn->vf_index, 0);
   445	#endif
   446		return NULL;
   447	}
   448	
   449	static void *eeh_rmv_device(void *data, void *userdata)
   450	{
   451		struct pci_driver *driver;
   452		struct eeh_dev *edev = (struct eeh_dev *)data;
   453		struct pci_dev *dev = eeh_dev_to_pci_dev(edev);
   454		struct eeh_rmv_data *rmv_data = (struct eeh_rmv_data *)userdata;
   455		int *removed = rmv_data ? &rmv_data->removed : NULL;
   456	
   457		/*
   458		 * Actually, we should remove the PCI bridges as well.
   459		 * However, that's lots of complexity to do that,
   460		 * particularly some of devices under the bridge might
   461		 * support EEH. So we just care about PCI devices for
   462		 * simplicity here.
   463		 */
   464		if (!dev || (dev->hdr_type == PCI_HEADER_TYPE_BRIDGE))
   465			return NULL;
   466	
   467		/*
   468		 * We rely on count-based pcibios_release_device() to
   469		 * detach permanently offlined PEs. Unfortunately, that's
   470		 * not reliable enough. We might have the permanently
   471		 * offlined PEs attached, but we needn't take care of
   472		 * them and their child devices.
   473		 */
   474		if (eeh_dev_removed(edev))
   475			return NULL;
   476	
   477		driver = eeh_pcid_get(dev);
   478		if (driver) {
   479			eeh_pcid_put(dev);
   480			if (removed &&
   481			    eeh_pe_passed(edev->pe))
   482				return NULL;
   483			if (removed &&
   484			    driver->err_handler &&
   485			    driver->err_handler->error_detected &&
   486			    driver->err_handler->slot_reset)
   487				return NULL;
   488		}
   489	
   490		/* Remove it from PCI subsystem */
   491		pr_debug("EEH: Removing %s without EEH sensitive driver\n",
   492			 pci_name(dev));
   493		edev->bus = dev->bus;
   494		edev->mode |= EEH_DEV_DISCONNECTED;
   495		if (removed)
   496			(*removed)++;
   497	
   498		if (edev->physfn) {
   499	#ifdef CONFIG_PPC_POWERNV
   500			struct pci_dn *pdn = eeh_dev_to_pdn(edev);
   501	
 > 502			pci_iov_remove_virtfn_locked(edev->physfn, pdn->vf_index, 0);
   503			edev->pdev = NULL;
   504	
   505			/*

---
0-DAY kernel test infrastructure                Open Source Technology Center
https://lists.01.org/pipermail/kbuild-all                   Intel Corporation

[-- Attachment #2: .config.gz --]
[-- Type: application/gzip, Size: 22266 bytes --]

RE: [PATCH 2/2] PCI: lock each enable/disable num_vfs operation in sysfs

[Tantilov, Emil S]

>-----Original Message-----
>From: Gavin Shan [mailto:gwshan@linux.vnet.ibm.com]
>Sent: Tuesday, January 03, 2017 6:16 PM
>To: Tantilov, Emil S <emil.s.tantilov@intel.com>
>Cc: linux-pci@vger.kernel.org; intel-wired-lan@lists.osuosl.org; Duyck,
>Alexander H <alexander.h.duyck@intel.com>; netdev@vger.kernel.org; linux-
>kernel@vger.kernel.org
>Subject: Re: [PATCH 2/2] PCI: lock each enable/disable num_vfs operation in
>sysfs
>
>On Tue, Jan 03, 2017 at 04:48:31PM -0800, Emil Tantilov wrote:
>>Enabling/disabling SRIOV via sysfs by echo-ing multiple values
>>simultaneously:
>>
>>echo 63 > /sys/class/net/ethX/device/sriov_numvfs&
>>echo 63 > /sys/class/net/ethX/device/sriov_numvfs
>>
>>sleep 5
>>
>>echo 0 > /sys/class/net/ethX/device/sriov_numvfs&
>>echo 0 > /sys/class/net/ethX/device/sriov_numvfs
>>
>>Results in the following bug:
>>
>>kernel BUG at drivers/pci/iov.c:495!
>>invalid opcode: 0000 [#1] SMP
>>CPU: 1 PID: 8050 Comm: bash Tainted: G   W   4.9.0-rc7-net-next #2092
>>RIP: 0010:[<ffffffff813b1647>]
>>	  [<ffffffff813b1647>] pci_iov_release+0x57/0x60
>>
>>Call Trace:
>> [<ffffffff81391726>] pci_release_dev+0x26/0x70
>> [<ffffffff8155be6e>] device_release+0x3e/0xb0
>> [<ffffffff81365ee7>] kobject_cleanup+0x67/0x180
>> [<ffffffff81365d9d>] kobject_put+0x2d/0x60
>> [<ffffffff8155bc27>] put_device+0x17/0x20
>> [<ffffffff8139c08a>] pci_dev_put+0x1a/0x20
>> [<ffffffff8139cb6b>] pci_get_dev_by_id+0x5b/0x90
>> [<ffffffff8139cca5>] pci_get_subsys+0x35/0x40
>> [<ffffffff8139ccc8>] pci_get_device+0x18/0x20
>> [<ffffffff8139ccfb>] pci_get_domain_bus_and_slot+0x2b/0x60
>> [<ffffffff813b09e7>] pci_iov_remove_virtfn+0x57/0x180
>> [<ffffffff813b0b95>] pci_disable_sriov+0x65/0x140
>> [<ffffffffa00a1af7>] ixgbe_disable_sriov+0xc7/0x1d0 [ixgbe]
>> [<ffffffffa00a1e9d>] ixgbe_pci_sriov_configure+0x3d/0x170 [ixgbe]
>> [<ffffffff8139d28c>] sriov_numvfs_store+0xdc/0x130
>>...
>>RIP  [<ffffffff813b1647>] pci_iov_release+0x57/0x60
>>
>>Use the existing mutex lock to protect each enable/disable operation.
>>
>>CC: Alexander Duyck <alexander.h.duyck@intel.com>
>>Signed-off-by: Emil Tantilov <emil.s.tantilov@intel.com>
>
>Emil, It's going to change semantics of pci_enable_sriov() and pci_disable_sriov().
>They can be invoked when writing to the sysfs entry, or loading PF's
>driver. With the change applied, the lock (pf->sriov->lock) isn't acquired and released
>in the PF's driver loading path.

The enablement of SRIOV on driver load is done via deprecated module parameter.
Perhaps we can just remove it, although there are probably still people that use it
and may not be happy if we get rid of it. 

>I think the reasonable way would be adding a flag in "struct sriov", to
>indicate someone is accessing the IOV capability through sysfs file. With this, the
>code returns with "-EBUSY" immediately for contenders. With it, nothing is going
>to be changed in PF's driver loading path.

Flag is what I initially had in mind, but did not want to add extra locking if we 
can make use of the existing.

>Also, there are some minor comments as below and I guess most of them won't
>be applied if you take my suggestion eventually. However, I'm trying to make
>the comments complete.

Thanks a lot for reviewing!

>
>>---
>> drivers/pci/pci-sysfs.c |   24 +++++++++++++++++-------
>> 1 file changed, 17 insertions(+), 7 deletions(-)
>>
>>diff --git a/drivers/pci/pci-sysfs.c b/drivers/pci/pci-sysfs.c
>>index 0666287..5b54cf5 100644
>>--- a/drivers/pci/pci-sysfs.c
>>+++ b/drivers/pci/pci-sysfs.c
>>@@ -472,7 +472,9 @@ static ssize_t sriov_numvfs_store(struct device *dev,
>> 				  const char *buf, size_t count)
>> {
>> 	struct pci_dev *pdev = to_pci_dev(dev);
>>+	struct pci_sriov *iov = pdev->sriov;
>> 	int ret;
>>+
>
>Unnecessary change.
>
>> 	u16 num_vfs;
>>
>> 	ret = kstrtou16(buf, 0, &num_vfs);
>>@@ -482,38 +484,46 @@ static ssize_t sriov_numvfs_store(struct device
>*dev,
>> 	if (num_vfs > pci_sriov_get_totalvfs(pdev))
>> 		return -ERANGE;
>>
>>+	mutex_lock(&iov->dev->sriov->lock);
>>+
>> 	if (num_vfs == pdev->sriov->num_VFs)
>>-		return count;		/* no change */
>>+		goto exit;
>>
>> 	/* is PF driver loaded w/callback */
>> 	if (!pdev->driver || !pdev->driver->sriov_configure) {
>> 		dev_info(&pdev->dev, "Driver doesn't support SRIOV
>configuration via sysfs\n");
>>-		return -ENOSYS;
>>+		ret = -EINVAL;
>>+		goto exit;
>
>Why we need change the error code here?

checkpatch was complaining about the use of the ENOSYS error code being specific
and even though it was not my patch introducing it I had to change it to shut it up.

>> 	}
>>
>> 	if (num_vfs == 0) {
>> 		/* disable VFs */
>> 		ret = pdev->driver->sriov_configure(pdev, 0);
>>-		if (ret < 0)
>>-			return ret;
>>-		return count;
>>+		goto exit;
>> 	}
>>
>> 	/* enable VFs */
>> 	if (pdev->sriov->num_VFs) {
>> 		dev_warn(&pdev->dev, "%d VFs already enabled. Disable before enabling %d VFs\n",
>> 			 pdev->sriov->num_VFs, num_vfs);
>>-		return -EBUSY;
>>+		ret = -EBUSY;
>>+		goto exit;
>> 	}
>>
>> 	ret = pdev->driver->sriov_configure(pdev, num_vfs);
>> 	if (ret < 0)
>>-		return ret;
>>+		goto exit;
>>
>> 	if (ret != num_vfs)
>> 		dev_warn(&pdev->dev, "%d VFs requested; only %d enabled\n",
>> 			 num_vfs, ret);
>>
>>+exit:
>>+	mutex_unlock(&iov->dev->sriov->lock);
>>+
>>+	if (ret < 0)
>>+		return ret;
>>+
>> 	return count;
>
>The code might be clearer if @ret is returned here. In that case, We need
>set it properly in error paths.

I played with different ways to handle this and this seemed the least intrusive.

Thanks,
Emil

Re: [PATCH 2/2] PCI: lock each enable/disable num_vfs operation in sysfs

[Gavin Shan]

On Wed, Jan 04, 2017 at 04:00:20PM +0000, Tantilov, Emil S wrote:
>>On Tue, Jan 03, 2017 at 04:48:31PM -0800, Emil Tantilov wrote:
>>>Enabling/disabling SRIOV via sysfs by echo-ing multiple values
>>>simultaneously:
>>>
>>>echo 63 > /sys/class/net/ethX/device/sriov_numvfs&
>>>echo 63 > /sys/class/net/ethX/device/sriov_numvfs
>>>
>>>sleep 5
>>>
>>>echo 0 > /sys/class/net/ethX/device/sriov_numvfs&
>>>echo 0 > /sys/class/net/ethX/device/sriov_numvfs
>>>
>>>Results in the following bug:
>>>
>>>kernel BUG at drivers/pci/iov.c:495!
>>>invalid opcode: 0000 [#1] SMP
>>>CPU: 1 PID: 8050 Comm: bash Tainted: G   W   4.9.0-rc7-net-next #2092
>>>RIP: 0010:[<ffffffff813b1647>]
>>>	  [<ffffffff813b1647>] pci_iov_release+0x57/0x60
>>>
>>>Call Trace:
>>> [<ffffffff81391726>] pci_release_dev+0x26/0x70
>>> [<ffffffff8155be6e>] device_release+0x3e/0xb0
>>> [<ffffffff81365ee7>] kobject_cleanup+0x67/0x180
>>> [<ffffffff81365d9d>] kobject_put+0x2d/0x60
>>> [<ffffffff8155bc27>] put_device+0x17/0x20
>>> [<ffffffff8139c08a>] pci_dev_put+0x1a/0x20
>>> [<ffffffff8139cb6b>] pci_get_dev_by_id+0x5b/0x90
>>> [<ffffffff8139cca5>] pci_get_subsys+0x35/0x40
>>> [<ffffffff8139ccc8>] pci_get_device+0x18/0x20
>>> [<ffffffff8139ccfb>] pci_get_domain_bus_and_slot+0x2b/0x60
>>> [<ffffffff813b09e7>] pci_iov_remove_virtfn+0x57/0x180
>>> [<ffffffff813b0b95>] pci_disable_sriov+0x65/0x140
>>> [<ffffffffa00a1af7>] ixgbe_disable_sriov+0xc7/0x1d0 [ixgbe]
>>> [<ffffffffa00a1e9d>] ixgbe_pci_sriov_configure+0x3d/0x170 [ixgbe]
>>> [<ffffffff8139d28c>] sriov_numvfs_store+0xdc/0x130
>>>...
>>>RIP  [<ffffffff813b1647>] pci_iov_release+0x57/0x60
>>>
>>>Use the existing mutex lock to protect each enable/disable operation.
>>>
>>>CC: Alexander Duyck <alexander.h.duyck@intel.com>
>>>Signed-off-by: Emil Tantilov <emil.s.tantilov@intel.com>
>>
>>Emil, It's going to change semantics of pci_enable_sriov() and pci_disable_sriov().
>>They can be invoked when writing to the sysfs entry, or loading PF's
>>driver. With the change applied, the lock (pf->sriov->lock) isn't acquired and released
>>in the PF's driver loading path.
>
>The enablement of SRIOV on driver load is done via deprecated module parameter.
>Perhaps we can just remove it, although there are probably still people that use it
>and may not be happy if we get rid of it. 
>

Yeah, some drivers are still using the interface. So we cannot affect it until
it can be droped.

>>I think the reasonable way would be adding a flag in "struct sriov", to
>>indicate someone is accessing the IOV capability through sysfs file. With this, the
>>code returns with "-EBUSY" immediately for contenders. With it, nothing is going
>>to be changed in PF's driver loading path.
>
>Flag is what I initially had in mind, but did not want to add extra locking if we 
>can make use of the existing.
>

The problem is sriov->lock wasn't introduced to protect the whole IOV capability.
Instead, it protects the allocation of virtual bus (if needed). In your patch,
it will be used to protect the whole IOV capability, ensure accessing the IOV
capability exclusively. So the usage of this lock is changed. 

     code extracted from pci.h:

     struct pci_sriov {
            :
            struct mutex lock;      /* lock for VF bus */
            :
     }

The lock was introduced by commit d1b054da8 ("PCI: initialize and release SR-IOV
capability"). If I'm correct enough, I don't think this lock is needed when
pci_enable_sriov() or pci_disable_sriov() are called in driver because of module
parameters. I don't see the usage case calling pci_disable_sriov() while previous
pci_enable_sriov() isn't finished yet. Also, it's not needed in EEH subsystem.
So I think the lock can be dropped, then it can be used to protect sysfs path. 

>>Also, there are some minor comments as below and I guess most of them won't
>>be applied if you take my suggestion eventually. However, I'm trying to make
>>the comments complete.
>
>Thanks a lot for reviewing!
>
>>
>>>---
>>> drivers/pci/pci-sysfs.c |   24 +++++++++++++++++-------
>>> 1 file changed, 17 insertions(+), 7 deletions(-)
>>>
>>>diff --git a/drivers/pci/pci-sysfs.c b/drivers/pci/pci-sysfs.c
>>>index 0666287..5b54cf5 100644
>>>--- a/drivers/pci/pci-sysfs.c
>>>+++ b/drivers/pci/pci-sysfs.c
>>>@@ -472,7 +472,9 @@ static ssize_t sriov_numvfs_store(struct device *dev,
>>> 				  const char *buf, size_t count)
>>> {
>>> 	struct pci_dev *pdev = to_pci_dev(dev);
>>>+	struct pci_sriov *iov = pdev->sriov;
>>> 	int ret;
>>>+
>>
>>Unnecessary change.
>>
>>> 	u16 num_vfs;
>>>
>>> 	ret = kstrtou16(buf, 0, &num_vfs);
>>>@@ -482,38 +484,46 @@ static ssize_t sriov_numvfs_store(struct device
>>*dev,
>>> 	if (num_vfs > pci_sriov_get_totalvfs(pdev))
>>> 		return -ERANGE;
>>>
>>>+	mutex_lock(&iov->dev->sriov->lock);
>>>+
>>> 	if (num_vfs == pdev->sriov->num_VFs)
>>>-		return count;		/* no change */
>>>+		goto exit;
>>>
>>> 	/* is PF driver loaded w/callback */
>>> 	if (!pdev->driver || !pdev->driver->sriov_configure) {
>>> 		dev_info(&pdev->dev, "Driver doesn't support SRIOV
>>configuration via sysfs\n");
>>>-		return -ENOSYS;
>>>+		ret = -EINVAL;
>>>+		goto exit;
>>
>>Why we need change the error code here?
>
>checkpatch was complaining about the use of the ENOSYS error code being specific
>and even though it was not my patch introducing it I had to change it to shut it up.
>

Right, it's reserved for attempt to call nonexisting syscall, but I think ENOENT
might be more indicative than EINVAL in this specific case?

>>> 	}
>>>
>>> 	if (num_vfs == 0) {
>>> 		/* disable VFs */
>>> 		ret = pdev->driver->sriov_configure(pdev, 0);
>>>-		if (ret < 0)
>>>-			return ret;
>>>-		return count;
>>>+		goto exit;
>>> 	}
>>>
>>> 	/* enable VFs */
>>> 	if (pdev->sriov->num_VFs) {
>>> 		dev_warn(&pdev->dev, "%d VFs already enabled. Disable before enabling %d VFs\n",
>>> 			 pdev->sriov->num_VFs, num_vfs);
>>>-		return -EBUSY;
>>>+		ret = -EBUSY;
>>>+		goto exit;
>>> 	}
>>>
>>> 	ret = pdev->driver->sriov_configure(pdev, num_vfs);
>>> 	if (ret < 0)
>>>-		return ret;
>>>+		goto exit;
>>>
>>> 	if (ret != num_vfs)
>>> 		dev_warn(&pdev->dev, "%d VFs requested; only %d enabled\n",
>>> 			 num_vfs, ret);
>>>
>>>+exit:
>>>+	mutex_unlock(&iov->dev->sriov->lock);
>>>+
>>>+	if (ret < 0)
>>>+		return ret;
>>>+
>>> 	return count;
>>
>>The code might be clearer if @ret is returned here. In that case, We need
>>set it properly in error paths.
>
>I played with different ways to handle this and this seemed the least intrusive.
>

Ok, both should be fine.

Thanks,
Gavin

RE: [PATCH 2/2] PCI: lock each enable/disable num_vfs operation in sysfs

[Tantilov, Emil S]

>-----Original Message-----
>From: Gavin Shan [mailto:gwshan@linux.vnet.ibm.com]
>Sent: Wednesday, January 04, 2017 3:12 PM
>To: Tantilov, Emil S <emil.s.tantilov@intel.com>
>Cc: Gavin Shan <gwshan@linux.vnet.ibm.com>; linux-pci@vger.kernel.org;
>intel-wired-lan@lists.osuosl.org; Duyck, Alexander H
><alexander.h.duyck@intel.com>; netdev@vger.kernel.org; linux-
>kernel@vger.kernel.org
>Subject: Re: [PATCH 2/2] PCI: lock each enable/disable num_vfs operation in
>sysfs
>
>On Wed, Jan 04, 2017 at 04:00:20PM +0000, Tantilov, Emil S wrote:
>>>On Tue, Jan 03, 2017 at 04:48:31PM -0800, Emil Tantilov wrote:
>>>>Enabling/disabling SRIOV via sysfs by echo-ing multiple values
>>>>simultaneously:
>>>>
>>>>echo 63 > /sys/class/net/ethX/device/sriov_numvfs&
>>>>echo 63 > /sys/class/net/ethX/device/sriov_numvfs
>>>>
>>>>sleep 5
>>>>
>>>>echo 0 > /sys/class/net/ethX/device/sriov_numvfs&
>>>>echo 0 > /sys/class/net/ethX/device/sriov_numvfs
>>>>
>>>>Results in the following bug:
>>>>
>>>>kernel BUG at drivers/pci/iov.c:495!
>>>>invalid opcode: 0000 [#1] SMP
>>>>CPU: 1 PID: 8050 Comm: bash Tainted: G   W   4.9.0-rc7-net-next #2092
>>>>RIP: 0010:[<ffffffff813b1647>]
>>>>	  [<ffffffff813b1647>] pci_iov_release+0x57/0x60
>>>>
>>>>Call Trace:
>>>> [<ffffffff81391726>] pci_release_dev+0x26/0x70
>>>> [<ffffffff8155be6e>] device_release+0x3e/0xb0
>>>> [<ffffffff81365ee7>] kobject_cleanup+0x67/0x180
>>>> [<ffffffff81365d9d>] kobject_put+0x2d/0x60
>>>> [<ffffffff8155bc27>] put_device+0x17/0x20
>>>> [<ffffffff8139c08a>] pci_dev_put+0x1a/0x20
>>>> [<ffffffff8139cb6b>] pci_get_dev_by_id+0x5b/0x90
>>>> [<ffffffff8139cca5>] pci_get_subsys+0x35/0x40
>>>> [<ffffffff8139ccc8>] pci_get_device+0x18/0x20
>>>> [<ffffffff8139ccfb>] pci_get_domain_bus_and_slot+0x2b/0x60
>>>> [<ffffffff813b09e7>] pci_iov_remove_virtfn+0x57/0x180
>>>> [<ffffffff813b0b95>] pci_disable_sriov+0x65/0x140
>>>> [<ffffffffa00a1af7>] ixgbe_disable_sriov+0xc7/0x1d0 [ixgbe]
>>>> [<ffffffffa00a1e9d>] ixgbe_pci_sriov_configure+0x3d/0x170 [ixgbe]
>>>> [<ffffffff8139d28c>] sriov_numvfs_store+0xdc/0x130
>>>>...
>>>>RIP  [<ffffffff813b1647>] pci_iov_release+0x57/0x60
>>>>
>>>>Use the existing mutex lock to protect each enable/disable operation.
>>>>
>>>>CC: Alexander Duyck <alexander.h.duyck@intel.com>
>>>>Signed-off-by: Emil Tantilov <emil.s.tantilov@intel.com>
>>>
>>>Emil, It's going to change semantics of pci_enable_sriov() and
>pci_disable_sriov().
>>>They can be invoked when writing to the sysfs entry, or loading PF's
>>>driver. With the change applied, the lock (pf->sriov->lock) isn't acquired and released
>>>in the PF's driver loading path.
>>
>>The enablement of SRIOV on driver load is done via deprecated module parameter.
>>Perhaps we can just remove it, although there are probably still people that use it
>>and may not be happy if we get rid of it.
>>
>
>Yeah, some drivers are still using the interface. So we cannot affect it
>until it can be droped.
>
>>>I think the reasonable way would be adding a flag in "struct sriov", to
>>>indicate someone is accessing the IOV capability through sysfs file. With this, the
>>>code returns with "-EBUSY" immediately for contenders. With it, nothing is going
>>>to be changed in PF's driver loading path.
>>
>>Flag is what I initially had in mind, but did not want to add extra locking if we
>>can make use of the existing.
>>
>
>The problem is sriov->lock wasn't introduced to protect the whole IOV capability.
>Instead, it protects the allocation of virtual bus (if needed). In your patch,
>it will be used to protect the whole IOV capability, ensure accessing the
>IOV capability exclusively. So the usage of this lock is changed.
>
>     code extracted from pci.h:
>
>     struct pci_sriov {
>            :
>            struct mutex lock;      /* lock for VF bus */
>            :
>     }
>
>The lock was introduced by commit d1b054da8 ("PCI: initialize and release
>SR-IOV capability"). If I'm correct enough, I don't think this lock is needed when
>pci_enable_sriov() or pci_disable_sriov() are called in driver because of
>module
>parameters. I don't see the usage case calling pci_disable_sriov() while
>previous pci_enable_sriov() isn't finished yet. Also, it's not needed in EEH
>subsystem.
>So I think the lock can be dropped, then it can be used to protect sysfs path.

That's pretty much what this patch does, except I kept the locking for EEH since 
it is the only driver that calls pci_iov_add/remove_virtfn() directly.

I'll write it up and run some tests, although I have no way to test EEH.
 
>>>Also, there are some minor comments as below and I guess most of them won't
>>>be applied if you take my suggestion eventually. However, I'm trying to make
>>>the comments complete.
>>
>>Thanks a lot for reviewing!
>>
>>>
>>>>---
>>>> drivers/pci/pci-sysfs.c |   24 +++++++++++++++++-------
>>>> 1 file changed, 17 insertions(+), 7 deletions(-)
>>>>
>>>>diff --git a/drivers/pci/pci-sysfs.c b/drivers/pci/pci-sysfs.c
>>>>index 0666287..5b54cf5 100644
>>>>--- a/drivers/pci/pci-sysfs.c
>>>>+++ b/drivers/pci/pci-sysfs.c
>>>>@@ -472,7 +472,9 @@ static ssize_t sriov_numvfs_store(struct device
>*dev,
>>>> 				  const char *buf, size_t count)
>>>> {
>>>> 	struct pci_dev *pdev = to_pci_dev(dev);
>>>>+	struct pci_sriov *iov = pdev->sriov;
>>>> 	int ret;
>>>>+
>>>
>>>Unnecessary change.
>>>
>>>> 	u16 num_vfs;
>>>>
>>>> 	ret = kstrtou16(buf, 0, &num_vfs);
>>>>@@ -482,38 +484,46 @@ static ssize_t sriov_numvfs_store(struct device
>>>*dev,
>>>> 	if (num_vfs > pci_sriov_get_totalvfs(pdev))
>>>> 		return -ERANGE;
>>>>
>>>>+	mutex_lock(&iov->dev->sriov->lock);
>>>>+
>>>> 	if (num_vfs == pdev->sriov->num_VFs)
>>>>-		return count;		/* no change */
>>>>+		goto exit;
>>>>
>>>> 	/* is PF driver loaded w/callback */
>>>> 	if (!pdev->driver || !pdev->driver->sriov_configure) {
>>>> 		dev_info(&pdev->dev, "Driver doesn't support SRIOV
>>>configuration via sysfs\n");
>>>>-		return -ENOSYS;
>>>>+		ret = -EINVAL;
>>>>+		goto exit;
>>>
>>>Why we need change the error code here?
>>
>>checkpatch was complaining about the use of the ENOSYS error code being specific
>>and even though it was not my patch introducing it I had to change it to shut it up.
>>
>
>Right, it's reserved for attempt to call nonexisting syscall, but I think
>ENOENT might be more indicative than EINVAL in this specific case?

ENOENT is for a missing file, but if we got this far in the code then there must've been 
a sysfs file. This is pretty straightforward "not supported" error, which is why I picked
EINVAL.

Thanks,
Emil 

Re: [PATCH 2/2] PCI: lock each enable/disable num_vfs operation in sysfs

[Gavin Shan]

On Fri, Jan 06, 2017 at 12:55:08AM +0000, Tantilov, Emil S wrote:
>>On Wed, Jan 04, 2017 at 04:00:20PM +0000, Tantilov, Emil S wrote:
>>>>On Tue, Jan 03, 2017 at 04:48:31PM -0800, Emil Tantilov wrote:
>>>>>Enabling/disabling SRIOV via sysfs by echo-ing multiple values
>>>>>simultaneously:
>>>>>
>>>>>echo 63 > /sys/class/net/ethX/device/sriov_numvfs&
>>>>>echo 63 > /sys/class/net/ethX/device/sriov_numvfs
>>>>>
>>>>>sleep 5
>>>>>
>>>>>echo 0 > /sys/class/net/ethX/device/sriov_numvfs&
>>>>>echo 0 > /sys/class/net/ethX/device/sriov_numvfs
>>>>>
>>>>>Results in the following bug:
>>>>>
>>>>>kernel BUG at drivers/pci/iov.c:495!
>>>>>invalid opcode: 0000 [#1] SMP
>>>>>CPU: 1 PID: 8050 Comm: bash Tainted: G   W   4.9.0-rc7-net-next #2092
>>>>>RIP: 0010:[<ffffffff813b1647>]
>>>>>	  [<ffffffff813b1647>] pci_iov_release+0x57/0x60
>>>>>
>>>>>Call Trace:
>>>>> [<ffffffff81391726>] pci_release_dev+0x26/0x70
>>>>> [<ffffffff8155be6e>] device_release+0x3e/0xb0
>>>>> [<ffffffff81365ee7>] kobject_cleanup+0x67/0x180
>>>>> [<ffffffff81365d9d>] kobject_put+0x2d/0x60
>>>>> [<ffffffff8155bc27>] put_device+0x17/0x20
>>>>> [<ffffffff8139c08a>] pci_dev_put+0x1a/0x20
>>>>> [<ffffffff8139cb6b>] pci_get_dev_by_id+0x5b/0x90
>>>>> [<ffffffff8139cca5>] pci_get_subsys+0x35/0x40
>>>>> [<ffffffff8139ccc8>] pci_get_device+0x18/0x20
>>>>> [<ffffffff8139ccfb>] pci_get_domain_bus_and_slot+0x2b/0x60
>>>>> [<ffffffff813b09e7>] pci_iov_remove_virtfn+0x57/0x180
>>>>> [<ffffffff813b0b95>] pci_disable_sriov+0x65/0x140
>>>>> [<ffffffffa00a1af7>] ixgbe_disable_sriov+0xc7/0x1d0 [ixgbe]
>>>>> [<ffffffffa00a1e9d>] ixgbe_pci_sriov_configure+0x3d/0x170 [ixgbe]
>>>>> [<ffffffff8139d28c>] sriov_numvfs_store+0xdc/0x130
>>>>>...
>>>>>RIP  [<ffffffff813b1647>] pci_iov_release+0x57/0x60
>>>>>
>>>>>Use the existing mutex lock to protect each enable/disable operation.
>>>>>
>>>>>CC: Alexander Duyck <alexander.h.duyck@intel.com>
>>>>>Signed-off-by: Emil Tantilov <emil.s.tantilov@intel.com>
>>>>
>>>>Emil, It's going to change semantics of pci_enable_sriov() and
>>pci_disable_sriov().
>>>>They can be invoked when writing to the sysfs entry, or loading PF's
>>>>driver. With the change applied, the lock (pf->sriov->lock) isn't acquired and released
>>>>in the PF's driver loading path.
>>>
>>>The enablement of SRIOV on driver load is done via deprecated module parameter.
>>>Perhaps we can just remove it, although there are probably still people that use it
>>>and may not be happy if we get rid of it.
>>>
>>
>>Yeah, some drivers are still using the interface. So we cannot affect it
>>until it can be droped.
>>
>>>>I think the reasonable way would be adding a flag in "struct sriov", to
>>>>indicate someone is accessing the IOV capability through sysfs file. With this, the
>>>>code returns with "-EBUSY" immediately for contenders. With it, nothing is going
>>>>to be changed in PF's driver loading path.
>>>
>>>Flag is what I initially had in mind, but did not want to add extra locking if we
>>>can make use of the existing.
>>>
>>
>>The problem is sriov->lock wasn't introduced to protect the whole IOV capability.
>>Instead, it protects the allocation of virtual bus (if needed). In your patch,
>>it will be used to protect the whole IOV capability, ensure accessing the
>>IOV capability exclusively. So the usage of this lock is changed.
>>
>>     code extracted from pci.h:
>>
>>     struct pci_sriov {
>>            :
>>            struct mutex lock;      /* lock for VF bus */
>>            :
>>     }
>>
>>The lock was introduced by commit d1b054da8 ("PCI: initialize and release
>>SR-IOV capability"). If I'm correct enough, I don't think this lock is needed when
>>pci_enable_sriov() or pci_disable_sriov() are called in driver because of
>>module
>>parameters. I don't see the usage case calling pci_disable_sriov() while
>>previous pci_enable_sriov() isn't finished yet. Also, it's not needed in EEH
>>subsystem.
>>So I think the lock can be dropped, then it can be used to protect sysfs path.
>
>That's pretty much what this patch does, except I kept the locking for EEH since 
>it is the only driver that calls pci_iov_add/remove_virtfn() directly.
>
>I'll write it up and run some tests, although I have no way to test EEH.
> 

Yes, Your patch is close to what I suggested. I'm pretty sure EEH needn't
this lock. I'll fix it up if EEH is broken because of it.

>>>>Also, there are some minor comments as below and I guess most of them won't
>>>>be applied if you take my suggestion eventually. However, I'm trying to make
>>>>the comments complete.
>>>
>>>Thanks a lot for reviewing!
>>>
>>>>
>>>>>---
>>>>> drivers/pci/pci-sysfs.c |   24 +++++++++++++++++-------
>>>>> 1 file changed, 17 insertions(+), 7 deletions(-)
>>>>>
>>>>>diff --git a/drivers/pci/pci-sysfs.c b/drivers/pci/pci-sysfs.c
>>>>>index 0666287..5b54cf5 100644
>>>>>--- a/drivers/pci/pci-sysfs.c
>>>>>+++ b/drivers/pci/pci-sysfs.c
>>>>>@@ -472,7 +472,9 @@ static ssize_t sriov_numvfs_store(struct device
>>*dev,
>>>>> 				  const char *buf, size_t count)
>>>>> {
>>>>> 	struct pci_dev *pdev = to_pci_dev(dev);
>>>>>+	struct pci_sriov *iov = pdev->sriov;
>>>>> 	int ret;
>>>>>+
>>>>
>>>>Unnecessary change.
>>>>
>>>>> 	u16 num_vfs;
>>>>>
>>>>> 	ret = kstrtou16(buf, 0, &num_vfs);
>>>>>@@ -482,38 +484,46 @@ static ssize_t sriov_numvfs_store(struct device
>>>>*dev,
>>>>> 	if (num_vfs > pci_sriov_get_totalvfs(pdev))
>>>>> 		return -ERANGE;
>>>>>
>>>>>+	mutex_lock(&iov->dev->sriov->lock);
>>>>>+
>>>>> 	if (num_vfs == pdev->sriov->num_VFs)
>>>>>-		return count;		/* no change */
>>>>>+		goto exit;
>>>>>
>>>>> 	/* is PF driver loaded w/callback */
>>>>> 	if (!pdev->driver || !pdev->driver->sriov_configure) {
>>>>> 		dev_info(&pdev->dev, "Driver doesn't support SRIOV
>>>>configuration via sysfs\n");
>>>>>-		return -ENOSYS;
>>>>>+		ret = -EINVAL;
>>>>>+		goto exit;
>>>>
>>>>Why we need change the error code here?
>>>
>>>checkpatch was complaining about the use of the ENOSYS error code being specific
>>>and even though it was not my patch introducing it I had to change it to shut it up.
>>>
>>
>>Right, it's reserved for attempt to call nonexisting syscall, but I think
>>ENOENT might be more indicative than EINVAL in this specific case?
>
>ENOENT is for a missing file, but if we got this far in the code then there must've been 
>a sysfs file. This is pretty straightforward "not supported" error, which is why I picked
>EINVAL.
>

EINVAL means "invalid arguments" in my mind. The understanding matches with what's
said in https://linux.die.net/man/3/errno  I also treated ENOENT as missed callbacks.
However, it's not a big deal. Just pick the errno you like. What I concerned is user
might has some script to map the errno and exact failure. Precise and unique errno
from error path isn't bad.

Thanks,
Gavin