[PATCH 1/2] pci: Cache the VF device ID in the SR-IOV structure

[PATCH 1/2] pci: Cache the VF device ID in the SR-IOV structure

[Filippo Sironi]

... and use it instead of reading it over and over from the PF config
space capability.

Signed-off-by: Filippo Sironi <sironi@amazon.de>
Cc: linux-pci@vger.kernel.org
Cc: linux-kernel@vger.kernel.org
---
 drivers/pci/iov.c | 5 +++--
 drivers/pci/pci.h | 1 +
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/drivers/pci/iov.c b/drivers/pci/iov.c
index 120485d6f352..e8f7eafaba6a 100644
--- a/drivers/pci/iov.c
+++ b/drivers/pci/iov.c
@@ -134,7 +134,7 @@ int pci_iov_add_virtfn(struct pci_dev *dev, int id, int reset)
 
 	virtfn->devfn = pci_iov_virtfn_devfn(dev, id);
 	virtfn->vendor = dev->vendor;
-	pci_read_config_word(dev, iov->pos + PCI_SRIOV_VF_DID, &virtfn->device);
+	virtfn->device = iov->vf_did;
 	rc = pci_setup_device(virtfn);
 	if (rc)
 		goto failed0;
@@ -448,6 +448,7 @@ static int sriov_init(struct pci_dev *dev, int pos)
 	iov->nres = nres;
 	iov->ctrl = ctrl;
 	iov->total_VFs = total;
+	pci_read_config_word(dev, pos + PCI_SRIOV_VF_DID, &iov->vf_did);
 	iov->pgsz = pgsz;
 	iov->self = dev;
 	iov->drivers_autoprobe = true;
@@ -723,7 +724,7 @@ int pci_vfs_assigned(struct pci_dev *dev)
 	 * determine the device ID for the VFs, the vendor ID will be the
 	 * same as the PF so there is no need to check for that one
 	 */
-	pci_read_config_word(dev, dev->sriov->pos + PCI_SRIOV_VF_DID, &dev_id);
+	dev_id = dev->sriov->vf_did;
 
 	/* loop through all the VFs to see if we own any that are assigned */
 	vfdev = pci_get_device(dev->vendor, dev_id, NULL);
diff --git a/drivers/pci/pci.h b/drivers/pci/pci.h
index 22e061738c6f..a7270e11e1ef 100644
--- a/drivers/pci/pci.h
+++ b/drivers/pci/pci.h
@@ -262,6 +262,7 @@ struct pci_sriov {
 	u16 num_VFs;		/* number of VFs available */
 	u16 offset;		/* first VF Routing ID offset */
 	u16 stride;		/* following VF stride */
+	u16 vf_did;		/* VF device ID */
 	u32 pgsz;		/* page size for BAR alignment */
 	u8 link;		/* Function Dependency Link */
 	u8 max_VF_buses;	/* max buses consumed by VFs */
-- 
2.7.4

[PATCH 2/2] pci: Expose offset, stride, and VF device ID via sysfs

[Filippo Sironi]

... to make it easier for userspace applications consumption.

Signed-off-by: Filippo Sironi <sironi@amazon.de>
Cc: linux-pci@vger.kernel.org
Cc: linux-kernel@vger.kernel.org
---
 drivers/pci/pci-sysfs.c | 33 +++++++++++++++++++++++++++++++++
 1 file changed, 33 insertions(+)

diff --git a/drivers/pci/pci-sysfs.c b/drivers/pci/pci-sysfs.c
index 2f3780b50723..f920afe7cff3 100644
--- a/drivers/pci/pci-sysfs.c
+++ b/drivers/pci/pci-sysfs.c
@@ -648,6 +648,33 @@ static ssize_t sriov_numvfs_store(struct device *dev,
 	return count;
 }
 
+static ssize_t sriov_offset_show(struct device *dev,
+				 struct device_attribute *attr,
+				 char *buf)
+{
+	struct pci_dev *pdev = to_pci_dev(dev);
+
+	return sprintf(buf, "%u\n", pdev->sriov->offset);
+}
+
+static ssize_t sriov_stride_show(struct device *dev,
+				 struct device_attribute *attr,
+				 char *buf)
+{
+	struct pci_dev *pdev = to_pci_dev(dev);
+
+	return sprintf(buf, "%u\n", pdev->sriov->stride);
+}
+
+static ssize_t sriov_vf_did_show(struct device *dev,
+				 struct device_attribute *attr,
+				 char *buf)
+{
+	struct pci_dev *pdev = to_pci_dev(dev);
+
+	return sprintf(buf, "%x\n", pdev->sriov->vf_did);
+}
+
 static ssize_t sriov_drivers_autoprobe_show(struct device *dev,
 					    struct device_attribute *attr,
 					    char *buf)
@@ -676,6 +703,9 @@ static struct device_attribute sriov_totalvfs_attr = __ATTR_RO(sriov_totalvfs);
 static struct device_attribute sriov_numvfs_attr =
 		__ATTR(sriov_numvfs, (S_IRUGO|S_IWUSR|S_IWGRP),
 		       sriov_numvfs_show, sriov_numvfs_store);
+static struct device_attribute sriov_offset_attr = __ATTR_RO(sriov_offset);
+static struct device_attribute sriov_stride_attr = __ATTR_RO(sriov_stride);
+static struct device_attribute sriov_vf_did_attr = __ATTR_RO(sriov_vf_did);
 static struct device_attribute sriov_drivers_autoprobe_attr =
 		__ATTR(sriov_drivers_autoprobe, (S_IRUGO|S_IWUSR|S_IWGRP),
 		       sriov_drivers_autoprobe_show, sriov_drivers_autoprobe_store);
@@ -1744,6 +1774,9 @@ static struct attribute_group pci_dev_hp_attr_group = {
 static struct attribute *sriov_dev_attrs[] = {
 	&sriov_totalvfs_attr.attr,
 	&sriov_numvfs_attr.attr,
+	&sriov_offset_attr.attr,
+	&sriov_stride_attr.attr,
+	&sriov_vf_did_attr.attr,
 	&sriov_drivers_autoprobe_attr.attr,
 	NULL,
 };
-- 
2.7.4

Re: [PATCH 2/2] pci: Expose offset, stride, and VF device ID via sysfs

[Bjorn Helgaas]

Hi Filippo,

On Mon, Aug 28, 2017 at 03:38:50PM +0200, Filippo Sironi wrote:
> +static ssize_t sriov_vf_did_show(struct device *dev,
> +				 struct device_attribute *attr,
> +				 char *buf)
> +{
> +	struct pci_dev *pdev = to_pci_dev(dev);
> +
> +	return sprintf(buf, "%x\n", pdev->sriov->vf_did);
> +}

What does the vf_did part look like in sysfs?  Do we have a directory with
both "device" and "vf_did" in it?  If so, why do we have both and do we
need both?  Could we put the vf_did in the "device" file?

>  static ssize_t sriov_drivers_autoprobe_show(struct device *dev,
>  					    struct device_attribute *attr,
>  					    char *buf)
> @@ -676,6 +703,9 @@ static struct device_attribute sriov_totalvfs_attr = __ATTR_RO(sriov_totalvfs);
>  static struct device_attribute sriov_numvfs_attr =
>  		__ATTR(sriov_numvfs, (S_IRUGO|S_IWUSR|S_IWGRP),
>  		       sriov_numvfs_show, sriov_numvfs_store);
> +static struct device_attribute sriov_offset_attr = __ATTR_RO(sriov_offset);
> +static struct device_attribute sriov_stride_attr = __ATTR_RO(sriov_stride);
> +static struct device_attribute sriov_vf_did_attr = __ATTR_RO(sriov_vf_did);
>  static struct device_attribute sriov_drivers_autoprobe_attr =
>  		__ATTR(sriov_drivers_autoprobe, (S_IRUGO|S_IWUSR|S_IWGRP),
>  		       sriov_drivers_autoprobe_show, sriov_drivers_autoprobe_store);
> @@ -1744,6 +1774,9 @@ static struct attribute_group pci_dev_hp_attr_group = {
>  static struct attribute *sriov_dev_attrs[] = {
>  	&sriov_totalvfs_attr.attr,
>  	&sriov_numvfs_attr.attr,
> +	&sriov_offset_attr.attr,
> +	&sriov_stride_attr.attr,
> +	&sriov_vf_did_attr.attr,
>  	&sriov_drivers_autoprobe_attr.attr,
>  	NULL,
>  };
> -- 
> 2.7.4
> 

Re: [PATCH 2/2] pci: Expose offset, stride, and VF device ID via sysfs

[Sironi, Filippo]

Hi Bjorn,

> On 25. Sep 2017, at 20:55, Bjorn Helgaas <helgaas@kernel.org> wrote:
> 
> Hi Filippo,
> 
> On Mon, Aug 28, 2017 at 03:38:50PM +0200, Filippo Sironi wrote:
>> +static ssize_t sriov_vf_did_show(struct device *dev,
>> +				 struct device_attribute *attr,
>> +				 char *buf)
>> +{
>> +	struct pci_dev *pdev = to_pci_dev(dev);
>> +
>> +	return sprintf(buf, "%x\n", pdev->sriov->vf_did);
>> +}
> 
> What does the vf_did part look like in sysfs?  Do we have a directory with
> both "device" and "vf_did" in it?  If so, why do we have both and do we
> need both?  Could we put the vf_did in the "device" file?

On my machine:

/sys/bus/pci/devices/0000:03:00.0# ls -l  # this is the PF
total 0
-rw-r--r-- 1 root root    4096 Sep 28 19:41 broken_parity_status
-r--r--r-- 1 root root    4096 Sep 28 19:41 class
-rw-r--r-- 1 root root    4096 Sep 28 19:41 config
-r--r--r-- 1 root root    4096 Sep 28 19:41 consistent_dma_mask_bits
-rw-r--r-- 1 root root    4096 Sep 28 19:41 d3cold_allowed
-r--r--r-- 1 root root    4096 Sep 28 19:41 device
-r--r--r-- 1 root root    4096 Sep 28 19:41 dma_mask_bits
lrwxrwxrwx 1 root root       0 Sep 28 19:41 driver -> ../../../../bus/pci/drivers/igb
-rw-r--r-- 1 root root    4096 Sep 28 19:41 driver_override
-rw-r--r-- 1 root root    4096 Sep 28 19:41 enable
lrwxrwxrwx 1 root root       0 Sep 28 19:41 firmware_node -> ../../../LNXSYSTM:00/LNXSYBUS:00/PNP0A08:00/device:4b/device:4c
-r--r--r-- 1 root root    4096 Sep 28 19:41 irq
-r--r--r-- 1 root root    4096 Sep 28 19:41 local_cpulist
-r--r--r-- 1 root root    4096 Sep 28 19:41 local_cpus
-r--r--r-- 1 root root    4096 Sep 28 19:41 modalias
-rw-r--r-- 1 root root    4096 Sep 28 19:41 msi_bus
drwxr-xr-x 2 root root       0 Sep 29 09:44 msi_irqs
drwxr-xr-x 3 root root       0 Sep 28 19:41 net
-rw-r--r-- 1 root root    4096 Sep 28 19:41 numa_node
-r--r--r-- 1 root root    4096 Sep 28 19:41 offset  # this is new
drwxr-xr-x 2 root root       0 Sep 28 19:41 power
drwxr-xr-x 3 root root       0 Sep 28 19:41 ptp
--w--w---- 1 root root    4096 Sep 28 19:41 remove
--w--w---- 1 root root    4096 Sep 28 19:41 rescan
--w------- 1 root root    4096 Sep 28 19:41 reset
-r--r--r-- 1 root root    4096 Sep 28 19:41 resource
-rw------- 1 root root  131072 Sep 28 19:41 resource0
-rw------- 1 root root 4194304 Sep 28 19:41 resource1
-rw------- 1 root root      32 Sep 28 19:41 resource2
-rw------- 1 root root   16384 Sep 28 19:41 resource3
-r--r--r-- 1 root root    4096 Sep 28 19:41 revision
-rw-rw-r-- 1 root root    4096 Sep 29 09:44 sriov_numvfs
-r--r--r-- 1 root root    4096 Sep 28 19:41 sriov_totalvfs
-r--r--r-- 1 root root    4096 Sep 28 19:41 stride  # this is new
lrwxrwxrwx 1 root root       0 Sep 28 19:41 subsystem -> ../../../../bus/pci
-r--r--r-- 1 root root    4096 Sep 28 19:41 subsystem_device
-r--r--r-- 1 root root    4096 Sep 28 19:41 subsystem_vendor
-rw-r--r-- 1 root root    4096 Sep 28 19:41 uevent
-r--r--r-- 1 root root    4096 Sep 28 19:41 vendor
-r--r--r-- 1 root root    4096 Sep 28 19:41 vf_did  # this is new
lrwxrwxrwx 1 root root       0 Sep 29 09:44 virtfn0 -> ../0000:03:10.0

nothing changes on for VFs.
Then:

/sys/bus/pci/devices/0000:03:00.0# cat device 
0x10c9

/sys/bus/pci/devices/0000:03:00.0# cat vf_did
0x10ca

Putting the VF device ID in the PF device file would be a change of
that we expose to userspace.  Something might break.

vf_did provides a easy way to retrieve the VF device ID without reading
the PF config (looking up the SR-IOV capability and reading it) or without
enabling SR-IOV to read for example virtfn0/device.

Similar considerations (ease of access) apply to offset and stride.


>> static ssize_t sriov_drivers_autoprobe_show(struct device *dev,
>> 					    struct device_attribute *attr,
>> 					    char *buf)
>> @@ -676,6 +703,9 @@ static struct device_attribute sriov_totalvfs_attr = __ATTR_RO(sriov_totalvfs);
>> static struct device_attribute sriov_numvfs_attr =
>> 		__ATTR(sriov_numvfs, (S_IRUGO|S_IWUSR|S_IWGRP),
>> 		       sriov_numvfs_show, sriov_numvfs_store);
>> +static struct device_attribute sriov_offset_attr = __ATTR_RO(sriov_offset);
>> +static struct device_attribute sriov_stride_attr = __ATTR_RO(sriov_stride);
>> +static struct device_attribute sriov_vf_did_attr = __ATTR_RO(sriov_vf_did);
>> static struct device_attribute sriov_drivers_autoprobe_attr =
>> 		__ATTR(sriov_drivers_autoprobe, (S_IRUGO|S_IWUSR|S_IWGRP),
>> 		       sriov_drivers_autoprobe_show, sriov_drivers_autoprobe_store);
>> @@ -1744,6 +1774,9 @@ static struct attribute_group pci_dev_hp_attr_group = {
>> static struct attribute *sriov_dev_attrs[] = {
>> 	&sriov_totalvfs_attr.attr,
>> 	&sriov_numvfs_attr.attr,
>> +	&sriov_offset_attr.attr,
>> +	&sriov_stride_attr.attr,
>> +	&sriov_vf_did_attr.attr,
>> 	&sriov_drivers_autoprobe_attr.attr,
>> 	NULL,
>> };
>> -- 
>> 2.7.4
>> 
> 

Amazon Development Center Germany GmbH
Berlin - Dresden - Aachen
main office: Krausenstr. 38, 10117 Berlin
Geschaeftsfuehrer: Dr. Ralf Herbrich, Christian Schlaeger
Ust-ID: DE289237879
Eingetragen am Amtsgericht Charlottenburg HRB 149173 B

Re: [PATCH 2/2] pci: Expose offset, stride, and VF device ID via sysfs

[Sironi, Filippo]

Hi Bjorn,

> On 25. Sep 2017, at 20:55, Bjorn Helgaas <helgaas@kernel.org> wrote:
> =

> Hi Filippo,
> =

> On Mon, Aug 28, 2017 at 03:38:50PM +0200, Filippo Sironi wrote:
>> +static ssize_t sriov_vf_did_show(struct device *dev,
>> +				 struct device_attribute *attr,
>> +				 char *buf)
>> +{
>> +	struct pci_dev *pdev =3D to_pci_dev(dev);
>> +
>> +	return sprintf(buf, "%x\n", pdev->sriov->vf_did);
>> +}
> =

> What does the vf_did part look like in sysfs?  Do we have a directory with
> both "device" and "vf_did" in it?  If so, why do we have both and do we
> need both?  Could we put the vf_did in the "device" file?

On my machine:

/sys/bus/pci/devices/0000:03:00.0# ls -l  # this is the PF
total 0
-rw-r--r-- 1 root root    4096 Sep 28 19:41 broken_parity_status
-r--r--r-- 1 root root    4096 Sep 28 19:41 class
-rw-r--r-- 1 root root    4096 Sep 28 19:41 config
-r--r--r-- 1 root root    4096 Sep 28 19:41 consistent_dma_mask_bits
-rw-r--r-- 1 root root    4096 Sep 28 19:41 d3cold_allowed
-r--r--r-- 1 root root    4096 Sep 28 19:41 device
-r--r--r-- 1 root root    4096 Sep 28 19:41 dma_mask_bits
lrwxrwxrwx 1 root root       0 Sep 28 19:41 driver -> ../../../../bus/pci/d=
rivers/igb
-rw-r--r-- 1 root root    4096 Sep 28 19:41 driver_override
-rw-r--r-- 1 root root    4096 Sep 28 19:41 enable
lrwxrwxrwx 1 root root       0 Sep 28 19:41 firmware_node -> ../../../LNXSY=
STM:00/LNXSYBUS:00/PNP0A08:00/device:4b/device:4c
-r--r--r-- 1 root root    4096 Sep 28 19:41 irq
-r--r--r-- 1 root root    4096 Sep 28 19:41 local_cpulist
-r--r--r-- 1 root root    4096 Sep 28 19:41 local_cpus
-r--r--r-- 1 root root    4096 Sep 28 19:41 modalias
-rw-r--r-- 1 root root    4096 Sep 28 19:41 msi_bus
drwxr-xr-x 2 root root       0 Sep 29 09:44 msi_irqs
drwxr-xr-x 3 root root       0 Sep 28 19:41 net
-rw-r--r-- 1 root root    4096 Sep 28 19:41 numa_node
-r--r--r-- 1 root root    4096 Sep 28 19:41 offset  # this is new
drwxr-xr-x 2 root root       0 Sep 28 19:41 power
drwxr-xr-x 3 root root       0 Sep 28 19:41 ptp
--w--w---- 1 root root    4096 Sep 28 19:41 remove
--w--w---- 1 root root    4096 Sep 28 19:41 rescan
--w------- 1 root root    4096 Sep 28 19:41 reset
-r--r--r-- 1 root root    4096 Sep 28 19:41 resource
-rw------- 1 root root  131072 Sep 28 19:41 resource0
-rw------- 1 root root 4194304 Sep 28 19:41 resource1
-rw------- 1 root root      32 Sep 28 19:41 resource2
-rw------- 1 root root   16384 Sep 28 19:41 resource3
-r--r--r-- 1 root root    4096 Sep 28 19:41 revision
-rw-rw-r-- 1 root root    4096 Sep 29 09:44 sriov_numvfs
-r--r--r-- 1 root root    4096 Sep 28 19:41 sriov_totalvfs
-r--r--r-- 1 root root    4096 Sep 28 19:41 stride  # this is new
lrwxrwxrwx 1 root root       0 Sep 28 19:41 subsystem -> ../../../../bus/pci
-r--r--r-- 1 root root    4096 Sep 28 19:41 subsystem_device
-r--r--r-- 1 root root    4096 Sep 28 19:41 subsystem_vendor
-rw-r--r-- 1 root root    4096 Sep 28 19:41 uevent
-r--r--r-- 1 root root    4096 Sep 28 19:41 vendor
-r--r--r-- 1 root root    4096 Sep 28 19:41 vf_did  # this is new
lrwxrwxrwx 1 root root       0 Sep 29 09:44 virtfn0 -> ../0000:03:10.0

nothing changes on for VFs.
Then:

/sys/bus/pci/devices/0000:03:00.0# cat device =

0x10c9

/sys/bus/pci/devices/0000:03:00.0# cat vf_did
0x10ca

Putting the VF device ID in the PF device file would be a change of
that we expose to userspace.  Something might break.

vf_did provides a easy way to retrieve the VF device ID without reading
the PF config (looking up the SR-IOV capability and reading it) or without
enabling SR-IOV to read for example virtfn0/device.

Similar considerations (ease of access) apply to offset and stride.


>> static ssize_t sriov_drivers_autoprobe_show(struct device *dev,
>> 					    struct device_attribute *attr,
>> 					    char *buf)
>> @@ -676,6 +703,9 @@ static struct device_attribute sriov_totalvfs_attr =
=3D __ATTR_RO(sriov_totalvfs);
>> static struct device_attribute sriov_numvfs_attr =3D
>> 		__ATTR(sriov_numvfs, (S_IRUGO|S_IWUSR|S_IWGRP),
>> 		       sriov_numvfs_show, sriov_numvfs_store);
>> +static struct device_attribute sriov_offset_attr =3D __ATTR_RO(sriov_of=
fset);
>> +static struct device_attribute sriov_stride_attr =3D __ATTR_RO(sriov_st=
ride);
>> +static struct device_attribute sriov_vf_did_attr =3D __ATTR_RO(sriov_vf=
_did);
>> static struct device_attribute sriov_drivers_autoprobe_attr =3D
>> 		__ATTR(sriov_drivers_autoprobe, (S_IRUGO|S_IWUSR|S_IWGRP),
>> 		       sriov_drivers_autoprobe_show, sriov_drivers_autoprobe_store);
>> @@ -1744,6 +1774,9 @@ static struct attribute_group pci_dev_hp_attr_grou=
p =3D {
>> static struct attribute *sriov_dev_attrs[] =3D {
>> 	&sriov_totalvfs_attr.attr,
>> 	&sriov_numvfs_attr.attr,
>> +	&sriov_offset_attr.attr,
>> +	&sriov_stride_attr.attr,
>> +	&sriov_vf_did_attr.attr,
>> 	&sriov_drivers_autoprobe_attr.attr,
>> 	NULL,
>> };
>> -- =

>> 2.7.4
>> =

> =


Amazon Development Center Germany GmbH
Berlin - Dresden - Aachen
main office: Krausenstr. 38, 10117 Berlin
Geschaeftsfuehrer: Dr. Ralf Herbrich, Christian Schlaeger
Ust-ID: DE289237879
Eingetragen am Amtsgericht Charlottenburg HRB 149173 B

Re: [PATCH 2/2] pci: Expose offset, stride, and VF device ID via sysfs

[Bjorn Helgaas]

On Fri, Sep 29, 2017 at 07:53:31AM +0000, Sironi, Filippo wrote:
> 
> Hi Bjorn,
> 
> > On 25. Sep 2017, at 20:55, Bjorn Helgaas <helgaas@kernel.org> wrote:
> > 
> > Hi Filippo,
> > 
> > On Mon, Aug 28, 2017 at 03:38:50PM +0200, Filippo Sironi wrote:
> >> +static ssize_t sriov_vf_did_show(struct device *dev,
> >> +				 struct device_attribute *attr,
> >> +				 char *buf)
> >> +{
> >> +	struct pci_dev *pdev = to_pci_dev(dev);
> >> +
> >> +	return sprintf(buf, "%x\n", pdev->sriov->vf_did);
> >> +}
> > 
> > What does the vf_did part look like in sysfs?  Do we have a directory with
> > both "device" and "vf_did" in it?  If so, why do we have both and do we
> > need both?  Could we put the vf_did in the "device" file?
> 
> On my machine:
> 
> /sys/bus/pci/devices/0000:03:00.0# ls -l  # this is the PF
> total 0
> -rw-r--r-- 1 root root    4096 Sep 28 19:41 broken_parity_status
> -r--r--r-- 1 root root    4096 Sep 28 19:41 class
> -rw-r--r-- 1 root root    4096 Sep 28 19:41 config
> -r--r--r-- 1 root root    4096 Sep 28 19:41 consistent_dma_mask_bits
> -rw-r--r-- 1 root root    4096 Sep 28 19:41 d3cold_allowed
> -r--r--r-- 1 root root    4096 Sep 28 19:41 device
> -r--r--r-- 1 root root    4096 Sep 28 19:41 dma_mask_bits
> lrwxrwxrwx 1 root root       0 Sep 28 19:41 driver -> ../../../../bus/pci/drivers/igb
> -rw-r--r-- 1 root root    4096 Sep 28 19:41 driver_override
> -rw-r--r-- 1 root root    4096 Sep 28 19:41 enable
> lrwxrwxrwx 1 root root       0 Sep 28 19:41 firmware_node -> ../../../LNXSYSTM:00/LNXSYBUS:00/PNP0A08:00/device:4b/device:4c
> -r--r--r-- 1 root root    4096 Sep 28 19:41 irq
> -r--r--r-- 1 root root    4096 Sep 28 19:41 local_cpulist
> -r--r--r-- 1 root root    4096 Sep 28 19:41 local_cpus
> -r--r--r-- 1 root root    4096 Sep 28 19:41 modalias
> -rw-r--r-- 1 root root    4096 Sep 28 19:41 msi_bus
> drwxr-xr-x 2 root root       0 Sep 29 09:44 msi_irqs
> drwxr-xr-x 3 root root       0 Sep 28 19:41 net
> -rw-r--r-- 1 root root    4096 Sep 28 19:41 numa_node
> -r--r--r-- 1 root root    4096 Sep 28 19:41 offset  # this is new
> drwxr-xr-x 2 root root       0 Sep 28 19:41 power
> drwxr-xr-x 3 root root       0 Sep 28 19:41 ptp
> --w--w---- 1 root root    4096 Sep 28 19:41 remove
> --w--w---- 1 root root    4096 Sep 28 19:41 rescan
> --w------- 1 root root    4096 Sep 28 19:41 reset
> -r--r--r-- 1 root root    4096 Sep 28 19:41 resource
> -rw------- 1 root root  131072 Sep 28 19:41 resource0
> -rw------- 1 root root 4194304 Sep 28 19:41 resource1
> -rw------- 1 root root      32 Sep 28 19:41 resource2
> -rw------- 1 root root   16384 Sep 28 19:41 resource3
> -r--r--r-- 1 root root    4096 Sep 28 19:41 revision
> -rw-rw-r-- 1 root root    4096 Sep 29 09:44 sriov_numvfs
> -r--r--r-- 1 root root    4096 Sep 28 19:41 sriov_totalvfs
> -r--r--r-- 1 root root    4096 Sep 28 19:41 stride  # this is new
> lrwxrwxrwx 1 root root       0 Sep 28 19:41 subsystem -> ../../../../bus/pci
> -r--r--r-- 1 root root    4096 Sep 28 19:41 subsystem_device
> -r--r--r-- 1 root root    4096 Sep 28 19:41 subsystem_vendor
> -rw-r--r-- 1 root root    4096 Sep 28 19:41 uevent
> -r--r--r-- 1 root root    4096 Sep 28 19:41 vendor
> -r--r--r-- 1 root root    4096 Sep 28 19:41 vf_did  # this is new
> lrwxrwxrwx 1 root root       0 Sep 29 09:44 virtfn0 -> ../0000:03:10.0
> 
> nothing changes on for VFs.
> Then:
> 
> /sys/bus/pci/devices/0000:03:00.0# cat device 
> 0x10c9
> 
> /sys/bus/pci/devices/0000:03:00.0# cat vf_did
> 0x10ca
> 
> Putting the VF device ID in the PF device file would be a change of
> that we expose to userspace.  Something might break.

Oh, sorry, I misunderstood!  I was thinking that you were adding
"vf_did" to the VF directory, not the PF directory.

Then I guess my only issue is that "vf_did" doesn't match the pattern
of other names.  I think "virtfn_device" would give more of a hint and
would match "device" and "subsystem_device".

Bjorn

> vf_did provides a easy way to retrieve the VF device ID without reading
> the PF config (looking up the SR-IOV capability and reading it) or without
> enabling SR-IOV to read for example virtfn0/device.
> 
> Similar considerations (ease of access) apply to offset and stride.
> 
> 
> >> static ssize_t sriov_drivers_autoprobe_show(struct device *dev,
> >> 					    struct device_attribute *attr,
> >> 					    char *buf)
> >> @@ -676,6 +703,9 @@ static struct device_attribute sriov_totalvfs_attr = __ATTR_RO(sriov_totalvfs);
> >> static struct device_attribute sriov_numvfs_attr =
> >> 		__ATTR(sriov_numvfs, (S_IRUGO|S_IWUSR|S_IWGRP),
> >> 		       sriov_numvfs_show, sriov_numvfs_store);
> >> +static struct device_attribute sriov_offset_attr = __ATTR_RO(sriov_offset);
> >> +static struct device_attribute sriov_stride_attr = __ATTR_RO(sriov_stride);
> >> +static struct device_attribute sriov_vf_did_attr = __ATTR_RO(sriov_vf_did);
> >> static struct device_attribute sriov_drivers_autoprobe_attr =
> >> 		__ATTR(sriov_drivers_autoprobe, (S_IRUGO|S_IWUSR|S_IWGRP),
> >> 		       sriov_drivers_autoprobe_show, sriov_drivers_autoprobe_store);
> >> @@ -1744,6 +1774,9 @@ static struct attribute_group pci_dev_hp_attr_group = {
> >> static struct attribute *sriov_dev_attrs[] = {
> >> 	&sriov_totalvfs_attr.attr,
> >> 	&sriov_numvfs_attr.attr,
> >> +	&sriov_offset_attr.attr,
> >> +	&sriov_stride_attr.attr,
> >> +	&sriov_vf_did_attr.attr,
> >> 	&sriov_drivers_autoprobe_attr.attr,
> >> 	NULL,
> >> };
> >> -- 
> >> 2.7.4
> >> 
> > 
> 
> Amazon Development Center Germany GmbH
> Berlin - Dresden - Aachen
> main office: Krausenstr. 38, 10117 Berlin
> Geschaeftsfuehrer: Dr. Ralf Herbrich, Christian Schlaeger
> Ust-ID: DE289237879
> Eingetragen am Amtsgericht Charlottenburg HRB 149173 B
> 

Re: [PATCH 1/2] pci: Cache the VF device ID in the SR-IOV structure

[Bjorn Helgaas]

On Mon, Aug 28, 2017 at 03:38:49PM +0200, Filippo Sironi wrote:
> ... and use it instead of reading it over and over from the PF config
> space capability.
> 
> Signed-off-by: Filippo Sironi <sironi@amazon.de>
> Cc: linux-pci@vger.kernel.org
> Cc: linux-kernel@vger.kernel.org

Applied to pci/virtualization for v4.15.

I renamed vf_did to vf_device for consistency with other existing fields.

> ---
>  drivers/pci/iov.c | 5 +++--
>  drivers/pci/pci.h | 1 +
>  2 files changed, 4 insertions(+), 2 deletions(-)
> 
> diff --git a/drivers/pci/iov.c b/drivers/pci/iov.c
> index 120485d6f352..e8f7eafaba6a 100644
> --- a/drivers/pci/iov.c
> +++ b/drivers/pci/iov.c
> @@ -134,7 +134,7 @@ int pci_iov_add_virtfn(struct pci_dev *dev, int id, int reset)
>  
>  	virtfn->devfn = pci_iov_virtfn_devfn(dev, id);
>  	virtfn->vendor = dev->vendor;
> -	pci_read_config_word(dev, iov->pos + PCI_SRIOV_VF_DID, &virtfn->device);
> +	virtfn->device = iov->vf_did;
>  	rc = pci_setup_device(virtfn);
>  	if (rc)
>  		goto failed0;
> @@ -448,6 +448,7 @@ static int sriov_init(struct pci_dev *dev, int pos)
>  	iov->nres = nres;
>  	iov->ctrl = ctrl;
>  	iov->total_VFs = total;
> +	pci_read_config_word(dev, pos + PCI_SRIOV_VF_DID, &iov->vf_did);
>  	iov->pgsz = pgsz;
>  	iov->self = dev;
>  	iov->drivers_autoprobe = true;
> @@ -723,7 +724,7 @@ int pci_vfs_assigned(struct pci_dev *dev)
>  	 * determine the device ID for the VFs, the vendor ID will be the
>  	 * same as the PF so there is no need to check for that one
>  	 */
> -	pci_read_config_word(dev, dev->sriov->pos + PCI_SRIOV_VF_DID, &dev_id);
> +	dev_id = dev->sriov->vf_did;
>  
>  	/* loop through all the VFs to see if we own any that are assigned */
>  	vfdev = pci_get_device(dev->vendor, dev_id, NULL);
> diff --git a/drivers/pci/pci.h b/drivers/pci/pci.h
> index 22e061738c6f..a7270e11e1ef 100644
> --- a/drivers/pci/pci.h
> +++ b/drivers/pci/pci.h
> @@ -262,6 +262,7 @@ struct pci_sriov {
>  	u16 num_VFs;		/* number of VFs available */
>  	u16 offset;		/* first VF Routing ID offset */
>  	u16 stride;		/* following VF stride */
> +	u16 vf_did;		/* VF device ID */
>  	u32 pgsz;		/* page size for BAR alignment */
>  	u8 link;		/* Function Dependency Link */
>  	u8 max_VF_buses;	/* max buses consumed by VFs */
> -- 
> 2.7.4
> 

Re: [PATCH 2/2] pci: Expose offset, stride, and VF device ID via sysfs

[Bjorn Helgaas]

On Tue, Oct 03, 2017 at 02:31:14PM -0500, Bjorn Helgaas wrote:
> On Fri, Sep 29, 2017 at 07:53:31AM +0000, Sironi, Filippo wrote:
> > 
> > Hi Bjorn,
> > 
> > > On 25. Sep 2017, at 20:55, Bjorn Helgaas <helgaas@kernel.org> wrote:
> > > 
> > > Hi Filippo,
> > > 
> > > On Mon, Aug 28, 2017 at 03:38:50PM +0200, Filippo Sironi wrote:
> > >> +static ssize_t sriov_vf_did_show(struct device *dev,
> > >> +				 struct device_attribute *attr,
> > >> +				 char *buf)
> > >> +{
> > >> +	struct pci_dev *pdev = to_pci_dev(dev);
> > >> +
> > >> +	return sprintf(buf, "%x\n", pdev->sriov->vf_did);
> > >> +}
> > > 
> > > What does the vf_did part look like in sysfs?  Do we have a directory with
> > > both "device" and "vf_did" in it?  If so, why do we have both and do we
> > > need both?  Could we put the vf_did in the "device" file?
> > 
> > On my machine:
> > 
> > /sys/bus/pci/devices/0000:03:00.0# ls -l  # this is the PF
> > total 0
> > -rw-r--r-- 1 root root    4096 Sep 28 19:41 broken_parity_status
> > -r--r--r-- 1 root root    4096 Sep 28 19:41 class
> > -rw-r--r-- 1 root root    4096 Sep 28 19:41 config
> > -r--r--r-- 1 root root    4096 Sep 28 19:41 consistent_dma_mask_bits
> > -rw-r--r-- 1 root root    4096 Sep 28 19:41 d3cold_allowed
> > -r--r--r-- 1 root root    4096 Sep 28 19:41 device
> > -r--r--r-- 1 root root    4096 Sep 28 19:41 dma_mask_bits
> > lrwxrwxrwx 1 root root       0 Sep 28 19:41 driver -> ../../../../bus/pci/drivers/igb
> > -rw-r--r-- 1 root root    4096 Sep 28 19:41 driver_override
> > -rw-r--r-- 1 root root    4096 Sep 28 19:41 enable
> > lrwxrwxrwx 1 root root       0 Sep 28 19:41 firmware_node -> ../../../LNXSYSTM:00/LNXSYBUS:00/PNP0A08:00/device:4b/device:4c
> > -r--r--r-- 1 root root    4096 Sep 28 19:41 irq
> > -r--r--r-- 1 root root    4096 Sep 28 19:41 local_cpulist
> > -r--r--r-- 1 root root    4096 Sep 28 19:41 local_cpus
> > -r--r--r-- 1 root root    4096 Sep 28 19:41 modalias
> > -rw-r--r-- 1 root root    4096 Sep 28 19:41 msi_bus
> > drwxr-xr-x 2 root root       0 Sep 29 09:44 msi_irqs
> > drwxr-xr-x 3 root root       0 Sep 28 19:41 net
> > -rw-r--r-- 1 root root    4096 Sep 28 19:41 numa_node
> > -r--r--r-- 1 root root    4096 Sep 28 19:41 offset  # this is new
> > drwxr-xr-x 2 root root       0 Sep 28 19:41 power
> > drwxr-xr-x 3 root root       0 Sep 28 19:41 ptp
> > --w--w---- 1 root root    4096 Sep 28 19:41 remove
> > --w--w---- 1 root root    4096 Sep 28 19:41 rescan
> > --w------- 1 root root    4096 Sep 28 19:41 reset
> > -r--r--r-- 1 root root    4096 Sep 28 19:41 resource
> > -rw------- 1 root root  131072 Sep 28 19:41 resource0
> > -rw------- 1 root root 4194304 Sep 28 19:41 resource1
> > -rw------- 1 root root      32 Sep 28 19:41 resource2
> > -rw------- 1 root root   16384 Sep 28 19:41 resource3
> > -r--r--r-- 1 root root    4096 Sep 28 19:41 revision
> > -rw-rw-r-- 1 root root    4096 Sep 29 09:44 sriov_numvfs
> > -r--r--r-- 1 root root    4096 Sep 28 19:41 sriov_totalvfs
> > -r--r--r-- 1 root root    4096 Sep 28 19:41 stride  # this is new
> > lrwxrwxrwx 1 root root       0 Sep 28 19:41 subsystem -> ../../../../bus/pci
> > -r--r--r-- 1 root root    4096 Sep 28 19:41 subsystem_device
> > -r--r--r-- 1 root root    4096 Sep 28 19:41 subsystem_vendor
> > -rw-r--r-- 1 root root    4096 Sep 28 19:41 uevent
> > -r--r--r-- 1 root root    4096 Sep 28 19:41 vendor
> > -r--r--r-- 1 root root    4096 Sep 28 19:41 vf_did  # this is new
> > lrwxrwxrwx 1 root root       0 Sep 29 09:44 virtfn0 -> ../0000:03:10.0
> > 
> > nothing changes on for VFs.
> > Then:
> > 
> > /sys/bus/pci/devices/0000:03:00.0# cat device 
> > 0x10c9
> > 
> > /sys/bus/pci/devices/0000:03:00.0# cat vf_did
> > 0x10ca
> > 
> > Putting the VF device ID in the PF device file would be a change of
> > that we expose to userspace.  Something might break.
> 
> Oh, sorry, I misunderstood!  I was thinking that you were adding
> "vf_did" to the VF directory, not the PF directory.
> 
> Then I guess my only issue is that "vf_did" doesn't match the pattern
> of other names.  I think "virtfn_device" would give more of a hint and
> would match "device" and "subsystem_device".

I was going to just make this tweak myself, but realized I'd actually
propose these changes as well:

  offset -> sriov_offset
  stride -> sriov_stride
  vf_did -> virtfn_device (could be sriov_device as well)

and I can't really test it to make sure I get all the details right.
Can you update those and repost this?

Bjorn

> > vf_did provides a easy way to retrieve the VF device ID without reading
> > the PF config (looking up the SR-IOV capability and reading it) or without
> > enabling SR-IOV to read for example virtfn0/device.
> > 
> > Similar considerations (ease of access) apply to offset and stride.
> > 
> > 
> > >> static ssize_t sriov_drivers_autoprobe_show(struct device *dev,
> > >> 					    struct device_attribute *attr,
> > >> 					    char *buf)
> > >> @@ -676,6 +703,9 @@ static struct device_attribute sriov_totalvfs_attr = __ATTR_RO(sriov_totalvfs);
> > >> static struct device_attribute sriov_numvfs_attr =
> > >> 		__ATTR(sriov_numvfs, (S_IRUGO|S_IWUSR|S_IWGRP),
> > >> 		       sriov_numvfs_show, sriov_numvfs_store);
> > >> +static struct device_attribute sriov_offset_attr = __ATTR_RO(sriov_offset);
> > >> +static struct device_attribute sriov_stride_attr = __ATTR_RO(sriov_stride);
> > >> +static struct device_attribute sriov_vf_did_attr = __ATTR_RO(sriov_vf_did);
> > >> static struct device_attribute sriov_drivers_autoprobe_attr =
> > >> 		__ATTR(sriov_drivers_autoprobe, (S_IRUGO|S_IWUSR|S_IWGRP),
> > >> 		       sriov_drivers_autoprobe_show, sriov_drivers_autoprobe_store);
> > >> @@ -1744,6 +1774,9 @@ static struct attribute_group pci_dev_hp_attr_group = {
> > >> static struct attribute *sriov_dev_attrs[] = {
> > >> 	&sriov_totalvfs_attr.attr,
> > >> 	&sriov_numvfs_attr.attr,
> > >> +	&sriov_offset_attr.attr,
> > >> +	&sriov_stride_attr.attr,
> > >> +	&sriov_vf_did_attr.attr,
> > >> 	&sriov_drivers_autoprobe_attr.attr,
> > >> 	NULL,
> > >> };
> > >> -- 
> > >> 2.7.4
> > >> 
> > > 
> > 
> > Amazon Development Center Germany GmbH
> > Berlin - Dresden - Aachen
> > main office: Krausenstr. 38, 10117 Berlin
> > Geschaeftsfuehrer: Dr. Ralf Herbrich, Christian Schlaeger
> > Ust-ID: DE289237879
> > Eingetragen am Amtsgericht Charlottenburg HRB 149173 B
> > 

Re: [PATCH 2/2] pci: Expose offset, stride, and VF device ID via sysfs

[Sironi, Filippo]

> On 3. Oct 2017, at 21:48, Bjorn Helgaas <helgaas@kernel.org> wrote:
> 
> On Tue, Oct 03, 2017 at 02:31:14PM -0500, Bjorn Helgaas wrote:
>> On Fri, Sep 29, 2017 at 07:53:31AM +0000, Sironi, Filippo wrote:
>>> 
>>> Hi Bjorn,
>>> 
>>>> On 25. Sep 2017, at 20:55, Bjorn Helgaas <helgaas@kernel.org> wrote:
>>>> 
>>>> Hi Filippo,
>>>> 
>>>> On Mon, Aug 28, 2017 at 03:38:50PM +0200, Filippo Sironi wrote:
>>>>> +static ssize_t sriov_vf_did_show(struct device *dev,
>>>>> +				 struct device_attribute *attr,
>>>>> +				 char *buf)
>>>>> +{
>>>>> +	struct pci_dev *pdev = to_pci_dev(dev);
>>>>> +
>>>>> +	return sprintf(buf, "%x\n", pdev->sriov->vf_did);
>>>>> +}
>>>> 
>>>> What does the vf_did part look like in sysfs?  Do we have a directory with
>>>> both "device" and "vf_did" in it?  If so, why do we have both and do we
>>>> need both?  Could we put the vf_did in the "device" file?
>>> 
>>> On my machine:
>>> 
>>> /sys/bus/pci/devices/0000:03:00.0# ls -l  # this is the PF
>>> total 0
>>> -rw-r--r-- 1 root root    4096 Sep 28 19:41 broken_parity_status
>>> -r--r--r-- 1 root root    4096 Sep 28 19:41 class
>>> -rw-r--r-- 1 root root    4096 Sep 28 19:41 config
>>> -r--r--r-- 1 root root    4096 Sep 28 19:41 consistent_dma_mask_bits
>>> -rw-r--r-- 1 root root    4096 Sep 28 19:41 d3cold_allowed
>>> -r--r--r-- 1 root root    4096 Sep 28 19:41 device
>>> -r--r--r-- 1 root root    4096 Sep 28 19:41 dma_mask_bits
>>> lrwxrwxrwx 1 root root       0 Sep 28 19:41 driver -> ../../../../bus/pci/drivers/igb
>>> -rw-r--r-- 1 root root    4096 Sep 28 19:41 driver_override
>>> -rw-r--r-- 1 root root    4096 Sep 28 19:41 enable
>>> lrwxrwxrwx 1 root root       0 Sep 28 19:41 firmware_node -> ../../../LNXSYSTM:00/LNXSYBUS:00/PNP0A08:00/device:4b/device:4c
>>> -r--r--r-- 1 root root    4096 Sep 28 19:41 irq
>>> -r--r--r-- 1 root root    4096 Sep 28 19:41 local_cpulist
>>> -r--r--r-- 1 root root    4096 Sep 28 19:41 local_cpus
>>> -r--r--r-- 1 root root    4096 Sep 28 19:41 modalias
>>> -rw-r--r-- 1 root root    4096 Sep 28 19:41 msi_bus
>>> drwxr-xr-x 2 root root       0 Sep 29 09:44 msi_irqs
>>> drwxr-xr-x 3 root root       0 Sep 28 19:41 net
>>> -rw-r--r-- 1 root root    4096 Sep 28 19:41 numa_node
>>> -r--r--r-- 1 root root    4096 Sep 28 19:41 offset  # this is new
>>> drwxr-xr-x 2 root root       0 Sep 28 19:41 power
>>> drwxr-xr-x 3 root root       0 Sep 28 19:41 ptp
>>> --w--w---- 1 root root    4096 Sep 28 19:41 remove
>>> --w--w---- 1 root root    4096 Sep 28 19:41 rescan
>>> --w------- 1 root root    4096 Sep 28 19:41 reset
>>> -r--r--r-- 1 root root    4096 Sep 28 19:41 resource
>>> -rw------- 1 root root  131072 Sep 28 19:41 resource0
>>> -rw------- 1 root root 4194304 Sep 28 19:41 resource1
>>> -rw------- 1 root root      32 Sep 28 19:41 resource2
>>> -rw------- 1 root root   16384 Sep 28 19:41 resource3
>>> -r--r--r-- 1 root root    4096 Sep 28 19:41 revision
>>> -rw-rw-r-- 1 root root    4096 Sep 29 09:44 sriov_numvfs
>>> -r--r--r-- 1 root root    4096 Sep 28 19:41 sriov_totalvfs
>>> -r--r--r-- 1 root root    4096 Sep 28 19:41 stride  # this is new
>>> lrwxrwxrwx 1 root root       0 Sep 28 19:41 subsystem -> ../../../../bus/pci
>>> -r--r--r-- 1 root root    4096 Sep 28 19:41 subsystem_device
>>> -r--r--r-- 1 root root    4096 Sep 28 19:41 subsystem_vendor
>>> -rw-r--r-- 1 root root    4096 Sep 28 19:41 uevent
>>> -r--r--r-- 1 root root    4096 Sep 28 19:41 vendor
>>> -r--r--r-- 1 root root    4096 Sep 28 19:41 vf_did  # this is new
>>> lrwxrwxrwx 1 root root       0 Sep 29 09:44 virtfn0 -> ../0000:03:10.0
>>> 
>>> nothing changes on for VFs.
>>> Then:
>>> 
>>> /sys/bus/pci/devices/0000:03:00.0# cat device 
>>> 0x10c9
>>> 
>>> /sys/bus/pci/devices/0000:03:00.0# cat vf_did
>>> 0x10ca
>>> 
>>> Putting the VF device ID in the PF device file would be a change of
>>> that we expose to userspace.  Something might break.
>> 
>> Oh, sorry, I misunderstood!  I was thinking that you were adding
>> "vf_did" to the VF directory, not the PF directory.
>> 
>> Then I guess my only issue is that "vf_did" doesn't match the pattern
>> of other names.  I think "virtfn_device" would give more of a hint and
>> would match "device" and "subsystem_device".
> 
> I was going to just make this tweak myself, but realized I'd actually
> propose these changes as well:
> 
>  offset -> sriov_offset
>  stride -> sriov_stride
>  vf_did -> virtfn_device (could be sriov_device as well)
> 
> and I can't really test it to make sure I get all the details right.
> Can you update those and repost this?
> 
> Bjorn

I'll give this a spin tomorrow.

Thanks,
Filippo


>>> vf_did provides a easy way to retrieve the VF device ID without reading
>>> the PF config (looking up the SR-IOV capability and reading it) or without
>>> enabling SR-IOV to read for example virtfn0/device.
>>> 
>>> Similar considerations (ease of access) apply to offset and stride.
>>> 
>>> 
>>>>> static ssize_t sriov_drivers_autoprobe_show(struct device *dev,
>>>>> 					    struct device_attribute *attr,
>>>>> 					    char *buf)
>>>>> @@ -676,6 +703,9 @@ static struct device_attribute sriov_totalvfs_attr = __ATTR_RO(sriov_totalvfs);
>>>>> static struct device_attribute sriov_numvfs_attr =
>>>>> 		__ATTR(sriov_numvfs, (S_IRUGO|S_IWUSR|S_IWGRP),
>>>>> 		       sriov_numvfs_show, sriov_numvfs_store);
>>>>> +static struct device_attribute sriov_offset_attr = __ATTR_RO(sriov_offset);
>>>>> +static struct device_attribute sriov_stride_attr = __ATTR_RO(sriov_stride);
>>>>> +static struct device_attribute sriov_vf_did_attr = __ATTR_RO(sriov_vf_did);
>>>>> static struct device_attribute sriov_drivers_autoprobe_attr =
>>>>> 		__ATTR(sriov_drivers_autoprobe, (S_IRUGO|S_IWUSR|S_IWGRP),
>>>>> 		       sriov_drivers_autoprobe_show, sriov_drivers_autoprobe_store);
>>>>> @@ -1744,6 +1774,9 @@ static struct attribute_group pci_dev_hp_attr_group = {
>>>>> static struct attribute *sriov_dev_attrs[] = {
>>>>> 	&sriov_totalvfs_attr.attr,
>>>>> 	&sriov_numvfs_attr.attr,
>>>>> +	&sriov_offset_attr.attr,
>>>>> +	&sriov_stride_attr.attr,
>>>>> +	&sriov_vf_did_attr.attr,
>>>>> 	&sriov_drivers_autoprobe_attr.attr,
>>>>> 	NULL,
>>>>> };
>>>>> -- 
>>>>> 2.7.4
>>>>> 
>>>> 
>>> 
>>> Amazon Development Center Germany GmbH
>>> Berlin - Dresden - Aachen
>>> main office: Krausenstr. 38, 10117 Berlin
>>> Geschaeftsfuehrer: Dr. Ralf Herbrich, Christian Schlaeger
>>> Ust-ID: DE289237879
>>> Eingetragen am Amtsgericht Charlottenburg HRB 149173 B

Amazon Development Center Germany GmbH
Berlin - Dresden - Aachen
main office: Krausenstr. 38, 10117 Berlin
Geschaeftsfuehrer: Dr. Ralf Herbrich, Christian Schlaeger
Ust-ID: DE289237879
Eingetragen am Amtsgericht Charlottenburg HRB 149173 B

Re: [PATCH 2/2] pci: Expose offset, stride, and VF device ID via sysfs

[Sironi, Filippo]

> On 3. Oct 2017, at 21:48, Bjorn Helgaas <helgaas@kernel.org> wrote:
> =

> On Tue, Oct 03, 2017 at 02:31:14PM -0500, Bjorn Helgaas wrote:
>> On Fri, Sep 29, 2017 at 07:53:31AM +0000, Sironi, Filippo wrote:
>>> =

>>> Hi Bjorn,
>>> =

>>>> On 25. Sep 2017, at 20:55, Bjorn Helgaas <helgaas@kernel.org> wrote:
>>>> =

>>>> Hi Filippo,
>>>> =

>>>> On Mon, Aug 28, 2017 at 03:38:50PM +0200, Filippo Sironi wrote:
>>>>> +static ssize_t sriov_vf_did_show(struct device *dev,
>>>>> +				 struct device_attribute *attr,
>>>>> +				 char *buf)
>>>>> +{
>>>>> +	struct pci_dev *pdev =3D to_pci_dev(dev);
>>>>> +
>>>>> +	return sprintf(buf, "%x\n", pdev->sriov->vf_did);
>>>>> +}
>>>> =

>>>> What does the vf_did part look like in sysfs?  Do we have a directory =
with
>>>> both "device" and "vf_did" in it?  If so, why do we have both and do we
>>>> need both?  Could we put the vf_did in the "device" file?
>>> =

>>> On my machine:
>>> =

>>> /sys/bus/pci/devices/0000:03:00.0# ls -l  # this is the PF
>>> total 0
>>> -rw-r--r-- 1 root root    4096 Sep 28 19:41 broken_parity_status
>>> -r--r--r-- 1 root root    4096 Sep 28 19:41 class
>>> -rw-r--r-- 1 root root    4096 Sep 28 19:41 config
>>> -r--r--r-- 1 root root    4096 Sep 28 19:41 consistent_dma_mask_bits
>>> -rw-r--r-- 1 root root    4096 Sep 28 19:41 d3cold_allowed
>>> -r--r--r-- 1 root root    4096 Sep 28 19:41 device
>>> -r--r--r-- 1 root root    4096 Sep 28 19:41 dma_mask_bits
>>> lrwxrwxrwx 1 root root       0 Sep 28 19:41 driver -> ../../../../bus/p=
ci/drivers/igb
>>> -rw-r--r-- 1 root root    4096 Sep 28 19:41 driver_override
>>> -rw-r--r-- 1 root root    4096 Sep 28 19:41 enable
>>> lrwxrwxrwx 1 root root       0 Sep 28 19:41 firmware_node -> ../../../L=
NXSYSTM:00/LNXSYBUS:00/PNP0A08:00/device:4b/device:4c
>>> -r--r--r-- 1 root root    4096 Sep 28 19:41 irq
>>> -r--r--r-- 1 root root    4096 Sep 28 19:41 local_cpulist
>>> -r--r--r-- 1 root root    4096 Sep 28 19:41 local_cpus
>>> -r--r--r-- 1 root root    4096 Sep 28 19:41 modalias
>>> -rw-r--r-- 1 root root    4096 Sep 28 19:41 msi_bus
>>> drwxr-xr-x 2 root root       0 Sep 29 09:44 msi_irqs
>>> drwxr-xr-x 3 root root       0 Sep 28 19:41 net
>>> -rw-r--r-- 1 root root    4096 Sep 28 19:41 numa_node
>>> -r--r--r-- 1 root root    4096 Sep 28 19:41 offset  # this is new
>>> drwxr-xr-x 2 root root       0 Sep 28 19:41 power
>>> drwxr-xr-x 3 root root       0 Sep 28 19:41 ptp
>>> --w--w---- 1 root root    4096 Sep 28 19:41 remove
>>> --w--w---- 1 root root    4096 Sep 28 19:41 rescan
>>> --w------- 1 root root    4096 Sep 28 19:41 reset
>>> -r--r--r-- 1 root root    4096 Sep 28 19:41 resource
>>> -rw------- 1 root root  131072 Sep 28 19:41 resource0
>>> -rw------- 1 root root 4194304 Sep 28 19:41 resource1
>>> -rw------- 1 root root      32 Sep 28 19:41 resource2
>>> -rw------- 1 root root   16384 Sep 28 19:41 resource3
>>> -r--r--r-- 1 root root    4096 Sep 28 19:41 revision
>>> -rw-rw-r-- 1 root root    4096 Sep 29 09:44 sriov_numvfs
>>> -r--r--r-- 1 root root    4096 Sep 28 19:41 sriov_totalvfs
>>> -r--r--r-- 1 root root    4096 Sep 28 19:41 stride  # this is new
>>> lrwxrwxrwx 1 root root       0 Sep 28 19:41 subsystem -> ../../../../bu=
s/pci
>>> -r--r--r-- 1 root root    4096 Sep 28 19:41 subsystem_device
>>> -r--r--r-- 1 root root    4096 Sep 28 19:41 subsystem_vendor
>>> -rw-r--r-- 1 root root    4096 Sep 28 19:41 uevent
>>> -r--r--r-- 1 root root    4096 Sep 28 19:41 vendor
>>> -r--r--r-- 1 root root    4096 Sep 28 19:41 vf_did  # this is new
>>> lrwxrwxrwx 1 root root       0 Sep 29 09:44 virtfn0 -> ../0000:03:10.0
>>> =

>>> nothing changes on for VFs.
>>> Then:
>>> =

>>> /sys/bus/pci/devices/0000:03:00.0# cat device =

>>> 0x10c9
>>> =

>>> /sys/bus/pci/devices/0000:03:00.0# cat vf_did
>>> 0x10ca
>>> =

>>> Putting the VF device ID in the PF device file would be a change of
>>> that we expose to userspace.  Something might break.
>> =

>> Oh, sorry, I misunderstood!  I was thinking that you were adding
>> "vf_did" to the VF directory, not the PF directory.
>> =

>> Then I guess my only issue is that "vf_did" doesn't match the pattern
>> of other names.  I think "virtfn_device" would give more of a hint and
>> would match "device" and "subsystem_device".
> =

> I was going to just make this tweak myself, but realized I'd actually
> propose these changes as well:
> =

>  offset -> sriov_offset
>  stride -> sriov_stride
>  vf_did -> virtfn_device (could be sriov_device as well)
> =

> and I can't really test it to make sure I get all the details right.
> Can you update those and repost this?
> =

> Bjorn

I'll give this a spin tomorrow.

Thanks,
Filippo


>>> vf_did provides a easy way to retrieve the VF device ID without reading
>>> the PF config (looking up the SR-IOV capability and reading it) or with=
out
>>> enabling SR-IOV to read for example virtfn0/device.
>>> =

>>> Similar considerations (ease of access) apply to offset and stride.
>>> =

>>> =

>>>>> static ssize_t sriov_drivers_autoprobe_show(struct device *dev,
>>>>> 					    struct device_attribute *attr,
>>>>> 					    char *buf)
>>>>> @@ -676,6 +703,9 @@ static struct device_attribute sriov_totalvfs_att=
r =3D __ATTR_RO(sriov_totalvfs);
>>>>> static struct device_attribute sriov_numvfs_attr =3D
>>>>> 		__ATTR(sriov_numvfs, (S_IRUGO|S_IWUSR|S_IWGRP),
>>>>> 		       sriov_numvfs_show, sriov_numvfs_store);
>>>>> +static struct device_attribute sriov_offset_attr =3D __ATTR_RO(sriov=
_offset);
>>>>> +static struct device_attribute sriov_stride_attr =3D __ATTR_RO(sriov=
_stride);
>>>>> +static struct device_attribute sriov_vf_did_attr =3D __ATTR_RO(sriov=
_vf_did);
>>>>> static struct device_attribute sriov_drivers_autoprobe_attr =3D
>>>>> 		__ATTR(sriov_drivers_autoprobe, (S_IRUGO|S_IWUSR|S_IWGRP),
>>>>> 		       sriov_drivers_autoprobe_show, sriov_drivers_autoprobe_store);
>>>>> @@ -1744,6 +1774,9 @@ static struct attribute_group pci_dev_hp_attr_g=
roup =3D {
>>>>> static struct attribute *sriov_dev_attrs[] =3D {
>>>>> 	&sriov_totalvfs_attr.attr,
>>>>> 	&sriov_numvfs_attr.attr,
>>>>> +	&sriov_offset_attr.attr,
>>>>> +	&sriov_stride_attr.attr,
>>>>> +	&sriov_vf_did_attr.attr,
>>>>> 	&sriov_drivers_autoprobe_attr.attr,
>>>>> 	NULL,
>>>>> };
>>>>> -- =

>>>>> 2.7.4
>>>>> =

>>>> =

>>> =

>>> Amazon Development Center Germany GmbH
>>> Berlin - Dresden - Aachen
>>> main office: Krausenstr. 38, 10117 Berlin
>>> Geschaeftsfuehrer: Dr. Ralf Herbrich, Christian Schlaeger
>>> Ust-ID: DE289237879
>>> Eingetragen am Amtsgericht Charlottenburg HRB 149173 B

Amazon Development Center Germany GmbH
Berlin - Dresden - Aachen
main office: Krausenstr. 38, 10117 Berlin
Geschaeftsfuehrer: Dr. Ralf Herbrich, Christian Schlaeger
Ust-ID: DE289237879
Eingetragen am Amtsgericht Charlottenburg HRB 149173 B

[PATCH v2] pci: Expose offset, stride, and VF device ID via sysfs

[Filippo Sironi]

Testing done:

$ ls -l /sys/bus/pci/devices/0000\:03\:00.0/
total 0
-rw-r--r-- 1 root root    4096 Oct  9 00:48 broken_parity_status
-r--r--r-- 1 root root    4096 Oct  9 00:48 class
-rw-r--r-- 1 root root    4096 Oct  9 00:46 config
-r--r--r-- 1 root root    4096 Oct  9 00:48 consistent_dma_mask_bits
-r--r--r-- 1 root root    4096 Oct  9 00:48 current_link_speed
-r--r--r-- 1 root root    4096 Oct  9 00:48 current_link_width
-rw-r--r-- 1 root root    4096 Oct  9 00:48 d3cold_allowed
-r--r--r-- 1 root root    4096 Oct  9 00:46 device
-r--r--r-- 1 root root    4096 Oct  9 00:48 dma_mask_bits
lrwxrwxrwx 1 root root       0 Oct  9 00:46 driver -> ../../../../bus/pci/drivers/igb
-rw-r--r-- 1 root root    4096 Oct  9 00:48 driver_override
-rw-r--r-- 1 root root    4096 Oct  9 00:48 enable
lrwxrwxrwx 1 root root       0 Oct  9 00:48 firmware_node -> ../../../LNXSYSTM:00/LNXSYBUS:00/PNP0A08:00/device:4b/device:4c
-r--r--r-- 1 root root    4096 Oct  9 00:46 irq
-r--r--r-- 1 root root    4096 Oct  9 00:48 local_cpulist
-r--r--r-- 1 root root    4096 Oct  9 00:48 local_cpus
-r--r--r-- 1 root root    4096 Oct  9 00:48 max_link_speed
-r--r--r-- 1 root root    4096 Oct  9 00:48 max_link_width
-r--r--r-- 1 root root    4096 Oct  9 00:48 modalias
-rw-r--r-- 1 root root    4096 Oct  9 00:48 msi_bus
drwxr-xr-x 2 root root       0 Oct  9 00:48 msi_irqs
drwxr-xr-x 3 root root       0 Oct  9 00:46 net
-rw-r--r-- 1 root root    4096 Oct  9 00:48 numa_node
drwxr-xr-x 2 root root       0 Oct  9 00:48 power
drwxr-xr-x 3 root root       0 Oct  9 00:46 ptp
--w--w---- 1 root root    4096 Oct  9 00:48 remove
--w--w---- 1 root root    4096 Oct  9 00:48 rescan
--w------- 1 root root    4096 Oct  9 00:48 reset
-r--r--r-- 1 root root    4096 Oct  9 00:46 resource
-rw------- 1 root root  131072 Oct  9 00:48 resource0
-rw------- 1 root root 4194304 Oct  9 00:48 resource1
-rw------- 1 root root      32 Oct  9 00:48 resource2
-rw------- 1 root root   16384 Oct  9 00:48 resource3
-r--r--r-- 1 root root    4096 Oct  9 00:48 revision
-rw-rw-r-- 1 root root    4096 Oct  9 00:48 sriov_drivers_autoprobe
-rw-rw-r-- 1 root root    4096 Oct  9 00:48 sriov_numvfs
-r--r--r-- 1 root root    4096 Oct  9 00:48 sriov_offset
-r--r--r-- 1 root root    4096 Oct  9 00:48 sriov_stride
-r--r--r-- 1 root root    4096 Oct  9 00:48 sriov_totalvfs
-r--r--r-- 1 root root    4096 Oct  9 00:48 sriov_vf_device
lrwxrwxrwx 1 root root       0 Oct  9 00:46 subsystem -> ../../../../bus/pci
-r--r--r-- 1 root root    4096 Oct  9 00:48 subsystem_device
-r--r--r-- 1 root root    4096 Oct  9 00:48 subsystem_vendor
-rw-r--r-- 1 root root    4096 Oct  9 00:46 uevent
-r--r--r-- 1 root root    4096 Oct  9 00:46 vendor
$ cat /sys/bus/pci/devices/0000\:03\:00.0/sriov_offset
128
$ cat /sys/bus/pci/devices/0000\:03\:00.0/sriov_stride
2
$ cat /sys/bus/pci/devices/0000\:03\:00.0/sriov_vf_device 
10ca

Filippo Sironi (1):
  pci: Expose offset, stride, and VF device ID via sysfs

 drivers/pci/pci-sysfs.c | 33 +++++++++++++++++++++++++++++++++
 1 file changed, 33 insertions(+)

-- 
2.7.4

[PATCH v2] pci: Expose offset, stride, and VF device ID via sysfs

[Filippo Sironi]

... to make it easier for userspace applications to consume them.

Signed-off-by: Filippo Sironi <sironi@amazon.de>
Cc: Bjorn Helgaas <helgaas@kernel.org>
Cc: linux-pci@vger.kernel.org
Cc: linux-kernel@vger.kernel.org
---
v2:
* follow up with the rename of vf_did to vf_device

 drivers/pci/pci-sysfs.c | 33 +++++++++++++++++++++++++++++++++
 1 file changed, 33 insertions(+)

diff --git a/drivers/pci/pci-sysfs.c b/drivers/pci/pci-sysfs.c
index 2f3780b50723..e6f4133f8992 100644
--- a/drivers/pci/pci-sysfs.c
+++ b/drivers/pci/pci-sysfs.c
@@ -648,6 +648,33 @@ static ssize_t sriov_numvfs_store(struct device *dev,
 	return count;
 }
 
+static ssize_t sriov_offset_show(struct device *dev,
+				 struct device_attribute *attr,
+				 char *buf)
+{
+	struct pci_dev *pdev = to_pci_dev(dev);
+
+	return sprintf(buf, "%u\n", pdev->sriov->offset);
+}
+
+static ssize_t sriov_stride_show(struct device *dev,
+				 struct device_attribute *attr,
+				 char *buf)
+{
+	struct pci_dev *pdev = to_pci_dev(dev);
+
+	return sprintf(buf, "%u\n", pdev->sriov->stride);
+}
+
+static ssize_t sriov_vf_device_show(struct device *dev,
+				    struct device_attribute *attr,
+				    char *buf)
+{
+	struct pci_dev *pdev = to_pci_dev(dev);
+
+	return sprintf(buf, "%x\n", pdev->sriov->vf_device);
+}
+
 static ssize_t sriov_drivers_autoprobe_show(struct device *dev,
 					    struct device_attribute *attr,
 					    char *buf)
@@ -676,6 +703,9 @@ static struct device_attribute sriov_totalvfs_attr = __ATTR_RO(sriov_totalvfs);
 static struct device_attribute sriov_numvfs_attr =
 		__ATTR(sriov_numvfs, (S_IRUGO|S_IWUSR|S_IWGRP),
 		       sriov_numvfs_show, sriov_numvfs_store);
+static struct device_attribute sriov_offset_attr = __ATTR_RO(sriov_offset);
+static struct device_attribute sriov_stride_attr = __ATTR_RO(sriov_stride);
+static struct device_attribute sriov_vf_device_attr = __ATTR_RO(sriov_vf_device);
 static struct device_attribute sriov_drivers_autoprobe_attr =
 		__ATTR(sriov_drivers_autoprobe, (S_IRUGO|S_IWUSR|S_IWGRP),
 		       sriov_drivers_autoprobe_show, sriov_drivers_autoprobe_store);
@@ -1744,6 +1774,9 @@ static struct attribute_group pci_dev_hp_attr_group = {
 static struct attribute *sriov_dev_attrs[] = {
 	&sriov_totalvfs_attr.attr,
 	&sriov_numvfs_attr.attr,
+	&sriov_offset_attr.attr,
+	&sriov_stride_attr.attr,
+	&sriov_vf_device_attr.attr,
 	&sriov_drivers_autoprobe_attr.attr,
 	NULL,
 };
-- 
2.7.4

Re: [PATCH v2] pci: Expose offset, stride, and VF device ID via sysfs

[Bjorn Helgaas]

On Mon, Oct 09, 2017 at 01:09:11AM +0200, Filippo Sironi wrote:
> ... to make it easier for userspace applications to consume them.
> 
> Signed-off-by: Filippo Sironi <sironi@amazon.de>
> Cc: Bjorn Helgaas <helgaas@kernel.org>
> Cc: linux-pci@vger.kernel.org
> Cc: linux-kernel@vger.kernel.org

Applied to pci/virtualization for v4.15, thanks!

> ---
> v2:
> * follow up with the rename of vf_did to vf_device
> 
>  drivers/pci/pci-sysfs.c | 33 +++++++++++++++++++++++++++++++++
>  1 file changed, 33 insertions(+)
> 
> diff --git a/drivers/pci/pci-sysfs.c b/drivers/pci/pci-sysfs.c
> index 2f3780b50723..e6f4133f8992 100644
> --- a/drivers/pci/pci-sysfs.c
> +++ b/drivers/pci/pci-sysfs.c
> @@ -648,6 +648,33 @@ static ssize_t sriov_numvfs_store(struct device *dev,
>  	return count;
>  }
>  
> +static ssize_t sriov_offset_show(struct device *dev,
> +				 struct device_attribute *attr,
> +				 char *buf)
> +{
> +	struct pci_dev *pdev = to_pci_dev(dev);
> +
> +	return sprintf(buf, "%u\n", pdev->sriov->offset);
> +}
> +
> +static ssize_t sriov_stride_show(struct device *dev,
> +				 struct device_attribute *attr,
> +				 char *buf)
> +{
> +	struct pci_dev *pdev = to_pci_dev(dev);
> +
> +	return sprintf(buf, "%u\n", pdev->sriov->stride);
> +}
> +
> +static ssize_t sriov_vf_device_show(struct device *dev,
> +				    struct device_attribute *attr,
> +				    char *buf)
> +{
> +	struct pci_dev *pdev = to_pci_dev(dev);
> +
> +	return sprintf(buf, "%x\n", pdev->sriov->vf_device);
> +}
> +
>  static ssize_t sriov_drivers_autoprobe_show(struct device *dev,
>  					    struct device_attribute *attr,
>  					    char *buf)
> @@ -676,6 +703,9 @@ static struct device_attribute sriov_totalvfs_attr = __ATTR_RO(sriov_totalvfs);
>  static struct device_attribute sriov_numvfs_attr =
>  		__ATTR(sriov_numvfs, (S_IRUGO|S_IWUSR|S_IWGRP),
>  		       sriov_numvfs_show, sriov_numvfs_store);
> +static struct device_attribute sriov_offset_attr = __ATTR_RO(sriov_offset);
> +static struct device_attribute sriov_stride_attr = __ATTR_RO(sriov_stride);
> +static struct device_attribute sriov_vf_device_attr = __ATTR_RO(sriov_vf_device);
>  static struct device_attribute sriov_drivers_autoprobe_attr =
>  		__ATTR(sriov_drivers_autoprobe, (S_IRUGO|S_IWUSR|S_IWGRP),
>  		       sriov_drivers_autoprobe_show, sriov_drivers_autoprobe_store);
> @@ -1744,6 +1774,9 @@ static struct attribute_group pci_dev_hp_attr_group = {
>  static struct attribute *sriov_dev_attrs[] = {
>  	&sriov_totalvfs_attr.attr,
>  	&sriov_numvfs_attr.attr,
> +	&sriov_offset_attr.attr,
> +	&sriov_stride_attr.attr,
> +	&sriov_vf_device_attr.attr,
>  	&sriov_drivers_autoprobe_attr.attr,
>  	NULL,
>  };
> -- 
> 2.7.4
>