[RFC] igb_uio: deprecate iomem and ioport mapping

[RFC] igb_uio: deprecate iomem and ioport mapping

[Jianfeng Tan]

Previously in igb_uio, iomem is mapped, and both ioport and io mem
are recorded into uio framework, which is duplicated and makes the
code too complex.

For iomem, DPDK user space code never opens or reads files under
/sys/pci/bus/devices/xxxx:xx:xx.x/uio/uioY/maps/. Instead,
/sys/pci/bus/devices/xxxx:xx:xx.x/resourceY are used to map device
memory.

For ioport, non-x86 platforms cannot read from files under
/sys/pci/bus/devices/xxxx:xx:xx.x/uio/uioY/portio/ directly, because
non-x86 platforms need to map port region for access in user space,
see non-x86 version pci_uio_ioport_map(). x86 platforms can use the
the same way as uio_pci_generic.

This patch deprecates iomem and ioport mapping in igb_uio kernel
module, and adjusts the iomem implementation in both igb_uio and
uio_pci_generic:
  - for x86 platform, get ports info from /proc/ioports;
  - for non-x86 platform, map and get ports info by pci_uio_ioport_map().

Note: this will affect those applications who are using files under
/sys/pci/bus/devices/xxxx:xx:xx.x/uio/uioY/maps/ and
/sys/pci/bus/devices/xxxx:xx:xx.x/uio/uioY/portio/.

Signed-off-by: Jianfeng Tan <jianfeng.tan@intel.com>
---
 lib/librte_eal/linuxapp/eal/eal_pci.c     |   4 -
 lib/librte_eal/linuxapp/eal/eal_pci_uio.c |  56 +-------------
 lib/librte_eal/linuxapp/igb_uio/igb_uio.c | 119 ++----------------------------
 3 files changed, 9 insertions(+), 170 deletions(-)

diff --git a/lib/librte_eal/linuxapp/eal/eal_pci.c b/lib/librte_eal/linuxapp/eal/eal_pci.c
index cd9de7c..f23e99d 100644
--- a/lib/librte_eal/linuxapp/eal/eal_pci.c
+++ b/lib/librte_eal/linuxapp/eal/eal_pci.c
@@ -629,8 +629,6 @@ rte_eal_pci_ioport_map(struct rte_pci_device *dev, int bar,
 		break;
 #endif
 	case RTE_KDRV_IGB_UIO:
-		ret = pci_uio_ioport_map(dev, bar, p);
-		break;
 	case RTE_KDRV_UIO_GENERIC:
 #if defined(RTE_ARCH_X86)
 		ret = pci_ioport_map(dev, bar, p);
@@ -718,8 +716,6 @@ rte_eal_pci_ioport_unmap(struct rte_pci_ioport *p)
 		break;
 #endif
 	case RTE_KDRV_IGB_UIO:
-		ret = pci_uio_ioport_unmap(p);
-		break;
 	case RTE_KDRV_UIO_GENERIC:
 #if defined(RTE_ARCH_X86)
 		ret = 0;
diff --git a/lib/librte_eal/linuxapp/eal/eal_pci_uio.c b/lib/librte_eal/linuxapp/eal/eal_pci_uio.c
index 1786b75..28d09ed 100644
--- a/lib/librte_eal/linuxapp/eal/eal_pci_uio.c
+++ b/lib/librte_eal/linuxapp/eal/eal_pci_uio.c
@@ -370,53 +370,7 @@ error:
 	return -1;
 }
 
-#if defined(RTE_ARCH_X86)
-int
-pci_uio_ioport_map(struct rte_pci_device *dev, int bar,
-		   struct rte_pci_ioport *p)
-{
-	char dirname[PATH_MAX];
-	char filename[PATH_MAX];
-	int uio_num;
-	unsigned long start;
-
-	uio_num = pci_get_uio_dev(dev, dirname, sizeof(dirname), 0);
-	if (uio_num < 0)
-		return -1;
-
-	/* get portio start */
-	snprintf(filename, sizeof(filename),
-		 "%s/portio/port%d/start", dirname, bar);
-	if (eal_parse_sysfs_value(filename, &start) < 0) {
-		RTE_LOG(ERR, EAL, "%s(): cannot parse portio start\n",
-			__func__);
-		return -1;
-	}
-	/* ensure we don't get anything funny here, read/write will cast to
-	 * uin16_t */
-	if (start > UINT16_MAX)
-		return -1;
-
-	/* FIXME only for primary process ? */
-	if (dev->intr_handle.type == RTE_INTR_HANDLE_UNKNOWN) {
-
-		snprintf(filename, sizeof(filename), "/dev/uio%u", uio_num);
-		dev->intr_handle.fd = open(filename, O_RDWR);
-		if (dev->intr_handle.fd < 0) {
-			RTE_LOG(ERR, EAL, "Cannot open %s: %s\n",
-				filename, strerror(errno));
-			return -1;
-		}
-		dev->intr_handle.type = RTE_INTR_HANDLE_UIO;
-	}
-
-	RTE_LOG(DEBUG, EAL, "PCI Port IO found start=0x%lx\n", start);
-
-	p->base = start;
-	p->len = 0;
-	return 0;
-}
-#else
+#if !defined(RTE_ARCH_X86)
 int
 pci_uio_ioport_map(struct rte_pci_device *dev, int bar,
 		   struct rte_pci_ioport *p)
@@ -553,14 +507,10 @@ pci_uio_ioport_write(struct rte_pci_ioport *p,
 	}
 }
 
+#if !defined(RTE_ARCH_X86)
 int
 pci_uio_ioport_unmap(struct rte_pci_ioport *p)
 {
-#if defined(RTE_ARCH_X86)
-	RTE_SET_USED(p);
-	/* FIXME close intr fd ? */
-	return 0;
-#else
 	return munmap((void *)(uintptr_t)p->base, p->len);
-#endif
 }
+#endif
diff --git a/lib/librte_eal/linuxapp/igb_uio/igb_uio.c b/lib/librte_eal/linuxapp/igb_uio/igb_uio.c
index df41e45..e9d78fb 100644
--- a/lib/librte_eal/linuxapp/igb_uio/igb_uio.c
+++ b/lib/librte_eal/linuxapp/igb_uio/igb_uio.c
@@ -216,107 +216,6 @@ igbuio_dom0_pci_mmap(struct uio_info *info, struct vm_area_struct *vma)
 }
 #endif
 
-/* Remap pci resources described by bar #pci_bar in uio resource n. */
-static int
-igbuio_pci_setup_iomem(struct pci_dev *dev, struct uio_info *info,
-		       int n, int pci_bar, const char *name)
-{
-	unsigned long addr, len;
-	void *internal_addr;
-
-	if (n >= ARRAY_SIZE(info->mem))
-		return -EINVAL;
-
-	addr = pci_resource_start(dev, pci_bar);
-	len = pci_resource_len(dev, pci_bar);
-	if (addr == 0 || len == 0)
-		return -1;
-	internal_addr = ioremap(addr, len);
-	if (internal_addr == NULL)
-		return -1;
-	info->mem[n].name = name;
-	info->mem[n].addr = addr;
-	info->mem[n].internal_addr = internal_addr;
-	info->mem[n].size = len;
-	info->mem[n].memtype = UIO_MEM_PHYS;
-	return 0;
-}
-
-/* Get pci port io resources described by bar #pci_bar in uio resource n. */
-static int
-igbuio_pci_setup_ioport(struct pci_dev *dev, struct uio_info *info,
-		int n, int pci_bar, const char *name)
-{
-	unsigned long addr, len;
-
-	if (n >= ARRAY_SIZE(info->port))
-		return -EINVAL;
-
-	addr = pci_resource_start(dev, pci_bar);
-	len = pci_resource_len(dev, pci_bar);
-	if (addr == 0 || len == 0)
-		return -EINVAL;
-
-	info->port[n].name = name;
-	info->port[n].start = addr;
-	info->port[n].size = len;
-	info->port[n].porttype = UIO_PORT_X86;
-
-	return 0;
-}
-
-/* Unmap previously ioremap'd resources */
-static void
-igbuio_pci_release_iomem(struct uio_info *info)
-{
-	int i;
-
-	for (i = 0; i < MAX_UIO_MAPS; i++) {
-		if (info->mem[i].internal_addr)
-			iounmap(info->mem[i].internal_addr);
-	}
-}
-
-static int
-igbuio_setup_bars(struct pci_dev *dev, struct uio_info *info)
-{
-	int i, iom, iop, ret;
-	unsigned long flags;
-	static const char *bar_names[PCI_STD_RESOURCE_END + 1]  = {
-		"BAR0",
-		"BAR1",
-		"BAR2",
-		"BAR3",
-		"BAR4",
-		"BAR5",
-	};
-
-	iom = 0;
-	iop = 0;
-
-	for (i = 0; i < ARRAY_SIZE(bar_names); i++) {
-		if (pci_resource_len(dev, i) != 0 &&
-				pci_resource_start(dev, i) != 0) {
-			flags = pci_resource_flags(dev, i);
-			if (flags & IORESOURCE_MEM) {
-				ret = igbuio_pci_setup_iomem(dev, info, iom,
-							     i, bar_names[i]);
-				if (ret != 0)
-					return ret;
-				iom++;
-			} else if (flags & IORESOURCE_IO) {
-				ret = igbuio_pci_setup_ioport(dev, info, iop,
-							      i, bar_names[i]);
-				if (ret != 0)
-					return ret;
-				iop++;
-			}
-		}
-	}
-
-	return (iom != 0) ? ret : -ENOENT;
-}
-
 #if LINUX_VERSION_CODE < KERNEL_VERSION(3, 8, 0)
 static int __devinit
 #else
@@ -345,22 +244,17 @@ igbuio_pci_probe(struct pci_dev *dev, const struct pci_device_id *id)
 	/* enable bus mastering on the device */
 	pci_set_master(dev);
 
-	/* remap IO memory */
-	err = igbuio_setup_bars(dev, &udev->info);
-	if (err != 0)
-		goto fail_release_iomem;
-
 	/* set 64-bit DMA mask */
 	err = pci_set_dma_mask(dev,  DMA_BIT_MASK(64));
 	if (err != 0) {
 		dev_err(&dev->dev, "Cannot set DMA mask\n");
-		goto fail_release_iomem;
+		goto fail_disable_dev;
 	}
 
 	err = pci_set_consistent_dma_mask(dev, DMA_BIT_MASK(64));
 	if (err != 0) {
 		dev_err(&dev->dev, "Cannot set consistent DMA mask\n");
-		goto fail_release_iomem;
+		goto fail_disable_dev;
 	}
 
 	/* fill uio infos */
@@ -406,12 +300,12 @@ igbuio_pci_probe(struct pci_dev *dev, const struct pci_device_id *id)
 		dev_err(&dev->dev, "invalid IRQ mode %u",
 			igbuio_intr_mode_preferred);
 		err = -EINVAL;
-		goto fail_release_iomem;
+		goto fail_disable_dev;
 	}
 
 	err = sysfs_create_group(&dev->dev.kobj, &dev_attr_grp);
 	if (err != 0)
-		goto fail_release_iomem;
+		goto fail_disable_irq;
 
 	/* register uio driver */
 	err = uio_register_device(&dev->dev, &udev->info);
@@ -427,10 +321,10 @@ igbuio_pci_probe(struct pci_dev *dev, const struct pci_device_id *id)
 
 fail_remove_group:
 	sysfs_remove_group(&dev->dev.kobj, &dev_attr_grp);
-fail_release_iomem:
-	igbuio_pci_release_iomem(&udev->info);
+fail_disable_irq:
 	if (udev->mode == RTE_INTR_MODE_MSIX)
 		pci_disable_msix(udev->pdev);
+fail_disable_dev:
 	pci_disable_device(dev);
 fail_free:
 	kfree(udev);
@@ -445,7 +339,6 @@ igbuio_pci_remove(struct pci_dev *dev)
 
 	sysfs_remove_group(&dev->dev.kobj, &dev_attr_grp);
 	uio_unregister_device(&udev->info);
-	igbuio_pci_release_iomem(&udev->info);
 	if (udev->mode == RTE_INTR_MODE_MSIX)
 		pci_disable_msix(dev);
 	pci_disable_device(dev);
-- 
2.7.4

Re: [RFC] igb_uio: deprecate iomem and ioport mapping

[Ferruh Yigit]

On 9/1/2016 3:16 AM, Jianfeng Tan wrote:
> Previously in igb_uio, iomem is mapped, and both ioport and io mem
> are recorded into uio framework, which is duplicated and makes the
> code too complex.
> 
> For iomem, DPDK user space code never opens or reads files under
> /sys/pci/bus/devices/xxxx:xx:xx.x/uio/uioY/maps/. Instead,
> /sys/pci/bus/devices/xxxx:xx:xx.x/resourceY are used to map device
> memory.
> 
> For ioport, non-x86 platforms cannot read from files under
> /sys/pci/bus/devices/xxxx:xx:xx.x/uio/uioY/portio/ directly, because
> non-x86 platforms need to map port region for access in user space,
> see non-x86 version pci_uio_ioport_map(). x86 platforms can use the
> the same way as uio_pci_generic.
> 
> This patch deprecates iomem and ioport mapping in igb_uio kernel
> module, and adjusts the iomem implementation in both igb_uio and
> uio_pci_generic:
>   - for x86 platform, get ports info from /proc/ioports;
>   - for non-x86 platform, map and get ports info by pci_uio_ioport_map().
> 
> Note: this will affect those applications who are using files under
> /sys/pci/bus/devices/xxxx:xx:xx.x/uio/uioY/maps/ and
> /sys/pci/bus/devices/xxxx:xx:xx.x/uio/uioY/portio/.
> 
> Signed-off-by: Jianfeng Tan <jianfeng.tan@intel.com>
> ---

I think it is good idea to simplify to code, that piece looks like can
go away.

But since sysfs interface exposed to the world, anybody can be using it,

what about this: for this release keep this patch as RFC and send
another deprecation notice patch?
And apply this patch in the following release (17.02).

Thanks,
ferruh

Re: [RFC] igb_uio: deprecate iomem and ioport mapping

[Tan, Jianfeng]

On 9/2/2016 8:31 PM, Ferruh Yigit wrote:
> On 9/1/2016 3:16 AM, Jianfeng Tan wrote:
>> Previously in igb_uio, iomem is mapped, and both ioport and io mem
>> are recorded into uio framework, which is duplicated and makes the
>> code too complex.
>>
>> For iomem, DPDK user space code never opens or reads files under
>> /sys/pci/bus/devices/xxxx:xx:xx.x/uio/uioY/maps/. Instead,
>> /sys/pci/bus/devices/xxxx:xx:xx.x/resourceY are used to map device
>> memory.
>>
>> For ioport, non-x86 platforms cannot read from files under
>> /sys/pci/bus/devices/xxxx:xx:xx.x/uio/uioY/portio/ directly, because
>> non-x86 platforms need to map port region for access in user space,
>> see non-x86 version pci_uio_ioport_map(). x86 platforms can use the
>> the same way as uio_pci_generic.
>>
>> This patch deprecates iomem and ioport mapping in igb_uio kernel
>> module, and adjusts the iomem implementation in both igb_uio and
>> uio_pci_generic:
>>    - for x86 platform, get ports info from /proc/ioports;
>>    - for non-x86 platform, map and get ports info by pci_uio_ioport_map().
>>
>> Note: this will affect those applications who are using files under
>> /sys/pci/bus/devices/xxxx:xx:xx.x/uio/uioY/maps/ and
>> /sys/pci/bus/devices/xxxx:xx:xx.x/uio/uioY/portio/.
>>
>> Signed-off-by: Jianfeng Tan <jianfeng.tan@intel.com>
>> ---
> I think it is good idea to simplify to code, that piece looks like can
> go away.
>
> But since sysfs interface exposed to the world, anybody can be using it,
>
> what about this: for this release keep this patch as RFC and send
> another deprecation notice patch?

Thank you Ferruh! Sounds good to me. And I will send out a deprecation 
notice patch later.

Thanks,
Jianfeng

> And apply this patch in the following release (17.02).
>
> Thanks,
> ferruh

Re: [RFC] igb_uio: deprecate iomem and ioport mapping

[David Marchand]

Hello Jianfeng,

On Thu, Sep 1, 2016 at 4:16 AM, Jianfeng Tan <jianfeng.tan@intel.com> wrote:
> Previously in igb_uio, iomem is mapped, and both ioport and io mem
> are recorded into uio framework, which is duplicated and makes the
> code too complex.
>
> For iomem, DPDK user space code never opens or reads files under
> /sys/pci/bus/devices/xxxx:xx:xx.x/uio/uioY/maps/. Instead,
> /sys/pci/bus/devices/xxxx:xx:xx.x/resourceY are used to map device
> memory.
>
> For ioport, non-x86 platforms cannot read from files under
> /sys/pci/bus/devices/xxxx:xx:xx.x/uio/uioY/portio/ directly, because
> non-x86 platforms need to map port region for access in user space,
> see non-x86 version pci_uio_ioport_map(). x86 platforms can use the
> the same way as uio_pci_generic.
>
> This patch deprecates iomem and ioport mapping in igb_uio kernel
> module, and adjusts the iomem implementation in both igb_uio and
> uio_pci_generic:
>   - for x86 platform, get ports info from /proc/ioports;
>   - for non-x86 platform, map and get ports info by pci_uio_ioport_map().
>
> Note: this will affect those applications who are using files under
> /sys/pci/bus/devices/xxxx:xx:xx.x/uio/uioY/maps/ and
> /sys/pci/bus/devices/xxxx:xx:xx.x/uio/uioY/portio/.
>
> Signed-off-by: Jianfeng Tan <jianfeng.tan@intel.com>
> ---
>  lib/librte_eal/linuxapp/eal/eal_pci.c     |   4 -
>  lib/librte_eal/linuxapp/eal/eal_pci_uio.c |  56 +-------------
>  lib/librte_eal/linuxapp/igb_uio/igb_uio.c | 119 ++----------------------------
>  3 files changed, 9 insertions(+), 170 deletions(-)

[snip]

> diff --git a/lib/librte_eal/linuxapp/eal/eal_pci_uio.c b/lib/librte_eal/linuxapp/eal/eal_pci_uio.c
> index 1786b75..28d09ed 100644
> --- a/lib/librte_eal/linuxapp/eal/eal_pci_uio.c
> +++ b/lib/librte_eal/linuxapp/eal/eal_pci_uio.c

[snip]

> -       /* FIXME only for primary process ? */
> -       if (dev->intr_handle.type == RTE_INTR_HANDLE_UNKNOWN) {
> -
> -               snprintf(filename, sizeof(filename), "/dev/uio%u", uio_num);
> -               dev->intr_handle.fd = open(filename, O_RDWR);
> -               if (dev->intr_handle.fd < 0) {
> -                       RTE_LOG(ERR, EAL, "Cannot open %s: %s\n",
> -                               filename, strerror(errno));
> -                       return -1;
> -               }
> -               dev->intr_handle.type = RTE_INTR_HANDLE_UIO;
> -       }

The only potential issue I can see removing this is that if a device
has no iomem resource, then its interrupt fd will never be
initialised.
I can see no problem at the moment, so let's go with this.
If such a problem arises later, we can separate this from the resource
mapping stuff (with a new api ?).

The rest looks good to me.

As Ferruh said, this should go through deprecation notices then go in 17.02.


-- 
David Marchand

Re: [RFC] igb_uio: deprecate iomem and ioport mapping

[Tan, Jianfeng]

Hi David,

Thank you for reviewing this.

On 9/9/2016 5:06 PM, David Marchand wrote:
> Hello Jianfeng,
>
> On Thu, Sep 1, 2016 at 4:16 AM, Jianfeng Tan <jianfeng.tan@intel.com> wrote:
>> Previously in igb_uio, iomem is mapped, and both ioport and io mem
>> are recorded into uio framework, which is duplicated and makes the
>> code too complex.
>>
>> For iomem, DPDK user space code never opens or reads files under
>> /sys/pci/bus/devices/xxxx:xx:xx.x/uio/uioY/maps/. Instead,
>> /sys/pci/bus/devices/xxxx:xx:xx.x/resourceY are used to map device
>> memory.
>>
>> For ioport, non-x86 platforms cannot read from files under
>> /sys/pci/bus/devices/xxxx:xx:xx.x/uio/uioY/portio/ directly, because
>> non-x86 platforms need to map port region for access in user space,
>> see non-x86 version pci_uio_ioport_map(). x86 platforms can use the
>> the same way as uio_pci_generic.
>>
>> This patch deprecates iomem and ioport mapping in igb_uio kernel
>> module, and adjusts the iomem implementation in both igb_uio and
>> uio_pci_generic:
>>    - for x86 platform, get ports info from /proc/ioports;
>>    - for non-x86 platform, map and get ports info by pci_uio_ioport_map().
>>
>> Note: this will affect those applications who are using files under
>> /sys/pci/bus/devices/xxxx:xx:xx.x/uio/uioY/maps/ and
>> /sys/pci/bus/devices/xxxx:xx:xx.x/uio/uioY/portio/.
>>
>> Signed-off-by: Jianfeng Tan <jianfeng.tan@intel.com>
>> ---
>>   lib/librte_eal/linuxapp/eal/eal_pci.c     |   4 -
>>   lib/librte_eal/linuxapp/eal/eal_pci_uio.c |  56 +-------------
>>   lib/librte_eal/linuxapp/igb_uio/igb_uio.c | 119 ++----------------------------
>>   3 files changed, 9 insertions(+), 170 deletions(-)
> [snip]
>
>> diff --git a/lib/librte_eal/linuxapp/eal/eal_pci_uio.c b/lib/librte_eal/linuxapp/eal/eal_pci_uio.c
>> index 1786b75..28d09ed 100644
>> --- a/lib/librte_eal/linuxapp/eal/eal_pci_uio.c
>> +++ b/lib/librte_eal/linuxapp/eal/eal_pci_uio.c
> [snip]
>
>> -       /* FIXME only for primary process ? */
>> -       if (dev->intr_handle.type == RTE_INTR_HANDLE_UNKNOWN) {
>> -
>> -               snprintf(filename, sizeof(filename), "/dev/uio%u", uio_num);
>> -               dev->intr_handle.fd = open(filename, O_RDWR);
>> -               if (dev->intr_handle.fd < 0) {
>> -                       RTE_LOG(ERR, EAL, "Cannot open %s: %s\n",
>> -                               filename, strerror(errno));
>> -                       return -1;
>> -               }
>> -               dev->intr_handle.type = RTE_INTR_HANDLE_UIO;
>> -       }
> The only potential issue I can see removing this is that if a device
> has no iomem resource, then its interrupt fd will never be
> initialised.

I'm catching it completely. IMO, dev->intr_handle.fd will be bound to be 
initialized in pci_uio_alloc_resource() <- pci_uio_map_resource() <- 
rte_eal_pci_map_device() after this patch, just like what we've done 
with uio-pci-generic. Or I'm missing something?

> I can see no problem at the moment, so let's go with this.
> If such a problem arises later, we can separate this from the resource
> mapping stuff (with a new api ?).
> The rest looks good to me.
>
> As Ferruh said, this should go through deprecation notices then go in 17.02.
>
Yes, no problem.

Thanks,
Jianfeng

[PATCH] doc: remove iomem and ioport handling in igb_uio

[Jianfeng Tan]

In igb_uio, iomem is mapped, and both ioport and io mem are recorded
into uio framework (then into sysfs files), which is duplicated with
what Linux has already provided for user space, and makes the code
too complex.

For iomem, DPDK user space code never opens or reads files under
/sys/pci/bus/devices/xxxx:xx:xx.x/uio/uioY/maps/. Instead,
/sys/pci/bus/devices/xxxx:xx:xx.x/resourceY are used to map device
memory.

For ioport, non-x86 platforms cannot read from files under
/sys/pci/bus/devices/xxxx:xx:xx.x/uio/uioY/portio/ directly, because
non-x86 platforms need to map port region for access in user space,
see non-x86 version pci_uio_ioport_map(). x86 platforms can use the
the same way as uio_pci_generic.

This will remove iomem and ioport mapping in igb_uio kernel module,
and adjusts the iomem implementation in both igb_uio and
uio_pci_generic:
  - for x86 platform, get ports info from /proc/ioports;
  - for non-x86 platform, map and get ports info by pci_uio_ioport_map().

Note: this will affect those applications who are using files under
/sys/pci/bus/devices/xxxx:xx:xx.x/uio/uioY/maps/ and
/sys/pci/bus/devices/xxxx:xx:xx.x/uio/uioY/portio/.

Suggested-by: Yigit, Ferruh <ferruh.yigit@intel.com>
Signed-off-by: Jianfeng Tan <jianfeng.tan@intel.com>
---
 doc/guides/rel_notes/deprecation.rst | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/doc/guides/rel_notes/deprecation.rst b/doc/guides/rel_notes/deprecation.rst
index 1a3831f..60f6e60 100644
--- a/doc/guides/rel_notes/deprecation.rst
+++ b/doc/guides/rel_notes/deprecation.rst
@@ -57,3 +57,8 @@ Deprecation Notices
 * API will change for ``rte_port_source_params`` and ``rte_port_sink_params``
   structures. The member ``file_name`` data type will be changed from
   ``char *`` to ``const char *``. This change targets release 16.11.
+
+* igb_uio: iomem mapping and sysfs files created for iomem and ioport in
+  igb_uio will be removed, because we are able to detect these from what Linux
+  has exposed, like the way we have done with uio-pci-generic. This change
+  targets release 17.02.
-- 
2.7.4

Re: [PATCH] doc: remove iomem and ioport handling in igb_uio

[Ferruh Yigit]

On 9/22/2016 6:44 AM, Jianfeng Tan wrote:
> In igb_uio, iomem is mapped, and both ioport and io mem are recorded
> into uio framework (then into sysfs files), which is duplicated with
> what Linux has already provided for user space, and makes the code
> too complex.
> 
> For iomem, DPDK user space code never opens or reads files under
> /sys/pci/bus/devices/xxxx:xx:xx.x/uio/uioY/maps/. Instead,
> /sys/pci/bus/devices/xxxx:xx:xx.x/resourceY are used to map device
> memory.
> 
> For ioport, non-x86 platforms cannot read from files under
> /sys/pci/bus/devices/xxxx:xx:xx.x/uio/uioY/portio/ directly, because
> non-x86 platforms need to map port region for access in user space,
> see non-x86 version pci_uio_ioport_map(). x86 platforms can use the
> the same way as uio_pci_generic.
> 
> This will remove iomem and ioport mapping in igb_uio kernel module,
> and adjusts the iomem implementation in both igb_uio and
> uio_pci_generic:
>   - for x86 platform, get ports info from /proc/ioports;
>   - for non-x86 platform, map and get ports info by pci_uio_ioport_map().
> 
> Note: this will affect those applications who are using files under
> /sys/pci/bus/devices/xxxx:xx:xx.x/uio/uioY/maps/ and
> /sys/pci/bus/devices/xxxx:xx:xx.x/uio/uioY/portio/.
> 
> Suggested-by: Yigit, Ferruh <ferruh.yigit@intel.com>
> Signed-off-by: Jianfeng Tan <jianfeng.tan@intel.com>
> ---

Acked-by: Ferruh Yigit <ferruh.yigit@intel.com>

Re: [PATCH] doc: remove iomem and ioport handling in igb_uio

[Remy Horton]

On 22/09/2016 13:44, Jianfeng Tan wrote:
[..]
>
> Suggested-by: Yigit, Ferruh <ferruh.yigit@intel.com>
> Signed-off-by: Jianfeng Tan <jianfeng.tan@intel.com>

Acked-by: Remy Horton <remy.horton@intel.com>


> +++ b/doc/guides/rel_notes/deprecation.rst
> @@ -57,3 +57,8 @@ Deprecation Notices
>  * API will change for ``rte_port_source_params`` and ``rte_port_sink_params``
>    structures. The member ``file_name`` data type will be changed from
>    ``char *`` to ``const char *``. This change targets release 16.11.

As an aside I don't think changing a structure entry to const will 
affect its binary layout, so this ought not be an ABI break..

Re: [PATCH] doc: remove iomem and ioport handling in igb_uio

[Thomas Monjalon]

> > Suggested-by: Yigit, Ferruh <ferruh.yigit@intel.com>
> > Signed-off-by: Jianfeng Tan <jianfeng.tan@intel.com>
> 
> Acked-by: Remy Horton <remy.horton@intel.com>

Applied

[PATCH] igb_uio: deprecate iomem and ioport mapping

[Jianfeng Tan]

Previously in igb_uio, iomem is mapped, and both ioport and io mem
are recorded into uio framework, which is duplicated and makes the
code too complex.

For iomem, DPDK user space code never opens or reads files under
/sys/pci/bus/devices/xxxx:xx:xx.x/uio/uioY/maps/. Instead,
/sys/pci/bus/devices/xxxx:xx:xx.x/resourceY are used to map device
memory.

For ioport, non-x86 platforms cannot read from files under
/sys/pci/bus/devices/xxxx:xx:xx.x/uio/uioY/portio/ directly, because
non-x86 platforms need to map port region for access in user space,
see non-x86 version pci_uio_ioport_map(). x86 platforms can use the
the same way as uio_pci_generic.

This patch deprecates iomem and ioport mapping in igb_uio kernel
module, and adjusts the iomem implementation in both igb_uio and
uio_pci_generic:
  - for x86 platform, get ports info from /proc/ioports;
  - for non-x86 platform, map and get ports info by pci_uio_ioport_map().

Note: this will affect those applications who are using files under
/sys/pci/bus/devices/xxxx:xx:xx.x/uio/uioY/maps/ and
/sys/pci/bus/devices/xxxx:xx:xx.x/uio/uioY/portio/.

Signed-off-by: Jianfeng Tan <jianfeng.tan@intel.com>
---
 lib/librte_eal/linuxapp/eal/eal_pci.c     |   4 -
 lib/librte_eal/linuxapp/eal/eal_pci_uio.c |  56 +-------------
 lib/librte_eal/linuxapp/igb_uio/igb_uio.c | 119 ++----------------------------
 3 files changed, 9 insertions(+), 170 deletions(-)

diff --git a/lib/librte_eal/linuxapp/eal/eal_pci.c b/lib/librte_eal/linuxapp/eal/eal_pci.c
index 876ba38..2110ad8 100644
--- a/lib/librte_eal/linuxapp/eal/eal_pci.c
+++ b/lib/librte_eal/linuxapp/eal/eal_pci.c
@@ -645,8 +645,6 @@ rte_eal_pci_ioport_map(struct rte_pci_device *dev, int bar,
 		break;
 #endif
 	case RTE_KDRV_IGB_UIO:
-		ret = pci_uio_ioport_map(dev, bar, p);
-		break;
 	case RTE_KDRV_UIO_GENERIC:
 #if defined(RTE_ARCH_X86)
 		ret = pci_ioport_map(dev, bar, p);
@@ -734,8 +732,6 @@ rte_eal_pci_ioport_unmap(struct rte_pci_ioport *p)
 		break;
 #endif
 	case RTE_KDRV_IGB_UIO:
-		ret = pci_uio_ioport_unmap(p);
-		break;
 	case RTE_KDRV_UIO_GENERIC:
 #if defined(RTE_ARCH_X86)
 		ret = 0;
diff --git a/lib/librte_eal/linuxapp/eal/eal_pci_uio.c b/lib/librte_eal/linuxapp/eal/eal_pci_uio.c
index 1786b75..28d09ed 100644
--- a/lib/librte_eal/linuxapp/eal/eal_pci_uio.c
+++ b/lib/librte_eal/linuxapp/eal/eal_pci_uio.c
@@ -370,53 +370,7 @@ pci_uio_map_resource_by_index(struct rte_pci_device *dev, int res_idx,
 	return -1;
 }
 
-#if defined(RTE_ARCH_X86)
-int
-pci_uio_ioport_map(struct rte_pci_device *dev, int bar,
-		   struct rte_pci_ioport *p)
-{
-	char dirname[PATH_MAX];
-	char filename[PATH_MAX];
-	int uio_num;
-	unsigned long start;
-
-	uio_num = pci_get_uio_dev(dev, dirname, sizeof(dirname), 0);
-	if (uio_num < 0)
-		return -1;
-
-	/* get portio start */
-	snprintf(filename, sizeof(filename),
-		 "%s/portio/port%d/start", dirname, bar);
-	if (eal_parse_sysfs_value(filename, &start) < 0) {
-		RTE_LOG(ERR, EAL, "%s(): cannot parse portio start\n",
-			__func__);
-		return -1;
-	}
-	/* ensure we don't get anything funny here, read/write will cast to
-	 * uin16_t */
-	if (start > UINT16_MAX)
-		return -1;
-
-	/* FIXME only for primary process ? */
-	if (dev->intr_handle.type == RTE_INTR_HANDLE_UNKNOWN) {
-
-		snprintf(filename, sizeof(filename), "/dev/uio%u", uio_num);
-		dev->intr_handle.fd = open(filename, O_RDWR);
-		if (dev->intr_handle.fd < 0) {
-			RTE_LOG(ERR, EAL, "Cannot open %s: %s\n",
-				filename, strerror(errno));
-			return -1;
-		}
-		dev->intr_handle.type = RTE_INTR_HANDLE_UIO;
-	}
-
-	RTE_LOG(DEBUG, EAL, "PCI Port IO found start=0x%lx\n", start);
-
-	p->base = start;
-	p->len = 0;
-	return 0;
-}
-#else
+#if !defined(RTE_ARCH_X86)
 int
 pci_uio_ioport_map(struct rte_pci_device *dev, int bar,
 		   struct rte_pci_ioport *p)
@@ -553,14 +507,10 @@ pci_uio_ioport_write(struct rte_pci_ioport *p,
 	}
 }
 
+#if !defined(RTE_ARCH_X86)
 int
 pci_uio_ioport_unmap(struct rte_pci_ioport *p)
 {
-#if defined(RTE_ARCH_X86)
-	RTE_SET_USED(p);
-	/* FIXME close intr fd ? */
-	return 0;
-#else
 	return munmap((void *)(uintptr_t)p->base, p->len);
-#endif
 }
+#endif
diff --git a/lib/librte_eal/linuxapp/igb_uio/igb_uio.c b/lib/librte_eal/linuxapp/igb_uio/igb_uio.c
index df41e45..e9d78fb 100644
--- a/lib/librte_eal/linuxapp/igb_uio/igb_uio.c
+++ b/lib/librte_eal/linuxapp/igb_uio/igb_uio.c
@@ -216,107 +216,6 @@ igbuio_dom0_pci_mmap(struct uio_info *info, struct vm_area_struct *vma)
 }
 #endif
 
-/* Remap pci resources described by bar #pci_bar in uio resource n. */
-static int
-igbuio_pci_setup_iomem(struct pci_dev *dev, struct uio_info *info,
-		       int n, int pci_bar, const char *name)
-{
-	unsigned long addr, len;
-	void *internal_addr;
-
-	if (n >= ARRAY_SIZE(info->mem))
-		return -EINVAL;
-
-	addr = pci_resource_start(dev, pci_bar);
-	len = pci_resource_len(dev, pci_bar);
-	if (addr == 0 || len == 0)
-		return -1;
-	internal_addr = ioremap(addr, len);
-	if (internal_addr == NULL)
-		return -1;
-	info->mem[n].name = name;
-	info->mem[n].addr = addr;
-	info->mem[n].internal_addr = internal_addr;
-	info->mem[n].size = len;
-	info->mem[n].memtype = UIO_MEM_PHYS;
-	return 0;
-}
-
-/* Get pci port io resources described by bar #pci_bar in uio resource n. */
-static int
-igbuio_pci_setup_ioport(struct pci_dev *dev, struct uio_info *info,
-		int n, int pci_bar, const char *name)
-{
-	unsigned long addr, len;
-
-	if (n >= ARRAY_SIZE(info->port))
-		return -EINVAL;
-
-	addr = pci_resource_start(dev, pci_bar);
-	len = pci_resource_len(dev, pci_bar);
-	if (addr == 0 || len == 0)
-		return -EINVAL;
-
-	info->port[n].name = name;
-	info->port[n].start = addr;
-	info->port[n].size = len;
-	info->port[n].porttype = UIO_PORT_X86;
-
-	return 0;
-}
-
-/* Unmap previously ioremap'd resources */
-static void
-igbuio_pci_release_iomem(struct uio_info *info)
-{
-	int i;
-
-	for (i = 0; i < MAX_UIO_MAPS; i++) {
-		if (info->mem[i].internal_addr)
-			iounmap(info->mem[i].internal_addr);
-	}
-}
-
-static int
-igbuio_setup_bars(struct pci_dev *dev, struct uio_info *info)
-{
-	int i, iom, iop, ret;
-	unsigned long flags;
-	static const char *bar_names[PCI_STD_RESOURCE_END + 1]  = {
-		"BAR0",
-		"BAR1",
-		"BAR2",
-		"BAR3",
-		"BAR4",
-		"BAR5",
-	};
-
-	iom = 0;
-	iop = 0;
-
-	for (i = 0; i < ARRAY_SIZE(bar_names); i++) {
-		if (pci_resource_len(dev, i) != 0 &&
-				pci_resource_start(dev, i) != 0) {
-			flags = pci_resource_flags(dev, i);
-			if (flags & IORESOURCE_MEM) {
-				ret = igbuio_pci_setup_iomem(dev, info, iom,
-							     i, bar_names[i]);
-				if (ret != 0)
-					return ret;
-				iom++;
-			} else if (flags & IORESOURCE_IO) {
-				ret = igbuio_pci_setup_ioport(dev, info, iop,
-							      i, bar_names[i]);
-				if (ret != 0)
-					return ret;
-				iop++;
-			}
-		}
-	}
-
-	return (iom != 0) ? ret : -ENOENT;
-}
-
 #if LINUX_VERSION_CODE < KERNEL_VERSION(3, 8, 0)
 static int __devinit
 #else
@@ -345,22 +244,17 @@ igbuio_pci_probe(struct pci_dev *dev, const struct pci_device_id *id)
 	/* enable bus mastering on the device */
 	pci_set_master(dev);
 
-	/* remap IO memory */
-	err = igbuio_setup_bars(dev, &udev->info);
-	if (err != 0)
-		goto fail_release_iomem;
-
 	/* set 64-bit DMA mask */
 	err = pci_set_dma_mask(dev,  DMA_BIT_MASK(64));
 	if (err != 0) {
 		dev_err(&dev->dev, "Cannot set DMA mask\n");
-		goto fail_release_iomem;
+		goto fail_disable_dev;
 	}
 
 	err = pci_set_consistent_dma_mask(dev, DMA_BIT_MASK(64));
 	if (err != 0) {
 		dev_err(&dev->dev, "Cannot set consistent DMA mask\n");
-		goto fail_release_iomem;
+		goto fail_disable_dev;
 	}
 
 	/* fill uio infos */
@@ -406,12 +300,12 @@ igbuio_pci_probe(struct pci_dev *dev, const struct pci_device_id *id)
 		dev_err(&dev->dev, "invalid IRQ mode %u",
 			igbuio_intr_mode_preferred);
 		err = -EINVAL;
-		goto fail_release_iomem;
+		goto fail_disable_dev;
 	}
 
 	err = sysfs_create_group(&dev->dev.kobj, &dev_attr_grp);
 	if (err != 0)
-		goto fail_release_iomem;
+		goto fail_disable_irq;
 
 	/* register uio driver */
 	err = uio_register_device(&dev->dev, &udev->info);
@@ -427,10 +321,10 @@ igbuio_pci_probe(struct pci_dev *dev, const struct pci_device_id *id)
 
 fail_remove_group:
 	sysfs_remove_group(&dev->dev.kobj, &dev_attr_grp);
-fail_release_iomem:
-	igbuio_pci_release_iomem(&udev->info);
+fail_disable_irq:
 	if (udev->mode == RTE_INTR_MODE_MSIX)
 		pci_disable_msix(udev->pdev);
+fail_disable_dev:
 	pci_disable_device(dev);
 fail_free:
 	kfree(udev);
@@ -445,7 +339,6 @@ igbuio_pci_remove(struct pci_dev *dev)
 
 	sysfs_remove_group(&dev->dev.kobj, &dev_attr_grp);
 	uio_unregister_device(&udev->info);
-	igbuio_pci_release_iomem(&udev->info);
 	if (udev->mode == RTE_INTR_MODE_MSIX)
 		pci_disable_msix(dev);
 	pci_disable_device(dev);
-- 
2.7.4

[PATCH] igb_uio: stop device when closing /dev/uioX

[Jianfeng Tan]

Depends-on: http://dpdk.org/dev/patchwork/patch/17493/

When a DPDK application grab a PCI device, device and driver work
together to Rx/Tx packets. If the DPDK app crashes or gets killed,
there's no chance for DPDK driver to stop the device, which means
rte_eth_dev_stop() will not be called.

This could result in problems. In virtio's case, the device (the
vhost backend), will keep working. If packets come, the vhost will
copy them into the vring, which is negotiated with the previous DPDK
app. But the resources, especially hugepages, are recycled by VM
kernel. What's worse, the memory could be allocated for other usage,
and re-written. This leads to an uncertain situation. Like this post
has reported, https://bugs.launchpad.net/qemu/+bug/1558175, QEMU's
vhost finds out wrong value, and exits the whole QEMU process.

To make it right, we need to restart (or reset) the device and make
the device go into the initial state, when uio-generic or igb_uio
recycles the PCI device. There's a chance in uio framework to disable
devices when /dev/uioX gets closed. Here we enable the pci device
in open() hook and disable it in release() hook.

However, if device is not enabled in probe() phase any more, we can
not register irq in probe() through uio_register_device(). To address
that, we invoke request_irq() to register callback directly.

Similar change needs to be done in uio-generic driver. And vfio-pci
seems to have done that already.

Signed-off-by: Jianfeng Tan <jianfeng.tan@intel.com>
---
 lib/librte_eal/linuxapp/igb_uio/igb_uio.c | 171 +++++++++++++++++++-----------
 1 file changed, 108 insertions(+), 63 deletions(-)

diff --git a/lib/librte_eal/linuxapp/igb_uio/igb_uio.c b/lib/librte_eal/linuxapp/igb_uio/igb_uio.c
index e9d78fb..048390d 100644
--- a/lib/librte_eal/linuxapp/igb_uio/igb_uio.c
+++ b/lib/librte_eal/linuxapp/igb_uio/igb_uio.c
@@ -157,8 +157,10 @@ igbuio_pci_irqcontrol(struct uio_info *info, s32 irq_state)
  * If yes, disable it here and will be enable later.
  */
 static irqreturn_t
-igbuio_pci_irqhandler(int irq, struct uio_info *info)
+igbuio_pci_irqhandler(int irq, void *dev_id)
 {
+	struct uio_device *idev = (struct uio_device *)dev_id;
+	struct uio_info *info = idev->info;
 	struct rte_uio_pci_dev *udev = info->priv;
 
 	/* Legacy mode need to mask in hardware */
@@ -166,6 +168,8 @@ igbuio_pci_irqhandler(int irq, struct uio_info *info)
 	    !pci_check_and_mask_intx(udev->pdev))
 		return IRQ_NONE;
 
+	uio_event_notify(info);
+
 	/* Message signal mode, no share IRQ and automasked */
 	return IRQ_HANDLED;
 }
@@ -216,20 +220,58 @@ igbuio_dom0_pci_mmap(struct uio_info *info, struct vm_area_struct *vma)
 }
 #endif
 
-#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 8, 0)
-static int __devinit
-#else
 static int
-#endif
-igbuio_pci_probe(struct pci_dev *dev, const struct pci_device_id *id)
+igbuio_intr_init(struct uio_info *info)
 {
-	struct rte_uio_pci_dev *udev;
+	struct rte_uio_pci_dev *udev = info->priv;
+	struct pci_dev *dev = udev->pdev;
 	struct msix_entry msix_entry;
-	int err;
+	int ret = 0;
 
-	udev = kzalloc(sizeof(struct rte_uio_pci_dev), GFP_KERNEL);
-	if (!udev)
-		return -ENOMEM;
+	switch (igbuio_intr_mode_preferred) {
+	case RTE_INTR_MODE_MSIX:
+		/* Only 1 msi-x vector needed */
+		msix_entry.entry = 0;
+		if (pci_enable_msix(dev, &msix_entry, 1) == 0) {
+			dev_dbg(&dev->dev, "using MSI-X");
+			info->irq = msix_entry.vector;
+			udev->mode = RTE_INTR_MODE_MSIX;
+			break;
+		}
+		/* fall back to INTX */
+	case RTE_INTR_MODE_LEGACY:
+		if (pci_intx_mask_supported(dev)) {
+			dev_dbg(&dev->dev, "using INTX");
+			info->irq_flags = IRQF_SHARED;
+			info->irq = dev->irq;
+			udev->mode = RTE_INTR_MODE_LEGACY;
+			break;
+		}
+		dev_notice(&dev->dev, "PCI INTX mask not supported\n");
+		ret = -EOPNOTSUPP;
+		/* fall back to no IRQ */
+	default:
+		udev->mode = RTE_INTR_MODE_NONE;
+		info->irq = 0;
+	}
+
+	if (info->irq) {
+		ret = request_irq(info->irq, igbuio_pci_irqhandler,
+				  info->irq_flags, info->name,
+				  info->uio_dev);
+		if (ret && udev->mode == RTE_INTR_MODE_MSIX)
+			pci_disable_msix(udev->pdev);
+	}
+
+	return ret;
+}
+
+static int
+igbuio_pci_open(struct uio_info *info, struct inode *inode)
+{
+	struct rte_uio_pci_dev *udev = info->priv;
+	struct pci_dev *dev = udev->pdev;
+	int err;
 
 	/*
 	 * enable device: ask low-level code to enable I/O and
@@ -238,30 +280,77 @@ igbuio_pci_probe(struct pci_dev *dev, const struct pci_device_id *id)
 	err = pci_enable_device(dev);
 	if (err != 0) {
 		dev_err(&dev->dev, "Cannot enable PCI device\n");
-		goto fail_free;
+		return err;
 	}
 
 	/* enable bus mastering on the device */
 	pci_set_master(dev);
 
 	/* set 64-bit DMA mask */
-	err = pci_set_dma_mask(dev,  DMA_BIT_MASK(64));
-	if (err != 0) {
+	err = pci_set_dma_mask(dev, DMA_BIT_MASK(64));
+	if (err) {
 		dev_err(&dev->dev, "Cannot set DMA mask\n");
-		goto fail_disable_dev;
+		goto error;
 	}
 
 	err = pci_set_consistent_dma_mask(dev, DMA_BIT_MASK(64));
-	if (err != 0) {
+	if (err) {
 		dev_err(&dev->dev, "Cannot set consistent DMA mask\n");
-		goto fail_disable_dev;
+		goto error;
+	}
+
+	err = igbuio_intr_init(info);
+	if (err) {
+		dev_err(&dev->dev, "Enable interrupt fails\n");
+		goto error;
+	} else {
+		dev_info(&dev->dev, "uio device registered with irq %lx\n",
+			 udev->info.irq);
+	}
+
+	return 0;
+error:
+	pci_disable_device(dev);
+	return err;
+}
+
+static int
+igbuio_pci_release(struct uio_info *info, struct inode *inode)
+{
+	struct rte_uio_pci_dev *udev = info->priv;
+	struct pci_dev *dev = udev->pdev;
+
+	dev_info(&udev->pdev->dev, "will be disable\n");
+	if (info->irq) {
+		free_irq(info->irq, info->uio_dev);
+		info->irq = 0;
 	}
+	if (udev->mode == RTE_INTR_MODE_MSIX)
+		pci_disable_msix(dev);
+	pci_disable_device(udev->pdev);
+	return 0;
+}
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 8, 0)
+static int __devinit
+#else
+static int
+#endif
+igbuio_pci_probe(struct pci_dev *dev, const struct pci_device_id *id)
+{
+	struct rte_uio_pci_dev *udev;
+	int err;
+
+	udev = kzalloc(sizeof(struct rte_uio_pci_dev), GFP_KERNEL);
+	if (!udev)
+		return -ENOMEM;
 
 	/* fill uio infos */
 	udev->info.name = "igb_uio";
 	udev->info.version = "0.1";
-	udev->info.handler = igbuio_pci_irqhandler;
 	udev->info.irqcontrol = igbuio_pci_irqcontrol;
+	udev->info.release = igbuio_pci_release;
+	udev->info.open = igbuio_pci_open;
 #ifdef CONFIG_XEN_DOM0
 	/* check if the driver run on Xen Dom0 */
 	if (xen_initial_domain())
@@ -270,42 +359,9 @@ igbuio_pci_probe(struct pci_dev *dev, const struct pci_device_id *id)
 	udev->info.priv = udev;
 	udev->pdev = dev;
 
-	switch (igbuio_intr_mode_preferred) {
-	case RTE_INTR_MODE_MSIX:
-		/* Only 1 msi-x vector needed */
-		msix_entry.entry = 0;
-		if (pci_enable_msix(dev, &msix_entry, 1) == 0) {
-			dev_dbg(&dev->dev, "using MSI-X");
-			udev->info.irq = msix_entry.vector;
-			udev->mode = RTE_INTR_MODE_MSIX;
-			break;
-		}
-		/* fall back to INTX */
-	case RTE_INTR_MODE_LEGACY:
-		if (pci_intx_mask_supported(dev)) {
-			dev_dbg(&dev->dev, "using INTX");
-			udev->info.irq_flags = IRQF_SHARED;
-			udev->info.irq = dev->irq;
-			udev->mode = RTE_INTR_MODE_LEGACY;
-			break;
-		}
-		dev_notice(&dev->dev, "PCI INTX mask not supported\n");
-		/* fall back to no IRQ */
-	case RTE_INTR_MODE_NONE:
-		udev->mode = RTE_INTR_MODE_NONE;
-		udev->info.irq = 0;
-		break;
-
-	default:
-		dev_err(&dev->dev, "invalid IRQ mode %u",
-			igbuio_intr_mode_preferred);
-		err = -EINVAL;
-		goto fail_disable_dev;
-	}
-
 	err = sysfs_create_group(&dev->dev.kobj, &dev_attr_grp);
 	if (err != 0)
-		goto fail_disable_irq;
+		goto fail_free;
 
 	/* register uio driver */
 	err = uio_register_device(&dev->dev, &udev->info);
@@ -314,18 +370,10 @@ igbuio_pci_probe(struct pci_dev *dev, const struct pci_device_id *id)
 
 	pci_set_drvdata(dev, udev);
 
-	dev_info(&dev->dev, "uio device registered with irq %lx\n",
-		 udev->info.irq);
-
 	return 0;
 
 fail_remove_group:
 	sysfs_remove_group(&dev->dev.kobj, &dev_attr_grp);
-fail_disable_irq:
-	if (udev->mode == RTE_INTR_MODE_MSIX)
-		pci_disable_msix(udev->pdev);
-fail_disable_dev:
-	pci_disable_device(dev);
 fail_free:
 	kfree(udev);
 
@@ -339,9 +387,6 @@ igbuio_pci_remove(struct pci_dev *dev)
 
 	sysfs_remove_group(&dev->dev.kobj, &dev_attr_grp);
 	uio_unregister_device(&udev->info);
-	if (udev->mode == RTE_INTR_MODE_MSIX)
-		pci_disable_msix(dev);
-	pci_disable_device(dev);
 	pci_set_drvdata(dev, NULL);
 	kfree(udev);
 }
-- 
2.7.4

Re: [PATCH] igb_uio: stop device when closing /dev/uioX

[Thomas Monjalon]

2016-12-02 16:45, Jianfeng Tan:
> Depends-on: http://dpdk.org/dev/patchwork/patch/17493/

The above patch is marked as rejected.
Do we want to reject also this patch?

> When a DPDK application grab a PCI device, device and driver work
> together to Rx/Tx packets. If the DPDK app crashes or gets killed,
> there's no chance for DPDK driver to stop the device, which means
> rte_eth_dev_stop() will not be called.
> 
> This could result in problems. In virtio's case, the device (the
> vhost backend), will keep working. If packets come, the vhost will
> copy them into the vring, which is negotiated with the previous DPDK
> app. But the resources, especially hugepages, are recycled by VM
> kernel. What's worse, the memory could be allocated for other usage,
> and re-written. This leads to an uncertain situation. Like this post
> has reported, https://bugs.launchpad.net/qemu/+bug/1558175, QEMU's
> vhost finds out wrong value, and exits the whole QEMU process.
> 
> To make it right, we need to restart (or reset) the device and make
> the device go into the initial state, when uio-generic or igb_uio
> recycles the PCI device. There's a chance in uio framework to disable
> devices when /dev/uioX gets closed. Here we enable the pci device
> in open() hook and disable it in release() hook.
> 
> However, if device is not enabled in probe() phase any more, we can
> not register irq in probe() through uio_register_device(). To address
> that, we invoke request_irq() to register callback directly.
> 
> Similar change needs to be done in uio-generic driver. And vfio-pci
> seems to have done that already.
> 
> Signed-off-by: Jianfeng Tan <jianfeng.tan@intel.com>

Re: [PATCH] igb_uio: stop device when closing /dev/uioX

[Ferruh Yigit]

On 3/30/2017 9:22 PM, Thomas Monjalon wrote:
> 2016-12-02 16:45, Jianfeng Tan:
>> Depends-on: http://dpdk.org/dev/patchwork/patch/17493/
> 
> The above patch is marked as rejected.
> Do we want to reject also this patch?

Above patch rejected because it may have backward compatibility issues
with the older DPDK (1.8 and older if I remember correct) with newer
kernels.

Plan was checking if this patch can be updated to remove dependency, and
if not accepting above patch.

But as far as I can see not worked on it. We may drop this if Jianfeng
don't have any plan to work on it.

> 
>> When a DPDK application grab a PCI device, device and driver work
>> together to Rx/Tx packets. If the DPDK app crashes or gets killed,
>> there's no chance for DPDK driver to stop the device, which means
>> rte_eth_dev_stop() will not be called.
>>
>> This could result in problems. In virtio's case, the device (the
>> vhost backend), will keep working. If packets come, the vhost will
>> copy them into the vring, which is negotiated with the previous DPDK
>> app. But the resources, especially hugepages, are recycled by VM
>> kernel. What's worse, the memory could be allocated for other usage,
>> and re-written. This leads to an uncertain situation. Like this post
>> has reported, https://bugs.launchpad.net/qemu/+bug/1558175, QEMU's
>> vhost finds out wrong value, and exits the whole QEMU process.
>>
>> To make it right, we need to restart (or reset) the device and make
>> the device go into the initial state, when uio-generic or igb_uio
>> recycles the PCI device. There's a chance in uio framework to disable
>> devices when /dev/uioX gets closed. Here we enable the pci device
>> in open() hook and disable it in release() hook.
>>
>> However, if device is not enabled in probe() phase any more, we can
>> not register irq in probe() through uio_register_device(). To address
>> that, we invoke request_irq() to register callback directly.
>>
>> Similar change needs to be done in uio-generic driver. And vfio-pci
>> seems to have done that already.
>>
>> Signed-off-by: Jianfeng Tan <jianfeng.tan@intel.com>
> 

Re: [RFC] igb_uio: deprecate iomem and ioport mapping

[Stephen Hemminger]

On Thu,  1 Sep 2016 02:16:37 +0000
Jianfeng Tan <jianfeng.tan@intel.com> wrote:

> Previously in igb_uio, iomem is mapped, and both ioport and io mem
> are recorded into uio framework, which is duplicated and makes the
> code too complex.
> 
> For iomem, DPDK user space code never opens or reads files under
> /sys/pci/bus/devices/xxxx:xx:xx.x/uio/uioY/maps/. Instead,
> /sys/pci/bus/devices/xxxx:xx:xx.x/resourceY are used to map device
> memory.
> 
> For ioport, non-x86 platforms cannot read from files under
> /sys/pci/bus/devices/xxxx:xx:xx.x/uio/uioY/portio/ directly, because
> non-x86 platforms need to map port region for access in user space,
> see non-x86 version pci_uio_ioport_map(). x86 platforms can use the
> the same way as uio_pci_generic.
> 
> This patch deprecates iomem and ioport mapping in igb_uio kernel
> module, and adjusts the iomem implementation in both igb_uio and
> uio_pci_generic:
>   - for x86 platform, get ports info from /proc/ioports;
>   - for non-x86 platform, map and get ports info by pci_uio_ioport_map().
> 
> Note: this will affect those applications who are using files under
> /sys/pci/bus/devices/xxxx:xx:xx.x/uio/uioY/maps/ and
> /sys/pci/bus/devices/xxxx:xx:xx.x/uio/uioY/portio/.
> 
> Signed-off-by: Jianfeng Tan <jianfeng.tan@intel.com>

What about people using older kernels with the new DPDK EAL and
vice versa?  It makes sense to make igb_uio generic for non DPDK
usage, so it should probably follow the other UIO drivers.

Re: [RFC] igb_uio: deprecate iomem and ioport mapping

[Tan, Jianfeng]

Hi Stephen,

> -----Original Message-----
> From: Stephen Hemminger [mailto:stephen@networkplumber.org]
> Sent: Saturday, December 3, 2016 7:47 AM
> To: Tan, Jianfeng
> Cc: dev@dpdk.org; david.marchand@6wind.com; Yigit, Ferruh
> Subject: Re: [RFC] igb_uio: deprecate iomem and ioport mapping
> 
> On Thu,  1 Sep 2016 02:16:37 +0000
> Jianfeng Tan <jianfeng.tan@intel.com> wrote:
> 
> > Previously in igb_uio, iomem is mapped, and both ioport and io mem
> > are recorded into uio framework, which is duplicated and makes the
> > code too complex.
> >
> > For iomem, DPDK user space code never opens or reads files under
> > /sys/pci/bus/devices/xxxx:xx:xx.x/uio/uioY/maps/. Instead,
> > /sys/pci/bus/devices/xxxx:xx:xx.x/resourceY are used to map device
> > memory.
> >
> > For ioport, non-x86 platforms cannot read from files under
> > /sys/pci/bus/devices/xxxx:xx:xx.x/uio/uioY/portio/ directly, because
> > non-x86 platforms need to map port region for access in user space,
> > see non-x86 version pci_uio_ioport_map(). x86 platforms can use the
> > the same way as uio_pci_generic.
> >
> > This patch deprecates iomem and ioport mapping in igb_uio kernel
> > module, and adjusts the iomem implementation in both igb_uio and
> > uio_pci_generic:
> >   - for x86 platform, get ports info from /proc/ioports;
> >   - for non-x86 platform, map and get ports info by pci_uio_ioport_map().
> >
> > Note: this will affect those applications who are using files under
> > /sys/pci/bus/devices/xxxx:xx:xx.x/uio/uioY/maps/ and
> > /sys/pci/bus/devices/xxxx:xx:xx.x/uio/uioY/portio/.
> >
> > Signed-off-by: Jianfeng Tan <jianfeng.tan@intel.com>
> 
> What about people using older kernels with the new DPDK EAL and
> vice versa? 

There are two things planned in this proposal:
(1) deprecating iomem mapping in igb_uio, which is not used in DPDK code anyway.
(2) deprecating ioport mapping in igb_uio, which has effect on ioport mapping for x86 platforms. The way we use to make up is to leverage how uio_pci_generic does, aka, based on /proc/ioports, and this proc file is available at very early stage of Linux (Even before 2.6.32).

So I don't see a problem there when running the new DPDK EAL on older kernels.

On the other way, running old DPDK EAL on new kernels, and using new igb_uio, right? Oops, this should have problem. Thank you for pointing out this. So how about just removing iomem? And my motivation to clean igb_uio like this way is to fix a problem here: http://dpdk.org/dev/patchwork/patch/17495/.

> It makes sense to make igb_uio generic for non DPDK
> usage, so it should probably follow the other UIO drivers.
> 

Then the problem would be backward compatibility. I might need to reconsider this.

Thank you for reviewing!

Thanks,
Jianfeng

Re: [RFC] igb_uio: deprecate iomem and ioport mapping

[Ferruh Yigit]

On 12/5/2016 7:04 AM, Tan, Jianfeng wrote:
> Hi Stephen,
> 
>> -----Original Message-----
>> From: Stephen Hemminger [mailto:stephen@networkplumber.org]
>> Sent: Saturday, December 3, 2016 7:47 AM
>> To: Tan, Jianfeng
>> Cc: dev@dpdk.org; david.marchand@6wind.com; Yigit, Ferruh
>> Subject: Re: [RFC] igb_uio: deprecate iomem and ioport mapping
>>
>> On Thu,  1 Sep 2016 02:16:37 +0000
>> Jianfeng Tan <jianfeng.tan@intel.com> wrote:
>>
>>> Previously in igb_uio, iomem is mapped, and both ioport and io mem
>>> are recorded into uio framework, which is duplicated and makes the
>>> code too complex.
>>>
>>> For iomem, DPDK user space code never opens or reads files under
>>> /sys/pci/bus/devices/xxxx:xx:xx.x/uio/uioY/maps/. Instead,
>>> /sys/pci/bus/devices/xxxx:xx:xx.x/resourceY are used to map device
>>> memory.
>>>
>>> For ioport, non-x86 platforms cannot read from files under
>>> /sys/pci/bus/devices/xxxx:xx:xx.x/uio/uioY/portio/ directly, because
>>> non-x86 platforms need to map port region for access in user space,
>>> see non-x86 version pci_uio_ioport_map(). x86 platforms can use the
>>> the same way as uio_pci_generic.
>>>
>>> This patch deprecates iomem and ioport mapping in igb_uio kernel
>>> module, and adjusts the iomem implementation in both igb_uio and
>>> uio_pci_generic:
>>>   - for x86 platform, get ports info from /proc/ioports;
>>>   - for non-x86 platform, map and get ports info by pci_uio_ioport_map().
>>>
>>> Note: this will affect those applications who are using files under
>>> /sys/pci/bus/devices/xxxx:xx:xx.x/uio/uioY/maps/ and
>>> /sys/pci/bus/devices/xxxx:xx:xx.x/uio/uioY/portio/.
>>>
>>> Signed-off-by: Jianfeng Tan <jianfeng.tan@intel.com>
>>
>> What about people using older kernels with the new DPDK EAL and
>> vice versa? 
> 
> There are two things planned in this proposal:
> (1) deprecating iomem mapping in igb_uio, which is not used in DPDK code anyway.
> (2) deprecating ioport mapping in igb_uio, which has effect on ioport mapping for x86 platforms. The way we use to make up is to leverage how uio_pci_generic does, aka, based on /proc/ioports, and this proc file is available at very early stage of Linux (Even before 2.6.32).
> 
> So I don't see a problem there when running the new DPDK EAL on older kernels.
> 
> On the other way, running old DPDK EAL on new kernels, and using new igb_uio, right? Oops, this should have problem. Thank you for pointing out this. So how about just removing iomem? And my motivation to clean igb_uio like this way is to fix a problem here: http://dpdk.org/dev/patchwork/patch/17495/.
> 
>> It makes sense to make igb_uio generic for non DPDK
>> usage, so it should probably follow the other UIO drivers.
>>
> 
> Then the problem would be backward compatibility. I might need to reconsider this.

Hi Jianfeng,

Taking into account that this patch is for cleanup, and there may be
some backward compatibility issues mentioned by Stephen, would you mind
dropping this patch?

If you agree to drop, would you mind sending a patch to remove existing
deprecation notice?

Thanks,
ferruh

> 
> Thank you for reviewing!
> 
> Thanks,
> Jianfeng
> 

Re: [RFC] igb_uio: deprecate iomem and ioport mapping

[Tan, Jianfeng]

Hi,

> -----Original Message-----
> From: Yigit, Ferruh
> Sent: Thursday, January 5, 2017 11:24 PM
> To: Tan, Jianfeng; Stephen Hemminger
> Cc: dev@dpdk.org; david.marchand@6wind.com
> Subject: Re: [RFC] igb_uio: deprecate iomem and ioport mapping
> 
> On 12/5/2016 7:04 AM, Tan, Jianfeng wrote:
> > Hi Stephen,
> >
> >> -----Original Message-----
> >> From: Stephen Hemminger [mailto:stephen@networkplumber.org]
> >> Sent: Saturday, December 3, 2016 7:47 AM
> >> To: Tan, Jianfeng
> >> Cc: dev@dpdk.org; david.marchand@6wind.com; Yigit, Ferruh
> >> Subject: Re: [RFC] igb_uio: deprecate iomem and ioport mapping
> >>
> >> On Thu,  1 Sep 2016 02:16:37 +0000
> >> Jianfeng Tan <jianfeng.tan@intel.com> wrote:
> >>
> >>> Previously in igb_uio, iomem is mapped, and both ioport and io mem
> >>> are recorded into uio framework, which is duplicated and makes the
> >>> code too complex.
> >>>
> >>> For iomem, DPDK user space code never opens or reads files under
> >>> /sys/pci/bus/devices/xxxx:xx:xx.x/uio/uioY/maps/. Instead,
> >>> /sys/pci/bus/devices/xxxx:xx:xx.x/resourceY are used to map device
> >>> memory.
> >>>
> >>> For ioport, non-x86 platforms cannot read from files under
> >>> /sys/pci/bus/devices/xxxx:xx:xx.x/uio/uioY/portio/ directly, because
> >>> non-x86 platforms need to map port region for access in user space,
> >>> see non-x86 version pci_uio_ioport_map(). x86 platforms can use the
> >>> the same way as uio_pci_generic.
> >>>
> >>> This patch deprecates iomem and ioport mapping in igb_uio kernel
> >>> module, and adjusts the iomem implementation in both igb_uio and
> >>> uio_pci_generic:
> >>>   - for x86 platform, get ports info from /proc/ioports;
> >>>   - for non-x86 platform, map and get ports info by pci_uio_ioport_map().
> >>>
> >>> Note: this will affect those applications who are using files under
> >>> /sys/pci/bus/devices/xxxx:xx:xx.x/uio/uioY/maps/ and
> >>> /sys/pci/bus/devices/xxxx:xx:xx.x/uio/uioY/portio/.
> >>>
> >>> Signed-off-by: Jianfeng Tan <jianfeng.tan@intel.com>
> >>
> >> What about people using older kernels with the new DPDK EAL and
> >> vice versa?
> >
> > There are two things planned in this proposal:
> > (1) deprecating iomem mapping in igb_uio, which is not used in DPDK code
> anyway.
> > (2) deprecating ioport mapping in igb_uio, which has effect on ioport
> mapping for x86 platforms. The way we use to make up is to leverage how
> uio_pci_generic does, aka, based on /proc/ioports, and this proc file is
> available at very early stage of Linux (Even before 2.6.32).
> >
> > So I don't see a problem there when running the new DPDK EAL on older
> kernels.
> >
> > On the other way, running old DPDK EAL on new kernels, and using new
> igb_uio, right? Oops, this should have problem. Thank you for pointing out
> this. So how about just removing iomem? And my motivation to clean
> igb_uio like this way is to fix a problem here:
> http://dpdk.org/dev/patchwork/patch/17495/.
> >
> >> It makes sense to make igb_uio generic for non DPDK
> >> usage, so it should probably follow the other UIO drivers.
> >>
> >
> > Then the problem would be backward compatibility. I might need to
> reconsider this.
> 
> Hi Jianfeng,
> 
> Taking into account that this patch is for cleanup, and there may be
> some backward compatibility issues mentioned by Stephen, would you mind
> dropping this patch?

I agree to drop this patch.

> 
> If you agree to drop, would you mind sending a patch to remove existing
> deprecation notice?

No problem, I'll do that.

Thanks,
Jianfeng

> 
> Thanks,
> ferruh
> 
> >
> > Thank you for reviewing!
> >
> > Thanks,
> > Jianfeng
> >