From mboxrd@z Thu Jan 1 00:00:00 1970 From: "Ananyev, Konstantin" Subject: Re: [PATCH 6/6] virtio: arm/arm64: memory mapped IO support in pmd driver Date: Tue, 8 Dec 2015 09:47:58 +0000 Message-ID: <2601191342CEEE43887BDE71AB97725836AD1894@irsmsx105.ger.corp.intel.com> References: <1449250519-28372-1-git-send-email-sshukla@mvista.com> <1449250519-28372-7-git-send-email-sshukla@mvista.com> Mime-Version: 1.0 Content-Type: text/plain; charset="us-ascii" Content-Transfer-Encoding: quoted-printable Cc: Rizwan Ansari To: Santosh Shukla , "dev@dpdk.org" Return-path: Received: from mga09.intel.com (mga09.intel.com [134.134.136.24]) by dpdk.org (Postfix) with ESMTP id A472A4A63 for ; Tue, 8 Dec 2015 10:50:32 +0100 (CET) In-Reply-To: <1449250519-28372-7-git-send-email-sshukla@mvista.com> Content-Language: en-US List-Id: patches and discussions about DPDK List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Sender: "dev" Hi, > -----Original Message----- > From: dev [mailto:dev-bounces@dpdk.org] On Behalf Of Santosh Shukla > Sent: Friday, December 04, 2015 5:35 PM > To: dev@dpdk.org > Cc: Rizwan Ansari > Subject: [dpdk-dev] [PATCH 6/6] virtio: arm/arm64: memory mapped IO suppo= rt in pmd driver >=20 > This patch set add memory-mapped-IO support for arm/arm64 archs in virtio= -pmd > driver. Patch creates ioport-mem device name /dev/igb_ioport. virtio_ethd= ev_init > function to open that device file for once, mappes 4K page_size memory su= ch that > all entry in cat /proc/ioport {all PCI_IOBAR entry} mapped for once. For = that > two ancessor api;s > - virtio_map_ioport : Maps all cat /proc/ioport pci iobars. > - virtio_set_ioport_addr : manages the each pci_iobar slot as an offset. >=20 > Tested for thunderX/arm64 platform, by creating maximum guest kernel supp= orted > virtio-net-pci device i.e.. 31 max, then attaching all 32 interface to ui= o, > Verified with tespmd io_fwd application. >=20 > Signed-off-by: Santosh Shukla > Signed-off-by: Rizwan Ansari > --- Is it possible to rework patch a bit to minimise number of #ifdef ARM ...=20 spread across the code?=20 Group arch related code into separate file(s), use union for fields with si= zes, etc? Konstantin =20 > drivers/net/virtio/virtio_ethdev.c | 138 +++++++++++++++++++++++= +++++- > lib/librte_eal/linuxapp/igb_uio/igb_uio.c | 80 ++++++++++++++++- > 2 files changed, 214 insertions(+), 4 deletions(-) >=20 > diff --git a/drivers/net/virtio/virtio_ethdev.c b/drivers/net/virtio/virt= io_ethdev.c > index 74c00ee..93b8784 100644 > --- a/drivers/net/virtio/virtio_ethdev.c > +++ b/drivers/net/virtio/virtio_ethdev.c > @@ -63,6 +63,30 @@ > #include "virtqueue.h" > #include "virtio_rxtx.h" >=20 > +#ifdef RTE_EXEC_ENV_LINUXAPP > +/* start address of first pci_iobar slot (user-space virtual-addres) */ > +void *ioport_map; > +#if defined(RTE_ARCH_ARM) || defined(RTE_ARCH_ARM64) > + > +#include > +#define DEV_NAME "/dev/igb_ioport" > + > +/* Keeping pci_ioport_size =3D 4k. > + * So maximum mmaped pci_iobar supported =3D > + * (ioport_size/pci_dev->mem_resource[0].len) > + * > + * note: kernel could allow maximum 32 virtio-net-pci interface, that me= an > + * maximum 32 PCI_IOBAR(s) where each PCI_IOBAR_LEN=3D0x20, so virtio_ma= p_ioport() > + * func by theory gonna support 4k/0x20 =3D=3D> 128 PCI_IOBAR(s), more t= han > + * max-virtio-net-pci interface. > + */ > +#define PAGE_SIZE 4096 > +#define PCI_IOPORT_SIZE PAGE_SIZE > +#define PCI_IOPORT_MAX 128 /* 4k / 0x20 */ > + > +int ioport_map_cnt; > +#endif /* ARM, ARM64 */ > +#endif /* RTE_EXEC_ENV_LINUXAPP */ >=20 > static int eth_virtio_dev_init(struct rte_eth_dev *eth_dev); > static int eth_virtio_dev_uninit(struct rte_eth_dev *eth_dev); > @@ -497,6 +521,18 @@ virtio_dev_close(struct rte_eth_dev *dev) > hw->started =3D 0; > virtio_dev_free_mbufs(dev); > virtio_free_queues(dev); > + > +#ifdef RTE_EXEC_ENV_LINUXAPP > +#if defined(RTE_ARCH_ARM) || defined(RTE_ARCH_ARM64) > + > + /* unmap ioport memory */ > + ioport_map_cnt--; > + if (!ioport_map_cnt) > + munmap(ioport_map, PCI_IOPORT_SIZE); > + > + PMD_INIT_LOG(DEBUG, "unmapping ioport_mem %d\n", ioport_map_cnt); > +#endif > +#endif > } >=20 > static void > @@ -1172,7 +1208,6 @@ static int virtio_resource_init_by_ioports(struct r= te_pci_device *pci_dev) >=20 > sscanf(ptr, "%04hx-%04hx", &start, &end); > size =3D end - start + 1; > - > break; > } > } > @@ -1256,6 +1291,81 @@ rx_func_get(struct rte_eth_dev *eth_dev) > eth_dev->rx_pkt_burst =3D &virtio_recv_pkts; > } >=20 > +#ifdef RTE_EXEC_ENV_LINUXAPP > +#if defined(RTE_ARCH_ARM) || defined(RTE_ARCH_ARM64) > + > +static int > +virtio_map_ioport(void **resource_addr) > +{ > + int fd; > + int ret =3D 0; > + > + fd =3D open(DEV_NAME, O_RDWR); > + if (fd < 0) { > + PMD_INIT_LOG(ERR, "device file %s open error: %d\n", > + DEV_NAME, fd); > + ret =3D -1; > + goto out; > + } > + > + ioport_map =3D mmap(NULL, PCI_IOPORT_SIZE, > + PROT_EXEC | PROT_WRITE | PROT_READ, MAP_SHARED, fd, 0); > + > + if (ioport_map =3D=3D MAP_FAILED) { > + PMD_INIT_LOG(ERR, "mmap: failed to map bar Address=3D%p\n", > + *resource_addr); > + ret =3D -ENOMEM; > + goto out1; > + } > + > + PMD_INIT_LOG(INFO, "First pci_iobar mapped at %p\n", ioport_map); > + > +out1: > + close(fd); > +out: > + return ret; > +} > + > +static int > +virtio_set_ioport_addr(void **resource_addr, unsigned long offset) > +{ > + int ret =3D 0; > + > + if (ioport_map_cnt >=3D PCI_IOPORT_MAX) { > + ret =3D -1; > + PMD_INIT_LOG(ERR, > + "ioport_map_cnt(%d) greater than" > + "PCI_IOPORT_MAX(%d)\n", > + ioport_map_cnt, PCI_IOPORT_MAX); > + goto out; > + } > + *resource_addr =3D (void *)((char *)ioport_map + (ioport_map_cnt)*offse= t); > + ioport_map_cnt++; > + > + PMD_INIT_LOG(DEBUG, "pci.resource_addr %p ioport_map_cnt %d\n", > + *resource_addr, ioport_map_cnt); > +out: > + return ret; > +} > +#else /* !ARM, !ARM64 */ > +static int > +virtio_map_ioport(void *resource_addr) > +{ > + (void)resource_addr; > + return 0; > +} > + > +static int > +virtio_set_ioport_addr(void *resource_addr, unsigned long offset) > +{ > + (void)resource_addr; > + (void)offset; > + return 0; > +} > + > +#endif /* ARM, ARM64 */ > +#endif /* RTE_EXEC_ENV_LINUXAPP */ > + > /* > * This function is based on probe() function in virtio_pci.c > * It returns 0 on success. > @@ -1294,8 +1404,31 @@ eth_virtio_dev_init(struct rte_eth_dev *eth_dev) > if (virtio_resource_init(pci_dev) < 0) > return -1; >=20 > +#ifdef RTE_EXEC_ENV_LINUXAPP > + int ret =3D 0; > + > + /* Map the all IOBAR entry from /proc/ioport to 4k page_size only once. > + * Later virtio_set_ioport_addr() func will update correct bar_addr > + * eachk ioport (i.e..pci_dev->mem_resource[0].addr) > + */ > + if (!ioport_map) { > + ret =3D virtio_map_ioport(&pci_dev->mem_resource[0].addr); > + if (ret < 0) > + return -1; > + } > + > + ret =3D virtio_set_ioport_addr(&pci_dev->mem_resource[0].addr, > + pci_dev->mem_resource[0].len); > + if (ret < 0) > + return -1; > + > + PMD_INIT_LOG(INFO, "ioport_map %p resource_addr %p resource_len :%ld\n"= , > + ioport_map, pci_dev->mem_resource[0].addr, > + (unsigned long)pci_dev->mem_resource[0].len); > + > +#endif > hw->use_msix =3D virtio_has_msix(&pci_dev->addr); > - hw->io_base =3D (uint32_t)(uintptr_t)pci_dev->mem_resource[0].addr; > + hw->io_base =3D (unsigned long)(uintptr_t)pci_dev->mem_resource[0].addr= ; >=20 > /* Reset the device although not necessary at startup */ > vtpci_reset(hw); > @@ -1430,7 +1563,6 @@ eth_virtio_dev_uninit(struct rte_eth_dev *eth_dev) > rte_intr_callback_unregister(&pci_dev->intr_handle, > virtio_interrupt_handler, > eth_dev); > - > PMD_INIT_LOG(DEBUG, "dev_uninit completed"); >=20 > return 0; > diff --git a/lib/librte_eal/linuxapp/igb_uio/igb_uio.c b/lib/librte_eal/l= inuxapp/igb_uio/igb_uio.c > index f5617d2..b154a44 100644 > --- a/lib/librte_eal/linuxapp/igb_uio/igb_uio.c > +++ b/lib/librte_eal/linuxapp/igb_uio/igb_uio.c > @@ -324,6 +324,61 @@ igbuio_dom0_pci_mmap(struct uio_info *info, struct v= m_area_struct *vma) > } > #endif >=20 > +#if defined(RTE_ARCH_ARM) || defined(RTE_ARCH_ARM64) > +#ifdef CONFIG_HAS_IOPORT_MAP > +/* > + * mmap driver to map x86-style PCI_IOBAR (i.e..cat /proc/ioport pci-bar= -memory) > + * from kernel-space virtual address to user-space virtual address. This= module > + * required for non-x86 archs example arm/arm64, as those archs donot do > + * IO_MAP_IO types access, Infact supports memory-mapped-IO. That is bec= ause > + * arm/arm64 doesn't support direct IO instruction, so the design approa= ch is to > + * map `cat /proc/ioport` PCI_IOBAR's kernel-space virtual-addr to user-= space > + * virtual-addr. Therefore the need for mmap-driver. > + */ > +#include /* file_operations */ > +#include > +#include /* VM_IO */ > +#include > +#include > +#include > +#include > + > +void *__iomem mapped_io; /* ioport addr of `cat /proc/ioport` */ > + > +static int igb_ioport_mmap(struct file *file, struct vm_area_struct *vma= ) > +{ > + struct page *npage; > + int ret =3D 0; > + > + vma->vm_page_prot =3D pgprot_noncached(vma->vm_page_prot); > + npage =3D vmalloc_to_page(mapped_io); > + ret =3D remap_pfn_range(vma, vma->vm_start, > + page_to_pfn(npage), > + vma->vm_end - vma->vm_start, > + vma->vm_page_prot); > + if (ret) { > + pr_info("Error: Failed to remap pfn=3D%lu error=3D%d\n", > + page_to_pfn(npage), ret); > + } > + return 0; > +} > + > +static const struct file_operations igb_ioport_fops =3D { > + .mmap =3D igb_ioport_mmap, > +}; > + > +static struct miscdevice igb_ioport_dev =3D { > + .minor =3D MISC_DYNAMIC_MINOR, > + .name =3D "igb_ioport", > + .fops =3D &igb_ioport_fops > +}; > +#else /* !CONFIG_HAS_IOPORT_MAP */ > + > +#error "CONFIG_HAS_IOPORT_MAP not supported for $RTE_ARCH" > + > +#endif /* CONFIG_HAS_IOPORT_MAP */ > +#endif /* RTE_ARCH_ARM, RTE_ARCH_ARM64 */ > + > /* Remap pci resources described by bar #pci_bar in uio resource n. */ > static int > igbuio_pci_setup_iomem(struct pci_dev *dev, struct uio_info *info, > @@ -365,11 +420,22 @@ igbuio_pci_setup_ioport(struct pci_dev *dev, struct= uio_info *info, > if (addr =3D=3D 0 || len =3D=3D 0) > return -EINVAL; >=20 > +#if defined(RTE_ARCH_ARM) || defined(RTE_ARCH_ARM64) > + /* > + * TODO: KNI/procfs:: porttype entry need to be added for ARM/ARM64, > + * for below mmap driver; > + * */ > + mapped_io =3D ioport_map(addr, 0); > + info->port[n].name =3D name; > + info->port[n].start =3D (unsigned long)(uintptr_t)mapped_io; > + info->port[n].size =3D len; > + info->port[n].porttype =3D UIO_PORT_X86; > +#else > info->port[n].name =3D name; > info->port[n].start =3D addr; > info->port[n].size =3D len; > info->port[n].porttype =3D UIO_PORT_X86; > - > +#endif > return 0; > } >=20 > @@ -615,6 +681,15 @@ igbuio_pci_init_module(void) > { > int ret; >=20 > +#if defined(RTE_ARCH_ARM) || defined(RTE_ARCH_ARM64) > + ret =3D misc_register(&igb_ioport_dev); > + if (ret < 0) { > + pr_info("Error: failed to register ioport map driver (%d)\n", > + ret); > + return ret; > + } > +#endif > + > ret =3D igbuio_config_intr_mode(intr_mode); > if (ret < 0) > return ret; > @@ -625,6 +700,9 @@ igbuio_pci_init_module(void) > static void __exit > igbuio_pci_exit_module(void) > { > +#if defined(RTE_ARCH_ARM) || defined(RTE_ARCH_ARM64) > + misc_deregister(&igb_ioport_dev); > +#endif > pci_unregister_driver(&igbuio_pci_driver); > } >=20 > -- > 1.7.9.5