Re: [ [PATCH v2] 03/13] rte_io: armv7/v8: Introduce api to emulate x86-style of PCI/ISA ioport access

[Santosh Shukla <sshukla@mvista.com>]

On Mon, Dec 14, 2015 at 7:55 PM, Jerin Jacob
<jerin.jacob@caviumnetworks.com> wrote:
> On Mon, Dec 14, 2015 at 06:30:22PM +0530, Santosh Shukla wrote:
>> Introducing rte_io.h header file to emulate x86-style of ioport rd/wr api
>> example {in,out}[bwl] and {in_p,out_p}[bwl]. Api support added for armv7 and
>> armv8 both.
>>
>> Current use-case for this api is for virtio_pci module that does x86-style
>> rd/wr.
>>
>> Tested for armv8/ThunderX platform and build successfully for armv7.
>>
>> Signed-off-by: Santosh Shukla <sshukla@mvista.com>
>> ---
>>  lib/librte_eal/common/Makefile                     |    1 +
>>  lib/librte_eal/common/include/arch/arm/rte_io.h    |   60 ++++++++
>>  lib/librte_eal/common/include/arch/arm/rte_io_32.h |  155 ++++++++++++++++++++
>>  lib/librte_eal/common/include/arch/arm/rte_io_64.h |  155 ++++++++++++++++++++
>>  lib/librte_eal/common/include/generic/rte_io.h     |   81 ++++++++++
>>  5 files changed, 452 insertions(+)
>>  create mode 100644 lib/librte_eal/common/include/arch/arm/rte_io.h
>>  create mode 100644 lib/librte_eal/common/include/arch/arm/rte_io_32.h
>>  create mode 100644 lib/librte_eal/common/include/arch/arm/rte_io_64.h
>>  create mode 100644 lib/librte_eal/common/include/generic/rte_io.h
>>
>> diff --git a/lib/librte_eal/common/Makefile b/lib/librte_eal/common/Makefile
>> index f5ea0ee..1021e1d 100644
>> --- a/lib/librte_eal/common/Makefile
>> +++ b/lib/librte_eal/common/Makefile
>> @@ -48,6 +48,7 @@ endif
>>
>>  GENERIC_INC := rte_atomic.h rte_byteorder.h rte_cycles.h rte_prefetch.h
>>  GENERIC_INC += rte_spinlock.h rte_memcpy.h rte_cpuflags.h rte_rwlock.h
>> +GENERIC_INC += rte_io.h
>>  # defined in mk/arch/$(RTE_ARCH)/rte.vars.mk
>>  ARCH_DIR ?= $(RTE_ARCH)
>>  ARCH_INC := $(notdir $(wildcard $(RTE_SDK)/lib/librte_eal/common/include/arch/$(ARCH_DIR)/*.h))
>> diff --git a/lib/librte_eal/common/include/arch/arm/rte_io.h b/lib/librte_eal/common/include/arch/arm/rte_io.h
>> new file mode 100644
>> index 0000000..b4f1613
>> --- /dev/null
>> +++ b/lib/librte_eal/common/include/arch/arm/rte_io.h
>> @@ -0,0 +1,60 @@
>> +/*
>> + *   BSD LICENSE
>> + *
>> + *   Copyright(c) 2015 Cavium Networks. All rights reserved.
>> + *   All rights reserved.
>> + *
>> + *   Copyright(c) 2010-2015 Intel Corporation. All rights reserved.
>> + *   All rights reserved.
>> + *
>> + *   ARM helper api to emulate x86-style of {in , out}[bwl] api used for
>> + *   accessing PCI/ISA IO address space.
>> + *
>> + *   Redistribution and use in source and binary forms, with or without
>> + *   modification, are permitted provided that the following conditions
>> + *   are met:
>> + *
>> + *     * Redistributions of source code must retain the above copyright
>> + *       notice, this list of conditions and the following disclaimer.
>> + *     * Redistributions in binary form must reproduce the above copyright
>> + *       notice, this list of conditions and the following disclaimer in
>> + *       the documentation and/or other materials provided with the
>> + *       distribution.
>> + *     * Neither the name of Intel Corporation nor the names of its
>> + *       contributors may be used to endorse or promote products derived
>> + *       from this software without specific prior written permission.
>> + *
>> + *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
>> + *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
>> + *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
>> + *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
>> + *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
>> + *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
>> + *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
>> + *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
>> + *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
>> + *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
>> + *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
>> + */
>> +
>> +#ifndef _RTE_IO_ARM_H_
>> +#define _RTE_IO_ARM_H_
>> +
>> +/*
>> + * @File
>> + * Use-case:
>> + * Currently virtio pci does IO access in x86-way i.e. IO_RESOURCE_IO way, It
>> + * access the pci address space by port_number. The ARM doesn't have
>> + * instructions for direct IO access. In ARM: IO's are memory mapped.
>> + *
>> + * Below helper api allow virtio_pci pmd driver to access IO's for arm/arm64
>> + * arch in x86-style of apis example: {in , out}[bwl] and {in_p , out_p}[bwl].
>> + */
>> +
>> +#ifdef RTE_ARCH_64
>> +#include <rte_io_64.h>
>> +#else
>> +#include <rte_io_32.h>
>> +#endif
>> +
>> +#endif /* _RTE_IO_ARM_H_ */
>> diff --git a/lib/librte_eal/common/include/arch/arm/rte_io_32.h b/lib/librte_eal/common/include/arch/arm/rte_io_32.h
>> new file mode 100644
>> index 0000000..0e79427
>> --- /dev/null
>> +++ b/lib/librte_eal/common/include/arch/arm/rte_io_32.h
>> @@ -0,0 +1,155 @@
>> +/*
>> + *   BSD LICENSE
>> + *
>> + *   Copyright(c) 2015 Cavium Networks. All rights reserved.
>> + *   All rights reserved.
>> + *
>> + *   Copyright(c) 2010-2015 Intel Corporation. All rights reserved.
>> + *   All rights reserved.
>> + *
>> + *   Redistribution and use in source and binary forms, with or without
>> + *   modification, are permitted provided that the following conditions
>> + *   are met:
>> + *
>> + *     * Redistributions of source code must retain the above copyright
>> + *       notice, this list of conditions and the following disclaimer.
>> + *     * Redistributions in binary form must reproduce the above copyright
>> + *       notice, this list of conditions and the following disclaimer in
>> + *       the documentation and/or other materials provided with the
>> + *       distribution.
>> + *     * Neither the name of Intel Corporation nor the names of its
>> + *       contributors may be used to endorse or promote products derived
>> + *       from this software without specific prior written permission.
>> + *
>> + *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
>> + *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
>> + *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
>> + *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
>> + *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
>> + *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
>> + *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
>> + *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
>> + *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
>> + *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
>> + *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
>> + */
>> +
>> +#ifndef _RTE_IO_ARM32_H_
>> +#define _RTE_IO_ARM32_H_
>> +
>> +#ifdef __cplusplus
>> +extern "C" {
>> +#endif
>> +
>> +#include "generic/rte_io.h"
>> +
>> +/*
>> + * Generic IO read/write api for arm: Refer TRM
>> + */
>> +static inline void raw_writeb(uint8_t val, uint32_t addr)
>> +{
>
> armv7 and armv8 instructions looks same for both raw_* implementation.
> The difference in length of "addr" can be abstracted as  uintptr_t to
> reuse the code.
>
> Use AT&T sytax on inline assembly as mentioned in
> doc/guides/contributing/coding_style.rst
>
> Inline ASM in C code
> ~~~~~~~~~~~~~~~~~~~~
>
> The ``asm`` and ``volatile`` keywords do not have underscores. The AT&T
> syntax should be used.
> Input and output operands should be named to avoid confusion, as shown
> in the following example:
>
> .. code-block:: c
>
>         asm volatile("outb %[val], %[port]"
>                 : :
>                 [port] "dN" (port),
>                 [val] "a" (val));
>
>

Ok.

>> +     asm volatile("strb %0, [%1]" : : "r" (val), "r" (addr));
>> +}
>> +
>> +static inline void raw_writew(uint16_t val, uint32_t addr)
>> +{
>> +     asm volatile("strh %0, [%1]" : : "r" (val), "r" (addr));
>> +}
>> +
>> +static inline void raw_writel(uint32_t val, uint32_t addr)
>> +{
>> +     asm volatile("str %0, [%1]" : : "r" (val), "r" (addr));
>> +}
>> +
>> +static inline uint8_t raw_readb(uint32_t addr)
>> +{
>> +     uint8_t val;
>> +     asm volatile("ldrb %0, [%1]" : "=r" (val) : "r" (addr));
>> +     return val;
>> +}
>> +
>> +static inline uint16_t raw_readw(uint32_t addr)
>> +{
>> +     uint16_t val;
>> +     asm volatile("ldrh %0, [%1]" : "=r" (val) : "r" (addr));
>> +     return val;
>> +}
>> +
>> +static inline uint32_t raw_readl(uint32_t addr)
>> +{
>> +     uint32_t val;
>> +     asm volatile("ldr %0, [%1]" : "=r" (val) : "r" (addr));
>> +     return val;
>> +}
>> +
>> +/**
>> + * Emulate x86-style of ioport api implementation for arm/arm64. Included API
>> + * - {in, out}{b, w, l}()
>> + * - {in_p, out_p} {b, w, l} ()
>> + */
>> +
>> +static inline uint8_t inb(unsigned long addr)
>> +{
>> +     return raw_readb(addr);
>> +}
>> +
>> +static inline uint16_t inw(unsigned long addr)
>> +{
>> +     return raw_readw(addr);
>> +}
>> +
>> +static inline uint32_t inl(unsigned long addr)
>> +{
>> +     return raw_readl(addr);
>> +}
>> +
>> +static inline void outb(uint8_t value, unsigned long addr)
>> +{
>> +     raw_writeb(value, addr);
>> +}
>> +
>> +static inline void outw(uint16_t value, unsigned long addr)
>> +{
>> +     raw_writew(value, addr);
>> +}
>> +
>> +static inline void outl(uint32_t value, unsigned long addr)
>> +{
>> +     raw_writel(value, addr);
>> +}
>> +
>> +static inline uint8_t inb_p(unsigned long addr)
>> +{
>> +     return inb(addr);
>> +}
>> +
>> +static inline uint16_t inw_p(unsigned long addr)
>> +{
>> +     return inw(addr);
>> +}
>> +
>> +static inline uint32_t inl_p(unsigned long addr)
>> +{
>> +     return inl(addr);
>> +}
>> +
>> +static inline void outb_p(uint8_t value, unsigned long addr)
>> +{
>> +     outb(value, addr);
>> +}
>> +
>> +static inline void outw_p(uint16_t value, unsigned long addr)
>> +{
>> +     outw(value, addr);
>> +}
>> +
>> +static inline void outl_p(uint32_t value, unsigned long addr)
>> +{
>> +     outl(value, addr);
>> +}
>> +
>> +#ifdef __cplusplus
>> +}
>> +#endif
>> +
>> +#endif /* _RTE_IO_ARM32_H_ */
>> diff --git a/lib/librte_eal/common/include/arch/arm/rte_io_64.h b/lib/librte_eal/common/include/arch/arm/rte_io_64.h
>> new file mode 100644
>> index 0000000..b601f2a
>> --- /dev/null
>> +++ b/lib/librte_eal/common/include/arch/arm/rte_io_64.h
>> @@ -0,0 +1,155 @@
>> +/*
>> + *   BSD LICENSE
>> + *
>> + *   Copyright(c) 2015 Cavium Networks. All rights reserved.
>> + *   All rights reserved.
>> + *
>> + *   Copyright(c) 2010-2015 Intel Corporation. All rights reserved.
>> + *   All rights reserved.
>> + *
>> + *   Redistribution and use in source and binary forms, with or without
>> + *   modification, are permitted provided that the following conditions
>> + *   are met:
>> + *
>> + *     * Redistributions of source code must retain the above copyright
>> + *       notice, this list of conditions and the following disclaimer.
>> + *     * Redistributions in binary form must reproduce the above copyright
>> + *       notice, this list of conditions and the following disclaimer in
>> + *       the documentation and/or other materials provided with the
>> + *       distribution.
>> + *     * Neither the name of Intel Corporation nor the names of its
>> + *       contributors may be used to endorse or promote products derived
>> + *       from this software without specific prior written permission.
>> + *
>> + *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
>> + *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
>> + *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
>> + *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
>> + *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
>> + *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
>> + *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
>> + *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
>> + *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
>> + *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
>> + *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
>> + */
>> +
>> +#ifndef _RTE_IO_ARM64_H_
>> +#define _RTE_IO_ARM64_H_
>> +
>> +#ifdef __cplusplus
>> +extern "C" {
>> +#endif
>> +
>> +#include "generic/rte_io.h"
>> +
>> +/*
>> + * Generic IO read/write api for arm64: Refer TRM
>> + */
>> +static inline void raw_writeb(uint8_t val, uint64_t addr)
>> +{
>> +     asm volatile("strb %w0, [%1]" : : "r" (val), "r" (addr));
>> +}
>> +
>> +static inline void raw_writew(uint16_t val, uint64_t addr)
>> +{
>> +     asm volatile("strh %w0, [%1]" : : "r" (val), "r" (addr));
>> +}
>> +
>> +static inline void raw_writel(uint32_t val, uint64_t addr)
>> +{
>> +     asm volatile("str %w0, [%1]" : : "r" (val), "r" (addr));
>> +}
>> +
>> +static inline uint8_t raw_readb(uint64_t addr)
>> +{
>> +     uint8_t val;
>> +     asm volatile("ldrb %w0, [%1]" : "=r" (val) : "r" (addr));
>> +     return val;
>> +}
>> +
>> +static inline uint16_t raw_readw(uint64_t addr)
>> +{
>> +     uint16_t val;
>> +     asm volatile("ldrh %w0, [%1]" : "=r" (val) : "r" (addr));
>> +     return val;
>> +}
>> +
>> +static inline uint32_t raw_readl(uint64_t addr)
>> +{
>> +     uint32_t val;
>> +     asm volatile("ldr %w0, [%1]" : "=r" (val) : "r" (addr));
>> +     return val;
>> +}
>> +
>> +/**
>> + * Emulate x86-style of ioport api implementation for arm/arm64. Included API
>> + * - {in, out}{b, w, l}()
>> + * - {in_p, out_p} {b, w, l} ()
>> + */
>> +
>> +static inline uint8_t inb(unsigned long addr)
>> +{
>> +     return raw_readb(addr);
>> +}
>> +
>> +static inline uint16_t inw(unsigned long addr)
>> +{
>> +     return raw_readw(addr);
>> +}
>> +
>> +static inline uint32_t inl(unsigned long addr)
>> +{
>> +     return raw_readl(addr);
>> +}
>> +
>> +static inline void outb(uint8_t value, unsigned long addr)
>> +{
>> +     raw_writeb(value, addr);
>> +}
>> +
>> +static inline void outw(uint16_t value, unsigned long addr)
>> +{
>> +     raw_writew(value, addr);
>> +}
>> +
>> +static inline void outl(uint32_t value, unsigned long addr)
>> +{
>> +     raw_writel(value, addr);
>> +}
>> +
>> +static inline uint8_t inb_p(unsigned long addr)
>> +{
>> +     return inb(addr);
>> +}
>> +
>> +static inline uint16_t inw_p(unsigned long addr)
>> +{
>> +     return inw(addr);
>> +}
>> +
>> +static inline uint32_t inl_p(unsigned long addr)
>> +{
>> +     return inl(addr);
>> +}
>> +
>> +static inline void outb_p(uint8_t value, unsigned long addr)
>> +{
>> +     outb(value, addr);
>> +}
>> +
>> +static inline void outw_p(uint16_t value, unsigned long addr)
>> +{
>> +     outw(value, addr);
>> +}
>> +
>> +static inline void outl_p(uint32_t value, unsigned long addr)
>> +{
>> +     outl(value, addr);
>> +}
>> +
>> +#ifdef __cplusplus
>> +}
>> +#endif
>> +
>> +#endif /* _RTE_IO_ARM64_H_ */
>> diff --git a/lib/librte_eal/common/include/generic/rte_io.h b/lib/librte_eal/common/include/generic/rte_io.h
>> new file mode 100644
>> index 0000000..7cc4279
>> --- /dev/null
>> +++ b/lib/librte_eal/common/include/generic/rte_io.h
>> @@ -0,0 +1,81 @@
>> +/*
>> + *   BSD LICENSE
>> + *
>> + *   Copyright(c) 2015 Cavium Networks. All rights reserved.
>> + *   All rights reserved.
>> + *
>> + *   Copyright(c) 2010-2015 Intel Corporation. All rights reserved.
>> + *   All rights reserved.
>> + *
>> + *   Redistribution and use in source and binary forms, with or without
>> + *   modification, are permitted provided that the following conditions
>> + *   are met:
>> + *
>> + *     * Redistributions of source code must retain the above copyright
>> + *       notice, this list of conditions and the following disclaimer.
>> + *     * Redistributions in binary form must reproduce the above copyright
>> + *       notice, this list of conditions and the following disclaimer in
>> + *       the documentation and/or other materials provided with the
>> + *       distribution.
>> + *     * Neither the name of Intel Corporation nor the names of its
>> + *       contributors may be used to endorse or promote products derived
>> + *       from this software without specific prior written permission.
>> + *
>> + *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
>> + *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
>> + *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
>> + *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
>> + *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
>> + *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
>> + *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
>> + *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
>> + *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
>> + *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
>> + *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
>> + */
>> +
>> +#ifndef _RTE_IO_H_
>> +#define _RTE_IO_H_
>> +
>> +/**
>> + * @file
>> + *
>> + * IO operations.
>> + *
>> + * This file defines an API for IO rd/wr inline-functions, API's of the style
>> + * {in , out}[bwl] and {in_p, out_p} [bwl].  which are architecture-dependent.
>> + * Used by non-x86 archs. In particular used by arm/arm64 arch.
>> + */
>> +
>> +#include <stdint.h>
>> +
>> +#if defined(RTE_ARCH_X86_64) || defined(RTE_ARCH_I686)
>
> Shouldn't be
> #if defined(RTE_ARCH_X86_64) || defined(RTE_ARCH_I686) ||
> defined(RTE_ARCH_X86_X32) ?
>

sys/io.h code incident in dpdk using similar ifdef so used, I guess
your are right on including X86_X32 as because it is now
include/generic/ direcotry. We'll do in v3. Thanks

>
>> +#include <sys/io.h>
>> +#else
>> +
>> +/**
>> + * Emulate x86-style of ioport api implementation for arm/arm64. Included API
>> + * - {in, out}{b, w, l}()
>> + * - {in_p, out_p} {b, w, l} ()
>> + */
>> +
>> +static inline uint8_t inb(unsigned long addr);
>> +static inline uint16_t inw(unsigned long addr);
>> +static inline uint32_t inl(unsigned long addr);
>> +
>> +static inline void outb(uint8_t value, unsigned long addr);
>> +static inline void outw(uint16_t value, unsigned long addr);
>> +static inline void outl(uint32_t value, unsigned long addr);
>> +
>> +static inline uint8_t inb_p(unsigned long addr);
>> +static inline uint16_t inw_p(unsigned long addr);
>> +static inline uint32_t inl_p(unsigned long addr);
>> +
>> +static inline void outb_p(uint8_t value, unsigned long addr);
>> +static inline void outw_p(uint16_t value, unsigned long addr);
>> +static inline void outl_p(uint32_t value, unsigned long addr);
>> +
>> +#endif
>> +
>> +#endif /* _RTE_IO_H_ */
>> +
>> --
>> 1.7.9.5
>>