[PATCH] RISC-V: Add PCIe I/O BAR memory mapping

[PATCH] RISC-V: Add PCIe I/O BAR memory mapping

[Yash Shah]

For I/O BARs to work correctly on RISC-V Linux, we need to establish a
reserved memory region for them, so that drivers that wish to use I/O BARs
can issue reads and writes against a memory region that is mapped to the
host PCIe controller's I/O BAR MMIO mapping.

Signed-off-by: Yash Shah <yash.shah@sifive.com>
---
 arch/riscv/include/asm/io.h      | 7 +++++++
 arch/riscv/include/asm/pgtable.h | 7 ++++++-
 2 files changed, 13 insertions(+), 1 deletion(-)

diff --git a/arch/riscv/include/asm/io.h b/arch/riscv/include/asm/io.h
index fc1189a..3ba4d93 100644
--- a/arch/riscv/include/asm/io.h
+++ b/arch/riscv/include/asm/io.h
@@ -13,6 +13,7 @@
 
 #include <linux/types.h>
 #include <asm/mmiowb.h>
+#include <asm/pgtable.h>
 
 extern void __iomem *ioremap(phys_addr_t offset, unsigned long size);
 
@@ -162,6 +163,12 @@ static inline u64 __raw_readq(const volatile void __iomem *addr)
 #endif
 
 /*
+ *  I/O port access constants.
+ */
+#define IO_SPACE_LIMIT		(PCI_IO_SIZE - 1)
+#define PCI_IOBASE		((void __iomem *)PCI_IO_START)
+
+/*
  * Emulation routines for the port-mapped IO space used by some PCI drivers.
  * These are defined as being "fully synchronous", but also "not guaranteed to
  * be fully ordered with respect to other memory and I/O operations".  We're
diff --git a/arch/riscv/include/asm/pgtable.h b/arch/riscv/include/asm/pgtable.h
index 7fc5e4a..d78cc74 100644
--- a/arch/riscv/include/asm/pgtable.h
+++ b/arch/riscv/include/asm/pgtable.h
@@ -7,6 +7,7 @@
 #define _ASM_RISCV_PGTABLE_H
 
 #include <linux/mmzone.h>
+#include <linux/sizes.h>
 
 #include <asm/pgtable-bits.h>
 
@@ -88,6 +89,7 @@ extern pgd_t swapper_pg_dir[];
 #define VMALLOC_SIZE     (KERN_VIRT_SIZE >> 1)
 #define VMALLOC_END      (PAGE_OFFSET - 1)
 #define VMALLOC_START    (PAGE_OFFSET - VMALLOC_SIZE)
+#define PCI_IO_SIZE      SZ_16M
 
 /*
  * Roughly size the vmemmap space to be large enough to fit enough
@@ -102,7 +104,10 @@ extern pgd_t swapper_pg_dir[];
 
 #define vmemmap		((struct page *)VMEMMAP_START)
 
-#define FIXADDR_TOP      (VMEMMAP_START)
+#define PCI_IO_END       VMEMMAP_START
+#define PCI_IO_START     (PCI_IO_END - PCI_IO_SIZE)
+#define FIXADDR_TOP      PCI_IO_START
+
 #ifdef CONFIG_64BIT
 #define FIXADDR_SIZE     PMD_SIZE
 #else
-- 
2.7.4


_______________________________________________
linux-riscv mailing list
linux-riscv@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-riscv

Re: [PATCH] RISC-V: Add PCIe I/O BAR memory mapping

[Logan Gunthorpe]

On 2019-10-24 3:14 a.m., Yash Shah wrote:
> For I/O BARs to work correctly on RISC-V Linux, we need to establish a
> reserved memory region for them, so that drivers that wish to use I/O BARs
> can issue reads and writes against a memory region that is mapped to the
> host PCIe controller's I/O BAR MMIO mapping.

I don't think other arches do this. ioremap() typically just uses
virtual address space in the VMALLOC region, PCI doesn't need it's own
range. As far as I know the ioremap() implementation in riscv already
does this.

In any case, 16MB for PCI bar space seems woefully inadequate.

Logan


> Signed-off-by: Yash Shah <yash.shah@sifive.com>
> ---
>  arch/riscv/include/asm/io.h      | 7 +++++++
>  arch/riscv/include/asm/pgtable.h | 7 ++++++-
>  2 files changed, 13 insertions(+), 1 deletion(-)
> 
> diff --git a/arch/riscv/include/asm/io.h b/arch/riscv/include/asm/io.h
> index fc1189a..3ba4d93 100644
> --- a/arch/riscv/include/asm/io.h
> +++ b/arch/riscv/include/asm/io.h
> @@ -13,6 +13,7 @@
>  
>  #include <linux/types.h>
>  #include <asm/mmiowb.h>
> +#include <asm/pgtable.h>
>  
>  extern void __iomem *ioremap(phys_addr_t offset, unsigned long size);
>  
> @@ -162,6 +163,12 @@ static inline u64 __raw_readq(const volatile void __iomem *addr)
>  #endif
>  
>  /*
> + *  I/O port access constants.
> + */
> +#define IO_SPACE_LIMIT		(PCI_IO_SIZE - 1)
> +#define PCI_IOBASE		((void __iomem *)PCI_IO_START)
> +
> +/*
>   * Emulation routines for the port-mapped IO space used by some PCI drivers.
>   * These are defined as being "fully synchronous", but also "not guaranteed to
>   * be fully ordered with respect to other memory and I/O operations".  We're
> diff --git a/arch/riscv/include/asm/pgtable.h b/arch/riscv/include/asm/pgtable.h
> index 7fc5e4a..d78cc74 100644
> --- a/arch/riscv/include/asm/pgtable.h
> +++ b/arch/riscv/include/asm/pgtable.h
> @@ -7,6 +7,7 @@
>  #define _ASM_RISCV_PGTABLE_H
>  
>  #include <linux/mmzone.h>
> +#include <linux/sizes.h>
>  
>  #include <asm/pgtable-bits.h>
>  
> @@ -88,6 +89,7 @@ extern pgd_t swapper_pg_dir[];
>  #define VMALLOC_SIZE     (KERN_VIRT_SIZE >> 1)
>  #define VMALLOC_END      (PAGE_OFFSET - 1)
>  #define VMALLOC_START    (PAGE_OFFSET - VMALLOC_SIZE)
> +#define PCI_IO_SIZE      SZ_16M
>  
>  /*
>   * Roughly size the vmemmap space to be large enough to fit enough
> @@ -102,7 +104,10 @@ extern pgd_t swapper_pg_dir[];
>  
>  #define vmemmap		((struct page *)VMEMMAP_START)
>  
> -#define FIXADDR_TOP      (VMEMMAP_START)
> +#define PCI_IO_END       VMEMMAP_START
> +#define PCI_IO_START     (PCI_IO_END - PCI_IO_SIZE)
> +#define FIXADDR_TOP      PCI_IO_START
> +
>  #ifdef CONFIG_64BIT
>  #define FIXADDR_SIZE     PMD_SIZE
>  #else
> 

_______________________________________________
linux-riscv mailing list
linux-riscv@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-riscv

Re: [PATCH] RISC-V: Add PCIe I/O BAR memory mapping

[Paul Walmsley]

On Thu, 24 Oct 2019, Logan Gunthorpe wrote:

> On 2019-10-24 3:14 a.m., Yash Shah wrote:
> > For I/O BARs to work correctly on RISC-V Linux, we need to establish a
> > reserved memory region for them, so that drivers that wish to use I/O BARs
> > can issue reads and writes against a memory region that is mapped to the
> > host PCIe controller's I/O BAR MMIO mapping.
> 
> I don't think other arches do this. 

$ git grep 'define PCI_IOBASE' arch/ 
arch/arm/include/asm/io.h:#define PCI_IOBASE            ((void __iomem *)PCI_IO_VIRT_BASE)
arch/arm64/include/asm/io.h:#define PCI_IOBASE          ((void __iomem *)PCI_IO_START)
arch/m68k/include/asm/io_no.h:#define PCI_IOBASE        ((void __iomem *) PCI_IO_PA)
arch/microblaze/include/asm/io.h:#define PCI_IOBASE     ((void __iomem *)_IO_BASE)
arch/unicore32/include/asm/io.h:#define PCI_IOBASE      PKUNITY_PCILIO_BASE
arch/xtensa/include/asm/io.h:#define PCI_IOBASE         ((void __iomem *)XCHAL_KIO_BYPASS_VADDR)
$

This is for the old x86-style, non-memory mapped I/O address space the 
legacy PCI stuff that one would use in{b,w,l}()/out{b,w,l}() for.

Yash, you might consider updating your patch description to note that this 
is for "legacy I/O BARs (i.e., non-MMIO BARs)" or something similar.  That 
might make things clearer.

> ioremap() typically just uses virtual address space in the VMALLOC 
> region, PCI doesn't need it's own range. As far as I know the ioremap() 
> implementation in riscv already does this.
> 
> In any case, 16MB for PCI bar space seems woefully inadequate.

The modern MMIO PCI resources wind up in jost controller apertures, which 
as you note, are usually much larger.  They don't go in this legacy space.

Regarding sizing - I haven't seen any PCIe cards with more than 64KiB of 
legacy I/O resources.  (16MiB / 64KiB) = 256, so 16MiB sounds reasonable 
from that point of view?  ARM64 is using that.


- Paul

_______________________________________________
linux-riscv mailing list
linux-riscv@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-riscv

Re: [PATCH] RISC-V: Add PCIe I/O BAR memory mapping

[Logan Gunthorpe]

On 2019-10-24 10:51 a.m., Paul Walmsley wrote:
> On Thu, 24 Oct 2019, Logan Gunthorpe wrote:
> 
>> On 2019-10-24 3:14 a.m., Yash Shah wrote:
>>> For I/O BARs to work correctly on RISC-V Linux, we need to establish a
>>> reserved memory region for them, so that drivers that wish to use I/O BARs
>>> can issue reads and writes against a memory region that is mapped to the
>>> host PCIe controller's I/O BAR MMIO mapping.
>>
>> I don't think other arches do this. 
> 
> $ git grep 'define PCI_IOBASE' arch/ 
> arch/arm/include/asm/io.h:#define PCI_IOBASE            ((void __iomem *)PCI_IO_VIRT_BASE)
> arch/arm64/include/asm/io.h:#define PCI_IOBASE          ((void __iomem *)PCI_IO_START)
> arch/m68k/include/asm/io_no.h:#define PCI_IOBASE        ((void __iomem *) PCI_IO_PA)
> arch/microblaze/include/asm/io.h:#define PCI_IOBASE     ((void __iomem *)_IO_BASE)
> arch/unicore32/include/asm/io.h:#define PCI_IOBASE      PKUNITY_PCILIO_BASE
> arch/xtensa/include/asm/io.h:#define PCI_IOBASE         ((void __iomem *)XCHAL_KIO_BYPASS_VADDR)
> $
> 
> This is for the old x86-style, non-memory mapped I/O address space the 
> legacy PCI stuff that one would use in{b,w,l}()/out{b,w,l}() for.
> 
> Yash, you might consider updating your patch description to note that this 
> is for "legacy I/O BARs (i.e., non-MMIO BARs)" or something similar.  That 
> might make things clearer.

Ah, right, yes, that would clear things up.

Logan

_______________________________________________
linux-riscv mailing list
linux-riscv@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-riscv

RE: [PATCH] RISC-V: Add PCIe I/O BAR memory mapping

[Yash Shah]

> On Thu, 24 Oct 2019, Logan Gunthorpe wrote:
> 
> > On 2019-10-24 3:14 a.m., Yash Shah wrote:
> > > For I/O BARs to work correctly on RISC-V Linux, we need to establish
> > > a reserved memory region for them, so that drivers that wish to use
> > > I/O BARs can issue reads and writes against a memory region that is
> > > mapped to the host PCIe controller's I/O BAR MMIO mapping.
> >
> > I don't think other arches do this.
> 
> $ git grep 'define PCI_IOBASE' arch/
> arch/arm/include/asm/io.h:#define PCI_IOBASE            ((void __iomem
> *)PCI_IO_VIRT_BASE)
> arch/arm64/include/asm/io.h:#define PCI_IOBASE          ((void __iomem
> *)PCI_IO_START)
> arch/m68k/include/asm/io_no.h:#define PCI_IOBASE        ((void __iomem *)
> PCI_IO_PA)
> arch/microblaze/include/asm/io.h:#define PCI_IOBASE     ((void __iomem
> *)_IO_BASE)
> arch/unicore32/include/asm/io.h:#define PCI_IOBASE
> PKUNITY_PCILIO_BASE
> arch/xtensa/include/asm/io.h:#define PCI_IOBASE         ((void __iomem
> *)XCHAL_KIO_BYPASS_VADDR)
> $
> 
> This is for the old x86-style, non-memory mapped I/O address space the
> legacy PCI stuff that one would use in{b,w,l}()/out{b,w,l}() for.
> 
> Yash, you might consider updating your patch description to note that this is
> for "legacy I/O BARs (i.e., non-MMIO BARs)" or something similar.  That
> might make things clearer.

Sure, will update the description and send v2.

- Yash

> 
> > ioremap() typically just uses virtual address space in the VMALLOC
> > region, PCI doesn't need it's own range. As far as I know the
> > ioremap() implementation in riscv already does this.
> >
> > In any case, 16MB for PCI bar space seems woefully inadequate.
> 
> The modern MMIO PCI resources wind up in jost controller apertures, which
> as you note, are usually much larger.  They don't go in this legacy space.
> 
> Regarding sizing - I haven't seen any PCIe cards with more than 64KiB of
> legacy I/O resources.  (16MiB / 64KiB) = 256, so 16MiB sounds reasonable
> from that point of view?  ARM64 is using that.
> 
> 
> - Paul

_______________________________________________
linux-riscv mailing list
linux-riscv@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-riscv