RE: [PATCH 05/25] dmaengine: dw-edma: Convert ll/dt phys-address to PCIe bus/DMA address

RE: [PATCH 05/25] dmaengine: dw-edma: Convert ll/dt phys-address to PCIe bus/DMA address

[Frank Li]

> -----Original Message-----
> From: Serge Semin <Sergey.Semin@baikalelectronics.ru>
> Sent: Wednesday, March 23, 2022 8:48 PM
> To: Gustavo Pimentel <gustavo.pimentel@synopsys.com>; Vinod Koul
> <vkoul@kernel.org>; Jingoo Han <jingoohan1@gmail.com>; Bjorn Helgaas
> <bhelgaas@google.com>; Frank Li <frank.li@nxp.com>; Manivannan Sadhasivam
> <manivannan.sadhasivam@linaro.org>
> Cc: Serge Semin <Sergey.Semin@baikalelectronics.ru>; Serge Semin
> <fancer.lancer@gmail.com>; Alexey Malahov
> <Alexey.Malahov@baikalelectronics.ru>; Pavel Parkhomenko
> <Pavel.Parkhomenko@baikalelectronics.ru>; Lorenzo Pieralisi
> <lorenzo.pieralisi@arm.com>; Rob Herring <robh@kernel.org>; Krzysztof
> Wilczyński <kw@linux.com>; linux-pci@vger.kernel.org;
> dmaengine@vger.kernel.org; linux-kernel@vger.kernel.org; Gustavo Pimentel
> <Gustavo.Pimentel@synopsys.com>
> Subject: [EXT] [PATCH 05/25] dmaengine: dw-edma: Convert ll/dt phys-address
> to PCIe bus/DMA address
> 
> Caution: EXT Email
> 
> In accordance with the dw_edma_region.paddr field semantics it is supposed
> to be initialized with a memory base address visible by the DW eDMA
> controller. If the DMA engine is embedded into the DW PCIe Host/EP
> controller, then the address should belong to the Local CPU/Application
> memory. If eDMA is remotely accessible across the PCIe bus via the PCIe
> memory IOs, then the address needs to be a part of the PCIe bus memory
> space. The later case hasn't been well covered in the corresponding
> glue-driver. Since in general the PCIe memory space doesn't have to match
> the CPU memory space and the pci_dev.resource[] arrays contain the
> resources defined in the CPU memory space, a proper conversion needs to be
> performed, otherwise either the driver won't properly work or much worse
> the memory corruption will happen. The conversion can be done by means of
> the pci_bus_address() method. Let's use it to retrieve the LL, DT and CSRs
> PCIe memory ranges.

Actually, This driver is dead driver. And no one use it and don't know
How to test it. 

I think pci_bus_address can't resolve this problem.

If remote side using iATU map bar 0 into 0x7000_0000

I supposed DMA link should be use 0x7000_0000, how host side pci_bus_address
know such iATU mapping information of EP side?

Best regards
Frank Li


> 
> Note in addition to that we need to extend the dw_edma_region.paddr field
> size. The field normally contains a memory range base address to be set in
> the DW eDMA Linked-List pointer register or as a base address of the
> Linked-List data buffer. In accordance with [1] the LL range is supposed
> to be created in the Local CPU/Application memory, but depending on the DW
> eDMA utilization the memory can be created as a part of the PCIe bus
> address space (as in the case of the DW PCIe EP prototype kit). Thus in
> the former case the dw_edma_region.paddr field should have the dma_addr_t
> type, while in the later one - pci_bus_addr_t. Seeing the corresponding
> CSRs are always 64-bits wide let's convert the dw_edma_region.paddr field
> type to be u64 and let the client code logic to make sure it has a valid
> address visible by the DW eDMA controller. For instance the DW eDMA PCIe
> glue-driver initializes the field with the addresses from the PCIe bus
> memory space.
> 
> [1] DesignWare Cores PCI Express Controller Databook - DWC PCIe Root Port,
>     v.5.40a, March 2019, p.1103
> 
> Fixes: 41aaff2a2ac0 ("dmaengine: Add Synopsys eDMA IP PCIe glue-logic")
> Signed-off-by: Serge Semin <Sergey.Semin@baikalelectronics.ru>
> ---
>  drivers/dma/dw-edma/dw-edma-pcie.c | 8 ++++----
>  include/linux/dma/edma.h           | 2 +-
>  2 files changed, 5 insertions(+), 5 deletions(-)
> 
> diff --git a/drivers/dma/dw-edma/dw-edma-pcie.c b/drivers/dma/dw-edma/dw-
> edma-pcie.c
> index d6b5e2463884..04c95cba1244 100644
> --- a/drivers/dma/dw-edma/dw-edma-pcie.c
> +++ b/drivers/dma/dw-edma/dw-edma-pcie.c
> @@ -231,7 +231,7 @@ static int dw_edma_pcie_probe(struct pci_dev *pdev,
>                         return -ENOMEM;
> 
>                 ll_region->vaddr += ll_block->off;
> -               ll_region->paddr = pdev->resource[ll_block->bar].start;
> +               ll_region->paddr = pci_bus_address(pdev, ll_block->bar);
>                 ll_region->paddr += ll_block->off;
>                 ll_region->sz = ll_block->sz;
> 
> @@ -240,7 +240,7 @@ static int dw_edma_pcie_probe(struct pci_dev *pdev,
>                         return -ENOMEM;
> 
>                 dt_region->vaddr += dt_block->off;
> -               dt_region->paddr = pdev->resource[dt_block->bar].start;
> +               dt_region->paddr = pci_bus_address(pdev, dt_block->bar);
>                 dt_region->paddr += dt_block->off;
>                 dt_region->sz = dt_block->sz;
>         }
> @@ -256,7 +256,7 @@ static int dw_edma_pcie_probe(struct pci_dev *pdev,
>                         return -ENOMEM;
> 
>                 ll_region->vaddr += ll_block->off;
> -               ll_region->paddr = pdev->resource[ll_block->bar].start;
> +               ll_region->paddr = pci_bus_address(pdev, ll_block->bar);
>                 ll_region->paddr += ll_block->off;
>                 ll_region->sz = ll_block->sz;
> 
> @@ -265,7 +265,7 @@ static int dw_edma_pcie_probe(struct pci_dev *pdev,
>                         return -ENOMEM;
> 
>                 dt_region->vaddr += dt_block->off;
> -               dt_region->paddr = pdev->resource[dt_block->bar].start;
> +               dt_region->paddr = pci_bus_address(pdev, dt_block->bar);
>                 dt_region->paddr += dt_block->off;
>                 dt_region->sz = dt_block->sz;
>         }
> diff --git a/include/linux/dma/edma.h b/include/linux/dma/edma.h
> index 8897f8a79b52..5abac9640a4e 100644
> --- a/include/linux/dma/edma.h
> +++ b/include/linux/dma/edma.h
> @@ -18,7 +18,7 @@
>  struct dw_edma;
> 
>  struct dw_edma_region {
> -       phys_addr_t     paddr;
> +       u64             paddr;
>         void __iomem    *vaddr;
>         size_t          sz;
>  };
> --
> 2.35.1

Re: [PATCH 05/25] dmaengine: dw-edma: Convert ll/dt phys-address to PCIe bus/DMA address

[Serge Semin]

On Thu, Mar 24, 2022 at 07:23:47PM +0000, Frank Li wrote:
> 
> 
> > -----Original Message-----
> > From: Serge Semin <Sergey.Semin@baikalelectronics.ru>
> > Sent: Wednesday, March 23, 2022 8:48 PM
> > To: Gustavo Pimentel <gustavo.pimentel@synopsys.com>; Vinod Koul
> > <vkoul@kernel.org>; Jingoo Han <jingoohan1@gmail.com>; Bjorn Helgaas
> > <bhelgaas@google.com>; Frank Li <frank.li@nxp.com>; Manivannan Sadhasivam
> > <manivannan.sadhasivam@linaro.org>
> > Cc: Serge Semin <Sergey.Semin@baikalelectronics.ru>; Serge Semin
> > <fancer.lancer@gmail.com>; Alexey Malahov
> > <Alexey.Malahov@baikalelectronics.ru>; Pavel Parkhomenko
> > <Pavel.Parkhomenko@baikalelectronics.ru>; Lorenzo Pieralisi
> > <lorenzo.pieralisi@arm.com>; Rob Herring <robh@kernel.org>; Krzysztof
> > Wilczyński <kw@linux.com>; linux-pci@vger.kernel.org;
> > dmaengine@vger.kernel.org; linux-kernel@vger.kernel.org; Gustavo Pimentel
> > <Gustavo.Pimentel@synopsys.com>
> > Subject: [EXT] [PATCH 05/25] dmaengine: dw-edma: Convert ll/dt phys-address
> > to PCIe bus/DMA address
> > 
> > Caution: EXT Email
> > 
> > In accordance with the dw_edma_region.paddr field semantics it is supposed
> > to be initialized with a memory base address visible by the DW eDMA
> > controller. If the DMA engine is embedded into the DW PCIe Host/EP
> > controller, then the address should belong to the Local CPU/Application
> > memory. If eDMA is remotely accessible across the PCIe bus via the PCIe
> > memory IOs, then the address needs to be a part of the PCIe bus memory
> > space. The later case hasn't been well covered in the corresponding
> > glue-driver. Since in general the PCIe memory space doesn't have to match
> > the CPU memory space and the pci_dev.resource[] arrays contain the
> > resources defined in the CPU memory space, a proper conversion needs to be
> > performed, otherwise either the driver won't properly work or much worse
> > the memory corruption will happen. The conversion can be done by means of
> > the pci_bus_address() method. Let's use it to retrieve the LL, DT and CSRs
> > PCIe memory ranges.
> 

> Actually, This driver is dead driver. And no one use it and don't know
> How to test it. 

My considerations were based on the commit logs, the databooks info and the
code logic.

> 
> I think pci_bus_address can't resolve this problem.
> 
> If remote side using iATU map bar 0 into 0x7000_0000
> 
> I supposed DMA link should be use 0x7000_0000, how host side pci_bus_address
> know such iATU mapping information of EP side?

pci_dev.resource[] entries contain the CPU-physical addresses of the
corresponding BARs. In general the CPU and PCIe bus address spaces may
mismatch. That is for instance the 0xF000_0000 CPU address can get to
be mapped to the PCIe address 0x7000_0000. The later address in its
turn can be used to initialize the eDMA BARx register. Since in case
of the remote eDMA setup the application SAR/DAR address is supposed
to be set with the value within the BARx range, we need to have the
BARx PCIe address, not the CPU one. In a cross-platform way the real
BAR' address range can be retrieved by means of the pci_bus_address()
method. That's what this patch is about.

-Sergey

> 
> Best regards
> Frank Li
> 
> 
> > 
> > Note in addition to that we need to extend the dw_edma_region.paddr field
> > size. The field normally contains a memory range base address to be set in
> > the DW eDMA Linked-List pointer register or as a base address of the
> > Linked-List data buffer. In accordance with [1] the LL range is supposed
> > to be created in the Local CPU/Application memory, but depending on the DW
> > eDMA utilization the memory can be created as a part of the PCIe bus
> > address space (as in the case of the DW PCIe EP prototype kit). Thus in
> > the former case the dw_edma_region.paddr field should have the dma_addr_t
> > type, while in the later one - pci_bus_addr_t. Seeing the corresponding
> > CSRs are always 64-bits wide let's convert the dw_edma_region.paddr field
> > type to be u64 and let the client code logic to make sure it has a valid
> > address visible by the DW eDMA controller. For instance the DW eDMA PCIe
> > glue-driver initializes the field with the addresses from the PCIe bus
> > memory space.
> > 
> > [1] DesignWare Cores PCI Express Controller Databook - DWC PCIe Root Port,
> >     v.5.40a, March 2019, p.1103
> > 
> > Fixes: 41aaff2a2ac0 ("dmaengine: Add Synopsys eDMA IP PCIe glue-logic")
> > Signed-off-by: Serge Semin <Sergey.Semin@baikalelectronics.ru>
> > ---
> >  drivers/dma/dw-edma/dw-edma-pcie.c | 8 ++++----
> >  include/linux/dma/edma.h           | 2 +-
> >  2 files changed, 5 insertions(+), 5 deletions(-)
> > 
> > diff --git a/drivers/dma/dw-edma/dw-edma-pcie.c b/drivers/dma/dw-edma/dw-
> > edma-pcie.c
> > index d6b5e2463884..04c95cba1244 100644
> > --- a/drivers/dma/dw-edma/dw-edma-pcie.c
> > +++ b/drivers/dma/dw-edma/dw-edma-pcie.c
> > @@ -231,7 +231,7 @@ static int dw_edma_pcie_probe(struct pci_dev *pdev,
> >                         return -ENOMEM;
> > 
> >                 ll_region->vaddr += ll_block->off;
> > -               ll_region->paddr = pdev->resource[ll_block->bar].start;
> > +               ll_region->paddr = pci_bus_address(pdev, ll_block->bar);
> >                 ll_region->paddr += ll_block->off;
> >                 ll_region->sz = ll_block->sz;
> > 
> > @@ -240,7 +240,7 @@ static int dw_edma_pcie_probe(struct pci_dev *pdev,
> >                         return -ENOMEM;
> > 
> >                 dt_region->vaddr += dt_block->off;
> > -               dt_region->paddr = pdev->resource[dt_block->bar].start;
> > +               dt_region->paddr = pci_bus_address(pdev, dt_block->bar);
> >                 dt_region->paddr += dt_block->off;
> >                 dt_region->sz = dt_block->sz;
> >         }
> > @@ -256,7 +256,7 @@ static int dw_edma_pcie_probe(struct pci_dev *pdev,
> >                         return -ENOMEM;
> > 
> >                 ll_region->vaddr += ll_block->off;
> > -               ll_region->paddr = pdev->resource[ll_block->bar].start;
> > +               ll_region->paddr = pci_bus_address(pdev, ll_block->bar);
> >                 ll_region->paddr += ll_block->off;
> >                 ll_region->sz = ll_block->sz;
> > 
> > @@ -265,7 +265,7 @@ static int dw_edma_pcie_probe(struct pci_dev *pdev,
> >                         return -ENOMEM;
> > 
> >                 dt_region->vaddr += dt_block->off;
> > -               dt_region->paddr = pdev->resource[dt_block->bar].start;
> > +               dt_region->paddr = pci_bus_address(pdev, dt_block->bar);
> >                 dt_region->paddr += dt_block->off;
> >                 dt_region->sz = dt_block->sz;
> >         }
> > diff --git a/include/linux/dma/edma.h b/include/linux/dma/edma.h
> > index 8897f8a79b52..5abac9640a4e 100644
> > --- a/include/linux/dma/edma.h
> > +++ b/include/linux/dma/edma.h
> > @@ -18,7 +18,7 @@
> >  struct dw_edma;
> > 
> >  struct dw_edma_region {
> > -       phys_addr_t     paddr;
> > +       u64             paddr;
> >         void __iomem    *vaddr;
> >         size_t          sz;
> >  };
> > --
> > 2.35.1
> 

[PATCH 00/25] dmaengine: dw-edma: Add RP/EP local DMA controllers support

[Serge Semin]

This is a final patchset in the series created in the framework of
my Baikal-T1 PCIe/eDMA-related work:

[1: In-progress] clk: Baikal-T1 DDR/PCIe resets and some xGMAC fixes
Link: --submitted--
[2: In-progress] PCI: dwc: Various fixes and cleanups
Link: --submitted--
[3: In-progress] PCI: dwc: Add dma-ranges/YAML-schema/Baikal-T1 support
Link: --submitted--
[4: In-progress] dmaengine: dw-edma: Add RP/EP local DMA controllers support
Link: --you are looking at it--

Note it is recommended to merge the last patchset after the former ones in
order to prevent merge conflicts. @Bjorn could you merge in this patchset
through your PCIe repo? After getting all the ack'es of course.

Please note originally this series was self content, but due to Frank
being a bit faster in his work submission I had to rebase my patchset onto
his one. So now this patchset turns to be dependent on the Frank' work:
Link: https://lore.kernel.org/dmaengine/20220310192457.3090-1-Frank.Li@nxp.com/
So please review and merge his series first before applying this one.

@Frank, @Manivannan as we agreed here:
Link: https://lore.kernel.org/dmaengine/20220309211204.26050-6-Frank.Li@nxp.com/
this series contains two patches with our joint work. Here they are:
[PATCH 1/25] dmaengine: dw-edma: Drop dma_slave_config.direction field usage
[PATCH 2/25] dmaengine: dw-edma: Fix eDMA Rd/Wr-channels and DMA-direction semantics
@Frank, could you please pick them up and add them to your series in place
of the patches:
[PATCH v5 6/9] dmaengine: dw-edma: Don't rely on the deprecated "direction" member
Link: https://lore.kernel.org/dmaengine/20220310192457.3090-7-Frank.Li@nxp.com/
[PATCH v5 5/9] dmaengine: dw-edma: Fix programming the source & dest addresses for ep
Link: https://lore.kernel.org/dmaengine/20220310192457.3090-6-Frank.Li@nxp.com/
respectively?

@Frank please don't forget to fix your series so the chip->dw field is
initialized after all the probe() initializations are done. For that sake
you also need to make sure that the dw_edma_irq_request(),
dw_edma_channel_setup() and dw_edma_v0_core_debugfs_on() methods take
dw_edma structure pointer as a parameter.

Here is a short summary regarding this patchset. The series starts with
fixes patches. The very first two patches have been modified based on
discussion with @Frank and @Manivannan as I noted in the previous
paragraph. They concern fixing the Read/Write channels xfer semantics.
See the patches description for more details. After that goes the fix of
the dma_direct_map_resource() method, which turned out to be not working
correctly for the case of having devive.dma_range_map being non-empty
(non-empty dma-ranges DT property). Then we discovered that the
dw-edma-pcie.c driver incorrectly initializes the LL/DT base addresses for
the platforms with not matching CPU and PCIe memory spaces. It is fixed by
using the pci_bus_address() method to get a correct base address. After
that you can find a series of interleaved xfers fixes. It turned out the
interleaved transfers implementation didn't work quite correctly from the
very beginning for instance missing src/dst addresses initialization, etc.
In the framework of the next two patches we suggest to add a new
platform-specific callback - pci_addrees() and use to convert the CPU
address to the PCIe space address. It is at least required for the DW eDAM
remote End-point setup on the platforms with not-matching address spaces.
In case of the DW eDMA local RP/EP setup the conversion will be done
automatically by the outbound iATU (if no DMA-bypass flag is specified for
the corresponding iATU window). Then we introduce a set of patches to make
the DebugFS part of the code supporting the multi-eDMA controllers
platforms. It starts with several cleanup patches and is closed joining
the Read/Write channels into a single DMA-device as they originally should
have been. After that you can find the patches with adding the non-atomic
io-64 methods usage, dropping DT-region descriptors allocation, replacing
chip IDs with device name. In addition to that in order to have the eDMA
embedded into the DW PCIe RP/EP supported we need to bypass the
dma-ranges-based memory ranges mapping since in case of the root port DT
node it's applicable for the peripheral PCIe devices only. Finally at the
series closure we introduce a generic DW eDMA controller support being
available in the DW PCIe Host/End-point driver.

Signed-off-by: Serge Semin <Sergey.Semin@baikalelectronics.ru>
Cc: Alexey Malahov <Alexey.Malahov@baikalelectronics.ru>
Cc: Pavel Parkhomenko <Pavel.Parkhomenko@baikalelectronics.ru>
Cc: Lorenzo Pieralisi <lorenzo.pieralisi@arm.com>
Cc: Rob Herring <robh@kernel.org>
Cc: "Krzysztof Wilczyński" <kw@linux.com>
Cc: linux-pci@vger.kernel.org
Cc: dmaengine@vger.kernel.org
Cc: linux-kernel@vger.kernel.org

Serge Semin (25):
  dmaengine: dw-edma: Drop dma_slave_config.direction field usage
  dmaengine: dw-edma: Fix eDMA Rd/Wr-channels and DMA-direction
    semantics
  dma-direct: take dma-ranges/offsets into account in resource mapping
  dmaengine: Fix dma_slave_config.dst_addr description
  dmaengine: dw-edma: Convert ll/dt phys-address to PCIe bus/DMA address
  dmaengine: dw-edma: Fix missing src/dst address of the interleaved
    xfers
  dmaengine: dw-edma: Don't permit non-inc interleaved xfers
  dmaengine: dw-edma: Fix invalid interleaved xfers semantics
  dmaengine: dw-edma: Add CPU to PCIe bus address translation
  dmaengine: dw-edma: Add PCIe bus address getter to the remote EP
    glue-driver
  dmaengine: dw-edma: Drop chancnt initialization
  dmaengine: dw-edma: Fix DebugFS reg entry type
  dmaengine: dw-edma: Stop checking debugfs_create_*() return value
  dmaengine: dw-edma: Add dw_edma prefix to the DebugFS nodes descriptor
  dmaengine: dw-edma: Convert DebugFS descs to being kz-allocated
  dmaengine: dw-edma: Simplify the DebugFS context CSRs init procedure
  dmaengine: dw-edma: Move eDMA data pointer to DebugFS node descriptor
  dmaengine: dw-edma: Join Write/Read channels into a single device
  dmaengine: dw-edma: Use DMA-engine device DebugFS subdirectory
  dmaengine: dw-edma: Use non-atomic io-64 methods
  dmaengine: dw-edma: Drop DT-region allocation
  dmaengine: dw-edma: Replace chip ID number with device name
  dmaengine: dw-edma: Bypass dma-ranges mapping for the local setup
  dmaengine: dw-edma: Skip cleanup procedure if no private data found
  PCI: dwc: Add DW eDMA engine support

 drivers/dma/dw-edma/dw-edma-core.c            | 249 +++++++------
 drivers/dma/dw-edma/dw-edma-core.h            |  10 +-
 drivers/dma/dw-edma/dw-edma-pcie.c            |  24 +-
 drivers/dma/dw-edma/dw-edma-v0-core.c         |  76 ++--
 drivers/dma/dw-edma/dw-edma-v0-core.h         |   1 -
 drivers/dma/dw-edma/dw-edma-v0-debugfs.c      | 350 ++++++++----------
 drivers/dma/dw-edma/dw-edma-v0-debugfs.h      |   5 -
 .../pci/controller/dwc/pcie-designware-ep.c   |   4 +
 .../pci/controller/dwc/pcie-designware-host.c |  13 +-
 drivers/pci/controller/dwc/pcie-designware.c  | 188 ++++++++++
 drivers/pci/controller/dwc/pcie-designware.h  |  23 +-
 include/linux/dma/edma.h                      |  18 +-
 include/linux/dmaengine.h                     |   2 +-
 kernel/dma/direct.c                           |   2 +-
 14 files changed, 598 insertions(+), 367 deletions(-)

-- 
2.35.1

[PATCH 05/25] dmaengine: dw-edma: Convert ll/dt phys-address to PCIe bus/DMA address

[Serge Semin]

In accordance with the dw_edma_region.paddr field semantics it is supposed
to be initialized with a memory base address visible by the DW eDMA
controller. If the DMA engine is embedded into the DW PCIe Host/EP
controller, then the address should belong to the Local CPU/Application
memory. If eDMA is remotely accessible across the PCIe bus via the PCIe
memory IOs, then the address needs to be a part of the PCIe bus memory
space. The later case hasn't been well covered in the corresponding
glue-driver. Since in general the PCIe memory space doesn't have to match
the CPU memory space and the pci_dev.resource[] arrays contain the
resources defined in the CPU memory space, a proper conversion needs to be
performed, otherwise either the driver won't properly work or much worse
the memory corruption will happen. The conversion can be done by means of
the pci_bus_address() method. Let's use it to retrieve the LL, DT and CSRs
PCIe memory ranges.

Note in addition to that we need to extend the dw_edma_region.paddr field
size. The field normally contains a memory range base address to be set in
the DW eDMA Linked-List pointer register or as a base address of the
Linked-List data buffer. In accordance with [1] the LL range is supposed
to be created in the Local CPU/Application memory, but depending on the DW
eDMA utilization the memory can be created as a part of the PCIe bus
address space (as in the case of the DW PCIe EP prototype kit). Thus in
the former case the dw_edma_region.paddr field should have the dma_addr_t
type, while in the later one - pci_bus_addr_t. Seeing the corresponding
CSRs are always 64-bits wide let's convert the dw_edma_region.paddr field
type to be u64 and let the client code logic to make sure it has a valid
address visible by the DW eDMA controller. For instance the DW eDMA PCIe
glue-driver initializes the field with the addresses from the PCIe bus
memory space.

[1] DesignWare Cores PCI Express Controller Databook - DWC PCIe Root Port,
    v.5.40a, March 2019, p.1103

Fixes: 41aaff2a2ac0 ("dmaengine: Add Synopsys eDMA IP PCIe glue-logic")
Signed-off-by: Serge Semin <Sergey.Semin@baikalelectronics.ru>
---
 drivers/dma/dw-edma/dw-edma-pcie.c | 8 ++++----
 include/linux/dma/edma.h           | 2 +-
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/drivers/dma/dw-edma/dw-edma-pcie.c b/drivers/dma/dw-edma/dw-edma-pcie.c
index d6b5e2463884..04c95cba1244 100644
--- a/drivers/dma/dw-edma/dw-edma-pcie.c
+++ b/drivers/dma/dw-edma/dw-edma-pcie.c
@@ -231,7 +231,7 @@ static int dw_edma_pcie_probe(struct pci_dev *pdev,
 			return -ENOMEM;
 
 		ll_region->vaddr += ll_block->off;
-		ll_region->paddr = pdev->resource[ll_block->bar].start;
+		ll_region->paddr = pci_bus_address(pdev, ll_block->bar);
 		ll_region->paddr += ll_block->off;
 		ll_region->sz = ll_block->sz;
 
@@ -240,7 +240,7 @@ static int dw_edma_pcie_probe(struct pci_dev *pdev,
 			return -ENOMEM;
 
 		dt_region->vaddr += dt_block->off;
-		dt_region->paddr = pdev->resource[dt_block->bar].start;
+		dt_region->paddr = pci_bus_address(pdev, dt_block->bar);
 		dt_region->paddr += dt_block->off;
 		dt_region->sz = dt_block->sz;
 	}
@@ -256,7 +256,7 @@ static int dw_edma_pcie_probe(struct pci_dev *pdev,
 			return -ENOMEM;
 
 		ll_region->vaddr += ll_block->off;
-		ll_region->paddr = pdev->resource[ll_block->bar].start;
+		ll_region->paddr = pci_bus_address(pdev, ll_block->bar);
 		ll_region->paddr += ll_block->off;
 		ll_region->sz = ll_block->sz;
 
@@ -265,7 +265,7 @@ static int dw_edma_pcie_probe(struct pci_dev *pdev,
 			return -ENOMEM;
 
 		dt_region->vaddr += dt_block->off;
-		dt_region->paddr = pdev->resource[dt_block->bar].start;
+		dt_region->paddr = pci_bus_address(pdev, dt_block->bar);
 		dt_region->paddr += dt_block->off;
 		dt_region->sz = dt_block->sz;
 	}
diff --git a/include/linux/dma/edma.h b/include/linux/dma/edma.h
index 8897f8a79b52..5abac9640a4e 100644
--- a/include/linux/dma/edma.h
+++ b/include/linux/dma/edma.h
@@ -18,7 +18,7 @@
 struct dw_edma;
 
 struct dw_edma_region {
-	phys_addr_t	paddr;
+	u64		paddr;
 	void __iomem	*vaddr;
 	size_t		sz;
 };
-- 
2.35.1

Re: [PATCH 05/25] dmaengine: dw-edma: Convert ll/dt phys-address to PCIe bus/DMA address

[Manivannan Sadhasivam]

On Thu, Mar 24, 2022 at 04:48:16AM +0300, Serge Semin wrote:
> In accordance with the dw_edma_region.paddr field semantics it is supposed
> to be initialized with a memory base address visible by the DW eDMA
> controller. If the DMA engine is embedded into the DW PCIe Host/EP
> controller, then the address should belong to the Local CPU/Application
> memory. If eDMA is remotely accessible across the PCIe bus via the PCIe
> memory IOs, then the address needs to be a part of the PCIe bus memory
> space. The later case hasn't been well covered in the corresponding
> glue-driver. Since in general the PCIe memory space doesn't have to match
> the CPU memory space and the pci_dev.resource[] arrays contain the
> resources defined in the CPU memory space, a proper conversion needs to be
> performed, otherwise either the driver won't properly work or much worse
> the memory corruption will happen. The conversion can be done by means of
> the pci_bus_address() method. Let's use it to retrieve the LL, DT and CSRs
> PCIe memory ranges.
> 
> Note in addition to that we need to extend the dw_edma_region.paddr field
> size. The field normally contains a memory range base address to be set in
> the DW eDMA Linked-List pointer register or as a base address of the
> Linked-List data buffer. In accordance with [1] the LL range is supposed
> to be created in the Local CPU/Application memory, but depending on the DW
> eDMA utilization the memory can be created as a part of the PCIe bus
> address space (as in the case of the DW PCIe EP prototype kit). Thus in
> the former case the dw_edma_region.paddr field should have the dma_addr_t
> type, while in the later one - pci_bus_addr_t. Seeing the corresponding
> CSRs are always 64-bits wide let's convert the dw_edma_region.paddr field
> type to be u64 and let the client code logic to make sure it has a valid
> address visible by the DW eDMA controller. For instance the DW eDMA PCIe
> glue-driver initializes the field with the addresses from the PCIe bus
> memory space.
> 
> [1] DesignWare Cores PCI Express Controller Databook - DWC PCIe Root Port,
>     v.5.40a, March 2019, p.1103
> 
> Fixes: 41aaff2a2ac0 ("dmaengine: Add Synopsys eDMA IP PCIe glue-logic")
> Signed-off-by: Serge Semin <Sergey.Semin@baikalelectronics.ru>

Reviewed-by: Manivannan Sadhasivam <manivannan.sadhasivam@linaro.org>

Thanks,
Mani

> ---
>  drivers/dma/dw-edma/dw-edma-pcie.c | 8 ++++----
>  include/linux/dma/edma.h           | 2 +-
>  2 files changed, 5 insertions(+), 5 deletions(-)
> 
> diff --git a/drivers/dma/dw-edma/dw-edma-pcie.c b/drivers/dma/dw-edma/dw-edma-pcie.c
> index d6b5e2463884..04c95cba1244 100644
> --- a/drivers/dma/dw-edma/dw-edma-pcie.c
> +++ b/drivers/dma/dw-edma/dw-edma-pcie.c
> @@ -231,7 +231,7 @@ static int dw_edma_pcie_probe(struct pci_dev *pdev,
>  			return -ENOMEM;
>  
>  		ll_region->vaddr += ll_block->off;
> -		ll_region->paddr = pdev->resource[ll_block->bar].start;
> +		ll_region->paddr = pci_bus_address(pdev, ll_block->bar);
>  		ll_region->paddr += ll_block->off;
>  		ll_region->sz = ll_block->sz;
>  
> @@ -240,7 +240,7 @@ static int dw_edma_pcie_probe(struct pci_dev *pdev,
>  			return -ENOMEM;
>  
>  		dt_region->vaddr += dt_block->off;
> -		dt_region->paddr = pdev->resource[dt_block->bar].start;
> +		dt_region->paddr = pci_bus_address(pdev, dt_block->bar);
>  		dt_region->paddr += dt_block->off;
>  		dt_region->sz = dt_block->sz;
>  	}
> @@ -256,7 +256,7 @@ static int dw_edma_pcie_probe(struct pci_dev *pdev,
>  			return -ENOMEM;
>  
>  		ll_region->vaddr += ll_block->off;
> -		ll_region->paddr = pdev->resource[ll_block->bar].start;
> +		ll_region->paddr = pci_bus_address(pdev, ll_block->bar);
>  		ll_region->paddr += ll_block->off;
>  		ll_region->sz = ll_block->sz;
>  
> @@ -265,7 +265,7 @@ static int dw_edma_pcie_probe(struct pci_dev *pdev,
>  			return -ENOMEM;
>  
>  		dt_region->vaddr += dt_block->off;
> -		dt_region->paddr = pdev->resource[dt_block->bar].start;
> +		dt_region->paddr = pci_bus_address(pdev, dt_block->bar);
>  		dt_region->paddr += dt_block->off;
>  		dt_region->sz = dt_block->sz;
>  	}
> diff --git a/include/linux/dma/edma.h b/include/linux/dma/edma.h
> index 8897f8a79b52..5abac9640a4e 100644
> --- a/include/linux/dma/edma.h
> +++ b/include/linux/dma/edma.h
> @@ -18,7 +18,7 @@
>  struct dw_edma;
>  
>  struct dw_edma_region {
> -	phys_addr_t	paddr;
> +	u64		paddr;
>  	void __iomem	*vaddr;
>  	size_t		sz;
>  };
> -- 
> 2.35.1
>