From: Logan Gunthorpe <logang@deltatee.com>
To: linux-kernel@vger.kernel.org, linux-nvme@lists.infradead.org,
linux-block@vger.kernel.org, linux-pci@vger.kernel.org,
linux-mm@kvack.org, iommu@lists.linux-foundation.org
Cc: "Stephen Bates" <sbates@raithlin.com>,
"Christoph Hellwig" <hch@lst.de>,
"Dan Williams" <dan.j.williams@intel.com>,
"Jason Gunthorpe" <jgg@ziepe.ca>,
"Christian König" <christian.koenig@amd.com>,
"Ira Weiny" <iweiny@intel.com>,
"John Hubbard" <jhubbard@nvidia.com>,
"Don Dutile" <ddutile@redhat.com>,
"Matthew Wilcox" <willy@infradead.org>,
"Daniel Vetter" <daniel.vetter@ffwll.ch>,
"Logan Gunthorpe" <logang@deltatee.com>
Subject: [RFC PATCH 07/15] iommu/dma: Support PCI P2PDMA pages in dma-iommu map_sg
Date: Fri, 6 Nov 2020 10:00:28 -0700 [thread overview]
Message-ID: <20201106170036.18713-8-logang@deltatee.com> (raw)
In-Reply-To: <20201106170036.18713-1-logang@deltatee.com>
When a PCI P2PDMA page is seen, set the IOVA length of the segment
to zero so that it is not mapped into the IOVA. Then, in finalise_sg(),
apply the appropriate bus address to the segment. The IOVA is not
created if the scatterlist only consists of P2PDMA pages.
Similar to dma-direct, DMA_ATTR_P2PDMA is used to indicate caller
support seeing the high bit of the dma_length is used as a flag
to indicate P2PDMA segments.
On unmap, P2PDMA segments are skipped over when determining the
start and end IOVA addresses.
With this change, the flags variable in the dma_map_ops is
set to DMA_F_PCI_P2PDMA_SUPPORTED to indicate support for
P2PDMA pages.
Signed-off-by: Logan Gunthorpe <logang@deltatee.com>
---
drivers/iommu/dma-iommu.c | 63 ++++++++++++++++++++++++++++++++-------
1 file changed, 53 insertions(+), 10 deletions(-)
diff --git a/drivers/iommu/dma-iommu.c b/drivers/iommu/dma-iommu.c
index 5591d6593583..1c8402474376 100644
--- a/drivers/iommu/dma-iommu.c
+++ b/drivers/iommu/dma-iommu.c
@@ -20,6 +20,7 @@
#include <linux/mm.h>
#include <linux/mutex.h>
#include <linux/pci.h>
+#include <linux/pci-p2pdma.h>
#include <linux/swiotlb.h>
#include <linux/scatterlist.h>
#include <linux/vmalloc.h>
@@ -872,13 +873,16 @@ static void iommu_dma_unmap_page(struct device *dev, dma_addr_t dma_handle,
* segment's start address to avoid concatenating across one.
*/
static int __finalise_sg(struct device *dev, struct scatterlist *sg, int nents,
- dma_addr_t dma_addr)
+ dma_addr_t dma_addr, unsigned long attrs)
{
struct scatterlist *s, *cur = sg;
unsigned long seg_mask = dma_get_seg_boundary(dev);
unsigned int cur_len = 0, max_len = dma_get_max_seg_size(dev);
int i, count = 0;
+ if (attrs & DMA_ATTR_P2PDMA && max_len >= SG_P2PDMA_FLAG)
+ max_len = SG_P2PDMA_FLAG - 1;
+
/*
* The Intel graphic driver is used to assume that the returned
* sg list is not combound. This blocks the efforts of converting
@@ -917,6 +921,19 @@ static int __finalise_sg(struct device *dev, struct scatterlist *sg, int nents,
sg_dma_address(s) = DMA_MAPPING_ERROR;
sg_dma_len(s) = 0;
+ if (is_pci_p2pdma_page(sg_page(s)) && !s_iova_len) {
+ if (i > 0)
+ cur = sg_next(cur);
+
+ sg_dma_address(cur) = sg_phys(s) + s->offset -
+ pci_p2pdma_bus_offset(sg_page(s));
+ sg_dma_len(cur) = s->length | SG_P2PDMA_FLAG;
+
+ count++;
+ cur_len = 0;
+ continue;
+ }
+
/*
* Now fill in the real DMA data. If...
* - there is a valid output segment to append to
@@ -1013,11 +1030,12 @@ static int iommu_dma_map_sg(struct device *dev, struct scatterlist *sg,
struct iommu_dma_cookie *cookie = domain->iova_cookie;
struct iova_domain *iovad = &cookie->iovad;
struct scatterlist *s, *prev = NULL;
+ struct dev_pagemap *pgmap = NULL;
int prot = dma_info_to_prot(dir, dev_is_dma_coherent(dev), attrs);
dma_addr_t iova;
size_t iova_len = 0;
unsigned long mask = dma_get_seg_boundary(dev);
- int i;
+ int i, map = -1;
if (unlikely(iommu_dma_deferred_attach(dev, domain)))
return 0;
@@ -1045,6 +1063,21 @@ static int iommu_dma_map_sg(struct device *dev, struct scatterlist *sg,
s_length = iova_align(iovad, s_length + s_iova_off);
s->length = s_length;
+ if (is_pci_p2pdma_page(sg_page(s))) {
+ if (sg_page(s)->pgmap != pgmap) {
+ pgmap = sg_page(s)->pgmap;
+ map = pci_p2pdma_should_map_bus(dev, pgmap);
+ }
+
+ if (map < 0 || !(attrs & DMA_ATTR_P2PDMA))
+ goto out_restore_sg;
+
+ if (map) {
+ s->length = 0;
+ continue;
+ }
+ }
+
/*
* Due to the alignment of our single IOVA allocation, we can
* depend on these assumptions about the segment boundary mask:
@@ -1067,6 +1100,9 @@ static int iommu_dma_map_sg(struct device *dev, struct scatterlist *sg,
prev = s;
}
+ if (!iova_len)
+ return __finalise_sg(dev, sg, nents, 0, attrs);
+
iova = iommu_dma_alloc_iova(domain, iova_len, dma_get_mask(dev), dev);
if (!iova)
goto out_restore_sg;
@@ -1078,7 +1114,7 @@ static int iommu_dma_map_sg(struct device *dev, struct scatterlist *sg,
if (iommu_map_sg_atomic(domain, iova, sg, nents, prot) < iova_len)
goto out_free_iova;
- return __finalise_sg(dev, sg, nents, iova);
+ return __finalise_sg(dev, sg, nents, iova, attrs);
out_free_iova:
iommu_dma_free_iova(cookie, iova, iova_len, NULL);
@@ -1090,7 +1126,7 @@ static int iommu_dma_map_sg(struct device *dev, struct scatterlist *sg,
static void iommu_dma_unmap_sg(struct device *dev, struct scatterlist *sg,
int nents, enum dma_data_direction dir, unsigned long attrs)
{
- dma_addr_t start, end;
+ dma_addr_t end, start = DMA_MAPPING_ERROR;
struct scatterlist *tmp;
int i;
@@ -1106,14 +1142,20 @@ static void iommu_dma_unmap_sg(struct device *dev, struct scatterlist *sg,
* The scatterlist segments are mapped into a single
* contiguous IOVA allocation, so this is incredibly easy.
*/
- start = sg_dma_address(sg);
- for_each_sg(sg_next(sg), tmp, nents - 1, i) {
- if (sg_dma_len(tmp) == 0)
+ for_each_sg(sg, tmp, nents, i) {
+ if ((attrs & DMA_ATTR_P2PDMA) && sg_dma_is_p2pdma(tmp))
+ continue;
+ if (sg_dma_p2pdma_len(tmp) == 0)
break;
- sg = tmp;
+
+ if (start == DMA_MAPPING_ERROR)
+ start = sg_dma_address(tmp);
+
+ end = sg_dma_address(tmp) + sg_dma_len(tmp);
}
- end = sg_dma_address(sg) + sg_dma_len(sg);
- __iommu_dma_unmap(dev, start, end - start);
+
+ if (start != DMA_MAPPING_ERROR)
+ __iommu_dma_unmap(dev, start, end - start);
}
static dma_addr_t iommu_dma_map_resource(struct device *dev, phys_addr_t phys,
@@ -1334,6 +1376,7 @@ static unsigned long iommu_dma_get_merge_boundary(struct device *dev)
}
static const struct dma_map_ops iommu_dma_ops = {
+ .flags = DMA_F_PCI_P2PDMA_SUPPORTED,
.alloc = iommu_dma_alloc,
.free = iommu_dma_free,
.alloc_pages = dma_common_alloc_pages,
--
2.20.1
next prev parent reply other threads:[~2020-11-06 17:01 UTC|newest]
Thread overview: 42+ messages / expand[flat|nested] mbox.gz Atom feed top
2020-11-06 17:00 [RFC PATCH 00/15] Userspace P2PDMA with O_DIRECT NVMe devices Logan Gunthorpe
2020-11-06 17:00 ` [RFC PATCH 01/15] PCI/P2PDMA: Don't sleep in upstream_bridge_distance_warn() Logan Gunthorpe
2020-11-09 9:12 ` Christoph Hellwig
2020-11-06 17:00 ` [RFC PATCH 02/15] PCI/P2PDMA: Attempt to set map_type if it has not been set Logan Gunthorpe
2020-11-06 17:00 ` [RFC PATCH 03/15] PCI/P2PDMA: Introduce pci_p2pdma_should_map_bus() and pci_p2pdma_bus_offset() Logan Gunthorpe
2020-11-10 23:25 ` Bjorn Helgaas
2020-11-10 23:42 ` Logan Gunthorpe
2020-11-06 17:00 ` [RFC PATCH 04/15] lib/scatterlist: Add flag for indicating P2PDMA segments in an SGL Logan Gunthorpe
2020-11-09 9:12 ` Christoph Hellwig
2020-11-09 14:02 ` Robin Murphy
2020-11-09 16:47 ` Logan Gunthorpe
2020-12-10 1:22 ` Dan Williams
2020-12-10 2:06 ` Logan Gunthorpe
2020-12-10 4:04 ` Dan Williams
2020-12-10 16:44 ` Logan Gunthorpe
2020-11-06 17:00 ` [RFC PATCH 05/15] dma-direct: Support PCI P2PDMA pages in dma-direct map_sg Logan Gunthorpe
2020-11-06 17:00 ` [RFC PATCH 06/15] dma-mapping: Add flags to dma_map_ops to indicate PCI P2PDMA support Logan Gunthorpe
2020-11-06 17:00 ` Logan Gunthorpe [this message]
2020-11-06 17:00 ` [RFC PATCH 08/15] nvme-pci: Check DMA ops when indicating support for PCI P2PDMA Logan Gunthorpe
2020-11-06 17:00 ` [RFC PATCH 09/15] nvme-pci: Convert to using dma_map_sg for p2pdma pages Logan Gunthorpe
2020-11-06 17:00 ` [RFC PATCH 10/15] mm: Introduce FOLL_PCI_P2PDMA to gate getting PCI P2PDMA pages Logan Gunthorpe
2020-11-06 17:00 ` [RFC PATCH 11/15] iov_iter: Introduce iov_iter_get_pages_[alloc_]flags() Logan Gunthorpe
2020-11-06 17:00 ` [RFC PATCH 12/15] block: Set FOLL_PCI_P2PDMA in __bio_iov_iter_get_pages() Logan Gunthorpe
2020-11-06 17:00 ` [RFC PATCH 13/15] block: Set FOLL_PCI_P2PDMA in bio_map_user_iov() Logan Gunthorpe
2020-11-06 17:00 ` [RFC PATCH 14/15] PCI/P2PDMA: Introduce pci_mmap_p2pmem() Logan Gunthorpe
2020-11-06 17:22 ` Jason Gunthorpe
2020-11-06 17:28 ` Logan Gunthorpe
2020-11-06 17:42 ` Jason Gunthorpe
2020-11-06 17:53 ` Logan Gunthorpe
2020-11-06 18:09 ` Jason Gunthorpe
2020-11-06 18:20 ` Logan Gunthorpe
2020-11-06 19:30 ` Jason Gunthorpe
2020-11-06 19:44 ` Logan Gunthorpe
2020-11-06 19:53 ` Jason Gunthorpe
2020-11-06 20:03 ` Logan Gunthorpe
2020-11-07 0:14 ` Jason Gunthorpe
2020-11-07 2:50 ` Logan Gunthorpe
2020-11-06 17:00 ` [RFC PATCH 15/15] nvme-pci: Allow mmaping the CMB in userspace Logan Gunthorpe
2020-11-06 17:39 ` Jason Gunthorpe
2020-11-06 17:43 ` Logan Gunthorpe
2020-11-09 15:03 ` Keith Busch
2020-11-09 16:50 ` Logan Gunthorpe
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20201106170036.18713-8-logang@deltatee.com \
--to=logang@deltatee.com \
--cc=christian.koenig@amd.com \
--cc=dan.j.williams@intel.com \
--cc=daniel.vetter@ffwll.ch \
--cc=ddutile@redhat.com \
--cc=hch@lst.de \
--cc=iommu@lists.linux-foundation.org \
--cc=iweiny@intel.com \
--cc=jgg@ziepe.ca \
--cc=jhubbard@nvidia.com \
--cc=linux-block@vger.kernel.org \
--cc=linux-kernel@vger.kernel.org \
--cc=linux-mm@kvack.org \
--cc=linux-nvme@lists.infradead.org \
--cc=linux-pci@vger.kernel.org \
--cc=sbates@raithlin.com \
--cc=willy@infradead.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).