* [PATCH V2 1/5] iommu: Handle freelists when using deferred flushing in iommu drivers
2020-09-03 20:18 [PATCH V2 0/5] Convert the intel iommu driver to the dma-iommu api Tom Murphy
@ 2020-09-03 20:18 ` Tom Murphy
2020-09-03 20:18 ` [PATCH V2 2/5] iommu: Add iommu_dma_free_cpu_cached_iovas function Tom Murphy
` (4 subsequent siblings)
5 siblings, 0 replies; 22+ messages in thread
From: Tom Murphy @ 2020-09-03 20:18 UTC (permalink / raw)
To: iommu; +Cc: David Woodhouse, linux-kernel, Tom Murphy
Allow the iommu_unmap_fast to return newly freed page table pages and
pass the freelist to queue_iova in the dma-iommu ops path.
This is useful for iommu drivers (in this case the intel iommu driver)
which need to wait for the ioTLB to be flushed before newly
free/unmapped page table pages can be freed. This way we can still batch
ioTLB free operations and handle the freelists.
Signed-off-by: Tom Murphy <murphyt7@tcd.ie>
---
drivers/iommu/dma-iommu.c | 30 ++++++++++++++------
drivers/iommu/intel/iommu.c | 55 ++++++++++++++++++++++++-------------
include/linux/iommu.h | 1 +
3 files changed, 59 insertions(+), 27 deletions(-)
diff --git a/drivers/iommu/dma-iommu.c b/drivers/iommu/dma-iommu.c
index 4959f5df21bd..f69dc9467d71 100644
--- a/drivers/iommu/dma-iommu.c
+++ b/drivers/iommu/dma-iommu.c
@@ -50,6 +50,18 @@ struct iommu_dma_cookie {
struct iommu_domain *fq_domain;
};
+static void iommu_dma_entry_dtor(unsigned long data)
+{
+ struct page *freelist = (struct page *)data;
+
+ while (freelist != NULL) {
+ unsigned long p = (unsigned long)page_address(freelist);
+
+ freelist = freelist->freelist;
+ free_page(p);
+ }
+}
+
static inline size_t cookie_msi_granule(struct iommu_dma_cookie *cookie)
{
if (cookie->type == IOMMU_DMA_IOVA_COOKIE)
@@ -344,7 +356,8 @@ static int iommu_dma_init_domain(struct iommu_domain *domain, dma_addr_t base,
if (!cookie->fq_domain && !iommu_domain_get_attr(domain,
DOMAIN_ATTR_DMA_USE_FLUSH_QUEUE, &attr) && attr) {
cookie->fq_domain = domain;
- init_iova_flush_queue(iovad, iommu_dma_flush_iotlb_all, NULL);
+ init_iova_flush_queue(iovad, iommu_dma_flush_iotlb_all,
+ iommu_dma_entry_dtor);
}
if (!dev)
@@ -438,7 +451,7 @@ static dma_addr_t iommu_dma_alloc_iova(struct iommu_domain *domain,
}
static void iommu_dma_free_iova(struct iommu_dma_cookie *cookie,
- dma_addr_t iova, size_t size)
+ dma_addr_t iova, size_t size, struct page *freelist)
{
struct iova_domain *iovad = &cookie->iovad;
@@ -447,7 +460,8 @@ static void iommu_dma_free_iova(struct iommu_dma_cookie *cookie,
cookie->msi_iova -= size;
else if (cookie->fq_domain) /* non-strict mode */
queue_iova(iovad, iova_pfn(iovad, iova),
- size >> iova_shift(iovad), 0);
+ size >> iova_shift(iovad),
+ (unsigned long) freelist);
else
free_iova_fast(iovad, iova_pfn(iovad, iova),
size >> iova_shift(iovad));
@@ -472,7 +486,7 @@ static void __iommu_dma_unmap(struct device *dev, dma_addr_t dma_addr,
if (!cookie->fq_domain)
iommu_tlb_sync(domain, &iotlb_gather);
- iommu_dma_free_iova(cookie, dma_addr, size);
+ iommu_dma_free_iova(cookie, dma_addr, size, iotlb_gather.freelist);
}
static dma_addr_t __iommu_dma_map(struct device *dev, phys_addr_t phys,
@@ -494,7 +508,7 @@ static dma_addr_t __iommu_dma_map(struct device *dev, phys_addr_t phys,
return DMA_MAPPING_ERROR;
if (iommu_map_atomic(domain, iova, phys - iova_off, size, prot)) {
- iommu_dma_free_iova(cookie, iova, size);
+ iommu_dma_free_iova(cookie, iova, size, NULL);
return DMA_MAPPING_ERROR;
}
return iova + iova_off;
@@ -649,7 +663,7 @@ static void *iommu_dma_alloc_remap(struct device *dev, size_t size,
out_free_sg:
sg_free_table(&sgt);
out_free_iova:
- iommu_dma_free_iova(cookie, iova, size);
+ iommu_dma_free_iova(cookie, iova, size, NULL);
out_free_pages:
__iommu_dma_free_pages(pages, count);
return NULL;
@@ -900,7 +914,7 @@ static int iommu_dma_map_sg(struct device *dev, struct scatterlist *sg,
return __finalise_sg(dev, sg, nents, iova);
out_free_iova:
- iommu_dma_free_iova(cookie, iova, iova_len);
+ iommu_dma_free_iova(cookie, iova, iova_len, NULL);
out_restore_sg:
__invalidate_sg(sg, nents);
return 0;
@@ -1194,7 +1208,7 @@ static struct iommu_dma_msi_page *iommu_dma_get_msi_page(struct device *dev,
return msi_page;
out_free_iova:
- iommu_dma_free_iova(cookie, iova, size);
+ iommu_dma_free_iova(cookie, iova, size, NULL);
out_free_page:
kfree(msi_page);
return NULL;
diff --git a/drivers/iommu/intel/iommu.c b/drivers/iommu/intel/iommu.c
index 237a470e1e9c..03699860880b 100644
--- a/drivers/iommu/intel/iommu.c
+++ b/drivers/iommu/intel/iommu.c
@@ -1160,17 +1160,17 @@ static struct page *dma_pte_clear_level(struct dmar_domain *domain, int level,
pages can only be freed after the IOTLB flush has been done. */
static struct page *domain_unmap(struct dmar_domain *domain,
unsigned long start_pfn,
- unsigned long last_pfn)
+ unsigned long last_pfn,
+ struct page *freelist)
{
- struct page *freelist;
-
BUG_ON(!domain_pfn_supported(domain, start_pfn));
BUG_ON(!domain_pfn_supported(domain, last_pfn));
BUG_ON(start_pfn > last_pfn);
/* we don't need lock here; nobody else touches the iova range */
freelist = dma_pte_clear_level(domain, agaw_to_level(domain->agaw),
- domain->pgd, 0, start_pfn, last_pfn, NULL);
+ domain->pgd, 0, start_pfn, last_pfn,
+ freelist);
/* free pgd */
if (start_pfn == 0 && last_pfn == DOMAIN_MAX_PFN(domain->gaw)) {
@@ -1924,7 +1924,8 @@ static void domain_exit(struct dmar_domain *domain)
if (domain->pgd) {
struct page *freelist;
- freelist = domain_unmap(domain, 0, DOMAIN_MAX_PFN(domain->gaw));
+ freelist = domain_unmap(domain, 0, DOMAIN_MAX_PFN(domain->gaw),
+ NULL);
dma_free_pagelist(freelist);
}
@@ -3480,7 +3481,7 @@ static void intel_unmap(struct device *dev, dma_addr_t dev_addr, size_t size)
if (dev_is_pci(dev))
pdev = to_pci_dev(dev);
- freelist = domain_unmap(domain, start_pfn, last_pfn);
+ freelist = domain_unmap(domain, start_pfn, last_pfn, NULL);
if (intel_iommu_strict || (pdev && pdev->untrusted) ||
!has_iova_flush_queue(&domain->iovad)) {
iommu_flush_iotlb_psi(iommu, domain, start_pfn,
@@ -4575,7 +4576,8 @@ static int intel_iommu_memory_notifier(struct notifier_block *nb,
struct page *freelist;
freelist = domain_unmap(si_domain,
- start_vpfn, last_vpfn);
+ start_vpfn, last_vpfn,
+ NULL);
rcu_read_lock();
for_each_active_iommu(iommu, drhd)
@@ -5543,10 +5545,8 @@ static size_t intel_iommu_unmap(struct iommu_domain *domain,
struct iommu_iotlb_gather *gather)
{
struct dmar_domain *dmar_domain = to_dmar_domain(domain);
- struct page *freelist = NULL;
unsigned long start_pfn, last_pfn;
- unsigned int npages;
- int iommu_id, level = 0;
+ int level = 0;
/* Cope with horrid API which requires us to unmap more than the
size argument if it happens to be a large-page mapping. */
@@ -5558,22 +5558,38 @@ static size_t intel_iommu_unmap(struct iommu_domain *domain,
start_pfn = iova >> VTD_PAGE_SHIFT;
last_pfn = (iova + size - 1) >> VTD_PAGE_SHIFT;
- freelist = domain_unmap(dmar_domain, start_pfn, last_pfn);
-
- npages = last_pfn - start_pfn + 1;
-
- for_each_domain_iommu(iommu_id, dmar_domain)
- iommu_flush_iotlb_psi(g_iommus[iommu_id], dmar_domain,
- start_pfn, npages, !freelist, 0);
-
- dma_free_pagelist(freelist);
+ gather->freelist = domain_unmap(dmar_domain, start_pfn, last_pfn,
+ gather->freelist);
if (dmar_domain->max_addr == iova + size)
dmar_domain->max_addr = iova;
+ iommu_iotlb_gather_add_page(domain, gather, iova, size);
+
return size;
}
+static void intel_iommu_tlb_sync(struct iommu_domain *domain,
+ struct iommu_iotlb_gather *gather)
+{
+ struct dmar_domain *dmar_domain = to_dmar_domain(domain);
+ unsigned long iova_pfn = IOVA_PFN(gather->start);
+ size_t size = gather->end - gather->start;
+ unsigned long start_pfn, last_pfn;
+ unsigned long nrpages;
+ int iommu_id;
+
+ nrpages = aligned_nrpages(gather->start, size);
+ start_pfn = mm_to_dma_pfn(iova_pfn);
+ last_pfn = start_pfn + nrpages - 1;
+
+ for_each_domain_iommu(iommu_id, dmar_domain)
+ iommu_flush_iotlb_psi(g_iommus[iommu_id], dmar_domain,
+ start_pfn, nrpages, !gather->freelist, 0);
+
+ dma_free_pagelist(gather->freelist);
+}
+
static phys_addr_t intel_iommu_iova_to_phys(struct iommu_domain *domain,
dma_addr_t iova)
{
@@ -6058,6 +6074,7 @@ const struct iommu_ops intel_iommu_ops = {
.aux_get_pasid = intel_iommu_aux_get_pasid,
.map = intel_iommu_map,
.unmap = intel_iommu_unmap,
+ .iotlb_sync = intel_iommu_tlb_sync,
.iova_to_phys = intel_iommu_iova_to_phys,
.probe_device = intel_iommu_probe_device,
.probe_finalize = intel_iommu_probe_finalize,
diff --git a/include/linux/iommu.h b/include/linux/iommu.h
index 5f0b7859d2eb..e3eafb3cf4ba 100644
--- a/include/linux/iommu.h
+++ b/include/linux/iommu.h
@@ -186,6 +186,7 @@ struct iommu_iotlb_gather {
unsigned long start;
unsigned long end;
size_t pgsize;
+ struct page *freelist;
};
/**
--
2.20.1
_______________________________________________
iommu mailing list
iommu@lists.linux-foundation.org
https://lists.linuxfoundation.org/mailman/listinfo/iommu
^ permalink raw reply related [flat|nested] 22+ messages in thread* [PATCH V2 3/5] iommu: allow the dma-iommu api to use bounce buffers
2020-09-03 20:18 [PATCH V2 0/5] Convert the intel iommu driver to the dma-iommu api Tom Murphy
2020-09-03 20:18 ` [PATCH V2 1/5] iommu: Handle freelists when using deferred flushing in iommu drivers Tom Murphy
2020-09-03 20:18 ` [PATCH V2 2/5] iommu: Add iommu_dma_free_cpu_cached_iovas function Tom Murphy
@ 2020-09-03 20:18 ` Tom Murphy
2020-09-09 1:34 ` Lu Baolu
2020-09-03 20:18 ` [PATCH V2 4/5] iommu/vt-d: Convert intel iommu driver to the iommu ops Tom Murphy
` (2 subsequent siblings)
5 siblings, 1 reply; 22+ messages in thread
From: Tom Murphy @ 2020-09-03 20:18 UTC (permalink / raw)
To: iommu; +Cc: David Woodhouse, linux-kernel, Tom Murphy
Allow the dma-iommu api to use bounce buffers for untrusted devices.
This is a copy of the intel bounce buffer code.
Signed-off-by: Tom Murphy <murphyt7@tcd.ie>
---
drivers/iommu/dma-iommu.c | 94 ++++++++++++++++++++++++++++++-------
drivers/iommu/intel/iommu.c | 6 +++
drivers/iommu/iommu.c | 10 ++++
include/linux/iommu.h | 7 +++
4 files changed, 101 insertions(+), 16 deletions(-)
diff --git a/drivers/iommu/dma-iommu.c b/drivers/iommu/dma-iommu.c
index 33f3f4f5edc5..185cd504ca5a 100644
--- a/drivers/iommu/dma-iommu.c
+++ b/drivers/iommu/dma-iommu.c
@@ -21,9 +21,11 @@
#include <linux/mm.h>
#include <linux/mutex.h>
#include <linux/pci.h>
+#include <linux/swiotlb.h>
#include <linux/scatterlist.h>
#include <linux/vmalloc.h>
#include <linux/crash_dump.h>
+#include <linux/dma-direct.h>
struct iommu_dma_msi_page {
struct list_head list;
@@ -498,26 +500,87 @@ static void __iommu_dma_unmap(struct device *dev, dma_addr_t dma_addr,
iommu_dma_free_iova(cookie, dma_addr, size, iotlb_gather.freelist);
}
+static void __iommu_dma_unmap_swiotlb(struct device *dev, dma_addr_t dma_addr,
+ size_t size, enum dma_data_direction dir,
+ unsigned long attrs)
+{
+ struct iommu_domain *domain = iommu_get_dma_domain(dev);
+ struct iommu_dma_cookie *cookie = domain->iova_cookie;
+ struct iova_domain *iovad = &cookie->iovad;
+ size_t iova_off = iova_offset(iovad, dma_addr);
+ size_t aligned_size = iova_align(iovad, size + iova_off);
+ phys_addr_t phys;
+
+ phys = iommu_iova_to_phys(domain, dma_addr);
+ if (WARN_ON(!phys))
+ return;
+
+ __iommu_dma_unmap(dev, dma_addr, size);
+
+#ifdef CONFIG_SWIOTLB
+ if (unlikely(is_swiotlb_buffer(phys)))
+ swiotlb_tbl_unmap_single(dev, phys, size,
+ aligned_size, dir, attrs);
+#endif
+}
+
static dma_addr_t __iommu_dma_map(struct device *dev, phys_addr_t phys,
- size_t size, int prot, u64 dma_mask)
+ size_t org_size, dma_addr_t dma_mask, bool coherent,
+ enum dma_data_direction dir, unsigned long attrs)
{
+ int prot = dma_info_to_prot(dir, coherent, attrs);
struct iommu_domain *domain = iommu_get_dma_domain(dev);
struct iommu_dma_cookie *cookie = domain->iova_cookie;
struct iova_domain *iovad = &cookie->iovad;
size_t iova_off = iova_offset(iovad, phys);
+ size_t aligned_size = iova_align(iovad, org_size + iova_off);
+ void *padding_start;
+ size_t padding_size;
dma_addr_t iova;
if (unlikely(iommu_dma_deferred_attach(dev, domain)))
return DMA_MAPPING_ERROR;
- size = iova_align(iovad, size + iova_off);
+#ifdef CONFIG_SWIOTLB
+ /*
+ * If both the physical buffer start address and size are
+ * page aligned, we don't need to use a bounce page.
+ */
+ if (iommu_needs_bounce_buffer(dev)
+ && !iova_offset(iovad, phys | org_size)) {
+ phys = swiotlb_tbl_map_single(dev,
+ __phys_to_dma(dev, io_tlb_start),
+ phys, org_size, aligned_size, dir, attrs);
+
+ if (phys == DMA_MAPPING_ERROR)
+ return DMA_MAPPING_ERROR;
+
+ /* Cleanup the padding area. */
+ padding_start = phys_to_virt(phys);
+ padding_size = aligned_size;
+
+ if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC) &&
+ (dir == DMA_TO_DEVICE ||
+ dir == DMA_BIDIRECTIONAL)) {
+ padding_start += org_size;
+ padding_size -= org_size;
+ }
- iova = iommu_dma_alloc_iova(domain, size, dma_mask, dev);
+ memset(padding_start, 0, padding_size);
+ }
+#endif
+
+ iova = iommu_dma_alloc_iova(domain, aligned_size, dma_mask, dev);
if (!iova)
return DMA_MAPPING_ERROR;
- if (iommu_map_atomic(domain, iova, phys - iova_off, size, prot)) {
- iommu_dma_free_iova(cookie, iova, size, NULL);
+ if (iommu_map_atomic(domain, iova, phys - iova_off, aligned_size,
+ prot)) {
+
+ if (unlikely(is_swiotlb_buffer(phys)))
+ swiotlb_tbl_unmap_single(dev, phys, aligned_size,
+ aligned_size, dir, attrs);
+ iommu_dma_free_iova(cookie, iova, aligned_size, NULL);
return DMA_MAPPING_ERROR;
}
return iova + iova_off;
@@ -751,10 +814,10 @@ static dma_addr_t iommu_dma_map_page(struct device *dev, struct page *page,
{
phys_addr_t phys = page_to_phys(page) + offset;
bool coherent = dev_is_dma_coherent(dev);
- int prot = dma_info_to_prot(dir, coherent, attrs);
dma_addr_t dma_handle;
- dma_handle = __iommu_dma_map(dev, phys, size, prot, dma_get_mask(dev));
+ dma_handle = __iommu_dma_map(dev, phys, size, dma_get_mask(dev),
+ coherent, dir, attrs);
if (!coherent && !(attrs & DMA_ATTR_SKIP_CPU_SYNC) &&
dma_handle != DMA_MAPPING_ERROR)
arch_sync_dma_for_device(phys, size, dir);
@@ -766,7 +829,7 @@ static void iommu_dma_unmap_page(struct device *dev, dma_addr_t dma_handle,
{
if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC))
iommu_dma_sync_single_for_cpu(dev, dma_handle, size, dir);
- __iommu_dma_unmap(dev, dma_handle, size);
+ __iommu_dma_unmap_swiotlb(dev, dma_handle, size, dir, attrs);
}
/*
@@ -950,21 +1013,20 @@ static void iommu_dma_unmap_sg(struct device *dev, struct scatterlist *sg,
sg = tmp;
}
end = sg_dma_address(sg) + sg_dma_len(sg);
- __iommu_dma_unmap(dev, start, end - start);
+ __iommu_dma_unmap_swiotlb(dev, start, end - start, dir, attrs);
}
static dma_addr_t iommu_dma_map_resource(struct device *dev, phys_addr_t phys,
size_t size, enum dma_data_direction dir, unsigned long attrs)
{
- return __iommu_dma_map(dev, phys, size,
- dma_info_to_prot(dir, false, attrs) | IOMMU_MMIO,
- dma_get_mask(dev));
+ return __iommu_dma_map(dev, phys, size, dma_get_mask(dev), false, dir,
+ attrs);
}
static void iommu_dma_unmap_resource(struct device *dev, dma_addr_t handle,
size_t size, enum dma_data_direction dir, unsigned long attrs)
{
- __iommu_dma_unmap(dev, handle, size);
+ __iommu_dma_unmap_swiotlb(dev, handle, size, dir, attrs);
}
static void __iommu_dma_free(struct device *dev, size_t size, void *cpu_addr)
@@ -1046,7 +1108,6 @@ static void *iommu_dma_alloc(struct device *dev, size_t size,
dma_addr_t *handle, gfp_t gfp, unsigned long attrs)
{
bool coherent = dev_is_dma_coherent(dev);
- int ioprot = dma_info_to_prot(DMA_BIDIRECTIONAL, coherent, attrs);
struct page *page = NULL;
void *cpu_addr;
@@ -1065,8 +1126,9 @@ static void *iommu_dma_alloc(struct device *dev, size_t size,
if (!cpu_addr)
return NULL;
- *handle = __iommu_dma_map(dev, page_to_phys(page), size, ioprot,
- dev->coherent_dma_mask);
+ *handle = __iommu_dma_map(dev, page_to_phys(page), size,
+ dev->coherent_dma_mask, coherent, DMA_BIDIRECTIONAL,
+ attrs);
if (*handle == DMA_MAPPING_ERROR) {
__iommu_dma_free(dev, size, cpu_addr);
return NULL;
diff --git a/drivers/iommu/intel/iommu.c b/drivers/iommu/intel/iommu.c
index 03699860880b..ba47623f0f12 100644
--- a/drivers/iommu/intel/iommu.c
+++ b/drivers/iommu/intel/iommu.c
@@ -5713,6 +5713,11 @@ static void intel_iommu_probe_finalize(struct device *dev)
set_dma_ops(dev, NULL);
}
+static int intel_iommu_needs_bounce_buffer(struct device *d)
+{
+ return !intel_no_bounce && dev_is_pci(d) && to_pci_dev(d)->untrusted;
+}
+
static void intel_iommu_get_resv_regions(struct device *device,
struct list_head *head)
{
@@ -6079,6 +6084,7 @@ const struct iommu_ops intel_iommu_ops = {
.probe_device = intel_iommu_probe_device,
.probe_finalize = intel_iommu_probe_finalize,
.release_device = intel_iommu_release_device,
+ .needs_bounce_buffer = intel_iommu_needs_bounce_buffer,
.get_resv_regions = intel_iommu_get_resv_regions,
.put_resv_regions = generic_iommu_put_resv_regions,
.apply_resv_region = intel_iommu_apply_resv_region,
diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c
index b6858adc4f17..8da26c73122f 100644
--- a/drivers/iommu/iommu.c
+++ b/drivers/iommu/iommu.c
@@ -2497,6 +2497,16 @@ int iommu_domain_set_attr(struct iommu_domain *domain,
}
EXPORT_SYMBOL_GPL(iommu_domain_set_attr);
+int iommu_needs_bounce_buffer(struct device *dev)
+{
+ const struct iommu_ops *ops = dev->bus->iommu_ops;
+
+ if (ops && ops->needs_bounce_buffer)
+ return ops->needs_bounce_buffer(dev);
+
+ return 0;
+}
+
void iommu_get_resv_regions(struct device *dev, struct list_head *list)
{
const struct iommu_ops *ops = dev->bus->iommu_ops;
diff --git a/include/linux/iommu.h b/include/linux/iommu.h
index e3eafb3cf4ba..4c2d2619fd8c 100644
--- a/include/linux/iommu.h
+++ b/include/linux/iommu.h
@@ -263,6 +263,7 @@ struct iommu_ops {
enum iommu_attr attr, void *data);
int (*domain_set_attr)(struct iommu_domain *domain,
enum iommu_attr attr, void *data);
+ int (*needs_bounce_buffer)(struct device *dev);
/* Request/Free a list of reserved regions for a device */
void (*get_resv_regions)(struct device *dev, struct list_head *list);
@@ -474,6 +475,7 @@ static inline size_t iommu_map_sgtable(struct iommu_domain *domain,
return iommu_map_sg(domain, iova, sgt->sgl, sgt->orig_nents, prot);
}
+extern int iommu_needs_bounce_buffer(struct device *dev);
extern void iommu_get_resv_regions(struct device *dev, struct list_head *list);
extern void iommu_put_resv_regions(struct device *dev, struct list_head *list);
extern void generic_iommu_put_resv_regions(struct device *dev,
@@ -779,6 +781,11 @@ static inline void iommu_set_fault_handler(struct iommu_domain *domain,
{
}
+static inline int iommu_needs_bounce_buffer(struct device *dev)
+{
+ return 0;
+}
+
static inline void iommu_get_resv_regions(struct device *dev,
struct list_head *list)
{
--
2.20.1
_______________________________________________
iommu mailing list
iommu@lists.linux-foundation.org
https://lists.linuxfoundation.org/mailman/listinfo/iommu
^ permalink raw reply related [flat|nested] 22+ messages in thread* [PATCH V2 4/5] iommu/vt-d: Convert intel iommu driver to the iommu ops
2020-09-03 20:18 [PATCH V2 0/5] Convert the intel iommu driver to the dma-iommu api Tom Murphy
` (2 preceding siblings ...)
2020-09-03 20:18 ` [PATCH V2 3/5] iommu: allow the dma-iommu api to use bounce buffers Tom Murphy
@ 2020-09-03 20:18 ` Tom Murphy
2020-09-09 1:59 ` Lu Baolu
2020-09-03 20:18 ` [PATCH V2 5/5] DO NOT MERGE: iommu: disable list appending in dma-iommu Tom Murphy
2020-09-04 10:03 ` [PATCH V2 0/5] Convert the intel iommu driver to the dma-iommu api Joerg Roedel
5 siblings, 1 reply; 22+ messages in thread
From: Tom Murphy @ 2020-09-03 20:18 UTC (permalink / raw)
To: iommu; +Cc: David Woodhouse, linux-kernel, Tom Murphy
Convert the intel iommu driver to the dma-iommu api. Remove the iova
handling and reserve region code from the intel iommu driver.
Signed-off-by: Tom Murphy <murphyt7@tcd.ie>
---
drivers/iommu/Kconfig | 1 +
drivers/iommu/intel/iommu.c | 756 +++---------------------------------
2 files changed, 51 insertions(+), 706 deletions(-)
diff --git a/drivers/iommu/Kconfig b/drivers/iommu/Kconfig
index b622af72448f..f1404fc4cc5f 100644
--- a/drivers/iommu/Kconfig
+++ b/drivers/iommu/Kconfig
@@ -191,6 +191,7 @@ config INTEL_IOMMU
select DMAR_TABLE
select SWIOTLB
select IOASID
+ select IOMMU_DMA
help
DMA remapping (DMAR) devices support enables independent address
translations for Direct Memory Access (DMA) from devices.
diff --git a/drivers/iommu/intel/iommu.c b/drivers/iommu/intel/iommu.c
index ba47623f0f12..98cda61681d2 100644
--- a/drivers/iommu/intel/iommu.c
+++ b/drivers/iommu/intel/iommu.c
@@ -31,6 +31,7 @@
#include <linux/io.h>
#include <linux/iova.h>
#include <linux/iommu.h>
+#include <linux/dma-iommu.h>
#include <linux/intel-iommu.h>
#include <linux/syscore_ops.h>
#include <linux/tboot.h>
@@ -41,7 +42,6 @@
#include <linux/dma-direct.h>
#include <linux/crash_dump.h>
#include <linux/numa.h>
-#include <linux/swiotlb.h>
#include <asm/irq_remapping.h>
#include <asm/cacheflush.h>
#include <asm/iommu.h>
@@ -383,9 +383,6 @@ struct device_domain_info *get_domain_info(struct device *dev)
DEFINE_SPINLOCK(device_domain_lock);
static LIST_HEAD(device_domain_list);
-#define device_needs_bounce(d) (!intel_no_bounce && dev_is_pci(d) && \
- to_pci_dev(d)->untrusted)
-
/*
* Iterate over elements in device_domain_list and call the specified
* callback @fn against each element.
@@ -748,7 +745,7 @@ static int iommu_dummy(struct device *dev)
static bool attach_deferred(struct device *dev)
{
- return dev->archdata.iommu == DEFER_DEVICE_DOMAIN_INFO;
+ return dev_iommu_priv_get(dev) == DEFER_DEVICE_DOMAIN_INFO;
}
/**
@@ -1194,13 +1191,6 @@ static void dma_free_pagelist(struct page *freelist)
}
}
-static void iova_entry_free(unsigned long data)
-{
- struct page *freelist = (struct page *)data;
-
- dma_free_pagelist(freelist);
-}
-
/* iommu handling */
static int iommu_alloc_root_entry(struct intel_iommu *iommu)
{
@@ -1565,19 +1555,17 @@ static inline void __mapping_notify_one(struct intel_iommu *iommu,
iommu_flush_write_buffer(iommu);
}
-static void iommu_flush_iova(struct iova_domain *iovad)
+static void intel_flush_iotlb_all(struct iommu_domain *domain)
{
- struct dmar_domain *domain;
+ struct dmar_domain *dmar_domain = to_dmar_domain(domain);
int idx;
- domain = container_of(iovad, struct dmar_domain, iovad);
-
- for_each_domain_iommu(idx, domain) {
+ for_each_domain_iommu(idx, dmar_domain) {
struct intel_iommu *iommu = g_iommus[idx];
- u16 did = domain->iommu_did[iommu->seq_id];
+ u16 did = dmar_domain->iommu_did[iommu->seq_id];
- if (domain_use_first_level(domain))
- domain_flush_piotlb(iommu, domain, 0, -1, 0);
+ if (domain_use_first_level(dmar_domain))
+ domain_flush_piotlb(iommu, dmar_domain, 0, -1, 0);
else
iommu->flush.flush_iotlb(iommu, did, 0, 0,
DMA_TLB_DSI_FLUSH);
@@ -1855,48 +1843,6 @@ static int domain_detach_iommu(struct dmar_domain *domain,
return count;
}
-static struct iova_domain reserved_iova_list;
-static struct lock_class_key reserved_rbtree_key;
-
-static int dmar_init_reserved_ranges(void)
-{
- struct pci_dev *pdev = NULL;
- struct iova *iova;
- int i;
-
- init_iova_domain(&reserved_iova_list, VTD_PAGE_SIZE, IOVA_START_PFN);
-
- lockdep_set_class(&reserved_iova_list.iova_rbtree_lock,
- &reserved_rbtree_key);
-
- /* IOAPIC ranges shouldn't be accessed by DMA */
- iova = reserve_iova(&reserved_iova_list, IOVA_PFN(IOAPIC_RANGE_START),
- IOVA_PFN(IOAPIC_RANGE_END));
- if (!iova) {
- pr_err("Reserve IOAPIC range failed\n");
- return -ENODEV;
- }
-
- /* Reserve all PCI MMIO to avoid peer-to-peer access */
- for_each_pci_dev(pdev) {
- struct resource *r;
-
- for (i = 0; i < PCI_NUM_RESOURCES; i++) {
- r = &pdev->resource[i];
- if (!r->flags || !(r->flags & IORESOURCE_MEM))
- continue;
- iova = reserve_iova(&reserved_iova_list,
- IOVA_PFN(r->start),
- IOVA_PFN(r->end));
- if (!iova) {
- pci_err(pdev, "Reserve iova for %pR failed\n", r);
- return -ENODEV;
- }
- }
- }
- return 0;
-}
-
static inline int guestwidth_to_adjustwidth(int gaw)
{
int agaw;
@@ -1919,7 +1865,7 @@ static void domain_exit(struct dmar_domain *domain)
/* destroy iovas */
if (domain->domain.type == IOMMU_DOMAIN_DMA)
- put_iova_domain(&domain->iovad);
+ iommu_put_dma_cookie(&domain->domain);
if (domain->pgd) {
struct page *freelist;
@@ -2450,16 +2396,6 @@ struct dmar_domain *find_domain(struct device *dev)
return NULL;
}
-static void do_deferred_attach(struct device *dev)
-{
- struct iommu_domain *domain;
-
- dev->archdata.iommu = NULL;
- domain = iommu_get_domain_for_dev(dev);
- if (domain)
- intel_iommu_attach_device(domain, dev);
-}
-
static inline struct device_domain_info *
dmar_search_domain_by_dev_info(int segment, int bus, int devfn)
{
@@ -3332,591 +3268,6 @@ static int __init init_dmars(void)
return ret;
}
-/* This takes a number of _MM_ pages, not VTD pages */
-static unsigned long intel_alloc_iova(struct device *dev,
- struct dmar_domain *domain,
- unsigned long nrpages, uint64_t dma_mask)
-{
- unsigned long iova_pfn;
-
- /*
- * Restrict dma_mask to the width that the iommu can handle.
- * First-level translation restricts the input-address to a
- * canonical address (i.e., address bits 63:N have the same
- * value as address bit [N-1], where N is 48-bits with 4-level
- * paging and 57-bits with 5-level paging). Hence, skip bit
- * [N-1].
- */
- if (domain_use_first_level(domain))
- dma_mask = min_t(uint64_t, DOMAIN_MAX_ADDR(domain->gaw - 1),
- dma_mask);
- else
- dma_mask = min_t(uint64_t, DOMAIN_MAX_ADDR(domain->gaw),
- dma_mask);
-
- /* Ensure we reserve the whole size-aligned region */
- nrpages = __roundup_pow_of_two(nrpages);
-
- if (!dmar_forcedac && dma_mask > DMA_BIT_MASK(32)) {
- /*
- * First try to allocate an io virtual address in
- * DMA_BIT_MASK(32) and if that fails then try allocating
- * from higher range
- */
- iova_pfn = alloc_iova_fast(&domain->iovad, nrpages,
- IOVA_PFN(DMA_BIT_MASK(32)), false);
- if (iova_pfn)
- return iova_pfn;
- }
- iova_pfn = alloc_iova_fast(&domain->iovad, nrpages,
- IOVA_PFN(dma_mask), true);
- if (unlikely(!iova_pfn)) {
- dev_err_once(dev, "Allocating %ld-page iova failed\n",
- nrpages);
- return 0;
- }
-
- return iova_pfn;
-}
-
-static dma_addr_t __intel_map_single(struct device *dev, phys_addr_t paddr,
- size_t size, int dir, u64 dma_mask)
-{
- struct dmar_domain *domain;
- phys_addr_t start_paddr;
- unsigned long iova_pfn;
- int prot = 0;
- int ret;
- struct intel_iommu *iommu;
- unsigned long paddr_pfn = paddr >> PAGE_SHIFT;
-
- BUG_ON(dir == DMA_NONE);
-
- if (unlikely(attach_deferred(dev)))
- do_deferred_attach(dev);
-
- domain = find_domain(dev);
- if (!domain)
- return DMA_MAPPING_ERROR;
-
- iommu = domain_get_iommu(domain);
- size = aligned_nrpages(paddr, size);
-
- iova_pfn = intel_alloc_iova(dev, domain, dma_to_mm_pfn(size), dma_mask);
- if (!iova_pfn)
- goto error;
-
- /*
- * Check if DMAR supports zero-length reads on write only
- * mappings..
- */
- if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
- !cap_zlr(iommu->cap))
- prot |= DMA_PTE_READ;
- if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
- prot |= DMA_PTE_WRITE;
- /*
- * paddr - (paddr + size) might be partial page, we should map the whole
- * page. Note: if two part of one page are separately mapped, we
- * might have two guest_addr mapping to the same host paddr, but this
- * is not a big problem
- */
- ret = domain_pfn_mapping(domain, mm_to_dma_pfn(iova_pfn),
- mm_to_dma_pfn(paddr_pfn), size, prot);
- if (ret)
- goto error;
-
- start_paddr = (phys_addr_t)iova_pfn << PAGE_SHIFT;
- start_paddr += paddr & ~PAGE_MASK;
-
- trace_map_single(dev, start_paddr, paddr, size << VTD_PAGE_SHIFT);
-
- return start_paddr;
-
-error:
- if (iova_pfn)
- free_iova_fast(&domain->iovad, iova_pfn, dma_to_mm_pfn(size));
- dev_err(dev, "Device request: %zx@%llx dir %d --- failed\n",
- size, (unsigned long long)paddr, dir);
- return DMA_MAPPING_ERROR;
-}
-
-static dma_addr_t intel_map_page(struct device *dev, struct page *page,
- unsigned long offset, size_t size,
- enum dma_data_direction dir,
- unsigned long attrs)
-{
- return __intel_map_single(dev, page_to_phys(page) + offset,
- size, dir, *dev->dma_mask);
-}
-
-static dma_addr_t intel_map_resource(struct device *dev, phys_addr_t phys_addr,
- size_t size, enum dma_data_direction dir,
- unsigned long attrs)
-{
- return __intel_map_single(dev, phys_addr, size, dir, *dev->dma_mask);
-}
-
-static void intel_unmap(struct device *dev, dma_addr_t dev_addr, size_t size)
-{
- struct dmar_domain *domain;
- unsigned long start_pfn, last_pfn;
- unsigned long nrpages;
- unsigned long iova_pfn;
- struct intel_iommu *iommu;
- struct page *freelist;
- struct pci_dev *pdev = NULL;
-
- domain = find_domain(dev);
- BUG_ON(!domain);
-
- iommu = domain_get_iommu(domain);
-
- iova_pfn = IOVA_PFN(dev_addr);
-
- nrpages = aligned_nrpages(dev_addr, size);
- start_pfn = mm_to_dma_pfn(iova_pfn);
- last_pfn = start_pfn + nrpages - 1;
-
- if (dev_is_pci(dev))
- pdev = to_pci_dev(dev);
-
- freelist = domain_unmap(domain, start_pfn, last_pfn, NULL);
- if (intel_iommu_strict || (pdev && pdev->untrusted) ||
- !has_iova_flush_queue(&domain->iovad)) {
- iommu_flush_iotlb_psi(iommu, domain, start_pfn,
- nrpages, !freelist, 0);
- /* free iova */
- free_iova_fast(&domain->iovad, iova_pfn, dma_to_mm_pfn(nrpages));
- dma_free_pagelist(freelist);
- } else {
- queue_iova(&domain->iovad, iova_pfn, nrpages,
- (unsigned long)freelist);
- /*
- * queue up the release of the unmap to save the 1/6th of the
- * cpu used up by the iotlb flush operation...
- */
- }
-
- trace_unmap_single(dev, dev_addr, size);
-}
-
-static void intel_unmap_page(struct device *dev, dma_addr_t dev_addr,
- size_t size, enum dma_data_direction dir,
- unsigned long attrs)
-{
- intel_unmap(dev, dev_addr, size);
-}
-
-static void intel_unmap_resource(struct device *dev, dma_addr_t dev_addr,
- size_t size, enum dma_data_direction dir, unsigned long attrs)
-{
- intel_unmap(dev, dev_addr, size);
-}
-
-static void *intel_alloc_coherent(struct device *dev, size_t size,
- dma_addr_t *dma_handle, gfp_t flags,
- unsigned long attrs)
-{
- struct page *page = NULL;
- int order;
-
- if (unlikely(attach_deferred(dev)))
- do_deferred_attach(dev);
-
- size = PAGE_ALIGN(size);
- order = get_order(size);
-
- if (gfpflags_allow_blocking(flags)) {
- unsigned int count = size >> PAGE_SHIFT;
-
- page = dma_alloc_from_contiguous(dev, count, order,
- flags & __GFP_NOWARN);
- }
-
- if (!page)
- page = alloc_pages(flags, order);
- if (!page)
- return NULL;
- memset(page_address(page), 0, size);
-
- *dma_handle = __intel_map_single(dev, page_to_phys(page), size,
- DMA_BIDIRECTIONAL,
- dev->coherent_dma_mask);
- if (*dma_handle != DMA_MAPPING_ERROR)
- return page_address(page);
- if (!dma_release_from_contiguous(dev, page, size >> PAGE_SHIFT))
- __free_pages(page, order);
-
- return NULL;
-}
-
-static void intel_free_coherent(struct device *dev, size_t size, void *vaddr,
- dma_addr_t dma_handle, unsigned long attrs)
-{
- int order;
- struct page *page = virt_to_page(vaddr);
-
- size = PAGE_ALIGN(size);
- order = get_order(size);
-
- intel_unmap(dev, dma_handle, size);
- if (!dma_release_from_contiguous(dev, page, size >> PAGE_SHIFT))
- __free_pages(page, order);
-}
-
-static void intel_unmap_sg(struct device *dev, struct scatterlist *sglist,
- int nelems, enum dma_data_direction dir,
- unsigned long attrs)
-{
- dma_addr_t startaddr = sg_dma_address(sglist) & PAGE_MASK;
- unsigned long nrpages = 0;
- struct scatterlist *sg;
- int i;
-
- for_each_sg(sglist, sg, nelems, i) {
- nrpages += aligned_nrpages(sg_dma_address(sg), sg_dma_len(sg));
- }
-
- intel_unmap(dev, startaddr, nrpages << VTD_PAGE_SHIFT);
-
- trace_unmap_sg(dev, startaddr, nrpages << VTD_PAGE_SHIFT);
-}
-
-static int intel_map_sg(struct device *dev, struct scatterlist *sglist, int nelems,
- enum dma_data_direction dir, unsigned long attrs)
-{
- int i;
- struct dmar_domain *domain;
- size_t size = 0;
- int prot = 0;
- unsigned long iova_pfn;
- int ret;
- struct scatterlist *sg;
- unsigned long start_vpfn;
- struct intel_iommu *iommu;
-
- BUG_ON(dir == DMA_NONE);
-
- if (unlikely(attach_deferred(dev)))
- do_deferred_attach(dev);
-
- domain = find_domain(dev);
- if (!domain)
- return 0;
-
- iommu = domain_get_iommu(domain);
-
- for_each_sg(sglist, sg, nelems, i)
- size += aligned_nrpages(sg->offset, sg->length);
-
- iova_pfn = intel_alloc_iova(dev, domain, dma_to_mm_pfn(size),
- *dev->dma_mask);
- if (!iova_pfn) {
- sglist->dma_length = 0;
- return 0;
- }
-
- /*
- * Check if DMAR supports zero-length reads on write only
- * mappings..
- */
- if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
- !cap_zlr(iommu->cap))
- prot |= DMA_PTE_READ;
- if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
- prot |= DMA_PTE_WRITE;
-
- start_vpfn = mm_to_dma_pfn(iova_pfn);
-
- ret = domain_sg_mapping(domain, start_vpfn, sglist, size, prot);
- if (unlikely(ret)) {
- dma_pte_free_pagetable(domain, start_vpfn,
- start_vpfn + size - 1,
- agaw_to_level(domain->agaw) + 1);
- free_iova_fast(&domain->iovad, iova_pfn, dma_to_mm_pfn(size));
- return 0;
- }
-
- for_each_sg(sglist, sg, nelems, i)
- trace_map_sg(dev, i + 1, nelems, sg);
-
- return nelems;
-}
-
-static u64 intel_get_required_mask(struct device *dev)
-{
- return DMA_BIT_MASK(32);
-}
-
-static const struct dma_map_ops intel_dma_ops = {
- .alloc = intel_alloc_coherent,
- .free = intel_free_coherent,
- .map_sg = intel_map_sg,
- .unmap_sg = intel_unmap_sg,
- .map_page = intel_map_page,
- .unmap_page = intel_unmap_page,
- .map_resource = intel_map_resource,
- .unmap_resource = intel_unmap_resource,
- .dma_supported = dma_direct_supported,
- .mmap = dma_common_mmap,
- .get_sgtable = dma_common_get_sgtable,
- .get_required_mask = intel_get_required_mask,
-};
-
-static void
-bounce_sync_single(struct device *dev, dma_addr_t addr, size_t size,
- enum dma_data_direction dir, enum dma_sync_target target)
-{
- struct dmar_domain *domain;
- phys_addr_t tlb_addr;
-
- domain = find_domain(dev);
- if (WARN_ON(!domain))
- return;
-
- tlb_addr = intel_iommu_iova_to_phys(&domain->domain, addr);
- if (is_swiotlb_buffer(tlb_addr))
- swiotlb_tbl_sync_single(dev, tlb_addr, size, dir, target);
-}
-
-static dma_addr_t
-bounce_map_single(struct device *dev, phys_addr_t paddr, size_t size,
- enum dma_data_direction dir, unsigned long attrs,
- u64 dma_mask)
-{
- size_t aligned_size = ALIGN(size, VTD_PAGE_SIZE);
- struct dmar_domain *domain;
- struct intel_iommu *iommu;
- unsigned long iova_pfn;
- unsigned long nrpages;
- phys_addr_t tlb_addr;
- int prot = 0;
- int ret;
-
- if (unlikely(attach_deferred(dev)))
- do_deferred_attach(dev);
-
- domain = find_domain(dev);
-
- if (WARN_ON(dir == DMA_NONE || !domain))
- return DMA_MAPPING_ERROR;
-
- iommu = domain_get_iommu(domain);
- if (WARN_ON(!iommu))
- return DMA_MAPPING_ERROR;
-
- nrpages = aligned_nrpages(0, size);
- iova_pfn = intel_alloc_iova(dev, domain,
- dma_to_mm_pfn(nrpages), dma_mask);
- if (!iova_pfn)
- return DMA_MAPPING_ERROR;
-
- /*
- * Check if DMAR supports zero-length reads on write only
- * mappings..
- */
- if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL ||
- !cap_zlr(iommu->cap))
- prot |= DMA_PTE_READ;
- if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
- prot |= DMA_PTE_WRITE;
-
- /*
- * If both the physical buffer start address and size are
- * page aligned, we don't need to use a bounce page.
- */
- if (!IS_ALIGNED(paddr | size, VTD_PAGE_SIZE)) {
- tlb_addr = swiotlb_tbl_map_single(dev,
- __phys_to_dma(dev, io_tlb_start),
- paddr, size, aligned_size, dir, attrs);
- if (tlb_addr == DMA_MAPPING_ERROR) {
- goto swiotlb_error;
- } else {
- /* Cleanup the padding area. */
- void *padding_start = phys_to_virt(tlb_addr);
- size_t padding_size = aligned_size;
-
- if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC) &&
- (dir == DMA_TO_DEVICE ||
- dir == DMA_BIDIRECTIONAL)) {
- padding_start += size;
- padding_size -= size;
- }
-
- memset(padding_start, 0, padding_size);
- }
- } else {
- tlb_addr = paddr;
- }
-
- ret = domain_pfn_mapping(domain, mm_to_dma_pfn(iova_pfn),
- tlb_addr >> VTD_PAGE_SHIFT, nrpages, prot);
- if (ret)
- goto mapping_error;
-
- trace_bounce_map_single(dev, iova_pfn << PAGE_SHIFT, paddr, size);
-
- return (phys_addr_t)iova_pfn << PAGE_SHIFT;
-
-mapping_error:
- if (is_swiotlb_buffer(tlb_addr))
- swiotlb_tbl_unmap_single(dev, tlb_addr, size,
- aligned_size, dir, attrs);
-swiotlb_error:
- free_iova_fast(&domain->iovad, iova_pfn, dma_to_mm_pfn(nrpages));
- dev_err(dev, "Device bounce map: %zx@%llx dir %d --- failed\n",
- size, (unsigned long long)paddr, dir);
-
- return DMA_MAPPING_ERROR;
-}
-
-static void
-bounce_unmap_single(struct device *dev, dma_addr_t dev_addr, size_t size,
- enum dma_data_direction dir, unsigned long attrs)
-{
- size_t aligned_size = ALIGN(size, VTD_PAGE_SIZE);
- struct dmar_domain *domain;
- phys_addr_t tlb_addr;
-
- domain = find_domain(dev);
- if (WARN_ON(!domain))
- return;
-
- tlb_addr = intel_iommu_iova_to_phys(&domain->domain, dev_addr);
- if (WARN_ON(!tlb_addr))
- return;
-
- intel_unmap(dev, dev_addr, size);
- if (is_swiotlb_buffer(tlb_addr))
- swiotlb_tbl_unmap_single(dev, tlb_addr, size,
- aligned_size, dir, attrs);
-
- trace_bounce_unmap_single(dev, dev_addr, size);
-}
-
-static dma_addr_t
-bounce_map_page(struct device *dev, struct page *page, unsigned long offset,
- size_t size, enum dma_data_direction dir, unsigned long attrs)
-{
- return bounce_map_single(dev, page_to_phys(page) + offset,
- size, dir, attrs, *dev->dma_mask);
-}
-
-static dma_addr_t
-bounce_map_resource(struct device *dev, phys_addr_t phys_addr, size_t size,
- enum dma_data_direction dir, unsigned long attrs)
-{
- return bounce_map_single(dev, phys_addr, size,
- dir, attrs, *dev->dma_mask);
-}
-
-static void
-bounce_unmap_page(struct device *dev, dma_addr_t dev_addr, size_t size,
- enum dma_data_direction dir, unsigned long attrs)
-{
- bounce_unmap_single(dev, dev_addr, size, dir, attrs);
-}
-
-static void
-bounce_unmap_resource(struct device *dev, dma_addr_t dev_addr, size_t size,
- enum dma_data_direction dir, unsigned long attrs)
-{
- bounce_unmap_single(dev, dev_addr, size, dir, attrs);
-}
-
-static void
-bounce_unmap_sg(struct device *dev, struct scatterlist *sglist, int nelems,
- enum dma_data_direction dir, unsigned long attrs)
-{
- struct scatterlist *sg;
- int i;
-
- for_each_sg(sglist, sg, nelems, i)
- bounce_unmap_page(dev, sg->dma_address,
- sg_dma_len(sg), dir, attrs);
-}
-
-static int
-bounce_map_sg(struct device *dev, struct scatterlist *sglist, int nelems,
- enum dma_data_direction dir, unsigned long attrs)
-{
- int i;
- struct scatterlist *sg;
-
- for_each_sg(sglist, sg, nelems, i) {
- sg->dma_address = bounce_map_page(dev, sg_page(sg),
- sg->offset, sg->length,
- dir, attrs);
- if (sg->dma_address == DMA_MAPPING_ERROR)
- goto out_unmap;
- sg_dma_len(sg) = sg->length;
- }
-
- for_each_sg(sglist, sg, nelems, i)
- trace_bounce_map_sg(dev, i + 1, nelems, sg);
-
- return nelems;
-
-out_unmap:
- bounce_unmap_sg(dev, sglist, i, dir, attrs | DMA_ATTR_SKIP_CPU_SYNC);
- return 0;
-}
-
-static void
-bounce_sync_single_for_cpu(struct device *dev, dma_addr_t addr,
- size_t size, enum dma_data_direction dir)
-{
- bounce_sync_single(dev, addr, size, dir, SYNC_FOR_CPU);
-}
-
-static void
-bounce_sync_single_for_device(struct device *dev, dma_addr_t addr,
- size_t size, enum dma_data_direction dir)
-{
- bounce_sync_single(dev, addr, size, dir, SYNC_FOR_DEVICE);
-}
-
-static void
-bounce_sync_sg_for_cpu(struct device *dev, struct scatterlist *sglist,
- int nelems, enum dma_data_direction dir)
-{
- struct scatterlist *sg;
- int i;
-
- for_each_sg(sglist, sg, nelems, i)
- bounce_sync_single(dev, sg_dma_address(sg),
- sg_dma_len(sg), dir, SYNC_FOR_CPU);
-}
-
-static void
-bounce_sync_sg_for_device(struct device *dev, struct scatterlist *sglist,
- int nelems, enum dma_data_direction dir)
-{
- struct scatterlist *sg;
- int i;
-
- for_each_sg(sglist, sg, nelems, i)
- bounce_sync_single(dev, sg_dma_address(sg),
- sg_dma_len(sg), dir, SYNC_FOR_DEVICE);
-}
-
-static const struct dma_map_ops bounce_dma_ops = {
- .alloc = intel_alloc_coherent,
- .free = intel_free_coherent,
- .map_sg = bounce_map_sg,
- .unmap_sg = bounce_unmap_sg,
- .map_page = bounce_map_page,
- .unmap_page = bounce_unmap_page,
- .sync_single_for_cpu = bounce_sync_single_for_cpu,
- .sync_single_for_device = bounce_sync_single_for_device,
- .sync_sg_for_cpu = bounce_sync_sg_for_cpu,
- .sync_sg_for_device = bounce_sync_sg_for_device,
- .map_resource = bounce_map_resource,
- .unmap_resource = bounce_unmap_resource,
- .dma_supported = dma_direct_supported,
-};
-
static inline int iommu_domain_cache_init(void)
{
int ret = 0;
@@ -4616,7 +3967,7 @@ static void free_all_cpu_cached_iovas(unsigned int cpu)
if (!domain || domain->domain.type != IOMMU_DOMAIN_DMA)
continue;
- free_cpu_cached_iovas(cpu, &domain->iovad);
+ iommu_dma_free_cpu_cached_iovas(cpu, &domain->domain);
}
}
}
@@ -4888,12 +4239,6 @@ int __init intel_iommu_init(void)
if (list_empty(&dmar_atsr_units))
pr_info("No ATSR found\n");
- if (dmar_init_reserved_ranges()) {
- if (force_on)
- panic("tboot: Failed to reserve iommu ranges\n");
- goto out_free_reserved_range;
- }
-
if (dmar_map_gfx)
intel_iommu_gfx_mapped = 1;
@@ -4904,7 +4249,7 @@ int __init intel_iommu_init(void)
if (force_on)
panic("tboot: Failed to initialize DMARs\n");
pr_err("Initialization failed\n");
- goto out_free_reserved_range;
+ goto out_free_dmar;
}
up_write(&dmar_global_lock);
@@ -4945,8 +4290,6 @@ int __init intel_iommu_init(void)
return 0;
-out_free_reserved_range:
- put_iova_domain(&reserved_iova_list);
out_free_dmar:
intel_iommu_free_dmars();
up_write(&dmar_global_lock);
@@ -5044,17 +4387,6 @@ static int md_domain_init(struct dmar_domain *domain, int guest_width)
return 0;
}
-static void intel_init_iova_domain(struct dmar_domain *dmar_domain)
-{
- init_iova_domain(&dmar_domain->iovad, VTD_PAGE_SIZE, IOVA_START_PFN);
- copy_reserved_iova(&reserved_iova_list, &dmar_domain->iovad);
-
- if (!intel_iommu_strict &&
- init_iova_flush_queue(&dmar_domain->iovad,
- iommu_flush_iova, iova_entry_free))
- pr_info("iova flush queue initialization failed\n");
-}
-
static struct iommu_domain *intel_iommu_domain_alloc(unsigned type)
{
struct dmar_domain *dmar_domain;
@@ -5075,8 +4407,9 @@ static struct iommu_domain *intel_iommu_domain_alloc(unsigned type)
return NULL;
}
- if (type == IOMMU_DOMAIN_DMA)
- intel_init_iova_domain(dmar_domain);
+ if (type == IOMMU_DOMAIN_DMA &&
+ iommu_get_dma_cookie(&dmar_domain->domain))
+ return NULL;
domain_update_iommu_cap(dmar_domain);
@@ -5700,24 +5033,26 @@ static void intel_iommu_release_device(struct device *dev)
set_dma_ops(dev, NULL);
}
+static int intel_iommu_needs_bounce_buffer(struct device *d)
+{
+ return !intel_no_bounce && dev_is_pci(d) && to_pci_dev(d)->untrusted;
+}
+
+
static void intel_iommu_probe_finalize(struct device *dev)
{
- struct iommu_domain *domain;
+ dma_addr_t base = IOVA_START_PFN << VTD_PAGE_SHIFT;
+ struct iommu_domain *domain = iommu_get_domain_for_dev(dev);
+ struct dmar_domain *dmar_domain = to_dmar_domain(domain);
- domain = iommu_get_domain_for_dev(dev);
- if (device_needs_bounce(dev))
- set_dma_ops(dev, &bounce_dma_ops);
- else if (domain && domain->type == IOMMU_DOMAIN_DMA)
- set_dma_ops(dev, &intel_dma_ops);
+ if (intel_iommu_needs_bounce_buffer(dev) ||
+ (domain && domain->type == IOMMU_DOMAIN_DMA))
+ iommu_setup_dma_ops(dev, base,
+ __DOMAIN_MAX_ADDR(dmar_domain->gaw) - base);
else
set_dma_ops(dev, NULL);
}
-static int intel_iommu_needs_bounce_buffer(struct device *d)
-{
- return !intel_no_bounce && dev_is_pci(d) && to_pci_dev(d)->untrusted;
-}
-
static void intel_iommu_get_resv_regions(struct device *device,
struct list_head *head)
{
@@ -5826,19 +5161,6 @@ int intel_iommu_enable_pasid(struct intel_iommu *iommu, struct device *dev)
return ret;
}
-static void intel_iommu_apply_resv_region(struct device *dev,
- struct iommu_domain *domain,
- struct iommu_resv_region *region)
-{
- struct dmar_domain *dmar_domain = to_dmar_domain(domain);
- unsigned long start, end;
-
- start = IOVA_PFN(region->start);
- end = IOVA_PFN(region->start + region->length - 1);
-
- WARN_ON_ONCE(!reserve_iova(&dmar_domain->iovad, start, end));
-}
-
static struct iommu_group *intel_iommu_device_group(struct device *dev)
{
if (dev_is_pci(dev))
@@ -6050,6 +5372,27 @@ intel_iommu_domain_set_attr(struct iommu_domain *domain,
return ret;
}
+static int intel_iommu_domain_get_attr(struct iommu_domain *domain,
+ enum iommu_attr attr, void *data)
+{
+ switch (domain->type) {
+ case IOMMU_DOMAIN_UNMANAGED:
+ return -ENODEV;
+ case IOMMU_DOMAIN_DMA:
+ switch (attr) {
+ case DOMAIN_ATTR_DMA_USE_FLUSH_QUEUE:
+ *(int *)data = !intel_iommu_strict;
+ return 0;
+ default:
+ return -ENODEV;
+ }
+ break;
+ default:
+ return -EINVAL;
+ }
+}
+
+
/*
* Check that the device does not live on an external facing PCI port that is
* marked as untrusted. Such devices should not be able to apply quirks and
@@ -6079,15 +5422,16 @@ const struct iommu_ops intel_iommu_ops = {
.aux_get_pasid = intel_iommu_aux_get_pasid,
.map = intel_iommu_map,
.unmap = intel_iommu_unmap,
+ .flush_iotlb_all = intel_flush_iotlb_all,
.iotlb_sync = intel_iommu_tlb_sync,
.iova_to_phys = intel_iommu_iova_to_phys,
.probe_device = intel_iommu_probe_device,
.probe_finalize = intel_iommu_probe_finalize,
.release_device = intel_iommu_release_device,
+ .domain_get_attr = intel_iommu_domain_get_attr,
.needs_bounce_buffer = intel_iommu_needs_bounce_buffer,
.get_resv_regions = intel_iommu_get_resv_regions,
.put_resv_regions = generic_iommu_put_resv_regions,
- .apply_resv_region = intel_iommu_apply_resv_region,
.device_group = intel_iommu_device_group,
.dev_has_feat = intel_iommu_dev_has_feat,
.dev_feat_enabled = intel_iommu_dev_feat_enabled,
--
2.20.1
_______________________________________________
iommu mailing list
iommu@lists.linux-foundation.org
https://lists.linuxfoundation.org/mailman/listinfo/iommu
^ permalink raw reply related [flat|nested] 22+ messages in thread