* [PATCH v3 01/14] iommu/amd: Re-define amd_iommu_domain_encode_pgtable as inline
2020-10-04 1:45 [PATCH v3 00/14] iommu/amd: Add Generic IO Page Table Framework Support Suravee Suthikulpanit
@ 2020-10-04 1:45 ` Suravee Suthikulpanit
2020-10-04 1:45 ` [PATCH v3 02/14] iommu/amd: Prepare for generic IO page table framework Suravee Suthikulpanit
` (13 subsequent siblings)
14 siblings, 0 replies; 20+ messages in thread
From: Suravee Suthikulpanit @ 2020-10-04 1:45 UTC (permalink / raw)
To: linux-kernel, iommu; +Cc: joro, robin.murphy, Suravee Suthikulpanit
Move the function to header file to allow inclusion in other files.
Signed-off-by: Suravee Suthikulpanit <suravee.suthikulpanit@amd.com>
---
drivers/iommu/amd/amd_iommu.h | 13 +++++++++++++
drivers/iommu/amd/iommu.c | 10 ----------
2 files changed, 13 insertions(+), 10 deletions(-)
diff --git a/drivers/iommu/amd/amd_iommu.h b/drivers/iommu/amd/amd_iommu.h
index 57309716fd18..97cdb235ce69 100644
--- a/drivers/iommu/amd/amd_iommu.h
+++ b/drivers/iommu/amd/amd_iommu.h
@@ -93,6 +93,19 @@ static inline void *iommu_phys_to_virt(unsigned long paddr)
return phys_to_virt(__sme_clr(paddr));
}
+static inline
+void amd_iommu_domain_set_pt_root(struct protection_domain *domain, u64 root)
+{
+ atomic64_set(&domain->pt_root, root);
+}
+
+static inline
+void amd_iommu_domain_clr_pt_root(struct protection_domain *domain)
+{
+ amd_iommu_domain_set_pt_root(domain, 0);
+}
+
+
extern bool translation_pre_enabled(struct amd_iommu *iommu);
extern bool amd_iommu_is_attach_deferred(struct iommu_domain *domain,
struct device *dev);
diff --git a/drivers/iommu/amd/iommu.c b/drivers/iommu/amd/iommu.c
index db4fb840c59c..e92b3f744292 100644
--- a/drivers/iommu/amd/iommu.c
+++ b/drivers/iommu/amd/iommu.c
@@ -162,16 +162,6 @@ static void amd_iommu_domain_get_pgtable(struct protection_domain *domain,
pgtable->mode = pt_root & 7; /* lowest 3 bits encode pgtable mode */
}
-static void amd_iommu_domain_set_pt_root(struct protection_domain *domain, u64 root)
-{
- atomic64_set(&domain->pt_root, root);
-}
-
-static void amd_iommu_domain_clr_pt_root(struct protection_domain *domain)
-{
- amd_iommu_domain_set_pt_root(domain, 0);
-}
-
static void amd_iommu_domain_set_pgtable(struct protection_domain *domain,
u64 *root, int mode)
{
--
2.17.1
^ permalink raw reply related [flat|nested] 20+ messages in thread
* [PATCH v3 02/14] iommu/amd: Prepare for generic IO page table framework
2020-10-04 1:45 [PATCH v3 00/14] iommu/amd: Add Generic IO Page Table Framework Support Suravee Suthikulpanit
2020-10-04 1:45 ` [PATCH v3 01/14] iommu/amd: Re-define amd_iommu_domain_encode_pgtable as inline Suravee Suthikulpanit
@ 2020-10-04 1:45 ` Suravee Suthikulpanit
2020-10-04 1:45 ` [PATCH v3 03/14] iommu/amd: Move pt_root to to struct amd_io_pgtable Suravee Suthikulpanit
` (12 subsequent siblings)
14 siblings, 0 replies; 20+ messages in thread
From: Suravee Suthikulpanit @ 2020-10-04 1:45 UTC (permalink / raw)
To: linux-kernel, iommu; +Cc: joro, robin.murphy, Suravee Suthikulpanit
Add initial hook up code to implement generic IO page table framework.
Signed-off-by: Suravee Suthikulpanit <suravee.suthikulpanit@amd.com>
---
drivers/iommu/amd/Kconfig | 1 +
drivers/iommu/amd/Makefile | 2 +-
drivers/iommu/amd/amd_iommu_types.h | 35 +++++++++++++++++++++++
drivers/iommu/amd/io_pgtable.c | 43 +++++++++++++++++++++++++++++
drivers/iommu/amd/iommu.c | 10 -------
drivers/iommu/io-pgtable.c | 3 ++
include/linux/io-pgtable.h | 2 ++
7 files changed, 85 insertions(+), 11 deletions(-)
create mode 100644 drivers/iommu/amd/io_pgtable.c
diff --git a/drivers/iommu/amd/Kconfig b/drivers/iommu/amd/Kconfig
index 626b97d0dd21..a3cbafb603f5 100644
--- a/drivers/iommu/amd/Kconfig
+++ b/drivers/iommu/amd/Kconfig
@@ -10,6 +10,7 @@ config AMD_IOMMU
select IOMMU_API
select IOMMU_IOVA
select IOMMU_DMA
+ select IOMMU_IO_PGTABLE
depends on X86_64 && PCI && ACPI && HAVE_CMPXCHG_DOUBLE
help
With this option you can enable support for AMD IOMMU hardware in
diff --git a/drivers/iommu/amd/Makefile b/drivers/iommu/amd/Makefile
index dc5a2fa4fd37..a935f8f4b974 100644
--- a/drivers/iommu/amd/Makefile
+++ b/drivers/iommu/amd/Makefile
@@ -1,4 +1,4 @@
# SPDX-License-Identifier: GPL-2.0-only
-obj-$(CONFIG_AMD_IOMMU) += iommu.o init.o quirks.o
+obj-$(CONFIG_AMD_IOMMU) += iommu.o init.o quirks.o io_pgtable.o
obj-$(CONFIG_AMD_IOMMU_DEBUGFS) += debugfs.o
obj-$(CONFIG_AMD_IOMMU_V2) += iommu_v2.o
diff --git a/drivers/iommu/amd/amd_iommu_types.h b/drivers/iommu/amd/amd_iommu_types.h
index f696ac7c5f89..e3ac3e57e507 100644
--- a/drivers/iommu/amd/amd_iommu_types.h
+++ b/drivers/iommu/amd/amd_iommu_types.h
@@ -15,6 +15,7 @@
#include <linux/spinlock.h>
#include <linux/pci.h>
#include <linux/irqreturn.h>
+#include <linux/io-pgtable.h>
/*
* Maximum number of IOMMUs supported
@@ -252,6 +253,19 @@
#define GA_GUEST_NR 0x1
+#define IOMMU_IN_ADDR_BIT_SIZE 52
+#define IOMMU_OUT_ADDR_BIT_SIZE 52
+
+/*
+ * This bitmap is used to advertise the page sizes our hardware support
+ * to the IOMMU core, which will then use this information to split
+ * physically contiguous memory regions it is mapping into page sizes
+ * that we support.
+ *
+ * 512GB Pages are not supported due to a hardware bug
+ */
+#define AMD_IOMMU_PGSIZES ((~0xFFFUL) & ~(2ULL << 38))
+
/* Bit value definition for dte irq remapping fields*/
#define DTE_IRQ_PHYS_ADDR_MASK (((1ULL << 45)-1) << 6)
#define DTE_IRQ_REMAP_INTCTL_MASK (0x3ULL << 60)
@@ -461,6 +475,26 @@ struct amd_irte_ops;
#define AMD_IOMMU_FLAG_TRANS_PRE_ENABLED (1 << 0)
+#define io_pgtable_to_data(x) \
+ container_of((x), struct amd_io_pgtable, iop)
+
+#define io_pgtable_ops_to_data(x) \
+ io_pgtable_to_data(io_pgtable_ops_to_pgtable(x))
+
+#define io_pgtable_ops_to_domain(x) \
+ container_of(io_pgtable_ops_to_data(x), \
+ struct protection_domain, iop)
+
+#define io_pgtable_cfg_to_data(x) \
+ container_of((x), struct amd_io_pgtable, pgtbl_cfg)
+
+struct amd_io_pgtable {
+ struct io_pgtable_cfg pgtbl_cfg;
+ struct io_pgtable iop;
+ int mode;
+ u64 *root;
+};
+
/*
* This structure contains generic data for IOMMU protection domains
* independent of their use.
@@ -469,6 +503,7 @@ struct protection_domain {
struct list_head dev_list; /* List of all devices in this domain */
struct iommu_domain domain; /* generic domain handle used by
iommu core code */
+ struct amd_io_pgtable iop;
spinlock_t lock; /* mostly used to lock the page table*/
u16 id; /* the domain id written to the device table */
atomic64_t pt_root; /* pgtable root and pgtable mode */
diff --git a/drivers/iommu/amd/io_pgtable.c b/drivers/iommu/amd/io_pgtable.c
new file mode 100644
index 000000000000..6b2de9e467d9
--- /dev/null
+++ b/drivers/iommu/amd/io_pgtable.c
@@ -0,0 +1,43 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * CPU-agnostic AMD IO page table allocator.
+ *
+ * Copyright (C) 2020 Advanced Micro Devices, Inc.
+ * Author: Suravee Suthikulpanit <suravee.suthikulpanit@amd.com>
+ */
+
+#define pr_fmt(fmt) "AMD-Vi: " fmt
+#define dev_fmt(fmt) pr_fmt(fmt)
+
+#include <linux/atomic.h>
+#include <linux/bitops.h>
+#include <linux/io-pgtable.h>
+#include <linux/kernel.h>
+#include <linux/sizes.h>
+#include <linux/slab.h>
+#include <linux/types.h>
+#include <linux/dma-mapping.h>
+
+#include <asm/barrier.h>
+
+#include "amd_iommu_types.h"
+#include "amd_iommu.h"
+
+/*
+ * ----------------------------------------------------
+ */
+static void v1_free_pgtable(struct io_pgtable *iop)
+{
+}
+
+static struct io_pgtable *v1_alloc_pgtable(struct io_pgtable_cfg *cfg, void *cookie)
+{
+ struct amd_io_pgtable *pgtable = io_pgtable_cfg_to_data(cfg);
+
+ return &pgtable->iop;
+}
+
+struct io_pgtable_init_fns io_pgtable_amd_iommu_v1_init_fns = {
+ .alloc = v1_alloc_pgtable,
+ .free = v1_free_pgtable,
+};
diff --git a/drivers/iommu/amd/iommu.c b/drivers/iommu/amd/iommu.c
index e92b3f744292..2b7eb51dcbb8 100644
--- a/drivers/iommu/amd/iommu.c
+++ b/drivers/iommu/amd/iommu.c
@@ -59,16 +59,6 @@
#define HT_RANGE_START (0xfd00000000ULL)
#define HT_RANGE_END (0xffffffffffULL)
-/*
- * This bitmap is used to advertise the page sizes our hardware support
- * to the IOMMU core, which will then use this information to split
- * physically contiguous memory regions it is mapping into page sizes
- * that we support.
- *
- * 512GB Pages are not supported due to a hardware bug
- */
-#define AMD_IOMMU_PGSIZES ((~0xFFFUL) & ~(2ULL << 38))
-
#define DEFAULT_PGTABLE_LEVEL PAGE_MODE_3_LEVEL
static DEFINE_SPINLOCK(pd_bitmap_lock);
diff --git a/drivers/iommu/io-pgtable.c b/drivers/iommu/io-pgtable.c
index 94394c81468f..6e9917ce980f 100644
--- a/drivers/iommu/io-pgtable.c
+++ b/drivers/iommu/io-pgtable.c
@@ -24,6 +24,9 @@ io_pgtable_init_table[IO_PGTABLE_NUM_FMTS] = {
#ifdef CONFIG_IOMMU_IO_PGTABLE_ARMV7S
[ARM_V7S] = &io_pgtable_arm_v7s_init_fns,
#endif
+#ifdef CONFIG_AMD_IOMMU
+ [AMD_IOMMU_V1] = &io_pgtable_amd_iommu_v1_init_fns,
+#endif
};
struct io_pgtable_ops *alloc_io_pgtable_ops(enum io_pgtable_fmt fmt,
diff --git a/include/linux/io-pgtable.h b/include/linux/io-pgtable.h
index 23285ba645db..db25d436cabd 100644
--- a/include/linux/io-pgtable.h
+++ b/include/linux/io-pgtable.h
@@ -15,6 +15,7 @@ enum io_pgtable_fmt {
ARM_64_LPAE_S2,
ARM_V7S,
ARM_MALI_LPAE,
+ AMD_IOMMU_V1,
IO_PGTABLE_NUM_FMTS,
};
@@ -254,5 +255,6 @@ extern struct io_pgtable_init_fns io_pgtable_arm_64_lpae_s1_init_fns;
extern struct io_pgtable_init_fns io_pgtable_arm_64_lpae_s2_init_fns;
extern struct io_pgtable_init_fns io_pgtable_arm_v7s_init_fns;
extern struct io_pgtable_init_fns io_pgtable_arm_mali_lpae_init_fns;
+extern struct io_pgtable_init_fns io_pgtable_amd_iommu_v1_init_fns;
#endif /* __IO_PGTABLE_H */
--
2.17.1
^ permalink raw reply related [flat|nested] 20+ messages in thread
* [PATCH v3 03/14] iommu/amd: Move pt_root to to struct amd_io_pgtable
2020-10-04 1:45 [PATCH v3 00/14] iommu/amd: Add Generic IO Page Table Framework Support Suravee Suthikulpanit
2020-10-04 1:45 ` [PATCH v3 01/14] iommu/amd: Re-define amd_iommu_domain_encode_pgtable as inline Suravee Suthikulpanit
2020-10-04 1:45 ` [PATCH v3 02/14] iommu/amd: Prepare for generic IO page table framework Suravee Suthikulpanit
@ 2020-10-04 1:45 ` Suravee Suthikulpanit
2020-10-04 1:45 ` [PATCH v3 04/14] iommu/amd: Convert to using amd_io_pgtable Suravee Suthikulpanit
` (11 subsequent siblings)
14 siblings, 0 replies; 20+ messages in thread
From: Suravee Suthikulpanit @ 2020-10-04 1:45 UTC (permalink / raw)
To: linux-kernel, iommu; +Cc: joro, robin.murphy, Suravee Suthikulpanit
To better organize the data structure since it contains IO page table
related information.
Signed-off-by: Suravee Suthikulpanit <suravee.suthikulpanit@amd.com>
---
drivers/iommu/amd/amd_iommu.h | 2 +-
drivers/iommu/amd/amd_iommu_types.h | 2 +-
drivers/iommu/amd/iommu.c | 2 +-
3 files changed, 3 insertions(+), 3 deletions(-)
diff --git a/drivers/iommu/amd/amd_iommu.h b/drivers/iommu/amd/amd_iommu.h
index 97cdb235ce69..da6e09657e00 100644
--- a/drivers/iommu/amd/amd_iommu.h
+++ b/drivers/iommu/amd/amd_iommu.h
@@ -96,7 +96,7 @@ static inline void *iommu_phys_to_virt(unsigned long paddr)
static inline
void amd_iommu_domain_set_pt_root(struct protection_domain *domain, u64 root)
{
- atomic64_set(&domain->pt_root, root);
+ atomic64_set(&domain->iop.pt_root, root);
}
static inline
diff --git a/drivers/iommu/amd/amd_iommu_types.h b/drivers/iommu/amd/amd_iommu_types.h
index e3ac3e57e507..80b5c34357ed 100644
--- a/drivers/iommu/amd/amd_iommu_types.h
+++ b/drivers/iommu/amd/amd_iommu_types.h
@@ -493,6 +493,7 @@ struct amd_io_pgtable {
struct io_pgtable iop;
int mode;
u64 *root;
+ atomic64_t pt_root; /* pgtable root and pgtable mode */
};
/*
@@ -506,7 +507,6 @@ struct protection_domain {
struct amd_io_pgtable iop;
spinlock_t lock; /* mostly used to lock the page table*/
u16 id; /* the domain id written to the device table */
- atomic64_t pt_root; /* pgtable root and pgtable mode */
int glx; /* Number of levels for GCR3 table */
u64 *gcr3_tbl; /* Guest CR3 table */
unsigned long flags; /* flags to find out type of domain */
diff --git a/drivers/iommu/amd/iommu.c b/drivers/iommu/amd/iommu.c
index 2b7eb51dcbb8..c8b8619cc744 100644
--- a/drivers/iommu/amd/iommu.c
+++ b/drivers/iommu/amd/iommu.c
@@ -146,7 +146,7 @@ static struct protection_domain *to_pdomain(struct iommu_domain *dom)
static void amd_iommu_domain_get_pgtable(struct protection_domain *domain,
struct domain_pgtable *pgtable)
{
- u64 pt_root = atomic64_read(&domain->pt_root);
+ u64 pt_root = atomic64_read(&domain->iop.pt_root);
pgtable->root = (u64 *)(pt_root & PAGE_MASK);
pgtable->mode = pt_root & 7; /* lowest 3 bits encode pgtable mode */
--
2.17.1
^ permalink raw reply related [flat|nested] 20+ messages in thread
* [PATCH v3 04/14] iommu/amd: Convert to using amd_io_pgtable
2020-10-04 1:45 [PATCH v3 00/14] iommu/amd: Add Generic IO Page Table Framework Support Suravee Suthikulpanit
` (2 preceding siblings ...)
2020-10-04 1:45 ` [PATCH v3 03/14] iommu/amd: Move pt_root to to struct amd_io_pgtable Suravee Suthikulpanit
@ 2020-10-04 1:45 ` Suravee Suthikulpanit
2020-10-04 1:45 ` [PATCH v3 05/14] iommu/amd: Declare functions as extern Suravee Suthikulpanit
` (10 subsequent siblings)
14 siblings, 0 replies; 20+ messages in thread
From: Suravee Suthikulpanit @ 2020-10-04 1:45 UTC (permalink / raw)
To: linux-kernel, iommu; +Cc: joro, robin.murphy, Suravee Suthikulpanit
Make use of the new struct amd_io_pgtable in preparation to remove
the struct domain_pgtable.
Signed-off-by: Suravee Suthikulpanit <suravee.suthikulpanit@amd.com>
---
drivers/iommu/amd/amd_iommu.h | 1 +
drivers/iommu/amd/iommu.c | 25 ++++++++++---------------
2 files changed, 11 insertions(+), 15 deletions(-)
diff --git a/drivers/iommu/amd/amd_iommu.h b/drivers/iommu/amd/amd_iommu.h
index da6e09657e00..22ecacb71675 100644
--- a/drivers/iommu/amd/amd_iommu.h
+++ b/drivers/iommu/amd/amd_iommu.h
@@ -47,6 +47,7 @@ extern void amd_iommu_domain_direct_map(struct iommu_domain *dom);
extern int amd_iommu_domain_enable_v2(struct iommu_domain *dom, int pasids);
extern int amd_iommu_flush_page(struct iommu_domain *dom, int pasid,
u64 address);
+extern void amd_iommu_update_and_flush_device_table(struct protection_domain *domain);
extern int amd_iommu_flush_tlb(struct iommu_domain *dom, int pasid);
extern int amd_iommu_domain_set_gcr3(struct iommu_domain *dom, int pasid,
unsigned long cr3);
diff --git a/drivers/iommu/amd/iommu.c b/drivers/iommu/amd/iommu.c
index c8b8619cc744..09da37c4c9c4 100644
--- a/drivers/iommu/amd/iommu.c
+++ b/drivers/iommu/amd/iommu.c
@@ -90,8 +90,6 @@ struct kmem_cache *amd_iommu_irq_cache;
static void update_domain(struct protection_domain *domain);
static void detach_device(struct device *dev);
-static void update_and_flush_device_table(struct protection_domain *domain,
- struct domain_pgtable *pgtable);
/****************************************************************************
*
@@ -1482,7 +1480,7 @@ static bool increase_address_space(struct protection_domain *domain,
pgtable.root = pte;
pgtable.mode += 1;
- update_and_flush_device_table(domain, &pgtable);
+ amd_iommu_update_and_flush_device_table(domain);
domain_flush_complete(domain);
/*
@@ -1857,17 +1855,16 @@ static void free_gcr3_table(struct protection_domain *domain)
}
static void set_dte_entry(u16 devid, struct protection_domain *domain,
- struct domain_pgtable *pgtable,
bool ats, bool ppr)
{
u64 pte_root = 0;
u64 flags = 0;
u32 old_domid;
- if (pgtable->mode != PAGE_MODE_NONE)
- pte_root = iommu_virt_to_phys(pgtable->root);
+ if (domain->iop.mode != PAGE_MODE_NONE)
+ pte_root = iommu_virt_to_phys(domain->iop.root);
- pte_root |= (pgtable->mode & DEV_ENTRY_MODE_MASK)
+ pte_root |= (domain->iop.mode & DEV_ENTRY_MODE_MASK)
<< DEV_ENTRY_MODE_SHIFT;
pte_root |= DTE_FLAG_IR | DTE_FLAG_IW | DTE_FLAG_V | DTE_FLAG_TV;
@@ -1957,7 +1954,7 @@ static void do_attach(struct iommu_dev_data *dev_data,
/* Update device table */
amd_iommu_domain_get_pgtable(domain, &pgtable);
- set_dte_entry(dev_data->devid, domain, &pgtable,
+ set_dte_entry(dev_data->devid, domain,
ats, dev_data->iommu_v2);
clone_aliases(dev_data->pdev);
@@ -2263,22 +2260,20 @@ static int amd_iommu_domain_get_attr(struct iommu_domain *domain,
*
*****************************************************************************/
-static void update_device_table(struct protection_domain *domain,
- struct domain_pgtable *pgtable)
+static void update_device_table(struct protection_domain *domain)
{
struct iommu_dev_data *dev_data;
list_for_each_entry(dev_data, &domain->dev_list, list) {
- set_dte_entry(dev_data->devid, domain, pgtable,
+ set_dte_entry(dev_data->devid, domain,
dev_data->ats.enabled, dev_data->iommu_v2);
clone_aliases(dev_data->pdev);
}
}
-static void update_and_flush_device_table(struct protection_domain *domain,
- struct domain_pgtable *pgtable)
+void amd_iommu_update_and_flush_device_table(struct protection_domain *domain)
{
- update_device_table(domain, pgtable);
+ update_device_table(domain);
domain_flush_devices(domain);
}
@@ -2288,7 +2283,7 @@ static void update_domain(struct protection_domain *domain)
/* Update device table */
amd_iommu_domain_get_pgtable(domain, &pgtable);
- update_and_flush_device_table(domain, &pgtable);
+ amd_iommu_update_and_flush_device_table(domain);
/* Flush domain TLB(s) and wait for completion */
domain_flush_tlb_pde(domain);
--
2.17.1
^ permalink raw reply related [flat|nested] 20+ messages in thread
* [PATCH v3 05/14] iommu/amd: Declare functions as extern
2020-10-04 1:45 [PATCH v3 00/14] iommu/amd: Add Generic IO Page Table Framework Support Suravee Suthikulpanit
` (3 preceding siblings ...)
2020-10-04 1:45 ` [PATCH v3 04/14] iommu/amd: Convert to using amd_io_pgtable Suravee Suthikulpanit
@ 2020-10-04 1:45 ` Suravee Suthikulpanit
2020-10-04 1:45 ` [PATCH v3 06/14] iommu/amd: Move IO page table related functions Suravee Suthikulpanit
` (9 subsequent siblings)
14 siblings, 0 replies; 20+ messages in thread
From: Suravee Suthikulpanit @ 2020-10-04 1:45 UTC (permalink / raw)
To: linux-kernel, iommu; +Cc: joro, robin.murphy, Suravee Suthikulpanit
And move declaration to header file so that they can be included across
multiple files. There is no functional change.
Signed-off-by: Suravee Suthikulpanit <suravee.suthikulpanit@amd.com>
---
drivers/iommu/amd/amd_iommu.h | 3 +++
drivers/iommu/amd/iommu.c | 39 +++++++++++++++++------------------
2 files changed, 22 insertions(+), 20 deletions(-)
diff --git a/drivers/iommu/amd/amd_iommu.h b/drivers/iommu/amd/amd_iommu.h
index 22ecacb71675..8b7be9171030 100644
--- a/drivers/iommu/amd/amd_iommu.h
+++ b/drivers/iommu/amd/amd_iommu.h
@@ -48,6 +48,9 @@ extern int amd_iommu_domain_enable_v2(struct iommu_domain *dom, int pasids);
extern int amd_iommu_flush_page(struct iommu_domain *dom, int pasid,
u64 address);
extern void amd_iommu_update_and_flush_device_table(struct protection_domain *domain);
+extern void amd_iommu_domain_update(struct protection_domain *domain);
+extern void amd_iommu_domain_flush_complete(struct protection_domain *domain);
+extern void amd_iommu_domain_flush_tlb_pde(struct protection_domain *domain);
extern int amd_iommu_flush_tlb(struct iommu_domain *dom, int pasid);
extern int amd_iommu_domain_set_gcr3(struct iommu_domain *dom, int pasid,
unsigned long cr3);
diff --git a/drivers/iommu/amd/iommu.c b/drivers/iommu/amd/iommu.c
index 09da37c4c9c4..f91f35edb7ba 100644
--- a/drivers/iommu/amd/iommu.c
+++ b/drivers/iommu/amd/iommu.c
@@ -88,7 +88,6 @@ struct iommu_cmd {
struct kmem_cache *amd_iommu_irq_cache;
-static void update_domain(struct protection_domain *domain);
static void detach_device(struct device *dev);
/****************************************************************************
@@ -1294,12 +1293,12 @@ static void domain_flush_pages(struct protection_domain *domain,
}
/* Flush the whole IO/TLB for a given protection domain - including PDE */
-static void domain_flush_tlb_pde(struct protection_domain *domain)
+void amd_iommu_domain_flush_tlb_pde(struct protection_domain *domain)
{
__domain_flush_pages(domain, 0, CMD_INV_IOMMU_ALL_PAGES_ADDRESS, 1);
}
-static void domain_flush_complete(struct protection_domain *domain)
+void amd_iommu_domain_flush_complete(struct protection_domain *domain)
{
int i;
@@ -1324,7 +1323,7 @@ static void domain_flush_np_cache(struct protection_domain *domain,
spin_lock_irqsave(&domain->lock, flags);
domain_flush_pages(domain, iova, size);
- domain_flush_complete(domain);
+ amd_iommu_domain_flush_complete(domain);
spin_unlock_irqrestore(&domain->lock, flags);
}
}
@@ -1481,7 +1480,7 @@ static bool increase_address_space(struct protection_domain *domain,
pgtable.root = pte;
pgtable.mode += 1;
amd_iommu_update_and_flush_device_table(domain);
- domain_flush_complete(domain);
+ amd_iommu_domain_flush_complete(domain);
/*
* Device Table needs to be updated and flushed before the new root can
@@ -1734,8 +1733,8 @@ static int iommu_map_page(struct protection_domain *dom,
* Updates and flushing already happened in
* increase_address_space().
*/
- domain_flush_tlb_pde(dom);
- domain_flush_complete(dom);
+ amd_iommu_domain_flush_tlb_pde(dom);
+ amd_iommu_domain_flush_complete(dom);
spin_unlock_irqrestore(&dom->lock, flags);
}
@@ -1978,10 +1977,10 @@ static void do_detach(struct iommu_dev_data *dev_data)
device_flush_dte(dev_data);
/* Flush IOTLB */
- domain_flush_tlb_pde(domain);
+ amd_iommu_domain_flush_tlb_pde(domain);
/* Wait for the flushes to finish */
- domain_flush_complete(domain);
+ amd_iommu_domain_flush_complete(domain);
/* decrease reference counters - needs to happen after the flushes */
domain->dev_iommu[iommu->index] -= 1;
@@ -2114,9 +2113,9 @@ static int attach_device(struct device *dev,
* left the caches in the IOMMU dirty. So we have to flush
* here to evict all dirty stuff.
*/
- domain_flush_tlb_pde(domain);
+ amd_iommu_domain_flush_tlb_pde(domain);
- domain_flush_complete(domain);
+ amd_iommu_domain_flush_complete(domain);
out:
spin_unlock(&dev_data->lock);
@@ -2277,7 +2276,7 @@ void amd_iommu_update_and_flush_device_table(struct protection_domain *domain)
domain_flush_devices(domain);
}
-static void update_domain(struct protection_domain *domain)
+void amd_iommu_domain_update(struct protection_domain *domain)
{
struct domain_pgtable pgtable;
@@ -2286,8 +2285,8 @@ static void update_domain(struct protection_domain *domain)
amd_iommu_update_and_flush_device_table(domain);
/* Flush domain TLB(s) and wait for completion */
- domain_flush_tlb_pde(domain);
- domain_flush_complete(domain);
+ amd_iommu_domain_flush_tlb_pde(domain);
+ amd_iommu_domain_flush_complete(domain);
}
int __init amd_iommu_init_api(void)
@@ -2675,8 +2674,8 @@ static void amd_iommu_flush_iotlb_all(struct iommu_domain *domain)
unsigned long flags;
spin_lock_irqsave(&dom->lock, flags);
- domain_flush_tlb_pde(dom);
- domain_flush_complete(dom);
+ amd_iommu_domain_flush_tlb_pde(dom);
+ amd_iommu_domain_flush_complete(dom);
spin_unlock_irqrestore(&dom->lock, flags);
}
@@ -2766,7 +2765,7 @@ void amd_iommu_domain_direct_map(struct iommu_domain *dom)
amd_iommu_domain_clr_pt_root(domain);
/* Make changes visible to IOMMUs */
- update_domain(domain);
+ amd_iommu_domain_update(domain);
/* Page-table is not visible to IOMMU anymore, so free it */
free_pagetable(&pgtable);
@@ -2810,7 +2809,7 @@ int amd_iommu_domain_enable_v2(struct iommu_domain *dom, int pasids)
domain->glx = levels;
domain->flags |= PD_IOMMUV2_MASK;
- update_domain(domain);
+ amd_iommu_domain_update(domain);
ret = 0;
@@ -2847,7 +2846,7 @@ static int __flush_pasid(struct protection_domain *domain, int pasid,
}
/* Wait until IOMMU TLB flushes are complete */
- domain_flush_complete(domain);
+ amd_iommu_domain_flush_complete(domain);
/* Now flush device TLBs */
list_for_each_entry(dev_data, &domain->dev_list, list) {
@@ -2873,7 +2872,7 @@ static int __flush_pasid(struct protection_domain *domain, int pasid,
}
/* Wait until all device TLBs are flushed */
- domain_flush_complete(domain);
+ amd_iommu_domain_flush_complete(domain);
ret = 0;
--
2.17.1
^ permalink raw reply related [flat|nested] 20+ messages in thread
* [PATCH v3 06/14] iommu/amd: Move IO page table related functions
2020-10-04 1:45 [PATCH v3 00/14] iommu/amd: Add Generic IO Page Table Framework Support Suravee Suthikulpanit
` (4 preceding siblings ...)
2020-10-04 1:45 ` [PATCH v3 05/14] iommu/amd: Declare functions as extern Suravee Suthikulpanit
@ 2020-10-04 1:45 ` Suravee Suthikulpanit
2020-10-04 1:45 ` [PATCH v3 07/14] iommu/amd: Restructure code for freeing page table Suravee Suthikulpanit
` (8 subsequent siblings)
14 siblings, 0 replies; 20+ messages in thread
From: Suravee Suthikulpanit @ 2020-10-04 1:45 UTC (permalink / raw)
To: linux-kernel, iommu; +Cc: joro, robin.murphy, Suravee Suthikulpanit
Preparing to migrate to use IO page table framework.
There is no functional change.
Signed-off-by: Suravee Suthikulpanit <suravee.suthikulpanit@amd.com>
---
drivers/iommu/amd/amd_iommu.h | 18 ++
drivers/iommu/amd/io_pgtable.c | 473 ++++++++++++++++++++++++++++++++
drivers/iommu/amd/iommu.c | 476 +--------------------------------
3 files changed, 493 insertions(+), 474 deletions(-)
diff --git a/drivers/iommu/amd/amd_iommu.h b/drivers/iommu/amd/amd_iommu.h
index 8b7be9171030..ee7ff4d827e1 100644
--- a/drivers/iommu/amd/amd_iommu.h
+++ b/drivers/iommu/amd/amd_iommu.h
@@ -122,4 +122,22 @@ void amd_iommu_apply_ivrs_quirks(void);
static inline void amd_iommu_apply_ivrs_quirks(void) { }
#endif
+/* TODO: These are temporary and will be removed once fully transition */
+extern void free_pagetable(struct domain_pgtable *pgtable);
+extern int iommu_map_page(struct protection_domain *dom,
+ unsigned long bus_addr,
+ unsigned long phys_addr,
+ unsigned long page_size,
+ int prot,
+ gfp_t gfp);
+extern unsigned long iommu_unmap_page(struct protection_domain *dom,
+ unsigned long bus_addr,
+ unsigned long page_size);
+extern u64 *fetch_pte(struct protection_domain *domain,
+ unsigned long address,
+ unsigned long *page_size);
+extern void amd_iommu_domain_get_pgtable(struct protection_domain *domain,
+ struct domain_pgtable *pgtable);
+extern void amd_iommu_domain_set_pgtable(struct protection_domain *domain,
+ u64 *root, int mode);
#endif
diff --git a/drivers/iommu/amd/io_pgtable.c b/drivers/iommu/amd/io_pgtable.c
index 6b2de9e467d9..c11355afe624 100644
--- a/drivers/iommu/amd/io_pgtable.c
+++ b/drivers/iommu/amd/io_pgtable.c
@@ -23,6 +23,479 @@
#include "amd_iommu_types.h"
#include "amd_iommu.h"
+/*
+ * Helper function to get the first pte of a large mapping
+ */
+static u64 *first_pte_l7(u64 *pte, unsigned long *page_size,
+ unsigned long *count)
+{
+ unsigned long pte_mask, pg_size, cnt;
+ u64 *fpte;
+
+ pg_size = PTE_PAGE_SIZE(*pte);
+ cnt = PAGE_SIZE_PTE_COUNT(pg_size);
+ pte_mask = ~((cnt << 3) - 1);
+ fpte = (u64 *)(((unsigned long)pte) & pte_mask);
+
+ if (page_size)
+ *page_size = pg_size;
+
+ if (count)
+ *count = cnt;
+
+ return fpte;
+}
+
+/****************************************************************************
+ *
+ * The functions below are used the create the page table mappings for
+ * unity mapped regions.
+ *
+ ****************************************************************************/
+
+static void free_page_list(struct page *freelist)
+{
+ while (freelist != NULL) {
+ unsigned long p = (unsigned long)page_address(freelist);
+
+ freelist = freelist->freelist;
+ free_page(p);
+ }
+}
+
+static struct page *free_pt_page(unsigned long pt, struct page *freelist)
+{
+ struct page *p = virt_to_page((void *)pt);
+
+ p->freelist = freelist;
+
+ return p;
+}
+
+#define DEFINE_FREE_PT_FN(LVL, FN) \
+static struct page *free_pt_##LVL (unsigned long __pt, struct page *freelist) \
+{ \
+ unsigned long p; \
+ u64 *pt; \
+ int i; \
+ \
+ pt = (u64 *)__pt; \
+ \
+ for (i = 0; i < 512; ++i) { \
+ /* PTE present? */ \
+ if (!IOMMU_PTE_PRESENT(pt[i])) \
+ continue; \
+ \
+ /* Large PTE? */ \
+ if (PM_PTE_LEVEL(pt[i]) == 0 || \
+ PM_PTE_LEVEL(pt[i]) == 7) \
+ continue; \
+ \
+ p = (unsigned long)IOMMU_PTE_PAGE(pt[i]); \
+ freelist = FN(p, freelist); \
+ } \
+ \
+ return free_pt_page((unsigned long)pt, freelist); \
+}
+
+DEFINE_FREE_PT_FN(l2, free_pt_page)
+DEFINE_FREE_PT_FN(l3, free_pt_l2)
+DEFINE_FREE_PT_FN(l4, free_pt_l3)
+DEFINE_FREE_PT_FN(l5, free_pt_l4)
+DEFINE_FREE_PT_FN(l6, free_pt_l5)
+
+static struct page *free_sub_pt(unsigned long root, int mode,
+ struct page *freelist)
+{
+ switch (mode) {
+ case PAGE_MODE_NONE:
+ case PAGE_MODE_7_LEVEL:
+ break;
+ case PAGE_MODE_1_LEVEL:
+ freelist = free_pt_page(root, freelist);
+ break;
+ case PAGE_MODE_2_LEVEL:
+ freelist = free_pt_l2(root, freelist);
+ break;
+ case PAGE_MODE_3_LEVEL:
+ freelist = free_pt_l3(root, freelist);
+ break;
+ case PAGE_MODE_4_LEVEL:
+ freelist = free_pt_l4(root, freelist);
+ break;
+ case PAGE_MODE_5_LEVEL:
+ freelist = free_pt_l5(root, freelist);
+ break;
+ case PAGE_MODE_6_LEVEL:
+ freelist = free_pt_l6(root, freelist);
+ break;
+ default:
+ BUG();
+ }
+
+ return freelist;
+}
+
+void free_pagetable(struct domain_pgtable *pgtable)
+{
+ struct page *freelist = NULL;
+ unsigned long root;
+
+ if (pgtable->mode == PAGE_MODE_NONE)
+ return;
+
+ BUG_ON(pgtable->mode < PAGE_MODE_NONE ||
+ pgtable->mode > PAGE_MODE_6_LEVEL);
+
+ root = (unsigned long)pgtable->root;
+ freelist = free_sub_pt(root, pgtable->mode, freelist);
+
+ free_page_list(freelist);
+}
+
+void amd_iommu_domain_set_pgtable(struct protection_domain *domain,
+ u64 *root, int mode)
+{
+ u64 pt_root;
+
+ /* lowest 3 bits encode pgtable mode */
+ pt_root = mode & 7;
+ pt_root |= (u64)root;
+
+ amd_iommu_domain_set_pt_root(domain, pt_root);
+}
+
+/*
+ * This function is used to add another level to an IO page table. Adding
+ * another level increases the size of the address space by 9 bits to a size up
+ * to 64 bits.
+ */
+static bool increase_address_space(struct protection_domain *domain,
+ unsigned long address,
+ gfp_t gfp)
+{
+ struct domain_pgtable pgtable;
+ unsigned long flags;
+ bool ret = true;
+ u64 *pte;
+
+ spin_lock_irqsave(&domain->lock, flags);
+
+ amd_iommu_domain_get_pgtable(domain, &pgtable);
+
+ if (address <= PM_LEVEL_SIZE(pgtable.mode))
+ goto out;
+
+ ret = false;
+ if (WARN_ON_ONCE(pgtable.mode == PAGE_MODE_6_LEVEL))
+ goto out;
+
+ pte = (void *)get_zeroed_page(gfp);
+ if (!pte)
+ goto out;
+
+ *pte = PM_LEVEL_PDE(pgtable.mode, iommu_virt_to_phys(pgtable.root));
+
+ pgtable.root = pte;
+ pgtable.mode += 1;
+ amd_iommu_update_and_flush_device_table(domain);
+ amd_iommu_domain_flush_complete(domain);
+
+ /*
+ * Device Table needs to be updated and flushed before the new root can
+ * be published.
+ */
+ amd_iommu_domain_set_pgtable(domain, pte, pgtable.mode);
+
+ ret = true;
+
+out:
+ spin_unlock_irqrestore(&domain->lock, flags);
+
+ return ret;
+}
+
+static u64 *alloc_pte(struct protection_domain *domain,
+ unsigned long address,
+ unsigned long page_size,
+ u64 **pte_page,
+ gfp_t gfp,
+ bool *updated)
+{
+ struct domain_pgtable pgtable;
+ int level, end_lvl;
+ u64 *pte, *page;
+
+ BUG_ON(!is_power_of_2(page_size));
+
+ amd_iommu_domain_get_pgtable(domain, &pgtable);
+
+ while (address > PM_LEVEL_SIZE(pgtable.mode)) {
+ /*
+ * Return an error if there is no memory to update the
+ * page-table.
+ */
+ if (!increase_address_space(domain, address, gfp))
+ return NULL;
+
+ /* Read new values to check if update was successful */
+ amd_iommu_domain_get_pgtable(domain, &pgtable);
+ }
+
+
+ level = pgtable.mode - 1;
+ pte = &pgtable.root[PM_LEVEL_INDEX(level, address)];
+ address = PAGE_SIZE_ALIGN(address, page_size);
+ end_lvl = PAGE_SIZE_LEVEL(page_size);
+
+ while (level > end_lvl) {
+ u64 __pte, __npte;
+ int pte_level;
+
+ __pte = *pte;
+ pte_level = PM_PTE_LEVEL(__pte);
+
+ /*
+ * If we replace a series of large PTEs, we need
+ * to tear down all of them.
+ */
+ if (IOMMU_PTE_PRESENT(__pte) &&
+ pte_level == PAGE_MODE_7_LEVEL) {
+ unsigned long count, i;
+ u64 *lpte;
+
+ lpte = first_pte_l7(pte, NULL, &count);
+
+ /*
+ * Unmap the replicated PTEs that still match the
+ * original large mapping
+ */
+ for (i = 0; i < count; ++i)
+ cmpxchg64(&lpte[i], __pte, 0ULL);
+
+ *updated = true;
+ continue;
+ }
+
+ if (!IOMMU_PTE_PRESENT(__pte) ||
+ pte_level == PAGE_MODE_NONE) {
+ page = (u64 *)get_zeroed_page(gfp);
+
+ if (!page)
+ return NULL;
+
+ __npte = PM_LEVEL_PDE(level, iommu_virt_to_phys(page));
+
+ /* pte could have been changed somewhere. */
+ if (cmpxchg64(pte, __pte, __npte) != __pte)
+ free_page((unsigned long)page);
+ else if (IOMMU_PTE_PRESENT(__pte))
+ *updated = true;
+
+ continue;
+ }
+
+ /* No level skipping support yet */
+ if (pte_level != level)
+ return NULL;
+
+ level -= 1;
+
+ pte = IOMMU_PTE_PAGE(__pte);
+
+ if (pte_page && level == end_lvl)
+ *pte_page = pte;
+
+ pte = &pte[PM_LEVEL_INDEX(level, address)];
+ }
+
+ return pte;
+}
+
+/*
+ * This function checks if there is a PTE for a given dma address. If
+ * there is one, it returns the pointer to it.
+ */
+u64 *fetch_pte(struct protection_domain *domain,
+ unsigned long address,
+ unsigned long *page_size)
+{
+ struct domain_pgtable pgtable;
+ int level;
+ u64 *pte;
+
+ *page_size = 0;
+
+ amd_iommu_domain_get_pgtable(domain, &pgtable);
+
+ if (address > PM_LEVEL_SIZE(pgtable.mode))
+ return NULL;
+
+ level = pgtable.mode - 1;
+ pte = &pgtable.root[PM_LEVEL_INDEX(level, address)];
+ *page_size = PTE_LEVEL_PAGE_SIZE(level);
+
+ while (level > 0) {
+
+ /* Not Present */
+ if (!IOMMU_PTE_PRESENT(*pte))
+ return NULL;
+
+ /* Large PTE */
+ if (PM_PTE_LEVEL(*pte) == 7 ||
+ PM_PTE_LEVEL(*pte) == 0)
+ break;
+
+ /* No level skipping support yet */
+ if (PM_PTE_LEVEL(*pte) != level)
+ return NULL;
+
+ level -= 1;
+
+ /* Walk to the next level */
+ pte = IOMMU_PTE_PAGE(*pte);
+ pte = &pte[PM_LEVEL_INDEX(level, address)];
+ *page_size = PTE_LEVEL_PAGE_SIZE(level);
+ }
+
+ /*
+ * If we have a series of large PTEs, make
+ * sure to return a pointer to the first one.
+ */
+ if (PM_PTE_LEVEL(*pte) == PAGE_MODE_7_LEVEL)
+ pte = first_pte_l7(pte, page_size, NULL);
+
+ return pte;
+}
+
+static struct page *free_clear_pte(u64 *pte, u64 pteval, struct page *freelist)
+{
+ unsigned long pt;
+ int mode;
+
+ while (cmpxchg64(pte, pteval, 0) != pteval) {
+ pr_warn("AMD-Vi: IOMMU pte changed since we read it\n");
+ pteval = *pte;
+ }
+
+ if (!IOMMU_PTE_PRESENT(pteval))
+ return freelist;
+
+ pt = (unsigned long)IOMMU_PTE_PAGE(pteval);
+ mode = IOMMU_PTE_MODE(pteval);
+
+ return free_sub_pt(pt, mode, freelist);
+}
+
+/*
+ * Generic mapping functions. It maps a physical address into a DMA
+ * address space. It allocates the page table pages if necessary.
+ * In the future it can be extended to a generic mapping function
+ * supporting all features of AMD IOMMU page tables like level skipping
+ * and full 64 bit address spaces.
+ */
+int iommu_map_page(struct protection_domain *dom,
+ unsigned long bus_addr,
+ unsigned long phys_addr,
+ unsigned long page_size,
+ int prot,
+ gfp_t gfp)
+{
+ struct page *freelist = NULL;
+ bool updated = false;
+ u64 __pte, *pte;
+ int ret, i, count;
+
+ BUG_ON(!IS_ALIGNED(bus_addr, page_size));
+ BUG_ON(!IS_ALIGNED(phys_addr, page_size));
+
+ ret = -EINVAL;
+ if (!(prot & IOMMU_PROT_MASK))
+ goto out;
+
+ count = PAGE_SIZE_PTE_COUNT(page_size);
+ pte = alloc_pte(dom, bus_addr, page_size, NULL, gfp, &updated);
+
+ ret = -ENOMEM;
+ if (!pte)
+ goto out;
+
+ for (i = 0; i < count; ++i)
+ freelist = free_clear_pte(&pte[i], pte[i], freelist);
+
+ if (freelist != NULL)
+ updated = true;
+
+ if (count > 1) {
+ __pte = PAGE_SIZE_PTE(__sme_set(phys_addr), page_size);
+ __pte |= PM_LEVEL_ENC(7) | IOMMU_PTE_PR | IOMMU_PTE_FC;
+ } else
+ __pte = __sme_set(phys_addr) | IOMMU_PTE_PR | IOMMU_PTE_FC;
+
+ if (prot & IOMMU_PROT_IR)
+ __pte |= IOMMU_PTE_IR;
+ if (prot & IOMMU_PROT_IW)
+ __pte |= IOMMU_PTE_IW;
+
+ for (i = 0; i < count; ++i)
+ pte[i] = __pte;
+
+ ret = 0;
+
+out:
+ if (updated) {
+ unsigned long flags;
+
+ spin_lock_irqsave(&dom->lock, flags);
+ /*
+ * Flush domain TLB(s) and wait for completion. Any Device-Table
+ * Updates and flushing already happened in
+ * increase_address_space().
+ */
+ amd_iommu_domain_flush_tlb_pde(dom);
+ amd_iommu_domain_flush_complete(dom);
+ spin_unlock_irqrestore(&dom->lock, flags);
+ }
+
+ /* Everything flushed out, free pages now */
+ free_page_list(freelist);
+
+ return ret;
+}
+
+unsigned long iommu_unmap_page(struct protection_domain *dom,
+ unsigned long bus_addr,
+ unsigned long page_size)
+{
+ unsigned long long unmapped;
+ unsigned long unmap_size;
+ u64 *pte;
+
+ BUG_ON(!is_power_of_2(page_size));
+
+ unmapped = 0;
+
+ while (unmapped < page_size) {
+
+ pte = fetch_pte(dom, bus_addr, &unmap_size);
+
+ if (pte) {
+ int i, count;
+
+ count = PAGE_SIZE_PTE_COUNT(unmap_size);
+ for (i = 0; i < count; i++)
+ pte[i] = 0ULL;
+ }
+
+ bus_addr = (bus_addr & ~(unmap_size - 1)) + unmap_size;
+ unmapped += unmap_size;
+ }
+
+ BUG_ON(unmapped && !is_power_of_2(unmapped));
+
+ return unmapped;
+}
+
/*
* ----------------------------------------------------
*/
diff --git a/drivers/iommu/amd/iommu.c b/drivers/iommu/amd/iommu.c
index f91f35edb7ba..4d65f64236b6 100644
--- a/drivers/iommu/amd/iommu.c
+++ b/drivers/iommu/amd/iommu.c
@@ -140,8 +140,8 @@ static struct protection_domain *to_pdomain(struct iommu_domain *dom)
return container_of(dom, struct protection_domain, domain);
}
-static void amd_iommu_domain_get_pgtable(struct protection_domain *domain,
- struct domain_pgtable *pgtable)
+void amd_iommu_domain_get_pgtable(struct protection_domain *domain,
+ struct domain_pgtable *pgtable)
{
u64 pt_root = atomic64_read(&domain->iop.pt_root);
@@ -149,18 +149,6 @@ static void amd_iommu_domain_get_pgtable(struct protection_domain *domain,
pgtable->mode = pt_root & 7; /* lowest 3 bits encode pgtable mode */
}
-static void amd_iommu_domain_set_pgtable(struct protection_domain *domain,
- u64 *root, int mode)
-{
- u64 pt_root;
-
- /* lowest 3 bits encode pgtable mode */
- pt_root = mode & 7;
- pt_root |= (u64)root;
-
- amd_iommu_domain_set_pt_root(domain, pt_root);
-}
-
static struct iommu_dev_data *alloc_dev_data(u16 devid)
{
struct iommu_dev_data *dev_data;
@@ -416,29 +404,6 @@ static void amd_iommu_uninit_device(struct device *dev)
*/
}
-/*
- * Helper function to get the first pte of a large mapping
- */
-static u64 *first_pte_l7(u64 *pte, unsigned long *page_size,
- unsigned long *count)
-{
- unsigned long pte_mask, pg_size, cnt;
- u64 *fpte;
-
- pg_size = PTE_PAGE_SIZE(*pte);
- cnt = PAGE_SIZE_PTE_COUNT(pg_size);
- pte_mask = ~((cnt << 3) - 1);
- fpte = (u64 *)(((unsigned long)pte) & pte_mask);
-
- if (page_size)
- *page_size = pg_size;
-
- if (count)
- *count = cnt;
-
- return fpte;
-}
-
/****************************************************************************
*
* Interrupt handling functions
@@ -1340,443 +1305,6 @@ static void domain_flush_devices(struct protection_domain *domain)
device_flush_dte(dev_data);
}
-/****************************************************************************
- *
- * The functions below are used the create the page table mappings for
- * unity mapped regions.
- *
- ****************************************************************************/
-
-static void free_page_list(struct page *freelist)
-{
- while (freelist != NULL) {
- unsigned long p = (unsigned long)page_address(freelist);
- freelist = freelist->freelist;
- free_page(p);
- }
-}
-
-static struct page *free_pt_page(unsigned long pt, struct page *freelist)
-{
- struct page *p = virt_to_page((void *)pt);
-
- p->freelist = freelist;
-
- return p;
-}
-
-#define DEFINE_FREE_PT_FN(LVL, FN) \
-static struct page *free_pt_##LVL (unsigned long __pt, struct page *freelist) \
-{ \
- unsigned long p; \
- u64 *pt; \
- int i; \
- \
- pt = (u64 *)__pt; \
- \
- for (i = 0; i < 512; ++i) { \
- /* PTE present? */ \
- if (!IOMMU_PTE_PRESENT(pt[i])) \
- continue; \
- \
- /* Large PTE? */ \
- if (PM_PTE_LEVEL(pt[i]) == 0 || \
- PM_PTE_LEVEL(pt[i]) == 7) \
- continue; \
- \
- p = (unsigned long)IOMMU_PTE_PAGE(pt[i]); \
- freelist = FN(p, freelist); \
- } \
- \
- return free_pt_page((unsigned long)pt, freelist); \
-}
-
-DEFINE_FREE_PT_FN(l2, free_pt_page)
-DEFINE_FREE_PT_FN(l3, free_pt_l2)
-DEFINE_FREE_PT_FN(l4, free_pt_l3)
-DEFINE_FREE_PT_FN(l5, free_pt_l4)
-DEFINE_FREE_PT_FN(l6, free_pt_l5)
-
-static struct page *free_sub_pt(unsigned long root, int mode,
- struct page *freelist)
-{
- switch (mode) {
- case PAGE_MODE_NONE:
- case PAGE_MODE_7_LEVEL:
- break;
- case PAGE_MODE_1_LEVEL:
- freelist = free_pt_page(root, freelist);
- break;
- case PAGE_MODE_2_LEVEL:
- freelist = free_pt_l2(root, freelist);
- break;
- case PAGE_MODE_3_LEVEL:
- freelist = free_pt_l3(root, freelist);
- break;
- case PAGE_MODE_4_LEVEL:
- freelist = free_pt_l4(root, freelist);
- break;
- case PAGE_MODE_5_LEVEL:
- freelist = free_pt_l5(root, freelist);
- break;
- case PAGE_MODE_6_LEVEL:
- freelist = free_pt_l6(root, freelist);
- break;
- default:
- BUG();
- }
-
- return freelist;
-}
-
-static void free_pagetable(struct domain_pgtable *pgtable)
-{
- struct page *freelist = NULL;
- unsigned long root;
-
- if (pgtable->mode == PAGE_MODE_NONE)
- return;
-
- BUG_ON(pgtable->mode < PAGE_MODE_NONE ||
- pgtable->mode > PAGE_MODE_6_LEVEL);
-
- root = (unsigned long)pgtable->root;
- freelist = free_sub_pt(root, pgtable->mode, freelist);
-
- free_page_list(freelist);
-}
-
-/*
- * This function is used to add another level to an IO page table. Adding
- * another level increases the size of the address space by 9 bits to a size up
- * to 64 bits.
- */
-static bool increase_address_space(struct protection_domain *domain,
- unsigned long address,
- gfp_t gfp)
-{
- struct domain_pgtable pgtable;
- unsigned long flags;
- bool ret = true;
- u64 *pte;
-
- spin_lock_irqsave(&domain->lock, flags);
-
- amd_iommu_domain_get_pgtable(domain, &pgtable);
-
- if (address <= PM_LEVEL_SIZE(pgtable.mode))
- goto out;
-
- ret = false;
- if (WARN_ON_ONCE(pgtable.mode == PAGE_MODE_6_LEVEL))
- goto out;
-
- pte = (void *)get_zeroed_page(gfp);
- if (!pte)
- goto out;
-
- *pte = PM_LEVEL_PDE(pgtable.mode, iommu_virt_to_phys(pgtable.root));
-
- pgtable.root = pte;
- pgtable.mode += 1;
- amd_iommu_update_and_flush_device_table(domain);
- amd_iommu_domain_flush_complete(domain);
-
- /*
- * Device Table needs to be updated and flushed before the new root can
- * be published.
- */
- amd_iommu_domain_set_pgtable(domain, pte, pgtable.mode);
-
- ret = true;
-
-out:
- spin_unlock_irqrestore(&domain->lock, flags);
-
- return ret;
-}
-
-static u64 *alloc_pte(struct protection_domain *domain,
- unsigned long address,
- unsigned long page_size,
- u64 **pte_page,
- gfp_t gfp,
- bool *updated)
-{
- struct domain_pgtable pgtable;
- int level, end_lvl;
- u64 *pte, *page;
-
- BUG_ON(!is_power_of_2(page_size));
-
- amd_iommu_domain_get_pgtable(domain, &pgtable);
-
- while (address > PM_LEVEL_SIZE(pgtable.mode)) {
- /*
- * Return an error if there is no memory to update the
- * page-table.
- */
- if (!increase_address_space(domain, address, gfp))
- return NULL;
-
- /* Read new values to check if update was successful */
- amd_iommu_domain_get_pgtable(domain, &pgtable);
- }
-
-
- level = pgtable.mode - 1;
- pte = &pgtable.root[PM_LEVEL_INDEX(level, address)];
- address = PAGE_SIZE_ALIGN(address, page_size);
- end_lvl = PAGE_SIZE_LEVEL(page_size);
-
- while (level > end_lvl) {
- u64 __pte, __npte;
- int pte_level;
-
- __pte = *pte;
- pte_level = PM_PTE_LEVEL(__pte);
-
- /*
- * If we replace a series of large PTEs, we need
- * to tear down all of them.
- */
- if (IOMMU_PTE_PRESENT(__pte) &&
- pte_level == PAGE_MODE_7_LEVEL) {
- unsigned long count, i;
- u64 *lpte;
-
- lpte = first_pte_l7(pte, NULL, &count);
-
- /*
- * Unmap the replicated PTEs that still match the
- * original large mapping
- */
- for (i = 0; i < count; ++i)
- cmpxchg64(&lpte[i], __pte, 0ULL);
-
- *updated = true;
- continue;
- }
-
- if (!IOMMU_PTE_PRESENT(__pte) ||
- pte_level == PAGE_MODE_NONE) {
- page = (u64 *)get_zeroed_page(gfp);
-
- if (!page)
- return NULL;
-
- __npte = PM_LEVEL_PDE(level, iommu_virt_to_phys(page));
-
- /* pte could have been changed somewhere. */
- if (cmpxchg64(pte, __pte, __npte) != __pte)
- free_page((unsigned long)page);
- else if (IOMMU_PTE_PRESENT(__pte))
- *updated = true;
-
- continue;
- }
-
- /* No level skipping support yet */
- if (pte_level != level)
- return NULL;
-
- level -= 1;
-
- pte = IOMMU_PTE_PAGE(__pte);
-
- if (pte_page && level == end_lvl)
- *pte_page = pte;
-
- pte = &pte[PM_LEVEL_INDEX(level, address)];
- }
-
- return pte;
-}
-
-/*
- * This function checks if there is a PTE for a given dma address. If
- * there is one, it returns the pointer to it.
- */
-static u64 *fetch_pte(struct protection_domain *domain,
- unsigned long address,
- unsigned long *page_size)
-{
- struct domain_pgtable pgtable;
- int level;
- u64 *pte;
-
- *page_size = 0;
-
- amd_iommu_domain_get_pgtable(domain, &pgtable);
-
- if (address > PM_LEVEL_SIZE(pgtable.mode))
- return NULL;
-
- level = pgtable.mode - 1;
- pte = &pgtable.root[PM_LEVEL_INDEX(level, address)];
- *page_size = PTE_LEVEL_PAGE_SIZE(level);
-
- while (level > 0) {
-
- /* Not Present */
- if (!IOMMU_PTE_PRESENT(*pte))
- return NULL;
-
- /* Large PTE */
- if (PM_PTE_LEVEL(*pte) == 7 ||
- PM_PTE_LEVEL(*pte) == 0)
- break;
-
- /* No level skipping support yet */
- if (PM_PTE_LEVEL(*pte) != level)
- return NULL;
-
- level -= 1;
-
- /* Walk to the next level */
- pte = IOMMU_PTE_PAGE(*pte);
- pte = &pte[PM_LEVEL_INDEX(level, address)];
- *page_size = PTE_LEVEL_PAGE_SIZE(level);
- }
-
- /*
- * If we have a series of large PTEs, make
- * sure to return a pointer to the first one.
- */
- if (PM_PTE_LEVEL(*pte) == PAGE_MODE_7_LEVEL)
- pte = first_pte_l7(pte, page_size, NULL);
-
- return pte;
-}
-
-static struct page *free_clear_pte(u64 *pte, u64 pteval, struct page *freelist)
-{
- unsigned long pt;
- int mode;
-
- while (cmpxchg64(pte, pteval, 0) != pteval) {
- pr_warn("AMD-Vi: IOMMU pte changed since we read it\n");
- pteval = *pte;
- }
-
- if (!IOMMU_PTE_PRESENT(pteval))
- return freelist;
-
- pt = (unsigned long)IOMMU_PTE_PAGE(pteval);
- mode = IOMMU_PTE_MODE(pteval);
-
- return free_sub_pt(pt, mode, freelist);
-}
-
-/*
- * Generic mapping functions. It maps a physical address into a DMA
- * address space. It allocates the page table pages if necessary.
- * In the future it can be extended to a generic mapping function
- * supporting all features of AMD IOMMU page tables like level skipping
- * and full 64 bit address spaces.
- */
-static int iommu_map_page(struct protection_domain *dom,
- unsigned long bus_addr,
- unsigned long phys_addr,
- unsigned long page_size,
- int prot,
- gfp_t gfp)
-{
- struct page *freelist = NULL;
- bool updated = false;
- u64 __pte, *pte;
- int ret, i, count;
-
- BUG_ON(!IS_ALIGNED(bus_addr, page_size));
- BUG_ON(!IS_ALIGNED(phys_addr, page_size));
-
- ret = -EINVAL;
- if (!(prot & IOMMU_PROT_MASK))
- goto out;
-
- count = PAGE_SIZE_PTE_COUNT(page_size);
- pte = alloc_pte(dom, bus_addr, page_size, NULL, gfp, &updated);
-
- ret = -ENOMEM;
- if (!pte)
- goto out;
-
- for (i = 0; i < count; ++i)
- freelist = free_clear_pte(&pte[i], pte[i], freelist);
-
- if (freelist != NULL)
- updated = true;
-
- if (count > 1) {
- __pte = PAGE_SIZE_PTE(__sme_set(phys_addr), page_size);
- __pte |= PM_LEVEL_ENC(7) | IOMMU_PTE_PR | IOMMU_PTE_FC;
- } else
- __pte = __sme_set(phys_addr) | IOMMU_PTE_PR | IOMMU_PTE_FC;
-
- if (prot & IOMMU_PROT_IR)
- __pte |= IOMMU_PTE_IR;
- if (prot & IOMMU_PROT_IW)
- __pte |= IOMMU_PTE_IW;
-
- for (i = 0; i < count; ++i)
- pte[i] = __pte;
-
- ret = 0;
-
-out:
- if (updated) {
- unsigned long flags;
-
- spin_lock_irqsave(&dom->lock, flags);
- /*
- * Flush domain TLB(s) and wait for completion. Any Device-Table
- * Updates and flushing already happened in
- * increase_address_space().
- */
- amd_iommu_domain_flush_tlb_pde(dom);
- amd_iommu_domain_flush_complete(dom);
- spin_unlock_irqrestore(&dom->lock, flags);
- }
-
- /* Everything flushed out, free pages now */
- free_page_list(freelist);
-
- return ret;
-}
-
-static unsigned long iommu_unmap_page(struct protection_domain *dom,
- unsigned long bus_addr,
- unsigned long page_size)
-{
- unsigned long long unmapped;
- unsigned long unmap_size;
- u64 *pte;
-
- BUG_ON(!is_power_of_2(page_size));
-
- unmapped = 0;
-
- while (unmapped < page_size) {
-
- pte = fetch_pte(dom, bus_addr, &unmap_size);
-
- if (pte) {
- int i, count;
-
- count = PAGE_SIZE_PTE_COUNT(unmap_size);
- for (i = 0; i < count; i++)
- pte[i] = 0ULL;
- }
-
- bus_addr = (bus_addr & ~(unmap_size - 1)) + unmap_size;
- unmapped += unmap_size;
- }
-
- BUG_ON(unmapped && !is_power_of_2(unmapped));
-
- return unmapped;
-}
-
/****************************************************************************
*
* The next functions belong to the domain allocation. A domain is
--
2.17.1
^ permalink raw reply related [flat|nested] 20+ messages in thread
* [PATCH v3 07/14] iommu/amd: Restructure code for freeing page table
2020-10-04 1:45 [PATCH v3 00/14] iommu/amd: Add Generic IO Page Table Framework Support Suravee Suthikulpanit
` (5 preceding siblings ...)
2020-10-04 1:45 ` [PATCH v3 06/14] iommu/amd: Move IO page table related functions Suravee Suthikulpanit
@ 2020-10-04 1:45 ` Suravee Suthikulpanit
2020-10-04 1:45 ` [PATCH v3 08/14] iommu/amd: Remove amd_iommu_domain_get_pgtable Suravee Suthikulpanit
` (7 subsequent siblings)
14 siblings, 0 replies; 20+ messages in thread
From: Suravee Suthikulpanit @ 2020-10-04 1:45 UTC (permalink / raw)
To: linux-kernel, iommu; +Cc: joro, robin.murphy, Suravee Suthikulpanit
Introduce amd_iommu_free_pgtable helper function, which consolidates
logic for freeing page table.
Signed-off-by: Suravee Suthikulpanit <suravee.suthikulpanit@amd.com>
---
drivers/iommu/amd/amd_iommu.h | 2 +-
drivers/iommu/amd/io_pgtable.c | 12 +++++++++++-
drivers/iommu/amd/iommu.c | 19 ++-----------------
3 files changed, 14 insertions(+), 19 deletions(-)
diff --git a/drivers/iommu/amd/amd_iommu.h b/drivers/iommu/amd/amd_iommu.h
index ee7ff4d827e1..8dff7d85be79 100644
--- a/drivers/iommu/amd/amd_iommu.h
+++ b/drivers/iommu/amd/amd_iommu.h
@@ -123,7 +123,6 @@ static inline void amd_iommu_apply_ivrs_quirks(void) { }
#endif
/* TODO: These are temporary and will be removed once fully transition */
-extern void free_pagetable(struct domain_pgtable *pgtable);
extern int iommu_map_page(struct protection_domain *dom,
unsigned long bus_addr,
unsigned long phys_addr,
@@ -140,4 +139,5 @@ extern void amd_iommu_domain_get_pgtable(struct protection_domain *domain,
struct domain_pgtable *pgtable);
extern void amd_iommu_domain_set_pgtable(struct protection_domain *domain,
u64 *root, int mode);
+extern void amd_iommu_free_pgtable(struct amd_io_pgtable *pgtable);
#endif
diff --git a/drivers/iommu/amd/io_pgtable.c b/drivers/iommu/amd/io_pgtable.c
index c11355afe624..23e82da2dea8 100644
--- a/drivers/iommu/amd/io_pgtable.c
+++ b/drivers/iommu/amd/io_pgtable.c
@@ -136,14 +136,24 @@ static struct page *free_sub_pt(unsigned long root, int mode,
return freelist;
}
-void free_pagetable(struct domain_pgtable *pgtable)
+void amd_iommu_free_pgtable(struct amd_io_pgtable *pgtable)
{
+ struct protection_domain *dom;
struct page *freelist = NULL;
unsigned long root;
if (pgtable->mode == PAGE_MODE_NONE)
return;
+ dom = container_of(pgtable, struct protection_domain, iop);
+
+ /* Update data structure */
+ amd_iommu_domain_clr_pt_root(dom);
+
+ /* Make changes visible to IOMMUs */
+ amd_iommu_domain_update(dom);
+
+ /* Page-table is not visible to IOMMU anymore, so free it */
BUG_ON(pgtable->mode < PAGE_MODE_NONE ||
pgtable->mode > PAGE_MODE_6_LEVEL);
diff --git a/drivers/iommu/amd/iommu.c b/drivers/iommu/amd/iommu.c
index 4d65f64236b6..cbbea7b952fb 100644
--- a/drivers/iommu/amd/iommu.c
+++ b/drivers/iommu/amd/iommu.c
@@ -1882,17 +1882,13 @@ static void cleanup_domain(struct protection_domain *domain)
static void protection_domain_free(struct protection_domain *domain)
{
- struct domain_pgtable pgtable;
-
if (!domain)
return;
if (domain->id)
domain_id_free(domain->id);
- amd_iommu_domain_get_pgtable(domain, &pgtable);
- amd_iommu_domain_clr_pt_root(domain);
- free_pagetable(&pgtable);
+ amd_iommu_free_pgtable(&domain->iop);
kfree(domain);
}
@@ -2281,22 +2277,11 @@ EXPORT_SYMBOL(amd_iommu_unregister_ppr_notifier);
void amd_iommu_domain_direct_map(struct iommu_domain *dom)
{
struct protection_domain *domain = to_pdomain(dom);
- struct domain_pgtable pgtable;
unsigned long flags;
spin_lock_irqsave(&domain->lock, flags);
- /* First save pgtable configuration*/
- amd_iommu_domain_get_pgtable(domain, &pgtable);
-
- /* Remove page-table from domain */
- amd_iommu_domain_clr_pt_root(domain);
-
- /* Make changes visible to IOMMUs */
- amd_iommu_domain_update(domain);
-
- /* Page-table is not visible to IOMMU anymore, so free it */
- free_pagetable(&pgtable);
+ amd_iommu_free_pgtable(&domain->iop);
spin_unlock_irqrestore(&domain->lock, flags);
}
--
2.17.1
^ permalink raw reply related [flat|nested] 20+ messages in thread
* [PATCH v3 08/14] iommu/amd: Remove amd_iommu_domain_get_pgtable
2020-10-04 1:45 [PATCH v3 00/14] iommu/amd: Add Generic IO Page Table Framework Support Suravee Suthikulpanit
` (6 preceding siblings ...)
2020-10-04 1:45 ` [PATCH v3 07/14] iommu/amd: Restructure code for freeing page table Suravee Suthikulpanit
@ 2020-10-04 1:45 ` Suravee Suthikulpanit
2020-10-04 1:45 ` [PATCH v3 09/14] iommu/amd: Rename variables to be consistent with struct io_pgtable_ops Suravee Suthikulpanit
` (6 subsequent siblings)
14 siblings, 0 replies; 20+ messages in thread
From: Suravee Suthikulpanit @ 2020-10-04 1:45 UTC (permalink / raw)
To: linux-kernel, iommu; +Cc: joro, robin.murphy, Suravee Suthikulpanit
Since the IO page table root and mode parameters have been moved into
the struct amd_io_pg, the function is no longer needed. Therefore,
remove it along with the struct domain_pgtable.
Signed-off-by: Suravee Suthikulpanit <suravee.suthikulpanit@amd.com>
---
drivers/iommu/amd/amd_iommu.h | 4 ++--
drivers/iommu/amd/amd_iommu_types.h | 6 -----
drivers/iommu/amd/io_pgtable.c | 36 ++++++++++-------------------
drivers/iommu/amd/iommu.c | 34 ++++-----------------------
4 files changed, 19 insertions(+), 61 deletions(-)
diff --git a/drivers/iommu/amd/amd_iommu.h b/drivers/iommu/amd/amd_iommu.h
index 8dff7d85be79..2059e64fdc53 100644
--- a/drivers/iommu/amd/amd_iommu.h
+++ b/drivers/iommu/amd/amd_iommu.h
@@ -101,6 +101,8 @@ static inline
void amd_iommu_domain_set_pt_root(struct protection_domain *domain, u64 root)
{
atomic64_set(&domain->iop.pt_root, root);
+ domain->iop.root = (u64 *)(root & PAGE_MASK);
+ domain->iop.mode = root & 7; /* lowest 3 bits encode pgtable mode */
}
static inline
@@ -135,8 +137,6 @@ extern unsigned long iommu_unmap_page(struct protection_domain *dom,
extern u64 *fetch_pte(struct protection_domain *domain,
unsigned long address,
unsigned long *page_size);
-extern void amd_iommu_domain_get_pgtable(struct protection_domain *domain,
- struct domain_pgtable *pgtable);
extern void amd_iommu_domain_set_pgtable(struct protection_domain *domain,
u64 *root, int mode);
extern void amd_iommu_free_pgtable(struct amd_io_pgtable *pgtable);
diff --git a/drivers/iommu/amd/amd_iommu_types.h b/drivers/iommu/amd/amd_iommu_types.h
index 80b5c34357ed..de3fe9433080 100644
--- a/drivers/iommu/amd/amd_iommu_types.h
+++ b/drivers/iommu/amd/amd_iommu_types.h
@@ -514,12 +514,6 @@ struct protection_domain {
unsigned dev_iommu[MAX_IOMMUS]; /* per-IOMMU reference count */
};
-/* For decocded pt_root */
-struct domain_pgtable {
- int mode;
- u64 *root;
-};
-
/*
* Structure where we save information about one hardware AMD IOMMU in the
* system.
diff --git a/drivers/iommu/amd/io_pgtable.c b/drivers/iommu/amd/io_pgtable.c
index 23e82da2dea8..6c063d2c8bf0 100644
--- a/drivers/iommu/amd/io_pgtable.c
+++ b/drivers/iommu/amd/io_pgtable.c
@@ -184,30 +184,27 @@ static bool increase_address_space(struct protection_domain *domain,
unsigned long address,
gfp_t gfp)
{
- struct domain_pgtable pgtable;
unsigned long flags;
bool ret = true;
u64 *pte;
spin_lock_irqsave(&domain->lock, flags);
- amd_iommu_domain_get_pgtable(domain, &pgtable);
-
- if (address <= PM_LEVEL_SIZE(pgtable.mode))
+ if (address <= PM_LEVEL_SIZE(domain->iop.mode))
goto out;
ret = false;
- if (WARN_ON_ONCE(pgtable.mode == PAGE_MODE_6_LEVEL))
+ if (WARN_ON_ONCE(domain->iop.mode == PAGE_MODE_6_LEVEL))
goto out;
pte = (void *)get_zeroed_page(gfp);
if (!pte)
goto out;
- *pte = PM_LEVEL_PDE(pgtable.mode, iommu_virt_to_phys(pgtable.root));
+ *pte = PM_LEVEL_PDE(domain->iop.mode, iommu_virt_to_phys(domain->iop.root));
- pgtable.root = pte;
- pgtable.mode += 1;
+ domain->iop.root = pte;
+ domain->iop.mode += 1;
amd_iommu_update_and_flush_device_table(domain);
amd_iommu_domain_flush_complete(domain);
@@ -215,7 +212,7 @@ static bool increase_address_space(struct protection_domain *domain,
* Device Table needs to be updated and flushed before the new root can
* be published.
*/
- amd_iommu_domain_set_pgtable(domain, pte, pgtable.mode);
+ amd_iommu_domain_set_pgtable(domain, pte, domain->iop.mode);
ret = true;
@@ -232,29 +229,23 @@ static u64 *alloc_pte(struct protection_domain *domain,
gfp_t gfp,
bool *updated)
{
- struct domain_pgtable pgtable;
int level, end_lvl;
u64 *pte, *page;
BUG_ON(!is_power_of_2(page_size));
- amd_iommu_domain_get_pgtable(domain, &pgtable);
-
- while (address > PM_LEVEL_SIZE(pgtable.mode)) {
+ while (address > PM_LEVEL_SIZE(domain->iop.mode)) {
/*
* Return an error if there is no memory to update the
* page-table.
*/
if (!increase_address_space(domain, address, gfp))
return NULL;
-
- /* Read new values to check if update was successful */
- amd_iommu_domain_get_pgtable(domain, &pgtable);
}
- level = pgtable.mode - 1;
- pte = &pgtable.root[PM_LEVEL_INDEX(level, address)];
+ level = domain->iop.mode - 1;
+ pte = &domain->iop.root[PM_LEVEL_INDEX(level, address)];
address = PAGE_SIZE_ALIGN(address, page_size);
end_lvl = PAGE_SIZE_LEVEL(page_size);
@@ -330,19 +321,16 @@ u64 *fetch_pte(struct protection_domain *domain,
unsigned long address,
unsigned long *page_size)
{
- struct domain_pgtable pgtable;
int level;
u64 *pte;
*page_size = 0;
- amd_iommu_domain_get_pgtable(domain, &pgtable);
-
- if (address > PM_LEVEL_SIZE(pgtable.mode))
+ if (address > PM_LEVEL_SIZE(domain->iop.mode))
return NULL;
- level = pgtable.mode - 1;
- pte = &pgtable.root[PM_LEVEL_INDEX(level, address)];
+ level = domain->iop.mode - 1;
+ pte = &domain->iop.root[PM_LEVEL_INDEX(level, address)];
*page_size = PTE_LEVEL_PAGE_SIZE(level);
while (level > 0) {
diff --git a/drivers/iommu/amd/iommu.c b/drivers/iommu/amd/iommu.c
index cbbea7b952fb..3f6ede1e572c 100644
--- a/drivers/iommu/amd/iommu.c
+++ b/drivers/iommu/amd/iommu.c
@@ -140,15 +140,6 @@ static struct protection_domain *to_pdomain(struct iommu_domain *dom)
return container_of(dom, struct protection_domain, domain);
}
-void amd_iommu_domain_get_pgtable(struct protection_domain *domain,
- struct domain_pgtable *pgtable)
-{
- u64 pt_root = atomic64_read(&domain->iop.pt_root);
-
- pgtable->root = (u64 *)(pt_root & PAGE_MASK);
- pgtable->mode = pt_root & 7; /* lowest 3 bits encode pgtable mode */
-}
-
static struct iommu_dev_data *alloc_dev_data(u16 devid)
{
struct iommu_dev_data *dev_data;
@@ -1464,7 +1455,6 @@ static void clear_dte_entry(u16 devid)
static void do_attach(struct iommu_dev_data *dev_data,
struct protection_domain *domain)
{
- struct domain_pgtable pgtable;
struct amd_iommu *iommu;
bool ats;
@@ -1480,7 +1470,6 @@ static void do_attach(struct iommu_dev_data *dev_data,
domain->dev_cnt += 1;
/* Update device table */
- amd_iommu_domain_get_pgtable(domain, &pgtable);
set_dte_entry(dev_data->devid, domain,
ats, dev_data->iommu_v2);
clone_aliases(dev_data->pdev);
@@ -1806,10 +1795,7 @@ void amd_iommu_update_and_flush_device_table(struct protection_domain *domain)
void amd_iommu_domain_update(struct protection_domain *domain)
{
- struct domain_pgtable pgtable;
-
/* Update device table */
- amd_iommu_domain_get_pgtable(domain, &pgtable);
amd_iommu_update_and_flush_device_table(domain);
/* Flush domain TLB(s) and wait for completion */
@@ -2058,12 +2044,10 @@ static int amd_iommu_map(struct iommu_domain *dom, unsigned long iova,
gfp_t gfp)
{
struct protection_domain *domain = to_pdomain(dom);
- struct domain_pgtable pgtable;
int prot = 0;
int ret;
- amd_iommu_domain_get_pgtable(domain, &pgtable);
- if (pgtable.mode == PAGE_MODE_NONE)
+ if (domain->iop.mode == PAGE_MODE_NONE)
return -EINVAL;
if (iommu_prot & IOMMU_READ)
@@ -2083,10 +2067,8 @@ static size_t amd_iommu_unmap(struct iommu_domain *dom, unsigned long iova,
struct iommu_iotlb_gather *gather)
{
struct protection_domain *domain = to_pdomain(dom);
- struct domain_pgtable pgtable;
- amd_iommu_domain_get_pgtable(domain, &pgtable);
- if (pgtable.mode == PAGE_MODE_NONE)
+ if (domain->iop.mode == PAGE_MODE_NONE)
return 0;
return iommu_unmap_page(domain, iova, page_size);
@@ -2097,11 +2079,9 @@ static phys_addr_t amd_iommu_iova_to_phys(struct iommu_domain *dom,
{
struct protection_domain *domain = to_pdomain(dom);
unsigned long offset_mask, pte_pgsize;
- struct domain_pgtable pgtable;
u64 *pte, __pte;
- amd_iommu_domain_get_pgtable(domain, &pgtable);
- if (pgtable.mode == PAGE_MODE_NONE)
+ if (domain->iop.mode == PAGE_MODE_NONE)
return iova;
pte = fetch_pte(domain, iova, &pte_pgsize);
@@ -2470,11 +2450,9 @@ static u64 *__get_gcr3_pte(u64 *root, int level, int pasid, bool alloc)
static int __set_gcr3(struct protection_domain *domain, int pasid,
unsigned long cr3)
{
- struct domain_pgtable pgtable;
u64 *pte;
- amd_iommu_domain_get_pgtable(domain, &pgtable);
- if (pgtable.mode != PAGE_MODE_NONE)
+ if (domain->iop.mode != PAGE_MODE_NONE)
return -EINVAL;
pte = __get_gcr3_pte(domain->gcr3_tbl, domain->glx, pasid, true);
@@ -2488,11 +2466,9 @@ static int __set_gcr3(struct protection_domain *domain, int pasid,
static int __clear_gcr3(struct protection_domain *domain, int pasid)
{
- struct domain_pgtable pgtable;
u64 *pte;
- amd_iommu_domain_get_pgtable(domain, &pgtable);
- if (pgtable.mode != PAGE_MODE_NONE)
+ if (domain->iop.mode != PAGE_MODE_NONE)
return -EINVAL;
pte = __get_gcr3_pte(domain->gcr3_tbl, domain->glx, pasid, false);
--
2.17.1
^ permalink raw reply related [flat|nested] 20+ messages in thread
* [PATCH v3 09/14] iommu/amd: Rename variables to be consistent with struct io_pgtable_ops
2020-10-04 1:45 [PATCH v3 00/14] iommu/amd: Add Generic IO Page Table Framework Support Suravee Suthikulpanit
` (7 preceding siblings ...)
2020-10-04 1:45 ` [PATCH v3 08/14] iommu/amd: Remove amd_iommu_domain_get_pgtable Suravee Suthikulpanit
@ 2020-10-04 1:45 ` Suravee Suthikulpanit
2020-10-04 1:45 ` [PATCH v3 10/14] iommu/amd: Refactor fetch_pte to use struct amd_io_pgtable Suravee Suthikulpanit
` (5 subsequent siblings)
14 siblings, 0 replies; 20+ messages in thread
From: Suravee Suthikulpanit @ 2020-10-04 1:45 UTC (permalink / raw)
To: linux-kernel, iommu; +Cc: joro, robin.murphy, Suravee Suthikulpanit
There is no functional change.
Signed-off-by: Suravee Suthikulpanit <suravee.suthikulpanit@amd.com>
---
drivers/iommu/amd/io_pgtable.c | 31 +++++++++++++++----------------
1 file changed, 15 insertions(+), 16 deletions(-)
diff --git a/drivers/iommu/amd/io_pgtable.c b/drivers/iommu/amd/io_pgtable.c
index 6c063d2c8bf0..989db64a89a7 100644
--- a/drivers/iommu/amd/io_pgtable.c
+++ b/drivers/iommu/amd/io_pgtable.c
@@ -393,9 +393,9 @@ static struct page *free_clear_pte(u64 *pte, u64 pteval, struct page *freelist)
* and full 64 bit address spaces.
*/
int iommu_map_page(struct protection_domain *dom,
- unsigned long bus_addr,
- unsigned long phys_addr,
- unsigned long page_size,
+ unsigned long iova,
+ unsigned long paddr,
+ unsigned long size,
int prot,
gfp_t gfp)
{
@@ -404,15 +404,15 @@ int iommu_map_page(struct protection_domain *dom,
u64 __pte, *pte;
int ret, i, count;
- BUG_ON(!IS_ALIGNED(bus_addr, page_size));
- BUG_ON(!IS_ALIGNED(phys_addr, page_size));
+ BUG_ON(!IS_ALIGNED(iova, size));
+ BUG_ON(!IS_ALIGNED(paddr, size));
ret = -EINVAL;
if (!(prot & IOMMU_PROT_MASK))
goto out;
- count = PAGE_SIZE_PTE_COUNT(page_size);
- pte = alloc_pte(dom, bus_addr, page_size, NULL, gfp, &updated);
+ count = PAGE_SIZE_PTE_COUNT(size);
+ pte = alloc_pte(dom, iova, size, NULL, gfp, &updated);
ret = -ENOMEM;
if (!pte)
@@ -425,10 +425,10 @@ int iommu_map_page(struct protection_domain *dom,
updated = true;
if (count > 1) {
- __pte = PAGE_SIZE_PTE(__sme_set(phys_addr), page_size);
+ __pte = PAGE_SIZE_PTE(__sme_set(paddr), size);
__pte |= PM_LEVEL_ENC(7) | IOMMU_PTE_PR | IOMMU_PTE_FC;
} else
- __pte = __sme_set(phys_addr) | IOMMU_PTE_PR | IOMMU_PTE_FC;
+ __pte = __sme_set(paddr) | IOMMU_PTE_PR | IOMMU_PTE_FC;
if (prot & IOMMU_PROT_IR)
__pte |= IOMMU_PTE_IR;
@@ -462,20 +462,19 @@ int iommu_map_page(struct protection_domain *dom,
}
unsigned long iommu_unmap_page(struct protection_domain *dom,
- unsigned long bus_addr,
- unsigned long page_size)
+ unsigned long iova,
+ unsigned long size)
{
unsigned long long unmapped;
unsigned long unmap_size;
u64 *pte;
- BUG_ON(!is_power_of_2(page_size));
+ BUG_ON(!is_power_of_2(size));
unmapped = 0;
- while (unmapped < page_size) {
-
- pte = fetch_pte(dom, bus_addr, &unmap_size);
+ while (unmapped < size) {
+ pte = fetch_pte(dom, iova, &unmap_size);
if (pte) {
int i, count;
@@ -485,7 +484,7 @@ unsigned long iommu_unmap_page(struct protection_domain *dom,
pte[i] = 0ULL;
}
- bus_addr = (bus_addr & ~(unmap_size - 1)) + unmap_size;
+ iova = (iova & ~(unmap_size - 1)) + unmap_size;
unmapped += unmap_size;
}
--
2.17.1
^ permalink raw reply related [flat|nested] 20+ messages in thread
* [PATCH v3 10/14] iommu/amd: Refactor fetch_pte to use struct amd_io_pgtable
2020-10-04 1:45 [PATCH v3 00/14] iommu/amd: Add Generic IO Page Table Framework Support Suravee Suthikulpanit
` (8 preceding siblings ...)
2020-10-04 1:45 ` [PATCH v3 09/14] iommu/amd: Rename variables to be consistent with struct io_pgtable_ops Suravee Suthikulpanit
@ 2020-10-04 1:45 ` Suravee Suthikulpanit
2020-10-04 1:45 ` [PATCH v3 11/14] iommu/amd: Introduce iommu_v1_iova_to_phys Suravee Suthikulpanit
` (4 subsequent siblings)
14 siblings, 0 replies; 20+ messages in thread
From: Suravee Suthikulpanit @ 2020-10-04 1:45 UTC (permalink / raw)
To: linux-kernel, iommu; +Cc: joro, robin.murphy, Suravee Suthikulpanit
To simplify the fetch_pte function. There is no functional change.
Signed-off-by: Suravee Suthikulpanit <suravee.suthikulpanit@amd.com>
---
drivers/iommu/amd/amd_iommu.h | 2 +-
drivers/iommu/amd/io_pgtable.c | 13 +++++++------
drivers/iommu/amd/iommu.c | 4 +++-
3 files changed, 11 insertions(+), 8 deletions(-)
diff --git a/drivers/iommu/amd/amd_iommu.h b/drivers/iommu/amd/amd_iommu.h
index 2059e64fdc53..69996e57fae2 100644
--- a/drivers/iommu/amd/amd_iommu.h
+++ b/drivers/iommu/amd/amd_iommu.h
@@ -134,7 +134,7 @@ extern int iommu_map_page(struct protection_domain *dom,
extern unsigned long iommu_unmap_page(struct protection_domain *dom,
unsigned long bus_addr,
unsigned long page_size);
-extern u64 *fetch_pte(struct protection_domain *domain,
+extern u64 *fetch_pte(struct amd_io_pgtable *pgtable,
unsigned long address,
unsigned long *page_size);
extern void amd_iommu_domain_set_pgtable(struct protection_domain *domain,
diff --git a/drivers/iommu/amd/io_pgtable.c b/drivers/iommu/amd/io_pgtable.c
index 989db64a89a7..93ff8cb452ed 100644
--- a/drivers/iommu/amd/io_pgtable.c
+++ b/drivers/iommu/amd/io_pgtable.c
@@ -317,7 +317,7 @@ static u64 *alloc_pte(struct protection_domain *domain,
* This function checks if there is a PTE for a given dma address. If
* there is one, it returns the pointer to it.
*/
-u64 *fetch_pte(struct protection_domain *domain,
+u64 *fetch_pte(struct amd_io_pgtable *pgtable,
unsigned long address,
unsigned long *page_size)
{
@@ -326,11 +326,11 @@ u64 *fetch_pte(struct protection_domain *domain,
*page_size = 0;
- if (address > PM_LEVEL_SIZE(domain->iop.mode))
+ if (address > PM_LEVEL_SIZE(pgtable->mode))
return NULL;
- level = domain->iop.mode - 1;
- pte = &domain->iop.root[PM_LEVEL_INDEX(level, address)];
+ level = pgtable->mode - 1;
+ pte = &pgtable->root[PM_LEVEL_INDEX(level, address)];
*page_size = PTE_LEVEL_PAGE_SIZE(level);
while (level > 0) {
@@ -465,6 +465,8 @@ unsigned long iommu_unmap_page(struct protection_domain *dom,
unsigned long iova,
unsigned long size)
{
+ struct io_pgtable_ops *ops = &dom->iop.iop.ops;
+ struct amd_io_pgtable *pgtable = io_pgtable_ops_to_data(ops);
unsigned long long unmapped;
unsigned long unmap_size;
u64 *pte;
@@ -474,8 +476,7 @@ unsigned long iommu_unmap_page(struct protection_domain *dom,
unmapped = 0;
while (unmapped < size) {
- pte = fetch_pte(dom, iova, &unmap_size);
-
+ pte = fetch_pte(pgtable, iova, &unmap_size);
if (pte) {
int i, count;
diff --git a/drivers/iommu/amd/iommu.c b/drivers/iommu/amd/iommu.c
index 3f6ede1e572c..87cea1cde414 100644
--- a/drivers/iommu/amd/iommu.c
+++ b/drivers/iommu/amd/iommu.c
@@ -2078,13 +2078,15 @@ static phys_addr_t amd_iommu_iova_to_phys(struct iommu_domain *dom,
dma_addr_t iova)
{
struct protection_domain *domain = to_pdomain(dom);
+ struct io_pgtable_ops *ops = &domain->iop.iop.ops;
+ struct amd_io_pgtable *pgtable = io_pgtable_ops_to_data(ops);
unsigned long offset_mask, pte_pgsize;
u64 *pte, __pte;
if (domain->iop.mode == PAGE_MODE_NONE)
return iova;
- pte = fetch_pte(domain, iova, &pte_pgsize);
+ pte = fetch_pte(pgtable, iova, &pte_pgsize);
if (!pte || !IOMMU_PTE_PRESENT(*pte))
return 0;
--
2.17.1
^ permalink raw reply related [flat|nested] 20+ messages in thread
* [PATCH v3 11/14] iommu/amd: Introduce iommu_v1_iova_to_phys
2020-10-04 1:45 [PATCH v3 00/14] iommu/amd: Add Generic IO Page Table Framework Support Suravee Suthikulpanit
` (9 preceding siblings ...)
2020-10-04 1:45 ` [PATCH v3 10/14] iommu/amd: Refactor fetch_pte to use struct amd_io_pgtable Suravee Suthikulpanit
@ 2020-10-04 1:45 ` Suravee Suthikulpanit
2020-10-04 1:45 ` [PATCH v3 12/14] iommu/amd: Introduce iommu_v1_map_page and iommu_v1_unmap_page Suravee Suthikulpanit
` (3 subsequent siblings)
14 siblings, 0 replies; 20+ messages in thread
From: Suravee Suthikulpanit @ 2020-10-04 1:45 UTC (permalink / raw)
To: linux-kernel, iommu; +Cc: joro, robin.murphy, Suravee Suthikulpanit
This implements iova_to_phys for AMD IOMMU v1 pagetable,
which will be used by the IO page table framework.
Signed-off-by: Suravee Suthikulpanit <suravee.suthikulpanit@amd.com>
---
drivers/iommu/amd/io_pgtable.c | 22 ++++++++++++++++++++++
drivers/iommu/amd/iommu.c | 16 +---------------
2 files changed, 23 insertions(+), 15 deletions(-)
diff --git a/drivers/iommu/amd/io_pgtable.c b/drivers/iommu/amd/io_pgtable.c
index 93ff8cb452ed..7841e5e1e563 100644
--- a/drivers/iommu/amd/io_pgtable.c
+++ b/drivers/iommu/amd/io_pgtable.c
@@ -494,6 +494,26 @@ unsigned long iommu_unmap_page(struct protection_domain *dom,
return unmapped;
}
+static phys_addr_t iommu_v1_iova_to_phys(struct io_pgtable_ops *ops, unsigned long iova)
+{
+ struct amd_io_pgtable *pgtable = io_pgtable_ops_to_data(ops);
+ unsigned long offset_mask, pte_pgsize;
+ u64 *pte, __pte;
+
+ if (pgtable->mode == PAGE_MODE_NONE)
+ return iova;
+
+ pte = fetch_pte(pgtable, iova, &pte_pgsize);
+
+ if (!pte || !IOMMU_PTE_PRESENT(*pte))
+ return 0;
+
+ offset_mask = pte_pgsize - 1;
+ __pte = __sme_clr(*pte & PM_ADDR_MASK);
+
+ return (__pte & ~offset_mask) | (iova & offset_mask);
+}
+
/*
* ----------------------------------------------------
*/
@@ -505,6 +525,8 @@ static struct io_pgtable *v1_alloc_pgtable(struct io_pgtable_cfg *cfg, void *coo
{
struct amd_io_pgtable *pgtable = io_pgtable_cfg_to_data(cfg);
+ pgtable->iop.ops.iova_to_phys = iommu_v1_iova_to_phys;
+
return &pgtable->iop;
}
diff --git a/drivers/iommu/amd/iommu.c b/drivers/iommu/amd/iommu.c
index 87cea1cde414..9a1a16031e00 100644
--- a/drivers/iommu/amd/iommu.c
+++ b/drivers/iommu/amd/iommu.c
@@ -2079,22 +2079,8 @@ static phys_addr_t amd_iommu_iova_to_phys(struct iommu_domain *dom,
{
struct protection_domain *domain = to_pdomain(dom);
struct io_pgtable_ops *ops = &domain->iop.iop.ops;
- struct amd_io_pgtable *pgtable = io_pgtable_ops_to_data(ops);
- unsigned long offset_mask, pte_pgsize;
- u64 *pte, __pte;
- if (domain->iop.mode == PAGE_MODE_NONE)
- return iova;
-
- pte = fetch_pte(pgtable, iova, &pte_pgsize);
-
- if (!pte || !IOMMU_PTE_PRESENT(*pte))
- return 0;
-
- offset_mask = pte_pgsize - 1;
- __pte = __sme_clr(*pte & PM_ADDR_MASK);
-
- return (__pte & ~offset_mask) | (iova & offset_mask);
+ return ops->iova_to_phys(ops, iova);
}
static bool amd_iommu_capable(enum iommu_cap cap)
--
2.17.1
^ permalink raw reply related [flat|nested] 20+ messages in thread
* [PATCH v3 12/14] iommu/amd: Introduce iommu_v1_map_page and iommu_v1_unmap_page
2020-10-04 1:45 [PATCH v3 00/14] iommu/amd: Add Generic IO Page Table Framework Support Suravee Suthikulpanit
` (10 preceding siblings ...)
2020-10-04 1:45 ` [PATCH v3 11/14] iommu/amd: Introduce iommu_v1_iova_to_phys Suravee Suthikulpanit
@ 2020-10-04 1:45 ` Suravee Suthikulpanit
2020-10-04 1:45 ` [PATCH v3 13/14] iommu/amd: Introduce IOMMU flush callbacks Suravee Suthikulpanit
` (2 subsequent siblings)
14 siblings, 0 replies; 20+ messages in thread
From: Suravee Suthikulpanit @ 2020-10-04 1:45 UTC (permalink / raw)
To: linux-kernel, iommu; +Cc: joro, robin.murphy, Suravee Suthikulpanit
These implement map and unmap for AMD IOMMU v1 pagetable, which
will be used by the IO pagetable framework.
Also clean up unused extern function declarations.
Signed-off-by: Suravee Suthikulpanit <suravee.suthikulpanit@amd.com>
---
drivers/iommu/amd/amd_iommu.h | 13 -------------
drivers/iommu/amd/io_pgtable.c | 25 ++++++++++++-------------
drivers/iommu/amd/iommu.c | 7 ++++---
3 files changed, 16 insertions(+), 29 deletions(-)
diff --git a/drivers/iommu/amd/amd_iommu.h b/drivers/iommu/amd/amd_iommu.h
index 69996e57fae2..2e8dc2a1ec0f 100644
--- a/drivers/iommu/amd/amd_iommu.h
+++ b/drivers/iommu/amd/amd_iommu.h
@@ -124,19 +124,6 @@ void amd_iommu_apply_ivrs_quirks(void);
static inline void amd_iommu_apply_ivrs_quirks(void) { }
#endif
-/* TODO: These are temporary and will be removed once fully transition */
-extern int iommu_map_page(struct protection_domain *dom,
- unsigned long bus_addr,
- unsigned long phys_addr,
- unsigned long page_size,
- int prot,
- gfp_t gfp);
-extern unsigned long iommu_unmap_page(struct protection_domain *dom,
- unsigned long bus_addr,
- unsigned long page_size);
-extern u64 *fetch_pte(struct amd_io_pgtable *pgtable,
- unsigned long address,
- unsigned long *page_size);
extern void amd_iommu_domain_set_pgtable(struct protection_domain *domain,
u64 *root, int mode);
extern void amd_iommu_free_pgtable(struct amd_io_pgtable *pgtable);
diff --git a/drivers/iommu/amd/io_pgtable.c b/drivers/iommu/amd/io_pgtable.c
index 7841e5e1e563..d8b329aa0bb2 100644
--- a/drivers/iommu/amd/io_pgtable.c
+++ b/drivers/iommu/amd/io_pgtable.c
@@ -317,9 +317,9 @@ static u64 *alloc_pte(struct protection_domain *domain,
* This function checks if there is a PTE for a given dma address. If
* there is one, it returns the pointer to it.
*/
-u64 *fetch_pte(struct amd_io_pgtable *pgtable,
- unsigned long address,
- unsigned long *page_size)
+static u64 *fetch_pte(struct amd_io_pgtable *pgtable,
+ unsigned long address,
+ unsigned long *page_size)
{
int level;
u64 *pte;
@@ -392,13 +392,10 @@ static struct page *free_clear_pte(u64 *pte, u64 pteval, struct page *freelist)
* supporting all features of AMD IOMMU page tables like level skipping
* and full 64 bit address spaces.
*/
-int iommu_map_page(struct protection_domain *dom,
- unsigned long iova,
- unsigned long paddr,
- unsigned long size,
- int prot,
- gfp_t gfp)
+static int iommu_v1_map_page(struct io_pgtable_ops *ops, unsigned long iova,
+ phys_addr_t paddr, size_t size, int prot, gfp_t gfp)
{
+ struct protection_domain *dom = io_pgtable_ops_to_domain(ops);
struct page *freelist = NULL;
bool updated = false;
u64 __pte, *pte;
@@ -461,11 +458,11 @@ int iommu_map_page(struct protection_domain *dom,
return ret;
}
-unsigned long iommu_unmap_page(struct protection_domain *dom,
- unsigned long iova,
- unsigned long size)
+static unsigned long iommu_v1_unmap_page(struct io_pgtable_ops *ops,
+ unsigned long iova,
+ size_t size,
+ struct iommu_iotlb_gather *gather)
{
- struct io_pgtable_ops *ops = &dom->iop.iop.ops;
struct amd_io_pgtable *pgtable = io_pgtable_ops_to_data(ops);
unsigned long long unmapped;
unsigned long unmap_size;
@@ -525,6 +522,8 @@ static struct io_pgtable *v1_alloc_pgtable(struct io_pgtable_cfg *cfg, void *coo
{
struct amd_io_pgtable *pgtable = io_pgtable_cfg_to_data(cfg);
+ pgtable->iop.ops.map = iommu_v1_map_page;
+ pgtable->iop.ops.unmap = iommu_v1_unmap_page;
pgtable->iop.ops.iova_to_phys = iommu_v1_iova_to_phys;
return &pgtable->iop;
diff --git a/drivers/iommu/amd/iommu.c b/drivers/iommu/amd/iommu.c
index 9a1a16031e00..77f44b927ae7 100644
--- a/drivers/iommu/amd/iommu.c
+++ b/drivers/iommu/amd/iommu.c
@@ -2044,6 +2044,7 @@ static int amd_iommu_map(struct iommu_domain *dom, unsigned long iova,
gfp_t gfp)
{
struct protection_domain *domain = to_pdomain(dom);
+ struct io_pgtable_ops *ops = &domain->iop.iop.ops;
int prot = 0;
int ret;
@@ -2055,8 +2056,7 @@ static int amd_iommu_map(struct iommu_domain *dom, unsigned long iova,
if (iommu_prot & IOMMU_WRITE)
prot |= IOMMU_PROT_IW;
- ret = iommu_map_page(domain, iova, paddr, page_size, prot, gfp);
-
+ ret = ops->map(ops, iova, paddr, page_size, prot, gfp);
domain_flush_np_cache(domain, iova, page_size);
return ret;
@@ -2067,11 +2067,12 @@ static size_t amd_iommu_unmap(struct iommu_domain *dom, unsigned long iova,
struct iommu_iotlb_gather *gather)
{
struct protection_domain *domain = to_pdomain(dom);
+ struct io_pgtable_ops *ops = &domain->iop.iop.ops;
if (domain->iop.mode == PAGE_MODE_NONE)
return 0;
- return iommu_unmap_page(domain, iova, page_size);
+ return ops->unmap(ops, iova, page_size, gather);
}
static phys_addr_t amd_iommu_iova_to_phys(struct iommu_domain *dom,
--
2.17.1
^ permalink raw reply related [flat|nested] 20+ messages in thread
* [PATCH v3 13/14] iommu/amd: Introduce IOMMU flush callbacks
2020-10-04 1:45 [PATCH v3 00/14] iommu/amd: Add Generic IO Page Table Framework Support Suravee Suthikulpanit
` (11 preceding siblings ...)
2020-10-04 1:45 ` [PATCH v3 12/14] iommu/amd: Introduce iommu_v1_map_page and iommu_v1_unmap_page Suravee Suthikulpanit
@ 2020-10-04 1:45 ` Suravee Suthikulpanit
2020-10-04 1:45 ` [PATCH v3 14/14] iommu/amd: Adopt IO page table framework Suravee Suthikulpanit
2020-11-02 3:16 ` [PATCH v3 00/14] iommu/amd: Add Generic IO Page Table Framework Support Suravee Suthikulpanit
14 siblings, 0 replies; 20+ messages in thread
From: Suravee Suthikulpanit @ 2020-10-04 1:45 UTC (permalink / raw)
To: linux-kernel, iommu; +Cc: joro, robin.murphy, Suravee Suthikulpanit
Add TLB flush callback functions, which are used by the IO
page table framework.
Signed-off-by: Suravee Suthikulpanit <suravee.suthikulpanit@amd.com>
---
drivers/iommu/amd/io_pgtable.c | 29 +++++++++++++++++++++++++++++
1 file changed, 29 insertions(+)
diff --git a/drivers/iommu/amd/io_pgtable.c b/drivers/iommu/amd/io_pgtable.c
index d8b329aa0bb2..3c2faa47ea5d 100644
--- a/drivers/iommu/amd/io_pgtable.c
+++ b/drivers/iommu/amd/io_pgtable.c
@@ -514,6 +514,33 @@ static phys_addr_t iommu_v1_iova_to_phys(struct io_pgtable_ops *ops, unsigned lo
/*
* ----------------------------------------------------
*/
+static void v1_tlb_flush_all(void *cookie)
+{
+}
+
+static void v1_tlb_flush_walk(unsigned long iova, size_t size,
+ size_t granule, void *cookie)
+{
+}
+
+static void v1_tlb_flush_leaf(unsigned long iova, size_t size,
+ size_t granule, void *cookie)
+{
+}
+
+static void v1_tlb_add_page(struct iommu_iotlb_gather *gather,
+ unsigned long iova, size_t granule,
+ void *cookie)
+{
+}
+
+const struct iommu_flush_ops v1_flush_ops = {
+ .tlb_flush_all = v1_tlb_flush_all,
+ .tlb_flush_walk = v1_tlb_flush_walk,
+ .tlb_flush_leaf = v1_tlb_flush_leaf,
+ .tlb_add_page = v1_tlb_add_page,
+};
+
static void v1_free_pgtable(struct io_pgtable *iop)
{
}
@@ -526,6 +553,8 @@ static struct io_pgtable *v1_alloc_pgtable(struct io_pgtable_cfg *cfg, void *coo
pgtable->iop.ops.unmap = iommu_v1_unmap_page;
pgtable->iop.ops.iova_to_phys = iommu_v1_iova_to_phys;
+ cfg->tlb = &v1_flush_ops;
+
return &pgtable->iop;
}
--
2.17.1
^ permalink raw reply related [flat|nested] 20+ messages in thread
* [PATCH v3 14/14] iommu/amd: Adopt IO page table framework
2020-10-04 1:45 [PATCH v3 00/14] iommu/amd: Add Generic IO Page Table Framework Support Suravee Suthikulpanit
` (12 preceding siblings ...)
2020-10-04 1:45 ` [PATCH v3 13/14] iommu/amd: Introduce IOMMU flush callbacks Suravee Suthikulpanit
@ 2020-10-04 1:45 ` Suravee Suthikulpanit
2020-10-05 10:33 ` Jonathan Cameron
2020-11-02 3:16 ` [PATCH v3 00/14] iommu/amd: Add Generic IO Page Table Framework Support Suravee Suthikulpanit
14 siblings, 1 reply; 20+ messages in thread
From: Suravee Suthikulpanit @ 2020-10-04 1:45 UTC (permalink / raw)
To: linux-kernel, iommu; +Cc: joro, robin.murphy, Suravee Suthikulpanit
Switch to using IO page table framework for AMD IOMMU v1 page table.
Signed-off-by: Suravee Suthikulpanit <suravee.suthikulpanit@amd.com>
---
drivers/iommu/amd/iommu.c | 26 ++++++++++++++++++++++++++
1 file changed, 26 insertions(+)
diff --git a/drivers/iommu/amd/iommu.c b/drivers/iommu/amd/iommu.c
index 77f44b927ae7..6f8316206fb8 100644
--- a/drivers/iommu/amd/iommu.c
+++ b/drivers/iommu/amd/iommu.c
@@ -32,6 +32,7 @@
#include <linux/irqdomain.h>
#include <linux/percpu.h>
#include <linux/iova.h>
+#include <linux/io-pgtable.h>
#include <asm/irq_remapping.h>
#include <asm/io_apic.h>
#include <asm/apic.h>
@@ -1573,6 +1574,22 @@ static int pdev_iommuv2_enable(struct pci_dev *pdev)
return ret;
}
+struct io_pgtable_ops *
+amd_iommu_setup_io_pgtable_ops(struct iommu_dev_data *dev_data,
+ struct protection_domain *domain)
+{
+ struct amd_iommu *iommu = amd_iommu_rlookup_table[dev_data->devid];
+
+ domain->iop.pgtbl_cfg = (struct io_pgtable_cfg) {
+ .pgsize_bitmap = AMD_IOMMU_PGSIZES,
+ .ias = IOMMU_IN_ADDR_BIT_SIZE,
+ .oas = IOMMU_OUT_ADDR_BIT_SIZE,
+ .iommu_dev = &iommu->dev->dev,
+ };
+
+ return alloc_io_pgtable_ops(AMD_IOMMU_V1, &domain->iop.pgtbl_cfg, domain);
+}
+
/*
* If a device is not yet associated with a domain, this function makes the
* device visible in the domain
@@ -1580,6 +1597,7 @@ static int pdev_iommuv2_enable(struct pci_dev *pdev)
static int attach_device(struct device *dev,
struct protection_domain *domain)
{
+ struct io_pgtable_ops *pgtbl_ops;
struct iommu_dev_data *dev_data;
struct pci_dev *pdev;
unsigned long flags;
@@ -1623,6 +1641,12 @@ static int attach_device(struct device *dev,
skip_ats_check:
ret = 0;
+ pgtbl_ops = amd_iommu_setup_io_pgtable_ops(dev_data, domain);
+ if (!pgtbl_ops) {
+ ret = -ENOMEM;
+ goto out;
+ }
+
do_attach(dev_data, domain);
/*
@@ -1958,6 +1982,8 @@ static void amd_iommu_domain_free(struct iommu_domain *dom)
if (domain->dev_cnt > 0)
cleanup_domain(domain);
+ free_io_pgtable_ops(&domain->iop.iop.ops);
+
BUG_ON(domain->dev_cnt != 0);
if (!dom)
--
2.17.1
^ permalink raw reply related [flat|nested] 20+ messages in thread
* Re: [PATCH v3 14/14] iommu/amd: Adopt IO page table framework
2020-10-04 1:45 ` [PATCH v3 14/14] iommu/amd: Adopt IO page table framework Suravee Suthikulpanit
@ 2020-10-05 10:33 ` Jonathan Cameron
0 siblings, 0 replies; 20+ messages in thread
From: Jonathan Cameron @ 2020-10-05 10:33 UTC (permalink / raw)
To: Suravee Suthikulpanit; +Cc: linux-kernel, iommu, robin.murphy
On Sun, 4 Oct 2020 01:45:49 +0000
Suravee Suthikulpanit <suravee.suthikulpanit@amd.com> wrote:
> Switch to using IO page table framework for AMD IOMMU v1 page table.
>
> Signed-off-by: Suravee Suthikulpanit <suravee.suthikulpanit@amd.com>
One minor thing inline.
> ---
> drivers/iommu/amd/iommu.c | 26 ++++++++++++++++++++++++++
> 1 file changed, 26 insertions(+)
>
> diff --git a/drivers/iommu/amd/iommu.c b/drivers/iommu/amd/iommu.c
> index 77f44b927ae7..6f8316206fb8 100644
> --- a/drivers/iommu/amd/iommu.c
> +++ b/drivers/iommu/amd/iommu.c
> @@ -32,6 +32,7 @@
> #include <linux/irqdomain.h>
> #include <linux/percpu.h>
> #include <linux/iova.h>
> +#include <linux/io-pgtable.h>
> #include <asm/irq_remapping.h>
> #include <asm/io_apic.h>
> #include <asm/apic.h>
> @@ -1573,6 +1574,22 @@ static int pdev_iommuv2_enable(struct pci_dev *pdev)
> return ret;
> }
>
> +struct io_pgtable_ops *
> +amd_iommu_setup_io_pgtable_ops(struct iommu_dev_data *dev_data,
> + struct protection_domain *domain)
> +{
> + struct amd_iommu *iommu = amd_iommu_rlookup_table[dev_data->devid];
> +
> + domain->iop.pgtbl_cfg = (struct io_pgtable_cfg) {
> + .pgsize_bitmap = AMD_IOMMU_PGSIZES,
> + .ias = IOMMU_IN_ADDR_BIT_SIZE,
> + .oas = IOMMU_OUT_ADDR_BIT_SIZE,
> + .iommu_dev = &iommu->dev->dev,
> + };
> +
> + return alloc_io_pgtable_ops(AMD_IOMMU_V1, &domain->iop.pgtbl_cfg, domain);
> +}
> +
> /*
> * If a device is not yet associated with a domain, this function makes the
> * device visible in the domain
> @@ -1580,6 +1597,7 @@ static int pdev_iommuv2_enable(struct pci_dev *pdev)
> static int attach_device(struct device *dev,
> struct protection_domain *domain)
> {
> + struct io_pgtable_ops *pgtbl_ops;
> struct iommu_dev_data *dev_data;
> struct pci_dev *pdev;
> unsigned long flags;
> @@ -1623,6 +1641,12 @@ static int attach_device(struct device *dev,
> skip_ats_check:
> ret = 0;
>
> + pgtbl_ops = amd_iommu_setup_io_pgtable_ops(dev_data, domain);
> + if (!pgtbl_ops) {
Perhaps cleaner to not store in a local variable if you aren't going to use it?
if (!amd_iommu_setup_io_pgtable_ops(dev_data, domain)) {
> + ret = -ENOMEM;
> + goto out;
> + }
> +
> do_attach(dev_data, domain);
>
> /*
> @@ -1958,6 +1982,8 @@ static void amd_iommu_domain_free(struct iommu_domain *dom)
> if (domain->dev_cnt > 0)
> cleanup_domain(domain);
>
> + free_io_pgtable_ops(&domain->iop.iop.ops);
> +
> BUG_ON(domain->dev_cnt != 0);
>
> if (!dom)
^ permalink raw reply [flat|nested] 20+ messages in thread
* Re: [PATCH v3 00/14] iommu/amd: Add Generic IO Page Table Framework Support
2020-10-04 1:45 [PATCH v3 00/14] iommu/amd: Add Generic IO Page Table Framework Support Suravee Suthikulpanit
` (13 preceding siblings ...)
2020-10-04 1:45 ` [PATCH v3 14/14] iommu/amd: Adopt IO page table framework Suravee Suthikulpanit
@ 2020-11-02 3:16 ` Suravee Suthikulpanit
2020-11-11 3:10 ` Suravee Suthikulpanit
14 siblings, 1 reply; 20+ messages in thread
From: Suravee Suthikulpanit @ 2020-11-02 3:16 UTC (permalink / raw)
To: linux-kernel, iommu; +Cc: joro, robin.murphy
Joerg,
You mentioned to remind you to pull this in to linux-next.
Thanks,
Suravee
On 10/4/20 8:45 AM, Suravee Suthikulpanit wrote:
> The framework allows callable implementation of IO page table.
> This allows AMD IOMMU driver to switch between different types
> of AMD IOMMU page tables (e.g. v1 vs. v2).
>
> This series refactors the current implementation of AMD IOMMU v1 page table
> to adopt the framework. There should be no functional change.
> Subsequent series will introduce support for the AMD IOMMU v2 page table.
>
> Thanks,
> Suravee
>
> Change from V2 (https://lore.kernel.org/lkml/835c0d46-ed96-9fbe-856a-777dcffac967@amd.com/T/#t)
> - Patch 2/14: Introduce helper function io_pgtable_cfg_to_data.
> - Patch 13/14: Put back the struct iommu_flush_ops since patch v2 would run into
> NULL pointer bug when calling free_io_pgtable_ops if not defined.
>
> Change from V1 (https://lkml.org/lkml/2020/9/23/251)
> - Do not specify struct io_pgtable_cfg.coherent_walk, since it is
> not currently used. (per Robin)
> - Remove unused struct iommu_flush_ops. (patch 2/13)
> - Move amd_iommu_setup_io_pgtable_ops to iommu.c instead of io_pgtable.c
> patch 13/13)
>
> Suravee Suthikulpanit (14):
> iommu/amd: Re-define amd_iommu_domain_encode_pgtable as inline
> iommu/amd: Prepare for generic IO page table framework
> iommu/amd: Move pt_root to to struct amd_io_pgtable
> iommu/amd: Convert to using amd_io_pgtable
> iommu/amd: Declare functions as extern
> iommu/amd: Move IO page table related functions
> iommu/amd: Restructure code for freeing page table
> iommu/amd: Remove amd_iommu_domain_get_pgtable
> iommu/amd: Rename variables to be consistent with struct
> io_pgtable_ops
> iommu/amd: Refactor fetch_pte to use struct amd_io_pgtable
> iommu/amd: Introduce iommu_v1_iova_to_phys
> iommu/amd: Introduce iommu_v1_map_page and iommu_v1_unmap_page
> iommu/amd: Introduce IOMMU flush callbacks
> iommu/amd: Adopt IO page table framework
>
> drivers/iommu/amd/Kconfig | 1 +
> drivers/iommu/amd/Makefile | 2 +-
> drivers/iommu/amd/amd_iommu.h | 22 +
> drivers/iommu/amd/amd_iommu_types.h | 43 +-
> drivers/iommu/amd/io_pgtable.c | 564 ++++++++++++++++++++++++
> drivers/iommu/amd/iommu.c | 646 +++-------------------------
> drivers/iommu/io-pgtable.c | 3 +
> include/linux/io-pgtable.h | 2 +
> 8 files changed, 691 insertions(+), 592 deletions(-)
> create mode 100644 drivers/iommu/amd/io_pgtable.c
>
^ permalink raw reply [flat|nested] 20+ messages in thread
* Re: [PATCH v3 00/14] iommu/amd: Add Generic IO Page Table Framework Support
2020-11-02 3:16 ` [PATCH v3 00/14] iommu/amd: Add Generic IO Page Table Framework Support Suravee Suthikulpanit
@ 2020-11-11 3:10 ` Suravee Suthikulpanit
2020-11-13 5:57 ` Suravee Suthikulpanit
0 siblings, 1 reply; 20+ messages in thread
From: Suravee Suthikulpanit @ 2020-11-11 3:10 UTC (permalink / raw)
To: linux-kernel, iommu; +Cc: joro, robin.murphy
Hi Joerg,
Do you have any update on this series?
Thanks,
Suravee
On 11/2/20 10:16 AM, Suravee Suthikulpanit wrote:
> Joerg,
>
> You mentioned to remind you to pull this in to linux-next.
>
> Thanks,
> Suravee
>
> On 10/4/20 8:45 AM, Suravee Suthikulpanit wrote:
>> The framework allows callable implementation of IO page table.
>> This allows AMD IOMMU driver to switch between different types
>> of AMD IOMMU page tables (e.g. v1 vs. v2).
>>
>> This series refactors the current implementation of AMD IOMMU v1 page table
>> to adopt the framework. There should be no functional change.
>> Subsequent series will introduce support for the AMD IOMMU v2 page table.
>>
>> Thanks,
>> Suravee
>>
>> Change from V2 (https://lore.kernel.org/lkml/835c0d46-ed96-9fbe-856a-777dcffac967@amd.com/T/#t)
>> - Patch 2/14: Introduce helper function io_pgtable_cfg_to_data.
>> - Patch 13/14: Put back the struct iommu_flush_ops since patch v2 would run into
>> NULL pointer bug when calling free_io_pgtable_ops if not defined.
>>
>> Change from V1 (https://lkml.org/lkml/2020/9/23/251)
>> - Do not specify struct io_pgtable_cfg.coherent_walk, since it is
>> not currently used. (per Robin)
>> - Remove unused struct iommu_flush_ops. (patch 2/13)
>> - Move amd_iommu_setup_io_pgtable_ops to iommu.c instead of io_pgtable.c
>> patch 13/13)
>>
>> Suravee Suthikulpanit (14):
>> iommu/amd: Re-define amd_iommu_domain_encode_pgtable as inline
>> iommu/amd: Prepare for generic IO page table framework
>> iommu/amd: Move pt_root to to struct amd_io_pgtable
>> iommu/amd: Convert to using amd_io_pgtable
>> iommu/amd: Declare functions as extern
>> iommu/amd: Move IO page table related functions
>> iommu/amd: Restructure code for freeing page table
>> iommu/amd: Remove amd_iommu_domain_get_pgtable
>> iommu/amd: Rename variables to be consistent with struct
>> io_pgtable_ops
>> iommu/amd: Refactor fetch_pte to use struct amd_io_pgtable
>> iommu/amd: Introduce iommu_v1_iova_to_phys
>> iommu/amd: Introduce iommu_v1_map_page and iommu_v1_unmap_page
>> iommu/amd: Introduce IOMMU flush callbacks
>> iommu/amd: Adopt IO page table framework
>>
>> drivers/iommu/amd/Kconfig | 1 +
>> drivers/iommu/amd/Makefile | 2 +-
>> drivers/iommu/amd/amd_iommu.h | 22 +
>> drivers/iommu/amd/amd_iommu_types.h | 43 +-
>> drivers/iommu/amd/io_pgtable.c | 564 ++++++++++++++++++++++++
>> drivers/iommu/amd/iommu.c | 646 +++-------------------------
>> drivers/iommu/io-pgtable.c | 3 +
>> include/linux/io-pgtable.h | 2 +
>> 8 files changed, 691 insertions(+), 592 deletions(-)
>> create mode 100644 drivers/iommu/amd/io_pgtable.c
>>
^ permalink raw reply [flat|nested] 20+ messages in thread
* Re: [PATCH v3 00/14] iommu/amd: Add Generic IO Page Table Framework Support
2020-11-11 3:10 ` Suravee Suthikulpanit
@ 2020-11-13 5:57 ` Suravee Suthikulpanit
2020-11-17 22:43 ` Will Deacon
0 siblings, 1 reply; 20+ messages in thread
From: Suravee Suthikulpanit @ 2020-11-13 5:57 UTC (permalink / raw)
To: linux-kernel, iommu; +Cc: joro, robin.murphy
Joerg,
Please ignore to include the V3. I am working on V4 to resubmit.
Thank you,
Suravee
On 11/11/20 10:10 AM, Suravee Suthikulpanit wrote:
> Hi Joerg,
>
> Do you have any update on this series?
>
> Thanks,
> Suravee
>
> On 11/2/20 10:16 AM, Suravee Suthikulpanit wrote:
>> Joerg,
>>
>> You mentioned to remind you to pull this in to linux-next.
>>
>> Thanks,
>> Suravee
>>
>> On 10/4/20 8:45 AM, Suravee Suthikulpanit wrote:
>>> The framework allows callable implementation of IO page table.
>>> This allows AMD IOMMU driver to switch between different types
>>> of AMD IOMMU page tables (e.g. v1 vs. v2).
>>>
>>> This series refactors the current implementation of AMD IOMMU v1 page table
>>> to adopt the framework. There should be no functional change.
>>> Subsequent series will introduce support for the AMD IOMMU v2 page table.
>>>
>>> Thanks,
>>> Suravee
>>>
>>> Change from V2 (https://lore.kernel.org/lkml/835c0d46-ed96-9fbe-856a-777dcffac967@amd.com/T/#t)
>>> - Patch 2/14: Introduce helper function io_pgtable_cfg_to_data.
>>> - Patch 13/14: Put back the struct iommu_flush_ops since patch v2 would run into
>>> NULL pointer bug when calling free_io_pgtable_ops if not defined.
>>>
>>> Change from V1 (https://lkml.org/lkml/2020/9/23/251)
>>> - Do not specify struct io_pgtable_cfg.coherent_walk, since it is
>>> not currently used. (per Robin)
>>> - Remove unused struct iommu_flush_ops. (patch 2/13)
>>> - Move amd_iommu_setup_io_pgtable_ops to iommu.c instead of io_pgtable.c
>>> patch 13/13)
>>>
>>> Suravee Suthikulpanit (14):
>>> iommu/amd: Re-define amd_iommu_domain_encode_pgtable as inline
>>> iommu/amd: Prepare for generic IO page table framework
>>> iommu/amd: Move pt_root to to struct amd_io_pgtable
>>> iommu/amd: Convert to using amd_io_pgtable
>>> iommu/amd: Declare functions as extern
>>> iommu/amd: Move IO page table related functions
>>> iommu/amd: Restructure code for freeing page table
>>> iommu/amd: Remove amd_iommu_domain_get_pgtable
>>> iommu/amd: Rename variables to be consistent with struct
>>> io_pgtable_ops
>>> iommu/amd: Refactor fetch_pte to use struct amd_io_pgtable
>>> iommu/amd: Introduce iommu_v1_iova_to_phys
>>> iommu/amd: Introduce iommu_v1_map_page and iommu_v1_unmap_page
>>> iommu/amd: Introduce IOMMU flush callbacks
>>> iommu/amd: Adopt IO page table framework
>>>
>>> drivers/iommu/amd/Kconfig | 1 +
>>> drivers/iommu/amd/Makefile | 2 +-
>>> drivers/iommu/amd/amd_iommu.h | 22 +
>>> drivers/iommu/amd/amd_iommu_types.h | 43 +-
>>> drivers/iommu/amd/io_pgtable.c | 564 ++++++++++++++++++++++++
>>> drivers/iommu/amd/iommu.c | 646 +++-------------------------
>>> drivers/iommu/io-pgtable.c | 3 +
>>> include/linux/io-pgtable.h | 2 +
>>> 8 files changed, 691 insertions(+), 592 deletions(-)
>>> create mode 100644 drivers/iommu/amd/io_pgtable.c
>>>
^ permalink raw reply [flat|nested] 20+ messages in thread