[PATCH 0/3] iommu/arm-smmu-v3: TLB invalidation for SVA

[PATCH 0/3] iommu/arm-smmu-v3: TLB invalidation for SVA

[Jean-Philippe Brucker]

To support sharing page tables with the CPU, the SMMU can participate in
Broadcast TLB Maintenance (BTM), where TLB invalidate instructions from
the CPU are received by the SMMU. For platforms that do no implement BTM
[1], it is still possible to use SVA, by sending all TLB invalidations
through the command queue. Patch 2 implements this.

This series also enables SVA for platforms that do support BTM, as an
intermediate step because properly supporting BTM requires cooperating
with KVM to allocate VMIDs [2]. With BTM enabled, the SMMU applies
broadcast invalidations by VMID to any matching TLB entry, because there
is no distinction between private and shared VMIDs like there is for
ASIDs. Therefore a stage-2 domain will need a VMID that doesn't conflict
with one allocated by KVM (or use the one from the corresponding VM,
pinned).

These patches, along with the IOPF series [3] and the quirks [4], enable
SVA for the hisi accelerator that's already supported upstream. My quick
performance comparison between BTM and !BTM on that platform were
inconclusive. Doing invalidations via cmdq seemed to slightly reduce
performance of some heavy compression jobs, but there was too much noise
and not enough invalidations in my tests.

This series does not depend on the IOPF one [3].

[1] https://lore.kernel.org/linux-iommu/BY5PR12MB37641E84D516054387FEE330B3CC0@BY5PR12MB3764.namprd12.prod.outlook.com/
[2] https://lore.kernel.org/linux-iommu/20200522101755.GA3453945@myrica/
[3] https://lore.kernel.org/linux-iommu/20210121123623.2060416-1-jean-philippe@linaro.org/
[4] https://lore.kernel.org/linux-pci/1610960316-28935-1-git-send-email-zhangfei.gao@linaro.org/

Jean-Philippe Brucker (3):
  iommu/arm-smmu-v3: Split arm_smmu_tlb_inv_range()
  iommu/arm-smmu-v3: Make BTM optional for SVA
  iommu/arm-smmu-v3: Add support for VHE

 drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h   |   6 +
 .../iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c   |  14 ++-
 drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c   | 104 ++++++++++++------
 3 files changed, 89 insertions(+), 35 deletions(-)

-- 
2.30.0


_______________________________________________
linux-arm-kernel mailing list
linux-arm-kernel@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel

[PATCH 1/3] iommu/arm-smmu-v3: Split arm_smmu_tlb_inv_range()

[Jean-Philippe Brucker]

Extract some of the cmd initialization and the ATC invalidation from
arm_smmu_tlb_inv_range(), to allow an MMU notifier to invalidate a VA
range by ASID.

Signed-off-by: Jean-Philippe Brucker <jean-philippe@linaro.org>
---
 drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 62 ++++++++++++---------
 1 file changed, 35 insertions(+), 27 deletions(-)

diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
index 740a3d487591..a27b074d5c0c 100644
--- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
+++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
@@ -1934,40 +1934,27 @@ static void arm_smmu_tlb_inv_context(void *cookie)
 	arm_smmu_atc_inv_domain(smmu_domain, 0, 0, 0);
 }
 
-static void arm_smmu_tlb_inv_range(unsigned long iova, size_t size,
-				   size_t granule, bool leaf,
+static void arm_smmu_tlb_inv_range(struct arm_smmu_cmdq_ent *cmd,
+				   unsigned long iova, size_t size,
+				   size_t granule,
 				   struct arm_smmu_domain *smmu_domain)
 {
 	struct arm_smmu_device *smmu = smmu_domain->smmu;
-	unsigned long start = iova, end = iova + size, num_pages = 0, tg = 0;
+	unsigned long end = iova + size, num_pages = 0, tg = 0;
 	size_t inv_range = granule;
 	struct arm_smmu_cmdq_batch cmds = {};
-	struct arm_smmu_cmdq_ent cmd = {
-		.tlbi = {
-			.leaf	= leaf,
-		},
-	};
-
 	if (!size)
 		return;
 
-	if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1) {
-		cmd.opcode	= CMDQ_OP_TLBI_NH_VA;
-		cmd.tlbi.asid	= smmu_domain->s1_cfg.cd.asid;
-	} else {
-		cmd.opcode	= CMDQ_OP_TLBI_S2_IPA;
-		cmd.tlbi.vmid	= smmu_domain->s2_cfg.vmid;
-	}
-
 	if (smmu->features & ARM_SMMU_FEAT_RANGE_INV) {
 		/* Get the leaf page size */
 		tg = __ffs(smmu_domain->domain.pgsize_bitmap);
 
 		/* Convert page size of 12,14,16 (log2) to 1,2,3 */
-		cmd.tlbi.tg = (tg - 10) / 2;
+		cmd->tlbi.tg = (tg - 10) / 2;
 
 		/* Determine what level the granule is at */
-		cmd.tlbi.ttl = 4 - ((ilog2(granule) - 3) / (tg - 3));
+		cmd->tlbi.ttl = 4 - ((ilog2(granule) - 3) / (tg - 3));
 
 		num_pages = size >> tg;
 	}
@@ -1985,11 +1972,11 @@ static void arm_smmu_tlb_inv_range(unsigned long iova, size_t size,
 
 			/* Determine the power of 2 multiple number of pages */
 			scale = __ffs(num_pages);
-			cmd.tlbi.scale = scale;
+			cmd->tlbi.scale = scale;
 
 			/* Determine how many chunks of 2^scale size we have */
 			num = (num_pages >> scale) & CMDQ_TLBI_RANGE_NUM_MAX;
-			cmd.tlbi.num = num - 1;
+			cmd->tlbi.num = num - 1;
 
 			/* range is num * 2^scale * pgsize */
 			inv_range = num << (scale + tg);
@@ -1998,17 +1985,37 @@ static void arm_smmu_tlb_inv_range(unsigned long iova, size_t size,
 			num_pages -= num << scale;
 		}
 
-		cmd.tlbi.addr = iova;
-		arm_smmu_cmdq_batch_add(smmu, &cmds, &cmd);
+		cmd->tlbi.addr = iova;
+		arm_smmu_cmdq_batch_add(smmu, &cmds, cmd);
 		iova += inv_range;
 	}
 	arm_smmu_cmdq_batch_submit(smmu, &cmds);
+}
+
+static void arm_smmu_tlb_inv_range_domain(unsigned long iova, size_t size,
+					  size_t granule, bool leaf,
+					  struct arm_smmu_domain *smmu_domain)
+{
+	struct arm_smmu_cmdq_ent cmd = {
+		.tlbi = {
+			.leaf	= leaf,
+		},
+	};
+
+	if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1) {
+		cmd.opcode	= CMDQ_OP_TLBI_NH_VA;
+		cmd.tlbi.asid	= smmu_domain->s1_cfg.cd.asid;
+	} else {
+		cmd.opcode	= CMDQ_OP_TLBI_S2_IPA;
+		cmd.tlbi.vmid	= smmu_domain->s2_cfg.vmid;
+	}
+	arm_smmu_tlb_inv_range(&cmd, iova, size, granule, smmu_domain);
 
 	/*
 	 * Unfortunately, this can't be leaf-only since we may have
 	 * zapped an entire table.
 	 */
-	arm_smmu_atc_inv_domain(smmu_domain, 0, start, size);
+	arm_smmu_atc_inv_domain(smmu_domain, 0, iova, size);
 }
 
 static void arm_smmu_tlb_inv_page_nosync(struct iommu_iotlb_gather *gather,
@@ -2024,7 +2031,7 @@ static void arm_smmu_tlb_inv_page_nosync(struct iommu_iotlb_gather *gather,
 static void arm_smmu_tlb_inv_walk(unsigned long iova, size_t size,
 				  size_t granule, void *cookie)
 {
-	arm_smmu_tlb_inv_range(iova, size, granule, false, cookie);
+	arm_smmu_tlb_inv_range_domain(iova, size, granule, false, cookie);
 }
 
 static const struct iommu_flush_ops arm_smmu_flush_ops = {
@@ -2622,8 +2629,9 @@ static void arm_smmu_iotlb_sync(struct iommu_domain *domain,
 {
 	struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
 
-	arm_smmu_tlb_inv_range(gather->start, gather->end - gather->start,
-			       gather->pgsize, true, smmu_domain);
+	arm_smmu_tlb_inv_range_domain(gather->start,
+				      gather->end - gather->start,
+				      gather->pgsize, true, smmu_domain);
 }
 
 static phys_addr_t
-- 
2.30.0


_______________________________________________
linux-arm-kernel mailing list
linux-arm-kernel@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel

Re: [PATCH 1/3] iommu/arm-smmu-v3: Split arm_smmu_tlb_inv_range()

[Will Deacon]

On Fri, Jan 22, 2021 at 12:52:56PM +0100, Jean-Philippe Brucker wrote:
> Extract some of the cmd initialization and the ATC invalidation from
> arm_smmu_tlb_inv_range(), to allow an MMU notifier to invalidate a VA
> range by ASID.
> 
> Signed-off-by: Jean-Philippe Brucker <jean-philippe@linaro.org>
> ---
>  drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 62 ++++++++++++---------
>  1 file changed, 35 insertions(+), 27 deletions(-)
> 
> diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
> index 740a3d487591..a27b074d5c0c 100644
> --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
> +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
> @@ -1934,40 +1934,27 @@ static void arm_smmu_tlb_inv_context(void *cookie)
>  	arm_smmu_atc_inv_domain(smmu_domain, 0, 0, 0);
>  }
>  
> -static void arm_smmu_tlb_inv_range(unsigned long iova, size_t size,
> -				   size_t granule, bool leaf,
> +static void arm_smmu_tlb_inv_range(struct arm_smmu_cmdq_ent *cmd,
> +				   unsigned long iova, size_t size,
> +				   size_t granule,
>  				   struct arm_smmu_domain *smmu_domain)

nit: please can you prefix this function with a couple of underscores and/or
change its name? It's now a low-level helper and trusts the caller to pass
in a TLBI command, so we really don't want people to call it directly!

Anyway, the series looks good to me. In fact, I tried to apply it but I get
a conflict with the last patch. Please can you rebase onto my
for-joerg/arm-smmu/updates branch? If you do that (plus the nit above) then
I can queue these right away.

Cheers,

Will

_______________________________________________
linux-arm-kernel mailing list
linux-arm-kernel@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel

Re: [PATCH 1/3] iommu/arm-smmu-v3: Split arm_smmu_tlb_inv_range()

[Robin Murphy]

On 2021-01-22 11:52, Jean-Philippe Brucker wrote:
> Extract some of the cmd initialization and the ATC invalidation from
> arm_smmu_tlb_inv_range(), to allow an MMU notifier to invalidate a VA
> range by ASID.
> 
> Signed-off-by: Jean-Philippe Brucker <jean-philippe@linaro.org>
> ---
>   drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 62 ++++++++++++---------
>   1 file changed, 35 insertions(+), 27 deletions(-)
> 
> diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
> index 740a3d487591..a27b074d5c0c 100644
> --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
> +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
> @@ -1934,40 +1934,27 @@ static void arm_smmu_tlb_inv_context(void *cookie)
>   	arm_smmu_atc_inv_domain(smmu_domain, 0, 0, 0);
>   }
>   
> -static void arm_smmu_tlb_inv_range(unsigned long iova, size_t size,
> -				   size_t granule, bool leaf,
> +static void arm_smmu_tlb_inv_range(struct arm_smmu_cmdq_ent *cmd,
> +				   unsigned long iova, size_t size,
> +				   size_t granule,
>   				   struct arm_smmu_domain *smmu_domain)

To keep with the specific low-level helper vibe, maybe we should just 
pass the pgsize_bitmap directly, since I think that's all we're using 
the domain for now.

Robin.

>   {
>   	struct arm_smmu_device *smmu = smmu_domain->smmu;
> -	unsigned long start = iova, end = iova + size, num_pages = 0, tg = 0;
> +	unsigned long end = iova + size, num_pages = 0, tg = 0;
>   	size_t inv_range = granule;
>   	struct arm_smmu_cmdq_batch cmds = {};
> -	struct arm_smmu_cmdq_ent cmd = {
> -		.tlbi = {
> -			.leaf	= leaf,
> -		},
> -	};
> -
>   	if (!size)
>   		return;
>   
> -	if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1) {
> -		cmd.opcode	= CMDQ_OP_TLBI_NH_VA;
> -		cmd.tlbi.asid	= smmu_domain->s1_cfg.cd.asid;
> -	} else {
> -		cmd.opcode	= CMDQ_OP_TLBI_S2_IPA;
> -		cmd.tlbi.vmid	= smmu_domain->s2_cfg.vmid;
> -	}
> -
>   	if (smmu->features & ARM_SMMU_FEAT_RANGE_INV) {
>   		/* Get the leaf page size */
>   		tg = __ffs(smmu_domain->domain.pgsize_bitmap);
>   
>   		/* Convert page size of 12,14,16 (log2) to 1,2,3 */
> -		cmd.tlbi.tg = (tg - 10) / 2;
> +		cmd->tlbi.tg = (tg - 10) / 2;
>   
>   		/* Determine what level the granule is at */
> -		cmd.tlbi.ttl = 4 - ((ilog2(granule) - 3) / (tg - 3));
> +		cmd->tlbi.ttl = 4 - ((ilog2(granule) - 3) / (tg - 3));
>   
>   		num_pages = size >> tg;
>   	}
> @@ -1985,11 +1972,11 @@ static void arm_smmu_tlb_inv_range(unsigned long iova, size_t size,
>   
>   			/* Determine the power of 2 multiple number of pages */
>   			scale = __ffs(num_pages);
> -			cmd.tlbi.scale = scale;
> +			cmd->tlbi.scale = scale;
>   
>   			/* Determine how many chunks of 2^scale size we have */
>   			num = (num_pages >> scale) & CMDQ_TLBI_RANGE_NUM_MAX;
> -			cmd.tlbi.num = num - 1;
> +			cmd->tlbi.num = num - 1;
>   
>   			/* range is num * 2^scale * pgsize */
>   			inv_range = num << (scale + tg);
> @@ -1998,17 +1985,37 @@ static void arm_smmu_tlb_inv_range(unsigned long iova, size_t size,
>   			num_pages -= num << scale;
>   		}
>   
> -		cmd.tlbi.addr = iova;
> -		arm_smmu_cmdq_batch_add(smmu, &cmds, &cmd);
> +		cmd->tlbi.addr = iova;
> +		arm_smmu_cmdq_batch_add(smmu, &cmds, cmd);
>   		iova += inv_range;
>   	}
>   	arm_smmu_cmdq_batch_submit(smmu, &cmds);
> +}
> +
> +static void arm_smmu_tlb_inv_range_domain(unsigned long iova, size_t size,
> +					  size_t granule, bool leaf,
> +					  struct arm_smmu_domain *smmu_domain)
> +{
> +	struct arm_smmu_cmdq_ent cmd = {
> +		.tlbi = {
> +			.leaf	= leaf,
> +		},
> +	};
> +
> +	if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1) {
> +		cmd.opcode	= CMDQ_OP_TLBI_NH_VA;
> +		cmd.tlbi.asid	= smmu_domain->s1_cfg.cd.asid;
> +	} else {
> +		cmd.opcode	= CMDQ_OP_TLBI_S2_IPA;
> +		cmd.tlbi.vmid	= smmu_domain->s2_cfg.vmid;
> +	}
> +	arm_smmu_tlb_inv_range(&cmd, iova, size, granule, smmu_domain);
>   
>   	/*
>   	 * Unfortunately, this can't be leaf-only since we may have
>   	 * zapped an entire table.
>   	 */
> -	arm_smmu_atc_inv_domain(smmu_domain, 0, start, size);
> +	arm_smmu_atc_inv_domain(smmu_domain, 0, iova, size);
>   }
>   
>   static void arm_smmu_tlb_inv_page_nosync(struct iommu_iotlb_gather *gather,
> @@ -2024,7 +2031,7 @@ static void arm_smmu_tlb_inv_page_nosync(struct iommu_iotlb_gather *gather,
>   static void arm_smmu_tlb_inv_walk(unsigned long iova, size_t size,
>   				  size_t granule, void *cookie)
>   {
> -	arm_smmu_tlb_inv_range(iova, size, granule, false, cookie);
> +	arm_smmu_tlb_inv_range_domain(iova, size, granule, false, cookie);
>   }
>   
>   static const struct iommu_flush_ops arm_smmu_flush_ops = {
> @@ -2622,8 +2629,9 @@ static void arm_smmu_iotlb_sync(struct iommu_domain *domain,
>   {
>   	struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
>   
> -	arm_smmu_tlb_inv_range(gather->start, gather->end - gather->start,
> -			       gather->pgsize, true, smmu_domain);
> +	arm_smmu_tlb_inv_range_domain(gather->start,
> +				      gather->end - gather->start,
> +				      gather->pgsize, true, smmu_domain);
>   }
>   
>   static phys_addr_t
> 

_______________________________________________
linux-arm-kernel mailing list
linux-arm-kernel@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel

[PATCH 2/3] iommu/arm-smmu-v3: Make BTM optional for SVA

[Jean-Philippe Brucker]

When BTM isn't supported by the SMMU, send invalidations on the
command queue.

Signed-off-by: Jean-Philippe Brucker <jean-philippe@linaro.org>
---
 drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h     |  3 +++
 drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c | 14 +++++++++++---
 drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c     | 14 ++++++++++++++
 3 files changed, 28 insertions(+), 3 deletions(-)

diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h
index a6536c2b32d0..652d03ad8ae6 100644
--- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h
+++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h
@@ -743,6 +743,9 @@ extern struct arm_smmu_ctx_desc quiet_cd;
 int arm_smmu_write_ctx_desc(struct arm_smmu_domain *smmu_domain, int ssid,
 			    struct arm_smmu_ctx_desc *cd);
 void arm_smmu_tlb_inv_asid(struct arm_smmu_device *smmu, u16 asid);
+void arm_smmu_tlb_inv_range_asid(unsigned long iova, size_t size, int asid,
+				 size_t granule, bool leaf,
+				 struct arm_smmu_domain *smmu_domain);
 bool arm_smmu_free_asid(struct arm_smmu_ctx_desc *cd);
 int arm_smmu_atc_inv_domain(struct arm_smmu_domain *smmu_domain, int ssid,
 			    unsigned long iova, size_t size);
diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c
index 642ce2c225b5..ad8cf62a8f83 100644
--- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c
+++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c
@@ -16,6 +16,7 @@ struct arm_smmu_mmu_notifier {
 	struct mmu_notifier		mn;
 	struct arm_smmu_ctx_desc	*cd;
 	bool				cleared;
+	bool				tlb_inv_command;
 	refcount_t			refs;
 	struct list_head		list;
 	struct arm_smmu_domain		*domain;
@@ -182,9 +183,13 @@ static void arm_smmu_mm_invalidate_range(struct mmu_notifier *mn,
 					 unsigned long start, unsigned long end)
 {
 	struct arm_smmu_mmu_notifier *smmu_mn = mn_to_smmu(mn);
+	struct arm_smmu_domain *smmu_domain = smmu_mn->domain;
+	size_t size = end - start + 1;
 
-	arm_smmu_atc_inv_domain(smmu_mn->domain, mm->pasid, start,
-				end - start + 1);
+	if (smmu_mn->tlb_inv_command)
+		arm_smmu_tlb_inv_range_asid(start, size, smmu_mn->cd->asid,
+					    PAGE_SIZE, false, smmu_domain);
+	arm_smmu_atc_inv_domain(smmu_domain, mm->pasid, start, size);
 }
 
 static void arm_smmu_mm_release(struct mmu_notifier *mn, struct mm_struct *mm)
@@ -253,6 +258,9 @@ arm_smmu_mmu_notifier_get(struct arm_smmu_domain *smmu_domain,
 	smmu_mn->domain = smmu_domain;
 	smmu_mn->mn.ops = &arm_smmu_mmu_notifier_ops;
 
+	if (!(smmu_domain->smmu->features & ARM_SMMU_FEAT_BTM))
+		smmu_mn->tlb_inv_command = true;
+
 	ret = mmu_notifier_register(&smmu_mn->mn, mm);
 	if (ret) {
 		kfree(smmu_mn);
@@ -404,7 +412,7 @@ bool arm_smmu_sva_supported(struct arm_smmu_device *smmu)
 	unsigned long reg, fld;
 	unsigned long oas;
 	unsigned long asid_bits;
-	u32 feat_mask = ARM_SMMU_FEAT_BTM | ARM_SMMU_FEAT_COHERENCY;
+	u32 feat_mask = ARM_SMMU_FEAT_COHERENCY;
 
 	if (vabits_actual == 52)
 		feat_mask |= ARM_SMMU_FEAT_VAX;
diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
index a27b074d5c0c..db545834493b 100644
--- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
+++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
@@ -2018,6 +2018,20 @@ static void arm_smmu_tlb_inv_range_domain(unsigned long iova, size_t size,
 	arm_smmu_atc_inv_domain(smmu_domain, 0, iova, size);
 }
 
+void arm_smmu_tlb_inv_range_asid(unsigned long iova, size_t size, int asid,
+				 size_t granule, bool leaf,
+				 struct arm_smmu_domain *smmu_domain)
+{
+	struct arm_smmu_cmdq_ent cmd = {
+		.opcode	= CMDQ_OP_TLBI_NH_VA,
+		.tlbi = {
+			.asid	= asid,
+			.leaf	= leaf,
+		},
+	};
+	arm_smmu_tlb_inv_range(&cmd, iova, size, granule, smmu_domain);
+}
+
 static void arm_smmu_tlb_inv_page_nosync(struct iommu_iotlb_gather *gather,
 					 unsigned long iova, size_t granule,
 					 void *cookie)
-- 
2.30.0


_______________________________________________
linux-arm-kernel mailing list
linux-arm-kernel@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel

Re: [PATCH 2/3] iommu/arm-smmu-v3: Make BTM optional for SVA

[Robin Murphy]

On 2021-01-22 11:52, Jean-Philippe Brucker wrote:
> When BTM isn't supported by the SMMU, send invalidations on the
> command queue.
> 
> Signed-off-by: Jean-Philippe Brucker <jean-philippe@linaro.org>
> ---
>   drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h     |  3 +++
>   drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c | 14 +++++++++++---
>   drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c     | 14 ++++++++++++++
>   3 files changed, 28 insertions(+), 3 deletions(-)
> 
> diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h
> index a6536c2b32d0..652d03ad8ae6 100644
> --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h
> +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h
> @@ -743,6 +743,9 @@ extern struct arm_smmu_ctx_desc quiet_cd;
>   int arm_smmu_write_ctx_desc(struct arm_smmu_domain *smmu_domain, int ssid,
>   			    struct arm_smmu_ctx_desc *cd);
>   void arm_smmu_tlb_inv_asid(struct arm_smmu_device *smmu, u16 asid);
> +void arm_smmu_tlb_inv_range_asid(unsigned long iova, size_t size, int asid,
> +				 size_t granule, bool leaf,
> +				 struct arm_smmu_domain *smmu_domain);
>   bool arm_smmu_free_asid(struct arm_smmu_ctx_desc *cd);
>   int arm_smmu_atc_inv_domain(struct arm_smmu_domain *smmu_domain, int ssid,
>   			    unsigned long iova, size_t size);
> diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c
> index 642ce2c225b5..ad8cf62a8f83 100644
> --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c
> +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c
> @@ -16,6 +16,7 @@ struct arm_smmu_mmu_notifier {
>   	struct mmu_notifier		mn;
>   	struct arm_smmu_ctx_desc	*cd;
>   	bool				cleared;
> +	bool				tlb_inv_command;
>   	refcount_t			refs;
>   	struct list_head		list;
>   	struct arm_smmu_domain		*domain;
> @@ -182,9 +183,13 @@ static void arm_smmu_mm_invalidate_range(struct mmu_notifier *mn,
>   					 unsigned long start, unsigned long end)
>   {
>   	struct arm_smmu_mmu_notifier *smmu_mn = mn_to_smmu(mn);
> +	struct arm_smmu_domain *smmu_domain = smmu_mn->domain;
> +	size_t size = end - start + 1;
>   
> -	arm_smmu_atc_inv_domain(smmu_mn->domain, mm->pasid, start,
> -				end - start + 1);
> +	if (smmu_mn->tlb_inv_command)

Since we're going to be drilling down to smmu_domain->smmu->features in 
the invalidate call anyway, perhaps we could just test for BTM directly 
here?

Obviously that also proves my previous comment involved a reading 
comprehension failure and too much haste, so should be discounted. Sorry 
about that :)

Robin.

> +		arm_smmu_tlb_inv_range_asid(start, size, smmu_mn->cd->asid,
> +					    PAGE_SIZE, false, smmu_domain);
> +	arm_smmu_atc_inv_domain(smmu_domain, mm->pasid, start, size);
>   }
>   
>   static void arm_smmu_mm_release(struct mmu_notifier *mn, struct mm_struct *mm)
> @@ -253,6 +258,9 @@ arm_smmu_mmu_notifier_get(struct arm_smmu_domain *smmu_domain,
>   	smmu_mn->domain = smmu_domain;
>   	smmu_mn->mn.ops = &arm_smmu_mmu_notifier_ops;
>   
> +	if (!(smmu_domain->smmu->features & ARM_SMMU_FEAT_BTM))
> +		smmu_mn->tlb_inv_command = true;
> +
>   	ret = mmu_notifier_register(&smmu_mn->mn, mm);
>   	if (ret) {
>   		kfree(smmu_mn);
> @@ -404,7 +412,7 @@ bool arm_smmu_sva_supported(struct arm_smmu_device *smmu)
>   	unsigned long reg, fld;
>   	unsigned long oas;
>   	unsigned long asid_bits;
> -	u32 feat_mask = ARM_SMMU_FEAT_BTM | ARM_SMMU_FEAT_COHERENCY;
> +	u32 feat_mask = ARM_SMMU_FEAT_COHERENCY;
>   
>   	if (vabits_actual == 52)
>   		feat_mask |= ARM_SMMU_FEAT_VAX;
> diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
> index a27b074d5c0c..db545834493b 100644
> --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
> +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
> @@ -2018,6 +2018,20 @@ static void arm_smmu_tlb_inv_range_domain(unsigned long iova, size_t size,
>   	arm_smmu_atc_inv_domain(smmu_domain, 0, iova, size);
>   }
>   
> +void arm_smmu_tlb_inv_range_asid(unsigned long iova, size_t size, int asid,
> +				 size_t granule, bool leaf,
> +				 struct arm_smmu_domain *smmu_domain)
> +{
> +	struct arm_smmu_cmdq_ent cmd = {
> +		.opcode	= CMDQ_OP_TLBI_NH_VA,
> +		.tlbi = {
> +			.asid	= asid,
> +			.leaf	= leaf,
> +		},
> +	};
> +	arm_smmu_tlb_inv_range(&cmd, iova, size, granule, smmu_domain);
> +}
> +
>   static void arm_smmu_tlb_inv_page_nosync(struct iommu_iotlb_gather *gather,
>   					 unsigned long iova, size_t granule,
>   					 void *cookie)
> 

_______________________________________________
linux-arm-kernel mailing list
linux-arm-kernel@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel

Re: [PATCH 2/3] iommu/arm-smmu-v3: Make BTM optional for SVA

[Jean-Philippe Brucker]

On Fri, Jan 22, 2021 at 02:04:55PM +0000, Robin Murphy wrote:
> > @@ -182,9 +183,13 @@ static void arm_smmu_mm_invalidate_range(struct mmu_notifier *mn,
> >   					 unsigned long start, unsigned long end)
> >   {
> >   	struct arm_smmu_mmu_notifier *smmu_mn = mn_to_smmu(mn);
> > +	struct arm_smmu_domain *smmu_domain = smmu_mn->domain;
> > +	size_t size = end - start + 1;
> > -	arm_smmu_atc_inv_domain(smmu_mn->domain, mm->pasid, start,
> > -				end - start + 1);
> > +	if (smmu_mn->tlb_inv_command)
> 
> Since we're going to be drilling down to smmu_domain->smmu->features in the
> invalidate call anyway, perhaps we could just test for BTM directly here?

Yes even with BTM enabled we'll still check features in atc_inv_domain(),
so this shortcut isn't useful.

Thanks,
Jean

_______________________________________________
linux-arm-kernel mailing list
linux-arm-kernel@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel

[PATCH 3/3] iommu/arm-smmu-v3: Add support for VHE

[Jean-Philippe Brucker]

ARMv8.1 extensions added Virtualization Host Extensions (VHE), which allow
to run a host kernel at EL2. When using normal DMA, Device and CPU address
spaces are dissociated, and do not need to implement the same
capabilities, so VHE hasn't been used in the SMMU until now.

With shared address spaces however, ASIDs are shared between MMU and SMMU,
and broadcast TLB invalidations issued by a CPU are taken into account by
the SMMU. TLB entries on both sides need to have identical exception level
in order to be cleared with a single invalidation.

When the CPU is using VHE, enable VHE in the SMMU for all STEs. Normal DMA
mappings will need to use TLBI_EL2 commands instead of TLBI_NH, but
shouldn't be otherwise affected by this change.

Acked-by: Will Deacon <will@kernel.org>
Signed-off-by: Jean-Philippe Brucker <jean-philippe@linaro.org>
---
 drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h |  3 ++
 drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 32 ++++++++++++++++-----
 2 files changed, 28 insertions(+), 7 deletions(-)

diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h
index 652d03ad8ae6..c01a65b4ae14 100644
--- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h
+++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h
@@ -454,6 +454,8 @@ struct arm_smmu_cmdq_ent {
 		#define CMDQ_OP_TLBI_NH_ASID	0x11
 		#define CMDQ_OP_TLBI_NH_VA	0x12
 		#define CMDQ_OP_TLBI_EL2_ALL	0x20
+		#define CMDQ_OP_TLBI_EL2_ASID	0x21
+		#define CMDQ_OP_TLBI_EL2_VA	0x22
 		#define CMDQ_OP_TLBI_S12_VMALL	0x28
 		#define CMDQ_OP_TLBI_S2_IPA	0x2a
 		#define CMDQ_OP_TLBI_NSNH_ALL	0x30
@@ -639,6 +641,7 @@ struct arm_smmu_device {
 #define ARM_SMMU_FEAT_RANGE_INV		(1 << 15)
 #define ARM_SMMU_FEAT_BTM		(1 << 16)
 #define ARM_SMMU_FEAT_SVA		(1 << 17)
+#define ARM_SMMU_FEAT_E2H		(1 << 18)
 	u32				features;
 
 #define ARM_SMMU_OPT_SKIP_PREFETCH	(1 << 0)
diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
index db545834493b..be762f8c1bc5 100644
--- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
+++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
@@ -273,9 +273,11 @@ static int arm_smmu_cmdq_build_cmd(u64 *cmd, struct arm_smmu_cmdq_ent *ent)
 		cmd[1] |= FIELD_PREP(CMDQ_CFGI_1_RANGE, 31);
 		break;
 	case CMDQ_OP_TLBI_NH_VA:
+		cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_VMID, ent->tlbi.vmid);
+		fallthrough;
+	case CMDQ_OP_TLBI_EL2_VA:
 		cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_NUM, ent->tlbi.num);
 		cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_SCALE, ent->tlbi.scale);
-		cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_VMID, ent->tlbi.vmid);
 		cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_ASID, ent->tlbi.asid);
 		cmd[1] |= FIELD_PREP(CMDQ_TLBI_1_LEAF, ent->tlbi.leaf);
 		cmd[1] |= FIELD_PREP(CMDQ_TLBI_1_TTL, ent->tlbi.ttl);
@@ -297,6 +299,9 @@ static int arm_smmu_cmdq_build_cmd(u64 *cmd, struct arm_smmu_cmdq_ent *ent)
 	case CMDQ_OP_TLBI_S12_VMALL:
 		cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_VMID, ent->tlbi.vmid);
 		break;
+	case CMDQ_OP_TLBI_EL2_ASID:
+		cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_ASID, ent->tlbi.asid);
+		break;
 	case CMDQ_OP_ATC_INV:
 		cmd[0] |= FIELD_PREP(CMDQ_0_SSV, ent->substream_valid);
 		cmd[0] |= FIELD_PREP(CMDQ_ATC_0_GLOBAL, ent->atc.global);
@@ -945,7 +950,8 @@ static int arm_smmu_page_response(struct device *dev,
 void arm_smmu_tlb_inv_asid(struct arm_smmu_device *smmu, u16 asid)
 {
 	struct arm_smmu_cmdq_ent cmd = {
-		.opcode = CMDQ_OP_TLBI_NH_ASID,
+		.opcode	= smmu->features & ARM_SMMU_FEAT_E2H ?
+			CMDQ_OP_TLBI_EL2_ASID : CMDQ_OP_TLBI_NH_ASID,
 		.tlbi.asid = asid,
 	};
 
@@ -1326,13 +1332,16 @@ static void arm_smmu_write_strtab_ent(struct arm_smmu_master *master, u32 sid,
 	}
 
 	if (s1_cfg) {
+		u64 strw = smmu->features & ARM_SMMU_FEAT_E2H ?
+			STRTAB_STE_1_STRW_EL2 : STRTAB_STE_1_STRW_NSEL1;
+
 		BUG_ON(ste_live);
 		dst[1] = cpu_to_le64(
 			 FIELD_PREP(STRTAB_STE_1_S1DSS, STRTAB_STE_1_S1DSS_SSID0) |
 			 FIELD_PREP(STRTAB_STE_1_S1CIR, STRTAB_STE_1_S1C_CACHE_WBRA) |
 			 FIELD_PREP(STRTAB_STE_1_S1COR, STRTAB_STE_1_S1C_CACHE_WBRA) |
 			 FIELD_PREP(STRTAB_STE_1_S1CSH, ARM_SMMU_SH_ISH) |
-			 FIELD_PREP(STRTAB_STE_1_STRW, STRTAB_STE_1_STRW_NSEL1));
+			 FIELD_PREP(STRTAB_STE_1_STRW, strw));
 
 		if (master->prg_resp_needs_ssid)
 			dst[1] |= cpu_to_le64(STRTAB_STE_1_PPAR);
@@ -2003,7 +2012,8 @@ static void arm_smmu_tlb_inv_range_domain(unsigned long iova, size_t size,
 	};
 
 	if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1) {
-		cmd.opcode	= CMDQ_OP_TLBI_NH_VA;
+		cmd.opcode	= smmu_domain->smmu->features & ARM_SMMU_FEAT_E2H ?
+				  CMDQ_OP_TLBI_EL2_VA : CMDQ_OP_TLBI_NH_VA;
 		cmd.tlbi.asid	= smmu_domain->s1_cfg.cd.asid;
 	} else {
 		cmd.opcode	= CMDQ_OP_TLBI_S2_IPA;
@@ -2023,7 +2033,8 @@ void arm_smmu_tlb_inv_range_asid(unsigned long iova, size_t size, int asid,
 				 struct arm_smmu_domain *smmu_domain)
 {
 	struct arm_smmu_cmdq_ent cmd = {
-		.opcode	= CMDQ_OP_TLBI_NH_VA,
+		.opcode	= smmu_domain->smmu->features & ARM_SMMU_FEAT_E2H ?
+			  CMDQ_OP_TLBI_EL2_VA : CMDQ_OP_TLBI_NH_VA,
 		.tlbi = {
 			.asid	= asid,
 			.leaf	= leaf,
@@ -3544,7 +3555,11 @@ static int arm_smmu_device_reset(struct arm_smmu_device *smmu, bool bypass)
 	writel_relaxed(reg, smmu->base + ARM_SMMU_CR1);
 
 	/* CR2 (random crap) */
-	reg = CR2_PTM | CR2_RECINVSID | CR2_E2H;
+	reg = CR2_PTM | CR2_RECINVSID;
+
+	if (smmu->features & ARM_SMMU_FEAT_E2H)
+		reg |= CR2_E2H;
+
 	writel_relaxed(reg, smmu->base + ARM_SMMU_CR2);
 
 	/* Stream table */
@@ -3705,8 +3720,11 @@ static int arm_smmu_device_hw_probe(struct arm_smmu_device *smmu)
 			smmu->options |= ARM_SMMU_OPT_MSIPOLL;
 	}
 
-	if (reg & IDR0_HYP)
+	if (reg & IDR0_HYP) {
 		smmu->features |= ARM_SMMU_FEAT_HYP;
+		if (cpus_have_cap(ARM64_HAS_VIRT_HOST_EXTN))
+			smmu->features |= ARM_SMMU_FEAT_E2H;
+	}
 
 	/*
 	 * The coherency feature as set by FW is used in preference to the ID
-- 
2.30.0


_______________________________________________
linux-arm-kernel mailing list
linux-arm-kernel@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel

Re: [PATCH 0/3] iommu/arm-smmu-v3: TLB invalidation for SVA

[Jonathan Cameron]

On Fri, 22 Jan 2021 12:52:55 +0100
Jean-Philippe Brucker <jean-philippe@linaro.org> wrote:

> To support sharing page tables with the CPU, the SMMU can participate in
> Broadcast TLB Maintenance (BTM), where TLB invalidate instructions from
> the CPU are received by the SMMU. For platforms that do no implement BTM
> [1], it is still possible to use SVA, by sending all TLB invalidations
> through the command queue. Patch 2 implements this.
> 
> This series also enables SVA for platforms that do support BTM, as an
> intermediate step because properly supporting BTM requires cooperating
> with KVM to allocate VMIDs [2]. With BTM enabled, the SMMU applies
> broadcast invalidations by VMID to any matching TLB entry, because there
> is no distinction between private and shared VMIDs like there is for
> ASIDs. Therefore a stage-2 domain will need a VMID that doesn't conflict
> with one allocated by KVM (or use the one from the corresponding VM,
> pinned).
> 
> These patches, along with the IOPF series [3] and the quirks [4], enable
> SVA for the hisi accelerator that's already supported upstream. My quick
> performance comparison between BTM and !BTM on that platform were
> inconclusive. Doing invalidations via cmdq seemed to slightly reduce
> performance of some heavy compression jobs, but there was too much noise
> and not enough invalidations in my tests.
> 
> This series does not depend on the IOPF one [3].
> 
> [1] https://lore.kernel.org/linux-iommu/BY5PR12MB37641E84D516054387FEE330B3CC0@BY5PR12MB3764.namprd12.prod.outlook.com/
> [2] https://lore.kernel.org/linux-iommu/20200522101755.GA3453945@myrica/
> [3] https://lore.kernel.org/linux-iommu/20210121123623.2060416-1-jean-philippe@linaro.org/
> [4] https://lore.kernel.org/linux-pci/1610960316-28935-1-git-send-email-zhangfei.gao@linaro.org/

Whole series looks good to me so FWIW
Reviewed-by: Jonathan Cameron <Jonathan.Cameron@huawei.com>

Will be very nice to have mainline support for SVA with those accelerators :)

> 
> Jean-Philippe Brucker (3):
>   iommu/arm-smmu-v3: Split arm_smmu_tlb_inv_range()
>   iommu/arm-smmu-v3: Make BTM optional for SVA
>   iommu/arm-smmu-v3: Add support for VHE
> 
>  drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h   |   6 +
>  .../iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c   |  14 ++-
>  drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c   | 104 ++++++++++++------
>  3 files changed, 89 insertions(+), 35 deletions(-)
> 


_______________________________________________
linux-arm-kernel mailing list
linux-arm-kernel@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel