Re: [PATCH v13 08/15] iommu/smmuv3: Implement cache_invalidate

["chenxiang (M)" <chenxiang66@hisilicon.com>]

Hi Eric,


在 2020/11/18 19:21, Eric Auger 写道:
> Implement domain-selective and page-selective IOTLB invalidations.
>
> Signed-off-by: Eric Auger <eric.auger@redhat.com>
>
> ---
> v7 -> v8:
> - ASID based invalidation using iommu_inv_pasid_info
> - check ARCHID/PASID flags in addr based invalidation
> - use __arm_smmu_tlb_inv_context and __arm_smmu_tlb_inv_range_nosync
>
> v6 -> v7
> - check the uapi version
>
> v3 -> v4:
> - adapt to changes in the uapi
> - add support for leaf parameter
> - do not use arm_smmu_tlb_inv_range_nosync or arm_smmu_tlb_inv_context
>    anymore
>
> v2 -> v3:
> - replace __arm_smmu_tlb_sync by arm_smmu_cmdq_issue_sync
>
> v1 -> v2:
> - properly pass the asid
> ---
>   drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 53 +++++++++++++++++++++
>   1 file changed, 53 insertions(+)
>
> diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
> index fdecc9f17b36..24124361dd3b 100644
> --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
> +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
> @@ -2771,6 +2771,58 @@ static void arm_smmu_detach_pasid_table(struct iommu_domain *domain)
>   	mutex_unlock(&smmu_domain->init_mutex);
>   }
>   
> +static int
> +arm_smmu_cache_invalidate(struct iommu_domain *domain, struct device *dev,
> +			  struct iommu_cache_invalidate_info *inv_info)
> +{
> +	struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
> +	struct arm_smmu_device *smmu = smmu_domain->smmu;
> +
> +	if (smmu_domain->stage != ARM_SMMU_DOMAIN_NESTED)
> +		return -EINVAL;
> +
> +	if (!smmu)
> +		return -EINVAL;
> +
> +	if (inv_info->version != IOMMU_CACHE_INVALIDATE_INFO_VERSION_1)
> +		return -EINVAL;
> +
> +	if (inv_info->cache & IOMMU_CACHE_INV_TYPE_IOTLB) {
> +		if (inv_info->granularity == IOMMU_INV_GRANU_PASID) {
> +			struct iommu_inv_pasid_info *info =
> +				&inv_info->granu.pasid_info;
> +
> +			if (!(info->flags & IOMMU_INV_PASID_FLAGS_ARCHID) ||
> +			     (info->flags & IOMMU_INV_PASID_FLAGS_PASID))
> +				return -EINVAL;
> +
> +			__arm_smmu_tlb_inv_context(smmu_domain, info->archid);
> +
> +		} else if (inv_info->granularity == IOMMU_INV_GRANU_ADDR) {
> +			struct iommu_inv_addr_info *info = &inv_info->granu.addr_info;
> +			size_t size = info->nb_granules * info->granule_size;
> +			bool leaf = info->flags & IOMMU_INV_ADDR_FLAGS_LEAF;
> +
> +			if (!(info->flags & IOMMU_INV_ADDR_FLAGS_ARCHID) ||
> +			     (info->flags & IOMMU_INV_ADDR_FLAGS_PASID))
> +				return -EINVAL;
> +
> +			__arm_smmu_tlb_inv_range(info->addr, size,
> +						 info->granule_size, leaf,
> +						  smmu_domain, info->archid);

When debugging with vSMMU on ARM64 huawei platform,  there is a issue:  
RIL feature is enabled on guest OS while it is not supported on host 
OS,  with some operations
(such as rmmod driver during iperf between guest and host), SMMU 
translation error occurs frequently, and it works well if RIL feature is 
disabled on guest OS.
We find that in function vfio_iommu_unmap_notify() (qemu code) it passes 
total size of tlb (num_pages * granule_size) as granule size to host OS :
addr_info->granules_size = size
addr_info->nb_granule = 1
So total size (num_pages * granule_size) is passed as granule size to 
function __arm_smmu_inv_range(), if RIL feature is not supported, then 
total size may be not
the granule size that host OS can recongize, i will only invalidate part 
of tlb it wants to invalidate (for example, 4K/2M/1G pagesize is 
supported on host OS,  if total_size = 8K, then addr_info->granule_size 
= 8K,
addr_info->nb_granule = 1, as RIL feature is not supported, it 
invalidates just 4K one time but it thought it had invalidated 8K).
Please have a check of the issue, and we have a temporary fix as follows 
on the issue.

hw/arm/smmuv3.c       | 3 ++-
  hw/vfio/common.c      | 6 +++---
  include/exec/memory.h | 1 +
  3 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/hw/arm/smmuv3.c b/hw/arm/smmuv3.c
index 6725019..891a65d 100644
--- a/hw/arm/smmuv3.c
+++ b/hw/arm/smmuv3.c
@@ -821,7 +821,8 @@ static void smmuv3_notify_iova(IOMMUMemoryRegion *mr,

      entry.target_as = &address_space_memory;
      entry.iova = iova;
-    entry.addr_mask = num_pages * (1 << granule) - 1;
+    entry.addr_mask = (1 << granule) - 1;
+    entry.num_pages = num_pages;
      entry.perm = IOMMU_NONE;
      entry.arch_id = asid;
      entry.flags = IOMMU_INV_GRANU_ADDR;
diff --git a/hw/vfio/common.c b/hw/vfio/common.c
index 5d365e0..c0164b1 100644
--- a/hw/vfio/common.c
+++ b/hw/vfio/common.c
@@ -604,7 +604,7 @@ static void vfio_iommu_unmap_notify(IOMMUNotifier 
*n, IOMMUTLBEntry *iotlb)
      hwaddr start = iotlb->iova + giommu->iommu_offset;
      VFIOContainer *container = giommu->container;
      struct vfio_iommu_type1_cache_invalidate ustruct = {};
-    size_t size = iotlb->addr_mask + 1;
+    size_t size = iotlb->num_pages * (iotlb->addr_mask + 1);
      int ret;

      assert(iotlb->perm == IOMMU_NONE);
@@ -632,8 +632,8 @@ static void vfio_iommu_unmap_notify(IOMMUNotifier 
*n, IOMMUTLBEntry *iotlb)
          }
          addr_info->archid = iotlb->arch_id;
          addr_info->addr = start;
-        addr_info->granule_size = size;
-        addr_info->nb_granules = 1;
+        addr_info->granule_size = iotlb->addr_mask + 1;
+        addr_info->nb_granules = iotlb->num_pages;
      }
      trace_vfio_iommu_addr_inv_iotlb(iotlb->arch_id, start, size, 1,
                                      iotlb->leaf);
diff --git a/include/exec/memory.h b/include/exec/memory.h
index 21959a7..aa8d43e 100644
--- a/include/exec/memory.h
+++ b/include/exec/memory.h
@@ -103,6 +103,7 @@ struct IOMMUTLBEntry {
      hwaddr           iova;
      hwaddr           translated_addr;
      hwaddr           addr_mask;
+    uint64_t         num_pages;
      IOMMUAccessFlags perm;
      uint32_t         arch_id;
      uint32_t         flags;



_______________________________________________
iommu mailing list
iommu@lists.linux-foundation.org
https://lists.linuxfoundation.org/mailman/listinfo/iommu