From: Jean-Philippe Brucker <jean-philippe@linaro.org>
To: iommu@lists.linux-foundation.org, devicetree@vger.kernel.org,
linux-arm-kernel@lists.infradead.org, linux-pci@vger.kernel.org,
linux-mm@kvack.org
Cc: fenghua.yu@intel.com, kevin.tian@intel.com,
jacob.jun.pan@linux.intel.com, jgg@ziepe.ca,
catalin.marinas@arm.com, joro@8bytes.org, robin.murphy@arm.com,
hch@infradead.org, zhangfei.gao@linaro.org,
Jean-Philippe Brucker <jean-philippe@linaro.org>,
Jonathan.Cameron@huawei.com, felix.kuehling@amd.com,
xuzaibo@huawei.com, will@kernel.org, christian.koenig@amd.com,
baolu.lu@linux.intel.com
Subject: [PATCH v6 18/25] iommu/arm-smmu-v3: Hook up ATC invalidation to mm ops
Date: Thu, 30 Apr 2020 16:34:17 +0200 [thread overview]
Message-ID: <20200430143424.2787566-19-jean-philippe@linaro.org> (raw)
In-Reply-To: <20200430143424.2787566-1-jean-philippe@linaro.org>
The invalidate_range() notifier is called for any change to the address
space. Perform the required ATC invalidations.
Signed-off-by: Jean-Philippe Brucker <jean-philippe@linaro.org>
---
drivers/iommu/arm-smmu-v3.c | 56 ++++++++++++++++++++++++++++++-------
1 file changed, 46 insertions(+), 10 deletions(-)
diff --git a/drivers/iommu/arm-smmu-v3.c b/drivers/iommu/arm-smmu-v3.c
index 00e5b69bb81a5..c65937d953b5f 100644
--- a/drivers/iommu/arm-smmu-v3.c
+++ b/drivers/iommu/arm-smmu-v3.c
@@ -742,7 +742,7 @@ struct arm_smmu_mmu_notifier {
struct arm_smmu_ctx_desc *cd;
bool cleared;
refcount_t refs;
- struct arm_smmu_domain *domain;
+ struct arm_smmu_domain __rcu *domain;
};
#define mn_to_smmu(mn) container_of(mn, struct arm_smmu_mmu_notifier, mn)
@@ -2396,6 +2396,20 @@ arm_smmu_atc_inv_to_cmd(int ssid, unsigned long iova, size_t size,
size_t inval_grain_shift = 12;
unsigned long page_start, page_end;
+ /*
+ * ATS and PASID:
+ *
+ * If substream_valid is clear, the PCIe TLP is sent without a PASID
+ * prefix. In that case all ATC entries within the address range are
+ * invalidated, including those that were requested with a PASID! There
+ * is no way to invalidate only entries without PASID.
+ *
+ * When using STRTAB_STE_1_S1DSS_SSID0 (reserving CD 0 for non-PASID
+ * traffic), translation requests without PASID create ATC entries
+ * without PASID, which must be invalidated with substream_valid clear.
+ * This has the unpleasant side-effect of invalidating all PASID-tagged
+ * ATC entries within the address range.
+ */
*cmd = (struct arm_smmu_cmdq_ent) {
.opcode = CMDQ_OP_ATC_INV,
.substream_valid = !!ssid,
@@ -2439,12 +2453,12 @@ arm_smmu_atc_inv_to_cmd(int ssid, unsigned long iova, size_t size,
cmd->atc.size = log2_span;
}
-static int arm_smmu_atc_inv_master(struct arm_smmu_master *master)
+static int arm_smmu_atc_inv_master(struct arm_smmu_master *master, int ssid)
{
int i;
struct arm_smmu_cmdq_ent cmd;
- arm_smmu_atc_inv_to_cmd(0, 0, 0, &cmd);
+ arm_smmu_atc_inv_to_cmd(ssid, 0, 0, &cmd);
for (i = 0; i < master->num_sids; i++) {
cmd.atc.sid = master->sids[i];
@@ -2958,7 +2972,7 @@ static void arm_smmu_disable_ats(struct arm_smmu_master *master)
* ATC invalidation via the SMMU.
*/
wmb();
- arm_smmu_atc_inv_master(master);
+ arm_smmu_atc_inv_master(master, 0);
atomic_dec(&smmu_domain->nr_ats_masters);
}
@@ -3187,7 +3201,22 @@ static void arm_smmu_mm_invalidate_range(struct mmu_notifier *mn,
struct mm_struct *mm,
unsigned long start, unsigned long end)
{
- /* TODO: invalidate ATS */
+ struct arm_smmu_mmu_notifier *smmu_mn = mn_to_smmu(mn);
+ struct arm_smmu_domain *smmu_domain;
+
+ rcu_read_lock();
+ smmu_domain = rcu_dereference(smmu_mn->domain);
+ if (smmu_domain) {
+ /*
+ * Ensure that mm->pasid is valid. Pairs with the
+ * smp_store_release() from rcu_assign_pointer() in
+ * __arm_smmu_sva_bind()
+ */
+ smp_rmb();
+ arm_smmu_atc_inv_domain(smmu_domain, mm->pasid, start,
+ end - start + 1);
+ }
+ rcu_read_unlock();
}
static void arm_smmu_mm_release(struct mmu_notifier *mn, struct mm_struct *mm)
@@ -3201,7 +3230,8 @@ static void arm_smmu_mm_release(struct mmu_notifier *mn, struct mm_struct *mm)
return;
}
- smmu_domain = smmu_mn->domain;
+ smmu_domain = rcu_dereference_protected(smmu_mn->domain,
+ lockdep_is_held(&arm_smmu_sva_lock));
/*
* DMA may still be running. Keep the cd valid but disable
@@ -3210,7 +3240,7 @@ static void arm_smmu_mm_release(struct mmu_notifier *mn, struct mm_struct *mm)
arm_smmu_write_ctx_desc(smmu_domain, mm->pasid, &invalid_cd);
arm_smmu_tlb_inv_asid(smmu_domain->smmu, smmu_mn->cd->asid);
- /* TODO: invalidate ATS */
+ arm_smmu_atc_inv_domain(smmu_domain, mm->pasid, 0, 0);
smmu_mn->cleared = true;
mutex_unlock(&arm_smmu_sva_lock);
@@ -3251,7 +3281,8 @@ __arm_smmu_sva_bind(struct device *dev, struct mm_struct *mm)
return ERR_CAST(mn);
smmu_mn = mn_to_smmu(mn);
- if (smmu_mn->domain)
+ if (rcu_dereference_protected(smmu_mn->domain,
+ lockdep_is_held(&arm_smmu_sva_lock)))
refcount_inc(&smmu_mn->refs);
bond = kzalloc(sizeof(*bond), GFP_KERNEL);
@@ -3277,7 +3308,11 @@ __arm_smmu_sva_bind(struct device *dev, struct mm_struct *mm)
bond->sva.dev = dev;
list_add(&bond->list, &master->bonds);
- smmu_mn->domain = smmu_domain;
+ /*
+ * Initialize domain last, since the invalidate() notifier assumes a
+ * valid mm->pasid after fetching a valid domain.
+ */
+ rcu_assign_pointer(smmu_mn->domain, smmu_domain);
return &bond->sva;
err_free_pasid:
@@ -3318,7 +3353,8 @@ static void __arm_smmu_sva_unbind(struct iommu_sva *handle)
if (!smmu_mn->cleared) {
arm_smmu_tlb_inv_asid(smmu_domain->smmu,
smmu_mn->cd->asid);
- /* TODO: invalidate ATS */
+ arm_smmu_atc_inv_domain(smmu_domain, bond->mm->pasid,
+ 0, 0);
}
}
--
2.26.2
_______________________________________________
linux-arm-kernel mailing list
linux-arm-kernel@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel
next prev parent reply other threads:[~2020-04-30 14:46 UTC|newest]
Thread overview: 55+ messages / expand[flat|nested] mbox.gz Atom feed top
2020-04-30 14:33 [PATCH v6 00/25] iommu: Shared Virtual Addressing for SMMUv3 Jean-Philippe Brucker
2020-04-30 14:34 ` [PATCH v6 01/25] mm: Add a PASID field to mm_struct Jean-Philippe Brucker
2020-05-04 1:52 ` Xu Zaibo
2020-05-04 16:29 ` Jean-Philippe Brucker
2020-04-30 14:34 ` [PATCH v6 02/25] iommu/ioasid: Add ioasid references Jean-Philippe Brucker
2020-04-30 18:39 ` Jacob Pan
2020-04-30 20:48 ` Jacob Pan
2020-05-04 14:39 ` Jean-Philippe Brucker
2020-05-04 15:22 ` Jacob Pan
2020-05-04 14:25 ` Jean-Philippe Brucker
2020-05-04 15:27 ` Jacob Pan
2020-04-30 14:34 ` [PATCH v6 03/25] iommu/sva: Add PASID helpers Jean-Philippe Brucker
2020-04-30 14:34 ` [PATCH v6 04/25] iommu: Add a page fault handler Jean-Philippe Brucker
2020-05-03 5:49 ` Lu Baolu
2020-05-04 16:22 ` Jean-Philippe Brucker
2020-04-30 14:34 ` [PATCH v6 05/25] iommu/iopf: Handle mm faults Jean-Philippe Brucker
2020-05-03 5:54 ` Lu Baolu
2020-05-04 16:25 ` Jean-Philippe Brucker
2020-04-30 14:34 ` [PATCH v6 06/25] arm64: mm: Add asid_gen_match() helper Jean-Philippe Brucker
2020-04-30 14:34 ` [PATCH v6 07/25] arm64: mm: Pin down ASIDs for sharing mm with devices Jean-Philippe Brucker
2020-04-30 14:34 ` [PATCH v6 08/25] iommu/io-pgtable-arm: Move some definitions to a header Jean-Philippe Brucker
2020-04-30 14:34 ` [PATCH v6 09/25] iommu/arm-smmu-v3: Manage ASIDs with xarray Jean-Philippe Brucker
2020-04-30 14:34 ` [PATCH v6 10/25] arm64: cpufeature: Export symbol read_sanitised_ftr_reg() Jean-Philippe Brucker
2020-04-30 15:18 ` Suzuki K Poulose
2020-04-30 14:34 ` [PATCH v6 11/25] iommu/arm-smmu-v3: Share process page tables Jean-Philippe Brucker
2020-04-30 15:39 ` Suzuki K Poulose
2020-05-04 14:11 ` Jean-Philippe Brucker
2020-05-04 14:42 ` Suzuki K Poulose
2020-04-30 14:34 ` [PATCH v6 12/25] iommu/arm-smmu-v3: Seize private ASID Jean-Philippe Brucker
2020-04-30 14:34 ` [PATCH v6 13/25] iommu/arm-smmu-v3: Add support for VHE Jean-Philippe Brucker
2020-04-30 14:34 ` [PATCH v6 14/25] iommu/arm-smmu-v3: Enable broadcast TLB maintenance Jean-Philippe Brucker
2020-04-30 14:34 ` [PATCH v6 15/25] iommu/arm-smmu-v3: Add SVA feature checking Jean-Philippe Brucker
2020-04-30 14:34 ` [PATCH v6 16/25] iommu/arm-smmu-v3: Add SVA device feature Jean-Philippe Brucker
2020-04-30 14:34 ` [PATCH v6 17/25] iommu/arm-smmu-v3: Implement iommu_sva_bind/unbind() Jean-Philippe Brucker
2020-04-30 21:16 ` Jacob Pan
2020-05-04 16:43 ` Jean-Philippe Brucker
2020-05-04 20:47 ` Jacob Pan
2020-05-05 9:15 ` Jean-Philippe Brucker
2020-05-07 16:31 ` Jacob Pan
2020-05-01 12:15 ` Christoph Hellwig
2020-05-01 12:55 ` Jason Gunthorpe
2020-05-04 16:07 ` Jean-Philippe Brucker
2020-05-04 16:06 ` Jean-Philippe Brucker
2020-04-30 14:34 ` Jean-Philippe Brucker [this message]
2020-04-30 14:34 ` [PATCH v6 19/25] iommu/arm-smmu-v3: Add support for Hardware Translation Table Update Jean-Philippe Brucker
2020-05-04 14:24 ` Prabhakar Kushwaha
2020-05-04 16:35 ` Jean-Philippe Brucker
2020-04-30 14:34 ` [PATCH v6 20/25] iommu/arm-smmu-v3: Maintain a SID->device structure Jean-Philippe Brucker
2020-04-30 14:34 ` [PATCH v6 21/25] dt-bindings: document stall property for IOMMU masters Jean-Philippe Brucker
2020-04-30 14:34 ` [PATCH v6 22/25] iommu/arm-smmu-v3: Add stall support for platform devices Jean-Philippe Brucker
2020-04-30 14:34 ` [PATCH v6 23/25] PCI/ATS: Add PRI stubs Jean-Philippe Brucker
2020-04-30 14:34 ` [PATCH v6 24/25] PCI/ATS: Export PRI functions Jean-Philippe Brucker
2020-04-30 14:34 ` [PATCH v6 25/25] iommu/arm-smmu-v3: Add support for PRI Jean-Philippe Brucker
2020-04-30 21:18 ` [PATCH v6 00/25] iommu: Shared Virtual Addressing for SMMUv3 Jacob Pan
2020-05-04 15:09 ` Jean-Philippe Brucker
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20200430143424.2787566-19-jean-philippe@linaro.org \
--to=jean-philippe@linaro.org \
--cc=Jonathan.Cameron@huawei.com \
--cc=baolu.lu@linux.intel.com \
--cc=catalin.marinas@arm.com \
--cc=christian.koenig@amd.com \
--cc=devicetree@vger.kernel.org \
--cc=felix.kuehling@amd.com \
--cc=fenghua.yu@intel.com \
--cc=hch@infradead.org \
--cc=iommu@lists.linux-foundation.org \
--cc=jacob.jun.pan@linux.intel.com \
--cc=jgg@ziepe.ca \
--cc=joro@8bytes.org \
--cc=kevin.tian@intel.com \
--cc=linux-arm-kernel@lists.infradead.org \
--cc=linux-mm@kvack.org \
--cc=linux-pci@vger.kernel.org \
--cc=robin.murphy@arm.com \
--cc=will@kernel.org \
--cc=xuzaibo@huawei.com \
--cc=zhangfei.gao@linaro.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).