[PATCH] iommu/dma: Resurrect the "forcedac" option

[PATCH] iommu/dma: Resurrect the "forcedac" option

[Robin Murphy]

In converting intel-iommu over to the common IOMMU DMA ops, it quietly
lost the functionality of its "forcedac" option. Since this is a handy
thing both for testing and for performance optimisation on certain
platforms, reimplement it under the common IOMMU parameter namespace.

For the sake of fixing the inadvertent breakage of the Intel-specific
parameter, remove the dmar_forcedac remnants and hook it up as an alias
while documenting the transition to the new common parameter.

Fixes: c588072bba6b ("iommu/vt-d: Convert intel iommu driver to the iommu ops")
Signed-off-by: Robin Murphy <robin.murphy@arm.com>
---
 Documentation/admin-guide/kernel-parameters.txt | 15 ++++++++-------
 drivers/iommu/dma-iommu.c                       | 13 ++++++++++++-
 drivers/iommu/intel/iommu.c                     |  5 ++---
 include/linux/dma-iommu.h                       |  2 ++
 4 files changed, 24 insertions(+), 11 deletions(-)

diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt
index 04545725f187..835f810f2f26 100644
--- a/Documentation/admin-guide/kernel-parameters.txt
+++ b/Documentation/admin-guide/kernel-parameters.txt
@@ -1869,13 +1869,6 @@
 			bypassed by not enabling DMAR with this option. In
 			this case, gfx device will use physical address for
 			DMA.
-		forcedac [X86-64]
-			With this option iommu will not optimize to look
-			for io virtual address below 32-bit forcing dual
-			address cycle on pci bus for cards supporting greater
-			than 32-bit addressing. The default is to look
-			for translation below 32-bit and if not available
-			then look in the higher range.
 		strict [Default Off]
 			With this option on every unmap_single operation will
 			result in a hardware IOTLB flush operation as opposed
@@ -1964,6 +1957,14 @@
 		nobypass	[PPC/POWERNV]
 			Disable IOMMU bypass, using IOMMU for PCI devices.
 
+	iommu.forcedac=	[ARM64, X86] Control IOVA allocation for PCI devices.
+			Format: { "0" | "1" }
+			0 - Try to allocate a 32-bit DMA address first, before
+			  falling back to the full range if needed.
+			1 - Allocate directly from the full usable range,
+			  forcing Dual Address Cycle for PCI cards supporting
+			  greater than 32-bit addressing.
+
 	iommu.strict=	[ARM64] Configure TLB invalidation behaviour
 			Format: { "0" | "1" }
 			0 - Lazy mode.
diff --git a/drivers/iommu/dma-iommu.c b/drivers/iommu/dma-iommu.c
index 9ab6ee22c110..260bf3de1992 100644
--- a/drivers/iommu/dma-iommu.c
+++ b/drivers/iommu/dma-iommu.c
@@ -52,6 +52,17 @@ struct iommu_dma_cookie {
 };
 
 static DEFINE_STATIC_KEY_FALSE(iommu_deferred_attach_enabled);
+bool iommu_dma_forcedac __read_mostly;
+
+static int __init iommu_dma_forcedac_setup(char *str)
+{
+	int ret = kstrtobool(str, &iommu_dma_forcedac);
+
+	if (!ret && iommu_dma_forcedac)
+		pr_info("Forcing DAC for PCI devices\n");
+	return ret;
+}
+early_param("iommu.forcedac", iommu_dma_forcedac_setup);
 
 void iommu_dma_free_cpu_cached_iovas(unsigned int cpu,
 		struct iommu_domain *domain)
@@ -438,7 +449,7 @@ static dma_addr_t iommu_dma_alloc_iova(struct iommu_domain *domain,
 		dma_limit = min(dma_limit, (u64)domain->geometry.aperture_end);
 
 	/* Try to get PCI devices a SAC address */
-	if (dma_limit > DMA_BIT_MASK(32) && dev_is_pci(dev))
+	if (dma_limit > DMA_BIT_MASK(32) && !iommu_dma_forcedac && dev_is_pci(dev))
 		iova = alloc_iova_fast(iovad, iova_len,
 				       DMA_BIT_MASK(32) >> shift, false);
 
diff --git a/drivers/iommu/intel/iommu.c b/drivers/iommu/intel/iommu.c
index ee0932307d64..1c32522220bc 100644
--- a/drivers/iommu/intel/iommu.c
+++ b/drivers/iommu/intel/iommu.c
@@ -360,7 +360,6 @@ int intel_iommu_enabled = 0;
 EXPORT_SYMBOL_GPL(intel_iommu_enabled);
 
 static int dmar_map_gfx = 1;
-static int dmar_forcedac;
 static int intel_iommu_strict;
 static int intel_iommu_superpage = 1;
 static int iommu_identity_mapping;
@@ -451,8 +450,8 @@ static int __init intel_iommu_setup(char *str)
 			dmar_map_gfx = 0;
 			pr_info("Disable GFX device mapping\n");
 		} else if (!strncmp(str, "forcedac", 8)) {
-			pr_info("Forcing DAC for PCI devices\n");
-			dmar_forcedac = 1;
+			pr_warn("intel_iommu=forcedac deprecated; use iommu.forcedac instead\n");
+			iommu_dma_forcedac = true;
 		} else if (!strncmp(str, "strict", 6)) {
 			pr_info("Disable batched IOTLB flush\n");
 			intel_iommu_strict = 1;
diff --git a/include/linux/dma-iommu.h b/include/linux/dma-iommu.h
index 706b68d1359b..13d1f4c14d7b 100644
--- a/include/linux/dma-iommu.h
+++ b/include/linux/dma-iommu.h
@@ -40,6 +40,8 @@ void iommu_dma_get_resv_regions(struct device *dev, struct list_head *list);
 void iommu_dma_free_cpu_cached_iovas(unsigned int cpu,
 		struct iommu_domain *domain);
 
+extern bool iommu_dma_forcedac;
+
 #else /* CONFIG_IOMMU_DMA */
 
 struct iommu_domain;
-- 
2.17.1

Re: [PATCH] iommu/dma: Resurrect the "forcedac" option

[John Garry]

On 05/03/2021 16:32, Robin Murphy wrote:
> In converting intel-iommu over to the common IOMMU DMA ops, it quietly
> lost the functionality of its "forcedac" option. Since this is a handy
> thing both for testing and for performance optimisation on certain
> platforms, reimplement it under the common IOMMU parameter namespace.
> 
> For the sake of fixing the inadvertent breakage of the Intel-specific
> parameter, remove the dmar_forcedac remnants and hook it up as an alias
> while documenting the transition to the new common parameter.
> 

Do you think that having a kconfig option to control the default for 
this can help identify the broken platforms which rely on forcedac=0? 
But seems a bit trivial for that, though.

Or are we bothered (finding them)?

Thanks,
john

> Fixes: c588072bba6b ("iommu/vt-d: Convert intel iommu driver to the iommu ops")
> Signed-off-by: Robin Murphy <robin.murphy@arm.com>
> ---
>   Documentation/admin-guide/kernel-parameters.txt | 15 ++++++++-------
>   drivers/iommu/dma-iommu.c                       | 13 ++++++++++++-
>   drivers/iommu/intel/iommu.c                     |  5 ++---
>   include/linux/dma-iommu.h                       |  2 ++
>   4 files changed, 24 insertions(+), 11 deletions(-)
> 
> diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt
> index 04545725f187..835f810f2f26 100644
> --- a/Documentation/admin-guide/kernel-parameters.txt
> +++ b/Documentation/admin-guide/kernel-parameters.txt
> @@ -1869,13 +1869,6 @@
>   			bypassed by not enabling DMAR with this option. In
>   			this case, gfx device will use physical address for
>   			DMA.
> -		forcedac [X86-64]
> -			With this option iommu will not optimize to look
> -			for io virtual address below 32-bit forcing dual
> -			address cycle on pci bus for cards supporting greater
> -			than 32-bit addressing. The default is to look
> -			for translation below 32-bit and if not available
> -			then look in the higher range.
>   		strict [Default Off]
>   			With this option on every unmap_single operation will
>   			result in a hardware IOTLB flush operation as opposed
> @@ -1964,6 +1957,14 @@
>   		nobypass	[PPC/POWERNV]
>   			Disable IOMMU bypass, using IOMMU for PCI devices.
>   
> +	iommu.forcedac=	[ARM64, X86] Control IOVA allocation for PCI devices.
> +			Format: { "0" | "1" }
> +			0 - Try to allocate a 32-bit DMA address first, before
> +			  falling back to the full range if needed.
> +			1 - Allocate directly from the full usable range,
> +			  forcing Dual Address Cycle for PCI cards supporting
> +			  greater than 32-bit addressing.
> +
>   	iommu.strict=	[ARM64] Configure TLB invalidation behaviour
>   			Format: { "0" | "1" }
>   			0 - Lazy mode.
> diff --git a/drivers/iommu/dma-iommu.c b/drivers/iommu/dma-iommu.c
> index 9ab6ee22c110..260bf3de1992 100644
> --- a/drivers/iommu/dma-iommu.c
> +++ b/drivers/iommu/dma-iommu.c
> @@ -52,6 +52,17 @@ struct iommu_dma_cookie {
>   };
>   
>   static DEFINE_STATIC_KEY_FALSE(iommu_deferred_attach_enabled);
> +bool iommu_dma_forcedac __read_mostly;
> +
> +static int __init iommu_dma_forcedac_setup(char *str)
> +{
> +	int ret = kstrtobool(str, &iommu_dma_forcedac);
> +
> +	if (!ret && iommu_dma_forcedac)
> +		pr_info("Forcing DAC for PCI devices\n");
> +	return ret;
> +}
> +early_param("iommu.forcedac", iommu_dma_forcedac_setup);
>   
>   void iommu_dma_free_cpu_cached_iovas(unsigned int cpu,
>   		struct iommu_domain *domain)
> @@ -438,7 +449,7 @@ static dma_addr_t iommu_dma_alloc_iova(struct iommu_domain *domain,
>   		dma_limit = min(dma_limit, (u64)domain->geometry.aperture_end);
>   
>   	/* Try to get PCI devices a SAC address */
> -	if (dma_limit > DMA_BIT_MASK(32) && dev_is_pci(dev))
> +	if (dma_limit > DMA_BIT_MASK(32) && !iommu_dma_forcedac && dev_is_pci(dev))
>   		iova = alloc_iova_fast(iovad, iova_len,
>   				       DMA_BIT_MASK(32) >> shift, false);
>   
> diff --git a/drivers/iommu/intel/iommu.c b/drivers/iommu/intel/iommu.c
> index ee0932307d64..1c32522220bc 100644
> --- a/drivers/iommu/intel/iommu.c
> +++ b/drivers/iommu/intel/iommu.c
> @@ -360,7 +360,6 @@ int intel_iommu_enabled = 0;
>   EXPORT_SYMBOL_GPL(intel_iommu_enabled);
>   
>   static int dmar_map_gfx = 1;
> -static int dmar_forcedac;
>   static int intel_iommu_strict;
>   static int intel_iommu_superpage = 1;
>   static int iommu_identity_mapping;
> @@ -451,8 +450,8 @@ static int __init intel_iommu_setup(char *str)
>   			dmar_map_gfx = 0;
>   			pr_info("Disable GFX device mapping\n");
>   		} else if (!strncmp(str, "forcedac", 8)) {
> -			pr_info("Forcing DAC for PCI devices\n");
> -			dmar_forcedac = 1;
> +			pr_warn("intel_iommu=forcedac deprecated; use iommu.forcedac instead\n");
> +			iommu_dma_forcedac = true;
>   		} else if (!strncmp(str, "strict", 6)) {
>   			pr_info("Disable batched IOTLB flush\n");
>   			intel_iommu_strict = 1;
> diff --git a/include/linux/dma-iommu.h b/include/linux/dma-iommu.h
> index 706b68d1359b..13d1f4c14d7b 100644
> --- a/include/linux/dma-iommu.h
> +++ b/include/linux/dma-iommu.h
> @@ -40,6 +40,8 @@ void iommu_dma_get_resv_regions(struct device *dev, struct list_head *list);
>   void iommu_dma_free_cpu_cached_iovas(unsigned int cpu,
>   		struct iommu_domain *domain);
>   
> +extern bool iommu_dma_forcedac;
> +
>   #else /* CONFIG_IOMMU_DMA */
>   
>   struct iommu_domain;
> 

Re: [PATCH] iommu/dma: Resurrect the "forcedac" option

[Lu Baolu]

Hi Robin,

On 3/6/21 12:32 AM, Robin Murphy wrote:
> In converting intel-iommu over to the common IOMMU DMA ops, it quietly
> lost the functionality of its "forcedac" option. Since this is a handy
> thing both for testing and for performance optimisation on certain
> platforms, reimplement it under the common IOMMU parameter namespace.
> 
> For the sake of fixing the inadvertent breakage of the Intel-specific
> parameter, remove the dmar_forcedac remnants and hook it up as an alias
> while documenting the transition to the new common parameter.
> 
> Fixes: c588072bba6b ("iommu/vt-d: Convert intel iommu driver to the iommu ops")
> Signed-off-by: Robin Murphy <robin.murphy@arm.com>
> ---
>   Documentation/admin-guide/kernel-parameters.txt | 15 ++++++++-------
>   drivers/iommu/dma-iommu.c                       | 13 ++++++++++++-
>   drivers/iommu/intel/iommu.c                     |  5 ++---
>   include/linux/dma-iommu.h                       |  2 ++
>   4 files changed, 24 insertions(+), 11 deletions(-)
> 
> diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt
> index 04545725f187..835f810f2f26 100644
> --- a/Documentation/admin-guide/kernel-parameters.txt
> +++ b/Documentation/admin-guide/kernel-parameters.txt
> @@ -1869,13 +1869,6 @@
>   			bypassed by not enabling DMAR with this option. In
>   			this case, gfx device will use physical address for
>   			DMA.
> -		forcedac [X86-64]
> -			With this option iommu will not optimize to look
> -			for io virtual address below 32-bit forcing dual
> -			address cycle on pci bus for cards supporting greater
> -			than 32-bit addressing. The default is to look
> -			for translation below 32-bit and if not available
> -			then look in the higher range.
>   		strict [Default Off]
>   			With this option on every unmap_single operation will
>   			result in a hardware IOTLB flush operation as opposed
> @@ -1964,6 +1957,14 @@
>   		nobypass	[PPC/POWERNV]
>   			Disable IOMMU bypass, using IOMMU for PCI devices.
>   
> +	iommu.forcedac=	[ARM64, X86] Control IOVA allocation for PCI devices.
> +			Format: { "0" | "1" }
> +			0 - Try to allocate a 32-bit DMA address first, before
> +			  falling back to the full range if needed.
> +			1 - Allocate directly from the full usable range,
> +			  forcing Dual Address Cycle for PCI cards supporting
> +			  greater than 32-bit addressing.
> +
>   	iommu.strict=	[ARM64] Configure TLB invalidation behaviour
>   			Format: { "0" | "1" }
>   			0 - Lazy mode.
> diff --git a/drivers/iommu/dma-iommu.c b/drivers/iommu/dma-iommu.c
> index 9ab6ee22c110..260bf3de1992 100644
> --- a/drivers/iommu/dma-iommu.c
> +++ b/drivers/iommu/dma-iommu.c
> @@ -52,6 +52,17 @@ struct iommu_dma_cookie {
>   };
>   
>   static DEFINE_STATIC_KEY_FALSE(iommu_deferred_attach_enabled);
> +bool iommu_dma_forcedac __read_mostly;
> +
> +static int __init iommu_dma_forcedac_setup(char *str)
> +{
> +	int ret = kstrtobool(str, &iommu_dma_forcedac);
> +
> +	if (!ret && iommu_dma_forcedac)
> +		pr_info("Forcing DAC for PCI devices\n");
> +	return ret;
> +}
> +early_param("iommu.forcedac", iommu_dma_forcedac_setup);
>   
>   void iommu_dma_free_cpu_cached_iovas(unsigned int cpu,
>   		struct iommu_domain *domain)
> @@ -438,7 +449,7 @@ static dma_addr_t iommu_dma_alloc_iova(struct iommu_domain *domain,
>   		dma_limit = min(dma_limit, (u64)domain->geometry.aperture_end);
>   
>   	/* Try to get PCI devices a SAC address */
> -	if (dma_limit > DMA_BIT_MASK(32) && dev_is_pci(dev))
> +	if (dma_limit > DMA_BIT_MASK(32) && !iommu_dma_forcedac && dev_is_pci(dev))
>   		iova = alloc_iova_fast(iovad, iova_len,
>   				       DMA_BIT_MASK(32) >> shift, false);
>   
> diff --git a/drivers/iommu/intel/iommu.c b/drivers/iommu/intel/iommu.c
> index ee0932307d64..1c32522220bc 100644
> --- a/drivers/iommu/intel/iommu.c
> +++ b/drivers/iommu/intel/iommu.c
> @@ -360,7 +360,6 @@ int intel_iommu_enabled = 0;
>   EXPORT_SYMBOL_GPL(intel_iommu_enabled);
>   
>   static int dmar_map_gfx = 1;
> -static int dmar_forcedac;
>   static int intel_iommu_strict;
>   static int intel_iommu_superpage = 1;
>   static int iommu_identity_mapping;
> @@ -451,8 +450,8 @@ static int __init intel_iommu_setup(char *str)
>   			dmar_map_gfx = 0;
>   			pr_info("Disable GFX device mapping\n");
>   		} else if (!strncmp(str, "forcedac", 8)) {
> -			pr_info("Forcing DAC for PCI devices\n");
> -			dmar_forcedac = 1;
> +			pr_warn("intel_iommu=forcedac deprecated; use iommu.forcedac instead\n");
> +			iommu_dma_forcedac = true;
>   		} else if (!strncmp(str, "strict", 6)) {
>   			pr_info("Disable batched IOTLB flush\n");
>   			intel_iommu_strict = 1;
> diff --git a/include/linux/dma-iommu.h b/include/linux/dma-iommu.h
> index 706b68d1359b..13d1f4c14d7b 100644
> --- a/include/linux/dma-iommu.h
> +++ b/include/linux/dma-iommu.h
> @@ -40,6 +40,8 @@ void iommu_dma_get_resv_regions(struct device *dev, struct list_head *list);
>   void iommu_dma_free_cpu_cached_iovas(unsigned int cpu,
>   		struct iommu_domain *domain);
>   
> +extern bool iommu_dma_forcedac;
> +
>   #else /* CONFIG_IOMMU_DMA */
>   
>   struct iommu_domain;
> 

Thanks for catching this.

Acked-by: Lu Baolu <baolu.lu@linux.intel.com>

Best regards,
baolu

Re: [PATCH] iommu/dma: Resurrect the "forcedac" option

[Robin Murphy]

On 2021-03-05 17:41, John Garry wrote:
> On 05/03/2021 16:32, Robin Murphy wrote:
>> In converting intel-iommu over to the common IOMMU DMA ops, it quietly
>> lost the functionality of its "forcedac" option. Since this is a handy
>> thing both for testing and for performance optimisation on certain
>> platforms, reimplement it under the common IOMMU parameter namespace.
>>
>> For the sake of fixing the inadvertent breakage of the Intel-specific
>> parameter, remove the dmar_forcedac remnants and hook it up as an alias
>> while documenting the transition to the new common parameter.
>>
> 
> Do you think that having a kconfig option to control the default for 
> this can help identify the broken platforms which rely on forcedac=0? 
> But seems a bit trivial for that, though.

I think it's still a sizeable can of worms - unlike, say, 
ARM_SMMU_DISABLE_BYPASS_BY_DEFAULT, we can't actually tell when things 
have gone awry and explicitly call it out. While I was getting the 
dma-ranges right on my Juno, everything broke differently - the SATA 
controller fails gracefully; the ethernet controller got the kernel tied 
up somewhere (to the point that the USB keyboard died) once it tried to 
brink up the link, but was at least spewing regular timeout backtraces 
that implicated the networking layer; having an (unused) NVMe plugged in 
simply wedged the boot process early on with no hint whatsoever of why.

TBH I'm not really sure what the best way forward is in terms of trying 
to weed out platforms that would need quirking. Our discussion just 
reminded me of this option and that it had gone AWOL, so bringing it 
back to be potentially *some* use to everyone seems justifiable on its own.

Thanks,
Robin.

> 
> Or are we bothered (finding them)?
> 
> Thanks,
> john
> 
>> Fixes: c588072bba6b ("iommu/vt-d: Convert intel iommu driver to the 
>> iommu ops")
>> Signed-off-by: Robin Murphy <robin.murphy@arm.com>
>> ---
>>   Documentation/admin-guide/kernel-parameters.txt | 15 ++++++++-------
>>   drivers/iommu/dma-iommu.c                       | 13 ++++++++++++-
>>   drivers/iommu/intel/iommu.c                     |  5 ++---
>>   include/linux/dma-iommu.h                       |  2 ++
>>   4 files changed, 24 insertions(+), 11 deletions(-)
>>
>> diff --git a/Documentation/admin-guide/kernel-parameters.txt 
>> b/Documentation/admin-guide/kernel-parameters.txt
>> index 04545725f187..835f810f2f26 100644
>> --- a/Documentation/admin-guide/kernel-parameters.txt
>> +++ b/Documentation/admin-guide/kernel-parameters.txt
>> @@ -1869,13 +1869,6 @@
>>               bypassed by not enabling DMAR with this option. In
>>               this case, gfx device will use physical address for
>>               DMA.
>> -        forcedac [X86-64]
>> -            With this option iommu will not optimize to look
>> -            for io virtual address below 32-bit forcing dual
>> -            address cycle on pci bus for cards supporting greater
>> -            than 32-bit addressing. The default is to look
>> -            for translation below 32-bit and if not available
>> -            then look in the higher range.
>>           strict [Default Off]
>>               With this option on every unmap_single operation will
>>               result in a hardware IOTLB flush operation as opposed
>> @@ -1964,6 +1957,14 @@
>>           nobypass    [PPC/POWERNV]
>>               Disable IOMMU bypass, using IOMMU for PCI devices.
>> +    iommu.forcedac=    [ARM64, X86] Control IOVA allocation for PCI 
>> devices.
>> +            Format: { "0" | "1" }
>> +            0 - Try to allocate a 32-bit DMA address first, before
>> +              falling back to the full range if needed.
>> +            1 - Allocate directly from the full usable range,
>> +              forcing Dual Address Cycle for PCI cards supporting
>> +              greater than 32-bit addressing.
>> +
>>       iommu.strict=    [ARM64] Configure TLB invalidation behaviour
>>               Format: { "0" | "1" }
>>               0 - Lazy mode.
>> diff --git a/drivers/iommu/dma-iommu.c b/drivers/iommu/dma-iommu.c
>> index 9ab6ee22c110..260bf3de1992 100644
>> --- a/drivers/iommu/dma-iommu.c
>> +++ b/drivers/iommu/dma-iommu.c
>> @@ -52,6 +52,17 @@ struct iommu_dma_cookie {
>>   };
>>   static DEFINE_STATIC_KEY_FALSE(iommu_deferred_attach_enabled);
>> +bool iommu_dma_forcedac __read_mostly;
>> +
>> +static int __init iommu_dma_forcedac_setup(char *str)
>> +{
>> +    int ret = kstrtobool(str, &iommu_dma_forcedac);
>> +
>> +    if (!ret && iommu_dma_forcedac)
>> +        pr_info("Forcing DAC for PCI devices\n");
>> +    return ret;
>> +}
>> +early_param("iommu.forcedac", iommu_dma_forcedac_setup);
>>   void iommu_dma_free_cpu_cached_iovas(unsigned int cpu,
>>           struct iommu_domain *domain)
>> @@ -438,7 +449,7 @@ static dma_addr_t iommu_dma_alloc_iova(struct 
>> iommu_domain *domain,
>>           dma_limit = min(dma_limit, (u64)domain->geometry.aperture_end);
>>       /* Try to get PCI devices a SAC address */
>> -    if (dma_limit > DMA_BIT_MASK(32) && dev_is_pci(dev))
>> +    if (dma_limit > DMA_BIT_MASK(32) && !iommu_dma_forcedac && 
>> dev_is_pci(dev))
>>           iova = alloc_iova_fast(iovad, iova_len,
>>                          DMA_BIT_MASK(32) >> shift, false);
>> diff --git a/drivers/iommu/intel/iommu.c b/drivers/iommu/intel/iommu.c
>> index ee0932307d64..1c32522220bc 100644
>> --- a/drivers/iommu/intel/iommu.c
>> +++ b/drivers/iommu/intel/iommu.c
>> @@ -360,7 +360,6 @@ int intel_iommu_enabled = 0;
>>   EXPORT_SYMBOL_GPL(intel_iommu_enabled);
>>   static int dmar_map_gfx = 1;
>> -static int dmar_forcedac;
>>   static int intel_iommu_strict;
>>   static int intel_iommu_superpage = 1;
>>   static int iommu_identity_mapping;
>> @@ -451,8 +450,8 @@ static int __init intel_iommu_setup(char *str)
>>               dmar_map_gfx = 0;
>>               pr_info("Disable GFX device mapping\n");
>>           } else if (!strncmp(str, "forcedac", 8)) {
>> -            pr_info("Forcing DAC for PCI devices\n");
>> -            dmar_forcedac = 1;
>> +            pr_warn("intel_iommu=forcedac deprecated; use 
>> iommu.forcedac instead\n");
>> +            iommu_dma_forcedac = true;
>>           } else if (!strncmp(str, "strict", 6)) {
>>               pr_info("Disable batched IOTLB flush\n");
>>               intel_iommu_strict = 1;
>> diff --git a/include/linux/dma-iommu.h b/include/linux/dma-iommu.h
>> index 706b68d1359b..13d1f4c14d7b 100644
>> --- a/include/linux/dma-iommu.h
>> +++ b/include/linux/dma-iommu.h
>> @@ -40,6 +40,8 @@ void iommu_dma_get_resv_regions(struct device *dev, 
>> struct list_head *list);
>>   void iommu_dma_free_cpu_cached_iovas(unsigned int cpu,
>>           struct iommu_domain *domain);
>> +extern bool iommu_dma_forcedac;
>> +
>>   #else /* CONFIG_IOMMU_DMA */
>>   struct iommu_domain;
>>
> 

Re: [PATCH] iommu/dma: Resurrect the "forcedac" option

[John Garry]

On 08/03/2021 13:08, Robin Murphy wrote:
> On 2021-03-05 17:41, John Garry wrote:
>> On 05/03/2021 16:32, Robin Murphy wrote:
>>> In converting intel-iommu over to the common IOMMU DMA ops, it quietly
>>> lost the functionality of its "forcedac" option. Since this is a handy
>>> thing both for testing and for performance optimisation on certain
>>> platforms, reimplement it under the common IOMMU parameter namespace.
>>>
>>> For the sake of fixing the inadvertent breakage of the Intel-specific
>>> parameter, remove the dmar_forcedac remnants and hook it up as an alias
>>> while documenting the transition to the new common parameter.
>>>
>>
>> Do you think that having a kconfig option to control the default for 
>> this can help identify the broken platforms which rely on forcedac=0? 
>> But seems a bit trivial for that, though.
> 
> I think it's still a sizeable can of worms - unlike, say, 
> ARM_SMMU_DISABLE_BYPASS_BY_DEFAULT, we can't actually tell when things 
> have gone awry and explicitly call it out. While I was getting the 
> dma-ranges right on my Juno, everything broke differently - the SATA 
> controller fails gracefully; the ethernet controller got the kernel tied 
> up somewhere (to the point that the USB keyboard died) once it tried to 
> brink up the link, but was at least spewing regular timeout backtraces 
> that implicated the networking layer; having an (unused) NVMe plugged in 
> simply wedged the boot process early on with no hint whatsoever of why.
> 
> TBH I'm not really sure what the best way forward is in terms of trying 
> to weed out platforms that would need quirking.

I was more thinking of an unstable TEST config, like 
DEBUG_TEST_DRIVER_REMOVE. So we know that this particular config breaks 
many platforms. But at least those in the know can turn it on locally 
and detect and fix issues, and strive towards having a platform for 
which it works.

But then it does become a little harder to justify such a config when we 
can enable via commadline.

> Our discussion just 
> reminded me of this option and that it had gone AWOL, so bringing it 
> back to be potentially *some* use to everyone seems justifiable on its own.

Of course.

Cheers,
John

Re: [PATCH] iommu/dma: Resurrect the "forcedac" option

[John Garry]

On 05/03/2021 16:32, Robin Murphy wrote:
> In converting intel-iommu over to the common IOMMU DMA ops, it quietly
> lost the functionality of its "forcedac" option. Since this is a handy
> thing both for testing and for performance optimisation on certain
> platforms, reimplement it under the common IOMMU parameter namespace.
> 
> For the sake of fixing the inadvertent breakage of the Intel-specific
> parameter, remove the dmar_forcedac remnants and hook it up as an alias
> while documenting the transition to the new common parameter.
> 
> Fixes: c588072bba6b ("iommu/vt-d: Convert intel iommu driver to the iommu ops")
> Signed-off-by: Robin Murphy <robin.murphy@arm.com>

If it's worth anything:
Reviewed-by: John Garry <john.garry@huawei.com>

> ---
>   Documentation/admin-guide/kernel-parameters.txt | 15 ++++++++-------
>   drivers/iommu/dma-iommu.c                       | 13 ++++++++++++-
>   drivers/iommu/intel/iommu.c                     |  5 ++---
>   include/linux/dma-iommu.h                       |  2 ++
>   4 files changed, 24 insertions(+), 11 deletions(-)
> 
> diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt
> index 04545725f187..835f810f2f26 100644
> --- a/Documentation/admin-guide/kernel-parameters.txt
> +++ b/Documentation/admin-guide/kernel-parameters.txt
> @@ -1869,13 +1869,6 @@
>   			bypassed by not enabling DMAR with this option. In
>   			this case, gfx device will use physical address for
>   			DMA.
> -		forcedac [X86-64]
> -			With this option iommu will not optimize to look
> -			for io virtual address below 32-bit forcing dual
> -			address cycle on pci bus for cards supporting greater
> -			than 32-bit addressing. The default is to look
> -			for translation below 32-bit and if not available
> -			then look in the higher range.
>   		strict [Default Off]
>   			With this option on every unmap_single operation will
>   			result in a hardware IOTLB flush operation as opposed
> @@ -1964,6 +1957,14 @@
>   		nobypass	[PPC/POWERNV]
>   			Disable IOMMU bypass, using IOMMU for PCI devices.
>   
> +	iommu.forcedac=	[ARM64, X86] Control IOVA allocation for PCI devices.
> +			Format: { "0" | "1" }
> +			0 - Try to allocate a 32-bit DMA address first, before
> +			  falling back to the full range if needed.
> +			1 - Allocate directly from the full usable range,
> +			  forcing Dual Address Cycle for PCI cards supporting
> +			  greater than 32-bit addressing.
> +
>   	iommu.strict=	[ARM64] Configure TLB invalidation behaviour
>   			Format: { "0" | "1" }
>   			0 - Lazy mode.
> diff --git a/drivers/iommu/dma-iommu.c b/drivers/iommu/dma-iommu.c
> index 9ab6ee22c110..260bf3de1992 100644
> --- a/drivers/iommu/dma-iommu.c
> +++ b/drivers/iommu/dma-iommu.c
> @@ -52,6 +52,17 @@ struct iommu_dma_cookie {
>   };
>   
>   static DEFINE_STATIC_KEY_FALSE(iommu_deferred_attach_enabled);
> +bool iommu_dma_forcedac __read_mostly;
> +
> +static int __init iommu_dma_forcedac_setup(char *str)
> +{
> +	int ret = kstrtobool(str, &iommu_dma_forcedac);
> +
> +	if (!ret && iommu_dma_forcedac)
> +		pr_info("Forcing DAC for PCI devices\n");

I seem to remember some disagreement on this sort of print some other 
time :)

> +	return ret;
> +}
> +early_param("iommu.forcedac", iommu_dma_forcedac_setup);
>   
>   void iommu_dma_free_cpu_cached_iovas(unsigned int cpu,
>   		struct iommu_domain *domain)
> @@ -438,7 +449,7 @@ static dma_addr_t iommu_dma_alloc_iova(struct iommu_domain *domain,
>   		dma_limit = min(dma_limit, (u64)domain->geometry.aperture_end);

Re: [PATCH] iommu/dma: Resurrect the "forcedac" option

[Joerg Roedel]

On Fri, Mar 05, 2021 at 04:32:34PM +0000, Robin Murphy wrote:
> In converting intel-iommu over to the common IOMMU DMA ops, it quietly
> lost the functionality of its "forcedac" option. Since this is a handy
> thing both for testing and for performance optimisation on certain
> platforms, reimplement it under the common IOMMU parameter namespace.
> 
> For the sake of fixing the inadvertent breakage of the Intel-specific
> parameter, remove the dmar_forcedac remnants and hook it up as an alias
> while documenting the transition to the new common parameter.
> 
> Fixes: c588072bba6b ("iommu/vt-d: Convert intel iommu driver to the iommu ops")
> Signed-off-by: Robin Murphy <robin.murphy@arm.com>
> ---
>  Documentation/admin-guide/kernel-parameters.txt | 15 ++++++++-------
>  drivers/iommu/dma-iommu.c                       | 13 ++++++++++++-
>  drivers/iommu/intel/iommu.c                     |  5 ++---
>  include/linux/dma-iommu.h                       |  2 ++
>  4 files changed, 24 insertions(+), 11 deletions(-)

Applied for v5.13, thanks.