[PATCH 1/1] powerpc/iommu: Enable remaining IOMMU Pagesizes present in LoPAR

[PATCH 1/1] powerpc/iommu: Enable remaining IOMMU Pagesizes present in LoPAR

[Leonardo Bras]

According to LoPAR, ibm,query-pe-dma-window output named "IO Page Sizes"
will let the OS know all possible pagesizes that can be used for creating a
new DDW.

Currently Linux will only try using 3 of the 8 available options:
4K, 64K and 16M. According to LoPAR, Hypervisor may also offer 32M, 64M,
128M, 256M and 16G.

Enabling bigger pages would be interesting for direct mapping systems
with a lot of RAM, while using less TCE entries.

Signed-off-by: Leonardo Bras <leobras.c@gmail.com>
---
 arch/powerpc/include/asm/iommu.h       |  8 ++++++++
 arch/powerpc/platforms/pseries/iommu.c | 28 +++++++++++++++++++-------
 2 files changed, 29 insertions(+), 7 deletions(-)

diff --git a/arch/powerpc/include/asm/iommu.h b/arch/powerpc/include/asm/iommu.h
index deef7c94d7b6..c170048b7a1b 100644
--- a/arch/powerpc/include/asm/iommu.h
+++ b/arch/powerpc/include/asm/iommu.h
@@ -19,6 +19,14 @@
 #include <asm/pci-bridge.h>
 #include <asm/asm-const.h>
 
+#define IOMMU_PAGE_SHIFT_16G	34
+#define IOMMU_PAGE_SHIFT_256M	28
+#define IOMMU_PAGE_SHIFT_128M	27
+#define IOMMU_PAGE_SHIFT_64M	26
+#define IOMMU_PAGE_SHIFT_32M	25
+#define IOMMU_PAGE_SHIFT_16M	24
+#define IOMMU_PAGE_SHIFT_64K	16
+
 #define IOMMU_PAGE_SHIFT_4K      12
 #define IOMMU_PAGE_SIZE_4K       (ASM_CONST(1) << IOMMU_PAGE_SHIFT_4K)
 #define IOMMU_PAGE_MASK_4K       (~((1 << IOMMU_PAGE_SHIFT_4K) - 1))
diff --git a/arch/powerpc/platforms/pseries/iommu.c b/arch/powerpc/platforms/pseries/iommu.c
index 9fc5217f0c8e..02958e80aa91 100644
--- a/arch/powerpc/platforms/pseries/iommu.c
+++ b/arch/powerpc/platforms/pseries/iommu.c
@@ -1099,6 +1099,24 @@ static void reset_dma_window(struct pci_dev *dev, struct device_node *par_dn)
 			 ret);
 }
 
+/* Returns page shift based on "IO Page Sizes" output at ibm,query-pe-dma-window. SeeL LoPAR */
+static int iommu_get_page_shift(u32 query_page_size)
+{
+	const int shift[] = {IOMMU_PAGE_SHIFT_4K,   IOMMU_PAGE_SHIFT_64K,  IOMMU_PAGE_SHIFT_16M,
+			     IOMMU_PAGE_SHIFT_32M,  IOMMU_PAGE_SHIFT_64M,  IOMMU_PAGE_SHIFT_128M,
+			     IOMMU_PAGE_SHIFT_256M, IOMMU_PAGE_SHIFT_16G};
+	int i = ARRAY_SIZE(shift) - 1;
+
+	/* Looks for the largest page size supported */
+	for (; i >= 0; i--) {
+		if (query_page_size & (1 << i))
+			return shift[i];
+	}
+
+	/* No valid page size found. */
+	return 0;
+}
+
 /*
  * If the PE supports dynamic dma windows, and there is space for a table
  * that can map all pages in a linear offset, then setup such a table,
@@ -1206,13 +1224,9 @@ static u64 enable_ddw(struct pci_dev *dev, struct device_node *pdn)
 			goto out_failed;
 		}
 	}
-	if (query.page_size & 4) {
-		page_shift = 24; /* 16MB */
-	} else if (query.page_size & 2) {
-		page_shift = 16; /* 64kB */
-	} else if (query.page_size & 1) {
-		page_shift = 12; /* 4kB */
-	} else {
+
+	page_shift = iommu_get_page_shift(query.page_size);
+	if (!page_shift) {
 		dev_dbg(&dev->dev, "no supported direct page size in mask %x",
 			  query.page_size);
 		goto out_failed;
-- 
2.29.2

Re: [PATCH 1/1] powerpc/iommu: Enable remaining IOMMU Pagesizes present in LoPAR

[Alexey Kardashevskiy]

On 23/03/2021 06:09, Leonardo Bras wrote:
> According to LoPAR, ibm,query-pe-dma-window output named "IO Page Sizes"
> will let the OS know all possible pagesizes that can be used for creating a
> new DDW.
> 
> Currently Linux will only try using 3 of the 8 available options:
> 4K, 64K and 16M. According to LoPAR, Hypervisor may also offer 32M, 64M,
> 128M, 256M and 16G.
> 
> Enabling bigger pages would be interesting for direct mapping systems
> with a lot of RAM, while using less TCE entries.
> > Signed-off-by: Leonardo Bras <leobras.c@gmail.com>
> ---
>   arch/powerpc/include/asm/iommu.h       |  8 ++++++++
>   arch/powerpc/platforms/pseries/iommu.c | 28 +++++++++++++++++++-------
>   2 files changed, 29 insertions(+), 7 deletions(-)
> 
> diff --git a/arch/powerpc/include/asm/iommu.h b/arch/powerpc/include/asm/iommu.h
> index deef7c94d7b6..c170048b7a1b 100644
> --- a/arch/powerpc/include/asm/iommu.h
> +++ b/arch/powerpc/include/asm/iommu.h
> @@ -19,6 +19,14 @@
>   #include <asm/pci-bridge.h>
>   #include <asm/asm-const.h>
>   
> +#define IOMMU_PAGE_SHIFT_16G	34
> +#define IOMMU_PAGE_SHIFT_256M	28
> +#define IOMMU_PAGE_SHIFT_128M	27
> +#define IOMMU_PAGE_SHIFT_64M	26
> +#define IOMMU_PAGE_SHIFT_32M	25
> +#define IOMMU_PAGE_SHIFT_16M	24
> +#define IOMMU_PAGE_SHIFT_64K	16


These are not very descriptive, these are just normal shifts, could be 
as simple as __builtin_ctz(SZ_4K) (gcc will optimize this) and so on.

OTOH the PAPR page sizes need macros as they are the ones which are 
weird and screaming for macros.

I'd steal/rework spapr_page_mask_to_query_mask() from QEMU. Thanks,




> +
>   #define IOMMU_PAGE_SHIFT_4K      12
>   #define IOMMU_PAGE_SIZE_4K       (ASM_CONST(1) << IOMMU_PAGE_SHIFT_4K)
>   #define IOMMU_PAGE_MASK_4K       (~((1 << IOMMU_PAGE_SHIFT_4K) - 1))
> diff --git a/arch/powerpc/platforms/pseries/iommu.c b/arch/powerpc/platforms/pseries/iommu.c
> index 9fc5217f0c8e..02958e80aa91 100644
> --- a/arch/powerpc/platforms/pseries/iommu.c
> +++ b/arch/powerpc/platforms/pseries/iommu.c
> @@ -1099,6 +1099,24 @@ static void reset_dma_window(struct pci_dev *dev, struct device_node *par_dn)
>   			 ret);
>   }
>   
> +/* Returns page shift based on "IO Page Sizes" output at ibm,query-pe-dma-window. SeeL LoPAR */
> +static int iommu_get_page_shift(u32 query_page_size)
> +{
> +	const int shift[] = {IOMMU_PAGE_SHIFT_4K,   IOMMU_PAGE_SHIFT_64K,  IOMMU_PAGE_SHIFT_16M,
> +			     IOMMU_PAGE_SHIFT_32M,  IOMMU_PAGE_SHIFT_64M,  IOMMU_PAGE_SHIFT_128M,
> +			     IOMMU_PAGE_SHIFT_256M, IOMMU_PAGE_SHIFT_16G};
> +	int i = ARRAY_SIZE(shift) - 1;
> +
> +	/* Looks for the largest page size supported */
> +	for (; i >= 0; i--) {
> +		if (query_page_size & (1 << i))
> +			return shift[i];
> +	}
> +
> +	/* No valid page size found. */
> +	return 0;
> +}
> +
>   /*
>    * If the PE supports dynamic dma windows, and there is space for a table
>    * that can map all pages in a linear offset, then setup such a table,
> @@ -1206,13 +1224,9 @@ static u64 enable_ddw(struct pci_dev *dev, struct device_node *pdn)
>   			goto out_failed;
>   		}
>   	}
> -	if (query.page_size & 4) {
> -		page_shift = 24; /* 16MB */
> -	} else if (query.page_size & 2) {
> -		page_shift = 16; /* 64kB */
> -	} else if (query.page_size & 1) {
> -		page_shift = 12; /* 4kB */
> -	} else {
> +
> +	page_shift = iommu_get_page_shift(query.page_size);
> +	if (!page_shift) {
>   		dev_dbg(&dev->dev, "no supported direct page size in mask %x",
>   			  query.page_size);
>   		goto out_failed;
> 

-- 
Alexey

Re: [PATCH 1/1] powerpc/iommu: Enable remaining IOMMU Pagesizes present in LoPAR

[Leonardo Bras]

Hello Alexey,

On Tue, 2021-03-23 at 18:41 +1100, Alexey Kardashevskiy wrote:
[...]
> > +#define IOMMU_PAGE_SHIFT_16G	34
> > +#define IOMMU_PAGE_SHIFT_256M	28
> > +#define IOMMU_PAGE_SHIFT_128M	27
> > +#define IOMMU_PAGE_SHIFT_64M	26
> > +#define IOMMU_PAGE_SHIFT_32M	25
> > +#define IOMMU_PAGE_SHIFT_16M	24
> > +#define IOMMU_PAGE_SHIFT_64K	16
> 
> 
> These are not very descriptive, these are just normal shifts, could be 
> as simple as __builtin_ctz(SZ_4K) (gcc will optimize this) and so on.
> 
> OTOH the PAPR page sizes need macros as they are the ones which are 
> weird and screaming for macros.
> 
> I'd steal/rework spapr_page_mask_to_query_mask() from QEMU. Thanks,
> 

Thanks for this feedback!
I just sent a v2 applying your suggestions.

Best regards,
Leonardo Bras