[PATCH 0/2 v2] Fix MAX_PHANDLE_ARGS limitations

[PATCH 0/2 v2] Fix MAX_PHANDLE_ARGS limitations

[Joerg Roedel]

From: Joerg Roedel <jroedel@suse.de>

Hi,

here is the second version of my patches to fix the
MAX_PHANDLE_ARGS limitation for the arm-smmu driver. On my
AMD Seattle system the max value of 16 is not enough to
initialize the SMMU, as the device tree node has an entry
with 25 phandle args (possible are up to 128). The result is
a WARNING and incorrect SMMU initialization.

On the other side MAX_PHANDLE_ARGS can't grow indefinitly,
as it causes 'struct of_phandle_args' to get bigger. This
struct is often allocated on-stack and should be small by
default.

So these patches introduce a way to make 'struct
of_phandle_args' bigger only when needed and change the
ARM-SMMU driver to make use of it.

The patches have been tested on my AMD Seattle system.

Please review.

Thanks,

	Joerg

Changes to v1:

	* Rebased to v4.2-rc2
	* Fixed leak in patch 2, as pointed out by Will
	  Deacon

Joerg Roedel (2):
  of: base: Allow more args than MAX_PHANDLE_ARGS if required
  arm/smmu: Make use of of_parse_phandle_with_var_args

 drivers/iommu/arm-smmu.c | 23 ++++++++++++++++-------
 drivers/of/base.c        | 43 ++++++++++++++++++++++++++++++++++++-------
 include/linux/of.h       |  7 +++++++
 3 files changed, 59 insertions(+), 14 deletions(-)

-- 
1.8.4.5

[PATCH 1/2] of: base: Allow more args than MAX_PHANDLE_ARGS if required

[Joerg Roedel]

From: Joerg Roedel <jroedel@suse.de>

The main use of MAX_PHANDLE_ARGS is to define the number of
args elements in 'struct of_phandle_args'. This struct is
often declared on the stack and thus it is impractical to
increase MAX_PHANDLE_ARGS again and again.

To handle situations where more than MAX_PHANDLE_ARGS
elements may appear in a device-tree, introduce functions
to allocate/free 'struct of_phandle_args' with more than
MAX_PHANDLE_ARGS elements and provide the new function
of_parse_phandle_with_var_args(), which can handle those
variable-size structs.

This is necessary for the ARM-SMMU driver, where the number
of mmu-masters can be up to 128.

Signed-off-by: Joerg Roedel <jroedel@suse.de>
---
 drivers/of/base.c  | 43 ++++++++++++++++++++++++++++++++++++-------
 include/linux/of.h |  7 +++++++
 2 files changed, 43 insertions(+), 7 deletions(-)

diff --git a/drivers/of/base.c b/drivers/of/base.c
index 8b5a187..2b288db 100644
--- a/drivers/of/base.c
+++ b/drivers/of/base.c
@@ -54,6 +54,24 @@ DEFINE_MUTEX(of_mutex);
  */
 DEFINE_RAW_SPINLOCK(devtree_lock);
 
+struct of_phandle_args *of_alloc_phandle_args(int size)
+{
+	struct of_phandle_args *args;
+	int e = max(0, size - MAX_PHANDLE_ARGS);
+
+	args =  kzalloc(sizeof(struct of_phandle_args) + e * sizeof(uint32_t),
+			GFP_KERNEL);
+
+	return args;
+}
+EXPORT_SYMBOL(of_alloc_phandle_args);
+
+void of_free_phandle_args(struct of_phandle_args *args)
+{
+	kfree(args);
+}
+EXPORT_SYMBOL(of_free_phandle_args);
+
 int of_n_addr_cells(struct device_node *np)
 {
 	const __be32 *ip;
@@ -1446,7 +1464,8 @@ static int __of_parse_phandle_with_args(const struct device_node *np,
 					const char *list_name,
 					const char *cells_name,
 					int cell_count, int index,
-					struct of_phandle_args *out_args)
+					struct of_phandle_args *out_args,
+					int elems)
 {
 	const __be32 *list, *list_end;
 	int rc = 0, size, cur_index = 0;
@@ -1525,8 +1544,8 @@ static int __of_parse_phandle_with_args(const struct device_node *np,
 
 			if (out_args) {
 				int i;
-				if (WARN_ON(count > MAX_PHANDLE_ARGS))
-					count = MAX_PHANDLE_ARGS;
+				if (WARN_ON(count > elems))
+					count = elems;
 				out_args->np = node;
 				out_args->args_count = count;
 				for (i = 0; i < count; i++)
@@ -1577,7 +1596,7 @@ struct device_node *of_parse_phandle(const struct device_node *np,
 		return NULL;
 
 	if (__of_parse_phandle_with_args(np, phandle_name, NULL, 0,
-					 index, &args))
+					 index, &args, MAX_PHANDLE_ARGS))
 		return NULL;
 
 	return args.np;
@@ -1623,10 +1642,20 @@ int of_parse_phandle_with_args(const struct device_node *np, const char *list_na
 	if (index < 0)
 		return -EINVAL;
 	return __of_parse_phandle_with_args(np, list_name, cells_name, 0,
-					    index, out_args);
+					    index, out_args, MAX_PHANDLE_ARGS);
 }
 EXPORT_SYMBOL(of_parse_phandle_with_args);
 
+int of_parse_phandle_with_var_args(const struct device_node *np, const char *list_name,
+				   const char *cells_name, int index,
+				   struct of_phandle_args *out_args, int elems)
+{
+	if (index < 0)
+		return -EINVAL;
+	return __of_parse_phandle_with_args(np, list_name, cells_name, 0,
+					    index, out_args, elems);
+}
+EXPORT_SYMBOL(of_parse_phandle_with_var_args);
 /**
  * of_parse_phandle_with_fixed_args() - Find a node pointed by phandle in a list
  * @np:		pointer to a device tree node containing a list
@@ -1664,7 +1693,7 @@ int of_parse_phandle_with_fixed_args(const struct device_node *np,
 	if (index < 0)
 		return -EINVAL;
 	return __of_parse_phandle_with_args(np, list_name, NULL, cell_count,
-					   index, out_args);
+					   index, out_args, MAX_PHANDLE_ARGS);
 }
 EXPORT_SYMBOL(of_parse_phandle_with_fixed_args);
 
@@ -1687,7 +1716,7 @@ int of_count_phandle_with_args(const struct device_node *np, const char *list_na
 				const char *cells_name)
 {
 	return __of_parse_phandle_with_args(np, list_name, cells_name, 0, -1,
-					    NULL);
+					    NULL, MAX_PHANDLE_ARGS);
 }
 EXPORT_SYMBOL(of_count_phandle_with_args);
 
diff --git a/include/linux/of.h b/include/linux/of.h
index edc068d..cd9f225 100644
--- a/include/linux/of.h
+++ b/include/linux/of.h
@@ -75,6 +75,10 @@ struct of_phandle_args {
 	uint32_t args[MAX_PHANDLE_ARGS];
 };
 
+/* Use these if you need more that MAX_PHANDLE_ARGS */
+extern struct of_phandle_args *of_alloc_phandle_args(int size);
+extern void of_free_phandle_args(struct of_phandle_args *args);
+
 struct of_reconfig_data {
 	struct device_node	*dn;
 	struct property		*prop;
@@ -327,6 +331,9 @@ extern struct device_node *of_parse_phandle(const struct device_node *np,
 extern int of_parse_phandle_with_args(const struct device_node *np,
 	const char *list_name, const char *cells_name, int index,
 	struct of_phandle_args *out_args);
+extern int of_parse_phandle_with_var_args(const struct device_node *np,
+	const char *list_name, const char *cells_name, int index,
+	struct of_phandle_args *out_args, int elems);
 extern int of_parse_phandle_with_fixed_args(const struct device_node *np,
 	const char *list_name, int cells_count, int index,
 	struct of_phandle_args *out_args);
-- 
1.9.1

[PATCH 2/2] arm/smmu: Make use of of_parse_phandle_with_var_args

[Joerg Roedel]

From: Joerg Roedel <jroedel@suse.de>

The function of_parse_phandle_with_args() can only handle 16
args, but there are systems that require more (25 in my
case). So use the newly introduced function
of_parse_phandle_with_var_args() instead.

Signed-off-by: Joerg Roedel <jroedel@suse.de>
---
 drivers/iommu/arm-smmu.c | 23 ++++++++++++++++-------
 1 file changed, 16 insertions(+), 7 deletions(-)

diff --git a/drivers/iommu/arm-smmu.c b/drivers/iommu/arm-smmu.c
index 4cd0c29..1fe7f35 100644
--- a/drivers/iommu/arm-smmu.c
+++ b/drivers/iommu/arm-smmu.c
@@ -47,7 +47,7 @@
 #include "io-pgtable.h"
 
 /* Maximum number of stream IDs assigned to a single device */
-#define MAX_MASTER_STREAMIDS		MAX_PHANDLE_ARGS
+#define MAX_MASTER_STREAMIDS		128
 
 /* Maximum number of context banks per SMMU */
 #define ARM_SMMU_MAX_CBS		128
@@ -1698,7 +1698,7 @@ static int arm_smmu_device_dt_probe(struct platform_device *pdev)
 	struct arm_smmu_device *smmu;
 	struct device *dev = &pdev->dev;
 	struct rb_node *node;
-	struct of_phandle_args masterspec;
+	struct of_phandle_args *masterspec;
 	int num_irqs, i, err;
 
 	smmu = devm_kzalloc(dev, sizeof(*smmu), GFP_KERNEL);
@@ -1757,15 +1757,19 @@ static int arm_smmu_device_dt_probe(struct platform_device *pdev)
 	if (err)
 		return err;
 
+	masterspec = of_alloc_phandle_args(MAX_MASTER_STREAMIDS);
+	if (!masterspec)
+		return -ENOMEM;
+
 	i = 0;
 	smmu->masters = RB_ROOT;
-	while (!of_parse_phandle_with_args(dev->of_node, "mmu-masters",
-					   "#stream-id-cells", i,
-					   &masterspec)) {
-		err = register_smmu_master(smmu, dev, &masterspec);
+	while (!of_parse_phandle_with_var_args(dev->of_node, "mmu-masters",
+					   "#stream-id-cells", i, masterspec,
+					   MAX_MASTER_STREAMIDS)) {
+		err = register_smmu_master(smmu, dev, masterspec);
 		if (err) {
 			dev_err(dev, "failed to add master %s\n",
-				masterspec.np->name);
+				masterspec->np->name);
 			goto out_put_masters;
 		}
 
@@ -1797,6 +1801,8 @@ static int arm_smmu_device_dt_probe(struct platform_device *pdev)
 		}
 	}
 
+	of_free_phandle_args(masterspec);
+
 	INIT_LIST_HEAD(&smmu->list);
 	spin_lock(&arm_smmu_devices_lock);
 	list_add(&smmu->list, &arm_smmu_devices);
@@ -1810,6 +1816,9 @@ out_free_irqs:
 		free_irq(smmu->irqs[i], smmu);
 
 out_put_masters:
+
+	of_free_phandle_args(masterspec);
+
 	for (node = rb_first(&smmu->masters); node; node = rb_next(node)) {
 		struct arm_smmu_master *master
 			= container_of(node, struct arm_smmu_master, node);
-- 
1.9.1

Re: [PATCH 2/2] arm/smmu: Make use of of_parse_phandle_with_var_args

[Will Deacon]

Hi Joerg,

On Thu, Jul 16, 2015 at 09:30:44AM +0100, Joerg Roedel wrote:
> From: Joerg Roedel <jroedel@suse.de>
> 
> The function of_parse_phandle_with_args() can only handle 16
> args, but there are systems that require more (25 in my
> case). So use the newly introduced function
> of_parse_phandle_with_var_args() instead.

Looks good to me:

  Acked-by: Will Deacon <will.deacon@arm.com>

Thanks!

Will

Re: [PATCH 1/2] of: base: Allow more args than MAX_PHANDLE_ARGS if required

[Will Deacon]

On Thu, Jul 16, 2015 at 09:30:43AM +0100, Joerg Roedel wrote:
> From: Joerg Roedel <jroedel@suse.de>
> 
> The main use of MAX_PHANDLE_ARGS is to define the number of
> args elements in 'struct of_phandle_args'. This struct is
> often declared on the stack and thus it is impractical to
> increase MAX_PHANDLE_ARGS again and again.
> 
> To handle situations where more than MAX_PHANDLE_ARGS
> elements may appear in a device-tree, introduce functions
> to allocate/free 'struct of_phandle_args' with more than
> MAX_PHANDLE_ARGS elements and provide the new function
> of_parse_phandle_with_var_args(), which can handle those
> variable-size structs.
> 
> This is necessary for the ARM-SMMU driver, where the number
> of mmu-masters can be up to 128.
> 
> Signed-off-by: Joerg Roedel <jroedel@suse.de>
> ---
>  drivers/of/base.c  | 43 ++++++++++++++++++++++++++++++++++++-------
>  include/linux/of.h |  7 +++++++
>  2 files changed, 43 insertions(+), 7 deletions(-)
> 
> diff --git a/drivers/of/base.c b/drivers/of/base.c
> index 8b5a187..2b288db 100644
> --- a/drivers/of/base.c
> +++ b/drivers/of/base.c
> @@ -54,6 +54,24 @@ DEFINE_MUTEX(of_mutex);
>   */
>  DEFINE_RAW_SPINLOCK(devtree_lock);
>  
> +struct of_phandle_args *of_alloc_phandle_args(int size)
> +{
> +	struct of_phandle_args *args;
> +	int e = max(0, size - MAX_PHANDLE_ARGS);
> +
> +	args =  kzalloc(sizeof(struct of_phandle_args) + e * sizeof(uint32_t),
> +			GFP_KERNEL);

Should you also update args->args_count to reflect the extended array?

That said, extending the fixed-size array member like this feels a bit
fragile. Does GCC not complain about out-of-bounds accesses if you
statically address args->args[MAX_PHANDLE_ARGS]? Admittedly, I can't
think *why* this would be break (things like additional padding will be
harmless), but I'm not intimate with the C standard.

I guess the more worrying possibility is if somebody adds a new member to
the end of of_phandle_args.

Will

Re: [PATCH 1/2] of: base: Allow more args than MAX_PHANDLE_ARGS if required

[Joerg Roedel]

Hi Will,

On Thu, Jul 16, 2015 at 11:23:26AM +0100, Will Deacon wrote:
> On Thu, Jul 16, 2015 at 09:30:43AM +0100, Joerg Roedel wrote:
> > +struct of_phandle_args *of_alloc_phandle_args(int size)
> > +{
> > +	struct of_phandle_args *args;
> > +	int e = max(0, size - MAX_PHANDLE_ARGS);
> > +
> > +	args =  kzalloc(sizeof(struct of_phandle_args) + e * sizeof(uint32_t),
> > +			GFP_KERNEL);
> 
> Should you also update args->args_count to reflect the extended array?

The args_count member just tells us how many of the array elements are
used and not how many there are. So it doesn't need to be updated here.

> That said, extending the fixed-size array member like this feels a bit
> fragile. Does GCC not complain about out-of-bounds accesses if you
> statically address args->args[MAX_PHANDLE_ARGS]? Admittedly, I can't
> think *why* this would be break (things like additional padding will be
> harmless), but I'm not intimate with the C standard.

Yeah, I agree, it is not the best possible solution. But this way I
don't need to update all callers, and thus it works better with our
development model.

But I am open for suggestions on how to solve this problem better. In
fact, my main motivation in sending this was to get the discussion about
an upstreamable solution started :)

Lets see what the device-tree maintainers have to say.

> I guess the more worrying possibility is if somebody adds a new member to
> the end of of_phandle_args.

I should probably add a comment there.



	Joerg

Re: [PATCH 1/2] of: base: Allow more args than MAX_PHANDLE_ARGS if required

[Rob Herring]

On Thu, Jul 16, 2015 at 6:09 AM, Joerg Roedel <jroedel@suse.de> wrote:
> Hi Will,
>
> On Thu, Jul 16, 2015 at 11:23:26AM +0100, Will Deacon wrote:
>> On Thu, Jul 16, 2015 at 09:30:43AM +0100, Joerg Roedel wrote:
>> > +struct of_phandle_args *of_alloc_phandle_args(int size)
>> > +{
>> > +   struct of_phandle_args *args;
>> > +   int e = max(0, size - MAX_PHANDLE_ARGS);
>> > +
>> > +   args =  kzalloc(sizeof(struct of_phandle_args) + e * sizeof(uint32_t),
>> > +                   GFP_KERNEL);
>>
>> Should you also update args->args_count to reflect the extended array?
>
> The args_count member just tells us how many of the array elements are
> used and not how many there are. So it doesn't need to be updated here.
>
>> That said, extending the fixed-size array member like this feels a bit
>> fragile. Does GCC not complain about out-of-bounds accesses if you
>> statically address args->args[MAX_PHANDLE_ARGS]? Admittedly, I can't
>> think *why* this would be break (things like additional padding will be
>> harmless), but I'm not intimate with the C standard.
>
> Yeah, I agree, it is not the best possible solution. But this way I
> don't need to update all callers, and thus it works better with our
> development model.

Our development model is not to work-around kernel APIs last time I checked.

> But I am open for suggestions on how to solve this problem better. In
> fact, my main motivation in sending this was to get the discussion about
> an upstreamable solution started :)
>
> Lets see what the device-tree maintainers have to say.

A good number of callers and all iommu callers loop thru the list of 
phandles which are just open coded ATM. So we should do loop iterators 
here. With iterators, we can do the allocation within the iterators. 
This can be much more efficient as we don't iterate thru the list from 
the start every time. Normally, the list is not big enough to matter, 
but in your case it may be.

I'm thinking something like this untested and not yet compiling patch. It 
still has the abuse of adding onto the end of the of_phandle_args 
struct which I don't really like. We could do a new struct, but I'd like 
to keep this code common.

I also need to refactor the existing code to use 
__of_parse_one_phandle_with_args.

Rob

8<--------------------------------------------------------------------------------------------
diff --git a/drivers/of/base.c b/drivers/of/base.c
index 8b5a187..c1c5a43 100644
--- a/drivers/of/base.c
+++ b/drivers/of/base.c
@@ -1442,6 +1442,96 @@ void of_print_phandle_args(const char *msg, const struct of_phandle_args *args)
 	printk("\n");
 }
 
+
+static int __of_parse_one_phandle_with_args(const __be32 **list,
+					const char *cells_name,
+					int cell_count,
+					struct of_phandle_args **out_args)
+{
+	struct device_node *node;
+	phandle phandle;
+	struct of_phandle_args *args = *out_args;
+	const __be32 *start_list = *list;
+	u32 i, count = 0;
+	int ret;
+
+	/*
+	 * If phandle is 0, then it is an empty entry with no
+	 * arguments.  Skip forward to the next entry.
+	 */
+	phandle = be32_to_cpup(*list++);
+	if (phandle) {
+		/*
+		 * Find the provider node and parse the #*-cells
+		 * property to determine the argument length.
+		 *
+		 * This is not needed if the cell count is hard-coded
+		 * (i.e. cells_name not set, but cell_count is set),
+		 * except when we're going to return the found node
+		 * below.
+		 */
+		node = of_find_node_by_phandle(phandle);
+		if (!node)
+			return -ENOENT;
+
+		if (cells_name) {
+			ret = of_property_read_u32(node, cells_name, &count);
+			if (ret) {
+				pr_err("could not get %s for %s\n",
+				       cells_name, node->full_name);
+				return ret;
+			}
+		} else {
+			count = cell_count;
+		}
+	}
+
+	if (args && WARN_ON(count > MAX_PHANDLE_ARGS))
+		count = MAX_PHANDLE_ARGS;
+
+	if (!args) {
+		args = kzalloc(sizeof(*args) + (count * sizeof(uint32_t))), GFP_KERNEL);
+		*out_args = args;
+	}
+	if (!args)
+		return -ENOMEM;
+
+	if (!phandle) {
+		memset(args, 0, sizeof(*args));
+		return -ENOENT;
+	}
+
+	args->np = node;
+	args->val = start_list;
+	args->args_count = count;
+	for (i = 0; i < count; i++)
+		args->args[i] = be32_to_cpup(*list++);
+
+	return 0;
+}
+
+of_phandle_args *of_prop_next_phandle_args(struct property *prop,
+						const char *cells_name,
+						int cell_count,
+						struct of_phandle_args *last)
+{
+	struct of_phandle_args *args = NULL;
+	const __be32 *start_list;
+
+	if (!prop)
+		return NULL;
+
+	if (!last) {
+		start_list = prop->value;
+	} else {
+		start_list = last->val;
+		kfree(last);
+	}
+	__of_parse_one_phandle_with_args(list, cells_name, cell_count, &args);
+	return args;
+}
+EXPORT_SYMBOL_GPL(of_prop_next_phandle_args);
+
 static int __of_parse_phandle_with_args(const struct device_node *np,
 					const char *list_name,
 					const char *cells_name,
diff --git a/include/linux/of.h b/include/linux/of.h
index edc068d..a0f432c 100644
--- a/include/linux/of.h
+++ b/include/linux/of.h
@@ -879,6 +879,13 @@ static inline int of_property_read_s32(const struct device_node *np,
 		s;						\
 		s = of_prop_next_string(prop, s))
 
+#define of_property_for_each_phandle_with_args(np, propname, cells_name, cell_count, prop, arg)	\
+	for (prop = of_find_property(np, propname, NULL),	\
+		arg = of_prop_next_phandle_args(prop, NULL);	\
+		arg;						\
+		arg = of_prop_next_phandle_args(prop, arg))
+
+
 #define for_each_node_by_name(dn, name) \
 	for (dn = of_find_node_by_name(NULL, name); dn; \
 	     dn = of_find_node_by_name(dn, name))