[PATCH v2 0/2] L2 ccache DT and cacheinfo support to read no. of L2 cache ways enabled

[PATCH v2 0/2] L2 ccache DT and cacheinfo support to read no. of L2 cache ways enabled

[Yash Shah]

The patchset includes the patch to implement a private attribute named
"number_of_ways_enabled" in the cacheinfo framework. Reading this
attribute returns the number of L2 cache ways enabled at runtime,
The patchset also include the patch to add DT node for SiFive L2 cache
controller.

This patchset is based on Linux v5.5-rc3 and tested on HiFive Unleashed
board.

Changes in v2:
- Rebase the series on v5.5-rc3
- Remove the reserved-memory node from DT

Yash Shah (2):
  riscv: dts: Add DT support for SiFive L2 cache controller
  riscv: cacheinfo: Add support to determine no. of L2 cache way enabled

 arch/riscv/boot/dts/sifive/fu540-c000.dtsi | 15 +++++++++++++++
 arch/riscv/include/asm/sifive_l2_cache.h   |  2 ++
 arch/riscv/kernel/cacheinfo.c              | 31 ++++++++++++++++++++++++++++++
 drivers/soc/sifive/sifive_l2_cache.c       |  5 +++++
 4 files changed, 53 insertions(+)

-- 
2.7.4

[PATCH v2 1/2] riscv: dts: Add DT support for SiFive L2 cache controller

[Yash Shah]

Add the L2 cache controller DT node in SiFive FU540 soc-specific DT file

Signed-off-by: Yash Shah <yash.shah@sifive.com>
Reviewed-by: Palmer Dabbelt <palmerdabbelt@google.com>
---
 arch/riscv/boot/dts/sifive/fu540-c000.dtsi | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/arch/riscv/boot/dts/sifive/fu540-c000.dtsi b/arch/riscv/boot/dts/sifive/fu540-c000.dtsi
index 70a1891..a2e3d54 100644
--- a/arch/riscv/boot/dts/sifive/fu540-c000.dtsi
+++ b/arch/riscv/boot/dts/sifive/fu540-c000.dtsi
@@ -54,6 +54,7 @@
 			reg = <1>;
 			riscv,isa = "rv64imafdc";
 			tlb-split;
+			next-level-cache = <&l2cache>;
 			cpu1_intc: interrupt-controller {
 				#interrupt-cells = <1>;
 				compatible = "riscv,cpu-intc";
@@ -77,6 +78,7 @@
 			reg = <2>;
 			riscv,isa = "rv64imafdc";
 			tlb-split;
+			next-level-cache = <&l2cache>;
 			cpu2_intc: interrupt-controller {
 				#interrupt-cells = <1>;
 				compatible = "riscv,cpu-intc";
@@ -100,6 +102,7 @@
 			reg = <3>;
 			riscv,isa = "rv64imafdc";
 			tlb-split;
+			next-level-cache = <&l2cache>;
 			cpu3_intc: interrupt-controller {
 				#interrupt-cells = <1>;
 				compatible = "riscv,cpu-intc";
@@ -123,6 +126,7 @@
 			reg = <4>;
 			riscv,isa = "rv64imafdc";
 			tlb-split;
+			next-level-cache = <&l2cache>;
 			cpu4_intc: interrupt-controller {
 				#interrupt-cells = <1>;
 				compatible = "riscv,cpu-intc";
@@ -253,6 +257,17 @@
 			#pwm-cells = <3>;
 			status = "disabled";
 		};
+		l2cache: cache-controller@2010000 {
+			compatible = "sifive,fu540-c000-ccache", "cache";
+			cache-block-size = <64>;
+			cache-level = <2>;
+			cache-sets = <1024>;
+			cache-size = <2097152>;
+			cache-unified;
+			interrupt-parent = <&plic0>;
+			interrupts = <1 2 3>;
+			reg = <0x0 0x2010000 0x0 0x1000>;
+		};
 
 	};
 };
-- 
2.7.4

[PATCH v2 2/2] riscv: cacheinfo: Add support to determine no. of L2 cache way enabled

[Yash Shah]

In order to determine the number of L2 cache ways enabled at runtime,
implement a private attribute using cache_get_priv_group() in cacheinfo
framework. Reading this attribute ("number_of_ways_enabled") will return
the number of enabled L2 cache ways at runtime.

Signed-off-by: Yash Shah <yash.shah@sifive.com>
---
 arch/riscv/include/asm/sifive_l2_cache.h |  2 ++
 arch/riscv/kernel/cacheinfo.c            | 31 +++++++++++++++++++++++++++++++
 drivers/soc/sifive/sifive_l2_cache.c     |  5 +++++
 3 files changed, 38 insertions(+)

diff --git a/arch/riscv/include/asm/sifive_l2_cache.h b/arch/riscv/include/asm/sifive_l2_cache.h
index 04f6748..217a42f 100644
--- a/arch/riscv/include/asm/sifive_l2_cache.h
+++ b/arch/riscv/include/asm/sifive_l2_cache.h
@@ -10,6 +10,8 @@
 extern int register_sifive_l2_error_notifier(struct notifier_block *nb);
 extern int unregister_sifive_l2_error_notifier(struct notifier_block *nb);
 
+int sifive_l2_largest_wayenabled(void);
+
 #define SIFIVE_L2_ERR_TYPE_CE 0
 #define SIFIVE_L2_ERR_TYPE_UE 1
 
diff --git a/arch/riscv/kernel/cacheinfo.c b/arch/riscv/kernel/cacheinfo.c
index 4c90c07..29bdb21 100644
--- a/arch/riscv/kernel/cacheinfo.c
+++ b/arch/riscv/kernel/cacheinfo.c
@@ -7,6 +7,7 @@
 #include <linux/cpu.h>
 #include <linux/of.h>
 #include <linux/of_device.h>
+#include <asm/sifive_l2_cache.h>
 
 static void ci_leaf_init(struct cacheinfo *this_leaf,
 			 struct device_node *node,
@@ -16,6 +17,36 @@ static void ci_leaf_init(struct cacheinfo *this_leaf,
 	this_leaf->type = type;
 }
 
+#ifdef CONFIG_SIFIVE_L2
+static ssize_t number_of_ways_enabled_show(struct device *dev,
+					   struct device_attribute *attr,
+					   char *buf)
+{
+	return sprintf(buf, "%u\n", sifive_l2_largest_wayenabled());
+}
+
+static DEVICE_ATTR_RO(number_of_ways_enabled);
+
+static struct attribute *priv_attrs[] = {
+	&dev_attr_number_of_ways_enabled.attr,
+	NULL,
+};
+
+static const struct attribute_group priv_attr_group = {
+	.attrs = priv_attrs,
+};
+
+const struct attribute_group *
+cache_get_priv_group(struct cacheinfo *this_leaf)
+{
+	/* We want to use private group for L2 cache only */
+	if (this_leaf->level == 2)
+		return &priv_attr_group;
+	else
+		return NULL;
+}
+#endif /* CONFIG_SIFIVE_L2 */
+
 static int __init_cache_level(unsigned int cpu)
 {
 	struct cpu_cacheinfo *this_cpu_ci = get_cpu_cacheinfo(cpu);
diff --git a/drivers/soc/sifive/sifive_l2_cache.c b/drivers/soc/sifive/sifive_l2_cache.c
index a9ffff3..f1a5f2c 100644
--- a/drivers/soc/sifive/sifive_l2_cache.c
+++ b/drivers/soc/sifive/sifive_l2_cache.c
@@ -107,6 +107,11 @@ int unregister_sifive_l2_error_notifier(struct notifier_block *nb)
 }
 EXPORT_SYMBOL_GPL(unregister_sifive_l2_error_notifier);
 
+int sifive_l2_largest_wayenabled(void)
+{
+	return readl(l2_base + SIFIVE_L2_WAYENABLE);
+}
+
 static irqreturn_t l2_int_handler(int irq, void *device)
 {
 	unsigned int add_h, add_l;
-- 
2.7.4

Re: [PATCH v2 1/2] riscv: dts: Add DT support for SiFive L2 cache controller

[Paul Walmsley]

On Fri, 3 Jan 2020, Yash Shah wrote:

> Add the L2 cache controller DT node in SiFive FU540 soc-specific DT file
> 
> Signed-off-by: Yash Shah <yash.shah@sifive.com>
> Reviewed-by: Palmer Dabbelt <palmerdabbelt@google.com>

Thanks, queued for v5.5-rc.


- Paul

Re: [PATCH v2 2/2] riscv: cacheinfo: Add support to determine no. of L2 cache way enabled

[Anup Patel]

On Fri, Jan 3, 2020 at 9:44 AM Yash Shah <yash.shah@sifive.com> wrote:
>
> In order to determine the number of L2 cache ways enabled at runtime,
> implement a private attribute using cache_get_priv_group() in cacheinfo
> framework. Reading this attribute ("number_of_ways_enabled") will return
> the number of enabled L2 cache ways at runtime.
>
> Signed-off-by: Yash Shah <yash.shah@sifive.com>
> ---
>  arch/riscv/include/asm/sifive_l2_cache.h |  2 ++
>  arch/riscv/kernel/cacheinfo.c            | 31 +++++++++++++++++++++++++++++++
>  drivers/soc/sifive/sifive_l2_cache.c     |  5 +++++
>  3 files changed, 38 insertions(+)
>
> diff --git a/arch/riscv/include/asm/sifive_l2_cache.h b/arch/riscv/include/asm/sifive_l2_cache.h
> index 04f6748..217a42f 100644
> --- a/arch/riscv/include/asm/sifive_l2_cache.h
> +++ b/arch/riscv/include/asm/sifive_l2_cache.h
> @@ -10,6 +10,8 @@
>  extern int register_sifive_l2_error_notifier(struct notifier_block *nb);
>  extern int unregister_sifive_l2_error_notifier(struct notifier_block *nb);
>
> +int sifive_l2_largest_wayenabled(void);
> +
>  #define SIFIVE_L2_ERR_TYPE_CE 0
>  #define SIFIVE_L2_ERR_TYPE_UE 1
>
> diff --git a/arch/riscv/kernel/cacheinfo.c b/arch/riscv/kernel/cacheinfo.c
> index 4c90c07..29bdb21 100644
> --- a/arch/riscv/kernel/cacheinfo.c
> +++ b/arch/riscv/kernel/cacheinfo.c
> @@ -7,6 +7,7 @@
>  #include <linux/cpu.h>
>  #include <linux/of.h>
>  #include <linux/of_device.h>
> +#include <asm/sifive_l2_cache.h>
>
>  static void ci_leaf_init(struct cacheinfo *this_leaf,
>                          struct device_node *node,
> @@ -16,6 +17,36 @@ static void ci_leaf_init(struct cacheinfo *this_leaf,
>         this_leaf->type = type;
>  }
>
> +#ifdef CONFIG_SIFIVE_L2
> +static ssize_t number_of_ways_enabled_show(struct device *dev,
> +                                          struct device_attribute *attr,
> +                                          char *buf)
> +{
> +       return sprintf(buf, "%u\n", sifive_l2_largest_wayenabled());
> +}
> +
> +static DEVICE_ATTR_RO(number_of_ways_enabled);
> +
> +static struct attribute *priv_attrs[] = {
> +       &dev_attr_number_of_ways_enabled.attr,
> +       NULL,
> +};
> +
> +static const struct attribute_group priv_attr_group = {
> +       .attrs = priv_attrs,
> +};
> +
> +const struct attribute_group *
> +cache_get_priv_group(struct cacheinfo *this_leaf)
> +{
> +       /* We want to use private group for L2 cache only */
> +       if (this_leaf->level == 2)
> +               return &priv_attr_group;
> +       else
> +               return NULL;
> +}
> +#endif /* CONFIG_SIFIVE_L2 */
> +

Instead of this, I would suggest to implement a generic ops
structure.

In arch/riscv/include/asm/cacheinfo.h:

struct riscv_caceinfo_ops {
    const struct attribute_group * (*get_priv_group)(struct cacheinfo
*this_leaf);
};

void riscv_set_cacheinfo_ops(struct riscv_caceinfo_ops *ops);

In arch/riscv/riscv/kernel/cacheinfo.h

static struct riscv_caceinfo_ops *rv_cache_ops;

void riscv_set_cacheinfo_ops(struct riscv_caceinfo_ops *ops)
{
    rv_cache_ops = ops;
}
EXPORT_SYMBOL_GPL(riscv_set_cacheinfo_ops);

const struct attribute_group *
cache_get_priv_group(struct cacheinfo *this_leaf)
{
    if (rv_cache_ops && rv_cache_ops->get_priv_group)
        return rv_cache_ops->get_priv_group(this_leaf)
    return NULL;
}

Above will be a separate patch. In future, we can add more
ops for SOC specific cacheinfo.

Using riscv_set_cacheinfo_ops() you can have another patch
to implement SiFive L2 info entirely in drivers/soc/sifive/sifive_l2_cache.c

Also, I would strongly recommend moving
arch/riscv/include/asm/sifive_l2_cache.h
TO
include/soc/sifive/sifive_l2_cache.h

>  static int __init_cache_level(unsigned int cpu)
>  {
>         struct cpu_cacheinfo *this_cpu_ci = get_cpu_cacheinfo(cpu);
> diff --git a/drivers/soc/sifive/sifive_l2_cache.c b/drivers/soc/sifive/sifive_l2_cache.c
> index a9ffff3..f1a5f2c 100644
> --- a/drivers/soc/sifive/sifive_l2_cache.c
> +++ b/drivers/soc/sifive/sifive_l2_cache.c
> @@ -107,6 +107,11 @@ int unregister_sifive_l2_error_notifier(struct notifier_block *nb)
>  }
>  EXPORT_SYMBOL_GPL(unregister_sifive_l2_error_notifier);
>
> +int sifive_l2_largest_wayenabled(void)
> +{
> +       return readl(l2_base + SIFIVE_L2_WAYENABLE);
> +}
> +
>  static irqreturn_t l2_int_handler(int irq, void *device)
>  {
>         unsigned int add_h, add_l;
> --
> 2.7.4
>

Regards,
Anup

RE: [PATCH v2 2/2] riscv: cacheinfo: Add support to determine no. of L2 cache way enabled

[Yash Shah]

> -----Original Message-----
> From: Anup Patel <anup@brainfault.org>
> Sent: 06 January 2020 14:40
> To: Yash Shah <yash.shah@sifive.com>
> Cc: Rob Herring <robh+dt@kernel.org>; Mark Rutland
> <mark.rutland@arm.com>; Paul Walmsley ( Sifive)
> <paul.walmsley@sifive.com>; Palmer Dabbelt <palmer@dabbelt.com>;
> Albert Ou <aou@eecs.berkeley.edu>; Bin Meng <bmeng.cn@gmail.com>;
> Green Wan <green.wan@sifive.com>; Allison Randal <allison@lohutok.net>;
> Alexios Zavras <alexios.zavras@intel.com>; Greg Kroah-Hartman
> <gregkh@linuxfoundation.org>; Thomas Gleixner <tglx@linutronix.de>;
> bp@suse.de; devicetree@vger.kernel.org; linux-riscv <linux-
> riscv@lists.infradead.org>; linux-kernel@vger.kernel.org List <linux-
> kernel@vger.kernel.org>; Sachin Ghadi <sachin.ghadi@sifive.com>
> Subject: Re: [PATCH v2 2/2] riscv: cacheinfo: Add support to determine no. of
> L2 cache way enabled
> 
> On Fri, Jan 3, 2020 at 9:44 AM Yash Shah <yash.shah@sifive.com> wrote:
> >
> > In order to determine the number of L2 cache ways enabled at runtime,
> > implement a private attribute using cache_get_priv_group() in
> > cacheinfo framework. Reading this attribute
> ("number_of_ways_enabled")
> > will return the number of enabled L2 cache ways at runtime.
> >
> > Signed-off-by: Yash Shah <yash.shah@sifive.com>
> > ---
> >  arch/riscv/include/asm/sifive_l2_cache.h |  2 ++
> >  arch/riscv/kernel/cacheinfo.c            | 31
> +++++++++++++++++++++++++++++++
> >  drivers/soc/sifive/sifive_l2_cache.c     |  5 +++++
> >  3 files changed, 38 insertions(+)
> >
> > diff --git a/arch/riscv/include/asm/sifive_l2_cache.h
> > b/arch/riscv/include/asm/sifive_l2_cache.h
> > index 04f6748..217a42f 100644
> > --- a/arch/riscv/include/asm/sifive_l2_cache.h
> > +++ b/arch/riscv/include/asm/sifive_l2_cache.h
> > @@ -10,6 +10,8 @@
> >  extern int register_sifive_l2_error_notifier(struct notifier_block
> > *nb);  extern int unregister_sifive_l2_error_notifier(struct
> > notifier_block *nb);
> >
> > +int sifive_l2_largest_wayenabled(void);
> > +
> >  #define SIFIVE_L2_ERR_TYPE_CE 0
> >  #define SIFIVE_L2_ERR_TYPE_UE 1
> >
> > diff --git a/arch/riscv/kernel/cacheinfo.c
> > b/arch/riscv/kernel/cacheinfo.c index 4c90c07..29bdb21 100644
> > --- a/arch/riscv/kernel/cacheinfo.c
> > +++ b/arch/riscv/kernel/cacheinfo.c
> > @@ -7,6 +7,7 @@
> >  #include <linux/cpu.h>
> >  #include <linux/of.h>
> >  #include <linux/of_device.h>
> > +#include <asm/sifive_l2_cache.h>
> >
> >  static void ci_leaf_init(struct cacheinfo *this_leaf,
> >                          struct device_node *node, @@ -16,6 +17,36 @@
> > static void ci_leaf_init(struct cacheinfo *this_leaf,
> >         this_leaf->type = type;
> >  }
> >
> > +#ifdef CONFIG_SIFIVE_L2
> > +static ssize_t number_of_ways_enabled_show(struct device *dev,
> > +                                          struct device_attribute *attr,
> > +                                          char *buf) {
> > +       return sprintf(buf, "%u\n", sifive_l2_largest_wayenabled()); }
> > +
> > +static DEVICE_ATTR_RO(number_of_ways_enabled);
> > +
> > +static struct attribute *priv_attrs[] = {
> > +       &dev_attr_number_of_ways_enabled.attr,
> > +       NULL,
> > +};
> > +
> > +static const struct attribute_group priv_attr_group = {
> > +       .attrs = priv_attrs,
> > +};
> > +
> > +const struct attribute_group *
> > +cache_get_priv_group(struct cacheinfo *this_leaf) {
> > +       /* We want to use private group for L2 cache only */
> > +       if (this_leaf->level == 2)
> > +               return &priv_attr_group;
> > +       else
> > +               return NULL;
> > +}
> > +#endif /* CONFIG_SIFIVE_L2 */
> > +
> 
> Instead of this, I would suggest to implement a generic ops structure.
> 
> In arch/riscv/include/asm/cacheinfo.h:
> 
> struct riscv_caceinfo_ops {
>     const struct attribute_group * (*get_priv_group)(struct cacheinfo
> *this_leaf); };
> 
> void riscv_set_cacheinfo_ops(struct riscv_caceinfo_ops *ops);
> 
> In arch/riscv/riscv/kernel/cacheinfo.h
> 
> static struct riscv_caceinfo_ops *rv_cache_ops;
> 
> void riscv_set_cacheinfo_ops(struct riscv_caceinfo_ops *ops) {
>     rv_cache_ops = ops;
> }
> EXPORT_SYMBOL_GPL(riscv_set_cacheinfo_ops);
> 
> const struct attribute_group *
> cache_get_priv_group(struct cacheinfo *this_leaf) {
>     if (rv_cache_ops && rv_cache_ops->get_priv_group)
>         return rv_cache_ops->get_priv_group(this_leaf)
>     return NULL;
> }
> 
> Above will be a separate patch. In future, we can add more ops for SOC
> specific cacheinfo.
> 
> Using riscv_set_cacheinfo_ops() you can have another patch to implement
> SiFive L2 info entirely in drivers/soc/sifive/sifive_l2_cache.c
> 

Yes I agree, the above approach is better. Will work on this approach and send new patches.

> Also, I would strongly recommend moving
> arch/riscv/include/asm/sifive_l2_cache.h
> TO
> include/soc/sifive/sifive_l2_cache.h

Sure, will send a separate patch for this.

> 
> >  static int __init_cache_level(unsigned int cpu)  {
> >         struct cpu_cacheinfo *this_cpu_ci = get_cpu_cacheinfo(cpu);
> > diff --git a/drivers/soc/sifive/sifive_l2_cache.c
> > b/drivers/soc/sifive/sifive_l2_cache.c
> > index a9ffff3..f1a5f2c 100644
> > --- a/drivers/soc/sifive/sifive_l2_cache.c
> > +++ b/drivers/soc/sifive/sifive_l2_cache.c
> > @@ -107,6 +107,11 @@ int unregister_sifive_l2_error_notifier(struct
> > notifier_block *nb)  }
> > EXPORT_SYMBOL_GPL(unregister_sifive_l2_error_notifier);
> >
> > +int sifive_l2_largest_wayenabled(void)
> > +{
> > +       return readl(l2_base + SIFIVE_L2_WAYENABLE); }
> > +
> >  static irqreturn_t l2_int_handler(int irq, void *device)  {
> >         unsigned int add_h, add_l;
> > --
> > 2.7.4
> >
> 
> Regards,
> Anup

Thanks for your comments

- Yash