[RFC/RFT PATCH 0/6] Unify NUMA implementation between ARM64 & RISC-V

[RFC/RFT PATCH 0/6] Unify NUMA implementation between ARM64 & RISC-V

[Atish Patra]

This series attempts to move the ARM64 numa implementation to common
code so that RISC-V can leverage that as well instead of reimplementing
it again.

RISC-V specific bits are based on initial work done by Greentime Hu [1] but
modified to reuse the common implementation to avoid duplication.

[1] https://lkml.org/lkml/2020/1/10/233

This series has been tested on qemu with numa enabled for both RISC-V & ARM64.
It would be great if somebody can test it on numa capable ARM64 hardware platforms.
This patch series doesn't modify the maintainers list for the common code (arch_numa)
as I am not sure if somebody from ARM64 community or Greg should take up the
maintainership. Ganapatrao was the original author of the arm64 version.
I would be happy to update that in the next revision once it is decided.

# numactl --hardware
available: 2 nodes (0-1)
node 0 cpus: 0 1 2 3
node 0 size: 486 MB
node 0 free: 470 MB
node 1 cpus: 4 5 6 7
node 1 size: 424 MB
node 1 free: 408 MB
node distances:
node   0   1 
  0:  10  20 
  1:  20  10 
# numactl -show
policy: default
preferred node: current
physcpubind: 0 1 2 3 4 5 6 7 
cpubind: 0 1 
nodebind: 0 1 
membind: 0 1 

For RISC-V, the following qemu series is a pre-requisite to test the patches in Qemu.

https://patchwork.kernel.org/project/qemu-devel/list/?series=303313

The patches are also available at

https://github.com/atishp04/linux/tree/5.10_numa_unified

There may be some minor conflicts with Mike's cleanup series [2] depending on the
order in which these two series are being accepted. I can rebase on top his series
if required.

[2] https://lkml.org/lkml/2020/7/28/39

Atish Patra (5):
numa: Move numa implementation to common code
arm64, numa: Change the numa init function name to be generic
arm64, numa: Move pcibus_to_node definition to generic numa code
riscv: Separate memory init from paging init
riscv: Add numa support for riscv64 platform

Greentime Hu (1):
riscv: Add support pte_protnone and pmd_protnone if
CONFIG_NUMA_BALANCING

arch/arm64/Kconfig                            |  1 +
arch/arm64/include/asm/numa.h                 | 45 +---------------
arch/arm64/kernel/pci.c                       | 10 ----
arch/arm64/mm/Makefile                        |  1 -
arch/arm64/mm/init.c                          |  4 +-
arch/riscv/Kconfig                            | 31 ++++++++++-
arch/riscv/include/asm/mmzone.h               | 13 +++++
arch/riscv/include/asm/numa.h                 |  8 +++
arch/riscv/include/asm/pci.h                  | 10 ++++
arch/riscv/include/asm/pgtable.h              | 21 ++++++++
arch/riscv/kernel/setup.c                     | 12 ++++-
arch/riscv/kernel/smpboot.c                   | 12 ++++-
arch/riscv/mm/init.c                          | 10 +++-
drivers/base/Kconfig                          |  6 +++
drivers/base/Makefile                         |  1 +
.../mm/numa.c => drivers/base/arch_numa.c     | 19 ++++++-
include/asm-generic/numa.h                    | 51 +++++++++++++++++++
17 files changed, 190 insertions(+), 65 deletions(-)
create mode 100644 arch/riscv/include/asm/mmzone.h
create mode 100644 arch/riscv/include/asm/numa.h
rename arch/arm64/mm/numa.c => drivers/base/arch_numa.c (97%)
create mode 100644 include/asm-generic/numa.h

--
2.24.0

[RFC/RFT PATCH 1/6] numa: Move numa implementation to common code

[Atish Patra]

ARM64 numa implementation is generic enough that RISC-V can reuse that
implementation with very minor cosmetic changes. This will help both
ARM64 and RISC-V in terms of maintanace and feature improvement

Move the numa implementation code to common directory so that both ISAs
can reuse this. This doesn't introduce any function changes for ARM64.

Signed-off-by: Atish Patra <atish.patra@wdc.com>
---
 arch/arm64/Kconfig                            |  1 +
 arch/arm64/include/asm/numa.h                 | 45 +---------------
 arch/arm64/mm/Makefile                        |  1 -
 drivers/base/Kconfig                          |  6 +++
 drivers/base/Makefile                         |  1 +
 .../mm/numa.c => drivers/base/arch_numa.c     |  0
 include/asm-generic/numa.h                    | 51 +++++++++++++++++++
 7 files changed, 60 insertions(+), 45 deletions(-)
 rename arch/arm64/mm/numa.c => drivers/base/arch_numa.c (100%)
 create mode 100644 include/asm-generic/numa.h

diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig
index 6d232837cbee..955a0cf75b16 100644
--- a/arch/arm64/Kconfig
+++ b/arch/arm64/Kconfig
@@ -960,6 +960,7 @@ config HOTPLUG_CPU
 # Common NUMA Features
 config NUMA
 	bool "NUMA Memory Allocation and Scheduler Support"
+	select GENERIC_ARCH_NUMA
 	select ACPI_NUMA if ACPI
 	select OF_NUMA
 	help
diff --git a/arch/arm64/include/asm/numa.h b/arch/arm64/include/asm/numa.h
index 626ad01e83bf..8c8cf4297cc3 100644
--- a/arch/arm64/include/asm/numa.h
+++ b/arch/arm64/include/asm/numa.h
@@ -3,49 +3,6 @@
 #define __ASM_NUMA_H
 
 #include <asm/topology.h>
-
-#ifdef CONFIG_NUMA
-
-#define NR_NODE_MEMBLKS		(MAX_NUMNODES * 2)
-
-int __node_distance(int from, int to);
-#define node_distance(a, b) __node_distance(a, b)
-
-extern nodemask_t numa_nodes_parsed __initdata;
-
-extern bool numa_off;
-
-/* Mappings between node number and cpus on that node. */
-extern cpumask_var_t node_to_cpumask_map[MAX_NUMNODES];
-void numa_clear_node(unsigned int cpu);
-
-#ifdef CONFIG_DEBUG_PER_CPU_MAPS
-const struct cpumask *cpumask_of_node(int node);
-#else
-/* Returns a pointer to the cpumask of CPUs on Node 'node'. */
-static inline const struct cpumask *cpumask_of_node(int node)
-{
-	return node_to_cpumask_map[node];
-}
-#endif
-
-void __init arm64_numa_init(void);
-int __init numa_add_memblk(int nodeid, u64 start, u64 end);
-void __init numa_set_distance(int from, int to, int distance);
-void __init numa_free_distance(void);
-void __init early_map_cpu_to_node(unsigned int cpu, int nid);
-void numa_store_cpu_info(unsigned int cpu);
-void numa_add_cpu(unsigned int cpu);
-void numa_remove_cpu(unsigned int cpu);
-
-#else	/* CONFIG_NUMA */
-
-static inline void numa_store_cpu_info(unsigned int cpu) { }
-static inline void numa_add_cpu(unsigned int cpu) { }
-static inline void numa_remove_cpu(unsigned int cpu) { }
-static inline void arm64_numa_init(void) { }
-static inline void early_map_cpu_to_node(unsigned int cpu, int nid) { }
-
-#endif	/* CONFIG_NUMA */
+#include <asm-generic/numa.h>
 
 #endif	/* __ASM_NUMA_H */
diff --git a/arch/arm64/mm/Makefile b/arch/arm64/mm/Makefile
index d91030f0ffee..928c308b044b 100644
--- a/arch/arm64/mm/Makefile
+++ b/arch/arm64/mm/Makefile
@@ -6,7 +6,6 @@ obj-y				:= dma-mapping.o extable.o fault.o init.o \
 obj-$(CONFIG_HUGETLB_PAGE)	+= hugetlbpage.o
 obj-$(CONFIG_PTDUMP_CORE)	+= dump.o
 obj-$(CONFIG_PTDUMP_DEBUGFS)	+= ptdump_debugfs.o
-obj-$(CONFIG_NUMA)		+= numa.o
 obj-$(CONFIG_DEBUG_VIRTUAL)	+= physaddr.o
 KASAN_SANITIZE_physaddr.o	+= n
 
diff --git a/drivers/base/Kconfig b/drivers/base/Kconfig
index 8d7001712062..73c2151de194 100644
--- a/drivers/base/Kconfig
+++ b/drivers/base/Kconfig
@@ -210,4 +210,10 @@ config GENERIC_ARCH_TOPOLOGY
 	  appropriate scaling, sysfs interface for reading capacity values at
 	  runtime.
 
+config GENERIC_ARCH_NUMA
+	bool
+	help
+	  Enable support for generic numa implementation. Currently, RISC-V
+	  and ARM64 uses it.
+
 endmenu
diff --git a/drivers/base/Makefile b/drivers/base/Makefile
index 157452080f3d..c3d02c644222 100644
--- a/drivers/base/Makefile
+++ b/drivers/base/Makefile
@@ -23,6 +23,7 @@ obj-$(CONFIG_PINCTRL) += pinctrl.o
 obj-$(CONFIG_DEV_COREDUMP) += devcoredump.o
 obj-$(CONFIG_GENERIC_MSI_IRQ_DOMAIN) += platform-msi.o
 obj-$(CONFIG_GENERIC_ARCH_TOPOLOGY) += arch_topology.o
+obj-$(CONFIG_GENERIC_ARCH_NUMA) += arch_numa.o
 
 obj-y			+= test/
 
diff --git a/arch/arm64/mm/numa.c b/drivers/base/arch_numa.c
similarity index 100%
rename from arch/arm64/mm/numa.c
rename to drivers/base/arch_numa.c
diff --git a/include/asm-generic/numa.h b/include/asm-generic/numa.h
new file mode 100644
index 000000000000..0635c0724b7c
--- /dev/null
+++ b/include/asm-generic/numa.h
@@ -0,0 +1,51 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef __ASM_GENERIC_NUMA_H
+#define __ASM_GENERIC_NUMA_H
+
+#ifdef CONFIG_NUMA
+
+#define NR_NODE_MEMBLKS		(MAX_NUMNODES * 2)
+
+int __node_distance(int from, int to);
+#define node_distance(a, b) __node_distance(a, b)
+
+extern nodemask_t numa_nodes_parsed __initdata;
+
+extern bool numa_off;
+
+/* Mappings between node number and cpus on that node. */
+extern cpumask_var_t node_to_cpumask_map[MAX_NUMNODES];
+void numa_clear_node(unsigned int cpu);
+
+#ifdef CONFIG_DEBUG_PER_CPU_MAPS
+const struct cpumask *cpumask_of_node(int node);
+#else
+/* Returns a pointer to the cpumask of CPUs on Node 'node'. */
+static inline const struct cpumask *cpumask_of_node(int node)
+{
+	return node_to_cpumask_map[node];
+}
+#endif
+
+void __init arm64_numa_init(void);
+int __init numa_add_memblk(int nodeid, u64 start, u64 end);
+void __init numa_set_distance(int from, int to, int distance);
+void __init numa_free_distance(void);
+void __init early_map_cpu_to_node(unsigned int cpu, int nid);
+void numa_store_cpu_info(unsigned int cpu);
+void numa_add_cpu(unsigned int cpu);
+void numa_remove_cpu(unsigned int cpu);
+
+#else	/* CONFIG_NUMA */
+
+static inline void numa_store_cpu_info(unsigned int cpu) { }
+static inline void numa_add_cpu(unsigned int cpu) { }
+static inline void numa_remove_cpu(unsigned int cpu) { }
+static inline void arm64_numa_init(void) { }
+static inline void early_map_cpu_to_node(unsigned int cpu, int nid) { }
+
+#endif	/* CONFIG_NUMA */
+
+#include <asm/topology.h>
+
+#endif	/* __ASM_GENERIC_NUMA_H */
-- 
2.24.0

Re: [RFC/RFT PATCH 1/6] numa: Move numa implementation to common code

[Randy Dunlap]

On 8/14/20 2:47 PM, Atish Patra wrote:
> diff --git a/drivers/base/Kconfig b/drivers/base/Kconfig
> index 8d7001712062..73c2151de194 100644
> --- a/drivers/base/Kconfig
> +++ b/drivers/base/Kconfig
> @@ -210,4 +210,10 @@ config GENERIC_ARCH_TOPOLOGY
>  	  appropriate scaling, sysfs interface for reading capacity values at
>  	  runtime.
>  
> +config GENERIC_ARCH_NUMA
> +	bool
> +	help
> +	  Enable support for generic numa implementation. Currently, RISC-V

	                             NUMA

> +	  and ARM64 uses it.

	            use it.

> +
>  endmenu

thanks.
-- 
~Randy

Re: [RFC/RFT PATCH 1/6] numa: Move numa implementation to common code

[Anshuman Khandual]

On 08/15/2020 03:17 AM, Atish Patra wrote:
> ARM64 numa implementation is generic enough that RISC-V can reuse that
> implementation with very minor cosmetic changes. This will help both
> ARM64 and RISC-V in terms of maintanace and feature improvement
> 
> Move the numa implementation code to common directory so that both ISAs
> can reuse this. This doesn't introduce any function changes for ARM64.
> 
> Signed-off-by: Atish Patra <atish.patra@wdc.com>
> ---
>  arch/arm64/Kconfig                            |  1 +
>  arch/arm64/include/asm/numa.h                 | 45 +---------------
>  arch/arm64/mm/Makefile                        |  1 -
>  drivers/base/Kconfig                          |  6 +++
>  drivers/base/Makefile                         |  1 +
>  .../mm/numa.c => drivers/base/arch_numa.c     |  0
>  include/asm-generic/numa.h                    | 51 +++++++++++++++++++
>  7 files changed, 60 insertions(+), 45 deletions(-)
>  rename arch/arm64/mm/numa.c => drivers/base/arch_numa.c (100%)
>  create mode 100644 include/asm-generic/numa.h
> 
> diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig
> index 6d232837cbee..955a0cf75b16 100644
> --- a/arch/arm64/Kconfig
> +++ b/arch/arm64/Kconfig
> @@ -960,6 +960,7 @@ config HOTPLUG_CPU
>  # Common NUMA Features
>  config NUMA
>  	bool "NUMA Memory Allocation and Scheduler Support"
> +	select GENERIC_ARCH_NUMA

So this introduces a generic NUMA framework selectable with GENERIC_ARCH_NUMA.

>  	select ACPI_NUMA if ACPI
>  	select OF_NUMA
>  	help
> diff --git a/arch/arm64/include/asm/numa.h b/arch/arm64/include/asm/numa.h
> index 626ad01e83bf..8c8cf4297cc3 100644
> --- a/arch/arm64/include/asm/numa.h
> +++ b/arch/arm64/include/asm/numa.h
> @@ -3,49 +3,6 @@
>  #define __ASM_NUMA_H
>  
>  #include <asm/topology.h>
> -
> -#ifdef CONFIG_NUMA
> -
> -#define NR_NODE_MEMBLKS		(MAX_NUMNODES * 2)
> -
> -int __node_distance(int from, int to);
> -#define node_distance(a, b) __node_distance(a, b)
> -
> -extern nodemask_t numa_nodes_parsed __initdata;
> -
> -extern bool numa_off;
> -
> -/* Mappings between node number and cpus on that node. */
> -extern cpumask_var_t node_to_cpumask_map[MAX_NUMNODES];
> -void numa_clear_node(unsigned int cpu);
> -
> -#ifdef CONFIG_DEBUG_PER_CPU_MAPS
> -const struct cpumask *cpumask_of_node(int node);
> -#else
> -/* Returns a pointer to the cpumask of CPUs on Node 'node'. */
> -static inline const struct cpumask *cpumask_of_node(int node)
> -{
> -	return node_to_cpumask_map[node];
> -}
> -#endif
> -
> -void __init arm64_numa_init(void);
> -int __init numa_add_memblk(int nodeid, u64 start, u64 end);
> -void __init numa_set_distance(int from, int to, int distance);
> -void __init numa_free_distance(void);
> -void __init early_map_cpu_to_node(unsigned int cpu, int nid);
> -void numa_store_cpu_info(unsigned int cpu);
> -void numa_add_cpu(unsigned int cpu);
> -void numa_remove_cpu(unsigned int cpu);
> -
> -#else	/* CONFIG_NUMA */
> -
> -static inline void numa_store_cpu_info(unsigned int cpu) { }
> -static inline void numa_add_cpu(unsigned int cpu) { }
> -static inline void numa_remove_cpu(unsigned int cpu) { }
> -static inline void arm64_numa_init(void) { }
> -static inline void early_map_cpu_to_node(unsigned int cpu, int nid) { }
> -
> -#endif	/* CONFIG_NUMA */
> +#include <asm-generic/numa.h>
>  
>  #endif	/* __ASM_NUMA_H */
> diff --git a/arch/arm64/mm/Makefile b/arch/arm64/mm/Makefile
> index d91030f0ffee..928c308b044b 100644
> --- a/arch/arm64/mm/Makefile
> +++ b/arch/arm64/mm/Makefile
> @@ -6,7 +6,6 @@ obj-y				:= dma-mapping.o extable.o fault.o init.o \
>  obj-$(CONFIG_HUGETLB_PAGE)	+= hugetlbpage.o
>  obj-$(CONFIG_PTDUMP_CORE)	+= dump.o
>  obj-$(CONFIG_PTDUMP_DEBUGFS)	+= ptdump_debugfs.o
> -obj-$(CONFIG_NUMA)		+= numa.o
>  obj-$(CONFIG_DEBUG_VIRTUAL)	+= physaddr.o
>  KASAN_SANITIZE_physaddr.o	+= n
>  
> diff --git a/drivers/base/Kconfig b/drivers/base/Kconfig
> index 8d7001712062..73c2151de194 100644
> --- a/drivers/base/Kconfig
> +++ b/drivers/base/Kconfig
> @@ -210,4 +210,10 @@ config GENERIC_ARCH_TOPOLOGY
>  	  appropriate scaling, sysfs interface for reading capacity values at
>  	  runtime.
>  
> +config GENERIC_ARCH_NUMA
> +	bool
> +	help
> +	  Enable support for generic numa implementation. Currently, RISC-V
> +	  and ARM64 uses it.
> +
>  endmenu
> diff --git a/drivers/base/Makefile b/drivers/base/Makefile
> index 157452080f3d..c3d02c644222 100644
> --- a/drivers/base/Makefile
> +++ b/drivers/base/Makefile
> @@ -23,6 +23,7 @@ obj-$(CONFIG_PINCTRL) += pinctrl.o
>  obj-$(CONFIG_DEV_COREDUMP) += devcoredump.o
>  obj-$(CONFIG_GENERIC_MSI_IRQ_DOMAIN) += platform-msi.o
>  obj-$(CONFIG_GENERIC_ARCH_TOPOLOGY) += arch_topology.o
> +obj-$(CONFIG_GENERIC_ARCH_NUMA) += arch_numa.o
>  
>  obj-y			+= test/
>  
> diff --git a/arch/arm64/mm/numa.c b/drivers/base/arch_numa.c
> similarity index 100%
> rename from arch/arm64/mm/numa.c
> rename to drivers/base/arch_numa.c

drivers/base/ does not seem right place to host generic NUMA code.
Probably it should be either mm/ or kernel/. The other question here
would be if existing arm64 NUMA implementation is sufficient enough
for generic NUMA. I would expect any platform selecting this config
should get some NUMA enabled, will be that be true with present code ?
Otherwise it will be difficult to name it as GENERIC_ARCH_NUMA.

Re: [RFC/RFT PATCH 1/6] numa: Move numa implementation to common code

[Atish Patra]

On Tue, Aug 18, 2020 at 8:19 PM Anshuman Khandual
<anshuman.khandual@arm.com> wrote:
>
>
>
> On 08/15/2020 03:17 AM, Atish Patra wrote:
> > ARM64 numa implementation is generic enough that RISC-V can reuse that
> > implementation with very minor cosmetic changes. This will help both
> > ARM64 and RISC-V in terms of maintanace and feature improvement
> >
> > Move the numa implementation code to common directory so that both ISAs
> > can reuse this. This doesn't introduce any function changes for ARM64.
> >
> > Signed-off-by: Atish Patra <atish.patra@wdc.com>
> > ---
> >  arch/arm64/Kconfig                            |  1 +
> >  arch/arm64/include/asm/numa.h                 | 45 +---------------
> >  arch/arm64/mm/Makefile                        |  1 -
> >  drivers/base/Kconfig                          |  6 +++
> >  drivers/base/Makefile                         |  1 +
> >  .../mm/numa.c => drivers/base/arch_numa.c     |  0
> >  include/asm-generic/numa.h                    | 51 +++++++++++++++++++
> >  7 files changed, 60 insertions(+), 45 deletions(-)
> >  rename arch/arm64/mm/numa.c => drivers/base/arch_numa.c (100%)
> >  create mode 100644 include/asm-generic/numa.h
> >
> > diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig
> > index 6d232837cbee..955a0cf75b16 100644
> > --- a/arch/arm64/Kconfig
> > +++ b/arch/arm64/Kconfig
> > @@ -960,6 +960,7 @@ config HOTPLUG_CPU
> >  # Common NUMA Features
> >  config NUMA
> >       bool "NUMA Memory Allocation and Scheduler Support"
> > +     select GENERIC_ARCH_NUMA
>
> So this introduces a generic NUMA framework selectable with GENERIC_ARCH_NUMA.
>
> >       select ACPI_NUMA if ACPI
> >       select OF_NUMA
> >       help
> > diff --git a/arch/arm64/include/asm/numa.h b/arch/arm64/include/asm/numa.h
> > index 626ad01e83bf..8c8cf4297cc3 100644
> > --- a/arch/arm64/include/asm/numa.h
> > +++ b/arch/arm64/include/asm/numa.h
> > @@ -3,49 +3,6 @@
> >  #define __ASM_NUMA_H
> >
> >  #include <asm/topology.h>
> > -
> > -#ifdef CONFIG_NUMA
> > -
> > -#define NR_NODE_MEMBLKS              (MAX_NUMNODES * 2)
> > -
> > -int __node_distance(int from, int to);
> > -#define node_distance(a, b) __node_distance(a, b)
> > -
> > -extern nodemask_t numa_nodes_parsed __initdata;
> > -
> > -extern bool numa_off;
> > -
> > -/* Mappings between node number and cpus on that node. */
> > -extern cpumask_var_t node_to_cpumask_map[MAX_NUMNODES];
> > -void numa_clear_node(unsigned int cpu);
> > -
> > -#ifdef CONFIG_DEBUG_PER_CPU_MAPS
> > -const struct cpumask *cpumask_of_node(int node);
> > -#else
> > -/* Returns a pointer to the cpumask of CPUs on Node 'node'. */
> > -static inline const struct cpumask *cpumask_of_node(int node)
> > -{
> > -     return node_to_cpumask_map[node];
> > -}
> > -#endif
> > -
> > -void __init arm64_numa_init(void);
> > -int __init numa_add_memblk(int nodeid, u64 start, u64 end);
> > -void __init numa_set_distance(int from, int to, int distance);
> > -void __init numa_free_distance(void);
> > -void __init early_map_cpu_to_node(unsigned int cpu, int nid);
> > -void numa_store_cpu_info(unsigned int cpu);
> > -void numa_add_cpu(unsigned int cpu);
> > -void numa_remove_cpu(unsigned int cpu);
> > -
> > -#else        /* CONFIG_NUMA */
> > -
> > -static inline void numa_store_cpu_info(unsigned int cpu) { }
> > -static inline void numa_add_cpu(unsigned int cpu) { }
> > -static inline void numa_remove_cpu(unsigned int cpu) { }
> > -static inline void arm64_numa_init(void) { }
> > -static inline void early_map_cpu_to_node(unsigned int cpu, int nid) { }
> > -
> > -#endif       /* CONFIG_NUMA */
> > +#include <asm-generic/numa.h>
> >
> >  #endif       /* __ASM_NUMA_H */
> > diff --git a/arch/arm64/mm/Makefile b/arch/arm64/mm/Makefile
> > index d91030f0ffee..928c308b044b 100644
> > --- a/arch/arm64/mm/Makefile
> > +++ b/arch/arm64/mm/Makefile
> > @@ -6,7 +6,6 @@ obj-y                         := dma-mapping.o extable.o fault.o init.o \
> >  obj-$(CONFIG_HUGETLB_PAGE)   += hugetlbpage.o
> >  obj-$(CONFIG_PTDUMP_CORE)    += dump.o
> >  obj-$(CONFIG_PTDUMP_DEBUGFS) += ptdump_debugfs.o
> > -obj-$(CONFIG_NUMA)           += numa.o
> >  obj-$(CONFIG_DEBUG_VIRTUAL)  += physaddr.o
> >  KASAN_SANITIZE_physaddr.o    += n
> >
> > diff --git a/drivers/base/Kconfig b/drivers/base/Kconfig
> > index 8d7001712062..73c2151de194 100644
> > --- a/drivers/base/Kconfig
> > +++ b/drivers/base/Kconfig
> > @@ -210,4 +210,10 @@ config GENERIC_ARCH_TOPOLOGY
> >         appropriate scaling, sysfs interface for reading capacity values at
> >         runtime.
> >
> > +config GENERIC_ARCH_NUMA
> > +     bool
> > +     help
> > +       Enable support for generic numa implementation. Currently, RISC-V
> > +       and ARM64 uses it.
> > +
> >  endmenu
> > diff --git a/drivers/base/Makefile b/drivers/base/Makefile
> > index 157452080f3d..c3d02c644222 100644
> > --- a/drivers/base/Makefile
> > +++ b/drivers/base/Makefile
> > @@ -23,6 +23,7 @@ obj-$(CONFIG_PINCTRL) += pinctrl.o
> >  obj-$(CONFIG_DEV_COREDUMP) += devcoredump.o
> >  obj-$(CONFIG_GENERIC_MSI_IRQ_DOMAIN) += platform-msi.o
> >  obj-$(CONFIG_GENERIC_ARCH_TOPOLOGY) += arch_topology.o
> > +obj-$(CONFIG_GENERIC_ARCH_NUMA) += arch_numa.o
> >
> >  obj-y                        += test/
> >
> > diff --git a/arch/arm64/mm/numa.c b/drivers/base/arch_numa.c
> > similarity index 100%
> > rename from arch/arm64/mm/numa.c
> > rename to drivers/base/arch_numa.c
>
> drivers/base/ does not seem right place to host generic NUMA code.

I chose drivers/base because the common topology code is also present there.
drivers/base/arch_topology.c under GENERIC_ARCH_TOPOLOGY
The idea is to keep all common arch(at least between RISC-V & ARM64)
related code at one place.

> Probably it should be either mm/ or kernel/. The other question here

I am fine with mm/arch_numa.c as well if that is preferred over driver/base.

> would be if existing arm64 NUMA implementation is sufficient enough
> for generic NUMA. I would expect any platform selecting this config
> should get some NUMA enabled, will be that be true with present code ?

It is for RISC-V. Here is the RISC-V support patch (last patch in the series)

http://lists.infradead.org/pipermail/linux-riscv/2020-August/001659.html

> Otherwise it will be difficult to name it as GENERIC_ARCH_NUMA.
>
> _______________________________________________
> linux-riscv mailing list
> linux-riscv@lists.infradead.org
> http://lists.infradead.org/mailman/listinfo/linux-riscv



-- 
Regards,
Atish

Re: [RFC/RFT PATCH 1/6] numa: Move numa implementation to common code

[Anshuman Khandual]

On 08/20/2020 12:48 AM, Atish Patra wrote:
> On Tue, Aug 18, 2020 at 8:19 PM Anshuman Khandual
> <anshuman.khandual@arm.com> wrote:
>>
>>
>>
>> On 08/15/2020 03:17 AM, Atish Patra wrote:
>>> ARM64 numa implementation is generic enough that RISC-V can reuse that
>>> implementation with very minor cosmetic changes. This will help both
>>> ARM64 and RISC-V in terms of maintanace and feature improvement
>>>
>>> Move the numa implementation code to common directory so that both ISAs
>>> can reuse this. This doesn't introduce any function changes for ARM64.
>>>
>>> Signed-off-by: Atish Patra <atish.patra@wdc.com>
>>> ---
>>>  arch/arm64/Kconfig                            |  1 +
>>>  arch/arm64/include/asm/numa.h                 | 45 +---------------
>>>  arch/arm64/mm/Makefile                        |  1 -
>>>  drivers/base/Kconfig                          |  6 +++
>>>  drivers/base/Makefile                         |  1 +
>>>  .../mm/numa.c => drivers/base/arch_numa.c     |  0
>>>  include/asm-generic/numa.h                    | 51 +++++++++++++++++++
>>>  7 files changed, 60 insertions(+), 45 deletions(-)
>>>  rename arch/arm64/mm/numa.c => drivers/base/arch_numa.c (100%)
>>>  create mode 100644 include/asm-generic/numa.h
>>>
>>> diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig
>>> index 6d232837cbee..955a0cf75b16 100644
>>> --- a/arch/arm64/Kconfig
>>> +++ b/arch/arm64/Kconfig
>>> @@ -960,6 +960,7 @@ config HOTPLUG_CPU
>>>  # Common NUMA Features
>>>  config NUMA
>>>       bool "NUMA Memory Allocation and Scheduler Support"
>>> +     select GENERIC_ARCH_NUMA
>>
>> So this introduces a generic NUMA framework selectable with GENERIC_ARCH_NUMA.
>>
>>>       select ACPI_NUMA if ACPI
>>>       select OF_NUMA
>>>       help
>>> diff --git a/arch/arm64/include/asm/numa.h b/arch/arm64/include/asm/numa.h
>>> index 626ad01e83bf..8c8cf4297cc3 100644
>>> --- a/arch/arm64/include/asm/numa.h
>>> +++ b/arch/arm64/include/asm/numa.h
>>> @@ -3,49 +3,6 @@
>>>  #define __ASM_NUMA_H
>>>
>>>  #include <asm/topology.h>
>>> -
>>> -#ifdef CONFIG_NUMA
>>> -
>>> -#define NR_NODE_MEMBLKS              (MAX_NUMNODES * 2)
>>> -
>>> -int __node_distance(int from, int to);
>>> -#define node_distance(a, b) __node_distance(a, b)
>>> -
>>> -extern nodemask_t numa_nodes_parsed __initdata;
>>> -
>>> -extern bool numa_off;
>>> -
>>> -/* Mappings between node number and cpus on that node. */
>>> -extern cpumask_var_t node_to_cpumask_map[MAX_NUMNODES];
>>> -void numa_clear_node(unsigned int cpu);
>>> -
>>> -#ifdef CONFIG_DEBUG_PER_CPU_MAPS
>>> -const struct cpumask *cpumask_of_node(int node);
>>> -#else
>>> -/* Returns a pointer to the cpumask of CPUs on Node 'node'. */
>>> -static inline const struct cpumask *cpumask_of_node(int node)
>>> -{
>>> -     return node_to_cpumask_map[node];
>>> -}
>>> -#endif
>>> -
>>> -void __init arm64_numa_init(void);
>>> -int __init numa_add_memblk(int nodeid, u64 start, u64 end);
>>> -void __init numa_set_distance(int from, int to, int distance);
>>> -void __init numa_free_distance(void);
>>> -void __init early_map_cpu_to_node(unsigned int cpu, int nid);
>>> -void numa_store_cpu_info(unsigned int cpu);
>>> -void numa_add_cpu(unsigned int cpu);
>>> -void numa_remove_cpu(unsigned int cpu);
>>> -
>>> -#else        /* CONFIG_NUMA */
>>> -
>>> -static inline void numa_store_cpu_info(unsigned int cpu) { }
>>> -static inline void numa_add_cpu(unsigned int cpu) { }
>>> -static inline void numa_remove_cpu(unsigned int cpu) { }
>>> -static inline void arm64_numa_init(void) { }
>>> -static inline void early_map_cpu_to_node(unsigned int cpu, int nid) { }
>>> -
>>> -#endif       /* CONFIG_NUMA */
>>> +#include <asm-generic/numa.h>
>>>
>>>  #endif       /* __ASM_NUMA_H */
>>> diff --git a/arch/arm64/mm/Makefile b/arch/arm64/mm/Makefile
>>> index d91030f0ffee..928c308b044b 100644
>>> --- a/arch/arm64/mm/Makefile
>>> +++ b/arch/arm64/mm/Makefile
>>> @@ -6,7 +6,6 @@ obj-y                         := dma-mapping.o extable.o fault.o init.o \
>>>  obj-$(CONFIG_HUGETLB_PAGE)   += hugetlbpage.o
>>>  obj-$(CONFIG_PTDUMP_CORE)    += dump.o
>>>  obj-$(CONFIG_PTDUMP_DEBUGFS) += ptdump_debugfs.o
>>> -obj-$(CONFIG_NUMA)           += numa.o
>>>  obj-$(CONFIG_DEBUG_VIRTUAL)  += physaddr.o
>>>  KASAN_SANITIZE_physaddr.o    += n
>>>
>>> diff --git a/drivers/base/Kconfig b/drivers/base/Kconfig
>>> index 8d7001712062..73c2151de194 100644
>>> --- a/drivers/base/Kconfig
>>> +++ b/drivers/base/Kconfig
>>> @@ -210,4 +210,10 @@ config GENERIC_ARCH_TOPOLOGY
>>>         appropriate scaling, sysfs interface for reading capacity values at
>>>         runtime.
>>>
>>> +config GENERIC_ARCH_NUMA
>>> +     bool
>>> +     help
>>> +       Enable support for generic numa implementation. Currently, RISC-V
>>> +       and ARM64 uses it.
>>> +
>>>  endmenu
>>> diff --git a/drivers/base/Makefile b/drivers/base/Makefile
>>> index 157452080f3d..c3d02c644222 100644
>>> --- a/drivers/base/Makefile
>>> +++ b/drivers/base/Makefile
>>> @@ -23,6 +23,7 @@ obj-$(CONFIG_PINCTRL) += pinctrl.o
>>>  obj-$(CONFIG_DEV_COREDUMP) += devcoredump.o
>>>  obj-$(CONFIG_GENERIC_MSI_IRQ_DOMAIN) += platform-msi.o
>>>  obj-$(CONFIG_GENERIC_ARCH_TOPOLOGY) += arch_topology.o
>>> +obj-$(CONFIG_GENERIC_ARCH_NUMA) += arch_numa.o
>>>
>>>  obj-y                        += test/
>>>
>>> diff --git a/arch/arm64/mm/numa.c b/drivers/base/arch_numa.c
>>> similarity index 100%
>>> rename from arch/arm64/mm/numa.c
>>> rename to drivers/base/arch_numa.c
>>
>> drivers/base/ does not seem right place to host generic NUMA code.
> 
> I chose drivers/base because the common topology code is also present there.
> drivers/base/arch_topology.c under GENERIC_ARCH_TOPOLOGY
> The idea is to keep all common arch(at least between RISC-V & ARM64)
> related code at one place.
> 
>> Probably it should be either mm/ or kernel/. The other question here
> 
> I am fine with mm/arch_numa.c as well if that is preferred over driver/base.

GENERIC_ARCH_NUMA being near other shared code such as GENERIC_ARCH_TOPOLOGY
do make sense. That being said, its a small nit and can be figured out later.

> 
>> would be if existing arm64 NUMA implementation is sufficient enough
>> for generic NUMA. I would expect any platform selecting this config
>> should get some NUMA enabled, will be that be true with present code ?
> 
> It is for RISC-V. Here is the RISC-V support patch (last patch in the series)
> 
> http://lists.infradead.org/pipermail/linux-riscv/2020-August/001659.html
>

+ Jonathan Cameron <Jonathan.Cameron@huawei.com>

There is another patch/discussion which is trying to unify ARM64 NUMA init
code with X86 (https://patchwork.kernel.org/patch/11651437/). I am wondering
if all three platforms could use GENERIC_ARCH_NUMA.
 
>> Otherwise it will be difficult to name it as GENERIC_ARCH_NUMA.
>>
>> _______________________________________________
>> linux-riscv mailing list
>> linux-riscv@lists.infradead.org
>> http://lists.infradead.org/mailman/listinfo/linux-riscv
> 
> 
> 

Re: [RFC/RFT PATCH 1/6] numa: Move numa implementation to common code

[Atish Patra]

On Wed, Aug 19, 2020 at 8:20 PM Anshuman Khandual
<anshuman.khandual@arm.com> wrote:
>
>
>
> On 08/20/2020 12:48 AM, Atish Patra wrote:
> > On Tue, Aug 18, 2020 at 8:19 PM Anshuman Khandual
> > <anshuman.khandual@arm.com> wrote:
> >>
> >>
> >>
> >> On 08/15/2020 03:17 AM, Atish Patra wrote:
> >>> ARM64 numa implementation is generic enough that RISC-V can reuse that
> >>> implementation with very minor cosmetic changes. This will help both
> >>> ARM64 and RISC-V in terms of maintanace and feature improvement
> >>>
> >>> Move the numa implementation code to common directory so that both ISAs
> >>> can reuse this. This doesn't introduce any function changes for ARM64.
> >>>
> >>> Signed-off-by: Atish Patra <atish.patra@wdc.com>
> >>> ---
> >>>  arch/arm64/Kconfig                            |  1 +
> >>>  arch/arm64/include/asm/numa.h                 | 45 +---------------
> >>>  arch/arm64/mm/Makefile                        |  1 -
> >>>  drivers/base/Kconfig                          |  6 +++
> >>>  drivers/base/Makefile                         |  1 +
> >>>  .../mm/numa.c => drivers/base/arch_numa.c     |  0
> >>>  include/asm-generic/numa.h                    | 51 +++++++++++++++++++
> >>>  7 files changed, 60 insertions(+), 45 deletions(-)
> >>>  rename arch/arm64/mm/numa.c => drivers/base/arch_numa.c (100%)
> >>>  create mode 100644 include/asm-generic/numa.h
> >>>
> >>> diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig
> >>> index 6d232837cbee..955a0cf75b16 100644
> >>> --- a/arch/arm64/Kconfig
> >>> +++ b/arch/arm64/Kconfig
> >>> @@ -960,6 +960,7 @@ config HOTPLUG_CPU
> >>>  # Common NUMA Features
> >>>  config NUMA
> >>>       bool "NUMA Memory Allocation and Scheduler Support"
> >>> +     select GENERIC_ARCH_NUMA
> >>
> >> So this introduces a generic NUMA framework selectable with GENERIC_ARCH_NUMA.
> >>
> >>>       select ACPI_NUMA if ACPI
> >>>       select OF_NUMA
> >>>       help
> >>> diff --git a/arch/arm64/include/asm/numa.h b/arch/arm64/include/asm/numa.h
> >>> index 626ad01e83bf..8c8cf4297cc3 100644
> >>> --- a/arch/arm64/include/asm/numa.h
> >>> +++ b/arch/arm64/include/asm/numa.h
> >>> @@ -3,49 +3,6 @@
> >>>  #define __ASM_NUMA_H
> >>>
> >>>  #include <asm/topology.h>
> >>> -
> >>> -#ifdef CONFIG_NUMA
> >>> -
> >>> -#define NR_NODE_MEMBLKS              (MAX_NUMNODES * 2)
> >>> -
> >>> -int __node_distance(int from, int to);
> >>> -#define node_distance(a, b) __node_distance(a, b)
> >>> -
> >>> -extern nodemask_t numa_nodes_parsed __initdata;
> >>> -
> >>> -extern bool numa_off;
> >>> -
> >>> -/* Mappings between node number and cpus on that node. */
> >>> -extern cpumask_var_t node_to_cpumask_map[MAX_NUMNODES];
> >>> -void numa_clear_node(unsigned int cpu);
> >>> -
> >>> -#ifdef CONFIG_DEBUG_PER_CPU_MAPS
> >>> -const struct cpumask *cpumask_of_node(int node);
> >>> -#else
> >>> -/* Returns a pointer to the cpumask of CPUs on Node 'node'. */
> >>> -static inline const struct cpumask *cpumask_of_node(int node)
> >>> -{
> >>> -     return node_to_cpumask_map[node];
> >>> -}
> >>> -#endif
> >>> -
> >>> -void __init arm64_numa_init(void);
> >>> -int __init numa_add_memblk(int nodeid, u64 start, u64 end);
> >>> -void __init numa_set_distance(int from, int to, int distance);
> >>> -void __init numa_free_distance(void);
> >>> -void __init early_map_cpu_to_node(unsigned int cpu, int nid);
> >>> -void numa_store_cpu_info(unsigned int cpu);
> >>> -void numa_add_cpu(unsigned int cpu);
> >>> -void numa_remove_cpu(unsigned int cpu);
> >>> -
> >>> -#else        /* CONFIG_NUMA */
> >>> -
> >>> -static inline void numa_store_cpu_info(unsigned int cpu) { }
> >>> -static inline void numa_add_cpu(unsigned int cpu) { }
> >>> -static inline void numa_remove_cpu(unsigned int cpu) { }
> >>> -static inline void arm64_numa_init(void) { }
> >>> -static inline void early_map_cpu_to_node(unsigned int cpu, int nid) { }
> >>> -
> >>> -#endif       /* CONFIG_NUMA */
> >>> +#include <asm-generic/numa.h>
> >>>
> >>>  #endif       /* __ASM_NUMA_H */
> >>> diff --git a/arch/arm64/mm/Makefile b/arch/arm64/mm/Makefile
> >>> index d91030f0ffee..928c308b044b 100644
> >>> --- a/arch/arm64/mm/Makefile
> >>> +++ b/arch/arm64/mm/Makefile
> >>> @@ -6,7 +6,6 @@ obj-y                         := dma-mapping.o extable.o fault.o init.o \
> >>>  obj-$(CONFIG_HUGETLB_PAGE)   += hugetlbpage.o
> >>>  obj-$(CONFIG_PTDUMP_CORE)    += dump.o
> >>>  obj-$(CONFIG_PTDUMP_DEBUGFS) += ptdump_debugfs.o
> >>> -obj-$(CONFIG_NUMA)           += numa.o
> >>>  obj-$(CONFIG_DEBUG_VIRTUAL)  += physaddr.o
> >>>  KASAN_SANITIZE_physaddr.o    += n
> >>>
> >>> diff --git a/drivers/base/Kconfig b/drivers/base/Kconfig
> >>> index 8d7001712062..73c2151de194 100644
> >>> --- a/drivers/base/Kconfig
> >>> +++ b/drivers/base/Kconfig
> >>> @@ -210,4 +210,10 @@ config GENERIC_ARCH_TOPOLOGY
> >>>         appropriate scaling, sysfs interface for reading capacity values at
> >>>         runtime.
> >>>
> >>> +config GENERIC_ARCH_NUMA
> >>> +     bool
> >>> +     help
> >>> +       Enable support for generic numa implementation. Currently, RISC-V
> >>> +       and ARM64 uses it.
> >>> +
> >>>  endmenu
> >>> diff --git a/drivers/base/Makefile b/drivers/base/Makefile
> >>> index 157452080f3d..c3d02c644222 100644
> >>> --- a/drivers/base/Makefile
> >>> +++ b/drivers/base/Makefile
> >>> @@ -23,6 +23,7 @@ obj-$(CONFIG_PINCTRL) += pinctrl.o
> >>>  obj-$(CONFIG_DEV_COREDUMP) += devcoredump.o
> >>>  obj-$(CONFIG_GENERIC_MSI_IRQ_DOMAIN) += platform-msi.o
> >>>  obj-$(CONFIG_GENERIC_ARCH_TOPOLOGY) += arch_topology.o
> >>> +obj-$(CONFIG_GENERIC_ARCH_NUMA) += arch_numa.o
> >>>
> >>>  obj-y                        += test/
> >>>
> >>> diff --git a/arch/arm64/mm/numa.c b/drivers/base/arch_numa.c
> >>> similarity index 100%
> >>> rename from arch/arm64/mm/numa.c
> >>> rename to drivers/base/arch_numa.c
> >>
> >> drivers/base/ does not seem right place to host generic NUMA code.
> >
> > I chose drivers/base because the common topology code is also present there.
> > drivers/base/arch_topology.c under GENERIC_ARCH_TOPOLOGY
> > The idea is to keep all common arch(at least between RISC-V & ARM64)
> > related code at one place.
> >
> >> Probably it should be either mm/ or kernel/. The other question here
> >
> > I am fine with mm/arch_numa.c as well if that is preferred over driver/base.
>
> GENERIC_ARCH_NUMA being near other shared code such as GENERIC_ARCH_TOPOLOGY
> do make sense. That being said, its a small nit and can be figured out later.
>
> >
> >> would be if existing arm64 NUMA implementation is sufficient enough
> >> for generic NUMA. I would expect any platform selecting this config
> >> should get some NUMA enabled, will be that be true with present code ?
> >
> > It is for RISC-V. Here is the RISC-V support patch (last patch in the series)
> >
> > http://lists.infradead.org/pipermail/linux-riscv/2020-August/001659.html
> >
>
> + Jonathan Cameron <Jonathan.Cameron@huawei.com>
>
> There is another patch/discussion which is trying to unify ARM64 NUMA init
> code with X86 (https://patchwork.kernel.org/patch/11651437/). I am wondering
> if all three platforms could use GENERIC_ARCH_NUMA.
>

Gmail decided to convert my previous reply to HTML for some reason and
was blocked by the mailing lists.
Here was my earlier response and apologies for the noise if you
received it twice.

That is certainly an awesome goal to achieve. I agree that there are a
lot of similarities between two implementations
that can be leveraged under common code. But the current arm64 or x86
numa implementation
have also enough differences that  can't just be reused for either.
This series did not introduce any functional
difference to arm64 numa code and just moved the existing code between
files. I don't think that's possible
for x86 under GENERIC_ARCH_NUMA. It requires a bit more effort to do
that and I am interested to explore that.

How about merging this series first and slowly moving pieces of x86
NUMA code to generic numa code as a separate series ?

> >> Otherwise it will be difficult to name it as GENERIC_ARCH_NUMA.
> >>
> >> _______________________________________________
> >> linux-riscv mailing list
> >> linux-riscv@lists.infradead.org
> >> http://lists.infradead.org/mailman/listinfo/linux-riscv
> >
> >
> >



-- 
Regards,
Atish

Re: [RFC/RFT PATCH 1/6] numa: Move numa implementation to common code

[Jonathan Cameron]

On Fri, 21 Aug 2020 14:58:22 -0700
Atish Patra <atishp@atishpatra.org> wrote:

> On Wed, Aug 19, 2020 at 8:20 PM Anshuman Khandual
> <anshuman.khandual@arm.com> wrote:
> >
> >
> >
> > On 08/20/2020 12:48 AM, Atish Patra wrote:  
> > > On Tue, Aug 18, 2020 at 8:19 PM Anshuman Khandual
> > > <anshuman.khandual@arm.com> wrote:  
> > >>
> > >>
> > >>
> > >> On 08/15/2020 03:17 AM, Atish Patra wrote:  
> > >>> ARM64 numa implementation is generic enough that RISC-V can reuse that
> > >>> implementation with very minor cosmetic changes. This will help both
> > >>> ARM64 and RISC-V in terms of maintanace and feature improvement
> > >>>
> > >>> Move the numa implementation code to common directory so that both ISAs
> > >>> can reuse this. This doesn't introduce any function changes for ARM64.
> > >>>
> > >>> Signed-off-by: Atish Patra <atish.patra@wdc.com>
> > >>> ---
> > >>>  arch/arm64/Kconfig                            |  1 +
> > >>>  arch/arm64/include/asm/numa.h                 | 45 +---------------
> > >>>  arch/arm64/mm/Makefile                        |  1 -
> > >>>  drivers/base/Kconfig                          |  6 +++
> > >>>  drivers/base/Makefile                         |  1 +
> > >>>  .../mm/numa.c => drivers/base/arch_numa.c     |  0
> > >>>  include/asm-generic/numa.h                    | 51 +++++++++++++++++++
> > >>>  7 files changed, 60 insertions(+), 45 deletions(-)
> > >>>  rename arch/arm64/mm/numa.c => drivers/base/arch_numa.c (100%)
> > >>>  create mode 100644 include/asm-generic/numa.h
> > >>>
> > >>> diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig
> > >>> index 6d232837cbee..955a0cf75b16 100644
> > >>> --- a/arch/arm64/Kconfig
> > >>> +++ b/arch/arm64/Kconfig
> > >>> @@ -960,6 +960,7 @@ config HOTPLUG_CPU
> > >>>  # Common NUMA Features
> > >>>  config NUMA
> > >>>       bool "NUMA Memory Allocation and Scheduler Support"
> > >>> +     select GENERIC_ARCH_NUMA  
> > >>
> > >> So this introduces a generic NUMA framework selectable with GENERIC_ARCH_NUMA.
> > >>  
> > >>>       select ACPI_NUMA if ACPI
> > >>>       select OF_NUMA
> > >>>       help
> > >>> diff --git a/arch/arm64/include/asm/numa.h b/arch/arm64/include/asm/numa.h
> > >>> index 626ad01e83bf..8c8cf4297cc3 100644
> > >>> --- a/arch/arm64/include/asm/numa.h
> > >>> +++ b/arch/arm64/include/asm/numa.h
> > >>> @@ -3,49 +3,6 @@
> > >>>  #define __ASM_NUMA_H
> > >>>
> > >>>  #include <asm/topology.h>
> > >>> -
> > >>> -#ifdef CONFIG_NUMA
> > >>> -
> > >>> -#define NR_NODE_MEMBLKS              (MAX_NUMNODES * 2)
> > >>> -
> > >>> -int __node_distance(int from, int to);
> > >>> -#define node_distance(a, b) __node_distance(a, b)
> > >>> -
> > >>> -extern nodemask_t numa_nodes_parsed __initdata;
> > >>> -
> > >>> -extern bool numa_off;
> > >>> -
> > >>> -/* Mappings between node number and cpus on that node. */
> > >>> -extern cpumask_var_t node_to_cpumask_map[MAX_NUMNODES];
> > >>> -void numa_clear_node(unsigned int cpu);
> > >>> -
> > >>> -#ifdef CONFIG_DEBUG_PER_CPU_MAPS
> > >>> -const struct cpumask *cpumask_of_node(int node);
> > >>> -#else
> > >>> -/* Returns a pointer to the cpumask of CPUs on Node 'node'. */
> > >>> -static inline const struct cpumask *cpumask_of_node(int node)
> > >>> -{
> > >>> -     return node_to_cpumask_map[node];
> > >>> -}
> > >>> -#endif
> > >>> -
> > >>> -void __init arm64_numa_init(void);
> > >>> -int __init numa_add_memblk(int nodeid, u64 start, u64 end);
> > >>> -void __init numa_set_distance(int from, int to, int distance);
> > >>> -void __init numa_free_distance(void);
> > >>> -void __init early_map_cpu_to_node(unsigned int cpu, int nid);
> > >>> -void numa_store_cpu_info(unsigned int cpu);
> > >>> -void numa_add_cpu(unsigned int cpu);
> > >>> -void numa_remove_cpu(unsigned int cpu);
> > >>> -
> > >>> -#else        /* CONFIG_NUMA */
> > >>> -
> > >>> -static inline void numa_store_cpu_info(unsigned int cpu) { }
> > >>> -static inline void numa_add_cpu(unsigned int cpu) { }
> > >>> -static inline void numa_remove_cpu(unsigned int cpu) { }
> > >>> -static inline void arm64_numa_init(void) { }
> > >>> -static inline void early_map_cpu_to_node(unsigned int cpu, int nid) { }
> > >>> -
> > >>> -#endif       /* CONFIG_NUMA */
> > >>> +#include <asm-generic/numa.h>
> > >>>
> > >>>  #endif       /* __ASM_NUMA_H */
> > >>> diff --git a/arch/arm64/mm/Makefile b/arch/arm64/mm/Makefile
> > >>> index d91030f0ffee..928c308b044b 100644
> > >>> --- a/arch/arm64/mm/Makefile
> > >>> +++ b/arch/arm64/mm/Makefile
> > >>> @@ -6,7 +6,6 @@ obj-y                         := dma-mapping.o extable.o fault.o init.o \
> > >>>  obj-$(CONFIG_HUGETLB_PAGE)   += hugetlbpage.o
> > >>>  obj-$(CONFIG_PTDUMP_CORE)    += dump.o
> > >>>  obj-$(CONFIG_PTDUMP_DEBUGFS) += ptdump_debugfs.o
> > >>> -obj-$(CONFIG_NUMA)           += numa.o
> > >>>  obj-$(CONFIG_DEBUG_VIRTUAL)  += physaddr.o
> > >>>  KASAN_SANITIZE_physaddr.o    += n
> > >>>
> > >>> diff --git a/drivers/base/Kconfig b/drivers/base/Kconfig
> > >>> index 8d7001712062..73c2151de194 100644
> > >>> --- a/drivers/base/Kconfig
> > >>> +++ b/drivers/base/Kconfig
> > >>> @@ -210,4 +210,10 @@ config GENERIC_ARCH_TOPOLOGY
> > >>>         appropriate scaling, sysfs interface for reading capacity values at
> > >>>         runtime.
> > >>>
> > >>> +config GENERIC_ARCH_NUMA
> > >>> +     bool
> > >>> +     help
> > >>> +       Enable support for generic numa implementation. Currently, RISC-V
> > >>> +       and ARM64 uses it.
> > >>> +
> > >>>  endmenu
> > >>> diff --git a/drivers/base/Makefile b/drivers/base/Makefile
> > >>> index 157452080f3d..c3d02c644222 100644
> > >>> --- a/drivers/base/Makefile
> > >>> +++ b/drivers/base/Makefile
> > >>> @@ -23,6 +23,7 @@ obj-$(CONFIG_PINCTRL) += pinctrl.o
> > >>>  obj-$(CONFIG_DEV_COREDUMP) += devcoredump.o
> > >>>  obj-$(CONFIG_GENERIC_MSI_IRQ_DOMAIN) += platform-msi.o
> > >>>  obj-$(CONFIG_GENERIC_ARCH_TOPOLOGY) += arch_topology.o
> > >>> +obj-$(CONFIG_GENERIC_ARCH_NUMA) += arch_numa.o
> > >>>
> > >>>  obj-y                        += test/
> > >>>
> > >>> diff --git a/arch/arm64/mm/numa.c b/drivers/base/arch_numa.c
> > >>> similarity index 100%
> > >>> rename from arch/arm64/mm/numa.c
> > >>> rename to drivers/base/arch_numa.c  
> > >>
> > >> drivers/base/ does not seem right place to host generic NUMA code.  
> > >
> > > I chose drivers/base because the common topology code is also present there.
> > > drivers/base/arch_topology.c under GENERIC_ARCH_TOPOLOGY
> > > The idea is to keep all common arch(at least between RISC-V & ARM64)
> > > related code at one place.
> > >  
> > >> Probably it should be either mm/ or kernel/. The other question here  
> > >
> > > I am fine with mm/arch_numa.c as well if that is preferred over driver/base.  
> >
> > GENERIC_ARCH_NUMA being near other shared code such as GENERIC_ARCH_TOPOLOGY
> > do make sense. That being said, its a small nit and can be figured out later.
> >  
> > >  
> > >> would be if existing arm64 NUMA implementation is sufficient enough
> > >> for generic NUMA. I would expect any platform selecting this config
> > >> should get some NUMA enabled, will be that be true with present code ?  
> > >
> > > It is for RISC-V. Here is the RISC-V support patch (last patch in the series)
> > >
> > > http://lists.infradead.org/pipermail/linux-riscv/2020-August/001659.html
> > >  
> >
> > + Jonathan Cameron <Jonathan.Cameron@huawei.com>
> >
> > There is another patch/discussion which is trying to unify ARM64 NUMA init
> > code with X86 (https://patchwork.kernel.org/patch/11651437/). I am wondering
> > if all three platforms could use GENERIC_ARCH_NUMA.
> >  
> 
> Gmail decided to convert my previous reply to HTML for some reason and
> was blocked by the mailing lists.
> Here was my earlier response and apologies for the noise if you
> received it twice.
> 
> That is certainly an awesome goal to achieve. I agree that there are a
> lot of similarities between two implementations
> that can be leveraged under common code. But the current arm64 or x86
> numa implementation
> have also enough differences that  can't just be reused for either.
> This series did not introduce any functional
> difference to arm64 numa code and just moved the existing code between
> files. I don't think that's possible
> for x86 under GENERIC_ARCH_NUMA. It requires a bit more effort to do
> that and I am interested to explore that.
> 
> How about merging this series first and slowly moving pieces of x86
> NUMA code to generic numa code as a separate series ?

I'm in favour of this step wise approach.  We aren't making things worse
by sharing this code between arm64 and riscv other than perhaps needing
to sanity check a few more platforms.

As discussed at Plumbers it might be a tall order to successfully share
all this code with x86 but perhaps there are some parts we can.

Jonathan

> 
> > >> Otherwise it will be difficult to name it as GENERIC_ARCH_NUMA.
> > >>
> > >> _______________________________________________
> > >> linux-riscv mailing list
> > >> linux-riscv@lists.infradead.org
> > >> http://lists.infradead.org/mailman/listinfo/linux-riscv  
> > >
> > >
> > >  
> 
> 
> 

Re: [RFC/RFT PATCH 1/6] numa: Move numa implementation to common code

[Jonathan Cameron]

On Fri, 14 Aug 2020 14:47:20 -0700
Atish Patra <atish.patra@wdc.com> wrote:

> ARM64 numa implementation is generic enough that RISC-V can reuse that
> implementation with very minor cosmetic changes. This will help both
> ARM64 and RISC-V in terms of maintanace and feature improvement
> 
> Move the numa implementation code to common directory so that both ISAs
> can reuse this. This doesn't introduce any function changes for ARM64.
> 
> Signed-off-by: Atish Patra <atish.patra@wdc.com>
Hi Atish,

One trivial question inline.  Otherwise subject to Anshuman's point about
location, this looks fine to me.

I'll run some sanity checks later.

Jonathan
> ---
>  arch/arm64/Kconfig                            |  1 +
>  arch/arm64/include/asm/numa.h                 | 45 +---------------
>  arch/arm64/mm/Makefile                        |  1 -
>  drivers/base/Kconfig                          |  6 +++
>  drivers/base/Makefile                         |  1 +
>  .../mm/numa.c => drivers/base/arch_numa.c     |  0
>  include/asm-generic/numa.h                    | 51 +++++++++++++++++++
>  7 files changed, 60 insertions(+), 45 deletions(-)
>  rename arch/arm64/mm/numa.c => drivers/base/arch_numa.c (100%)
>  create mode 100644 include/asm-generic/numa.h
> 
> diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig
> index 6d232837cbee..955a0cf75b16 100644
> --- a/arch/arm64/Kconfig
> +++ b/arch/arm64/Kconfig
> @@ -960,6 +960,7 @@ config HOTPLUG_CPU
>  # Common NUMA Features
>  config NUMA
>  	bool "NUMA Memory Allocation and Scheduler Support"
> +	select GENERIC_ARCH_NUMA
>  	select ACPI_NUMA if ACPI
>  	select OF_NUMA
>  	help
> diff --git a/arch/arm64/include/asm/numa.h b/arch/arm64/include/asm/numa.h
> index 626ad01e83bf..8c8cf4297cc3 100644
> --- a/arch/arm64/include/asm/numa.h
> +++ b/arch/arm64/include/asm/numa.h
> @@ -3,49 +3,6 @@
>  #define __ASM_NUMA_H
>  
>  #include <asm/topology.h>
> -
> -#ifdef CONFIG_NUMA
> -
> -#define NR_NODE_MEMBLKS		(MAX_NUMNODES * 2)
> -
> -int __node_distance(int from, int to);
> -#define node_distance(a, b) __node_distance(a, b)
> -
> -extern nodemask_t numa_nodes_parsed __initdata;
> -
> -extern bool numa_off;
> -
> -/* Mappings between node number and cpus on that node. */
> -extern cpumask_var_t node_to_cpumask_map[MAX_NUMNODES];
> -void numa_clear_node(unsigned int cpu);
> -
> -#ifdef CONFIG_DEBUG_PER_CPU_MAPS
> -const struct cpumask *cpumask_of_node(int node);
> -#else
> -/* Returns a pointer to the cpumask of CPUs on Node 'node'. */
> -static inline const struct cpumask *cpumask_of_node(int node)
> -{
> -	return node_to_cpumask_map[node];
> -}
> -#endif
> -
> -void __init arm64_numa_init(void);
> -int __init numa_add_memblk(int nodeid, u64 start, u64 end);
> -void __init numa_set_distance(int from, int to, int distance);
> -void __init numa_free_distance(void);
> -void __init early_map_cpu_to_node(unsigned int cpu, int nid);
> -void numa_store_cpu_info(unsigned int cpu);
> -void numa_add_cpu(unsigned int cpu);
> -void numa_remove_cpu(unsigned int cpu);
> -
> -#else	/* CONFIG_NUMA */
> -
> -static inline void numa_store_cpu_info(unsigned int cpu) { }
> -static inline void numa_add_cpu(unsigned int cpu) { }
> -static inline void numa_remove_cpu(unsigned int cpu) { }
> -static inline void arm64_numa_init(void) { }
> -static inline void early_map_cpu_to_node(unsigned int cpu, int nid) { }
> -
> -#endif	/* CONFIG_NUMA */
> +#include <asm-generic/numa.h>
>  
>  #endif	/* __ASM_NUMA_H */
> diff --git a/arch/arm64/mm/Makefile b/arch/arm64/mm/Makefile
> index d91030f0ffee..928c308b044b 100644
> --- a/arch/arm64/mm/Makefile
> +++ b/arch/arm64/mm/Makefile
> @@ -6,7 +6,6 @@ obj-y				:= dma-mapping.o extable.o fault.o init.o \
>  obj-$(CONFIG_HUGETLB_PAGE)	+= hugetlbpage.o
>  obj-$(CONFIG_PTDUMP_CORE)	+= dump.o
>  obj-$(CONFIG_PTDUMP_DEBUGFS)	+= ptdump_debugfs.o
> -obj-$(CONFIG_NUMA)		+= numa.o
>  obj-$(CONFIG_DEBUG_VIRTUAL)	+= physaddr.o
>  KASAN_SANITIZE_physaddr.o	+= n
>  
> diff --git a/drivers/base/Kconfig b/drivers/base/Kconfig
> index 8d7001712062..73c2151de194 100644
> --- a/drivers/base/Kconfig
> +++ b/drivers/base/Kconfig
> @@ -210,4 +210,10 @@ config GENERIC_ARCH_TOPOLOGY
>  	  appropriate scaling, sysfs interface for reading capacity values at
>  	  runtime.
>  
> +config GENERIC_ARCH_NUMA
> +	bool
> +	help
> +	  Enable support for generic numa implementation. Currently, RISC-V
> +	  and ARM64 uses it.
> +
>  endmenu
> diff --git a/drivers/base/Makefile b/drivers/base/Makefile
> index 157452080f3d..c3d02c644222 100644
> --- a/drivers/base/Makefile
> +++ b/drivers/base/Makefile
> @@ -23,6 +23,7 @@ obj-$(CONFIG_PINCTRL) += pinctrl.o
>  obj-$(CONFIG_DEV_COREDUMP) += devcoredump.o
>  obj-$(CONFIG_GENERIC_MSI_IRQ_DOMAIN) += platform-msi.o
>  obj-$(CONFIG_GENERIC_ARCH_TOPOLOGY) += arch_topology.o
> +obj-$(CONFIG_GENERIC_ARCH_NUMA) += arch_numa.o
>  
>  obj-y			+= test/
>  
> diff --git a/arch/arm64/mm/numa.c b/drivers/base/arch_numa.c
> similarity index 100%
> rename from arch/arm64/mm/numa.c
> rename to drivers/base/arch_numa.c
> diff --git a/include/asm-generic/numa.h b/include/asm-generic/numa.h
> new file mode 100644
> index 000000000000..0635c0724b7c
> --- /dev/null
> +++ b/include/asm-generic/numa.h
> @@ -0,0 +1,51 @@
> +/* SPDX-License-Identifier: GPL-2.0 */
> +#ifndef __ASM_GENERIC_NUMA_H
> +#define __ASM_GENERIC_NUMA_H
> +
> +#ifdef CONFIG_NUMA
> +
> +#define NR_NODE_MEMBLKS		(MAX_NUMNODES * 2)
> +
> +int __node_distance(int from, int to);
> +#define node_distance(a, b) __node_distance(a, b)
> +
> +extern nodemask_t numa_nodes_parsed __initdata;
> +
> +extern bool numa_off;
> +
> +/* Mappings between node number and cpus on that node. */
> +extern cpumask_var_t node_to_cpumask_map[MAX_NUMNODES];
> +void numa_clear_node(unsigned int cpu);
> +
> +#ifdef CONFIG_DEBUG_PER_CPU_MAPS
> +const struct cpumask *cpumask_of_node(int node);
> +#else
> +/* Returns a pointer to the cpumask of CPUs on Node 'node'. */
> +static inline const struct cpumask *cpumask_of_node(int node)
> +{
> +	return node_to_cpumask_map[node];
> +}
> +#endif
> +
> +void __init arm64_numa_init(void);
> +int __init numa_add_memblk(int nodeid, u64 start, u64 end);
> +void __init numa_set_distance(int from, int to, int distance);
> +void __init numa_free_distance(void);
> +void __init early_map_cpu_to_node(unsigned int cpu, int nid);
> +void numa_store_cpu_info(unsigned int cpu);
> +void numa_add_cpu(unsigned int cpu);
> +void numa_remove_cpu(unsigned int cpu);
> +
> +#else	/* CONFIG_NUMA */
> +
> +static inline void numa_store_cpu_info(unsigned int cpu) { }
> +static inline void numa_add_cpu(unsigned int cpu) { }
> +static inline void numa_remove_cpu(unsigned int cpu) { }
> +static inline void arm64_numa_init(void) { }
> +static inline void early_map_cpu_to_node(unsigned int cpu, int nid) { }
> +
> +#endif	/* CONFIG_NUMA */
> +
> +#include <asm/topology.h>

Why the include here?

> +
> +#endif	/* __ASM_GENERIC_NUMA_H */

Re: [RFC/RFT PATCH 1/6] numa: Move numa implementation to common code

[Atish Patra]

On Fri, Aug 28, 2020 at 2:24 AM Jonathan Cameron
<Jonathan.Cameron@huawei.com> wrote:
>
> On Fri, 14 Aug 2020 14:47:20 -0700
> Atish Patra <atish.patra@wdc.com> wrote:
>
> > ARM64 numa implementation is generic enough that RISC-V can reuse that
> > implementation with very minor cosmetic changes. This will help both
> > ARM64 and RISC-V in terms of maintanace and feature improvement
> >
> > Move the numa implementation code to common directory so that both ISAs
> > can reuse this. This doesn't introduce any function changes for ARM64.
> >
> > Signed-off-by: Atish Patra <atish.patra@wdc.com>
> Hi Atish,
>
> One trivial question inline.  Otherwise subject to Anshuman's point about
> location, this looks fine to me.
>
> I'll run some sanity checks later.
>
> Jonathan
> > ---
> >  arch/arm64/Kconfig                            |  1 +
> >  arch/arm64/include/asm/numa.h                 | 45 +---------------
> >  arch/arm64/mm/Makefile                        |  1 -
> >  drivers/base/Kconfig                          |  6 +++
> >  drivers/base/Makefile                         |  1 +
> >  .../mm/numa.c => drivers/base/arch_numa.c     |  0
> >  include/asm-generic/numa.h                    | 51 +++++++++++++++++++
> >  7 files changed, 60 insertions(+), 45 deletions(-)
> >  rename arch/arm64/mm/numa.c => drivers/base/arch_numa.c (100%)
> >  create mode 100644 include/asm-generic/numa.h
> >
> > diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig
> > index 6d232837cbee..955a0cf75b16 100644
> > --- a/arch/arm64/Kconfig
> > +++ b/arch/arm64/Kconfig
> > @@ -960,6 +960,7 @@ config HOTPLUG_CPU
> >  # Common NUMA Features
> >  config NUMA
> >       bool "NUMA Memory Allocation and Scheduler Support"
> > +     select GENERIC_ARCH_NUMA
> >       select ACPI_NUMA if ACPI
> >       select OF_NUMA
> >       help
> > diff --git a/arch/arm64/include/asm/numa.h b/arch/arm64/include/asm/numa.h
> > index 626ad01e83bf..8c8cf4297cc3 100644
> > --- a/arch/arm64/include/asm/numa.h
> > +++ b/arch/arm64/include/asm/numa.h
> > @@ -3,49 +3,6 @@
> >  #define __ASM_NUMA_H
> >
> >  #include <asm/topology.h>
> > -
> > -#ifdef CONFIG_NUMA
> > -
> > -#define NR_NODE_MEMBLKS              (MAX_NUMNODES * 2)
> > -
> > -int __node_distance(int from, int to);
> > -#define node_distance(a, b) __node_distance(a, b)
> > -
> > -extern nodemask_t numa_nodes_parsed __initdata;
> > -
> > -extern bool numa_off;
> > -
> > -/* Mappings between node number and cpus on that node. */
> > -extern cpumask_var_t node_to_cpumask_map[MAX_NUMNODES];
> > -void numa_clear_node(unsigned int cpu);
> > -
> > -#ifdef CONFIG_DEBUG_PER_CPU_MAPS
> > -const struct cpumask *cpumask_of_node(int node);
> > -#else
> > -/* Returns a pointer to the cpumask of CPUs on Node 'node'. */
> > -static inline const struct cpumask *cpumask_of_node(int node)
> > -{
> > -     return node_to_cpumask_map[node];
> > -}
> > -#endif
> > -
> > -void __init arm64_numa_init(void);
> > -int __init numa_add_memblk(int nodeid, u64 start, u64 end);
> > -void __init numa_set_distance(int from, int to, int distance);
> > -void __init numa_free_distance(void);
> > -void __init early_map_cpu_to_node(unsigned int cpu, int nid);
> > -void numa_store_cpu_info(unsigned int cpu);
> > -void numa_add_cpu(unsigned int cpu);
> > -void numa_remove_cpu(unsigned int cpu);
> > -
> > -#else        /* CONFIG_NUMA */
> > -
> > -static inline void numa_store_cpu_info(unsigned int cpu) { }
> > -static inline void numa_add_cpu(unsigned int cpu) { }
> > -static inline void numa_remove_cpu(unsigned int cpu) { }
> > -static inline void arm64_numa_init(void) { }
> > -static inline void early_map_cpu_to_node(unsigned int cpu, int nid) { }
> > -
> > -#endif       /* CONFIG_NUMA */
> > +#include <asm-generic/numa.h>
> >
> >  #endif       /* __ASM_NUMA_H */
> > diff --git a/arch/arm64/mm/Makefile b/arch/arm64/mm/Makefile
> > index d91030f0ffee..928c308b044b 100644
> > --- a/arch/arm64/mm/Makefile
> > +++ b/arch/arm64/mm/Makefile
> > @@ -6,7 +6,6 @@ obj-y                         := dma-mapping.o extable.o fault.o init.o \
> >  obj-$(CONFIG_HUGETLB_PAGE)   += hugetlbpage.o
> >  obj-$(CONFIG_PTDUMP_CORE)    += dump.o
> >  obj-$(CONFIG_PTDUMP_DEBUGFS) += ptdump_debugfs.o
> > -obj-$(CONFIG_NUMA)           += numa.o
> >  obj-$(CONFIG_DEBUG_VIRTUAL)  += physaddr.o
> >  KASAN_SANITIZE_physaddr.o    += n
> >
> > diff --git a/drivers/base/Kconfig b/drivers/base/Kconfig
> > index 8d7001712062..73c2151de194 100644
> > --- a/drivers/base/Kconfig
> > +++ b/drivers/base/Kconfig
> > @@ -210,4 +210,10 @@ config GENERIC_ARCH_TOPOLOGY
> >         appropriate scaling, sysfs interface for reading capacity values at
> >         runtime.
> >
> > +config GENERIC_ARCH_NUMA
> > +     bool
> > +     help
> > +       Enable support for generic numa implementation. Currently, RISC-V
> > +       and ARM64 uses it.
> > +
> >  endmenu
> > diff --git a/drivers/base/Makefile b/drivers/base/Makefile
> > index 157452080f3d..c3d02c644222 100644
> > --- a/drivers/base/Makefile
> > +++ b/drivers/base/Makefile
> > @@ -23,6 +23,7 @@ obj-$(CONFIG_PINCTRL) += pinctrl.o
> >  obj-$(CONFIG_DEV_COREDUMP) += devcoredump.o
> >  obj-$(CONFIG_GENERIC_MSI_IRQ_DOMAIN) += platform-msi.o
> >  obj-$(CONFIG_GENERIC_ARCH_TOPOLOGY) += arch_topology.o
> > +obj-$(CONFIG_GENERIC_ARCH_NUMA) += arch_numa.o
> >
> >  obj-y                        += test/
> >
> > diff --git a/arch/arm64/mm/numa.c b/drivers/base/arch_numa.c
> > similarity index 100%
> > rename from arch/arm64/mm/numa.c
> > rename to drivers/base/arch_numa.c
> > diff --git a/include/asm-generic/numa.h b/include/asm-generic/numa.h
> > new file mode 100644
> > index 000000000000..0635c0724b7c
> > --- /dev/null
> > +++ b/include/asm-generic/numa.h
> > @@ -0,0 +1,51 @@
> > +/* SPDX-License-Identifier: GPL-2.0 */
> > +#ifndef __ASM_GENERIC_NUMA_H
> > +#define __ASM_GENERIC_NUMA_H
> > +
> > +#ifdef CONFIG_NUMA
> > +
> > +#define NR_NODE_MEMBLKS              (MAX_NUMNODES * 2)
> > +
> > +int __node_distance(int from, int to);
> > +#define node_distance(a, b) __node_distance(a, b)
> > +
> > +extern nodemask_t numa_nodes_parsed __initdata;
> > +
> > +extern bool numa_off;
> > +
> > +/* Mappings between node number and cpus on that node. */
> > +extern cpumask_var_t node_to_cpumask_map[MAX_NUMNODES];
> > +void numa_clear_node(unsigned int cpu);
> > +
> > +#ifdef CONFIG_DEBUG_PER_CPU_MAPS
> > +const struct cpumask *cpumask_of_node(int node);
> > +#else
> > +/* Returns a pointer to the cpumask of CPUs on Node 'node'. */
> > +static inline const struct cpumask *cpumask_of_node(int node)
> > +{
> > +     return node_to_cpumask_map[node];
> > +}
> > +#endif
> > +
> > +void __init arm64_numa_init(void);
> > +int __init numa_add_memblk(int nodeid, u64 start, u64 end);
> > +void __init numa_set_distance(int from, int to, int distance);
> > +void __init numa_free_distance(void);
> > +void __init early_map_cpu_to_node(unsigned int cpu, int nid);
> > +void numa_store_cpu_info(unsigned int cpu);
> > +void numa_add_cpu(unsigned int cpu);
> > +void numa_remove_cpu(unsigned int cpu);
> > +
> > +#else        /* CONFIG_NUMA */
> > +
> > +static inline void numa_store_cpu_info(unsigned int cpu) { }
> > +static inline void numa_add_cpu(unsigned int cpu) { }
> > +static inline void numa_remove_cpu(unsigned int cpu) { }
> > +static inline void arm64_numa_init(void) { }
> > +static inline void early_map_cpu_to_node(unsigned int cpu, int nid) { }
> > +
> > +#endif       /* CONFIG_NUMA */
> > +
> > +#include <asm/topology.h>
>
> Why the include here?
>

Sorry. This was a spillover from the previous patch. Thanks for catching it.
I will fix it in v2.
> > +
> > +#endif       /* __ASM_GENERIC_NUMA_H */
>
>
>
> _______________________________________________
> linux-riscv mailing list
> linux-riscv@lists.infradead.org
> http://lists.infradead.org/mailman/listinfo/linux-riscv



-- 
Regards,
Atish

[RFC/RFT PATCH 2/6] arm64, numa: Change the numa init function name to be generic

[Atish Patra]

As we are using generic numa implementation code, modify the init function
name to indicate that generic implementation.

Signed-off-by: Atish Patra <atish.patra@wdc.com>
---
 arch/arm64/mm/init.c       | 4 ++--
 drivers/base/arch_numa.c   | 8 ++++++--
 include/asm-generic/numa.h | 4 ++--
 3 files changed, 10 insertions(+), 6 deletions(-)

diff --git a/arch/arm64/mm/init.c b/arch/arm64/mm/init.c
index 481d22c32a2e..93b660229e1d 100644
--- a/arch/arm64/mm/init.c
+++ b/arch/arm64/mm/init.c
@@ -418,10 +418,10 @@ void __init bootmem_init(void)
 	max_pfn = max_low_pfn = max;
 	min_low_pfn = min;
 
-	arm64_numa_init();
+	arch_numa_init();
 
 	/*
-	 * must be done after arm64_numa_init() which calls numa_init() to
+	 * must be done after arch_numa_init() which calls numa_init() to
 	 * initialize node_online_map that gets used in hugetlb_cma_reserve()
 	 * while allocating required CMA size across online nodes.
 	 */
diff --git a/drivers/base/arch_numa.c b/drivers/base/arch_numa.c
index 73f8b49d485c..83341c807240 100644
--- a/drivers/base/arch_numa.c
+++ b/drivers/base/arch_numa.c
@@ -13,7 +13,9 @@
 #include <linux/module.h>
 #include <linux/of.h>
 
+#ifdef CONFIG_ARM64
 #include <asm/acpi.h>
+#endif
 #include <asm/sections.h>
 
 struct pglist_data *node_data[MAX_NUMNODES] __read_mostly;
@@ -445,16 +447,18 @@ static int __init dummy_numa_init(void)
 }
 
 /**
- * arm64_numa_init() - Initialize NUMA
+ * arch_numa_init() - Initialize NUMA
  *
  * Try each configured NUMA initialization method until one succeeds. The
  * last fallback is dummy single node config encomapssing whole memory.
  */
-void __init arm64_numa_init(void)
+void __init arch_numa_init(void)
 {
 	if (!numa_off) {
+#ifdef CONFIG_ARM64
 		if (!acpi_disabled && !numa_init(arm64_acpi_numa_init))
 			return;
+#endif
 		if (acpi_disabled && !numa_init(of_numa_init))
 			return;
 	}
diff --git a/include/asm-generic/numa.h b/include/asm-generic/numa.h
index 0635c0724b7c..309eca8c0b5d 100644
--- a/include/asm-generic/numa.h
+++ b/include/asm-generic/numa.h
@@ -27,7 +27,7 @@ static inline const struct cpumask *cpumask_of_node(int node)
 }
 #endif
 
-void __init arm64_numa_init(void);
+void __init arch_numa_init(void);
 int __init numa_add_memblk(int nodeid, u64 start, u64 end);
 void __init numa_set_distance(int from, int to, int distance);
 void __init numa_free_distance(void);
@@ -41,7 +41,7 @@ void numa_remove_cpu(unsigned int cpu);
 static inline void numa_store_cpu_info(unsigned int cpu) { }
 static inline void numa_add_cpu(unsigned int cpu) { }
 static inline void numa_remove_cpu(unsigned int cpu) { }
-static inline void arm64_numa_init(void) { }
+static inline void arch_numa_init(void) { }
 static inline void early_map_cpu_to_node(unsigned int cpu, int nid) { }
 
 #endif	/* CONFIG_NUMA */
-- 
2.24.0

Re: [RFC/RFT PATCH 2/6] arm64, numa: Change the numa init function name to be generic

[Jonathan Cameron]

On Fri, 14 Aug 2020 14:47:21 -0700
Atish Patra <atish.patra@wdc.com> wrote:

> As we are using generic numa implementation code, modify the init function
> name to indicate that generic implementation.
> 
> Signed-off-by: Atish Patra <atish.patra@wdc.com>
> ---
>  arch/arm64/mm/init.c       | 4 ++--
>  drivers/base/arch_numa.c   | 8 ++++++--
>  include/asm-generic/numa.h | 4 ++--
>  3 files changed, 10 insertions(+), 6 deletions(-)
> 
> diff --git a/arch/arm64/mm/init.c b/arch/arm64/mm/init.c
> index 481d22c32a2e..93b660229e1d 100644
> --- a/arch/arm64/mm/init.c
> +++ b/arch/arm64/mm/init.c
> @@ -418,10 +418,10 @@ void __init bootmem_init(void)
>  	max_pfn = max_low_pfn = max;
>  	min_low_pfn = min;
>  
> -	arm64_numa_init();
> +	arch_numa_init();
>  
>  	/*
> -	 * must be done after arm64_numa_init() which calls numa_init() to
> +	 * must be done after arch_numa_init() which calls numa_init() to
>  	 * initialize node_online_map that gets used in hugetlb_cma_reserve()
>  	 * while allocating required CMA size across online nodes.
>  	 */
> diff --git a/drivers/base/arch_numa.c b/drivers/base/arch_numa.c
> index 73f8b49d485c..83341c807240 100644
> --- a/drivers/base/arch_numa.c
> +++ b/drivers/base/arch_numa.c
> @@ -13,7 +13,9 @@
>  #include <linux/module.h>
>  #include <linux/of.h>
>  
> +#ifdef CONFIG_ARM64
>  #include <asm/acpi.h>
> +#endif

This highlights an issue.  We really don't want to define 'generic' arch
code then match on individual architectures if at all possible.

I'm also not sure we need to. 

The arm64_acpi_numa_init() code is just a light wrapper around the
standard acpi_init() call so should work fine on riscv (once ACPI
support is ready).

Can we pull that function into here or perhaps a generic
arch_numa_acpi.c?

There is probably a bit of a dance needed around acpi_disabled
though as that can be defined in entirely different places
depending on whether acpi is enabled or not.
Possibly just adding an

extern int acpi_disabled to include/linux/acpi.h when acpi is enabled
will be sufficient (if ugly)?


>  #include <asm/sections.h>
>  
>  struct pglist_data *node_data[MAX_NUMNODES] __read_mostly;
> @@ -445,16 +447,18 @@ static int __init dummy_numa_init(void)
>  }
>  
>  /**
> - * arm64_numa_init() - Initialize NUMA
> + * arch_numa_init() - Initialize NUMA
>   *
>   * Try each configured NUMA initialization method until one succeeds. The
>   * last fallback is dummy single node config encomapssing whole memory.
>   */
> -void __init arm64_numa_init(void)
> +void __init arch_numa_init(void)
>  {
>  	if (!numa_off) {
> +#ifdef CONFIG_ARM64
>  		if (!acpi_disabled && !numa_init(arm64_acpi_numa_init))
>  			return;
> +#endif
>  		if (acpi_disabled && !numa_init(of_numa_init))
>  			return;
>  	}
> diff --git a/include/asm-generic/numa.h b/include/asm-generic/numa.h
> index 0635c0724b7c..309eca8c0b5d 100644
> --- a/include/asm-generic/numa.h
> +++ b/include/asm-generic/numa.h
> @@ -27,7 +27,7 @@ static inline const struct cpumask *cpumask_of_node(int node)
>  }
>  #endif
>  
> -void __init arm64_numa_init(void);
> +void __init arch_numa_init(void);
>  int __init numa_add_memblk(int nodeid, u64 start, u64 end);
>  void __init numa_set_distance(int from, int to, int distance);
>  void __init numa_free_distance(void);
> @@ -41,7 +41,7 @@ void numa_remove_cpu(unsigned int cpu);
>  static inline void numa_store_cpu_info(unsigned int cpu) { }
>  static inline void numa_add_cpu(unsigned int cpu) { }
>  static inline void numa_remove_cpu(unsigned int cpu) { }
> -static inline void arm64_numa_init(void) { }
> +static inline void arch_numa_init(void) { }
>  static inline void early_map_cpu_to_node(unsigned int cpu, int nid) { }
>  
>  #endif	/* CONFIG_NUMA */

Re: [RFC/RFT PATCH 2/6] arm64, numa: Change the numa init function name to be generic

[Atish Patra]

On Fri, Aug 28, 2020 at 2:37 AM Jonathan Cameron
<Jonathan.Cameron@huawei.com> wrote:
>
> On Fri, 14 Aug 2020 14:47:21 -0700
> Atish Patra <atish.patra@wdc.com> wrote:
>
> > As we are using generic numa implementation code, modify the init function
> > name to indicate that generic implementation.
> >
> > Signed-off-by: Atish Patra <atish.patra@wdc.com>
> > ---
> >  arch/arm64/mm/init.c       | 4 ++--
> >  drivers/base/arch_numa.c   | 8 ++++++--
> >  include/asm-generic/numa.h | 4 ++--
> >  3 files changed, 10 insertions(+), 6 deletions(-)
> >
> > diff --git a/arch/arm64/mm/init.c b/arch/arm64/mm/init.c
> > index 481d22c32a2e..93b660229e1d 100644
> > --- a/arch/arm64/mm/init.c
> > +++ b/arch/arm64/mm/init.c
> > @@ -418,10 +418,10 @@ void __init bootmem_init(void)
> >       max_pfn = max_low_pfn = max;
> >       min_low_pfn = min;
> >
> > -     arm64_numa_init();
> > +     arch_numa_init();
> >
> >       /*
> > -      * must be done after arm64_numa_init() which calls numa_init() to
> > +      * must be done after arch_numa_init() which calls numa_init() to
> >        * initialize node_online_map that gets used in hugetlb_cma_reserve()
> >        * while allocating required CMA size across online nodes.
> >        */
> > diff --git a/drivers/base/arch_numa.c b/drivers/base/arch_numa.c
> > index 73f8b49d485c..83341c807240 100644
> > --- a/drivers/base/arch_numa.c
> > +++ b/drivers/base/arch_numa.c
> > @@ -13,7 +13,9 @@
> >  #include <linux/module.h>
> >  #include <linux/of.h>
> >
> > +#ifdef CONFIG_ARM64
> >  #include <asm/acpi.h>
> > +#endif
>
> This highlights an issue.  We really don't want to define 'generic' arch
> code then match on individual architectures if at all possible.
>

I agree.

> I'm also not sure we need to.
>
> The arm64_acpi_numa_init() code is just a light wrapper around the
> standard acpi_init() call so should work fine on riscv (once ACPI
> support is ready).
>
> Can we pull that function into here

Sure. We can move the arm64_acpi_numa_init to here and rename it to
arch_acpi_numa_init.
We can keep arch_acpi_numa_init and the acpi.h included under CONFIG_ACPI_NUMA.
If RISC-V becomes ACPI ready one day, they always need to enable
CONFIG_ACPI_NUMA and reuse the generic functions.

> or perhaps a generic arch_numa_acpi.c?
>
There has not been much discussion about ACPI for RISC-V. So moving
the arm64 acpi code now to generic code would be premature
in my opinion. Currently, we don't even know how ACPI will look like
for RISC-V.

> There is probably a bit of a dance needed around acpi_disabled
> though as that can be defined in entirely different places
> depending on whether acpi is enabled or not.
> Possibly just adding an
>
> extern int acpi_disabled to include/linux/acpi.h when acpi is enabled
> will be sufficient (if ugly)?
>

We don't need to do that now unless we are moving arm64 ACPI code
implementation to generic code.
If ACPI is not enabled, it is already defined as a macro in
include/linux/acpi.h.

>
> >  #include <asm/sections.h>
> >
> >  struct pglist_data *node_data[MAX_NUMNODES] __read_mostly;
> > @@ -445,16 +447,18 @@ static int __init dummy_numa_init(void)
> >  }
> >
> >  /**
> > - * arm64_numa_init() - Initialize NUMA
> > + * arch_numa_init() - Initialize NUMA
> >   *
> >   * Try each configured NUMA initialization method until one succeeds. The
> >   * last fallback is dummy single node config encomapssing whole memory.
> >   */
> > -void __init arm64_numa_init(void)
> > +void __init arch_numa_init(void)
> >  {
> >       if (!numa_off) {
> > +#ifdef CONFIG_ARM64
> >               if (!acpi_disabled && !numa_init(arm64_acpi_numa_init))
> >                       return;
> > +#endif
> >               if (acpi_disabled && !numa_init(of_numa_init))
> >                       return;
> >       }
> > diff --git a/include/asm-generic/numa.h b/include/asm-generic/numa.h
> > index 0635c0724b7c..309eca8c0b5d 100644
> > --- a/include/asm-generic/numa.h
> > +++ b/include/asm-generic/numa.h
> > @@ -27,7 +27,7 @@ static inline const struct cpumask *cpumask_of_node(int node)
> >  }
> >  #endif
> >
> > -void __init arm64_numa_init(void);
> > +void __init arch_numa_init(void);
> >  int __init numa_add_memblk(int nodeid, u64 start, u64 end);
> >  void __init numa_set_distance(int from, int to, int distance);
> >  void __init numa_free_distance(void);
> > @@ -41,7 +41,7 @@ void numa_remove_cpu(unsigned int cpu);
> >  static inline void numa_store_cpu_info(unsigned int cpu) { }
> >  static inline void numa_add_cpu(unsigned int cpu) { }
> >  static inline void numa_remove_cpu(unsigned int cpu) { }
> > -static inline void arm64_numa_init(void) { }
> > +static inline void arch_numa_init(void) { }
> >  static inline void early_map_cpu_to_node(unsigned int cpu, int nid) { }
> >
> >  #endif       /* CONFIG_NUMA */
>
>
>
> _______________________________________________
> linux-riscv mailing list
> linux-riscv@lists.infradead.org
> http://lists.infradead.org/mailman/listinfo/linux-riscv



-- 
Regards,
Atish

[RFC/RFT PATCH 3/6] arm64, numa: Move pcibus_to_node definition to generic numa code

[Atish Patra]

pcibus_to_node is used only when numa is enabled and does not depend
on ISA. Thus, it can be moved the generic numa implementation.

Signed-off-by: Atish Patra <atish.patra@wdc.com>
---
 arch/arm64/kernel/pci.c  | 10 ----------
 drivers/base/arch_numa.c | 11 +++++++++++
 2 files changed, 11 insertions(+), 10 deletions(-)

diff --git a/arch/arm64/kernel/pci.c b/arch/arm64/kernel/pci.c
index 1006ed2d7c60..07c122946c11 100644
--- a/arch/arm64/kernel/pci.c
+++ b/arch/arm64/kernel/pci.c
@@ -54,16 +54,6 @@ int raw_pci_write(unsigned int domain, unsigned int bus,
 	return b->ops->write(b, devfn, reg, len, val);
 }
 
-#ifdef CONFIG_NUMA
-
-int pcibus_to_node(struct pci_bus *bus)
-{
-	return dev_to_node(&bus->dev);
-}
-EXPORT_SYMBOL(pcibus_to_node);
-
-#endif
-
 #ifdef CONFIG_ACPI
 
 struct acpi_pci_generic_root_info {
diff --git a/drivers/base/arch_numa.c b/drivers/base/arch_numa.c
index 83341c807240..4ab1b20a615d 100644
--- a/drivers/base/arch_numa.c
+++ b/drivers/base/arch_numa.c
@@ -11,6 +11,7 @@
 #include <linux/acpi.h>
 #include <linux/memblock.h>
 #include <linux/module.h>
+#include <linux/pci.h>
 #include <linux/of.h>
 
 #ifdef CONFIG_ARM64
@@ -60,6 +61,16 @@ EXPORT_SYMBOL(cpumask_of_node);
 
 #endif
 
+#ifdef CONFIG_PCI
+
+int pcibus_to_node(struct pci_bus *bus)
+{
+	return dev_to_node(&bus->dev);
+}
+EXPORT_SYMBOL(pcibus_to_node);
+
+#endif
+
 static void numa_update_cpu(unsigned int cpu, bool remove)
 {
 	int nid = cpu_to_node(cpu);
-- 
2.24.0

Re: [RFC/RFT PATCH 3/6] arm64, numa: Move pcibus_to_node definition to generic numa code

[Jonathan Cameron]

On Fri, 14 Aug 2020 14:47:22 -0700
Atish Patra <atish.patra@wdc.com> wrote:

> pcibus_to_node is used only when numa is enabled and does not depend
> on ISA. Thus, it can be moved the generic numa implementation.
> 
> Signed-off-by: Atish Patra <atish.patra@wdc.com>

From a more general unification point of view, there seem to
be two ways architectures implement this.
Either

bus->sysdata.node

Or as here.
There are weird other options, but let us ignore those :)

That is going to take a bit of unwinding should we
want to take this unification further and perhaps we want to think
about doing this in pci generic code rather than here?

Perhaps this is one we are better keeping architecture specific for
now?

+CC Bjorn and Linux-pci


> ---
>  arch/arm64/kernel/pci.c  | 10 ----------
>  drivers/base/arch_numa.c | 11 +++++++++++
>  2 files changed, 11 insertions(+), 10 deletions(-)
> 
> diff --git a/arch/arm64/kernel/pci.c b/arch/arm64/kernel/pci.c
> index 1006ed2d7c60..07c122946c11 100644
> --- a/arch/arm64/kernel/pci.c
> +++ b/arch/arm64/kernel/pci.c
> @@ -54,16 +54,6 @@ int raw_pci_write(unsigned int domain, unsigned int bus,
>  	return b->ops->write(b, devfn, reg, len, val);
>  }
>  
> -#ifdef CONFIG_NUMA
> -
> -int pcibus_to_node(struct pci_bus *bus)
> -{
> -	return dev_to_node(&bus->dev);
> -}
> -EXPORT_SYMBOL(pcibus_to_node);
> -
> -#endif
> -
>  #ifdef CONFIG_ACPI
>  
>  struct acpi_pci_generic_root_info {
> diff --git a/drivers/base/arch_numa.c b/drivers/base/arch_numa.c
> index 83341c807240..4ab1b20a615d 100644
> --- a/drivers/base/arch_numa.c
> +++ b/drivers/base/arch_numa.c
> @@ -11,6 +11,7 @@
>  #include <linux/acpi.h>
>  #include <linux/memblock.h>
>  #include <linux/module.h>
> +#include <linux/pci.h>
>  #include <linux/of.h>
>  
>  #ifdef CONFIG_ARM64
> @@ -60,6 +61,16 @@ EXPORT_SYMBOL(cpumask_of_node);
>  
>  #endif
>  
> +#ifdef CONFIG_PCI
> +
> +int pcibus_to_node(struct pci_bus *bus)
> +{
> +	return dev_to_node(&bus->dev);
> +}
> +EXPORT_SYMBOL(pcibus_to_node);
> +
> +#endif
> +
>  static void numa_update_cpu(unsigned int cpu, bool remove)
>  {
>  	int nid = cpu_to_node(cpu);

Re: [RFC/RFT PATCH 3/6] arm64, numa: Move pcibus_to_node definition to generic numa code

[Bjorn Helgaas]

On Fri, Aug 28, 2020 at 10:48:30AM +0100, Jonathan Cameron wrote:
> On Fri, 14 Aug 2020 14:47:22 -0700
> Atish Patra <atish.patra@wdc.com> wrote:
> 
> > pcibus_to_node is used only when numa is enabled and does not depend
> > on ISA. Thus, it can be moved the generic numa implementation.
> > 
> > Signed-off-by: Atish Patra <atish.patra@wdc.com>
> 
> From a more general unification point of view, there seem to
> be two ways architectures implement this.
> Either
> 
> bus->sysdata.node
> 
> Or as here.
> There are weird other options, but let us ignore those :)
> 
> That is going to take a bit of unwinding should we
> want to take this unification further and perhaps we want to think
> about doing this in pci generic code rather than here?
> 
> Perhaps this is one we are better keeping architecture specific for
> now?
> 
> +CC Bjorn and Linux-pci
> 
> 
> > ---
> >  arch/arm64/kernel/pci.c  | 10 ----------
> >  drivers/base/arch_numa.c | 11 +++++++++++
> >  2 files changed, 11 insertions(+), 10 deletions(-)
> > 
> > diff --git a/arch/arm64/kernel/pci.c b/arch/arm64/kernel/pci.c
> > index 1006ed2d7c60..07c122946c11 100644
> > --- a/arch/arm64/kernel/pci.c
> > +++ b/arch/arm64/kernel/pci.c
> > @@ -54,16 +54,6 @@ int raw_pci_write(unsigned int domain, unsigned int bus,
> >  	return b->ops->write(b, devfn, reg, len, val);
> >  }
> >  
> > -#ifdef CONFIG_NUMA
> > -
> > -int pcibus_to_node(struct pci_bus *bus)
> > -{
> > -	return dev_to_node(&bus->dev);
> > -}
> > -EXPORT_SYMBOL(pcibus_to_node);
> > -
> > -#endif
> > -
> >  #ifdef CONFIG_ACPI
> >  
> >  struct acpi_pci_generic_root_info {
> > diff --git a/drivers/base/arch_numa.c b/drivers/base/arch_numa.c
> > index 83341c807240..4ab1b20a615d 100644
> > --- a/drivers/base/arch_numa.c
> > +++ b/drivers/base/arch_numa.c
> > @@ -11,6 +11,7 @@
> >  #include <linux/acpi.h>
> >  #include <linux/memblock.h>
> >  #include <linux/module.h>
> > +#include <linux/pci.h>
> >  #include <linux/of.h>
> >  
> >  #ifdef CONFIG_ARM64
> > @@ -60,6 +61,16 @@ EXPORT_SYMBOL(cpumask_of_node);
> >  
> >  #endif
> >  
> > +#ifdef CONFIG_PCI
> > +
> > +int pcibus_to_node(struct pci_bus *bus)
> > +{
> > +	return dev_to_node(&bus->dev);
> > +}
> > +EXPORT_SYMBOL(pcibus_to_node);
> > +
> > +#endif

I certainly agree that this should not be arch-specific, but I'm not
really in favor of adding this PCI gunk in drivers/base.

I think we can do better (eventually) by getting rid of
pcibus_to_node() completely.  It's not used very much except by
cpumask_of_pcibus(), which itself is hardly used at all.

> >  static void numa_update_cpu(unsigned int cpu, bool remove)
> >  {
> >  	int nid = cpu_to_node(cpu);
> 
> 

Re: [RFC/RFT PATCH 3/6] arm64, numa: Move pcibus_to_node definition to generic numa code

[Atish Patra]

On Fri, Aug 28, 2020 at 9:15 AM Bjorn Helgaas <helgaas@kernel.org> wrote:
>
> On Fri, Aug 28, 2020 at 10:48:30AM +0100, Jonathan Cameron wrote:
> > On Fri, 14 Aug 2020 14:47:22 -0700
> > Atish Patra <atish.patra@wdc.com> wrote:
> >
> > > pcibus_to_node is used only when numa is enabled and does not depend
> > > on ISA. Thus, it can be moved the generic numa implementation.
> > >
> > > Signed-off-by: Atish Patra <atish.patra@wdc.com>
> >
> > From a more general unification point of view, there seem to
> > be two ways architectures implement this.
> > Either
> >
> > bus->sysdata.node
> >
> > Or as here.
> > There are weird other options, but let us ignore those :)
> >
> > That is going to take a bit of unwinding should we
> > want to take this unification further and perhaps we want to think
> > about doing this in pci generic code rather than here?
> >
> > Perhaps this is one we are better keeping architecture specific for
> > now?
> >
> > +CC Bjorn and Linux-pci
> >
> >
> > > ---
> > >  arch/arm64/kernel/pci.c  | 10 ----------
> > >  drivers/base/arch_numa.c | 11 +++++++++++
> > >  2 files changed, 11 insertions(+), 10 deletions(-)
> > >
> > > diff --git a/arch/arm64/kernel/pci.c b/arch/arm64/kernel/pci.c
> > > index 1006ed2d7c60..07c122946c11 100644
> > > --- a/arch/arm64/kernel/pci.c
> > > +++ b/arch/arm64/kernel/pci.c
> > > @@ -54,16 +54,6 @@ int raw_pci_write(unsigned int domain, unsigned int bus,
> > >     return b->ops->write(b, devfn, reg, len, val);
> > >  }
> > >
> > > -#ifdef CONFIG_NUMA
> > > -
> > > -int pcibus_to_node(struct pci_bus *bus)
> > > -{
> > > -   return dev_to_node(&bus->dev);
> > > -}
> > > -EXPORT_SYMBOL(pcibus_to_node);
> > > -
> > > -#endif
> > > -
> > >  #ifdef CONFIG_ACPI
> > >
> > >  struct acpi_pci_generic_root_info {
> > > diff --git a/drivers/base/arch_numa.c b/drivers/base/arch_numa.c
> > > index 83341c807240..4ab1b20a615d 100644
> > > --- a/drivers/base/arch_numa.c
> > > +++ b/drivers/base/arch_numa.c
> > > @@ -11,6 +11,7 @@
> > >  #include <linux/acpi.h>
> > >  #include <linux/memblock.h>
> > >  #include <linux/module.h>
> > > +#include <linux/pci.h>
> > >  #include <linux/of.h>
> > >
> > >  #ifdef CONFIG_ARM64
> > > @@ -60,6 +61,16 @@ EXPORT_SYMBOL(cpumask_of_node);
> > >
> > >  #endif
> > >
> > > +#ifdef CONFIG_PCI
> > > +
> > > +int pcibus_to_node(struct pci_bus *bus)
> > > +{
> > > +   return dev_to_node(&bus->dev);
> > > +}
> > > +EXPORT_SYMBOL(pcibus_to_node);
> > > +
> > > +#endif
>
> I certainly agree that this should not be arch-specific, but I'm not
> really in favor of adding this PCI gunk in drivers/base.
>
> I think we can do better (eventually) by getting rid of
> pcibus_to_node() completely.  It's not used very much except by
> cpumask_of_pcibus(), which itself is hardly used at all.
>
I am a bit confused here. A quick grep suggested that pcibus_to_node()
is also called from generic pci probe,
controller and few drivers(block, infiniband) as well. Maybe I am
missing something here ?

We can move the pcibus_to_node to arch specific code for now if that's
what is preferred.

> > >  static void numa_update_cpu(unsigned int cpu, bool remove)
> > >  {
> > >     int nid = cpu_to_node(cpu);
> >
> >
>
> _______________________________________________
> linux-riscv mailing list
> linux-riscv@lists.infradead.org
> http://lists.infradead.org/mailman/listinfo/linux-riscv



--
Regards,
Atish

Re: [RFC/RFT PATCH 3/6] arm64, numa: Move pcibus_to_node definition to generic numa code

[Bjorn Helgaas]

On Fri, Aug 28, 2020 at 06:11:50PM -0700, Atish Patra wrote:
> On Fri, Aug 28, 2020 at 9:15 AM Bjorn Helgaas <helgaas@kernel.org> wrote:
> > On Fri, Aug 28, 2020 at 10:48:30AM +0100, Jonathan Cameron wrote:
> > > On Fri, 14 Aug 2020 14:47:22 -0700
> > > Atish Patra <atish.patra@wdc.com> wrote:
> > >
> > > > pcibus_to_node is used only when numa is enabled and does not depend
> > > > on ISA. Thus, it can be moved the generic numa implementation.
> > > >
> > > > Signed-off-by: Atish Patra <atish.patra@wdc.com>
> > >
> > > From a more general unification point of view, there seem to
> > > be two ways architectures implement this.
> > > Either
> > >
> > > bus->sysdata.node
> > >
> > > Or as here.
> > > There are weird other options, but let us ignore those :)
> > >
> > > That is going to take a bit of unwinding should we
> > > want to take this unification further and perhaps we want to think
> > > about doing this in pci generic code rather than here?
> > >
> > > Perhaps this is one we are better keeping architecture specific for
> > > now?
> > >
> > > +CC Bjorn and Linux-pci
> > >
> > >
> > > > ---
> > > >  arch/arm64/kernel/pci.c  | 10 ----------
> > > >  drivers/base/arch_numa.c | 11 +++++++++++
> > > >  2 files changed, 11 insertions(+), 10 deletions(-)
> > > >
> > > > diff --git a/arch/arm64/kernel/pci.c b/arch/arm64/kernel/pci.c
> > > > index 1006ed2d7c60..07c122946c11 100644
> > > > --- a/arch/arm64/kernel/pci.c
> > > > +++ b/arch/arm64/kernel/pci.c
> > > > @@ -54,16 +54,6 @@ int raw_pci_write(unsigned int domain, unsigned int bus,
> > > >     return b->ops->write(b, devfn, reg, len, val);
> > > >  }
> > > >
> > > > -#ifdef CONFIG_NUMA
> > > > -
> > > > -int pcibus_to_node(struct pci_bus *bus)
> > > > -{
> > > > -   return dev_to_node(&bus->dev);
> > > > -}
> > > > -EXPORT_SYMBOL(pcibus_to_node);
> > > > -
> > > > -#endif
> > > > -
> > > >  #ifdef CONFIG_ACPI
> > > >
> > > >  struct acpi_pci_generic_root_info {
> > > > diff --git a/drivers/base/arch_numa.c b/drivers/base/arch_numa.c
> > > > index 83341c807240..4ab1b20a615d 100644
> > > > --- a/drivers/base/arch_numa.c
> > > > +++ b/drivers/base/arch_numa.c
> > > > @@ -11,6 +11,7 @@
> > > >  #include <linux/acpi.h>
> > > >  #include <linux/memblock.h>
> > > >  #include <linux/module.h>
> > > > +#include <linux/pci.h>
> > > >  #include <linux/of.h>
> > > >
> > > >  #ifdef CONFIG_ARM64
> > > > @@ -60,6 +61,16 @@ EXPORT_SYMBOL(cpumask_of_node);
> > > >
> > > >  #endif
> > > >
> > > > +#ifdef CONFIG_PCI
> > > > +
> > > > +int pcibus_to_node(struct pci_bus *bus)
> > > > +{
> > > > +   return dev_to_node(&bus->dev);
> > > > +}
> > > > +EXPORT_SYMBOL(pcibus_to_node);
> > > > +
> > > > +#endif
> >
> > I certainly agree that this should not be arch-specific, but I'm not
> > really in favor of adding this PCI gunk in drivers/base.
> >
> > I think we can do better (eventually) by getting rid of
> > pcibus_to_node() completely.  It's not used very much except by
> > cpumask_of_pcibus(), which itself is hardly used at all.
> >
> I am a bit confused here. A quick grep suggested that pcibus_to_node()
> is also called from generic pci probe,
> controller and few drivers(block, infiniband) as well. Maybe I am
> missing something here ?

I didn't say it was *only* used by cpumask_of_pcibus().  13 of the 29
calls are from cpumask_of_pcibus().

As you point out, there are a few drivers that use it.  They typically
have a pci_dev, so they do the equivalent of pcibus_to_node(pdev->bus).
That seems silly; they should just do dev_to_node(&pdev->dev) instead.

I looked at this once, and it seems like there might have been a
wrinkle like the pdev->dev node not being set correctly or something.
If that's the case, I think it should be fixed.

> We can move the pcibus_to_node to arch specific code for now if that's
> what is preferred.

Now I'm the one who's confused :)  Most arches, including arm64,
already have arch-specific implementations of pcibus_to_node().  I
didn't look at the rest of the series to see if there's a reason you
need to move pcibus_to_node() from arch/arm64/kernel/pci.c to
drivers//base/arch_numa.c.  If you don't need to, I would just leave
it where it is.

> > > >  static void numa_update_cpu(unsigned int cpu, bool remove)
> > > >  {
> > > >     int nid = cpu_to_node(cpu);

Re: [RFC/RFT PATCH 3/6] arm64, numa: Move pcibus_to_node definition to generic numa code

[Atish Patra]

On Sat, Aug 29, 2020 at 7:54 PM Bjorn Helgaas <helgaas@kernel.org> wrote:
>
> On Fri, Aug 28, 2020 at 06:11:50PM -0700, Atish Patra wrote:
> > On Fri, Aug 28, 2020 at 9:15 AM Bjorn Helgaas <helgaas@kernel.org> wrote:
> > > On Fri, Aug 28, 2020 at 10:48:30AM +0100, Jonathan Cameron wrote:
> > > > On Fri, 14 Aug 2020 14:47:22 -0700
> > > > Atish Patra <atish.patra@wdc.com> wrote:
> > > >
> > > > > pcibus_to_node is used only when numa is enabled and does not depend
> > > > > on ISA. Thus, it can be moved the generic numa implementation.
> > > > >
> > > > > Signed-off-by: Atish Patra <atish.patra@wdc.com>
> > > >
> > > > From a more general unification point of view, there seem to
> > > > be two ways architectures implement this.
> > > > Either
> > > >
> > > > bus->sysdata.node
> > > >
> > > > Or as here.
> > > > There are weird other options, but let us ignore those :)
> > > >
> > > > That is going to take a bit of unwinding should we
> > > > want to take this unification further and perhaps we want to think
> > > > about doing this in pci generic code rather than here?
> > > >
> > > > Perhaps this is one we are better keeping architecture specific for
> > > > now?
> > > >
> > > > +CC Bjorn and Linux-pci
> > > >
> > > >
> > > > > ---
> > > > >  arch/arm64/kernel/pci.c  | 10 ----------
> > > > >  drivers/base/arch_numa.c | 11 +++++++++++
> > > > >  2 files changed, 11 insertions(+), 10 deletions(-)
> > > > >
> > > > > diff --git a/arch/arm64/kernel/pci.c b/arch/arm64/kernel/pci.c
> > > > > index 1006ed2d7c60..07c122946c11 100644
> > > > > --- a/arch/arm64/kernel/pci.c
> > > > > +++ b/arch/arm64/kernel/pci.c
> > > > > @@ -54,16 +54,6 @@ int raw_pci_write(unsigned int domain, unsigned int bus,
> > > > >     return b->ops->write(b, devfn, reg, len, val);
> > > > >  }
> > > > >
> > > > > -#ifdef CONFIG_NUMA
> > > > > -
> > > > > -int pcibus_to_node(struct pci_bus *bus)
> > > > > -{
> > > > > -   return dev_to_node(&bus->dev);
> > > > > -}
> > > > > -EXPORT_SYMBOL(pcibus_to_node);
> > > > > -
> > > > > -#endif
> > > > > -
> > > > >  #ifdef CONFIG_ACPI
> > > > >
> > > > >  struct acpi_pci_generic_root_info {
> > > > > diff --git a/drivers/base/arch_numa.c b/drivers/base/arch_numa.c
> > > > > index 83341c807240..4ab1b20a615d 100644
> > > > > --- a/drivers/base/arch_numa.c
> > > > > +++ b/drivers/base/arch_numa.c
> > > > > @@ -11,6 +11,7 @@
> > > > >  #include <linux/acpi.h>
> > > > >  #include <linux/memblock.h>
> > > > >  #include <linux/module.h>
> > > > > +#include <linux/pci.h>
> > > > >  #include <linux/of.h>
> > > > >
> > > > >  #ifdef CONFIG_ARM64
> > > > > @@ -60,6 +61,16 @@ EXPORT_SYMBOL(cpumask_of_node);
> > > > >
> > > > >  #endif
> > > > >
> > > > > +#ifdef CONFIG_PCI
> > > > > +
> > > > > +int pcibus_to_node(struct pci_bus *bus)
> > > > > +{
> > > > > +   return dev_to_node(&bus->dev);
> > > > > +}
> > > > > +EXPORT_SYMBOL(pcibus_to_node);
> > > > > +
> > > > > +#endif
> > >
> > > I certainly agree that this should not be arch-specific, but I'm not
> > > really in favor of adding this PCI gunk in drivers/base.
> > >
> > > I think we can do better (eventually) by getting rid of
> > > pcibus_to_node() completely.  It's not used very much except by
> > > cpumask_of_pcibus(), which itself is hardly used at all.
> > >
> > I am a bit confused here. A quick grep suggested that pcibus_to_node()
> > is also called from generic pci probe,
> > controller and few drivers(block, infiniband) as well. Maybe I am
> > missing something here ?
>
> I didn't say it was *only* used by cpumask_of_pcibus().  13 of the 29
> calls are from cpumask_of_pcibus().
>
Ahh okay. Sorry I misunderstood that.

> As you point out, there are a few drivers that use it.  They typically
> have a pci_dev, so they do the equivalent of pcibus_to_node(pdev->bus).
> That seems silly; they should just do dev_to_node(&pdev->dev) instead.
>

That covers the use case for ARM64. There are other arch implementations
as well which retrieve node information from sysdata which seems to be
type casted to different structures on different arch.
What should be done for those ?

> I looked at this once, and it seems like there might have been a
> wrinkle like the pdev->dev node not being set correctly or something.
> If that's the case, I think it should be fixed.
>
> > We can move the pcibus_to_node to arch specific code for now if that's
> > what is preferred.
>
> Now I'm the one who's confused :)  Most arches, including arm64,
> already have arch-specific implementations of pcibus_to_node().  I
> didn't look at the rest of the series to see if there's a reason you
> need to move pcibus_to_node() from arch/arm64/kernel/pci.c to
> drivers//base/arch_numa.c.  If you don't need to, I would just leave
> it where it is.
>

The reason I moved it from arch/arm64/kernel/pci.c to drivers//base/arch_numa.c.
so that we don't have to define it for RISC-V. But it's just a single
line function and
we can define it in RISC-V as well. I will try that in v2.

> > > > >  static void numa_update_cpu(unsigned int cpu, bool remove)
> > > > >  {
> > > > >     int nid = cpu_to_node(cpu);



-- 
Regards,
Atish

[RFC/RFT PATCH 4/6] riscv: Separate memory init from paging init

[Atish Patra]

Currently, we perform some memory init functions in paging init. But,
that will be an issue for NUMA support where DT needs to be flattened
before numa initialization and memblock_present can only be called
after numa initialization.

Move memory initialization related functions to a separate function.

Signed-off-by: Atish Patra <atish.patra@wdc.com>
---
 arch/riscv/include/asm/pgtable.h | 1 +
 arch/riscv/kernel/setup.c        | 2 ++
 arch/riscv/mm/init.c             | 6 +++++-
 3 files changed, 8 insertions(+), 1 deletion(-)

diff --git a/arch/riscv/include/asm/pgtable.h b/arch/riscv/include/asm/pgtable.h
index eaea1f717010..515b42f98d34 100644
--- a/arch/riscv/include/asm/pgtable.h
+++ b/arch/riscv/include/asm/pgtable.h
@@ -466,6 +466,7 @@ static inline void __kernel_map_pages(struct page *page, int numpages, int enabl
 extern void *dtb_early_va;
 void setup_bootmem(void);
 void paging_init(void);
+void misc_mem_init(void);
 
 #define FIRST_USER_ADDRESS  0
 
diff --git a/arch/riscv/kernel/setup.c b/arch/riscv/kernel/setup.c
index f04373be54a6..32bb5a1bea05 100644
--- a/arch/riscv/kernel/setup.c
+++ b/arch/riscv/kernel/setup.c
@@ -79,6 +79,8 @@ void __init setup_arch(char **cmdline_p)
 #else
 	unflatten_device_tree();
 #endif
+	misc_mem_init();
+
 	clint_init_boot_cpu();
 
 #ifdef CONFIG_SWIOTLB
diff --git a/arch/riscv/mm/init.c b/arch/riscv/mm/init.c
index 787c75f751a5..b8905ae2bbe7 100644
--- a/arch/riscv/mm/init.c
+++ b/arch/riscv/mm/init.c
@@ -570,8 +570,12 @@ static void __init resource_init(void)
 void __init paging_init(void)
 {
 	setup_vm_final();
-	sparse_init();
 	setup_zero_page();
+}
+
+void __init misc_mem_init(void)
+{
+	sparse_init();
 	zone_sizes_init();
 	resource_init();
 }
-- 
2.24.0

[RFC/RFT PATCH 5/6] riscv: Add support pte_protnone and pmd_protnone if CONFIG_NUMA_BALANCING

[Atish Patra]

From: Greentime Hu <greentime.hu@sifive.com>

These two functions are used to distinguish between PROT_NONENUMA
protections and hinting fault protections.

Signed-off-by: Greentime Hu <greentime.hu@sifive.com>
---
 arch/riscv/include/asm/pgtable.h | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)

diff --git a/arch/riscv/include/asm/pgtable.h b/arch/riscv/include/asm/pgtable.h
index 515b42f98d34..2751110675e6 100644
--- a/arch/riscv/include/asm/pgtable.h
+++ b/arch/riscv/include/asm/pgtable.h
@@ -183,6 +183,11 @@ static inline unsigned long pmd_page_vaddr(pmd_t pmd)
 	return (unsigned long)pfn_to_virt(pmd_val(pmd) >> _PAGE_PFN_SHIFT);
 }
 
+static inline pte_t pmd_pte(pmd_t pmd)
+{
+	return __pte(pmd_val(pmd));
+}
+
 /* Yields the page frame number (PFN) of a page table entry */
 static inline unsigned long pte_pfn(pte_t pte)
 {
@@ -286,6 +291,21 @@ static inline pte_t pte_mkhuge(pte_t pte)
 	return pte;
 }
 
+#ifdef CONFIG_NUMA_BALANCING
+/*
+ * See the comment in include/asm-generic/pgtable.h
+ */
+static inline int pte_protnone(pte_t pte)
+{
+	return (pte_val(pte) & (_PAGE_PRESENT | _PAGE_PROT_NONE)) == _PAGE_PROT_NONE;
+}
+
+static inline int pmd_protnone(pmd_t pmd)
+{
+	return pte_protnone(pmd_pte(pmd));
+}
+#endif
+
 /* Modify page protection bits */
 static inline pte_t pte_modify(pte_t pte, pgprot_t newprot)
 {
-- 
2.24.0

[RFC/RFT PATCH 6/6] riscv: Add numa support for riscv64 platform

[Atish Patra]

Use the generic numa implementation to add NUMA support for RISC-V.
This is based on Greentime's patch[1] but modified to use generic NUMA
implementation and few more fixes.

[1] https://lkml.org/lkml/2020/1/10/233

Co-developed-by: Greentime Hu <greentime.hu@sifive.com>
Signed-off-by: Greentime Hu <greentime.hu@sifive.com>
Signed-off-by: Atish Patra <atish.patra@wdc.com>
---
 arch/riscv/Kconfig              | 31 ++++++++++++++++++++++++++++++-
 arch/riscv/include/asm/mmzone.h | 13 +++++++++++++
 arch/riscv/include/asm/numa.h   |  8 ++++++++
 arch/riscv/include/asm/pci.h    | 10 ++++++++++
 arch/riscv/kernel/setup.c       | 10 ++++++++--
 arch/riscv/kernel/smpboot.c     | 12 +++++++++++-
 arch/riscv/mm/init.c            |  4 +++-
 7 files changed, 83 insertions(+), 5 deletions(-)
 create mode 100644 arch/riscv/include/asm/mmzone.h
 create mode 100644 arch/riscv/include/asm/numa.h

diff --git a/arch/riscv/Kconfig b/arch/riscv/Kconfig
index 7b5905529146..4bd67f94aaac 100644
--- a/arch/riscv/Kconfig
+++ b/arch/riscv/Kconfig
@@ -137,7 +137,7 @@ config PAGE_OFFSET
 	default 0xffffffe000000000 if 64BIT && MAXPHYSMEM_128GB
 
 config ARCH_FLATMEM_ENABLE
-	def_bool y
+	def_bool !NUMA
 
 config ARCH_SPARSEMEM_ENABLE
 	def_bool y
@@ -295,6 +295,35 @@ config TUNE_GENERIC
 
 endchoice
 
+# Common NUMA Features
+config NUMA
+	bool "Numa Memory Allocation and Scheduler Support"
+	select GENERIC_ARCH_NUMA
+	select OF_NUMA
+	select ARCH_SUPPORTS_NUMA_BALANCING
+	help
+	  Enable NUMA (Non-Uniform Memory Access) support.
+
+	  The kernel will try to allocate memory used by a CPU on the
+	  local memory of the CPU and add some more NUMA awareness to the kernel.
+
+config NODES_SHIFT
+	int "Maximum NUMA Nodes (as a power of 2)"
+	range 1 10
+	default "2"
+	depends on NEED_MULTIPLE_NODES
+	help
+	  Specify the maximum number of NUMA Nodes available on the target
+	  system.  Increases memory reserved to accommodate various tables.
+
+config USE_PERCPU_NUMA_NODE_ID
+	def_bool y
+	depends on NUMA
+
+config NEED_PER_CPU_EMBED_FIRST_CHUNK
+	def_bool y
+	depends on NUMA
+
 config RISCV_ISA_C
 	bool "Emit compressed instructions when building Linux"
 	default y
diff --git a/arch/riscv/include/asm/mmzone.h b/arch/riscv/include/asm/mmzone.h
new file mode 100644
index 000000000000..fa17e01d9ab2
--- /dev/null
+++ b/arch/riscv/include/asm/mmzone.h
@@ -0,0 +1,13 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef __ASM_MMZONE_H
+#define __ASM_MMZONE_H
+
+#ifdef CONFIG_NUMA
+
+#include <asm/numa.h>
+
+extern struct pglist_data *node_data[];
+#define NODE_DATA(nid)		(node_data[(nid)])
+
+#endif /* CONFIG_NUMA */
+#endif /* __ASM_MMZONE_H */
diff --git a/arch/riscv/include/asm/numa.h b/arch/riscv/include/asm/numa.h
new file mode 100644
index 000000000000..8c8cf4297cc3
--- /dev/null
+++ b/arch/riscv/include/asm/numa.h
@@ -0,0 +1,8 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef __ASM_NUMA_H
+#define __ASM_NUMA_H
+
+#include <asm/topology.h>
+#include <asm-generic/numa.h>
+
+#endif	/* __ASM_NUMA_H */
diff --git a/arch/riscv/include/asm/pci.h b/arch/riscv/include/asm/pci.h
index 1c473a1bd986..d6a0e59638c0 100644
--- a/arch/riscv/include/asm/pci.h
+++ b/arch/riscv/include/asm/pci.h
@@ -32,6 +32,16 @@ static inline int pci_proc_domain(struct pci_bus *bus)
 	/* always show the domain in /proc */
 	return 1;
 }
+
+#ifdef	CONFIG_NUMA
+int pcibus_to_node(struct pci_bus *bus);
+#ifndef cpumask_of_pcibus
+#define cpumask_of_pcibus(bus)	(pcibus_to_node(bus) == -1 ?		\
+				 cpu_all_mask :				\
+				 cpumask_of_node(pcibus_to_node(bus)))
+#endif
+#endif	/* CONFIG_NUMA */
+
 #endif  /* CONFIG_PCI */
 
 #endif  /* _ASM_RISCV_PCI_H */
diff --git a/arch/riscv/kernel/setup.c b/arch/riscv/kernel/setup.c
index 32bb5a1bea05..1533ee5c6e56 100644
--- a/arch/riscv/kernel/setup.c
+++ b/arch/riscv/kernel/setup.c
@@ -104,13 +104,19 @@ void __init setup_arch(char **cmdline_p)
 
 static int __init topology_init(void)
 {
-	int i;
+	int i, ret;
+
+	for_each_online_node(i)
+		register_one_node(i);
 
 	for_each_possible_cpu(i) {
 		struct cpu *cpu = &per_cpu(cpu_devices, i);
 
 		cpu->hotpluggable = cpu_has_hotplug(i);
-		register_cpu(cpu, i);
+		ret = register_cpu(cpu, i);
+		if (unlikely(ret))
+			pr_warn("Warning: %s: register_cpu %d failed (%d)\n",
+			       __func__, i, ret);
 	}
 
 	return 0;
diff --git a/arch/riscv/kernel/smpboot.c b/arch/riscv/kernel/smpboot.c
index 356825a57551..4e9bdb6230de 100644
--- a/arch/riscv/kernel/smpboot.c
+++ b/arch/riscv/kernel/smpboot.c
@@ -28,6 +28,7 @@
 #include <asm/cpu_ops.h>
 #include <asm/irq.h>
 #include <asm/mmu_context.h>
+#include <asm/numa.h>
 #include <asm/tlbflush.h>
 #include <asm/sections.h>
 #include <asm/sbi.h>
@@ -46,13 +47,18 @@ void __init smp_prepare_cpus(unsigned int max_cpus)
 {
 	int cpuid;
 	int ret;
+	unsigned int curr_cpuid;
+
+	curr_cpuid = smp_processor_id();
+	numa_store_cpu_info(curr_cpuid);
+	numa_add_cpu(curr_cpuid);
 
 	/* This covers non-smp usecase mandated by "nosmp" option */
 	if (max_cpus == 0)
 		return;
 
 	for_each_possible_cpu(cpuid) {
-		if (cpuid == smp_processor_id())
+		if (cpuid == curr_cpuid)
 			continue;
 		if (cpu_ops[cpuid]->cpu_prepare) {
 			ret = cpu_ops[cpuid]->cpu_prepare(cpuid);
@@ -60,6 +66,7 @@ void __init smp_prepare_cpus(unsigned int max_cpus)
 				continue;
 		}
 		set_cpu_present(cpuid, true);
+		numa_store_cpu_info(cpuid);
 	}
 }
 
@@ -80,6 +87,7 @@ void __init setup_smp(void)
 		if (hart == cpuid_to_hartid_map(0)) {
 			BUG_ON(found_boot_cpu);
 			found_boot_cpu = 1;
+			early_map_cpu_to_node(0, of_node_to_nid(dn));
 			continue;
 		}
 		if (cpuid >= NR_CPUS) {
@@ -89,6 +97,7 @@ void __init setup_smp(void)
 		}
 
 		cpuid_to_hartid_map(cpuid) = hart;
+		early_map_cpu_to_node(cpuid, of_node_to_nid(dn));
 		cpuid++;
 	}
 
@@ -155,6 +164,7 @@ asmlinkage __visible void smp_callin(void)
 	current->active_mm = mm;
 
 	notify_cpu_starting(curr_cpuid);
+	numa_add_cpu(curr_cpuid);
 	update_siblings_masks(curr_cpuid);
 	set_cpu_online(curr_cpuid, 1);
 
diff --git a/arch/riscv/mm/init.c b/arch/riscv/mm/init.c
index b8905ae2bbe7..d1f4e94626aa 100644
--- a/arch/riscv/mm/init.c
+++ b/arch/riscv/mm/init.c
@@ -20,6 +20,7 @@
 #include <asm/soc.h>
 #include <asm/io.h>
 #include <asm/ptdump.h>
+#include <asm/numa.h>
 
 #include "../kernel/head.h"
 
@@ -190,7 +191,6 @@ void __init setup_bootmem(void)
 
 	early_init_fdt_scan_reserved_mem();
 	memblock_allow_resize();
-	memblock_dump_all();
 
 	for_each_memblock(memory, reg) {
 		unsigned long start_pfn = memblock_region_memory_base_pfn(reg);
@@ -575,9 +575,11 @@ void __init paging_init(void)
 
 void __init misc_mem_init(void)
 {
+	arch_numa_init();
 	sparse_init();
 	zone_sizes_init();
 	resource_init();
+	memblock_dump_all();
 }
 
 #ifdef CONFIG_SPARSEMEM_VMEMMAP
-- 
2.24.0

Re: [RFC/RFT PATCH 6/6] riscv: Add numa support for riscv64 platform

[Randy Dunlap]

On 8/14/20 2:47 PM, Atish Patra wrote:
> diff --git a/arch/riscv/Kconfig b/arch/riscv/Kconfig
> index 7b5905529146..4bd67f94aaac 100644
> --- a/arch/riscv/Kconfig
> +++ b/arch/riscv/Kconfig
> @@ -137,7 +137,7 @@ config PAGE_OFFSET
>  	default 0xffffffe000000000 if 64BIT && MAXPHYSMEM_128GB
>  
>  config ARCH_FLATMEM_ENABLE
> -	def_bool y
> +	def_bool !NUMA
>  
>  config ARCH_SPARSEMEM_ENABLE
>  	def_bool y
> @@ -295,6 +295,35 @@ config TUNE_GENERIC
>  
>  endchoice
>  
> +# Common NUMA Features
> +config NUMA
> +	bool "Numa Memory Allocation and Scheduler Support"

	      NUMA

> +	select GENERIC_ARCH_NUMA
> +	select OF_NUMA
> +	select ARCH_SUPPORTS_NUMA_BALANCING
> +	help
> +	  Enable NUMA (Non-Uniform Memory Access) support.
> +
> +	  The kernel will try to allocate memory used by a CPU on the
> +	  local memory of the CPU and add some more NUMA awareness to the kernel.
> +
> +config NODES_SHIFT
> +	int "Maximum NUMA Nodes (as a power of 2)"
> +	range 1 10
> +	default "2"
> +	depends on NEED_MULTIPLE_NODES
> +	help
> +	  Specify the maximum number of NUMA Nodes available on the target
> +	  system.  Increases memory reserved to accommodate various tables.
> +
> +config USE_PERCPU_NUMA_NODE_ID
> +	def_bool y
> +	depends on NUMA
> +
> +config NEED_PER_CPU_EMBED_FIRST_CHUNK
> +	def_bool y
> +	depends on NUMA
> +
>  config RISCV_ISA_C
>  	bool "Emit compressed instructions when building Linux"
>  	default y


thanks.
-- 
~Randy