[RFC PATCH v2 14/25] ARM: NUMA: Parse NUMA distance information

From: vijay.kilari@gmail.com
To: xen-devel@lists.xen.org
Cc: sstabellini@kernel.org, wei.liu2@citrix.com,
	George.Dunlap@eu.citrix.com, andrew.cooper3@citrix.com,
	ian.jackson@eu.citrix.com, tim@xen.org, julien.grall@arm.com,
	jbeulich@suse.com, Vijaya Kumar K <Vijaya.Kumar@cavium.com>
Subject: [RFC PATCH v2 14/25] ARM: NUMA: Parse NUMA distance information
Date: Tue, 28 Mar 2017 21:23:22 +0530	[thread overview]
Message-ID: <1490716413-19796-15-git-send-email-vijay.kilari@gmail.com> (raw)
In-Reply-To: <1490716413-19796-1-git-send-email-vijay.kilari@gmail.com>

From: Vijaya Kumar K <Vijaya.Kumar@cavium.com>

Parse distance-matrix and fetch node distance information.
Store distance information in node_distance[].

Register dt_node_distance() function pointer with
the ARM numa code. This approach can be later used for
ACPI.

Signed-off-by: Vijaya Kumar K <Vijaya.Kumar@cavium.com>
---
 xen/arch/arm/bootfdt.c      |   4 +-
 xen/arch/arm/numa/dt_numa.c | 133 ++++++++++++++++++++++++++++++++++++++++++++
 xen/arch/arm/numa/numa.c    |  21 +++++++
 xen/include/asm-arm/numa.h  |   3 +
 xen/include/asm-arm/setup.h |   2 +
 5 files changed, 161 insertions(+), 2 deletions(-)

diff --git a/xen/arch/arm/bootfdt.c b/xen/arch/arm/bootfdt.c
index 993760a..c72300c 100644
--- a/xen/arch/arm/bootfdt.c
+++ b/xen/arch/arm/bootfdt.c
@@ -17,8 +17,8 @@
 #include <xsm/xsm.h>
 #include <asm/setup.h>
 
-static bool_t __init device_tree_node_matches(const void *fdt, int node,
-                                              const char *match)
+bool_t __init device_tree_node_matches(const void *fdt, int node,
+                                       const char *match)
 {
     const char *name;
     size_t match_len;
diff --git a/xen/arch/arm/numa/dt_numa.c b/xen/arch/arm/numa/dt_numa.c
index 593c647..c2dcfa1 100644
--- a/xen/arch/arm/numa/dt_numa.c
+++ b/xen/arch/arm/numa/dt_numa.c
@@ -27,6 +27,48 @@
 extern nodemask_t processor_nodes_parsed;
 extern nodemask_t memory_nodes_parsed;
 
+static uint8_t node_distance[MAX_NUMNODES][MAX_NUMNODES];
+
+static uint8_t dt_node_distance(nodeid_t nodea, nodeid_t nodeb)
+{
+    if ( nodea >= MAX_NUMNODES || nodeb >= MAX_NUMNODES )
+        return nodea == nodeb ? LOCAL_DISTANCE : REMOTE_DISTANCE;
+
+    return node_distance[nodea][nodeb];
+}
+
+static int dt_numa_set_distance(uint32_t nodea, uint32_t nodeb,
+                                uint32_t distance)
+{
+   /* node_distance is uint8_t. Ensure distance is less than 255 */
+   if ( nodea >= MAX_NUMNODES || nodeb >= MAX_NUMNODES || distance > 255 )
+       return -EINVAL;
+
+   node_distance[nodea][nodeb] = distance;
+
+   return 0;
+}
+
+void init_dt_numa_distance(void)
+{
+    int i, j;
+
+    for ( i = 0; i < MAX_NUMNODES; i++ )
+    {
+        for ( j = 0; j < MAX_NUMNODES; j++ )
+        {
+            /*
+             * Initialize distance 10 for local distance and
+             * 20 for remote distance.
+             */
+            if ( i  == j )
+                node_distance[i][j] = LOCAL_DISTANCE;
+            else
+                node_distance[i][j] = REMOTE_DISTANCE;
+        }
+    }
+}
+
 /*
  * Even though we connect cpus to numa domains later in SMP
  * init, we need to know the node ids now for all cpus.
@@ -48,6 +90,76 @@ static int __init dt_numa_process_cpu_node(const void *fdt, int node,
     return 0;
 }
 
+static int __init dt_numa_parse_distance_map(const void *fdt, int node,
+                                             const char *name,
+                                             uint32_t address_cells,
+                                             uint32_t size_cells)
+{
+    const struct fdt_property *prop;
+    const __be32 *matrix;
+    int entry_count, len, i;
+
+    printk(XENLOG_INFO "NUMA: parsing numa-distance-map\n");
+
+    prop = fdt_get_property(fdt, node, "distance-matrix", &len);
+    if ( !prop )
+    {
+        printk(XENLOG_WARNING
+               "NUMA: No distance-matrix property in distance-map\n");
+
+        return -EINVAL;
+    }
+
+    if ( len % sizeof(uint32_t) != 0 )
+    {
+         printk(XENLOG_WARNING
+                "distance-matrix in node is not a multiple of u32\n");
+
+        return -EINVAL;
+    }
+
+    entry_count = len / sizeof(uint32_t);
+    if ( entry_count <= 0 )
+    {
+        printk(XENLOG_WARNING "NUMA: Invalid distance-matrix\n");
+
+        return -EINVAL;
+    }
+
+    matrix = (const __be32 *)prop->data;
+    for ( i = 0; i + 2 < entry_count; i += 3 )
+    {
+        uint32_t nodea, nodeb, distance;
+
+        nodea = dt_read_number(matrix, 1);
+        matrix++;
+        nodeb = dt_read_number(matrix, 1);
+        matrix++;
+        distance = dt_read_number(matrix, 1);
+        matrix++;
+
+        if ( dt_numa_set_distance(nodea, nodeb, distance) )
+        {
+            printk(XENLOG_WARNING
+                   "NUMA: node-id out of range in distance matrix for [node%d -> node%d]\n",
+                   nodea, nodeb);
+            return -EINVAL;
+
+        }
+        printk(XENLOG_INFO "NUMA: distance[node%d -> node%d] = %d\n",
+               nodea, nodeb, distance);
+
+        /*
+         * Set default distance of node B->A same as A->B.
+         * No need to check for return value of numa_set_distance.
+         */
+        if ( nodeb > nodea )
+            dt_numa_set_distance(nodeb, nodea, distance);
+    }
+
+    return 0;
+}
+
 static int __init dt_numa_scan_cpu_node(const void *fdt, int node,
                                         const char *name, int depth,
                                         uint32_t address_cells,
@@ -92,12 +204,33 @@ void __init dt_numa_process_memory_node(uint32_t nid, paddr_t start,
     return;
 }
 
+static int __init dt_numa_scan_distance_node(const void *fdt, int node,
+                                             const char *name, int depth,
+                                             uint32_t address_cells,
+                                             uint32_t size_cells, void *data)
+{
+    if ( device_tree_node_matches(fdt, node, "numa-distance-map-v1") )
+        return dt_numa_parse_distance_map(fdt, node, name, address_cells,
+                                          size_cells);
+
+    return 0;
+}
+
 int __init dt_numa_init(void)
 {
     int ret;
 
     ret = device_tree_for_each_node((void *)device_tree_flattened,
                                     dt_numa_scan_cpu_node, NULL);
+    if ( ret )
+        return ret;
+
+    ret = device_tree_for_each_node((void *)device_tree_flattened,
+                                    dt_numa_scan_distance_node, NULL);
+
+    if ( !ret )
+        register_node_distance(&dt_node_distance);
+
     return ret;
 }
 
diff --git a/xen/arch/arm/numa/numa.c b/xen/arch/arm/numa/numa.c
index b333453..0ee89da 100644
--- a/xen/arch/arm/numa/numa.c
+++ b/xen/arch/arm/numa/numa.c
@@ -21,13 +21,33 @@
 #include <asm/mm.h>
 #include <xen/numa.h>
 #include <asm/acpi.h>
+#include <xen/errno.h>
+
+static uint8_t (*node_distance_fn)(nodeid_t a, nodeid_t b);
 
 extern nodemask_t processor_nodes_parsed;
 static bool_t dt_numa = 1;
 
+uint8_t __node_distance(nodeid_t a, nodeid_t b)
+{
+    if ( node_distance_fn != NULL);
+        return node_distance_fn(a, b);
+
+    return a == b ? LOCAL_DISTANCE : REMOTE_DISTANCE;
+}
+
+EXPORT_SYMBOL(__node_distance);
+
+void register_node_distance(uint8_t (fn)(nodeid_t a, nodeid_t b))
+{
+    node_distance_fn = fn;
+}
+
 void numa_failed(void)
 {
     dt_numa = 0;
+    init_dt_numa_distance();
+    node_distance_fn = NULL;
 }
 
 void __init numa_init(void)
@@ -35,6 +55,7 @@ void __init numa_init(void)
     int ret = 0;
 
     nodes_clear(processor_nodes_parsed);
+    init_dt_numa_distance();
     if ( is_numa_off() )
         goto no_numa;
 
diff --git a/xen/include/asm-arm/numa.h b/xen/include/asm-arm/numa.h
index 962a214..c390a0e 100644
--- a/xen/include/asm-arm/numa.h
+++ b/xen/include/asm-arm/numa.h
@@ -8,6 +8,9 @@ typedef uint8_t nodeid_t;
 
 extern void dt_numa_process_memory_node(uint32_t nid,paddr_t start,
                                         paddr_t size);
+extern void register_node_distance(uint8_t (fn)(nodeid_t a, nodeid_t b));
+extern void init_dt_numa_distance(void);
+extern uint8_t __node_distance(nodeid_t a, nodeid_t b);
 #ifdef CONFIG_NUMA
 extern void numa_init(void);
 extern int dt_numa_init(void);
diff --git a/xen/include/asm-arm/setup.h b/xen/include/asm-arm/setup.h
index b1022a3..8713657 100644
--- a/xen/include/asm-arm/setup.h
+++ b/xen/include/asm-arm/setup.h
@@ -82,6 +82,8 @@ const char * __init boot_module_kind_as_string(bootmodule_kind kind);
 u32 device_tree_get_u32(const void *fdt, int node,
                         const char *prop_name, u32 dflt);
 bool_t device_tree_type_matches(const void *fdt, int node, const char *match);
+bool_t __init device_tree_node_matches(const void *fdt, int node,
+                                       const char *match);
 #endif
 /*
  * Local variables:
-- 
2.7.4


_______________________________________________
Xen-devel mailing list
Xen-devel@lists.xen.org
https://lists.xen.org/xen-devel
