[PATCH] - Externel SLIT table information thru sysfs

[PATCH] - Externel SLIT table information thru sysfs

[Jack Steiner]

The ACPI SLIT table provides useful information on internode distances.
Here is a patch to externalize the SLIT information. 

For example:

        # cd /sys/devices/system
        # find .
        ./node
        ./node/node5
        ./node/node5/distance
        ./node/node5/numastat
        ./node/node5/meminfo
        ./node/node5/cpumap

        # cat ./node/node0/distance
        10 20 64 42 42 22

        # cat node/*/distance
        10 20 64 42 42 22
        20 10 42 22 64 84
        64 42 10 20 22 42
        42 22 20 10 42 62
        42 64 22 42 10 20
        22 84 42 62 20 10





Signed-off-by: Jack Steiner <steiner@sgi.com>


Add SLIT (inter node distance) information to sysfs. 



Index: linux/drivers/base/node.c
===================================================================
--- linux.orig/drivers/base/node.c	2004-11-05 08:34:42.461312000 -0600
+++ linux/drivers/base/node.c	2004-11-05 15:56:23.345662000 -0600
@@ -111,6 +111,24 @@ static ssize_t node_read_numastat(struct
 }
 static SYSDEV_ATTR(numastat, S_IRUGO, node_read_numastat, NULL);
 
+static ssize_t node_read_distance(struct sys_device * dev, char * buf)
+{
+	int nid = dev->id;
+	int len = 0;
+	int i;
+
+	/* buf currently PAGE_SIZE, need ~4 chars per node */
+	BUILD_BUG_ON(NR_NODES*4 > PAGE_SIZE/2);
+
+	for (i = 0; i < numnodes; i++)
+		len += sprintf(buf + len, "%s%d", i ? " " : "", node_distance(nid, i));
+		
+	len += sprintf(buf + len, "\n");
+	return len;
+}
+static SYSDEV_ATTR(distance, S_IRUGO, node_read_distance, NULL);
+
+
 /*
  * register_node - Setup a driverfs device for a node.
  * @num - Node number to use when creating the device.
@@ -129,6 +147,7 @@ int __init register_node(struct node *no
 		sysdev_create_file(&node->sysdev, &attr_cpumap);
 		sysdev_create_file(&node->sysdev, &attr_meminfo);
 		sysdev_create_file(&node->sysdev, &attr_numastat);
+		sysdev_create_file(&node->sysdev, &attr_distance);
 	}
 	return error;
 }
Index: linux/include/asm-i386/topology.h
===================================================================
--- linux.orig/include/asm-i386/topology.h	2004-11-05 08:34:53.713053000 -0600
+++ linux/include/asm-i386/topology.h	2004-11-23 09:59:43.574062951 -0600
@@ -66,9 +66,6 @@ static inline cpumask_t pcibus_to_cpumas
 	return node_to_cpumask(mp_bus_id_to_node[bus]);
 }
 
-/* Node-to-Node distance */
-#define node_distance(from, to) ((from) != (to))
-
 /* sched_domains SD_NODE_INIT for NUMAQ machines */
 #define SD_NODE_INIT (struct sched_domain) {		\
 	.span			= CPU_MASK_NONE,	\
Index: linux/include/linux/topology.h
===================================================================
--- linux.orig/include/linux/topology.h	2004-11-05 08:34:57.492932000 -0600
+++ linux/include/linux/topology.h	2004-11-23 10:03:26.700821978 -0600
@@ -55,7 +55,10 @@ static inline int __next_node_with_cpus(
 	for (node = 0; node < numnodes; node = __next_node_with_cpus(node))
 
 #ifndef node_distance
-#define node_distance(from,to)	((from) != (to))
+/* Conform to ACPI 2.0 SLIT distance definitions */
+#define LOCAL_DISTANCE		10
+#define REMOTE_DISTANCE		20
+#define node_distance(from,to)	((from) == (to) ? LOCAL_DISTANCE : REMOTE_DISTANCE)
 #endif
 #ifndef PENALTY_FOR_NODE_WITH_CPUS
 #define PENALTY_FOR_NODE_WITH_CPUS	(1)
-- 
Thanks

Jack Steiner (steiner@sgi.com)          651-683-5302
Principal Engineer                      SGI - Silicon Graphics, Inc.

Re: [PATCH] - Externel SLIT table information thru sysfs

[Randy.Dunlap]

Jack Steiner wrote:
> The ACPI SLIT table provides useful information on internode distances.
> Here is a patch to externalize the SLIT information. 
> 
> For example:
> 
>         # cd /sys/devices/system
>         # find .
>         ./node
>         ./node/node5
>         ./node/node5/distance
>         ./node/node5/numastat
>         ./node/node5/meminfo
>         ./node/node5/cpumap
> 
>         # cat ./node/node0/distance
>         10 20 64 42 42 22
> 
>         # cat node/*/distance
>         10 20 64 42 42 22
>         20 10 42 22 64 84
>         64 42 10 20 22 42
>         42 22 20 10 42 62
>         42 64 22 42 10 20
>         22 84 42 62 20 10

Apparently I'm easily confused, but node_distance() {near end}
seems to evaluate to either 10 or 20 (only), so what are
all of these other numbers here?

And how many nodes are in this example?
Maybe 6, numbered 0 thru 5?  Plz correct this guess....

> Index: linux/drivers/base/node.c
> ===================================================================
> --- linux.orig/drivers/base/node.c	2004-11-05 08:34:42.461312000 -0600
> +++ linux/drivers/base/node.c	2004-11-05 15:56:23.345662000 -0600
> @@ -111,6 +111,24 @@ static ssize_t node_read_numastat(struct
>  }
>  static SYSDEV_ATTR(numastat, S_IRUGO, node_read_numastat, NULL);
>  
> +static ssize_t node_read_distance(struct sys_device * dev, char * buf)
> +{
> +	int nid = dev->id;
> +	int len = 0;
> +	int i;
> +
> +	/* buf currently PAGE_SIZE, need ~4 chars per node */
> +	BUILD_BUG_ON(NR_NODES*4 > PAGE_SIZE/2);
> +
> +	for (i = 0; i < numnodes; i++)
> +		len += sprintf(buf + len, "%s%d", i ? " " : "", node_distance(nid, i));
> +		
> +	len += sprintf(buf + len, "\n");
> +	return len;
> +}
> +static SYSDEV_ATTR(distance, S_IRUGO, node_read_distance, NULL);
> +
> +
> Index: linux/include/linux/topology.h
> ===================================================================
> --- linux.orig/include/linux/topology.h	2004-11-05 08:34:57.492932000 -0600
> +++ linux/include/linux/topology.h	2004-11-23 10:03:26.700821978 -0600
> @@ -55,7 +55,10 @@ static inline int __next_node_with_cpus(
>  	for (node = 0; node < numnodes; node = __next_node_with_cpus(node))
>  
>  #ifndef node_distance
> -#define node_distance(from,to)	((from) != (to))
> +/* Conform to ACPI 2.0 SLIT distance definitions */
> +#define LOCAL_DISTANCE		10
> +#define REMOTE_DISTANCE		20
> +#define node_distance(from,to)	((from) == (to) ? LOCAL_DISTANCE : REMOTE_DISTANCE)
Please add a space after "from,".

>  #endif

-- 
~Randy

Re: [PATCH] - Externel SLIT table information thru sysfs

[Greg KH]

On Wed, Nov 24, 2004 at 06:04:03PM -0800, Randy.Dunlap wrote:
> Jack Steiner wrote:
> >The ACPI SLIT table provides useful information on internode distances.
> >Here is a patch to externalize the SLIT information. 
> >
> >For example:
> >
> >        # cd /sys/devices/system
> >        # find .
> >        ./node
> >        ./node/node5
> >        ./node/node5/distance
> >        ./node/node5/numastat
> >        ./node/node5/meminfo
> >        ./node/node5/cpumap
> >
> >        # cat ./node/node0/distance
> >        10 20 64 42 42 22

Care to turn this into a one value per file implementation instead of
this?  That will make it easier to determine exactly what the data in
each file is, and follow the sysfs rules.

thanks,

greg k-h

Re: [PATCH] - Externel SLIT table information thru sysfs

[Jack Steiner]

On Wed, Nov 24, 2004 at 06:04:03PM -0800, Randy.Dunlap wrote:
> Jack Steiner wrote:
> >The ACPI SLIT table provides useful information on internode distances.
> >Here is a patch to externalize the SLIT information. 
> >
> >For example:
> >
> >        # cd /sys/devices/system
> >        # find .
> >        ./node
> >        ./node/node5
> >        ./node/node5/distance
> >        ./node/node5/numastat
> >        ./node/node5/meminfo
> >        ./node/node5/cpumap
> >
> >        # cat ./node/node0/distance
> >        10 20 64 42 42 22
> >
> >        # cat node/*/distance
> >        10 20 64 42 42 22
> >        20 10 42 22 64 84
> >        64 42 10 20 22 42
> >        42 22 20 10 42 62
> >        42 64 22 42 10 20
> >        22 84 42 62 20 10
> 
> Apparently I'm easily confused, but node_distance() {near end}
> seems to evaluate to either 10 or 20 (only), so what are
> all of these other numbers here?

On systems that provide the ACPI SLIT table, the distances come from
the SLIT table.  (See the ACPI spec for the full definition).
The example above is from the ACPI SLIT table of a 6-node SGI Altix 
system and the table is provided by the BIOS.

If the BIOS does not provide a SLIT table, the default distance is 10 for
local & 20 for remote. The value of 10 conforms to the spec for
local distance, 20 is arbitrary but indicates further than local.

> 
> And how many nodes are in this example?
> Maybe 6, numbered 0 thru 5?  Plz correct this guess....

Good guess.



> 
> >Index: linux/drivers/base/node.c
> >===================================================================
> >--- linux.orig/drivers/base/node.c	2004-11-05 08:34:42.461312000 -0600
> >+++ linux/drivers/base/node.c	2004-11-05 15:56:23.345662000 -0600
> >@@ -111,6 +111,24 @@ static ssize_t node_read_numastat(struct
> > }
> > static SYSDEV_ATTR(numastat, S_IRUGO, node_read_numastat, NULL);
> > 
> >+static ssize_t node_read_distance(struct sys_device * dev, char * buf)
> >+{
> >+	int nid = dev->id;
> >+	int len = 0;
> >+	int i;
> >+
> >+	/* buf currently PAGE_SIZE, need ~4 chars per node */
> >+	BUILD_BUG_ON(NR_NODES*4 > PAGE_SIZE/2);
> >+
> >+	for (i = 0; i < numnodes; i++)
> >+		len += sprintf(buf + len, "%s%d", i ? " " : "", 
> >node_distance(nid, i));
> >+		
> >+	len += sprintf(buf + len, "\n");
> >+	return len;
> >+}
> >+static SYSDEV_ATTR(distance, S_IRUGO, node_read_distance, NULL);
> >+
> >+
> >Index: linux/include/linux/topology.h
> >===================================================================
> >--- linux.orig/include/linux/topology.h	2004-11-05 
> >08:34:57.492932000 -0600
> >+++ linux/include/linux/topology.h	2004-11-23 10:03:26.700821978 -0600
> >@@ -55,7 +55,10 @@ static inline int __next_node_with_cpus(
> > 	for (node = 0; node < numnodes; node = __next_node_with_cpus(node))
> > 
> > #ifndef node_distance
> >-#define node_distance(from,to)	((from) != (to))
> >+/* Conform to ACPI 2.0 SLIT distance definitions */
> >+#define LOCAL_DISTANCE		10
> >+#define REMOTE_DISTANCE		20
> >+#define node_distance(from,to)	((from) == (to) ? LOCAL_DISTANCE : 
> >REMOTE_DISTANCE)
> Please add a space after "from,".
> 
> > #endif
> 
> -- 
> ~Randy

-- 
Thanks

Jack Steiner (steiner@sgi.com)          651-683-5302
Principal Engineer                      SGI - Silicon Graphics, Inc.

Re: [PATCH] - Externel SLIT table information thru sysfs

[Jack Steiner]

(resend - first bounced)

On Wed, Nov 24, 2004 at 06:04:03PM -0800, Randy.Dunlap wrote:
> Jack Steiner wrote:
> >The ACPI SLIT table provides useful information on internode distances.
> >Here is a patch to externalize the SLIT information. 
> >
> >For example:
> >
> >        # cd /sys/devices/system
> >        # find .
> >        ./node
> >        ./node/node5
> >        ./node/node5/distance
> >        ./node/node5/numastat
> >        ./node/node5/meminfo
> >        ./node/node5/cpumap
> >
> >        # cat ./node/node0/distance
> >        10 20 64 42 42 22
> >
> >        # cat node/*/distance
> >        10 20 64 42 42 22
> >        20 10 42 22 64 84
> >        64 42 10 20 22 42
> >        42 22 20 10 42 62
> >        42 64 22 42 10 20
> >        22 84 42 62 20 10
> 
> Apparently I'm easily confused, but node_distance() {near end}
> seems to evaluate to either 10 or 20 (only), so what are
> all of these other numbers here?

On systems that provide the ACPI SLIT table, the distances come from
the SLIT table.  (See the ACPI spec for the full definition).
The example above is from the ACPI SLIT table of a 6-node SGI Altix 
system and the table is provided by the BIOS.

If the BIOS does not provide a SLIT table, the default distance is 10 for
local & 20 for remote. The value of 10 conforms to the spec for
local distance, 20 is arbitrary but indicates further than local.

> 
> And how many nodes are in this example?
> Maybe 6, numbered 0 thru 5?  Plz correct this guess....

Good guess.



> 
> >Index: linux/drivers/base/node.c
> >===================================================================
> >--- linux.orig/drivers/base/node.c	2004-11-05 08:34:42.461312000 -0600
> >+++ linux/drivers/base/node.c	2004-11-05 15:56:23.345662000 -0600
> >@@ -111,6 +111,24 @@ static ssize_t node_read_numastat(struct
> > }
> > static SYSDEV_ATTR(numastat, S_IRUGO, node_read_numastat, NULL);
> > 
> >+static ssize_t node_read_distance(struct sys_device * dev, char * buf)
> >+{
> >+	int nid = dev->id;
> >+	int len = 0;
> >+	int i;
> >+
> >+	/* buf currently PAGE_SIZE, need ~4 chars per node */
> >+	BUILD_BUG_ON(NR_NODES*4 > PAGE_SIZE/2);
> >+
> >+	for (i = 0; i < numnodes; i++)
> >+		len += sprintf(buf + len, "%s%d", i ? " " : "", 
> >node_distance(nid, i));
> >+		
> >+	len += sprintf(buf + len, "\n");
> >+	return len;
> >+}
> >+static SYSDEV_ATTR(distance, S_IRUGO, node_read_distance, NULL);
> >+
> >+
> >Index: linux/include/linux/topology.h
> >===================================================================
> >--- linux.orig/include/linux/topology.h	2004-11-05 
> >08:34:57.492932000 -0600
> >+++ linux/include/linux/topology.h	2004-11-23 10:03:26.700821978 -0600
> >@@ -55,7 +55,10 @@ static inline int __next_node_with_cpus(
> > 	for (node = 0; node < numnodes; node = __next_node_with_cpus(node))
> > 
> > #ifndef node_distance
> >-#define node_distance(from,to)	((from) != (to))
> >+/* Conform to ACPI 2.0 SLIT distance definitions */
> >+#define LOCAL_DISTANCE		10
> >+#define REMOTE_DISTANCE		20
> >+#define node_distance(from,to)	((from) == (to) ? LOCAL_DISTANCE : 
> >REMOTE_DISTANCE)
> Please add a space after "from,".
> 
> > #endif
> 
> -- 
> ~Randy

-- 
Thanks

Jack Steiner (steiner@sgi.com)          651-683-5302
Principal Engineer                      SGI - Silicon Graphics, Inc.

Re: [PATCH] - Externel SLIT table information thru sysfs

[Andi Kleen]

Greg KH <greg@kroah.com> writes:

> > >        10 20 64 42 42 22
> 
> Care to turn this into a one value per file implementation instead of
> this?  That will make it easier to determine exactly what the data in
> each file is, and follow the sysfs rules.

No, we discussed this already earlier (please check the thread)

One file for each would lead to excessive memory consumption on large
systems, and it would make it much more costly for programs to read
this information.

Also the sysfs documentation allows for this case where it makes
sense even.

-Andi