Re: [patch] Summit support for pcibus <-> cpumask topology

Re: [patch] Summit support for pcibus <-> cpumask topology

[James Bottomley]

> This patch adds a new file, arch/i386/kernel/summit.c, for 
> summit-specific code.  Adds some structures to mach_mpparse.h.  Also 
> adds a hook in setup_arch() to dig out the PCI info, and stores it in 
> the mp_bus_id_to_node[] array, where it can be read by the topology 
> functions.

Wouldn't this file be better in arch/i386/mach-summit in keeping with
all the other subarch stuff?

While you're creating a separate file for summit, could you move the
summit specific variables (mpparse.c:x86_summit is the only one, I
think) into it so we can clean all the summit references out of the main
line?

Thanks,

James

Re: [patch] Summit support for pcibus <-> cpumask topology [1/2]

[Matthew Dobson]

[-- Attachment #1: Type: text/plain, Size: 1047 bytes --]

James Bottomley wrote:
> 
>>This patch adds a new file, arch/i386/kernel/summit.c, for 
>>summit-specific code.  Adds some structures to mach_mpparse.h.  Also 
>>adds a hook in setup_arch() to dig out the PCI info, and stores it in 
>>the mp_bus_id_to_node[] array, where it can be read by the topology 
>>functions.
> 
> 
> Wouldn't this file be better in arch/i386/mach-summit in keeping with
> all the other subarch stuff?
> 
> While you're creating a separate file for summit, could you move the
> summit specific variables (mpparse.c:x86_summit is the only one, I
> think) into it so we can clean all the summit references out of the main
> line?
> 
> Thanks,
> 
> James

Ok...  Split into two patches now.  Patch 1 moves summit stuff into 
subarch.  Just creates the directory and populates it with setup.c, 
topology.c, and the Makefile from mach-default.  Also creates a summit.c 
which only has the x86_summit variable you referred to.

Patch 2 adds all pcimapping code, but adds it in the subarch dir, not 
i386/kernel/.

Cheers!

-Matt

[-- Attachment #2: summit_subarch-2.5.64.patch --]
[-- Type: text/plain, Size: 8168 bytes --]

diff -Nur --exclude-from=/usr/src/.dontdiff linux-2.5.64-vanilla/arch/i386/Makefile linux-2.5.64-summit_subarch/arch/i386/Makefile
--- linux-2.5.64-vanilla/arch/i386/Makefile	Tue Mar  4 19:29:16 2003
+++ linux-2.5.64-summit_subarch/arch/i386/Makefile	Fri Mar 14 17:32:48 2003
@@ -71,7 +71,7 @@
 
 #Summit subarch support
 mflags-$(CONFIG_X86_SUMMIT) := -Iinclude/asm-i386/mach-summit
-mcore-$(CONFIG_X86_SUMMIT)  := mach-default
+mcore-$(CONFIG_X86_SUMMIT)  := mach-summit
 
 # default subarch .h files
 mflags-y += -Iinclude/asm-i386/mach-default
diff -Nur --exclude-from=/usr/src/.dontdiff linux-2.5.64-vanilla/arch/i386/kernel/mpparse.c linux-2.5.64-summit_subarch/arch/i386/kernel/mpparse.c
--- linux-2.5.64-vanilla/arch/i386/kernel/mpparse.c	Tue Mar  4 19:29:05 2003
+++ linux-2.5.64-summit_subarch/arch/i386/kernel/mpparse.c	Fri Mar 14 17:32:48 2003
@@ -72,7 +72,6 @@
 /* Bitmask of physically existing CPUs */
 unsigned long phys_cpu_present_map;
 
-int x86_summit = 0;
 u8 bios_cpu_apicid[NR_CPUS] = { [0 ... NR_CPUS-1] = BAD_APICID };
 
 /*
diff -Nur --exclude-from=/usr/src/.dontdiff linux-2.5.64-vanilla/arch/i386/mach-summit/Makefile linux-2.5.64-summit_subarch/arch/i386/mach-summit/Makefile
--- linux-2.5.64-vanilla/arch/i386/mach-summit/Makefile	Wed Dec 31 16:00:00 1969
+++ linux-2.5.64-summit_subarch/arch/i386/mach-summit/Makefile	Fri Mar 14 17:32:48 2003
@@ -0,0 +1,7 @@
+#
+# Makefile for the linux kernel.
+#
+
+EXTRA_CFLAGS	+= -I../kernel
+
+obj-y				:= setup.o topology.o summit.o
diff -Nur --exclude-from=/usr/src/.dontdiff linux-2.5.64-vanilla/arch/i386/mach-summit/setup.c linux-2.5.64-summit_subarch/arch/i386/mach-summit/setup.c
--- linux-2.5.64-vanilla/arch/i386/mach-summit/setup.c	Wed Dec 31 16:00:00 1969
+++ linux-2.5.64-summit_subarch/arch/i386/mach-summit/setup.c	Fri Mar 14 17:32:48 2003
@@ -0,0 +1,104 @@
+/*
+ *	Machine specific setup for generic
+ */
+
+#include <linux/config.h>
+#include <linux/smp.h>
+#include <linux/init.h>
+#include <linux/irq.h>
+#include <linux/interrupt.h>
+#include <asm/arch_hooks.h>
+
+/**
+ * pre_intr_init_hook - initialisation prior to setting up interrupt vectors
+ *
+ * Description:
+ *	Perform any necessary interrupt initialisation prior to setting up
+ *	the "ordinary" interrupt call gates.  For legacy reasons, the ISA
+ *	interrupts should be initialised here if the machine emulates a PC
+ *	in any way.
+ **/
+void __init pre_intr_init_hook(void)
+{
+	init_ISA_irqs();
+}
+
+/*
+ * IRQ2 is cascade interrupt to second interrupt controller
+ */
+static struct irqaction irq2 = { no_action, 0, 0, "cascade", NULL, NULL};
+
+/**
+ * intr_init_hook - post gate setup interrupt initialisation
+ *
+ * Description:
+ *	Fill in any interrupts that may have been left out by the general
+ *	init_IRQ() routine.  interrupts having to do with the machine rather
+ *	than the devices on the I/O bus (like APIC interrupts in intel MP
+ *	systems) are started here.
+ **/
+void __init intr_init_hook(void)
+{
+#ifdef CONFIG_X86_LOCAL_APIC
+	apic_intr_init();
+#endif
+
+	setup_irq(2, &irq2);
+}
+
+/**
+ * pre_setup_arch_hook - hook called prior to any setup_arch() execution
+ *
+ * Description:
+ *	generally used to activate any machine specific identification
+ *	routines that may be needed before setup_arch() runs.  On VISWS
+ *	this is used to get the board revision and type.
+ **/
+void __init pre_setup_arch_hook(void)
+{
+}
+
+/**
+ * trap_init_hook - initialise system specific traps
+ *
+ * Description:
+ *	Called as the final act of trap_init().  Used in VISWS to initialise
+ *	the various board specific APIC traps.
+ **/
+void __init trap_init_hook(void)
+{
+}
+
+static struct irqaction irq0  = { timer_interrupt, SA_INTERRUPT, 0, "timer", NULL, NULL};
+
+/**
+ * time_init_hook - do any specific initialisations for the system timer.
+ *
+ * Description:
+ *	Must plug the system timer interrupt source at HZ into the IRQ listed
+ *	in irq_vectors.h:TIMER_IRQ
+ **/
+void __init time_init_hook(void)
+{
+	setup_irq(0, &irq0);
+}
+
+#ifdef CONFIG_MCA
+/**
+ * mca_nmi_hook - hook into MCA specific NMI chain
+ *
+ * Description:
+ *	The MCA (Microchannel Arcitecture) has an NMI chain for NMI sources
+ *	along the MCA bus.  Use this to hook into that chain if you will need
+ *	it.
+ **/
+void __init mca_nmi_hook(void)
+{
+	/* If I recall correctly, there's a whole bunch of other things that
+	 * we can do to check for NMI problems, but that's all I know about
+	 * at the moment.
+	 */
+
+	printk("NMI generated from unknown source!\n");
+}
+#endif
diff -Nur --exclude-from=/usr/src/.dontdiff linux-2.5.64-vanilla/arch/i386/mach-summit/summit.c linux-2.5.64-summit_subarch/arch/i386/mach-summit/summit.c
--- linux-2.5.64-vanilla/arch/i386/mach-summit/summit.c	Wed Dec 31 16:00:00 1969
+++ linux-2.5.64-summit_subarch/arch/i386/mach-summit/summit.c	Fri Mar 14 17:32:48 2003
@@ -0,0 +1,29 @@
+/*
+ * arch/i386/kernel/summit.c - IBM Summit-Specific Code
+ *
+ * Written By: Matthew Dobson, IBM Corporation
+ *
+ * Copyright (c) 2003 IBM Corp.
+ *
+ * All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or (at
+ * your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or
+ * NON INFRINGEMENT.  See the GNU General Public License for more
+ * details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * Send feedback to <colpatch@us.ibm.com>
+ *
+ */
+
+int x86_summit = 0;
diff -Nur --exclude-from=/usr/src/.dontdiff linux-2.5.64-vanilla/arch/i386/mach-summit/topology.c linux-2.5.64-summit_subarch/arch/i386/mach-summit/topology.c
--- linux-2.5.64-vanilla/arch/i386/mach-summit/topology.c	Wed Dec 31 16:00:00 1969
+++ linux-2.5.64-summit_subarch/arch/i386/mach-summit/topology.c	Fri Mar 14 17:32:48 2003
@@ -0,0 +1,68 @@
+/*
+ * arch/i386/mach-generic/topology.c - Populate driverfs with topology information
+ *
+ * Written by: Matthew Dobson, IBM Corporation
+ * Original Code: Paul Dorwin, IBM Corporation, Patrick Mochel, OSDL
+ *
+ * Copyright (C) 2002, IBM Corp.
+ *
+ * All rights reserved.          
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or
+ * NON INFRINGEMENT.  See the GNU General Public License for more
+ * details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * Send feedback to <colpatch@us.ibm.com>
+ */
+#include <linux/init.h>
+#include <linux/smp.h>
+#include <asm/cpu.h>
+
+struct i386_cpu cpu_devices[NR_CPUS];
+
+#ifdef CONFIG_NUMA
+#include <linux/mmzone.h>
+#include <asm/node.h>
+#include <asm/memblk.h>
+
+struct i386_node node_devices[MAX_NUMNODES];
+struct i386_memblk memblk_devices[MAX_NR_MEMBLKS];
+
+static int __init topology_init(void)
+{
+	int i;
+
+	for (i = 0; i < num_online_nodes(); i++)
+		arch_register_node(i);
+	for (i = 0; i < NR_CPUS; i++)
+		if (cpu_possible(i)) arch_register_cpu(i);
+	for (i = 0; i < num_online_memblks(); i++)
+		arch_register_memblk(i);
+	return 0;
+}
+
+#else /* !CONFIG_NUMA */
+
+static int __init topology_init(void)
+{
+	int i;
+
+	for (i = 0; i < NR_CPUS; i++)
+		if (cpu_possible(i)) arch_register_cpu(i);
+	return 0;
+}
+
+#endif /* CONFIG_NUMA */
+
+subsys_initcall(topology_init);

Re: [patch] Summit support for pcibus <-> cpumask topology [2/2]

[Matthew Dobson]

[-- Attachment #1: Type: text/plain, Size: 1040 bytes --]

James Bottomley wrote:
>> This patch adds a new file, arch/i386/kernel/summit.c, for 
>> summit-specific code.  Adds some structures to mach_mpparse.h.  Also 
>> adds a hook in setup_arch() to dig out the PCI info, and stores it in 
>> the mp_bus_id_to_node[] array, where it can be read by the topology 
>> functions.
>
>
> Wouldn't this file be better in arch/i386/mach-summit in keeping with
> all the other subarch stuff?
>
> While you're creating a separate file for summit, could you move the
> summit specific variables (mpparse.c:x86_summit is the only one, I
> think) into it so we can clean all the summit references out of the main
> line?
>
> Thanks,
>
> James

Ok...  Split into two patches now.  Patch 1 moves summit stuff into
subarch.  Just creates the directory and populates it with setup.c,
topology.c, and the Makefile from mach-default.  Also creates a summit.c
which only has the x86_summit variable you referred to.

Patch 2 adds all pcimapping code, but adds it in the subarch dir, not
i386/kernel/.

Cheers!

-Matt

[-- Attachment #2: summit_pcimap-2.5.64-patch --]
[-- Type: text/plain, Size: 10689 bytes --]

diff -Nur --exclude-from=/usr/src/.dontdiff linux-2.5.64-summit_subarch/arch/i386/kernel/setup.c linux-2.5.64-summit_pcimap/arch/i386/kernel/setup.c
--- linux-2.5.64-summit_subarch/arch/i386/kernel/setup.c	Tue Mar  4 19:29:18 2003
+++ linux-2.5.64-summit_pcimap/arch/i386/kernel/setup.c	Fri Mar 14 09:56:21 2003
@@ -918,6 +918,9 @@
 	if (smp_found_config)
 		get_smp_config();
 #endif
+#ifdef CONFIG_X86_SUMMIT
+	setup_summit();
+#endif
 
 	register_memory(max_low_pfn);
 
diff -Nur --exclude-from=/usr/src/.dontdiff linux-2.5.64-summit_subarch/arch/i386/mach-summit/summit.c linux-2.5.64-summit_pcimap/arch/i386/mach-summit/summit.c
--- linux-2.5.64-summit_subarch/arch/i386/mach-summit/summit.c	Fri Mar 14 17:32:48 2003
+++ linux-2.5.64-summit_pcimap/arch/i386/mach-summit/summit.c	Fri Mar 14 17:09:02 2003
@@ -26,4 +26,139 @@
  *
  */
 
+#include <linux/mm.h>
+#include <linux/init.h>
+#include <asm/io.h>
+#include <mach_mpparse.h>
+
 int x86_summit = 0;
+
+static void __init setup_pci_node_map_for_wpeg(int wpeg_num, struct rio_table_hdr *rth, 
+		struct scal_detail **scal_nodes, struct rio_detail **rio_nodes){
+	int twst_num = 0, node = 0, first_bus = 0;
+	int i, bus, num_busses;
+
+	for(i = 0; i < rth->num_rio_dev; i++){
+		if (rio_nodes[i]->node_id == rio_nodes[wpeg_num]->owner_id){
+			twst_num = rio_nodes[i]->owner_id;
+			break;
+		}
+	}
+	if (i == rth->num_rio_dev){
+		printk("%s: Couldn't find owner Cyclone for Winnipeg!\n", __FUNCTION__);
+		return;
+	}
+
+	for(i = 0; i < rth->num_scal_dev; i++){
+		if (scal_nodes[i]->node_id == twst_num){
+			node = scal_nodes[i]->node_id;
+			break;
+		}
+	}
+	if (i == rth->num_scal_dev){
+		printk("%s: Couldn't find owner Twister for Cyclone!\n", __FUNCTION__);
+		return;
+	}
+
+	switch (rio_nodes[wpeg_num]->type){
+	case CompatWPEG:
+		/* The Compatability Winnipeg controls the legacy busses
+		   (busses 0 & 1), the 66MHz PCI bus [2 slots] (bus 2), 
+		   and the "extra" busses in case a PCI-PCI bridge card is 
+		   used in either slot (busses 3 & 4): total 5 busses. */
+		num_busses = 5;
+		/* The BIOS numbers the busses starting at 1, and in a 
+		   slightly wierd manner.  You'll have to trust that 
+		   the math used below to determine the number of the 
+		   first bus works. */
+		first_bus = (rio_nodes[wpeg_num]->first_slot - 1) * 2;
+		break;
+	case AltWPEG:
+		/* The Alternate/Secondary Winnipeg controls the 1st 133MHz 
+		   bus [1 slot] & its "extra" bus (busses 0 & 1), the 2nd 
+		   133MHz bus [1 slot] & its "extra" bus (busses 2 & 3), the 
+		   100MHz bus [2 slots] (bus 4), and the "extra" busses for 
+		   the 2 100MHz slots (busses 5 & 6): total 7 busses. */
+		num_busses = 7;
+		first_bus = (rio_nodes[wpeg_num]->first_slot * 2) - 1;
+		break;
+	case LookOutAWPEG:
+	case LookOutBWPEG:
+		printk("%s: LookOut Winnipegs not supported yet!\n", __FUNCTION__);
+		return;
+	default:
+		printk("%s: Unsupported Winnipeg type!\n", __FUNCTION__);
+		return;
+	}
+
+	for(bus = first_bus; bus < first_bus + num_busses; bus++)
+		mp_bus_id_to_node[bus] = node;
+}
+
+static void __init build_detail_arrays(struct rio_table_hdr *rth,
+		struct scal_detail **sd, struct rio_detail **rd){
+	unsigned long ptr;
+	int i, scal_detail_size, rio_detail_size;
+
+	switch (rth->version){
+	default:
+		printk("%s: Bad Rio Grande Table Version: %d\n", __FUNCTION__, rth->version);
+		/* Fall through to default to version 2 spec */
+	case 2:
+		scal_detail_size = 11;
+		rio_detail_size = 13;
+		break;
+	case 3:
+		scal_detail_size = 12;
+		rio_detail_size = 15;
+		break;
+	}
+
+	ptr = (unsigned long)rth + 3;
+	for(i = 0; i < rth->num_scal_dev; i++)
+		sd[i] = (struct scal_detail *)(ptr + (scal_detail_size * i));
+
+	ptr += scal_detail_size * rth->num_scal_dev;
+	for(i = 0; i < rth->num_rio_dev; i++)
+		rd[i] = (struct rio_detail *)(ptr + (rio_detail_size * i));
+}
+
+void __init setup_summit(void)
+{
+	struct rio_table_hdr	*rio_table_hdr = NULL;
+	struct scal_detail	*scal_devs[MAX_NUMNODES];
+	struct rio_detail	*rio_devs[MAX_NUMNODES*2];
+	unsigned long		ptr;
+	unsigned short		offset;
+	int			i;
+
+	memset(mp_bus_id_to_node, -1, sizeof(mp_bus_id_to_node));
+
+	/* The pointer to the EBDA is stored in the word @ phys 0x40E(40:0E) */
+	ptr = *(unsigned short *)phys_to_virt(0x40Eul);
+	ptr = (unsigned long)phys_to_virt(ptr << 4);
+
+	offset = 0x180;
+	while (offset){
+		/* The block id is stored in the 2nd word */
+		if (*((unsigned short *)(ptr + offset + 2)) == 0x4752){
+			/* set the pointer past the offset & block id */
+			rio_table_hdr = (struct rio_table_hdr *)(ptr + offset + 4);
+			break;
+		}
+		/* The next offset is stored in the 1st word.  0 means no more */
+		offset = *((unsigned short *)(ptr + offset));
+	}
+	if (!rio_table_hdr){
+		printk("%s: Unable to locate Rio Grande Table in EBDA - bailing!\n", __FUNCTION__);
+		return;
+	}
+
+	/* Deal with the ugly version 2/3 pointer arithmetic */
+	build_detail_arrays(rio_table_hdr, scal_devs, rio_devs);
+
+	for(i = 0; i < rio_table_hdr->num_rio_dev; i++)
+		if (is_WPEG(rio_devs[i]->type))
+			/* It's a Winnipeg, it's got PCI Busses */
+			setup_pci_node_map_for_wpeg(i, rio_table_hdr, scal_devs, rio_devs);
+}
diff -Nur --exclude-from=/usr/src/.dontdiff linux-2.5.64-summit_subarch/include/asm-i386/mach-summit/mach_mpparse.h linux-2.5.64-summit_pcimap/include/asm-i386/mach-summit/mach_mpparse.h
--- linux-2.5.64-summit_subarch/include/asm-i386/mach-summit/mach_mpparse.h	Tue Mar  4 19:29:17 2003
+++ linux-2.5.64-summit_pcimap/include/asm-i386/mach-summit/mach_mpparse.h	Fri Mar 14 12:20:38 2003
@@ -1,6 +1,8 @@
 #ifndef __ASM_MACH_MPPARSE_H
 #define __ASM_MACH_MPPARSE_H
 
+#include <mach_apic.h>
+
 extern int use_cyclone;
 
 static inline void mpc_oem_bus_info(struct mpc_config_bus *m, char *name, 
@@ -33,4 +35,71 @@
 		use_cyclone = 1; /*enable cyclone-timer*/
 	}
 }
+
+struct rio_table_hdr {
+	unsigned char version;      /* Version number of this data structure           */
+	                            /* Version 3 adds chassis_num & WP_index           */
+	unsigned char num_scal_dev; /* # of Scalability devices (Twisters for Vigil)   */
+	unsigned char num_rio_dev;  /* # of RIO I/O devices (Cyclones and Winnipegs)   */
+} __attribute__((packed));
+
+struct scal_detail {
+	unsigned char node_id;      /* Scalability Node ID                             */
+	unsigned long CBAR;         /* Address of 1MB register space                   */
+	unsigned char port0node;    /* Node ID port connected to: 0xFF=None            */
+	unsigned char port0port;    /* Port num port connected to: 0,1,2, or 0xFF=None */
+	unsigned char port1node;    /* Node ID port connected to: 0xFF = None          */
+	unsigned char port1port;    /* Port num port connected to: 0,1,2, or 0xFF=None */
+	unsigned char port2node;    /* Node ID port connected to: 0xFF = None          */
+	unsigned char port2port;    /* Port num port connected to: 0,1,2, or 0xFF=None */
+	unsigned char chassis_num;  /* 1 based Chassis number (1 = boot node)          */
+} __attribute__((packed));
+
+struct rio_detail {
+	unsigned char node_id;      /* RIO Node ID                                     */
+	unsigned long BBAR;         /* Address of 1MB register space                   */
+	unsigned char type;         /* Type of device                                  */
+	unsigned char owner_id;     /* For WPEG: Node ID of Cyclone that owns this WPEG*/
+	                            /* For CYC:  Node ID of Twister that owns this CYC */
+	unsigned char port0node;    /* Node ID port connected to: 0xFF=None            */
+	unsigned char port0port;    /* Port num port connected to: 0,1,2, or 0xFF=None */
+	unsigned char port1node;    /* Node ID port connected to: 0xFF=None            */
+	unsigned char port1port;    /* Port num port connected to: 0,1,2, or 0xFF=None */
+	unsigned char first_slot;   /* For WPEG: Lowest slot number below this WPEG    */
+	                            /* For CYC:  0                                     */
+	unsigned char status;       /* For WPEG: Bit 0 = 1 : the XAPIC is used         */
+	                            /*                 = 0 : the XAPIC is not used, ie:*/
+	                            /*                     ints fwded to another XAPIC */
+	                            /*           Bits1:7 Reserved                      */
+	                            /* For CYC:  Bits0:7 Reserved                      */
+	unsigned char WP_index;     /* For WPEG: WPEG instance index - lower ones have */
+	                            /*           lower slot numbers/PCI bus numbers    */
+	                            /* For CYC:  No meaning                            */
+	unsigned char chassis_num;  /* 1 based Chassis number                          */
+	                            /* For LookOut WPEGs this field indicates the      */
+	                            /* Expansion Chassis #, enumerated from Boot       */
+	                            /* Node WPEG external port, then Boot Node CYC     */
+	                            /* external port, then Next Vigil chassis WPEG     */
+	                            /* external port, etc.                             */
+	                            /* Shared Lookouts have only 1 chassis number (the */
+	                            /* first one assigned)                             */
+} __attribute__((packed));
+
+
+typedef enum {
+	CompatTwister = 0,  /* Compatibility Twister               */
+	AltTwister    = 1,  /* Alternate Twister of internal 8-way */
+	CompatCyclone = 2,  /* Compatibility Cyclone               */
+	AltCyclone    = 3,  /* Alternate Cyclone of internal 8-way */
+	CompatWPEG    = 4,  /* Compatibility WPEG                  */
+	AltWPEG       = 5,  /* Second Planar WPEG                  */
+	LookOutAWPEG  = 6,  /* LookOut WPEG                        */
+	LookOutBWPEG  = 7,  /* LookOut WPEG                        */
+} node_type;
+
+static inline int is_WPEG(node_type type){
+	return (type == CompatWPEG || type == AltWPEG ||
+		type == LookOutAWPEG || type == LookOutBWPEG);
+}
+
 #endif /* __ASM_MACH_MPPARSE_H */
diff -Nur --exclude-from=/usr/src/.dontdiff linux-2.5.64-summit_subarch/include/asm-i386/mpspec.h linux-2.5.64-summit_pcimap/include/asm-i386/mpspec.h
--- linux-2.5.64-summit_subarch/include/asm-i386/mpspec.h	Tue Mar  4 19:28:53 2003
+++ linux-2.5.64-summit_pcimap/include/asm-i386/mpspec.h	Fri Mar 14 09:58:46 2003
@@ -222,6 +222,10 @@
 extern int pic_mode;
 extern int using_apic_timer;
 
+#ifdef CONFIG_X86_SUMMIT
+extern void setup_summit (void);
+#endif
+
 #ifdef CONFIG_ACPI_BOOT
 extern void mp_register_lapic (u8 id, u8 enabled);
 extern void mp_register_lapic_address (u64 address);

[patch] NUMAQ subarchification

[Matthew Dobson]

[-- Attachment #1: Type: text/plain, Size: 896 bytes --]

James Bottomley wrote:
>> This patch adds a new file, arch/i386/kernel/summit.c, for 
>> summit-specific code.  Adds some structures to mach_mpparse.h.  Also 
>> adds a hook in setup_arch() to dig out the PCI info, and stores it in 
>> the mp_bus_id_to_node[] array, where it can be read by the topology 
>> functions.
>
> Wouldn't this file be better in arch/i386/mach-summit in keeping with
> all the other subarch stuff?
>
> While you're creating a separate file for summit, could you move the
> summit specific variables (mpparse.c:x86_summit is the only one, I
> think) into it so we can clean all the summit references out of the main
> line?
>
> Thanks,
>
> James

While I was at it, I subarchified (I'll cc Websters with the new word ;) 
numaq as well.  Copied mach-defaults setup.c, topology.c, and Makefile. 
  Moved arch/i386/kernel/numaq.c into mach-numaq.  Compiles.

Cheers!

-Matt

[-- Attachment #2: numaq_subarch-2.5.64.patch --]
[-- Type: text/plain, Size: 15307 bytes --]

diff -Nur --exclude-from=/usr/src/.dontdiff linux-2.5.64-vanilla/arch/i386/Makefile linux-2.5.64-numaq_subarch/arch/i386/Makefile
--- linux-2.5.64-vanilla/arch/i386/Makefile	Tue Mar  4 19:29:16 2003
+++ linux-2.5.64-numaq_subarch/arch/i386/Makefile	Fri Mar 14 15:28:53 2003
@@ -61,14 +61,14 @@
 mflags-$(CONFIG_X86_VISWS)	:= -Iinclude/asm-i386/mach-visws
 mcore-$(CONFIG_X86_VISWS)	:= mach-visws
 
-# NUMAQ subarch support
-mflags-$(CONFIG_X86_NUMAQ)	:= -Iinclude/asm-i386/mach-numaq
-mcore-$(CONFIG_X86_NUMAQ)	:= mach-default
-
 # BIGSMP subarch support
 mflags-$(CONFIG_X86_BIGSMP)	:= -Iinclude/asm-i386/mach-bigsmp
 mcore-$(CONFIG_X86_BIGSMP)	:= mach-default
 
+# NUMAQ subarch support
+mflags-$(CONFIG_X86_NUMAQ)	:= -Iinclude/asm-i386/mach-numaq
+mcore-$(CONFIG_X86_NUMAQ)	:= mach-numaq
+
 #Summit subarch support
 mflags-$(CONFIG_X86_SUMMIT) := -Iinclude/asm-i386/mach-summit
 mcore-$(CONFIG_X86_SUMMIT)  := mach-default
diff -Nur --exclude-from=/usr/src/.dontdiff linux-2.5.64-vanilla/arch/i386/kernel/Makefile linux-2.5.64-numaq_subarch/arch/i386/kernel/Makefile
--- linux-2.5.64-vanilla/arch/i386/kernel/Makefile	Tue Mar  4 19:29:01 2003
+++ linux-2.5.64-numaq_subarch/arch/i386/kernel/Makefile	Fri Mar 14 15:29:40 2003
@@ -24,7 +24,6 @@
 obj-$(CONFIG_X86_LOCAL_APIC)	+= apic.o nmi.o
 obj-$(CONFIG_X86_IO_APIC)	+= io_apic.o
 obj-$(CONFIG_SOFTWARE_SUSPEND)	+= suspend.o suspend_asm.o
-obj-$(CONFIG_X86_NUMAQ)		+= numaq.o
 obj-$(CONFIG_EDD)             	+= edd.o
 obj-$(CONFIG_MODULES)		+= module.o
 obj-y				+= sysenter.o
diff -Nur --exclude-from=/usr/src/.dontdiff linux-2.5.64-vanilla/arch/i386/kernel/numaq.c linux-2.5.64-numaq_subarch/arch/i386/kernel/numaq.c
--- linux-2.5.64-vanilla/arch/i386/kernel/numaq.c	Tue Mar  4 19:29:32 2003
+++ linux-2.5.64-numaq_subarch/arch/i386/kernel/numaq.c	Wed Dec 31 16:00:00 1969
@@ -1,127 +0,0 @@
-/*
- * Written by: Patricia Gaughen, IBM Corporation
- *
- * Copyright (C) 2002, IBM Corp.
- *
- * All rights reserved.          
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or
- * NON INFRINGEMENT.  See the GNU General Public License for more
- * details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
- *
- * Send feedback to <gone@us.ibm.com>
- */
-
-#include <linux/config.h>
-#include <linux/mm.h>
-#include <linux/bootmem.h>
-#include <linux/mmzone.h>
-#include <linux/module.h>
-#include <asm/numaq.h>
-
-/* These are needed before the pgdat's are created */
-unsigned long node_start_pfn[MAX_NUMNODES];
-unsigned long node_end_pfn[MAX_NUMNODES];
-
-#define	MB_TO_PAGES(addr) ((addr) << (20 - PAGE_SHIFT))
-
-/*
- * Function: smp_dump_qct()
- *
- * Description: gets memory layout from the quad config table.  This
- * function also increments numnodes with the number of nodes (quads)
- * present.
- */
-static void __init smp_dump_qct(void)
-{
-	int node;
-	struct eachquadmem *eq;
-	struct sys_cfg_data *scd =
-		(struct sys_cfg_data *)__va(SYS_CFG_DATA_PRIV_ADDR);
-
-	numnodes = 0;
-	for(node = 0; node < MAX_NUMNODES; node++) {
-		if(scd->quads_present31_0 & (1 << node)) {
-			node_set_online(node);
-			numnodes++;
-			eq = &scd->eq[node];
-			/* Convert to pages */
-			node_start_pfn[node] = MB_TO_PAGES(
-				eq->hi_shrd_mem_start - eq->priv_mem_size);
-			node_end_pfn[node] = MB_TO_PAGES(
-				eq->hi_shrd_mem_start + eq->hi_shrd_mem_size);
-		}
-	}
-}
-
-/*
- * -----------------------------------------
- *
- * functions related to physnode_map
- *
- * -----------------------------------------
- */
-/*
- * physnode_map keeps track of the physical memory layout of the
- * numaq nodes on a 256Mb break (each element of the array will
- * represent 256Mb of memory and will be marked by the node id.  so,
- * if the first gig is on node 0, and the second gig is on node 1
- * physnode_map will contain:
- * physnode_map[0-3] = 0;
- * physnode_map[4-7] = 1;
- * physnode_map[8- ] = -1;
- */
-int physnode_map[MAX_ELEMENTS] = { [0 ... (MAX_ELEMENTS - 1)] = -1};
-EXPORT_SYMBOL(physnode_map);
-
-/*
- * for each node mark the regions
- *        TOPOFMEM = hi_shrd_mem_start + hi_shrd_mem_size
- *
- * need to be very careful to not mark 1024+ as belonging
- * to node 0. will want 1027 to show as belonging to node 1
- * example:
- *  TOPOFMEM = 1024
- * 1024 >> 8 = 4 (subtract 1 for starting at 0]
- * tmpvar = TOPOFMEM - 256 = 768
- * 1024 >> 8 = 4 (subtract 1 for starting at 0]
- * 
- */
-static void __init initialize_physnode_map(void)
-{
-	int nid;
-	unsigned int topofmem, cur;
-	struct eachquadmem *eq;
- 	struct sys_cfg_data *scd =
-		(struct sys_cfg_data *)__va(SYS_CFG_DATA_PRIV_ADDR);
-
-	
-	for(nid = 0; nid < numnodes; nid++) {
-		if(scd->quads_present31_0 & (1 << nid)) {
-			eq = &scd->eq[nid];
-			cur = eq->hi_shrd_mem_start;
-			topofmem = eq->hi_shrd_mem_start + eq->hi_shrd_mem_size;
-			while (cur < topofmem) {
-				physnode_map[cur >> 8] = nid;
-				cur ++;
-			}
-		}
-	}
-}
-
-void __init get_memcfg_numaq(void)
-{
-	smp_dump_qct();
-	initialize_physnode_map();
-}
diff -Nur --exclude-from=/usr/src/.dontdiff linux-2.5.64-vanilla/arch/i386/mach-numaq/Makefile linux-2.5.64-numaq_subarch/arch/i386/mach-numaq/Makefile
--- linux-2.5.64-vanilla/arch/i386/mach-numaq/Makefile	Wed Dec 31 16:00:00 1969
+++ linux-2.5.64-numaq_subarch/arch/i386/mach-numaq/Makefile	Fri Mar 14 15:40:55 2003
@@ -0,0 +1,7 @@
+#
+# Makefile for the linux kernel.
+#
+
+EXTRA_CFLAGS	+= -I../kernel
+
+obj-y				:= setup.o topology.o numaq.o
diff -Nur --exclude-from=/usr/src/.dontdiff linux-2.5.64-vanilla/arch/i386/mach-numaq/numaq.c linux-2.5.64-numaq_subarch/arch/i386/mach-numaq/numaq.c
--- linux-2.5.64-vanilla/arch/i386/mach-numaq/numaq.c	Wed Dec 31 16:00:00 1969
+++ linux-2.5.64-numaq_subarch/arch/i386/mach-numaq/numaq.c	Tue Mar  4 19:29:32 2003
@@ -0,0 +1,127 @@
+/*
+ * Written by: Patricia Gaughen, IBM Corporation
+ *
+ * Copyright (C) 2002, IBM Corp.
+ *
+ * All rights reserved.          
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or
+ * NON INFRINGEMENT.  See the GNU General Public License for more
+ * details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * Send feedback to <gone@us.ibm.com>
+ */
+
+#include <linux/config.h>
+#include <linux/mm.h>
+#include <linux/bootmem.h>
+#include <linux/mmzone.h>
+#include <linux/module.h>
+#include <asm/numaq.h>
+
+/* These are needed before the pgdat's are created */
+unsigned long node_start_pfn[MAX_NUMNODES];
+unsigned long node_end_pfn[MAX_NUMNODES];
+
+#define	MB_TO_PAGES(addr) ((addr) << (20 - PAGE_SHIFT))
+
+/*
+ * Function: smp_dump_qct()
+ *
+ * Description: gets memory layout from the quad config table.  This
+ * function also increments numnodes with the number of nodes (quads)
+ * present.
+ */
+static void __init smp_dump_qct(void)
+{
+	int node;
+	struct eachquadmem *eq;
+	struct sys_cfg_data *scd =
+		(struct sys_cfg_data *)__va(SYS_CFG_DATA_PRIV_ADDR);
+
+	numnodes = 0;
+	for(node = 0; node < MAX_NUMNODES; node++) {
+		if(scd->quads_present31_0 & (1 << node)) {
+			node_set_online(node);
+			numnodes++;
+			eq = &scd->eq[node];
+			/* Convert to pages */
+			node_start_pfn[node] = MB_TO_PAGES(
+				eq->hi_shrd_mem_start - eq->priv_mem_size);
+			node_end_pfn[node] = MB_TO_PAGES(
+				eq->hi_shrd_mem_start + eq->hi_shrd_mem_size);
+		}
+	}
+}
+
+/*
+ * -----------------------------------------
+ *
+ * functions related to physnode_map
+ *
+ * -----------------------------------------
+ */
+/*
+ * physnode_map keeps track of the physical memory layout of the
+ * numaq nodes on a 256Mb break (each element of the array will
+ * represent 256Mb of memory and will be marked by the node id.  so,
+ * if the first gig is on node 0, and the second gig is on node 1
+ * physnode_map will contain:
+ * physnode_map[0-3] = 0;
+ * physnode_map[4-7] = 1;
+ * physnode_map[8- ] = -1;
+ */
+int physnode_map[MAX_ELEMENTS] = { [0 ... (MAX_ELEMENTS - 1)] = -1};
+EXPORT_SYMBOL(physnode_map);
+
+/*
+ * for each node mark the regions
+ *        TOPOFMEM = hi_shrd_mem_start + hi_shrd_mem_size
+ *
+ * need to be very careful to not mark 1024+ as belonging
+ * to node 0. will want 1027 to show as belonging to node 1
+ * example:
+ *  TOPOFMEM = 1024
+ * 1024 >> 8 = 4 (subtract 1 for starting at 0]
+ * tmpvar = TOPOFMEM - 256 = 768
+ * 1024 >> 8 = 4 (subtract 1 for starting at 0]
+ * 
+ */
+static void __init initialize_physnode_map(void)
+{
+	int nid;
+	unsigned int topofmem, cur;
+	struct eachquadmem *eq;
+ 	struct sys_cfg_data *scd =
+		(struct sys_cfg_data *)__va(SYS_CFG_DATA_PRIV_ADDR);
+
+	
+	for(nid = 0; nid < numnodes; nid++) {
+		if(scd->quads_present31_0 & (1 << nid)) {
+			eq = &scd->eq[nid];
+			cur = eq->hi_shrd_mem_start;
+			topofmem = eq->hi_shrd_mem_start + eq->hi_shrd_mem_size;
+			while (cur < topofmem) {
+				physnode_map[cur >> 8] = nid;
+				cur ++;
+			}
+		}
+	}
+}
+
+void __init get_memcfg_numaq(void)
+{
+	smp_dump_qct();
+	initialize_physnode_map();
+}
diff -Nur --exclude-from=/usr/src/.dontdiff linux-2.5.64-vanilla/arch/i386/mach-numaq/setup.c linux-2.5.64-numaq_subarch/arch/i386/mach-numaq/setup.c
--- linux-2.5.64-vanilla/arch/i386/mach-numaq/setup.c	Wed Dec 31 16:00:00 1969
+++ linux-2.5.64-numaq_subarch/arch/i386/mach-numaq/setup.c	Fri Mar 14 15:42:17 2003
@@ -0,0 +1,104 @@
+/*
+ *	Machine specific setup for generic
+ */
+
+#include <linux/config.h>
+#include <linux/smp.h>
+#include <linux/init.h>
+#include <linux/irq.h>
+#include <linux/interrupt.h>
+#include <asm/arch_hooks.h>
+
+/**
+ * pre_intr_init_hook - initialisation prior to setting up interrupt vectors
+ *
+ * Description:
+ *	Perform any necessary interrupt initialisation prior to setting up
+ *	the "ordinary" interrupt call gates.  For legacy reasons, the ISA
+ *	interrupts should be initialised here if the machine emulates a PC
+ *	in any way.
+ **/
+void __init pre_intr_init_hook(void)
+{
+	init_ISA_irqs();
+}
+
+/*
+ * IRQ2 is cascade interrupt to second interrupt controller
+ */
+static struct irqaction irq2 = { no_action, 0, 0, "cascade", NULL, NULL};
+
+/**
+ * intr_init_hook - post gate setup interrupt initialisation
+ *
+ * Description:
+ *	Fill in any interrupts that may have been left out by the general
+ *	init_IRQ() routine.  interrupts having to do with the machine rather
+ *	than the devices on the I/O bus (like APIC interrupts in intel MP
+ *	systems) are started here.
+ **/
+void __init intr_init_hook(void)
+{
+#ifdef CONFIG_X86_LOCAL_APIC
+	apic_intr_init();
+#endif
+
+	setup_irq(2, &irq2);
+}
+
+/**
+ * pre_setup_arch_hook - hook called prior to any setup_arch() execution
+ *
+ * Description:
+ *	generally used to activate any machine specific identification
+ *	routines that may be needed before setup_arch() runs.  On VISWS
+ *	this is used to get the board revision and type.
+ **/
+void __init pre_setup_arch_hook(void)
+{
+}
+
+/**
+ * trap_init_hook - initialise system specific traps
+ *
+ * Description:
+ *	Called as the final act of trap_init().  Used in VISWS to initialise
+ *	the various board specific APIC traps.
+ **/
+void __init trap_init_hook(void)
+{
+}
+
+static struct irqaction irq0  = { timer_interrupt, SA_INTERRUPT, 0, "timer", NULL, NULL};
+
+/**
+ * time_init_hook - do any specific initialisations for the system timer.
+ *
+ * Description:
+ *	Must plug the system timer interrupt source at HZ into the IRQ listed
+ *	in irq_vectors.h:TIMER_IRQ
+ **/
+void __init time_init_hook(void)
+{
+	setup_irq(0, &irq0);
+}
+
+#ifdef CONFIG_MCA
+/**
+ * mca_nmi_hook - hook into MCA specific NMI chain
+ *
+ * Description:
+ *	The MCA (Microchannel Arcitecture) has an NMI chain for NMI sources
+ *	along the MCA bus.  Use this to hook into that chain if you will need
+ *	it.
+ **/
+void __init mca_nmi_hook(void)
+{
+	/* If I recall correctly, there's a whole bunch of other things that
+	 * we can do to check for NMI problems, but that's all I know about
+	 * at the moment.
+	 */
+
+	printk("NMI generated from unknown source!\n");
+}
+#endif
diff -Nur --exclude-from=/usr/src/.dontdiff linux-2.5.64-vanilla/arch/i386/mach-numaq/topology.c linux-2.5.64-numaq_subarch/arch/i386/mach-numaq/topology.c
--- linux-2.5.64-vanilla/arch/i386/mach-numaq/topology.c	Wed Dec 31 16:00:00 1969
+++ linux-2.5.64-numaq_subarch/arch/i386/mach-numaq/topology.c	Fri Mar 14 15:42:17 2003
@@ -0,0 +1,68 @@
+/*
+ * arch/i386/mach-generic/topology.c - Populate driverfs with topology information
+ *
+ * Written by: Matthew Dobson, IBM Corporation
+ * Original Code: Paul Dorwin, IBM Corporation, Patrick Mochel, OSDL
+ *
+ * Copyright (C) 2002, IBM Corp.
+ *
+ * All rights reserved.          
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or
+ * NON INFRINGEMENT.  See the GNU General Public License for more
+ * details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * Send feedback to <colpatch@us.ibm.com>
+ */
+#include <linux/init.h>
+#include <linux/smp.h>
+#include <asm/cpu.h>
+
+struct i386_cpu cpu_devices[NR_CPUS];
+
+#ifdef CONFIG_NUMA
+#include <linux/mmzone.h>
+#include <asm/node.h>
+#include <asm/memblk.h>
+
+struct i386_node node_devices[MAX_NUMNODES];
+struct i386_memblk memblk_devices[MAX_NR_MEMBLKS];
+
+static int __init topology_init(void)
+{
+	int i;
+
+	for (i = 0; i < num_online_nodes(); i++)
+		arch_register_node(i);
+	for (i = 0; i < NR_CPUS; i++)
+		if (cpu_possible(i)) arch_register_cpu(i);
+	for (i = 0; i < num_online_memblks(); i++)
+		arch_register_memblk(i);
+	return 0;
+}
+
+#else /* !CONFIG_NUMA */
+
+static int __init topology_init(void)
+{
+	int i;
+
+	for (i = 0; i < NR_CPUS; i++)
+		if (cpu_possible(i)) arch_register_cpu(i);
+	return 0;
+}
+
+#endif /* CONFIG_NUMA */
+
+subsys_initcall(topology_init);

Re: [patch] NUMAQ subarchification

[Martin J. Bligh]

>>> This patch adds a new file, arch/i386/kernel/summit.c, for 
>>> summit-specific code.  Adds some structures to mach_mpparse.h.  Also 
>>> adds a hook in setup_arch() to dig out the PCI info, and stores it in 
>>> the mp_bus_id_to_node[] array, where it can be read by the topology 
>>> functions.
>> 
>> Wouldn't this file be better in arch/i386/mach-summit in keeping with
>> all the other subarch stuff?
>> 
>> While you're creating a separate file for summit, could you move the
>> summit specific variables (mpparse.c:x86_summit is the only one, I
>> think) into it so we can clean all the summit references out of the main
>> line?
>> 
>> Thanks,
>> 
>> James
> 
> While I was at it, I subarchified (I'll cc Websters with the new word ;) numaq as well.  Copied mach-defaults setup.c, topology.c, and Makefile.   Moved arch/i386/kernel/numaq.c into mach-numaq.  Compiles.


No, *please* don't do this. Subarch for .c files is *broken*.

Last time I looked (and I don't think anyone has fixed it since) 
it requires copying files all over the place, making an unmaintainable
nightmare. Either subarch needs fixing first, or we don't use it.

Let's just stick with your original patch - it's fine.

M.

Re: [patch] NUMAQ subarchification

[James Bottomley]

On Fri, 2003-03-14 at 20:05, Martin J. Bligh wrote:
> No, *please* don't do this. Subarch for .c files is *broken*.

It is the place designed for code belonging only to one subarch.

> Last time I looked (and I don't think anyone has fixed it since) 
> it requires copying files all over the place, making an unmaintainable
> nightmare. Either subarch needs fixing first, or we don't use it.

It's design is identical to the arch directories, except that it has far
fewer hooks.  People grumble about having to change 20 arch files when
they want to alter the interface, but no-one's yet called it
unmaintainable.

The subarch split is designed to support machines with radically
different architectures like voyager and to a lesser extent visws.  Just
because summit isn't a radically different architecture doesn't make the
subarch concept broken.  I think other people have mentioned before that
what you probably need for summit is a modular apic driver.  However, if
you want to propose changes to the subarch setup, you're welcome to do
that too.

The problem you have (your setup.c and topology.c are identical to the
default) was originally going to be solved using VPATH.  Unfortunately,
that got broken along the way in the new build scheme, so the best I
think you can do is add this to the summit Makefile

$(obj)/setup.c: $(src)/../mach-default/setup.c
	cat $< $@

etc.

Kai isn't going to like this, but hopefully he will be able to come up
with a better solution.

However, you could also take this opportunity to remove the NUMA
pollution from mach-generic/topology.c.

> Let's just stick with your original patch - it's fine.

No, it's not.  The object of the subarch is to remove all subarch
specific files from the main i386/kernel directory.

James

Re: [patch] NUMAQ subarchification

[Martin J. Bligh]

>> No, *please* don't do this. Subarch for .c files is *broken*.
> 
> It is the place designed for code belonging only to one subarch.

That doesn't mean it's not broken.
 
>> Let's just stick with your original patch - it's fine.
> 
> No, it's not.  The object of the subarch is to remove all subarch
> specific files from the main i386/kernel directory.

It needs fixing to avoid the duplication first ... after that it'll 
be a fine idea. Until that, I'm not moving any code under there.

M.

Re: [patch] NUMAQ subarchification

[Alan Cox]

On Sat, 2003-03-15 at 02:05, Martin J. Bligh wrote:
> No, *please* don't do this. Subarch for .c files is *broken*.
> Last time I looked (and I don't think anyone has fixed it since) 
> it requires copying files all over the place, making an unmaintainable
> nightmare. Either subarch needs fixing first, or we don't use it.

It was fixed in about 2.5.50-ac. I thought Linus had picked up the 
improved version of mach-default* too. Its used extensively for stuff
like PC9800 which is deeply un-PC

Re: [patch] NUMAQ subarchification

[Martin J. Bligh]

>> No, *please* don't do this. Subarch for .c files is *broken*.
>> Last time I looked (and I don't think anyone has fixed it since) 
>> it requires copying files all over the place, making an unmaintainable
>> nightmare. Either subarch needs fixing first, or we don't use it.
> 
> It was fixed in about 2.5.50-ac. I thought Linus had picked up the 
> improved version of mach-default* too. Its used extensively for stuff
> like PC9800 which is deeply un-PC

Cool, if that's in mainline, we're set ... if not, I guess we need to
pull it out and steal it ;-)

M.

Re: [patch] NUMAQ subarchification

[Kai Germaschewski]

On 15 Mar 2003, James Bottomley wrote:

> It is the place designed for code belonging only to one subarch.
> 
> > Last time I looked (and I don't think anyone has fixed it since) 
> > it requires copying files all over the place, making an unmaintainable
> > nightmare. Either subarch needs fixing first, or we don't use it.

I agree that duplicating files into different subarch dirs sure isn't the 
way to go.

> The problem you have (your setup.c and topology.c are identical to the
> default) was originally going to be solved using VPATH.  Unfortunately,
> that got broken along the way in the new build scheme, so the best I
> think you can do is add this to the summit Makefile

I think VPATH has never been meant to be used for anything like this, it 
could be make to work, though it would interfere with the separate src/obj 
thing. But I don't think it's a good idea, we'll have object files 
magically appear without any visible source file, that's just too obscure.

I can basically see two options at this point:

> $(obj)/setup.c: $(src)/../mach-default/setup.c
> 	cat $< $@

Something like this, but using ln -sf would be nicer, since that avoids 
someone editing the wrong file by mistake.

Or:

Make the build enter mach-default always. However, make compilation of
setup.c and topology.c conditional on CONFIG_X86_DEFAULT_SETUP and
CONFIG_X86_DEFAULT_TOPOLOGY (or combine those two into one config
variable). Then, you'll have to deal with those two variables explicitly
from Kconfig. Basically, set them to n for voyager and visws, y otherwise.

Yes, that means changing another line in Kconfig when you add a new 
subarch, but that doesn't happen all that frequently. And it's sure better 
than crashes on boot because your setup.c disappeared after a bk commit 
and the build chose to use the mach-default one instead...

--Kai

Re: [patch] NUMAQ subarchification

[Martin J. Bligh]

>> It is the place designed for code belonging only to one subarch.
>> 
>> > Last time I looked (and I don't think anyone has fixed it since) 
>> > it requires copying files all over the place, making an unmaintainable
>> > nightmare. Either subarch needs fixing first, or we don't use it.
> 
> I agree that duplicating files into different subarch dirs sure isn't the 
> way to go.
> 
>> The problem you have (your setup.c and topology.c are identical to the
>> default) was originally going to be solved using VPATH.  Unfortunately,
>> that got broken along the way in the new build scheme, so the best I
>> think you can do is add this to the summit Makefile
> 
> I think VPATH has never been meant to be used for anything like this, it 
> could be make to work, though it would interfere with the separate src/obj 
> thing. But I don't think it's a good idea, we'll have object files 
> magically appear without any visible source file, that's just too obscure.
> 
> I can basically see two options at this point:
> 
>> $(obj)/setup.c: $(src)/../mach-default/setup.c
>> 	cat $< $@
> 
> Something like this, but using ln -sf would be nicer, since that avoids 
> someone editing the wrong file by mistake.

GNU diff doesn't seem to understand soft links, so I don't think that's
feasible unfortunately. The ../mach-default stuff still requires you to
edit each subarch's makefile when adding a file to any subarch, which is
a pain in the butt, but better than the duplicated files by a long way.

Another comprimise solution for now might be to leave the Makefile entries
with ifdefs in the main Makefile (just as we do without subarch), but just
move the subarch files into the subarch dirs ... at least until we get a 
better plan sorted out? That'll mean much less reshuffling when it's fixed.

M.

Re: [patch] NUMAQ subarchification

[James Bottomley]

On Sat, 2003-03-15 at 19:07, Alan Cox wrote:
> It was fixed in about 2.5.50-ac. I thought Linus had picked up the 
> improved version of mach-default* too. Its used extensively for stuff
> like PC9800 which is deeply un-PC

Actually, we have all that in the mainline.  It was the .h fallback to
mach-defaults.

The issue here is that mach-numaq and mach-summit are very PC like and
so don't need any of the hooks that are in setup.c, thus they'd like to
share setup.c from mach-default.  Likewise, they have the same topology
file, I think (although this time it would be different from
mach-default since we can pull the numa pieces out).  Prior to the build
changes, VPATH would solve this problem (I used to use it to get
trampoline.o for voyager).  However, now, the best I think we can do is
to cat the file across.

James

Re: [patch] NUMAQ subarchification

[James Bottomley]

On Sat, 2003-03-15 at 20:53, Kai Germaschewski wrote:
> I think VPATH has never been meant to be used for anything like this, it 
> could be make to work, though it would interfere with the separate src/obj 
> thing. But I don't think it's a good idea, we'll have object files 
> magically appear without any visible source file, that's just too obscure.

Well...There is a slightly different solution.

What if the summit/numaq setup.c simply contained 

#include "../mach-default/setup.c"

?

Not that I like doing this, but it solves the "magic" appearace of the
object file and it's perfectly clear to anyone editing the file where it
really comes from.

James

Re: [patch] NUMAQ subarchification

[Kai Germaschewski]

On 15 Mar 2003, James Bottomley wrote:

> On Sat, 2003-03-15 at 20:53, Kai Germaschewski wrote:
> > I think VPATH has never been meant to be used for anything like this, it 
> > could be make to work, though it would interfere with the separate src/obj 
> > thing. But I don't think it's a good idea, we'll have object files 
> > magically appear without any visible source file, that's just too obscure.
> 
> Well...There is a slightly different solution.
> 
> What if the summit/numaq setup.c simply contained 
> 
> #include "../mach-default/setup.c"
> 
> ?
> 
> Not that I like doing this, but it solves the "magic" appearace of the
> object file and it's perfectly clear to anyone editing the file where it
> really comes from.

Yes, that'd work as well. Just for reference, my suggestion: - Take
whatever you like best ;)

--Kai


===== arch/i386/Kconfig 1.48 vs edited =====
--- 1.48/arch/i386/Kconfig	Sat Mar  8 16:50:37 2003
+++ edited/arch/i386/Kconfig	Sat Mar 15 23:00:18 2003
@@ -97,6 +97,16 @@
 
 endchoice
 
+config X86_DEFAULT_SETUP
+	bool
+	default y
+	depends on X86_PC || X86_NUMAQ || X86_SUMMIT || X86_BIGSMP
+
+config X86_DEFAULT_TOPOLOGY
+	bool
+	default y
+	depends on X86_PC || X86_NUMAQ || X86_SUMMIT || X86_BIGSMP
+
 
 choice
 	prompt "Processor family"
===== arch/i386/Makefile 1.48 vs edited =====
--- 1.48/arch/i386/Makefile	Tue Mar  4 17:09:44 2003
+++ edited/arch/i386/Makefile	Sat Mar 15 23:29:32 2003
@@ -50,38 +50,33 @@
 
 CFLAGS += $(cflags-y)
 
-# Default subarch .c files
-mcore-y  := mach-default
-
 # Voyager subarch support
 mflags-$(CONFIG_X86_VOYAGER)	:= -Iinclude/asm-i386/mach-voyager
-mcore-$(CONFIG_X86_VOYAGER)	:= mach-voyager
+mcore-$(CONFIG_X86_VOYAGER)	:= arch/i386/mach-voyager/
 
 # VISWS subarch support
 mflags-$(CONFIG_X86_VISWS)	:= -Iinclude/asm-i386/mach-visws
-mcore-$(CONFIG_X86_VISWS)	:= mach-visws
+mcore-$(CONFIG_X86_VISWS)	:= arch/i386/mach-visws/
 
 # NUMAQ subarch support
 mflags-$(CONFIG_X86_NUMAQ)	:= -Iinclude/asm-i386/mach-numaq
-mcore-$(CONFIG_X86_NUMAQ)	:= mach-default
 
 # BIGSMP subarch support
 mflags-$(CONFIG_X86_BIGSMP)	:= -Iinclude/asm-i386/mach-bigsmp
-mcore-$(CONFIG_X86_BIGSMP)	:= mach-default
 
-#Summit subarch support
-mflags-$(CONFIG_X86_SUMMIT) := -Iinclude/asm-i386/mach-summit
-mcore-$(CONFIG_X86_SUMMIT)  := mach-default
+# Summit subarch support
+mflags-$(CONFIG_X86_SUMMIT)	:= -Iinclude/asm-i386/mach-summit
 
 # default subarch .h files
-mflags-y += -Iinclude/asm-i386/mach-default
+mflags-y 			+= -Iinclude/asm-i386/mach-default
 
 head-y := arch/i386/kernel/head.o arch/i386/kernel/init_task.o
 
 libs-y 					+= arch/i386/lib/
 core-y					+= arch/i386/kernel/ \
 					   arch/i386/mm/ \
-					   arch/i386/$(mcore-y)/
+					   arch/i386/mach-default/ \
+					   $(mcore-y)
 drivers-$(CONFIG_MATH_EMULATION)	+= arch/i386/math-emu/
 drivers-$(CONFIG_PCI)			+= arch/i386/pci/
 # FIXME: is drivers- right ?
===== arch/i386/mach-default/Makefile 1.8 vs edited =====
--- 1.8/arch/i386/mach-default/Makefile	Sun Dec 22 06:08:42 2002
+++ edited/arch/i386/mach-default/Makefile	Sat Mar 15 22:57:13 2003
@@ -4,4 +4,5 @@
 
 EXTRA_CFLAGS	+= -I../kernel
 
-obj-y				:= setup.o topology.o
+obj-$(CONFIG_X86_DEFAULT_SETUP)		+= setup.o
+obj-$(CONFIG_X86_DEFAULT_TOPOLOGY)	+= topology.o

Re: [patch] NUMAQ subarchification

[James Bottomley]

On Sat, 2003-03-15 at 23:55, Kai Germaschewski wrote:
> On 15 Mar 2003, James Bottomley wrote:
> +config X86_DEFAULT_TOPOLOGY
> +	bool
> +	default y
> +	depends on X86_PC || X86_NUMAQ || X86_SUMMIT || X86_BIGSMP
> +

The slight problem here is that topology isn't shared between default,
summit and numaq (or at least won't be when the #ifdef mess is removed
from the default), but summit and numaq would still like to share the
same topology file between them, which is going to add another level of
complexity to the Makefile solution.

On the whole, I think the least of the three evils is #including a .c
file.

James

Re: [patch] NUMAQ subarchification

[Matthew Dobson]

[-- Attachment #1: Type: text/plain, Size: 1859 bytes --]

Martin J. Bligh wrote:
>>>>This patch adds a new file, arch/i386/kernel/summit.c, for 
>>>>summit-specific code.  Adds some structures to mach_mpparse.h.  Also 
>>>>adds a hook in setup_arch() to dig out the PCI info, and stores it in 
>>>>the mp_bus_id_to_node[] array, where it can be read by the topology 
>>>>functions.
>>>
>>>Wouldn't this file be better in arch/i386/mach-summit in keeping with
>>>all the other subarch stuff?
>>>
>>>While you're creating a separate file for summit, could you move the
>>>summit specific variables (mpparse.c:x86_summit is the only one, I
>>>think) into it so we can clean all the summit references out of the main
>>>line?
>>>
>>>Thanks,
>>>
>>>James
>>
>>While I was at it, I subarchified (I'll cc Websters with the new word ;) numaq as well.  Copied mach-defaults setup.c, topology.c, and Makefile.   Moved arch/i386/kernel/numaq.c into mach-numaq.  Compiles.
> 
> 
> 
> No, *please* don't do this. Subarch for .c files is *broken*.
> 
> Last time I looked (and I don't think anyone has fixed it since) 
> it requires copying files all over the place, making an unmaintainable
> nightmare. Either subarch needs fixing first, or we don't use it.
> 
> Let's just stick with your original patch - it's fine.
> 
> M.

I'm inclined to agree with Martin on this one.  The useless code 
duplication is outright stupidity.  Makefile hackery would work, as 
would James' suggestion of #include'ing the .c files.  I tend to agree 
with his assessment of that as the least of evils.  But until we have a 
good way of falling back to mach-default for .c files, I'm going to 
leave the one (relatively) small .c file in arch/i386/kernel.  It will 
be trivial to move it later, and for now it sits nicely next to it's 
kin: arch/i386/kernel/numaq.c.

Original patch rediffed against 2.5.65 (just offsets) below.

Cheers!

-Matt



[-- Attachment #2: summit_pcimap-no_subarch-2.5.65-patch --]
[-- Type: text/plain, Size: 12159 bytes --]

diff -Nur --exclude-from=/usr/src/.dontdiff linux-2.5.65-vanilla/arch/i386/kernel/Makefile linux-2.5.65-summit_pcimap/arch/i386/kernel/Makefile
--- linux-2.5.65-vanilla/arch/i386/kernel/Makefile	Mon Mar 17 13:43:44 2003
+++ linux-2.5.65-summit_pcimap/arch/i386/kernel/Makefile	Mon Mar 17 17:34:03 2003
@@ -25,6 +25,7 @@
 obj-$(CONFIG_X86_IO_APIC)	+= io_apic.o
 obj-$(CONFIG_SOFTWARE_SUSPEND)	+= suspend.o suspend_asm.o
 obj-$(CONFIG_X86_NUMAQ)		+= numaq.o
+obj-$(CONFIG_X86_SUMMIT)	+= summit.o
 obj-$(CONFIG_EDD)             	+= edd.o
 obj-$(CONFIG_MODULES)		+= module.o
 obj-y				+= sysenter.o
diff -Nur --exclude-from=/usr/src/.dontdiff linux-2.5.65-vanilla/arch/i386/kernel/setup.c linux-2.5.65-summit_pcimap/arch/i386/kernel/setup.c
--- linux-2.5.65-vanilla/arch/i386/kernel/setup.c	Mon Mar 17 13:44:05 2003
+++ linux-2.5.65-summit_pcimap/arch/i386/kernel/setup.c	Mon Mar 17 17:34:03 2003
@@ -939,6 +939,9 @@
 	if (smp_found_config)
 		get_smp_config();
 #endif
+#ifdef CONFIG_X86_SUMMIT
+	setup_summit();
+#endif
 
 	register_memory(max_low_pfn);
 
diff -Nur --exclude-from=/usr/src/.dontdiff linux-2.5.65-vanilla/arch/i386/kernel/summit.c linux-2.5.65-summit_pcimap/arch/i386/kernel/summit.c
--- linux-2.5.65-vanilla/arch/i386/kernel/summit.c	Wed Dec 31 16:00:00 1969
+++ linux-2.5.65-summit_pcimap/arch/i386/kernel/summit.c	Mon Mar 17 17:34:03 2003
@@ -0,0 +1,162 @@
+/*
+ * arch/i386/kernel/summit.c - IBM Summit-Specific Code
+ *
+ * Written By: Matthew Dobson, IBM Corporation
+ *
+ * Copyright (c) 2003 IBM Corp.
+ *
+ * All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or (at
+ * your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or
+ * NON INFRINGEMENT.  See the GNU General Public License for more
+ * details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * Send feedback to <colpatch@us.ibm.com>
+ *
+ */
+
+#include <linux/mm.h>
+#include <linux/init.h>
+#include <asm/io.h>
+#include <mach_mpparse.h>
+
+static void __init setup_pci_node_map_for_wpeg(int wpeg_num, struct rio_table_hdr *rth, 
+		struct scal_detail **scal_nodes, struct rio_detail **rio_nodes){
+	int twst_num = 0, node = 0, first_bus = 0;
+	int i, bus, num_busses;
+
+	for(i = 0; i < rth->num_rio_dev; i++){
+		if (rio_nodes[i]->node_id == rio_nodes[wpeg_num]->owner_id){
+			twst_num = rio_nodes[i]->owner_id;
+			break;
+		}
+	}
+	if (i == rth->num_rio_dev){
+		printk("%s: Couldn't find owner Cyclone for Winnipeg!\n", __FUNCTION__);
+		return;
+	}
+
+	for(i = 0; i < rth->num_scal_dev; i++){
+		if (scal_nodes[i]->node_id == twst_num){
+			node = scal_nodes[i]->node_id;
+			break;
+		}
+	}
+	if (i == rth->num_scal_dev){
+		printk("%s: Couldn't find owner Twister for Cyclone!\n", __FUNCTION__);
+		return;
+	}
+
+	switch (rio_nodes[wpeg_num]->type){
+	case CompatWPEG:
+		/* The Compatability Winnipeg controls the legacy busses
+		   (busses 0 & 1), the 66MHz PCI bus [2 slots] (bus 2), 
+		   and the "extra" busses in case a PCI-PCI bridge card is 
+		   used in either slot (busses 3 & 4): total 5 busses. */
+		num_busses = 5;
+		/* The BIOS numbers the busses starting at 1, and in a 
+		   slightly wierd manner.  You'll have to trust that 
+		   the math used below to determine the number of the 
+		   first bus works. */
+		first_bus = (rio_nodes[wpeg_num]->first_slot - 1) * 2;
+		break;
+	case AltWPEG:
+		/* The Alternate/Secondary Winnipeg controls the 1st 133MHz 
+		   bus [1 slot] & its "extra" bus (busses 0 & 1), the 2nd 
+		   133MHz bus [1 slot] & its "extra" bus (busses 2 & 3), the 
+		   100MHz bus [2 slots] (bus 4), and the "extra" busses for 
+		   the 2 100MHz slots (busses 5 & 6): total 7 busses. */
+		num_busses = 7;
+		first_bus = (rio_nodes[wpeg_num]->first_slot * 2) - 1;
+		break;
+	case LookOutAWPEG:
+	case LookOutBWPEG:
+		printk("%s: LookOut Winnipegs not supported yet!\n", __FUNCTION__);
+		return;
+	default:
+		printk("%s: Unsupported Winnipeg type!\n", __FUNCTION__);
+		return;
+	}
+
+	for(bus = first_bus; bus < first_bus + num_busses; bus++)
+		mp_bus_id_to_node[bus] = node;
+}
+
+static void __init build_detail_arrays(struct rio_table_hdr *rth,
+		struct scal_detail **sd, struct rio_detail **rd){
+	unsigned long ptr;
+	int i, scal_detail_size, rio_detail_size;
+
+	switch (rth->version){
+	default:
+		printk("%s: Bad Rio Grande Table Version: %d\n", __FUNCTION__, rth->version);
+		/* Fall through to default to version 2 spec */
+	case 2:
+		scal_detail_size = 11;
+		rio_detail_size = 13;
+		break;
+	case 3:
+		scal_detail_size = 12;
+		rio_detail_size = 15;
+		break;
+	}
+
+	ptr = (unsigned long)rth + 3;
+	for(i = 0; i < rth->num_scal_dev; i++)
+		sd[i] = (struct scal_detail *)(ptr + (scal_detail_size * i));
+
+	ptr += scal_detail_size * rth->num_scal_dev;
+	for(i = 0; i < rth->num_rio_dev; i++)
+		rd[i] = (struct rio_detail *)(ptr + (rio_detail_size * i));
+}
+
+void __init setup_summit(void)
+{
+	struct rio_table_hdr	*rio_table_hdr = NULL;
+	struct scal_detail	*scal_devs[MAX_NUMNODES];
+	struct rio_detail	*rio_devs[MAX_NUMNODES*2];
+	unsigned long		ptr;
+	unsigned short		offset;
+	int			i;
+
+	memset(mp_bus_id_to_node, -1, sizeof(mp_bus_id_to_node));
+
+	/* The pointer to the EBDA is stored in the word @ phys 0x40E(40:0E) */
+	ptr = *(unsigned short *)phys_to_virt(0x40Eul);
+	ptr = (unsigned long)phys_to_virt(ptr << 4);
+
+	offset = 0x180;
+	while (offset){
+		/* The block id is stored in the 2nd word */
+		if (*((unsigned short *)(ptr + offset + 2)) == 0x4752){
+			/* set the pointer past the offset & block id */
+			rio_table_hdr = (struct rio_table_hdr *)(ptr + offset + 4);
+			break;
+		}
+		/* The next offset is stored in the 1st word.  0 means no more */
+		offset = *((unsigned short *)(ptr + offset));
+	}
+	if (!rio_table_hdr){
+		printk("%s: Unable to locate Rio Grande Table in EBDA - bailing!\n", __FUNCTION__);
+		return;
+	}
+
+	/* Deal with the ugly version 2/3 pointer arithmetic */
+	build_detail_arrays(rio_table_hdr, scal_devs, rio_devs);
+
+	for(i = 0; i < rio_table_hdr->num_rio_dev; i++)
+		if (is_WPEG(rio_devs[i]->type))
+			/* It's a Winnipeg, it's got PCI Busses */
+			setup_pci_node_map_for_wpeg(i, rio_table_hdr, scal_devs, rio_devs);
+}
diff -Nur --exclude-from=/usr/src/.dontdiff linux-2.5.65-vanilla/include/asm-i386/mach-summit/mach_mpparse.h linux-2.5.65-summit_pcimap/include/asm-i386/mach-summit/mach_mpparse.h
--- linux-2.5.65-vanilla/include/asm-i386/mach-summit/mach_mpparse.h	Mon Mar 17 13:44:04 2003
+++ linux-2.5.65-summit_pcimap/include/asm-i386/mach-summit/mach_mpparse.h	Mon Mar 17 17:34:03 2003
@@ -1,6 +1,8 @@
 #ifndef __ASM_MACH_MPPARSE_H
 #define __ASM_MACH_MPPARSE_H
 
+#include <mach_apic.h>
+
 extern int use_cyclone;
 
 static inline void mpc_oem_bus_info(struct mpc_config_bus *m, char *name, 
@@ -33,4 +35,71 @@
 		use_cyclone = 1; /*enable cyclone-timer*/
 	}
 }
+
+struct rio_table_hdr {
+	unsigned char version;      /* Version number of this data structure           */
+	                            /* Version 3 adds chassis_num & WP_index           */
+	unsigned char num_scal_dev; /* # of Scalability devices (Twisters for Vigil)   */
+	unsigned char num_rio_dev;  /* # of RIO I/O devices (Cyclones and Winnipegs)   */
+} __attribute__((packed));
+
+struct scal_detail {
+	unsigned char node_id;      /* Scalability Node ID                             */
+	unsigned long CBAR;         /* Address of 1MB register space                   */
+	unsigned char port0node;    /* Node ID port connected to: 0xFF=None            */
+	unsigned char port0port;    /* Port num port connected to: 0,1,2, or 0xFF=None */
+	unsigned char port1node;    /* Node ID port connected to: 0xFF = None          */
+	unsigned char port1port;    /* Port num port connected to: 0,1,2, or 0xFF=None */
+	unsigned char port2node;    /* Node ID port connected to: 0xFF = None          */
+	unsigned char port2port;    /* Port num port connected to: 0,1,2, or 0xFF=None */
+	unsigned char chassis_num;  /* 1 based Chassis number (1 = boot node)          */
+} __attribute__((packed));
+
+struct rio_detail {
+	unsigned char node_id;      /* RIO Node ID                                     */
+	unsigned long BBAR;         /* Address of 1MB register space                   */
+	unsigned char type;         /* Type of device                                  */
+	unsigned char owner_id;     /* For WPEG: Node ID of Cyclone that owns this WPEG*/
+	                            /* For CYC:  Node ID of Twister that owns this CYC */
+	unsigned char port0node;    /* Node ID port connected to: 0xFF=None            */
+	unsigned char port0port;    /* Port num port connected to: 0,1,2, or 0xFF=None */
+	unsigned char port1node;    /* Node ID port connected to: 0xFF=None            */
+	unsigned char port1port;    /* Port num port connected to: 0,1,2, or 0xFF=None */
+	unsigned char first_slot;   /* For WPEG: Lowest slot number below this WPEG    */
+	                            /* For CYC:  0                                     */
+	unsigned char status;       /* For WPEG: Bit 0 = 1 : the XAPIC is used         */
+	                            /*                 = 0 : the XAPIC is not used, ie:*/
+	                            /*                     ints fwded to another XAPIC */
+	                            /*           Bits1:7 Reserved                      */
+	                            /* For CYC:  Bits0:7 Reserved                      */
+	unsigned char WP_index;     /* For WPEG: WPEG instance index - lower ones have */
+	                            /*           lower slot numbers/PCI bus numbers    */
+	                            /* For CYC:  No meaning                            */
+	unsigned char chassis_num;  /* 1 based Chassis number                          */
+	                            /* For LookOut WPEGs this field indicates the      */
+	                            /* Expansion Chassis #, enumerated from Boot       */
+	                            /* Node WPEG external port, then Boot Node CYC     */
+	                            /* external port, then Next Vigil chassis WPEG     */
+	                            /* external port, etc.                             */
+	                            /* Shared Lookouts have only 1 chassis number (the */
+	                            /* first one assigned)                             */
+} __attribute__((packed));
+
+
+typedef enum {
+	CompatTwister = 0,  /* Compatibility Twister               */
+	AltTwister    = 1,  /* Alternate Twister of internal 8-way */
+	CompatCyclone = 2,  /* Compatibility Cyclone               */
+	AltCyclone    = 3,  /* Alternate Cyclone of internal 8-way */
+	CompatWPEG    = 4,  /* Compatibility WPEG                  */
+	AltWPEG       = 5,  /* Second Planar WPEG                  */
+	LookOutAWPEG  = 6,  /* LookOut WPEG                        */
+	LookOutBWPEG  = 7,  /* LookOut WPEG                        */
+} node_type;
+
+static inline int is_WPEG(node_type type){
+	return (type == CompatWPEG || type == AltWPEG ||
+		type == LookOutAWPEG || type == LookOutBWPEG);
+}
+
 #endif /* __ASM_MACH_MPPARSE_H */
diff -Nur --exclude-from=/usr/src/.dontdiff linux-2.5.65-vanilla/include/asm-i386/mpspec.h linux-2.5.65-summit_pcimap/include/asm-i386/mpspec.h
--- linux-2.5.65-vanilla/include/asm-i386/mpspec.h	Mon Mar 17 13:43:37 2003
+++ linux-2.5.65-summit_pcimap/include/asm-i386/mpspec.h	Mon Mar 17 17:34:03 2003
@@ -222,6 +222,10 @@
 extern int pic_mode;
 extern int using_apic_timer;
 
+#ifdef CONFIG_X86_SUMMIT
+extern void setup_summit (void);
+#endif
+
 #ifdef CONFIG_ACPI_BOOT
 extern void mp_register_lapic (u8 id, u8 enabled);
 extern void mp_register_lapic_address (u64 address);

Re: [patch] NUMAQ subarchification

[James Bottomley]

On Mon, 2003-03-17 at 20:58, Matthew Dobson wrote:
> 
> I'm inclined to agree with Martin on this one.  The useless code 
> duplication is outright stupidity.  Makefile hackery would work, as 
> would James' suggestion of #include'ing the .c files.  I tend to agree 
> with his assessment of that as the least of evils.  But until we have a 
> good way of falling back to mach-default for .c files, I'm going to 
> leave the one (relatively) small .c file in arch/i386/kernel.  It will 
> be trivial to move it later, and for now it sits nicely next to it's 
> kin: arch/i386/kernel/numaq.c.

There are five hooks in setup.c (since I assume SUMMIT isn't MCA),
amounting to about 20 lines of code in all.  There is no extra code in
the hooks which doesn't serve a purpose to the rest of the subarch
implementations.

I also notice there's some summit code that *should* be in an arch hook:

> diff -Nur --exclude-from=/usr/src/.dontdiff linux-2.5.65-vanilla/arch/i386/kernel/setup.c linux-2.5.65-summit_pcimap/arch/i386/kernel/setup.c
> --- linux-2.5.65-vanilla/arch/i386/kernel/setup.c	Mon Mar 17 13:44:05 2003
> +++ linux-2.5.65-summit_pcimap/arch/i386/kernel/setup.c	Mon Mar 17 17:34:03 2003
> @@ -939,6 +939,9 @@
>  	if (smp_found_config)
>  		get_smp_config();
>  #endif
> +#ifdef CONFIG_X86_SUMMIT
> +	setup_summit();
> +#endif
>  
>  	register_memory(max_low_pfn);
>  
> --- linux-2.5.65-vanilla/include/asm-i386/mpspec.h	Mon Mar 17 13:43:37 2003
> +++ linux-2.5.65-summit_pcimap/include/asm-i386/mpspec.h	Mon Mar 17 17:34:03 2003
> @@ -222,6 +222,10 @@
>  extern int pic_mode;
>  extern int using_apic_timer;
>  
> +#ifdef CONFIG_X86_SUMMIT
> +extern void setup_summit (void);
> +#endif
> +

which is already there waiting for you to use it.

As I've already pointed out, the summit topology.c file should be split
out from the default one and the #ifdef CONFIG_NUMA removed.

All in all, properly done, there would be very little code duplication
even for something as PC like as the summit.

If I read correctly between the whines, the real issue is that subarch
is used to support things that vary from uniquely oddball to extremely
weird and are all ancient and highly unlikely ever to be found in
nature.  IBM, understandably, doesn't want its brand new and expensive
summit lumped with such company.

I'm not the gatekeeper of code purity in arch/i386/kernel, so if you can
convince Linus, he'll probably take it.  However, if you could come up
with a cogent technical argument why you can't use the subarch (or
modify it for your needs), it would probably be helpful.  Not wanting to
duplicate 15 lines of code isn't really that convincing...

James

Re: [patch] NUMAQ subarchification

[Martin J. Bligh]

> All in all, properly done, there would be very little code duplication
> even for something as PC like as the summit.
> 
> If I read correctly between the whines, the real issue is that subarch
> is used to support things that vary from uniquely oddball to extremely
> weird and are all ancient and highly unlikely ever to be found in
> nature.  IBM, understandably, doesn't want its brand new and expensive
> summit lumped with such company.

The company argument really doesn't bother me at all ;-)
I'm much more concerned with making things easy to maintain ... the .h
file stuff under subarch is *superb* as long as you don't have to build
one kernel that supports many machines. If the .c stuff worked like that,
I'd have no issues.

The function vector stuff is another issue ... we need to make the distros
happy at some point soon for the mainline arches that need to be supported
at least (ie NUMA-Q et al don't matter so much).

M.

[patch] Summit support for pcibus <-> cpumask topology

[Matthew Dobson]

[-- Attachment #1: Type: text/plain, Size: 934 bytes --]

Here's a patch to dig the PCI bus to node mapping information out of the 
deep, dank recesses of the EBDA tables for the Summit machines.  This 
info is then used for the pcibus_to_cpumask() topology function.

brief summary:
This patch adds a new file, arch/i386/kernel/summit.c, for 
summit-specific code.  Adds some structures to mach_mpparse.h.  Also 
adds a hook in setup_arch() to dig out the PCI info, and stores it in 
the mp_bus_id_to_node[] array, where it can be read by the topology 
functions.

As always, questions, comments, or praise (yeah, right) welcome!

  arch/i386/kernel/Makefile                   |    1
  arch/i386/kernel/setup.c                    |    3
  arch/i386/kernel/summit.c                   |  162 
++++++++++++++++++++++++++++
  include/asm-i386/mach-summit/mach_mpparse.h |   69 +++++++++++
  include/asm-i386/mpspec.h                   |    4
  5 files changed, 239 insertions(+)

Cheers!

-Matt

[-- Attachment #2: summit_pcimap-2.5.64-patch --]
[-- Type: text/plain, Size: 12159 bytes --]

diff -Nur --exclude-from=/usr/src/.dontdiff linux-2.5.64-vanilla/arch/i386/kernel/Makefile linux-2.5.64-summit_pcimap/arch/i386/kernel/Makefile
--- linux-2.5.64-vanilla/arch/i386/kernel/Makefile	Tue Mar  4 19:29:01 2003
+++ linux-2.5.64-summit_pcimap/arch/i386/kernel/Makefile	Fri Mar 14 09:54:18 2003
@@ -25,6 +25,7 @@
 obj-$(CONFIG_X86_IO_APIC)	+= io_apic.o
 obj-$(CONFIG_SOFTWARE_SUSPEND)	+= suspend.o suspend_asm.o
 obj-$(CONFIG_X86_NUMAQ)		+= numaq.o
+obj-$(CONFIG_X86_SUMMIT)	+= summit.o
 obj-$(CONFIG_EDD)             	+= edd.o
 obj-$(CONFIG_MODULES)		+= module.o
 obj-y				+= sysenter.o
diff -Nur --exclude-from=/usr/src/.dontdiff linux-2.5.64-vanilla/arch/i386/kernel/setup.c linux-2.5.64-summit_pcimap/arch/i386/kernel/setup.c
--- linux-2.5.64-vanilla/arch/i386/kernel/setup.c	Tue Mar  4 19:29:18 2003
+++ linux-2.5.64-summit_pcimap/arch/i386/kernel/setup.c	Fri Mar 14 09:56:21 2003
@@ -918,6 +918,9 @@
 	if (smp_found_config)
 		get_smp_config();
 #endif
+#ifdef CONFIG_X86_SUMMIT
+	setup_summit();
+#endif
 
 	register_memory(max_low_pfn);
 
diff -Nur --exclude-from=/usr/src/.dontdiff linux-2.5.64-vanilla/arch/i386/kernel/summit.c linux-2.5.64-summit_pcimap/arch/i386/kernel/summit.c
--- linux-2.5.64-vanilla/arch/i386/kernel/summit.c	Wed Dec 31 16:00:00 1969
+++ linux-2.5.64-summit_pcimap/arch/i386/kernel/summit.c	Fri Mar 14 11:03:15 2003
@@ -0,0 +1,162 @@
+/*
+ * arch/i386/kernel/summit.c - IBM Summit-Specific Code
+ *
+ * Written By: Matthew Dobson, IBM Corporation
+ *
+ * Copyright (c) 2003 IBM Corp.
+ *
+ * All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or (at
+ * your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or
+ * NON INFRINGEMENT.  See the GNU General Public License for more
+ * details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * Send feedback to <colpatch@us.ibm.com>
+ *
+ */
+
+#include <linux/mm.h>
+#include <linux/init.h>
+#include <asm/io.h>
+#include <mach_mpparse.h>
+
+static void __init setup_pci_node_map_for_wpeg(int wpeg_num, struct rio_table_hdr *rth, 
+		struct scal_detail **scal_nodes, struct rio_detail **rio_nodes){
+	int twst_num = 0, node = 0, first_bus = 0;
+	int i, bus, num_busses;
+
+	for(i = 0; i < rth->num_rio_dev; i++){
+		if (rio_nodes[i]->node_id == rio_nodes[wpeg_num]->owner_id){
+			twst_num = rio_nodes[i]->owner_id;
+			break;
+		}
+	}
+	if (i == rth->num_rio_dev){
+		printk("%s: Couldn't find owner Cyclone for Winnipeg!\n", __FUNCTION__);
+		return;
+	}
+
+	for(i = 0; i < rth->num_scal_dev; i++){
+		if (scal_nodes[i]->node_id == twst_num){
+			node = scal_nodes[i]->node_id;
+			break;
+		}
+	}
+	if (i == rth->num_scal_dev){
+		printk("%s: Couldn't find owner Twister for Cyclone!\n", __FUNCTION__);
+		return;
+	}
+
+	switch (rio_nodes[wpeg_num]->type){
+	case CompatWPEG:
+		/* The Compatability Winnipeg controls the legacy busses
+		   (busses 0 & 1), the 66MHz PCI bus [2 slots] (bus 2), 
+		   and the "extra" busses in case a PCI-PCI bridge card is 
+		   used in either slot (busses 3 & 4): total 5 busses. */
+		num_busses = 5;
+		/* The BIOS numbers the busses starting at 1, and in a 
+		   slightly wierd manner.  You'll have to trust that 
+		   the math used below to determine the number of the 
+		   first bus works. */
+		first_bus = (rio_nodes[wpeg_num]->first_slot - 1) * 2;
+		break;
+	case AltWPEG:
+		/* The Alternate/Secondary Winnipeg controls the 1st 133MHz 
+		   bus [1 slot] & its "extra" bus (busses 0 & 1), the 2nd 
+		   133MHz bus [1 slot] & its "extra" bus (busses 2 & 3), the 
+		   100MHz bus [2 slots] (bus 4), and the "extra" busses for 
+		   the 2 100MHz slots (busses 5 & 6): total 7 busses. */
+		num_busses = 7;
+		first_bus = (rio_nodes[wpeg_num]->first_slot * 2) - 1;
+		break;
+	case LookOutAWPEG:
+	case LookOutBWPEG:
+		printk("%s: LookOut Winnipegs not supported yet!\n", __FUNCTION__);
+		return;
+	default:
+		printk("%s: Unsupported Winnipeg type!\n", __FUNCTION__);
+		return;
+	}
+
+	for(bus = first_bus; bus < first_bus + num_busses; bus++)
+		mp_bus_id_to_node[bus] = node;
+}
+
+static void __init build_detail_arrays(struct rio_table_hdr *rth,
+		struct scal_detail **sd, struct rio_detail **rd){
+	unsigned long ptr;
+	int i, scal_detail_size, rio_detail_size;
+
+	switch (rth->version){
+	default:
+		printk("%s: Bad Rio Grande Table Version: %d\n", __FUNCTION__, rth->version);
+		/* Fall through to default to version 2 spec */
+	case 2:
+		scal_detail_size = 11;
+		rio_detail_size = 13;
+		break;
+	case 3:
+		scal_detail_size = 12;
+		rio_detail_size = 15;
+		break;
+	}
+
+	ptr = (unsigned long)rth + 3;
+	for(i = 0; i < rth->num_scal_dev; i++)
+		sd[i] = (struct scal_detail *)(ptr + (scal_detail_size * i));
+
+	ptr += scal_detail_size * rth->num_scal_dev;
+	for(i = 0; i < rth->num_rio_dev; i++)
+		rd[i] = (struct rio_detail *)(ptr + (rio_detail_size * i));
+}
+
+void __init setup_summit(void)
+{
+	struct rio_table_hdr	*rio_table_hdr = NULL;
+	struct scal_detail	*scal_devs[MAX_NUMNODES];
+	struct rio_detail	*rio_devs[MAX_NUMNODES*2];
+	unsigned long		ptr;
+	unsigned short		offset;
+	int			i;
+
+	memset(mp_bus_id_to_node, -1, sizeof(mp_bus_id_to_node));
+
+	/* The pointer to the EBDA is stored in the word @ phys 0x40E(40:0E) */
+	ptr = *(unsigned short *)phys_to_virt(0x40Eul);
+	ptr = (unsigned long)phys_to_virt(ptr << 4);
+
+	offset = 0x180;
+	while (offset){
+		/* The block id is stored in the 2nd word */
+		if (*((unsigned short *)(ptr + offset + 2)) == 0x4752){
+			/* set the pointer past the offset & block id */
+			rio_table_hdr = (struct rio_table_hdr *)(ptr + offset + 4);
+			break;
+		}
+		/* The next offset is stored in the 1st word.  0 means no more */
+		offset = *((unsigned short *)(ptr + offset));
+	}
+	if (!rio_table_hdr){
+		printk("%s: Unable to locate Rio Grande Table in EBDA - bailing!\n", __FUNCTION__);
+		return;
+	}
+
+	/* Deal with the ugly version 2/3 pointer arithmetic */
+	build_detail_arrays(rio_table_hdr, scal_devs, rio_devs);
+
+	for(i = 0; i < rio_table_hdr->num_rio_dev; i++)
+		if (is_WPEG(rio_devs[i]->type))
+			/* It's a Winnipeg, it's got PCI Busses */
+			setup_pci_node_map_for_wpeg(i, rio_table_hdr, scal_devs, rio_devs);
+}
diff -Nur --exclude-from=/usr/src/.dontdiff linux-2.5.64-vanilla/include/asm-i386/mach-summit/mach_mpparse.h linux-2.5.64-summit_pcimap/include/asm-i386/mach-summit/mach_mpparse.h
--- linux-2.5.64-vanilla/include/asm-i386/mach-summit/mach_mpparse.h	Tue Mar  4 19:29:17 2003
+++ linux-2.5.64-summit_pcimap/include/asm-i386/mach-summit/mach_mpparse.h	Fri Mar 14 12:20:38 2003
@@ -1,6 +1,8 @@
 #ifndef __ASM_MACH_MPPARSE_H
 #define __ASM_MACH_MPPARSE_H
 
+#include <mach_apic.h>
+
 extern int use_cyclone;
 
 static inline void mpc_oem_bus_info(struct mpc_config_bus *m, char *name, 
@@ -33,4 +35,71 @@
 		use_cyclone = 1; /*enable cyclone-timer*/
 	}
 }
+
+struct rio_table_hdr {
+	unsigned char version;      /* Version number of this data structure           */
+	                            /* Version 3 adds chassis_num & WP_index           */
+	unsigned char num_scal_dev; /* # of Scalability devices (Twisters for Vigil)   */
+	unsigned char num_rio_dev;  /* # of RIO I/O devices (Cyclones and Winnipegs)   */
+} __attribute__((packed));
+
+struct scal_detail {
+	unsigned char node_id;      /* Scalability Node ID                             */
+	unsigned long CBAR;         /* Address of 1MB register space                   */
+	unsigned char port0node;    /* Node ID port connected to: 0xFF=None            */
+	unsigned char port0port;    /* Port num port connected to: 0,1,2, or 0xFF=None */
+	unsigned char port1node;    /* Node ID port connected to: 0xFF = None          */
+	unsigned char port1port;    /* Port num port connected to: 0,1,2, or 0xFF=None */
+	unsigned char port2node;    /* Node ID port connected to: 0xFF = None          */
+	unsigned char port2port;    /* Port num port connected to: 0,1,2, or 0xFF=None */
+	unsigned char chassis_num;  /* 1 based Chassis number (1 = boot node)          */
+} __attribute__((packed));
+
+struct rio_detail {
+	unsigned char node_id;      /* RIO Node ID                                     */
+	unsigned long BBAR;         /* Address of 1MB register space                   */
+	unsigned char type;         /* Type of device                                  */
+	unsigned char owner_id;     /* For WPEG: Node ID of Cyclone that owns this WPEG*/
+	                            /* For CYC:  Node ID of Twister that owns this CYC */
+	unsigned char port0node;    /* Node ID port connected to: 0xFF=None            */
+	unsigned char port0port;    /* Port num port connected to: 0,1,2, or 0xFF=None */
+	unsigned char port1node;    /* Node ID port connected to: 0xFF=None            */
+	unsigned char port1port;    /* Port num port connected to: 0,1,2, or 0xFF=None */
+	unsigned char first_slot;   /* For WPEG: Lowest slot number below this WPEG    */
+	                            /* For CYC:  0                                     */
+	unsigned char status;       /* For WPEG: Bit 0 = 1 : the XAPIC is used         */
+	                            /*                 = 0 : the XAPIC is not used, ie:*/
+	                            /*                     ints fwded to another XAPIC */
+	                            /*           Bits1:7 Reserved                      */
+	                            /* For CYC:  Bits0:7 Reserved                      */
+	unsigned char WP_index;     /* For WPEG: WPEG instance index - lower ones have */
+	                            /*           lower slot numbers/PCI bus numbers    */
+	                            /* For CYC:  No meaning                            */
+	unsigned char chassis_num;  /* 1 based Chassis number                          */
+	                            /* For LookOut WPEGs this field indicates the      */
+	                            /* Expansion Chassis #, enumerated from Boot       */
+	                            /* Node WPEG external port, then Boot Node CYC     */
+	                            /* external port, then Next Vigil chassis WPEG     */
+	                            /* external port, etc.                             */
+	                            /* Shared Lookouts have only 1 chassis number (the */
+	                            /* first one assigned)                             */
+} __attribute__((packed));
+
+
+typedef enum {
+	CompatTwister = 0,  /* Compatibility Twister               */
+	AltTwister    = 1,  /* Alternate Twister of internal 8-way */
+	CompatCyclone = 2,  /* Compatibility Cyclone               */
+	AltCyclone    = 3,  /* Alternate Cyclone of internal 8-way */
+	CompatWPEG    = 4,  /* Compatibility WPEG                  */
+	AltWPEG       = 5,  /* Second Planar WPEG                  */
+	LookOutAWPEG  = 6,  /* LookOut WPEG                        */
+	LookOutBWPEG  = 7,  /* LookOut WPEG                        */
+} node_type;
+
+static inline int is_WPEG(node_type type){
+	return (type == CompatWPEG || type == AltWPEG ||
+		type == LookOutAWPEG || type == LookOutBWPEG);
+}
+
 #endif /* __ASM_MACH_MPPARSE_H */
diff -Nur --exclude-from=/usr/src/.dontdiff linux-2.5.64-vanilla/include/asm-i386/mpspec.h linux-2.5.64-summit_pcimap/include/asm-i386/mpspec.h
--- linux-2.5.64-vanilla/include/asm-i386/mpspec.h	Tue Mar  4 19:28:53 2003
+++ linux-2.5.64-summit_pcimap/include/asm-i386/mpspec.h	Fri Mar 14 09:58:46 2003
@@ -222,6 +222,10 @@
 extern int pic_mode;
 extern int using_apic_timer;
 
+#ifdef CONFIG_X86_SUMMIT
+extern void setup_summit (void);
+#endif
+
 #ifdef CONFIG_ACPI_BOOT
 extern void mp_register_lapic (u8 id, u8 enabled);
 extern void mp_register_lapic_address (u64 address);