(no subject)

[Andrew Banman]

>From Andrew Banman <abanman@sgi.com> # This line is ignored.
From: Andrew Banman <abanman@sgi.com>
Subject: [PATCH 0/9] arch/x86/platform/uv: add UV4 support to BAU
In-Reply-To: 

The following patch set adds support for UV4 architecture to the Broadcast
Assist Unit (BAU). Major hardware changes to the BAU require these fixes to
ensure correct operation and to avoid illegal MMR writes.

 arch/x86/include/asm/uv/uv_bau.h |  45 ++----------------------------
 arch/x86/platform/uv/tlb_uv.c    | 114 ++++++++++++++++++++++++---------------------------------------
-------------

The patch set can be thought of in three logical groups:

1) General cleanup.

 [PATCH 1/9] arch/x86/platform/uv: BAU cleanup: update printks
 [PATCH 2/9] arch/x86/platform/uv: BAU cleanup: pq_init
 [PATCH 3/9] arch/x86/platform/uv: BAU replace uv_physnodeaddr

 These housekeeping patches make the subsequent UV4 patches clearer,
 and they should be done in any case.


2) Implement a new scheme to abstract UV version-specific functions.

 [PATCH 4/9] arch/x86/platform/uv: BAU add generic function pointers
 [PATCH 5/9] arch/x86/platform/uv: BAU use generic function pointers

 We add a struct of function pointers to define version-specific BAU
 operations. The philosophy is to abstract functions that perform the same
 operation on all UV versions but have different implementations. This will
 simplify their use in the body of the driver code and greatly simplify the
 UV4 patches to follow.


3) Add UV4 functionality.

 [PATCH 6/9] arch/x86/platform/uv: BAU UV4 populate uvhub_version
 [PATCH 7/9] arch/x86/platform/uv: BAU UV4 disable software timeout
 [PATCH 8/9] arch/x86/platform/uv: BAU UV4 fix payload queue setup
 [PATCH 9/9] arch/x86/platform/uv: BAU UV4 add version-specific

 These patches feature a minimal set of changes to make the BAU on UV4
 operational.


This patch set has been tested for regressions on pre-UV4 architectures and
for correct functionality on UV4. The patches apply cleanly to 4.8-rc7.
Fine-tuned performance tweaking for UV4 will come in a future patch set.


Thank you,

Andrew Banman

[PATCH 01/10] arch/x86/platform/uv: BAU cleanup: update printks

[Andrew Banman]

Replace all uses of printk with the appropriate pr_* function.

Signed-off-by: Andrew Banman <abanman@sgi.com>
Acked-by: Mike Travis <travis@sgi.com>
Acked-by: Dimitri Sivanich <sivanich@sgi.com>
---
 arch/x86/platform/uv/tlb_uv.c | 26 +++++++++++++-------------
 1 file changed, 13 insertions(+), 13 deletions(-)

diff --git a/arch/x86/platform/uv/tlb_uv.c b/arch/x86/platform/uv/tlb_uv.c
index fdb4d42..7baea46 100644
--- a/arch/x86/platform/uv/tlb_uv.c
+++ b/arch/x86/platform/uv/tlb_uv.c
@@ -1497,16 +1497,16 @@ static ssize_t ptc_proc_write(struct file *file, const char __user *user,
 	}
 
 	if (kstrtol(optstr, 10, &input_arg) < 0) {
-		printk(KERN_DEBUG "%s is invalid\n", optstr);
+		pr_debug("%s is invalid\n", optstr);
 		return -EINVAL;
 	}
 
 	if (input_arg == 0) {
 		elements = ARRAY_SIZE(stat_description);
-		printk(KERN_DEBUG "# cpu:      cpu number\n");
-		printk(KERN_DEBUG "Sender statistics:\n");
+		pr_debug("# cpu:      cpu number\n");
+		pr_debug("Sender statistics:\n");
 		for (i = 0; i < elements; i++)
-			printk(KERN_DEBUG "%s\n", stat_description[i]);
+			pr_debug("%s\n", stat_description[i]);
 	} else if (input_arg == -1) {
 		for_each_present_cpu(cpu) {
 			stat = &per_cpu(ptcstats, cpu);
@@ -1554,7 +1554,7 @@ static int parse_tunables_write(struct bau_control *bcp, char *instr,
 			break;
 	}
 	if (cnt != e) {
-		printk(KERN_INFO "bau tunable error: should be %d values\n", e);
+		pr_info("bau tunable error: should be %d values\n", e);
 		return -EINVAL;
 	}
 
@@ -1571,7 +1571,7 @@ static int parse_tunables_write(struct bau_control *bcp, char *instr,
 				continue;
 			}
 			if (val < 1 || val > bcp->cpus_in_uvhub) {
-				printk(KERN_DEBUG
+				pr_debug(
 				"Error: BAU max concurrent %d is invalid\n",
 				val);
 				return -EINVAL;
@@ -1676,21 +1676,21 @@ static int __init uv_ptc_init(void)
 	proc_uv_ptc = proc_create(UV_PTC_BASENAME, 0444, NULL,
 				  &proc_uv_ptc_operations);
 	if (!proc_uv_ptc) {
-		printk(KERN_ERR "unable to create %s proc entry\n",
+		pr_err("unable to create %s proc entry\n",
 		       UV_PTC_BASENAME);
 		return -EINVAL;
 	}
 
 	tunables_dir = debugfs_create_dir(UV_BAU_TUNABLES_DIR, NULL);
 	if (!tunables_dir) {
-		printk(KERN_ERR "unable to create debugfs directory %s\n",
+		pr_err("unable to create debugfs directory %s\n",
 		       UV_BAU_TUNABLES_DIR);
 		return -EINVAL;
 	}
 	tunables_file = debugfs_create_file(UV_BAU_TUNABLES_FILE, 0600,
 					tunables_dir, NULL, &tunables_fops);
 	if (!tunables_file) {
-		printk(KERN_ERR "unable to create debugfs file %s\n",
+		pr_err("unable to create debugfs file %s\n",
 		       UV_BAU_TUNABLES_FILE);
 		return -EINVAL;
 	}
@@ -1944,7 +1944,7 @@ static int __init get_cpu_topology(int base_pnode,
 
 		pnode = uv_cpu_hub_info(cpu)->pnode;
 		if ((pnode - base_pnode) >= UV_DISTRIBUTION_SIZE) {
-			printk(KERN_EMERG
+			pr_emerg(
 				"cpu %d pnode %d-%d beyond %d; BAU disabled\n",
 				cpu, pnode, base_pnode, UV_DISTRIBUTION_SIZE);
 			return 1;
@@ -1969,7 +1969,7 @@ static int __init get_cpu_topology(int base_pnode,
 		sdp->cpu_number[sdp->num_cpus] = cpu;
 		sdp->num_cpus++;
 		if (sdp->num_cpus > MAX_CPUS_PER_SOCKET) {
-			printk(KERN_EMERG "%d cpus per socket invalid\n",
+			pr_emerg("%d cpus per socket invalid\n",
 				sdp->num_cpus);
 			return 1;
 		}
@@ -2036,14 +2036,14 @@ static int scan_sock(struct socket_desc *sdp, struct uvhub_desc *bdp,
 		else if (is_uv3_hub())
 			bcp->uvhub_version = 3;
 		else {
-			printk(KERN_EMERG "uvhub version not 1, 2 or 3\n");
+			pr_emerg("uvhub version not 1, 2, or 3\n");
 			return 1;
 		}
 		bcp->uvhub_master = *hmasterp;
 		bcp->uvhub_cpu = uv_cpu_blade_processor_id(cpu);
 
 		if (bcp->uvhub_cpu >= MAX_CPUS_PER_UVHUB) {
-			printk(KERN_EMERG "%d cpus per uvhub invalid\n",
+			pr_emerg("%d cpus per uvhub invalid\n",
 				bcp->uvhub_cpu);
 			return 1;
 		}
-- 
1.8.5.6

[PATCH 02/10] arch/x86/platform/uv: BAU cleanup: pq_init

[Andrew Banman]

The payload queue first MMR requires the physical memory address and hub
GNODE where the payload queue, but the associated variables are named as if
the PNODE were used. Rename gnode-related variables and clarify the
definitions of the payload queue head, last, and tail pointers.

Signed-off-by: Andrew Banman <abanman@sgi.com>
Acked-by: Mike Travis <travis@sgi.com>
Acked-by: Dimitri Sivanich <sivanich@sgi.com>
---
 arch/x86/include/asm/uv/uv_bau.h |  2 +-
 arch/x86/platform/uv/tlb_uv.c    | 19 ++++++++-----------
 2 files changed, 9 insertions(+), 12 deletions(-)

diff --git a/arch/x86/include/asm/uv/uv_bau.h b/arch/x86/include/asm/uv/uv_bau.h
index cc44d92..cc058c6 100644
--- a/arch/x86/include/asm/uv/uv_bau.h
+++ b/arch/x86/include/asm/uv/uv_bau.h
@@ -49,7 +49,7 @@
 #define UV_NET_ENDPOINT_INTD		(is_uv1_hub() ?			\
 			UV1_NET_ENDPOINT_INTD : UV2_NET_ENDPOINT_INTD)
 #define UV_DESC_PSHIFT			49
-#define UV_PAYLOADQ_PNODE_SHIFT		49
+#define UV_PAYLOADQ_GNODE_SHIFT		49
 #define UV_PTC_BASENAME			"sgi_uv/ptc_statistics"
 #define UV_BAU_BASENAME			"sgi_uv/bau_tunables"
 #define UV_BAU_TUNABLES_DIR		"sgi_uv"
diff --git a/arch/x86/platform/uv/tlb_uv.c b/arch/x86/platform/uv/tlb_uv.c
index 7baea46..e8c2d2f 100644
--- a/arch/x86/platform/uv/tlb_uv.c
+++ b/arch/x86/platform/uv/tlb_uv.c
@@ -1790,10 +1790,7 @@ static void pq_init(int node, int pnode)
 	size_t plsize;
 	char *cp;
 	void *vp;
-	unsigned long pn;
-	unsigned long first;
-	unsigned long pn_first;
-	unsigned long last;
+	unsigned long gnode, first, last, tail;
 	struct bau_pq_entry *pqp;
 	struct bau_control *bcp;
 
@@ -1814,16 +1811,16 @@ static void pq_init(int node, int pnode)
 		bcp->bau_msg_head	= pqp;
 		bcp->queue_last		= pqp + (DEST_Q_SIZE - 1);
 	}
-	/*
-	 * need the gnode of where the memory was really allocated
-	 */
-	pn = uv_gpa_to_gnode(uv_gpa(pqp));
+
 	first = uv_physnodeaddr(pqp);
-	pn_first = ((unsigned long)pn << UV_PAYLOADQ_PNODE_SHIFT) | first;
 	last = uv_physnodeaddr(pqp + (DEST_Q_SIZE - 1));
-	write_mmr_payload_first(pnode, pn_first);
-	write_mmr_payload_tail(pnode, first);
+	tail = first;
+	gnode = uv_gpa_to_gnode(uv_gpa(pqp));
+	first = (gnode << UV_PAYLOADQ_GNODE_SHIFT) | tail;
+
+	write_mmr_payload_first(pnode, first);
 	write_mmr_payload_last(pnode, last);
+	write_mmr_payload_tail(pnode, tail);
 	write_gmmr_sw_ack(pnode, 0xffffUL);
 
 	/* in effect, all msg_type's are set to MSG_NOOP */
-- 
1.8.5.6

[PATCH 03/10] arch/x86/platform/uv: BAU replace uv_physnodeaddr

[Andrew Banman]

The BAU driver should use the functions provided by uv_hub.h rather than
its own implementations. uv_physnodeaddr converts vaddrs to paddrs for
BAU MMR fields, but this is done better by uv_gpa_to_offset.

Signed-off-by: Andrew Banman <abanman@sgi.com>
Acked-by: Mike Travis <travis@sgi.com>
Acked-by: Dimitri Sivanich <sivanich@sgi.com>
---
 arch/x86/include/asm/uv/uv_bau.h | 2 --
 arch/x86/platform/uv/tlb_uv.c    | 4 ++--
 2 files changed, 2 insertions(+), 4 deletions(-)

diff --git a/arch/x86/include/asm/uv/uv_bau.h b/arch/x86/include/asm/uv/uv_bau.h
index cc058c6..a46f270 100644
--- a/arch/x86/include/asm/uv/uv_bau.h
+++ b/arch/x86/include/asm/uv/uv_bau.h
@@ -55,8 +55,6 @@
 #define UV_BAU_TUNABLES_DIR		"sgi_uv"
 #define UV_BAU_TUNABLES_FILE		"bau_tunables"
 #define WHITESPACE			" \t\n"
-#define uv_mmask			((1UL << uv_hub_info->m_val) - 1)
-#define uv_physnodeaddr(x)		((__pa((unsigned long)(x)) & uv_mmask))
 #define cpubit_isset(cpu, bau_local_cpumask) \
 	test_bit((cpu), (bau_local_cpumask).bits)
 
diff --git a/arch/x86/platform/uv/tlb_uv.c b/arch/x86/platform/uv/tlb_uv.c
index e8c2d2f..98f1d43 100644
--- a/arch/x86/platform/uv/tlb_uv.c
+++ b/arch/x86/platform/uv/tlb_uv.c
@@ -1812,8 +1812,8 @@ static void pq_init(int node, int pnode)
 		bcp->queue_last		= pqp + (DEST_Q_SIZE - 1);
 	}
 
-	first = uv_physnodeaddr(pqp);
-	last = uv_physnodeaddr(pqp + (DEST_Q_SIZE - 1));
+	first = uv_gpa_to_offset(uv_gpa(pqp));
+	last = uv_gpa_to_offset(uv_gpa(pqp + (DEST_Q_SIZE - 1)));
 	tail = first;
 	gnode = uv_gpa_to_gnode(uv_gpa(pqp));
 	first = (gnode << UV_PAYLOADQ_GNODE_SHIFT) | tail;
-- 
1.8.5.6

[PATCH 04/10] arch/x86/platform/uv: BAU add generic function pointers

[Andrew Banman]

Many BAU functions have different implementations depending on the UV
version. Rather than switching on the uvhub_version throughout the driver,
we can define a set of operations for each version. This is especially
beneficial for UV4, which will require many new MMR read/write functions.

Currently, the set of abstracted functions are the same for UV1, UV2, and
UV3. The functions were chosen because each one will have a different
implementation for UV4. Other functions will be added as needed to handle
new implementations or to cleanup the existing differences between UV1,
UV2, and UV3, i.e. read_status and wait_completion.

Signed-off-by: Andrew Banman <abanman@sgi.com>
Acked-by: Mike Travis <travis@sgi.com>
Acked-by: Dimitri Sivanich <sivanich@sgi.com>
---
 arch/x86/include/asm/uv/uv_bau.h | 11 +++++++++++
 arch/x86/platform/uv/tlb_uv.c    | 19 +++++++++++++++++++
 2 files changed, 30 insertions(+)

diff --git a/arch/x86/include/asm/uv/uv_bau.h b/arch/x86/include/asm/uv/uv_bau.h
index a46f270..a7a93a5 100644
--- a/arch/x86/include/asm/uv/uv_bau.h
+++ b/arch/x86/include/asm/uv/uv_bau.h
@@ -385,6 +385,17 @@ struct uv2_3_bau_msg_header {
 	/* bits 127:120 */
 };
 
+/* Abstracted BAU functions */
+struct bau_operations {
+	unsigned long (*read_l_sw_ack)(void);
+	unsigned long (*read_g_sw_ack)(int pnode);
+	unsigned long (*bau_gpa_to_offset)(unsigned long vaddr);
+	void (*write_l_sw_ack)(unsigned long mmr);
+	void (*write_g_sw_ack)(int pnode, unsigned long mmr);
+	void (*write_payload_first)(int pnode, unsigned long mmr);
+	void (*write_payload_last)(int pnode, unsigned long mmr);
+};
+
 /*
  * The activation descriptor:
  * The format of the message to send, plus all accompanying control
diff --git a/arch/x86/platform/uv/tlb_uv.c b/arch/x86/platform/uv/tlb_uv.c
index 98f1d43..045da97 100644
--- a/arch/x86/platform/uv/tlb_uv.c
+++ b/arch/x86/platform/uv/tlb_uv.c
@@ -24,6 +24,18 @@
 #include <asm/irq_vectors.h>
 #include <asm/timer.h>
 
+static struct bau_operations ops;
+
+static struct bau_operations uv123_bau_ops = {
+	.bau_gpa_to_offset = uv_gpa_to_offset,
+	.read_l_sw_ack = read_mmr_sw_ack,
+	.read_g_sw_ack = read_gmmr_sw_ack,
+	.write_l_sw_ack = write_mmr_sw_ack,
+	.write_g_sw_ack = write_gmmr_sw_ack,
+	.write_payload_first = write_mmr_payload_first,
+	.write_payload_last = write_mmr_payload_last,
+};
+
 /* timeouts in nanoseconds (indexed by UVH_AGING_PRESCALE_SEL urgency7 30:28) */
 static int timeout_base_ns[] = {
 		20,
@@ -2135,6 +2147,13 @@ static int __init uv_bau_init(void)
 	if (!is_uv_system())
 		return 0;
 
+	if (is_uv3_hub())
+		ops = uv123_bau_ops;
+	else if (is_uv2_hub())
+		ops = uv123_bau_ops;
+	else if (is_uv1_hub())
+		ops = uv123_bau_ops;
+
 	for_each_possible_cpu(cur_cpu) {
 		mask = &per_cpu(uv_flush_tlb_mask, cur_cpu);
 		zalloc_cpumask_var_node(mask, GFP_KERNEL, cpu_to_node(cur_cpu));
-- 
1.8.5.6

[PATCH 05/10] arch/x86/platform/uv: BAU use generic function pointers

[Andrew Banman]

Convert the use of UV version-specific functions to their abstracted
counterparts.

Signed-off-by: Andrew Banman <abanman@sgi.com>
Acked-by: Mike Travis <travis@sgi.com>
Acked-by: Dimitri Sivanich <sivanich@sgi.com>
---
 arch/x86/platform/uv/tlb_uv.c | 26 +++++++++++++-------------
 1 file changed, 13 insertions(+), 13 deletions(-)

diff --git a/arch/x86/platform/uv/tlb_uv.c b/arch/x86/platform/uv/tlb_uv.c
index 045da97..bfeb382 100644
--- a/arch/x86/platform/uv/tlb_uv.c
+++ b/arch/x86/platform/uv/tlb_uv.c
@@ -228,7 +228,7 @@ static void reply_to_message(struct msg_desc *mdp, struct bau_control *bcp,
 	msg = mdp->msg;
 	if (!msg->canceled && do_acknowledge) {
 		dw = (msg->swack_vec << UV_SW_ACK_NPENDING) | msg->swack_vec;
-		write_mmr_sw_ack(dw);
+		ops.write_l_sw_ack(dw);
 	}
 	msg->replied_to = 1;
 	msg->swack_vec = 0;
@@ -264,7 +264,7 @@ static void bau_process_retry_msg(struct msg_desc *mdp,
 			msg->swack_vec) == 0) &&
 		    (msg2->sending_cpu == msg->sending_cpu) &&
 		    (msg2->msg_type != MSG_NOOP)) {
-			mmr = read_mmr_sw_ack();
+			mmr = ops.read_l_sw_ack();
 			msg_res = msg2->swack_vec;
 			/*
 			 * This is a message retry; clear the resources held
@@ -282,7 +282,7 @@ static void bau_process_retry_msg(struct msg_desc *mdp,
 				stat->d_canceled++;
 				cancel_count++;
 				mr = (msg_res << UV_SW_ACK_NPENDING) | msg_res;
-				write_mmr_sw_ack(mr);
+				ops.write_l_sw_ack(mr);
 			}
 		}
 	}
@@ -415,12 +415,12 @@ static void do_reset(void *ptr)
 			/*
 			 * only reset the resource if it is still pending
 			 */
-			mmr = read_mmr_sw_ack();
+			mmr = ops.read_l_sw_ack();
 			msg_res = msg->swack_vec;
 			mr = (msg_res << UV_SW_ACK_NPENDING) | msg_res;
 			if (mmr & msg_res) {
 				stat->d_rcanceled++;
-				write_mmr_sw_ack(mr);
+				ops.write_l_sw_ack(mr);
 			}
 		}
 	}
@@ -1214,7 +1214,7 @@ void process_uv2_message(struct msg_desc *mdp, struct bau_control *bcp)
 	struct bau_pq_entry *msg = mdp->msg;
 	struct bau_pq_entry *other_msg;
 
-	mmr_image = read_mmr_sw_ack();
+	mmr_image = ops.read_l_sw_ack();
 	swack_vec = msg->swack_vec;
 
 	if ((swack_vec & mmr_image) == 0) {
@@ -1443,7 +1443,7 @@ static int ptc_seq_show(struct seq_file *file, void *data)
 		/* destination side statistics */
 		seq_printf(file,
 			"%lx %ld %ld %ld %ld %ld %ld %ld %ld %ld %ld %ld\n",
-			   read_gmmr_sw_ack(uv_cpu_to_pnode(cpu)),
+			   ops.read_g_sw_ack(uv_cpu_to_pnode(cpu)),
 			   stat->d_requestee, cycles_2_us(stat->d_time),
 			   stat->d_alltlb, stat->d_onetlb, stat->d_multmsg,
 			   stat->d_nomsg, stat->d_retries, stat->d_canceled,
@@ -1737,7 +1737,7 @@ static void activation_descriptor_init(int node, int pnode, int base_pnode)
 
 	gpa = uv_gpa(bau_desc);
 	n = uv_gpa_to_gnode(gpa);
-	m = uv_gpa_to_offset(gpa);
+	m = ops.bau_gpa_to_offset(gpa);
 	if (is_uv1_hub())
 		uv1 = 1;
 
@@ -1824,16 +1824,16 @@ static void pq_init(int node, int pnode)
 		bcp->queue_last		= pqp + (DEST_Q_SIZE - 1);
 	}
 
-	first = uv_gpa_to_offset(uv_gpa(pqp));
-	last = uv_gpa_to_offset(uv_gpa(pqp + (DEST_Q_SIZE - 1)));
+	first = ops.bau_gpa_to_offset(uv_gpa(pqp));
+	last = ops.bau_gpa_to_offset(uv_gpa(pqp + (DEST_Q_SIZE - 1)));
 	tail = first;
 	gnode = uv_gpa_to_gnode(uv_gpa(pqp));
 	first = (gnode << UV_PAYLOADQ_GNODE_SHIFT) | tail;
 
-	write_mmr_payload_first(pnode, first);
-	write_mmr_payload_last(pnode, last);
 	write_mmr_payload_tail(pnode, tail);
-	write_gmmr_sw_ack(pnode, 0xffffUL);
+	ops.write_payload_first(pnode, first);
+	ops.write_payload_last(pnode, last);
+	ops.write_g_sw_ack(pnode, 0xffffUL);
 
 	/* in effect, all msg_type's are set to MSG_NOOP */
 	memset(pqp, 0, sizeof(struct bau_pq_entry) * DEST_Q_SIZE);
-- 
1.8.5.6

[PATCH 06/10] arch/x86/platform/uv: BAU UV4 populate uvhub_version

[Andrew Banman]

Signed-off-by: Andrew Banman <abanman@sgi.com>
Acked-by: Mike Travis <travis@sgi.com>
Acked-by: Dimitri Sivanich <sivanich@sgi.com>
---
 arch/x86/platform/uv/tlb_uv.c | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/arch/x86/platform/uv/tlb_uv.c b/arch/x86/platform/uv/tlb_uv.c
index bfeb382..67a31d0 100644
--- a/arch/x86/platform/uv/tlb_uv.c
+++ b/arch/x86/platform/uv/tlb_uv.c
@@ -2044,8 +2044,10 @@ static int scan_sock(struct socket_desc *sdp, struct uvhub_desc *bdp,
 			bcp->uvhub_version = 2;
 		else if (is_uv3_hub())
 			bcp->uvhub_version = 3;
+		else if (is_uv4_hub())
+			bcp->uvhub_version = 4;
 		else {
-			pr_emerg("uvhub version not 1, 2, or 3\n");
+			pr_emerg("uvhub version not 1, 2, 3, or 4\n");
 			return 1;
 		}
 		bcp->uvhub_master = *hmasterp;
-- 
1.8.5.6

[PATCH 07/10] arch/x86/platform/uv: BAU UV4 disable software timeout

[Andrew Banman]

Software timeouts are not currently supported on BAU for UV4. Instead, the
BAU will rely on hardware-level fairness protocols to determine broadcast
timeouts.

Do not call enable_timeouts or calculate_destination_timeout on UV4. These
functions write to pre-UV4 MMRs so they generate error messages on UV4.

Signed-off-by: Andrew Banman <abanman@sgi.com>
Acked-by: Mike Travis <travis@sgi.com>
Acked-by: Dimitri Sivanich <sivanich@sgi.com>
---
 arch/x86/platform/uv/tlb_uv.c | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/arch/x86/platform/uv/tlb_uv.c b/arch/x86/platform/uv/tlb_uv.c
index 67a31d0..bfd03db 100644
--- a/arch/x86/platform/uv/tlb_uv.c
+++ b/arch/x86/platform/uv/tlb_uv.c
@@ -2109,7 +2109,8 @@ static int __init init_per_cpu(int nuvhubs, int base_part_pnode)
 	void *vp;
 	struct uvhub_desc *uvhub_descs;
 
-	timeout_us = calculate_destination_timeout();
+	if (is_uv3_hub() || is_uv2_hub() || is_uv1_hub())
+		timeout_us = calculate_destination_timeout();
 
 	vp = kmalloc(nuvhubs * sizeof(struct uvhub_desc), GFP_KERNEL);
 	uvhub_descs = (struct uvhub_desc *)vp;
@@ -2171,7 +2172,9 @@ static int __init uv_bau_init(void)
 			uv_base_pnode = uv_blade_to_pnode(uvhub);
 	}
 
-	enable_timeouts();
+	/* software timeouts are not supported on UV4 */
+	if (is_uv3_hub() || is_uv2_hub() || is_uv1_hub())
+		enable_timeouts();
 
 	if (init_per_cpu(nuvhubs, uv_base_pnode)) {
 		set_bau_off();
-- 
1.8.5.6

[PATCH 08/10] arch/x86/platform/uv: BAU UV4 fix payload queue setup

[Andrew Banman]

The BAU on UV4 does not need to maintain the payload queue tail pointer. Do
not initialize the tail pointer MMR on UV4.

Note that write_payload_tail is not an abstracted BAU function since it is
an operation specific to pre-UV4 versions. Then we must switch on the UV
version to control its usage, for which we use uvhub_version rather than
is_uv*_hub because it is quicker/more concise.

Signed-off-by: Andrew Banman <abanman@sgi.com>
Acked-by: Mike Travis <travis@sgi.com>
Acked-by: Dimitri Sivanich <sivanich@sgi.com>
---
 arch/x86/platform/uv/tlb_uv.c | 16 ++++++++++++----
 1 file changed, 12 insertions(+), 4 deletions(-)

diff --git a/arch/x86/platform/uv/tlb_uv.c b/arch/x86/platform/uv/tlb_uv.c
index bfd03db..56d12eb 100644
--- a/arch/x86/platform/uv/tlb_uv.c
+++ b/arch/x86/platform/uv/tlb_uv.c
@@ -1826,11 +1826,19 @@ static void pq_init(int node, int pnode)
 
 	first = ops.bau_gpa_to_offset(uv_gpa(pqp));
 	last = ops.bau_gpa_to_offset(uv_gpa(pqp + (DEST_Q_SIZE - 1)));
-	tail = first;
-	gnode = uv_gpa_to_gnode(uv_gpa(pqp));
-	first = (gnode << UV_PAYLOADQ_GNODE_SHIFT) | tail;
 
-	write_mmr_payload_tail(pnode, tail);
+	/*
+	 * Pre UV4, the gnode is required to locate the payload queue
+	 * and the payload queue tail must be maintained by the kernel.
+	 */
+	bcp = &per_cpu(bau_control, smp_processor_id());
+	if (bcp->uvhub_version <= 3) {
+		tail = first;
+		gnode = uv_gpa_to_gnode(uv_gpa(pqp));
+		first = (gnode << UV_PAYLOADQ_GNODE_SHIFT) | tail;
+		write_mmr_payload_tail(pnode, tail);
+	}
+
 	ops.write_payload_first(pnode, first);
 	ops.write_payload_last(pnode, last);
 	ops.write_g_sw_ack(pnode, 0xffffUL);
-- 
1.8.5.6

[PATCH 09/10] arch/x86/platform/uv: BAU UV4 add version-specific functions

[Andrew Banman]

Add the UV4-specific function definitions and define an operations struct
to implement them in the BAU driver.

Many BAU MMRs, although functionally the same, have new addresses on UV4
due to hardware changes. Each MMR requires new read/write functions, but
their implementation in the driver does not change. Thus, it is enough to
enumerate them in the operations struct for the changes to take effect.

Signed-off-by: Andrew Banman <abanman@sgi.com>
Acked-by: Mike Travis <travis@sgi.com>
Acked-by: Dimitri Sivanich <sivanich@sgi.com>
---
 arch/x86/include/asm/uv/uv_bau.h | 30 ++++++++++++++++++++++++++++++
 arch/x86/platform/uv/tlb_uv.c    | 15 ++++++++++++++-
 2 files changed, 44 insertions(+), 1 deletion(-)

diff --git a/arch/x86/include/asm/uv/uv_bau.h b/arch/x86/include/asm/uv/uv_bau.h
index a7a93a5..57ab86d 100644
--- a/arch/x86/include/asm/uv/uv_bau.h
+++ b/arch/x86/include/asm/uv/uv_bau.h
@@ -664,6 +664,16 @@ static inline void write_gmmr_activation(int pnode, unsigned long mmr_image)
 	write_gmmr(pnode, UVH_LB_BAU_SB_ACTIVATION_CONTROL, mmr_image);
 }
 
+static inline void write_mmr_proc_payload_first(int pnode, unsigned long mmr_image)
+{
+	write_gmmr(pnode, UV4H_LB_PROC_INTD_QUEUE_FIRST, mmr_image);
+}
+
+static inline void write_mmr_proc_payload_last(int pnode, unsigned long mmr_image)
+{
+	write_gmmr(pnode, UV4H_LB_PROC_INTD_QUEUE_LAST, mmr_image);
+}
+
 static inline void write_mmr_payload_first(int pnode, unsigned long mmr_image)
 {
 	write_gmmr(pnode, UVH_LB_BAU_INTD_PAYLOAD_QUEUE_FIRST, mmr_image);
@@ -709,6 +719,26 @@ static inline unsigned long read_gmmr_sw_ack(int pnode)
 	return read_gmmr(pnode, UVH_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE);
 }
 
+static inline void write_mmr_proc_sw_ack(unsigned long mr)
+{
+	uv_write_local_mmr(UV4H_LB_PROC_INTD_SOFT_ACK_CLEAR, mr);
+}
+
+static inline void write_gmmr_proc_sw_ack(int pnode, unsigned long mr)
+{
+	write_gmmr(pnode, UV4H_LB_PROC_INTD_SOFT_ACK_CLEAR, mr);
+}
+
+static inline unsigned long read_mmr_proc_sw_ack(void)
+{
+	return read_lmmr(UV4H_LB_PROC_INTD_SOFT_ACK_PENDING);
+}
+
+static inline unsigned long read_gmmr_proc_sw_ack(int pnode)
+{
+	return read_gmmr(pnode, UV4H_LB_PROC_INTD_SOFT_ACK_PENDING);
+}
+
 static inline void write_mmr_data_config(int pnode, unsigned long mr)
 {
 	uv_write_global_mmr64(pnode, UVH_BAU_DATA_CONFIG, mr);
diff --git a/arch/x86/platform/uv/tlb_uv.c b/arch/x86/platform/uv/tlb_uv.c
index 56d12eb..470d73c 100644
--- a/arch/x86/platform/uv/tlb_uv.c
+++ b/arch/x86/platform/uv/tlb_uv.c
@@ -36,6 +36,17 @@ static struct bau_operations uv123_bau_ops = {
 	.write_payload_last = write_mmr_payload_last,
 };
 
+static struct bau_operations uv4_bau_ops = {
+	.bau_gpa_to_offset = uv_gpa_to_soc_phys_ram,
+	.read_l_sw_ack = read_mmr_proc_sw_ack,
+	.read_g_sw_ack = read_gmmr_proc_sw_ack,
+	.write_l_sw_ack = write_mmr_proc_sw_ack,
+	.write_g_sw_ack = write_gmmr_proc_sw_ack,
+	.write_payload_first = write_mmr_proc_payload_first,
+	.write_payload_last = write_mmr_proc_payload_last,
+};
+
+
 /* timeouts in nanoseconds (indexed by UVH_AGING_PRESCALE_SEL urgency7 30:28) */
 static int timeout_base_ns[] = {
 		20,
@@ -2158,7 +2169,9 @@ static int __init uv_bau_init(void)
 	if (!is_uv_system())
 		return 0;
 
-	if (is_uv3_hub())
+	if (is_uv4_hub())
+		ops = uv4_bau_ops;
+	else if (is_uv3_hub())
 		ops = uv123_bau_ops;
 	else if (is_uv2_hub())
 		ops = uv123_bau_ops;
-- 
1.8.5.6

[PATCH 10/10] Apply Mike's UV4 fixup patches

[Andrew Banman]

uv4_fix_gam_range_lookup_table
uv4_fix_bios_init_error
uv4_remove_uvsystab_pxm
uv4_disable_is_uv_system
---
 arch/x86/include/asm/uv/bios.h     |  5 +++--
 arch/x86/kernel/apic/x2apic_uv_x.c | 42 ++++++++++++++++++--------------------
 arch/x86/platform/uv/bios_uv.c     |  8 +++++---
 3 files changed, 28 insertions(+), 27 deletions(-)

diff --git a/arch/x86/include/asm/uv/bios.h b/arch/x86/include/asm/uv/bios.h
index c852590..e652a7c 100644
--- a/arch/x86/include/asm/uv/bios.h
+++ b/arch/x86/include/asm/uv/bios.h
@@ -79,7 +79,7 @@ struct uv_gam_range_entry {
 	u16	nasid;		/* HNasid */
 	u16	sockid;		/* Socket ID, high bits of APIC ID */
 	u16	pnode;		/* Index to MMR and GRU spaces */
-	u32	pxm;		/* ACPI proximity domain number */
+	u32	unused2;
 	u32	limit;		/* PA bits 56:26 (UV_GAM_RANGE_SHFT) */
 };
 
@@ -88,7 +88,8 @@ struct uv_gam_range_entry {
 #define	UV_SYSTAB_VERSION_UV4		0x400	/* UV4 BIOS base version */
 #define	UV_SYSTAB_VERSION_UV4_1		0x401	/* + gpa_shift */
 #define	UV_SYSTAB_VERSION_UV4_2		0x402	/* + TYPE_NVRAM/WINDOW/MBOX */
-#define	UV_SYSTAB_VERSION_UV4_LATEST	UV_SYSTAB_VERSION_UV4_2
+#define	UV_SYSTAB_VERSION_UV4_3		0x403	/* - GAM Range PXM Value */
+#define	UV_SYSTAB_VERSION_UV4_LATEST	UV_SYSTAB_VERSION_UV4_3
 
 #define	UV_SYSTAB_TYPE_UNUSED		0	/* End of table (offset == 0) */
 #define	UV_SYSTAB_TYPE_GAM_PARAMS	1	/* GAM PARAM conversions */
diff --git a/arch/x86/kernel/apic/x2apic_uv_x.c b/arch/x86/kernel/apic/x2apic_uv_x.c
index f5e943b..b07161d 100644
--- a/arch/x86/kernel/apic/x2apic_uv_x.c
+++ b/arch/x86/kernel/apic/x2apic_uv_x.c
@@ -225,6 +225,11 @@ pr_crit("UV: OEM_IDS %s/%s\n", oem_id, oem_table_id);
 	if (strncmp(oem_id, "SGI", 3) != 0)
 		return 0;
 
+	if (numa_off) {
+		pr_err("UV: NUMA is off, disabling UV support\n");
+		return 0;
+	}
+
 	/* Setup early hub type field in uv_hub_info for Node 0 */
 	uv_cpu_info->p_uv_hub_info = &uv_hub_info_node0;
 
@@ -327,7 +332,7 @@ static __init void build_uv_gr_table(void)
 	struct uv_gam_range_entry *gre = uv_gre_table;
 	struct uv_gam_range_s *grt;
 	unsigned long last_limit = 0, ram_limit = 0;
-	int bytes, i, sid, lsid = -1;
+	int bytes, i, sid, lsid = -1, indx = 0, lindx = -1;
 
 	if (!gre)
 		return;
@@ -358,11 +363,12 @@ static __init void build_uv_gr_table(void)
 		}
 		sid = gre->sockid - _min_socket;
 		if (lsid < sid) {		/* new range */
-			grt = &_gr_table[sid];
-			grt->base = lsid;
+			grt = &_gr_table[indx];
+			grt->base = lindx;
 			grt->nasid = gre->nasid;
 			grt->limit = last_limit = gre->limit;
 			lsid = sid;
+			lindx = indx++;
 			continue;
 		}
 		if (lsid == sid && !ram_limit) {	/* update range */
@@ -373,7 +379,7 @@ static __init void build_uv_gr_table(void)
 		}
 		if (!ram_limit) {		/* non-contiguous ram range */
 			grt++;
-			grt->base = sid - 1;
+			grt->base = lindx;
 			grt->nasid = gre->nasid;
 			grt->limit = last_limit = gre->limit;
 			continue;
@@ -1163,19 +1169,18 @@ static void __init decode_gam_rng_tbl(unsigned long ptr)
 	for (; gre->type != UV_GAM_RANGE_TYPE_UNUSED; gre++) {
 		if (!index) {
 			pr_info("UV: GAM Range Table...\n");
-			pr_info("UV:  # %20s %14s %5s %4s %5s %3s %2s %3s\n",
+			pr_info("UV:  # %20s %14s %5s %4s %5s %3s %2s\n",
 				"Range", "", "Size", "Type", "NASID",
-				"SID", "PN", "PXM");
+				"SID", "PN");
 		}
 		pr_info(
-		"UV: %2d: 0x%014lx-0x%014lx %5luG %3d   %04x  %02x %02x %3d\n",
+		"UV: %2d: 0x%014lx-0x%014lx %5luG %3d   %04x  %02x %02x\n",
 			index++,
 			(unsigned long)lgre << UV_GAM_RANGE_SHFT,
 			(unsigned long)gre->limit << UV_GAM_RANGE_SHFT,
 			((unsigned long)(gre->limit - lgre)) >>
 				(30 - UV_GAM_RANGE_SHFT), /* 64M -> 1G */
-			gre->type, gre->nasid, gre->sockid,
-			gre->pnode, gre->pxm);
+			gre->type, gre->nasid, gre->sockid, gre->pnode);
 
 		lgre = gre->limit;
 		if (sock_min > gre->sockid)
@@ -1294,7 +1299,7 @@ static void __init build_socket_tables(void)
 		_pnode_to_socket[i] = SOCK_EMPTY;
 
 	/* fill in pnode/node/addr conversion list values */
-	pr_info("UV: GAM Building socket/pnode/pxm conversion tables\n");
+	pr_info("UV: GAM Building socket/pnode conversion tables\n");
 	for (; gre->type != UV_GAM_RANGE_TYPE_UNUSED; gre++) {
 		if (gre->type == UV_GAM_RANGE_TYPE_HOLE)
 			continue;
@@ -1302,20 +1307,18 @@ static void __init build_socket_tables(void)
 		if (_socket_to_pnode[i] != SOCK_EMPTY)
 			continue;	/* duplicate */
 		_socket_to_pnode[i] = gre->pnode;
-		_socket_to_node[i] = gre->pxm;
 
 		i = gre->pnode - minpnode;
 		_pnode_to_socket[i] = gre->sockid;
 
 		pr_info(
-		"UV: sid:%02x type:%d nasid:%04x pn:%02x pxm:%2d pn2s:%2x\n",
+		"UV: sid:%02x type:%d nasid:%04x pn:%02x pn2s:%2x\n",
 			gre->sockid, gre->type, gre->nasid,
 			_socket_to_pnode[gre->sockid - minsock],
-			_socket_to_node[gre->sockid - minsock],
 			_pnode_to_socket[gre->pnode - minpnode]);
 	}
 
-	/* check socket -> node values */
+	/* Set socket -> node values */
 	lnid = -1;
 	for_each_present_cpu(cpu) {
 		int nid = cpu_to_node(cpu);
@@ -1326,14 +1329,9 @@ static void __init build_socket_tables(void)
 		lnid = nid;
 		apicid = per_cpu(x86_cpu_to_apicid, cpu);
 		sockid = apicid >> uv_cpuid.socketid_shift;
-		i = sockid - minsock;
-
-		if (nid != _socket_to_node[i]) {
-			pr_warn(
-			"UV: %02x: type:%d socket:%02x PXM:%02x != node:%2d\n",
-				i, sockid, gre->type, _socket_to_node[i], nid);
-			_socket_to_node[i] = nid;
-		}
+		_socket_to_node[sockid - minsock] = nid;
+		pr_info("UV: sid:%02x: apicid:%04x node:%2d\n",
+			sockid, apicid, nid);
 	}
 
 	/* Setup physical blade to pnode translation from GAM Range Table */
diff --git a/arch/x86/platform/uv/bios_uv.c b/arch/x86/platform/uv/bios_uv.c
index 815fec6..ccd2603 100644
--- a/arch/x86/platform/uv/bios_uv.c
+++ b/arch/x86/platform/uv/bios_uv.c
@@ -200,12 +200,14 @@ void uv_bios_init(void)
 		return;
 	}
 
+	/* Starting with UV4 the UV systab size is variable */
 	if (uv_systab->revision >= UV_SYSTAB_VERSION_UV4) {
+		int size = uv_systab->size;
+
 		iounmap(uv_systab);
-		uv_systab = ioremap(efi.uv_systab, uv_systab->size);
+		uv_systab = ioremap(efi.uv_systab, size);
 		if (!uv_systab) {
-			pr_err("UV: UVsystab: ioremap(%d) failed!\n",
-				uv_systab->size);
+			pr_err("UV: UVsystab: ioremap(%d) failed!\n", size);
 			return;
 		}
 	}
-- 
1.8.5.6

Re: your mail

[andrew banman]

Subject line got dropped the first time around. Will send again.

Apologies for the chatter,

Andrew

On Tue, Sep 20, 2016 at 05:21:06PM -0500, Andrew Banman wrote:
> From Andrew Banman <abanman@sgi.com> # This line is ignored.
> From: Andrew Banman <abanman@sgi.com>
> Subject: [PATCH 0/9] arch/x86/platform/uv: add UV4 support to BAU
> In-Reply-To: 
> 
> The following patch set adds support for UV4 architecture to the Broadcast
> Assist Unit (BAU). Major hardware changes to the BAU require these fixes to
> ensure correct operation and to avoid illegal MMR writes.
> 
>  arch/x86/include/asm/uv/uv_bau.h |  45 ++----------------------------
>  arch/x86/platform/uv/tlb_uv.c    | 114 ++++++++++++++++++++++++---------------------------------------
> -------------
> 
> The patch set can be thought of in three logical groups:
> 
> 1) General cleanup.
> 
>  [PATCH 1/9] arch/x86/platform/uv: BAU cleanup: update printks
>  [PATCH 2/9] arch/x86/platform/uv: BAU cleanup: pq_init
>  [PATCH 3/9] arch/x86/platform/uv: BAU replace uv_physnodeaddr
> 
>  These housekeeping patches make the subsequent UV4 patches clearer,
>  and they should be done in any case.
> 
> 
> 2) Implement a new scheme to abstract UV version-specific functions.
> 
>  [PATCH 4/9] arch/x86/platform/uv: BAU add generic function pointers
>  [PATCH 5/9] arch/x86/platform/uv: BAU use generic function pointers
> 
>  We add a struct of function pointers to define version-specific BAU
>  operations. The philosophy is to abstract functions that perform the same
>  operation on all UV versions but have different implementations. This will
>  simplify their use in the body of the driver code and greatly simplify the
>  UV4 patches to follow.
> 
> 
> 3) Add UV4 functionality.
> 
>  [PATCH 6/9] arch/x86/platform/uv: BAU UV4 populate uvhub_version
>  [PATCH 7/9] arch/x86/platform/uv: BAU UV4 disable software timeout
>  [PATCH 8/9] arch/x86/platform/uv: BAU UV4 fix payload queue setup
>  [PATCH 9/9] arch/x86/platform/uv: BAU UV4 add version-specific
> 
>  These patches feature a minimal set of changes to make the BAU on UV4
>  operational.
> 
> 
> This patch set has been tested for regressions on pre-UV4 architectures and
> for correct functionality on UV4. The patches apply cleanly to 4.8-rc7.
> Fine-tuned performance tweaking for UV4 will come in a future patch set.
> 
> 
> Thank you,
> 
> Andrew Banman