[Patch 00/25] GRU - GRU Updates - Production Driver

[Patch 00/25] GRU - GRU Updates - Production Driver

[steiner]

This patch set contains a collection of patches to the SGI UV GRU
driver. These patches fix various bugs found in the testing
of the driver on real hardware.

No changes are made to non-UV files.

[Patch 01/25] GRU - delete obsolete gru instruction opcodes

[steiner]

[-- Attachment #1: uv_gru_no_poll --]
[-- Type: text/plain, Size: 999 bytes --]

From: Jack Steiner <steiner@sgi.com>

Update the GRU instruction macros to match what is actually
implemented in initial silicon.

Signed-off-by: Jack Steiner <steiner@sgi.com>


---
 drivers/misc/sgi-gru/gru_instructions.h |    2 --
 1 file changed, 2 deletions(-)

Index: linux/drivers/misc/sgi-gru/gru_instructions.h
===================================================================
--- linux.orig/drivers/misc/sgi-gru/gru_instructions.h	2010-06-09 08:11:31.528038315 -0500
+++ linux/drivers/misc/sgi-gru/gru_instructions.h	2010-06-09 08:11:38.907960603 -0500
@@ -217,8 +217,6 @@ struct gru_instruction {
 #define EOP_ERR_XOR	0x03 /* Logical XOR with memory */
 #define EOP_ERR_ADD	0x04 /* Add value to memory */
 #define EOP_ERR_CSWAP	0x08 /* Compare with operand2, write operand1 if match*/
-#define EOP_ERR_EPOLL	0x09 /* Poll for equality */
-#define EOP_ERR_NPOLL	0x0a /* Poll for inequality */
 
 /* GAMXR - SGI Arithmetic unit */
 #define EOP_XR_CSWAP	0x0b /* Masked compare exchange */

[Patch 02/25] GRU - skip gru tlb purging of gru contexts:w

[steiner]

[-- Attachment #1: uv_gru_no_tlb_purge_gseg --]
[-- Type: text/plain, Size: 5566 bytes --]

From: Jack Steiner <steiner@sgi.com>

Skip purging the GRU TLB if the purge request is for the GRU
itself. The driver will not create TLB mappings for vmas
that map the GRU address space. Purging them is unnecessary.

This is only an optimization and is not functionally required. However, if
GRU resources are oversubscribed (unusual except for stress tests),
purging can be frequent.

Signed-off-by: Jack Steiner <steiner@sgi.com>


---
 drivers/misc/sgi-gru/grufault.c    |    8 ++++----
 drivers/misc/sgi-gru/grufile.c     |    2 +-
 drivers/misc/sgi-gru/gruprocfs.c   |    1 +
 drivers/misc/sgi-gru/grutables.h   |    3 ++-
 drivers/misc/sgi-gru/grutlbpurge.c |   11 +++++++++--
 5 files changed, 17 insertions(+), 8 deletions(-)

Index: linux/drivers/misc/sgi-gru/grufault.c
===================================================================
--- linux.orig/drivers/misc/sgi-gru/grufault.c	2010-06-09 08:11:31.480037430 -0500
+++ linux/drivers/misc/sgi-gru/grufault.c	2010-06-09 08:11:39.747460083 -0500
@@ -57,11 +57,11 @@ static inline int is_gru_paddr(unsigned
 /*
  * Find the vma of a GRU segment. Caller must hold mmap_sem.
  */
-struct vm_area_struct *gru_find_vma(unsigned long vaddr)
+struct vm_area_struct *gru_find_vma(struct mm_struct *mm, unsigned long vaddr)
 {
 	struct vm_area_struct *vma;
 
-	vma = find_vma(current->mm, vaddr);
+	vma = find_vma(mm, vaddr);
 	if (vma && vma->vm_start <= vaddr && vma->vm_ops == &gru_vm_ops)
 		return vma;
 	return NULL;
@@ -82,7 +82,7 @@ static struct gru_thread_state *gru_find
 	struct gru_thread_state *gts = NULL;
 
 	down_read(&mm->mmap_sem);
-	vma = gru_find_vma(vaddr);
+	vma = gru_find_vma(current->mm, vaddr);
 	if (vma)
 		gts = gru_find_thread_state(vma, TSID(vaddr, vma));
 	if (gts)
@@ -99,7 +99,7 @@ static struct gru_thread_state *gru_allo
 	struct gru_thread_state *gts = ERR_PTR(-EINVAL);
 
 	down_write(&mm->mmap_sem);
-	vma = gru_find_vma(vaddr);
+	vma = gru_find_vma(current->mm, vaddr);
 	if (!vma)
 		goto err;
 
Index: linux/drivers/misc/sgi-gru/grufile.c
===================================================================
--- linux.orig/drivers/misc/sgi-gru/grufile.c	2010-06-09 08:11:31.480037430 -0500
+++ linux/drivers/misc/sgi-gru/grufile.c	2010-06-09 08:11:39.751457793 -0500
@@ -145,7 +145,7 @@ static int gru_create_new_context(unsign
 		req.options |= GRU_OPT_MISS_FMM_INTR;
 
 	down_write(&current->mm->mmap_sem);
-	vma = gru_find_vma(req.gseg);
+	vma = gru_find_vma(current->mm, req.gseg);
 	if (vma) {
 		vdata = vma->vm_private_data;
 		vdata->vd_user_options = req.options;
Index: linux/drivers/misc/sgi-gru/gruprocfs.c
===================================================================
--- linux.orig/drivers/misc/sgi-gru/gruprocfs.c	2010-06-09 08:11:31.480037430 -0500
+++ linux/drivers/misc/sgi-gru/gruprocfs.c	2010-06-09 08:11:39.763413322 -0500
@@ -87,6 +87,7 @@ static int statistics_show(struct seq_fi
 	printstat(s, tlb_dropin_fail_no_exception);
 	printstat(s, tfh_stale_on_fault);
 	printstat(s, mmu_invalidate_range);
+	printstat(s, mmu_invalidate_range_flush);
 	printstat(s, mmu_invalidate_page);
 	printstat(s, flush_tlb);
 	printstat(s, flush_tlb_gru);
Index: linux/drivers/misc/sgi-gru/grutables.h
===================================================================
--- linux.orig/drivers/misc/sgi-gru/grutables.h	2010-06-09 08:11:31.480037430 -0500
+++ linux/drivers/misc/sgi-gru/grutables.h	2010-06-09 08:11:39.779413278 -0500
@@ -213,6 +213,7 @@ struct gru_stats_s {
 	atomic_long_t tlb_dropin_fail_no_exception;
 	atomic_long_t tfh_stale_on_fault;
 	atomic_long_t mmu_invalidate_range;
+	atomic_long_t mmu_invalidate_range_flush;
 	atomic_long_t mmu_invalidate_page;
 	atomic_long_t flush_tlb;
 	atomic_long_t flush_tlb_gru;
@@ -653,7 +654,7 @@ extern int gru_get_exception_detail(unsi
 extern int gru_set_context_option(unsigned long address);
 extern void gru_check_context_placement(struct gru_thread_state *gts);
 extern int gru_cpu_fault_map_id(void);
-extern struct vm_area_struct *gru_find_vma(unsigned long vaddr);
+extern struct vm_area_struct *gru_find_vma(struct mm_struct *mm, unsigned long vaddr);
 extern void gru_flush_all_tlb(struct gru_state *gru);
 extern int gru_proc_init(void);
 extern void gru_proc_exit(void);
Index: linux/drivers/misc/sgi-gru/grutlbpurge.c
===================================================================
--- linux.orig/drivers/misc/sgi-gru/grutlbpurge.c	2010-06-09 08:11:31.480037430 -0500
+++ linux/drivers/misc/sgi-gru/grutlbpurge.c	2010-06-09 08:11:39.795536964 -0500
@@ -226,11 +226,19 @@ static void gru_invalidate_range_start(s
 	struct gru_mm_struct *gms = container_of(mn, struct gru_mm_struct,
 						 ms_notifier);
 
+	if (mm && atomic_read(&mm->mm_users) && unlikely(down_write_trylock(&mm->mmap_sem))) {
+		up_write(&mm->mmap_sem);
+	}
 	STAT(mmu_invalidate_range);
 	atomic_inc(&gms->ms_range_active);
 	gru_dbg(grudev, "gms %p, start 0x%lx, end 0x%lx, act %d\n", gms,
 		start, end, atomic_read(&gms->ms_range_active));
-	gru_flush_tlb_range(gms, start, end - start);
+
+	/* No need to flush if unmapping a GRU context */
+	if (!gru_find_vma(mm, start)) {
+		STAT(mmu_invalidate_range_flush);
+		gru_flush_tlb_range(gms, start, end - start);
+	}
 }
 
 static void gru_invalidate_range_end(struct mmu_notifier *mn,
@@ -242,7 +250,6 @@ static void gru_invalidate_range_end(str
 
 	/* ..._and_test() provides needed barrier */
 	(void)atomic_dec_and_test(&gms->ms_range_active);
-
 	wake_up_all(&gms->ms_wait_queue);
 	gru_dbg(grudev, "gms %p, start 0x%lx, end 0x%lx\n", gms, start, end);
 }

[Patch 03/25] GRU - update gru tlb miss statistics

[steiner]

[-- Attachment #1: uv_gru_tlb_statistics --]
[-- Type: text/plain, Size: 5398 bytes --]

From: Jack Steiner <steiner@sgi.com>

Update the GRU TLB dropin statistics. Add counts to indicate
number of TLB miss interrupts and a separate count of TLB
entries actually written. Also minor code cleanup of the TLB fault path.


Signed-off-by: Jack Steiner <steiner@sgi.com>

---
 drivers/misc/sgi-gru/gru.h       |    3 ++-
 drivers/misc/sgi-gru/grufault.c  |   33 +++++++++++++++++++++++++--------
 drivers/misc/sgi-gru/gruprocfs.c |    1 +
 drivers/misc/sgi-gru/grutables.h |    1 +
 4 files changed, 29 insertions(+), 9 deletions(-)

Index: linux/drivers/misc/sgi-gru/gru.h
===================================================================
--- linux.orig/drivers/misc/sgi-gru/gru.h	2010-06-09 08:11:31.372037434 -0500
+++ linux/drivers/misc/sgi-gru/gru.h	2010-06-09 08:11:40.281083913 -0500
@@ -61,7 +61,8 @@ struct gru_gseg_statistics {
 	unsigned long	upm_tlbmiss;
 	unsigned long	tlbdropin;
 	unsigned long	context_stolen;
-	unsigned long	reserved[10];
+	unsigned long	interrupts;
+	unsigned long	reserved[9];
 };
 
 /* Flags for GRU options on the gru_create_context() call */
Index: linux/drivers/misc/sgi-gru/grufault.c
===================================================================
--- linux.orig/drivers/misc/sgi-gru/grufault.c	2010-06-09 08:11:39.747460083 -0500
+++ linux/drivers/misc/sgi-gru/grufault.c	2010-06-09 08:11:40.285085853 -0500
@@ -289,8 +289,10 @@ static int gru_vtop(struct gru_thread_st
 	return VTOP_SUCCESS;
 
 inval:
+	STAT(tlb_dropin_fail_invalid);
 	return VTOP_INVALID;
 upm:
+	STAT(tlb_dropin_fail_retry);
 	return VTOP_RETRY;
 }
 
@@ -422,7 +424,7 @@ static int gru_try_dropin(struct gru_sta
 	if (ret == VTOP_INVALID)
 		goto failinval;
 	if (ret == VTOP_RETRY)
-		goto failupm;
+		goto failretry;
 
 	if (!(gts->ts_sizeavail & GRU_SIZEAVAIL(pageshift))) {
 		gts->ts_sizeavail |= GRU_SIZEAVAIL(pageshift);
@@ -438,9 +440,9 @@ static int gru_try_dropin(struct gru_sta
 	}
 
 	gru_cb_set_istatus_active(cbk);
-	gts->ustats.tlbdropin++;
 	tfh_write_restart(tfh, gpa, GAA_RAM, vaddr, asid, write,
 			  GRU_PAGESIZE(pageshift));
+	gts->ustats.tlbdropin++;
 	gru_dbg(grudev,
 		"%s: gid %d, gts 0x%p, tfh 0x%p, vaddr 0x%lx, asid 0x%x, indexway 0x%x,"
 		" rw %d, ps %d, gpa 0x%lx\n",
@@ -453,20 +455,33 @@ failnoasid:
 	/* No asid (delayed unload). */
 	STAT(tlb_dropin_fail_no_asid);
 	gru_dbg(grudev, "FAILED no_asid tfh: 0x%p, vaddr 0x%lx\n", tfh, vaddr);
-	if (!cbk)
+	if (atomic)
 		tfh_user_polling_mode(tfh);
 	else
 		gru_flush_cache(tfh);
 	gru_flush_cache_cbe(cbe);
 	return -EAGAIN;
 
+failretry:
+	/* vtop failed - retry */
+	if (atomic)
+		tfh_user_polling_mode(tfh);
+	else
+		gru_flush_cache(tfh);
+	gru_flush_cache_cbe(cbe);
+	gru_dbg(grudev, "FAILED retry tfh: 0x%p, vaddr 0x%lx\n", tfh, vaddr);
+	return -EAGAIN;
+
 failupm:
 	/* Atomic failure switch CBR to UPM */
-	tfh_user_polling_mode(tfh);
+	if (atomic)
+		tfh_user_polling_mode(tfh);
+	else
+		gru_flush_cache(tfh);
 	gru_flush_cache_cbe(cbe);
 	STAT(tlb_dropin_fail_upm);
 	gru_dbg(grudev, "FAILED upm tfh: 0x%p, vaddr 0x%lx\n", tfh, vaddr);
-	return 1;
+	return -EAGAIN;
 
 failfmm:
 	/* FMM state on UPM call */
@@ -501,13 +516,12 @@ failinval:
 	/* All errors (atomic & non-atomic) switch CBR to EXCEPTION state */
 	tfh_exception(tfh);
 	gru_flush_cache_cbe(cbe);
-	STAT(tlb_dropin_fail_invalid);
 	gru_dbg(grudev, "FAILED inval tfh: 0x%p, vaddr 0x%lx\n", tfh, vaddr);
 	return -EFAULT;
 
 failactive:
 	/* Range invalidate active. Switch to UPM iff atomic */
-	if (!cbk)
+	if (atomic)
 		tfh_user_polling_mode(tfh);
 	else
 		gru_flush_cache(tfh);
@@ -531,7 +545,7 @@ static irqreturn_t gru_intr(int chiplet,
 	struct gru_thread_state *gts;
 	struct gru_tlb_fault_handle *tfh = NULL;
 	struct completion *cmp;
-	int cbrnum, ctxnum;
+	int cbrnum, ctxnum, multi = 0;
 
 	STAT(intr);
 
@@ -581,7 +595,10 @@ static irqreturn_t gru_intr(int chiplet,
 		 * This is running in interrupt context. Trylock the mmap_sem.
 		 * If it fails, retry the fault in user context.
 		 */
+		if (!multi)
+			gts->ustats.interrupts++;
 		gts->ustats.fmm_tlbmiss++;
+		multi = 1;
 		if (!gts->ts_force_cch_reload &&
 					down_read_trylock(&gts->ts_mm->mmap_sem)) {
 			gru_try_dropin(gru, gts, tfh, NULL);
Index: linux/drivers/misc/sgi-gru/gruprocfs.c
===================================================================
--- linux.orig/drivers/misc/sgi-gru/gruprocfs.c	2010-06-09 08:11:39.763413322 -0500
+++ linux/drivers/misc/sgi-gru/gruprocfs.c	2010-06-09 08:11:40.301082823 -0500
@@ -79,6 +79,7 @@ static int statistics_show(struct seq_fi
 	printstat(s, tlb_dropin);
 	printstat(s, tlb_preload_page);
 	printstat(s, tlb_dropin_fail_no_asid);
+	printstat(s, tlb_dropin_fail_retry);
 	printstat(s, tlb_dropin_fail_upm);
 	printstat(s, tlb_dropin_fail_invalid);
 	printstat(s, tlb_dropin_fail_range_active);
Index: linux/drivers/misc/sgi-gru/grutables.h
===================================================================
--- linux.orig/drivers/misc/sgi-gru/grutables.h	2010-06-09 08:11:39.779413278 -0500
+++ linux/drivers/misc/sgi-gru/grutables.h	2010-06-09 08:11:40.315537614 -0500
@@ -205,6 +205,7 @@ struct gru_stats_s {
 	atomic_long_t tlb_dropin;
 	atomic_long_t tlb_preload_page;
 	atomic_long_t tlb_dropin_fail_no_asid;
+	atomic_long_t tlb_dropin_fail_retry;
 	atomic_long_t tlb_dropin_fail_upm;
 	atomic_long_t tlb_dropin_fail_invalid;
 	atomic_long_t tlb_dropin_fail_range_active;

[Patch 04/25] GRU - mmap gru contexts using nonlinear

[steiner]

[-- Attachment #1: uv_gru_nonlinear --]
[-- Type: text/plain, Size: 774 bytes --]

From: Jack Steiner <steiner@sgi.com>

Add VM_NONLINEAR to the vm_flags used to map the GRU.

Signed-off-by: Jack Steiner <steiner@sgi.com>

---
 drivers/misc/sgi-gru/grufile.c |    2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

Index: linux/drivers/misc/sgi-gru/grufile.c
===================================================================
--- linux.orig/drivers/misc/sgi-gru/grufile.c	2010-06-09 08:11:39.751457793 -0500
+++ linux/drivers/misc/sgi-gru/grufile.c	2010-06-09 08:11:40.740099769 -0500
@@ -110,7 +110,7 @@ static int gru_file_mmap(struct file *fi
 
 	vma->vm_flags |=
 	    (VM_IO | VM_DONTCOPY | VM_LOCKED | VM_DONTEXPAND | VM_PFNMAP |
-			VM_RESERVED);
+			VM_RESERVED | VM_NONLINEAR);
 	vma->vm_page_prot = PAGE_SHARED;
 	vma->vm_ops = &gru_vm_ops;
 

[Patch 05/25] GRU - cbe cache flush

[steiner]

[-- Attachment #1: uv_gru_cbe_flush --]
[-- Type: text/plain, Size: 1187 bytes --]

From: Jack Steiner <steiner@sgi.com>

In some cases, for performance reasons flushing the SGI GRU cbe
needs to dirty the cacheline before doing the cacheline flush. This ensures
that the dirty cacheline actually goes back to the caching home agent.
Update a reserved field in the CBE instead of a field that might be used. This
_should_ not be required (dirtying any field should be ok) but early
silicon has an issue here.

Note that flushing the CL is not a requirement - only a performance enhancement.

Signed-off-by: Jack Steiner <steiner@sgi.com>


---
 drivers/misc/sgi-gru/grufault.c |    2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

Index: linux/drivers/misc/sgi-gru/grufault.c
===================================================================
--- linux.orig/drivers/misc/sgi-gru/grufault.c	2010-06-09 08:11:40.285085853 -0500
+++ linux/drivers/misc/sgi-gru/grufault.c	2010-06-09 08:11:41.157110138 -0500
@@ -306,7 +306,7 @@ upm:
 static void gru_flush_cache_cbe(struct gru_control_block_extended *cbe)
 {
 	if (unlikely(cbe)) {
-		cbe->cbrexecstatus = 0;         /* make CL dirty */
+		cbe->reserved6 = 0;         /* make CL dirty */
 		gru_flush_cache(cbe);
 	}
 }

[Patch 06/25] GRU - change context stealing

[steiner]

[-- Attachment #1: uv_gru_kservices --]
[-- Type: text/plain, Size: 22012 bytes --]

From: Jack Steiner <steiner@sgi.com>

This patch substantially reworks the way the GRU driver steals
contexts from users in order to use the GRU resources for kernel use.

The new version fixes some ugly locking bugs and is overall more efficient.

Signed-off-by: Jack Steiner <steiner@sgi.com>

---
 drivers/misc/sgi-gru/grufault.c     |    2 
 drivers/misc/sgi-gru/grukservices.c |  271 +++++++++++++++++-------------------
 drivers/misc/sgi-gru/gruprocfs.c    |   12 +
 drivers/misc/sgi-gru/grutables.h    |   37 +++-
 drivers/misc/sgi-gru/grutlbpurge.c  |    2 
 5 files changed, 167 insertions(+), 157 deletions(-)

Index: linux/drivers/misc/sgi-gru/grufault.c
===================================================================
--- linux.orig/drivers/misc/sgi-gru/grufault.c	2010-07-19 10:23:21.000000000 -0500
+++ linux/drivers/misc/sgi-gru/grufault.c	2010-07-19 10:23:54.846243726 -0500
@@ -564,7 +564,7 @@ static irqreturn_t gru_intr(int chiplet,
 
 	for_each_cbr_in_tfm(cbrnum, dmap.fault_bits) {
 		STAT(intr_cbr);
-		cmp = gru->gs_blade->bs_async_wq;
+		cmp = gru->gs_async_wq[cbrnum];
 		if (cmp)
 			complete(cmp);
 		gru_dbg(grudev, "gid %d, cbr_done %d, done %d\n",
Index: linux/drivers/misc/sgi-gru/grukservices.c
===================================================================
--- linux.orig/drivers/misc/sgi-gru/grukservices.c	2010-07-19 10:23:10.000000000 -0500
+++ linux/drivers/misc/sgi-gru/grukservices.c	2010-07-19 10:23:54.854242947 -0500
@@ -95,14 +95,20 @@
  */
 
 
-#define ASYNC_HAN_TO_BID(h)	((h) - 1)
-#define ASYNC_BID_TO_HAN(b)	((b) + 1)
-#define ASYNC_HAN_TO_BS(h)	gru_base[ASYNC_HAN_TO_BID(h)]
-
-#define GRU_NUM_KERNEL_CBR	1
-#define GRU_NUM_KERNEL_DSR_BYTES 256
-#define GRU_NUM_KERNEL_DSR_CL	(GRU_NUM_KERNEL_DSR_BYTES /		\
-					GRU_CACHE_LINE_BYTES)
+#define HAN_TO_BID(h)	((h) - 1)
+#define BID_TO_HAN(b)	((b) + 1)
+#define HAN_TO_BS(h)	gru_base[HAN_TO_BID(h)]
+
+/* Total CBRs in a kernel context for a blade */
+#define GRU_NUM_KERNEL_CBRS	GRU_CBR_AU_SIZE
+
+/* Total DSR bytes in a kernel context for a blade */
+#define GRU_NUM_KERNEL_DSR_BYTES GRU_DSR_AU_BYTES
+
+/* Total DSR bytes/CL per CBR */
+#define GRU_CBR_DSR_BYTES	(GRU_NUM_KERNEL_DSR_BYTES /		\
+					GRU_NUM_KERNEL_CBRS)
+#define GRU_CBR_DSR_CL		(GRU_CBR_DSR_BYTES / GRU_CACHE_LINE_BYTES)
 
 /* GRU instruction attributes for all instructions */
 #define IMA			IMA_CB_DELAY
@@ -153,28 +159,32 @@ static void gru_load_kernel_context(stru
 	struct gru_state *gru;
 	struct gru_thread_state *kgts;
 	void *vaddr;
-	int ctxnum, ncpus;
+	int ctxnum;
 
 	up_read(&bs->bs_kgts_sema);
+	STAT(load_kcontext);
 	down_write(&bs->bs_kgts_sema);
 
+again:
 	if (!bs->bs_kgts) {
-		bs->bs_kgts = gru_alloc_gts(NULL, 0, 0, 0, 0, 0);
+		int cbrs = GRU_CB_COUNT_TO_AU(GRU_NUM_KERNEL_CBRS);
+		int dsrs = GRU_DS_BYTES_TO_AU(GRU_NUM_KERNEL_DSR_BYTES);
+		bs->bs_kgts = gru_alloc_gts(NULL, cbrs, dsrs, 0, 0, 0);
 		bs->bs_kgts->ts_user_blade_id = blade_id;
 	}
 	kgts = bs->bs_kgts;
 
 	if (!kgts->ts_gru) {
-		STAT(load_kernel_context);
-		ncpus = uv_blade_nr_possible_cpus(blade_id);
-		kgts->ts_cbr_au_count = GRU_CB_COUNT_TO_AU(
-			GRU_NUM_KERNEL_CBR * ncpus + bs->bs_async_cbrs);
-		kgts->ts_dsr_au_count = GRU_DS_BYTES_TO_AU(
-			GRU_NUM_KERNEL_DSR_BYTES * ncpus +
-				bs->bs_async_dsr_bytes);
-		while (!gru_assign_gru_context(kgts)) {
+		STAT(load_kcontext_assign);
+		if (!gru_assign_gru_context(kgts)) {
+			STAT(load_kcontext_steal);
+
+			up_write(&bs->bs_kgts_sema);
 			msleep(1);
-			gru_steal_context(kgts);
+			down_write(&bs->bs_kgts_sema);
+			if (bs->bs_kgts)
+				gru_steal_context(kgts);
+			goto again;
 		}
 		gru_load_context(kgts);
 		gru = bs->bs_kgts->ts_gru;
@@ -224,7 +234,7 @@ static struct gru_blade_state *gru_lock_
 	struct gru_blade_state *bs;
 	int bid;
 
-	STAT(lock_kernel_context);
+	STAT(lock_kcontext);
 again:
 	bid = blade_id < 0 ? uv_numa_blade_id() : blade_id;
 	bs = gru_base[bid];
@@ -251,33 +261,51 @@ static void gru_unlock_kernel_context(in
 
 	bs = gru_base[blade_id];
 	up_read(&bs->bs_kgts_sema);
-	STAT(unlock_kernel_context);
+	STAT(unlock_kcontext);
 }
 
 /*
  * Reserve & get pointers to the DSR/CBRs reserved for the current cpu.
  * 	- returns with preemption disabled
  */
-static int gru_get_cpu_resources(int dsr_bytes, void **cb, void **dsr)
+static int gru_get_cpu_resources(int blade_id, int dsr_bytes, void **cb, void **dsr)
 {
 	struct gru_blade_state *bs;
-	int lcpu;
+	int n, busy = 0;
+	void *cb0;
 
 	BUG_ON(dsr_bytes > GRU_NUM_KERNEL_DSR_BYTES);
 	preempt_disable();
-	bs = gru_lock_kernel_context(-1);
-	lcpu = uv_blade_processor_id();
-	*cb = bs->kernel_cb + lcpu * GRU_HANDLE_STRIDE;
-	*dsr = bs->kernel_dsr + lcpu * GRU_NUM_KERNEL_DSR_BYTES;
-	return 0;
+	bs = gru_lock_kernel_context(blade_id);
+
+	/*
+	 * Starting at a random CBR, lock a CBR for
+	 * this task.
+	 */
+	n = gru_random() % GRU_NUM_KERNEL_CBRS;
+	STAT(get_kcontext_cbr);
+	cb0 = bs->kernel_cb + n * GRU_HANDLE_STRIDE;
+	while (!trylock_cbr_handle(cb0)) {
+		n = (n + 1) % GRU_NUM_KERNEL_CBRS;
+		cb0 = bs->kernel_cb + n * GRU_HANDLE_STRIDE;
+		if (!busy)
+			STAT(get_kcontext_cbr_busy);
+		busy = 1;
+		cpu_relax();
+	}
+	*cb = cb0;
+	if (dsr)
+		*dsr = bs->kernel_dsr + n * GRU_CBR_DSR_BYTES;
+	return BID_TO_HAN(bs->bs_grus[0].gs_blade_id);
 }
 
 /*
  * Free the current cpus reserved DSR/CBR resources.
  */
-static void gru_free_cpu_resources(void *cb, void *dsr)
+static void gru_free_cpu_resources(int han, void *cb)
 {
-	gru_unlock_kernel_context(uv_numa_blade_id());
+	unlock_cbr_handle(cb);
+	gru_unlock_kernel_context(HAN_TO_BID(han));
 	preempt_enable();
 }
 
@@ -297,28 +325,11 @@ unsigned long gru_reserve_async_resource
 			struct completion *cmp)
 {
 	struct gru_blade_state *bs;
-	struct gru_thread_state *kgts;
 	int ret = 0;
 
 	bs = gru_base[blade_id];
-
-	down_write(&bs->bs_kgts_sema);
-
-	/* Verify no resources already reserved */
-	if (bs->bs_async_dsr_bytes + bs->bs_async_cbrs)
-		goto done;
-	bs->bs_async_dsr_bytes = dsr_bytes;
-	bs->bs_async_cbrs = cbrs;
-	bs->bs_async_wq = cmp;
-	kgts = bs->bs_kgts;
-
-	/* Resources changed. Unload context if already loaded */
-	if (kgts && kgts->ts_gru)
-		gru_unload_context(kgts, 0);
-	ret = ASYNC_BID_TO_HAN(blade_id);
-
-done:
-	up_write(&bs->bs_kgts_sema);
+	if (cmpxchg(&bs->bs_async_wq, 0, cmp) == 0)
+		ret = BID_TO_HAN(blade_id);
 	return ret;
 }
 
@@ -330,13 +341,9 @@ done:
  */
 void gru_release_async_resources(unsigned long han)
 {
-	struct gru_blade_state *bs = ASYNC_HAN_TO_BS(han);
+	struct gru_blade_state *bs = HAN_TO_BS(han);
 
-	down_write(&bs->bs_kgts_sema);
-	bs->bs_async_dsr_bytes = 0;
-	bs->bs_async_cbrs = 0;
 	bs->bs_async_wq = NULL;
-	up_write(&bs->bs_kgts_sema);
 }
 
 /*
@@ -347,7 +354,7 @@ void gru_release_async_resources(unsigne
  */
 void gru_wait_async_cbr(unsigned long han)
 {
-	struct gru_blade_state *bs = ASYNC_HAN_TO_BS(han);
+	struct gru_blade_state *bs = HAN_TO_BS(han);
 
 	wait_for_completion(bs->bs_async_wq);
 	mb();
@@ -364,16 +371,15 @@ void gru_wait_async_cbr(unsigned long ha
  */
 void gru_lock_async_resource(unsigned long han,  void **cb, void **dsr)
 {
-	struct gru_blade_state *bs = ASYNC_HAN_TO_BS(han);
-	int blade_id = ASYNC_HAN_TO_BID(han);
-	int ncpus;
-
-	gru_lock_kernel_context(blade_id);
-	ncpus = uv_blade_nr_possible_cpus(blade_id);
-	if (cb)
-		*cb = bs->kernel_cb + ncpus * GRU_HANDLE_STRIDE;
-	if (dsr)
-		*dsr = bs->kernel_dsr + ncpus * GRU_NUM_KERNEL_DSR_BYTES;
+	int blade_id = HAN_TO_BID(han);
+	struct gru_blade_state *bs = HAN_TO_BS(han);
+	int cbrnum;
+
+	STAT(lock_async_resource);
+	han = gru_get_cpu_resources(blade_id, 1, cb, dsr);
+	bs->bs_async_cbr = *cb;
+	cbrnum = thread_cbr_number(bs->bs_kgts, get_cb_number(*cb));
+	bs->bs_kgts->ts_gru->gs_async_wq[cbrnum] = bs->bs_async_wq;
 }
 
 /*
@@ -384,9 +390,11 @@ void gru_lock_async_resource(unsigned lo
  */
 void gru_unlock_async_resource(unsigned long han)
 {
-	int blade_id = ASYNC_HAN_TO_BID(han);
+	struct gru_blade_state *bs = HAN_TO_BS(han);
 
-	gru_unlock_kernel_context(blade_id);
+	STAT(unlock_async_resource);
+	gru_free_cpu_resources(han, bs->bs_async_cbr);
+	bs->bs_async_cbr = NULL;
 }
 
 /*----------------------------------------------------------------------*/
@@ -434,20 +442,29 @@ char *gru_get_cb_exception_detail_str(in
 	struct gru_control_block_status *gen = (void *)cb;
 	struct control_block_extended_exc_detail excdet;
 
-	if (ret > 0 && gen->istatus == CBS_EXCEPTION) {
+	if (gen->istatus == CBS_EXCEPTION) {
 		gru_get_cb_exception_detail(cb, &excdet);
 		snprintf(buf, size,
 			"GRU:%d exception: cb %p, opc %d, exopc %d, ecause 0x%x,"
-			"excdet0 0x%lx, excdet1 0x%x", smp_processor_id(),
+			"excdet0 0x%lx, excdet1 0x%x, execstatus 0x%x, state 0x%x", smp_processor_id(),
 			gen, excdet.opc, excdet.exopc, excdet.ecause,
-			excdet.exceptdet0, excdet.exceptdet1);
+			excdet.exceptdet0, excdet.exceptdet1,
+			excdet.cbrexecstatus, excdet.cbrexecstatus);
 	} else {
 		snprintf(buf, size, "No exception");
 	}
 	return buf;
 }
 
-static int gru_wait_idle_or_exception(struct gru_control_block_status *gen)
+static void gru_print_cb_exception_detail(char *id, void *cb)
+{
+	char buf[GRU_EXC_STR_SIZE];
+
+	gru_get_cb_exception_detail_str(1, cb, buf, sizeof(buf));
+	printk(KERN_ERR "GRU: %d  %s\n", smp_processor_id(), buf);
+}
+
+static int gru_wait_idle_or_exception(struct gru_instruction_bits *gen)
 {
 	while (gen->istatus >= CBS_ACTIVE) {
 		cpu_relax();
@@ -458,7 +475,7 @@ static int gru_wait_idle_or_exception(st
 
 static int gru_retry_exception(void *cb)
 {
-	struct gru_control_block_status *gen = (void *)cb;
+	struct gru_instruction_bits *gen = (void *)cb;
 	struct control_block_extended_exc_detail excdet;
 	int retry = EXCEPTION_RETRY_LIMIT;
 
@@ -475,13 +492,14 @@ static int gru_retry_exception(void *cb)
 			break;
 		gen->icmd = 1;
 		gru_flush_cache(gen);
+		printk(KERN_ERR "GRU: %d retry exception 0x%p\n", smp_processor_id(), cb);
 	}
 	return CBS_EXCEPTION;
 }
 
 int gru_check_status_proc(void *cb)
 {
-	struct gru_control_block_status *gen = (void *)cb;
+	struct gru_instruction_bits *gen = (void *)cb;
 	int ret;
 
 	ret = gen->istatus;
@@ -494,7 +512,7 @@ int gru_check_status_proc(void *cb)
 
 int gru_wait_proc(void *cb)
 {
-	struct gru_control_block_status *gen = (void *)cb;
+	struct gru_instruction_bits *gen = (void *)cb;
 	int ret;
 
 	ret = gru_wait_idle_or_exception(gen);
@@ -517,8 +535,10 @@ void gru_wait_abort_proc(void *cb)
 	int ret;
 
 	ret = gru_wait_proc(cb);
-	if (ret)
-		gru_abort(ret, cb, "gru_wait_abort");
+	if (ret) {
+		gru_print_cb_exception_detail("abort", cb);
+		panic("gru_wait_abort");
+	}
 }
 
 
@@ -799,14 +819,13 @@ int gru_send_message_gpa(struct gru_mess
 	struct message_header *mhdr;
 	void *cb;
 	void *dsr;
-	int istatus, clines, ret;
+	int han, istatus, clines, ret;
 
 	STAT(mesq_send);
 	BUG_ON(bytes < sizeof(int) || bytes > 2 * GRU_CACHE_LINE_BYTES);
 
 	clines = DIV_ROUND_UP(bytes, GRU_CACHE_LINE_BYTES);
-	if (gru_get_cpu_resources(bytes, &cb, &dsr))
-		return MQE_BUG_NO_RESOURCES;
+	han = gru_get_cpu_resources(-1, bytes, &cb, &dsr);
 	memcpy(dsr, mesg, bytes);
 	mhdr = dsr;
 	mhdr->present = MQS_FULL;
@@ -823,7 +842,7 @@ int gru_send_message_gpa(struct gru_mess
 		if (istatus != CBS_IDLE)
 			ret = send_message_failure(cb, mqd, dsr, clines);
 	} while (ret == MQIE_AGAIN);
-	gru_free_cpu_resources(cb, dsr);
+	gru_free_cpu_resources(han, cb);
 
 	if (ret)
 		STAT(mesq_send_failed);
@@ -906,22 +925,20 @@ int gru_read_gpa(unsigned long *value, u
 {
 	void *cb;
 	void *dsr;
-	int ret, iaa;
+	int han, ret, iaa;
 
 	STAT(read_gpa);
-	if (gru_get_cpu_resources(GRU_NUM_KERNEL_DSR_BYTES, &cb, &dsr))
-		return MQE_BUG_NO_RESOURCES;
+	han = gru_get_cpu_resources(-1, GRU_NUM_KERNEL_DSR_BYTES, &cb, &dsr);
 	iaa = gpa >> 62;
 	gru_vload_phys(cb, gpa, gru_get_tri(dsr), iaa, IMA);
 	ret = gru_wait(cb);
 	if (ret == CBS_IDLE)
 		*value = *(unsigned long *)dsr;
-	gru_free_cpu_resources(cb, dsr);
+	gru_free_cpu_resources(han, cb);
 	return ret;
 }
 EXPORT_SYMBOL_GPL(gru_read_gpa);
 
-
 /*
  * Copy a block of data using the GRU resources
  */
@@ -930,22 +947,20 @@ int gru_copy_gpa(unsigned long dest_gpa,
 {
 	void *cb;
 	void *dsr;
-	int ret;
+	int han, ret;
 
 	STAT(copy_gpa);
-	if (gru_get_cpu_resources(GRU_NUM_KERNEL_DSR_BYTES, &cb, &dsr))
-		return MQE_BUG_NO_RESOURCES;
+	han = gru_get_cpu_resources(-1, GRU_NUM_KERNEL_DSR_BYTES, &cb, &dsr);
 	gru_bcopy(cb, src_gpa, dest_gpa, gru_get_tri(dsr),
-		  XTYPE_B, bytes, GRU_NUM_KERNEL_DSR_CL, IMA);
+		  XTYPE_B, bytes, GRU_CBR_DSR_CL, IMA);
 	ret = gru_wait(cb);
-	gru_free_cpu_resources(cb, dsr);
+	gru_free_cpu_resources(han, cb);
 	return ret;
 }
 EXPORT_SYMBOL_GPL(gru_copy_gpa);
 
 /* ------------------- KERNEL QUICKTESTS RUN AT STARTUP ----------------*/
 /* 	Temp - will delete after we gain confidence in the GRU		*/
-
 static int quicktest0(unsigned long arg)
 {
 	unsigned long word0;
@@ -953,40 +968,39 @@ static int quicktest0(unsigned long arg)
 	void *cb;
 	void *dsr;
 	unsigned long *p;
-	int ret = -EIO;
+	int han, ret = -EIO;
 
-	if (gru_get_cpu_resources(GRU_CACHE_LINE_BYTES, &cb, &dsr))
-		return MQE_BUG_NO_RESOURCES;
+	han = gru_get_cpu_resources(-1, GRU_CACHE_LINE_BYTES, &cb, &dsr);
 	p = dsr;
 	word0 = MAGIC;
 	word1 = 0;
 
 	gru_vload(cb, uv_gpa(&word0), gru_get_tri(dsr), XTYPE_DW, 1, 1, IMA);
 	if (gru_wait(cb) != CBS_IDLE) {
-		printk(KERN_DEBUG "GRU:%d quicktest0: CBR failure 1\n", smp_processor_id());
+		printk(KERN_DEBUG "GRU: %d quicktest0: 0x%p CBR failure 1\n", smp_processor_id(), cb);
 		goto done;
 	}
 
 	if (*p != MAGIC) {
-		printk(KERN_DEBUG "GRU:%d quicktest0 bad magic 0x%lx\n", smp_processor_id(), *p);
+		printk(KERN_DEBUG "GRU: %d quicktest0: 0x%p bad magic 0x%lx\n", smp_processor_id(), cb, *p);
 		goto done;
 	}
 	gru_vstore(cb, uv_gpa(&word1), gru_get_tri(dsr), XTYPE_DW, 1, 1, IMA);
 	if (gru_wait(cb) != CBS_IDLE) {
-		printk(KERN_DEBUG "GRU:%d quicktest0: CBR failure 2\n", smp_processor_id());
+		printk(KERN_DEBUG "GRU: %d quicktest0: 0x%p CBR failure 2\n", smp_processor_id(), cb);
 		goto done;
 	}
 
 	if (word0 != word1 || word1 != MAGIC) {
 		printk(KERN_DEBUG
-		       "GRU:%d quicktest0 err: found 0x%lx, expected 0x%lx\n",
-		     smp_processor_id(), word1, MAGIC);
+		       "GRU: %d quicktest0 err 0x%p: found 0x%lx, expected 0x%lx\n",
+		     smp_processor_id(), cb, word1, MAGIC);
 		goto done;
 	}
 	ret = 0;
 
 done:
-	gru_free_cpu_resources(cb, dsr);
+	gru_free_cpu_resources(han, cb);
 	return ret;
 }
 
@@ -1018,7 +1032,7 @@ static int quicktest1(unsigned long arg)
 			break;
 	}
 	if (ret != MQE_QUEUE_FULL || i != 4) {
-		printk(KERN_DEBUG "GRU:%d quicktest1: unexpect status %d, i %d\n",
+		printk(KERN_DEBUG "GRU: %d quicktest1: unexpect status %d, msg %d\n",
 		       smp_processor_id(), ret, i);
 		goto done;
 	}
@@ -1030,7 +1044,7 @@ static int quicktest1(unsigned long arg)
 		gru_free_message(&mqd, m);
 	}
 	if (i != 4) {
-		printk(KERN_DEBUG "GRU:%d quicktest2: bad message, i %d, m %p, m8 %d\n",
+		printk(KERN_DEBUG "GRU: %d quicktest2: bad message, i %d, m %p, m8 %d\n",
 			smp_processor_id(), i, m, m ? m[8] : -1);
 		goto done;
 	}
@@ -1046,54 +1060,30 @@ static int quicktest2(unsigned long arg)
 	static DECLARE_COMPLETION(cmp);
 	unsigned long han;
 	int blade_id = 0;
-	int numcb = 4;
 	int ret = 0;
 	unsigned long *buf;
-	void *cb0, *cb;
-	struct gru_control_block_status *gen;
-	int i, k, istatus, bytes;
+	void *cb;
+	int bytes;
 
-	bytes = numcb * 4 * 8;
+	bytes = 4 * 8;
 	buf = kmalloc(bytes, GFP_KERNEL);
 	if (!buf)
 		return -ENOMEM;
 
 	ret = -EBUSY;
-	han = gru_reserve_async_resources(blade_id, numcb, 0, &cmp);
+	han = gru_reserve_async_resources(blade_id, 1, 0, &cmp);
 	if (!han)
 		goto done;
 
-	gru_lock_async_resource(han, &cb0, NULL);
-	memset(buf, 0xee, bytes);
-	for (i = 0; i < numcb; i++)
-		gru_vset(cb0 + i * GRU_HANDLE_STRIDE, uv_gpa(&buf[i * 4]), 0,
-				XTYPE_DW, 4, 1, IMA_INTERRUPT);
-
-	ret = 0;
-	k = numcb;
-	do {
-		gru_wait_async_cbr(han);
-		for (i = 0; i < numcb; i++) {
-			cb = cb0 + i * GRU_HANDLE_STRIDE;
-			istatus = gru_check_status(cb);
-			if (istatus != CBS_ACTIVE && istatus != CBS_CALL_OS)
-				break;
-		}
-		if (i == numcb)
-			continue;
-		if (istatus != CBS_IDLE) {
-			printk(KERN_DEBUG "GRU:%d quicktest2: cb %d, exception\n", smp_processor_id(), i);
-			ret = -EFAULT;
-		} else if (buf[4 * i] || buf[4 * i + 1] || buf[4 * i + 2] ||
-				buf[4 * i + 3]) {
-			printk(KERN_DEBUG "GRU:%d quicktest2:cb %d,  buf 0x%lx, 0x%lx, 0x%lx, 0x%lx\n",
-			       smp_processor_id(), i, buf[4 * i], buf[4 * i + 1], buf[4 * i + 2], buf[4 * i + 3]);
-			ret = -EIO;
-		}
-		k--;
-		gen = cb;
-		gen->istatus = CBS_CALL_OS; /* don't handle this CBR again */
-	} while (k);
+	memset(buf, 0xef, bytes);
+	gru_lock_async_resource(han, &cb, NULL);
+	gru_vset(cb, uv_gpa(buf), 0, XTYPE_DW, 4, 1, IMA_INTERRUPT);
+	gru_wait_async_cbr(han);
+	if (buf[0] || buf[1] || buf[2] || buf[3]) {
+		printk(KERN_DEBUG "GRU: %d quicktest2:cb 0x%p (0x%lx, 0x%lx, 0x%lx 0x%lx)\n",
+		       smp_processor_id(), cb, buf[0], buf[1], buf[2], buf[3]);
+		ret = -EIO;
+	}
 	BUG_ON(cmp.done);
 
 	gru_unlock_async_resource(han);
@@ -1113,7 +1103,7 @@ static int quicktest3(unsigned long arg)
 	memset(buf1, get_cycles() & 255, sizeof(buf1));
 	gru_copy_gpa(uv_gpa(buf2), uv_gpa(buf1), BUFSIZE);
 	if (memcmp(buf1, buf2, BUFSIZE)) {
-		printk(KERN_DEBUG "GRU:%d quicktest3 error\n", smp_processor_id());
+		printk(KERN_DEBUG "GRU: %d quicktest3 error\n", smp_processor_id());
 		ret = -EIO;
 	}
 	return ret;
@@ -1158,4 +1148,3 @@ void gru_kservices_exit(void)
 	if (gru_free_kernel_contexts())
 		BUG();
 }
-
Index: linux/drivers/misc/sgi-gru/gruprocfs.c
===================================================================
--- linux.orig/drivers/misc/sgi-gru/gruprocfs.c	2010-07-19 10:23:16.000000000 -0500
+++ linux/drivers/misc/sgi-gru/gruprocfs.c	2010-07-19 10:23:54.870243453 -0500
@@ -52,9 +52,15 @@ static int statistics_show(struct seq_fi
 	printstat(s, assign_context_failed);
 	printstat(s, free_context);
 	printstat(s, load_user_context);
-	printstat(s, load_kernel_context);
-	printstat(s, lock_kernel_context);
-	printstat(s, unlock_kernel_context);
+	printstat(s, load_kcontext);
+	printstat(s, load_kcontext_assign);
+	printstat(s, load_kcontext_steal);
+	printstat(s, lock_kcontext);
+	printstat(s, unlock_kcontext);
+	printstat(s, get_kcontext_cbr);
+	printstat(s, get_kcontext_cbr_busy);
+	printstat(s, lock_async_resource);
+	printstat(s, unlock_async_resource);
 	printstat(s, steal_user_context);
 	printstat(s, steal_kernel_context);
 	printstat(s, steal_context_failed);
Index: linux/drivers/misc/sgi-gru/grutables.h
===================================================================
--- linux.orig/drivers/misc/sgi-gru/grutables.h	2010-07-19 10:23:21.000000000 -0500
+++ linux/drivers/misc/sgi-gru/grutables.h	2010-07-19 10:25:12.510295762 -0500
@@ -163,6 +163,8 @@ extern unsigned int gru_max_gids;
 #define GRU_DRIVER_ID_STR	"SGI GRU Device Driver"
 #define GRU_DRIVER_VERSION_STR	"0.85"
 
+#define gru_random()    get_cycles()
+
 /*
  * GRU statistics.
  */
@@ -178,9 +180,15 @@ struct gru_stats_s {
 	atomic_long_t assign_context_failed;
 	atomic_long_t free_context;
 	atomic_long_t load_user_context;
-	atomic_long_t load_kernel_context;
-	atomic_long_t lock_kernel_context;
-	atomic_long_t unlock_kernel_context;
+	atomic_long_t load_kcontext;
+	atomic_long_t load_kcontext_assign;
+	atomic_long_t load_kcontext_steal;
+	atomic_long_t lock_kcontext;
+	atomic_long_t unlock_kcontext;
+	atomic_long_t get_kcontext_cbr;
+	atomic_long_t get_kcontext_cbr_busy;
+	atomic_long_t lock_async_resource;
+	atomic_long_t unlock_async_resource;
 	atomic_long_t steal_user_context;
 	atomic_long_t steal_kernel_context;
 	atomic_long_t steal_context_failed;
@@ -443,14 +451,11 @@ struct gru_state {
 							   resources */
 	unsigned long		gs_dsr_map;		/* bitmap used to manage
 							   DATA resources */
-	unsigned int		gs_reserved_cbrs;	/* Number of kernel-
-							   reserved cbrs */
-	unsigned int		gs_reserved_dsr_bytes;	/* Bytes of kernel-
-							   reserved dsrs */
 	unsigned short		gs_active_contexts;	/* number of contexts
 							   in use */
 	struct gru_thread_state	*gs_gts[GRU_NUM_CCH];	/* GTS currently using
 							   the context */
+	struct completion	*gs_async_wq[GRU_NUM_CB];
 	int			gs_irq[GRU_NUM_TFM];	/* Interrupt irqs */
 };
 
@@ -466,8 +471,7 @@ struct gru_blade_state {
 	struct gru_thread_state *bs_kgts;		/* GTS for kernel use */
 
 	/* ---- the following are used for managing kernel async GRU CBRs --- */
-	int			bs_async_dsr_bytes;	/* DSRs for async */
-	int			bs_async_cbrs;		/* CBRs AU for async */
+	void			*bs_async_cbr;		/* CBR for async */
 	struct completion	*bs_async_wq;
 
 	/* ---- the following are protected by the bs_lock spinlock ---- */
@@ -558,11 +562,24 @@ struct gru_blade_state {
 
 /*-----------------------------------------------------------------------------
  * Lock / Unlock GRU handles
- * 	Use the "delresp" bit in the handle as a "lock" bit.
+ * 	Use the "delresp" bit in MCS handles as a "lock" bit.
+ * 	Use the "unmapped" bit in CBRs as a "lock" bit.
+ *
+ *		Return: 0 = lock failed, 1 = locked
  */
 
 /* Lock hierarchy checking enabled only in emulator */
 
+static inline int trylock_cbr_handle(void *h)
+{
+	return !test_and_set_bit(2, h);
+}
+
+static inline void unlock_cbr_handle(void *h)
+{
+	clear_bit(2, h);
+}
+
 /* 0 = lock failed, 1 = locked */
 static inline int __trylock_handle(void *h)
 {
Index: linux/drivers/misc/sgi-gru/grutlbpurge.c
===================================================================
--- linux.orig/drivers/misc/sgi-gru/grutlbpurge.c	2010-07-19 10:23:15.000000000 -0500
+++ linux/drivers/misc/sgi-gru/grutlbpurge.c	2010-07-19 10:23:54.902272728 -0500
@@ -40,8 +40,6 @@
 #include "grutables.h"
 #include <asm/uv/uv_hub.h>
 
-#define gru_random()	get_cycles()
-
 /* ---------------------------------- TLB Invalidation functions --------
  * get_tgh_handle
  *

[Patch 07/25] GRU - add context lock flag to gru status

[steiner]

[-- Attachment #1: uv_gru_proc_cch_locked --]
[-- Type: text/plain, Size: 1660 bytes --]

From: Jack Steiner <steiner@sgi.com>

Add a flag to the GRU cch_status file to indicate if the cch is locked.
This is useful for debugging.

Signed-off-by: Jack Steiner <steiner@sgi.com>


---
 drivers/misc/sgi-gru/gruprocfs.c |   11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

Index: linux/drivers/misc/sgi-gru/gruprocfs.c
===================================================================
--- linux.orig/drivers/misc/sgi-gru/gruprocfs.c	2010-06-09 08:11:42.043954832 -0500
+++ linux/drivers/misc/sgi-gru/gruprocfs.c	2010-06-09 08:11:42.456084620 -0500
@@ -190,21 +190,22 @@ static int cch_seq_show(struct seq_file
 	const char *mode[] = { "??", "UPM", "INTR", "OS_POLL" };
 
 	if (gid == 0)
-		seq_printf(file, "#%5s%5s%6s%7s%9s%6s%8s%8s\n", "gid", "bid",
-			   "ctx#", "asid", "pid", "cbrs", "dsbytes", "mode");
+		seq_printf(file, "#%5s%5s%6s%7s%9s%6s%8s%8s%7s\n", "gid", "bid",
+			   "ctx#", "asid", "pid", "cbrs", "dsbytes", "mode", "locked");
 	if (gru)
 		for (i = 0; i < GRU_NUM_CCH; i++) {
 			ts = gru->gs_gts[i];
 			if (!ts)
 				continue;
-			seq_printf(file, " %5d%5d%6d%7d%9d%6d%8d%8s\n",
+			seq_printf(file, " %5d%5d%6d%7d%9d%6d%8d%8s%7s\n",
 				   gru->gs_gid, gru->gs_blade_id, i,
 				   is_kernel_context(ts) ? 0 : ts->ts_gms->ms_asids[gid].mt_asid,
 				   is_kernel_context(ts) ? 0 : ts->ts_tgid_owner,
 				   ts->ts_cbr_au_count * GRU_CBR_AU_SIZE,
-				   ts->ts_cbr_au_count * GRU_DSR_AU_BYTES,
+				   ts->ts_dsr_au_count * GRU_DSR_AU_BYTES,
 				   mode[ts->ts_user_options &
-					GRU_OPT_MISS_MASK]);
+					GRU_OPT_MISS_MASK],
+				   mutex_is_locked(&ts->ts_ctxlock) ? "LOCKED" : "-");
 		}
 
 	return 0;

[Patch 08/25] GRU - flush gru tlb when driver is loaded

[steiner]

[-- Attachment #1: uv_gru_init_flush_tlb --]
[-- Type: text/plain, Size: 886 bytes --]

From: Jack Steiner <steiner@sgi.com>

Flush the GRU TLB when the GRU driver is loaded. There may be stale
entries in the TLB if a previous GRU was unloaded since the system
was last reset.

Signed-off-by: Jack Steiner <steiner@sgi.com>

---
 drivers/misc/sgi-gru/grutlbpurge.c |    2 ++
 1 file changed, 2 insertions(+)

Index: linux/drivers/misc/sgi-gru/grutlbpurge.c
===================================================================
--- linux.orig/drivers/misc/sgi-gru/grutlbpurge.c	2010-06-09 08:11:42.087537647 -0500
+++ linux/drivers/misc/sgi-gru/grutlbpurge.c	2010-06-09 08:11:42.891554573 -0500
@@ -380,4 +380,6 @@ void gru_tgh_flush_init(struct gru_state
 	/* first starting TGH index to use for remote purges */
 	gru->gs_tgh_first_remote = (cpus + (1 << shift) - 1) >> shift;
 
+	/* flush the GRU TLB in case there are stale entries present */
+	gru_flush_all_tlb(gru);
 }

[Patch 09/25] GRU - add software reserved bits to cbr definition

[steiner]

[-- Attachment #1: uv_gru_software_reserved --]
[-- Type: text/plain, Size: 2607 bytes --]

From: Jack Steiner <steiner@sgi.com>

The GRU hardware has reserved a few bits for software use.
Add these "software reserved" fields to the structure definitions.

Signed-off-by: Jack Steiner <steiner@sgi.com>

---
 drivers/misc/sgi-gru/gru_instructions.h |   19 +++++++++++++------
 1 file changed, 13 insertions(+), 6 deletions(-)

Index: linux/drivers/misc/sgi-gru/gru_instructions.h
===================================================================
--- linux.orig/drivers/misc/sgi-gru/gru_instructions.h	2010-06-09 08:11:38.907960603 -0500
+++ linux/drivers/misc/sgi-gru/gru_instructions.h	2010-06-09 08:11:43.303963161 -0500
@@ -112,7 +112,8 @@ struct gru_instruction_bits {
     unsigned char		reserved3: 1;
     unsigned char		tlb_fault_color: 1;
     /* DW 1 */
-    unsigned long		idef4;		/* 42 bits: TRi1, BufSize */
+    unsigned int		idef4;		/* 32 bits: TRi1, BufSize */
+    unsigned int		sw_reserved;
     /* DW 2-6 */
     unsigned long		idef1;		/* BAddr0 */
     unsigned long		idef5;		/* Nelem */
@@ -136,7 +137,13 @@ struct gru_instruction {
 		unsigned int	tri0;
 	};
     };
-    unsigned long		tri1_bufsize;		/* DW 1 */
+    union {
+	unsigned long		tri1_bufsize_64;	/* DW 1 */
+	struct {
+    		unsigned int	tri1_bufsize_32;
+		unsigned int	sw_reserved;
+	};
+    };
     unsigned long		baddr0;			/* DW 2 */
     unsigned long		nelem;			/* DW 3 */
     unsigned long		op1_stride;		/* DW 4 */
@@ -411,7 +418,7 @@ static inline void gru_ivload(void *cb,
 
 	ins->baddr0 = (long)mem_addr;
 	ins->nelem = nelem;
-	ins->tri1_bufsize = tri1;
+	ins->tri1_bufsize_64 = tri1;
 	gru_start_instruction(ins, __opdword(OP_IVLOAD, 0, xtype, IAA_RAM, 0,
 					tri0, CB_IMA(hints)));
 }
@@ -424,7 +431,7 @@ static inline void gru_ivstore(void *cb,
 
 	ins->baddr0 = (long)mem_addr;
 	ins->nelem = nelem;
-	ins->tri1_bufsize = tri1;
+	ins->tri1_bufsize_64 = tri1;
 	gru_start_instruction(ins, __opdword(OP_IVSTORE, 0, xtype, IAA_RAM, 0,
 					tri0, CB_IMA(hints)));
 }
@@ -452,7 +459,7 @@ static inline void gru_ivset(void *cb, u
 	ins->baddr0 = (long)mem_addr;
 	ins->op2_value_baddr1 = value;
 	ins->nelem = nelem;
-	ins->tri1_bufsize = tri1;
+	ins->tri1_bufsize_64 = tri1;
 	gru_start_instruction(ins, __opdword(OP_IVSET, 0, xtype, IAA_RAM, 0,
 					0, CB_IMA(hints)));
 }
@@ -488,7 +495,7 @@ static inline void gru_bcopy(void *cb, c
 	ins->baddr0 = (long)src;
 	ins->op2_value_baddr1 = (long)dest;
 	ins->nelem = nelem;
-	ins->tri1_bufsize = bufsize;
+	ins->tri1_bufsize_64 = bufsize;
 	gru_start_instruction(ins, __opdword(OP_BCOPY, 0, xtype, IAA_RAM,
 					IAA_RAM, tri0, CB_IMA(hints)));
 }

[Patch 10/25] GRU - eliminate gru contention on mmap_sem

[steiner]

[-- Attachment #1: uv_gru_mmap_lock --]
[-- Type: text/plain, Size: 13372 bytes --]

From: Jack Steiner <steiner@sgi.com>

Rework the way the mmap_sem is used by the GRU driver. The old code
had a few pathes that acquired the lock for write. By changing
the place that the mmu_notifier was allocated/freed, the mmap_sem
is no longer acquired for write. This eliminates some contention on the lock
and improves performance of threaded tests that use the gru.
This also eliminates an endcase where zap_vma_range() could be called without
holding the mmap_sem .

Signed-off-by: Jack Steiner <steiner@sgi.com>

---
 drivers/misc/sgi-gru/grufault.c    |   11 ++++---
 drivers/misc/sgi-gru/grufile.c     |   16 +++++++----
 drivers/misc/sgi-gru/gruhandles.c  |    5 ---
 drivers/misc/sgi-gru/grumain.c     |   51 ++++++++++++++++++++++---------------
 drivers/misc/sgi-gru/grutables.h   |   11 +++++++
 drivers/misc/sgi-gru/grutlbpurge.c |   18 +++----------
 6 files changed, 63 insertions(+), 49 deletions(-)

Index: linux/drivers/misc/sgi-gru/grufault.c
===================================================================
--- linux.orig/drivers/misc/sgi-gru/grufault.c	2010-07-19 10:23:54.846243726 -0500
+++ linux/drivers/misc/sgi-gru/grufault.c	2010-07-19 10:25:31.203387741 -0500
@@ -98,7 +98,7 @@ static struct gru_thread_state *gru_allo
 	struct vm_area_struct *vma;
 	struct gru_thread_state *gts = ERR_PTR(-EINVAL);
 
-	down_write(&mm->mmap_sem);
+	down_read(&mm->mmap_sem);
 	vma = gru_find_vma(current->mm, vaddr);
 	if (!vma)
 		goto err;
@@ -107,11 +107,10 @@ static struct gru_thread_state *gru_allo
 	if (IS_ERR(gts))
 		goto err;
 	mutex_lock(&gts->ts_ctxlock);
-	downgrade_write(&mm->mmap_sem);
 	return gts;
 
 err:
-	up_write(&mm->mmap_sem);
+	up_read(&mm->mmap_sem);
 	return gts;
 }
 
@@ -764,18 +763,20 @@ static int gru_unload_all_contexts(void)
 	struct gru_thread_state *gts;
 	struct gru_state *gru;
 	int gid, ctxnum;
+	struct gru_blade_state *blade;
 
 	if (!capable(CAP_SYS_ADMIN))
 		return -EPERM;
 	foreach_gid(gid) {
 		gru = GID_TO_GRU(gid);
+		blade = gru->gs_blade;
 		spin_lock(&gru->gs_lock);
 		for (ctxnum = 0; ctxnum < GRU_NUM_CCH; ctxnum++) {
 			gts = gru->gs_gts[ctxnum];
-			if (gts && mutex_trylock(&gts->ts_ctxlock)) {
+			if (gts && gru_is_gts_stealable(gts, blade)) {
 				spin_unlock(&gru->gs_lock);
 				gru_unload_context(gts, 1);
-				mutex_unlock(&gts->ts_ctxlock);
+				gru_gts_stolen(gts, blade);
 				spin_lock(&gru->gs_lock);
 			}
 		}
Index: linux/drivers/misc/sgi-gru/grufile.c
===================================================================
--- linux.orig/drivers/misc/sgi-gru/grufile.c	2010-07-19 10:23:21.622879486 -0500
+++ linux/drivers/misc/sgi-gru/grufile.c	2010-07-19 10:25:31.203387741 -0500
@@ -88,6 +88,7 @@ static void gru_vma_close(struct vm_area
 		mutex_unlock(&gts->ts_ctxlock);
 		gts_drop(gts);
 	}
+	gru_drop_mmu_notifier(vdata->vd_gms);
 	kfree(vdata);
 	STAT(vdata_free);
 }
@@ -101,6 +102,8 @@ static void gru_vma_close(struct vm_area
  */
 static int gru_file_mmap(struct file *file, struct vm_area_struct *vma)
 {
+	struct gru_vma_data *vdata;
+
 	if ((vma->vm_flags & (VM_SHARED | VM_WRITE)) != (VM_SHARED | VM_WRITE))
 		return -EPERM;
 
@@ -114,9 +117,10 @@ static int gru_file_mmap(struct file *fi
 	vma->vm_page_prot = PAGE_SHARED;
 	vma->vm_ops = &gru_vm_ops;
 
-	vma->vm_private_data = gru_alloc_vma_data(vma, 0);
-	if (!vma->vm_private_data)
-		return -ENOMEM;
+	vdata = gru_alloc_vma_data(vma, 0);
+	if (IS_ERR(vdata))
+		return PTR_ERR(vdata);
+	vma->vm_private_data = vdata;
 
 	gru_dbg(grudev, "file %p, vaddr 0x%lx, vma %p, vdata %p\n",
 		file, vma->vm_start, vma, vma->vm_private_data);
@@ -144,18 +148,20 @@ static int gru_create_new_context(unsign
 	if (!(req.options & GRU_OPT_MISS_MASK))
 		req.options |= GRU_OPT_MISS_FMM_INTR;
 
-	down_write(&current->mm->mmap_sem);
+	down_read(&current->mm->mmap_sem);
 	vma = gru_find_vma(current->mm, req.gseg);
 	if (vma) {
 		vdata = vma->vm_private_data;
+		spin_lock(&vdata->vd_lock);
 		vdata->vd_user_options = req.options;
 		vdata->vd_dsr_au_count =
 		    GRU_DS_BYTES_TO_AU(req.data_segment_bytes);
 		vdata->vd_cbr_au_count = GRU_CB_COUNT_TO_AU(req.control_blocks);
 		vdata->vd_tlb_preload_count = req.tlb_preload_count;
+		spin_unlock(&vdata->vd_lock);
 		ret = 0;
 	}
-	up_write(&current->mm->mmap_sem);
+	up_read(&current->mm->mmap_sem);
 
 	return ret;
 }
Index: linux/drivers/misc/sgi-gru/gruhandles.c
===================================================================
--- linux.orig/drivers/misc/sgi-gru/gruhandles.c	2010-07-19 10:23:10.078252250 -0500
+++ linux/drivers/misc/sgi-gru/gruhandles.c	2010-07-19 10:25:31.223491144 -0500
@@ -132,11 +132,6 @@ int cch_deallocate(struct gru_context_co
 	start_instruction(cch);
 	ret = wait_instruction_complete(cch, cchop_deallocate);
 
-	/*
-	 * Stop speculation into the GSEG being unmapped by the previous
-	 * DEALLOCATE.
-	 */
-	sync_core();
 	return ret;
 }
 
Index: linux/drivers/misc/sgi-gru/grumain.c
===================================================================
--- linux.orig/drivers/misc/sgi-gru/grumain.c	2010-07-19 10:23:10.078252250 -0500
+++ linux/drivers/misc/sgi-gru/grumain.c	2010-07-19 10:25:31.247511575 -0500
@@ -295,8 +295,6 @@ static void gru_unload_mm_tracker(struct
 void gts_drop(struct gru_thread_state *gts)
 {
 	if (gts && atomic_dec_return(&gts->ts_refcnt) == 0) {
-		if (gts->ts_gms)
-			gru_drop_mmu_notifier(gts->ts_gms);
 		kfree(gts);
 		STAT(gts_free);
 	}
@@ -324,7 +322,6 @@ struct gru_thread_state *gru_alloc_gts(s
 		unsigned char tlb_preload_count, int options, int tsid)
 {
 	struct gru_thread_state *gts;
-	struct gru_mm_struct *gms;
 	int bytes;
 
 	bytes = DSR_BYTES(dsr_au_count) + CBR_BYTES(cbr_au_count);
@@ -349,20 +346,15 @@ struct gru_thread_state *gru_alloc_gts(s
 	gts->ts_cch_req_slice = -1;
 	gts->ts_sizeavail = GRU_SIZEAVAIL(PAGE_SHIFT);
 	if (vma) {
+		struct gru_vma_data *vdata = vma->vm_private_data;
+
+		gts->ts_gms = vdata->vd_gms;;
 		gts->ts_mm = current->mm;
 		gts->ts_vma = vma;
-		gms = gru_register_mmu_notifier();
-		if (IS_ERR(gms))
-			goto err;
-		gts->ts_gms = gms;
 	}
 
 	gru_dbg(grudev, "alloc gts %p\n", gts);
 	return gts;
-
-err:
-	gts_drop(gts);
-	return ERR_CAST(gms);
 }
 
 /*
@@ -371,16 +363,25 @@ err:
 struct gru_vma_data *gru_alloc_vma_data(struct vm_area_struct *vma, int tsid)
 {
 	struct gru_vma_data *vdata = NULL;
+	struct gru_mm_struct *gms;
 
 	vdata = kmalloc(sizeof(*vdata), GFP_KERNEL);
 	if (!vdata)
-		return NULL;
+		return ERR_PTR(-ENOMEM);
+	gms = gru_register_mmu_notifier();
+	if (IS_ERR(gms))
+		goto err;
+	vdata->vd_gms = gms;
 
 	STAT(vdata_alloc);
 	INIT_LIST_HEAD(&vdata->vd_head);
 	spin_lock_init(&vdata->vd_lock);
 	gru_dbg(grudev, "alloc vdata %p\n", vdata);
 	return vdata;
+
+err:
+	kfree(vdata);
+	return ERR_PTR(PTR_ERR(gms));
 }
 
 /*
@@ -758,16 +759,23 @@ void gru_check_context_placement(struct
 #define next_gru(b, g)	(((g) < &(b)->bs_grus[GRU_CHIPLETS_PER_BLADE - 1]) ?  \
 				 ((g)+1) : &(b)->bs_grus[0])
 
-static int is_gts_stealable(struct gru_thread_state *gts,
+int gru_is_gts_stealable(struct gru_thread_state *gts,
 		struct gru_blade_state *bs)
 {
-	if (is_kernel_context(gts))
-		return down_write_trylock(&bs->bs_kgts_sema);
-	else
-		return mutex_trylock(&gts->ts_ctxlock);
+	int ret = 0, ret2;
+	if (is_kernel_context(gts)) {
+		ret = down_write_trylock(&bs->bs_kgts_sema);
+	} else {
+		ret2 = down_read_trylock(&gts->ts_mm->mmap_sem);
+		if (ret2)
+			ret = mutex_trylock(&gts->ts_ctxlock);
+		if (ret2 && !ret)
+			up_read(&gts->ts_mm->mmap_sem);
+	}
+	return ret;
 }
 
-static void gts_stolen(struct gru_thread_state *gts,
+void gru_gts_stolen(struct gru_thread_state *gts,
 		struct gru_blade_state *bs)
 {
 	if (is_kernel_context(gts)) {
@@ -775,6 +783,7 @@ static void gts_stolen(struct gru_thread
 		STAT(steal_kernel_context);
 	} else {
 		mutex_unlock(&gts->ts_ctxlock);
+		up_read(&gts->ts_mm->mmap_sem);
 		STAT(steal_user_context);
 	}
 }
@@ -819,7 +828,7 @@ void gru_steal_context(struct gru_thread
 			 	* success are high. If trylock fails, try to steal a
 			 	* different GSEG.
 			 	*/
-				if (ngts && is_gts_stealable(ngts, blade))
+				if (ngts && gru_is_gts_stealable(ngts, blade))
 					break;
 				ngts = NULL;
 			}
@@ -839,7 +848,7 @@ void gru_steal_context(struct gru_thread
 		gts->ustats.context_stolen++;
 		ngts->ts_steal_jiffies = jiffies;
 		gru_unload_context(ngts, is_kernel_context(ngts) ? 0 : 1);
-		gts_stolen(ngts, blade);
+		gru_gts_stolen(ngts, blade);
 	} else {
 		STAT(steal_context_failed);
 	}
@@ -951,6 +960,8 @@ again:
 		if (!gru_assign_gru_context(gts)) {
 			preempt_enable();
 			mutex_unlock(&gts->ts_ctxlock);
+			if (signal_pending(current))
+				return VM_FAULT_NOPAGE;
 			set_current_state(TASK_INTERRUPTIBLE);
 			schedule_timeout(GRU_ASSIGN_DELAY);  /* true hack ZZZ */
 			if (gts->ts_steal_jiffies + GRU_STEAL_DELAY < jiffies)
Index: linux/drivers/misc/sgi-gru/grutables.h
===================================================================
--- linux.orig/drivers/misc/sgi-gru/grutables.h	2010-07-19 10:25:12.510295762 -0500
+++ linux/drivers/misc/sgi-gru/grutables.h	2010-07-19 10:25:31.274286154 -0500
@@ -115,10 +115,14 @@
  *       task -->|
  *       task ---+---> mm ->------ (notifier) -------+-> gms
  *                     |                             |
+ *                     |              +------------->|
+ *                     |             /               |
  *                     |--> vma -> vdata ---> gts--->|		GSEG1 (thread1)
  *                     |                  |          |
  *                     |                  +-> gts--->|		GSEG1 (thread2)
  *                     |                             |
+ *                     |              +------------->|
+ *                     |             /               |
  *                     |--> vma -> vdata ---> gts--->|		GSEG2 (thread2)
  *                     .
  *                     .
@@ -336,6 +340,7 @@ struct gru_mm_struct {
 	wait_queue_head_t	ms_wait_queue;
 	DECLARE_BITMAP(ms_asidmap, GRU_MAX_GRUS);
 	struct gru_mm_tracker	ms_asids[GRU_MAX_GRUS];
+	struct mm_struct	*ms_mm;		/* for mmu_notifier_unreg */
 };
 
 /*
@@ -345,6 +350,7 @@ struct gru_mm_struct {
 struct gru_vma_data {
 	spinlock_t		vd_lock;	/* Serialize access to vma */
 	struct list_head	vd_head;	/* head of linked list of gts */
+	struct gru_mm_struct	*vd_gms;	/* asid & ioproc struct */
 	long			vd_user_options;/* misc user option flags */
 	int			vd_cbr_au_count;
 	int			vd_dsr_au_count;
@@ -469,7 +475,6 @@ struct gru_blade_state {
 							   reserved DSR */
 	struct rw_semaphore	bs_kgts_sema;		/* lock for kgts */
 	struct gru_thread_state *bs_kgts;		/* GTS for kernel use */
-
 	/* ---- the following are used for managing kernel async GRU CBRs --- */
 	void			*bs_async_cbr;		/* CBR for async */
 	struct completion	*bs_async_wq;
@@ -670,6 +675,10 @@ extern int gru_user_flush_tlb(unsigned l
 extern int gru_user_unload_context(unsigned long arg);
 extern int gru_get_exception_detail(unsigned long arg);
 extern int gru_set_context_option(unsigned long address);
+extern int gru_is_gts_stealable(struct gru_thread_state *gts,
+				struct gru_blade_state *blade);
+extern void gru_gts_stolen(struct gru_thread_state *gts,
+				struct gru_blade_state *blade);
 extern void gru_check_context_placement(struct gru_thread_state *gts);
 extern int gru_cpu_fault_map_id(void);
 extern struct vm_area_struct *gru_find_vma(struct mm_struct *mm, unsigned long vaddr);
Index: linux/drivers/misc/sgi-gru/grutlbpurge.c
===================================================================
--- linux.orig/drivers/misc/sgi-gru/grutlbpurge.c	2010-07-19 10:25:29.491266141 -0500
+++ linux/drivers/misc/sgi-gru/grutlbpurge.c	2010-07-19 10:25:31.298287925 -0500
@@ -263,21 +263,10 @@ static void gru_invalidate_page(struct m
 	gru_dbg(grudev, "gms %p, address 0x%lx\n", gms, address);
 }
 
-static void gru_release(struct mmu_notifier *mn, struct mm_struct *mm)
-{
-	struct gru_mm_struct *gms = container_of(mn, struct gru_mm_struct,
-						 ms_notifier);
-
-	gms->ms_released = 1;
-	gru_dbg(grudev, "gms %p\n", gms);
-}
-
-
 static const struct mmu_notifier_ops gru_mmuops = {
 	.invalidate_page	= gru_invalidate_page,
 	.invalidate_range_start	= gru_invalidate_range_start,
 	.invalidate_range_end	= gru_invalidate_range_end,
-	.release		= gru_release,
 };
 
 /* Move this to the basic mmu_notifier file. But for now... */
@@ -316,6 +305,7 @@ struct gru_mm_struct *gru_register_mmu_n
 			STAT(gms_alloc);
 			spin_lock_init(&gms->ms_asid_lock);
 			gms->ms_notifier.ops = &gru_mmuops;
+			gms->ms_mm = current->mm;
 			atomic_set(&gms->ms_refcnt, 1);
 			init_waitqueue_head(&gms->ms_wait_queue);
 			err = __mmu_notifier_register(&gms->ms_notifier, current->mm);
@@ -328,6 +318,9 @@ struct gru_mm_struct *gru_register_mmu_n
 	return gms;
 error:
 	kfree(gms);
+	/* mmu_notifier_register EINTR is reported as EAGAIN */
+	if (err == -EINTR)
+		err = -EAGAIN;
 	return ERR_PTR(err);
 }
 
@@ -336,8 +329,7 @@ void gru_drop_mmu_notifier(struct gru_mm
 	gru_dbg(grudev, "gms %p, refcnt %d, released %d\n", gms,
 		atomic_read(&gms->ms_refcnt), gms->ms_released);
 	if (atomic_dec_return(&gms->ms_refcnt) == 0) {
-		if (!gms->ms_released)
-			mmu_notifier_unregister(&gms->ms_notifier, current->mm);
+		mmu_notifier_unregister(&gms->ms_notifier, gms->ms_mm);
 		kfree(gms);
 		STAT(gms_free);
 	}

[Patch 11/25] GRU - interrupt fix for processors without core 0

[steiner]

[-- Attachment #1: uv_gru_interrupt_0 --]
[-- Type: text/plain, Size: 1119 bytes --]

From: Jack Steiner <steiner@sgi.com>

GRU interrupts for the kernel context are targeted to core 0
of the lowest socket on the blade.

Fix an endcase for 6-socket processors that do not have a
APICID for core 0. (There are potentially 8 cores in the
socket. A 6 core socket has 2 cores disabled. If core 0 was
disabled, the old code did not work).

Signed-off-by: Jack Steiner <steiner@sgi.com>

---
 drivers/misc/sgi-gru/grumain.c |    2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

Index: linux/drivers/misc/sgi-gru/grumain.c
===================================================================
--- linux.orig/drivers/misc/sgi-gru/grumain.c	2010-06-09 08:11:43.740083860 -0500
+++ linux/drivers/misc/sgi-gru/grumain.c	2010-06-09 08:11:44.128107324 -0500
@@ -618,7 +618,7 @@ void gru_load_context(struct gru_thread_
 		cch->unmap_enable = 1;
 		cch->tfm_done_bit_enable = 1;
 		cch->cb_int_enable = 1;
-		cch->tlb_int_select = 0;	/* For now, ints go to cpu 0 */
+		cch->tlb_int_select = uv_cpu_core_number(0); /* For now, ints go to cpu 0 */
 	} else {
 		cch->unmap_enable = 0;
 		cch->tfm_done_bit_enable = 0;

[Patch 12/25] GRU - add gru hub number to context status

[steiner]

[-- Attachment #1: uv_gru_statistics --]
[-- Type: text/plain, Size: 1662 bytes --]

From: Jack Steiner <steiner@sgi.com>

Add the GRU hub number to the output of the GRU cch_status &
gru_status files (in /proc/sgi_uv/gru).

Signed-off-by: Jack Steiner <steiner@sgi.com>


---
 drivers/misc/sgi-gru/gruprocfs.c |   12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

Index: linux/drivers/misc/sgi-gru/gruprocfs.c
===================================================================
--- linux.orig/drivers/misc/sgi-gru/gruprocfs.c	2010-06-09 08:11:42.456084620 -0500
+++ linux/drivers/misc/sgi-gru/gruprocfs.c	2010-06-09 08:11:44.561086699 -0500
@@ -217,18 +217,18 @@ static int gru_seq_show(struct seq_file
 	struct gru_state *gru = GID_TO_GRU(gid);
 
 	if (gid == 0) {
-		seq_printf(file, "#%5s%5s%7s%6s%6s%8s%6s%6s\n", "gid", "nid",
+		seq_printf(file, "#%5s%5s%7s%6s%6s%8s%6s%6s\n", "hub", "gid",
 			   "ctx", "cbr", "dsr", "ctx", "cbr", "dsr");
-		seq_printf(file, "#%5s%5s%7s%6s%6s%8s%6s%6s\n", "", "", "busy",
-			   "busy", "busy", "free", "free", "free");
+		seq_printf(file, "#%5s%5s%7s%6s%6s%8s%6s%6s\n", "", "", "total",
+			   "total", "total", "free", "free", "free");
 	}
 	if (gru) {
 		ctxfree = GRU_NUM_CCH - gru->gs_active_contexts;
 		cbrfree = hweight64(gru->gs_cbr_map) * GRU_CBR_AU_SIZE;
 		dsrfree = hweight64(gru->gs_dsr_map) * GRU_DSR_AU_BYTES;
-		seq_printf(file, " %5d%5d%7ld%6ld%6ld%8ld%6ld%6ld\n",
-			   gru->gs_gid, gru->gs_blade_id, GRU_NUM_CCH - ctxfree,
-			   GRU_NUM_CBE - cbrfree, GRU_NUM_DSR_BYTES - dsrfree,
+		seq_printf(file, " %5d%5d%7d%6d%6d%8ld%6ld%6ld\n",
+			   gru->gs_blade_id, gru->gs_gid,
+			   GRU_NUM_CCH, GRU_NUM_CBE, GRU_NUM_DSR_BYTES,
 			   ctxfree, cbrfree, dsrfree);
 	}
 

[Patch 13/25] GRU - delete obsolete debug code

[steiner]

[-- Attachment #1: uv_gru_delete_debug_config_info --]
[-- Type: text/plain, Size: 2204 bytes --]

From: Jack Steiner <steiner@sgi.com>

SGI UV driver. Delete some debug code that has outlived it's usefullness.
The code is not part of a supported API

Signed-off-by: Jack Steiner <steiner@sgi.com>

---
 drivers/misc/sgi-gru/grufile.c |   25 -------------------------
 drivers/misc/sgi-gru/grulib.h  |    3 ---
 2 files changed, 28 deletions(-)

Index: linux/drivers/misc/sgi-gru/grufile.c
===================================================================
--- linux.orig/drivers/misc/sgi-gru/grufile.c	2010-06-09 08:11:43.712036004 -0500
+++ linux/drivers/misc/sgi-gru/grufile.c	2010-06-09 08:11:44.973081222 -0500
@@ -166,28 +166,6 @@ static int gru_create_new_context(unsign
 	return ret;
 }
 
-/*
- * Get GRU configuration info (temp - for emulator testing)
- */
-static long gru_get_config_info(unsigned long arg)
-{
-	struct gru_config_info info;
-	int nodesperblade;
-
-	if (num_online_nodes() > 1 &&
-			(uv_node_to_blade_id(1) == uv_node_to_blade_id(0)))
-		nodesperblade = 2;
-	else
-		nodesperblade = 1;
-	info.cpus = num_online_cpus();
-	info.nodes = num_online_nodes();
-	info.blades = info.nodes / nodesperblade;
-	info.chiplets = GRU_CHIPLETS_PER_BLADE * info.blades;
-
-	if (copy_to_user((void __user *)arg, &info, sizeof(info)))
-		return -EFAULT;
-	return 0;
-}
 
 /*
  * gru_file_unlocked_ioctl
@@ -226,9 +204,6 @@ static long gru_file_unlocked_ioctl(stru
 	case GRU_KTEST:
 		err = gru_ktest(arg);
 		break;
-	case GRU_GET_CONFIG_INFO:
-		err = gru_get_config_info(arg);
-		break;
 	case GRU_DUMP_CHIPLET_STATE:
 		err = gru_dump_chiplet_request(arg);
 		break;
Index: linux/drivers/misc/sgi-gru/grulib.h
===================================================================
--- linux.orig/drivers/misc/sgi-gru/grulib.h	2010-06-09 08:11:41.655538228 -0500
+++ linux/drivers/misc/sgi-gru/grulib.h	2010-06-09 08:11:44.973081222 -0500
@@ -53,9 +53,6 @@
 /* For user TLB flushing (primarily for tests) */
 #define GRU_USER_FLUSH_TLB		_IOWR(GRU_IOCTL_NUM, 50, void *)
 
-/* Get some config options (primarily for tests & emulator) */
-#define GRU_GET_CONFIG_INFO		_IOWR(GRU_IOCTL_NUM, 51, void *)
-
 /* Various kernel self-tests */
 #define GRU_KTEST			_IOWR(GRU_IOCTL_NUM, 52, void *)
 

[Patch 14/25] GRU - add polling for tlb misses

[steiner]

[-- Attachment #1: uv_gru_tlb_polling --]
[-- Type: text/plain, Size: 11660 bytes --]

From: Jack Steiner <steiner@sgi.com>

Currently, the GRU driver processes TLB misses by sending an interrupt to the cpu.
The TLB is updated from the interrupt handler.

Some workloads have unused cpus. This patch (still experimental) uses idle
cpus to poll for TLB misses. When a miss occurs, the TLB is updated directly
w/o having to send interrupts to the cpu.

Signed-off-by: Jack Steiner <steiner@sgi.com>

---
 drivers/misc/sgi-gru/gru.h       |    1 
 drivers/misc/sgi-gru/grufault.c  |   50 ++++++++++++++++++++++++++++++++++-----
 drivers/misc/sgi-gru/grufile.c   |   20 +++++++++++++++
 drivers/misc/sgi-gru/grulib.h    |   10 +++++++
 drivers/misc/sgi-gru/grumain.c   |   33 +++++++++++--------------
 drivers/misc/sgi-gru/gruprocfs.c |    2 +
 drivers/misc/sgi-gru/grutables.h |    7 ++++-
 7 files changed, 97 insertions(+), 26 deletions(-)

Index: linux/drivers/misc/sgi-gru/gru.h
===================================================================
--- linux.orig/drivers/misc/sgi-gru/gru.h	2010-07-19 10:23:16.266244020 -0500
+++ linux/drivers/misc/sgi-gru/gru.h	2010-07-19 10:25:40.974376072 -0500
@@ -71,7 +71,6 @@ struct gru_gseg_statistics {
 #define GRU_OPT_MISS_USER_POLL	0x0001	/* User will poll CB for faults */
 #define GRU_OPT_MISS_FMM_INTR	0x0002	/* Send interrupt to cpu to
 					   handle fault */
-#define GRU_OPT_MISS_FMM_POLL	0x0003	/* Use system polling thread */
 #define GRU_OPT_MISS_MASK	0x0003	/* Mask for TLB MISS option */
 
 
Index: linux/drivers/misc/sgi-gru/grufault.c
===================================================================
--- linux.orig/drivers/misc/sgi-gru/grufault.c	2010-07-19 10:25:31.203387741 -0500
+++ linux/drivers/misc/sgi-gru/grufault.c	2010-07-19 10:25:40.978392010 -0500
@@ -156,7 +156,7 @@ static void get_clear_fault_map(struct g
 	unsigned long i, k;
 	struct gru_tlb_fault_map *tfm;
 
-	tfm = get_tfm_for_cpu(gru, gru_cpu_fault_map_id());
+	tfm = get_tfm_for_cpu(gru, gru_cpu_fault_map_id(gru));
 	prefetchw(tfm);		/* Helps on hardware, required for emulator */
 	for (i = 0; i < BITS_TO_LONGS(GRU_NUM_CBE); i++) {
 		k = tfm->fault_bits[i];
@@ -325,7 +325,7 @@ static void gru_preload_tlb(struct gru_s
 	unsigned long vaddr = 0, gpa;
 	int ret, pageshift;
 
-	if (cbe->opccpy != OP_BCOPY)
+	if (cbe->opccpy != OP_BCOPY || (cbe->cbrexecstatus & CBR_EXS_TLB_INVAL))
 		return;
 
 	if (fault_vaddr == cbe->cbe_baddr0)
@@ -546,8 +546,6 @@ static irqreturn_t gru_intr(int chiplet,
 	struct completion *cmp;
 	int cbrnum, ctxnum, multi = 0;
 
-	STAT(intr);
-
 	gru = &gru_base[blade]->bs_grus[chiplet];
 	if (!gru) {
 		dev_err(grudev, "GRU: invalid interrupt: cpu %d, chiplet %d\n",
@@ -610,14 +608,54 @@ static irqreturn_t gru_intr(int chiplet,
 	return IRQ_HANDLED;
 }
 
+#define YIELD_TICKS	(HZ / 20)
+void gru_intr_poll(int chiplet, int blade)
+{
+	struct gru_state *gru;
+	struct gru_tlb_fault_map *tfm;
+	unsigned long j, j_yield = 0, j_intr = 0;
+
+	gru = &gru_base[blade]->bs_grus[chiplet];
+	tfm = get_tfm_for_cpu(gru, 0);
+	gru->gs_fmm_polling_mode = 1;
+
+	while (likely(!signal_pending(current))) {
+		j = jiffies;
+		if (j_intr == j) {
+			cpu_relax();
+		} else {
+			__monitor(tfm, 0, 0);
+			smp_mb();
+			if (likely(tfm->fault_bits[0] + tfm->fault_bits[1] + tfm->done_bits[0] + tfm->done_bits[1]) == 0) {
+				__mwait(0x0, 0);
+				STAT(intr_poll);
+			}
+		}
+		if (likely(tfm->fault_bits[0] + tfm->fault_bits[1] + tfm->done_bits[0] + tfm->done_bits[1])) {
+			gru_intr(chiplet, blade);
+			STAT(intr_poll_found);
+			j_intr = j;
+		}
+		if (unlikely(j < j_yield)) {
+			yield();
+			j_yield = jiffies + YIELD_TICKS;
+		}
+	}
+	gru->gs_fmm_polling_mode = 0;
+}
+
 irqreturn_t gru0_intr(int irq, void *dev_id)
 {
-	return gru_intr(0, uv_numa_blade_id());
+	gru_intr(0, uv_numa_blade_id());
+	STAT(intr);
+	return IRQ_HANDLED;
 }
 
 irqreturn_t gru1_intr(int irq, void *dev_id)
 {
-	return gru_intr(1, uv_numa_blade_id());
+	gru_intr(1, uv_numa_blade_id());
+	STAT(intr);
+	return IRQ_HANDLED;
 }
 
 irqreturn_t gru_intr_mblade(int irq, void *dev_id)
Index: linux/drivers/misc/sgi-gru/grufile.c
===================================================================
--- linux.orig/drivers/misc/sgi-gru/grufile.c	2010-07-19 10:25:39.950898415 -0500
+++ linux/drivers/misc/sgi-gru/grufile.c	2010-07-19 10:25:41.002455321 -0500
@@ -127,6 +127,23 @@ static int gru_file_mmap(struct file *fi
 	return 0;
 }
 
+int gru_enable_polling_mode(unsigned long arg)
+{
+	struct gru_fmm_polling_req req;
+	struct gru_state *gru;
+
+	if (copy_from_user(&req, (void __user *)arg, sizeof(req)))
+		return -EFAULT;
+
+	if (req.gid >= gru_max_gids)
+		return -EINVAL;
+
+	gru = GID_TO_GRU(req.gid);
+	gru_intr_poll(gru->gs_chiplet_id, gru->gs_blade_id);
+	return 0;
+}
+
+
 /*
  * Create a new GRU context
  */
@@ -186,6 +203,9 @@ static long gru_file_unlocked_ioctl(stru
 	case GRU_SET_CONTEXT_OPTION:
 		err = gru_set_context_option(arg);
 		break;
+	case GRU_FMM_POLLING_MODE:
+		err = gru_enable_polling_mode(arg);
+		break;
 	case GRU_USER_GET_EXCEPTION_DETAIL:
 		err = gru_get_exception_detail(arg);
 		break;
Index: linux/drivers/misc/sgi-gru/grulib.h
===================================================================
--- linux.orig/drivers/misc/sgi-gru/grulib.h	2010-07-19 10:25:39.950898415 -0500
+++ linux/drivers/misc/sgi-gru/grulib.h	2010-07-19 10:25:41.042456010 -0500
@@ -50,6 +50,9 @@
 /* For getting gseg statistics */
 #define GRU_GET_GSEG_STATISTICS		_IOWR(GRU_IOCTL_NUM, 12, void *)
 
+/* For switching a GRU to FMM polling mode */
+#define GRU_FMM_POLLING_MODE		_IOWR(GRU_IOCTL_NUM, 13, void *)
+
 /* For user TLB flushing (primarily for tests) */
 #define GRU_USER_FLUSH_TLB		_IOWR(GRU_IOCTL_NUM, 50, void *)
 
@@ -85,6 +88,13 @@ struct gru_unload_context_req {
 };
 
 /*
+ * Structure used to initiate GRU polling for TLB misses
+ */
+struct gru_fmm_polling_req {
+	unsigned int	gid;
+};
+
+/*
  * Structure used to set context options
  */
 enum {sco_gseg_owner, sco_cch_req_slice, sco_blade_chiplet};
Index: linux/drivers/misc/sgi-gru/grumain.c
===================================================================
--- linux.orig/drivers/misc/sgi-gru/grumain.c	2010-07-19 10:25:37.570888442 -0500
+++ linux/drivers/misc/sgi-gru/grumain.c	2010-07-19 10:25:41.058388796 -0500
@@ -51,7 +51,7 @@ struct device *grudev = &gru_device;
  * multiple cpus may be using the same map.
  *	ZZZ should be inline but did not work on emulator
  */
-int gru_cpu_fault_map_id(void)
+int gru_cpu_fault_map_id(struct gru_state *gru)
 {
 #ifdef CONFIG_IA64
 	return uv_blade_processor_id() % GRU_NUM_TFM;
@@ -59,6 +59,8 @@ int gru_cpu_fault_map_id(void)
 	int cpu = smp_processor_id();
 	int id, core;
 
+	if (gru->gs_fmm_polling_mode)
+		return 0;
 	core = uv_cpu_core_number(cpu);
 	id = core + UV_MAX_INT_CORES * uv_cpu_socket_number(cpu);
 	return id;
@@ -596,14 +598,11 @@ void gru_load_context(struct gru_thread_
 
 	cch = get_cch(gru->gs_gru_base_vaddr, ctxnum);
 	lock_cch_handle(cch);
-	cch->tfm_fault_bit_enable =
-	    (gts->ts_user_options == GRU_OPT_MISS_FMM_POLL
-	     || gts->ts_user_options == GRU_OPT_MISS_FMM_INTR);
-	cch->tlb_int_enable = (gts->ts_user_options == GRU_OPT_MISS_FMM_INTR);
-	if (cch->tlb_int_enable) {
-		gts->ts_tlb_int_select = gru_cpu_fault_map_id();
-		cch->tlb_int_select = gts->ts_tlb_int_select;
-	}
+	cch->tfm_fault_bit_enable = gts->ts_user_options == GRU_OPT_MISS_FMM_INTR;
+	gts->ts_tlb_int_select = gru_cpu_fault_map_id(gru);
+	cch->tlb_int_select = gts->ts_tlb_int_select;
+	cch->tlb_int_enable = !gru->gs_fmm_polling_mode &&
+			gts->ts_user_options == GRU_OPT_MISS_FMM_INTR;
 	if (gts->ts_cch_req_slice >= 0) {
 		cch->req_slice_set_enable = 1;
 		cch->req_slice = gts->ts_cch_req_slice;
@@ -671,11 +670,9 @@ int gru_update_cch(struct gru_thread_sta
 			BUG();
 		for (i = 0; i < 8; i++)
 			cch->sizeavail[i] = gts->ts_sizeavail;
-		gts->ts_tlb_int_select = gru_cpu_fault_map_id();
-		cch->tlb_int_select = gru_cpu_fault_map_id();
-		cch->tfm_fault_bit_enable =
-		  (gts->ts_user_options == GRU_OPT_MISS_FMM_POLL
-		    || gts->ts_user_options == GRU_OPT_MISS_FMM_INTR);
+		gts->ts_tlb_int_select = gru_cpu_fault_map_id(gru);
+		cch->tlb_int_select = gts->ts_tlb_int_select;
+		cch->tfm_fault_bit_enable = gts->ts_user_options == GRU_OPT_MISS_FMM_INTR;
 		if (cch_start(cch))
 			BUG();
 		ret = 1;
@@ -692,14 +689,14 @@ exit:
  * 	- task has migrated to a different cpu on the same blade where
  * 	  it was previously running.
  */
-static int gru_retarget_intr(struct gru_thread_state *gts)
+static int gru_retarget_intr(struct gru_state *gru, struct gru_thread_state *gts)
 {
 	if (gts->ts_tlb_int_select < 0
-	    || gts->ts_tlb_int_select == gru_cpu_fault_map_id())
+	    || gts->ts_tlb_int_select == gru_cpu_fault_map_id(gru))
 		return 0;
 
 	gru_dbg(grudev, "retarget from %d to %d\n", gts->ts_tlb_int_select,
-		gru_cpu_fault_map_id());
+		gru_cpu_fault_map_id(gru));
 	return gru_update_cch(gts);
 }
 
@@ -745,7 +742,7 @@ void gru_check_context_placement(struct
 	if (!gru_check_chiplet_assignment(gru, gts)) {
 		STAT(check_context_unload);
 		gru_unload_context(gts, 1);
-	} else if (gru_retarget_intr(gts)) {
+	} else if (gru_retarget_intr(gru, gts)) {
 		STAT(check_context_retarget_intr);
 	}
 }
Index: linux/drivers/misc/sgi-gru/gruprocfs.c
===================================================================
--- linux.orig/drivers/misc/sgi-gru/gruprocfs.c	2010-07-19 10:25:39.034257217 -0500
+++ linux/drivers/misc/sgi-gru/gruprocfs.c	2010-07-19 10:25:41.078288158 -0500
@@ -72,6 +72,8 @@ static int statistics_show(struct seq_fi
 	printstat(s, intr);
 	printstat(s, intr_cbr);
 	printstat(s, intr_tfh);
+	printstat(s, intr_poll);
+	printstat(s, intr_poll_found);
 	printstat(s, intr_spurious);
 	printstat(s, intr_mm_lock_failed);
 	printstat(s, call_os);
Index: linux/drivers/misc/sgi-gru/grutables.h
===================================================================
--- linux.orig/drivers/misc/sgi-gru/grutables.h	2010-07-19 10:25:31.274286154 -0500
+++ linux/drivers/misc/sgi-gru/grutables.h	2010-07-19 10:25:41.130287807 -0500
@@ -204,6 +204,8 @@ struct gru_stats_s {
 	atomic_long_t intr;
 	atomic_long_t intr_cbr;
 	atomic_long_t intr_tfh;
+	atomic_long_t intr_poll;
+	atomic_long_t intr_poll_found;
 	atomic_long_t intr_spurious;
 	atomic_long_t intr_mm_lock_failed;
 	atomic_long_t call_os;
@@ -438,6 +440,8 @@ struct gru_state {
 							   local flush */
 	unsigned char		gs_tgh_first_remote;	/* starting TGH# for
 							   remote flush */
+	unsigned char		gs_fmm_polling_mode;	/* Chiplet is in TFH polling mode
+							   for TLB misses */
 	spinlock_t		gs_asid_lock;		/* lock used for
 							   assigning asids */
 	spinlock_t		gs_lock;		/* lock used for
@@ -668,6 +672,7 @@ extern void gru_kservices_exit(void);
 extern irqreturn_t gru0_intr(int irq, void *dev_id);
 extern irqreturn_t gru1_intr(int irq, void *dev_id);
 extern irqreturn_t gru_intr_mblade(int irq, void *dev_id);
+extern void gru_intr_poll(int chiplet, int blade);
 extern int gru_dump_chiplet_request(unsigned long arg);
 extern long gru_get_gseg_statistics(unsigned long arg);
 extern int gru_handle_user_call_os(unsigned long address);
@@ -680,7 +685,7 @@ extern int gru_is_gts_stealable(struct g
 extern void gru_gts_stolen(struct gru_thread_state *gts,
 				struct gru_blade_state *blade);
 extern void gru_check_context_placement(struct gru_thread_state *gts);
-extern int gru_cpu_fault_map_id(void);
+extern int gru_cpu_fault_map_id(struct gru_state *gru);
 extern struct vm_area_struct *gru_find_vma(struct mm_struct *mm, unsigned long vaddr);
 extern void gru_flush_all_tlb(struct gru_state *gru);
 extern int gru_proc_init(void);

[Patch 15/25] GRU - reorder interrupt processing

[steiner]

[-- Attachment #1: uv_gru_preload_tlb_inval --]
[-- Type: text/plain, Size: 2372 bytes --]

From: Jack Steiner <steiner@sgi.com>

Reorder the servicing of CBR completion interrupts & TLB miss interrupts.
There is a small performance gain to service TLB misses first. Completion
interrupts are rare.


Signed-off-by: Jack Steiner <steiner@sgi.com>

---
 drivers/misc/sgi-gru/grufault.c |   24 +++++++++++++-----------
 1 file changed, 13 insertions(+), 11 deletions(-)

Index: linux/drivers/misc/sgi-gru/grufault.c
===================================================================
--- linux.orig/drivers/misc/sgi-gru/grufault.c	2010-07-19 10:25:40.978392010 -0500
+++ linux/drivers/misc/sgi-gru/grufault.c	2010-07-19 10:25:45.626240841 -0500
@@ -7,7 +7,7 @@
  * These misses are reported either via interrupts or user polling of
  * the user CB.
  *
- *  Copyright (c) 2008 Silicon Graphics, Inc.  All Rights Reserved.
+ *  Copyright (c) 2008-2010 Silicon Graphics, Inc.  All Rights Reserved.
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -559,19 +559,10 @@ static irqreturn_t gru_intr(int chiplet,
 		imap.fault_bits[0], imap.fault_bits[1],
 		dmap.fault_bits[0], dmap.fault_bits[1]);
 
-	for_each_cbr_in_tfm(cbrnum, dmap.fault_bits) {
-		STAT(intr_cbr);
-		cmp = gru->gs_async_wq[cbrnum];
-		if (cmp)
-			complete(cmp);
-		gru_dbg(grudev, "gid %d, cbr_done %d, done %d\n",
-			gru->gs_gid, cbrnum, cmp ? cmp->done : -1);
-	}
-
 	for_each_cbr_in_tfm(cbrnum, imap.fault_bits) {
-		STAT(intr_tfh);
 		tfh = get_tfh_by_index(gru, cbrnum);
 		prefetchw(tfh);	/* Helps on hdw, required for emulator */
+		STAT(intr_tfh);
 
 		/*
 		 * When hardware sets a bit in the faultmap, it implicitly
@@ -596,6 +587,8 @@ static irqreturn_t gru_intr(int chiplet,
 			gts->ustats.interrupts++;
 		gts->ustats.fmm_tlbmiss++;
 		multi = 1;
+		if (!gts->ts_mm)
+			continue;
 		if (!gts->ts_force_cch_reload &&
 					down_read_trylock(&gts->ts_mm->mmap_sem)) {
 			gru_try_dropin(gru, gts, tfh, NULL);
@@ -605,6 +598,15 @@ static irqreturn_t gru_intr(int chiplet,
 			STAT(intr_mm_lock_failed);
 		}
 	}
+	for_each_cbr_in_tfm(cbrnum, dmap.fault_bits) {
+		STAT(intr_cbr);
+		cmp = gru->gs_async_wq[cbrnum];
+		if (cmp)
+			complete(cmp);
+		gru_dbg(grudev, "gid %d, cbr_done %d, done %d\n",
+			gru->gs_gid, cbrnum, cmp ? cmp->done : -1);
+	}
+
 	return IRQ_HANDLED;
 }
 

[Patch 16/25] GRU - add refcnt to vdata structure

[steiner]

[-- Attachment #1: uv_gru_refcnt_vdata --]
[-- Type: text/plain, Size: 3770 bytes --]

From: Jack Steiner <steiner@sgi.com>

The GRU vdata structure must have a reference count. 

Signed-off-by: Jack Steiner <steiner@sgi.com>

---
 drivers/misc/sgi-gru/grufile.c   |   21 +++++++++++++++++++--
 drivers/misc/sgi-gru/grumain.c   |    1 +
 drivers/misc/sgi-gru/gruprocfs.c |    1 +
 drivers/misc/sgi-gru/grutables.h |    2 ++
 4 files changed, 23 insertions(+), 2 deletions(-)

Index: linux/drivers/misc/sgi-gru/grufile.c
===================================================================
--- linux.orig/drivers/misc/sgi-gru/grufile.c	2010-07-19 10:25:41.002455321 -0500
+++ linux/drivers/misc/sgi-gru/grufile.c	2010-07-19 10:25:49.182243660 -0500
@@ -65,6 +65,19 @@ static struct miscdevice gru_miscdev;
  * Called when unmapping a device mapping. Frees all gru resources
  * and tables belonging to the vma.
  */
+static void gru_vma_open(struct vm_area_struct *vma)
+{
+	struct gru_vma_data *vdata;
+
+	if (!vma->vm_private_data)
+		return;
+	STAT(vdata_open);
+	vdata = vma->vm_private_data;
+	atomic_inc(&vdata->vd_refcnt);
+	gru_dbg(grudev, "vma %p, file %p, vdata %p\n", vma, vma->vm_file,
+				vdata);
+}
+
 static void gru_vma_close(struct vm_area_struct *vma)
 {
 	struct gru_vma_data *vdata;
@@ -75,9 +88,12 @@ static void gru_vma_close(struct vm_area
 		return;
 
 	vdata = vma->vm_private_data;
+	gru_dbg(grudev, "vma %p, file %p, vdata %p, refcnt %d\n", vma, vma->vm_file,
+				vdata, atomic_read(&vdata->vd_refcnt));
+	if (atomic_dec_return(&vdata->vd_refcnt) != 0)
+		return;
+
 	vma->vm_private_data = NULL;
-	gru_dbg(grudev, "vma %p, file %p, vdata %p\n", vma, vma->vm_file,
-				vdata);
 	list_for_each_safe(entry, next, &vdata->vd_head) {
 		gts =
 		    list_entry(entry, struct gru_thread_state, ts_next);
@@ -597,6 +613,7 @@ static struct miscdevice gru_miscdev = {
 };
 
 const struct vm_operations_struct gru_vm_ops = {
+	.open		= gru_vma_open,
 	.close		= gru_vma_close,
 	.fault		= gru_fault,
 };
Index: linux/drivers/misc/sgi-gru/grumain.c
===================================================================
--- linux.orig/drivers/misc/sgi-gru/grumain.c	2010-07-19 10:25:41.058388796 -0500
+++ linux/drivers/misc/sgi-gru/grumain.c	2010-07-19 10:25:49.186243696 -0500
@@ -374,6 +374,7 @@ struct gru_vma_data *gru_alloc_vma_data(
 	if (IS_ERR(gms))
 		goto err;
 	vdata->vd_gms = gms;
+	atomic_set(&vdata->vd_refcnt, 1);
 
 	STAT(vdata_alloc);
 	INIT_LIST_HEAD(&vdata->vd_head);
Index: linux/drivers/misc/sgi-gru/gruprocfs.c
===================================================================
--- linux.orig/drivers/misc/sgi-gru/gruprocfs.c	2010-07-19 10:25:41.078288158 -0500

+++ linux/drivers/misc/sgi-gru/gruprocfs.c	2010-07-19 10:25:49.214243610 -0500
@@ -42,6 +42,7 @@ static void printstat_val(struct seq_fil
 static int statistics_show(struct seq_file *s, void *p)
 {
 	printstat(s, vdata_alloc);
+	printstat(s, vdata_open);
 	printstat(s, vdata_free);
 	printstat(s, gts_alloc);
 	printstat(s, gts_free);
Index: linux/drivers/misc/sgi-gru/grutables.h
===================================================================
--- linux.orig/drivers/misc/sgi-gru/grutables.h	2010-07-19 10:25:41.130287807 -0500
+++ linux/drivers/misc/sgi-gru/grutables.h	2010-07-19 10:25:49.242242927 -0500
@@ -174,6 +174,7 @@ extern unsigned int gru_max_gids;
  */
 struct gru_stats_s {
 	atomic_long_t vdata_alloc;
+	atomic_long_t vdata_open;
 	atomic_long_t vdata_free;
 	atomic_long_t gts_alloc;
 	atomic_long_t gts_free;
@@ -351,6 +352,7 @@ struct gru_mm_struct {
  */
 struct gru_vma_data {
 	spinlock_t		vd_lock;	/* Serialize access to vma */
+	atomic_t		vd_refcnt;
 	struct list_head	vd_head;	/* head of linked list of gts */
 	struct gru_mm_struct	*vd_gms;	/* asid & ioproc struct */
 	long			vd_user_options;/* misc user option flags */

[Patch 17/25] GRU - no panic on gru malfunction

[steiner]

[-- Attachment #1: uv_gru_no_panic_gru_failure --]
[-- Type: text/plain, Size: 1343 bytes --]

From: Jack Steiner <steiner@sgi.com>

If the GRU malfunctions, print a message instead of panicing the system.
This simplifies debugging since some of the debug tools can be used on
a live system. Flush the cache on instruction timeouts in case the
malfunction is related to a coherency issue (never seen this but I'm paranoid).

Signed-off-by: Jack Steiner <steiner@sgi.com>

---
 drivers/misc/sgi-gru/gruhandles.c |    3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

Index: linux/drivers/misc/sgi-gru/gruhandles.c
===================================================================
--- linux.orig/drivers/misc/sgi-gru/gruhandles.c	2010-06-09 08:11:43.724081727 -0500
+++ linux/drivers/misc/sgi-gru/gruhandles.c	2010-06-09 08:11:46.697237522 -0500
@@ -71,7 +71,7 @@ static void report_instruction_timeout(v
 	else if (TYPE_IS(TFH, goff))
 		id = "TFH";
 
-	panic(KERN_ALERT "GRU %p (%s) is malfunctioning\n", h, id);
+	printk(KERN_ALERT "GRU:%d %p (%s) is malfunctioning\n", smp_processor_id(), h, id);
 }
 
 static int wait_instruction_complete(void *h, enum mcs_op opc)
@@ -85,6 +85,7 @@ static int wait_instruction_complete(voi
 		if (status != CCHSTATUS_ACTIVE)
 			break;
 		if (GRU_OPERATION_TIMEOUT < (get_cycles() - start_time)) {
+			gru_flush_cache(h);
 			report_instruction_timeout(h);
 			start_time = get_cycles();
 		}

[Patch 18/25] GRU - contexts must contain cbrs

[steiner]

[-- Attachment #1: uv_gru_cbr_check --]
[-- Type: text/plain, Size: 1002 bytes --]

From: Jack Steiner <steiner@sgi.com>

Don't allow users to create GRU contexts with 0 CBRs. This does
not make sense & is not allowed by the chiplet. The chiplet will flag this
as an error and fail to allocate the context.

Signed-off-by: Jack Steiner <steiner@sgi.com>

---
 drivers/misc/sgi-gru/grufile.c |    2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

Index: linux/drivers/misc/sgi-gru/grufile.c
===================================================================
--- linux.orig/drivers/misc/sgi-gru/grufile.c	2010-06-09 08:11:46.261068458 -0500
+++ linux/drivers/misc/sgi-gru/grufile.c	2010-06-09 08:11:47.151963284 -0500
@@ -175,7 +175,7 @@ static int gru_create_new_context(unsign
 
 	if (req.data_segment_bytes > max_user_dsr_bytes)
 		return -EINVAL;
-	if (req.control_blocks > max_user_cbrs || !req.maximum_thread_count)
+	if (req.control_blocks == 0 || req.control_blocks > max_user_cbrs || !req.maximum_thread_count)
 		return -EINVAL;
 
 	if (!(req.options & GRU_OPT_MISS_MASK))

[Patch 19/25] GRU - update debug messages and comments

[steiner]

[-- Attachment #1: uv_gru_misc_cleanup --]
[-- Type: text/plain, Size: 6843 bytes --]

From: Jack Steiner <steiner@sgi.com>

Misc clean:
	- update debug messages
	- update stale comments in the code


Signed-off-by: Jack Steiner <steiner@sgi.com>

---
 drivers/misc/sgi-gru/grufault.c     |    1 -
 drivers/misc/sgi-gru/grukservices.h |    3 +--
 drivers/misc/sgi-gru/grulib.h       |    1 -
 drivers/misc/sgi-gru/grumain.c      |   26 +++++++++++++-------------
 drivers/misc/sgi-gru/grutlbpurge.c  |    1 -
 5 files changed, 14 insertions(+), 18 deletions(-)

Index: linux/drivers/misc/sgi-gru/grufault.c
===================================================================
--- linux.orig/drivers/misc/sgi-gru/grufault.c	2010-06-09 08:11:45.803958078 -0500
+++ linux/drivers/misc/sgi-gru/grufault.c	2010-06-09 08:11:47.573040296 -0500
@@ -209,7 +209,6 @@ static int non_atomic_pte_lookup(struct
  *
  * Convert a user virtual address to a physical address
  * Only supports Intel large pages (2MB only) on x86_64.
- *	ZZZ - hugepage support is incomplete
  *
  * NOTE: mmap_sem is already held on entry to this function. This
  * guarantees existence of the page tables.
Index: linux/drivers/misc/sgi-gru/grukservices.h
===================================================================
--- linux.orig/drivers/misc/sgi-gru/grukservices.h	2010-06-09 08:11:30.008045922 -0500
+++ linux/drivers/misc/sgi-gru/grukservices.h	2010-06-09 08:11:47.573040296 -0500
@@ -36,9 +36,8 @@
  * 	- multiple senders
  *	- cross partition message
  *
- * Missing features ZZZ:
+ * Missing features:
  * 	- user options for dealing with timeouts, queue full, etc.
- * 	- gru_create_message_queue() needs interrupt vector info
  */
 
 struct gru_message_queue_desc {
Index: linux/drivers/misc/sgi-gru/grulib.h
===================================================================
--- linux.orig/drivers/misc/sgi-gru/grulib.h	2010-06-09 08:11:45.407956625 -0500
+++ linux/drivers/misc/sgi-gru/grulib.h	2010-06-09 08:11:47.593081003 -0500
@@ -25,7 +25,6 @@
 
 /*
  * Maximum number of GRU segments that a user can have open
- * ZZZ temp - set high for testing. Revisit.
  */
 #define GRU_MAX_OPEN_CONTEXTS		32
 
Index: linux/drivers/misc/sgi-gru/grumain.c
===================================================================
--- linux.orig/drivers/misc/sgi-gru/grumain.c	2010-06-09 08:11:46.261068458 -0500
+++ linux/drivers/misc/sgi-gru/grumain.c	2010-06-09 08:11:47.621041821 -0500
@@ -265,7 +265,7 @@ static int gru_load_mm_tracker(struct gr
 	spin_unlock(&gms->ms_asid_lock);
 
 	gru_dbg(grudev,
-		"gid %d, gts %p, gms %p, ctxnum %d, asid 0x%x, asidmap 0x%lx\n",
+		"gid %d, gts 0x%p, gms %p, ctxnum %d, asid 0x%x, asidmap 0x%lx\n",
 		gru->gs_gid, gts, gms, gts->ts_ctxnum, asid,
 		gms->ms_asidmap[0]);
 	return asid;
@@ -284,7 +284,7 @@ static void gru_unload_mm_tracker(struct
 	spin_lock(&gru->gs_asid_lock);
 	BUG_ON((asids->mt_ctxbitmap & ctxbitmap) != ctxbitmap);
 	asids->mt_ctxbitmap ^= ctxbitmap;
-	gru_dbg(grudev, "gid %d, gts %p, gms %p, ctxnum 0x%d, asidmap 0x%lx\n",
+	gru_dbg(grudev, "gid %d, gts 0x%p, gms %p, ctxnum 0x%d, asidmap 0x%lx\n",
 		gru->gs_gid, gts, gms, gts->ts_ctxnum, gms->ms_asidmap[0]);
 	spin_unlock(&gru->gs_asid_lock);
 	spin_unlock(&gms->ms_asid_lock);
@@ -350,12 +350,12 @@ struct gru_thread_state *gru_alloc_gts(s
 	if (vma) {
 		struct gru_vma_data *vdata = vma->vm_private_data;
 
-		gts->ts_gms = vdata->vd_gms;;
+		gts->ts_gms = vdata->vd_gms;
 		gts->ts_mm = current->mm;
 		gts->ts_vma = vma;
 	}
 
-	gru_dbg(grudev, "alloc gts %p\n", gts);
+	gru_dbg(grudev, "alloc gts 0x%p\n", gts);
 	return gts;
 }
 
@@ -399,7 +399,7 @@ struct gru_thread_state *gru_find_thread
 	spin_lock(&vdata->vd_lock);
 	gts = gru_find_current_gts_nolock(vdata, tsid);
 	spin_unlock(&vdata->vd_lock);
-	gru_dbg(grudev, "vma %p, gts %p\n", vma, gts);
+	gru_dbg(grudev, "vma %p, gts 0x%p\n", vma, gts);
 	return gts;
 }
 
@@ -430,7 +430,7 @@ struct gru_thread_state *gru_alloc_threa
 		list_add(&gts->ts_next, &vdata->vd_head);
 	}
 	spin_unlock(&vdata->vd_lock);
-	gru_dbg(grudev, "vma %p, gts %p\n", vma, gts);
+	gru_dbg(grudev, "vma %p, gts 0x%p\n", vma, gts);
 	return gts;
 }
 
@@ -442,7 +442,7 @@ static void gru_free_gru_context(struct
 	struct gru_state *gru;
 
 	gru = gts->ts_gru;
-	gru_dbg(grudev, "gts %p, gid %d\n", gts, gru->gs_gid);
+	gru_dbg(grudev, "gts 0x%p, gid %d\n", gts, gru->gs_gid);
 
 	spin_lock(&gru->gs_lock);
 	gru->gs_gts[gts->ts_ctxnum] = NULL;
@@ -565,7 +565,7 @@ void gru_unload_context(struct gru_threa
 		zap_vma_ptes(gts->ts_vma, UGRUADDR(gts), GRU_GSEG_PAGESIZE);
 	cch = get_cch(gru->gs_gru_base_vaddr, ctxnum);
 
-	gru_dbg(grudev, "gts %p, cbrmap 0x%lx, dsrmap 0x%lx\n",
+	gru_dbg(grudev, "gts 0x%p, cbrmap 0x%lx, dsrmap 0x%lx\n",
 		gts, gts->ts_cbr_map, gts->ts_dsr_map);
 	lock_cch_handle(cch);
 	if (cch_interrupt_sync(cch))
@@ -633,7 +633,7 @@ void gru_load_context(struct gru_thread_
 	err = cch_allocate(cch);
 	if (err) {
 		gru_dbg(grudev,
-			"err %d: cch %p, gts %p, cbr 0x%lx, dsr 0x%lx\n",
+			"err %d: cch %p, gts 0x%p, cbr 0x%lx, dsr 0x%lx\n",
 			err, cch, gts, gts->ts_cbr_map, gts->ts_dsr_map);
 		BUG();
 	}
@@ -645,7 +645,7 @@ void gru_load_context(struct gru_thread_
 		BUG();
 	unlock_cch_handle(cch);
 
-	gru_dbg(grudev, "gid %d, gts %p, cbrmap 0x%lx, dsrmap 0x%lx, tie %d, tis %d\n",
+	gru_dbg(grudev, "gid %d, gts 0x%p, cbrmap 0x%lx, dsrmap 0x%lx, tie %d, tis %d\n",
 		gts->ts_gru->gs_gid, gts, gts->ts_cbr_map, gts->ts_dsr_map,
 		(gts->ts_user_options == GRU_OPT_MISS_FMM_INTR), gts->ts_tlb_int_select);
 }
@@ -851,7 +851,7 @@ void gru_steal_context(struct gru_thread
 		STAT(steal_context_failed);
 	}
 	gru_dbg(grudev,
-		"stole gid %d, ctxnum %d from gts %p. Need cb %d, ds %d;"
+		"stole gid %d, ctxnum %d from gts 0x%p. Need cb %d, ds %d;"
 		" avail cb %ld, ds %ld\n",
 		gru->gs_gid, ctxnum, ngts, cbr, dsr, hweight64(gru->gs_cbr_map),
 		hweight64(gru->gs_dsr_map));
@@ -913,8 +913,8 @@ again:
 
 		STAT(assign_context);
 		gru_dbg(grudev,
-			"gseg %p, gts %p, gid %d, ctx %d, cbr %d, dsr %d\n",
-			gseg_virtual_address(gts->ts_gru, gts->ts_ctxnum), gts,
+			"gseg %p, pid %d, gts 0x%p, gid %d, ctx %d, cbr %d, dsr %d\n",
+			gseg_virtual_address(gts->ts_gru, gts->ts_ctxnum), current->pid, gts,
 			gts->ts_gru->gs_gid, gts->ts_ctxnum,
 			gts->ts_cbr_au_count, gts->ts_dsr_au_count);
 	} else {
Index: linux/drivers/misc/sgi-gru/grutlbpurge.c
===================================================================
--- linux.orig/drivers/misc/sgi-gru/grutlbpurge.c	2010-06-09 08:11:43.772080689 -0500
+++ linux/drivers/misc/sgi-gru/grutlbpurge.c	2010-06-09 08:11:47.645042345 -0500
@@ -162,7 +162,6 @@ void gru_flush_tlb_range(struct gru_mm_s
 	unsigned long num;
 	int grupagesize, pagesize, pageshift, gid, asid;
 
-	/* ZZZ TODO - handle huge pages */
 	pageshift = PAGE_SHIFT;
 	pagesize = (1UL << pageshift);
 	grupagesize = GRU_PAGESIZE(pageshift);

[Patch 20/25] GRU - add gsh information to gru dumps

[steiner]

[-- Attachment #1: uv_gru_kdump --]
[-- Type: text/plain, Size: 2741 bytes --]

From: Jack Steiner <steiner@sgi.com>

Add support for dumpping the GRU GSH state. This provides addition debugging information
for GRU failures.

Add missing cache flush instructions. For performance reasons, the CBE does not
alway maintain cache-coherency.

Signed-off-by: Jack Steiner <steiner@sgi.com>

---
 drivers/misc/sgi-gru/grukdump.c |   17 ++++++++++++++++-
 drivers/misc/sgi-gru/grulib.h   |    2 ++
 2 files changed, 18 insertions(+), 1 deletion(-)

Index: linux/drivers/misc/sgi-gru/grukdump.c
===================================================================
--- linux.orig/drivers/misc/sgi-gru/grukdump.c	2010-07-19 10:23:08.926244572 -0500
+++ linux/drivers/misc/sgi-gru/grukdump.c	2010-07-19 13:10:56.766719840 -0500
@@ -54,10 +54,16 @@ static int gru_dump_context_data(void *g
 	cb = gseg + GRU_CB_BASE;
 	cbe = grubase + GRU_CBE_BASE;
 	tfh = grubase + GRU_TFH_BASE;
+	sync_core();
+	gru_flush_cache(cbe);
 
 	for_each_cbr_in_allocation_map(i, &cch->cbr_allocation_map, scr) {
-		if (flush_cbrs)
+		if (flush_cbrs) {
 			gru_flush_cache(cb);
+			gru_flush_cache(tfh + i * GRU_HANDLE_STRIDE);
+		}
+		gru_flush_cache(cbe + i * GRU_HANDLE_STRIDE);
+		sync_core();
 		if (gru_user_copy_handle(&ubuf, cb))
 			goto fail;
 		if (gru_user_copy_handle(&ubuf, tfh + i * GRU_HANDLE_STRIDE))
@@ -130,6 +136,7 @@ static int gru_dump_context(struct gru_s
 	memset(&hdr, 0, sizeof(hdr));
 	grubase = gru->gs_gru_base_vaddr;
 	cch = get_cch(grubase, ctxnum);
+	gru_flush_cache(cch);
 	for (try = 0; try < CCH_LOCK_ATTEMPTS; try++) {
 		cch_locked =  trylock_cch_handle(cch);
 		if (cch_locked)
@@ -184,6 +191,12 @@ fail:
 	unlock_cch_handle(cch);
 	return -EFAULT;
 }
+static void dump_gsh_state(struct gru_state *gru,
+			   struct gru_dump_chiplet_state_req *req)
+{
+	memset(&req->gsh_cb_state, 0, sizeof(req->gsh_cb_state));
+	memset(&req->gsh_cch_state, 0, sizeof(req->gsh_cch_state));
+}
 
 int gru_dump_chiplet_request(unsigned long arg)
 {
@@ -214,6 +227,8 @@ int gru_dump_chiplet_request(unsigned lo
 		goto fail;
 	ubuf += ret;
 
+	dump_gsh_state(gru, &req);
+
 	for (ctxnum = 0; ctxnum < GRU_NUM_CCH; ctxnum++) {
 		if (req.ctxnum == ctxnum || req.ctxnum < 0) {
 			ret = gru_dump_context(gru, ctxnum, ubuf, ubufend,
Index: linux/drivers/misc/sgi-gru/grulib.h
===================================================================
--- linux.orig/drivers/misc/sgi-gru/grulib.h	2010-07-19 10:25:54.334243996 -0500
+++ linux/drivers/misc/sgi-gru/grulib.h	2010-07-19 13:10:35.154917927 -0500
@@ -130,6 +130,8 @@ struct gru_dump_chiplet_state_req {
 	size_t		buflen;
 	/* ---- output --- */
 	unsigned int	num_contexts;
+	unsigned long	gsh_cb_state[16];
+	unsigned long	gsh_cch_state[16];
 };
 
 #define GRU_DUMP_MAGIC	0x3474ab6c

[Patch 21/25] GRU - delete unused gru statistics structure

[steiner]

[-- Attachment #1: uv_delete_chiplet_info --]
[-- Type: text/plain, Size: 873 bytes --]

From: Jack Steiner <steiner@sgi.com>

Delete unused structure that was left over from debugging.

Signed-off-by: Jack Steiner <steiner@sgi.com>

---
 drivers/misc/sgi-gru/gru.h |   15 ---------------
 1 file changed, 15 deletions(-)

Index: linux/drivers/misc/sgi-gru/gru.h
===================================================================
--- linux.orig/drivers/misc/sgi-gru/gru.h	2010-06-09 08:11:45.375993599 -0500
+++ linux/drivers/misc/sgi-gru/gru.h	2010-06-09 08:11:48.856038221 -0500
@@ -39,21 +39,6 @@
 #endif
 
 /*
- * Structure for obtaining GRU resource information
- */
-struct gru_chiplet_info {
-	int	node;
-	int	chiplet;
-	int	blade;
-	int	total_dsr_bytes;
-	int	total_cbr;
-	int	total_user_dsr_bytes;
-	int	total_user_cbr;
-	int	free_user_dsr_bytes;
-	int	free_user_cbr;
-};
-
-/*
  * Statictics kept for each context.
  */
 struct gru_gseg_statistics {

[Patch 22/25] GRU - gru api cleanup

[steiner]

[-- Attachment #1: uv_gru_instruction_macros --]
[-- Type: text/plain, Size: 11571 bytes --]

From: Jack Steiner <steiner@sgi.com>

Header file cleanup & reorganization. A variant of the gru_instructions.h
header file is part of the user API. Move items that are NOT part of the
user API from this header file to internal header files.

	- move the CBX definition
	- move the exception retrybits
	- remove the option for mapped/unmapped mode from the
	  instruction macros. Users always run in mapped mode; the
	  kernel always runs in ummapped mode. This option does not
	  need to an option exposed via the macros

Signed-off-by: Jack Steiner <steiner@sgi.com>

---
 drivers/misc/sgi-gru/gru_instructions.h |   88 +++++++-------------------------
 drivers/misc/sgi-gru/gruhandles.h       |    1 
 drivers/misc/sgi-gru/grulib.h           |   34 ++++++++++++
 3 files changed, 55 insertions(+), 68 deletions(-)

Index: linux/drivers/misc/sgi-gru/gru_instructions.h
===================================================================
--- linux.orig/drivers/misc/sgi-gru/gru_instructions.h	2010-06-09 08:11:43.303963161 -0500
+++ linux/drivers/misc/sgi-gru/gru_instructions.h	2010-06-09 08:11:50.264036100 -0500
@@ -71,21 +71,6 @@ extern void gru_wait_abort_proc(void *cb
 #define CBSS_PUT_NACKED			5
 
 /*
- * Structure used to fetch exception detail for CBs that terminate with
- * CBS_EXCEPTION
- */
-struct control_block_extended_exc_detail {
-	unsigned long	cb;
-	int		opc;
-	int		ecause;
-	int		exopc;
-	long		exceptdet0;
-	int		exceptdet1;
-	int		cbrstate;
-	int		cbrexecstatus;
-};
-
-/*
  * Instruction formats
  */
 
@@ -289,19 +274,6 @@ struct gru_instruction {
 #define CBR_EXS_EXCEPTION			(1 << CBR_EXS_EXCEPTION_BIT)
 #define CBR_EXS_CB_INT_PENDING			(1 << CBR_EXS_CB_INT_PENDING_BIT)
 
-/*
- * Exceptions are retried for the following cases. If any OTHER bits are set
- * in ecause, the exception is not retryable.
- */
-#define EXCEPTION_RETRY_BITS (CBE_CAUSE_EXECUTION_HW_ERROR |		\
-			      CBE_CAUSE_TLBHW_ERROR |			\
-			      CBE_CAUSE_RA_REQUEST_TIMEOUT |		\
-			      CBE_CAUSE_RA_RESPONSE_NON_FATAL |		\
-			      CBE_CAUSE_HA_RESPONSE_NON_FATAL |		\
-			      CBE_CAUSE_RA_RESPONSE_DATA_ERROR |	\
-			      CBE_CAUSE_HA_RESPONSE_DATA_ERROR		\
-			      )
-
 /* Message queue head structure */
 union gru_mesqhead {
 	unsigned long	val;
@@ -311,7 +283,6 @@ union gru_mesqhead {
 	};
 };
 
-
 /* Generate the low word of a GRU instruction */
 static inline unsigned long
 __opdword(unsigned char opcode, unsigned char exopc, unsigned char xtype,
@@ -324,6 +295,7 @@ __opdword(unsigned char opcode, unsigned
 	   (iaa0 << GRU_CB_IAA0_SHFT) |
 	   (iaa1 << GRU_CB_IAA1_SHFT) |
 	   (ima << GRU_CB_IMA_SHFT) |
+	   (IMA_UNMAPPED << GRU_CB_IMA_SHFT) |
 	   (xtype << GRU_CB_XTYPE_SHFT) |
 	   (opcode << GRU_CB_OPC_SHFT) |
 	   (exopc << GRU_CB_EXOPC_SHFT);
@@ -349,12 +321,6 @@ static inline void gru_start_instruction
 }
 
 
-/* Convert "hints" to IMA */
-#define CB_IMA(h)		((h) | IMA_UNMAPPED)
-
-/* Convert data segment cache line index into TRI0 / TRI1 value */
-#define GRU_DINDEX(i)		((i) * GRU_CACHE_LINE_BYTES)
-
 /* Inline functions for GRU instructions.
  *     Note:
  *     	- nelem and stride are in elements
@@ -369,7 +335,7 @@ static inline void gru_vload_phys(void *
 	ins->nelem = 1;
 	ins->op1_stride = 1;
 	gru_start_instruction(ins, __opdword(OP_VLOAD, 0, XTYPE_DW, iaa, 0,
-					(unsigned long)tri0, CB_IMA(hints)));
+					(unsigned long)tri0, hints));
 }
 
 static inline void gru_vstore_phys(void *cb, unsigned long gpa,
@@ -381,7 +347,7 @@ static inline void gru_vstore_phys(void
 	ins->nelem = 1;
 	ins->op1_stride = 1;
 	gru_start_instruction(ins, __opdword(OP_VSTORE, 0, XTYPE_DW, iaa, 0,
-					(unsigned long)tri0, CB_IMA(hints)));
+					(unsigned long)tri0, hints));
 }
 
 static inline void gru_vload(void *cb, unsigned long mem_addr,
@@ -394,7 +360,7 @@ static inline void gru_vload(void *cb, u
 	ins->nelem = nelem;
 	ins->op1_stride = stride;
 	gru_start_instruction(ins, __opdword(OP_VLOAD, 0, xtype, IAA_RAM, 0,
-					(unsigned long)tri0, CB_IMA(hints)));
+					(unsigned long)tri0, hints));
 }
 
 static inline void gru_vstore(void *cb, unsigned long mem_addr,
@@ -407,7 +373,7 @@ static inline void gru_vstore(void *cb,
 	ins->nelem = nelem;
 	ins->op1_stride = stride;
 	gru_start_instruction(ins, __opdword(OP_VSTORE, 0, xtype, IAA_RAM, 0,
-					tri0, CB_IMA(hints)));
+					tri0, hints));
 }
 
 static inline void gru_ivload(void *cb, unsigned long mem_addr,
@@ -420,7 +386,7 @@ static inline void gru_ivload(void *cb,
 	ins->nelem = nelem;
 	ins->tri1_bufsize_64 = tri1;
 	gru_start_instruction(ins, __opdword(OP_IVLOAD, 0, xtype, IAA_RAM, 0,
-					tri0, CB_IMA(hints)));
+					tri0, hints));
 }
 
 static inline void gru_ivstore(void *cb, unsigned long mem_addr,
@@ -433,7 +399,7 @@ static inline void gru_ivstore(void *cb,
 	ins->nelem = nelem;
 	ins->tri1_bufsize_64 = tri1;
 	gru_start_instruction(ins, __opdword(OP_IVSTORE, 0, xtype, IAA_RAM, 0,
-					tri0, CB_IMA(hints)));
+					tri0, hints));
 }
 
 static inline void gru_vset(void *cb, unsigned long mem_addr,
@@ -447,7 +413,7 @@ static inline void gru_vset(void *cb, un
 	ins->nelem = nelem;
 	ins->op1_stride = stride;
 	gru_start_instruction(ins, __opdword(OP_VSET, 0, xtype, IAA_RAM, 0,
-					 0, CB_IMA(hints)));
+					 0, hints));
 }
 
 static inline void gru_ivset(void *cb, unsigned long mem_addr,
@@ -461,7 +427,7 @@ static inline void gru_ivset(void *cb, u
 	ins->nelem = nelem;
 	ins->tri1_bufsize_64 = tri1;
 	gru_start_instruction(ins, __opdword(OP_IVSET, 0, xtype, IAA_RAM, 0,
-					0, CB_IMA(hints)));
+					0, hints));
 }
 
 static inline void gru_vflush(void *cb, unsigned long mem_addr,
@@ -474,14 +440,14 @@ static inline void gru_vflush(void *cb,
 	ins->op1_stride = stride;
 	ins->nelem = nelem;
 	gru_start_instruction(ins, __opdword(OP_VFLUSH, 0, xtype, IAA_RAM, 0,
-					0, CB_IMA(hints)));
+					0, hints));
 }
 
 static inline void gru_nop(void *cb, int hints)
 {
 	struct gru_instruction *ins = (void *)cb;
 
-	gru_start_instruction(ins, __opdword(OP_NOP, 0, 0, 0, 0, 0, CB_IMA(hints)));
+	gru_start_instruction(ins, __opdword(OP_NOP, 0, 0, 0, 0, 0, hints));
 }
 
 
@@ -497,7 +463,7 @@ static inline void gru_bcopy(void *cb, c
 	ins->nelem = nelem;
 	ins->tri1_bufsize_64 = bufsize;
 	gru_start_instruction(ins, __opdword(OP_BCOPY, 0, xtype, IAA_RAM,
-					IAA_RAM, tri0, CB_IMA(hints)));
+					IAA_RAM, tri0, hints));
 }
 
 static inline void gru_bstore(void *cb, const unsigned long src,
@@ -510,7 +476,7 @@ static inline void gru_bstore(void *cb,
 	ins->op2_value_baddr1 = (long)dest;
 	ins->nelem = nelem;
 	gru_start_instruction(ins, __opdword(OP_BSTORE, 0, xtype, 0, IAA_RAM,
-					tri0, CB_IMA(hints)));
+					tri0, hints));
 }
 
 static inline void gru_gamir(void *cb, int exopc, unsigned long src,
@@ -520,7 +486,7 @@ static inline void gru_gamir(void *cb, i
 
 	ins->baddr0 = (long)src;
 	gru_start_instruction(ins, __opdword(OP_GAMIR, exopc, xtype, IAA_RAM, 0,
-					0, CB_IMA(hints)));
+					0, hints));
 }
 
 static inline void gru_gamirr(void *cb, int exopc, unsigned long src,
@@ -530,7 +496,7 @@ static inline void gru_gamirr(void *cb,
 
 	ins->baddr0 = (long)src;
 	gru_start_instruction(ins, __opdword(OP_GAMIRR, exopc, xtype, IAA_RAM, 0,
-					0, CB_IMA(hints)));
+					0, hints));
 }
 
 static inline void gru_gamer(void *cb, int exopc, unsigned long src,
@@ -544,7 +510,7 @@ static inline void gru_gamer(void *cb, i
 	ins->op1_stride = operand1;
 	ins->op2_value_baddr1 = operand2;
 	gru_start_instruction(ins, __opdword(OP_GAMER, exopc, xtype, IAA_RAM, 0,
-					0, CB_IMA(hints)));
+					0, hints));
 }
 
 static inline void gru_gamerr(void *cb, int exopc, unsigned long src,
@@ -557,7 +523,7 @@ static inline void gru_gamerr(void *cb,
 	ins->op1_stride = operand1;
 	ins->op2_value_baddr1 = operand2;
 	gru_start_instruction(ins, __opdword(OP_GAMERR, exopc, xtype, IAA_RAM, 0,
-					0, CB_IMA(hints)));
+					0, hints));
 }
 
 static inline void gru_gamxr(void *cb, unsigned long src,
@@ -568,7 +534,7 @@ static inline void gru_gamxr(void *cb, u
 	ins->baddr0 = (long)src;
 	ins->nelem = 4;
 	gru_start_instruction(ins, __opdword(OP_GAMXR, EOP_XR_CSWAP, XTYPE_DW,
-				 IAA_RAM, 0, 0, CB_IMA(hints)));
+				 IAA_RAM, 0, 0, hints));
 }
 
 static inline void gru_mesq(void *cb, unsigned long queue,
@@ -580,7 +546,7 @@ static inline void gru_mesq(void *cb, un
 	ins->baddr0 = (long)queue;
 	ins->nelem = nelem;
 	gru_start_instruction(ins, __opdword(OP_MESQ, 0, XTYPE_CL, IAA_RAM, 0,
-					tri0, CB_IMA(hints)));
+					tri0, hints));
 }
 
 static inline unsigned long gru_get_amo_value(void *cb)
@@ -613,12 +579,6 @@ static inline union gru_mesqhead  gru_me
 	return mqh;
 }
 
-/*
- * Get struct control_block_extended_exc_detail for CB.
- */
-extern int gru_get_cb_exception_detail(void *cb,
-		       struct control_block_extended_exc_detail *excdet);
-
 #define GRU_EXC_STR_SIZE		256
 
 
@@ -671,6 +631,7 @@ static inline int gru_check_status(void
 	struct gru_control_block_status *cbs = (void *)cb;
 	int ret;
 
+	barrier();
 	ret = cbs->istatus;
 	if (ret != CBS_ACTIVE)
 		ret = gru_check_status_proc(cb);
@@ -700,15 +661,6 @@ static inline void gru_wait_abort(void *
 }
 
 /*
- * Get a pointer to the start of a gseg
- * 	p	- Any valid pointer within the gseg
- */
-static inline void *gru_get_gseg_pointer (void *p)
-{
-	return (void *)((unsigned long)p & ~(GRU_GSEG_PAGESIZE - 1));
-}
-
-/*
  * Get a pointer to a control block
  * 	gseg	- GSeg address returned from gru_get_thread_gru_segment()
  * 	index	- index of desired CB
Index: linux/drivers/misc/sgi-gru/gruhandles.h
===================================================================
--- linux.orig/drivers/misc/sgi-gru/gruhandles.h	2010-06-09 08:11:29.799955128 -0500
+++ linux/drivers/misc/sgi-gru/gruhandles.h	2010-06-09 08:11:50.268036119 -0500
@@ -22,6 +22,7 @@
 
 #ifndef __GRUHANDLES_H__
 #define __GRUHANDLES_H__
+#include "grulib.h"
 #include "gru_instructions.h"
 
 /*
Index: linux/drivers/misc/sgi-gru/grulib.h
===================================================================
--- linux.orig/drivers/misc/sgi-gru/grulib.h	2010-06-09 08:11:47.993040877 -0500
+++ linux/drivers/misc/sgi-gru/grulib.h	2010-06-09 08:11:50.280038484 -0500
@@ -62,6 +62,25 @@
 #define THREAD_POINTER(p, th)		(p + GRU_GSEG_PAGESIZE * (th))
 #define GSEG_START(cb)			((void *)((unsigned long)(cb) & ~(GRU_GSEG_PAGESIZE - 1)))
 
+/*
+ * Size used to map GRU GSeg
+ */
+#define GRU_GSEG_PAGESIZE    (256 * 1024UL)
+
+/*
+ * Exceptions are retried for the following cases. If any OTHER bits are set
+ * in ecause, the exception is not retryable.
+ */
+#define EXCEPTION_RETRY_BITS (CBE_CAUSE_EXECUTION_HW_ERROR |            \
+			      CBE_CAUSE_TLBHW_ERROR |                   \
+			      CBE_CAUSE_RA_REQUEST_TIMEOUT |            \
+			      CBE_CAUSE_RA_RESPONSE_NON_FATAL |         \
+			      CBE_CAUSE_HA_RESPONSE_NON_FATAL |         \
+			      CBE_CAUSE_RA_RESPONSE_DATA_ERROR |        \
+			      CBE_CAUSE_HA_RESPONSE_DATA_ERROR          \
+			     )
+
+
 struct gru_get_gseg_statistics_req {
 	unsigned long			gseg;
 	struct gru_gseg_statistics	stats;
@@ -134,6 +153,21 @@ struct gru_dump_chiplet_state_req {
 	unsigned long	gsh_cch_state[16];
 };
 
+/*
+ * Structure used to fetch exception detail for CBs that terminate with
+ * CBS_EXCEPTION
+ */
+struct control_block_extended_exc_detail {
+	unsigned long	cb;
+	int		opc;
+	int		ecause;
+	int		exopc;
+	long		exceptdet0;
+	int		exceptdet1;
+	int		cbrstate;
+	int		cbrexecstatus;
+};
+
 #define GRU_DUMP_MAGIC	0x3474ab6c
 struct gru_dump_context_header {
 	unsigned int	magic;

[Patch 23/25] GRU - update driverr version

[steiner]

[-- Attachment #1: uv_gru_version --]
[-- Type: text/plain, Size: 790 bytes --]

From: Jack Steiner <steiner@sgi.com>

Update the GRU driver version to 1.0.

Signed-off-by: Jack Steiner <steiner@sgi.com>

---
 drivers/misc/sgi-gru/grutables.h |    2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

Index: linux/drivers/misc/sgi-gru/grutables.h
===================================================================
--- linux.orig/drivers/misc/sgi-gru/grutables.h	2010-06-09 08:11:46.307537752 -0500
+++ linux/drivers/misc/sgi-gru/grutables.h	2010-06-09 08:11:50.819414779 -0500
@@ -165,7 +165,7 @@ extern unsigned int gru_max_gids;
 #define GRU_MAX_GRUS		(GRU_MAX_BLADES * GRU_CHIPLETS_PER_BLADE)
 
 #define GRU_DRIVER_ID_STR	"SGI GRU Device Driver"
-#define GRU_DRIVER_VERSION_STR	"0.85"
+#define GRU_DRIVER_VERSION_STR	"1.00"
 
 #define gru_random()    get_cycles()
 

[Patch 24/25] GRU - rename gru pagesize defines

[steiner]

[-- Attachment #1: uv_gru_min_pagesize --]
[-- Type: text/plain, Size: 1714 bytes --]

From: Jack Steiner <steiner@sgi.com>

Change the name of GRU_GSEG_PAGESIZE to GRU_MIN_GSEG_PAGESIZE
to more accurately reflect it's use. This is in preparation for
allocating contexts using hugepages.

Signed-off-by: Jack Steiner <steiner@sgi.com>

---
 drivers/misc/sgi-gru/gru.h              |   10 ++--------
 drivers/misc/sgi-gru/gru_instructions.h |    2 +-
 2 files changed, 3 insertions(+), 9 deletions(-)

Index: linux/drivers/misc/sgi-gru/gru.h
===================================================================
--- linux.orig/drivers/misc/sgi-gru/gru.h	2010-06-09 08:11:48.856038221 -0500
+++ linux/drivers/misc/sgi-gru/gru.h	2010-06-09 08:11:51.328084904 -0500
@@ -28,15 +28,9 @@
 #define GRU_DS_BASE			0x20000
 
 /*
- * Size used to map GRU GSeg
+ * Minimum page size used to map a GRU context
  */
-#if defined(CONFIG_IA64)
-#define GRU_GSEG_PAGESIZE	(256 * 1024UL)
-#elif defined(CONFIG_X86_64)
-#define GRU_GSEG_PAGESIZE	(256 * 1024UL)		/* ZZZ 2MB ??? */
-#else
-#error "Unsupported architecture"
-#endif
+#define GRU_MIN_GSEG_PAGESIZE	(256 * 1024UL)
 
 /*
  * Statictics kept for each context.
Index: linux/drivers/misc/sgi-gru/gru_instructions.h
===================================================================
--- linux.orig/drivers/misc/sgi-gru/gru_instructions.h	2010-06-09 08:11:50.264036100 -0500
+++ linux/drivers/misc/sgi-gru/gru_instructions.h	2010-06-09 08:11:51.332083779 -0500
@@ -687,6 +687,6 @@ static inline void *gru_get_data_pointer
  */
 static inline int gru_get_tri(void *vaddr)
 {
-	return ((unsigned long)vaddr & (GRU_GSEG_PAGESIZE - 1)) - GRU_DS_BASE;
+	return ((unsigned long)vaddr & (GRU_MIN_GSEG_PAGESIZE - 1)) - GRU_DS_BASE;
 }
 #endif		/* __GRU_INSTRUCTIONS_H__ */

[Patch 25/25] GRU - update cbrstate definitions

[steiner]

[-- Attachment #1: uv_gru_update_cbrstate --]
[-- Type: text/plain, Size: 1190 bytes --]

From: Jack Steiner <steiner@sgi.com>

Update the CBRSTATE definitions to match what was actually
implemented in hardware.

Signed-off-by: Jack Steiner <steiner@sgi.com>

---
 drivers/misc/sgi-gru/gruhandles.h |   12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

Index: linux/drivers/misc/sgi-gru/gruhandles.h
===================================================================
--- linux.orig/drivers/misc/sgi-gru/gruhandles.h	2010-07-19 10:25:57.000000000 -0500
+++ linux/drivers/misc/sgi-gru/gruhandles.h	2010-07-19 13:03:39.982295167 -0500
@@ -478,15 +478,19 @@ enum gru_cbr_state {
 	CBRSTATE_INACTIVE,
 	CBRSTATE_IDLE,
 	CBRSTATE_PE_CHECK,
+	CBRSTATE_DSR_PRE,
 	CBRSTATE_QUEUED,
+	CBRSTATE_ISSUED,
 	CBRSTATE_WAIT_RESPONSE,
-	CBRSTATE_INTERRUPTED,
-	CBRSTATE_INTERRUPTED_MISS_FMM,
+	CBRSTATE_DSR_POST,
 	CBRSTATE_BUSY_INTERRUPT_MISS_FMM,
+	CBRSTATE_INTERRUPTED_MISS_FMM,
+	CBRSTATE_BUSY_INTERRUPT_MISS_UPM,
 	CBRSTATE_INTERRUPTED_MISS_UPM,
-	CBRSTATE_BUSY_INTERRUPTED_MISS_UPM,
-	CBRSTATE_REQUEST_ISSUE,
 	CBRSTATE_BUSY_INTERRUPT,
+	CBRSTATE_INTERRUPTED,
+	CBRSTATE_BCPY_CHECK,
+	CBRSTATE_WAIT_DONE,
 };
 
 /* CBE cbrexecstatus bits  - defined in gru_instructions.h*/

[Patch 08/25] GRU - flush gru tlb when driver is loaded

[steiner]

[-- Attachment #1: uv_gru_init_flush_tlb --]
[-- Type: text/plain, Size: 886 bytes --]

From: Jack Steiner <steiner@sgi.com>

Flush the GRU TLB when the GRU driver is loaded. There may be stale
entries in the TLB if a previous GRU was unloaded since the system
was last reset.

Signed-off-by: Jack Steiner <steiner@sgi.com>

---
 drivers/misc/sgi-gru/grutlbpurge.c |    2 ++
 1 file changed, 2 insertions(+)

Index: linux/drivers/misc/sgi-gru/grutlbpurge.c
===================================================================
--- linux.orig/drivers/misc/sgi-gru/grutlbpurge.c	2010-06-09 08:11:42.087537647 -0500
+++ linux/drivers/misc/sgi-gru/grutlbpurge.c	2010-06-09 08:11:42.891554573 -0500
@@ -380,4 +380,6 @@ void gru_tgh_flush_init(struct gru_state
 	/* first starting TGH index to use for remote purges */
 	gru->gs_tgh_first_remote = (cpus + (1 << shift) - 1) >> shift;
 
+	/* flush the GRU TLB in case there are stale entries present */
+	gru_flush_all_tlb(gru);
 }