Re: Xen's use of PAT and PV guests

[Ian Campbell <Ian.Campbell@citrix.com>]

[-- Attachment #1: Type: text/plain, Size: 585 bytes --]

On Tue, 2010-03-30 at 22:47 +0100, Jeremy Fitzhardinge wrote:
> 
> > I'm not sure that this scheme is at all upstreamable though.
> >
> 
> I don't see why not; it would all be hidden away in the Xen code, and 
> maintains the normal x86 illusion.   It's just a matter of hooking 
> wrmsr, make_pte and pte_val, which we do already. 

I was referring to my patch which wasn't at all hidden away in the Xen
code. Anyway I've found and attached it for your amusement, it's not
nearly as fully baked as I remembered ;-) In my defence the date stamp
on the patch file is May 2009...

Ian.


[-- Attachment #2: pat --]
[-- Type: text/x-patch, Size: 9467 bytes --]

diff --git a/arch/x86/include/asm/pgtable_types.h b/arch/x86/include/asm/pgtable_types.h
index 54cb697..ef947d8 100644
--- a/arch/x86/include/asm/pgtable_types.h
+++ b/arch/x86/include/asm/pgtable_types.h
@@ -68,14 +68,22 @@
 			 _PAGE_DIRTY)
 
 /* Set of bits not changed in pte_modify */
-#define _PAGE_CHG_MASK	(PTE_PFN_MASK | _PAGE_PCD | _PAGE_PWT |		\
+#define _PAGE_CHG_MASK	(PTE_PFN_MASK | _PAGE_PAT | _PAGE_PCD | _PAGE_PWT | \
 			 _PAGE_SPECIAL | _PAGE_ACCESSED | _PAGE_DIRTY)
 
-#define _PAGE_CACHE_MASK	(_PAGE_PCD | _PAGE_PWT)
-#define _PAGE_CACHE_WB		(0)
-#define _PAGE_CACHE_WC		(_PAGE_PWT)
-#define _PAGE_CACHE_UC_MINUS	(_PAGE_PCD)
-#define _PAGE_CACHE_UC		(_PAGE_PCD | _PAGE_PWT)
+#define _PAGE_CACHE_MASK	(_PAGE_PAT | _PAGE_PCD | _PAGE_PWT)
+
+#ifndef __ASSEMBLY__
+extern unsigned __page_cache_wb;
+extern unsigned __page_cache_wc;
+extern unsigned __page_cache_uc_minus;
+extern unsigned __page_cache_uc;
+#endif
+
+#define _PAGE_CACHE_WB		__page_cache_wb
+#define _PAGE_CACHE_WC		__page_cache_wc
+#define _PAGE_CACHE_UC_MINUS	__page_cache_uc_minus
+#define _PAGE_CACHE_UC		__page_cache_uc
 
 #define PAGE_NONE	__pgprot(_PAGE_PROTNONE | _PAGE_ACCESSED)
 #define PAGE_SHARED	__pgprot(_PAGE_PRESENT | _PAGE_RW | _PAGE_USER | \
diff --git a/arch/x86/kernel/cpu/mtrr/generic.c b/arch/x86/kernel/cpu/mtrr/generic.c
index c66dda1..0137c05 100644
--- a/arch/x86/kernel/cpu/mtrr/generic.c
+++ b/arch/x86/kernel/cpu/mtrr/generic.c
@@ -304,6 +304,8 @@ void __init get_mtrr_state(void)
 	unsigned lo, dummy;
 	unsigned long flags;
 
+	printk(KERN_CRIT "pat: made it to get_mtrr_state\n");
+
 	vrs = mtrr_state.var_ranges;
 
 	rdmsr(MTRRcap_MSR, lo, dummy);
@@ -336,6 +338,7 @@ void __init get_mtrr_state(void)
 	local_irq_save(flags);
 	prepare_set();
 
+	printk(KERN_CRIT "pat: get_mtrr_state calling pat_init()\n");
 	pat_init();
 
 	post_set();
@@ -614,6 +617,7 @@ static void generic_set_all(void)
 	mask = set_mtrr_state();
 
 	/* also set PAT */
+	printk(KERN_CRIT "pat: generic_set_all() calling pat_init()\n");
 	pat_init();
 
 	post_set();
diff --git a/arch/x86/kernel/cpu/mtrr/main.c b/arch/x86/kernel/cpu/mtrr/main.c
index fd5ac04..6eee754 100644
--- a/arch/x86/kernel/cpu/mtrr/main.c
+++ b/arch/x86/kernel/cpu/mtrr/main.c
@@ -623,6 +623,8 @@ void __init mtrr_bp_init(void)
 {
 	u32 phys_addr;
 
+	printk(KERN_CRIT "pat: made it to mtrr_bp_init\n");
+
 	init_ifs();
 
 	phys_addr = 32;
@@ -690,16 +692,22 @@ void __init mtrr_bp_init(void)
 	}
 
 	if (mtrr_if) {
+		printk(KERN_CRIT "pat: in mtrr_bp_init and mtrr_if is true\n");
+
 		num_var_ranges = mtrr_if->num_var_ranges();
 		init_table();
 		if (use_intel()) {
+			printk(KERN_CRIT "pat: mtrr_bp_init calling get_mtrr_state\n");
 			get_mtrr_state();
 
 			if (mtrr_cleanup(phys_addr)) {
 				changed_by_mtrr_cleanup = 1;
+				printk(KERN_CRIT "pat: mtrr_bp_init calling %pF via set_all hook\n", &mtrr_if->set_all);
 				mtrr_if->set_all();
 			}
 
+		} else {
+			printk(KERN_CRIT "pat: in mtrr_bp_init but not use_intel()\n");
 		}
 	}
 }
@@ -720,6 +728,7 @@ void mtrr_ap_init(void)
 	 */
 	local_irq_save(flags);
 
+	printk(KERN_CRIT "pat: mtrr_ap_init calling %pF via set_all hook\n", &mtrr_if->set_all);
 	mtrr_if->set_all();
 
 	local_irq_restore(flags);
diff --git a/arch/x86/kernel/cpu/mtrr/xen.c b/arch/x86/kernel/cpu/mtrr/xen.c
index 50a45db..63f86aa 100644
--- a/arch/x86/kernel/cpu/mtrr/xen.c
+++ b/arch/x86/kernel/cpu/mtrr/xen.c
@@ -15,6 +15,8 @@ static void xen_set_mtrr(unsigned int reg, unsigned long base,
 	struct xen_platform_op op;
 	int error;
 
+	printk(KERN_CRIT "pat: xen_set_mtrr\n");
+
 	/* mtrr_ops->set() is called once per CPU,
 	 * but Xen's ops apply to all CPUs.
 	 */
@@ -87,7 +89,7 @@ static struct mtrr_ops xen_mtrr_ops = {
 	.get_free_region   = xen_get_free_region,
 	.validate_add_page = generic_validate_add_page,
 	.have_wrcomb       = positive_have_wrcomb,
-	.use_intel_if	   = 0,
+	.use_intel_if	   = 1/*0*/,
 	.num_var_ranges	   = xen_num_var_ranges,
 };
 
diff --git a/arch/x86/kernel/setup.c b/arch/x86/kernel/setup.c
index 1b1c851..f704078 100644
--- a/arch/x86/kernel/setup.c
+++ b/arch/x86/kernel/setup.c
@@ -834,6 +834,7 @@ void __init setup_arch(char **cmdline_p)
 	/* preallocate 4k for mptable mpc */
 	early_reserve_e820_mpc_new();
 	/* update e820 for memory not covered by WB MTRRs */
+	printk(KERN_CRIT "pat: calling mtrr_bp_init from setup_arch()\n");
 	mtrr_bp_init();
 	if (mtrr_trim_uncached_memory(max_pfn))
 		max_pfn = e820_end_of_ram_pfn();
diff --git a/arch/x86/mm/ioremap.c b/arch/x86/mm/ioremap.c
index 8a45093..3c13d64 100644
--- a/arch/x86/mm/ioremap.c
+++ b/arch/x86/mm/ioremap.c
@@ -141,18 +141,12 @@ int ioremap_change_attr(unsigned long vaddr, unsigned long size,
 	unsigned long nrpages = size >> PAGE_SHIFT;
 	int err;
 
-	switch (prot_val) {
-	case _PAGE_CACHE_UC:
-	default:
-		err = _set_memory_uc(vaddr, nrpages);
-		break;
-	case _PAGE_CACHE_WC:
+	if (prot_val == _PAGE_CACHE_WC)
 		err = _set_memory_wc(vaddr, nrpages);
-		break;
-	case _PAGE_CACHE_WB:
+	else if (prot_val == _PAGE_CACHE_WB)
 		err = _set_memory_wb(vaddr, nrpages);
-		break;
-	}
+	else /* _PAGE_CACHE_UC or "other" */
+		err = _set_memory_uc(vaddr, nrpages);
 
 	return err;
 }
@@ -256,21 +250,14 @@ static void __iomem *__ioremap_caller(resource_size_t phys_addr,
 		prot_val = new_prot_val;
 	}
 
-	switch (prot_val) {
-	case _PAGE_CACHE_UC:
-	default:
-		prot = PAGE_KERNEL_IO_NOCACHE;
-		break;
-	case _PAGE_CACHE_UC_MINUS:
-		prot = PAGE_KERNEL_IO_UC_MINUS;
-		break;
-	case _PAGE_CACHE_WC:
+	if (prot_val == _PAGE_CACHE_WC)
 		prot = PAGE_KERNEL_IO_WC;
-		break;
-	case _PAGE_CACHE_WB:
+	else if (prot_val == _PAGE_CACHE_WB)
 		prot = PAGE_KERNEL_IO;
-		break;
-	}
+	else if (prot_val == _PAGE_CACHE_UC_MINUS)
+		prot = PAGE_KERNEL_IO_UC_MINUS;
+	else /* _PAGE_CACHE_UC or "other" */
+		prot = PAGE_KERNEL_IO_NOCACHE;
 
 	/*
 	 * Ok, go for it..
diff --git a/arch/x86/mm/pat.c b/arch/x86/mm/pat.c
index 3f7886f..f85a273 100644
--- a/arch/x86/mm/pat.c
+++ b/arch/x86/mm/pat.c
@@ -61,6 +61,11 @@ __setup("debugpat", pat_debug_setup);
 
 static u64 __read_mostly boot_pat_state;
 
+unsigned __page_cache_wb;
+unsigned __page_cache_wc;
+unsigned __page_cache_uc_minus;
+unsigned __page_cache_uc;
+
 enum {
 	PAT_UC = 0,		/* uncached */
 	PAT_WC = 1,		/* Write combining */
@@ -70,15 +75,43 @@ enum {
 	PAT_UC_MINUS = 7,	/* UC, but can be overriden by MTRR */
 };
 
+const char *pat_labels[8] = {
+	"broken",
+	"broken",
+	"broken",
+	"broken",
+	"broken",
+	"broken",
+	"broken",
+	"broken"
+};
+
 #define PAT(x, y)	((u64)PAT_ ## y << ((x)*8))
 
+static unsigned find_pat_index(const u64 pat, int mode)
+{
+	u64 mask;
+	int i;
+
+	for (mask = 0x7, i = 0; i < 8; mask <<= 8, i++) {
+		if (((pat & mask)>>(i*8)) == mode)
+			return i;
+	}
+	return -1; /* OR??? */
+}
+
 void pat_init(void)
 {
 	u64 pat;
 
-	if (!pat_enabled)
+	printk(KERN_CRIT "%s\n", __func__);
+	dump_stack();
+	if (!pat_enabled) {
+		printk(KERN_CRIT "%s pat not enabled\n", __func__);
 		return;
+	}
 
+	printk(KERN_CRIT "%s:%d\n", __func__, __LINE__);
 	if (!cpu_has_pat) {
 		if (!boot_pat_state) {
 			pat_disable("PAT not supported by CPU.");
@@ -95,6 +128,8 @@ void pat_init(void)
 		}
 	}
 
+	printk(KERN_CRIT "%s:%d\n", __func__, __LINE__);
+
 	/* Set PWT to Write-Combining. All other bits stay the same */
 	/*
 	 * PTE encoding used in Linux:
@@ -105,32 +140,41 @@ void pat_init(void)
 	 *      000 WB		_PAGE_CACHE_WB
 	 *      001 WC		_PAGE_CACHE_WC
 	 *      010 UC-		_PAGE_CACHE_UC_MINUS
-	 *      011 UC		_PAGE_CACHE_UC
-	 * PAT bit unused
+	 *      011 UC		_PAGE_CACHE_UC PAT bit unused
 	 */
 	pat = PAT(0, WB) | PAT(1, WC) | PAT(2, UC_MINUS) | PAT(3, UC) |
-	      PAT(4, WB) | PAT(5, WC) | PAT(6, UC_MINUS) | PAT(7, UC);
+		PAT(4, WB) | PAT(5, WC) | PAT(6, UC_MINUS) | PAT(7, UC);
 
 	/* Boot CPU check */
 	if (!boot_pat_state)
 		rdmsrl(MSR_IA32_CR_PAT, boot_pat_state);
 
 	wrmsrl(MSR_IA32_CR_PAT, pat);
+
+
+	/* Now see what we actually got */
+	rdmsrl(MSR_IA32_CR_PAT, pat);
+	__page_cache_wb = find_pat_index(pat, PAT_WB);
+	__page_cache_wc = find_pat_index(pat, PAT_WC);
+	__page_cache_uc_minus = find_pat_index(pat, PAT_UC_MINUS);
+	__page_cache_uc = find_pat_index(pat, PAT_UC);
+
+	pat_labels[__page_cache_wb] = "write-back";
+	pat_labels[__page_cache_wc] = "write-combining";
+	pat_labels[__page_cache_uc_minus] = "uncached-minus";
+	pat_labels[__page_cache_uc] = "uncached";
+
 	printk(KERN_INFO "x86 PAT enabled: cpu %d, old 0x%Lx, new 0x%Lx\n",
 	       smp_processor_id(), boot_pat_state, pat);
+	printk(KERN_INFO "Indexes: WB:%d; WC:%d; UC-:%d; UC:%d\n",
+	       __page_cache_wb, __page_cache_wc, __page_cache_uc_minus, __page_cache_uc);
 }
 
 #undef PAT
 
 static char *cattr_name(unsigned long flags)
 {
-	switch (flags & _PAGE_CACHE_MASK) {
-	case _PAGE_CACHE_UC:		return "uncached";
-	case _PAGE_CACHE_UC_MINUS:	return "uncached-minus";
-	case _PAGE_CACHE_WB:		return "write-back";
-	case _PAGE_CACHE_WC:		return "write-combining";
-	default:			return "broken";
-	}
+	return pat_labels[flags & _PAGE_CACHE_MASK];
 }
 
 /*
diff --git a/arch/x86/xen/enlighten.c b/arch/x86/xen/enlighten.c
index e099e44..232ed3e 100644
--- a/arch/x86/xen/enlighten.c
+++ b/arch/x86/xen/enlighten.c
@@ -1121,7 +1121,7 @@ asmlinkage void __init xen_start_kernel(void)
 		xen_start_info->console.domU.evtchn = 0;
 	}
 
-	pat_disable("PAT disabled on Xen");
+	//pat_disable("PAT disabled on Xen");
 
 	xen_raw_console_write("about to get started...\n");
 

[-- Attachment #3: Type: text/plain, Size: 138 bytes --]

_______________________________________________
Xen-devel mailing list
Xen-devel@lists.xensource.com
http://lists.xensource.com/xen-devel