[PATCH v2 1/2] xen/setup: Remove Identity Map Debug Message

[PATCH v2 1/2] xen/setup: Remove Identity Map Debug Message

[Matt Rushton]

Removing a debug message for setting the identity map since it becomes
rather noisy after rework of the identity map code.

Signed-off-by: Matthew Rushton <mrushton@amazon.com>
---
 arch/x86/xen/p2m.c |    5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/arch/x86/xen/p2m.c b/arch/x86/xen/p2m.c
index 9bb3d82..3172692 100644
--- a/arch/x86/xen/p2m.c
+++ b/arch/x86/xen/p2m.c
@@ -841,10 +841,9 @@ unsigned long __init set_phys_range_identity(unsigned long pfn_s,
 			pfn = ALIGN(pfn, P2M_PER_PAGE);
 	}
 
-	if (!WARN((pfn - pfn_s) != (pfn_e - pfn_s),
+	WARN((pfn - pfn_s) != (pfn_e - pfn_s),
 		"Identity mapping failed. We are %ld short of 1-1 mappings!\n",
-		(pfn_e - pfn_s) - (pfn - pfn_s)))
-		printk(KERN_DEBUG "1-1 mapping on %lx->%lx\n", pfn_s, pfn);
+		(pfn_e - pfn_s) - (pfn - pfn_s));
 
 	return pfn - pfn_s;
 }
-- 
1.7.9.5

[PATCH v2 2/2] xen/setup: Remap Xen Identity Mapped RAM

[Matt Rushton]

Instead of ballooning up and down dom0 memory this remaps the existing mfns
that were replaced by the identity map. The reason for this is that the
existing implementation ballooned memory up and and down which caused dom0
to have discontiguous pages. In some cases this resulted in the use of bounce
buffers which reduced network I/O performance significantly. This change will
honor the existing order of the pages with the exception of some boundary
conditions.

To do this we need to update both the Linux p2m table and the Xen m2p table.
Particular care must be taken when updating the p2m table since it's important
to limit table memory consumption and reuse the existing leaf pages which get
freed when an entire leaf page is set to the identity map. To implement this,
mapping updates are grouped into blocks with table entries getting cached
temporarily and then released.

On my test system before:
Total pages: 2105014
Total contiguous: 1640635

After:
Total pages: 2105014
Total contiguous: 2098904

Signed-off-by: Matthew Rushton <mrushton@amazon.com>
---
 arch/x86/xen/p2m.c   |   23 +---
 arch/x86/xen/p2m.h   |   15 ++
 arch/x86/xen/setup.c |  370 +++++++++++++++++++++++++++++++++++++++-----------
 3 files changed, 314 insertions(+), 94 deletions(-)
 create mode 100644 arch/x86/xen/p2m.h

diff --git a/arch/x86/xen/p2m.c b/arch/x86/xen/p2m.c
index 3172692..9f5983b 100644
--- a/arch/x86/xen/p2m.c
+++ b/arch/x86/xen/p2m.c
@@ -173,6 +173,7 @@
 #include <xen/balloon.h>
 #include <xen/grant_table.h>
 
+#include "p2m.h"
 #include "multicalls.h"
 #include "xen-ops.h"
 
@@ -180,12 +181,6 @@ static void __init m2p_override_init(void);
 
 unsigned long xen_max_p2m_pfn __read_mostly;
 
-#define P2M_PER_PAGE		(PAGE_SIZE / sizeof(unsigned long))
-#define P2M_MID_PER_PAGE	(PAGE_SIZE / sizeof(unsigned long *))
-#define P2M_TOP_PER_PAGE	(PAGE_SIZE / sizeof(unsigned long **))
-
-#define MAX_P2M_PFN		(P2M_TOP_PER_PAGE * P2M_MID_PER_PAGE * P2M_PER_PAGE)
-
 /* Placeholders for holes in the address space */
 static RESERVE_BRK_ARRAY(unsigned long, p2m_missing, P2M_PER_PAGE);
 static RESERVE_BRK_ARRAY(unsigned long *, p2m_mid_missing, P2M_MID_PER_PAGE);
@@ -202,16 +197,12 @@ static RESERVE_BRK_ARRAY(unsigned long, p2m_mid_identity_mfn, P2M_MID_PER_PAGE);
 RESERVE_BRK(p2m_mid, PAGE_SIZE * (MAX_DOMAIN_PAGES / (P2M_PER_PAGE * P2M_MID_PER_PAGE)));
 RESERVE_BRK(p2m_mid_mfn, PAGE_SIZE * (MAX_DOMAIN_PAGES / (P2M_PER_PAGE * P2M_MID_PER_PAGE)));
 
-/* We might hit two boundary violations at the start and end, at max each
- * boundary violation will require three middle nodes. */
-RESERVE_BRK(p2m_mid_extra, PAGE_SIZE * 2 * 3);
-
-/* When we populate back during bootup, the amount of pages can vary. The
- * max we have is seen is 395979, but that does not mean it can't be more.
- * Some machines can have 3GB I/O holes even. With early_can_reuse_p2m_middle
- * it can re-use Xen provided mfn_list array, so we only need to allocate at
- * most three P2M top nodes. */
-RESERVE_BRK(p2m_populated, PAGE_SIZE * 3);
+/* For each I/O range remapped we may lose up to two leaf pages for the boundary
+ * violations and three mid pages to cover up to 3GB. With
+ * early_can_reuse_p2m_middle() most of the leaf pages will be reused by the
+ * remapped region.
+ */
+RESERVE_BRK(p2m_identity_remap, PAGE_SIZE * 2 * 3 * MAX_REMAP_RANGES);
 
 static inline unsigned p2m_top_index(unsigned long pfn)
 {
diff --git a/arch/x86/xen/p2m.h b/arch/x86/xen/p2m.h
new file mode 100644
index 0000000..ad8aee2
--- /dev/null
+++ b/arch/x86/xen/p2m.h
@@ -0,0 +1,15 @@
+#ifndef _XEN_P2M_H
+#define _XEN_P2M_H
+
+#define P2M_PER_PAGE        (PAGE_SIZE / sizeof(unsigned long))
+#define P2M_MID_PER_PAGE    (PAGE_SIZE / sizeof(unsigned long *))
+#define P2M_TOP_PER_PAGE    (PAGE_SIZE / sizeof(unsigned long **))
+
+#define MAX_P2M_PFN         (P2M_TOP_PER_PAGE * P2M_MID_PER_PAGE * P2M_PER_PAGE)
+
+#define MAX_REMAP_RANGES    10
+
+extern unsigned long __init set_phys_range_identity(unsigned long pfn_s,
+                                      unsigned long pfn_e);
+
+#endif  /* _XEN_P2M_H */
diff --git a/arch/x86/xen/setup.c b/arch/x86/xen/setup.c
index 2e555163..a58f16a 100644
--- a/arch/x86/xen/setup.c
+++ b/arch/x86/xen/setup.c
@@ -29,6 +29,7 @@
 #include <xen/features.h>
 #include "xen-ops.h"
 #include "vdso.h"
+#include "p2m.h"
 
 /* These are code, but not functions.  Defined in entry.S */
 extern const char xen_hypervisor_callback[];
@@ -46,6 +47,9 @@ struct xen_memory_region xen_extra_mem[XEN_EXTRA_MEM_MAX_REGIONS] __initdata;
 /* Number of pages released from the initial allocation. */
 unsigned long xen_released_pages;
 
+/* Buffer used to remap identity mapped pages */
+unsigned long xen_remap_buf[P2M_PER_PAGE] __initdata;
+
 /* 
  * The maximum amount of extra memory compared to the base size.  The
  * main scaling factor is the size of struct page.  At extreme ratios
@@ -151,107 +155,325 @@ static unsigned long __init xen_do_chunk(unsigned long start,
 	return len;
 }
 
-static unsigned long __init xen_release_chunk(unsigned long start,
-					      unsigned long end)
-{
-	return xen_do_chunk(start, end, true);
-}
-
-static unsigned long __init xen_populate_chunk(
+/*
+ * Finds the next RAM pfn available in the E820 map after min_pfn.
+ * This function updates min_pfn with the pfn found and returns
+ * the size of that range or zero if not found.
+ */
+static unsigned long __init xen_find_pfn_range(
 	const struct e820entry *list, size_t map_size,
-	unsigned long max_pfn, unsigned long *last_pfn,
-	unsigned long credits_left)
+	unsigned long *min_pfn)
 {
 	const struct e820entry *entry;
 	unsigned int i;
 	unsigned long done = 0;
-	unsigned long dest_pfn;
 
 	for (i = 0, entry = list; i < map_size; i++, entry++) {
 		unsigned long s_pfn;
 		unsigned long e_pfn;
-		unsigned long pfns;
-		long capacity;
-
-		if (credits_left <= 0)
-			break;
 
 		if (entry->type != E820_RAM)
 			continue;
 
 		e_pfn = PFN_DOWN(entry->addr + entry->size);
 
-		/* We only care about E820 after the xen_start_info->nr_pages */
-		if (e_pfn <= max_pfn)
+		/* We only care about E820 after this */
+		if (e_pfn < *min_pfn)
 			continue;
 
 		s_pfn = PFN_UP(entry->addr);
-		/* If the E820 falls within the nr_pages, we want to start
-		 * at the nr_pages PFN.
-		 * If that would mean going past the E820 entry, skip it
+
+		/* If min_pfn falls within the E820 entry, we want to start
+		 * at the min_pfn PFN.
 		 */
-		if (s_pfn <= max_pfn) {
-			capacity = e_pfn - max_pfn;
-			dest_pfn = max_pfn;
+		if (s_pfn <= *min_pfn) {
+			done = e_pfn - *min_pfn;
 		} else {
-			capacity = e_pfn - s_pfn;
-			dest_pfn = s_pfn;
+			done = e_pfn - s_pfn;
+			*min_pfn = s_pfn;
 		}
+		break;
+	}
 
-		if (credits_left < capacity)
-			capacity = credits_left;
+	return done;
+}
 
-		pfns = xen_do_chunk(dest_pfn, dest_pfn + capacity, false);
-		done += pfns;
-		*last_pfn = (dest_pfn + pfns);
-		if (pfns < capacity)
-			break;
-		credits_left -= pfns;
+/*
+ * This releases a chunk of memory and then does the identity map. It's used as
+ * as a fallback if the remapping fails.
+ */
+static void __init xen_set_identity_and_release_chunk(unsigned long start_pfn,
+	unsigned long end_pfn, unsigned long nr_pages, unsigned long *identity,
+	unsigned long *released)
+{
+	WARN_ON(start_pfn > end_pfn);
+
+	/* Need to release pages first */
+	*released += xen_do_chunk(start_pfn, min(end_pfn, nr_pages), true);
+	*identity += set_phys_range_identity(start_pfn, end_pfn);
+}
+
+/*
+ * Helper function to update both the p2m and m2p tables.
+ */
+static unsigned long __init xen_update_mem_tables(unsigned long pfn,
+						  unsigned long mfn)
+{
+	struct mmu_update update = {
+		.ptr = (mfn << PAGE_SHIFT) | MMU_MACHPHYS_UPDATE,
+		.val = pfn
+	};
+
+	/* Update p2m */
+	if (!early_set_phys_to_machine(pfn, mfn)) {
+		WARN(1, "Failed to set p2m mapping for pfn=%ld mfn=%ld\n",
+		     pfn, mfn);
+		return false;
 	}
-	return done;
+
+	/* Update m2p */
+	if (HYPERVISOR_mmu_update(&update, 1, NULL, DOMID_SELF) < 0) {
+		WARN(1, "Failed to set m2p mapping for mfn=%ld pfn=%ld\n",
+		     mfn, pfn);
+		return false;
+	}
+
+	return true;
 }
 
-static void __init xen_set_identity_and_release_chunk(
-	unsigned long start_pfn, unsigned long end_pfn, unsigned long nr_pages,
-	unsigned long *released, unsigned long *identity)
+/*
+ * This function updates the p2m and m2p tables with an identity map from
+ * start_pfn to start_pfn+size and remaps the underlying RAM of the original
+ * allocation at remap_pfn. It must do so carefully in P2M_PER_PAGE sized blocks
+ * to not exhaust the reserved brk space. Doing it in properly aligned blocks
+ * ensures we only allocate the minimum required leaf pages in the p2m table. It
+ * copies the existing mfns from the p2m table under the 1:1 map, overwrites
+ * them with the identity map and then updates the p2m and m2p tables with the
+ * remapped memory.
+ */
+static unsigned long __init xen_do_set_identity_and_remap_chunk(
+        unsigned long start_pfn, unsigned long size, unsigned long remap_pfn)
 {
-	unsigned long pfn;
+	unsigned long ident_pfn_iter, remap_pfn_iter;
+	unsigned long ident_start_pfn_align, remap_start_pfn_align;
+	unsigned long ident_end_pfn_align, remap_end_pfn_align;
+	unsigned long ident_boundary_pfn, remap_boundary_pfn;
+	unsigned long ident_cnt = 0;
+	unsigned long remap_cnt = 0;
+	unsigned long left = size;
+	unsigned long mod;
+	int i;
+
+	WARN_ON(size == 0);
+
+	BUG_ON(xen_feature(XENFEAT_auto_translated_physmap));
 
 	/*
-	 * If the PFNs are currently mapped, clear the mappings
-	 * (except for the ISA region which must be 1:1 mapped) to
-	 * release the refcounts (in Xen) on the original frames.
+	 * Determine the proper alignment to remap memory in P2M_PER_PAGE sized
+	 * blocks. We need to keep track of both the existing pfn mapping and
+	 * the new pfn remapping.
 	 */
-	for (pfn = start_pfn; pfn <= max_pfn_mapped && pfn < end_pfn; pfn++) {
-		pte_t pte = __pte_ma(0);
+	mod = start_pfn % P2M_PER_PAGE;
+	ident_start_pfn_align =
+		mod ? (start_pfn - mod + P2M_PER_PAGE) : start_pfn;
+	mod = remap_pfn % P2M_PER_PAGE;
+	remap_start_pfn_align =
+		mod ? (remap_pfn - mod + P2M_PER_PAGE) : remap_pfn;
+	mod = (start_pfn + size) % P2M_PER_PAGE;
+	ident_end_pfn_align = start_pfn + size - mod;
+	mod = (remap_pfn + size) % P2M_PER_PAGE;
+	remap_end_pfn_align = remap_pfn + size - mod;
+
+	/* Iterate over each p2m leaf node in each range */
+	for (ident_pfn_iter = ident_start_pfn_align, remap_pfn_iter = remap_start_pfn_align;
+	     ident_pfn_iter < ident_end_pfn_align && remap_pfn_iter < remap_end_pfn_align;
+	     ident_pfn_iter += P2M_PER_PAGE, remap_pfn_iter += P2M_PER_PAGE) {
+		/* Check we aren't past the end */
+		BUG_ON(ident_pfn_iter + P2M_PER_PAGE > start_pfn + size);
+		BUG_ON(remap_pfn_iter + P2M_PER_PAGE > remap_pfn + size);
+
+		/* Save p2m mappings */
+		for (i = 0; i < P2M_PER_PAGE; i++)
+			xen_remap_buf[i] = pfn_to_mfn(ident_pfn_iter + i);
+
+		/* Set identity map which will free a p2m leaf */
+		ident_cnt += set_phys_range_identity(ident_pfn_iter,
+			ident_pfn_iter + P2M_PER_PAGE);
+
+#ifdef DEBUG
+		/* Helps verify a p2m leaf has been freed */
+		for (i = 0; i < P2M_PER_PAGE; i++) {
+			unsigned int pfn = ident_pfn_iter + i;
+			BUG_ON(pfn_to_mfn(pfn) != pfn);
+		}
+#endif
+		/* Now remap memory */
+		for (i = 0; i < P2M_PER_PAGE; i++) {
+			unsigned long mfn = xen_remap_buf[i];
+
+			/* This will use the p2m leaf freed above */
+			if (!xen_update_mem_tables(remap_pfn_iter + i, mfn)) {
+				WARN(1, "Failed to update mem mapping for pfn=%ld mfn=%ld\n",
+					remap_pfn_iter + i, mfn);
+				return 0;
+			}
+
+			remap_cnt++;
+		}
 
-		if (pfn < PFN_UP(ISA_END_ADDRESS))
-			pte = mfn_pte(pfn, PAGE_KERNEL_IO);
+		left -= P2M_PER_PAGE;
+	}
 
-		(void)HYPERVISOR_update_va_mapping(
-			(unsigned long)__va(pfn << PAGE_SHIFT), pte, 0);
+	/* Max boundary space possible */
+	BUG_ON(left > (P2M_PER_PAGE - 1) * 2);
+
+	/* Now handle the boundary conditions */
+	ident_boundary_pfn = start_pfn;
+	remap_boundary_pfn = remap_pfn;
+	for (i = 0; i < left; i++) {
+		unsigned long mfn;
+
+		/* These two checks move from the start to end boundaries */
+		if (ident_boundary_pfn == ident_start_pfn_align)
+			ident_boundary_pfn = ident_pfn_iter;
+		if (remap_boundary_pfn == remap_start_pfn_align)
+			remap_boundary_pfn = remap_pfn_iter;
+
+		/* Check we aren't past the end */
+		BUG_ON(ident_boundary_pfn >= start_pfn + size);
+		BUG_ON(remap_boundary_pfn >= remap_pfn + size);
+
+		mfn = pfn_to_mfn(ident_boundary_pfn);
+
+		if (!xen_update_mem_tables(remap_boundary_pfn, mfn)) {
+			WARN(1, "Failed to update mem mapping for pfn=%ld mfn=%ld\n",
+				remap_pfn_iter + i, mfn);
+			return 0;
+		}
+		remap_cnt++;
+
+		ident_boundary_pfn++;
+		remap_boundary_pfn++;
 	}
 
-	if (start_pfn < nr_pages)
-		*released += xen_release_chunk(
-			start_pfn, min(end_pfn, nr_pages));
+	/* Finish up the identity map */
+	if (ident_start_pfn_align >= ident_end_pfn_align) {
+		/*
+                 * In this case we have an identity range which does not span an
+                 * aligned block so everything needs to be identity mapped here.
+                 * If we didn't check this we might remap too many pages since
+                 * the align boundaries are not meaningful in this case.
+	         */
+		ident_cnt += set_phys_range_identity(start_pfn,
+			start_pfn + size);
+	} else {
+		/* Remapped above so check each end of the chunk */
+		if (start_pfn < ident_start_pfn_align)
+			ident_cnt += set_phys_range_identity(start_pfn,
+				ident_start_pfn_align);
+		if (start_pfn + size > ident_pfn_iter)
+			ident_cnt += set_phys_range_identity(ident_pfn_iter,
+				start_pfn + size);
+	}
 
-	*identity += set_phys_range_identity(start_pfn, end_pfn);
+	BUG_ON(ident_cnt != size);
+	BUG_ON(remap_cnt != size);
+
+	return size;
 }
 
-static unsigned long __init xen_set_identity_and_release(
-	const struct e820entry *list, size_t map_size, unsigned long nr_pages)
+/*
+ * This function takes a contiguous pfn range that needs to be identity mapped
+ * and:
+ *
+ *  1) Finds a new range of pfns to use to remap based on E820 and remap_pfn.
+ *  2) Calls the do_ function to actually do the mapping/remapping work.
+ *
+ * The goal is to not allocate additional memory but to remap the existing
+ * pages. In the case of an error the underlying memory is simply released back
+ * to Xen and not remapped.
+ */
+static unsigned long __init xen_set_identity_and_remap_chunk(
+        const struct e820entry *list, size_t map_size, unsigned long start_pfn,
+	unsigned long end_pfn, unsigned long nr_pages, unsigned long remap_pfn,
+	unsigned long *identity, unsigned long *remapped,
+	unsigned long *released)
+{
+	unsigned long pfn;
+	unsigned long i = 0;
+	unsigned long n = end_pfn - start_pfn;
+
+	while (i < n) {
+		unsigned long cur_pfn = start_pfn + i;
+		unsigned long left = n - i;
+		unsigned long size = left;
+		unsigned long remap_range_size;
+
+		/* Do not remap pages beyond the current allocation */
+		if (cur_pfn >= nr_pages) {
+			/* Identity map remaining pages */
+			*identity += set_phys_range_identity(cur_pfn,
+				cur_pfn + size);
+			break;
+		}
+		if (cur_pfn + size > nr_pages)
+			size = nr_pages - cur_pfn;
+
+		remap_range_size = xen_find_pfn_range(list, map_size,
+						      &remap_pfn);
+		if (!remap_range_size) {
+			pr_warning("Unable to find available pfn range, not remapping identity pages\n");
+			xen_set_identity_and_release_chunk(cur_pfn,
+				cur_pfn + left, nr_pages, identity, released);
+			break;
+		}
+		/* Adjust size to fit in current e820 RAM region */
+		if (size > remap_range_size)
+			size = remap_range_size;
+
+		if (!xen_do_set_identity_and_remap_chunk(cur_pfn, size, remap_pfn)) {
+			WARN(1, "Failed to remap 1:1 memory cur_pfn=%ld size=%ld remap_pfn=%ld\n",
+				cur_pfn, size, remap_pfn);
+			xen_set_identity_and_release_chunk(cur_pfn,
+				cur_pfn + left, nr_pages, identity, released);
+			break;
+		}
+
+		/* Update variables to reflect new mappings. */
+		i += size;
+		remap_pfn += size;
+		*identity += size;
+		*remapped += size;
+	}
+
+	/*
+	 * If the PFNs are currently mapped, the VA mapping also needs
+	 * to be updated to be 1:1.
+	 */
+	for (pfn = start_pfn; pfn <= max_pfn_mapped && pfn < end_pfn; pfn++)
+		(void)HYPERVISOR_update_va_mapping(
+			(unsigned long)__va(pfn << PAGE_SHIFT),
+			mfn_pte(pfn, PAGE_KERNEL_IO), 0);
+
+	return remap_pfn;
+}
+
+static unsigned long __init xen_set_identity_and_remap(
+	const struct e820entry *list, size_t map_size, unsigned long nr_pages,
+	unsigned long *released)
 {
 	phys_addr_t start = 0;
-	unsigned long released = 0;
 	unsigned long identity = 0;
+	unsigned long remapped = 0;
+	unsigned long last_pfn = nr_pages;
 	const struct e820entry *entry;
+	unsigned long num_released = 0;
 	int i;
 
 	/*
 	 * Combine non-RAM regions and gaps until a RAM region (or the
 	 * end of the map) is reached, then set the 1:1 map and
-	 * release the pages (if available) in those non-RAM regions.
+	 * remap the memory in those non-RAM regions.
 	 *
 	 * The combined non-RAM regions are rounded to a whole number
 	 * of pages so any partial pages are accessible via the 1:1
@@ -269,22 +491,24 @@ static unsigned long __init xen_set_identity_and_release(
 				end_pfn = PFN_UP(entry->addr);
 
 			if (start_pfn < end_pfn)
-				xen_set_identity_and_release_chunk(
-					start_pfn, end_pfn, nr_pages,
-					&released, &identity);
-
+				last_pfn = xen_set_identity_and_remap_chunk(
+						list, map_size, start_pfn,
+						end_pfn, nr_pages, last_pfn,
+						&identity, &remapped,
+						&num_released);
 			start = end;
 		}
 	}
 
-	if (released)
-		printk(KERN_INFO "Released %lu pages of unused memory\n", released);
-	if (identity)
-		printk(KERN_INFO "Set %ld page(s) to 1-1 mapping\n", identity);
+	*released = num_released;
 
-	return released;
-}
+	pr_info("Set %ld page(s) to 1-1 mapping\n", identity);
+	pr_info("Remapped %ld page(s), last_pfn=%ld\n", remapped,
+		last_pfn);
+	pr_info("Released %ld page(s)\n", num_released);
 
+	return last_pfn;
+}
 static unsigned long __init xen_get_max_pages(void)
 {
 	unsigned long max_pages = MAX_DOMAIN_PAGES;
@@ -347,7 +571,6 @@ char * __init xen_memory_setup(void)
 	unsigned long max_pages;
 	unsigned long last_pfn = 0;
 	unsigned long extra_pages = 0;
-	unsigned long populated;
 	int i;
 	int op;
 
@@ -392,20 +615,11 @@ char * __init xen_memory_setup(void)
 		extra_pages += max_pages - max_pfn;
 
 	/*
-	 * Set P2M for all non-RAM pages and E820 gaps to be identity
-	 * type PFNs.  Any RAM pages that would be made inaccesible by
-	 * this are first released.
+	 * Set identity map on non-RAM pages and remap the underlying RAM.
 	 */
-	xen_released_pages = xen_set_identity_and_release(
-		map, memmap.nr_entries, max_pfn);
-
-	/*
-	 * Populate back the non-RAM pages and E820 gaps that had been
-	 * released. */
-	populated = xen_populate_chunk(map, memmap.nr_entries,
-			max_pfn, &last_pfn, xen_released_pages);
+	last_pfn = xen_set_identity_and_remap(map, memmap.nr_entries, max_pfn,
+					      &xen_released_pages);
 
-	xen_released_pages -= populated;
 	extra_pages += xen_released_pages;
 
 	if (last_pfn > max_pfn) {
-- 
1.7.9.5

Re: [PATCH v2 1/2] xen/setup: Remove Identity Map Debug Message

[David Vrabel]

On 20/07/14 01:01, Matt Rushton wrote:
> Removing a debug message for setting the identity map since it becomes
> rather noisy after rework of the identity map code.

Reviewed-by: David Vrabel <david.vrabel@citrix.com>

David

> --- a/arch/x86/xen/p2m.c
> +++ b/arch/x86/xen/p2m.c
> @@ -841,10 +841,9 @@ unsigned long __init set_phys_range_identity(unsigned long pfn_s,
>  			pfn = ALIGN(pfn, P2M_PER_PAGE);
>  	}
>  
> -	if (!WARN((pfn - pfn_s) != (pfn_e - pfn_s),
> +	WARN((pfn - pfn_s) != (pfn_e - pfn_s),
>  		"Identity mapping failed. We are %ld short of 1-1 mappings!\n",
> -		(pfn_e - pfn_s) - (pfn - pfn_s)))
> -		printk(KERN_DEBUG "1-1 mapping on %lx->%lx\n", pfn_s, pfn);
> +		(pfn_e - pfn_s) - (pfn - pfn_s));
>  
>  	return pfn - pfn_s;
>  }
> 

Re: [PATCH v2 1/2] xen/setup: Remove Identity Map Debug Message

[David Vrabel]

On 20/07/14 01:01, Matt Rushton wrote:
> Removing a debug message for setting the identity map since it becomes
> rather noisy after rework of the identity map code.

Reviewed-by: David Vrabel <david.vrabel@citrix.com>

David

> --- a/arch/x86/xen/p2m.c
> +++ b/arch/x86/xen/p2m.c
> @@ -841,10 +841,9 @@ unsigned long __init set_phys_range_identity(unsigned long pfn_s,
>  			pfn = ALIGN(pfn, P2M_PER_PAGE);
>  	}
>  
> -	if (!WARN((pfn - pfn_s) != (pfn_e - pfn_s),
> +	WARN((pfn - pfn_s) != (pfn_e - pfn_s),
>  		"Identity mapping failed. We are %ld short of 1-1 mappings!\n",
> -		(pfn_e - pfn_s) - (pfn - pfn_s)))
> -		printk(KERN_DEBUG "1-1 mapping on %lx->%lx\n", pfn_s, pfn);
> +		(pfn_e - pfn_s) - (pfn - pfn_s));
>  
>  	return pfn - pfn_s;
>  }
> 

Re: [PATCH v2 2/2] xen/setup: Remap Xen Identity Mapped RAM

[David Vrabel]

On 20/07/14 01:01, Matt Rushton wrote:
> Instead of ballooning up and down dom0 memory this remaps the existing mfns
> that were replaced by the identity map. The reason for this is that the
> existing implementation ballooned memory up and and down which caused dom0
> to have discontiguous pages. In some cases this resulted in the use of bounce
> buffers which reduced network I/O performance significantly. This change will
> honor the existing order of the pages with the exception of some boundary
> conditions.
> 
> To do this we need to update both the Linux p2m table and the Xen m2p table.
> Particular care must be taken when updating the p2m table since it's important
> to limit table memory consumption and reuse the existing leaf pages which get
> freed when an entire leaf page is set to the identity map. To implement this,
> mapping updates are grouped into blocks with table entries getting cached
> temporarily and then released.
> 
> On my test system before:
> Total pages: 2105014
> Total contiguous: 1640635
> 
> After:
> Total pages: 2105014
> Total contiguous: 2098904

This looks like a worthwhile improvement but...

>  arch/x86/xen/setup.c |  370 +++++++++++++++++++++++++++++++++++++++-----------

... Ew :(

It will take me a bit longer before I can give this a proper review.

David

Re: [PATCH v2 2/2] xen/setup: Remap Xen Identity Mapped RAM

[David Vrabel]

On 20/07/14 01:01, Matt Rushton wrote:
> Instead of ballooning up and down dom0 memory this remaps the existing mfns
> that were replaced by the identity map. The reason for this is that the
> existing implementation ballooned memory up and and down which caused dom0
> to have discontiguous pages. In some cases this resulted in the use of bounce
> buffers which reduced network I/O performance significantly. This change will
> honor the existing order of the pages with the exception of some boundary
> conditions.
> 
> To do this we need to update both the Linux p2m table and the Xen m2p table.
> Particular care must be taken when updating the p2m table since it's important
> to limit table memory consumption and reuse the existing leaf pages which get
> freed when an entire leaf page is set to the identity map. To implement this,
> mapping updates are grouped into blocks with table entries getting cached
> temporarily and then released.
> 
> On my test system before:
> Total pages: 2105014
> Total contiguous: 1640635
> 
> After:
> Total pages: 2105014
> Total contiguous: 2098904

This looks like a worthwhile improvement but...

>  arch/x86/xen/setup.c |  370 +++++++++++++++++++++++++++++++++++++++-----------

... Ew :(

It will take me a bit longer before I can give this a proper review.

David

Re: [PATCH v2 2/2] xen/setup: Remap Xen Identity Mapped RAM

[David Vrabel]

On 20/07/14 01:01, Matt Rushton wrote:
> Instead of ballooning up and down dom0 memory this remaps the existing mfns
> that were replaced by the identity map. The reason for this is that the
> existing implementation ballooned memory up and and down which caused dom0
> to have discontiguous pages. In some cases this resulted in the use of bounce
> buffers which reduced network I/O performance significantly. This change will
> honor the existing order of the pages with the exception of some boundary
> conditions.
> 
> To do this we need to update both the Linux p2m table and the Xen m2p table.
> Particular care must be taken when updating the p2m table since it's important
> to limit table memory consumption and reuse the existing leaf pages which get
> freed when an entire leaf page is set to the identity map. To implement this,
> mapping updates are grouped into blocks with table entries getting cached
> temporarily and then released.
> 
> On my test system before:
> Total pages: 2105014
> Total contiguous: 1640635
> 
> After:
> Total pages: 2105014
> Total contiguous: 2098904

Applied to devel/for-linus-3.17

Thanks.

David

Re: [PATCH v2 2/2] xen/setup: Remap Xen Identity Mapped RAM

[David Vrabel]

On 20/07/14 01:01, Matt Rushton wrote:
> Instead of ballooning up and down dom0 memory this remaps the existing mfns
> that were replaced by the identity map. The reason for this is that the
> existing implementation ballooned memory up and and down which caused dom0
> to have discontiguous pages. In some cases this resulted in the use of bounce
> buffers which reduced network I/O performance significantly. This change will
> honor the existing order of the pages with the exception of some boundary
> conditions.
> 
> To do this we need to update both the Linux p2m table and the Xen m2p table.
> Particular care must be taken when updating the p2m table since it's important
> to limit table memory consumption and reuse the existing leaf pages which get
> freed when an entire leaf page is set to the identity map. To implement this,
> mapping updates are grouped into blocks with table entries getting cached
> temporarily and then released.
> 
> On my test system before:
> Total pages: 2105014
> Total contiguous: 1640635
> 
> After:
> Total pages: 2105014
> Total contiguous: 2098904

Applied to devel/for-linus-3.17

Thanks.

David

Re: [PATCH v2 2/2] xen/setup: Remap Xen Identity Mapped RAM

[David Vrabel]

On 31/07/14 18:43, David Vrabel wrote:
> On 20/07/14 01:01, Matt Rushton wrote:
>> Instead of ballooning up and down dom0 memory this remaps the existing mfns
>> that were replaced by the identity map. The reason for this is that the
>> existing implementation ballooned memory up and and down which caused dom0
>> to have discontiguous pages. In some cases this resulted in the use of bounce
>> buffers which reduced network I/O performance significantly. This change will
>> honor the existing order of the pages with the exception of some boundary
>> conditions.
>>
>> To do this we need to update both the Linux p2m table and the Xen m2p table.
>> Particular care must be taken when updating the p2m table since it's important
>> to limit table memory consumption and reuse the existing leaf pages which get
>> freed when an entire leaf page is set to the identity map. To implement this,
>> mapping updates are grouped into blocks with table entries getting cached
>> temporarily and then released.
>>
>> On my test system before:
>> Total pages: 2105014
>> Total contiguous: 1640635
>>
>> After:
>> Total pages: 2105014
>> Total contiguous: 2098904
> 
> Applied to devel/for-linus-3.17

Unfortunately, this produces too many WARNINGs on some boxes or
with certain configurations.

I will remove this patch from the queue for 3.17.

[    0.000000] ------------[ cut here ]------------
[    0.000000] WARNING: CPU: 0 PID: 0 at /home/build/linux-boris/arch/x86/xen/setup.c:237 xen_update_mem_tables+0xae/0xb8()
[    0.000000] Failed to set m2p mapping for mfn=2302110 pfn=255744
[    0.000000] Modules linked in:
[    0.000000] CPU: 0 PID: 0 Comm: swapper Not tainted 3.16.0-rc5upstream-00266-gf83c52f #1
[    0.000000]  000000ed c17e9d2c c1615056 c1730590 c17e9d5c c109032f c1730618 c17e9d88
[    0.000000]  00000000 c1730590 000000ed c185b4c3 c185b4c3 deadbeef deadbeef 0000009e
[    0.000000]  c17e9d74 c10903ce 00000009 c17e9d6c c1730618 c17e9d88 c17e9db4 c185b4c3
[    0.000000] Call Trace:
[    0.000000]  [<c1615056>] dump_stack+0x48/0x60
[    0.000000]  [<c109032f>] warn_slowpath_common+0x7f/0xa0
[    0.000000]  [<c185b4c3>] ? xen_update_mem_tables+0xae/0xb8
[    0.000000]  [<c185b4c3>] ? xen_update_mem_tables+0xae/0xb8
[    0.000000]  [<c10903ce>] warn_slowpath_fmt+0x2e/0x30
[    0.000000]  [<c185b4c3>] xen_update_mem_tables+0xae/0xb8
[    0.000000]  [<c185bbbf>] xen_memory_setup+0x544/0xb33
[    0.000000]  [<c12de82c>] ? sprintf+0x1c/0x20
[    0.000000]  [<c1861b32>] setup_memory_map+0x19/0x3b
[    0.000000]  [<c185b332>] ? xen_arch_setup+0xad/0xe1
[    0.000000]  [<c185fc33>] setup_arch+0x246/0xe68
[    0.000000]  [<c103bd7e>] ? __raw_callee_save_xen_restore_fl+0x6/0x8
[    0.000000]  [<c10dc5ec>] ? vprintk_emit+0x19c/0x4a0
[    0.000000]  [<c1614efe>] ? printk+0x48/0x4a
[    0.000000]  [<c18569dc>] start_kernel+0x76/0x435
[    0.000000]  [<c103bf77>] ? xen_setup_runstate_info+0x27/0x40
[    0.000000]  [<c1856373>] i386_start_kernel+0x12e/0x131
[    0.000000]  [<c185b106>] xen_start_kernel+0x661/0x66c
[    0.000000] ---[ end trace db4b7fde7786bb07 ]---
[    0.000000] ------------[ cut here ]------------
[    0.000000] WARNING: CPU: 0 PID: 0 at /home/build/linux-boris/arch/x86/xen/setup.c:350 xen_memory_setup+0x580/0xb33()
[    0.000000] Failed to update mem mapping for pfn=256000 mfn=2302110
[    0.000000] Modules linked in:
[    0.000000] CPU: 0 PID: 0 Comm: swapper Tainted: G        W     3.16.0-rc5upstream-00266-gf83c52f #1
[    0.000000]  0000015e c17e9d6c c1615056 c1730590 c17e9d9c c109032f c17306b4 c17e9dc8
[    0.000000]  00000000 c1730590 0000015e c185bbfb c185bbfb 00000062 0003e700 0023209e
[    0.000000]  c17e9db4 c10903ce 00000009 c17e9dac c17306b4 c17e9dc8 c17e9e80 c185bbfb
[    0.000000] Call Trace:
[    0.000000]  [<c1615056>] dump_stack+0x48/0x60
[    0.000000]  [<c109032f>] warn_slowpath_common+0x7f/0xa0
[    0.000000]  [<c185bbfb>] ? xen_memory_setup+0x580/0xb33
[    0.000000]  [<c185bbfb>] ? xen_memory_setup+0x580/0xb33
[    0.000000]  [<c10903ce>] warn_slowpath_fmt+0x2e/0x30
[    0.000000]  [<c185bbfb>] xen_memory_setup+0x580/0xb33
[    0.000000]  [<c12de82c>] ? sprintf+0x1c/0x20
[    0.000000]  [<c1861b32>] setup_memory_map+0x19/0x3b
[    0.000000]  [<c185b332>] ? xen_arch_setup+0xad/0xe1
[    0.000000]  [<c185fc33>] setup_arch+0x246/0xe68
[    0.000000]  [<c103bd7e>] ? __raw_callee_save_xen_restore_fl+0x6/0x8
[    0.000000]  [<c10dc5ec>] ? vprintk_emit+0x19c/0x4a0
[    0.000000]  [<c1614efe>] ? printk+0x48/0x4a
[    0.000000]  [<c18569dc>] start_kernel+0x76/0x435
[    0.000000]  [<c103bf77>] ? xen_setup_runstate_info+0x27/0x40
[    0.000000]  [<c1856373>] i386_start_kernel+0x12e/0x131
[    0.000000]  [<c185b106>] xen_start_kernel+0x661/0x66c
[    0.000000] ---[ end trace db4b7fde7786bb08 ]---
[    0.000000] ------------[ cut here ]------------
[    0.000000] WARNING: CPU: 0 PID: 0 at /home/build/linux-boris/arch/x86/xen/setup.c:436 xen_memory_setup+0x654/0xb33()
[    0.000000] Failed to remap 1:1 memory cur_pfn=158 size=98 remap_pfn=255744
[    0.000000] Modules linked in:
[    0.000000] CPU: 0 PID: 0 Comm: swapper Tainted: G        W     3.16.0-rc5upstream-00266-gf83c52f #1
[    0.000000]  000001b4 c17e9d6c c1615056 c1730590 c17e9d9c c109032f c17306e8 c17e9dc8
[    0.000000]  00000000 c1730590 000001b4 c185bccf c185bccf c17e9e70 0003e700 0023209e
[    0.000000]  c17e9db4 c10903ce 00000009 c17e9dac c17306e8 c17e9dc8 c17e9e80 c185bccf
[    0.000000] Call Trace:
[    0.000000]  [<c1615056>] dump_stack+0x48/0x60
[    0.000000]  [<c109032f>] warn_slowpath_common+0x7f/0xa0
[    0.000000]  [<c185bccf>] ? xen_memory_setup+0x654/0xb33
[    0.000000]  [<c185bccf>] ? xen_memory_setup+0x654/0xb33
[    0.000000]  [<c10903ce>] warn_slowpath_fmt+0x2e/0x30
[    0.000000]  [<c185bccf>] xen_memory_setup+0x654/0xb33
[    0.000000]  [<c1861b32>] setup_memory_map+0x19/0x3b
[    0.000000]  [<c185b332>] ? xen_arch_setup+0xad/0xe1
[    0.000000]  [<c185fc33>] setup_arch+0x246/0xe68
[    0.000000]  [<c103bd7e>] ? __raw_callee_save_xen_restore_fl+0x6/0x8
[    0.000000]  [<c10dc5ec>] ? vprintk_emit+0x19c/0x4a0
[    0.000000]  [<c1614efe>] ? printk+0x48/0x4a
[    0.000000]  [<c18569dc>] start_kernel+0x76/0x435
[    0.000000]  [<c103bf77>] ? xen_setup_runstate_info+0x27/0x40
[    0.000000]  [<c1856373>] i386_start_kernel+0x12e/0x131
[    0.000000]  [<c185b106>] xen_start_kernel+0x661/0x66c
[    0.000000] ---[ end trace db4b7fde7786bb09 ]---
[    0.000000] Freeing 9e-100 pfn range: 98 pages freed
[    0.000000] Set 266437667 page(s) to 1-1 mapping
[    0.000000] Remapped 0 page(s), last_pfn=255744
[    0.000000] Released 98 page(s)
[    0.000000] ------------[ cut here ]------------
[    0.000000] WARNING: CPU: 0 PID: 0 at /home/build/linux-boris/arch/x86/xen/setup.c:95 xen_memory_setup+0x9c4/0xb33()
[    0.000000] Trying to remove 3e700 which has 23209e mfn!
[    0.000000] Modules linked in:
[    0.000000] CPU: 0 PID: 0 Comm: swapper Tainted: G        W     3.16.0-rc5upstream-00266-gf83c52f #1
[    0.000000]  0000005f c17e9d6c c1615056 c1730590 c17e9d9c c109032f c17307d0 c17e9dc8
[    0.000000]  00000000 c1730590 0000005f c185c03f c185c03f 3e700000 00000000 00000003
[    0.000000]  c17e9db4 c10903ce 00000009 c17e9dac c17307d0 c17e9dc8 c17e9e80 c185c03f
[    0.000000] Call Trace:
[    0.000000]  [<c1615056>] dump_stack+0x48/0x60
[    0.000000]  [<c109032f>] warn_slowpath_common+0x7f/0xa0
[    0.000000]  [<c185c03f>] ? xen_memory_setup+0x9c4/0xb33
[    0.000000]  [<c185c03f>] ? xen_memory_setup+0x9c4/0xb33
[    0.000000]  [<c10903ce>] warn_slowpath_fmt+0x2e/0x30
[    0.000000]  [<c185c03f>] xen_memory_setup+0x9c4/0xb33
[    0.000000]  [<c1861b32>] setup_memory_map+0x19/0x3b
[    0.000000]  [<c185b332>] ? xen_arch_setup+0xad/0xe1
[    0.000000]  [<c185fc33>] setup_arch+0x246/0xe68
[    0.000000]  [<c103bd7e>] ? __raw_callee_save_xen_restore_fl+0x6/0x8
[    0.000000]  [<c10dc5ec>] ? vprintk_emit+0x19c/0x4a0
[    0.000000]  [<c1614efe>] ? printk+0x48/0x4a
[    0.000000]  [<c18569dc>] start_kernel+0x76/0x435
[    0.000000]  [<c103bf77>] ? xen_setup_runstate_info+0x27/0x40
[    0.000000]  [<c1856373>] i386_start_kernel+0x12e/0x131
[    0.000000]  [<c185b106>] xen_start_kernel+0x661/0x66c
[    0.000000] ---[ end trace db4b7fde7786bb0a ]---
[    0.000000] e820: BIOS-provided physical RAM map:
[    0.000000] Xen: [mem 0x0000000000000000-0x000000000009dfff] usable
[    0.000000] Xen: [mem 0x000000000009ec00-0x00000000000fffff] reserved
[    0.000000] Xen: [mem 0x0000000000100000-0x000000004d061fff] usable
[    0.000000] Xen: [mem 0x000000004d062000-0x00000000a79b7fff] unusable
[    0.000000] Xen: [mem 0x00000000a79b8000-0x00000000a79fefff] ACPI NVS
[    0.000000] Xen: [mem 0x00000000a79ff000-0x00000000a7a08fff] ACPI data
[    0.000000] Xen: [mem 0x00000000a7a09000-0x00000000a7a65fff] reserved
[    0.000000] Xen: [mem 0x00000000a7a66000-0x00000000a7a68fff] ACPI NVS
[    0.000000] Xen: [mem 0x00000000a7a69000-0x00000000a7a6afff] unusable
[    0.000000] Xen: [mem 0x00000000a7a6b000-0x00000000a7a6bfff] reserved
[    0.000000] Xen: [mem 0x00000000a7a6c000-0x00000000a7a6cfff] ACPI NVS
[    0.000000] Xen: [mem 0x00000000a7a6d000-0x00000000a7aa3fff] reserved
[    0.000000] Xen: [mem 0x00000000a7aa4000-0x00000000a7aaafff] ACPI NVS
[    0.000000] Xen: [mem 0x00000000a7aab000-0x00000000a7ad1fff] reserved
[    0.000000] Xen: [mem 0x00000000a7ad2000-0x00000000a7b14fff] ACPI NVS
[    0.000000] Xen: [mem 0x00000000a7b15000-0x00000000a7d91fff] unusable
[    0.000000] Xen: [mem 0x00000000a7d92000-0x00000000a7ef6fff] reserved
[    0.000000] Xen: [mem 0x00000000a7ef7000-0x00000000a7efffff] unusable
[    0.000000] Xen: [mem 0x00000000fec00000-0x00000000fec00fff] reserved
[    0.000000] Xen: [mem 0x00000000fec10000-0x00000000fec10fff] reserved
[    0.000000] Xen: [mem 0x00000000fed00000-0x00000000fed00fff] reserved
[    0.000000] Xen: [mem 0x00000000fed61000-0x00000000fed70fff] reserved
[    0.000000] Xen: [mem 0x00000000fed80000-0x00000000fed8ffff] reserved
[    0.000000] Xen: [mem 0x00000000fee00000-0x00000000ffffffff] reserved
[    0.000000] Xen: [mem 0x0000000100001000-0x000000023fffffff] unusable
[    0.000000] Xen: [mem 0x000000fd00000000-0x000000ffffffffff] reserved

David

Re: [PATCH v2 2/2] xen/setup: Remap Xen Identity Mapped RAM

[David Vrabel]

On 31/07/14 18:43, David Vrabel wrote:
> On 20/07/14 01:01, Matt Rushton wrote:
>> Instead of ballooning up and down dom0 memory this remaps the existing mfns
>> that were replaced by the identity map. The reason for this is that the
>> existing implementation ballooned memory up and and down which caused dom0
>> to have discontiguous pages. In some cases this resulted in the use of bounce
>> buffers which reduced network I/O performance significantly. This change will
>> honor the existing order of the pages with the exception of some boundary
>> conditions.
>>
>> To do this we need to update both the Linux p2m table and the Xen m2p table.
>> Particular care must be taken when updating the p2m table since it's important
>> to limit table memory consumption and reuse the existing leaf pages which get
>> freed when an entire leaf page is set to the identity map. To implement this,
>> mapping updates are grouped into blocks with table entries getting cached
>> temporarily and then released.
>>
>> On my test system before:
>> Total pages: 2105014
>> Total contiguous: 1640635
>>
>> After:
>> Total pages: 2105014
>> Total contiguous: 2098904
> 
> Applied to devel/for-linus-3.17

Unfortunately, this produces too many WARNINGs on some boxes or
with certain configurations.

I will remove this patch from the queue for 3.17.

[    0.000000] ------------[ cut here ]------------
[    0.000000] WARNING: CPU: 0 PID: 0 at /home/build/linux-boris/arch/x86/xen/setup.c:237 xen_update_mem_tables+0xae/0xb8()
[    0.000000] Failed to set m2p mapping for mfn=2302110 pfn=255744
[    0.000000] Modules linked in:
[    0.000000] CPU: 0 PID: 0 Comm: swapper Not tainted 3.16.0-rc5upstream-00266-gf83c52f #1
[    0.000000]  000000ed c17e9d2c c1615056 c1730590 c17e9d5c c109032f c1730618 c17e9d88
[    0.000000]  00000000 c1730590 000000ed c185b4c3 c185b4c3 deadbeef deadbeef 0000009e
[    0.000000]  c17e9d74 c10903ce 00000009 c17e9d6c c1730618 c17e9d88 c17e9db4 c185b4c3
[    0.000000] Call Trace:
[    0.000000]  [<c1615056>] dump_stack+0x48/0x60
[    0.000000]  [<c109032f>] warn_slowpath_common+0x7f/0xa0
[    0.000000]  [<c185b4c3>] ? xen_update_mem_tables+0xae/0xb8
[    0.000000]  [<c185b4c3>] ? xen_update_mem_tables+0xae/0xb8
[    0.000000]  [<c10903ce>] warn_slowpath_fmt+0x2e/0x30
[    0.000000]  [<c185b4c3>] xen_update_mem_tables+0xae/0xb8
[    0.000000]  [<c185bbbf>] xen_memory_setup+0x544/0xb33
[    0.000000]  [<c12de82c>] ? sprintf+0x1c/0x20
[    0.000000]  [<c1861b32>] setup_memory_map+0x19/0x3b
[    0.000000]  [<c185b332>] ? xen_arch_setup+0xad/0xe1
[    0.000000]  [<c185fc33>] setup_arch+0x246/0xe68
[    0.000000]  [<c103bd7e>] ? __raw_callee_save_xen_restore_fl+0x6/0x8
[    0.000000]  [<c10dc5ec>] ? vprintk_emit+0x19c/0x4a0
[    0.000000]  [<c1614efe>] ? printk+0x48/0x4a
[    0.000000]  [<c18569dc>] start_kernel+0x76/0x435
[    0.000000]  [<c103bf77>] ? xen_setup_runstate_info+0x27/0x40
[    0.000000]  [<c1856373>] i386_start_kernel+0x12e/0x131
[    0.000000]  [<c185b106>] xen_start_kernel+0x661/0x66c
[    0.000000] ---[ end trace db4b7fde7786bb07 ]---
[    0.000000] ------------[ cut here ]------------
[    0.000000] WARNING: CPU: 0 PID: 0 at /home/build/linux-boris/arch/x86/xen/setup.c:350 xen_memory_setup+0x580/0xb33()
[    0.000000] Failed to update mem mapping for pfn=256000 mfn=2302110
[    0.000000] Modules linked in:
[    0.000000] CPU: 0 PID: 0 Comm: swapper Tainted: G        W     3.16.0-rc5upstream-00266-gf83c52f #1
[    0.000000]  0000015e c17e9d6c c1615056 c1730590 c17e9d9c c109032f c17306b4 c17e9dc8
[    0.000000]  00000000 c1730590 0000015e c185bbfb c185bbfb 00000062 0003e700 0023209e
[    0.000000]  c17e9db4 c10903ce 00000009 c17e9dac c17306b4 c17e9dc8 c17e9e80 c185bbfb
[    0.000000] Call Trace:
[    0.000000]  [<c1615056>] dump_stack+0x48/0x60
[    0.000000]  [<c109032f>] warn_slowpath_common+0x7f/0xa0
[    0.000000]  [<c185bbfb>] ? xen_memory_setup+0x580/0xb33
[    0.000000]  [<c185bbfb>] ? xen_memory_setup+0x580/0xb33
[    0.000000]  [<c10903ce>] warn_slowpath_fmt+0x2e/0x30
[    0.000000]  [<c185bbfb>] xen_memory_setup+0x580/0xb33
[    0.000000]  [<c12de82c>] ? sprintf+0x1c/0x20
[    0.000000]  [<c1861b32>] setup_memory_map+0x19/0x3b
[    0.000000]  [<c185b332>] ? xen_arch_setup+0xad/0xe1
[    0.000000]  [<c185fc33>] setup_arch+0x246/0xe68
[    0.000000]  [<c103bd7e>] ? __raw_callee_save_xen_restore_fl+0x6/0x8
[    0.000000]  [<c10dc5ec>] ? vprintk_emit+0x19c/0x4a0
[    0.000000]  [<c1614efe>] ? printk+0x48/0x4a
[    0.000000]  [<c18569dc>] start_kernel+0x76/0x435
[    0.000000]  [<c103bf77>] ? xen_setup_runstate_info+0x27/0x40
[    0.000000]  [<c1856373>] i386_start_kernel+0x12e/0x131
[    0.000000]  [<c185b106>] xen_start_kernel+0x661/0x66c
[    0.000000] ---[ end trace db4b7fde7786bb08 ]---
[    0.000000] ------------[ cut here ]------------
[    0.000000] WARNING: CPU: 0 PID: 0 at /home/build/linux-boris/arch/x86/xen/setup.c:436 xen_memory_setup+0x654/0xb33()
[    0.000000] Failed to remap 1:1 memory cur_pfn=158 size=98 remap_pfn=255744
[    0.000000] Modules linked in:
[    0.000000] CPU: 0 PID: 0 Comm: swapper Tainted: G        W     3.16.0-rc5upstream-00266-gf83c52f #1
[    0.000000]  000001b4 c17e9d6c c1615056 c1730590 c17e9d9c c109032f c17306e8 c17e9dc8
[    0.000000]  00000000 c1730590 000001b4 c185bccf c185bccf c17e9e70 0003e700 0023209e
[    0.000000]  c17e9db4 c10903ce 00000009 c17e9dac c17306e8 c17e9dc8 c17e9e80 c185bccf
[    0.000000] Call Trace:
[    0.000000]  [<c1615056>] dump_stack+0x48/0x60
[    0.000000]  [<c109032f>] warn_slowpath_common+0x7f/0xa0
[    0.000000]  [<c185bccf>] ? xen_memory_setup+0x654/0xb33
[    0.000000]  [<c185bccf>] ? xen_memory_setup+0x654/0xb33
[    0.000000]  [<c10903ce>] warn_slowpath_fmt+0x2e/0x30
[    0.000000]  [<c185bccf>] xen_memory_setup+0x654/0xb33
[    0.000000]  [<c1861b32>] setup_memory_map+0x19/0x3b
[    0.000000]  [<c185b332>] ? xen_arch_setup+0xad/0xe1
[    0.000000]  [<c185fc33>] setup_arch+0x246/0xe68
[    0.000000]  [<c103bd7e>] ? __raw_callee_save_xen_restore_fl+0x6/0x8
[    0.000000]  [<c10dc5ec>] ? vprintk_emit+0x19c/0x4a0
[    0.000000]  [<c1614efe>] ? printk+0x48/0x4a
[    0.000000]  [<c18569dc>] start_kernel+0x76/0x435
[    0.000000]  [<c103bf77>] ? xen_setup_runstate_info+0x27/0x40
[    0.000000]  [<c1856373>] i386_start_kernel+0x12e/0x131
[    0.000000]  [<c185b106>] xen_start_kernel+0x661/0x66c
[    0.000000] ---[ end trace db4b7fde7786bb09 ]---
[    0.000000] Freeing 9e-100 pfn range: 98 pages freed
[    0.000000] Set 266437667 page(s) to 1-1 mapping
[    0.000000] Remapped 0 page(s), last_pfn=255744
[    0.000000] Released 98 page(s)
[    0.000000] ------------[ cut here ]------------
[    0.000000] WARNING: CPU: 0 PID: 0 at /home/build/linux-boris/arch/x86/xen/setup.c:95 xen_memory_setup+0x9c4/0xb33()
[    0.000000] Trying to remove 3e700 which has 23209e mfn!
[    0.000000] Modules linked in:
[    0.000000] CPU: 0 PID: 0 Comm: swapper Tainted: G        W     3.16.0-rc5upstream-00266-gf83c52f #1
[    0.000000]  0000005f c17e9d6c c1615056 c1730590 c17e9d9c c109032f c17307d0 c17e9dc8
[    0.000000]  00000000 c1730590 0000005f c185c03f c185c03f 3e700000 00000000 00000003
[    0.000000]  c17e9db4 c10903ce 00000009 c17e9dac c17307d0 c17e9dc8 c17e9e80 c185c03f
[    0.000000] Call Trace:
[    0.000000]  [<c1615056>] dump_stack+0x48/0x60
[    0.000000]  [<c109032f>] warn_slowpath_common+0x7f/0xa0
[    0.000000]  [<c185c03f>] ? xen_memory_setup+0x9c4/0xb33
[    0.000000]  [<c185c03f>] ? xen_memory_setup+0x9c4/0xb33
[    0.000000]  [<c10903ce>] warn_slowpath_fmt+0x2e/0x30
[    0.000000]  [<c185c03f>] xen_memory_setup+0x9c4/0xb33
[    0.000000]  [<c1861b32>] setup_memory_map+0x19/0x3b
[    0.000000]  [<c185b332>] ? xen_arch_setup+0xad/0xe1
[    0.000000]  [<c185fc33>] setup_arch+0x246/0xe68
[    0.000000]  [<c103bd7e>] ? __raw_callee_save_xen_restore_fl+0x6/0x8
[    0.000000]  [<c10dc5ec>] ? vprintk_emit+0x19c/0x4a0
[    0.000000]  [<c1614efe>] ? printk+0x48/0x4a
[    0.000000]  [<c18569dc>] start_kernel+0x76/0x435
[    0.000000]  [<c103bf77>] ? xen_setup_runstate_info+0x27/0x40
[    0.000000]  [<c1856373>] i386_start_kernel+0x12e/0x131
[    0.000000]  [<c185b106>] xen_start_kernel+0x661/0x66c
[    0.000000] ---[ end trace db4b7fde7786bb0a ]---
[    0.000000] e820: BIOS-provided physical RAM map:
[    0.000000] Xen: [mem 0x0000000000000000-0x000000000009dfff] usable
[    0.000000] Xen: [mem 0x000000000009ec00-0x00000000000fffff] reserved
[    0.000000] Xen: [mem 0x0000000000100000-0x000000004d061fff] usable
[    0.000000] Xen: [mem 0x000000004d062000-0x00000000a79b7fff] unusable
[    0.000000] Xen: [mem 0x00000000a79b8000-0x00000000a79fefff] ACPI NVS
[    0.000000] Xen: [mem 0x00000000a79ff000-0x00000000a7a08fff] ACPI data
[    0.000000] Xen: [mem 0x00000000a7a09000-0x00000000a7a65fff] reserved
[    0.000000] Xen: [mem 0x00000000a7a66000-0x00000000a7a68fff] ACPI NVS
[    0.000000] Xen: [mem 0x00000000a7a69000-0x00000000a7a6afff] unusable
[    0.000000] Xen: [mem 0x00000000a7a6b000-0x00000000a7a6bfff] reserved
[    0.000000] Xen: [mem 0x00000000a7a6c000-0x00000000a7a6cfff] ACPI NVS
[    0.000000] Xen: [mem 0x00000000a7a6d000-0x00000000a7aa3fff] reserved
[    0.000000] Xen: [mem 0x00000000a7aa4000-0x00000000a7aaafff] ACPI NVS
[    0.000000] Xen: [mem 0x00000000a7aab000-0x00000000a7ad1fff] reserved
[    0.000000] Xen: [mem 0x00000000a7ad2000-0x00000000a7b14fff] ACPI NVS
[    0.000000] Xen: [mem 0x00000000a7b15000-0x00000000a7d91fff] unusable
[    0.000000] Xen: [mem 0x00000000a7d92000-0x00000000a7ef6fff] reserved
[    0.000000] Xen: [mem 0x00000000a7ef7000-0x00000000a7efffff] unusable
[    0.000000] Xen: [mem 0x00000000fec00000-0x00000000fec00fff] reserved
[    0.000000] Xen: [mem 0x00000000fec10000-0x00000000fec10fff] reserved
[    0.000000] Xen: [mem 0x00000000fed00000-0x00000000fed00fff] reserved
[    0.000000] Xen: [mem 0x00000000fed61000-0x00000000fed70fff] reserved
[    0.000000] Xen: [mem 0x00000000fed80000-0x00000000fed8ffff] reserved
[    0.000000] Xen: [mem 0x00000000fee00000-0x00000000ffffffff] reserved
[    0.000000] Xen: [mem 0x0000000100001000-0x000000023fffffff] unusable
[    0.000000] Xen: [mem 0x000000fd00000000-0x000000ffffffffff] reserved

David

Re: [PATCH v2 2/2] xen/setup: Remap Xen Identity Mapped RAM

[Matt Wilson]

On Fri, Aug 01, 2014 at 03:52:28PM +0100, David Vrabel wrote:
> On 31/07/14 18:43, David Vrabel wrote:
> > On 20/07/14 01:01, Matt Rushton wrote:
> >> Instead of ballooning up and down dom0 memory this remaps the existing mfns
> >> that were replaced by the identity map. The reason for this is that the
> >> existing implementation ballooned memory up and and down which caused dom0
> >> to have discontiguous pages. In some cases this resulted in the use of bounce
> >> buffers which reduced network I/O performance significantly. This change will
> >> honor the existing order of the pages with the exception of some boundary
> >> conditions.
> >>
> >> To do this we need to update both the Linux p2m table and the Xen m2p table.
> >> Particular care must be taken when updating the p2m table since it's important
> >> to limit table memory consumption and reuse the existing leaf pages which get
> >> freed when an entire leaf page is set to the identity map. To implement this,
> >> mapping updates are grouped into blocks with table entries getting cached
> >> temporarily and then released.
> >>
> >> On my test system before:
> >> Total pages: 2105014
> >> Total contiguous: 1640635
> >>
> >> After:
> >> Total pages: 2105014
> >> Total contiguous: 2098904
> > 
> > Applied to devel/for-linus-3.17
> 
> Unfortunately, this produces too many WARNINGs on some boxes or
> with certain configurations.

Hi David,

Do you have more information about the systems or configurations that
showed a problem?

--msw

Re: [PATCH v2 2/2] xen/setup: Remap Xen Identity Mapped RAM

[Boris Ostrovsky]

On 08/01/2014 11:52 AM, Matt Wilson wrote:
> On Fri, Aug 01, 2014 at 03:52:28PM +0100, David Vrabel wrote:
>> On 31/07/14 18:43, David Vrabel wrote:
>>> On 20/07/14 01:01, Matt Rushton wrote:
>>>> Instead of ballooning up and down dom0 memory this remaps the existing mfns
>>>> that were replaced by the identity map. The reason for this is that the
>>>> existing implementation ballooned memory up and and down which caused dom0
>>>> to have discontiguous pages. In some cases this resulted in the use of bounce
>>>> buffers which reduced network I/O performance significantly. This change will
>>>> honor the existing order of the pages with the exception of some boundary
>>>> conditions.
>>>>
>>>> To do this we need to update both the Linux p2m table and the Xen m2p table.
>>>> Particular care must be taken when updating the p2m table since it's important
>>>> to limit table memory consumption and reuse the existing leaf pages which get
>>>> freed when an entire leaf page is set to the identity map. To implement this,
>>>> mapping updates are grouped into blocks with table entries getting cached
>>>> temporarily and then released.
>>>>
>>>> On my test system before:
>>>> Total pages: 2105014
>>>> Total contiguous: 1640635
>>>>
>>>> After:
>>>> Total pages: 2105014
>>>> Total contiguous: 2098904
>>> Applied to devel/for-linus-3.17
>> Unfortunately, this produces too many WARNINGs on some boxes or
>> with certain configurations.
> Hi David,
>
> Do you have more information about the systems or configurations that
> showed a problem?
>


This appears to be happening on 32-bit dom0.

-boris

Re: [PATCH v2 2/2] xen/setup: Remap Xen Identity Mapped RAM

[Matthew Rushton]

On 08/01/14 08:56, Boris Ostrovsky wrote:
> On 08/01/2014 11:52 AM, Matt Wilson wrote:
>> On Fri, Aug 01, 2014 at 03:52:28PM +0100, David Vrabel wrote:
>>> On 31/07/14 18:43, David Vrabel wrote:
>>>> On 20/07/14 01:01, Matt Rushton wrote:
>>>>> Instead of ballooning up and down dom0 memory this remaps the 
>>>>> existing mfns
>>>>> that were replaced by the identity map. The reason for this is 
>>>>> that the
>>>>> existing implementation ballooned memory up and and down which 
>>>>> caused dom0
>>>>> to have discontiguous pages. In some cases this resulted in the 
>>>>> use of bounce
>>>>> buffers which reduced network I/O performance significantly. This 
>>>>> change will
>>>>> honor the existing order of the pages with the exception of some 
>>>>> boundary
>>>>> conditions.
>>>>>
>>>>> To do this we need to update both the Linux p2m table and the Xen 
>>>>> m2p table.
>>>>> Particular care must be taken when updating the p2m table since 
>>>>> it's important
>>>>> to limit table memory consumption and reuse the existing leaf 
>>>>> pages which get
>>>>> freed when an entire leaf page is set to the identity map. To 
>>>>> implement this,
>>>>> mapping updates are grouped into blocks with table entries getting 
>>>>> cached
>>>>> temporarily and then released.
>>>>>
>>>>> On my test system before:
>>>>> Total pages: 2105014
>>>>> Total contiguous: 1640635
>>>>>
>>>>> After:
>>>>> Total pages: 2105014
>>>>> Total contiguous: 2098904
>>>> Applied to devel/for-linus-3.17
>>> Unfortunately, this produces too many WARNINGs on some boxes or
>>> with certain configurations.
>> Hi David,
>>
>> Do you have more information about the systems or configurations that
>> showed a problem?
>>
>
>
> This appears to be happening on 32-bit dom0.
>
> -boris

It's unclear to me why the m2p update would be failing in this case (on 
first attempt) which causes the subsequent errors. I'll try to repro on 
a 32-bit dom0.

Re: [PATCH v2 2/2] xen/setup: Remap Xen Identity Mapped RAM

[Matthew Rushton]

On 08/01/14 08:56, Boris Ostrovsky wrote:
> On 08/01/2014 11:52 AM, Matt Wilson wrote:
>> On Fri, Aug 01, 2014 at 03:52:28PM +0100, David Vrabel wrote:
>>> On 31/07/14 18:43, David Vrabel wrote:
>>>> On 20/07/14 01:01, Matt Rushton wrote:
>>>>> Instead of ballooning up and down dom0 memory this remaps the 
>>>>> existing mfns
>>>>> that were replaced by the identity map. The reason for this is 
>>>>> that the
>>>>> existing implementation ballooned memory up and and down which 
>>>>> caused dom0
>>>>> to have discontiguous pages. In some cases this resulted in the 
>>>>> use of bounce
>>>>> buffers which reduced network I/O performance significantly. This 
>>>>> change will
>>>>> honor the existing order of the pages with the exception of some 
>>>>> boundary
>>>>> conditions.
>>>>>
>>>>> To do this we need to update both the Linux p2m table and the Xen 
>>>>> m2p table.
>>>>> Particular care must be taken when updating the p2m table since 
>>>>> it's important
>>>>> to limit table memory consumption and reuse the existing leaf 
>>>>> pages which get
>>>>> freed when an entire leaf page is set to the identity map. To 
>>>>> implement this,
>>>>> mapping updates are grouped into blocks with table entries getting 
>>>>> cached
>>>>> temporarily and then released.
>>>>>
>>>>> On my test system before:
>>>>> Total pages: 2105014
>>>>> Total contiguous: 1640635
>>>>>
>>>>> After:
>>>>> Total pages: 2105014
>>>>> Total contiguous: 2098904
>>>> Applied to devel/for-linus-3.17
>>> Unfortunately, this produces too many WARNINGs on some boxes or
>>> with certain configurations.
>> Hi David,
>>
>> Do you have more information about the systems or configurations that
>> showed a problem?
>>
>
>
> This appears to be happening on 32-bit dom0.
>
> -boris

I just pushed a v3 version of the patch set which should fix the issue. 
There was casting issue that would affect 32-bit builds.

Re: [PATCH v2 2/2] xen/setup: Remap Xen Identity Mapped RAM

[Matthew Rushton]

On 08/01/14 07:52, David Vrabel wrote:
> On 31/07/14 18:43, David Vrabel wrote:
>> On 20/07/14 01:01, Matt Rushton wrote:
>>> Instead of ballooning up and down dom0 memory this remaps the existing mfns
>>> that were replaced by the identity map. The reason for this is that the
>>> existing implementation ballooned memory up and and down which caused dom0
>>> to have discontiguous pages. In some cases this resulted in the use of bounce
>>> buffers which reduced network I/O performance significantly. This change will
>>> honor the existing order of the pages with the exception of some boundary
>>> conditions.
>>>
>>> To do this we need to update both the Linux p2m table and the Xen m2p table.
>>> Particular care must be taken when updating the p2m table since it's important
>>> to limit table memory consumption and reuse the existing leaf pages which get
>>> freed when an entire leaf page is set to the identity map. To implement this,
>>> mapping updates are grouped into blocks with table entries getting cached
>>> temporarily and then released.
>>>
>>> On my test system before:
>>> Total pages: 2105014
>>> Total contiguous: 1640635
>>>
>>> After:
>>> Total pages: 2105014
>>> Total contiguous: 2098904
>> Applied to devel/for-linus-3.17
> Unfortunately, this produces too many WARNINGs on some boxes or
> with certain configurations.
>
> I will remove this patch from the queue for 3.17.
>

Have you had a chance to try the v3 patch set?