[RFC 4/4] Implement VMF for arm64

["Smith, Jackson" <rsmith@RiversideResearch.org>]

Implements the functions from xen/vmf.h for arm64.
Introduces an xen/arch/arm/mm-walk.c helper file for
walking an entire page table structure.
---
 xen/arch/arm/Makefile              |   1 +
 xen/arch/arm/include/asm/mm-walk.h |  53 ++++++++++
 xen/arch/arm/include/asm/mm.h      |  11 +++
 xen/arch/arm/mm-walk.c             | 181 +++++++++++++++++++++++++++++++++
 xen/arch/arm/mm.c                  | 198 ++++++++++++++++++++++++++++++++++++-
 xen/common/Kconfig                 |   2 +
 6 files changed, 445 insertions(+), 1 deletion(-)
 create mode 100644 xen/arch/arm/include/asm/mm-walk.h
 create mode 100644 xen/arch/arm/mm-walk.c

diff --git a/xen/arch/arm/Makefile b/xen/arch/arm/Makefile
index 4d076b2..e358452 100644
--- a/xen/arch/arm/Makefile
+++ b/xen/arch/arm/Makefile
@@ -37,6 +37,7 @@ obj-y += kernel.init.o
 obj-$(CONFIG_LIVEPATCH) += livepatch.o
 obj-y += mem_access.o
 obj-y += mm.o
+obj-y += mm-walk.o
 obj-y += monitor.o
 obj-y += p2m.o
 obj-y += percpu.o
diff --git a/xen/arch/arm/include/asm/mm-walk.h b/xen/arch/arm/include/asm/mm-walk.h
new file mode 100644
index 0000000..770cc89
--- /dev/null
+++ b/xen/arch/arm/include/asm/mm-walk.h
@@ -0,0 +1,53 @@
+#ifndef __ARM_MM_WALK_H__
+#define __ARM_MM_WALK_H__
+
+#include <asm/lpae.h>
+
+#define RECURSIVE_IDX ((unsigned long)(XEN_PT_LPAE_ENTRIES-1))
+#define RECURSIVE_VA (RECURSIVE_IDX << ZEROETH_SHIFT)
+
+/*
+ * Remove all mappings in these tables from Xen's address space
+ * Only makes sense if walking a guest's tables
+ */
+#define WALK_HIDE_GUEST_MAPPING (1U << 0)
+/*
+ * Remove all mappings to these tables from Xen's address space
+ * Makes sense if walking a guest's table (hide guest tables from Xen)
+ * Or if walking Xen's tables (lock Xen's virtual memory configuration)
+ */
+#define WALK_HIDE_GUEST_TABLE (1U << 1)
+
+/*
+ * Before we can hide individual table entires,
+ * we need to split the directmap superpages
+ */
+#define WALK_SPLIT_DIRECTMAP_TABLE (1U << 2)
+/*
+ * Like walk table hide, but using recursive mapping
+ * to bypass walking directmap when table is in the directmap
+ */
+#define WALK_HIDE_DIRECTMAP_TABLE (1U << 3)
+
+/* These are useful for development/debug */
+/* Show all pte's for a given address space */
+#define WALK_DUMP_ENTRIES (1U << 4)
+/* Show all mappings for a given address space */
+#define WALK_DUMP_MAPPINGS (1U << 5)
+
+/*
+ * Given the value of a ttbr register, this function walks every valid entry in the trie
+ * (As opposed to dump_pt_walk, which follows a single address from root to leaf)
+ */
+void do_walk_tables(paddr_t ttbr, int root_level, int nr_root_tables, int flags);
+
+#endif /*  __ARM_MM_WALK_H__ */
+/*
+ * Local variables:
+ * mode: C
+ * c-file-style: "BSD"
+ * c-basic-offset: 4
+ * tab-width: 4
+ * indent-tabs-mode: nil
+ * End:
+ */
diff --git a/xen/arch/arm/include/asm/mm.h b/xen/arch/arm/include/asm/mm.h
index 68adcac..2e85885 100644
--- a/xen/arch/arm/include/asm/mm.h
+++ b/xen/arch/arm/include/asm/mm.h
@@ -209,6 +209,17 @@ extern void mmu_init_secondary_cpu(void);
  * For Arm64, map the region in the directmap area.
  */
 extern void setup_directmap_mappings(unsigned long base_mfn, unsigned long nr_mfns);
+/* Shatter superpages for these mfns if needed */
+extern int split_directmap_mapping(unsigned long mfn, unsigned long nr_mfns);
+/* Remove these mfns from the directmap */
+extern int destroy_directmap_mapping(unsigned long mfn, unsigned long nr_mfns);
+/*
+ * Remove this mfn from the directmap (bypassing normal update code)
+ * This is a workaround for current pgtable update code, which cannot be used
+ * to remove directmap table entries from the directmap (because they are
+ * needed to walk the directmap)
+ */
+extern void destroy_directmap_table(unsigned long mfn);
 /* Map a frame table to cover physical addresses ps through pe */
 extern void setup_frametable_mappings(paddr_t ps, paddr_t pe);
 /* map a physical range in virtual memory */
diff --git a/xen/arch/arm/mm-walk.c b/xen/arch/arm/mm-walk.c
new file mode 100644
index 0000000..48f9b2d
--- /dev/null
+++ b/xen/arch/arm/mm-walk.c
@@ -0,0 +1,181 @@
+/*
+ * xen/arch/arm/mm-walk.c
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#include <xen/lib.h>
+#include <xen/domain_page.h>
+
+#include <asm/page.h>
+#include <asm/mm-walk.h>
+
+typedef struct {
+    /* Keeps track of all the table offsets so we can reconstruct the VA if we need to */
+    int off[4];
+
+    /* Keeps track of root level so we can make sense of the table offsets */
+    int root_level;
+    int root_table_idx; /* only meaningful when nr_root_tables > 1 */
+} walk_info_t;
+
+/*
+ * Turn a walk_info_t into a virtual address
+ *
+ * XXX: This only applies to the lower VA range
+ * Ie. if you are looking at a table in ttbr1, this is different
+ * XXX: doesn't work for concat tables right now either
+ */
+static unsigned long walk_to_va(int level, walk_info_t *walk)
+{
+/* #define off_valid(x) (((x) <= level) && ((x) >= walk->root_level)) */
+#define off_valid(x) ((x) <= level)
+#define off_val(x) ((u64)(off_valid(x) ? walk->off[x] : 0))
+
+    return (off_val(0) << ZEROETH_SHIFT)  \
+           | (off_val(1) << FIRST_SHIFT)  \
+           | (off_val(2) << SECOND_SHIFT) \
+           | (off_val(3) << THIRD_SHIFT);
+}
+
+/* Prints each entry in the form "\t @XTH TABLE:0.0.0.0 = 0xENTRY" */
+static void dump_entry(int level, lpae_t pte, walk_info_t *walk)
+{
+    int i;
+    static const char *level_strs[4] = { "0TH", "1ST", "2ND", "3RD" };
+    ASSERT(level <= 3);
+
+    for (i = 0; i < level; i++)
+        printk("  ");
+
+    printk("@%s %i:", level_strs[level], walk->root_table_idx);
+
+    for (i = walk->root_level; i < level; i++)
+        printk("%d.", walk->off[i]);
+
+    printk("%d = 0x%lx\n", walk->off[level], pte.bits);
+}
+
+/* Prints each mapping in the form IA:0xIA -> OFN:0xOFN XG,M,K */
+static void dump_mapping(int level, lpae_t pte, walk_info_t *walk)
+{
+    unsigned long va;
+    unsigned long ofn = pte.walk.base;
+    const char *size[4] = {"??", "1G", "2M", "4K"};
+
+    ASSERT(level >= 1);
+    ASSERT(level <= 3);
+
+    va = walk_to_va(level, walk);
+
+    /* ofn stands for output frame number.. I just made it up. */
+    printk("0x%lx -> 0x%lx %s\n", va, ofn, size[level]);
+}
+
+/* Recursive walk function */
+static void walk_table(mfn_t mfn, int level, walk_info_t *walk, int flags)
+{
+    lpae_t *table;
+
+    #define i (walk->off[level])
+
+    BUG_ON(level > 3);
+
+    table = map_domain_page(mfn);
+    for ( i = 0; i < XEN_PT_LPAE_ENTRIES; i++ )
+    {
+        lpae_t pte = table[i];
+        if ( !lpae_is_valid(pte) )
+            continue;
+
+        /* Skip recursive mapping */
+        if ( level == 0 && i == RECURSIVE_IDX )
+            continue;
+
+        if ( flags & WALK_DUMP_ENTRIES )
+            dump_entry(level, pte, walk);
+
+        if ( lpae_is_mapping(pte, level) )
+        {
+            /* Do mapping related things */
+            if ( flags & WALK_DUMP_MAPPINGS )
+                dump_mapping(level, pte, walk);
+            if ( flags & WALK_HIDE_GUEST_MAPPING )
+                /* Destroy all of Xen's mappings to the physical frames covered by this entry */
+                destroy_directmap_mapping(pte.walk.base, 1 << XEN_PT_LEVEL_ORDER(level));
+        }
+        else if ( lpae_is_table(pte, level) )
+        {
+            /* else, pte is a table: recurse! */
+            walk_table(lpae_get_mfn(pte), level + 1, walk, flags);
+
+            /* Note that the entry is a normal entry in xen's page tables */
+            if ( flags & WALK_HIDE_GUEST_TABLE )
+                /*
+                 * This call will look up the table pointed to by this entry in the directmap
+                 * and remove it in the typical way
+                 * This leaves the table intact, but removes the directmap mapping to it, hiding it from xen
+                 */
+                destroy_directmap_mapping(pte.walk.base, 1);
+            if ( flags & WALK_SPLIT_DIRECTMAP_TABLE )
+                /*
+                 * This call will look up the table pointed to by this entry in the directmap
+                 * and make sure that it has it's own l3 entry, splitting superpages if needed
+                 */
+                split_directmap_mapping(pte.walk.base, 1);
+            if ( flags & WALK_HIDE_DIRECTMAP_TABLE )
+                /*
+                 * This call will look up the table pointed to by this entry in the directmap
+                 * and (now that it has it's own l3 entry) overwrite that entry with 0's
+                 * This leaves the table intact, but removes the directmap mapping to it, hiding it from xen
+                 */
+                destroy_directmap_table(pte.walk.base);
+        }
+        /* else, invalid pte, level == 3, vaild == true, table = false */
+    }
+    unmap_domain_page(table);
+
+    #undef i
+}
+
+void do_walk_tables(paddr_t ttbr, int root_level, int nr_root_tables, int flags)
+{
+    int i;
+    mfn_t root = maddr_to_mfn(ttbr & PADDR_MASK);
+    walk_info_t walk = {
+        .off = {0},
+        .root_level = root_level,
+    };
+
+    BUG_ON( !mfn_x(root) || !mfn_valid(root) );
+
+    for ( i = 0; i < nr_root_tables; i++, root = mfn_add(root, 1) ) {
+        walk.root_table_idx = i;
+        walk_table(root, root_level, &walk, flags);
+
+        /* Our walk doesn't consider the root table, so do that here */
+        if ( flags & WALK_SPLIT_DIRECTMAP_TABLE )
+            split_directmap_mapping(mfn_x(root), 1);
+        if ( flags & WALK_HIDE_GUEST_TABLE )
+            destroy_directmap_mapping(mfn_x(root), 1);
+        if ( flags & WALK_HIDE_DIRECTMAP_TABLE )
+            destroy_directmap_table(mfn_x(root));
+    }
+}
+
+/*
+ * Local variables:
+ * mode: C
+ * c-file-style: "BSD"
+ * c-basic-offset: 4
+ * indent-tabs-mode: nil
+ * End:
+ */
diff --git a/xen/arch/arm/mm.c b/xen/arch/arm/mm.c
index 91b9c2b..64e9efd 100644
--- a/xen/arch/arm/mm.c
+++ b/xen/arch/arm/mm.c
@@ -21,11 +21,13 @@
 #include <xen/sizes.h>
 #include <xen/types.h>
 #include <xen/vmap.h>
+#include <xen/vmf.h>
 
 #include <xsm/xsm.h>
 
 #include <asm/fixmap.h>
 #include <asm/setup.h>
+#include <asm/mm-walk.h>
 
 #include <public/memory.h>
 
@@ -1164,7 +1166,8 @@ static int xen_pt_update(unsigned long virt,
      *
      * XXX: Add a check.
      */
-    const mfn_t root = virt_to_mfn(THIS_CPU_PGTABLE);
+    /* TODO: does this change have a negative performance impact? */
+    const mfn_t root = maddr_to_mfn(READ_SYSREG64(TTBR0_EL2));
 
     /*
      * The hardware was configured to forbid mapping both writeable and
@@ -1273,6 +1276,199 @@ int modify_xen_mappings(unsigned long s, unsigned long e, unsigned int flags)
     return xen_pt_update(s, INVALID_MFN, (e - s) >> PAGE_SHIFT, flags);
 }
 
+static void insert_recursive_mapping(void)
+{
+    uint64_t ttbr = READ_SYSREG64(TTBR0_EL2);
+    const mfn_t root_mfn = maddr_to_mfn(ttbr & PADDR_MASK);
+    lpae_t *pgtable = map_domain_page(root_mfn);
+
+    lpae_t pte = mfn_to_xen_entry(root_mfn, MT_NORMAL);
+    pte.pt.table = 1;
+
+    spin_lock(&xen_pt_lock);
+
+    write_pte(&pgtable[RECURSIVE_IDX], pte);
+    clean_dcache(pgtable[RECURSIVE_IDX]);
+
+    unmap_domain_page(pgtable);
+    spin_unlock(&xen_pt_lock);
+}
+
+/*
+ * Converts va to a table pointer through the recursive mapping
+ * Only valid for the current address space obviously
+ */
+static lpae_t *va_to_table(int level, unsigned long va)
+{
+    /* Shift everything by 9 for each walk we skip */
+    /* Last off shifted out becomes becomes offset into page */
+    for ( ;level <= 3; level++ ) {
+        va >>= XEN_PT_LPAE_SHIFT;
+        va |= RECURSIVE_VA;
+    }
+
+    /* Mask out any offset, in case caller is asking about a misalligned va */
+    va &= ~0x7;
+    return (lpae_t *)va;
+}
+
+/*
+ * Zero out the table at level when walking to virt
+ * Do this through the recursive mapping, in case we have already
+ * removed part of the directmap and can't walk to that entry
+ */
+static void clear_pte_directly(int level, void *virt)
+{
+    unsigned long va = (unsigned long)virt;
+    lpae_t empty = {.pt = {0x0}};
+    lpae_t *table;
+
+    spin_lock(&xen_pt_lock);
+
+    /* We're assuming we can safely remove an entry at `level` */
+    /* This depends on va not living in a superpage */
+    BUG_ON(level > 1 && !va_to_table(1, va)->pt.table);
+    BUG_ON(level > 2 && !va_to_table(2, va)->pt.table);
+
+    table = va_to_table(level, va);
+    write_pte(table, empty);
+    clean_dcache(*table);
+    flush_xen_tlb_range_va((vaddr_t)table, sizeof(*table));
+
+    spin_unlock(&xen_pt_lock);
+}
+
+static void remove_recursive_mapping(void)
+{
+    clear_pte_directly(0, (void *)RECURSIVE_VA);
+}
+
+static int modify_virt_mapping(void *virt, int nr_pages, int flags)
+{
+    unsigned long va = (unsigned long)virt;
+    return modify_xen_mappings(va, va + (PAGE_SIZE * nr_pages), flags);
+}
+
+static int destroy_virt_mapping(void *virt, int nr_pages)
+{
+    return modify_virt_mapping(virt, nr_pages, 0);
+}
+
+static int modify_directmap_mapping(unsigned long mfn, unsigned long nr_mfns, int flags)
+{
+    if ( mfn & pfn_hole_mask )
+    {
+        printk("** Skipping mfn 0x%lx because it lives in the pfn hole **\n", mfn);
+        return 0;
+    }
+
+    return modify_virt_mapping(__mfn_to_virt(mfn), nr_mfns, flags);
+}
+
+int split_directmap_mapping(unsigned long mfn, unsigned long nr_mfns)
+{
+    return modify_directmap_mapping(mfn, nr_mfns, PAGE_HYPERVISOR);
+}
+
+int destroy_directmap_mapping(unsigned long mfn, unsigned long nr_mfns)
+{
+    return modify_directmap_mapping(mfn, nr_mfns, 0);
+}
+
+void destroy_directmap_table(unsigned long mfn)
+{
+    BUG_ON(mfn & pfn_hole_mask);
+    clear_pte_directly(3, __mfn_to_virt(mfn));
+}
+
+static void unmap_xen_root_tables(void)
+{
+    destroy_virt_mapping(xen_xenmap, 1);
+    destroy_virt_mapping(xen_fixmap, 1);
+    destroy_virt_mapping(xen_second, 1);
+#if defined(CONFIG_ARM_64)
+    destroy_virt_mapping(xen_first, 1);
+    destroy_virt_mapping(xen_pgtable, 1);
+#endif
+}
+
+static void walk_hyp_tables(int flags)
+{
+    uint64_t httbr = READ_SYSREG64(TTBR0_EL2);
+    do_walk_tables(httbr, HYP_PT_ROOT_LEVEL, 1, flags);
+}
+
+static void walk_guest_tables(struct domain *d, int flags)
+{
+    uint64_t vttbr = d->arch.p2m.vttbr;
+    do_walk_tables(vttbr, P2M_ROOT_LEVEL, 1<<P2M_ROOT_ORDER, flags);
+}
+
+
+void vmf_unmap_guest(struct domain *d)
+{
+    /* Remove all of directmap mappings to guest */
+    walk_guest_tables(d, WALK_HIDE_GUEST_MAPPING);
+
+    /* Remove all mappings to guest second stage tables */
+    walk_guest_tables(d, WALK_HIDE_GUEST_TABLE);
+}
+
+void vmf_lock_xen_pgtables(void)
+{
+    /* Remove all of the static allocated root tables */
+    unmap_xen_root_tables();
+
+    /*
+     * Remove all tables from directmap
+     * Becuase we can't use the directmap to walk tables while we are removing
+     * the directmap, add a recursive pointer and use that to erase pte's
+     */
+    insert_recursive_mapping();
+    walk_hyp_tables(WALK_SPLIT_DIRECTMAP_TABLE);
+    walk_hyp_tables(WALK_HIDE_DIRECTMAP_TABLE);
+    remove_recursive_mapping();
+}
+
+void vmf_dump_xen_info()
+{
+    printk("Dump reg info...\n");
+    printk("current httbr0 is 0x%lx\n", READ_SYSREG64(TTBR0_EL2));
+    printk("current vttbr is 0x%lx\n", READ_SYSREG64(VTTBR_EL2));
+    printk("current ttbr0 is 0x%lx\n", READ_SYSREG64(TTBR0_EL1));
+    printk("current ttbr1 is 0x%lx\n", READ_SYSREG64(TTBR1_EL1));
+    printk("\n");
+
+    printk("Dump xen table info...\n");
+#if defined(CONFIG_ARM_64)
+    printk("xen_pgtable: 0x%"PRIvaddr"\n", (vaddr_t)xen_pgtable);
+    printk("xen_first: 0x%"PRIvaddr"\n", (vaddr_t)xen_first);
+#endif
+    printk("xen_second: 0x%"PRIvaddr"\n", (vaddr_t)xen_second);
+    printk("xen_xenmap: 0x%"PRIvaddr"\n", (vaddr_t)xen_xenmap);
+    printk("xen_fixmap: 0x%"PRIvaddr"\n", (vaddr_t)xen_fixmap);
+}
+
+void vmf_dump_domain_info(struct domain *d)
+{
+    uint64_t vttbr = d->arch.p2m.vttbr;
+    uint64_t httbr = READ_SYSREG64(TTBR0_EL2);
+
+    printk("Dump domain info...\n");
+    printk("guest mfn = 0x%lx\n", paddr_to_pfn(vttbr & PADDR_MASK));
+    printk("xen mfn = 0x%lx\n", paddr_to_pfn(httbr & PADDR_MASK));
+}
+
+void vmf_dump_xen_tables()
+{
+    walk_hyp_tables(WALK_DUMP_MAPPINGS | WALK_DUMP_ENTRIES);
+}
+
+void vmf_dump_domain_tables(struct domain *d)
+{
+    walk_guest_tables(d, WALK_DUMP_MAPPINGS | WALK_DUMP_ENTRIES);
+}
+
 /* Release all __init and __initdata ranges to be reused */
 void free_init_memory(void)
 {
diff --git a/xen/common/Kconfig b/xen/common/Kconfig
index 3bf92b8..c087371 100644
--- a/xen/common/Kconfig
+++ b/xen/common/Kconfig
@@ -94,6 +94,8 @@ config STATIC_MEMORY
 
 config VMF
 	bool "Virtual Memory Fuse Support"
+	depends on ARM_64
+	default y
 
 menu "Speculative hardening"
 
-- 
2.7.4