[PATCH 04/12] Nested Virtualization: core

[PATCH 04/12] Nested Virtualization: core

[Christoph Egger]

[-- Attachment #1: Type: text/plain, Size: 264 bytes --]


-- 
---to satisfy European Law for business letters:
Advanced Micro Devices GmbH
Einsteinring 24, 85609 Dornach b. Muenchen
Geschaeftsfuehrer: Alberto Bozzo, Andrew Bowd
Sitz: Dornach, Gemeinde Aschheim, Landkreis Muenchen
Registergericht Muenchen, HRB Nr. 43632

[-- Attachment #2: xen_nh04_core.diff --]
[-- Type: text/x-diff, Size: 8289 bytes --]

# HG changeset patch
# User cegger
# Date 1292839432 -3600
Nested Virtualization core implementation

Signed-off-by: Christoph Egger <Christoph.Egger@amd.com>

diff -r e43ab6fb0ee2 -r a9465de5a794 xen/arch/x86/hvm/Makefile
--- a/xen/arch/x86/hvm/Makefile
+++ b/xen/arch/x86/hvm/Makefile
@@ -10,6 +10,7 @@ obj-y += intercept.o
 obj-y += io.o
 obj-y += irq.o
 obj-y += mtrr.o
+obj-y += nestedhvm.o
 obj-y += pmtimer.o
 obj-y += quirks.o
 obj-y += rtc.o
diff -r e43ab6fb0ee2 -r a9465de5a794 xen/arch/x86/hvm/nestedhvm.c
--- /dev/null
+++ b/xen/arch/x86/hvm/nestedhvm.c
@@ -0,0 +1,198 @@
+/*
+ * Nested HVM
+ * Copyright (c) 2010, Advanced Micro Devices, Inc.
+ * Author: Christoph Egger <Christoph.Egger@amd.com>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program; if not, write to the Free Software Foundation, Inc., 59 Temple
+ * Place - Suite 330, Boston, MA 02111-1307 USA.
+ */
+
+#include <asm/msr.h>
+#include <asm/hvm/support.h>	/* for HVM_DELIVER_NO_ERROR_CODE */
+#include <asm/hvm/hvm.h>
+#include <asm/hvm/nestedhvm.h>
+#include <asm/event.h>  /* for local_event_delivery_(en|dis)able */
+#include <asm/paging.h> /* for paging_mode_hap() */
+
+
+/* Nested HVM on/off per domain */
+bool_t
+nestedhvm_enabled(struct domain *d)
+{
+    bool_t enabled;
+
+    enabled = !!(d->arch.hvm_domain.params[HVM_PARAM_NESTEDHVM]);
+    /* sanity check */
+    BUG_ON(enabled && !is_hvm_domain(d));
+
+    if (!is_hvm_domain(d))
+        return 0;
+
+    return enabled;
+}
+
+/* Nested VCPU */
+bool_t
+nestedhvm_vcpu_in_guestmode(struct vcpu *v)
+{
+    return vcpu_nestedhvm(v).nv_guestmode;
+}
+
+void
+nestedhvm_vcpu_reset(struct vcpu *v)
+{
+    struct nestedvcpu *nv = &vcpu_nestedhvm(v);
+
+    if (nv->nv_vmcx)
+        hvm_unmap_guest_frame(nv->nv_vmcx);
+    nv->nv_vmcx = NULL;
+    nv->nv_vmcxaddr = VMCX_EADDR;
+    nv->nv_flushp2m = 0;
+    nv->nv_p2m = NULL;
+
+    nhvm_vcpu_reset(v);
+
+    /* vcpu is in host mode */
+    nestedhvm_vcpu_exit_guestmode(v);
+}
+
+int
+nestedhvm_vcpu_initialise(struct vcpu *v)
+{
+    int rc;
+    struct nestedvcpu *nv = &vcpu_nestedhvm(v);
+
+    if (!nestedhvm_enabled(v->domain))
+        return 0;
+
+    memset(nv, 0x0, sizeof(struct nestedvcpu));
+
+    /* initialise hostsave, for example */
+    rc = nhvm_vcpu_initialise(v); 
+    if (rc) {
+        nhvm_vcpu_destroy(v);
+        return rc;
+    }
+
+    nestedhvm_vcpu_reset(v);
+    return 0;
+}
+
+int
+nestedhvm_vcpu_destroy(struct vcpu *v)
+{
+    if (!nestedhvm_enabled(v->domain))
+        return 0;
+
+    return nhvm_vcpu_destroy(v);
+}
+
+void
+nestedhvm_vcpu_enter_guestmode(struct vcpu *v)
+{
+    vcpu_nestedhvm(v).nv_guestmode = 1;
+}
+
+void
+nestedhvm_vcpu_exit_guestmode(struct vcpu *v)
+{
+    vcpu_nestedhvm(v).nv_guestmode = 0;
+}
+
+/* Common shadow IO Permission bitmap */
+
+struct shadow_iomap {
+    /* same format and size as hvm_io_bitmap */
+    unsigned long iomap[3*PAGE_SIZE/BYTES_PER_LONG];
+    int refcnt;
+};
+
+/* There four global patterns of io bitmap each guest can
+ * choose one of them depending on interception of io port 0x80 and/or
+ * 0xED (shown in table below). Each shadow iomap pattern is
+ * implemented as a singleton to minimize memory consumption while
+ * providing a provider/consumer interface to the users.
+ * The users are in SVM/VMX specific code.
+ *
+ * bitmap        port 0x80  port 0xed
+ * hvm_io_bitmap cleared    cleared
+ * iomap[0]      cleared    set
+ * iomap[1]      set        cleared
+ * iomap[2]      set        set
+ */
+static struct shadow_iomap *nhvm_io_bitmap[3];
+
+unsigned long *
+nestedhvm_vcpu_iomap_get(bool_t port_80, bool_t port_ed)
+{
+    int i;
+    extern int hvm_port80_allowed;
+
+    if (!hvm_port80_allowed)
+        port_80 = 1;
+
+    if (port_80 == 0) {
+        if (port_ed == 0)
+            return hvm_io_bitmap;
+        i = 0;
+    } else {
+        if (port_ed == 0)
+            i = 1;
+        else
+            i = 2;
+    }
+
+    if (nhvm_io_bitmap[i] == NULL) {
+        nhvm_io_bitmap[i] =
+            _xmalloc(sizeof(struct shadow_iomap), PAGE_SIZE);
+        nhvm_io_bitmap[i]->refcnt = 0;
+        /* set all bits */
+        memset(nhvm_io_bitmap[i]->iomap, ~0, sizeof(nhvm_io_bitmap[i]->iomap));
+        switch (i) {
+        case 0:
+            __clear_bit(0x80, nhvm_io_bitmap[i]->iomap);
+            break;
+        case 1:
+            __clear_bit(0xed, nhvm_io_bitmap[i]->iomap);
+            break;
+        case 2:
+            break;
+        }
+    }
+
+    nhvm_io_bitmap[i]->refcnt++;
+    return nhvm_io_bitmap[i]->iomap;
+}
+
+void
+nestedhvm_vcpu_iomap_put(unsigned long *iomap)
+{
+    int i;
+
+    if (iomap == hvm_io_bitmap)
+        return;
+
+    for (i = 0; i < 3; i++) {
+        struct shadow_iomap *map;
+        if (iomap != nhvm_io_bitmap[i]->iomap)
+            continue;
+        map = nhvm_io_bitmap[i];
+        map->refcnt--;
+        if (map->refcnt == 0) {
+            xfree(map);
+            nhvm_io_bitmap[i] = NULL;
+        }
+        return;
+    }
+    BUG();
+}
diff -r e43ab6fb0ee2 -r a9465de5a794 xen/include/asm-x86/hvm/nestedhvm.h
--- /dev/null
+++ b/xen/include/asm-x86/hvm/nestedhvm.h
@@ -0,0 +1,67 @@
+/*
+ * Nested HVM
+ * Copyright (c) 2010, Advanced Micro Devices, Inc.
+ * Author: Christoph Egger <Christoph.Egger@amd.com>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program; if not, write to the Free Software Foundation, Inc., 59 Temple
+ * Place - Suite 330, Boston, MA 02111-1307 USA.
+ */
+
+#ifndef _HVM_NESTEDHVM_H
+#define _HVM_NESTEDHVM_H
+
+#include <xen/types.h>         /* for uintNN_t */
+#include <xen/sched.h>         /* for struct vcpu, struct domain */
+#include <asm/hvm/vcpu.h>      /* for vcpu_nestedhvm */
+
+enum nestedhvm_vmexits {
+    NESTEDHVM_VMEXIT_ERROR = 0, /* inject VMEXIT w/ invalid VMCB */
+    NESTEDHVM_VMEXIT_FATALERROR = 1, /* crash first level guest */
+    NESTEDHVM_VMEXIT_HOST = 2,  /* exit handled on host level */
+    NESTEDHVM_VMEXIT_CONTINUE = 3, /* further handling */
+    NESTEDHVM_VMEXIT_INJECT = 4, /* inject VMEXIT */
+    NESTEDHVM_VMEXIT_DONE = 5, /* VMEXIT handled */
+};
+
+#define VMCX_EADDR    (~0ULL)
+
+/* Nested HVM on/off per domain */
+bool_t nestedhvm_enabled(struct domain *d);
+int nestedhvm_initialise(struct domain *d);
+
+/* Nested VCPU */
+int nestedhvm_vcpu_initialise(struct vcpu *v);
+int nestedhvm_vcpu_destroy(struct vcpu *v);
+void nestedhvm_vcpu_reset(struct vcpu *v);
+bool_t nestedhvm_vcpu_in_guestmode(struct vcpu *v);
+void nestedhvm_vcpu_enter_guestmode(struct vcpu *v);
+void nestedhvm_vcpu_exit_guestmode(struct vcpu *v);
+
+/* Nested paging */
+#define NESTEDHVM_PAGEFAULT_DONE   0
+#define NESTEDHVM_PAGEFAULT_INJECT 1
+#define NESTEDHVM_PAGEFAULT_ERROR  2
+int nestedhvm_hap_nested_page_fault(struct vcpu *v, paddr_t L2_gpa);
+
+/* IO permission map */
+unsigned long *nestedhvm_vcpu_iomap_get(bool_t ioport_80, bool_t ioport_ed);
+void nestedhvm_vcpu_iomap_put(unsigned long *iomap);
+
+/* Misc */
+#define nestedhvm_paging_mode_hap(v) (!!nhvm_vmcx_hap_enabled(v))
+#define nestedhvm_vmswitch_in_progress(v)   \
+    (!!vcpu_nestedhvm((v)).nv_vmswitch_in_progress)
+#define nestedhvm_vmcx_flushtlb(d) \
+    flush_tlb_mask(&(d)->arch.hvm_domain.nh_dirty_cpumask)
+
+#endif /* _HVM_NESTEDHVM_H */

[-- Attachment #3: Type: text/plain, Size: 138 bytes --]

_______________________________________________
Xen-devel mailing list
Xen-devel@lists.xensource.com
http://lists.xensource.com/xen-devel

RE: [PATCH 04/12] Nested Virtualization: core

[Dong, Eddie]

Dong, Eddie wrote:
> # HG changeset patch
> # User cegger
> # Date 1292839432 -3600
> Nested Virtualization core implementation
> 
> Signed-off-by: Christoph Egger <Christoph.Egger@amd.com>
> 
> diff -r e43ab6fb0ee2 -r a9465de5a794 xen/arch/x86/hvm/Makefile
> --- a/xen/arch/x86/hvm/Makefile
> +++ b/xen/arch/x86/hvm/Makefile
> @@ -10,6 +10,7 @@ obj-y += intercept.o
>  obj-y += io.o
>  obj-y += irq.o
>  obj-y += mtrr.o
> +obj-y += nestedhvm.o
>  obj-y += pmtimer.o
>  obj-y += quirks.o
>  obj-y += rtc.o
> diff -r e43ab6fb0ee2 -r a9465de5a794 xen/arch/x86/hvm/nestedhvm.c
> --- /dev/null
> +++ b/xen/arch/x86/hvm/nestedhvm.c
> @@ -0,0 +1,198 @@
> +/*
> + * Nested HVM
> + * Copyright (c) 2010, Advanced Micro Devices, Inc.
> + * Author: Christoph Egger <Christoph.Egger@amd.com>
> + *
> + * This program is free software; you can redistribute it and/or
> modify it + * under the terms and conditions of the GNU General
> Public License, + * version 2, as published by the Free Software
> Foundation. + *
> + * This program is distributed in the hope it will be useful, but
> WITHOUT + * ANY WARRANTY; without even the implied warranty of
> MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
> General Public License for + * more details.
> + *
> + * You should have received a copy of the GNU General Public License
> along with + * this program; if not, write to the Free Software
> Foundation, Inc., 59 Temple + * Place - Suite 330, Boston, MA
> 02111-1307 USA. + */
> +
> +#include <asm/msr.h>
> +#include <asm/hvm/support.h>	/* for HVM_DELIVER_NO_ERROR_CODE */
> +#include <asm/hvm/hvm.h>
> +#include <asm/hvm/nestedhvm.h>
> +#include <asm/event.h>  /* for local_event_delivery_(en|dis)able */
> +#include <asm/paging.h> /* for paging_mode_hap() */
> +
> +
> +/* Nested HVM on/off per domain */
> +bool_t
> +nestedhvm_enabled(struct domain *d)
> +{
> +    bool_t enabled;
> +
> +    enabled = !!(d->arch.hvm_domain.params[HVM_PARAM_NESTEDHVM]);
> +    /* sanity check */
> +    BUG_ON(enabled && !is_hvm_domain(d));
> +
> +    if (!is_hvm_domain(d))
> +        return 0;
> +
> +    return enabled;
> +}
> +
> +/* Nested VCPU */
> +bool_t
> +nestedhvm_vcpu_in_guestmode(struct vcpu *v)
> +{
> +    return vcpu_nestedhvm(v).nv_guestmode;
> +}
> +
> +void
> +nestedhvm_vcpu_reset(struct vcpu *v)
> +{
> +    struct nestedvcpu *nv = &vcpu_nestedhvm(v);
> +
> +    if (nv->nv_vmcx)
> +        hvm_unmap_guest_frame(nv->nv_vmcx);
> +    nv->nv_vmcx = NULL;
> +    nv->nv_vmcxaddr = VMCX_EADDR;
> +    nv->nv_flushp2m = 0;
> +    nv->nv_p2m = NULL;
> +
> +    nhvm_vcpu_reset(v);
> +
> +    /* vcpu is in host mode */
> +    nestedhvm_vcpu_exit_guestmode(v);
> +}
> +
> +int
> +nestedhvm_vcpu_initialise(struct vcpu *v)
> +{
> +    int rc;
> +    struct nestedvcpu *nv = &vcpu_nestedhvm(v);
> +
> +    if (!nestedhvm_enabled(v->domain))
> +        return 0;
> +
> +    memset(nv, 0x0, sizeof(struct nestedvcpu));
> +
> +    /* initialise hostsave, for example */
> +    rc = nhvm_vcpu_initialise(v);
> +    if (rc) {
> +        nhvm_vcpu_destroy(v);
> +        return rc;
> +    }
> +
> +    nestedhvm_vcpu_reset(v);
> +    return 0;
> +}
> +
> +int
> +nestedhvm_vcpu_destroy(struct vcpu *v)
> +{
> +    if (!nestedhvm_enabled(v->domain))
> +        return 0;
> +
> +    return nhvm_vcpu_destroy(v);
> +}
> +
> +void
> +nestedhvm_vcpu_enter_guestmode(struct vcpu *v)
> +{
> +    vcpu_nestedhvm(v).nv_guestmode = 1;
> +}
> +
> +void
> +nestedhvm_vcpu_exit_guestmode(struct vcpu *v)
> +{
> +    vcpu_nestedhvm(v).nv_guestmode = 0;
> +}
> +
> +/* Common shadow IO Permission bitmap */
> +
> +struct shadow_iomap {
> +    /* same format and size as hvm_io_bitmap */
> +    unsigned long iomap[3*PAGE_SIZE/BYTES_PER_LONG];
> +    int refcnt;
> +};
> +
> +/* There four global patterns of io bitmap each guest can
> + * choose one of them depending on interception of io port 0x80
> and/or + * 0xED (shown in table below). Each shadow iomap pattern is
> + * implemented as a singleton to minimize memory consumption while
> + * providing a provider/consumer interface to the users.
> + * The users are in SVM/VMX specific code.
> + *
> + * bitmap        port 0x80  port 0xed
> + * hvm_io_bitmap cleared    cleared
> + * iomap[0]      cleared    set
> + * iomap[1]      set        cleared
> + * iomap[2]      set        set
> + */
> +static struct shadow_iomap *nhvm_io_bitmap[3];
> +
> +unsigned long *
> +nestedhvm_vcpu_iomap_get(bool_t port_80, bool_t port_ed)
> +{
> +    int i;
> +    extern int hvm_port80_allowed;
> +
> +    if (!hvm_port80_allowed)
> +        port_80 = 1;
> +
> +    if (port_80 == 0) {
> +        if (port_ed == 0)
> +            return hvm_io_bitmap;
> +        i = 0;
> +    } else {
> +        if (port_ed == 0)
> +            i = 1;
> +        else
> +            i = 2;
> +    }
> +
> +    if (nhvm_io_bitmap[i] == NULL) {
> +        nhvm_io_bitmap[i] =
> +            _xmalloc(sizeof(struct shadow_iomap), PAGE_SIZE);
> +        nhvm_io_bitmap[i]->refcnt = 0;
> +        /* set all bits */
> +        memset(nhvm_io_bitmap[i]->iomap, ~0,
> sizeof(nhvm_io_bitmap[i]->iomap)); +        switch (i) {
> +        case 0:
> +            __clear_bit(0x80, nhvm_io_bitmap[i]->iomap);
> +            break;
> +        case 1:
> +            __clear_bit(0xed, nhvm_io_bitmap[i]->iomap);
> +            break;
> +        case 2:
> +            break;
> +        }
> +    }
> +


This is overcomplicated. Static table should serve this much simple and efficient.

Re: [PATCH 04/12] Nested Virtualization: core

[Christoph Egger]

On Monday 27 December 2010 08:54:16 Dong, Eddie wrote:
> Dong, Eddie wrote:
> > # HG changeset patch
> > # User cegger
> > # Date 1292839432 -3600
> > Nested Virtualization core implementation
> >
> > Signed-off-by: Christoph Egger <Christoph.Egger@amd.com>
> >
> > diff -r e43ab6fb0ee2 -r a9465de5a794 xen/arch/x86/hvm/Makefile
> > --- a/xen/arch/x86/hvm/Makefile
> > +++ b/xen/arch/x86/hvm/Makefile
> > @@ -10,6 +10,7 @@ obj-y += intercept.o
> >  obj-y += io.o
> >  obj-y += irq.o
> >  obj-y += mtrr.o
> > +obj-y += nestedhvm.o
> >  obj-y += pmtimer.o
> >  obj-y += quirks.o
> >  obj-y += rtc.o
> > diff -r e43ab6fb0ee2 -r a9465de5a794 xen/arch/x86/hvm/nestedhvm.c
> > --- /dev/null
> > +++ b/xen/arch/x86/hvm/nestedhvm.c
> > @@ -0,0 +1,198 @@
> > +/*
> > + * Nested HVM
> > + * Copyright (c) 2010, Advanced Micro Devices, Inc.
> > + * Author: Christoph Egger <Christoph.Egger@amd.com>
> > + *
> > + * This program is free software; you can redistribute it and/or
> > modify it + * under the terms and conditions of the GNU General
> > Public License, + * version 2, as published by the Free Software
> > Foundation. + *
> > + * This program is distributed in the hope it will be useful, but
> > WITHOUT + * ANY WARRANTY; without even the implied warranty of
> > MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
> > General Public License for + * more details.
> > + *
> > + * You should have received a copy of the GNU General Public License
> > along with + * this program; if not, write to the Free Software
> > Foundation, Inc., 59 Temple + * Place - Suite 330, Boston, MA
> > 02111-1307 USA. + */
> > +
> > +#include <asm/msr.h>
> > +#include <asm/hvm/support.h>	/* for HVM_DELIVER_NO_ERROR_CODE */
> > +#include <asm/hvm/hvm.h>
> > +#include <asm/hvm/nestedhvm.h>
> > +#include <asm/event.h>  /* for local_event_delivery_(en|dis)able */
> > +#include <asm/paging.h> /* for paging_mode_hap() */
> > +
> > +
> > +/* Nested HVM on/off per domain */
> > +bool_t
> > +nestedhvm_enabled(struct domain *d)
> > +{
> > +    bool_t enabled;
> > +
> > +    enabled = !!(d->arch.hvm_domain.params[HVM_PARAM_NESTEDHVM]);
> > +    /* sanity check */
> > +    BUG_ON(enabled && !is_hvm_domain(d));
> > +
> > +    if (!is_hvm_domain(d))
> > +        return 0;
> > +
> > +    return enabled;
> > +}
> > +
> > +/* Nested VCPU */
> > +bool_t
> > +nestedhvm_vcpu_in_guestmode(struct vcpu *v)
> > +{
> > +    return vcpu_nestedhvm(v).nv_guestmode;
> > +}
> > +
> > +void
> > +nestedhvm_vcpu_reset(struct vcpu *v)
> > +{
> > +    struct nestedvcpu *nv = &vcpu_nestedhvm(v);
> > +
> > +    if (nv->nv_vmcx)
> > +        hvm_unmap_guest_frame(nv->nv_vmcx);
> > +    nv->nv_vmcx = NULL;
> > +    nv->nv_vmcxaddr = VMCX_EADDR;
> > +    nv->nv_flushp2m = 0;
> > +    nv->nv_p2m = NULL;
> > +
> > +    nhvm_vcpu_reset(v);
> > +
> > +    /* vcpu is in host mode */
> > +    nestedhvm_vcpu_exit_guestmode(v);
> > +}
> > +
> > +int
> > +nestedhvm_vcpu_initialise(struct vcpu *v)
> > +{
> > +    int rc;
> > +    struct nestedvcpu *nv = &vcpu_nestedhvm(v);
> > +
> > +    if (!nestedhvm_enabled(v->domain))
> > +        return 0;
> > +
> > +    memset(nv, 0x0, sizeof(struct nestedvcpu));
> > +
> > +    /* initialise hostsave, for example */
> > +    rc = nhvm_vcpu_initialise(v);
> > +    if (rc) {
> > +        nhvm_vcpu_destroy(v);
> > +        return rc;
> > +    }
> > +
> > +    nestedhvm_vcpu_reset(v);
> > +    return 0;
> > +}
> > +
> > +int
> > +nestedhvm_vcpu_destroy(struct vcpu *v)
> > +{
> > +    if (!nestedhvm_enabled(v->domain))
> > +        return 0;
> > +
> > +    return nhvm_vcpu_destroy(v);
> > +}
> > +
> > +void
> > +nestedhvm_vcpu_enter_guestmode(struct vcpu *v)
> > +{
> > +    vcpu_nestedhvm(v).nv_guestmode = 1;
> > +}
> > +
> > +void
> > +nestedhvm_vcpu_exit_guestmode(struct vcpu *v)
> > +{
> > +    vcpu_nestedhvm(v).nv_guestmode = 0;
> > +}
> > +
> > +/* Common shadow IO Permission bitmap */
> > +
> > +struct shadow_iomap {
> > +    /* same format and size as hvm_io_bitmap */
> > +    unsigned long iomap[3*PAGE_SIZE/BYTES_PER_LONG];
> > +    int refcnt;
> > +};
> > +
> > +/* There four global patterns of io bitmap each guest can
> > + * choose one of them depending on interception of io port 0x80
> > and/or + * 0xED (shown in table below). Each shadow iomap pattern is
> > + * implemented as a singleton to minimize memory consumption while
> > + * providing a provider/consumer interface to the users.
> > + * The users are in SVM/VMX specific code.
> > + *
> > + * bitmap        port 0x80  port 0xed
> > + * hvm_io_bitmap cleared    cleared
> > + * iomap[0]      cleared    set
> > + * iomap[1]      set        cleared
> > + * iomap[2]      set        set
> > + */
> > +static struct shadow_iomap *nhvm_io_bitmap[3];
> > +
> > +unsigned long *
> > +nestedhvm_vcpu_iomap_get(bool_t port_80, bool_t port_ed)
> > +{
> > +    int i;
> > +    extern int hvm_port80_allowed;
> > +
> > +    if (!hvm_port80_allowed)
> > +        port_80 = 1;
> > +
> > +    if (port_80 == 0) {
> > +        if (port_ed == 0)
> > +            return hvm_io_bitmap;
> > +        i = 0;
> > +    } else {
> > +        if (port_ed == 0)
> > +            i = 1;
> > +        else
> > +            i = 2;
> > +    }
> > +
> > +    if (nhvm_io_bitmap[i] == NULL) {
> > +        nhvm_io_bitmap[i] =
> > +            _xmalloc(sizeof(struct shadow_iomap), PAGE_SIZE);
> > +        nhvm_io_bitmap[i]->refcnt = 0;
> > +        /* set all bits */
> > +        memset(nhvm_io_bitmap[i]->iomap, ~0,
> > sizeof(nhvm_io_bitmap[i]->iomap)); +        switch (i) {
> > +        case 0:
> > +            __clear_bit(0x80, nhvm_io_bitmap[i]->iomap);
> > +            break;
> > +        case 1:
> > +            __clear_bit(0xed, nhvm_io_bitmap[i]->iomap);
> > +            break;
> > +        case 2:
> > +            break;
> > +        }
> > +    }
> > +
>
> This is overcomplicated. Static table should serve this much simple and
> efficient.

The logic to select the right static table will be still needed. I am not sure
if removing the _xmalloc() call simplifies this part a lot.

I appreciate opinions from other people on this.

Christoph



-- 
---to satisfy European Law for business letters:
Advanced Micro Devices GmbH
Einsteinring 24, 85609 Dornach b. Muenchen
Geschaeftsfuehrer: Alberto Bozzo, Andrew Bowd
Sitz: Dornach, Gemeinde Aschheim, Landkreis Muenchen
Registergericht Muenchen, HRB Nr. 43632

RE: [PATCH 04/12] Nested Virtualization: core

[Dong, Eddie]

>> This is overcomplicated. Static table should serve this much simple
>> and efficient.
> 
> The logic to select the right static table will be still needed. I am
> not sure if removing the _xmalloc() call simplifies this part a lot.

It will be much simple. You don't need the nestedhvm_vcpu_iomap_get/put api, nor the refcnt.

The thing more important is policy: If you are in favoring of memory size or simplicity. If it is for memory size, then you should only allocate 2 io_bitmap pages for VMX.


> 
> I appreciate opinions from other people on this.
> 

Besides, ideally we should implement per guest io bitmap page, by reusing L1 guest io_bitmap + write protection of the page table. At least for both Xen & KVM, the io bitmap is not modified at runtime once it is initialized. The readibility can be improved & the memory page can be saved. We only need 2 bits per L1 guest. 

But if we want simplicity, I am ok too, however the current patch doesn't fit for either of the goal.

thx, eddie

Re: [PATCH 04/12] Nested Virtualization: core

[Christoph Egger]

On Thursday 06 January 2011 18:33:56 Dong, Eddie wrote:
> >> This is overcomplicated. Static table should serve this much simple
> >> and efficient.
> >
> > The logic to select the right static table will be still needed. I am
> > not sure if removing the _xmalloc() call simplifies this part a lot.
>
> It will be much simple. You don't need the nestedhvm_vcpu_iomap_get/put
> api, nor the refcnt.

It is intended that the api behaves like a pool from the caller site
while it is implemented as a singleton.
The refcnt (or should I call it usagecnt) is needed by the singleton design 
pattern.

When I remove the refcnt then I have to implement the api as a real pool
which will result in allocating an io bitmap for each vcpu for each l1 guest
at runtime.

>
> The thing more important is policy: If you are in favoring of memory size
> or simplicity. If it is for memory size, then you should only allocate 2
> io_bitmap pages for VMX.
>
> > I appreciate opinions from other people on this.
>
> Besides, ideally we should implement per guest io bitmap page, by reusing
> L1 guest io_bitmap + write protection of the page table.

That will work fine with 4kb pages but I guess it won't be very
efficient with 2MB and 1GB pages. Most time will be spent with emulating write 
accesses to the address ranges outside of the io bitmaps with large pages.

> At least for both Xen & KVM, the io bitmap is not modified at runtime once
> it is initialized. 

Yep, that's why we only need to deal with four possible patterns of
shadow io bitmaps in Xen. We can't assume the l1 guest is not modifying it.

> The readibility can be improved & the memory page can be saved. We only
> need 2 bits per L1 guest.
>
> But if we want simplicity, I am ok too, however the current patch doesn't
> fit for either of the goal.

hmm... I think, I need to move that part of common logic into SVM to reach
consensus... pity.

Christoph


-- 
---to satisfy European Law for business letters:
Advanced Micro Devices GmbH
Einsteinring 24, 85609 Dornach b. Muenchen
Geschaeftsfuehrer: Alberto Bozzo, Andrew Bowd
Sitz: Dornach, Gemeinde Aschheim, Landkreis Muenchen
Registergericht Muenchen, HRB Nr. 43632

Re: [PATCH 04/12] Nested Virtualization: core

[Tim Deegan]

At 15:58 +0000 on 03 Jan (1294070335), Christoph Egger wrote:
> On Monday 27 December 2010 08:54:16 Dong, Eddie wrote:
> > This is overcomplicated. Static table should serve this much simple and
> > efficient.
> 
> The logic to select the right static table will be still needed. I am
> not sure if removing the _xmalloc() call simplifies this part a lot.
> 
> I appreciate opinions from other people on this.

I think that you should allocate the three static bitmaps once at boot
time and not bother refcounting them.  It's only 36KiB of overhead for
the entire host.

Otherwise you'd have to decide what to do if _xmalloc() returned NULL. 

Cheers,

Tim.

-- 
Tim Deegan <Tim.Deegan@citrix.com>
Principal Software Engineer, Xen Platform Team
Citrix Systems UK Ltd.  (Company #02937203, SL9 0BG)

Re: [PATCH 04/12] Nested Virtualization: core

[Christoph Egger]

On Friday 07 January 2011 15:12:51 Tim Deegan wrote:
> At 15:58 +0000 on 03 Jan (1294070335), Christoph Egger wrote:
> > On Monday 27 December 2010 08:54:16 Dong, Eddie wrote:
> > > This is overcomplicated. Static table should serve this much simple and
> > > efficient.
> >
> > The logic to select the right static table will be still needed. I am
> > not sure if removing the _xmalloc() call simplifies this part a lot.
> >
> > I appreciate opinions from other people on this.
>
> I think that you should allocate the three static bitmaps once at boot
> time and not bother refcounting them.  It's only 36KiB of overhead for
> the entire host.
>
> Otherwise you'd have to decide what to do if _xmalloc() returned NULL.

I did not want to waste memory in the non-nested-virtualization case
but ok, I will go that way then.

Christoph


-- 
---to satisfy European Law for business letters:
Advanced Micro Devices GmbH
Einsteinring 24, 85609 Dornach b. Muenchen
Geschaeftsfuehrer: Alberto Bozzo, Andrew Bowd
Sitz: Dornach, Gemeinde Aschheim, Landkreis Muenchen
Registergericht Muenchen, HRB Nr. 43632

Re: [PATCH 04/12] Nested Virtualization: core

[Christoph Egger]

On Thursday 06 January 2011 18:33:56 Dong, Eddie wrote:
> >> This is overcomplicated. Static table should serve this much simple
> >> and efficient.
> >
> > The logic to select the right static table will be still needed. I am
> > not sure if removing the _xmalloc() call simplifies this part a lot.
>
> It will be much simple. You don't need the nestedhvm_vcpu_iomap_get/put
> api, nor the refcnt.
>
> The thing more important is policy: If you are in favoring of memory size
> or simplicity. If it is for memory size, then you should only allocate 2
> io_bitmap pages for VMX.
>
> > I appreciate opinions from other people on this.
>
> Besides, ideally we should implement per guest io bitmap page, by reusing
> L1 guest io_bitmap + write protection of the page table. At least for both
> Xen & KVM, the io bitmap is not modified at runtime once it is initialized.
> The readibility can be improved & the memory page can be saved. We only
> need 2 bits per L1 guest.
>
> But if we want simplicity, I am ok too, however the current patch doesn't
> fit for either of the goal.

I think, I need to excuse. Now that I did the change in my local tree
I have to mention that did not realize that I can remove the put function
completely. This in fact is a simplification.

Christoph


-- 
---to satisfy European Law for business letters:
Advanced Micro Devices GmbH
Einsteinring 24, 85609 Dornach b. Muenchen
Geschaeftsfuehrer: Alberto Bozzo, Andrew Bowd
Sitz: Dornach, Gemeinde Aschheim, Landkreis Muenchen
Registergericht Muenchen, HRB Nr. 43632

RE: [PATCH 04/12] Nested Virtualization: core

[Dong, Eddie]

Glad to see that you eventually take our proposal. Simple is beautiful, that is my truth.

BTW, comments to your previous question in case you are still interested in them.

>> 
>> It will be much simple. You don't need the
>> nestedhvm_vcpu_iomap_get/put api, nor the refcnt.
> 
> It is intended that the api behaves like a pool from the caller site
> while it is implemented as a singleton.
> The refcnt (or should I call it usagecnt) is needed by the singleton
> design pattern.
> 
> When I remove the refcnt then I have to implement the api as a real
> pool which will result in allocating an io bitmap for each vcpu for
> each l1 guest at runtime.

You don't need the apis w/ pre-allocated pages.

> 
>> 
>> The thing more important is policy: If you are in favoring of memory
>> size or simplicity. If it is for memory size, then you should only
>> allocate 2 io_bitmap pages for VMX. 
>> 
>>> I appreciate opinions from other people on this.
>> 
>> Besides, ideally we should implement per guest io bitmap page, by
>> reusing L1 guest io_bitmap + write protection of the page table.
> 
> That will work fine with 4kb pages but I guess it won't be very
> efficient with 2MB and 1GB pages. Most time will be spent with
> emulating write accesses to the address ranges outside of the io
> bitmaps with large pages. 

That is not true. Even the guest got contuguous machine large pages, but the host should still be able to handle mixed page size. a typical usage for this is that host may not always be able to get contiguous large page such as after migration.

In this case, of course we only protect 2* 4K pages. That doesn't introduce any additional issues.

> 
>> At least for both Xen & KVM, the io bitmap is not modified at
>> runtime once it is initialized.
> 
> Yep, that's why we only need to deal with four possible patterns of
> shadow io bitmaps in Xen. We can't assume the l1 guest is not
> modifying it. 

While, for performance, we can assume. For correctness, we need to handle the rare situation. That is also why we need to write-protect the bitmap pages, and create another seperate shadow io bitmap pages if the guest want to do that for correctness. However in dominant case, the host can reuse guest io butmap pages for host usage with 2 bits indicating original guest state.

Four pattern really is not related with the topic.

> 
>> The readibility can be improved & the memory page can be saved. We
>> only need 2 bits per L1 guest. 
>> 
>> But if we want simplicity, I am ok too, however the current patch
>> doesn't fit for either of the goal.
> 
> hmm... I think, I need to move that part of common logic into SVM to
> reach consensus... pity.
> 

My idea was given twice before you publically post it :(

Thx, Eddie

RE: [PATCH 04/12] Nested Virtualization: core

[Dong, Eddie]

Acked by eddie.dong@intel.com

-----Original Message-----
From: xen-devel-bounces@lists.xensource.com [mailto:xen-devel-bounces@lists.xensource.com] On Behalf Of Christoph Egger
Sent: Wednesday, March 09, 2011 10:23 PM
To: xen-devel@lists.xensource.com
Subject: [Xen-devel] [PATCH 04/12] Nested Virtualization: core


--
---to satisfy European Law for business letters:
Advanced Micro Devices GmbH
Einsteinring 24, 85609 Dornach b. Muenchen
Geschaeftsfuehrer: Alberto Bozzo, Andrew Bowd
Sitz: Dornach, Gemeinde Aschheim, Landkreis Muenchen Registergericht Muenchen, HRB Nr. 43632

[PATCH 04/12] Nested Virtualization: core

[Christoph Egger]

[-- Attachment #1: Type: text/plain, Size: 264 bytes --]


-- 
---to satisfy European Law for business letters:
Advanced Micro Devices GmbH
Einsteinring 24, 85609 Dornach b. Muenchen
Geschaeftsfuehrer: Alberto Bozzo, Andrew Bowd
Sitz: Dornach, Gemeinde Aschheim, Landkreis Muenchen
Registergericht Muenchen, HRB Nr. 43632

[-- Attachment #2: xen_nh04_core.diff --]
[-- Type: text/x-diff, Size: 8992 bytes --]

# HG changeset patch
# User cegger
# Date 1298892106 -3600
Nested Virtualization core implementation

Signed-off-by: Christoph Egger <Christoph.Egger@amd.com>

diff -r 216ffee07118 -r 4c51c9128e19 xen/arch/x86/hvm/Makefile
--- a/xen/arch/x86/hvm/Makefile
+++ b/xen/arch/x86/hvm/Makefile
@@ -10,6 +10,7 @@ obj-y += intercept.o
 obj-y += io.o
 obj-y += irq.o
 obj-y += mtrr.o
+obj-y += nestedhvm.o
 obj-y += pmtimer.o
 obj-y += quirks.o
 obj-y += rtc.o
diff -r 216ffee07118 -r 4c51c9128e19 xen/arch/x86/hvm/nestedhvm.c
--- /dev/null
+++ b/xen/arch/x86/hvm/nestedhvm.c
@@ -0,0 +1,177 @@
+/*
+ * Nested HVM
+ * Copyright (c) 2011, Advanced Micro Devices, Inc.
+ * Author: Christoph Egger <Christoph.Egger@amd.com>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program; if not, write to the Free Software Foundation, Inc., 59 Temple
+ * Place - Suite 330, Boston, MA 02111-1307 USA.
+ */
+
+#include <asm/msr.h>
+#include <asm/hvm/support.h>	/* for HVM_DELIVER_NO_ERROR_CODE */
+#include <asm/hvm/hvm.h>
+#include <asm/hvm/nestedhvm.h>
+#include <asm/event.h>  /* for local_event_delivery_(en|dis)able */
+#include <asm/paging.h> /* for paging_mode_hap() */
+
+
+/* Nested HVM on/off per domain */
+bool_t
+nestedhvm_enabled(struct domain *d)
+{
+    bool_t enabled;
+
+    enabled = !!(d->arch.hvm_domain.params[HVM_PARAM_NESTEDHVM]);
+    /* sanity check */
+    BUG_ON(enabled && !is_hvm_domain(d));
+
+    if (!is_hvm_domain(d))
+        return 0;
+
+    return enabled;
+}
+
+/* Nested VCPU */
+bool_t
+nestedhvm_vcpu_in_guestmode(struct vcpu *v)
+{
+    return vcpu_nestedhvm(v).nv_guestmode;
+}
+
+void
+nestedhvm_vcpu_reset(struct vcpu *v)
+{
+    struct nestedvcpu *nv = &vcpu_nestedhvm(v);
+
+    nv->nv_vmentry_pending = 0;
+    nv->nv_vmexit_pending = 0;
+    nv->nv_vmswitch_in_progress = 0;
+    nv->nv_ioport80 = 0;
+    nv->nv_ioportED = 0;
+
+    if (nv->nv_vvmcx)
+        hvm_unmap_guest_frame(nv->nv_vvmcx);
+    nv->nv_vvmcx = NULL;
+    nv->nv_vvmcxaddr = VMCX_EADDR;
+    nv->nv_flushp2m = 0;
+    nv->nv_p2m = NULL;
+
+    nhvm_vcpu_reset(v);
+
+    /* vcpu is in host mode */
+    nestedhvm_vcpu_exit_guestmode(v);
+}
+
+int
+nestedhvm_vcpu_initialise(struct vcpu *v)
+{
+    int rc;
+
+    rc = nhvm_vcpu_initialise(v); 
+    if (rc) {
+        nhvm_vcpu_destroy(v);
+        return rc;
+    }
+
+    nestedhvm_vcpu_reset(v);
+    return 0;
+}
+
+int
+nestedhvm_vcpu_destroy(struct vcpu *v)
+{
+    if (!nestedhvm_enabled(v->domain))
+        return 0;
+
+    return nhvm_vcpu_destroy(v);
+}
+
+/* Common shadow IO Permission bitmap */
+
+/* There four global patterns of io bitmap each guest can
+ * choose depending on interception of io port 0x80 and/or
+ * 0xED (shown in table below).
+ * The users of the bitmap patterns are in SVM/VMX specific code.
+ *
+ * bitmap        port 0x80  port 0xed
+ * hvm_io_bitmap cleared    cleared
+ * iomap[0]      cleared    set
+ * iomap[1]      set        cleared
+ * iomap[2]      set        set
+ */
+
+/* same format and size as hvm_io_bitmap */
+#define IOBITMAP_SIZE		3*PAGE_SIZE/BYTES_PER_LONG
+/* same format as hvm_io_bitmap */
+#define IOBITMAP_VMX_SIZE	2*PAGE_SIZE/BYTES_PER_LONG
+
+static unsigned long *shadow_io_bitmap[3];
+
+void
+nestedhvm_setup(void)
+{
+    /* shadow_io_bitmaps can't be declared static because
+     *   they must fulfill hw requirements (page aligned section)
+     *   and doing so triggers the ASSERT(va >= XEN_VIRT_START)
+     *   in __virt_to_maddr()
+     *
+     * So as a compromise pre-allocate them when xen boots.
+     * This function must be called from within start_xen() when
+     * it is valid to use _xmalloc()
+     */
+
+    /* shadow I/O permission bitmaps */
+    if (boot_cpu_data.x86_vendor == X86_VENDOR_INTEL) {
+        /* Same format as hvm_io_bitmap */
+        shadow_io_bitmap[0] = _xmalloc(IOBITMAP_VMX_SIZE, PAGE_SIZE);
+        shadow_io_bitmap[1] = _xmalloc(IOBITMAP_VMX_SIZE, PAGE_SIZE);
+        shadow_io_bitmap[2] = _xmalloc(IOBITMAP_VMX_SIZE, PAGE_SIZE);
+        memset(shadow_io_bitmap[0], ~0U, IOBITMAP_VMX_SIZE);
+        memset(shadow_io_bitmap[1], ~0U, IOBITMAP_VMX_SIZE);
+        memset(shadow_io_bitmap[2], ~0U, IOBITMAP_VMX_SIZE);
+    } else {
+        /* Same size and format as hvm_io_bitmap */
+        shadow_io_bitmap[0] = _xmalloc(IOBITMAP_SIZE, PAGE_SIZE);
+        shadow_io_bitmap[1] = _xmalloc(IOBITMAP_SIZE, PAGE_SIZE);
+        shadow_io_bitmap[2] = _xmalloc(IOBITMAP_SIZE, PAGE_SIZE);
+        memset(shadow_io_bitmap[0], ~0U, IOBITMAP_SIZE);
+        memset(shadow_io_bitmap[1], ~0U, IOBITMAP_SIZE);
+        memset(shadow_io_bitmap[2], ~0U, IOBITMAP_SIZE);
+    }
+
+    __clear_bit(0x80, shadow_io_bitmap[0]);
+    __clear_bit(0xed, shadow_io_bitmap[1]);
+}
+
+unsigned long *
+nestedhvm_vcpu_iomap_get(bool_t port_80, bool_t port_ed)
+{
+    int i;
+    extern int hvm_port80_allowed;
+
+    if (!hvm_port80_allowed)
+        port_80 = 1;
+
+    if (port_80 == 0) {
+        if (port_ed == 0)
+            return hvm_io_bitmap;
+        i = 0;
+    } else {
+        if (port_ed == 0)
+            i = 1;
+        else
+            i = 2;
+    }
+
+    return shadow_io_bitmap[i];
+}
diff -r 216ffee07118 -r 4c51c9128e19 xen/arch/x86/setup.c
--- a/xen/arch/x86/setup.c
+++ b/xen/arch/x86/setup.c
@@ -1260,6 +1260,8 @@ void __init __start_xen(unsigned long mb
 
     if ( opt_watchdog ) 
         watchdog_enable();
+
+    nestedhvm_setup();
     
     if ( !tboot_protect_mem_regions() )
         panic("Could not protect TXT memory regions\n");
diff -r 216ffee07118 -r 4c51c9128e19 xen/include/asm-x86/hvm/nestedhvm.h
--- /dev/null
+++ b/xen/include/asm-x86/hvm/nestedhvm.h
@@ -0,0 +1,63 @@
+/*
+ * Nested HVM
+ * Copyright (c) 2011, Advanced Micro Devices, Inc.
+ * Author: Christoph Egger <Christoph.Egger@amd.com>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program; if not, write to the Free Software Foundation, Inc., 59 Temple
+ * Place - Suite 330, Boston, MA 02111-1307 USA.
+ */
+
+#ifndef _HVM_NESTEDHVM_H
+#define _HVM_NESTEDHVM_H
+
+#include <xen/types.h>         /* for uintNN_t */
+#include <xen/sched.h>         /* for struct vcpu, struct domain */
+#include <asm/hvm/vcpu.h>      /* for vcpu_nestedhvm */
+
+enum nestedhvm_vmexits {
+    NESTEDHVM_VMEXIT_ERROR = 0, /* inject VMEXIT w/ invalid VMCB */
+    NESTEDHVM_VMEXIT_FATALERROR = 1, /* crash first level guest */
+    NESTEDHVM_VMEXIT_HOST = 2,  /* exit handled on host level */
+    NESTEDHVM_VMEXIT_CONTINUE = 3, /* further handling */
+    NESTEDHVM_VMEXIT_INJECT = 4, /* inject VMEXIT */
+    NESTEDHVM_VMEXIT_DONE = 5, /* VMEXIT handled */
+};
+
+/* Nested HVM on/off per domain */
+bool_t nestedhvm_enabled(struct domain *d);
+
+/* Nested VCPU */
+int nestedhvm_vcpu_initialise(struct vcpu *v);
+int nestedhvm_vcpu_destroy(struct vcpu *v);
+void nestedhvm_vcpu_reset(struct vcpu *v);
+bool_t nestedhvm_vcpu_in_guestmode(struct vcpu *v);
+#define nestedhvm_vcpu_enter_guestmode(v) \
+    vcpu_nestedhvm(v).nv_guestmode = 1
+#define nestedhvm_vcpu_exit_guestmode(v)  \
+    vcpu_nestedhvm(v).nv_guestmode = 0
+
+/* Nested paging */
+#define NESTEDHVM_PAGEFAULT_DONE   0
+#define NESTEDHVM_PAGEFAULT_INJECT 1
+#define NESTEDHVM_PAGEFAULT_ERROR  2
+int nestedhvm_hap_nested_page_fault(struct vcpu *v, paddr_t L2_gpa);
+
+/* IO permission map */
+unsigned long *nestedhvm_vcpu_iomap_get(bool_t ioport_80, bool_t ioport_ed);
+
+/* Misc */
+#define nestedhvm_paging_mode_hap(v) (!!nhvm_vmcx_hap_enabled(v))
+#define nestedhvm_vmswitch_in_progress(v)   \
+    (!!vcpu_nestedhvm((v)).nv_vmswitch_in_progress)
+
+#endif /* _HVM_NESTEDHVM_H */
diff -r 216ffee07118 -r 4c51c9128e19 xen/include/asm-x86/setup.h
--- a/xen/include/asm-x86/setup.h
+++ b/xen/include/asm-x86/setup.h
@@ -23,6 +23,7 @@ int transmeta_init_cpu(void);
 void numa_initmem_init(unsigned long start_pfn, unsigned long end_pfn);
 void arch_init_memory(void);
 void subarch_init_memory(void);
+void nestedhvm_setup(void);
 
 void init_IRQ(void);
 void vesa_init(void);

[-- Attachment #3: Type: text/plain, Size: 138 bytes --]

_______________________________________________
Xen-devel mailing list
Xen-devel@lists.xensource.com
http://lists.xensource.com/xen-devel