[PATCH v2 0/9] xen/arm: guest SMP support

[PATCH v2 0/9] xen/arm: guest SMP support

[Stefano Stabellini]

Hi all,
this patch series implement guest SMP support for ARM, using the ARM
PSCI interface for secondary cpu bringup.


Changes in v2:
- set is_initialised in arch_set_info_guest;
- zero vcpu_guest_context before using it;
- add a patch to allocate secondaries dom0 vcpus;
- split "move VCPUOP_register_vcpu_info to common code" in two;
- add a patch to start the vtimer Xen timers on the processor they should
  be running on;
- add a patch to initialize virt_timer and phys_timer with the same
  values on all vcpus.


Stefano Stabellini (9):
      xen/arm: basic PSCI support, implement cpu_on
      xen/arm: allocate secondaries dom0 vcpus
      xen/arm: support for guest SGI
      xen/arm: support vcpu_op hypercalls
      xen/arm: implement map_domain_page_global and unmap_domain_page_global
      xen: move VCPUOP_register_vcpu_info to common code
      xen/arm: send IPIs to inject irqs into guest vcpus running on different pcpus
      xen/arm: start the vtimer Xen timers on the processor they should be running on
      xen/arm: initialize virt_timer and phys_timer with the same values on all vcpus

 xen/arch/arm/domain.c        |   69 +++++++++++++++++++++++++
 xen/arch/arm/domain_build.c  |    8 +++-
 xen/arch/arm/mm.c            |   10 ++++
 xen/arch/arm/traps.c         |   42 ++++++++++++++++
 xen/arch/arm/vgic.c          |   54 ++++++++++++++++++--
 xen/arch/arm/vtimer.c        |   22 ++++++--
 xen/arch/x86/domain.c        |  113 ------------------------------------------
 xen/common/domain.c          |  110 ++++++++++++++++++++++++++++++++++++++++
 xen/include/asm-arm/domain.h |   30 ++++++++---
 xen/include/asm-arm/gic.h    |    3 +
 xen/include/asm-x86/domain.h |    3 -
 xen/include/xen/domain.h     |    3 +
 xen/include/xen/sched.h      |    3 +
 13 files changed, 333 insertions(+), 137 deletions(-)


Cheers,

Stefano

[PATCH v2 1/9] xen/arm: basic PSCI support, implement cpu_on

[Stefano Stabellini]

Implement support for ARM Power State Coordination Interface, PSCI in
short. The current implementation is based on HVC and only supports the
cpu_on call.

Changes in v2:
- set is_initialised in arch_set_info_guest;
- zero vcpu_guest_context before using it.

Signed-off-by: Stefano Stabellini <stefano.stabellini@eu.citrix.com>
---
 xen/arch/arm/domain.c        |   69 ++++++++++++++++++++++++++++++++++++++++++
 xen/arch/arm/traps.c         |   41 +++++++++++++++++++++++++
 xen/include/asm-arm/domain.h |    4 ++
 3 files changed, 114 insertions(+), 0 deletions(-)

diff --git a/xen/arch/arm/domain.c b/xen/arch/arm/domain.c
index eae42af..f71b582 100644
--- a/xen/arch/arm/domain.c
+++ b/xen/arch/arm/domain.c
@@ -561,6 +561,8 @@ int arch_set_info_guest(
     else
         set_bit(_VPF_down, &v->pause_flags);
 
+    v->is_initialised = 1;
+
     return 0;
 }
 
@@ -635,6 +637,73 @@ void arch_dump_vcpu_info(struct vcpu *v)
 {
 }
 
+int do_psci_cpu_on(uint32_t vcpuid, uint32_t entry_point)
+{
+    struct vcpu *v;
+    struct domain *d = current->domain;
+    struct vcpu_guest_context *ctxt;
+    int rc;
+
+    if ( (vcpuid < 0) || (vcpuid >= MAX_VIRT_CPUS) )
+        return -EINVAL;
+
+    if ( vcpuid >= d->max_vcpus || (v = d->vcpu[vcpuid]) == NULL )
+        return -ENOENT;
+
+    if ( v->is_initialised )
+        return -EEXIST;
+    if ( (ctxt = alloc_vcpu_guest_context()) == NULL )
+        return -ENOMEM;
+
+    memset(ctxt, 0, sizeof(*ctxt));
+    ctxt->user_regs.pc32 = entry_point;
+    /* Linux boot protocol. See linux.Documentation/arm/Booting. */
+    ctxt->user_regs.x0 = 0; /* SBZ */
+    /* Machine ID: We use DTB therefore no machine id */
+    ctxt->user_regs.x1 = 0xffffffff;
+    /* ATAGS/DTB: We currently require that the guest kernel to be
+     * using CONFIG_ARM_APPENDED_DTB. Ensure that r2 does not look
+     * like a valid pointer to a set of ATAGS or a DTB.
+     */
+    ctxt->user_regs.x2 = 0xffffffff;
+    ctxt->sctlr = SCTLR_BASE;
+    ctxt->ttbr0 = 0;
+    ctxt->ttbr1 = 0;
+    ctxt->ttbcr = 0; /* Defined Reset Value */
+    ctxt->user_regs.cpsr = PSR_ABT_MASK|PSR_FIQ_MASK|PSR_IRQ_MASK|PSR_MODE_SVC;
+    ctxt->flags = VGCF_online;
+
+    domain_lock(d);
+    rc = arch_set_info_guest(v, ctxt);
+    free_vcpu_guest_context(ctxt);
+
+    if ( rc < 0 )
+    {
+        domain_unlock(d);
+        return rc;
+    }
+
+    clear_bit(_VPF_down, &v->pause_flags);
+    vcpu_wake(v);
+
+    domain_unlock(d);
+
+    return 0;
+}
+
+int do_psci_cpu_off(uint32_t power_state)
+{
+    return -ENOSYS;
+}
+int do_psci_cpu_suspend(uint32_t power_state, uint32_t entry_point)
+{
+    return -ENOSYS;
+}
+int do_psci_migrate(uint32_t vcpuid)
+{
+    return -ENOSYS;
+}
+
 void vcpu_mark_events_pending(struct vcpu *v)
 {
     int already_pending = test_and_set_bit(
diff --git a/xen/arch/arm/traps.c b/xen/arch/arm/traps.c
index e9c83ed..8b5d0fe 100644
--- a/xen/arch/arm/traps.c
+++ b/xen/arch/arm/traps.c
@@ -617,6 +617,31 @@ static arm_hypercall_t arm_hypercall_table[] = {
     HYPERCALL(grant_table_op, 3),
 };
 
+#define __PSCI_cpu_suspend 0
+#define __PSCI_cpu_off     1
+#define __PSCI_cpu_on      2
+#define __PSCI_migrate     3
+
+typedef int (*arm_psci_fn_t)(uint32_t, uint32_t);
+
+typedef struct {
+    arm_psci_fn_t fn;
+    int nr_args;
+} arm_psci_t;
+
+#define PSCI(_name, _nr_args)                                  \
+    [ __PSCI_ ## _name ] =  {                                  \
+        .fn = (arm_psci_fn_t) &do_psci_ ## _name,              \
+        .nr_args = _nr_args,                                   \
+    }
+
+static arm_psci_t arm_psci_table[] = {
+    PSCI(cpu_suspend, 2),
+    PSCI(cpu_off, 1),
+    PSCI(cpu_on, 2),
+    PSCI(migrate, 1),
+};
+
 static void do_debug_trap(struct cpu_user_regs *regs, unsigned int code)
 {
     register_t *r;
@@ -646,6 +671,20 @@ static void do_debug_trap(struct cpu_user_regs *regs, unsigned int code)
     }
 }
 
+static void do_trap_psci(struct cpu_user_regs *regs)
+{
+    arm_psci_fn_t psci_call = NULL;
+
+    if ( regs->r0 >= ARRAY_SIZE(arm_psci_table) )
+    {
+        regs->r0 = -ENOSYS;
+        return;
+    }
+
+    psci_call = arm_psci_table[regs->r0].fn;
+    regs->r0 = psci_call(regs->r1, regs->r2);
+}
+
 static void do_trap_hypercall(struct cpu_user_regs *regs, unsigned long iss)
 {
     arm_hypercall_fn_t call = NULL;
@@ -950,6 +989,8 @@ asmlinkage void do_trap_hypervisor(struct cpu_user_regs *regs)
     case HSR_EC_HVC:
         if ( (hsr.iss & 0xff00) == 0xff00 )
             return do_debug_trap(regs, hsr.iss & 0x00ff);
+        if ( !hsr.iss )
+            return do_trap_psci(regs);
         do_trap_hypercall(regs, hsr.iss);
         break;
     case HSR_EC_DATA_ABORT_GUEST:
diff --git a/xen/include/asm-arm/domain.h b/xen/include/asm-arm/domain.h
index 3fa266c2..85b5d40 100644
--- a/xen/include/asm-arm/domain.h
+++ b/xen/include/asm-arm/domain.h
@@ -220,6 +220,10 @@ struct arch_vcpu
 void vcpu_show_execution_state(struct vcpu *);
 void vcpu_show_registers(const struct vcpu *);
 
+int do_psci_cpu_on(uint32_t vcpuid, uint32_t entry_point);
+int do_psci_cpu_off(uint32_t power_state);
+int do_psci_cpu_suspend(uint32_t power_state, uint32_t entry_point);
+int do_psci_migrate(uint32_t vcpuid);
 #endif /* __ASM_DOMAIN_H__ */
 
 /*
-- 
1.7.2.5

[PATCH v2 2/9] xen/arm: allocate secondaries dom0 vcpus

[Stefano Stabellini]

Signed-off-by: Stefano Stabellini <stefano.stabellini@eu.citrix.com>
---
 xen/arch/arm/domain_build.c |    8 +++++++-
 1 files changed, 7 insertions(+), 1 deletions(-)

diff --git a/xen/arch/arm/domain_build.c b/xen/arch/arm/domain_build.c
index a6d8e9d..5dfa592 100644
--- a/xen/arch/arm/domain_build.c
+++ b/xen/arch/arm/domain_build.c
@@ -363,7 +363,7 @@ static void dtb_load(struct kernel_info *kinfo)
 int construct_dom0(struct domain *d)
 {
     struct kernel_info kinfo = {};
-    int rc;
+    int rc, i, cpu = 0;
 
     struct vcpu *v = d->vcpu[0];
     struct cpu_user_regs *regs = &v->arch.cpu_info->guest_cpu_user_regs;
@@ -451,6 +451,12 @@ int construct_dom0(struct domain *d)
     }
 #endif
 
+    for ( i = 1; i < d->max_vcpus; i++ )
+    {
+        cpu = cpumask_cycle(cpu, &cpu_online_map);
+        (void)alloc_vcpu(d, i, cpu);
+    }
+
     local_abort_enable();
 
     return 0;
-- 
1.7.2.5

[PATCH v2 3/9] xen/arm: support for guest SGI

[Stefano Stabellini]

Trap writes to GICD_SGIR, parse the requests, inject SGIs into the right
guest vcpu.

Signed-off-by: Stefano Stabellini <stefano.stabellini@eu.citrix.com>
---
 xen/arch/arm/vgic.c       |   52 ++++++++++++++++++++++++++++++++++++++++----
 xen/include/asm-arm/gic.h |    3 ++
 2 files changed, 50 insertions(+), 5 deletions(-)

diff --git a/xen/arch/arm/vgic.c b/xen/arch/arm/vgic.c
index b30da78..acdd732 100644
--- a/xen/arch/arm/vgic.c
+++ b/xen/arch/arm/vgic.c
@@ -370,6 +370,7 @@ static void vgic_enable_irqs(struct vcpu *v, uint32_t r, int n)
 
 static int vgic_distr_mmio_write(struct vcpu *v, mmio_info_t *info)
 {
+    struct domain *d = v->domain;
     struct hsr_dabt dabt = info->dabt;
     struct cpu_user_regs *regs = guest_cpu_user_regs();
     register_t *r = select_user_reg(regs, dabt.reg);
@@ -498,11 +499,52 @@ static int vgic_distr_mmio_write(struct vcpu *v, mmio_info_t *info)
         goto write_ignore;
 
     case GICD_SGIR:
-        if ( dabt.size != 2 ) goto bad_width;
-        printk("vGICD: unhandled write %#"PRIregister" to ICFGR%d\n",
-               *r, gicd_reg - GICD_ICFGR);
-        return 0;
-
+        {
+            cpumask_t vcpu_mask;
+            int virtual_irq;
+            int filter;
+            int vcpuid;
+            struct vcpu *vt;
+            int i;
+
+            if ( dabt.size != 2 ) goto bad_width;
+
+            filter = (*r & GICD_SGI_TARGET_LIST_MASK);
+            virtual_irq = (*r & GICD_SGI_INTID_MASK);
+
+            cpumask_clear(&vcpu_mask);
+            switch ( filter )
+            {
+                case GICD_SGI_TARGET_LIST:
+                    cpumask_bits(&vcpu_mask)[0] = (*r & GICD_SGI_TARGET_MASK) >> GICD_SGI_TARGET_SHIFT;
+                    break;
+                case GICD_SGI_TARGET_OTHERS:
+                    for ( i = 0; i < d->max_vcpus; i++ )
+                    {
+                        if ( i != current->vcpu_id && d->vcpu[i] != NULL )
+                            cpumask_set_cpu(i, &vcpu_mask);
+                    }
+                case GICD_SGI_TARGET_SELF:
+                    cpumask_of(current->vcpu_id);
+                    break;
+                default:
+                    printk("vGICD: unhandled GICD_SGIR write %x with wrong TargetListFilter field\n", *r);
+                    return 0;
+            }
+
+            for_each_cpu( vcpuid, &vcpu_mask )
+            {
+                if ( vcpuid >= d->max_vcpus || (vt = d->vcpu[vcpuid]) == NULL ||
+                        virtual_irq >= 16 )
+                {
+                    printk("vGICD: GICD_SGIR write %x, wrong CPUTargetList\n", *r);
+                    return 0;
+                }
+                vgic_vcpu_inject_irq(vt, virtual_irq, 1);
+            }
+            return 1;
+        }
+ 
     case GICD_CPENDSGIR ... GICD_CPENDSGIRN:
         if ( dabt.size != 0 && dabt.size != 2 ) goto bad_width;
         printk("vGICD: unhandled %s write %#"PRIregister" to ICPENDSGIR%d\n",
diff --git a/xen/include/asm-arm/gic.h b/xen/include/asm-arm/gic.h
index 24c0d5c..b23b747 100644
--- a/xen/include/asm-arm/gic.h
+++ b/xen/include/asm-arm/gic.h
@@ -51,12 +51,15 @@
 #define GICD_SPENDSGIRN (0xF2C/4)
 #define GICD_ICPIDR2    (0xFE8/4)
 
+#define GICD_SGI_TARGET_LIST_SHIFT   (24)
+#define GICD_SGI_TARGET_LIST_MASK    (0x3UL << GICD_SGI_TARGET_LIST_SHIFT)
 #define GICD_SGI_TARGET_LIST   (0UL<<24)
 #define GICD_SGI_TARGET_OTHERS (1UL<<24)
 #define GICD_SGI_TARGET_SELF   (2UL<<24)
 #define GICD_SGI_TARGET_SHIFT  (16)
 #define GICD_SGI_TARGET_MASK   (0xFFUL<<GICD_SGI_TARGET_SHIFT)
 #define GICD_SGI_GROUP1        (1UL<<15)
+#define GICD_SGI_INTID_MASK    (0xFUL)
 
 #define GICC_CTLR       (0x0000/4)
 #define GICC_PMR        (0x0004/4)
-- 
1.7.2.5

[PATCH v2 4/9] xen/arm: support vcpu_op hypercalls

[Stefano Stabellini]

Signed-off-by: Stefano Stabellini <stefano.stabellini@eu.citrix.com>
---
 xen/arch/arm/traps.c |    1 +
 1 files changed, 1 insertions(+), 0 deletions(-)

diff --git a/xen/arch/arm/traps.c b/xen/arch/arm/traps.c
index 8b5d0fe..fd6d315 100644
--- a/xen/arch/arm/traps.c
+++ b/xen/arch/arm/traps.c
@@ -615,6 +615,7 @@ static arm_hypercall_t arm_hypercall_table[] = {
     HYPERCALL(sysctl, 2),
     HYPERCALL(hvm_op, 2),
     HYPERCALL(grant_table_op, 3),
+    HYPERCALL(vcpu_op, 3),
 };
 
 #define __PSCI_cpu_suspend 0
-- 
1.7.2.5

[PATCH v2 5/9] xen/arm: implement map_domain_page_global and unmap_domain_page_global

[Stefano Stabellini]

Signed-off-by: Stefano Stabellini <stefano.stabellini@eu.citrix.com>
---
 xen/arch/arm/mm.c |   10 ++++++++++
 1 files changed, 10 insertions(+), 0 deletions(-)

diff --git a/xen/arch/arm/mm.c b/xen/arch/arm/mm.c
index ba3140d..ae0394c 100644
--- a/xen/arch/arm/mm.c
+++ b/xen/arch/arm/mm.c
@@ -130,6 +130,16 @@ void clear_fixmap(unsigned map)
     flush_xen_data_tlb_range_va(FIXMAP_ADDR(map), PAGE_SIZE);
 }
 
+void *map_domain_page_global(unsigned long mfn)
+{
+    return map_domain_page(mfn);
+}
+    
+void unmap_domain_page_global(const void *va)
+{
+    unmap_domain_page(va);
+}
+
 /* Map a page of domheap memory */
 void *map_domain_page(unsigned long mfn)
 {
-- 
1.7.2.5

[PATCH v2 6/9] xen: move VCPUOP_register_vcpu_info to common code

[Stefano Stabellini]

Move the implementation of VCPUOP_register_vcpu_info from x86 specific
to commmon code.

Move vcpu_info_mfn from an arch specific vcpu sub-field to the common
vcpu struct.
Move the initialization of vcpu_info_mfn to common code.

Move unmap_vcpu_info and the call to unmap_vcpu_info at domain
destruction time to common code.

Signed-off-by: Stefano Stabellini <stefano.stabellini@eu.citrix.com>
---
 xen/arch/x86/domain.c        |  113 ------------------------------------------
 xen/common/domain.c          |  110 ++++++++++++++++++++++++++++++++++++++++
 xen/include/asm-x86/domain.h |    3 -
 xen/include/xen/domain.h     |    3 +
 xen/include/xen/sched.h      |    3 +
 5 files changed, 116 insertions(+), 116 deletions(-)

diff --git a/xen/arch/x86/domain.c b/xen/arch/x86/domain.c
index 8d30d08..ee22869 100644
--- a/xen/arch/x86/domain.c
+++ b/xen/arch/x86/domain.c
@@ -393,8 +393,6 @@ int vcpu_initialise(struct vcpu *v)
         goto done;
     }
 
-    v->arch.pv_vcpu.vcpu_info_mfn = INVALID_MFN;
-
     spin_lock_init(&v->arch.pv_vcpu.shadow_ldt_lock);
 
     if ( !is_idle_domain(d) )
@@ -953,99 +951,6 @@ void arch_vcpu_reset(struct vcpu *v)
     }
 }
 
-/* 
- * Unmap the vcpu info page if the guest decided to place it somewhere
- * else.  This is only used from arch_domain_destroy, so there's no
- * need to do anything clever.
- */
-static void
-unmap_vcpu_info(struct vcpu *v)
-{
-    unsigned long mfn;
-
-    if ( v->arch.pv_vcpu.vcpu_info_mfn == INVALID_MFN )
-        return;
-
-    mfn = v->arch.pv_vcpu.vcpu_info_mfn;
-    unmap_domain_page_global(v->vcpu_info);
-
-    v->vcpu_info = &dummy_vcpu_info;
-    v->arch.pv_vcpu.vcpu_info_mfn = INVALID_MFN;
-
-    put_page_and_type(mfn_to_page(mfn));
-}
-
-/* 
- * Map a guest page in and point the vcpu_info pointer at it.  This
- * makes sure that the vcpu_info is always pointing at a valid piece
- * of memory, and it sets a pending event to make sure that a pending
- * event doesn't get missed.
- */
-static int
-map_vcpu_info(struct vcpu *v, unsigned long gfn, unsigned offset)
-{
-    struct domain *d = v->domain;
-    void *mapping;
-    vcpu_info_t *new_info;
-    struct page_info *page;
-    int i;
-
-    if ( offset > (PAGE_SIZE - sizeof(vcpu_info_t)) )
-        return -EINVAL;
-
-    if ( v->arch.pv_vcpu.vcpu_info_mfn != INVALID_MFN )
-        return -EINVAL;
-
-    /* Run this command on yourself or on other offline VCPUS. */
-    if ( (v != current) && !test_bit(_VPF_down, &v->pause_flags) )
-        return -EINVAL;
-
-    page = get_page_from_gfn(d, gfn, NULL, P2M_ALLOC);
-    if ( !page )
-        return -EINVAL;
-
-    if ( !get_page_type(page, PGT_writable_page) )
-    {
-        put_page(page);
-        return -EINVAL;
-    }
-
-    mapping = __map_domain_page_global(page);
-    if ( mapping == NULL )
-    {
-        put_page_and_type(page);
-        return -ENOMEM;
-    }
-
-    new_info = (vcpu_info_t *)(mapping + offset);
-
-    if ( v->vcpu_info == &dummy_vcpu_info )
-    {
-        memset(new_info, 0, sizeof(*new_info));
-        __vcpu_info(v, new_info, evtchn_upcall_mask) = 1;
-    }
-    else
-    {
-        memcpy(new_info, v->vcpu_info, sizeof(*new_info));
-    }
-
-    v->vcpu_info = new_info;
-    v->arch.pv_vcpu.vcpu_info_mfn = page_to_mfn(page);
-
-    /* Set new vcpu_info pointer /before/ setting pending flags. */
-    wmb();
-
-    /*
-     * Mark everything as being pending just to make sure nothing gets
-     * lost.  The domain will get a spurious event, but it can cope.
-     */
-    vcpu_info(v, evtchn_upcall_pending) = 1;
-    for ( i = 0; i < BITS_PER_EVTCHN_WORD(d); i++ )
-        set_bit(i, &vcpu_info(v, evtchn_pending_sel));
-
-    return 0;
-}
-
 long
 arch_do_vcpu_op(
     int cmd, struct vcpu *v, XEN_GUEST_HANDLE_PARAM(void) arg)
@@ -1082,22 +987,6 @@ arch_do_vcpu_op(
         break;
     }
 
-    case VCPUOP_register_vcpu_info:
-    {
-        struct domain *d = v->domain;
-        struct vcpu_register_vcpu_info info;
-
-        rc = -EFAULT;
-        if ( copy_from_guest(&info, arg, 1) )
-            break;
-
-        domain_lock(d);
-        rc = map_vcpu_info(v, info.mfn, info.offset);
-        domain_unlock(d);
-
-        break;
-    }
-
     /*
      * XXX Disable for 4.0.0: __update_vcpu_system_time() writes to the given
      * virtual address even when running in another domain's address space.
@@ -2014,8 +1903,6 @@ int domain_relinquish_resources(struct domain *d)
                  * mappings.
                  */
                 destroy_gdt(v);
-
-                unmap_vcpu_info(v);
             }
 
             if ( d->arch.pv_domain.pirq_eoi_map != NULL )
diff --git a/xen/common/domain.c b/xen/common/domain.c
index 64ee29d..7aad36a 100644
--- a/xen/common/domain.c
+++ b/xen/common/domain.c
@@ -141,6 +141,7 @@ struct vcpu *alloc_vcpu(
         v->vcpu_info = ((vcpu_id < XEN_LEGACY_MAX_VCPUS)
                         ? (vcpu_info_t *)&shared_info(d, vcpu_info[vcpu_id])
                         : &dummy_vcpu_info);
+        v->vcpu_info_mfn = INVALID_MFN;
         init_waitqueue_vcpu(v);
     }
 
@@ -493,6 +494,7 @@ int rcu_lock_live_remote_domain_by_id(domid_t dom, struct domain **d)
 int domain_kill(struct domain *d)
 {
     int rc = 0;
+    struct vcpu *v;
 
     if ( d == current->domain )
         return -EINVAL;
@@ -517,6 +519,8 @@ int domain_kill(struct domain *d)
             BUG_ON(rc != -EAGAIN);
             break;
         }
+        for_each_vcpu ( d, v )
+            unmap_vcpu_info(v);
         d->is_dying = DOMDYING_dead;
         /* Mem event cleanup has to go here because the rings 
          * have to be put before we call put_domain. */
@@ -842,6 +846,96 @@ void vcpu_reset(struct vcpu *v)
     vcpu_unpause(v);
 }
 
+/* 
+ * Map a guest page in and point the vcpu_info pointer at it.  This
+ * makes sure that the vcpu_info is always pointing at a valid piece
+ * of memory, and it sets a pending event to make sure that a pending
+ * event doesn't get missed.
+ */
+int map_vcpu_info(struct vcpu *v, unsigned long gfn, unsigned offset)
+{
+    struct domain *d = v->domain;
+    void *mapping;
+    vcpu_info_t *new_info;
+    struct page_info *page;
+    int i;
+
+    if ( offset > (PAGE_SIZE - sizeof(vcpu_info_t)) )
+        return -EINVAL;
+
+    if ( v->vcpu_info_mfn != INVALID_MFN )
+        return -EINVAL;
+
+    /* Run this command on yourself or on other offline VCPUS. */
+    if ( (v != current) && !test_bit(_VPF_down, &v->pause_flags) )
+        return -EINVAL;
+
+    page = get_page_from_gfn(d, gfn, NULL, P2M_ALLOC);
+    if ( !page )
+        return -EINVAL;
+
+    if ( !get_page_type(page, PGT_writable_page) )
+    {
+        put_page(page);
+        return -EINVAL;
+    }
+
+    mapping = __map_domain_page_global(page);
+    if ( mapping == NULL )
+    {
+        put_page_and_type(page);
+        return -ENOMEM;
+    }
+
+    new_info = (vcpu_info_t *)(mapping + offset);
+
+    if ( v->vcpu_info == &dummy_vcpu_info )
+    {
+        memset(new_info, 0, sizeof(*new_info));
+        __vcpu_info(v, new_info, evtchn_upcall_mask) = 1;
+    }
+    else
+    {
+        memcpy(new_info, v->vcpu_info, sizeof(*new_info));
+    }
+
+    v->vcpu_info = new_info;
+    v->vcpu_info_mfn = page_to_mfn(page);
+
+    /* Set new vcpu_info pointer /before/ setting pending flags. */
+    wmb();
+
+    /*
+     * Mark everything as being pending just to make sure nothing gets
+     * lost.  The domain will get a spurious event, but it can cope.
+     */
+    vcpu_info(v, evtchn_upcall_pending) = 1;
+    for ( i = 0; i < BITS_PER_EVTCHN_WORD(d); i++ )
+        set_bit(i, &vcpu_info(v, evtchn_pending_sel));
+
+    return 0;
+}
+
+/* 
+ * Unmap the vcpu info page if the guest decided to place it somewhere
+ * else.  This is only used from arch_domain_destroy, so there's no
+ * need to do anything clever.
+ */
+void unmap_vcpu_info(struct vcpu *v)
+{
+    unsigned long mfn;
+
+    if ( v->vcpu_info_mfn == INVALID_MFN )
+        return;
+
+    mfn = v->vcpu_info_mfn;
+    unmap_domain_page_global(v->vcpu_info);
+
+    v->vcpu_info = &dummy_vcpu_info;
+    v->vcpu_info_mfn = INVALID_MFN;
+
+    put_page_and_type(mfn_to_page(mfn));
+}
 
 long do_vcpu_op(int cmd, int vcpuid, XEN_GUEST_HANDLE_PARAM(void) arg)
 {
@@ -961,6 +1055,22 @@ long do_vcpu_op(int cmd, int vcpuid, XEN_GUEST_HANDLE_PARAM(void) arg)
 
         break;
 
+    case VCPUOP_register_vcpu_info:
+    {
+        struct domain *d = v->domain;
+        struct vcpu_register_vcpu_info info;
+
+        rc = -EFAULT;
+        if ( copy_from_guest(&info, arg, 1) )
+            break;
+
+        domain_lock(d);
+        rc = map_vcpu_info(v, info.mfn, info.offset);
+        domain_unlock(d);
+
+        break;
+    }
+
 #ifdef VCPU_TRAP_NMI
     case VCPUOP_send_nmi:
         if ( !guest_handle_is_null(arg) )
diff --git a/xen/include/asm-x86/domain.h b/xen/include/asm-x86/domain.h
index 6f9744a..e7ed7c3 100644
--- a/xen/include/asm-x86/domain.h
+++ b/xen/include/asm-x86/domain.h
@@ -374,9 +374,6 @@ struct pv_vcpu
     /* Current LDT details. */
     unsigned long shadow_ldt_mapcnt;
     spinlock_t shadow_ldt_lock;
-
-    /* Guest-specified relocation of vcpu_info. */
-    unsigned long vcpu_info_mfn;
 };
 
 struct arch_vcpu
diff --git a/xen/include/xen/domain.h b/xen/include/xen/domain.h
index d4ac50f..d0263a9 100644
--- a/xen/include/xen/domain.h
+++ b/xen/include/xen/domain.h
@@ -52,6 +52,9 @@ void free_pirq_struct(void *);
 int  vcpu_initialise(struct vcpu *v);
 void vcpu_destroy(struct vcpu *v);
 
+int map_vcpu_info(struct vcpu *v, unsigned long gfn, unsigned offset);
+void unmap_vcpu_info(struct vcpu *v);
+
 int arch_domain_create(struct domain *d, unsigned int domcr_flags);
 
 void arch_domain_destroy(struct domain *d);
diff --git a/xen/include/xen/sched.h b/xen/include/xen/sched.h
index cabaf27..504c3d6 100644
--- a/xen/include/xen/sched.h
+++ b/xen/include/xen/sched.h
@@ -187,6 +187,9 @@ struct vcpu
 
     struct waitqueue_vcpu *waitqueue_vcpu;
 
+    /* Guest-specified relocation of vcpu_info. */
+    unsigned long vcpu_info_mfn;
+
     struct arch_vcpu arch;
 };
 
-- 
1.7.2.5

[PATCH v2 7/9] xen/arm: send IPIs to inject irqs into guest vcpus running on different pcpus

[Stefano Stabellini]

If we need to inject an irq into a guest that is running on a different
processor, we shouldn't just enqueue the irq into the lr_pending and
inflight lists and wait for something to interrupt the guest execution.
Send an IPI to the target pcpu so that Xen can inject the new interrupt
returning to guest.

Signed-off-by: Stefano Stabellini <stefano.stabellini@eu.citrix.com>
---
 xen/arch/arm/vgic.c |    2 ++
 1 files changed, 2 insertions(+), 0 deletions(-)

diff --git a/xen/arch/arm/vgic.c b/xen/arch/arm/vgic.c
index acdd732..0025e28 100644
--- a/xen/arch/arm/vgic.c
+++ b/xen/arch/arm/vgic.c
@@ -661,6 +661,8 @@ out:
     spin_unlock_irqrestore(&v->arch.vgic.lock, flags);
     /* we have a new higher priority irq, inject it into the guest */
     vcpu_unblock(v);
+    if ( v != current && v->is_running )
+        smp_send_event_check_mask(cpumask_of(v->processor));
 }
 
 /*
-- 
1.7.2.5

[PATCH v2 8/9] xen/arm: start the vtimer Xen timers on the processor they should be running on

[Stefano Stabellini]

The Xen physical timer emulator and virtual timer driver use two
internal Xen timers: initialize them on the processor the vcpu is
going to be running on, rather than the processor that it's creating the
vcpu.

Signed-off-by: Stefano Stabellini <stefano.stabellini@eu.citrix.com>
---
 xen/arch/arm/vtimer.c |    4 ++--
 1 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/xen/arch/arm/vtimer.c b/xen/arch/arm/vtimer.c
index 1cb365e..2444851 100644
--- a/xen/arch/arm/vtimer.c
+++ b/xen/arch/arm/vtimer.c
@@ -48,7 +48,7 @@ int vcpu_vtimer_init(struct vcpu *v)
 {
     struct vtimer *t = &v->arch.phys_timer;
 
-    init_timer(&t->timer, phys_timer_expired, t, smp_processor_id());
+    init_timer(&t->timer, phys_timer_expired, t, v->processor);
     t->ctl = 0;
     t->offset = NOW();
     t->cval = NOW();
@@ -56,7 +56,7 @@ int vcpu_vtimer_init(struct vcpu *v)
     t->v = v;
 
     t = &v->arch.virt_timer;
-    init_timer(&t->timer, virt_timer_expired, t, smp_processor_id());
+    init_timer(&t->timer, virt_timer_expired, t, v->processor);
     t->ctl = 0;
     t->offset = READ_SYSREG64(CNTVCT_EL0) + READ_SYSREG64(CNTVOFF_EL2);
     t->cval = 0;
-- 
1.7.2.5

[PATCH v2 9/9] xen/arm: initialize virt_timer and phys_timer with the same values on all vcpus

[Stefano Stabellini]

Signed-off-by: Stefano Stabellini <stefano.stabellini@eu.citrix.com>
---
 xen/arch/arm/vtimer.c        |   18 ++++++++++++++----
 xen/include/asm-arm/domain.h |   26 +++++++++++++++++---------
 2 files changed, 31 insertions(+), 13 deletions(-)

diff --git a/xen/arch/arm/vtimer.c b/xen/arch/arm/vtimer.c
index 2444851..49858e7 100644
--- a/xen/arch/arm/vtimer.c
+++ b/xen/arch/arm/vtimer.c
@@ -46,20 +46,30 @@ static void virt_timer_expired(void *data)
 
 int vcpu_vtimer_init(struct vcpu *v)
 {
+    struct vtimer_base *b = &v->domain->arch.phys_timer_base;
     struct vtimer *t = &v->arch.phys_timer;
 
+    if ( !b->offset )
+        b->offset = NOW();
+    if ( !b->cval )
+        b->cval = NOW();
     init_timer(&t->timer, phys_timer_expired, t, v->processor);
     t->ctl = 0;
-    t->offset = NOW();
-    t->cval = NOW();
+    t->offset = b->offset;
+    t->cval = b->cval;
     t->irq = 30;
     t->v = v;
 
+    b = &v->domain->arch.virt_timer_base;
+    if ( !b->offset )
+        b->offset = READ_SYSREG64(CNTVCT_EL0) + READ_SYSREG64(CNTVOFF_EL2);
+    if ( !b->cval )
+        b->cval = 0;
     t = &v->arch.virt_timer;
     init_timer(&t->timer, virt_timer_expired, t, v->processor);
     t->ctl = 0;
-    t->offset = READ_SYSREG64(CNTVCT_EL0) + READ_SYSREG64(CNTVOFF_EL2);
-    t->cval = 0;
+    t->offset = b->offset;
+    t->cval = b->cval;
     t->irq = 27;
     t->v = v;
 
diff --git a/xen/include/asm-arm/domain.h b/xen/include/asm-arm/domain.h
index 85b5d40..410b8e7 100644
--- a/xen/include/asm-arm/domain.h
+++ b/xen/include/asm-arm/domain.h
@@ -47,6 +47,20 @@ enum domain_type {
 #define is_pv64_domain(d) (0)
 #endif
 
+struct vtimer {
+        struct vcpu *v;
+        int irq;
+        struct timer timer;
+        uint32_t ctl;
+        uint64_t offset;
+        uint64_t cval;
+};
+
+struct vtimer_base {
+        uint64_t offset;
+        uint64_t cval;
+};
+
 struct arch_domain
 {
 #ifdef CONFIG_ARM_64
@@ -61,6 +75,9 @@ struct arch_domain
     uint32_t vpidr;
     register_t vmpidr;
 
+    struct vtimer_base phys_timer_base;
+    struct vtimer_base virt_timer_base;
+
     struct {
         /*
          * Covers access to other members of this struct _except_ for
@@ -91,15 +108,6 @@ struct arch_domain
 
 }  __cacheline_aligned;
 
-struct vtimer {
-        struct vcpu *v;
-        int irq;
-        struct timer timer;
-        uint32_t ctl;
-        uint64_t offset;
-        uint64_t cval;
-};
-
 struct arch_vcpu
 {
     struct {
-- 
1.7.2.5

Re: [PATCH v2 6/9] xen: move VCPUOP_register_vcpu_info to common code

[Julien Grall]

On 03/26/2013 02:47 PM, Stefano Stabellini wrote:

> +int map_vcpu_info(struct vcpu *v, unsigned long gfn, unsigned offset)
> +{
> +    struct domain *d = v->domain;
> +    void *mapping;
> +    vcpu_info_t *new_info;
> +    struct page_info *page;
> +    int i;
> +
> +    if ( offset > (PAGE_SIZE - sizeof(vcpu_info_t)) )
> +        return -EINVAL;
> +
> +    if ( v->vcpu_info_mfn != INVALID_MFN )
> +        return -EINVAL;
> +
> +    /* Run this command on yourself or on other offline VCPUS. */
> +    if ( (v != current) && !test_bit(_VPF_down, &v->pause_flags) )
> +        return -EINVAL;
> +
> +    page = get_page_from_gfn(d, gfn, NULL, P2M_ALLOC);

This patch doesn't compile on x86. get_page_from_gfn is missing.

Adding #include <asm/p2m.h> on top of the file (common/domain.c)
resolves the compilation error.

Julien

Re: [PATCH v2 6/9] xen: move VCPUOP_register_vcpu_info to common code

[Stefano Stabellini]

On Tue, 16 Apr 2013, Julien Grall wrote:
> On 03/26/2013 02:47 PM, Stefano Stabellini wrote:
> 
> > +int map_vcpu_info(struct vcpu *v, unsigned long gfn, unsigned offset)
> > +{
> > +    struct domain *d = v->domain;
> > +    void *mapping;
> > +    vcpu_info_t *new_info;
> > +    struct page_info *page;
> > +    int i;
> > +
> > +    if ( offset > (PAGE_SIZE - sizeof(vcpu_info_t)) )
> > +        return -EINVAL;
> > +
> > +    if ( v->vcpu_info_mfn != INVALID_MFN )
> > +        return -EINVAL;
> > +
> > +    /* Run this command on yourself or on other offline VCPUS. */
> > +    if ( (v != current) && !test_bit(_VPF_down, &v->pause_flags) )
> > +        return -EINVAL;
> > +
> > +    page = get_page_from_gfn(d, gfn, NULL, P2M_ALLOC);
> 
> This patch doesn't compile on x86. get_page_from_gfn is missing.
> 
> Adding #include <asm/p2m.h> on top of the file (common/domain.c)
> resolves the compilation error.

Thanks, I'll do that.