[PATCH] Fixed legacy issues when extends number of vcpus > 32

* [PATCH] Fixed legacy issues when extends number of vcpus > 32
@ 2009-08-16  2:17 Zhang, Xiantao
  2009-08-16  7:56 ` Keir Fraser
  0 siblings, 1 reply; 6+ messages in thread
From: Zhang, Xiantao @ 2009-08-16  2:17 UTC (permalink / raw)
  To: Keir Fraser; +Cc: xen-devel

[-- Attachment #1: Type: text/plain, Size: 265 bytes --]

Hi, Keir
  Attached two patches fixed the legacy issues after extending number of vcpus to more for HVM domain.  The first one fixed vcpu bitmask issue for interrupt delivery, and the second one fixed vpid shortage issue after extending number of vcpus.  
Xiantao

[-- Attachment #2: 0002-only_allocate_vpid_for_intilized_vcpus.patch --]
[-- Type: application/octet-stream, Size: 4650 bytes --]

# HG changeset patch
# User Xiantao Zhang <xiantao.zhang@intel.com>
# Date 1250148086 -28800
# Node ID aea1e831ee747ee62733d6d7cd2e3ac9b7425f6c
# Parent  64d5844f21055698aaa765ea880b0f58e169601a
x86: Only allocate vpid for intilized vcpus.

Currently, 32 vpids are allocated for each
domain statically, it blocks to support more
vcpus for HVM domain, so remove the limit and
only allocate vpid for intilized vcpus. In this
way, vpid can be non-contiguous for vcpus of one
single domain.

Signed-off-by: Xiantao Zhang <xiantao.zhang@intel.com>

diff -r 64d5844f2105 -r aea1e831ee74 xen/arch/x86/hvm/vmx/vmcs.c

--- a/xen/arch/x86/hvm/vmx/vmcs.c	Thu Aug 13 13:35:59 2009 +0800
+++ b/xen/arch/x86/hvm/vmx/vmcs.c	Thu Aug 13 15:21:26 2009 +0800
@@ -732,11 +732,7 @@ static int construct_vmcs(struct vcpu *v
     }
 
     if ( cpu_has_vmx_vpid )
-    {
-        v->arch.hvm_vmx.vpid =
-            v->domain->arch.hvm_domain.vmx.vpid_base + v->vcpu_id;
         __vmwrite(VIRTUAL_PROCESSOR_ID, v->arch.hvm_vmx.vpid);
-    }
 
     if ( cpu_has_vmx_pat && paging_mode_hap(d) )
     {
diff -r 64d5844f2105 -r aea1e831ee74 xen/arch/x86/hvm/vmx/vmx.c
--- a/xen/arch/x86/hvm/vmx/vmx.c	Thu Aug 13 13:35:59 2009 +0800
+++ b/xen/arch/x86/hvm/vmx/vmx.c	Thu Aug 13 15:21:26 2009 +0800
@@ -59,8 +59,8 @@ static void vmx_ctxt_switch_to(struct vc
 
 static int  vmx_alloc_vlapic_mapping(struct domain *d);
 static void vmx_free_vlapic_mapping(struct domain *d);
-static int  vmx_alloc_vpid(struct domain *d);
-static void vmx_free_vpid(struct domain *d);
+static int  vmx_alloc_vpid(struct vcpu *v);
+static void vmx_free_vpid(struct vcpu *v);
 static void vmx_install_vlapic_mapping(struct vcpu *v);
 static void vmx_update_guest_cr(struct vcpu *v, unsigned int cr);
 static void vmx_update_guest_efer(struct vcpu *v);
@@ -82,14 +82,9 @@ static int vmx_domain_initialise(struct 
     d->arch.hvm_domain.vmx.ept_control.asr  =
         pagetable_get_pfn(d->arch.phys_table);
 
-    if ( (rc = vmx_alloc_vpid(d)) != 0 )
+
+    if ( (rc = vmx_alloc_vlapic_mapping(d)) != 0 )
         return rc;
-
-    if ( (rc = vmx_alloc_vlapic_mapping(d)) != 0 )
-    {
-        vmx_free_vpid(d);
-        return rc;
-    }
 
     return 0;
 }
@@ -98,7 +93,6 @@ static void vmx_domain_destroy(struct do
 {
     ept_sync_domain(d);
     vmx_free_vlapic_mapping(d);
-    vmx_free_vpid(d);
 }
 
 static int vmx_vcpu_initialise(struct vcpu *v)
@@ -106,6 +100,9 @@ static int vmx_vcpu_initialise(struct vc
     int rc;
 
     spin_lock_init(&v->arch.hvm_vmx.vmcs_lock);
+
+    if ( (rc = vmx_alloc_vpid(v)) != 0 )
+        return rc;
 
     v->arch.schedule_tail    = vmx_do_resume;
     v->arch.ctxt_switch_from = vmx_ctxt_switch_from;
@@ -116,6 +113,7 @@ static int vmx_vcpu_initialise(struct vc
         dprintk(XENLOG_WARNING,
                 "Failed to create VMCS for vcpu %d: err=%d.\n",
                 v->vcpu_id, rc);
+        vmx_free_vpid(v);
         return rc;
     }
 
@@ -135,6 +133,7 @@ static void vmx_vcpu_destroy(struct vcpu
     vmx_destroy_vmcs(v);
     vpmu_destroy(v);
     passive_domain_destroy(v);
+    vmx_free_vpid(v);
 }
 
 #ifdef __x86_64__
@@ -1398,7 +1397,7 @@ static struct hvm_function_table vmx_fun
 };
 
 static unsigned long *vpid_bitmap;
-#define VPID_BITMAP_SIZE ((1u << VMCS_VPID_WIDTH) / XEN_LEGACY_MAX_VCPUS)
+#define VPID_BITMAP_SIZE (1u << VMCS_VPID_WIDTH)
 
 void start_vmx(void)
 {
@@ -1903,7 +1902,7 @@ static void vmx_free_vlapic_mapping(stru
         free_xenheap_page(mfn_to_virt(mfn));
 }
 
-static int vmx_alloc_vpid(struct domain *d)
+static int vmx_alloc_vpid(struct vcpu *v)
 {
     int idx;
 
@@ -1920,17 +1919,17 @@ static int vmx_alloc_vpid(struct domain 
     }
     while ( test_and_set_bit(idx, vpid_bitmap) );
 
-    d->arch.hvm_domain.vmx.vpid_base = idx * XEN_LEGACY_MAX_VCPUS;
+    v->arch.hvm_vmx.vpid = idx;
     return 0;
 }
 
-static void vmx_free_vpid(struct domain *d)
+static void vmx_free_vpid(struct vcpu *v)
 {
     if ( !cpu_has_vmx_vpid )
         return;
 
-    clear_bit(d->arch.hvm_domain.vmx.vpid_base / XEN_LEGACY_MAX_VCPUS,
-              vpid_bitmap);
+    if ( v->arch.hvm_vmx.vpid )
+        clear_bit(v->arch.hvm_vmx.vpid, vpid_bitmap);
 }
 
 static void vmx_install_vlapic_mapping(struct vcpu *v)
diff -r 64d5844f2105 -r aea1e831ee74 xen/include/asm-x86/hvm/vmx/vmcs.h
--- a/xen/include/asm-x86/hvm/vmx/vmcs.h	Thu Aug 13 13:35:59 2009 +0800
+++ b/xen/include/asm-x86/hvm/vmx/vmcs.h	Thu Aug 13 15:21:26 2009 +0800
@@ -58,7 +58,6 @@ struct vmx_msr_state {
 
 struct vmx_domain {
     unsigned long apic_access_mfn;
-    unsigned long vpid_base;
     union {
         struct {
             u64 etmt :3,

[-- Attachment #3: 0001-use_cpumask_t_as_vcpu_bitmap.patch --]
[-- Type: application/octet-stream, Size: 11907 bytes --]

# HG changeset patch
# User Xiantao Zhang <xiantao.zhang@intel.com>
# Date 1250388568 -28800
# Node ID fa701b6d4a56470ef9626f083e57796ab20cfcbc
# Parent  da620c4549166a436424fecf922d14f35d2522b7
x86: Fix vcpu bitmap issue when delivery interrupts.

Currently, since only use uint32_t as bitmap for vcpu's interrupt
delivery bitmask, so vcpus whose vcpu_id is bigger than 31 can't
receive any interrupts, and it may crash guests in some cases.
Insteadly, this patch uses cpumask_t to replace uint32_t as vcpu
bitmask, but it is required to constrain MAX_VIRT_VCPUS < NR_CPUS.

Signed-off-by: Xiantao Zhang <xiantao.zhang@intel.com>

diff -r da620c454916 -r fa701b6d4a56 xen/arch/x86/hvm/vioapic.c
--- a/xen/arch/x86/hvm/vioapic.c	Thu Aug 13 08:40:39 2009 +0100
+++ b/xen/arch/x86/hvm/vioapic.c	Sun Aug 16 10:09:28 2009 +0800
@@ -263,21 +263,23 @@ static void ioapic_inj_irq(
         vcpu_kick(vlapic_vcpu(target));
 }
 
-static uint32_t ioapic_get_delivery_bitmask(
+static cpumask_t ioapic_get_delivery_bitmask(
     struct hvm_hw_vioapic *vioapic, uint16_t dest, uint8_t dest_mode)
 {
-    uint32_t mask = 0;
+    cpumask_t mask;
     struct vcpu *v;
 
     HVM_DBG_LOG(DBG_LEVEL_IOAPIC, "dest %d dest_mode %d",
                 dest, dest_mode);
 
+    cpus_clear(mask);
+
     if ( dest_mode == 0 ) /* Physical mode. */
     {
         if ( dest == 0xFF ) /* Broadcast. */
         {
             for_each_vcpu ( vioapic_domain(vioapic), v )
-                mask |= 1 << v->vcpu_id;
+                cpu_set(v->vcpu_id, mask);
             goto out;
         }
 
@@ -285,7 +287,7 @@ static uint32_t ioapic_get_delivery_bitm
         {
             if ( VLAPIC_ID(vcpu_vlapic(v)) == dest )
             {
-                mask = 1 << v->vcpu_id;
+                cpu_set(v->vcpu_id, mask);
                 break;
             }
         }
@@ -294,12 +296,12 @@ static uint32_t ioapic_get_delivery_bitm
     {
         for_each_vcpu ( vioapic_domain(vioapic), v )
             if ( vlapic_match_logical_addr(vcpu_vlapic(v), dest) )
-                mask |= 1 << v->vcpu_id;
+                cpu_set(v->vcpu_id, mask);
     }
 
  out:
-    HVM_DBG_LOG(DBG_LEVEL_IOAPIC, "mask %x",
-                mask);
+    HVM_DBG_LOG(DBG_LEVEL_IOAPIC, "mask :0x%x",
+                *(int*)mask.bits);
     return mask;
 }
 
@@ -316,7 +318,7 @@ static void vioapic_deliver(struct hvm_h
     uint8_t delivery_mode = vioapic->redirtbl[irq].fields.delivery_mode;
     uint8_t vector = vioapic->redirtbl[irq].fields.vector;
     uint8_t trig_mode = vioapic->redirtbl[irq].fields.trig_mode;
-    uint32_t deliver_bitmask;
+    cpumask_t  deliver_bitmask;
     struct vlapic *target;
     struct vcpu *v;
 
@@ -328,7 +330,7 @@ static void vioapic_deliver(struct hvm_h
                 dest, dest_mode, delivery_mode, vector, trig_mode);
 
     deliver_bitmask = ioapic_get_delivery_bitmask(vioapic, dest, dest_mode);
-    if ( !deliver_bitmask )
+    if ( cpus_empty(deliver_bitmask) )
     {
         HVM_DBG_LOG(DBG_LEVEL_IOAPIC, "no target on destination");
         return;
@@ -358,19 +360,19 @@ static void vioapic_deliver(struct hvm_h
         {
             HVM_DBG_LOG(DBG_LEVEL_IOAPIC, "null round robin: "
                         "mask=%x vector=%x delivery_mode=%x",
-                        deliver_bitmask, vector, dest_LowestPrio);
+                        *(int*)deliver_bitmask.bits, vector, dest_LowestPrio);
         }
         break;
     }
 
     case dest_Fixed:
     {
-        uint8_t bit;
-        for ( bit = 0; deliver_bitmask != 0; bit++ )
-        {
-            if ( !(deliver_bitmask & (1 << bit)) )
-                continue;
-            deliver_bitmask &= ~(1 << bit);
+        uint16_t vcpu_id;
+        for ( vcpu_id = 0; !cpus_empty(deliver_bitmask); vcpu_id++ )
+        {
+            if ( !cpu_test_and_clear(vcpu_id, deliver_bitmask) )
+               continue;
+
             if ( vioapic_domain(vioapic)->vcpu == NULL )
                 v = NULL;
 #ifdef IRQ0_SPECIAL_ROUTING
@@ -379,7 +381,7 @@ static void vioapic_deliver(struct hvm_h
                 v = vioapic_domain(vioapic)->vcpu[0];
 #endif
             else
-                v = vioapic_domain(vioapic)->vcpu[bit];
+                v = vioapic_domain(vioapic)->vcpu[vcpu_id];
             if ( v != NULL )
             {
                 target = vcpu_vlapic(v);
@@ -392,14 +394,14 @@ static void vioapic_deliver(struct hvm_h
 
     case dest_NMI:
     {
-        uint8_t bit;
-        for ( bit = 0; deliver_bitmask != 0; bit++ )
-        {
-            if ( !(deliver_bitmask & (1 << bit)) )
-                continue;
-            deliver_bitmask &= ~(1 << bit);
+        uint16_t vcpu_id;
+        for ( vcpu_id = 0; !cpus_empty(deliver_bitmask); vcpu_id++ )
+        {
+            if ( !cpu_test_and_clear(vcpu_id, deliver_bitmask) )
+               continue;
+
             if ( (vioapic_domain(vioapic)->vcpu != NULL) &&
-                 ((v = vioapic_domain(vioapic)->vcpu[bit]) != NULL) &&
+                 ((v = vioapic_domain(vioapic)->vcpu[vcpu_id]) != NULL) &&
                  !test_and_set_bool(v->nmi_pending) )
                 vcpu_kick(v);
         }
diff -r da620c454916 -r fa701b6d4a56 xen/arch/x86/hvm/vlapic.c
--- a/xen/arch/x86/hvm/vlapic.c	Thu Aug 13 08:40:39 2009 +0100
+++ b/xen/arch/x86/hvm/vlapic.c	Sun Aug 16 10:09:28 2009 +0800
@@ -377,7 +377,7 @@ static int vlapic_accept_irq(struct vcpu
 }
 
 /* This function is used by both ioapic and lapic.The bitmap is for vcpu_id. */
-struct vlapic *apic_lowest_prio(struct domain *d, uint32_t bitmap)
+struct vlapic *apic_lowest_prio(struct domain *d, cpumask_t mask)
 {
     int old = d->arch.hvm_domain.irq.round_robin_prev_vcpu;
     uint32_t ppr, target_ppr = UINT_MAX;
@@ -390,7 +390,7 @@ struct vlapic *apic_lowest_prio(struct d
     do {
         v = v->next_in_list ? : d->vcpu[0];
         vlapic = vcpu_vlapic(v);
-        if ( test_bit(v->vcpu_id, &bitmap) && vlapic_enabled(vlapic) &&
+        if ( test_bit(v->vcpu_id, mask.bits) && vlapic_enabled(vlapic) &&
              ((ppr = vlapic_get_ppr(vlapic)) < target_ppr) )
         {
             target = vlapic;
@@ -434,7 +434,7 @@ int vlapic_ipi(
 
     struct vlapic *target;
     struct vcpu *v;
-    uint32_t lpr_map = 0;
+    cpumask_t lpr_cpu_map;
     int rc = X86EMUL_OKAY;
 
     HVM_DBG_LOG(DBG_LEVEL_VLAPIC, "icr_high 0x%x, icr_low 0x%x, "
@@ -443,12 +443,14 @@ int vlapic_ipi(
                 icr_high, icr_low, short_hand, dest,
                 trig_mode, level, dest_mode, delivery_mode, vector);
 
+    cpus_clear(lpr_cpu_map);
+
     for_each_vcpu ( vlapic_domain(vlapic), v )
     {
         if ( vlapic_match_dest(v, vlapic, short_hand, dest, dest_mode) )
         {
             if ( delivery_mode == APIC_DM_LOWEST )
-                __set_bit(v->vcpu_id, &lpr_map);
+                cpu_set(v->vcpu_id, lpr_cpu_map);
             else
                 rc = vlapic_accept_irq(v, delivery_mode,
                                        vector, level, trig_mode);
@@ -460,7 +462,7 @@ int vlapic_ipi(
 
     if ( delivery_mode == APIC_DM_LOWEST )
     {
-        target = apic_lowest_prio(vlapic_domain(vlapic), lpr_map);
+        target = apic_lowest_prio(vlapic_domain(vlapic), lpr_cpu_map);
         if ( target != NULL )
             rc = vlapic_accept_irq(vlapic_vcpu(target), delivery_mode,
                                    vector, level, trig_mode);
diff -r da620c454916 -r fa701b6d4a56 xen/arch/x86/hvm/vmsi.c
--- a/xen/arch/x86/hvm/vmsi.c	Thu Aug 13 08:40:39 2009 +0100
+++ b/xen/arch/x86/hvm/vmsi.c	Sun Aug 16 10:09:28 2009 +0800
@@ -40,10 +40,10 @@
 #include <asm/current.h>
 #include <asm/event.h>
 
-static uint32_t vmsi_get_delivery_bitmask(
+static cpumask_t vmsi_get_delivery_bitmask(
     struct domain *d, uint16_t dest, uint8_t dest_mode)
 {
-    uint32_t mask = 0;
+    cpumask_t mask;
     struct vcpu *v;
 
     HVM_DBG_LOG(DBG_LEVEL_IOAPIC, "ioapic_get_delivery_bitmask "
@@ -54,7 +54,7 @@ static uint32_t vmsi_get_delivery_bitmas
         if ( dest == 0xFF ) /* Broadcast. */
         {
             for_each_vcpu ( d, v )
-                mask |= 1 << v->vcpu_id;
+                cpu_set(v->vcpu_id, mask);
             goto out;
         }
 
@@ -62,7 +62,7 @@ static uint32_t vmsi_get_delivery_bitmas
         {
             if ( VLAPIC_ID(vcpu_vlapic(v)) == dest )
             {
-                mask = 1 << v->vcpu_id;
+                cpu_set(v->vcpu_id, mask);
                 break;
             }
         }
@@ -71,12 +71,12 @@ static uint32_t vmsi_get_delivery_bitmas
     {
         for_each_vcpu ( d, v )
             if ( vlapic_match_logical_addr(vcpu_vlapic(v), dest) )
-                mask |= 1 << v->vcpu_id;
+                cpu_set(v->vcpu_id, mask);
     }
 
  out:
-    HVM_DBG_LOG(DBG_LEVEL_IOAPIC, "ioapic_get_delivery_bitmask mask %x\n",
-                mask);
+    HVM_DBG_LOG(DBG_LEVEL_IOAPIC, "ioapic_get_delivery_bitmask mask 0x%x\n",
+                *(int*)mask.bits);
     return mask;
 }
 
@@ -125,7 +125,7 @@ int vmsi_deliver(struct domain *d, int p
     uint8_t dest_mode = (flags & VMSI_DM_MASK) >> GFLAGS_SHIFT_DM;
     uint8_t delivery_mode = (flags & VMSI_DELIV_MASK) >> GLFAGS_SHIFT_DELIV_MODE;
     uint8_t trig_mode = (flags & VMSI_TRIG_MODE) >> GLFAGS_SHIFT_TRG_MODE;
-    uint32_t deliver_bitmask;
+    cpumask_t deliver_bitmask;
     struct vlapic *target;
     struct vcpu *v;
 
@@ -141,7 +141,7 @@ int vmsi_deliver(struct domain *d, int p
     }
 
     deliver_bitmask = vmsi_get_delivery_bitmask(d, dest, dest_mode);
-    if ( !deliver_bitmask )
+    if ( cpus_empty(deliver_bitmask) )
     {
         HVM_DBG_LOG(DBG_LEVEL_IOAPIC, "ioapic deliver "
                     "no target on destination\n");
@@ -158,20 +158,20 @@ int vmsi_deliver(struct domain *d, int p
         else
             HVM_DBG_LOG(DBG_LEVEL_IOAPIC, "null round robin: "
                         "mask=%x vector=%x delivery_mode=%x\n",
-                        deliver_bitmask, vector, dest_LowestPrio);
+                        *(int*)deliver_bitmask.bits, vector, dest_LowestPrio);
         break;
     }
 
     case dest_Fixed:
     case dest_ExtINT:
     {
-        uint8_t bit;
-        for ( bit = 0; deliver_bitmask != 0; bit++ )
+        uint16_t vcpu_id;
+        for ( vcpu_id = 0; !cpus_empty(deliver_bitmask); vcpu_id++ )
         {
-            if ( !(deliver_bitmask & (1 << bit)) )
-                continue;
-            deliver_bitmask &= ~(1 << bit);
-            v = d->vcpu[bit];
+            if ( !cpu_test_and_clear(vcpu_id, deliver_bitmask) )
+               continue;
+
+            v = d->vcpu[vcpu_id];
             if ( v != NULL )
             {
                 target = vcpu_vlapic(v);
diff -r da620c454916 -r fa701b6d4a56 xen/include/asm-x86/config.h
--- a/xen/include/asm-x86/config.h	Thu Aug 13 08:40:39 2009 +0100
+++ b/xen/include/asm-x86/config.h	Sun Aug 16 10:09:28 2009 +0800
@@ -358,7 +358,7 @@ extern unsigned long xenheap_phys_end;
 #define GDT_LDT_MBYTES           (MAX_VIRT_CPUS >> (20-GDT_LDT_VCPU_VA_SHIFT))
 #else
 #define GDT_LDT_MBYTES           PERDOMAIN_MBYTES
-#define MAX_VIRT_CPUS            (GDT_LDT_MBYTES << (20-GDT_LDT_VCPU_VA_SHIFT))
+#define MAX_VIRT_CPUS            NR_CPUS
 #endif
 #define GDT_LDT_VIRT_START       PERDOMAIN_VIRT_START
 #define GDT_LDT_VIRT_END         (GDT_LDT_VIRT_START + (GDT_LDT_MBYTES << 20))
diff -r da620c454916 -r fa701b6d4a56 xen/include/asm-x86/hvm/vlapic.h
--- a/xen/include/asm-x86/hvm/vlapic.h	Thu Aug 13 08:40:39 2009 +0100
+++ b/xen/include/asm-x86/hvm/vlapic.h	Sun Aug 16 10:09:28 2009 +0800
@@ -94,7 +94,7 @@ int vlapic_accept_pic_intr(struct vcpu *
 
 void vlapic_adjust_i8259_target(struct domain *d);
 
-struct vlapic *apic_lowest_prio(struct domain *d, uint32_t bitmap);
+struct vlapic *apic_lowest_prio(struct domain *d, cpumask_t mask);
 
 int vlapic_match_logical_addr(struct vlapic *vlapic, uint8_t mda);
 

[-- Attachment #4: Type: text/plain, Size: 138 bytes --]

_______________________________________________
Xen-devel mailing list
Xen-devel@lists.xensource.com
http://lists.xensource.com/xen-devel

^ permalink raw reply	[flat|nested] 6+ messages in thread