[PATCH 2/5] x86/vioapic: allow the vIO APIC to have a variable number of pins

[Roger Pau Monne <roger.pau@citrix.com>]

Altough it's still always set to VIOAPIC_NUM_PINS (48).

Add a new field to the hvm_hw_ioapic struct to contain the number of pins
(number of IO redirection table entries), and add the migration compatibility
code.

Signed-off-by: Roger Pau Monné <roger.pau@citrix.com>
---
Cc: Ian Jackson <ian.jackson@eu.citrix.com>
Cc: Wei Liu <wei.liu2@citrix.com>
Cc: Jan Beulich <jbeulich@suse.com>
Cc: Andrew Cooper <andrew.cooper3@citrix.com>
---
 tools/misc/xen-hvmctx.c                |  25 +++++-
 xen/arch/x86/hvm/vioapic.c             | 138 +++++++++++++++++++++++++++++----
 xen/include/public/arch-x86/hvm/save.h |  50 +++++++-----
 3 files changed, 177 insertions(+), 36 deletions(-)

diff --git a/tools/misc/xen-hvmctx.c b/tools/misc/xen-hvmctx.c
index 32be120..8135a0e 100644
--- a/tools/misc/xen-hvmctx.c
+++ b/tools/misc/xen-hvmctx.c
@@ -227,14 +227,31 @@ static void dump_pic(void)
 static void dump_ioapic(void) 
 {
     int i;
-    HVM_SAVE_TYPE(IOAPIC) p;
-    READ(p);
+    struct hvm_hw_vioapic p;
+    union vioapic_redir_entry redirtbl[VIOAPIC_NUM_PINS];
+
+    /*
+     * NB: due to the fact that the IO APIC struct can have a variable number
+     * of pins (in order to support PVHv2 Dom0), the migration code needs to
+     * support this structure, although migration of guests with a number of
+     * pins different than VIOAPIC_NUM_PINS is not supported.
+     */
+    memcpy(&p, buf + off, offsetof(struct hvm_hw_vioapic, redirtbl));
+    off += offsetof(struct hvm_hw_vioapic, redirtbl);
+    if ( p.nr_pins != VIOAPIC_NUM_PINS )
+    {
+        printf("Invalid number of IO APIC pins %u\n", p.nr_pins);
+        exit(EXIT_FAILURE);
+    }
+    memcpy(redirtbl, buf + off, sizeof(redirtbl));
+    off += sizeof(redirtbl);
+
     printf("    IOAPIC: base_address %#llx, ioregsel %#x id %#x\n",
            (unsigned long long) p.base_address, p.ioregsel, p.id);
     for ( i = 0; i < VIOAPIC_NUM_PINS; i++ )
     {
         printf("            pin %.2i: 0x%.16llx\n", i, 
-               (unsigned long long) p.redirtbl[i].bits);
+               (unsigned long long) redirtbl[i].bits);
     }
 }
 
@@ -453,7 +470,7 @@ int main(int argc, char **argv)
         case HVM_SAVE_CODE(HEADER): dump_header(); break;
         case HVM_SAVE_CODE(CPU): dump_cpu(); break;
         case HVM_SAVE_CODE(PIC): dump_pic(); break;
-        case HVM_SAVE_CODE(IOAPIC): dump_ioapic(); break;
+        case IOAPIC_CODE: dump_ioapic(); break;
         case HVM_SAVE_CODE(LAPIC): dump_lapic(); break;
         case HVM_SAVE_CODE(LAPIC_REGS): dump_lapic_regs(); break;
         case HVM_SAVE_CODE(PCI_IRQ): dump_pci_irq(); break;
diff --git a/xen/arch/x86/hvm/vioapic.c b/xen/arch/x86/hvm/vioapic.c
index 9677227..f469cbf 100644
--- a/xen/arch/x86/hvm/vioapic.c
+++ b/xen/arch/x86/hvm/vioapic.c
@@ -53,7 +53,7 @@ static uint32_t vioapic_read_indirect(const struct hvm_hw_vioapic *vioapic)
     case VIOAPIC_REG_VERSION:
         result = ((union IO_APIC_reg_01){
                   .bits = { .version = VIOAPIC_VERSION_ID,
-                            .entries = VIOAPIC_NUM_PINS - 1 }
+                            .entries = vioapic->nr_pins - 1 }
                   }).raw;
         break;
 
@@ -198,7 +198,7 @@ static void vioapic_write_indirect(struct domain *d, uint32_t val)
         HVM_DBG_LOG(DBG_LEVEL_IOAPIC, "rte[%02x].%s = %08x",
                     redir_index, vioapic->ioregsel & 1 ? "hi" : "lo", val);
 
-        if ( redir_index >= VIOAPIC_NUM_PINS )
+        if ( redir_index >= vioapic->nr_pins )
         {
             gdprintk(XENLOG_WARNING, "vioapic_write_indirect "
                      "error register %x\n", vioapic->ioregsel);
@@ -368,7 +368,7 @@ void vioapic_irq_positive_edge(struct domain *d, unsigned int irq)
 
     HVM_DBG_LOG(DBG_LEVEL_IOAPIC, "irq %x", irq);
 
-    ASSERT(irq < VIOAPIC_NUM_PINS);
+    ASSERT(irq < vioapic->nr_pins);
     ASSERT(spin_is_locked(&d->arch.hvm_domain.irq_lock));
 
     ent = &vioapic->redirtbl[irq];
@@ -397,7 +397,7 @@ void vioapic_update_EOI(struct domain *d, u8 vector)
 
     spin_lock(&d->arch.hvm_domain.irq_lock);
 
-    for ( gsi = 0; gsi < VIOAPIC_NUM_PINS; gsi++ )
+    for ( gsi = 0; gsi < vioapic->nr_pins; gsi++ )
     {
         ent = &vioapic->redirtbl[gsi];
         if ( ent->fields.vector != vector )
@@ -424,40 +424,141 @@ void vioapic_update_EOI(struct domain *d, u8 vector)
     spin_unlock(&d->arch.hvm_domain.irq_lock);
 }
 
-static int ioapic_save(struct domain *d, hvm_domain_context_t *h)
+#define VIOAPIC_SAVE_CONST offsetof(struct hvm_hw_vioapic, redirtbl)
+#define VIOAPIC_SAVE_VAR(cnt) (sizeof(union vioapic_redir_entry) * (cnt))
+#define VIOAPIC_SAVE_SIZE(cnt) (VIOAPIC_SAVE_CONST + VIOAPIC_SAVE_VAR(cnt))
+
+static int vioapic_save(struct domain *d, hvm_domain_context_t *h)
 {
-    struct hvm_hw_vioapic *s = domain_vioapic(d);
+    struct hvm_hw_vioapic *vioapic = domain_vioapic(d);
 
     if ( !has_vioapic(d) )
         return 0;
 
-    return hvm_save_entry(IOAPIC, 0, h, s);
+    if ( vioapic->nr_pins != VIOAPIC_NUM_PINS )
+        return -ENOSYS;
+
+    if ( _hvm_init_entry(h, IOAPIC_CODE, 0,
+                         VIOAPIC_SAVE_SIZE(vioapic->nr_pins)) )
+        return 1;
+
+    memcpy(&h->data[h->cur], vioapic, VIOAPIC_SAVE_CONST);
+    h->cur += VIOAPIC_SAVE_CONST;
+    memcpy(&h->data[h->cur], vioapic->redirtbl,
+           VIOAPIC_SAVE_VAR(vioapic->nr_pins));
+    h->cur += VIOAPIC_SAVE_VAR(vioapic->nr_pins);
+
+    return 0;
 }
 
-static int ioapic_load(struct domain *d, hvm_domain_context_t *h)
+static int vioapic_load(struct domain *d, hvm_domain_context_t *h)
 {
-    struct hvm_hw_vioapic *s = domain_vioapic(d);
+    unsigned int ioapic_nr = hvm_load_instance(h);
+    const struct hvm_save_descriptor *desc;
+    struct hvm_hw_vioapic_compat *ioapic_compat;
+    struct hvm_hw_vioapic *ioapic = domain_vioapic(d);
 
     if ( !has_vioapic(d) )
         return -ENODEV;
 
-    return hvm_load_entry(IOAPIC, h, s);
+    if ( ioapic_nr != 0 )
+        return -ENOSYS;
+
+    desc = (struct hvm_save_descriptor *)&h->data[h->cur];
+    if ( sizeof (*desc) > h->size - h->cur)
+    {
+        printk(XENLOG_G_WARNING
+               "HVM%d restore: not enough data left to read IOAPIC descriptor\n",
+               d->domain_id);
+        return -ENODATA;
+    }
+    if ( desc->length + sizeof (*desc) > h->size - h->cur)
+    {
+        printk(XENLOG_G_WARNING
+               "HVM%d restore: not enough data left to read %u IOAPIC bytes\n",
+               d->domain_id, desc->length);
+        return -ENODATA;
+    }
+    if ( desc->length < sizeof(*ioapic_compat) )
+    {
+        printk(XENLOG_G_WARNING
+               "HVM%d restore mismatch: IOAPIC length %u < %lu\n",
+               d->domain_id, desc->length, sizeof(*ioapic_compat));
+        return -EINVAL;
+    }
+
+    h->cur += sizeof(*desc);
+
+    switch ( desc->length )
+    {
+    case sizeof(*ioapic_compat):
+        ioapic_compat = (struct hvm_hw_vioapic_compat *)&h->data[h->cur];
+        ioapic->base_address = ioapic_compat->base_address;
+        ioapic->ioregsel = ioapic_compat->ioregsel;
+        ioapic->id = ioapic_compat->id;
+        ioapic->nr_pins = VIOAPIC_NUM_PINS;
+        memcpy(ioapic->redirtbl, ioapic_compat->redirtbl,
+               sizeof(ioapic_compat->redirtbl));
+        h->cur += sizeof(*ioapic_compat);
+        break;
+    case VIOAPIC_SAVE_SIZE(VIOAPIC_NUM_PINS):
+        memcpy(ioapic, &h->data[h->cur], VIOAPIC_SAVE_CONST);
+        h->cur += VIOAPIC_SAVE_CONST;
+        if ( ioapic->nr_pins != VIOAPIC_NUM_PINS )
+        {
+            printk(XENLOG_G_WARNING
+                   "HVM%d restore mismatch: unexpected number of IO APIC entries: %u\n",
+                   d->domain_id, ioapic->nr_pins);
+            return -EINVAL;
+        }
+        memcpy(ioapic->redirtbl, &h->data[h->cur],
+               VIOAPIC_SAVE_VAR(ioapic->nr_pins));
+        h->cur += VIOAPIC_SAVE_VAR(ioapic->nr_pins);
+        break;
+    default:
+        printk(XENLOG_G_WARNING "HVM%d restore mismatch: IO APIC length\n",
+               d->domain_id);
+        return -EINVAL;
+    }
+
+    return 0;
+}
+
+/*
+ * We need variable length (variable number of pins) IO APICs, although
+ * those would only be used by the hardware domain, so migration wise
+ * we are always going to use VIOAPIC_NUM_PINS.
+ */
+static int __init vioapic_register_save_and_restore(void)
+{
+    hvm_register_savevm(IOAPIC_CODE, "IOAPIC", vioapic_save, vioapic_load,
+                        VIOAPIC_SAVE_SIZE(VIOAPIC_NUM_PINS) +
+                            sizeof(struct hvm_save_descriptor),
+                        HVMSR_PER_DOM);
+
+    return 0;
 }
+__initcall(vioapic_register_save_and_restore);
 
-HVM_REGISTER_SAVE_RESTORE(IOAPIC, ioapic_save, ioapic_load, 1, HVMSR_PER_DOM);
+#undef VIOAPIC_SAVE_CONST
+#undef VIOAPIC_SAVE_VAR
+#undef VIOAPIC_SAVE_SIZE
 
 void vioapic_reset(struct domain *d)
 {
     struct hvm_hw_vioapic *vioapic = domain_vioapic(d);
-    int i;
+    unsigned int i;
 
     if ( !has_vioapic(d) )
         return;
 
-    memset(vioapic, 0, sizeof(*vioapic));
-    for ( i = 0; i < VIOAPIC_NUM_PINS; i++ )
+    memset(vioapic->redirtbl, 0,
+           sizeof(*vioapic->redirtbl) * vioapic->nr_pins);
+    for ( i = 0; i < vioapic->nr_pins; i++ )
         vioapic->redirtbl[i].fields.mask = 1;
     vioapic->base_address = VIOAPIC_DEFAULT_BASE_ADDRESS;
+    vioapic->id = 0;
+    vioapic->ioregsel = 0;
 }
 
 int vioapic_init(struct domain *d)
@@ -470,6 +571,15 @@ int vioapic_init(struct domain *d)
            xmalloc(struct hvm_hw_vioapic)) == NULL) )
         return -ENOMEM;
 
+    domain_vioapic(d)->redirtbl = xmalloc_array(union vioapic_redir_entry,
+                                                VIOAPIC_NUM_PINS);
+    if ( !domain_vioapic(d)->redirtbl )
+    {
+        xfree(d->arch.hvm_domain.vioapic);
+        return -ENOMEM;
+    }
+
+    domain_vioapic(d)->nr_pins = VIOAPIC_NUM_PINS;
     vioapic_reset(d);
 
     register_mmio_handler(d, &vioapic_mmio_ops);
diff --git a/xen/include/public/arch-x86/hvm/save.h b/xen/include/public/arch-x86/hvm/save.h
index 419a3b2..a218804 100644
--- a/xen/include/public/arch-x86/hvm/save.h
+++ b/xen/include/public/arch-x86/hvm/save.h
@@ -363,30 +363,44 @@ DECLARE_HVM_SAVE_TYPE(PIC, 3, struct hvm_hw_vpic);
 
 #define VIOAPIC_NUM_PINS  48 /* 16 ISA IRQs, 32 non-legacy PCI IRQS. */
 
+/*
+ * Needs to be declared independently of the ioapic structs, or else the
+ * compat check fails.
+ */
+union vioapic_redir_entry
+{
+    uint64_t bits;
+    struct {
+        uint8_t vector;
+        uint8_t delivery_mode:3;
+        uint8_t dest_mode:1;
+        uint8_t delivery_status:1;
+        uint8_t polarity:1;
+        uint8_t remote_irr:1;
+        uint8_t trig_mode:1;
+        uint8_t mask:1;
+        uint8_t reserve:7;
+        uint8_t reserved[4];
+        uint8_t dest_id;
+    } fields;
+};
+
 struct hvm_hw_vioapic {
     uint64_t base_address;
     uint32_t ioregsel;
     uint32_t id;
-    union vioapic_redir_entry
-    {
-        uint64_t bits;
-        struct {
-            uint8_t vector;
-            uint8_t delivery_mode:3;
-            uint8_t dest_mode:1;
-            uint8_t delivery_status:1;
-            uint8_t polarity:1;
-            uint8_t remote_irr:1;
-            uint8_t trig_mode:1;
-            uint8_t mask:1;
-            uint8_t reserve:7;
-            uint8_t reserved[4];
-            uint8_t dest_id;
-        } fields;
-    } redirtbl[VIOAPIC_NUM_PINS];
+    uint32_t nr_pins;
+    union vioapic_redir_entry *redirtbl;
+};
+
+struct hvm_hw_vioapic_compat {
+    uint64_t base_address;
+    uint32_t ioregsel;
+    uint32_t id;
+    union vioapic_redir_entry redirtbl[VIOAPIC_NUM_PINS];
 };
 
-DECLARE_HVM_SAVE_TYPE(IOAPIC, 4, struct hvm_hw_vioapic);
+#define IOAPIC_CODE  4
 
 
 /*
-- 
2.10.1 (Apple Git-78)


_______________________________________________
Xen-devel mailing list
Xen-devel@lists.xen.org
https://lists.xen.org/xen-devel