[PATCH v2 0/4] x86/MSI-X: XSA-120 follow-up

[PATCH v2 0/4] x86/MSI-X: XSA-120 follow-up

[Jan Beulich]

The problem requiring the first patch here is actually what lead to
XSA-120.

1: be more careful during teardown
2: access MSI-X table only after having enabled MSI-X
3: reduce fiddling with control register during restore
4: cleanup

Signed-off-by: Jan Beulich <jbeulich@suse.com>

(Only patch 1 really changed - see there for details.)

[PATCH v2 1/4] x86/MSI-X: be more careful during teardown

[Jan Beulich]

[-- Attachment #1: Type: text/plain, Size: 11096 bytes --]

When a device gets detached from a guest, pciback will clear its
command register, thus disabling both memory and I/O decoding. The
disabled memory decoding, however, has an effect on the MSI-X table
accesses the hypervisor does: These won't have the intended effect
anymore. Even worse, for PCIe devices (but not SR-IOV virtual
functions) such accesses may (will?) be treated as Unsupported
Requests, causing respective errors to be surfaced, potentially in the
form of NMIs that may be fatal to the hypervisor or Dom0 is different
ways. Hence rather than carrying out these accesses, we should avoid
them where we can, and use alternative (e.g. PCI config space based)
mechanisms to achieve at least the same effect.

Signed-off-by: Jan Beulich <jbeulich@suse.com>
---
v2: mask_msi_irq()'s BUG() needs to be conditional upon IRQ_DISABLED
    not already being set, as ->shutdown() may be called on error paths
    when the IRQ never got enabled. This in turn required the changes
    to irq.c.

Backporting note (largely to myself):
   Depends on (not yet backported) commit 061eebe0e "x86/MSI: drop
   workaround for insecure Dom0 kernels" (due to re-use of struct
   arch_msix's warned field).

--- a/xen/arch/x86/irq.c
+++ b/xen/arch/x86/irq.c
@@ -217,9 +217,9 @@ void destroy_irq(unsigned int irq)
     }
 
     spin_lock_irqsave(&desc->lock, flags);
-    desc->status  |= IRQ_DISABLED;
     desc->status  &= ~IRQ_GUEST;
     desc->handler->shutdown(desc);
+    desc->status |= IRQ_DISABLED;
     action = desc->action;
     desc->action  = NULL;
     desc->msi_desc = NULL;
@@ -995,8 +995,8 @@ void __init release_irq(unsigned int irq
     spin_lock_irqsave(&desc->lock,flags);
     action = desc->action;
     desc->action  = NULL;
-    desc->status |= IRQ_DISABLED;
     desc->handler->shutdown(desc);
+    desc->status |= IRQ_DISABLED;
     spin_unlock_irqrestore(&desc->lock,flags);
 
     /* Wait to make sure it's not being used on another CPU */
@@ -1725,8 +1725,8 @@ static irq_guest_action_t *__pirq_guest_
     BUG_ON(action->in_flight != 0);
 
     /* Disabling IRQ before releasing the desc_lock avoids an IRQ storm. */
-    desc->status |= IRQ_DISABLED;
     desc->handler->disable(desc);
+    desc->status |= IRQ_DISABLED;
 
     /*
      * Mark any remaining pending EOIs as ready to flush.
--- a/xen/arch/x86/msi.c
+++ b/xen/arch/x86/msi.c
@@ -121,6 +121,27 @@ static void msix_put_fixmap(struct arch_
     spin_unlock(&msix->table_lock);
 }
 
+static bool_t memory_decoded(const struct pci_dev *dev)
+{
+    u8 bus, slot, func;
+
+    if ( !dev->info.is_virtfn )
+    {
+        bus = dev->bus;
+        slot = PCI_SLOT(dev->devfn);
+        func = PCI_FUNC(dev->devfn);
+    }
+    else
+    {
+        bus = dev->info.physfn.bus;
+        slot = PCI_SLOT(dev->info.physfn.devfn);
+        func = PCI_FUNC(dev->info.physfn.devfn);
+    }
+
+    return !!(pci_conf_read16(dev->seg, bus, slot, func, PCI_COMMAND) &
+              PCI_COMMAND_MEMORY);
+}
+
 /*
  * MSI message composition
  */
@@ -162,7 +183,7 @@ void msi_compose_msg(unsigned vector, co
     }
 }
 
-static void read_msi_msg(struct msi_desc *entry, struct msi_msg *msg)
+static bool_t read_msi_msg(struct msi_desc *entry, struct msi_msg *msg)
 {
     switch ( entry->msi_attrib.type )
     {
@@ -198,6 +219,8 @@ static void read_msi_msg(struct msi_desc
         void __iomem *base;
         base = entry->mask_base;
 
+        if ( unlikely(!memory_decoded(entry->dev)) )
+            return 0;
         msg->address_lo = readl(base + PCI_MSIX_ENTRY_LOWER_ADDR_OFFSET);
         msg->address_hi = readl(base + PCI_MSIX_ENTRY_UPPER_ADDR_OFFSET);
         msg->data = readl(base + PCI_MSIX_ENTRY_DATA_OFFSET);
@@ -209,6 +232,8 @@ static void read_msi_msg(struct msi_desc
 
     if ( iommu_intremap )
         iommu_read_msi_from_ire(entry, msg);
+
+    return 1;
 }
 
 static int write_msi_msg(struct msi_desc *entry, struct msi_msg *msg)
@@ -260,6 +285,8 @@ static int write_msi_msg(struct msi_desc
         void __iomem *base;
         base = entry->mask_base;
 
+        if ( unlikely(!memory_decoded(entry->dev)) )
+            return -ENXIO;
         writel(msg->address_lo,
                base + PCI_MSIX_ENTRY_LOWER_ADDR_OFFSET);
         writel(msg->address_hi,
@@ -287,7 +314,8 @@ void set_msi_affinity(struct irq_desc *d
     ASSERT(spin_is_locked(&desc->lock));
 
     memset(&msg, 0, sizeof(msg));
-    read_msi_msg(msi_desc, &msg);
+    if ( !read_msi_msg(msi_desc, &msg) )
+        return;
 
     msg.data &= ~MSI_DATA_VECTOR_MASK;
     msg.data |= MSI_DATA_VECTOR(desc->arch.vector);
@@ -347,20 +375,24 @@ int msi_maskable_irq(const struct msi_de
            || entry->msi_attrib.maskbit;
 }
 
-static void msi_set_mask_bit(struct irq_desc *desc, int flag)
+static bool_t msi_set_mask_bit(struct irq_desc *desc, int flag)
 {
     struct msi_desc *entry = desc->msi_desc;
+    struct pci_dev *pdev;
+    u16 seg;
+    u8 bus, slot, func;
 
     ASSERT(spin_is_locked(&desc->lock));
     BUG_ON(!entry || !entry->dev);
+    pdev = entry->dev;
+    seg = pdev->seg;
+    bus = pdev->bus;
+    slot = PCI_SLOT(pdev->devfn);
+    func = PCI_FUNC(pdev->devfn);
     switch (entry->msi_attrib.type) {
     case PCI_CAP_ID_MSI:
         if (entry->msi_attrib.maskbit) {
             u32 mask_bits;
-            u16 seg = entry->dev->seg;
-            u8 bus = entry->dev->bus;
-            u8 slot = PCI_SLOT(entry->dev->devfn);
-            u8 func = PCI_FUNC(entry->dev->devfn);
 
             mask_bits = pci_conf_read32(seg, bus, slot, func, entry->msi.mpos);
             mask_bits &= ~((u32)1 << entry->msi_attrib.entry_nr);
@@ -369,24 +401,52 @@ static void msi_set_mask_bit(struct irq_
         }
         break;
     case PCI_CAP_ID_MSIX:
-    {
-        int offset = PCI_MSIX_ENTRY_VECTOR_CTRL_OFFSET;
-        writel(flag, entry->mask_base + offset);
-        readl(entry->mask_base + offset);
-        break;
-    }
+        if ( likely(memory_decoded(pdev)) )
+        {
+            writel(flag, entry->mask_base + PCI_MSIX_ENTRY_VECTOR_CTRL_OFFSET);
+            readl(entry->mask_base + PCI_MSIX_ENTRY_VECTOR_CTRL_OFFSET);
+            break;
+        }
+        if ( flag )
+        {
+            u16 control;
+            domid_t domid = pdev->domain->domain_id;
+
+            control = pci_conf_read16(seg, bus, slot, func,
+                                      msix_control_reg(entry->msi_attrib.pos));
+            if ( control & PCI_MSIX_FLAGS_MASKALL )
+                break;
+            pci_conf_write16(seg, bus, slot, func,
+                             msix_control_reg(entry->msi_attrib.pos),
+                             control | PCI_MSIX_FLAGS_MASKALL);
+            if ( pdev->msix->warned != domid )
+            {
+                pdev->msix->warned = domid;
+                printk(XENLOG_G_WARNING
+                       "cannot mask IRQ %d: masked MSI-X on Dom%d's %04x:%02x:%02x.%u\n",
+                       desc->irq, domid, pdev->seg, pdev->bus,
+                       PCI_SLOT(pdev->devfn), PCI_FUNC(pdev->devfn));
+            }
+            break;
+        }
+        /* fall through */
     default:
-        BUG();
-        break;
+        return 0;
     }
     entry->msi_attrib.masked = !!flag;
+
+    return 1;
 }
 
 static int msi_get_mask_bit(const struct msi_desc *entry)
 {
-    switch (entry->msi_attrib.type) {
+    if ( !entry->dev )
+        return -1;
+
+    switch ( entry->msi_attrib.type )
+    {
     case PCI_CAP_ID_MSI:
-        if (!entry->dev || !entry->msi_attrib.maskbit)
+        if ( !entry->msi_attrib.maskbit )
             break;
         return (pci_conf_read32(entry->dev->seg, entry->dev->bus,
                                 PCI_SLOT(entry->dev->devfn),
@@ -394,6 +454,8 @@ static int msi_get_mask_bit(const struct
                                 entry->msi.mpos) >>
                 entry->msi_attrib.entry_nr) & 1;
     case PCI_CAP_ID_MSIX:
+        if ( unlikely(!memory_decoded(entry->dev)) )
+            break;
         return readl(entry->mask_base + PCI_MSIX_ENTRY_VECTOR_CTRL_OFFSET) & 1;
     }
     return -1;
@@ -401,12 +463,14 @@ static int msi_get_mask_bit(const struct
 
 void mask_msi_irq(struct irq_desc *desc)
 {
-    msi_set_mask_bit(desc, 1);
+    if ( unlikely(!msi_set_mask_bit(desc, 1)) )
+        BUG_ON(!(desc->status & IRQ_DISABLED));
 }
 
 void unmask_msi_irq(struct irq_desc *desc)
 {
-    msi_set_mask_bit(desc, 0);
+    if ( unlikely(!msi_set_mask_bit(desc, 0)) )
+        WARN();
 }
 
 static unsigned int startup_msi_irq(struct irq_desc *desc)
@@ -713,6 +777,9 @@ static int msix_capability_init(struct p
     control = pci_conf_read16(seg, bus, slot, func, msix_control_reg(pos));
     msix_set_enable(dev, 0);/* Ensure msix is disabled as I set it up */
 
+    if ( unlikely(!memory_decoded(dev)) )
+        return -ENXIO;
+
     if ( desc )
     {
         entry = alloc_msi_entry(1);
@@ -845,7 +912,8 @@ static int msix_capability_init(struct p
     ++msix->used_entries;
 
     /* Restore MSI-X enabled bits */
-    pci_conf_write16(seg, bus, slot, func, msix_control_reg(pos), control);
+    pci_conf_write16(seg, bus, slot, func, msix_control_reg(pos),
+                     control & ~PCI_MSIX_FLAGS_MASKALL);
 
     return 0;
 }
@@ -998,8 +1066,16 @@ static void __pci_disable_msix(struct ms
 
     BUG_ON(list_empty(&dev->msi_list));
 
-    writel(1, entry->mask_base + PCI_MSIX_ENTRY_VECTOR_CTRL_OFFSET);
-
+    if ( likely(memory_decoded(dev)) )
+        writel(1, entry->mask_base + PCI_MSIX_ENTRY_VECTOR_CTRL_OFFSET);
+    else if ( !(control & PCI_MSIX_FLAGS_MASKALL) )
+    {
+        printk(XENLOG_WARNING
+               "cannot disable IRQ %d: masking MSI-X on %04x:%02x:%02x.%u\n",
+               entry->irq, dev->seg, dev->bus,
+               PCI_SLOT(dev->devfn), PCI_FUNC(dev->devfn));
+        control |= PCI_MSIX_FLAGS_MASKALL;
+    }
     pci_conf_write16(seg, bus, slot, func, msix_control_reg(pos), control);
 
     _pci_cleanup_msix(dev->msix);
@@ -1137,14 +1213,23 @@ int pci_restore_msi_state(struct pci_dev
             nr = entry->msi.nvec;
         }
         else if ( entry->msi_attrib.type == PCI_CAP_ID_MSIX )
+        {
             msix_set_enable(pdev, 0);
+            if ( unlikely(!memory_decoded(pdev)) )
+            {
+                spin_unlock_irqrestore(&desc->lock, flags);
+                return -ENXIO;
+            }
+        }
 
         msg = entry->msg;
         write_msi_msg(entry, &msg);
 
         for ( i = 0; ; )
         {
-            msi_set_mask_bit(desc, entry[i].msi_attrib.masked);
+            if ( unlikely(!msi_set_mask_bit(desc,
+                                            entry[i].msi_attrib.masked)) )
+                BUG();
 
             if ( !--nr )
                 break;



[-- Attachment #2: x86-MSI-X-teardown.patch --]
[-- Type: text/plain, Size: 11138 bytes --]

x86/MSI-X: be more careful during teardown

When a device gets detached from a guest, pciback will clear its
command register, thus disabling both memory and I/O decoding. The
disabled memory decoding, however, has an effect on the MSI-X table
accesses the hypervisor does: These won't have the intended effect
anymore. Even worse, for PCIe devices (but not SR-IOV virtual
functions) such accesses may (will?) be treated as Unsupported
Requests, causing respective errors to be surfaced, potentially in the
form of NMIs that may be fatal to the hypervisor or Dom0 is different
ways. Hence rather than carrying out these accesses, we should avoid
them where we can, and use alternative (e.g. PCI config space based)
mechanisms to achieve at least the same effect.

Signed-off-by: Jan Beulich <jbeulich@suse.com>
---
v2: mask_msi_irq()'s BUG() needs to be conditional upon IRQ_DISABLED
    not already being set, as ->shutdown() may be called on error paths
    when the IRQ never got enabled. This in turn required the changes
    to irq.c.

Backporting note (largely to myself):
   Depends on (not yet backported) commit 061eebe0e "x86/MSI: drop
   workaround for insecure Dom0 kernels" (due to re-use of struct
   arch_msix's warned field).

--- a/xen/arch/x86/irq.c
+++ b/xen/arch/x86/irq.c
@@ -217,9 +217,9 @@ void destroy_irq(unsigned int irq)
     }
 
     spin_lock_irqsave(&desc->lock, flags);
-    desc->status  |= IRQ_DISABLED;
     desc->status  &= ~IRQ_GUEST;
     desc->handler->shutdown(desc);
+    desc->status |= IRQ_DISABLED;
     action = desc->action;
     desc->action  = NULL;
     desc->msi_desc = NULL;
@@ -995,8 +995,8 @@ void __init release_irq(unsigned int irq
     spin_lock_irqsave(&desc->lock,flags);
     action = desc->action;
     desc->action  = NULL;
-    desc->status |= IRQ_DISABLED;
     desc->handler->shutdown(desc);
+    desc->status |= IRQ_DISABLED;
     spin_unlock_irqrestore(&desc->lock,flags);
 
     /* Wait to make sure it's not being used on another CPU */
@@ -1725,8 +1725,8 @@ static irq_guest_action_t *__pirq_guest_
     BUG_ON(action->in_flight != 0);
 
     /* Disabling IRQ before releasing the desc_lock avoids an IRQ storm. */
-    desc->status |= IRQ_DISABLED;
     desc->handler->disable(desc);
+    desc->status |= IRQ_DISABLED;
 
     /*
      * Mark any remaining pending EOIs as ready to flush.
--- a/xen/arch/x86/msi.c
+++ b/xen/arch/x86/msi.c
@@ -121,6 +121,27 @@ static void msix_put_fixmap(struct arch_
     spin_unlock(&msix->table_lock);
 }
 
+static bool_t memory_decoded(const struct pci_dev *dev)
+{
+    u8 bus, slot, func;
+
+    if ( !dev->info.is_virtfn )
+    {
+        bus = dev->bus;
+        slot = PCI_SLOT(dev->devfn);
+        func = PCI_FUNC(dev->devfn);
+    }
+    else
+    {
+        bus = dev->info.physfn.bus;
+        slot = PCI_SLOT(dev->info.physfn.devfn);
+        func = PCI_FUNC(dev->info.physfn.devfn);
+    }
+
+    return !!(pci_conf_read16(dev->seg, bus, slot, func, PCI_COMMAND) &
+              PCI_COMMAND_MEMORY);
+}
+
 /*
  * MSI message composition
  */
@@ -162,7 +183,7 @@ void msi_compose_msg(unsigned vector, co
     }
 }
 
-static void read_msi_msg(struct msi_desc *entry, struct msi_msg *msg)
+static bool_t read_msi_msg(struct msi_desc *entry, struct msi_msg *msg)
 {
     switch ( entry->msi_attrib.type )
     {
@@ -198,6 +219,8 @@ static void read_msi_msg(struct msi_desc
         void __iomem *base;
         base = entry->mask_base;
 
+        if ( unlikely(!memory_decoded(entry->dev)) )
+            return 0;
         msg->address_lo = readl(base + PCI_MSIX_ENTRY_LOWER_ADDR_OFFSET);
         msg->address_hi = readl(base + PCI_MSIX_ENTRY_UPPER_ADDR_OFFSET);
         msg->data = readl(base + PCI_MSIX_ENTRY_DATA_OFFSET);
@@ -209,6 +232,8 @@ static void read_msi_msg(struct msi_desc
 
     if ( iommu_intremap )
         iommu_read_msi_from_ire(entry, msg);
+
+    return 1;
 }
 
 static int write_msi_msg(struct msi_desc *entry, struct msi_msg *msg)
@@ -260,6 +285,8 @@ static int write_msi_msg(struct msi_desc
         void __iomem *base;
         base = entry->mask_base;
 
+        if ( unlikely(!memory_decoded(entry->dev)) )
+            return -ENXIO;
         writel(msg->address_lo,
                base + PCI_MSIX_ENTRY_LOWER_ADDR_OFFSET);
         writel(msg->address_hi,
@@ -287,7 +314,8 @@ void set_msi_affinity(struct irq_desc *d
     ASSERT(spin_is_locked(&desc->lock));
 
     memset(&msg, 0, sizeof(msg));
-    read_msi_msg(msi_desc, &msg);
+    if ( !read_msi_msg(msi_desc, &msg) )
+        return;
 
     msg.data &= ~MSI_DATA_VECTOR_MASK;
     msg.data |= MSI_DATA_VECTOR(desc->arch.vector);
@@ -347,20 +375,24 @@ int msi_maskable_irq(const struct msi_de
            || entry->msi_attrib.maskbit;
 }
 
-static void msi_set_mask_bit(struct irq_desc *desc, int flag)
+static bool_t msi_set_mask_bit(struct irq_desc *desc, int flag)
 {
     struct msi_desc *entry = desc->msi_desc;
+    struct pci_dev *pdev;
+    u16 seg;
+    u8 bus, slot, func;
 
     ASSERT(spin_is_locked(&desc->lock));
     BUG_ON(!entry || !entry->dev);
+    pdev = entry->dev;
+    seg = pdev->seg;
+    bus = pdev->bus;
+    slot = PCI_SLOT(pdev->devfn);
+    func = PCI_FUNC(pdev->devfn);
     switch (entry->msi_attrib.type) {
     case PCI_CAP_ID_MSI:
         if (entry->msi_attrib.maskbit) {
             u32 mask_bits;
-            u16 seg = entry->dev->seg;
-            u8 bus = entry->dev->bus;
-            u8 slot = PCI_SLOT(entry->dev->devfn);
-            u8 func = PCI_FUNC(entry->dev->devfn);
 
             mask_bits = pci_conf_read32(seg, bus, slot, func, entry->msi.mpos);
             mask_bits &= ~((u32)1 << entry->msi_attrib.entry_nr);
@@ -369,24 +401,52 @@ static void msi_set_mask_bit(struct irq_
         }
         break;
     case PCI_CAP_ID_MSIX:
-    {
-        int offset = PCI_MSIX_ENTRY_VECTOR_CTRL_OFFSET;
-        writel(flag, entry->mask_base + offset);
-        readl(entry->mask_base + offset);
-        break;
-    }
+        if ( likely(memory_decoded(pdev)) )
+        {
+            writel(flag, entry->mask_base + PCI_MSIX_ENTRY_VECTOR_CTRL_OFFSET);
+            readl(entry->mask_base + PCI_MSIX_ENTRY_VECTOR_CTRL_OFFSET);
+            break;
+        }
+        if ( flag )
+        {
+            u16 control;
+            domid_t domid = pdev->domain->domain_id;
+
+            control = pci_conf_read16(seg, bus, slot, func,
+                                      msix_control_reg(entry->msi_attrib.pos));
+            if ( control & PCI_MSIX_FLAGS_MASKALL )
+                break;
+            pci_conf_write16(seg, bus, slot, func,
+                             msix_control_reg(entry->msi_attrib.pos),
+                             control | PCI_MSIX_FLAGS_MASKALL);
+            if ( pdev->msix->warned != domid )
+            {
+                pdev->msix->warned = domid;
+                printk(XENLOG_G_WARNING
+                       "cannot mask IRQ %d: masked MSI-X on Dom%d's %04x:%02x:%02x.%u\n",
+                       desc->irq, domid, pdev->seg, pdev->bus,
+                       PCI_SLOT(pdev->devfn), PCI_FUNC(pdev->devfn));
+            }
+            break;
+        }
+        /* fall through */
     default:
-        BUG();
-        break;
+        return 0;
     }
     entry->msi_attrib.masked = !!flag;
+
+    return 1;
 }
 
 static int msi_get_mask_bit(const struct msi_desc *entry)
 {
-    switch (entry->msi_attrib.type) {
+    if ( !entry->dev )
+        return -1;
+
+    switch ( entry->msi_attrib.type )
+    {
     case PCI_CAP_ID_MSI:
-        if (!entry->dev || !entry->msi_attrib.maskbit)
+        if ( !entry->msi_attrib.maskbit )
             break;
         return (pci_conf_read32(entry->dev->seg, entry->dev->bus,
                                 PCI_SLOT(entry->dev->devfn),
@@ -394,6 +454,8 @@ static int msi_get_mask_bit(const struct
                                 entry->msi.mpos) >>
                 entry->msi_attrib.entry_nr) & 1;
     case PCI_CAP_ID_MSIX:
+        if ( unlikely(!memory_decoded(entry->dev)) )
+            break;
         return readl(entry->mask_base + PCI_MSIX_ENTRY_VECTOR_CTRL_OFFSET) & 1;
     }
     return -1;
@@ -401,12 +463,14 @@ static int msi_get_mask_bit(const struct
 
 void mask_msi_irq(struct irq_desc *desc)
 {
-    msi_set_mask_bit(desc, 1);
+    if ( unlikely(!msi_set_mask_bit(desc, 1)) )
+        BUG_ON(!(desc->status & IRQ_DISABLED));
 }
 
 void unmask_msi_irq(struct irq_desc *desc)
 {
-    msi_set_mask_bit(desc, 0);
+    if ( unlikely(!msi_set_mask_bit(desc, 0)) )
+        WARN();
 }
 
 static unsigned int startup_msi_irq(struct irq_desc *desc)
@@ -713,6 +777,9 @@ static int msix_capability_init(struct p
     control = pci_conf_read16(seg, bus, slot, func, msix_control_reg(pos));
     msix_set_enable(dev, 0);/* Ensure msix is disabled as I set it up */
 
+    if ( unlikely(!memory_decoded(dev)) )
+        return -ENXIO;
+
     if ( desc )
     {
         entry = alloc_msi_entry(1);
@@ -845,7 +912,8 @@ static int msix_capability_init(struct p
     ++msix->used_entries;
 
     /* Restore MSI-X enabled bits */
-    pci_conf_write16(seg, bus, slot, func, msix_control_reg(pos), control);
+    pci_conf_write16(seg, bus, slot, func, msix_control_reg(pos),
+                     control & ~PCI_MSIX_FLAGS_MASKALL);
 
     return 0;
 }
@@ -998,8 +1066,16 @@ static void __pci_disable_msix(struct ms
 
     BUG_ON(list_empty(&dev->msi_list));
 
-    writel(1, entry->mask_base + PCI_MSIX_ENTRY_VECTOR_CTRL_OFFSET);
-
+    if ( likely(memory_decoded(dev)) )
+        writel(1, entry->mask_base + PCI_MSIX_ENTRY_VECTOR_CTRL_OFFSET);
+    else if ( !(control & PCI_MSIX_FLAGS_MASKALL) )
+    {
+        printk(XENLOG_WARNING
+               "cannot disable IRQ %d: masking MSI-X on %04x:%02x:%02x.%u\n",
+               entry->irq, dev->seg, dev->bus,
+               PCI_SLOT(dev->devfn), PCI_FUNC(dev->devfn));
+        control |= PCI_MSIX_FLAGS_MASKALL;
+    }
     pci_conf_write16(seg, bus, slot, func, msix_control_reg(pos), control);
 
     _pci_cleanup_msix(dev->msix);
@@ -1137,14 +1213,23 @@ int pci_restore_msi_state(struct pci_dev
             nr = entry->msi.nvec;
         }
         else if ( entry->msi_attrib.type == PCI_CAP_ID_MSIX )
+        {
             msix_set_enable(pdev, 0);
+            if ( unlikely(!memory_decoded(pdev)) )
+            {
+                spin_unlock_irqrestore(&desc->lock, flags);
+                return -ENXIO;
+            }
+        }
 
         msg = entry->msg;
         write_msi_msg(entry, &msg);
 
         for ( i = 0; ; )
         {
-            msi_set_mask_bit(desc, entry[i].msi_attrib.masked);
+            if ( unlikely(!msi_set_mask_bit(desc,
+                                            entry[i].msi_attrib.masked)) )
+                BUG();
 
             if ( !--nr )
                 break;

[-- Attachment #3: Type: text/plain, Size: 126 bytes --]

_______________________________________________
Xen-devel mailing list
Xen-devel@lists.xen.org
http://lists.xen.org/xen-devel

Re: [PATCH v2 1/4] x86/MSI-X: be more careful during teardown

[Andrew Cooper]

On 25/03/15 16:39, Jan Beulich wrote:
> When a device gets detached from a guest, pciback will clear its
> command register, thus disabling both memory and I/O decoding. The
> disabled memory decoding, however, has an effect on the MSI-X table
> accesses the hypervisor does: These won't have the intended effect
> anymore. Even worse, for PCIe devices (but not SR-IOV virtual
> functions) such accesses may (will?) be treated as Unsupported
> Requests, causing respective errors to be surfaced, potentially in the
> form of NMIs that may be fatal to the hypervisor or Dom0 is different
> ways. Hence rather than carrying out these accesses, we should avoid
> them where we can, and use alternative (e.g. PCI config space based)
> mechanisms to achieve at least the same effect.
>
> Signed-off-by: Jan Beulich <jbeulich@suse.com>

Reviewed-by: Andrew Cooper <andrew.cooper3@citrix.com>

Re: [PATCH v2 1/4] x86/MSI-X: be more careful during teardown

[Stefano Stabellini]

On Wed, 25 Mar 2015, Jan Beulich wrote:
> When a device gets detached from a guest, pciback will clear its
> command register, thus disabling both memory and I/O decoding. The
> disabled memory decoding, however, has an effect on the MSI-X table
> accesses the hypervisor does: These won't have the intended effect
> anymore. Even worse, for PCIe devices (but not SR-IOV virtual
> functions) such accesses may (will?) be treated as Unsupported
> Requests, causing respective errors to be surfaced, potentially in the
> form of NMIs that may be fatal to the hypervisor or Dom0 is different
> ways. Hence rather than carrying out these accesses, we should avoid
> them where we can, and use alternative (e.g. PCI config space based)
> mechanisms to achieve at least the same effect.

I don't think that it is a good idea for both Xen and Linux to access
the command register simultaneously.  Working around Linux in Xen
doesn't sound like an optimal solution.   Maybe we could just fix the
pciback and that would be enough.

In any case we should make it clear somewhere who is supposed to write
to the command register (and other PCI reigsters) at any given time,
otherwise it would be very easy for a new kernel update to break the
hypervisor and we wouldn't even know why it happened.

 
> @@ -369,24 +401,52 @@ static void msi_set_mask_bit(struct irq_
>          }
>          break;
>      case PCI_CAP_ID_MSIX:
> -    {
> -        int offset = PCI_MSIX_ENTRY_VECTOR_CTRL_OFFSET;
> -        writel(flag, entry->mask_base + offset);
> -        readl(entry->mask_base + offset);
> -        break;
> -    }
> +        if ( likely(memory_decoded(pdev)) )
> +        {
> +            writel(flag, entry->mask_base + PCI_MSIX_ENTRY_VECTOR_CTRL_OFFSET);
> +            readl(entry->mask_base + PCI_MSIX_ENTRY_VECTOR_CTRL_OFFSET);
> +            break;
> +        }
> +        if ( flag )
> +        {
> +            u16 control;
> +            domid_t domid = pdev->domain->domain_id;
> +
> +            control = pci_conf_read16(seg, bus, slot, func,
> +                                      msix_control_reg(entry->msi_attrib.pos));
> +            if ( control & PCI_MSIX_FLAGS_MASKALL )
> +                break;
> +            pci_conf_write16(seg, bus, slot, func,
> +                             msix_control_reg(entry->msi_attrib.pos),
> +                             control | PCI_MSIX_FLAGS_MASKALL);

How is that going to interact with Linux (Dom0) writing to the command
register? Moreover QEMU writes to the PCI_MSIX_FLAGS_MASKALL bit for
devices assigned to HVM guests. Could this cause any conflicts?

One might argue that QEMU should not touch PCI_MSIX_FLAGS_MASKALL, but
as a matter of fact QEMU has done that for years and we cannot break
that behaviour without introducing regressions.  In fact as it stands
QEMU is the owner of PCI_MSIX_FLAGS_MASKALL for devices assigned to HVM
guests, not Xen.

Can we avoid messing with PCI_MSIX_FLAGS_MASKALL in Xen for passed
through devices to running domains?  I think that might be a good enough
separation of responsibilities between Xen and QEMU.

Re: [PATCH v2 1/4] x86/MSI-X: be more careful during teardown

[Jan Beulich]

>>> On 02.04.15 at 18:49, <stefano.stabellini@eu.citrix.com> wrote:
> On Wed, 25 Mar 2015, Jan Beulich wrote:
>> When a device gets detached from a guest, pciback will clear its
>> command register, thus disabling both memory and I/O decoding. The
>> disabled memory decoding, however, has an effect on the MSI-X table
>> accesses the hypervisor does: These won't have the intended effect
>> anymore. Even worse, for PCIe devices (but not SR-IOV virtual
>> functions) such accesses may (will?) be treated as Unsupported
>> Requests, causing respective errors to be surfaced, potentially in the
>> form of NMIs that may be fatal to the hypervisor or Dom0 is different
>> ways. Hence rather than carrying out these accesses, we should avoid
>> them where we can, and use alternative (e.g. PCI config space based)
>> mechanisms to achieve at least the same effect.
> 
> I don't think that it is a good idea for both Xen and Linux to access
> the command register simultaneously.  Working around Linux in Xen
> doesn't sound like an optimal solution.   Maybe we could just fix the
> pciback and that would be enough.

I'm afraid that would just eliminate the specific case, but not the
general issue. While we trust Dom0 to not do outright bad things,
the hypervisor should still avoid doing things that can go wrong
due to the state a device is put (or left) in by Dom0.

> In any case we should make it clear somewhere who is supposed to write
> to the command register (and other PCI reigsters) at any given time,
> otherwise it would be very easy for a new kernel update to break the
> hypervisor and we wouldn't even know why it happened.

We should, but at this point in time this is going to be rather
problematic. Such a separation of responsibilities should have been
done before all the pass-through code got written.

>> @@ -369,24 +401,52 @@ static void msi_set_mask_bit(struct irq_
>>          }
>>          break;
>>      case PCI_CAP_ID_MSIX:
>> -    {
>> -        int offset = PCI_MSIX_ENTRY_VECTOR_CTRL_OFFSET;
>> -        writel(flag, entry->mask_base + offset);
>> -        readl(entry->mask_base + offset);
>> -        break;
>> -    }
>> +        if ( likely(memory_decoded(pdev)) )
>> +        {
>> +            writel(flag, entry->mask_base + PCI_MSIX_ENTRY_VECTOR_CTRL_OFFSET);
>> +            readl(entry->mask_base + PCI_MSIX_ENTRY_VECTOR_CTRL_OFFSET);
>> +            break;
>> +        }
>> +        if ( flag )
>> +        {
>> +            u16 control;
>> +            domid_t domid = pdev->domain->domain_id;
>> +
>> +            control = pci_conf_read16(seg, bus, slot, func,
>> +                                      msix_control_reg(entry->msi_attrib.pos));
>> +            if ( control & PCI_MSIX_FLAGS_MASKALL )
>> +                break;
>> +            pci_conf_write16(seg, bus, slot, func,
>> +                             msix_control_reg(entry->msi_attrib.pos),
>> +                             control | PCI_MSIX_FLAGS_MASKALL);
> 
> How is that going to interact with Linux (Dom0) writing to the command
> register? Moreover QEMU writes to the PCI_MSIX_FLAGS_MASKALL bit for
> devices assigned to HVM guests. Could this cause any conflicts?

I don't see the relation to the comment register here - all quoted code
only deals with the MSI-X control register.

> One might argue that QEMU should not touch PCI_MSIX_FLAGS_MASKALL, but
> as a matter of fact QEMU has done that for years and we cannot break
> that behaviour without introducing regressions.  In fact as it stands
> QEMU is the owner of PCI_MSIX_FLAGS_MASKALL for devices assigned to HVM
> guests, not Xen.
> 
> Can we avoid messing with PCI_MSIX_FLAGS_MASKALL in Xen for passed
> through devices to running domains?  I think that might be a good enough
> separation of responsibilities between Xen and QEMU.

No, the bits has to be under hypervisor control (just like any other
hardware bits controlling interrupts). Dealing with this is the subject
of patches I have ready, but didn't post yet.

Jan

Re: [PATCH v2 1/4] x86/MSI-X: be more careful during teardown

[Stefano Stabellini]

On Mon, 13 Apr 2015, Jan Beulich wrote:
> >>> On 02.04.15 at 18:49, <stefano.stabellini@eu.citrix.com> wrote:
> > On Wed, 25 Mar 2015, Jan Beulich wrote:
> >> When a device gets detached from a guest, pciback will clear its
> >> command register, thus disabling both memory and I/O decoding. The
> >> disabled memory decoding, however, has an effect on the MSI-X table
> >> accesses the hypervisor does: These won't have the intended effect
> >> anymore. Even worse, for PCIe devices (but not SR-IOV virtual
> >> functions) such accesses may (will?) be treated as Unsupported
> >> Requests, causing respective errors to be surfaced, potentially in the
> >> form of NMIs that may be fatal to the hypervisor or Dom0 is different
> >> ways. Hence rather than carrying out these accesses, we should avoid
> >> them where we can, and use alternative (e.g. PCI config space based)
> >> mechanisms to achieve at least the same effect.
> > 
> > I don't think that it is a good idea for both Xen and Linux to access
> > the command register simultaneously.  Working around Linux in Xen
> > doesn't sound like an optimal solution.   Maybe we could just fix the
> > pciback and that would be enough.
> 
> I'm afraid that would just eliminate the specific case, but not the
> general issue.

If we trust Dom0 to do the right thing, then I don't think there is a
general issue to be solved. Dom0 can break the system at any time, I
don't see any differences here, unless we have a plan to actually be
able to handle a misbehaving dom0, in that case I am all for it.


> While we trust Dom0 to not do outright bad things,
> the hypervisor should still avoid doing things that can go wrong
> due to the state a device is put (or left) in by Dom0.

Xen should also avoid doing things that can go wrong because of the
state a device is put in by QEMU or other components in the system.
There isn't much room for Xen to play with.


> > In any case we should make it clear somewhere who is supposed to write
> > to the command register (and other PCI reigsters) at any given time,
> > otherwise it would be very easy for a new kernel update to break the
> > hypervisor and we wouldn't even know why it happened.
> 
> We should, but at this point in time this is going to be rather
> problematic. Such a separation of responsibilities should have been
> done before all the pass-through code got written.

That is true.


> >> @@ -369,24 +401,52 @@ static void msi_set_mask_bit(struct irq_
> >>          }
> >>          break;
> >>      case PCI_CAP_ID_MSIX:
> >> -    {
> >> -        int offset = PCI_MSIX_ENTRY_VECTOR_CTRL_OFFSET;
> >> -        writel(flag, entry->mask_base + offset);
> >> -        readl(entry->mask_base + offset);
> >> -        break;
> >> -    }
> >> +        if ( likely(memory_decoded(pdev)) )
> >> +        {
> >> +            writel(flag, entry->mask_base + PCI_MSIX_ENTRY_VECTOR_CTRL_OFFSET);
> >> +            readl(entry->mask_base + PCI_MSIX_ENTRY_VECTOR_CTRL_OFFSET);
> >> +            break;
> >> +        }
> >> +        if ( flag )
> >> +        {
> >> +            u16 control;
> >> +            domid_t domid = pdev->domain->domain_id;
> >> +
> >> +            control = pci_conf_read16(seg, bus, slot, func,
> >> +                                      msix_control_reg(entry->msi_attrib.pos));
> >> +            if ( control & PCI_MSIX_FLAGS_MASKALL )
> >> +                break;
> >> +            pci_conf_write16(seg, bus, slot, func,
> >> +                             msix_control_reg(entry->msi_attrib.pos),
> >> +                             control | PCI_MSIX_FLAGS_MASKALL);
> > 
> > How is that going to interact with Linux (Dom0) writing to the command
> > register? Moreover QEMU writes to the PCI_MSIX_FLAGS_MASKALL bit for
> > devices assigned to HVM guests. Could this cause any conflicts?
> 
> I don't see the relation to the comment register here - all quoted code
> only deals with the MSI-X control register.

I meant control register, sorry for the confusion.


> > One might argue that QEMU should not touch PCI_MSIX_FLAGS_MASKALL, but
> > as a matter of fact QEMU has done that for years and we cannot break
> > that behaviour without introducing regressions.  In fact as it stands
> > QEMU is the owner of PCI_MSIX_FLAGS_MASKALL for devices assigned to HVM
> > guests, not Xen.
> > 
> > Can we avoid messing with PCI_MSIX_FLAGS_MASKALL in Xen for passed
> > through devices to running domains?  I think that might be a good enough
> > separation of responsibilities between Xen and QEMU.
> 
> No, the bits has to be under hypervisor control (just like any other
> hardware bits controlling interrupts).

Maybe they should be under hypervisor control, but keep in mind that
they are not and they haven't been for years now.  We cannot go ahead
and break existing code paths and use cases. As it stands, QEMU is in
charge of PCI_MSIX_FLAGS_MASKALL for passthrough devices and this is a
regression.


> Dealing with this is the subject of patches I have ready, but didn't post yet.

OK, that is very good. But if we really want to make this change,
then I think that those patches would need to be part of this series so
that they can be committed at the same time to minimize the breakage.

And how are we going to deal with older "unfixed" QEMUs?
So far we have been using the same policy for QEMU and the Dom0 kernel:
Xen doesn't break them -- old Linux kernels and QEMUs are supposed to
just work.

However this change would subtly break QEMU, so we would need to
introduce a feature flag in QEMU to advertise that is new enough and
capable of leaving the control register alone. If the flag is missing,
Xen would refuse to do PCI passthrough?  It seems awful.

Alternatively we would have to introduce a minimum supported QEMU
version for Xen, that is unpleasant.  Do we want to do that? I think it
is worth a thought before going ahead.

Re: [PATCH v2 1/4] x86/MSI-X: be more careful during teardown

[Jan Beulich]

>>> On 13.04.15 at 12:50, <stefano.stabellini@eu.citrix.com> wrote:
> On Mon, 13 Apr 2015, Jan Beulich wrote:
>> >>> On 02.04.15 at 18:49, <stefano.stabellini@eu.citrix.com> wrote:
>> > On Wed, 25 Mar 2015, Jan Beulich wrote:
>> >> When a device gets detached from a guest, pciback will clear its
>> >> command register, thus disabling both memory and I/O decoding. The
>> >> disabled memory decoding, however, has an effect on the MSI-X table
>> >> accesses the hypervisor does: These won't have the intended effect
>> >> anymore. Even worse, for PCIe devices (but not SR-IOV virtual
>> >> functions) such accesses may (will?) be treated as Unsupported
>> >> Requests, causing respective errors to be surfaced, potentially in the
>> >> form of NMIs that may be fatal to the hypervisor or Dom0 is different
>> >> ways. Hence rather than carrying out these accesses, we should avoid
>> >> them where we can, and use alternative (e.g. PCI config space based)
>> >> mechanisms to achieve at least the same effect.
>> > 
>> > I don't think that it is a good idea for both Xen and Linux to access
>> > the command register simultaneously.  Working around Linux in Xen
>> > doesn't sound like an optimal solution.   Maybe we could just fix the
>> > pciback and that would be enough.
>> 
>> I'm afraid that would just eliminate the specific case, but not the
>> general issue.
> 
> If we trust Dom0 to do the right thing, then I don't think there is a
> general issue to be solved. Dom0 can break the system at any time, I
> don't see any differences here, unless we have a plan to actually be
> able to handle a misbehaving dom0, in that case I am all for it.

No, that gets us in the wrong direction. Dom0 can have legitimate
reasons to have to clear memory or I/O decoding on a device at
run time (even if current Linux doesn't do so). The more general
problem we may need to solve is that of racing config space
accesses (one by Dom0, the other by the hypervisor). But that's
beyond this series' scope.

>> While we trust Dom0 to not do outright bad things,
>> the hypervisor should still avoid doing things that can go wrong
>> due to the state a device is put (or left) in by Dom0.
> 
> Xen should also avoid doing things that can go wrong because of the
> state a device is put in by QEMU or other components in the system.
> There isn't much room for Xen to play with.

Qemu is either part of Dom0, or doesn't play with devices directly.

> And how are we going to deal with older "unfixed" QEMUs?
> So far we have been using the same policy for QEMU and the Dom0 kernel:
> Xen doesn't break them -- old Linux kernels and QEMUs are supposed to
> just work.

I'm not sure that's really true for qemu, or if it is, then only by pure
luck: The tool stack interface of the hypervisor as well as the libxc
interfaces are subject to change between any two releases. I view
it as unavoidable to break older qemu here.

Jan

Re: [PATCH v2 1/4] x86/MSI-X: be more careful during teardown

[Stefano Stabellini]

On Mon, 13 Apr 2015, Jan Beulich wrote:
> >>> On 13.04.15 at 12:50, <stefano.stabellini@eu.citrix.com> wrote:
> > On Mon, 13 Apr 2015, Jan Beulich wrote:
> >> >>> On 02.04.15 at 18:49, <stefano.stabellini@eu.citrix.com> wrote:
> >> > On Wed, 25 Mar 2015, Jan Beulich wrote:
> >> >> When a device gets detached from a guest, pciback will clear its
> >> >> command register, thus disabling both memory and I/O decoding. The
> >> >> disabled memory decoding, however, has an effect on the MSI-X table
> >> >> accesses the hypervisor does: These won't have the intended effect
> >> >> anymore. Even worse, for PCIe devices (but not SR-IOV virtual
> >> >> functions) such accesses may (will?) be treated as Unsupported
> >> >> Requests, causing respective errors to be surfaced, potentially in the
> >> >> form of NMIs that may be fatal to the hypervisor or Dom0 is different
> >> >> ways. Hence rather than carrying out these accesses, we should avoid
> >> >> them where we can, and use alternative (e.g. PCI config space based)
> >> >> mechanisms to achieve at least the same effect.
> >> > 
> >> > I don't think that it is a good idea for both Xen and Linux to access
> >> > the command register simultaneously.  Working around Linux in Xen
> >> > doesn't sound like an optimal solution.   Maybe we could just fix the
> >> > pciback and that would be enough.
> >> 
> >> I'm afraid that would just eliminate the specific case, but not the
> >> general issue.
> > 
> > If we trust Dom0 to do the right thing, then I don't think there is a
> > general issue to be solved. Dom0 can break the system at any time, I
> > don't see any differences here, unless we have a plan to actually be
> > able to handle a misbehaving dom0, in that case I am all for it.
> 
> No, that gets us in the wrong direction. Dom0 can have legitimate
> reasons to have to clear memory or I/O decoding on a device at
> run time (even if current Linux doesn't do so). The more general
> problem we may need to solve is that of racing config space
> accesses (one by Dom0, the other by the hypervisor). But that's
> beyond this series' scope.

This is why I was asking for a document that describes who is in charge
of what and when. I don't think we can move forward without it.


> >> While we trust Dom0 to not do outright bad things,
> >> the hypervisor should still avoid doing things that can go wrong
> >> due to the state a device is put (or left) in by Dom0.
> > 
> > Xen should also avoid doing things that can go wrong because of the
> > state a device is put in by QEMU or other components in the system.
> > There isn't much room for Xen to play with.
> 
> Qemu is either part of Dom0, or doesn't play with devices directly.

I don't understand the point you are trying to make here.
If your intention is to point out that QEMU shouldn't be writing to the
control register, as I wrote earlier, the current codebase disagrees
with you and has been that way for years.


> > And how are we going to deal with older "unfixed" QEMUs?
> > So far we have been using the same policy for QEMU and the Dom0 kernel:
> > Xen doesn't break them -- old Linux kernels and QEMUs are supposed to
> > just work.
> 
> I'm not sure that's really true for qemu, or if it is, then only by pure
> luck:

This is false, it is so by design.  QEMU has an internal libxc
compatibility layer.


> The tool stack interface of the hypervisor as well as the libxc
> interfaces are subject to change between any two releases.

QEMU knows how to cope with libxc interface changes.


> I view it as unavoidable to break older qemu here.

I disagree and I am opposed to any patches series that would
deliberately break QEMU.

Re: [PATCH v2 1/4] x86/MSI-X: be more careful during teardown

[Jan Beulich]

>>> On 13.04.15 at 14:01, <stefano.stabellini@eu.citrix.com> wrote:
> On Mon, 13 Apr 2015, Jan Beulich wrote:
>> >>> On 13.04.15 at 12:50, <stefano.stabellini@eu.citrix.com> wrote:
>> > On Mon, 13 Apr 2015, Jan Beulich wrote:
>> >> While we trust Dom0 to not do outright bad things,
>> >> the hypervisor should still avoid doing things that can go wrong
>> >> due to the state a device is put (or left) in by Dom0.
>> > 
>> > Xen should also avoid doing things that can go wrong because of the
>> > state a device is put in by QEMU or other components in the system.
>> > There isn't much room for Xen to play with.
>> 
>> Qemu is either part of Dom0, or doesn't play with devices directly.
> 
> I don't understand the point you are trying to make here.
> If your intention is to point out that QEMU shouldn't be writing to the
> control register, as I wrote earlier, the current codebase disagrees
> with you and has been that way for years.

Yes, this was precisely my intention. Qemu having done a certain
thing for many years doesn't mean this is correct, and shouldn't
be fixed if broken.

>> > And how are we going to deal with older "unfixed" QEMUs?
>> > So far we have been using the same policy for QEMU and the Dom0 kernel:
>> > Xen doesn't break them -- old Linux kernels and QEMUs are supposed to
>> > just work.
>> 
>> I'm not sure that's really true for qemu, or if it is, then only by pure
>> luck:
> 
> This is false, it is so by design.  QEMU has an internal libxc
> compatibility layer.

Which could necessarily only ever be updated after the fact (of
a libxc change) and only ever in maintained branches. I.e. as
widely or as narrow as fixing the issues here would require.

>> I view it as unavoidable to break older qemu here.
> 
> I disagree and I am opposed to any patches series that would
> deliberately break QEMU.

If that was without also adjusting qemu I would understand you.
Considering that the issues here haven't been brought up just
yesterday, the lack of any substantial counter proposal on how
to fix this is signaling to me that there are little alternatives. Or
do you have any going beyond "do it another way"?

Jan

Re: [PATCH v2 1/4] x86/MSI-X: be more careful during teardown

[Stefano Stabellini]

On Mon, 13 Apr 2015, Jan Beulich wrote:
> >>> On 13.04.15 at 14:01, <stefano.stabellini@eu.citrix.com> wrote:
> > On Mon, 13 Apr 2015, Jan Beulich wrote:
> >> >>> On 13.04.15 at 12:50, <stefano.stabellini@eu.citrix.com> wrote:
> >> > On Mon, 13 Apr 2015, Jan Beulich wrote:
> >> >> While we trust Dom0 to not do outright bad things,
> >> >> the hypervisor should still avoid doing things that can go wrong
> >> >> due to the state a device is put (or left) in by Dom0.
> >> > 
> >> > Xen should also avoid doing things that can go wrong because of the
> >> > state a device is put in by QEMU or other components in the system.
> >> > There isn't much room for Xen to play with.
> >> 
> >> Qemu is either part of Dom0, or doesn't play with devices directly.
> > 
> > I don't understand the point you are trying to make here.
> > If your intention is to point out that QEMU shouldn't be writing to the
> > control register, as I wrote earlier, the current codebase disagrees
> > with you and has been that way for years.
> 
> Yes, this was precisely my intention. Qemu having done a certain
> thing for many years doesn't mean this is correct, and shouldn't
> be fixed if broken.

Sure, as long as the fix doesn't cause regressions.

Do you have a specific example of how QEMU's behaviour is "broken" and
this patch is supposed to correct it?  I thought you were trying to
work-around a dom0 kernel issue with this patch, not a QEMU bug.

Working-around a Dom0 kernel bug by creating a QEMU bug is not a great
move.


> >> > And how are we going to deal with older "unfixed" QEMUs?
> >> > So far we have been using the same policy for QEMU and the Dom0 kernel:
> >> > Xen doesn't break them -- old Linux kernels and QEMUs are supposed to
> >> > just work.
> >> 
> >> I'm not sure that's really true for qemu, or if it is, then only by pure
> >> luck:
> > 
> > This is false, it is so by design.  QEMU has an internal libxc
> > compatibility layer.
> 
> Which could necessarily only ever be updated after the fact (of
> a libxc change) and only ever in maintained branches. I.e. as
> widely or as narrow as fixing the issues here would require.

True, in fact we used it with older Xen releases, when upstream QEMU
wasn't the default device model yet. Nothing from Xen 4.2 onward has
required a change in QEMU and its compatibility shim and I would like to
keep it that way (note that Paul's ioreq server changes are not required
to build older QEMU versions against new Xen versions), unless we have a
very compelling reason.


> >> I view it as unavoidable to break older qemu here.
> > 
> > I disagree and I am opposed to any patches series that would
> > deliberately break QEMU.
> 
> If that was without also adjusting qemu I would understand you.

You say that, but the initial submission only included the Xen
hypervisor changes.

I am not completely against stopping QEMU from writing to the control
register, but we cannot go ahead and commit patches to Xen that break
QEMU compatibility willy nilly.

At least we need to:

- review the QEMU and Xen patches together
- commit the Xen patches only after the QEMU patches have been reviewed
and acked
- we should consider making the QEMU changes acceptable for QEMU stable
trees and mark them for backport
- we need to write down which is the minimal version of QEMU that can
  work for PCI passthrough


> Considering that the issues here haven't been brought up just
> yesterday, the lack of any substantial counter proposal on how
> to fix this is signaling to me that there are little alternatives. Or
> do you have any going beyond "do it another way"?

TBH in case of this issue, I think that having the Dom0 kernel do the
right thing is good enough.

Re: [PATCH v2 1/4] x86/MSI-X: be more careful during teardown

[Ian Campbell]

On Mon, 2015-04-13 at 10:11 +0100, Jan Beulich wrote:
> >>> On 02.04.15 at 18:49, <stefano.stabellini@eu.citrix.com> wrote:
> > In any case we should make it clear somewhere who is supposed to write
> > to the command register (and other PCI reigsters) at any given time,
> > otherwise it would be very easy for a new kernel update to break the
> > hypervisor and we wouldn't even know why it happened.
> 
> We should, but at this point in time this is going to be rather
> problematic. Such a separation of responsibilities should have been
> done before all the pass-through code got written.

Wasn't Stefano just asking to write down the current semantics, problems
and all?

I think that would be a wonderful idea even if for some parts the
documentation is things like:
      * "we don't really know";
      * "there are multiple parties X, Y and Z involved and it's a
        mess";
      * "currently qemu, but really ought to be Xen";
      * etc.

I think half the problem with coming up with a plan to move forward is
that exactly where we are now isn't very clear, so having a thing to
refer to there might be helpful above and beyond it being a good idea to
write these things down on principal.

Ian.

[PATCH v2 2/4] x86/MSI-X: access MSI-X table only after having enabled MSI-X

[Jan Beulich]

[-- Attachment #1: Type: text/plain, Size: 10474 bytes --]

As done in Linux by f598282f51 ("PCI: Fix the NIU MSI-X problem in a
better way") and its broken predecessor, make sure we don't access the
MSI-X table without having enabled MSI-X first, using the mask-all flag
instead to prevent interrupts from occurring.

Signed-off-by: Jan Beulich <jbeulich@suse.com>
Reviewed-by: Andrew Cooper <andrew.cooper3@citrix.com>

--- a/xen/arch/x86/msi.c
+++ b/xen/arch/x86/msi.c
@@ -142,6 +142,19 @@ static bool_t memory_decoded(const struc
               PCI_COMMAND_MEMORY);
 }
 
+static bool_t msix_memory_decoded(const struct pci_dev *dev, unsigned int pos)
+{
+    u16 control = pci_conf_read16(dev->seg, dev->bus,
+                                  PCI_SLOT(dev->devfn),
+                                  PCI_FUNC(dev->devfn),
+                                  msix_control_reg(pos));
+
+    if ( !(control & PCI_MSIX_FLAGS_ENABLE) )
+        return 0;
+
+    return memory_decoded(dev);
+}
+
 /*
  * MSI message composition
  */
@@ -219,7 +236,8 @@ static bool_t read_msi_msg(struct msi_de
         void __iomem *base;
         base = entry->mask_base;
 
-        if ( unlikely(!memory_decoded(entry->dev)) )
+        if ( unlikely(!msix_memory_decoded(entry->dev,
+                                           entry->msi_attrib.pos)) )
             return 0;
         msg->address_lo = readl(base + PCI_MSIX_ENTRY_LOWER_ADDR_OFFSET);
         msg->address_hi = readl(base + PCI_MSIX_ENTRY_UPPER_ADDR_OFFSET);
@@ -285,7 +303,8 @@ static int write_msi_msg(struct msi_desc
         void __iomem *base;
         base = entry->mask_base;
 
-        if ( unlikely(!memory_decoded(entry->dev)) )
+        if ( unlikely(!msix_memory_decoded(entry->dev,
+                                           entry->msi_attrib.pos)) )
             return -ENXIO;
         writel(msg->address_lo,
                base + PCI_MSIX_ENTRY_LOWER_ADDR_OFFSET);
@@ -379,7 +398,7 @@ static bool_t msi_set_mask_bit(struct ir
 {
     struct msi_desc *entry = desc->msi_desc;
     struct pci_dev *pdev;
-    u16 seg;
+    u16 seg, control;
     u8 bus, slot, func;
 
     ASSERT(spin_is_locked(&desc->lock));
@@ -401,35 +420,38 @@ static bool_t msi_set_mask_bit(struct ir
         }
         break;
     case PCI_CAP_ID_MSIX:
+        control = pci_conf_read16(seg, bus, slot, func,
+                                  msix_control_reg(entry->msi_attrib.pos));
+        if ( unlikely(!(control & PCI_MSIX_FLAGS_ENABLE)) )
+            pci_conf_write16(seg, bus, slot, func,
+                             msix_control_reg(entry->msi_attrib.pos),
+                             control | (PCI_MSIX_FLAGS_ENABLE |
+                                        PCI_MSIX_FLAGS_MASKALL));
         if ( likely(memory_decoded(pdev)) )
         {
             writel(flag, entry->mask_base + PCI_MSIX_ENTRY_VECTOR_CTRL_OFFSET);
             readl(entry->mask_base + PCI_MSIX_ENTRY_VECTOR_CTRL_OFFSET);
-            break;
+            if ( likely(control & PCI_MSIX_FLAGS_ENABLE) )
+                break;
+            flag = 1;
         }
-        if ( flag )
+        else if ( flag && !(control & PCI_MSIX_FLAGS_MASKALL) )
         {
-            u16 control;
             domid_t domid = pdev->domain->domain_id;
 
-            control = pci_conf_read16(seg, bus, slot, func,
-                                      msix_control_reg(entry->msi_attrib.pos));
-            if ( control & PCI_MSIX_FLAGS_MASKALL )
-                break;
-            pci_conf_write16(seg, bus, slot, func,
-                             msix_control_reg(entry->msi_attrib.pos),
-                             control | PCI_MSIX_FLAGS_MASKALL);
+            control |= PCI_MSIX_FLAGS_MASKALL;
             if ( pdev->msix->warned != domid )
             {
                 pdev->msix->warned = domid;
                 printk(XENLOG_G_WARNING
-                       "cannot mask IRQ %d: masked MSI-X on Dom%d's %04x:%02x:%02x.%u\n",
+                       "cannot mask IRQ %d: masking MSI-X on Dom%d's %04x:%02x:%02x.%u\n",
                        desc->irq, domid, pdev->seg, pdev->bus,
                        PCI_SLOT(pdev->devfn), PCI_FUNC(pdev->devfn));
             }
-            break;
         }
-        /* fall through */
+        pci_conf_write16(seg, bus, slot, func,
+                         msix_control_reg(entry->msi_attrib.pos), control);
+        return flag;
     default:
         return 0;
     }
@@ -454,7 +476,8 @@ static int msi_get_mask_bit(const struct
                                 entry->msi.mpos) >>
                 entry->msi_attrib.entry_nr) & 1;
     case PCI_CAP_ID_MSIX:
-        if ( unlikely(!memory_decoded(entry->dev)) )
+        if ( unlikely(!msix_memory_decoded(entry->dev,
+                                           entry->msi_attrib.pos)) )
             break;
         return readl(entry->mask_base + PCI_MSIX_ENTRY_VECTOR_CTRL_OFFSET) & 1;
     }
@@ -775,16 +798,32 @@ static int msix_capability_init(struct p
 
     pos = pci_find_cap_offset(seg, bus, slot, func, PCI_CAP_ID_MSIX);
     control = pci_conf_read16(seg, bus, slot, func, msix_control_reg(pos));
-    msix_set_enable(dev, 0);/* Ensure msix is disabled as I set it up */
+    /*
+     * Ensure MSI-X interrupts are masked during setup. Some devices require
+     * MSI-X to be enabled before we can touch the MSI-X registers. We need
+     * to mask all the vectors to prevent interrupts coming in before they're
+     * fully set up.
+     */
+    pci_conf_write16(seg, bus, slot, func, msix_control_reg(pos),
+                     control | (PCI_MSIX_FLAGS_ENABLE |
+                                PCI_MSIX_FLAGS_MASKALL));
 
     if ( unlikely(!memory_decoded(dev)) )
+    {
+        pci_conf_write16(seg, bus, slot, func, msix_control_reg(pos),
+                         control & ~PCI_MSIX_FLAGS_ENABLE);
         return -ENXIO;
+    }
 
     if ( desc )
     {
         entry = alloc_msi_entry(1);
         if ( !entry )
+        {
+            pci_conf_write16(seg, bus, slot, func, msix_control_reg(pos),
+                             control & ~PCI_MSIX_FLAGS_ENABLE);
             return -ENOMEM;
+        }
         ASSERT(msi);
     }
 
@@ -815,6 +854,8 @@ static int msix_capability_init(struct p
     {
         if ( !msi || !msi->table_base )
         {
+            pci_conf_write16(seg, bus, slot, func, msix_control_reg(pos),
+                             control & ~PCI_MSIX_FLAGS_ENABLE);
             xfree(entry);
             return -ENXIO;
         }
@@ -857,6 +898,8 @@ static int msix_capability_init(struct p
 
         if ( idx < 0 )
         {
+            pci_conf_write16(seg, bus, slot, func, msix_control_reg(pos),
+                             control & ~PCI_MSIX_FLAGS_ENABLE);
             xfree(entry);
             return idx;
         }
@@ -912,8 +955,7 @@ static int msix_capability_init(struct p
     ++msix->used_entries;
 
     /* Restore MSI-X enabled bits */
-    pci_conf_write16(seg, bus, slot, func, msix_control_reg(pos),
-                     control & ~PCI_MSIX_FLAGS_MASKALL);
+    pci_conf_write16(seg, bus, slot, func, msix_control_reg(pos), control);
 
     return 0;
 }
@@ -1062,7 +1104,10 @@ static void __pci_disable_msix(struct ms
 
     pos = pci_find_cap_offset(seg, bus, slot, func, PCI_CAP_ID_MSIX);
     control = pci_conf_read16(seg, bus, slot, func, msix_control_reg(pos));
-    msix_set_enable(dev, 0);
+    if ( unlikely(!(control & PCI_MSIX_FLAGS_ENABLE)) )
+        pci_conf_write16(seg, bus, slot, func, msix_control_reg(pos),
+                         control | (PCI_MSIX_FLAGS_ENABLE |
+                                    PCI_MSIX_FLAGS_MASKALL));
 
     BUG_ON(list_empty(&dev->msi_list));
 
@@ -1188,6 +1234,8 @@ int pci_restore_msi_state(struct pci_dev
     list_for_each_entry_safe( entry, tmp, &pdev->msi_list, list )
     {
         unsigned int i = 0, nr = 1;
+        u16 control = 0;
+        u8 slot = PCI_SLOT(pdev->devfn), func = PCI_FUNC(pdev->devfn);
 
         irq = entry->irq;
         desc = &irq_desc[irq];
@@ -1214,10 +1262,18 @@ int pci_restore_msi_state(struct pci_dev
         }
         else if ( entry->msi_attrib.type == PCI_CAP_ID_MSIX )
         {
-            msix_set_enable(pdev, 0);
+            control = pci_conf_read16(pdev->seg, pdev->bus, slot, func,
+                                      msix_control_reg(entry->msi_attrib.pos));
+            pci_conf_write16(pdev->seg, pdev->bus, slot, func,
+                             msix_control_reg(entry->msi_attrib.pos),
+                             control | (PCI_MSIX_FLAGS_ENABLE |
+                                        PCI_MSIX_FLAGS_MASKALL));
             if ( unlikely(!memory_decoded(pdev)) )
             {
                 spin_unlock_irqrestore(&desc->lock, flags);
+                pci_conf_write16(pdev->seg, pdev->bus, slot, func,
+                                 msix_control_reg(entry->msi_attrib.pos),
+                                 control & ~PCI_MSIX_FLAGS_ENABLE);
                 return -ENXIO;
             }
         }
@@ -1246,11 +1302,9 @@ int pci_restore_msi_state(struct pci_dev
         if ( entry->msi_attrib.type == PCI_CAP_ID_MSI )
         {
             unsigned int cpos = msi_control_reg(entry->msi_attrib.pos);
-            u16 control = pci_conf_read16(pdev->seg, pdev->bus,
-                                          PCI_SLOT(pdev->devfn),
-                                          PCI_FUNC(pdev->devfn), cpos);
 
-            control &= ~PCI_MSI_FLAGS_QSIZE;
+            control = pci_conf_read16(pdev->seg, pdev->bus, slot, func, cpos) &
+                      ~PCI_MSI_FLAGS_QSIZE;
             multi_msi_enable(control, entry->msi.nvec);
             pci_conf_write16(pdev->seg, pdev->bus, PCI_SLOT(pdev->devfn),
                              PCI_FUNC(pdev->devfn), cpos, control);
@@ -1258,7 +1312,9 @@ int pci_restore_msi_state(struct pci_dev
             msi_set_enable(pdev, 1);
         }
         else if ( entry->msi_attrib.type == PCI_CAP_ID_MSIX )
-            msix_set_enable(pdev, 1);
+            pci_conf_write16(pdev->seg, pdev->bus, slot, func,
+                             msix_control_reg(entry->msi_attrib.pos),
+                             control | PCI_MSIX_FLAGS_ENABLE);
     }
 
     return 0;



[-- Attachment #2: x86-MSI-X-enable.patch --]
[-- Type: text/plain, Size: 10535 bytes --]

x86/MSI-X: access MSI-X table only after having enabled MSI-X

As done in Linux by f598282f51 ("PCI: Fix the NIU MSI-X problem in a
better way") and its broken predecessor, make sure we don't access the
MSI-X table without having enabled MSI-X first, using the mask-all flag
instead to prevent interrupts from occurring.

Signed-off-by: Jan Beulich <jbeulich@suse.com>
Reviewed-by: Andrew Cooper <andrew.cooper3@citrix.com>

--- a/xen/arch/x86/msi.c
+++ b/xen/arch/x86/msi.c
@@ -142,6 +142,19 @@ static bool_t memory_decoded(const struc
               PCI_COMMAND_MEMORY);
 }
 
+static bool_t msix_memory_decoded(const struct pci_dev *dev, unsigned int pos)
+{
+    u16 control = pci_conf_read16(dev->seg, dev->bus,
+                                  PCI_SLOT(dev->devfn),
+                                  PCI_FUNC(dev->devfn),
+                                  msix_control_reg(pos));
+
+    if ( !(control & PCI_MSIX_FLAGS_ENABLE) )
+        return 0;
+
+    return memory_decoded(dev);
+}
+
 /*
  * MSI message composition
  */
@@ -219,7 +236,8 @@ static bool_t read_msi_msg(struct msi_de
         void __iomem *base;
         base = entry->mask_base;
 
-        if ( unlikely(!memory_decoded(entry->dev)) )
+        if ( unlikely(!msix_memory_decoded(entry->dev,
+                                           entry->msi_attrib.pos)) )
             return 0;
         msg->address_lo = readl(base + PCI_MSIX_ENTRY_LOWER_ADDR_OFFSET);
         msg->address_hi = readl(base + PCI_MSIX_ENTRY_UPPER_ADDR_OFFSET);
@@ -285,7 +303,8 @@ static int write_msi_msg(struct msi_desc
         void __iomem *base;
         base = entry->mask_base;
 
-        if ( unlikely(!memory_decoded(entry->dev)) )
+        if ( unlikely(!msix_memory_decoded(entry->dev,
+                                           entry->msi_attrib.pos)) )
             return -ENXIO;
         writel(msg->address_lo,
                base + PCI_MSIX_ENTRY_LOWER_ADDR_OFFSET);
@@ -379,7 +398,7 @@ static bool_t msi_set_mask_bit(struct ir
 {
     struct msi_desc *entry = desc->msi_desc;
     struct pci_dev *pdev;
-    u16 seg;
+    u16 seg, control;
     u8 bus, slot, func;
 
     ASSERT(spin_is_locked(&desc->lock));
@@ -401,35 +420,38 @@ static bool_t msi_set_mask_bit(struct ir
         }
         break;
     case PCI_CAP_ID_MSIX:
+        control = pci_conf_read16(seg, bus, slot, func,
+                                  msix_control_reg(entry->msi_attrib.pos));
+        if ( unlikely(!(control & PCI_MSIX_FLAGS_ENABLE)) )
+            pci_conf_write16(seg, bus, slot, func,
+                             msix_control_reg(entry->msi_attrib.pos),
+                             control | (PCI_MSIX_FLAGS_ENABLE |
+                                        PCI_MSIX_FLAGS_MASKALL));
         if ( likely(memory_decoded(pdev)) )
         {
             writel(flag, entry->mask_base + PCI_MSIX_ENTRY_VECTOR_CTRL_OFFSET);
             readl(entry->mask_base + PCI_MSIX_ENTRY_VECTOR_CTRL_OFFSET);
-            break;
+            if ( likely(control & PCI_MSIX_FLAGS_ENABLE) )
+                break;
+            flag = 1;
         }
-        if ( flag )
+        else if ( flag && !(control & PCI_MSIX_FLAGS_MASKALL) )
         {
-            u16 control;
             domid_t domid = pdev->domain->domain_id;
 
-            control = pci_conf_read16(seg, bus, slot, func,
-                                      msix_control_reg(entry->msi_attrib.pos));
-            if ( control & PCI_MSIX_FLAGS_MASKALL )
-                break;
-            pci_conf_write16(seg, bus, slot, func,
-                             msix_control_reg(entry->msi_attrib.pos),
-                             control | PCI_MSIX_FLAGS_MASKALL);
+            control |= PCI_MSIX_FLAGS_MASKALL;
             if ( pdev->msix->warned != domid )
             {
                 pdev->msix->warned = domid;
                 printk(XENLOG_G_WARNING
-                       "cannot mask IRQ %d: masked MSI-X on Dom%d's %04x:%02x:%02x.%u\n",
+                       "cannot mask IRQ %d: masking MSI-X on Dom%d's %04x:%02x:%02x.%u\n",
                        desc->irq, domid, pdev->seg, pdev->bus,
                        PCI_SLOT(pdev->devfn), PCI_FUNC(pdev->devfn));
             }
-            break;
         }
-        /* fall through */
+        pci_conf_write16(seg, bus, slot, func,
+                         msix_control_reg(entry->msi_attrib.pos), control);
+        return flag;
     default:
         return 0;
     }
@@ -454,7 +476,8 @@ static int msi_get_mask_bit(const struct
                                 entry->msi.mpos) >>
                 entry->msi_attrib.entry_nr) & 1;
     case PCI_CAP_ID_MSIX:
-        if ( unlikely(!memory_decoded(entry->dev)) )
+        if ( unlikely(!msix_memory_decoded(entry->dev,
+                                           entry->msi_attrib.pos)) )
             break;
         return readl(entry->mask_base + PCI_MSIX_ENTRY_VECTOR_CTRL_OFFSET) & 1;
     }
@@ -775,16 +798,32 @@ static int msix_capability_init(struct p
 
     pos = pci_find_cap_offset(seg, bus, slot, func, PCI_CAP_ID_MSIX);
     control = pci_conf_read16(seg, bus, slot, func, msix_control_reg(pos));
-    msix_set_enable(dev, 0);/* Ensure msix is disabled as I set it up */
+    /*
+     * Ensure MSI-X interrupts are masked during setup. Some devices require
+     * MSI-X to be enabled before we can touch the MSI-X registers. We need
+     * to mask all the vectors to prevent interrupts coming in before they're
+     * fully set up.
+     */
+    pci_conf_write16(seg, bus, slot, func, msix_control_reg(pos),
+                     control | (PCI_MSIX_FLAGS_ENABLE |
+                                PCI_MSIX_FLAGS_MASKALL));
 
     if ( unlikely(!memory_decoded(dev)) )
+    {
+        pci_conf_write16(seg, bus, slot, func, msix_control_reg(pos),
+                         control & ~PCI_MSIX_FLAGS_ENABLE);
         return -ENXIO;
+    }
 
     if ( desc )
     {
         entry = alloc_msi_entry(1);
         if ( !entry )
+        {
+            pci_conf_write16(seg, bus, slot, func, msix_control_reg(pos),
+                             control & ~PCI_MSIX_FLAGS_ENABLE);
             return -ENOMEM;
+        }
         ASSERT(msi);
     }
 
@@ -815,6 +854,8 @@ static int msix_capability_init(struct p
     {
         if ( !msi || !msi->table_base )
         {
+            pci_conf_write16(seg, bus, slot, func, msix_control_reg(pos),
+                             control & ~PCI_MSIX_FLAGS_ENABLE);
             xfree(entry);
             return -ENXIO;
         }
@@ -857,6 +898,8 @@ static int msix_capability_init(struct p
 
         if ( idx < 0 )
         {
+            pci_conf_write16(seg, bus, slot, func, msix_control_reg(pos),
+                             control & ~PCI_MSIX_FLAGS_ENABLE);
             xfree(entry);
             return idx;
         }
@@ -912,8 +955,7 @@ static int msix_capability_init(struct p
     ++msix->used_entries;
 
     /* Restore MSI-X enabled bits */
-    pci_conf_write16(seg, bus, slot, func, msix_control_reg(pos),
-                     control & ~PCI_MSIX_FLAGS_MASKALL);
+    pci_conf_write16(seg, bus, slot, func, msix_control_reg(pos), control);
 
     return 0;
 }
@@ -1062,7 +1104,10 @@ static void __pci_disable_msix(struct ms
 
     pos = pci_find_cap_offset(seg, bus, slot, func, PCI_CAP_ID_MSIX);
     control = pci_conf_read16(seg, bus, slot, func, msix_control_reg(pos));
-    msix_set_enable(dev, 0);
+    if ( unlikely(!(control & PCI_MSIX_FLAGS_ENABLE)) )
+        pci_conf_write16(seg, bus, slot, func, msix_control_reg(pos),
+                         control | (PCI_MSIX_FLAGS_ENABLE |
+                                    PCI_MSIX_FLAGS_MASKALL));
 
     BUG_ON(list_empty(&dev->msi_list));
 
@@ -1188,6 +1234,8 @@ int pci_restore_msi_state(struct pci_dev
     list_for_each_entry_safe( entry, tmp, &pdev->msi_list, list )
     {
         unsigned int i = 0, nr = 1;
+        u16 control = 0;
+        u8 slot = PCI_SLOT(pdev->devfn), func = PCI_FUNC(pdev->devfn);
 
         irq = entry->irq;
         desc = &irq_desc[irq];
@@ -1214,10 +1262,18 @@ int pci_restore_msi_state(struct pci_dev
         }
         else if ( entry->msi_attrib.type == PCI_CAP_ID_MSIX )
         {
-            msix_set_enable(pdev, 0);
+            control = pci_conf_read16(pdev->seg, pdev->bus, slot, func,
+                                      msix_control_reg(entry->msi_attrib.pos));
+            pci_conf_write16(pdev->seg, pdev->bus, slot, func,
+                             msix_control_reg(entry->msi_attrib.pos),
+                             control | (PCI_MSIX_FLAGS_ENABLE |
+                                        PCI_MSIX_FLAGS_MASKALL));
             if ( unlikely(!memory_decoded(pdev)) )
             {
                 spin_unlock_irqrestore(&desc->lock, flags);
+                pci_conf_write16(pdev->seg, pdev->bus, slot, func,
+                                 msix_control_reg(entry->msi_attrib.pos),
+                                 control & ~PCI_MSIX_FLAGS_ENABLE);
                 return -ENXIO;
             }
         }
@@ -1246,11 +1302,9 @@ int pci_restore_msi_state(struct pci_dev
         if ( entry->msi_attrib.type == PCI_CAP_ID_MSI )
         {
             unsigned int cpos = msi_control_reg(entry->msi_attrib.pos);
-            u16 control = pci_conf_read16(pdev->seg, pdev->bus,
-                                          PCI_SLOT(pdev->devfn),
-                                          PCI_FUNC(pdev->devfn), cpos);
 
-            control &= ~PCI_MSI_FLAGS_QSIZE;
+            control = pci_conf_read16(pdev->seg, pdev->bus, slot, func, cpos) &
+                      ~PCI_MSI_FLAGS_QSIZE;
             multi_msi_enable(control, entry->msi.nvec);
             pci_conf_write16(pdev->seg, pdev->bus, PCI_SLOT(pdev->devfn),
                              PCI_FUNC(pdev->devfn), cpos, control);
@@ -1258,7 +1312,9 @@ int pci_restore_msi_state(struct pci_dev
             msi_set_enable(pdev, 1);
         }
         else if ( entry->msi_attrib.type == PCI_CAP_ID_MSIX )
-            msix_set_enable(pdev, 1);
+            pci_conf_write16(pdev->seg, pdev->bus, slot, func,
+                             msix_control_reg(entry->msi_attrib.pos),
+                             control | PCI_MSIX_FLAGS_ENABLE);
     }
 
     return 0;

[-- Attachment #3: Type: text/plain, Size: 126 bytes --]

_______________________________________________
Xen-devel mailing list
Xen-devel@lists.xen.org
http://lists.xen.org/xen-devel

Re: [PATCH v2 2/4] x86/MSI-X: access MSI-X table only after having enabled MSI-X

[Konrad Rzeszutek Wilk]

[-- Attachment #1: Type: text/plain, Size: 1364 bytes --]

On Wed, Mar 25, 2015 at 04:39:49PM +0000, Jan Beulich wrote:
> As done in Linux by f598282f51 ("PCI: Fix the NIU MSI-X problem in a
> better way") and its broken predecessor, make sure we don't access the
> MSI-X table without having enabled MSI-X first, using the mask-all flag
> instead to prevent interrupts from occurring.

This causes an regression with an Linux guest that has the XSA120 + XSA120
addendum with PV guests (hadn't tried yet HVM).

When PV guest requests an MSI-X, pciback gets:

[  122.778654] xen-pciback: 0000:0a:00.0: enable MSI-X
[  122.778861] pciback 0000:0a:00.0: xen map irq failed -6 for 1 domain
[  122.778887] xen_pciback: 0000:0a:00.0: error enabling MSI-X for guest 1: err -6!

The device has the PCI_COMMAND enabled correctly:

# lspci -s 0a:0.0 -vvv | head
0a:00.0 Ethernet controller: Intel Corporation 82576 Gigabit Network Connection (rev 01)
        Subsystem: Super Micro Computer Inc Device 10c9
        Control: I/O+ Mem+ BusMaster- SpecCycle- MemWINV- VGASnoop- ParErr- Stepping- SERR- FastB2B- DisINTx-

Attaching the 'xl dmesg', 'dmesg', and 'lspci'

The guest config is
[konrad@build-external tst036]$ more pf-pv.xm
extra="console=hvc0 debug "
kernel="/mnt/lab/bootstrap-x86_64/vmlinuz"
ramdisk="/mnt/lab/bootstrap-x86_64/initramfs.cpio.gz"
memory=2048
vcpus=6
name="pv-pf"
on_crash="preserve"
pci=["0000:0a:0.0"];



[-- Attachment #2: d --]
[-- Type: text/plain, Size: 120961 bytes --]

[    0.000000] PAT configuration [0-7]: WB  WT  UC- UC  WC  WP  UC  UC  
[    0.000000] Initializing cgroup subsys cpuset
[    0.000000] Initializing cgroup subsys cpu
[    0.000000] Initializing cgroup subsys cpuacct
[    0.000000] Linux version 4.0.0-rc6upstream-00005-gc5be3c5-dirty (konrad@build-external.dumpdata.com) (gcc version 4.4.4 20100503 (Red Hat 4.4.4-2) (GCC) ) #1 SMP Fri Apr 10 10:28:01 EDT 2015
[    0.000000] Command line: console=hvc0 earlyprintk=xen xen-pciback.hide=(0a:00.0)(0a:00.1) debug
[    0.000000] Released 0 page(s)
[    0.000000] e820: BIOS-provided physical RAM map:
[    0.000000] Xen: [mem 0x0000000000000000-0x000000000009dfff] usable
[    0.000000] Xen: [mem 0x000000000009e800-0x00000000000fffff] reserved
[    0.000000] Xen: [mem 0x0000000000100000-0x000000004d061fff] usable
[    0.000000] Xen: [mem 0x000000004d062000-0x00000000bf76ffff] unusable
[    0.000000] Xen: [mem 0x00000000bf770000-0x00000000bf77dfff] ACPI data
[    0.000000] Xen: [mem 0x00000000bf77e000-0x00000000bf7cffff] ACPI NVS
[    0.000000] Xen: [mem 0x00000000bf7d0000-0x00000000bf7dffff] reserved
[    0.000000] Xen: [mem 0x00000000bf7ec000-0x00000000bfffffff] reserved
[    0.000000] Xen: [mem 0x00000000e0000000-0x00000000efffffff] reserved
[    0.000000] Xen: [mem 0x00000000fbffe000-0x00000000fbffefff] reserved
[    0.000000] Xen: [mem 0x00000000fec00000-0x00000000fec00fff] reserved
[    0.000000] Xen: [mem 0x00000000fec80000-0x00000000fec80fff] reserved
[    0.000000] Xen: [mem 0x00000000fec8a000-0x00000000fec8afff] reserved
[    0.000000] Xen: [mem 0x00000000fee00000-0x00000000feefffff] reserved
[    0.000000] Xen: [mem 0x00000000ffc00000-0x00000000ffffffff] reserved
[    0.000000] Xen: [mem 0x0000000100000000-0x000000033fffffff] unusable
[    0.000000] bootconsole [xenboot0] enabled
[    0.000000] NX (Execute Disable) protection: active
[    0.000000] SMBIOS 2.6 present.
[    0.000000] DMI: Supermicro X8DTN/X8DTN, BIOS 2.1c       10/28/2011
[    0.000000] e820: update [mem 0x00000000-0x00000fff] usable ==> reserved
[    0.000000] e820: remove [mem 0x000a0000-0x000fffff] usable
[    0.000000] e820: last_pfn = 0x4d062 max_arch_pfn = 0x400000000
[    0.000000] Scanning 1 areas for low memory corruption
[    0.000000] Base memory trampoline at [ffff880000098000] 98000 size 24576
[    0.000000] init_memory_mapping: [mem 0x00000000-0x000fffff]
[    0.000000]  [mem 0x00000000-0x000fffff] page 4k
[    0.000000] init_memory_mapping: [mem 0x3e400000-0x3e5fffff]
[    0.000000]  [mem 0x3e400000-0x3e5fffff] page 4k
[    0.000000] BRK [0x0200b000, 0x0200bfff] PGTABLE
[    0.000000] init_memory_mapping: [mem 0x20000000-0x3e3fffff]
[    0.000000]  [mem 0x20000000-0x3e3fffff] page 4k
[    0.000000] BRK [0x0200c000, 0x0200cfff] PGTABLE
[    0.000000] BRK [0x0200d000, 0x0200dfff] PGTABLE
[    0.000000] BRK [0x0200e000, 0x0200efff] PGTABLE
[    0.000000] BRK [0x0200f000, 0x0200ffff] PGTABLE
[    0.000000] BRK [0x02010000, 0x02010fff] PGTABLE
[    0.000000] init_memory_mapping: [mem 0x00100000-0x1fffffff]
[    0.000000]  [mem 0x00100000-0x1fffffff] page 4k
[    0.000000] init_memory_mapping: [mem 0x3e600000-0x4d061fff]
[    0.000000]  [mem 0x3e600000-0x4d061fff] page 4k
[    0.000000] RAMDISK: [mem 0x04000000-0x09839fff]
[    0.000000] ACPI: Early table checksum verification disabled
[    0.000000] ACPI: RSDP 0x00000000000FA180 000024 (v02 ACPIAM)
[    0.000000] ACPI: XSDT 0x00000000BF770100 00006C (v01 SMCI            20111028 MSFT 00000097)
[    0.000000] ACPI: FACP 0x00000000BF770290 0000F4 (v04 102811 FACP1450 20111028 MSFT 00000097)
[    0.000000] ACPI BIOS Warning (bug): 32/64X length mismatch in FADT/Gpe0Block: 128/64 (20150204/tbfadt-618)
[    0.000000] ACPI: DSDT 0x00000000BF7706C0 005CBB (v02 1F280  1F280000 00000000 INTL 20051117)
[    0.000000] ACPI: FACS 0x00000000BF77E000 000040
[    0.000000] ACPI: APIC 0x00000000BF770390 000136 (v02 102811 APIC1450 20111028 MSFT 00000097)
[    0.000000] ACPI: MCFG 0x00000000BF7704D0 00003C (v01 102811 OEMMCFG  20111028 MSFT 00000097)
[    0.000000] ACPI: SLIT 0x00000000BF770510 000030 (v01 102811 OEMSLIT  20111028 MSFT 00000097)
[    0.000000] ACPI: OEMB 0x00000000BF77E040 000092 (v01 102811 OEMB1450 20111028 MSFT 00000097)
[    0.000000] ACPI: SRAT 0x00000000BF77A6C0 0001A8 (v02 102811 OEMSRAT  00000001 INTL 00000001)
[    0.000000] ACPI: HPET 0x00000000BF77A870 000038 (v01 102811 OEMHPET  20111028 MSFT 00000097)
[    0.000000] ACPI: XMAR 0x00000000BF77E0E0 000120 (v01 AMI    OEMDMAR  00000001 MSFT 00000097)
[    0.000000] ACPI: SSDT 0x00000000BF781720 000363 (v01 DpgPmm CpuPm    00000012 INTL 20051117)
[    0.000000] ACPI: Local APIC address 0xfee00000
[    0.000000] NUMA turned off
[    0.000000] Faking a node at [mem 0x0000000000000000-0x000000004d061fff]
[    0.000000] NODE_DATA(0) allocated [mem 0x3e6fc000-0x3e6fffff]
[    0.000000] Zone ranges:
[    0.000000]   DMA      [mem 0x0000000000001000-0x0000000000ffffff]
[    0.000000]   DMA32    [mem 0x0000000001000000-0x000000004d061fff]
[    0.000000]   Normal   empty
[    0.000000] Movable zone start for each node
[    0.000000] Early memory node ranges
[    0.000000]   node   0: [mem 0x0000000000001000-0x000000000009dfff]
[    0.000000]   node   0: [mem 0x0000000000100000-0x000000004d061fff]
[    0.000000] Initmem setup node 0 [mem 0x0000000000001000-0x000000004d061fff]
[    0.000000] On node 0 totalpages: 315391
[    0.000000]   DMA zone: 56 pages used for memmap
[    0.000000]   DMA zone: 21 pages reserved
[    0.000000]   DMA zone: 3997 pages, LIFO batch:0
[    0.000000]   DMA32 zone: 4258 pages used for memmap
[    0.000000]   DMA32 zone: 311394 pages, LIFO batch:31
[    0.000000] p2m virtual area at ffffc90000000000, size is 400000
[    0.000000] Remapped 98 page(s)
[    0.000000] ACPI: PM-Timer IO Port: 0x808
[    0.000000] ACPI: Local APIC address 0xfee00000
[    0.000000] ACPI: LAPIC (acpi_id[0x01] lapic_id[0x00] enabled)
[    0.000000] ACPI: LAPIC (acpi_id[0x02] lapic_id[0x02] enabled)
[    0.000000] ACPI: LAPIC (acpi_id[0x03] lapic_id[0x04] enabled)
[    0.000000] ACPI: LAPIC (acpi_id[0x04] lapic_id[0x06] enabled)
[    0.000000] ACPI: LAPIC (acpi_id[0x05] lapic_id[0x10] enabled)
[    0.000000] ACPI: LAPIC (acpi_id[0x06] lapic_id[0x12] enabled)
[    0.000000] ACPI: LAPIC (acpi_id[0x07] lapic_id[0x14] enabled)
[    0.000000] ACPI: LAPIC (acpi_id[0x08] lapic_id[0x16] enabled)
[    0.000000] ACPI: LAPIC (acpi_id[0x09] lapic_id[0x01] enabled)
[    0.000000] ACPI: LAPIC (acpi_id[0x0a] lapic_id[0x03] enabled)
[    0.000000] ACPI: LAPIC (acpi_id[0x0b] lapic_id[0x05] enabled)
[    0.000000] ACPI: LAPIC (acpi_id[0x0c] lapic_id[0x07] enabled)
[    0.000000] ACPI: LAPIC (acpi_id[0x0d] lapic_id[0x11] enabled)
[    0.000000] ACPI: LAPIC (acpi_id[0x0e] lapic_id[0x13] enabled)
[    0.000000] ACPI: LAPIC (acpi_id[0x0f] lapic_id[0x15] enabled)
[    0.000000] ACPI: LAPIC (acpi_id[0x10] lapic_id[0x17] enabled)
[    0.000000] ACPI: LAPIC (acpi_id[0x11] lapic_id[0x90] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0x12] lapic_id[0x91] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0x13] lapic_id[0x92] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0x14] lapic_id[0x93] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0x15] lapic_id[0x94] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0x16] lapic_id[0x95] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0x17] lapic_id[0x96] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0x18] lapic_id[0x97] disabled)
[    0.000000] ACPI: LAPIC_NMI (acpi_id[0xff] dfl dfl lint[0x1])
[    0.000000] ACPI: IOAPIC (id[0x08] address[0xfec00000] gsi_base[0])
[    0.000000] IOAPIC[0]: apic_id 8, version 32, address 0xfec00000, GSI 0-23
[    0.000000] ACPI: IOAPIC (id[0x09] address[0xfec8a000] gsi_base[24])
[    0.000000] IOAPIC[1]: apic_id 9, version 32, address 0xfec8a000, GSI 24-47
[    0.000000] ACPI: IOAPIC (id[0x0a] address[0xfec80000] gsi_base[48])
[    0.000000] IOAPIC[2]: apic_id 10, version 32, address 0xfec80000, GSI 48-71
[    0.000000] ACPI: IOAPIC (id[0x0b] address[0xfec80400] gsi_base[72])
[    0.000000] IOAPIC[3]: apic_id 11, version 32, address 0xfec80400, GSI 72-95
[    0.000000] ACPI: INT_SRC_OVR (bus 0 bus_irq 0 global_irq 2 dfl dfl)
[    0.000000] ACPI: INT_SRC_OVR (bus 0 bus_irq 9 global_irq 20 low level)
[    0.000000] ACPI: IRQ0 used by override.
[    0.000000] ACPI: IRQ9 used by override.
[    0.000000] Using ACPI (MADT) for SMP configuration information
[    0.000000] ACPI: HPET id: 0x8086a301 base: 0xfed00000
[    0.000000] smpboot: Allowing 24 CPUs, 8 hotplug CPUs
[    0.000000] PM: Registered nosave memory: [mem 0x00000000-0x00000fff]
[    0.000000] PM: Registered nosave memory: [mem 0x0009e000-0x0009efff]
[    0.000000] PM: Registered nosave memory: [mem 0x0009f000-0x000fffff]
[    0.000000] e820: [mem 0xc0000000-0xdfffffff] available for PCI devices
[    0.000000] Booting paravirtualized kernel on Xen
[    0.000000] Xen version: 4.6-unstable (preserve-AD)
[    0.000000] setup_percpu: NR_CPUS:512 nr_cpumask_bits:512 nr_cpu_ids:24 nr_node_ids:1
[    0.000000] PERCPU: Embedded 31 pages/cpu @ffff88003dc00000 s86296 r8192 d32488 u131072
[    0.000000] pcpu-alloc: s86296 r8192 d32488 u131072 alloc=1*2097152
[    0.000000] pcpu-alloc: [0] 00 01 02 03 04 05 06 07 08 09 10 11 12 13 14 15 
[    0.000000] pcpu-alloc: [0] 16 17 18 19 20 21 22 23 -- -- -- -- -- -- -- -- 
[    0.000000] xen: PV spinlocks enabled
[    0.000000] Built 1 zonelists in Node order, mobility grouping on.  Total pages: 311056
[    0.000000] Policy zone: DMA32
[    0.000000] Kernel command line: console=hvc0 earlyprintk=xen xen-pciback.hide=(0a:00.0)(0a:00.1) debug
[    0.000000] PID hash table entries: 4096 (order: 3, 32768 bytes)
[    0.000000] software IO TLB [mem 0x38600000-0x3c600000] (64MB) mapped at [ffff880038600000-ffff88003c5fffff]
[    0.000000] Memory: 823948K/1261564K available (7560K kernel code, 838K rwdata, 2408K rodata, 1996K init, 1264K bss, 437616K reserved, 0K cma-reserved)
[    0.000000] Hierarchical RCU implementation.
[    0.000000] 	Additional per-CPU info printed with stalls.
[    0.000000] 	RCU restricting CPUs from NR_CPUS=512 to nr_cpu_ids=3.
[    0.000000] RCU: Adjusting geometry for rcu_fanout_leaf=16, nr_cpu_ids=3
[    0.000000] NR_IRQS:33024 nr_irqs:768 16
[    0.000000] xen:events: Using FIFO-based ABI
[    0.000000] xen: --> pirq=1 -> irq=1 (gsi=1)
[    0.000000] xen: --> pirq=2 -> irq=2 (gsi=2)
[    0.000000] xen: --> pirq=3 -> irq=3 (gsi=3)
[    0.000000] xen: --> pirq=4 -> irq=4 (gsi=4)
[    0.000000] xen: --> pirq=5 -> irq=5 (gsi=5)
[    0.000000] xen: --> pirq=6 -> irq=6 (gsi=6)
[    0.000000] xen: --> pirq=7 -> irq=7 (gsi=7)
[    0.000000] xen: --> pirq=8 -> irq=8 (gsi=8)
[    0.000000] xen: --> pirq=10 -> irq=10 (gsi=10)
[    0.000000] xen: --> pirq=11 -> irq=11 (gsi=11)
[    0.000000] xen: --> pirq=12 -> irq=12 (gsi=12)
[    0.000000] xen: --> pirq=13 -> irq=13 (gsi=13)
[    0.000000] xen: --> pirq=14 -> irq=14 (gsi=14)
[    0.000000] xen: --> pirq=15 -> irq=15 (gsi=15)
[    0.000000] Console: colour VGA+ 80x25
[    0.000000] console [hvc0] enabled
[    0.000000] bootconsole [xenboot0] disabled
[    0.000000] Xen: using vcpuop timer interface
[    0.000000] installing Xen timer for CPU 0
[    0.000000] tsc: Detected 2266.802 MHz processor
[    6.767083] Calibrating delay loop (skipped), value calculated using timer frequency.. 4533.60 BogoMIPS (lpj=2266802)
[    6.767091] pid_max: default: 32768 minimum: 301
[    6.767152] ACPI: Core revision 20150204
[    6.870633] ACPI: All ACPI Tables successfully acquired
[    6.911351] Security Framework initialized
[    6.911359] SELinux:  Initializing.
[    6.911380] SELinux:  Starting in permissive mode
[    6.912121] Dentry cache hash table entries: 262144 (order: 9, 2097152 bytes)
[    6.912808] Inode-cache hash table entries: 131072 (order: 8, 1048576 bytes)
[    6.913243] Mount-cache hash table entries: 4096 (order: 3, 32768 bytes)
[    6.913276] Mountpoint-cache hash table entries: 4096 (order: 3, 32768 bytes)
[    6.915042] Initializing cgroup subsys freezer
[    6.915205] CPU: Physical Processor ID: 0
[    6.915208] CPU: Processor Core ID: 0
[    6.915214] mce: CPU supports 2 MCE banks
[    6.915238] Last level iTLB entries: 4KB 512, 2MB 7, 4MB 7
[    6.915242] Last level dTLB entries: 4KB 512, 2MB 32, 4MB 32, 1GB 0
[    6.915352] Freeing SMP alternatives memory: 28K (ffffffff81ec6000 - ffffffff81ecd000)
[    6.917168] ftrace: allocating 28210 entries in 111 pages
[    6.932468] cpu 0 spinlock event irq 97
[    6.966063] Performance Events: unsupported p6 CPU model 26 no PMU driver, software events only.
[    6.967197] NMI watchdog: disabled (cpu0): hardware events not enabled
[    6.967841] installing Xen timer for CPU 1
[    6.967924] cpu 1 spinlock event irq 104
[    6.969939] installing Xen timer for CPU 2
[    6.970022] cpu 2 spinlock event irq 111
[    6.971307] x86: Booted up 1 node, 3 CPUs
[    6.973670] devtmpfs: initialized
[    6.980649] PM: Registering ACPI NVS region [mem 0xbf77e000-0xbf7cffff] (335872 bytes)
[    6.981300] kworker/u6:0 (25) used greatest stack depth: 13776 bytes left
[    6.982068] RTC time: 19:53:29, date: 04/10/15
[    6.982729] NET: Registered protocol family 16
[    6.982805] xen:grant_table: Grant tables using version 1 layout
[    6.982839] Grant table initialized
[    6.984890] ACPI: bus type PCI registered
[    6.984895] acpiphp: ACPI Hot Plug PCI Controller Driver version: 0.5
[    6.988750] dca service started, version 1.12.1
[    6.988846] PCI: MMCONFIG for domain 0000 [bus 00-ff] at [mem 0xe0000000-0xefffffff] (base 0xe0000000)
[    6.988853] PCI: MMCONFIG at [mem 0xe0000000-0xefffffff] reserved in E820
[    7.023192] PCI: Using configuration type 1 for base access
[    7.032136] kworker/u6:0 (157) used greatest stack depth: 13688 bytes left
[    7.052838] ACPI: Added _OSI(Module Device)
[    7.052847] ACPI: Added _OSI(Processor Device)
[    7.052851] ACPI: Added _OSI(3.0 _SCP Extensions)
[    7.052854] ACPI: Added _OSI(Processor Aggregator Device)
[    7.055395] xen: registering gsi 20 triggering 0 polarity 1
[    7.055452] xen: --> pirq=20 -> irq=20 (gsi=20)
[    7.061170] ACPI: Executed 1 blocks of module-level executable AML code
[    7.123840] ACPI: Dynamic OEM Table Load:
[    7.123861] ACPI: SSDT 0xFFFF880037064000 002B94 (v01 DpgPmm P001Ist  00000011 INTL 20051117)
[    7.156398] ACPI: Dynamic OEM Table Load:
[    7.156437] ACPI: SSDT 0xFFFF880037C51000 00097E (v01 PmRef  P001Cst  00003001 INTL 20051117)
[    7.173696] ACPI: Interpreter enabled
[    7.173733] ACPI Exception: AE_NOT_FOUND, While evaluating Sleep State [\_S2_] (20150204/hwxface-580)
[    7.173751] ACPI Exception: AE_NOT_FOUND, While evaluating Sleep State [\_S3_] (20150204/hwxface-580)
[    7.173793] ACPI: (supports S0 S1 S4 S5)
[    7.173798] ACPI: Using IOAPIC for interrupt routing
[    7.174215] PCI: Using host bridge windows from ACPI; if necessary, use "pci=nocrs" and report a bug
[    7.325229] ACPI: PCI Root Bridge [PCI0] (domain 0000 [bus 00-ff])
[    7.325260] acpi PNP0A08:00: _OSC: OS supports [ExtendedConfig ASPM ClockPM Segments MSI]
[    7.331244] acpi PNP0A08:00: _OSC: platform does not support [PCIeHotplug]
[    7.337043] acpi PNP0A08:00: _OSC: OS now controls [PME AER PCIeCapability]
[    7.340955] PCI host bridge to bus 0000:00
[    7.340968] pci_bus 0000:00: root bus resource [bus 00-ff]
[    7.340978] pci_bus 0000:00: root bus resource [io  0x0000-0x0cf7 window]
[    7.340989] pci_bus 0000:00: root bus resource [io  0x0d00-0xffff window]
[    7.340999] pci_bus 0000:00: root bus resource [mem 0x000a0000-0x000bffff window]
[    7.341011] pci_bus 0000:00: root bus resource [mem 0x000d0000-0x000dffff window]
[    7.341022] pci_bus 0000:00: root bus resource [mem 0xc0000000-0xdfffffff window]
[    7.341033] pci_bus 0000:00: root bus resource [mem 0xf0000000-0xfed8ffff window]
[    7.341075] pci 0000:00:00.0: [8086:3406] type 00 class 0x060000
[    7.341259] pci 0000:00:00.0: PME# supported from D0 D3hot D3cold
[    7.342719] pci 0000:00:01.0: [8086:3408] type 01 class 0x060400
[    7.342900] pci 0000:00:01.0: PME# supported from D0 D3hot D3cold
[    7.343767] pci 0000:00:01.0: System wakeup disabled by ACPI
[    7.344537] pci 0000:00:03.0: [8086:340a] type 01 class 0x060400
[    7.344708] pci 0000:00:03.0: PME# supported from D0 D3hot D3cold
[    7.345574] pci 0000:00:03.0: System wakeup disabled by ACPI
[    7.346342] pci 0000:00:05.0: [8086:340c] type 01 class 0x060400
[    7.346510] pci 0000:00:05.0: PME# supported from D0 D3hot D3cold
[    7.347369] pci 0000:00:05.0: System wakeup disabled by ACPI
[    7.348134] pci 0000:00:07.0: [8086:340e] type 01 class 0x060400
[    7.348313] pci 0000:00:07.0: PME# supported from D0 D3hot D3cold
[    7.349175] pci 0000:00:07.0: System wakeup disabled by ACPI
[    7.349952] pci 0000:00:09.0: [8086:3410] type 01 class 0x060400
[    7.350133] pci 0000:00:09.0: PME# supported from D0 D3hot D3cold
[    7.350997] pci 0000:00:09.0: System wakeup disabled by ACPI
[    7.351790] pci 0000:00:13.0: [8086:342d] type 00 class 0x080020
[    7.351817] pci 0000:00:13.0: reg 0x10: [mem 0xfec8a000-0xfec8afff]
[    7.351967] pci 0000:00:13.0: PME# supported from D0 D3hot D3cold
[    7.353413] pci 0000:00:14.0: [8086:342e] type 00 class 0x080000
[    7.354825] pci 0000:00:14.1: [8086:3422] type 00 class 0x080000
[    7.356229] pci 0000:00:14.2: [8086:3423] type 00 class 0x080000
[    7.357640] pci 0000:00:14.3: [8086:3438] type 00 class 0x080000
[    7.359043] pci 0000:00:16.0: [8086:3430] type 00 class 0x088000
[    7.359075] pci 0000:00:16.0: reg 0x10: [mem 0xfb8d8000-0xfb8dbfff 64bit]
[    7.360533] pci 0000:00:16.1: [8086:3431] type 00 class 0x088000
[    7.360564] pci 0000:00:16.1: reg 0x10: [mem 0xfb8dc000-0xfb8dffff 64bit]
[    7.362009] pci 0000:00:16.2: [8086:3432] type 00 class 0x088000
[    7.362040] pci 0000:00:16.2: reg 0x10: [mem 0xfb8e0000-0xfb8e3fff 64bit]
[    7.363493] pci 0000:00:16.3: [8086:3433] type 00 class 0x088000
[    7.363524] pci 0000:00:16.3: reg 0x10: [mem 0xfb8e4000-0xfb8e7fff 64bit]
[    7.364972] pci 0000:00:16.4: [8086:3429] type 00 class 0x088000
[    7.365003] pci 0000:00:16.4: reg 0x10: [mem 0xfb8e8000-0xfb8ebfff 64bit]
[    7.366446] pci 0000:00:16.5: [8086:342a] type 00 class 0x088000
[    7.366478] pci 0000:00:16.5: reg 0x10: [mem 0xfb8ec000-0xfb8effff 64bit]
[    7.367927] pci 0000:00:16.6: [8086:342b] type 00 class 0x088000
[    7.367959] pci 0000:00:16.6: reg 0x10: [mem 0xfb8f0000-0xfb8f3fff 64bit]
[    7.369412] pci 0000:00:16.7: [8086:342c] type 00 class 0x088000
[    7.369443] pci 0000:00:16.7: reg 0x10: [mem 0xfb8f4000-0xfb8f7fff 64bit]
[    7.370925] pci 0000:00:1a.0: [8086:3a37] type 00 class 0x0c0300
[    7.371184] pci 0000:00:1a.0: reg 0x20: [io  0x8400-0x841f]
[    7.372494] pci 0000:00:1a.0: System wakeup disabled by ACPI
[    7.373330] pci 0000:00:1a.1: [8086:3a38] type 00 class 0x0c0300
[    7.373556] pci 0000:00:1a.1: reg 0x20: [io  0x8480-0x849f]
[    7.374885] pci 0000:00:1a.1: System wakeup disabled by ACPI
[    7.375681] pci 0000:00:1a.2: [8086:3a39] type 00 class 0x0c0300
[    7.375911] pci 0000:00:1a.2: reg 0x20: [io  0x8800-0x881f]
[    7.377235] pci 0000:00:1a.2: System wakeup disabled by ACPI
[    7.378070] pci 0000:00:1a.7: [8086:3a3c] type 00 class 0x0c0320
[    7.378174] pci 0000:00:1a.7: reg 0x10: [mem 0xfb8fa000-0xfb8fa3ff]
[    7.378653] pci 0000:00:1a.7: PME# supported from D0 D3hot D3cold
[    7.379549] pci 0000:00:1a.7: System wakeup disabled by ACPI
[    7.380332] pci 0000:00:1c.0: [8086:3a40] type 01 class 0x060400
[    7.380822] pci 0000:00:1c.0: PME# supported from D0 D3hot D3cold
[    7.381747] pci 0000:00:1c.0: System wakeup disabled by ACPI
[    7.382557] pci 0000:00:1c.4: [8086:3a48] type 01 class 0x060400
[    7.383067] pci 0000:00:1c.4: PME# supported from D0 D3hot D3cold
[    7.384716] pci 0000:00:1d.0: [8086:3a34] type 00 class 0x0c0300
[    7.384947] pci 0000:00:1d.0: reg 0x20: [io  0x8880-0x889f]
[    7.386283] pci 0000:00:1d.0: System wakeup disabled by ACPI
[    7.387078] pci 0000:00:1d.1: [8086:3a35] type 00 class 0x0c0300
[    7.387309] pci 0000:00:1d.1: reg 0x20: [io  0x8c00-0x8c1f]
[    7.388684] pci 0000:00:1d.1: System wakeup disabled by ACPI
[    7.389481] pci 0000:00:1d.2: [8086:3a36] type 00 class 0x0c0300
[    7.389718] pci 0000:00:1d.2: reg 0x20: [io  0x9000-0x901f]
[    7.391078] pci 0000:00:1d.2: System wakeup disabled by ACPI
[    7.391919] pci 0000:00:1d.7: [8086:3a3a] type 00 class 0x0c0320
[    7.392053] pci 0000:00:1d.7: reg 0x10: [mem 0xfb8fc000-0xfb8fc3ff]
[    7.392547] pci 0000:00:1d.7: PME# supported from D0 D3hot D3cold
[    7.393426] pci 0000:00:1d.7: System wakeup disabled by ACPI
[    7.394230] pci 0000:00:1e.0: [8086:244e] type 01 class 0x060401
[    7.395395] pci 0000:00:1e.0: System wakeup disabled by ACPI
[    7.396225] pci 0000:00:1f.0: [8086:3a16] type 00 class 0x060100
[    7.396615] pci 0000:00:1f.0: can't claim BAR 13 [io  0x0800-0x087f]: address conflict with ACPI CPU throttle [io  0x0810-0x0815]
[    7.396634] pci 0000:00:1f.0: quirk: [io  0x0500-0x053f] claimed by ICH6 GPIO
[    7.396646] pci 0000:00:1f.0: ICH7 LPC Generic IO decode 1 PIO at 0a00 (mask 00ff)
[    7.396662] pci 0000:00:1f.0: ICH7 LPC Generic IO decode 3 PIO at 0290 (mask 001f)
[    7.396669] pci 0000:00:1f.0: ICH7 LPC Generic IO decode 4 PIO at 0ca0 (mask 000f)
[    7.398219] pci 0000:00:1f.2: [8086:3a20] type 00 class 0x01018f
[    7.398323] pci 0000:00:1f.2: reg 0x10: [io  0xac00-0xac07]
[    7.398369] pci 0000:00:1f.2: reg 0x14: [io  0xa880-0xa883]
[    7.398414] pci 0000:00:1f.2: reg 0x18: [io  0xa800-0xa807]
[    7.398459] pci 0000:00:1f.2: reg 0x1c: [io  0xa480-0xa483]
[    7.398495] pci 0000:00:1f.2: reg 0x20: [io  0xa400-0xa40f]
[    7.398538] pci 0000:00:1f.2: reg 0x24: [io  0xa080-0xa08f]
[    7.400205] pci 0000:00:1f.3: [8086:3a30] type 00 class 0x0c0500
[    7.400295] pci 0000:00:1f.3: reg 0x10: [mem 0xfb8fe000-0xfb8fe0ff 64bit]
[    7.400418] pci 0000:00:1f.3: reg 0x20: [io  0x0400-0x041f]
[    7.401977] pci 0000:00:1f.5: [8086:3a26] type 00 class 0x010185
[    7.402073] pci 0000:00:1f.5: reg 0x10: [io  0x9c00-0x9c07]
[    7.402119] pci 0000:00:1f.5: reg 0x14: [io  0x9880-0x9883]
[    7.402163] pci 0000:00:1f.5: reg 0x18: [io  0x9800-0x9807]
[    7.402209] pci 0000:00:1f.5: reg 0x1c: [io  0x9480-0x9483]
[    7.402253] pci 0000:00:1f.5: reg 0x20: [io  0x9400-0x940f]
[    7.402299] pci 0000:00:1f.5: reg 0x24: [io  0x9080-0x908f]
[    7.404550] pci 0000:0a:00.0: [8086:10c9] type 00 class 0x020000
[    7.404579] pci 0000:0a:00.0: reg 0x10: [mem 0xfbe20000-0xfbe3ffff]
[    7.404598] pci 0000:0a:00.0: reg 0x14: [mem 0xfbe00000-0xfbe1ffff]
[    7.404616] pci 0000:0a:00.0: reg 0x18: [io  0xe880-0xe89f]
[    7.404653] pci 0000:0a:00.0: reg 0x1c: [mem 0xfbddc000-0xfbddffff]
[    7.404700] pci 0000:0a:00.0: reg 0x30: [mem 0xfbde0000-0xfbdfffff pref]
[    7.404877] pci 0000:0a:00.0: PME# supported from D0 D3hot D3cold
[    7.404936] pci 0000:0a:00.0: reg 0x184: [mem 0xfbda0000-0xfbda3fff 64bit]
[    7.404962] pci 0000:0a:00.0: reg 0x190: [mem 0xfbd80000-0xfbd83fff 64bit]
[    7.406047] pci 0000:0a:00.1: [8086:10c9] type 00 class 0x020000
[    7.406076] pci 0000:0a:00.1: reg 0x10: [mem 0xfbee0000-0xfbefffff]
[    7.406094] pci 0000:0a:00.1: reg 0x14: [mem 0xfbec0000-0xfbedffff]
[    7.406113] pci 0000:0a:00.1: reg 0x18: [io  0xec00-0xec1f]
[    7.406131] pci 0000:0a:00.1: reg 0x1c: [mem 0xfbe9c000-0xfbe9ffff]
[    7.406178] pci 0000:0a:00.1: reg 0x30: [mem 0xfbea0000-0xfbebffff pref]
[    7.406355] pci 0000:0a:00.1: PME# supported from D0 D3hot D3cold
[    7.406407] pci 0000:0a:00.1: reg 0x184: [mem 0xfbe60000-0xfbe63fff 64bit]
[    7.406433] pci 0000:0a:00.1: reg 0x190: [mem 0xfbe40000-0xfbe43fff 64bit]
[    7.409866] pci 0000:00:01.0: PCI bridge to [bus 0a-0b]
[    7.409881] pci 0000:00:01.0:   bridge window [io  0xe000-0xefff]
[    7.409892] pci 0000:00:01.0:   bridge window [mem 0xfbd00000-0xfbefffff]
[    7.410357] pci 0000:00:03.0: PCI bridge to [bus 09]
[    7.410783] pci 0000:00:05.0: PCI bridge to [bus 08]
[    7.411232] pci 0000:00:07.0: PCI bridge to [bus 07]
[    7.411869] pci 0000:04:00.0: [8086:0329] type 01 class 0x060400
[    7.411907] pci 0000:04:00.0: PXH quirk detected; SHPC device MSI disabled
[    7.412124] pci 0000:04:00.0: PME# supported from D0 D3hot D3cold
[    7.412422] pci 0000:04:00.0: System wakeup disabled by ACPI
[    7.413281] pci 0000:04:00.1: [8086:0326] type 00 class 0x080020
[    7.413305] pci 0000:04:00.1: reg 0x10: [mem 0xfbbfc000-0xfbbfcfff]
[    7.414312] pci 0000:04:00.2: [8086:032a] type 01 class 0x060400
[    7.414372] pci 0000:04:00.2: PXH quirk detected; SHPC device MSI disabled
[    7.414580] pci 0000:04:00.2: PME# supported from D0 D3hot D3cold
[    7.414883] pci 0000:04:00.2: System wakeup disabled by ACPI
[    7.415686] pci 0000:04:00.3: [8086:0327] type 00 class 0x080020
[    7.415710] pci 0000:04:00.3: reg 0x10: [mem 0xfbbfe000-0xfbbfefff]
[    7.416666] pci 0000:04:00.0: disabling ASPM on pre-1.1 PCIe device.  You can enable it with 'pcie_aspm=force'
[    7.416700] pci 0000:00:09.0: PCI bridge to [bus 04-06]
[    7.416708] pci 0000:00:09.0:   bridge window [io  0xd000-0xdfff]
[    7.416716] pci 0000:00:09.0:   bridge window [mem 0xfbb00000-0xfbcfffff]
[    7.417197] pci 0000:06:01.0: [8086:1010] type 00 class 0x020000
[    7.417239] pci 0000:06:01.0: reg 0x10: [mem 0xfbcc0000-0xfbcdffff 64bit]
[    7.417283] pci 0000:06:01.0: reg 0x20: [io  0xd880-0xd8bf]
[    7.417501] pci 0000:06:01.0: PME# supported from D0 D3hot D3cold
[    7.418433] pci 0000:06:01.1: [8086:1010] type 00 class 0x020000
[    7.418475] pci 0000:06:01.1: reg 0x10: [mem 0xfbce0000-0xfbcfffff 64bit]
[    7.418518] pci 0000:06:01.1: reg 0x20: [io  0xdc00-0xdc3f]
[    7.418734] pci 0000:06:01.1: PME# supported from D0 D3hot D3cold
[    7.419696] pci 0000:04:00.0: PCI bridge to [bus 06]
[    7.419705] pci 0000:04:00.0:   bridge window [io  0xd000-0xdfff]
[    7.419713] pci 0000:04:00.0:   bridge window [mem 0xfbc00000-0xfbcfffff]
[    7.420217] pci 0000:04:00.2: PCI bridge to [bus 05]
[    7.420797] pci 0000:00:1c.0: PCI bridge to [bus 03]
[    7.421487] pci 0000:02:00.0: [197b:2368] type 00 class 0x010185
[    7.421599] pci 0000:02:00.0: reg 0x10: [io  0xc400-0xc407]
[    7.421647] pci 0000:02:00.0: reg 0x14: [io  0xcc00-0xcc03]
[    7.421692] pci 0000:02:00.0: reg 0x18: [io  0xc880-0xc887]
[    7.421737] pci 0000:02:00.0: reg 0x1c: [io  0xc800-0xc803]
[    7.421782] pci 0000:02:00.0: reg 0x20: [io  0xc480-0xc48f]
[    7.421914] pci 0000:02:00.0: reg 0x30: [mem 0xfbaf0000-0xfbafffff pref]
[    7.422979] pci 0000:02:00.0: disabling ASPM on pre-1.1 PCIe device.  You can enable it with 'pcie_aspm=force'
[    7.423024] pci 0000:00:1c.4: PCI bridge to [bus 02]
[    7.423040] pci 0000:00:1c.4:   bridge window [io  0xc000-0xcfff]
[    7.423057] pci 0000:00:1c.4:   bridge window [mem 0xfba00000-0xfbafffff]
[    7.423577] pci 0000:01:01.0: [1002:515e] type 00 class 0x030000
[    7.423661] pci 0000:01:01.0: reg 0x10: [mem 0xd8000000-0xdfffffff pref]
[    7.423699] pci 0000:01:01.0: reg 0x14: [io  0xb000-0xb0ff]
[    7.423737] pci 0000:01:01.0: reg 0x18: [mem 0xfb9d0000-0xfb9dffff]
[    7.423900] pci 0000:01:01.0: reg 0x30: [mem 0xfb9e0000-0xfb9fffff pref]
[    7.424092] pci 0000:01:01.0: supports D1 D2
[    7.424969] pci 0000:00:1e.0: PCI bridge to [bus 01] (subtractive decode)
[    7.424986] pci 0000:00:1e.0:   bridge window [io  0xb000-0xbfff]
[    7.425003] pci 0000:00:1e.0:   bridge window [mem 0xfb900000-0xfb9fffff]
[    7.425031] pci 0000:00:1e.0:   bridge window [mem 0xd8000000-0xdfffffff 64bit pref]
[    7.425044] pci 0000:00:1e.0:   bridge window [io  0x0000-0x0cf7 window] (subtractive decode)
[    7.425055] pci 0000:00:1e.0:   bridge window [io  0x0d00-0xffff window] (subtractive decode)
[    7.425067] pci 0000:00:1e.0:   bridge window [mem 0x000a0000-0x000bffff window] (subtractive decode)
[    7.425079] pci 0000:00:1e.0:   bridge window [mem 0x000d0000-0x000dffff window] (subtractive decode)
[    7.425090] pci 0000:00:1e.0:   bridge window [mem 0xc0000000-0xdfffffff window] (subtractive decode)
[    7.425102] pci 0000:00:1e.0:   bridge window [mem 0xf0000000-0xfed8ffff window] (subtractive decode)
[    7.426436] xen: registering gsi 13 triggering 1 polarity 0
[    7.436082] ACPI: PCI Interrupt Link [LNKA] (IRQs 3 4 5 6 *7 10 11 12 14 15)
[    7.436968] ACPI: PCI Interrupt Link [LNKB] (IRQs 3 4 5 6 7 *10 11 12 14 15)
[    7.437870] ACPI: PCI Interrupt Link [LNKC] (IRQs 3 4 5 6 7 10 *11 12 14 15)
[    7.438758] ACPI: PCI Interrupt Link [LNKD] (IRQs 3 4 *5 6 7 10 11 12 14 15)
[    7.439649] ACPI: PCI Interrupt Link [LNKE] (IRQs 3 4 5 6 7 10 11 12 14 15) *0, disabled.
[    7.440581] ACPI: PCI Interrupt Link [LNKF] (IRQs 3 4 5 6 7 10 11 12 *14 15)
[    7.441470] ACPI: PCI Interrupt Link [LNKG] (IRQs 3 4 5 6 7 10 11 12 14 15) *0, disabled.
[    7.442370] ACPI: PCI Interrupt Link [LNKH] (IRQs 3 4 5 6 7 10 11 12 14 *15)
[    7.442958] ACPI: Enabled 1 GPEs in block 00 to 3F
[    7.443261] xen:balloon: Initialising balloon driver
[    7.445008] xen_balloon: Initialising balloon driver
[    7.446343] vgaarb: setting as boot device: PCI:0000:01:01.0
[    7.446350] vgaarb: device added: PCI:0000:01:01.0,decodes=io+mem,owns=io+mem,locks=none
[    7.446354] vgaarb: loaded
[    7.446356] vgaarb: bridge control possible 0000:01:01.0
[    7.446848] ACPI: bus type USB registered
[    7.447123] usbcore: registered new interface driver usbfs
[    7.447253] usbcore: registered new interface driver hub
[    7.447363] usbcore: registered new device driver usb
[    7.447651] pps_core: LinuxPPS API ver. 1 registered
[    7.447654] pps_core: Software ver. 5.3.6 - Copyright 2005-2007 Rodolfo Giometti <giometti@linux.it>
[    7.447709] PTP clock support registered
[    7.447921] PCI: Using ACPI for IRQ routing
[    7.492286] PCI: Discovered peer bus fe
[    7.492297] PCI: root bus fe: using default resources
[    7.492309] PCI: Probing PCI hardware (bus fe)
[    7.492696] PCI host bridge to bus 0000:fe
[    7.492707] pci_bus 0000:fe: root bus resource [io  0x0000-0xffff]
[    7.492716] pci_bus 0000:fe: root bus resource [mem 0x00000000-0xffffffffff]
[    7.492720] pci_bus 0000:fe: No busn resource found for root bus, will use [bus fe-ff]
[    7.492728] pci_bus 0000:fe: busn_res: can not insert [bus fe-ff] under domain [bus 00-ff] (conflicts with (null) [bus 00-ff])
[    7.492756] pci 0000:fe:00.0: [8086:2c40] type 00 class 0x060000
[    7.493301] pci 0000:fe:00.1: [8086:2c01] type 00 class 0x060000
[    7.493821] pci 0000:fe:02.0: [8086:2c10] type 00 class 0x060000
[    7.494345] pci 0000:fe:02.1: [8086:2c11] type 00 class 0x060000
[    7.494851] pci 0000:fe:02.4: [8086:2c14] type 00 class 0x060000
[    7.495380] pci 0000:fe:02.5: [8086:2c15] type 00 class 0x060000
[    7.495894] pci 0000:fe:03.0: [8086:2c18] type 00 class 0x060000
[    7.496408] pci 0000:fe:03.1: [8086:2c19] type 00 class 0x060000
[    7.496912] pci 0000:fe:03.2: [8086:2c1a] type 00 class 0x060000
[    7.497429] pci 0000:fe:03.4: [8086:2c1c] type 00 class 0x060000
[    7.497942] pci 0000:fe:04.0: [8086:2c20] type 00 class 0x060000
[    7.498452] pci 0000:fe:04.1: [8086:2c21] type 00 class 0x060000
[    7.498965] pci 0000:fe:04.2: [8086:2c22] type 00 class 0x060000
[    7.499492] pci 0000:fe:04.3: [8086:2c23] type 00 class 0x060000
[    7.500001] pci 0000:fe:05.0: [8086:2c28] type 00 class 0x060000
[    7.500516] pci 0000:fe:05.1: [8086:2c29] type 00 class 0x060000
[    7.501014] pci 0000:fe:05.2: [8086:2c2a] type 00 class 0x060000
[    7.501526] pci 0000:fe:05.3: [8086:2c2b] type 00 class 0x060000
[    7.502032] pci 0000:fe:06.0: [8086:2c30] type 00 class 0x060000
[    7.502543] pci 0000:fe:06.1: [8086:2c31] type 00 class 0x060000
[    7.503059] pci 0000:fe:06.2: [8086:2c32] type 00 class 0x060000
[    7.503583] pci 0000:fe:06.3: [8086:2c33] type 00 class 0x060000
[    7.504121] pci_bus 0000:fe: busn_res: [bus fe-ff] end is updated to fe
[    7.504127] pci_bus 0000:fe: busn_res: can not insert [bus fe] under domain [bus 00-ff] (conflicts with (null) [bus 00-ff])
[    7.504140] PCI: Discovered peer bus ff
[    7.504146] PCI: root bus ff: using default resources
[    7.504158] PCI: Probing PCI hardware (bus ff)
[    7.504538] PCI host bridge to bus 0000:ff
[    7.504548] pci_bus 0000:ff: root bus resource [io  0x0000-0xffff]
[    7.504557] pci_bus 0000:ff: root bus resource [mem 0x00000000-0xffffffffff]
[    7.504561] pci_bus 0000:ff: No busn resource found for root bus, will use [bus ff-ff]
[    7.504567] pci_bus 0000:ff: busn_res: can not insert [bus ff] under domain [bus 00-ff] (conflicts with (null) [bus 00-ff])
[    7.504597] pci 0000:ff:00.0: [8086:2c40] type 00 class 0x060000
[    7.505114] pci 0000:ff:00.1: [8086:2c01] type 00 class 0x060000
[    7.505647] pci 0000:ff:02.0: [8086:2c10] type 00 class 0x060000
[    7.506145] pci 0000:ff:02.1: [8086:2c11] type 00 class 0x060000
[    7.506655] pci 0000:ff:02.4: [8086:2c14] type 00 class 0x060000
[    7.507147] pci 0000:ff:02.5: [8086:2c15] type 00 class 0x060000
[    7.507657] pci 0000:ff:03.0: [8086:2c18] type 00 class 0x060000
[    7.508152] pci 0000:ff:03.1: [8086:2c19] type 00 class 0x060000
[    7.508644] pci 0000:ff:03.2: [8086:2c1a] type 00 class 0x060000
[    7.509151] pci 0000:ff:03.4: [8086:2c1c] type 00 class 0x060000
[    7.509674] pci 0000:ff:04.0: [8086:2c20] type 00 class 0x060000
[    7.510184] pci 0000:ff:04.1: [8086:2c21] type 00 class 0x060000
[    7.510699] pci 0000:ff:04.2: [8086:2c22] type 00 class 0x060000
[    7.511205] pci 0000:ff:04.3: [8086:2c23] type 00 class 0x060000
[    7.511740] pci 0000:ff:05.0: [8086:2c28] type 00 class 0x060000
[    7.512244] pci 0000:ff:05.1: [8086:2c29] type 00 class 0x060000
[    7.512738] pci 0000:ff:05.2: [8086:2c2a] type 00 class 0x060000
[    7.513264] pci 0000:ff:05.3: [8086:2c2b] type 00 class 0x060000
[    7.513766] pci 0000:ff:06.0: [8086:2c30] type 00 class 0x060000
[    7.514269] pci 0000:ff:06.1: [8086:2c31] type 00 class 0x060000
[    7.514763] pci 0000:ff:06.2: [8086:2c32] type 00 class 0x060000
[    7.515279] pci 0000:ff:06.3: [8086:2c33] type 00 class 0x060000
[    7.515811] pci_bus 0000:ff: busn_res: [bus ff] end is updated to ff
[    7.515817] pci_bus 0000:ff: busn_res: can not insert [bus ff] under domain [bus 00-ff] (conflicts with (null) [bus 00-ff])
[    7.515828] PCI: pci_cache_line_size set to 64 bytes
[    7.516282] e820: reserve RAM buffer [mem 0x0009e000-0x0009ffff]
[    7.516293] e820: reserve RAM buffer [mem 0x4d062000-0x4fffffff]
[    7.517766] NetLabel: Initializing
[    7.517771] NetLabel:  domain hash size = 128
[    7.517774] NetLabel:  protocols = UNLABELED CIPSOv4
[    7.517871] NetLabel:  unlabeled traffic allowed by default
[    7.518566] Switched to clocksource xen
[    7.570138] pnp: PnP ACPI init
[    7.570808] system 00:00: [mem 0xfbf00000-0xfbffffff] could not be reserved
[    7.570828] system 00:00: [mem 0xfc000000-0xfcffffff] has been reserved
[    7.570855] system 00:00: [mem 0xfd000000-0xfdffffff] has been reserved
[    7.570873] system 00:00: [mem 0xfe000000-0xfebfffff] has been reserved
[    7.570893] system 00:00: [mem 0xfec8a000-0xfec8afff] could not be reserved
[    7.570910] system 00:00: [mem 0xfed10000-0xfed10fff] has been reserved
[    7.570917] system 00:00: Plug and Play ACPI device, IDs PNP0c01 (active)
[    7.571365] xen: registering gsi 8 triggering 1 polarity 0
[    7.571628] pnp 00:01: Plug and Play ACPI device, IDs PNP0b00 (active)
[    7.574772] xen: registering gsi 4 triggering 1 polarity 0
[    7.574778] Already setup the GSI :4
[    7.574794] pnp 00:02: [dma 0 disabled]
[    7.575252] pnp 00:02: Plug and Play ACPI device, IDs PNP0501 (active)
[    7.578340] xen: registering gsi 3 triggering 1 polarity 0
[    7.578365] pnp 00:03: [dma 0 disabled]
[    7.578914] pnp 00:03: Plug and Play ACPI device, IDs PNP0501 (active)
[    7.582981] system 00:04: [io  0x04d0-0x04d1] has been reserved
[    7.583003] system 00:04: [io  0x0800-0x087f] could not be reserved
[    7.583021] system 00:04: [io  0x0500-0x057f] could not be reserved
[    7.583041] system 00:04: [mem 0xfed1c000-0xfed1ffff] has been reserved
[    7.583059] system 00:04: [mem 0xfed20000-0xfed3ffff] has been reserved
[    7.583077] system 00:04: [mem 0xfed40000-0xfed8ffff] has been reserved
[    7.583083] system 00:04: Plug and Play ACPI device, IDs PNP0c02 (active)
[    7.585655] system 00:05: [mem 0xfec00000-0xfec00fff] could not be reserved
[    7.585675] system 00:05: [mem 0xfee00000-0xfee00fff] has been reserved
[    7.585680] system 00:05: Plug and Play ACPI device, IDs PNP0c02 (active)
[    7.587656] system 00:06: [io  0x0a00-0x0a0f] has been reserved
[    7.587663] system 00:06: Plug and Play ACPI device, IDs PNP0c02 (active)
[    7.588298] system 00:07: [io  0x0ca2-0x0ca3] has been reserved
[    7.588304] system 00:07: Plug and Play ACPI device, IDs PNP0c02 (active)
[    7.589355] system 00:08: [mem 0xe0000000-0xefffffff] has been reserved
[    7.589362] system 00:08: Plug and Play ACPI device, IDs PNP0c02 (active)
[    7.592615] system 00:09: [mem 0x00000000-0x0009ffff] could not be reserved
[    7.592638] system 00:09: [mem 0x000c0000-0x000cffff] could not be reserved
[    7.592656] system 00:09: [mem 0x000e0000-0x000fffff] could not be reserved
[    7.592676] system 00:09: [mem 0x00100000-0xbfffffff] could not be reserved
[    7.592695] system 00:09: [mem 0xfed90000-0xffffffff] could not be reserved
[    7.592700] system 00:09: Plug and Play ACPI device, IDs PNP0c01 (active)
[    7.594614] pnp: PnP ACPI: found 10 devices
[    7.595738] pciback 0000:0a:00.0: seizing device
[    7.595778] pciback 0000:0a:00.1: seizing device
[    7.600168] kworker/u6:0 (581) used greatest stack depth: 13568 bytes left
[    7.624628] PM-Timer failed consistency check  (0xffffff) - aborting.
[    7.624778] pci 0000:00:1c.0: bridge window [io  0x1000-0x0fff] to [bus 03] add_size 1000
[    7.624790] pci 0000:00:1c.0: bridge window [mem 0x00100000-0x000fffff 64bit pref] to [bus 03] add_size 200000
[    7.624801] pci 0000:00:1c.0: bridge window [mem 0x00100000-0x000fffff] to [bus 03] add_size 200000
[    7.624849] pci 0000:00:1c.4: bridge window [mem 0x00100000-0x000fffff 64bit pref] to [bus 02] add_size 200000
[    7.624926] pci 0000:00:1f.0: BAR 13: [io  0x0800-0x087f] has bogus alignment
[    7.624948] pci 0000:00:1c.0: res[14]=[mem 0x00100000-0x000fffff] get_res_add_size add_size 200000
[    7.624954] pci 0000:00:1c.0: res[15]=[mem 0x00100000-0x000fffff 64bit pref] get_res_add_size add_size 200000
[    7.624959] pci 0000:00:1c.4: res[15]=[mem 0x00100000-0x000fffff 64bit pref] get_res_add_size add_size 200000
[    7.624965] pci 0000:00:1c.0: res[13]=[io  0x1000-0x0fff] get_res_add_size add_size 1000
[    7.624975] pci 0000:00:1c.0: BAR 14: assigned [mem 0xc0000000-0xc01fffff]
[    7.624983] pci 0000:00:1c.0: BAR 15: assigned [mem 0xc0200000-0xc03fffff 64bit pref]
[    7.624990] pci 0000:00:1c.4: BAR 15: assigned [mem 0xc0400000-0xc05fffff 64bit pref]
[    7.624995] pci 0000:00:1c.0: BAR 13: assigned [io  0x1000-0x1fff]
[    7.625005] pci 0000:00:01.0: PCI bridge to [bus 0a-0b]
[    7.625011] pci 0000:00:01.0:   bridge window [io  0xe000-0xefff]
[    7.625021] pci 0000:00:01.0:   bridge window [mem 0xfbd00000-0xfbefffff]
[    7.625035] pci 0000:00:03.0: PCI bridge to [bus 09]
[    7.625055] pci 0000:00:05.0: PCI bridge to [bus 08]
[    7.625074] pci 0000:00:07.0: PCI bridge to [bus 07]
[    7.625097] pci 0000:04:00.0: PCI bridge to [bus 06]
[    7.625124] pci 0000:04:00.0:   bridge window [io  0xd000-0xdfff]
[    7.625134] pci 0000:04:00.0:   bridge window [mem 0xfbc00000-0xfbcfffff]
[    7.625148] pci 0000:04:00.2: PCI bridge to [bus 05]
[    7.625168] pci 0000:00:09.0: PCI bridge to [bus 04-06]
[    7.625174] pci 0000:00:09.0:   bridge window [io  0xd000-0xdfff]
[    7.625183] pci 0000:00:09.0:   bridge window [mem 0xfbb00000-0xfbcfffff]
[    7.625197] pci 0000:00:1c.0: PCI bridge to [bus 03]
[    7.625207] pci 0000:00:1c.0:   bridge window [io  0x1000-0x1fff]
[    7.625229] pci 0000:00:1c.0:   bridge window [mem 0xc0000000-0xc01fffff]
[    7.625245] pci 0000:00:1c.0:   bridge window [mem 0xc0200000-0xc03fffff 64bit pref]
[    7.625272] pci 0000:00:1c.4: PCI bridge to [bus 02]
[    7.625282] pci 0000:00:1c.4:   bridge window [io  0xc000-0xcfff]
[    7.625300] pci 0000:00:1c.4:   bridge window [mem 0xfba00000-0xfbafffff]
[    7.625316] pci 0000:00:1c.4:   bridge window [mem 0xc0400000-0xc05fffff 64bit pref]
[    7.625344] pci 0000:00:1e.0: PCI bridge to [bus 01]
[    7.625354] pci 0000:00:1e.0:   bridge window [io  0xb000-0xbfff]
[    7.625376] pci 0000:00:1e.0:   bridge window [mem 0xfb900000-0xfb9fffff]
[    7.625392] pci 0000:00:1e.0:   bridge window [mem 0xd8000000-0xdfffffff 64bit pref]
[    7.625421] pci_bus 0000:00: resource 4 [io  0x0000-0x0cf7 window]
[    7.625426] pci_bus 0000:00: resource 5 [io  0x0d00-0xffff window]
[    7.625430] pci_bus 0000:00: resource 6 [mem 0x000a0000-0x000bffff window]
[    7.625434] pci_bus 0000:00: resource 7 [mem 0x000d0000-0x000dffff window]
[    7.625438] pci_bus 0000:00: resource 8 [mem 0xc0000000-0xdfffffff window]
[    7.625442] pci_bus 0000:00: resource 9 [mem 0xf0000000-0xfed8ffff window]
[    7.625447] pci_bus 0000:0a: resource 0 [io  0xe000-0xefff]
[    7.625450] pci_bus 0000:0a: resource 1 [mem 0xfbd00000-0xfbefffff]
[    7.625455] pci_bus 0000:04: resource 0 [io  0xd000-0xdfff]
[    7.625458] pci_bus 0000:04: resource 1 [mem 0xfbb00000-0xfbcfffff]
[    7.625463] pci_bus 0000:06: resource 0 [io  0xd000-0xdfff]
[    7.625466] pci_bus 0000:06: resource 1 [mem 0xfbc00000-0xfbcfffff]
[    7.625471] pci_bus 0000:03: resource 0 [io  0x1000-0x1fff]
[    7.625474] pci_bus 0000:03: resource 1 [mem 0xc0000000-0xc01fffff]
[    7.625479] pci_bus 0000:03: resource 2 [mem 0xc0200000-0xc03fffff 64bit pref]
[    7.625483] pci_bus 0000:02: resource 0 [io  0xc000-0xcfff]
[    7.625487] pci_bus 0000:02: resource 1 [mem 0xfba00000-0xfbafffff]
[    7.625491] pci_bus 0000:02: resource 2 [mem 0xc0400000-0xc05fffff 64bit pref]
[    7.625495] pci_bus 0000:01: resource 0 [io  0xb000-0xbfff]
[    7.625499] pci_bus 0000:01: resource 1 [mem 0xfb900000-0xfb9fffff]
[    7.625503] pci_bus 0000:01: resource 2 [mem 0xd8000000-0xdfffffff 64bit pref]
[    7.625508] pci_bus 0000:01: resource 4 [io  0x0000-0x0cf7 window]
[    7.625512] pci_bus 0000:01: resource 5 [io  0x0d00-0xffff window]
[    7.625516] pci_bus 0000:01: resource 6 [mem 0x000a0000-0x000bffff window]
[    7.625520] pci_bus 0000:01: resource 7 [mem 0x000d0000-0x000dffff window]
[    7.625525] pci_bus 0000:01: resource 8 [mem 0xc0000000-0xdfffffff window]
[    7.625529] pci_bus 0000:01: resource 9 [mem 0xf0000000-0xfed8ffff window]
[    7.625535] pci_bus 0000:fe: resource 4 [io  0x0000-0xffff]
[    7.625539] pci_bus 0000:fe: resource 5 [mem 0x00000000-0xffffffffff]
[    7.625545] pci_bus 0000:ff: resource 4 [io  0x0000-0xffff]
[    7.625548] pci_bus 0000:ff: resource 5 [mem 0x00000000-0xffffffffff]
[    7.625922] NET: Registered protocol family 2
[    7.626814] TCP established hash table entries: 16384 (order: 5, 131072 bytes)
[    7.626920] TCP bind hash table entries: 16384 (order: 6, 262144 bytes)
[    7.626961] TCP: Hash tables configured (established 16384 bind 16384)
[    7.627022] TCP: reno registered
[    7.627054] UDP hash table entries: 1024 (order: 3, 32768 bytes)
[    7.627090] UDP-Lite hash table entries: 1024 (order: 3, 32768 bytes)
[    7.627552] NET: Registered protocol family 1
[    7.628085] RPC: Registered named UNIX socket transport module.
[    7.628090] RPC: Registered udp transport module.
[    7.628093] RPC: Registered tcp transport module.
[    7.628096] RPC: Registered tcp NFSv4.1 backchannel transport module.
[    7.629914] xen: registering gsi 16 triggering 0 polarity 1
[    7.629970] xen: --> pirq=16 -> irq=16 (gsi=16)
[    7.633323] xen: registering gsi 21 triggering 0 polarity 1
[    7.633365] xen: --> pirq=21 -> irq=21 (gsi=21)
[    7.636697] xen: registering gsi 19 triggering 0 polarity 1
[    7.636747] xen: --> pirq=19 -> irq=19 (gsi=19)
[    7.640100] xen: registering gsi 18 triggering 0 polarity 1
[    7.640146] xen: --> pirq=18 -> irq=18 (gsi=18)
[    7.643516] xen: registering gsi 23 triggering 0 polarity 1
[    7.643569] xen: --> pirq=23 -> irq=23 (gsi=23)
[    7.646938] xen: registering gsi 19 triggering 0 polarity 1
[    7.646944] Already setup the GSI :19
[    7.650273] xen: registering gsi 18 triggering 0 polarity 1
[    7.650279] Already setup the GSI :18
[    7.653611] xen: registering gsi 23 triggering 0 polarity 1
[    7.653617] Already setup the GSI :23
[    7.655317] pci 0000:04:00.0: rerouting interrupts for [8086:0329]
[    7.655329] pci 0000:04:00.2: rerouting interrupts for [8086:032a]
[    7.655365] pci 0000:01:01.0: Video device with shadowed ROM
[    7.655484] PCI: CLS 256 bytes, default 64
[    7.655777] Unpacking initramfs...
[   10.052805] Freeing initrd memory: 90344K (ffff880004000000 - ffff88000983a000)
[   10.054700] Machine check injector initialized
[   10.057898] Scanning for low memory corruption every 60 seconds
[   10.060539] futex hash table entries: 1024 (order: 4, 65536 bytes)
[   10.060740] audit: initializing netlink subsys (disabled)
[   10.060826] audit: type=2000 audit(1428695612.625:1): initialized
[   10.062716] HugeTLB registered 2 MB page size, pre-allocated 0 pages
[   10.063506] VFS: Disk quotas dquot_6.5.2
[   10.063715] VFS: Dquot-cache hash table entries: 512 (order 0, 4096 bytes)
[   10.065017] NFS: Registering the id_resolver key type
[   10.065054] Key type id_resolver registered
[   10.065057] Key type id_legacy registered
[   10.065142] ntfs: driver 2.1.31 [Flags: R/W].
[   10.065900] SELinux:  Registering netfilter hooks
[   10.068340] bounce: pool size: 64 pages
[   10.068454] Block layer SCSI generic (bsg) driver version 0.4 loaded (major 250)
[   10.068466] io scheduler noop registered
[   10.068474] io scheduler deadline registered
[   10.068584] io scheduler cfq registered (default)
[   10.073230] pcieport 0000:00:1c.0: enabling device (0104 -> 0107)
[   10.075338] xen: registering gsi 17 triggering 0 polarity 1
[   10.075393] xen: --> pirq=17 -> irq=17 (gsi=17)
[   10.078350] xen: registering gsi 17 triggering 0 polarity 1
[   10.078357] Already setup the GSI :17
[   10.079611] aer 0000:00:01.0:pcie02: service driver aer loaded
[   10.079768] aer 0000:00:03.0:pcie02: service driver aer loaded
[   10.079936] aer 0000:00:05.0:pcie02: service driver aer loaded
[   10.080099] aer 0000:00:07.0:pcie02: service driver aer loaded
[   10.080261] aer 0000:00:09.0:pcie02: service driver aer loaded
[   10.080409] pcieport 0000:00:01.0: Signaling PME through PCIe PME interrupt
[   10.080415] pciback 0000:0a:00.0: Signaling PME through PCIe PME interrupt
[   10.080420] pciback 0000:0a:00.1: Signaling PME through PCIe PME interrupt
[   10.080428] pcie_pme 0000:00:01.0:pcie01: service driver pcie_pme loaded
[   10.080493] pcieport 0000:00:03.0: Signaling PME through PCIe PME interrupt
[   10.080514] pcie_pme 0000:00:03.0:pcie01: service driver pcie_pme loaded
[   10.080588] pcieport 0000:00:05.0: Signaling PME through PCIe PME interrupt
[   10.080598] pcie_pme 0000:00:05.0:pcie01: service driver pcie_pme loaded
[   10.080699] pcieport 0000:00:07.0: Signaling PME through PCIe PME interrupt
[   10.080707] pcie_pme 0000:00:07.0:pcie01: service driver pcie_pme loaded
[   10.080770] pcieport 0000:00:09.0: Signaling PME through PCIe PME interrupt
[   10.080775] pci 0000:04:00.0: Signaling PME through PCIe PME interrupt
[   10.080780] pci 0000:06:01.0: Signaling PME through PCIe PME interrupt
[   10.080784] pci 0000:06:01.1: Signaling PME through PCIe PME interrupt
[   10.080789] pci 0000:04:00.1: Signaling PME through PCIe PME interrupt
[   10.080793] pci 0000:04:00.2: Signaling PME through PCIe PME interrupt
[   10.080798] pci 0000:04:00.3: Signaling PME through PCIe PME interrupt
[   10.080806] pcie_pme 0000:00:09.0:pcie01: service driver pcie_pme loaded
[   10.080974] pcieport 0000:00:1c.0: Signaling PME through PCIe PME interrupt
[   10.080988] pcie_pme 0000:00:1c.0:pcie01: service driver pcie_pme loaded
[   10.081151] pcieport 0000:00:1c.4: Signaling PME through PCIe PME interrupt
[   10.081157] pci 0000:02:00.0: Signaling PME through PCIe PME interrupt
[   10.081174] pcie_pme 0000:00:1c.4:pcie01: service driver pcie_pme loaded
[   10.081250] pci_hotplug: PCI Hot Plug PCI Core version: 0.5
[   10.081432] pciehp: PCI Express Hot Plug Controller Driver version: 0.4
[   10.081760] intel_idle: MWAIT substates: 0x1120
[   10.081799] intel_idle: v0.4 model 0x1A
[   10.081802] intel_idle: lapic_timer_reliable_states 0x2
[   10.081823] intel_idle: intel_idle yielding to none
[   10.082643] input: Power Button as /devices/LNXSYSTM:00/LNXSYBUS:00/PNP0C0C:00/input/input0
[   10.082660] ACPI: Power Button [PWRB]
[   10.083222] input: Power Button as /devices/LNXSYSTM:00/LNXPWRBN:00/input/input1
[   10.083234] ACPI: Power Button [PWRF]
[   10.086006] Monitor-Mwait will be used to enter C-3 state
[   10.086074] Monitor-Mwait will be used to enter C-3 state
[   10.090821] Warning: Processor Platform Limit not supported.
[   10.091177] GHES: HEST is not enabled!
[   10.091193] ioatdma: Intel(R) QuickData Technology Driver 4.00
[   10.093284] xen: registering gsi 43 triggering 0 polarity 1
[   10.093332] xen: --> pirq=43 -> irq=43 (gsi=43)
[   10.093971] ioatdma 0000:00:16.0: xen map irq failed -6 for 32752 domain
[   10.098244] xen: registering gsi 44 triggering 0 polarity 1
[   10.098292] xen: --> pirq=44 -> irq=44 (gsi=44)
[   10.098908] ioatdma 0000:00:16.1: xen map irq failed -6 for 32752 domain
[   10.103091] xen: registering gsi 45 triggering 0 polarity 1
[   10.103148] xen: --> pirq=45 -> irq=45 (gsi=45)
[   10.103719] ioatdma 0000:00:16.2: xen map irq failed -6 for 32752 domain
[   10.107852] xen: registering gsi 46 triggering 0 polarity 1
[   10.107902] xen: --> pirq=46 -> irq=46 (gsi=46)
[   10.108640] ioatdma 0000:00:16.3: xen map irq failed -6 for 32752 domain
[   10.112723] xen: registering gsi 43 triggering 0 polarity 1
[   10.112731] Already setup the GSI :43
[   10.113295] ioatdma 0000:00:16.4: xen map irq failed -6 for 32752 domain
[   10.117307] xen: registering gsi 44 triggering 0 polarity 1
[   10.117314] Already setup the GSI :44
[   10.117959] ioatdma 0000:00:16.5: xen map irq failed -6 for 32752 domain
[   10.121996] xen: registering gsi 45 triggering 0 polarity 1
[   10.122003] Already setup the GSI :45
[   10.122597] ioatdma 0000:00:16.6: xen map irq failed -6 for 32752 domain
[   10.126597] xen: registering gsi 46 triggering 0 polarity 1
[   10.126604] Already setup the GSI :46
[   10.127198] ioatdma 0000:00:16.7: xen map irq failed -6 for 32752 domain
[   10.132282] xen: registering gsi 40 triggering 0 polarity 1
[   10.132323] xen: --> pirq=40 -> irq=40 (gsi=40)
[   10.233549] xen: registering gsi 28 triggering 0 polarity 1
[   10.233610] xen: --> pirq=28 -> irq=28 (gsi=28)
[   10.335278] xen_pciback: backend is vpci
[   10.345540] xen_acpi_processor: Uploading Xen processor PM info
[   10.434431] random: nonblocking pool is initialized
[   10.482487] Serial: 8250/16550 driver, 4 ports, IRQ sharing enabled
[   10.504432] 00:03: ttyS1 at I/O 0x2f8 (irq = 3, base_baud = 115200) is a 16550A
[   10.508021] hpet_acpi_add: no address or irqs in _CRS
[   10.508332] Non-volatile memory driver v1.3
[   10.508631] Linux agpgart interface v0.103
[   10.509113] AMD IOMMUv2 driver by Joerg Roedel <jroedel@suse.de>
[   10.509117] AMD IOMMUv2 functionality not available on this system
[   10.509214] [drm] Initialized drm 1.1.0 20060810
[   10.521681] loop: module loaded
[   10.523093] zram: Created 1 device(s)
[   10.523587] libphy: Fixed MDIO Bus: probed
[   10.523602] tun: Universal TUN/TAP device driver, 1.6
[   10.523606] tun: (C) 1999-2004 Max Krasnyansky <maxk@qualcomm.com>
[   10.524031] ixgbevf: Intel(R) 10 Gigabit PCI Express Virtual Function Network Driver - version 2.12.1-k
[   10.524037] ixgbevf: Copyright (c) 2009 - 2012 Intel Corporation.
[   10.525235] ehci_hcd: USB 2.0 'Enhanced' Host Controller (EHCI) Driver
[   10.525247] ehci-pci: EHCI PCI platform driver
[   10.527985] xen: registering gsi 18 triggering 0 polarity 1
[   10.527997] Already setup the GSI :18
[   10.528165] ehci-pci 0000:00:1a.7: EHCI Host Controller
[   10.528989] ehci-pci 0000:00:1a.7: new USB bus registered, assigned bus number 1
[   10.529071] ehci-pci 0000:00:1a.7: debug port 1
[   10.533178] ehci-pci 0000:00:1a.7: cache line size of 256 is not supported
[   10.533359] ehci-pci 0000:00:1a.7: irq 18, io mem 0xfb8fa000
[   10.539666] ehci-pci 0000:00:1a.7: USB 2.0 started, EHCI 1.00
[   10.540493] usb usb1: New USB device found, idVendor=1d6b, idProduct=0002
[   10.540504] usb usb1: New USB device strings: Mfr=3, Product=2, SerialNumber=1
[   10.540511] usb usb1: Product: EHCI Host Controller
[   10.540516] usb usb1: Manufacturer: Linux 4.0.0-rc6upstream-00005-gc5be3c5-dirty ehci_hcd
[   10.540523] usb usb1: SerialNumber: 0000:00:1a.7
[   10.542605] hub 1-0:1.0: USB hub found
[   10.542732] hub 1-0:1.0: 6 ports detected
[   10.549435] xen: registering gsi 23 triggering 0 polarity 1
[   10.549445] Already setup the GSI :23
[   10.549592] ehci-pci 0000:00:1d.7: EHCI Host Controller
[   10.550326] ehci-pci 0000:00:1d.7: new USB bus registered, assigned bus number 2
[   10.550407] ehci-pci 0000:00:1d.7: debug port 1
[   10.554504] ehci-pci 0000:00:1d.7: cache line size of 256 is not supported
[   10.554685] ehci-pci 0000:00:1d.7: irq 23, io mem 0xfb8fc000
[   10.560740] ehci-pci 0000:00:1d.7: USB 2.0 started, EHCI 1.00
[   10.561579] usb usb2: New USB device found, idVendor=1d6b, idProduct=0002
[   10.561588] usb usb2: New USB device strings: Mfr=3, Product=2, SerialNumber=1
[   10.561595] usb usb2: Product: EHCI Host Controller
[   10.561600] usb usb2: Manufacturer: Linux 4.0.0-rc6upstream-00005-gc5be3c5-dirty ehci_hcd
[   10.561607] usb usb2: SerialNumber: 0000:00:1d.7
[   10.563608] hub 2-0:1.0: USB hub found
[   10.563729] hub 2-0:1.0: 6 ports detected
[   10.567851] ohci_hcd: USB 1.1 'Open' Host Controller (OHCI) Driver
[   10.567891] ohci-pci: OHCI PCI platform driver
[   10.568048] uhci_hcd: USB Universal Host Controller Interface driver
[   10.570865] xen: registering gsi 16 triggering 0 polarity 1
[   10.570873] Already setup the GSI :16
[   10.570945] uhci_hcd 0000:00:1a.0: UHCI Host Controller
[   10.571673] uhci_hcd 0000:00:1a.0: new USB bus registered, assigned bus number 3
[   10.571993] uhci_hcd 0000:00:1a.0: irq 16, io base 0x00008400
[   10.572940] usb usb3: New USB device found, idVendor=1d6b, idProduct=0001
[   10.572950] usb usb3: New USB device strings: Mfr=3, Product=2, SerialNumber=1
[   10.572956] usb usb3: Product: UHCI Host Controller
[   10.572962] usb usb3: Manufacturer: Linux 4.0.0-rc6upstream-00005-gc5be3c5-dirty uhci_hcd
[   10.572969] usb usb3: SerialNumber: 0000:00:1a.0
[   10.575009] hub 3-0:1.0: USB hub found
[   10.575119] hub 3-0:1.0: 2 ports detected
[   10.580048] xen: registering gsi 21 triggering 0 polarity 1
[   10.580057] Already setup the GSI :21
[   10.580129] uhci_hcd 0000:00:1a.1: UHCI Host Controller
[   10.580930] uhci_hcd 0000:00:1a.1: new USB bus registered, assigned bus number 4
[   10.581211] uhci_hcd 0000:00:1a.1: irq 21, io base 0x00008480
[   10.582191] usb usb4: New USB device found, idVendor=1d6b, idProduct=0001
[   10.582200] usb usb4: New USB device strings: Mfr=3, Product=2, SerialNumber=1
[   10.582207] usb usb4: Product: UHCI Host Controller
[   10.582213] usb usb4: Manufacturer: Linux 4.0.0-rc6upstream-00005-gc5be3c5-dirty uhci_hcd
[   10.582220] usb usb4: SerialNumber: 0000:00:1a.1
[   10.584242] hub 4-0:1.0: USB hub found
[   10.584346] hub 4-0:1.0: 2 ports detected
[   10.589178] xen: registering gsi 19 triggering 0 polarity 1
[   10.589187] Already setup the GSI :19
[   10.589259] uhci_hcd 0000:00:1a.2: UHCI Host Controller
[   10.590037] uhci_hcd 0000:00:1a.2: new USB bus registered, assigned bus number 5
[   10.590309] uhci_hcd 0000:00:1a.2: irq 19, io base 0x00008800
[   10.591199] usb usb5: New USB device found, idVendor=1d6b, idProduct=0001
[   10.591208] usb usb5: New USB device strings: Mfr=3, Product=2, SerialNumber=1
[   10.591215] usb usb5: Product: UHCI Host Controller
[   10.591220] usb usb5: Manufacturer: Linux 4.0.0-rc6upstream-00005-gc5be3c5-dirty uhci_hcd
[   10.591227] usb usb5: SerialNumber: 0000:00:1a.2
[   10.593268] hub 5-0:1.0: USB hub found
[   10.593371] hub 5-0:1.0: 2 ports detected
[   10.598095] xen: registering gsi 23 triggering 0 polarity 1
[   10.598105] Already setup the GSI :23
[   10.598173] uhci_hcd 0000:00:1d.0: UHCI Host Controller
[   10.598902] uhci_hcd 0000:00:1d.0: new USB bus registered, assigned bus number 6
[   10.599047] uhci_hcd 0000:00:1d.0: irq 23, io base 0x00008880
[   10.599947] usb usb6: New USB device found, idVendor=1d6b, idProduct=0001
[   10.599956] usb usb6: New USB device strings: Mfr=3, Product=2, SerialNumber=1
[   10.599963] usb usb6: Product: UHCI Host Controller
[   10.599969] usb usb6: Manufacturer: Linux 4.0.0-rc6upstream-00005-gc5be3c5-dirty uhci_hcd
[   10.599975] usb usb6: SerialNumber: 0000:00:1d.0
[   10.602055] hub 6-0:1.0: USB hub found
[   10.602178] hub 6-0:1.0: 2 ports detected
[   10.607176] xen: registering gsi 19 triggering 0 polarity 1
[   10.607184] Already setup the GSI :19
[   10.607255] uhci_hcd 0000:00:1d.1: UHCI Host Controller
[   10.607991] uhci_hcd 0000:00:1d.1: new USB bus registered, assigned bus number 7
[   10.608176] uhci_hcd 0000:00:1d.1: irq 19, io base 0x00008c00
[   10.609048] usb usb7: New USB device found, idVendor=1d6b, idProduct=0001
[   10.609058] usb usb7: New USB device strings: Mfr=3, Product=2, SerialNumber=1
[   10.609065] usb usb7: Product: UHCI Host Controller
[   10.609070] usb usb7: Manufacturer: Linux 4.0.0-rc6upstream-00005-gc5be3c5-dirty uhci_hcd
[   10.609077] usb usb7: SerialNumber: 0000:00:1d.1
[   10.611129] hub 7-0:1.0: USB hub found
[   10.611234] hub 7-0:1.0: 2 ports detected
[   10.616088] xen: registering gsi 18 triggering 0 polarity 1
[   10.616097] Already setup the GSI :18
[   10.616170] uhci_hcd 0000:00:1d.2: UHCI Host Controller
[   10.617004] uhci_hcd 0000:00:1d.2: new USB bus registered, assigned bus number 8
[   10.617155] uhci_hcd 0000:00:1d.2: irq 18, io base 0x00009000
[   10.618073] usb usb8: New USB device found, idVendor=1d6b, idProduct=0001
[   10.618083] usb usb8: New USB device strings: Mfr=3, Product=2, SerialNumber=1
[   10.618089] usb usb8: Product: UHCI Host Controller
[   10.618095] usb usb8: Manufacturer: Linux 4.0.0-rc6upstream-00005-gc5be3c5-dirty uhci_hcd
[   10.618102] usb usb8: SerialNumber: 0000:00:1d.2
[   10.620147] hub 8-0:1.0: USB hub found
[   10.620268] hub 8-0:1.0: 2 ports detected
[   10.622818] usbcore: registered new interface driver usblp
[   10.623138] i8042: PNP: No PS/2 controller found. Probing ports directly.
[   10.626354] serio: i8042 KBD port at 0x60,0x64 irq 1
[   10.626383] serio: i8042 AUX port at 0x60,0x64 irq 12
[   10.627041] mousedev: PS/2 mouse device common for all mice
[   10.628761] rtc_cmos 00:01: RTC can wake from S4
[   10.629918] rtc_cmos 00:01: rtc core: registered rtc_cmos as rtc0
[   10.630037] rtc_cmos 00:01: alarms up to one month, y3k, 114 bytes nvram
[   10.633039] xen: registering gsi 18 triggering 0 polarity 1
[   10.633048] Already setup the GSI :18
[   10.633153] i801_smbus 0000:00:1f.3: SMBus using PCI interrupt
[   10.634634] Netfilter messages via NETLINK v0.30.
[   10.634730] nf_conntrack version 0.5.0 (7143 buckets, 28572 max)
[   10.635063] ctnetlink v0.93: registering with nfnetlink.
[   10.635826] ip_tables: (C) 2000-2006 Netfilter Core Team
[   10.636082] TCP: cubic registered
[   10.636094] Initializing XFRM netlink socket
[   10.636416] NET: Registered protocol family 10
[   10.637958] ip6_tables: (C) 2000-2006 Netfilter Core Team
[   10.638813] sit: IPv6 over IPv4 tunneling driver
[   10.640839] NET: Registered protocol family 17
[   10.640884] bridge: automatic filtering via arp/ip/ip6tables has been deprecated. Update your scripts to load br_netfilter if you need this.
[   10.641083] Key type dns_resolver registered
[   10.642602] mce: Unable to init device /dev/mcelog (rc: -16)
[   10.644508] registered taskstats version 1
[   10.644720] kmemleak: Kernel memory leak detector initialized
[   10.644727] kmemleak: Automatic memory scanning thread started
[   10.651439]   Magic number: 11:94:900
[   10.651456] usb usb2-port1: hash matches
[   10.651541] tty ptyvd: hash matches
[   10.651683] acpi PNP0501:00: hash matches
[   10.652904] PM: Hibernation image not present or could not be loaded.
[   10.654731] Freeing unused kernel memory: 1996K (ffffffff81cd3000 - ffffffff81ec6000)
[   10.654740] Write protecting the kernel read-only data: 12288k
[   10.661151] Freeing unused kernel memory: 620K (ffff880001765000 - ffff880001800000)
[   10.662157] Freeing unused kernel memory: 1688K (ffff880001a5a000 - ffff880001c00000)
[   10.847700] usb 1-5: new high-speed USB device number 2 using ehci-pci
[   10.967062] usb 1-5: New USB device found, idVendor=14dd, idProduct=0002
[   10.967073] usb 1-5: New USB device strings: Mfr=1, Product=2, SerialNumber=3
[   10.967078] usb 1-5: Product: Multidevice
[   10.967081] usb 1-5: Manufacturer: Peppercon AG
[   10.967084] usb 1-5: SerialNumber: 0e55d5019f4991f87CE4F555C62C01B7
[   11.094536] xen_netfront: Initialising Xen virtual ethernet driver
[   11.127211] udevd (1468): /proc/1468/oom_adj is deprecated, please use /proc/1468/oom_score_adj instead.
[   11.349828] udevadm (1469) used greatest stack depth: 13504 bytes left
[   11.365892] modprobe (1996) used greatest stack depth: 13456 bytes left
[   11.440929] e1000: Intel(R) PRO/1000 Network Driver - version 7.3.21-k8-NAPI
[   11.440940] e1000: Copyright (c) 1999-2006 Intel Corporation.
[   11.441783] e1000 0000:06:01.0: PCI IRQ 48 -> rerouted to legacy IRQ 16
[   11.441790] xen: registering gsi 16 triggering 0 polarity 1
[   11.441799] Already setup the GSI :16
[   11.460785] SCSI subsystem initialized
[   11.506832] libata version 3.00 loaded.
[   11.509119] ata_piix 0000:00:1f.2: version 2.13
[   11.511453] xen: registering gsi 19 triggering 0 polarity 1
[   11.511464] Already setup the GSI :19
[   11.511512] ata_piix 0000:00:1f.2: MAP [ P0 P2 P1 P3 ]
[   11.564633] scsi host0: ata_piix
[   11.566244] scsi host1: ata_piix
[   11.568832] ata1: SATA max UDMA/133 cmd 0xac00 ctl 0xa880 bmdma 0xa400 irq 19
[   11.568861] ata2: SATA max UDMA/133 cmd 0xa800 ctl 0xa480 bmdma 0xa408 irq 19
[   11.571262] xen: registering gsi 19 triggering 0 polarity 1
[   11.571272] Already setup the GSI :19
[   11.571311] ata_piix 0000:00:1f.5: MAP [ P0 -- P1 -- ]
[   11.577992] modprobe (2695) used greatest stack depth: 13360 bytes left
[   11.605670] hidraw: raw HID events driver (C) Jiri Kosina
[   11.615437] scsi host2: ata_piix
[   11.627814] scsi host3: ata_piix
[   11.629025] ata3: SATA max UDMA/133 cmd 0x9c00 ctl 0x9880 bmdma 0x9400 irq 19
[   11.629035] ata4: SATA max UDMA/133 cmd 0x9800 ctl 0x9480 bmdma 0x9408 irq 19
[   11.636356] usbcore: registered new interface driver usbhid
[   11.636365] usbhid: USB HID core driver
[   11.636881] modprobe (2714) used greatest stack depth: 13160 bytes left
[   11.699039] [drm] radeon kernel modesetting enabled.
[   11.700537] xen: registering gsi 18 triggering 0 polarity 1
[   11.700547] Already setup the GSI :18
[   11.706699] [drm] initializing kernel modesetting (RV100 0x1002:0x515E 0x15D9:0xF280).
[   11.706761] [drm] register mmio base: 0xFB9D0000
[   11.706764] [drm] register mmio size: 65536
[   11.707070] radeon 0000:01:01.0: VRAM: 128M 0x00000000D8000000 - 0x00000000DFFFFFFF (32M used)
[   11.707077] radeon 0000:01:01.0: GTT: 512M 0x00000000B8000000 - 0x00000000D7FFFFFF
[   11.707215] [drm] Detected VRAM RAM=128M, BAR=128M
[   11.707219] [drm] RAM width 16bits DDR
[   11.711513] [TTM] Zone  kernel: Available graphics memory: 459312 kiB
[   11.711519] [TTM] Initializing pool allocator
[   11.711683] [TTM] Initializing DMA pool allocator
[   11.712009] [drm] radeon: 32M of VRAM memory ready
[   11.712020] [drm] radeon: 512M of GTT memory ready.
[   11.712353] [drm] GART: num cpu pages 131072, num gpu pages 131072
[   11.737919] [drm] PCI GART of 512M enabled (table at 0x00000000BF580000).
[   11.738165] radeon 0000:01:01.0: WB disabled
[   11.738172] radeon 0000:01:01.0: fence driver on ring 0 use gpu addr 0x00000000b8000000 and cpu addr 0xffff880028c87000
[   11.738186] [drm] Supports vblank timestamp caching Rev 2 (21.10.2013).
[   11.738190] [drm] Driver supports precise vblank timestamp query.
[   11.738237] [drm] radeon: irq initialized.
[   11.738395] [drm] Loading R100 Microcode
[   11.750631] e1000 0000:06:01.0 eth0: (PCI-X:133MHz:64-bit) 00:07:e9:19:e1:9e
[   11.750642] e1000 0000:06:01.0 eth0: Intel(R) PRO/1000 Network Connection
[   11.751665] e1000 0000:06:01.1: PCI IRQ 49 -> rerouted to legacy IRQ 17
[   11.751671] xen: registering gsi 17 triggering 0 polarity 1
[   11.751681] Already setup the GSI :17
[   11.763473] [drm] radeon: ring at 0x00000000B8001000
[   11.763530] [drm] ring test succeeded in 1 usecs
[   11.769555] [drm] ib test succeeded in 0 usecs
[   11.774990] [drm] No TV DAC info found in BIOS
[   11.775031] [drm] No valid Ext TMDS info found in BIOS
[   11.775820] [drm] Radeon Display Connectors
[   11.775824] [drm] Connector 0:
[   11.775826] [drm]   VGA-1
[   11.775829] [drm]   DDC: 0x60 0x60 0x60 0x60 0x60 0x60 0x60 0x60
[   11.775832] [drm]   Encoders:
[   11.775835] [drm]     CRT1: INTERNAL_DAC1
[   11.775837] [drm] Connector 1:
[   11.775840] [drm]   DVI-I-1
[   11.775842] [drm]   HPD2
[   11.775844] [drm]   DDC: 0x6c 0x6c 0x6c 0x6c 0x6c 0x6c 0x6c 0x6c
[   11.775847] [drm]   Encoders:
[   11.775849] [drm]     CRT2: INTERNAL_DAC2
[   11.775852] [drm]     DFP2: INTERNAL_DVO1
[   11.819296] [drm] fb mappable at 0xD8040000
[   11.819296] [drm] vram apper at 0xD8000000
[   11.819297] [drm] size 786432
[   11.819297] [drm] fb depth is 8
[   11.819298] [drm]    pitch is 1024
[   11.861853] fbcon: radeondrmfb (fb0) is primary device
[   11.940948] input: Peppercon AG Multidevice as /devices/pci0000:00/0000:00:1a.7/usb1/1-5/1-5:1.0/0003:14DD:0002.0001/input/input5
[   11.957063] ata3: SATA link down (SStatus 0 SControl 300)
[   11.959104] hid-generic 0003:14DD:0002.0001: input,hidraw0: USB HID v1.01 Mouse [Peppercon AG Multidevice] on usb-0000:00:1a.7-5/input0
[   11.960269] input: Peppercon AG Multidevice as /devices/pci0000:00/0000:00:1a.7/usb1/1-5/1-5:1.1/0003:14DD:0002.0002/input/input6
[   12.011593] hid-generic 0003:14DD:0002.0002: input,hidraw1: USB HID v1.01 Keyboard [Peppercon AG Multidevice] on usb-0000:00:1a.7-5/input1
[   12.012112] modprobe (2825) used greatest stack depth: 12632 bytes left
[   12.019006] Console: switching to colour frame buffer device 128x48
[   12.026813] radeon 0000:01:01.0: fb0: radeondrmfb frame buffer device
[   12.026817] radeon 0000:01:01.0: registered panic notifier
[   12.035681] [drm] Initialized radeon 2.41.0 20080528 for 0000:01:01.0 on minor 0
[   12.041668] modprobe (2716) used greatest stack depth: 12128 bytes left
[   12.051716] e1000 0000:06:01.1 eth1: (PCI-X:133MHz:64-bit) 00:07:e9:19:e1:9f
[   12.051726] e1000 0000:06:01.1 eth1: Intel(R) PRO/1000 Network Connection
[   12.052855] modprobe (2616) used greatest stack depth: 12088 bytes left
[   12.055883] Switched to clocksource tsc
[   12.088789] ata4: SATA link up 3.0 Gbps (SStatus 123 SControl 300)
[   12.106978] ata4.00: ATA-8: WDC WD3200AAJS-00VWA0, 12.01B02, max UDMA/133
[   12.106986] ata4.00: 625142448 sectors, multi 16: LBA48 NCQ (depth 0/32)
[   12.110180] ata4.00: configured for UDMA/133
[   12.222028] ata1.00: SATA link down (SStatus 0 SControl 300)
[   12.222124] ata1.01: SATA link down (SStatus 0 SControl 300)
[   12.357919] ata2.00: SATA link up 3.0 Gbps (SStatus 123 SControl 300)
[   12.358014] ata2.01: SATA link down (SStatus 0 SControl 300)
[   12.411356] ata2.00: ATA-7: ST380815AS, 3.AAC, max UDMA/133
[   12.411374] ata2.00: 156301488 sectors, multi 16: LBA48 NCQ (depth 0/32)
[   12.486308] ata2.00: configured for UDMA/133
[   12.487054] scsi 1:0:0:0: Direct-Access     ATA      ST380815AS       C    PQ: 0 ANSI: 5
[   12.491038] scsi 3:0:0:0: Direct-Access     ATA      WDC WD3200AAJS-0 1B02 PQ: 0 ANSI: 5
[   12.496429] modprobe (2631) used greatest stack depth: 10984 bytes left
[   12.498100] sd 1:0:0:0: [sda] 156301488 512-byte logical blocks: (80.0 GB/74.5 GiB)
[   12.498534] sd 1:0:0:0: [sda] Write Protect is off
[   12.498543] sd 1:0:0:0: [sda] Mode Sense: 00 3a 00 00
[   12.498737] sd 1:0:0:0: [sda] Write cache: enabled, read cache: enabled, doesn't support DPO or FUA
[   12.499174] sd 3:0:0:0: [sdb] 625142448 512-byte logical blocks: (320 GB/298 GiB)
[   12.499647] sd 3:0:0:0: [sdb] Write Protect is off
[   12.499656] sd 3:0:0:0: [sdb] Mode Sense: 00 3a 00 00
[   12.499831] sd 3:0:0:0: [sdb] Write cache: enabled, read cache: enabled, doesn't support DPO or FUA
[   12.511628] sd 1:0:0:0: Attached scsi generic sg0 type 0
[   12.512177] sd 3:0:0:0: Attached scsi generic sg1 type 0
[   12.551876]  sda: sda1 sda2 sda3 sda4
[   12.551995]  sdb: sdb1 sdb2 sdb3
[   12.556403] sd 3:0:0:0: [sdb] Attached SCSI disk
[   12.557068] sd 1:0:0:0: [sda] Attached SCSI disk
[   13.107829] IPv6: ADDRCONF(NETDEV_UP): eth0: link is not ready
[   13.112642] device eth0 entered promiscuous mode
[   13.209553] IPv6: ADDRCONF(NETDEV_UP): eth1: link is not ready
[   17.116157] e1000: eth0 NIC Link is Up 1000 Mbps Full Duplex, Flow Control: RX/TX
[   17.116335] IPv6: ADDRCONF(NETDEV_CHANGE): eth0: link becomes ready
[   17.220068] e1000: eth1 NIC Link is Up 1000 Mbps Full Duplex, Flow Control: RX/TX
[   17.220242] IPv6: ADDRCONF(NETDEV_CHANGE): eth1: link becomes ready
[   19.245227] switch: port 1(eth0) entered forwarding state
[   19.245278] switch: port 1(eth0) entered forwarding state
[   21.939946] Loading iSCSI transport class v2.0-870.
[   21.945115] iscsi: registered transport (tcp)
[   21.995848] xen:xen_evtchn: Event-channel device installed
[   25.352689] scsi host4: iSCSI Initiator over TCP/IP
[   25.397785]  connection1:0: detected conn error (1020)
[   25.660634] device-mapper: ioctl: 4.30.0-ioctl (2014-12-22) initialised: dm-devel@redhat.com
[   25.662537] device-mapper: multipath: version 1.8.0 loaded
[   34.280108] switch: port 1(eth0) entered forwarding state
[  118.128267] xen_pciback: vpci: 0000:0a:00.0: assign to virtual slot 0
[  118.129445] pciback 0000:0a:00.0: registering for 1
[  121.110113] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x0
[  121.110130] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x0 = 10c98086
[  121.110165] xen-pciback: 0000:0a:00.0: read 1 bytes at 0xe
[  121.110179] xen-pciback: 0000:0a:00.0: read 1 bytes at 0xe = 80
[  121.110211] xen-pciback: 0000:0a:00.0: read 2 bytes at 0x6
[  121.110246] xen-pciback: 0000:0a:00.0: read 2 bytes at 0x6 = 10
[  121.110259] xen-pciback: 0000:0a:00.0: read 1 bytes at 0x34
[  121.110268] xen-pciback: 0000:0a:00.0: read 1 bytes at 0x34 = 40
[  121.110286] xen-pciback: 0000:0a:00.0: read 1 bytes at 0x40
[  121.110297] xen-pciback: 0000:0a:00.0: read 1 bytes at 0x40 = 5001
[  121.110327] xen-pciback: 0000:0a:00.0: read 1 bytes at 0x41
[  121.110339] xen-pciback: 0000:0a:00.0: read 1 bytes at 0x41 = 50
[  121.110357] xen-pciback: 0000:0a:00.0: read 1 bytes at 0x50
[  121.110365] xen-pciback: 0000:0a:00.0: read 1 bytes at 0x50 = 5
[  121.110382] xen-pciback: 0000:0a:00.0: read 1 bytes at 0x51
[  121.110390] xen-pciback: 0000:0a:00.0: read 1 bytes at 0x51 = 70
[  121.110409] xen-pciback: 0000:0a:00.0: read 1 bytes at 0x70
[  121.110418] xen-pciback: 0000:0a:00.0: read 1 bytes at 0x70 = 11
[  121.110430] xen-pciback: 0000:0a:00.0: read 1 bytes at 0x71
[  121.110438] xen-pciback: 0000:0a:00.0: read 1 bytes at 0x71 = a0
[  121.110465] xen-pciback: 0000:0a:00.0: read 1 bytes at 0xa0
[  121.110474] xen-pciback: 0000:0a:00.0: read 1 bytes at 0xa0 = 10
[  121.110492] xen-pciback: 0000:0a:00.0: read 2 bytes at 0xa2
[  121.110501] xen-pciback: 0000:0a:00.0: read 2 bytes at 0xa2 = 2
[  121.110523] xen-pciback: 0000:0a:00.0: read 2 bytes at 0xa4
[  121.110532] xen-pciback: 0000:0a:00.0: read 2 bytes at 0xa4 = 8cc2
[  121.110573] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x8
[  121.110583] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x8 = 2000001
[  121.110638] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x100
[  121.110647] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x100 = 14010001
[  121.110669] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x0
[  121.110678] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x0 = 10c98086
[  121.110687] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x100
[  121.110696] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x100 = 14010001
[  121.110716] xen-pciback: 0000:0a:00.0: read 1 bytes at 0x3d
[  121.110728] xen-pciback: 0000:0a:00.0: read 1 bytes at 0x3d = 1
[  121.110751] xen-pciback: 0000:0a:00.0: read 1 bytes at 0x3c
[  121.110797] xen-pciback: 0000:0a:00.0: read 1 bytes at 0x3c = 1c
[  121.110822] xen-pciback: 0000:0a:00.0: read 2 bytes at 0x4
[  121.110835] xen-pciback: 0000:0a:00.0: read 2 bytes at 0x4 = 3
[  121.110870] xen-pciback: 0000:0a:00.0: write request 2 bytes at 0x4 = 0
[  121.110876] xen-pciback: 0000:0a:00.0: read 2 bytes at 0x4
[  121.110887] xen-pciback: 0000:0a:00.0: read 2 bytes at 0x4 = 3
[  121.110904] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x10
[  121.110913] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x10 = fbe20000
[  121.110934] xen-pciback: 0000:0a:00.0: write request 4 bytes at 0x10 = ffffffff
[  121.110940] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x10
[  121.110948] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x10 = fbe20000
[  121.110958] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x10
[  121.110969] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x10 = 20000
[  121.110992] xen-pciback: 0000:0a:00.0: write request 4 bytes at 0x10 = fbe20000
[  121.110999] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x10
[  121.111007] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x10 = 20000
[  121.111022] xen-pciback: 0000:0a:00.0: write request 2 bytes at 0x4 = 3
[  121.111027] xen-pciback: 0000:0a:00.0: read 2 bytes at 0x4
[  121.111038] xen-pciback: 0000:0a:00.0: read 2 bytes at 0x4 = 0
[  121.111043] xen-pciback: 0000:0a:00.0: enable
[  121.111626] xen: registering gsi 28 triggering 0 polarity 1
[  121.111639] Already setup the GSI :28
[  121.111859] xen-pciback: 0000:0a:00.0: read 2 bytes at 0x4
[  121.111872] xen-pciback: 0000:0a:00.0: read 2 bytes at 0x4 = 3
[  121.111901] xen-pciback: 0000:0a:00.0: write request 2 bytes at 0x4 = 0
[  121.111907] xen-pciback: 0000:0a:00.0: read 2 bytes at 0x4
[  121.111917] xen-pciback: 0000:0a:00.0: read 2 bytes at 0x4 = 3
[  121.111922] xen-pciback: 0000:0a:00.0: disable
[  121.112575] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x14
[  121.112586] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x14 = fbe00000
[  121.112622] xen-pciback: 0000:0a:00.0: write request 4 bytes at 0x14 = ffffffff
[  121.112629] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x14
[  121.112637] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x14 = fbe00000
[  121.112667] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x14
[  121.112676] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x14 = 20000
[  121.112703] xen-pciback: 0000:0a:00.0: write request 4 bytes at 0x14 = fbe00000
[  121.112710] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x14
[  121.112718] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x14 = 20000
[  121.112747] xen-pciback: 0000:0a:00.0: write request 2 bytes at 0x4 = 3
[  121.112753] xen-pciback: 0000:0a:00.0: read 2 bytes at 0x4
[  121.112764] xen-pciback: 0000:0a:00.0: read 2 bytes at 0x4 = 0
[  121.112769] xen-pciback: 0000:0a:00.0: enable
[  121.113277] xen: registering gsi 28 triggering 0 polarity 1
[  121.113285] Already setup the GSI :28
[  121.113381] xen-pciback: 0000:0a:00.0: read 2 bytes at 0x4
[  121.113399] xen-pciback: 0000:0a:00.0: read 2 bytes at 0x4 = 3
[  121.113423] xen-pciback: 0000:0a:00.0: write request 2 bytes at 0x4 = 0
[  121.113432] xen-pciback: 0000:0a:00.0: read 2 bytes at 0x4
[  121.113447] xen-pciback: 0000:0a:00.0: read 2 bytes at 0x4 = 3
[  121.113453] xen-pciback: 0000:0a:00.0: disable
[  121.114153] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x18
[  121.114168] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x18 = e881
[  121.114194] xen-pciback: 0000:0a:00.0: write request 4 bytes at 0x18 = ffffffff
[  121.114203] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x18
[  121.114218] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x18 = e881
[  121.114244] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x18
[  121.114253] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x18 = 20
[  121.114264] xen-pciback: 0000:0a:00.0: write request 4 bytes at 0x18 = e881
[  121.114269] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x18
[  121.114277] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x18 = 20
[  121.114297] xen-pciback: 0000:0a:00.0: write request 2 bytes at 0x4 = 3
[  121.114302] xen-pciback: 0000:0a:00.0: read 2 bytes at 0x4
[  121.114313] xen-pciback: 0000:0a:00.0: read 2 bytes at 0x4 = 0
[  121.114318] xen-pciback: 0000:0a:00.0: enable
[  121.114857] xen: registering gsi 28 triggering 0 polarity 1
[  121.114866] Already setup the GSI :28
[  121.114966] xen-pciback: 0000:0a:00.0: read 2 bytes at 0x4
[  121.114978] xen-pciback: 0000:0a:00.0: read 2 bytes at 0x4 = 3
[  121.114998] xen-pciback: 0000:0a:00.0: write request 2 bytes at 0x4 = 0
[  121.115004] xen-pciback: 0000:0a:00.0: read 2 bytes at 0x4
[  121.115014] xen-pciback: 0000:0a:00.0: read 2 bytes at 0x4 = 3
[  121.115019] xen-pciback: 0000:0a:00.0: disable
[  121.115633] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x1c
[  121.115645] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x1c = fbddc000
[  121.115657] xen-pciback: 0000:0a:00.0: write request 4 bytes at 0x1c = ffffffff
[  121.115663] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x1c
[  121.115671] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x1c = fbddc000
[  121.115682] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x1c
[  121.115690] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x1c = 4000
[  121.115712] xen-pciback: 0000:0a:00.0: write request 4 bytes at 0x1c = fbddc000
[  121.115718] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x1c
[  121.115727] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x1c = 4000
[  121.115742] xen-pciback: 0000:0a:00.0: write request 2 bytes at 0x4 = 3
[  121.115748] xen-pciback: 0000:0a:00.0: read 2 bytes at 0x4
[  121.115758] xen-pciback: 0000:0a:00.0: read 2 bytes at 0x4 = 0
[  121.115763] xen-pciback: 0000:0a:00.0: enable
[  121.116275] xen: registering gsi 28 triggering 0 polarity 1
[  121.116283] Already setup the GSI :28
[  121.116410] xen-pciback: 0000:0a:00.0: read 2 bytes at 0x4
[  121.116423] xen-pciback: 0000:0a:00.0: read 2 bytes at 0x4 = 3
[  121.116451] xen-pciback: 0000:0a:00.0: write request 2 bytes at 0x4 = 0
[  121.116456] xen-pciback: 0000:0a:00.0: read 2 bytes at 0x4
[  121.116467] xen-pciback: 0000:0a:00.0: read 2 bytes at 0x4 = 3
[  121.116472] xen-pciback: 0000:0a:00.0: disable
[  121.117015] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x20
[  121.117027] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x20 = 0
[  121.117054] xen-pciback: 0000:0a:00.0: write request 4 bytes at 0x20 = ffffffff
[  121.117060] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x20
[  121.117069] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x20 = 0
[  121.117085] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x20
[  121.117093] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x20 = 1
[  121.117112] xen-pciback: 0000:0a:00.0: write request 4 bytes at 0x20 = 0
[  121.117118] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x20
[  121.117126] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x20 = 1
[  121.117146] xen-pciback: 0000:0a:00.0: write request 2 bytes at 0x4 = 3
[  121.117152] xen-pciback: 0000:0a:00.0: read 2 bytes at 0x4
[  121.117163] xen-pciback: 0000:0a:00.0: read 2 bytes at 0x4 = 0
[  121.117168] xen-pciback: 0000:0a:00.0: enable
[  121.117835] xen: registering gsi 28 triggering 0 polarity 1
[  121.117846] Already setup the GSI :28
[  121.117938] xen-pciback: 0000:0a:00.0: read 2 bytes at 0x4
[  121.117955] xen-pciback: 0000:0a:00.0: read 2 bytes at 0x4 = 3
[  121.117972] xen-pciback: 0000:0a:00.0: write request 2 bytes at 0x4 = 0
[  121.117979] xen-pciback: 0000:0a:00.0: read 2 bytes at 0x4
[  121.117994] xen-pciback: 0000:0a:00.0: read 2 bytes at 0x4 = 3
[  121.118001] xen-pciback: 0000:0a:00.0: disable
[  121.118608] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x24
[  121.118620] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x24 = 0
[  121.118638] xen-pciback: 0000:0a:00.0: write request 4 bytes at 0x24 = ffffffff
[  121.118645] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x24
[  121.118653] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x24 = 0
[  121.118680] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x24
[  121.118689] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x24 = 1
[  121.118716] xen-pciback: 0000:0a:00.0: write request 4 bytes at 0x24 = 0
[  121.118723] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x24
[  121.118731] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x24 = 1
[  121.118760] xen-pciback: 0000:0a:00.0: write request 2 bytes at 0x4 = 3
[  121.118766] xen-pciback: 0000:0a:00.0: read 2 bytes at 0x4
[  121.118777] xen-pciback: 0000:0a:00.0: read 2 bytes at 0x4 = 0
[  121.118782] xen-pciback: 0000:0a:00.0: enable
[  121.119353] xen: registering gsi 28 triggering 0 polarity 1
[  121.119362] Already setup the GSI :28
[  121.119435] xen-pciback: 0000:0a:00.0: read 2 bytes at 0x4
[  121.119447] xen-pciback: 0000:0a:00.0: read 2 bytes at 0x4 = 3
[  121.119468] xen-pciback: 0000:0a:00.0: write request 2 bytes at 0x4 = 0
[  121.119474] xen-pciback: 0000:0a:00.0: read 2 bytes at 0x4
[  121.119485] xen-pciback: 0000:0a:00.0: read 2 bytes at 0x4 = 3
[  121.119490] xen-pciback: 0000:0a:00.0: disable
[  121.120000] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x30
[  121.120012] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x30 = fbde0000
[  121.120025] xen-pciback: 0000:0a:00.0: write request 4 bytes at 0x30 = fffff800
[  121.120031] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x30
[  121.120045] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x30 = fbde0000
[  121.120077] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x30
[  121.120086] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x30 = fbde0000
[  121.120112] xen-pciback: 0000:0a:00.0: write request 4 bytes at 0x30 = fbde0000
[  121.120119] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x30
[  121.120127] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x30 = fbde0000
[  121.120158] xen-pciback: 0000:0a:00.0: write request 2 bytes at 0x4 = 3
[  121.120164] xen-pciback: 0000:0a:00.0: read 2 bytes at 0x4
[  121.120175] xen-pciback: 0000:0a:00.0: read 2 bytes at 0x4 = 0
[  121.120180] xen-pciback: 0000:0a:00.0: enable
[  121.120771] xen: registering gsi 28 triggering 0 polarity 1
[  121.120781] Already setup the GSI :28
[  121.120872] xen-pciback: 0000:0a:00.0: read 2 bytes at 0x2c
[  121.120883] xen-pciback: 0000:0a:00.0: read 2 bytes at 0x2c = 15d9
[  121.120899] xen-pciback: 0000:0a:00.0: read 2 bytes at 0x2e
[  121.120908] xen-pciback: 0000:0a:00.0: read 2 bytes at 0x2e = 10c9
[  121.120979] xen-pciback: 0000:0a:00.0: read 2 bytes at 0x6
[  121.120988] xen-pciback: 0000:0a:00.0: read 2 bytes at 0x6 = 10
[  121.121000] xen-pciback: 0000:0a:00.0: read 1 bytes at 0x34
[  121.121008] xen-pciback: 0000:0a:00.0: read 1 bytes at 0x34 = 40
[  121.121025] xen-pciback: 0000:0a:00.0: read 1 bytes at 0x40
[  121.121036] xen-pciback: 0000:0a:00.0: read 1 bytes at 0x40 = 5001
[  121.121053] xen-pciback: 0000:0a:00.0: read 1 bytes at 0x41
[  121.121064] xen-pciback: 0000:0a:00.0: read 1 bytes at 0x41 = 50
[  121.121083] xen-pciback: 0000:0a:00.0: read 1 bytes at 0x50
[  121.121092] xen-pciback: 0000:0a:00.0: read 1 bytes at 0x50 = 5
[  121.121103] xen-pciback: 0000:0a:00.0: read 2 bytes at 0x52
[  121.121112] xen-pciback: 0000:0a:00.0: read 2 bytes at 0x52 = 180
[  121.121137] xen-pciback: 0000:0a:00.0: write request 2 bytes at 0x52 = 180
[  121.121146] pciback 0000:0a:00.0: Driver tried to write to a read-only configuration space field at offset 0x52, size 2. This may be harmless, but if you have problems with your device:
[  121.121146] 1) see permissive attribute in sysfs
[  121.121146] 2) report problems to the xen-devel mailing list along with details of your device obtained from lspci.
[  121.121178] xen-pciback: 0000:0a:00.0: read 2 bytes at 0x6
[  121.121186] xen-pciback: 0000:0a:00.0: read 2 bytes at 0x6 = 10
[  121.121212] xen-pciback: 0000:0a:00.0: read 1 bytes at 0x34
[  121.121234] xen-pciback: 0000:0a:00.0: read 1 bytes at 0x34 = 40
[  121.121252] xen-pciback: 0000:0a:00.0: read 1 bytes at 0x40
[  121.121263] xen-pciback: 0000:0a:00.0: read 1 bytes at 0x40 = 5001
[  121.121290] xen-pciback: 0000:0a:00.0: read 1 bytes at 0x41
[  121.121301] xen-pciback: 0000:0a:00.0: read 1 bytes at 0x41 = 50
[  121.121315] xen-pciback: 0000:0a:00.0: read 1 bytes at 0x50
[  121.121323] xen-pciback: 0000:0a:00.0: read 1 bytes at 0x50 = 5
[  121.121349] xen-pciback: 0000:0a:00.0: read 1 bytes at 0x51
[  121.121357] xen-pciback: 0000:0a:00.0: read 1 bytes at 0x51 = 70
[  121.121372] xen-pciback: 0000:0a:00.0: read 1 bytes at 0x70
[  121.121380] xen-pciback: 0000:0a:00.0: read 1 bytes at 0x70 = 11
[  121.121391] xen-pciback: 0000:0a:00.0: read 2 bytes at 0x72
[  121.121399] xen-pciback: 0000:0a:00.0: read 2 bytes at 0x72 = 9
[  121.121423] xen-pciback: 0000:0a:00.0: write request 2 bytes at 0x72 = 9
[  121.121441] xen-pciback: 0000:0a:00.0: read 2 bytes at 0x6
[  121.121450] xen-pciback: 0000:0a:00.0: read 2 bytes at 0x6 = 10
[  121.121461] xen-pciback: 0000:0a:00.0: read 1 bytes at 0x34
[  121.121468] xen-pciback: 0000:0a:00.0: read 1 bytes at 0x34 = 40
[  121.121487] xen-pciback: 0000:0a:00.0: read 1 bytes at 0x40
[  121.121498] xen-pciback: 0000:0a:00.0: read 1 bytes at 0x40 = 5001
[  121.121517] xen-pciback: 0000:0a:00.0: read 1 bytes at 0x41
[  121.121528] xen-pciback: 0000:0a:00.0: read 1 bytes at 0x41 = 50
[  121.121547] xen-pciback: 0000:0a:00.0: read 1 bytes at 0x50
[  121.121555] xen-pciback: 0000:0a:00.0: read 1 bytes at 0x50 = 5
[  121.121581] xen-pciback: 0000:0a:00.0: read 1 bytes at 0x51
[  121.121589] xen-pciback: 0000:0a:00.0: read 1 bytes at 0x51 = 70
[  121.121619] xen-pciback: 0000:0a:00.0: read 1 bytes at 0x70
[  121.121628] xen-pciback: 0000:0a:00.0: read 1 bytes at 0x70 = 11
[  121.121639] xen-pciback: 0000:0a:00.0: read 1 bytes at 0x71
[  121.121647] xen-pciback: 0000:0a:00.0: read 1 bytes at 0x71 = a0
[  121.121671] xen-pciback: 0000:0a:00.0: read 1 bytes at 0xa0
[  121.121679] xen-pciback: 0000:0a:00.0: read 1 bytes at 0xa0 = 10
[  121.121706] xen-pciback: 0000:0a:00.0: read 2 bytes at 0x6
[  121.121715] xen-pciback: 0000:0a:00.0: read 2 bytes at 0x6 = 10
[  121.121724] xen-pciback: 0000:0a:00.0: read 1 bytes at 0x34
[  121.121735] xen-pciback: 0000:0a:00.0: read 1 bytes at 0x34 = 40
[  121.121753] xen-pciback: 0000:0a:00.0: read 1 bytes at 0x40
[  121.121765] xen-pciback: 0000:0a:00.0: read 1 bytes at 0x40 = 5001
[  121.121783] xen-pciback: 0000:0a:00.0: read 1 bytes at 0x41
[  121.121794] xen-pciback: 0000:0a:00.0: read 1 bytes at 0x41 = 50
[  121.121820] xen-pciback: 0000:0a:00.0: read 1 bytes at 0x50
[  121.121829] xen-pciback: 0000:0a:00.0: read 1 bytes at 0x50 = 5
[  121.121845] xen-pciback: 0000:0a:00.0: read 1 bytes at 0x51
[  121.121890] xen-pciback: 0000:0a:00.0: read 1 bytes at 0x51 = 70
[  121.121909] xen-pciback: 0000:0a:00.0: read 1 bytes at 0x70
[  121.121917] xen-pciback: 0000:0a:00.0: read 1 bytes at 0x70 = 11
[  121.121929] xen-pciback: 0000:0a:00.0: read 1 bytes at 0x71
[  121.121937] xen-pciback: 0000:0a:00.0: read 1 bytes at 0x71 = a0
[  121.121962] xen-pciback: 0000:0a:00.0: read 1 bytes at 0xa0
[  121.121971] xen-pciback: 0000:0a:00.0: read 1 bytes at 0xa0 = 10
[  121.121986] xen-pciback: 0000:0a:00.0: read 1 bytes at 0xa1
[  121.121994] xen-pciback: 0000:0a:00.0: read 1 bytes at 0xa1 = 0
[  121.122005] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x100
[  121.122012] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x100 = 14010001
[  121.122037] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x140
[  121.122044] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x140 = 15010003
[  121.122069] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x150
[  121.122076] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x150 = 1601000e
[  121.122100] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x160
[  121.122106] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x160 = 10010
[  121.122130] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x100
[  121.122137] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x100 = 14010001
[  121.122161] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x140
[  121.122168] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x140 = 15010003
[  121.122193] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x150
[  121.122199] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x150 = 1601000e
[  121.122255] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x160
[  121.122262] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x160 = 10010
[  121.122284] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x100
[  121.122290] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x100 = 14010001
[  121.122309] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x140
[  121.122319] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x140 = 15010003
[  121.122343] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x150
[  121.122353] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x150 = 1601000e
[  121.122372] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x160
[  121.122381] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x160 = 10010
[  121.122396] xen-pciback: 0000:0a:00.0: read 2 bytes at 0x6
[  121.122409] xen-pciback: 0000:0a:00.0: read 2 bytes at 0x6 = 10
[  121.122425] xen-pciback: 0000:0a:00.0: read 1 bytes at 0x34
[  121.122437] xen-pciback: 0000:0a:00.0: read 1 bytes at 0x34 = 40
[  121.122450] xen-pciback: 0000:0a:00.0: read 1 bytes at 0x40
[  121.122469] xen-pciback: 0000:0a:00.0: read 1 bytes at 0x40 = 5001
[  121.122495] xen-pciback: 0000:0a:00.0: read 2 bytes at 0x42
[  121.122511] xen-pciback: 0000:0a:00.0: read 2 bytes at 0x42 = 23
[  121.122535] xen-pciback: 0000:0a:00.0: read 2 bytes at 0x6
[  121.122547] xen-pciback: 0000:0a:00.0: read 2 bytes at 0x6 = 10
[  121.122574] xen-pciback: 0000:0a:00.0: read 1 bytes at 0x34
[  121.122586] xen-pciback: 0000:0a:00.0: read 1 bytes at 0x34 = 40
[  121.122619] xen-pciback: 0000:0a:00.0: read 1 bytes at 0x40
[  121.122635] xen-pciback: 0000:0a:00.0: read 1 bytes at 0x40 = 5001
[  121.122655] xen-pciback: 0000:0a:00.0: read 1 bytes at 0x41
[  121.122669] xen-pciback: 0000:0a:00.0: read 1 bytes at 0x41 = 50
[  121.122695] xen-pciback: 0000:0a:00.0: read 1 bytes at 0x50
[  121.122707] xen-pciback: 0000:0a:00.0: read 1 bytes at 0x50 = 5
[  121.122730] xen-pciback: 0000:0a:00.0: read 1 bytes at 0x51
[  121.122742] xen-pciback: 0000:0a:00.0: read 1 bytes at 0x51 = 70
[  121.122765] xen-pciback: 0000:0a:00.0: read 1 bytes at 0x70
[  121.122777] xen-pciback: 0000:0a:00.0: read 1 bytes at 0x70 = 11
[  121.122803] xen-pciback: 0000:0a:00.0: read 1 bytes at 0x71
[  121.122816] xen-pciback: 0000:0a:00.0: read 1 bytes at 0x71 = a0
[  121.122837] xen-pciback: 0000:0a:00.0: read 1 bytes at 0xa0
[  121.122849] xen-pciback: 0000:0a:00.0: read 1 bytes at 0xa0 = 10
[  121.122873] xen-pciback: 0000:0a:00.0: read 1 bytes at 0xa1
[  121.122885] xen-pciback: 0000:0a:00.0: read 1 bytes at 0xa1 = 0
[  121.122909] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x100
[  121.122919] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x100 = 14010001
[  121.122947] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x140
[  121.122956] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x140 = 15010003
[  121.122975] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x150
[  121.122984] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x150 = 1601000e
[  121.123009] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x160
[  121.123018] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x160 = 10010
[  121.123034] xen-pciback: 0000:0a:00.0: read 2 bytes at 0x168
[  121.123043] xen-pciback: 0000:0a:00.0: read 2 bytes at 0x168 = 10
[  121.123066] xen-pciback: 0000:0a:00.0: read 2 bytes at 0x16e
[  121.123076] xen-pciback: 0000:0a:00.0: read 2 bytes at 0x16e = 8
[  121.123097] xen-pciback: 0000:0a:00.0: write request 2 bytes at 0x168 = 0
[  121.123120] xen-pciback: 0000:0a:00.0: write request 2 bytes at 0x170 = 0
[  121.123144] xen-pciback: 0000:0a:00.0: read 2 bytes at 0x174
[  121.123154] xen-pciback: 0000:0a:00.0: read 2 bytes at 0x174 = 80
[  121.123178] xen-pciback: 0000:0a:00.0: read 2 bytes at 0x176
[  121.123187] xen-pciback: 0000:0a:00.0: read 2 bytes at 0x176 = 2
[  121.123204] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x17c
[  121.123217] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x17c = 553
[  121.123288] xen-pciback: 0000:0a:00.0: write request 4 bytes at 0x180 = 1
[  121.123313] xen-pciback: 0000:0a:00.0: read 2 bytes at 0x4
[  121.123329] xen-pciback: 0000:0a:00.0: read 2 bytes at 0x4 = 3
[  121.123354] xen-pciback: 0000:0a:00.0: write request 2 bytes at 0x4 = 0
[  121.123363] xen-pciback: 0000:0a:00.0: read 2 bytes at 0x4
[  121.123378] xen-pciback: 0000:0a:00.0: read 2 bytes at 0x4 = 3
[  121.123386] xen-pciback: 0000:0a:00.0: disable
[  121.124073] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x184
[  121.124084] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x184 = fbda0004
[  121.124103] xen-pciback: 0000:0a:00.0: write request 4 bytes at 0x184 = ffffffff
[  121.124122] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x184
[  121.124131] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x184 = fbda0004
[  121.124154] xen-pciback: 0000:0a:00.0: write request 4 bytes at 0x184 = fbda0004
[  121.124172] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x188
[  121.124181] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x188 = 0
[  121.124206] xen-pciback: 0000:0a:00.0: write request 4 bytes at 0x188 = ffffffff
[  121.124242] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x188
[  121.124251] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x188 = 0
[  121.124269] xen-pciback: 0000:0a:00.0: write request 4 bytes at 0x188 = 0
[  121.124298] xen-pciback: 0000:0a:00.0: write request 2 bytes at 0x4 = 3
[  121.124307] xen-pciback: 0000:0a:00.0: read 2 bytes at 0x4
[  121.124323] xen-pciback: 0000:0a:00.0: read 2 bytes at 0x4 = 0
[  121.124330] xen-pciback: 0000:0a:00.0: enable
[  121.125036] xen: registering gsi 28 triggering 0 polarity 1
[  121.125048] Already setup the GSI :28
[  121.125157] xen-pciback: 0000:0a:00.0: read 2 bytes at 0x4
[  121.125175] xen-pciback: 0000:0a:00.0: read 2 bytes at 0x4 = 3
[  121.125199] xen-pciback: 0000:0a:00.0: write request 2 bytes at 0x4 = 0
[  121.125207] xen-pciback: 0000:0a:00.0: read 2 bytes at 0x4
[  121.125256] xen-pciback: 0000:0a:00.0: read 2 bytes at 0x4 = 3
[  121.125264] xen-pciback: 0000:0a:00.0: disable
[  121.125940] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x188
[  121.125951] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x188 = 0
[  121.125978] xen-pciback: 0000:0a:00.0: write request 4 bytes at 0x188 = ffffffff
[  121.126002] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x188
[  121.126012] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x188 = 0
[  121.126069] xen-pciback: 0000:0a:00.0: write request 4 bytes at 0x188 = 0
[  121.126089] xen-pciback: 0000:0a:00.0: write request 2 bytes at 0x4 = 3
[  121.126097] xen-pciback: 0000:0a:00.0: read 2 bytes at 0x4
[  121.126113] xen-pciback: 0000:0a:00.0: read 2 bytes at 0x4 = 0
[  121.126120] xen-pciback: 0000:0a:00.0: enable
[  121.126798] xen: registering gsi 28 triggering 0 polarity 1
[  121.126809] Already setup the GSI :28
[  121.126897] xen-pciback: 0000:0a:00.0: read 2 bytes at 0x4
[  121.126915] xen-pciback: 0000:0a:00.0: read 2 bytes at 0x4 = 3
[  121.126932] xen-pciback: 0000:0a:00.0: write request 2 bytes at 0x4 = 0
[  121.126940] xen-pciback: 0000:0a:00.0: read 2 bytes at 0x4
[  121.126955] xen-pciback: 0000:0a:00.0: read 2 bytes at 0x4 = 3
[  121.126963] xen-pciback: 0000:0a:00.0: disable
[  121.127698] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x18c
[  121.127710] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x18c = 0
[  121.127729] xen-pciback: 0000:0a:00.0: write request 4 bytes at 0x18c = ffffffff
[  121.127747] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x18c
[  121.127755] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x18c = 0
[  121.127779] xen-pciback: 0000:0a:00.0: write request 4 bytes at 0x18c = 0
[  121.127802] xen-pciback: 0000:0a:00.0: write request 2 bytes at 0x4 = 3
[  121.127810] xen-pciback: 0000:0a:00.0: read 2 bytes at 0x4
[  121.127826] xen-pciback: 0000:0a:00.0: read 2 bytes at 0x4 = 0
[  121.127833] xen-pciback: 0000:0a:00.0: enable
[  121.128525] xen: registering gsi 28 triggering 0 polarity 1
[  121.128537] Already setup the GSI :28
[  121.128624] xen-pciback: 0000:0a:00.0: read 2 bytes at 0x4
[  121.128642] xen-pciback: 0000:0a:00.0: read 2 bytes at 0x4 = 3
[  121.128660] xen-pciback: 0000:0a:00.0: write request 2 bytes at 0x4 = 0
[  121.128668] xen-pciback: 0000:0a:00.0: read 2 bytes at 0x4
[  121.128682] xen-pciback: 0000:0a:00.0: read 2 bytes at 0x4 = 3
[  121.128690] xen-pciback: 0000:0a:00.0: disable
[  121.129432] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x190
[  121.129444] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x190 = fbd80004
[  121.129472] xen-pciback: 0000:0a:00.0: write request 4 bytes at 0x190 = ffffffff
[  121.129500] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x190
[  121.129511] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x190 = fbd80004
[  121.129536] xen-pciback: 0000:0a:00.0: write request 4 bytes at 0x190 = fbd80004
[  121.129557] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x194
[  121.129566] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x194 = 0
[  121.129582] xen-pciback: 0000:0a:00.0: write request 4 bytes at 0x194 = ffffffff
[  121.129619] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x194
[  121.129629] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x194 = 0
[  121.129654] xen-pciback: 0000:0a:00.0: write request 4 bytes at 0x194 = 0
[  121.129671] xen-pciback: 0000:0a:00.0: write request 2 bytes at 0x4 = 3
[  121.129679] xen-pciback: 0000:0a:00.0: read 2 bytes at 0x4
[  121.129695] xen-pciback: 0000:0a:00.0: read 2 bytes at 0x4 = 0
[  121.129703] xen-pciback: 0000:0a:00.0: enable
[  121.130426] xen: registering gsi 28 triggering 0 polarity 1
[  121.130437] Already setup the GSI :28
[  121.130541] xen-pciback: 0000:0a:00.0: read 2 bytes at 0x4
[  121.130559] xen-pciback: 0000:0a:00.0: read 2 bytes at 0x4 = 3
[  121.130583] xen-pciback: 0000:0a:00.0: write request 2 bytes at 0x4 = 0
[  121.130591] xen-pciback: 0000:0a:00.0: read 2 bytes at 0x4
[  121.130605] xen-pciback: 0000:0a:00.0: read 2 bytes at 0x4 = 3
[  121.130612] xen-pciback: 0000:0a:00.0: disable
[  121.131311] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x194
[  121.131323] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x194 = 0
[  121.131339] xen-pciback: 0000:0a:00.0: write request 4 bytes at 0x194 = ffffffff
[  121.131355] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x194
[  121.131365] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x194 = 0
[  121.131380] xen-pciback: 0000:0a:00.0: write request 4 bytes at 0x194 = 0
[  121.131397] xen-pciback: 0000:0a:00.0: write request 2 bytes at 0x4 = 3
[  121.131405] xen-pciback: 0000:0a:00.0: read 2 bytes at 0x4
[  121.131421] xen-pciback: 0000:0a:00.0: read 2 bytes at 0x4 = 0
[  121.131427] xen-pciback: 0000:0a:00.0: enable
[  121.132105] xen: registering gsi 28 triggering 0 polarity 1
[  121.132116] Already setup the GSI :28
[  121.132202] xen-pciback: 0000:0a:00.0: read 2 bytes at 0x4
[  121.132238] xen-pciback: 0000:0a:00.0: read 2 bytes at 0x4 = 3
[  121.132265] xen-pciback: 0000:0a:00.0: write request 2 bytes at 0x4 = 0
[  121.132274] xen-pciback: 0000:0a:00.0: read 2 bytes at 0x4
[  121.132289] xen-pciback: 0000:0a:00.0: read 2 bytes at 0x4 = 3
[  121.132295] xen-pciback: 0000:0a:00.0: disable
[  121.133028] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x198
[  121.133038] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x198 = 0
[  121.133064] xen-pciback: 0000:0a:00.0: write request 4 bytes at 0x198 = ffffffff
[  121.133090] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x198
[  121.133098] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x198 = 0
[  121.133134] xen-pciback: 0000:0a:00.0: write request 4 bytes at 0x198 = 0
[  121.133162] xen-pciback: 0000:0a:00.0: write request 2 bytes at 0x4 = 3
[  121.133170] xen-pciback: 0000:0a:00.0: read 2 bytes at 0x4
[  121.133186] xen-pciback: 0000:0a:00.0: read 2 bytes at 0x4 = 0
[  121.133193] xen-pciback: 0000:0a:00.0: enable
[  121.133847] xen: registering gsi 28 triggering 0 polarity 1
[  121.133859] Already setup the GSI :28
[  121.133968] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x164
[  121.133978] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x164 = 0
[  121.134013] xen-pciback: 0000:0a:00.0: read 1 bytes at 0x172
[  121.134023] xen-pciback: 0000:0a:00.0: read 1 bytes at 0x172 = 0
[  121.134049] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x100
[  121.134058] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x100 = 14010001
[  121.134082] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x140
[  121.134091] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x140 = 15010003
[  121.134114] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x150
[  121.134123] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x150 = 1601000e
[  121.134146] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x160
[  121.134155] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x160 = 10010
[  122.305563] xen-pciback: 0000:0a:00.0: read 4 bytes at 0xa4
[  122.305576] xen-pciback: 0000:0a:00.0: read 4 bytes at 0xa4 = 10008cc2
[  122.775858] xen-pciback: 0000:0a:00.0: read 2 bytes at 0x44
[  122.775879] xen-pciback: 0000:0a:00.0: read 2 bytes at 0x44 = 2000
[  122.775906] xen-pciback: 0000:0a:00.0: read 2 bytes at 0x4
[  122.775918] xen-pciback: 0000:0a:00.0: read 2 bytes at 0x4 = 3
[  122.775949] xen-pciback: 0000:0a:00.0: read 1 bytes at 0x3c
[  122.775959] xen-pciback: 0000:0a:00.0: read 1 bytes at 0x3c = 1c
[  122.776211] xen-pciback: 0000:0a:00.0: read 1 bytes at 0x3d
[  122.776224] xen-pciback: 0000:0a:00.0: read 1 bytes at 0x3d = 1
[  122.776270] xen-pciback: 0000:0a:00.0: read 2 bytes at 0x4
[  122.776282] xen-pciback: 0000:0a:00.0: read 2 bytes at 0x4 = 3
[  122.776344] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x100
[  122.776351] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x100 = 14010001
[  122.776361] xen-pciback: 0000:0a:00.0: read 2 bytes at 0xa8
[  122.776372] xen-pciback: 0000:0a:00.0: read 2 bytes at 0xa8 = 2830
[  122.776389] xen-pciback: 0000:0a:00.0: write request 2 bytes at 0xa8 = 283f
[  122.776408] xen-pciback: 0000:0a:00.0: read 2 bytes at 0x4
[  122.776419] xen-pciback: 0000:0a:00.0: read 2 bytes at 0x4 = 3
[  122.776434] xen-pciback: 0000:0a:00.0: write request 2 bytes at 0x4 = 7
[  122.776440] xen-pciback: 0000:0a:00.0: read 2 bytes at 0x4
[  122.776450] xen-pciback: 0000:0a:00.0: read 2 bytes at 0x4 = 3
[  122.776455] xen-pciback: 0000:0a:00.0: set bus master
[  122.776488] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x0
[  122.776497] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x0 = 10c98086
[  122.776508] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x4
[  122.776519] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x4 = 100007
[  122.776545] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x8
[  122.776554] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x8 = 2000001
[  122.776566] xen-pciback: 0000:0a:00.0: read 4 bytes at 0xc
[  122.776579] xen-pciback: 0000:0a:00.0: read 4 bytes at 0xc = 800040
[  122.776645] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x10
[  122.776654] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x10 = fbe20000
[  122.776670] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x14
[  122.776679] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x14 = fbe00000
[  122.776697] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x18
[  122.776706] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x18 = e881
[  122.776732] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x1c
[  122.776740] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x1c = fbddc000
[  122.776756] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x20
[  122.776764] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x20 = 0
[  122.776790] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x24
[  122.776799] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x24 = 0
[  122.776827] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x28
[  122.776836] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x28 = 0
[  122.776865] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x2c
[  122.776874] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x2c = 10c915d9
[  122.776901] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x30
[  122.776910] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x30 = fbde0000
[  122.776935] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x34
[  122.776944] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x34 = 40
[  122.776958] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x38
[  122.776967] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x38 = 0
[  122.776991] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x3c
[  122.777000] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x3c = 11c
[  122.777012] xen-pciback: 0000:0a:00.0: read 2 bytes at 0xa8
[  122.777020] xen-pciback: 0000:0a:00.0: read 2 bytes at 0xa8 = 2830
[  122.777045] xen-pciback: 0000:0a:00.0: read 2 bytes at 0xb0
[  122.777054] xen-pciback: 0000:0a:00.0: read 2 bytes at 0xb0 = 40
[  122.777080] xen-pciback: 0000:0a:00.0: read 2 bytes at 0xc8
[  122.777088] xen-pciback: 0000:0a:00.0: read 2 bytes at 0xc8 = 0
[  122.777105] xen-pciback: 0000:0a:00.0: read 2 bytes at 0xd0
[  122.777113] xen-pciback: 0000:0a:00.0: read 2 bytes at 0xd0 = 0
[  122.777138] xen-pciback: 0000:0a:00.0: read 2 bytes at 0x6
[  122.777147] xen-pciback: 0000:0a:00.0: read 2 bytes at 0x6 = 10
[  122.777172] xen-pciback: 0000:0a:00.0: read 1 bytes at 0x34
[  122.777181] xen-pciback: 0000:0a:00.0: read 1 bytes at 0x34 = 40
[  122.777200] xen-pciback: 0000:0a:00.0: read 1 bytes at 0x40
[  122.777211] xen-pciback: 0000:0a:00.0: read 1 bytes at 0x40 = 5001
[  122.777228] xen-pciback: 0000:0a:00.0: read 1 bytes at 0x41
[  122.777251] xen-pciback: 0000:0a:00.0: read 1 bytes at 0x41 = 50
[  122.777281] xen-pciback: 0000:0a:00.0: read 1 bytes at 0x50
[  122.777290] xen-pciback: 0000:0a:00.0: read 1 bytes at 0x50 = 5
[  122.777307] xen-pciback: 0000:0a:00.0: read 1 bytes at 0x51
[  122.777315] xen-pciback: 0000:0a:00.0: read 1 bytes at 0x51 = 70
[  122.777340] xen-pciback: 0000:0a:00.0: read 1 bytes at 0x70
[  122.777349] xen-pciback: 0000:0a:00.0: read 1 bytes at 0x70 = 11
[  122.777365] xen-pciback: 0000:0a:00.0: read 1 bytes at 0x71
[  122.777374] xen-pciback: 0000:0a:00.0: read 1 bytes at 0x71 = a0
[  122.777399] xen-pciback: 0000:0a:00.0: read 1 bytes at 0xa0
[  122.777407] xen-pciback: 0000:0a:00.0: read 1 bytes at 0xa0 = 10
[  122.777424] xen-pciback: 0000:0a:00.0: read 1 bytes at 0xa1
[  122.777432] xen-pciback: 0000:0a:00.0: read 1 bytes at 0xa1 = 0
[  122.777457] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x100
[  122.777464] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x100 = 14010001
[  122.777490] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x140
[  122.777496] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x140 = 15010003
[  122.777562] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x150
[  122.777569] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x150 = 1601000e
[  122.777637] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x160
[  122.777644] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x160 = 10010
[  122.777653] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x100
[  122.777662] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x100 = 14010001
[  122.777687] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x140
[  122.777694] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x140 = 15010003
[  122.777719] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x150
[  122.777726] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x150 = 1601000e
[  122.777751] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x160
[  122.777758] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x160 = 10010
[  122.777784] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x100
[  122.777790] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x100 = 14010001
[  122.777815] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x140
[  122.777822] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x140 = 15010003
[  122.777840] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x150
[  122.777847] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x150 = 1601000e
[  122.777872] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x160
[  122.777879] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x160 = 10010
[  122.778258] xen-pciback: 0000:0a:00.0: read 2 bytes at 0x4
[  122.778270] xen-pciback: 0000:0a:00.0: read 2 bytes at 0x4 = 7
[  122.778310] xen-pciback: 0000:0a:00.0: read 2 bytes at 0x72
[  122.778319] xen-pciback: 0000:0a:00.0: read 2 bytes at 0x72 = 9
[  122.778346] xen-pciback: 0000:0a:00.0: read 2 bytes at 0x72
[  122.778355] xen-pciback: 0000:0a:00.0: read 2 bytes at 0x72 = 9
[  122.778371] xen-pciback: 0000:0a:00.0: write request 2 bytes at 0x72 = 9
[  122.778397] xen-pciback: 0000:0a:00.0: read 2 bytes at 0x72
[  122.778406] xen-pciback: 0000:0a:00.0: read 2 bytes at 0x72 = 9
[  122.778431] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x74
[  122.778440] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x74 = 3
[  122.778654] xen-pciback: 0000:0a:00.0: enable MSI-X
[  122.778861] pciback 0000:0a:00.0: xen map irq failed -6 for 1 domain
[  122.778887] xen_pciback: 0000:0a:00.0: error enabling MSI-X for guest 1: err -6!
[  122.779026] xen-pciback: 0000:0a:00.0: disable MSI-X
[  122.779034] xen-pciback: 0000:0a:00.0: MSI-X: 28
[  122.779093] xen-pciback: 0000:0a:00.0: read 2 bytes at 0x52
[  122.779107] xen-pciback: 0000:0a:00.0: read 2 bytes at 0x52 = 180
[  122.779130] xen-pciback: 0000:0a:00.0: read 2 bytes at 0x52
[  122.779142] xen-pciback: 0000:0a:00.0: read 2 bytes at 0x52 = 180
[  122.779168] xen-pciback: 0000:0a:00.0: write request 2 bytes at 0x52 = 180
[  122.779210] xen-pciback: 0000:0a:00.0: read 2 bytes at 0x52
[  122.779223] xen-pciback: 0000:0a:00.0: read 2 bytes at 0x52 = 180
[  122.779245] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x60
[  122.779255] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x60 = 0
[  122.779301] xen-pciback: 0000:0a:00.0: enable MSI
[  122.779492] xen-pciback: 0000:0a:00.0: MSI: 127
[  122.779770] xen-pciback: 0000:0a:00.0: read 2 bytes at 0x4
[  122.779783] xen-pciback: 0000:0a:00.0: read 2 bytes at 0x4 = 7
[  122.779802] xen-pciback: 0000:0a:00.0: write request 2 bytes at 0x4 = 407
[  122.779808] xen-pciback: 0000:0a:00.0: read 2 bytes at 0x4
[  122.779819] xen-pciback: 0000:0a:00.0: read 2 bytes at 0x4 = 7
[  122.779850] xen-pciback: 0000:0a:00.0: read 2 bytes at 0x52
[  122.779860] xen-pciback: 0000:0a:00.0: read 2 bytes at 0x52 = 181
[  122.779871] xen-pciback: 0000:0a:00.0: write request 2 bytes at 0x52 = 181
[  122.779910] xen-pciback: 0000:0a:00.0: read 2 bytes at 0xb2
[  122.779920] xen-pciback: 0000:0a:00.0: read 2 bytes at 0xb2 = 1041
[  122.780059] xen-pciback: 0000:0a:00.0: read 2 bytes at 0xc8
[  122.780068] xen-pciback: 0000:0a:00.0: read 2 bytes at 0xc8 = 0
[  122.780084] xen-pciback: 0000:0a:00.0: write request 2 bytes at 0xc8 = 5
[  122.804329] xen-pciback: 0000:0a:00.0: read 2 bytes at 0xc8
[  122.804340] xen-pciback: 0000:0a:00.0: read 2 bytes at 0xc8 = 0
[  122.804358] xen-pciback: 0000:0a:00.0: write request 2 bytes at 0xc8 = 5
[  128.751418] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x0
[  128.751442] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x0 = 10c98086
[  128.751462] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x4
[  128.751477] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x4 = 100407
[  128.751507] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x8
[  128.751519] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x8 = 2000001
[  128.751588] xen-pciback: 0000:0a:00.0: read 4 bytes at 0xc
[  128.751606] xen-pciback: 0000:0a:00.0: read 4 bytes at 0xc = 800040
[  128.751653] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x10
[  128.751665] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x10 = fbe20000
[  128.751687] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x14
[  128.751698] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x14 = fbe00000
[  128.751719] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x18
[  128.751729] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x18 = e881
[  128.751757] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x1c
[  128.751769] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x1c = fbddc000
[  128.751786] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x20
[  128.751796] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x20 = 0
[  128.751811] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x24
[  128.751822] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x24 = 0
[  128.751836] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x28
[  128.751848] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x28 = 0
[  128.751862] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x2c
[  128.751873] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x2c = 10c915d9
[  128.751885] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x30
[  128.751899] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x30 = fbde0000
[  128.751911] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x34
[  128.751925] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x34 = 40
[  128.751939] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x38
[  128.751951] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x38 = 0
[  128.751973] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x3c
[  128.751984] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x3c = 185
[  151.452407] xen-pciback: 0000:0a:00.0: read 2 bytes at 0xc8
[  151.452439] xen-pciback: 0000:0a:00.0: read 2 bytes at 0xc8 = 0
[  151.452467] xen-pciback: 0000:0a:00.0: write request 2 bytes at 0xc8 = 5
[  151.616169] xen-pciback: 0000:0a:00.0: read 2 bytes at 0x52
[  151.616194] xen-pciback: 0000:0a:00.0: read 2 bytes at 0x52 = 181
[  151.616229] xen-pciback: 0000:0a:00.0: write request 2 bytes at 0x52 = 180
[  151.616262] xen-pciback: 0000:0a:00.0: read 2 bytes at 0x4
[  151.616283] xen-pciback: 0000:0a:00.0: read 2 bytes at 0x4 = 407
[  151.616326] xen-pciback: 0000:0a:00.0: write request 2 bytes at 0x4 = 7
[  151.616334] xen-pciback: 0000:0a:00.0: read 2 bytes at 0x4
[  151.616350] xen-pciback: 0000:0a:00.0: read 2 bytes at 0x4 = 407
[  151.616383] xen-pciback: 0000:0a:00.0: disable MSI
[  151.616484] xen-pciback: 0000:0a:00.0: MSI: 28
[  151.616562] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x0
[  151.616577] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x0 = 10c98086
[  151.616627] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x4
[  151.616643] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x4 = 100007
[  151.616657] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x8
[  151.616673] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x8 = 2000001
[  151.616692] xen-pciback: 0000:0a:00.0: read 4 bytes at 0xc
[  151.616712] xen-pciback: 0000:0a:00.0: read 4 bytes at 0xc = 800040
[  151.616742] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x10
[  151.616755] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x10 = fbe20000
[  151.616773] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x14
[  151.616785] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x14 = fbe00000
[  151.616802] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x18
[  151.616815] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x18 = e881
[  151.616844] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x1c
[  151.616857] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x1c = fbddc000
[  151.616881] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x20
[  151.616894] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x20 = 0
[  151.616918] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x24
[  151.616931] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x24 = 0
[  151.616978] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x28
[  151.616991] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x28 = 0
[  151.617017] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x2c
[  151.617029] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x2c = 10c915d9
[  151.617054] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x30
[  151.617067] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x30 = fbde0000
[  151.617091] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x34
[  151.617103] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x34 = 40
[  151.617127] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x38
[  151.617139] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x38 = 0
[  151.617163] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x3c
[  151.617176] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x3c = 11c
[  151.617199] xen-pciback: 0000:0a:00.0: read 2 bytes at 0xa8
[  151.617212] xen-pciback: 0000:0a:00.0: read 2 bytes at 0xa8 = 2830
[  151.617236] xen-pciback: 0000:0a:00.0: read 2 bytes at 0xb0
[  151.617248] xen-pciback: 0000:0a:00.0: read 2 bytes at 0xb0 = 40
[  151.617272] xen-pciback: 0000:0a:00.0: read 2 bytes at 0xc8
[  151.617287] xen-pciback: 0000:0a:00.0: read 2 bytes at 0xc8 = 0
[  151.617310] xen-pciback: 0000:0a:00.0: read 2 bytes at 0xd0
[  151.617323] xen-pciback: 0000:0a:00.0: read 2 bytes at 0xd0 = 0
[  151.617346] xen-pciback: 0000:0a:00.0: read 2 bytes at 0x6
[  151.617359] xen-pciback: 0000:0a:00.0: read 2 bytes at 0x6 = 10
[  151.617382] xen-pciback: 0000:0a:00.0: read 1 bytes at 0x34
[  151.617394] xen-pciback: 0000:0a:00.0: read 1 bytes at 0x34 = 40
[  151.617420] xen-pciback: 0000:0a:00.0: read 1 bytes at 0x40
[  151.617437] xen-pciback: 0000:0a:00.0: read 1 bytes at 0x40 = 5001
[  151.617466] xen-pciback: 0000:0a:00.0: read 1 bytes at 0x41
[  151.617483] xen-pciback: 0000:0a:00.0: read 1 bytes at 0x41 = 50
[  151.617512] xen-pciback: 0000:0a:00.0: read 1 bytes at 0x50
[  151.617524] xen-pciback: 0000:0a:00.0: read 1 bytes at 0x50 = 5
[  151.617549] xen-pciback: 0000:0a:00.0: read 1 bytes at 0x51
[  151.617562] xen-pciback: 0000:0a:00.0: read 1 bytes at 0x51 = 70
[  151.617585] xen-pciback: 0000:0a:00.0: read 1 bytes at 0x70
[  151.617597] xen-pciback: 0000:0a:00.0: read 1 bytes at 0x70 = 11
[  151.617642] xen-pciback: 0000:0a:00.0: read 1 bytes at 0x71
[  151.617661] xen-pciback: 0000:0a:00.0: read 1 bytes at 0x71 = a0
[  151.617685] xen-pciback: 0000:0a:00.0: read 1 bytes at 0xa0
[  151.617698] xen-pciback: 0000:0a:00.0: read 1 bytes at 0xa0 = 10
[  151.617715] xen-pciback: 0000:0a:00.0: read 1 bytes at 0xa1
[  151.617728] xen-pciback: 0000:0a:00.0: read 1 bytes at 0xa1 = 0
[  151.617744] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x100
[  151.617755] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x100 = 14010001
[  151.617772] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x140
[  151.617781] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x140 = 15010003
[  151.617805] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x150
[  151.617815] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x150 = 1601000e
[  151.617843] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x160
[  151.617853] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x160 = 10010
[  151.617881] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x100
[  151.617892] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x100 = 14010001
[  151.617921] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x140
[  151.617931] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x140 = 15010003
[  151.617973] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x150
[  151.617984] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x150 = 1601000e
[  151.618014] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x160
[  151.618024] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x160 = 10010
[  151.618051] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x100
[  151.618061] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x100 = 14010001
[  151.618087] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x140
[  151.618097] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x140 = 15010003
[  151.618123] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x150
[  151.618133] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x150 = 1601000e
[  151.618160] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x160
[  151.618171] xen-pciback: 0000:0a:00.0: read 4 bytes at 0x160 = 10010
[  151.619618] xen-pciback: 0000:0a:00.0: read 2 bytes at 0x4
[  151.619635] xen-pciback: 0000:0a:00.0: read 2 bytes at 0x4 = 7
[  151.619652] xen-pciback: 0000:0a:00.0: write request 2 bytes at 0x4 = 3
[  151.619661] xen-pciback: 0000:0a:00.0: read 2 bytes at 0x4
[  151.619676] xen-pciback: 0000:0a:00.0: read 2 bytes at 0x4 = 7

[-- Attachment #3: x --]
[-- Type: text/plain, Size: 30640 bytes --]

 Xen 4.6-unstable
(XEN) Xen version 4.6-unstable (konrad@(none)) (gcc (GCC) 4.4.4 20100503 (Red Hat 4.4.4-2)) debug=y Fri Apr 10 16:56:20 EDT 2015
(XEN) Latest ChangeSet: Fri Apr 10 16:55:58 2015 -0400 git:4a63ef1
(XEN) Bootloader: unknown
(XEN) Command line: com1=115200,8n1 dom0_mem=999M,max:1232M dom0_max_vcpus=3 cpufreq=performance,verbose iommu=verbose,no-intremap console=com1,vga loglvl=all guest_loglvl=all
(XEN) Video information:
(XEN)  VGA is text mode 80x25, font 8x16
(XEN)  VBE/DDC methods: none; EDID transfer time: 2 seconds
(XEN)  EDID info not retrieved because no DDC retrieval method detected
(XEN) Disc information:
(XEN)  Found 2 MBR signatures
(XEN)  Found 2 EDD information structures
(XEN) Xen-e820 RAM map:
(XEN)  0000000000000000 - 000000000009e800 (usable)
(XEN)  000000000009e800 - 00000000000a0000 (reserved)
(XEN)  00000000000e6000 - 0000000000100000 (reserved)
(XEN)  0000000000100000 - 00000000bf770000 (usable)
(XEN)  00000000bf770000 - 00000000bf77e000 (ACPI data)
(XEN)  00000000bf77e000 - 00000000bf7d0000 (ACPI NVS)
(XEN)  00000000bf7d0000 - 00000000bf7e0000 (reserved)
(XEN)  00000000bf7ec000 - 00000000c0000000 (reserved)
(XEN)  00000000e0000000 - 00000000f0000000 (reserved)
(XEN)  00000000fee00000 - 00000000fee01000 (reserved)
(XEN)  00000000ffc00000 - 0000000100000000 (reserved)
(XEN)  0000000100000000 - 0000000340000000 (usable)
(XEN) ACPI: RSDP 000FA180, 0024 (r2 ACPIAM)
(XEN) ACPI: XSDT BF770100, 006C (r1 SMCI            20111028 MSFT       97)
(XEN) ACPI: FACP BF770290, 00F4 (r4 102811 FACP1450 20111028 MSFT       97)
(XEN) ACPI: DSDT BF7706C0, 5CBB (r2  1F280 1F280000        0 INTL 20051117)
(XEN) ACPI: FACS BF77E000, 0040
(XEN) ACPI: APIC BF770390, 0136 (r2 102811 APIC1450 20111028 MSFT       97)
(XEN) ACPI: MCFG BF7704D0, 003C (r1 102811 OEMMCFG  20111028 MSFT       97)
(XEN) ACPI: SLIT BF770510, 0030 (r1 102811 OEMSLIT  20111028 MSFT       97)
(XEN) ACPI: OEMB BF77E040, 0092 (r1 102811 OEMB1450 20111028 MSFT       97)
(XEN) ACPI: SRAT BF77A6C0, 01A8 (r2 102811 OEMSRAT         1 INTL        1)
(XEN) ACPI: HPET BF77A870, 0038 (r1 102811 OEMHPET  20111028 MSFT       97)
(XEN) ACPI: DMAR BF77E0E0, 0120 (r1    AMI  OEMDMAR        1 MSFT       97)
(XEN) ACPI: SSDT BF781720, 0363 (r1 DpgPmm    CpuPm       12 INTL 20051117)
(XEN) System RAM: 12279MB (12573752kB)
(XEN) SRAT: PXM 0 -> APIC 0 -> Node 0
(XEN) SRAT: PXM 0 -> APIC 2 -> Node 0
(XEN) SRAT: PXM 0 -> APIC 4 -> Node 0
(XEN) SRAT: PXM 0 -> APIC 6 -> Node 0
(XEN) SRAT: PXM 0 -> APIC 1 -> Node 0
(XEN) SRAT: PXM 0 -> APIC 3 -> Node 0
(XEN) SRAT: PXM 0 -> APIC 5 -> Node 0
(XEN) SRAT: PXM 0 -> APIC 7 -> Node 0
(XEN) SRAT: PXM 1 -> APIC 16 -> Node 1
(XEN) SRAT: PXM 1 -> APIC 18 -> Node 1
(XEN) SRAT: PXM 1 -> APIC 20 -> Node 1
(XEN) SRAT: PXM 1 -> APIC 22 -> Node 1
(XEN) SRAT: PXM 1 -> APIC 17 -> Node 1
(XEN) SRAT: PXM 1 -> APIC 19 -> Node 1
(XEN) SRAT: PXM 1 -> APIC 21 -> Node 1
(XEN) SRAT: PXM 1 -> APIC 23 -> Node 1
(XEN) SRAT: Node 1 PXM 1 0-a0000
(XEN) SRAT: Node 1 PXM 1 100000-c0000000
(XEN) SRAT: Node 1 PXM 1 100000000-340000000
(XEN) NUMA: Allocated memnodemap from 338f49000 - 338f4d000
(XEN) NUMA: Using 8 for the hash shift.
(XEN) SRAT: Node 0 has no memory. BIOS Bug or mis-configured hardware?
(XEN) Domain heap initialised DMA width 9 bits
(XEN) found SMP MP-table at 000ff780
(XEN) DMI present.
(XEN) Using APIC driver default
(XEN) ACPI: PM-Timer IO Port: 0x808
(XEN) ACPI: SLEEP INFO: pm1x_cnt[1:804,1:0], pm1x_evt[1:800,1:0]
(XEN) ACPI:             wakeup_vec[bf77e00c], vec_size[20]
(XEN) ACPI: Local APIC address 0xfee00000
(XEN) ACPI: LAPIC (acpi_id[0x01] lapic_id[0x00] enabled)
(XEN) Processor #0 7:10 APIC version 21
(XEN) ACPI: LAPIC (acpi_id[0x02] lapic_id[0x02] enabled)
(XEN) Processor #2 7:10 APIC version 21
(XEN) ACPI: LAPIC (acpi_id[0x03] lapic_id[0x04] enabled)
(XEN) Processor #4 7:10 APIC version 21
(XEN) ACPI: LAPIC (acpi_id[0x04] lapic_id[0x06] enabled)
(XEN) Processor #6 7:10 APIC version 21
(XEN) ACPI: LAPIC (acpi_id[0x05] lapic_id[0x10] enabled)
(XEN) Processor #16 7:10 APIC version 21
(XEN) ACPI: LAPIC (acpi_id[0x06] lapic_id[0x12] enabled)
(XEN) Processor #18 7:10 APIC version 21
(XEN) ACPI: LAPIC (acpi_id[0x07] lapic_id[0x14] enabled)
(XEN) Processor #20 7:10 APIC version 21
(XEN) ACPI: LAPIC (acpi_id[0x08] lapic_id[0x16] enabled)
(XEN) Processor #22 7:10 APIC version 21
(XEN) ACPI: LAPIC (acpi_id[0x09] lapic_id[0x01] enabled)
(XEN) Processor #1 7:10 APIC version 21
(XEN) ACPI: LAPIC (acpi_id[0x0a] lapic_id[0x03] enabled)
(XEN) Processor #3 7:10 APIC version 21
(XEN) ACPI: LAPIC (acpi_id[0x0b] lapic_id[0x05] enabled)
(XEN) Processor #5 7:10 APIC version 21
(XEN) ACPI: LAPIC (acpi_id[0x0c] lapic_id[0x07] enabled)
(XEN) Processor #7 7:10 APIC version 21
(XEN) ACPI: LAPIC (acpi_id[0x0d] lapic_id[0x11] enabled)
(XEN) Processor #17 7:10 APIC version 21
(XEN) ACPI: LAPIC (acpi_id[0x0e] lapic_id[0x13] enabled)
(XEN) Processor #19 7:10 APIC version 21
(XEN) ACPI: LAPIC (acpi_id[0x0f] lapic_id[0x15] enabled)
(XEN) Processor #21 7:10 APIC version 21
(XEN) ACPI: LAPIC (acpi_id[0x10] lapic_id[0x17] enabled)
(XEN) Processor #23 7:10 APIC version 21
(XEN) ACPI: LAPIC (acpi_id[0x11] lapic_id[0x90] disabled)
(XEN) ACPI: LAPIC (acpi_id[0x12] lapic_id[0x91] disabled)
(XEN) ACPI: LAPIC (acpi_id[0x13] lapic_id[0x92] disabled)
(XEN) ACPI: LAPIC (acpi_id[0x14] lapic_id[0x93] disabled)
(XEN) ACPI: LAPIC (acpi_id[0x15] lapic_id[0x94] disabled)
(XEN) ACPI: LAPIC (acpi_id[0x16] lapic_id[0x95] disabled)
(XEN) ACPI: LAPIC (acpi_id[0x17] lapic_id[0x96] disabled)
(XEN) ACPI: LAPIC (acpi_id[0x18] lapic_id[0x97] disabled)
(XEN) ACPI: LAPIC_NMI (acpi_id[0xff] dfl dfl lint[0x1])
(XEN) Overriding APIC driver with bigsmp
(XEN) ACPI: IOAPIC (id[0x08] address[0xfec00000] gsi_base[0])
(XEN) IOAPIC[0]: apic_id 8, version 32, address 0xfec00000, GSI 0-23
(XEN) ACPI: IOAPIC (id[0x09] address[0xfec8a000] gsi_base[24])
(XEN) IOAPIC[1]: apic_id 9, version 32, address 0xfec8a000, GSI 24-47
(XEN) ACPI: IOAPIC (id[0x0a] address[0xfec80000] gsi_base[48])
(XEN) IOAPIC[2]: apic_id 10, version 32, address 0xfec80000, GSI 48-71
(XEN) ACPI: IOAPIC (id[0x0b] address[0xfec80400] gsi_base[72])
(XEN) IOAPIC[3]: apic_id 11, version 32, address 0xfec80400, GSI 72-95
(XEN) ACPI: INT_SRC_OVR (bus 0 bus_irq 0 global_irq 2 dfl dfl)
(XEN) ACPI: INT_SRC_OVR (bus 0 bus_irq 9 global_irq 20 low level)
(XEN) ACPI: IRQ0 used by override.
(XEN) ACPI: IRQ2 used by override.
(XEN) ACPI: IRQ9 used by override.
(XEN) Enabling APIC mode:  Phys.  Using 4 I/O APICs
(XEN) ACPI: HPET id: 0x8086a301 base: 0xfed00000
(XEN) [VT-D]dmar.c:788: Host address width 40
(XEN) [VT-D]dmar.c:802: found ACPI_DMAR_DRHD:
(XEN) [VT-D]dmar.c:472:   dmaru->address = fbffe000
(XEN) [VT-D]iommu.c:1151: drhd->address = fbffe000 iommu->reg = ffff82c000201000
(XEN) [VT-D]iommu.c:1153: cap = c90780106f0462 ecap = f020f6
(XEN) [VT-D]dmar.c:486:   flags: INCLUDE_ALL
(XEN) [VT-D]dmar.c:807: found ACPI_DMAR_RMRR:
(XEN) [VT-D]dmar.c:383:  endpoint: 0000:00:1d.0
(XEN) [VT-D]dmar.c:383:  endpoint: 0000:00:1d.1
(XEN) [VT-D]dmar.c:383:  endpoint: 0000:00:1d.2
(XEN) [VT-D]dmar.c:383:  endpoint: 0000:00:1d.7
(XEN) [VT-D]dmar.c:383:  endpoint: 0000:00:1a.0
(XEN) [VT-D]dmar.c:383:  endpoint: 0000:00:1a.1
(XEN) [VT-D]dmar.c:383:  endpoint: 0000:00:1a.2
(XEN) [VT-D]dmar.c:383:  endpoint: 0000:00:1a.7
(XEN) [VT-D]dmar.c:676:   RMRR region: base_addr e6000 end_address e9fff
(XEN) [VT-D]dmar.c:807: found ACPI_DMAR_RMRR:
(XEN) [VT-D]dmar.c:383:  endpoint: 0000:00:1d.0
(XEN) [VT-D]dmar.c:383:  endpoint: 0000:00:1d.1
(XEN) [VT-D]dmar.c:383:  endpoint: 0000:00:1d.2
(XEN) [VT-D]dmar.c:383:  endpoint: 0000:00:1d.7
(XEN) [VT-D]dmar.c:383:  endpoint: 0000:00:1a.0
(XEN) [VT-D]dmar.c:383:  endpoint: 0000:00:1a.1
(XEN) [VT-D]dmar.c:383:  endpoint: 0000:00:1a.2
(XEN) [VT-D]dmar.c:383:  endpoint: 0000:00:1a.7
(XEN) [VT-D]dmar.c:676:   RMRR region: base_addr bf7ec000 end_address bf7fffff
(XEN) [VT-D]dmar.c:812: found ACPI_DMAR_ATSR:
(XEN) [VT-D]dmar.c:705:   atsru->all_ports: 0
(XEN) [VT-D]dmar.c:353:  bridge: 0000:00:01.0 start=0 sec=a sub=b
(XEN) [VT-D]dmar.c:353:  bridge: 0000:00:03.0 start=0 sec=9 sub=9
(XEN) [VT-D]dmar.c:353:  bridge: 0000:00:05.0 start=0 sec=8 sub=8
(XEN) [VT-D]dmar.c:353:  bridge: 0000:00:07.0 start=0 sec=7 sub=7
(XEN) [VT-D]dmar.c:353:  bridge: 0000:00:09.0 start=0 sec=4 sub=6
(XEN) ERST table was not found
(XEN) Using ACPI (MADT) for SMP configuration information
(XEN) SMP: Allowing 24 CPUs (8 hotplug CPUs)
(XEN) IRQ limits: 96 GSI, 2992 MSI/MSI-X
(XEN) Using scheduler: SMP Credit Scheduler (credit)
(XEN) Detected 2266.803 MHz processor.
(XEN) Initing memory sharing.
(XEN) mce_intel.c:735: MCA Capability: BCAST 1 SER 0 CMCI 1 firstbank 0 extended MCE MSR 0
(XEN) Intel machine check reporting enabled
(XEN) alt table ffff82d0802d9e30 -> ffff82d0802db03c
(XEN) PCI: MCFG configuration 0: base e0000000 segment 0000 buses 00 - ff
(XEN) PCI: MCFG area at e0000000 reserved in E820
(XEN) PCI: Using MCFG for segment 0000 bus 00-ff
(XEN) Intel VT-d iommu 0 supported page sizes: 4kB.
(XEN) Intel VT-d Snoop Control enabled.
(XEN) Intel VT-d Dom0 DMA Passthrough not enabled.
(XEN) Intel VT-d Queued Invalidation enabled.
(XEN) Intel VT-d Interrupt Remapping not enabled.
(XEN) Intel VT-d Shared EPT tables not enabled.
(XEN) I/O virtualisation enabled
(XEN)  - Dom0 mode: Relaxed
(XEN) Interrupt remapping disabled
(XEN) ENABLING IO-APIC IRQs
(XEN)  -> Using new ACK method
(XEN) ..TIMER: vector=0xF0 apic1=0 pin1=2 apic2=-1 pin2=-1
(XEN) Platform timer is 14.318MHz HPET
(XEN) Allocated console ring of 128 KiB.
(XEN) mwait-idle: MWAIT substates: 0x1120
(XEN) mwait-idle: v0.4 model 0x1a
(XEN) mwait-idle: lapic_timer_reliable_states 0x2
(XEN) HPET: 0 timers usable for broadcast (4 total)
(XEN) VMX: Supported advanced features:
(XEN)  - APIC MMIO access virtualisation
(XEN)  - APIC TPR shadow
(XEN)  - Extended Page Tables (EPT)
(XEN)  - Virtual-Processor Identifiers (VPID)
(XEN)  - Virtual NMI
(XEN)  - MSR direct-access bitmap
(XEN) HVM: ASIDs enabled.
(XEN) HVM: VMX enabled
(XEN) HVM: Hardware Assisted Paging (HAP) detected
(XEN) HVM: HAP page sizes: 4kB, 2MB
(XEN) Brought up 16 CPUs
(XEN) ACPI sleep modes: S3
(XEN) mcheck_poll: Machine check polling timer started.
(XEN) Dom0 has maximum 672 PIRQs
(XEN) Multiple initrd candidates, picking module #1
(XEN) *** LOADING DOMAIN 0 ***
(XEN) elf_parse_binary: phdr: paddr=0x1000000 memsz=0xa5a000
(XEN) elf_parse_binary: phdr: paddr=0x1c00000 memsz=0xd3000
(XEN) elf_parse_binary: phdr: paddr=0x1cd3000 memsz=0x15118
(XEN) elf_parse_binary: phdr: paddr=0x1ce9000 memsz=0x348000
(XEN) elf_parse_binary: memory: 0x1000000 -> 0x2031000
(XEN) elf_xen_parse_note: GUEST_OS = "linux"
(XEN) elf_xen_parse_note: GUEST_VERSION = "2.6"
(XEN) elf_xen_parse_note: XEN_VERSION = "xen-3.0"
(XEN) elf_xen_parse_note: VIRT_BASE = 0xffffffff80000000
(XEN) elf_xen_parse_note: ENTRY = 0xffffffff81ce91f0
(XEN) elf_xen_parse_note: HYPERCALL_PAGE = 0xffffffff81001000
(XEN) elf_xen_parse_note: FEATURES = "!writable_page_tables|pae_pgdir_above_4gb|writable_descriptor_tables|auto_translated_physmap|supervisor_mode_kernel"
(XEN) elf_xen_parse_note: SUPPORTED_FEATURES = 0x90d
(XEN) elf_xen_parse_note: PAE_MODE = "yes"
(XEN) elf_xen_parse_note: LOADER = "generic"
(XEN) elf_xen_parse_note: unknown xen elf note (0xd)
(XEN) elf_xen_parse_note: SUSPEND_CANCEL = 0x1
(XEN) elf_xen_parse_note: MOD_START_PFN = 0x1
(XEN) elf_xen_parse_note: HV_START_LOW = 0xffff800000000000
(XEN) elf_xen_parse_note: PADDR_OFFSET = 0x0
(XEN) elf_xen_addr_calc_check: addresses:
(XEN)     virt_base        = 0xffffffff80000000
(XEN)     elf_paddr_offset = 0x0
(XEN)     virt_offset      = 0xffffffff80000000
(XEN)     virt_kstart      = 0xffffffff81000000
(XEN)     virt_kend        = 0xffffffff82031000
(XEN)     virt_entry       = 0xffffffff81ce91f0
(XEN)     p2m_base         = 0xffffffffffffffff
(XEN)  Xen  kernel: 64-bit, lsb, compat32
(XEN)  Dom0 kernel: 64-bit, PAE, lsb, paddr 0x1000000 -> 0x2031000
(XEN) PHYSICAL MEMORY ARRANGEMENT:
(XEN)  Dom0 alloc.:   000000032c000000->0000000330000000 (216774 pages to be allocated)
(XEN)  Init. ramdisk: 000000033a74d000->000000033ff8699a
(XEN) VIRTUAL MEMORY ARRANGEMENT:
(XEN)  Loaded kernel: ffffffff81000000->ffffffff82031000
(XEN)  Init. ramdisk: 0000000000000000->0000000000000000
(XEN)  Phys-Mach map: ffffffff82031000->ffffffff82224800
(XEN)  Start info:    ffffffff82225000->ffffffff822254b4
(XEN)  Page tables:   ffffffff82226000->ffffffff8223b000
(XEN)  Boot stack:    ffffffff8223b000->ffffffff8223c000
(XEN)  TOTAL:         ffffffff80000000->ffffffff82400000
(XEN)  ENTRY ADDRESS: ffffffff81ce91f0
(XEN) Dom0 has maximum 3 VCPUs
(XEN) elf_load_binary: phdr 0 at 0xffffffff81000000 -> 0xffffffff81a5a000
(XEN) elf_load_binary: phdr 1 at 0xffffffff81c00000 -> 0xffffffff81cd3000
(XEN) elf_load_binary: phdr 2 at 0xffffffff81cd3000 -> 0xffffffff81ce8118
(XEN) elf_load_binary: phdr 3 at 0xffffffff81ce9000 -> 0xffffffff81ece000
(XEN) [VT-D]iommu.c:1435: d0:Hostbridge: skip 0000:00:00.0 map
(XEN) Found masked UR signaling on 0000:00:00.0
(XEN) Found masked UR signaling on 0000:00:01.0
(XEN) Found masked UR signaling on 0000:00:03.0
(XEN) Found masked UR signaling on 0000:00:05.0
(XEN) Found masked UR signaling on 0000:00:07.0
(XEN) Found masked UR signaling on 0000:00:09.0
(XEN) [VT-D]iommu.c:1461: d0:PCI: map 0000:00:13.0
(XEN) [VT-D]iommu.c:1449: d0:PCIe: map 0000:00:14.0
(XEN) Masked VT-d error signaling on 0000:00:14.0
(XEN) [VT-D]iommu.c:1449: d0:PCIe: map 0000:00:14.1
(XEN) [VT-D]iommu.c:1449: d0:PCIe: map 0000:00:14.2
(XEN) [VT-D]iommu.c:1461: d0:PCI: map 0000:00:14.3
(XEN) [VT-D]iommu.c:1449: d0:PCIe: map 0000:00:16.0
(XEN) [VT-D]iommu.c:1449: d0:PCIe: map 0000:00:16.1
(XEN) [VT-D]iommu.c:1449: d0:PCIe: map 0000:00:16.2
(XEN) [VT-D]iommu.c:1449: d0:PCIe: map 0000:00:16.3
(XEN) [VT-D]iommu.c:1449: d0:PCIe: map 0000:00:16.4
(XEN) [VT-D]iommu.c:1449: d0:PCIe: map 0000:00:16.5
(XEN) [VT-D]iommu.c:1449: d0:PCIe: map 0000:00:16.6
(XEN) [VT-D]iommu.c:1449: d0:PCIe: map 0000:00:16.7
(XEN) [VT-D]iommu.c:1461: d0:PCI: map 0000:00:1a.0
(XEN) [VT-D]iommu.c:1461: d0:PCI: map 0000:00:1a.1
(XEN) [VT-D]iommu.c:1461: d0:PCI: map 0000:00:1a.2
(XEN) [VT-D]iommu.c:1461: d0:PCI: map 0000:00:1a.7
(XEN) [VT-D]iommu.c:1461: d0:PCI: map 0000:00:1d.0
(XEN) [VT-D]iommu.c:1461: d0:PCI: map 0000:00:1d.1
(XEN) [VT-D]iommu.c:1461: d0:PCI: map 0000:00:1d.2
(XEN) [VT-D]iommu.c:1461: d0:PCI: map 0000:00:1d.7
(XEN) [VT-D]iommu.c:1461: d0:PCI: map 0000:00:1f.0
(XEN) [VT-D]iommu.c:1461: d0:PCI: map 0000:00:1f.2
(XEN) [VT-D]iommu.c:1461: d0:PCI: map 0000:00:1f.3
(XEN) [VT-D]iommu.c:1461: d0:PCI: map 0000:00:1f.5
(XEN) [VT-D]iommu.c:1461: d0:PCI: map 0000:01:01.0
(XEN) [VT-D]iommu.c:1449: d0:PCIe: map 0000:02:00.0
(XEN) [VT-D]iommu.c:1449: d0:PCIe: map 0000:04:00.1
(XEN) [VT-D]iommu.c:1449: d0:PCIe: map 0000:04:00.3
(XEN) [VT-D]iommu.c:1461: d0:PCI: map 0000:06:01.0
(XEN) [VT-D]iommu.c:1461: d0:PCI: map 0000:06:01.1
(XEN) [VT-D]iommu.c:1449: d0:PCIe: map 0000:0a:00.0
(XEN) [VT-D]iommu.c:1449: d0:PCIe: map 0000:0a:00.1
(XEN) [VT-D]iommu.c:1435: d0:Hostbridge: skip 0000:fe:00.0 map
(XEN) [VT-D]iommu.c:1435: d0:Hostbridge: skip 0000:fe:00.1 map
(XEN) [VT-D]iommu.c:1435: d0:Hostbridge: skip 0000:fe:02.0 map
(XEN) [VT-D]iommu.c:1435: d0:Hostbridge: skip 0000:fe:02.1 map
(XEN) [VT-D]iommu.c:1435: d0:Hostbridge: skip 0000:fe:02.4 map
(XEN) [VT-D]iommu.c:1435: d0:Hostbridge: skip 0000:fe:02.5 map
(XEN) [VT-D]iommu.c:1435: d0:Hostbridge: skip 0000:fe:03.0 map
(XEN) [VT-D]iommu.c:1435: d0:Hostbridge: skip 0000:fe:03.1 map
(XEN) [VT-D]iommu.c:1435: d0:Hostbridge: skip 0000:fe:03.2 map
(XEN) [VT-D]iommu.c:1435: d0:Hostbridge: skip 0000:fe:03.4 map
(XEN) [VT-D]iommu.c:1435: d0:Hostbridge: skip 0000:fe:04.0 map
(XEN) [VT-D]iommu.c:1435: d0:Hostbridge: skip 0000:fe:04.1 map
(XEN) [VT-D]iommu.c:1435: d0:Hostbridge: skip 0000:fe:04.2 map
(XEN) [VT-D]iommu.c:1435: d0:Hostbridge: skip 0000:fe:04.3 map
(XEN) [VT-D]iommu.c:1435: d0:Hostbridge: skip 0000:fe:05.0 map
(XEN) [VT-D]iommu.c:1435: d0:Hostbridge: skip 0000:fe:05.1 map
(XEN) [VT-D]iommu.c:1435: d0:Hostbridge: skip 0000:fe:05.2 map
(XEN) [VT-D]iommu.c:1435: d0:Hostbridge: skip 0000:fe:05.3 map
(XEN) [VT-D]iommu.c:1435: d0:Hostbridge: skip 0000:fe:06.0 map
(XEN) [VT-D]iommu.c:1435: d0:Hostbridge: skip 0000:fe:06.1 map
(XEN) [VT-D]iommu.c:1435: d0:Hostbridge: skip 0000:fe:06.2 map
(XEN) [VT-D]iommu.c:1435: d0:Hostbridge: skip 0000:fe:06.3 map
(XEN) [VT-D]iommu.c:1435: d0:Hostbridge: skip 0000:ff:00.0 map
(XEN) [VT-D]iommu.c:1435: d0:Hostbridge: skip 0000:ff:00.1 map
(XEN) [VT-D]iommu.c:1435: d0:Hostbridge: skip 0000:ff:02.0 map
(XEN) [VT-D]iommu.c:1435: d0:Hostbridge: skip 0000:ff:02.1 map
(XEN) [VT-D]iommu.c:1435: d0:Hostbridge: skip 0000:ff:02.4 map
(XEN) [VT-D]iommu.c:1435: d0:Hostbridge: skip 0000:ff:02.5 map
(XEN) [VT-D]iommu.c:1435: d0:Hostbridge: skip 0000:ff:03.0 map
(XEN) [VT-D]iommu.c:1435: d0:Hostbridge: skip 0000:ff:03.1 map
(XEN) [VT-D]iommu.c:1435: d0:Hostbridge: skip 0000:ff:03.2 map
(XEN) [VT-D]iommu.c:1435: d0:Hostbridge: skip 0000:ff:03.4 map
(XEN) [VT-D]iommu.c:1435: d0:Hostbridge: skip 0000:ff:04.0 map
(XEN) [VT-D]iommu.c:1435: d0:Hostbridge: skip 0000:ff:04.1 map
(XEN) [VT-D]iommu.c:1435: d0:Hostbridge: skip 0000:ff:04.2 map
(XEN) [VT-D]iommu.c:1435: d0:Hostbridge: skip 0000:ff:04.3 map
(XEN) [VT-D]iommu.c:1435: d0:Hostbridge: skip 0000:ff:05.0 map
(XEN) [VT-D]iommu.c:1435: d0:Hostbridge: skip 0000:ff:05.1 map
(XEN) [VT-D]iommu.c:1435: d0:Hostbridge: skip 0000:ff:05.2 map
(XEN) [VT-D]iommu.c:1435: d0:Hostbridge: skip 0000:ff:05.3 map
(XEN) [VT-D]iommu.c:1435: d0:Hostbridge: skip 0000:ff:06.0 map
(XEN) [VT-D]iommu.c:1435: d0:Hostbridge: skip 0000:ff:06.1 map
(XEN) [VT-D]iommu.c:1435: d0:Hostbridge: skip 0000:ff:06.2 map
(XEN) [VT-D]iommu.c:1435: d0:Hostbridge: skip 0000:ff:06.3 map
(XEN) [VT-D]iommu.c:730: iommu_enable_translation: iommu->reg = ffff82c000201000
(XEN) Scrubbing Free RAM on 2 nodes using 4 CPUs
(XEN) ..........................done.
(XEN) Initial low memory virq threshold set at 0x4000 pages.
(XEN) Std. Loglevel: All
(XEN) Guest Loglevel: All
(XEN) Xen is relinquishing VGA console.
(XEN) *** Serial input -> DOM0 (type 'CTRL-a' three times to switch input to Xen)
(XEN) Freed 292kB init memory.
(XEN) traps.c:2654:d0v0 Domain attempted WRMSR 00000000c0000081 from 0xe023e00800000000 to 0x0023001000000000.
(XEN) traps.c:2654:d0v0 Domain attempted WRMSR 00000000c0000082 from 0xffff82d0802e3000 to 0xffffffff8175c690.
(XEN) traps.c:2654:d0v0 Domain attempted WRMSR 00000000c0000083 from 0xffff82d0802e3080 to 0xffffffff8175e9c0.
(XEN) traps.c:2654:d0v0 Domain attempted WRMSR 0000000000000174 from 0x000000000000e008 to 0x0000000000000010.
(XEN) traps.c:2654:d0v0 Domain attempted WRMSR 0000000000000175 from 0xffff82d0802e7fc0 to 0x0000000000000000.
(XEN) traps.c:2654:d0v0 Domain attempted WRMSR 0000000000000176 from 0xffff82d080230920 to 0xffffffff8175eca0.
(XEN) traps.c:2654:d0v0 Domain attempted WRMSR 00000000c0000083 from 0xffff82d0802e3080 to 0xffffffff8175eee0.
(XEN) traps.c:2654:d0v0 Domain attempted WRMSR 00000000c0000084 from 0x0000000000074700 to 0x0000000000047700.
(XEN) traps.c:2654:d0v1 Domain attempted WRMSR 00000000c0000081 from 0xe023e00800000000 to 0x0023001000000000.
(XEN) traps.c:2654:d0v1 Domain attempted WRMSR 00000000c0000082 from 0xffff830338edb000 to 0xffffffff8175c690.
(XEN) traps.c:2654:d0v1 Domain attempted WRMSR 00000000c0000083 from 0xffff830338edb080 to 0xffffffff8175e9c0.
(XEN) traps.c:2654:d0v1 Domain attempted WRMSR 0000000000000174 from 0x000000000000e008 to 0x0000000000000010.
(XEN) traps.c:2654:d0v1 Domain attempted WRMSR 0000000000000175 from 0xffff830338edffc0 to 0x0000000000000000.
(XEN) traps.c:2654:d0v1 Domain attempted WRMSR 0000000000000176 from 0xffff82d080230920 to 0xffffffff8175eca0.
(XEN) traps.c:2654:d0v1 Domain attempted WRMSR 00000000c0000083 from 0xffff830338edb080 to 0xffffffff8175eee0.
(XEN) traps.c:2654:d0v1 Domain attempted WRMSR 00000000c0000084 from 0x0000000000074700 to 0x0000000000047700.
(XEN) traps.c:2654:d0v2 Domain attempted WRMSR 00000000c0000081 from 0xe023e00800000000 to 0x0023001000000000.
(XEN) traps.c:2654:d0v2 Domain attempted WRMSR 00000000c0000082 from 0xffff830338ed3000 to 0xffffffff8175c690.
(XEN) traps.c:2654:d0v2 Domain attempted WRMSR 00000000c0000083 from 0xffff830338ed3080 to 0xffffffff8175e9c0.
(XEN) traps.c:2654:d0v2 Domain attempted WRMSR 0000000000000174 from 0x000000000000e008 to 0x0000000000000010.
(XEN) traps.c:2654:d0v2 Domain attempted WRMSR 0000000000000175 from 0xffff830338ed7fc0 to 0x0000000000000000.
(XEN) traps.c:2654:d0v2 Domain attempted WRMSR 0000000000000176 from 0xffff82d080230920 to 0xffffffff8175eca0.
(XEN) traps.c:2654:d0v2 Domain attempted WRMSR 00000000c0000083 from 0xffff830338ed3080 to 0xffffffff8175eee0.
(XEN) traps.c:2654:d0v2 Domain attempted WRMSR 00000000c0000084 from 0x0000000000074700 to 0x0000000000047700.
(XEN) Found masked UR signaling on 0000:00:00.0
(XEN) PCI add device 0000:00:00.0
(XEN) Found masked UR signaling on 0000:00:01.0
(XEN) PCI add device 0000:00:01.0
(XEN) Found masked UR signaling on 0000:00:03.0
(XEN) PCI add device 0000:00:03.0
(XEN) Found masked UR signaling on 0000:00:05.0
(XEN) PCI add device 0000:00:05.0
(XEN) Found masked UR signaling on 0000:00:07.0
(XEN) PCI add device 0000:00:07.0
(XEN) Found masked UR signaling on 0000:00:09.0
(XEN) PCI add device 0000:00:09.0
(XEN) PCI add device 0000:00:13.0
(XEN) Masked VT-d error signaling on 0000:00:14.0
(XEN) PCI add device 0000:00:14.0
(XEN) PCI add device 0000:00:14.1
(XEN) PCI add device 0000:00:14.2
(XEN) PCI add device 0000:00:14.3
(XEN) PCI add device 0000:00:16.0
(XEN) PCI add device 0000:00:16.1
(XEN) PCI add device 0000:00:16.2
(XEN) PCI add device 0000:00:16.3
(XEN) PCI add device 0000:00:16.4
(XEN) PCI add device 0000:00:16.5
(XEN) PCI add device 0000:00:16.6
(XEN) PCI add device 0000:00:16.7
(XEN) PCI add device 0000:00:1a.0
(XEN) PCI add device 0000:00:1a.1
(XEN) PCI add device 0000:00:1a.2
(XEN) PCI add device 0000:00:1a.7
(XEN) PCI add device 0000:00:1c.0
(XEN) PCI add device 0000:00:1c.4
(XEN) PCI add device 0000:00:1d.0
(XEN) PCI add device 0000:00:1d.1
(XEN) PCI add device 0000:00:1d.2
(XEN) PCI add device 0000:00:1d.7
(XEN) PCI add device 0000:00:1e.0
(XEN) PCI add device 0000:00:1f.0
(XEN) PCI add device 0000:00:1f.2
(XEN) PCI add device 0000:00:1f.3
(XEN) PCI add device 0000:00:1f.5
(XEN) PCI add device 0000:0a:00.0
(XEN) PCI add device 0000:0a:00.1
(XEN) PCI add device 0000:04:00.0
(XEN) PCI add device 0000:04:00.1
(XEN) PCI add device 0000:04:00.2
(XEN) PCI add device 0000:04:00.3
(XEN) PCI add device 0000:06:01.0
(XEN) PCI add device 0000:06:01.1
(XEN) PCI add device 0000:02:00.0
(XEN) PCI add device 0000:01:01.0
(XEN) PCI add device 0000:fe:00.0
(XEN) PCI add device 0000:fe:00.1
(XEN) PCI add device 0000:fe:02.0
(XEN) PCI add device 0000:fe:02.1
(XEN) PCI add device 0000:fe:02.4
(XEN) PCI add device 0000:fe:02.5
(XEN) PCI add device 0000:fe:03.0
(XEN) PCI add device 0000:fe:03.1
(XEN) PCI add device 0000:fe:03.2
(XEN) PCI add device 0000:fe:03.4
(XEN) PCI add device 0000:fe:04.0
(XEN) PCI add device 0000:fe:04.1
(XEN) PCI add device 0000:fe:04.2
(XEN) PCI add device 0000:fe:04.3
(XEN) PCI add device 0000:fe:05.0
(XEN) PCI add device 0000:fe:05.1
(XEN) PCI add device 0000:fe:05.2
(XEN) PCI add device 0000:fe:05.3
(XEN) PCI add device 0000:fe:06.0
(XEN) PCI add device 0000:fe:06.1
(XEN) PCI add device 0000:fe:06.2
(XEN) PCI add device 0000:fe:06.3
(XEN) PCI add device 0000:ff:00.0
(XEN) PCI add device 0000:ff:00.1
(XEN) PCI add device 0000:ff:02.0
(XEN) PCI add device 0000:ff:02.1
(XEN) PCI add device 0000:ff:02.4
(XEN) PCI add device 0000:ff:02.5
(XEN) PCI add device 0000:ff:03.0
(XEN) PCI add device 0000:ff:03.1
(XEN) PCI add device 0000:ff:03.2
(XEN) PCI add device 0000:ff:03.4
(XEN) PCI add device 0000:ff:04.0
(XEN) PCI add device 0000:ff:04.1
(XEN) PCI add device 0000:ff:04.2
(XEN) PCI add device 0000:ff:04.3
(XEN) PCI add device 0000:ff:05.0
(XEN) PCI add device 0000:ff:05.1
(XEN) PCI add device 0000:ff:05.2
(XEN) PCI add device 0000:ff:05.3
(XEN) PCI add device 0000:ff:06.0
(XEN) PCI add device 0000:ff:06.1
(XEN) PCI add device 0000:ff:06.2
(XEN) PCI add device 0000:ff:06.3
(XEN) mm.c:803: d0: Forcing read-only access to MFN fed00
(XEN) d0 attempted to change d0v2's CR4 flags 00002660 -> 00002760
(XEN) d0 attempted to change d0v0's CR4 flags 00002660 -> 00002760
(XEN) d0 attempted to change d0v1's CR4 flags 00002660 -> 00002760
(XEN) [VT-D]iommu.c:1588: d0:PCIe: unmap 0000:0a:00.0
(XEN) [VT-D]iommu.c:1449: d1:PCIe: map 0000:0a:00.0
(d1) mapping kernel into physical memory
(d1) about to get started...
(XEN) traps.c:2654:d1v0 Domain attempted WRMSR 00000000c0000081 from 0xe023e00800000000 to 0x0023001000000000.
(XEN) traps.c:2654:d1v0 Domain attempted WRMSR 00000000c0000082 from 0xffff83033076b000 to 0xffffffff8175c690.
(XEN) traps.c:2654:d1v0 Domain attempted WRMSR 00000000c0000083 from 0xffff83033076b080 to 0xffffffff8175e9c0.
(XEN) traps.c:2654:d1v0 Domain attempted WRMSR 0000000000000174 from 0x000000000000e008 to 0x0000000000000010.
(XEN) traps.c:2654:d1v0 Domain attempted WRMSR 0000000000000175 from 0xffff83033076ffc0 to 0x0000000000000000.
(XEN) traps.c:2654:d1v0 Domain attempted WRMSR 0000000000000176 from 0xffff82d080230920 to 0xffffffff8175eca0.
(XEN) traps.c:2654:d1v0 Domain attempted WRMSR 00000000c0000083 from 0xffff83033076b080 to 0xffffffff8175eee0.
(XEN) traps.c:2654:d1v0 Domain attempted WRMSR 00000000c0000084 from 0x0000000000074700 to 0x0000000000047700.
(XEN) traps.c:2654:d1v1 Domain attempted WRMSR 00000000c0000081 from 0xe023e00800000000 to 0x0023001000000000.
(XEN) traps.c:2654:d1v1 Domain attempted WRMSR 00000000c0000082 from 0xffff8303307cb000 to 0xffffffff8175c690.
(XEN) traps.c:2654:d1v1 Domain attempted WRMSR 00000000c0000083 from 0xffff8303307cb080 to 0xffffffff8175e9c0.
(XEN) traps.c:2654:d1v1 Domain attempted WRMSR 0000000000000174 from 0x000000000000e008 to 0x0000000000000010.
(XEN) traps.c:2654:d1v1 Domain attempted WRMSR 0000000000000175 from 0xffff8303307cffc0 to 0x0000000000000000.
(XEN) traps.c:2654:d1v1 Domain attempted WRMSR 0000000000000176 from 0xffff82d080230920 to 0xffffffff8175eca0.
(XEN) traps.c:2654:d1v1 Domain attempted WRMSR 00000000c0000083 from 0xffff8303307cb080 to 0xffffffff8175eee0.
(XEN) traps.c:2654:d1v1 Domain attempted WRMSR 00000000c0000084 from 0x0000000000074700 to 0x0000000000047700.
(XEN) traps.c:2654:d1v2 Domain attempted WRMSR 00000000c0000081 from 0xe023e00800000000 to 0x0023001000000000.
(XEN) traps.c:2654:d1v2 Domain attempted WRMSR 00000000c0000082 from 0xffff8303307cb000 to 0xffffffff8175c690.
(XEN) traps.c:2654:d1v2 Domain attempted WRMSR 00000000c0000083 from 0xffff8303307cb080 to 0xffffffff8175e9c0.
(XEN) traps.c:2654:d1v2 Domain attempted WRMSR 0000000000000174 from 0x000000000000e008 to 0x0000000000000010.
(XEN) traps.c:2654:d1v2 Domain attempted WRMSR 0000000000000175 from 0xffff8303307cffc0 to 0x0000000000000000.
(XEN) traps.c:2654:d1v2 Domain attempted WRMSR 0000000000000176 from 0xffff82d080230920 to 0xffffffff8175eca0.
(XEN) traps.c:2654:d1v2 Domain attempted WRMSR 00000000c0000083 from 0xffff8303307cb080 to 0xffffffff8175eee0.
(XEN) traps.c:2654:d1v2 Domain attempted WRMSR 00000000c0000084 from 0x0000000000074700 to 0x0000000000047700.
(XEN) traps.c:2654:d1v3 Domain attempted WRMSR 00000000c0000081 from 0xe023e00800000000 to 0x0023001000000000.
(XEN) traps.c:2654:d1v3 Domain attempted WRMSR 00000000c0000082 from 0xffff8303307cb000 to 0xffffffff8175c690.
(XEN) traps.c:2654:d1v3 Domain attempted WRMSR 00000000c0000083 from 0xffff8303307cb080 to 0xffffffff8175e9c0.
(XEN) traps.c:2654:d1v3 Domain attempted WRMSR 0000000000000174 from 0x000000000000e008 to 0x0000000000000010.
(XEN) traps.c:2654:d1v3 Domain attempted WRMSR 0000000000000175 from 0xffff8303307cffc0 to 0x0000000000000000.
(XEN) traps.c:2654:d1v3 Domain attempted WRMSR 0000000000000176 from 0xffff82d080230920 to 0xffffffff8175eca0.
(XEN) traps.c:2654:d1v3 Domain attempted WRMSR 00000000c0000083 from 0xffff8303307cb080 to 0xffffffff8175eee0.
(XEN) traps.c:2654:d1v3 Domain attempted WRMSR 00000000c0000084 from 0x0000000000074700 to 0x0000000000047700.
(XEN) traps.c:2654:d1v4 Domain attempted WRMSR 00000000c0000081 from 0xe023e00800000000 to 0x0023001000000000.
(XEN) traps.c:2654:d1v4 Domain attempted WRMSR 00000000c0000082 from 0xffff8303307cb000 to 0xffffffff8175c690.
(XEN) traps.c:2654:d1v4 Domain attempted WRMSR 00000000c0000083 from 0xffff8303307cb080 to 0xffffffff8175e9c0.
(XEN) traps.c:2654:d1v4 Domain attempted WRMSR 0000000000000174 from 0x000000000000e008 to 0x0000000000000010.
(XEN) traps.c:2654:d1v4 Domain attempted WRMSR 0000000000000175 from 0xffff8303307cffc0 to 0x0000000000000000.
(XEN) traps.c:2654:d1v4 Domain attempted WRMSR 0000000000000176 from 0xffff82d080230920 to 0xffffffff8175eca0.
(XEN) traps.c:2654:d1v4 Domain attempted WRMSR 00000000c0000083 from 0xffff8303307cb080 to 0xffffffff8175eee0.
(XEN) traps.c:2654:d1v4 Domain attempted WRMSR 00000000c0000084 from 0x0000000000074700 to 0x0000000000047700.
(XEN) traps.c:2654:d1v5 Domain attempted WRMSR 00000000c0000081 from 0xe023e00800000000 to 0x0023001000000000.
(XEN) traps.c:2654:d1v5 Domain attempted WRMSR 00000000c0000082 from 0xffff8303307cb000 to 0xffffffff8175c690.
(XEN) traps.c:2654:d1v5 Domain attempted WRMSR 00000000c0000083 from 0xffff8303307cb080 to 0xffffffff8175e9c0.
(XEN) traps.c:2654:d1v5 Domain attempted WRMSR 0000000000000174 from 0x000000000000e008 to 0x0000000000000010.
(XEN) traps.c:2654:d1v5 Domain attempted WRMSR 0000000000000175 from 0xffff8303307cffc0 to 0x0000000000000000.
(XEN) traps.c:2654:d1v5 Domain attempted WRMSR 0000000000000176 from 0xffff82d080230920 to 0xffffffff8175eca0.
(XEN) traps.c:2654:d1v5 Domain attempted WRMSR 00000000c0000083 from 0xffff8303307cb080 to 0xffffffff8175eee0.
(XEN) traps.c:2654:d1v5 Domain attempted WRMSR 00000000c0000084 from 0x0000000000074700 to 0x0000000000047700.
(XEN) d1 attempted to change d1v4's CR4 flags 00002660 -> 00002760
(XEN) d1 attempted to change d1v5's CR4 flags 00002660 -> 00002760
(XEN) d1 attempted to change d1v0's CR4 flags 00002660 -> 00002760
(XEN) d1 attempted to change d1v1's CR4 flags 00002660 -> 00002760
(XEN) d1 attempted to change d1v2's CR4 flags 00002660 -> 00002760
(XEN) d1 attempted to change d1v3's CR4 flags 00002660 -> 00002760
(XEN) mm.c:803: d1: Forcing read-only access to MFN fbddc
(XEN) mm.c:803: d0: Forcing read-only access to MFN fbddc
(XEN) [VT-D]iommu.c:1588: d1:PCIe: unmap 0000:0a:00.0
(XEN) [VT-D]iommu.c:1449: d0:PCIe: map 0000:0a:00.0

[-- Attachment #4: lspci --]
[-- Type: text/plain, Size: 17120 bytes --]

0a:00.0 Ethernet controller: Intel Corporation 82576 Gigabit Network Connection (rev 01)
	Subsystem: Super Micro Computer Inc Device 10c9
	Control: I/O+ Mem+ BusMaster- SpecCycle- MemWINV- VGASnoop- ParErr- Stepping- SERR- FastB2B- DisINTx-
	Status: Cap+ 66MHz- UDF- FastB2B- ParErr- DEVSEL=fast >TAbort- <TAbort- <MAbort- >SERR- <PERR- INTx-
	Interrupt: pin A routed to IRQ 28
	Region 0: Memory at fbe20000 (32-bit, non-prefetchable) [size=128K]
	Region 1: Memory at fbe00000 (32-bit, non-prefetchable) [size=128K]
	Region 2: I/O ports at e880 [size=32]
	Region 3: Memory at fbddc000 (32-bit, non-prefetchable) [size=16K]
	Expansion ROM at fbde0000 [disabled] [size=128K]
	Capabilities: [40] Power Management version 3
		Flags: PMEClk- DSI+ D1- D2- AuxCurrent=0mA PME(D0+,D1-,D2-,D3hot+,D3cold+)
		Status: D0 NoSoftRst- PME-Enable- DSel=0 DScale=1 PME-
	Capabilities: [50] MSI: Enable- Count=1/1 Maskable+ 64bit+
		Address: 0000000000000000  Data: 0000
		Masking: 00000000  Pending: 00000000
	Capabilities: [70] MSI-X: Enable- Count=10 Masked-
		Vector table: BAR=3 offset=00000000
		PBA: BAR=3 offset=00002000
	Capabilities: [a0] Express (v2) Endpoint, MSI 00
		DevCap:	MaxPayload 512 bytes, PhantFunc 0, Latency L0s <512ns, L1 <64us
			ExtTag- AttnBtn- AttnInd- PwrInd- RBE+ FLReset+
		DevCtl:	Report errors: Correctable- Non-Fatal- Fatal- Unsupported-
			RlxdOrd+ ExtTag- PhantFunc- AuxPwr- NoSnoop+ FLReset-
			MaxPayload 256 bytes, MaxReadReq 512 bytes
		DevSta:	CorrErr- UncorrErr- FatalErr- UnsuppReq- AuxPwr+ TransPend-
		LnkCap:	Port #0, Speed 2.5GT/s, Width x4, ASPM L0s L1, Latency L0 <4us, L1 <64us
			ClockPM- Surprise- LLActRep- BwNot-
		LnkCtl:	ASPM Disabled; RCB 64 bytes Disabled- Retrain- CommClk+
			ExtSynch- ClockPM- AutWidDis- BWInt- AutBWInt-
		LnkSta:	Speed 2.5GT/s, Width x4, TrErr- Train- SlotClk+ DLActive- BWMgmt- ABWMgmt-
		DevCap2: Completion Timeout: Range ABCD, TimeoutDis+
		DevCtl2: Completion Timeout: 50us to 50ms, TimeoutDis-
		LnkCtl2: Target Link Speed: 2.5GT/s, EnterCompliance- SpeedDis-, Selectable De-emphasis: -6dB
			 Transmit Margin: Normal Operating Range, EnterModifiedCompliance- ComplianceSOS-
			 Compliance De-emphasis: -6dB
		LnkSta2: Current De-emphasis Level: -6dB
	Capabilities: [100 v1] Advanced Error Reporting
		UESta:	DLP- SDES- TLP- FCP- CmpltTO- CmpltAbrt- UnxCmplt- RxOF- MalfTLP- ECRC- UnsupReq- ACSViol-
		UEMsk:	DLP- SDES- TLP- FCP- CmpltTO- CmpltAbrt- UnxCmplt- RxOF- MalfTLP- ECRC- UnsupReq- ACSViol-
		UESvrt:	DLP+ SDES- TLP- FCP+ CmpltTO- CmpltAbrt- UnxCmplt- RxOF+ MalfTLP+ ECRC- UnsupReq- ACSViol-
		CESta:	RxErr- BadTLP- BadDLLP- Rollover- Timeout- NonFatalErr+
		CEMsk:	RxErr- BadTLP- BadDLLP- Rollover- Timeout- NonFatalErr+
		AERCap:	First Error Pointer: 00, GenCap- CGenEn- ChkCap- ChkEn-
	Capabilities: [140 v1] Device Serial Number 00-30-48-ff-ff-f1-03-06
	Capabilities: [150 v1] Alternative Routing-ID Interpretation (ARI)
		ARICap:	MFVC- ACS-, Next Function: 1
		ARICtl:	MFVC- ACS-, Function Group: 0
	Capabilities: [160 v1] Single Root I/O Virtualization (SR-IOV)
		IOVCap:	Migration-, Interrupt Message Number: 000
		IOVCtl:	Enable- Migration- Interrupt- MSE- ARIHierarchy+
		IOVSta:	Migration-
		Initial VFs: 8, Total VFs: 8, Number of VFs: 0, Function Dependency Link: 00
		VF offset: 128, stride: 2, Device ID: 10ca
		Supported Page Size: 00000553, System Page Size: 00000001
		Region 0: Memory at 00000000fbda0000 (64-bit, non-prefetchable)
		Region 3: Memory at 00000000fbd80000 (64-bit, non-prefetchable)
		VF Migration: offset: 00000000, BIR: 0
	Kernel driver in use: pciback
00: 86 80 c9 10 03 00 10 00 01 00 00 02 40 00 80 00
10: 00 00 e2 fb 00 00 e0 fb 81 e8 00 00 00 c0 dd fb
20: 00 00 00 00 00 00 00 00 00 00 00 00 d9 15 c9 10
30: 00 00 de fb 40 00 00 00 00 00 00 00 07 01 00 00
40: 01 50 23 c8 00 20 00 1a 00 00 00 00 00 00 00 00
50: 05 70 80 01 00 00 00 00 00 00 00 00 00 00 00 00
60: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
70: 11 a0 09 00 03 00 00 00 03 20 00 00 00 00 00 00
80: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
90: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
a0: 10 00 02 00 c2 8c 00 10 30 28 10 00 41 6c 03 00
b0: 40 00 41 10 00 00 00 00 00 00 00 00 00 00 00 00
c0: 00 00 00 00 1f 00 00 00 00 00 00 00 00 00 00 00
d0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
e0: 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
f0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
100: 01 00 01 14 00 00 00 00 00 00 00 00 11 20 06 00
110: 00 20 00 00 00 20 00 00 00 00 00 00 00 00 00 00
120: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
130: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
140: 03 00 01 15 06 03 f1 ff ff 48 30 00 00 00 00 00
150: 0e 00 01 16 00 01 00 00 00 00 00 00 00 00 00 00
160: 10 00 01 00 00 00 00 00 10 00 00 00 08 00 08 00
170: 00 00 00 00 80 00 02 00 00 00 ca 10 53 05 00 00
180: 01 00 00 00 04 00 da fb 00 00 00 00 00 00 00 00
190: 04 00 d8 fb 00 00 00 00 00 00 00 00 00 00 00 00
1a0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
1b0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
1c0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
1d0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
1e0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
1f0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
200: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
210: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
220: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
230: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
240: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
250: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
260: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
270: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
280: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
290: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
2a0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
2b0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
2c0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
2d0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
2e0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
2f0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
300: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
310: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
320: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
330: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
340: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
350: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
360: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
370: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
380: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
390: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
3a0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
3b0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
3c0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
3d0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
3e0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
3f0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
400: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
410: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
420: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
430: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
440: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
450: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
460: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
470: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
480: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
490: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
4a0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
4b0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
4c0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
4d0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
4e0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
4f0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
500: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
510: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
520: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
530: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
540: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
550: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
560: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
570: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
580: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
590: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
5a0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
5b0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
5c0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
5d0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
5e0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
5f0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
600: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
610: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
620: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
630: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
640: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
650: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
660: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
670: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
680: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
690: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
6a0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
6b0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
6c0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
6d0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
6e0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
6f0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
700: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
710: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
720: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
730: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
740: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
750: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
760: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
770: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
780: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
790: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
7a0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
7b0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
7c0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
7d0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
7e0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
7f0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
800: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
810: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
820: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
830: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
840: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
850: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
860: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
870: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
880: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
890: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
8a0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
8b0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
8c0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
8d0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
8e0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
8f0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
900: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
910: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
920: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
930: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
940: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
950: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
960: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
970: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
980: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
990: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
9a0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
9b0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
9c0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
9d0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
9e0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
9f0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
a00: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
a10: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
a20: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
a30: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
a40: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
a50: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
a60: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
a70: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
a80: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
a90: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
aa0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
ab0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
ac0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
ad0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
ae0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
af0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
b00: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
b10: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
b20: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
b30: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
b40: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
b50: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
b60: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
b70: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
b80: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
b90: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
ba0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
bb0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
bc0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
bd0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
be0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
bf0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
c00: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
c10: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
c20: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
c30: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
c40: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
c50: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
c60: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
c70: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
c80: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
c90: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
ca0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
cb0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
cc0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
cd0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
ce0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
cf0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
d00: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
d10: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
d20: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
d30: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
d40: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
d50: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
d60: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
d70: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
d80: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
d90: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
da0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
db0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
dc0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
dd0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
de0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
df0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
e00: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
e10: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
e20: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
e30: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
e40: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
e50: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
e60: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
e70: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
e80: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
e90: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
ea0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
eb0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
ec0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
ed0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
ee0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
ef0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
f00: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
f10: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
f20: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
f30: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
f40: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
f50: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
f60: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
f70: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
f80: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
f90: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
fa0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
fb0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
fc0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
fd0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
fe0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
ff0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00


[-- Attachment #5: Type: text/plain, Size: 126 bytes --]

_______________________________________________
Xen-devel mailing list
Xen-devel@lists.xen.org
http://lists.xen.org/xen-devel

Re: [PATCH v2 2/4] x86/MSI-X: access MSI-X table only after having enabled MSI-X

[Jan Beulich]

[-- Attachment #1: Type: text/plain, Size: 1958 bytes --]

>>> On 10.04.15 at 22:02, <konrad.wilk@oracle.com> wrote:
> On Wed, Mar 25, 2015 at 04:39:49PM +0000, Jan Beulich wrote:
>> As done in Linux by f598282f51 ("PCI: Fix the NIU MSI-X problem in a
>> better way") and its broken predecessor, make sure we don't access the
>> MSI-X table without having enabled MSI-X first, using the mask-all flag
>> instead to prevent interrupts from occurring.
> 
> This causes an regression with an Linux guest that has the XSA120 + XSA120
> addendum with PV guests (hadn't tried yet HVM).

You mentioning XSA-120 and its addendum - are these requirements
for the problem to be seen? I admit I may have tested a PV guest
only with an SR-IOV VF (and only a HVM guest also with an "ordinary"
device), but I'd like to be clear about the validity of the connection.

> When PV guest requests an MSI-X, pciback gets:
> 
> [  122.778654] xen-pciback: 0000:0a:00.0: enable MSI-X
> [  122.778861] pciback 0000:0a:00.0: xen map irq failed -6 for 1 domain
> [  122.778887] xen_pciback: 0000:0a:00.0: error enabling MSI-X for guest 1: err -6!

Yeah, there were a few adjustments necessary to get similar issues
under control for HVM guests - maybe there's a path left not covered
by what's done for HVM guests.

> The device has the PCI_COMMAND enabled correctly:
> 
> # lspci -s 0a:0.0 -vvv | head
> 0a:00.0 Ethernet controller: Intel Corporation 82576 Gigabit Network 
> Connection (rev 01)
>         Subsystem: Super Micro Computer Inc Device 10c9
>         Control: I/O+ Mem+ BusMaster- SpecCycle- MemWINV- VGASnoop- ParErr- 
> Stepping- SERR- FastB2B- DisINTx-
> 
> Attaching the 'xl dmesg', 'dmesg', and 'lspci'

Right - MSI-X is still disabled if the lspci really matches the state at
the time the failure occurred. I'm attaching the original patch with
some debugging code left in, which I suppose would show where
the problematic path is in case you can get to it before I would.

Jan


[-- Attachment #2: x86-MSI-X-enable.patch --]
[-- Type: text/plain, Size: 10559 bytes --]


TODO: drop //temp-s

As done in Linux by f598282f51 ("PCI: Fix the NIU MSI-X problem in a
better way") and its broken predecessor, make sure we don't access the
MSI-X table without having enabled MSI-X first, using the mask-all flag
instead to prevent interrupts from occurring.

--- unstable.orig/xen/arch/x86/msi.c	2015-02-10 08:19:17.000000000 +0100
+++ unstable/xen/arch/x86/msi.c	2015-02-10 09:05:12.000000000 +0100
@@ -142,6 +142,23 @@ static bool_t memory_decoded(const struc
               PCI_COMMAND_MEMORY);
 }
 
+static bool_t msix_memory_decoded(const struct pci_dev *dev, unsigned int pos)
+{
+    u16 control = pci_conf_read16(dev->seg, dev->bus,
+                                  PCI_SLOT(dev->devfn),
+                                  PCI_FUNC(dev->devfn),
+                                  msix_control_reg(pos));
+
+    if ( !(control & PCI_MSIX_FLAGS_ENABLE) )
+{//temp
+ static bool_t warned;
+ WARN_ON(!test_and_set_bool(warned));
+        return 0;
+}
+
+    return memory_decoded(dev);
+}
+
 /*
  * MSI message composition
  */
@@ -219,7 +236,8 @@ static bool_t read_msi_msg(struct msi_de
         void __iomem *base;
         base = entry->mask_base;
 
-        if ( unlikely(!memory_decoded(entry->dev)) )
+        if ( unlikely(!msix_memory_decoded(entry->dev,
+                                           entry->msi_attrib.pos)) )
             return 0;
         msg->address_lo = readl(base + PCI_MSIX_ENTRY_LOWER_ADDR_OFFSET);
         msg->address_hi = readl(base + PCI_MSIX_ENTRY_UPPER_ADDR_OFFSET);
@@ -285,7 +303,8 @@ static int write_msi_msg(struct msi_desc
         void __iomem *base;
         base = entry->mask_base;
 
-        if ( unlikely(!memory_decoded(entry->dev)) )
+        if ( unlikely(!msix_memory_decoded(entry->dev,
+                                           entry->msi_attrib.pos)) )
             return -ENXIO;
         writel(msg->address_lo,
                base + PCI_MSIX_ENTRY_LOWER_ADDR_OFFSET);
@@ -379,7 +398,7 @@ static bool_t msi_set_mask_bit(struct ir
 {
     struct msi_desc *entry = desc->msi_desc;
     struct pci_dev *pdev;
-    u16 seg;
+    u16 seg, control;
     u8 bus, slot, func;
 
     ASSERT(spin_is_locked(&desc->lock));
@@ -401,35 +420,38 @@ static bool_t msi_set_mask_bit(struct ir
         }
         break;
     case PCI_CAP_ID_MSIX:
+        control = pci_conf_read16(seg, bus, slot, func,
+                                  msix_control_reg(entry->msi_attrib.pos));
+        if ( unlikely(!(control & PCI_MSIX_FLAGS_ENABLE)) )
+            pci_conf_write16(seg, bus, slot, func,
+                             msix_control_reg(entry->msi_attrib.pos),
+                             control | (PCI_MSIX_FLAGS_ENABLE |
+                                        PCI_MSIX_FLAGS_MASKALL));
         if ( likely(memory_decoded(pdev)) )
         {
             writel(flag, entry->mask_base + PCI_MSIX_ENTRY_VECTOR_CTRL_OFFSET);
             readl(entry->mask_base + PCI_MSIX_ENTRY_VECTOR_CTRL_OFFSET);
-            break;
+            if ( likely(control & PCI_MSIX_FLAGS_ENABLE) )
+                break;
+            flag = 1;
         }
-        if ( flag )
+        else if ( flag && !(control & PCI_MSIX_FLAGS_MASKALL) )
         {
-            u16 control;
             domid_t domid = pdev->domain->domain_id;
 
-            control = pci_conf_read16(seg, bus, slot, func,
-                                      msix_control_reg(entry->msi_attrib.pos));
-            if ( control & PCI_MSIX_FLAGS_MASKALL )
-                break;
-            pci_conf_write16(seg, bus, slot, func,
-                             msix_control_reg(entry->msi_attrib.pos),
-                             control | PCI_MSIX_FLAGS_MASKALL);
+            control |= PCI_MSIX_FLAGS_MASKALL;
             if ( pdev->msix->warned != domid )
             {
                 pdev->msix->warned = domid;
                 printk(XENLOG_G_WARNING
-                       "cannot mask IRQ %d: masked MSI-X on Dom%d's %04x:%02x:%02x.%u\n",
+                       "cannot mask IRQ %d: masking MSI-X on Dom%d's %04x:%02x:%02x.%u\n",
                        desc->irq, domid, pdev->seg, pdev->bus,
                        PCI_SLOT(pdev->devfn), PCI_FUNC(pdev->devfn));
             }
-            break;
         }
-        /* fall through */
+        pci_conf_write16(seg, bus, slot, func,
+                         msix_control_reg(entry->msi_attrib.pos), control);
+        return flag;
     default:
         return 0;
     }
@@ -454,7 +476,8 @@ static int msi_get_mask_bit(const struct
                                 entry->msi.mpos) >>
                 entry->msi_attrib.entry_nr) & 1;
     case PCI_CAP_ID_MSIX:
-        if ( unlikely(!memory_decoded(entry->dev)) )
+        if ( unlikely(!msix_memory_decoded(entry->dev,
+                                           entry->msi_attrib.pos)) )
             break;
         return readl(entry->mask_base + PCI_MSIX_ENTRY_VECTOR_CTRL_OFFSET) & 1;
     }
@@ -775,16 +798,32 @@ static int msix_capability_init(struct p
 
     pos = pci_find_cap_offset(seg, bus, slot, func, PCI_CAP_ID_MSIX);
     control = pci_conf_read16(seg, bus, slot, func, msix_control_reg(pos));
-    msix_set_enable(dev, 0);/* Ensure msix is disabled as I set it up */
+    /*
+     * Ensure MSI-X interrupts are masked during setup. Some devices require
+     * MSI-X to be enabled before we can touch the MSI-X registers. We need
+     * to mask all the vectors to prevent interrupts coming in before they're
+     * fully set up.
+     */
+    pci_conf_write16(seg, bus, slot, func, msix_control_reg(pos),
+                     control | (PCI_MSIX_FLAGS_ENABLE |
+                                PCI_MSIX_FLAGS_MASKALL));
 
     if ( unlikely(!memory_decoded(dev)) )
+    {
+        pci_conf_write16(seg, bus, slot, func, msix_control_reg(pos),
+                         control & ~PCI_MSIX_FLAGS_ENABLE);
         return -ENXIO;
+    }
 
     if ( desc )
     {
         entry = alloc_msi_entry(1);
         if ( !entry )
+        {
+            pci_conf_write16(seg, bus, slot, func, msix_control_reg(pos),
+                             control & ~PCI_MSIX_FLAGS_ENABLE);
             return -ENOMEM;
+        }
         ASSERT(msi);
     }
 
@@ -815,6 +854,8 @@ static int msix_capability_init(struct p
     {
         if ( !msi || !msi->table_base )
         {
+            pci_conf_write16(seg, bus, slot, func, msix_control_reg(pos),
+                             control & ~PCI_MSIX_FLAGS_ENABLE);
             xfree(entry);
             return -ENXIO;
         }
@@ -857,6 +898,8 @@ static int msix_capability_init(struct p
 
         if ( idx < 0 )
         {
+            pci_conf_write16(seg, bus, slot, func, msix_control_reg(pos),
+                             control & ~PCI_MSIX_FLAGS_ENABLE);
             xfree(entry);
             return idx;
         }
@@ -912,8 +955,7 @@ static int msix_capability_init(struct p
     ++msix->used_entries;
 
     /* Restore MSI-X enabled bits */
-    pci_conf_write16(seg, bus, slot, func, msix_control_reg(pos),
-                     control & ~PCI_MSIX_FLAGS_MASKALL);
+    pci_conf_write16(seg, bus, slot, func, msix_control_reg(pos), control);
 
     return 0;
 }
@@ -1062,7 +1104,10 @@ static void __pci_disable_msix(struct ms
 
     pos = pci_find_cap_offset(seg, bus, slot, func, PCI_CAP_ID_MSIX);
     control = pci_conf_read16(seg, bus, slot, func, msix_control_reg(pos));
-    msix_set_enable(dev, 0);
+    if ( unlikely(!(control & PCI_MSIX_FLAGS_ENABLE)) )
+        pci_conf_write16(seg, bus, slot, func, msix_control_reg(pos),
+                         control | (PCI_MSIX_FLAGS_ENABLE |
+                                    PCI_MSIX_FLAGS_MASKALL));
 
     BUG_ON(list_empty(&dev->msi_list));
 
@@ -1188,6 +1234,8 @@ int pci_restore_msi_state(struct pci_dev
     list_for_each_entry_safe( entry, tmp, &pdev->msi_list, list )
     {
         unsigned int i = 0, nr = 1;
+        u16 control = 0;
+        u8 slot = PCI_SLOT(pdev->devfn), func = PCI_FUNC(pdev->devfn);
 
         irq = entry->irq;
         desc = &irq_desc[irq];
@@ -1214,10 +1262,18 @@ int pci_restore_msi_state(struct pci_dev
         }
         else if ( entry->msi_attrib.type == PCI_CAP_ID_MSIX )
         {
-            msix_set_enable(pdev, 0);
+            control = pci_conf_read16(pdev->seg, pdev->bus, slot, func,
+                                      msix_control_reg(entry->msi_attrib.pos));
+            pci_conf_write16(pdev->seg, pdev->bus, slot, func,
+                             msix_control_reg(entry->msi_attrib.pos),
+                             control | (PCI_MSIX_FLAGS_ENABLE |
+                                        PCI_MSIX_FLAGS_MASKALL));
             if ( unlikely(!memory_decoded(pdev)) )
             {
                 spin_unlock_irqrestore(&desc->lock, flags);
+                pci_conf_write16(pdev->seg, pdev->bus, slot, func,
+                                 msix_control_reg(entry->msi_attrib.pos),
+                                 control & ~PCI_MSIX_FLAGS_ENABLE);
                 return -ENXIO;
             }
         }
@@ -1246,11 +1302,9 @@ int pci_restore_msi_state(struct pci_dev
         if ( entry->msi_attrib.type == PCI_CAP_ID_MSI )
         {
             unsigned int cpos = msi_control_reg(entry->msi_attrib.pos);
-            u16 control = pci_conf_read16(pdev->seg, pdev->bus,
-                                          PCI_SLOT(pdev->devfn),
-                                          PCI_FUNC(pdev->devfn), cpos);
 
-            control &= ~PCI_MSI_FLAGS_QSIZE;
+            control = pci_conf_read16(pdev->seg, pdev->bus, slot, func, cpos) &
+                      ~PCI_MSI_FLAGS_QSIZE;
             multi_msi_enable(control, entry->msi.nvec);
             pci_conf_write16(pdev->seg, pdev->bus, PCI_SLOT(pdev->devfn),
                              PCI_FUNC(pdev->devfn), cpos, control);
@@ -1258,7 +1312,9 @@ int pci_restore_msi_state(struct pci_dev
             msi_set_enable(pdev, 1);
         }
         else if ( entry->msi_attrib.type == PCI_CAP_ID_MSIX )
-            msix_set_enable(pdev, 1);
+            pci_conf_write16(pdev->seg, pdev->bus, slot, func,
+                             msix_control_reg(entry->msi_attrib.pos),
+                             control | PCI_MSIX_FLAGS_ENABLE);
     }
 
     return 0;

[-- Attachment #3: Type: text/plain, Size: 126 bytes --]

_______________________________________________
Xen-devel mailing list
Xen-devel@lists.xen.org
http://lists.xen.org/xen-devel

Re: [PATCH v2 2/4] x86/MSI-X: access MSI-X table only after having enabled MSI-X

[Konrad Rzeszutek Wilk]

On Mon, Apr 13, 2015 at 10:05:14AM +0100, Jan Beulich wrote:
> >>> On 10.04.15 at 22:02, <konrad.wilk@oracle.com> wrote:
> > On Wed, Mar 25, 2015 at 04:39:49PM +0000, Jan Beulich wrote:
> >> As done in Linux by f598282f51 ("PCI: Fix the NIU MSI-X problem in a
> >> better way") and its broken predecessor, make sure we don't access the
> >> MSI-X table without having enabled MSI-X first, using the mask-all flag
> >> instead to prevent interrupts from occurring.
> > 
> > This causes an regression with an Linux guest that has the XSA120 + XSA120
> > addendum with PV guests (hadn't tried yet HVM).
> 
> You mentioning XSA-120 and its addendum - are these requirements
> for the problem to be seen? I admit I may have tested a PV guest
> only with an SR-IOV VF (and only a HVM guest also with an "ordinary"
> device), but I'd like to be clear about the validity of the connection.

No. I just tried with v4.0-rc5 (and then also v4.0) and just 
using SR-IOV to make this simpler.

With staging  + two of your patches:
a10cc68 TODO: drop //temp-s
1b8721c x86/MSI-X: be more careful during teardown

When trying to enable SR-IOV I get this error:

failed to echo 1 > /sys/devices/pci0000:00/0000:00:01.0/0000:0a:00.0/sriov_numvfs, rc: 1
(hadn't tried just passing in an HVM guest).

Attached is the 'xl dmesg'.

Please keep in mind that if I do this based on

70a3cbb x86/vMSI-X: honor all mask requests

or 
df9f567 x86/vMSI-X: add valid bits for read acceleration

It works and I can setup VFs (and do PCI operations).


Here is the 'xl dmesg' output.

 Xen 4.6-unstable
(XEN) Xen version 4.6-unstable (konrad@(none)) (gcc (GCC) 4.4.4 20100503 (Red Hat 4.4.4-2)) debug=y Wed Apr 15 11:44:44 EDT 2015
(XEN) Latest ChangeSet: Wed Apr 15 11:43:31 2015 -0400 git:a10cc68
(XEN) Bootloader: unknown
(XEN) Command line: com1=115200,8n1 dom0_mem=999M,max:1232M dom0_max_vcpus=3 cpufreq=performance,verbose iommu=verbose,no-intremap console=com1,vga loglvl=all guest_loglvl=all
(XEN) Video information:
(XEN)  VGA is text mode 80x25, font 8x16
(XEN)  VBE/DDC methods: none; EDID transfer time: 2 seconds
(XEN)  EDID info not retrieved because no DDC retrieval method detected
(XEN) Disc information:
(XEN)  Found 2 MBR signatures
(XEN)  Found 2 EDD information structures
(XEN) Xen-e820 RAM map:
(XEN)  0000000000000000 - 000000000009e800 (usable)
(XEN)  000000000009e800 - 00000000000a0000 (reserved)
(XEN)  00000000000e6000 - 0000000000100000 (reserved)
(XEN)  0000000000100000 - 00000000bf770000 (usable)
(XEN)  00000000bf770000 - 00000000bf77e000 (ACPI data)
(XEN)  00000000bf77e000 - 00000000bf7d0000 (ACPI NVS)
(XEN)  00000000bf7d0000 - 00000000bf7e0000 (reserved)
(XEN)  00000000bf7ec000 - 00000000c0000000 (reserved)
(XEN)  00000000e0000000 - 00000000f0000000 (reserved)
(XEN)  00000000fee00000 - 00000000fee01000 (reserved)
(XEN)  00000000ffc00000 - 0000000100000000 (reserved)
(XEN)  0000000100000000 - 0000000340000000 (usable)
(XEN) ACPI: RSDP 000FA180, 0024 (r2 ACPIAM)
(XEN) ACPI: XSDT BF770100, 006C (r1 SMCI            20111028 MSFT       97)
(XEN) ACPI: FACP BF770290, 00F4 (r4 102811 FACP1450 20111028 MSFT       97)
(XEN) ACPI: DSDT BF7706C0, 5CBB (r2  1F280 1F280000        0 INTL 20051117)
(XEN) ACPI: FACS BF77E000, 0040
(XEN) ACPI: APIC BF770390, 0136 (r2 102811 APIC1450 20111028 MSFT       97)
(XEN) ACPI: MCFG BF7704D0, 003C (r1 102811 OEMMCFG  20111028 MSFT       97)
(XEN) ACPI: SLIT BF770510, 0030 (r1 102811 OEMSLIT  20111028 MSFT       97)
(XEN) ACPI: OEMB BF77E040, 0092 (r1 102811 OEMB1450 20111028 MSFT       97)
(XEN) ACPI: SRAT BF77A6C0, 01A8 (r2 102811 OEMSRAT         1 INTL        1)
(XEN) ACPI: HPET BF77A870, 0038 (r1 102811 OEMHPET  20111028 MSFT       97)
(XEN) ACPI: DMAR BF77E0E0, 0120 (r1    AMI  OEMDMAR        1 MSFT       97)
(XEN) ACPI: SSDT BF781720, 0363 (r1 DpgPmm    CpuPm       12 INTL 20051117)
(XEN) System RAM: 12279MB (12573752kB)
(XEN) SRAT: PXM 0 -> APIC 0 -> Node 0
(XEN) SRAT: PXM 0 -> APIC 2 -> Node 0
(XEN) SRAT: PXM 0 -> APIC 4 -> Node 0
(XEN) SRAT: PXM 0 -> APIC 6 -> Node 0
(XEN) SRAT: PXM 0 -> APIC 1 -> Node 0
(XEN) SRAT: PXM 0 -> APIC 3 -> Node 0
(XEN) SRAT: PXM 0 -> APIC 5 -> Node 0
(XEN) SRAT: PXM 0 -> APIC 7 -> Node 0
(XEN) SRAT: PXM 1 -> APIC 16 -> Node 1
(XEN) SRAT: PXM 1 -> APIC 18 -> Node 1
(XEN) SRAT: PXM 1 -> APIC 20 -> Node 1
(XEN) SRAT: PXM 1 -> APIC 22 -> Node 1
(XEN) SRAT: PXM 1 -> APIC 17 -> Node 1
(XEN) SRAT: PXM 1 -> APIC 19 -> Node 1
(XEN) SRAT: PXM 1 -> APIC 21 -> Node 1
(XEN) SRAT: PXM 1 -> APIC 23 -> Node 1
(XEN) SRAT: Node 1 PXM 1 0-a0000
(XEN) SRAT: Node 1 PXM 1 100000-c0000000
(XEN) SRAT: Node 1 PXM 1 100000000-340000000
(XEN) NUMA: Allocated memnodemap from 338f4f000 - 338f53000
(XEN) NUMA: Using 8 for the hash shift.
(XEN) SRAT: Node 0 has no memory. BIOS Bug or mis-configured hardware?
(XEN) Domain heap initialised DMA width 9 bits
(XEN) found SMP MP-table at 000ff780
(XEN) DMI present.
(XEN) Using APIC driver default
(XEN) ACPI: PM-Timer IO Port: 0x808
(XEN) ACPI: SLEEP INFO: pm1x_cnt[1:804,1:0], pm1x_evt[1:800,1:0]
(XEN) ACPI:             wakeup_vec[bf77e00c], vec_size[20]
(XEN) ACPI: Local APIC address 0xfee00000
(XEN) ACPI: LAPIC (acpi_id[0x01] lapic_id[0x00] enabled)
(XEN) Processor #0 7:10 APIC version 21
(XEN) ACPI: LAPIC (acpi_id[0x02] lapic_id[0x02] enabled)
(XEN) Processor #2 7:10 APIC version 21
(XEN) ACPI: LAPIC (acpi_id[0x03] lapic_id[0x04] enabled)
(XEN) Processor #4 7:10 APIC version 21
(XEN) ACPI: LAPIC (acpi_id[0x04] lapic_id[0x06] enabled)
(XEN) Processor #6 7:10 APIC version 21
(XEN) ACPI: LAPIC (acpi_id[0x05] lapic_id[0x10] enabled)
(XEN) Processor #16 7:10 APIC version 21
(XEN) ACPI: LAPIC (acpi_id[0x06] lapic_id[0x12] enabled)
(XEN) Processor #18 7:10 APIC version 21
(XEN) ACPI: LAPIC (acpi_id[0x07] lapic_id[0x14] enabled)
(XEN) Processor #20 7:10 APIC version 21
(XEN) ACPI: LAPIC (acpi_id[0x08] lapic_id[0x16] enabled)
(XEN) Processor #22 7:10 APIC version 21
(XEN) ACPI: LAPIC (acpi_id[0x09] lapic_id[0x01] enabled)
(XEN) Processor #1 7:10 APIC version 21
(XEN) ACPI: LAPIC (acpi_id[0x0a] lapic_id[0x03] enabled)
(XEN) Processor #3 7:10 APIC version 21
(XEN) ACPI: LAPIC (acpi_id[0x0b] lapic_id[0x05] enabled)
(XEN) Processor #5 7:10 APIC version 21
(XEN) ACPI: LAPIC (acpi_id[0x0c] lapic_id[0x07] enabled)
(XEN) Processor #7 7:10 APIC version 21
(XEN) ACPI: LAPIC (acpi_id[0x0d] lapic_id[0x11] enabled)
(XEN) Processor #17 7:10 APIC version 21
(XEN) ACPI: LAPIC (acpi_id[0x0e] lapic_id[0x13] enabled)
(XEN) Processor #19 7:10 APIC version 21
(XEN) ACPI: LAPIC (acpi_id[0x0f] lapic_id[0x15] enabled)
(XEN) Processor #21 7:10 APIC version 21
(XEN) ACPI: LAPIC (acpi_id[0x10] lapic_id[0x17] enabled)
(XEN) Processor #23 7:10 APIC version 21
(XEN) ACPI: LAPIC (acpi_id[0x11] lapic_id[0x90] disabled)
(XEN) ACPI: LAPIC (acpi_id[0x12] lapic_id[0x91] disabled)
(XEN) ACPI: LAPIC (acpi_id[0x13] lapic_id[0x92] disabled)
(XEN) ACPI: LAPIC (acpi_id[0x14] lapic_id[0x93] disabled)
(XEN) ACPI: LAPIC (acpi_id[0x15] lapic_id[0x94] disabled)
(XEN) ACPI: LAPIC (acpi_id[0x16] lapic_id[0x95] disabled)
(XEN) ACPI: LAPIC (acpi_id[0x17] lapic_id[0x96] disabled)
(XEN) ACPI: LAPIC (acpi_id[0x18] lapic_id[0x97] disabled)
(XEN) ACPI: LAPIC_NMI (acpi_id[0xff] dfl dfl lint[0x1])
(XEN) Overriding APIC driver with bigsmp
(XEN) ACPI: IOAPIC (id[0x08] address[0xfec00000] gsi_base[0])
(XEN) IOAPIC[0]: apic_id 8, version 32, address 0xfec00000, GSI 0-23
(XEN) ACPI: IOAPIC (id[0x09] address[0xfec8a000] gsi_base[24])
(XEN) IOAPIC[1]: apic_id 9, version 32, address 0xfec8a000, GSI 24-47
(XEN) ACPI: IOAPIC (id[0x0a] address[0xfec80000] gsi_base[48])
(XEN) IOAPIC[2]: apic_id 10, version 32, address 0xfec80000, GSI 48-71
(XEN) ACPI: IOAPIC (id[0x0b] address[0xfec80400] gsi_base[72])
(XEN) IOAPIC[3]: apic_id 11, version 32, address 0xfec80400, GSI 72-95
(XEN) ACPI: INT_SRC_OVR (bus 0 bus_irq 0 global_irq 2 dfl dfl)
(XEN) ACPI: INT_SRC_OVR (bus 0 bus_irq 9 global_irq 20 low level)
(XEN) ACPI: IRQ0 used by override.
(XEN) ACPI: IRQ2 used by override.
(XEN) ACPI: IRQ9 used by override.
(XEN) Enabling APIC mode:  Phys.  Using 4 I/O APICs
(XEN) ACPI: HPET id: 0x8086a301 base: 0xfed00000
(XEN) [VT-D]dmar.c:789: Host address width 40
(XEN) [VT-D]dmar.c:803: found ACPI_DMAR_DRHD:
(XEN) [VT-D]dmar.c:473:   dmaru->address = fbffe000
(XEN) [VT-D]iommu.c:1151: drhd->address = fbffe000 iommu->reg = ffff82c000201000
(XEN) [VT-D]iommu.c:1153: cap = c90780106f0462 ecap = f020f6
(XEN) [VT-D]dmar.c:487:   flags: INCLUDE_ALL
(XEN) [VT-D]dmar.c:808: found ACPI_DMAR_RMRR:
(XEN) [VT-D]dmar.c:384:  endpoint: 0000:00:1d.0
(XEN) [VT-D]dmar.c:384:  endpoint: 0000:00:1d.1
(XEN) [VT-D]dmar.c:384:  endpoint: 0000:00:1d.2
(XEN) [VT-D]dmar.c:384:  endpoint: 0000:00:1d.7
(XEN) [VT-D]dmar.c:384:  endpoint: 0000:00:1a.0
(XEN) [VT-D]dmar.c:384:  endpoint: 0000:00:1a.1
(XEN) [VT-D]dmar.c:384:  endpoint: 0000:00:1a.2
(XEN) [VT-D]dmar.c:384:  endpoint: 0000:00:1a.7
(XEN) [VT-D]dmar.c:677:   RMRR region: base_addr e6000 end_address e9fff
(XEN) [VT-D]dmar.c:808: found ACPI_DMAR_RMRR:
(XEN) [VT-D]dmar.c:384:  endpoint: 0000:00:1d.0
(XEN) [VT-D]dmar.c:384:  endpoint: 0000:00:1d.1
(XEN) [VT-D]dmar.c:384:  endpoint: 0000:00:1d.2
(XEN) [VT-D]dmar.c:384:  endpoint: 0000:00:1d.7
(XEN) [VT-D]dmar.c:384:  endpoint: 0000:00:1a.0
(XEN) [VT-D]dmar.c:384:  endpoint: 0000:00:1a.1
(XEN) [VT-D]dmar.c:384:  endpoint: 0000:00:1a.2
(XEN) [VT-D]dmar.c:384:  endpoint: 0000:00:1a.7
(XEN) [VT-D]dmar.c:677:   RMRR region: base_addr bf7ec000 end_address bf7fffff
(XEN) [VT-D]dmar.c:813: found ACPI_DMAR_ATSR:
(XEN) [VT-D]dmar.c:706:   atsru->all_ports: 0
(XEN) [VT-D]dmar.c:354:  bridge: 0000:00:01.0 start=0 sec=a sub=b
(XEN) [VT-D]dmar.c:354:  bridge: 0000:00:03.0 start=0 sec=9 sub=9
(XEN) [VT-D]dmar.c:354:  bridge: 0000:00:05.0 start=0 sec=8 sub=8
(XEN) [VT-D]dmar.c:354:  bridge: 0000:00:07.0 start=0 sec=7 sub=7
(XEN) [VT-D]dmar.c:354:  bridge: 0000:00:09.0 start=0 sec=4 sub=6
(XEN) ERST table was not found
(XEN) Using ACPI (MADT) for SMP configuration information
(XEN) SMP: Allowing 24 CPUs (8 hotplug CPUs)
(XEN) IRQ limits: 96 GSI, 2992 MSI/MSI-X
(XEN) Using scheduler: SMP Credit Scheduler (credit)
(XEN) Detected 2266.812 MHz processor.
(XEN) Initing memory sharing.
(XEN) mce_intel.c:735: MCA Capability: BCAST 1 SER 0 CMCI 1 firstbank 0 extended MCE MSR 0
(XEN) Intel machine check reporting enabled
(XEN) alt table ffff82d0802d9e30 -> ffff82d0802db03c
(XEN) PCI: MCFG configuration 0: base e0000000 segment 0000 buses 00 - ff
(XEN) PCI: MCFG area at e0000000 reserved in E820
(XEN) PCI: Using MCFG for segment 0000 bus 00-ff
(XEN) Intel VT-d iommu 0 supported page sizes: 4kB.
(XEN) Intel VT-d Snoop Control enabled.
(XEN) Intel VT-d Dom0 DMA Passthrough not enabled.
(XEN) Intel VT-d Queued Invalidation enabled.
(XEN) Intel VT-d Interrupt Remapping not enabled.
(XEN) Intel VT-d Shared EPT tables not enabled.
(XEN) I/O virtualisation enabled
(XEN)  - Dom0 mode: Relaxed
(XEN) Interrupt remapping disabled
(XEN) ENABLING IO-APIC IRQs
(XEN)  -> Using new ACK method
(XEN) ..TIMER: vector=0xF0 apic1=0 pin1=2 apic2=-1 pin2=-1
(XEN) Platform timer is 14.318MHz HPET
(XEN) Allocated console ring of 128 KiB.
(XEN) mwait-idle: MWAIT substates: 0x1120
(XEN) mwait-idle: v0.4 model 0x1a
(XEN) mwait-idle: lapic_timer_reliable_states 0x2
(XEN) HPET: 0 timers usable for broadcast (4 total)
(XEN) VMX: Supported advanced features:
(XEN)  - APIC MMIO access virtualisation
(XEN)  - APIC TPR shadow
(XEN)  - Extended Page Tables (EPT)
(XEN)  - Virtual-Processor Identifiers (VPID)
(XEN)  - Virtual NMI
(XEN)  - MSR direct-access bitmap
(XEN) HVM: ASIDs enabled.
(XEN) HVM: VMX enabled
(XEN) HVM: Hardware Assisted Paging (HAP) detected
(XEN) HVM: HAP page sizes: 4kB, 2MB
(XEN) Brought up 16 CPUs
(XEN) ACPI sleep modes: S3
(XEN) mcheck_poll: Machine check polling timer started.
(XEN) Dom0 has maximum 672 PIRQs
(XEN) Multiple initrd candidates, picking module #1
(XEN) *** LOADING DOMAIN 0 ***
(XEN) elf_parse_binary: phdr: paddr=0x1000000 memsz=0xa5a000
(XEN) elf_parse_binary: phdr: paddr=0x1c00000 memsz=0xd3000
(XEN) elf_parse_binary: phdr: paddr=0x1cd3000 memsz=0x15118
(XEN) elf_parse_binary: phdr: paddr=0x1ce9000 memsz=0x348000
(XEN) elf_parse_binary: memory: 0x1000000 -> 0x2031000
(XEN) elf_xen_parse_note: GUEST_OS = "linux"
(XEN) elf_xen_parse_note: GUEST_VERSION = "2.6"
(XEN) elf_xen_parse_note: XEN_VERSION = "xen-3.0"
(XEN) elf_xen_parse_note: VIRT_BASE = 0xffffffff80000000
(XEN) elf_xen_parse_note: ENTRY = 0xffffffff81ce91f0
(XEN) elf_xen_parse_note: HYPERCALL_PAGE = 0xffffffff81001000
(XEN) elf_xen_parse_note: FEATURES = "!writable_page_tables|pae_pgdir_above_4gb|writable_descriptor_tables|auto_translated_physmap|supervisor_mode_kernel"
(XEN) elf_xen_parse_note: SUPPORTED_FEATURES = 0x90d
(XEN) elf_xen_parse_note: PAE_MODE = "yes"
(XEN) elf_xen_parse_note: LOADER = "generic"
(XEN) elf_xen_parse_note: unknown xen elf note (0xd)
(XEN) elf_xen_parse_note: SUSPEND_CANCEL = 0x1
(XEN) elf_xen_parse_note: MOD_START_PFN = 0x1
(XEN) elf_xen_parse_note: HV_START_LOW = 0xffff800000000000
(XEN) elf_xen_parse_note: PADDR_OFFSET = 0x0
(XEN) elf_xen_addr_calc_check: addresses:
(XEN)     virt_base        = 0xffffffff80000000
(XEN)     elf_paddr_offset = 0x0
(XEN)     virt_offset      = 0xffffffff80000000
(XEN)     virt_kstart      = 0xffffffff81000000
(XEN)     virt_kend        = 0xffffffff82031000
(XEN)     virt_entry       = 0xffffffff81ce91f0
(XEN)     p2m_base         = 0xffffffffffffffff
(XEN)  Xen  kernel: 64-bit, lsb, compat32
(XEN)  Dom0 kernel: 64-bit, PAE, lsb, paddr 0x1000000 -> 0x2031000
(XEN) PHYSICAL MEMORY ARRANGEMENT:
(XEN)  Dom0 alloc.:   000000032c000000->0000000330000000 (216780 pages to be allocated)
(XEN)  Init. ramdisk: 000000033a753000->000000033ff86f2d
(XEN) VIRTUAL MEMORY ARRANGEMENT:
(XEN)  Loaded kernel: ffffffff81000000->ffffffff82031000
(XEN)  Init. ramdisk: 0000000000000000->0000000000000000
(XEN)  Phys-Mach map: ffffffff82031000->ffffffff82224800
(XEN)  Start info:    ffffffff82225000->ffffffff822254b4
(XEN)  Page tables:   ffffffff82226000->ffffffff8223b000
(XEN)  Boot stack:    ffffffff8223b000->ffffffff8223c000
(XEN)  TOTAL:         ffffffff80000000->ffffffff82400000
(XEN)  ENTRY ADDRESS: ffffffff81ce91f0
(XEN) Dom0 has maximum 3 VCPUs
(XEN) elf_load_binary: phdr 0 at 0xffffffff81000000 -> 0xffffffff81a5a000
(XEN) elf_load_binary: phdr 1 at 0xffffffff81c00000 -> 0xffffffff81cd3000
(XEN) elf_load_binary: phdr 2 at 0xffffffff81cd3000 -> 0xffffffff81ce8118
(XEN) elf_load_binary: phdr 3 at 0xffffffff81ce9000 -> 0xffffffff81ece000
(XEN) [VT-D]iommu.c:1435: d0:Hostbridge: skip 0000:00:00.0 map
(XEN) Found masked UR signaling on 0000:00:00.0
(XEN) Found masked UR signaling on 0000:00:01.0
(XEN) Found masked UR signaling on 0000:00:03.0
(XEN) Found masked UR signaling on 0000:00:05.0
(XEN) Found masked UR signaling on 0000:00:07.0
(XEN) Found masked UR signaling on 0000:00:09.0
(XEN) [VT-D]iommu.c:1461: d0:PCI: map 0000:00:13.0
(XEN) [VT-D]iommu.c:1449: d0:PCIe: map 0000:00:14.0
(XEN) Masked VT-d error signaling on 0000:00:14.0
(XEN) [VT-D]iommu.c:1449: d0:PCIe: map 0000:00:14.1
(XEN) [VT-D]iommu.c:1449: d0:PCIe: map 0000:00:14.2
(XEN) [VT-D]iommu.c:1461: d0:PCI: map 0000:00:14.3
(XEN) [VT-D]iommu.c:1449: d0:PCIe: map 0000:00:16.0
(XEN) [VT-D]iommu.c:1449: d0:PCIe: map 0000:00:16.1
(XEN) [VT-D]iommu.c:1449: d0:PCIe: map 0000:00:16.2
(XEN) [VT-D]iommu.c:1449: d0:PCIe: map 0000:00:16.3
(XEN) [VT-D]iommu.c:1449: d0:PCIe: map 0000:00:16.4
(XEN) [VT-D]iommu.c:1449: d0:PCIe: map 0000:00:16.5
(XEN) [VT-D]iommu.c:1449: d0:PCIe: map 0000:00:16.6
(XEN) [VT-D]iommu.c:1449: d0:PCIe: map 0000:00:16.7
(XEN) [VT-D]iommu.c:1461: d0:PCI: map 0000:00:1a.0
(XEN) [VT-D]iommu.c:1461: d0:PCI: map 0000:00:1a.1
(XEN) [VT-D]iommu.c:1461: d0:PCI: map 0000:00:1a.2
(XEN) [VT-D]iommu.c:1461: d0:PCI: map 0000:00:1a.7
(XEN) [VT-D]iommu.c:1461: d0:PCI: map 0000:00:1d.0
(XEN) [VT-D]iommu.c:1461: d0:PCI: map 0000:00:1d.1
(XEN) [VT-D]iommu.c:1461: d0:PCI: map 0000:00:1d.2
(XEN) [VT-D]iommu.c:1461: d0:PCI: map 0000:00:1d.7
(XEN) [VT-D]iommu.c:1461: d0:PCI: map 0000:00:1f.0
(XEN) [VT-D]iommu.c:1461: d0:PCI: map 0000:00:1f.2
(XEN) [VT-D]iommu.c:1461: d0:PCI: map 0000:00:1f.3
(XEN) [VT-D]iommu.c:1461: d0:PCI: map 0000:00:1f.5
(XEN) [VT-D]iommu.c:1461: d0:PCI: map 0000:01:01.0
(XEN) [VT-D]iommu.c:1449: d0:PCIe: map 0000:02:00.0
(XEN) [VT-D]iommu.c:1449: d0:PCIe: map 0000:04:00.1
(XEN) [VT-D]iommu.c:1449: d0:PCIe: map 0000:04:00.3
(XEN) [VT-D]iommu.c:1461: d0:PCI: map 0000:06:01.0
(XEN) [VT-D]iommu.c:1461: d0:PCI: map 0000:06:01.1
(XEN) [VT-D]iommu.c:1449: d0:PCIe: map 0000:0a:00.0
(XEN) [VT-D]iommu.c:1449: d0:PCIe: map 0000:0a:00.1
(XEN) [VT-D]iommu.c:1435: d0:Hostbridge: skip 0000:fe:00.0 map
(XEN) [VT-D]iommu.c:1435: d0:Hostbridge: skip 0000:fe:00.1 map
(XEN) [VT-D]iommu.c:1435: d0:Hostbridge: skip 0000:fe:02.0 map
(XEN) [VT-D]iommu.c:1435: d0:Hostbridge: skip 0000:fe:02.1 map
(XEN) [VT-D]iommu.c:1435: d0:Hostbridge: skip 0000:fe:02.4 map
(XEN) [VT-D]iommu.c:1435: d0:Hostbridge: skip 0000:fe:02.5 map
(XEN) [VT-D]iommu.c:1435: d0:Hostbridge: skip 0000:fe:03.0 map
(XEN) [VT-D]iommu.c:1435: d0:Hostbridge: skip 0000:fe:03.1 map
(XEN) [VT-D]iommu.c:1435: d0:Hostbridge: skip 0000:fe:03.2 map
(XEN) [VT-D]iommu.c:1435: d0:Hostbridge: skip 0000:fe:03.4 map
(XEN) [VT-D]iommu.c:1435: d0:Hostbridge: skip 0000:fe:04.0 map
(XEN) [VT-D]iommu.c:1435: d0:Hostbridge: skip 0000:fe:04.1 map
(XEN) [VT-D]iommu.c:1435: d0:Hostbridge: skip 0000:fe:04.2 map
(XEN) [VT-D]iommu.c:1435: d0:Hostbridge: skip 0000:fe:04.3 map
(XEN) [VT-D]iommu.c:1435: d0:Hostbridge: skip 0000:fe:05.0 map
(XEN) [VT-D]iommu.c:1435: d0:Hostbridge: skip 0000:fe:05.1 map
(XEN) [VT-D]iommu.c:1435: d0:Hostbridge: skip 0000:fe:05.2 map
(XEN) [VT-D]iommu.c:1435: d0:Hostbridge: skip 0000:fe:05.3 map
(XEN) [VT-D]iommu.c:1435: d0:Hostbridge: skip 0000:fe:06.0 map
(XEN) [VT-D]iommu.c:1435: d0:Hostbridge: skip 0000:fe:06.1 map
(XEN) [VT-D]iommu.c:1435: d0:Hostbridge: skip 0000:fe:06.2 map
(XEN) [VT-D]iommu.c:1435: d0:Hostbridge: skip 0000:fe:06.3 map
(XEN) [VT-D]iommu.c:1435: d0:Hostbridge: skip 0000:ff:00.0 map
(XEN) [VT-D]iommu.c:1435: d0:Hostbridge: skip 0000:ff:00.1 map
(XEN) [VT-D]iommu.c:1435: d0:Hostbridge: skip 0000:ff:02.0 map
(XEN) [VT-D]iommu.c:1435: d0:Hostbridge: skip 0000:ff:02.1 map
(XEN) [VT-D]iommu.c:1435: d0:Hostbridge: skip 0000:ff:02.4 map
(XEN) [VT-D]iommu.c:1435: d0:Hostbridge: skip 0000:ff:02.5 map
(XEN) [VT-D]iommu.c:1435: d0:Hostbridge: skip 0000:ff:03.0 map
(XEN) [VT-D]iommu.c:1435: d0:Hostbridge: skip 0000:ff:03.1 map
(XEN) [VT-D]iommu.c:1435: d0:Hostbridge: skip 0000:ff:03.2 map
(XEN) [VT-D]iommu.c:1435: d0:Hostbridge: skip 0000:ff:03.4 map
(XEN) [VT-D]iommu.c:1435: d0:Hostbridge: skip 0000:ff:04.0 map
(XEN) [VT-D]iommu.c:1435: d0:Hostbridge: skip 0000:ff:04.1 map
(XEN) [VT-D]iommu.c:1435: d0:Hostbridge: skip 0000:ff:04.2 map
(XEN) [VT-D]iommu.c:1435: d0:Hostbridge: skip 0000:ff:04.3 map
(XEN) [VT-D]iommu.c:1435: d0:Hostbridge: skip 0000:ff:05.0 map
(XEN) [VT-D]iommu.c:1435: d0:Hostbridge: skip 0000:ff:05.1 map
(XEN) [VT-D]iommu.c:1435: d0:Hostbridge: skip 0000:ff:05.2 map
(XEN) [VT-D]iommu.c:1435: d0:Hostbridge: skip 0000:ff:05.3 map
(XEN) [VT-D]iommu.c:1435: d0:Hostbridge: skip 0000:ff:06.0 map
(XEN) [VT-D]iommu.c:1435: d0:Hostbridge: skip 0000:ff:06.1 map
(XEN) [VT-D]iommu.c:1435: d0:Hostbridge: skip 0000:ff:06.2 map
(XEN) [VT-D]iommu.c:1435: d0:Hostbridge: skip 0000:ff:06.3 map
(XEN) [VT-D]iommu.c:730: iommu_enable_translation: iommu->reg = ffff82c000201000
(XEN) Scrubbing Free RAM on 2 nodes using 4 CPUs
(XEN) ..........................done.
(XEN) Initial low memory virq threshold set at 0x4000 pages.
(XEN) Std. Loglevel: All
(XEN) Guest Loglevel: All
(XEN) Xen is relinquishing VGA console.
(XEN) *** Serial input -> DOM0 (type 'CTRL-a' three times to switch input to Xen)
(XEN) Freed 292kB init memory.
(XEN) traps.c:2654:d0v0 Domain attempted WRMSR 00000000c0000081 from 0xe023e00800000000 to 0x0023001000000000.
(XEN) traps.c:2654:d0v0 Domain attempted WRMSR 00000000c0000082 from 0xffff82d0802e3000 to 0xffffffff8175c390.
(XEN) traps.c:2654:d0v0 Domain attempted WRMSR 00000000c0000083 from 0xffff82d0802e3080 to 0xffffffff8175e6c0.
(XEN) traps.c:2654:d0v0 Domain attempted WRMSR 0000000000000174 from 0x000000000000e008 to 0x0000000000000010.
(XEN) traps.c:2654:d0v0 Domain attempted WRMSR 0000000000000175 from 0xffff82d0802e7fc0 to 0x0000000000000000.
(XEN) traps.c:2654:d0v0 Domain attempted WRMSR 0000000000000176 from 0xffff82d080230960 to 0xffffffff8175e9a0.
(XEN) traps.c:2654:d0v0 Domain attempted WRMSR 00000000c0000083 from 0xffff82d0802e3080 to 0xffffffff8175ebe0.
(XEN) traps.c:2654:d0v0 Domain attempted WRMSR 00000000c0000084 from 0x0000000000074700 to 0x0000000000047700.
(XEN) traps.c:2654:d0v1 Domain attempted WRMSR 00000000c0000081 from 0xe023e00800000000 to 0x0023001000000000.
(XEN) traps.c:2654:d0v1 Domain attempted WRMSR 00000000c0000082 from 0xffff830338f23000 to 0xffffffff8175c390.
(XEN) traps.c:2654:d0v1 Domain attempted WRMSR 00000000c0000083 from 0xffff830338f23080 to 0xffffffff8175e6c0.
(XEN) traps.c:2654:d0v1 Domain attempted WRMSR 0000000000000174 from 0x000000000000e008 to 0x0000000000000010.
(XEN) traps.c:2654:d0v1 Domain attempted WRMSR 0000000000000175 from 0xffff830338f27fc0 to 0x0000000000000000.
(XEN) traps.c:2654:d0v1 Domain attempted WRMSR 0000000000000176 from 0xffff82d080230960 to 0xffffffff8175e9a0.
(XEN) traps.c:2654:d0v1 Domain attempted WRMSR 00000000c0000083 from 0xffff830338f23080 to 0xffffffff8175ebe0.
(XEN) traps.c:2654:d0v1 Domain attempted WRMSR 00000000c0000084 from 0x0000000000074700 to 0x0000000000047700.
(XEN) traps.c:2654:d0v2 Domain attempted WRMSR 00000000c0000081 from 0xe023e00800000000 to 0x0023001000000000.
(XEN) traps.c:2654:d0v2 Domain attempted WRMSR 00000000c0000082 from 0xffff830338ed3000 to 0xffffffff8175c390.
(XEN) traps.c:2654:d0v2 Domain attempted WRMSR 00000000c0000083 from 0xffff830338ed3080 to 0xffffffff8175e6c0.
(XEN) traps.c:2654:d0v2 Domain attempted WRMSR 0000000000000174 from 0x000000000000e008 to 0x0000000000000010.
(XEN) traps.c:2654:d0v2 Domain attempted WRMSR 0000000000000175 from 0xffff830338ed7fc0 to 0x0000000000000000.
(XEN) traps.c:2654:d0v2 Domain attempted WRMSR 0000000000000176 from 0xffff82d080230960 to 0xffffffff8175e9a0.
(XEN) traps.c:2654:d0v2 Domain attempted WRMSR 00000000c0000083 from 0xffff830338ed3080 to 0xffffffff8175ebe0.
(XEN) traps.c:2654:d0v2 Domain attempted WRMSR 00000000c0000084 from 0x0000000000074700 to 0x0000000000047700.
(XEN) Found masked UR signaling on 0000:00:00.0
(XEN) PCI add device 0000:00:00.0
(XEN) Found masked UR signaling on 0000:00:01.0
(XEN) PCI add device 0000:00:01.0
(XEN) Found masked UR signaling on 0000:00:03.0
(XEN) PCI add device 0000:00:03.0
(XEN) Found masked UR signaling on 0000:00:05.0
(XEN) PCI add device 0000:00:05.0
(XEN) Found masked UR signaling on 0000:00:07.0
(XEN) PCI add device 0000:00:07.0
(XEN) Found masked UR signaling on 0000:00:09.0
(XEN) PCI add device 0000:00:09.0
(XEN) PCI add device 0000:00:13.0
(XEN) Masked VT-d error signaling on 0000:00:14.0
(XEN) PCI add device 0000:00:14.0
(XEN) PCI add device 0000:00:14.1
(XEN) PCI add device 0000:00:14.2
(XEN) PCI add device 0000:00:14.3
(XEN) PCI add device 0000:00:16.0
(XEN) PCI add device 0000:00:16.1
(XEN) PCI add device 0000:00:16.2
(XEN) PCI add device 0000:00:16.3
(XEN) PCI add device 0000:00:16.4
(XEN) PCI add device 0000:00:16.5
(XEN) PCI add device 0000:00:16.6
(XEN) PCI add device 0000:00:16.7
(XEN) PCI add device 0000:00:1a.0
(XEN) PCI add device 0000:00:1a.1
(XEN) PCI add device 0000:00:1a.2
(XEN) PCI add device 0000:00:1a.7
(XEN) PCI add device 0000:00:1c.0
(XEN) PCI add device 0000:00:1c.4
(XEN) PCI add device 0000:00:1d.0
(XEN) PCI add device 0000:00:1d.1
(XEN) PCI add device 0000:00:1d.2
(XEN) PCI add device 0000:00:1d.7
(XEN) PCI add device 0000:00:1e.0
(XEN) PCI add device 0000:00:1f.0
(XEN) PCI add device 0000:00:1f.2
(XEN) PCI add device 0000:00:1f.3
(XEN) PCI add device 0000:00:1f.5
(XEN) PCI add device 0000:0a:00.0
(XEN) PCI add device 0000:0a:00.1
(XEN) PCI add device 0000:04:00.0
(XEN) PCI add device 0000:04:00.1
(XEN) PCI add device 0000:04:00.2
(XEN) PCI add device 0000:04:00.3
(XEN) PCI add device 0000:06:01.0
(XEN) PCI add device 0000:06:01.1
(XEN) PCI add device 0000:02:00.0
(XEN) PCI add device 0000:01:01.0
(XEN) PCI add device 0000:fe:00.0
(XEN) PCI add device 0000:fe:00.1
(XEN) PCI add device 0000:fe:02.0
(XEN) PCI add device 0000:fe:02.1
(XEN) PCI add device 0000:fe:02.4
(XEN) PCI add device 0000:fe:02.5
(XEN) PCI add device 0000:fe:03.0
(XEN) PCI add device 0000:fe:03.1
(XEN) PCI add device 0000:fe:03.2
(XEN) PCI add device 0000:fe:03.4
(XEN) PCI add device 0000:fe:04.0
(XEN) PCI add device 0000:fe:04.1
(XEN) PCI add device 0000:fe:04.2
(XEN) PCI add device 0000:fe:04.3
(XEN) PCI add device 0000:fe:05.0
(XEN) PCI add device 0000:fe:05.1
(XEN) PCI add device 0000:fe:05.2
(XEN) PCI add device 0000:fe:05.3
(XEN) PCI add device 0000:fe:06.0
(XEN) PCI add device 0000:fe:06.1
(XEN) PCI add device 0000:fe:06.2
(XEN) PCI add device 0000:fe:06.3
(XEN) PCI add device 0000:ff:00.0
(XEN) PCI add device 0000:ff:00.1
(XEN) PCI add device 0000:ff:02.0
(XEN) PCI add device 0000:ff:02.1
(XEN) PCI add device 0000:ff:02.4
(XEN) PCI add device 0000:ff:02.5
(XEN) PCI add device 0000:ff:03.0
(XEN) PCI add device 0000:ff:03.1
(XEN) PCI add device 0000:ff:03.2
(XEN) PCI add device 0000:ff:03.4
(XEN) PCI add device 0000:ff:04.0
(XEN) PCI add device 0000:ff:04.1
(XEN) PCI add device 0000:ff:04.2
(XEN) PCI add device 0000:ff:04.3
(XEN) PCI add device 0000:ff:05.0
(XEN) PCI add device 0000:ff:05.1
(XEN) PCI add device 0000:ff:05.2
(XEN) PCI add device 0000:ff:05.3
(XEN) PCI add device 0000:ff:06.0
(XEN) PCI add device 0000:ff:06.1
(XEN) PCI add device 0000:ff:06.2
(XEN) PCI add device 0000:ff:06.3
(XEN) Xen WARN at msi.c:155
(XEN) ----[ Xen-4.6-unstable  x86_64  debug=y  Not tainted ]----
(XEN) CPU:    3
(XEN) RIP:    e008:[<ffff82d08016cd23>] msix_memory_decoded+0x4a/0x62
(XEN) RFLAGS: 0000000000010046   CONTEXT: hypervisor (d0v1)
(XEN) rax: 0000000000000000   rbx: ffff830338f38ea0   rcx: 0000000000000000
(XEN) rdx: ffff830338ef8000   rsi: 0000000000000000   rdi: ffff82d080294f20
(XEN) rbp: ffff830338effc78   rsp: ffff830338effc68   r8:  0000000000000002
(XEN) r9:  0000000000000000   r10: ffff82d08024a1e0   r11: 0000000000000202
(XEN) r12: ffff830338effcd8   r13: ffff8303276c2d50   r14: ffff830338e06824
(XEN) r15: ffff830338e06800   cr0: 000000008005003b   cr4: 00000000000026f0
(XEN) cr3: 000000032dc0f000   cr2: 0000000000000000
(XEN) ds: 0000   es: 0000   fs: 0000   gs: 0000   ss: e010   cs: e008
(XEN) Xen stack trace from rsp=ffff830338effc68:
(XEN)    ffff830338effc98 ffff82cfffd74000 ffff830338effcc8 ffff82d08016d3bc
(XEN)    ffff8303276c2c20 ffff830338eda6b0 ffff830338effcc8 ffff830338e06800
(XEN)    ffff8303276c2d50 ffff830338effcd8 ffff830338e06824 ffff830338e06800
(XEN)    ffff830338effd08 ffff82d08016f1fe 00000000fee00000 0000000000004059
(XEN)    ffff830338effd18 ffff8303276c2d50 ffff830338e06800 0000000000000068
(XEN)    ffff830338effd28 ffff82d08016f261 ffff830330769000 0000000000000298
(XEN)    ffff830338effdc8 ffff82d080173281 ffff8303276c2d00 0000000000000298
(XEN)    ffff830338effd90 ffff830338f38ef0 ffff830338f38ef0 0000000000000246
(XEN)    0000029801e06800 ffff8303307690d8 ffff830338effe98 ffff830300000000
(XEN)    ffff8303276c2d50 ffff8303276c2d00 ffff830338effdc8 ffff830330769000
(XEN)    00000000fffffffd 0000000000000000 ffff830338effe98 ffff830338effe70
(XEN)    ffff830338effe48 ffff82d080185924 ffff830338efff18 ffffffff815db58e
(XEN)    ffff830338effe98 ffff8303307690b8 ffff830300000298 ffff82d080192bda
(XEN)    000000680000f800 ffff830338effe74 ffff8300bf2f6000 000000000000000d
(XEN)    ffff8800379bfa78 ffff830330769000 ffff880037556540 0000000000000000
(XEN)    ffff830338effef8 ffff82d080185fa3 0000000700000000 0000000438effeb8
(XEN)    0000000000007ff0 ffffffffffffffff 000000b000000000 0000000000000000
(XEN)    00000000fb8d8000 ffff830330769000 00000068b0000000 ffff820000000000
(XEN)    00000000fb8d8000 0000000000000000 ffffffffffffffff 0000000000000000
(XEN)    0000000000000282 ffff8300bf2f6000 ffff880037c0a000 0000000000000001
(XEN) Xen call trace:
(XEN)    [<ffff82d08016cd23>] msix_memory_decoded+0x4a/0x62
(XEN)    [<ffff82d08016d3bc>] write_msi_msg+0x153/0x194
(XEN)    [<ffff82d08016f1fe>] __setup_msi_irq+0x41/0x66
(XEN)    [<ffff82d08016f261>] setup_msi_irq+0x3e/0x49
(XEN)    [<ffff82d080173281>] map_domain_pirq+0x459/0x628
(XEN)    [<ffff82d080185924>] physdev_map_pirq+0x509/0x5d2
(XEN)    [<ffff82d080185fa3>] do_physdev_op+0x5b6/0x112b
(XEN)    [<ffff82d0802307cb>] syscall_enter+0xeb/0x145
(XEN) 
(XEN) mm.c:803: d0: Forcing read-only access to MFN fed00
(XEN) d0 attempted to change d0v2's CR4 flags 00002660 -> 00002760
(XEN) d0 attempted to change d0v0's CR4 flags 00002660 -> 00002760
(XEN) d0 attempted to change d0v1's CR4 flags 00002660 -> 00002760

Re: [PATCH v2 2/4] x86/MSI-X: access MSI-X table only after having enabled MSI-X

[Jan Beulich]

>>> On 15.04.15 at 19:41, <konrad.wilk@oracle.com> wrote:
> On Mon, Apr 13, 2015 at 10:05:14AM +0100, Jan Beulich wrote:
>> You mentioning XSA-120 and its addendum - are these requirements
>> for the problem to be seen? I admit I may have tested a PV guest
>> only with an SR-IOV VF (and only a HVM guest also with an "ordinary"
>> device), but I'd like to be clear about the validity of the connection.
> 
> No. I just tried with v4.0-rc5 (and then also v4.0) and just 
> using SR-IOV to make this simpler.

Good.

> With staging  + two of your patches:
> a10cc68 TODO: drop //temp-s
> 1b8721c x86/MSI-X: be more careful during teardown
> 
> When trying to enable SR-IOV I get this error:

Okay, this definitely works for me, albeit I know I had to do
adjustments to avoid running into the (debug) warning you've
hit. But (looking at the call stack) it surely would be a mistake to
set up an MSI-X IRQ on the device without first enabling MSI-X
on the it (i.e. the error returned could be considered legitimate).
While we may want the hypervisor to cope with this (by enabling
MSI-X on this path, which I'd have to add to that patch), is this
hypervisor change perhaps uncovering a pv-ops kernel issue (in
that other than what drivers/pci/msi.c does as of the commit
mentioned in the description of that second patch some Xen-
specific path fails to enable MSI-X before setting up any of the
entries)?

Jan

Re: [PATCH v2 2/4] x86/MSI-X: access MSI-X table only after having enabled MSI-X

[Konrad Rzeszutek Wilk]

On Thu, Apr 16, 2015 at 08:43:30AM +0100, Jan Beulich wrote:
> >>> On 15.04.15 at 19:41, <konrad.wilk@oracle.com> wrote:
> > On Mon, Apr 13, 2015 at 10:05:14AM +0100, Jan Beulich wrote:
> >> You mentioning XSA-120 and its addendum - are these requirements
> >> for the problem to be seen? I admit I may have tested a PV guest
> >> only with an SR-IOV VF (and only a HVM guest also with an "ordinary"
> >> device), but I'd like to be clear about the validity of the connection.
> > 
> > No. I just tried with v4.0-rc5 (and then also v4.0) and just 
> > using SR-IOV to make this simpler.
> 
> Good.
> 
> > With staging  + two of your patches:
> > a10cc68 TODO: drop //temp-s
> > 1b8721c x86/MSI-X: be more careful during teardown
> > 
> > When trying to enable SR-IOV I get this error:
> 
> Okay, this definitely works for me, albeit I know I had to do
> adjustments to avoid running into the (debug) warning you've
> hit. But (looking at the call stack) it surely would be a mistake to
> set up an MSI-X IRQ on the device without first enabling MSI-X
> on the it (i.e. the error returned could be considered legitimate).
> While we may want the hypervisor to cope with this (by enabling
> MSI-X on this path, which I'd have to add to that patch), is this
> hypervisor change perhaps uncovering a pv-ops kernel issue (in
> that other than what drivers/pci/msi.c does as of the commit
> mentioned in the description of that second patch some Xen-
> specific path fails to enable MSI-X before setting up any of the
> entries)?

Everything is possible :-)

What kernel are you using? Or better yet - what branch/tree could
one find it at?
> 
> Jan
> 

Re: [PATCH v2 2/4] x86/MSI-X: access MSI-X table only after having enabled MSI-X

[Jan Beulich]

>>> On 16.04.15 at 20:21, <konrad.wilk@oracle.com> wrote:
> What kernel are you using? Or better yet - what branch/tree could
> one find it at?

As always, I'm personally using my own kernel, not in any branch/tree.
But for all purposes here, our "Kernel-of-the-Day" should be good
enough, i.e. the stuff under
http://download.opensuse.org/repositories/Kernel:/HEAD/standard/.

But anyway I hope to get you a replacement patch later today for
the one I had previously handed you with some debugging left in.

Jan

Re: [PATCH v2 2/4] x86/MSI-X: access MSI-X table only after having enabled MSI-X

[Jan Beulich]

>>> On 15.04.15 at 19:41, <konrad.wilk@oracle.com> wrote:
> On Mon, Apr 13, 2015 at 10:05:14AM +0100, Jan Beulich wrote:
>> >>> On 10.04.15 at 22:02, <konrad.wilk@oracle.com> wrote:
>> > On Wed, Mar 25, 2015 at 04:39:49PM +0000, Jan Beulich wrote:
>> >> As done in Linux by f598282f51 ("PCI: Fix the NIU MSI-X problem in a
>> >> better way") and its broken predecessor, make sure we don't access the
>> >> MSI-X table without having enabled MSI-X first, using the mask-all flag
>> >> instead to prevent interrupts from occurring.
>> > 
>> > This causes an regression with an Linux guest that has the XSA120 + XSA120
>> > addendum with PV guests (hadn't tried yet HVM).
>> 
>> You mentioning XSA-120 and its addendum - are these requirements
>> for the problem to be seen? I admit I may have tested a PV guest
>> only with an SR-IOV VF (and only a HVM guest also with an "ordinary"
>> device), but I'd like to be clear about the validity of the connection.
> 
> No. I just tried with v4.0-rc5 (and then also v4.0) and just 
> using SR-IOV to make this simpler.
> 
> With staging  + two of your patches:
> a10cc68 TODO: drop //temp-s
> 1b8721c x86/MSI-X: be more careful during teardown
> 
> When trying to enable SR-IOV I get this error:
> 
> failed to echo 1 > 
> /sys/devices/pci0000:00/0000:00:01.0/0000:0a:00.0/sriov_numvfs, rc: 1
> (hadn't tried just passing in an HVM guest).
> 
> Attached is the 'xl dmesg'.

Could you replace the patch I handed you earlier on by this one
and try again? I actually was able to determine that I did try a
(SUSE) PV guest without seeing an issue. I just now tried again,
and I don't see either of the two debug warnings. So quite clear
any indication towards a pvops problem.

Jan

x86/MSI-X: access MSI-X table only after having enabled MSI-X

As done in Linux by f598282f51 ("PCI: Fix the NIU MSI-X problem in a
better way") and its broken predecessor, make sure we don't access the
MSI-X table without having enabled MSI-X first, using the mask-all flag
instead to prevent interrupts from occurring.

Signed-off-by: Jan Beulich <jbeulich@suse.com>
---
v3: temporarily enable MSI-X in setup_msi_irq() if not already enabled

--- unstable.orig/xen/arch/x86/msi.c
+++ unstable/xen/arch/x86/msi.c
@@ -142,6 +142,21 @@ static bool_t memory_decoded(const struc
               PCI_COMMAND_MEMORY);
 }
 
+static bool_t msix_memory_decoded(const struct pci_dev *dev, unsigned int pos)
+{
+    u16 control = pci_conf_read16(dev->seg, dev->bus, PCI_SLOT(dev->devfn),
+                                  PCI_FUNC(dev->devfn), msix_control_reg(pos));
+
+    if ( !(control & PCI_MSIX_FLAGS_ENABLE) )
+{//temp
+ static bool_t warned;
+ WARN_ON(!test_and_set_bool(warned));
+        return 0;
+}
+
+    return memory_decoded(dev);
+}
+
 /*
  * MSI message composition
  */
@@ -219,7 +234,8 @@ static bool_t read_msi_msg(struct msi_de
         void __iomem *base;
         base = entry->mask_base;
 
-        if ( unlikely(!memory_decoded(entry->dev)) )
+        if ( unlikely(!msix_memory_decoded(entry->dev,
+                                           entry->msi_attrib.pos)) )
             return 0;
         msg->address_lo = readl(base + PCI_MSIX_ENTRY_LOWER_ADDR_OFFSET);
         msg->address_hi = readl(base + PCI_MSIX_ENTRY_UPPER_ADDR_OFFSET);
@@ -285,7 +301,8 @@ static int write_msi_msg(struct msi_desc
         void __iomem *base;
         base = entry->mask_base;
 
-        if ( unlikely(!memory_decoded(entry->dev)) )
+        if ( unlikely(!msix_memory_decoded(entry->dev,
+                                           entry->msi_attrib.pos)) )
             return -ENXIO;
         writel(msg->address_lo,
                base + PCI_MSIX_ENTRY_LOWER_ADDR_OFFSET);
@@ -379,7 +396,7 @@ static bool_t msi_set_mask_bit(struct ir
 {
     struct msi_desc *entry = desc->msi_desc;
     struct pci_dev *pdev;
-    u16 seg;
+    u16 seg, control;
     u8 bus, slot, func;
 
     ASSERT(spin_is_locked(&desc->lock));
@@ -401,35 +418,38 @@ static bool_t msi_set_mask_bit(struct ir
         }
         break;
     case PCI_CAP_ID_MSIX:
+        control = pci_conf_read16(seg, bus, slot, func,
+                                  msix_control_reg(entry->msi_attrib.pos));
+        if ( unlikely(!(control & PCI_MSIX_FLAGS_ENABLE)) )
+            pci_conf_write16(seg, bus, slot, func,
+                             msix_control_reg(entry->msi_attrib.pos),
+                             control | (PCI_MSIX_FLAGS_ENABLE |
+                                        PCI_MSIX_FLAGS_MASKALL));
         if ( likely(memory_decoded(pdev)) )
         {
             writel(flag, entry->mask_base + PCI_MSIX_ENTRY_VECTOR_CTRL_OFFSET);
             readl(entry->mask_base + PCI_MSIX_ENTRY_VECTOR_CTRL_OFFSET);
-            break;
+            if ( likely(control & PCI_MSIX_FLAGS_ENABLE) )
+                break;
+            flag = 1;
         }
-        if ( flag )
+        else if ( flag && !(control & PCI_MSIX_FLAGS_MASKALL) )
         {
-            u16 control;
             domid_t domid = pdev->domain->domain_id;
 
-            control = pci_conf_read16(seg, bus, slot, func,
-                                      msix_control_reg(entry->msi_attrib.pos));
-            if ( control & PCI_MSIX_FLAGS_MASKALL )
-                break;
-            pci_conf_write16(seg, bus, slot, func,
-                             msix_control_reg(entry->msi_attrib.pos),
-                             control | PCI_MSIX_FLAGS_MASKALL);
+            control |= PCI_MSIX_FLAGS_MASKALL;
             if ( pdev->msix->warned != domid )
             {
                 pdev->msix->warned = domid;
                 printk(XENLOG_G_WARNING
-                       "cannot mask IRQ %d: masked MSI-X on Dom%d's %04x:%02x:%02x.%u\n",
+                       "cannot mask IRQ %d: masking MSI-X on Dom%d's %04x:%02x:%02x.%u\n",
                        desc->irq, domid, pdev->seg, pdev->bus,
                        PCI_SLOT(pdev->devfn), PCI_FUNC(pdev->devfn));
             }
-            break;
         }
-        /* fall through */
+        pci_conf_write16(seg, bus, slot, func,
+                         msix_control_reg(entry->msi_attrib.pos), control);
+        return flag;
     default:
         return 0;
     }
@@ -454,7 +474,8 @@ static int msi_get_mask_bit(const struct
                                 entry->msi.mpos) >>
                 entry->msi_attrib.entry_nr) & 1;
     case PCI_CAP_ID_MSIX:
-        if ( unlikely(!memory_decoded(entry->dev)) )
+        if ( unlikely(!msix_memory_decoded(entry->dev,
+                                           entry->msi_attrib.pos)) )
             break;
         return readl(entry->mask_base + PCI_MSIX_ENTRY_VECTOR_CTRL_OFFSET) & 1;
     }
@@ -542,9 +563,35 @@ static struct msi_desc *alloc_msi_entry(
 
 int setup_msi_irq(struct irq_desc *desc, struct msi_desc *msidesc)
 {
-    return __setup_msi_irq(desc, msidesc,
-                           msi_maskable_irq(msidesc) ? &pci_msi_maskable
-                                                     : &pci_msi_nonmaskable);
+    const struct pci_dev *pdev = msidesc->dev;
+    unsigned int cpos = msix_control_reg(msidesc->msi_attrib.pos);
+    u16 control = ~0;
+    int rc;
+
+    if ( msidesc->msi_attrib.type == PCI_CAP_ID_MSIX )
+    {
+        control = pci_conf_read16(pdev->seg, pdev->bus, PCI_SLOT(pdev->devfn),
+                                  PCI_FUNC(pdev->devfn), cpos);
+        if ( !(control & PCI_MSIX_FLAGS_ENABLE) )
+{//temp
+ static bool_t warned;
+ WARN_ON(!test_and_set_bool(warned));
+            pci_conf_write16(pdev->seg, pdev->bus, PCI_SLOT(pdev->devfn),
+                             PCI_FUNC(pdev->devfn), cpos,
+                             control | (PCI_MSIX_FLAGS_ENABLE |
+                                        PCI_MSIX_FLAGS_MASKALL));
+}
+    }
+
+    rc = __setup_msi_irq(desc, msidesc,
+                         msi_maskable_irq(msidesc) ? &pci_msi_maskable
+                                                   : &pci_msi_nonmaskable);
+
+    if ( !(control & PCI_MSIX_FLAGS_ENABLE) )
+        pci_conf_write16(pdev->seg, pdev->bus, PCI_SLOT(pdev->devfn),
+                         PCI_FUNC(pdev->devfn), cpos, control);
+
+    return rc;
 }
 
 int __setup_msi_irq(struct irq_desc *desc, struct msi_desc *msidesc,
@@ -775,16 +822,32 @@ static int msix_capability_init(struct p
 
     pos = pci_find_cap_offset(seg, bus, slot, func, PCI_CAP_ID_MSIX);
     control = pci_conf_read16(seg, bus, slot, func, msix_control_reg(pos));
-    msix_set_enable(dev, 0);/* Ensure msix is disabled as I set it up */
+    /*
+     * Ensure MSI-X interrupts are masked during setup. Some devices require
+     * MSI-X to be enabled before we can touch the MSI-X registers. We need
+     * to mask all the vectors to prevent interrupts coming in before they're
+     * fully set up.
+     */
+    pci_conf_write16(seg, bus, slot, func, msix_control_reg(pos),
+                     control | (PCI_MSIX_FLAGS_ENABLE |
+                                PCI_MSIX_FLAGS_MASKALL));
 
     if ( unlikely(!memory_decoded(dev)) )
+    {
+        pci_conf_write16(seg, bus, slot, func, msix_control_reg(pos),
+                         control & ~PCI_MSIX_FLAGS_ENABLE);
         return -ENXIO;
+    }
 
     if ( desc )
     {
         entry = alloc_msi_entry(1);
         if ( !entry )
+        {
+            pci_conf_write16(seg, bus, slot, func, msix_control_reg(pos),
+                             control & ~PCI_MSIX_FLAGS_ENABLE);
             return -ENOMEM;
+        }
         ASSERT(msi);
     }
 
@@ -815,6 +878,8 @@ static int msix_capability_init(struct p
     {
         if ( !msi || !msi->table_base )
         {
+            pci_conf_write16(seg, bus, slot, func, msix_control_reg(pos),
+                             control & ~PCI_MSIX_FLAGS_ENABLE);
             xfree(entry);
             return -ENXIO;
         }
@@ -857,6 +922,8 @@ static int msix_capability_init(struct p
 
         if ( idx < 0 )
         {
+            pci_conf_write16(seg, bus, slot, func, msix_control_reg(pos),
+                             control & ~PCI_MSIX_FLAGS_ENABLE);
             xfree(entry);
             return idx;
         }
@@ -912,8 +979,7 @@ static int msix_capability_init(struct p
     ++msix->used_entries;
 
     /* Restore MSI-X enabled bits */
-    pci_conf_write16(seg, bus, slot, func, msix_control_reg(pos),
-                     control & ~PCI_MSIX_FLAGS_MASKALL);
+    pci_conf_write16(seg, bus, slot, func, msix_control_reg(pos), control);
 
     return 0;
 }
@@ -1062,7 +1128,10 @@ static void __pci_disable_msix(struct ms
 
     pos = pci_find_cap_offset(seg, bus, slot, func, PCI_CAP_ID_MSIX);
     control = pci_conf_read16(seg, bus, slot, func, msix_control_reg(pos));
-    msix_set_enable(dev, 0);
+    if ( unlikely(!(control & PCI_MSIX_FLAGS_ENABLE)) )
+        pci_conf_write16(seg, bus, slot, func, msix_control_reg(pos),
+                         control | (PCI_MSIX_FLAGS_ENABLE |
+                                    PCI_MSIX_FLAGS_MASKALL));
 
     BUG_ON(list_empty(&dev->msi_list));
 
@@ -1188,6 +1257,8 @@ int pci_restore_msi_state(struct pci_dev
     list_for_each_entry_safe( entry, tmp, &pdev->msi_list, list )
     {
         unsigned int i = 0, nr = 1;
+        u16 control = 0;
+        u8 slot = PCI_SLOT(pdev->devfn), func = PCI_FUNC(pdev->devfn);
 
         irq = entry->irq;
         desc = &irq_desc[irq];
@@ -1214,10 +1285,18 @@ int pci_restore_msi_state(struct pci_dev
         }
         else if ( entry->msi_attrib.type == PCI_CAP_ID_MSIX )
         {
-            msix_set_enable(pdev, 0);
+            control = pci_conf_read16(pdev->seg, pdev->bus, slot, func,
+                                      msix_control_reg(entry->msi_attrib.pos));
+            pci_conf_write16(pdev->seg, pdev->bus, slot, func,
+                             msix_control_reg(entry->msi_attrib.pos),
+                             control | (PCI_MSIX_FLAGS_ENABLE |
+                                        PCI_MSIX_FLAGS_MASKALL));
             if ( unlikely(!memory_decoded(pdev)) )
             {
                 spin_unlock_irqrestore(&desc->lock, flags);
+                pci_conf_write16(pdev->seg, pdev->bus, slot, func,
+                                 msix_control_reg(entry->msi_attrib.pos),
+                                 control & ~PCI_MSIX_FLAGS_ENABLE);
                 return -ENXIO;
             }
         }
@@ -1246,11 +1325,9 @@ int pci_restore_msi_state(struct pci_dev
         if ( entry->msi_attrib.type == PCI_CAP_ID_MSI )
         {
             unsigned int cpos = msi_control_reg(entry->msi_attrib.pos);
-            u16 control = pci_conf_read16(pdev->seg, pdev->bus,
-                                          PCI_SLOT(pdev->devfn),
-                                          PCI_FUNC(pdev->devfn), cpos);
 
-            control &= ~PCI_MSI_FLAGS_QSIZE;
+            control = pci_conf_read16(pdev->seg, pdev->bus, slot, func, cpos) &
+                      ~PCI_MSI_FLAGS_QSIZE;
             multi_msi_enable(control, entry->msi.nvec);
             pci_conf_write16(pdev->seg, pdev->bus, PCI_SLOT(pdev->devfn),
                              PCI_FUNC(pdev->devfn), cpos, control);
@@ -1258,7 +1335,9 @@ int pci_restore_msi_state(struct pci_dev
             msi_set_enable(pdev, 1);
         }
         else if ( entry->msi_attrib.type == PCI_CAP_ID_MSIX )
-            msix_set_enable(pdev, 1);
+            pci_conf_write16(pdev->seg, pdev->bus, slot, func,
+                             msix_control_reg(entry->msi_attrib.pos),
+                             control | PCI_MSIX_FLAGS_ENABLE);
     }
 
     return 0;

[PATCH v2 3/4] x86/MSI-X: reduce fiddling with control register during restore

[Jan Beulich]

[-- Attachment #1: Type: text/plain, Size: 4108 bytes --]

Rather than disabling and enabling MSI-X once per vector, do it just
once per device.

Signed-off-by: Jan Beulich <jbeulich@suse.com>
Reviewed-by: Andrew Cooper <andrew.cooper3@citrix.com>

--- a/xen/arch/x86/msi.c
+++ b/xen/arch/x86/msi.c
@@ -1217,6 +1217,9 @@ int pci_restore_msi_state(struct pci_dev
     struct msi_desc *entry, *tmp;
     struct irq_desc *desc;
     struct msi_msg msg;
+    u8 slot = PCI_SLOT(pdev->devfn), func = PCI_FUNC(pdev->devfn);
+    unsigned int type = 0, pos = 0;
+    u16 control = 0;
 
     ASSERT(spin_is_locked(&pcidevs_lock));
 
@@ -1233,8 +1236,6 @@ int pci_restore_msi_state(struct pci_dev
     list_for_each_entry_safe( entry, tmp, &pdev->msi_list, list )
     {
         unsigned int i = 0, nr = 1;
-        u16 control = 0;
-        u8 slot = PCI_SLOT(pdev->devfn), func = PCI_FUNC(pdev->devfn);
 
         irq = entry->irq;
         desc = &irq_desc[irq];
@@ -1251,31 +1252,38 @@ int pci_restore_msi_state(struct pci_dev
                     pdev->seg, pdev->bus, PCI_SLOT(pdev->devfn),
                     PCI_FUNC(pdev->devfn), i);
             spin_unlock_irqrestore(&desc->lock, flags);
+            if ( type == PCI_CAP_ID_MSIX )
+                pci_conf_write16(pdev->seg, pdev->bus, slot, func,
+                                 msix_control_reg(pos),
+                                 control & ~PCI_MSIX_FLAGS_ENABLE);
             return -EINVAL;
         }
 
+        ASSERT(!type || type == entry->msi_attrib.type);
+        pos = entry->msi_attrib.pos;
         if ( entry->msi_attrib.type == PCI_CAP_ID_MSI )
         {
             msi_set_enable(pdev, 0);
             nr = entry->msi.nvec;
         }
-        else if ( entry->msi_attrib.type == PCI_CAP_ID_MSIX )
+        else if ( !type && entry->msi_attrib.type == PCI_CAP_ID_MSIX )
         {
             control = pci_conf_read16(pdev->seg, pdev->bus, slot, func,
-                                      msix_control_reg(entry->msi_attrib.pos));
+                                      msix_control_reg(pos));
             pci_conf_write16(pdev->seg, pdev->bus, slot, func,
-                             msix_control_reg(entry->msi_attrib.pos),
+                             msix_control_reg(pos),
                              control | (PCI_MSIX_FLAGS_ENABLE |
                                         PCI_MSIX_FLAGS_MASKALL));
             if ( unlikely(!memory_decoded(pdev)) )
             {
                 spin_unlock_irqrestore(&desc->lock, flags);
                 pci_conf_write16(pdev->seg, pdev->bus, slot, func,
-                                 msix_control_reg(entry->msi_attrib.pos),
+                                 msix_control_reg(pos),
                                  control & ~PCI_MSIX_FLAGS_ENABLE);
                 return -ENXIO;
             }
         }
+        type = entry->msi_attrib.type;
 
         msg = entry->msg;
         write_msi_msg(entry, &msg);
@@ -1298,9 +1306,9 @@ int pci_restore_msi_state(struct pci_dev
 
         spin_unlock_irqrestore(&desc->lock, flags);
 
-        if ( entry->msi_attrib.type == PCI_CAP_ID_MSI )
+        if ( type == PCI_CAP_ID_MSI )
         {
-            unsigned int cpos = msi_control_reg(entry->msi_attrib.pos);
+            unsigned int cpos = msi_control_reg(pos);
 
             control = pci_conf_read16(pdev->seg, pdev->bus, slot, func, cpos) &
                       ~PCI_MSI_FLAGS_QSIZE;
@@ -1310,12 +1318,13 @@ int pci_restore_msi_state(struct pci_dev
 
             msi_set_enable(pdev, 1);
         }
-        else if ( entry->msi_attrib.type == PCI_CAP_ID_MSIX )
-            pci_conf_write16(pdev->seg, pdev->bus, slot, func,
-                             msix_control_reg(entry->msi_attrib.pos),
-                             control | PCI_MSIX_FLAGS_ENABLE);
     }
 
+    if ( type == PCI_CAP_ID_MSIX )
+        pci_conf_write16(pdev->seg, pdev->bus, slot, func,
+                         msix_control_reg(pos),
+                         control | PCI_MSIX_FLAGS_ENABLE);
+
     return 0;
 }
 



[-- Attachment #2: x86-MSI-X-restore-once.patch --]
[-- Type: text/plain, Size: 4171 bytes --]

x86/MSI-X: reduce fiddling with control register during restore

Rather than disabling and enabling MSI-X once per vector, do it just
once per device.

Signed-off-by: Jan Beulich <jbeulich@suse.com>
Reviewed-by: Andrew Cooper <andrew.cooper3@citrix.com>

--- a/xen/arch/x86/msi.c
+++ b/xen/arch/x86/msi.c
@@ -1217,6 +1217,9 @@ int pci_restore_msi_state(struct pci_dev
     struct msi_desc *entry, *tmp;
     struct irq_desc *desc;
     struct msi_msg msg;
+    u8 slot = PCI_SLOT(pdev->devfn), func = PCI_FUNC(pdev->devfn);
+    unsigned int type = 0, pos = 0;
+    u16 control = 0;
 
     ASSERT(spin_is_locked(&pcidevs_lock));
 
@@ -1233,8 +1236,6 @@ int pci_restore_msi_state(struct pci_dev
     list_for_each_entry_safe( entry, tmp, &pdev->msi_list, list )
     {
         unsigned int i = 0, nr = 1;
-        u16 control = 0;
-        u8 slot = PCI_SLOT(pdev->devfn), func = PCI_FUNC(pdev->devfn);
 
         irq = entry->irq;
         desc = &irq_desc[irq];
@@ -1251,31 +1252,38 @@ int pci_restore_msi_state(struct pci_dev
                     pdev->seg, pdev->bus, PCI_SLOT(pdev->devfn),
                     PCI_FUNC(pdev->devfn), i);
             spin_unlock_irqrestore(&desc->lock, flags);
+            if ( type == PCI_CAP_ID_MSIX )
+                pci_conf_write16(pdev->seg, pdev->bus, slot, func,
+                                 msix_control_reg(pos),
+                                 control & ~PCI_MSIX_FLAGS_ENABLE);
             return -EINVAL;
         }
 
+        ASSERT(!type || type == entry->msi_attrib.type);
+        pos = entry->msi_attrib.pos;
         if ( entry->msi_attrib.type == PCI_CAP_ID_MSI )
         {
             msi_set_enable(pdev, 0);
             nr = entry->msi.nvec;
         }
-        else if ( entry->msi_attrib.type == PCI_CAP_ID_MSIX )
+        else if ( !type && entry->msi_attrib.type == PCI_CAP_ID_MSIX )
         {
             control = pci_conf_read16(pdev->seg, pdev->bus, slot, func,
-                                      msix_control_reg(entry->msi_attrib.pos));
+                                      msix_control_reg(pos));
             pci_conf_write16(pdev->seg, pdev->bus, slot, func,
-                             msix_control_reg(entry->msi_attrib.pos),
+                             msix_control_reg(pos),
                              control | (PCI_MSIX_FLAGS_ENABLE |
                                         PCI_MSIX_FLAGS_MASKALL));
             if ( unlikely(!memory_decoded(pdev)) )
             {
                 spin_unlock_irqrestore(&desc->lock, flags);
                 pci_conf_write16(pdev->seg, pdev->bus, slot, func,
-                                 msix_control_reg(entry->msi_attrib.pos),
+                                 msix_control_reg(pos),
                                  control & ~PCI_MSIX_FLAGS_ENABLE);
                 return -ENXIO;
             }
         }
+        type = entry->msi_attrib.type;
 
         msg = entry->msg;
         write_msi_msg(entry, &msg);
@@ -1298,9 +1306,9 @@ int pci_restore_msi_state(struct pci_dev
 
         spin_unlock_irqrestore(&desc->lock, flags);
 
-        if ( entry->msi_attrib.type == PCI_CAP_ID_MSI )
+        if ( type == PCI_CAP_ID_MSI )
         {
-            unsigned int cpos = msi_control_reg(entry->msi_attrib.pos);
+            unsigned int cpos = msi_control_reg(pos);
 
             control = pci_conf_read16(pdev->seg, pdev->bus, slot, func, cpos) &
                       ~PCI_MSI_FLAGS_QSIZE;
@@ -1310,12 +1318,13 @@ int pci_restore_msi_state(struct pci_dev
 
             msi_set_enable(pdev, 1);
         }
-        else if ( entry->msi_attrib.type == PCI_CAP_ID_MSIX )
-            pci_conf_write16(pdev->seg, pdev->bus, slot, func,
-                             msix_control_reg(entry->msi_attrib.pos),
-                             control | PCI_MSIX_FLAGS_ENABLE);
     }
 
+    if ( type == PCI_CAP_ID_MSIX )
+        pci_conf_write16(pdev->seg, pdev->bus, slot, func,
+                         msix_control_reg(pos),
+                         control | PCI_MSIX_FLAGS_ENABLE);
+
     return 0;
 }
 

[-- Attachment #3: Type: text/plain, Size: 126 bytes --]

_______________________________________________
Xen-devel mailing list
Xen-devel@lists.xen.org
http://lists.xen.org/xen-devel

[PATCH v2 4/4] x86/MSI-X: cleanup

[Jan Beulich]

[-- Attachment #1: Type: text/plain, Size: 10722 bytes --]

- __pci_enable_msix() now checks that an MSI-X capability was actually
  found
- pass "pos" to msix_capability_init() as both callers already know it
  (and hence there's no need to re-obtain it)
- call __pci_disable_msi{,x}() directly instead of via
  pci_disable_msi() from __pci_enable_msi{x,}() state validation paths
- use msix_control_reg() instead of open coding it
- log message adjustments
- coding style corrections

Signed-off-by: Jan Beulich <jbeulich@suse.com>
Reviewed-by: Andrew Cooper <andrew.cooper3@citrix.com>

--- a/xen/arch/x86/msi.c
+++ b/xen/arch/x86/msi.c
@@ -35,6 +35,8 @@
 static s8 __read_mostly use_msi = -1;
 boolean_param("msi", use_msi);
 
+static void __pci_disable_msix(struct msi_desc *);
+
 /* bitmap indicate which fixed map is free */
 static DEFINE_SPINLOCK(msix_fixmap_lock);
 static DECLARE_BITMAP(msix_fixmap_pages, FIX_MSIX_MAX_PAGES);
@@ -167,12 +169,14 @@ void msi_compose_msg(unsigned vector, co
     unsigned dest;
 
     memset(msg, 0, sizeof(*msg));
-    if ( !cpumask_intersects(cpu_mask, &cpu_online_map) ) {
+    if ( !cpumask_intersects(cpu_mask, &cpu_online_map) )
+    {
         dprintk(XENLOG_ERR,"%s, compose msi message error!!\n", __func__);
         return;
     }
 
-    if ( vector ) {
+    if ( vector )
+    {
         cpumask_t *mask = this_cpu(scratch_mask);
 
         cpumask_and(mask, cpu_mask, &cpu_online_map);
@@ -233,8 +237,7 @@ static bool_t read_msi_msg(struct msi_de
     }
     case PCI_CAP_ID_MSIX:
     {
-        void __iomem *base;
-        base = entry->mask_base;
+        void __iomem *base = entry->mask_base;
 
         if ( unlikely(!msix_memory_decoded(entry->dev,
                                            entry->msi_attrib.pos)) )
@@ -300,8 +303,7 @@ static int write_msi_msg(struct msi_desc
     }
     case PCI_CAP_ID_MSIX:
     {
-        void __iomem *base;
-        base = entry->mask_base;
+        void __iomem *base = entry->mask_base;
 
         if ( unlikely(!msix_memory_decoded(entry->dev,
                                            entry->msi_attrib.pos)) )
@@ -327,7 +329,7 @@ void set_msi_affinity(struct irq_desc *d
     struct msi_desc *msi_desc = desc->msi_desc;
 
     dest = set_desc_affinity(desc, mask);
-    if (dest == BAD_APICID || !msi_desc)
+    if ( dest == BAD_APICID || !msi_desc )
         return;
 
     ASSERT(spin_is_locked(&desc->lock));
@@ -379,11 +381,11 @@ static void msix_set_enable(struct pci_d
     pos = pci_find_cap_offset(seg, bus, slot, func, PCI_CAP_ID_MSIX);
     if ( pos )
     {
-        control = pci_conf_read16(seg, bus, slot, func, pos + PCI_MSIX_FLAGS);
+        control = pci_conf_read16(seg, bus, slot, func, msix_control_reg(pos));
         control &= ~PCI_MSIX_FLAGS_ENABLE;
         if ( enable )
             control |= PCI_MSIX_FLAGS_ENABLE;
-        pci_conf_write16(seg, bus, slot, func, pos + PCI_MSIX_FLAGS, control);
+        pci_conf_write16(seg, bus, slot, func, msix_control_reg(pos), control);
     }
 }
 
@@ -408,9 +410,11 @@ static bool_t msi_set_mask_bit(struct ir
     bus = pdev->bus;
     slot = PCI_SLOT(pdev->devfn);
     func = PCI_FUNC(pdev->devfn);
-    switch (entry->msi_attrib.type) {
+    switch ( entry->msi_attrib.type )
+    {
     case PCI_CAP_ID_MSI:
-        if (entry->msi_attrib.maskbit) {
+        if ( entry->msi_attrib.maskbit )
+        {
             u32 mask_bits;
 
             mask_bits = pci_conf_read32(seg, bus, slot, func, entry->msi.mpos);
@@ -778,13 +782,14 @@ static u64 read_pci_mem_bar(u16 seg, u8 
  * requested MSI-X entries with allocated irqs or non-zero for otherwise.
  **/
 static int msix_capability_init(struct pci_dev *dev,
+                                unsigned int pos,
                                 struct msi_info *msi,
                                 struct msi_desc **desc,
                                 unsigned int nr_entries)
 {
     struct arch_msix *msix = dev->msix;
     struct msi_desc *entry = NULL;
-    int pos, vf;
+    int vf;
     u16 control;
     u64 table_paddr;
     u32 table_offset;
@@ -796,7 +801,6 @@ static int msix_capability_init(struct p
 
     ASSERT(spin_is_locked(&pcidevs_lock));
 
-    pos = pci_find_cap_offset(seg, bus, slot, func, PCI_CAP_ID_MSIX);
     control = pci_conf_read16(seg, bus, slot, func, msix_control_reg(pos));
     /*
      * Ensure MSI-X interrupts are masked during setup. Some devices require
@@ -984,10 +988,9 @@ static int __pci_enable_msi(struct msi_i
     old_desc = find_msi_entry(pdev, msi->irq, PCI_CAP_ID_MSI);
     if ( old_desc )
     {
-        dprintk(XENLOG_WARNING, "irq %d has already mapped to MSI on "
-                "device %04x:%02x:%02x.%01x\n",
-                msi->irq, msi->seg, msi->bus,
-                PCI_SLOT(msi->devfn), PCI_FUNC(msi->devfn));
+        printk(XENLOG_WARNING "irq %d already mapped to MSI on %04x:%02x:%02x.%u\n",
+               msi->irq, msi->seg, msi->bus,
+               PCI_SLOT(msi->devfn), PCI_FUNC(msi->devfn));
         *desc = old_desc;
         return 0;
     }
@@ -995,10 +998,10 @@ static int __pci_enable_msi(struct msi_i
     old_desc = find_msi_entry(pdev, -1, PCI_CAP_ID_MSIX);
     if ( old_desc )
     {
-        dprintk(XENLOG_WARNING, "MSI-X is already in use on "
-                "device %04x:%02x:%02x.%01x\n", msi->seg, msi->bus,
-                PCI_SLOT(msi->devfn), PCI_FUNC(msi->devfn));
-        pci_disable_msi(old_desc);
+        printk(XENLOG_WARNING "MSI-X already in use on %04x:%02x:%02x.%u\n",
+               msi->seg, msi->bus,
+               PCI_SLOT(msi->devfn), PCI_FUNC(msi->devfn));
+        __pci_disable_msix(old_desc);
     }
 
     return msi_capability_init(pdev, msi->irq, desc, msi->entry_nr);
@@ -1012,7 +1015,6 @@ static void __pci_disable_msi(struct msi
     msi_set_enable(dev, 0);
 
     BUG_ON(list_empty(&dev->msi_list));
-
 }
 
 /**
@@ -1032,7 +1034,7 @@ static void __pci_disable_msi(struct msi
  **/
 static int __pci_enable_msix(struct msi_info *msi, struct msi_desc **desc)
 {
-    int status, pos, nr_entries;
+    int pos, nr_entries;
     struct pci_dev *pdev;
     u16 control;
     u8 slot = PCI_SLOT(msi->devfn);
@@ -1041,23 +1043,22 @@ static int __pci_enable_msix(struct msi_
 
     ASSERT(spin_is_locked(&pcidevs_lock));
     pdev = pci_get_pdev(msi->seg, msi->bus, msi->devfn);
-    if ( !pdev )
+    pos = pci_find_cap_offset(msi->seg, msi->bus, slot, func, PCI_CAP_ID_MSIX);
+    if ( !pdev || !pos )
         return -ENODEV;
 
-    pos = pci_find_cap_offset(msi->seg, msi->bus, slot, func, PCI_CAP_ID_MSIX);
     control = pci_conf_read16(msi->seg, msi->bus, slot, func,
                               msix_control_reg(pos));
     nr_entries = multi_msix_capable(control);
-    if (msi->entry_nr >= nr_entries)
+    if ( msi->entry_nr >= nr_entries )
         return -EINVAL;
 
     old_desc = find_msi_entry(pdev, msi->irq, PCI_CAP_ID_MSIX);
     if ( old_desc )
     {
-        dprintk(XENLOG_WARNING, "irq %d has already mapped to MSIX on "
-                "device %04x:%02x:%02x.%01x\n",
-                msi->irq, msi->seg, msi->bus,
-                PCI_SLOT(msi->devfn), PCI_FUNC(msi->devfn));
+        printk(XENLOG_WARNING "irq %d already mapped to MSI-X on %04x:%02x:%02x.%u\n",
+               msi->irq, msi->seg, msi->bus,
+               PCI_SLOT(msi->devfn), PCI_FUNC(msi->devfn));
         *desc = old_desc;
         return 0;
     }
@@ -1065,15 +1066,13 @@ static int __pci_enable_msix(struct msi_
     old_desc = find_msi_entry(pdev, -1, PCI_CAP_ID_MSI);
     if ( old_desc )
     {
-        dprintk(XENLOG_WARNING, "MSI is already in use on "
-                "device %04x:%02x:%02x.%01x\n", msi->seg, msi->bus,
-                PCI_SLOT(msi->devfn), PCI_FUNC(msi->devfn));
-        pci_disable_msi(old_desc);
-
+        printk(XENLOG_WARNING "MSI already in use on %04x:%02x:%02x.%u\n",
+               msi->seg, msi->bus,
+               PCI_SLOT(msi->devfn), PCI_FUNC(msi->devfn));
+        __pci_disable_msi(old_desc);
     }
 
-    status = msix_capability_init(pdev, msi, desc, nr_entries);
-    return status;
+    return msix_capability_init(pdev, pos, msi, desc, nr_entries);
 }
 
 static void _pci_cleanup_msix(struct arch_msix *msix)
@@ -1091,19 +1090,16 @@ static void _pci_cleanup_msix(struct arc
 
 static void __pci_disable_msix(struct msi_desc *entry)
 {
-    struct pci_dev *dev;
-    int pos;
-    u16 control, seg;
-    u8 bus, slot, func;
-
-    dev = entry->dev;
-    seg = dev->seg;
-    bus = dev->bus;
-    slot = PCI_SLOT(dev->devfn);
-    func = PCI_FUNC(dev->devfn);
+    struct pci_dev *dev = entry->dev;
+    u16 seg = dev->seg;
+    u8 bus = dev->bus;
+    u8 slot = PCI_SLOT(dev->devfn);
+    u8 func = PCI_FUNC(dev->devfn);
+    unsigned int pos = pci_find_cap_offset(seg, bus, slot, func,
+                                           PCI_CAP_ID_MSIX);
+    u16 control = pci_conf_read16(seg, bus, slot, func,
+                                  msix_control_reg(entry->msi_attrib.pos));
 
-    pos = pci_find_cap_offset(seg, bus, slot, func, PCI_CAP_ID_MSIX);
-    control = pci_conf_read16(seg, bus, slot, func, msix_control_reg(pos));
     if ( unlikely(!(control & PCI_MSIX_FLAGS_ENABLE)) )
         pci_conf_write16(seg, bus, slot, func, msix_control_reg(pos),
                          control | (PCI_MSIX_FLAGS_ENABLE |
@@ -1156,7 +1152,7 @@ int pci_prepare_msix(u16 seg, u8 bus, u8
         u16 control = pci_conf_read16(seg, bus, slot, func,
                                       msix_control_reg(pos));
 
-        rc = msix_capability_init(pdev, NULL, NULL,
+        rc = msix_capability_init(pdev, pos, NULL, NULL,
                                   multi_msix_capable(control));
     }
     spin_unlock(&pcidevs_lock);
@@ -1175,8 +1171,8 @@ int pci_enable_msi(struct msi_info *msi,
     if ( !use_msi )
         return -EPERM;
 
-    return  msi->table_base ? __pci_enable_msix(msi, desc) :
-        __pci_enable_msi(msi, desc);
+    return msi->table_base ? __pci_enable_msix(msi, desc) :
+                             __pci_enable_msi(msi, desc);
 }
 
 /*
@@ -1229,7 +1225,9 @@ int pci_restore_msi_state(struct pci_dev
     if ( !pdev )
         return -EINVAL;
 
-    ret = xsm_resource_setup_pci(XSM_PRIV, (pdev->seg << 16) | (pdev->bus << 8) | pdev->devfn);
+    ret = xsm_resource_setup_pci(XSM_PRIV,
+                                (pdev->seg << 16) | (pdev->bus << 8) |
+                                pdev->devfn);
     if ( ret )
         return ret;
 



[-- Attachment #2: x86-MSI-X-cleanup.patch --]
[-- Type: text/plain, Size: 10740 bytes --]

x86/MSI-X: cleanup

- __pci_enable_msix() now checks that an MSI-X capability was actually
  found
- pass "pos" to msix_capability_init() as both callers already know it
  (and hence there's no need to re-obtain it)
- call __pci_disable_msi{,x}() directly instead of via
  pci_disable_msi() from __pci_enable_msi{x,}() state validation paths
- use msix_control_reg() instead of open coding it
- log message adjustments
- coding style corrections

Signed-off-by: Jan Beulich <jbeulich@suse.com>
Reviewed-by: Andrew Cooper <andrew.cooper3@citrix.com>

--- a/xen/arch/x86/msi.c
+++ b/xen/arch/x86/msi.c
@@ -35,6 +35,8 @@
 static s8 __read_mostly use_msi = -1;
 boolean_param("msi", use_msi);
 
+static void __pci_disable_msix(struct msi_desc *);
+
 /* bitmap indicate which fixed map is free */
 static DEFINE_SPINLOCK(msix_fixmap_lock);
 static DECLARE_BITMAP(msix_fixmap_pages, FIX_MSIX_MAX_PAGES);
@@ -167,12 +169,14 @@ void msi_compose_msg(unsigned vector, co
     unsigned dest;
 
     memset(msg, 0, sizeof(*msg));
-    if ( !cpumask_intersects(cpu_mask, &cpu_online_map) ) {
+    if ( !cpumask_intersects(cpu_mask, &cpu_online_map) )
+    {
         dprintk(XENLOG_ERR,"%s, compose msi message error!!\n", __func__);
         return;
     }
 
-    if ( vector ) {
+    if ( vector )
+    {
         cpumask_t *mask = this_cpu(scratch_mask);
 
         cpumask_and(mask, cpu_mask, &cpu_online_map);
@@ -233,8 +237,7 @@ static bool_t read_msi_msg(struct msi_de
     }
     case PCI_CAP_ID_MSIX:
     {
-        void __iomem *base;
-        base = entry->mask_base;
+        void __iomem *base = entry->mask_base;
 
         if ( unlikely(!msix_memory_decoded(entry->dev,
                                            entry->msi_attrib.pos)) )
@@ -300,8 +303,7 @@ static int write_msi_msg(struct msi_desc
     }
     case PCI_CAP_ID_MSIX:
     {
-        void __iomem *base;
-        base = entry->mask_base;
+        void __iomem *base = entry->mask_base;
 
         if ( unlikely(!msix_memory_decoded(entry->dev,
                                            entry->msi_attrib.pos)) )
@@ -327,7 +329,7 @@ void set_msi_affinity(struct irq_desc *d
     struct msi_desc *msi_desc = desc->msi_desc;
 
     dest = set_desc_affinity(desc, mask);
-    if (dest == BAD_APICID || !msi_desc)
+    if ( dest == BAD_APICID || !msi_desc )
         return;
 
     ASSERT(spin_is_locked(&desc->lock));
@@ -379,11 +381,11 @@ static void msix_set_enable(struct pci_d
     pos = pci_find_cap_offset(seg, bus, slot, func, PCI_CAP_ID_MSIX);
     if ( pos )
     {
-        control = pci_conf_read16(seg, bus, slot, func, pos + PCI_MSIX_FLAGS);
+        control = pci_conf_read16(seg, bus, slot, func, msix_control_reg(pos));
         control &= ~PCI_MSIX_FLAGS_ENABLE;
         if ( enable )
             control |= PCI_MSIX_FLAGS_ENABLE;
-        pci_conf_write16(seg, bus, slot, func, pos + PCI_MSIX_FLAGS, control);
+        pci_conf_write16(seg, bus, slot, func, msix_control_reg(pos), control);
     }
 }
 
@@ -408,9 +410,11 @@ static bool_t msi_set_mask_bit(struct ir
     bus = pdev->bus;
     slot = PCI_SLOT(pdev->devfn);
     func = PCI_FUNC(pdev->devfn);
-    switch (entry->msi_attrib.type) {
+    switch ( entry->msi_attrib.type )
+    {
     case PCI_CAP_ID_MSI:
-        if (entry->msi_attrib.maskbit) {
+        if ( entry->msi_attrib.maskbit )
+        {
             u32 mask_bits;
 
             mask_bits = pci_conf_read32(seg, bus, slot, func, entry->msi.mpos);
@@ -778,13 +782,14 @@ static u64 read_pci_mem_bar(u16 seg, u8 
  * requested MSI-X entries with allocated irqs or non-zero for otherwise.
  **/
 static int msix_capability_init(struct pci_dev *dev,
+                                unsigned int pos,
                                 struct msi_info *msi,
                                 struct msi_desc **desc,
                                 unsigned int nr_entries)
 {
     struct arch_msix *msix = dev->msix;
     struct msi_desc *entry = NULL;
-    int pos, vf;
+    int vf;
     u16 control;
     u64 table_paddr;
     u32 table_offset;
@@ -796,7 +801,6 @@ static int msix_capability_init(struct p
 
     ASSERT(spin_is_locked(&pcidevs_lock));
 
-    pos = pci_find_cap_offset(seg, bus, slot, func, PCI_CAP_ID_MSIX);
     control = pci_conf_read16(seg, bus, slot, func, msix_control_reg(pos));
     /*
      * Ensure MSI-X interrupts are masked during setup. Some devices require
@@ -984,10 +988,9 @@ static int __pci_enable_msi(struct msi_i
     old_desc = find_msi_entry(pdev, msi->irq, PCI_CAP_ID_MSI);
     if ( old_desc )
     {
-        dprintk(XENLOG_WARNING, "irq %d has already mapped to MSI on "
-                "device %04x:%02x:%02x.%01x\n",
-                msi->irq, msi->seg, msi->bus,
-                PCI_SLOT(msi->devfn), PCI_FUNC(msi->devfn));
+        printk(XENLOG_WARNING "irq %d already mapped to MSI on %04x:%02x:%02x.%u\n",
+               msi->irq, msi->seg, msi->bus,
+               PCI_SLOT(msi->devfn), PCI_FUNC(msi->devfn));
         *desc = old_desc;
         return 0;
     }
@@ -995,10 +998,10 @@ static int __pci_enable_msi(struct msi_i
     old_desc = find_msi_entry(pdev, -1, PCI_CAP_ID_MSIX);
     if ( old_desc )
     {
-        dprintk(XENLOG_WARNING, "MSI-X is already in use on "
-                "device %04x:%02x:%02x.%01x\n", msi->seg, msi->bus,
-                PCI_SLOT(msi->devfn), PCI_FUNC(msi->devfn));
-        pci_disable_msi(old_desc);
+        printk(XENLOG_WARNING "MSI-X already in use on %04x:%02x:%02x.%u\n",
+               msi->seg, msi->bus,
+               PCI_SLOT(msi->devfn), PCI_FUNC(msi->devfn));
+        __pci_disable_msix(old_desc);
     }
 
     return msi_capability_init(pdev, msi->irq, desc, msi->entry_nr);
@@ -1012,7 +1015,6 @@ static void __pci_disable_msi(struct msi
     msi_set_enable(dev, 0);
 
     BUG_ON(list_empty(&dev->msi_list));
-
 }
 
 /**
@@ -1032,7 +1034,7 @@ static void __pci_disable_msi(struct msi
  **/
 static int __pci_enable_msix(struct msi_info *msi, struct msi_desc **desc)
 {
-    int status, pos, nr_entries;
+    int pos, nr_entries;
     struct pci_dev *pdev;
     u16 control;
     u8 slot = PCI_SLOT(msi->devfn);
@@ -1041,23 +1043,22 @@ static int __pci_enable_msix(struct msi_
 
     ASSERT(spin_is_locked(&pcidevs_lock));
     pdev = pci_get_pdev(msi->seg, msi->bus, msi->devfn);
-    if ( !pdev )
+    pos = pci_find_cap_offset(msi->seg, msi->bus, slot, func, PCI_CAP_ID_MSIX);
+    if ( !pdev || !pos )
         return -ENODEV;
 
-    pos = pci_find_cap_offset(msi->seg, msi->bus, slot, func, PCI_CAP_ID_MSIX);
     control = pci_conf_read16(msi->seg, msi->bus, slot, func,
                               msix_control_reg(pos));
     nr_entries = multi_msix_capable(control);
-    if (msi->entry_nr >= nr_entries)
+    if ( msi->entry_nr >= nr_entries )
         return -EINVAL;
 
     old_desc = find_msi_entry(pdev, msi->irq, PCI_CAP_ID_MSIX);
     if ( old_desc )
     {
-        dprintk(XENLOG_WARNING, "irq %d has already mapped to MSIX on "
-                "device %04x:%02x:%02x.%01x\n",
-                msi->irq, msi->seg, msi->bus,
-                PCI_SLOT(msi->devfn), PCI_FUNC(msi->devfn));
+        printk(XENLOG_WARNING "irq %d already mapped to MSI-X on %04x:%02x:%02x.%u\n",
+               msi->irq, msi->seg, msi->bus,
+               PCI_SLOT(msi->devfn), PCI_FUNC(msi->devfn));
         *desc = old_desc;
         return 0;
     }
@@ -1065,15 +1066,13 @@ static int __pci_enable_msix(struct msi_
     old_desc = find_msi_entry(pdev, -1, PCI_CAP_ID_MSI);
     if ( old_desc )
     {
-        dprintk(XENLOG_WARNING, "MSI is already in use on "
-                "device %04x:%02x:%02x.%01x\n", msi->seg, msi->bus,
-                PCI_SLOT(msi->devfn), PCI_FUNC(msi->devfn));
-        pci_disable_msi(old_desc);
-
+        printk(XENLOG_WARNING "MSI already in use on %04x:%02x:%02x.%u\n",
+               msi->seg, msi->bus,
+               PCI_SLOT(msi->devfn), PCI_FUNC(msi->devfn));
+        __pci_disable_msi(old_desc);
     }
 
-    status = msix_capability_init(pdev, msi, desc, nr_entries);
-    return status;
+    return msix_capability_init(pdev, pos, msi, desc, nr_entries);
 }
 
 static void _pci_cleanup_msix(struct arch_msix *msix)
@@ -1091,19 +1090,16 @@ static void _pci_cleanup_msix(struct arc
 
 static void __pci_disable_msix(struct msi_desc *entry)
 {
-    struct pci_dev *dev;
-    int pos;
-    u16 control, seg;
-    u8 bus, slot, func;
-
-    dev = entry->dev;
-    seg = dev->seg;
-    bus = dev->bus;
-    slot = PCI_SLOT(dev->devfn);
-    func = PCI_FUNC(dev->devfn);
+    struct pci_dev *dev = entry->dev;
+    u16 seg = dev->seg;
+    u8 bus = dev->bus;
+    u8 slot = PCI_SLOT(dev->devfn);
+    u8 func = PCI_FUNC(dev->devfn);
+    unsigned int pos = pci_find_cap_offset(seg, bus, slot, func,
+                                           PCI_CAP_ID_MSIX);
+    u16 control = pci_conf_read16(seg, bus, slot, func,
+                                  msix_control_reg(entry->msi_attrib.pos));
 
-    pos = pci_find_cap_offset(seg, bus, slot, func, PCI_CAP_ID_MSIX);
-    control = pci_conf_read16(seg, bus, slot, func, msix_control_reg(pos));
     if ( unlikely(!(control & PCI_MSIX_FLAGS_ENABLE)) )
         pci_conf_write16(seg, bus, slot, func, msix_control_reg(pos),
                          control | (PCI_MSIX_FLAGS_ENABLE |
@@ -1156,7 +1152,7 @@ int pci_prepare_msix(u16 seg, u8 bus, u8
         u16 control = pci_conf_read16(seg, bus, slot, func,
                                       msix_control_reg(pos));
 
-        rc = msix_capability_init(pdev, NULL, NULL,
+        rc = msix_capability_init(pdev, pos, NULL, NULL,
                                   multi_msix_capable(control));
     }
     spin_unlock(&pcidevs_lock);
@@ -1175,8 +1171,8 @@ int pci_enable_msi(struct msi_info *msi,
     if ( !use_msi )
         return -EPERM;
 
-    return  msi->table_base ? __pci_enable_msix(msi, desc) :
-        __pci_enable_msi(msi, desc);
+    return msi->table_base ? __pci_enable_msix(msi, desc) :
+                             __pci_enable_msi(msi, desc);
 }
 
 /*
@@ -1229,7 +1225,9 @@ int pci_restore_msi_state(struct pci_dev
     if ( !pdev )
         return -EINVAL;
 
-    ret = xsm_resource_setup_pci(XSM_PRIV, (pdev->seg << 16) | (pdev->bus << 8) | pdev->devfn);
+    ret = xsm_resource_setup_pci(XSM_PRIV,
+                                (pdev->seg << 16) | (pdev->bus << 8) |
+                                pdev->devfn);
     if ( ret )
         return ret;
 

[-- Attachment #3: Type: text/plain, Size: 126 bytes --]

_______________________________________________
Xen-devel mailing list
Xen-devel@lists.xen.org
http://lists.xen.org/xen-devel