[XEN PATCH v2 19/25] arm: new VGIC: its: Add LPI translation cache definition

[Mykyta Poturai <Mykyta_Poturai@epam.com>]

Add the basic data structure that expresses an MSI to LPI
translation as well as the allocation/release hooks.

Implement cache invalidation, lookup and storage.

The size of the cache is arbitrarily defined as 16*nr_vcpus.

This is based on Linux commits 24cab82c34aa6f, 7d825fd6eaa7467,
89489ee9ced8 and 86a7dae884f38 by Marc Zyngier

Signed-off-by: Mykyta Poturai <mykyta_poturai@epam.com>
---
 xen/arch/arm/include/asm/new_vgic.h |   3 +
 xen/arch/arm/vgic/vgic-its.c        | 195 ++++++++++++++++++++++++++++
 2 files changed, 198 insertions(+)

diff --git a/xen/arch/arm/include/asm/new_vgic.h b/xen/arch/arm/include/asm/new_vgic.h
index d0fd15e154..b038fb7861 100644
--- a/xen/arch/arm/include/asm/new_vgic.h
+++ b/xen/arch/arm/include/asm/new_vgic.h
@@ -212,6 +212,9 @@ struct vgic_dist {
     spinlock_t          lpi_list_lock;
     struct list_head    lpi_list_head;
     unsigned int        lpi_list_count;
+
+    /* LPI translation cache */
+    struct list_head	lpi_translation_cache;
 };
 
 struct vgic_cpu {
diff --git a/xen/arch/arm/vgic/vgic-its.c b/xen/arch/arm/vgic/vgic-its.c
index 6c726dde3a..48dfa09115 100644
--- a/xen/arch/arm/vgic/vgic-its.c
+++ b/xen/arch/arm/vgic/vgic-its.c
@@ -44,6 +44,14 @@ struct its_ite {
     u32 event_id;
 };
 
+struct vgic_translation_cache_entry {
+    struct list_head entry;
+    paddr_t db;
+    u32 devid;
+    u32 eventid;
+    struct vgic_irq *irq;
+};
+
 /*
  * Find and returns a device in the device table for an ITS.
  * Must be called with the its_devices_lock mutex held.
@@ -152,6 +160,144 @@ int vgic_copy_lpi_list(struct domain *d, struct vcpu *vcpu, u32 **intid_ptr)
     return i;
 }
 
+void __vgic_put_lpi_locked(struct domain *d, struct vgic_irq *irq)
+{
+    struct vgic_dist *dist = &d->arch.vgic;
+
+    if ( !atomic_dec_and_test(&irq->refcount) )
+    {
+        return;
+    };
+
+    list_del(&irq->lpi_list);
+    dist->lpi_list_count--;
+
+    xfree(irq);
+}
+
+static struct vgic_irq *__vgic_its_check_cache(struct vgic_dist *dist,
+                                               paddr_t db, u32 devid,
+                                               u32 eventid)
+{
+    struct vgic_translation_cache_entry *cte, *fcte;
+
+    list_for_each_entry(cte, &dist->lpi_translation_cache, entry)
+    {
+        /*
+         * If we hit a NULL entry, there is nothing after this
+         * point.
+         */
+        if ( !cte->irq )
+            break;
+
+        if ( cte->db != db || cte->devid != devid || cte->eventid != eventid )
+            continue;
+
+        /*
+         * Move this entry to the head, as it is the most
+         * recently used.
+         */
+        fcte = list_first_entry(&dist->lpi_translation_cache,
+                                struct vgic_translation_cache_entry, entry);
+
+        if ( fcte->irq != cte->irq )
+            list_move(&cte->entry, &dist->lpi_translation_cache);
+
+        return cte->irq;
+    }
+
+    return NULL;
+}
+
+static struct vgic_irq *vgic_its_check_cache(struct domain *d, paddr_t db,
+					     u32 devid, u32 eventid)
+{
+	struct vgic_dist *dist = &d->arch.vgic;
+	struct vgic_irq *irq;
+
+	spin_lock(&dist->lpi_list_lock);
+	irq = __vgic_its_check_cache(dist, db, devid, eventid);
+	spin_unlock(&dist->lpi_list_lock);
+
+	return irq;
+}
+
+static void vgic_its_cache_translation(struct domain *d, struct vgic_its *its,
+                                       u32 devid, u32 eventid,
+                                       struct vgic_irq *irq)
+{
+    struct vgic_dist *dist = &d->arch.vgic;
+    struct vgic_translation_cache_entry *cte;
+    unsigned long flags;
+    paddr_t db;
+
+    /* Do not cache a directly injected interrupt */
+    if ( irq->hw )
+        return;
+
+    spin_lock_irqsave(&dist->lpi_list_lock, flags);
+
+    if ( unlikely(list_empty(&dist->lpi_translation_cache)) )
+        goto out;
+
+    /*
+     * We could have raced with another CPU caching the same
+     * translation behind our back, so let's check it is not in
+     * already
+     */
+    db = its->vgic_its_base + GITS_TRANSLATER;
+    if ( __vgic_its_check_cache(dist, db, devid, eventid) )
+        goto out;
+
+    /* Always reuse the last entry (LRU policy) */
+    cte = list_last_entry(&dist->lpi_translation_cache, typeof(*cte), entry);
+
+    /*
+     * Caching the translation implies having an extra reference
+     * to the interrupt, so drop the potential reference on what
+     * was in the cache, and increment it on the new interrupt.
+     */
+    if ( cte->irq )
+        __vgic_put_lpi_locked(d, cte->irq);
+
+    vgic_get_irq_kref(irq);
+
+    cte->db      = db;
+    cte->devid   = devid;
+    cte->eventid = eventid;
+    cte->irq     = irq;
+
+    /* Move the new translation to the head of the list */
+    list_move(&cte->entry, &dist->lpi_translation_cache);
+
+out:
+    spin_unlock_irqrestore(&dist->lpi_list_lock, flags);
+}
+
+void vgic_its_invalidate_cache(struct domain *d)
+{
+    struct vgic_dist *dist = &d->arch.vgic;
+    struct vgic_translation_cache_entry *cte;
+    unsigned long flags;
+
+    spin_lock_irqsave(&dist->lpi_list_lock, flags);
+
+    list_for_each_entry(cte, &dist->lpi_translation_cache, entry)
+    {
+        /*
+         * If we hit a NULL entry, there is nothing after this
+         * point.
+         */
+        if ( !cte->irq )
+            break;
+
+        __vgic_put_lpi_locked(d, cte->irq);
+        cte->irq = NULL;
+    }
+
+    spin_unlock_irqrestore(&dist->lpi_list_lock, flags);
+}
+
 /* Requires the its_lock to be held. */
 static void its_free_ite(struct domain *d, struct its_ite *ite)
 {
@@ -184,6 +330,8 @@ void vgic_its_free_device(struct vgic_its_device *device)
     list_for_each_entry_safe(ite, temp, &device->itt_head, ite_list)
         its_free_ite(d, ite);
 
+    vgic_its_invalidate_cache(d);
+
     list_del(&device->dev_list);
     xfree(device);
 }
@@ -351,6 +499,8 @@ static void vgic_mmio_write_its_ctlr(struct domain *d, struct vgic_its *its,
         goto out;
 
     its->enabled = !!(val & GITS_CTLR_ENABLE);
+    if ( !its->enabled )
+        vgic_its_invalidate_cache(d);
 
     /*
      * Try to process any pending commands. This function bails out early
@@ -742,6 +892,48 @@ out:
     return ret;
 }
 
+/* Default is 16 cached LPIs per vcpu */
+#define LPI_DEFAULT_PCPU_CACHE_SIZE 16
+
+void vgic_lpi_translation_cache_init(struct domain *d)
+{
+    struct vgic_dist *dist = &d->arch.vgic;
+    unsigned int sz;
+    int i;
+
+    if ( !list_empty(&dist->lpi_translation_cache) )
+        return;
+
+    sz = d->max_vcpus * LPI_DEFAULT_PCPU_CACHE_SIZE;
+
+    for ( i = 0; i < sz; i++ )
+    {
+        struct vgic_translation_cache_entry *cte;
+
+        /* An allocation failure is not fatal */
+        cte = xzalloc(struct vgic_translation_cache_entry);
+        if ( WARN_ON(!cte) )
+            break;
+
+        INIT_LIST_HEAD(&cte->entry);
+        list_add(&cte->entry, &dist->lpi_translation_cache);
+    }
+}
+
+void vgic_lpi_translation_cache_destroy(struct domain *d)
+{
+    struct vgic_dist *dist = &d->arch.vgic;
+    struct vgic_translation_cache_entry *cte, *tmp;
+
+    vgic_its_invalidate_cache(d);
+
+    list_for_each_entry_safe(cte, tmp, &dist->lpi_translation_cache, entry)
+    {
+        list_del(&cte->entry);
+        xfree(cte);
+    }
+}
+
 #define INITIAL_BASER_VALUE                                                    \
     (GIC_BASER_CACHEABILITY(GITS_BASER, INNER, RaWb) |                         \
      GIC_BASER_CACHEABILITY(GITS_BASER, OUTER, SameAsInner) |                  \
@@ -763,6 +955,8 @@ static int vgic_its_create(struct domain *d, u64 addr)
 
     d->arch.vgic.its = its;
 
+    vgic_lpi_translation_cache_init(d);
+
     spin_lock_init(&its->its_lock);
     spin_lock_init(&its->cmd_lock);
 
@@ -829,6 +1023,7 @@ void vgic_v3_its_free_domain(struct domain *d)
 
     vgic_its_free_device_list(d, its);
     vgic_its_free_collection_list(d, its);
+    vgic_lpi_translation_cache_destroy(d);
 
     spin_unlock(&d->arch.vgic.its_devices_lock);
     spin_unlock(&its->its_lock);
-- 
2.34.1