[PATCH v2 0/3] Automatically derive soft affinity from vnuma information

[PATCH v2 0/3] Automatically derive soft affinity from vnuma information

[Dario Faggioli]

Round 2. All patches have Wei's Ack.

Wei, I modified the comment of libxl__vnuma_config_check() and added your ack
on patch 1, as you said during review.

This is available as a git branch here:

  git://xenbits.xen.org/people/dariof/xen.git  rel/numa/vnuma/softaffinity-v2

Regards,
Dario
---
Dario Faggioli (3):
      libxl: check whether vcpu affinity and vnuma info match
      libxl: automatically set soft affinity after vnuma info
      libxl: cleanup some misuse of 'cpumap' as parameter

 tools/libxl/libxl_dom.c   |   46 ++++++++++++++++++++++++++++++++++-
 tools/libxl/libxl_utils.h |    6 ++---
 tools/libxl/libxl_vnuma.c |   60 +++++++++++++++++++++++++++++++++++++++++++++
 3 files changed, 108 insertions(+), 4 deletions(-)

[PATCH v2 1/3] libxl: check whether vcpu affinity and vnuma info match

[Dario Faggioli]

More specifically, vcpus are assigned to a vnode, which in
turn is associated with a pnode. If a vcpu also has, in its
(hard or soft) affinity, some pcpus that are not part of the
said pnode, print a warning to the user.

Signed-off-by: Dario Faggioli <dario.faggioli@citrix.com>
Cc: Ian Campbell <ian.campbell@citrix.com>
Cc: Ian Jackson <Ian.Jackson@eu.citrix.com>
Cc: Stefano Stabellini <stefano.stabellini@eu.citrix.com>
Cc: Wei Liu <wei.liu2@citrix.com>
Acked-by: Wei Liu <wei.liu2@citrix.com>
---
Changes from v1:
 * updated the doc comment of libxl__vnuma_config_check(),
   as requested during review.
---
 tools/libxl/libxl_vnuma.c |   60 +++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 60 insertions(+)

diff --git a/tools/libxl/libxl_vnuma.c b/tools/libxl/libxl_vnuma.c
index aad297e..cac78d7 100644
--- a/tools/libxl/libxl_vnuma.c
+++ b/tools/libxl/libxl_vnuma.c
@@ -33,11 +33,54 @@ static int compare_vmemrange(const void *a, const void *b)
     return 0;
 }
 
+/* Check if a vcpu has an hard (or soft) affinity set in such
+ * a way that it does not match the pnode to which the vcpu itself
+ * is assigned to.
+ */
+static int check_vnuma_affinity(libxl__gc *gc,
+                                 unsigned int vcpu,
+                                 unsigned int pnode,
+                                 unsigned int num_affinity,
+                                 const libxl_bitmap *affinity,
+                                 const char *kind)
+{
+    libxl_bitmap nodemap;
+    int rc = 0;
+
+    libxl_bitmap_init(&nodemap);
+
+    rc = libxl_node_bitmap_alloc(CTX, &nodemap, 0);
+    if (rc) {
+        LOG(ERROR, "Can't allocate nodemap");
+        goto out;
+    }
+
+    rc = libxl_cpumap_to_nodemap(CTX, affinity, &nodemap);
+    if (rc) {
+        LOG(ERROR, "Can't convert Vcpu %d affinity to nodemap", vcpu);
+        goto out;
+    }
+
+    if (libxl_bitmap_count_set(&nodemap) != 1 ||
+        !libxl_bitmap_test(&nodemap, pnode))
+        LOG(WARN, "Vcpu %d %s affinity and vnuma info mismatch", vcpu, kind);
+
+out:
+    libxl_bitmap_dispose(&nodemap);
+    return rc;
+}
+
 /* Check if vNUMA configuration is valid:
  *  1. all pnodes inside vnode_to_pnode array are valid
  *  2. each vcpu belongs to one and only one vnode
  *  3. each vmemrange is valid and doesn't overlap with any other
  *  4. local distance cannot be larger than remote distance
+ *
+ * Check also, if any hard or soft affinity is specified, whether
+ * they match with the vNUMA related bits (namely vcpus to vnodes
+ * mappings and vnodes to pnodes association). If that does not
+ * hold, however, just print a warning, as that has "only"
+ * performance implications.
  */
 int libxl__vnuma_config_check(libxl__gc *gc,
                               const libxl_domain_build_info *b_info,
@@ -101,6 +144,23 @@ int libxl__vnuma_config_check(libxl__gc *gc,
         }
     }
 
+    /* Check whether vcpu affinity (if any) matches vnuma configuration */
+    for (i = 0; i < b_info->num_vnuma_nodes; i++) {
+        v = &b_info->vnuma_nodes[i];
+        libxl_for_each_set_bit(j, v->vcpus) {
+            if (b_info->num_vcpu_hard_affinity > j)
+                check_vnuma_affinity(gc, j, v->pnode,
+                                     b_info->num_vcpu_hard_affinity,
+                                     &b_info->vcpu_hard_affinity[j],
+                                     "hard");
+            if (b_info->num_vcpu_soft_affinity > j)
+                check_vnuma_affinity(gc, j, v->pnode,
+                                     b_info->num_vcpu_soft_affinity,
+                                     &b_info->vcpu_soft_affinity[j],
+                                     "soft");
+        }
+    }
+
     /* Check vmemranges */
     qsort(state->vmemranges, state->num_vmemranges, sizeof(xen_vmemrange_t),
           compare_vmemrange);

[PATCH v2 2/3] libxl: automatically set soft affinity after vnuma info

[Dario Faggioli]

More specifically, vcpus are assigned to a vnode, which in
turn is associated with a pnode. If a vcpu does not have any
soft affinity, automatically build up one, matching the pcpus
of the said pnode.

Signed-off-by: Dario Faggioli <dario.faggioli@citrix.com>
Cc: Ian Campbell <ian.campbell@citrix.com>
Cc: Ian Jackson <Ian.Jackson@eu.citrix.com>
Cc: Stefano Stabellini <stefano.stabellini@eu.citrix.com>
Cc: Wei Liu <wei.liu2@citrix.com>
Acked-by: Wei Liu <wei.liu2@citrix.com>
---
 tools/libxl/libxl_dom.c |   44 ++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 44 insertions(+)

diff --git a/tools/libxl/libxl_dom.c b/tools/libxl/libxl_dom.c
index ace8a66..554ea68 100644
--- a/tools/libxl/libxl_dom.c
+++ b/tools/libxl/libxl_dom.c
@@ -439,6 +439,44 @@ int libxl__build_pre(libxl__gc *gc, uint32_t domid,
     return rc;
 }
 
+static int set_vnuma_affinity(libxl__gc *gc, uint32_t domid,
+                              libxl_domain_build_info *info)
+{
+    libxl_bitmap cpumap;
+    libxl_vnode_info *v;
+    unsigned int i, j;
+    int rc = 0;
+
+    libxl_bitmap_init(&cpumap);
+
+    rc = libxl_cpu_bitmap_alloc(CTX, &cpumap, 0);
+    if (rc) {
+        LOG(ERROR, "Can't allocate nodemap");
+        goto out;
+    }
+
+    /*
+     * For each vcpu in each vnode, set its soft affinity to
+     * the pcpus belonging to the pnode the vnode is on
+     */
+    for (i = 0; i < info->num_vnuma_nodes; i++) {
+        v = &info->vnuma_nodes[i];
+
+        rc = libxl_node_to_cpumap(CTX, v->pnode, &cpumap);
+        if (rc) {
+            LOG(ERROR, "Can't get cpumap for vnode %d", i);
+            goto out;
+        }
+
+        libxl_for_each_set_bit(j, v->vcpus)
+            libxl_set_vcpuaffinity(CTX, domid, j, NULL, &cpumap);
+    }
+
+out:
+    libxl_bitmap_dispose(&cpumap);
+    return rc;
+}
+
 int libxl__build_post(libxl__gc *gc, uint32_t domid,
                       libxl_domain_build_info *info,
                       libxl__domain_build_state *state,
@@ -450,6 +488,12 @@ int libxl__build_post(libxl__gc *gc, uint32_t domid,
     char **ents;
     int i, rc;
 
+    if (info->num_vnuma_nodes && !info->num_vcpu_soft_affinity) {
+        rc = set_vnuma_affinity(gc, domid, info);
+        if (rc)
+            return rc;
+    }
+
     rc = libxl_domain_sched_params_set(CTX, domid, &info->sched_params);
     if (rc)
         return rc;

[PATCH v2 3/3] libxl: cleanup some misuse of 'cpumap' as parameter

[Dario Faggioli]

in favour of the more generic 'bitmap', which is better
since these are generic libxl_bitmap_* functions.

Also fix a typo, and remove a stale (and wrong) comment.

No functional change intended.

Signed-off-by: Dario Faggioli <dario.faggioli@citrix.com>
Cc: Ian Campbell <ian.campbell@citrix.com>
Cc: Ian Jackson <Ian.Jackson@eu.citrix.com>
Cc: Stefano Stabellini <stefano.stabellini@eu.citrix.com>
Cc: Wei Liu <wei.liu2@citrix.com>
Acked-by: Wei Liu <wei.liu2@citrix.com>
---
 tools/libxl/libxl_dom.c   |    2 +-
 tools/libxl/libxl_utils.h |    6 +++---
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/tools/libxl/libxl_dom.c b/tools/libxl/libxl_dom.c
index 554ea68..26a0382 100644
--- a/tools/libxl/libxl_dom.c
+++ b/tools/libxl/libxl_dom.c
@@ -376,7 +376,7 @@ int libxl__build_pre(libxl__gc *gc, uint32_t domid,
     }
     if (info->nodemap.size)
         libxl_domain_set_nodeaffinity(ctx, domid, &info->nodemap);
-    /* As mentioned in libxl.h, vcpu_hard_array takes precedence */
+
     if (info->num_vcpu_hard_affinity || info->num_vcpu_soft_affinity) {
         libxl_bitmap *hard_affinity, *soft_affinity;
         int i, n_vcpus;
diff --git a/tools/libxl/libxl_utils.h b/tools/libxl/libxl_utils.h
index acacdd9..68b5580 100644
--- a/tools/libxl/libxl_utils.h
+++ b/tools/libxl/libxl_utils.h
@@ -89,8 +89,8 @@ int libxl_bitmap_is_empty(const libxl_bitmap *bitmap);
 int libxl_bitmap_test(const libxl_bitmap *bitmap, int bit);
 void libxl_bitmap_set(libxl_bitmap *bitmap, int bit);
 void libxl_bitmap_reset(libxl_bitmap *bitmap, int bit);
-int libxl_bitmap_count_set(const libxl_bitmap *cpumap);
-char *libxl_bitmap_to_hex_string(libxl_ctx *ctx, const libxl_bitmap *cpumap);
+int libxl_bitmap_count_set(const libxl_bitmap *bitmap);
+char *libxl_bitmap_to_hex_string(libxl_ctx *ctx, const libxl_bitmap *bitmap);
 static inline void libxl_bitmap_set_any(libxl_bitmap *bitmap)
 {
     memset(bitmap->map, -1, bitmap->size);
@@ -145,7 +145,7 @@ int libxl_node_to_cpumap(libxl_ctx *ctx, int node,
                          libxl_bitmap *cpumap);
 /* Populate nodemap with the nodes of the cpus in cpumap */
 int libxl_cpumap_to_nodemap(libxl_ctx *ctx,
-                            const libxl_bitmap *cpuemap,
+                            const libxl_bitmap *cpumap,
                             libxl_bitmap *nodemap);
 
  static inline uint32_t libxl__sizekb_to_mb(uint32_t s) {

Re: [PATCH v2 0/3] Automatically derive soft affinity from vnuma information

[Ian Campbell]

On Thu, 2015-03-26 at 09:54 +0100, Dario Faggioli wrote:
> Round 2. All patches have Wei's Ack.

Applied.

Ian.