[Qemu-devel] [PATCH v7 0/4] s390: Support for Hotplug of Standby Memory

[Qemu-devel] [PATCH v7 0/4] s390: Support for Hotplug of Standby Memory

[Matthew Rosato]

This patchset adds support in s390 for a pool of standby memory,
which can be set online/offline by the guest (ie, via chmem).
The standby pool of memory is allocated as the difference between 
the initial memory setting and the maxmem setting.
As part of this work, additional results are provided for the 
Read SCP Information SCLP, and new implentation is added for the 
Read Storage Element Information, Attach Storage Element, 
Assign Storage and Unassign Storage SCLPs, which enables the s390 
guest to manipulate the standby memory pool.

This patchset is based on work originally done by Jeng-Fang (Nick)
Wang.

Sample qemu command snippet:

qemu -machine s390-ccw-virtio  -m 1024M,maxmem=2048M,slots=32 -enable-kvm

This will allocate 1024M of active memory, and another 1024M 
of standby memory.  Example output from s390-tools lsmem:
=============================================================================
0x0000000000000000-0x000000000fffffff        256  online   no         0-127
0x0000000010000000-0x000000001fffffff        256  online   yes        128-255
0x0000000020000000-0x000000003fffffff        512  online   no         256-511
0x0000000040000000-0x000000007fffffff       1024  offline  -          512-1023

Memory device size  : 2 MB
Memory block size   : 256 MB
Total online memory : 1024 MB
Total offline memory: 1024 MB


The guest can dynamically enable part or all of the standby pool 
via the s390-tools chmem, for example:

chmem -e 512M

And can attempt to dynamically disable:

chmem -d 512M

Changes for v7:
 * Added patch to enforce the same memory alignments in s390-virtio.c,
   so that shared code (like sclp) doesn't need to be dual paths.  

Changes for v6:
 * Fix in sclp.h - DeviceState parent --> SysBusDevice parent 
   in struct sclpMemoryHotplugDev.
 * Fix in assign_storage - int this_subregion_size, should 
   be uint64_t.
 * Added information on how to test in the cover letter.  

Changes for v5:
 * Since ACPI memory hotplug is now in, removed Igor's patches 
   from this set.
 * Updated sclp.c to use object_resolve_path() instead of 
   object_property_find().

Matthew Rosato (4):
  sclp-s390: Add device to manage s390 memory hotplug
  virtio-ccw: Include standby memory when calculating storage increment
  s390-virtio: Apply same memory boundaries as virtio-ccw
  sclp-s390: Add memory hotplug SCLPs

 hw/s390x/s390-virtio-ccw.c |   46 +++++--
 hw/s390x/s390-virtio.c     |   15 ++-
 hw/s390x/sclp.c            |  289 +++++++++++++++++++++++++++++++++++++++++++-
 include/hw/s390x/sclp.h    |   20 +++
 qemu-options.hx            |    3 +-
 target-s390x/cpu.h         |   18 +++
 target-s390x/kvm.c         |    5 +
 7 files changed, 375 insertions(+), 21 deletions(-)

-- 
1.7.9.5

[Qemu-devel] [PATCH v7 1/4] sclp-s390: Add device to manage s390 memory hotplug

[Matthew Rosato]

Add sclpMemoryHotplugDev to contain associated data structures, etc.

Signed-off-by: Matthew Rosato <mjrosato@linux.vnet.ibm.com>
---
 hw/s390x/sclp.c         |   30 ++++++++++++++++++++++++++++++
 include/hw/s390x/sclp.h |   20 ++++++++++++++++++++
 2 files changed, 50 insertions(+)

diff --git a/hw/s390x/sclp.c b/hw/s390x/sclp.c
index d8ddf35..769d7c3 100644
--- a/hw/s390x/sclp.c
+++ b/hw/s390x/sclp.c
@@ -183,3 +183,33 @@ void s390_sclp_init(void)
                               OBJECT(dev), NULL);
     qdev_init_nofail(dev);
 }
+
+sclpMemoryHotplugDev *init_sclp_memory_hotplug_dev(void)
+{
+    DeviceState *dev;
+    dev = qdev_create(NULL, TYPE_SCLP_MEMORY_HOTPLUG_DEV);
+    object_property_add_child(qdev_get_machine(),
+                              TYPE_SCLP_MEMORY_HOTPLUG_DEV,
+                              OBJECT(dev), NULL);
+    qdev_init_nofail(dev);
+    return SCLP_MEMORY_HOTPLUG_DEV(object_resolve_path(
+                                   TYPE_SCLP_MEMORY_HOTPLUG_DEV, NULL));
+}
+
+sclpMemoryHotplugDev *get_sclp_memory_hotplug_dev(void)
+{
+    return SCLP_MEMORY_HOTPLUG_DEV(object_resolve_path(
+                                   TYPE_SCLP_MEMORY_HOTPLUG_DEV, NULL));
+}
+
+static TypeInfo sclp_memory_hotplug_dev_info = {
+    .name = TYPE_SCLP_MEMORY_HOTPLUG_DEV,
+    .parent = TYPE_SYS_BUS_DEVICE,
+    .instance_size = sizeof(sclpMemoryHotplugDev),
+};
+
+static void register_types(void)
+{
+    type_register_static(&sclp_memory_hotplug_dev_info);
+}
+type_init(register_types);
diff --git a/include/hw/s390x/sclp.h b/include/hw/s390x/sclp.h
index 7ef1622..5c43574 100644
--- a/include/hw/s390x/sclp.h
+++ b/include/hw/s390x/sclp.h
@@ -37,6 +37,7 @@
 #define SCLP_STARTING_SUBINCREMENT_ID           0x10001
 #define SCLP_INCREMENT_UNIT                     0x10000
 #define MAX_AVAIL_SLOTS                         32
+#define MAX_STORAGE_INCREMENTS                  1020
 
 /* CPU hotplug SCLP codes */
 #define SCLP_HAS_CPU_INFO                       0x0C00000000000000ULL
@@ -156,6 +157,23 @@ typedef struct SCCB {
     char data[SCCB_DATA_LEN];
  } QEMU_PACKED SCCB;
 
+typedef struct sclpMemoryHotplugDev sclpMemoryHotplugDev;
+
+#define TYPE_SCLP_MEMORY_HOTPLUG_DEV "sclp-memory-hotplug-dev"
+#define SCLP_MEMORY_HOTPLUG_DEV(obj) \
+  OBJECT_CHECK(sclpMemoryHotplugDev, (obj), TYPE_SCLP_MEMORY_HOTPLUG_DEV)
+
+struct sclpMemoryHotplugDev {
+    SysBusDevice parent;
+    ram_addr_t standby_mem_size;
+    ram_addr_t padded_ram_size;
+    ram_addr_t pad_size;
+    ram_addr_t standby_subregion_size;
+    ram_addr_t rzm;
+    int increment_size;
+    char *standby_state_map;
+};
+
 static inline int sccb_data_len(SCCB *sccb)
 {
     return be16_to_cpu(sccb->h.length) - sizeof(sccb->h);
@@ -163,6 +181,8 @@ static inline int sccb_data_len(SCCB *sccb)
 
 
 void s390_sclp_init(void);
+sclpMemoryHotplugDev *init_sclp_memory_hotplug_dev(void);
+sclpMemoryHotplugDev *get_sclp_memory_hotplug_dev(void);
 void sclp_service_interrupt(uint32_t sccb);
 void raise_irq_cpu_hotplug(void);
 
-- 
1.7.9.5

[Qemu-devel] [PATCH v7 2/4] virtio-ccw: Include standby memory when calculating storage increment

[Matthew Rosato]

When determining the memory increment size, use the maxmem size if
it was specified.

Signed-off-by: Matthew Rosato <mjrosato@linux.vnet.ibm.com>
---
 hw/s390x/s390-virtio-ccw.c |   46 ++++++++++++++++++++++++++++++++++++--------
 qemu-options.hx            |    3 ++-
 target-s390x/cpu.h         |    3 +++
 3 files changed, 43 insertions(+), 9 deletions(-)

diff --git a/hw/s390x/s390-virtio-ccw.c b/hw/s390x/s390-virtio-ccw.c
index 42f5cec..e736b24 100644
--- a/hw/s390x/s390-virtio-ccw.c
+++ b/hw/s390x/s390-virtio-ccw.c
@@ -17,6 +17,7 @@
 #include "ioinst.h"
 #include "css.h"
 #include "virtio-ccw.h"
+#include "qemu/config-file.h"
 
 void io_subsystem_reset(void)
 {
@@ -84,17 +85,35 @@ static void ccw_init(MachineState *machine)
     ram_addr_t my_ram_size = machine->ram_size;
     MemoryRegion *sysmem = get_system_memory();
     MemoryRegion *ram = g_new(MemoryRegion, 1);
-    int shift = 0;
+    sclpMemoryHotplugDev *mhd = init_sclp_memory_hotplug_dev();
     uint8_t *storage_keys;
     int ret;
     VirtualCssBus *css_bus;
-
-    /* s390x ram size detection needs a 16bit multiplier + an increment. So
-       guests > 64GB can be specified in 2MB steps etc. */
-    while ((my_ram_size >> (20 + shift)) > 65535) {
-        shift++;
+    QemuOpts *opts = qemu_opts_find(qemu_find_opts("memory"), NULL);
+    ram_addr_t pad_size = 0;
+    ram_addr_t maxmem = qemu_opt_get_size(opts, "maxmem", my_ram_size);
+    ram_addr_t standby_mem_size = maxmem - my_ram_size;
+
+    /* The storage increment size is a multiple of 1M and is a power of 2.
+     * The number of storage increments must be MAX_STORAGE_INCREMENTS or fewer.
+     * The variable 'mhd->increment_size' is an exponent of 2 that can be
+     * used to calculate the size (in bytes) of an increment. */
+    mhd->increment_size = 20;
+    while ((my_ram_size >> mhd->increment_size) > MAX_STORAGE_INCREMENTS) {
+        mhd->increment_size++;
+    }
+    while ((standby_mem_size >> mhd->increment_size) > MAX_STORAGE_INCREMENTS) {
+        mhd->increment_size++;
     }
-    my_ram_size = my_ram_size >> (20 + shift) << (20 + shift);
+
+    /* The core and standby memory areas need to be aligned with
+     * the increment size.  In effect, this can cause the
+     * user-specified memory size to be rounded down to align
+     * with the nearest increment boundary. */
+    standby_mem_size = standby_mem_size >> mhd->increment_size
+                                        << mhd->increment_size;
+    my_ram_size = my_ram_size >> mhd->increment_size
+                              << mhd->increment_size;
 
     /* let's propagate the changed ram size into the global variable. */
     ram_size = my_ram_size;
@@ -109,11 +128,22 @@ static void ccw_init(MachineState *machine)
     /* register hypercalls */
     virtio_ccw_register_hcalls();
 
-    /* allocate RAM */
+    /* allocate RAM for core */
     memory_region_init_ram(ram, NULL, "s390.ram", my_ram_size);
     vmstate_register_ram_global(ram);
     memory_region_add_subregion(sysmem, 0, ram);
 
+    /* If the size of ram is not on a MEM_SECTION_SIZE boundary,
+       calculate the pad size necessary to force this boundary. */
+    if (standby_mem_size) {
+        if (my_ram_size % MEM_SECTION_SIZE) {
+            pad_size = MEM_SECTION_SIZE - my_ram_size % MEM_SECTION_SIZE;
+        }
+        my_ram_size += standby_mem_size + pad_size;
+        mhd->pad_size = pad_size;
+        mhd->standby_mem_size = standby_mem_size;
+    }
+
     /* allocate storage keys */
     storage_keys = g_malloc0(my_ram_size / TARGET_PAGE_SIZE);
 
diff --git a/qemu-options.hx b/qemu-options.hx
index 1549625..ab69efd 100644
--- a/qemu-options.hx
+++ b/qemu-options.hx
@@ -225,7 +225,8 @@ DEF("m", HAS_ARG, QEMU_OPTION_m,
     "                size: initial amount of guest memory (default: "
     stringify(DEFAULT_RAM_SIZE) "MiB)\n"
     "                slots: number of hotplug slots (default: none)\n"
-    "                maxmem: maximum amount of guest memory (default: none)\n",
+    "                maxmem: maximum amount of guest memory (default: none)\n"
+    "NOTE: Some architectures might enforce a specific granularity\n",
     QEMU_ARCH_ALL)
 STEXI
 @item -m [size=]@var{megs}
diff --git a/target-s390x/cpu.h b/target-s390x/cpu.h
index b13761d..ba0e4b4 100644
--- a/target-s390x/cpu.h
+++ b/target-s390x/cpu.h
@@ -1045,6 +1045,9 @@ static inline void cpu_inject_crw_mchk(S390CPU *cpu)
     cpu_interrupt(CPU(cpu), CPU_INTERRUPT_HARD);
 }
 
+/* from s390-virtio-ccw */
+#define MEM_SECTION_SIZE             0x10000000UL
+
 /* fpu_helper.c */
 uint32_t set_cc_nz_f32(float32 v);
 uint32_t set_cc_nz_f64(float64 v);
-- 
1.7.9.5

[Qemu-devel] [PATCH v7 3/4] s390-virtio: Apply same memory boundaries as virtio-ccw

[Matthew Rosato]

Although s390-virtio won't support memory hotplug, it should
enforce the same memory boundaries so that it can use shared codepaths
(like read_SCP_info).

Signed-off-by: Matthew Rosato <mjrosato@linux.vnet.ibm.com>
---
 hw/s390x/s390-virtio.c |   15 +++++++++------
 1 file changed, 9 insertions(+), 6 deletions(-)

diff --git a/hw/s390x/s390-virtio.c b/hw/s390x/s390-virtio.c
index 93c7ace..857f8d3 100644
--- a/hw/s390x/s390-virtio.c
+++ b/hw/s390x/s390-virtio.c
@@ -229,18 +229,21 @@ static void s390_init(MachineState *machine)
     ram_addr_t my_ram_size = machine->ram_size;
     MemoryRegion *sysmem = get_system_memory();
     MemoryRegion *ram = g_new(MemoryRegion, 1);
-    int shift = 0;
+    int increment_size = 20;
     uint8_t *storage_keys;
     void *virtio_region;
     hwaddr virtio_region_len;
     hwaddr virtio_region_start;
 
-    /* s390x ram size detection needs a 16bit multiplier + an increment. So
-       guests > 64GB can be specified in 2MB steps etc. */
-    while ((my_ram_size >> (20 + shift)) > 65535) {
-        shift++;
+    /*
+     * The storage increment size is a multiple of 1M and is a power of 2.
+     * The number of storage increments must be MAX_STORAGE_INCREMENTS or
+     * fewer.
+     */
+    while ((my_ram_size >> increment_size) > MAX_STORAGE_INCREMENTS) {
+        increment_size++;
     }
-    my_ram_size = my_ram_size >> (20 + shift) << (20 + shift);
+    my_ram_size = my_ram_size >> increment_size << increment_size;
 
     /* let's propagate the changed ram size into the global variable. */
     ram_size = my_ram_size;
-- 
1.7.9.5

[Qemu-devel] [PATCH v7 4/4] sclp-s390: Add memory hotplug SCLPs

[Matthew Rosato]

Add memory information to read SCP info and add handlers for
Read Storage Element Information, Attach Storage Element,
Assign Storage and Unassign Storage.

Signed-off-by: Matthew Rosato <mjrosato@linux.vnet.ibm.com>
---
 hw/s390x/sclp.c    |  259 ++++++++++++++++++++++++++++++++++++++++++++++++++--
 target-s390x/cpu.h |   15 +++
 target-s390x/kvm.c |    5 +
 3 files changed, 273 insertions(+), 6 deletions(-)

diff --git a/hw/s390x/sclp.c b/hw/s390x/sclp.c
index 769d7c3..936b189 100644
--- a/hw/s390x/sclp.c
+++ b/hw/s390x/sclp.c
@@ -16,7 +16,8 @@
 #include "sysemu/kvm.h"
 #include "exec/memory.h"
 #include "sysemu/sysemu.h"
-
+#include "exec/address-spaces.h"
+#include "qemu/config-file.h"
 #include "hw/s390x/sclp.h"
 #include "hw/s390x/event-facility.h"
 
@@ -33,10 +34,19 @@ static inline SCLPEventFacility *get_event_facility(void)
 static void read_SCP_info(SCCB *sccb)
 {
     ReadInfo *read_info = (ReadInfo *) sccb;
+    sclpMemoryHotplugDev *mhd = get_sclp_memory_hotplug_dev();
     CPUState *cpu;
-    int shift = 0;
     int cpu_count = 0;
     int i = 0;
+    int increment_size = 20;
+    int rnsize, rnmax;
+    QemuOpts *opts = qemu_opts_find(qemu_find_opts("memory"), NULL);
+    int slots = qemu_opt_get_number(opts, "slots", 0);
+    int max_avail_slots = s390_get_memslot_count(kvm_state);
+
+    if (slots > max_avail_slots) {
+        slots = max_avail_slots;
+    }
 
     CPU_FOREACH(cpu) {
         cpu_count++;
@@ -54,14 +64,235 @@ static void read_SCP_info(SCCB *sccb)
 
     read_info->facilities = cpu_to_be64(SCLP_HAS_CPU_INFO);
 
-    while ((ram_size >> (20 + shift)) > 65535) {
-        shift++;
+    /*
+     * The storage increment size is a multiple of 1M and is a power of 2.
+     * The number of storage increments must be MAX_STORAGE_INCREMENTS or fewer.
+     */
+    while ((ram_size >> increment_size) > MAX_STORAGE_INCREMENTS) {
+        increment_size++;
+    }
+    rnmax = ram_size >> increment_size;
+
+    /* Memory Hotplug is only supported for the ccw machine type */
+    if (mhd) {
+        while ((mhd->standby_mem_size >> increment_size) >
+               MAX_STORAGE_INCREMENTS) {
+            increment_size++;
+        }
+        assert(increment_size == mhd->increment_size);
+
+        mhd->standby_subregion_size = MEM_SECTION_SIZE;
+        /* Deduct the memory slot already used for core */
+        if (slots > 0) {
+            while ((mhd->standby_subregion_size * (slots - 1)
+                    < mhd->standby_mem_size)) {
+                mhd->standby_subregion_size = mhd->standby_subregion_size << 1;
+            }
+        }
+        /*
+         * Initialize mapping of guest standby memory sections indicating which
+         * are and are not online. Assume all standby memory begins offline.
+         */
+        if (mhd->standby_state_map == 0) {
+            if (mhd->standby_mem_size % mhd->standby_subregion_size) {
+                mhd->standby_state_map = g_malloc0((mhd->standby_mem_size /
+                                             mhd->standby_subregion_size + 1) *
+                                             (mhd->standby_subregion_size /
+                                             MEM_SECTION_SIZE));
+            } else {
+                mhd->standby_state_map = g_malloc0(mhd->standby_mem_size /
+                                                   MEM_SECTION_SIZE);
+            }
+        }
+        mhd->padded_ram_size = ram_size + mhd->pad_size;
+        mhd->rzm = 1 << mhd->increment_size;
+        rnmax = ((ram_size + mhd->standby_mem_size + mhd->pad_size)
+             >> mhd->increment_size);
+
+        read_info->facilities |= cpu_to_be64(SCLP_FC_ASSIGN_ATTACH_READ_STOR);
+    }
+
+    rnsize = 1 << (increment_size - 20);
+    if (rnsize <= 128) {
+        read_info->rnsize = rnsize;
+    } else {
+        read_info->rnsize = 0;
+        read_info->rnsize2 = cpu_to_be32(rnsize);
     }
-    read_info->rnmax = cpu_to_be16(ram_size >> (20 + shift));
-    read_info->rnsize = 1 << shift;
+
+    if (rnmax < 0x10000) {
+        read_info->rnmax = cpu_to_be16(rnmax);
+    } else {
+        read_info->rnmax = cpu_to_be16(0);
+        read_info->rnmax2 = cpu_to_be64(rnmax);
+    }
+
     sccb->h.response_code = cpu_to_be16(SCLP_RC_NORMAL_READ_COMPLETION);
 }
 
+static void read_storage_element0_info(SCCB *sccb)
+{
+    int i, assigned;
+    int subincrement_id = SCLP_STARTING_SUBINCREMENT_ID;
+    ReadStorageElementInfo *storage_info = (ReadStorageElementInfo *) sccb;
+    sclpMemoryHotplugDev *mhd = get_sclp_memory_hotplug_dev();
+
+    assert(mhd);
+
+    if ((ram_size >> mhd->increment_size) >= 0x10000) {
+        sccb->h.response_code = cpu_to_be16(SCLP_RC_SCCB_BOUNDARY_VIOLATION);
+        return;
+    }
+
+    /* Return information regarding core memory */
+    storage_info->max_id = cpu_to_be16(mhd->standby_mem_size ? 1 : 0);
+    assigned = ram_size >> mhd->increment_size;
+    storage_info->assigned = cpu_to_be16(assigned);
+
+    for (i = 0; i < assigned; i++) {
+        storage_info->entries[i] = cpu_to_be32(subincrement_id);
+        subincrement_id += SCLP_INCREMENT_UNIT;
+    }
+    sccb->h.response_code = cpu_to_be16(SCLP_RC_NORMAL_READ_COMPLETION);
+}
+
+static void read_storage_element1_info(SCCB *sccb)
+{
+    ReadStorageElementInfo *storage_info = (ReadStorageElementInfo *) sccb;
+    sclpMemoryHotplugDev *mhd = get_sclp_memory_hotplug_dev();
+
+    assert(mhd);
+
+    if ((mhd->standby_mem_size >> mhd->increment_size) >= 0x10000) {
+        sccb->h.response_code = cpu_to_be16(SCLP_RC_SCCB_BOUNDARY_VIOLATION);
+        return;
+    }
+
+    /* Return information regarding standby memory */
+    storage_info->max_id = cpu_to_be16(mhd->standby_mem_size ? 1 : 0);
+    storage_info->assigned = cpu_to_be16(mhd->standby_mem_size >>
+                                         mhd->increment_size);
+    storage_info->standby = cpu_to_be16(mhd->standby_mem_size >>
+                                        mhd->increment_size);
+    sccb->h.response_code = cpu_to_be16(SCLP_RC_STANDBY_READ_COMPLETION);
+}
+
+static void attach_storage_element(SCCB *sccb, uint16_t element)
+{
+    int i, assigned, subincrement_id;
+    AttachStorageElement *attach_info = (AttachStorageElement *) sccb;
+    sclpMemoryHotplugDev *mhd = get_sclp_memory_hotplug_dev();
+
+    assert(mhd);
+
+    if (element != 1) {
+        sccb->h.response_code = cpu_to_be16(SCLP_RC_INVALID_SCLP_COMMAND);
+        return;
+    }
+
+    assigned = mhd->standby_mem_size >> mhd->increment_size;
+    attach_info->assigned = cpu_to_be16(assigned);
+    subincrement_id = ((ram_size >> mhd->increment_size) << 16)
+                      + SCLP_STARTING_SUBINCREMENT_ID;
+    for (i = 0; i < assigned; i++) {
+        attach_info->entries[i] = cpu_to_be32(subincrement_id);
+        subincrement_id += SCLP_INCREMENT_UNIT;
+    }
+    sccb->h.response_code = cpu_to_be16(SCLP_RC_NORMAL_COMPLETION);
+}
+
+static void assign_storage(SCCB *sccb)
+{
+    MemoryRegion *mr = NULL;
+    uint64_t this_subregion_size;
+    AssignStorage *assign_info = (AssignStorage *) sccb;
+    sclpMemoryHotplugDev *mhd = get_sclp_memory_hotplug_dev();
+    assert(mhd);
+    ram_addr_t assign_addr = (assign_info->rn - 1) * mhd->rzm;
+    MemoryRegion *sysmem = get_system_memory();
+
+    if ((assign_addr % MEM_SECTION_SIZE == 0) &&
+        (assign_addr >= mhd->padded_ram_size)) {
+        /* Re-use existing memory region if found */
+        mr = memory_region_find(sysmem, assign_addr, 1).mr;
+        if (!mr) {
+
+            MemoryRegion *standby_ram = g_new(MemoryRegion, 1);
+
+            /* offset to align to standby_subregion_size for allocation */
+            ram_addr_t offset = assign_addr -
+                                (assign_addr - mhd->padded_ram_size)
+                                % mhd->standby_subregion_size;
+
+            /* strlen("standby.ram") + 4 (Max of KVM_MEMORY_SLOTS) +  NULL */
+            char id[16];
+            snprintf(id, 16, "standby.ram%d",
+                     (int)((offset - mhd->padded_ram_size) /
+                     mhd->standby_subregion_size) + 1);
+
+            /* Allocate a subregion of the calculated standby_subregion_size */
+            if (offset + mhd->standby_subregion_size >
+                mhd->padded_ram_size + mhd->standby_mem_size) {
+                this_subregion_size = mhd->padded_ram_size +
+                  mhd->standby_mem_size - offset;
+            } else {
+                this_subregion_size = mhd->standby_subregion_size;
+            }
+
+            memory_region_init_ram(standby_ram, NULL, id, this_subregion_size);
+            vmstate_register_ram_global(standby_ram);
+            memory_region_add_subregion(sysmem, offset, standby_ram);
+        }
+        /* The specified subregion is no longer in standby */
+        mhd->standby_state_map[(assign_addr - mhd->padded_ram_size)
+                               / MEM_SECTION_SIZE] = 1;
+    }
+    sccb->h.response_code = cpu_to_be16(SCLP_RC_NORMAL_COMPLETION);
+}
+
+static void unassign_storage(SCCB *sccb)
+{
+    MemoryRegion *mr = NULL;
+    AssignStorage *assign_info = (AssignStorage *) sccb;
+    sclpMemoryHotplugDev *mhd = get_sclp_memory_hotplug_dev();
+    assert(mhd);
+    ram_addr_t unassign_addr = (assign_info->rn - 1) * mhd->rzm;
+    MemoryRegion *sysmem = get_system_memory();
+
+    /* if the addr is a multiple of 256 MB */
+    if ((unassign_addr % MEM_SECTION_SIZE == 0) &&
+        (unassign_addr >= mhd->padded_ram_size)) {
+        mhd->standby_state_map[(unassign_addr -
+                           mhd->padded_ram_size) / MEM_SECTION_SIZE] = 0;
+
+        /* find the specified memory region and destroy it */
+        mr = memory_region_find(sysmem, unassign_addr, 1).mr;
+        if (mr) {
+            int i;
+            int is_removable = 1;
+            ram_addr_t map_offset = (unassign_addr - mhd->padded_ram_size -
+                                     (unassign_addr - mhd->padded_ram_size)
+                                     % mhd->standby_subregion_size);
+            /* Mark all affected subregions as 'standby' once again */
+            for (i = 0;
+                 i < (mhd->standby_subregion_size / MEM_SECTION_SIZE);
+                 i++) {
+
+                if (mhd->standby_state_map[i + map_offset / MEM_SECTION_SIZE]) {
+                    is_removable = 0;
+                    break;
+                }
+            }
+            if (is_removable) {
+                memory_region_del_subregion(sysmem, mr);
+                memory_region_destroy(mr);
+                g_free(mr);
+            }
+        }
+    }
+    sccb->h.response_code = cpu_to_be16(SCLP_RC_NORMAL_COMPLETION);
+}
+
 /* Provide information about the CPU */
 static void sclp_read_cpu_info(SCCB *sccb)
 {
@@ -103,6 +334,22 @@ static void sclp_execute(SCCB *sccb, uint32_t code)
     case SCLP_CMDW_READ_CPU_INFO:
         sclp_read_cpu_info(sccb);
         break;
+    case SCLP_READ_STORAGE_ELEMENT_INFO:
+        if (code & 0xff00) {
+            read_storage_element1_info(sccb);
+        } else {
+            read_storage_element0_info(sccb);
+        }
+        break;
+    case SCLP_ATTACH_STORAGE_ELEMENT:
+        attach_storage_element(sccb, (code & 0xff00) >> 8);
+        break;
+    case SCLP_ASSIGN_STORAGE:
+        assign_storage(sccb);
+        break;
+    case SCLP_UNASSIGN_STORAGE:
+        unassign_storage(sccb);
+        break;
     default:
         efc->command_handler(ef, sccb, code);
         break;
diff --git a/target-s390x/cpu.h b/target-s390x/cpu.h
index ba0e4b4..1c1681c 100644
--- a/target-s390x/cpu.h
+++ b/target-s390x/cpu.h
@@ -1047,6 +1047,7 @@ static inline void cpu_inject_crw_mchk(S390CPU *cpu)
 
 /* from s390-virtio-ccw */
 #define MEM_SECTION_SIZE             0x10000000UL
+#define MAX_AVAIL_SLOTS              32
 
 /* fpu_helper.c */
 uint32_t set_cc_nz_f32(float32 v);
@@ -1070,6 +1071,7 @@ void kvm_s390_enable_css_support(S390CPU *cpu);
 int kvm_s390_assign_subch_ioeventfd(EventNotifier *notifier, uint32_t sch,
                                     int vq, bool assign);
 int kvm_s390_cpu_restart(S390CPU *cpu);
+int kvm_s390_get_memslot_count(KVMState *s);
 void kvm_s390_clear_cmma_callback(void *opaque);
 #else
 static inline void kvm_s390_io_interrupt(uint16_t subchannel_id,
@@ -1097,6 +1099,10 @@ static inline int kvm_s390_cpu_restart(S390CPU *cpu)
 static inline void kvm_s390_clear_cmma_callback(void *opaque)
 {
 }
+static inline int kvm_s390_get_memslot_count(KVMState *s)
+{
+  return MAX_AVAIL_SLOTS;
+}
 #endif
 
 static inline void cmma_reset(S390CPU *cpu)
@@ -1115,6 +1121,15 @@ static inline int s390_cpu_restart(S390CPU *cpu)
     return -ENOSYS;
 }
 
+static inline int s390_get_memslot_count(KVMState *s)
+{
+    if (kvm_enabled()) {
+        return kvm_s390_get_memslot_count(s);
+    } else {
+        return MAX_AVAIL_SLOTS;
+    }
+}
+
 void s390_io_interrupt(uint16_t subchannel_id, uint16_t subchannel_nr,
                        uint32_t io_int_parm, uint32_t io_int_word);
 void s390_crw_mchk(void);
diff --git a/target-s390x/kvm.c b/target-s390x/kvm.c
index a32d91a..9f2b586 100644
--- a/target-s390x/kvm.c
+++ b/target-s390x/kvm.c
@@ -1306,3 +1306,8 @@ int kvm_s390_assign_subch_ioeventfd(EventNotifier *notifier, uint32_t sch,
     }
     return kvm_vm_ioctl(kvm_state, KVM_IOEVENTFD, &kick);
 }
+
+int kvm_s390_get_memslot_count(KVMState *s)
+{
+    return kvm_check_extension(s, KVM_CAP_NR_MEMSLOTS);
+}
-- 
1.7.9.5

Re: [Qemu-devel] [PATCH v7 0/4] s390: Support for Hotplug of Standby Memory

[Matthew Rosato]

On 07/30/2014 02:15 PM, Matthew Rosato wrote:
> This patchset adds support in s390 for a pool of standby memory,
> which can be set online/offline by the guest (ie, via chmem).
> The standby pool of memory is allocated as the difference between 
> the initial memory setting and the maxmem setting.
> As part of this work, additional results are provided for the 
> Read SCP Information SCLP, and new implentation is added for the 
> Read Storage Element Information, Attach Storage Element, 
> Assign Storage and Unassign Storage SCLPs, which enables the s390 
> guest to manipulate the standby memory pool.
> 
> This patchset is based on work originally done by Jeng-Fang (Nick)
> Wang.
> 
> Sample qemu command snippet:
> 
> qemu -machine s390-ccw-virtio  -m 1024M,maxmem=2048M,slots=32 -enable-kvm
> 
> This will allocate 1024M of active memory, and another 1024M 
> of standby memory.  Example output from s390-tools lsmem:
> =============================================================================
> 0x0000000000000000-0x000000000fffffff        256  online   no         0-127
> 0x0000000010000000-0x000000001fffffff        256  online   yes        128-255
> 0x0000000020000000-0x000000003fffffff        512  online   no         256-511
> 0x0000000040000000-0x000000007fffffff       1024  offline  -          512-1023
> 
> Memory device size  : 2 MB
> Memory block size   : 256 MB
> Total online memory : 1024 MB
> Total offline memory: 1024 MB
> 
> 
> The guest can dynamically enable part or all of the standby pool 
> via the s390-tools chmem, for example:
> 
> chmem -e 512M
> 
> And can attempt to dynamically disable:
> 
> chmem -d 512M
> 
> Changes for v7:
>  * Added patch to enforce the same memory alignments in s390-virtio.c,
>    so that shared code (like sclp) doesn't need to be dual paths.  

Ping...

> 
> Changes for v6:
>  * Fix in sclp.h - DeviceState parent --> SysBusDevice parent 
>    in struct sclpMemoryHotplugDev.
>  * Fix in assign_storage - int this_subregion_size, should 
>    be uint64_t.
>  * Added information on how to test in the cover letter.  
> 
> Changes for v5:
>  * Since ACPI memory hotplug is now in, removed Igor's patches 
>    from this set.
>  * Updated sclp.c to use object_resolve_path() instead of 
>    object_property_find().
> 
> Matthew Rosato (4):
>   sclp-s390: Add device to manage s390 memory hotplug
>   virtio-ccw: Include standby memory when calculating storage increment
>   s390-virtio: Apply same memory boundaries as virtio-ccw
>   sclp-s390: Add memory hotplug SCLPs
> 
>  hw/s390x/s390-virtio-ccw.c |   46 +++++--
>  hw/s390x/s390-virtio.c     |   15 ++-
>  hw/s390x/sclp.c            |  289 +++++++++++++++++++++++++++++++++++++++++++-
>  include/hw/s390x/sclp.h    |   20 +++
>  qemu-options.hx            |    3 +-
>  target-s390x/cpu.h         |   18 +++
>  target-s390x/kvm.c         |    5 +
>  7 files changed, 375 insertions(+), 21 deletions(-)
> 

Re: [Qemu-devel] [PATCH v7 0/4] s390: Support for Hotplug of Standby Memory

[Christian Borntraeger]

On 30/07/14 20:15, Matthew Rosato wrote:
> This patchset adds support in s390 for a pool of standby memory,
> which can be set online/offline by the guest (ie, via chmem).
> The standby pool of memory is allocated as the difference between 
> the initial memory setting and the maxmem setting.
> As part of this work, additional results are provided for the 
> Read SCP Information SCLP, and new implentation is added for the 
> Read Storage Element Information, Attach Storage Element, 
> Assign Storage and Unassign Storage SCLPs, which enables the s390 
> guest to manipulate the standby memory pool.
> 
> This patchset is based on work originally done by Jeng-Fang (Nick)
> Wang.

Can you respin against latest upstream.

commit d8d95814609e89e5438a3318a647ec322fc4ff16
    memory: convert memory_region_destroy to object_unparent

and followon now cause:

  CC    s390x-softmmu/trace/generated-helpers.o
qemu/hw/s390x/sclp.c: In function ‘unassign_storage’:
qemu/hw/s390x/sclp.c:288:17: error: implicit declaration of function ‘memory_region_destroy’ [-Werror=implicit-function-declaration]
qemu/hw/s390x/sclp.c:288:17: error: nested extern declaration of ‘memory_region_destroy’ [-Werror=nested-externs]
cc1: all warnings being treated as errors



> 
> Sample qemu command snippet:
> 
> qemu -machine s390-ccw-virtio  -m 1024M,maxmem=2048M,slots=32 -enable-kvm
> 
> This will allocate 1024M of active memory, and another 1024M 
> of standby memory.  Example output from s390-tools lsmem:
> =============================================================================
> 0x0000000000000000-0x000000000fffffff        256  online   no         0-127
> 0x0000000010000000-0x000000001fffffff        256  online   yes        128-255
> 0x0000000020000000-0x000000003fffffff        512  online   no         256-511
> 0x0000000040000000-0x000000007fffffff       1024  offline  -          512-1023
> 
> Memory device size  : 2 MB
> Memory block size   : 256 MB
> Total online memory : 1024 MB
> Total offline memory: 1024 MB
> 
> 
> The guest can dynamically enable part or all of the standby pool 
> via the s390-tools chmem, for example:
> 
> chmem -e 512M
> 
> And can attempt to dynamically disable:
> 
> chmem -d 512M
> 
> Changes for v7:
>  * Added patch to enforce the same memory alignments in s390-virtio.c,
>    so that shared code (like sclp) doesn't need to be dual paths.  
> 
> Changes for v6:
>  * Fix in sclp.h - DeviceState parent --> SysBusDevice parent 
>    in struct sclpMemoryHotplugDev.
>  * Fix in assign_storage - int this_subregion_size, should 
>    be uint64_t.
>  * Added information on how to test in the cover letter.  
> 
> Changes for v5:
>  * Since ACPI memory hotplug is now in, removed Igor's patches 
>    from this set.
>  * Updated sclp.c to use object_resolve_path() instead of 
>    object_property_find().
> 
> Matthew Rosato (4):
>   sclp-s390: Add device to manage s390 memory hotplug
>   virtio-ccw: Include standby memory when calculating storage increment
>   s390-virtio: Apply same memory boundaries as virtio-ccw
>   sclp-s390: Add memory hotplug SCLPs
> 
>  hw/s390x/s390-virtio-ccw.c |   46 +++++--
>  hw/s390x/s390-virtio.c     |   15 ++-
>  hw/s390x/sclp.c            |  289 +++++++++++++++++++++++++++++++++++++++++++-
>  include/hw/s390x/sclp.h    |   20 +++
>  qemu-options.hx            |    3 +-
>  target-s390x/cpu.h         |   18 +++
>  target-s390x/kvm.c         |    5 +
>  7 files changed, 375 insertions(+), 21 deletions(-)
>