[PATCH v9 00/15] s390x: Protected Virtualization support

[PATCH v9 00/15] s390x: Protected Virtualization support

[Janosch Frank]

Most of the QEMU changes for PV are related to the new IPL type with
subcodes 8 - 10 and the execution of the necessary Ultravisor calls to
IPL secure guests. Note that we can only boot into secure mode from
normal mode, i.e. stfle 161 is not active in secure mode.

The other changes related to data gathering for emulation and
disabling addressing checks in secure mode, as well as CPU resets.

v9:
	* Moved pv.h into include/hw/s390x/
	* Replaced cmd strings with macro
	* Moved s390_is_pv() to pv.h
	* Added new copyright dates and authors
v8:
	* Removed the iplb_valid changes as they are picked
	* Checkpatch fixes
	* Review fixes
	* Replaced env/ms->pv with s390_is_pv()
v7:
	* Merged the diag 308 subcode patches and the unpack
	* Moved the SIDA memops into the sync patch
	* Bailout for the none machien and fencing of CONFIG_USER_ONLY
	* Changes due to review

v6:
	* diag308 rc numbers were changed by architecture
	* IPL pv block received one more reserved field by architecture
	* Officially added the bios patch to the series
	* Dropped picked constant rename patch

v5:
	* Moved docs into docs/system
	* Some more enable/disable changes
	* Moved enablement/disablement of pv in separate functions
	* Some review fixes

v4:
	* Sync with KVM changes
	* Review changes

V3:
	* Use dedicated functions to access SIDA
	* Smaller cleanups and segfault fixes
	* Error reporting for Ultravisor calls
	* Inject of RC of diag308 subcode 10 fails

V2:
	* Split out cleanups
	* Internal PV state tracking
	* Review feedback

Christian Borntraeger (1):
  s390x: Add unpack facility feature to GA1

Janosch Frank (14):
  Sync pv
  s390x: protvirt: Support unpack facility
  s390x: protvirt: Add migration blocker
  s390x: protvirt: Inhibit balloon when switching to protected mode
  s390x: protvirt: KVM intercept changes
  s390x: Add SIDA memory ops
  s390x: protvirt: Move STSI data over SIDAD
  s390x: protvirt: SCLP interpretation
  s390x: protvirt: Set guest IPL PSW
  s390x: protvirt: Move diag 308 data over SIDA
  s390x: protvirt: Disable address checks for PV guest IO emulation
  s390x: protvirt: Move IO control structures over SIDA
  s390x: protvirt: Handle SIGP store status correctly
  docs: Add protvirt docs

 docs/system/index.rst               |   1 +
 docs/system/protvirt.rst            |  56 +++++++++++
 hw/s390x/Makefile.objs              |   1 +
 hw/s390x/ipl.c                      |  56 ++++++++++-
 hw/s390x/ipl.h                      |  79 ++++++++++++++++
 hw/s390x/pv.c                       |  98 ++++++++++++++++++++
 hw/s390x/s390-virtio-ccw.c          | 139 +++++++++++++++++++++++++++-
 hw/s390x/sclp.c                     |  30 ++++++
 include/hw/s390x/pv.h               |  57 ++++++++++++
 include/hw/s390x/s390-virtio-ccw.h  |   1 +
 include/hw/s390x/sclp.h             |   2 +
 linux-headers/linux/kvm.h           |  45 ++++++++-
 target/s390x/cpu.c                  |  24 +++--
 target/s390x/cpu.h                  |   7 +-
 target/s390x/cpu_features_def.inc.h |   1 +
 target/s390x/diag.c                 |  58 ++++++++++--
 target/s390x/gen-features.c         |   1 +
 target/s390x/helper.c               |   6 ++
 target/s390x/ioinst.c               | 120 +++++++++++++++++-------
 target/s390x/kvm.c                  |  69 ++++++++++++--
 target/s390x/kvm_s390x.h            |   2 +
 target/s390x/mmu_helper.c           |  14 +++
 22 files changed, 803 insertions(+), 64 deletions(-)
 create mode 100644 docs/system/protvirt.rst
 create mode 100644 hw/s390x/pv.c
 create mode 100644 include/hw/s390x/pv.h

-- 
2.25.1

[PATCH v9 01/15] Sync pv

[Janosch Frank]

Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
---
 linux-headers/linux/kvm.h | 45 +++++++++++++++++++++++++++++++++++++--
 1 file changed, 43 insertions(+), 2 deletions(-)

diff --git a/linux-headers/linux/kvm.h b/linux-headers/linux/kvm.h
index 265099100e65b3e8..c30344ab0095e856 100644
--- a/linux-headers/linux/kvm.h
+++ b/linux-headers/linux/kvm.h
@@ -474,12 +474,17 @@ struct kvm_s390_mem_op {
 	__u32 size;		/* amount of bytes */
 	__u32 op;		/* type of operation */
 	__u64 buf;		/* buffer in userspace */
-	__u8 ar;		/* the access register number */
-	__u8 reserved[31];	/* should be set to 0 */
+	union {
+		__u8 ar;	/* the access register number */
+		__u32 sida_offset; /* offset into the sida */
+		__u8 reserved[32]; /* should be set to 0 */
+	};
 };
 /* types for kvm_s390_mem_op->op */
 #define KVM_S390_MEMOP_LOGICAL_READ	0
 #define KVM_S390_MEMOP_LOGICAL_WRITE	1
+#define KVM_S390_MEMOP_SIDA_READ	2
+#define KVM_S390_MEMOP_SIDA_WRITE	3
 /* flags for kvm_s390_mem_op->flags */
 #define KVM_S390_MEMOP_F_CHECK_ONLY		(1ULL << 0)
 #define KVM_S390_MEMOP_F_INJECT_EXCEPTION	(1ULL << 1)
@@ -1010,6 +1015,7 @@ struct kvm_ppc_resize_hpt {
 #define KVM_CAP_ARM_NISV_TO_USER 177
 #define KVM_CAP_ARM_INJECT_EXT_DABT 178
 #define KVM_CAP_S390_VCPU_RESETS 179
+#define KVM_CAP_S390_PROTECTED 180
 
 #ifdef KVM_CAP_IRQ_ROUTING
 
@@ -1478,6 +1484,41 @@ struct kvm_enc_region {
 #define KVM_S390_NORMAL_RESET	_IO(KVMIO,   0xc3)
 #define KVM_S390_CLEAR_RESET	_IO(KVMIO,   0xc4)
 
+struct kvm_s390_pv_sec_parm {
+	__u64 origin;
+	__u64 length;
+};
+
+struct kvm_s390_pv_unp {
+	__u64 addr;
+	__u64 size;
+	__u64 tweak;
+};
+
+enum pv_cmd_id {
+	KVM_PV_ENABLE,
+	KVM_PV_DISABLE,
+	KVM_PV_VM_SET_SEC_PARMS,
+	KVM_PV_VM_UNPACK,
+	KVM_PV_VM_VERIFY,
+	KVM_PV_VM_PREP_RESET,
+	KVM_PV_VM_UNSHARE_ALL,
+	KVM_PV_VCPU_CREATE,
+	KVM_PV_VCPU_DESTROY,
+};
+
+struct kvm_pv_cmd {
+	__u32 cmd;	/* Command to be executed */
+	__u16 rc;	/* Ultravisor return code */
+	__u16 rrc;	/* Ultravisor return reason code */
+	__u64 data;	/* Data or address */
+	__u32 flags;    /* flags for future extensions. Must be 0 for now */
+	__u32 reserved[3];
+};
+
+/* Available with KVM_CAP_S390_PROTECTED */
+#define KVM_S390_PV_COMMAND		_IOWR(KVMIO, 0xc5, struct kvm_pv_cmd)
+
 /* Secure Encrypted Virtualization command */
 enum sev_cmd_id {
 	/* Guest initialization commands */
-- 
2.25.1

[PATCH v9 02/15] s390x: protvirt: Support unpack facility

[Janosch Frank]

The unpack facility provides the means to setup a protected guest. A
protected guest cannot be introspected by the hypervisor or any
user/administrator of the machine it is running on.

Protected guests are encrypted at rest and need a special boot
mechanism via diag308 subcode 8 and 10.

Code 8 sets the PV specific IPLB which is retained separately from
those set via code 5.

Code 10 is used to unpack the VM into protected memory, verify its
integrity and start it.

Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
Co-developed-by: Christian Borntraeger <borntraeger@de.ibm.com> [Changes
to machine]
---
 hw/s390x/Makefile.objs              |   1 +
 hw/s390x/ipl.c                      |  56 +++++++++++++-
 hw/s390x/ipl.h                      |  79 +++++++++++++++++++
 hw/s390x/pv.c                       |  98 +++++++++++++++++++++++
 hw/s390x/s390-virtio-ccw.c          | 116 +++++++++++++++++++++++++++-
 include/hw/s390x/pv.h               |  57 ++++++++++++++
 include/hw/s390x/s390-virtio-ccw.h  |   1 +
 target/s390x/cpu.c                  |   2 +
 target/s390x/cpu_features_def.inc.h |   1 +
 target/s390x/diag.c                 |  31 +++++++-
 10 files changed, 436 insertions(+), 6 deletions(-)
 create mode 100644 hw/s390x/pv.c
 create mode 100644 include/hw/s390x/pv.h

diff --git a/hw/s390x/Makefile.objs b/hw/s390x/Makefile.objs
index e02ed80b6829a511..a46a1c7894e0f612 100644
--- a/hw/s390x/Makefile.objs
+++ b/hw/s390x/Makefile.objs
@@ -31,6 +31,7 @@ obj-y += tod-qemu.o
 obj-$(CONFIG_KVM) += tod-kvm.o
 obj-$(CONFIG_KVM) += s390-skeys-kvm.o
 obj-$(CONFIG_KVM) += s390-stattrib-kvm.o
+obj-$(CONFIG_KVM) += pv.o
 obj-y += s390-ccw.o
 obj-y += ap-device.o
 obj-y += ap-bridge.o
diff --git a/hw/s390x/ipl.c b/hw/s390x/ipl.c
index b81942e1e6f9002e..98df89e62c25f583 100644
--- a/hw/s390x/ipl.c
+++ b/hw/s390x/ipl.c
@@ -27,6 +27,7 @@
 #include "hw/s390x/vfio-ccw.h"
 #include "hw/s390x/css.h"
 #include "hw/s390x/ebcdic.h"
+#include "hw/s390x/pv.h"
 #include "ipl.h"
 #include "qemu/error-report.h"
 #include "qemu/config-file.h"
@@ -566,12 +567,31 @@ void s390_ipl_update_diag308(IplParameterBlock *iplb)
 {
     S390IPLState *ipl = get_ipl_device();
 
-    ipl->iplb = *iplb;
-    ipl->iplb_valid = true;
+    /*
+     * The IPLB set and retrieved by subcodes 8/9 is completely
+     * separate from the one managed via subcodes 5/6.
+     */
+    if (iplb->pbt == S390_IPL_TYPE_PV) {
+        ipl->iplb_pv = *iplb;
+        ipl->iplb_valid_pv = true;
+    } else {
+        ipl->iplb = *iplb;
+        ipl->iplb_valid = true;
+    }
     ipl->netboot = is_virtio_net_device(iplb);
     update_machine_ipl_properties(iplb);
 }
 
+IplParameterBlock *s390_ipl_get_iplb_pv(void)
+{
+    S390IPLState *ipl = get_ipl_device();
+
+    if (!ipl->iplb_valid_pv) {
+        return NULL;
+    }
+    return &ipl->iplb_pv;
+}
+
 IplParameterBlock *s390_ipl_get_iplb(void)
 {
     S390IPLState *ipl = get_ipl_device();
@@ -660,6 +680,38 @@ static void s390_ipl_prepare_qipl(S390CPU *cpu)
     cpu_physical_memory_unmap(addr, len, 1, len);
 }
 
+int s390_ipl_prepare_pv_header(void)
+{
+    IplParameterBlock *ipib = s390_ipl_get_iplb_pv();
+    IPLBlockPV *ipib_pv = &ipib->pv;
+    void *hdr = g_malloc(ipib_pv->pv_header_len);
+    int rc;
+
+    cpu_physical_memory_read(ipib_pv->pv_header_addr, hdr,
+                             ipib_pv->pv_header_len);
+    rc = s390_pv_set_sec_parms((uint64_t)hdr,
+                               ipib_pv->pv_header_len);
+    g_free(hdr);
+    return rc;
+}
+
+int s390_ipl_pv_unpack(void)
+{
+    IplParameterBlock *ipib = s390_ipl_get_iplb_pv();
+    IPLBlockPV *ipib_pv = &ipib->pv;
+    int i, rc = 0;
+
+    for (i = 0; i < ipib_pv->num_comp; i++) {
+        rc = s390_pv_unpack(ipib_pv->components[i].addr,
+                            TARGET_PAGE_ALIGN(ipib_pv->components[i].size),
+                            ipib_pv->components[i].tweak_pref);
+        if (rc) {
+            break;
+        }
+    }
+    return rc;
+}
+
 void s390_ipl_prepare_cpu(S390CPU *cpu)
 {
     S390IPLState *ipl = get_ipl_device();
diff --git a/hw/s390x/ipl.h b/hw/s390x/ipl.h
index 3e44abe1c651d8a0..919f9e69131b5207 100644
--- a/hw/s390x/ipl.h
+++ b/hw/s390x/ipl.h
@@ -15,6 +15,24 @@
 #include "cpu.h"
 #include "hw/qdev-core.h"
 
+struct IPLBlockPVComp {
+    uint64_t tweak_pref;
+    uint64_t addr;
+    uint64_t size;
+} QEMU_PACKED;
+typedef struct IPLBlockPVComp IPLBlockPVComp;
+
+struct IPLBlockPV {
+    uint8_t  reserved18[87];    /* 0x18 */
+    uint8_t  version;           /* 0x6f */
+    uint32_t reserved70;        /* 0x70 */
+    uint32_t num_comp;          /* 0x74 */
+    uint64_t pv_header_addr;    /* 0x78 */
+    uint64_t pv_header_len;     /* 0x80 */
+    struct IPLBlockPVComp components[];
+} QEMU_PACKED;
+typedef struct IPLBlockPV IPLBlockPV;
+
 struct IplBlockCcw {
     uint8_t  reserved0[85];
     uint8_t  ssid;
@@ -71,6 +89,7 @@ union IplParameterBlock {
         union {
             IplBlockCcw ccw;
             IplBlockFcp fcp;
+            IPLBlockPV pv;
             IplBlockQemuScsi scsi;
         };
     } QEMU_PACKED;
@@ -85,8 +104,11 @@ typedef union IplParameterBlock IplParameterBlock;
 
 int s390_ipl_set_loadparm(uint8_t *loadparm);
 void s390_ipl_update_diag308(IplParameterBlock *iplb);
+int s390_ipl_prepare_pv_header(void);
+int s390_ipl_pv_unpack(void);
 void s390_ipl_prepare_cpu(S390CPU *cpu);
 IplParameterBlock *s390_ipl_get_iplb(void);
+IplParameterBlock *s390_ipl_get_iplb_pv(void);
 
 enum s390_reset {
     /* default is a reset not triggered by a CPU e.g. issued by QMP */
@@ -94,6 +116,7 @@ enum s390_reset {
     S390_RESET_REIPL,
     S390_RESET_MODIFIED_CLEAR,
     S390_RESET_LOAD_NORMAL,
+    S390_RESET_PV,
 };
 void s390_ipl_reset_request(CPUState *cs, enum s390_reset reset_type);
 void s390_ipl_get_reset_request(CPUState **cs, enum s390_reset *reset_type);
@@ -133,6 +156,7 @@ struct S390IPLState {
     /*< private >*/
     DeviceState parent_obj;
     IplParameterBlock iplb;
+    IplParameterBlock iplb_pv;
     QemuIplParameters qipl;
     uint64_t start_addr;
     uint64_t compat_start_addr;
@@ -140,6 +164,7 @@ struct S390IPLState {
     uint64_t compat_bios_start_addr;
     bool enforce_bios;
     bool iplb_valid;
+    bool iplb_valid_pv;
     bool netboot;
     /* reset related properties don't have to be migrated or reset */
     enum s390_reset reset_type;
@@ -161,9 +186,11 @@ QEMU_BUILD_BUG_MSG(offsetof(S390IPLState, iplb) & 3, "alignment of iplb wrong");
 
 #define S390_IPL_TYPE_FCP 0x00
 #define S390_IPL_TYPE_CCW 0x02
+#define S390_IPL_TYPE_PV 0x05
 #define S390_IPL_TYPE_QEMU_SCSI 0xff
 
 #define S390_IPLB_HEADER_LEN 8
+#define S390_IPLB_MIN_PV_LEN 148
 #define S390_IPLB_MIN_CCW_LEN 200
 #define S390_IPLB_MIN_FCP_LEN 384
 #define S390_IPLB_MIN_QEMU_SCSI_LEN 200
@@ -173,6 +200,50 @@ static inline bool iplb_valid_len(IplParameterBlock *iplb)
     return be32_to_cpu(iplb->len) <= sizeof(IplParameterBlock);
 }
 
+static inline bool ipl_valid_pv_components(IplParameterBlock *iplb)
+{
+    IPLBlockPV *ipib_pv = &iplb->pv;
+    int i;
+
+    if (ipib_pv->num_comp == 0) {
+        return false;
+    }
+
+    for (i = 0; i < ipib_pv->num_comp; i++) {
+        /* Addr must be 4k aligned */
+        if (ipib_pv->components[i].addr & ~TARGET_PAGE_MASK) {
+            return false;
+        }
+
+        /* Tweak prefix is monotonically increasing with each component */
+        if (i < ipib_pv->num_comp - 1 &&
+            ipib_pv->components[i].tweak_pref >=
+            ipib_pv->components[i + 1].tweak_pref) {
+            return false;
+        }
+    }
+    return true;
+}
+
+static inline bool ipl_valid_pv_header(IplParameterBlock *iplb)
+{
+        IPLBlockPV *ipib_pv = &iplb->pv;
+
+        if (ipib_pv->pv_header_len > 2 * TARGET_PAGE_SIZE) {
+            return false;
+        }
+
+        if (!address_space_access_valid(&address_space_memory,
+                                        ipib_pv->pv_header_addr,
+                                        ipib_pv->pv_header_len,
+                                        false,
+                                        MEMTXATTRS_UNSPECIFIED)) {
+            return false;
+        }
+
+        return true;
+}
+
 static inline bool iplb_valid(IplParameterBlock *iplb)
 {
     switch (iplb->pbt) {
@@ -180,6 +251,14 @@ static inline bool iplb_valid(IplParameterBlock *iplb)
         return be32_to_cpu(iplb->len) >= S390_IPLB_MIN_FCP_LEN;
     case S390_IPL_TYPE_CCW:
         return be32_to_cpu(iplb->len) >= S390_IPLB_MIN_CCW_LEN;
+    case S390_IPL_TYPE_PV:
+        if (be32_to_cpu(iplb->len) < S390_IPLB_MIN_PV_LEN) {
+            return false;
+        }
+        if (!ipl_valid_pv_header(iplb)) {
+            return false;
+        }
+        return ipl_valid_pv_components(iplb);
     default:
         return false;
     }
diff --git a/hw/s390x/pv.c b/hw/s390x/pv.c
new file mode 100644
index 0000000000000000..8cf5cd2c9bcd48b0
--- /dev/null
+++ b/hw/s390x/pv.c
@@ -0,0 +1,98 @@
+/*
+ * Protected Virtualization functions
+ *
+ * Copyright IBM Corp. 2020
+ * Author(s):
+ *  Janosch Frank <frankja@linux.ibm.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or (at
+ * your option) any later version. See the COPYING file in the top-level
+ * directory.
+ */
+#include "qemu/osdep.h"
+
+#include <linux/kvm.h>
+
+#include "qemu/error-report.h"
+#include "sysemu/kvm.h"
+#include "hw/s390x/pv.h"
+
+static int __s390_pv_cmd(uint32_t cmd, const char *cmdname, void *data)
+{
+    struct kvm_pv_cmd pv_cmd = {
+        .cmd = cmd,
+        .data = (uint64_t)data,
+    };
+    int rc = kvm_vm_ioctl(kvm_state, KVM_S390_PV_COMMAND, &pv_cmd);
+
+    if (rc) {
+        error_report("KVM PV command %d (%s) failed: header rc %x rrc %x "
+                     "IOCTL rc: %d", cmd, cmdname, pv_cmd.rc, pv_cmd.rrc,
+                     rc);
+    }
+    return rc;
+}
+
+/*
+ * This macro lets us pass the command as a string to the function so
+ * we can print it on an error.
+ */
+#define s390_pv_cmd(cmd, data) __s390_pv_cmd(cmd, #cmd, data);
+#define s390_pv_cmd_exit(cmd, data)    \
+{                                      \
+    int rc;                            \
+                                       \
+    rc = __s390_pv_cmd(cmd, #cmd, data);\
+    if (rc) {                          \
+        exit(1);                       \
+    }                                  \
+}
+
+int s390_pv_vm_enable(void)
+{
+    return s390_pv_cmd(KVM_PV_ENABLE, NULL);
+}
+
+void s390_pv_vm_disable(void)
+{
+     s390_pv_cmd_exit(KVM_PV_DISABLE, NULL);
+}
+
+int s390_pv_set_sec_parms(uint64_t origin, uint64_t length)
+{
+    struct kvm_s390_pv_sec_parm args = {
+        .origin = origin,
+        .length = length,
+    };
+
+    return s390_pv_cmd(KVM_PV_VM_SET_SEC_PARMS, &args);
+}
+
+/*
+ * Called for each component in the SE type IPL parameter block 0.
+ */
+int s390_pv_unpack(uint64_t addr, uint64_t size, uint64_t tweak)
+{
+    struct kvm_s390_pv_unp args = {
+        .addr = addr,
+        .size = size,
+        .tweak = tweak,
+    };
+
+    return s390_pv_cmd(KVM_PV_VM_UNPACK, &args);
+}
+
+void s390_pv_perf_clear_reset(void)
+{
+    s390_pv_cmd_exit(KVM_PV_VM_PREP_RESET, NULL);
+}
+
+int s390_pv_verify(void)
+{
+    return s390_pv_cmd(KVM_PV_VM_VERIFY, NULL);
+}
+
+void s390_pv_unshare(void)
+{
+    s390_pv_cmd_exit(KVM_PV_VM_UNSHARE_ALL, NULL);
+}
diff --git a/hw/s390x/s390-virtio-ccw.c b/hw/s390x/s390-virtio-ccw.c
index 895498cca6199c16..9569b777a0e1abd6 100644
--- a/hw/s390x/s390-virtio-ccw.c
+++ b/hw/s390x/s390-virtio-ccw.c
@@ -41,6 +41,7 @@
 #include "hw/qdev-properties.h"
 #include "hw/s390x/tod.h"
 #include "sysemu/sysemu.h"
+#include "hw/s390x/pv.h"
 
 S390CPU *s390_cpu_addr2state(uint16_t cpu_addr)
 {
@@ -316,10 +317,79 @@ static inline void s390_do_cpu_ipl(CPUState *cs, run_on_cpu_data arg)
     s390_cpu_set_state(S390_CPU_STATE_OPERATING, cpu);
 }
 
+static void s390_machine_unprotect(S390CcwMachineState *ms)
+{
+    s390_pv_vm_disable();
+    ms->pv = false;
+}
+
+static int s390_machine_protect(S390CcwMachineState *ms)
+{
+    int rc;
+
+    /* Create SE VM */
+    rc = s390_pv_vm_enable();
+    if (rc) {
+        return rc;
+    }
+
+    ms->pv = true;
+
+    /* Set SE header and unpack */
+    rc = s390_ipl_prepare_pv_header();
+    if (rc) {
+        goto out_err;
+    }
+
+    /* Decrypt image */
+    rc = s390_ipl_pv_unpack();
+    if (rc) {
+        goto out_err;
+    }
+
+    /* Verify integrity */
+    rc = s390_pv_verify();
+    if (rc) {
+        goto out_err;
+    }
+    return rc;
+
+out_err:
+    s390_machine_unprotect(ms);
+    return rc;
+}
+
+#define DIAG_308_RC_INVAL_FOR_PV    0x0a02
+static void s390_machine_inject_pv_error(CPUState *cs)
+{
+    int r1 = (cs->kvm_run->s390_sieic.ipa & 0x00f0) >> 4;
+    CPUS390XState *env = &S390_CPU(cs)->env;
+
+    /* Report that we are unable to enter protected mode */
+    env->regs[r1 + 1] = DIAG_308_RC_INVAL_FOR_PV;
+}
+
+static void s390_pv_prepare_reset(S390CcwMachineState *ms)
+{
+    CPUState *cs;
+
+    if (!s390_is_pv()) {
+        return;
+    }
+    /* Unsharing requires all cpus to be stopped */
+    CPU_FOREACH(cs) {
+        s390_cpu_set_state(S390_CPU_STATE_STOPPED, S390_CPU(cs));
+    }
+    s390_pv_unshare();
+    s390_pv_perf_clear_reset();
+}
+
 static void s390_machine_reset(MachineState *machine)
 {
+    S390CcwMachineState *ms = S390_CCW_MACHINE(machine);
     enum s390_reset reset_type;
     CPUState *cs, *t;
+    S390CPU *cpu;
 
     /* get the reset parameters, reset them once done */
     s390_ipl_get_reset_request(&cs, &reset_type);
@@ -327,9 +397,15 @@ static void s390_machine_reset(MachineState *machine)
     /* all CPUs are paused and synchronized at this point */
     s390_cmma_reset();
 
+    cpu = S390_CPU(cs);
+
     switch (reset_type) {
     case S390_RESET_EXTERNAL:
     case S390_RESET_REIPL:
+        if (s390_is_pv()) {
+            s390_machine_unprotect(ms);
+        }
+
         qemu_devices_reset();
         s390_crypto_reset();
 
@@ -337,22 +413,56 @@ static void s390_machine_reset(MachineState *machine)
         run_on_cpu(cs, s390_do_cpu_ipl, RUN_ON_CPU_NULL);
         break;
     case S390_RESET_MODIFIED_CLEAR:
+        /*
+         * Susbsystem reset needs to be done before we unshare memory
+         * and loose access to VIRTIO structures in guest memory.
+         */
+        subsystem_reset();
+        s390_crypto_reset();
+        s390_pv_prepare_reset(ms);
         CPU_FOREACH(t) {
             run_on_cpu(t, s390_do_cpu_full_reset, RUN_ON_CPU_NULL);
         }
-        subsystem_reset();
-        s390_crypto_reset();
         run_on_cpu(cs, s390_do_cpu_load_normal, RUN_ON_CPU_NULL);
         break;
     case S390_RESET_LOAD_NORMAL:
+        /*
+         * Susbsystem reset needs to be done before we unshare memory
+         * and loose access to VIRTIO structures in guest memory.
+         */
+        subsystem_reset();
+        s390_pv_prepare_reset(ms);
         CPU_FOREACH(t) {
             if (t == cs) {
                 continue;
             }
             run_on_cpu(t, s390_do_cpu_reset, RUN_ON_CPU_NULL);
         }
-        subsystem_reset();
         run_on_cpu(cs, s390_do_cpu_initial_reset, RUN_ON_CPU_NULL);
+        run_on_cpu(cs, s390_do_cpu_load_normal, RUN_ON_CPU_NULL);
+        break;
+    case S390_RESET_PV: /* Subcode 10 */
+        subsystem_reset();
+        s390_crypto_reset();
+
+        CPU_FOREACH(t) {
+            if (t == cs) {
+                continue;
+            }
+            run_on_cpu(t, s390_do_cpu_full_reset, RUN_ON_CPU_NULL);
+        }
+        run_on_cpu(cs, s390_do_cpu_reset, RUN_ON_CPU_NULL);
+
+        if (s390_machine_protect(ms)) {
+            s390_machine_inject_pv_error(cs);
+            /*
+             * Continue after the diag308 so the guest knows something
+             * went wrong.
+             */
+            s390_cpu_set_state(S390_CPU_STATE_OPERATING, cpu);
+            return;
+        }
+
         run_on_cpu(cs, s390_do_cpu_load_normal, RUN_ON_CPU_NULL);
         break;
     default:
diff --git a/include/hw/s390x/pv.h b/include/hw/s390x/pv.h
new file mode 100644
index 0000000000000000..43298171a9e23c76
--- /dev/null
+++ b/include/hw/s390x/pv.h
@@ -0,0 +1,57 @@
+/*
+ * Protected Virtualization header
+ *
+ * Copyright IBM Corp. 2020
+ * Author(s):
+ *  Janosch Frank <frankja@linux.ibm.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or (at
+ * your option) any later version. See the COPYING file in the top-level
+ * directory.
+ */
+#ifndef HW_S390_PV_H
+#define HW_S390_PV_H
+
+#ifdef CONFIG_KVM
+#include "hw/s390x/s390-virtio-ccw.h"
+
+static inline bool s390_is_pv(void)
+{
+    static S390CcwMachineState *ccw;
+    Object *obj;
+
+    if (ccw) {
+        return ccw->pv;
+    }
+
+    /* we have to bail out for the "none" machine */
+    obj = object_dynamic_cast(qdev_get_machine(),
+                              TYPE_S390_CCW_MACHINE);
+    if (!obj) {
+        return false;
+    }
+    ccw = S390_CCW_MACHINE(obj);
+    return ccw->pv;
+}
+
+int s390_pv_vm_enable(void);
+void s390_pv_vm_disable(void);
+int s390_pv_set_sec_parms(uint64_t origin, uint64_t length);
+int s390_pv_unpack(uint64_t addr, uint64_t size, uint64_t tweak);
+void s390_pv_perf_clear_reset(void);
+int s390_pv_verify(void);
+void s390_pv_unshare(void);
+#else
+static inline bool s390_is_pv(void) { return false; }
+static inline int s390_pv_vm_enable(void) { return 0; }
+static inline void s390_pv_vm_disable(void) {}
+static inline int s390_pv_set_sec_parms(uint64_t origin, uint64_t length) { return 0; }
+static inline int s390_pv_unpack(uint64_t addr, uint64_t size, uint64_t tweak) { return 0; }
+static inline void s390_pv_perf_clear_reset(void) {}
+static inline int s390_pv_verify(void) { return 0; }
+static inline void s390_pv_unshare(void) {}
+#endif
+
+
+
+#endif /* HW_S390_PV_H */
diff --git a/include/hw/s390x/s390-virtio-ccw.h b/include/hw/s390x/s390-virtio-ccw.h
index 8aa27199c9123bab..cd1dccc6e3ba8645 100644
--- a/include/hw/s390x/s390-virtio-ccw.h
+++ b/include/hw/s390x/s390-virtio-ccw.h
@@ -28,6 +28,7 @@ typedef struct S390CcwMachineState {
     /*< public >*/
     bool aes_key_wrap;
     bool dea_key_wrap;
+    bool pv;
     uint8_t loadparm[8];
 } S390CcwMachineState;
 
diff --git a/target/s390x/cpu.c b/target/s390x/cpu.c
index 3dd396e870357944..84029f14814b4980 100644
--- a/target/s390x/cpu.c
+++ b/target/s390x/cpu.c
@@ -37,6 +37,8 @@
 #include "sysemu/hw_accel.h"
 #include "hw/qdev-properties.h"
 #ifndef CONFIG_USER_ONLY
+#include "hw/s390x/s390-virtio-ccw.h"
+#include "hw/s390x/pv.h"
 #include "hw/boards.h"
 #include "sysemu/arch_init.h"
 #include "sysemu/sysemu.h"
diff --git a/target/s390x/cpu_features_def.inc.h b/target/s390x/cpu_features_def.inc.h
index 31dff0d84e972451..60db28351d059091 100644
--- a/target/s390x/cpu_features_def.inc.h
+++ b/target/s390x/cpu_features_def.inc.h
@@ -107,6 +107,7 @@ DEF_FEAT(DEFLATE_BASE, "deflate-base", STFL, 151, "Deflate-conversion facility (
 DEF_FEAT(VECTOR_PACKED_DECIMAL_ENH, "vxpdeh", STFL, 152, "Vector-Packed-Decimal-Enhancement Facility")
 DEF_FEAT(MSA_EXT_9, "msa9-base", STFL, 155, "Message-security-assist-extension-9 facility (excluding subfunctions)")
 DEF_FEAT(ETOKEN, "etoken", STFL, 156, "Etoken facility")
+DEF_FEAT(UNPACK, "unpack", STFL, 161, "Unpack facility")
 
 /* Features exposed via SCLP SCCB Byte 80 - 98  (bit numbers relative to byte-80) */
 DEF_FEAT(SIE_GSLS, "gsls", SCLP_CONF_CHAR, 40, "SIE: Guest-storage-limit-suppression facility")
diff --git a/target/s390x/diag.c b/target/s390x/diag.c
index 54e5670b3fd6d960..b245e557037ded06 100644
--- a/target/s390x/diag.c
+++ b/target/s390x/diag.c
@@ -20,6 +20,7 @@
 #include "sysemu/cpus.h"
 #include "hw/s390x/ipl.h"
 #include "hw/s390x/s390-virtio-ccw.h"
+#include "hw/s390x/pv.h"
 
 int handle_diag_288(CPUS390XState *env, uint64_t r1, uint64_t r3)
 {
@@ -52,6 +53,7 @@ int handle_diag_288(CPUS390XState *env, uint64_t r1, uint64_t r3)
 #define DIAG_308_RC_OK              0x0001
 #define DIAG_308_RC_NO_CONF         0x0102
 #define DIAG_308_RC_INVALID         0x0402
+#define DIAG_308_RC_NO_PV_CONF      0x0902
 
 #define DIAG308_RESET_MOD_CLR       0
 #define DIAG308_RESET_LOAD_NORM     1
@@ -59,10 +61,17 @@ int handle_diag_288(CPUS390XState *env, uint64_t r1, uint64_t r3)
 #define DIAG308_LOAD_NORMAL_DUMP    4
 #define DIAG308_SET                 5
 #define DIAG308_STORE               6
+#define DIAG308_PV_SET              8
+#define DIAG308_PV_STORE            9
+#define DIAG308_PV_START            10
 
 static int diag308_parm_check(CPUS390XState *env, uint64_t r1, uint64_t addr,
                               uintptr_t ra, bool write)
 {
+    /* Handled by the Ultravisor */
+    if (s390_is_pv()) {
+        return 0;
+    }
     if ((r1 & 1) || (addr & ~TARGET_PAGE_MASK)) {
         s390_program_interrupt(env, PGM_SPECIFICATION, ra);
         return -1;
@@ -93,6 +102,11 @@ void handle_diag_308(CPUS390XState *env, uint64_t r1, uint64_t r3, uintptr_t ra)
         return;
     }
 
+    if (subcode >= DIAG308_PV_SET && !s390_has_feat(S390_FEAT_UNPACK)) {
+        s390_program_interrupt(env, PGM_SPECIFICATION, ra);
+        return;
+    }
+
     switch (subcode) {
     case DIAG308_RESET_MOD_CLR:
         s390_ipl_reset_request(cs, S390_RESET_MODIFIED_CLEAR);
@@ -105,6 +119,7 @@ void handle_diag_308(CPUS390XState *env, uint64_t r1, uint64_t r3, uintptr_t ra)
         s390_ipl_reset_request(cs, S390_RESET_REIPL);
         break;
     case DIAG308_SET:
+    case DIAG308_PV_SET:
         if (diag308_parm_check(env, r1, addr, ra, false)) {
             return;
         }
@@ -128,10 +143,15 @@ out:
         g_free(iplb);
         return;
     case DIAG308_STORE:
+    case DIAG308_PV_STORE:
         if (diag308_parm_check(env, r1, addr, ra, true)) {
             return;
         }
-        iplb = s390_ipl_get_iplb();
+        if (subcode == DIAG308_PV_STORE) {
+            iplb = s390_ipl_get_iplb_pv();
+        } else {
+            iplb = s390_ipl_get_iplb();
+        }
         if (iplb) {
             cpu_physical_memory_write(addr, iplb, be32_to_cpu(iplb->len));
             env->regs[r1 + 1] = DIAG_308_RC_OK;
@@ -139,6 +159,15 @@ out:
             env->regs[r1 + 1] = DIAG_308_RC_NO_CONF;
         }
         return;
+    case DIAG308_PV_START:
+        iplb = s390_ipl_get_iplb_pv();
+        if (!iplb) {
+            env->regs[r1 + 1] = DIAG_308_RC_NO_PV_CONF;
+            return;
+        }
+
+        s390_ipl_reset_request(cs, S390_RESET_PV);
+        break;
     default:
         s390_program_interrupt(env, PGM_SPECIFICATION, ra);
         break;
-- 
2.25.1

[PATCH v9 03/15] s390x: protvirt: Add migration blocker

[Janosch Frank]

Migration is not yet supported.

Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
Reviewed-by: David Hildenbrand <david@redhat.com>
---
 hw/s390x/s390-virtio-ccw.c | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)

diff --git a/hw/s390x/s390-virtio-ccw.c b/hw/s390x/s390-virtio-ccw.c
index 9569b777a0e1abd6..deb31e060052d279 100644
--- a/hw/s390x/s390-virtio-ccw.c
+++ b/hw/s390x/s390-virtio-ccw.c
@@ -42,6 +42,9 @@
 #include "hw/s390x/tod.h"
 #include "sysemu/sysemu.h"
 #include "hw/s390x/pv.h"
+#include "migration/blocker.h"
+
+static Error *pv_mig_blocker;
 
 S390CPU *s390_cpu_addr2state(uint16_t cpu_addr)
 {
@@ -321,15 +324,30 @@ static void s390_machine_unprotect(S390CcwMachineState *ms)
 {
     s390_pv_vm_disable();
     ms->pv = false;
+    migrate_del_blocker(pv_mig_blocker);
+    error_free_or_abort(&pv_mig_blocker);
 }
 
 static int s390_machine_protect(S390CcwMachineState *ms)
 {
+    Error *local_err = NULL;
     int rc;
 
+    error_setg(&pv_mig_blocker,
+               "protected VMs are currently not migrateable.");
+    rc = migrate_add_blocker(pv_mig_blocker, &local_err);
+    if (local_err) {
+        error_report_err(local_err);
+        error_free_or_abort(&pv_mig_blocker);
+        return rc;
+    }
+
     /* Create SE VM */
     rc = s390_pv_vm_enable();
     if (rc) {
+        error_report_err(local_err);
+        migrate_del_blocker(pv_mig_blocker);
+        error_free_or_abort(&pv_mig_blocker);
         return rc;
     }
 
-- 
2.25.1

[PATCH v9 04/15] s390x: protvirt: Inhibit balloon when switching to protected mode

[Janosch Frank]

Ballooning in protected VMs can only be done when the guest shares the
pages it gives to the host. If pages are not shared, the integrity
checks will fail once those pages have been altered and are given back
to the guest.

As we currently do not yet have a solution for this we will continue
like this:

1. We block ballooning now in QEMU (with this patch)

2. Later we will provide a change to virtio that removes the blocker
and adds VIRTIO_F_IOMMU_PLATFORM automatically by QEMU when doing the
protvirt switch. This is ok as the guest balloon driver will reject to
work with the IOMMU change

3. Later we can fix the guest balloon driver to accept the IOMMU
feature bit and correctly exercise sharing and unsharing of balloon
pages

Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
Reviewed-by: David Hildenbrand <david@redhat.com>
Reviewed-by: Christian Borntraeger <borntraeger@de.ibm.com>
---
 hw/s390x/s390-virtio-ccw.c | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/hw/s390x/s390-virtio-ccw.c b/hw/s390x/s390-virtio-ccw.c
index deb31e060052d279..066e01f303c35671 100644
--- a/hw/s390x/s390-virtio-ccw.c
+++ b/hw/s390x/s390-virtio-ccw.c
@@ -41,6 +41,7 @@
 #include "hw/qdev-properties.h"
 #include "hw/s390x/tod.h"
 #include "sysemu/sysemu.h"
+#include "sysemu/balloon.h"
 #include "hw/s390x/pv.h"
 #include "migration/blocker.h"
 
@@ -326,6 +327,7 @@ static void s390_machine_unprotect(S390CcwMachineState *ms)
     ms->pv = false;
     migrate_del_blocker(pv_mig_blocker);
     error_free_or_abort(&pv_mig_blocker);
+    qemu_balloon_inhibit(false);
 }
 
 static int s390_machine_protect(S390CcwMachineState *ms)
@@ -333,10 +335,12 @@ static int s390_machine_protect(S390CcwMachineState *ms)
     Error *local_err = NULL;
     int rc;
 
+    qemu_balloon_inhibit(true);
     error_setg(&pv_mig_blocker,
                "protected VMs are currently not migrateable.");
     rc = migrate_add_blocker(pv_mig_blocker, &local_err);
     if (local_err) {
+        qemu_balloon_inhibit(false);
         error_report_err(local_err);
         error_free_or_abort(&pv_mig_blocker);
         return rc;
@@ -345,6 +349,7 @@ static int s390_machine_protect(S390CcwMachineState *ms)
     /* Create SE VM */
     rc = s390_pv_vm_enable();
     if (rc) {
+        qemu_balloon_inhibit(false);
         error_report_err(local_err);
         migrate_del_blocker(pv_mig_blocker);
         error_free_or_abort(&pv_mig_blocker);
-- 
2.25.1

[PATCH v9 05/15] s390x: protvirt: KVM intercept changes

[Janosch Frank]

Protected VMs no longer intercept with code 4 for an instruction
interception. Instead they have codes 104 and 108 for protected
instruction interception and protected instruction notification
respectively.

The 104 mirrors the 4 interception.

The 108 is a notification interception to let KVM and QEMU know that
something changed and we need to update tracking information or
perform specific tasks. It's currently taken for the following
instructions:

* spx (To inform about the changed prefix location)
* sclp (On incorrect SCCB values, so we can inject a IRQ)
* sigp (All but "stop and store status")
* diag308 (Subcodes 0/1)

Of these exits only sclp errors, state changing sigps and diag308 will
reach QEMU. QEMU will do its parts of the job, while the ultravisor
has done the instruction part of the job.

Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
Reviewed-by: David Hildenbrand <david@redhat.com>
Reviewed-by: Christian Borntraeger <borntraeger@de.ibm.com>
---
 target/s390x/kvm.c | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/target/s390x/kvm.c b/target/s390x/kvm.c
index 1d6fd6a27b48e35f..eec0b92479465b9c 100644
--- a/target/s390x/kvm.c
+++ b/target/s390x/kvm.c
@@ -115,6 +115,8 @@
 #define ICPT_CPU_STOP                   0x28
 #define ICPT_OPEREXC                    0x2c
 #define ICPT_IO                         0x40
+#define ICPT_PV_INSTR                   0x68
+#define ICPT_PV_INSTR_NOTIFICATION      0x6c
 
 #define NR_LOCAL_IRQS 32
 /*
@@ -1693,6 +1695,8 @@ static int handle_intercept(S390CPU *cpu)
             (long)cs->kvm_run->psw_addr);
     switch (icpt_code) {
         case ICPT_INSTRUCTION:
+        case ICPT_PV_INSTR:
+        case ICPT_PV_INSTR_NOTIFICATION:
             r = handle_instruction(cpu, run);
             break;
         case ICPT_PROGRAM:
-- 
2.25.1

[PATCH v9 06/15] s390x: Add SIDA memory ops

[Janosch Frank]

Protected guests save the instruction control blocks in the SIDA
instead of QEMU/KVM directly accessing the guest's memory.

Let's introduce new functions to access the SIDA.

Also the new memops are available with KVM_CAP_S390_PROTECTED, so
let's check for that.

Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
Reviewed-by: David Hildenbrand <david@redhat.com>
---
 target/s390x/cpu.h        |  7 ++++++-
 target/s390x/kvm.c        | 25 +++++++++++++++++++++++++
 target/s390x/kvm_s390x.h  |  2 ++
 target/s390x/mmu_helper.c | 14 ++++++++++++++
 4 files changed, 47 insertions(+), 1 deletion(-)

diff --git a/target/s390x/cpu.h b/target/s390x/cpu.h
index 1d17709d6e10b5e0..035427521cec2528 100644
--- a/target/s390x/cpu.h
+++ b/target/s390x/cpu.h
@@ -823,7 +823,12 @@ int s390_cpu_virt_mem_rw(S390CPU *cpu, vaddr laddr, uint8_t ar, void *hostbuf,
 #define s390_cpu_virt_mem_check_write(cpu, laddr, ar, len)   \
         s390_cpu_virt_mem_rw(cpu, laddr, ar, NULL, len, true)
 void s390_cpu_virt_mem_handle_exc(S390CPU *cpu, uintptr_t ra);
-
+int s390_cpu_pv_mem_rw(S390CPU *cpu, unsigned int offset, void *hostbuf,
+                       int len, bool is_write);
+#define s390_cpu_pv_mem_read(cpu, offset, dest, len)    \
+        s390_cpu_pv_mem_rw(cpu, offset, dest, len, false)
+#define s390_cpu_pv_mem_write(cpu, offset, dest, len)       \
+        s390_cpu_pv_mem_rw(cpu, offset, dest, len, true)
 
 /* sigp.c */
 int s390_cpu_restart(S390CPU *cpu);
diff --git a/target/s390x/kvm.c b/target/s390x/kvm.c
index eec0b92479465b9c..cdcd538b4f7fb318 100644
--- a/target/s390x/kvm.c
+++ b/target/s390x/kvm.c
@@ -154,6 +154,7 @@ static int cap_ri;
 static int cap_gs;
 static int cap_hpage_1m;
 static int cap_vcpu_resets;
+static int cap_protected;
 
 static int active_cmma;
 
@@ -346,6 +347,7 @@ int kvm_arch_init(MachineState *ms, KVMState *s)
     cap_mem_op = kvm_check_extension(s, KVM_CAP_S390_MEM_OP);
     cap_s390_irq = kvm_check_extension(s, KVM_CAP_S390_INJECT_IRQ);
     cap_vcpu_resets = kvm_check_extension(s, KVM_CAP_S390_VCPU_RESETS);
+    cap_protected = kvm_check_extension(s, KVM_CAP_S390_PROTECTED);
 
     if (!kvm_check_extension(s, KVM_CAP_S390_GMAP)
         || !kvm_check_extension(s, KVM_CAP_S390_COW)) {
@@ -846,6 +848,29 @@ int kvm_s390_mem_op(S390CPU *cpu, vaddr addr, uint8_t ar, void *hostbuf,
     return ret;
 }
 
+int kvm_s390_mem_op_pv(S390CPU *cpu, uint64_t offset, void *hostbuf,
+                       int len, bool is_write)
+{
+    struct kvm_s390_mem_op mem_op = {
+        .sida_offset = offset,
+        .size = len,
+        .op = is_write ? KVM_S390_MEMOP_SIDA_WRITE
+                       : KVM_S390_MEMOP_SIDA_READ,
+        .buf = (uint64_t)hostbuf,
+    };
+    int ret;
+
+    if (!cap_mem_op || !cap_protected) {
+        return -ENOSYS;
+    }
+
+    ret = kvm_vcpu_ioctl(CPU(cpu), KVM_S390_MEM_OP, &mem_op);
+    if (ret < 0) {
+        error_report("KVM_S390_MEM_OP failed: %s", strerror(-ret));
+    }
+    return ret;
+}
+
 /*
  * Legacy layout for s390:
  * Older S390 KVM requires the topmost vma of the RAM to be
diff --git a/target/s390x/kvm_s390x.h b/target/s390x/kvm_s390x.h
index 0b21789796d7c462..9c38f6ccce83e39e 100644
--- a/target/s390x/kvm_s390x.h
+++ b/target/s390x/kvm_s390x.h
@@ -19,6 +19,8 @@ void kvm_s390_vcpu_interrupt(S390CPU *cpu, struct kvm_s390_irq *irq);
 void kvm_s390_access_exception(S390CPU *cpu, uint16_t code, uint64_t te_code);
 int kvm_s390_mem_op(S390CPU *cpu, vaddr addr, uint8_t ar, void *hostbuf,
                     int len, bool is_write);
+int kvm_s390_mem_op_pv(S390CPU *cpu, vaddr addr, void *hostbuf, int len,
+                       bool is_write);
 void kvm_s390_program_interrupt(S390CPU *cpu, uint16_t code);
 int kvm_s390_set_cpu_state(S390CPU *cpu, uint8_t cpu_state);
 void kvm_s390_vcpu_interrupt_pre_save(S390CPU *cpu);
diff --git a/target/s390x/mmu_helper.c b/target/s390x/mmu_helper.c
index 0be2f300bbe4ac8b..7d9f3059cd502c49 100644
--- a/target/s390x/mmu_helper.c
+++ b/target/s390x/mmu_helper.c
@@ -474,6 +474,20 @@ static int translate_pages(S390CPU *cpu, vaddr addr, int nr_pages,
     return 0;
 }
 
+int s390_cpu_pv_mem_rw(S390CPU *cpu, unsigned int offset, void *hostbuf,
+                       int len, bool is_write)
+{
+    int ret;
+
+    if (kvm_enabled()) {
+        ret = kvm_s390_mem_op_pv(cpu, offset, hostbuf, len, is_write);
+    } else {
+        /* Protected Virtualization is a KVM/Hardware only feature */
+        g_assert_not_reached();
+    }
+    return ret;
+}
+
 /**
  * s390_cpu_virt_mem_rw:
  * @laddr:     the logical start address
-- 
2.25.1

[PATCH v9 07/15] s390x: protvirt: Move STSI data over SIDAD

[Janosch Frank]

For protected guests, we need to put the STSI emulation results into
the SIDA, so SIE will write them into the guest at the next entry.

Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
Reviewed-by: David Hildenbrand <david@redhat.com>
---
 target/s390x/kvm.c | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/target/s390x/kvm.c b/target/s390x/kvm.c
index cdcd538b4f7fb318..8085d5030e7c6454 100644
--- a/target/s390x/kvm.c
+++ b/target/s390x/kvm.c
@@ -50,6 +50,7 @@
 #include "exec/memattrs.h"
 #include "hw/s390x/s390-virtio-ccw.h"
 #include "hw/s390x/s390-virtio-hcall.h"
+#include "hw/s390x/pv.h"
 
 #ifndef DEBUG_KVM
 #define DEBUG_KVM  0
@@ -1800,7 +1801,9 @@ static void insert_stsi_3_2_2(S390CPU *cpu, __u64 addr, uint8_t ar)
     SysIB_322 sysib;
     int del;
 
-    if (s390_cpu_virt_mem_read(cpu, addr, ar, &sysib, sizeof(sysib))) {
+    if (s390_is_pv()) {
+        s390_cpu_pv_mem_read(cpu, 0, &sysib, sizeof(sysib));
+    } else if (s390_cpu_virt_mem_read(cpu, addr, ar, &sysib, sizeof(sysib))) {
         return;
     }
     /* Shift the stack of Extended Names to prepare for our own data */
@@ -1840,7 +1843,11 @@ static void insert_stsi_3_2_2(S390CPU *cpu, __u64 addr, uint8_t ar)
     /* Insert UUID */
     memcpy(sysib.vm[0].uuid, &qemu_uuid, sizeof(sysib.vm[0].uuid));
 
-    s390_cpu_virt_mem_write(cpu, addr, ar, &sysib, sizeof(sysib));
+    if (s390_is_pv()) {
+        s390_cpu_pv_mem_write(cpu, 0, &sysib, sizeof(sysib));
+    } else {
+        s390_cpu_virt_mem_write(cpu, addr, ar, &sysib, sizeof(sysib));
+    }
 }
 
 static int handle_stsi(S390CPU *cpu)
-- 
2.25.1

[PATCH v9 08/15] s390x: protvirt: SCLP interpretation

[Janosch Frank]

SCLP for a protected guest is done over the SIDAD, so we need to use
the s390_cpu_pv_mem_* functions to access the SIDAD instead of guest
memory when reading/writing SCBs.

To not confuse the sclp emulation, we set 0x4000 as the SCCB address,
since the function that injects the sclp external interrupt would
reject a zero sccb address.

Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
Reviewed-by: David Hildenbrand <david@redhat.com>
---
 hw/s390x/sclp.c         | 30 ++++++++++++++++++++++++++++++
 include/hw/s390x/sclp.h |  2 ++
 target/s390x/kvm.c      | 24 +++++++++++++++++++-----
 3 files changed, 51 insertions(+), 5 deletions(-)

diff --git a/hw/s390x/sclp.c b/hw/s390x/sclp.c
index af0bfbc2eca74767..5f3aa30d6283dce5 100644
--- a/hw/s390x/sclp.c
+++ b/hw/s390x/sclp.c
@@ -193,6 +193,36 @@ static void sclp_execute(SCLPDevice *sclp, SCCB *sccb, uint32_t code)
     }
 }
 
+/*
+ * We only need the address to have something valid for the
+ * service_interrupt call.
+ */
+#define SCLP_PV_DUMMY_ADDR 0x4000
+int sclp_service_call_protected(CPUS390XState *env, uint64_t sccb,
+                                uint32_t code)
+{
+    SCLPDevice *sclp = get_sclp_device();
+    SCLPDeviceClass *sclp_c = SCLP_GET_CLASS(sclp);
+    SCCB work_sccb;
+    hwaddr sccb_len = sizeof(SCCB);
+
+    /*
+     * Only a very limited amount of calls is permitted by the
+     * Ultravisor and we support all of them, so we don't check for
+     * them. All other specification exceptions are also interpreted
+     * by the Ultravisor and hence never cause an exit we need to
+     * handle.
+     *
+     * Setting the CC is also done by the Ultravisor.
+     */
+    s390_cpu_pv_mem_read(env_archcpu(env), 0, &work_sccb, sccb_len);
+    sclp_c->execute(sclp, &work_sccb, code);
+    s390_cpu_pv_mem_write(env_archcpu(env), 0, &work_sccb,
+                          be16_to_cpu(work_sccb.h.length));
+    sclp_c->service_interrupt(sclp, SCLP_PV_DUMMY_ADDR);
+    return 0;
+}
+
 int sclp_service_call(CPUS390XState *env, uint64_t sccb, uint32_t code)
 {
     SCLPDevice *sclp = get_sclp_device();
diff --git a/include/hw/s390x/sclp.h b/include/hw/s390x/sclp.h
index c54413b78cf01b27..c0a3faa37d730453 100644
--- a/include/hw/s390x/sclp.h
+++ b/include/hw/s390x/sclp.h
@@ -217,5 +217,7 @@ void s390_sclp_init(void);
 void sclp_service_interrupt(uint32_t sccb);
 void raise_irq_cpu_hotplug(void);
 int sclp_service_call(CPUS390XState *env, uint64_t sccb, uint32_t code);
+int sclp_service_call_protected(CPUS390XState *env, uint64_t sccb,
+                                uint32_t code);
 
 #endif
diff --git a/target/s390x/kvm.c b/target/s390x/kvm.c
index 8085d5030e7c6454..ff6027036ec2f14a 100644
--- a/target/s390x/kvm.c
+++ b/target/s390x/kvm.c
@@ -1227,12 +1227,26 @@ static void kvm_sclp_service_call(S390CPU *cpu, struct kvm_run *run,
     sccb = env->regs[ipbh0 & 0xf];
     code = env->regs[(ipbh0 & 0xf0) >> 4];
 
-    r = sclp_service_call(env, sccb, code);
-    if (r < 0) {
-        kvm_s390_program_interrupt(cpu, -r);
-        return;
+    switch (run->s390_sieic.icptcode) {
+    case ICPT_PV_INSTR_NOTIFICATION:
+        g_assert(s390_is_pv());
+        /* The notification intercepts are currently handled by KVM */
+        error_report("unexpected SCLP PV notification");
+        exit(1);
+        break;
+    case ICPT_PV_INSTR:
+        g_assert(s390_is_pv());
+        sclp_service_call_protected(env, sccb, code);
+        break;
+    case ICPT_INSTRUCTION:
+        g_assert(!s390_is_pv());
+        r = sclp_service_call(env, sccb, code);
+        if (r < 0) {
+            kvm_s390_program_interrupt(cpu, -r);
+            return;
+        }
+        setcc(cpu, r);
     }
-    setcc(cpu, r);
 }
 
 static int handle_b2(S390CPU *cpu, struct kvm_run *run, uint8_t ipa1)
-- 
2.25.1

[PATCH v9 09/15] s390x: protvirt: Set guest IPL PSW

[Janosch Frank]

Handling of CPU reset and setting of the IPL psw from guest storage at
offset 0 is done by a Ultravisor call. Let's only fetch it if
necessary.

Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
Reviewed-by: Thomas Huth <thuth@redhat.com>
Reviewed-by: David Hildenbrand <david@redhat.com>
---
 target/s390x/cpu.c | 22 +++++++++++++---------
 1 file changed, 13 insertions(+), 9 deletions(-)

diff --git a/target/s390x/cpu.c b/target/s390x/cpu.c
index 84029f14814b4980..a48d39f139cdc1c4 100644
--- a/target/s390x/cpu.c
+++ b/target/s390x/cpu.c
@@ -78,16 +78,20 @@ static bool s390_cpu_has_work(CPUState *cs)
 static void s390_cpu_load_normal(CPUState *s)
 {
     S390CPU *cpu = S390_CPU(s);
-    uint64_t spsw = ldq_phys(s->as, 0);
-
-    cpu->env.psw.mask = spsw & PSW_MASK_SHORT_CTRL;
-    /*
-     * Invert short psw indication, so SIE will report a specification
-     * exception if it was not set.
-     */
-    cpu->env.psw.mask ^= PSW_MASK_SHORTPSW;
-    cpu->env.psw.addr = spsw & PSW_MASK_SHORT_ADDR;
+    uint64_t spsw;
 
+    if (!s390_is_pv()) {
+        spsw = ldq_phys(s->as, 0);
+        cpu->env.psw.mask = spsw & PSW_MASK_SHORT_CTRL;
+        /*
+         * Invert short psw indication, so SIE will report a specification
+         * exception if it was not set.
+         */
+        cpu->env.psw.mask ^= PSW_MASK_SHORTPSW;
+        cpu->env.psw.addr = spsw & PSW_MASK_SHORT_ADDR;
+    } else {
+        s390_cpu_set_state(S390_CPU_STATE_LOAD, cpu);
+    }
     s390_cpu_set_state(S390_CPU_STATE_OPERATING, cpu);
 }
 #endif
-- 
2.25.1

[PATCH v9 10/15] s390x: protvirt: Move diag 308 data over SIDA

[Janosch Frank]

For protected guests the IPIB is written/read to/from the SIDA, so we
need those accesses to go through s390_cpu_pv_mem_read/write().

Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
Reviewed-by: David Hildenbrand <david@redhat.com>
---
 target/s390x/diag.c | 27 +++++++++++++++++++++------
 1 file changed, 21 insertions(+), 6 deletions(-)

diff --git a/target/s390x/diag.c b/target/s390x/diag.c
index b245e557037ded06..a733485caf162111 100644
--- a/target/s390x/diag.c
+++ b/target/s390x/diag.c
@@ -88,6 +88,7 @@ static int diag308_parm_check(CPUS390XState *env, uint64_t r1, uint64_t addr,
 void handle_diag_308(CPUS390XState *env, uint64_t r1, uint64_t r3, uintptr_t ra)
 {
     CPUState *cs = env_cpu(env);
+    S390CPU *cpu = S390_CPU(cs);
     uint64_t addr =  env->regs[r1];
     uint64_t subcode = env->regs[r3];
     IplParameterBlock *iplb;
@@ -124,13 +125,22 @@ void handle_diag_308(CPUS390XState *env, uint64_t r1, uint64_t r3, uintptr_t ra)
             return;
         }
         iplb = g_new0(IplParameterBlock, 1);
-        cpu_physical_memory_read(addr, iplb, sizeof(iplb->len));
+        if (!s390_is_pv()) {
+            cpu_physical_memory_read(addr, iplb, sizeof(iplb->len));
+        } else {
+            s390_cpu_pv_mem_read(cpu, 0, iplb, sizeof(iplb->len));
+        }
+
         if (!iplb_valid_len(iplb)) {
             env->regs[r1 + 1] = DIAG_308_RC_INVALID;
             goto out;
         }
 
-        cpu_physical_memory_read(addr, iplb, be32_to_cpu(iplb->len));
+        if (!s390_is_pv()) {
+            cpu_physical_memory_read(addr, iplb, be32_to_cpu(iplb->len));
+        } else {
+            s390_cpu_pv_mem_read(cpu, 0, iplb, be32_to_cpu(iplb->len));
+        }
 
         if (!iplb_valid(iplb)) {
             env->regs[r1 + 1] = DIAG_308_RC_INVALID;
@@ -152,12 +162,17 @@ out:
         } else {
             iplb = s390_ipl_get_iplb();
         }
-        if (iplb) {
-            cpu_physical_memory_write(addr, iplb, be32_to_cpu(iplb->len));
-            env->regs[r1 + 1] = DIAG_308_RC_OK;
-        } else {
+        if (!iplb) {
             env->regs[r1 + 1] = DIAG_308_RC_NO_CONF;
+            return;
         }
+
+        if (!s390_is_pv()) {
+            cpu_physical_memory_write(addr, iplb, be32_to_cpu(iplb->len));
+        } else {
+            s390_cpu_pv_mem_write(cpu, 0, iplb, be32_to_cpu(iplb->len));
+        }
+        env->regs[r1 + 1] = DIAG_308_RC_OK;
         return;
     case DIAG308_PV_START:
         iplb = s390_ipl_get_iplb_pv();
-- 
2.25.1

[PATCH v9 11/15] s390x: protvirt: Disable address checks for PV guest IO emulation

[Janosch Frank]

IO instruction data is routed through SIDAD for protected guests, so
adresses do not need to be checked, as this is kernel memory.

Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
Reviewed-by: Thomas Huth <thuth@redhat.com>
Reviewed-by: David Hildenbrand <david@redhat.com>
---
 target/s390x/ioinst.c | 33 ++++++++++++++++++++++++++-------
 1 file changed, 26 insertions(+), 7 deletions(-)

diff --git a/target/s390x/ioinst.c b/target/s390x/ioinst.c
index c437a1d8c6afed80..481d789de9e09a04 100644
--- a/target/s390x/ioinst.c
+++ b/target/s390x/ioinst.c
@@ -16,6 +16,23 @@
 #include "hw/s390x/ioinst.h"
 #include "trace.h"
 #include "hw/s390x/s390-pci-bus.h"
+#include "hw/s390x/pv.h"
+
+static uint64_t get_address_from_regs(CPUS390XState *env, uint32_t ipb,
+                                      uint8_t *ar)
+{
+    /*
+     * Addresses for protected guests are all offsets into the
+     * satellite block which holds the IO control structures. Those
+     * control structures are always aligned and accessible, so we can
+     * return 0 here which will pass the following address checks.
+     */
+    if (s390_is_pv()) {
+        *ar = 0;
+        return 0;
+    }
+    return decode_basedisp_s(env, ipb, ar);
+}
 
 int ioinst_disassemble_sch_ident(uint32_t value, int *m, int *cssid, int *ssid,
                                  int *schid)
@@ -114,7 +131,7 @@ void ioinst_handle_msch(S390CPU *cpu, uint64_t reg1, uint32_t ipb, uintptr_t ra)
     CPUS390XState *env = &cpu->env;
     uint8_t ar;
 
-    addr = decode_basedisp_s(env, ipb, &ar);
+    addr = get_address_from_regs(env, ipb, &ar);
     if (addr & 3) {
         s390_program_interrupt(env, PGM_SPECIFICATION, ra);
         return;
@@ -171,7 +188,7 @@ void ioinst_handle_ssch(S390CPU *cpu, uint64_t reg1, uint32_t ipb, uintptr_t ra)
     CPUS390XState *env = &cpu->env;
     uint8_t ar;
 
-    addr = decode_basedisp_s(env, ipb, &ar);
+    addr = get_address_from_regs(env, ipb, &ar);
     if (addr & 3) {
         s390_program_interrupt(env, PGM_SPECIFICATION, ra);
         return;
@@ -203,7 +220,7 @@ void ioinst_handle_stcrw(S390CPU *cpu, uint32_t ipb, uintptr_t ra)
     CPUS390XState *env = &cpu->env;
     uint8_t ar;
 
-    addr = decode_basedisp_s(env, ipb, &ar);
+    addr = get_address_from_regs(env, ipb, &ar);
     if (addr & 3) {
         s390_program_interrupt(env, PGM_SPECIFICATION, ra);
         return;
@@ -234,7 +251,7 @@ void ioinst_handle_stsch(S390CPU *cpu, uint64_t reg1, uint32_t ipb,
     CPUS390XState *env = &cpu->env;
     uint8_t ar;
 
-    addr = decode_basedisp_s(env, ipb, &ar);
+    addr = get_address_from_regs(env, ipb, &ar);
     if (addr & 3) {
         s390_program_interrupt(env, PGM_SPECIFICATION, ra);
         return;
@@ -303,7 +320,7 @@ int ioinst_handle_tsch(S390CPU *cpu, uint64_t reg1, uint32_t ipb, uintptr_t ra)
         return -EIO;
     }
     trace_ioinst_sch_id("tsch", cssid, ssid, schid);
-    addr = decode_basedisp_s(env, ipb, &ar);
+    addr = get_address_from_regs(env, ipb, &ar);
     if (addr & 3) {
         s390_program_interrupt(env, PGM_SPECIFICATION, ra);
         return -EIO;
@@ -601,7 +618,7 @@ void ioinst_handle_chsc(S390CPU *cpu, uint32_t ipb, uintptr_t ra)
 {
     ChscReq *req;
     ChscResp *res;
-    uint64_t addr;
+    uint64_t addr = 0;
     int reg;
     uint16_t len;
     uint16_t command;
@@ -610,7 +627,9 @@ void ioinst_handle_chsc(S390CPU *cpu, uint32_t ipb, uintptr_t ra)
 
     trace_ioinst("chsc");
     reg = (ipb >> 20) & 0x00f;
-    addr = env->regs[reg];
+    if (!s390_is_pv()) {
+        addr = env->regs[reg];
+    }
     /* Page boundary? */
     if (addr & 0xfff) {
         s390_program_interrupt(env, PGM_SPECIFICATION, ra);
-- 
2.25.1

[PATCH v9 12/15] s390x: protvirt: Move IO control structures over SIDA

[Janosch Frank]

For protected guests, we need to put the IO emulation results into the
SIDA, so SIE will write them into the guest at the next entry.

Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
---
 target/s390x/ioinst.c | 87 ++++++++++++++++++++++++++++++-------------
 1 file changed, 61 insertions(+), 26 deletions(-)

diff --git a/target/s390x/ioinst.c b/target/s390x/ioinst.c
index 481d789de9e09a04..61095bdc9ffef436 100644
--- a/target/s390x/ioinst.c
+++ b/target/s390x/ioinst.c
@@ -136,9 +136,13 @@ void ioinst_handle_msch(S390CPU *cpu, uint64_t reg1, uint32_t ipb, uintptr_t ra)
         s390_program_interrupt(env, PGM_SPECIFICATION, ra);
         return;
     }
-    if (s390_cpu_virt_mem_read(cpu, addr, ar, &schib, sizeof(schib))) {
-        s390_cpu_virt_mem_handle_exc(cpu, ra);
-        return;
+    if (s390_is_pv()) {
+        s390_cpu_pv_mem_read(cpu, addr, &schib, sizeof(schib));
+    } else {
+        if (s390_cpu_virt_mem_read(cpu, addr, ar, &schib, sizeof(schib))) {
+            s390_cpu_virt_mem_handle_exc(cpu, ra);
+            return;
+        }
     }
     if (ioinst_disassemble_sch_ident(reg1, &m, &cssid, &ssid, &schid) ||
         !ioinst_schib_valid(&schib)) {
@@ -193,9 +197,13 @@ void ioinst_handle_ssch(S390CPU *cpu, uint64_t reg1, uint32_t ipb, uintptr_t ra)
         s390_program_interrupt(env, PGM_SPECIFICATION, ra);
         return;
     }
-    if (s390_cpu_virt_mem_read(cpu, addr, ar, &orig_orb, sizeof(orb))) {
-        s390_cpu_virt_mem_handle_exc(cpu, ra);
-        return;
+    if (s390_is_pv()) {
+        s390_cpu_pv_mem_read(cpu, addr, &orig_orb, sizeof(orb));
+    } else {
+        if (s390_cpu_virt_mem_read(cpu, addr, ar, &orig_orb, sizeof(orb))) {
+            s390_cpu_virt_mem_handle_exc(cpu, ra);
+            return;
+        }
     }
     copy_orb_from_guest(&orb, &orig_orb);
     if (ioinst_disassemble_sch_ident(reg1, &m, &cssid, &ssid, &schid) ||
@@ -229,14 +237,19 @@ void ioinst_handle_stcrw(S390CPU *cpu, uint32_t ipb, uintptr_t ra)
     cc = css_do_stcrw(&crw);
     /* 0 - crw stored, 1 - zeroes stored */
 
-    if (s390_cpu_virt_mem_write(cpu, addr, ar, &crw, sizeof(crw)) == 0) {
+    if (s390_is_pv()) {
+        s390_cpu_pv_mem_write(cpu, addr, &crw, sizeof(crw));
         setcc(cpu, cc);
     } else {
-        if (cc == 0) {
-            /* Write failed: requeue CRW since STCRW is suppressing */
-            css_undo_stcrw(&crw);
+        if (s390_cpu_virt_mem_write(cpu, addr, ar, &crw, sizeof(crw)) == 0) {
+            setcc(cpu, cc);
+        } else {
+            if (cc == 0) {
+                /* Write failed: requeue CRW since STCRW is suppressing */
+                css_undo_stcrw(&crw);
+            }
+            s390_cpu_virt_mem_handle_exc(cpu, ra);
         }
-        s390_cpu_virt_mem_handle_exc(cpu, ra);
     }
 }
 
@@ -258,6 +271,9 @@ void ioinst_handle_stsch(S390CPU *cpu, uint64_t reg1, uint32_t ipb,
     }
 
     if (ioinst_disassemble_sch_ident(reg1, &m, &cssid, &ssid, &schid)) {
+        if (s390_is_pv()) {
+            return;
+        }
         /*
          * As operand exceptions have a lower priority than access exceptions,
          * we check whether the memory area is writeable (injecting the
@@ -290,14 +306,19 @@ void ioinst_handle_stsch(S390CPU *cpu, uint64_t reg1, uint32_t ipb,
         }
     }
     if (cc != 3) {
-        if (s390_cpu_virt_mem_write(cpu, addr, ar, &schib,
-                                    sizeof(schib)) != 0) {
-            s390_cpu_virt_mem_handle_exc(cpu, ra);
-            return;
+        if (s390_is_pv()) {
+            s390_cpu_pv_mem_write(cpu, addr, &schib, sizeof(schib));
+        } else {
+            if (s390_cpu_virt_mem_write(cpu, addr, ar, &schib,
+                                        sizeof(schib)) != 0) {
+                s390_cpu_virt_mem_handle_exc(cpu, ra);
+                return;
+            }
         }
     } else {
         /* Access exceptions have a higher priority than cc3 */
-        if (s390_cpu_virt_mem_check_write(cpu, addr, ar, sizeof(schib)) != 0) {
+        if (!s390_is_pv() &&
+            s390_cpu_virt_mem_check_write(cpu, addr, ar, sizeof(schib)) != 0) {
             s390_cpu_virt_mem_handle_exc(cpu, ra);
             return;
         }
@@ -334,15 +355,20 @@ int ioinst_handle_tsch(S390CPU *cpu, uint64_t reg1, uint32_t ipb, uintptr_t ra)
     }
     /* 0 - status pending, 1 - not status pending, 3 - not operational */
     if (cc != 3) {
-        if (s390_cpu_virt_mem_write(cpu, addr, ar, &irb, irb_len) != 0) {
-            s390_cpu_virt_mem_handle_exc(cpu, ra);
-            return -EFAULT;
+        if (s390_is_pv()) {
+            s390_cpu_pv_mem_write(cpu, addr, &irb, irb_len);
+        } else {
+            if (s390_cpu_virt_mem_write(cpu, addr, ar, &irb, irb_len) != 0) {
+                s390_cpu_virt_mem_handle_exc(cpu, ra);
+                return -EFAULT;
+            }
         }
         css_do_tsch_update_subch(sch);
     } else {
         irb_len = sizeof(irb) - sizeof(irb.emw);
         /* Access exceptions have a higher priority than cc3 */
-        if (s390_cpu_virt_mem_check_write(cpu, addr, ar, irb_len) != 0) {
+        if (!s390_is_pv() &&
+            s390_cpu_virt_mem_check_write(cpu, addr, ar, irb_len) != 0) {
             s390_cpu_virt_mem_handle_exc(cpu, ra);
             return -EFAULT;
         }
@@ -640,9 +666,13 @@ void ioinst_handle_chsc(S390CPU *cpu, uint32_t ipb, uintptr_t ra)
      * present CHSC sub-handlers ... if we ever need more, we should take
      * care of req->len here first.
      */
-    if (s390_cpu_virt_mem_read(cpu, addr, reg, buf, sizeof(ChscReq))) {
-        s390_cpu_virt_mem_handle_exc(cpu, ra);
-        return;
+    if (s390_is_pv()) {
+        s390_cpu_pv_mem_read(cpu, addr, buf, sizeof(ChscReq));
+    } else {
+        if (s390_cpu_virt_mem_read(cpu, addr, reg, buf, sizeof(ChscReq))) {
+            s390_cpu_virt_mem_handle_exc(cpu, ra);
+            return;
+        }
     }
     req = (ChscReq *)buf;
     len = be16_to_cpu(req->len);
@@ -673,11 +703,16 @@ void ioinst_handle_chsc(S390CPU *cpu, uint32_t ipb, uintptr_t ra)
         break;
     }
 
-    if (!s390_cpu_virt_mem_write(cpu, addr + len, reg, res,
-                                 be16_to_cpu(res->len))) {
+    if (s390_is_pv()) {
+        s390_cpu_pv_mem_write(cpu, addr + len, res, be16_to_cpu(res->len));
         setcc(cpu, 0);    /* Command execution complete */
     } else {
-        s390_cpu_virt_mem_handle_exc(cpu, ra);
+        if (!s390_cpu_virt_mem_write(cpu, addr + len, reg, res,
+                                     be16_to_cpu(res->len))) {
+            setcc(cpu, 0);    /* Command execution complete */
+        } else {
+            s390_cpu_virt_mem_handle_exc(cpu, ra);
+        }
     }
 }
 
-- 
2.25.1

[PATCH v9 13/15] s390x: protvirt: Handle SIGP store status correctly

[Janosch Frank]

For protected VMs status storing is not done by QEMU anymore.

Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
Reviewed-by: Thomas Huth <thuth@redhat.com>
Reviewed-by: David Hildenbrand <david@redhat.com>
---
 target/s390x/helper.c | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/target/s390x/helper.c b/target/s390x/helper.c
index ed726849114f2f35..5022df8812d406c9 100644
--- a/target/s390x/helper.c
+++ b/target/s390x/helper.c
@@ -25,6 +25,7 @@
 #include "qemu/timer.h"
 #include "qemu/qemu-print.h"
 #include "hw/s390x/ioinst.h"
+#include "hw/s390x/pv.h"
 #include "sysemu/hw_accel.h"
 #include "sysemu/runstate.h"
 #ifndef CONFIG_USER_ONLY
@@ -246,6 +247,11 @@ int s390_store_status(S390CPU *cpu, hwaddr addr, bool store_arch)
     hwaddr len = sizeof(*sa);
     int i;
 
+    /* Storing will occur on next SIE entry for protected VMs */
+    if (s390_is_pv()) {
+        return 0;
+    }
+
     sa = cpu_physical_memory_map(addr, &len, true);
     if (!sa) {
         return -EFAULT;
-- 
2.25.1

[PATCH v9 14/15] docs: Add protvirt docs

[Janosch Frank]

Lets add some documentation for the Protected VM functionality.

Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
---
 docs/system/index.rst    |  1 +
 docs/system/protvirt.rst | 56 ++++++++++++++++++++++++++++++++++++++++
 2 files changed, 57 insertions(+)
 create mode 100644 docs/system/protvirt.rst

diff --git a/docs/system/index.rst b/docs/system/index.rst
index 6e5f20fa1333ce23..74afbd7cc3fc0296 100644
--- a/docs/system/index.rst
+++ b/docs/system/index.rst
@@ -34,3 +34,4 @@ Contents:
    deprecated
    build-platforms
    license
+   protvirt
diff --git a/docs/system/protvirt.rst b/docs/system/protvirt.rst
new file mode 100644
index 0000000000000000..6c8cf0f7910eae86
--- /dev/null
+++ b/docs/system/protvirt.rst
@@ -0,0 +1,56 @@
+Protected Virtualization on s390x
+=================================
+
+The memory and most of the registers of Protected Virtual Machines
+(PVMs) are encrypted or inaccessible to the hypervisor, effectively
+prohibiting VM introspection when the VM is running. At rest, PVMs are
+encrypted and can only be decrypted by the firmware, represented by an
+entity called Ultravisor, of specific IBM Z machines.
+
+
+Prerequisites
+-------------
+
+To run PVMs a machine with the Protected Virtualization feature
+which is indicated by the Ultravisor Call facility (stfle bit
+158) is required. The Ultravisor needs to be initialized at boot by
+setting `prot_virt=1` on the kernel command line.
+
+If those requirements are met, the capability `KVM_CAP_S390_PROTECTED`
+will indicate that KVM can support PVMs on that LPAR.
+
+
+QEMU Settings
+-------------
+
+To indicate to the VM that it can transition into protected mode, the
+`Unpack facility` (stfle bit 161 represented by the feature
+`S390_FEAT_UNPACK`) needs to be part of the cpu model of the VM.
+
+All I/O devices need to use the IOMMU.
+Passthrough (vfio) devices are currently not supported.
+
+Host huge page backings are not supported. However guests can use huge
+pages as indicated by its facilities.
+
+
+Boot Process
+------------
+
+A secure guest image can either be loaded from disk or supplied on the
+QEMU command line. Booting from disk is done by the unmodified
+s390-ccw BIOS. I.e., the bootmap is interpreted, multiple components
+are read into memory and control is transferred to one of the
+components (zipl stage3). Stag3 does some fixups and then transfers
+control to some program residing in guest memory, which is normally
+the OS kernel. The secure image has another component prepended
+(stage3a) that uses the new diag308 subcodes 8 and 10 to trigger the
+transition into secure mode.
+
+Booting from the image supplied via the QEMU command line requires
+that the file passed via -kernel has the same memory layout as would
+result from the disk boot. This memory layout includes the encrypted
+components (kernel, initrd, cmdline), the stage3a loader and
+metadata. In case this boot method is used, the command line
+options -initrd and -cmdline are ineffective. The preparation of a PVM
+image is done by genprotimg of the s390-tools package.
-- 
2.25.1

[PATCH v9 15/15] s390x: Add unpack facility feature to GA1

[Janosch Frank]

From: Christian Borntraeger <borntraeger@de.ibm.com>

The unpack facility is an indication that diagnose 308 subcodes 8-10
are available to the guest. That means, that the guest can put itself
into protected mode.

Once it is in protected mode, the hardware stops any attempt of VM
introspection by the hypervisor.

Some features are currently not supported in protected mode:
     * Passthrough devices
     * Migration
     * Huge page backings

Signed-off-by: Christian Borntraeger <borntraeger@de.ibm.com>
Reviewed-by: David Hildenbrand <david@redhat.com>
---
 target/s390x/gen-features.c | 1 +
 target/s390x/kvm.c          | 5 +++++
 2 files changed, 6 insertions(+)

diff --git a/target/s390x/gen-features.c b/target/s390x/gen-features.c
index 6278845b12b8dee8..8ddeebc54419a3e2 100644
--- a/target/s390x/gen-features.c
+++ b/target/s390x/gen-features.c
@@ -562,6 +562,7 @@ static uint16_t full_GEN15_GA1[] = {
     S390_FEAT_GROUP_MSA_EXT_9,
     S390_FEAT_GROUP_MSA_EXT_9_PCKMO,
     S390_FEAT_ETOKEN,
+    S390_FEAT_UNPACK,
 };
 
 /* Default features (in order of release)
diff --git a/target/s390x/kvm.c b/target/s390x/kvm.c
index ff6027036ec2f14a..e11e895a3d9038bb 100644
--- a/target/s390x/kvm.c
+++ b/target/s390x/kvm.c
@@ -2403,6 +2403,11 @@ void kvm_s390_get_host_cpu_model(S390CPUModel *model, Error **errp)
         clear_bit(S390_FEAT_BPB, model->features);
     }
 
+    /* we do have the IPL enhancements */
+    if (cap_protected) {
+        set_bit(S390_FEAT_UNPACK, model->features);
+    }
+
     /* We emulate a zPCI bus and AEN, therefore we don't need HW support */
     set_bit(S390_FEAT_ZPCI, model->features);
     set_bit(S390_FEAT_ADAPTER_EVENT_NOTIFICATION, model->features);
-- 
2.25.1

Re: [PATCH v9 08/15] s390x: protvirt: SCLP interpretation

[David Hildenbrand]

> + * We only need the address to have something valid for the
> + * service_interrupt call.
> + */
> +#define SCLP_PV_DUMMY_ADDR 0x4000
> +int sclp_service_call_protected(CPUS390XState *env, uint64_t sccb,
> +                                uint32_t code)
> +{
> +    SCLPDevice *sclp = get_sclp_device();
> +    SCLPDeviceClass *sclp_c = SCLP_GET_CLASS(sclp);
> +    SCCB work_sccb;
> +    hwaddr sccb_len = sizeof(SCCB);
> +
> +    /*
> +     * Only a very limited amount of calls is permitted by the

s/amount/number/ ?


-- 
Thanks,

David / dhildenb

Re: [PATCH v9 08/15] s390x: protvirt: SCLP interpretation

[Janosch Frank]

[-- Attachment #1.1: Type: text/plain, Size: 625 bytes --]

On 3/11/20 2:24 PM, David Hildenbrand wrote:
> 
>> + * We only need the address to have something valid for the
>> + * service_interrupt call.
>> + */
>> +#define SCLP_PV_DUMMY_ADDR 0x4000
>> +int sclp_service_call_protected(CPUS390XState *env, uint64_t sccb,
>> +                                uint32_t code)
>> +{
>> +    SCLPDevice *sclp = get_sclp_device();
>> +    SCLPDeviceClass *sclp_c = SCLP_GET_CLASS(sclp);
>> +    SCCB work_sccb;
>> +    hwaddr sccb_len = sizeof(SCCB);
>> +
>> +    /*
>> +     * Only a very limited amount of calls is permitted by the
> 
> s/amount/number/ ?
> 
> 

Ack


[-- Attachment #2: OpenPGP digital signature --]
[-- Type: application/pgp-signature, Size: 833 bytes --]

Re: [PATCH v9 00/15] s390x: Protected Virtualization support

[no-reply]

Patchew URL: https://patchew.org/QEMU/20200311132151.172389-1-frankja@linux.ibm.com/



Hi,

This series seems to have some coding style problems. See output below for
more information:

Subject: [PATCH v9 00/15] s390x: Protected Virtualization support
Message-id: 20200311132151.172389-1-frankja@linux.ibm.com
Type: series

=== TEST SCRIPT BEGIN ===
#!/bin/bash
git rev-parse base > /dev/null || exit 0
git config --local diff.renamelimit 0
git config --local diff.renames True
git config --local diff.algorithm histogram
./scripts/checkpatch.pl --mailback base..
=== TEST SCRIPT END ===

Updating 3c8cf5a9c21ff8782164d1def7f44bd888713384
Switched to a new branch 'test'
5091111 s390x: Add unpack facility feature to GA1
3017941 docs: Add protvirt docs
8971626 s390x: protvirt: Handle SIGP store status correctly
80534f0 s390x: protvirt: Move IO control structures over SIDA
6469c9e s390x: protvirt: Disable address checks for PV guest IO emulation
fcf17b9 s390x: protvirt: Move diag 308 data over SIDA
bc7ccf1 s390x: protvirt: Set guest IPL PSW
0dc6509 s390x: protvirt: SCLP interpretation
55b33fb s390x: protvirt: Move STSI data over SIDAD
cd885bc s390x: Add SIDA memory ops
420161d s390x: protvirt: KVM intercept changes
df0e73f s390x: protvirt: Inhibit balloon when switching to protected mode
1a573ba s390x: protvirt: Add migration blocker
f214aa5 s390x: protvirt: Support unpack facility
9eb0129 Sync pv

=== OUTPUT BEGIN ===
1/15 Checking commit 9eb01294c7d7 (Sync pv)
2/15 Checking commit f214aa5c8796 (s390x: protvirt: Support unpack facility)
WARNING: added, moved or deleted file(s), does MAINTAINERS need updating?
#274: 
new file mode 100644

WARNING: line over 80 characters
#600: FILE: include/hw/s390x/pv.h:48:
+static inline int s390_pv_set_sec_parms(uint64_t origin, uint64_t length) { return 0; }

ERROR: line over 90 characters
#601: FILE: include/hw/s390x/pv.h:49:
+static inline int s390_pv_unpack(uint64_t addr, uint64_t size, uint64_t tweak) { return 0; }

total: 1 errors, 2 warnings, 642 lines checked

Patch 2/15 has style problems, please review.  If any of these errors
are false positives report them to the maintainer, see
CHECKPATCH in MAINTAINERS.

3/15 Checking commit 1a573ba2b449 (s390x: protvirt: Add migration blocker)
4/15 Checking commit df0e73f31901 (s390x: protvirt: Inhibit balloon when switching to protected mode)
5/15 Checking commit 420161d14911 (s390x: protvirt: KVM intercept changes)
ERROR: switch and case should be at the same indent
#48: FILE: target/s390x/kvm.c:1696:
     switch (icpt_code) {
[...]
+        case ICPT_PV_INSTR:
+        case ICPT_PV_INSTR_NOTIFICATION:

total: 1 errors, 0 warnings, 16 lines checked

Patch 5/15 has style problems, please review.  If any of these errors
are false positives report them to the maintainer, see
CHECKPATCH in MAINTAINERS.

6/15 Checking commit cd885bcb5756 (s390x: Add SIDA memory ops)
7/15 Checking commit 55b33fb822b7 (s390x: protvirt: Move STSI data over SIDAD)
8/15 Checking commit 0dc650979f79 (s390x: protvirt: SCLP interpretation)
9/15 Checking commit bc7ccf1d1d99 (s390x: protvirt: Set guest IPL PSW)
10/15 Checking commit fcf17b96e7cc (s390x: protvirt: Move diag 308 data over SIDA)
11/15 Checking commit 6469c9e84dab (s390x: protvirt: Disable address checks for PV guest IO emulation)
12/15 Checking commit 80534f0309d2 (s390x: protvirt: Move IO control structures over SIDA)
13/15 Checking commit 8971626e3e69 (s390x: protvirt: Handle SIGP store status correctly)
14/15 Checking commit 3017941d6e2c (docs: Add protvirt docs)
WARNING: added, moved or deleted file(s), does MAINTAINERS need updating?
#22: 
new file mode 100644

total: 0 errors, 1 warnings, 60 lines checked

Patch 14/15 has style problems, please review.  If any of these errors
are false positives report them to the maintainer, see
CHECKPATCH in MAINTAINERS.
15/15 Checking commit 509111178918 (s390x: Add unpack facility feature to GA1)
=== OUTPUT END ===

Test command exited with code: 1


The full log is available at
http://patchew.org/logs/20200311132151.172389-1-frankja@linux.ibm.com/testing.checkpatch/?type=message.
---
Email generated automatically by Patchew [https://patchew.org/].
Please send your feedback to patchew-devel@redhat.com

Re: [PATCH v9 00/15] s390x: Protected Virtualization support

[no-reply]

Patchew URL: https://patchew.org/QEMU/20200311132151.172389-1-frankja@linux.ibm.com/



Hi,

This series failed the asan build test. Please find the testing commands and
their output below. If you have Docker installed, you can probably reproduce it
locally.

=== TEST SCRIPT BEGIN ===
#!/bin/bash
export ARCH=x86_64
make docker-image-fedora V=1 NETWORK=1
time make docker-test-debug@fedora TARGET_LIST=x86_64-softmmu J=14 NETWORK=1
=== TEST SCRIPT END ===

PASS 1 fdc-test /x86_64/fdc/cmos
PASS 2 fdc-test /x86_64/fdc/no_media_on_start
PASS 3 fdc-test /x86_64/fdc/read_without_media
==6307==WARNING: ASan doesn't fully support makecontext/swapcontext functions and may produce false positives in some cases!
PASS 4 fdc-test /x86_64/fdc/media_change
PASS 5 fdc-test /x86_64/fdc/sense_interrupt
PASS 6 fdc-test /x86_64/fdc/relative_seek
---
PASS 32 test-opts-visitor /visitor/opts/range/beyond
PASS 33 test-opts-visitor /visitor/opts/dict/unvisited
MALLOC_PERTURB_=${MALLOC_PERTURB_:-$(( ${RANDOM:-0} % 255 + 1))}  tests/test-coroutine -m=quick -k --tap < /dev/null | ./scripts/tap-driver.pl --test-name="test-coroutine" 
==6345==WARNING: ASan doesn't fully support makecontext/swapcontext functions and may produce false positives in some cases!
==6345==WARNING: ASan is ignoring requested __asan_handle_no_return: stack top: 0x7ffc06eb9000; bottom 0x7f1283fb7000; size: 0x00e982f02000 (1002924154880)
False positive error reports may follow
For details see https://github.com/google/sanitizers/issues/189
PASS 1 test-coroutine /basic/no-dangling-access
---
PASS 13 test-aio /aio/event/wait/no-flush-cb
PASS 11 fdc-test /x86_64/fdc/read_no_dma_18
PASS 14 test-aio /aio/timer/schedule
==6360==WARNING: ASan doesn't fully support makecontext/swapcontext functions and may produce false positives in some cases!
PASS 15 test-aio /aio/coroutine/queue-chaining
PASS 16 test-aio /aio-gsource/flush
PASS 17 test-aio /aio-gsource/bh/schedule
---
PASS 28 test-aio /aio-gsource/timer/schedule
MALLOC_PERTURB_=${MALLOC_PERTURB_:-$(( ${RANDOM:-0} % 255 + 1))}  QTEST_QEMU_BINARY=x86_64-softmmu/qemu-system-x86_64 QTEST_QEMU_IMG=qemu-img tests/qtest/ide-test -m=quick -k --tap < /dev/null | ./scripts/tap-driver.pl --test-name="ide-test" 
MALLOC_PERTURB_=${MALLOC_PERTURB_:-$(( ${RANDOM:-0} % 255 + 1))}  tests/test-aio-multithread -m=quick -k --tap < /dev/null | ./scripts/tap-driver.pl --test-name="test-aio-multithread" 
==6368==WARNING: ASan doesn't fully support makecontext/swapcontext functions and may produce false positives in some cases!
PASS 1 test-aio-multithread /aio/multi/lifecycle
==6371==WARNING: ASan doesn't fully support makecontext/swapcontext functions and may produce false positives in some cases!
PASS 1 ide-test /x86_64/ide/identify
PASS 2 test-aio-multithread /aio/multi/schedule
==6388==WARNING: ASan doesn't fully support makecontext/swapcontext functions and may produce false positives in some cases!
PASS 2 ide-test /x86_64/ide/flush
==6399==WARNING: ASan doesn't fully support makecontext/swapcontext functions and may produce false positives in some cases!
PASS 3 ide-test /x86_64/ide/bmdma/simple_rw
PASS 3 test-aio-multithread /aio/multi/mutex/contended
==6405==WARNING: ASan doesn't fully support makecontext/swapcontext functions and may produce false positives in some cases!
PASS 4 ide-test /x86_64/ide/bmdma/trim
==6416==WARNING: ASan doesn't fully support makecontext/swapcontext functions and may produce false positives in some cases!
PASS 4 test-aio-multithread /aio/multi/mutex/handoff
PASS 5 test-aio-multithread /aio/multi/mutex/mcs
PASS 6 test-aio-multithread /aio/multi/mutex/pthread
MALLOC_PERTURB_=${MALLOC_PERTURB_:-$(( ${RANDOM:-0} % 255 + 1))}  tests/test-throttle -m=quick -k --tap < /dev/null | ./scripts/tap-driver.pl --test-name="test-throttle" 
==6433==WARNING: ASan doesn't fully support makecontext/swapcontext functions and may produce false positives in some cases!
PASS 1 test-throttle /throttle/leak_bucket
PASS 2 test-throttle /throttle/compute_wait
PASS 3 test-throttle /throttle/init
---
PASS 14 test-throttle /throttle/config/max
PASS 15 test-throttle /throttle/config/iops_size
MALLOC_PERTURB_=${MALLOC_PERTURB_:-$(( ${RANDOM:-0} % 255 + 1))}  tests/test-thread-pool -m=quick -k --tap < /dev/null | ./scripts/tap-driver.pl --test-name="test-thread-pool" 
==6437==WARNING: ASan doesn't fully support makecontext/swapcontext functions and may produce false positives in some cases!
PASS 1 test-thread-pool /thread-pool/submit
PASS 2 test-thread-pool /thread-pool/submit-aio
PASS 3 test-thread-pool /thread-pool/submit-co
PASS 4 test-thread-pool /thread-pool/submit-many
==6504==WARNING: ASan doesn't fully support makecontext/swapcontext functions and may produce false positives in some cases!
PASS 5 test-thread-pool /thread-pool/cancel
PASS 6 test-thread-pool /thread-pool/cancel-async
MALLOC_PERTURB_=${MALLOC_PERTURB_:-$(( ${RANDOM:-0} % 255 + 1))}  tests/test-hbitmap -m=quick -k --tap < /dev/null | ./scripts/tap-driver.pl --test-name="test-hbitmap" 
---
PASS 15 test-hbitmap /hbitmap/set/overlap
PASS 16 test-hbitmap /hbitmap/reset/empty
PASS 17 test-hbitmap /hbitmap/reset/general
==6514==WARNING: ASan doesn't fully support makecontext/swapcontext functions and may produce false positives in some cases!
PASS 18 test-hbitmap /hbitmap/reset/all
PASS 19 test-hbitmap /hbitmap/truncate/nop
PASS 20 test-hbitmap /hbitmap/truncate/grow/negligible
---
PASS 31 test-hbitmap /hbitmap/meta/one
PASS 32 test-hbitmap /hbitmap/meta/byte
PASS 33 test-hbitmap /hbitmap/meta/word
==6520==WARNING: ASan doesn't fully support makecontext/swapcontext functions and may produce false positives in some cases!
PASS 34 test-hbitmap /hbitmap/meta/sector
PASS 35 test-hbitmap /hbitmap/serialize/align
PASS 36 test-hbitmap /hbitmap/serialize/basic
---
PASS 44 test-hbitmap /hbitmap/next_dirty_area/next_dirty_area_4
PASS 45 test-hbitmap /hbitmap/next_dirty_area/next_dirty_area_after_truncate
MALLOC_PERTURB_=${MALLOC_PERTURB_:-$(( ${RANDOM:-0} % 255 + 1))}  tests/test-bdrv-drain -m=quick -k --tap < /dev/null | ./scripts/tap-driver.pl --test-name="test-bdrv-drain" 
==6527==WARNING: ASan doesn't fully support makecontext/swapcontext functions and may produce false positives in some cases!
PASS 1 test-bdrv-drain /bdrv-drain/nested
PASS 2 test-bdrv-drain /bdrv-drain/multiparent
PASS 3 test-bdrv-drain /bdrv-drain/set_aio_context
---
PASS 41 test-bdrv-drain /bdrv-drain/bdrv_drop_intermediate/poll
PASS 42 test-bdrv-drain /bdrv-drain/replace_child/mid-drain
MALLOC_PERTURB_=${MALLOC_PERTURB_:-$(( ${RANDOM:-0} % 255 + 1))}  tests/test-bdrv-graph-mod -m=quick -k --tap < /dev/null | ./scripts/tap-driver.pl --test-name="test-bdrv-graph-mod" 
==6566==WARNING: ASan doesn't fully support makecontext/swapcontext functions and may produce false positives in some cases!
PASS 1 test-bdrv-graph-mod /bdrv-graph-mod/update-perm-tree
PASS 2 test-bdrv-graph-mod /bdrv-graph-mod/should-update-child
MALLOC_PERTURB_=${MALLOC_PERTURB_:-$(( ${RANDOM:-0} % 255 + 1))}  tests/test-blockjob -m=quick -k --tap < /dev/null | ./scripts/tap-driver.pl --test-name="test-blockjob" 
==6570==WARNING: ASan doesn't fully support makecontext/swapcontext functions and may produce false positives in some cases!
PASS 1 test-blockjob /blockjob/ids
PASS 2 test-blockjob /blockjob/cancel/created
PASS 3 test-blockjob /blockjob/cancel/running
---
PASS 7 test-blockjob /blockjob/cancel/pending
PASS 8 test-blockjob /blockjob/cancel/concluded
MALLOC_PERTURB_=${MALLOC_PERTURB_:-$(( ${RANDOM:-0} % 255 + 1))}  tests/test-blockjob-txn -m=quick -k --tap < /dev/null | ./scripts/tap-driver.pl --test-name="test-blockjob-txn" 
==6574==WARNING: ASan doesn't fully support makecontext/swapcontext functions and may produce false positives in some cases!
PASS 1 test-blockjob-txn /single/success
PASS 2 test-blockjob-txn /single/failure
PASS 3 test-blockjob-txn /single/cancel
---
PASS 6 test-blockjob-txn /pair/cancel
PASS 7 test-blockjob-txn /pair/fail-cancel-race
MALLOC_PERTURB_=${MALLOC_PERTURB_:-$(( ${RANDOM:-0} % 255 + 1))}  tests/test-block-backend -m=quick -k --tap < /dev/null | ./scripts/tap-driver.pl --test-name="test-block-backend" 
==6578==WARNING: ASan doesn't fully support makecontext/swapcontext functions and may produce false positives in some cases!
PASS 1 test-block-backend /block-backend/drain_aio_error
PASS 2 test-block-backend /block-backend/drain_all_aio_error
MALLOC_PERTURB_=${MALLOC_PERTURB_:-$(( ${RANDOM:-0} % 255 + 1))}  tests/test-block-iothread -m=quick -k --tap < /dev/null | ./scripts/tap-driver.pl --test-name="test-block-iothread" 
==6582==WARNING: ASan doesn't fully support makecontext/swapcontext functions and may produce false positives in some cases!
PASS 1 test-block-iothread /sync-op/pread
PASS 2 test-block-iothread /sync-op/pwrite
PASS 3 test-block-iothread /sync-op/load_vmstate
---
PASS 15 test-block-iothread /propagate/diamond
PASS 16 test-block-iothread /propagate/mirror
MALLOC_PERTURB_=${MALLOC_PERTURB_:-$(( ${RANDOM:-0} % 255 + 1))}  tests/test-image-locking -m=quick -k --tap < /dev/null | ./scripts/tap-driver.pl --test-name="test-image-locking" 
==6602==WARNING: ASan doesn't fully support makecontext/swapcontext functions and may produce false positives in some cases!
PASS 1 test-image-locking /image-locking/basic
PASS 2 test-image-locking /image-locking/set-perm-abort
MALLOC_PERTURB_=${MALLOC_PERTURB_:-$(( ${RANDOM:-0} % 255 + 1))}  tests/test-x86-cpuid -m=quick -k --tap < /dev/null | ./scripts/tap-driver.pl --test-name="test-x86-cpuid" 
---
PASS 5 test-xbzrle /xbzrle/encode_decode_overflow
PASS 6 test-xbzrle /xbzrle/encode_decode
MALLOC_PERTURB_=${MALLOC_PERTURB_:-$(( ${RANDOM:-0} % 255 + 1))}  tests/test-vmstate -m=quick -k --tap < /dev/null | ./scripts/tap-driver.pl --test-name="test-vmstate" 
==6611==WARNING: ASan doesn't fully support makecontext/swapcontext functions and may produce false positives in some cases!
PASS 1 test-vmstate /vmstate/tmp_struct
PASS 2 test-vmstate /vmstate/simple/primitive
PASS 3 test-vmstate /vmstate/simple/array
---
MALLOC_PERTURB_=${MALLOC_PERTURB_:-$(( ${RANDOM:-0} % 255 + 1))}  tests/test-rcu-list -m=quick -k --tap < /dev/null | ./scripts/tap-driver.pl --test-name="test-rcu-list" 
PASS 1 test-rcu-list /rcu/qlist/single-threaded
PASS 2 test-rcu-list /rcu/qlist/short-few
==6672==WARNING: ASan doesn't fully support makecontext/swapcontext functions and may produce false positives in some cases!
PASS 3 test-rcu-list /rcu/qlist/long-many
MALLOC_PERTURB_=${MALLOC_PERTURB_:-$(( ${RANDOM:-0} % 255 + 1))}  tests/test-rcu-simpleq -m=quick -k --tap < /dev/null | ./scripts/tap-driver.pl --test-name="test-rcu-simpleq" 
PASS 1 test-rcu-simpleq /rcu/qsimpleq/single-threaded
PASS 2 test-rcu-simpleq /rcu/qsimpleq/short-few
PASS 3 test-rcu-simpleq /rcu/qsimpleq/long-many
MALLOC_PERTURB_=${MALLOC_PERTURB_:-$(( ${RANDOM:-0} % 255 + 1))}  tests/test-rcu-tailq -m=quick -k --tap < /dev/null | ./scripts/tap-driver.pl --test-name="test-rcu-tailq" 
==6732==WARNING: ASan doesn't fully support makecontext/swapcontext functions and may produce false positives in some cases!
PASS 1 test-rcu-tailq /rcu/qtailq/single-threaded
PASS 2 test-rcu-tailq /rcu/qtailq/short-few
PASS 3 test-rcu-tailq /rcu/qtailq/long-many
MALLOC_PERTURB_=${MALLOC_PERTURB_:-$(( ${RANDOM:-0} % 255 + 1))}  tests/test-rcu-slist -m=quick -k --tap < /dev/null | ./scripts/tap-driver.pl --test-name="test-rcu-slist" 
PASS 1 test-rcu-slist /rcu/qslist/single-threaded
PASS 2 test-rcu-slist /rcu/qslist/short-few
==6783==WARNING: ASan doesn't fully support makecontext/swapcontext functions and may produce false positives in some cases!
PASS 3 test-rcu-slist /rcu/qslist/long-many
MALLOC_PERTURB_=${MALLOC_PERTURB_:-$(( ${RANDOM:-0} % 255 + 1))}  tests/test-qdist -m=quick -k --tap < /dev/null | ./scripts/tap-driver.pl --test-name="test-qdist" 
PASS 1 test-qdist /qdist/none
---
PASS 7 test-qdist /qdist/binning/expand
PASS 8 test-qdist /qdist/binning/shrink
MALLOC_PERTURB_=${MALLOC_PERTURB_:-$(( ${RANDOM:-0} % 255 + 1))}  tests/test-qht -m=quick -k --tap < /dev/null | ./scripts/tap-driver.pl --test-name="test-qht" 
==6817==WARNING: ASan doesn't fully support makecontext/swapcontext functions and may produce false positives in some cases!
PASS 5 ide-test /x86_64/ide/bmdma/various_prdts
==6823==WARNING: ASan doesn't fully support makecontext/swapcontext functions and may produce false positives in some cases!
==6823==WARNING: ASan is ignoring requested __asan_handle_no_return: stack top: 0x7fff14e49000; bottom 0x7f70779fe000; size: 0x008e9d44b000 (612523880448)
False positive error reports may follow
For details see https://github.com/google/sanitizers/issues/189
PASS 6 ide-test /x86_64/ide/bmdma/no_busmaster
PASS 7 ide-test /x86_64/ide/flush/nodev
==6834==WARNING: ASan doesn't fully support makecontext/swapcontext functions and may produce false positives in some cases!
PASS 8 ide-test /x86_64/ide/flush/empty_drive
==6839==WARNING: ASan doesn't fully support makecontext/swapcontext functions and may produce false positives in some cases!
PASS 9 ide-test /x86_64/ide/flush/retry_pci
==6845==WARNING: ASan doesn't fully support makecontext/swapcontext functions and may produce false positives in some cases!
PASS 10 ide-test /x86_64/ide/flush/retry_isa
==6851==WARNING: ASan doesn't fully support makecontext/swapcontext functions and may produce false positives in some cases!
PASS 11 ide-test /x86_64/ide/cdrom/pio
==6857==WARNING: ASan doesn't fully support makecontext/swapcontext functions and may produce false positives in some cases!
PASS 1 test-qht /qht/mode/default
PASS 2 test-qht /qht/mode/resize
MALLOC_PERTURB_=${MALLOC_PERTURB_:-$(( ${RANDOM:-0} % 255 + 1))}  tests/test-qht-par -m=quick -k --tap < /dev/null | ./scripts/tap-driver.pl --test-name="test-qht-par" 
PASS 12 ide-test /x86_64/ide/cdrom/pio_large
==6872==WARNING: ASan doesn't fully support makecontext/swapcontext functions and may produce false positives in some cases!
PASS 13 ide-test /x86_64/ide/cdrom/dma
MALLOC_PERTURB_=${MALLOC_PERTURB_:-$(( ${RANDOM:-0} % 255 + 1))}  QTEST_QEMU_BINARY=x86_64-softmmu/qemu-system-x86_64 QTEST_QEMU_IMG=qemu-img tests/qtest/ahci-test -m=quick -k --tap < /dev/null | ./scripts/tap-driver.pl --test-name="ahci-test" 
PASS 1 test-qht-par /qht/parallel/2threads-0%updates-1s
PASS 2 test-qht-par /qht/parallel/2threads-20%updates-1s
MALLOC_PERTURB_=${MALLOC_PERTURB_:-$(( ${RANDOM:-0} % 255 + 1))}  tests/test-bitops -m=quick -k --tap < /dev/null | ./scripts/tap-driver.pl --test-name="test-bitops" 
==6892==WARNING: ASan doesn't fully support makecontext/swapcontext functions and may produce false positives in some cases!
PASS 1 test-bitops /bitops/sextract32
PASS 2 test-bitops /bitops/sextract64
PASS 3 test-bitops /bitops/half_shuffle32
---
PASS 3 test-qdev-global-props /qdev/properties/dynamic/global
PASS 4 test-qdev-global-props /qdev/properties/global/subclass
MALLOC_PERTURB_=${MALLOC_PERTURB_:-$(( ${RANDOM:-0} % 255 + 1))}  tests/check-qom-interface -m=quick -k --tap < /dev/null | ./scripts/tap-driver.pl --test-name="check-qom-interface" 
==6904==WARNING: ASan doesn't fully support makecontext/swapcontext functions and may produce false positives in some cases!
PASS 1 check-qom-interface /qom/interface/direct_impl
PASS 2 check-qom-interface /qom/interface/intermediate_impl
MALLOC_PERTURB_=${MALLOC_PERTURB_:-$(( ${RANDOM:-0} % 255 + 1))}  tests/check-qom-proplist -m=quick -k --tap < /dev/null | ./scripts/tap-driver.pl --test-name="check-qom-proplist" 
---
PASS 18 test-qemu-opts /qemu-opts/to_qdict/filtered
PASS 19 test-qemu-opts /qemu-opts/to_qdict/duplicates
MALLOC_PERTURB_=${MALLOC_PERTURB_:-$(( ${RANDOM:-0} % 255 + 1))}  tests/test-keyval -m=quick -k --tap < /dev/null | ./scripts/tap-driver.pl --test-name="test-keyval" 
==6932==WARNING: ASan doesn't fully support makecontext/swapcontext functions and may produce false positives in some cases!
PASS 1 test-keyval /keyval/keyval_parse
PASS 2 test-keyval /keyval/keyval_parse/list
PASS 3 test-keyval /keyval/visit/bool
---
PASS 4 test-write-threshold /write-threshold/not-trigger
PASS 5 test-write-threshold /write-threshold/trigger
MALLOC_PERTURB_=${MALLOC_PERTURB_:-$(( ${RANDOM:-0} % 255 + 1))}  tests/test-crypto-hash -m=quick -k --tap < /dev/null | ./scripts/tap-driver.pl --test-name="test-crypto-hash" 
==6949==WARNING: ASan doesn't fully support makecontext/swapcontext functions and may produce false positives in some cases!
PASS 1 test-crypto-hash /crypto/hash/iov
PASS 2 test-crypto-hash /crypto/hash/alloc
PASS 3 test-crypto-hash /crypto/hash/prealloc
---
PASS 15 test-crypto-secret /crypto/secret/crypt/missingiv
PASS 16 test-crypto-secret /crypto/secret/crypt/badiv
MALLOC_PERTURB_=${MALLOC_PERTURB_:-$(( ${RANDOM:-0} % 255 + 1))}  tests/test-crypto-tlscredsx509 -m=quick -k --tap < /dev/null | ./scripts/tap-driver.pl --test-name="test-crypto-tlscredsx509" 
==6964==WARNING: ASan doesn't fully support makecontext/swapcontext functions and may produce false positives in some cases!
PASS 5 ahci-test /x86_64/ahci/hba_enable
==6982==WARNING: ASan doesn't fully support makecontext/swapcontext functions and may produce false positives in some cases!
PASS 1 test-crypto-tlscredsx509 /qcrypto/tlscredsx509/perfectserver
PASS 2 test-crypto-tlscredsx509 /qcrypto/tlscredsx509/perfectclient
PASS 6 ahci-test /x86_64/ahci/identify
==6988==WARNING: ASan doesn't fully support makecontext/swapcontext functions and may produce false positives in some cases!
PASS 3 test-crypto-tlscredsx509 /qcrypto/tlscredsx509/goodca1
PASS 7 ahci-test /x86_64/ahci/max
PASS 4 test-crypto-tlscredsx509 /qcrypto/tlscredsx509/goodca2
==6994==WARNING: ASan doesn't fully support makecontext/swapcontext functions and may produce false positives in some cases!
PASS 8 ahci-test /x86_64/ahci/reset
PASS 5 test-crypto-tlscredsx509 /qcrypto/tlscredsx509/goodca3
PASS 6 test-crypto-tlscredsx509 /qcrypto/tlscredsx509/badca1
PASS 7 test-crypto-tlscredsx509 /qcrypto/tlscredsx509/badca2
PASS 8 test-crypto-tlscredsx509 /qcrypto/tlscredsx509/badca3
==7000==WARNING: ASan doesn't fully support makecontext/swapcontext functions and may produce false positives in some cases!
==7000==WARNING: ASan is ignoring requested __asan_handle_no_return: stack top: 0x7ffd76fe2000; bottom 0x7f548edfe000; size: 0x00a8e81e4000 (725448802304)
False positive error reports may follow
For details see https://github.com/google/sanitizers/issues/189
PASS 9 ahci-test /x86_64/ahci/io/pio/lba28/simple/zero
PASS 9 test-crypto-tlscredsx509 /qcrypto/tlscredsx509/goodserver1
==7006==WARNING: ASan doesn't fully support makecontext/swapcontext functions and may produce false positives in some cases!
PASS 10 test-crypto-tlscredsx509 /qcrypto/tlscredsx509/goodserver2
==7006==WARNING: ASan is ignoring requested __asan_handle_no_return: stack top: 0x7ffea6ecf000; bottom 0x7f6d285fe000; size: 0x00917e8d1000 (624893431808)
False positive error reports may follow
For details see https://github.com/google/sanitizers/issues/189
PASS 10 ahci-test /x86_64/ahci/io/pio/lba28/simple/low
==7012==WARNING: ASan doesn't fully support makecontext/swapcontext functions and may produce false positives in some cases!
==7012==WARNING: ASan is ignoring requested __asan_handle_no_return: stack top: 0x7ffc06174000; bottom 0x7fdd933fe000; size: 0x001e72d76000 (130775736320)
False positive error reports may follow
For details see https://github.com/google/sanitizers/issues/189
PASS 11 test-crypto-tlscredsx509 /qcrypto/tlscredsx509/goodserver3
PASS 11 ahci-test /x86_64/ahci/io/pio/lba28/simple/high
==7018==WARNING: ASan doesn't fully support makecontext/swapcontext functions and may produce false positives in some cases!
==7018==WARNING: ASan is ignoring requested __asan_handle_no_return: stack top: 0x7fff25f61000; bottom 0x7f70eadfe000; size: 0x008e3b163000 (610876665856)
False positive error reports may follow
For details see https://github.com/google/sanitizers/issues/189
PASS 12 test-crypto-tlscredsx509 /qcrypto/tlscredsx509/goodserver4
PASS 12 ahci-test /x86_64/ahci/io/pio/lba28/double/zero
==7024==WARNING: ASan doesn't fully support makecontext/swapcontext functions and may produce false positives in some cases!
==7024==WARNING: ASan is ignoring requested __asan_handle_no_return: stack top: 0x7fff57e6b000; bottom 0x7f94c1dfe000; size: 0x006a9606d000 (457783562240)
False positive error reports may follow
For details see https://github.com/google/sanitizers/issues/189
PASS 13 test-crypto-tlscredsx509 /qcrypto/tlscredsx509/goodserver5
PASS 13 ahci-test /x86_64/ahci/io/pio/lba28/double/low
==7030==WARNING: ASan doesn't fully support makecontext/swapcontext functions and may produce false positives in some cases!
==7030==WARNING: ASan is ignoring requested __asan_handle_no_return: stack top: 0x7fff59f94000; bottom 0x7f9c875fe000; size: 0x0062d2996000 (424440061952)
False positive error reports may follow
For details see https://github.com/google/sanitizers/issues/189
PASS 14 ahci-test /x86_64/ahci/io/pio/lba28/double/high
==7036==WARNING: ASan doesn't fully support makecontext/swapcontext functions and may produce false positives in some cases!
PASS 14 test-crypto-tlscredsx509 /qcrypto/tlscredsx509/goodserver6
==7036==WARNING: ASan is ignoring requested __asan_handle_no_return: stack top: 0x7fff040bf000; bottom 0x7f4ded9fe000; size: 0x00b1166c1000 (760585392128)
False positive error reports may follow
For details see https://github.com/google/sanitizers/issues/189
PASS 15 test-crypto-tlscredsx509 /qcrypto/tlscredsx509/goodserver7
---
PASS 33 test-crypto-tlscredsx509 /qcrypto/tlscredsx509/inactive2
PASS 34 test-crypto-tlscredsx509 /qcrypto/tlscredsx509/inactive3
PASS 15 ahci-test /x86_64/ahci/io/pio/lba28/long/zero
==7042==WARNING: ASan doesn't fully support makecontext/swapcontext functions and may produce false positives in some cases!
==7042==WARNING: ASan is ignoring requested __asan_handle_no_return: stack top: 0x7fff6e46f000; bottom 0x7fb6e35fe000; size: 0x00488ae71000 (311568044032)
False positive error reports may follow
For details see https://github.com/google/sanitizers/issues/189
PASS 16 ahci-test /x86_64/ahci/io/pio/lba28/long/low
---
PASS 38 test-crypto-tlscredsx509 /qcrypto/tlscredsx509/missingserver
PASS 39 test-crypto-tlscredsx509 /qcrypto/tlscredsx509/missingclient
MALLOC_PERTURB_=${MALLOC_PERTURB_:-$(( ${RANDOM:-0} % 255 + 1))}  tests/test-crypto-tlssession -m=quick -k --tap < /dev/null | ./scripts/tap-driver.pl --test-name="test-crypto-tlssession" 
==7048==WARNING: ASan doesn't fully support makecontext/swapcontext functions and may produce false positives in some cases!
==7048==WARNING: ASan is ignoring requested __asan_handle_no_return: stack top: 0x7ffe1734b000; bottom 0x7f483e9fe000; size: 0x00b5d894d000 (781022711808)
False positive error reports may follow
For details see https://github.com/google/sanitizers/issues/189
PASS 17 ahci-test /x86_64/ahci/io/pio/lba28/long/high
PASS 1 test-crypto-tlssession /qcrypto/tlssession/psk
PASS 2 test-crypto-tlssession /qcrypto/tlssession/basicca
==7058==WARNING: ASan doesn't fully support makecontext/swapcontext functions and may produce false positives in some cases!
PASS 18 ahci-test /x86_64/ahci/io/pio/lba28/short/zero
==7064==WARNING: ASan doesn't fully support makecontext/swapcontext functions and may produce false positives in some cases!
PASS 3 test-crypto-tlssession /qcrypto/tlssession/differentca
PASS 19 ahci-test /x86_64/ahci/io/pio/lba28/short/low
==7070==WARNING: ASan doesn't fully support makecontext/swapcontext functions and may produce false positives in some cases!
PASS 4 test-crypto-tlssession /qcrypto/tlssession/altname1
PASS 20 ahci-test /x86_64/ahci/io/pio/lba28/short/high
PASS 5 test-crypto-tlssession /qcrypto/tlssession/altname2
PASS 6 test-crypto-tlssession /qcrypto/tlssession/altname3
==7076==WARNING: ASan doesn't fully support makecontext/swapcontext functions and may produce false positives in some cases!
==7076==WARNING: ASan is ignoring requested __asan_handle_no_return: stack top: 0x7ffe06d0f000; bottom 0x7fd4239fe000; size: 0x0029e3311000 (179905302528)
False positive error reports may follow
For details see https://github.com/google/sanitizers/issues/189
PASS 21 ahci-test /x86_64/ahci/io/pio/lba48/simple/zero
PASS 7 test-crypto-tlssession /qcrypto/tlssession/altname4
==7082==WARNING: ASan doesn't fully support makecontext/swapcontext functions and may produce false positives in some cases!
==7082==WARNING: ASan is ignoring requested __asan_handle_no_return: stack top: 0x7ffddcd98000; bottom 0x7fe45c9fe000; size: 0x00198039a000 (109525442560)
False positive error reports may follow
For details see https://github.com/google/sanitizers/issues/189
PASS 8 test-crypto-tlssession /qcrypto/tlssession/altname5
PASS 22 ahci-test /x86_64/ahci/io/pio/lba48/simple/low
==7088==WARNING: ASan doesn't fully support makecontext/swapcontext functions and may produce false positives in some cases!
==7088==WARNING: ASan is ignoring requested __asan_handle_no_return: stack top: 0x7ffd965c3000; bottom 0x7fe0cbdfe000; size: 0x001cca7c5000 (123656228864)
False positive error reports may follow
For details see https://github.com/google/sanitizers/issues/189
PASS 23 ahci-test /x86_64/ahci/io/pio/lba48/simple/high
PASS 9 test-crypto-tlssession /qcrypto/tlssession/altname6
==7094==WARNING: ASan doesn't fully support makecontext/swapcontext functions and may produce false positives in some cases!
PASS 10 test-crypto-tlssession /qcrypto/tlssession/wildcard1
==7094==WARNING: ASan is ignoring requested __asan_handle_no_return: stack top: 0x7ffcb1a2c000; bottom 0x7f4c711fe000; size: 0x00b04082e000 (756996562944)
False positive error reports may follow
For details see https://github.com/google/sanitizers/issues/189
PASS 24 ahci-test /x86_64/ahci/io/pio/lba48/double/zero
==7100==WARNING: ASan doesn't fully support makecontext/swapcontext functions and may produce false positives in some cases!
PASS 11 test-crypto-tlssession /qcrypto/tlssession/wildcard2
==7100==WARNING: ASan is ignoring requested __asan_handle_no_return: stack top: 0x7ffc15737000; bottom 0x7fc4ec1fe000; size: 0x003729539000 (236916543488)
False positive error reports may follow
For details see https://github.com/google/sanitizers/issues/189
PASS 25 ahci-test /x86_64/ahci/io/pio/lba48/double/low
PASS 12 test-crypto-tlssession /qcrypto/tlssession/wildcard3
PASS 13 test-crypto-tlssession /qcrypto/tlssession/wildcard4
==7106==WARNING: ASan doesn't fully support makecontext/swapcontext functions and may produce false positives in some cases!
==7106==WARNING: ASan is ignoring requested __asan_handle_no_return: stack top: 0x7ffcbccb9000; bottom 0x7f68911fe000; size: 0x00942babb000 (636387831808)
False positive error reports may follow
For details see https://github.com/google/sanitizers/issues/189
PASS 14 test-crypto-tlssession /qcrypto/tlssession/wildcard5
PASS 26 ahci-test /x86_64/ahci/io/pio/lba48/double/high
PASS 15 test-crypto-tlssession /qcrypto/tlssession/wildcard6
==7112==WARNING: ASan doesn't fully support makecontext/swapcontext functions and may produce false positives in some cases!
==7112==WARNING: ASan is ignoring requested __asan_handle_no_return: stack top: 0x7ffc766a8000; bottom 0x7f6ce41fe000; size: 0x008f924aa000 (616634687488)
False positive error reports may follow
For details see https://github.com/google/sanitizers/issues/189
PASS 27 ahci-test /x86_64/ahci/io/pio/lba48/long/zero
PASS 16 test-crypto-tlssession /qcrypto/tlssession/cachain
MALLOC_PERTURB_=${MALLOC_PERTURB_:-$(( ${RANDOM:-0} % 255 + 1))}  tests/test-qga -m=quick -k --tap < /dev/null | ./scripts/tap-driver.pl --test-name="test-qga" 
==7118==WARNING: ASan doesn't fully support makecontext/swapcontext functions and may produce false positives in some cases!
==7118==WARNING: ASan is ignoring requested __asan_handle_no_return: stack top: 0x7fffeec3d000; bottom 0x7f9ca25fe000; size: 0x00634c63f000 (426483380224)
False positive error reports may follow
For details see https://github.com/google/sanitizers/issues/189
PASS 28 ahci-test /x86_64/ahci/io/pio/lba48/long/low
---
PASS 15 test-qga /qga/invalid-cmd
PASS 16 test-qga /qga/invalid-args
PASS 17 test-qga /qga/fsfreeze-status
==7132==WARNING: ASan doesn't fully support makecontext/swapcontext functions and may produce false positives in some cases!
==7132==WARNING: ASan is ignoring requested __asan_handle_no_return: stack top: 0x7ffd625e7000; bottom 0x7f27eab7c000; size: 0x00d577a6b000 (916835446784)
False positive error reports may follow
For details see https://github.com/google/sanitizers/issues/189
PASS 29 ahci-test /x86_64/ahci/io/pio/lba48/long/high
---
PASS 19 test-qga /qga/config
PASS 20 test-qga /qga/guest-exec
PASS 21 test-qga /qga/guest-exec-invalid
==7141==WARNING: ASan doesn't fully support makecontext/swapcontext functions and may produce false positives in some cases!
PASS 30 ahci-test /x86_64/ahci/io/pio/lba48/short/zero
PASS 22 test-qga /qga/guest-get-osinfo
PASS 23 test-qga /qga/guest-get-host-name
PASS 24 test-qga /qga/guest-get-timezone
PASS 25 test-qga /qga/guest-get-users
MALLOC_PERTURB_=${MALLOC_PERTURB_:-$(( ${RANDOM:-0} % 255 + 1))}  tests/test-timed-average -m=quick -k --tap < /dev/null | ./scripts/tap-driver.pl --test-name="test-timed-average" 
==7159==WARNING: ASan doesn't fully support makecontext/swapcontext functions and may produce false positives in some cases!
PASS 1 test-timed-average /timed-average/average
MALLOC_PERTURB_=${MALLOC_PERTURB_:-$(( ${RANDOM:-0} % 255 + 1))}  tests/test-util-filemonitor -m=quick -k --tap < /dev/null | ./scripts/tap-driver.pl --test-name="test-util-filemonitor" 
PASS 1 test-util-filemonitor /util/filemonitor
---
PASS 8 test-util-sockets /socket/fd-pass/num/nocli
MALLOC_PERTURB_=${MALLOC_PERTURB_:-$(( ${RANDOM:-0} % 255 + 1))}  tests/test-authz-simple -m=quick -k --tap < /dev/null | ./scripts/tap-driver.pl --test-name="test-authz-simple" 
PASS 1 test-authz-simple /authz/simple
==7178==WARNING: ASan doesn't fully support makecontext/swapcontext functions and may produce false positives in some cases!
MALLOC_PERTURB_=${MALLOC_PERTURB_:-$(( ${RANDOM:-0} % 255 + 1))}  tests/test-authz-list -m=quick -k --tap < /dev/null | ./scripts/tap-driver.pl --test-name="test-authz-list" 
PASS 1 test-authz-list /auth/list/complex
PASS 2 test-authz-list /auth/list/add-remove
---
PASS 3 test-io-task /crypto/task/failure
PASS 4 test-io-task /crypto/task/thread_complete
PASS 5 test-io-task /crypto/task/thread_failure
==7197==WARNING: ASan doesn't fully support makecontext/swapcontext functions and may produce false positives in some cases!
MALLOC_PERTURB_=${MALLOC_PERTURB_:-$(( ${RANDOM:-0} % 255 + 1))}  tests/test-io-channel-socket -m=quick -k --tap < /dev/null | ./scripts/tap-driver.pl --test-name="test-io-channel-socket" 
PASS 1 test-io-channel-socket /io/channel/socket/ipv4-sync
PASS 2 test-io-channel-socket /io/channel/socket/ipv4-async
---
PASS 4 test-io-channel-file /io/channel/pipe/sync
PASS 5 test-io-channel-file /io/channel/pipe/async
MALLOC_PERTURB_=${MALLOC_PERTURB_:-$(( ${RANDOM:-0} % 255 + 1))}  tests/test-io-channel-tls -m=quick -k --tap < /dev/null | ./scripts/tap-driver.pl --test-name="test-io-channel-tls" 
==7257==WARNING: ASan doesn't fully support makecontext/swapcontext functions and may produce false positives in some cases!
PASS 1 test-io-channel-tls /qio/channel/tls/basic
MALLOC_PERTURB_=${MALLOC_PERTURB_:-$(( ${RANDOM:-0} % 255 + 1))}  tests/test-io-channel-command -m=quick -k --tap < /dev/null | ./scripts/tap-driver.pl --test-name="test-io-channel-command" 
PASS 34 ahci-test /x86_64/ahci/io/dma/lba28/retry
---
MALLOC_PERTURB_=${MALLOC_PERTURB_:-$(( ${RANDOM:-0} % 255 + 1))}  tests/test-io-channel-buffer -m=quick -k --tap < /dev/null | ./scripts/tap-driver.pl --test-name="test-io-channel-buffer" 
PASS 1 test-io-channel-buffer /io/channel/buf
MALLOC_PERTURB_=${MALLOC_PERTURB_:-$(( ${RANDOM:-0} % 255 + 1))}  tests/test-base64 -m=quick -k --tap < /dev/null | ./scripts/tap-driver.pl --test-name="test-base64" 
==7274==WARNING: ASan doesn't fully support makecontext/swapcontext functions and may produce false positives in some cases!
PASS 1 test-base64 /util/base64/good
PASS 2 test-base64 /util/base64/embedded-nul
PASS 3 test-base64 /util/base64/not-nul-terminated
---
PASS 3 test-crypto-afsplit /crypto/afsplit/sha256/big
PASS 4 test-crypto-afsplit /crypto/afsplit/sha1/1000
MALLOC_PERTURB_=${MALLOC_PERTURB_:-$(( ${RANDOM:-0} % 255 + 1))}  tests/test-crypto-xts -m=quick -k --tap < /dev/null | ./scripts/tap-driver.pl --test-name="test-crypto-xts" 
==7292==WARNING: ASan doesn't fully support makecontext/swapcontext functions and may produce false positives in some cases!
PASS 1 test-crypto-xts /crypto/xts/t-1-key-32-ptx-32/basic
PASS 2 test-crypto-xts /crypto/xts/t-1-key-32-ptx-32/split
PASS 3 test-crypto-xts /crypto/xts/t-1-key-32-ptx-32/unaligned
---
PASS 3 test-logging /logging/logfile_write_path
PASS 4 test-logging /logging/logfile_lock_path
MALLOC_PERTURB_=${MALLOC_PERTURB_:-$(( ${RANDOM:-0} % 255 + 1))}  tests/test-replication -m=quick -k --tap < /dev/null | ./scripts/tap-driver.pl --test-name="test-replication" 
==7314==WARNING: ASan doesn't fully support makecontext/swapcontext functions and may produce false positives in some cases!
==7321==WARNING: ASan doesn't fully support makecontext/swapcontext functions and may produce false positives in some cases!
PASS 37 ahci-test /x86_64/ahci/io/dma/lba28/simple/high
PASS 1 test-replication /replication/primary/read
PASS 2 test-replication /replication/primary/write
==7329==WARNING: ASan doesn't fully support makecontext/swapcontext functions and may produce false positives in some cases!
PASS 38 ahci-test /x86_64/ahci/io/dma/lba28/double/zero
PASS 3 test-replication /replication/primary/start
PASS 4 test-replication /replication/primary/stop
==7335==WARNING: ASan doesn't fully support makecontext/swapcontext functions and may produce false positives in some cases!
PASS 5 test-replication /replication/primary/do_checkpoint
PASS 6 test-replication /replication/primary/get_error_all
PASS 39 ahci-test /x86_64/ahci/io/dma/lba28/double/low
==7341==WARNING: ASan doesn't fully support makecontext/swapcontext functions and may produce false positives in some cases!
PASS 7 test-replication /replication/secondary/read
PASS 40 ahci-test /x86_64/ahci/io/dma/lba28/double/high
==7347==WARNING: ASan doesn't fully support makecontext/swapcontext functions and may produce false positives in some cases!
PASS 8 test-replication /replication/secondary/write
==7347==WARNING: ASan is ignoring requested __asan_handle_no_return: stack top: 0x7ffc9eea6000; bottom 0x7fc9ff3fd000; size: 0x00329faa9000 (217427120128)
False positive error reports may follow
For details see https://github.com/google/sanitizers/issues/189
PASS 41 ahci-test /x86_64/ahci/io/dma/lba28/long/zero
==7354==WARNING: ASan doesn't fully support makecontext/swapcontext functions and may produce false positives in some cases!
==7354==WARNING: ASan is ignoring requested __asan_handle_no_return: stack top: 0x7ffef2f64000; bottom 0x7fc0bc97b000; size: 0x003e365e9000 (267200139264)
False positive error reports may follow
For details see https://github.com/google/sanitizers/issues/189
PASS 42 ahci-test /x86_64/ahci/io/dma/lba28/long/low
==7361==WARNING: ASan doesn't fully support makecontext/swapcontext functions and may produce false positives in some cases!
==7361==WARNING: ASan is ignoring requested __asan_handle_no_return: stack top: 0x7ffe1ae51000; bottom 0x7fe846ffd000; size: 0x0015d3e54000 (93749329920)
False positive error reports may follow
For details see https://github.com/google/sanitizers/issues/189
==7321==WARNING: ASan is ignoring requested __asan_handle_no_return: stack top: 0x7ffe47bd5000; bottom 0x7f854543c000; size: 0x007902799000 (519732563968)
False positive error reports may follow
For details see https://github.com/google/sanitizers/issues/189
PASS 43 ahci-test /x86_64/ahci/io/dma/lba28/long/high
PASS 9 test-replication /replication/secondary/start
==7390==WARNING: ASan doesn't fully support makecontext/swapcontext functions and may produce false positives in some cases!
PASS 44 ahci-test /x86_64/ahci/io/dma/lba28/short/zero
==7396==WARNING: ASan doesn't fully support makecontext/swapcontext functions and may produce false positives in some cases!
PASS 45 ahci-test /x86_64/ahci/io/dma/lba28/short/low
==7402==WARNING: ASan doesn't fully support makecontext/swapcontext functions and may produce false positives in some cases!
PASS 46 ahci-test /x86_64/ahci/io/dma/lba28/short/high
==7408==WARNING: ASan doesn't fully support makecontext/swapcontext functions and may produce false positives in some cases!
PASS 10 test-replication /replication/secondary/stop
PASS 47 ahci-test /x86_64/ahci/io/dma/lba48/simple/zero
==7414==WARNING: ASan doesn't fully support makecontext/swapcontext functions and may produce false positives in some cases!
PASS 48 ahci-test /x86_64/ahci/io/dma/lba48/simple/low
==7420==WARNING: ASan doesn't fully support makecontext/swapcontext functions and may produce false positives in some cases!
PASS 49 ahci-test /x86_64/ahci/io/dma/lba48/simple/high
==7426==WARNING: ASan doesn't fully support makecontext/swapcontext functions and may produce false positives in some cases!
PASS 50 ahci-test /x86_64/ahci/io/dma/lba48/double/zero
PASS 11 test-replication /replication/secondary/continuous_replication
==7432==WARNING: ASan doesn't fully support makecontext/swapcontext functions and may produce false positives in some cases!
PASS 51 ahci-test /x86_64/ahci/io/dma/lba48/double/low
==7438==WARNING: ASan doesn't fully support makecontext/swapcontext functions and may produce false positives in some cases!
PASS 52 ahci-test /x86_64/ahci/io/dma/lba48/double/high
==7444==WARNING: ASan doesn't fully support makecontext/swapcontext functions and may produce false positives in some cases!
==7444==WARNING: ASan is ignoring requested __asan_handle_no_return: stack top: 0x7ffc76516000; bottom 0x7f8443ffd000; size: 0x007832519000 (516240281600)
False positive error reports may follow
For details see https://github.com/google/sanitizers/issues/189
PASS 12 test-replication /replication/secondary/do_checkpoint
PASS 53 ahci-test /x86_64/ahci/io/dma/lba48/long/zero
==7451==WARNING: ASan doesn't fully support makecontext/swapcontext functions and may produce false positives in some cases!
==7451==WARNING: ASan is ignoring requested __asan_handle_no_return: stack top: 0x7fff0d748000; bottom 0x7f1e5ebfd000; size: 0x00e0aeb4b000 (965003751424)
False positive error reports may follow
For details see https://github.com/google/sanitizers/issues/189
PASS 13 test-replication /replication/secondary/get_error_all
PASS 54 ahci-test /x86_64/ahci/io/dma/lba48/long/low
MALLOC_PERTURB_=${MALLOC_PERTURB_:-$(( ${RANDOM:-0} % 255 + 1))}  tests/test-bufferiszero -m=quick -k --tap < /dev/null | ./scripts/tap-driver.pl --test-name="test-bufferiszero" 
==7458==WARNING: ASan doesn't fully support makecontext/swapcontext functions and may produce false positives in some cases!
==7458==WARNING: ASan is ignoring requested __asan_handle_no_return: stack top: 0x7ffe0f28e000; bottom 0x7f62bbd7b000; size: 0x009b53513000 (667117760512)
False positive error reports may follow
For details see https://github.com/google/sanitizers/issues/189
PASS 55 ahci-test /x86_64/ahci/io/dma/lba48/long/high
==7468==WARNING: ASan doesn't fully support makecontext/swapcontext functions and may produce false positives in some cases!
PASS 56 ahci-test /x86_64/ahci/io/dma/lba48/short/zero
==7474==WARNING: ASan doesn't fully support makecontext/swapcontext functions and may produce false positives in some cases!
PASS 57 ahci-test /x86_64/ahci/io/dma/lba48/short/low
==7480==WARNING: ASan doesn't fully support makecontext/swapcontext functions and may produce false positives in some cases!
PASS 58 ahci-test /x86_64/ahci/io/dma/lba48/short/high
==7486==WARNING: ASan doesn't fully support makecontext/swapcontext functions and may produce false positives in some cases!
PASS 59 ahci-test /x86_64/ahci/io/ncq/simple
==7492==WARNING: ASan doesn't fully support makecontext/swapcontext functions and may produce false positives in some cases!
PASS 60 ahci-test /x86_64/ahci/io/ncq/retry
==7498==WARNING: ASan doesn't fully support makecontext/swapcontext functions and may produce false positives in some cases!
PASS 61 ahci-test /x86_64/ahci/flush/simple
==7504==WARNING: ASan doesn't fully support makecontext/swapcontext functions and may produce false positives in some cases!
PASS 62 ahci-test /x86_64/ahci/flush/retry
==7510==WARNING: ASan doesn't fully support makecontext/swapcontext functions and may produce false positives in some cases!
==7516==WARNING: ASan doesn't fully support makecontext/swapcontext functions and may produce false positives in some cases!
PASS 63 ahci-test /x86_64/ahci/flush/migrate
==7524==WARNING: ASan doesn't fully support makecontext/swapcontext functions and may produce false positives in some cases!
==7530==WARNING: ASan doesn't fully support makecontext/swapcontext functions and may produce false positives in some cases!
PASS 64 ahci-test /x86_64/ahci/migrate/sanity
==7538==WARNING: ASan doesn't fully support makecontext/swapcontext functions and may produce false positives in some cases!
==7544==WARNING: ASan doesn't fully support makecontext/swapcontext functions and may produce false positives in some cases!
PASS 65 ahci-test /x86_64/ahci/migrate/dma/simple
==7552==WARNING: ASan doesn't fully support makecontext/swapcontext functions and may produce false positives in some cases!
==7558==WARNING: ASan doesn't fully support makecontext/swapcontext functions and may produce false positives in some cases!
PASS 66 ahci-test /x86_64/ahci/migrate/dma/halted
==7566==WARNING: ASan doesn't fully support makecontext/swapcontext functions and may produce false positives in some cases!
==7572==WARNING: ASan doesn't fully support makecontext/swapcontext functions and may produce false positives in some cases!
PASS 67 ahci-test /x86_64/ahci/migrate/ncq/simple
==7580==WARNING: ASan doesn't fully support makecontext/swapcontext functions and may produce false positives in some cases!
==7586==WARNING: ASan doesn't fully support makecontext/swapcontext functions and may produce false positives in some cases!
PASS 68 ahci-test /x86_64/ahci/migrate/ncq/halted
==7594==WARNING: ASan doesn't fully support makecontext/swapcontext functions and may produce false positives in some cases!
PASS 69 ahci-test /x86_64/ahci/cdrom/eject
==7599==WARNING: ASan doesn't fully support makecontext/swapcontext functions and may produce false positives in some cases!
PASS 70 ahci-test /x86_64/ahci/cdrom/dma/single
==7605==WARNING: ASan doesn't fully support makecontext/swapcontext functions and may produce false positives in some cases!
PASS 71 ahci-test /x86_64/ahci/cdrom/dma/multi
==7611==WARNING: ASan doesn't fully support makecontext/swapcontext functions and may produce false positives in some cases!
PASS 72 ahci-test /x86_64/ahci/cdrom/pio/single
==7617==WARNING: ASan doesn't fully support makecontext/swapcontext functions and may produce false positives in some cases!
==7617==WARNING: ASan is ignoring requested __asan_handle_no_return: stack top: 0x7ffedd775000; bottom 0x7fee367fe000; size: 0x0010a6f77000 (71520710656)
False positive error reports may follow
For details see https://github.com/google/sanitizers/issues/189
PASS 73 ahci-test /x86_64/ahci/cdrom/pio/multi
==7623==WARNING: ASan doesn't fully support makecontext/swapcontext functions and may produce false positives in some cases!
PASS 74 ahci-test /x86_64/ahci/cdrom/pio/bcl
MALLOC_PERTURB_=${MALLOC_PERTURB_:-$(( ${RANDOM:-0} % 255 + 1))}  QTEST_QEMU_BINARY=x86_64-softmmu/qemu-system-x86_64 QTEST_QEMU_IMG=qemu-img tests/qtest/hd-geo-test -m=quick -k --tap < /dev/null | ./scripts/tap-driver.pl --test-name="hd-geo-test" 
PASS 1 hd-geo-test /x86_64/hd-geo/ide/none
---
PASS 527 ptimer-test /ptimer/periodic_with_load_0 policy=wrap_after_one_period,continuous_trigger,no_immediate_reload,no_counter_rounddown,trigger_only_on_decrement,
PASS 528 ptimer-test /ptimer/oneshot_with_load_0 policy=wrap_after_one_period,continuous_trigger,no_immediate_reload,no_counter_rounddown,trigger_only_on_decrement,
MALLOC_PERTURB_=${MALLOC_PERTURB_:-$(( ${RANDOM:-0} % 255 + 1))}  tests/test-qapi-util -m=quick -k --tap < /dev/null | ./scripts/tap-driver.pl --test-name="test-qapi-util" 
==7637==WARNING: ASan doesn't fully support makecontext/swapcontext functions and may produce false positives in some cases!
PASS 1 test-qapi-util /qapi/util/qapi_enum_parse
PASS 2 test-qapi-util /qapi/util/parse_qapi_name
MALLOC_PERTURB_=${MALLOC_PERTURB_:-$(( ${RANDOM:-0} % 255 + 1))}  tests/test-qgraph -m=quick -k --tap < /dev/null | ./scripts/tap-driver.pl --test-name="test-qgraph" 
---
PASS 21 test-qgraph /qgraph/test_two_test_same_interface
PASS 22 test-qgraph /qgraph/test_test_in_path
PASS 23 test-qgraph /qgraph/test_double_edge
==7656==WARNING: ASan doesn't fully support makecontext/swapcontext functions and may produce false positives in some cases!
PASS 3 hd-geo-test /x86_64/hd-geo/ide/drive/mbr/blank
==7662==WARNING: ASan doesn't fully support makecontext/swapcontext functions and may produce false positives in some cases!
PASS 4 hd-geo-test /x86_64/hd-geo/ide/drive/mbr/lba
==7668==WARNING: ASan doesn't fully support makecontext/swapcontext functions and may produce false positives in some cases!
PASS 5 hd-geo-test /x86_64/hd-geo/ide/drive/mbr/chs
==7674==WARNING: ASan doesn't fully support makecontext/swapcontext functions and may produce false positives in some cases!
PASS 6 hd-geo-test /x86_64/hd-geo/ide/device/mbr/blank
==7680==WARNING: ASan doesn't fully support makecontext/swapcontext functions and may produce false positives in some cases!
PASS 7 hd-geo-test /x86_64/hd-geo/ide/device/mbr/lba
==7686==WARNING: ASan doesn't fully support makecontext/swapcontext functions and may produce false positives in some cases!
PASS 8 hd-geo-test /x86_64/hd-geo/ide/device/mbr/chs
==7692==WARNING: ASan doesn't fully support makecontext/swapcontext functions and may produce false positives in some cases!
PASS 9 hd-geo-test /x86_64/hd-geo/ide/device/user/chs
==7697==WARNING: ASan doesn't fully support makecontext/swapcontext functions and may produce false positives in some cases!
PASS 10 hd-geo-test /x86_64/hd-geo/ide/device/user/chst
==7703==WARNING: ASan doesn't fully support makecontext/swapcontext functions and may produce false positives in some cases!
==7707==WARNING: ASan doesn't fully support makecontext/swapcontext functions and may produce false positives in some cases!
==7711==WARNING: ASan doesn't fully support makecontext/swapcontext functions and may produce false positives in some cases!
==7715==WARNING: ASan doesn't fully support makecontext/swapcontext functions and may produce false positives in some cases!
==7719==WARNING: ASan doesn't fully support makecontext/swapcontext functions and may produce false positives in some cases!
==7723==WARNING: ASan doesn't fully support makecontext/swapcontext functions and may produce false positives in some cases!
==7727==WARNING: ASan doesn't fully support makecontext/swapcontext functions and may produce false positives in some cases!
==7731==WARNING: ASan doesn't fully support makecontext/swapcontext functions and may produce false positives in some cases!
==7734==WARNING: ASan doesn't fully support makecontext/swapcontext functions and may produce false positives in some cases!
PASS 11 hd-geo-test /x86_64/hd-geo/override/ide
==7741==WARNING: ASan doesn't fully support makecontext/swapcontext functions and may produce false positives in some cases!
==7745==WARNING: ASan doesn't fully support makecontext/swapcontext functions and may produce false positives in some cases!
==7749==WARNING: ASan doesn't fully support makecontext/swapcontext functions and may produce false positives in some cases!
==7753==WARNING: ASan doesn't fully support makecontext/swapcontext functions and may produce false positives in some cases!
==7757==WARNING: ASan doesn't fully support makecontext/swapcontext functions and may produce false positives in some cases!
==7761==WARNING: ASan doesn't fully support makecontext/swapcontext functions and may produce false positives in some cases!
==7765==WARNING: ASan doesn't fully support makecontext/swapcontext functions and may produce false positives in some cases!
==7769==WARNING: ASan doesn't fully support makecontext/swapcontext functions and may produce false positives in some cases!
==7772==WARNING: ASan doesn't fully support makecontext/swapcontext functions and may produce false positives in some cases!
PASS 12 hd-geo-test /x86_64/hd-geo/override/scsi
==7779==WARNING: ASan doesn't fully support makecontext/swapcontext functions and may produce false positives in some cases!
==7783==WARNING: ASan doesn't fully support makecontext/swapcontext functions and may produce false positives in some cases!
==7787==WARNING: ASan doesn't fully support makecontext/swapcontext functions and may produce false positives in some cases!
==7791==WARNING: ASan doesn't fully support makecontext/swapcontext functions and may produce false positives in some cases!
==7795==WARNING: ASan doesn't fully support makecontext/swapcontext functions and may produce false positives in some cases!
==7799==WARNING: ASan doesn't fully support makecontext/swapcontext functions and may produce false positives in some cases!
==7803==WARNING: ASan doesn't fully support makecontext/swapcontext functions and may produce false positives in some cases!
==7807==WARNING: ASan doesn't fully support makecontext/swapcontext functions and may produce false positives in some cases!
==7810==WARNING: ASan doesn't fully support makecontext/swapcontext functions and may produce false positives in some cases!
PASS 13 hd-geo-test /x86_64/hd-geo/override/scsi_2_controllers
==7817==WARNING: ASan doesn't fully support makecontext/swapcontext functions and may produce false positives in some cases!
==7821==WARNING: ASan doesn't fully support makecontext/swapcontext functions and may produce false positives in some cases!
==7825==WARNING: ASan doesn't fully support makecontext/swapcontext functions and may produce false positives in some cases!
==7829==WARNING: ASan doesn't fully support makecontext/swapcontext functions and may produce false positives in some cases!
==7832==WARNING: ASan doesn't fully support makecontext/swapcontext functions and may produce false positives in some cases!
PASS 14 hd-geo-test /x86_64/hd-geo/override/virtio_blk
==7839==WARNING: ASan doesn't fully support makecontext/swapcontext functions and may produce false positives in some cases!
==7843==WARNING: ASan doesn't fully support makecontext/swapcontext functions and may produce false positives in some cases!
==7846==WARNING: ASan doesn't fully support makecontext/swapcontext functions and may produce false positives in some cases!
PASS 15 hd-geo-test /x86_64/hd-geo/override/zero_chs
==7853==WARNING: ASan doesn't fully support makecontext/swapcontext functions and may produce false positives in some cases!
==7857==WARNING: ASan doesn't fully support makecontext/swapcontext functions and may produce false positives in some cases!
==7861==WARNING: ASan doesn't fully support makecontext/swapcontext functions and may produce false positives in some cases!
==7865==WARNING: ASan doesn't fully support makecontext/swapcontext functions and may produce false positives in some cases!
==7868==WARNING: ASan doesn't fully support makecontext/swapcontext functions and may produce false positives in some cases!
PASS 16 hd-geo-test /x86_64/hd-geo/override/scsi_hot_unplug
==7875==WARNING: ASan doesn't fully support makecontext/swapcontext functions and may produce false positives in some cases!
==7879==WARNING: ASan doesn't fully support makecontext/swapcontext functions and may produce false positives in some cases!
==7883==WARNING: ASan doesn't fully support makecontext/swapcontext functions and may produce false positives in some cases!
==7887==WARNING: ASan doesn't fully support makecontext/swapcontext functions and may produce false positives in some cases!
==7890==WARNING: ASan doesn't fully support makecontext/swapcontext functions and may produce false positives in some cases!
PASS 17 hd-geo-test /x86_64/hd-geo/override/virtio_hot_unplug
MALLOC_PERTURB_=${MALLOC_PERTURB_:-$(( ${RANDOM:-0} % 255 + 1))}  QTEST_QEMU_BINARY=x86_64-softmmu/qemu-system-x86_64 QTEST_QEMU_IMG=qemu-img tests/qtest/boot-order-test -m=quick -k --tap < /dev/null | ./scripts/tap-driver.pl --test-name="boot-order-test" 
PASS 1 boot-order-test /x86_64/boot-order/pc
---
Could not access KVM kernel module: No such file or directory
qemu-system-x86_64: -accel kvm: failed to initialize kvm: No such file or directory
qemu-system-x86_64: falling back to tcg
==7959==WARNING: ASan doesn't fully support makecontext/swapcontext functions and may produce false positives in some cases!

Looking for expected file 'tests/data/acpi/pc/FACP'
Using expected file 'tests/data/acpi/pc/FACP'
---
Could not access KVM kernel module: No such file or directory
qemu-system-x86_64: -accel kvm: failed to initialize kvm: No such file or directory
qemu-system-x86_64: falling back to tcg
==7965==WARNING: ASan doesn't fully support makecontext/swapcontext functions and may produce false positives in some cases!

Looking for expected file 'tests/data/acpi/q35/FACP'
Using expected file 'tests/data/acpi/q35/FACP'
---
Could not access KVM kernel module: No such file or directory
qemu-system-x86_64: -accel kvm: failed to initialize kvm: No such file or directory
qemu-system-x86_64: falling back to tcg
==7971==WARNING: ASan doesn't fully support makecontext/swapcontext functions and may produce false positives in some cases!

Looking for expected file 'tests/data/acpi/pc/FACP.bridge'
Looking for expected file 'tests/data/acpi/pc/FACP'
---
Could not access KVM kernel module: No such file or directory
qemu-system-x86_64: -accel kvm: failed to initialize kvm: No such file or directory
qemu-system-x86_64: falling back to tcg
==7977==WARNING: ASan doesn't fully support makecontext/swapcontext functions and may produce false positives in some cases!

Looking for expected file 'tests/data/acpi/pc/FACP.ipmikcs'
Looking for expected file 'tests/data/acpi/pc/FACP'
---
Could not access KVM kernel module: No such file or directory
qemu-system-x86_64: -accel kvm: failed to initialize kvm: No such file or directory
qemu-system-x86_64: falling back to tcg
==7983==WARNING: ASan doesn't fully support makecontext/swapcontext functions and may produce false positives in some cases!

Looking for expected file 'tests/data/acpi/pc/FACP.cphp'
Looking for expected file 'tests/data/acpi/pc/FACP'
---
Could not access KVM kernel module: No such file or directory
qemu-system-x86_64: -accel kvm: failed to initialize kvm: No such file or directory
qemu-system-x86_64: falling back to tcg
==7990==WARNING: ASan doesn't fully support makecontext/swapcontext functions and may produce false positives in some cases!

Looking for expected file 'tests/data/acpi/pc/FACP.memhp'
Looking for expected file 'tests/data/acpi/pc/FACP'
---
Could not access KVM kernel module: No such file or directory
qemu-system-x86_64: -accel kvm: failed to initialize kvm: No such file or directory
qemu-system-x86_64: falling back to tcg
==7996==WARNING: ASan doesn't fully support makecontext/swapcontext functions and may produce false positives in some cases!

Looking for expected file 'tests/data/acpi/pc/FACP.numamem'
Looking for expected file 'tests/data/acpi/pc/FACP'
---
Could not access KVM kernel module: No such file or directory
qemu-system-x86_64: -accel kvm: failed to initialize kvm: No such file or directory
qemu-system-x86_64: falling back to tcg
==8002==WARNING: ASan doesn't fully support makecontext/swapcontext functions and may produce false positives in some cases!

Looking for expected file 'tests/data/acpi/pc/FACP.dimmpxm'
Looking for expected file 'tests/data/acpi/pc/FACP'
---
Could not access KVM kernel module: No such file or directory
qemu-system-x86_64: -accel kvm: failed to initialize kvm: No such file or directory
qemu-system-x86_64: falling back to tcg
==8011==WARNING: ASan doesn't fully support makecontext/swapcontext functions and may produce false positives in some cases!

Looking for expected file 'tests/data/acpi/pc/FACP.acpihmat'
Looking for expected file 'tests/data/acpi/pc/FACP'
---
Could not access KVM kernel module: No such file or directory
qemu-system-x86_64: -accel kvm: failed to initialize kvm: No such file or directory
qemu-system-x86_64: falling back to tcg
==8018==WARNING: ASan doesn't fully support makecontext/swapcontext functions and may produce false positives in some cases!

Looking for expected file 'tests/data/acpi/q35/FACP.bridge'
Looking for expected file 'tests/data/acpi/q35/FACP'
---
Could not access KVM kernel module: No such file or directory
qemu-system-x86_64: -accel kvm: failed to initialize kvm: No such file or directory
qemu-system-x86_64: falling back to tcg
==8024==WARNING: ASan doesn't fully support makecontext/swapcontext functions and may produce false positives in some cases!

Looking for expected file 'tests/data/acpi/q35/FACP.mmio64'
Looking for expected file 'tests/data/acpi/q35/FACP'
---
Could not access KVM kernel module: No such file or directory
qemu-system-x86_64: -accel kvm: failed to initialize kvm: No such file or directory
qemu-system-x86_64: falling back to tcg
==8030==WARNING: ASan doesn't fully support makecontext/swapcontext functions and may produce false positives in some cases!

Looking for expected file 'tests/data/acpi/q35/FACP.ipmibt'
Looking for expected file 'tests/data/acpi/q35/FACP'
---
Could not access KVM kernel module: No such file or directory
qemu-system-x86_64: -accel kvm: failed to initialize kvm: No such file or directory
qemu-system-x86_64: falling back to tcg
==8036==WARNING: ASan doesn't fully support makecontext/swapcontext functions and may produce false positives in some cases!

Looking for expected file 'tests/data/acpi/q35/FACP.cphp'
Looking for expected file 'tests/data/acpi/q35/FACP'
---
Could not access KVM kernel module: No such file or directory
qemu-system-x86_64: -accel kvm: failed to initialize kvm: No such file or directory
qemu-system-x86_64: falling back to tcg
==8043==WARNING: ASan doesn't fully support makecontext/swapcontext functions and may produce false positives in some cases!

Looking for expected file 'tests/data/acpi/q35/FACP.memhp'
Looking for expected file 'tests/data/acpi/q35/FACP'
---
Could not access KVM kernel module: No such file or directory
qemu-system-x86_64: -accel kvm: failed to initialize kvm: No such file or directory
qemu-system-x86_64: falling back to tcg
==8049==WARNING: ASan doesn't fully support makecontext/swapcontext functions and may produce false positives in some cases!

Looking for expected file 'tests/data/acpi/q35/FACP.numamem'
Looking for expected file 'tests/data/acpi/q35/FACP'
---
Could not access KVM kernel module: No such file or directory
qemu-system-x86_64: -accel kvm: failed to initialize kvm: No such file or directory
qemu-system-x86_64: falling back to tcg
==8055==WARNING: ASan doesn't fully support makecontext/swapcontext functions and may produce false positives in some cases!

Looking for expected file 'tests/data/acpi/q35/FACP.dimmpxm'
Looking for expected file 'tests/data/acpi/q35/FACP'
---
Could not access KVM kernel module: No such file or directory
qemu-system-x86_64: -accel kvm: failed to initialize kvm: No such file or directory
qemu-system-x86_64: falling back to tcg
==8064==WARNING: ASan doesn't fully support makecontext/swapcontext functions and may produce false positives in some cases!

Looking for expected file 'tests/data/acpi/q35/FACP.acpihmat'
Looking for expected file 'tests/data/acpi/q35/FACP'
---
PASS 1 i440fx-test /x86_64/i440fx/defaults
PASS 2 i440fx-test /x86_64/i440fx/pam
PASS 3 i440fx-test /x86_64/i440fx/firmware/bios
==8156==WARNING: ASan doesn't fully support makecontext/swapcontext functions and may produce false positives in some cases!
PASS 4 i440fx-test /x86_64/i440fx/firmware/pflash
MALLOC_PERTURB_=${MALLOC_PERTURB_:-$(( ${RANDOM:-0} % 255 + 1))}  QTEST_QEMU_BINARY=x86_64-softmmu/qemu-system-x86_64 QTEST_QEMU_IMG=qemu-img tests/qtest/fw_cfg-test -m=quick -k --tap < /dev/null | ./scripts/tap-driver.pl --test-name="fw_cfg-test" 
PASS 1 fw_cfg-test /x86_64/fw_cfg/signature
---
MALLOC_PERTURB_=${MALLOC_PERTURB_:-$(( ${RANDOM:-0} % 255 + 1))}  QTEST_QEMU_BINARY=x86_64-softmmu/qemu-system-x86_64 QTEST_QEMU_IMG=qemu-img tests/qtest/drive_del-test -m=quick -k --tap < /dev/null | ./scripts/tap-driver.pl --test-name="drive_del-test" 
PASS 1 drive_del-test /x86_64/drive_del/without-dev
PASS 2 drive_del-test /x86_64/drive_del/after_failed_device_add
==8249==WARNING: ASan doesn't fully support makecontext/swapcontext functions and may produce false positives in some cases!
PASS 3 drive_del-test /x86_64/blockdev/drive_del_device_del
MALLOC_PERTURB_=${MALLOC_PERTURB_:-$(( ${RANDOM:-0} % 255 + 1))}  QTEST_QEMU_BINARY=x86_64-softmmu/qemu-system-x86_64 QTEST_QEMU_IMG=qemu-img tests/qtest/wdt_ib700-test -m=quick -k --tap < /dev/null | ./scripts/tap-driver.pl --test-name="wdt_ib700-test" 
PASS 1 wdt_ib700-test /x86_64/wdt_ib700/pause
---
dbus-daemon[8419]: Could not get password database information for UID of current process: User "???" unknown or no memory to allocate password entry

**
ERROR:/tmp/qemu-test/src/tests/qtest/dbus-vmstate-test.c:114:get_connection: assertion failed (err == NULL): The connection is closed (g-io-error-quark, 18)
cleaning up pid 8419
ERROR - Bail out! ERROR:/tmp/qemu-test/src/tests/qtest/dbus-vmstate-test.c:114:get_connection: assertion failed (err == NULL): The connection is closed (g-io-error-quark, 18)
make: *** [/tmp/qemu-test/src/tests/Makefile.include:632: check-qtest-x86_64] Error 1
make: *** Waiting for unfinished jobs....
Traceback (most recent call last):
  File "./tests/docker/docker.py", line 664, in <module>
---
    raise CalledProcessError(retcode, cmd)
subprocess.CalledProcessError: Command '['sudo', '-n', 'docker', 'run', '--label', 'com.qemu.instance.uuid=ae2152fca69f479fa27d706640eba29f', '-u', '1003', '--security-opt', 'seccomp=unconfined', '--rm', '-e', 'TARGET_LIST=x86_64-softmmu', '-e', 'EXTRA_CONFIGURE_OPTS=', '-e', 'V=', '-e', 'J=14', '-e', 'DEBUG=', '-e', 'SHOW_ENV=', '-e', 'CCACHE_DIR=/var/tmp/ccache', '-v', '/home/patchew2/.cache/qemu-docker-ccache:/var/tmp/ccache:z', '-v', '/var/tmp/patchew-tester-tmp-mb0pm6mw/src/docker-src.2020-03-11-10.08.24.30837:/var/tmp/qemu:z,ro', 'qemu:fedora', '/var/tmp/qemu/run', 'test-debug']' returned non-zero exit status 2.
filter=--filter=label=com.qemu.instance.uuid=ae2152fca69f479fa27d706640eba29f
make[1]: *** [docker-run] Error 1
make[1]: Leaving directory `/var/tmp/patchew-tester-tmp-mb0pm6mw/src'
make: *** [docker-run-test-debug@fedora] Error 2

real    27m46.347s
user    0m8.512s


The full log is available at
http://patchew.org/logs/20200311132151.172389-1-frankja@linux.ibm.com/testing.asan/?type=message.
---
Email generated automatically by Patchew [https://patchew.org/].
Please send your feedback to patchew-devel@redhat.com

Re: [PATCH v9 02/15] s390x: protvirt: Support unpack facility

[Christian Borntraeger]

On 11.03.20 14:21, Janosch Frank wrote:
[...]
> diff --git a/hw/s390x/ipl.c b/hw/s390x/ipl.c
> index b81942e1e6f9002e..98df89e62c25f583 100644
> --- a/hw/s390x/ipl.c
> +++ b/hw/s390x/ipl.c
> @@ -27,6 +27,7 @@
>  #include "hw/s390x/vfio-ccw.h"
>  #include "hw/s390x/css.h"
>  #include "hw/s390x/ebcdic.h"
> +#include "hw/s390x/pv.h"
>  #include "ipl.h"
>  #include "qemu/error-report.h"
>  #include "qemu/config-file.h"
> @@ -566,12 +567,31 @@ void s390_ipl_update_diag308(IplParameterBlock *iplb)
>  {
>      S390IPLState *ipl = get_ipl_device();
>  
> -    ipl->iplb = *iplb;
> -    ipl->iplb_valid = true;
> +    /*
> +     * The IPLB set and retrieved by subcodes 8/9 is completely
> +     * separate from the one managed via subcodes 5/6.
> +     */
> +    if (iplb->pbt == S390_IPL_TYPE_PV) {
> +        ipl->iplb_pv = *iplb;
> +        ipl->iplb_valid_pv = true;
> +    } else {
> +        ipl->iplb = *iplb;
> +        ipl->iplb_valid = true;
> +    }

We call this for DIAG308_SET and DIAG308_PV_SET in diag.c (see below).
Doesnt this allow to set S390_IPL_TYPE_PV via subcode 5 and an CCW type
via subcode 8. It is certainly not an issue security-wise, but it seems to violate
the architecture.
Shouldnt we add a check in diag.c?

[...]
> diff --git a/target/s390x/diag.c b/target/s390x/diag.c
[..]
> @@ -93,6 +102,11 @@ void handle_diag_308(CPUS390XState *env, uint64_t r1, uint64_t r3, uintptr_t ra)
>          return;
>      }
>  
> +    if (subcode >= DIAG308_PV_SET && !s390_has_feat(S390_FEAT_UNPACK)) {
> +        s390_program_interrupt(env, PGM_SPECIFICATION, ra);
> +        return;
> +    }
> +
>      switch (subcode) {
>      case DIAG308_RESET_MOD_CLR:
>          s390_ipl_reset_request(cs, S390_RESET_MODIFIED_CLEAR);
> @@ -105,6 +119,7 @@ void handle_diag_308(CPUS390XState *env, uint64_t r1, uint64_t r3, uintptr_t ra)
>          s390_ipl_reset_request(cs, S390_RESET_REIPL);
>          break;
>      case DIAG308_SET:
> +    case DIAG308_PV_SET:

somewhere here after we have loaded the block. 



Other than that this looks good.

Re: [PATCH v9 03/15] s390x: protvirt: Add migration blocker

[Christian Borntraeger]

On 11.03.20 14:21, Janosch Frank wrote:
> Migration is not yet supported.
> 
> Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
> Reviewed-by: David Hildenbrand <david@redhat.com>
> ---
>  hw/s390x/s390-virtio-ccw.c | 18 ++++++++++++++++++
>  1 file changed, 18 insertions(+)
> 
> diff --git a/hw/s390x/s390-virtio-ccw.c b/hw/s390x/s390-virtio-ccw.c
> index 9569b777a0e1abd6..deb31e060052d279 100644
> --- a/hw/s390x/s390-virtio-ccw.c
> +++ b/hw/s390x/s390-virtio-ccw.c
> @@ -42,6 +42,9 @@
>  #include "hw/s390x/tod.h"
>  #include "sysemu/sysemu.h"
>  #include "hw/s390x/pv.h"
> +#include "migration/blocker.h"
> +
> +static Error *pv_mig_blocker;
>  
>  S390CPU *s390_cpu_addr2state(uint16_t cpu_addr)
>  {
> @@ -321,15 +324,30 @@ static void s390_machine_unprotect(S390CcwMachineState *ms)
>  {
>      s390_pv_vm_disable();
>      ms->pv = false;
> +    migrate_del_blocker(pv_mig_blocker);
> +    error_free_or_abort(&pv_mig_blocker);
>  }
>  
>  static int s390_machine_protect(S390CcwMachineState *ms)
>  {
> +    Error *local_err = NULL;
>      int rc;
>  
> +    error_setg(&pv_mig_blocker,
> +               "protected VMs are currently not migrateable.");
> +    rc = migrate_add_blocker(pv_mig_blocker, &local_err);
> +    if (local_err) {

Shall we rather use rc here? local_err should also work I guess.

Anyway
Reviewed-by: Christian Borntraeger <borntraeger@de.ibm.com>


> +        error_report_err(local_err);
> +        error_free_or_abort(&pv_mig_blocker);
> +        return rc;
> +    }
> +
>      /* Create SE VM */
>      rc = s390_pv_vm_enable();
>      if (rc) {
> +        error_report_err(local_err);
> +        migrate_del_blocker(pv_mig_blocker);
> +        error_free_or_abort(&pv_mig_blocker);
>          return rc;
>      }
>  
> 

Re: [PATCH v9 06/15] s390x: Add SIDA memory ops

[Christian Borntraeger]

On 11.03.20 14:21, Janosch Frank wrote:
> Protected guests save the instruction control blocks in the SIDA
> instead of QEMU/KVM directly accessing the guest's memory.
> 
> Let's introduce new functions to access the SIDA.
> 
> Also the new memops are available with KVM_CAP_S390_PROTECTED, so
> let's check for that.
> 
> Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
> Reviewed-by: David Hildenbrand <david@redhat.com>

Reviewed-by: Christian Borntraeger <borntraeger@de.ibm.com>

> ---
>  target/s390x/cpu.h        |  7 ++++++-
>  target/s390x/kvm.c        | 25 +++++++++++++++++++++++++
>  target/s390x/kvm_s390x.h  |  2 ++
>  target/s390x/mmu_helper.c | 14 ++++++++++++++
>  4 files changed, 47 insertions(+), 1 deletion(-)
> 
> diff --git a/target/s390x/cpu.h b/target/s390x/cpu.h
> index 1d17709d6e10b5e0..035427521cec2528 100644
> --- a/target/s390x/cpu.h
> +++ b/target/s390x/cpu.h
> @@ -823,7 +823,12 @@ int s390_cpu_virt_mem_rw(S390CPU *cpu, vaddr laddr, uint8_t ar, void *hostbuf,
>  #define s390_cpu_virt_mem_check_write(cpu, laddr, ar, len)   \
>          s390_cpu_virt_mem_rw(cpu, laddr, ar, NULL, len, true)
>  void s390_cpu_virt_mem_handle_exc(S390CPU *cpu, uintptr_t ra);
> -
> +int s390_cpu_pv_mem_rw(S390CPU *cpu, unsigned int offset, void *hostbuf,
> +                       int len, bool is_write);
> +#define s390_cpu_pv_mem_read(cpu, offset, dest, len)    \
> +        s390_cpu_pv_mem_rw(cpu, offset, dest, len, false)
> +#define s390_cpu_pv_mem_write(cpu, offset, dest, len)       \
> +        s390_cpu_pv_mem_rw(cpu, offset, dest, len, true)
>  
>  /* sigp.c */
>  int s390_cpu_restart(S390CPU *cpu);
> diff --git a/target/s390x/kvm.c b/target/s390x/kvm.c
> index eec0b92479465b9c..cdcd538b4f7fb318 100644
> --- a/target/s390x/kvm.c
> +++ b/target/s390x/kvm.c
> @@ -154,6 +154,7 @@ static int cap_ri;
>  static int cap_gs;
>  static int cap_hpage_1m;
>  static int cap_vcpu_resets;
> +static int cap_protected;
>  
>  static int active_cmma;
>  
> @@ -346,6 +347,7 @@ int kvm_arch_init(MachineState *ms, KVMState *s)
>      cap_mem_op = kvm_check_extension(s, KVM_CAP_S390_MEM_OP);
>      cap_s390_irq = kvm_check_extension(s, KVM_CAP_S390_INJECT_IRQ);
>      cap_vcpu_resets = kvm_check_extension(s, KVM_CAP_S390_VCPU_RESETS);
> +    cap_protected = kvm_check_extension(s, KVM_CAP_S390_PROTECTED);
>  
>      if (!kvm_check_extension(s, KVM_CAP_S390_GMAP)
>          || !kvm_check_extension(s, KVM_CAP_S390_COW)) {
> @@ -846,6 +848,29 @@ int kvm_s390_mem_op(S390CPU *cpu, vaddr addr, uint8_t ar, void *hostbuf,
>      return ret;
>  }
>  
> +int kvm_s390_mem_op_pv(S390CPU *cpu, uint64_t offset, void *hostbuf,
> +                       int len, bool is_write)
> +{
> +    struct kvm_s390_mem_op mem_op = {
> +        .sida_offset = offset,
> +        .size = len,
> +        .op = is_write ? KVM_S390_MEMOP_SIDA_WRITE
> +                       : KVM_S390_MEMOP_SIDA_READ,
> +        .buf = (uint64_t)hostbuf,
> +    };
> +    int ret;
> +
> +    if (!cap_mem_op || !cap_protected) {
> +        return -ENOSYS;
> +    }
> +
> +    ret = kvm_vcpu_ioctl(CPU(cpu), KVM_S390_MEM_OP, &mem_op);
> +    if (ret < 0) {
> +        error_report("KVM_S390_MEM_OP failed: %s", strerror(-ret));
> +    }
> +    return ret;
> +}
> +
>  /*
>   * Legacy layout for s390:
>   * Older S390 KVM requires the topmost vma of the RAM to be
> diff --git a/target/s390x/kvm_s390x.h b/target/s390x/kvm_s390x.h
> index 0b21789796d7c462..9c38f6ccce83e39e 100644
> --- a/target/s390x/kvm_s390x.h
> +++ b/target/s390x/kvm_s390x.h
> @@ -19,6 +19,8 @@ void kvm_s390_vcpu_interrupt(S390CPU *cpu, struct kvm_s390_irq *irq);
>  void kvm_s390_access_exception(S390CPU *cpu, uint16_t code, uint64_t te_code);
>  int kvm_s390_mem_op(S390CPU *cpu, vaddr addr, uint8_t ar, void *hostbuf,
>                      int len, bool is_write);
> +int kvm_s390_mem_op_pv(S390CPU *cpu, vaddr addr, void *hostbuf, int len,
> +                       bool is_write);
>  void kvm_s390_program_interrupt(S390CPU *cpu, uint16_t code);
>  int kvm_s390_set_cpu_state(S390CPU *cpu, uint8_t cpu_state);
>  void kvm_s390_vcpu_interrupt_pre_save(S390CPU *cpu);
> diff --git a/target/s390x/mmu_helper.c b/target/s390x/mmu_helper.c
> index 0be2f300bbe4ac8b..7d9f3059cd502c49 100644
> --- a/target/s390x/mmu_helper.c
> +++ b/target/s390x/mmu_helper.c
> @@ -474,6 +474,20 @@ static int translate_pages(S390CPU *cpu, vaddr addr, int nr_pages,
>      return 0;
>  }
>  
> +int s390_cpu_pv_mem_rw(S390CPU *cpu, unsigned int offset, void *hostbuf,
> +                       int len, bool is_write)
> +{
> +    int ret;
> +
> +    if (kvm_enabled()) {
> +        ret = kvm_s390_mem_op_pv(cpu, offset, hostbuf, len, is_write);
> +    } else {
> +        /* Protected Virtualization is a KVM/Hardware only feature */
> +        g_assert_not_reached();
> +    }
> +    return ret;
> +}
> +
>  /**
>   * s390_cpu_virt_mem_rw:
>   * @laddr:     the logical start address
> 

Re: [PATCH v9 02/15] s390x: protvirt: Support unpack facility

[Janosch Frank]

[-- Attachment #1.1: Type: text/plain, Size: 2370 bytes --]

On 3/11/20 5:07 PM, David Hildenbrand wrote:
> 
>> +++ b/include/hw/s390x/pv.h
>> @@ -0,0 +1,57 @@
>> +/*
>> + * Protected Virtualization header
>> + *
>> + * Copyright IBM Corp. 2020
>> + * Author(s):
>> + *  Janosch Frank <frankja@linux.ibm.com>
>> + *
>> + * This work is licensed under the terms of the GNU GPL, version 2 or (at
>> + * your option) any later version. See the COPYING file in the top-level
>> + * directory.
>> + */
>> +#ifndef HW_S390_PV_H
>> +#define HW_S390_PV_H
>> +
>> +#ifdef CONFIG_KVM
>> +#include "hw/s390x/s390-virtio-ccw.h"
>> +
>> +static inline bool s390_is_pv(void)
>> +{
>> +    static S390CcwMachineState *ccw;
>> +    Object *obj;
>> +
>> +    if (ccw) {
>> +        return ccw->pv;
>> +    }
>> +
>> +    /* we have to bail out for the "none" machine */
>> +    obj = object_dynamic_cast(qdev_get_machine(),
>> +                              TYPE_S390_CCW_MACHINE);
>> +    if (!obj) {
>> +        return false;
>> +    }
>> +    ccw = S390_CCW_MACHINE(obj);
>> +    return ccw->pv;
>> +}
>> +
>> +int s390_pv_vm_enable(void);
>> +void s390_pv_vm_disable(void);
>> +int s390_pv_set_sec_parms(uint64_t origin, uint64_t length);
>> +int s390_pv_unpack(uint64_t addr, uint64_t size, uint64_t tweak);
>> +void s390_pv_perf_clear_reset(void);
>> +int s390_pv_verify(void);
>> +void s390_pv_unshare(void);
>> +#else
>> +static inline bool s390_is_pv(void) { return false; }
>> +static inline int s390_pv_vm_enable(void) { return 0; }
>> +static inline void s390_pv_vm_disable(void) {}
>> +static inline int s390_pv_set_sec_parms(uint64_t origin, uint64_t length) { return 0; }
>> +static inline int s390_pv_unpack(uint64_t addr, uint64_t size, uint64_t tweak) { return 0; }
>> +static inline void s390_pv_perf_clear_reset(void) {}
>> +static inline int s390_pv_verify(void) { return 0; }
>> +static inline void s390_pv_unshare(void) {}
>> +#endif
> 
> Maybe /* CONFIG_KVM */ for the else and endif. Whatever you prefer.

Ack

> 
>> +
>> +
>> +
> 
> Nit: too many newlines.

Ack

> 
>> +#endif /* HW_S390_PV_H */
>> diff --git a/include/hw/s390x/s390-virtio-ccw.h b/include/hw/s390x/s390-virtio-ccw.h
>> index 8aa27199c9123bab..cd1dccc6e3ba8645 100644
> 
> 
> Nothing jumped at me :)
> 
> Reviewed-by: David Hildenbrand <david@redhat.com>

Thanks

> 



[-- Attachment #2: OpenPGP digital signature --]
[-- Type: application/pgp-signature, Size: 833 bytes --]

Re: [PATCH v9 03/15] s390x: protvirt: Add migration blocker

[Janosch Frank]

[-- Attachment #1.1: Type: text/plain, Size: 2016 bytes --]

On 3/12/20 9:42 AM, Christian Borntraeger wrote:
> 
> 
> On 11.03.20 14:21, Janosch Frank wrote:
>> Migration is not yet supported.
>>
>> Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
>> Reviewed-by: David Hildenbrand <david@redhat.com>
>> ---
>>  hw/s390x/s390-virtio-ccw.c | 18 ++++++++++++++++++
>>  1 file changed, 18 insertions(+)
>>
>> diff --git a/hw/s390x/s390-virtio-ccw.c b/hw/s390x/s390-virtio-ccw.c
>> index 9569b777a0e1abd6..deb31e060052d279 100644
>> --- a/hw/s390x/s390-virtio-ccw.c
>> +++ b/hw/s390x/s390-virtio-ccw.c
>> @@ -42,6 +42,9 @@
>>  #include "hw/s390x/tod.h"
>>  #include "sysemu/sysemu.h"
>>  #include "hw/s390x/pv.h"
>> +#include "migration/blocker.h"
>> +
>> +static Error *pv_mig_blocker;
>>  
>>  S390CPU *s390_cpu_addr2state(uint16_t cpu_addr)
>>  {
>> @@ -321,15 +324,30 @@ static void s390_machine_unprotect(S390CcwMachineState *ms)
>>  {
>>      s390_pv_vm_disable();
>>      ms->pv = false;
>> +    migrate_del_blocker(pv_mig_blocker);
>> +    error_free_or_abort(&pv_mig_blocker);
>>  }
>>  
>>  static int s390_machine_protect(S390CcwMachineState *ms)
>>  {
>> +    Error *local_err = NULL;
>>      int rc;
>>  
>> +    error_setg(&pv_mig_blocker,
>> +               "protected VMs are currently not migrateable.");
>> +    rc = migrate_add_blocker(pv_mig_blocker, &local_err);
>> +    if (local_err) {
> 
> Shall we rather use rc here? local_err should also work I guess.
> 
> Anyway
> Reviewed-by: Christian Borntraeger <borntraeger@de.ibm.com>
> 

Hmm, yeah I guess that would make more sense.
Thanks!

> 
>> +        error_report_err(local_err);
>> +        error_free_or_abort(&pv_mig_blocker);
>> +        return rc;
>> +    }
>> +
>>      /* Create SE VM */
>>      rc = s390_pv_vm_enable();
>>      if (rc) {
>> +        error_report_err(local_err);
>> +        migrate_del_blocker(pv_mig_blocker);
>> +        error_free_or_abort(&pv_mig_blocker);
>>          return rc;
>>      }
>>  
>>



[-- Attachment #2: OpenPGP digital signature --]
[-- Type: application/pgp-signature, Size: 833 bytes --]

Re: [PATCH v9 07/15] s390x: protvirt: Move STSI data over SIDAD

[Christian Borntraeger]

On 11.03.20 14:21, Janosch Frank wrote:
> For protected guests, we need to put the STSI emulation results into
> the SIDA, so SIE will write them into the guest at the next entry.
> 
> Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
> Reviewed-by: David Hildenbrand <david@redhat.com>
> ---
>  target/s390x/kvm.c | 11 +++++++++--
>  1 file changed, 9 insertions(+), 2 deletions(-)
> 
> diff --git a/target/s390x/kvm.c b/target/s390x/kvm.c
> index cdcd538b4f7fb318..8085d5030e7c6454 100644
> --- a/target/s390x/kvm.c
> +++ b/target/s390x/kvm.c
> @@ -50,6 +50,7 @@
>  #include "exec/memattrs.h"
>  #include "hw/s390x/s390-virtio-ccw.h"
>  #include "hw/s390x/s390-virtio-hcall.h"
> +#include "hw/s390x/pv.h"
>  
>  #ifndef DEBUG_KVM
>  #define DEBUG_KVM  0
> @@ -1800,7 +1801,9 @@ static void insert_stsi_3_2_2(S390CPU *cpu, __u64 addr, uint8_t ar)
>      SysIB_322 sysib;
>      int del;
>  
> -    if (s390_cpu_virt_mem_read(cpu, addr, ar, &sysib, sizeof(sysib))) {
> +    if (s390_is_pv()) {
> +        s390_cpu_pv_mem_read(cpu, 0, &sysib, sizeof(sysib));

Not strictly necessary, but do we also want to do an early exit if the pv case fails?

Re: [PATCH v9 07/15] s390x: protvirt: Move STSI data over SIDAD

[Janosch Frank]

[-- Attachment #1.1: Type: text/plain, Size: 1344 bytes --]

On 3/12/20 11:42 AM, Christian Borntraeger wrote:
> 
> 
> On 11.03.20 14:21, Janosch Frank wrote:
>> For protected guests, we need to put the STSI emulation results into
>> the SIDA, so SIE will write them into the guest at the next entry.
>>
>> Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
>> Reviewed-by: David Hildenbrand <david@redhat.com>
>> ---
>>  target/s390x/kvm.c | 11 +++++++++--
>>  1 file changed, 9 insertions(+), 2 deletions(-)
>>
>> diff --git a/target/s390x/kvm.c b/target/s390x/kvm.c
>> index cdcd538b4f7fb318..8085d5030e7c6454 100644
>> --- a/target/s390x/kvm.c
>> +++ b/target/s390x/kvm.c
>> @@ -50,6 +50,7 @@
>>  #include "exec/memattrs.h"
>>  #include "hw/s390x/s390-virtio-ccw.h"
>>  #include "hw/s390x/s390-virtio-hcall.h"
>> +#include "hw/s390x/pv.h"
>>  
>>  #ifndef DEBUG_KVM
>>  #define DEBUG_KVM  0
>> @@ -1800,7 +1801,9 @@ static void insert_stsi_3_2_2(S390CPU *cpu, __u64 addr, uint8_t ar)
>>      SysIB_322 sysib;
>>      int del;
>>  
>> -    if (s390_cpu_virt_mem_read(cpu, addr, ar, &sysib, sizeof(sysib))) {
>> +    if (s390_is_pv()) {
>> +        s390_cpu_pv_mem_read(cpu, 0, &sysib, sizeof(sysib));
> 
> Not strictly necessary, but do we also want to do an early exit if the pv case fails?
> 

I'd rather do an early exit for the SIDA read/write ioctl itself


[-- Attachment #2: OpenPGP digital signature --]
[-- Type: application/pgp-signature, Size: 833 bytes --]

Re: [PATCH v9 14/15] docs: Add protvirt docs

[Janosch Frank]

[-- Attachment #1.1: Type: text/plain, Size: 3215 bytes --]

On 3/11/20 5:16 PM, David Hildenbrand wrote:
> 
> I don't have to warn you that I am not a native speaker ;)
> 
>> +Prerequisites
>> +-------------
>> +
>> +To run PVMs a machine with the Protected Virtualization feature
> 
> PVMs, a
> 
>> +which is indicated by the Ultravisor Call facility (stfle bit
> 
> , which ..., is required
> 
>> +158) is required. The Ultravisor needs to be initialized at boot by
>> +setting `prot_virt=1` on the kernel command line.
>> +
>> +If those requirements are met, the capability `KVM_CAP_S390_PROTECTED`
>> +will indicate that KVM can support PVMs on that LPAR.
>> +
>> +
>> +QEMU Settings
>> +-------------
>> +
>> +To indicate to the VM that it can transition into protected mode, the
>> +`Unpack facility` (stfle bit 161 represented by the feature
>> +`S390_FEAT_UNPACK`) needs to be part of the cpu model of the VM.
> 
> maybe mention the feature name instead of S390_FEAT_UNPACK ? "unpack"
> 
>> +
>> +All I/O devices need to use the IOMMU.
> 
> need to/have to ?
> 
>> +Passthrough (vfio) devices are currently not supported.
> 
> Does that have to be fenced or will they simply not be detected/not work?

I guess they will lead to the VM being killed by the kernel, since it
can't access protected memory. I need to check that with Halil to
confirm though.

> 
>> +
>> +Host huge page backings are not supported. However guests can use huge
>> +pages as indicated by its facilities.
> 
> Maybe mention what will happen if huge pages are used.

Currently we would fail the unpack process, which I dislike because it
would only log a cryptic error code. I will have a look if I can instead
print an error when the subcode 10 is issued and return an error.


> 
>> +
>> +
>> +Boot Process
>> +------------
>> +
>> +A secure guest image can either be loaded from disk or supplied on the
>> +QEMU command line. Booting from disk is done by the unmodified
>> +s390-ccw BIOS. I.e., the bootmap is interpreted, multiple components
>> +are read into memory and control is transferred to one of the
>> +components (zipl stage3). Stag3 does some fixups and then transfers
>> +control to some program residing in guest memory, which is normally
>> +the OS kernel. The secure image has another component prepended
>> +(stage3a) that uses the new diag308 subcodes 8 and 10 to trigger the
>> +transition into secure mode.
>> +
>> +Booting from the image supplied via the QEMU command line requires
> 
> via/on as above?

Sure

> 
>> +that the file passed via -kernel has the same memory layout as would
>> +result from the disk boot. This memory layout includes the encrypted
>> +components (kernel, initrd, cmdline), the stage3a loader and
>> +metadata. In case this boot method is used, the command line
>> +options -initrd and -cmdline are ineffective. The preparation of a PVM
> 
> Is there way we could warn if these would be set?

I'll have a look, but I'm unsure where I'll need to hook into to make
such a check and it the effort is worth it.

> 
>> +image is done by genprotimg of the s390-tools package.
>>
> 
> 
> Acked-by: David Hildenbrand <david@redhat.com>
> 

Thanks!


[-- Attachment #2: OpenPGP digital signature --]
[-- Type: application/pgp-signature, Size: 833 bytes --]

Re: [PATCH v9 09/15] s390x: protvirt: Set guest IPL PSW

[Christian Borntraeger]

On 11.03.20 14:21, Janosch Frank wrote:
> Handling of CPU reset and setting of the IPL psw from guest storage at
> offset 0 is done by a Ultravisor call. Let's only fetch it if
> necessary.
> 
> Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
> Reviewed-by: Thomas Huth <thuth@redhat.com>
> Reviewed-by: David Hildenbrand <david@redhat.com>

Reviewed-by: Christian Borntraeger <borntraeger@de.ibm.com>

> ---
>  target/s390x/cpu.c | 22 +++++++++++++---------
>  1 file changed, 13 insertions(+), 9 deletions(-)
> 
> diff --git a/target/s390x/cpu.c b/target/s390x/cpu.c
> index 84029f14814b4980..a48d39f139cdc1c4 100644
> --- a/target/s390x/cpu.c
> +++ b/target/s390x/cpu.c
> @@ -78,16 +78,20 @@ static bool s390_cpu_has_work(CPUState *cs)
>  static void s390_cpu_load_normal(CPUState *s)
>  {
>      S390CPU *cpu = S390_CPU(s);
> -    uint64_t spsw = ldq_phys(s->as, 0);
> -
> -    cpu->env.psw.mask = spsw & PSW_MASK_SHORT_CTRL;
> -    /*
> -     * Invert short psw indication, so SIE will report a specification
> -     * exception if it was not set.
> -     */
> -    cpu->env.psw.mask ^= PSW_MASK_SHORTPSW;
> -    cpu->env.psw.addr = spsw & PSW_MASK_SHORT_ADDR;
> +    uint64_t spsw;
>  
> +    if (!s390_is_pv()) {
> +        spsw = ldq_phys(s->as, 0);
> +        cpu->env.psw.mask = spsw & PSW_MASK_SHORT_CTRL;
> +        /*
> +         * Invert short psw indication, so SIE will report a specification
> +         * exception if it was not set.
> +         */
> +        cpu->env.psw.mask ^= PSW_MASK_SHORTPSW;
> +        cpu->env.psw.addr = spsw & PSW_MASK_SHORT_ADDR;
> +    } else {
> +        s390_cpu_set_state(S390_CPU_STATE_LOAD, cpu);
> +    }
>      s390_cpu_set_state(S390_CPU_STATE_OPERATING, cpu);
>  }
>  #endif
> 

Re: [PATCH v9 10/15] s390x: protvirt: Move diag 308 data over SIDA

[Christian Borntraeger]

On 11.03.20 14:21, Janosch Frank wrote:
> For protected guests the IPIB is written/read to/from the SIDA, so we

I think we need to make up our mind regarding IPLB vs. IPIB....

Otherwise
Reviewed-by: Christian Borntraeger <borntraeger@de.ibm.com>
				
> need those accesses to go through s390_cpu_pv_mem_read/write().
> 
> Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
> Reviewed-by: David Hildenbrand <david@redhat.com>
> ---
>  target/s390x/diag.c | 27 +++++++++++++++++++++------
>  1 file changed, 21 insertions(+), 6 deletions(-)
> 
> diff --git a/target/s390x/diag.c b/target/s390x/diag.c
> index b245e557037ded06..a733485caf162111 100644
> --- a/target/s390x/diag.c
> +++ b/target/s390x/diag.c
> @@ -88,6 +88,7 @@ static int diag308_parm_check(CPUS390XState *env, uint64_t r1, uint64_t addr,
>  void handle_diag_308(CPUS390XState *env, uint64_t r1, uint64_t r3, uintptr_t ra)
>  {
>      CPUState *cs = env_cpu(env);
> +    S390CPU *cpu = S390_CPU(cs);
>      uint64_t addr =  env->regs[r1];
>      uint64_t subcode = env->regs[r3];
>      IplParameterBlock *iplb;
> @@ -124,13 +125,22 @@ void handle_diag_308(CPUS390XState *env, uint64_t r1, uint64_t r3, uintptr_t ra)
>              return;
>          }
>          iplb = g_new0(IplParameterBlock, 1);
> -        cpu_physical_memory_read(addr, iplb, sizeof(iplb->len));
> +        if (!s390_is_pv()) {
> +            cpu_physical_memory_read(addr, iplb, sizeof(iplb->len));
> +        } else {
> +            s390_cpu_pv_mem_read(cpu, 0, iplb, sizeof(iplb->len));
> +        }
> +
>          if (!iplb_valid_len(iplb)) {
>              env->regs[r1 + 1] = DIAG_308_RC_INVALID;
>              goto out;
>          }
>  
> -        cpu_physical_memory_read(addr, iplb, be32_to_cpu(iplb->len));
> +        if (!s390_is_pv()) {
> +            cpu_physical_memory_read(addr, iplb, be32_to_cpu(iplb->len));
> +        } else {
> +            s390_cpu_pv_mem_read(cpu, 0, iplb, be32_to_cpu(iplb->len));
> +        }
>  
>          if (!iplb_valid(iplb)) {
>              env->regs[r1 + 1] = DIAG_308_RC_INVALID;
> @@ -152,12 +162,17 @@ out:
>          } else {
>              iplb = s390_ipl_get_iplb();
>          }
> -        if (iplb) {
> -            cpu_physical_memory_write(addr, iplb, be32_to_cpu(iplb->len));
> -            env->regs[r1 + 1] = DIAG_308_RC_OK;
> -        } else {
> +        if (!iplb) {
>              env->regs[r1 + 1] = DIAG_308_RC_NO_CONF;
> +            return;
>          }
> +
> +        if (!s390_is_pv()) {
> +            cpu_physical_memory_write(addr, iplb, be32_to_cpu(iplb->len));
> +        } else {
> +            s390_cpu_pv_mem_write(cpu, 0, iplb, be32_to_cpu(iplb->len));
> +        }
> +        env->regs[r1 + 1] = DIAG_308_RC_OK;
>          return;
>      case DIAG308_PV_START:
>          iplb = s390_ipl_get_iplb_pv();
> 

Re: [PATCH v9 11/15] s390x: protvirt: Disable address checks for PV guest IO emulation

[Christian Borntraeger]

On 11.03.20 14:21, Janosch Frank wrote:
> IO instruction data is routed through SIDAD for protected guests, so
> adresses do not need to be checked, as this is kernel memory.

Maybe also mention that the data structures for  the I/O instrutions
start at offset 0 of the SIDA?
> 
> Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
> Reviewed-by: Thomas Huth <thuth@redhat.com>
> Reviewed-by: David Hildenbrand <david@redhat.com>


Reviewed-by: Christian Borntraeger <borntraeger@de.ibm.com>



> ---
>  target/s390x/ioinst.c | 33 ++++++++++++++++++++++++++-------
>  1 file changed, 26 insertions(+), 7 deletions(-)
> 
> diff --git a/target/s390x/ioinst.c b/target/s390x/ioinst.c
> index c437a1d8c6afed80..481d789de9e09a04 100644
> --- a/target/s390x/ioinst.c
> +++ b/target/s390x/ioinst.c
> @@ -16,6 +16,23 @@
>  #include "hw/s390x/ioinst.h"
>  #include "trace.h"
>  #include "hw/s390x/s390-pci-bus.h"
> +#include "hw/s390x/pv.h"
> +
> +static uint64_t get_address_from_regs(CPUS390XState *env, uint32_t ipb,
> +                                      uint8_t *ar)
> +{
> +    /*
> +     * Addresses for protected guests are all offsets into the
> +     * satellite block which holds the IO control structures. Those
> +     * control structures are always aligned and accessible, so we can
> +     * return 0 here which will pass the following address checks.
> +     */
> +    if (s390_is_pv()) {
> +        *ar = 0;
> +        return 0;
> +    }
> +    return decode_basedisp_s(env, ipb, ar);
> +}
>  
>  int ioinst_disassemble_sch_ident(uint32_t value, int *m, int *cssid, int *ssid,
>                                   int *schid)
> @@ -114,7 +131,7 @@ void ioinst_handle_msch(S390CPU *cpu, uint64_t reg1, uint32_t ipb, uintptr_t ra)
>      CPUS390XState *env = &cpu->env;
>      uint8_t ar;
>  
> -    addr = decode_basedisp_s(env, ipb, &ar);
> +    addr = get_address_from_regs(env, ipb, &ar);
>      if (addr & 3) {
>          s390_program_interrupt(env, PGM_SPECIFICATION, ra);
>          return;
> @@ -171,7 +188,7 @@ void ioinst_handle_ssch(S390CPU *cpu, uint64_t reg1, uint32_t ipb, uintptr_t ra)
>      CPUS390XState *env = &cpu->env;
>      uint8_t ar;
>  
> -    addr = decode_basedisp_s(env, ipb, &ar);
> +    addr = get_address_from_regs(env, ipb, &ar);
>      if (addr & 3) {
>          s390_program_interrupt(env, PGM_SPECIFICATION, ra);
>          return;
> @@ -203,7 +220,7 @@ void ioinst_handle_stcrw(S390CPU *cpu, uint32_t ipb, uintptr_t ra)
>      CPUS390XState *env = &cpu->env;
>      uint8_t ar;
>  
> -    addr = decode_basedisp_s(env, ipb, &ar);
> +    addr = get_address_from_regs(env, ipb, &ar);
>      if (addr & 3) {
>          s390_program_interrupt(env, PGM_SPECIFICATION, ra);
>          return;
> @@ -234,7 +251,7 @@ void ioinst_handle_stsch(S390CPU *cpu, uint64_t reg1, uint32_t ipb,
>      CPUS390XState *env = &cpu->env;
>      uint8_t ar;
>  
> -    addr = decode_basedisp_s(env, ipb, &ar);
> +    addr = get_address_from_regs(env, ipb, &ar);
>      if (addr & 3) {
>          s390_program_interrupt(env, PGM_SPECIFICATION, ra);
>          return;
> @@ -303,7 +320,7 @@ int ioinst_handle_tsch(S390CPU *cpu, uint64_t reg1, uint32_t ipb, uintptr_t ra)
>          return -EIO;
>      }
>      trace_ioinst_sch_id("tsch", cssid, ssid, schid);
> -    addr = decode_basedisp_s(env, ipb, &ar);
> +    addr = get_address_from_regs(env, ipb, &ar);
>      if (addr & 3) {
>          s390_program_interrupt(env, PGM_SPECIFICATION, ra);
>          return -EIO;
> @@ -601,7 +618,7 @@ void ioinst_handle_chsc(S390CPU *cpu, uint32_t ipb, uintptr_t ra)
>  {
>      ChscReq *req;
>      ChscResp *res;
> -    uint64_t addr;
> +    uint64_t addr = 0;
>      int reg;
>      uint16_t len;
>      uint16_t command;
> @@ -610,7 +627,9 @@ void ioinst_handle_chsc(S390CPU *cpu, uint32_t ipb, uintptr_t ra)
>  
>      trace_ioinst("chsc");
>      reg = (ipb >> 20) & 0x00f;
> -    addr = env->regs[reg];
> +    if (!s390_is_pv()) {
> +        addr = env->regs[reg];
> +    }
>      /* Page boundary? */
>      if (addr & 0xfff) {
>          s390_program_interrupt(env, PGM_SPECIFICATION, ra);
> 

Re: [PATCH v9 13/15] s390x: protvirt: Handle SIGP store status correctly

[Christian Borntraeger]

On 11.03.20 14:21, Janosch Frank wrote:
> For protected VMs status storing is not done by QEMU anymore.
> 
> Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
> Reviewed-by: Thomas Huth <thuth@redhat.com>
> Reviewed-by: David Hildenbrand <david@redhat.com>


> ---
>  target/s390x/helper.c | 6 ++++++
>  1 file changed, 6 insertions(+)
> 
> diff --git a/target/s390x/helper.c b/target/s390x/helper.c
> index ed726849114f2f35..5022df8812d406c9 100644
> --- a/target/s390x/helper.c
> +++ b/target/s390x/helper.c
> @@ -25,6 +25,7 @@
>  #include "qemu/timer.h"
>  #include "qemu/qemu-print.h"
>  #include "hw/s390x/ioinst.h"
> +#include "hw/s390x/pv.h"
>  #include "sysemu/hw_accel.h"
>  #include "sysemu/runstate.h"
>  #ifndef CONFIG_USER_ONLY
> @@ -246,6 +247,11 @@ int s390_store_status(S390CPU *cpu, hwaddr addr, bool store_arch)
>      hwaddr len = sizeof(*sa);
>      int i;
>  
> +    /* Storing will occur on next SIE entry for protected VMs */

Maybe ... next SIE entry of the sending CPU .... 
?

Reviewed-by: Christian Borntraeger <borntraeger@de.ibm.com>


> +    if (s390_is_pv()) {
> +        return 0;
> +    }
> +
>      sa = cpu_physical_memory_map(addr, &len, true);
>      if (!sa) {
>          return -EFAULT;
> 

Re: [PATCH v9 10/15] s390x: protvirt: Move diag 308 data over SIDA

[Janosch Frank]

[-- Attachment #1.1: Type: text/plain, Size: 3129 bytes --]

On 3/12/20 4:17 PM, Christian Borntraeger wrote:
> 
> 
> On 11.03.20 14:21, Janosch Frank wrote:
>> For protected guests the IPIB is written/read to/from the SIDA, so we
> 
> I think we need to make up our mind regarding IPLB vs. IPIB....
> 
> Otherwise
> Reviewed-by: Christian Borntraeger <borntraeger@de.ibm.com>

Shall I put that onto the list of things I need to fix in my cleanup series?

> 				
>> need those accesses to go through s390_cpu_pv_mem_read/write().
>>
>> Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
>> Reviewed-by: David Hildenbrand <david@redhat.com>
>> ---
>>  target/s390x/diag.c | 27 +++++++++++++++++++++------
>>  1 file changed, 21 insertions(+), 6 deletions(-)
>>
>> diff --git a/target/s390x/diag.c b/target/s390x/diag.c
>> index b245e557037ded06..a733485caf162111 100644
>> --- a/target/s390x/diag.c
>> +++ b/target/s390x/diag.c
>> @@ -88,6 +88,7 @@ static int diag308_parm_check(CPUS390XState *env, uint64_t r1, uint64_t addr,
>>  void handle_diag_308(CPUS390XState *env, uint64_t r1, uint64_t r3, uintptr_t ra)
>>  {
>>      CPUState *cs = env_cpu(env);
>> +    S390CPU *cpu = S390_CPU(cs);
>>      uint64_t addr =  env->regs[r1];
>>      uint64_t subcode = env->regs[r3];
>>      IplParameterBlock *iplb;
>> @@ -124,13 +125,22 @@ void handle_diag_308(CPUS390XState *env, uint64_t r1, uint64_t r3, uintptr_t ra)
>>              return;
>>          }
>>          iplb = g_new0(IplParameterBlock, 1);
>> -        cpu_physical_memory_read(addr, iplb, sizeof(iplb->len));
>> +        if (!s390_is_pv()) {
>> +            cpu_physical_memory_read(addr, iplb, sizeof(iplb->len));
>> +        } else {
>> +            s390_cpu_pv_mem_read(cpu, 0, iplb, sizeof(iplb->len));
>> +        }
>> +
>>          if (!iplb_valid_len(iplb)) {
>>              env->regs[r1 + 1] = DIAG_308_RC_INVALID;
>>              goto out;
>>          }
>>  
>> -        cpu_physical_memory_read(addr, iplb, be32_to_cpu(iplb->len));
>> +        if (!s390_is_pv()) {
>> +            cpu_physical_memory_read(addr, iplb, be32_to_cpu(iplb->len));
>> +        } else {
>> +            s390_cpu_pv_mem_read(cpu, 0, iplb, be32_to_cpu(iplb->len));
>> +        }
>>  
>>          if (!iplb_valid(iplb)) {
>>              env->regs[r1 + 1] = DIAG_308_RC_INVALID;
>> @@ -152,12 +162,17 @@ out:
>>          } else {
>>              iplb = s390_ipl_get_iplb();
>>          }
>> -        if (iplb) {
>> -            cpu_physical_memory_write(addr, iplb, be32_to_cpu(iplb->len));
>> -            env->regs[r1 + 1] = DIAG_308_RC_OK;
>> -        } else {
>> +        if (!iplb) {
>>              env->regs[r1 + 1] = DIAG_308_RC_NO_CONF;
>> +            return;
>>          }
>> +
>> +        if (!s390_is_pv()) {
>> +            cpu_physical_memory_write(addr, iplb, be32_to_cpu(iplb->len));
>> +        } else {
>> +            s390_cpu_pv_mem_write(cpu, 0, iplb, be32_to_cpu(iplb->len));
>> +        }
>> +        env->regs[r1 + 1] = DIAG_308_RC_OK;
>>          return;
>>      case DIAG308_PV_START:
>>          iplb = s390_ipl_get_iplb_pv();
>>



[-- Attachment #2: OpenPGP digital signature --]
[-- Type: application/pgp-signature, Size: 833 bytes --]

Re: [PATCH v9 11/15] s390x: protvirt: Disable address checks for PV guest IO emulation

[Janosch Frank]

[-- Attachment #1.1: Type: text/plain, Size: 4547 bytes --]

On 3/12/20 4:41 PM, Christian Borntraeger wrote:
> On 11.03.20 14:21, Janosch Frank wrote:
>> IO instruction data is routed through SIDAD for protected guests, so
>> adresses do not need to be checked, as this is kernel memory.
> 
> Maybe also mention that the data structures for  the I/O instrutions
> start at offset 0 of the SIDA?

You mean in the comment for get_address_from_regs() or in the commit
message? It would certainly make sense for get_address_from_regs().

>>
>> Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
>> Reviewed-by: Thomas Huth <thuth@redhat.com>
>> Reviewed-by: David Hildenbrand <david@redhat.com>
> 
> 
> Reviewed-by: Christian Borntraeger <borntraeger@de.ibm.com>
> 
> 
> 
>> ---
>>  target/s390x/ioinst.c | 33 ++++++++++++++++++++++++++-------
>>  1 file changed, 26 insertions(+), 7 deletions(-)
>>
>> diff --git a/target/s390x/ioinst.c b/target/s390x/ioinst.c
>> index c437a1d8c6afed80..481d789de9e09a04 100644
>> --- a/target/s390x/ioinst.c
>> +++ b/target/s390x/ioinst.c
>> @@ -16,6 +16,23 @@
>>  #include "hw/s390x/ioinst.h"
>>  #include "trace.h"
>>  #include "hw/s390x/s390-pci-bus.h"
>> +#include "hw/s390x/pv.h"
>> +
>> +static uint64_t get_address_from_regs(CPUS390XState *env, uint32_t ipb,
>> +                                      uint8_t *ar)
>> +{
>> +    /*
>> +     * Addresses for protected guests are all offsets into the
>> +     * satellite block which holds the IO control structures. Those
>> +     * control structures are always aligned and accessible, so we can
>> +     * return 0 here which will pass the following address checks.
>> +     */
>> +    if (s390_is_pv()) {
>> +        *ar = 0;
>> +        return 0;
>> +    }
>> +    return decode_basedisp_s(env, ipb, ar);
>> +}
>>  
>>  int ioinst_disassemble_sch_ident(uint32_t value, int *m, int *cssid, int *ssid,
>>                                   int *schid)
>> @@ -114,7 +131,7 @@ void ioinst_handle_msch(S390CPU *cpu, uint64_t reg1, uint32_t ipb, uintptr_t ra)
>>      CPUS390XState *env = &cpu->env;
>>      uint8_t ar;
>>  
>> -    addr = decode_basedisp_s(env, ipb, &ar);
>> +    addr = get_address_from_regs(env, ipb, &ar);
>>      if (addr & 3) {
>>          s390_program_interrupt(env, PGM_SPECIFICATION, ra);
>>          return;
>> @@ -171,7 +188,7 @@ void ioinst_handle_ssch(S390CPU *cpu, uint64_t reg1, uint32_t ipb, uintptr_t ra)
>>      CPUS390XState *env = &cpu->env;
>>      uint8_t ar;
>>  
>> -    addr = decode_basedisp_s(env, ipb, &ar);
>> +    addr = get_address_from_regs(env, ipb, &ar);
>>      if (addr & 3) {
>>          s390_program_interrupt(env, PGM_SPECIFICATION, ra);
>>          return;
>> @@ -203,7 +220,7 @@ void ioinst_handle_stcrw(S390CPU *cpu, uint32_t ipb, uintptr_t ra)
>>      CPUS390XState *env = &cpu->env;
>>      uint8_t ar;
>>  
>> -    addr = decode_basedisp_s(env, ipb, &ar);
>> +    addr = get_address_from_regs(env, ipb, &ar);
>>      if (addr & 3) {
>>          s390_program_interrupt(env, PGM_SPECIFICATION, ra);
>>          return;
>> @@ -234,7 +251,7 @@ void ioinst_handle_stsch(S390CPU *cpu, uint64_t reg1, uint32_t ipb,
>>      CPUS390XState *env = &cpu->env;
>>      uint8_t ar;
>>  
>> -    addr = decode_basedisp_s(env, ipb, &ar);
>> +    addr = get_address_from_regs(env, ipb, &ar);
>>      if (addr & 3) {
>>          s390_program_interrupt(env, PGM_SPECIFICATION, ra);
>>          return;
>> @@ -303,7 +320,7 @@ int ioinst_handle_tsch(S390CPU *cpu, uint64_t reg1, uint32_t ipb, uintptr_t ra)
>>          return -EIO;
>>      }
>>      trace_ioinst_sch_id("tsch", cssid, ssid, schid);
>> -    addr = decode_basedisp_s(env, ipb, &ar);
>> +    addr = get_address_from_regs(env, ipb, &ar);
>>      if (addr & 3) {
>>          s390_program_interrupt(env, PGM_SPECIFICATION, ra);
>>          return -EIO;
>> @@ -601,7 +618,7 @@ void ioinst_handle_chsc(S390CPU *cpu, uint32_t ipb, uintptr_t ra)
>>  {
>>      ChscReq *req;
>>      ChscResp *res;
>> -    uint64_t addr;
>> +    uint64_t addr = 0;
>>      int reg;
>>      uint16_t len;
>>      uint16_t command;
>> @@ -610,7 +627,9 @@ void ioinst_handle_chsc(S390CPU *cpu, uint32_t ipb, uintptr_t ra)
>>  
>>      trace_ioinst("chsc");
>>      reg = (ipb >> 20) & 0x00f;
>> -    addr = env->regs[reg];
>> +    if (!s390_is_pv()) {
>> +        addr = env->regs[reg];
>> +    }
>>      /* Page boundary? */
>>      if (addr & 0xfff) {
>>          s390_program_interrupt(env, PGM_SPECIFICATION, ra);
>>



[-- Attachment #2: OpenPGP digital signature --]
[-- Type: application/pgp-signature, Size: 833 bytes --]

Re: [PATCH v9 10/15] s390x: protvirt: Move diag 308 data over SIDA

[Christian Borntraeger]

On 12.03.20 17:05, Janosch Frank wrote:
> On 3/12/20 4:17 PM, Christian Borntraeger wrote:
>>
>>
>> On 11.03.20 14:21, Janosch Frank wrote:
>>> For protected guests the IPIB is written/read to/from the SIDA, so we
>>
>> I think we need to make up our mind regarding IPLB vs. IPIB....
>>
>> Otherwise
>> Reviewed-by: Christian Borntraeger <borntraeger@de.ibm.com>
> 
> Shall I put that onto the list of things I need to fix in my cleanup series?

Yes, nothing urgent but if you have free cycles somewhen, go ahead. 

Re: [PATCH v9 11/15] s390x: protvirt: Disable address checks for PV guest IO emulation

[Christian Borntraeger]

On 12.03.20 17:07, Janosch Frank wrote:
> On 3/12/20 4:41 PM, Christian Borntraeger wrote:
>> On 11.03.20 14:21, Janosch Frank wrote:
>>> IO instruction data is routed through SIDAD for protected guests, so
>>> adresses do not need to be checked, as this is kernel memory.
>>
>> Maybe also mention that the data structures for  the I/O instrutions
>> start at offset 0 of the SIDA?
> 
> You mean in the comment for get_address_from_regs() or in the commit
> message? It would certainly make sense for get_address_from_regs().

Maybe both?

Re: [PATCH v9 13/15] s390x: protvirt: Handle SIGP store status correctly

[Janosch Frank]

[-- Attachment #1.1: Type: text/plain, Size: 1500 bytes --]

On 3/12/20 4:51 PM, Christian Borntraeger wrote:
> On 11.03.20 14:21, Janosch Frank wrote:
>> For protected VMs status storing is not done by QEMU anymore.
>>
>> Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
>> Reviewed-by: Thomas Huth <thuth@redhat.com>
>> Reviewed-by: David Hildenbrand <david@redhat.com>
> 
> 
>> ---
>>  target/s390x/helper.c | 6 ++++++
>>  1 file changed, 6 insertions(+)
>>
>> diff --git a/target/s390x/helper.c b/target/s390x/helper.c
>> index ed726849114f2f35..5022df8812d406c9 100644
>> --- a/target/s390x/helper.c
>> +++ b/target/s390x/helper.c
>> @@ -25,6 +25,7 @@
>>  #include "qemu/timer.h"
>>  #include "qemu/qemu-print.h"
>>  #include "hw/s390x/ioinst.h"
>> +#include "hw/s390x/pv.h"
>>  #include "sysemu/hw_accel.h"
>>  #include "sysemu/runstate.h"
>>  #ifndef CONFIG_USER_ONLY
>> @@ -246,6 +247,11 @@ int s390_store_status(S390CPU *cpu, hwaddr addr, bool store_arch)
>>      hwaddr len = sizeof(*sa);
>>      int i;
>>  
>> +    /* Storing will occur on next SIE entry for protected VMs */
> 
> Maybe ... next SIE entry of the sending CPU .... 
> ?

Well that would be the current cpu, right?
So:
/* For PVMs storing will occur when this cpu enters SIE again */

> 
> Reviewed-by: Christian Borntraeger <borntraeger@de.ibm.com>
> 
> 
>> +    if (s390_is_pv()) {
>> +        return 0;
>> +    }
>> +
>>      sa = cpu_physical_memory_map(addr, &len, true);
>>      if (!sa) {
>>          return -EFAULT;
>>



[-- Attachment #2: OpenPGP digital signature --]
[-- Type: application/pgp-signature, Size: 833 bytes --]

Re: [PATCH v9 11/15] s390x: protvirt: Disable address checks for PV guest IO emulation

[Janosch Frank]

[-- Attachment #1.1: Type: text/plain, Size: 1546 bytes --]

On 3/12/20 5:10 PM, Christian Borntraeger wrote:
> 
> 
> On 12.03.20 17:07, Janosch Frank wrote:
>> On 3/12/20 4:41 PM, Christian Borntraeger wrote:
>>> On 11.03.20 14:21, Janosch Frank wrote:
>>>> IO instruction data is routed through SIDAD for protected guests, so
>>>> adresses do not need to be checked, as this is kernel memory.
>>>
>>> Maybe also mention that the data structures for  the I/O instrutions
>>> start at offset 0 of the SIDA?
>>
>> You mean in the comment for get_address_from_regs() or in the commit
>> message? It would certainly make sense for get_address_from_regs().
> 
> Maybe both?
> 

This is what I have now:

s390x: protvirt: Disable address checks for PV guest IO emulation

IO instruction data is routed through SIDAD for protected guests, so
adresses do not need to be checked, as this is kernel memory which is
always available.

Also the instruction data always starts at offset 0 of the SIDAD.

Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
Reviewed-by: Thomas Huth <thuth@redhat.com>
Reviewed-by: David Hildenbrand <david@redhat.com>
Reviewed-by: Christian Borntraeger <borntraeger@de.ibm.com>

and

+    /*


+     * Addresses for protected guests are all offsets into the


+     * satellite block which holds the IO control structures. Those


+     * control structures are always starting at offset 0 and are


+     * always aligned and accessible. So we can return 0 here which


+     * will pass the following address checks.


+     */


[-- Attachment #2: OpenPGP digital signature --]
[-- Type: application/pgp-signature, Size: 833 bytes --]

[PATCH v9] s390x: protvirt: Fence huge pages

[Janosch Frank]

Let's bail out of the protected transition if we detect that huge
pages might be in use.

Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
---

I'd like to squash this into the unpack patch to give a proper error
message if we try to transition into the protected mode while being
backed by huge pages. 

---
 hw/s390x/ipl.h             | 16 ++++++++++++++++
 hw/s390x/s390-virtio-ccw.c |  1 -
 target/s390x/diag.c        | 23 ++++++++---------------
 target/s390x/kvm-stub.c    |  5 +++++
 target/s390x/kvm.c         |  5 +++++
 target/s390x/kvm_s390x.h   |  1 +
 6 files changed, 35 insertions(+), 16 deletions(-)

diff --git a/hw/s390x/ipl.h b/hw/s390x/ipl.h
index af5bb130a6334821..95e3183c9cccf8b6 100644
--- a/hw/s390x/ipl.h
+++ b/hw/s390x/ipl.h
@@ -185,6 +185,22 @@ struct S390IPLState {
 typedef struct S390IPLState S390IPLState;
 QEMU_BUILD_BUG_MSG(offsetof(S390IPLState, iplb) & 3, "alignment of iplb wrong");
 
+#define DIAG_308_RC_OK              0x0001
+#define DIAG_308_RC_NO_CONF         0x0102
+#define DIAG_308_RC_INVALID         0x0402
+#define DIAG_308_RC_NO_PV_CONF      0x0902
+#define DIAG_308_RC_INVAL_FOR_PV    0x0a02
+
+#define DIAG308_RESET_MOD_CLR       0
+#define DIAG308_RESET_LOAD_NORM     1
+#define DIAG308_LOAD_CLEAR          3
+#define DIAG308_LOAD_NORMAL_DUMP    4
+#define DIAG308_SET                 5
+#define DIAG308_STORE               6
+#define DIAG308_PV_SET              8
+#define DIAG308_PV_STORE            9
+#define DIAG308_PV_START            10
+
 #define S390_IPL_TYPE_FCP 0x00
 #define S390_IPL_TYPE_CCW 0x02
 #define S390_IPL_TYPE_PV 0x05
diff --git a/hw/s390x/s390-virtio-ccw.c b/hw/s390x/s390-virtio-ccw.c
index ebdaaa3a001f6e8c..d32f35c7f47b9c1d 100644
--- a/hw/s390x/s390-virtio-ccw.c
+++ b/hw/s390x/s390-virtio-ccw.c
@@ -361,7 +361,6 @@ out_err:
     return rc;
 }
 
-#define DIAG_308_RC_INVAL_FOR_PV    0x0a02
 static void s390_machine_inject_pv_error(CPUState *cs)
 {
     int r1 = (cs->kvm_run->s390_sieic.ipa & 0x00f0) >> 4;
diff --git a/target/s390x/diag.c b/target/s390x/diag.c
index b245e557037ded06..b1ca81633b83bbdc 100644
--- a/target/s390x/diag.c
+++ b/target/s390x/diag.c
@@ -21,6 +21,7 @@
 #include "hw/s390x/ipl.h"
 #include "hw/s390x/s390-virtio-ccw.h"
 #include "hw/s390x/pv.h"
+#include "kvm_s390x.h"
 
 int handle_diag_288(CPUS390XState *env, uint64_t r1, uint64_t r3)
 {
@@ -50,21 +51,6 @@ int handle_diag_288(CPUS390XState *env, uint64_t r1, uint64_t r3)
     return diag288_class->handle_timer(diag288, func, timeout);
 }
 
-#define DIAG_308_RC_OK              0x0001
-#define DIAG_308_RC_NO_CONF         0x0102
-#define DIAG_308_RC_INVALID         0x0402
-#define DIAG_308_RC_NO_PV_CONF      0x0902
-
-#define DIAG308_RESET_MOD_CLR       0
-#define DIAG308_RESET_LOAD_NORM     1
-#define DIAG308_LOAD_CLEAR          3
-#define DIAG308_LOAD_NORMAL_DUMP    4
-#define DIAG308_SET                 5
-#define DIAG308_STORE               6
-#define DIAG308_PV_SET              8
-#define DIAG308_PV_STORE            9
-#define DIAG308_PV_START            10
-
 static int diag308_parm_check(CPUS390XState *env, uint64_t r1, uint64_t addr,
                               uintptr_t ra, bool write)
 {
@@ -166,6 +152,13 @@ out:
             return;
         }
 
+        if (kvm_s390_get_hpage_1m()) {
+            error_report("Protected VMs can currently not be backed with "
+                         "huge pages");
+            env->regs[r1 + 1] = DIAG_308_RC_INVAL_FOR_PV;
+            return;
+        }
+
         s390_ipl_reset_request(cs, S390_RESET_PV);
         break;
     default:
diff --git a/target/s390x/kvm-stub.c b/target/s390x/kvm-stub.c
index c4cd497f850eb9c7..aa185017a2a886ca 100644
--- a/target/s390x/kvm-stub.c
+++ b/target/s390x/kvm-stub.c
@@ -39,6 +39,11 @@ int kvm_s390_vcpu_interrupt_post_load(S390CPU *cpu)
     return 0;
 }
 
+int kvm_s390_get_hpage_1m(void)
+{
+    return 0;
+}
+
 int kvm_s390_get_ri(void)
 {
     return 0;
diff --git a/target/s390x/kvm.c b/target/s390x/kvm.c
index 1d6fd6a27b48e35f..c695941076b7aead 100644
--- a/target/s390x/kvm.c
+++ b/target/s390x/kvm.c
@@ -321,6 +321,11 @@ void kvm_s390_set_max_pagesize(uint64_t pagesize, Error **errp)
     cap_hpage_1m = 1;
 }
 
+int kvm_s390_get_hpage_1m(void)
+{
+    return cap_hpage_1m;
+}
+
 static void ccw_machine_class_foreach(ObjectClass *oc, void *opaque)
 {
     MachineClass *mc = MACHINE_CLASS(oc);
diff --git a/target/s390x/kvm_s390x.h b/target/s390x/kvm_s390x.h
index 0b21789796d7c462..dea813f450153c34 100644
--- a/target/s390x/kvm_s390x.h
+++ b/target/s390x/kvm_s390x.h
@@ -23,6 +23,7 @@ void kvm_s390_program_interrupt(S390CPU *cpu, uint16_t code);
 int kvm_s390_set_cpu_state(S390CPU *cpu, uint8_t cpu_state);
 void kvm_s390_vcpu_interrupt_pre_save(S390CPU *cpu);
 int kvm_s390_vcpu_interrupt_post_load(S390CPU *cpu);
+int kvm_s390_get_hpage_1m(void);
 int kvm_s390_get_ri(void);
 int kvm_s390_get_gs(void);
 int kvm_s390_get_clock(uint8_t *tod_high, uint64_t *tod_clock);
-- 
2.25.1

Re: [PATCH v9 11/15] s390x: protvirt: Disable address checks for PV guest IO emulation

[Christian Borntraeger]

On 12.03.20 17:17, Janosch Frank wrote:
> On 3/12/20 5:10 PM, Christian Borntraeger wrote:
>>
>>
>> On 12.03.20 17:07, Janosch Frank wrote:
>>> On 3/12/20 4:41 PM, Christian Borntraeger wrote:
>>>> On 11.03.20 14:21, Janosch Frank wrote:
>>>>> IO instruction data is routed through SIDAD for protected guests, so
>>>>> adresses do not need to be checked, as this is kernel memory.
>>>>
>>>> Maybe also mention that the data structures for  the I/O instrutions
>>>> start at offset 0 of the SIDA?
>>>
>>> You mean in the comment for get_address_from_regs() or in the commit
>>> message? It would certainly make sense for get_address_from_regs().
>>
>> Maybe both?
>>
> 
> This is what I have now:
> 
> s390x: protvirt: Disable address checks for PV guest IO emulation
> 
> IO instruction data is routed through SIDAD for protected guests, so
> adresses do not need to be checked, as this is kernel memory which is
> always available.
> 
> Also the instruction data always starts at offset 0 of the SIDAD.
> 
> Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
> Reviewed-by: Thomas Huth <thuth@redhat.com>
> Reviewed-by: David Hildenbrand <david@redhat.com>
> Reviewed-by: Christian Borntraeger <borntraeger@de.ibm.com>
> 
> and
> 
> +    /*
> 
> 
> +     * Addresses for protected guests are all offsets into the
> 
> 
> +     * satellite block which holds the IO control structures. Those
> 
> 
> +     * control structures are always starting at offset 0 and are
> 
> 
> +     * always aligned and accessible. So we can return 0 here which
> 
> 
> +     * will pass the following address checks.
> 
> 
> +     */
> 

Looks perfect without the empty lines.

Re: [PATCH v9 13/15] s390x: protvirt: Handle SIGP store status correctly

[Christian Borntraeger]

On 12.03.20 17:13, Janosch Frank wrote:
> On 3/12/20 4:51 PM, Christian Borntraeger wrote:
>> On 11.03.20 14:21, Janosch Frank wrote:
>>> For protected VMs status storing is not done by QEMU anymore.
>>>
>>> Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
>>> Reviewed-by: Thomas Huth <thuth@redhat.com>
>>> Reviewed-by: David Hildenbrand <david@redhat.com>
>>
>>
>>> ---
>>>  target/s390x/helper.c | 6 ++++++
>>>  1 file changed, 6 insertions(+)
>>>
>>> diff --git a/target/s390x/helper.c b/target/s390x/helper.c
>>> index ed726849114f2f35..5022df8812d406c9 100644
>>> --- a/target/s390x/helper.c
>>> +++ b/target/s390x/helper.c
>>> @@ -25,6 +25,7 @@
>>>  #include "qemu/timer.h"
>>>  #include "qemu/qemu-print.h"
>>>  #include "hw/s390x/ioinst.h"
>>> +#include "hw/s390x/pv.h"
>>>  #include "sysemu/hw_accel.h"
>>>  #include "sysemu/runstate.h"
>>>  #ifndef CONFIG_USER_ONLY
>>> @@ -246,6 +247,11 @@ int s390_store_status(S390CPU *cpu, hwaddr addr, bool store_arch)
>>>      hwaddr len = sizeof(*sa);
>>>      int i;
>>>  
>>> +    /* Storing will occur on next SIE entry for protected VMs */
>>
>> Maybe ... next SIE entry of the sending CPU .... 
>> ?
> 
> Well that would be the current cpu, right?
> So:
> /* For PVMs storing will occur when this cpu enters SIE again */

ack.

Re: [PATCH v9 13/15] s390x: protvirt: Handle SIGP store status correctly

[Christian Borntraeger]

On 12.03.20 17:13, Janosch Frank wrote:
> On 3/12/20 4:51 PM, Christian Borntraeger wrote:
>> On 11.03.20 14:21, Janosch Frank wrote:
>>> For protected VMs status storing is not done by QEMU anymore.
>>>
>>> Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
>>> Reviewed-by: Thomas Huth <thuth@redhat.com>
>>> Reviewed-by: David Hildenbrand <david@redhat.com>
>>
>>
>>> ---
>>>  target/s390x/helper.c | 6 ++++++
>>>  1 file changed, 6 insertions(+)
>>>
>>> diff --git a/target/s390x/helper.c b/target/s390x/helper.c
>>> index ed726849114f2f35..5022df8812d406c9 100644
>>> --- a/target/s390x/helper.c
>>> +++ b/target/s390x/helper.c
>>> @@ -25,6 +25,7 @@
>>>  #include "qemu/timer.h"
>>>  #include "qemu/qemu-print.h"
>>>  #include "hw/s390x/ioinst.h"
>>> +#include "hw/s390x/pv.h"
>>>  #include "sysemu/hw_accel.h"
>>>  #include "sysemu/runstate.h"
>>>  #ifndef CONFIG_USER_ONLY
>>> @@ -246,6 +247,11 @@ int s390_store_status(S390CPU *cpu, hwaddr addr, bool store_arch)
>>>      hwaddr len = sizeof(*sa);
>>>      int i;
>>>  
>>> +    /* Storing will occur on next SIE entry for protected VMs */
>>
>> Maybe ... next SIE entry of the sending CPU .... 
>> ?
> 
> Well that would be the current cpu, right?
> So:
> /* For PVMs storing will occur when this cpu enters SIE again */

ack.

Re: [PATCH v9] s390x: protvirt: Fence huge pages

[Christian Borntraeger]

On 12.03.20 17:25, Janosch Frank wrote:
> Let's bail out of the protected transition if we detect that huge
> pages might be in use.
> 
> Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
> ---
> 
> I'd like to squash this into the unpack patch to give a proper error
> message if we try to transition into the protected mode while being
> backed by huge pages. 

Looks good.
But maybe we can do it better. Why not reverse the logic and
instead of having kvm_s390_get_hpage_1m, let us define an protvirt_allowed
that as of today only returns hugepages != 1:
Then we could (for kvm-stub.c) also say protvirt_allowed=false;
And if other reasons come along we can extend.

We could also keep this patch separate, does not really matter.

> 
> ---
>  hw/s390x/ipl.h             | 16 ++++++++++++++++
>  hw/s390x/s390-virtio-ccw.c |  1 -
>  target/s390x/diag.c        | 23 ++++++++---------------
>  target/s390x/kvm-stub.c    |  5 +++++
>  target/s390x/kvm.c         |  5 +++++
>  target/s390x/kvm_s390x.h   |  1 +
>  6 files changed, 35 insertions(+), 16 deletions(-)
> 
> diff --git a/hw/s390x/ipl.h b/hw/s390x/ipl.h
> index af5bb130a6334821..95e3183c9cccf8b6 100644
> --- a/hw/s390x/ipl.h
> +++ b/hw/s390x/ipl.h
> @@ -185,6 +185,22 @@ struct S390IPLState {
>  typedef struct S390IPLState S390IPLState;
>  QEMU_BUILD_BUG_MSG(offsetof(S390IPLState, iplb) & 3, "alignment of iplb wrong");
>  
> +#define DIAG_308_RC_OK              0x0001
> +#define DIAG_308_RC_NO_CONF         0x0102
> +#define DIAG_308_RC_INVALID         0x0402
> +#define DIAG_308_RC_NO_PV_CONF      0x0902
> +#define DIAG_308_RC_INVAL_FOR_PV    0x0a02
> +
> +#define DIAG308_RESET_MOD_CLR       0
> +#define DIAG308_RESET_LOAD_NORM     1
> +#define DIAG308_LOAD_CLEAR          3
> +#define DIAG308_LOAD_NORMAL_DUMP    4
> +#define DIAG308_SET                 5
> +#define DIAG308_STORE               6
> +#define DIAG308_PV_SET              8
> +#define DIAG308_PV_STORE            9
> +#define DIAG308_PV_START            10
> +
>  #define S390_IPL_TYPE_FCP 0x00
>  #define S390_IPL_TYPE_CCW 0x02
>  #define S390_IPL_TYPE_PV 0x05
> diff --git a/hw/s390x/s390-virtio-ccw.c b/hw/s390x/s390-virtio-ccw.c
> index ebdaaa3a001f6e8c..d32f35c7f47b9c1d 100644
> --- a/hw/s390x/s390-virtio-ccw.c
> +++ b/hw/s390x/s390-virtio-ccw.c
> @@ -361,7 +361,6 @@ out_err:
>      return rc;
>  }
>  
> -#define DIAG_308_RC_INVAL_FOR_PV    0x0a02
>  static void s390_machine_inject_pv_error(CPUState *cs)
>  {
>      int r1 = (cs->kvm_run->s390_sieic.ipa & 0x00f0) >> 4;
> diff --git a/target/s390x/diag.c b/target/s390x/diag.c
> index b245e557037ded06..b1ca81633b83bbdc 100644
> --- a/target/s390x/diag.c
> +++ b/target/s390x/diag.c
> @@ -21,6 +21,7 @@
>  #include "hw/s390x/ipl.h"
>  #include "hw/s390x/s390-virtio-ccw.h"
>  #include "hw/s390x/pv.h"
> +#include "kvm_s390x.h"
>  
>  int handle_diag_288(CPUS390XState *env, uint64_t r1, uint64_t r3)
>  {
> @@ -50,21 +51,6 @@ int handle_diag_288(CPUS390XState *env, uint64_t r1, uint64_t r3)
>      return diag288_class->handle_timer(diag288, func, timeout);
>  }
>  
> -#define DIAG_308_RC_OK              0x0001
> -#define DIAG_308_RC_NO_CONF         0x0102
> -#define DIAG_308_RC_INVALID         0x0402
> -#define DIAG_308_RC_NO_PV_CONF      0x0902
> -
> -#define DIAG308_RESET_MOD_CLR       0
> -#define DIAG308_RESET_LOAD_NORM     1
> -#define DIAG308_LOAD_CLEAR          3
> -#define DIAG308_LOAD_NORMAL_DUMP    4
> -#define DIAG308_SET                 5
> -#define DIAG308_STORE               6
> -#define DIAG308_PV_SET              8
> -#define DIAG308_PV_STORE            9
> -#define DIAG308_PV_START            10
> -
>  static int diag308_parm_check(CPUS390XState *env, uint64_t r1, uint64_t addr,
>                                uintptr_t ra, bool write)
>  {
> @@ -166,6 +152,13 @@ out:
>              return;
>          }
>  
> +        if (kvm_s390_get_hpage_1m()) {
> +            error_report("Protected VMs can currently not be backed with "
> +                         "huge pages");
> +            env->regs[r1 + 1] = DIAG_308_RC_INVAL_FOR_PV;
> +            return;
> +        }
> +
>          s390_ipl_reset_request(cs, S390_RESET_PV);
>          break;
>      default:
> diff --git a/target/s390x/kvm-stub.c b/target/s390x/kvm-stub.c
> index c4cd497f850eb9c7..aa185017a2a886ca 100644
> --- a/target/s390x/kvm-stub.c
> +++ b/target/s390x/kvm-stub.c
> @@ -39,6 +39,11 @@ int kvm_s390_vcpu_interrupt_post_load(S390CPU *cpu)
>      return 0;
>  }
>  
> +int kvm_s390_get_hpage_1m(void)
> +{
> +    return 0;
> +}
> +
>  int kvm_s390_get_ri(void)
>  {
>      return 0;
> diff --git a/target/s390x/kvm.c b/target/s390x/kvm.c
> index 1d6fd6a27b48e35f..c695941076b7aead 100644
> --- a/target/s390x/kvm.c
> +++ b/target/s390x/kvm.c
> @@ -321,6 +321,11 @@ void kvm_s390_set_max_pagesize(uint64_t pagesize, Error **errp)
>      cap_hpage_1m = 1;
>  }
>  
> +int kvm_s390_get_hpage_1m(void)
> +{
> +    return cap_hpage_1m;
> +}
> +
>  static void ccw_machine_class_foreach(ObjectClass *oc, void *opaque)
>  {
>      MachineClass *mc = MACHINE_CLASS(oc);
> diff --git a/target/s390x/kvm_s390x.h b/target/s390x/kvm_s390x.h
> index 0b21789796d7c462..dea813f450153c34 100644
> --- a/target/s390x/kvm_s390x.h
> +++ b/target/s390x/kvm_s390x.h
> @@ -23,6 +23,7 @@ void kvm_s390_program_interrupt(S390CPU *cpu, uint16_t code);
>  int kvm_s390_set_cpu_state(S390CPU *cpu, uint8_t cpu_state);
>  void kvm_s390_vcpu_interrupt_pre_save(S390CPU *cpu);
>  int kvm_s390_vcpu_interrupt_post_load(S390CPU *cpu);
> +int kvm_s390_get_hpage_1m(void);
>  int kvm_s390_get_ri(void);
>  int kvm_s390_get_gs(void);
>  int kvm_s390_get_clock(uint8_t *tod_high, uint64_t *tod_clock);
> 

Re: [PATCH v9 02/15] s390x: protvirt: Support unpack facility

[Janosch Frank]

[-- Attachment #1.1: Type: text/plain, Size: 2312 bytes --]

On 3/12/20 9:33 AM, Christian Borntraeger wrote:
> 
> 
> On 11.03.20 14:21, Janosch Frank wrote:
> [...]
>> diff --git a/hw/s390x/ipl.c b/hw/s390x/ipl.c
>> index b81942e1e6f9002e..98df89e62c25f583 100644
>> --- a/hw/s390x/ipl.c
>> +++ b/hw/s390x/ipl.c
>> @@ -27,6 +27,7 @@
>>  #include "hw/s390x/vfio-ccw.h"
>>  #include "hw/s390x/css.h"
>>  #include "hw/s390x/ebcdic.h"
>> +#include "hw/s390x/pv.h"
>>  #include "ipl.h"
>>  #include "qemu/error-report.h"
>>  #include "qemu/config-file.h"
>> @@ -566,12 +567,31 @@ void s390_ipl_update_diag308(IplParameterBlock *iplb)
>>  {
>>      S390IPLState *ipl = get_ipl_device();
>>  
>> -    ipl->iplb = *iplb;
>> -    ipl->iplb_valid = true;
>> +    /*
>> +     * The IPLB set and retrieved by subcodes 8/9 is completely
>> +     * separate from the one managed via subcodes 5/6.
>> +     */
>> +    if (iplb->pbt == S390_IPL_TYPE_PV) {
>> +        ipl->iplb_pv = *iplb;
>> +        ipl->iplb_valid_pv = true;
>> +    } else {
>> +        ipl->iplb = *iplb;
>> +        ipl->iplb_valid = true;
>> +    }
> 
> We call this for DIAG308_SET and DIAG308_PV_SET in diag.c (see below).
> Doesnt this allow to set S390_IPL_TYPE_PV via subcode 5 and an CCW type
> via subcode 8. It is certainly not an issue security-wise, but it seems to violate
> the architecture.
> Shouldnt we add a check in diag.c?

That would make sense, I'll add it

> 
> [...]
>> diff --git a/target/s390x/diag.c b/target/s390x/diag.c
> [..]
>> @@ -93,6 +102,11 @@ void handle_diag_308(CPUS390XState *env, uint64_t r1, uint64_t r3, uintptr_t ra)
>>          return;
>>      }
>>  
>> +    if (subcode >= DIAG308_PV_SET && !s390_has_feat(S390_FEAT_UNPACK)) {
>> +        s390_program_interrupt(env, PGM_SPECIFICATION, ra);
>> +        return;
>> +    }
>> +
>>      switch (subcode) {
>>      case DIAG308_RESET_MOD_CLR:
>>          s390_ipl_reset_request(cs, S390_RESET_MODIFIED_CLEAR);
>> @@ -105,6 +119,7 @@ void handle_diag_308(CPUS390XState *env, uint64_t r1, uint64_t r3, uintptr_t ra)
>>          s390_ipl_reset_request(cs, S390_RESET_REIPL);
>>          break;
>>      case DIAG308_SET:
>> +    case DIAG308_PV_SET:
> 
> somewhere here after we have loaded the block. 
> 
> 
> 
> Other than that this looks good.
> 
> 



[-- Attachment #2: OpenPGP digital signature --]
[-- Type: application/pgp-signature, Size: 833 bytes --]

[PATCH v9] fixup! Fix subcode/pbt

[Janosch Frank]

Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
---
 hw/s390x/ipl.h      | 11 +++++++----
 target/s390x/diag.c |  2 +-
 2 files changed, 8 insertions(+), 5 deletions(-)

diff --git a/hw/s390x/ipl.h b/hw/s390x/ipl.h
index 95e3183c9cccf8b6..f799f7cfcf4763b1 100644
--- a/hw/s390x/ipl.h
+++ b/hw/s390x/ipl.h
@@ -261,15 +261,18 @@ static inline bool ipl_valid_pv_header(IplParameterBlock *iplb)
         return true;
 }
 
-static inline bool iplb_valid(IplParameterBlock *iplb)
+static inline bool iplb_valid(IplParameterBlock *iplb, uint64_t subcode)
 {
     switch (iplb->pbt) {
     case S390_IPL_TYPE_FCP:
-        return be32_to_cpu(iplb->len) >= S390_IPLB_MIN_FCP_LEN;
+        return (subcode == DIAG308_SET &&
+                be32_to_cpu(iplb->len) >= S390_IPLB_MIN_FCP_LEN);
     case S390_IPL_TYPE_CCW:
-        return be32_to_cpu(iplb->len) >= S390_IPLB_MIN_CCW_LEN;
+        return (subcode == DIAG308_SET &&
+                be32_to_cpu(iplb->len) >= S390_IPLB_MIN_CCW_LEN);
     case S390_IPL_TYPE_PV:
-        if (be32_to_cpu(iplb->len) < S390_IPLB_MIN_PV_LEN) {
+        if (subcode != DIAG308_PV_SET ||
+            be32_to_cpu(iplb->len) < S390_IPLB_MIN_PV_LEN) {
             return false;
         }
         if (!ipl_valid_pv_header(iplb)) {
diff --git a/target/s390x/diag.c b/target/s390x/diag.c
index b1ca81633b83bbdc..d4f33db5c23c818d 100644
--- a/target/s390x/diag.c
+++ b/target/s390x/diag.c
@@ -118,7 +118,7 @@ void handle_diag_308(CPUS390XState *env, uint64_t r1, uint64_t r3, uintptr_t ra)
 
         cpu_physical_memory_read(addr, iplb, be32_to_cpu(iplb->len));
 
-        if (!iplb_valid(iplb)) {
+        if (!iplb_valid(iplb, subcode)) {
             env->regs[r1 + 1] = DIAG_308_RC_INVALID;
             goto out;
         }
-- 
2.25.1

Re: [PATCH v9 02/15] s390x: protvirt: Support unpack facility

[Claudio Imbrenda]

On Wed, 11 Mar 2020 09:21:38 -0400
Janosch Frank <frankja@linux.ibm.com> wrote:

> The unpack facility provides the means to setup a protected guest. A
> protected guest cannot be introspected by the hypervisor or any
> user/administrator of the machine it is running on.
> 
> Protected guests are encrypted at rest and need a special boot
> mechanism via diag308 subcode 8 and 10.
> 
> Code 8 sets the PV specific IPLB which is retained separately from
> those set via code 5.
> 
> Code 10 is used to unpack the VM into protected memory, verify its
> integrity and start it.
> 
> Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
> Co-developed-by: Christian Borntraeger <borntraeger@de.ibm.com>
> [Changes to machine]
> ---
>  hw/s390x/Makefile.objs              |   1 +
>  hw/s390x/ipl.c                      |  56 +++++++++++++-
>  hw/s390x/ipl.h                      |  79 +++++++++++++++++++
>  hw/s390x/pv.c                       |  98 +++++++++++++++++++++++
>  hw/s390x/s390-virtio-ccw.c          | 116
> +++++++++++++++++++++++++++- include/hw/s390x/pv.h               |
> 57 ++++++++++++++ include/hw/s390x/s390-virtio-ccw.h  |   1 +
>  target/s390x/cpu.c                  |   2 +
>  target/s390x/cpu_features_def.inc.h |   1 +
>  target/s390x/diag.c                 |  31 +++++++-
>  10 files changed, 436 insertions(+), 6 deletions(-)
>  create mode 100644 hw/s390x/pv.c
>  create mode 100644 include/hw/s390x/pv.h
> 
> diff --git a/hw/s390x/Makefile.objs b/hw/s390x/Makefile.objs
> index e02ed80b6829a511..a46a1c7894e0f612 100644
> --- a/hw/s390x/Makefile.objs
> +++ b/hw/s390x/Makefile.objs
> @@ -31,6 +31,7 @@ obj-y += tod-qemu.o
>  obj-$(CONFIG_KVM) += tod-kvm.o
>  obj-$(CONFIG_KVM) += s390-skeys-kvm.o
>  obj-$(CONFIG_KVM) += s390-stattrib-kvm.o
> +obj-$(CONFIG_KVM) += pv.o
>  obj-y += s390-ccw.o
>  obj-y += ap-device.o
>  obj-y += ap-bridge.o
> diff --git a/hw/s390x/ipl.c b/hw/s390x/ipl.c
> index b81942e1e6f9002e..98df89e62c25f583 100644
> --- a/hw/s390x/ipl.c
> +++ b/hw/s390x/ipl.c
> @@ -27,6 +27,7 @@
>  #include "hw/s390x/vfio-ccw.h"
>  #include "hw/s390x/css.h"
>  #include "hw/s390x/ebcdic.h"
> +#include "hw/s390x/pv.h"
>  #include "ipl.h"
>  #include "qemu/error-report.h"
>  #include "qemu/config-file.h"
> @@ -566,12 +567,31 @@ void s390_ipl_update_diag308(IplParameterBlock
> *iplb) {
>      S390IPLState *ipl = get_ipl_device();
>  
> -    ipl->iplb = *iplb;
> -    ipl->iplb_valid = true;
> +    /*
> +     * The IPLB set and retrieved by subcodes 8/9 is completely
> +     * separate from the one managed via subcodes 5/6.
> +     */
> +    if (iplb->pbt == S390_IPL_TYPE_PV) {
> +        ipl->iplb_pv = *iplb;
> +        ipl->iplb_valid_pv = true;
> +    } else {
> +        ipl->iplb = *iplb;
> +        ipl->iplb_valid = true;
> +    }
>      ipl->netboot = is_virtio_net_device(iplb);
>      update_machine_ipl_properties(iplb);
>  }
>  
> +IplParameterBlock *s390_ipl_get_iplb_pv(void)
> +{
> +    S390IPLState *ipl = get_ipl_device();
> +
> +    if (!ipl->iplb_valid_pv) {
> +        return NULL;
> +    }
> +    return &ipl->iplb_pv;
> +}
> +
>  IplParameterBlock *s390_ipl_get_iplb(void)
>  {
>      S390IPLState *ipl = get_ipl_device();
> @@ -660,6 +680,38 @@ static void s390_ipl_prepare_qipl(S390CPU *cpu)
>      cpu_physical_memory_unmap(addr, len, 1, len);
>  }
>  
> +int s390_ipl_prepare_pv_header(void)
> +{
> +    IplParameterBlock *ipib = s390_ipl_get_iplb_pv();
> +    IPLBlockPV *ipib_pv = &ipib->pv;
> +    void *hdr = g_malloc(ipib_pv->pv_header_len);
> +    int rc;
> +
> +    cpu_physical_memory_read(ipib_pv->pv_header_addr, hdr,
> +                             ipib_pv->pv_header_len);
> +    rc = s390_pv_set_sec_parms((uint64_t)hdr,
> +                               ipib_pv->pv_header_len);
> +    g_free(hdr);
> +    return rc;
> +}
> +
> +int s390_ipl_pv_unpack(void)
> +{
> +    IplParameterBlock *ipib = s390_ipl_get_iplb_pv();
> +    IPLBlockPV *ipib_pv = &ipib->pv;
> +    int i, rc = 0;
> +
> +    for (i = 0; i < ipib_pv->num_comp; i++) {
> +        rc = s390_pv_unpack(ipib_pv->components[i].addr,
> +
> TARGET_PAGE_ALIGN(ipib_pv->components[i].size),
> +                            ipib_pv->components[i].tweak_pref);
> +        if (rc) {
> +            break;
> +        }
> +    }
> +    return rc;
> +}
> +
>  void s390_ipl_prepare_cpu(S390CPU *cpu)
>  {
>      S390IPLState *ipl = get_ipl_device();
> diff --git a/hw/s390x/ipl.h b/hw/s390x/ipl.h
> index 3e44abe1c651d8a0..919f9e69131b5207 100644
> --- a/hw/s390x/ipl.h
> +++ b/hw/s390x/ipl.h
> @@ -15,6 +15,24 @@
>  #include "cpu.h"
>  #include "hw/qdev-core.h"
>  
> +struct IPLBlockPVComp {
> +    uint64_t tweak_pref;
> +    uint64_t addr;
> +    uint64_t size;
> +} QEMU_PACKED;
> +typedef struct IPLBlockPVComp IPLBlockPVComp;
> +
> +struct IPLBlockPV {
> +    uint8_t  reserved18[87];    /* 0x18 */
> +    uint8_t  version;           /* 0x6f */
> +    uint32_t reserved70;        /* 0x70 */
> +    uint32_t num_comp;          /* 0x74 */
> +    uint64_t pv_header_addr;    /* 0x78 */
> +    uint64_t pv_header_len;     /* 0x80 */
> +    struct IPLBlockPVComp components[];
> +} QEMU_PACKED;
> +typedef struct IPLBlockPV IPLBlockPV;
> +
>  struct IplBlockCcw {
>      uint8_t  reserved0[85];
>      uint8_t  ssid;
> @@ -71,6 +89,7 @@ union IplParameterBlock {
>          union {
>              IplBlockCcw ccw;
>              IplBlockFcp fcp;
> +            IPLBlockPV pv;
>              IplBlockQemuScsi scsi;
>          };
>      } QEMU_PACKED;
> @@ -85,8 +104,11 @@ typedef union IplParameterBlock IplParameterBlock;
>  
>  int s390_ipl_set_loadparm(uint8_t *loadparm);
>  void s390_ipl_update_diag308(IplParameterBlock *iplb);
> +int s390_ipl_prepare_pv_header(void);
> +int s390_ipl_pv_unpack(void);
>  void s390_ipl_prepare_cpu(S390CPU *cpu);
>  IplParameterBlock *s390_ipl_get_iplb(void);
> +IplParameterBlock *s390_ipl_get_iplb_pv(void);
>  
>  enum s390_reset {
>      /* default is a reset not triggered by a CPU e.g. issued by QMP
> */ @@ -94,6 +116,7 @@ enum s390_reset {
>      S390_RESET_REIPL,
>      S390_RESET_MODIFIED_CLEAR,
>      S390_RESET_LOAD_NORMAL,
> +    S390_RESET_PV,
>  };
>  void s390_ipl_reset_request(CPUState *cs, enum s390_reset
> reset_type); void s390_ipl_get_reset_request(CPUState **cs, enum
> s390_reset *reset_type); @@ -133,6 +156,7 @@ struct S390IPLState {
>      /*< private >*/
>      DeviceState parent_obj;
>      IplParameterBlock iplb;
> +    IplParameterBlock iplb_pv;
>      QemuIplParameters qipl;
>      uint64_t start_addr;
>      uint64_t compat_start_addr;
> @@ -140,6 +164,7 @@ struct S390IPLState {
>      uint64_t compat_bios_start_addr;
>      bool enforce_bios;
>      bool iplb_valid;
> +    bool iplb_valid_pv;
>      bool netboot;
>      /* reset related properties don't have to be migrated or reset */
>      enum s390_reset reset_type;
> @@ -161,9 +186,11 @@ QEMU_BUILD_BUG_MSG(offsetof(S390IPLState, iplb)
> & 3, "alignment of iplb wrong"); 
>  #define S390_IPL_TYPE_FCP 0x00
>  #define S390_IPL_TYPE_CCW 0x02
> +#define S390_IPL_TYPE_PV 0x05
>  #define S390_IPL_TYPE_QEMU_SCSI 0xff
>  
>  #define S390_IPLB_HEADER_LEN 8
> +#define S390_IPLB_MIN_PV_LEN 148
>  #define S390_IPLB_MIN_CCW_LEN 200
>  #define S390_IPLB_MIN_FCP_LEN 384
>  #define S390_IPLB_MIN_QEMU_SCSI_LEN 200
> @@ -173,6 +200,50 @@ static inline bool
> iplb_valid_len(IplParameterBlock *iplb) return be32_to_cpu(iplb->len)
> <= sizeof(IplParameterBlock); }
>  
> +static inline bool ipl_valid_pv_components(IplParameterBlock *iplb)
> +{
> +    IPLBlockPV *ipib_pv = &iplb->pv;
> +    int i;
> +
> +    if (ipib_pv->num_comp == 0) {
> +        return false;
> +    }
> +
> +    for (i = 0; i < ipib_pv->num_comp; i++) {
> +        /* Addr must be 4k aligned */
> +        if (ipib_pv->components[i].addr & ~TARGET_PAGE_MASK) {
> +            return false;
> +        }
> +
> +        /* Tweak prefix is monotonically increasing with each
> component */
> +        if (i < ipib_pv->num_comp - 1 &&
> +            ipib_pv->components[i].tweak_pref >=
> +            ipib_pv->components[i + 1].tweak_pref) {
> +            return false;
> +        }
> +    }
> +    return true;
> +}
> +
> +static inline bool ipl_valid_pv_header(IplParameterBlock *iplb)
> +{
> +        IPLBlockPV *ipib_pv = &iplb->pv;
> +
> +        if (ipib_pv->pv_header_len > 2 * TARGET_PAGE_SIZE) {
> +            return false;
> +        }
> +
> +        if (!address_space_access_valid(&address_space_memory,
> +                                        ipib_pv->pv_header_addr,
> +                                        ipib_pv->pv_header_len,
> +                                        false,
> +                                        MEMTXATTRS_UNSPECIFIED)) {
> +            return false;
> +        }
> +
> +        return true;
> +}
> +
>  static inline bool iplb_valid(IplParameterBlock *iplb)
>  {
>      switch (iplb->pbt) {
> @@ -180,6 +251,14 @@ static inline bool iplb_valid(IplParameterBlock
> *iplb) return be32_to_cpu(iplb->len) >= S390_IPLB_MIN_FCP_LEN;
>      case S390_IPL_TYPE_CCW:
>          return be32_to_cpu(iplb->len) >= S390_IPLB_MIN_CCW_LEN;
> +    case S390_IPL_TYPE_PV:
> +        if (be32_to_cpu(iplb->len) < S390_IPLB_MIN_PV_LEN) {
> +            return false;
> +        }
> +        if (!ipl_valid_pv_header(iplb)) {
> +            return false;
> +        }
> +        return ipl_valid_pv_components(iplb);
>      default:
>          return false;
>      }
> diff --git a/hw/s390x/pv.c b/hw/s390x/pv.c
> new file mode 100644
> index 0000000000000000..8cf5cd2c9bcd48b0
> --- /dev/null
> +++ b/hw/s390x/pv.c
> @@ -0,0 +1,98 @@
> +/*
> + * Protected Virtualization functions
> + *
> + * Copyright IBM Corp. 2020
> + * Author(s):
> + *  Janosch Frank <frankja@linux.ibm.com>
> + *
> + * This work is licensed under the terms of the GNU GPL, version 2
> or (at
> + * your option) any later version. See the COPYING file in the
> top-level
> + * directory.
> + */
> +#include "qemu/osdep.h"
> +
> +#include <linux/kvm.h>
> +
> +#include "qemu/error-report.h"
> +#include "sysemu/kvm.h"
> +#include "hw/s390x/pv.h"
> +
> +static int __s390_pv_cmd(uint32_t cmd, const char *cmdname, void
> *data) +{
> +    struct kvm_pv_cmd pv_cmd = {
> +        .cmd = cmd,
> +        .data = (uint64_t)data,
> +    };
> +    int rc = kvm_vm_ioctl(kvm_state, KVM_S390_PV_COMMAND, &pv_cmd);
> +
> +    if (rc) {
> +        error_report("KVM PV command %d (%s) failed: header rc %x
> rrc %x "
> +                     "IOCTL rc: %d", cmd, cmdname, pv_cmd.rc,
> pv_cmd.rrc,
> +                     rc);
> +    }
> +    return rc;
> +}
> +
> +/*
> + * This macro lets us pass the command as a string to the function so
> + * we can print it on an error.
> + */
> +#define s390_pv_cmd(cmd, data) __s390_pv_cmd(cmd, #cmd, data);
> +#define s390_pv_cmd_exit(cmd, data)    \
> +{                                      \
> +    int rc;                            \
> +                                       \
> +    rc = __s390_pv_cmd(cmd, #cmd, data);\
> +    if (rc) {                          \
> +        exit(1);                       \
> +    }                                  \
> +}
> +
> +int s390_pv_vm_enable(void)
> +{
> +    return s390_pv_cmd(KVM_PV_ENABLE, NULL);
> +}
> +
> +void s390_pv_vm_disable(void)
> +{
> +     s390_pv_cmd_exit(KVM_PV_DISABLE, NULL);
> +}
> +
> +int s390_pv_set_sec_parms(uint64_t origin, uint64_t length)
> +{
> +    struct kvm_s390_pv_sec_parm args = {
> +        .origin = origin,
> +        .length = length,
> +    };
> +
> +    return s390_pv_cmd(KVM_PV_VM_SET_SEC_PARMS, &args);
> +}
> +
> +/*
> + * Called for each component in the SE type IPL parameter block 0.
> + */
> +int s390_pv_unpack(uint64_t addr, uint64_t size, uint64_t tweak)
> +{
> +    struct kvm_s390_pv_unp args = {
> +        .addr = addr,
> +        .size = size,
> +        .tweak = tweak,
> +    };
> +
> +    return s390_pv_cmd(KVM_PV_VM_UNPACK, &args);
> +}
> +
> +void s390_pv_perf_clear_reset(void)
> +{
> +    s390_pv_cmd_exit(KVM_PV_VM_PREP_RESET, NULL);
> +}
> +
> +int s390_pv_verify(void)
> +{
> +    return s390_pv_cmd(KVM_PV_VM_VERIFY, NULL);
> +}
> +
> +void s390_pv_unshare(void)
> +{
> +    s390_pv_cmd_exit(KVM_PV_VM_UNSHARE_ALL, NULL);
> +}
> diff --git a/hw/s390x/s390-virtio-ccw.c b/hw/s390x/s390-virtio-ccw.c
> index 895498cca6199c16..9569b777a0e1abd6 100644
> --- a/hw/s390x/s390-virtio-ccw.c
> +++ b/hw/s390x/s390-virtio-ccw.c
> @@ -41,6 +41,7 @@
>  #include "hw/qdev-properties.h"
>  #include "hw/s390x/tod.h"
>  #include "sysemu/sysemu.h"
> +#include "hw/s390x/pv.h"
>  
>  S390CPU *s390_cpu_addr2state(uint16_t cpu_addr)
>  {
> @@ -316,10 +317,79 @@ static inline void s390_do_cpu_ipl(CPUState
> *cs, run_on_cpu_data arg)
> s390_cpu_set_state(S390_CPU_STATE_OPERATING, cpu); }
>  
> +static void s390_machine_unprotect(S390CcwMachineState *ms)
> +{
> +    s390_pv_vm_disable();
> +    ms->pv = false;
> +}
> +
> +static int s390_machine_protect(S390CcwMachineState *ms)
> +{
> +    int rc;
> +
> +    /* Create SE VM */
> +    rc = s390_pv_vm_enable();
> +    if (rc) {
> +        return rc;
> +    }
> +
> +    ms->pv = true;
> +
> +    /* Set SE header and unpack */
> +    rc = s390_ipl_prepare_pv_header();
> +    if (rc) {
> +        goto out_err;
> +    }
> +
> +    /* Decrypt image */
> +    rc = s390_ipl_pv_unpack();
> +    if (rc) {
> +        goto out_err;
> +    }
> +
> +    /* Verify integrity */
> +    rc = s390_pv_verify();
> +    if (rc) {
> +        goto out_err;
> +    }
> +    return rc;
> +
> +out_err:
> +    s390_machine_unprotect(ms);
> +    return rc;
> +}
> +
> +#define DIAG_308_RC_INVAL_FOR_PV    0x0a02
> +static void s390_machine_inject_pv_error(CPUState *cs)
> +{
> +    int r1 = (cs->kvm_run->s390_sieic.ipa & 0x00f0) >> 4;
> +    CPUS390XState *env = &S390_CPU(cs)->env;
> +
> +    /* Report that we are unable to enter protected mode */
> +    env->regs[r1 + 1] = DIAG_308_RC_INVAL_FOR_PV;
> +}
> +
> +static void s390_pv_prepare_reset(S390CcwMachineState *ms)
> +{
> +    CPUState *cs;
> +
> +    if (!s390_is_pv()) {
> +        return;
> +    }
> +    /* Unsharing requires all cpus to be stopped */
> +    CPU_FOREACH(cs) {
> +        s390_cpu_set_state(S390_CPU_STATE_STOPPED, S390_CPU(cs));
> +    }
> +    s390_pv_unshare();
> +    s390_pv_perf_clear_reset();
> +}
> +
>  static void s390_machine_reset(MachineState *machine)
>  {
> +    S390CcwMachineState *ms = S390_CCW_MACHINE(machine);
>      enum s390_reset reset_type;
>      CPUState *cs, *t;
> +    S390CPU *cpu;
>  
>      /* get the reset parameters, reset them once done */
>      s390_ipl_get_reset_request(&cs, &reset_type);
> @@ -327,9 +397,15 @@ static void s390_machine_reset(MachineState
> *machine) /* all CPUs are paused and synchronized at this point */
>      s390_cmma_reset();
>  
> +    cpu = S390_CPU(cs);
> +
>      switch (reset_type) {
>      case S390_RESET_EXTERNAL:
>      case S390_RESET_REIPL:
> +        if (s390_is_pv()) {
> +            s390_machine_unprotect(ms);
> +        }
> +
>          qemu_devices_reset();
>          s390_crypto_reset();
>  
> @@ -337,22 +413,56 @@ static void s390_machine_reset(MachineState
> *machine) run_on_cpu(cs, s390_do_cpu_ipl, RUN_ON_CPU_NULL);
>          break;
>      case S390_RESET_MODIFIED_CLEAR:
> +        /*
> +         * Susbsystem reset needs to be done before we unshare memory
> +         * and loose access to VIRTIO structures in guest memory.

lose

> +         */
> +        subsystem_reset();
> +        s390_crypto_reset();
> +        s390_pv_prepare_reset(ms);
>          CPU_FOREACH(t) {
>              run_on_cpu(t, s390_do_cpu_full_reset, RUN_ON_CPU_NULL);
>          }
> -        subsystem_reset();
> -        s390_crypto_reset();
>          run_on_cpu(cs, s390_do_cpu_load_normal, RUN_ON_CPU_NULL);
>          break;
>      case S390_RESET_LOAD_NORMAL:
> +        /*
> +         * Susbsystem reset needs to be done before we unshare memory
> +         * and loose access to VIRTIO structures in guest memory.

lose

> +         */
> +        subsystem_reset();
> +        s390_pv_prepare_reset(ms);
>          CPU_FOREACH(t) {
>              if (t == cs) {
>                  continue;
>              }
>              run_on_cpu(t, s390_do_cpu_reset, RUN_ON_CPU_NULL);
>          }
> -        subsystem_reset();
>          run_on_cpu(cs, s390_do_cpu_initial_reset, RUN_ON_CPU_NULL);
> +        run_on_cpu(cs, s390_do_cpu_load_normal, RUN_ON_CPU_NULL);
> +        break;
> +    case S390_RESET_PV: /* Subcode 10 */
> +        subsystem_reset();
> +        s390_crypto_reset();
> +
> +        CPU_FOREACH(t) {
> +            if (t == cs) {
> +                continue;
> +            }
> +            run_on_cpu(t, s390_do_cpu_full_reset, RUN_ON_CPU_NULL);
> +        }
> +        run_on_cpu(cs, s390_do_cpu_reset, RUN_ON_CPU_NULL);
> +
> +        if (s390_machine_protect(ms)) {
> +            s390_machine_inject_pv_error(cs);
> +            /*
> +             * Continue after the diag308 so the guest knows
> something
> +             * went wrong.
> +             */
> +            s390_cpu_set_state(S390_CPU_STATE_OPERATING, cpu);
> +            return;
> +        }
> +
>          run_on_cpu(cs, s390_do_cpu_load_normal, RUN_ON_CPU_NULL);
>          break;
>      default:
> diff --git a/include/hw/s390x/pv.h b/include/hw/s390x/pv.h
> new file mode 100644
> index 0000000000000000..43298171a9e23c76
> --- /dev/null
> +++ b/include/hw/s390x/pv.h
> @@ -0,0 +1,57 @@
> +/*
> + * Protected Virtualization header
> + *
> + * Copyright IBM Corp. 2020
> + * Author(s):
> + *  Janosch Frank <frankja@linux.ibm.com>
> + *
> + * This work is licensed under the terms of the GNU GPL, version 2
> or (at
> + * your option) any later version. See the COPYING file in the
> top-level
> + * directory.
> + */
> +#ifndef HW_S390_PV_H
> +#define HW_S390_PV_H
> +
> +#ifdef CONFIG_KVM
> +#include "hw/s390x/s390-virtio-ccw.h"
> +
> +static inline bool s390_is_pv(void)
> +{
> +    static S390CcwMachineState *ccw;
> +    Object *obj;
> +
> +    if (ccw) {
> +        return ccw->pv;
> +    }
> +
> +    /* we have to bail out for the "none" machine */
> +    obj = object_dynamic_cast(qdev_get_machine(),
> +                              TYPE_S390_CCW_MACHINE);
> +    if (!obj) {
> +        return false;
> +    }
> +    ccw = S390_CCW_MACHINE(obj);
> +    return ccw->pv;
> +}
> +
> +int s390_pv_vm_enable(void);
> +void s390_pv_vm_disable(void);
> +int s390_pv_set_sec_parms(uint64_t origin, uint64_t length);
> +int s390_pv_unpack(uint64_t addr, uint64_t size, uint64_t tweak);
> +void s390_pv_perf_clear_reset(void);
> +int s390_pv_verify(void);
> +void s390_pv_unshare(void);
> +#else
> +static inline bool s390_is_pv(void) { return false; }
> +static inline int s390_pv_vm_enable(void) { return 0; }
> +static inline void s390_pv_vm_disable(void) {}
> +static inline int s390_pv_set_sec_parms(uint64_t origin, uint64_t
> length) { return 0; } +static inline int s390_pv_unpack(uint64_t
> addr, uint64_t size, uint64_t tweak) { return 0; } +static inline
> void s390_pv_perf_clear_reset(void) {} +static inline int
> s390_pv_verify(void) { return 0; } +static inline void
> s390_pv_unshare(void) {} +#endif
> +
> +
> +
> +#endif /* HW_S390_PV_H */
> diff --git a/include/hw/s390x/s390-virtio-ccw.h
> b/include/hw/s390x/s390-virtio-ccw.h index
> 8aa27199c9123bab..cd1dccc6e3ba8645 100644 ---
> a/include/hw/s390x/s390-virtio-ccw.h +++
> b/include/hw/s390x/s390-virtio-ccw.h @@ -28,6 +28,7 @@ typedef struct
> S390CcwMachineState { /*< public >*/
>      bool aes_key_wrap;
>      bool dea_key_wrap;
> +    bool pv;
>      uint8_t loadparm[8];
>  } S390CcwMachineState;
>  
> diff --git a/target/s390x/cpu.c b/target/s390x/cpu.c
> index 3dd396e870357944..84029f14814b4980 100644
> --- a/target/s390x/cpu.c
> +++ b/target/s390x/cpu.c
> @@ -37,6 +37,8 @@
>  #include "sysemu/hw_accel.h"
>  #include "hw/qdev-properties.h"
>  #ifndef CONFIG_USER_ONLY
> +#include "hw/s390x/s390-virtio-ccw.h"
> +#include "hw/s390x/pv.h"
>  #include "hw/boards.h"
>  #include "sysemu/arch_init.h"
>  #include "sysemu/sysemu.h"
> diff --git a/target/s390x/cpu_features_def.inc.h
> b/target/s390x/cpu_features_def.inc.h index
> 31dff0d84e972451..60db28351d059091 100644 ---
> a/target/s390x/cpu_features_def.inc.h +++
> b/target/s390x/cpu_features_def.inc.h @@ -107,6 +107,7 @@
> DEF_FEAT(DEFLATE_BASE, "deflate-base", STFL, 151, "Deflate-conversion
> facility ( DEF_FEAT(VECTOR_PACKED_DECIMAL_ENH, "vxpdeh", STFL, 152,
> "Vector-Packed-Decimal-Enhancement Facility") DEF_FEAT(MSA_EXT_9,
> "msa9-base", STFL, 155, "Message-security-assist-extension-9 facility
> (excluding subfunctions)") DEF_FEAT(ETOKEN, "etoken", STFL, 156,
> "Etoken facility") +DEF_FEAT(UNPACK, "unpack", STFL, 161, "Unpack
> facility") /* Features exposed via SCLP SCCB Byte 80 - 98  (bit
> numbers relative to byte-80) */ DEF_FEAT(SIE_GSLS, "gsls",
> SCLP_CONF_CHAR, 40, "SIE: Guest-storage-limit-suppression facility")
> diff --git a/target/s390x/diag.c b/target/s390x/diag.c index
> 54e5670b3fd6d960..b245e557037ded06 100644 --- a/target/s390x/diag.c
> +++ b/target/s390x/diag.c @@ -20,6 +20,7 @@
>  #include "sysemu/cpus.h"
>  #include "hw/s390x/ipl.h"
>  #include "hw/s390x/s390-virtio-ccw.h"
> +#include "hw/s390x/pv.h"
>  
>  int handle_diag_288(CPUS390XState *env, uint64_t r1, uint64_t r3)
>  {
> @@ -52,6 +53,7 @@ int handle_diag_288(CPUS390XState *env, uint64_t
> r1, uint64_t r3) #define DIAG_308_RC_OK              0x0001
>  #define DIAG_308_RC_NO_CONF         0x0102
>  #define DIAG_308_RC_INVALID         0x0402
> +#define DIAG_308_RC_NO_PV_CONF      0x0902
>  
>  #define DIAG308_RESET_MOD_CLR       0
>  #define DIAG308_RESET_LOAD_NORM     1
> @@ -59,10 +61,17 @@ int handle_diag_288(CPUS390XState *env, uint64_t
> r1, uint64_t r3) #define DIAG308_LOAD_NORMAL_DUMP    4
>  #define DIAG308_SET                 5
>  #define DIAG308_STORE               6
> +#define DIAG308_PV_SET              8
> +#define DIAG308_PV_STORE            9
> +#define DIAG308_PV_START            10
>  
>  static int diag308_parm_check(CPUS390XState *env, uint64_t r1,
> uint64_t addr, uintptr_t ra, bool write)
>  {
> +    /* Handled by the Ultravisor */
> +    if (s390_is_pv()) {
> +        return 0;
> +    }
>      if ((r1 & 1) || (addr & ~TARGET_PAGE_MASK)) {
>          s390_program_interrupt(env, PGM_SPECIFICATION, ra);
>          return -1;
> @@ -93,6 +102,11 @@ void handle_diag_308(CPUS390XState *env, uint64_t
> r1, uint64_t r3, uintptr_t ra) return;
>      }
>  
> +    if (subcode >= DIAG308_PV_SET &&
> !s390_has_feat(S390_FEAT_UNPACK)) {
> +        s390_program_interrupt(env, PGM_SPECIFICATION, ra);
> +        return;
> +    }
> +
>      switch (subcode) {
>      case DIAG308_RESET_MOD_CLR:
>          s390_ipl_reset_request(cs, S390_RESET_MODIFIED_CLEAR);
> @@ -105,6 +119,7 @@ void handle_diag_308(CPUS390XState *env, uint64_t
> r1, uint64_t r3, uintptr_t ra) s390_ipl_reset_request(cs,
> S390_RESET_REIPL); break;
>      case DIAG308_SET:
> +    case DIAG308_PV_SET:
>          if (diag308_parm_check(env, r1, addr, ra, false)) {
>              return;
>          }
> @@ -128,10 +143,15 @@ out:
>          g_free(iplb);
>          return;
>      case DIAG308_STORE:
> +    case DIAG308_PV_STORE:
>          if (diag308_parm_check(env, r1, addr, ra, true)) {
>              return;
>          }
> -        iplb = s390_ipl_get_iplb();
> +        if (subcode == DIAG308_PV_STORE) {
> +            iplb = s390_ipl_get_iplb_pv();
> +        } else {
> +            iplb = s390_ipl_get_iplb();
> +        }
>          if (iplb) {
>              cpu_physical_memory_write(addr, iplb,
> be32_to_cpu(iplb->len)); env->regs[r1 + 1] = DIAG_308_RC_OK;
> @@ -139,6 +159,15 @@ out:
>              env->regs[r1 + 1] = DIAG_308_RC_NO_CONF;
>          }
>          return;
> +    case DIAG308_PV_START:
> +        iplb = s390_ipl_get_iplb_pv();
> +        if (!iplb) {
> +            env->regs[r1 + 1] = DIAG_308_RC_NO_PV_CONF;
> +            return;
> +        }
> +
> +        s390_ipl_reset_request(cs, S390_RESET_PV);
> +        break;
>      default:
>          s390_program_interrupt(env, PGM_SPECIFICATION, ra);
>          break;

with the two typos fixed and the fixup:

Reviewed-by: Claudio Imbrenda <imbrenda@linux.ibm.com>

Re: [PATCH v9 03/15] s390x: protvirt: Add migration blocker

[Claudio Imbrenda]

On Wed, 11 Mar 2020 09:21:39 -0400
Janosch Frank <frankja@linux.ibm.com> wrote:

> Migration is not yet supported.
> 
> Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
> Reviewed-by: David Hildenbrand <david@redhat.com>
> ---
>  hw/s390x/s390-virtio-ccw.c | 18 ++++++++++++++++++
>  1 file changed, 18 insertions(+)
> 
> diff --git a/hw/s390x/s390-virtio-ccw.c b/hw/s390x/s390-virtio-ccw.c
> index 9569b777a0e1abd6..deb31e060052d279 100644
> --- a/hw/s390x/s390-virtio-ccw.c
> +++ b/hw/s390x/s390-virtio-ccw.c
> @@ -42,6 +42,9 @@
>  #include "hw/s390x/tod.h"
>  #include "sysemu/sysemu.h"
>  #include "hw/s390x/pv.h"
> +#include "migration/blocker.h"
> +
> +static Error *pv_mig_blocker;
>  
>  S390CPU *s390_cpu_addr2state(uint16_t cpu_addr)
>  {
> @@ -321,15 +324,30 @@ static void
> s390_machine_unprotect(S390CcwMachineState *ms) {
>      s390_pv_vm_disable();
>      ms->pv = false;
> +    migrate_del_blocker(pv_mig_blocker);
> +    error_free_or_abort(&pv_mig_blocker);
>  }
>  
>  static int s390_machine_protect(S390CcwMachineState *ms)
>  {
> +    Error *local_err = NULL;
>      int rc;
>  
> +    error_setg(&pv_mig_blocker,
> +               "protected VMs are currently not migrateable.");
> +    rc = migrate_add_blocker(pv_mig_blocker, &local_err);
> +    if (local_err) {
> +        error_report_err(local_err);
> +        error_free_or_abort(&pv_mig_blocker);
> +        return rc;
> +    }
> +
>      /* Create SE VM */
>      rc = s390_pv_vm_enable();
>      if (rc) {
> +        error_report_err(local_err);
> +        migrate_del_blocker(pv_mig_blocker);
> +        error_free_or_abort(&pv_mig_blocker);
>          return rc;
>      }
>  

looks rather straightforward

Reviewed-by: Claudio Imbrenda <imbrenda@linux.ibm.com>

Re: [PATCH v9 04/15] s390x: protvirt: Inhibit balloon when switching to protected mode

[Claudio Imbrenda]

On Wed, 11 Mar 2020 09:21:40 -0400
Janosch Frank <frankja@linux.ibm.com> wrote:

> Ballooning in protected VMs can only be done when the guest shares the
> pages it gives to the host. If pages are not shared, the integrity
> checks will fail once those pages have been altered and are given back
> to the guest.
> 
> As we currently do not yet have a solution for this we will continue
> like this:
> 
> 1. We block ballooning now in QEMU (with this patch)
> 
> 2. Later we will provide a change to virtio that removes the blocker
> and adds VIRTIO_F_IOMMU_PLATFORM automatically by QEMU when doing the
> protvirt switch. This is ok as the guest balloon driver will reject to
> work with the IOMMU change
> 
> 3. Later we can fix the guest balloon driver to accept the IOMMU
> feature bit and correctly exercise sharing and unsharing of balloon
> pages
> 
> Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
> Reviewed-by: David Hildenbrand <david@redhat.com>
> Reviewed-by: Christian Borntraeger <borntraeger@de.ibm.com>
> ---
>  hw/s390x/s390-virtio-ccw.c | 5 +++++
>  1 file changed, 5 insertions(+)
> 
> diff --git a/hw/s390x/s390-virtio-ccw.c b/hw/s390x/s390-virtio-ccw.c
> index deb31e060052d279..066e01f303c35671 100644
> --- a/hw/s390x/s390-virtio-ccw.c
> +++ b/hw/s390x/s390-virtio-ccw.c
> @@ -41,6 +41,7 @@
>  #include "hw/qdev-properties.h"
>  #include "hw/s390x/tod.h"
>  #include "sysemu/sysemu.h"
> +#include "sysemu/balloon.h"
>  #include "hw/s390x/pv.h"
>  #include "migration/blocker.h"
>  
> @@ -326,6 +327,7 @@ static void
> s390_machine_unprotect(S390CcwMachineState *ms) ms->pv = false;
>      migrate_del_blocker(pv_mig_blocker);
>      error_free_or_abort(&pv_mig_blocker);
> +    qemu_balloon_inhibit(false);
>  }
>  
>  static int s390_machine_protect(S390CcwMachineState *ms)
> @@ -333,10 +335,12 @@ static int
> s390_machine_protect(S390CcwMachineState *ms) Error *local_err = NULL;
>      int rc;
>  
> +    qemu_balloon_inhibit(true);
>      error_setg(&pv_mig_blocker,
>                 "protected VMs are currently not migrateable.");
>      rc = migrate_add_blocker(pv_mig_blocker, &local_err);
>      if (local_err) {
> +        qemu_balloon_inhibit(false);
>          error_report_err(local_err);
>          error_free_or_abort(&pv_mig_blocker);
>          return rc;
> @@ -345,6 +349,7 @@ static int
> s390_machine_protect(S390CcwMachineState *ms) /* Create SE VM */
>      rc = s390_pv_vm_enable();
>      if (rc) {
> +        qemu_balloon_inhibit(false);
>          error_report_err(local_err);
>          migrate_del_blocker(pv_mig_blocker);
>          error_free_or_abort(&pv_mig_blocker);

looks straightforward

Reviewed-by: Claudio Imbrenda <imbrenda@linux.ibm.com>

Re: [PATCH v9 05/15] s390x: protvirt: KVM intercept changes

[Claudio Imbrenda]

On Wed, 11 Mar 2020 09:21:41 -0400
Janosch Frank <frankja@linux.ibm.com> wrote:

> Protected VMs no longer intercept with code 4 for an instruction
> interception. Instead they have codes 104 and 108 for protected
> instruction interception and protected instruction notification
> respectively.
> 
> The 104 mirrors the 4 interception.
> 
> The 108 is a notification interception to let KVM and QEMU know that
> something changed and we need to update tracking information or
> perform specific tasks. It's currently taken for the following
> instructions:
> 
> * spx (To inform about the changed prefix location)
> * sclp (On incorrect SCCB values, so we can inject a IRQ)
> * sigp (All but "stop and store status")
> * diag308 (Subcodes 0/1)
> 
> Of these exits only sclp errors, state changing sigps and diag308 will
> reach QEMU. QEMU will do its parts of the job, while the ultravisor
> has done the instruction part of the job.
> 
> Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
> Reviewed-by: David Hildenbrand <david@redhat.com>
> Reviewed-by: Christian Borntraeger <borntraeger@de.ibm.com>
> ---
>  target/s390x/kvm.c | 4 ++++
>  1 file changed, 4 insertions(+)
> 
> diff --git a/target/s390x/kvm.c b/target/s390x/kvm.c
> index 1d6fd6a27b48e35f..eec0b92479465b9c 100644
> --- a/target/s390x/kvm.c
> +++ b/target/s390x/kvm.c
> @@ -115,6 +115,8 @@
>  #define ICPT_CPU_STOP                   0x28
>  #define ICPT_OPEREXC                    0x2c
>  #define ICPT_IO                         0x40
> +#define ICPT_PV_INSTR                   0x68
> +#define ICPT_PV_INSTR_NOTIFICATION      0x6c
>  
>  #define NR_LOCAL_IRQS 32
>  /*
> @@ -1693,6 +1695,8 @@ static int handle_intercept(S390CPU *cpu)
>              (long)cs->kvm_run->psw_addr);
>      switch (icpt_code) {
>          case ICPT_INSTRUCTION:
> +        case ICPT_PV_INSTR:
> +        case ICPT_PV_INSTR_NOTIFICATION:
>              r = handle_instruction(cpu, run);
>              break;
>          case ICPT_PROGRAM:

very straightforward

Reviewed-by: Claudio Imbrenda <imbrenda@linux.ibm.com>

Re: [PATCH v9 06/15] s390x: Add SIDA memory ops

[Claudio Imbrenda]

On Wed, 11 Mar 2020 09:21:42 -0400
Janosch Frank <frankja@linux.ibm.com> wrote:

> Protected guests save the instruction control blocks in the SIDA
> instead of QEMU/KVM directly accessing the guest's memory.
> 
> Let's introduce new functions to access the SIDA.
> 
> Also the new memops are available with KVM_CAP_S390_PROTECTED, so
> let's check for that.
> 
> Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
> Reviewed-by: David Hildenbrand <david@redhat.com>
> ---
>  target/s390x/cpu.h        |  7 ++++++-
>  target/s390x/kvm.c        | 25 +++++++++++++++++++++++++
>  target/s390x/kvm_s390x.h  |  2 ++
>  target/s390x/mmu_helper.c | 14 ++++++++++++++
>  4 files changed, 47 insertions(+), 1 deletion(-)
> 
> diff --git a/target/s390x/cpu.h b/target/s390x/cpu.h
> index 1d17709d6e10b5e0..035427521cec2528 100644
> --- a/target/s390x/cpu.h
> +++ b/target/s390x/cpu.h
> @@ -823,7 +823,12 @@ int s390_cpu_virt_mem_rw(S390CPU *cpu, vaddr
> laddr, uint8_t ar, void *hostbuf, #define
> s390_cpu_virt_mem_check_write(cpu, laddr, ar, len)   \
> s390_cpu_virt_mem_rw(cpu, laddr, ar, NULL, len, true) void
> s390_cpu_virt_mem_handle_exc(S390CPU *cpu, uintptr_t ra); -
> +int s390_cpu_pv_mem_rw(S390CPU *cpu, unsigned int offset, void
> *hostbuf,
> +                       int len, bool is_write);
> +#define s390_cpu_pv_mem_read(cpu, offset, dest, len)    \
> +        s390_cpu_pv_mem_rw(cpu, offset, dest, len, false)
> +#define s390_cpu_pv_mem_write(cpu, offset, dest, len)       \
> +        s390_cpu_pv_mem_rw(cpu, offset, dest, len, true)
>  
>  /* sigp.c */
>  int s390_cpu_restart(S390CPU *cpu);
> diff --git a/target/s390x/kvm.c b/target/s390x/kvm.c
> index eec0b92479465b9c..cdcd538b4f7fb318 100644
> --- a/target/s390x/kvm.c
> +++ b/target/s390x/kvm.c
> @@ -154,6 +154,7 @@ static int cap_ri;
>  static int cap_gs;
>  static int cap_hpage_1m;
>  static int cap_vcpu_resets;
> +static int cap_protected;
>  
>  static int active_cmma;
>  
> @@ -346,6 +347,7 @@ int kvm_arch_init(MachineState *ms, KVMState *s)
>      cap_mem_op = kvm_check_extension(s, KVM_CAP_S390_MEM_OP);
>      cap_s390_irq = kvm_check_extension(s, KVM_CAP_S390_INJECT_IRQ);
>      cap_vcpu_resets = kvm_check_extension(s,
> KVM_CAP_S390_VCPU_RESETS);
> +    cap_protected = kvm_check_extension(s, KVM_CAP_S390_PROTECTED);
>  
>      if (!kvm_check_extension(s, KVM_CAP_S390_GMAP)
>          || !kvm_check_extension(s, KVM_CAP_S390_COW)) {
> @@ -846,6 +848,29 @@ int kvm_s390_mem_op(S390CPU *cpu, vaddr addr,
> uint8_t ar, void *hostbuf, return ret;
>  }
>  
> +int kvm_s390_mem_op_pv(S390CPU *cpu, uint64_t offset, void *hostbuf,
> +                       int len, bool is_write)
> +{
> +    struct kvm_s390_mem_op mem_op = {
> +        .sida_offset = offset,
> +        .size = len,
> +        .op = is_write ? KVM_S390_MEMOP_SIDA_WRITE
> +                       : KVM_S390_MEMOP_SIDA_READ,
> +        .buf = (uint64_t)hostbuf,
> +    };
> +    int ret;
> +
> +    if (!cap_mem_op || !cap_protected) {
> +        return -ENOSYS;
> +    }
> +
> +    ret = kvm_vcpu_ioctl(CPU(cpu), KVM_S390_MEM_OP, &mem_op);
> +    if (ret < 0) {
> +        error_report("KVM_S390_MEM_OP failed: %s", strerror(-ret));
> +    }
> +    return ret;
> +}
> +
>  /*
>   * Legacy layout for s390:
>   * Older S390 KVM requires the topmost vma of the RAM to be
> diff --git a/target/s390x/kvm_s390x.h b/target/s390x/kvm_s390x.h
> index 0b21789796d7c462..9c38f6ccce83e39e 100644
> --- a/target/s390x/kvm_s390x.h
> +++ b/target/s390x/kvm_s390x.h
> @@ -19,6 +19,8 @@ void kvm_s390_vcpu_interrupt(S390CPU *cpu, struct
> kvm_s390_irq *irq); void kvm_s390_access_exception(S390CPU *cpu,
> uint16_t code, uint64_t te_code); int kvm_s390_mem_op(S390CPU *cpu,
> vaddr addr, uint8_t ar, void *hostbuf, int len, bool is_write);
> +int kvm_s390_mem_op_pv(S390CPU *cpu, vaddr addr, void *hostbuf, int
> len,
> +                       bool is_write);
>  void kvm_s390_program_interrupt(S390CPU *cpu, uint16_t code);
>  int kvm_s390_set_cpu_state(S390CPU *cpu, uint8_t cpu_state);
>  void kvm_s390_vcpu_interrupt_pre_save(S390CPU *cpu);
> diff --git a/target/s390x/mmu_helper.c b/target/s390x/mmu_helper.c
> index 0be2f300bbe4ac8b..7d9f3059cd502c49 100644
> --- a/target/s390x/mmu_helper.c
> +++ b/target/s390x/mmu_helper.c
> @@ -474,6 +474,20 @@ static int translate_pages(S390CPU *cpu, vaddr
> addr, int nr_pages, return 0;
>  }
>  
> +int s390_cpu_pv_mem_rw(S390CPU *cpu, unsigned int offset, void
> *hostbuf,
> +                       int len, bool is_write)
> +{
> +    int ret;
> +
> +    if (kvm_enabled()) {
> +        ret = kvm_s390_mem_op_pv(cpu, offset, hostbuf, len,
> is_write);
> +    } else {
> +        /* Protected Virtualization is a KVM/Hardware only feature */
> +        g_assert_not_reached();
> +    }
> +    return ret;
> +}
> +
>  /**
>   * s390_cpu_virt_mem_rw:
>   * @laddr:     the logical start address

Reviewed-by: Claudio Imbrenda <imbrenda@linux.ibm.com>

Re: [PATCH v9 07/15] s390x: protvirt: Move STSI data over SIDAD

[Claudio Imbrenda]

On Wed, 11 Mar 2020 09:21:43 -0400
Janosch Frank <frankja@linux.ibm.com> wrote:

> For protected guests, we need to put the STSI emulation results into
> the SIDA, so SIE will write them into the guest at the next entry.
> 
> Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
> Reviewed-by: David Hildenbrand <david@redhat.com>
> ---
>  target/s390x/kvm.c | 11 +++++++++--
>  1 file changed, 9 insertions(+), 2 deletions(-)
> 
> diff --git a/target/s390x/kvm.c b/target/s390x/kvm.c
> index cdcd538b4f7fb318..8085d5030e7c6454 100644
> --- a/target/s390x/kvm.c
> +++ b/target/s390x/kvm.c
> @@ -50,6 +50,7 @@
>  #include "exec/memattrs.h"
>  #include "hw/s390x/s390-virtio-ccw.h"
>  #include "hw/s390x/s390-virtio-hcall.h"
> +#include "hw/s390x/pv.h"
>  
>  #ifndef DEBUG_KVM
>  #define DEBUG_KVM  0
> @@ -1800,7 +1801,9 @@ static void insert_stsi_3_2_2(S390CPU *cpu,
> __u64 addr, uint8_t ar) SysIB_322 sysib;
>      int del;
>  
> -    if (s390_cpu_virt_mem_read(cpu, addr, ar, &sysib,
> sizeof(sysib))) {
> +    if (s390_is_pv()) {
> +        s390_cpu_pv_mem_read(cpu, 0, &sysib, sizeof(sysib));
> +    } else if (s390_cpu_virt_mem_read(cpu, addr, ar, &sysib,
> sizeof(sysib))) { return;
>      }
>      /* Shift the stack of Extended Names to prepare for our own data
> */ @@ -1840,7 +1843,11 @@ static void insert_stsi_3_2_2(S390CPU *cpu,
> __u64 addr, uint8_t ar) /* Insert UUID */
>      memcpy(sysib.vm[0].uuid, &qemu_uuid, sizeof(sysib.vm[0].uuid));
>  
> -    s390_cpu_virt_mem_write(cpu, addr, ar, &sysib, sizeof(sysib));
> +    if (s390_is_pv()) {
> +        s390_cpu_pv_mem_write(cpu, 0, &sysib, sizeof(sysib));
> +    } else {
> +        s390_cpu_virt_mem_write(cpu, addr, ar, &sysib,
> sizeof(sysib));
> +    }
>  }
>  
>  static int handle_stsi(S390CPU *cpu)

Reviewed-by: Claudio Imbrenda <imbrenda@linux.ibm.com>

Re: [PATCH v9 08/15] s390x: protvirt: SCLP interpretation

[Claudio Imbrenda]

On Wed, 11 Mar 2020 09:21:44 -0400
Janosch Frank <frankja@linux.ibm.com> wrote:

> SCLP for a protected guest is done over the SIDAD, so we need to use
> the s390_cpu_pv_mem_* functions to access the SIDAD instead of guest
> memory when reading/writing SCBs.
> 
> To not confuse the sclp emulation, we set 0x4000 as the SCCB address,
> since the function that injects the sclp external interrupt would
> reject a zero sccb address.
> 
> Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
> Reviewed-by: David Hildenbrand <david@redhat.com>
> ---
>  hw/s390x/sclp.c         | 30 ++++++++++++++++++++++++++++++
>  include/hw/s390x/sclp.h |  2 ++
>  target/s390x/kvm.c      | 24 +++++++++++++++++++-----
>  3 files changed, 51 insertions(+), 5 deletions(-)
> 
> diff --git a/hw/s390x/sclp.c b/hw/s390x/sclp.c
> index af0bfbc2eca74767..5f3aa30d6283dce5 100644
> --- a/hw/s390x/sclp.c
> +++ b/hw/s390x/sclp.c
> @@ -193,6 +193,36 @@ static void sclp_execute(SCLPDevice *sclp, SCCB
> *sccb, uint32_t code) }
>  }
>  
> +/*
> + * We only need the address to have something valid for the
> + * service_interrupt call.
> + */
> +#define SCLP_PV_DUMMY_ADDR 0x4000
> +int sclp_service_call_protected(CPUS390XState *env, uint64_t sccb,
> +                                uint32_t code)
> +{
> +    SCLPDevice *sclp = get_sclp_device();
> +    SCLPDeviceClass *sclp_c = SCLP_GET_CLASS(sclp);
> +    SCCB work_sccb;
> +    hwaddr sccb_len = sizeof(SCCB);
> +
> +    /*
> +     * Only a very limited amount of calls is permitted by the
> +     * Ultravisor and we support all of them, so we don't check for
> +     * them. All other specification exceptions are also interpreted
> +     * by the Ultravisor and hence never cause an exit we need to
> +     * handle.
> +     *
> +     * Setting the CC is also done by the Ultravisor.
> +     */
> +    s390_cpu_pv_mem_read(env_archcpu(env), 0, &work_sccb, sccb_len);
> +    sclp_c->execute(sclp, &work_sccb, code);
> +    s390_cpu_pv_mem_write(env_archcpu(env), 0, &work_sccb,
> +                          be16_to_cpu(work_sccb.h.length));
> +    sclp_c->service_interrupt(sclp, SCLP_PV_DUMMY_ADDR);
> +    return 0;
> +}
> +
>  int sclp_service_call(CPUS390XState *env, uint64_t sccb, uint32_t
> code) {
>      SCLPDevice *sclp = get_sclp_device();
> diff --git a/include/hw/s390x/sclp.h b/include/hw/s390x/sclp.h
> index c54413b78cf01b27..c0a3faa37d730453 100644
> --- a/include/hw/s390x/sclp.h
> +++ b/include/hw/s390x/sclp.h
> @@ -217,5 +217,7 @@ void s390_sclp_init(void);
>  void sclp_service_interrupt(uint32_t sccb);
>  void raise_irq_cpu_hotplug(void);
>  int sclp_service_call(CPUS390XState *env, uint64_t sccb, uint32_t
> code); +int sclp_service_call_protected(CPUS390XState *env, uint64_t
> sccb,
> +                                uint32_t code);
>  
>  #endif
> diff --git a/target/s390x/kvm.c b/target/s390x/kvm.c
> index 8085d5030e7c6454..ff6027036ec2f14a 100644
> --- a/target/s390x/kvm.c
> +++ b/target/s390x/kvm.c
> @@ -1227,12 +1227,26 @@ static void kvm_sclp_service_call(S390CPU
> *cpu, struct kvm_run *run, sccb = env->regs[ipbh0 & 0xf];
>      code = env->regs[(ipbh0 & 0xf0) >> 4];
>  
> -    r = sclp_service_call(env, sccb, code);
> -    if (r < 0) {
> -        kvm_s390_program_interrupt(cpu, -r);
> -        return;
> +    switch (run->s390_sieic.icptcode) {
> +    case ICPT_PV_INSTR_NOTIFICATION:
> +        g_assert(s390_is_pv());
> +        /* The notification intercepts are currently handled by KVM
> */
> +        error_report("unexpected SCLP PV notification");
> +        exit(1);
> +        break;
> +    case ICPT_PV_INSTR:
> +        g_assert(s390_is_pv());
> +        sclp_service_call_protected(env, sccb, code);
> +        break;
> +    case ICPT_INSTRUCTION:
> +        g_assert(!s390_is_pv());
> +        r = sclp_service_call(env, sccb, code);
> +        if (r < 0) {
> +            kvm_s390_program_interrupt(cpu, -r);
> +            return;
> +        }
> +        setcc(cpu, r);
>      }
> -    setcc(cpu, r);
>  }
>  
>  static int handle_b2(S390CPU *cpu, struct kvm_run *run, uint8_t ipa1)


Reviewed-by: Claudio Imbrenda <imbrenda@linux.ibm.com>

Re: [PATCH v9 09/15] s390x: protvirt: Set guest IPL PSW

[Claudio Imbrenda]

On Wed, 11 Mar 2020 09:21:45 -0400
Janosch Frank <frankja@linux.ibm.com> wrote:

> Handling of CPU reset and setting of the IPL psw from guest storage at
> offset 0 is done by a Ultravisor call. Let's only fetch it if
> necessary.
> 
> Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
> Reviewed-by: Thomas Huth <thuth@redhat.com>
> Reviewed-by: David Hildenbrand <david@redhat.com>
> ---
>  target/s390x/cpu.c | 22 +++++++++++++---------
>  1 file changed, 13 insertions(+), 9 deletions(-)
> 
> diff --git a/target/s390x/cpu.c b/target/s390x/cpu.c
> index 84029f14814b4980..a48d39f139cdc1c4 100644
> --- a/target/s390x/cpu.c
> +++ b/target/s390x/cpu.c
> @@ -78,16 +78,20 @@ static bool s390_cpu_has_work(CPUState *cs)
>  static void s390_cpu_load_normal(CPUState *s)
>  {
>      S390CPU *cpu = S390_CPU(s);
> -    uint64_t spsw = ldq_phys(s->as, 0);
> -
> -    cpu->env.psw.mask = spsw & PSW_MASK_SHORT_CTRL;
> -    /*
> -     * Invert short psw indication, so SIE will report a
> specification
> -     * exception if it was not set.
> -     */
> -    cpu->env.psw.mask ^= PSW_MASK_SHORTPSW;
> -    cpu->env.psw.addr = spsw & PSW_MASK_SHORT_ADDR;
> +    uint64_t spsw;
>  
> +    if (!s390_is_pv()) {
> +        spsw = ldq_phys(s->as, 0);
> +        cpu->env.psw.mask = spsw & PSW_MASK_SHORT_CTRL;
> +        /*
> +         * Invert short psw indication, so SIE will report a
> specification
> +         * exception if it was not set.
> +         */
> +        cpu->env.psw.mask ^= PSW_MASK_SHORTPSW;
> +        cpu->env.psw.addr = spsw & PSW_MASK_SHORT_ADDR;
> +    } else {
> +        s390_cpu_set_state(S390_CPU_STATE_LOAD, cpu);
> +    }
>      s390_cpu_set_state(S390_CPU_STATE_OPERATING, cpu);
>  }
>  #endif

I don't understand why you set the state to S390_CPU_STATE_LOAD and
then immediately afterwards to S390_CPU_STATE_OPERATING, especially
considering that both do the same

Re: [PATCH v9 10/15] s390x: protvirt: Move diag 308 data over SIDA

[Claudio Imbrenda]

On Wed, 11 Mar 2020 09:21:46 -0400
Janosch Frank <frankja@linux.ibm.com> wrote:

> For protected guests the IPIB is written/read to/from the SIDA, so we
> need those accesses to go through s390_cpu_pv_mem_read/write().
> 
> Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
> Reviewed-by: David Hildenbrand <david@redhat.com>
> ---
>  target/s390x/diag.c | 27 +++++++++++++++++++++------
>  1 file changed, 21 insertions(+), 6 deletions(-)
> 
> diff --git a/target/s390x/diag.c b/target/s390x/diag.c
> index b245e557037ded06..a733485caf162111 100644
> --- a/target/s390x/diag.c
> +++ b/target/s390x/diag.c
> @@ -88,6 +88,7 @@ static int diag308_parm_check(CPUS390XState *env,
> uint64_t r1, uint64_t addr, void handle_diag_308(CPUS390XState *env,
> uint64_t r1, uint64_t r3, uintptr_t ra) {
>      CPUState *cs = env_cpu(env);
> +    S390CPU *cpu = S390_CPU(cs);
>      uint64_t addr =  env->regs[r1];
>      uint64_t subcode = env->regs[r3];
>      IplParameterBlock *iplb;
> @@ -124,13 +125,22 @@ void handle_diag_308(CPUS390XState *env,
> uint64_t r1, uint64_t r3, uintptr_t ra) return;
>          }
>          iplb = g_new0(IplParameterBlock, 1);
> -        cpu_physical_memory_read(addr, iplb, sizeof(iplb->len));
> +        if (!s390_is_pv()) {
> +            cpu_physical_memory_read(addr, iplb, sizeof(iplb->len));
> +        } else {
> +            s390_cpu_pv_mem_read(cpu, 0, iplb, sizeof(iplb->len));
> +        }
> +
>          if (!iplb_valid_len(iplb)) {
>              env->regs[r1 + 1] = DIAG_308_RC_INVALID;
>              goto out;
>          }
>  
> -        cpu_physical_memory_read(addr, iplb, be32_to_cpu(iplb->len));
> +        if (!s390_is_pv()) {
> +            cpu_physical_memory_read(addr, iplb,
> be32_to_cpu(iplb->len));
> +        } else {
> +            s390_cpu_pv_mem_read(cpu, 0, iplb,
> be32_to_cpu(iplb->len));
> +        }
>  
>          if (!iplb_valid(iplb)) {
>              env->regs[r1 + 1] = DIAG_308_RC_INVALID;
> @@ -152,12 +162,17 @@ out:
>          } else {
>              iplb = s390_ipl_get_iplb();
>          }
> -        if (iplb) {
> -            cpu_physical_memory_write(addr, iplb,
> be32_to_cpu(iplb->len));
> -            env->regs[r1 + 1] = DIAG_308_RC_OK;
> -        } else {
> +        if (!iplb) {
>              env->regs[r1 + 1] = DIAG_308_RC_NO_CONF;
> +            return;
>          }
> +
> +        if (!s390_is_pv()) {
> +            cpu_physical_memory_write(addr, iplb,
> be32_to_cpu(iplb->len));
> +        } else {
> +            s390_cpu_pv_mem_write(cpu, 0, iplb,
> be32_to_cpu(iplb->len));
> +        }
> +        env->regs[r1 + 1] = DIAG_308_RC_OK;
>          return;
>      case DIAG308_PV_START:
>          iplb = s390_ipl_get_iplb_pv();

Reviewed-by: Claudio Imbrenda <imbrenda@linux.ibm.com>

Re: [PATCH v9 11/15] s390x: protvirt: Disable address checks for PV guest IO emulation

[Claudio Imbrenda]

On Wed, 11 Mar 2020 09:21:47 -0400
Janosch Frank <frankja@linux.ibm.com> wrote:

> IO instruction data is routed through SIDAD for protected guests, so
> adresses do not need to be checked, as this is kernel memory.
> 
> Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
> Reviewed-by: Thomas Huth <thuth@redhat.com>
> Reviewed-by: David Hildenbrand <david@redhat.com>
> ---
>  target/s390x/ioinst.c | 33 ++++++++++++++++++++++++++-------
>  1 file changed, 26 insertions(+), 7 deletions(-)
> 
> diff --git a/target/s390x/ioinst.c b/target/s390x/ioinst.c
> index c437a1d8c6afed80..481d789de9e09a04 100644
> --- a/target/s390x/ioinst.c
> +++ b/target/s390x/ioinst.c
> @@ -16,6 +16,23 @@
>  #include "hw/s390x/ioinst.h"
>  #include "trace.h"
>  #include "hw/s390x/s390-pci-bus.h"
> +#include "hw/s390x/pv.h"
> +
> +static uint64_t get_address_from_regs(CPUS390XState *env, uint32_t
> ipb,
> +                                      uint8_t *ar)
> +{
> +    /*
> +     * Addresses for protected guests are all offsets into the
> +     * satellite block which holds the IO control structures. Those
> +     * control structures are always aligned and accessible, so we
> can
> +     * return 0 here which will pass the following address checks.
> +     */
> +    if (s390_is_pv()) {
> +        *ar = 0;
> +        return 0;
> +    }
> +    return decode_basedisp_s(env, ipb, ar);
> +}
>  
>  int ioinst_disassemble_sch_ident(uint32_t value, int *m, int *cssid,
> int *ssid, int *schid)
> @@ -114,7 +131,7 @@ void ioinst_handle_msch(S390CPU *cpu, uint64_t
> reg1, uint32_t ipb, uintptr_t ra) CPUS390XState *env = &cpu->env;
>      uint8_t ar;
>  
> -    addr = decode_basedisp_s(env, ipb, &ar);
> +    addr = get_address_from_regs(env, ipb, &ar);
>      if (addr & 3) {
>          s390_program_interrupt(env, PGM_SPECIFICATION, ra);
>          return;
> @@ -171,7 +188,7 @@ void ioinst_handle_ssch(S390CPU *cpu, uint64_t
> reg1, uint32_t ipb, uintptr_t ra) CPUS390XState *env = &cpu->env;
>      uint8_t ar;
>  
> -    addr = decode_basedisp_s(env, ipb, &ar);
> +    addr = get_address_from_regs(env, ipb, &ar);
>      if (addr & 3) {
>          s390_program_interrupt(env, PGM_SPECIFICATION, ra);
>          return;
> @@ -203,7 +220,7 @@ void ioinst_handle_stcrw(S390CPU *cpu, uint32_t
> ipb, uintptr_t ra) CPUS390XState *env = &cpu->env;
>      uint8_t ar;
>  
> -    addr = decode_basedisp_s(env, ipb, &ar);
> +    addr = get_address_from_regs(env, ipb, &ar);
>      if (addr & 3) {
>          s390_program_interrupt(env, PGM_SPECIFICATION, ra);
>          return;
> @@ -234,7 +251,7 @@ void ioinst_handle_stsch(S390CPU *cpu, uint64_t
> reg1, uint32_t ipb, CPUS390XState *env = &cpu->env;
>      uint8_t ar;
>  
> -    addr = decode_basedisp_s(env, ipb, &ar);
> +    addr = get_address_from_regs(env, ipb, &ar);
>      if (addr & 3) {
>          s390_program_interrupt(env, PGM_SPECIFICATION, ra);
>          return;
> @@ -303,7 +320,7 @@ int ioinst_handle_tsch(S390CPU *cpu, uint64_t
> reg1, uint32_t ipb, uintptr_t ra) return -EIO;
>      }
>      trace_ioinst_sch_id("tsch", cssid, ssid, schid);
> -    addr = decode_basedisp_s(env, ipb, &ar);
> +    addr = get_address_from_regs(env, ipb, &ar);
>      if (addr & 3) {
>          s390_program_interrupt(env, PGM_SPECIFICATION, ra);
>          return -EIO;
> @@ -601,7 +618,7 @@ void ioinst_handle_chsc(S390CPU *cpu, uint32_t
> ipb, uintptr_t ra) {
>      ChscReq *req;
>      ChscResp *res;
> -    uint64_t addr;
> +    uint64_t addr = 0;
>      int reg;
>      uint16_t len;
>      uint16_t command;
> @@ -610,7 +627,9 @@ void ioinst_handle_chsc(S390CPU *cpu, uint32_t
> ipb, uintptr_t ra) 
>      trace_ioinst("chsc");
>      reg = (ipb >> 20) & 0x00f;
> -    addr = env->regs[reg];
> +    if (!s390_is_pv()) {
> +        addr = env->regs[reg];
> +    }
>      /* Page boundary? */
>      if (addr & 0xfff) {
>          s390_program_interrupt(env, PGM_SPECIFICATION, ra);

Reviewed-by: Claudio Imbrenda <imbrenda@linux.ibm.com>

Re: [PATCH v9 12/15] s390x: protvirt: Move IO control structures over SIDA

[Claudio Imbrenda]

On Wed, 11 Mar 2020 09:21:48 -0400
Janosch Frank <frankja@linux.ibm.com> wrote:

> For protected guests, we need to put the IO emulation results into the
> SIDA, so SIE will write them into the guest at the next entry.
> 
> Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
> ---
>  target/s390x/ioinst.c | 87
> ++++++++++++++++++++++++++++++------------- 1 file changed, 61
> insertions(+), 26 deletions(-)
> 
> diff --git a/target/s390x/ioinst.c b/target/s390x/ioinst.c
> index 481d789de9e09a04..61095bdc9ffef436 100644
> --- a/target/s390x/ioinst.c
> +++ b/target/s390x/ioinst.c
> @@ -136,9 +136,13 @@ void ioinst_handle_msch(S390CPU *cpu, uint64_t
> reg1, uint32_t ipb, uintptr_t ra) s390_program_interrupt(env,
> PGM_SPECIFICATION, ra); return;
>      }
> -    if (s390_cpu_virt_mem_read(cpu, addr, ar, &schib,
> sizeof(schib))) {
> -        s390_cpu_virt_mem_handle_exc(cpu, ra);
> -        return;
> +    if (s390_is_pv()) {
> +        s390_cpu_pv_mem_read(cpu, addr, &schib, sizeof(schib));
> +    } else {
> +        if (s390_cpu_virt_mem_read(cpu, addr, ar, &schib,
> sizeof(schib))) {
> +            s390_cpu_virt_mem_handle_exc(cpu, ra);
> +            return;
> +        }
>      }
>      if (ioinst_disassemble_sch_ident(reg1, &m, &cssid, &ssid,
> &schid) || !ioinst_schib_valid(&schib)) {
> @@ -193,9 +197,13 @@ void ioinst_handle_ssch(S390CPU *cpu, uint64_t
> reg1, uint32_t ipb, uintptr_t ra) s390_program_interrupt(env,
> PGM_SPECIFICATION, ra); return;
>      }
> -    if (s390_cpu_virt_mem_read(cpu, addr, ar, &orig_orb,
> sizeof(orb))) {
> -        s390_cpu_virt_mem_handle_exc(cpu, ra);
> -        return;
> +    if (s390_is_pv()) {
> +        s390_cpu_pv_mem_read(cpu, addr, &orig_orb, sizeof(orb));
> +    } else {
> +        if (s390_cpu_virt_mem_read(cpu, addr, ar, &orig_orb,
> sizeof(orb))) {
> +            s390_cpu_virt_mem_handle_exc(cpu, ra);
> +            return;
> +        }
>      }
>      copy_orb_from_guest(&orb, &orig_orb);
>      if (ioinst_disassemble_sch_ident(reg1, &m, &cssid, &ssid,
> &schid) || @@ -229,14 +237,19 @@ void ioinst_handle_stcrw(S390CPU
> *cpu, uint32_t ipb, uintptr_t ra) cc = css_do_stcrw(&crw);
>      /* 0 - crw stored, 1 - zeroes stored */
>  
> -    if (s390_cpu_virt_mem_write(cpu, addr, ar, &crw, sizeof(crw)) ==
> 0) {
> +    if (s390_is_pv()) {
> +        s390_cpu_pv_mem_write(cpu, addr, &crw, sizeof(crw));
>          setcc(cpu, cc);
>      } else {
> -        if (cc == 0) {
> -            /* Write failed: requeue CRW since STCRW is suppressing
> */
> -            css_undo_stcrw(&crw);
> +        if (s390_cpu_virt_mem_write(cpu, addr, ar, &crw,
> sizeof(crw)) == 0) {
> +            setcc(cpu, cc);
> +        } else {
> +            if (cc == 0) {
> +                /* Write failed: requeue CRW since STCRW is
> suppressing */
> +                css_undo_stcrw(&crw);
> +            }
> +            s390_cpu_virt_mem_handle_exc(cpu, ra);
>          }
> -        s390_cpu_virt_mem_handle_exc(cpu, ra);
>      }
>  }
>  
> @@ -258,6 +271,9 @@ void ioinst_handle_stsch(S390CPU *cpu, uint64_t
> reg1, uint32_t ipb, }
>  
>      if (ioinst_disassemble_sch_ident(reg1, &m, &cssid, &ssid,
> &schid)) {
> +        if (s390_is_pv()) {
> +            return;
> +        }

If the operand is invalid, should we not inject an operand exception?

If instead the Ultravisor or KVM make sure that we don't end up here,
then, why handle this at all?

>          /*
>           * As operand exceptions have a lower priority than access
> exceptions,
>           * we check whether the memory area is writeable (injecting
> the @@ -290,14 +306,19 @@ void ioinst_handle_stsch(S390CPU *cpu,
> uint64_t reg1, uint32_t ipb, }
>      }
>      if (cc != 3) {
> -        if (s390_cpu_virt_mem_write(cpu, addr, ar, &schib,
> -                                    sizeof(schib)) != 0) {
> -            s390_cpu_virt_mem_handle_exc(cpu, ra);
> -            return;
> +        if (s390_is_pv()) {
> +            s390_cpu_pv_mem_write(cpu, addr, &schib, sizeof(schib));
> +        } else {
> +            if (s390_cpu_virt_mem_write(cpu, addr, ar, &schib,
> +                                        sizeof(schib)) != 0) {
> +                s390_cpu_virt_mem_handle_exc(cpu, ra);
> +                return;
> +            }
>          }
>      } else {
>          /* Access exceptions have a higher priority than cc3 */
> -        if (s390_cpu_virt_mem_check_write(cpu, addr, ar,
> sizeof(schib)) != 0) {
> +        if (!s390_is_pv() &&
> +            s390_cpu_virt_mem_check_write(cpu, addr, ar,
> sizeof(schib)) != 0) { s390_cpu_virt_mem_handle_exc(cpu, ra);
>              return;
>          }
> @@ -334,15 +355,20 @@ int ioinst_handle_tsch(S390CPU *cpu, uint64_t
> reg1, uint32_t ipb, uintptr_t ra) }
>      /* 0 - status pending, 1 - not status pending, 3 - not
> operational */ if (cc != 3) {
> -        if (s390_cpu_virt_mem_write(cpu, addr, ar, &irb, irb_len) !=
> 0) {
> -            s390_cpu_virt_mem_handle_exc(cpu, ra);
> -            return -EFAULT;
> +        if (s390_is_pv()) {
> +            s390_cpu_pv_mem_write(cpu, addr, &irb, irb_len);
> +        } else {
> +            if (s390_cpu_virt_mem_write(cpu, addr, ar, &irb,
> irb_len) != 0) {
> +                s390_cpu_virt_mem_handle_exc(cpu, ra);
> +                return -EFAULT;
> +            }
>          }
>          css_do_tsch_update_subch(sch);
>      } else {
>          irb_len = sizeof(irb) - sizeof(irb.emw);
>          /* Access exceptions have a higher priority than cc3 */
> -        if (s390_cpu_virt_mem_check_write(cpu, addr, ar, irb_len) !=
> 0) {
> +        if (!s390_is_pv() &&
> +            s390_cpu_virt_mem_check_write(cpu, addr, ar, irb_len) !=
> 0) { s390_cpu_virt_mem_handle_exc(cpu, ra);
>              return -EFAULT;
>          }
> @@ -640,9 +666,13 @@ void ioinst_handle_chsc(S390CPU *cpu, uint32_t
> ipb, uintptr_t ra)
>       * present CHSC sub-handlers ... if we ever need more, we should
> take
>       * care of req->len here first.
>       */
> -    if (s390_cpu_virt_mem_read(cpu, addr, reg, buf,
> sizeof(ChscReq))) {
> -        s390_cpu_virt_mem_handle_exc(cpu, ra);
> -        return;
> +    if (s390_is_pv()) {
> +        s390_cpu_pv_mem_read(cpu, addr, buf, sizeof(ChscReq));
> +    } else {
> +        if (s390_cpu_virt_mem_read(cpu, addr, reg, buf,
> sizeof(ChscReq))) {
> +            s390_cpu_virt_mem_handle_exc(cpu, ra);
> +            return;
> +        }
>      }
>      req = (ChscReq *)buf;
>      len = be16_to_cpu(req->len);
> @@ -673,11 +703,16 @@ void ioinst_handle_chsc(S390CPU *cpu, uint32_t
> ipb, uintptr_t ra) break;
>      }
>  
> -    if (!s390_cpu_virt_mem_write(cpu, addr + len, reg, res,
> -                                 be16_to_cpu(res->len))) {
> +    if (s390_is_pv()) {
> +        s390_cpu_pv_mem_write(cpu, addr + len, res,
> be16_to_cpu(res->len)); setcc(cpu, 0);    /* Command execution
> complete */ } else {
> -        s390_cpu_virt_mem_handle_exc(cpu, ra);
> +        if (!s390_cpu_virt_mem_write(cpu, addr + len, reg, res,
> +                                     be16_to_cpu(res->len))) {
> +            setcc(cpu, 0);    /* Command execution complete */
> +        } else {
> +            s390_cpu_virt_mem_handle_exc(cpu, ra);
> +        }
>      }
>  }
>  

Re: [PATCH v9 13/15] s390x: protvirt: Handle SIGP store status correctly

[Claudio Imbrenda]

On Wed, 11 Mar 2020 09:21:49 -0400
Janosch Frank <frankja@linux.ibm.com> wrote:

> For protected VMs status storing is not done by QEMU anymore.
> 
> Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
> Reviewed-by: Thomas Huth <thuth@redhat.com>
> Reviewed-by: David Hildenbrand <david@redhat.com>
> ---
>  target/s390x/helper.c | 6 ++++++
>  1 file changed, 6 insertions(+)
> 
> diff --git a/target/s390x/helper.c b/target/s390x/helper.c
> index ed726849114f2f35..5022df8812d406c9 100644
> --- a/target/s390x/helper.c
> +++ b/target/s390x/helper.c
> @@ -25,6 +25,7 @@
>  #include "qemu/timer.h"
>  #include "qemu/qemu-print.h"
>  #include "hw/s390x/ioinst.h"
> +#include "hw/s390x/pv.h"
>  #include "sysemu/hw_accel.h"
>  #include "sysemu/runstate.h"
>  #ifndef CONFIG_USER_ONLY
> @@ -246,6 +247,11 @@ int s390_store_status(S390CPU *cpu, hwaddr addr,
> bool store_arch) hwaddr len = sizeof(*sa);
>      int i;
>  
> +    /* Storing will occur on next SIE entry for protected VMs */
> +    if (s390_is_pv()) {
> +        return 0;
> +    }
> +
>      sa = cpu_physical_memory_map(addr, &len, true);
>      if (!sa) {
>          return -EFAULT;

Reviewed-by: Claudio Imbrenda <imbrenda@linux.ibm.com>

Re: [PATCH v9 14/15] docs: Add protvirt docs

[Claudio Imbrenda]

On Wed, 11 Mar 2020 09:21:50 -0400
Janosch Frank <frankja@linux.ibm.com> wrote:

> Lets add some documentation for the Protected VM functionality.
> 
> Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
> ---
>  docs/system/index.rst    |  1 +
>  docs/system/protvirt.rst | 56
> ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 57
> insertions(+) create mode 100644 docs/system/protvirt.rst
> 
> diff --git a/docs/system/index.rst b/docs/system/index.rst
> index 6e5f20fa1333ce23..74afbd7cc3fc0296 100644
> --- a/docs/system/index.rst
> +++ b/docs/system/index.rst
> @@ -34,3 +34,4 @@ Contents:
>     deprecated
>     build-platforms
>     license
> +   protvirt
> diff --git a/docs/system/protvirt.rst b/docs/system/protvirt.rst
> new file mode 100644
> index 0000000000000000..6c8cf0f7910eae86
> --- /dev/null
> +++ b/docs/system/protvirt.rst
> @@ -0,0 +1,56 @@
> +Protected Virtualization on s390x
> +=================================
> +
> +The memory and most of the registers of Protected Virtual Machines
> +(PVMs) are encrypted or inaccessible to the hypervisor, effectively
> +prohibiting VM introspection when the VM is running. At rest, PVMs
> are +encrypted and can only be decrypted by the firmware, represented
> by an +entity called Ultravisor, of specific IBM Z machines.
> +
> +
> +Prerequisites
> +-------------
> +
> +To run PVMs a machine with the Protected Virtualization feature
> +which is indicated by the Ultravisor Call facility (stfle bit
> +158) is required. The Ultravisor needs to be initialized at boot by
> +setting `prot_virt=1` on the kernel command line.

I'd add "of the host" just to make it extra clear

> +
> +If those requirements are met, the capability
> `KVM_CAP_S390_PROTECTED` +will indicate that KVM can support PVMs on
> that LPAR. +
> +
> +QEMU Settings
> +-------------
> +
> +To indicate to the VM that it can transition into protected mode, the
> +`Unpack facility` (stfle bit 161 represented by the feature
> +`S390_FEAT_UNPACK`) needs to be part of the cpu model of the VM.
> +
> +All I/O devices need to use the IOMMU.
> +Passthrough (vfio) devices are currently not supported.
> +
> +Host huge page backings are not supported. However guests can use
> huge +pages as indicated by its facilities.
> +
> +
> +Boot Process
> +------------
> +
> +A secure guest image can either be loaded from disk or supplied on
> the +QEMU command line. Booting from disk is done by the unmodified
> +s390-ccw BIOS. I.e., the bootmap is interpreted, multiple components
> +are read into memory and control is transferred to one of the
> +components (zipl stage3). Stag3 does some fixups and then transfers
> +control to some program residing in guest memory, which is normally
> +the OS kernel. The secure image has another component prepended
> +(stage3a) that uses the new diag308 subcodes 8 and 10 to trigger the
> +transition into secure mode.
> +
> +Booting from the image supplied via the QEMU command line requires
> +that the file passed via -kernel has the same memory layout as would
> +result from the disk boot. This memory layout includes the encrypted
> +components (kernel, initrd, cmdline), the stage3a loader and
> +metadata. In case this boot method is used, the command line
> +options -initrd and -cmdline are ineffective. The preparation of a
> PVM +image is done by genprotimg of the s390-tools package.

Reviewed-by: Claudio Imbrenda <imbrenda@linux.ibm.com>

Re: [PATCH v9 15/15] s390x: Add unpack facility feature to GA1

[Claudio Imbrenda]

On Wed, 11 Mar 2020 09:21:51 -0400
Janosch Frank <frankja@linux.ibm.com> wrote:

> From: Christian Borntraeger <borntraeger@de.ibm.com>
> 
> The unpack facility is an indication that diagnose 308 subcodes 8-10
> are available to the guest. That means, that the guest can put itself
> into protected mode.
> 
> Once it is in protected mode, the hardware stops any attempt of VM
> introspection by the hypervisor.
> 
> Some features are currently not supported in protected mode:
>      * Passthrough devices
>      * Migration
>      * Huge page backings
> 
> Signed-off-by: Christian Borntraeger <borntraeger@de.ibm.com>
> Reviewed-by: David Hildenbrand <david@redhat.com>
> ---
>  target/s390x/gen-features.c | 1 +
>  target/s390x/kvm.c          | 5 +++++
>  2 files changed, 6 insertions(+)
> 
> diff --git a/target/s390x/gen-features.c b/target/s390x/gen-features.c
> index 6278845b12b8dee8..8ddeebc54419a3e2 100644
> --- a/target/s390x/gen-features.c
> +++ b/target/s390x/gen-features.c
> @@ -562,6 +562,7 @@ static uint16_t full_GEN15_GA1[] = {
>      S390_FEAT_GROUP_MSA_EXT_9,
>      S390_FEAT_GROUP_MSA_EXT_9_PCKMO,
>      S390_FEAT_ETOKEN,
> +    S390_FEAT_UNPACK,
>  };
>  
>  /* Default features (in order of release)
> diff --git a/target/s390x/kvm.c b/target/s390x/kvm.c
> index ff6027036ec2f14a..e11e895a3d9038bb 100644
> --- a/target/s390x/kvm.c
> +++ b/target/s390x/kvm.c
> @@ -2403,6 +2403,11 @@ void kvm_s390_get_host_cpu_model(S390CPUModel
> *model, Error **errp) clear_bit(S390_FEAT_BPB, model->features);
>      }
>  
> +    /* we do have the IPL enhancements */
> +    if (cap_protected) {
> +        set_bit(S390_FEAT_UNPACK, model->features);
> +    }
> +
>      /* We emulate a zPCI bus and AEN, therefore we don't need HW
> support */ set_bit(S390_FEAT_ZPCI, model->features);
>      set_bit(S390_FEAT_ADAPTER_EVENT_NOTIFICATION, model->features);

Reviewed-by: Claudio Imbrenda <imbrenda@linux.ibm.com>

Re: [PATCH v9 08/15] s390x: protvirt: SCLP interpretation

[Christian Borntraeger]

On 11.03.20 14:21, Janosch Frank wrote:
> SCLP for a protected guest is done over the SIDAD, so we need to use
> the s390_cpu_pv_mem_* functions to access the SIDAD instead of guest
> memory when reading/writing SCBs.
> 
> To not confuse the sclp emulation, we set 0x4000 as the SCCB address,
> since the function that injects the sclp external interrupt would
> reject a zero sccb address.
> 
> Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
> Reviewed-by: David Hildenbrand <david@redhat.com>
> ---
>  hw/s390x/sclp.c         | 30 ++++++++++++++++++++++++++++++
>  include/hw/s390x/sclp.h |  2 ++
>  target/s390x/kvm.c      | 24 +++++++++++++++++++-----
>  3 files changed, 51 insertions(+), 5 deletions(-)
> 
> diff --git a/hw/s390x/sclp.c b/hw/s390x/sclp.c
> index af0bfbc2eca74767..5f3aa30d6283dce5 100644
> --- a/hw/s390x/sclp.c
> +++ b/hw/s390x/sclp.c
> @@ -193,6 +193,36 @@ static void sclp_execute(SCLPDevice *sclp, SCCB *sccb, uint32_t code)
>      }
>  }
>  
> +/*
> + * We only need the address to have something valid for the
> + * service_interrupt call.
> + */
> +#define SCLP_PV_DUMMY_ADDR 0x4000
> +int sclp_service_call_protected(CPUS390XState *env, uint64_t sccb,
> +                                uint32_t code)
> +{
> +    SCLPDevice *sclp = get_sclp_device();
> +    SCLPDeviceClass *sclp_c = SCLP_GET_CLASS(sclp);
> +    SCCB work_sccb;
> +    hwaddr sccb_len = sizeof(SCCB);
> +
> +    /*
> +     * Only a very limited amount of calls is permitted by the
> +     * Ultravisor and we support all of them, so we don't check for
> +     * them. All other specification exceptions are also interpreted
> +     * by the Ultravisor and hence never cause an exit we need to
> +     * handle.
> +     *
> +     * Setting the CC is also done by the Ultravisor.
> +     */

This is fine for the current architecture which specifies a list of sclp 
commands that are passed through (and this is fine). Question is still if
we replace this comment with an assertion that this is the case?
Or maybe even really do the same as sclp_service_call and return 0x1f0 for
unknown commands?

Anyway, whatever you decide.

Reviewed-by: Christian Borntraeger <borntraeger@de.ibm.com>

> +    s390_cpu_pv_mem_read(env_archcpu(env), 0, &work_sccb, sccb_len);
> +    sclp_c->execute(sclp, &work_sccb, code);
> +    s390_cpu_pv_mem_write(env_archcpu(env), 0, &work_sccb,
> +                          be16_to_cpu(work_sccb.h.length));
> +    sclp_c->service_interrupt(sclp, SCLP_PV_DUMMY_ADDR);
> +    return 0;
> +}
> +
>  int sclp_service_call(CPUS390XState *env, uint64_t sccb, uint32_t code)
>  {
>      SCLPDevice *sclp = get_sclp_device();
> diff --git a/include/hw/s390x/sclp.h b/include/hw/s390x/sclp.h
> index c54413b78cf01b27..c0a3faa37d730453 100644
> --- a/include/hw/s390x/sclp.h
> +++ b/include/hw/s390x/sclp.h
> @@ -217,5 +217,7 @@ void s390_sclp_init(void);
>  void sclp_service_interrupt(uint32_t sccb);
>  void raise_irq_cpu_hotplug(void);
>  int sclp_service_call(CPUS390XState *env, uint64_t sccb, uint32_t code);
> +int sclp_service_call_protected(CPUS390XState *env, uint64_t sccb,
> +                                uint32_t code);
>  
>  #endif
> diff --git a/target/s390x/kvm.c b/target/s390x/kvm.c
> index 8085d5030e7c6454..ff6027036ec2f14a 100644
> --- a/target/s390x/kvm.c
> +++ b/target/s390x/kvm.c
> @@ -1227,12 +1227,26 @@ static void kvm_sclp_service_call(S390CPU *cpu, struct kvm_run *run,
>      sccb = env->regs[ipbh0 & 0xf];
>      code = env->regs[(ipbh0 & 0xf0) >> 4];
>  
> -    r = sclp_service_call(env, sccb, code);
> -    if (r < 0) {
> -        kvm_s390_program_interrupt(cpu, -r);
> -        return;
> +    switch (run->s390_sieic.icptcode) {
> +    case ICPT_PV_INSTR_NOTIFICATION:
> +        g_assert(s390_is_pv());
> +        /* The notification intercepts are currently handled by KVM */
> +        error_report("unexpected SCLP PV notification");
> +        exit(1);
> +        break;
> +    case ICPT_PV_INSTR:
> +        g_assert(s390_is_pv());
> +        sclp_service_call_protected(env, sccb, code);
> +        break;
> +    case ICPT_INSTRUCTION:
> +        g_assert(!s390_is_pv());
> +        r = sclp_service_call(env, sccb, code);
> +        if (r < 0) {
> +            kvm_s390_program_interrupt(cpu, -r);
> +            return;
> +        }
> +        setcc(cpu, r);
>      }
> -    setcc(cpu, r);
>  }
>  
>  static int handle_b2(S390CPU *cpu, struct kvm_run *run, uint8_t ipa1)
> 

Re: [PATCH v9 12/15] s390x: protvirt: Move IO control structures over SIDA

[Janosch Frank]

[-- Attachment #1.1: Type: text/plain, Size: 8073 bytes --]

On 3/13/20 1:57 PM, Claudio Imbrenda wrote:
> On Wed, 11 Mar 2020 09:21:48 -0400
> Janosch Frank <frankja@linux.ibm.com> wrote:
> 
>> For protected guests, we need to put the IO emulation results into the
>> SIDA, so SIE will write them into the guest at the next entry.
>>
>> Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
>> ---
>>  target/s390x/ioinst.c | 87
>> ++++++++++++++++++++++++++++++------------- 1 file changed, 61
>> insertions(+), 26 deletions(-)
>>
>> diff --git a/target/s390x/ioinst.c b/target/s390x/ioinst.c
>> index 481d789de9e09a04..61095bdc9ffef436 100644
>> --- a/target/s390x/ioinst.c
>> +++ b/target/s390x/ioinst.c
>> @@ -136,9 +136,13 @@ void ioinst_handle_msch(S390CPU *cpu, uint64_t
>> reg1, uint32_t ipb, uintptr_t ra) s390_program_interrupt(env,
>> PGM_SPECIFICATION, ra); return;
>>      }
>> -    if (s390_cpu_virt_mem_read(cpu, addr, ar, &schib,
>> sizeof(schib))) {
>> -        s390_cpu_virt_mem_handle_exc(cpu, ra);
>> -        return;
>> +    if (s390_is_pv()) {
>> +        s390_cpu_pv_mem_read(cpu, addr, &schib, sizeof(schib));
>> +    } else {
>> +        if (s390_cpu_virt_mem_read(cpu, addr, ar, &schib,
>> sizeof(schib))) {
>> +            s390_cpu_virt_mem_handle_exc(cpu, ra);
>> +            return;
>> +        }
>>      }
>>      if (ioinst_disassemble_sch_ident(reg1, &m, &cssid, &ssid,
>> &schid) || !ioinst_schib_valid(&schib)) {
>> @@ -193,9 +197,13 @@ void ioinst_handle_ssch(S390CPU *cpu, uint64_t
>> reg1, uint32_t ipb, uintptr_t ra) s390_program_interrupt(env,
>> PGM_SPECIFICATION, ra); return;
>>      }
>> -    if (s390_cpu_virt_mem_read(cpu, addr, ar, &orig_orb,
>> sizeof(orb))) {
>> -        s390_cpu_virt_mem_handle_exc(cpu, ra);
>> -        return;
>> +    if (s390_is_pv()) {
>> +        s390_cpu_pv_mem_read(cpu, addr, &orig_orb, sizeof(orb));
>> +    } else {
>> +        if (s390_cpu_virt_mem_read(cpu, addr, ar, &orig_orb,
>> sizeof(orb))) {
>> +            s390_cpu_virt_mem_handle_exc(cpu, ra);
>> +            return;
>> +        }
>>      }
>>      copy_orb_from_guest(&orb, &orig_orb);
>>      if (ioinst_disassemble_sch_ident(reg1, &m, &cssid, &ssid,
>> &schid) || @@ -229,14 +237,19 @@ void ioinst_handle_stcrw(S390CPU
>> *cpu, uint32_t ipb, uintptr_t ra) cc = css_do_stcrw(&crw);
>>      /* 0 - crw stored, 1 - zeroes stored */
>>  
>> -    if (s390_cpu_virt_mem_write(cpu, addr, ar, &crw, sizeof(crw)) ==
>> 0) {
>> +    if (s390_is_pv()) {
>> +        s390_cpu_pv_mem_write(cpu, addr, &crw, sizeof(crw));
>>          setcc(cpu, cc);
>>      } else {
>> -        if (cc == 0) {
>> -            /* Write failed: requeue CRW since STCRW is suppressing
>> */
>> -            css_undo_stcrw(&crw);
>> +        if (s390_cpu_virt_mem_write(cpu, addr, ar, &crw,
>> sizeof(crw)) == 0) {
>> +            setcc(cpu, cc);
>> +        } else {
>> +            if (cc == 0) {
>> +                /* Write failed: requeue CRW since STCRW is
>> suppressing */
>> +                css_undo_stcrw(&crw);
>> +            }
>> +            s390_cpu_virt_mem_handle_exc(cpu, ra);
>>          }
>> -        s390_cpu_virt_mem_handle_exc(cpu, ra);
>>      }
>>  }
>>  
>> @@ -258,6 +271,9 @@ void ioinst_handle_stsch(S390CPU *cpu, uint64_t
>> reg1, uint32_t ipb, }
>>  
>>      if (ioinst_disassemble_sch_ident(reg1, &m, &cssid, &ssid,
>> &schid)) {
>> +        if (s390_is_pv()) {
>> +            return;
>> +        }
> 
> If the operand is invalid, should we not inject an operand exception?
> 
> If instead the Ultravisor or KVM make sure that we don't end up here,
> then, why handle this at all?

Currently the specs specify that we may inject an operand exception but
I'm not able to find any exception that would still apply after all of
the ones that SIE already handles.

I've already asked for clarification and I'm considering to make this a
g_assert(!s390_is_pv())

So I hope we can clear this up in the next architecture call.

> 
>>          /*
>>           * As operand exceptions have a lower priority than access
>> exceptions,
>>           * we check whether the memory area is writeable (injecting
>> the @@ -290,14 +306,19 @@ void ioinst_handle_stsch(S390CPU *cpu,
>> uint64_t reg1, uint32_t ipb, }
>>      }
>>      if (cc != 3) {
>> -        if (s390_cpu_virt_mem_write(cpu, addr, ar, &schib,
>> -                                    sizeof(schib)) != 0) {
>> -            s390_cpu_virt_mem_handle_exc(cpu, ra);
>> -            return;
>> +        if (s390_is_pv()) {
>> +            s390_cpu_pv_mem_write(cpu, addr, &schib, sizeof(schib));
>> +        } else {
>> +            if (s390_cpu_virt_mem_write(cpu, addr, ar, &schib,
>> +                                        sizeof(schib)) != 0) {
>> +                s390_cpu_virt_mem_handle_exc(cpu, ra);
>> +                return;
>> +            }
>>          }
>>      } else {
>>          /* Access exceptions have a higher priority than cc3 */
>> -        if (s390_cpu_virt_mem_check_write(cpu, addr, ar,
>> sizeof(schib)) != 0) {
>> +        if (!s390_is_pv() &&
>> +            s390_cpu_virt_mem_check_write(cpu, addr, ar,
>> sizeof(schib)) != 0) { s390_cpu_virt_mem_handle_exc(cpu, ra);
>>              return;
>>          }
>> @@ -334,15 +355,20 @@ int ioinst_handle_tsch(S390CPU *cpu, uint64_t
>> reg1, uint32_t ipb, uintptr_t ra) }
>>      /* 0 - status pending, 1 - not status pending, 3 - not
>> operational */ if (cc != 3) {
>> -        if (s390_cpu_virt_mem_write(cpu, addr, ar, &irb, irb_len) !=
>> 0) {
>> -            s390_cpu_virt_mem_handle_exc(cpu, ra);
>> -            return -EFAULT;
>> +        if (s390_is_pv()) {
>> +            s390_cpu_pv_mem_write(cpu, addr, &irb, irb_len);
>> +        } else {
>> +            if (s390_cpu_virt_mem_write(cpu, addr, ar, &irb,
>> irb_len) != 0) {
>> +                s390_cpu_virt_mem_handle_exc(cpu, ra);
>> +                return -EFAULT;
>> +            }
>>          }
>>          css_do_tsch_update_subch(sch);
>>      } else {
>>          irb_len = sizeof(irb) - sizeof(irb.emw);
>>          /* Access exceptions have a higher priority than cc3 */
>> -        if (s390_cpu_virt_mem_check_write(cpu, addr, ar, irb_len) !=
>> 0) {
>> +        if (!s390_is_pv() &&
>> +            s390_cpu_virt_mem_check_write(cpu, addr, ar, irb_len) !=
>> 0) { s390_cpu_virt_mem_handle_exc(cpu, ra);
>>              return -EFAULT;
>>          }
>> @@ -640,9 +666,13 @@ void ioinst_handle_chsc(S390CPU *cpu, uint32_t
>> ipb, uintptr_t ra)
>>       * present CHSC sub-handlers ... if we ever need more, we should
>> take
>>       * care of req->len here first.
>>       */
>> -    if (s390_cpu_virt_mem_read(cpu, addr, reg, buf,
>> sizeof(ChscReq))) {
>> -        s390_cpu_virt_mem_handle_exc(cpu, ra);
>> -        return;
>> +    if (s390_is_pv()) {
>> +        s390_cpu_pv_mem_read(cpu, addr, buf, sizeof(ChscReq));
>> +    } else {
>> +        if (s390_cpu_virt_mem_read(cpu, addr, reg, buf,
>> sizeof(ChscReq))) {
>> +            s390_cpu_virt_mem_handle_exc(cpu, ra);
>> +            return;
>> +        }
>>      }
>>      req = (ChscReq *)buf;
>>      len = be16_to_cpu(req->len);
>> @@ -673,11 +703,16 @@ void ioinst_handle_chsc(S390CPU *cpu, uint32_t
>> ipb, uintptr_t ra) break;
>>      }
>>  
>> -    if (!s390_cpu_virt_mem_write(cpu, addr + len, reg, res,
>> -                                 be16_to_cpu(res->len))) {
>> +    if (s390_is_pv()) {
>> +        s390_cpu_pv_mem_write(cpu, addr + len, res,
>> be16_to_cpu(res->len)); setcc(cpu, 0);    /* Command execution
>> complete */ } else {
>> -        s390_cpu_virt_mem_handle_exc(cpu, ra);
>> +        if (!s390_cpu_virt_mem_write(cpu, addr + len, reg, res,
>> +                                     be16_to_cpu(res->len))) {
>> +            setcc(cpu, 0);    /* Command execution complete */
>> +        } else {
>> +            s390_cpu_virt_mem_handle_exc(cpu, ra);
>> +        }
>>      }
>>  }
>>  
> 
> 



[-- Attachment #2: OpenPGP digital signature --]
[-- Type: application/pgp-signature, Size: 833 bytes --]

Re: [PATCH v9 14/15] docs: Add protvirt docs

[Janosch Frank]

[-- Attachment #1.1: Type: text/plain, Size: 3724 bytes --]

On 3/13/20 2:00 PM, Claudio Imbrenda wrote:
> On Wed, 11 Mar 2020 09:21:50 -0400
> Janosch Frank <frankja@linux.ibm.com> wrote:
> 
>> Lets add some documentation for the Protected VM functionality.
>>
>> Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
>> ---
>>  docs/system/index.rst    |  1 +
>>  docs/system/protvirt.rst | 56
>> ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 57
>> insertions(+) create mode 100644 docs/system/protvirt.rst
>>
>> diff --git a/docs/system/index.rst b/docs/system/index.rst
>> index 6e5f20fa1333ce23..74afbd7cc3fc0296 100644
>> --- a/docs/system/index.rst
>> +++ b/docs/system/index.rst
>> @@ -34,3 +34,4 @@ Contents:
>>     deprecated
>>     build-platforms
>>     license
>> +   protvirt
>> diff --git a/docs/system/protvirt.rst b/docs/system/protvirt.rst
>> new file mode 100644
>> index 0000000000000000..6c8cf0f7910eae86
>> --- /dev/null
>> +++ b/docs/system/protvirt.rst
>> @@ -0,0 +1,56 @@
>> +Protected Virtualization on s390x
>> +=================================
>> +
>> +The memory and most of the registers of Protected Virtual Machines
>> +(PVMs) are encrypted or inaccessible to the hypervisor, effectively
>> +prohibiting VM introspection when the VM is running. At rest, PVMs
>> are +encrypted and can only be decrypted by the firmware, represented
>> by an +entity called Ultravisor, of specific IBM Z machines.
>> +
>> +
>> +Prerequisites
>> +-------------
>> +
>> +To run PVMs a machine with the Protected Virtualization feature
>> +which is indicated by the Ultravisor Call facility (stfle bit
>> +158) is required. The Ultravisor needs to be initialized at boot by
>> +setting `prot_virt=1` on the kernel command line.
> 
> I'd add "of the host" just to make it extra clear

How about:
...on the host kernel command line

> 
>> +
>> +If those requirements are met, the capability
>> `KVM_CAP_S390_PROTECTED` +will indicate that KVM can support PVMs on
>> that LPAR. +
>> +
>> +QEMU Settings
>> +-------------
>> +
>> +To indicate to the VM that it can transition into protected mode, the
>> +`Unpack facility` (stfle bit 161 represented by the feature
>> +`S390_FEAT_UNPACK`) needs to be part of the cpu model of the VM.
>> +
>> +All I/O devices need to use the IOMMU.
>> +Passthrough (vfio) devices are currently not supported.
>> +
>> +Host huge page backings are not supported. However guests can use
>> huge +pages as indicated by its facilities.
>> +
>> +
>> +Boot Process
>> +------------
>> +
>> +A secure guest image can either be loaded from disk or supplied on
>> the +QEMU command line. Booting from disk is done by the unmodified
>> +s390-ccw BIOS. I.e., the bootmap is interpreted, multiple components
>> +are read into memory and control is transferred to one of the
>> +components (zipl stage3). Stag3 does some fixups and then transfers
>> +control to some program residing in guest memory, which is normally
>> +the OS kernel. The secure image has another component prepended
>> +(stage3a) that uses the new diag308 subcodes 8 and 10 to trigger the
>> +transition into secure mode.
>> +
>> +Booting from the image supplied via the QEMU command line requires
>> +that the file passed via -kernel has the same memory layout as would
>> +result from the disk boot. This memory layout includes the encrypted
>> +components (kernel, initrd, cmdline), the stage3a loader and
>> +metadata. In case this boot method is used, the command line
>> +options -initrd and -cmdline are ineffective. The preparation of a
>> PVM +image is done by genprotimg of the s390-tools package.
> 
> Reviewed-by: Claudio Imbrenda <imbrenda@linux.ibm.com>

Thanks!



[-- Attachment #2: OpenPGP digital signature --]
[-- Type: application/pgp-signature, Size: 833 bytes --]

Re: [PATCH v9 14/15] docs: Add protvirt docs

[Peter Maydell]

On Wed, 11 Mar 2020 at 13:31, Janosch Frank <frankja@linux.ibm.com> wrote:
>
> Lets add some documentation for the Protected VM functionality.
>
> Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
> ---
>  docs/system/index.rst    |  1 +
>  docs/system/protvirt.rst | 56 ++++++++++++++++++++++++++++++++++++++++
>  2 files changed, 57 insertions(+)
>  create mode 100644 docs/system/protvirt.rst
>
> diff --git a/docs/system/index.rst b/docs/system/index.rst
> index 6e5f20fa1333ce23..74afbd7cc3fc0296 100644
> --- a/docs/system/index.rst
> +++ b/docs/system/index.rst
> @@ -34,3 +34,4 @@ Contents:
>     deprecated
>     build-platforms
>     license
> +   protvirt

The order of this list in index.rst determines the order
of the table of contents in the manual. Could you put
your new document at a reasonable place in the manual,
not just at the bottom of the list, please?

For something that is s390 specific, the ideal would
be to create a new docs/system/target-s390x.rst
which is then the place where all s390 docs can
hang off of. This is how we're handling various
other guest architecture docs. It doesn't all
have to go in a single page -- eg target-arm.rst
is an example of putting some sub-documents into
docs/system/arm/ and referring to them from
target-arm.rst.

thanks
-- PMM

Re: [PATCH v9 09/15] s390x: protvirt: Set guest IPL PSW

[Janosch Frank]

[-- Attachment #1.1: Type: text/plain, Size: 2210 bytes --]

On 3/13/20 1:57 PM, Claudio Imbrenda wrote:
> On Wed, 11 Mar 2020 09:21:45 -0400
> Janosch Frank <frankja@linux.ibm.com> wrote:
> 
>> Handling of CPU reset and setting of the IPL psw from guest storage at
>> offset 0 is done by a Ultravisor call. Let's only fetch it if
>> necessary.
>>
>> Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
>> Reviewed-by: Thomas Huth <thuth@redhat.com>
>> Reviewed-by: David Hildenbrand <david@redhat.com>
>> ---
>>  target/s390x/cpu.c | 22 +++++++++++++---------
>>  1 file changed, 13 insertions(+), 9 deletions(-)
>>
>> diff --git a/target/s390x/cpu.c b/target/s390x/cpu.c
>> index 84029f14814b4980..a48d39f139cdc1c4 100644
>> --- a/target/s390x/cpu.c
>> +++ b/target/s390x/cpu.c
>> @@ -78,16 +78,20 @@ static bool s390_cpu_has_work(CPUState *cs)
>>  static void s390_cpu_load_normal(CPUState *s)
>>  {
>>      S390CPU *cpu = S390_CPU(s);
>> -    uint64_t spsw = ldq_phys(s->as, 0);
>> -
>> -    cpu->env.psw.mask = spsw & PSW_MASK_SHORT_CTRL;
>> -    /*
>> -     * Invert short psw indication, so SIE will report a
>> specification
>> -     * exception if it was not set.
>> -     */
>> -    cpu->env.psw.mask ^= PSW_MASK_SHORTPSW;
>> -    cpu->env.psw.addr = spsw & PSW_MASK_SHORT_ADDR;
>> +    uint64_t spsw;
>>  
>> +    if (!s390_is_pv()) {
>> +        spsw = ldq_phys(s->as, 0);
>> +        cpu->env.psw.mask = spsw & PSW_MASK_SHORT_CTRL;
>> +        /*
>> +         * Invert short psw indication, so SIE will report a
>> specification
>> +         * exception if it was not set.
>> +         */
>> +        cpu->env.psw.mask ^= PSW_MASK_SHORTPSW;
>> +        cpu->env.psw.addr = spsw & PSW_MASK_SHORT_ADDR;
>> +    } else {
>> +        s390_cpu_set_state(S390_CPU_STATE_LOAD, cpu);
>> +    }
>>      s390_cpu_set_state(S390_CPU_STATE_OPERATING, cpu);
>>  }
>>  #endif
> 
> I don't understand why you set the state to S390_CPU_STATE_LOAD and
> then immediately afterwards to S390_CPU_STATE_OPERATING, especially
> considering that both do the same
> 

Have a look at the specs, wee need to set the load state before setting
the cpu to operating.

I can add a comment to make it clearer if you want.



[-- Attachment #2: OpenPGP digital signature --]
[-- Type: application/pgp-signature, Size: 833 bytes --]

Re: [PATCH v9 09/15] s390x: protvirt: Set guest IPL PSW

[Christian Borntraeger]

On 13.03.20 15:21, Janosch Frank wrote:
[..]
>>> +    } else {
>>> +        s390_cpu_set_state(S390_CPU_STATE_LOAD, cpu);
>>> +    }
>>>      s390_cpu_set_state(S390_CPU_STATE_OPERATING, cpu);
>>>  }
>>>  #endif
>>
>> I don't understand why you set the state to S390_CPU_STATE_LOAD and
>> then immediately afterwards to S390_CPU_STATE_OPERATING, especially
>> considering that both do the same
>>
> 
> Have a look at the specs, wee need to set the load state before setting
> the cpu to operating.
> 
> I can add a comment to make it clearer if you want.

Yes please. I stumbled over this as well before I read the specs.

Re: [PATCH v9 09/15] s390x: protvirt: Set guest IPL PSW

[Claudio Imbrenda]

On Fri, 13 Mar 2020 15:21:07 +0100
Janosch Frank <frankja@linux.ibm.com> wrote:

> On 3/13/20 1:57 PM, Claudio Imbrenda wrote:
> > On Wed, 11 Mar 2020 09:21:45 -0400
> > Janosch Frank <frankja@linux.ibm.com> wrote:
> >   
> >> Handling of CPU reset and setting of the IPL psw from guest
> >> storage at offset 0 is done by a Ultravisor call. Let's only fetch
> >> it if necessary.
> >>
> >> Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
> >> Reviewed-by: Thomas Huth <thuth@redhat.com>
> >> Reviewed-by: David Hildenbrand <david@redhat.com>
> >> ---
> >>  target/s390x/cpu.c | 22 +++++++++++++---------
> >>  1 file changed, 13 insertions(+), 9 deletions(-)
> >>
> >> diff --git a/target/s390x/cpu.c b/target/s390x/cpu.c
> >> index 84029f14814b4980..a48d39f139cdc1c4 100644
> >> --- a/target/s390x/cpu.c
> >> +++ b/target/s390x/cpu.c
> >> @@ -78,16 +78,20 @@ static bool s390_cpu_has_work(CPUState *cs)
> >>  static void s390_cpu_load_normal(CPUState *s)
> >>  {
> >>      S390CPU *cpu = S390_CPU(s);
> >> -    uint64_t spsw = ldq_phys(s->as, 0);
> >> -
> >> -    cpu->env.psw.mask = spsw & PSW_MASK_SHORT_CTRL;
> >> -    /*
> >> -     * Invert short psw indication, so SIE will report a
> >> specification
> >> -     * exception if it was not set.
> >> -     */
> >> -    cpu->env.psw.mask ^= PSW_MASK_SHORTPSW;
> >> -    cpu->env.psw.addr = spsw & PSW_MASK_SHORT_ADDR;
> >> +    uint64_t spsw;
> >>  
> >> +    if (!s390_is_pv()) {
> >> +        spsw = ldq_phys(s->as, 0);
> >> +        cpu->env.psw.mask = spsw & PSW_MASK_SHORT_CTRL;
> >> +        /*
> >> +         * Invert short psw indication, so SIE will report a
> >> specification
> >> +         * exception if it was not set.
> >> +         */
> >> +        cpu->env.psw.mask ^= PSW_MASK_SHORTPSW;
> >> +        cpu->env.psw.addr = spsw & PSW_MASK_SHORT_ADDR;
> >> +    } else {
> >> +        s390_cpu_set_state(S390_CPU_STATE_LOAD, cpu);
> >> +    }
> >>      s390_cpu_set_state(S390_CPU_STATE_OPERATING, cpu);
> >>  }
> >>  #endif  
> > 
> > I don't understand why you set the state to S390_CPU_STATE_LOAD and
> > then immediately afterwards to S390_CPU_STATE_OPERATING, especially
> > considering that both do the same
> >   
> 
> Have a look at the specs, wee need to set the load state before
> setting the cpu to operating.
> 
> I can add a comment to make it clearer if you want.

yes please. 

Re: [PATCH v9] fixup! Fix subcode/pbt

[Christian Borntraeger]

On 13.03.20 10:52, Janosch Frank wrote:
> Signed-off-by: Janosch Frank <frankja@linux.ibm.com>

looks sane. You will merge this?

> ---
>  hw/s390x/ipl.h      | 11 +++++++----
>  target/s390x/diag.c |  2 +-
>  2 files changed, 8 insertions(+), 5 deletions(-)
> 
> diff --git a/hw/s390x/ipl.h b/hw/s390x/ipl.h
> index 95e3183c9cccf8b6..f799f7cfcf4763b1 100644
> --- a/hw/s390x/ipl.h
> +++ b/hw/s390x/ipl.h
> @@ -261,15 +261,18 @@ static inline bool ipl_valid_pv_header(IplParameterBlock *iplb)
>          return true;
>  }
>  
> -static inline bool iplb_valid(IplParameterBlock *iplb)
> +static inline bool iplb_valid(IplParameterBlock *iplb, uint64_t subcode)
>  {
>      switch (iplb->pbt) {
>      case S390_IPL_TYPE_FCP:
> -        return be32_to_cpu(iplb->len) >= S390_IPLB_MIN_FCP_LEN;
> +        return (subcode == DIAG308_SET &&
> +                be32_to_cpu(iplb->len) >= S390_IPLB_MIN_FCP_LEN);
>      case S390_IPL_TYPE_CCW:
> -        return be32_to_cpu(iplb->len) >= S390_IPLB_MIN_CCW_LEN;
> +        return (subcode == DIAG308_SET &&
> +                be32_to_cpu(iplb->len) >= S390_IPLB_MIN_CCW_LEN);
>      case S390_IPL_TYPE_PV:
> -        if (be32_to_cpu(iplb->len) < S390_IPLB_MIN_PV_LEN) {
> +        if (subcode != DIAG308_PV_SET ||
> +            be32_to_cpu(iplb->len) < S390_IPLB_MIN_PV_LEN) {
>              return false;
>          }
>          if (!ipl_valid_pv_header(iplb)) {
> diff --git a/target/s390x/diag.c b/target/s390x/diag.c
> index b1ca81633b83bbdc..d4f33db5c23c818d 100644
> --- a/target/s390x/diag.c
> +++ b/target/s390x/diag.c
> @@ -118,7 +118,7 @@ void handle_diag_308(CPUS390XState *env, uint64_t r1, uint64_t r3, uintptr_t ra)
>  
>          cpu_physical_memory_read(addr, iplb, be32_to_cpu(iplb->len));
>  
> -        if (!iplb_valid(iplb)) {
> +        if (!iplb_valid(iplb, subcode)) {
>              env->regs[r1 + 1] = DIAG_308_RC_INVALID;
>              goto out;
>          }
> 

Re: [PATCH v9 09/15] s390x: protvirt: Set guest IPL PSW

[Claudio Imbrenda]

On Fri, 13 Mar 2020 15:21:07 +0100
Janosch Frank <frankja@linux.ibm.com> wrote:

> On 3/13/20 1:57 PM, Claudio Imbrenda wrote:
> > On Wed, 11 Mar 2020 09:21:45 -0400
> > Janosch Frank <frankja@linux.ibm.com> wrote:
> >   
> >> Handling of CPU reset and setting of the IPL psw from guest
> >> storage at offset 0 is done by a Ultravisor call. Let's only fetch
> >> it if necessary.
> >>
> >> Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
> >> Reviewed-by: Thomas Huth <thuth@redhat.com>
> >> Reviewed-by: David Hildenbrand <david@redhat.com>
> >> ---
> >>  target/s390x/cpu.c | 22 +++++++++++++---------
> >>  1 file changed, 13 insertions(+), 9 deletions(-)
> >>
> >> diff --git a/target/s390x/cpu.c b/target/s390x/cpu.c
> >> index 84029f14814b4980..a48d39f139cdc1c4 100644
> >> --- a/target/s390x/cpu.c
> >> +++ b/target/s390x/cpu.c
> >> @@ -78,16 +78,20 @@ static bool s390_cpu_has_work(CPUState *cs)
> >>  static void s390_cpu_load_normal(CPUState *s)
> >>  {
> >>      S390CPU *cpu = S390_CPU(s);
> >> -    uint64_t spsw = ldq_phys(s->as, 0);
> >> -
> >> -    cpu->env.psw.mask = spsw & PSW_MASK_SHORT_CTRL;
> >> -    /*
> >> -     * Invert short psw indication, so SIE will report a
> >> specification
> >> -     * exception if it was not set.
> >> -     */
> >> -    cpu->env.psw.mask ^= PSW_MASK_SHORTPSW;
> >> -    cpu->env.psw.addr = spsw & PSW_MASK_SHORT_ADDR;
> >> +    uint64_t spsw;
> >>  
> >> +    if (!s390_is_pv()) {
> >> +        spsw = ldq_phys(s->as, 0);
> >> +        cpu->env.psw.mask = spsw & PSW_MASK_SHORT_CTRL;
> >> +        /*
> >> +         * Invert short psw indication, so SIE will report a
> >> specification
> >> +         * exception if it was not set.
> >> +         */
> >> +        cpu->env.psw.mask ^= PSW_MASK_SHORTPSW;
> >> +        cpu->env.psw.addr = spsw & PSW_MASK_SHORT_ADDR;
> >> +    } else {
> >> +        s390_cpu_set_state(S390_CPU_STATE_LOAD, cpu);
> >> +    }
> >>      s390_cpu_set_state(S390_CPU_STATE_OPERATING, cpu);
> >>  }
> >>  #endif  
> > 
> > I don't understand why you set the state to S390_CPU_STATE_LOAD and
> > then immediately afterwards to S390_CPU_STATE_OPERATING, especially
> > considering that both do the same
> >   
> 
> Have a look at the specs, wee need to set the load state before
> setting the cpu to operating.
> 
> I can add a comment to make it clearer if you want.

once you have added the comment, you can also add:

Reviewed-by: Claudio Imbrenda <imbrenda@linux.ibm.com>

Re: [PATCH v9] fixup! Fix subcode/pbt

[Janosch Frank]

[-- Attachment #1.1: Type: text/plain, Size: 2125 bytes --]

On 3/13/20 3:30 PM, Christian Borntraeger wrote:
> 
> 
> On 13.03.20 10:52, Janosch Frank wrote:
>> Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
> 
> looks sane. You will merge this?

Yes, just did.

> 
>> ---
>>  hw/s390x/ipl.h      | 11 +++++++----
>>  target/s390x/diag.c |  2 +-
>>  2 files changed, 8 insertions(+), 5 deletions(-)
>>
>> diff --git a/hw/s390x/ipl.h b/hw/s390x/ipl.h
>> index 95e3183c9cccf8b6..f799f7cfcf4763b1 100644
>> --- a/hw/s390x/ipl.h
>> +++ b/hw/s390x/ipl.h
>> @@ -261,15 +261,18 @@ static inline bool ipl_valid_pv_header(IplParameterBlock *iplb)
>>          return true;
>>  }
>>  
>> -static inline bool iplb_valid(IplParameterBlock *iplb)
>> +static inline bool iplb_valid(IplParameterBlock *iplb, uint64_t subcode)
>>  {
>>      switch (iplb->pbt) {
>>      case S390_IPL_TYPE_FCP:
>> -        return be32_to_cpu(iplb->len) >= S390_IPLB_MIN_FCP_LEN;
>> +        return (subcode == DIAG308_SET &&
>> +                be32_to_cpu(iplb->len) >= S390_IPLB_MIN_FCP_LEN);
>>      case S390_IPL_TYPE_CCW:
>> -        return be32_to_cpu(iplb->len) >= S390_IPLB_MIN_CCW_LEN;
>> +        return (subcode == DIAG308_SET &&
>> +                be32_to_cpu(iplb->len) >= S390_IPLB_MIN_CCW_LEN);
>>      case S390_IPL_TYPE_PV:
>> -        if (be32_to_cpu(iplb->len) < S390_IPLB_MIN_PV_LEN) {
>> +        if (subcode != DIAG308_PV_SET ||
>> +            be32_to_cpu(iplb->len) < S390_IPLB_MIN_PV_LEN) {
>>              return false;
>>          }
>>          if (!ipl_valid_pv_header(iplb)) {
>> diff --git a/target/s390x/diag.c b/target/s390x/diag.c
>> index b1ca81633b83bbdc..d4f33db5c23c818d 100644
>> --- a/target/s390x/diag.c
>> +++ b/target/s390x/diag.c
>> @@ -118,7 +118,7 @@ void handle_diag_308(CPUS390XState *env, uint64_t r1, uint64_t r3, uintptr_t ra)
>>  
>>          cpu_physical_memory_read(addr, iplb, be32_to_cpu(iplb->len));
>>  
>> -        if (!iplb_valid(iplb)) {
>> +        if (!iplb_valid(iplb, subcode)) {
>>              env->regs[r1 + 1] = DIAG_308_RC_INVALID;
>>              goto out;
>>          }
>>
> 
> 



[-- Attachment #2: OpenPGP digital signature --]
[-- Type: application/pgp-signature, Size: 833 bytes --]

Re: [PATCH v9] s390x: protvirt: Fence huge pages

[Janosch Frank]

[-- Attachment #1.1: Type: text/plain, Size: 6444 bytes --]

On 3/13/20 9:21 AM, Christian Borntraeger wrote:
> 
> 
> On 12.03.20 17:25, Janosch Frank wrote:
>> Let's bail out of the protected transition if we detect that huge
>> pages might be in use.
>>
>> Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
>> ---
>>
>> I'd like to squash this into the unpack patch to give a proper error
>> message if we try to transition into the protected mode while being
>> backed by huge pages. 
> 
> Looks good.
> But maybe we can do it better. Why not reverse the logic and
> instead of having kvm_s390_get_hpage_1m, let us define an protvirt_allowed
> that as of today only returns hugepages != 1:
> Then we could (for kvm-stub.c) also say protvirt_allowed=false;
> And if other reasons come along we can extend.
> 
> We could also keep this patch separate, does not really matter.

The *_allowed() functions are all based on the machine and part of
s390-virtio-ccw.c so having one in kvm.c looks strange.

!protvirt_allowed could have any number of reasons in the future, I
introduced this patch to give a specific error message that can help the
user to chose the right options when looking for the error.

Other ideas or a revised one?

> 
>>
>> ---
>>  hw/s390x/ipl.h             | 16 ++++++++++++++++
>>  hw/s390x/s390-virtio-ccw.c |  1 -
>>  target/s390x/diag.c        | 23 ++++++++---------------
>>  target/s390x/kvm-stub.c    |  5 +++++
>>  target/s390x/kvm.c         |  5 +++++
>>  target/s390x/kvm_s390x.h   |  1 +
>>  6 files changed, 35 insertions(+), 16 deletions(-)
>>
>> diff --git a/hw/s390x/ipl.h b/hw/s390x/ipl.h
>> index af5bb130a6334821..95e3183c9cccf8b6 100644
>> --- a/hw/s390x/ipl.h
>> +++ b/hw/s390x/ipl.h
>> @@ -185,6 +185,22 @@ struct S390IPLState {
>>  typedef struct S390IPLState S390IPLState;
>>  QEMU_BUILD_BUG_MSG(offsetof(S390IPLState, iplb) & 3, "alignment of iplb wrong");
>>  
>> +#define DIAG_308_RC_OK              0x0001
>> +#define DIAG_308_RC_NO_CONF         0x0102
>> +#define DIAG_308_RC_INVALID         0x0402
>> +#define DIAG_308_RC_NO_PV_CONF      0x0902
>> +#define DIAG_308_RC_INVAL_FOR_PV    0x0a02
>> +
>> +#define DIAG308_RESET_MOD_CLR       0
>> +#define DIAG308_RESET_LOAD_NORM     1
>> +#define DIAG308_LOAD_CLEAR          3
>> +#define DIAG308_LOAD_NORMAL_DUMP    4
>> +#define DIAG308_SET                 5
>> +#define DIAG308_STORE               6
>> +#define DIAG308_PV_SET              8
>> +#define DIAG308_PV_STORE            9
>> +#define DIAG308_PV_START            10
>> +
>>  #define S390_IPL_TYPE_FCP 0x00
>>  #define S390_IPL_TYPE_CCW 0x02
>>  #define S390_IPL_TYPE_PV 0x05
>> diff --git a/hw/s390x/s390-virtio-ccw.c b/hw/s390x/s390-virtio-ccw.c
>> index ebdaaa3a001f6e8c..d32f35c7f47b9c1d 100644
>> --- a/hw/s390x/s390-virtio-ccw.c
>> +++ b/hw/s390x/s390-virtio-ccw.c
>> @@ -361,7 +361,6 @@ out_err:
>>      return rc;
>>  }
>>  
>> -#define DIAG_308_RC_INVAL_FOR_PV    0x0a02
>>  static void s390_machine_inject_pv_error(CPUState *cs)
>>  {
>>      int r1 = (cs->kvm_run->s390_sieic.ipa & 0x00f0) >> 4;
>> diff --git a/target/s390x/diag.c b/target/s390x/diag.c
>> index b245e557037ded06..b1ca81633b83bbdc 100644
>> --- a/target/s390x/diag.c
>> +++ b/target/s390x/diag.c
>> @@ -21,6 +21,7 @@
>>  #include "hw/s390x/ipl.h"
>>  #include "hw/s390x/s390-virtio-ccw.h"
>>  #include "hw/s390x/pv.h"
>> +#include "kvm_s390x.h"
>>  
>>  int handle_diag_288(CPUS390XState *env, uint64_t r1, uint64_t r3)
>>  {
>> @@ -50,21 +51,6 @@ int handle_diag_288(CPUS390XState *env, uint64_t r1, uint64_t r3)
>>      return diag288_class->handle_timer(diag288, func, timeout);
>>  }
>>  
>> -#define DIAG_308_RC_OK              0x0001
>> -#define DIAG_308_RC_NO_CONF         0x0102
>> -#define DIAG_308_RC_INVALID         0x0402
>> -#define DIAG_308_RC_NO_PV_CONF      0x0902
>> -
>> -#define DIAG308_RESET_MOD_CLR       0
>> -#define DIAG308_RESET_LOAD_NORM     1
>> -#define DIAG308_LOAD_CLEAR          3
>> -#define DIAG308_LOAD_NORMAL_DUMP    4
>> -#define DIAG308_SET                 5
>> -#define DIAG308_STORE               6
>> -#define DIAG308_PV_SET              8
>> -#define DIAG308_PV_STORE            9
>> -#define DIAG308_PV_START            10
>> -
>>  static int diag308_parm_check(CPUS390XState *env, uint64_t r1, uint64_t addr,
>>                                uintptr_t ra, bool write)
>>  {
>> @@ -166,6 +152,13 @@ out:
>>              return;
>>          }
>>  
>> +        if (kvm_s390_get_hpage_1m()) {
>> +            error_report("Protected VMs can currently not be backed with "
>> +                         "huge pages");
>> +            env->regs[r1 + 1] = DIAG_308_RC_INVAL_FOR_PV;
>> +            return;
>> +        }
>> +
>>          s390_ipl_reset_request(cs, S390_RESET_PV);
>>          break;
>>      default:
>> diff --git a/target/s390x/kvm-stub.c b/target/s390x/kvm-stub.c
>> index c4cd497f850eb9c7..aa185017a2a886ca 100644
>> --- a/target/s390x/kvm-stub.c
>> +++ b/target/s390x/kvm-stub.c
>> @@ -39,6 +39,11 @@ int kvm_s390_vcpu_interrupt_post_load(S390CPU *cpu)
>>      return 0;
>>  }
>>  
>> +int kvm_s390_get_hpage_1m(void)
>> +{
>> +    return 0;
>> +}
>> +
>>  int kvm_s390_get_ri(void)
>>  {
>>      return 0;
>> diff --git a/target/s390x/kvm.c b/target/s390x/kvm.c
>> index 1d6fd6a27b48e35f..c695941076b7aead 100644
>> --- a/target/s390x/kvm.c
>> +++ b/target/s390x/kvm.c
>> @@ -321,6 +321,11 @@ void kvm_s390_set_max_pagesize(uint64_t pagesize, Error **errp)
>>      cap_hpage_1m = 1;
>>  }
>>  
>> +int kvm_s390_get_hpage_1m(void)
>> +{
>> +    return cap_hpage_1m;
>> +}
>> +
>>  static void ccw_machine_class_foreach(ObjectClass *oc, void *opaque)
>>  {
>>      MachineClass *mc = MACHINE_CLASS(oc);
>> diff --git a/target/s390x/kvm_s390x.h b/target/s390x/kvm_s390x.h
>> index 0b21789796d7c462..dea813f450153c34 100644
>> --- a/target/s390x/kvm_s390x.h
>> +++ b/target/s390x/kvm_s390x.h
>> @@ -23,6 +23,7 @@ void kvm_s390_program_interrupt(S390CPU *cpu, uint16_t code);
>>  int kvm_s390_set_cpu_state(S390CPU *cpu, uint8_t cpu_state);
>>  void kvm_s390_vcpu_interrupt_pre_save(S390CPU *cpu);
>>  int kvm_s390_vcpu_interrupt_post_load(S390CPU *cpu);
>> +int kvm_s390_get_hpage_1m(void);
>>  int kvm_s390_get_ri(void);
>>  int kvm_s390_get_gs(void);
>>  int kvm_s390_get_clock(uint8_t *tod_high, uint64_t *tod_clock);
>>
> 
> 



[-- Attachment #2: OpenPGP digital signature --]
[-- Type: application/pgp-signature, Size: 833 bytes --]

Re: [PATCH v9] fixup! Fix subcode/pbt

[Cornelia Huck]

On Fri, 13 Mar 2020 05:52:32 -0400
Janosch Frank <frankja@linux.ibm.com> wrote:

> Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
> ---
>  hw/s390x/ipl.h      | 11 +++++++----
>  target/s390x/diag.c |  2 +-
>  2 files changed, 8 insertions(+), 5 deletions(-)
> 
> diff --git a/hw/s390x/ipl.h b/hw/s390x/ipl.h
> index 95e3183c9cccf8b6..f799f7cfcf4763b1 100644
> --- a/hw/s390x/ipl.h
> +++ b/hw/s390x/ipl.h
> @@ -261,15 +261,18 @@ static inline bool ipl_valid_pv_header(IplParameterBlock *iplb)
>          return true;
>  }
>  
> -static inline bool iplb_valid(IplParameterBlock *iplb)
> +static inline bool iplb_valid(IplParameterBlock *iplb, uint64_t subcode)
>  {
>      switch (iplb->pbt) {
>      case S390_IPL_TYPE_FCP:
> -        return be32_to_cpu(iplb->len) >= S390_IPLB_MIN_FCP_LEN;
> +        return (subcode == DIAG308_SET &&
> +                be32_to_cpu(iplb->len) >= S390_IPLB_MIN_FCP_LEN);
>      case S390_IPL_TYPE_CCW:
> -        return be32_to_cpu(iplb->len) >= S390_IPLB_MIN_CCW_LEN;
> +        return (subcode == DIAG308_SET &&
> +                be32_to_cpu(iplb->len) >= S390_IPLB_MIN_CCW_LEN);
>      case S390_IPL_TYPE_PV:
> -        if (be32_to_cpu(iplb->len) < S390_IPLB_MIN_PV_LEN) {
> +        if (subcode != DIAG308_PV_SET ||
> +            be32_to_cpu(iplb->len) < S390_IPLB_MIN_PV_LEN) {
>              return false;

I'm not sure I like passing the subcode here...

>          }
>          if (!ipl_valid_pv_header(iplb)) {
> diff --git a/target/s390x/diag.c b/target/s390x/diag.c
> index b1ca81633b83bbdc..d4f33db5c23c818d 100644
> --- a/target/s390x/diag.c
> +++ b/target/s390x/diag.c
> @@ -118,7 +118,7 @@ void handle_diag_308(CPUS390XState *env, uint64_t r1, uint64_t r3, uintptr_t ra)
>  
>          cpu_physical_memory_read(addr, iplb, be32_to_cpu(iplb->len));
>  
> -        if (!iplb_valid(iplb)) {
> +        if (!iplb_valid(iplb, subcode)) {
>              env->regs[r1 + 1] = DIAG_308_RC_INVALID;
>              goto out;
>          }

...because you're basically checking whether you either have a valid
normal iplb, or a valid pv iplb, with the two being mutually exclusive,
IIUC. So what about introducing iplb_valid_pv and calling that for the
pv case? Would be a bit nicer to read, I think, and also matches what
you do for the STORE case.

Re: [PATCH v9] fixup! Fix subcode/pbt

[Janosch Frank]

[-- Attachment #1.1: Type: text/plain, Size: 2713 bytes --]

On 3/16/20 3:27 PM, Cornelia Huck wrote:
> On Fri, 13 Mar 2020 05:52:32 -0400
> Janosch Frank <frankja@linux.ibm.com> wrote:
> 
>> Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
>> ---
>>  hw/s390x/ipl.h      | 11 +++++++----
>>  target/s390x/diag.c |  2 +-
>>  2 files changed, 8 insertions(+), 5 deletions(-)
>>
>> diff --git a/hw/s390x/ipl.h b/hw/s390x/ipl.h
>> index 95e3183c9cccf8b6..f799f7cfcf4763b1 100644
>> --- a/hw/s390x/ipl.h
>> +++ b/hw/s390x/ipl.h
>> @@ -261,15 +261,18 @@ static inline bool ipl_valid_pv_header(IplParameterBlock *iplb)
>>          return true;
>>  }
>>  
>> -static inline bool iplb_valid(IplParameterBlock *iplb)
>> +static inline bool iplb_valid(IplParameterBlock *iplb, uint64_t subcode)
>>  {
>>      switch (iplb->pbt) {
>>      case S390_IPL_TYPE_FCP:
>> -        return be32_to_cpu(iplb->len) >= S390_IPLB_MIN_FCP_LEN;
>> +        return (subcode == DIAG308_SET &&
>> +                be32_to_cpu(iplb->len) >= S390_IPLB_MIN_FCP_LEN);
>>      case S390_IPL_TYPE_CCW:
>> -        return be32_to_cpu(iplb->len) >= S390_IPLB_MIN_CCW_LEN;
>> +        return (subcode == DIAG308_SET &&
>> +                be32_to_cpu(iplb->len) >= S390_IPLB_MIN_CCW_LEN);
>>      case S390_IPL_TYPE_PV:
>> -        if (be32_to_cpu(iplb->len) < S390_IPLB_MIN_PV_LEN) {
>> +        if (subcode != DIAG308_PV_SET ||
>> +            be32_to_cpu(iplb->len) < S390_IPLB_MIN_PV_LEN) {
>>              return false;
> 
> I'm not sure I like passing the subcode here...

I could move this to diag.c and call it iplb_valid_for_subcode()

> 
>>          }
>>          if (!ipl_valid_pv_header(iplb)) {
>> diff --git a/target/s390x/diag.c b/target/s390x/diag.c
>> index b1ca81633b83bbdc..d4f33db5c23c818d 100644
>> --- a/target/s390x/diag.c
>> +++ b/target/s390x/diag.c
>> @@ -118,7 +118,7 @@ void handle_diag_308(CPUS390XState *env, uint64_t r1, uint64_t r3, uintptr_t ra)
>>  
>>          cpu_physical_memory_read(addr, iplb, be32_to_cpu(iplb->len));
>>  
>> -        if (!iplb_valid(iplb)) {
>> +        if (!iplb_valid(iplb, subcode)) {
>>              env->regs[r1 + 1] = DIAG_308_RC_INVALID;
>>              goto out;
>>          }
> 
> ...because you're basically checking whether you either have a valid
> normal iplb, or a valid pv iplb, with the two being mutually exclusive,
> IIUC. So what about introducing iplb_valid_pv and calling that for the
> pv case? Would be a bit nicer to read, I think, and also matches what
> you do for the STORE case.
> 

The idea was to get rid of all of these ifs and elses and only have one
iplb_valid function. Your suggestion would defeat hiding that complexity
behind this function.



[-- Attachment #2: OpenPGP digital signature --]
[-- Type: application/pgp-signature, Size: 833 bytes --]

Re: [PATCH v9 03/15] s390x: protvirt: Add migration blocker

[Cornelia Huck]

On Wed, 11 Mar 2020 09:21:39 -0400
Janosch Frank <frankja@linux.ibm.com> wrote:

> Migration is not yet supported.
> 
> Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
> Reviewed-by: David Hildenbrand <david@redhat.com>
> ---
>  hw/s390x/s390-virtio-ccw.c | 18 ++++++++++++++++++
>  1 file changed, 18 insertions(+)

Reviewed-by: Cornelia Huck <cohuck@redhat.com>

Re: [PATCH v9] fixup! Fix subcode/pbt

[Cornelia Huck]

[-- Attachment #1: Type: text/plain, Size: 1543 bytes --]

On Mon, 16 Mar 2020 15:47:41 +0100
Janosch Frank <frankja@linux.ibm.com> wrote:

> On 3/16/20 3:27 PM, Cornelia Huck wrote:
> > On Fri, 13 Mar 2020 05:52:32 -0400
> > Janosch Frank <frankja@linux.ibm.com> wrote:
> >   
> >> Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
> >> ---
> >>  hw/s390x/ipl.h      | 11 +++++++----
> >>  target/s390x/diag.c |  2 +-
> >>  2 files changed, 8 insertions(+), 5 deletions(-)


> >> @@ -118,7 +118,7 @@ void handle_diag_308(CPUS390XState *env, uint64_t r1, uint64_t r3, uintptr_t ra)
> >>  
> >>          cpu_physical_memory_read(addr, iplb, be32_to_cpu(iplb->len));
> >>  
> >> -        if (!iplb_valid(iplb)) {
> >> +        if (!iplb_valid(iplb, subcode)) {
> >>              env->regs[r1 + 1] = DIAG_308_RC_INVALID;
> >>              goto out;
> >>          }  
> > 
> > ...because you're basically checking whether you either have a valid
> > normal iplb, or a valid pv iplb, with the two being mutually exclusive,
> > IIUC. So what about introducing iplb_valid_pv and calling that for the
> > pv case? Would be a bit nicer to read, I think, and also matches what
> > you do for the STORE case.
> >   
> 
> The idea was to get rid of all of these ifs and elses and only have one
> iplb_valid function. Your suggestion would defeat hiding that complexity
> behind this function.

I'd argue that this is a complexity we should not hide; for non-pv, we
can have several formats, for pv, only one, and we cannot use a pv iplb
in a non-pv context and vice versa.

[-- Attachment #2: OpenPGP digital signature --]
[-- Type: application/pgp-signature, Size: 833 bytes --]

Re: [PATCH v9] fixup! Fix subcode/pbt

[Christian Borntraeger]

On 16.03.20 15:54, Cornelia Huck wrote:
> On Mon, 16 Mar 2020 15:47:41 +0100
> Janosch Frank <frankja@linux.ibm.com> wrote:
> 
>> On 3/16/20 3:27 PM, Cornelia Huck wrote:
>>> On Fri, 13 Mar 2020 05:52:32 -0400
>>> Janosch Frank <frankja@linux.ibm.com> wrote:
>>>   
>>>> Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
>>>> ---
>>>>  hw/s390x/ipl.h      | 11 +++++++----
>>>>  target/s390x/diag.c |  2 +-
>>>>  2 files changed, 8 insertions(+), 5 deletions(-)
> 
> 
>>>> @@ -118,7 +118,7 @@ void handle_diag_308(CPUS390XState *env, uint64_t r1, uint64_t r3, uintptr_t ra)
>>>>  
>>>>          cpu_physical_memory_read(addr, iplb, be32_to_cpu(iplb->len));
>>>>  
>>>> -        if (!iplb_valid(iplb)) {
>>>> +        if (!iplb_valid(iplb, subcode)) {
>>>>              env->regs[r1 + 1] = DIAG_308_RC_INVALID;
>>>>              goto out;
>>>>          }  
>>>
>>> ...because you're basically checking whether you either have a valid
>>> normal iplb, or a valid pv iplb, with the two being mutually exclusive,
>>> IIUC. So what about introducing iplb_valid_pv and calling that for the
>>> pv case? Would be a bit nicer to read, I think, and also matches what
>>> you do for the STORE case.
>>>   
>>
>> The idea was to get rid of all of these ifs and elses and only have one
>> iplb_valid function. Your suggestion would defeat hiding that complexity
>> behind this function.
> 
> I'd argue that this is a complexity we should not hide; for non-pv, we
> can have several formats, for pv, only one, and we cannot use a pv iplb
> in a non-pv context and vice versa.

So you suggest to split these case statements?
case DIAG308_STORE:
case DIAG308_PV_STORE:

Re: [PATCH v9] fixup! Fix subcode/pbt

[Janosch Frank]

[-- Attachment #1.1: Type: text/plain, Size: 2005 bytes --]

On 3/16/20 3:54 PM, Cornelia Huck wrote:
> On Mon, 16 Mar 2020 15:47:41 +0100
> Janosch Frank <frankja@linux.ibm.com> wrote:
> 
>> On 3/16/20 3:27 PM, Cornelia Huck wrote:
>>> On Fri, 13 Mar 2020 05:52:32 -0400
>>> Janosch Frank <frankja@linux.ibm.com> wrote:
>>>   
>>>> Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
>>>> ---
>>>>  hw/s390x/ipl.h      | 11 +++++++----
>>>>  target/s390x/diag.c |  2 +-
>>>>  2 files changed, 8 insertions(+), 5 deletions(-)
> 
> 
>>>> @@ -118,7 +118,7 @@ void handle_diag_308(CPUS390XState *env, uint64_t r1, uint64_t r3, uintptr_t ra)
>>>>  
>>>>          cpu_physical_memory_read(addr, iplb, be32_to_cpu(iplb->len));
>>>>  
>>>> -        if (!iplb_valid(iplb)) {
>>>> +        if (!iplb_valid(iplb, subcode)) {
>>>>              env->regs[r1 + 1] = DIAG_308_RC_INVALID;
>>>>              goto out;
>>>>          }  
>>>
>>> ...because you're basically checking whether you either have a valid
>>> normal iplb, or a valid pv iplb, with the two being mutually exclusive,
>>> IIUC. So what about introducing iplb_valid_pv and calling that for the
>>> pv case? Would be a bit nicer to read, I think, and also matches what
>>> you do for the STORE case.
>>>   
>>S390_IPL_TYPE_CCW
>> The idea was to get rid of all of these ifs and elses and only have one
>> iplb_valid function. Your suggestion would defeat hiding that complexity
>> behind this function.
> 
> I'd argue that this is a complexity we should not hide; for non-pv, we
> can have several formats, for pv, only one, and we cannot use a pv iplb
> in a non-pv context and vice versa.
> 

Ok, then please let me split this out into a new function within diag.c.
Something like:

static bool diag308_pbt_subcode_validity(uint8_t pbt, uint64_t subcode)
{
	if (subcode == DIAG308_SET) {
		return (pbt == S390_IPL_TYPE_FCP || pbt == S390_IPL_TYPE_CCW)
	} else if (subcode == DIAG308_PV_SET && pbt == S390_IPL_TYPE_PV) {
	return true;
}

	return false;
}


[-- Attachment #2: OpenPGP digital signature --]
[-- Type: application/pgp-signature, Size: 833 bytes --]

Re: [PATCH v9] s390x: protvirt: Fence huge pages

[Christian Borntraeger]

On 16.03.20 11:06, Janosch Frank wrote:
> On 3/13/20 9:21 AM, Christian Borntraeger wrote:
>>
>>
>> On 12.03.20 17:25, Janosch Frank wrote:
>>> Let's bail out of the protected transition if we detect that huge
>>> pages might be in use.
>>>
>>> Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
>>> ---
>>>
>>> I'd like to squash this into the unpack patch to give a proper error
>>> message if we try to transition into the protected mode while being
>>> backed by huge pages. 
>>
>> Looks good.
>> But maybe we can do it better. Why not reverse the logic and
>> instead of having kvm_s390_get_hpage_1m, let us define an protvirt_allowed
>> that as of today only returns hugepages != 1:
>> Then we could (for kvm-stub.c) also say protvirt_allowed=false;
>> And if other reasons come along we can extend.
>>
>> We could also keep this patch separate, does not really matter.
> 
> The *_allowed() functions are all based on the machine and part of
> s390-virtio-ccw.c so having one in kvm.c looks strange.
> 
> !protvirt_allowed could have any number of reasons in the future, I
> introduced this patch to give a specific error message that can help the
> user to chose the right options when looking for the error.
> 
> Other ideas or a revised one?

I am fine with either variant. If a future machine ever supports protected
guests backed by large pages we can refactor again. In the mean time the kernel
will also return with an error (I think EFAULT on the initial unpack) so this
extra check is just a mean to improve usability. So if you prefer this variant
its fine with me. 

Christian

> 
>>
>>>
>>> ---
>>>  hw/s390x/ipl.h             | 16 ++++++++++++++++
>>>  hw/s390x/s390-virtio-ccw.c |  1 -
>>>  target/s390x/diag.c        | 23 ++++++++---------------
>>>  target/s390x/kvm-stub.c    |  5 +++++
>>>  target/s390x/kvm.c         |  5 +++++
>>>  target/s390x/kvm_s390x.h   |  1 +
>>>  6 files changed, 35 insertions(+), 16 deletions(-)
>>>
>>> diff --git a/hw/s390x/ipl.h b/hw/s390x/ipl.h
>>> index af5bb130a6334821..95e3183c9cccf8b6 100644
>>> --- a/hw/s390x/ipl.h
>>> +++ b/hw/s390x/ipl.h
>>> @@ -185,6 +185,22 @@ struct S390IPLState {
>>>  typedef struct S390IPLState S390IPLState;
>>>  QEMU_BUILD_BUG_MSG(offsetof(S390IPLState, iplb) & 3, "alignment of iplb wrong");
>>>  
>>> +#define DIAG_308_RC_OK              0x0001
>>> +#define DIAG_308_RC_NO_CONF         0x0102
>>> +#define DIAG_308_RC_INVALID         0x0402
>>> +#define DIAG_308_RC_NO_PV_CONF      0x0902
>>> +#define DIAG_308_RC_INVAL_FOR_PV    0x0a02
>>> +
>>> +#define DIAG308_RESET_MOD_CLR       0
>>> +#define DIAG308_RESET_LOAD_NORM     1
>>> +#define DIAG308_LOAD_CLEAR          3
>>> +#define DIAG308_LOAD_NORMAL_DUMP    4
>>> +#define DIAG308_SET                 5
>>> +#define DIAG308_STORE               6
>>> +#define DIAG308_PV_SET              8
>>> +#define DIAG308_PV_STORE            9
>>> +#define DIAG308_PV_START            10
>>> +
>>>  #define S390_IPL_TYPE_FCP 0x00
>>>  #define S390_IPL_TYPE_CCW 0x02
>>>  #define S390_IPL_TYPE_PV 0x05
>>> diff --git a/hw/s390x/s390-virtio-ccw.c b/hw/s390x/s390-virtio-ccw.c
>>> index ebdaaa3a001f6e8c..d32f35c7f47b9c1d 100644
>>> --- a/hw/s390x/s390-virtio-ccw.c
>>> +++ b/hw/s390x/s390-virtio-ccw.c
>>> @@ -361,7 +361,6 @@ out_err:
>>>      return rc;
>>>  }
>>>  
>>> -#define DIAG_308_RC_INVAL_FOR_PV    0x0a02
>>>  static void s390_machine_inject_pv_error(CPUState *cs)
>>>  {
>>>      int r1 = (cs->kvm_run->s390_sieic.ipa & 0x00f0) >> 4;
>>> diff --git a/target/s390x/diag.c b/target/s390x/diag.c
>>> index b245e557037ded06..b1ca81633b83bbdc 100644
>>> --- a/target/s390x/diag.c
>>> +++ b/target/s390x/diag.c
>>> @@ -21,6 +21,7 @@
>>>  #include "hw/s390x/ipl.h"
>>>  #include "hw/s390x/s390-virtio-ccw.h"
>>>  #include "hw/s390x/pv.h"
>>> +#include "kvm_s390x.h"
>>>  
>>>  int handle_diag_288(CPUS390XState *env, uint64_t r1, uint64_t r3)
>>>  {
>>> @@ -50,21 +51,6 @@ int handle_diag_288(CPUS390XState *env, uint64_t r1, uint64_t r3)
>>>      return diag288_class->handle_timer(diag288, func, timeout);
>>>  }
>>>  
>>> -#define DIAG_308_RC_OK              0x0001
>>> -#define DIAG_308_RC_NO_CONF         0x0102
>>> -#define DIAG_308_RC_INVALID         0x0402
>>> -#define DIAG_308_RC_NO_PV_CONF      0x0902
>>> -
>>> -#define DIAG308_RESET_MOD_CLR       0
>>> -#define DIAG308_RESET_LOAD_NORM     1
>>> -#define DIAG308_LOAD_CLEAR          3
>>> -#define DIAG308_LOAD_NORMAL_DUMP    4
>>> -#define DIAG308_SET                 5
>>> -#define DIAG308_STORE               6
>>> -#define DIAG308_PV_SET              8
>>> -#define DIAG308_PV_STORE            9
>>> -#define DIAG308_PV_START            10
>>> -
>>>  static int diag308_parm_check(CPUS390XState *env, uint64_t r1, uint64_t addr,
>>>                                uintptr_t ra, bool write)
>>>  {
>>> @@ -166,6 +152,13 @@ out:
>>>              return;
>>>          }
>>>  
>>> +        if (kvm_s390_get_hpage_1m()) {
>>> +            error_report("Protected VMs can currently not be backed with "
>>> +                         "huge pages");
>>> +            env->regs[r1 + 1] = DIAG_308_RC_INVAL_FOR_PV;
>>> +            return;
>>> +        }
>>> +
>>>          s390_ipl_reset_request(cs, S390_RESET_PV);
>>>          break;
>>>      default:
>>> diff --git a/target/s390x/kvm-stub.c b/target/s390x/kvm-stub.c
>>> index c4cd497f850eb9c7..aa185017a2a886ca 100644
>>> --- a/target/s390x/kvm-stub.c
>>> +++ b/target/s390x/kvm-stub.c
>>> @@ -39,6 +39,11 @@ int kvm_s390_vcpu_interrupt_post_load(S390CPU *cpu)
>>>      return 0;
>>>  }
>>>  
>>> +int kvm_s390_get_hpage_1m(void)
>>> +{
>>> +    return 0;
>>> +}
>>> +
>>>  int kvm_s390_get_ri(void)
>>>  {
>>>      return 0;
>>> diff --git a/target/s390x/kvm.c b/target/s390x/kvm.c
>>> index 1d6fd6a27b48e35f..c695941076b7aead 100644
>>> --- a/target/s390x/kvm.c
>>> +++ b/target/s390x/kvm.c
>>> @@ -321,6 +321,11 @@ void kvm_s390_set_max_pagesize(uint64_t pagesize, Error **errp)
>>>      cap_hpage_1m = 1;
>>>  }
>>>  
>>> +int kvm_s390_get_hpage_1m(void)
>>> +{
>>> +    return cap_hpage_1m;
>>> +}
>>> +
>>>  static void ccw_machine_class_foreach(ObjectClass *oc, void *opaque)
>>>  {
>>>      MachineClass *mc = MACHINE_CLASS(oc);
>>> diff --git a/target/s390x/kvm_s390x.h b/target/s390x/kvm_s390x.h
>>> index 0b21789796d7c462..dea813f450153c34 100644
>>> --- a/target/s390x/kvm_s390x.h
>>> +++ b/target/s390x/kvm_s390x.h
>>> @@ -23,6 +23,7 @@ void kvm_s390_program_interrupt(S390CPU *cpu, uint16_t code);
>>>  int kvm_s390_set_cpu_state(S390CPU *cpu, uint8_t cpu_state);
>>>  void kvm_s390_vcpu_interrupt_pre_save(S390CPU *cpu);
>>>  int kvm_s390_vcpu_interrupt_post_load(S390CPU *cpu);
>>> +int kvm_s390_get_hpage_1m(void);
>>>  int kvm_s390_get_ri(void);
>>>  int kvm_s390_get_gs(void);
>>>  int kvm_s390_get_clock(uint8_t *tod_high, uint64_t *tod_clock);
>>>
>>
>>
> 
> 

Re: [PATCH v9] fixup! Fix subcode/pbt

[Cornelia Huck]

On Mon, 16 Mar 2020 16:04:00 +0100
Christian Borntraeger <borntraeger@de.ibm.com> wrote:

> On 16.03.20 15:54, Cornelia Huck wrote:
> > On Mon, 16 Mar 2020 15:47:41 +0100
> > Janosch Frank <frankja@linux.ibm.com> wrote:
> >   
> >> On 3/16/20 3:27 PM, Cornelia Huck wrote:  
> >>> On Fri, 13 Mar 2020 05:52:32 -0400
> >>> Janosch Frank <frankja@linux.ibm.com> wrote:
> >>>     
> >>>> Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
> >>>> ---
> >>>>  hw/s390x/ipl.h      | 11 +++++++----
> >>>>  target/s390x/diag.c |  2 +-
> >>>>  2 files changed, 8 insertions(+), 5 deletions(-)  
> > 
> >   
> >>>> @@ -118,7 +118,7 @@ void handle_diag_308(CPUS390XState *env, uint64_t r1, uint64_t r3, uintptr_t ra)
> >>>>  
> >>>>          cpu_physical_memory_read(addr, iplb, be32_to_cpu(iplb->len));
> >>>>  
> >>>> -        if (!iplb_valid(iplb)) {
> >>>> +        if (!iplb_valid(iplb, subcode)) {
> >>>>              env->regs[r1 + 1] = DIAG_308_RC_INVALID;
> >>>>              goto out;
> >>>>          }    
> >>>
> >>> ...because you're basically checking whether you either have a valid
> >>> normal iplb, or a valid pv iplb, with the two being mutually exclusive,
> >>> IIUC. So what about introducing iplb_valid_pv and calling that for the
> >>> pv case? Would be a bit nicer to read, I think, and also matches what
> >>> you do for the STORE case.
> >>>     
> >>
> >> The idea was to get rid of all of these ifs and elses and only have one
> >> iplb_valid function. Your suggestion would defeat hiding that complexity
> >> behind this function.  
> > 
> > I'd argue that this is a complexity we should not hide; for non-pv, we
> > can have several formats, for pv, only one, and we cannot use a pv iplb
> > in a non-pv context and vice versa.  
> 
> So you suggest to split these case statements?
> case DIAG308_STORE:
> case DIAG308_PV_STORE:

Why? Those cases are already done in the way I suggest for these here
as well (i.e. keep common checks, just split the iplb handling.)

Re: [PATCH v9] fixup! Fix subcode/pbt

[Cornelia Huck]

[-- Attachment #1: Type: text/plain, Size: 2298 bytes --]

On Mon, 16 Mar 2020 16:05:03 +0100
Janosch Frank <frankja@linux.ibm.com> wrote:

> On 3/16/20 3:54 PM, Cornelia Huck wrote:
> > On Mon, 16 Mar 2020 15:47:41 +0100
> > Janosch Frank <frankja@linux.ibm.com> wrote:
> >   
> >> On 3/16/20 3:27 PM, Cornelia Huck wrote:  
> >>> On Fri, 13 Mar 2020 05:52:32 -0400
> >>> Janosch Frank <frankja@linux.ibm.com> wrote:
> >>>     
> >>>> Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
> >>>> ---
> >>>>  hw/s390x/ipl.h      | 11 +++++++----
> >>>>  target/s390x/diag.c |  2 +-
> >>>>  2 files changed, 8 insertions(+), 5 deletions(-)  
> > 
> >   
> >>>> @@ -118,7 +118,7 @@ void handle_diag_308(CPUS390XState *env, uint64_t r1, uint64_t r3, uintptr_t ra)
> >>>>  
> >>>>          cpu_physical_memory_read(addr, iplb, be32_to_cpu(iplb->len));
> >>>>  
> >>>> -        if (!iplb_valid(iplb)) {
> >>>> +        if (!iplb_valid(iplb, subcode)) {
> >>>>              env->regs[r1 + 1] = DIAG_308_RC_INVALID;
> >>>>              goto out;
> >>>>          }    
> >>>
> >>> ...because you're basically checking whether you either have a valid
> >>> normal iplb, or a valid pv iplb, with the two being mutually exclusive,
> >>> IIUC. So what about introducing iplb_valid_pv and calling that for the
> >>> pv case? Would be a bit nicer to read, I think, and also matches what
> >>> you do for the STORE case.
> >>>     
> >>S390_IPL_TYPE_CCW
> >> The idea was to get rid of all of these ifs and elses and only have one
> >> iplb_valid function. Your suggestion would defeat hiding that complexity
> >> behind this function.  
> > 
> > I'd argue that this is a complexity we should not hide; for non-pv, we
> > can have several formats, for pv, only one, and we cannot use a pv iplb
> > in a non-pv context and vice versa.
> >   
> 
> Ok, then please let me split this out into a new function within diag.c.
> Something like:
> 
> static bool diag308_pbt_subcode_validity(uint8_t pbt, uint64_t subcode)
> {
> 	if (subcode == DIAG308_SET) {
> 		return (pbt == S390_IPL_TYPE_FCP || pbt == S390_IPL_TYPE_CCW)
> 	} else if (subcode == DIAG308_PV_SET && pbt == S390_IPL_TYPE_PV) {
> 	return true;
> }
> 
> 	return false;
> }
> 

Sorry, you now managed to confuse me... where is that supposed to be
called?

[-- Attachment #2: OpenPGP digital signature --]
[-- Type: application/pgp-signature, Size: 833 bytes --]

Re: [PATCH v9] fixup! Fix subcode/pbt

[Christian Borntraeger]

On 16.03.20 18:57, Cornelia Huck wrote:
> On Mon, 16 Mar 2020 16:04:00 +0100
> Christian Borntraeger <borntraeger@de.ibm.com> wrote:
> 
>> On 16.03.20 15:54, Cornelia Huck wrote:
>>> On Mon, 16 Mar 2020 15:47:41 +0100
>>> Janosch Frank <frankja@linux.ibm.com> wrote:
>>>   
>>>> On 3/16/20 3:27 PM, Cornelia Huck wrote:  
>>>>> On Fri, 13 Mar 2020 05:52:32 -0400
>>>>> Janosch Frank <frankja@linux.ibm.com> wrote:
>>>>>     
>>>>>> Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
>>>>>> ---
>>>>>>  hw/s390x/ipl.h      | 11 +++++++----
>>>>>>  target/s390x/diag.c |  2 +-
>>>>>>  2 files changed, 8 insertions(+), 5 deletions(-)  
>>>
>>>   
>>>>>> @@ -118,7 +118,7 @@ void handle_diag_308(CPUS390XState *env, uint64_t r1, uint64_t r3, uintptr_t ra)
>>>>>>  
>>>>>>          cpu_physical_memory_read(addr, iplb, be32_to_cpu(iplb->len));
>>>>>>  
>>>>>> -        if (!iplb_valid(iplb)) {
>>>>>> +        if (!iplb_valid(iplb, subcode)) {
>>>>>>              env->regs[r1 + 1] = DIAG_308_RC_INVALID;
>>>>>>              goto out;
>>>>>>          }    
>>>>>
>>>>> ...because you're basically checking whether you either have a valid
>>>>> normal iplb, or a valid pv iplb, with the two being mutually exclusive,
>>>>> IIUC. So what about introducing iplb_valid_pv and calling that for the
>>>>> pv case? Would be a bit nicer to read, I think, and also matches what
>>>>> you do for the STORE case.
>>>>>     
>>>>
>>>> The idea was to get rid of all of these ifs and elses and only have one
>>>> iplb_valid function. Your suggestion would defeat hiding that complexity
>>>> behind this function.  
>>>
>>> I'd argue that this is a complexity we should not hide; for non-pv, we
>>> can have several formats, for pv, only one, and we cannot use a pv iplb
>>> in a non-pv context and vice versa.  
>>
>> So you suggest to split these case statements?
>> case DIAG308_STORE:
>> case DIAG308_PV_STORE:
> 
> Why? Those cases are already done in the way I suggest for these here
> as well (i.e. keep common checks, just split the iplb handling.)

This was more of a question. I was not sure what your suggestion was.

Re: [PATCH v9] fixup! Fix subcode/pbt

[Cornelia Huck]

On Mon, 16 Mar 2020 20:42:33 +0100
Christian Borntraeger <borntraeger@de.ibm.com> wrote:

> On 16.03.20 18:57, Cornelia Huck wrote:
> > On Mon, 16 Mar 2020 16:04:00 +0100
> > Christian Borntraeger <borntraeger@de.ibm.com> wrote:
> >   
> >> On 16.03.20 15:54, Cornelia Huck wrote:  
> >>> On Mon, 16 Mar 2020 15:47:41 +0100
> >>> Janosch Frank <frankja@linux.ibm.com> wrote:
> >>>     
> >>>> On 3/16/20 3:27 PM, Cornelia Huck wrote:    
> >>>>> On Fri, 13 Mar 2020 05:52:32 -0400
> >>>>> Janosch Frank <frankja@linux.ibm.com> wrote:
> >>>>>       
> >>>>>> Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
> >>>>>> ---
> >>>>>>  hw/s390x/ipl.h      | 11 +++++++----
> >>>>>>  target/s390x/diag.c |  2 +-
> >>>>>>  2 files changed, 8 insertions(+), 5 deletions(-)    
> >>>
> >>>     
> >>>>>> @@ -118,7 +118,7 @@ void handle_diag_308(CPUS390XState *env, uint64_t r1, uint64_t r3, uintptr_t ra)
> >>>>>>  
> >>>>>>          cpu_physical_memory_read(addr, iplb, be32_to_cpu(iplb->len));
> >>>>>>  
> >>>>>> -        if (!iplb_valid(iplb)) {
> >>>>>> +        if (!iplb_valid(iplb, subcode)) {
> >>>>>>              env->regs[r1 + 1] = DIAG_308_RC_INVALID;
> >>>>>>              goto out;
> >>>>>>          }      
> >>>>>
> >>>>> ...because you're basically checking whether you either have a valid
> >>>>> normal iplb, or a valid pv iplb, with the two being mutually exclusive,
> >>>>> IIUC. So what about introducing iplb_valid_pv and calling that for the
> >>>>> pv case? Would be a bit nicer to read, I think, and also matches what
> >>>>> you do for the STORE case.
> >>>>>       
> >>>>
> >>>> The idea was to get rid of all of these ifs and elses and only have one
> >>>> iplb_valid function. Your suggestion would defeat hiding that complexity
> >>>> behind this function.    
> >>>
> >>> I'd argue that this is a complexity we should not hide; for non-pv, we
> >>> can have several formats, for pv, only one, and we cannot use a pv iplb
> >>> in a non-pv context and vice versa.    
> >>
> >> So you suggest to split these case statements?
> >> case DIAG308_STORE:
> >> case DIAG308_PV_STORE:  
> > 
> > Why? Those cases are already done in the way I suggest for these here
> > as well (i.e. keep common checks, just split the iplb handling.)  
> 
> This was more of a question. I was not sure what your suggestion was.

Sorry if I wasn't clear enough.

For the store case, you have

        if (subcode == DIAG308_PV_STORE) {
            iplb = s390_ipl_get_iplb_pv();
        } else {
            iplb = s390_ipl_get_iplb();
        }

with the rest of the handling being identical. My suggestion was to use
something like

        valid = subcode == DIAG308_PV_SET ? iplb_valid_pv(iplb) : iplb_valid(iplb);
        if (!valid) {
             env->regs[r1 + 1] = DIAG_308_RC_INVALID;
             goto out;
         }

Re: [PATCH v9 05/15] s390x: protvirt: KVM intercept changes

[Cornelia Huck]

On Wed, 11 Mar 2020 09:21:41 -0400
Janosch Frank <frankja@linux.ibm.com> wrote:

> Protected VMs no longer intercept with code 4 for an instruction
> interception. Instead they have codes 104 and 108 for protected
> instruction interception and protected instruction notification
> respectively.
> 
> The 104 mirrors the 4 interception.
> 
> The 108 is a notification interception to let KVM and QEMU know that
> something changed and we need to update tracking information or
> perform specific tasks. It's currently taken for the following
> instructions:
> 
> * spx (To inform about the changed prefix location)
> * sclp (On incorrect SCCB values, so we can inject a IRQ)
> * sigp (All but "stop and store status")
> * diag308 (Subcodes 0/1)
> 
> Of these exits only sclp errors, state changing sigps and diag308 will
> reach QEMU. QEMU will do its parts of the job, while the ultravisor
> has done the instruction part of the job.
> 
> Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
> Reviewed-by: David Hildenbrand <david@redhat.com>
> Reviewed-by: Christian Borntraeger <borntraeger@de.ibm.com>
> ---
>  target/s390x/kvm.c | 4 ++++
>  1 file changed, 4 insertions(+)

Reviewed-by: Cornelia Huck <cohuck@redhat.com>

Re: [PATCH v9 06/15] s390x: Add SIDA memory ops

[Cornelia Huck]

On Wed, 11 Mar 2020 09:21:42 -0400
Janosch Frank <frankja@linux.ibm.com> wrote:

> Protected guests save the instruction control blocks in the SIDA
> instead of QEMU/KVM directly accessing the guest's memory.
> 
> Let's introduce new functions to access the SIDA.
> 
> Also the new memops are available with KVM_CAP_S390_PROTECTED, so

"The memops for doing so are available..." ?

> let's check for that.
> 
> Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
> Reviewed-by: David Hildenbrand <david@redhat.com>
> ---
>  target/s390x/cpu.h        |  7 ++++++-
>  target/s390x/kvm.c        | 25 +++++++++++++++++++++++++
>  target/s390x/kvm_s390x.h  |  2 ++
>  target/s390x/mmu_helper.c | 14 ++++++++++++++
>  4 files changed, 47 insertions(+), 1 deletion(-)

Reviewed-by: Cornelia Huck <cohuck@redhat.com>

Re: [PATCH v9 07/15] s390x: protvirt: Move STSI data over SIDAD

[Cornelia Huck]

[-- Attachment #1: Type: text/plain, Size: 1630 bytes --]

On Thu, 12 Mar 2020 12:20:25 +0100
Janosch Frank <frankja@linux.ibm.com> wrote:

> On 3/12/20 11:42 AM, Christian Borntraeger wrote:
> > 
> > 
> > On 11.03.20 14:21, Janosch Frank wrote:  
> >> For protected guests, we need to put the STSI emulation results into
> >> the SIDA, so SIE will write them into the guest at the next entry.
> >>
> >> Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
> >> Reviewed-by: David Hildenbrand <david@redhat.com>
> >> ---
> >>  target/s390x/kvm.c | 11 +++++++++--
> >>  1 file changed, 9 insertions(+), 2 deletions(-)
> >>
> >> diff --git a/target/s390x/kvm.c b/target/s390x/kvm.c
> >> index cdcd538b4f7fb318..8085d5030e7c6454 100644
> >> --- a/target/s390x/kvm.c
> >> +++ b/target/s390x/kvm.c
> >> @@ -50,6 +50,7 @@
> >>  #include "exec/memattrs.h"
> >>  #include "hw/s390x/s390-virtio-ccw.h"
> >>  #include "hw/s390x/s390-virtio-hcall.h"
> >> +#include "hw/s390x/pv.h"
> >>  
> >>  #ifndef DEBUG_KVM
> >>  #define DEBUG_KVM  0
> >> @@ -1800,7 +1801,9 @@ static void insert_stsi_3_2_2(S390CPU *cpu, __u64 addr, uint8_t ar)
> >>      SysIB_322 sysib;
> >>      int del;
> >>  
> >> -    if (s390_cpu_virt_mem_read(cpu, addr, ar, &sysib, sizeof(sysib))) {
> >> +    if (s390_is_pv()) {
> >> +        s390_cpu_pv_mem_read(cpu, 0, &sysib, sizeof(sysib));  
> > 
> > Not strictly necessary, but do we also want to do an early exit if the pv case fails?
> >   
> 
> I'd rather do an early exit for the SIDA read/write ioctl itself

Early exit in what respect? Abort?

If not, checking the return code here and returning looks like
something we want.

[-- Attachment #2: OpenPGP digital signature --]
[-- Type: application/pgp-signature, Size: 833 bytes --]

Re: [PATCH v9 07/15] s390x: protvirt: Move STSI data over SIDAD

[Janosch Frank]

[-- Attachment #1.1: Type: text/plain, Size: 1852 bytes --]

On 3/17/20 11:28 AM, Cornelia Huck wrote:
> On Thu, 12 Mar 2020 12:20:25 +0100
> Janosch Frank <frankja@linux.ibm.com> wrote:
> 
>> On 3/12/20 11:42 AM, Christian Borntraeger wrote:
>>>
>>>
>>> On 11.03.20 14:21, Janosch Frank wrote:  
>>>> For protected guests, we need to put the STSI emulation results into
>>>> the SIDA, so SIE will write them into the guest at the next entry.
>>>>
>>>> Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
>>>> Reviewed-by: David Hildenbrand <david@redhat.com>
>>>> ---
>>>>  target/s390x/kvm.c | 11 +++++++++--
>>>>  1 file changed, 9 insertions(+), 2 deletions(-)
>>>>
>>>> diff --git a/target/s390x/kvm.c b/target/s390x/kvm.c
>>>> index cdcd538b4f7fb318..8085d5030e7c6454 100644
>>>> --- a/target/s390x/kvm.c
>>>> +++ b/target/s390x/kvm.c
>>>> @@ -50,6 +50,7 @@
>>>>  #include "exec/memattrs.h"
>>>>  #include "hw/s390x/s390-virtio-ccw.h"
>>>>  #include "hw/s390x/s390-virtio-hcall.h"
>>>> +#include "hw/s390x/pv.h"
>>>>  
>>>>  #ifndef DEBUG_KVM
>>>>  #define DEBUG_KVM  0
>>>> @@ -1800,7 +1801,9 @@ static void insert_stsi_3_2_2(S390CPU *cpu, __u64 addr, uint8_t ar)
>>>>      SysIB_322 sysib;
>>>>      int del;
>>>>  
>>>> -    if (s390_cpu_virt_mem_read(cpu, addr, ar, &sysib, sizeof(sysib))) {
>>>> +    if (s390_is_pv()) {
>>>> +        s390_cpu_pv_mem_read(cpu, 0, &sysib, sizeof(sysib));  
>>>
>>> Not strictly necessary, but do we also want to do an early exit if the pv case fails?
>>>   
>>
>> I'd rather do an early exit for the SIDA read/write ioctl itself
> 
> Early exit in what respect? Abort?

Yes, abort
If a write fails we most likely will not succeed on the continuation
check and if a read fails we will error out somewhere in qemu anyway

> 
> If not, checking the return code here and returning looks like
> something we want.
> 



[-- Attachment #2: OpenPGP digital signature --]
[-- Type: application/pgp-signature, Size: 833 bytes --]

Re: [PATCH v9 07/15] s390x: protvirt: Move STSI data over SIDAD

[Cornelia Huck]

[-- Attachment #1: Type: text/plain, Size: 1999 bytes --]

On Tue, 17 Mar 2020 11:32:03 +0100
Janosch Frank <frankja@linux.ibm.com> wrote:

> On 3/17/20 11:28 AM, Cornelia Huck wrote:
> > On Thu, 12 Mar 2020 12:20:25 +0100
> > Janosch Frank <frankja@linux.ibm.com> wrote:
> >   
> >> On 3/12/20 11:42 AM, Christian Borntraeger wrote:  
> >>>
> >>>
> >>> On 11.03.20 14:21, Janosch Frank wrote:    
> >>>> For protected guests, we need to put the STSI emulation results into
> >>>> the SIDA, so SIE will write them into the guest at the next entry.
> >>>>
> >>>> Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
> >>>> Reviewed-by: David Hildenbrand <david@redhat.com>
> >>>> ---
> >>>>  target/s390x/kvm.c | 11 +++++++++--
> >>>>  1 file changed, 9 insertions(+), 2 deletions(-)
> >>>>
> >>>> diff --git a/target/s390x/kvm.c b/target/s390x/kvm.c
> >>>> index cdcd538b4f7fb318..8085d5030e7c6454 100644
> >>>> --- a/target/s390x/kvm.c
> >>>> +++ b/target/s390x/kvm.c
> >>>> @@ -50,6 +50,7 @@
> >>>>  #include "exec/memattrs.h"
> >>>>  #include "hw/s390x/s390-virtio-ccw.h"
> >>>>  #include "hw/s390x/s390-virtio-hcall.h"
> >>>> +#include "hw/s390x/pv.h"
> >>>>  
> >>>>  #ifndef DEBUG_KVM
> >>>>  #define DEBUG_KVM  0
> >>>> @@ -1800,7 +1801,9 @@ static void insert_stsi_3_2_2(S390CPU *cpu, __u64 addr, uint8_t ar)
> >>>>      SysIB_322 sysib;
> >>>>      int del;
> >>>>  
> >>>> -    if (s390_cpu_virt_mem_read(cpu, addr, ar, &sysib, sizeof(sysib))) {
> >>>> +    if (s390_is_pv()) {
> >>>> +        s390_cpu_pv_mem_read(cpu, 0, &sysib, sizeof(sysib));    
> >>>
> >>> Not strictly necessary, but do we also want to do an early exit if the pv case fails?
> >>>     
> >>
> >> I'd rather do an early exit for the SIDA read/write ioctl itself  
> > 
> > Early exit in what respect? Abort?  
> 
> Yes, abort
> If a write fails we most likely will not succeed on the continuation
> check and if a read fails we will error out somewhere in qemu anyway

Ok, so this will go into the previous patch, I guess?

[-- Attachment #2: OpenPGP digital signature --]
[-- Type: application/pgp-signature, Size: 833 bytes --]

Re: [PATCH v9 08/15] s390x: protvirt: SCLP interpretation

[Cornelia Huck]

On Fri, 13 Mar 2020 14:14:35 +0100
Christian Borntraeger <borntraeger@de.ibm.com> wrote:

> On 11.03.20 14:21, Janosch Frank wrote:
> > SCLP for a protected guest is done over the SIDAD, so we need to use
> > the s390_cpu_pv_mem_* functions to access the SIDAD instead of guest
> > memory when reading/writing SCBs.
> > 
> > To not confuse the sclp emulation, we set 0x4000 as the SCCB address,
> > since the function that injects the sclp external interrupt would
> > reject a zero sccb address.
> > 
> > Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
> > Reviewed-by: David Hildenbrand <david@redhat.com>
> > ---
> >  hw/s390x/sclp.c         | 30 ++++++++++++++++++++++++++++++
> >  include/hw/s390x/sclp.h |  2 ++
> >  target/s390x/kvm.c      | 24 +++++++++++++++++++-----
> >  3 files changed, 51 insertions(+), 5 deletions(-)

> > +int sclp_service_call_protected(CPUS390XState *env, uint64_t sccb,
> > +                                uint32_t code)
> > +{
> > +    SCLPDevice *sclp = get_sclp_device();
> > +    SCLPDeviceClass *sclp_c = SCLP_GET_CLASS(sclp);
> > +    SCCB work_sccb;
> > +    hwaddr sccb_len = sizeof(SCCB);
> > +
> > +    /*
> > +     * Only a very limited amount of calls is permitted by the
> > +     * Ultravisor and we support all of them, so we don't check for
> > +     * them. All other specification exceptions are also interpreted
> > +     * by the Ultravisor and hence never cause an exit we need to
> > +     * handle.
> > +     *
> > +     * Setting the CC is also done by the Ultravisor.
> > +     */  
> 
> This is fine for the current architecture which specifies a list of sclp 
> commands that are passed through (and this is fine). Question is still if
> we replace this comment with an assertion that this is the case?
> Or maybe even really do the same as sclp_service_call and return 0x1f0 for
> unknown commands?

That would be a case of older QEMU on newer hardware, right? Signaling
that the command is unsupported seems the most reasonable to me
(depending on what the architecture allows.)

> 
> Anyway, whatever you decide.
> 
> Reviewed-by: Christian Borntraeger <borntraeger@de.ibm.com>
> 
> > +    s390_cpu_pv_mem_read(env_archcpu(env), 0, &work_sccb, sccb_len);
> > +    sclp_c->execute(sclp, &work_sccb, code);
> > +    s390_cpu_pv_mem_write(env_archcpu(env), 0, &work_sccb,
> > +                          be16_to_cpu(work_sccb.h.length));
> > +    sclp_c->service_interrupt(sclp, SCLP_PV_DUMMY_ADDR);
> > +    return 0;
> > +}
> > +

Re: [PATCH v9 11/15] s390x: protvirt: Disable address checks for PV guest IO emulation

[Cornelia Huck]

On Wed, 11 Mar 2020 09:21:47 -0400
Janosch Frank <frankja@linux.ibm.com> wrote:

> IO instruction data is routed through SIDAD for protected guests, so
> adresses do not need to be checked, as this is kernel memory.
> 
> Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
> Reviewed-by: Thomas Huth <thuth@redhat.com>
> Reviewed-by: David Hildenbrand <david@redhat.com>
> ---
>  target/s390x/ioinst.c | 33 ++++++++++++++++++++++++++-------
>  1 file changed, 26 insertions(+), 7 deletions(-)
> 
> diff --git a/target/s390x/ioinst.c b/target/s390x/ioinst.c
> index c437a1d8c6afed80..481d789de9e09a04 100644
> --- a/target/s390x/ioinst.c
> +++ b/target/s390x/ioinst.c
> @@ -16,6 +16,23 @@
>  #include "hw/s390x/ioinst.h"
>  #include "trace.h"
>  #include "hw/s390x/s390-pci-bus.h"
> +#include "hw/s390x/pv.h"
> +

Maybe add

/* all I/O instructions but chsc use the s format */

to emphasize that this is not a generic decoder?

> +static uint64_t get_address_from_regs(CPUS390XState *env, uint32_t ipb,
> +                                      uint8_t *ar)
> +{
> +    /*
> +     * Addresses for protected guests are all offsets into the
> +     * satellite block which holds the IO control structures. Those
> +     * control structures are always aligned and accessible, so we can
> +     * return 0 here which will pass the following address checks.
> +     */
> +    if (s390_is_pv()) {
> +        *ar = 0;
> +        return 0;
> +    }
> +    return decode_basedisp_s(env, ipb, ar);
> +}
>  
>  int ioinst_disassemble_sch_ident(uint32_t value, int *m, int *cssid, int *ssid,
>                                   int *schid)

With the other comment updates,

Reviewed-by: Cornelia Huck <cohuck@redhat.com>

Re: [PATCH v9 08/15] s390x: protvirt: SCLP interpretation

[Janosch Frank]

[-- Attachment #1.1: Type: text/plain, Size: 2847 bytes --]

On 3/17/20 12:05 PM, Cornelia Huck wrote:
> On Fri, 13 Mar 2020 14:14:35 +0100
> Christian Borntraeger <borntraeger@de.ibm.com> wrote:
> 
>> On 11.03.20 14:21, Janosch Frank wrote:
>>> SCLP for a protected guest is done over the SIDAD, so we need to use
>>> the s390_cpu_pv_mem_* functions to access the SIDAD instead of guest
>>> memory when reading/writing SCBs.
>>>
>>> To not confuse the sclp emulation, we set 0x4000 as the SCCB address,
>>> since the function that injects the sclp external interrupt would
>>> reject a zero sccb address.
>>>
>>> Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
>>> Reviewed-by: David Hildenbrand <david@redhat.com>
>>> ---
>>>  hw/s390x/sclp.c         | 30 ++++++++++++++++++++++++++++++
>>>  include/hw/s390x/sclp.h |  2 ++
>>>  target/s390x/kvm.c      | 24 +++++++++++++++++++-----
>>>  3 files changed, 51 insertions(+), 5 deletions(-)
> 
>>> +int sclp_service_call_protected(CPUS390XState *env, uint64_t sccb,
>>> +                                uint32_t code)
>>> +{
>>> +    SCLPDevice *sclp = get_sclp_device();
>>> +    SCLPDeviceClass *sclp_c = SCLP_GET_CLASS(sclp);
>>> +    SCCB work_sccb;
>>> +    hwaddr sccb_len = sizeof(SCCB);
>>> +
>>> +    /*
>>> +     * Only a very limited amount of calls is permitted by the
>>> +     * Ultravisor and we support all of them, so we don't check for
>>> +     * them. All other specification exceptions are also interpreted
>>> +     * by the Ultravisor and hence never cause an exit we need to
>>> +     * handle.
>>> +     *
>>> +     * Setting the CC is also done by the Ultravisor.
>>> +     */  
>>
>> This is fine for the current architecture which specifies a list of sclp 
>> commands that are passed through (and this is fine). Question is still if
>> we replace this comment with an assertion that this is the case?
>> Or maybe even really do the same as sclp_service_call and return 0x1f0 for
>> unknown commands?
> 
> That would be a case of older QEMU on newer hardware, right? Signaling
> that the command is unsupported seems the most reasonable to me
> (depending on what the architecture allows.)

Question is if we want to check for the non-pv codes as the hardware
will currently only allow a smaller subset anyway. Then if the IO codes
are passed through by SIE we would support them right away.

> 
>>
>> Anyway, whatever you decide.
>>
>> Reviewed-by: Christian Borntraeger <borntraeger@de.ibm.com>
>>
>>> +    s390_cpu_pv_mem_read(env_archcpu(env), 0, &work_sccb, sccb_len);
>>> +    sclp_c->execute(sclp, &work_sccb, code);
>>> +    s390_cpu_pv_mem_write(env_archcpu(env), 0, &work_sccb,
>>> +                          be16_to_cpu(work_sccb.h.length));
>>> +    sclp_c->service_interrupt(sclp, SCLP_PV_DUMMY_ADDR);
>>> +    return 0;
>>> +}
>>> +
> 
> 



[-- Attachment #2: OpenPGP digital signature --]
[-- Type: application/pgp-signature, Size: 833 bytes --]

Re: [PATCH v9 06/15] s390x: Add SIDA memory ops

[Janosch Frank]

[-- Attachment #1.1: Type: text/plain, Size: 932 bytes --]

On 3/17/20 11:24 AM, Cornelia Huck wrote:
> On Wed, 11 Mar 2020 09:21:42 -0400
> Janosch Frank <frankja@linux.ibm.com> wrote:
> 
>> Protected guests save the instruction control blocks in the SIDA
>> instead of QEMU/KVM directly accessing the guest's memory.
>>
>> Let's introduce new functions to access the SIDA.
>>
>> Also the new memops are available with KVM_CAP_S390_PROTECTED, so

Ack

> 
> "The memops for doing so are available..." ?
> 
>> let's check for that.
>>
>> Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
>> Reviewed-by: David Hildenbrand <david@redhat.com>
>> ---
>>  target/s390x/cpu.h        |  7 ++++++-
>>  target/s390x/kvm.c        | 25 +++++++++++++++++++++++++
>>  target/s390x/kvm_s390x.h  |  2 ++
>>  target/s390x/mmu_helper.c | 14 ++++++++++++++
>>  4 files changed, 47 insertions(+), 1 deletion(-)
> 
> Reviewed-by: Cornelia Huck <cohuck@redhat.com>
> 

Thanks!


[-- Attachment #2: OpenPGP digital signature --]
[-- Type: application/pgp-signature, Size: 833 bytes --]

Re: [PATCH v9 11/15] s390x: protvirt: Disable address checks for PV guest IO emulation

[Janosch Frank]

On 3/17/20 12:36 PM, Cornelia Huck wrote:
> On Wed, 11 Mar 2020 09:21:47 -0400
> Janosch Frank <frankja@linux.ibm.com> wrote:
> 
>> IO instruction data is routed through SIDAD for protected guests, so
>> adresses do not need to be checked, as this is kernel memory.
>>
>> Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
>> Reviewed-by: Thomas Huth <thuth@redhat.com>
>> Reviewed-by: David Hildenbrand <david@redhat.com>
>> ---
>>  target/s390x/ioinst.c | 33 ++++++++++++++++++++++++++-------
>>  1 file changed, 26 insertions(+), 7 deletions(-)
>>
>> diff --git a/target/s390x/ioinst.c b/target/s390x/ioinst.c
>> index c437a1d8c6afed80..481d789de9e09a04 100644
>> --- a/target/s390x/ioinst.c
>> +++ b/target/s390x/ioinst.c
>> @@ -16,6 +16,23 @@
>>  #include "hw/s390x/ioinst.h"
>>  #include "trace.h"
>>  #include "hw/s390x/s390-pci-bus.h"
>> +#include "hw/s390x/pv.h"
>> +
> 
> Maybe add
> 
> /* all I/O instructions but chsc use the s format *>
> to emphasize that this is not a generic decoder?

Ack

> 
>> +static uint64_t get_address_from_regs(CPUS390XState *env, uint32_t ipb,
>> +                                      uint8_t *ar)
>> +{
>> +    /*
>> +     * Addresses for protected guests are all offsets into the
>> +     * satellite block which holds the IO control structures. Those
>> +     * control structures are always aligned and accessible, so we can
>> +     * return 0 here which will pass the following address checks.
>> +     */
>> +    if (s390_is_pv()) {
>> +        *ar = 0;
>> +        return 0;
>> +    }
>> +    return decode_basedisp_s(env, ipb, ar);
>> +}
>>  
>>  int ioinst_disassemble_sch_ident(uint32_t value, int *m, int *cssid, int *ssid,
>>                                   int *schid)
> 
> With the other comment updates,
> 
> Reviewed-by: Cornelia Huck <cohuck@redhat.com>
> 
> 
Thanks

Re: [PATCH v9 08/15] s390x: protvirt: SCLP interpretation

[Cornelia Huck]

[-- Attachment #1: Type: text/plain, Size: 3261 bytes --]

On Tue, 17 Mar 2020 12:54:54 +0100
Janosch Frank <frankja@linux.ibm.com> wrote:

> On 3/17/20 12:05 PM, Cornelia Huck wrote:
> > On Fri, 13 Mar 2020 14:14:35 +0100
> > Christian Borntraeger <borntraeger@de.ibm.com> wrote:
> >   
> >> On 11.03.20 14:21, Janosch Frank wrote:  
> >>> SCLP for a protected guest is done over the SIDAD, so we need to use
> >>> the s390_cpu_pv_mem_* functions to access the SIDAD instead of guest
> >>> memory when reading/writing SCBs.
> >>>
> >>> To not confuse the sclp emulation, we set 0x4000 as the SCCB address,
> >>> since the function that injects the sclp external interrupt would
> >>> reject a zero sccb address.
> >>>
> >>> Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
> >>> Reviewed-by: David Hildenbrand <david@redhat.com>
> >>> ---
> >>>  hw/s390x/sclp.c         | 30 ++++++++++++++++++++++++++++++
> >>>  include/hw/s390x/sclp.h |  2 ++
> >>>  target/s390x/kvm.c      | 24 +++++++++++++++++++-----
> >>>  3 files changed, 51 insertions(+), 5 deletions(-)  
> >   
> >>> +int sclp_service_call_protected(CPUS390XState *env, uint64_t sccb,
> >>> +                                uint32_t code)
> >>> +{
> >>> +    SCLPDevice *sclp = get_sclp_device();
> >>> +    SCLPDeviceClass *sclp_c = SCLP_GET_CLASS(sclp);
> >>> +    SCCB work_sccb;
> >>> +    hwaddr sccb_len = sizeof(SCCB);
> >>> +
> >>> +    /*
> >>> +     * Only a very limited amount of calls is permitted by the
> >>> +     * Ultravisor and we support all of them, so we don't check for
> >>> +     * them. All other specification exceptions are also interpreted
> >>> +     * by the Ultravisor and hence never cause an exit we need to
> >>> +     * handle.
> >>> +     *
> >>> +     * Setting the CC is also done by the Ultravisor.
> >>> +     */    
> >>
> >> This is fine for the current architecture which specifies a list of sclp 
> >> commands that are passed through (and this is fine). Question is still if
> >> we replace this comment with an assertion that this is the case?
> >> Or maybe even really do the same as sclp_service_call and return 0x1f0 for
> >> unknown commands?  
> > 
> > That would be a case of older QEMU on newer hardware, right? Signaling
> > that the command is unsupported seems the most reasonable to me
> > (depending on what the architecture allows.)  
> 
> Question is if we want to check for the non-pv codes as the hardware
> will currently only allow a smaller subset anyway. Then if the IO codes
> are passed through by SIE we would support them right away.

Depending on if the passed-through codes would work without any further
changes, I guess (which seems likely?) You probably have a better idea
about that :)

> 
> >   
> >>
> >> Anyway, whatever you decide.
> >>
> >> Reviewed-by: Christian Borntraeger <borntraeger@de.ibm.com>
> >>  
> >>> +    s390_cpu_pv_mem_read(env_archcpu(env), 0, &work_sccb, sccb_len);
> >>> +    sclp_c->execute(sclp, &work_sccb, code);
> >>> +    s390_cpu_pv_mem_write(env_archcpu(env), 0, &work_sccb,
> >>> +                          be16_to_cpu(work_sccb.h.length));
> >>> +    sclp_c->service_interrupt(sclp, SCLP_PV_DUMMY_ADDR);
> >>> +    return 0;
> >>> +}
> >>> +  
> > 
> >   
> 
> 


[-- Attachment #2: OpenPGP digital signature --]
[-- Type: application/pgp-signature, Size: 833 bytes --]

Re: [PATCH v9 13/15] s390x: protvirt: Handle SIGP store status correctly

[Cornelia Huck]

[-- Attachment #1: Type: text/plain, Size: 1484 bytes --]

On Thu, 12 Mar 2020 17:13:10 +0100
Janosch Frank <frankja@linux.ibm.com> wrote:

> On 3/12/20 4:51 PM, Christian Borntraeger wrote:
> > On 11.03.20 14:21, Janosch Frank wrote:  
> >> For protected VMs status storing is not done by QEMU anymore.
> >>
> >> Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
> >> Reviewed-by: Thomas Huth <thuth@redhat.com>
> >> Reviewed-by: David Hildenbrand <david@redhat.com>  
> > 
> >   
> >> ---
> >>  target/s390x/helper.c | 6 ++++++
> >>  1 file changed, 6 insertions(+)
> >>
> >> diff --git a/target/s390x/helper.c b/target/s390x/helper.c
> >> index ed726849114f2f35..5022df8812d406c9 100644
> >> --- a/target/s390x/helper.c
> >> +++ b/target/s390x/helper.c
> >> @@ -25,6 +25,7 @@
> >>  #include "qemu/timer.h"
> >>  #include "qemu/qemu-print.h"
> >>  #include "hw/s390x/ioinst.h"
> >> +#include "hw/s390x/pv.h"
> >>  #include "sysemu/hw_accel.h"
> >>  #include "sysemu/runstate.h"
> >>  #ifndef CONFIG_USER_ONLY
> >> @@ -246,6 +247,11 @@ int s390_store_status(S390CPU *cpu, hwaddr addr, bool store_arch)
> >>      hwaddr len = sizeof(*sa);
> >>      int i;
> >>  
> >> +    /* Storing will occur on next SIE entry for protected VMs */  
> > 
> > Maybe ... next SIE entry of the sending CPU .... 
> > ?  
> 
> Well that would be the current cpu, right?
> So:
> /* For PVMs storing will occur when this cpu enters SIE again */

With that comment tweak,
Reviewed-by: Cornelia Huck <cohuck@redhat.com>

[-- Attachment #2: OpenPGP digital signature --]
[-- Type: application/pgp-signature, Size: 833 bytes --]

Re: [PATCH v9] s390x: protvirt: Fence huge pages

[Cornelia Huck]

On Thu, 12 Mar 2020 12:25:10 -0400
Janosch Frank <frankja@linux.ibm.com> wrote:

> Let's bail out of the protected transition if we detect that huge
> pages might be in use.
> 
> Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
> ---
> 
> I'd like to squash this into the unpack patch to give a proper error
> message if we try to transition into the protected mode while being
> backed by huge pages. 

Looks sane to me. Folding this into the unpack patch will probably
create less churn.

> 
> ---
>  hw/s390x/ipl.h             | 16 ++++++++++++++++
>  hw/s390x/s390-virtio-ccw.c |  1 -
>  target/s390x/diag.c        | 23 ++++++++---------------
>  target/s390x/kvm-stub.c    |  5 +++++
>  target/s390x/kvm.c         |  5 +++++
>  target/s390x/kvm_s390x.h   |  1 +
>  6 files changed, 35 insertions(+), 16 deletions(-)

Re: [PATCH v9 15/15] s390x: Add unpack facility feature to GA1

[Cornelia Huck]

On Wed, 11 Mar 2020 09:21:51 -0400
Janosch Frank <frankja@linux.ibm.com> wrote:

> From: Christian Borntraeger <borntraeger@de.ibm.com>
> 
> The unpack facility is an indication that diagnose 308 subcodes 8-10
> are available to the guest. That means, that the guest can put itself
> into protected mode.
> 
> Once it is in protected mode, the hardware stops any attempt of VM
> introspection by the hypervisor.
> 
> Some features are currently not supported in protected mode:
>      * Passthrough devices

s/Passthrough/vfio/ ?

>      * Migration
>      * Huge page backings
> 
> Signed-off-by: Christian Borntraeger <borntraeger@de.ibm.com>
> Reviewed-by: David Hildenbrand <david@redhat.com>

Btw: please add your s-o-b if you're passing on patches from others.

> ---
>  target/s390x/gen-features.c | 1 +
>  target/s390x/kvm.c          | 5 +++++
>  2 files changed, 6 insertions(+)
> 
> diff --git a/target/s390x/gen-features.c b/target/s390x/gen-features.c
> index 6278845b12b8dee8..8ddeebc54419a3e2 100644
> --- a/target/s390x/gen-features.c
> +++ b/target/s390x/gen-features.c
> @@ -562,6 +562,7 @@ static uint16_t full_GEN15_GA1[] = {
>      S390_FEAT_GROUP_MSA_EXT_9,
>      S390_FEAT_GROUP_MSA_EXT_9_PCKMO,
>      S390_FEAT_ETOKEN,
> +    S390_FEAT_UNPACK,
>  };
>  
>  /* Default features (in order of release)
> diff --git a/target/s390x/kvm.c b/target/s390x/kvm.c
> index ff6027036ec2f14a..e11e895a3d9038bb 100644
> --- a/target/s390x/kvm.c
> +++ b/target/s390x/kvm.c
> @@ -2403,6 +2403,11 @@ void kvm_s390_get_host_cpu_model(S390CPUModel *model, Error **errp)
>          clear_bit(S390_FEAT_BPB, model->features);
>      }
>  
> +    /* we do have the IPL enhancements */

I'm more confused by that comment than educated :) Not sure what 'IPL
enhancements' means in this context.

> +    if (cap_protected) {
> +        set_bit(S390_FEAT_UNPACK, model->features);
> +    }
> +
>      /* We emulate a zPCI bus and AEN, therefore we don't need HW support */
>      set_bit(S390_FEAT_ZPCI, model->features);
>      set_bit(S390_FEAT_ADAPTER_EVENT_NOTIFICATION, model->features);

Re: [PATCH v9 14/15] docs: Add protvirt docs

[Cornelia Huck]

On Fri, 13 Mar 2020 13:28:56 +0000
Peter Maydell <peter.maydell@linaro.org> wrote:

> On Wed, 11 Mar 2020 at 13:31, Janosch Frank <frankja@linux.ibm.com> wrote:
> >
> > Lets add some documentation for the Protected VM functionality.
> >
> > Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
> > ---
> >  docs/system/index.rst    |  1 +
> >  docs/system/protvirt.rst | 56 ++++++++++++++++++++++++++++++++++++++++
> >  2 files changed, 57 insertions(+)
> >  create mode 100644 docs/system/protvirt.rst
> >
> > diff --git a/docs/system/index.rst b/docs/system/index.rst
> > index 6e5f20fa1333ce23..74afbd7cc3fc0296 100644
> > --- a/docs/system/index.rst
> > +++ b/docs/system/index.rst
> > @@ -34,3 +34,4 @@ Contents:
> >     deprecated
> >     build-platforms
> >     license
> > +   protvirt  
> 
> The order of this list in index.rst determines the order
> of the table of contents in the manual. Could you put
> your new document at a reasonable place in the manual,
> not just at the bottom of the list, please?
> 
> For something that is s390 specific, the ideal would
> be to create a new docs/system/target-s390x.rst
> which is then the place where all s390 docs can
> hang off of. This is how we're handling various
> other guest architecture docs. It doesn't all
> have to go in a single page -- eg target-arm.rst
> is an example of putting some sub-documents into
> docs/system/arm/ and referring to them from
> target-arm.rst.

That sounds like a good idea; let me see whether I can come up with
something that Janosch can base his change on.

(There's already the vfio-ap documentation that's s390x specific; I'll
also move it.)

Re: [PATCH v9 14/15] docs: Add protvirt docs

[Janosch Frank]

[-- Attachment #1.1: Type: text/plain, Size: 1925 bytes --]

On 3/17/20 7:23 PM, Cornelia Huck wrote:
> On Fri, 13 Mar 2020 13:28:56 +0000
> Peter Maydell <peter.maydell@linaro.org> wrote:
> 
>> On Wed, 11 Mar 2020 at 13:31, Janosch Frank <frankja@linux.ibm.com> wrote:
>>>
>>> Lets add some documentation for the Protected VM functionality.
>>>
>>> Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
>>> ---
>>>  docs/system/index.rst    |  1 +
>>>  docs/system/protvirt.rst | 56 ++++++++++++++++++++++++++++++++++++++++
>>>  2 files changed, 57 insertions(+)
>>>  create mode 100644 docs/system/protvirt.rst
>>>
>>> diff --git a/docs/system/index.rst b/docs/system/index.rst
>>> index 6e5f20fa1333ce23..74afbd7cc3fc0296 100644
>>> --- a/docs/system/index.rst
>>> +++ b/docs/system/index.rst
>>> @@ -34,3 +34,4 @@ Contents:
>>>     deprecated
>>>     build-platforms
>>>     license
>>> +   protvirt  
>>
>> The order of this list in index.rst determines the order
>> of the table of contents in the manual. Could you put
>> your new document at a reasonable place in the manual,
>> not just at the bottom of the list, please?
>>
>> For something that is s390 specific, the ideal would
>> be to create a new docs/system/target-s390x.rst
>> which is then the place where all s390 docs can
>> hang off of. This is how we're handling various
>> other guest architecture docs. It doesn't all
>> have to go in a single page -- eg target-arm.rst
>> is an example of putting some sub-documents into
>> docs/system/arm/ and referring to them from
>> target-arm.rst.
> 
> That sounds like a good idea; let me see whether I can come up with
> something that Janosch can base his change on.
> 
> (There's already the vfio-ap documentation that's s390x specific; I'll
> also move it.)
> 

Oh great, I already tried moving the ap documentation but if I can
rebase onto something usable I would be even happier as we save some
mail round trips!


[-- Attachment #2: OpenPGP digital signature --]
[-- Type: application/pgp-signature, Size: 833 bytes --]

Re: [PATCH v9 15/15] s390x: Add unpack facility feature to GA1

[Janosch Frank]

[-- Attachment #1.1: Type: text/plain, Size: 2366 bytes --]

On 3/17/20 7:06 PM, Cornelia Huck wrote:
> On Wed, 11 Mar 2020 09:21:51 -0400
> Janosch Frank <frankja@linux.ibm.com> wrote:
> 
>> From: Christian Borntraeger <borntraeger@de.ibm.com>
>>
>> The unpack facility is an indication that diagnose 308 subcodes 8-10
>> are available to the guest. That means, that the guest can put itself
>> into protected mode.
>>
>> Once it is in protected mode, the hardware stops any attempt of VM
>> introspection by the hypervisor.
>>
>> Some features are currently not supported in protected mode:
>>      * Passthrough devices
> 
> s/Passthrough/vfio/ ?

Ack

> 
>>      * Migration
>>      * Huge page backings
>>
>> Signed-off-by: Christian Borntraeger <borntraeger@de.ibm.com>
>> Reviewed-by: David Hildenbrand <david@redhat.com>
> 
> Btw: please add your s-o-b if you're passing on patches from others.

Ack

> 
>> ---
>>  target/s390x/gen-features.c | 1 +
>>  target/s390x/kvm.c          | 5 +++++
>>  2 files changed, 6 insertions(+)
>>
>> diff --git a/target/s390x/gen-features.c b/target/s390x/gen-features.c
>> index 6278845b12b8dee8..8ddeebc54419a3e2 100644
>> --- a/target/s390x/gen-features.c
>> +++ b/target/s390x/gen-features.c
>> @@ -562,6 +562,7 @@ static uint16_t full_GEN15_GA1[] = {
>>      S390_FEAT_GROUP_MSA_EXT_9,
>>      S390_FEAT_GROUP_MSA_EXT_9_PCKMO,
>>      S390_FEAT_ETOKEN,
>> +    S390_FEAT_UNPACK,
>>  };
>>  
>>  /* Default features (in order of release)
>> diff --git a/target/s390x/kvm.c b/target/s390x/kvm.c
>> index ff6027036ec2f14a..e11e895a3d9038bb 100644
>> --- a/target/s390x/kvm.c
>> +++ b/target/s390x/kvm.c
>> @@ -2403,6 +2403,11 @@ void kvm_s390_get_host_cpu_model(S390CPUModel *model, Error **errp)
>>          clear_bit(S390_FEAT_BPB, model->features);
>>      }
>>  
>> +    /* we do have the IPL enhancements */
> 
> I'm more confused by that comment than educated :) Not sure what 'IPL
> enhancements' means in this context.

/* We do have the protected virtualization ipl unpack facility */
?

> 
>> +    if (cap_protected) {
>> +        set_bit(S390_FEAT_UNPACK, model->features);
>> +    }
>> +
>>      /* We emulate a zPCI bus and AEN, therefore we don't need HW support */
>>      set_bit(S390_FEAT_ZPCI, model->features);
>>      set_bit(S390_FEAT_ADAPTER_EVENT_NOTIFICATION, model->features);
> 



[-- Attachment #2: OpenPGP digital signature --]
[-- Type: application/pgp-signature, Size: 833 bytes --]

Re: [PATCH v9 15/15] s390x: Add unpack facility feature to GA1

[Cornelia Huck]

[-- Attachment #1: Type: text/plain, Size: 1337 bytes --]

On Wed, 18 Mar 2020 09:44:08 +0100
Janosch Frank <frankja@linux.ibm.com> wrote:

> On 3/17/20 7:06 PM, Cornelia Huck wrote:
> > On Wed, 11 Mar 2020 09:21:51 -0400
> > Janosch Frank <frankja@linux.ibm.com> wrote:

> >> diff --git a/target/s390x/kvm.c b/target/s390x/kvm.c
> >> index ff6027036ec2f14a..e11e895a3d9038bb 100644
> >> --- a/target/s390x/kvm.c
> >> +++ b/target/s390x/kvm.c
> >> @@ -2403,6 +2403,11 @@ void kvm_s390_get_host_cpu_model(S390CPUModel *model, Error **errp)
> >>          clear_bit(S390_FEAT_BPB, model->features);
> >>      }
> >>  
> >> +    /* we do have the IPL enhancements */  
> > 
> > I'm more confused by that comment than educated :) Not sure what 'IPL
> > enhancements' means in this context.  
> 
> /* We do have the protected virtualization ipl unpack facility */
> ?

Ah :)

What about

/*
 * If we have support for protected virtualization, indicate
 * the protected virtualization IPL unpack facility.
 */

?

> 
> >   
> >> +    if (cap_protected) {
> >> +        set_bit(S390_FEAT_UNPACK, model->features);
> >> +    }
> >> +
> >>      /* We emulate a zPCI bus and AEN, therefore we don't need HW support */
> >>      set_bit(S390_FEAT_ZPCI, model->features);
> >>      set_bit(S390_FEAT_ADAPTER_EVENT_NOTIFICATION, model->features);  
> >   
> 
> 


[-- Attachment #2: OpenPGP digital signature --]
[-- Type: application/pgp-signature, Size: 833 bytes --]

Re: [PATCH v9 15/15] s390x: Add unpack facility feature to GA1

[Janosch Frank]

[-- Attachment #1.1: Type: text/plain, Size: 206 bytes --]

On 3/18/20 10:27 AM, Cornelia Huck wrote:
> What about
> 
> /*
>  * If we have support for protected virtualization, indicate
>  * the protected virtualization IPL unpack facility.
>  */
> 
Sure


[-- Attachment #2: OpenPGP digital signature --]
[-- Type: application/pgp-signature, Size: 833 bytes --]

Re: [PATCH v9 14/15] docs: Add protvirt docs

[Cornelia Huck]

On Wed, 11 Mar 2020 09:21:50 -0400
Janosch Frank <frankja@linux.ibm.com> wrote:

> Lets add some documentation for the Protected VM functionality.

s/Lets/Let's/ :)

> 
> Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
> ---
>  docs/system/index.rst    |  1 +
>  docs/system/protvirt.rst | 56 ++++++++++++++++++++++++++++++++++++++++
>  2 files changed, 57 insertions(+)
>  create mode 100644 docs/system/protvirt.rst
> 
(...)
> diff --git a/docs/system/protvirt.rst b/docs/system/protvirt.rst
> new file mode 100644
> index 0000000000000000..6c8cf0f7910eae86
> --- /dev/null
> +++ b/docs/system/protvirt.rst
> @@ -0,0 +1,56 @@
> +Protected Virtualization on s390x
> +=================================
> +
> +The memory and most of the registers of Protected Virtual Machines
> +(PVMs) are encrypted or inaccessible to the hypervisor, effectively
> +prohibiting VM introspection when the VM is running. At rest, PVMs are
> +encrypted and can only be decrypted by the firmware, represented by an
> +entity called Ultravisor, of specific IBM Z machines.

Is that specific _generations_ or specific _machines_ (i.e. those with
an appropriate key)? If the latter, maybe add a note that a specific
image is bound to (a) specific machine(s)?


> +
> +
> +Prerequisites
> +-------------

Also, maybe add

"Running PVMs requires using the KVM hypervisor."

?

> +
> +To run PVMs a machine with the Protected Virtualization feature
> +which is indicated by the Ultravisor Call facility (stfle bit

maybe s/which is/as/ ?

> +158) is required. The Ultravisor needs to be initialized at boot by
> +setting `prot_virt=1` on the kernel command line.
> +
> +If those requirements are met, the capability `KVM_CAP_S390_PROTECTED`
> +will indicate that KVM can support PVMs on that LPAR.
> +
> +
> +QEMU Settings
> +-------------
> +
> +To indicate to the VM that it can transition into protected mode, the
> +`Unpack facility` (stfle bit 161 represented by the feature
> +`S390_FEAT_UNPACK`) needs to be part of the cpu model of the VM.
> +
> +All I/O devices need to use the IOMMU.
> +Passthrough (vfio) devices are currently not supported.
> +
> +Host huge page backings are not supported. However guests can use huge
> +pages as indicated by its facilities.
> +
> +
> +Boot Process
> +------------
> +
> +A secure guest image can either be loaded from disk or supplied on the
> +QEMU command line. Booting from disk is done by the unmodified
> +s390-ccw BIOS. I.e., the bootmap is interpreted, multiple components
> +are read into memory and control is transferred to one of the
> +components (zipl stage3). Stag3 does some fixups and then transfers

s/Stag3/Stage3/

> +control to some program residing in guest memory, which is normally
> +the OS kernel. The secure image has another component prepended
> +(stage3a) that uses the new diag308 subcodes 8 and 10 to trigger the
> +transition into secure mode.
> +
> +Booting from the image supplied via the QEMU command line requires
> +that the file passed via -kernel has the same memory layout as would
> +result from the disk boot. This memory layout includes the encrypted
> +components (kernel, initrd, cmdline), the stage3a loader and
> +metadata. In case this boot method is used, the command line
> +options -initrd and -cmdline are ineffective. The preparation of a PVM
> +image is done by genprotimg of the s390-tools package.

"is done via the `genprotimg` tool from the s390-tools collection"

?

Re: [PATCH v9 04/15] s390x: protvirt: Inhibit balloon when switching to protected mode

[Cornelia Huck]

On Wed, 11 Mar 2020 09:21:40 -0400
Janosch Frank <frankja@linux.ibm.com> wrote:

> Ballooning in protected VMs can only be done when the guest shares the
> pages it gives to the host. If pages are not shared, the integrity
> checks will fail once those pages have been altered and are given back
> to the guest.
> 
> As we currently do not yet have a solution for this we will continue
> like this:
> 
> 1. We block ballooning now in QEMU (with this patch)

add trailing '.'

> 
> 2. Later we will provide a change to virtio that removes the blocker

s/Later/Later,/

> and adds VIRTIO_F_IOMMU_PLATFORM automatically by QEMU when doing the
> protvirt switch. This is ok as the guest balloon driver will reject to
> work with the IOMMU change

"This is OK, as the balloon driver in Linux (the only supported guest)
will refuse to work with the IOMMU_PLATFORM feature bit set."

?

> 
> 3. Later we can fix the guest balloon driver to accept the IOMMU

"Later, we can fix the Linux guest balloon driver..."

> feature bit and correctly exercise sharing and unsharing of balloon
> pages

again, missing trailing '.'

> 
> Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
> Reviewed-by: David Hildenbrand <david@redhat.com>
> Reviewed-by: Christian Borntraeger <borntraeger@de.ibm.com>
> ---
>  hw/s390x/s390-virtio-ccw.c | 5 +++++
>  1 file changed, 5 insertions(+)
> 
> diff --git a/hw/s390x/s390-virtio-ccw.c b/hw/s390x/s390-virtio-ccw.c
> index deb31e060052d279..066e01f303c35671 100644
> --- a/hw/s390x/s390-virtio-ccw.c
> +++ b/hw/s390x/s390-virtio-ccw.c
> @@ -41,6 +41,7 @@
>  #include "hw/qdev-properties.h"
>  #include "hw/s390x/tod.h"
>  #include "sysemu/sysemu.h"
> +#include "sysemu/balloon.h"
>  #include "hw/s390x/pv.h"
>  #include "migration/blocker.h"
>  
> @@ -326,6 +327,7 @@ static void s390_machine_unprotect(S390CcwMachineState *ms)
>      ms->pv = false;
>      migrate_del_blocker(pv_mig_blocker);
>      error_free_or_abort(&pv_mig_blocker);
> +    qemu_balloon_inhibit(false);
>  }
>  
>  static int s390_machine_protect(S390CcwMachineState *ms)
> @@ -333,10 +335,12 @@ static int s390_machine_protect(S390CcwMachineState *ms)
>      Error *local_err = NULL;
>      int rc;
>

Maybe add a comment?

/*
 * Ballooning on protected VMs needs support in the guest for
 * sharing and unsharing balloon pages. Block ballooning for
 * now, until we have a solution to make at least Linux guests
 * either support it or fail gracefully.
 */
  
> +    qemu_balloon_inhibit(true);
>      error_setg(&pv_mig_blocker,
>                 "protected VMs are currently not migrateable.");
>      rc = migrate_add_blocker(pv_mig_blocker, &local_err);
>      if (local_err) {
> +        qemu_balloon_inhibit(false);
>          error_report_err(local_err);
>          error_free_or_abort(&pv_mig_blocker);
>          return rc;
> @@ -345,6 +349,7 @@ static int s390_machine_protect(S390CcwMachineState *ms)
>      /* Create SE VM */
>      rc = s390_pv_vm_enable();
>      if (rc) {
> +        qemu_balloon_inhibit(false);
>          error_report_err(local_err);
>          migrate_del_blocker(pv_mig_blocker);
>          error_free_or_abort(&pv_mig_blocker);

Re: [PATCH v9 04/15] s390x: protvirt: Inhibit balloon when switching to protected mode

[Janosch Frank]

[-- Attachment #1.1: Type: text/plain, Size: 3458 bytes --]

On 3/18/20 12:42 PM, Cornelia Huck wrote:
> On Wed, 11 Mar 2020 09:21:40 -0400
> Janosch Frank <frankja@linux.ibm.com> wrote:
> 
>> Ballooning in protected VMs can only be done when the guest shares the
>> pages it gives to the host. If pages are not shared, the integrity
>> checks will fail once those pages have been altered and are given back
>> to the guest.
>>
>> As we currently do not yet have a solution for this we will continue
>> like this:
>>
>> 1. We block ballooning now in QEMU (with this patch)
> 
> add trailing '.'
> 
>>
>> 2. Later we will provide a change to virtio that removes the blocker
> 
> s/Later/Later,/
> 
>> and adds VIRTIO_F_IOMMU_PLATFORM automatically by QEMU when doing the
>> protvirt switch. This is ok as the guest balloon driver will reject to
>> work with the IOMMU change
> 
> "This is OK, as the balloon driver in Linux (the only supported guest)
> will refuse to work with the IOMMU_PLATFORM feature bit set."
> 
> ?
> 
>>
>> 3. Later we can fix the guest balloon driver to accept the IOMMU
> 
> "Later, we can fix the Linux guest balloon driver..."
> 
>> feature bit and correctly exercise sharing and unsharing of balloon
>> pages
> 
> again, missing trailing '.'

All nits fixed

> 
>>
>> Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
>> Reviewed-by: David Hildenbrand <david@redhat.com>
>> Reviewed-by: Christian Borntraeger <borntraeger@de.ibm.com>
>> ---
>>  hw/s390x/s390-virtio-ccw.c | 5 +++++
>>  1 file changed, 5 insertions(+)
>>
>> diff --git a/hw/s390x/s390-virtio-ccw.c b/hw/s390x/s390-virtio-ccw.c
>> index deb31e060052d279..066e01f303c35671 100644
>> --- a/hw/s390x/s390-virtio-ccw.c
>> +++ b/hw/s390x/s390-virtio-ccw.c
>> @@ -41,6 +41,7 @@
>>  #include "hw/qdev-properties.h"
>>  #include "hw/s390x/tod.h"
>>  #include "sysemu/sysemu.h"
>> +#include "sysemu/balloon.h"
>>  #include "hw/s390x/pv.h"
>>  #include "migration/blocker.h"
>>  
>> @@ -326,6 +327,7 @@ static void s390_machine_unprotect(S390CcwMachineState *ms)
>>      ms->pv = false;
>>      migrate_del_blocker(pv_mig_blocker);
>>      error_free_or_abort(&pv_mig_blocker);
>> +    qemu_balloon_inhibit(false);
>>  }
>>  
>>  static int s390_machine_protect(S390CcwMachineState *ms)
>> @@ -333,10 +335,12 @@ static int s390_machine_protect(S390CcwMachineState *ms)
>>      Error *local_err = NULL;
>>      int rc;
>>
> 
> Maybe add a comment?
> 
> /*
>  * Ballooning on protected VMs needs support in the guest for
>  * sharing and unsharing balloon pages. Block ballooning for
>  * now, until we have a solution to make at least Linux guests
>  * either support it or fail gracefully.
>  */
>   
>> +    qemu_balloon_inhibit(true);
>>      error_setg(&pv_mig_blocker,
>>                 "protected VMs are currently not migrateable.");
>>      rc = migrate_add_blocker(pv_mig_blocker, &local_err);
>>      if (local_err) {
>> +        qemu_balloon_inhibit(false);
>>          error_report_err(local_err);
>>          error_free_or_abort(&pv_mig_blocker);
>>          return rc;
>> @@ -345,6 +349,7 @@ static int s390_machine_protect(S390CcwMachineState *ms)
>>      /* Create SE VM */
>>      rc = s390_pv_vm_enable();
>>      if (rc) {
>> +        qemu_balloon_inhibit(false);
>>          error_report_err(local_err);
>>          migrate_del_blocker(pv_mig_blocker);
>>          error_free_or_abort(&pv_mig_blocker);
> 



[-- Attachment #2: OpenPGP digital signature --]
[-- Type: application/pgp-signature, Size: 833 bytes --]

Re: [PATCH v9 14/15] docs: Add protvirt docs

[Janosch Frank]

[-- Attachment #1.1: Type: text/plain, Size: 3762 bytes --]

On 3/18/20 12:25 PM, Cornelia Huck wrote:
> On Wed, 11 Mar 2020 09:21:50 -0400
> Janosch Frank <frankja@linux.ibm.com> wrote:
> 
>> Lets add some documentation for the Protected VM functionality.
> 
> s/Lets/Let's/ :)
> 
>>
>> Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
>> ---
>>  docs/system/index.rst    |  1 +
>>  docs/system/protvirt.rst | 56 ++++++++++++++++++++++++++++++++++++++++
>>  2 files changed, 57 insertions(+)
>>  create mode 100644 docs/system/protvirt.rst
>>
> (...)
>> diff --git a/docs/system/protvirt.rst b/docs/system/protvirt.rst
>> new file mode 100644
>> index 0000000000000000..6c8cf0f7910eae86
>> --- /dev/null
>> +++ b/docs/system/protvirt.rst
>> @@ -0,0 +1,56 @@
>> +Protected Virtualization on s390x
>> +=================================
>> +
>> +The memory and most of the registers of Protected Virtual Machines
>> +(PVMs) are encrypted or inaccessible to the hypervisor, effectively
>> +prohibiting VM introspection when the VM is running. At rest, PVMs are
>> +encrypted and can only be decrypted by the firmware, represented by an
>> +entity called Ultravisor, of specific IBM Z machines.
> 
> Is that specific _generations_ or specific _machines_ (i.e. those with
> an appropriate key)? If the latter, maybe add a note that a specific
> image is bound to (a) specific machine(s)?
> 
> 
>> +
>> +
>> +Prerequisites
>> +-------------
> 
> Also, maybe add
> 
> "Running PVMs requires using the KVM hypervisor."
> 
> ?
> 
>> +
>> +To run PVMs a machine with the Protected Virtualization feature
>> +which is indicated by the Ultravisor Call facility (stfle bit
> 
> maybe s/which is/as/ ?
> 
>> +158) is required. The Ultravisor needs to be initialized at boot by
>> +setting `prot_virt=1` on the kernel command line.
>> +
>> +If those requirements are met, the capability `KVM_CAP_S390_PROTECTED`
>> +will indicate that KVM can support PVMs on that LPAR.
>> +
>> +
>> +QEMU Settings
>> +-------------
>> +
>> +To indicate to the VM that it can transition into protected mode, the
>> +`Unpack facility` (stfle bit 161 represented by the feature
>> +`S390_FEAT_UNPACK`) needs to be part of the cpu model of the VM.
>> +
>> +All I/O devices need to use the IOMMU.
>> +Passthrough (vfio) devices are currently not supported.
>> +
>> +Host huge page backings are not supported. However guests can use huge
>> +pages as indicated by its facilities.
>> +
>> +
>> +Boot Process
>> +------------
>> +
>> +A secure guest image can either be loaded from disk or supplied on the
>> +QEMU command line. Booting from disk is done by the unmodified
>> +s390-ccw BIOS. I.e., the bootmap is interpreted, multiple components
>> +are read into memory and control is transferred to one of the
>> +components (zipl stage3). Stag3 does some fixups and then transfers
> 
> s/Stag3/Stage3/
> 
>> +control to some program residing in guest memory, which is normally
>> +the OS kernel. The secure image has another component prepended
>> +(stage3a) that uses the new diag308 subcodes 8 and 10 to trigger the
>> +transition into secure mode.
>> +
>> +Booting from the image supplied via the QEMU command line requires
>> +that the file passed via -kernel has the same memory layout as would
>> +result from the disk boot. This memory layout includes the encrypted
>> +components (kernel, initrd, cmdline), the stage3a loader and
>> +metadata. In case this boot method is used, the command line
>> +options -initrd and -cmdline are ineffective. The preparation of a PVM
>> +image is done by genprotimg of the s390-tools package.
> 
> "is done via the `genprotimg` tool from the s390-tools collection"
> 
> ?

Too many cooks :)
Ack to all



[-- Attachment #2: OpenPGP digital signature --]
[-- Type: application/pgp-signature, Size: 833 bytes --]