[PATCH 00/15] s390x: Protected Virtualization support

[PATCH 00/15] s390x: Protected Virtualization support

[Janosch Frank]

Most of the QEMU changes for PV are related to the new IPL type with
subcodes 8 - 10 and the execution of the necessary Ultravisor calls to
IPL secure guests. Note that we can only boot into secure mode from
normal mode, i.e. stfle 161 is not active in secure mode.

The other changes related to data gathering for emulation and
disabling addressing checks in secure mode, as well as CPU resets.

While working on this I sprinkled in some cleanups, as we sometimes
significantly increase line count of some functions and they got
unreadable.

Janosch Frank (15):
  s390x: Cleanup cpu resets
  s390x: Beautify diag308 handling
  s390x: protvirt: Add diag308 subcodes 8 - 10
  Header sync protvirt
  s390x: protvirt: Sync PV state
  s390x: protvirt: Support unpack facility
  s390x: protvirt: Handle diag 308 subcodes 0,1,3,4
  s390x: protvirt: KVM intercept changes
  s390x: protvirt: SCLP interpretation
  s390x: protvirt: Add new VCPU reset functions
  RFC: s390x: Exit on vcpu reset error
  s390x: protvirt: Set guest IPL PSW
  s390x: protvirt: Move diag 308 data over SIDAD
  s390x: protvirt: Disable address checks for PV guest IO emulation
  s390x: protvirt: Handle SIGP store status correctly

 hw/s390x/Makefile.objs              |   1 +
 hw/s390x/ipl.c                      |  81 +++++++++++++++++-
 hw/s390x/ipl.h                      |  35 ++++++++
 hw/s390x/pv.c                       | 123 +++++++++++++++++++++++++++
 hw/s390x/pv.h                       |  27 ++++++
 hw/s390x/s390-virtio-ccw.c          |  79 ++++++++++++++---
 hw/s390x/sclp.c                     |  16 ++++
 include/hw/s390x/sclp.h             |   2 +
 linux-headers/asm-s390/kvm.h        |   4 +-
 linux-headers/linux/kvm.h           |  43 ++++++++++
 target/s390x/cpu.c                  | 127 ++++++++++++++--------------
 target/s390x/cpu.h                  |   1 +
 target/s390x/cpu_features_def.inc.h |   1 +
 target/s390x/diag.c                 | 108 +++++++++++++++++------
 target/s390x/ioinst.c               |  46 ++++++----
 target/s390x/kvm-stub.c             |  10 ++-
 target/s390x/kvm.c                  |  58 +++++++++++--
 target/s390x/kvm_s390x.h            |   4 +-
 target/s390x/sigp.c                 |   7 +-
 19 files changed, 640 insertions(+), 133 deletions(-)
 create mode 100644 hw/s390x/pv.c
 create mode 100644 hw/s390x/pv.h

-- 
2.20.1

[PATCH 01/15] s390x: Cleanup cpu resets

[Janosch Frank]

Let's move the resets into one function and switch by type, so we can
use fallthroughs for shared reset actions.

Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
---
 hw/s390x/s390-virtio-ccw.c |   3 +
 target/s390x/cpu.c         | 111 ++++++++++++++++---------------------
 2 files changed, 52 insertions(+), 62 deletions(-)

diff --git a/hw/s390x/s390-virtio-ccw.c b/hw/s390x/s390-virtio-ccw.c
index d3edeef0ad..c1d1440272 100644
--- a/hw/s390x/s390-virtio-ccw.c
+++ b/hw/s390x/s390-virtio-ccw.c
@@ -348,6 +348,9 @@ static void s390_machine_reset(MachineState *machine)
         break;
     case S390_RESET_LOAD_NORMAL:
         CPU_FOREACH(t) {
+            if (t == cs) {
+                continue;
+            }
             run_on_cpu(t, s390_do_cpu_reset, RUN_ON_CPU_NULL);
         }
         subsystem_reset();
diff --git a/target/s390x/cpu.c b/target/s390x/cpu.c
index 3abe7e80fd..10d5b915d8 100644
--- a/target/s390x/cpu.c
+++ b/target/s390x/cpu.c
@@ -82,67 +82,53 @@ static void s390_cpu_load_normal(CPUState *s)
 }
 #endif
 
-/* S390CPUClass::cpu_reset() */
-static void s390_cpu_reset(CPUState *s)
+enum {
+    S390_CPU_RESET_NORMAL,
+    S390_CPU_RESET_INITIAL,
+    S390_CPU_RESET_CLEAR,
+};
+
+static void s390_cpu_reset(CPUState *s, uint8_t type)
 {
     S390CPU *cpu = S390_CPU(s);
     S390CPUClass *scc = S390_CPU_GET_CLASS(cpu);
     CPUS390XState *env = &cpu->env;
 
-    env->pfault_token = -1UL;
-    env->bpbc = false;
     scc->parent_reset(s);
     cpu->env.sigp_order = 0;
     s390_cpu_set_state(S390_CPU_STATE_STOPPED, cpu);
-}
 
-/* S390CPUClass::initial_reset() */
-static void s390_cpu_initial_reset(CPUState *s)
-{
-    S390CPU *cpu = S390_CPU(s);
-    CPUS390XState *env = &cpu->env;
+    /* Set initial values after clearing */
+    switch (type) {
+    case S390_CPU_RESET_CLEAR:
+        /* Fallthrough will clear the rest */
+        memset(env, 0, offsetof(CPUS390XState, start_initial_reset_fields));
+        /* Fallthrough */
+    case S390_CPU_RESET_INITIAL:
+        memset(&env->start_initial_reset_fields, 0,
+               offsetof(CPUS390XState, end_reset_fields) -
+               offsetof(CPUS390XState, start_initial_reset_fields));
+        /* architectured initial values for CR 0 and 14 */
+        env->cregs[0] = CR0_RESET;
+        env->cregs[14] = CR14_RESET;
 
-    s390_cpu_reset(s);
-    /* initial reset does not clear everything! */
-    memset(&env->start_initial_reset_fields, 0,
-        offsetof(CPUS390XState, end_reset_fields) -
-        offsetof(CPUS390XState, start_initial_reset_fields));
-
-    /* architectured initial values for CR 0 and 14 */
-    env->cregs[0] = CR0_RESET;
-    env->cregs[14] = CR14_RESET;
-
-    /* architectured initial value for Breaking-Event-Address register */
-    env->gbea = 1;
-
-    env->pfault_token = -1UL;
-
-    /* tininess for underflow is detected before rounding */
-    set_float_detect_tininess(float_tininess_before_rounding,
-                              &env->fpu_status);
+        /* architectured initial value for Breaking-Event-Address register */
+        env->gbea = 1;
+        /* tininess for underflow is detected before rounding */
+        set_float_detect_tininess(float_tininess_before_rounding,
+                                  &env->fpu_status);
+        /* Fallthrough */
+    case S390_CPU_RESET_NORMAL:
+        env->pfault_token = -1UL;
+        env->bpbc = false;
+        break;
+    }
 
     /* Reset state inside the kernel that we cannot access yet from QEMU. */
-    if (kvm_enabled()) {
-        kvm_s390_reset_vcpu(cpu);
+    if (kvm_enabled() && (type == S390_CPU_RESET_CLEAR ||
+                          type == S390_CPU_RESET_INITIAL)) {
+            kvm_s390_reset_vcpu(cpu);
     }
-}
-
-/* CPUClass:reset() */
-static void s390_cpu_full_reset(CPUState *s)
-{
-    S390CPU *cpu = S390_CPU(s);
-    S390CPUClass *scc = S390_CPU_GET_CLASS(cpu);
-    CPUS390XState *env = &cpu->env;
-
-    scc->parent_reset(s);
-    cpu->env.sigp_order = 0;
-    s390_cpu_set_state(S390_CPU_STATE_STOPPED, cpu);
-
-    memset(env, 0, offsetof(CPUS390XState, end_reset_fields));
-
-    /* architectured initial values for CR 0 and 14 */
-    env->cregs[0] = CR0_RESET;
-    env->cregs[14] = CR14_RESET;
 
 #if defined(CONFIG_USER_ONLY)
     /* user mode should always be allowed to use the full FPU */
@@ -151,20 +137,21 @@ static void s390_cpu_full_reset(CPUState *s)
         env->cregs[0] |= CR0_VECTOR;
     }
 #endif
+}
 
-    /* architectured initial value for Breaking-Event-Address register */
-    env->gbea = 1;
+static void s390_cpu_reset_normal(CPUState *s)
+{
+    return s390_cpu_reset(s, S390_CPU_RESET_NORMAL);
+}
 
-    env->pfault_token = -1UL;
+static void s390_cpu_reset_initial(CPUState *s)
+{
+    return s390_cpu_reset(s, S390_CPU_RESET_INITIAL);
+}
 
-    /* tininess for underflow is detected before rounding */
-    set_float_detect_tininess(float_tininess_before_rounding,
-                              &env->fpu_status);
-
-    /* Reset state inside the kernel that we cannot access yet from QEMU. */
-    if (kvm_enabled()) {
-        kvm_s390_reset_vcpu(cpu);
-    }
+static void s390_cpu_reset_clear(CPUState *s)
+{
+    return s390_cpu_reset(s, S390_CPU_RESET_CLEAR);
 }
 
 #if !defined(CONFIG_USER_ONLY)
@@ -473,9 +460,9 @@ static void s390_cpu_class_init(ObjectClass *oc, void *data)
 #if !defined(CONFIG_USER_ONLY)
     scc->load_normal = s390_cpu_load_normal;
 #endif
-    scc->cpu_reset = s390_cpu_reset;
-    scc->initial_cpu_reset = s390_cpu_initial_reset;
-    cc->reset = s390_cpu_full_reset;
+    scc->cpu_reset = s390_cpu_reset_normal;
+    scc->initial_cpu_reset = s390_cpu_reset_initial;
+    cc->reset = s390_cpu_reset_clear;
     cc->class_by_name = s390_cpu_class_by_name,
     cc->has_work = s390_cpu_has_work;
 #ifdef CONFIG_TCG
-- 
2.20.1

[PATCH 02/15] s390x: Beautify diag308 handling

[Janosch Frank]

Let's improve readability by:
* Using constants for the subcodes
* Moving parameter checking into a function
* Removing subcode > 6 check as the default case catches that

Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
---
 target/s390x/diag.c | 54 +++++++++++++++++++++++++++------------------
 1 file changed, 32 insertions(+), 22 deletions(-)

diff --git a/target/s390x/diag.c b/target/s390x/diag.c
index 53c2f81f2a..067c667ba7 100644
--- a/target/s390x/diag.c
+++ b/target/s390x/diag.c
@@ -53,6 +53,29 @@ int handle_diag_288(CPUS390XState *env, uint64_t r1, uint64_t r3)
 #define DIAG_308_RC_NO_CONF         0x0102
 #define DIAG_308_RC_INVALID         0x0402
 
+#define DIAG308_RES_MOD_CLR		0
+#define DIAG308_RES_LOAD_NORM		1
+#define DIAG308_LOAD_CLEAR		3
+#define DIAG308_LOAD_NORMAL_DUMP	4
+#define DIAG308_SET			5
+#define DIAG308_STORE			6
+
+static int diag308_parm_check(CPUS390XState *env, uint64_t r1, uint64_t addr,
+                              uintptr_t ra, bool write)
+{
+    if ((r1 & 1) || (addr & ~TARGET_PAGE_MASK)) {
+        s390_program_interrupt(env, PGM_SPECIFICATION, ra);
+        return -EINVAL;
+    }
+    if (!address_space_access_valid(&address_space_memory, addr,
+                                    sizeof(IplParameterBlock), write,
+                                    MEMTXATTRS_UNSPECIFIED)) {
+        s390_program_interrupt(env, PGM_ADDRESSING, ra);
+        return -EINVAL;
+    }
+    return 0;
+}
+
 void handle_diag_308(CPUS390XState *env, uint64_t r1, uint64_t r3, uintptr_t ra)
 {
     CPUState *cs = env_cpu(env);
@@ -65,30 +88,24 @@ void handle_diag_308(CPUS390XState *env, uint64_t r1, uint64_t r3, uintptr_t ra)
         return;
     }
 
-    if ((subcode & ~0x0ffffULL) || (subcode > 6)) {
+    if (subcode & ~0x0ffffULL) {
         s390_program_interrupt(env, PGM_SPECIFICATION, ra);
         return;
     }
 
     switch (subcode) {
-    case 0:
+    case DIAG308_RES_MOD_CLR:
         s390_ipl_reset_request(cs, S390_RESET_MODIFIED_CLEAR);
         break;
-    case 1:
+    case DIAG308_RES_LOAD_NORM:
         s390_ipl_reset_request(cs, S390_RESET_LOAD_NORMAL);
         break;
-    case 3:
+    case DIAG308_LOAD_CLEAR:
+        /* Well we still lack the clearing bit... */
         s390_ipl_reset_request(cs, S390_RESET_REIPL);
         break;
-    case 5:
-        if ((r1 & 1) || (addr & 0x0fffULL)) {
-            s390_program_interrupt(env, PGM_SPECIFICATION, ra);
-            return;
-        }
-        if (!address_space_access_valid(&address_space_memory, addr,
-                                        sizeof(IplParameterBlock), false,
-                                        MEMTXATTRS_UNSPECIFIED)) {
-            s390_program_interrupt(env, PGM_ADDRESSING, ra);
+    case DIAG308_SET:
+        if (diag308_parm_check(env, r1, addr, ra, false)) {
             return;
         }
         iplb = g_new0(IplParameterBlock, 1);
@@ -110,15 +127,8 @@ void handle_diag_308(CPUS390XState *env, uint64_t r1, uint64_t r3, uintptr_t ra)
 out:
         g_free(iplb);
         return;
-    case 6:
-        if ((r1 & 1) || (addr & 0x0fffULL)) {
-            s390_program_interrupt(env, PGM_SPECIFICATION, ra);
-            return;
-        }
-        if (!address_space_access_valid(&address_space_memory, addr,
-                                        sizeof(IplParameterBlock), true,
-                                        MEMTXATTRS_UNSPECIFIED)) {
-            s390_program_interrupt(env, PGM_ADDRESSING, ra);
+    case DIAG308_STORE:
+        if (diag308_parm_check(env, r1, addr, ra, true)) {
             return;
         }
         iplb = s390_ipl_get_iplb();
-- 
2.20.1

[PATCH 03/15] s390x: protvirt: Add diag308 subcodes 8 - 10

[Janosch Frank]

For diag308 subcodes 8 - 10 we have a new ipib of type 5. The ipib
holds the address and length of the secure execution header, as well
as a list of guest components.

Each component is a block of memory, for example kernel or initrd,
which needs to be decrypted by the Ultravisor in order to run a
protected VM. The secure execution header instructs the Ultravisor on
how to handle the protected VM and its components.

Subcodes 8 and 9 are similiar to 5 and 6 and subcode 10 will finally
start the protected guest.

Subcodes 8-10 are not valid in protected mode, we have to do a subcode
3 and then the 8 and 10 combination for a protected reboot.

Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
---
 hw/s390x/ipl.c      | 48 ++++++++++++++++++++++++++++++++++++++++++---
 hw/s390x/ipl.h      | 33 +++++++++++++++++++++++++++++++
 target/s390x/diag.c | 26 ++++++++++++++++++++++--
 3 files changed, 102 insertions(+), 5 deletions(-)

diff --git a/hw/s390x/ipl.c b/hw/s390x/ipl.c
index ca544d64c5..a077926f36 100644
--- a/hw/s390x/ipl.c
+++ b/hw/s390x/ipl.c
@@ -529,15 +529,56 @@ static bool is_virtio_scsi_device(IplParameterBlock *iplb)
     return is_virtio_ccw_device_of_type(iplb, VIRTIO_ID_SCSI);
 }
 
+int s390_ipl_pv_check_comp(IplParameterBlock *iplb)
+{
+    int i;
+    IPLBlockPV *ipib_pv = &iplb->pv;
+
+    if (ipib_pv->num_comp == 0) {
+        return -EINVAL;
+    }
+
+    for (i = 0; i < ipib_pv->num_comp; i++) {
+
+        /* Addr must be 4k aligned */
+        if (ipib_pv->components[i].addr & ~TARGET_PAGE_MASK) {
+            return -EINVAL;
+        }
+
+        /* Tweak prefix is monotonously increasing with each component */
+        if (i < ipib_pv->num_comp - 1 &&
+            ipib_pv->components[i].tweak_pref >
+            ipib_pv->components[i + 1].tweak_pref) {
+            return -EINVAL;
+        }
+    }
+    return 1;
+}
+
 void s390_ipl_update_diag308(IplParameterBlock *iplb)
 {
     S390IPLState *ipl = get_ipl_device();
 
-    ipl->iplb = *iplb;
-    ipl->iplb_valid = true;
+    if (iplb->pbt == 5) {
+        ipl->iplb_pbt5 = *iplb;
+        ipl->iplb_valid_pbt5 = true;
+    } else {
+        ipl->iplb = *iplb;
+        ipl->iplb_valid = true;
+    }
     ipl->netboot = is_virtio_net_device(iplb);
 }
 
+IplParameterBlock *s390_ipl_get_iplb_secure(void)
+{
+    S390IPLState *ipl = get_ipl_device();
+
+    if (!ipl->iplb_valid_pbt5) {
+        return NULL;
+    }
+    return &ipl->iplb_pbt5;
+}
+
 IplParameterBlock *s390_ipl_get_iplb(void)
 {
     S390IPLState *ipl = get_ipl_device();
@@ -552,7 +593,8 @@ void s390_ipl_reset_request(CPUState *cs, enum s390_reset reset_type)
 {
     S390IPLState *ipl = get_ipl_device();
 
-    if (reset_type == S390_RESET_EXTERNAL || reset_type == S390_RESET_REIPL) {
+    if (reset_type == S390_RESET_EXTERNAL || reset_type == S390_RESET_REIPL ||
+        reset_type == S390_RESET_PV) {
         /* use CPU 0 for full resets */
         ipl->reset_cpu_index = 0;
     } else {
diff --git a/hw/s390x/ipl.h b/hw/s390x/ipl.h
index d4813105db..7b8a493509 100644
--- a/hw/s390x/ipl.h
+++ b/hw/s390x/ipl.h
@@ -15,6 +15,24 @@
 #include "cpu.h"
 #include "hw/qdev-core.h"
 
+struct IPLBlockPVComp {
+    uint64_t tweak_pref;
+    uint64_t addr;
+    uint64_t size;
+} QEMU_PACKED;
+typedef struct IPLBlockPVComp IPLBlockPVComp;
+
+struct IPLBlockPV {
+    uint8_t  reserved[84];
+    uint8_t  reserved67[3];
+    uint8_t  version;
+    uint32_t num_comp;
+    uint64_t pv_header_addr;
+    uint64_t pv_header_len;
+    struct IPLBlockPVComp components[];
+} QEMU_PACKED;
+typedef struct IPLBlockPV IPLBlockPV;
+
 struct IplBlockCcw {
     uint8_t  reserved0[85];
     uint8_t  ssid;
@@ -71,6 +89,7 @@ union IplParameterBlock {
         union {
             IplBlockCcw ccw;
             IplBlockFcp fcp;
+            IPLBlockPV pv;
             IplBlockQemuScsi scsi;
         };
     } QEMU_PACKED;
@@ -84,9 +103,11 @@ union IplParameterBlock {
 typedef union IplParameterBlock IplParameterBlock;
 
 int s390_ipl_set_loadparm(uint8_t *loadparm);
+int s390_ipl_pv_check_comp(IplParameterBlock *iplb);
 void s390_ipl_update_diag308(IplParameterBlock *iplb);
 void s390_ipl_prepare_cpu(S390CPU *cpu);
 IplParameterBlock *s390_ipl_get_iplb(void);
+IplParameterBlock *s390_ipl_get_iplb_secure(void);
 
 enum s390_reset {
     /* default is a reset not triggered by a CPU e.g. issued by QMP */
@@ -94,6 +115,7 @@ enum s390_reset {
     S390_RESET_REIPL,
     S390_RESET_MODIFIED_CLEAR,
     S390_RESET_LOAD_NORMAL,
+    S390_RESET_PV,
 };
 void s390_ipl_reset_request(CPUState *cs, enum s390_reset reset_type);
 void s390_ipl_get_reset_request(CPUState **cs, enum s390_reset *reset_type);
@@ -133,6 +155,7 @@ struct S390IPLState {
     /*< private >*/
     DeviceState parent_obj;
     IplParameterBlock iplb;
+    IplParameterBlock iplb_pbt5;
     QemuIplParameters qipl;
     uint64_t start_addr;
     uint64_t compat_start_addr;
@@ -140,6 +163,7 @@ struct S390IPLState {
     uint64_t compat_bios_start_addr;
     bool enforce_bios;
     bool iplb_valid;
+    bool iplb_valid_pbt5;
     bool netboot;
     /* reset related properties don't have to be migrated or reset */
     enum s390_reset reset_type;
@@ -161,9 +185,11 @@ QEMU_BUILD_BUG_MSG(offsetof(S390IPLState, iplb) & 3, "alignment of iplb wrong");
 
 #define S390_IPL_TYPE_FCP 0x00
 #define S390_IPL_TYPE_CCW 0x02
+#define S390_IPL_TYPE_PV 0x05
 #define S390_IPL_TYPE_QEMU_SCSI 0xff
 
 #define S390_IPLB_HEADER_LEN 8
+#define S390_IPLB_MIN_PV_LEN 148
 #define S390_IPLB_MIN_CCW_LEN 200
 #define S390_IPLB_MIN_FCP_LEN 384
 #define S390_IPLB_MIN_QEMU_SCSI_LEN 200
@@ -185,4 +211,11 @@ static inline bool iplb_valid_fcp(IplParameterBlock *iplb)
            iplb->pbt == S390_IPL_TYPE_FCP;
 }
 
+static inline bool iplb_valid_se(IplParameterBlock *iplb)
+{
+    return be32_to_cpu(iplb->len) >= S390_IPLB_MIN_PV_LEN &&
+           iplb->pbt == S390_IPL_TYPE_PV;
+}
+
+
 #endif
diff --git a/target/s390x/diag.c b/target/s390x/diag.c
index 067c667ba7..32049bb4ee 100644
--- a/target/s390x/diag.c
+++ b/target/s390x/diag.c
@@ -52,6 +52,8 @@ int handle_diag_288(CPUS390XState *env, uint64_t r1, uint64_t r3)
 #define DIAG_308_RC_OK              0x0001
 #define DIAG_308_RC_NO_CONF         0x0102
 #define DIAG_308_RC_INVALID         0x0402
+#define DIAG_308_RC_NO_PV_CONF      0x0a02
+#define DIAG_308_RC_INV_FOR_PV      0x0b02
 
 #define DIAG308_RES_MOD_CLR		0
 #define DIAG308_RES_LOAD_NORM		1
@@ -59,6 +61,9 @@ int handle_diag_288(CPUS390XState *env, uint64_t r1, uint64_t r3)
 #define DIAG308_LOAD_NORMAL_DUMP	4
 #define DIAG308_SET			5
 #define DIAG308_STORE			6
+#define DIAG308_PV_SET			8
+#define DIAG308_PV_STORE		9
+#define DIAG308_PV_START		10
 
 static int diag308_parm_check(CPUS390XState *env, uint64_t r1, uint64_t addr,
                               uintptr_t ra, bool write)
@@ -105,6 +110,7 @@ void handle_diag_308(CPUS390XState *env, uint64_t r1, uint64_t r3, uintptr_t ra)
         s390_ipl_reset_request(cs, S390_RESET_REIPL);
         break;
     case DIAG308_SET:
+    case DIAG308_PV_SET: /* Set SE parms */
         if (diag308_parm_check(env, r1, addr, ra, false)) {
             return;
         }
@@ -117,7 +123,8 @@ void handle_diag_308(CPUS390XState *env, uint64_t r1, uint64_t r3, uintptr_t ra)
 
         cpu_physical_memory_read(addr, iplb, be32_to_cpu(iplb->len));
 
-        if (!iplb_valid_ccw(iplb) && !iplb_valid_fcp(iplb)) {
+        if (!iplb_valid_ccw(iplb) && !iplb_valid_fcp(iplb) &&
+            !(iplb_valid_se(iplb) && s390_ipl_pv_check_comp(iplb) >= 0)) {
             env->regs[r1 + 1] = DIAG_308_RC_INVALID;
             goto out;
         }
@@ -128,10 +135,15 @@ out:
         g_free(iplb);
         return;
     case DIAG308_STORE:
+    case DIAG308_PV_STORE: /* Get SE parms */
         if (diag308_parm_check(env, r1, addr, ra, true)) {
             return;
         }
-        iplb = s390_ipl_get_iplb();
+        if (subcode == DIAG308_PV_STORE) {
+            iplb = s390_ipl_get_iplb_secure();
+        } else {
+            iplb = s390_ipl_get_iplb();
+        }
         if (iplb) {
             cpu_physical_memory_write(addr, iplb, be32_to_cpu(iplb->len));
             env->regs[r1 + 1] = DIAG_308_RC_OK;
@@ -139,6 +151,16 @@ out:
             env->regs[r1 + 1] = DIAG_308_RC_NO_CONF;
         }
         return;
+        break;
+    case DIAG308_PV_START: /* SE start */
+        iplb = s390_ipl_get_iplb_secure();
+        if (!iplb_valid_se(iplb)) {
+            env->regs[r1 + 1] = DIAG_308_RC_NO_PV_CONF;
+            return;
+        }
+
+        s390_ipl_reset_request(cs, S390_RESET_PV);
+        break;
     default:
         s390_program_interrupt(env, PGM_SPECIFICATION, ra);
         break;
-- 
2.20.1

[PATCH 04/15] Header sync protvirt

[Janosch Frank]

Let's sync all the protvirt header changes

Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
---
 linux-headers/asm-s390/kvm.h |  3 ++-
 linux-headers/linux/kvm.h    | 42 ++++++++++++++++++++++++++++++++++++
 2 files changed, 44 insertions(+), 1 deletion(-)

diff --git a/linux-headers/asm-s390/kvm.h b/linux-headers/asm-s390/kvm.h
index 03ab5968c7..41976d33f0 100644
--- a/linux-headers/asm-s390/kvm.h
+++ b/linux-headers/asm-s390/kvm.h
@@ -255,7 +255,8 @@ struct kvm_sync_regs {
 	__u8  reserved[512];	/* for future vector expansion */
 	__u32 fpc;		/* valid on KVM_SYNC_VRS or KVM_SYNC_FPRS */
 	__u8 bpbc : 1;		/* bp mode */
-	__u8 reserved2 : 7;
+	__u8 pv : 1;		/* pv mode */
+	__u8 reserved2 : 6;
 	__u8 padding1[51];	/* riccb needs to be 64byte aligned */
 	__u8 riccb[64];		/* runtime instrumentation controls block */
 	__u8 padding2[192];	/* sdnx needs to be 256byte aligned */
diff --git a/linux-headers/linux/kvm.h b/linux-headers/linux/kvm.h
index 18892d6541..d031051601 100644
--- a/linux-headers/linux/kvm.h
+++ b/linux-headers/linux/kvm.h
@@ -995,6 +995,8 @@ struct kvm_ppc_resize_hpt {
 #define KVM_CAP_ARM_SVE 170
 #define KVM_CAP_ARM_PTRAUTH_ADDRESS 171
 #define KVM_CAP_ARM_PTRAUTH_GENERIC 172
+#define KVM_CAP_S390_PROTECTED 180
+#define KVM_CAP_S390_VCPU_RESETS 181
 
 #ifdef KVM_CAP_IRQ_ROUTING
 
@@ -1453,6 +1455,46 @@ struct kvm_enc_region {
 /* Available with KVM_CAP_ARM_SVE */
 #define KVM_ARM_VCPU_FINALIZE	  _IOW(KVMIO,  0xc2, int)
 
+struct kvm_s390_pv_sec_parm {
+	__u64	origin;
+	__u64	length;
+};
+
+struct kvm_s390_pv_unp {
+	__u64 addr;
+	__u64 size;
+	__u64 tweak;
+};
+
+enum pv_cmd_id {
+	KVM_PV_VM_CREATE,
+	KVM_PV_VM_DESTROY,
+	KVM_PV_VM_SET_SEC_PARMS,
+	KVM_PV_VM_UNPACK,
+	KVM_PV_VM_VERIFY,
+	KVM_PV_VM_PERF_CLEAR_RESET,
+	KVM_PV_VM_UNSHARE,
+	KVM_PV_VCPU_CREATE,
+	KVM_PV_VCPU_DESTROY,
+};
+
+struct kvm_pv_cmd {
+	__u32	cmd;
+	__u16	rc;
+	__u16	rrc;
+	__u64	data;
+};
+
+/* Available with KVM_CAP_S390_PROTECTED */
+#define KVM_S390_PV_COMMAND		_IOW(KVMIO, 0xc3, struct kvm_pv_cmd)
+#define KVM_S390_PV_COMMAND_VCPU	_IOW(KVMIO, 0xc4, struct kvm_pv_cmd)
+
+#define KVM_S390_VCPU_RESET_NORMAL	0
+#define KVM_S390_VCPU_RESET_INITIAL	1
+#define KVM_S390_VCPU_RESET_CLEAR	2
+
+#define KVM_S390_VCPU_RESET    _IO(KVMIO,   0xc5)
+
 /* Secure Encrypted Virtualization command */
 enum sev_cmd_id {
 	/* Guest initialization commands */
-- 
2.20.1

[PATCH 05/15] s390x: protvirt: Sync PV state

[Janosch Frank]

We do not always have the SIE intercept code handy at each place where
we do emulation. Unfortunately emulation for secure guests often
differ slightly from normal emulation and we need to make decisions
based on the protected state of the VCPU.

Let's sync the protected state and make it available.

Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
---
 linux-headers/asm-s390/kvm.h | 1 +
 target/s390x/cpu.h           | 1 +
 target/s390x/kvm.c           | 4 ++++
 3 files changed, 6 insertions(+)

diff --git a/linux-headers/asm-s390/kvm.h b/linux-headers/asm-s390/kvm.h
index 41976d33f0..7c46cf6078 100644
--- a/linux-headers/asm-s390/kvm.h
+++ b/linux-headers/asm-s390/kvm.h
@@ -231,6 +231,7 @@ struct kvm_guest_debug_arch {
 #define KVM_SYNC_GSCB   (1UL << 9)
 #define KVM_SYNC_BPBC   (1UL << 10)
 #define KVM_SYNC_ETOKEN (1UL << 11)
+#define KVM_SYNC_PV	(1UL << 12)
 /* length and alignment of the sdnx as a power of two */
 #define SDNXC 8
 #define SDNXL (1UL << SDNXC)
diff --git a/target/s390x/cpu.h b/target/s390x/cpu.h
index 17460ed7b3..a787221772 100644
--- a/target/s390x/cpu.h
+++ b/target/s390x/cpu.h
@@ -116,6 +116,7 @@ struct CPUS390XState {
 
     /* Fields up to this point are cleared by a CPU reset */
     struct {} end_reset_fields;
+    bool pv; /* protected virtualization */
 
 #if !defined(CONFIG_USER_ONLY)
     uint32_t core_id; /* PoP "CPU address", same as cpu_index */
diff --git a/target/s390x/kvm.c b/target/s390x/kvm.c
index c24c869e77..418154ccfe 100644
--- a/target/s390x/kvm.c
+++ b/target/s390x/kvm.c
@@ -676,6 +676,10 @@ int kvm_arch_get_registers(CPUState *cs)
         env->etoken_extension = cs->kvm_run->s.regs.etoken_extension;
     }
 
+    if (can_sync_regs(cs, KVM_SYNC_PV)) {
+        env->pv = !!cs->kvm_run->s.regs.pv;
+    }
+
     /* pfault parameters */
     if (can_sync_regs(cs, KVM_SYNC_PFAULT)) {
         env->pfault_token = cs->kvm_run->s.regs.pft;
-- 
2.20.1

[PATCH 06/15] s390x: protvirt: Support unpack facility

[Janosch Frank]

When a guest has saved a ipib of type 5 and call diagnose308 with
subcode 10, we have to setup the protected processing environment via
Ultravisor calls. The calls are done by KVM and are exposed via an API.

The following steps are necessary:
1. Create a VM (register it with the Ultravisor)
2. Create secure CPUs for all of our current cpus
3. Forward the secure header to the Ultravisor (has all information on
how to decrypt the image and VM information)
4. Protect image pages from the host and decrypt them
5. Verify the image integrity

Only after step 5 a protected VM is allowed to run.

Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
---
 hw/s390x/Makefile.objs              |   1 +
 hw/s390x/ipl.c                      |  33 ++++++++
 hw/s390x/ipl.h                      |   2 +
 hw/s390x/pv.c                       | 118 ++++++++++++++++++++++++++++
 hw/s390x/pv.h                       |  26 ++++++
 hw/s390x/s390-virtio-ccw.c          |  45 ++++++++---
 target/s390x/cpu_features_def.inc.h |   1 +
 7 files changed, 216 insertions(+), 10 deletions(-)
 create mode 100644 hw/s390x/pv.c
 create mode 100644 hw/s390x/pv.h

diff --git a/hw/s390x/Makefile.objs b/hw/s390x/Makefile.objs
index 94e57113d8..568bab9711 100644
--- a/hw/s390x/Makefile.objs
+++ b/hw/s390x/Makefile.objs
@@ -31,6 +31,7 @@ obj-y += tod-qemu.o
 obj-$(CONFIG_KVM) += tod-kvm.o
 obj-$(CONFIG_KVM) += s390-skeys-kvm.o
 obj-$(CONFIG_KVM) += s390-stattrib-kvm.o s390-mchk.o
+obj-$(CONFIG_KVM) += pv.o
 obj-y += s390-ccw.o
 obj-y += ap-device.o
 obj-y += ap-bridge.o
diff --git a/hw/s390x/ipl.c b/hw/s390x/ipl.c
index a077926f36..50501fcd27 100644
--- a/hw/s390x/ipl.c
+++ b/hw/s390x/ipl.c
@@ -33,6 +33,7 @@
 #include "qemu/cutils.h"
 #include "qemu/option.h"
 #include "exec/exec-all.h"
+#include "pv.h"
 
 #define KERN_IMAGE_START                0x010000UL
 #define LINUX_MAGIC_ADDR                0x010008UL
@@ -668,6 +669,38 @@ static void s390_ipl_prepare_qipl(S390CPU *cpu)
     cpu_physical_memory_unmap(addr, len, 1, len);
 }
 
+int s390_ipl_prepare_pv_header(void)
+{
+    int rc;
+    IplParameterBlock *iplb = s390_ipl_get_iplb_secure();
+    IPLBlockPV *ipib_pv = &iplb->pv;
+    void *hdr = g_malloc(ipib_pv->pv_header_len);
+
+    cpu_physical_memory_read(ipib_pv->pv_header_addr, hdr,
+                             ipib_pv->pv_header_len);
+    rc = s390_pv_set_sec_parms((uint64_t)hdr,
+                               ipib_pv->pv_header_len);
+    g_free(hdr);
+    return rc;
+}
+
+int s390_ipl_pv_unpack(void)
+{
+    int i, rc;
+    IplParameterBlock *iplb = s390_ipl_get_iplb_secure();
+    IPLBlockPV *ipib_pv = &iplb->pv;
+
+    for (i = 0; i < ipib_pv->num_comp; i++) {
+        rc = s390_pv_unpack(ipib_pv->components[i].addr,
+                            TARGET_PAGE_ALIGN(ipib_pv->components[i].size),
+                            ipib_pv->components[i].tweak_pref);
+        if (rc) {
+            return rc;
+        }
+    }
+    return rc;
+}
+
 void s390_ipl_prepare_cpu(S390CPU *cpu)
 {
     S390IPLState *ipl = get_ipl_device();
diff --git a/hw/s390x/ipl.h b/hw/s390x/ipl.h
index 7b8a493509..e848602c16 100644
--- a/hw/s390x/ipl.h
+++ b/hw/s390x/ipl.h
@@ -105,6 +105,8 @@ typedef union IplParameterBlock IplParameterBlock;
 int s390_ipl_set_loadparm(uint8_t *loadparm);
 int s390_ipl_pv_check_comp(IplParameterBlock *iplb);
 void s390_ipl_update_diag308(IplParameterBlock *iplb);
+int s390_ipl_prepare_pv_header(void);
+int s390_ipl_pv_unpack(void);
 void s390_ipl_prepare_cpu(S390CPU *cpu);
 IplParameterBlock *s390_ipl_get_iplb(void);
 IplParameterBlock *s390_ipl_get_iplb_secure(void);
diff --git a/hw/s390x/pv.c b/hw/s390x/pv.c
new file mode 100644
index 0000000000..0218070322
--- /dev/null
+++ b/hw/s390x/pv.c
@@ -0,0 +1,118 @@
+/*
+ * Secure execution functions
+ *
+ * Copyright IBM Corp. 2019
+ * Author(s):
+ *  Janosch Frank <frankja@linux.ibm.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or (at
+ * your option) any later version. See the COPYING file in the top-level
+ * directory.
+ */
+#include "qemu/osdep.h"
+#include <sys/ioctl.h>
+
+#include <linux/kvm.h>
+
+#include "qemu/error-report.h"
+#include "sysemu/kvm.h"
+#include "pv.h"
+
+static int s390_pv_cmd(uint32_t cmd, void *data)
+{
+    int rc;
+    struct kvm_pv_cmd pv_cmd = {
+        .cmd = cmd,
+        .data = (uint64_t)data,
+    };
+
+    rc = kvm_vm_ioctl(kvm_state, KVM_S390_PV_COMMAND, &pv_cmd);
+    if (rc) {
+        error_report("KVM PV command failed cmd: %d rc: %d", cmd, rc);
+        exit(1);
+    }
+    return rc;
+}
+
+static int s390_pv_cmd_vcpu(CPUState *cs, uint32_t cmd, void *data)
+{
+    int rc;
+    struct kvm_pv_cmd pv_cmd = {
+        .cmd = cmd,
+        .data = (uint64_t)data,
+    };
+
+    rc = kvm_vcpu_ioctl(cs, KVM_S390_PV_COMMAND_VCPU, &pv_cmd);
+    if (rc) {
+        error_report("KVM PV VCPU command failed cmd: %d rc: %d", cmd, rc);
+        exit(1);
+    }
+    return rc;
+}
+
+int s390_pv_vm_create(void)
+{
+    return s390_pv_cmd(KVM_PV_VM_CREATE, NULL);
+}
+
+int s390_pv_vm_destroy(void)
+{
+    return s390_pv_cmd(KVM_PV_VM_DESTROY, NULL);
+}
+
+int s390_pv_vcpu_create(CPUState *cs)
+{
+    return s390_pv_cmd_vcpu(cs, KVM_PV_VCPU_CREATE, NULL);
+}
+
+int s390_pv_vcpu_destroy(CPUState *cs)
+{
+    S390CPU *cpu = S390_CPU(cs);
+    CPUS390XState *env = &cpu->env;
+    int rc;
+
+    rc = s390_pv_cmd_vcpu(cs, KVM_PV_VCPU_DESTROY, NULL);
+    if (!rc) {
+        env->pv = false;
+    }
+    return rc;
+}
+
+int s390_pv_set_sec_parms(uint64_t origin, uint64_t length)
+{
+    struct kvm_s390_pv_sec_parm args = {
+        .origin = origin,
+        .length = length,
+    };
+
+    return s390_pv_cmd(KVM_PV_VM_SET_SEC_PARMS, &args);
+}
+
+/*
+ * Called for each component in the SE type IPL parameter block 0.
+ */
+int s390_pv_unpack(uint64_t addr, uint64_t size, uint64_t tweak)
+{
+    struct kvm_s390_pv_unp args = {
+        .addr = addr,
+        .size = size,
+        .tweak = tweak,
+    };
+
+    return s390_pv_cmd(KVM_PV_VM_UNPACK, &args);
+}
+
+int s390_pv_perf_clear_reset(void)
+{
+    return s390_pv_cmd(KVM_PV_VM_PERF_CLEAR_RESET, NULL);
+}
+
+int s390_pv_verify(void)
+{
+    return s390_pv_cmd(KVM_PV_VM_VERIFY, NULL);
+}
+
+int s390_pv_unshare(void)
+{
+    return s390_pv_cmd(KVM_PV_VM_UNSHARE, NULL);
+}
diff --git a/hw/s390x/pv.h b/hw/s390x/pv.h
new file mode 100644
index 0000000000..eb074e4bc9
--- /dev/null
+++ b/hw/s390x/pv.h
@@ -0,0 +1,26 @@
+/*
+ * Protected Virtualization header
+ *
+ * Copyright IBM Corp. 2019
+ * Author(s):
+ *  Janosch Frank <frankja@linux.ibm.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or (at
+ * your option) any later version. See the COPYING file in the top-level
+ * directory.
+ */
+
+#ifndef HW_S390_PV_H
+#define HW_S390_PV_H
+
+int s390_pv_vm_create(void);
+int s390_pv_vm_destroy(void);
+int s390_pv_vcpu_destroy(CPUState *cs);
+int s390_pv_vcpu_create(CPUState *cs);
+int s390_pv_set_sec_parms(uint64_t origin, uint64_t length);
+int s390_pv_unpack(uint64_t addr, uint64_t size, uint64_t tweak);
+int s390_pv_perf_clear_reset(void);
+int s390_pv_verify(void);
+int s390_pv_unshare(void);
+
+#endif /* HW_S390_PV_H */
diff --git a/hw/s390x/s390-virtio-ccw.c b/hw/s390x/s390-virtio-ccw.c
index c1d1440272..7262453616 100644
--- a/hw/s390x/s390-virtio-ccw.c
+++ b/hw/s390x/s390-virtio-ccw.c
@@ -41,6 +41,7 @@
 #include "hw/qdev-properties.h"
 #include "hw/s390x/tod.h"
 #include "sysemu/sysemu.h"
+#include "hw/s390x/pv.h"
 
 S390CPU *s390_cpu_addr2state(uint16_t cpu_addr)
 {
@@ -322,6 +323,7 @@ static void s390_machine_reset(MachineState *machine)
 {
     enum s390_reset reset_type;
     CPUState *cs, *t;
+    S390CPU *cpu;
 
     /* get the reset parameters, reset them once done */
     s390_ipl_get_reset_request(&cs, &reset_type);
@@ -329,16 +331,10 @@ static void s390_machine_reset(MachineState *machine)
     /* all CPUs are paused and synchronized at this point */
     s390_cmma_reset();
 
-    switch (reset_type) {
-    case S390_RESET_EXTERNAL:
-    case S390_RESET_REIPL:
-        qemu_devices_reset();
-        s390_crypto_reset();
+    cpu = S390_CPU(cs);
 
-        /* configure and start the ipl CPU only */
-        run_on_cpu(cs, s390_do_cpu_ipl, RUN_ON_CPU_NULL);
-        break;
-    case S390_RESET_MODIFIED_CLEAR:
+    switch (reset_type) {
+    case S390_RESET_MODIFIED_CLEAR:  /* Subcode 0 */
         CPU_FOREACH(t) {
             run_on_cpu(t, s390_do_cpu_full_reset, RUN_ON_CPU_NULL);
         }
@@ -346,7 +342,7 @@ static void s390_machine_reset(MachineState *machine)
         s390_crypto_reset();
         run_on_cpu(cs, s390_do_cpu_load_normal, RUN_ON_CPU_NULL);
         break;
-    case S390_RESET_LOAD_NORMAL:
+    case S390_RESET_LOAD_NORMAL: /* Subcode 1*/
         CPU_FOREACH(t) {
             if (t == cs) {
                 continue;
@@ -357,6 +353,35 @@ static void s390_machine_reset(MachineState *machine)
         run_on_cpu(cs, s390_do_cpu_initial_reset, RUN_ON_CPU_NULL);
         run_on_cpu(cs, s390_do_cpu_load_normal, RUN_ON_CPU_NULL);
         break;
+    case S390_RESET_EXTERNAL:
+    case S390_RESET_REIPL: /* Subcode 4 */
+        qemu_devices_reset();
+        s390_crypto_reset();
+        /* configure and start the ipl CPU only */
+        run_on_cpu(cs, s390_do_cpu_ipl, RUN_ON_CPU_NULL);
+        break;
+    case S390_RESET_PV: /* Subcode 10 */
+        subsystem_reset();
+        s390_crypto_reset();
+
+        CPU_FOREACH(t) {
+            run_on_cpu(t, s390_do_cpu_full_reset, RUN_ON_CPU_NULL);
+        }
+
+        /* Create SE VM */
+        s390_pv_vm_create();
+        CPU_FOREACH(t) {
+            s390_pv_vcpu_create(t);
+        }
+
+        /* Set SE header and unpack */
+        s390_ipl_prepare_pv_header();
+        /* Decrypt image */
+        s390_ipl_pv_unpack();
+        /* Verify integrity */
+        s390_pv_verify();
+        s390_cpu_set_state(S390_CPU_STATE_OPERATING, cpu);
+        break;
     default:
         g_assert_not_reached();
     }
diff --git a/target/s390x/cpu_features_def.inc.h b/target/s390x/cpu_features_def.inc.h
index 31dff0d84e..60db28351d 100644
--- a/target/s390x/cpu_features_def.inc.h
+++ b/target/s390x/cpu_features_def.inc.h
@@ -107,6 +107,7 @@ DEF_FEAT(DEFLATE_BASE, "deflate-base", STFL, 151, "Deflate-conversion facility (
 DEF_FEAT(VECTOR_PACKED_DECIMAL_ENH, "vxpdeh", STFL, 152, "Vector-Packed-Decimal-Enhancement Facility")
 DEF_FEAT(MSA_EXT_9, "msa9-base", STFL, 155, "Message-security-assist-extension-9 facility (excluding subfunctions)")
 DEF_FEAT(ETOKEN, "etoken", STFL, 156, "Etoken facility")
+DEF_FEAT(UNPACK, "unpack", STFL, 161, "Unpack facility")
 
 /* Features exposed via SCLP SCCB Byte 80 - 98  (bit numbers relative to byte-80) */
 DEF_FEAT(SIE_GSLS, "gsls", SCLP_CONF_CHAR, 40, "SIE: Guest-storage-limit-suppression facility")
-- 
2.20.1

[PATCH 07/15] s390x: protvirt: Handle diag 308 subcodes 0,1,3,4

[Janosch Frank]

Now that we know the protection state off the cpus, we can start
handling all diag 308 subcodes in the protected state.

For subcodes 0 and 1 we need to unshare all pages before continuing,
so the guest doesn't accidently expose data when dumping.

For subcode 3/4 we tear down the protected VM and reboot into
unprotected mode. We do not provide a secure reboot.

Before we can do the unshare calls, we need to mark all cpus as
stopped.

Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
---
 hw/s390x/s390-virtio-ccw.c | 30 +++++++++++++++++++++++++++---
 target/s390x/diag.c        |  4 ++++
 2 files changed, 31 insertions(+), 3 deletions(-)

diff --git a/hw/s390x/s390-virtio-ccw.c b/hw/s390x/s390-virtio-ccw.c
index 7262453616..6fd50b4c42 100644
--- a/hw/s390x/s390-virtio-ccw.c
+++ b/hw/s390x/s390-virtio-ccw.c
@@ -319,11 +319,26 @@ static inline void s390_do_cpu_ipl(CPUState *cs, run_on_cpu_data arg)
     s390_cpu_set_state(S390_CPU_STATE_OPERATING, cpu);
 }
 
+static void s390_pv_prepare_reset(CPUS390XState *env)
+{
+    CPUState *cs;
+
+    if (!env->pv) {
+        return;
+    }
+    CPU_FOREACH(cs) {
+        s390_cpu_set_state(S390_CPU_STATE_STOPPED, S390_CPU(cs));
+    }
+    s390_pv_unshare();
+    s390_pv_perf_clear_reset();
+}
+
 static void s390_machine_reset(MachineState *machine)
 {
     enum s390_reset reset_type;
     CPUState *cs, *t;
     S390CPU *cpu;
+    CPUS390XState *env;
 
     /* get the reset parameters, reset them once done */
     s390_ipl_get_reset_request(&cs, &reset_type);
@@ -332,29 +347,38 @@ static void s390_machine_reset(MachineState *machine)
     s390_cmma_reset();
 
     cpu = S390_CPU(cs);
+    env = &cpu->env;
 
     switch (reset_type) {
     case S390_RESET_MODIFIED_CLEAR:  /* Subcode 0 */
+        subsystem_reset();
+        s390_crypto_reset();
+        s390_pv_prepare_reset(env);
         CPU_FOREACH(t) {
             run_on_cpu(t, s390_do_cpu_full_reset, RUN_ON_CPU_NULL);
         }
-        subsystem_reset();
-        s390_crypto_reset();
         run_on_cpu(cs, s390_do_cpu_load_normal, RUN_ON_CPU_NULL);
         break;
     case S390_RESET_LOAD_NORMAL: /* Subcode 1*/
+        subsystem_reset();
+        s390_pv_prepare_reset(env);
         CPU_FOREACH(t) {
             if (t == cs) {
                 continue;
             }
             run_on_cpu(t, s390_do_cpu_reset, RUN_ON_CPU_NULL);
         }
-        subsystem_reset();
         run_on_cpu(cs, s390_do_cpu_initial_reset, RUN_ON_CPU_NULL);
         run_on_cpu(cs, s390_do_cpu_load_normal, RUN_ON_CPU_NULL);
         break;
     case S390_RESET_EXTERNAL:
     case S390_RESET_REIPL: /* Subcode 4 */
+        if (env->pv) {
+            CPU_FOREACH(t) {
+                s390_pv_vcpu_destroy(t);
+            }
+            s390_pv_vm_destroy();
+        }
         qemu_devices_reset();
         s390_crypto_reset();
         /* configure and start the ipl CPU only */
diff --git a/target/s390x/diag.c b/target/s390x/diag.c
index 32049bb4ee..db6d79cef3 100644
--- a/target/s390x/diag.c
+++ b/target/s390x/diag.c
@@ -68,6 +68,10 @@ int handle_diag_288(CPUS390XState *env, uint64_t r1, uint64_t r3)
 static int diag308_parm_check(CPUS390XState *env, uint64_t r1, uint64_t addr,
                               uintptr_t ra, bool write)
 {
+    /* Handled by the Ultravisor */
+    if (env->pv) {
+        return 0;
+    }
     if ((r1 & 1) || (addr & ~TARGET_PAGE_MASK)) {
         s390_program_interrupt(env, PGM_SPECIFICATION, ra);
         return -EINVAL;
-- 
2.20.1

[PATCH 08/15] s390x: protvirt: KVM intercept changes

[Janosch Frank]

Secure guests no longer intercept with code 4 for an instruction
interception. Instead they have codes 104 and 108 for secure
instruction interception and secure instruction notification
respectively.

The 104 mirrors the 4, but the 108 is a notification, that something
happened and the hypervisor might need to adjust its tracking data to
that fact. An example for that is the set prefix notification
interception, where KVM only reads the new prefix, but does not update
the prefix in the state description.

Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
---
 target/s390x/kvm.c | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/target/s390x/kvm.c b/target/s390x/kvm.c
index 418154ccfe..58251c0229 100644
--- a/target/s390x/kvm.c
+++ b/target/s390x/kvm.c
@@ -115,6 +115,8 @@
 #define ICPT_CPU_STOP                   0x28
 #define ICPT_OPEREXC                    0x2c
 #define ICPT_IO                         0x40
+#define ICPT_PV_INSTR                   0x68
+#define ICPT_PV_INSTR_NOT               0x6c
 
 #define NR_LOCAL_IRQS 32
 /*
@@ -151,6 +153,7 @@ static int cap_s390_irq;
 static int cap_ri;
 static int cap_gs;
 static int cap_hpage_1m;
+static int cap_protvirt;
 
 static int active_cmma;
 
@@ -336,6 +339,7 @@ int kvm_arch_init(MachineState *ms, KVMState *s)
     cap_async_pf = kvm_check_extension(s, KVM_CAP_ASYNC_PF);
     cap_mem_op = kvm_check_extension(s, KVM_CAP_S390_MEM_OP);
     cap_s390_irq = kvm_check_extension(s, KVM_CAP_S390_INJECT_IRQ);
+    cap_protvirt = kvm_check_extension(s, KVM_CAP_S390_PROTECTED);
 
     if (!kvm_check_extension(s, KVM_CAP_S390_GMAP)
         || !kvm_check_extension(s, KVM_CAP_S390_COW)) {
@@ -1664,6 +1668,8 @@ static int handle_intercept(S390CPU *cpu)
             (long)cs->kvm_run->psw_addr);
     switch (icpt_code) {
         case ICPT_INSTRUCTION:
+        case ICPT_PV_INSTR:
+        case ICPT_PV_INSTR_NOT:
             r = handle_instruction(cpu, run);
             break;
         case ICPT_PROGRAM:
-- 
2.20.1

[PATCH 09/15] s390x: protvirt: SCLP interpretation

[Janosch Frank]

SCLP for a protected guest is done over the SIDAD, so we need to use
the s390_cpu_virt_mem_* functions to access the SIDAD instead of guest
memory when reading/writing SCBs.

To not confuse the sclp emulation, we set 0x42000 as the address, but
ignore it for reading/writing the SCCB.

Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
---
 hw/s390x/sclp.c         | 16 ++++++++++++++++
 include/hw/s390x/sclp.h |  2 ++
 target/s390x/kvm.c      |  8 +++++++-
 3 files changed, 25 insertions(+), 1 deletion(-)

diff --git a/hw/s390x/sclp.c b/hw/s390x/sclp.c
index f57ce7b739..00d08adc7f 100644
--- a/hw/s390x/sclp.c
+++ b/hw/s390x/sclp.c
@@ -193,6 +193,22 @@ static void sclp_execute(SCLPDevice *sclp, SCCB *sccb, uint32_t code)
     }
 }
 
+int sclp_service_call_protected(CPUS390XState *env, uint64_t sccb,
+                                uint32_t code)
+{
+    SCLPDevice *sclp = get_sclp_device();
+    SCLPDeviceClass *sclp_c = SCLP_GET_CLASS(sclp);
+    SCCB work_sccb;
+    hwaddr sccb_len = sizeof(SCCB);
+
+    s390_cpu_virt_mem_read(env_archcpu(env), 0, 0, &work_sccb, sccb_len);
+    sclp_c->execute(sclp, &work_sccb, code);
+    s390_cpu_virt_mem_write(env_archcpu(env), 0, 0, &work_sccb,
+                            be16_to_cpu(work_sccb.h.length));
+    sclp_c->service_interrupt(sclp, sccb);
+    return 0;
+}
+
 int sclp_service_call(CPUS390XState *env, uint64_t sccb, uint32_t code)
 {
     SCLPDevice *sclp = get_sclp_device();
diff --git a/include/hw/s390x/sclp.h b/include/hw/s390x/sclp.h
index c54413b78c..c0a3faa37d 100644
--- a/include/hw/s390x/sclp.h
+++ b/include/hw/s390x/sclp.h
@@ -217,5 +217,7 @@ void s390_sclp_init(void);
 void sclp_service_interrupt(uint32_t sccb);
 void raise_irq_cpu_hotplug(void);
 int sclp_service_call(CPUS390XState *env, uint64_t sccb, uint32_t code);
+int sclp_service_call_protected(CPUS390XState *env, uint64_t sccb,
+                                uint32_t code);
 
 #endif
diff --git a/target/s390x/kvm.c b/target/s390x/kvm.c
index 58251c0229..0f2458b553 100644
--- a/target/s390x/kvm.c
+++ b/target/s390x/kvm.c
@@ -1172,7 +1172,13 @@ static int kvm_sclp_service_call(S390CPU *cpu, struct kvm_run *run,
     sccb = env->regs[ipbh0 & 0xf];
     code = env->regs[(ipbh0 & 0xf0) >> 4];
 
-    r = sclp_service_call(env, sccb, code);
+    if (run->s390_sieic.icptcode == ICPT_PV_INSTR ||
+        run->s390_sieic.icptcode == ICPT_PV_INSTR_NOT) {
+        r = sclp_service_call_protected(env, 0x42000, code);
+    } else {
+        r = sclp_service_call(env, sccb, code);
+    }
+
     if (r < 0) {
         kvm_s390_program_interrupt(cpu, -r);
     } else {
-- 
2.20.1

[PATCH 10/15] s390x: protvirt: Add new VCPU reset functions

[Janosch Frank]

CPU resets for protected guests need to be done via Ultravisor
calls. Hence we need a way to issue these calls for each reset.

As we formerly had only one reset function and it was called for
initial, as well as for the clear reset, we now need a new interface.

Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
---
 target/s390x/cpu.c       | 15 ++++++++++++---
 target/s390x/kvm-stub.c  | 10 +++++++++-
 target/s390x/kvm.c       | 38 ++++++++++++++++++++++++++++++++------
 target/s390x/kvm_s390x.h |  4 +++-
 4 files changed, 56 insertions(+), 11 deletions(-)

diff --git a/target/s390x/cpu.c b/target/s390x/cpu.c
index 10d5b915d8..ac04fdbba9 100644
--- a/target/s390x/cpu.c
+++ b/target/s390x/cpu.c
@@ -125,9 +125,18 @@ static void s390_cpu_reset(CPUState *s, uint8_t type)
     }
 
     /* Reset state inside the kernel that we cannot access yet from QEMU. */
-    if (kvm_enabled() && (type == S390_CPU_RESET_CLEAR ||
-                          type == S390_CPU_RESET_INITIAL)) {
-            kvm_s390_reset_vcpu(cpu);
+    if (kvm_enabled()) {
+        switch (type) {
+        case S390_CPU_RESET_CLEAR:
+            kvm_s390_reset_vcpu_clear(cpu);
+            break;
+        case S390_CPU_RESET_INITIAL:
+            kvm_s390_reset_vcpu_initial(cpu);
+            break;
+        case S390_CPU_RESET_NORMAL:
+            kvm_s390_reset_vcpu_normal(cpu);
+            break;
+        }
     }
 
 #if defined(CONFIG_USER_ONLY)
diff --git a/target/s390x/kvm-stub.c b/target/s390x/kvm-stub.c
index 5152e2bdf1..c4cd497f85 100644
--- a/target/s390x/kvm-stub.c
+++ b/target/s390x/kvm-stub.c
@@ -83,7 +83,15 @@ void kvm_s390_cmma_reset(void)
 {
 }
 
-void kvm_s390_reset_vcpu(S390CPU *cpu)
+void kvm_s390_reset_vcpu_initial(S390CPU *cpu)
+{
+}
+
+void kvm_s390_reset_vcpu_clear(S390CPU *cpu)
+{
+}
+
+void kvm_s390_reset_vcpu_normal(S390CPU *cpu)
 {
 }
 
diff --git a/target/s390x/kvm.c b/target/s390x/kvm.c
index 0f2458b553..190400df55 100644
--- a/target/s390x/kvm.c
+++ b/target/s390x/kvm.c
@@ -154,6 +154,7 @@ static int cap_ri;
 static int cap_gs;
 static int cap_hpage_1m;
 static int cap_protvirt;
+static int cap_vcpu_resets;
 
 static int active_cmma;
 
@@ -340,6 +341,7 @@ int kvm_arch_init(MachineState *ms, KVMState *s)
     cap_mem_op = kvm_check_extension(s, KVM_CAP_S390_MEM_OP);
     cap_s390_irq = kvm_check_extension(s, KVM_CAP_S390_INJECT_IRQ);
     cap_protvirt = kvm_check_extension(s, KVM_CAP_S390_PROTECTED);
+    cap_vcpu_resets = kvm_check_extension(s, KVM_CAP_S390_VCPU_RESETS);
 
     if (!kvm_check_extension(s, KVM_CAP_S390_GMAP)
         || !kvm_check_extension(s, KVM_CAP_S390_COW)) {
@@ -401,20 +403,44 @@ int kvm_arch_destroy_vcpu(CPUState *cs)
     return 0;
 }
 
-void kvm_s390_reset_vcpu(S390CPU *cpu)
+static void kvm_s390_reset_vcpu(S390CPU *cpu, unsigned long type)
 {
     CPUState *cs = CPU(cpu);
 
-    /* The initial reset call is needed here to reset in-kernel
-     * vcpu data that we can't access directly from QEMU
-     * (i.e. with older kernels which don't support sync_regs/ONE_REG).
-     * Before this ioctl cpu_synchronize_state() is called in common kvm
-     * code (kvm-all) */
+    /*
+     * The reset call is needed here to reset in-kernel vcpu data that
+     * we can't access directly from QEMU (i.e. with older kernels
+     * which don't support sync_regs/ONE_REG).  Before this ioctl
+     * cpu_synchronize_state() is called in common kvm code
+     * (kvm-all).
+     */
+    if (cap_vcpu_resets) {
+        if (kvm_vcpu_ioctl(cs, KVM_S390_VCPU_RESET, type)) {
+            error_report("CPU reset type %ld failed on CPU %i",
+                         type, cs->cpu_index);
+        }
+        return;
+    }
     if (kvm_vcpu_ioctl(cs, KVM_S390_INITIAL_RESET, NULL)) {
         error_report("Initial CPU reset failed on CPU %i", cs->cpu_index);
     }
 }
 
+void kvm_s390_reset_vcpu_initial(S390CPU *cpu)
+{
+    kvm_s390_reset_vcpu(cpu, KVM_S390_VCPU_RESET_INITIAL);
+}
+
+void kvm_s390_reset_vcpu_clear(S390CPU *cpu)
+{
+    kvm_s390_reset_vcpu(cpu, KVM_S390_VCPU_RESET_CLEAR);
+}
+
+void kvm_s390_reset_vcpu_normal(S390CPU *cpu)
+{
+    kvm_s390_reset_vcpu(cpu, KVM_S390_VCPU_RESET_NORMAL);
+}
+
 static int can_sync_regs(CPUState *cs, int regs)
 {
     return cap_sync_regs && (cs->kvm_run->kvm_valid_regs & regs) == regs;
diff --git a/target/s390x/kvm_s390x.h b/target/s390x/kvm_s390x.h
index caf985955b..0b21789796 100644
--- a/target/s390x/kvm_s390x.h
+++ b/target/s390x/kvm_s390x.h
@@ -34,7 +34,9 @@ int kvm_s390_assign_subch_ioeventfd(EventNotifier *notifier, uint32_t sch,
                                     int vq, bool assign);
 int kvm_s390_cmma_active(void);
 void kvm_s390_cmma_reset(void);
-void kvm_s390_reset_vcpu(S390CPU *cpu);
+void kvm_s390_reset_vcpu_clear(S390CPU *cpu);
+void kvm_s390_reset_vcpu_normal(S390CPU *cpu);
+void kvm_s390_reset_vcpu_initial(S390CPU *cpu);
 int kvm_s390_set_mem_limit(uint64_t new_limit, uint64_t *hw_limit);
 void kvm_s390_set_max_pagesize(uint64_t pagesize, Error **errp);
 void kvm_s390_crypto_reset(void);
-- 
2.20.1

[PATCH 11/15] RFC: s390x: Exit on vcpu reset error

[Janosch Frank]

If a vcpu is not properly reset it might be better to just end the VM.

Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
---
 target/s390x/kvm.c | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/target/s390x/kvm.c b/target/s390x/kvm.c
index 190400df55..0210b54157 100644
--- a/target/s390x/kvm.c
+++ b/target/s390x/kvm.c
@@ -418,11 +418,13 @@ static void kvm_s390_reset_vcpu(S390CPU *cpu, unsigned long type)
         if (kvm_vcpu_ioctl(cs, KVM_S390_VCPU_RESET, type)) {
             error_report("CPU reset type %ld failed on CPU %i",
                          type, cs->cpu_index);
+            exit(1);
         }
         return;
     }
     if (kvm_vcpu_ioctl(cs, KVM_S390_INITIAL_RESET, NULL)) {
         error_report("Initial CPU reset failed on CPU %i", cs->cpu_index);
+        exit(1);
     }
 }
 
-- 
2.20.1

[PATCH 12/15] s390x: protvirt: Set guest IPL PSW

[Janosch Frank]

Handling of CPU reset and setting of the IPL psw from guest storage at
offset 0 is done by a Ultravisor call. Let's only fetch it if
necessary.

Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
---
 hw/s390x/pv.c              | 5 +++++
 hw/s390x/pv.h              | 1 +
 hw/s390x/s390-virtio-ccw.c | 3 ++-
 linux-headers/linux/kvm.h  | 1 +
 target/s390x/cpu.c         | 9 ++++++++-
 5 files changed, 17 insertions(+), 2 deletions(-)

diff --git a/hw/s390x/pv.c b/hw/s390x/pv.c
index 0218070322..106252833f 100644
--- a/hw/s390x/pv.c
+++ b/hw/s390x/pv.c
@@ -88,6 +88,11 @@ int s390_pv_set_sec_parms(uint64_t origin, uint64_t length)
     return s390_pv_cmd(KVM_PV_VM_SET_SEC_PARMS, &args);
 }
 
+int s390_pv_set_ipl_psw(CPUState *cs)
+{
+    return s390_pv_cmd_vcpu(cs, KVM_PV_VCPU_SET_IPL_PSW, NULL);
+}
+
 /*
  * Called for each component in the SE type IPL parameter block 0.
  */
diff --git a/hw/s390x/pv.h b/hw/s390x/pv.h
index eb074e4bc9..e670c67270 100644
--- a/hw/s390x/pv.h
+++ b/hw/s390x/pv.h
@@ -18,6 +18,7 @@ int s390_pv_vm_destroy(void);
 int s390_pv_vcpu_destroy(CPUState *cs);
 int s390_pv_vcpu_create(CPUState *cs);
 int s390_pv_set_sec_parms(uint64_t origin, uint64_t length);
+int s390_pv_set_ipl_psw(CPUState *cs);
 int s390_pv_unpack(uint64_t addr, uint64_t size, uint64_t tweak);
 int s390_pv_perf_clear_reset(void);
 int s390_pv_verify(void);
diff --git a/hw/s390x/s390-virtio-ccw.c b/hw/s390x/s390-virtio-ccw.c
index 6fd50b4c42..e020b92854 100644
--- a/hw/s390x/s390-virtio-ccw.c
+++ b/hw/s390x/s390-virtio-ccw.c
@@ -404,7 +404,8 @@ static void s390_machine_reset(MachineState *machine)
         s390_ipl_pv_unpack();
         /* Verify integrity */
         s390_pv_verify();
-        s390_cpu_set_state(S390_CPU_STATE_OPERATING, cpu);
+        env->pv = true;
+        run_on_cpu(cs, s390_do_cpu_load_normal, RUN_ON_CPU_NULL);
         break;
     default:
         g_assert_not_reached();
diff --git a/linux-headers/linux/kvm.h b/linux-headers/linux/kvm.h
index d031051601..b5c7552016 100644
--- a/linux-headers/linux/kvm.h
+++ b/linux-headers/linux/kvm.h
@@ -1476,6 +1476,7 @@ enum pv_cmd_id {
 	KVM_PV_VM_UNSHARE,
 	KVM_PV_VCPU_CREATE,
 	KVM_PV_VCPU_DESTROY,
+	KVM_PV_VCPU_SET_IPL_PSW,
 };
 
 struct kvm_pv_cmd {
diff --git a/target/s390x/cpu.c b/target/s390x/cpu.c
index ac04fdbba9..c004987a87 100644
--- a/target/s390x/cpu.c
+++ b/target/s390x/cpu.c
@@ -37,6 +37,7 @@
 #include "sysemu/hw_accel.h"
 #include "hw/qdev-properties.h"
 #ifndef CONFIG_USER_ONLY
+#include "hw/s390x/pv.h"
 #include "hw/boards.h"
 #include "sysemu/arch_init.h"
 #include "sysemu/sysemu.h"
@@ -76,7 +77,13 @@ static bool s390_cpu_has_work(CPUState *cs)
 static void s390_cpu_load_normal(CPUState *s)
 {
     S390CPU *cpu = S390_CPU(s);
-    cpu->env.psw.addr = ldl_phys(s->as, 4) & PSW_MASK_ESA_ADDR;
+    CPUS390XState *env = &cpu->env;
+
+    if (!env->pv) {
+        cpu->env.psw.addr = ldl_phys(s->as, 4) & PSW_MASK_ESA_ADDR;
+    } else {
+        s390_pv_set_ipl_psw(s);
+    }
     cpu->env.psw.mask = PSW_MASK_32 | PSW_MASK_64;
     s390_cpu_set_state(S390_CPU_STATE_OPERATING, cpu);
 }
-- 
2.20.1

[PATCH 13/15] s390x: protvirt: Move diag 308 data over SIDAD

[Janosch Frank]

For protected guests the IPIB is written/read to/from the sattelite
block, so we need to make those accesses virtual to make them go
through KBM mem ops.

Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
---
 target/s390x/diag.c | 26 +++++++++++++++++++++++---
 1 file changed, 23 insertions(+), 3 deletions(-)

diff --git a/target/s390x/diag.c b/target/s390x/diag.c
index db6d79cef3..d96d8bdc6c 100644
--- a/target/s390x/diag.c
+++ b/target/s390x/diag.c
@@ -88,6 +88,7 @@ static int diag308_parm_check(CPUS390XState *env, uint64_t r1, uint64_t addr,
 void handle_diag_308(CPUS390XState *env, uint64_t r1, uint64_t r3, uintptr_t ra)
 {
     CPUState *cs = env_cpu(env);
+    S390CPU *cpu = S390_CPU(cs);
     uint64_t addr =  env->regs[r1];
     uint64_t subcode = env->regs[r3];
     IplParameterBlock *iplb;
@@ -118,14 +119,27 @@ void handle_diag_308(CPUS390XState *env, uint64_t r1, uint64_t r3, uintptr_t ra)
         if (diag308_parm_check(env, r1, addr, ra, false)) {
             return;
         }
+
         iplb = g_new0(IplParameterBlock, 1);
-        cpu_physical_memory_read(addr, iplb, sizeof(iplb->len));
+        if (!env->pv) {
+            cpu_physical_memory_read(addr, iplb, sizeof(iplb->len));
+        } else {
+            s390_cpu_virt_mem_read(cpu, 0, 0, iplb, sizeof(iplb->len));
+            s390_cpu_virt_mem_handle_exc(cpu, ra);
+        }
+
         if (!iplb_valid_len(iplb)) {
             env->regs[r1 + 1] = DIAG_308_RC_INVALID;
             goto out;
         }
 
-        cpu_physical_memory_read(addr, iplb, be32_to_cpu(iplb->len));
+        if (!env->pv) {
+            cpu_physical_memory_read(addr, iplb, be32_to_cpu(iplb->len));
+        } else {
+            s390_cpu_virt_mem_read(cpu, 0, 0, iplb, be32_to_cpu(iplb->len));
+            s390_cpu_virt_mem_handle_exc(cpu, ra);
+        }
+
 
         if (!iplb_valid_ccw(iplb) && !iplb_valid_fcp(iplb) &&
             !(iplb_valid_se(iplb) && s390_ipl_pv_check_comp(iplb) >= 0)) {
@@ -149,7 +163,13 @@ out:
             iplb = s390_ipl_get_iplb();
         }
         if (iplb) {
-            cpu_physical_memory_write(addr, iplb, be32_to_cpu(iplb->len));
+            if (!env->pv) {
+                cpu_physical_memory_write(addr, iplb, be32_to_cpu(iplb->len));
+            } else {
+                s390_cpu_virt_mem_write(cpu, 0, 0, iplb,
+                                        be32_to_cpu(iplb->len));
+                s390_cpu_virt_mem_handle_exc(cpu, ra);
+            }
             env->regs[r1 + 1] = DIAG_308_RC_OK;
         } else {
             env->regs[r1 + 1] = DIAG_308_RC_NO_CONF;
-- 
2.20.1

[PATCH 14/15] s390x: protvirt: Disable address checks for PV guest IO emulation

[Janosch Frank]

IO instruction data is routed through SIDAD for protected guests, so
adresses do not need to be checked, as this is kernel memory.

Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
---
 target/s390x/ioinst.c | 46 +++++++++++++++++++++++++++----------------
 1 file changed, 29 insertions(+), 17 deletions(-)

diff --git a/target/s390x/ioinst.c b/target/s390x/ioinst.c
index c437a1d8c6..d3bd422ddd 100644
--- a/target/s390x/ioinst.c
+++ b/target/s390x/ioinst.c
@@ -110,11 +110,13 @@ void ioinst_handle_msch(S390CPU *cpu, uint64_t reg1, uint32_t ipb, uintptr_t ra)
     int cssid, ssid, schid, m;
     SubchDev *sch;
     SCHIB schib;
-    uint64_t addr;
+    uint64_t addr = 0;
     CPUS390XState *env = &cpu->env;
-    uint8_t ar;
+    uint8_t ar = 0;
 
-    addr = decode_basedisp_s(env, ipb, &ar);
+    if (!env->pv) {
+        addr = decode_basedisp_s(env, ipb, &ar);
+    }
     if (addr & 3) {
         s390_program_interrupt(env, PGM_SPECIFICATION, ra);
         return;
@@ -167,11 +169,13 @@ void ioinst_handle_ssch(S390CPU *cpu, uint64_t reg1, uint32_t ipb, uintptr_t ra)
     int cssid, ssid, schid, m;
     SubchDev *sch;
     ORB orig_orb, orb;
-    uint64_t addr;
+    uint64_t addr = 0;
     CPUS390XState *env = &cpu->env;
-    uint8_t ar;
+    uint8_t ar = 0;
 
-    addr = decode_basedisp_s(env, ipb, &ar);
+    if (!env->pv) {
+        addr = decode_basedisp_s(env, ipb, &ar);
+    }
     if (addr & 3) {
         s390_program_interrupt(env, PGM_SPECIFICATION, ra);
         return;
@@ -198,12 +202,14 @@ void ioinst_handle_ssch(S390CPU *cpu, uint64_t reg1, uint32_t ipb, uintptr_t ra)
 void ioinst_handle_stcrw(S390CPU *cpu, uint32_t ipb, uintptr_t ra)
 {
     CRW crw;
-    uint64_t addr;
+    uint64_t addr = 0;
     int cc;
     CPUS390XState *env = &cpu->env;
-    uint8_t ar;
+    uint8_t ar = 0;
 
-    addr = decode_basedisp_s(env, ipb, &ar);
+    if (!env->pv) {
+        addr = decode_basedisp_s(env, ipb, &ar);
+    }
     if (addr & 3) {
         s390_program_interrupt(env, PGM_SPECIFICATION, ra);
         return;
@@ -228,13 +234,15 @@ void ioinst_handle_stsch(S390CPU *cpu, uint64_t reg1, uint32_t ipb,
 {
     int cssid, ssid, schid, m;
     SubchDev *sch;
-    uint64_t addr;
+    uint64_t addr = 0;
     int cc;
     SCHIB schib;
     CPUS390XState *env = &cpu->env;
-    uint8_t ar;
+    uint8_t ar = 0;
 
-    addr = decode_basedisp_s(env, ipb, &ar);
+    if (!env->pv) {
+        addr = decode_basedisp_s(env, ipb, &ar);
+    }
     if (addr & 3) {
         s390_program_interrupt(env, PGM_SPECIFICATION, ra);
         return;
@@ -294,16 +302,18 @@ int ioinst_handle_tsch(S390CPU *cpu, uint64_t reg1, uint32_t ipb, uintptr_t ra)
     int cssid, ssid, schid, m;
     SubchDev *sch;
     IRB irb;
-    uint64_t addr;
+    uint64_t addr = 0;
     int cc, irb_len;
-    uint8_t ar;
+    uint8_t ar = 0;
 
     if (ioinst_disassemble_sch_ident(reg1, &m, &cssid, &ssid, &schid)) {
         s390_program_interrupt(env, PGM_OPERAND, ra);
         return -EIO;
     }
     trace_ioinst_sch_id("tsch", cssid, ssid, schid);
-    addr = decode_basedisp_s(env, ipb, &ar);
+    if (!env->pv) {
+        addr = decode_basedisp_s(env, ipb, &ar);
+    }
     if (addr & 3) {
         s390_program_interrupt(env, PGM_SPECIFICATION, ra);
         return -EIO;
@@ -601,7 +611,7 @@ void ioinst_handle_chsc(S390CPU *cpu, uint32_t ipb, uintptr_t ra)
 {
     ChscReq *req;
     ChscResp *res;
-    uint64_t addr;
+    uint64_t addr = 0;
     int reg;
     uint16_t len;
     uint16_t command;
@@ -610,7 +620,9 @@ void ioinst_handle_chsc(S390CPU *cpu, uint32_t ipb, uintptr_t ra)
 
     trace_ioinst("chsc");
     reg = (ipb >> 20) & 0x00f;
-    addr = env->regs[reg];
+    if (!env->pv) {
+        addr = env->regs[reg];
+    }
     /* Page boundary? */
     if (addr & 0xfff) {
         s390_program_interrupt(env, PGM_SPECIFICATION, ra);
-- 
2.20.1

[PATCH 15/15] s390x: protvirt: Handle SIGP store status correctly

[Janosch Frank]

Status storing is obviously not done by qemu anymore.

Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
---
 target/s390x/sigp.c | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/target/s390x/sigp.c b/target/s390x/sigp.c
index 2ce22d4dc1..68634d694a 100644
--- a/target/s390x/sigp.c
+++ b/target/s390x/sigp.c
@@ -144,7 +144,9 @@ static void sigp_stop_and_store_status(CPUState *cs, run_on_cpu_data arg)
     case S390_CPU_STATE_STOPPED:
         /* already stopped, just store the status */
         cpu_synchronize_state(cs);
-        s390_store_status(cpu, S390_STORE_STATUS_DEF_ADDR, true);
+        if (!cpu->env.pv) {
+            s390_store_status(cpu, S390_STORE_STATUS_DEF_ADDR, true);
+        }
         break;
     }
     si->cc = SIGP_CC_ORDER_CODE_ACCEPTED;
@@ -497,7 +499,8 @@ void do_stop_interrupt(CPUS390XState *env)
     if (s390_cpu_set_state(S390_CPU_STATE_STOPPED, cpu) == 0) {
         qemu_system_shutdown_request(SHUTDOWN_CAUSE_GUEST_SHUTDOWN);
     }
-    if (cpu->env.sigp_order == SIGP_STOP_STORE_STATUS) {
+    /* Storing will occur on next SIE entry for fmt 4 */
+    if (cpu->env.sigp_order == SIGP_STOP_STORE_STATUS && !env->pv) {
         s390_store_status(cpu, S390_STORE_STATUS_DEF_ADDR, true);
     }
     env->sigp_order = 0;
-- 
2.20.1

Re: [PATCH 00/15] s390x: Protected Virtualization support

[Cornelia Huck]

On Wed, 20 Nov 2019 06:43:19 -0500
Janosch Frank <frankja@linux.ibm.com> wrote:

Do you have a branch with this somewhere?

> Most of the QEMU changes for PV are related to the new IPL type with
> subcodes 8 - 10 and the execution of the necessary Ultravisor calls to
> IPL secure guests. Note that we can only boot into secure mode from
> normal mode, i.e. stfle 161 is not active in secure mode.
> 
> The other changes related to data gathering for emulation and
> disabling addressing checks in secure mode, as well as CPU resets.
> 
> While working on this I sprinkled in some cleanups, as we sometimes
> significantly increase line count of some functions and they got
> unreadable.

Any other cleanups than in the first two patches? I.e., anything that
could be picked up independently?

> 
> Janosch Frank (15):
>   s390x: Cleanup cpu resets
>   s390x: Beautify diag308 handling
>   s390x: protvirt: Add diag308 subcodes 8 - 10
>   Header sync protvirt
>   s390x: protvirt: Sync PV state
>   s390x: protvirt: Support unpack facility
>   s390x: protvirt: Handle diag 308 subcodes 0,1,3,4
>   s390x: protvirt: KVM intercept changes
>   s390x: protvirt: SCLP interpretation
>   s390x: protvirt: Add new VCPU reset functions
>   RFC: s390x: Exit on vcpu reset error
>   s390x: protvirt: Set guest IPL PSW
>   s390x: protvirt: Move diag 308 data over SIDAD
>   s390x: protvirt: Disable address checks for PV guest IO emulation
>   s390x: protvirt: Handle SIGP store status correctly
> 
>  hw/s390x/Makefile.objs              |   1 +
>  hw/s390x/ipl.c                      |  81 +++++++++++++++++-
>  hw/s390x/ipl.h                      |  35 ++++++++
>  hw/s390x/pv.c                       | 123 +++++++++++++++++++++++++++
>  hw/s390x/pv.h                       |  27 ++++++
>  hw/s390x/s390-virtio-ccw.c          |  79 ++++++++++++++---
>  hw/s390x/sclp.c                     |  16 ++++
>  include/hw/s390x/sclp.h             |   2 +
>  linux-headers/asm-s390/kvm.h        |   4 +-
>  linux-headers/linux/kvm.h           |  43 ++++++++++
>  target/s390x/cpu.c                  | 127 ++++++++++++++--------------
>  target/s390x/cpu.h                  |   1 +
>  target/s390x/cpu_features_def.inc.h |   1 +
>  target/s390x/diag.c                 | 108 +++++++++++++++++------
>  target/s390x/ioinst.c               |  46 ++++++----
>  target/s390x/kvm-stub.c             |  10 ++-
>  target/s390x/kvm.c                  |  58 +++++++++++--
>  target/s390x/kvm_s390x.h            |   4 +-
>  target/s390x/sigp.c                 |   7 +-
>  19 files changed, 640 insertions(+), 133 deletions(-)
>  create mode 100644 hw/s390x/pv.c
>  create mode 100644 hw/s390x/pv.h
> 

Re: [PATCH 00/15] s390x: Protected Virtualization support

[Janosch Frank]

[-- Attachment #1.1: Type: text/plain, Size: 2889 bytes --]

On 11/20/19 2:26 PM, Cornelia Huck wrote:
> On Wed, 20 Nov 2019 06:43:19 -0500
> Janosch Frank <frankja@linux.ibm.com> wrote:
> 
> Do you have a branch with this somewhere?
> 
>> Most of the QEMU changes for PV are related to the new IPL type with
>> subcodes 8 - 10 and the execution of the necessary Ultravisor calls to
>> IPL secure guests. Note that we can only boot into secure mode from
>> normal mode, i.e. stfle 161 is not active in secure mode.
>>
>> The other changes related to data gathering for emulation and
>> disabling addressing checks in secure mode, as well as CPU resets.
>>
>> While working on this I sprinkled in some cleanups, as we sometimes
>> significantly increase line count of some functions and they got
>> unreadable.
> 
> Any other cleanups than in the first two patches? I.e., anything that
> could be picked up independently?

Maybe patch #11, but that's RFC

> 
>>
>> Janosch Frank (15):
>>   s390x: Cleanup cpu resets
>>   s390x: Beautify diag308 handling
>>   s390x: protvirt: Add diag308 subcodes 8 - 10
>>   Header sync protvirt
>>   s390x: protvirt: Sync PV state
>>   s390x: protvirt: Support unpack facility
>>   s390x: protvirt: Handle diag 308 subcodes 0,1,3,4
>>   s390x: protvirt: KVM intercept changes
>>   s390x: protvirt: SCLP interpretation
>>   s390x: protvirt: Add new VCPU reset functions
>>   RFC: s390x: Exit on vcpu reset error
>>   s390x: protvirt: Set guest IPL PSW
>>   s390x: protvirt: Move diag 308 data over SIDAD
>>   s390x: protvirt: Disable address checks for PV guest IO emulation
>>   s390x: protvirt: Handle SIGP store status correctly
>>
>>  hw/s390x/Makefile.objs              |   1 +
>>  hw/s390x/ipl.c                      |  81 +++++++++++++++++-
>>  hw/s390x/ipl.h                      |  35 ++++++++
>>  hw/s390x/pv.c                       | 123 +++++++++++++++++++++++++++
>>  hw/s390x/pv.h                       |  27 ++++++
>>  hw/s390x/s390-virtio-ccw.c          |  79 ++++++++++++++---
>>  hw/s390x/sclp.c                     |  16 ++++
>>  include/hw/s390x/sclp.h             |   2 +
>>  linux-headers/asm-s390/kvm.h        |   4 +-
>>  linux-headers/linux/kvm.h           |  43 ++++++++++
>>  target/s390x/cpu.c                  | 127 ++++++++++++++--------------
>>  target/s390x/cpu.h                  |   1 +
>>  target/s390x/cpu_features_def.inc.h |   1 +
>>  target/s390x/diag.c                 | 108 +++++++++++++++++------
>>  target/s390x/ioinst.c               |  46 ++++++----
>>  target/s390x/kvm-stub.c             |  10 ++-
>>  target/s390x/kvm.c                  |  58 +++++++++++--
>>  target/s390x/kvm_s390x.h            |   4 +-
>>  target/s390x/sigp.c                 |   7 +-
>>  19 files changed, 640 insertions(+), 133 deletions(-)
>>  create mode 100644 hw/s390x/pv.c
>>  create mode 100644 hw/s390x/pv.h
>>
> 
> 



[-- Attachment #2: OpenPGP digital signature --]
[-- Type: application/pgp-signature, Size: 833 bytes --]

Re: [PATCH 06/15] s390x: protvirt: Support unpack facility

[Cornelia Huck]

On Wed, 20 Nov 2019 06:43:25 -0500
Janosch Frank <frankja@linux.ibm.com> wrote:

> When a guest has saved a ipib of type 5 and call diagnose308 with
> subcode 10, we have to setup the protected processing environment via
> Ultravisor calls. The calls are done by KVM and are exposed via an API.
> 
> The following steps are necessary:
> 1. Create a VM (register it with the Ultravisor)
> 2. Create secure CPUs for all of our current cpus
> 3. Forward the secure header to the Ultravisor (has all information on
> how to decrypt the image and VM information)
> 4. Protect image pages from the host and decrypt them
> 5. Verify the image integrity
> 
> Only after step 5 a protected VM is allowed to run.
> 
> Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
> ---
>  hw/s390x/Makefile.objs              |   1 +
>  hw/s390x/ipl.c                      |  33 ++++++++
>  hw/s390x/ipl.h                      |   2 +
>  hw/s390x/pv.c                       | 118 ++++++++++++++++++++++++++++
>  hw/s390x/pv.h                       |  26 ++++++
>  hw/s390x/s390-virtio-ccw.c          |  45 ++++++++---
>  target/s390x/cpu_features_def.inc.h |   1 +
>  7 files changed, 216 insertions(+), 10 deletions(-)
>  create mode 100644 hw/s390x/pv.c
>  create mode 100644 hw/s390x/pv.h
> 
> diff --git a/hw/s390x/Makefile.objs b/hw/s390x/Makefile.objs
> index 94e57113d8..568bab9711 100644
> --- a/hw/s390x/Makefile.objs
> +++ b/hw/s390x/Makefile.objs
> @@ -31,6 +31,7 @@ obj-y += tod-qemu.o
>  obj-$(CONFIG_KVM) += tod-kvm.o
>  obj-$(CONFIG_KVM) += s390-skeys-kvm.o
>  obj-$(CONFIG_KVM) += s390-stattrib-kvm.o s390-mchk.o
> +obj-$(CONFIG_KVM) += pv.o

As this is kvm only...

>  obj-y += s390-ccw.o
>  obj-y += ap-device.o
>  obj-y += ap-bridge.o
> diff --git a/hw/s390x/ipl.c b/hw/s390x/ipl.c
> index a077926f36..50501fcd27 100644
> --- a/hw/s390x/ipl.c
> +++ b/hw/s390x/ipl.c
> @@ -33,6 +33,7 @@
>  #include "qemu/cutils.h"
>  #include "qemu/option.h"
>  #include "exec/exec-all.h"
> +#include "pv.h"
>  
>  #define KERN_IMAGE_START                0x010000UL
>  #define LINUX_MAGIC_ADDR                0x010008UL
> @@ -668,6 +669,38 @@ static void s390_ipl_prepare_qipl(S390CPU *cpu)
>      cpu_physical_memory_unmap(addr, len, 1, len);
>  }
>  
> +int s390_ipl_prepare_pv_header(void)
> +{
> +    int rc;
> +    IplParameterBlock *iplb = s390_ipl_get_iplb_secure();
> +    IPLBlockPV *ipib_pv = &iplb->pv;
> +    void *hdr = g_malloc(ipib_pv->pv_header_len);
> +
> +    cpu_physical_memory_read(ipib_pv->pv_header_addr, hdr,
> +                             ipib_pv->pv_header_len);
> +    rc = s390_pv_set_sec_parms((uint64_t)hdr,
> +                               ipib_pv->pv_header_len);
> +    g_free(hdr);
> +    return rc;
> +}
> +
> +int s390_ipl_pv_unpack(void)
> +{
> +    int i, rc;
> +    IplParameterBlock *iplb = s390_ipl_get_iplb_secure();
> +    IPLBlockPV *ipib_pv = &iplb->pv;
> +
> +    for (i = 0; i < ipib_pv->num_comp; i++) {
> +        rc = s390_pv_unpack(ipib_pv->components[i].addr,
> +                            TARGET_PAGE_ALIGN(ipib_pv->components[i].size),
> +                            ipib_pv->components[i].tweak_pref);

...you probably need a stub version of the pv functions as well, right?

> +        if (rc) {
> +            return rc;
> +        }
> +    }
> +    return rc;
> +}
> +
>  void s390_ipl_prepare_cpu(S390CPU *cpu)
>  {
>      S390IPLState *ipl = get_ipl_device();

Re: [PATCH 00/15] s390x: Protected Virtualization support

[Janosch Frank]

On 11/20/19 2:26 PM, Cornelia Huck wrote:
> On Wed, 20 Nov 2019 06:43:19 -0500
> Janosch Frank <frankja@linux.ibm.com> wrote:
> 
> Do you have a branch with this somewhere?

Just for you:
https://github.com/frankjaa/qemu/tree/protvirt

Re: [PATCH 00/15] s390x: Protected Virtualization support

[Cornelia Huck]

On Thu, 21 Nov 2019 10:13:29 +0100
Janosch Frank <frankja@linux.ibm.com> wrote:

> On 11/20/19 2:26 PM, Cornelia Huck wrote:
> > On Wed, 20 Nov 2019 06:43:19 -0500
> > Janosch Frank <frankja@linux.ibm.com> wrote:
> > 
> > Do you have a branch with this somewhere?  
> 
> Just for you:
> https://github.com/frankjaa/qemu/tree/protvirt
> 

Thanks!

Re: [PATCH 01/15] s390x: Cleanup cpu resets

[Cornelia Huck]

On Wed, 20 Nov 2019 06:43:20 -0500
Janosch Frank <frankja@linux.ibm.com> wrote:

> Let's move the resets into one function and switch by type, so we can
> use fallthroughs for shared reset actions.

Doing that makes sense.

> 
> Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
> ---
>  hw/s390x/s390-virtio-ccw.c |   3 +
>  target/s390x/cpu.c         | 111 ++++++++++++++++---------------------
>  2 files changed, 52 insertions(+), 62 deletions(-)
> 
> diff --git a/hw/s390x/s390-virtio-ccw.c b/hw/s390x/s390-virtio-ccw.c
> index d3edeef0ad..c1d1440272 100644
> --- a/hw/s390x/s390-virtio-ccw.c
> +++ b/hw/s390x/s390-virtio-ccw.c
> @@ -348,6 +348,9 @@ static void s390_machine_reset(MachineState *machine)
>          break;
>      case S390_RESET_LOAD_NORMAL:
>          CPU_FOREACH(t) {
> +            if (t == cs) {
> +                continue;
> +            }

Hm, why is this needed now?

>              run_on_cpu(t, s390_do_cpu_reset, RUN_ON_CPU_NULL);
>          }
>          subsystem_reset();
> diff --git a/target/s390x/cpu.c b/target/s390x/cpu.c
> index 3abe7e80fd..10d5b915d8 100644
> --- a/target/s390x/cpu.c
> +++ b/target/s390x/cpu.c
> @@ -82,67 +82,53 @@ static void s390_cpu_load_normal(CPUState *s)
>  }
>  #endif
>  
> -/* S390CPUClass::cpu_reset() */

Not sure if it would be worth keeping these comments near by the
calling functions.

> -static void s390_cpu_reset(CPUState *s)
> +enum {
> +    S390_CPU_RESET_NORMAL,
> +    S390_CPU_RESET_INITIAL,
> +    S390_CPU_RESET_CLEAR,
> +};

Maybe make this into a proper type, so you can use type checking?

(...)

The diff is a bit hard to read, but the change seems fine at a glance.

Re: [PATCH 02/15] s390x: Beautify diag308 handling

[Cornelia Huck]

On Wed, 20 Nov 2019 06:43:21 -0500
Janosch Frank <frankja@linux.ibm.com> wrote:

> Let's improve readability by:
> * Using constants for the subcodes
> * Moving parameter checking into a function
> * Removing subcode > 6 check as the default case catches that
> 
> Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
> ---
>  target/s390x/diag.c | 54 +++++++++++++++++++++++++++------------------
>  1 file changed, 32 insertions(+), 22 deletions(-)
> 
> diff --git a/target/s390x/diag.c b/target/s390x/diag.c
> index 53c2f81f2a..067c667ba7 100644
> --- a/target/s390x/diag.c
> +++ b/target/s390x/diag.c
> @@ -53,6 +53,29 @@ int handle_diag_288(CPUS390XState *env, uint64_t r1, uint64_t r3)
>  #define DIAG_308_RC_NO_CONF         0x0102
>  #define DIAG_308_RC_INVALID         0x0402
>  
> +#define DIAG308_RES_MOD_CLR		0
> +#define DIAG308_RES_LOAD_NORM		1

Maybe s/RES/RESET/ ? Not that much longer, but clearer IMHO.

> +#define DIAG308_LOAD_CLEAR		3
> +#define DIAG308_LOAD_NORMAL_DUMP	4
> +#define DIAG308_SET			5
> +#define DIAG308_STORE			6

(...)

Really more readable afterwards.

Reviewed-by: Cornelia Huck <cohuck@redhat.com>

Re: [PATCH 02/15] s390x: Beautify diag308 handling

[David Hildenbrand]

On 20.11.19 12:43, Janosch Frank wrote:
> Let's improve readability by:
> * Using constants for the subcodes
> * Moving parameter checking into a function
> * Removing subcode > 6 check as the default case catches that
> 
> Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
> ---
>   target/s390x/diag.c | 54 +++++++++++++++++++++++++++------------------
>   1 file changed, 32 insertions(+), 22 deletions(-)
> 
> diff --git a/target/s390x/diag.c b/target/s390x/diag.c
> index 53c2f81f2a..067c667ba7 100644
> --- a/target/s390x/diag.c
> +++ b/target/s390x/diag.c
> @@ -53,6 +53,29 @@ int handle_diag_288(CPUS390XState *env, uint64_t r1, uint64_t r3)
>   #define DIAG_308_RC_NO_CONF         0x0102
>   #define DIAG_308_RC_INVALID         0x0402
>   
> +#define DIAG308_RES_MOD_CLR		0
> +#define DIAG308_RES_LOAD_NORM		1
> +#define DIAG308_LOAD_CLEAR		3
> +#define DIAG308_LOAD_NORMAL_DUMP	4
> +#define DIAG308_SET			5
> +#define DIAG308_STORE			6
> +
> +static int diag308_parm_check(CPUS390XState *env, uint64_t r1, uint64_t addr,
> +                              uintptr_t ra, bool write)
> +{
> +    if ((r1 & 1) || (addr & ~TARGET_PAGE_MASK)) {
> +        s390_program_interrupt(env, PGM_SPECIFICATION, ra);
> +        return -EINVAL;
> +    }
> +    if (!address_space_access_valid(&address_space_memory, addr,
> +                                    sizeof(IplParameterBlock), write,
> +                                    MEMTXATTRS_UNSPECIFIED)) {
> +        s390_program_interrupt(env, PGM_ADDRESSING, ra);
> +        return -EINVAL;
> +    }
> +    return 0;
> +}
> +
>   void handle_diag_308(CPUS390XState *env, uint64_t r1, uint64_t r3, uintptr_t ra)
>   {
>       CPUState *cs = env_cpu(env);
> @@ -65,30 +88,24 @@ void handle_diag_308(CPUS390XState *env, uint64_t r1, uint64_t r3, uintptr_t ra)
>           return;
>       }
>   
> -    if ((subcode & ~0x0ffffULL) || (subcode > 6)) {
> +    if (subcode & ~0x0ffffULL) {

Strange, the default case in the switch was basically dead code.

>           s390_program_interrupt(env, PGM_SPECIFICATION, ra);
>           return;
>       }
>   
>       switch (subcode) {
> -    case 0:
> +    case DIAG308_RES_MOD_CLR:
>           s390_ipl_reset_request(cs, S390_RESET_MODIFIED_CLEAR);
>           break;
> -    case 1:
> +    case DIAG308_RES_LOAD_NORM:
>           s390_ipl_reset_request(cs, S390_RESET_LOAD_NORMAL);
>           break;
> -    case 3:
> +    case DIAG308_LOAD_CLEAR:
> +        /* Well we still lack the clearing bit... */
>           s390_ipl_reset_request(cs, S390_RESET_REIPL);
>           break;
> -    case 5:
> -        if ((r1 & 1) || (addr & 0x0fffULL)) {
> -            s390_program_interrupt(env, PGM_SPECIFICATION, ra);
> -            return;
> -        }
> -        if (!address_space_access_valid(&address_space_memory, addr,
> -                                        sizeof(IplParameterBlock), false,
> -                                        MEMTXATTRS_UNSPECIFIED)) {
> -            s390_program_interrupt(env, PGM_ADDRESSING, ra);
> +    case DIAG308_SET:
> +        if (diag308_parm_check(env, r1, addr, ra, false)) {
>               return;
>           }
>           iplb = g_new0(IplParameterBlock, 1);
> @@ -110,15 +127,8 @@ void handle_diag_308(CPUS390XState *env, uint64_t r1, uint64_t r3, uintptr_t ra)
>   out:
>           g_free(iplb);
>           return;
> -    case 6:
> -        if ((r1 & 1) || (addr & 0x0fffULL)) {
> -            s390_program_interrupt(env, PGM_SPECIFICATION, ra);
> -            return;
> -        }
> -        if (!address_space_access_valid(&address_space_memory, addr,
> -                                        sizeof(IplParameterBlock), true,
> -                                        MEMTXATTRS_UNSPECIFIED)) {
> -            s390_program_interrupt(env, PGM_ADDRESSING, ra);
> +    case DIAG308_STORE:
> +        if (diag308_parm_check(env, r1, addr, ra, true)) {
>               return;
>           }
>           iplb = s390_ipl_get_iplb();
> 

Reviewed-by: David Hildenbrand <david@redhat.com>

-- 

Thanks,

David / dhildenb

Re: [PATCH 15/15] s390x: protvirt: Handle SIGP store status correctly

[David Hildenbrand]

On 20.11.19 12:43, Janosch Frank wrote:
> Status storing is obviously not done by qemu anymore.
> 
> Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
> ---
>   target/s390x/sigp.c | 7 +++++--
>   1 file changed, 5 insertions(+), 2 deletions(-)
> 
> diff --git a/target/s390x/sigp.c b/target/s390x/sigp.c
> index 2ce22d4dc1..68634d694a 100644
> --- a/target/s390x/sigp.c
> +++ b/target/s390x/sigp.c
> @@ -144,7 +144,9 @@ static void sigp_stop_and_store_status(CPUState *cs, run_on_cpu_data arg)
>       case S390_CPU_STATE_STOPPED:
>           /* already stopped, just store the status */
>           cpu_synchronize_state(cs);
> -        s390_store_status(cpu, S390_STORE_STATUS_DEF_ADDR, true);
> +        if (!cpu->env.pv) {
> +            s390_store_status(cpu, S390_STORE_STATUS_DEF_ADDR, true);
> +        }

Confused, how is that case handled? The CPU is already stopped, so it 
won't be run again (consequently, next SIE entry can't store it). Who 
will store the status?

>           break;
>       }
>       si->cc = SIGP_CC_ORDER_CODE_ACCEPTED;
> @@ -497,7 +499,8 @@ void do_stop_interrupt(CPUS390XState *env)
>       if (s390_cpu_set_state(S390_CPU_STATE_STOPPED, cpu) == 0) {
>           qemu_system_shutdown_request(SHUTDOWN_CAUSE_GUEST_SHUTDOWN);
>       }
> -    if (cpu->env.sigp_order == SIGP_STOP_STORE_STATUS) {
> +    /* Storing will occur on next SIE entry for fmt 4 */
> +    if (cpu->env.sigp_order == SIGP_STOP_STORE_STATUS && !env->pv) {
>           s390_store_status(cpu, S390_STORE_STATUS_DEF_ADDR, true);
>       }
>       env->sigp_order = 0;
> 


-- 

Thanks,

David / dhildenb

Re: [PATCH 02/15] s390x: Beautify diag308 handling

[Janosch Frank]

[-- Attachment #1.1: Type: text/plain, Size: 1288 bytes --]

On 11/21/19 12:17 PM, Cornelia Huck wrote:
> On Wed, 20 Nov 2019 06:43:21 -0500
> Janosch Frank <frankja@linux.ibm.com> wrote:
> 
>> Let's improve readability by:
>> * Using constants for the subcodes
>> * Moving parameter checking into a function
>> * Removing subcode > 6 check as the default case catches that
>>
>> Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
>> ---
>>  target/s390x/diag.c | 54 +++++++++++++++++++++++++++------------------
>>  1 file changed, 32 insertions(+), 22 deletions(-)
>>
>> diff --git a/target/s390x/diag.c b/target/s390x/diag.c
>> index 53c2f81f2a..067c667ba7 100644
>> --- a/target/s390x/diag.c
>> +++ b/target/s390x/diag.c
>> @@ -53,6 +53,29 @@ int handle_diag_288(CPUS390XState *env, uint64_t r1, uint64_t r3)
>>  #define DIAG_308_RC_NO_CONF         0x0102
>>  #define DIAG_308_RC_INVALID         0x0402
>>  
>> +#define DIAG308_RES_MOD_CLR		0
>> +#define DIAG308_RES_LOAD_NORM		1
> 
> Maybe s/RES/RESET/ ? Not that much longer, but clearer IMHO.

Sure

> 
>> +#define DIAG308_LOAD_CLEAR		3
>> +#define DIAG308_LOAD_NORMAL_DUMP	4
>> +#define DIAG308_SET			5
>> +#define DIAG308_STORE			6
> 
> (...)
> 
> Really more readable afterwards.
> 
> Reviewed-by: Cornelia Huck <cohuck@redhat.com>

Thanks!



[-- Attachment #2: OpenPGP digital signature --]
[-- Type: application/pgp-signature, Size: 833 bytes --]

Re: [PATCH 06/15] s390x: protvirt: Support unpack facility

[David Hildenbrand]

On 20.11.19 12:43, Janosch Frank wrote:
> When a guest has saved a ipib of type 5 and call diagnose308 with
> subcode 10, we have to setup the protected processing environment via
> Ultravisor calls. The calls are done by KVM and are exposed via an API.
> 
> The following steps are necessary:
> 1. Create a VM (register it with the Ultravisor)
> 2. Create secure CPUs for all of our current cpus
> 3. Forward the secure header to the Ultravisor (has all information on
> how to decrypt the image and VM information)
> 4. Protect image pages from the host and decrypt them
> 5. Verify the image integrity
> 
> Only after step 5 a protected VM is allowed to run.
> 
> Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
> ---
>   hw/s390x/Makefile.objs              |   1 +
>   hw/s390x/ipl.c                      |  33 ++++++++
>   hw/s390x/ipl.h                      |   2 +
>   hw/s390x/pv.c                       | 118 ++++++++++++++++++++++++++++
>   hw/s390x/pv.h                       |  26 ++++++
>   hw/s390x/s390-virtio-ccw.c          |  45 ++++++++---
>   target/s390x/cpu_features_def.inc.h |   1 +
>   7 files changed, 216 insertions(+), 10 deletions(-)
>   create mode 100644 hw/s390x/pv.c
>   create mode 100644 hw/s390x/pv.h
> 
> diff --git a/hw/s390x/Makefile.objs b/hw/s390x/Makefile.objs
> index 94e57113d8..568bab9711 100644
> --- a/hw/s390x/Makefile.objs
> +++ b/hw/s390x/Makefile.objs
> @@ -31,6 +31,7 @@ obj-y += tod-qemu.o
>   obj-$(CONFIG_KVM) += tod-kvm.o
>   obj-$(CONFIG_KVM) += s390-skeys-kvm.o
>   obj-$(CONFIG_KVM) += s390-stattrib-kvm.o s390-mchk.o
> +obj-$(CONFIG_KVM) += pv.o
>   obj-y += s390-ccw.o
>   obj-y += ap-device.o
>   obj-y += ap-bridge.o
> diff --git a/hw/s390x/ipl.c b/hw/s390x/ipl.c
> index a077926f36..50501fcd27 100644
> --- a/hw/s390x/ipl.c
> +++ b/hw/s390x/ipl.c
> @@ -33,6 +33,7 @@
>   #include "qemu/cutils.h"
>   #include "qemu/option.h"
>   #include "exec/exec-all.h"
> +#include "pv.h"
>   
>   #define KERN_IMAGE_START                0x010000UL
>   #define LINUX_MAGIC_ADDR                0x010008UL
> @@ -668,6 +669,38 @@ static void s390_ipl_prepare_qipl(S390CPU *cpu)
>       cpu_physical_memory_unmap(addr, len, 1, len);
>   }
>   
> +int s390_ipl_prepare_pv_header(void)
> +{
> +    int rc;
> +    IplParameterBlock *iplb = s390_ipl_get_iplb_secure();
> +    IPLBlockPV *ipib_pv = &iplb->pv;
> +    void *hdr = g_malloc(ipib_pv->pv_header_len);
> +
> +    cpu_physical_memory_read(ipib_pv->pv_header_addr, hdr,
> +                             ipib_pv->pv_header_len);
> +    rc = s390_pv_set_sec_parms((uint64_t)hdr,
> +                               ipib_pv->pv_header_len);
> +    g_free(hdr);
> +    return rc;
> +}
> +
> +int s390_ipl_pv_unpack(void)
> +{
> +    int i, rc;
> +    IplParameterBlock *iplb = s390_ipl_get_iplb_secure();
> +    IPLBlockPV *ipib_pv = &iplb->pv;
> +
> +    for (i = 0; i < ipib_pv->num_comp; i++) {
> +        rc = s390_pv_unpack(ipib_pv->components[i].addr,
> +                            TARGET_PAGE_ALIGN(ipib_pv->components[i].size),
> +                            ipib_pv->components[i].tweak_pref);
> +        if (rc) {
> +            return rc;
> +        }
> +    }
> +    return rc;
> +}
> +
>   void s390_ipl_prepare_cpu(S390CPU *cpu)
>   {
>       S390IPLState *ipl = get_ipl_device();
> diff --git a/hw/s390x/ipl.h b/hw/s390x/ipl.h
> index 7b8a493509..e848602c16 100644
> --- a/hw/s390x/ipl.h
> +++ b/hw/s390x/ipl.h
> @@ -105,6 +105,8 @@ typedef union IplParameterBlock IplParameterBlock;
>   int s390_ipl_set_loadparm(uint8_t *loadparm);
>   int s390_ipl_pv_check_comp(IplParameterBlock *iplb);
>   void s390_ipl_update_diag308(IplParameterBlock *iplb);
> +int s390_ipl_prepare_pv_header(void);
> +int s390_ipl_pv_unpack(void);
>   void s390_ipl_prepare_cpu(S390CPU *cpu);
>   IplParameterBlock *s390_ipl_get_iplb(void);
>   IplParameterBlock *s390_ipl_get_iplb_secure(void);
> diff --git a/hw/s390x/pv.c b/hw/s390x/pv.c
> new file mode 100644
> index 0000000000..0218070322
> --- /dev/null
> +++ b/hw/s390x/pv.c
> @@ -0,0 +1,118 @@
> +/*
> + * Secure execution functions
> + *
> + * Copyright IBM Corp. 2019
> + * Author(s):
> + *  Janosch Frank <frankja@linux.ibm.com>
> + *
> + * This work is licensed under the terms of the GNU GPL, version 2 or (at
> + * your option) any later version. See the COPYING file in the top-level
> + * directory.
> + */
> +#include "qemu/osdep.h"
> +#include <sys/ioctl.h>
> +
> +#include <linux/kvm.h>
> +
> +#include "qemu/error-report.h"
> +#include "sysemu/kvm.h"
> +#include "pv.h"
> +
> +static int s390_pv_cmd(uint32_t cmd, void *data)
> +{
> +    int rc;
> +    struct kvm_pv_cmd pv_cmd = {
> +        .cmd = cmd,
> +        .data = (uint64_t)data,
> +    };
> +
> +    rc = kvm_vm_ioctl(kvm_state, KVM_S390_PV_COMMAND, &pv_cmd);
> +    if (rc) {
> +        error_report("KVM PV command failed cmd: %d rc: %d", cmd, rc);
> +        exit(1);
> +    }
> +    return rc;
> +}
> +
> +static int s390_pv_cmd_vcpu(CPUState *cs, uint32_t cmd, void *data)
> +{
> +    int rc;
> +    struct kvm_pv_cmd pv_cmd = {
> +        .cmd = cmd,
> +        .data = (uint64_t)data,
> +    };
> +
> +    rc = kvm_vcpu_ioctl(cs, KVM_S390_PV_COMMAND_VCPU, &pv_cmd);
> +    if (rc) {
> +        error_report("KVM PV VCPU command failed cmd: %d rc: %d", cmd, rc);
> +        exit(1);
> +    }
> +    return rc;
> +}
> +
> +int s390_pv_vm_create(void)
> +{
> +    return s390_pv_cmd(KVM_PV_VM_CREATE, NULL);
> +}
> +
> +int s390_pv_vm_destroy(void)
> +{
> +    return s390_pv_cmd(KVM_PV_VM_DESTROY, NULL);
> +}
> +
> +int s390_pv_vcpu_create(CPUState *cs)
> +{
> +    return s390_pv_cmd_vcpu(cs, KVM_PV_VCPU_CREATE, NULL);
> +}
> +
> +int s390_pv_vcpu_destroy(CPUState *cs)
> +{
> +    S390CPU *cpu = S390_CPU(cs);
> +    CPUS390XState *env = &cpu->env;
> +    int rc;
> +
> +    rc = s390_pv_cmd_vcpu(cs, KVM_PV_VCPU_DESTROY, NULL);
> +    if (!rc) {
> +        env->pv = false;
> +    }
> +    return rc;
> +}
> +
> +int s390_pv_set_sec_parms(uint64_t origin, uint64_t length)
> +{
> +    struct kvm_s390_pv_sec_parm args = {
> +        .origin = origin,
> +        .length = length,
> +    };
> +
> +    return s390_pv_cmd(KVM_PV_VM_SET_SEC_PARMS, &args);
> +}
> +
> +/*
> + * Called for each component in the SE type IPL parameter block 0.
> + */
> +int s390_pv_unpack(uint64_t addr, uint64_t size, uint64_t tweak)
> +{
> +    struct kvm_s390_pv_unp args = {
> +        .addr = addr,
> +        .size = size,
> +        .tweak = tweak,
> +    };
> +
> +    return s390_pv_cmd(KVM_PV_VM_UNPACK, &args);
> +}
> +
> +int s390_pv_perf_clear_reset(void)
> +{
> +    return s390_pv_cmd(KVM_PV_VM_PERF_CLEAR_RESET, NULL);
> +}
> +
> +int s390_pv_verify(void)
> +{
> +    return s390_pv_cmd(KVM_PV_VM_VERIFY, NULL);
> +}
> +
> +int s390_pv_unshare(void)
> +{
> +    return s390_pv_cmd(KVM_PV_VM_UNSHARE, NULL);
> +}
> diff --git a/hw/s390x/pv.h b/hw/s390x/pv.h
> new file mode 100644
> index 0000000000..eb074e4bc9
> --- /dev/null
> +++ b/hw/s390x/pv.h
> @@ -0,0 +1,26 @@
> +/*
> + * Protected Virtualization header
> + *
> + * Copyright IBM Corp. 2019
> + * Author(s):
> + *  Janosch Frank <frankja@linux.ibm.com>
> + *
> + * This work is licensed under the terms of the GNU GPL, version 2 or (at
> + * your option) any later version. See the COPYING file in the top-level
> + * directory.
> + */
> +
> +#ifndef HW_S390_PV_H
> +#define HW_S390_PV_H
> +
> +int s390_pv_vm_create(void);
> +int s390_pv_vm_destroy(void);
> +int s390_pv_vcpu_destroy(CPUState *cs);
> +int s390_pv_vcpu_create(CPUState *cs);
> +int s390_pv_set_sec_parms(uint64_t origin, uint64_t length);
> +int s390_pv_unpack(uint64_t addr, uint64_t size, uint64_t tweak);
> +int s390_pv_perf_clear_reset(void);
> +int s390_pv_verify(void);
> +int s390_pv_unshare(void);
> +
> +#endif /* HW_S390_PV_H */
> diff --git a/hw/s390x/s390-virtio-ccw.c b/hw/s390x/s390-virtio-ccw.c
> index c1d1440272..7262453616 100644
> --- a/hw/s390x/s390-virtio-ccw.c
> +++ b/hw/s390x/s390-virtio-ccw.c
> @@ -41,6 +41,7 @@
>   #include "hw/qdev-properties.h"
>   #include "hw/s390x/tod.h"
>   #include "sysemu/sysemu.h"
> +#include "hw/s390x/pv.h"
>   
>   S390CPU *s390_cpu_addr2state(uint16_t cpu_addr)
>   {
> @@ -322,6 +323,7 @@ static void s390_machine_reset(MachineState *machine)
>   {
>       enum s390_reset reset_type;
>       CPUState *cs, *t;
> +    S390CPU *cpu;
>   
>       /* get the reset parameters, reset them once done */
>       s390_ipl_get_reset_request(&cs, &reset_type);
> @@ -329,16 +331,10 @@ static void s390_machine_reset(MachineState *machine)
>       /* all CPUs are paused and synchronized at this point */
>       s390_cmma_reset();
>   
> -    switch (reset_type) {
> -    case S390_RESET_EXTERNAL:
> -    case S390_RESET_REIPL:
> -        qemu_devices_reset();
> -        s390_crypto_reset();
> +    cpu = S390_CPU(cs);
>   
> -        /* configure and start the ipl CPU only */
> -        run_on_cpu(cs, s390_do_cpu_ipl, RUN_ON_CPU_NULL);
> -        break;
> -    case S390_RESET_MODIFIED_CLEAR:
> +    switch (reset_type) {
> +    case S390_RESET_MODIFIED_CLEAR:  /* Subcode 0 */
>           CPU_FOREACH(t) {
>               run_on_cpu(t, s390_do_cpu_full_reset, RUN_ON_CPU_NULL);
>           }
> @@ -346,7 +342,7 @@ static void s390_machine_reset(MachineState *machine)
>           s390_crypto_reset();
>           run_on_cpu(cs, s390_do_cpu_load_normal, RUN_ON_CPU_NULL);
>           break;
> -    case S390_RESET_LOAD_NORMAL:
> +    case S390_RESET_LOAD_NORMAL: /* Subcode 1*/
>           CPU_FOREACH(t) {
>               if (t == cs) {
>                   continue;
> @@ -357,6 +353,35 @@ static void s390_machine_reset(MachineState *machine)
>           run_on_cpu(cs, s390_do_cpu_initial_reset, RUN_ON_CPU_NULL);
>           run_on_cpu(cs, s390_do_cpu_load_normal, RUN_ON_CPU_NULL);
>           break;
> +    case S390_RESET_EXTERNAL:
> +    case S390_RESET_REIPL: /* Subcode 4 */
> +        qemu_devices_reset();
> +        s390_crypto_reset();
> +        /* configure and start the ipl CPU only */
> +        run_on_cpu(cs, s390_do_cpu_ipl, RUN_ON_CPU_NULL);
> +        break;

Is there a way to modify this patch to not change unrelated code that 
heavily? Makes it harder to review.

> +    case S390_RESET_PV: /* Subcode 10 */
> +        subsystem_reset();
> +        s390_crypto_reset();
> +
> +        CPU_FOREACH(t) {
> +            run_on_cpu(t, s390_do_cpu_full_reset, RUN_ON_CPU_NULL);
> +        }
> +
> +        /* Create SE VM */
> +        s390_pv_vm_create();
> +        CPU_FOREACH(t) {
> +            s390_pv_vcpu_create(t);
> +        }
> +
> +        /* Set SE header and unpack */
> +        s390_ipl_prepare_pv_header();
> +        /* Decrypt image */
> +        s390_ipl_pv_unpack();
> +        /* Verify integrity */
> +        s390_pv_verify();
> +        s390_cpu_set_state(S390_CPU_STATE_OPERATING, cpu);
> +        break;
>       default:
>           g_assert_not_reached();
>       }
> diff --git a/target/s390x/cpu_features_def.inc.h b/target/s390x/cpu_features_def.inc.h
> index 31dff0d84e..60db28351d 100644
> --- a/target/s390x/cpu_features_def.inc.h
> +++ b/target/s390x/cpu_features_def.inc.h
> @@ -107,6 +107,7 @@ DEF_FEAT(DEFLATE_BASE, "deflate-base", STFL, 151, "Deflate-conversion facility (
>   DEF_FEAT(VECTOR_PACKED_DECIMAL_ENH, "vxpdeh", STFL, 152, "Vector-Packed-Decimal-Enhancement Facility")
>   DEF_FEAT(MSA_EXT_9, "msa9-base", STFL, 155, "Message-security-assist-extension-9 facility (excluding subfunctions)")
>   DEF_FEAT(ETOKEN, "etoken", STFL, 156, "Etoken facility")
> +DEF_FEAT(UNPACK, "unpack", STFL, 161, "Unpack facility")
>   
>   /* Features exposed via SCLP SCCB Byte 80 - 98  (bit numbers relative to byte-80) */
>   DEF_FEAT(SIE_GSLS, "gsls", SCLP_CONF_CHAR, 40, "SIE: Guest-storage-limit-suppression facility")
> 


-- 

Thanks,

David / dhildenb

Re: [PATCH 02/15] s390x: Beautify diag308 handling

[Janosch Frank]

[-- Attachment #1.1: Type: text/plain, Size: 4312 bytes --]

On 11/21/19 12:21 PM, David Hildenbrand wrote:
> On 20.11.19 12:43, Janosch Frank wrote:
>> Let's improve readability by:
>> * Using constants for the subcodes
>> * Moving parameter checking into a function
>> * Removing subcode > 6 check as the default case catches that
>>
>> Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
>> ---
>>   target/s390x/diag.c | 54 +++++++++++++++++++++++++++------------------
>>   1 file changed, 32 insertions(+), 22 deletions(-)
>>
>> diff --git a/target/s390x/diag.c b/target/s390x/diag.c
>> index 53c2f81f2a..067c667ba7 100644
>> --- a/target/s390x/diag.c
>> +++ b/target/s390x/diag.c
>> @@ -53,6 +53,29 @@ int handle_diag_288(CPUS390XState *env, uint64_t r1, uint64_t r3)
>>   #define DIAG_308_RC_NO_CONF         0x0102
>>   #define DIAG_308_RC_INVALID         0x0402
>>   
>> +#define DIAG308_RES_MOD_CLR		0
>> +#define DIAG308_RES_LOAD_NORM		1
>> +#define DIAG308_LOAD_CLEAR		3
>> +#define DIAG308_LOAD_NORMAL_DUMP	4
>> +#define DIAG308_SET			5
>> +#define DIAG308_STORE			6
>> +
>> +static int diag308_parm_check(CPUS390XState *env, uint64_t r1, uint64_t addr,
>> +                              uintptr_t ra, bool write)
>> +{
>> +    if ((r1 & 1) || (addr & ~TARGET_PAGE_MASK)) {
>> +        s390_program_interrupt(env, PGM_SPECIFICATION, ra);
>> +        return -EINVAL;
>> +    }
>> +    if (!address_space_access_valid(&address_space_memory, addr,
>> +                                    sizeof(IplParameterBlock), write,
>> +                                    MEMTXATTRS_UNSPECIFIED)) {
>> +        s390_program_interrupt(env, PGM_ADDRESSING, ra);
>> +        return -EINVAL;
>> +    }
>> +    return 0;
>> +}
>> +
>>   void handle_diag_308(CPUS390XState *env, uint64_t r1, uint64_t r3, uintptr_t ra)
>>   {
>>       CPUState *cs = env_cpu(env);
>> @@ -65,30 +88,24 @@ void handle_diag_308(CPUS390XState *env, uint64_t r1, uint64_t r3, uintptr_t ra)
>>           return;
>>       }
>>   
>> -    if ((subcode & ~0x0ffffULL) || (subcode > 6)) {
>> +    if (subcode & ~0x0ffffULL) {
> 
> Strange, the default case in the switch was basically dead code.
> 
>>           s390_program_interrupt(env, PGM_SPECIFICATION, ra);
>>           return;
>>       }
>>   
>>       switch (subcode) {
>> -    case 0:
>> +    case DIAG308_RES_MOD_CLR:
>>           s390_ipl_reset_request(cs, S390_RESET_MODIFIED_CLEAR);
>>           break;
>> -    case 1:
>> +    case DIAG308_RES_LOAD_NORM:
>>           s390_ipl_reset_request(cs, S390_RESET_LOAD_NORMAL);
>>           break;
>> -    case 3:
>> +    case DIAG308_LOAD_CLEAR:
>> +        /* Well we still lack the clearing bit... */
>>           s390_ipl_reset_request(cs, S390_RESET_REIPL);
>>           break;
>> -    case 5:
>> -        if ((r1 & 1) || (addr & 0x0fffULL)) {
>> -            s390_program_interrupt(env, PGM_SPECIFICATION, ra);
>> -            return;
>> -        }
>> -        if (!address_space_access_valid(&address_space_memory, addr,
>> -                                        sizeof(IplParameterBlock), false,
>> -                                        MEMTXATTRS_UNSPECIFIED)) {
>> -            s390_program_interrupt(env, PGM_ADDRESSING, ra);
>> +    case DIAG308_SET:
>> +        if (diag308_parm_check(env, r1, addr, ra, false)) {
>>               return;
>>           }
>>           iplb = g_new0(IplParameterBlock, 1);
>> @@ -110,15 +127,8 @@ void handle_diag_308(CPUS390XState *env, uint64_t r1, uint64_t r3, uintptr_t ra)
>>   out:
>>           g_free(iplb);
>>           return;
>> -    case 6:
>> -        if ((r1 & 1) || (addr & 0x0fffULL)) {
>> -            s390_program_interrupt(env, PGM_SPECIFICATION, ra);
>> -            return;
>> -        }
>> -        if (!address_space_access_valid(&address_space_memory, addr,
>> -                                        sizeof(IplParameterBlock), true,
>> -                                        MEMTXATTRS_UNSPECIFIED)) {
>> -            s390_program_interrupt(env, PGM_ADDRESSING, ra);
>> +    case DIAG308_STORE:
>> +        if (diag308_parm_check(env, r1, addr, ra, true)) {
>>               return;
>>           }
>>           iplb = s390_ipl_get_iplb();
>>
> 
> Reviewed-by: David Hildenbrand <david@redhat.com>

Thanks!




[-- Attachment #2: OpenPGP digital signature --]
[-- Type: application/pgp-signature, Size: 833 bytes --]

Re: [PATCH 15/15] s390x: protvirt: Handle SIGP store status correctly

[Janosch Frank]

[-- Attachment #1.1: Type: text/plain, Size: 1750 bytes --]

On 11/21/19 12:24 PM, David Hildenbrand wrote:
> On 20.11.19 12:43, Janosch Frank wrote:
>> Status storing is obviously not done by qemu anymore.
>>
>> Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
>> ---
>>   target/s390x/sigp.c | 7 +++++--
>>   1 file changed, 5 insertions(+), 2 deletions(-)
>>
>> diff --git a/target/s390x/sigp.c b/target/s390x/sigp.c
>> index 2ce22d4dc1..68634d694a 100644
>> --- a/target/s390x/sigp.c
>> +++ b/target/s390x/sigp.c
>> @@ -144,7 +144,9 @@ static void sigp_stop_and_store_status(CPUState *cs, run_on_cpu_data arg)
>>       case S390_CPU_STATE_STOPPED:
>>           /* already stopped, just store the status */
>>           cpu_synchronize_state(cs);
>> -        s390_store_status(cpu, S390_STORE_STATUS_DEF_ADDR, true);
>> +        if (!cpu->env.pv) {
>> +            s390_store_status(cpu, S390_STORE_STATUS_DEF_ADDR, true);
>> +        }
> 
> Confused, how is that case handled? The CPU is already stopped, so it 
> won't be run again (consequently, next SIE entry can't store it). Who 
> will store the status?

Firmware does some magic

> 
>>           break;
>>       }
>>       si->cc = SIGP_CC_ORDER_CODE_ACCEPTED;
>> @@ -497,7 +499,8 @@ void do_stop_interrupt(CPUS390XState *env)
>>       if (s390_cpu_set_state(S390_CPU_STATE_STOPPED, cpu) == 0) {
>>           qemu_system_shutdown_request(SHUTDOWN_CAUSE_GUEST_SHUTDOWN);
>>       }
>> -    if (cpu->env.sigp_order == SIGP_STOP_STORE_STATUS) {
>> +    /* Storing will occur on next SIE entry for fmt 4 */
>> +    if (cpu->env.sigp_order == SIGP_STOP_STORE_STATUS && !env->pv) {
>>           s390_store_status(cpu, S390_STORE_STATUS_DEF_ADDR, true);
>>       }
>>       env->sigp_order = 0;
>>
> 
> 



[-- Attachment #2: OpenPGP digital signature --]
[-- Type: application/pgp-signature, Size: 833 bytes --]

Re: [PATCH 01/15] s390x: Cleanup cpu resets

[Janosch Frank]

[-- Attachment #1.1: Type: text/plain, Size: 2025 bytes --]

On 11/21/19 12:10 PM, Cornelia Huck wrote:
> On Wed, 20 Nov 2019 06:43:20 -0500
> Janosch Frank <frankja@linux.ibm.com> wrote:
> 
>> Let's move the resets into one function and switch by type, so we can
>> use fallthroughs for shared reset actions.
> 
> Doing that makes sense.
> 
>>
>> Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
>> ---
>>  hw/s390x/s390-virtio-ccw.c |   3 +
>>  target/s390x/cpu.c         | 111 ++++++++++++++++---------------------
>>  2 files changed, 52 insertions(+), 62 deletions(-)
>>
>> diff --git a/hw/s390x/s390-virtio-ccw.c b/hw/s390x/s390-virtio-ccw.c
>> index d3edeef0ad..c1d1440272 100644
>> --- a/hw/s390x/s390-virtio-ccw.c
>> +++ b/hw/s390x/s390-virtio-ccw.c
>> @@ -348,6 +348,9 @@ static void s390_machine_reset(MachineState *machine)
>>          break;
>>      case S390_RESET_LOAD_NORMAL:
>>          CPU_FOREACH(t) {
>> +            if (t == cs) {
>> +                continue;
>> +            }
> 
> Hm, why is this needed now?

The Ultravisor checks which reset is done to which cpu.
So blindly resetting the calling cpu with a normal reset to then do a
clear/initial reset will return an error.

> 
>>              run_on_cpu(t, s390_do_cpu_reset, RUN_ON_CPU_NULL);
>>          }
>>          subsystem_reset();
>> diff --git a/target/s390x/cpu.c b/target/s390x/cpu.c
>> index 3abe7e80fd..10d5b915d8 100644
>> --- a/target/s390x/cpu.c
>> +++ b/target/s390x/cpu.c
>> @@ -82,67 +82,53 @@ static void s390_cpu_load_normal(CPUState *s)
>>  }
>>  #endif
>>  
>> -/* S390CPUClass::cpu_reset() */
> 
> Not sure if it would be worth keeping these comments near by the
> calling functions.
> 
>> -static void s390_cpu_reset(CPUState *s)
>> +enum {
>> +    S390_CPU_RESET_NORMAL,
>> +    S390_CPU_RESET_INITIAL,
>> +    S390_CPU_RESET_CLEAR,
>> +};
> 
> Maybe make this into a proper type, so you can use type checking?

Ok

> 
> (...)
> 
> The diff is a bit hard to read, but the change seems fine at a glance.
> 



[-- Attachment #2: OpenPGP digital signature --]
[-- Type: application/pgp-signature, Size: 833 bytes --]

Re: [PATCH 06/15] s390x: protvirt: Support unpack facility

[Janosch Frank]

[-- Attachment #1.1: Type: text/plain, Size: 3687 bytes --]

On 11/20/19 2:43 PM, Cornelia Huck wrote:
> On Wed, 20 Nov 2019 06:43:25 -0500
> Janosch Frank <frankja@linux.ibm.com> wrote:
> 
>> When a guest has saved a ipib of type 5 and call diagnose308 with
>> subcode 10, we have to setup the protected processing environment via
>> Ultravisor calls. The calls are done by KVM and are exposed via an API.
>>
>> The following steps are necessary:
>> 1. Create a VM (register it with the Ultravisor)
>> 2. Create secure CPUs for all of our current cpus
>> 3. Forward the secure header to the Ultravisor (has all information on
>> how to decrypt the image and VM information)
>> 4. Protect image pages from the host and decrypt them
>> 5. Verify the image integrity
>>
>> Only after step 5 a protected VM is allowed to run.
>>
>> Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
>> ---
>>  hw/s390x/Makefile.objs              |   1 +
>>  hw/s390x/ipl.c                      |  33 ++++++++
>>  hw/s390x/ipl.h                      |   2 +
>>  hw/s390x/pv.c                       | 118 ++++++++++++++++++++++++++++
>>  hw/s390x/pv.h                       |  26 ++++++
>>  hw/s390x/s390-virtio-ccw.c          |  45 ++++++++---
>>  target/s390x/cpu_features_def.inc.h |   1 +
>>  7 files changed, 216 insertions(+), 10 deletions(-)
>>  create mode 100644 hw/s390x/pv.c
>>  create mode 100644 hw/s390x/pv.h
>>
>> diff --git a/hw/s390x/Makefile.objs b/hw/s390x/Makefile.objs
>> index 94e57113d8..568bab9711 100644
>> --- a/hw/s390x/Makefile.objs
>> +++ b/hw/s390x/Makefile.objs
>> @@ -31,6 +31,7 @@ obj-y += tod-qemu.o
>>  obj-$(CONFIG_KVM) += tod-kvm.o
>>  obj-$(CONFIG_KVM) += s390-skeys-kvm.o
>>  obj-$(CONFIG_KVM) += s390-stattrib-kvm.o s390-mchk.o
>> +obj-$(CONFIG_KVM) += pv.o
> 
> As this is kvm only...
> 
>>  obj-y += s390-ccw.o
>>  obj-y += ap-device.o
>>  obj-y += ap-bridge.o
>> diff --git a/hw/s390x/ipl.c b/hw/s390x/ipl.c
>> index a077926f36..50501fcd27 100644
>> --- a/hw/s390x/ipl.c
>> +++ b/hw/s390x/ipl.c
>> @@ -33,6 +33,7 @@
>>  #include "qemu/cutils.h"
>>  #include "qemu/option.h"
>>  #include "exec/exec-all.h"
>> +#include "pv.h"
>>  
>>  #define KERN_IMAGE_START                0x010000UL
>>  #define LINUX_MAGIC_ADDR                0x010008UL
>> @@ -668,6 +669,38 @@ static void s390_ipl_prepare_qipl(S390CPU *cpu)
>>      cpu_physical_memory_unmap(addr, len, 1, len);
>>  }
>>  
>> +int s390_ipl_prepare_pv_header(void)
>> +{
>> +    int rc;
>> +    IplParameterBlock *iplb = s390_ipl_get_iplb_secure();
>> +    IPLBlockPV *ipib_pv = &iplb->pv;
>> +    void *hdr = g_malloc(ipib_pv->pv_header_len);
>> +
>> +    cpu_physical_memory_read(ipib_pv->pv_header_addr, hdr,
>> +                             ipib_pv->pv_header_len);
>> +    rc = s390_pv_set_sec_parms((uint64_t)hdr,
>> +                               ipib_pv->pv_header_len);
>> +    g_free(hdr);
>> +    return rc;
>> +}
>> +
>> +int s390_ipl_pv_unpack(void)
>> +{
>> +    int i, rc;
>> +    IplParameterBlock *iplb = s390_ipl_get_iplb_secure();
>> +    IPLBlockPV *ipib_pv = &iplb->pv;
>> +
>> +    for (i = 0; i < ipib_pv->num_comp; i++) {
>> +        rc = s390_pv_unpack(ipib_pv->components[i].addr,
>> +                            TARGET_PAGE_ALIGN(ipib_pv->components[i].size),
>> +                            ipib_pv->components[i].tweak_pref);
> 
> ...you probably need a stub version of the pv functions as well, right?

Yes, I'm working on it

> 
>> +        if (rc) {
>> +            return rc;
>> +        }
>> +    }
>> +    return rc;
>> +}
>> +
>>  void s390_ipl_prepare_cpu(S390CPU *cpu)
>>  {
>>      S390IPLState *ipl = get_ipl_device();
> 



[-- Attachment #2: OpenPGP digital signature --]
[-- Type: application/pgp-signature, Size: 833 bytes --]

Re: [PATCH 11/15] RFC: s390x: Exit on vcpu reset error

[David Hildenbrand]

On 20.11.19 12:43, Janosch Frank wrote:
> If a vcpu is not properly reset it might be better to just end the VM.
> 
> Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
> ---
>   target/s390x/kvm.c | 2 ++
>   1 file changed, 2 insertions(+)
> 
> diff --git a/target/s390x/kvm.c b/target/s390x/kvm.c
> index 190400df55..0210b54157 100644
> --- a/target/s390x/kvm.c
> +++ b/target/s390x/kvm.c
> @@ -418,11 +418,13 @@ static void kvm_s390_reset_vcpu(S390CPU *cpu, unsigned long type)
>           if (kvm_vcpu_ioctl(cs, KVM_S390_VCPU_RESET, type)) {
>               error_report("CPU reset type %ld failed on CPU %i",
>                            type, cs->cpu_index);
> +            exit(1);
>           }
>           return;
>       }
>       if (kvm_vcpu_ioctl(cs, KVM_S390_INITIAL_RESET, NULL)) {
>           error_report("Initial CPU reset failed on CPU %i", cs->cpu_index);
> +        exit(1);
>       }
>   }
>   
> 

According to the comment in include/qapi/error.h

"Please don't error_setg(&error_fatal, ...), use error_report() and 
exit(), because that's more obvious."

This is the right thing to do.

... and it's a fairly pathological thing to happen either way.

Reviewed-by: David Hildenbrand <david@redhat.com>

-- 

Thanks,

David / dhildenb

Re: [PATCH 01/15] s390x: Cleanup cpu resets

[Cornelia Huck]

[-- Attachment #1: Type: text/plain, Size: 1424 bytes --]

On Thu, 21 Nov 2019 12:32:38 +0100
Janosch Frank <frankja@linux.ibm.com> wrote:

> On 11/21/19 12:10 PM, Cornelia Huck wrote:
> > On Wed, 20 Nov 2019 06:43:20 -0500
> > Janosch Frank <frankja@linux.ibm.com> wrote:
> >   
> >> Let's move the resets into one function and switch by type, so we can
> >> use fallthroughs for shared reset actions.  
> > 
> > Doing that makes sense.
> >   
> >>
> >> Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
> >> ---
> >>  hw/s390x/s390-virtio-ccw.c |   3 +
> >>  target/s390x/cpu.c         | 111 ++++++++++++++++---------------------
> >>  2 files changed, 52 insertions(+), 62 deletions(-)
> >>
> >> diff --git a/hw/s390x/s390-virtio-ccw.c b/hw/s390x/s390-virtio-ccw.c
> >> index d3edeef0ad..c1d1440272 100644
> >> --- a/hw/s390x/s390-virtio-ccw.c
> >> +++ b/hw/s390x/s390-virtio-ccw.c
> >> @@ -348,6 +348,9 @@ static void s390_machine_reset(MachineState *machine)
> >>          break;
> >>      case S390_RESET_LOAD_NORMAL:
> >>          CPU_FOREACH(t) {
> >> +            if (t == cs) {
> >> +                continue;
> >> +            }  
> > 
> > Hm, why is this needed now?  
> 
> The Ultravisor checks which reset is done to which cpu.
> So blindly resetting the calling cpu with a normal reset to then do a
> clear/initial reset will return an error.

Let's split this change out, then? The rest of the patch is a simple
refactoring.

[-- Attachment #2: OpenPGP digital signature --]
[-- Type: application/pgp-signature, Size: 833 bytes --]

Re: [PATCH 11/15] RFC: s390x: Exit on vcpu reset error

[Janosch Frank]

[-- Attachment #1.1: Type: text/plain, Size: 1450 bytes --]

On 11/21/19 1:14 PM, David Hildenbrand wrote:
> On 20.11.19 12:43, Janosch Frank wrote:
>> If a vcpu is not properly reset it might be better to just end the VM.
>>
>> Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
>> ---
>>   target/s390x/kvm.c | 2 ++
>>   1 file changed, 2 insertions(+)
>>
>> diff --git a/target/s390x/kvm.c b/target/s390x/kvm.c
>> index 190400df55..0210b54157 100644
>> --- a/target/s390x/kvm.c
>> +++ b/target/s390x/kvm.c
>> @@ -418,11 +418,13 @@ static void kvm_s390_reset_vcpu(S390CPU *cpu, unsigned long type)
>>           if (kvm_vcpu_ioctl(cs, KVM_S390_VCPU_RESET, type)) {
>>               error_report("CPU reset type %ld failed on CPU %i",
>>                            type, cs->cpu_index);
>> +            exit(1);
>>           }
>>           return;
>>       }
>>       if (kvm_vcpu_ioctl(cs, KVM_S390_INITIAL_RESET, NULL)) {
>>           error_report("Initial CPU reset failed on CPU %i", cs->cpu_index);
>> +        exit(1);
>>       }
>>   }
>>   
>>
> 
> According to the comment in include/qapi/error.h
> 
> "Please don't error_setg(&error_fatal, ...), use error_report() and 
> exit(), because that's more obvious."
> 
> This is the right thing to do.
> 
> ... and it's a fairly pathological thing to happen either way.
> 
> Reviewed-by: David Hildenbrand <david@redhat.com>
> 

Do we want to have that separate or should I squash it into the reset
changes?



[-- Attachment #2: OpenPGP digital signature --]
[-- Type: application/pgp-signature, Size: 833 bytes --]

Re: [PATCH 11/15] RFC: s390x: Exit on vcpu reset error

[David Hildenbrand]

On 21.11.19 13:19, Janosch Frank wrote:
> On 11/21/19 1:14 PM, David Hildenbrand wrote:
>> On 20.11.19 12:43, Janosch Frank wrote:
>>> If a vcpu is not properly reset it might be better to just end the VM.
>>>
>>> Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
>>> ---
>>>    target/s390x/kvm.c | 2 ++
>>>    1 file changed, 2 insertions(+)
>>>
>>> diff --git a/target/s390x/kvm.c b/target/s390x/kvm.c
>>> index 190400df55..0210b54157 100644
>>> --- a/target/s390x/kvm.c
>>> +++ b/target/s390x/kvm.c
>>> @@ -418,11 +418,13 @@ static void kvm_s390_reset_vcpu(S390CPU *cpu, unsigned long type)
>>>            if (kvm_vcpu_ioctl(cs, KVM_S390_VCPU_RESET, type)) {
>>>                error_report("CPU reset type %ld failed on CPU %i",
>>>                             type, cs->cpu_index);
>>> +            exit(1);
>>>            }
>>>            return;
>>>        }
>>>        if (kvm_vcpu_ioctl(cs, KVM_S390_INITIAL_RESET, NULL)) {
>>>            error_report("Initial CPU reset failed on CPU %i", cs->cpu_index);
>>> +        exit(1);
>>>        }
>>>    }
>>>    
>>>
>>
>> According to the comment in include/qapi/error.h
>>
>> "Please don't error_setg(&error_fatal, ...), use error_report() and
>> exit(), because that's more obvious."
>>
>> This is the right thing to do.
>>
>> ... and it's a fairly pathological thing to happen either way.
>>
>> Reviewed-by: David Hildenbrand <david@redhat.com>
>>
> 
> Do we want to have that separate or should I squash it into the reset
> changes?
> 
> 

I' keep it separated.

-- 

Thanks,

David / dhildenb

Re: [PATCH 03/15] s390x: protvirt: Add diag308 subcodes 8 - 10

[Cornelia Huck]

On Wed, 20 Nov 2019 06:43:22 -0500
Janosch Frank <frankja@linux.ibm.com> wrote:

> For diag308 subcodes 8 - 10 we have a new ipib of type 5. The ipib
> holds the address and length of the secure execution header, as well
> as a list of guest components.
> 
> Each component is a block of memory, for example kernel or initrd,
> which needs to be decrypted by the Ultravisor in order to run a
> protected VM. The secure execution header instructs the Ultravisor on
> how to handle the protected VM and its components.
> 
> Subcodes 8 and 9 are similiar to 5 and 6 and subcode 10 will finally
> start the protected guest.
> 
> Subcodes 8-10 are not valid in protected mode, we have to do a subcode
> 3 and then the 8 and 10 combination for a protected reboot.
> 
> Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
> ---
>  hw/s390x/ipl.c      | 48 ++++++++++++++++++++++++++++++++++++++++++---
>  hw/s390x/ipl.h      | 33 +++++++++++++++++++++++++++++++
>  target/s390x/diag.c | 26 ++++++++++++++++++++++--
>  3 files changed, 102 insertions(+), 5 deletions(-)
> 
> diff --git a/hw/s390x/ipl.c b/hw/s390x/ipl.c
> index ca544d64c5..a077926f36 100644
> --- a/hw/s390x/ipl.c
> +++ b/hw/s390x/ipl.c
> @@ -529,15 +529,56 @@ static bool is_virtio_scsi_device(IplParameterBlock *iplb)
>      return is_virtio_ccw_device_of_type(iplb, VIRTIO_ID_SCSI);
>  }
>  
> +int s390_ipl_pv_check_comp(IplParameterBlock *iplb)

s390_ipl_pv_check_components() ?

> +{
> +    int i;
> +    IPLBlockPV *ipib_pv = &iplb->pv;
> +
> +    if (ipib_pv->num_comp == 0) {
> +        return -EINVAL;
> +    }
> +
> +    for (i = 0; i < ipib_pv->num_comp; i++) {
> +
> +        /* Addr must be 4k aligned */
> +        if (ipib_pv->components[i].addr & ~TARGET_PAGE_MASK) {
> +            return -EINVAL;
> +        }
> +
> +        /* Tweak prefix is monotonously increasing with each component */
> +        if (i < ipib_pv->num_comp - 1 &&
> +            ipib_pv->components[i].tweak_pref >
> +            ipib_pv->components[i + 1].tweak_pref) {
> +            return -EINVAL;
> +        }
> +    }
> +    return 1;
> +}
> +

(...)

> @@ -185,4 +211,11 @@ static inline bool iplb_valid_fcp(IplParameterBlock *iplb)
>             iplb->pbt == S390_IPL_TYPE_FCP;
>  }
>  
> +static inline bool iplb_valid_se(IplParameterBlock *iplb)

iplb_valid_pv() ?

> +{
> +    return be32_to_cpu(iplb->len) >= S390_IPLB_MIN_PV_LEN &&
> +           iplb->pbt == S390_IPL_TYPE_PV;
> +}
> +
> +
>  #endif

(...)

> @@ -105,6 +110,7 @@ void handle_diag_308(CPUS390XState *env, uint64_t r1, uint64_t r3, uintptr_t ra)
>          s390_ipl_reset_request(cs, S390_RESET_REIPL);
>          break;
>      case DIAG308_SET:
> +    case DIAG308_PV_SET: /* Set SE parms */

PV parms?

('SE' makes me think of 'service element' :)

>          if (diag308_parm_check(env, r1, addr, ra, false)) {
>              return;
>          }
> @@ -117,7 +123,8 @@ void handle_diag_308(CPUS390XState *env, uint64_t r1, uint64_t r3, uintptr_t ra)
>  
>          cpu_physical_memory_read(addr, iplb, be32_to_cpu(iplb->len));
>  
> -        if (!iplb_valid_ccw(iplb) && !iplb_valid_fcp(iplb)) {
> +        if (!iplb_valid_ccw(iplb) && !iplb_valid_fcp(iplb) &&
> +            !(iplb_valid_se(iplb) && s390_ipl_pv_check_comp(iplb) >= 0)) {
>              env->regs[r1 + 1] = DIAG_308_RC_INVALID;
>              goto out;
>          }
> @@ -128,10 +135,15 @@ out:
>          g_free(iplb);
>          return;
>      case DIAG308_STORE:
> +    case DIAG308_PV_STORE: /* Get SE parms */

Same here.

>          if (diag308_parm_check(env, r1, addr, ra, true)) {
>              return;
>          }
> -        iplb = s390_ipl_get_iplb();
> +        if (subcode == DIAG308_PV_STORE) {
> +            iplb = s390_ipl_get_iplb_secure();
> +        } else {
> +            iplb = s390_ipl_get_iplb();
> +        }
>          if (iplb) {
>              cpu_physical_memory_write(addr, iplb, be32_to_cpu(iplb->len));
>              env->regs[r1 + 1] = DIAG_308_RC_OK;
> @@ -139,6 +151,16 @@ out:
>              env->regs[r1 + 1] = DIAG_308_RC_NO_CONF;
>          }
>          return;
> +        break;
> +    case DIAG308_PV_START: /* SE start */

Also here :)

> +        iplb = s390_ipl_get_iplb_secure();
> +        if (!iplb_valid_se(iplb)) {
> +            env->regs[r1 + 1] = DIAG_308_RC_NO_PV_CONF;
> +            return;
> +        }
> +
> +        s390_ipl_reset_request(cs, S390_RESET_PV);
> +        break;
>      default:
>          s390_program_interrupt(env, PGM_SPECIFICATION, ra);
>          break;

Did not spot anything else.

Re: [PATCH 01/15] s390x: Cleanup cpu resets

[Thomas Huth]

On 20/11/2019 12.43, Janosch Frank wrote:
> Let's move the resets into one function and switch by type, so we can
> use fallthroughs for shared reset actions.
> 
> Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
> ---
>  hw/s390x/s390-virtio-ccw.c |   3 +
>  target/s390x/cpu.c         | 111 ++++++++++++++++---------------------
>  2 files changed, 52 insertions(+), 62 deletions(-)
[...]
> diff --git a/target/s390x/cpu.c b/target/s390x/cpu.c
> index 3abe7e80fd..10d5b915d8 100644
> --- a/target/s390x/cpu.c
> +++ b/target/s390x/cpu.c
> @@ -82,67 +82,53 @@ static void s390_cpu_load_normal(CPUState *s)
>  }
>  #endif
>  
> -/* S390CPUClass::cpu_reset() */
> -static void s390_cpu_reset(CPUState *s)
> +enum {
> +    S390_CPU_RESET_NORMAL,
> +    S390_CPU_RESET_INITIAL,
> +    S390_CPU_RESET_CLEAR,
> +};
> +
> +static void s390_cpu_reset(CPUState *s, uint8_t type)

Please give the enum a name and use that instead of uint8_t for "type".
Or at least make it an "int". uint8_t is not really appropriate here.

>  {
>      S390CPU *cpu = S390_CPU(s);
>      S390CPUClass *scc = S390_CPU_GET_CLASS(cpu);
>      CPUS390XState *env = &cpu->env;
>  
> -    env->pfault_token = -1UL;
> -    env->bpbc = false;
>      scc->parent_reset(s);
>      cpu->env.sigp_order = 0;
>      s390_cpu_set_state(S390_CPU_STATE_STOPPED, cpu);
> -}
>  
> -/* S390CPUClass::initial_reset() */
> -static void s390_cpu_initial_reset(CPUState *s)
> -{
> -    S390CPU *cpu = S390_CPU(s);
> -    CPUS390XState *env = &cpu->env;
> +    /* Set initial values after clearing */
> +    switch (type) {
> +    case S390_CPU_RESET_CLEAR:
> +        /* Fallthrough will clear the rest */

I think you could drop the above comment, since /* Fallthrough */ two
lines later should be enough.

> +        memset(env, 0, offsetof(CPUS390XState, start_initial_reset_fields));
> +        /* Fallthrough */
> +    case S390_CPU_RESET_INITIAL:
> +        memset(&env->start_initial_reset_fields, 0,
> +               offsetof(CPUS390XState, end_reset_fields) -
> +               offsetof(CPUS390XState, start_initial_reset_fields));
> +        /* architectured initial values for CR 0 and 14 */
> +        env->cregs[0] = CR0_RESET;
> +        env->cregs[14] = CR14_RESET;
>  
> -    s390_cpu_reset(s);
> -    /* initial reset does not clear everything! */
> -    memset(&env->start_initial_reset_fields, 0,
> -        offsetof(CPUS390XState, end_reset_fields) -
> -        offsetof(CPUS390XState, start_initial_reset_fields));
> -
> -    /* architectured initial values for CR 0 and 14 */
> -    env->cregs[0] = CR0_RESET;
> -    env->cregs[14] = CR14_RESET;
> -
> -    /* architectured initial value for Breaking-Event-Address register */
> -    env->gbea = 1;
> -
> -    env->pfault_token = -1UL;
> -
> -    /* tininess for underflow is detected before rounding */
> -    set_float_detect_tininess(float_tininess_before_rounding,
> -                              &env->fpu_status);
> +        /* architectured initial value for Breaking-Event-Address register */
> +        env->gbea = 1;
> +        /* tininess for underflow is detected before rounding */
> +        set_float_detect_tininess(float_tininess_before_rounding,
> +                                  &env->fpu_status);
> +        /* Fallthrough */
> +    case S390_CPU_RESET_NORMAL:
> +        env->pfault_token = -1UL;
> +        env->bpbc = false;
> +        break;
> +    }
>  
>      /* Reset state inside the kernel that we cannot access yet from QEMU. */
> -    if (kvm_enabled()) {
> -        kvm_s390_reset_vcpu(cpu);
> +    if (kvm_enabled() && (type == S390_CPU_RESET_CLEAR ||
> +                          type == S390_CPU_RESET_INITIAL)) {
> +            kvm_s390_reset_vcpu(cpu);
>      }

Why don't you simply move that into the switch-case statement, too?

[...]

Anyway, re-using code is of course a good idea, but I wonder whether it
would be nicer to keep most things in place, and then simply chain the
functions like this:

static void s390_cpu_reset_normal(CPUState *s)
{
   ...
}

static void s390_cpu_reset_initial(CPUState *s)
{
    ...
    s390_cpu_reset_normal(s);
    ...
}

static void s390_cpu_reset_clear(CPUState *s)
{
    ...
    s390_cpu_reset_initial()
    ...
}

Just my 0.02 €, but at least for me, that's easier to understand than a
switch-case statement with fallthroughs inbetween.

 Thomas

Re: [PATCH 04/15] Header sync protvirt

[Cornelia Huck]

On Wed, 20 Nov 2019 06:43:23 -0500
Janosch Frank <frankja@linux.ibm.com> wrote:

> Let's sync all the protvirt header changes
> 
> Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
> ---
>  linux-headers/asm-s390/kvm.h |  3 ++-
>  linux-headers/linux/kvm.h    | 42 ++++++++++++++++++++++++++++++++++++
>  2 files changed, 44 insertions(+), 1 deletion(-)
> 

> diff --git a/linux-headers/linux/kvm.h b/linux-headers/linux/kvm.h
> index 18892d6541..d031051601 100644
> --- a/linux-headers/linux/kvm.h
> +++ b/linux-headers/linux/kvm.h
> @@ -995,6 +995,8 @@ struct kvm_ppc_resize_hpt {
>  #define KVM_CAP_ARM_SVE 170
>  #define KVM_CAP_ARM_PTRAUTH_ADDRESS 171
>  #define KVM_CAP_ARM_PTRAUTH_GENERIC 172
> +#define KVM_CAP_S390_PROTECTED 180
> +#define KVM_CAP_S390_VCPU_RESETS 181

Hm, where does this cap come from? I did not see it in the kernel
patches.

>  
>  #ifdef KVM_CAP_IRQ_ROUTING
>  

Re: [PATCH 01/15] s390x: Cleanup cpu resets

[Janosch Frank]

[-- Attachment #1.1: Type: text/plain, Size: 4856 bytes --]

On 11/21/19 1:53 PM, Thomas Huth wrote:
> On 20/11/2019 12.43, Janosch Frank wrote:
>> Let's move the resets into one function and switch by type, so we can
>> use fallthroughs for shared reset actions.
>>
>> Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
>> ---
>>  hw/s390x/s390-virtio-ccw.c |   3 +
>>  target/s390x/cpu.c         | 111 ++++++++++++++++---------------------
>>  2 files changed, 52 insertions(+), 62 deletions(-)
> [...]
>> diff --git a/target/s390x/cpu.c b/target/s390x/cpu.c
>> index 3abe7e80fd..10d5b915d8 100644
>> --- a/target/s390x/cpu.c
>> +++ b/target/s390x/cpu.c
>> @@ -82,67 +82,53 @@ static void s390_cpu_load_normal(CPUState *s)
>>  }
>>  #endif
>>  
>> -/* S390CPUClass::cpu_reset() */
>> -static void s390_cpu_reset(CPUState *s)
>> +enum {
>> +    S390_CPU_RESET_NORMAL,
>> +    S390_CPU_RESET_INITIAL,
>> +    S390_CPU_RESET_CLEAR,
>> +};
>> +
>> +static void s390_cpu_reset(CPUState *s, uint8_t type)
> 
> Please give the enum a name and use that instead of uint8_t for "type".
> Or at least make it an "int". uint8_t is not really appropriate here.

Sure

> 
>>  {
>>      S390CPU *cpu = S390_CPU(s);
>>      S390CPUClass *scc = S390_CPU_GET_CLASS(cpu);
>>      CPUS390XState *env = &cpu->env;
>>  
>> -    env->pfault_token = -1UL;
>> -    env->bpbc = false;
>>      scc->parent_reset(s);
>>      cpu->env.sigp_order = 0;
>>      s390_cpu_set_state(S390_CPU_STATE_STOPPED, cpu);
>> -}
>>  
>> -/* S390CPUClass::initial_reset() */
>> -static void s390_cpu_initial_reset(CPUState *s)
>> -{
>> -    S390CPU *cpu = S390_CPU(s);
>> -    CPUS390XState *env = &cpu->env;
>> +    /* Set initial values after clearing */
>> +    switch (type) {
>> +    case S390_CPU_RESET_CLEAR:
>> +        /* Fallthrough will clear the rest */
> 
> I think you could drop the above comment, since /* Fallthrough */ two
> lines later should be enough.
> 
>> +        memset(env, 0, offsetof(CPUS390XState, start_initial_reset_fields));
>> +        /* Fallthrough */
>> +    case S390_CPU_RESET_INITIAL:
>> +        memset(&env->start_initial_reset_fields, 0,
>> +               offsetof(CPUS390XState, end_reset_fields) -
>> +               offsetof(CPUS390XState, start_initial_reset_fields));
>> +        /* architectured initial values for CR 0 and 14 */
>> +        env->cregs[0] = CR0_RESET;
>> +        env->cregs[14] = CR14_RESET;
>>  
>> -    s390_cpu_reset(s);
>> -    /* initial reset does not clear everything! */
>> -    memset(&env->start_initial_reset_fields, 0,
>> -        offsetof(CPUS390XState, end_reset_fields) -
>> -        offsetof(CPUS390XState, start_initial_reset_fields));
>> -
>> -    /* architectured initial values for CR 0 and 14 */
>> -    env->cregs[0] = CR0_RESET;
>> -    env->cregs[14] = CR14_RESET;
>> -
>> -    /* architectured initial value for Breaking-Event-Address register */
>> -    env->gbea = 1;
>> -
>> -    env->pfault_token = -1UL;
>> -
>> -    /* tininess for underflow is detected before rounding */
>> -    set_float_detect_tininess(float_tininess_before_rounding,
>> -                              &env->fpu_status);
>> +        /* architectured initial value for Breaking-Event-Address register */
>> +        env->gbea = 1;
>> +        /* tininess for underflow is detected before rounding */
>> +        set_float_detect_tininess(float_tininess_before_rounding,
>> +                                  &env->fpu_status);
>> +        /* Fallthrough */
>> +    case S390_CPU_RESET_NORMAL:
>> +        env->pfault_token = -1UL;
>> +        env->bpbc = false;
>> +        break;
>> +    }
>>  
>>      /* Reset state inside the kernel that we cannot access yet from QEMU. */
>> -    if (kvm_enabled()) {
>> -        kvm_s390_reset_vcpu(cpu);
>> +    if (kvm_enabled() && (type == S390_CPU_RESET_CLEAR ||
>> +                          type == S390_CPU_RESET_INITIAL)) {
>> +            kvm_s390_reset_vcpu(cpu);
>>      }
> 
> Why don't you simply move that into the switch-case statement, too?

There was a reason for that, time to load it from cold storage...

> 
> [...]
> 
> Anyway, re-using code is of course a good idea, but I wonder whether it
> would be nicer to keep most things in place, and then simply chain the
> functions like this:

I tried that and I prefer the version in the patch.

> 
> static void s390_cpu_reset_normal(CPUState *s)
> {
>    ...
> }
> 
> static void s390_cpu_reset_initial(CPUState *s)
> {
>     ...
>     s390_cpu_reset_normal(s);
>     ...
> }
> 
> static void s390_cpu_reset_clear(CPUState *s)
> {
>     ...
>     s390_cpu_reset_initial()
>     ...
> }
> 
> Just my 0.02 €, but at least for me, that's easier to understand than a
> switch-case statement with fallthroughs inbetween.
> 
>  Thomas
> 



[-- Attachment #2: OpenPGP digital signature --]
[-- Type: application/pgp-signature, Size: 833 bytes --]

Re: [PATCH 02/15] s390x: Beautify diag308 handling

[Thomas Huth]

On 21/11/2019 12.21, David Hildenbrand wrote:
> On 20.11.19 12:43, Janosch Frank wrote:
>> Let's improve readability by:
>> * Using constants for the subcodes
>> * Moving parameter checking into a function
>> * Removing subcode > 6 check as the default case catches that
>>
>> Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
>> ---
>>   target/s390x/diag.c | 54 +++++++++++++++++++++++++++------------------
>>   1 file changed, 32 insertions(+), 22 deletions(-)
>>
>> diff --git a/target/s390x/diag.c b/target/s390x/diag.c
>> index 53c2f81f2a..067c667ba7 100644
>> --- a/target/s390x/diag.c
>> +++ b/target/s390x/diag.c
>> @@ -53,6 +53,29 @@ int handle_diag_288(CPUS390XState *env, uint64_t
>> r1, uint64_t r3)
>>   #define DIAG_308_RC_NO_CONF         0x0102
>>   #define DIAG_308_RC_INVALID         0x0402
>>   +#define DIAG308_RES_MOD_CLR        0
>> +#define DIAG308_RES_LOAD_NORM        1
>> +#define DIAG308_LOAD_CLEAR        3
>> +#define DIAG308_LOAD_NORMAL_DUMP    4
>> +#define DIAG308_SET            5
>> +#define DIAG308_STORE            6
>> +
>> +static int diag308_parm_check(CPUS390XState *env, uint64_t r1,
>> uint64_t addr,
>> +                              uintptr_t ra, bool write)
>> +{
>> +    if ((r1 & 1) || (addr & ~TARGET_PAGE_MASK)) {
>> +        s390_program_interrupt(env, PGM_SPECIFICATION, ra);
>> +        return -EINVAL;
>> +    }
>> +    if (!address_space_access_valid(&address_space_memory, addr,
>> +                                    sizeof(IplParameterBlock), write,
>> +                                    MEMTXATTRS_UNSPECIFIED)) {
>> +        s390_program_interrupt(env, PGM_ADDRESSING, ra);
>> +        return -EINVAL;
>> +    }
>> +    return 0;
>> +}
>> +
>>   void handle_diag_308(CPUS390XState *env, uint64_t r1, uint64_t r3,
>> uintptr_t ra)
>>   {
>>       CPUState *cs = env_cpu(env);
>> @@ -65,30 +88,24 @@ void handle_diag_308(CPUS390XState *env, uint64_t
>> r1, uint64_t r3, uintptr_t ra)
>>           return;
>>       }
>>   -    if ((subcode & ~0x0ffffULL) || (subcode > 6)) {
>> +    if (subcode & ~0x0ffffULL) {
> 
> Strange, the default case in the switch was basically dead code.

Not really, case 2 and case 4 are not handled there. We even had a funny
bug some months ago, where a guest could terminate QEMU with one of
these codes, see commit 37dbd1f4d4805edcd18d94eb202bb3461b3cd52d.

 Thomas

Re: [PATCH 04/15] Header sync protvirt

[Janosch Frank]

[-- Attachment #1.1: Type: text/plain, Size: 1137 bytes --]

On 11/21/19 1:59 PM, Cornelia Huck wrote:
> On Wed, 20 Nov 2019 06:43:23 -0500
> Janosch Frank <frankja@linux.ibm.com> wrote:
> 
>> Let's sync all the protvirt header changes
>>
>> Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
>> ---
>>  linux-headers/asm-s390/kvm.h |  3 ++-
>>  linux-headers/linux/kvm.h    | 42 ++++++++++++++++++++++++++++++++++++
>>  2 files changed, 44 insertions(+), 1 deletion(-)
>>
> 
>> diff --git a/linux-headers/linux/kvm.h b/linux-headers/linux/kvm.h
>> index 18892d6541..d031051601 100644
>> --- a/linux-headers/linux/kvm.h
>> +++ b/linux-headers/linux/kvm.h
>> @@ -995,6 +995,8 @@ struct kvm_ppc_resize_hpt {
>>  #define KVM_CAP_ARM_SVE 170
>>  #define KVM_CAP_ARM_PTRAUTH_ADDRESS 171
>>  #define KVM_CAP_ARM_PTRAUTH_GENERIC 172
>> +#define KVM_CAP_S390_PROTECTED 180
>> +#define KVM_CAP_S390_VCPU_RESETS 181
> 
> Hm, where does this cap come from? I did not see it in the kernel
> patches.

It'll be part of the V1 once I send it out.
Pierre found the problem while reviewing QEMU, so I fixed it in both repos.

> 
>>  
>>  #ifdef KVM_CAP_IRQ_ROUTING
>>  
> 



[-- Attachment #2: OpenPGP digital signature --]
[-- Type: application/pgp-signature, Size: 833 bytes --]

Re: [PATCH 04/15] Header sync protvirt

[Cornelia Huck]

[-- Attachment #1: Type: text/plain, Size: 1303 bytes --]

On Thu, 21 Nov 2019 14:12:21 +0100
Janosch Frank <frankja@linux.ibm.com> wrote:

> On 11/21/19 1:59 PM, Cornelia Huck wrote:
> > On Wed, 20 Nov 2019 06:43:23 -0500
> > Janosch Frank <frankja@linux.ibm.com> wrote:
> >   
> >> Let's sync all the protvirt header changes
> >>
> >> Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
> >> ---
> >>  linux-headers/asm-s390/kvm.h |  3 ++-
> >>  linux-headers/linux/kvm.h    | 42 ++++++++++++++++++++++++++++++++++++
> >>  2 files changed, 44 insertions(+), 1 deletion(-)
> >>  
> >   
> >> diff --git a/linux-headers/linux/kvm.h b/linux-headers/linux/kvm.h
> >> index 18892d6541..d031051601 100644
> >> --- a/linux-headers/linux/kvm.h
> >> +++ b/linux-headers/linux/kvm.h
> >> @@ -995,6 +995,8 @@ struct kvm_ppc_resize_hpt {
> >>  #define KVM_CAP_ARM_SVE 170
> >>  #define KVM_CAP_ARM_PTRAUTH_ADDRESS 171
> >>  #define KVM_CAP_ARM_PTRAUTH_GENERIC 172
> >> +#define KVM_CAP_S390_PROTECTED 180
> >> +#define KVM_CAP_S390_VCPU_RESETS 181  
> > 
> > Hm, where does this cap come from? I did not see it in the kernel
> > patches.  
> 
> It'll be part of the V1 once I send it out.
> Pierre found the problem while reviewing QEMU, so I fixed it in both repos.

Ok, that cap probably makes sense. It was just surprising to find :)

[-- Attachment #2: OpenPGP digital signature --]
[-- Type: application/pgp-signature, Size: 833 bytes --]

Re: [PATCH 01/15] s390x: Cleanup cpu resets

[Thomas Huth]

On 21/11/2019 14.11, Janosch Frank wrote:
> On 11/21/19 1:53 PM, Thomas Huth wrote:
>> On 20/11/2019 12.43, Janosch Frank wrote:
>>> Let's move the resets into one function and switch by type, so we can
>>> use fallthroughs for shared reset actions.
[...]
>>> +        memset(env, 0, offsetof(CPUS390XState, start_initial_reset_fields));
>>> +        /* Fallthrough */
>>> +    case S390_CPU_RESET_INITIAL:
>>> +        memset(&env->start_initial_reset_fields, 0,
>>> +               offsetof(CPUS390XState, end_reset_fields) -
>>> +               offsetof(CPUS390XState, start_initial_reset_fields));
>>> +        /* architectured initial values for CR 0 and 14 */
>>> +        env->cregs[0] = CR0_RESET;
>>> +        env->cregs[14] = CR14_RESET;
>>>  
>>> -    s390_cpu_reset(s);
>>> -    /* initial reset does not clear everything! */
>>> -    memset(&env->start_initial_reset_fields, 0,
>>> -        offsetof(CPUS390XState, end_reset_fields) -
>>> -        offsetof(CPUS390XState, start_initial_reset_fields));
>>> -
>>> -    /* architectured initial values for CR 0 and 14 */
>>> -    env->cregs[0] = CR0_RESET;
>>> -    env->cregs[14] = CR14_RESET;
>>> -
>>> -    /* architectured initial value for Breaking-Event-Address register */
>>> -    env->gbea = 1;
>>> -
>>> -    env->pfault_token = -1UL;
>>> -
>>> -    /* tininess for underflow is detected before rounding */
>>> -    set_float_detect_tininess(float_tininess_before_rounding,
>>> -                              &env->fpu_status);
>>> +        /* architectured initial value for Breaking-Event-Address register */
>>> +        env->gbea = 1;
>>> +        /* tininess for underflow is detected before rounding */
>>> +        set_float_detect_tininess(float_tininess_before_rounding,
>>> +                                  &env->fpu_status);
>>> +        /* Fallthrough */
>>> +    case S390_CPU_RESET_NORMAL:
>>> +        env->pfault_token = -1UL;
>>> +        env->bpbc = false;
>>> +        break;
>>> +    }
>>>  
>>>      /* Reset state inside the kernel that we cannot access yet from QEMU. */
>>> -    if (kvm_enabled()) {
>>> -        kvm_s390_reset_vcpu(cpu);
>>> +    if (kvm_enabled() && (type == S390_CPU_RESET_CLEAR ||
>>> +                          type == S390_CPU_RESET_INITIAL)) {
>>> +            kvm_s390_reset_vcpu(cpu);
>>>      }
>>
>> Why don't you simply move that into the switch-case statement, too?
> 
> There was a reason for that, time to load it from cold storage...

I just noticed that you rework this in patch 10/15, so it indeed makes
sense to keep it outside of the switch-case-statement above...

>> [...]
>>
>> Anyway, re-using code is of course a good idea, but I wonder whether it
>> would be nicer to keep most things in place, and then simply chain the
>> functions like this:
> 
> I tried that and I prefer the version in the patch.

... and with patch 10 in mind, it indeed also makes more sense to *not*
use the chaining that I've suggested. So never mind, your switch with
the fallthrough statements is just fine.

 Thomas

Re: [PATCH 02/15] s390x: Beautify diag308 handling

[Thomas Huth]

On 20/11/2019 12.43, Janosch Frank wrote:
> Let's improve readability by:
> * Using constants for the subcodes
> * Moving parameter checking into a function
> * Removing subcode > 6 check as the default case catches that
> 
> Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
> ---
>  target/s390x/diag.c | 54 +++++++++++++++++++++++++++------------------
>  1 file changed, 32 insertions(+), 22 deletions(-)
> 
> diff --git a/target/s390x/diag.c b/target/s390x/diag.c
> index 53c2f81f2a..067c667ba7 100644
> --- a/target/s390x/diag.c
> +++ b/target/s390x/diag.c
> @@ -53,6 +53,29 @@ int handle_diag_288(CPUS390XState *env, uint64_t r1, uint64_t r3)
>  #define DIAG_308_RC_NO_CONF         0x0102
>  #define DIAG_308_RC_INVALID         0x0402
>  
> +#define DIAG308_RES_MOD_CLR		0
> +#define DIAG308_RES_LOAD_NORM		1

I think I'd also prefer RESET instead of RES here.

> +#define DIAG308_LOAD_CLEAR		3
> +#define DIAG308_LOAD_NORMAL_DUMP	4
> +#define DIAG308_SET			5
> +#define DIAG308_STORE			6
> +
> +static int diag308_parm_check(CPUS390XState *env, uint64_t r1, uint64_t addr,
> +                              uintptr_t ra, bool write)
> +{
> +    if ((r1 & 1) || (addr & ~TARGET_PAGE_MASK)) {
> +        s390_program_interrupt(env, PGM_SPECIFICATION, ra);
> +        return -EINVAL;
> +    }
> +    if (!address_space_access_valid(&address_space_memory, addr,
> +                                    sizeof(IplParameterBlock), write,
> +                                    MEMTXATTRS_UNSPECIFIED)) {
> +        s390_program_interrupt(env, PGM_ADDRESSING, ra);
> +        return -EINVAL;

or maybe -EFAULT ? ;-)

> +    }
> +    return 0;
> +}
> +
>  void handle_diag_308(CPUS390XState *env, uint64_t r1, uint64_t r3, uintptr_t ra)
>  {
>      CPUState *cs = env_cpu(env);
> @@ -65,30 +88,24 @@ void handle_diag_308(CPUS390XState *env, uint64_t r1, uint64_t r3, uintptr_t ra)
>          return;
>      }
>  
> -    if ((subcode & ~0x0ffffULL) || (subcode > 6)) {
> +    if (subcode & ~0x0ffffULL) {
>          s390_program_interrupt(env, PGM_SPECIFICATION, ra);
>          return;
>      }
>  
>      switch (subcode) {
> -    case 0:
> +    case DIAG308_RES_MOD_CLR:
>          s390_ipl_reset_request(cs, S390_RESET_MODIFIED_CLEAR);
>          break;
> -    case 1:
> +    case DIAG308_RES_LOAD_NORM:
>          s390_ipl_reset_request(cs, S390_RESET_LOAD_NORMAL);
>          break;
> -    case 3:
> +    case DIAG308_LOAD_CLEAR:
> +        /* Well we still lack the clearing bit... */
>          s390_ipl_reset_request(cs, S390_RESET_REIPL);
>          break;
> -    case 5:
> -        if ((r1 & 1) || (addr & 0x0fffULL)) {
> -            s390_program_interrupt(env, PGM_SPECIFICATION, ra);
> -            return;
> -        }
> -        if (!address_space_access_valid(&address_space_memory, addr,
> -                                        sizeof(IplParameterBlock), false,
> -                                        MEMTXATTRS_UNSPECIFIED)) {
> -            s390_program_interrupt(env, PGM_ADDRESSING, ra);
> +    case DIAG308_SET:
> +        if (diag308_parm_check(env, r1, addr, ra, false)) {
>              return;
>          }
>          iplb = g_new0(IplParameterBlock, 1);
> @@ -110,15 +127,8 @@ void handle_diag_308(CPUS390XState *env, uint64_t r1, uint64_t r3, uintptr_t ra)
>  out:
>          g_free(iplb);
>          return;
> -    case 6:
> -        if ((r1 & 1) || (addr & 0x0fffULL)) {
> -            s390_program_interrupt(env, PGM_SPECIFICATION, ra);
> -            return;
> -        }
> -        if (!address_space_access_valid(&address_space_memory, addr,
> -                                        sizeof(IplParameterBlock), true,
> -                                        MEMTXATTRS_UNSPECIFIED)) {
> -            s390_program_interrupt(env, PGM_ADDRESSING, ra);
> +    case DIAG308_STORE:
> +        if (diag308_parm_check(env, r1, addr, ra, true)) {
>              return;
>          }
>          iplb = s390_ipl_get_iplb();
> 

With RESET instead of RES:

Reviewed-by: Thomas Huth <thuth@redhat.com>

Re: [PATCH 05/15] s390x: protvirt: Sync PV state

[Cornelia Huck]

On Wed, 20 Nov 2019 06:43:24 -0500
Janosch Frank <frankja@linux.ibm.com> wrote:

> We do not always have the SIE intercept code handy at each place where
> we do emulation. Unfortunately emulation for secure guests often
> differ slightly from normal emulation and we need to make decisions
> based on the protected state of the VCPU.
> 
> Let's sync the protected state and make it available.
> 
> Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
> ---
>  linux-headers/asm-s390/kvm.h | 1 +
>  target/s390x/cpu.h           | 1 +
>  target/s390x/kvm.c           | 4 ++++
>  3 files changed, 6 insertions(+)
> 
> diff --git a/linux-headers/asm-s390/kvm.h b/linux-headers/asm-s390/kvm.h
> index 41976d33f0..7c46cf6078 100644
> --- a/linux-headers/asm-s390/kvm.h
> +++ b/linux-headers/asm-s390/kvm.h
> @@ -231,6 +231,7 @@ struct kvm_guest_debug_arch {
>  #define KVM_SYNC_GSCB   (1UL << 9)
>  #define KVM_SYNC_BPBC   (1UL << 10)
>  #define KVM_SYNC_ETOKEN (1UL << 11)
> +#define KVM_SYNC_PV	(1UL << 12)

That should go into the previous patch (will be picked up by header
sync).

>  /* length and alignment of the sdnx as a power of two */
>  #define SDNXC 8
>  #define SDNXL (1UL << SDNXC)
> diff --git a/target/s390x/cpu.h b/target/s390x/cpu.h
> index 17460ed7b3..a787221772 100644
> --- a/target/s390x/cpu.h
> +++ b/target/s390x/cpu.h
> @@ -116,6 +116,7 @@ struct CPUS390XState {
>  
>      /* Fields up to this point are cleared by a CPU reset */
>      struct {} end_reset_fields;
> +    bool pv; /* protected virtualization */
>  
>  #if !defined(CONFIG_USER_ONLY)
>      uint32_t core_id; /* PoP "CPU address", same as cpu_index */
> diff --git a/target/s390x/kvm.c b/target/s390x/kvm.c
> index c24c869e77..418154ccfe 100644
> --- a/target/s390x/kvm.c
> +++ b/target/s390x/kvm.c
> @@ -676,6 +676,10 @@ int kvm_arch_get_registers(CPUState *cs)
>          env->etoken_extension = cs->kvm_run->s.regs.etoken_extension;
>      }
>  
> +    if (can_sync_regs(cs, KVM_SYNC_PV)) {
> +        env->pv = !!cs->kvm_run->s.regs.pv;
> +    }
> +
>      /* pfault parameters */
>      if (can_sync_regs(cs, KVM_SYNC_PFAULT)) {
>          env->pfault_token = cs->kvm_run->s.regs.pft;

As you add a new field to the cpu state... you probably can't migrate
protected guests, so you don't need a new vmstate subsection?

Re: [PATCH 05/15] s390x: protvirt: Sync PV state

[Janosch Frank]

[-- Attachment #1.1: Type: text/plain, Size: 2586 bytes --]

On 11/21/19 2:25 PM, Cornelia Huck wrote:
> On Wed, 20 Nov 2019 06:43:24 -0500
> Janosch Frank <frankja@linux.ibm.com> wrote:
> 
>> We do not always have the SIE intercept code handy at each place where
>> we do emulation. Unfortunately emulation for secure guests often
>> differ slightly from normal emulation and we need to make decisions
>> based on the protected state of the VCPU.
>>
>> Let's sync the protected state and make it available.
>>
>> Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
>> ---
>>  linux-headers/asm-s390/kvm.h | 1 +
>>  target/s390x/cpu.h           | 1 +
>>  target/s390x/kvm.c           | 4 ++++
>>  3 files changed, 6 insertions(+)
>>
>> diff --git a/linux-headers/asm-s390/kvm.h b/linux-headers/asm-s390/kvm.h
>> index 41976d33f0..7c46cf6078 100644
>> --- a/linux-headers/asm-s390/kvm.h
>> +++ b/linux-headers/asm-s390/kvm.h
>> @@ -231,6 +231,7 @@ struct kvm_guest_debug_arch {
>>  #define KVM_SYNC_GSCB   (1UL << 9)
>>  #define KVM_SYNC_BPBC   (1UL << 10)
>>  #define KVM_SYNC_ETOKEN (1UL << 11)
>> +#define KVM_SYNC_PV	(1UL << 12)
> 
> That should go into the previous patch (will be picked up by header
> sync).

Yes, will be fixed in a second

> 
>>  /* length and alignment of the sdnx as a power of two */
>>  #define SDNXC 8
>>  #define SDNXL (1UL << SDNXC)
>> diff --git a/target/s390x/cpu.h b/target/s390x/cpu.h
>> index 17460ed7b3..a787221772 100644
>> --- a/target/s390x/cpu.h
>> +++ b/target/s390x/cpu.h
>> @@ -116,6 +116,7 @@ struct CPUS390XState {
>>  
>>      /* Fields up to this point are cleared by a CPU reset */
>>      struct {} end_reset_fields;
>> +    bool pv; /* protected virtualization */
>>  
>>  #if !defined(CONFIG_USER_ONLY)
>>      uint32_t core_id; /* PoP "CPU address", same as cpu_index */
>> diff --git a/target/s390x/kvm.c b/target/s390x/kvm.c
>> index c24c869e77..418154ccfe 100644
>> --- a/target/s390x/kvm.c
>> +++ b/target/s390x/kvm.c
>> @@ -676,6 +676,10 @@ int kvm_arch_get_registers(CPUState *cs)
>>          env->etoken_extension = cs->kvm_run->s.regs.etoken_extension;
>>      }
>>  
>> +    if (can_sync_regs(cs, KVM_SYNC_PV)) {
>> +        env->pv = !!cs->kvm_run->s.regs.pv;
>> +    }
>> +
>>      /* pfault parameters */
>>      if (can_sync_regs(cs, KVM_SYNC_PFAULT)) {
>>          env->pfault_token = cs->kvm_run->s.regs.pft;
> 
> As you add a new field to the cpu state... you probably can't migrate
> protected guests, so you don't need a new vmstate subsection?
> 

We can't migrate currently, but it's on out todo list


[-- Attachment #2: OpenPGP digital signature --]
[-- Type: application/pgp-signature, Size: 833 bytes --]

Re: [PATCH 07/15] s390x: protvirt: Handle diag 308 subcodes 0,1,3,4

[Cornelia Huck]

On Wed, 20 Nov 2019 06:43:26 -0500
Janosch Frank <frankja@linux.ibm.com> wrote:

> Now that we know the protection state off the cpus, we can start
> handling all diag 308 subcodes in the protected state.

"As we now have access to the protection state of the cpus, we can
implement special handling of diag 308 subcodes for cpus in the
protected state."

?

> 
> For subcodes 0 and 1 we need to unshare all pages before continuing,
> so the guest doesn't accidently expose data when dumping.
> 
> For subcode 3/4 we tear down the protected VM and reboot into
> unprotected mode. We do not provide a secure reboot.
> 
> Before we can do the unshare calls, we need to mark all cpus as
> stopped.
> 
> Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
> ---
>  hw/s390x/s390-virtio-ccw.c | 30 +++++++++++++++++++++++++++---
>  target/s390x/diag.c        |  4 ++++
>  2 files changed, 31 insertions(+), 3 deletions(-)
> 
> diff --git a/hw/s390x/s390-virtio-ccw.c b/hw/s390x/s390-virtio-ccw.c
> index 7262453616..6fd50b4c42 100644
> --- a/hw/s390x/s390-virtio-ccw.c
> +++ b/hw/s390x/s390-virtio-ccw.c
> @@ -319,11 +319,26 @@ static inline void s390_do_cpu_ipl(CPUState *cs, run_on_cpu_data arg)
>      s390_cpu_set_state(S390_CPU_STATE_OPERATING, cpu);
>  }
>  
> +static void s390_pv_prepare_reset(CPUS390XState *env)
> +{
> +    CPUState *cs;
> +
> +    if (!env->pv) {
> +        return;
> +    }
> +    CPU_FOREACH(cs) {
> +        s390_cpu_set_state(S390_CPU_STATE_STOPPED, S390_CPU(cs));
> +    }
> +    s390_pv_unshare();
> +    s390_pv_perf_clear_reset();
> +}
> +
>  static void s390_machine_reset(MachineState *machine)
>  {
>      enum s390_reset reset_type;
>      CPUState *cs, *t;
>      S390CPU *cpu;
> +    CPUS390XState *env;
>  
>      /* get the reset parameters, reset them once done */
>      s390_ipl_get_reset_request(&cs, &reset_type);
> @@ -332,29 +347,38 @@ static void s390_machine_reset(MachineState *machine)
>      s390_cmma_reset();
>  
>      cpu = S390_CPU(cs);
> +    env = &cpu->env;
>  
>      switch (reset_type) {
>      case S390_RESET_MODIFIED_CLEAR:  /* Subcode 0 */
> +        subsystem_reset();
> +        s390_crypto_reset();
> +        s390_pv_prepare_reset(env);
>          CPU_FOREACH(t) {
>              run_on_cpu(t, s390_do_cpu_full_reset, RUN_ON_CPU_NULL);
>          }
> -        subsystem_reset();
> -        s390_crypto_reset();

This also switches the order in which different parts are reset for
normal guests. I presume that order doesn't matter here?

>          run_on_cpu(cs, s390_do_cpu_load_normal, RUN_ON_CPU_NULL);
>          break;
>      case S390_RESET_LOAD_NORMAL: /* Subcode 1*/

missing blank before */ (introduced in a previous patch)


> +        subsystem_reset();
> +        s390_pv_prepare_reset(env);
>          CPU_FOREACH(t) {
>              if (t == cs) {
>                  continue;
>              }
>              run_on_cpu(t, s390_do_cpu_reset, RUN_ON_CPU_NULL);
>          }
> -        subsystem_reset();
>          run_on_cpu(cs, s390_do_cpu_initial_reset, RUN_ON_CPU_NULL);
>          run_on_cpu(cs, s390_do_cpu_load_normal, RUN_ON_CPU_NULL);
>          break;
>      case S390_RESET_EXTERNAL:

Annotate this with the subcode as well? (in the patch introducing it)

>      case S390_RESET_REIPL: /* Subcode 4 */
> +        if (env->pv) {
> +            CPU_FOREACH(t) {
> +                s390_pv_vcpu_destroy(t);
> +            }
> +            s390_pv_vm_destroy();
> +        }
>          qemu_devices_reset();
>          s390_crypto_reset();
>          /* configure and start the ipl CPU only */
> diff --git a/target/s390x/diag.c b/target/s390x/diag.c
> index 32049bb4ee..db6d79cef3 100644
> --- a/target/s390x/diag.c
> +++ b/target/s390x/diag.c
> @@ -68,6 +68,10 @@ int handle_diag_288(CPUS390XState *env, uint64_t r1, uint64_t r3)
>  static int diag308_parm_check(CPUS390XState *env, uint64_t r1, uint64_t addr,
>                                uintptr_t ra, bool write)
>  {
> +    /* Handled by the Ultravisor */
> +    if (env->pv) {
> +        return 0;
> +    }
>      if ((r1 & 1) || (addr & ~TARGET_PAGE_MASK)) {
>          s390_program_interrupt(env, PGM_SPECIFICATION, ra);
>          return -EINVAL;

Re: [PATCH 02/15] s390x: Beautify diag308 handling

[Janosch Frank]

[-- Attachment #1.1: Type: text/plain, Size: 4478 bytes --]

On 11/21/19 2:20 PM, Thomas Huth wrote:
> On 20/11/2019 12.43, Janosch Frank wrote:
>> Let's improve readability by:
>> * Using constants for the subcodes
>> * Moving parameter checking into a function
>> * Removing subcode > 6 check as the default case catches that
>>
>> Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
>> ---
>>  target/s390x/diag.c | 54 +++++++++++++++++++++++++++------------------
>>  1 file changed, 32 insertions(+), 22 deletions(-)
>>
>> diff --git a/target/s390x/diag.c b/target/s390x/diag.c
>> index 53c2f81f2a..067c667ba7 100644
>> --- a/target/s390x/diag.c
>> +++ b/target/s390x/diag.c
>> @@ -53,6 +53,29 @@ int handle_diag_288(CPUS390XState *env, uint64_t r1, uint64_t r3)
>>  #define DIAG_308_RC_NO_CONF         0x0102
>>  #define DIAG_308_RC_INVALID         0x0402
>>  
>> +#define DIAG308_RES_MOD_CLR		0
>> +#define DIAG308_RES_LOAD_NORM		1
> 
> I think I'd also prefer RESET instead of RES here.
> 
>> +#define DIAG308_LOAD_CLEAR		3
>> +#define DIAG308_LOAD_NORMAL_DUMP	4
>> +#define DIAG308_SET			5
>> +#define DIAG308_STORE			6
>> +
>> +static int diag308_parm_check(CPUS390XState *env, uint64_t r1, uint64_t addr,
>> +                              uintptr_t ra, bool write)
>> +{
>> +    if ((r1 & 1) || (addr & ~TARGET_PAGE_MASK)) {
>> +        s390_program_interrupt(env, PGM_SPECIFICATION, ra);
>> +        return -EINVAL;
>> +    }
>> +    if (!address_space_access_valid(&address_space_memory, addr,
>> +                                    sizeof(IplParameterBlock), write,
>> +                                    MEMTXATTRS_UNSPECIFIED)) {
>> +        s390_program_interrupt(env, PGM_ADDRESSING, ra);
>> +        return -EINVAL;
> 
> or maybe -EFAULT ? ;-)

Honestly, I'm asking myself why I'm even bothering to return specific
error codes when all I check is rc != 0. So I guess I just make it -1
for all errors.

> 
>> +    }
>> +    return 0;
>> +}
>> +
>>  void handle_diag_308(CPUS390XState *env, uint64_t r1, uint64_t r3, uintptr_t ra)
>>  {
>>      CPUState *cs = env_cpu(env);
>> @@ -65,30 +88,24 @@ void handle_diag_308(CPUS390XState *env, uint64_t r1, uint64_t r3, uintptr_t ra)
>>          return;
>>      }
>>  
>> -    if ((subcode & ~0x0ffffULL) || (subcode > 6)) {
>> +    if (subcode & ~0x0ffffULL) {
>>          s390_program_interrupt(env, PGM_SPECIFICATION, ra);
>>          return;
>>      }
>>  
>>      switch (subcode) {
>> -    case 0:
>> +    case DIAG308_RES_MOD_CLR:
>>          s390_ipl_reset_request(cs, S390_RESET_MODIFIED_CLEAR);
>>          break;
>> -    case 1:
>> +    case DIAG308_RES_LOAD_NORM:
>>          s390_ipl_reset_request(cs, S390_RESET_LOAD_NORMAL);
>>          break;
>> -    case 3:
>> +    case DIAG308_LOAD_CLEAR:
>> +        /* Well we still lack the clearing bit... */
>>          s390_ipl_reset_request(cs, S390_RESET_REIPL);
>>          break;
>> -    case 5:
>> -        if ((r1 & 1) || (addr & 0x0fffULL)) {
>> -            s390_program_interrupt(env, PGM_SPECIFICATION, ra);
>> -            return;
>> -        }
>> -        if (!address_space_access_valid(&address_space_memory, addr,
>> -                                        sizeof(IplParameterBlock), false,
>> -                                        MEMTXATTRS_UNSPECIFIED)) {
>> -            s390_program_interrupt(env, PGM_ADDRESSING, ra);
>> +    case DIAG308_SET:
>> +        if (diag308_parm_check(env, r1, addr, ra, false)) {
>>              return;
>>          }
>>          iplb = g_new0(IplParameterBlock, 1);
>> @@ -110,15 +127,8 @@ void handle_diag_308(CPUS390XState *env, uint64_t r1, uint64_t r3, uintptr_t ra)
>>  out:
>>          g_free(iplb);
>>          return;
>> -    case 6:
>> -        if ((r1 & 1) || (addr & 0x0fffULL)) {
>> -            s390_program_interrupt(env, PGM_SPECIFICATION, ra);
>> -            return;
>> -        }
>> -        if (!address_space_access_valid(&address_space_memory, addr,
>> -                                        sizeof(IplParameterBlock), true,
>> -                                        MEMTXATTRS_UNSPECIFIED)) {
>> -            s390_program_interrupt(env, PGM_ADDRESSING, ra);
>> +    case DIAG308_STORE:
>> +        if (diag308_parm_check(env, r1, addr, ra, true)) {
>>              return;
>>          }
>>          iplb = s390_ipl_get_iplb();
>>
> 
> With RESET instead of RES:
> 
> Reviewed-by: Thomas Huth <thuth@redhat.com>
> 



[-- Attachment #2: OpenPGP digital signature --]
[-- Type: application/pgp-signature, Size: 833 bytes --]

Re: [PATCH 07/15] s390x: protvirt: Handle diag 308 subcodes 0,1,3,4

[Janosch Frank]

[-- Attachment #1.1: Type: text/plain, Size: 4663 bytes --]

On 11/21/19 2:50 PM, Cornelia Huck wrote:
> On Wed, 20 Nov 2019 06:43:26 -0500
> Janosch Frank <frankja@linux.ibm.com> wrote:
> 
>> Now that we know the protection state off the cpus, we can start
>> handling all diag 308 subcodes in the protected state.
> 
> "As we now have access to the protection state of the cpus, we can
> implement special handling of diag 308 subcodes for cpus in the
> protected state."
> 
> ?

Sure

> 
>>
>> For subcodes 0 and 1 we need to unshare all pages before continuing,
>> so the guest doesn't accidently expose data when dumping.
>>
>> For subcode 3/4 we tear down the protected VM and reboot into
>> unprotected mode. We do not provide a secure reboot.
>>
>> Before we can do the unshare calls, we need to mark all cpus as
>> stopped.
>>
>> Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
>> ---
>>  hw/s390x/s390-virtio-ccw.c | 30 +++++++++++++++++++++++++++---
>>  target/s390x/diag.c        |  4 ++++
>>  2 files changed, 31 insertions(+), 3 deletions(-)
>>
>> diff --git a/hw/s390x/s390-virtio-ccw.c b/hw/s390x/s390-virtio-ccw.c
>> index 7262453616..6fd50b4c42 100644
>> --- a/hw/s390x/s390-virtio-ccw.c
>> +++ b/hw/s390x/s390-virtio-ccw.c
>> @@ -319,11 +319,26 @@ static inline void s390_do_cpu_ipl(CPUState *cs, run_on_cpu_data arg)
>>      s390_cpu_set_state(S390_CPU_STATE_OPERATING, cpu);
>>  }
>>  
>> +static void s390_pv_prepare_reset(CPUS390XState *env)
>> +{
>> +    CPUState *cs;
>> +
>> +    if (!env->pv) {
>> +        return;
>> +    }
>> +    CPU_FOREACH(cs) {
>> +        s390_cpu_set_state(S390_CPU_STATE_STOPPED, S390_CPU(cs));
>> +    }
>> +    s390_pv_unshare();
>> +    s390_pv_perf_clear_reset();
>> +}
>> +
>>  static void s390_machine_reset(MachineState *machine)
>>  {
>>      enum s390_reset reset_type;
>>      CPUState *cs, *t;
>>      S390CPU *cpu;
>> +    CPUS390XState *env;
>>  
>>      /* get the reset parameters, reset them once done */
>>      s390_ipl_get_reset_request(&cs, &reset_type);
>> @@ -332,29 +347,38 @@ static void s390_machine_reset(MachineState *machine)
>>      s390_cmma_reset();
>>  
>>      cpu = S390_CPU(cs);
>> +    env = &cpu->env;
>>  
>>      switch (reset_type) {
>>      case S390_RESET_MODIFIED_CLEAR:  /* Subcode 0 */
>> +        subsystem_reset();
>> +        s390_crypto_reset();
>> +        s390_pv_prepare_reset(env);
>>          CPU_FOREACH(t) {
>>              run_on_cpu(t, s390_do_cpu_full_reset, RUN_ON_CPU_NULL);
>>          }
>> -        subsystem_reset();
>> -        s390_crypto_reset();
> 
> This also switches the order in which different parts are reset for
> normal guests. I presume that order doesn't matter here?

I don't think that the order is specified in the POP, it is however
specified for protected VMs...

> 
>>          run_on_cpu(cs, s390_do_cpu_load_normal, RUN_ON_CPU_NULL);
>>          break;
>>      case S390_RESET_LOAD_NORMAL: /* Subcode 1*/
> 
> missing blank before */ (introduced in a previous patch)

Will fix

> 
> 
>> +        subsystem_reset();
>> +        s390_pv_prepare_reset(env);
>>          CPU_FOREACH(t) {
>>              if (t == cs) {
>>                  continue;
>>              }
>>              run_on_cpu(t, s390_do_cpu_reset, RUN_ON_CPU_NULL);
>>          }
>> -        subsystem_reset();
>>          run_on_cpu(cs, s390_do_cpu_initial_reset, RUN_ON_CPU_NULL);
>>          run_on_cpu(cs, s390_do_cpu_load_normal, RUN_ON_CPU_NULL);
>>          break;
>>      case S390_RESET_EXTERNAL:
> 
> Annotate this with the subcode as well? (in the patch introducing it)

Sure

> 
>>      case S390_RESET_REIPL: /* Subcode 4 */
>> +        if (env->pv) {
>> +            CPU_FOREACH(t) {
>> +                s390_pv_vcpu_destroy(t);
>> +            }
>> +            s390_pv_vm_destroy();
>> +        }
>>          qemu_devices_reset();
>>          s390_crypto_reset();
>>          /* configure and start the ipl CPU only */
>> diff --git a/target/s390x/diag.c b/target/s390x/diag.c
>> index 32049bb4ee..db6d79cef3 100644
>> --- a/target/s390x/diag.c
>> +++ b/target/s390x/diag.c
>> @@ -68,6 +68,10 @@ int handle_diag_288(CPUS390XState *env, uint64_t r1, uint64_t r3)
>>  static int diag308_parm_check(CPUS390XState *env, uint64_t r1, uint64_t addr,
>>                                uintptr_t ra, bool write)
>>  {
>> +    /* Handled by the Ultravisor */
>> +    if (env->pv) {
>> +        return 0;
>> +    }
>>      if ((r1 & 1) || (addr & ~TARGET_PAGE_MASK)) {
>>          s390_program_interrupt(env, PGM_SPECIFICATION, ra);
>>          return -EINVAL;
> 



[-- Attachment #2: OpenPGP digital signature --]
[-- Type: application/pgp-signature, Size: 833 bytes --]

Re: [PATCH 07/15] s390x: protvirt: Handle diag 308 subcodes 0,1,3,4

[Janosch Frank]

[-- Attachment #1.1: Type: text/plain, Size: 1000 bytes --]

On 11/21/19 2:50 PM, Cornelia Huck wrote:
> On Wed, 20 Nov 2019 06:43:26 -0500
> Janosch Frank <frankja@linux.ibm.com> wrote:

> 
>>          run_on_cpu(cs, s390_do_cpu_load_normal, RUN_ON_CPU_NULL);
>>          break;
>>      case S390_RESET_LOAD_NORMAL: /* Subcode 1*/
> 
> missing blank before */ (introduced in a previous patch)
> 
> 
>> +        subsystem_reset();
>> +        s390_pv_prepare_reset(env);
>>          CPU_FOREACH(t) {
>>              if (t == cs) {
>>                  continue;
>>              }
>>              run_on_cpu(t, s390_do_cpu_reset, RUN_ON_CPU_NULL);
>>          }
>> -        subsystem_reset();
>>          run_on_cpu(cs, s390_do_cpu_initial_reset, RUN_ON_CPU_NULL);
>>          run_on_cpu(cs, s390_do_cpu_load_normal, RUN_ON_CPU_NULL);
>>          break;
>>      case S390_RESET_EXTERNAL:
> 
> Annotate this with the subcode as well? (in the patch introducing it)

I think this has no diag 308 subcode and is triggered by qemu





[-- Attachment #2: OpenPGP digital signature --]
[-- Type: application/pgp-signature, Size: 833 bytes --]

Re: [PATCH 08/15] s390x: protvirt: KVM intercept changes

[Cornelia Huck]

On Wed, 20 Nov 2019 06:43:27 -0500
Janosch Frank <frankja@linux.ibm.com> wrote:

> Secure guests no longer intercept with code 4 for an instruction
> interception. Instead they have codes 104 and 108 for secure
> instruction interception and secure instruction notification
> respectively.
> 
> The 104 mirrors the 4, but the 108 is a notification, that something
> happened and the hypervisor might need to adjust its tracking data to
> that fact. An example for that is the set prefix notification
> interception, where KVM only reads the new prefix, but does not update
> the prefix in the state description.
> 
> Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
> ---
>  target/s390x/kvm.c | 6 ++++++
>  1 file changed, 6 insertions(+)
> 
> diff --git a/target/s390x/kvm.c b/target/s390x/kvm.c
> index 418154ccfe..58251c0229 100644
> --- a/target/s390x/kvm.c
> +++ b/target/s390x/kvm.c
> @@ -115,6 +115,8 @@
>  #define ICPT_CPU_STOP                   0x28
>  #define ICPT_OPEREXC                    0x2c
>  #define ICPT_IO                         0x40
> +#define ICPT_PV_INSTR                   0x68
> +#define ICPT_PV_INSTR_NOT               0x6c

_NOTIF ?

>  
>  #define NR_LOCAL_IRQS 32
>  /*
> @@ -151,6 +153,7 @@ static int cap_s390_irq;
>  static int cap_ri;
>  static int cap_gs;
>  static int cap_hpage_1m;
> +static int cap_protvirt;
>  
>  static int active_cmma;
>  
> @@ -336,6 +339,7 @@ int kvm_arch_init(MachineState *ms, KVMState *s)
>      cap_async_pf = kvm_check_extension(s, KVM_CAP_ASYNC_PF);
>      cap_mem_op = kvm_check_extension(s, KVM_CAP_S390_MEM_OP);
>      cap_s390_irq = kvm_check_extension(s, KVM_CAP_S390_INJECT_IRQ);
> +    cap_protvirt = kvm_check_extension(s, KVM_CAP_S390_PROTECTED);

You don't seem to do anything with this yet?

>  
>      if (!kvm_check_extension(s, KVM_CAP_S390_GMAP)
>          || !kvm_check_extension(s, KVM_CAP_S390_COW)) {
> @@ -1664,6 +1668,8 @@ static int handle_intercept(S390CPU *cpu)
>              (long)cs->kvm_run->psw_addr);
>      switch (icpt_code) {
>          case ICPT_INSTRUCTION:
> +        case ICPT_PV_INSTR:
> +        case ICPT_PV_INSTR_NOT:
>              r = handle_instruction(cpu, run);

Doesn't handle_instruction() want to know whether it got a request for
emulation vs a notification?

>              break;
>          case ICPT_PROGRAM:

Re: [PATCH 09/15] s390x: protvirt: SCLP interpretation

[Cornelia Huck]

On Wed, 20 Nov 2019 06:43:28 -0500
Janosch Frank <frankja@linux.ibm.com> wrote:

> SCLP for a protected guest is done over the SIDAD, so we need to use
> the s390_cpu_virt_mem_* functions to access the SIDAD instead of guest
> memory when reading/writing SCBs.
> 
> To not confuse the sclp emulation, we set 0x42000 as the address, but
> ignore it for reading/writing the SCCB.
> 
> Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
> ---
>  hw/s390x/sclp.c         | 16 ++++++++++++++++
>  include/hw/s390x/sclp.h |  2 ++
>  target/s390x/kvm.c      |  8 +++++++-
>  3 files changed, 25 insertions(+), 1 deletion(-)
> 
> diff --git a/hw/s390x/sclp.c b/hw/s390x/sclp.c
> index f57ce7b739..00d08adc7f 100644
> --- a/hw/s390x/sclp.c
> +++ b/hw/s390x/sclp.c
> @@ -193,6 +193,22 @@ static void sclp_execute(SCLPDevice *sclp, SCCB *sccb, uint32_t code)
>      }
>  }
>  
> +int sclp_service_call_protected(CPUS390XState *env, uint64_t sccb,
> +                                uint32_t code)
> +{
> +    SCLPDevice *sclp = get_sclp_device();
> +    SCLPDeviceClass *sclp_c = SCLP_GET_CLASS(sclp);
> +    SCCB work_sccb;
> +    hwaddr sccb_len = sizeof(SCCB);
> +
> +    s390_cpu_virt_mem_read(env_archcpu(env), 0, 0, &work_sccb, sccb_len);
> +    sclp_c->execute(sclp, &work_sccb, code);
> +    s390_cpu_virt_mem_write(env_archcpu(env), 0, 0, &work_sccb,
> +                            be16_to_cpu(work_sccb.h.length));
> +    sclp_c->service_interrupt(sclp, sccb);
> +    return 0;
> +}
> +
>  int sclp_service_call(CPUS390XState *env, uint64_t sccb, uint32_t code)
>  {
>      SCLPDevice *sclp = get_sclp_device();
> diff --git a/include/hw/s390x/sclp.h b/include/hw/s390x/sclp.h
> index c54413b78c..c0a3faa37d 100644
> --- a/include/hw/s390x/sclp.h
> +++ b/include/hw/s390x/sclp.h
> @@ -217,5 +217,7 @@ void s390_sclp_init(void);
>  void sclp_service_interrupt(uint32_t sccb);
>  void raise_irq_cpu_hotplug(void);
>  int sclp_service_call(CPUS390XState *env, uint64_t sccb, uint32_t code);
> +int sclp_service_call_protected(CPUS390XState *env, uint64_t sccb,
> +                                uint32_t code);
>  
>  #endif
> diff --git a/target/s390x/kvm.c b/target/s390x/kvm.c
> index 58251c0229..0f2458b553 100644
> --- a/target/s390x/kvm.c
> +++ b/target/s390x/kvm.c
> @@ -1172,7 +1172,13 @@ static int kvm_sclp_service_call(S390CPU *cpu, struct kvm_run *run,
>      sccb = env->regs[ipbh0 & 0xf];
>      code = env->regs[(ipbh0 & 0xf0) >> 4];
>  
> -    r = sclp_service_call(env, sccb, code);
> +    if (run->s390_sieic.icptcode == ICPT_PV_INSTR ||
> +        run->s390_sieic.icptcode == ICPT_PV_INSTR_NOT) {
> +        r = sclp_service_call_protected(env, 0x42000, code);

I fear that confuses the reader instead of the emulation :)

Especially as you end up passing that value to
sclp_c->service_interrupt()...

> +    } else {
> +        r = sclp_service_call(env, sccb, code);
> +    }
> +
>      if (r < 0) {
>          kvm_s390_program_interrupt(cpu, -r);
>      } else {

Re: [PATCH 07/15] s390x: protvirt: Handle diag 308 subcodes 0,1,3,4

[Cornelia Huck]

[-- Attachment #1: Type: text/plain, Size: 1249 bytes --]

On Thu, 21 Nov 2019 15:04:31 +0100
Janosch Frank <frankja@linux.ibm.com> wrote:

> On 11/21/19 2:50 PM, Cornelia Huck wrote:
> > On Wed, 20 Nov 2019 06:43:26 -0500
> > Janosch Frank <frankja@linux.ibm.com> wrote:  
> 
> >   
> >>          run_on_cpu(cs, s390_do_cpu_load_normal, RUN_ON_CPU_NULL);
> >>          break;
> >>      case S390_RESET_LOAD_NORMAL: /* Subcode 1*/  
> > 
> > missing blank before */ (introduced in a previous patch)
> > 
> >   
> >> +        subsystem_reset();
> >> +        s390_pv_prepare_reset(env);
> >>          CPU_FOREACH(t) {
> >>              if (t == cs) {
> >>                  continue;
> >>              }
> >>              run_on_cpu(t, s390_do_cpu_reset, RUN_ON_CPU_NULL);
> >>          }
> >> -        subsystem_reset();
> >>          run_on_cpu(cs, s390_do_cpu_initial_reset, RUN_ON_CPU_NULL);
> >>          run_on_cpu(cs, s390_do_cpu_load_normal, RUN_ON_CPU_NULL);
> >>          break;
> >>      case S390_RESET_EXTERNAL:  
> > 
> > Annotate this with the subcode as well? (in the patch introducing it)  
> 
> I think this has no diag 308 subcode and is triggered by qemu

-ENOCOFFE

But even more reason to annotate this with the fact that this is
triggered by QEMU :)

[-- Attachment #2: OpenPGP digital signature --]
[-- Type: application/pgp-signature, Size: 833 bytes --]

Re: [PATCH 07/15] s390x: protvirt: Handle diag 308 subcodes 0,1,3,4

[Janosch Frank]

[-- Attachment #1.1: Type: text/plain, Size: 1505 bytes --]

On 11/21/19 3:17 PM, Cornelia Huck wrote:
> On Thu, 21 Nov 2019 15:04:31 +0100
> Janosch Frank <frankja@linux.ibm.com> wrote:
> 
>> On 11/21/19 2:50 PM, Cornelia Huck wrote:
>>> On Wed, 20 Nov 2019 06:43:26 -0500
>>> Janosch Frank <frankja@linux.ibm.com> wrote:  
>>
>>>   
>>>>          run_on_cpu(cs, s390_do_cpu_load_normal, RUN_ON_CPU_NULL);
>>>>          break;
>>>>      case S390_RESET_LOAD_NORMAL: /* Subcode 1*/  
>>>
>>> missing blank before */ (introduced in a previous patch)
>>>
>>>   
>>>> +        subsystem_reset();
>>>> +        s390_pv_prepare_reset(env);
>>>>          CPU_FOREACH(t) {
>>>>              if (t == cs) {
>>>>                  continue;
>>>>              }
>>>>              run_on_cpu(t, s390_do_cpu_reset, RUN_ON_CPU_NULL);
>>>>          }
>>>> -        subsystem_reset();
>>>>          run_on_cpu(cs, s390_do_cpu_initial_reset, RUN_ON_CPU_NULL);
>>>>          run_on_cpu(cs, s390_do_cpu_load_normal, RUN_ON_CPU_NULL);
>>>>          break;
>>>>      case S390_RESET_EXTERNAL:  
>>>
>>> Annotate this with the subcode as well? (in the patch introducing it)  
>>
>> I think this has no diag 308 subcode and is triggered by qemu
> 
> -ENOCOFFE
> 
> But even more reason to annotate this with the fact that this is
> triggered by QEMU :)
> 

You're too late with that idea :)
I just split out the reordering and the annotation into a new commit:

https://github.com/frankjaa/qemu/commit/8c53d5c8a6bbcc53496c7a2877c7cbffc435b708


[-- Attachment #2: OpenPGP digital signature --]
[-- Type: application/pgp-signature, Size: 833 bytes --]

Re: [PATCH 09/15] s390x: protvirt: SCLP interpretation

[Janosch Frank]

[-- Attachment #1.1: Type: text/plain, Size: 3309 bytes --]

On 11/21/19 3:11 PM, Cornelia Huck wrote:
> On Wed, 20 Nov 2019 06:43:28 -0500
> Janosch Frank <frankja@linux.ibm.com> wrote:
> 
>> SCLP for a protected guest is done over the SIDAD, so we need to use
>> the s390_cpu_virt_mem_* functions to access the SIDAD instead of guest
>> memory when reading/writing SCBs.
>>
>> To not confuse the sclp emulation, we set 0x42000 as the address, but
>> ignore it for reading/writing the SCCB.
>>
>> Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
>> ---
>>  hw/s390x/sclp.c         | 16 ++++++++++++++++
>>  include/hw/s390x/sclp.h |  2 ++
>>  target/s390x/kvm.c      |  8 +++++++-
>>  3 files changed, 25 insertions(+), 1 deletion(-)
>>
>> diff --git a/hw/s390x/sclp.c b/hw/s390x/sclp.c
>> index f57ce7b739..00d08adc7f 100644
>> --- a/hw/s390x/sclp.c
>> +++ b/hw/s390x/sclp.c
>> @@ -193,6 +193,22 @@ static void sclp_execute(SCLPDevice *sclp, SCCB *sccb, uint32_t code)
>>      }
>>  }
>>  
>> +int sclp_service_call_protected(CPUS390XState *env, uint64_t sccb,
>> +                                uint32_t code)
>> +{
>> +    SCLPDevice *sclp = get_sclp_device();
>> +    SCLPDeviceClass *sclp_c = SCLP_GET_CLASS(sclp);
>> +    SCCB work_sccb;
>> +    hwaddr sccb_len = sizeof(SCCB);
>> +
>> +    s390_cpu_virt_mem_read(env_archcpu(env), 0, 0, &work_sccb, sccb_len);
>> +    sclp_c->execute(sclp, &work_sccb, code);
>> +    s390_cpu_virt_mem_write(env_archcpu(env), 0, 0, &work_sccb,
>> +                            be16_to_cpu(work_sccb.h.length));
>> +    sclp_c->service_interrupt(sclp, sccb);
>> +    return 0;
>> +}
>> +
>>  int sclp_service_call(CPUS390XState *env, uint64_t sccb, uint32_t code)
>>  {
>>      SCLPDevice *sclp = get_sclp_device();
>> diff --git a/include/hw/s390x/sclp.h b/include/hw/s390x/sclp.h
>> index c54413b78c..c0a3faa37d 100644
>> --- a/include/hw/s390x/sclp.h
>> +++ b/include/hw/s390x/sclp.h
>> @@ -217,5 +217,7 @@ void s390_sclp_init(void);
>>  void sclp_service_interrupt(uint32_t sccb);
>>  void raise_irq_cpu_hotplug(void);
>>  int sclp_service_call(CPUS390XState *env, uint64_t sccb, uint32_t code);
>> +int sclp_service_call_protected(CPUS390XState *env, uint64_t sccb,
>> +                                uint32_t code);
>>  
>>  #endif
>> diff --git a/target/s390x/kvm.c b/target/s390x/kvm.c
>> index 58251c0229..0f2458b553 100644
>> --- a/target/s390x/kvm.c
>> +++ b/target/s390x/kvm.c
>> @@ -1172,7 +1172,13 @@ static int kvm_sclp_service_call(S390CPU *cpu, struct kvm_run *run,
>>      sccb = env->regs[ipbh0 & 0xf];
>>      code = env->regs[(ipbh0 & 0xf0) >> 4];
>>  
>> -    r = sclp_service_call(env, sccb, code);
>> +    if (run->s390_sieic.icptcode == ICPT_PV_INSTR ||
>> +        run->s390_sieic.icptcode == ICPT_PV_INSTR_NOT) {
>> +        r = sclp_service_call_protected(env, 0x42000, code);
> 
> I fear that confuses the reader instead of the emulation :)
> 
> Especially as you end up passing that value to
> sclp_c->service_interrupt()...

Pierre has some more opinions on that, so I'll let him present his
planned changes to this patch :)

> 
>> +    } else {
>> +        r = sclp_service_call(env, sccb, code);
>> +    }
>> +
>>      if (r < 0) {
>>          kvm_s390_program_interrupt(cpu, -r);
>>      } else {
> 



[-- Attachment #2: OpenPGP digital signature --]
[-- Type: application/pgp-signature, Size: 833 bytes --]

Re: [PATCH 06/15] s390x: protvirt: Support unpack facility

[Janosch Frank]

[-- Attachment #1.1: Type: text/plain, Size: 2545 bytes --]

On 11/21/19 12:27 PM, David Hildenbrand wrote:
> On 20.11.19 12:43, Janosch Frank wrote:

>> @@ -357,6 +353,35 @@ static void s390_machine_reset(MachineState *machine)
>>           run_on_cpu(cs, s390_do_cpu_initial_reset, RUN_ON_CPU_NULL);
>>           run_on_cpu(cs, s390_do_cpu_load_normal, RUN_ON_CPU_NULL);
>>           break;
>> +    case S390_RESET_EXTERNAL:
>> +    case S390_RESET_REIPL: /* Subcode 4 */
>> +        qemu_devices_reset();
>> +        s390_crypto_reset();
>> +        /* configure and start the ipl CPU only */
>> +        run_on_cpu(cs, s390_do_cpu_ipl, RUN_ON_CPU_NULL);
>> +        break;
> 
> Is there a way to modify this patch to not change unrelated code that 
> heavily? Makes it harder to review.

https://github.com/frankjaa/qemu/commit/8c53d5c8a6bbcc53496c7a2877c7cbffc435b708

And please trim your emails.

> 
>> +    case S390_RESET_PV: /* Subcode 10 */
>> +        subsystem_reset();
>> +        s390_crypto_reset();
>> +
>> +        CPU_FOREACH(t) {
>> +            run_on_cpu(t, s390_do_cpu_full_reset, RUN_ON_CPU_NULL);
>> +        }
>> +
>> +        /* Create SE VM */
>> +        s390_pv_vm_create();
>> +        CPU_FOREACH(t) {
>> +            s390_pv_vcpu_create(t);
>> +        }
>> +
>> +        /* Set SE header and unpack */
>> +        s390_ipl_prepare_pv_header();
>> +        /* Decrypt image */
>> +        s390_ipl_pv_unpack();
>> +        /* Verify integrity */
>> +        s390_pv_verify();
>> +        s390_cpu_set_state(S390_CPU_STATE_OPERATING, cpu);
>> +        break;
>>       default:
>>           g_assert_not_reached();
>>       }
>> diff --git a/target/s390x/cpu_features_def.inc.h b/target/s390x/cpu_features_def.inc.h
>> index 31dff0d84e..60db28351d 100644
>> --- a/target/s390x/cpu_features_def.inc.h
>> +++ b/target/s390x/cpu_features_def.inc.h
>> @@ -107,6 +107,7 @@ DEF_FEAT(DEFLATE_BASE, "deflate-base", STFL, 151, "Deflate-conversion facility (
>>   DEF_FEAT(VECTOR_PACKED_DECIMAL_ENH, "vxpdeh", STFL, 152, "Vector-Packed-Decimal-Enhancement Facility")
>>   DEF_FEAT(MSA_EXT_9, "msa9-base", STFL, 155, "Message-security-assist-extension-9 facility (excluding subfunctions)")
>>   DEF_FEAT(ETOKEN, "etoken", STFL, 156, "Etoken facility")
>> +DEF_FEAT(UNPACK, "unpack", STFL, 161, "Unpack facility")
>>   
>>   /* Features exposed via SCLP SCCB Byte 80 - 98  (bit numbers relative to byte-80) */
>>   DEF_FEAT(SIE_GSLS, "gsls", SCLP_CONF_CHAR, 40, "SIE: Guest-storage-limit-suppression facility")
>>
> 
> 



[-- Attachment #2: OpenPGP digital signature --]
[-- Type: application/pgp-signature, Size: 833 bytes --]

Re: [PATCH 06/15] s390x: protvirt: Support unpack facility

[David Hildenbrand]

On 21.11.19 15:25, Janosch Frank wrote:
> On 11/21/19 12:27 PM, David Hildenbrand wrote:
>> On 20.11.19 12:43, Janosch Frank wrote:
> 
>>> @@ -357,6 +353,35 @@ static void s390_machine_reset(MachineState *machine)
>>>            run_on_cpu(cs, s390_do_cpu_initial_reset, RUN_ON_CPU_NULL);
>>>            run_on_cpu(cs, s390_do_cpu_load_normal, RUN_ON_CPU_NULL);
>>>            break;
>>> +    case S390_RESET_EXTERNAL:
>>> +    case S390_RESET_REIPL: /* Subcode 4 */
>>> +        qemu_devices_reset();
>>> +        s390_crypto_reset();
>>> +        /* configure and start the ipl CPU only */
>>> +        run_on_cpu(cs, s390_do_cpu_ipl, RUN_ON_CPU_NULL);
>>> +        break;
>>
>> Is there a way to modify this patch to not change unrelated code that
>> heavily? Makes it harder to review.
> 
> https://github.com/frankjaa/qemu/commit/8c53d5c8a6bbcc53496c7a2877c7cbffc435b708
> 
> And please trim your emails.
> 

If you use Thunderbird I suggest QuoteCollapse ... because nobody got 
time for that ;)


-- 

Thanks,

David / dhildenb

Re: [PATCH 08/15] s390x: protvirt: KVM intercept changes

[Janosch Frank]

[-- Attachment #1.1: Type: text/plain, Size: 2742 bytes --]

On 11/21/19 3:07 PM, Cornelia Huck wrote:
> On Wed, 20 Nov 2019 06:43:27 -0500
> Janosch Frank <frankja@linux.ibm.com> wrote:
> 
>> Secure guests no longer intercept with code 4 for an instruction
>> interception. Instead they have codes 104 and 108 for secure
>> instruction interception and secure instruction notification
>> respectively.
>>
>> The 104 mirrors the 4, but the 108 is a notification, that something
>> happened and the hypervisor might need to adjust its tracking data to
>> that fact. An example for that is the set prefix notification
>> interception, where KVM only reads the new prefix, but does not update
>> the prefix in the state description.
>>
>> Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
>> ---
>>  target/s390x/kvm.c | 6 ++++++
>>  1 file changed, 6 insertions(+)
>>
>> diff --git a/target/s390x/kvm.c b/target/s390x/kvm.c
>> index 418154ccfe..58251c0229 100644
>> --- a/target/s390x/kvm.c
>> +++ b/target/s390x/kvm.c
>> @@ -115,6 +115,8 @@
>>  #define ICPT_CPU_STOP                   0x28
>>  #define ICPT_OPEREXC                    0x2c
>>  #define ICPT_IO                         0x40
>> +#define ICPT_PV_INSTR                   0x68
>> +#define ICPT_PV_INSTR_NOT               0x6c
> 
> _NOTIF ?

Yeah, forgot about that

> 
>>  
>>  #define NR_LOCAL_IRQS 32
>>  /*
>> @@ -151,6 +153,7 @@ static int cap_s390_irq;
>>  static int cap_ri;
>>  static int cap_gs;
>>  static int cap_hpage_1m;
>> +static int cap_protvirt;
>>  
>>  static int active_cmma;
>>  
>> @@ -336,6 +339,7 @@ int kvm_arch_init(MachineState *ms, KVMState *s)
>>      cap_async_pf = kvm_check_extension(s, KVM_CAP_ASYNC_PF);
>>      cap_mem_op = kvm_check_extension(s, KVM_CAP_S390_MEM_OP);
>>      cap_s390_irq = kvm_check_extension(s, KVM_CAP_S390_INJECT_IRQ);
>> +    cap_protvirt = kvm_check_extension(s, KVM_CAP_S390_PROTECTED);
> 
> You don't seem to do anything with this yet?

No, I'm still a bit in the dark about how we want to tie protvirt into qemu.

> 
>>  
>>      if (!kvm_check_extension(s, KVM_CAP_S390_GMAP)
>>          || !kvm_check_extension(s, KVM_CAP_S390_COW)) {
>> @@ -1664,6 +1668,8 @@ static int handle_intercept(S390CPU *cpu)
>>              (long)cs->kvm_run->psw_addr);
>>      switch (icpt_code) {
>>          case ICPT_INSTRUCTION:
>> +        case ICPT_PV_INSTR:
>> +        case ICPT_PV_INSTR_NOT:
>>              r = handle_instruction(cpu, run);
> 
> Doesn't handle_instruction() want to know whether it got a request for
> emulation vs a notification?

Currently not, the sclp patch looks at the vcpu run icptcode to figure
out what's going on.

> 
>>              break;
>>          case ICPT_PROGRAM:
> 
> 



[-- Attachment #2: OpenPGP digital signature --]
[-- Type: application/pgp-signature, Size: 833 bytes --]

Re: [PATCH 06/15] s390x: protvirt: Support unpack facility

[Christian Borntraeger]

On 21.11.19 15:28, David Hildenbrand wrote:
>> And please trim your emails.
>>
> 
> If you use Thunderbird I suggest QuoteCollapse ... because nobody got time for that ;)

neat.

Re: [PATCH 06/15] s390x: protvirt: Support unpack facility

[Janosch Frank]

[-- Attachment #1.1: Type: text/plain, Size: 344 bytes --]

On 11/21/19 3:31 PM, Christian Borntraeger wrote:
> 
> 
> On 21.11.19 15:28, David Hildenbrand wrote:
>>> And please trim your emails.
>>>
>>
>> If you use Thunderbird I suggest QuoteCollapse ... because nobody got time for that ;)
> 
> neat.
> 

Yeah, seems like I'm already too old-school for fancy addons at my young
age ;-)


[-- Attachment #2: OpenPGP digital signature --]
[-- Type: application/pgp-signature, Size: 833 bytes --]

Re: [PATCH 03/15] s390x: protvirt: Add diag308 subcodes 8 - 10

[Thomas Huth]

On 20/11/2019 12.43, Janosch Frank wrote:
> For diag308 subcodes 8 - 10 we have a new ipib of type 5. The ipib
> holds the address and length of the secure execution header, as well
> as a list of guest components.
> 
> Each component is a block of memory, for example kernel or initrd,
> which needs to be decrypted by the Ultravisor in order to run a
> protected VM. The secure execution header instructs the Ultravisor on
> how to handle the protected VM and its components.
> 
> Subcodes 8 and 9 are similiar to 5 and 6 and subcode 10 will finally
> start the protected guest.
> 
> Subcodes 8-10 are not valid in protected mode, we have to do a subcode
> 3 and then the 8 and 10 combination for a protected reboot.
> 
> Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
> ---
[...]
> diff --git a/hw/s390x/ipl.h b/hw/s390x/ipl.h
> index d4813105db..7b8a493509 100644
> --- a/hw/s390x/ipl.h
> +++ b/hw/s390x/ipl.h
> @@ -15,6 +15,24 @@
>  #include "cpu.h"
>  #include "hw/qdev-core.h"
>  
> +struct IPLBlockPVComp {
> +    uint64_t tweak_pref;
> +    uint64_t addr;
> +    uint64_t size;
> +} QEMU_PACKED;
> +typedef struct IPLBlockPVComp IPLBlockPVComp;
> +
> +struct IPLBlockPV {
> +    uint8_t  reserved[84];
> +    uint8_t  reserved67[3];

What does the "67" mean here?

> +    uint8_t  version;
> +    uint32_t num_comp;
> +    uint64_t pv_header_addr;
> +    uint64_t pv_header_len;
> +    struct IPLBlockPVComp components[];
> +} QEMU_PACKED;
> +typedef struct IPLBlockPV IPLBlockPV;

Given the fact that we had quite some headaches with QEMU_PACKED structs
in the past already, and the structs seem to be naturally aligned ...
what about dropping the QEMU_PACKED here and using QEMU_BUILD_BUG() to
assert that the struct has the correct size?

[...]
> @@ -185,4 +211,11 @@ static inline bool iplb_valid_fcp(IplParameterBlock *iplb)
>             iplb->pbt == S390_IPL_TYPE_FCP;
>  }
>  
> +static inline bool iplb_valid_se(IplParameterBlock *iplb)
> +{
> +    return be32_to_cpu(iplb->len) >= S390_IPLB_MIN_PV_LEN &&
> +           iplb->pbt == S390_IPL_TYPE_PV;
> +}
> +
> +

Drop one empty line?

>  #endif
> diff --git a/target/s390x/diag.c b/target/s390x/diag.c
> index 067c667ba7..32049bb4ee 100644
> --- a/target/s390x/diag.c
> +++ b/target/s390x/diag.c
> @@ -52,6 +52,8 @@ int handle_diag_288(CPUS390XState *env, uint64_t r1, uint64_t r3)
>  #define DIAG_308_RC_OK              0x0001
>  #define DIAG_308_RC_NO_CONF         0x0102
>  #define DIAG_308_RC_INVALID         0x0402
> +#define DIAG_308_RC_NO_PV_CONF      0x0a02
> +#define DIAG_308_RC_INV_FOR_PV      0x0b02
>  
>  #define DIAG308_RES_MOD_CLR		0
>  #define DIAG308_RES_LOAD_NORM		1
> @@ -59,6 +61,9 @@ int handle_diag_288(CPUS390XState *env, uint64_t r1, uint64_t r3)
>  #define DIAG308_LOAD_NORMAL_DUMP	4
>  #define DIAG308_SET			5
>  #define DIAG308_STORE			6
> +#define DIAG308_PV_SET			8
> +#define DIAG308_PV_STORE		9
> +#define DIAG308_PV_START		10
>  
>  static int diag308_parm_check(CPUS390XState *env, uint64_t r1, uint64_t addr,
>                                uintptr_t ra, bool write)
> @@ -105,6 +110,7 @@ void handle_diag_308(CPUS390XState *env, uint64_t r1, uint64_t r3, uintptr_t ra)
>          s390_ipl_reset_request(cs, S390_RESET_REIPL);
>          break;
>      case DIAG308_SET:
> +    case DIAG308_PV_SET: /* Set SE parms */
>          if (diag308_parm_check(env, r1, addr, ra, false)) {
>              return;
>          }
> @@ -117,7 +123,8 @@ void handle_diag_308(CPUS390XState *env, uint64_t r1, uint64_t r3, uintptr_t ra)
>  
>          cpu_physical_memory_read(addr, iplb, be32_to_cpu(iplb->len));
>  
> -        if (!iplb_valid_ccw(iplb) && !iplb_valid_fcp(iplb)) {
> +        if (!iplb_valid_ccw(iplb) && !iplb_valid_fcp(iplb) &&
> +            !(iplb_valid_se(iplb) && s390_ipl_pv_check_comp(iplb) >= 0)) {
>              env->regs[r1 + 1] = DIAG_308_RC_INVALID;
>              goto out;
>          }
> @@ -128,10 +135,15 @@ out:
>          g_free(iplb);
>          return;
>      case DIAG308_STORE:
> +    case DIAG308_PV_STORE: /* Get SE parms */
>          if (diag308_parm_check(env, r1, addr, ra, true)) {
>              return;
>          }
> -        iplb = s390_ipl_get_iplb();
> +        if (subcode == DIAG308_PV_STORE) {
> +            iplb = s390_ipl_get_iplb_secure();
> +        } else {
> +            iplb = s390_ipl_get_iplb();
> +        }
>          if (iplb) {
>              cpu_physical_memory_write(addr, iplb, be32_to_cpu(iplb->len));
>              env->regs[r1 + 1] = DIAG_308_RC_OK;
> @@ -139,6 +151,16 @@ out:
>              env->regs[r1 + 1] = DIAG_308_RC_NO_CONF;
>          }
>          return;
> +        break;

<hamletmode>
To return or to break, that's the question...
</hamletmode>

... please choose one of the two.

> +    case DIAG308_PV_START: /* SE start */
> +        iplb = s390_ipl_get_iplb_secure();
> +        if (!iplb_valid_se(iplb)) {
> +            env->regs[r1 + 1] = DIAG_308_RC_NO_PV_CONF;
> +            return;
> +        }
> +
> +        s390_ipl_reset_request(cs, S390_RESET_PV);
> +        break;
>      default:
>          s390_program_interrupt(env, PGM_SPECIFICATION, ra);
>          break;
> 

 Thomas

Re: [PATCH 05/15] s390x: protvirt: Sync PV state

[Thomas Huth]

On 20/11/2019 12.43, Janosch Frank wrote:
> We do not always have the SIE intercept code handy at each place where
> we do emulation. Unfortunately emulation for secure guests often
> differ slightly from normal emulation and we need to make decisions
> based on the protected state of the VCPU.
> 
> Let's sync the protected state and make it available.
> 
> Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
> ---
>  linux-headers/asm-s390/kvm.h | 1 +
>  target/s390x/cpu.h           | 1 +
>  target/s390x/kvm.c           | 4 ++++
>  3 files changed, 6 insertions(+)
> 
> diff --git a/linux-headers/asm-s390/kvm.h b/linux-headers/asm-s390/kvm.h
> index 41976d33f0..7c46cf6078 100644
> --- a/linux-headers/asm-s390/kvm.h
> +++ b/linux-headers/asm-s390/kvm.h
> @@ -231,6 +231,7 @@ struct kvm_guest_debug_arch {
>  #define KVM_SYNC_GSCB   (1UL << 9)
>  #define KVM_SYNC_BPBC   (1UL << 10)
>  #define KVM_SYNC_ETOKEN (1UL << 11)
> +#define KVM_SYNC_PV	(1UL << 12)
>  /* length and alignment of the sdnx as a power of two */
>  #define SDNXC 8
>  #define SDNXL (1UL << SDNXC)
> diff --git a/target/s390x/cpu.h b/target/s390x/cpu.h
> index 17460ed7b3..a787221772 100644
> --- a/target/s390x/cpu.h
> +++ b/target/s390x/cpu.h
> @@ -116,6 +116,7 @@ struct CPUS390XState {
>  
>      /* Fields up to this point are cleared by a CPU reset */
>      struct {} end_reset_fields;
> +    bool pv; /* protected virtualization */
>  
>  #if !defined(CONFIG_USER_ONLY)
>      uint32_t core_id; /* PoP "CPU address", same as cpu_index */
> diff --git a/target/s390x/kvm.c b/target/s390x/kvm.c
> index c24c869e77..418154ccfe 100644
> --- a/target/s390x/kvm.c
> +++ b/target/s390x/kvm.c
> @@ -676,6 +676,10 @@ int kvm_arch_get_registers(CPUState *cs)
>          env->etoken_extension = cs->kvm_run->s.regs.etoken_extension;
>      }
>  
> +    if (can_sync_regs(cs, KVM_SYNC_PV)) {
> +        env->pv = !!cs->kvm_run->s.regs.pv;
> +    }

Does this really need to be sync'ed each time during
kvm_arch_get_registers()? I mean, this is not a state that continuously
changes ... so when I guest enters PV mode, I assume that we have to do
some stuff in QEMU anyway, so we could set the "pv = true" in that case.
And during reset, we could clear it again. Or do I miss something?

 Thomas

Re: [PATCH 08/15] s390x: protvirt: KVM intercept changes

[Thomas Huth]

On 20/11/2019 12.43, Janosch Frank wrote:
> Secure guests no longer intercept with code 4 for an instruction
> interception. Instead they have codes 104 and 108 for secure
> instruction interception and secure instruction notification
> respectively.
> 
> The 104 mirrors the 4, but the 108 is a notification, that something
> happened and the hypervisor might need to adjust its tracking data to
> that fact. An example for that is the set prefix notification
> interception, where KVM only reads the new prefix, but does not update
> the prefix in the state description.
> 
> Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
> ---
>  target/s390x/kvm.c | 6 ++++++
>  1 file changed, 6 insertions(+)
> 
> diff --git a/target/s390x/kvm.c b/target/s390x/kvm.c
> index 418154ccfe..58251c0229 100644
> --- a/target/s390x/kvm.c
> +++ b/target/s390x/kvm.c
> @@ -115,6 +115,8 @@
>  #define ICPT_CPU_STOP                   0x28
>  #define ICPT_OPEREXC                    0x2c
>  #define ICPT_IO                         0x40
> +#define ICPT_PV_INSTR                   0x68
> +#define ICPT_PV_INSTR_NOT               0x6c
>  
>  #define NR_LOCAL_IRQS 32
>  /*
> @@ -151,6 +153,7 @@ static int cap_s390_irq;
>  static int cap_ri;
>  static int cap_gs;
>  static int cap_hpage_1m;
> +static int cap_protvirt;
>  
>  static int active_cmma;
>  
> @@ -336,6 +339,7 @@ int kvm_arch_init(MachineState *ms, KVMState *s)
>      cap_async_pf = kvm_check_extension(s, KVM_CAP_ASYNC_PF);
>      cap_mem_op = kvm_check_extension(s, KVM_CAP_S390_MEM_OP);
>      cap_s390_irq = kvm_check_extension(s, KVM_CAP_S390_INJECT_IRQ);
> +    cap_protvirt = kvm_check_extension(s, KVM_CAP_S390_PROTECTED);
>  
>      if (!kvm_check_extension(s, KVM_CAP_S390_GMAP)
>          || !kvm_check_extension(s, KVM_CAP_S390_COW)) {
> @@ -1664,6 +1668,8 @@ static int handle_intercept(S390CPU *cpu)
>              (long)cs->kvm_run->psw_addr);
>      switch (icpt_code) {
>          case ICPT_INSTRUCTION:
> +        case ICPT_PV_INSTR:
> +        case ICPT_PV_INSTR_NOT:
>              r = handle_instruction(cpu, run);

Even if this works by default, my gut feeling tells me that it would be
safer and cleaner to have a separate handler for this...
Otherwise we might get surprising results if future machine generations
intercept/notify for more or different instructions, I guess?

However, it's just a gut feeling ... I really don't have much experience
with this PV stuff yet ... what do the others here think?

 Thomas

Re: [PATCH 06/15] s390x: protvirt: Support unpack facility

[Cornelia Huck]

On Wed, 20 Nov 2019 06:43:25 -0500
Janosch Frank <frankja@linux.ibm.com> wrote:

> When a guest has saved a ipib of type 5 and call diagnose308 with
> subcode 10, we have to setup the protected processing environment via
> Ultravisor calls. The calls are done by KVM and are exposed via an API.
> 
> The following steps are necessary:
> 1. Create a VM (register it with the Ultravisor)
> 2. Create secure CPUs for all of our current cpus
> 3. Forward the secure header to the Ultravisor (has all information on
> how to decrypt the image and VM information)
> 4. Protect image pages from the host and decrypt them
> 5. Verify the image integrity
> 
> Only after step 5 a protected VM is allowed to run.
> 
> Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
> ---
>  hw/s390x/Makefile.objs              |   1 +
>  hw/s390x/ipl.c                      |  33 ++++++++
>  hw/s390x/ipl.h                      |   2 +
>  hw/s390x/pv.c                       | 118 ++++++++++++++++++++++++++++
>  hw/s390x/pv.h                       |  26 ++++++
>  hw/s390x/s390-virtio-ccw.c          |  45 ++++++++---
>  target/s390x/cpu_features_def.inc.h |   1 +
>  7 files changed, 216 insertions(+), 10 deletions(-)
>  create mode 100644 hw/s390x/pv.c
>  create mode 100644 hw/s390x/pv.h
> 

> +static int s390_pv_cmd(uint32_t cmd, void *data)
> +{
> +    int rc;
> +    struct kvm_pv_cmd pv_cmd = {
> +        .cmd = cmd,
> +        .data = (uint64_t)data,
> +    };
> +
> +    rc = kvm_vm_ioctl(kvm_state, KVM_S390_PV_COMMAND, &pv_cmd);
> +    if (rc) {
> +        error_report("KVM PV command failed cmd: %d rc: %d", cmd, rc);
> +        exit(1);
> +    }
> +    return rc;
> +}
> +
> +static int s390_pv_cmd_vcpu(CPUState *cs, uint32_t cmd, void *data)
> +{
> +    int rc;
> +    struct kvm_pv_cmd pv_cmd = {
> +        .cmd = cmd,
> +        .data = (uint64_t)data,
> +    };
> +
> +    rc = kvm_vcpu_ioctl(cs, KVM_S390_PV_COMMAND_VCPU, &pv_cmd);
> +    if (rc) {
> +        error_report("KVM PV VCPU command failed cmd: %d rc: %d", cmd, rc);
> +        exit(1);
> +    }
> +    return rc;
> +}

If you do a hard exit for any rc != 0 anyway, returning rc does not
sound very useful :)

Re: [PATCH 09/15] s390x: protvirt: SCLP interpretation

[Pierre Morel]

On 2019-11-21 15:24, Janosch Frank wrote:
> On 11/21/19 3:11 PM, Cornelia Huck wrote:
>> On Wed, 20 Nov 2019 06:43:28 -0500
>> Janosch Frank <frankja@linux.ibm.com> wrote:
>>
>>> SCLP for a protected guest is done over the SIDAD, so we need to use
>>> the s390_cpu_virt_mem_* functions to access the SIDAD instead of guest
>>> memory when reading/writing SCBs.
>>>
>>> To not confuse the sclp emulation, we set 0x42000 as the address, but
>>> ignore it for reading/writing the SCCB.
>>>
>>> Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
>>> ---
>>>   hw/s390x/sclp.c         | 16 ++++++++++++++++
>>>   include/hw/s390x/sclp.h |  2 ++
>>>   target/s390x/kvm.c      |  8 +++++++-
>>>   3 files changed, 25 insertions(+), 1 deletion(-)
>>>
>>> diff --git a/hw/s390x/sclp.c b/hw/s390x/sclp.c
>>> index f57ce7b739..00d08adc7f 100644
>>> --- a/hw/s390x/sclp.c
>>> +++ b/hw/s390x/sclp.c
>>> @@ -193,6 +193,22 @@ static void sclp_execute(SCLPDevice *sclp, SCCB *sccb, uint32_t code)
>>>       }
>>>   }
>>>   
>>> +int sclp_service_call_protected(CPUS390XState *env, uint64_t sccb,
>>> +                                uint32_t code)
>>> +{
>>> +    SCLPDevice *sclp = get_sclp_device();
>>> +    SCLPDeviceClass *sclp_c = SCLP_GET_CLASS(sclp);
>>> +    SCCB work_sccb;
>>> +    hwaddr sccb_len = sizeof(SCCB);
>>> +
>>> +    s390_cpu_virt_mem_read(env_archcpu(env), 0, 0, &work_sccb, sccb_len);
>>> +    sclp_c->execute(sclp, &work_sccb, code);
>>> +    s390_cpu_virt_mem_write(env_archcpu(env), 0, 0, &work_sccb,
>>> +                            be16_to_cpu(work_sccb.h.length));
>>> +    sclp_c->service_interrupt(sclp, sccb);
>>> +    return 0;
>>> +}
>>> +
>>>   int sclp_service_call(CPUS390XState *env, uint64_t sccb, uint32_t code)
>>>   {
>>>       SCLPDevice *sclp = get_sclp_device();
>>> diff --git a/include/hw/s390x/sclp.h b/include/hw/s390x/sclp.h
>>> index c54413b78c..c0a3faa37d 100644
>>> --- a/include/hw/s390x/sclp.h
>>> +++ b/include/hw/s390x/sclp.h
>>> @@ -217,5 +217,7 @@ void s390_sclp_init(void);
>>>   void sclp_service_interrupt(uint32_t sccb);
>>>   void raise_irq_cpu_hotplug(void);
>>>   int sclp_service_call(CPUS390XState *env, uint64_t sccb, uint32_t code);
>>> +int sclp_service_call_protected(CPUS390XState *env, uint64_t sccb,
>>> +                                uint32_t code);
>>>   
>>>   #endif
>>> diff --git a/target/s390x/kvm.c b/target/s390x/kvm.c
>>> index 58251c0229..0f2458b553 100644
>>> --- a/target/s390x/kvm.c
>>> +++ b/target/s390x/kvm.c
>>> @@ -1172,7 +1172,13 @@ static int kvm_sclp_service_call(S390CPU *cpu, struct kvm_run *run,
>>>       sccb = env->regs[ipbh0 & 0xf];
>>>       code = env->regs[(ipbh0 & 0xf0) >> 4];
>>>   
>>> -    r = sclp_service_call(env, sccb, code);
>>> +    if (run->s390_sieic.icptcode == ICPT_PV_INSTR ||
>>> +        run->s390_sieic.icptcode == ICPT_PV_INSTR_NOT) {
>>> +        r = sclp_service_call_protected(env, 0x42000, code);
>> I fear that confuses the reader instead of the emulation :)
>>
>> Especially as you end up passing that value to
>> sclp_c->service_interrupt()...
> Pierre has some more opinions on that, so I'll let him present his
> planned changes to this patch :)


Hum, sorry but I can not present any thing at the moment.
I will study the problem, it will take some time.


>
>>> +    } else {
>>> +        r = sclp_service_call(env, sccb, code);
>>> +    }
>>> +
>>>       if (r < 0) {
>>>           kvm_s390_program_interrupt(cpu, -r);
>>>       } else {
>
-- 
Pierre Morel
IBM Lab Boeblingen

Re: [PATCH 06/15] s390x: protvirt: Support unpack facility

[Janosch Frank]

[-- Attachment #1.1: Type: text/plain, Size: 2517 bytes --]

On 11/22/19 2:39 PM, Cornelia Huck wrote:
> On Wed, 20 Nov 2019 06:43:25 -0500
> Janosch Frank <frankja@linux.ibm.com> wrote:
> 
>> When a guest has saved a ipib of type 5 and call diagnose308 with
>> subcode 10, we have to setup the protected processing environment via
>> Ultravisor calls. The calls are done by KVM and are exposed via an API.
>>
>> The following steps are necessary:
>> 1. Create a VM (register it with the Ultravisor)
>> 2. Create secure CPUs for all of our current cpus
>> 3. Forward the secure header to the Ultravisor (has all information on
>> how to decrypt the image and VM information)
>> 4. Protect image pages from the host and decrypt them
>> 5. Verify the image integrity
>>
>> Only after step 5 a protected VM is allowed to run.
>>
>> Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
>> ---
>>  hw/s390x/Makefile.objs              |   1 +
>>  hw/s390x/ipl.c                      |  33 ++++++++
>>  hw/s390x/ipl.h                      |   2 +
>>  hw/s390x/pv.c                       | 118 ++++++++++++++++++++++++++++
>>  hw/s390x/pv.h                       |  26 ++++++
>>  hw/s390x/s390-virtio-ccw.c          |  45 ++++++++---
>>  target/s390x/cpu_features_def.inc.h |   1 +
>>  7 files changed, 216 insertions(+), 10 deletions(-)
>>  create mode 100644 hw/s390x/pv.c
>>  create mode 100644 hw/s390x/pv.h
>>
> 
>> +static int s390_pv_cmd(uint32_t cmd, void *data)
>> +{
>> +    int rc;
>> +    struct kvm_pv_cmd pv_cmd = {
>> +        .cmd = cmd,
>> +        .data = (uint64_t)data,
>> +    };
>> +
>> +    rc = kvm_vm_ioctl(kvm_state, KVM_S390_PV_COMMAND, &pv_cmd);
>> +    if (rc) {
>> +        error_report("KVM PV command failed cmd: %d rc: %d", cmd, rc);
>> +        exit(1);
>> +    }
>> +    return rc;
>> +}
>> +
>> +static int s390_pv_cmd_vcpu(CPUState *cs, uint32_t cmd, void *data)
>> +{
>> +    int rc;
>> +    struct kvm_pv_cmd pv_cmd = {
>> +        .cmd = cmd,
>> +        .data = (uint64_t)data,
>> +    };
>> +
>> +    rc = kvm_vcpu_ioctl(cs, KVM_S390_PV_COMMAND_VCPU, &pv_cmd);
>> +    if (rc) {
>> +        error_report("KVM PV VCPU command failed cmd: %d rc: %d", cmd, rc);
>> +        exit(1);
>> +    }
>> +    return rc;
>> +}
> 
> If you do a hard exit for any rc != 0 anyway, returning rc does not
> sound very useful :)
> 

Yeah, that's also what Pierre said.
I'll need to rethink if there's actually a call where we could recover
from an error and if not make everything void.



[-- Attachment #2: OpenPGP digital signature --]
[-- Type: application/pgp-signature, Size: 833 bytes --]

Re: [PATCH 06/15] s390x: protvirt: Support unpack facility

[Thomas Huth]

On 20/11/2019 12.43, Janosch Frank wrote:
> When a guest has saved a ipib of type 5 and call diagnose308 with
> subcode 10, we have to setup the protected processing environment via
> Ultravisor calls. The calls are done by KVM and are exposed via an API.
> 
> The following steps are necessary:
> 1. Create a VM (register it with the Ultravisor)
> 2. Create secure CPUs for all of our current cpus
> 3. Forward the secure header to the Ultravisor (has all information on
> how to decrypt the image and VM information)
> 4. Protect image pages from the host and decrypt them
> 5. Verify the image integrity
> 
> Only after step 5 a protected VM is allowed to run.
> 
> Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
> ---
[...]
> diff --git a/hw/s390x/ipl.c b/hw/s390x/ipl.c
> index a077926f36..50501fcd27 100644
> --- a/hw/s390x/ipl.c
> +++ b/hw/s390x/ipl.c
> @@ -33,6 +33,7 @@
>  #include "qemu/cutils.h"
>  #include "qemu/option.h"
>  #include "exec/exec-all.h"
> +#include "pv.h"
>  
>  #define KERN_IMAGE_START                0x010000UL
>  #define LINUX_MAGIC_ADDR                0x010008UL
> @@ -668,6 +669,38 @@ static void s390_ipl_prepare_qipl(S390CPU *cpu)
>      cpu_physical_memory_unmap(addr, len, 1, len);
>  }
>  
> +int s390_ipl_prepare_pv_header(void)
> +{
> +    int rc;
> +    IplParameterBlock *iplb = s390_ipl_get_iplb_secure();
> +    IPLBlockPV *ipib_pv = &iplb->pv;
> +    void *hdr = g_malloc(ipib_pv->pv_header_len);
> +
> +    cpu_physical_memory_read(ipib_pv->pv_header_addr, hdr,
> +                             ipib_pv->pv_header_len);
> +    rc = s390_pv_set_sec_parms((uint64_t)hdr,
> +                               ipib_pv->pv_header_len);
> +    g_free(hdr);
> +    return rc;
> +}
> +
> +int s390_ipl_pv_unpack(void)
> +{
> +    int i, rc;
> +    IplParameterBlock *iplb = s390_ipl_get_iplb_secure();
> +    IPLBlockPV *ipib_pv = &iplb->pv;
> +
> +    for (i = 0; i < ipib_pv->num_comp; i++) {
> +        rc = s390_pv_unpack(ipib_pv->components[i].addr,
> +                            TARGET_PAGE_ALIGN(ipib_pv->components[i].size),
> +                            ipib_pv->components[i].tweak_pref);
> +        if (rc) {
> +            return rc;
> +        }
> +    }
> +    return rc;
> +}

For both functions, s390_ipl_prepare_pv_header() and
s390_ipl_pv_unpack(), you're ignoring the return code at the calling
site, so errors go completely unnoticed. I suggest to either do an
error_report() here or at the calling site...

> diff --git a/hw/s390x/pv.c b/hw/s390x/pv.c
> new file mode 100644
> index 0000000000..0218070322
> --- /dev/null
> +++ b/hw/s390x/pv.c
> @@ -0,0 +1,118 @@
> +/*
> + * Secure execution functions
> + *
> + * Copyright IBM Corp. 2019
> + * Author(s):
> + *  Janosch Frank <frankja@linux.ibm.com>
> + *
> + * This work is licensed under the terms of the GNU GPL, version 2 or (at
> + * your option) any later version. See the COPYING file in the top-level
> + * directory.
> + */
> +#include "qemu/osdep.h"
> +#include <sys/ioctl.h>
> +
> +#include <linux/kvm.h>
> +
> +#include "qemu/error-report.h"
> +#include "sysemu/kvm.h"
> +#include "pv.h"
> +
> +static int s390_pv_cmd(uint32_t cmd, void *data)
> +{
> +    int rc;
> +    struct kvm_pv_cmd pv_cmd = {
> +        .cmd = cmd,
> +        .data = (uint64_t)data,
> +    };
> +
> +    rc = kvm_vm_ioctl(kvm_state, KVM_S390_PV_COMMAND, &pv_cmd);
> +    if (rc) {
> +        error_report("KVM PV command failed cmd: %d rc: %d", cmd, rc);
> +        exit(1);
> +    }
> +    return rc;
> +}
> +
> +static int s390_pv_cmd_vcpu(CPUState *cs, uint32_t cmd, void *data)
> +{
> +    int rc;
> +    struct kvm_pv_cmd pv_cmd = {
> +        .cmd = cmd,
> +        .data = (uint64_t)data,
> +    };
> +
> +    rc = kvm_vcpu_ioctl(cs, KVM_S390_PV_COMMAND_VCPU, &pv_cmd);
> +    if (rc) {
> +        error_report("KVM PV VCPU command failed cmd: %d rc: %d", cmd, rc);
> +        exit(1);
> +    }
> +    return rc;
> +}

... or even better, since these functions exit on error anyway, make
these functions and all the others "void" instead?

 Thomas

Re: [PATCH 06/15] s390x: protvirt: Support unpack facility

[Janosch Frank]

[-- Attachment #1.1: Type: text/plain, Size: 4660 bytes --]

On 11/28/19 3:07 PM, Thomas Huth wrote:
> On 20/11/2019 12.43, Janosch Frank wrote:
>> When a guest has saved a ipib of type 5 and call diagnose308 with
>> subcode 10, we have to setup the protected processing environment via
>> Ultravisor calls. The calls are done by KVM and are exposed via an API.
>>
>> The following steps are necessary:
>> 1. Create a VM (register it with the Ultravisor)
>> 2. Create secure CPUs for all of our current cpus
>> 3. Forward the secure header to the Ultravisor (has all information on
>> how to decrypt the image and VM information)
>> 4. Protect image pages from the host and decrypt them
>> 5. Verify the image integrity
>>
>> Only after step 5 a protected VM is allowed to run.
>>
>> Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
>> ---
> [...]
>> diff --git a/hw/s390x/ipl.c b/hw/s390x/ipl.c
>> index a077926f36..50501fcd27 100644
>> --- a/hw/s390x/ipl.c
>> +++ b/hw/s390x/ipl.c
>> @@ -33,6 +33,7 @@
>>  #include "qemu/cutils.h"
>>  #include "qemu/option.h"
>>  #include "exec/exec-all.h"
>> +#include "pv.h"
>>  
>>  #define KERN_IMAGE_START                0x010000UL
>>  #define LINUX_MAGIC_ADDR                0x010008UL
>> @@ -668,6 +669,38 @@ static void s390_ipl_prepare_qipl(S390CPU *cpu)
>>      cpu_physical_memory_unmap(addr, len, 1, len);
>>  }
>>  
>> +int s390_ipl_prepare_pv_header(void)
>> +{
>> +    int rc;
>> +    IplParameterBlock *iplb = s390_ipl_get_iplb_secure();
>> +    IPLBlockPV *ipib_pv = &iplb->pv;
>> +    void *hdr = g_malloc(ipib_pv->pv_header_len);
>> +
>> +    cpu_physical_memory_read(ipib_pv->pv_header_addr, hdr,
>> +                             ipib_pv->pv_header_len);
>> +    rc = s390_pv_set_sec_parms((uint64_t)hdr,
>> +                               ipib_pv->pv_header_len);
>> +    g_free(hdr);
>> +    return rc;
>> +}
>> +
>> +int s390_ipl_pv_unpack(void)
>> +{
>> +    int i, rc;
>> +    IplParameterBlock *iplb = s390_ipl_get_iplb_secure();
>> +    IPLBlockPV *ipib_pv = &iplb->pv;
>> +
>> +    for (i = 0; i < ipib_pv->num_comp; i++) {
>> +        rc = s390_pv_unpack(ipib_pv->components[i].addr,
>> +                            TARGET_PAGE_ALIGN(ipib_pv->components[i].size),
>> +                            ipib_pv->components[i].tweak_pref);
>> +        if (rc) {
>> +            return rc;
>> +        }
>> +    }
>> +    return rc;
>> +}
> 
> For both functions, s390_ipl_prepare_pv_header() and
> s390_ipl_pv_unpack(), you're ignoring the return code at the calling
> site, so errors go completely unnoticed. I suggest to either do an
> error_report() here or at the calling site...
> 
>> diff --git a/hw/s390x/pv.c b/hw/s390x/pv.c
>> new file mode 100644
>> index 0000000000..0218070322
>> --- /dev/null
>> +++ b/hw/s390x/pv.c
>> @@ -0,0 +1,118 @@
>> +/*
>> + * Secure execution functions
>> + *
>> + * Copyright IBM Corp. 2019
>> + * Author(s):
>> + *  Janosch Frank <frankja@linux.ibm.com>
>> + *
>> + * This work is licensed under the terms of the GNU GPL, version 2 or (at
>> + * your option) any later version. See the COPYING file in the top-level
>> + * directory.
>> + */
>> +#include "qemu/osdep.h"
>> +#include <sys/ioctl.h>
>> +
>> +#include <linux/kvm.h>
>> +
>> +#include "qemu/error-report.h"
>> +#include "sysemu/kvm.h"
>> +#include "pv.h"
>> +
>> +static int s390_pv_cmd(uint32_t cmd, void *data)
>> +{
>> +    int rc;
>> +    struct kvm_pv_cmd pv_cmd = {
>> +        .cmd = cmd,
>> +        .data = (uint64_t)data,
>> +    };
>> +
>> +    rc = kvm_vm_ioctl(kvm_state, KVM_S390_PV_COMMAND, &pv_cmd);
>> +    if (rc) {
>> +        error_report("KVM PV command failed cmd: %d rc: %d", cmd, rc);
>> +        exit(1);
>> +    }
>> +    return rc;
>> +}
>> +
>> +static int s390_pv_cmd_vcpu(CPUState *cs, uint32_t cmd, void *data)
>> +{
>> +    int rc;
>> +    struct kvm_pv_cmd pv_cmd = {
>> +        .cmd = cmd,
>> +        .data = (uint64_t)data,
>> +    };
>> +
>> +    rc = kvm_vcpu_ioctl(cs, KVM_S390_PV_COMMAND_VCPU, &pv_cmd);
>> +    if (rc) {
>> +        error_report("KVM PV VCPU command failed cmd: %d rc: %d", cmd, rc);
>> +        exit(1);
>> +    }
>> +    return rc;
>> +}
> 
> ... or even better, since these functions exit on error anyway, make
> these functions and all the others "void" instead?
> 
>  Thomas
> 

You're not the first stating this idea :-)
We might get a return code for diag308 that signals that we failed to
start the secure guest and then I might need them again.

If we don't get a rc, I'll definitely make all functions void, if we do
I'll make nearly all void:)


[-- Attachment #2: OpenPGP digital signature --]
[-- Type: application/pgp-signature, Size: 833 bytes --]

Re: [PATCH 12/15] s390x: protvirt: Set guest IPL PSW

[Thomas Huth]

On 20/11/2019 12.43, Janosch Frank wrote:
> Handling of CPU reset and setting of the IPL psw from guest storage at
> offset 0 is done by a Ultravisor call. Let's only fetch it if
> necessary.
> 
> Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
> ---
>  hw/s390x/pv.c              | 5 +++++
>  hw/s390x/pv.h              | 1 +
>  hw/s390x/s390-virtio-ccw.c | 3 ++-
>  linux-headers/linux/kvm.h  | 1 +
>  target/s390x/cpu.c         | 9 ++++++++-
>  5 files changed, 17 insertions(+), 2 deletions(-)
[...]
> diff --git a/hw/s390x/s390-virtio-ccw.c b/hw/s390x/s390-virtio-ccw.c
> index 6fd50b4c42..e020b92854 100644
> --- a/hw/s390x/s390-virtio-ccw.c
> +++ b/hw/s390x/s390-virtio-ccw.c
> @@ -404,7 +404,8 @@ static void s390_machine_reset(MachineState *machine)
>          s390_ipl_pv_unpack();
>          /* Verify integrity */
>          s390_pv_verify();
> -        s390_cpu_set_state(S390_CPU_STATE_OPERATING, cpu);
> +        env->pv = true;

Ah, here you set env->pv = true manually ... so maybe that's another
good reason to get rid of patch 05/15 ("s390x: protvirt: Sync PV state")
in the end?

Reviewed-by: Thomas Huth <thuth@redhat.com>

Re: [PATCH 13/15] s390x: protvirt: Move diag 308 data over SIDAD

[Thomas Huth]

On 20/11/2019 12.43, Janosch Frank wrote:
> For protected guests the IPIB is written/read to/from the sattelite

satellite

> block, so we need to make those accesses virtual to make them go
> through KBM mem ops.

What's KBM ?

> Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
> ---
>  target/s390x/diag.c | 26 +++++++++++++++++++++++---
>  1 file changed, 23 insertions(+), 3 deletions(-)
> 
> diff --git a/target/s390x/diag.c b/target/s390x/diag.c
> index db6d79cef3..d96d8bdc6c 100644
> --- a/target/s390x/diag.c
> +++ b/target/s390x/diag.c
> @@ -88,6 +88,7 @@ static int diag308_parm_check(CPUS390XState *env, uint64_t r1, uint64_t addr,
>  void handle_diag_308(CPUS390XState *env, uint64_t r1, uint64_t r3, uintptr_t ra)
>  {
>      CPUState *cs = env_cpu(env);
> +    S390CPU *cpu = S390_CPU(cs);
>      uint64_t addr =  env->regs[r1];
>      uint64_t subcode = env->regs[r3];
>      IplParameterBlock *iplb;
> @@ -118,14 +119,27 @@ void handle_diag_308(CPUS390XState *env, uint64_t r1, uint64_t r3, uintptr_t ra)
>          if (diag308_parm_check(env, r1, addr, ra, false)) {
>              return;
>          }
> +
>          iplb = g_new0(IplParameterBlock, 1);
> -        cpu_physical_memory_read(addr, iplb, sizeof(iplb->len));
> +        if (!env->pv) {
> +            cpu_physical_memory_read(addr, iplb, sizeof(iplb->len));
> +        } else {
> +            s390_cpu_virt_mem_read(cpu, 0, 0, iplb, sizeof(iplb->len));
> +            s390_cpu_virt_mem_handle_exc(cpu, ra);

I'm looking forward to protected virt support in TCG ;-)

> +        }
> +
>          if (!iplb_valid_len(iplb)) {
>              env->regs[r1 + 1] = DIAG_308_RC_INVALID;
>              goto out;
>          }
>  
> -        cpu_physical_memory_read(addr, iplb, be32_to_cpu(iplb->len));
> +        if (!env->pv) {
> +            cpu_physical_memory_read(addr, iplb, be32_to_cpu(iplb->len));
> +        } else {
> +            s390_cpu_virt_mem_read(cpu, 0, 0, iplb, be32_to_cpu(iplb->len));
> +            s390_cpu_virt_mem_handle_exc(cpu, ra);
> +        }
> +
>  
>          if (!iplb_valid_ccw(iplb) && !iplb_valid_fcp(iplb) &&
>              !(iplb_valid_se(iplb) && s390_ipl_pv_check_comp(iplb) >= 0)) {
> @@ -149,7 +163,13 @@ out:
>              iplb = s390_ipl_get_iplb();
>          }
>          if (iplb) {
> -            cpu_physical_memory_write(addr, iplb, be32_to_cpu(iplb->len));
> +            if (!env->pv) {
> +                cpu_physical_memory_write(addr, iplb, be32_to_cpu(iplb->len));
> +            } else {
> +                s390_cpu_virt_mem_write(cpu, 0, 0, iplb,
> +                                        be32_to_cpu(iplb->len));
> +                s390_cpu_virt_mem_handle_exc(cpu, ra);
> +            }
>              env->regs[r1 + 1] = DIAG_308_RC_OK;
>          } else {
>              env->regs[r1 + 1] = DIAG_308_RC_NO_CONF;
> 

I wonder whether we maybe want to have some wrapper functions for these
reads and writes, something like:

void s390_cpu_physical_memory_write(...)
{
    if (!env->pv) {
        cpu_physical_memory_write(...);
    } else {
        s390_cpu_virt_mem_write(...);
    }
}

?

 Thomas

Re: [PATCH 14/15] s390x: protvirt: Disable address checks for PV guest IO emulation

[Thomas Huth]

On 20/11/2019 12.43, Janosch Frank wrote:
> IO instruction data is routed through SIDAD for protected guests, so
> adresses do not need to be checked, as this is kernel memory.
> 
> Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
> ---
>  target/s390x/ioinst.c | 46 +++++++++++++++++++++++++++----------------
>  1 file changed, 29 insertions(+), 17 deletions(-)
> 
> diff --git a/target/s390x/ioinst.c b/target/s390x/ioinst.c
> index c437a1d8c6..d3bd422ddd 100644
> --- a/target/s390x/ioinst.c
> +++ b/target/s390x/ioinst.c
> @@ -110,11 +110,13 @@ void ioinst_handle_msch(S390CPU *cpu, uint64_t reg1, uint32_t ipb, uintptr_t ra)
>      int cssid, ssid, schid, m;
>      SubchDev *sch;
>      SCHIB schib;
> -    uint64_t addr;
> +    uint64_t addr = 0;
>      CPUS390XState *env = &cpu->env;
> -    uint8_t ar;
> +    uint8_t ar = 0;
>  
> -    addr = decode_basedisp_s(env, ipb, &ar);
> +    if (!env->pv) {
> +        addr = decode_basedisp_s(env, ipb, &ar);
> +    }
>      if (addr & 3) {
>          s390_program_interrupt(env, PGM_SPECIFICATION, ra);
>          return;
> @@ -167,11 +169,13 @@ void ioinst_handle_ssch(S390CPU *cpu, uint64_t reg1, uint32_t ipb, uintptr_t ra)
>      int cssid, ssid, schid, m;
>      SubchDev *sch;
>      ORB orig_orb, orb;
> -    uint64_t addr;
> +    uint64_t addr = 0;
>      CPUS390XState *env = &cpu->env;
> -    uint8_t ar;
> +    uint8_t ar = 0;
>  
> -    addr = decode_basedisp_s(env, ipb, &ar);
> +    if (!env->pv) {
> +        addr = decode_basedisp_s(env, ipb, &ar);
> +    }
>      if (addr & 3) {
>          s390_program_interrupt(env, PGM_SPECIFICATION, ra);
>          return;
> @@ -198,12 +202,14 @@ void ioinst_handle_ssch(S390CPU *cpu, uint64_t reg1, uint32_t ipb, uintptr_t ra)
>  void ioinst_handle_stcrw(S390CPU *cpu, uint32_t ipb, uintptr_t ra)
>  {
>      CRW crw;
> -    uint64_t addr;
> +    uint64_t addr = 0;
>      int cc;
>      CPUS390XState *env = &cpu->env;
> -    uint8_t ar;
> +    uint8_t ar = 0;
>  
> -    addr = decode_basedisp_s(env, ipb, &ar);
> +    if (!env->pv) {
> +        addr = decode_basedisp_s(env, ipb, &ar);
> +    }
>      if (addr & 3) {
>          s390_program_interrupt(env, PGM_SPECIFICATION, ra);
>          return;
> @@ -228,13 +234,15 @@ void ioinst_handle_stsch(S390CPU *cpu, uint64_t reg1, uint32_t ipb,
>  {
>      int cssid, ssid, schid, m;
>      SubchDev *sch;
> -    uint64_t addr;
> +    uint64_t addr = 0;
>      int cc;
>      SCHIB schib;
>      CPUS390XState *env = &cpu->env;
> -    uint8_t ar;
> +    uint8_t ar = 0;
>  
> -    addr = decode_basedisp_s(env, ipb, &ar);
> +    if (!env->pv) {
> +        addr = decode_basedisp_s(env, ipb, &ar);
> +    }
>      if (addr & 3) {
>          s390_program_interrupt(env, PGM_SPECIFICATION, ra);
>          return;
> @@ -294,16 +302,18 @@ int ioinst_handle_tsch(S390CPU *cpu, uint64_t reg1, uint32_t ipb, uintptr_t ra)
>      int cssid, ssid, schid, m;
>      SubchDev *sch;
>      IRB irb;
> -    uint64_t addr;
> +    uint64_t addr = 0;
>      int cc, irb_len;
> -    uint8_t ar;
> +    uint8_t ar = 0;
>  
>      if (ioinst_disassemble_sch_ident(reg1, &m, &cssid, &ssid, &schid)) {
>          s390_program_interrupt(env, PGM_OPERAND, ra);
>          return -EIO;
>      }
>      trace_ioinst_sch_id("tsch", cssid, ssid, schid);
> -    addr = decode_basedisp_s(env, ipb, &ar);
> +    if (!env->pv) {
> +        addr = decode_basedisp_s(env, ipb, &ar);
> +    }
>      if (addr & 3) {
>          s390_program_interrupt(env, PGM_SPECIFICATION, ra);
>          return -EIO;

Would it make sense to hide all these changes in decode_basedisp_s()
instead? ... so that decode_basedisp_s() returns 0 if env->pv == true ?
... or are there still cases where we need real values from
decode_basedisp_s() in case of env->pv==true?

Anyway,
Reviewed-by: Thomas Huth <thuth@redhat.com>

Re: [PATCH 15/15] s390x: protvirt: Handle SIGP store status correctly

[Thomas Huth]

On 20/11/2019 12.43, Janosch Frank wrote:
> Status storing is obviously not done by qemu anymore.
> 
> Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
> ---
>  target/s390x/sigp.c | 7 +++++--
>  1 file changed, 5 insertions(+), 2 deletions(-)
> 
> diff --git a/target/s390x/sigp.c b/target/s390x/sigp.c
> index 2ce22d4dc1..68634d694a 100644
> --- a/target/s390x/sigp.c
> +++ b/target/s390x/sigp.c
> @@ -144,7 +144,9 @@ static void sigp_stop_and_store_status(CPUState *cs, run_on_cpu_data arg)
>      case S390_CPU_STATE_STOPPED:
>          /* already stopped, just store the status */
>          cpu_synchronize_state(cs);
> -        s390_store_status(cpu, S390_STORE_STATUS_DEF_ADDR, true);
> +        if (!cpu->env.pv) {
> +            s390_store_status(cpu, S390_STORE_STATUS_DEF_ADDR, true);
> +        }
>          break;
>      }
>      si->cc = SIGP_CC_ORDER_CODE_ACCEPTED;
> @@ -497,7 +499,8 @@ void do_stop_interrupt(CPUS390XState *env)
>      if (s390_cpu_set_state(S390_CPU_STATE_STOPPED, cpu) == 0) {
>          qemu_system_shutdown_request(SHUTDOWN_CAUSE_GUEST_SHUTDOWN);
>      }
> -    if (cpu->env.sigp_order == SIGP_STOP_STORE_STATUS) {
> +    /* Storing will occur on next SIE entry for fmt 4 */
> +    if (cpu->env.sigp_order == SIGP_STOP_STORE_STATUS && !env->pv) {
>          s390_store_status(cpu, S390_STORE_STATUS_DEF_ADDR, true);
>      }
>      env->sigp_order = 0;
> 

Maybe move the check into the s390_store_status() function instead?

Anyway,
Reviewed-by: Thomas Huth <thuth@redhat.com>

Re: [PATCH 14/15] s390x: protvirt: Disable address checks for PV guest IO emulation

[Janosch Frank]

[-- Attachment #1.1: Type: text/plain, Size: 4309 bytes --]

On 11/28/19 4:28 PM, Thomas Huth wrote:
> On 20/11/2019 12.43, Janosch Frank wrote:
>> IO instruction data is routed through SIDAD for protected guests, so
>> adresses do not need to be checked, as this is kernel memory.
>>
>> Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
>> ---
>>  target/s390x/ioinst.c | 46 +++++++++++++++++++++++++++----------------
>>  1 file changed, 29 insertions(+), 17 deletions(-)
>>
>> diff --git a/target/s390x/ioinst.c b/target/s390x/ioinst.c
>> index c437a1d8c6..d3bd422ddd 100644
>> --- a/target/s390x/ioinst.c
>> +++ b/target/s390x/ioinst.c
>> @@ -110,11 +110,13 @@ void ioinst_handle_msch(S390CPU *cpu, uint64_t reg1, uint32_t ipb, uintptr_t ra)
>>      int cssid, ssid, schid, m;
>>      SubchDev *sch;
>>      SCHIB schib;
>> -    uint64_t addr;
>> +    uint64_t addr = 0;
>>      CPUS390XState *env = &cpu->env;
>> -    uint8_t ar;
>> +    uint8_t ar = 0;
>>  
>> -    addr = decode_basedisp_s(env, ipb, &ar);
>> +    if (!env->pv) {
>> +        addr = decode_basedisp_s(env, ipb, &ar);
>> +    }
>>      if (addr & 3) {
>>          s390_program_interrupt(env, PGM_SPECIFICATION, ra);
>>          return;
>> @@ -167,11 +169,13 @@ void ioinst_handle_ssch(S390CPU *cpu, uint64_t reg1, uint32_t ipb, uintptr_t ra)
>>      int cssid, ssid, schid, m;
>>      SubchDev *sch;
>>      ORB orig_orb, orb;
>> -    uint64_t addr;
>> +    uint64_t addr = 0;
>>      CPUS390XState *env = &cpu->env;
>> -    uint8_t ar;
>> +    uint8_t ar = 0;
>>  
>> -    addr = decode_basedisp_s(env, ipb, &ar);
>> +    if (!env->pv) {
>> +        addr = decode_basedisp_s(env, ipb, &ar);
>> +    }
>>      if (addr & 3) {
>>          s390_program_interrupt(env, PGM_SPECIFICATION, ra);
>>          return;
>> @@ -198,12 +202,14 @@ void ioinst_handle_ssch(S390CPU *cpu, uint64_t reg1, uint32_t ipb, uintptr_t ra)
>>  void ioinst_handle_stcrw(S390CPU *cpu, uint32_t ipb, uintptr_t ra)
>>  {
>>      CRW crw;
>> -    uint64_t addr;
>> +    uint64_t addr = 0;
>>      int cc;
>>      CPUS390XState *env = &cpu->env;
>> -    uint8_t ar;
>> +    uint8_t ar = 0;
>>  
>> -    addr = decode_basedisp_s(env, ipb, &ar);
>> +    if (!env->pv) {
>> +        addr = decode_basedisp_s(env, ipb, &ar);
>> +    }
>>      if (addr & 3) {
>>          s390_program_interrupt(env, PGM_SPECIFICATION, ra);
>>          return;
>> @@ -228,13 +234,15 @@ void ioinst_handle_stsch(S390CPU *cpu, uint64_t reg1, uint32_t ipb,
>>  {
>>      int cssid, ssid, schid, m;
>>      SubchDev *sch;
>> -    uint64_t addr;
>> +    uint64_t addr = 0;
>>      int cc;
>>      SCHIB schib;
>>      CPUS390XState *env = &cpu->env;
>> -    uint8_t ar;
>> +    uint8_t ar = 0;
>>  
>> -    addr = decode_basedisp_s(env, ipb, &ar);
>> +    if (!env->pv) {
>> +        addr = decode_basedisp_s(env, ipb, &ar);
>> +    }
>>      if (addr & 3) {
>>          s390_program_interrupt(env, PGM_SPECIFICATION, ra);
>>          return;
>> @@ -294,16 +302,18 @@ int ioinst_handle_tsch(S390CPU *cpu, uint64_t reg1, uint32_t ipb, uintptr_t ra)
>>      int cssid, ssid, schid, m;
>>      SubchDev *sch;
>>      IRB irb;
>> -    uint64_t addr;
>> +    uint64_t addr = 0;
>>      int cc, irb_len;
>> -    uint8_t ar;
>> +    uint8_t ar = 0;
>>  
>>      if (ioinst_disassemble_sch_ident(reg1, &m, &cssid, &ssid, &schid)) {
>>          s390_program_interrupt(env, PGM_OPERAND, ra);
>>          return -EIO;
>>      }
>>      trace_ioinst_sch_id("tsch", cssid, ssid, schid);
>> -    addr = decode_basedisp_s(env, ipb, &ar);
>> +    if (!env->pv) {
>> +        addr = decode_basedisp_s(env, ipb, &ar);
>> +    }
>>      if (addr & 3) {
>>          s390_program_interrupt(env, PGM_SPECIFICATION, ra);
>>          return -EIO;
> 
> Would it make sense to hide all these changes in decode_basedisp_s()
> instead? ... so that decode_basedisp_s() returns 0 if env->pv == true ?
> ... or are there still cases where we need real values from
> decode_basedisp_s() in case of env->pv==true?

Pierre already suggested that, I'll look into it.
Hopefully we have no addr != 0 or addr > 2 * PAGE_SIZE checks.
Because of that it might make more sense to just rip out the checks.

> 
> Anyway,
> Reviewed-by: Thomas Huth <thuth@redhat.com>
> 



[-- Attachment #2: OpenPGP digital signature --]
[-- Type: application/pgp-signature, Size: 833 bytes --]

Re: [PATCH 12/15] s390x: protvirt: Set guest IPL PSW

[Janosch Frank]

[-- Attachment #1.1: Type: text/plain, Size: 1353 bytes --]

On 11/28/19 3:30 PM, Thomas Huth wrote:
> On 20/11/2019 12.43, Janosch Frank wrote:
>> Handling of CPU reset and setting of the IPL psw from guest storage at
>> offset 0 is done by a Ultravisor call. Let's only fetch it if
>> necessary.
>>
>> Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
>> ---
>>  hw/s390x/pv.c              | 5 +++++
>>  hw/s390x/pv.h              | 1 +
>>  hw/s390x/s390-virtio-ccw.c | 3 ++-
>>  linux-headers/linux/kvm.h  | 1 +
>>  target/s390x/cpu.c         | 9 ++++++++-
>>  5 files changed, 17 insertions(+), 2 deletions(-)
> [...]
>> diff --git a/hw/s390x/s390-virtio-ccw.c b/hw/s390x/s390-virtio-ccw.c
>> index 6fd50b4c42..e020b92854 100644
>> --- a/hw/s390x/s390-virtio-ccw.c
>> +++ b/hw/s390x/s390-virtio-ccw.c
>> @@ -404,7 +404,8 @@ static void s390_machine_reset(MachineState *machine)
>>          s390_ipl_pv_unpack();
>>          /* Verify integrity */
>>          s390_pv_verify();
>> -        s390_cpu_set_state(S390_CPU_STATE_OPERATING, cpu);
>> +        env->pv = true;
> 
> Ah, here you set env->pv = true manually ... so maybe that's another
> good reason to get rid of patch 05/15 ("s390x: protvirt: Sync PV state")
> in the end?

Given that QEMU triggers the state changes that would make sense.

> 
> Reviewed-by: Thomas Huth <thuth@redhat.com>

Thanks!

> 
> 



[-- Attachment #2: OpenPGP digital signature --]
[-- Type: application/pgp-signature, Size: 833 bytes --]

Re: [PATCH 13/15] s390x: protvirt: Move diag 308 data over SIDAD

[Janosch Frank]

[-- Attachment #1.1: Type: text/plain, Size: 3724 bytes --]

On 11/28/19 3:40 PM, Thomas Huth wrote:
> On 20/11/2019 12.43, Janosch Frank wrote:
>> For protected guests the IPIB is written/read to/from the sattelite
> 
> satellite
> 
>> block, so we need to make those accesses virtual to make them go
>> through KBM mem ops.
> 
> What's KBM ?

A totally new hypervisor that's much faster than KVM because b comes
before v in the alphabet.

> 
>> Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
>> ---
>>  target/s390x/diag.c | 26 +++++++++++++++++++++++---
>>  1 file changed, 23 insertions(+), 3 deletions(-)
>>
>> diff --git a/target/s390x/diag.c b/target/s390x/diag.c
>> index db6d79cef3..d96d8bdc6c 100644
>> --- a/target/s390x/diag.c
>> +++ b/target/s390x/diag.c
>> @@ -88,6 +88,7 @@ static int diag308_parm_check(CPUS390XState *env, uint64_t r1, uint64_t addr,
>>  void handle_diag_308(CPUS390XState *env, uint64_t r1, uint64_t r3, uintptr_t ra)
>>  {
>>      CPUState *cs = env_cpu(env);
>> +    S390CPU *cpu = S390_CPU(cs);
>>      uint64_t addr =  env->regs[r1];
>>      uint64_t subcode = env->regs[r3];
>>      IplParameterBlock *iplb;
>> @@ -118,14 +119,27 @@ void handle_diag_308(CPUS390XState *env, uint64_t r1, uint64_t r3, uintptr_t ra)
>>          if (diag308_parm_check(env, r1, addr, ra, false)) {
>>              return;
>>          }
>> +
>>          iplb = g_new0(IplParameterBlock, 1);
>> -        cpu_physical_memory_read(addr, iplb, sizeof(iplb->len));
>> +        if (!env->pv) {
>> +            cpu_physical_memory_read(addr, iplb, sizeof(iplb->len));
>> +        } else {
>> +            s390_cpu_virt_mem_read(cpu, 0, 0, iplb, sizeof(iplb->len));
>> +            s390_cpu_virt_mem_handle_exc(cpu, ra);
> 
> I'm looking forward to protected virt support in TCG ;-)

Who doesn't?

> 
>> +        }
>> +
>>          if (!iplb_valid_len(iplb)) {
>>              env->regs[r1 + 1] = DIAG_308_RC_INVALID;
>>              goto out;
>>          }
>>  
>> -        cpu_physical_memory_read(addr, iplb, be32_to_cpu(iplb->len));
>> +        if (!env->pv) {
>> +            cpu_physical_memory_read(addr, iplb, be32_to_cpu(iplb->len));
>> +        } else {
>> +            s390_cpu_virt_mem_read(cpu, 0, 0, iplb, be32_to_cpu(iplb->len));
>> +            s390_cpu_virt_mem_handle_exc(cpu, ra);
>> +        }
>> +
>>  
>>          if (!iplb_valid_ccw(iplb) && !iplb_valid_fcp(iplb) &&
>>              !(iplb_valid_se(iplb) && s390_ipl_pv_check_comp(iplb) >= 0)) {
>> @@ -149,7 +163,13 @@ out:
>>              iplb = s390_ipl_get_iplb();
>>          }
>>          if (iplb) {
>> -            cpu_physical_memory_write(addr, iplb, be32_to_cpu(iplb->len));
>> +            if (!env->pv) {
>> +                cpu_physical_memory_write(addr, iplb, be32_to_cpu(iplb->len));
>> +            } else {
>> +                s390_cpu_virt_mem_write(cpu, 0, 0, iplb,
>> +                                        be32_to_cpu(iplb->len));
>> +                s390_cpu_virt_mem_handle_exc(cpu, ra);
>> +            }
>>              env->regs[r1 + 1] = DIAG_308_RC_OK;
>>          } else {
>>              env->regs[r1 + 1] = DIAG_308_RC_NO_CONF;
>>
> 
> I wonder whether we maybe want to have some wrapper functions for these
> reads and writes, something like:

diag308 is currently the only instruction where we would need it.

sclp needs a new handler, because we can skip lots of checks if pv is
enabled.
IO instructions are already using logical addressing

> 
> void s390_cpu_physical_memory_write(...)
> {
>     if (!env->pv) {
>         cpu_physical_memory_write(...);
>     } else {
>         s390_cpu_virt_mem_write(...);
>     }
> }
> 
> ?
> 
>  Thomas
> 



[-- Attachment #2: OpenPGP digital signature --]
[-- Type: application/pgp-signature, Size: 833 bytes --]

Re: [PATCH 14/15] s390x: protvirt: Disable address checks for PV guest IO emulation

[Janosch Frank]

[-- Attachment #1.1: Type: text/plain, Size: 4321 bytes --]

On 11/28/19 4:28 PM, Thomas Huth wrote:
> On 20/11/2019 12.43, Janosch Frank wrote:
>> IO instruction data is routed through SIDAD for protected guests, so
>> adresses do not need to be checked, as this is kernel memory.
>>
>> Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
>> ---
>>  target/s390x/ioinst.c | 46 +++++++++++++++++++++++++++----------------
>>  1 file changed, 29 insertions(+), 17 deletions(-)
>>
>> diff --git a/target/s390x/ioinst.c b/target/s390x/ioinst.c
>> index c437a1d8c6..d3bd422ddd 100644
>> --- a/target/s390x/ioinst.c
>> +++ b/target/s390x/ioinst.c
>> @@ -110,11 +110,13 @@ void ioinst_handle_msch(S390CPU *cpu, uint64_t reg1, uint32_t ipb, uintptr_t ra)
>>      int cssid, ssid, schid, m;
>>      SubchDev *sch;
>>      SCHIB schib;
>> -    uint64_t addr;
>> +    uint64_t addr = 0;
>>      CPUS390XState *env = &cpu->env;
>> -    uint8_t ar;
>> +    uint8_t ar = 0;
>>  
>> -    addr = decode_basedisp_s(env, ipb, &ar);
>> +    if (!env->pv) {
>> +        addr = decode_basedisp_s(env, ipb, &ar);
>> +    }
>>      if (addr & 3) {
>>          s390_program_interrupt(env, PGM_SPECIFICATION, ra);
>>          return;
>> @@ -167,11 +169,13 @@ void ioinst_handle_ssch(S390CPU *cpu, uint64_t reg1, uint32_t ipb, uintptr_t ra)
>>      int cssid, ssid, schid, m;
>>      SubchDev *sch;
>>      ORB orig_orb, orb;
>> -    uint64_t addr;
>> +    uint64_t addr = 0;
>>      CPUS390XState *env = &cpu->env;
>> -    uint8_t ar;
>> +    uint8_t ar = 0;
>>  
>> -    addr = decode_basedisp_s(env, ipb, &ar);
>> +    if (!env->pv) {
>> +        addr = decode_basedisp_s(env, ipb, &ar);
>> +    }
>>      if (addr & 3) {
>>          s390_program_interrupt(env, PGM_SPECIFICATION, ra);
>>          return;
>> @@ -198,12 +202,14 @@ void ioinst_handle_ssch(S390CPU *cpu, uint64_t reg1, uint32_t ipb, uintptr_t ra)
>>  void ioinst_handle_stcrw(S390CPU *cpu, uint32_t ipb, uintptr_t ra)
>>  {
>>      CRW crw;
>> -    uint64_t addr;
>> +    uint64_t addr = 0;
>>      int cc;
>>      CPUS390XState *env = &cpu->env;
>> -    uint8_t ar;
>> +    uint8_t ar = 0;
>>  
>> -    addr = decode_basedisp_s(env, ipb, &ar);
>> +    if (!env->pv) {
>> +        addr = decode_basedisp_s(env, ipb, &ar);
>> +    }
>>      if (addr & 3) {
>>          s390_program_interrupt(env, PGM_SPECIFICATION, ra);
>>          return;
>> @@ -228,13 +234,15 @@ void ioinst_handle_stsch(S390CPU *cpu, uint64_t reg1, uint32_t ipb,
>>  {
>>      int cssid, ssid, schid, m;
>>      SubchDev *sch;
>> -    uint64_t addr;
>> +    uint64_t addr = 0;
>>      int cc;
>>      SCHIB schib;
>>      CPUS390XState *env = &cpu->env;
>> -    uint8_t ar;
>> +    uint8_t ar = 0;
>>  
>> -    addr = decode_basedisp_s(env, ipb, &ar);
>> +    if (!env->pv) {
>> +        addr = decode_basedisp_s(env, ipb, &ar);
>> +    }
>>      if (addr & 3) {
>>          s390_program_interrupt(env, PGM_SPECIFICATION, ra);
>>          return;
>> @@ -294,16 +302,18 @@ int ioinst_handle_tsch(S390CPU *cpu, uint64_t reg1, uint32_t ipb, uintptr_t ra)
>>      int cssid, ssid, schid, m;
>>      SubchDev *sch;
>>      IRB irb;
>> -    uint64_t addr;
>> +    uint64_t addr = 0;
>>      int cc, irb_len;
>> -    uint8_t ar;
>> +    uint8_t ar = 0;
>>  
>>      if (ioinst_disassemble_sch_ident(reg1, &m, &cssid, &ssid, &schid)) {
>>          s390_program_interrupt(env, PGM_OPERAND, ra);
>>          return -EIO;
>>      }
>>      trace_ioinst_sch_id("tsch", cssid, ssid, schid);
>> -    addr = decode_basedisp_s(env, ipb, &ar);
>> +    if (!env->pv) {
>> +        addr = decode_basedisp_s(env, ipb, &ar);
>> +    }
>>      if (addr & 3) {
>>          s390_program_interrupt(env, PGM_SPECIFICATION, ra);
>>          return -EIO;
> 
> Would it make sense to hide all these changes in decode_basedisp_s()
> instead? ... so that decode_basedisp_s() returns 0 if env->pv == true ?
> ... or are there still cases where we need real values from
> decode_basedisp_s() in case of env->pv==true?

I'd like to keep decode_basedisp_s() as is, but how about a static
function in ioinst.c called something like get_address_from_regs()?

It'll call decode_basedisp_s() or return 0.


> 
> Anyway,
> Reviewed-by: Thomas Huth <thuth@redhat.com>
> 

Thanks!



[-- Attachment #2: OpenPGP digital signature --]
[-- Type: application/pgp-signature, Size: 833 bytes --]

Re: [PATCH 13/15] s390x: protvirt: Move diag 308 data over SIDAD

[David Hildenbrand]

On 28.11.19 17:08, Janosch Frank wrote:
> On 11/28/19 3:40 PM, Thomas Huth wrote:
>> On 20/11/2019 12.43, Janosch Frank wrote:
>>> For protected guests the IPIB is written/read to/from the sattelite
>>
>> satellite
>>
>>> block, so we need to make those accesses virtual to make them go
>>> through KBM mem ops.
>>
>> What's KBM ?
> 
> A totally new hypervisor that's much faster than KVM because b comes
> before v in the alphabet.


Sounds more lime IBM is slowly taking over KVM :D


-- 
Thanks,

David / dhildenb

Re: [PATCH 14/15] s390x: protvirt: Disable address checks for PV guest IO emulation

[Cornelia Huck]

[-- Attachment #1: Type: text/plain, Size: 738 bytes --]

On Thu, 28 Nov 2019 17:10:38 +0100
Janosch Frank <frankja@linux.ibm.com> wrote:

> On 11/28/19 4:28 PM, Thomas Huth wrote:

> > Would it make sense to hide all these changes in decode_basedisp_s()
> > instead? ... so that decode_basedisp_s() returns 0 if env->pv == true ?
> > ... or are there still cases where we need real values from
> > decode_basedisp_s() in case of env->pv==true?  
> 
> I'd like to keep decode_basedisp_s() as is, but how about a static
> function in ioinst.c called something like get_address_from_regs()?
> 
> It'll call decode_basedisp_s() or return 0.

We could do something like that; but do we ever get there for other
instruction formats as well? It feels a bit odd to single out this one.

[-- Attachment #2: OpenPGP digital signature --]
[-- Type: application/pgp-signature, Size: 833 bytes --]

Re: [PATCH 14/15] s390x: protvirt: Disable address checks for PV guest IO emulation

[Janosch Frank]

[-- Attachment #1.1: Type: text/plain, Size: 880 bytes --]

On 11/28/19 5:18 PM, Cornelia Huck wrote:
> On Thu, 28 Nov 2019 17:10:38 +0100
> Janosch Frank <frankja@linux.ibm.com> wrote:
> 
>> On 11/28/19 4:28 PM, Thomas Huth wrote:
> 
>>> Would it make sense to hide all these changes in decode_basedisp_s()
>>> instead? ... so that decode_basedisp_s() returns 0 if env->pv == true ?
>>> ... or are there still cases where we need real values from
>>> decode_basedisp_s() in case of env->pv==true?  
>>
>> I'd like to keep decode_basedisp_s() as is, but how about a static
>> function in ioinst.c called something like get_address_from_regs()?
>>
>> It'll call decode_basedisp_s() or return 0.
> 
> We could do something like that; but do we ever get there for other
> instruction formats as well? It feels a bit odd to single out this one.
> 

sclp is rre (register address) and diag308 is rs-a (r1 is the address).


[-- Attachment #2: OpenPGP digital signature --]
[-- Type: application/pgp-signature, Size: 833 bytes --]

Re: [PATCH 08/15] s390x: protvirt: KVM intercept changes

[Janosch Frank]

[-- Attachment #1.1: Type: text/plain, Size: 2984 bytes --]

On 11/21/19 4:11 PM, Thomas Huth wrote:
> On 20/11/2019 12.43, Janosch Frank wrote:
>> Secure guests no longer intercept with code 4 for an instruction
>> interception. Instead they have codes 104 and 108 for secure
>> instruction interception and secure instruction notification
>> respectively.
>>
>> The 104 mirrors the 4, but the 108 is a notification, that something
>> happened and the hypervisor might need to adjust its tracking data to
>> that fact. An example for that is the set prefix notification
>> interception, where KVM only reads the new prefix, but does not update
>> the prefix in the state description.
>>
>> Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
>> ---
>>  target/s390x/kvm.c | 6 ++++++
>>  1 file changed, 6 insertions(+)
>>
>> diff --git a/target/s390x/kvm.c b/target/s390x/kvm.c
>> index 418154ccfe..58251c0229 100644
>> --- a/target/s390x/kvm.c
>> +++ b/target/s390x/kvm.c
>> @@ -115,6 +115,8 @@
>>  #define ICPT_CPU_STOP                   0x28
>>  #define ICPT_OPEREXC                    0x2c
>>  #define ICPT_IO                         0x40
>> +#define ICPT_PV_INSTR                   0x68
>> +#define ICPT_PV_INSTR_NOT               0x6c
>>  
>>  #define NR_LOCAL_IRQS 32
>>  /*
>> @@ -151,6 +153,7 @@ static int cap_s390_irq;
>>  static int cap_ri;
>>  static int cap_gs;
>>  static int cap_hpage_1m;
>> +static int cap_protvirt;
>>  
>>  static int active_cmma;
>>  
>> @@ -336,6 +339,7 @@ int kvm_arch_init(MachineState *ms, KVMState *s)
>>      cap_async_pf = kvm_check_extension(s, KVM_CAP_ASYNC_PF);
>>      cap_mem_op = kvm_check_extension(s, KVM_CAP_S390_MEM_OP);
>>      cap_s390_irq = kvm_check_extension(s, KVM_CAP_S390_INJECT_IRQ);
>> +    cap_protvirt = kvm_check_extension(s, KVM_CAP_S390_PROTECTED);
>>  
>>      if (!kvm_check_extension(s, KVM_CAP_S390_GMAP)
>>          || !kvm_check_extension(s, KVM_CAP_S390_COW)) {
>> @@ -1664,6 +1668,8 @@ static int handle_intercept(S390CPU *cpu)
>>              (long)cs->kvm_run->psw_addr);
>>      switch (icpt_code) {
>>          case ICPT_INSTRUCTION:
>> +        case ICPT_PV_INSTR:
>> +        case ICPT_PV_INSTR_NOT:
>>              r = handle_instruction(cpu, run);
> 
> Even if this works by default, my gut feeling tells me that it would be
> safer and cleaner to have a separate handler for this...
> Otherwise we might get surprising results if future machine generations
> intercept/notify for more or different instructions, I guess?
> 
> However, it's just a gut feeling ... I really don't have much experience
> with this PV stuff yet ... what do the others here think?
> 
>  Thomas


Adding a handle_instruction_pv doesn't hurt me too much.
The default case can then do an error_report() and exit(1);

PV was designed in a way that we can re-use as much code as possible, so
I tried using the normal instruction handlers and only change as little
as possible in the instructions themselves.



[-- Attachment #2: OpenPGP digital signature --]
[-- Type: application/pgp-signature, Size: 833 bytes --]

Re: [PATCH 08/15] s390x: protvirt: KVM intercept changes

[Cornelia Huck]

[-- Attachment #1: Type: text/plain, Size: 3660 bytes --]

On Thu, 28 Nov 2019 17:38:19 +0100
Janosch Frank <frankja@linux.ibm.com> wrote:

> On 11/21/19 4:11 PM, Thomas Huth wrote:
> > On 20/11/2019 12.43, Janosch Frank wrote:  
> >> Secure guests no longer intercept with code 4 for an instruction
> >> interception. Instead they have codes 104 and 108 for secure
> >> instruction interception and secure instruction notification
> >> respectively.
> >>
> >> The 104 mirrors the 4, but the 108 is a notification, that something
> >> happened and the hypervisor might need to adjust its tracking data to
> >> that fact. An example for that is the set prefix notification
> >> interception, where KVM only reads the new prefix, but does not update
> >> the prefix in the state description.
> >>
> >> Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
> >> ---
> >>  target/s390x/kvm.c | 6 ++++++
> >>  1 file changed, 6 insertions(+)
> >>
> >> diff --git a/target/s390x/kvm.c b/target/s390x/kvm.c
> >> index 418154ccfe..58251c0229 100644
> >> --- a/target/s390x/kvm.c
> >> +++ b/target/s390x/kvm.c
> >> @@ -115,6 +115,8 @@
> >>  #define ICPT_CPU_STOP                   0x28
> >>  #define ICPT_OPEREXC                    0x2c
> >>  #define ICPT_IO                         0x40
> >> +#define ICPT_PV_INSTR                   0x68
> >> +#define ICPT_PV_INSTR_NOT               0x6c
> >>  
> >>  #define NR_LOCAL_IRQS 32
> >>  /*
> >> @@ -151,6 +153,7 @@ static int cap_s390_irq;
> >>  static int cap_ri;
> >>  static int cap_gs;
> >>  static int cap_hpage_1m;
> >> +static int cap_protvirt;
> >>  
> >>  static int active_cmma;
> >>  
> >> @@ -336,6 +339,7 @@ int kvm_arch_init(MachineState *ms, KVMState *s)
> >>      cap_async_pf = kvm_check_extension(s, KVM_CAP_ASYNC_PF);
> >>      cap_mem_op = kvm_check_extension(s, KVM_CAP_S390_MEM_OP);
> >>      cap_s390_irq = kvm_check_extension(s, KVM_CAP_S390_INJECT_IRQ);
> >> +    cap_protvirt = kvm_check_extension(s, KVM_CAP_S390_PROTECTED);
> >>  
> >>      if (!kvm_check_extension(s, KVM_CAP_S390_GMAP)
> >>          || !kvm_check_extension(s, KVM_CAP_S390_COW)) {
> >> @@ -1664,6 +1668,8 @@ static int handle_intercept(S390CPU *cpu)
> >>              (long)cs->kvm_run->psw_addr);
> >>      switch (icpt_code) {
> >>          case ICPT_INSTRUCTION:
> >> +        case ICPT_PV_INSTR:
> >> +        case ICPT_PV_INSTR_NOT:
> >>              r = handle_instruction(cpu, run);  
> > 
> > Even if this works by default, my gut feeling tells me that it would be
> > safer and cleaner to have a separate handler for this...
> > Otherwise we might get surprising results if future machine generations
> > intercept/notify for more or different instructions, I guess?
> > 
> > However, it's just a gut feeling ... I really don't have much experience
> > with this PV stuff yet ... what do the others here think?
> > 
> >  Thomas  
> 
> 
> Adding a handle_instruction_pv doesn't hurt me too much.
> The default case can then do an error_report() and exit(1);
> 
> PV was designed in a way that we can re-use as much code as possible, so
> I tried using the normal instruction handlers and only change as little
> as possible in the instructions themselves.

I think we could argue that handling 4 and 104 in the same function
makes sense; but the 108 notification should really be separate, I
think. From what I've seen, the expectation of what the hypervisor
needs to do is just something else in this case ("hey, I did something;
just to let you know").

Is the set of instructions you get a 104 for always supposed to be a
subset of the instructions you get a 4 for? I'd expect it to be so.

[-- Attachment #2: OpenPGP digital signature --]
[-- Type: application/pgp-signature, Size: 833 bytes --]

Re: [PATCH 08/15] s390x: protvirt: KVM intercept changes

[Janosch Frank]

[-- Attachment #1.1: Type: text/plain, Size: 4314 bytes --]

On 11/28/19 5:45 PM, Cornelia Huck wrote:
> On Thu, 28 Nov 2019 17:38:19 +0100
> Janosch Frank <frankja@linux.ibm.com> wrote:
> 
>> On 11/21/19 4:11 PM, Thomas Huth wrote:
>>> On 20/11/2019 12.43, Janosch Frank wrote:  
>>>> Secure guests no longer intercept with code 4 for an instruction
>>>> interception. Instead they have codes 104 and 108 for secure
>>>> instruction interception and secure instruction notification
>>>> respectively.
>>>>
>>>> The 104 mirrors the 4, but the 108 is a notification, that something
>>>> happened and the hypervisor might need to adjust its tracking data to
>>>> that fact. An example for that is the set prefix notification
>>>> interception, where KVM only reads the new prefix, but does not update
>>>> the prefix in the state description.
>>>>
>>>> Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
>>>> ---
>>>>  target/s390x/kvm.c | 6 ++++++
>>>>  1 file changed, 6 insertions(+)
>>>>
>>>> diff --git a/target/s390x/kvm.c b/target/s390x/kvm.c
>>>> index 418154ccfe..58251c0229 100644
>>>> --- a/target/s390x/kvm.c
>>>> +++ b/target/s390x/kvm.c
>>>> @@ -115,6 +115,8 @@
>>>>  #define ICPT_CPU_STOP                   0x28
>>>>  #define ICPT_OPEREXC                    0x2c
>>>>  #define ICPT_IO                         0x40
>>>> +#define ICPT_PV_INSTR                   0x68
>>>> +#define ICPT_PV_INSTR_NOT               0x6c
>>>>  
>>>>  #define NR_LOCAL_IRQS 32
>>>>  /*
>>>> @@ -151,6 +153,7 @@ static int cap_s390_irq;
>>>>  static int cap_ri;
>>>>  static int cap_gs;
>>>>  static int cap_hpage_1m;
>>>> +static int cap_protvirt;
>>>>  
>>>>  static int active_cmma;
>>>>  
>>>> @@ -336,6 +339,7 @@ int kvm_arch_init(MachineState *ms, KVMState *s)
>>>>      cap_async_pf = kvm_check_extension(s, KVM_CAP_ASYNC_PF);
>>>>      cap_mem_op = kvm_check_extension(s, KVM_CAP_S390_MEM_OP);
>>>>      cap_s390_irq = kvm_check_extension(s, KVM_CAP_S390_INJECT_IRQ);
>>>> +    cap_protvirt = kvm_check_extension(s, KVM_CAP_S390_PROTECTED);
>>>>  
>>>>      if (!kvm_check_extension(s, KVM_CAP_S390_GMAP)
>>>>          || !kvm_check_extension(s, KVM_CAP_S390_COW)) {
>>>> @@ -1664,6 +1668,8 @@ static int handle_intercept(S390CPU *cpu)
>>>>              (long)cs->kvm_run->psw_addr);
>>>>      switch (icpt_code) {
>>>>          case ICPT_INSTRUCTION:
>>>> +        case ICPT_PV_INSTR:
>>>> +        case ICPT_PV_INSTR_NOT:
>>>>              r = handle_instruction(cpu, run);  
>>>
>>> Even if this works by default, my gut feeling tells me that it would be
>>> safer and cleaner to have a separate handler for this...
>>> Otherwise we might get surprising results if future machine generations
>>> intercept/notify for more or different instructions, I guess?
>>>
>>> However, it's just a gut feeling ... I really don't have much experience
>>> with this PV stuff yet ... what do the others here think?
>>>
>>>  Thomas  
>>
>>
>> Adding a handle_instruction_pv doesn't hurt me too much.
>> The default case can then do an error_report() and exit(1);
>>
>> PV was designed in a way that we can re-use as much code as possible, so
>> I tried using the normal instruction handlers and only change as little
>> as possible in the instructions themselves.
> 
> I think we could argue that handling 4 and 104 in the same function
> makes sense; but the 108 notification should really be separate, I

In my latest answer to Thomas I stated that we could move to a separate
pv instruction handler. I just had another look and rediscovered, that
it would mean a lot more changes. I would need to duplicate the ipa/b
parsing and for diagnose even the base+disp parsing.

So yes, I'd like to treat the 104 like the 4 intercept...

> think. From what I've seen, the expectation of what the hypervisor
> needs to do is just something else in this case ("hey, I did something;
> just to let you know").

We can remove the notification from QEMU, as far is I know, we moved the
instruction that used this path to a 104 code.

> 
> Is the set of instructions you get a 104 for always supposed to be a
> subset of the instructions you get a 4 for? I'd expect it to be so.
> 

Yes
I'll ask if we'll get a new code for instructions that are only valid in
PV mode; currently there are none.


[-- Attachment #2: OpenPGP digital signature --]
[-- Type: application/pgp-signature, Size: 833 bytes --]

Re: [PATCH 14/15] s390x: protvirt: Disable address checks for PV guest IO emulation

[Thomas Huth]

[-- Attachment #1.1: Type: text/plain, Size: 4442 bytes --]

On 28/11/2019 17.10, Janosch Frank wrote:
> On 11/28/19 4:28 PM, Thomas Huth wrote:
>> On 20/11/2019 12.43, Janosch Frank wrote:
>>> IO instruction data is routed through SIDAD for protected guests, so
>>> adresses do not need to be checked, as this is kernel memory.
>>>
>>> Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
>>> ---
>>>  target/s390x/ioinst.c | 46 +++++++++++++++++++++++++++----------------
>>>  1 file changed, 29 insertions(+), 17 deletions(-)
>>>
>>> diff --git a/target/s390x/ioinst.c b/target/s390x/ioinst.c
>>> index c437a1d8c6..d3bd422ddd 100644
>>> --- a/target/s390x/ioinst.c
>>> +++ b/target/s390x/ioinst.c
>>> @@ -110,11 +110,13 @@ void ioinst_handle_msch(S390CPU *cpu, uint64_t reg1, uint32_t ipb, uintptr_t ra)
>>>      int cssid, ssid, schid, m;
>>>      SubchDev *sch;
>>>      SCHIB schib;
>>> -    uint64_t addr;
>>> +    uint64_t addr = 0;
>>>      CPUS390XState *env = &cpu->env;
>>> -    uint8_t ar;
>>> +    uint8_t ar = 0;
>>>  
>>> -    addr = decode_basedisp_s(env, ipb, &ar);
>>> +    if (!env->pv) {
>>> +        addr = decode_basedisp_s(env, ipb, &ar);
>>> +    }
>>>      if (addr & 3) {
>>>          s390_program_interrupt(env, PGM_SPECIFICATION, ra);
>>>          return;
>>> @@ -167,11 +169,13 @@ void ioinst_handle_ssch(S390CPU *cpu, uint64_t reg1, uint32_t ipb, uintptr_t ra)
>>>      int cssid, ssid, schid, m;
>>>      SubchDev *sch;
>>>      ORB orig_orb, orb;
>>> -    uint64_t addr;
>>> +    uint64_t addr = 0;
>>>      CPUS390XState *env = &cpu->env;
>>> -    uint8_t ar;
>>> +    uint8_t ar = 0;
>>>  
>>> -    addr = decode_basedisp_s(env, ipb, &ar);
>>> +    if (!env->pv) {
>>> +        addr = decode_basedisp_s(env, ipb, &ar);
>>> +    }
>>>      if (addr & 3) {
>>>          s390_program_interrupt(env, PGM_SPECIFICATION, ra);
>>>          return;
>>> @@ -198,12 +202,14 @@ void ioinst_handle_ssch(S390CPU *cpu, uint64_t reg1, uint32_t ipb, uintptr_t ra)
>>>  void ioinst_handle_stcrw(S390CPU *cpu, uint32_t ipb, uintptr_t ra)
>>>  {
>>>      CRW crw;
>>> -    uint64_t addr;
>>> +    uint64_t addr = 0;
>>>      int cc;
>>>      CPUS390XState *env = &cpu->env;
>>> -    uint8_t ar;
>>> +    uint8_t ar = 0;
>>>  
>>> -    addr = decode_basedisp_s(env, ipb, &ar);
>>> +    if (!env->pv) {
>>> +        addr = decode_basedisp_s(env, ipb, &ar);
>>> +    }
>>>      if (addr & 3) {
>>>          s390_program_interrupt(env, PGM_SPECIFICATION, ra);
>>>          return;
>>> @@ -228,13 +234,15 @@ void ioinst_handle_stsch(S390CPU *cpu, uint64_t reg1, uint32_t ipb,
>>>  {
>>>      int cssid, ssid, schid, m;
>>>      SubchDev *sch;
>>> -    uint64_t addr;
>>> +    uint64_t addr = 0;
>>>      int cc;
>>>      SCHIB schib;
>>>      CPUS390XState *env = &cpu->env;
>>> -    uint8_t ar;
>>> +    uint8_t ar = 0;
>>>  
>>> -    addr = decode_basedisp_s(env, ipb, &ar);
>>> +    if (!env->pv) {
>>> +        addr = decode_basedisp_s(env, ipb, &ar);
>>> +    }
>>>      if (addr & 3) {
>>>          s390_program_interrupt(env, PGM_SPECIFICATION, ra);
>>>          return;
>>> @@ -294,16 +302,18 @@ int ioinst_handle_tsch(S390CPU *cpu, uint64_t reg1, uint32_t ipb, uintptr_t ra)
>>>      int cssid, ssid, schid, m;
>>>      SubchDev *sch;
>>>      IRB irb;
>>> -    uint64_t addr;
>>> +    uint64_t addr = 0;
>>>      int cc, irb_len;
>>> -    uint8_t ar;
>>> +    uint8_t ar = 0;
>>>  
>>>      if (ioinst_disassemble_sch_ident(reg1, &m, &cssid, &ssid, &schid)) {
>>>          s390_program_interrupt(env, PGM_OPERAND, ra);
>>>          return -EIO;
>>>      }
>>>      trace_ioinst_sch_id("tsch", cssid, ssid, schid);
>>> -    addr = decode_basedisp_s(env, ipb, &ar);
>>> +    if (!env->pv) {
>>> +        addr = decode_basedisp_s(env, ipb, &ar);
>>> +    }
>>>      if (addr & 3) {
>>>          s390_program_interrupt(env, PGM_SPECIFICATION, ra);
>>>          return -EIO;
>>
>> Would it make sense to hide all these changes in decode_basedisp_s()
>> instead? ... so that decode_basedisp_s() returns 0 if env->pv == true ?
>> ... or are there still cases where we need real values from
>> decode_basedisp_s() in case of env->pv==true?
> 
> I'd like to keep decode_basedisp_s() as is, but how about a static
> function in ioinst.c called something like get_address_from_regs()?
> 
> It'll call decode_basedisp_s() or return 0.

Sounds fine for me, too!

 Thomas


[-- Attachment #2: OpenPGP digital signature --]
[-- Type: application/pgp-signature, Size: 833 bytes --]

Re: [PATCH 00/15] s390x: Protected Virtualization support

[Daniel P. Berrangé]

On Wed, Nov 20, 2019 at 06:43:19AM -0500, Janosch Frank wrote:
> Most of the QEMU changes for PV are related to the new IPL type with
> subcodes 8 - 10 and the execution of the necessary Ultravisor calls to
> IPL secure guests. Note that we can only boot into secure mode from
> normal mode, i.e. stfle 161 is not active in secure mode.
> 
> The other changes related to data gathering for emulation and
> disabling addressing checks in secure mode, as well as CPU resets.
> 
> While working on this I sprinkled in some cleanups, as we sometimes
> significantly increase line count of some functions and they got
> unreadable.

Can you give some guidance on how management applications including
libvirt & layers above (oVirt, OpenStack, etc) would/should use this
feature ?  What new command line / monitor calls are needed, and
what feature restrictions are there on its use ?

Regards,
Daniel
-- 
|: https://berrange.com      -o-    https://www.flickr.com/photos/dberrange :|
|: https://libvirt.org         -o-            https://fstop138.berrange.com :|
|: https://entangle-photo.org    -o-    https://www.instagram.com/dberrange :|

Re: [PATCH 00/15] s390x: Protected Virtualization support

[Janosch Frank]

[-- Attachment #1.1: Type: text/plain, Size: 2057 bytes --]

On 11/29/19 12:08 PM, Daniel P. Berrangé wrote:
> On Wed, Nov 20, 2019 at 06:43:19AM -0500, Janosch Frank wrote:
>> Most of the QEMU changes for PV are related to the new IPL type with
>> subcodes 8 - 10 and the execution of the necessary Ultravisor calls to
>> IPL secure guests. Note that we can only boot into secure mode from
>> normal mode, i.e. stfle 161 is not active in secure mode.
>>
>> The other changes related to data gathering for emulation and
>> disabling addressing checks in secure mode, as well as CPU resets.
>>
>> While working on this I sprinkled in some cleanups, as we sometimes
>> significantly increase line count of some functions and they got
>> unreadable.
> 
> Can you give some guidance on how management applications including
> libvirt & layers above (oVirt, OpenStack, etc) would/should use this
> feature ?  What new command line / monitor calls are needed, and
> what feature restrictions are there on its use ?
> 
> Regards,
> Daniel
> 

Hey Daniel,

management applications generally do not need to know about this
feature. Most of the magic is in the guest image, which boots up in a
certain way to become a protected machine.

The requirements for that to happen are:
* Machine/firmware support
* KVM & QEMU support
* IO only with iommu
* Guest needs to use IO bounce buffers
* A kernel image or a kernel on a disk that was prepared with special
tooling

Such VMs are started like any other VM and run a short "normal" stub
that will prepare some things and then requests to be protected.

Most of the restrictions are memory related and might be lifted in the
future:
* No paging
* No migration
* No huge page backings
* No collaborative memory management


There are no monitor changes or cmd additions currently.
We're trying to insert protected VMs into the normal VM flow as much as
possible. You can even do a memory dump without any segfault or
protection exception for QEMU, however the guest's memory content will
be unreadable because it's encrypted.


[-- Attachment #2: OpenPGP digital signature --]
[-- Type: application/pgp-signature, Size: 833 bytes --]

Re: [PATCH 00/15] s390x: Protected Virtualization support

[Daniel P. Berrangé]

On Fri, Nov 29, 2019 at 01:14:27PM +0100, Janosch Frank wrote:
> On 11/29/19 12:08 PM, Daniel P. Berrangé wrote:
> > On Wed, Nov 20, 2019 at 06:43:19AM -0500, Janosch Frank wrote:
> >> Most of the QEMU changes for PV are related to the new IPL type with
> >> subcodes 8 - 10 and the execution of the necessary Ultravisor calls to
> >> IPL secure guests. Note that we can only boot into secure mode from
> >> normal mode, i.e. stfle 161 is not active in secure mode.
> >>
> >> The other changes related to data gathering for emulation and
> >> disabling addressing checks in secure mode, as well as CPU resets.
> >>
> >> While working on this I sprinkled in some cleanups, as we sometimes
> >> significantly increase line count of some functions and they got
> >> unreadable.
> > 
> > Can you give some guidance on how management applications including
> > libvirt & layers above (oVirt, OpenStack, etc) would/should use this
> > feature ?  What new command line / monitor calls are needed, and
> > what feature restrictions are there on its use ?
> 
> management applications generally do not need to know about this
> feature. Most of the magic is in the guest image, which boots up in a
> certain way to become a protected machine.
> 
> The requirements for that to happen are:
> * Machine/firmware support
> * KVM & QEMU support
> * IO only with iommu
> * Guest needs to use IO bounce buffers
> * A kernel image or a kernel on a disk that was prepared with special
> tooling

If the user has a guest image that's expecting to run in protected
machine mode, presumably this will fail to boot if run on a host
which doesn't support this feature ?

As a mgmt app I think there will be a need to be able to determine
whether a host + QEMU combo is actually able to support protected
machines. If the mgmt app is given an image and the users says it
required protected mode, then the mgmt app needs to know which
host(s) are able to run it.

Doing version number checks is not particularly desirable, so is
there a way libvirt can determine if QEMU + host in general supports
protected machines, so that we can report this feature to mgmt apps ?


If a guest has booted & activated protected mode is there any way
for libvirt to query that status ? This would allow the mgmt app
to know that the guest is not going to be migratable thereafter.

Is there any way to prevent a guest from using protected mode even
if QEMU supports it ?  eg the mgmt app may want to be able to
guarantee that all VMs are migratable, so don't want a guest OS
secretly activating protected mode which blocks migration.

> Such VMs are started like any other VM and run a short "normal" stub
> that will prepare some things and then requests to be protected.
> 
> Most of the restrictions are memory related and might be lifted in the
> future:
> * No paging
> * No migration

Presumably QEMU is going to set a migration blocker when a guest
activates protected mode ?

> * No huge page backings
> * No collaborative memory management

> There are no monitor changes or cmd additions currently.
> We're trying to insert protected VMs into the normal VM flow as much as
> possible. You can even do a memory dump without any segfault or
> protection exception for QEMU, however the guest's memory content will
> be unreadable because it's encrypted.

Is there any way to securely acquire a key needed to interpret this,
or is the memory dump completely useless ?

Regards,
Daniel
-- 
|: https://berrange.com      -o-    https://www.flickr.com/photos/dberrange :|
|: https://libvirt.org         -o-            https://fstop138.berrange.com :|
|: https://entangle-photo.org    -o-    https://www.instagram.com/dberrange :|

Re: [PATCH 00/15] s390x: Protected Virtualization support

[Janosch Frank]

[-- Attachment #1.1: Type: text/plain, Size: 4168 bytes --]

On 11/29/19 1:35 PM, Daniel P. Berrangé wrote:
> On Fri, Nov 29, 2019 at 01:14:27PM +0100, Janosch Frank wrote:
>> On 11/29/19 12:08 PM, Daniel P. Berrangé wrote:
>>> On Wed, Nov 20, 2019 at 06:43:19AM -0500, Janosch Frank wrote:
>>>> Most of the QEMU changes for PV are related to the new IPL type with
>>>> subcodes 8 - 10 and the execution of the necessary Ultravisor calls to
>>>> IPL secure guests. Note that we can only boot into secure mode from
>>>> normal mode, i.e. stfle 161 is not active in secure mode.
>>>>
>>>> The other changes related to data gathering for emulation and
>>>> disabling addressing checks in secure mode, as well as CPU resets.
>>>>
>>>> While working on this I sprinkled in some cleanups, as we sometimes
>>>> significantly increase line count of some functions and they got
>>>> unreadable.
>>>
>>> Can you give some guidance on how management applications including
>>> libvirt & layers above (oVirt, OpenStack, etc) would/should use this
>>> feature ?  What new command line / monitor calls are needed, and
>>> what feature restrictions are there on its use ?
>>
>> management applications generally do not need to know about this
>> feature. Most of the magic is in the guest image, which boots up in a
>> certain way to become a protected machine.
>>
>> The requirements for that to happen are:
>> * Machine/firmware support
>> * KVM & QEMU support
>> * IO only with iommu
>> * Guest needs to use IO bounce buffers
>> * A kernel image or a kernel on a disk that was prepared with special
>> tooling
> 
> If the user has a guest image that's expecting to run in protected
> machine mode, presumably this will fail to boot if run on a host
> which doesn't support this feature ?

Yes, the guest will lack stfle facility 161 and KVM will report a
specification exception on the diagnose subcode 8 - 10.

> 
> As a mgmt app I think there will be a need to be able to determine
> whether a host + QEMU combo is actually able to support protected
> machines. If the mgmt app is given an image and the users says it
> required protected mode, then the mgmt app needs to know which
> host(s) are able to run it.
> 
> Doing version number checks is not particularly desirable, so is
> there a way libvirt can determine if QEMU + host in general supports
> protected machines, so that we can report this feature to mgmt apps ?

I thought that would be visible via the cpu model by checking for the
unpack facility (161)?
Time for somebody else to explain that.


@Viktor @Boris: This one's for you.

> 
> 
> If a guest has booted & activated protected mode is there any way
> for libvirt to query that status ? This would allow the mgmt app
> to know that the guest is not going to be migratable thereafter.

Currently not

> 
> Is there any way to prevent a guest from using protected mode even
> if QEMU supports it ?  eg the mgmt app may want to be able to
> guarantee that all VMs are migratable, so don't want a guest OS
> secretly activating protected mode which blocks migration.

Not enabling facility 161 is enough.

> 
>> Such VMs are started like any other VM and run a short "normal" stub
>> that will prepare some things and then requests to be protected.
>>
>> Most of the restrictions are memory related and might be lifted in the
>> future:
>> * No paging
>> * No migration
> 
> Presumably QEMU is going to set a migration blocker when a guest
> activates protected mode ?

Well, that's stuff I still need to figure out :)

> 
>> * No huge page backings
>> * No collaborative memory management
> 
>> There are no monitor changes or cmd additions currently.
>> We're trying to insert protected VMs into the normal VM flow as much as
>> possible. You can even do a memory dump without any segfault or
>> protection exception for QEMU, however the guest's memory content will
>> be unreadable because it's encrypted.
> 
> Is there any way to securely acquire a key needed to interpret this,
> or is the memory dump completely useless ?

It's part of the design, but not yet implemented.

> 
> Regards,
> Daniel
> 



[-- Attachment #2: OpenPGP digital signature --]
[-- Type: application/pgp-signature, Size: 833 bytes --]

Re: [PATCH 00/15] s390x: Protected Virtualization support

[Viktor Mihajlovski]

On 11/29/19 3:02 PM, Janosch Frank wrote:
[...]
>>
>> As a mgmt app I think there will be a need to be able to determine
>> whether a host + QEMU combo is actually able to support protected
>> machines. If the mgmt app is given an image and the users says it
>> required protected mode, then the mgmt app needs to know which
>> host(s) are able to run it.
>>
>> Doing version number checks is not particularly desirable, so is
>> there a way libvirt can determine if QEMU + host in general supports
>> protected machines, so that we can report this feature to mgmt apps ?
> 
> I thought that would be visible via the cpu model by checking for the
> unpack facility (161)?
> Time for somebody else to explain that.
> 
> 
> @Viktor @Boris: This one's for you.
> 
Right, a management app could check the supported CPU model, with 
something like virsh domcapabilities. The domain's CPU model would have 
to require the 'unpack' facility. So, in theory any management app 
establishing CPU model compatibility using the libvirt APIs should be 
able to find appropriate hosts.
[...]
-- 
Kind Regards,
    Viktor

Re: [PATCH 00/15] s390x: Protected Virtualization support

[Daniel P. Berrangé]

On Fri, Nov 29, 2019 at 03:02:41PM +0100, Janosch Frank wrote:
> On 11/29/19 1:35 PM, Daniel P. Berrangé wrote:

> > Is there any way to prevent a guest from using protected mode even
> > if QEMU supports it ?  eg the mgmt app may want to be able to
> > guarantee that all VMs are migratable, so don't want a guest OS
> > secretly activating protected mode which blocks migration.
> 
> Not enabling facility 161 is enough.

Is this facility enabled by default in any scenario ?

What happens if the feature is enabled & QEMU is also
coinfigured to use huge pages or does not have memory
pinned into RAM, given that those features are said to
be incompatible ?

> 
> > 
> >> Such VMs are started like any other VM and run a short "normal" stub
> >> that will prepare some things and then requests to be protected.
> >>
> >> Most of the restrictions are memory related and might be lifted in the
> >> future:
> >> * No paging
> >> * No migration
> > 
> > Presumably QEMU is going to set a migration blocker when a guest
> > activates protected mode ?
> 
> Well, that's stuff I still need to figure out :)
> 
> > 
> >> * No huge page backings
> >> * No collaborative memory management

Regards,
Daniel
-- 
|: https://berrange.com      -o-    https://www.flickr.com/photos/dberrange :|
|: https://libvirt.org         -o-            https://fstop138.berrange.com :|
|: https://entangle-photo.org    -o-    https://www.instagram.com/dberrange :|

Re: [PATCH 03/15] s390x: protvirt: Add diag308 subcodes 8 - 10

[Janosch Frank]

[-- Attachment #1.1: Type: text/plain, Size: 5900 bytes --]

On 11/21/19 3:36 PM, Thomas Huth wrote:
> On 20/11/2019 12.43, Janosch Frank wrote:
>> For diag308 subcodes 8 - 10 we have a new ipib of type 5. The ipib
>> holds the address and length of the secure execution header, as well
>> as a list of guest components.
>>
>> Each component is a block of memory, for example kernel or initrd,
>> which needs to be decrypted by the Ultravisor in order to run a
>> protected VM. The secure execution header instructs the Ultravisor on
>> how to handle the protected VM and its components.
>>
>> Subcodes 8 and 9 are similiar to 5 and 6 and subcode 10 will finally
>> start the protected guest.
>>
>> Subcodes 8-10 are not valid in protected mode, we have to do a subcode
>> 3 and then the 8 and 10 combination for a protected reboot.
>>
>> Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
>> ---
> [...]
>> diff --git a/hw/s390x/ipl.h b/hw/s390x/ipl.h
>> index d4813105db..7b8a493509 100644
>> --- a/hw/s390x/ipl.h
>> +++ b/hw/s390x/ipl.h
>> @@ -15,6 +15,24 @@
>>  #include "cpu.h"
>>  #include "hw/qdev-core.h"
>>  
>> +struct IPLBlockPVComp {
>> +    uint64_t tweak_pref;
>> +    uint64_t addr;
>> +    uint64_t size;
>> +} QEMU_PACKED;
>> +typedef struct IPLBlockPVComp IPLBlockPVComp;
>> +
>> +struct IPLBlockPV {
>> +    uint8_t  reserved[84];
>> +    uint8_t  reserved67[3];
> 
> What does the "67" mean here?

Was compacted to reserved[87]

> 
>> +    uint8_t  version;
>> +    uint32_t num_comp;
>> +    uint64_t pv_header_addr;
>> +    uint64_t pv_header_len;
>> +    struct IPLBlockPVComp components[];
>> +} QEMU_PACKED;
>> +typedef struct IPLBlockPV IPLBlockPV;
> 
> Given the fact that we had quite some headaches with QEMU_PACKED structs
> in the past already, and the structs seem to be naturally aligned ...
> what about dropping the QEMU_PACKED here and using QEMU_BUILD_BUG() to
> assert that the struct has the correct size?

As the struct in which IPLBlockPV is embeded are also packed, removing
the PACKED annotation results in this:

/mnt/dev/qemu/hw/s390x/ipl.c:544:27: error: taking address of packed
member of ‘struct <anonymous>’ may result in an unaligned pointer value
[-Werror=address-of-packed-member]


> 
> [...]
>> @@ -185,4 +211,11 @@ static inline bool iplb_valid_fcp(IplParameterBlock *iplb)
>>             iplb->pbt == S390_IPL_TYPE_FCP;
>>  }
>>  
>> +static inline bool iplb_valid_se(IplParameterBlock *iplb)
>> +{
>> +    return be32_to_cpu(iplb->len) >= S390_IPLB_MIN_PV_LEN &&
>> +           iplb->pbt == S390_IPL_TYPE_PV;
>> +}
>> +
>> +
> 
> Drop one empty line?
ack.

> 
>>  #endif
>> diff --git a/target/s390x/diag.c b/target/s390x/diag.c
>> index 067c667ba7..32049bb4ee 100644
>> --- a/target/s390x/diag.c
>> +++ b/target/s390x/diag.c
>> @@ -52,6 +52,8 @@ int handle_diag_288(CPUS390XState *env, uint64_t r1, uint64_t r3)
>>  #define DIAG_308_RC_OK              0x0001
>>  #define DIAG_308_RC_NO_CONF         0x0102
>>  #define DIAG_308_RC_INVALID         0x0402
>> +#define DIAG_308_RC_NO_PV_CONF      0x0a02
>> +#define DIAG_308_RC_INV_FOR_PV      0x0b02
>>  
>>  #define DIAG308_RES_MOD_CLR		0
>>  #define DIAG308_RES_LOAD_NORM		1
>> @@ -59,6 +61,9 @@ int handle_diag_288(CPUS390XState *env, uint64_t r1, uint64_t r3)
>>  #define DIAG308_LOAD_NORMAL_DUMP	4
>>  #define DIAG308_SET			5
>>  #define DIAG308_STORE			6
>> +#define DIAG308_PV_SET			8
>> +#define DIAG308_PV_STORE		9
>> +#define DIAG308_PV_START		10
>>  
>>  static int diag308_parm_check(CPUS390XState *env, uint64_t r1, uint64_t addr,
>>                                uintptr_t ra, bool write)
>> @@ -105,6 +110,7 @@ void handle_diag_308(CPUS390XState *env, uint64_t r1, uint64_t r3, uintptr_t ra)
>>          s390_ipl_reset_request(cs, S390_RESET_REIPL);
>>          break;
>>      case DIAG308_SET:
>> +    case DIAG308_PV_SET: /* Set SE parms */
>>          if (diag308_parm_check(env, r1, addr, ra, false)) {
>>              return;
>>          }
>> @@ -117,7 +123,8 @@ void handle_diag_308(CPUS390XState *env, uint64_t r1, uint64_t r3, uintptr_t ra)
>>  
>>          cpu_physical_memory_read(addr, iplb, be32_to_cpu(iplb->len));
>>  
>> -        if (!iplb_valid_ccw(iplb) && !iplb_valid_fcp(iplb)) {
>> +        if (!iplb_valid_ccw(iplb) && !iplb_valid_fcp(iplb) &&
>> +            !(iplb_valid_se(iplb) && s390_ipl_pv_check_comp(iplb) >= 0)) {
>>              env->regs[r1 + 1] = DIAG_308_RC_INVALID;
>>              goto out;
>>          }
>> @@ -128,10 +135,15 @@ out:
>>          g_free(iplb);
>>          return;
>>      case DIAG308_STORE:
>> +    case DIAG308_PV_STORE: /* Get SE parms */
>>          if (diag308_parm_check(env, r1, addr, ra, true)) {
>>              return;
>>          }
>> -        iplb = s390_ipl_get_iplb();
>> +        if (subcode == DIAG308_PV_STORE) {
>> +            iplb = s390_ipl_get_iplb_secure();
>> +        } else {
>> +            iplb = s390_ipl_get_iplb();
>> +        }
>>          if (iplb) {
>>              cpu_physical_memory_write(addr, iplb, be32_to_cpu(iplb->len));
>>              env->regs[r1 + 1] = DIAG_308_RC_OK;
>> @@ -139,6 +151,16 @@ out:
>>              env->regs[r1 + 1] = DIAG_308_RC_NO_CONF;
>>          }
>>          return;
>> +        break;
> 
> <hamletmode>
> To return or to break, that's the question...
> </hamletmode>
> 
> ... please choose one of the two.
> 
>> +    case DIAG308_PV_START: /* SE start */
>> +        iplb = s390_ipl_get_iplb_secure();
>> +        if (!iplb_valid_se(iplb)) {
>> +            env->regs[r1 + 1] = DIAG_308_RC_NO_PV_CONF;
>> +            return;
>> +        }
>> +
>> +        s390_ipl_reset_request(cs, S390_RESET_PV);
>> +        break;
>>      default:
>>          s390_program_interrupt(env, PGM_SPECIFICATION, ra);
>>          break;
>>
> 
>  Thomas
> 
> 



[-- Attachment #2: OpenPGP digital signature --]
[-- Type: application/pgp-signature, Size: 833 bytes --]