[PATCH v4] KVM: s390: Add new reset vcpu API

[PATCH v4] KVM: s390: Add new reset vcpu API

[Janosch Frank]

The architecture states that we need to reset local IRQs for all CPU
resets. Because the old reset interface did not support the normal CPU
reset we never did that on a normal reset.

Let's implement an interface for the missing normal and clear resets
and reset all local IRQs, registers and control structures as stated
in the architecture.

Userspace might already reset the registers via the vcpu run struct,
but as we need the interface for the interrupt clearing part anyway,
we implement the resets fully and don't rely on userspace to reset the
rest.

Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
---

I dropped the reviews, as I changed quite a lot.  

Keep in mind, that now we'll need a new parameter in normal and
initial reset for protected virtualization to indicate that we need to
do the reset via the UV call. The Ultravisor does only accept the
needed reset, not any subset resets.

---
 Documentation/virt/kvm/api.txt |  46 ++++++++++++++
 arch/s390/kvm/kvm-s390.c       | 106 +++++++++++++++++++++++----------
 include/uapi/linux/kvm.h       |   5 ++
 3 files changed, 127 insertions(+), 30 deletions(-)

diff --git a/Documentation/virt/kvm/api.txt b/Documentation/virt/kvm/api.txt
index ebb37b34dcfc..734fbe992ed6 100644
--- a/Documentation/virt/kvm/api.txt
+++ b/Documentation/virt/kvm/api.txt
@@ -4168,6 +4168,45 @@ This ioctl issues an ultravisor call to terminate the secure guest,
 unpins the VPA pages and releases all the device pages that are used to
 track the secure pages by hypervisor.
 
+4.122 KVM_S390_NORMAL_RESET
+
+Capability: KVM_CAP_S390_VCPU_RESETS
+Architectures: s390
+Type: vcpu ioctl
+Parameters: none
+Returns: 0
+
+This ioctl resets VCPU registers and control structures. It is
+intended to be called when a normal reset is performed on the vcpu and
+clears local interrupts, the riccb and PSW bit 24.
+
+4.123 KVM_S390_INITIAL_RESET
+
+Capability: none
+Architectures: s390
+Type: vcpu ioctl
+Parameters: none
+Returns: 0
+
+This ioctl resets VCPU registers and control structures. It is
+intended to be called when an initial reset (which is a superset of
+the normal reset) is performed on the vcpu and additionally clears the
+psw, prefix, timing related registers, as well as setting the control
+registers to their initial value.
+
+4.124 KVM_S390_CLEAR_RESET
+
+Capability: KVM_CAP_S390_VCPU_RESETS
+Architectures: s390
+Type: vcpu ioctl
+Parameters: none
+Returns: 0
+
+This ioctl resets VCPU registers and control structures. It is
+intended to be called when a clear reset (which is a superset of the
+initial reset) is performed on the vcpu and additionally clears
+general, access, floating point and vector registers.
+
 5. The kvm_run structure
 ------------------------
 
@@ -5396,3 +5435,10 @@ handling by KVM (as some KVM hypercall may be mistakenly treated as TLB
 flush hypercalls by Hyper-V) so userspace should disable KVM identification
 in CPUID and only exposes Hyper-V identification. In this case, guest
 thinks it's running on Hyper-V and only use Hyper-V hypercalls.
+
+8.22 KVM_CAP_S390_VCPU_RESETS
+
+Architectures: s390
+
+This capability indicates that the KVM_S390_NORMAL_RESET and
+KVM_S390_CLEAR_RESET ioctls are available.
diff --git a/arch/s390/kvm/kvm-s390.c b/arch/s390/kvm/kvm-s390.c
index d9e6bf3d54f0..c338a49331e5 100644
--- a/arch/s390/kvm/kvm-s390.c
+++ b/arch/s390/kvm/kvm-s390.c
@@ -529,6 +529,7 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)
 	case KVM_CAP_S390_CMMA_MIGRATION:
 	case KVM_CAP_S390_AIS:
 	case KVM_CAP_S390_AIS_MIGRATION:
+	case KVM_CAP_S390_VCPU_RESETS:
 		r = 1;
 		break;
 	case KVM_CAP_S390_HPAGE_1M:
@@ -2844,35 +2845,6 @@ void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
 
 }
 
-static void kvm_s390_vcpu_initial_reset(struct kvm_vcpu *vcpu)
-{
-	/* this equals initial cpu reset in pop, but we don't switch to ESA */
-	vcpu->arch.sie_block->gpsw.mask = 0UL;
-	vcpu->arch.sie_block->gpsw.addr = 0UL;
-	kvm_s390_set_prefix(vcpu, 0);
-	kvm_s390_set_cpu_timer(vcpu, 0);
-	vcpu->arch.sie_block->ckc       = 0UL;
-	vcpu->arch.sie_block->todpr     = 0;
-	memset(vcpu->arch.sie_block->gcr, 0, 16 * sizeof(__u64));
-	vcpu->arch.sie_block->gcr[0]  = CR0_UNUSED_56 |
-					CR0_INTERRUPT_KEY_SUBMASK |
-					CR0_MEASUREMENT_ALERT_SUBMASK;
-	vcpu->arch.sie_block->gcr[14] = CR14_UNUSED_32 |
-					CR14_UNUSED_33 |
-					CR14_EXTERNAL_DAMAGE_SUBMASK;
-	/* make sure the new fpc will be lazily loaded */
-	save_fpu_regs();
-	current->thread.fpu.fpc = 0;
-	vcpu->arch.sie_block->gbea = 1;
-	vcpu->arch.sie_block->pp = 0;
-	vcpu->arch.sie_block->fpf &= ~FPF_BPBC;
-	vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID;
-	kvm_clear_async_pf_completion_queue(vcpu);
-	if (!kvm_s390_user_cpu_state_ctrl(vcpu->kvm))
-		kvm_s390_vcpu_stop(vcpu);
-	kvm_s390_clear_local_irqs(vcpu);
-}
-
 void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu)
 {
 	mutex_lock(&vcpu->kvm->lock);
@@ -3287,9 +3259,76 @@ static int kvm_arch_vcpu_ioctl_set_one_reg(struct kvm_vcpu *vcpu,
 	return r;
 }
 
+static int kvm_arch_vcpu_ioctl_normal_reset(struct kvm_vcpu *vcpu)
+{
+	vcpu->arch.sie_block->gpsw.mask = ~PSW_MASK_RI;
+	vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID;
+	memset(vcpu->run->s.regs.riccb, 0, sizeof(vcpu->run->s.regs.riccb));
+
+	kvm_clear_async_pf_completion_queue(vcpu);
+	if (!kvm_s390_user_cpu_state_ctrl(vcpu->kvm))
+		kvm_s390_vcpu_stop(vcpu);
+	kvm_s390_clear_local_irqs(vcpu);
+
+	return 0;
+}
+
 static int kvm_arch_vcpu_ioctl_initial_reset(struct kvm_vcpu *vcpu)
 {
-	kvm_s390_vcpu_initial_reset(vcpu);
+	/* this equals initial cpu reset in pop, but we don't switch to ESA */
+	vcpu->arch.sie_block->gpsw.mask = 0UL;
+	vcpu->arch.sie_block->gpsw.addr = 0UL;
+	kvm_s390_set_prefix(vcpu, 0);
+	kvm_s390_set_cpu_timer(vcpu, 0);
+	vcpu->arch.sie_block->ckc       = 0UL;
+	vcpu->arch.sie_block->todpr     = 0;
+	memset(vcpu->arch.sie_block->gcr, 0, 16 * sizeof(__u64));
+	vcpu->arch.sie_block->gcr[0]  = CR0_UNUSED_56 |
+					CR0_INTERRUPT_KEY_SUBMASK |
+					CR0_MEASUREMENT_ALERT_SUBMASK;
+	vcpu->arch.sie_block->gcr[14] = CR14_UNUSED_32 |
+					CR14_UNUSED_33 |
+					CR14_EXTERNAL_DAMAGE_SUBMASK;
+	/* make sure the new fpc will be lazily loaded */
+	save_fpu_regs();
+	current->thread.fpu.fpc = 0;
+	vcpu->arch.sie_block->gbea = 1;
+	vcpu->arch.sie_block->pp = 0;
+	vcpu->arch.sie_block->fpf &= ~FPF_BPBC;
+
+	return 0;
+}
+
+static int kvm_arch_vcpu_ioctl_clear_reset(struct kvm_vcpu *vcpu)
+{
+	struct kvm_sync_regs *regs = &vcpu->run->s.regs;
+
+	memset(&regs->gprs, 0, sizeof(regs->gprs));
+	/*
+	 * Will be picked up via save_fpu_regs() in the initial reset
+	 * fallthrough.
+	 */
+	memset(&regs->vrs, 0, sizeof(regs->vrs));
+	memset(&regs->acrs, 0, sizeof(regs->acrs));
+
+	regs->etoken = 0;
+	regs->etoken_extension = 0;
+
+	memset(&regs->gscb, 0, sizeof(regs->gscb));
+	if (MACHINE_HAS_GS) {
+		preempt_disable();
+		__ctl_set_bit(2, 4);
+		if (current->thread.gs_cb) {
+			vcpu->arch.host_gscb = current->thread.gs_cb;
+			save_gs_cb(vcpu->arch.host_gscb);
+		}
+		if (vcpu->arch.gs_enabled) {
+			current->thread.gs_cb = (struct gs_cb *)
+				&vcpu->run->s.regs.gscb;
+			restore_gs_cb(current->thread.gs_cb);
+		}
+		preempt_enable();
+	}
 	return 0;
 }
 
@@ -4363,8 +4402,15 @@ long kvm_arch_vcpu_ioctl(struct file *filp,
 		r = kvm_arch_vcpu_ioctl_set_initial_psw(vcpu, psw);
 		break;
 	}
+
+	case KVM_S390_CLEAR_RESET:
+		r = kvm_arch_vcpu_ioctl_clear_reset(vcpu);
+		/* fallthrough */
 	case KVM_S390_INITIAL_RESET:
 		r = kvm_arch_vcpu_ioctl_initial_reset(vcpu);
+		/* fallthrough */
+	case KVM_S390_NORMAL_RESET:
+		r = kvm_arch_vcpu_ioctl_normal_reset(vcpu);
 		break;
 	case KVM_SET_ONE_REG:
 	case KVM_GET_ONE_REG: {
diff --git a/include/uapi/linux/kvm.h b/include/uapi/linux/kvm.h
index f0a16b4adbbd..4b95f9a31a2f 100644
--- a/include/uapi/linux/kvm.h
+++ b/include/uapi/linux/kvm.h
@@ -1009,6 +1009,7 @@ struct kvm_ppc_resize_hpt {
 #define KVM_CAP_PPC_GUEST_DEBUG_SSTEP 176
 #define KVM_CAP_ARM_NISV_TO_USER 177
 #define KVM_CAP_ARM_INJECT_EXT_DABT 178
+#define KVM_CAP_S390_VCPU_RESETS 179
 
 #ifdef KVM_CAP_IRQ_ROUTING
 
@@ -1473,6 +1474,10 @@ struct kvm_enc_region {
 /* Available with KVM_CAP_ARM_SVE */
 #define KVM_ARM_VCPU_FINALIZE	  _IOW(KVMIO,  0xc2, int)
 
+/* Available with  KVM_CAP_S390_VCPU_RESETS */
+#define KVM_S390_NORMAL_RESET	_IO(KVMIO,   0xc3)
+#define KVM_S390_CLEAR_RESET	_IO(KVMIO,   0xc4)
+
 /* Secure Encrypted Virtualization command */
 enum sev_cmd_id {
 	/* Guest initialization commands */
-- 
2.20.1

Re: [PATCH v4] KVM: s390: Add new reset vcpu API

[Cornelia Huck]

On Thu,  9 Jan 2020 10:56:01 -0500
Janosch Frank <frankja@linux.ibm.com> wrote:

> The architecture states that we need to reset local IRQs for all CPU
> resets. Because the old reset interface did not support the normal CPU
> reset we never did that on a normal reset.
> 
> Let's implement an interface for the missing normal and clear resets
> and reset all local IRQs, registers and control structures as stated
> in the architecture.
> 
> Userspace might already reset the registers via the vcpu run struct,
> but as we need the interface for the interrupt clearing part anyway,
> we implement the resets fully and don't rely on userspace to reset the
> rest.
> 
> Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
> ---
> 
> I dropped the reviews, as I changed quite a lot.  
> 
> Keep in mind, that now we'll need a new parameter in normal and
> initial reset for protected virtualization to indicate that we need to
> do the reset via the UV call. The Ultravisor does only accept the
> needed reset, not any subset resets.

In the interface, or externally?

[Apologies, but the details of the protected virt stuff are no longer
in my cache.]

> 
> ---
>  Documentation/virt/kvm/api.txt |  46 ++++++++++++++
>  arch/s390/kvm/kvm-s390.c       | 106 +++++++++++++++++++++++----------
>  include/uapi/linux/kvm.h       |   5 ++
>  3 files changed, 127 insertions(+), 30 deletions(-)
> 
> diff --git a/Documentation/virt/kvm/api.txt b/Documentation/virt/kvm/api.txt
> index ebb37b34dcfc..734fbe992ed6 100644
> --- a/Documentation/virt/kvm/api.txt
> +++ b/Documentation/virt/kvm/api.txt
> @@ -4168,6 +4168,45 @@ This ioctl issues an ultravisor call to terminate the secure guest,
>  unpins the VPA pages and releases all the device pages that are used to
>  track the secure pages by hypervisor.
>  
> +4.122 KVM_S390_NORMAL_RESET
> +
> +Capability: KVM_CAP_S390_VCPU_RESETS
> +Architectures: s390
> +Type: vcpu ioctl
> +Parameters: none
> +Returns: 0
> +
> +This ioctl resets VCPU registers and control structures. It is
> +intended to be called when a normal reset is performed on the vcpu and
> +clears local interrupts, the riccb and PSW bit 24.

I'm not sure you'd want to specify the actual values to be reset here;
you'd always need to remember to update them when the architecture is
extended... just refer to the POP instead?

> +
> +4.123 KVM_S390_INITIAL_RESET
> +
> +Capability: none
> +Architectures: s390
> +Type: vcpu ioctl
> +Parameters: none
> +Returns: 0
> +
> +This ioctl resets VCPU registers and control structures. It is
> +intended to be called when an initial reset (which is a superset of
> +the normal reset) is performed on the vcpu and additionally clears the
> +psw, prefix, timing related registers, as well as setting the control
> +registers to their initial value.

Same here.

> +
> +4.124 KVM_S390_CLEAR_RESET
> +
> +Capability: KVM_CAP_S390_VCPU_RESETS
> +Architectures: s390
> +Type: vcpu ioctl
> +Parameters: none
> +Returns: 0
> +
> +This ioctl resets VCPU registers and control structures. It is
> +intended to be called when a clear reset (which is a superset of the
> +initial reset) is performed on the vcpu and additionally clears
> +general, access, floating point and vector registers.

And here.

> +
>  5. The kvm_run structure
>  ------------------------
>  
> @@ -5396,3 +5435,10 @@ handling by KVM (as some KVM hypercall may be mistakenly treated as TLB
>  flush hypercalls by Hyper-V) so userspace should disable KVM identification
>  in CPUID and only exposes Hyper-V identification. In this case, guest
>  thinks it's running on Hyper-V and only use Hyper-V hypercalls.
> +
> +8.22 KVM_CAP_S390_VCPU_RESETS
> +
> +Architectures: s390
> +
> +This capability indicates that the KVM_S390_NORMAL_RESET and
> +KVM_S390_CLEAR_RESET ioctls are available.
> diff --git a/arch/s390/kvm/kvm-s390.c b/arch/s390/kvm/kvm-s390.c
> index d9e6bf3d54f0..c338a49331e5 100644
> --- a/arch/s390/kvm/kvm-s390.c
> +++ b/arch/s390/kvm/kvm-s390.c
> @@ -529,6 +529,7 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)
>  	case KVM_CAP_S390_CMMA_MIGRATION:
>  	case KVM_CAP_S390_AIS:
>  	case KVM_CAP_S390_AIS_MIGRATION:
> +	case KVM_CAP_S390_VCPU_RESETS:
>  		r = 1;
>  		break;
>  	case KVM_CAP_S390_HPAGE_1M:
> @@ -2844,35 +2845,6 @@ void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
>  
>  }
>  
> -static void kvm_s390_vcpu_initial_reset(struct kvm_vcpu *vcpu)
> -{
> -	/* this equals initial cpu reset in pop, but we don't switch to ESA */
> -	vcpu->arch.sie_block->gpsw.mask = 0UL;
> -	vcpu->arch.sie_block->gpsw.addr = 0UL;
> -	kvm_s390_set_prefix(vcpu, 0);
> -	kvm_s390_set_cpu_timer(vcpu, 0);
> -	vcpu->arch.sie_block->ckc       = 0UL;
> -	vcpu->arch.sie_block->todpr     = 0;
> -	memset(vcpu->arch.sie_block->gcr, 0, 16 * sizeof(__u64));
> -	vcpu->arch.sie_block->gcr[0]  = CR0_UNUSED_56 |
> -					CR0_INTERRUPT_KEY_SUBMASK |
> -					CR0_MEASUREMENT_ALERT_SUBMASK;
> -	vcpu->arch.sie_block->gcr[14] = CR14_UNUSED_32 |
> -					CR14_UNUSED_33 |
> -					CR14_EXTERNAL_DAMAGE_SUBMASK;
> -	/* make sure the new fpc will be lazily loaded */
> -	save_fpu_regs();
> -	current->thread.fpu.fpc = 0;
> -	vcpu->arch.sie_block->gbea = 1;
> -	vcpu->arch.sie_block->pp = 0;
> -	vcpu->arch.sie_block->fpf &= ~FPF_BPBC;
> -	vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID;
> -	kvm_clear_async_pf_completion_queue(vcpu);
> -	if (!kvm_s390_user_cpu_state_ctrl(vcpu->kvm))
> -		kvm_s390_vcpu_stop(vcpu);
> -	kvm_s390_clear_local_irqs(vcpu);
> -}
> -
>  void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu)
>  {
>  	mutex_lock(&vcpu->kvm->lock);
> @@ -3287,9 +3259,76 @@ static int kvm_arch_vcpu_ioctl_set_one_reg(struct kvm_vcpu *vcpu,
>  	return r;
>  }
>  
> +static int kvm_arch_vcpu_ioctl_normal_reset(struct kvm_vcpu *vcpu)
> +{
> +	vcpu->arch.sie_block->gpsw.mask = ~PSW_MASK_RI;
> +	vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID;
> +	memset(vcpu->run->s.regs.riccb, 0, sizeof(vcpu->run->s.regs.riccb));
> +
> +	kvm_clear_async_pf_completion_queue(vcpu);
> +	if (!kvm_s390_user_cpu_state_ctrl(vcpu->kvm))
> +		kvm_s390_vcpu_stop(vcpu);
> +	kvm_s390_clear_local_irqs(vcpu);
> +
> +	return 0;
> +}
> +
>  static int kvm_arch_vcpu_ioctl_initial_reset(struct kvm_vcpu *vcpu)
>  {
> -	kvm_s390_vcpu_initial_reset(vcpu);
> +	/* this equals initial cpu reset in pop, but we don't switch to ESA */

Maybe also mention that in the documentation?

> +	vcpu->arch.sie_block->gpsw.mask = 0UL;
> +	vcpu->arch.sie_block->gpsw.addr = 0UL;
> +	kvm_s390_set_prefix(vcpu, 0);
> +	kvm_s390_set_cpu_timer(vcpu, 0);
> +	vcpu->arch.sie_block->ckc       = 0UL;
> +	vcpu->arch.sie_block->todpr     = 0;
> +	memset(vcpu->arch.sie_block->gcr, 0, 16 * sizeof(__u64));
> +	vcpu->arch.sie_block->gcr[0]  = CR0_UNUSED_56 |
> +					CR0_INTERRUPT_KEY_SUBMASK |
> +					CR0_MEASUREMENT_ALERT_SUBMASK;
> +	vcpu->arch.sie_block->gcr[14] = CR14_UNUSED_32 |
> +					CR14_UNUSED_33 |
> +					CR14_EXTERNAL_DAMAGE_SUBMASK;
> +	/* make sure the new fpc will be lazily loaded */
> +	save_fpu_regs();
> +	current->thread.fpu.fpc = 0;
> +	vcpu->arch.sie_block->gbea = 1;
> +	vcpu->arch.sie_block->pp = 0;
> +	vcpu->arch.sie_block->fpf &= ~FPF_BPBC;
> +

Add a comment that the remaining work will be done in normal_reset?

> +	return 0;
> +}
> +
> +static int kvm_arch_vcpu_ioctl_clear_reset(struct kvm_vcpu *vcpu)
> +{
> +	struct kvm_sync_regs *regs = &vcpu->run->s.regs;
> +
> +	memset(&regs->gprs, 0, sizeof(regs->gprs));
> +	/*
> +	 * Will be picked up via save_fpu_regs() in the initial reset
> +	 * fallthrough.
> +	 */

This comment is a bit confusing... what does 'picked up' mean?

(Maybe I'm just too tired, sorry...)

> +	memset(&regs->vrs, 0, sizeof(regs->vrs));
> +	memset(&regs->acrs, 0, sizeof(regs->acrs));
> +
> +	regs->etoken = 0;
> +	regs->etoken_extension = 0;
> +
> +	memset(&regs->gscb, 0, sizeof(regs->gscb));
> +	if (MACHINE_HAS_GS) {
> +		preempt_disable();
> +		__ctl_set_bit(2, 4);
> +		if (current->thread.gs_cb) {
> +			vcpu->arch.host_gscb = current->thread.gs_cb;
> +			save_gs_cb(vcpu->arch.host_gscb);
> +		}
> +		if (vcpu->arch.gs_enabled) {
> +			current->thread.gs_cb = (struct gs_cb *)
> +				&vcpu->run->s.regs.gscb;
> +			restore_gs_cb(current->thread.gs_cb);
> +		}
> +		preempt_enable();
> +	}

And here that the remaining work will be done in initial_reset and
normal_reset?

>  	return 0;
>  }
>  
> @@ -4363,8 +4402,15 @@ long kvm_arch_vcpu_ioctl(struct file *filp,
>  		r = kvm_arch_vcpu_ioctl_set_initial_psw(vcpu, psw);
>  		break;
>  	}
> +
> +	case KVM_S390_CLEAR_RESET:
> +		r = kvm_arch_vcpu_ioctl_clear_reset(vcpu);
> +		/* fallthrough */
>  	case KVM_S390_INITIAL_RESET:
>  		r = kvm_arch_vcpu_ioctl_initial_reset(vcpu);
> +		/* fallthrough */
> +	case KVM_S390_NORMAL_RESET:
> +		r = kvm_arch_vcpu_ioctl_normal_reset(vcpu);

Can any of these functions return !0 when the protected virt stuff is
done on top? If not, can we make them void and just set r=0; here?

>  		break;
>  	case KVM_SET_ONE_REG:
>  	case KVM_GET_ONE_REG: {
> diff --git a/include/uapi/linux/kvm.h b/include/uapi/linux/kvm.h
> index f0a16b4adbbd..4b95f9a31a2f 100644
> --- a/include/uapi/linux/kvm.h
> +++ b/include/uapi/linux/kvm.h
> @@ -1009,6 +1009,7 @@ struct kvm_ppc_resize_hpt {
>  #define KVM_CAP_PPC_GUEST_DEBUG_SSTEP 176
>  #define KVM_CAP_ARM_NISV_TO_USER 177
>  #define KVM_CAP_ARM_INJECT_EXT_DABT 178
> +#define KVM_CAP_S390_VCPU_RESETS 179
>  
>  #ifdef KVM_CAP_IRQ_ROUTING
>  
> @@ -1473,6 +1474,10 @@ struct kvm_enc_region {
>  /* Available with KVM_CAP_ARM_SVE */
>  #define KVM_ARM_VCPU_FINALIZE	  _IOW(KVMIO,  0xc2, int)
>  
> +/* Available with  KVM_CAP_S390_VCPU_RESETS */
> +#define KVM_S390_NORMAL_RESET	_IO(KVMIO,   0xc3)
> +#define KVM_S390_CLEAR_RESET	_IO(KVMIO,   0xc4)
> +
>  /* Secure Encrypted Virtualization command */
>  enum sev_cmd_id {
>  	/* Guest initialization commands */

Re: [PATCH v4] KVM: s390: Add new reset vcpu API

[Janosch Frank]

[-- Attachment #1.1: Type: text/plain, Size: 5816 bytes --]

On 1/9/20 6:08 PM, Cornelia Huck wrote:
> On Thu,  9 Jan 2020 10:56:01 -0500
> Janosch Frank <frankja@linux.ibm.com> wrote:
> 
>> The architecture states that we need to reset local IRQs for all CPU
>> resets. Because the old reset interface did not support the normal CPU
>> reset we never did that on a normal reset.
>>
>> Let's implement an interface for the missing normal and clear resets
>> and reset all local IRQs, registers and control structures as stated
>> in the architecture.
>>
>> Userspace might already reset the registers via the vcpu run struct,
>> but as we need the interface for the interrupt clearing part anyway,
>> we implement the resets fully and don't rely on userspace to reset the
>> rest.
>>
>> Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
>> ---
>>
>> I dropped the reviews, as I changed quite a lot.  
>>
>> Keep in mind, that now we'll need a new parameter in normal and
>> initial reset for protected virtualization to indicate that we need to
>> do the reset via the UV call. The Ultravisor does only accept the
>> needed reset, not any subset resets.
> 
> In the interface, or externally?

?

> 
> [Apologies, but the details of the protected virt stuff are no longer
> in my cache.
Reworded explanation:
I can't use a fallthrough, because the UV will reject the normal reset
if we do an initial reset (same goes for the clear reset). To address
this issue, I added a boolean to the normal and initial reset functions
which tells the function if it was called directly or was called because
of the fallthrough.

Only if called directly a UV call for the reset is done, that way we can
keep the fallthrough.

>>  static int kvm_arch_vcpu_ioctl_initial_reset(struct kvm_vcpu *vcpu)
>>  {
>> -	kvm_s390_vcpu_initial_reset(vcpu);
>> +	/* this equals initial cpu reset in pop, but we don't switch to ESA */
> 
> Maybe also mention that in the documentation?

Sure

> 
>> +	vcpu->arch.sie_block->gpsw.mask = 0UL;
>> +	vcpu->arch.sie_block->gpsw.addr = 0UL;
>> +	kvm_s390_set_prefix(vcpu, 0);
>> +	kvm_s390_set_cpu_timer(vcpu, 0);
>> +	vcpu->arch.sie_block->ckc       = 0UL;
>> +	vcpu->arch.sie_block->todpr     = 0;
>> +	memset(vcpu->arch.sie_block->gcr, 0, 16 * sizeof(__u64));
>> +	vcpu->arch.sie_block->gcr[0]  = CR0_UNUSED_56 |
>> +					CR0_INTERRUPT_KEY_SUBMASK |
>> +					CR0_MEASUREMENT_ALERT_SUBMASK;
>> +	vcpu->arch.sie_block->gcr[14] = CR14_UNUSED_32 |
>> +					CR14_UNUSED_33 |
>> +					CR14_EXTERNAL_DAMAGE_SUBMASK;
>> +	/* make sure the new fpc will be lazily loaded */
>> +	save_fpu_regs();
>> +	current->thread.fpu.fpc = 0;
>> +	vcpu->arch.sie_block->gbea = 1;
>> +	vcpu->arch.sie_block->pp = 0;
>> +	vcpu->arch.sie_block->fpf &= ~FPF_BPBC;
>> +
> 
> Add a comment that the remaining work will be done in normal_reset?

Will do

> 
>> +	return 0;
>> +}
>> +
>> +static int kvm_arch_vcpu_ioctl_clear_reset(struct kvm_vcpu *vcpu)
>> +{
>> +	struct kvm_sync_regs *regs = &vcpu->run->s.regs;
>> +
>> +	memset(&regs->gprs, 0, sizeof(regs->gprs));
>> +	/*
>> +	 * Will be picked up via save_fpu_regs() in the initial reset
>> +	 * fallthrough.
>> +	 */
> 
> This comment is a bit confusing... what does 'picked up' mean?
> 
> (Maybe I'm just too tired, sorry...)

fpus are loaded lazily, maybe I should just remove the comment.

> 
>> +	memset(&regs->vrs, 0, sizeof(regs->vrs));
>> +	memset(&regs->acrs, 0, sizeof(regs->acrs));
>> +
>> +	regs->etoken = 0;
>> +	regs->etoken_extension = 0;
>> +
>> +	memset(&regs->gscb, 0, sizeof(regs->gscb));
>> +	if (MACHINE_HAS_GS) {
>> +		preempt_disable();
>> +		__ctl_set_bit(2, 4);
>> +		if (current->thread.gs_cb) {
>> +			vcpu->arch.host_gscb = current->thread.gs_cb;
>> +			save_gs_cb(vcpu->arch.host_gscb);
>> +		}
>> +		if (vcpu->arch.gs_enabled) {
>> +			current->thread.gs_cb = (struct gs_cb *)
>> +				&vcpu->run->s.regs.gscb;
>> +			restore_gs_cb(current->thread.gs_cb);
>> +		}
>> +		preempt_enable();
>> +	}
> 
> And here that the remaining work will be done in initial_reset and
> normal_reset?
> 
>>  	return 0;
>>  }
>>  
>> @@ -4363,8 +4402,15 @@ long kvm_arch_vcpu_ioctl(struct file *filp,
>>  		r = kvm_arch_vcpu_ioctl_set_initial_psw(vcpu, psw);
>>  		break;
>>  	}
>> +
>> +	case KVM_S390_CLEAR_RESET:
>> +		r = kvm_arch_vcpu_ioctl_clear_reset(vcpu);
>> +		/* fallthrough */
>>  	case KVM_S390_INITIAL_RESET:
>>  		r = kvm_arch_vcpu_ioctl_initial_reset(vcpu);
>> +		/* fallthrough */
>> +	case KVM_S390_NORMAL_RESET:
>> +		r = kvm_arch_vcpu_ioctl_normal_reset(vcpu);
> 
> Can any of these functions return !0 when the protected virt stuff is
> done on top? If not, can we make them void and just set r=0; here?

They do return > 0 if the UV call fails, so I need those r values.

> 
>>  		break;
>>  	case KVM_SET_ONE_REG:
>>  	case KVM_GET_ONE_REG: {
>> diff --git a/include/uapi/linux/kvm.h b/include/uapi/linux/kvm.h
>> index f0a16b4adbbd..4b95f9a31a2f 100644
>> --- a/include/uapi/linux/kvm.h
>> +++ b/include/uapi/linux/kvm.h
>> @@ -1009,6 +1009,7 @@ struct kvm_ppc_resize_hpt {
>>  #define KVM_CAP_PPC_GUEST_DEBUG_SSTEP 176
>>  #define KVM_CAP_ARM_NISV_TO_USER 177
>>  #define KVM_CAP_ARM_INJECT_EXT_DABT 178
>> +#define KVM_CAP_S390_VCPU_RESETS 179
>>  
>>  #ifdef KVM_CAP_IRQ_ROUTING
>>  
>> @@ -1473,6 +1474,10 @@ struct kvm_enc_region {
>>  /* Available with KVM_CAP_ARM_SVE */
>>  #define KVM_ARM_VCPU_FINALIZE	  _IOW(KVMIO,  0xc2, int)
>>  
>> +/* Available with  KVM_CAP_S390_VCPU_RESETS */
>> +#define KVM_S390_NORMAL_RESET	_IO(KVMIO,   0xc3)
>> +#define KVM_S390_CLEAR_RESET	_IO(KVMIO,   0xc4)
>> +
>>  /* Secure Encrypted Virtualization command */
>>  enum sev_cmd_id {
>>  	/* Guest initialization commands */
> 



[-- Attachment #2: OpenPGP digital signature --]
[-- Type: application/pgp-signature, Size: 833 bytes --]

Re: [PATCH v4] KVM: s390: Add new reset vcpu API

[Thomas Huth]

[-- Attachment #1.1: Type: text/plain, Size: 2863 bytes --]

On 09/01/2020 18.51, Janosch Frank wrote:
> On 1/9/20 6:08 PM, Cornelia Huck wrote:
>> On Thu,  9 Jan 2020 10:56:01 -0500
>> Janosch Frank <frankja@linux.ibm.com> wrote:
>>
>>> The architecture states that we need to reset local IRQs for all CPU
>>> resets. Because the old reset interface did not support the normal CPU
>>> reset we never did that on a normal reset.
>>>
>>> Let's implement an interface for the missing normal and clear resets
>>> and reset all local IRQs, registers and control structures as stated
>>> in the architecture.
>>>
>>> Userspace might already reset the registers via the vcpu run struct,
>>> but as we need the interface for the interrupt clearing part anyway,
>>> we implement the resets fully and don't rely on userspace to reset the
>>> rest.
>>>
>>> Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
>>> ---
>>>
>>> I dropped the reviews, as I changed quite a lot.  
>>>
>>> Keep in mind, that now we'll need a new parameter in normal and
>>> initial reset for protected virtualization to indicate that we need to
>>> do the reset via the UV call. The Ultravisor does only accept the
>>> needed reset, not any subset resets.
>>
>> In the interface, or externally?
> 
> ?
> 
>>
>> [Apologies, but the details of the protected virt stuff are no longer
>> in my cache.
> Reworded explanation:
> I can't use a fallthrough, because the UV will reject the normal reset
> if we do an initial reset (same goes for the clear reset). To address
> this issue, I added a boolean to the normal and initial reset functions
> which tells the function if it was called directly or was called because
> of the fallthrough.
> 
> Only if called directly a UV call for the reset is done, that way we can
> keep the fallthrough.

Sounds complicated. And do we need the fallthrough stuff here at all?
What about doing something like:

static int kvm_arch_vcpu_ioctl_normal_reset(struct kvm_vcpu *vcpu)
{
	...
}

static int kvm_arch_vcpu_ioctl_initial_reset(struct kvm_vcpu *vcpu)
{
	kvm_arch_vcpu_ioctl_normal_reset(vcpu);
	...
}

static int kvm_arch_vcpu_ioctl_clear_reset(struct kvm_vcpu *vcpu)
{
	kvm_arch_vcpu_ioctl_initial_reset(vcpu);
	...
}

...

	case KVM_S390_CLEAR_RESET:
		r = kvm_arch_vcpu_ioctl_clear_reset(vcpu);
		if (!r && protected) {
			r = uv_cmd_nodata(...,
 				UVC_CMD_CPU_RESET_CLEAR, ...);
		}
		break;
 	case KVM_S390_INITIAL_RESET:
 		r = kvm_arch_vcpu_ioctl_initial_reset(vcpu);
		if (!r && protected) {
			r = uv_cmd_nodata(...,
 				UVC_CMD_CPU_RESET_INITIAL, ...);
		}
	case KVM_S390_NORMAL_RESET:
		r = kvm_arch_vcpu_ioctl_normal_reset(vcpu);
		if (!r && protected) {
			r = uv_cmd_nodata(...,
 				UVC_CMD_CPU_RESET, ...);
		}
 		break;

... or does that not work due to some other constraints that I've missed?

 Thomas


[-- Attachment #2: OpenPGP digital signature --]
[-- Type: application/pgp-signature, Size: 833 bytes --]

Re: [PATCH v4] KVM: s390: Add new reset vcpu API

[Janosch Frank]

[-- Attachment #1.1: Type: text/plain, Size: 3187 bytes --]

On 1/10/20 8:03 AM, Thomas Huth wrote:
> On 09/01/2020 18.51, Janosch Frank wrote:
>> On 1/9/20 6:08 PM, Cornelia Huck wrote:
>>> On Thu,  9 Jan 2020 10:56:01 -0500
>>> Janosch Frank <frankja@linux.ibm.com> wrote:
>>>
>>>> The architecture states that we need to reset local IRQs for all CPU
>>>> resets. Because the old reset interface did not support the normal CPU
>>>> reset we never did that on a normal reset.
>>>>
>>>> Let's implement an interface for the missing normal and clear resets
>>>> and reset all local IRQs, registers and control structures as stated
>>>> in the architecture.
>>>>
>>>> Userspace might already reset the registers via the vcpu run struct,
>>>> but as we need the interface for the interrupt clearing part anyway,
>>>> we implement the resets fully and don't rely on userspace to reset the
>>>> rest.
>>>>
>>>> Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
>>>> ---
>>>>
>>>> I dropped the reviews, as I changed quite a lot.  
>>>>
>>>> Keep in mind, that now we'll need a new parameter in normal and
>>>> initial reset for protected virtualization to indicate that we need to
>>>> do the reset via the UV call. The Ultravisor does only accept the
>>>> needed reset, not any subset resets.
>>>
>>> In the interface, or externally?
>>
>> ?
>>
>>>
>>> [Apologies, but the details of the protected virt stuff are no longer
>>> in my cache.
>> Reworded explanation:
>> I can't use a fallthrough, because the UV will reject the normal reset
>> if we do an initial reset (same goes for the clear reset). To address
>> this issue, I added a boolean to the normal and initial reset functions
>> which tells the function if it was called directly or was called because
>> of the fallthrough.
>>
>> Only if called directly a UV call for the reset is done, that way we can
>> keep the fallthrough.
> 
> Sounds complicated. And do we need the fallthrough stuff here at all?
> What about doing something like:

That would work and I thought about it, it just comes down to taste :-)
I don't have any strong feelings for a specific implementation.

> 
> static int kvm_arch_vcpu_ioctl_normal_reset(struct kvm_vcpu *vcpu)
> {
> 	...
> }
> 
> static int kvm_arch_vcpu_ioctl_initial_reset(struct kvm_vcpu *vcpu)
> {
> 	kvm_arch_vcpu_ioctl_normal_reset(vcpu);
> 	...
> }
> 
> static int kvm_arch_vcpu_ioctl_clear_reset(struct kvm_vcpu *vcpu)
> {
> 	kvm_arch_vcpu_ioctl_initial_reset(vcpu);
> 	...
> }
> 
> ...
> 
> 	case KVM_S390_CLEAR_RESET:
> 		r = kvm_arch_vcpu_ioctl_clear_reset(vcpu);
> 		if (!r && protected) {
> 			r = uv_cmd_nodata(...,
>  				UVC_CMD_CPU_RESET_CLEAR, ...);
> 		}
> 		break;
>  	case KVM_S390_INITIAL_RESET:
>  		r = kvm_arch_vcpu_ioctl_initial_reset(vcpu);
> 		if (!r && protected) {
> 			r = uv_cmd_nodata(...,
>  				UVC_CMD_CPU_RESET_INITIAL, ...);
> 		}
> 	case KVM_S390_NORMAL_RESET:
> 		r = kvm_arch_vcpu_ioctl_normal_reset(vcpu);
> 		if (!r && protected) {
> 			r = uv_cmd_nodata(...,
>  				UVC_CMD_CPU_RESET, ...);
> 		}
>  		break;
> 
> ... or does that not work due to some other constraints that I've missed?
> 
>  Thomas
> 



[-- Attachment #2: OpenPGP digital signature --]
[-- Type: application/pgp-signature, Size: 833 bytes --]

Re: [PATCH v4] KVM: s390: Add new reset vcpu API

[Janosch Frank]

[-- Attachment #1.1: Type: text/plain, Size: 6265 bytes --]

On 1/10/20 8:14 AM, Janosch Frank wrote:
> On 1/10/20 8:03 AM, Thomas Huth wrote:
>> On 09/01/2020 18.51, Janosch Frank wrote:
>>> On 1/9/20 6:08 PM, Cornelia Huck wrote:
>>>> On Thu,  9 Jan 2020 10:56:01 -0500
>>>> Janosch Frank <frankja@linux.ibm.com> wrote:
>>>>
>>>>> The architecture states that we need to reset local IRQs for all CPU
>>>>> resets. Because the old reset interface did not support the normal CPU
>>>>> reset we never did that on a normal reset.
>>>>>
>>>>> Let's implement an interface for the missing normal and clear resets
>>>>> and reset all local IRQs, registers and control structures as stated
>>>>> in the architecture.
>>>>>
>>>>> Userspace might already reset the registers via the vcpu run struct,
>>>>> but as we need the interface for the interrupt clearing part anyway,
>>>>> we implement the resets fully and don't rely on userspace to reset the
>>>>> rest.
>>>>>
>>>>> Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
>>>>> ---
>>>>>
>>>>> I dropped the reviews, as I changed quite a lot.  
>>>>>
>>>>> Keep in mind, that now we'll need a new parameter in normal and
>>>>> initial reset for protected virtualization to indicate that we need to
>>>>> do the reset via the UV call. The Ultravisor does only accept the
>>>>> needed reset, not any subset resets.
>>>>
>>>> In the interface, or externally?
>>>
>>> ?
>>>
>>>>
>>>> [Apologies, but the details of the protected virt stuff are no longer
>>>> in my cache.
>>> Reworded explanation:
>>> I can't use a fallthrough, because the UV will reject the normal reset
>>> if we do an initial reset (same goes for the clear reset). To address
>>> this issue, I added a boolean to the normal and initial reset functions
>>> which tells the function if it was called directly or was called because
>>> of the fallthrough.
>>>
>>> Only if called directly a UV call for the reset is done, that way we can
>>> keep the fallthrough.
>>
>> Sounds complicated. And do we need the fallthrough stuff here at all?
>> What about doing something like:
> 
> That would work and I thought about it, it just comes down to taste :-)
> I don't have any strong feelings for a specific implementation.

To be more specific:


Commit c72db49c098bceb8b73c2e9d305caf37a41fb3bf
Author: Janosch Frank <frankja@linux.ibm.com>
Date:   Thu Jan 9 04:37:50 2020 -0500

    KVM: s390: protvirt: Add UV cpu reset calls

    For protected VMs, the VCPU resets are done by the Ultravisor, as KVM
    has no access to the VCPU registers.

    As the Ultravisor will only accept a call for the reset that is
    needed, we need to fence the UV calls when chaining resets.

    Signed-off-by: Janosch Frank <frankja@linux.ibm.com>

diff --git a/arch/s390/kvm/kvm-s390.c b/arch/s390/kvm/kvm-s390.c
index 63dc2bd97582..d5876527e464 100644
--- a/arch/s390/kvm/kvm-s390.c
+++ b/arch/s390/kvm/kvm-s390.c
@@ -3476,8 +3476,11 @@ static int kvm_arch_vcpu_ioctl_set_one_reg(struct
kvm_vcpu *vcpu,
 	return r;
 }

-static int kvm_arch_vcpu_ioctl_normal_reset(struct kvm_vcpu *vcpu)
+static int kvm_arch_vcpu_ioctl_normal_reset(struct kvm_vcpu *vcpu, bool
chain)
 {
+	int rc = 0;
+	u32 ret;
+
 	vcpu->arch.sie_block->gpsw.mask = ~PSW_MASK_RI;
 	vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID;
 	memset(vcpu->run->s.regs.riccb, 0, sizeof(vcpu->run->s.regs.riccb));
@@ -3487,11 +3490,21 @@ static int
kvm_arch_vcpu_ioctl_normal_reset(struct kvm_vcpu *vcpu)
 		kvm_s390_vcpu_stop(vcpu);
 	kvm_s390_clear_local_irqs(vcpu);

-	return 0;
+	if (kvm_s390_pv_handle_cpu(vcpu) && !chain) {
+		rc = uv_cmd_nodata(kvm_s390_pv_handle_cpu(vcpu),
+				   UVC_CMD_CPU_RESET, &ret);
+		VCPU_EVENT(vcpu, 3, "PROTVIRT RESET NORMAL VCPU: cpu %d rc %x rrc %x",
+			   vcpu->vcpu_id, ret >> 16, ret & 0x0000ffff);
+	}
+
+	return rc;
 }

-static int kvm_arch_vcpu_ioctl_initial_reset(struct kvm_vcpu *vcpu)
+static int kvm_arch_vcpu_ioctl_initial_reset(struct kvm_vcpu *vcpu,
bool chain)
 {
+	int rc = 0;
+	u32 ret;
+
 	/* this equals initial cpu reset in pop, but we don't switch to ESA */
 	vcpu->arch.sie_block->gpsw.mask = 0UL;
 	vcpu->arch.sie_block->gpsw.addr = 0UL;
@@ -3509,16 +3522,26 @@ static int
kvm_arch_vcpu_ioctl_initial_reset(struct kvm_vcpu *vcpu)
 	/* make sure the new fpc will be lazily loaded */
 	save_fpu_regs();
 	current->thread.fpu.fpc = 0;
-	if (!kvm_s390_pv_is_protected(vcpu->kvm))
+	if (!kvm_s390_pv_handle_cpu(vcpu))
 		vcpu->arch.sie_block->gbea = 1;
 	vcpu->arch.sie_block->pp = 0;
 	vcpu->arch.sie_block->fpf &= ~FPF_BPBC;

-	return 0;
+	if (kvm_s390_pv_handle_cpu(vcpu) && !chain) {
+		rc = uv_cmd_nodata(kvm_s390_pv_handle_cpu(vcpu),
+				   UVC_CMD_CPU_RESET_INITIAL,
+				   &ret);
+		VCPU_EVENT(vcpu, 3, "PROTVIRT RESET INITIAL VCPU: cpu %d rc %x rrc %x",
+			   vcpu->vcpu_id, ret >> 16, ret & 0x0000ffff);
+	}
+
+	return rc;
 }

 static int kvm_arch_vcpu_ioctl_clear_reset(struct kvm_vcpu *vcpu)
 {
+	int rc = 0;
+	u32 ret;
 	struct kvm_sync_regs *regs = &vcpu->run->s.regs;

 	memset(&regs->gprs, 0, sizeof(regs->gprs));
@@ -3547,7 +3570,13 @@ static int kvm_arch_vcpu_ioctl_clear_reset(struct
kvm_vcpu *vcpu)
 		}
 		preempt_enable();
 	}
-	return 0;
+	if (kvm_s390_pv_handle_cpu(vcpu)) {
+		rc = uv_cmd_nodata(kvm_s390_pv_handle_cpu(vcpu),
+				   UVC_CMD_CPU_RESET_CLEAR, &ret);
+		VCPU_EVENT(vcpu, 3, "PROTVIRT RESET CLEAR VCPU: cpu %d rc %x rrc %x",
+			   vcpu->vcpu_id, ret >> 16, ret & 0x0000ffff);
+	}
+	return rc;
 }

 int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs
*regs)
@@ -4738,12 +4767,16 @@ long kvm_arch_vcpu_ioctl(struct file *filp,

 	case KVM_S390_CLEAR_RESET:
 		r = kvm_arch_vcpu_ioctl_clear_reset(vcpu);
+		if (r)
+			break;
 		/* fallthrough */
 	case KVM_S390_INITIAL_RESET:
-		r = kvm_arch_vcpu_ioctl_initial_reset(vcpu);
+		r = kvm_arch_vcpu_ioctl_initial_reset(vcpu, ioctl !=
KVM_S390_INITIAL_RESET);
+		if (r)
+			break;
 		/* fallthrough */
 	case KVM_S390_NORMAL_RESET:
-		r = kvm_arch_vcpu_ioctl_normal_reset(vcpu);
+		r = kvm_arch_vcpu_ioctl_normal_reset(vcpu, ioctl !=
KVM_S390_NORMAL_RESET);
 		break;
 	case KVM_SET_ONE_REG:
 	case KVM_GET_ONE_REG: {


[-- Attachment #2: OpenPGP digital signature --]
[-- Type: application/pgp-signature, Size: 833 bytes --]

Re: [PATCH v4] KVM: s390: Add new reset vcpu API

[Thomas Huth]

On 10/01/2020 09.43, Janosch Frank wrote:
> On 1/10/20 8:14 AM, Janosch Frank wrote:
>> On 1/10/20 8:03 AM, Thomas Huth wrote:
>>> On 09/01/2020 18.51, Janosch Frank wrote:
>>>> On 1/9/20 6:08 PM, Cornelia Huck wrote:
>>>>> On Thu,  9 Jan 2020 10:56:01 -0500
>>>>> Janosch Frank <frankja@linux.ibm.com> wrote:
>>>>>
>>>>>> The architecture states that we need to reset local IRQs for all CPU
>>>>>> resets. Because the old reset interface did not support the normal CPU
>>>>>> reset we never did that on a normal reset.
>>>>>>
>>>>>> Let's implement an interface for the missing normal and clear resets
>>>>>> and reset all local IRQs, registers and control structures as stated
>>>>>> in the architecture.
>>>>>>
>>>>>> Userspace might already reset the registers via the vcpu run struct,
>>>>>> but as we need the interface for the interrupt clearing part anyway,
>>>>>> we implement the resets fully and don't rely on userspace to reset the
>>>>>> rest.
>>>>>>
>>>>>> Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
>>>>>> ---
>>>>>>
>>>>>> I dropped the reviews, as I changed quite a lot.  
>>>>>>
>>>>>> Keep in mind, that now we'll need a new parameter in normal and
>>>>>> initial reset for protected virtualization to indicate that we need to
>>>>>> do the reset via the UV call. The Ultravisor does only accept the
>>>>>> needed reset, not any subset resets.
>>>>>
>>>>> In the interface, or externally?
>>>>
>>>> ?
>>>>
>>>>>
>>>>> [Apologies, but the details of the protected virt stuff are no longer
>>>>> in my cache.
>>>> Reworded explanation:
>>>> I can't use a fallthrough, because the UV will reject the normal reset
>>>> if we do an initial reset (same goes for the clear reset). To address
>>>> this issue, I added a boolean to the normal and initial reset functions
>>>> which tells the function if it was called directly or was called because
>>>> of the fallthrough.
>>>>
>>>> Only if called directly a UV call for the reset is done, that way we can
>>>> keep the fallthrough.
>>>
>>> Sounds complicated. And do we need the fallthrough stuff here at all?
>>> What about doing something like:
>>
>> That would work and I thought about it, it just comes down to taste :-)
>> I don't have any strong feelings for a specific implementation.
> 
> To be more specific:
> 
> 
> Commit c72db49c098bceb8b73c2e9d305caf37a41fb3bf
> Author: Janosch Frank <frankja@linux.ibm.com>
> Date:   Thu Jan 9 04:37:50 2020 -0500
> 
>     KVM: s390: protvirt: Add UV cpu reset calls
> 
>     For protected VMs, the VCPU resets are done by the Ultravisor, as KVM
>     has no access to the VCPU registers.
> 
>     As the Ultravisor will only accept a call for the reset that is
>     needed, we need to fence the UV calls when chaining resets.
> 
>     Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
> 
> diff --git a/arch/s390/kvm/kvm-s390.c b/arch/s390/kvm/kvm-s390.c
> index 63dc2bd97582..d5876527e464 100644
> --- a/arch/s390/kvm/kvm-s390.c
> +++ b/arch/s390/kvm/kvm-s390.c
> @@ -3476,8 +3476,11 @@ static int kvm_arch_vcpu_ioctl_set_one_reg(struct
> kvm_vcpu *vcpu,
>  	return r;
>  }
> 
> -static int kvm_arch_vcpu_ioctl_normal_reset(struct kvm_vcpu *vcpu)
> +static int kvm_arch_vcpu_ioctl_normal_reset(struct kvm_vcpu *vcpu, bool
> chain)
>  {
> +	int rc = 0;
> +	u32 ret;
> +
>  	vcpu->arch.sie_block->gpsw.mask = ~PSW_MASK_RI;
>  	vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID;
>  	memset(vcpu->run->s.regs.riccb, 0, sizeof(vcpu->run->s.regs.riccb));
> @@ -3487,11 +3490,21 @@ static int
> kvm_arch_vcpu_ioctl_normal_reset(struct kvm_vcpu *vcpu)
>  		kvm_s390_vcpu_stop(vcpu);
>  	kvm_s390_clear_local_irqs(vcpu);
> 
> -	return 0;
> +	if (kvm_s390_pv_handle_cpu(vcpu) && !chain) {
> +		rc = uv_cmd_nodata(kvm_s390_pv_handle_cpu(vcpu),
> +				   UVC_CMD_CPU_RESET, &ret);
> +		VCPU_EVENT(vcpu, 3, "PROTVIRT RESET NORMAL VCPU: cpu %d rc %x rrc %x",
> +			   vcpu->vcpu_id, ret >> 16, ret & 0x0000ffff);
> +	}
> +
> +	return rc;
>  }
[...]
> @@ -4738,12 +4767,16 @@ long kvm_arch_vcpu_ioctl(struct file *filp,
> 
>  	case KVM_S390_CLEAR_RESET:
>  		r = kvm_arch_vcpu_ioctl_clear_reset(vcpu);
> +		if (r)
> +			break;
>  		/* fallthrough */
>  	case KVM_S390_INITIAL_RESET:
> -		r = kvm_arch_vcpu_ioctl_initial_reset(vcpu);
> +		r = kvm_arch_vcpu_ioctl_initial_reset(vcpu, ioctl !=
> KVM_S390_INITIAL_RESET);
> +		if (r)
> +			break;
>  		/* fallthrough */
>  	case KVM_S390_NORMAL_RESET:
> -		r = kvm_arch_vcpu_ioctl_normal_reset(vcpu);
> +		r = kvm_arch_vcpu_ioctl_normal_reset(vcpu, ioctl !=
> KVM_S390_NORMAL_RESET);
>  		break;
>  	case KVM_SET_ONE_REG:
>  	case KVM_GET_ONE_REG: {
> 

As you said, it's mostly a matter of taste, but at least in my eyes this
approach with fallthroughs and the additional parameter looks rather
harder to understand compared to what I've suggested.

 Thomas

Re: [PATCH v4] KVM: s390: Add new reset vcpu API

[Christian Borntraeger]

On 10.01.20 09:49, Thomas Huth wrote:

> 
> As you said, it's mostly a matter of taste, but at least in my eyes this
> approach with fallthroughs and the additional parameter looks rather
> harder to understand compared to what I've suggested.

Agreed. A parameter makes the code harder to grasp.

Re: [PATCH v4] KVM: s390: Add new reset vcpu API

[Cornelia Huck]

[-- Attachment #1: Type: text/plain, Size: 3255 bytes --]

On Fri, 10 Jan 2020 09:43:33 +0100
Janosch Frank <frankja@linux.ibm.com> wrote:

> On 1/10/20 8:14 AM, Janosch Frank wrote:
> > On 1/10/20 8:03 AM, Thomas Huth wrote:  
> >> On 09/01/2020 18.51, Janosch Frank wrote:  

> >>> Reworded explanation:
> >>> I can't use a fallthrough, because the UV will reject the normal reset
> >>> if we do an initial reset (same goes for the clear reset). To address
> >>> this issue, I added a boolean to the normal and initial reset functions
> >>> which tells the function if it was called directly or was called because
> >>> of the fallthrough.
> >>>
> >>> Only if called directly a UV call for the reset is done, that way we can
> >>> keep the fallthrough.  
> >>
> >> Sounds complicated. And do we need the fallthrough stuff here at all?
> >> What about doing something like:  
> > 
> > That would work and I thought about it, it just comes down to taste :-)
> > I don't have any strong feelings for a specific implementation.  

(...)

> +	if (kvm_s390_pv_handle_cpu(vcpu) && !chain) {

I find this 'chain' thingy a bit unwieldy...

> +		rc = uv_cmd_nodata(kvm_s390_pv_handle_cpu(vcpu),
> +				   UVC_CMD_CPU_RESET, &ret);
> +		VCPU_EVENT(vcpu, 3, "PROTVIRT RESET NORMAL VCPU: cpu %d rc %x rrc %x",
> +			   vcpu->vcpu_id, ret >> 16, ret & 0x0000ffff);
> +	}
> +
> +	return rc;
>  }

(...)

> 
>  	case KVM_S390_CLEAR_RESET:
>  		r = kvm_arch_vcpu_ioctl_clear_reset(vcpu);
> +		if (r)
> +			break;
>  		/* fallthrough */
>  	case KVM_S390_INITIAL_RESET:
> -		r = kvm_arch_vcpu_ioctl_initial_reset(vcpu);
> +		r = kvm_arch_vcpu_ioctl_initial_reset(vcpu, ioctl !=
> KVM_S390_INITIAL_RESET);
> +		if (r)
> +			break;
>  		/* fallthrough */
>  	case KVM_S390_NORMAL_RESET:
> -		r = kvm_arch_vcpu_ioctl_normal_reset(vcpu);
> +		r = kvm_arch_vcpu_ioctl_normal_reset(vcpu, ioctl !=
> KVM_S390_NORMAL_RESET);

...especially looking at the invocations.

>  		break;
>  	case KVM_SET_ONE_REG:
>  	case KVM_GET_ONE_REG: {
> 

<bikeshed>
What about the following?

static void _do_normal_reset(struct kvm_vcpu *vcpu)
{
	/* do normal reset */
}

static int kvm_arch_vcpu_ioctl_normal_reset(struct kvm_vcpu *vcpu)
{
	_do_normal_reset(vcpu);
        if (kvm_s390_pv_handle_cpu(vcpu)) {
		/* do protected virt normal reset */
	}
}

static void _do_initial_reset(struct kvm_vcpu *vcpu)
{
	/* do initial reset */
}

static int kvm_arch_vcpu_ioctl_initial_reset(struct kvm_vcpu *vcpu)
{
	_do_initial_reset(vcpu);
	if (kvm_set_pv_handle_cpu(vcpu)) {
		/* do protected virt initial reset */
	}
	_do_normal_reset(vcpu);
}

static void _do_clear_reset(struct kvm_vcpu *vcpu)
{
	/* do clear reset */
}

static int kvm_arch_vcpu_ioctl_clear_reset(struct kvm_vcpu *vcpu)
{
	_do_clear_reset(vcpu);
	if (kvm_set_pv_handle_cpu(vcpu)) {
		/* do protected virt clear reset */
	}
	_do_initial_reset(vcpu);
	_do_normal_reset(vcpu);
}

And call the *_ioctl_* functions directly without fallthrough.

The nice thing about this is that it makes the call chain explicit and
does not require parameters.

The drawback is that we need more functions, and that it looks a bit
overcomplicated before the pv stuff is added.

</bikeshed>

[-- Attachment #2: OpenPGP digital signature --]
[-- Type: application/pgp-signature, Size: 833 bytes --]

Re: [PATCH v4] KVM: s390: Add new reset vcpu API

[Thomas Huth]

On 05/12/2019 13.28, Janosch Frank wrote:
> The architecture states that we need to reset local IRQs for all CPU
> resets. Because the old reset interface did not support the normal CPU
> reset we never did that. Now that we have a new interface, let's
> properly clear out local IRQs.
> 
> Also we add a ioctl for the clear reset to have all resets exposed to
> userspace. Currently the clear reset falls back to the initial reset,
> but we plan to have clear reset specific code in the future.
> 
> Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
> Reviewed-by: David Hildenbrand <david@redhat.com>
> ---
>  Documentation/virt/kvm/api.txt | 48 ++++++++++++++++++++++++++++++++++
>  arch/s390/kvm/kvm-s390.c       | 14 ++++++++++
>  include/uapi/linux/kvm.h       |  5 ++++
>  3 files changed, 67 insertions(+)
[...]
> +4.123 KVM_S390_CLEAR_RESET
> +
> +Capability: KVM_CAP_S390_VCPU_RESETS
> +Architectures: s390
> +Type: vcpu ioctl
> +Parameters: none
> +Returns: 0
> +
> +This ioctl resets VCPU registers and control structures that userspace
> +can't access via the kvm_run structure. It is intended to be called
> +when an initial reset (which is a superset of the normal reset) is
> +performed on the vcpu and additionally clears general, access,
> +floating and vector registers.

So now you've documented that this ioctl clears the GPRs, ARs and
FRs/VRs ... but the implementation does not! That's quite ugly.
Can you please state clearly that it is the job of userspace to clear
these registers (in the default, non-protected case) and that this ioctl
should be called on top?

Same problem with the PSW bit 24 and riccb during normal reset.

Or should the kernel code maybe also clear these in addition to
userspace, just to be in line with the initial reset?

 Thomas

Re: [PATCH v4] KVM: s390: Add new reset vcpu API

[Cornelia Huck]

On Thu,  5 Dec 2019 07:28:10 -0500
Janosch Frank <frankja@linux.ibm.com> wrote:

> The architecture states that we need to reset local IRQs for all CPU
> resets. Because the old reset interface did not support the normal CPU
> reset we never did that. Now that we have a new interface, let's
> properly clear out local IRQs.
> 
> Also we add a ioctl for the clear reset to have all resets exposed to
> userspace. Currently the clear reset falls back to the initial reset,
> but we plan to have clear reset specific code in the future.
> 
> Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
> Reviewed-by: David Hildenbrand <david@redhat.com>
> ---
>  Documentation/virt/kvm/api.txt | 48 ++++++++++++++++++++++++++++++++++
>  arch/s390/kvm/kvm-s390.c       | 14 ++++++++++
>  include/uapi/linux/kvm.h       |  5 ++++
>  3 files changed, 67 insertions(+)
> 
> diff --git a/Documentation/virt/kvm/api.txt b/Documentation/virt/kvm/api.txt
> index 4833904d32a5..296e51f9df70 100644
> --- a/Documentation/virt/kvm/api.txt
> +++ b/Documentation/virt/kvm/api.txt
> @@ -4126,6 +4126,47 @@ Valid values for 'action':
>  #define KVM_PMU_EVENT_ALLOW 0
>  #define KVM_PMU_EVENT_DENY 1
>  
> +4.121 KVM_S390_NORMAL_RESET
> +
> +Capability: KVM_CAP_S390_VCPU_RESETS
> +Architectures: s390
> +Type: vcpu ioctl
> +Parameters: none
> +Returns: 0
> +
> +This ioctl resets VCPU registers and control structures that userspace
> +can't access via the kvm_run structure. It is intended to be called
> +when a normal reset is performed on the vcpu and clears local
> +interrupts, the riccb and PSW bit 24.
> +
> +4.122 KVM_S390_INITIAL_RESET
> +
> +Capability: none
> +Architectures: s390
> +Type: vcpu ioctl
> +Parameters: none
> +Returns: 0
> +
> +This ioctl resets VCPU registers and control structures that userspace
> +can't access via the kvm_run structure. It is intended to be called
> +when an initial reset (which is a superset of the normal reset) is
> +performed on the vcpu and additionally clears the psw, prefix, timing
> +related registers, as well as setting the control registers to their
> +initial value.
> +
> +4.123 KVM_S390_CLEAR_RESET
> +
> +Capability: KVM_CAP_S390_VCPU_RESETS
> +Architectures: s390
> +Type: vcpu ioctl
> +Parameters: none
> +Returns: 0
> +
> +This ioctl resets VCPU registers and control structures that userspace
> +can't access via the kvm_run structure. It is intended to be called
> +when an initial reset (which is a superset of the normal reset) is

s/initial/clear/
s/normal/initial/

(no need to respin, just fix up while applying :)

> +performed on the vcpu and additionally clears general, access,
> +floating and vector registers.
>  
>  5. The kvm_run structure
>  ------------------------
> @@ -5322,3 +5363,10 @@ handling by KVM (as some KVM hypercall may be mistakenly treated as TLB
>  flush hypercalls by Hyper-V) so userspace should disable KVM identification
>  in CPUID and only exposes Hyper-V identification. In this case, guest
>  thinks it's running on Hyper-V and only use Hyper-V hypercalls.
> +
> +8.22 KVM_CAP_S390_VCPU_RESETS
> +
> +Architectures: s390
> +
> +This capability indicates that the KVM_S390_NORMAL_RESET and
> +KVM_S390_CLEAR_RESET ioctls are available.

Reviewed-by: Cornelia Huck <cohuck@redhat.com>

[PATCH v4] KVM: s390: Add new reset vcpu API

[Janosch Frank]

The architecture states that we need to reset local IRQs for all CPU
resets. Because the old reset interface did not support the normal CPU
reset we never did that. Now that we have a new interface, let's
properly clear out local IRQs.

Also we add a ioctl for the clear reset to have all resets exposed to
userspace. Currently the clear reset falls back to the initial reset,
but we plan to have clear reset specific code in the future.

Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
Reviewed-by: David Hildenbrand <david@redhat.com>
---
 Documentation/virt/kvm/api.txt | 48 ++++++++++++++++++++++++++++++++++
 arch/s390/kvm/kvm-s390.c       | 14 ++++++++++
 include/uapi/linux/kvm.h       |  5 ++++
 3 files changed, 67 insertions(+)

diff --git a/Documentation/virt/kvm/api.txt b/Documentation/virt/kvm/api.txt
index 4833904d32a5..296e51f9df70 100644
--- a/Documentation/virt/kvm/api.txt
+++ b/Documentation/virt/kvm/api.txt
@@ -4126,6 +4126,47 @@ Valid values for 'action':
 #define KVM_PMU_EVENT_ALLOW 0
 #define KVM_PMU_EVENT_DENY 1
 
+4.121 KVM_S390_NORMAL_RESET
+
+Capability: KVM_CAP_S390_VCPU_RESETS
+Architectures: s390
+Type: vcpu ioctl
+Parameters: none
+Returns: 0
+
+This ioctl resets VCPU registers and control structures that userspace
+can't access via the kvm_run structure. It is intended to be called
+when a normal reset is performed on the vcpu and clears local
+interrupts, the riccb and PSW bit 24.
+
+4.122 KVM_S390_INITIAL_RESET
+
+Capability: none
+Architectures: s390
+Type: vcpu ioctl
+Parameters: none
+Returns: 0
+
+This ioctl resets VCPU registers and control structures that userspace
+can't access via the kvm_run structure. It is intended to be called
+when an initial reset (which is a superset of the normal reset) is
+performed on the vcpu and additionally clears the psw, prefix, timing
+related registers, as well as setting the control registers to their
+initial value.
+
+4.123 KVM_S390_CLEAR_RESET
+
+Capability: KVM_CAP_S390_VCPU_RESETS
+Architectures: s390
+Type: vcpu ioctl
+Parameters: none
+Returns: 0
+
+This ioctl resets VCPU registers and control structures that userspace
+can't access via the kvm_run structure. It is intended to be called
+when an initial reset (which is a superset of the normal reset) is
+performed on the vcpu and additionally clears general, access,
+floating and vector registers.
 
 5. The kvm_run structure
 ------------------------
@@ -5322,3 +5363,10 @@ handling by KVM (as some KVM hypercall may be mistakenly treated as TLB
 flush hypercalls by Hyper-V) so userspace should disable KVM identification
 in CPUID and only exposes Hyper-V identification. In this case, guest
 thinks it's running on Hyper-V and only use Hyper-V hypercalls.
+
+8.22 KVM_CAP_S390_VCPU_RESETS
+
+Architectures: s390
+
+This capability indicates that the KVM_S390_NORMAL_RESET and
+KVM_S390_CLEAR_RESET ioctls are available.
diff --git a/arch/s390/kvm/kvm-s390.c b/arch/s390/kvm/kvm-s390.c
index d9e6bf3d54f0..7f3ede0b2715 100644
--- a/arch/s390/kvm/kvm-s390.c
+++ b/arch/s390/kvm/kvm-s390.c
@@ -529,6 +529,7 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)
 	case KVM_CAP_S390_CMMA_MIGRATION:
 	case KVM_CAP_S390_AIS:
 	case KVM_CAP_S390_AIS_MIGRATION:
+	case KVM_CAP_S390_VCPU_RESETS:
 		r = 1;
 		break;
 	case KVM_CAP_S390_HPAGE_1M:
@@ -3287,6 +3288,13 @@ static int kvm_arch_vcpu_ioctl_set_one_reg(struct kvm_vcpu *vcpu,
 	return r;
 }
 
+static int kvm_arch_vcpu_ioctl_normal_reset(struct kvm_vcpu *vcpu)
+{
+	kvm_clear_async_pf_completion_queue(vcpu);
+	kvm_s390_clear_local_irqs(vcpu);
+	return 0;
+}
+
 static int kvm_arch_vcpu_ioctl_initial_reset(struct kvm_vcpu *vcpu)
 {
 	kvm_s390_vcpu_initial_reset(vcpu);
@@ -4363,9 +4371,15 @@ long kvm_arch_vcpu_ioctl(struct file *filp,
 		r = kvm_arch_vcpu_ioctl_set_initial_psw(vcpu, psw);
 		break;
 	}
+
+	case KVM_S390_CLEAR_RESET:
+		/* fallthrough */
 	case KVM_S390_INITIAL_RESET:
 		r = kvm_arch_vcpu_ioctl_initial_reset(vcpu);
 		break;
+	case KVM_S390_NORMAL_RESET:
+		r = kvm_arch_vcpu_ioctl_normal_reset(vcpu);
+		break;
 	case KVM_SET_ONE_REG:
 	case KVM_GET_ONE_REG: {
 		struct kvm_one_reg reg;
diff --git a/include/uapi/linux/kvm.h b/include/uapi/linux/kvm.h
index 52641d8ca9e8..edbb2da43f02 100644
--- a/include/uapi/linux/kvm.h
+++ b/include/uapi/linux/kvm.h
@@ -1000,6 +1000,7 @@ struct kvm_ppc_resize_hpt {
 #define KVM_CAP_PMU_EVENT_FILTER 173
 #define KVM_CAP_ARM_IRQ_LINE_LAYOUT_2 174
 #define KVM_CAP_HYPERV_DIRECT_TLBFLUSH 175
+#define KVM_CAP_S390_VCPU_RESETS 176
 
 #ifdef KVM_CAP_IRQ_ROUTING
 
@@ -1461,6 +1462,10 @@ struct kvm_enc_region {
 /* Available with KVM_CAP_ARM_SVE */
 #define KVM_ARM_VCPU_FINALIZE	  _IOW(KVMIO,  0xc2, int)
 
+/* Available with  KVM_CAP_S390_VCPU_RESETS */
+#define KVM_S390_NORMAL_RESET	_IO(KVMIO,   0xc3)
+#define KVM_S390_CLEAR_RESET	_IO(KVMIO,   0xc4)
+
 /* Secure Encrypted Virtualization command */
 enum sev_cmd_id {
 	/* Guest initialization commands */
-- 
2.20.1