Re: [PATCH v9 16/21] KVM: s390: pci: add routines to start/stop interpretive execution

[Pierre Morel <pmorel@linux.ibm.com>]

On 6/6/22 22:33, Matthew Rosato wrote:
> These routines will be invoked at the time an s390x vfio-pci device is
> associated with a KVM (or when the association is removed), allowing
> the zPCI device to enable or disable load/store intepretation mode;
> this requires the host zPCI device to inform firmware of the unique
> token (GISA designation) that is associated with the owning KVM.
> 
> Signed-off-by: Matthew Rosato <mjrosato@linux.ibm.com>
> ---
>   arch/s390/include/asm/kvm_host.h |  18 ++++
>   arch/s390/include/asm/pci.h      |   1 +
>   arch/s390/kvm/kvm-s390.c         |  15 +++
>   arch/s390/kvm/pci.c              | 162 +++++++++++++++++++++++++++++++
>   arch/s390/kvm/pci.h              |   5 +
>   arch/s390/pci/pci.c              |   4 +
>   6 files changed, 205 insertions(+)
> 
> diff --git a/arch/s390/include/asm/kvm_host.h b/arch/s390/include/asm/kvm_host.h
> index 8e381603b6a7..6e83d746bae2 100644
> --- a/arch/s390/include/asm/kvm_host.h
> +++ b/arch/s390/include/asm/kvm_host.h
> @@ -19,6 +19,7 @@
>   #include <linux/kvm.h>
>   #include <linux/seqlock.h>
>   #include <linux/module.h>
> +#include <linux/pci.h>
>   #include <asm/debug.h>
>   #include <asm/cpu.h>
>   #include <asm/fpu/api.h>
> @@ -967,6 +968,8 @@ struct kvm_arch{
>   	DECLARE_BITMAP(idle_mask, KVM_MAX_VCPUS);
>   	struct kvm_s390_gisa_interrupt gisa_int;
>   	struct kvm_s390_pv pv;
> +	struct list_head kzdev_list;
> +	spinlock_t kzdev_list_lock;
>   };
>   
>   #define KVM_HVA_ERR_BAD		(-1UL)
> @@ -1017,4 +1020,19 @@ static inline void kvm_arch_flush_shadow_memslot(struct kvm *kvm,
>   static inline void kvm_arch_vcpu_blocking(struct kvm_vcpu *vcpu) {}
>   static inline void kvm_arch_vcpu_unblocking(struct kvm_vcpu *vcpu) {}
>   
> +#define __KVM_HAVE_ARCH_VM_FREE
> +void kvm_arch_free_vm(struct kvm *kvm);
> +
> +#ifdef CONFIG_VFIO_PCI_ZDEV_KVM
> +int kvm_s390_pci_register_kvm(struct zpci_dev *zdev, struct kvm *kvm);
> +void kvm_s390_pci_unregister_kvm(struct zpci_dev *zdev);
> +#else
> +static inline int kvm_s390_pci_register_kvm(struct zpci_dev *dev,
> +					    struct kvm *kvm)
> +{
> +	return -EPERM;
> +}
> +static inline void kvm_s390_pci_unregister_kvm(struct zpci_dev *dev) {}
> +#endif
> +
>   #endif
> diff --git a/arch/s390/include/asm/pci.h b/arch/s390/include/asm/pci.h
> index 322060a75d9f..85eb0ef9d4c3 100644
> --- a/arch/s390/include/asm/pci.h
> +++ b/arch/s390/include/asm/pci.h
> @@ -194,6 +194,7 @@ struct zpci_dev {
>   	/* IOMMU and passthrough */
>   	struct s390_domain *s390_domain; /* s390 IOMMU domain data */
>   	struct kvm_zdev *kzdev;
> +	struct mutex kzdev_lock;

I guess that since it did not exist before the lock is not there to 
protect the zpci_dev struct.
May be add a comment to say what it is protecting.


>   };
>   
>   static inline bool zdev_enabled(struct zpci_dev *zdev)
> diff --git a/arch/s390/kvm/kvm-s390.c b/arch/s390/kvm/kvm-s390.c
> index a66da3f66114..4758bb731199 100644
> --- a/arch/s390/kvm/kvm-s390.c
> +++ b/arch/s390/kvm/kvm-s390.c
> @@ -2790,6 +2790,14 @@ static void sca_dispose(struct kvm *kvm)
>   	kvm->arch.sca = NULL;
>   }
>   
> +void kvm_arch_free_vm(struct kvm *kvm)
> +{
> +	if (IS_ENABLED(CONFIG_VFIO_PCI_ZDEV_KVM))
> +		kvm_s390_pci_clear_list(kvm);
> +
> +	__kvm_arch_free_vm(kvm);
> +}
> +
>   int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
>   {
>   	gfp_t alloc_flags = GFP_KERNEL_ACCOUNT;
> @@ -2872,6 +2880,13 @@ int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
>   
>   	kvm_s390_crypto_init(kvm);
>   
> +	if (IS_ENABLED(CONFIG_VFIO_PCI_ZDEV_KVM)) {
> +		mutex_lock(&kvm->lock);
> +		kvm_s390_pci_init_list(kvm);
> +		kvm_s390_vcpu_pci_enable_interp(kvm);
> +		mutex_unlock(&kvm->lock);
> +	}
> +
>   	mutex_init(&kvm->arch.float_int.ais_lock);
>   	spin_lock_init(&kvm->arch.float_int.lock);
>   	for (i = 0; i < FIRQ_LIST_COUNT; i++)
> diff --git a/arch/s390/kvm/pci.c b/arch/s390/kvm/pci.c
> index b232c8cbaa81..24211741deb0 100644
> --- a/arch/s390/kvm/pci.c
> +++ b/arch/s390/kvm/pci.c
> @@ -12,7 +12,9 @@
>   #include <asm/pci.h>
>   #include <asm/pci_insn.h>
>   #include <asm/pci_io.h>
> +#include <asm/sclp.h>
>   #include "pci.h"
> +#include "kvm-s390.h"
>   
>   struct zpci_aift *aift;
>   
> @@ -423,6 +425,166 @@ static void kvm_s390_pci_dev_release(struct zpci_dev *zdev)
>   	kfree(kzdev);
>   }
>   
> +
> +/*
> + * Register device with the specified KVM. If interpetation facilities are
> + * available, enable them and let userspace indicate whether or not they will
> + * be used (specify SHM bit to disable).
> + */
> +int kvm_s390_pci_register_kvm(struct zpci_dev *zdev, struct kvm *kvm)
> +{
> +	int rc;
> +
> +	if (!zdev)
> +		return -EINVAL;
> +
> +	mutex_lock(&zdev->kzdev_lock);
> +
> +	if (zdev->kzdev || zdev->gisa != 0 || !kvm) {
> +		mutex_unlock(&zdev->kzdev_lock);
> +		return -EINVAL;
> +	}
> +
> +	kvm_get_kvm(kvm);
> +
> +	mutex_lock(&kvm->lock);

Why do we need to lock KVM here?

just a question, I do not think it is a big problem.

> +
> +	rc = kvm_s390_pci_dev_open(zdev);
> +	if (rc)
> +		goto err;
> +
> +	/*
> +	 * If interpretation facilities aren't available, add the device to
> +	 * the kzdev list but don't enable for interpretation.
> +	 */
> +	if (!kvm_s390_pci_interp_allowed())
> +		goto out;
> +
> +	/*
> +	 * If this is the first request to use an interpreted device, make the
> +	 * necessary vcpu changes
> +	 */
> +	if (!kvm->arch.use_zpci_interp)
> +		kvm_s390_vcpu_pci_enable_interp(kvm);
> +
> +	if (zdev_enabled(zdev)) {
> +		rc = zpci_disable_device(zdev);
> +		if (rc)
> +			goto err;
> +	}
> +
> +	/*
> +	 * Store information about the identity of the kvm guest allowed to
> +	 * access this device via interpretation to be used by host CLP
> +	 */
> +	zdev->gisa = (u32)virt_to_phys(&kvm->arch.sie_page2->gisa);
> +
> +	rc = zpci_enable_device(zdev);
> +	if (rc)
> +		goto clear_gisa;
> +
> +	/* Re-register the IOMMU that was already created */
> +	rc = zpci_register_ioat(zdev, 0, zdev->start_dma, zdev->end_dma,
> +				virt_to_phys(zdev->dma_table));
> +	if (rc)
> +		goto clear_gisa;
> +
> +out:
> +	zdev->kzdev->kvm = kvm;
> +
> +	spin_lock(&kvm->arch.kzdev_list_lock);
> +	list_add_tail(&zdev->kzdev->entry, &kvm->arch.kzdev_list);
> +	spin_unlock(&kvm->arch.kzdev_list_lock);
> +
> +	mutex_unlock(&kvm->lock);
> +	mutex_unlock(&zdev->kzdev_lock);
> +	return 0;
> +
> +clear_gisa:
> +	zdev->gisa = 0;
> +err:
> +	if (zdev->kzdev)
> +		kvm_s390_pci_dev_release(zdev);
> +	mutex_unlock(&kvm->lock);
> +	mutex_unlock(&zdev->kzdev_lock);
> +	kvm_put_kvm(kvm);
> +	return rc;
> +}
> +EXPORT_SYMBOL_GPL(kvm_s390_pci_register_kvm);
> +
> +void kvm_s390_pci_unregister_kvm(struct zpci_dev *zdev)
> +{
> +	struct kvm *kvm;
> +
> +	if (!zdev)
> +		return;
> +
> +	mutex_lock(&zdev->kzdev_lock);
> +
> +	if (WARN_ON(!zdev->kzdev)) {

When can this happen ?

> +		mutex_unlock(&zdev->kzdev_lock);
> +		return;
> +	}
> +
> +	kvm = zdev->kzdev->kvm;
> +	mutex_lock(&kvm->lock);
> +
> +	/*
> +	 * A 0 gisa means interpretation was never enabled, just remove the
> +	 * device from the list.
> +	 */
> +	if (zdev->gisa == 0)
> +		goto out;
> +
> +	/* Forwarding must be turned off before interpretation */
> +	if (zdev->kzdev->fib.fmt0.aibv != 0)
> +		kvm_s390_pci_aif_disable(zdev, true);
> +
> +	/* Remove the host CLP guest designation */
> +	zdev->gisa = 0;
> +
> +	if (zdev_enabled(zdev)) {
> +		if (zpci_disable_device(zdev))
> +			goto out;

NIT debug trace ?

> +	}
> +
> +	if (zpci_enable_device(zdev))
> +		goto out;

NIT debug trace?

Only some questions, otherwise, LGTM

Acked-by: Pierre Morel <pmorel@linux.ibm.com>

-- 
Pierre Morel
IBM Lab Boeblingen