Re: [PATCH v3 14/22] KVM: Move out KVM arch PM hooks and hardware enable/disable logic

[Isaku Yamahata <isaku.yamahata@gmail.com>]

On Tue, Sep 06, 2022 at 03:43:58PM +0800,
Yuan Yao <yuan.yao@linux.intel.com> wrote:

> On Thu, Sep 01, 2022 at 07:17:49PM -0700, isaku.yamahata@intel.com wrote:
> > From: Isaku Yamahata <isaku.yamahata@intel.com>
> >
> > To make clear that those files are default implementation that KVM/x86 (and
> > other KVM arch in future) will override them, split out those into a single
> > file. Once conversions for all kvm archs are done, the file will be
> > deleted.  kvm_arch_pre_hardware_unsetup() is introduced to avoid cross-arch
> > code churn for now.  Once it's settled down,
> > kvm_arch_pre_hardware_unsetup() can be merged into
> > kvm_arch_hardware_unsetup() in each arch code.
> >
> > Signed-off-by: Isaku Yamahata <isaku.yamahata@intel.com>
> > ---
> >  include/linux/kvm_host.h |   1 +
> >  virt/kvm/kvm_arch.c      | 103 ++++++++++++++++++++++-
> >  virt/kvm/kvm_main.c      | 172 +++++----------------------------------
> >  3 files changed, 124 insertions(+), 152 deletions(-)
> >
> > diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h
> > index f78364e01ca9..60f4ae9d6f48 100644
> > --- a/include/linux/kvm_host.h
> > +++ b/include/linux/kvm_host.h
> > @@ -1437,6 +1437,7 @@ static inline void kvm_create_vcpu_debugfs(struct kvm_vcpu *vcpu) {}
> >  int kvm_arch_hardware_enable(void);
> >  void kvm_arch_hardware_disable(void);
> >  int kvm_arch_hardware_setup(void *opaque);
> > +void kvm_arch_pre_hardware_unsetup(void);
> >  void kvm_arch_hardware_unsetup(void);
> >  int kvm_arch_check_processor_compat(void);
> >  int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu);
> > diff --git a/virt/kvm/kvm_arch.c b/virt/kvm/kvm_arch.c
> > index 0eac996f4981..0648d4463d9e 100644
> > --- a/virt/kvm/kvm_arch.c
> > +++ b/virt/kvm/kvm_arch.c
> > @@ -6,49 +6,148 @@
> >   * Author:
> >   *   Isaku Yamahata <isaku.yamahata@intel.com>
> >   *                  <isaku.yamahata@gmail.com>
> > + *
> > + * TODO: Delete this file once the conversion of all KVM arch is done.
> >   */
> >
> >  #include <linux/kvm_host.h>
> >
> > +static cpumask_t cpus_hardware_enabled = CPU_MASK_NONE;
> > +static atomic_t hardware_enable_failed;
> > +
> >  __weak int kvm_arch_post_init_vm(struct kvm *kvm)
> >  {
> >  	return 0;
> >  }
> >
> > +static void hardware_enable_nolock(void *caller_name)
> > +{
> > +	int cpu = raw_smp_processor_id();
> > +	int r;
> > +
> > +	WARN_ON_ONCE(preemptible());
> > +
> > +	if (cpumask_test_cpu(cpu, &cpus_hardware_enabled))
> > +		return;
> > +
> > +	cpumask_set_cpu(cpu, &cpus_hardware_enabled);
> > +
> > +	r = kvm_arch_hardware_enable();
> > +
> > +	if (r) {
> > +		cpumask_clear_cpu(cpu, &cpus_hardware_enabled);
> > +		atomic_inc(&hardware_enable_failed);
> > +		pr_warn("kvm: enabling virtualization on CPU%d failed during %s()\n",
> > +			cpu, (const char *)caller_name);
> > +	}
> > +}
> > +
> > +static void hardware_disable_nolock(void *junk)
> > +{
> > +	int cpu = raw_smp_processor_id();
> > +
> > +	WARN_ON_ONCE(preemptible());
> > +
> > +	if (!cpumask_test_cpu(cpu, &cpus_hardware_enabled))
> > +		return;
> > +	cpumask_clear_cpu(cpu, &cpus_hardware_enabled);
> > +	kvm_arch_hardware_disable();
> > +}
> > +
> > +__weak void kvm_arch_pre_hardware_unsetup(void)
> > +{
> > +	on_each_cpu(hardware_disable_nolock, NULL, 1);
> > +}
> > +
> >  /*
> >   * Called after the VM is otherwise initialized, but just before adding it to
> >   * the vm_list.
> >   */
> >  __weak int kvm_arch_add_vm(struct kvm *kvm, int usage_count)
> >  {
> > -	return kvm_arch_post_init_vm(kvm);
> > +	int r = 0;
> > +
> > +	if (usage_count != 1)
> > +		return 0;
> > +
> > +	atomic_set(&hardware_enable_failed, 0);
> > +	on_each_cpu(hardware_enable_nolock, (void *)__func__, 1);
> 
> 
> This function is called in kvm_create_vm:
> 
>  kvm_create_vm {
>  ...
>    enable_hardware_all()
>  ...
>    kvm_arch_add_vm()
>  ...
> }
> 
> so don't need on_each_cpu(enable_hardware_nolock) here, or the
> enable_hardware_all() shuold be removed from kvm_create_vm().


Yes, it's removed. Please notice the following hunk.

@@ -1196,10 +1191,6 @@ static struct kvm *kvm_create_vm(unsigned long type, const char *fdname)
      if (r)
              goto out_err_no_arch_destroy_vm;

-     r = hardware_enable_all();
-     if (r)
-             goto out_err_no_disable;
-
 #ifdef CONFIG_HAVE_KVM_IRQFD
      INIT_HLIST_HEAD(&kvm->irq_ack_notifier_list);
 #endif
-- 
Isaku Yamahata <isaku.yamahata@gmail.com>