[PATCH v2 0/2] disabling halt polling for nested virtualization

[PATCH v2 0/2] disabling halt polling for nested virtualization

[Christian Borntraeger]

v1->v2:
	- add kvm_vcpu parameter to kvm_arch_no_poll
	- add kvm stat in the s390 implementation
	- make the value tunable
	
Folks,

this is a very simple variant to disable halt polling when the KVM host
is already running virtualized. We could imagine more complex variants
(like tuning down the halt polling value) but this seems to do the trick
for some kvm deployment scenarios on s390x. (e.g. having multiple LPARS
with KVMs that are in itself already overcommitted). 


Christian Borntraeger (2):
  KVM: polling: add architecture backend to disable polling
  KVM: s390: provide kvm_arch_no_poll function

 arch/s390/include/asm/kvm_host.h |  1 +
 arch/s390/kvm/Kconfig            |  1 +
 arch/s390/kvm/kvm-s390.c         | 17 +++++++++++++++++
 include/linux/kvm_host.h         | 10 ++++++++++
 virt/kvm/Kconfig                 |  3 +++
 virt/kvm/kvm_main.c              |  2 +-
 6 files changed, 33 insertions(+), 1 deletion(-)

-- 
2.19.1

[PATCH v2 1/2] KVM: polling: add architecture backend to disable polling

[Christian Borntraeger]

There are cases where halt polling is unwanted. For example when running
KVM on an over committed LPAR we rather want to give back the CPU to
neighbour LPARs instead of polling. Let us provide a callback that
allows architectures to disable polling.

Signed-off-by: Christian Borntraeger <borntraeger@de.ibm.com>
Acked-by: Paolo Bonzini <pbonzini@redhat.com>
---
 include/linux/kvm_host.h | 10 ++++++++++
 virt/kvm/Kconfig         |  3 +++
 virt/kvm/kvm_main.c      |  2 +-
 3 files changed, 14 insertions(+), 1 deletion(-)

diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h
index 9d55c63db09b..b3aff1a3f633 100644
--- a/include/linux/kvm_host.h
+++ b/include/linux/kvm_host.h
@@ -1305,6 +1305,16 @@ static inline bool vcpu_valid_wakeup(struct kvm_vcpu *vcpu)
 }
 #endif /* CONFIG_HAVE_KVM_INVALID_WAKEUPS */
 
+#ifdef CONFIG_HAVE_KVM_NO_POLL
+/* Callback that tells if we must not poll */
+bool kvm_arch_no_poll(struct kvm_vcpu *vcpu);
+#else
+static inline bool kvm_arch_no_poll(struct kvm_vcpu *vcpu)
+{
+	return false;
+}
+#endif /* CONFIG_HAVE_KVM_NO_POLL */
+
 #ifdef CONFIG_HAVE_KVM_VCPU_ASYNC_IOCTL
 long kvm_arch_vcpu_async_ioctl(struct file *filp,
 			       unsigned int ioctl, unsigned long arg);
diff --git a/virt/kvm/Kconfig b/virt/kvm/Kconfig
index ea434ddc8499..aad9284c043a 100644
--- a/virt/kvm/Kconfig
+++ b/virt/kvm/Kconfig
@@ -57,3 +57,6 @@ config HAVE_KVM_VCPU_ASYNC_IOCTL
 
 config HAVE_KVM_VCPU_RUN_PID_CHANGE
        bool
+
+config HAVE_KVM_NO_POLL
+       bool
diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c
index 55fe8e20d8fd..23aec2f4ba71 100644
--- a/virt/kvm/kvm_main.c
+++ b/virt/kvm/kvm_main.c
@@ -2253,7 +2253,7 @@ void kvm_vcpu_block(struct kvm_vcpu *vcpu)
 	u64 block_ns;
 
 	start = cur = ktime_get();
-	if (vcpu->halt_poll_ns) {
+	if (vcpu->halt_poll_ns && !kvm_arch_no_poll(vcpu)) {
 		ktime_t stop = ktime_add_ns(ktime_get(), vcpu->halt_poll_ns);
 
 		++vcpu->stat.halt_attempted_poll;
-- 
2.19.1

[PATCH v2 2/2] KVM: s390: provide kvm_arch_no_poll function

[Christian Borntraeger]

We do track the current steal time of the host CPUs. Let us use
this value to disable halt polling if the steal time goes beyond
a configured value.

Signed-off-by: Christian Borntraeger <borntraeger@de.ibm.com>
Acked-by: Paolo Bonzini <pbonzini@redhat.com>
---
 arch/s390/include/asm/kvm_host.h |  1 +
 arch/s390/kvm/Kconfig            |  1 +
 arch/s390/kvm/kvm-s390.c         | 17 +++++++++++++++++
 3 files changed, 19 insertions(+)

diff --git a/arch/s390/include/asm/kvm_host.h b/arch/s390/include/asm/kvm_host.h
index e70f6ce14267..dbe254847e0d 100644
--- a/arch/s390/include/asm/kvm_host.h
+++ b/arch/s390/include/asm/kvm_host.h
@@ -313,6 +313,7 @@ struct kvm_vcpu_stat {
 	u64 halt_successful_poll;
 	u64 halt_attempted_poll;
 	u64 halt_poll_invalid;
+	u64 halt_no_poll_steal;
 	u64 halt_wakeup;
 	u64 instruction_lctl;
 	u64 instruction_lctlg;
diff --git a/arch/s390/kvm/Kconfig b/arch/s390/kvm/Kconfig
index 767453faacfc..e987e73738b5 100644
--- a/arch/s390/kvm/Kconfig
+++ b/arch/s390/kvm/Kconfig
@@ -31,6 +31,7 @@ config KVM
 	select HAVE_KVM_IRQFD
 	select HAVE_KVM_IRQ_ROUTING
 	select HAVE_KVM_INVALID_WAKEUPS
+	select HAVE_KVM_NO_POLL
 	select SRCU
 	select KVM_VFIO
 	---help---
diff --git a/arch/s390/kvm/kvm-s390.c b/arch/s390/kvm/kvm-s390.c
index d7423fdf5f68..8d0e2416f905 100644
--- a/arch/s390/kvm/kvm-s390.c
+++ b/arch/s390/kvm/kvm-s390.c
@@ -75,6 +75,7 @@ struct kvm_stats_debugfs_item debugfs_entries[] = {
 	{ "halt_successful_poll", VCPU_STAT(halt_successful_poll) },
 	{ "halt_attempted_poll", VCPU_STAT(halt_attempted_poll) },
 	{ "halt_poll_invalid", VCPU_STAT(halt_poll_invalid) },
+	{ "halt_no_poll_steal", VCPU_STAT(halt_no_poll_steal) },
 	{ "halt_wakeup", VCPU_STAT(halt_wakeup) },
 	{ "instruction_lctlg", VCPU_STAT(instruction_lctlg) },
 	{ "instruction_lctl", VCPU_STAT(instruction_lctl) },
@@ -182,6 +183,11 @@ static int hpage;
 module_param(hpage, int, 0444);
 MODULE_PARM_DESC(hpage, "1m huge page backing support");
 
+/* maximum percentage of steal time for polling */
+static u8 halt_poll_max_steal = 10;
+module_param(halt_poll_max_steal, byte, 0644);
+MODULE_PARM_DESC(hpage, "Maximum percentage of steal time to allow pollinh");
+
 /*
  * For now we handle at most 16 double words as this is what the s390 base
  * kernel handles and stores in the prefix page. If we ever need to go beyond
@@ -3150,6 +3156,17 @@ static void kvm_gmap_notifier(struct gmap *gmap, unsigned long start,
 	}
 }
 
+bool kvm_arch_no_poll(struct kvm_vcpu *vcpu)
+{
+	/* do not poll with more than halt_poll_max_steal percent of steal time */
+	if (S390_lowcore.avg_steal_timer * 100 / (TICK_USEC << 12) >=
+	    halt_poll_max_steal) {
+		vcpu->stat.halt_no_poll_steal++;
+		return true;
+	}
+	return false;
+}
+
 int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu)
 {
 	/* kvm common code refers to this, but never calls it */
-- 
2.19.1

Re: [PATCH v2 2/2] KVM: s390: provide kvm_arch_no_poll function

[Christian Borntraeger]

FWiw, this patch need 5.1-rc3 or later.

On 16.04.19 20:01, Christian Borntraeger wrote:
> We do track the current steal time of the host CPUs. Let us use
> this value to disable halt polling if the steal time goes beyond
> a configured value.
> 
> Signed-off-by: Christian Borntraeger <borntraeger@de.ibm.com>
> Acked-by: Paolo Bonzini <pbonzini@redhat.com>
> ---
>  arch/s390/include/asm/kvm_host.h |  1 +
>  arch/s390/kvm/Kconfig            |  1 +
>  arch/s390/kvm/kvm-s390.c         | 17 +++++++++++++++++
>  3 files changed, 19 insertions(+)
> 
> diff --git a/arch/s390/include/asm/kvm_host.h b/arch/s390/include/asm/kvm_host.h
> index e70f6ce14267..dbe254847e0d 100644
> --- a/arch/s390/include/asm/kvm_host.h
> +++ b/arch/s390/include/asm/kvm_host.h
> @@ -313,6 +313,7 @@ struct kvm_vcpu_stat {
>  	u64 halt_successful_poll;
>  	u64 halt_attempted_poll;
>  	u64 halt_poll_invalid;
> +	u64 halt_no_poll_steal;
>  	u64 halt_wakeup;
>  	u64 instruction_lctl;
>  	u64 instruction_lctlg;
> diff --git a/arch/s390/kvm/Kconfig b/arch/s390/kvm/Kconfig
> index 767453faacfc..e987e73738b5 100644
> --- a/arch/s390/kvm/Kconfig
> +++ b/arch/s390/kvm/Kconfig
> @@ -31,6 +31,7 @@ config KVM
>  	select HAVE_KVM_IRQFD
>  	select HAVE_KVM_IRQ_ROUTING
>  	select HAVE_KVM_INVALID_WAKEUPS
> +	select HAVE_KVM_NO_POLL
>  	select SRCU
>  	select KVM_VFIO
>  	---help---
> diff --git a/arch/s390/kvm/kvm-s390.c b/arch/s390/kvm/kvm-s390.c
> index d7423fdf5f68..8d0e2416f905 100644
> --- a/arch/s390/kvm/kvm-s390.c
> +++ b/arch/s390/kvm/kvm-s390.c
> @@ -75,6 +75,7 @@ struct kvm_stats_debugfs_item debugfs_entries[] = {
>  	{ "halt_successful_poll", VCPU_STAT(halt_successful_poll) },
>  	{ "halt_attempted_poll", VCPU_STAT(halt_attempted_poll) },
>  	{ "halt_poll_invalid", VCPU_STAT(halt_poll_invalid) },
> +	{ "halt_no_poll_steal", VCPU_STAT(halt_no_poll_steal) },
>  	{ "halt_wakeup", VCPU_STAT(halt_wakeup) },
>  	{ "instruction_lctlg", VCPU_STAT(instruction_lctlg) },
>  	{ "instruction_lctl", VCPU_STAT(instruction_lctl) },
> @@ -182,6 +183,11 @@ static int hpage;
>  module_param(hpage, int, 0444);
>  MODULE_PARM_DESC(hpage, "1m huge page backing support");
>  
> +/* maximum percentage of steal time for polling */
> +static u8 halt_poll_max_steal = 10;
> +module_param(halt_poll_max_steal, byte, 0644);
> +MODULE_PARM_DESC(hpage, "Maximum percentage of steal time to allow pollinh");
> +
>  /*
>   * For now we handle at most 16 double words as this is what the s390 base
>   * kernel handles and stores in the prefix page. If we ever need to go beyond
> @@ -3150,6 +3156,17 @@ static void kvm_gmap_notifier(struct gmap *gmap, unsigned long start,
>  	}
>  }
>  
> +bool kvm_arch_no_poll(struct kvm_vcpu *vcpu)
> +{
> +	/* do not poll with more than halt_poll_max_steal percent of steal time */
> +	if (S390_lowcore.avg_steal_timer * 100 / (TICK_USEC << 12) >=
> +	    halt_poll_max_steal) {
> +		vcpu->stat.halt_no_poll_steal++;
> +		return true;
> +	}
> +	return false;
> +}
> +
>  int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu)
>  {
>  	/* kvm common code refers to this, but never calls it */
> 

Re: [PATCH v2 0/2] disabling halt polling for nested virtualization

[Paolo Bonzini]

On 16/04/19 20:01, Christian Borntraeger wrote:
> v1->v2:
> 	- add kvm_vcpu parameter to kvm_arch_no_poll
> 	- add kvm stat in the s390 implementation
> 	- make the value tunable
> 	
> Folks,
> 
> this is a very simple variant to disable halt polling when the KVM host
> is already running virtualized. We could imagine more complex variants
> (like tuning down the halt polling value) but this seems to do the trick
> for some kvm deployment scenarios on s390x. (e.g. having multiple LPARS
> with KVMs that are in itself already overcommitted). 

Ok to queue this for 5.2?

(Stupid question, I thought LPARs are partitions and as such were never
overcommitted. What is it that separates an LPAR from a zVM or KVM guest?)

Paolo

Re: [PATCH v2 0/2] disabling halt polling for nested virtualization

[Christian Borntraeger]

On 17.04.19 15:20, Paolo Bonzini wrote:
> On 16/04/19 20:01, Christian Borntraeger wrote:
>> v1->v2:
>> 	- add kvm_vcpu parameter to kvm_arch_no_poll
>> 	- add kvm stat in the s390 implementation
>> 	- make the value tunable
>> 	
>> Folks,
>>
>> this is a very simple variant to disable halt polling when the KVM host
>> is already running virtualized. We could imagine more complex variants
>> (like tuning down the halt polling value) but this seems to do the trick
>> for some kvm deployment scenarios on s390x. (e.g. having multiple LPARS
>> with KVMs that are in itself already overcommitted). 
> 
> Ok to queue this for 5.2?

Yes, after review. I can queue that in the s390 tree if you want.
 
> (Stupid question, I thought LPARs are partitions and as such were never
> overcommitted. What is it that separates an LPAR from a zVM or KVM guest?)

LPAR can do CPU overcommitment as well (or dedicated whatever you configure). Its the
memory that is hard partitioned on LPAR.

Re: [PATCH v2 0/2] disabling halt polling for nested virtualization

[Paolo Bonzini]

On 17/04/19 15:23, Christian Borntraeger wrote:
> 
> 
> On 17.04.19 15:20, Paolo Bonzini wrote:
>> On 16/04/19 20:01, Christian Borntraeger wrote:
>>> v1->v2:
>>> 	- add kvm_vcpu parameter to kvm_arch_no_poll
>>> 	- add kvm stat in the s390 implementation
>>> 	- make the value tunable
>>> 	
>>> Folks,
>>>
>>> this is a very simple variant to disable halt polling when the KVM host
>>> is already running virtualized. We could imagine more complex variants
>>> (like tuning down the halt polling value) but this seems to do the trick
>>> for some kvm deployment scenarios on s390x. (e.g. having multiple LPARS
>>> with KVMs that are in itself already overcommitted). 
>>
>> Ok to queue this for 5.2?
> 
> Yes, after review. I can queue that in the s390 tree if you want.

Sure, that's okay.

Paolo

>> (Stupid question, I thought LPARs are partitions and as such were never
>> overcommitted. What is it that separates an LPAR from a zVM or KVM guest?)
> 
> LPAR can do CPU overcommitment as well (or dedicated whatever you configure). Its the
> memory that is hard partitioned on LPAR.
> 

Re: [PATCH v2 1/2] KVM: polling: add architecture backend to disable polling

[Cornelia Huck]

On Tue, 16 Apr 2019 20:01:57 +0200
Christian Borntraeger <borntraeger@de.ibm.com> wrote:

> There are cases where halt polling is unwanted. For example when running
> KVM on an over committed LPAR we rather want to give back the CPU to
> neighbour LPARs instead of polling. Let us provide a callback that
> allows architectures to disable polling.
> 
> Signed-off-by: Christian Borntraeger <borntraeger@de.ibm.com>
> Acked-by: Paolo Bonzini <pbonzini@redhat.com>
> ---
>  include/linux/kvm_host.h | 10 ++++++++++
>  virt/kvm/Kconfig         |  3 +++
>  virt/kvm/kvm_main.c      |  2 +-
>  3 files changed, 14 insertions(+), 1 deletion(-)

Reviewed-by: Cornelia Huck <cohuck@redhat.com>

Re: [PATCH v2 2/2] KVM: s390: provide kvm_arch_no_poll function

[Cornelia Huck]

On Tue, 16 Apr 2019 20:01:58 +0200
Christian Borntraeger <borntraeger@de.ibm.com> wrote:

> We do track the current steal time of the host CPUs. Let us use
> this value to disable halt polling if the steal time goes beyond
> a configured value.
> 
> Signed-off-by: Christian Borntraeger <borntraeger@de.ibm.com>
> Acked-by: Paolo Bonzini <pbonzini@redhat.com>
> ---
>  arch/s390/include/asm/kvm_host.h |  1 +
>  arch/s390/kvm/Kconfig            |  1 +
>  arch/s390/kvm/kvm-s390.c         | 17 +++++++++++++++++
>  3 files changed, 19 insertions(+)
> 

> @@ -182,6 +183,11 @@ static int hpage;
>  module_param(hpage, int, 0444);
>  MODULE_PARM_DESC(hpage, "1m huge page backing support");
>  
> +/* maximum percentage of steal time for polling */
> +static u8 halt_poll_max_steal = 10;
> +module_param(halt_poll_max_steal, byte, 0644);
> +MODULE_PARM_DESC(hpage, "Maximum percentage of steal time to allow pollinh");

s/pollinh/polling/

> +
>  /*
>   * For now we handle at most 16 double words as this is what the s390 base
>   * kernel handles and stores in the prefix page. If we ever need to go beyond

Reviewed-by: Cornelia Huck <cohuck@redhat.com>

Re: [PATCH v2 0/2] disabling halt polling for nested virtualization

[Christian Borntraeger]

applied and pushed out to kvms390/next.

On 16.04.19 20:01, Christian Borntraeger wrote:
> v1->v2:
> 	- add kvm_vcpu parameter to kvm_arch_no_poll
> 	- add kvm stat in the s390 implementation
> 	- make the value tunable
> 	
> Folks,
> 
> this is a very simple variant to disable halt polling when the KVM host
> is already running virtualized. We could imagine more complex variants
> (like tuning down the halt polling value) but this seems to do the trick
> for some kvm deployment scenarios on s390x. (e.g. having multiple LPARS
> with KVMs that are in itself already overcommitted). 
> 
> 
> Christian Borntraeger (2):
>   KVM: polling: add architecture backend to disable polling
>   KVM: s390: provide kvm_arch_no_poll function
> 
>  arch/s390/include/asm/kvm_host.h |  1 +
>  arch/s390/kvm/Kconfig            |  1 +
>  arch/s390/kvm/kvm-s390.c         | 17 +++++++++++++++++
>  include/linux/kvm_host.h         | 10 ++++++++++
>  virt/kvm/Kconfig                 |  3 +++
>  virt/kvm/kvm_main.c              |  2 +-
>  6 files changed, 33 insertions(+), 1 deletion(-)
>