[patch 0/3] Patches for KVM & RT

[patch 0/3] Patches for KVM & RT

[Jan Blunck]

Here are some patches that are necessary to get KVM running with the -rt4
patchset.

Cheers,
Jan

[patch 1/3] Make rt_down_read_trylock() behave like down_read_trylock()

[Jan Blunck]

[-- Attachment #1: patches.rt/rt_down_read_trylock-fix.diff --]
[-- Type: text/plain, Size: 962 bytes --]

This patch removes the stupid "Read locks within the self-held write lock
succeed" behaviour. This is breaking in mm_take_all_locks() since it is quite
common to ensure that a lock is taken with BUG_ON(down_read_trylock(&mm->mmap_sem)).

Signed-off-by: Jan Blunck <jblunck@suse.de>
---
 kernel/rt.c |   12 ------------
 1 file changed, 12 deletions(-)

Index: b/kernel/rt.c
===================================================================
--- a/kernel/rt.c
+++ b/kernel/rt.c
@@ -381,18 +381,6 @@ int  rt_down_read_trylock(struct rw_sema
 	unsigned long flags;
 	int ret;
 
-	/*
-	 * Read locks within the self-held write lock succeed.
-	 */
-	spin_lock_irqsave(&rwsem->lock.wait_lock, flags);
-	if (rt_mutex_real_owner(&rwsem->lock) == current) {
-		spin_unlock_irqrestore(&rwsem->lock.wait_lock, flags);
-		rwsem_acquire_read(&rwsem->dep_map, 0, 1, _RET_IP_);
-		rwsem->read_depth++;
-		return 1;
-	}
-	spin_unlock_irqrestore(&rwsem->lock.wait_lock, flags);

[patch 2/3] kvm: convert spin-locks to raw_spinlock_t

[Jan Blunck]

[-- Attachment #1: patches.rt/kvm-pic.diff --]
[-- Type: text/plain, Size: 953 bytes --]

This patch converts some KVM spin-locks to be of type raw_spinlock_t.

Signed-off-by: Jan Blunck <jblunck@suse.de>
---
 arch/x86/kvm/i8254.h |    2 +-
 arch/x86/kvm/irq.h   |    2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

Index: b/arch/x86/kvm/i8254.h
===================================================================
--- a/arch/x86/kvm/i8254.h
+++ b/arch/x86/kvm/i8254.h
@@ -33,7 +33,7 @@ struct kvm_kpit_state {
 	u32    speaker_data_on;
 	struct mutex lock;
 	struct kvm_pit *pit;
-	spinlock_t inject_lock;
+	raw_spinlock_t inject_lock;
 	unsigned long irq_ack;
 	struct kvm_irq_ack_notifier irq_ack_notifier;
 };
Index: b/arch/x86/kvm/irq.h
===================================================================
--- a/arch/x86/kvm/irq.h
+++ b/arch/x86/kvm/irq.h
@@ -60,7 +60,7 @@ struct kvm_kpic_state {
 };
 
 struct kvm_pic {
-	spinlock_t lock;
+	raw_spinlock_t lock;
 	bool wakeup_needed;
 	unsigned pending_acks;
 	struct kvm *kvm;

[patch 3/3] kvm: wake up waitqueue before calling get_cpu()

[Jan Blunck]

[-- Attachment #1: patches.rt/kvm-wake-up-waitqueue-before-calling-get_cpu.patch --]
[-- Type: text/plain, Size: 1066 bytes --]

This moves the get_cpu() call down to be called after we wake up the
waiters. Therefore the waitqueue locks can savely be rt mutex.

Signed-off-by: Jan Blunck <jblunck@suse.de>
Signed-off-by: Sven-Thorsten Dietrich <sven@thebigcorporation.com>
---
 arch/x86/kvm/x86.c |    3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

Index: b/arch/x86/kvm/x86.c
===================================================================
--- a/arch/x86/kvm/x86.c
+++ b/arch/x86/kvm/x86.c
@@ -4229,7 +4229,7 @@ static void vcpu_kick_intr(void *info)
 void kvm_vcpu_kick(struct kvm_vcpu *vcpu)
 {
 	int ipi_pcpu = vcpu->cpu;
-	int cpu = get_cpu();
+	int cpu;
 
 	if (waitqueue_active(&vcpu->wq)) {
 		wake_up_interruptible(&vcpu->wq);
@@ -4239,6 +4239,7 @@ void kvm_vcpu_kick(struct kvm_vcpu *vcpu
 	 * We may be called synchronously with irqs disabled in guest mode,
 	 * So need not to call smp_call_function_single() in that case.
 	 */
+	cpu = get_cpu();
 	if (vcpu->guest_mode && vcpu->cpu != cpu)
 		smp_call_function_single(ipi_pcpu, vcpu_kick_intr, vcpu, 0);
 	put_cpu();

Re: [patch 2/3] kvm: convert spin-locks to raw_spinlock_t

[Marcelo Tosatti]

Acked-by: Marcelo Tosatti <mtosatti@redhat.com>

On Wed, Apr 08, 2009 at 01:58:55AM +0200, Jan Blunck wrote:
> This patch converts some KVM spin-locks to be of type raw_spinlock_t.
> 
> Signed-off-by: Jan Blunck <jblunck@suse.de>
> ---
>  arch/x86/kvm/i8254.h |    2 +-
>  arch/x86/kvm/irq.h   |    2 +-
>  2 files changed, 2 insertions(+), 2 deletions(-)
> 
> Index: b/arch/x86/kvm/i8254.h
> ===================================================================
> --- a/arch/x86/kvm/i8254.h
> +++ b/arch/x86/kvm/i8254.h
> @@ -33,7 +33,7 @@ struct kvm_kpit_state {
>  	u32    speaker_data_on;
>  	struct mutex lock;
>  	struct kvm_pit *pit;
> -	spinlock_t inject_lock;
> +	raw_spinlock_t inject_lock;
>  	unsigned long irq_ack;
>  	struct kvm_irq_ack_notifier irq_ack_notifier;
>  };
> Index: b/arch/x86/kvm/irq.h
> ===================================================================
> --- a/arch/x86/kvm/irq.h
> +++ b/arch/x86/kvm/irq.h
> @@ -60,7 +60,7 @@ struct kvm_kpic_state {
>  };
>  
>  struct kvm_pic {
> -	spinlock_t lock;
> +	raw_spinlock_t lock;
>  	bool wakeup_needed;
>  	unsigned pending_acks;
>  	struct kvm *kvm;
> 
> 
> --
> To unsubscribe from this list: send the line "unsubscribe kvm" in
> the body of a message to majordomo@vger.kernel.org
> More majordomo info at  http://vger.kernel.org/majordomo-info.html

Re: [patch 3/3] kvm: wake up waitqueue before calling get_cpu()

[Marcelo Tosatti]

Acked-by: Marcelo Tosatti <mtosatti@redhat.com>

On Wed, Apr 08, 2009 at 01:58:56AM +0200, Jan Blunck wrote:
> This moves the get_cpu() call down to be called after we wake up the
> waiters. Therefore the waitqueue locks can savely be rt mutex.
> 
> Signed-off-by: Jan Blunck <jblunck@suse.de>
> Signed-off-by: Sven-Thorsten Dietrich <sven@thebigcorporation.com>
> ---
>  arch/x86/kvm/x86.c |    3 ++-
>  1 file changed, 2 insertions(+), 1 deletion(-)
> 
> Index: b/arch/x86/kvm/x86.c
> ===================================================================
> --- a/arch/x86/kvm/x86.c
> +++ b/arch/x86/kvm/x86.c
> @@ -4229,7 +4229,7 @@ static void vcpu_kick_intr(void *info)
>  void kvm_vcpu_kick(struct kvm_vcpu *vcpu)
>  {
>  	int ipi_pcpu = vcpu->cpu;
> -	int cpu = get_cpu();
> +	int cpu;
>  
>  	if (waitqueue_active(&vcpu->wq)) {
>  		wake_up_interruptible(&vcpu->wq);
> @@ -4239,6 +4239,7 @@ void kvm_vcpu_kick(struct kvm_vcpu *vcpu
>  	 * We may be called synchronously with irqs disabled in guest mode,
>  	 * So need not to call smp_call_function_single() in that case.
>  	 */
> +	cpu = get_cpu();
>  	if (vcpu->guest_mode && vcpu->cpu != cpu)
>  		smp_call_function_single(ipi_pcpu, vcpu_kick_intr, vcpu, 0);
>  	put_cpu();
> 
> 
> --
> To unsubscribe from this list: send the line "unsubscribe kvm" in
> the body of a message to majordomo@vger.kernel.org
> More majordomo info at  http://vger.kernel.org/majordomo-info.html

Re: [patch 1/3] Make rt_down_read_trylock() behave like down_read_trylock()

[Thomas Gleixner]

On Wed, 8 Apr 2009, Jan Blunck wrote:

> This patch removes the stupid "Read locks within the self-held write lock
> succeed" behaviour. This is breaking in mm_take_all_locks() since it is quite
> common to ensure that a lock is taken with BUG_ON(down_read_trylock(&mm->mmap_sem)).

Good catch. There are some more oddballs in that file, which we missed
when we moved from 26 to 29 - we dropped the rwlocks stuff and that
cleaned up that area.

Thanks,

	tglx

Re: [patch 2/3] kvm: convert spin-locks to raw_spinlock_t

[Thomas Gleixner]

On Wed, 8 Apr 2009, Jan Blunck wrote:

> This patch converts some KVM spin-locks to be of type raw_spinlock_t.

Applied. Thanks,

	 tglx

Re: [patch 3/3] kvm: wake up waitqueue before calling get_cpu()

[Thomas Gleixner]

On Wed, 8 Apr 2009, Jan Blunck wrote:

> This moves the get_cpu() call down to be called after we wake up the
> waiters. Therefore the waitqueue locks can savely be rt mutex.

Applied. Thanks,

	 tglx

Re: [patch 0/3] Patches for KVM & RT

[Carsten Emde]

[-- Attachment #1: Type: text/plain, Size: 330 bytes --]

Jan,

> Here are some patches that are necessary to get KVM running with the -rt4
> patchset.
Thanks a lot.

Unfortunaately, there is still a last one at kernel/smp.c:288
        /* Can deadlock when called with interrupts disabled */
        WARN_ON_ONCE(irqs_disabled() && !oops_in_progress);

Do we get another fix?

	Carsten.

[-- Attachment #2: warn-kvm-2.6.29.1-rt5 --]
[-- Type: text/plain, Size: 4183 bytes --]

Apr  9 22:57:55 deliv1 kernel: ------------[ cut here ]------------
Apr  9 22:57:55 deliv1 kernel: WARNING: at kernel/smp.c:289 smp_call_function_single+0x5a/0x11c()
Apr  9 22:57:55 deliv1 kernel: Hardware name: SD37V10
Apr  9 22:57:55 deliv1 kernel: Modules linked in: tun fuse radeon drm nfsd lockd nfs_acl auth_rpcgss exportfs sco bridge stp bnep l2cap bluetooth coretemp it87 hwmon_vid hwmon sunrpc zd1211rw mac80211 cfg80211 ipt_MASQUERADE iptable_nat nf_nat ipt_REJECT ipt_LOG nf_conntrack_ipv4 nf_defrag_ipv4 iptable_filter ip_tables nf_conntrack_netbios_ns ip6t_REJECT xt_tcpudp nf_conntrack_ipv6 xt_state nf_conntrack ip6table_filter ip6_tables x_tables ipv6 cpufreq_ondemand acpi_cpufreq dm_mirror dm_region_hash dm_log dm_multipath scsi_dh dm_mod kvm_intel kvm uinput snd_hda_codec_realtek snd_hda_intel snd_hda_codec snd_usb_audio snd_usb_lib snd_rawmidi snd_hwdep snd_seq_dummy snd_seq_oss snd_seq_midi_event snd_seq snd_seq_device snd_pcm_oss snd_mixer_oss snd_pcm snd_timer snd soundcore iTCO_wdt i2c_i801 tg3 i82975x_edac ohci1394 snd_page_alloc iTCO_vendor_support i2c_core libphy edac_core ieee1394 auerswald sr_mod sg button cdrom pcspkr ata_generic pata_acpi ata_piix libata sd_mod scsi_mod raid456 async_xor async_memcpy async
Apr  9 22:57:55 deliv1 kernel: _tx xor raid1 ext3 jbd mbcache uhci_hcd ohci_hcd ehci_hcd [last unloaded: microcode]
Apr  9 22:57:55 deliv1 kernel: Pid: 4085, comm: qemu-kvm Not tainted 2.6.29.1-rt5-32bit #39
Apr  9 22:57:55 deliv1 kernel: Call Trace:
Apr  9 22:57:55 deliv1 kernel: [<c022e89e>] warn_slowpath+0x76/0xad
Apr  9 22:57:55 deliv1 kernel: [<c0227688>] ? sub_preempt_count+0x94/0xa3
Apr  9 22:57:55 deliv1 kernel: [<f8352622>] ? set_spte+0xe/0x5dd [kvm]
Apr  9 22:57:55 deliv1 kernel: [<c048757f>] ? __schedule+0x9b0/0x9f9
Apr  9 22:57:55 deliv1 kernel: [<f834a626>] ? unalias_gfn+0xe/0x54 [kvm]
Apr  9 22:57:55 deliv1 kernel: [<c026c4cc>] ? __rcu_read_unlock+0x6e/0x7d
Apr  9 22:57:55 deliv1 kernel: [<c022185b>] ? cpuacct_charge+0x52/0x58
Apr  9 22:57:55 deliv1 kernel: [<c0221f0b>] ? update_curr+0x136/0x177
Apr  9 22:57:55 deliv1 kernel: [<c0207f87>] ? sched_clock+0x44/0x64
Apr  9 22:57:55 deliv1 kernel: [<c026c4cc>] ? __rcu_read_unlock+0x6e/0x7d
Apr  9 22:57:55 deliv1 kernel: [<c022185b>] ? cpuacct_charge+0x52/0x58
Apr  9 22:57:55 deliv1 kernel: [<c024fc95>] smp_call_function_single+0x5a/0x11c
Apr  9 22:57:55 deliv1 kernel: [<f81cc72f>] ? __vcpu_clear+0x0/0xbc [kvm_intel]
Apr  9 22:57:55 deliv1 kernel: [<f81cf19d>] vmx_vcpu_load+0x46/0x191 [kvm_intel]
Apr  9 22:57:55 deliv1 kernel: [<c0489067>] ? __spin_unlock+0x14/0x28
Apr  9 22:57:55 deliv1 kernel: [<f834db41>] kvm_arch_vcpu_load+0x14/0x1e [kvm]
Apr  9 22:57:55 deliv1 kernel: [<f83481d7>] kvm_sched_in+0x10/0x12 [kvm]
Apr  9 22:57:55 deliv1 kernel: [<c02275a5>] finish_task_switch+0x97/0xe6
Apr  9 22:57:55 deliv1 kernel: [<c0487570>] __schedule+0x9a1/0x9f9
Apr  9 22:57:55 deliv1 kernel: [<c024430f>] ? hrtimer_interrupt+0x18a/0x192
Apr  9 22:57:55 deliv1 kernel: [<c0202c9d>] ? restore_nocheck_notrace+0x0/0xe
Apr  9 22:57:55 deliv1 kernel: [<f834e7c3>] ? kvm_arch_vcpu_ioctl_run+0x406/0x634 [kvm]
Apr  9 22:57:55 deliv1 kernel: [<c048774f>] ? preempt_schedule+0x3e/0x67
Apr  9 22:57:55 deliv1 kernel: [<c048775e>] preempt_schedule+0x4d/0x67
Apr  9 22:57:55 deliv1 kernel: [<f834e7c3>] kvm_arch_vcpu_ioctl_run+0x406/0x634 [kvm]
Apr  9 22:57:55 deliv1 kernel: [<f834740d>] kvm_vcpu_ioctl+0xf4/0x659 [kvm]
Apr  9 22:57:55 deliv1 kernel: [<c048757f>] ? __schedule+0x9b0/0x9f9
Apr  9 22:57:55 deliv1 kernel: [<c024b232>] ? tick_dev_program_event+0x2d/0x9a
Apr  9 22:57:55 deliv1 kernel: [<f8347319>] ? kvm_vcpu_ioctl+0x0/0x659 [kvm]
Apr  9 22:57:55 deliv1 kernel: [<c02b8a8d>] vfs_ioctl+0x27/0x6e
Apr  9 22:57:55 deliv1 kernel: [<c02b9038>] do_vfs_ioctl+0x49b/0x4d4
Apr  9 22:57:55 deliv1 kernel: [<c0335058>] ? trace_hardirqs_on_thunk+0xc/0x10
Apr  9 22:57:55 deliv1 kernel: [<c02ae507>] ? fget_light+0x85/0x9a
Apr  9 22:57:55 deliv1 kernel: [<c02b90b6>] sys_ioctl+0x45/0x5f
Apr  9 22:57:55 deliv1 kernel: [<c0202b88>] sysenter_do_call+0x12/0x27
Apr  9 22:57:55 deliv1 kernel: ---[ end trace ea071060aeca7fc5 ]---

Re: [patch 0/3] Patches for KVM & RT

[Thomas Gleixner]

On Thu, 9 Apr 2009, Carsten Emde wrote:

> Jan,
> 
> > Here are some patches that are necessary to get KVM running with the -rt4
> > patchset.
> Thanks a lot.
> 
> Unfortunaately, there is still a last one at kernel/smp.c:288
>         /* Can deadlock when called with interrupts disabled */
>         WARN_ON_ONCE(irqs_disabled() && !oops_in_progress);
> 
> Do we get another fix?

I think I have seen that before. Just remembered that I fixed that
with Avi last year. Patch got dropped in the 26->29 move.

Thanks,

	tglx

----

From: Thomas Gleixner <tglx@linutronix.de>
Date: Mon, 14 Jan 2008 14:02:44 +0200
Subject: CFS: enable irqs in fire_sched_in_preempt_notifier

KVM expects the notifier call with irqs enabled. It's necessary due
to a possible IPI call. Make the preempt-rt version behave the same
way as mainline.

Signed-off-by: Thomas Gleixner <tgxl@linutronix.de>

---
 kernel/sched.c |    9 +++++++++
 1 file changed, 9 insertions(+)

Index: linux-2.6.24.7-rt27/kernel/sched.c
===================================================================
--- linux-2.6.24.7-rt27.orig/kernel/sched.c	2009-02-08 00:01:16.000000000 -0500
+++ linux-2.6.24.7-rt27/kernel/sched.c	2009-02-08 00:01:22.000000000 -0500
@@ -1821,8 +1821,17 @@ static void fire_sched_in_preempt_notifi
 	struct preempt_notifier *notifier;
 	struct hlist_node *node;
 
+	if (hlist_empty(&curr->preempt_notifiers))
+		return;
+
+	/*
+	 * The KVM sched in notifier expects to be called with
+	 * interrupts enabled.
+	 */
+	local_irq_enable();
 	hlist_for_each_entry(notifier, node, &curr->preempt_notifiers, link)
 		notifier->ops->sched_in(notifier, raw_smp_processor_id());
+	local_irq_disable();
 }
 
 static void