From mboxrd@z Thu Jan 1 00:00:00 1970 From: Avi Kivity Subject: [PATCH v2 2/5] KVM: MMU: Convert remote flushes to kvm_mark_tlb_dirty() and a conditional flush Date: Thu, 17 May 2012 13:24:41 +0300 Message-ID: <1337250284-18607-2-git-send-email-avi@redhat.com> References: <1337250284-18607-1-git-send-email-avi@redhat.com> To: Marcelo Tosatti , Xiao Guangrong , Takuya Yoshikawa , kvm@vger.kernel.org Return-path: Received: from mx1.redhat.com ([209.132.183.28]:28233 "EHLO mx1.redhat.com" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S1759475Ab2EQKYw (ORCPT ); Thu, 17 May 2012 06:24:52 -0400 In-Reply-To: <1337250284-18607-1-git-send-email-avi@redhat.com> Sender: kvm-owner@vger.kernel.org List-ID: This allows us to later move the actual flush out of protection of the mmu spinlock, provided there are no additional dependencies. Constructs of the form if (pred) kvm_flush_remote_tlbs(kvm) are converted to if (pred) kvm_mark_tlb_dirty(kvm) kvm_cond_flush_remote_tlbs(kvm) so that if another thread caused pred to transition from true to false, but has not yet flushed the tlb, we notice it and flush it before proceeding. Signed-off-by: Avi Kivity --- arch/ia64/kvm/kvm-ia64.c | 6 ++++-- arch/x86/kvm/mmu.c | 45 +++++++++++++++++++++++++++----------------- arch/x86/kvm/paging_tmpl.h | 4 +++- arch/x86/kvm/x86.c | 4 +++- virt/kvm/kvm_main.c | 4 +++- 5 files changed, 41 insertions(+), 22 deletions(-) diff --git a/arch/ia64/kvm/kvm-ia64.c b/arch/ia64/kvm/kvm-ia64.c index bd77cb5..a4a92d8 100644 --- a/arch/ia64/kvm/kvm-ia64.c +++ b/arch/ia64/kvm/kvm-ia64.c @@ -1628,7 +1628,8 @@ void kvm_arch_commit_memory_region(struct kvm *kvm, void kvm_arch_flush_shadow(struct kvm *kvm) { - kvm_flush_remote_tlbs(kvm); + kvm_mark_tlb_dirty(kvm); + kvm_cond_flush_remote_tlbs(kvm); } long kvm_arch_dev_ioctl(struct file *filp, @@ -1853,10 +1854,11 @@ int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm, /* If nothing is dirty, don't bother messing with page tables. */ if (is_dirty) { - kvm_flush_remote_tlbs(kvm); + kvm_mark_tlb_dirty(kvm); n = kvm_dirty_bitmap_bytes(memslot); memset(memslot->dirty_bitmap, 0, n); } + kvm_cond_flush_remote_tlbs(kvm); r = 0; out: mutex_unlock(&kvm->slots_lock); diff --git a/arch/x86/kvm/mmu.c b/arch/x86/kvm/mmu.c index 07424cf..03a9c80 100644 --- a/arch/x86/kvm/mmu.c +++ b/arch/x86/kvm/mmu.c @@ -1170,7 +1170,9 @@ static int kvm_set_pte_rmapp(struct kvm *kvm, unsigned long *rmapp, } if (need_flush) - kvm_flush_remote_tlbs(kvm); + kvm_mark_tlb_dirty(kvm); + + kvm_cond_flush_remote_tlbs(kvm); return 0; } @@ -1294,7 +1296,8 @@ static void rmap_recycle(struct kvm_vcpu *vcpu, u64 *spte, gfn_t gfn) rmapp = gfn_to_rmap(vcpu->kvm, gfn, sp->role.level); kvm_unmap_rmapp(vcpu->kvm, rmapp, 0); - kvm_flush_remote_tlbs(vcpu->kvm); + kvm_mark_tlb_dirty(vcpu->kvm); + kvm_cond_flush_remote_tlbs(vcpu->kvm); } int kvm_age_hva(struct kvm *kvm, unsigned long hva) @@ -1697,7 +1700,9 @@ static void mmu_sync_children(struct kvm_vcpu *vcpu, protected |= rmap_write_protect(vcpu->kvm, sp->gfn); if (protected) - kvm_flush_remote_tlbs(vcpu->kvm); + kvm_mark_tlb_dirty(vcpu->kvm); + + kvm_cond_flush_remote_tlbs(vcpu->kvm); for_each_sp(pages, sp, parents, i) { kvm_sync_page(vcpu, sp, &invalid_list); @@ -1786,7 +1791,8 @@ static struct kvm_mmu_page *kvm_mmu_get_page(struct kvm_vcpu *vcpu, &vcpu->kvm->arch.mmu_page_hash[kvm_page_table_hashfn(gfn)]); if (!direct) { if (rmap_write_protect(vcpu->kvm, gfn)) - kvm_flush_remote_tlbs(vcpu->kvm); + kvm_mark_tlb_dirty(vcpu->kvm); + kvm_cond_flush_remote_tlbs(vcpu->kvm); if (level > PT_PAGE_TABLE_LEVEL && need_sync) kvm_sync_pages(vcpu, gfn); @@ -1861,7 +1867,8 @@ static void drop_large_spte(struct kvm_vcpu *vcpu, u64 *sptep) if (is_large_pte(*sptep)) { drop_spte(vcpu->kvm, sptep); --vcpu->kvm->stat.lpages; - kvm_flush_remote_tlbs(vcpu->kvm); + kvm_mark_tlb_dirty(vcpu->kvm); + kvm_cond_flush_remote_tlbs(vcpu->kvm); } } @@ -1883,7 +1890,8 @@ static void validate_direct_spte(struct kvm_vcpu *vcpu, u64 *sptep, return; drop_parent_pte(child, sptep); - kvm_flush_remote_tlbs(vcpu->kvm); + kvm_mark_tlb_dirty(vcpu->kvm); + kvm_cond_flush_remote_tlbs(vcpu->kvm); } } @@ -2021,7 +2029,8 @@ static void kvm_mmu_commit_zap_page(struct kvm *kvm, if (list_empty(invalid_list)) return; - kvm_flush_remote_tlbs(kvm); + kvm_mark_tlb_dirty(kvm); + kvm_cond_flush_remote_tlbs(kvm); if (atomic_read(&kvm->arch.reader_counter)) { kvm_mmu_isolate_pages(invalid_list); @@ -2344,7 +2353,9 @@ static int set_spte(struct kvm_vcpu *vcpu, u64 *sptep, * might be cached on a CPU's TLB. */ if (is_writable_pte(entry) && !is_writable_pte(*sptep)) - kvm_flush_remote_tlbs(vcpu->kvm); + kvm_mark_tlb_dirty(vcpu->kvm); + + kvm_cond_flush_remote_tlbs(vcpu->kvm); done: return ret; } @@ -2376,15 +2387,15 @@ static void mmu_set_spte(struct kvm_vcpu *vcpu, u64 *sptep, child = page_header(pte & PT64_BASE_ADDR_MASK); drop_parent_pte(child, sptep); - kvm_flush_remote_tlbs(vcpu->kvm); } else if (pfn != spte_to_pfn(*sptep)) { pgprintk("hfn old %llx new %llx\n", spte_to_pfn(*sptep), pfn); drop_spte(vcpu->kvm, sptep); - kvm_flush_remote_tlbs(vcpu->kvm); + kvm_mark_tlb_dirty(vcpu->kvm); } else was_rmapped = 1; } + kvm_cond_flush_remote_tlbs(vcpu->kvm); if (set_spte(vcpu, sptep, pte_access, user_fault, write_fault, level, gfn, pfn, speculative, true, @@ -3561,14 +3572,13 @@ static bool need_remote_flush(u64 old, u64 new) } static void mmu_pte_write_flush_tlb(struct kvm_vcpu *vcpu, bool zap_page, - bool remote_flush, bool local_flush) + bool local_flush) { if (zap_page) return; - if (remote_flush) - kvm_flush_remote_tlbs(vcpu->kvm); - else if (local_flush) + kvm_cond_flush_remote_tlbs(vcpu->kvm); + if (local_flush) kvm_mmu_flush_tlb(vcpu); } @@ -3742,11 +3752,11 @@ void kvm_mmu_pte_write(struct kvm_vcpu *vcpu, gpa_t gpa, & mask.word) && rmap_can_add(vcpu)) mmu_pte_write_new_pte(vcpu, sp, spte, &gentry); if (!remote_flush && need_remote_flush(entry, *spte)) - remote_flush = true; + kvm_mark_tlb_dirty(vcpu->kvm); ++spte; } } - mmu_pte_write_flush_tlb(vcpu, zap_page, remote_flush, local_flush); + mmu_pte_write_flush_tlb(vcpu, zap_page, local_flush); kvm_mmu_commit_zap_page(vcpu->kvm, &invalid_list); kvm_mmu_audit(vcpu, AUDIT_POST_PTE_WRITE); spin_unlock(&vcpu->kvm->mmu_lock); @@ -3927,7 +3937,8 @@ void kvm_mmu_slot_remove_write_access(struct kvm *kvm, int slot) pt[i] & ~PT_WRITABLE_MASK); } } - kvm_flush_remote_tlbs(kvm); + kvm_mark_tlb_dirty(kvm); + kvm_cond_flush_remote_tlbs(kvm); } void kvm_mmu_zap_all(struct kvm *kvm) diff --git a/arch/x86/kvm/paging_tmpl.h b/arch/x86/kvm/paging_tmpl.h index 97e2a81..9fe5f72 100644 --- a/arch/x86/kvm/paging_tmpl.h +++ b/arch/x86/kvm/paging_tmpl.h @@ -698,7 +698,9 @@ static void FNAME(invlpg)(struct kvm_vcpu *vcpu, gva_t gva) pte_gpa += (sptep - sp->spt) * sizeof(pt_element_t); if (mmu_page_zap_pte(vcpu->kvm, sp, sptep)) - kvm_flush_remote_tlbs(vcpu->kvm); + kvm_mark_tlb_dirty(vcpu->kvm); + + kvm_cond_flush_remote_tlbs(vcpu->kvm); if (!rmap_can_add(vcpu)) break; diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index 2256f51..a2149d8 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -3130,7 +3130,9 @@ int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm, struct kvm_dirty_log *log) kvm_mmu_write_protect_pt_masked(kvm, memslot, offset, mask); } if (is_dirty) - kvm_flush_remote_tlbs(kvm); + kvm_mark_tlb_dirty(kvm); + + kvm_cond_flush_remote_tlbs(kvm); spin_unlock(&kvm->mmu_lock); diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c index 4113a36..585ab45 100644 --- a/virt/kvm/kvm_main.c +++ b/virt/kvm/kvm_main.c @@ -392,7 +392,9 @@ static int kvm_mmu_notifier_clear_flush_young(struct mmu_notifier *mn, young = kvm_age_hva(kvm, address); if (young) - kvm_flush_remote_tlbs(kvm); + kvm_mark_tlb_dirty(kvm); + + kvm_cond_flush_remote_tlbs(kvm); spin_unlock(&kvm->mmu_lock); srcu_read_unlock(&kvm->srcu, idx); -- 1.7.10.1