From f54d23ec258da7c631c7b059048fb383b8255079 Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Wed, 2 Mar 2022 08:44:22 -0500 Subject: [PATCH 1/2] KVM: x86/mmu: only perform eager page splitting on valid roots Eager page splitting is an optimization; it does not have to be performed on invalid roots. By only operating on the valid roots, this removes the only case in which a reader might acquire a reference to an invalid root. Signed-off-by: Paolo Bonzini --- arch/x86/kvm/mmu/tdp_mmu.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/x86/kvm/mmu/tdp_mmu.c b/arch/x86/kvm/mmu/tdp_mmu.c index 941e25fd14bd..e3b104448e14 100644 --- a/arch/x86/kvm/mmu/tdp_mmu.c +++ b/arch/x86/kvm/mmu/tdp_mmu.c @@ -1541,7 +1541,7 @@ void kvm_tdp_mmu_try_split_huge_pages(struct kvm *kvm, kvm_lockdep_assert_mmu_lock_held(kvm, shared); - for_each_tdp_mmu_root_yield_safe(kvm, root, slot->as_id, shared) { + for_each_valid_tdp_mmu_root_yield_safe(kvm, root, slot->as_id, shared) { r = tdp_mmu_split_huge_pages_root(kvm, root, start, end, target_level, shared); if (r) { kvm_tdp_mmu_put_root(kvm, root, shared); -- 2.31.1 From 95557483c2a32386cb0769a61f838550d9467434 Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Wed, 2 Mar 2022 08:51:05 -0500 Subject: [PATCH 2/2] KVM: x86/mmu: do not allow readers to acquire references to invalid roots Remove the "shared" argument of for_each_tdp_mmu_root_yield_safe, thus ensuring that readers do not ever acquire a reference to an invalid root. After this patch, all readers except kvm_tdp_mmu_zap_invalidated_roots() treat refcount=0/valid, refcount=0/invalid and refcount=1/invalid in exactly the same way. kvm_tdp_mmu_zap_invalidated_roots() is different but it also does not acquire a reference to the invalid root, and it cannot see refcount=0/invalid because it is guaranteed to run after kvm_tdp_mmu_invalidate_all_roots(). Signed-off-by: Paolo Bonzini --- arch/x86/kvm/mmu/tdp_mmu.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/arch/x86/kvm/mmu/tdp_mmu.c b/arch/x86/kvm/mmu/tdp_mmu.c index e3b104448e14..2e935edd3f6c 100644 --- a/arch/x86/kvm/mmu/tdp_mmu.c +++ b/arch/x86/kvm/mmu/tdp_mmu.c @@ -171,8 +171,8 @@ static struct kvm_mmu_page *tdp_mmu_next_root(struct kvm *kvm, #define for_each_valid_tdp_mmu_root_yield_safe(_kvm, _root, _as_id, _shared) \ __for_each_tdp_mmu_root_yield_safe(_kvm, _root, _as_id, _shared, true) -#define for_each_tdp_mmu_root_yield_safe(_kvm, _root, _as_id, _shared) \ - __for_each_tdp_mmu_root_yield_safe(_kvm, _root, _as_id, _shared, false) +#define for_each_tdp_mmu_root_yield_safe(_kvm, _root, _as_id) \ + __for_each_tdp_mmu_root_yield_safe(_kvm, _root, _as_id, false, false) /* * Iterate over all TDP MMU roots. Requires that mmu_lock be held for write, @@ -879,7 +879,8 @@ bool kvm_tdp_mmu_zap_leafs(struct kvm *kvm, int as_id, gfn_t start, gfn_t end, { struct kvm_mmu_page *root; - for_each_tdp_mmu_root_yield_safe(kvm, root, as_id, false) + lockdep_assert_held_write(&kvm->mmu_lock); + for_each_tdp_mmu_root_yield_safe(kvm, root, as_id) flush = tdp_mmu_zap_leafs(kvm, root, start, end, can_yield, false); return flush; @@ -895,8 +896,9 @@ void kvm_tdp_mmu_zap_all(struct kvm *kvm) * is being destroyed or the userspace VMM has exited. In both cases, * KVM_RUN is unreachable, i.e. no vCPUs will ever service the request. */ + lockdep_assert_held_write(&kvm->mmu_lock); for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++) { - for_each_tdp_mmu_root_yield_safe(kvm, root, i, false) + for_each_tdp_mmu_root_yield_safe(kvm, root, i) tdp_mmu_zap_root(kvm, root, false); } } -- 2.31.1