CC: kbuild-all(a)lists.01.org CC: linux-kernel(a)vger.kernel.org TO: Ben Gardon CC: Paolo Bonzini CC: Peter Feiner tree: https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git master head: 16fc44d6387e260f4932e9248b985837324705d8 commit: 9a77daacc87dee9fd63e31243f21894132ed8407 KVM: x86/mmu: Use atomic ops to set SPTEs in TDP MMU map date: 3 months ago :::::: branch date: 9 hours ago :::::: commit date: 3 months ago config: x86_64-randconfig-s021-20210422 (attached as .config) compiler: gcc-9 (Debian 9.3.0-22) 9.3.0 reproduce: # apt-get install sparse # sparse version: v0.6.3-341-g8af24329-dirty # https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/commit/?id=9a77daacc87dee9fd63e31243f21894132ed8407 git remote add linus https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git git fetch --no-tags linus master git checkout 9a77daacc87dee9fd63e31243f21894132ed8407 # save the attached .config to linux build tree make W=1 C=1 CF='-fdiagnostic-prefix -D__CHECK_ENDIAN__' W=1 ARCH=x86_64 If you fix the issue, kindly add following tag as appropriate Reported-by: kernel test robot sparse warnings: (new ones prefixed by >>) arch/x86/kvm/mmu/tdp_mmu.c:455:49: sparse: sparse: incorrect type in argument 2 (different address spaces) @@ expected unsigned long long [usertype] *pt @@ got unsigned long long [noderef] [usertype] __rcu * @@ arch/x86/kvm/mmu/tdp_mmu.c:455:49: sparse: expected unsigned long long [usertype] *pt arch/x86/kvm/mmu/tdp_mmu.c:455:49: sparse: got unsigned long long [noderef] [usertype] __rcu * >> arch/x86/kvm/mmu/tdp_mmu.c:291:9: sparse: sparse: context imbalance in 'tdp_mmu_link_page' - different lock contexts for basic block >> arch/x86/kvm/mmu/tdp_mmu.c:316:9: sparse: sparse: context imbalance in 'tdp_mmu_unlink_page' - different lock contexts for basic block arch/x86/kvm/mmu/tdp_mmu.c:606:51: sparse: sparse: incorrect type in argument 2 (different address spaces) @@ expected unsigned long long [usertype] *root_pt @@ got unsigned long long [noderef] [usertype] __rcu * @@ arch/x86/kvm/mmu/tdp_mmu.c:606:51: sparse: expected unsigned long long [usertype] *root_pt arch/x86/kvm/mmu/tdp_mmu.c:606:51: sparse: got unsigned long long [noderef] [usertype] __rcu * arch/x86/kvm/mmu/tdp_mmu.c:521:49: sparse: sparse: incorrect type in argument 1 (different address spaces) @@ expected unsigned long long [usertype] *sptep @@ got unsigned long long [noderef] [usertype] __rcu *[usertype] root_pt @@ arch/x86/kvm/mmu/tdp_mmu.c:521:49: sparse: expected unsigned long long [usertype] *sptep arch/x86/kvm/mmu/tdp_mmu.c:521:49: sparse: got unsigned long long [noderef] [usertype] __rcu *[usertype] root_pt >> arch/x86/kvm/mmu/tdp_mmu.c:483:40: sparse: sparse: incorrect type in initializer (different address spaces) @@ expected unsigned long long [usertype] *root_pt @@ got unsigned long long [noderef] [usertype] __rcu * @@ arch/x86/kvm/mmu/tdp_mmu.c:483:40: sparse: expected unsigned long long [usertype] *root_pt arch/x86/kvm/mmu/tdp_mmu.c:483:40: sparse: got unsigned long long [noderef] [usertype] __rcu * >> arch/x86/kvm/mmu/tdp_mmu.c:483:40: sparse: sparse: incorrect type in initializer (different address spaces) @@ expected unsigned long long [usertype] *root_pt @@ got unsigned long long [noderef] [usertype] __rcu * @@ arch/x86/kvm/mmu/tdp_mmu.c:483:40: sparse: expected unsigned long long [usertype] *root_pt arch/x86/kvm/mmu/tdp_mmu.c:483:40: sparse: got unsigned long long [noderef] [usertype] __rcu * >> arch/x86/kvm/mmu/tdp_mmu.c:483:40: sparse: sparse: incorrect type in initializer (different address spaces) @@ expected unsigned long long [usertype] *root_pt @@ got unsigned long long [noderef] [usertype] __rcu * @@ arch/x86/kvm/mmu/tdp_mmu.c:483:40: sparse: expected unsigned long long [usertype] *root_pt arch/x86/kvm/mmu/tdp_mmu.c:483:40: sparse: got unsigned long long [noderef] [usertype] __rcu * arch/x86/kvm/mmu/tdp_mmu.c:521:49: sparse: sparse: incorrect type in argument 1 (different address spaces) @@ expected unsigned long long [usertype] *sptep @@ got unsigned long long [noderef] [usertype] __rcu *[usertype] root_pt @@ arch/x86/kvm/mmu/tdp_mmu.c:521:49: sparse: expected unsigned long long [usertype] *sptep arch/x86/kvm/mmu/tdp_mmu.c:521:49: sparse: got unsigned long long [noderef] [usertype] __rcu *[usertype] root_pt arch/x86/kvm/mmu/tdp_mmu.c:521:49: sparse: sparse: incorrect type in argument 1 (different address spaces) @@ expected unsigned long long [usertype] *sptep @@ got unsigned long long [noderef] [usertype] __rcu *[usertype] root_pt @@ arch/x86/kvm/mmu/tdp_mmu.c:521:49: sparse: expected unsigned long long [usertype] *sptep arch/x86/kvm/mmu/tdp_mmu.c:521:49: sparse: got unsigned long long [noderef] [usertype] __rcu *[usertype] root_pt arch/x86/kvm/mmu/tdp_mmu.c:521:49: sparse: sparse: incorrect type in argument 1 (different address spaces) @@ expected unsigned long long [usertype] *sptep @@ got unsigned long long [noderef] [usertype] __rcu *[usertype] root_pt @@ arch/x86/kvm/mmu/tdp_mmu.c:521:49: sparse: expected unsigned long long [usertype] *sptep arch/x86/kvm/mmu/tdp_mmu.c:521:49: sparse: got unsigned long long [noderef] [usertype] __rcu *[usertype] root_pt arch/x86/kvm/mmu/tdp_mmu.c:606:51: sparse: sparse: incorrect type in argument 2 (different address spaces) @@ expected unsigned long long [usertype] *root_pt @@ got unsigned long long [noderef] [usertype] __rcu * @@ arch/x86/kvm/mmu/tdp_mmu.c:606:51: sparse: expected unsigned long long [usertype] *root_pt arch/x86/kvm/mmu/tdp_mmu.c:606:51: sparse: got unsigned long long [noderef] [usertype] __rcu * arch/x86/kvm/mmu/tdp_mmu.c:521:49: sparse: sparse: incorrect type in argument 1 (different address spaces) @@ expected unsigned long long [usertype] *sptep @@ got unsigned long long [noderef] [usertype] __rcu *[usertype] root_pt @@ arch/x86/kvm/mmu/tdp_mmu.c:521:49: sparse: expected unsigned long long [usertype] *sptep arch/x86/kvm/mmu/tdp_mmu.c:521:49: sparse: got unsigned long long [noderef] [usertype] __rcu *[usertype] root_pt arch/x86/kvm/mmu/tdp_mmu.c:606:51: sparse: sparse: incorrect type in argument 2 (different address spaces) @@ expected unsigned long long [usertype] *root_pt @@ got unsigned long long [noderef] [usertype] __rcu * @@ arch/x86/kvm/mmu/tdp_mmu.c:606:51: sparse: expected unsigned long long [usertype] *root_pt arch/x86/kvm/mmu/tdp_mmu.c:606:51: sparse: got unsigned long long [noderef] [usertype] __rcu * arch/x86/kvm/mmu/tdp_mmu.c:521:49: sparse: sparse: incorrect type in argument 1 (different address spaces) @@ expected unsigned long long [usertype] *sptep @@ got unsigned long long [noderef] [usertype] __rcu *[usertype] root_pt @@ arch/x86/kvm/mmu/tdp_mmu.c:521:49: sparse: expected unsigned long long [usertype] *sptep arch/x86/kvm/mmu/tdp_mmu.c:521:49: sparse: got unsigned long long [noderef] [usertype] __rcu *[usertype] root_pt arch/x86/kvm/mmu/tdp_mmu.c:521:49: sparse: sparse: incorrect type in argument 1 (different address spaces) @@ expected unsigned long long [usertype] *sptep @@ got unsigned long long [noderef] [usertype] __rcu *[usertype] root_pt @@ arch/x86/kvm/mmu/tdp_mmu.c:521:49: sparse: expected unsigned long long [usertype] *sptep arch/x86/kvm/mmu/tdp_mmu.c:521:49: sparse: got unsigned long long [noderef] [usertype] __rcu *[usertype] root_pt arch/x86/kvm/mmu/tdp_mmu.c:606:51: sparse: sparse: incorrect type in argument 2 (different address spaces) @@ expected unsigned long long [usertype] *root_pt @@ got unsigned long long [noderef] [usertype] __rcu * @@ arch/x86/kvm/mmu/tdp_mmu.c:606:51: sparse: expected unsigned long long [usertype] *root_pt arch/x86/kvm/mmu/tdp_mmu.c:606:51: sparse: got unsigned long long [noderef] [usertype] __rcu * arch/x86/kvm/mmu/tdp_mmu.c:521:49: sparse: sparse: incorrect type in argument 1 (different address spaces) @@ expected unsigned long long [usertype] *sptep @@ got unsigned long long [noderef] [usertype] __rcu *[usertype] root_pt @@ arch/x86/kvm/mmu/tdp_mmu.c:521:49: sparse: expected unsigned long long [usertype] *sptep arch/x86/kvm/mmu/tdp_mmu.c:521:49: sparse: got unsigned long long [noderef] [usertype] __rcu *[usertype] root_pt arch/x86/kvm/mmu/tdp_mmu.c:606:51: sparse: sparse: incorrect type in argument 2 (different address spaces) @@ expected unsigned long long [usertype] *root_pt @@ got unsigned long long [noderef] [usertype] __rcu * @@ arch/x86/kvm/mmu/tdp_mmu.c:606:51: sparse: expected unsigned long long [usertype] *root_pt arch/x86/kvm/mmu/tdp_mmu.c:606:51: sparse: got unsigned long long [noderef] [usertype] __rcu * arch/x86/kvm/mmu/tdp_mmu.c:521:49: sparse: sparse: incorrect type in argument 1 (different address spaces) @@ expected unsigned long long [usertype] *sptep @@ got unsigned long long [noderef] [usertype] __rcu *[usertype] root_pt @@ arch/x86/kvm/mmu/tdp_mmu.c:521:49: sparse: expected unsigned long long [usertype] *sptep arch/x86/kvm/mmu/tdp_mmu.c:521:49: sparse: got unsigned long long [noderef] [usertype] __rcu *[usertype] root_pt arch/x86/kvm/mmu/tdp_mmu.c:521:49: sparse: sparse: incorrect type in argument 1 (different address spaces) @@ expected unsigned long long [usertype] *sptep @@ got unsigned long long [noderef] [usertype] __rcu *[usertype] root_pt @@ arch/x86/kvm/mmu/tdp_mmu.c:521:49: sparse: expected unsigned long long [usertype] *sptep arch/x86/kvm/mmu/tdp_mmu.c:521:49: sparse: got unsigned long long [noderef] [usertype] __rcu *[usertype] root_pt vim +/tdp_mmu_link_page +291 arch/x86/kvm/mmu/tdp_mmu.c a6a0b05da9f37f Ben Gardon 2020-10-14 267 a9442f594147f9 Ben Gardon 2021-02-02 268 /** a9442f594147f9 Ben Gardon 2021-02-02 269 * tdp_mmu_link_page - Add a new page to the list of pages used by the TDP MMU a9442f594147f9 Ben Gardon 2021-02-02 270 * a9442f594147f9 Ben Gardon 2021-02-02 271 * @kvm: kvm instance a9442f594147f9 Ben Gardon 2021-02-02 272 * @sp: the new page 9a77daacc87dee Ben Gardon 2021-02-02 273 * @shared: This operation may not be running under the exclusive use of 9a77daacc87dee Ben Gardon 2021-02-02 274 * the MMU lock and the operation must synchronize with other 9a77daacc87dee Ben Gardon 2021-02-02 275 * threads that might be adding or removing pages. a9442f594147f9 Ben Gardon 2021-02-02 276 * @account_nx: This page replaces a NX large page and should be marked for a9442f594147f9 Ben Gardon 2021-02-02 277 * eventual reclaim. a9442f594147f9 Ben Gardon 2021-02-02 278 */ a9442f594147f9 Ben Gardon 2021-02-02 279 static void tdp_mmu_link_page(struct kvm *kvm, struct kvm_mmu_page *sp, 9a77daacc87dee Ben Gardon 2021-02-02 280 bool shared, bool account_nx) a9442f594147f9 Ben Gardon 2021-02-02 281 { 9a77daacc87dee Ben Gardon 2021-02-02 282 if (shared) 9a77daacc87dee Ben Gardon 2021-02-02 283 spin_lock(&kvm->arch.tdp_mmu_pages_lock); 9a77daacc87dee Ben Gardon 2021-02-02 284 else a9442f594147f9 Ben Gardon 2021-02-02 285 lockdep_assert_held_write(&kvm->mmu_lock); a9442f594147f9 Ben Gardon 2021-02-02 286 a9442f594147f9 Ben Gardon 2021-02-02 287 list_add(&sp->link, &kvm->arch.tdp_mmu_pages); a9442f594147f9 Ben Gardon 2021-02-02 288 if (account_nx) a9442f594147f9 Ben Gardon 2021-02-02 289 account_huge_nx_page(kvm, sp); 9a77daacc87dee Ben Gardon 2021-02-02 290 9a77daacc87dee Ben Gardon 2021-02-02 @291 if (shared) 9a77daacc87dee Ben Gardon 2021-02-02 292 spin_unlock(&kvm->arch.tdp_mmu_pages_lock); a9442f594147f9 Ben Gardon 2021-02-02 293 } a9442f594147f9 Ben Gardon 2021-02-02 294 a9442f594147f9 Ben Gardon 2021-02-02 295 /** a9442f594147f9 Ben Gardon 2021-02-02 296 * tdp_mmu_unlink_page - Remove page from the list of pages used by the TDP MMU a9442f594147f9 Ben Gardon 2021-02-02 297 * a9442f594147f9 Ben Gardon 2021-02-02 298 * @kvm: kvm instance a9442f594147f9 Ben Gardon 2021-02-02 299 * @sp: the page to be removed 9a77daacc87dee Ben Gardon 2021-02-02 300 * @shared: This operation may not be running under the exclusive use of 9a77daacc87dee Ben Gardon 2021-02-02 301 * the MMU lock and the operation must synchronize with other 9a77daacc87dee Ben Gardon 2021-02-02 302 * threads that might be adding or removing pages. a9442f594147f9 Ben Gardon 2021-02-02 303 */ 9a77daacc87dee Ben Gardon 2021-02-02 304 static void tdp_mmu_unlink_page(struct kvm *kvm, struct kvm_mmu_page *sp, 9a77daacc87dee Ben Gardon 2021-02-02 305 bool shared) a9442f594147f9 Ben Gardon 2021-02-02 306 { 9a77daacc87dee Ben Gardon 2021-02-02 307 if (shared) 9a77daacc87dee Ben Gardon 2021-02-02 308 spin_lock(&kvm->arch.tdp_mmu_pages_lock); 9a77daacc87dee Ben Gardon 2021-02-02 309 else a9442f594147f9 Ben Gardon 2021-02-02 310 lockdep_assert_held_write(&kvm->mmu_lock); a9442f594147f9 Ben Gardon 2021-02-02 311 a9442f594147f9 Ben Gardon 2021-02-02 312 list_del(&sp->link); a9442f594147f9 Ben Gardon 2021-02-02 313 if (sp->lpage_disallowed) a9442f594147f9 Ben Gardon 2021-02-02 314 unaccount_huge_nx_page(kvm, sp); 9a77daacc87dee Ben Gardon 2021-02-02 315 9a77daacc87dee Ben Gardon 2021-02-02 @316 if (shared) 9a77daacc87dee Ben Gardon 2021-02-02 317 spin_unlock(&kvm->arch.tdp_mmu_pages_lock); a9442f594147f9 Ben Gardon 2021-02-02 318 } a9442f594147f9 Ben Gardon 2021-02-02 319 a066e61f13cf4b Ben Gardon 2021-02-02 320 /** a066e61f13cf4b Ben Gardon 2021-02-02 321 * handle_removed_tdp_mmu_page - handle a pt removed from the TDP structure a066e61f13cf4b Ben Gardon 2021-02-02 322 * a066e61f13cf4b Ben Gardon 2021-02-02 323 * @kvm: kvm instance a066e61f13cf4b Ben Gardon 2021-02-02 324 * @pt: the page removed from the paging structure 9a77daacc87dee Ben Gardon 2021-02-02 325 * @shared: This operation may not be running under the exclusive use 9a77daacc87dee Ben Gardon 2021-02-02 326 * of the MMU lock and the operation must synchronize with other 9a77daacc87dee Ben Gardon 2021-02-02 327 * threads that might be modifying SPTEs. a066e61f13cf4b Ben Gardon 2021-02-02 328 * a066e61f13cf4b Ben Gardon 2021-02-02 329 * Given a page table that has been removed from the TDP paging structure, a066e61f13cf4b Ben Gardon 2021-02-02 330 * iterates through the page table to clear SPTEs and free child page tables. a066e61f13cf4b Ben Gardon 2021-02-02 331 */ 9a77daacc87dee Ben Gardon 2021-02-02 332 static void handle_removed_tdp_mmu_page(struct kvm *kvm, u64 *pt, 9a77daacc87dee Ben Gardon 2021-02-02 333 bool shared) a066e61f13cf4b Ben Gardon 2021-02-02 334 { a066e61f13cf4b Ben Gardon 2021-02-02 335 struct kvm_mmu_page *sp = sptep_to_sp(pt); a066e61f13cf4b Ben Gardon 2021-02-02 336 int level = sp->role.level; a066e61f13cf4b Ben Gardon 2021-02-02 337 gfn_t gfn = sp->gfn; a066e61f13cf4b Ben Gardon 2021-02-02 338 u64 old_child_spte; 9a77daacc87dee Ben Gardon 2021-02-02 339 u64 *sptep; a066e61f13cf4b Ben Gardon 2021-02-02 340 int i; a066e61f13cf4b Ben Gardon 2021-02-02 341 a066e61f13cf4b Ben Gardon 2021-02-02 342 trace_kvm_mmu_prepare_zap_page(sp); a066e61f13cf4b Ben Gardon 2021-02-02 343 9a77daacc87dee Ben Gardon 2021-02-02 344 tdp_mmu_unlink_page(kvm, sp, shared); a066e61f13cf4b Ben Gardon 2021-02-02 345 a066e61f13cf4b Ben Gardon 2021-02-02 346 for (i = 0; i < PT64_ENT_PER_PAGE; i++) { 9a77daacc87dee Ben Gardon 2021-02-02 347 sptep = pt + i; 9a77daacc87dee Ben Gardon 2021-02-02 348 9a77daacc87dee Ben Gardon 2021-02-02 349 if (shared) { 9a77daacc87dee Ben Gardon 2021-02-02 350 old_child_spte = xchg(sptep, 0); 9a77daacc87dee Ben Gardon 2021-02-02 351 } else { 9a77daacc87dee Ben Gardon 2021-02-02 352 old_child_spte = READ_ONCE(*sptep); 9a77daacc87dee Ben Gardon 2021-02-02 353 WRITE_ONCE(*sptep, 0); 9a77daacc87dee Ben Gardon 2021-02-02 354 } a066e61f13cf4b Ben Gardon 2021-02-02 355 handle_changed_spte(kvm, kvm_mmu_page_as_id(sp), a066e61f13cf4b Ben Gardon 2021-02-02 356 gfn + (i * KVM_PAGES_PER_HPAGE(level - 1)), 9a77daacc87dee Ben Gardon 2021-02-02 357 old_child_spte, 0, level - 1, shared); a066e61f13cf4b Ben Gardon 2021-02-02 358 } a066e61f13cf4b Ben Gardon 2021-02-02 359 a066e61f13cf4b Ben Gardon 2021-02-02 360 kvm_flush_remote_tlbs_with_address(kvm, gfn, a066e61f13cf4b Ben Gardon 2021-02-02 361 KVM_PAGES_PER_HPAGE(level)); a066e61f13cf4b Ben Gardon 2021-02-02 362 7cca2d0b7e7d9f Ben Gardon 2021-02-02 363 call_rcu(&sp->rcu_head, tdp_mmu_free_sp_rcu_callback); a066e61f13cf4b Ben Gardon 2021-02-02 364 } a066e61f13cf4b Ben Gardon 2021-02-02 365 2f2fad0897cbfd Ben Gardon 2020-10-14 366 /** 2f2fad0897cbfd Ben Gardon 2020-10-14 367 * handle_changed_spte - handle bookkeeping associated with an SPTE change 2f2fad0897cbfd Ben Gardon 2020-10-14 368 * @kvm: kvm instance 2f2fad0897cbfd Ben Gardon 2020-10-14 369 * @as_id: the address space of the paging structure the SPTE was a part of 2f2fad0897cbfd Ben Gardon 2020-10-14 370 * @gfn: the base GFN that was mapped by the SPTE 2f2fad0897cbfd Ben Gardon 2020-10-14 371 * @old_spte: The value of the SPTE before the change 2f2fad0897cbfd Ben Gardon 2020-10-14 372 * @new_spte: The value of the SPTE after the change 2f2fad0897cbfd Ben Gardon 2020-10-14 373 * @level: the level of the PT the SPTE is part of in the paging structure 9a77daacc87dee Ben Gardon 2021-02-02 374 * @shared: This operation may not be running under the exclusive use of 9a77daacc87dee Ben Gardon 2021-02-02 375 * the MMU lock and the operation must synchronize with other 9a77daacc87dee Ben Gardon 2021-02-02 376 * threads that might be modifying SPTEs. 2f2fad0897cbfd Ben Gardon 2020-10-14 377 * 2f2fad0897cbfd Ben Gardon 2020-10-14 378 * Handle bookkeeping that might result from the modification of a SPTE. 2f2fad0897cbfd Ben Gardon 2020-10-14 379 * This function must be called for all TDP SPTE modifications. 2f2fad0897cbfd Ben Gardon 2020-10-14 380 */ 2f2fad0897cbfd Ben Gardon 2020-10-14 381 static void __handle_changed_spte(struct kvm *kvm, int as_id, gfn_t gfn, 9a77daacc87dee Ben Gardon 2021-02-02 382 u64 old_spte, u64 new_spte, int level, 9a77daacc87dee Ben Gardon 2021-02-02 383 bool shared) 2f2fad0897cbfd Ben Gardon 2020-10-14 384 { 2f2fad0897cbfd Ben Gardon 2020-10-14 385 bool was_present = is_shadow_present_pte(old_spte); 2f2fad0897cbfd Ben Gardon 2020-10-14 386 bool is_present = is_shadow_present_pte(new_spte); 2f2fad0897cbfd Ben Gardon 2020-10-14 387 bool was_leaf = was_present && is_last_spte(old_spte, level); 2f2fad0897cbfd Ben Gardon 2020-10-14 388 bool is_leaf = is_present && is_last_spte(new_spte, level); 2f2fad0897cbfd Ben Gardon 2020-10-14 389 bool pfn_changed = spte_to_pfn(old_spte) != spte_to_pfn(new_spte); 2f2fad0897cbfd Ben Gardon 2020-10-14 390 2f2fad0897cbfd Ben Gardon 2020-10-14 391 WARN_ON(level > PT64_ROOT_MAX_LEVEL); 2f2fad0897cbfd Ben Gardon 2020-10-14 392 WARN_ON(level < PG_LEVEL_4K); 764388ce598f0c Sean Christopherson 2020-10-23 393 WARN_ON(gfn & (KVM_PAGES_PER_HPAGE(level) - 1)); 2f2fad0897cbfd Ben Gardon 2020-10-14 394 2f2fad0897cbfd Ben Gardon 2020-10-14 395 /* 2f2fad0897cbfd Ben Gardon 2020-10-14 396 * If this warning were to trigger it would indicate that there was a 2f2fad0897cbfd Ben Gardon 2020-10-14 397 * missing MMU notifier or a race with some notifier handler. 2f2fad0897cbfd Ben Gardon 2020-10-14 398 * A present, leaf SPTE should never be directly replaced with another 2f2fad0897cbfd Ben Gardon 2020-10-14 399 * present leaf SPTE pointing to a differnt PFN. A notifier handler 2f2fad0897cbfd Ben Gardon 2020-10-14 400 * should be zapping the SPTE before the main MM's page table is 2f2fad0897cbfd Ben Gardon 2020-10-14 401 * changed, or the SPTE should be zeroed, and the TLBs flushed by the 2f2fad0897cbfd Ben Gardon 2020-10-14 402 * thread before replacement. 2f2fad0897cbfd Ben Gardon 2020-10-14 403 */ 2f2fad0897cbfd Ben Gardon 2020-10-14 404 if (was_leaf && is_leaf && pfn_changed) { 2f2fad0897cbfd Ben Gardon 2020-10-14 405 pr_err("Invalid SPTE change: cannot replace a present leaf\n" 2f2fad0897cbfd Ben Gardon 2020-10-14 406 "SPTE with another present leaf SPTE mapping a\n" 2f2fad0897cbfd Ben Gardon 2020-10-14 407 "different PFN!\n" 2f2fad0897cbfd Ben Gardon 2020-10-14 408 "as_id: %d gfn: %llx old_spte: %llx new_spte: %llx level: %d", 2f2fad0897cbfd Ben Gardon 2020-10-14 409 as_id, gfn, old_spte, new_spte, level); 2f2fad0897cbfd Ben Gardon 2020-10-14 410 2f2fad0897cbfd Ben Gardon 2020-10-14 411 /* 2f2fad0897cbfd Ben Gardon 2020-10-14 412 * Crash the host to prevent error propagation and guest data 2f2fad0897cbfd Ben Gardon 2020-10-14 413 * courruption. 2f2fad0897cbfd Ben Gardon 2020-10-14 414 */ 2f2fad0897cbfd Ben Gardon 2020-10-14 415 BUG(); 2f2fad0897cbfd Ben Gardon 2020-10-14 416 } 2f2fad0897cbfd Ben Gardon 2020-10-14 417 2f2fad0897cbfd Ben Gardon 2020-10-14 418 if (old_spte == new_spte) 2f2fad0897cbfd Ben Gardon 2020-10-14 419 return; 2f2fad0897cbfd Ben Gardon 2020-10-14 420 b9a98c3437e353 Ben Gardon 2020-10-27 421 trace_kvm_tdp_mmu_spte_changed(as_id, gfn, level, old_spte, new_spte); b9a98c3437e353 Ben Gardon 2020-10-27 422 2f2fad0897cbfd Ben Gardon 2020-10-14 423 /* 2f2fad0897cbfd Ben Gardon 2020-10-14 424 * The only times a SPTE should be changed from a non-present to 2f2fad0897cbfd Ben Gardon 2020-10-14 425 * non-present state is when an MMIO entry is installed/modified/ 2f2fad0897cbfd Ben Gardon 2020-10-14 426 * removed. In that case, there is nothing to do here. 2f2fad0897cbfd Ben Gardon 2020-10-14 427 */ 2f2fad0897cbfd Ben Gardon 2020-10-14 428 if (!was_present && !is_present) { 2f2fad0897cbfd Ben Gardon 2020-10-14 429 /* 2f2fad0897cbfd Ben Gardon 2020-10-14 430 * If this change does not involve a MMIO SPTE, it is 2f2fad0897cbfd Ben Gardon 2020-10-14 431 * unexpected. Log the change, though it should not impact the 2f2fad0897cbfd Ben Gardon 2020-10-14 432 * guest since both the former and current SPTEs are nonpresent. 2f2fad0897cbfd Ben Gardon 2020-10-14 433 */ 2f2fad0897cbfd Ben Gardon 2020-10-14 434 if (WARN_ON(!is_mmio_spte(old_spte) && !is_mmio_spte(new_spte))) 2f2fad0897cbfd Ben Gardon 2020-10-14 435 pr_err("Unexpected SPTE change! Nonpresent SPTEs\n" 2f2fad0897cbfd Ben Gardon 2020-10-14 436 "should not be replaced with another,\n" 2f2fad0897cbfd Ben Gardon 2020-10-14 437 "different nonpresent SPTE, unless one or both\n" 2f2fad0897cbfd Ben Gardon 2020-10-14 438 "are MMIO SPTEs.\n" 2f2fad0897cbfd Ben Gardon 2020-10-14 439 "as_id: %d gfn: %llx old_spte: %llx new_spte: %llx level: %d", 2f2fad0897cbfd Ben Gardon 2020-10-14 440 as_id, gfn, old_spte, new_spte, level); 2f2fad0897cbfd Ben Gardon 2020-10-14 441 return; 2f2fad0897cbfd Ben Gardon 2020-10-14 442 } 2f2fad0897cbfd Ben Gardon 2020-10-14 443 2f2fad0897cbfd Ben Gardon 2020-10-14 444 2f2fad0897cbfd Ben Gardon 2020-10-14 445 if (was_leaf && is_dirty_spte(old_spte) && 2f2fad0897cbfd Ben Gardon 2020-10-14 446 (!is_dirty_spte(new_spte) || pfn_changed)) 2f2fad0897cbfd Ben Gardon 2020-10-14 447 kvm_set_pfn_dirty(spte_to_pfn(old_spte)); 2f2fad0897cbfd Ben Gardon 2020-10-14 448 2f2fad0897cbfd Ben Gardon 2020-10-14 449 /* 2f2fad0897cbfd Ben Gardon 2020-10-14 450 * Recursively handle child PTs if the change removed a subtree from 2f2fad0897cbfd Ben Gardon 2020-10-14 451 * the paging structure. 2f2fad0897cbfd Ben Gardon 2020-10-14 452 */ a066e61f13cf4b Ben Gardon 2021-02-02 453 if (was_present && !was_leaf && (pfn_changed || !is_present)) a066e61f13cf4b Ben Gardon 2021-02-02 454 handle_removed_tdp_mmu_page(kvm, 9a77daacc87dee Ben Gardon 2021-02-02 455 spte_to_child_pt(old_spte, level), shared); 2f2fad0897cbfd Ben Gardon 2020-10-14 456 } 2f2fad0897cbfd Ben Gardon 2020-10-14 457 2f2fad0897cbfd Ben Gardon 2020-10-14 458 static void handle_changed_spte(struct kvm *kvm, int as_id, gfn_t gfn, 9a77daacc87dee Ben Gardon 2021-02-02 459 u64 old_spte, u64 new_spte, int level, 9a77daacc87dee Ben Gardon 2021-02-02 460 bool shared) 2f2fad0897cbfd Ben Gardon 2020-10-14 461 { 9a77daacc87dee Ben Gardon 2021-02-02 462 __handle_changed_spte(kvm, as_id, gfn, old_spte, new_spte, level, 9a77daacc87dee Ben Gardon 2021-02-02 463 shared); f8e144971c6834 Ben Gardon 2020-10-14 464 handle_changed_spte_acc_track(old_spte, new_spte, level); a6a0b05da9f37f Ben Gardon 2020-10-14 465 handle_changed_spte_dirty_log(kvm, as_id, gfn, old_spte, a6a0b05da9f37f Ben Gardon 2020-10-14 466 new_spte, level); 2f2fad0897cbfd Ben Gardon 2020-10-14 467 } faaf05b00aecdb Ben Gardon 2020-10-14 468 9a77daacc87dee Ben Gardon 2021-02-02 469 /* 9a77daacc87dee Ben Gardon 2021-02-02 470 * tdp_mmu_set_spte_atomic - Set a TDP MMU SPTE atomically and handle the 9a77daacc87dee Ben Gardon 2021-02-02 471 * associated bookkeeping 9a77daacc87dee Ben Gardon 2021-02-02 472 * 9a77daacc87dee Ben Gardon 2021-02-02 473 * @kvm: kvm instance 9a77daacc87dee Ben Gardon 2021-02-02 474 * @iter: a tdp_iter instance currently on the SPTE that should be set 9a77daacc87dee Ben Gardon 2021-02-02 475 * @new_spte: The value the SPTE should be set to 9a77daacc87dee Ben Gardon 2021-02-02 476 * Returns: true if the SPTE was set, false if it was not. If false is returned, 9a77daacc87dee Ben Gardon 2021-02-02 477 * this function will have no side-effects. 9a77daacc87dee Ben Gardon 2021-02-02 478 */ 9a77daacc87dee Ben Gardon 2021-02-02 479 static inline bool tdp_mmu_set_spte_atomic(struct kvm *kvm, 9a77daacc87dee Ben Gardon 2021-02-02 480 struct tdp_iter *iter, 9a77daacc87dee Ben Gardon 2021-02-02 481 u64 new_spte) 9a77daacc87dee Ben Gardon 2021-02-02 482 { 9a77daacc87dee Ben Gardon 2021-02-02 @483 u64 *root_pt = tdp_iter_root_pt(iter); 9a77daacc87dee Ben Gardon 2021-02-02 484 struct kvm_mmu_page *root = sptep_to_sp(root_pt); 9a77daacc87dee Ben Gardon 2021-02-02 485 int as_id = kvm_mmu_page_as_id(root); 9a77daacc87dee Ben Gardon 2021-02-02 486 9a77daacc87dee Ben Gardon 2021-02-02 487 lockdep_assert_held_read(&kvm->mmu_lock); 9a77daacc87dee Ben Gardon 2021-02-02 488 9a77daacc87dee Ben Gardon 2021-02-02 489 if (cmpxchg64(rcu_dereference(iter->sptep), iter->old_spte, 9a77daacc87dee Ben Gardon 2021-02-02 490 new_spte) != iter->old_spte) 9a77daacc87dee Ben Gardon 2021-02-02 491 return false; 9a77daacc87dee Ben Gardon 2021-02-02 492 9a77daacc87dee Ben Gardon 2021-02-02 493 handle_changed_spte(kvm, as_id, iter->gfn, iter->old_spte, new_spte, 9a77daacc87dee Ben Gardon 2021-02-02 494 iter->level, true); 9a77daacc87dee Ben Gardon 2021-02-02 495 9a77daacc87dee Ben Gardon 2021-02-02 496 return true; 9a77daacc87dee Ben Gardon 2021-02-02 497 } 9a77daacc87dee Ben Gardon 2021-02-02 498 --- 0-DAY CI Kernel Test Service, Intel Corporation https://lists.01.org/hyperkitty/list/kbuild-all(a)lists.01.org