All of lore.kernel.org
 help / color / mirror / Atom feed
From: Hugh Dickins <hughd@google.com>
To: Andrew Morton <akpm@linux-foundation.org>
Cc: Mike Kravetz <mike.kravetz@oracle.com>,
	Mike Rapoport <rppt@kernel.org>,
	"Kirill A. Shutemov" <kirill.shutemov@linux.intel.com>,
	Matthew Wilcox <willy@infradead.org>,
	David Hildenbrand <david@redhat.com>,
	Suren Baghdasaryan <surenb@google.com>,
	Qi Zheng <zhengqi.arch@bytedance.com>,
	Yang Shi <shy828301@gmail.com>,
	Mel Gorman <mgorman@techsingularity.net>,
	Peter Xu <peterx@redhat.com>,
	Peter Zijlstra <peterz@infradead.org>,
	Will Deacon <will@kernel.org>, Yu Zhao <yuzhao@google.com>,
	Alistair Popple <apopple@nvidia.com>,
	Ralph Campbell <rcampbell@nvidia.com>,
	Ira Weiny <ira.weiny@intel.com>,
	Steven Price <steven.price@arm.com>,
	SeongJae Park <sj@kernel.org>,
	Lorenzo Stoakes <lstoakes@gmail.com>,
	Huang Ying <ying.huang@intel.com>,
	Naoya Horiguchi <naoya.horiguchi@nec.com>,
	Christophe Leroy <christophe.leroy@csgroup.eu>,
	Zack Rusin <zackr@vmware.com>, Jason Gunthorpe <jgg@ziepe.ca>,
	Axel Rasmussen <axelrasmussen@google.com>,
	Anshuman Khandual <anshuman.khandual@arm.com>,
	Pasha Tatashin <pasha.tatashin@soleen.com>,
	Miaohe Lin <linmiaohe@huawei.com>,
	Minchan Kim <minchan@kernel.org>,
	Christoph Hellwig <hch@infradead.org>, Song Liu <song@kernel.org>,
	Thomas Hellstrom <thomas.hellstrom@linux.intel.com>,
	Ryan Roberts <ryan.roberts@arm.com>,
	linux-kernel@vger.kernel.org, linux-mm@kvack.org
Subject: [PATCH v2 18/32] mm/mprotect: delete pmd_none_or_clear_bad_unless_trans_huge()
Date: Thu, 8 Jun 2023 18:30:48 -0700 (PDT)	[thread overview]
Message-ID: <725a42a9-91e9-c868-925-e3a5fd40bb4f@google.com> (raw)
In-Reply-To: <c1c9a74a-bc5b-15ea-e5d2-8ec34bc921d@google.com>

change_pmd_range() had special pmd_none_or_clear_bad_unless_trans_huge(),
required to avoid "bad" choices when setting automatic NUMA hinting under
mmap_read_lock(); but most of that is already covered in pte_offset_map()
now.  change_pmd_range() just wants a pmd_none() check before wasting
time on MMU notifiers, then checks on the read-once _pmd value to work
out what's needed for huge cases.  If change_pte_range() returns -EAGAIN
to retry if pte_offset_map_lock() fails, nothing more special is needed.

Signed-off-by: Hugh Dickins <hughd@google.com>
---
 mm/mprotect.c | 74 ++++++++++++---------------------------------------
 1 file changed, 17 insertions(+), 57 deletions(-)

diff --git a/mm/mprotect.c b/mm/mprotect.c
index c5a13c0f1017..64e1df0af514 100644
--- a/mm/mprotect.c
+++ b/mm/mprotect.c
@@ -93,22 +93,9 @@ static long change_pte_range(struct mmu_gather *tlb,
 	bool uffd_wp_resolve = cp_flags & MM_CP_UFFD_WP_RESOLVE;
 
 	tlb_change_page_size(tlb, PAGE_SIZE);
-
-	/*
-	 * Can be called with only the mmap_lock for reading by
-	 * prot_numa so we must check the pmd isn't constantly
-	 * changing from under us from pmd_none to pmd_trans_huge
-	 * and/or the other way around.
-	 */
-	if (pmd_trans_unstable(pmd))
-		return 0;
-
-	/*
-	 * The pmd points to a regular pte so the pmd can't change
-	 * from under us even if the mmap_lock is only hold for
-	 * reading.
-	 */
 	pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
+	if (!pte)
+		return -EAGAIN;
 
 	/* Get target node for single threaded private VMAs */
 	if (prot_numa && !(vma->vm_flags & VM_SHARED) &&
@@ -301,26 +288,6 @@ static long change_pte_range(struct mmu_gather *tlb,
 	return pages;
 }
 
-/*
- * Used when setting automatic NUMA hinting protection where it is
- * critical that a numa hinting PMD is not confused with a bad PMD.
- */
-static inline int pmd_none_or_clear_bad_unless_trans_huge(pmd_t *pmd)
-{
-	pmd_t pmdval = pmdp_get_lockless(pmd);
-
-	if (pmd_none(pmdval))
-		return 1;
-	if (pmd_trans_huge(pmdval))
-		return 0;
-	if (unlikely(pmd_bad(pmdval))) {
-		pmd_clear_bad(pmd);
-		return 1;
-	}
-
-	return 0;
-}
-
 /*
  * Return true if we want to split THPs into PTE mappings in change
  * protection procedure, false otherwise.
@@ -398,7 +365,8 @@ static inline long change_pmd_range(struct mmu_gather *tlb,
 	pmd = pmd_offset(pud, addr);
 	do {
 		long ret;
-
+		pmd_t _pmd;
+again:
 		next = pmd_addr_end(addr, end);
 
 		ret = change_pmd_prepare(vma, pmd, cp_flags);
@@ -406,16 +374,8 @@ static inline long change_pmd_range(struct mmu_gather *tlb,
 			pages = ret;
 			break;
 		}
-		/*
-		 * Automatic NUMA balancing walks the tables with mmap_lock
-		 * held for read. It's possible a parallel update to occur
-		 * between pmd_trans_huge() and a pmd_none_or_clear_bad()
-		 * check leading to a false positive and clearing.
-		 * Hence, it's necessary to atomically read the PMD value
-		 * for all the checks.
-		 */
-		if (!is_swap_pmd(*pmd) && !pmd_devmap(*pmd) &&
-		     pmd_none_or_clear_bad_unless_trans_huge(pmd))
+
+		if (pmd_none(*pmd))
 			goto next;
 
 		/* invoke the mmu notifier if the pmd is populated */
@@ -426,7 +386,8 @@ static inline long change_pmd_range(struct mmu_gather *tlb,
 			mmu_notifier_invalidate_range_start(&range);
 		}
 
-		if (is_swap_pmd(*pmd) || pmd_trans_huge(*pmd) || pmd_devmap(*pmd)) {
+		_pmd = pmdp_get_lockless(pmd);
+		if (is_swap_pmd(_pmd) || pmd_trans_huge(_pmd) || pmd_devmap(_pmd)) {
 			if ((next - addr != HPAGE_PMD_SIZE) ||
 			    pgtable_split_needed(vma, cp_flags)) {
 				__split_huge_pmd(vma, pmd, addr, false, NULL);
@@ -441,15 +402,10 @@ static inline long change_pmd_range(struct mmu_gather *tlb,
 					break;
 				}
 			} else {
-				/*
-				 * change_huge_pmd() does not defer TLB flushes,
-				 * so no need to propagate the tlb argument.
-				 */
-				int nr_ptes = change_huge_pmd(tlb, vma, pmd,
+				ret = change_huge_pmd(tlb, vma, pmd,
 						addr, newprot, cp_flags);
-
-				if (nr_ptes) {
-					if (nr_ptes == HPAGE_PMD_NR) {
+				if (ret) {
+					if (ret == HPAGE_PMD_NR) {
 						pages += HPAGE_PMD_NR;
 						nr_huge_updates++;
 					}
@@ -460,8 +416,12 @@ static inline long change_pmd_range(struct mmu_gather *tlb,
 			}
 			/* fall through, the trans huge pmd just split */
 		}
-		pages += change_pte_range(tlb, vma, pmd, addr, next,
-					  newprot, cp_flags);
+
+		ret = change_pte_range(tlb, vma, pmd, addr, next, newprot,
+				       cp_flags);
+		if (ret < 0)
+			goto again;
+		pages += ret;
 next:
 		cond_resched();
 	} while (pmd++, addr = next, addr != end);
-- 
2.35.3


  parent reply	other threads:[~2023-06-09  1:30 UTC|newest]

Thread overview: 82+ messages / expand[flat|nested]  mbox.gz  Atom feed  top
2023-06-09  0:54 [PATCH v2 00/32] mm: allow pte_offset_map[_lock]() to fail Hugh Dickins
2023-06-09  1:06 ` [PATCH v2 01/32] mm: use pmdp_get_lockless() without surplus barrier() Hugh Dickins
2023-06-09  1:08 ` [PATCH v2 02/32] mm/migrate: remove cruft from migration_entry_wait()s Hugh Dickins
2023-06-09  1:09 ` [PATCH v2 03/32] mm/pgtable: kmap_local_page() instead of kmap_atomic() Hugh Dickins
2023-06-09  1:10 ` [PATCH v2 04/32] mm/pgtable: allow pte_offset_map[_lock]() to fail Hugh Dickins
2023-07-11  1:23   ` Zi Yan
2023-07-28 13:53   ` Yongqin Liu
2023-07-28 14:05     ` Matthew Wilcox
2023-07-28 16:58       ` Hugh Dickins
2023-08-05 16:06         ` Yongqin Liu
2023-08-05 17:07           ` Matthew Wilcox
2023-08-08  0:29             ` John Hubbard
2023-06-09  1:11 ` [PATCH v2 05/32] mm/filemap: allow pte_offset_map_lock() " Hugh Dickins
2023-07-11  1:34   ` Zi Yan
2023-07-11  5:21     ` Hugh Dickins
2023-06-09  1:12 ` [PATCH v2 06/32] mm/page_vma_mapped: delete bogosity in page_vma_mapped_walk() Hugh Dickins
2023-07-11  1:47   ` Zi Yan
2023-06-09  1:14 ` [PATCH v2 07/32] mm/page_vma_mapped: reformat map_pte() with less indentation Hugh Dickins
2023-07-11  1:56   ` Zi Yan
2023-06-09  1:15 ` [PATCH v2 08/32] mm/page_vma_mapped: pte_offset_map_nolock() not pte_lockptr() Hugh Dickins
2023-06-09  1:17 ` [PATCH v2 09/32] mm/pagewalkers: ACTION_AGAIN if pte_offset_map_lock() fails Hugh Dickins
2023-06-09  1:18 ` [PATCH v2 10/32] mm/pagewalk: walk_pte_range() allow for pte_offset_map() Hugh Dickins
2023-06-09  1:20 ` [PATCH v2 11/32] mm/vmwgfx: simplify pmd & pud mapping dirty helpers Hugh Dickins
2023-06-09  1:21 ` [PATCH v2 12/32] mm/vmalloc: vmalloc_to_page() use pte_offset_kernel() Hugh Dickins
2023-07-10 14:42   ` Mark Brown
2023-07-10 14:42     ` Mark Brown
2023-07-10 17:18     ` Lorenzo Stoakes
2023-07-10 17:18       ` Lorenzo Stoakes
2023-07-10 17:33       ` Mark Brown
2023-07-10 17:33         ` Mark Brown
2023-07-11  4:34         ` Hugh Dickins
2023-07-11  4:34           ` Hugh Dickins
2023-07-11 15:34           ` Mark Brown
2023-07-11 15:34             ` Mark Brown
2023-07-11 16:13             ` Hugh Dickins
2023-07-11 16:13               ` Hugh Dickins
2023-07-11 16:34               ` Mark Brown
2023-07-11 16:34                 ` Mark Brown
2023-07-11 17:57               ` Mark Brown
2023-07-11 17:57                 ` Mark Brown
2023-07-13 11:19                 ` Linux regression tracking #update (Thorsten Leemhuis)
2023-07-13 11:19                   ` Linux regression tracking #update (Thorsten Leemhuis)
2023-07-20 10:32                 ` Will Deacon
2023-07-20 10:32                   ` Will Deacon
2023-07-20 12:06                   ` Mark Brown
2023-07-20 12:06                     ` Mark Brown
2023-08-08  5:52                     ` Linux regression tracking (Thorsten Leemhuis)
2023-08-08  5:52                       ` Linux regression tracking (Thorsten Leemhuis)
2023-08-08 11:09                       ` Mark Brown
2023-08-08 11:09                         ` Mark Brown
2023-08-11  8:00                         ` Linux regression tracking #update (Thorsten Leemhuis)
2023-08-11  8:00                           ` Linux regression tracking #update (Thorsten Leemhuis)
2023-07-11 14:48     ` Linux regression tracking #adding (Thorsten Leemhuis)
2023-07-11 14:48       ` Linux regression tracking #adding (Thorsten Leemhuis)
2023-06-09  1:23 ` [PATCH v2 13/32] mm/hmm: retry if pte_offset_map() fails Hugh Dickins
2023-06-09  1:24 ` [PATCH v2 14/32] mm/userfaultfd: " Hugh Dickins
2023-06-09  1:26 ` [PATCH v2 15/32] mm/userfaultfd: allow pte_offset_map_lock() to fail Hugh Dickins
2023-06-09  1:27 ` [PATCH v2 16/32] mm/debug_vm_pgtable,page_table_check: warn pte map fails Hugh Dickins
2023-06-09  1:29 ` [PATCH v2 17/32] mm/various: give up if pte_offset_map[_lock]() fails Hugh Dickins
2023-06-09  1:30 ` Hugh Dickins [this message]
2023-06-09  1:32 ` [PATCH v2 19/32] mm/mremap: retry if either pte_offset_map_*lock() fails Hugh Dickins
2023-06-09  1:34 ` [PATCH v2 20/32] mm/madvise: clean up pte_offset_map_lock() scans Hugh Dickins
2023-06-09  1:35 ` [PATCH v2 21/32] mm/madvise: clean up force_shm_swapin_readahead() Hugh Dickins
2023-06-09  1:36 ` [PATCH v2 22/32] mm/swapoff: allow pte_offset_map[_lock]() to fail Hugh Dickins
2023-06-09  1:37 ` [PATCH v2 23/32] mm/mglru: allow pte_offset_map_nolock() " Hugh Dickins
2023-06-09  1:38 ` [PATCH v2 24/32] mm/migrate_device: allow pte_offset_map_lock() " Hugh Dickins
2023-06-09  1:39 ` [PATCH v2 25/32] mm/gup: remove FOLL_SPLIT_PMD use of pmd_trans_unstable() Hugh Dickins
2023-06-09 18:24   ` Yang Shi
2023-06-09  1:41 ` [PATCH v2 26/32] mm/huge_memory: split huge pmd under one pte_offset_map() Hugh Dickins
2023-06-09  1:42 ` [PATCH v2 27/32] mm/khugepaged: allow pte_offset_map[_lock]() to fail Hugh Dickins
2023-06-09  1:43 ` [PATCH v2 28/32] mm/memory: " Hugh Dickins
2023-06-09 20:06   ` Andrew Morton
2023-06-09 20:11     ` Hugh Dickins
2023-06-12  9:10       ` Ryan Roberts
2023-06-15 23:10   ` [PATCH v2 28/32 fix] mm/memory: allow pte_offset_map[_lock]() to fail: fix Hugh Dickins
2023-06-09  1:45 ` [PATCH v2 29/32] mm/memory: handle_pte_fault() use pte_offset_map_nolock() Hugh Dickins
2023-06-09  1:50 ` [PATCH v2 30/32] mm/pgtable: delete pmd_trans_unstable() and friends Hugh Dickins
2023-06-09  1:52 ` [PATCH v2 31/32] mm/swap: swap_vma_readahead() do the pte_offset_map() Hugh Dickins
2023-06-12  8:03   ` Huang, Ying
2023-06-14  3:58     ` Hugh Dickins
2023-06-09  1:53 ` [PATCH v2 32/32] perf/core: Allow pte_offset_map() to fail Hugh Dickins
2023-06-20  6:50 ` [PATCH] mm/swapfile: delete outdated pte_offset_map() comment Hugh Dickins

Reply instructions:

You may reply publicly to this message via plain-text email
using any one of the following methods:

* Save the following mbox file, import it into your mail client,
  and reply-to-all from there: mbox

  Avoid top-posting and favor interleaved quoting:
  https://en.wikipedia.org/wiki/Posting_style#Interleaved_style

* Reply using the --to, --cc, and --in-reply-to
  switches of git-send-email(1):

  git send-email \
    --in-reply-to=725a42a9-91e9-c868-925-e3a5fd40bb4f@google.com \
    --to=hughd@google.com \
    --cc=akpm@linux-foundation.org \
    --cc=anshuman.khandual@arm.com \
    --cc=apopple@nvidia.com \
    --cc=axelrasmussen@google.com \
    --cc=christophe.leroy@csgroup.eu \
    --cc=david@redhat.com \
    --cc=hch@infradead.org \
    --cc=ira.weiny@intel.com \
    --cc=jgg@ziepe.ca \
    --cc=kirill.shutemov@linux.intel.com \
    --cc=linmiaohe@huawei.com \
    --cc=linux-kernel@vger.kernel.org \
    --cc=linux-mm@kvack.org \
    --cc=lstoakes@gmail.com \
    --cc=mgorman@techsingularity.net \
    --cc=mike.kravetz@oracle.com \
    --cc=minchan@kernel.org \
    --cc=naoya.horiguchi@nec.com \
    --cc=pasha.tatashin@soleen.com \
    --cc=peterx@redhat.com \
    --cc=peterz@infradead.org \
    --cc=rcampbell@nvidia.com \
    --cc=rppt@kernel.org \
    --cc=ryan.roberts@arm.com \
    --cc=shy828301@gmail.com \
    --cc=sj@kernel.org \
    --cc=song@kernel.org \
    --cc=steven.price@arm.com \
    --cc=surenb@google.com \
    --cc=thomas.hellstrom@linux.intel.com \
    --cc=will@kernel.org \
    --cc=willy@infradead.org \
    --cc=ying.huang@intel.com \
    --cc=yuzhao@google.com \
    --cc=zackr@vmware.com \
    --cc=zhengqi.arch@bytedance.com \
    /path/to/YOUR_REPLY

  https://kernel.org/pub/software/scm/git/docs/git-send-email.html

* If your mail client supports setting the In-Reply-To header
  via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line before the message body.
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.