* [PATCH 1/7] mm: khugepaged: check THP flag in hugepage_vma_check()
2022-05-20 21:15 [mm-unstable PATCH 0/7] Cleanup transhuge_xxx helpers Yang Shi
@ 2022-05-20 21:15 ` Yang Shi
2022-05-20 21:16 ` [PATCH 2/7] mm: thp: introduce transhuge_vma_size_ok() helper Yang Shi
` (5 subsequent siblings)
6 siblings, 0 replies; 10+ messages in thread
From: Yang Shi @ 2022-05-20 21:15 UTC (permalink / raw)
To: vbabka, kirill.shutemov, akpm; +Cc: shy828301, linux-mm, linux-kernel
Currently the THP flag check in hugepage_vma_check() will fallthrough if
the flag is NEVER and VM_HUGEPAGE is set. This is not a problem for now
since all the callers have the flag checked before or can't be invoked if
the flag is NEVER.
However, the following patch will call hugepage_vma_check() in more
places, for example, page fault, so this flag must be checked in
hugepge_vma_check().
Signed-off-by: Yang Shi <shy828301@gmail.com>
---
mm/khugepaged.c | 3 +++
1 file changed, 3 insertions(+)
diff --git a/mm/khugepaged.c b/mm/khugepaged.c
index 637bfecd6bf5..92e91c08d96a 100644
--- a/mm/khugepaged.c
+++ b/mm/khugepaged.c
@@ -458,6 +458,9 @@ bool hugepage_vma_check(struct vm_area_struct *vma,
if (shmem_file(vma->vm_file))
return shmem_huge_enabled(vma);
+ if (!khugepaged_enabled())
+ return false;
+
/* THP settings require madvise. */
if (!(vm_flags & VM_HUGEPAGE) && !khugepaged_always())
return false;
--
2.26.3
^ permalink raw reply related [flat|nested] 10+ messages in thread
* [PATCH 2/7] mm: thp: introduce transhuge_vma_size_ok() helper
2022-05-20 21:15 [mm-unstable PATCH 0/7] Cleanup transhuge_xxx helpers Yang Shi
2022-05-20 21:15 ` [PATCH 1/7] mm: khugepaged: check THP flag in hugepage_vma_check() Yang Shi
@ 2022-05-20 21:16 ` Yang Shi
2022-05-21 1:39 ` Hillf Danton
2022-05-20 21:16 ` [PATCH 3/7] mm: khugepaged: remove the redundant anon vma check Yang Shi
` (4 subsequent siblings)
6 siblings, 1 reply; 10+ messages in thread
From: Yang Shi @ 2022-05-20 21:16 UTC (permalink / raw)
To: vbabka, kirill.shutemov, akpm; +Cc: shy828301, linux-mm, linux-kernel
There are couple of places that check whether the vma size is ok for
THP or not, they are open coded and duplicate, introduce
transhuge_vma_size_ok() helper to do the job.
Signed-off-by: Yang Shi <shy828301@gmail.com>
---
include/linux/huge_mm.h | 17 +++++++++++++++++
mm/huge_memory.c | 5 +----
mm/khugepaged.c | 12 ++++++------
3 files changed, 24 insertions(+), 10 deletions(-)
diff --git a/include/linux/huge_mm.h b/include/linux/huge_mm.h
index 648cb3ce7099..a8f61db47f2a 100644
--- a/include/linux/huge_mm.h
+++ b/include/linux/huge_mm.h
@@ -116,6 +116,18 @@ extern struct kobj_attribute shmem_enabled_attr;
extern unsigned long transparent_hugepage_flags;
+/*
+ * The vma size has to be large enough to hold an aligned HPAGE_PMD_SIZE area.
+ */
+static inline bool transhuge_vma_size_ok(struct vm_area_struct *vma)
+{
+ if (round_up(vma->vm_start, HPAGE_PMD_SIZE) <
+ (vma->vm_end & HPAGE_PMD_MASK))
+ return true;
+
+ return false;
+}
+
static inline bool transhuge_vma_suitable(struct vm_area_struct *vma,
unsigned long addr)
{
@@ -345,6 +357,11 @@ static inline bool transparent_hugepage_active(struct vm_area_struct *vma)
return false;
}
+static inline bool transhuge_vma_size_ok(struct vm_area_struct *vma)
+{
+ return false;
+}
+
static inline bool transhuge_vma_suitable(struct vm_area_struct *vma,
unsigned long addr)
{
diff --git a/mm/huge_memory.c b/mm/huge_memory.c
index 80e8b58b4f39..d633f97452c1 100644
--- a/mm/huge_memory.c
+++ b/mm/huge_memory.c
@@ -71,10 +71,7 @@ unsigned long huge_zero_pfn __read_mostly = ~0UL;
bool transparent_hugepage_active(struct vm_area_struct *vma)
{
- /* The addr is used to check if the vma size fits */
- unsigned long addr = (vma->vm_end & HPAGE_PMD_MASK) - HPAGE_PMD_SIZE;
-
- if (!transhuge_vma_suitable(vma, addr))
+ if (!transhuge_vma_size_ok(vma))
return false;
if (vma_is_anonymous(vma))
return __transparent_hugepage_enabled(vma);
diff --git a/mm/khugepaged.c b/mm/khugepaged.c
index 92e91c08d96a..5bdb30a76f05 100644
--- a/mm/khugepaged.c
+++ b/mm/khugepaged.c
@@ -454,6 +454,9 @@ bool hugepage_vma_check(struct vm_area_struct *vma,
vma->vm_pgoff, HPAGE_PMD_NR))
return false;
+ if (!transhuge_vma_size_ok(vma))
+ return false;
+
/* Enabled via shmem mount options or sysfs settings. */
if (shmem_file(vma->vm_file))
return shmem_huge_enabled(vma);
@@ -512,9 +515,7 @@ void khugepaged_enter_vma(struct vm_area_struct *vma,
unsigned long vm_flags)
{
if (!test_bit(MMF_VM_HUGEPAGE, &vma->vm_mm->flags) &&
- khugepaged_enabled() &&
- (((vma->vm_start + ~HPAGE_PMD_MASK) & HPAGE_PMD_MASK) <
- (vma->vm_end & HPAGE_PMD_MASK))) {
+ khugepaged_enabled()) {
if (hugepage_vma_check(vma, vm_flags))
__khugepaged_enter(vma->vm_mm);
}
@@ -2144,10 +2145,9 @@ static unsigned int khugepaged_scan_mm_slot(unsigned int pages,
progress++;
continue;
}
- hstart = (vma->vm_start + ~HPAGE_PMD_MASK) & HPAGE_PMD_MASK;
+
+ hstart = round_up(vma->vm_start, HPAGE_PMD_SIZE);
hend = vma->vm_end & HPAGE_PMD_MASK;
- if (hstart >= hend)
- goto skip;
if (khugepaged_scan.address > hend)
goto skip;
if (khugepaged_scan.address < hstart)
--
2.26.3
^ permalink raw reply related [flat|nested] 10+ messages in thread
* Re: [PATCH 2/7] mm: thp: introduce transhuge_vma_size_ok() helper
2022-05-20 21:16 ` [PATCH 2/7] mm: thp: introduce transhuge_vma_size_ok() helper Yang Shi
@ 2022-05-21 1:39 ` Hillf Danton
2022-05-23 18:12 ` Yang Shi
0 siblings, 1 reply; 10+ messages in thread
From: Hillf Danton @ 2022-05-21 1:39 UTC (permalink / raw)
To: Yang Shi; +Cc: vbabka, kirill.shutemov, akpm, linux-mm, linux-kernel
On Fri, 20 May 2022 14:16:00 -0700 Yang Shi wrote:
> There are couple of places that check whether the vma size is ok for
> THP or not, they are open coded and duplicate, introduce
> transhuge_vma_size_ok() helper to do the job.
>
> Signed-off-by: Yang Shi <shy828301@gmail.com>
> ---
> include/linux/huge_mm.h | 17 +++++++++++++++++
> mm/huge_memory.c | 5 +----
> mm/khugepaged.c | 12 ++++++------
> 3 files changed, 24 insertions(+), 10 deletions(-)
>
> diff --git a/include/linux/huge_mm.h b/include/linux/huge_mm.h
> index 648cb3ce7099..a8f61db47f2a 100644
> --- a/include/linux/huge_mm.h
> +++ b/include/linux/huge_mm.h
> @@ -116,6 +116,18 @@ extern struct kobj_attribute shmem_enabled_attr;
>
> extern unsigned long transparent_hugepage_flags;
>
> +/*
> + * The vma size has to be large enough to hold an aligned HPAGE_PMD_SIZE area.
> + */
> +static inline bool transhuge_vma_size_ok(struct vm_area_struct *vma)
> +{
> + if (round_up(vma->vm_start, HPAGE_PMD_SIZE) <
> + (vma->vm_end & HPAGE_PMD_MASK))
> + return true;
> +
> + return false;
> +}
> +
> static inline bool transhuge_vma_suitable(struct vm_area_struct *vma,
> unsigned long addr)
> {
> @@ -345,6 +357,11 @@ static inline bool transparent_hugepage_active(struct vm_area_struct *vma)
> return false;
> }
>
> +static inline bool transhuge_vma_size_ok(struct vm_area_struct *vma)
> +{
> + return false;
> +}
> +
> static inline bool transhuge_vma_suitable(struct vm_area_struct *vma,
> unsigned long addr)
> {
> diff --git a/mm/huge_memory.c b/mm/huge_memory.c
> index 80e8b58b4f39..d633f97452c1 100644
> --- a/mm/huge_memory.c
> +++ b/mm/huge_memory.c
> @@ -71,10 +71,7 @@ unsigned long huge_zero_pfn __read_mostly = ~0UL;
>
> bool transparent_hugepage_active(struct vm_area_struct *vma)
> {
> - /* The addr is used to check if the vma size fits */
> - unsigned long addr = (vma->vm_end & HPAGE_PMD_MASK) - HPAGE_PMD_SIZE;
> -
> - if (!transhuge_vma_suitable(vma, addr))
> + if (!transhuge_vma_size_ok(vma))
> return false;
Given the comment added in 4/7, this patch adds change in semantic.
If that is intened, add some words in log message for it.
> if (vma_is_anonymous(vma))
> return __transparent_hugepage_enabled(vma);
^ permalink raw reply [flat|nested] 10+ messages in thread
* Re: [PATCH 2/7] mm: thp: introduce transhuge_vma_size_ok() helper
2022-05-21 1:39 ` Hillf Danton
@ 2022-05-23 18:12 ` Yang Shi
0 siblings, 0 replies; 10+ messages in thread
From: Yang Shi @ 2022-05-23 18:12 UTC (permalink / raw)
To: Hillf Danton
Cc: Vlastimil Babka, Kirill A. Shutemov, Andrew Morton, Linux MM,
Linux Kernel Mailing List
On Fri, May 20, 2022 at 6:40 PM Hillf Danton <hdanton@sina.com> wrote:
>
> On Fri, 20 May 2022 14:16:00 -0700 Yang Shi wrote:
> > There are couple of places that check whether the vma size is ok for
> > THP or not, they are open coded and duplicate, introduce
> > transhuge_vma_size_ok() helper to do the job.
> >
> > Signed-off-by: Yang Shi <shy828301@gmail.com>
> > ---
> > include/linux/huge_mm.h | 17 +++++++++++++++++
> > mm/huge_memory.c | 5 +----
> > mm/khugepaged.c | 12 ++++++------
> > 3 files changed, 24 insertions(+), 10 deletions(-)
> >
> > diff --git a/include/linux/huge_mm.h b/include/linux/huge_mm.h
> > index 648cb3ce7099..a8f61db47f2a 100644
> > --- a/include/linux/huge_mm.h
> > +++ b/include/linux/huge_mm.h
> > @@ -116,6 +116,18 @@ extern struct kobj_attribute shmem_enabled_attr;
> >
> > extern unsigned long transparent_hugepage_flags;
> >
> > +/*
> > + * The vma size has to be large enough to hold an aligned HPAGE_PMD_SIZE area.
> > + */
> > +static inline bool transhuge_vma_size_ok(struct vm_area_struct *vma)
> > +{
> > + if (round_up(vma->vm_start, HPAGE_PMD_SIZE) <
> > + (vma->vm_end & HPAGE_PMD_MASK))
> > + return true;
> > +
> > + return false;
> > +}
> > +
> > static inline bool transhuge_vma_suitable(struct vm_area_struct *vma,
> > unsigned long addr)
> > {
> > @@ -345,6 +357,11 @@ static inline bool transparent_hugepage_active(struct vm_area_struct *vma)
> > return false;
> > }
> >
> > +static inline bool transhuge_vma_size_ok(struct vm_area_struct *vma)
> > +{
> > + return false;
> > +}
> > +
> > static inline bool transhuge_vma_suitable(struct vm_area_struct *vma,
> > unsigned long addr)
> > {
> > diff --git a/mm/huge_memory.c b/mm/huge_memory.c
> > index 80e8b58b4f39..d633f97452c1 100644
> > --- a/mm/huge_memory.c
> > +++ b/mm/huge_memory.c
> > @@ -71,10 +71,7 @@ unsigned long huge_zero_pfn __read_mostly = ~0UL;
> >
> > bool transparent_hugepage_active(struct vm_area_struct *vma)
> > {
> > - /* The addr is used to check if the vma size fits */
> > - unsigned long addr = (vma->vm_end & HPAGE_PMD_MASK) - HPAGE_PMD_SIZE;
> > -
> > - if (!transhuge_vma_suitable(vma, addr))
> > + if (!transhuge_vma_size_ok(vma))
> > return false;
>
> Given the comment added in 4/7, this patch adds change in semantic.
> If that is intened, add some words in log message for it.
It should not change the semantic even with 4/7. The old comment says
"The addr is used to check if the vma size fits", so the purpose of
transhuge_vma_suitable() here is used to check if the vma size is ok
or not.
>
> > if (vma_is_anonymous(vma))
> > return __transparent_hugepage_enabled(vma);
>
^ permalink raw reply [flat|nested] 10+ messages in thread
* [PATCH 3/7] mm: khugepaged: remove the redundant anon vma check
2022-05-20 21:15 [mm-unstable PATCH 0/7] Cleanup transhuge_xxx helpers Yang Shi
2022-05-20 21:15 ` [PATCH 1/7] mm: khugepaged: check THP flag in hugepage_vma_check() Yang Shi
2022-05-20 21:16 ` [PATCH 2/7] mm: thp: introduce transhuge_vma_size_ok() helper Yang Shi
@ 2022-05-20 21:16 ` Yang Shi
2022-05-20 21:16 ` [PATCH 4/7] mm: khugepaged: use transhuge_vma_suitable replace open-code Yang Shi
` (3 subsequent siblings)
6 siblings, 0 replies; 10+ messages in thread
From: Yang Shi @ 2022-05-20 21:16 UTC (permalink / raw)
To: vbabka, kirill.shutemov, akpm; +Cc: shy828301, linux-mm, linux-kernel
The hugepage_vma_check() already checked it, so remove the redundant
check.
Signed-off-by: Yang Shi <shy828301@gmail.com>
---
mm/khugepaged.c | 3 ---
1 file changed, 3 deletions(-)
diff --git a/mm/khugepaged.c b/mm/khugepaged.c
index 5bdb30a76f05..4fe1dd3d8ef4 100644
--- a/mm/khugepaged.c
+++ b/mm/khugepaged.c
@@ -966,9 +966,6 @@ static int hugepage_vma_revalidate(struct mm_struct *mm, unsigned long address,
return SCAN_ADDRESS_RANGE;
if (!hugepage_vma_check(vma, vma->vm_flags))
return SCAN_VMA_CHECK;
- /* Anon VMA expected */
- if (!vma->anon_vma || !vma_is_anonymous(vma))
- return SCAN_VMA_CHECK;
return 0;
}
--
2.26.3
^ permalink raw reply related [flat|nested] 10+ messages in thread
* [PATCH 4/7] mm: khugepaged: use transhuge_vma_suitable replace open-code
2022-05-20 21:15 [mm-unstable PATCH 0/7] Cleanup transhuge_xxx helpers Yang Shi
` (2 preceding siblings ...)
2022-05-20 21:16 ` [PATCH 3/7] mm: khugepaged: remove the redundant anon vma check Yang Shi
@ 2022-05-20 21:16 ` Yang Shi
2022-05-20 21:16 ` [PATCH 5/7] mm: thp: kill transparent_hugepage_active() Yang Shi
` (2 subsequent siblings)
6 siblings, 0 replies; 10+ messages in thread
From: Yang Shi @ 2022-05-20 21:16 UTC (permalink / raw)
To: vbabka, kirill.shutemov, akpm; +Cc: shy828301, linux-mm, linux-kernel
The hugepage_vma_revalidate() needs to check if the address is still in
the aligned HPAGE_PMD_SIZE area of the vma when reacquiring mmap_lock,
but it was open-coded, use transhuge_vma_suitable() to do the job. And
add proper comments for transhuge_vma_suitable().
Signed-off-by: Yang Shi <shy828301@gmail.com>
---
include/linux/huge_mm.h | 6 ++++++
mm/khugepaged.c | 5 +----
2 files changed, 7 insertions(+), 4 deletions(-)
diff --git a/include/linux/huge_mm.h b/include/linux/huge_mm.h
index a8f61db47f2a..79d5919beb83 100644
--- a/include/linux/huge_mm.h
+++ b/include/linux/huge_mm.h
@@ -128,6 +128,12 @@ static inline bool transhuge_vma_size_ok(struct vm_area_struct *vma)
return false;
}
+/*
+ * Do the below checks:
+ * - For non-anon vma, check if the vm_pgoff is HPAGE_PMD_NR aligned.
+ * - For all vmas, check if the haddr is in an aligned HPAGE_PMD_SIZE
+ * area.
+ */
static inline bool transhuge_vma_suitable(struct vm_area_struct *vma,
unsigned long addr)
{
diff --git a/mm/khugepaged.c b/mm/khugepaged.c
index 4fe1dd3d8ef4..51f0e6ea3977 100644
--- a/mm/khugepaged.c
+++ b/mm/khugepaged.c
@@ -951,7 +951,6 @@ static int hugepage_vma_revalidate(struct mm_struct *mm, unsigned long address,
struct vm_area_struct **vmap)
{
struct vm_area_struct *vma;
- unsigned long hstart, hend;
if (unlikely(khugepaged_test_exit(mm)))
return SCAN_ANY_PROCESS;
@@ -960,9 +959,7 @@ static int hugepage_vma_revalidate(struct mm_struct *mm, unsigned long address,
if (!vma)
return SCAN_VMA_NULL;
- hstart = (vma->vm_start + ~HPAGE_PMD_MASK) & HPAGE_PMD_MASK;
- hend = vma->vm_end & HPAGE_PMD_MASK;
- if (address < hstart || address + HPAGE_PMD_SIZE > hend)
+ if (!transhuge_vma_suitable(vma, address))
return SCAN_ADDRESS_RANGE;
if (!hugepage_vma_check(vma, vma->vm_flags))
return SCAN_VMA_CHECK;
--
2.26.3
^ permalink raw reply related [flat|nested] 10+ messages in thread
* [PATCH 5/7] mm: thp: kill transparent_hugepage_active()
2022-05-20 21:15 [mm-unstable PATCH 0/7] Cleanup transhuge_xxx helpers Yang Shi
` (3 preceding siblings ...)
2022-05-20 21:16 ` [PATCH 4/7] mm: khugepaged: use transhuge_vma_suitable replace open-code Yang Shi
@ 2022-05-20 21:16 ` Yang Shi
2022-05-20 21:16 ` [PATCH 6/7] mm: thp: kill __transhuge_page_enabled() Yang Shi
2022-05-20 21:16 ` [PATCH 7/7] mm: khugepaged: reorg some khugepaged helpers Yang Shi
6 siblings, 0 replies; 10+ messages in thread
From: Yang Shi @ 2022-05-20 21:16 UTC (permalink / raw)
To: vbabka, kirill.shutemov, akpm; +Cc: shy828301, linux-mm, linux-kernel
The transparent_hugepage_active() was introduced to show THP eligibility
bit in smaps in proc, smaps is the only user. But it actually does the
similar check as hugepage_vma_check() which is used by khugepaged. We
definitely don't have to maintain two similar checks, so kill
transparent_hugepage_active().
Also move hugepage_vma_check() to huge_memory.c and huge_mm.h since it
is not only for khugepaged anymore.
Signed-off-by: Yang Shi <shy828301@gmail.com>
---
fs/proc/task_mmu.c | 2 +-
include/linux/huge_mm.h | 14 +++++++------
include/linux/khugepaged.h | 2 --
mm/huge_memory.c | 41 ++++++++++++++++++++++++++++++------
mm/khugepaged.c | 43 --------------------------------------
5 files changed, 44 insertions(+), 58 deletions(-)
diff --git a/fs/proc/task_mmu.c b/fs/proc/task_mmu.c
index f9c9abb50bb7..ab4849d07a1f 100644
--- a/fs/proc/task_mmu.c
+++ b/fs/proc/task_mmu.c
@@ -863,7 +863,7 @@ static int show_smap(struct seq_file *m, void *v)
__show_smap(m, &mss, false);
seq_printf(m, "THPeligible: %d\n",
- transparent_hugepage_active(vma));
+ hugepage_vma_check(vma, vma->vm_flags));
if (arch_pkeys_enabled())
seq_printf(m, "ProtectionKey: %8u\n", vma_pkey(vma));
diff --git a/include/linux/huge_mm.h b/include/linux/huge_mm.h
index 79d5919beb83..2da52ca8fd52 100644
--- a/include/linux/huge_mm.h
+++ b/include/linux/huge_mm.h
@@ -209,7 +209,8 @@ static inline bool file_thp_enabled(struct vm_area_struct *vma)
!inode_is_open_for_write(inode) && S_ISREG(inode->i_mode);
}
-bool transparent_hugepage_active(struct vm_area_struct *vma);
+extern bool hugepage_vma_check(struct vm_area_struct *vma,
+ unsigned long vm_flags);
#define transparent_hugepage_use_zero_page() \
(transparent_hugepage_flags & \
@@ -358,11 +359,6 @@ static inline bool __transparent_hugepage_enabled(struct vm_area_struct *vma)
return false;
}
-static inline bool transparent_hugepage_active(struct vm_area_struct *vma)
-{
- return false;
-}
-
static inline bool transhuge_vma_size_ok(struct vm_area_struct *vma)
{
return false;
@@ -380,6 +376,12 @@ static inline bool transhuge_vma_enabled(struct vm_area_struct *vma,
return false;
}
+static inline bool hugepage_vma_check(struct vm_area_struct *vma,
+ unsigned long vm_flags)
+{
+ return false;
+}
+
static inline void prep_transhuge_page(struct page *page) {}
#define transparent_hugepage_flags 0UL
diff --git a/include/linux/khugepaged.h b/include/linux/khugepaged.h
index 392d34c3c59a..795c8d2170b0 100644
--- a/include/linux/khugepaged.h
+++ b/include/linux/khugepaged.h
@@ -10,8 +10,6 @@ extern struct attribute_group khugepaged_attr_group;
extern int khugepaged_init(void);
extern void khugepaged_destroy(void);
extern int start_stop_khugepaged(void);
-extern bool hugepage_vma_check(struct vm_area_struct *vma,
- unsigned long vm_flags);
extern void __khugepaged_enter(struct mm_struct *mm);
extern void __khugepaged_exit(struct mm_struct *mm);
extern void khugepaged_enter_vma(struct vm_area_struct *vma,
diff --git a/mm/huge_memory.c b/mm/huge_memory.c
index d633f97452c1..d9ec82892b05 100644
--- a/mm/huge_memory.c
+++ b/mm/huge_memory.c
@@ -69,18 +69,47 @@ static atomic_t huge_zero_refcount;
struct page *huge_zero_page __read_mostly;
unsigned long huge_zero_pfn __read_mostly = ~0UL;
-bool transparent_hugepage_active(struct vm_area_struct *vma)
+bool hugepage_vma_check(struct vm_area_struct *vma,
+ unsigned long vm_flags)
{
+ if (!transhuge_vma_enabled(vma, vm_flags))
+ return false;
+
+ if (vm_flags & VM_NO_KHUGEPAGED)
+ return false;
+
+ /* Don't run khugepaged against DAX vma */
+ if (vma_is_dax(vma))
+ return false;
+
+ if (vma->vm_file && !IS_ALIGNED((vma->vm_start >> PAGE_SHIFT) -
+ vma->vm_pgoff, HPAGE_PMD_NR))
+ return false;
+
if (!transhuge_vma_size_ok(vma))
return false;
- if (vma_is_anonymous(vma))
- return __transparent_hugepage_enabled(vma);
- if (vma_is_shmem(vma))
+
+ /* Enabled via shmem mount options or sysfs settings. */
+ if (shmem_file(vma->vm_file))
return shmem_huge_enabled(vma);
- if (transhuge_vma_enabled(vma, vma->vm_flags) && file_thp_enabled(vma))
+
+ if (!khugepaged_enabled())
+ return false;
+
+ /* THP settings require madvise. */
+ if (!(vm_flags & VM_HUGEPAGE) && !khugepaged_always())
+ return false;
+
+ /* Only regular file is valid */
+ if (file_thp_enabled(vma))
return true;
- return false;
+ if (!vma->anon_vma || !vma_is_anonymous(vma))
+ return false;
+ if (vma_is_temporary_stack(vma))
+ return false;
+
+ return true;
}
static bool get_huge_zero_page(void)
diff --git a/mm/khugepaged.c b/mm/khugepaged.c
index 51f0e6ea3977..a7ac1ae418f8 100644
--- a/mm/khugepaged.c
+++ b/mm/khugepaged.c
@@ -437,49 +437,6 @@ static inline int khugepaged_test_exit(struct mm_struct *mm)
return atomic_read(&mm->mm_users) == 0;
}
-bool hugepage_vma_check(struct vm_area_struct *vma,
- unsigned long vm_flags)
-{
- if (!transhuge_vma_enabled(vma, vm_flags))
- return false;
-
- if (vm_flags & VM_NO_KHUGEPAGED)
- return false;
-
- /* Don't run khugepaged against DAX vma */
- if (vma_is_dax(vma))
- return false;
-
- if (vma->vm_file && !IS_ALIGNED((vma->vm_start >> PAGE_SHIFT) -
- vma->vm_pgoff, HPAGE_PMD_NR))
- return false;
-
- if (!transhuge_vma_size_ok(vma))
- return false;
-
- /* Enabled via shmem mount options or sysfs settings. */
- if (shmem_file(vma->vm_file))
- return shmem_huge_enabled(vma);
-
- if (!khugepaged_enabled())
- return false;
-
- /* THP settings require madvise. */
- if (!(vm_flags & VM_HUGEPAGE) && !khugepaged_always())
- return false;
-
- /* Only regular file is valid */
- if (file_thp_enabled(vma))
- return true;
-
- if (!vma->anon_vma || !vma_is_anonymous(vma))
- return false;
- if (vma_is_temporary_stack(vma))
- return false;
-
- return true;
-}
-
void __khugepaged_enter(struct mm_struct *mm)
{
struct mm_slot *mm_slot;
--
2.26.3
^ permalink raw reply related [flat|nested] 10+ messages in thread
* [PATCH 6/7] mm: thp: kill __transhuge_page_enabled()
2022-05-20 21:15 [mm-unstable PATCH 0/7] Cleanup transhuge_xxx helpers Yang Shi
` (4 preceding siblings ...)
2022-05-20 21:16 ` [PATCH 5/7] mm: thp: kill transparent_hugepage_active() Yang Shi
@ 2022-05-20 21:16 ` Yang Shi
2022-05-20 21:16 ` [PATCH 7/7] mm: khugepaged: reorg some khugepaged helpers Yang Shi
6 siblings, 0 replies; 10+ messages in thread
From: Yang Shi @ 2022-05-20 21:16 UTC (permalink / raw)
To: vbabka, kirill.shutemov, akpm; +Cc: shy828301, linux-mm, linux-kernel
The page fault path checks THP eligibility with
__transhuge_page_enabled() which does the similar thing as
hugepage_vma_check(), so use hugepage_vma_check() instead.
However page fault allows DAX and !anon_vma cases, so added a new flag,
in_pf, to hugepage_vma_check() to make page fault work correctly.
The in_pf flag is also used to skip shmem and file THP for page fault
since shmem handles THP in its own shmem_fault() and file THP allocation
on fault is not supported yet.
Also remove hugepage_vma_enabled() since hugepage_vma_check() is the
only caller now, it is not necessary to have a helper function.
Signed-off-by: Yang Shi <shy828301@gmail.com>
---
fs/proc/task_mmu.c | 2 +-
include/linux/huge_mm.h | 57 ++------------------------------------
include/linux/khugepaged.h | 2 +-
mm/huge_memory.c | 24 +++++++++++-----
mm/khugepaged.c | 8 +++---
mm/memory.c | 6 ++--
6 files changed, 29 insertions(+), 70 deletions(-)
diff --git a/fs/proc/task_mmu.c b/fs/proc/task_mmu.c
index ab4849d07a1f..28f472a2eb51 100644
--- a/fs/proc/task_mmu.c
+++ b/fs/proc/task_mmu.c
@@ -863,7 +863,7 @@ static int show_smap(struct seq_file *m, void *v)
__show_smap(m, &mss, false);
seq_printf(m, "THPeligible: %d\n",
- hugepage_vma_check(vma, vma->vm_flags));
+ hugepage_vma_check(vma, vma->vm_flags, false));
if (arch_pkeys_enabled())
seq_printf(m, "ProtectionKey: %8u\n", vma_pkey(vma));
diff --git a/include/linux/huge_mm.h b/include/linux/huge_mm.h
index 2da52ca8fd52..67c913e66e60 100644
--- a/include/linux/huge_mm.h
+++ b/include/linux/huge_mm.h
@@ -153,48 +153,6 @@ static inline bool transhuge_vma_suitable(struct vm_area_struct *vma,
return true;
}
-static inline bool transhuge_vma_enabled(struct vm_area_struct *vma,
- unsigned long vm_flags)
-{
- /* Explicitly disabled through madvise. */
- if ((vm_flags & VM_NOHUGEPAGE) ||
- test_bit(MMF_DISABLE_THP, &vma->vm_mm->flags))
- return false;
- return true;
-}
-
-/*
- * to be used on vmas which are known to support THP.
- * Use transparent_hugepage_active otherwise
- */
-static inline bool __transparent_hugepage_enabled(struct vm_area_struct *vma)
-{
-
- /*
- * If the hardware/firmware marked hugepage support disabled.
- */
- if (transparent_hugepage_flags & (1 << TRANSPARENT_HUGEPAGE_NEVER_DAX))
- return false;
-
- if (!transhuge_vma_enabled(vma, vma->vm_flags))
- return false;
-
- if (vma_is_temporary_stack(vma))
- return false;
-
- if (transparent_hugepage_flags & (1 << TRANSPARENT_HUGEPAGE_FLAG))
- return true;
-
- if (vma_is_dax(vma))
- return true;
-
- if (transparent_hugepage_flags &
- (1 << TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG))
- return !!(vma->vm_flags & VM_HUGEPAGE);
-
- return false;
-}
-
static inline bool file_thp_enabled(struct vm_area_struct *vma)
{
struct inode *inode;
@@ -210,7 +168,7 @@ static inline bool file_thp_enabled(struct vm_area_struct *vma)
}
extern bool hugepage_vma_check(struct vm_area_struct *vma,
- unsigned long vm_flags);
+ unsigned long vm_flags, bool in_pf);
#define transparent_hugepage_use_zero_page() \
(transparent_hugepage_flags & \
@@ -354,11 +312,6 @@ static inline bool folio_test_pmd_mappable(struct folio *folio)
return false;
}
-static inline bool __transparent_hugepage_enabled(struct vm_area_struct *vma)
-{
- return false;
-}
-
static inline bool transhuge_vma_size_ok(struct vm_area_struct *vma)
{
return false;
@@ -370,14 +323,8 @@ static inline bool transhuge_vma_suitable(struct vm_area_struct *vma,
return false;
}
-static inline bool transhuge_vma_enabled(struct vm_area_struct *vma,
- unsigned long vm_flags)
-{
- return false;
-}
-
static inline bool hugepage_vma_check(struct vm_area_struct *vma,
- unsigned long vm_flags)
+ unsigned long vm_flags, bool in_pf)
{
return false;
}
diff --git a/include/linux/khugepaged.h b/include/linux/khugepaged.h
index 795c8d2170b0..8a6452e089ca 100644
--- a/include/linux/khugepaged.h
+++ b/include/linux/khugepaged.h
@@ -55,7 +55,7 @@ static inline void khugepaged_enter(struct vm_area_struct *vma,
{
if (!test_bit(MMF_VM_HUGEPAGE, &vma->vm_mm->flags) &&
khugepaged_enabled()) {
- if (hugepage_vma_check(vma, vm_flags))
+ if (hugepage_vma_check(vma, vm_flags, false))
__khugepaged_enter(vma->vm_mm);
}
}
diff --git a/mm/huge_memory.c b/mm/huge_memory.c
index d9ec82892b05..b4cab3aa2534 100644
--- a/mm/huge_memory.c
+++ b/mm/huge_memory.c
@@ -70,17 +70,26 @@ struct page *huge_zero_page __read_mostly;
unsigned long huge_zero_pfn __read_mostly = ~0UL;
bool hugepage_vma_check(struct vm_area_struct *vma,
- unsigned long vm_flags)
+ unsigned long vm_flags, bool in_pf)
{
- if (!transhuge_vma_enabled(vma, vm_flags))
+ /* Explicitly disabled through madvise or prctl. */
+ if ((vm_flags & VM_NOHUGEPAGE) ||
+ test_bit(MMF_DISABLE_THP, &vma->vm_mm->flags))
return false;
+ /*
+ * If the hardware/firmware marked hugepage support disabled.
+ */
+ if (transparent_hugepage_flags & (1 << TRANSPARENT_HUGEPAGE_NEVER_DAX))
+ return false;
+
+ /* Special VMA and hugetlb VMA */
if (vm_flags & VM_NO_KHUGEPAGED)
return false;
- /* Don't run khugepaged against DAX vma */
+ /* khugepaged doesn't collapse DAX vma, but page fault is fine. */
if (vma_is_dax(vma))
- return false;
+ return in_pf;
if (vma->vm_file && !IS_ALIGNED((vma->vm_start >> PAGE_SHIFT) -
vma->vm_pgoff, HPAGE_PMD_NR))
@@ -90,7 +99,7 @@ bool hugepage_vma_check(struct vm_area_struct *vma,
return false;
/* Enabled via shmem mount options or sysfs settings. */
- if (shmem_file(vma->vm_file))
+ if (!in_pf && shmem_file(vma->vm_file))
return shmem_huge_enabled(vma);
if (!khugepaged_enabled())
@@ -101,10 +110,11 @@ bool hugepage_vma_check(struct vm_area_struct *vma,
return false;
/* Only regular file is valid */
- if (file_thp_enabled(vma))
+ if (!in_pf && file_thp_enabled(vma))
return true;
- if (!vma->anon_vma || !vma_is_anonymous(vma))
+ /* anon_vma may be not prepared before the first page fault */
+ if ((!in_pf && !vma->anon_vma) || !vma_is_anonymous(vma))
return false;
if (vma_is_temporary_stack(vma))
return false;
diff --git a/mm/khugepaged.c b/mm/khugepaged.c
index a7ac1ae418f8..292ed9c0db1e 100644
--- a/mm/khugepaged.c
+++ b/mm/khugepaged.c
@@ -473,7 +473,7 @@ void khugepaged_enter_vma(struct vm_area_struct *vma,
{
if (!test_bit(MMF_VM_HUGEPAGE, &vma->vm_mm->flags) &&
khugepaged_enabled()) {
- if (hugepage_vma_check(vma, vm_flags))
+ if (hugepage_vma_check(vma, vm_flags, false))
__khugepaged_enter(vma->vm_mm);
}
}
@@ -918,7 +918,7 @@ static int hugepage_vma_revalidate(struct mm_struct *mm, unsigned long address,
if (!transhuge_vma_suitable(vma, address))
return SCAN_ADDRESS_RANGE;
- if (!hugepage_vma_check(vma, vma->vm_flags))
+ if (!hugepage_vma_check(vma, vma->vm_flags, false))
return SCAN_VMA_CHECK;
return 0;
}
@@ -1399,7 +1399,7 @@ void collapse_pte_mapped_thp(struct mm_struct *mm, unsigned long addr)
* the valid THP. Add extra VM_HUGEPAGE so hugepage_vma_check()
* will not fail the vma for missing VM_HUGEPAGE
*/
- if (!hugepage_vma_check(vma, vma->vm_flags | VM_HUGEPAGE))
+ if (!hugepage_vma_check(vma, vma->vm_flags | VM_HUGEPAGE, false))
return;
/* Keep pmd pgtable for uffd-wp; see comment in retract_page_tables() */
@@ -2091,7 +2091,7 @@ static unsigned int khugepaged_scan_mm_slot(unsigned int pages,
progress++;
break;
}
- if (!hugepage_vma_check(vma, vma->vm_flags)) {
+ if (!hugepage_vma_check(vma, vma->vm_flags, false)) {
skip:
progress++;
continue;
diff --git a/mm/memory.c b/mm/memory.c
index 95735bee3876..0850b3e38dbf 100644
--- a/mm/memory.c
+++ b/mm/memory.c
@@ -4979,7 +4979,8 @@ static vm_fault_t __handle_mm_fault(struct vm_area_struct *vma,
if (!vmf.pud)
return VM_FAULT_OOM;
retry_pud:
- if (pud_none(*vmf.pud) && __transparent_hugepage_enabled(vma)) {
+ if (pud_none(*vmf.pud) &&
+ hugepage_vma_check(vma, vma->vm_flags, true)) {
ret = create_huge_pud(&vmf);
if (!(ret & VM_FAULT_FALLBACK))
return ret;
@@ -5012,7 +5013,8 @@ static vm_fault_t __handle_mm_fault(struct vm_area_struct *vma,
if (pud_trans_unstable(vmf.pud))
goto retry_pud;
- if (pmd_none(*vmf.pmd) && __transparent_hugepage_enabled(vma)) {
+ if (pmd_none(*vmf.pmd) &&
+ hugepage_vma_check(vma, vma->vm_flags, true)) {
ret = create_huge_pmd(&vmf);
if (!(ret & VM_FAULT_FALLBACK))
return ret;
--
2.26.3
^ permalink raw reply related [flat|nested] 10+ messages in thread
* [PATCH 7/7] mm: khugepaged: reorg some khugepaged helpers
2022-05-20 21:15 [mm-unstable PATCH 0/7] Cleanup transhuge_xxx helpers Yang Shi
` (5 preceding siblings ...)
2022-05-20 21:16 ` [PATCH 6/7] mm: thp: kill __transhuge_page_enabled() Yang Shi
@ 2022-05-20 21:16 ` Yang Shi
6 siblings, 0 replies; 10+ messages in thread
From: Yang Shi @ 2022-05-20 21:16 UTC (permalink / raw)
To: vbabka, kirill.shutemov, akpm; +Cc: shy828301, linux-mm, linux-kernel
The khugepaged_{enabled|always|req_madv} are not khugepaged only
anymore, move them to huge_mm.h and rename to hugepage_flags_xxx, and
remove khugepaged_req_madv due to no users.
Also move khugepaged_defrag to khugepaged.c since its only caller is in
that file, it doesn't have to be in a header file.
Signed-off-by: Yang Shi <shy828301@gmail.com>
---
include/linux/huge_mm.h | 8 ++++++++
include/linux/khugepaged.h | 17 +----------------
mm/huge_memory.c | 4 ++--
mm/khugepaged.c | 18 +++++++++++-------
4 files changed, 22 insertions(+), 25 deletions(-)
diff --git a/include/linux/huge_mm.h b/include/linux/huge_mm.h
index 67c913e66e60..e6f89d8ad5f8 100644
--- a/include/linux/huge_mm.h
+++ b/include/linux/huge_mm.h
@@ -116,6 +116,14 @@ extern struct kobj_attribute shmem_enabled_attr;
extern unsigned long transparent_hugepage_flags;
+#define hugepage_flags_enabled() \
+ (transparent_hugepage_flags & \
+ ((1<<TRANSPARENT_HUGEPAGE_FLAG) | \
+ (1<<TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG)))
+#define hugepage_flags_always() \
+ (transparent_hugepage_flags & \
+ (1<<TRANSPARENT_HUGEPAGE_FLAG))
+
/*
* The vma size has to be large enough to hold an aligned HPAGE_PMD_SIZE area.
*/
diff --git a/include/linux/khugepaged.h b/include/linux/khugepaged.h
index 8a6452e089ca..ca54e5c42ed0 100644
--- a/include/linux/khugepaged.h
+++ b/include/linux/khugepaged.h
@@ -24,20 +24,6 @@ static inline void collapse_pte_mapped_thp(struct mm_struct *mm,
}
#endif
-#define khugepaged_enabled() \
- (transparent_hugepage_flags & \
- ((1<<TRANSPARENT_HUGEPAGE_FLAG) | \
- (1<<TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG)))
-#define khugepaged_always() \
- (transparent_hugepage_flags & \
- (1<<TRANSPARENT_HUGEPAGE_FLAG))
-#define khugepaged_req_madv() \
- (transparent_hugepage_flags & \
- (1<<TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG))
-#define khugepaged_defrag() \
- (transparent_hugepage_flags & \
- (1<<TRANSPARENT_HUGEPAGE_DEFRAG_KHUGEPAGED_FLAG))
-
static inline void khugepaged_fork(struct mm_struct *mm, struct mm_struct *oldmm)
{
if (test_bit(MMF_VM_HUGEPAGE, &oldmm->flags))
@@ -53,8 +39,7 @@ static inline void khugepaged_exit(struct mm_struct *mm)
static inline void khugepaged_enter(struct vm_area_struct *vma,
unsigned long vm_flags)
{
- if (!test_bit(MMF_VM_HUGEPAGE, &vma->vm_mm->flags) &&
- khugepaged_enabled()) {
+ if (!test_bit(MMF_VM_HUGEPAGE, &vma->vm_mm->flags)) {
if (hugepage_vma_check(vma, vm_flags, false))
__khugepaged_enter(vma->vm_mm);
}
diff --git a/mm/huge_memory.c b/mm/huge_memory.c
index b4cab3aa2534..f4d7b6378b83 100644
--- a/mm/huge_memory.c
+++ b/mm/huge_memory.c
@@ -102,11 +102,11 @@ bool hugepage_vma_check(struct vm_area_struct *vma,
if (!in_pf && shmem_file(vma->vm_file))
return shmem_huge_enabled(vma);
- if (!khugepaged_enabled())
+ if (!hugepage_flags_enabled())
return false;
/* THP settings require madvise. */
- if (!(vm_flags & VM_HUGEPAGE) && !khugepaged_always())
+ if (!(vm_flags & VM_HUGEPAGE) && !hugepage_flags_always())
return false;
/* Only regular file is valid */
diff --git a/mm/khugepaged.c b/mm/khugepaged.c
index 292ed9c0db1e..37ea5a5396cd 100644
--- a/mm/khugepaged.c
+++ b/mm/khugepaged.c
@@ -472,7 +472,7 @@ void khugepaged_enter_vma(struct vm_area_struct *vma,
unsigned long vm_flags)
{
if (!test_bit(MMF_VM_HUGEPAGE, &vma->vm_mm->flags) &&
- khugepaged_enabled()) {
+ hugepage_flags_enabled()) {
if (hugepage_vma_check(vma, vm_flags, false))
__khugepaged_enter(vma->vm_mm);
}
@@ -763,6 +763,10 @@ static bool khugepaged_scan_abort(int nid)
return false;
}
+#define khugepaged_defrag() \
+ (transparent_hugepage_flags & \
+ (1<<TRANSPARENT_HUGEPAGE_DEFRAG_KHUGEPAGED_FLAG))
+
/* Defrag for khugepaged will enter direct reclaim/compaction if necessary */
static inline gfp_t alloc_hugepage_khugepaged_gfpmask(void)
{
@@ -860,7 +864,7 @@ static struct page *khugepaged_alloc_hugepage(bool *wait)
khugepaged_alloc_sleep();
} else
count_vm_event(THP_COLLAPSE_ALLOC);
- } while (unlikely(!hpage) && likely(khugepaged_enabled()));
+ } while (unlikely(!hpage) && likely(hugepage_flags_enabled()));
return hpage;
}
@@ -2175,7 +2179,7 @@ static unsigned int khugepaged_scan_mm_slot(unsigned int pages,
static int khugepaged_has_work(void)
{
return !list_empty(&khugepaged_scan.mm_head) &&
- khugepaged_enabled();
+ hugepage_flags_enabled();
}
static int khugepaged_wait_event(void)
@@ -2240,7 +2244,7 @@ static void khugepaged_wait_work(void)
return;
}
- if (khugepaged_enabled())
+ if (hugepage_flags_enabled())
wait_event_freezable(khugepaged_wait, khugepaged_wait_event());
}
@@ -2271,7 +2275,7 @@ static void set_recommended_min_free_kbytes(void)
int nr_zones = 0;
unsigned long recommended_min;
- if (!khugepaged_enabled()) {
+ if (!hugepage_flags_enabled()) {
calculate_min_free_kbytes();
goto update_wmarks;
}
@@ -2321,7 +2325,7 @@ int start_stop_khugepaged(void)
int err = 0;
mutex_lock(&khugepaged_mutex);
- if (khugepaged_enabled()) {
+ if (hugepage_flags_enabled()) {
if (!khugepaged_thread)
khugepaged_thread = kthread_run(khugepaged, NULL,
"khugepaged");
@@ -2347,7 +2351,7 @@ int start_stop_khugepaged(void)
void khugepaged_min_free_kbytes_update(void)
{
mutex_lock(&khugepaged_mutex);
- if (khugepaged_enabled() && khugepaged_thread)
+ if (hugepage_flags_enabled() && khugepaged_thread)
set_recommended_min_free_kbytes();
mutex_unlock(&khugepaged_mutex);
}
--
2.26.3
^ permalink raw reply related [flat|nested] 10+ messages in thread