* [PATCH] x86/vmemmap: Handle unpopulated sub-pmd ranges
@ 2021-01-27 11:23 Oscar Salvador
2021-01-27 11:29 ` David Hildenbrand
0 siblings, 1 reply; 3+ messages in thread
From: Oscar Salvador @ 2021-01-27 11:23 UTC (permalink / raw)
To: Andrew Morton
Cc: David Hildenbrand, Dave Hansen, Andy Lutomirski, Peter Zijlstra,
Thomas Gleixner, Ingo Molnar, Borislav Petkov, x86,
H . Peter Anvin, Michal Hocko, linux-mm, linux-kernel,
Oscar Salvador
When the size of a struct page is not multiple of 2MB, sections do
not span a PMD anymore and so when populating them some parts of the
PMD will remain unused.
Because of this, PMDs will be left behind when depopulating sections
since remove_pmd_table() thinks that those unused parts are still in
use.
Fix this by marking the unused parts with PAGE_INUSE, so memchr_inv() will
do the right thing and will let us free the PMD when the last user of it
is gone.
This patch is based on a similar patch by David Hildenbrand:
https://lore.kernel.org/linux-mm/20200722094558.9828-9-david@redhat.com/
https://lore.kernel.org/linux-mm/20200722094558.9828-10-david@redhat.com/
Signed-off-by: Oscar Salvador <osalvador@suse.de>
---
arch/x86/mm/init_64.c | 81 +++++++++++++++++++++++++++++++++++++++----
1 file changed, 74 insertions(+), 7 deletions(-)
diff --git a/arch/x86/mm/init_64.c b/arch/x86/mm/init_64.c
index b5a3fa4033d3..9b6d45ea353b 100644
--- a/arch/x86/mm/init_64.c
+++ b/arch/x86/mm/init_64.c
@@ -873,6 +873,71 @@ int arch_add_memory(int nid, u64 start, u64 size,
#define PAGE_INUSE 0xFD
+/*
+ * The unused vmemmap range, which was not yet memset(PAGE_INUSE) ranges
+ * from unused_pmd_start to next PMD_SIZE boundary.
+ */
+static unsigned long unused_pmd_start __meminitdata;
+
+static void __meminit vmemmap_flush_unused_pmd(void)
+{
+ if (!unused_pmd_start)
+ return;
+ /*
+ * Clears (unused_pmd_start, PMD_END]
+ */
+ memset((void *)unused_pmd_start, PAGE_INUSE,
+ ALIGN(unused_pmd_start, PMD_SIZE) - unused_pmd_start);
+ unused_pmd_start = 0;
+}
+
+/* Returns true if the PMD is completely unused and thus it can be freed */
+static bool __meminit vmemmap_unuse_sub_pmd(unsigned long addr, unsigned long end)
+{
+ unsigned long start = ALIGN_DOWN(addr, PMD_SIZE);
+
+ vmemmap_flush_unused_pmd();
+ memset((void *)addr, PAGE_INUSE, end - addr);
+
+ return !memchr_inv((void *)start, PAGE_INUSE, PMD_SIZE);
+}
+
+static void __meminit vmemmap_use_sub_pmd(unsigned long start, unsigned long end)
+{
+ /*
+ * We only optimize if the new used range directly follows the
+ * previously unused range (esp., when populating consecutive sections).
+ */
+ if (unused_pmd_start == start) {
+ if (likely(IS_ALIGNED(end, PMD_SIZE)))
+ unused_pmd_start = 0;
+ else
+ unused_pmd_start = end;
+ return;
+ }
+
+ vmemmap_flush_unused_pmd();
+}
+
+static void __meminit vmemmap_use_new_sub_pmd(unsigned long start, unsigned long end)
+{
+ vmemmap_flush_unused_pmd();
+
+ /*
+ * Mark the unused parts of the new memmap range
+ */
+ if (!IS_ALIGNED(start, PMD_SIZE))
+ memset((void *)start, PAGE_INUSE,
+ start - ALIGN_DOWN(start, PMD_SIZE));
+ /*
+ * We want to avoid memset(PAGE_UNUSED) when populating the vmemmap of
+ * consecutive sections. Remember for the last added PMD the last
+ * unused range in the populated PMD.
+ */
+ if (!IS_ALIGNED(end, PMD_SIZE))
+ unused_pmd_start = end;
+}
+
static void __meminit free_pagetable(struct page *page, int order)
{
unsigned long magic;
@@ -1034,7 +1099,6 @@ remove_pmd_table(pmd_t *pmd_start, unsigned long addr, unsigned long end,
unsigned long next, pages = 0;
pte_t *pte_base;
pmd_t *pmd;
- void *page_addr;
pmd = pmd_start + pmd_index(addr);
for (; addr < end; addr = next, pmd++) {
@@ -1055,12 +1119,10 @@ remove_pmd_table(pmd_t *pmd_start, unsigned long addr, unsigned long end,
spin_unlock(&init_mm.page_table_lock);
pages++;
} else {
- /* If here, we are freeing vmemmap pages. */
- memset((void *)addr, PAGE_INUSE, next - addr);
-
- page_addr = page_address(pmd_page(*pmd));
- if (!memchr_inv(page_addr, PAGE_INUSE,
- PMD_SIZE)) {
+ /*
+ * Free the PMD if the whole range is unused.
+ */
+ if (vmemmap_unuse_sub_pmd(addr, next)) {
free_hugepage_table(pmd_page(*pmd),
altmap);
@@ -1538,11 +1600,16 @@ static int __meminit vmemmap_populate_hugepages(unsigned long start,
addr_end = addr + PMD_SIZE;
p_end = p + PMD_SIZE;
+
+ if (!IS_ALIGNED(addr, PMD_SIZE) ||
+ !IS_ALIGNED(next, PMD_SIZE))
+ vmemmap_use_new_sub_pmd(addr, next);
continue;
} else if (altmap)
return -ENOMEM; /* no fallback */
} else if (pmd_large(*pmd)) {
vmemmap_verify((pte_t *)pmd, node, addr, next);
+ vmemmap_use_sub_pmd(addr, next);
continue;
}
if (vmemmap_populate_basepages(addr, next, node, NULL))
--
2.26.2
^ permalink raw reply related [flat|nested] 3+ messages in thread
* Re: [PATCH] x86/vmemmap: Handle unpopulated sub-pmd ranges
2021-01-27 11:23 [PATCH] x86/vmemmap: Handle unpopulated sub-pmd ranges Oscar Salvador
@ 2021-01-27 11:29 ` David Hildenbrand
2021-01-27 11:34 ` Oscar Salvador
0 siblings, 1 reply; 3+ messages in thread
From: David Hildenbrand @ 2021-01-27 11:29 UTC (permalink / raw)
To: Oscar Salvador, Andrew Morton
Cc: Dave Hansen, Andy Lutomirski, Peter Zijlstra, Thomas Gleixner,
Ingo Molnar, Borislav Petkov, x86, H . Peter Anvin, Michal Hocko,
linux-mm, linux-kernel
On 27.01.21 12:23, Oscar Salvador wrote:
> When the size of a struct page is not multiple of 2MB, sections do
> not span a PMD anymore and so when populating them some parts of the
> PMD will remain unused.
> Because of this, PMDs will be left behind when depopulating sections
> since remove_pmd_table() thinks that those unused parts are still in
> use.
>
> Fix this by marking the unused parts with PAGE_INUSE, so memchr_inv() will
> do the right thing and will let us free the PMD when the last user of it
> is gone.
>
> This patch is based on a similar patch by David Hildenbrand:
>
> https://lore.kernel.org/linux-mm/20200722094558.9828-9-david@redhat.com/
> https://lore.kernel.org/linux-mm/20200722094558.9828-10-david@redhat.com/
>
> Signed-off-by: Oscar Salvador <osalvador@suse.de>
> ---
> arch/x86/mm/init_64.c | 81 +++++++++++++++++++++++++++++++++++++++----
> 1 file changed, 74 insertions(+), 7 deletions(-)
>
> diff --git a/arch/x86/mm/init_64.c b/arch/x86/mm/init_64.c
> index b5a3fa4033d3..9b6d45ea353b 100644
> --- a/arch/x86/mm/init_64.c
> +++ b/arch/x86/mm/init_64.c
> @@ -873,6 +873,71 @@ int arch_add_memory(int nid, u64 start, u64 size,
>
> #define PAGE_INUSE 0xFD
Note that I called it "PAGE_UNUSED", using "PAGE_INUSE" might make it a
little harder to digest.
--
Thanks,
David / dhildenb
^ permalink raw reply [flat|nested] 3+ messages in thread
* Re: [PATCH] x86/vmemmap: Handle unpopulated sub-pmd ranges
2021-01-27 11:29 ` David Hildenbrand
@ 2021-01-27 11:34 ` Oscar Salvador
0 siblings, 0 replies; 3+ messages in thread
From: Oscar Salvador @ 2021-01-27 11:34 UTC (permalink / raw)
To: David Hildenbrand
Cc: Andrew Morton, Dave Hansen, Andy Lutomirski, Peter Zijlstra,
Thomas Gleixner, Ingo Molnar, Borislav Petkov, x86,
H . Peter Anvin, Michal Hocko, linux-mm, linux-kernel
On Wed, Jan 27, 2021 at 12:29:27PM +0100, David Hildenbrand wrote:
> > diff --git a/arch/x86/mm/init_64.c b/arch/x86/mm/init_64.c
> > index b5a3fa4033d3..9b6d45ea353b 100644
> > --- a/arch/x86/mm/init_64.c
> > +++ b/arch/x86/mm/init_64.c
> > @@ -873,6 +873,71 @@ int arch_add_memory(int nid, u64 start, u64 size,
> > #define PAGE_INUSE 0xFD
>
> Note that I called it "PAGE_UNUSED", using "PAGE_INUSE" might make it a
> little harder to digest.
Yeah, I wanted to recycle the PAGE_INUSE macro for x86, but on a second though
we might just call it PAGE_UNUSE as it gives a better perspective.
Let us see if there are other pushbacks/reviews, and if not, I shall re-submit
with that addressed.
Thanks
--
Oscar Salvador
SUSE L3
^ permalink raw reply [flat|nested] 3+ messages in thread
end of thread, other threads:[~2021-01-27 11:39 UTC | newest]
Thread overview: 3+ messages (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2021-01-27 11:23 [PATCH] x86/vmemmap: Handle unpopulated sub-pmd ranges Oscar Salvador
2021-01-27 11:29 ` David Hildenbrand
2021-01-27 11:34 ` Oscar Salvador
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).