* [PATCH 1/5] mm/vmalloc: remove map_kernel_range
2021-01-26 4:53 [PATCH 0/5] mm/vmalloc: cleanup after hugepage series Nicholas Piggin
@ 2021-01-26 4:54 ` Nicholas Piggin
2021-01-26 6:37 ` Christoph Hellwig
2021-01-26 4:54 ` [PATCH 2/5] kernel/dma: remove unnecessary unmap_kernel_range Nicholas Piggin
` (3 subsequent siblings)
4 siblings, 1 reply; 15+ messages in thread
From: Nicholas Piggin @ 2021-01-26 4:54 UTC (permalink / raw)
To: linux-mm, Andrew Morton; +Cc: Nicholas Piggin, linux-kernel, Christoph Hellwig
This is a shim around vmap_pages_range, get rid of it.
Move the main API comment from the _noflush variant to the normal
variant, and make _noflush internal to mm/.
Signed-off-by: Nicholas Piggin <npiggin@gmail.com>
---
Documentation/core-api/cachetlb.rst | 2 +-
include/linux/vmalloc.h | 11 -----
mm/internal.h | 6 +++
mm/percpu-vm.c | 5 ++-
mm/vmalloc.c | 65 +++++++++++++----------------
5 files changed, 38 insertions(+), 51 deletions(-)
diff --git a/Documentation/core-api/cachetlb.rst b/Documentation/core-api/cachetlb.rst
index a1582cc79f0f..756f7bcf8191 100644
--- a/Documentation/core-api/cachetlb.rst
+++ b/Documentation/core-api/cachetlb.rst
@@ -213,7 +213,7 @@ Here are the routines, one by one:
there will be no entries in the cache for the kernel address
space for virtual addresses in the range 'start' to 'end-1'.
- The first of these two routines is invoked after map_kernel_range()
+ The first of these two routines is invoked after vmap_range()
has installed the page table entries. The second is invoked
before unmap_kernel_range() deletes the page table entries.
diff --git a/include/linux/vmalloc.h b/include/linux/vmalloc.h
index 93270adf5db5..913c9d4f5e03 100644
--- a/include/linux/vmalloc.h
+++ b/include/linux/vmalloc.h
@@ -217,10 +217,6 @@ static inline bool is_vm_area_hugepages(const void *addr)
int vmap_range(unsigned long addr, unsigned long end,
phys_addr_t phys_addr, pgprot_t prot,
unsigned int max_page_shift);
-extern int map_kernel_range_noflush(unsigned long start, unsigned long size,
- pgprot_t prot, struct page **pages);
-int map_kernel_range(unsigned long start, unsigned long size, pgprot_t prot,
- struct page **pages);
extern void unmap_kernel_range_noflush(unsigned long addr, unsigned long size);
extern void unmap_kernel_range(unsigned long addr, unsigned long size);
static inline void set_vm_flush_reset_perms(void *addr)
@@ -232,13 +228,6 @@ static inline void set_vm_flush_reset_perms(void *addr)
}
#else
-static inline int
-map_kernel_range_noflush(unsigned long start, unsigned long size,
- pgprot_t prot, struct page **pages)
-{
- return size >> PAGE_SHIFT;
-}
-#define map_kernel_range map_kernel_range_noflush
static inline void
unmap_kernel_range_noflush(unsigned long addr, unsigned long size)
{
diff --git a/mm/internal.h b/mm/internal.h
index 25d2b2439f19..3ce246945882 100644
--- a/mm/internal.h
+++ b/mm/internal.h
@@ -623,4 +623,10 @@ struct migration_target_control {
gfp_t gfp_mask;
};
+/*
+ * mm/vmalloc.c
+ */
+int vmap_pages_range_noflush(unsigned long addr, unsigned long end,
+ pgprot_t prot, struct page **pages, unsigned int page_shift);
+
#endif /* __MM_INTERNAL_H */
diff --git a/mm/percpu-vm.c b/mm/percpu-vm.c
index e46f7a6917f9..88a53eb68a94 100644
--- a/mm/percpu-vm.c
+++ b/mm/percpu-vm.c
@@ -8,6 +8,7 @@
* Chunks are mapped into vmalloc areas and populated page by page.
* This is the default chunk allocator.
*/
+#include "internal.h"
static struct page *pcpu_chunk_page(struct pcpu_chunk *chunk,
unsigned int cpu, int page_idx)
@@ -192,8 +193,8 @@ static void pcpu_post_unmap_tlb_flush(struct pcpu_chunk *chunk,
static int __pcpu_map_pages(unsigned long addr, struct page **pages,
int nr_pages)
{
- return map_kernel_range_noflush(addr, nr_pages << PAGE_SHIFT,
- PAGE_KERNEL, pages);
+ return vmap_pages_range_noflush(addr, addr + (nr_pages << PAGE_SHIFT),
+ PAGE_KERNEL, pages, PAGE_SHIFT);
}
/**
diff --git a/mm/vmalloc.c b/mm/vmalloc.c
index e9a28de04182..78dc3ec457ef 100644
--- a/mm/vmalloc.c
+++ b/mm/vmalloc.c
@@ -523,7 +523,16 @@ static int vmap_small_pages_range_noflush(unsigned long addr, unsigned long end,
return 0;
}
-static int vmap_pages_range_noflush(unsigned long addr, unsigned long end,
+/*
+ * vmap_pages_range_noflush is similar to vmap_pages_range, but does not
+ * flush caches.
+ *
+ * The caller is responsible for calling flush_cache_vmap() after this
+ * function returns successfully and before the addresses are accessed.
+ *
+ * This is an internal function only. Do not use outside mm/.
+ */
+int vmap_pages_range_noflush(unsigned long addr, unsigned long end,
pgprot_t prot, struct page **pages, unsigned int page_shift)
{
unsigned int i, nr = (end - addr) >> PAGE_SHIFT;
@@ -549,48 +558,26 @@ static int vmap_pages_range_noflush(unsigned long addr, unsigned long end,
return 0;
}
-static int vmap_pages_range(unsigned long addr, unsigned long end,
- pgprot_t prot, struct page **pages, unsigned int page_shift)
-{
- int err;
-
- err = vmap_pages_range_noflush(addr, end, prot, pages, page_shift);
- flush_cache_vmap(addr, end);
- return err;
-}
-
/**
- * map_kernel_range_noflush - map kernel VM area with the specified pages
+ * vmap_pages_range - map pages to a kernel virtual address
* @addr: start of the VM area to map
- * @size: size of the VM area to map
+ * @end: end of the VM area to map (non-inclusive)
* @prot: page protection flags to use
- * @pages: pages to map
- *
- * Map PFN_UP(@size) pages at @addr. The VM area @addr and @size specify should
- * have been allocated using get_vm_area() and its friends.
- *
- * NOTE:
- * This function does NOT do any cache flushing. The caller is responsible for
- * calling flush_cache_vmap() on to-be-mapped areas before calling this
- * function.
+ * @pages: pages to map (always PAGE_SIZE pages)
+ * @page_shift: maximum shift that the pages may be mapped with, @pages must
+ * be aligned and contiguous up to at least this shift.
*
* RETURNS:
* 0 on success, -errno on failure.
*/
-int map_kernel_range_noflush(unsigned long addr, unsigned long size,
- pgprot_t prot, struct page **pages)
-{
- return vmap_pages_range_noflush(addr, addr + size, prot, pages, PAGE_SHIFT);
-}
-
-int map_kernel_range(unsigned long start, unsigned long size, pgprot_t prot,
- struct page **pages)
+static int vmap_pages_range(unsigned long addr, unsigned long end,
+ pgprot_t prot, struct page **pages, unsigned int page_shift)
{
- int ret;
+ int err;
- ret = map_kernel_range_noflush(start, size, prot, pages);
- flush_cache_vmap(start, start + size);
- return ret;
+ err = vmap_pages_range_noflush(addr, end, prot, pages, page_shift);
+ flush_cache_vmap(addr, end);
+ return err;
}
int is_vmalloc_or_module_addr(const void *x)
@@ -2156,10 +2143,12 @@ void *vm_map_ram(struct page **pages, unsigned int count, int node)
kasan_unpoison_vmalloc(mem, size);
- if (map_kernel_range(addr, size, PAGE_KERNEL, pages) < 0) {
+ if (vmap_pages_range(addr, addr + size, PAGE_KERNEL,
+ pages, PAGE_SHIFT) < 0) {
vm_unmap_ram(mem, count);
return NULL;
}
+
return mem;
}
EXPORT_SYMBOL(vm_map_ram);
@@ -2703,6 +2692,7 @@ void *vmap(struct page **pages, unsigned int count,
unsigned long flags, pgprot_t prot)
{
struct vm_struct *area;
+ unsigned long addr;
unsigned long size; /* In bytes */
might_sleep();
@@ -2715,8 +2705,9 @@ void *vmap(struct page **pages, unsigned int count,
if (!area)
return NULL;
- if (map_kernel_range((unsigned long)area->addr, size, pgprot_nx(prot),
- pages) < 0) {
+ addr = (unsigned long)area->addr;
+ if (vmap_pages_range(addr, addr + size, pgprot_nx(prot),
+ pages, PAGE_SHIFT) < 0) {
vunmap(area->addr);
return NULL;
}
--
2.23.0
^ permalink raw reply related [flat|nested] 15+ messages in thread* [PATCH 2/5] kernel/dma: remove unnecessary unmap_kernel_range
2021-01-26 4:53 [PATCH 0/5] mm/vmalloc: cleanup after hugepage series Nicholas Piggin
2021-01-26 4:54 ` [PATCH 1/5] mm/vmalloc: remove map_kernel_range Nicholas Piggin
@ 2021-01-26 4:54 ` Nicholas Piggin
2021-01-26 6:38 ` Christoph Hellwig
2021-01-26 22:08 ` Konrad Rzeszutek Wilk
2021-01-26 4:54 ` [PATCH 3/5] powerpc/xive: " Nicholas Piggin
` (2 subsequent siblings)
4 siblings, 2 replies; 15+ messages in thread
From: Nicholas Piggin @ 2021-01-26 4:54 UTC (permalink / raw)
To: linux-mm, Andrew Morton
Cc: Nicholas Piggin, linux-kernel, Christoph Hellwig,
Marek Szyprowski, Robin Murphy, iommu
vunmap will remove ptes.
Cc: Christoph Hellwig <hch@lst.de>
Cc: Marek Szyprowski <m.szyprowski@samsung.com>
Cc: Robin Murphy <robin.murphy@arm.com>
Cc: iommu@lists.linux-foundation.org
Signed-off-by: Nicholas Piggin <npiggin@gmail.com>
---
kernel/dma/remap.c | 1 -
1 file changed, 1 deletion(-)
diff --git a/kernel/dma/remap.c b/kernel/dma/remap.c
index 905c3fa005f1..b4526668072e 100644
--- a/kernel/dma/remap.c
+++ b/kernel/dma/remap.c
@@ -66,6 +66,5 @@ void dma_common_free_remap(void *cpu_addr, size_t size)
return;
}
- unmap_kernel_range((unsigned long)cpu_addr, PAGE_ALIGN(size));
vunmap(cpu_addr);
}
--
2.23.0
^ permalink raw reply related [flat|nested] 15+ messages in thread* Re: [PATCH 2/5] kernel/dma: remove unnecessary unmap_kernel_range
2021-01-26 4:54 ` [PATCH 2/5] kernel/dma: remove unnecessary unmap_kernel_range Nicholas Piggin
@ 2021-01-26 6:38 ` Christoph Hellwig
2021-01-26 22:08 ` Konrad Rzeszutek Wilk
1 sibling, 0 replies; 15+ messages in thread
From: Christoph Hellwig @ 2021-01-26 6:38 UTC (permalink / raw)
To: Nicholas Piggin
Cc: linux-mm, Andrew Morton, linux-kernel, Christoph Hellwig,
Marek Szyprowski, Robin Murphy, iommu
On Tue, Jan 26, 2021 at 02:54:01PM +1000, Nicholas Piggin wrote:
> vunmap will remove ptes.
Looks good,
Reviewed-by: Christoph Hellwig <hch@lst.de>
^ permalink raw reply [flat|nested] 15+ messages in thread
* Re: [PATCH 2/5] kernel/dma: remove unnecessary unmap_kernel_range
2021-01-26 4:54 ` [PATCH 2/5] kernel/dma: remove unnecessary unmap_kernel_range Nicholas Piggin
2021-01-26 6:38 ` Christoph Hellwig
@ 2021-01-26 22:08 ` Konrad Rzeszutek Wilk
2021-01-27 7:10 ` Christoph Hellwig
1 sibling, 1 reply; 15+ messages in thread
From: Konrad Rzeszutek Wilk @ 2021-01-26 22:08 UTC (permalink / raw)
To: Nicholas Piggin
Cc: linux-mm, Andrew Morton, linux-kernel, iommu, Robin Murphy,
Christoph Hellwig
On Tue, Jan 26, 2021 at 02:54:01PM +1000, Nicholas Piggin wrote:
> vunmap will remove ptes.
Should there be some ASSERT after the vunmap to make sure that is the
case?
>
> Cc: Christoph Hellwig <hch@lst.de>
> Cc: Marek Szyprowski <m.szyprowski@samsung.com>
> Cc: Robin Murphy <robin.murphy@arm.com>
> Cc: iommu@lists.linux-foundation.org
> Signed-off-by: Nicholas Piggin <npiggin@gmail.com>
> ---
> kernel/dma/remap.c | 1 -
> 1 file changed, 1 deletion(-)
>
> diff --git a/kernel/dma/remap.c b/kernel/dma/remap.c
> index 905c3fa005f1..b4526668072e 100644
> --- a/kernel/dma/remap.c
> +++ b/kernel/dma/remap.c
> @@ -66,6 +66,5 @@ void dma_common_free_remap(void *cpu_addr, size_t size)
> return;
> }
>
> - unmap_kernel_range((unsigned long)cpu_addr, PAGE_ALIGN(size));
> vunmap(cpu_addr);
> }
> --
> 2.23.0
>
> _______________________________________________
> iommu mailing list
> iommu@lists.linux-foundation.org
> https://lists.linuxfoundation.org/mailman/listinfo/iommu
^ permalink raw reply [flat|nested] 15+ messages in thread
* Re: [PATCH 2/5] kernel/dma: remove unnecessary unmap_kernel_range
2021-01-26 22:08 ` Konrad Rzeszutek Wilk
@ 2021-01-27 7:10 ` Christoph Hellwig
2021-01-27 23:44 ` Nicholas Piggin
0 siblings, 1 reply; 15+ messages in thread
From: Christoph Hellwig @ 2021-01-27 7:10 UTC (permalink / raw)
To: Konrad Rzeszutek Wilk
Cc: Nicholas Piggin, linux-mm, Andrew Morton, linux-kernel, iommu,
Robin Murphy, Christoph Hellwig
On Tue, Jan 26, 2021 at 05:08:46PM -0500, Konrad Rzeszutek Wilk wrote:
> On Tue, Jan 26, 2021 at 02:54:01PM +1000, Nicholas Piggin wrote:
> > vunmap will remove ptes.
>
> Should there be some ASSERT after the vunmap to make sure that is the
> case?
Not really. removing the PTEs is the whole point of vunmap. Everything
else is just house keeping.
^ permalink raw reply [flat|nested] 15+ messages in thread
* Re: [PATCH 2/5] kernel/dma: remove unnecessary unmap_kernel_range
2021-01-27 7:10 ` Christoph Hellwig
@ 2021-01-27 23:44 ` Nicholas Piggin
0 siblings, 0 replies; 15+ messages in thread
From: Nicholas Piggin @ 2021-01-27 23:44 UTC (permalink / raw)
To: Christoph Hellwig, Konrad Rzeszutek Wilk
Cc: Andrew Morton, iommu, linux-kernel, linux-mm, Robin Murphy
Excerpts from Christoph Hellwig's message of January 27, 2021 5:10 pm:
> On Tue, Jan 26, 2021 at 05:08:46PM -0500, Konrad Rzeszutek Wilk wrote:
>> On Tue, Jan 26, 2021 at 02:54:01PM +1000, Nicholas Piggin wrote:
>> > vunmap will remove ptes.
>>
>> Should there be some ASSERT after the vunmap to make sure that is the
>> case?
>
> Not really. removing the PTEs is the whole point of vunmap. Everything
> else is just house keeping.
Agree. I did double check this and wrote a quick test to check ptes were
there before the vunmap and cleared after, just to make sure I didn't
make a silly mistake with the patch. But in general drivers should be
able to trust code behind the API call will do the right thing. Such
assertions should go in the vunmap() implementation as appropriate.
Thanks,
Nick
^ permalink raw reply [flat|nested] 15+ messages in thread
* [PATCH 3/5] powerpc/xive: remove unnecessary unmap_kernel_range
2021-01-26 4:53 [PATCH 0/5] mm/vmalloc: cleanup after hugepage series Nicholas Piggin
2021-01-26 4:54 ` [PATCH 1/5] mm/vmalloc: remove map_kernel_range Nicholas Piggin
2021-01-26 4:54 ` [PATCH 2/5] kernel/dma: remove unnecessary unmap_kernel_range Nicholas Piggin
@ 2021-01-26 4:54 ` Nicholas Piggin
2021-01-26 6:38 ` Christoph Hellwig
2021-01-26 7:25 ` Cédric Le Goater
2021-01-26 4:54 ` [PATCH 4/5] mm/vmalloc: remove unmap_kernel_range Nicholas Piggin
2021-01-26 4:54 ` [PATCH 5/5] mm/vmalloc: improve allocation failure error messages Nicholas Piggin
4 siblings, 2 replies; 15+ messages in thread
From: Nicholas Piggin @ 2021-01-26 4:54 UTC (permalink / raw)
To: linux-mm, Andrew Morton
Cc: Nicholas Piggin, linux-kernel, Christoph Hellwig,
Cédric Le Goater, linuxppc-dev
iounmap will remove ptes.
Cc: "Cédric Le Goater" <clg@kaod.org>
Cc: linuxppc-dev@lists.ozlabs.org
Signed-off-by: Nicholas Piggin <npiggin@gmail.com>
---
arch/powerpc/sysdev/xive/common.c | 4 ----
1 file changed, 4 deletions(-)
diff --git a/arch/powerpc/sysdev/xive/common.c b/arch/powerpc/sysdev/xive/common.c
index 595310e056f4..d6c2069cc828 100644
--- a/arch/powerpc/sysdev/xive/common.c
+++ b/arch/powerpc/sysdev/xive/common.c
@@ -959,16 +959,12 @@ EXPORT_SYMBOL_GPL(is_xive_irq);
void xive_cleanup_irq_data(struct xive_irq_data *xd)
{
if (xd->eoi_mmio) {
- unmap_kernel_range((unsigned long)xd->eoi_mmio,
- 1u << xd->esb_shift);
iounmap(xd->eoi_mmio);
if (xd->eoi_mmio == xd->trig_mmio)
xd->trig_mmio = NULL;
xd->eoi_mmio = NULL;
}
if (xd->trig_mmio) {
- unmap_kernel_range((unsigned long)xd->trig_mmio,
- 1u << xd->esb_shift);
iounmap(xd->trig_mmio);
xd->trig_mmio = NULL;
}
--
2.23.0
^ permalink raw reply related [flat|nested] 15+ messages in thread* Re: [PATCH 3/5] powerpc/xive: remove unnecessary unmap_kernel_range
2021-01-26 4:54 ` [PATCH 3/5] powerpc/xive: " Nicholas Piggin
@ 2021-01-26 6:38 ` Christoph Hellwig
2021-01-26 7:25 ` Cédric Le Goater
1 sibling, 0 replies; 15+ messages in thread
From: Christoph Hellwig @ 2021-01-26 6:38 UTC (permalink / raw)
To: Nicholas Piggin
Cc: linux-mm, Andrew Morton, linux-kernel, Christoph Hellwig,
Cédric Le Goater, linuxppc-dev
On Tue, Jan 26, 2021 at 02:54:02PM +1000, Nicholas Piggin wrote:
> iounmap will remove ptes.
Looks good,
Reviewed-by: Christoph Hellwig <hch@lst.de>
^ permalink raw reply [flat|nested] 15+ messages in thread
* Re: [PATCH 3/5] powerpc/xive: remove unnecessary unmap_kernel_range
2021-01-26 4:54 ` [PATCH 3/5] powerpc/xive: " Nicholas Piggin
2021-01-26 6:38 ` Christoph Hellwig
@ 2021-01-26 7:25 ` Cédric Le Goater
1 sibling, 0 replies; 15+ messages in thread
From: Cédric Le Goater @ 2021-01-26 7:25 UTC (permalink / raw)
To: Nicholas Piggin, linux-mm, Andrew Morton
Cc: linux-kernel, Christoph Hellwig, linuxppc-dev
On 1/26/21 5:54 AM, Nicholas Piggin wrote:
> iounmap will remove ptes.
>
> Cc: "Cédric Le Goater" <clg@kaod.org>
> Cc: linuxppc-dev@lists.ozlabs.org
> Signed-off-by: Nicholas Piggin <npiggin@gmail.com>
Looks good.
Acked-by: Cédric Le Goater <clg@kaod.org>
Thanks,
C.
> ---
> arch/powerpc/sysdev/xive/common.c | 4 ----
> 1 file changed, 4 deletions(-)
>
> diff --git a/arch/powerpc/sysdev/xive/common.c b/arch/powerpc/sysdev/xive/common.c
> index 595310e056f4..d6c2069cc828 100644
> --- a/arch/powerpc/sysdev/xive/common.c
> +++ b/arch/powerpc/sysdev/xive/common.c
> @@ -959,16 +959,12 @@ EXPORT_SYMBOL_GPL(is_xive_irq);
> void xive_cleanup_irq_data(struct xive_irq_data *xd)
> {
> if (xd->eoi_mmio) {
> - unmap_kernel_range((unsigned long)xd->eoi_mmio,
> - 1u << xd->esb_shift);
> iounmap(xd->eoi_mmio);
> if (xd->eoi_mmio == xd->trig_mmio)
> xd->trig_mmio = NULL;
> xd->eoi_mmio = NULL;
> }
> if (xd->trig_mmio) {
> - unmap_kernel_range((unsigned long)xd->trig_mmio,
> - 1u << xd->esb_shift);
> iounmap(xd->trig_mmio);
> xd->trig_mmio = NULL;
> }
>
^ permalink raw reply [flat|nested] 15+ messages in thread
* [PATCH 4/5] mm/vmalloc: remove unmap_kernel_range
2021-01-26 4:53 [PATCH 0/5] mm/vmalloc: cleanup after hugepage series Nicholas Piggin
` (2 preceding siblings ...)
2021-01-26 4:54 ` [PATCH 3/5] powerpc/xive: " Nicholas Piggin
@ 2021-01-26 4:54 ` Nicholas Piggin
2021-01-26 6:38 ` Christoph Hellwig
2021-01-26 4:54 ` [PATCH 5/5] mm/vmalloc: improve allocation failure error messages Nicholas Piggin
4 siblings, 1 reply; 15+ messages in thread
From: Nicholas Piggin @ 2021-01-26 4:54 UTC (permalink / raw)
To: linux-mm, Andrew Morton; +Cc: Nicholas Piggin, linux-kernel, Christoph Hellwig
This is a shim around vunmap_range, get rid of it.
Move the main API comment from the _noflush variant to the normal
variant, and make _noflush internal to mm/.
Signed-off-by: Nicholas Piggin <npiggin@gmail.com>
---
Documentation/core-api/cachetlb.rst | 2 +-
arch/arm64/mm/init.c | 2 +-
arch/powerpc/kernel/isa-bridge.c | 4 +-
arch/powerpc/kernel/pci_64.c | 2 +-
arch/powerpc/mm/ioremap.c | 2 +-
drivers/pci/pci.c | 2 +-
include/linux/vmalloc.h | 8 +---
mm/internal.h | 1 +
mm/percpu-vm.c | 2 +-
mm/vmalloc.c | 59 ++++++++++++++---------------
10 files changed, 38 insertions(+), 46 deletions(-)
diff --git a/Documentation/core-api/cachetlb.rst b/Documentation/core-api/cachetlb.rst
index 756f7bcf8191..fe4290e26729 100644
--- a/Documentation/core-api/cachetlb.rst
+++ b/Documentation/core-api/cachetlb.rst
@@ -215,7 +215,7 @@ Here are the routines, one by one:
The first of these two routines is invoked after vmap_range()
has installed the page table entries. The second is invoked
- before unmap_kernel_range() deletes the page table entries.
+ before vunmap_range() deletes the page table entries.
There exists another whole class of cpu cache issues which currently
require a whole different set of interfaces to handle properly.
diff --git a/arch/arm64/mm/init.c b/arch/arm64/mm/init.c
index 709d98fea90c..7fe0a5074205 100644
--- a/arch/arm64/mm/init.c
+++ b/arch/arm64/mm/init.c
@@ -498,7 +498,7 @@ void free_initmem(void)
* prevents the region from being reused for kernel modules, which
* is not supported by kallsyms.
*/
- unmap_kernel_range((u64)__init_begin, (u64)(__init_end - __init_begin));
+ vunmap_range((u64)__init_begin, (u64)__init_end);
}
void dump_mem_limit(void)
diff --git a/arch/powerpc/kernel/isa-bridge.c b/arch/powerpc/kernel/isa-bridge.c
index 2257d24e6a26..39c625737c09 100644
--- a/arch/powerpc/kernel/isa-bridge.c
+++ b/arch/powerpc/kernel/isa-bridge.c
@@ -48,7 +48,7 @@ static void remap_isa_base(phys_addr_t pa, unsigned long size)
if (slab_is_available()) {
if (ioremap_page_range(ISA_IO_BASE, ISA_IO_BASE + size, pa,
pgprot_noncached(PAGE_KERNEL)))
- unmap_kernel_range(ISA_IO_BASE, size);
+ vunmap_range(ISA_IO_BASE, ISA_IO_BASE + size);
} else {
early_ioremap_range(ISA_IO_BASE, pa, size,
pgprot_noncached(PAGE_KERNEL));
@@ -311,7 +311,7 @@ static void isa_bridge_remove(void)
isa_bridge_pcidev = NULL;
/* Unmap the ISA area */
- unmap_kernel_range(ISA_IO_BASE, 0x10000);
+ vunmap_range(ISA_IO_BASE, ISA_IO_BASE + 0x10000);
}
/**
diff --git a/arch/powerpc/kernel/pci_64.c b/arch/powerpc/kernel/pci_64.c
index 9312e6eda7ff..3fb7e572abed 100644
--- a/arch/powerpc/kernel/pci_64.c
+++ b/arch/powerpc/kernel/pci_64.c
@@ -140,7 +140,7 @@ void __iomem *ioremap_phb(phys_addr_t paddr, unsigned long size)
addr = (unsigned long)area->addr;
if (ioremap_page_range(addr, addr + size, paddr,
pgprot_noncached(PAGE_KERNEL))) {
- unmap_kernel_range(addr, size);
+ vunmap_range(addr, addr + size);
return NULL;
}
diff --git a/arch/powerpc/mm/ioremap.c b/arch/powerpc/mm/ioremap.c
index b1a0aebe8c48..57342154d2b0 100644
--- a/arch/powerpc/mm/ioremap.c
+++ b/arch/powerpc/mm/ioremap.c
@@ -93,7 +93,7 @@ void __iomem *do_ioremap(phys_addr_t pa, phys_addr_t offset, unsigned long size,
if (!ret)
return (void __iomem *)area->addr + offset;
- unmap_kernel_range(va, size);
+ vunmap_range(va, va + size);
free_vm_area(area);
return NULL;
diff --git a/drivers/pci/pci.c b/drivers/pci/pci.c
index b9fecc25d213..d1e5ee09b381 100644
--- a/drivers/pci/pci.c
+++ b/drivers/pci/pci.c
@@ -4107,7 +4107,7 @@ void pci_unmap_iospace(struct resource *res)
#if defined(PCI_IOBASE) && defined(CONFIG_MMU)
unsigned long vaddr = (unsigned long)PCI_IOBASE + res->start;
- unmap_kernel_range(vaddr, resource_size(res));
+ vunmap_range(vaddr, vaddr + resource_size(res));
#endif
}
EXPORT_SYMBOL(pci_unmap_iospace);
diff --git a/include/linux/vmalloc.h b/include/linux/vmalloc.h
index 913c9d4f5e03..b569a13c9960 100644
--- a/include/linux/vmalloc.h
+++ b/include/linux/vmalloc.h
@@ -217,8 +217,7 @@ static inline bool is_vm_area_hugepages(const void *addr)
int vmap_range(unsigned long addr, unsigned long end,
phys_addr_t phys_addr, pgprot_t prot,
unsigned int max_page_shift);
-extern void unmap_kernel_range_noflush(unsigned long addr, unsigned long size);
-extern void unmap_kernel_range(unsigned long addr, unsigned long size);
+void vunmap_range(unsigned long addr, unsigned long end);
static inline void set_vm_flush_reset_perms(void *addr)
{
struct vm_struct *vm = find_vm_area(addr);
@@ -228,11 +227,6 @@ static inline void set_vm_flush_reset_perms(void *addr)
}
#else
-static inline void
-unmap_kernel_range_noflush(unsigned long addr, unsigned long size)
-{
-}
-#define unmap_kernel_range unmap_kernel_range_noflush
static inline void set_vm_flush_reset_perms(void *addr)
{
}
diff --git a/mm/internal.h b/mm/internal.h
index 3ce246945882..3d4449f79582 100644
--- a/mm/internal.h
+++ b/mm/internal.h
@@ -628,5 +628,6 @@ struct migration_target_control {
*/
int vmap_pages_range_noflush(unsigned long addr, unsigned long end,
pgprot_t prot, struct page **pages, unsigned int page_shift);
+void vunmap_range_noflush(unsigned long start, unsigned long end);
#endif /* __MM_INTERNAL_H */
diff --git a/mm/percpu-vm.c b/mm/percpu-vm.c
index 88a53eb68a94..8d3844bc0c7c 100644
--- a/mm/percpu-vm.c
+++ b/mm/percpu-vm.c
@@ -134,7 +134,7 @@ static void pcpu_pre_unmap_flush(struct pcpu_chunk *chunk,
static void __pcpu_unmap_pages(unsigned long addr, int nr_pages)
{
- unmap_kernel_range_noflush(addr, nr_pages << PAGE_SHIFT);
+ vunmap_range_noflush(addr, addr + (nr_pages << PAGE_SHIFT));
}
/**
diff --git a/mm/vmalloc.c b/mm/vmalloc.c
index 78dc3ec457ef..5ff190590fe4 100644
--- a/mm/vmalloc.c
+++ b/mm/vmalloc.c
@@ -378,22 +378,20 @@ static void vunmap_p4d_range(pgd_t *pgd, unsigned long addr, unsigned long end,
} while (p4d++, addr = next, addr != end);
}
-/**
- * unmap_kernel_range_noflush - unmap kernel VM area
- * @start: start of the VM area to unmap
- * @size: size of the VM area to unmap
+/*
+ * vunmap_range_noflush is similar to vunmap_range, but does not
+ * flush caches or TLBs.
*
- * Unmap PFN_UP(@size) pages at @addr. The VM area @addr and @size specify
- * should have been allocated using get_vm_area() and its friends.
+ * The caller is responsible for calling flush_cache_vmap() before calling
+ * this function, and flush_tlb_kernel_range after it has returned
+ * successfully (and before the addresses are expected to cause a page fault
+ * or be re-mapped for something else, if TLB flushes are being delayed or
+ * coalesced).
*
- * NOTE:
- * This function does NOT do any cache flushing. The caller is responsible
- * for calling flush_cache_vunmap() on to-be-mapped areas before calling this
- * function and flush_tlb_kernel_range() after.
+ * This is an internal function only. Do not use outside mm/.
*/
-void unmap_kernel_range_noflush(unsigned long start, unsigned long size)
+void vunmap_range_noflush(unsigned long start, unsigned long end)
{
- unsigned long end = start + size;
unsigned long next;
pgd_t *pgd;
unsigned long addr = start;
@@ -414,6 +412,22 @@ void unmap_kernel_range_noflush(unsigned long start, unsigned long size)
arch_sync_kernel_mappings(start, end);
}
+/**
+ * vunmap_range_noflush - unmap kernel virtual addresses
+ * @addr: start of the VM area to unmap
+ * @end: end of the VM area to unmap (non-inclusive)
+ *
+ * Clears any present PTEs in the virtual address range, flushes TLBs and
+ * caches. Any subsequent access to the address before it has been re-mapped
+ * is a kernel bug.
+ */
+void vunmap_range(unsigned long addr, unsigned long end)
+{
+ flush_cache_vunmap(addr, end);
+ vunmap_range_noflush(addr, end);
+ flush_tlb_kernel_range(addr, end);
+}
+
static int vmap_pages_pte_range(pmd_t *pmd, unsigned long addr,
unsigned long end, pgprot_t prot, struct page **pages, int *nr,
pgtbl_mod_mask *mask)
@@ -1712,7 +1726,7 @@ static void free_vmap_area_noflush(struct vmap_area *va)
static void free_unmap_vmap_area(struct vmap_area *va)
{
flush_cache_vunmap(va->va_start, va->va_end);
- unmap_kernel_range_noflush(va->va_start, va->va_end - va->va_start);
+ vunmap_range_noflush(va->va_start, va->va_end);
if (debug_pagealloc_enabled_static())
flush_tlb_kernel_range(va->va_start, va->va_end);
@@ -1990,7 +2004,7 @@ static void vb_free(unsigned long addr, unsigned long size)
offset = (addr & (VMAP_BLOCK_SIZE - 1)) >> PAGE_SHIFT;
vb = xa_load(&vmap_blocks, addr_to_vb_idx(addr));
- unmap_kernel_range_noflush(addr, size);
+ vunmap_range_noflush(addr, addr + size);
if (debug_pagealloc_enabled_static())
flush_tlb_kernel_range(addr, addr + size);
@@ -2307,23 +2321,6 @@ void __init vmalloc_init(void)
vmap_initialized = true;
}
-/**
- * unmap_kernel_range - unmap kernel VM area and flush cache and TLB
- * @addr: start of the VM area to unmap
- * @size: size of the VM area to unmap
- *
- * Similar to unmap_kernel_range_noflush() but flushes vcache before
- * the unmapping and tlb after.
- */
-void unmap_kernel_range(unsigned long addr, unsigned long size)
-{
- unsigned long end = addr + size;
-
- flush_cache_vunmap(addr, end);
- unmap_kernel_range_noflush(addr, size);
- flush_tlb_kernel_range(addr, end);
-}
-
static inline void setup_vmalloc_vm_locked(struct vm_struct *vm,
struct vmap_area *va, unsigned long flags, const void *caller)
{
--
2.23.0
^ permalink raw reply related [flat|nested] 15+ messages in thread* Re: [PATCH 4/5] mm/vmalloc: remove unmap_kernel_range
2021-01-26 4:54 ` [PATCH 4/5] mm/vmalloc: remove unmap_kernel_range Nicholas Piggin
@ 2021-01-26 6:38 ` Christoph Hellwig
0 siblings, 0 replies; 15+ messages in thread
From: Christoph Hellwig @ 2021-01-26 6:38 UTC (permalink / raw)
To: Nicholas Piggin; +Cc: linux-mm, Andrew Morton, linux-kernel, Christoph Hellwig
On Tue, Jan 26, 2021 at 02:54:03PM +1000, Nicholas Piggin wrote:
> This is a shim around vunmap_range, get rid of it.
>
> Move the main API comment from the _noflush variant to the normal
> variant, and make _noflush internal to mm/.
Looks good,
Reviewed-by: Christoph Hellwig <hch@lst.de>
^ permalink raw reply [flat|nested] 15+ messages in thread
* [PATCH 5/5] mm/vmalloc: improve allocation failure error messages
2021-01-26 4:53 [PATCH 0/5] mm/vmalloc: cleanup after hugepage series Nicholas Piggin
` (3 preceding siblings ...)
2021-01-26 4:54 ` [PATCH 4/5] mm/vmalloc: remove unmap_kernel_range Nicholas Piggin
@ 2021-01-26 4:54 ` Nicholas Piggin
2021-01-26 6:39 ` Christoph Hellwig
4 siblings, 1 reply; 15+ messages in thread
From: Nicholas Piggin @ 2021-01-26 4:54 UTC (permalink / raw)
To: linux-mm, Andrew Morton; +Cc: Nicholas Piggin, linux-kernel, Christoph Hellwig
There are several reasons why a vmalloc can fail, virtual space
exhausted, page array allocation failure, page allocation failure,
and kernel page table allocation failure.
Add distinct warning messages for the main causes of failure, with
some added information like page order or allocation size where
applicable.
Signed-off-by: Nicholas Piggin <npiggin@gmail.com>
---
mm/vmalloc.c | 40 ++++++++++++++++++++++++++++------------
1 file changed, 28 insertions(+), 12 deletions(-)
diff --git a/mm/vmalloc.c b/mm/vmalloc.c
index 5ff190590fe4..4facf582a3be 100644
--- a/mm/vmalloc.c
+++ b/mm/vmalloc.c
@@ -2790,6 +2790,10 @@ static void *__vmalloc_area_node(struct vm_struct *area, gfp_t gfp_mask,
if (!pages) {
free_vm_area(area);
+ warn_alloc(gfp_mask, NULL,
+ "vmalloc size %lu allocation failure: "
+ "page array size %lu allocation failed",
+ area->nr_pages * PAGE_SIZE, array_size);
return NULL;
}
@@ -2813,6 +2817,10 @@ static void *__vmalloc_area_node(struct vm_struct *area, gfp_t gfp_mask,
/* Successfully allocated i pages, free them in __vfree() */
area->nr_pages = i;
atomic_long_add(area->nr_pages, &nr_vmalloc_pages);
+ warn_alloc(gfp_mask, NULL,
+ "vmalloc size %lu allocation failure: "
+ "page order %u allocation failed",
+ area->nr_pages * PAGE_SIZE, page_order);
goto fail;
}
@@ -2824,15 +2832,17 @@ static void *__vmalloc_area_node(struct vm_struct *area, gfp_t gfp_mask,
}
atomic_long_add(area->nr_pages, &nr_vmalloc_pages);
- if (vmap_pages_range(addr, addr + size, prot, pages, page_shift) < 0)
+ if (vmap_pages_range(addr, addr + size, prot, pages, page_shift) < 0) {
+ warn_alloc(gfp_mask, NULL,
+ "vmalloc size %lu allocation failure: "
+ "failed to map pages",
+ area->nr_pages * PAGE_SIZE);
goto fail;
+ }
return area->addr;
fail:
- warn_alloc(gfp_mask, NULL,
- "vmalloc: allocation failure, allocated %ld of %ld bytes",
- (area->nr_pages*PAGE_SIZE), size);
__vfree(area->addr);
return NULL;
}
@@ -2866,8 +2876,15 @@ void *__vmalloc_node_range(unsigned long size, unsigned long align,
unsigned long real_align = align;
unsigned int shift = PAGE_SHIFT;
- if (!size || (size >> PAGE_SHIFT) > totalram_pages())
- goto fail;
+ if (WARN_ON_ONCE(!size))
+ return NULL;
+
+ if ((size >> PAGE_SHIFT) > totalram_pages()) {
+ warn_alloc(gfp_mask, NULL,
+ "vmalloc size %lu allocation failure: "
+ "exceeds total pages", real_size);
+ return NULL;
+ }
if (vmap_allow_huge && !(vm_flags & VM_NO_HUGE_VMAP) &&
arch_vmap_pmd_supported(prot)) {
@@ -2894,8 +2911,12 @@ void *__vmalloc_node_range(unsigned long size, unsigned long align,
size = PAGE_ALIGN(size);
area = __get_vm_area_node(size, align, VM_ALLOC | VM_UNINITIALIZED |
vm_flags, start, end, node, gfp_mask, caller);
- if (!area)
+ if (!area) {
+ warn_alloc(gfp_mask, NULL,
+ "vmalloc size %lu allocation failure: "
+ "vm_struct allocation failed", real_size);
goto fail;
+ }
addr = __vmalloc_area_node(area, gfp_mask, prot, shift, node);
if (!addr)
@@ -2920,11 +2941,6 @@ void *__vmalloc_node_range(unsigned long size, unsigned long align,
goto again;
}
- if (!area) {
- /* Warn for area allocation, page allocations already warn */
- warn_alloc(gfp_mask, NULL,
- "vmalloc: allocation failure: %lu bytes", real_size);
- }
return NULL;
}
--
2.23.0
^ permalink raw reply related [flat|nested] 15+ messages in thread* Re: [PATCH 5/5] mm/vmalloc: improve allocation failure error messages
2021-01-26 4:54 ` [PATCH 5/5] mm/vmalloc: improve allocation failure error messages Nicholas Piggin
@ 2021-01-26 6:39 ` Christoph Hellwig
0 siblings, 0 replies; 15+ messages in thread
From: Christoph Hellwig @ 2021-01-26 6:39 UTC (permalink / raw)
To: Nicholas Piggin; +Cc: linux-mm, Andrew Morton, linux-kernel, Christoph Hellwig
On Tue, Jan 26, 2021 at 02:54:04PM +1000, Nicholas Piggin wrote:
> There are several reasons why a vmalloc can fail, virtual space
> exhausted, page array allocation failure, page allocation failure,
> and kernel page table allocation failure.
>
> Add distinct warning messages for the main causes of failure, with
> some added information like page order or allocation size where
> applicable.
>
> Signed-off-by: Nicholas Piggin <npiggin@gmail.com>
Looks good,
Reviewed-by: Christoph Hellwig <hch@lst.de>
^ permalink raw reply [flat|nested] 15+ messages in thread