* [PATCH v3 01/12] mm: fix type width of section to/from pfn conversion macros
2017-01-19 22:06 [PATCH v3 00/12] mm: sub-section memory hotplug support Dan Williams
@ 2017-01-19 22:06 ` Dan Williams
2017-01-19 22:06 ` [PATCH v3 02/12] mm, devm_memremap_pages: use multi-order radix for ZONE_DEVICE lookups Dan Williams
` (10 subsequent siblings)
11 siblings, 0 replies; 15+ messages in thread
From: Dan Williams @ 2017-01-19 22:06 UTC (permalink / raw)
To: akpm
Cc: Michal Hocko, linux-nvdimm, Logan Gunthorpe, linux-kernel,
Stephen Bates, linux-mm, Johannes Weiner, Mel Gorman,
Vlastimil Babka
section_nr_to_pfn() will silently accept an argument that is too small
to contain a pfn. Cast the argument to an unsigned long, similar to
PFN_PHYS(). Fix up pfn_to_section_nr() in the same way.
This was discovered in __add_pages() when converting it to use an
signed integer for the loop variable.
Cc: Michal Hocko <mhocko@suse.com>
Cc: Vlastimil Babka <vbabka@suse.cz>
Cc: Johannes Weiner <hannes@cmpxchg.org>
Cc: Logan Gunthorpe <logang@deltatee.com>
Cc: Mel Gorman <mgorman@techsingularity.net>
Cc: Andrew Morton <akpm@linux-foundation.org>
Cc: Stephen Bates <stephen.bates@microsemi.com>
Signed-off-by: Dan Williams <dan.j.williams@intel.com>
---
include/linux/mmzone.h | 4 ++--
mm/memory_hotplug.c | 4 +---
2 files changed, 3 insertions(+), 5 deletions(-)
diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h
index 36d9896fbc1e..df36d21e0247 100644
--- a/include/linux/mmzone.h
+++ b/include/linux/mmzone.h
@@ -1062,8 +1062,8 @@ static inline unsigned long early_pfn_to_nid(unsigned long pfn)
#error Allocator MAX_ORDER exceeds SECTION_SIZE
#endif
-#define pfn_to_section_nr(pfn) ((pfn) >> PFN_SECTION_SHIFT)
-#define section_nr_to_pfn(sec) ((sec) << PFN_SECTION_SHIFT)
+#define pfn_to_section_nr(pfn) ((unsigned long)(pfn) >> PFN_SECTION_SHIFT)
+#define section_nr_to_pfn(sec) ((unsigned long)(sec) << PFN_SECTION_SHIFT)
#define SECTION_ALIGN_UP(pfn) (((pfn) + PAGES_PER_SECTION - 1) & PAGE_SECTION_MASK)
#define SECTION_ALIGN_DOWN(pfn) ((pfn) & PAGE_SECTION_MASK)
diff --git a/mm/memory_hotplug.c b/mm/memory_hotplug.c
index e43142c15631..de34951bdf41 100644
--- a/mm/memory_hotplug.c
+++ b/mm/memory_hotplug.c
@@ -528,9 +528,7 @@ static int __meminit __add_section(int nid, struct zone *zone,
int __ref __add_pages(int nid, struct zone *zone, unsigned long phys_start_pfn,
unsigned long nr_pages)
{
- unsigned long i;
- int err = 0;
- int start_sec, end_sec;
+ int err = 0, i, start_sec, end_sec;
struct vmem_altmap *altmap;
clear_zone_contiguous(zone);
^ permalink raw reply related [flat|nested] 15+ messages in thread* [PATCH v3 02/12] mm, devm_memremap_pages: use multi-order radix for ZONE_DEVICE lookups
2017-01-19 22:06 [PATCH v3 00/12] mm: sub-section memory hotplug support Dan Williams
2017-01-19 22:06 ` [PATCH v3 01/12] mm: fix type width of section to/from pfn conversion macros Dan Williams
@ 2017-01-19 22:06 ` Dan Williams
2017-01-19 22:06 ` [PATCH v3 03/12] mm: introduce struct mem_section_usage to track partial population of a section Dan Williams
` (9 subsequent siblings)
11 siblings, 0 replies; 15+ messages in thread
From: Dan Williams @ 2017-01-19 22:06 UTC (permalink / raw)
To: akpm; +Cc: linux-mm, linux-kernel, Toshi Kani, linux-nvdimm
devm_memremap_pages() records mapped ranges in pgmap_radix with an entry
per section's worth of memory (128MB). The key for each of those
entries is a section number.
This leads to false positives when devm_memremap_pages() is passed a
section-unaligned range as lookups in the misalignment fail to return
NULL. We can close this hole by using the pfn as the key for entries in
the tree. The number of entries required to describe a remapped range
is reduced by leveraging multi-order entries.
In practice this approach usually yields just one entry in the tree if
the size and starting address are of the same power-of-2 alignment.
Previously we always needed nr_entries = mapping_size / 128MB.
Link: https://lists.01.org/pipermail/linux-nvdimm/2016-August/006666.html
Reported-by: Toshi Kani <toshi.kani@hpe.com>
Cc: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Dan Williams <dan.j.williams@intel.com>
---
kernel/memremap.c | 52 ++++++++++++++++++++++++++++++++++++++--------------
mm/Kconfig | 1 +
2 files changed, 39 insertions(+), 14 deletions(-)
diff --git a/kernel/memremap.c b/kernel/memremap.c
index 9ecedc28b928..e0a2de6a168b 100644
--- a/kernel/memremap.c
+++ b/kernel/memremap.c
@@ -194,18 +194,41 @@ void put_zone_device_page(struct page *page)
}
EXPORT_SYMBOL(put_zone_device_page);
-static void pgmap_radix_release(struct resource *res)
+static unsigned long order_at(struct resource *res, unsigned long pgoff)
{
- resource_size_t key, align_start, align_size, align_end;
+ unsigned long phys_pgoff = PHYS_PFN(res->start) + pgoff;
+ unsigned long nr_pages, mask;
- align_start = res->start & ~(SECTION_SIZE - 1);
- align_size = ALIGN(resource_size(res), SECTION_SIZE);
- align_end = align_start + align_size - 1;
+ nr_pages = PHYS_PFN(resource_size(res));
+ if (nr_pages == pgoff)
+ return ULONG_MAX;
+
+ /*
+ * What is the largest aligned power-of-2 range available from
+ * this resource pgoff to the end of the resource range,
+ * considering the alignment of the current pgoff?
+ */
+ mask = phys_pgoff | rounddown_pow_of_two(nr_pages - pgoff);
+ if (!mask)
+ return ULONG_MAX;
+
+ return find_first_bit(&mask, BITS_PER_LONG);
+}
+
+#define foreach_order_pgoff(res, order, pgoff) \
+ for (pgoff = 0, order = order_at((res), pgoff); order < ULONG_MAX; \
+ pgoff += 1UL << order, order = order_at((res), pgoff))
+
+static void pgmap_radix_release(struct resource *res)
+{
+ unsigned long pgoff, order;
mutex_lock(&pgmap_lock);
- for (key = res->start; key <= res->end; key += SECTION_SIZE)
- radix_tree_delete(&pgmap_radix, key >> PA_SECTION_SHIFT);
+ foreach_order_pgoff(res, order, pgoff)
+ radix_tree_delete(&pgmap_radix, PHYS_PFN(res->start) + pgoff);
mutex_unlock(&pgmap_lock);
+
+ synchronize_rcu();
}
static unsigned long pfn_first(struct page_map *page_map)
@@ -262,7 +285,7 @@ struct dev_pagemap *find_dev_pagemap(resource_size_t phys)
WARN_ON_ONCE(!rcu_read_lock_held());
- page_map = radix_tree_lookup(&pgmap_radix, phys >> PA_SECTION_SHIFT);
+ page_map = radix_tree_lookup(&pgmap_radix, PHYS_PFN(phys));
return page_map ? &page_map->pgmap : NULL;
}
@@ -284,12 +307,12 @@ struct dev_pagemap *find_dev_pagemap(resource_size_t phys)
void *devm_memremap_pages(struct device *dev, struct resource *res,
struct percpu_ref *ref, struct vmem_altmap *altmap)
{
- resource_size_t key, align_start, align_size, align_end;
+ resource_size_t align_start, align_size, align_end;
+ unsigned long pfn, pgoff, order;
pgprot_t pgprot = PAGE_KERNEL;
struct dev_pagemap *pgmap;
struct page_map *page_map;
int error, nid, is_ram;
- unsigned long pfn;
align_start = res->start & ~(SECTION_SIZE - 1);
align_size = ALIGN(res->start + resource_size(res), SECTION_SIZE)
@@ -328,11 +351,12 @@ void *devm_memremap_pages(struct device *dev, struct resource *res,
mutex_lock(&pgmap_lock);
error = 0;
align_end = align_start + align_size - 1;
- for (key = align_start; key <= align_end; key += SECTION_SIZE) {
+
+ foreach_order_pgoff(res, order, pgoff) {
struct dev_pagemap *dup;
rcu_read_lock();
- dup = find_dev_pagemap(key);
+ dup = find_dev_pagemap(res->start + PFN_PHYS(pgoff));
rcu_read_unlock();
if (dup) {
dev_err(dev, "%s: %pr collides with mapping for %s\n",
@@ -340,8 +364,8 @@ void *devm_memremap_pages(struct device *dev, struct resource *res,
error = -EBUSY;
break;
}
- error = radix_tree_insert(&pgmap_radix, key >> PA_SECTION_SHIFT,
- page_map);
+ error = __radix_tree_insert(&pgmap_radix,
+ PHYS_PFN(res->start) + pgoff, order, page_map);
if (error) {
dev_err(dev, "%s: failed: %d\n", __func__, error);
break;
diff --git a/mm/Kconfig b/mm/Kconfig
index 9b8fccb969dc..4aff67e042f1 100644
--- a/mm/Kconfig
+++ b/mm/Kconfig
@@ -690,6 +690,7 @@ config ZONE_DEVICE
depends on MEMORY_HOTREMOVE
depends on SPARSEMEM_VMEMMAP
depends on X86_64 #arch_add_memory() comprehends device memory
+ select RADIX_TREE_MULTIORDER
help
Device memory hotplug support allows for establishing pmem,
^ permalink raw reply related [flat|nested] 15+ messages in thread* [PATCH v3 03/12] mm: introduce struct mem_section_usage to track partial population of a section
2017-01-19 22:06 [PATCH v3 00/12] mm: sub-section memory hotplug support Dan Williams
2017-01-19 22:06 ` [PATCH v3 01/12] mm: fix type width of section to/from pfn conversion macros Dan Williams
2017-01-19 22:06 ` [PATCH v3 02/12] mm, devm_memremap_pages: use multi-order radix for ZONE_DEVICE lookups Dan Williams
@ 2017-01-19 22:06 ` Dan Williams
2017-01-19 22:07 ` [PATCH v3 04/12] mm: introduce common definitions for the size and mask " Dan Williams
` (8 subsequent siblings)
11 siblings, 0 replies; 15+ messages in thread
From: Dan Williams @ 2017-01-19 22:06 UTC (permalink / raw)
To: akpm
Cc: Michal Hocko, linux-nvdimm, Logan Gunthorpe, linux-kernel,
Stephen Bates, linux-mm, Johannes Weiner, Mel Gorman,
Vlastimil Babka
'struct mem_section_usage' combines the existing 'pageblock_flags' bitmap
with a new 'map_active' bitmap. The new bitmap enables the memory
hot{plug,remove} implementation to act on incremental sub-divisions of
a section. The primary impetus for this functionality is to support
platforms that mix "System RAM" and "Persistent Memory" within a single
section. We want to be able to hotplug "Persistent Memory" to extend a
partially populated section and share that section between ZONE_DEVICE and
ZONE_NORMAL/MOVABLE memory.
This introduces a pointer to the new 'map_active' bitmap through struct
mem_section, but otherwise should not change any behavior.
Cc: Michal Hocko <mhocko@suse.com>
Cc: Vlastimil Babka <vbabka@suse.cz>
Cc: Johannes Weiner <hannes@cmpxchg.org>
Cc: Logan Gunthorpe <logang@deltatee.com>
Cc: Mel Gorman <mgorman@techsingularity.net>
Cc: Andrew Morton <akpm@linux-foundation.org>
Cc: Stephen Bates <stephen.bates@microsemi.com>
Signed-off-by: Dan Williams <dan.j.williams@intel.com>
---
include/linux/mmzone.h | 21 +++++++++-
mm/memory_hotplug.c | 4 +-
mm/page_alloc.c | 2 -
mm/sparse.c | 98 ++++++++++++++++++++++++++----------------------
4 files changed, 75 insertions(+), 50 deletions(-)
diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h
index df36d21e0247..18cc15b93d19 100644
--- a/include/linux/mmzone.h
+++ b/include/linux/mmzone.h
@@ -1068,6 +1068,19 @@ static inline unsigned long early_pfn_to_nid(unsigned long pfn)
#define SECTION_ALIGN_UP(pfn) (((pfn) + PAGES_PER_SECTION - 1) & PAGE_SECTION_MASK)
#define SECTION_ALIGN_DOWN(pfn) ((pfn) & PAGE_SECTION_MASK)
+#define SECTION_ACTIVE_SIZE ((1UL << SECTION_SIZE_BITS) / BITS_PER_LONG)
+#define SECTION_ACTIVE_MASK (~(SECTION_ACTIVE_SIZE - 1))
+
+struct mem_section_usage {
+ /*
+ * SECTION_ACTIVE_SIZE portions of the section that are populated in
+ * the memmap
+ */
+ unsigned long map_active;
+ /* See declaration of similar field in struct zone */
+ unsigned long pageblock_flags[0];
+};
+
struct page;
struct page_ext;
struct mem_section {
@@ -1085,8 +1098,7 @@ struct mem_section {
*/
unsigned long section_mem_map;
- /* See declaration of similar field in struct zone */
- unsigned long *pageblock_flags;
+ struct mem_section_usage *usage;
#ifdef CONFIG_PAGE_EXTENSION
/*
* If SPARSEMEM, pgdat doesn't have page_ext pointer. We use
@@ -1117,6 +1129,11 @@ extern struct mem_section *mem_section[NR_SECTION_ROOTS];
extern struct mem_section mem_section[NR_SECTION_ROOTS][SECTIONS_PER_ROOT];
#endif
+static inline unsigned long *section_to_usemap(struct mem_section *ms)
+{
+ return ms->usage->pageblock_flags;
+}
+
static inline struct mem_section *__nr_to_section(unsigned long nr)
{
if (!mem_section[SECTION_NR_TO_ROOT(nr)])
diff --git a/mm/memory_hotplug.c b/mm/memory_hotplug.c
index de34951bdf41..a73a1fcaa181 100644
--- a/mm/memory_hotplug.c
+++ b/mm/memory_hotplug.c
@@ -227,7 +227,7 @@ static void register_page_bootmem_info_section(unsigned long start_pfn)
for (i = 0; i < mapsize; i++, page++)
get_page_bootmem(section_nr, page, SECTION_INFO);
- usemap = __nr_to_section(section_nr)->pageblock_flags;
+ usemap = section_to_usemap(__nr_to_section(section_nr));
page = virt_to_page(usemap);
mapsize = PAGE_ALIGN(usemap_size()) >> PAGE_SHIFT;
@@ -253,7 +253,7 @@ static void register_page_bootmem_info_section(unsigned long start_pfn)
register_page_bootmem_memmap(section_nr, memmap, PAGES_PER_SECTION);
- usemap = __nr_to_section(section_nr)->pageblock_flags;
+ usemap = section_to_usemap(__nr_to_section(section_nr));
page = virt_to_page(usemap);
mapsize = PAGE_ALIGN(usemap_size()) >> PAGE_SHIFT;
diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index d604d2596b7b..68ccf5bcdbb2 100644
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -352,7 +352,7 @@ static inline unsigned long *get_pageblock_bitmap(struct page *page,
unsigned long pfn)
{
#ifdef CONFIG_SPARSEMEM
- return __pfn_to_section(pfn)->pageblock_flags;
+ return section_to_usemap(__pfn_to_section(pfn));
#else
return page_zone(page)->pageblock_flags;
#endif /* CONFIG_SPARSEMEM */
diff --git a/mm/sparse.c b/mm/sparse.c
index 1e168bf2779a..91e1908db23d 100644
--- a/mm/sparse.c
+++ b/mm/sparse.c
@@ -233,15 +233,15 @@ struct page *sparse_decode_mem_map(unsigned long coded_mem_map, unsigned long pn
static int __meminit sparse_init_one_section(struct mem_section *ms,
unsigned long pnum, struct page *mem_map,
- unsigned long *pageblock_bitmap)
+ struct mem_section_usage *usage)
{
if (!present_section(ms))
return -EINVAL;
ms->section_mem_map &= ~SECTION_MAP_MASK;
ms->section_mem_map |= sparse_encode_mem_map(mem_map, pnum) |
- SECTION_HAS_MEM_MAP;
- ms->pageblock_flags = pageblock_bitmap;
+ SECTION_HAS_MEM_MAP;
+ ms->usage = usage;
return 1;
}
@@ -255,9 +255,13 @@ unsigned long usemap_size(void)
}
#ifdef CONFIG_MEMORY_HOTPLUG
-static unsigned long *__kmalloc_section_usemap(void)
+static struct mem_section_usage *__alloc_section_usage(void)
{
- return kmalloc(usemap_size(), GFP_KERNEL);
+ struct mem_section_usage *usage;
+
+ usage = kzalloc(sizeof(*usage) + usemap_size(), GFP_KERNEL);
+ /* TODO: allocate the map_active bitmap */
+ return usage;
}
#endif /* CONFIG_MEMORY_HOTPLUG */
@@ -293,7 +297,8 @@ sparse_early_usemaps_alloc_pgdat_section(struct pglist_data *pgdat,
return p;
}
-static void __init check_usemap_section_nr(int nid, unsigned long *usemap)
+static void __init check_usemap_section_nr(int nid,
+ struct mem_section_usage *usage)
{
unsigned long usemap_snr, pgdat_snr;
static unsigned long old_usemap_snr = NR_MEM_SECTIONS;
@@ -301,7 +306,7 @@ static void __init check_usemap_section_nr(int nid, unsigned long *usemap)
struct pglist_data *pgdat = NODE_DATA(nid);
int usemap_nid;
- usemap_snr = pfn_to_section_nr(__pa(usemap) >> PAGE_SHIFT);
+ usemap_snr = pfn_to_section_nr(__pa(usage) >> PAGE_SHIFT);
pgdat_snr = pfn_to_section_nr(__pa(pgdat) >> PAGE_SHIFT);
if (usemap_snr == pgdat_snr)
return;
@@ -336,7 +341,8 @@ sparse_early_usemaps_alloc_pgdat_section(struct pglist_data *pgdat,
return memblock_virt_alloc_node_nopanic(size, pgdat->node_id);
}
-static void __init check_usemap_section_nr(int nid, unsigned long *usemap)
+static void __init check_usemap_section_nr(int nid,
+ struct mem_section_usage *usage)
{
}
#endif /* CONFIG_MEMORY_HOTREMOVE */
@@ -344,26 +350,27 @@ static void __init check_usemap_section_nr(int nid, unsigned long *usemap)
static void __init sparse_early_usemaps_alloc_node(void *data,
unsigned long pnum_begin,
unsigned long pnum_end,
- unsigned long usemap_count, int nodeid)
+ unsigned long usage_count, int nodeid)
{
- void *usemap;
+ void *usage;
unsigned long pnum;
- unsigned long **usemap_map = (unsigned long **)data;
- int size = usemap_size();
+ struct mem_section_usage **usage_map = data;
+ int size = sizeof(struct mem_section_usage) + usemap_size();
- usemap = sparse_early_usemaps_alloc_pgdat_section(NODE_DATA(nodeid),
- size * usemap_count);
- if (!usemap) {
+ usage = sparse_early_usemaps_alloc_pgdat_section(NODE_DATA(nodeid),
+ size * usage_count);
+ if (!usage) {
pr_warn("%s: allocation failed\n", __func__);
return;
}
+ memset(usage, 0, size * usage_count);
for (pnum = pnum_begin; pnum < pnum_end; pnum++) {
if (!present_section_nr(pnum))
continue;
- usemap_map[pnum] = usemap;
- usemap += size;
- check_usemap_section_nr(nodeid, usemap_map[pnum]);
+ usage_map[pnum] = usage;
+ usage += size;
+ check_usemap_section_nr(nodeid, usage_map[pnum]);
}
}
@@ -468,7 +475,7 @@ void __weak __meminit vmemmap_populate_print_last(void)
/**
* alloc_usemap_and_memmap - memory alloction for pageblock flags and vmemmap
- * @map: usemap_map for pageblock flags or mmap_map for vmemmap
+ * @map: usage_map for mem_section_usage or mmap_map for vmemmap
*/
static void __init alloc_usemap_and_memmap(void (*alloc_func)
(void *, unsigned long, unsigned long,
@@ -521,10 +528,9 @@ static void __init alloc_usemap_and_memmap(void (*alloc_func)
*/
void __init sparse_init(void)
{
+ struct mem_section_usage *usage, **usage_map;
unsigned long pnum;
struct page *map;
- unsigned long *usemap;
- unsigned long **usemap_map;
int size;
#ifdef CONFIG_SPARSEMEM_ALLOC_MEM_MAP_TOGETHER
int size2;
@@ -539,21 +545,21 @@ void __init sparse_init(void)
/*
* map is using big page (aka 2M in x86 64 bit)
- * usemap is less one page (aka 24 bytes)
+ * usage is less one page (aka 24 bytes)
* so alloc 2M (with 2M align) and 24 bytes in turn will
* make next 2M slip to one more 2M later.
* then in big system, the memory will have a lot of holes...
* here try to allocate 2M pages continuously.
*
* powerpc need to call sparse_init_one_section right after each
- * sparse_early_mem_map_alloc, so allocate usemap_map at first.
+ * sparse_early_mem_map_alloc, so allocate usage_map at first.
*/
- size = sizeof(unsigned long *) * NR_MEM_SECTIONS;
- usemap_map = memblock_virt_alloc(size, 0);
- if (!usemap_map)
- panic("can not allocate usemap_map\n");
+ size = sizeof(struct mem_section_usage *) * NR_MEM_SECTIONS;
+ usage_map = memblock_virt_alloc(size, 0);
+ if (!usage_map)
+ panic("can not allocate usage_map\n");
alloc_usemap_and_memmap(sparse_early_usemaps_alloc_node,
- (void *)usemap_map);
+ (void *)usage_map);
#ifdef CONFIG_SPARSEMEM_ALLOC_MEM_MAP_TOGETHER
size2 = sizeof(struct page *) * NR_MEM_SECTIONS;
@@ -568,8 +574,8 @@ void __init sparse_init(void)
if (!present_section_nr(pnum))
continue;
- usemap = usemap_map[pnum];
- if (!usemap)
+ usage = usage_map[pnum];
+ if (!usage)
continue;
#ifdef CONFIG_SPARSEMEM_ALLOC_MEM_MAP_TOGETHER
@@ -581,7 +587,7 @@ void __init sparse_init(void)
continue;
sparse_init_one_section(__nr_to_section(pnum), pnum, map,
- usemap);
+ usage);
}
vmemmap_populate_print_last();
@@ -589,7 +595,7 @@ void __init sparse_init(void)
#ifdef CONFIG_SPARSEMEM_ALLOC_MEM_MAP_TOGETHER
memblock_free_early(__pa(map_map), size2);
#endif
- memblock_free_early(__pa(usemap_map), size);
+ memblock_free_early(__pa(usage_map), size);
}
#ifdef CONFIG_MEMORY_HOTPLUG
@@ -693,9 +699,9 @@ int __meminit sparse_add_one_section(struct zone *zone, unsigned long start_pfn)
{
unsigned long section_nr = pfn_to_section_nr(start_pfn);
struct pglist_data *pgdat = zone->zone_pgdat;
+ static struct mem_section_usage *usage;
struct mem_section *ms;
struct page *memmap;
- unsigned long *usemap;
unsigned long flags;
int ret;
@@ -709,8 +715,8 @@ int __meminit sparse_add_one_section(struct zone *zone, unsigned long start_pfn)
memmap = kmalloc_section_memmap(section_nr, pgdat->node_id);
if (!memmap)
return -ENOMEM;
- usemap = __kmalloc_section_usemap();
- if (!usemap) {
+ usage = __alloc_section_usage();
+ if (!usage) {
__kfree_section_memmap(memmap);
return -ENOMEM;
}
@@ -727,12 +733,12 @@ int __meminit sparse_add_one_section(struct zone *zone, unsigned long start_pfn)
ms->section_mem_map |= SECTION_MARKED_PRESENT;
- ret = sparse_init_one_section(ms, section_nr, memmap, usemap);
+ ret = sparse_init_one_section(ms, section_nr, memmap, usage);
out:
pgdat_resize_unlock(pgdat, &flags);
if (ret <= 0) {
- kfree(usemap);
+ kfree(usage);
__kfree_section_memmap(memmap);
}
return ret;
@@ -760,19 +766,20 @@ static inline void clear_hwpoisoned_pages(struct page *memmap, int nr_pages)
}
#endif
-static void free_section_usemap(struct page *memmap, unsigned long *usemap)
+static void free_section_usage(struct page *memmap,
+ struct mem_section_usage *usage)
{
struct page *usemap_page;
- if (!usemap)
+ if (!usage)
return;
- usemap_page = virt_to_page(usemap);
+ usemap_page = virt_to_page(usage->pageblock_flags);
/*
* Check to see if allocation came from hot-plug-add
*/
if (PageSlab(usemap_page) || PageCompound(usemap_page)) {
- kfree(usemap);
+ kfree(usage);
if (memmap)
__kfree_section_memmap(memmap);
return;
@@ -790,23 +797,24 @@ static void free_section_usemap(struct page *memmap, unsigned long *usemap)
void sparse_remove_one_section(struct zone *zone, struct mem_section *ms,
unsigned long map_offset)
{
+ unsigned long flags;
struct page *memmap = NULL;
- unsigned long *usemap = NULL, flags;
+ struct mem_section_usage *usage = NULL;
struct pglist_data *pgdat = zone->zone_pgdat;
pgdat_resize_lock(pgdat, &flags);
if (ms->section_mem_map) {
- usemap = ms->pageblock_flags;
+ usage = ms->usage;
memmap = sparse_decode_mem_map(ms->section_mem_map,
__section_nr(ms));
ms->section_mem_map = 0;
- ms->pageblock_flags = NULL;
+ ms->usage = NULL;
}
pgdat_resize_unlock(pgdat, &flags);
clear_hwpoisoned_pages(memmap + map_offset,
PAGES_PER_SECTION - map_offset);
- free_section_usemap(memmap, usemap);
+ free_section_usage(memmap, usage);
}
#endif /* CONFIG_MEMORY_HOTREMOVE */
#endif /* CONFIG_MEMORY_HOTPLUG */
^ permalink raw reply related [flat|nested] 15+ messages in thread* [PATCH v3 04/12] mm: introduce common definitions for the size and mask of a section
2017-01-19 22:06 [PATCH v3 00/12] mm: sub-section memory hotplug support Dan Williams
` (2 preceding siblings ...)
2017-01-19 22:06 ` [PATCH v3 03/12] mm: introduce struct mem_section_usage to track partial population of a section Dan Williams
@ 2017-01-19 22:07 ` Dan Williams
2017-01-19 22:07 ` [PATCH v3 05/12] mm: cleanup sparse_init_one_section() return value Dan Williams
` (7 subsequent siblings)
11 siblings, 0 replies; 15+ messages in thread
From: Dan Williams @ 2017-01-19 22:07 UTC (permalink / raw)
To: akpm
Cc: Michal Hocko, linux-nvdimm, Logan Gunthorpe, linux-kernel,
Stephen Bates, linux-mm, Johannes Weiner, Mel Gorman,
Vlastimil Babka
Up-level the local section size and mask from kernel/memremap.c to
global definitions. These will be used by the new sub-section hotplug
support.
Cc: Michal Hocko <mhocko@suse.com>
Cc: Vlastimil Babka <vbabka@suse.cz>
Cc: Johannes Weiner <hannes@cmpxchg.org>
Cc: Logan Gunthorpe <logang@deltatee.com>
Cc: Mel Gorman <mgorman@techsingularity.net>
Cc: Andrew Morton <akpm@linux-foundation.org>
Cc: Stephen Bates <stephen.bates@microsemi.com>
Signed-off-by: Dan Williams <dan.j.williams@intel.com>
---
include/linux/mmzone.h | 2 ++
kernel/memremap.c | 10 ++++------
2 files changed, 6 insertions(+), 6 deletions(-)
diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h
index 18cc15b93d19..aa37e2e860ed 100644
--- a/include/linux/mmzone.h
+++ b/include/linux/mmzone.h
@@ -1048,6 +1048,8 @@ static inline unsigned long early_pfn_to_nid(unsigned long pfn)
* PFN_SECTION_SHIFT pfn to/from section number
*/
#define PA_SECTION_SHIFT (SECTION_SIZE_BITS)
+#define PA_SECTION_SIZE (1UL << PA_SECTION_SHIFT)
+#define PA_SECTION_MASK (~(PA_SECTION_SIZE-1))
#define PFN_SECTION_SHIFT (SECTION_SIZE_BITS - PAGE_SHIFT)
#define NR_MEM_SECTIONS (1UL << SECTIONS_SHIFT)
diff --git a/kernel/memremap.c b/kernel/memremap.c
index e0a2de6a168b..e58d7828ff9a 100644
--- a/kernel/memremap.c
+++ b/kernel/memremap.c
@@ -172,8 +172,6 @@ EXPORT_SYMBOL(devm_memunmap);
#ifdef CONFIG_ZONE_DEVICE
static DEFINE_MUTEX(pgmap_lock);
static RADIX_TREE(pgmap_radix, GFP_KERNEL);
-#define SECTION_MASK ~((1UL << PA_SECTION_SHIFT) - 1)
-#define SECTION_SIZE (1UL << PA_SECTION_SHIFT)
struct page_map {
struct resource res;
@@ -267,8 +265,8 @@ static void devm_memremap_pages_release(struct device *dev, void *data)
}
/* pages are dead and unused, undo the arch mapping */
- align_start = res->start & ~(SECTION_SIZE - 1);
- align_size = ALIGN(resource_size(res), SECTION_SIZE);
+ align_start = res->start & PA_SECTION_MASK;
+ align_size = ALIGN(resource_size(res), PA_SECTION_SIZE);
mem_hotplug_begin();
arch_remove_memory(align_start, align_size);
mem_hotplug_done();
@@ -314,8 +312,8 @@ void *devm_memremap_pages(struct device *dev, struct resource *res,
struct page_map *page_map;
int error, nid, is_ram;
- align_start = res->start & ~(SECTION_SIZE - 1);
- align_size = ALIGN(res->start + resource_size(res), SECTION_SIZE)
+ align_start = res->start & PA_SECTION_MASK;
+ align_size = ALIGN(res->start + resource_size(res), PA_SECTION_SIZE)
- align_start;
is_ram = region_intersects(align_start, align_size,
IORESOURCE_SYSTEM_RAM, IORES_DESC_NONE);
^ permalink raw reply related [flat|nested] 15+ messages in thread* [PATCH v3 05/12] mm: cleanup sparse_init_one_section() return value
2017-01-19 22:06 [PATCH v3 00/12] mm: sub-section memory hotplug support Dan Williams
` (3 preceding siblings ...)
2017-01-19 22:07 ` [PATCH v3 04/12] mm: introduce common definitions for the size and mask " Dan Williams
@ 2017-01-19 22:07 ` Dan Williams
2017-01-19 22:07 ` [PATCH v3 06/12] mm: track active portions of a section at boot Dan Williams
` (6 subsequent siblings)
11 siblings, 0 replies; 15+ messages in thread
From: Dan Williams @ 2017-01-19 22:07 UTC (permalink / raw)
To: akpm
Cc: Michal Hocko, linux-nvdimm, Logan Gunthorpe, linux-kernel,
Stephen Bates, linux-mm, Johannes Weiner, Mel Gorman,
Vlastimil Babka
We mark and check that the section is present under a spin_lock() in
sparse_add_one_section(), so the lock ensures it will not change between
those 2 events. Also, we do not check the -EBUSY return value in
sparse_init(). Just make sparse_init_one_section() return void and clean
up the error handling.
Cc: Michal Hocko <mhocko@suse.com>
Cc: Vlastimil Babka <vbabka@suse.cz>
Cc: Johannes Weiner <hannes@cmpxchg.org>
Cc: Logan Gunthorpe <logang@deltatee.com>
Cc: Mel Gorman <mgorman@techsingularity.net>
Cc: Andrew Morton <akpm@linux-foundation.org>
Cc: Stephen Bates <stephen.bates@microsemi.com>
Signed-off-by: Dan Williams <dan.j.williams@intel.com>
---
mm/sparse.c | 21 ++++++---------------
1 file changed, 6 insertions(+), 15 deletions(-)
diff --git a/mm/sparse.c b/mm/sparse.c
index 91e1908db23d..59966a3e8ff0 100644
--- a/mm/sparse.c
+++ b/mm/sparse.c
@@ -231,19 +231,14 @@ struct page *sparse_decode_mem_map(unsigned long coded_mem_map, unsigned long pn
return ((struct page *)coded_mem_map) + section_nr_to_pfn(pnum);
}
-static int __meminit sparse_init_one_section(struct mem_section *ms,
+static void __meminit sparse_init_one_section(struct mem_section *ms,
unsigned long pnum, struct page *mem_map,
struct mem_section_usage *usage)
{
- if (!present_section(ms))
- return -EINVAL;
-
ms->section_mem_map &= ~SECTION_MAP_MASK;
ms->section_mem_map |= sparse_encode_mem_map(mem_map, pnum) |
SECTION_HAS_MEM_MAP;
ms->usage = usage;
-
- return 1;
}
unsigned long usemap_size(void)
@@ -690,11 +685,6 @@ static void free_map_bootmem(struct page *memmap)
#endif /* CONFIG_MEMORY_HOTREMOVE */
#endif /* CONFIG_SPARSEMEM_VMEMMAP */
-/*
- * returns the number of sections whose mem_maps were properly
- * set. If this is <=0, then that means that the passed-in
- * map was not consumed and must be freed.
- */
int __meminit sparse_add_one_section(struct zone *zone, unsigned long start_pfn)
{
unsigned long section_nr = pfn_to_section_nr(start_pfn);
@@ -725,7 +715,7 @@ int __meminit sparse_add_one_section(struct zone *zone, unsigned long start_pfn)
ms = __pfn_to_section(start_pfn);
if (ms->section_mem_map & SECTION_MARKED_PRESENT) {
- ret = -EEXIST;
+ ret = -EBUSY;
goto out;
}
@@ -733,15 +723,16 @@ int __meminit sparse_add_one_section(struct zone *zone, unsigned long start_pfn)
ms->section_mem_map |= SECTION_MARKED_PRESENT;
- ret = sparse_init_one_section(ms, section_nr, memmap, usage);
+ sparse_init_one_section(ms, section_nr, memmap, usage);
out:
pgdat_resize_unlock(pgdat, &flags);
- if (ret <= 0) {
+ if (ret < 0 && ret != -EEXIST) {
kfree(usage);
__kfree_section_memmap(memmap);
+ return ret;
}
- return ret;
+ return 0;
}
#ifdef CONFIG_MEMORY_HOTREMOVE
^ permalink raw reply related [flat|nested] 15+ messages in thread* [PATCH v3 06/12] mm: track active portions of a section at boot
2017-01-19 22:06 [PATCH v3 00/12] mm: sub-section memory hotplug support Dan Williams
` (4 preceding siblings ...)
2017-01-19 22:07 ` [PATCH v3 05/12] mm: cleanup sparse_init_one_section() return value Dan Williams
@ 2017-01-19 22:07 ` Dan Williams
2017-01-20 0:05 ` Andrew Morton
2017-01-19 22:07 ` [PATCH v3 07/12] mm: fix register_new_memory() zone type detection Dan Williams
` (5 subsequent siblings)
11 siblings, 1 reply; 15+ messages in thread
From: Dan Williams @ 2017-01-19 22:07 UTC (permalink / raw)
To: akpm
Cc: Michal Hocko, linux-nvdimm, Logan Gunthorpe, linux-kernel,
Stephen Bates, linux-mm, Johannes Weiner, Mel Gorman,
Vlastimil Babka
Prepare for hot{plug,remove} of sub-ranges of a section by tracking a
section active bitmask, each bit representing 2MB (SECTION_SIZE (128M) /
map_active bitmask length (64)).
Cc: Michal Hocko <mhocko@suse.com>
Cc: Vlastimil Babka <vbabka@suse.cz>
Cc: Johannes Weiner <hannes@cmpxchg.org>
Cc: Logan Gunthorpe <logang@deltatee.com>
Cc: Mel Gorman <mgorman@techsingularity.net>
Cc: Andrew Morton <akpm@linux-foundation.org>
Cc: Stephen Bates <stephen.bates@microsemi.com>
Signed-off-by: Dan Williams <dan.j.williams@intel.com>
---
include/linux/mmzone.h | 3 +++
mm/page_alloc.c | 4 +++-
mm/sparse.c | 53 ++++++++++++++++++++++++++++++++++++++++++++++++
3 files changed, 59 insertions(+), 1 deletion(-)
diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h
index aa37e2e860ed..6b9d44ce1e2e 100644
--- a/include/linux/mmzone.h
+++ b/include/linux/mmzone.h
@@ -1083,6 +1083,8 @@ struct mem_section_usage {
unsigned long pageblock_flags[0];
};
+void section_active_init(unsigned long pfn, unsigned long nr_pages);
+
struct page;
struct page_ext;
struct mem_section {
@@ -1224,6 +1226,7 @@ void sparse_init(void);
#else
#define sparse_init() do {} while (0)
#define sparse_index_init(_sec, _nid) do {} while (0)
+#define section_active_init(_pfn, _nr_pages) do {} while (0)
#endif /* CONFIG_SPARSEMEM */
/*
diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index 68ccf5bcdbb2..9a3ab6c245a8 100644
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -6352,10 +6352,12 @@ void __init free_area_init_nodes(unsigned long *max_zone_pfn)
/* Print out the early node map */
pr_info("Early memory node ranges\n");
- for_each_mem_pfn_range(i, MAX_NUMNODES, &start_pfn, &end_pfn, &nid)
+ for_each_mem_pfn_range(i, MAX_NUMNODES, &start_pfn, &end_pfn, &nid) {
pr_info(" node %3d: [mem %#018Lx-%#018Lx]\n", nid,
(u64)start_pfn << PAGE_SHIFT,
((u64)end_pfn << PAGE_SHIFT) - 1);
+ section_active_init(start_pfn, end_pfn - start_pfn);
+ }
/* Initialise every node */
mminit_verify_pageflags_layout();
diff --git a/mm/sparse.c b/mm/sparse.c
index 59966a3e8ff0..00fdb5d04680 100644
--- a/mm/sparse.c
+++ b/mm/sparse.c
@@ -168,6 +168,59 @@ void __meminit mminit_validate_memmodel_limits(unsigned long *start_pfn,
}
}
+static int section_active_index(phys_addr_t phys)
+{
+ return (phys & ~(PA_SECTION_MASK)) / SECTION_ACTIVE_SIZE;
+}
+
+static unsigned long section_active_mask(unsigned long pfn,
+ unsigned long nr_pages)
+{
+ int idx_start, idx_size;
+ phys_addr_t start, size;
+
+ if (!nr_pages)
+ return 0;
+
+ start = PFN_PHYS(pfn);
+ size = PFN_PHYS(min(nr_pages, PAGES_PER_SECTION
+ - (pfn & ~PAGE_SECTION_MASK)));
+ size = ALIGN(size, SECTION_ACTIVE_SIZE);
+
+ idx_start = section_active_index(start);
+ idx_size = section_active_index(size);
+
+ if (idx_size == 0)
+ return -1;
+ return ((1UL << idx_size) - 1) << idx_start;
+}
+
+void section_active_init(unsigned long pfn, unsigned long nr_pages)
+{
+ int end_sec = pfn_to_section_nr(pfn + nr_pages - 1);
+ int i, start_sec = pfn_to_section_nr(pfn);
+
+ if (!nr_pages)
+ return;
+
+ for (i = start_sec; i <= end_sec; i++) {
+ struct mem_section *ms;
+ unsigned long mask;
+ unsigned long pfns;
+
+ pfns = min(nr_pages, PAGES_PER_SECTION
+ - (pfn & ~PAGE_SECTION_MASK));
+ mask = section_active_mask(pfn, pfns);
+
+ ms = __nr_to_section(i);
+ pr_debug("%s: sec: %d mask: %#018lx\n", __func__, i, mask);
+ ms->usage->map_active = mask;
+
+ pfn += pfns;
+ nr_pages -= pfns;
+ }
+}
+
/* Record a memory area against a node. */
void __init memory_present(int nid, unsigned long start, unsigned long end)
{
^ permalink raw reply related [flat|nested] 15+ messages in thread* Re: [PATCH v3 06/12] mm: track active portions of a section at boot
2017-01-19 22:07 ` [PATCH v3 06/12] mm: track active portions of a section at boot Dan Williams
@ 2017-01-20 0:05 ` Andrew Morton
2017-01-20 17:56 ` Dan Williams
0 siblings, 1 reply; 15+ messages in thread
From: Andrew Morton @ 2017-01-20 0:05 UTC (permalink / raw)
To: Dan Williams
Cc: Michal Hocko, linux-nvdimm, Logan Gunthorpe, linux-kernel,
Stephen Bates, linux-mm, Johannes Weiner, Mel Gorman,
Vlastimil Babka
On Thu, 19 Jan 2017 14:07:13 -0800 Dan Williams <dan.j.williams@intel.com> wrote:
> Prepare for hot{plug,remove} of sub-ranges of a section by tracking a
> section active bitmask, each bit representing 2MB (SECTION_SIZE (128M) /
> map_active bitmask length (64)).
>
> --- a/include/linux/mmzone.h
> +++ b/include/linux/mmzone.h
> @@ -1083,6 +1083,8 @@ struct mem_section_usage {
> unsigned long pageblock_flags[0];
> };
>
> +void section_active_init(unsigned long pfn, unsigned long nr_pages);
> +
> struct page;
> struct page_ext;
> struct mem_section {
> @@ -1224,6 +1226,7 @@ void sparse_init(void);
> #else
> #define sparse_init() do {} while (0)
> #define sparse_index_init(_sec, _nid) do {} while (0)
> +#define section_active_init(_pfn, _nr_pages) do {} while (0)
Using a #define for this is crappy. A static inline does typechecking
and can suppress unused-var warnings in callers.
> #endif /* CONFIG_SPARSEMEM */
>
> /*
> diff --git a/mm/page_alloc.c b/mm/page_alloc.c
> index 68ccf5bcdbb2..9a3ab6c245a8 100644
> --- a/mm/page_alloc.c
> +++ b/mm/page_alloc.c
> @@ -6352,10 +6352,12 @@ void __init free_area_init_nodes(unsigned long *max_zone_pfn)
>
> /* Print out the early node map */
> pr_info("Early memory node ranges\n");
> - for_each_mem_pfn_range(i, MAX_NUMNODES, &start_pfn, &end_pfn, &nid)
> + for_each_mem_pfn_range(i, MAX_NUMNODES, &start_pfn, &end_pfn, &nid) {
> pr_info(" node %3d: [mem %#018Lx-%#018Lx]\n", nid,
> (u64)start_pfn << PAGE_SHIFT,
> ((u64)end_pfn << PAGE_SHIFT) - 1);
> + section_active_init(start_pfn, end_pfn - start_pfn);
section_active_init() can be __init, methinks. We don't want to carry
the extra .text after boot.
--- a/include/linux/mmzone.h~mm-track-active-portions-of-a-section-at-boot-fix
+++ a/include/linux/mmzone.h
@@ -1083,7 +1083,7 @@ struct mem_section_usage {
unsigned long pageblock_flags[0];
};
-void section_active_init(unsigned long pfn, unsigned long nr_pages);
+void __init section_active_init(unsigned long pfn, unsigned long nr_pages);
struct page;
struct page_ext;
@@ -1226,6 +1226,10 @@ void sparse_init(void);
#else
#define sparse_init() do {} while (0)
#define sparse_index_init(_sec, _nid) do {} while (0)
+static inline void section_active_init(unsigned long pfn,
+ unsigned long nr_pages)
+{
+}
#define section_active_init(_pfn, _nr_pages) do {} while (0)
#endif /* CONFIG_SPARSEMEM */
--- a/mm/sparse.c~mm-track-active-portions-of-a-section-at-boot-fix
+++ a/mm/sparse.c
@@ -168,13 +168,13 @@ void __meminit mminit_validate_memmodel_
}
}
-static int section_active_index(phys_addr_t phys)
+static int __init section_active_index(phys_addr_t phys)
{
return (phys & ~(PA_SECTION_MASK)) / SECTION_ACTIVE_SIZE;
}
-static unsigned long section_active_mask(unsigned long pfn,
- unsigned long nr_pages)
+static unsigned long __init section_active_mask(unsigned long pfn,
+ unsigned long nr_pages)
{
int idx_start, idx_size;
phys_addr_t start, size;
@@ -195,7 +195,7 @@ static unsigned long section_active_mask
return ((1UL << idx_size) - 1) << idx_start;
}
-void section_active_init(unsigned long pfn, unsigned long nr_pages)
+void __init section_active_init(unsigned long pfn, unsigned long nr_pages)
{
int end_sec = pfn_to_section_nr(pfn + nr_pages - 1);
int i, start_sec = pfn_to_section_nr(pfn);
_
^ permalink raw reply [flat|nested] 15+ messages in thread* Re: [PATCH v3 06/12] mm: track active portions of a section at boot
2017-01-20 0:05 ` Andrew Morton
@ 2017-01-20 17:56 ` Dan Williams
0 siblings, 0 replies; 15+ messages in thread
From: Dan Williams @ 2017-01-20 17:56 UTC (permalink / raw)
To: Andrew Morton
Cc: Michal Hocko, linux-nvdimm, Logan Gunthorpe, linux-kernel,
Stephen Bates, Linux MM, Johannes Weiner, Mel Gorman,
Vlastimil Babka
On Thu, Jan 19, 2017 at 4:05 PM, Andrew Morton
<akpm@linux-foundation.org> wrote:
> On Thu, 19 Jan 2017 14:07:13 -0800 Dan Williams <dan.j.williams@intel.com> wrote:
>
>> Prepare for hot{plug,remove} of sub-ranges of a section by tracking a
>> section active bitmask, each bit representing 2MB (SECTION_SIZE (128M) /
>> map_active bitmask length (64)).
>>
>> --- a/include/linux/mmzone.h
>> +++ b/include/linux/mmzone.h
>> @@ -1083,6 +1083,8 @@ struct mem_section_usage {
>> unsigned long pageblock_flags[0];
>> };
>>
>> +void section_active_init(unsigned long pfn, unsigned long nr_pages);
>> +
>> struct page;
>> struct page_ext;
>> struct mem_section {
>> @@ -1224,6 +1226,7 @@ void sparse_init(void);
>> #else
>> #define sparse_init() do {} while (0)
>> #define sparse_index_init(_sec, _nid) do {} while (0)
>> +#define section_active_init(_pfn, _nr_pages) do {} while (0)
>
> Using a #define for this is crappy. A static inline does typechecking
> and can suppress unused-var warnings in callers.
>
>> #endif /* CONFIG_SPARSEMEM */
>>
>> /*
>> diff --git a/mm/page_alloc.c b/mm/page_alloc.c
>> index 68ccf5bcdbb2..9a3ab6c245a8 100644
>> --- a/mm/page_alloc.c
>> +++ b/mm/page_alloc.c
>> @@ -6352,10 +6352,12 @@ void __init free_area_init_nodes(unsigned long *max_zone_pfn)
>>
>> /* Print out the early node map */
>> pr_info("Early memory node ranges\n");
>> - for_each_mem_pfn_range(i, MAX_NUMNODES, &start_pfn, &end_pfn, &nid)
>> + for_each_mem_pfn_range(i, MAX_NUMNODES, &start_pfn, &end_pfn, &nid) {
>> pr_info(" node %3d: [mem %#018Lx-%#018Lx]\n", nid,
>> (u64)start_pfn << PAGE_SHIFT,
>> ((u64)end_pfn << PAGE_SHIFT) - 1);
>> + section_active_init(start_pfn, end_pfn - start_pfn);
>
> section_active_init() can be __init, methinks. We don't want to carry
> the extra .text after boot.
>
> --- a/include/linux/mmzone.h~mm-track-active-portions-of-a-section-at-boot-fix
> +++ a/include/linux/mmzone.h
> @@ -1083,7 +1083,7 @@ struct mem_section_usage {
> unsigned long pageblock_flags[0];
> };
>
> -void section_active_init(unsigned long pfn, unsigned long nr_pages);
> +void __init section_active_init(unsigned long pfn, unsigned long nr_pages);
>
> struct page;
> struct page_ext;
> @@ -1226,6 +1226,10 @@ void sparse_init(void);
> #else
> #define sparse_init() do {} while (0)
> #define sparse_index_init(_sec, _nid) do {} while (0)
> +static inline void section_active_init(unsigned long pfn,
> + unsigned long nr_pages)
> +{
> +}
> #define section_active_init(_pfn, _nr_pages) do {} while (0)
> #endif /* CONFIG_SPARSEMEM */
>
> --- a/mm/sparse.c~mm-track-active-portions-of-a-section-at-boot-fix
> +++ a/mm/sparse.c
> @@ -168,13 +168,13 @@ void __meminit mminit_validate_memmodel_
> }
> }
>
> -static int section_active_index(phys_addr_t phys)
> +static int __init section_active_index(phys_addr_t phys)
> {
> return (phys & ~(PA_SECTION_MASK)) / SECTION_ACTIVE_SIZE;
> }
>
> -static unsigned long section_active_mask(unsigned long pfn,
> - unsigned long nr_pages)
> +static unsigned long __init section_active_mask(unsigned long pfn,
> + unsigned long nr_pages)
> {
> int idx_start, idx_size;
> phys_addr_t start, size;
> @@ -195,7 +195,7 @@ static unsigned long section_active_mask
> return ((1UL << idx_size) - 1) << idx_start;
> }
>
> -void section_active_init(unsigned long pfn, unsigned long nr_pages)
> +void __init section_active_init(unsigned long pfn, unsigned long nr_pages)
> {
> int end_sec = pfn_to_section_nr(pfn + nr_pages - 1);
> int i, start_sec = pfn_to_section_nr(pfn);
> _
>
Yes, looks good to me.
^ permalink raw reply [flat|nested] 15+ messages in thread
* [PATCH v3 07/12] mm: fix register_new_memory() zone type detection
2017-01-19 22:06 [PATCH v3 00/12] mm: sub-section memory hotplug support Dan Williams
` (5 preceding siblings ...)
2017-01-19 22:07 ` [PATCH v3 06/12] mm: track active portions of a section at boot Dan Williams
@ 2017-01-19 22:07 ` Dan Williams
2017-01-19 22:07 ` [PATCH v3 08/12] mm: convert kmalloc_section_memmap() to populate_section_memmap() Dan Williams
` (4 subsequent siblings)
11 siblings, 0 replies; 15+ messages in thread
From: Dan Williams @ 2017-01-19 22:07 UTC (permalink / raw)
To: akpm
Cc: Michal Hocko, linux-nvdimm, Logan Gunthorpe, linux-kernel,
Stephen Bates, linux-mm, Johannes Weiner, Mel Gorman,
Vlastimil Babka
In preparation for sub-section memory hotplug support, remove a
dependency on ->section_mem_map being populated. In SPARSEMEM_VMEMMAP=y
configurations pfn_to_page() does not use ->section_mem_map. The
sub-section hotplug support relies on this fact and skips initializing
it. Without ->section_mem_map populated, or aligned to section boundary,
conversions of mem_section instances to zones is not possible.
So, this removes a false dependency on a structure field that will only
be valid in the SPARSEMEM_VMEMMAP=n case, and only used for
pfn_to_page() (and similar) operations.
Cc: Michal Hocko <mhocko@suse.com>
Cc: Vlastimil Babka <vbabka@suse.cz>
Cc: Johannes Weiner <hannes@cmpxchg.org>
Cc: Logan Gunthorpe <logang@deltatee.com>
Cc: Mel Gorman <mgorman@techsingularity.net>
Cc: Andrew Morton <akpm@linux-foundation.org>
Cc: Stephen Bates <stephen.bates@microsemi.com>
Signed-off-by: Dan Williams <dan.j.williams@intel.com>
---
drivers/base/memory.c | 26 +++++++++-----------------
include/linux/memory.h | 4 ++--
mm/memory_hotplug.c | 4 ++--
3 files changed, 13 insertions(+), 21 deletions(-)
diff --git a/drivers/base/memory.c b/drivers/base/memory.c
index 8ab8ea1253e6..be0c4e860188 100644
--- a/drivers/base/memory.c
+++ b/drivers/base/memory.c
@@ -685,24 +685,16 @@ static int add_memory_block(int base_section_nr)
return 0;
}
-static bool is_zone_device_section(struct mem_section *ms)
-{
- struct page *page;
-
- page = sparse_decode_mem_map(ms->section_mem_map, __section_nr(ms));
- return is_zone_device_page(page);
-}
-
/*
* need an interface for the VM to add new memory regions,
* but without onlining it.
*/
-int register_new_memory(int nid, struct mem_section *section)
+int register_new_memory(struct zone *zone, int nid, struct mem_section *section)
{
int ret = 0;
struct memory_block *mem;
- if (is_zone_device_section(section))
+ if (is_dev_zone(zone))
return 0;
mutex_lock(&mem_sysfs_mutex);
@@ -736,14 +728,11 @@ unregister_memory(struct memory_block *memory)
device_unregister(&memory->dev);
}
-static int remove_memory_section(unsigned long node_id,
- struct mem_section *section, int phys_device)
+static int remove_memory_section(struct zone *zone, unsigned long node_id,
+ struct mem_section *section, int phys_device)
{
struct memory_block *mem;
- if (is_zone_device_section(section))
- return 0;
-
mutex_lock(&mem_sysfs_mutex);
mem = find_memory_block(section);
unregister_mem_sect_under_nodes(mem, __section_nr(section));
@@ -758,12 +747,15 @@ static int remove_memory_section(unsigned long node_id,
return 0;
}
-int unregister_memory_section(struct mem_section *section)
+int unregister_memory_section(struct zone *zone, struct mem_section *section)
{
+ if (is_dev_zone(zone))
+ return 0;
+
if (!present_section(section))
return -EINVAL;
- return remove_memory_section(0, section, 0);
+ return remove_memory_section(zone, 0, section, 0);
}
#endif /* CONFIG_MEMORY_HOTREMOVE */
diff --git a/include/linux/memory.h b/include/linux/memory.h
index 093607f90b91..301dfb03ecb7 100644
--- a/include/linux/memory.h
+++ b/include/linux/memory.h
@@ -108,12 +108,12 @@ extern int register_memory_notifier(struct notifier_block *nb);
extern void unregister_memory_notifier(struct notifier_block *nb);
extern int register_memory_isolate_notifier(struct notifier_block *nb);
extern void unregister_memory_isolate_notifier(struct notifier_block *nb);
-extern int register_new_memory(int, struct mem_section *);
+extern int register_new_memory(struct zone *, int, struct mem_section *);
extern int memory_block_change_state(struct memory_block *mem,
unsigned long to_state,
unsigned long from_state_req);
#ifdef CONFIG_MEMORY_HOTREMOVE
-extern int unregister_memory_section(struct mem_section *);
+extern int unregister_memory_section(struct zone *, struct mem_section *);
#endif
extern int memory_dev_init(void);
extern int memory_notify(unsigned long val, void *v);
diff --git a/mm/memory_hotplug.c b/mm/memory_hotplug.c
index a73a1fcaa181..684addad9708 100644
--- a/mm/memory_hotplug.c
+++ b/mm/memory_hotplug.c
@@ -516,7 +516,7 @@ static int __meminit __add_section(int nid, struct zone *zone,
if (ret < 0)
return ret;
- return register_new_memory(nid, __pfn_to_section(phys_start_pfn));
+ return register_new_memory(zone, nid, __pfn_to_section(phys_start_pfn));
}
/*
@@ -783,7 +783,7 @@ static int __remove_section(struct zone *zone, struct mem_section *ms,
if (!valid_section(ms))
return ret;
- ret = unregister_memory_section(ms);
+ ret = unregister_memory_section(zone, ms);
if (ret)
return ret;
^ permalink raw reply related [flat|nested] 15+ messages in thread* [PATCH v3 08/12] mm: convert kmalloc_section_memmap() to populate_section_memmap()
2017-01-19 22:06 [PATCH v3 00/12] mm: sub-section memory hotplug support Dan Williams
` (6 preceding siblings ...)
2017-01-19 22:07 ` [PATCH v3 07/12] mm: fix register_new_memory() zone type detection Dan Williams
@ 2017-01-19 22:07 ` Dan Williams
2017-01-19 22:07 ` [PATCH v3 09/12] mm: prepare for hot-{add, remove} of sub-section ranges Dan Williams
` (3 subsequent siblings)
11 siblings, 0 replies; 15+ messages in thread
From: Dan Williams @ 2017-01-19 22:07 UTC (permalink / raw)
To: akpm
Cc: Michal Hocko, linux-nvdimm, Logan Gunthorpe, linux-kernel,
Stephen Bates, linux-mm, Johannes Weiner, Mel Gorman,
Vlastimil Babka
Allow sub-section sized ranges to be added to the memmap.
populate_section_memmap() takes an explict pfn range rather than
assuming a full section, and those parameters are plumbed all the way
through to vmmemap_populate(). There should be no sub-section in
current code. New warnings are added to clarify which memmap allocation
paths are sub-section capable.
Cc: Michal Hocko <mhocko@suse.com>
Cc: Vlastimil Babka <vbabka@suse.cz>
Cc: Johannes Weiner <hannes@cmpxchg.org>
Cc: Logan Gunthorpe <logang@deltatee.com>
Cc: Mel Gorman <mgorman@techsingularity.net>
Cc: Andrew Morton <akpm@linux-foundation.org>
Cc: Stephen Bates <stephen.bates@microsemi.com>
Signed-off-by: Dan Williams <dan.j.williams@intel.com>
---
arch/x86/mm/init_64.c | 4 ++-
include/linux/mm.h | 3 ++
mm/sparse-vmemmap.c | 24 ++++++++++++++------
mm/sparse.c | 60 ++++++++++++++++++++++++++++++++-----------------
4 files changed, 61 insertions(+), 30 deletions(-)
diff --git a/arch/x86/mm/init_64.c b/arch/x86/mm/init_64.c
index af85b686a7b0..3d719d48f11d 100644
--- a/arch/x86/mm/init_64.c
+++ b/arch/x86/mm/init_64.c
@@ -1220,7 +1220,9 @@ int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node)
struct vmem_altmap *altmap = to_vmem_altmap(start);
int err;
- if (boot_cpu_has(X86_FEATURE_PSE))
+ if (end - start < PAGES_PER_SECTION * sizeof(struct page))
+ err = vmemmap_populate_basepages(start, end, node);
+ else if (boot_cpu_has(X86_FEATURE_PSE))
err = vmemmap_populate_hugepages(start, end, node, altmap);
else if (altmap) {
pr_err_once("%s: no cpu support for altmap allocations\n",
diff --git a/include/linux/mm.h b/include/linux/mm.h
index b84615b0f64c..3a555c420f66 100644
--- a/include/linux/mm.h
+++ b/include/linux/mm.h
@@ -2313,7 +2313,8 @@ void sparse_mem_maps_populate_node(struct page **map_map,
unsigned long map_count,
int nodeid);
-struct page *sparse_mem_map_populate(unsigned long pnum, int nid);
+struct page *__populate_section_memmap(unsigned long pfn,
+ unsigned long nr_pages, int nid);
pgd_t *vmemmap_pgd_populate(unsigned long addr, int node);
pud_t *vmemmap_pud_populate(pgd_t *pgd, unsigned long addr, int node);
pmd_t *vmemmap_pmd_populate(pud_t *pud, unsigned long addr, int node);
diff --git a/mm/sparse-vmemmap.c b/mm/sparse-vmemmap.c
index 574c67b663fe..8679d4a81b98 100644
--- a/mm/sparse-vmemmap.c
+++ b/mm/sparse-vmemmap.c
@@ -248,20 +248,28 @@ int __meminit vmemmap_populate_basepages(unsigned long start,
return 0;
}
-struct page * __meminit sparse_mem_map_populate(unsigned long pnum, int nid)
+struct page * __meminit __populate_section_memmap(unsigned long pfn,
+ unsigned long nr_pages, int nid)
{
unsigned long start;
unsigned long end;
- struct page *map;
- map = pfn_to_page(pnum * PAGES_PER_SECTION);
- start = (unsigned long)map;
- end = (unsigned long)(map + PAGES_PER_SECTION);
+ /*
+ * The minimum granularity of memmap extensions is
+ * SECTION_ACTIVE_SIZE as allocations are tracked in the
+ * 'map_active' bitmap of the section.
+ */
+ end = ALIGN(pfn + nr_pages, PHYS_PFN(SECTION_ACTIVE_SIZE));
+ pfn &= PHYS_PFN(SECTION_ACTIVE_MASK);
+ nr_pages = end - pfn;
+
+ start = (unsigned long) pfn_to_page(pfn);
+ end = start + nr_pages * sizeof(struct page);
if (vmemmap_populate(start, end, nid))
return NULL;
- return map;
+ return pfn_to_page(pfn);
}
void __init sparse_mem_maps_populate_node(struct page **map_map,
@@ -284,11 +292,13 @@ void __init sparse_mem_maps_populate_node(struct page **map_map,
for (pnum = pnum_begin; pnum < pnum_end; pnum++) {
struct mem_section *ms;
+ unsigned long pfn = section_nr_to_pfn(pnum);
if (!present_section_nr(pnum))
continue;
- map_map[pnum] = sparse_mem_map_populate(pnum, nodeid);
+ map_map[pnum] = __populate_section_memmap(pfn,
+ PAGES_PER_SECTION, nodeid);
if (map_map[pnum])
continue;
ms = __nr_to_section(pnum);
diff --git a/mm/sparse.c b/mm/sparse.c
index 00fdb5d04680..97f91770e3d0 100644
--- a/mm/sparse.c
+++ b/mm/sparse.c
@@ -423,7 +423,8 @@ static void __init sparse_early_usemaps_alloc_node(void *data,
}
#ifndef CONFIG_SPARSEMEM_VMEMMAP
-struct page __init *sparse_mem_map_populate(unsigned long pnum, int nid)
+struct page __init *__populate_section_memmap(unsigned long pfn,
+ unsigned long nr_pages, int nid)
{
struct page *map;
unsigned long size;
@@ -475,10 +476,12 @@ void __init sparse_mem_maps_populate_node(struct page **map_map,
/* fallback */
for (pnum = pnum_begin; pnum < pnum_end; pnum++) {
struct mem_section *ms;
+ unsigned long pfn = section_nr_to_pfn(pnum);
if (!present_section_nr(pnum))
continue;
- map_map[pnum] = sparse_mem_map_populate(pnum, nodeid);
+ map_map[pnum] = __populate_section_memmap(pfn,
+ PAGES_PER_SECTION, nodeid);
if (map_map[pnum])
continue;
ms = __nr_to_section(pnum);
@@ -506,7 +509,8 @@ static struct page __init *sparse_early_mem_map_alloc(unsigned long pnum)
struct mem_section *ms = __nr_to_section(pnum);
int nid = sparse_early_nid(ms);
- map = sparse_mem_map_populate(pnum, nid);
+ map = __populate_section_memmap(section_nr_to_pfn(pnum),
+ PAGES_PER_SECTION, nid);
if (map)
return map;
@@ -648,15 +652,16 @@ void __init sparse_init(void)
#ifdef CONFIG_MEMORY_HOTPLUG
#ifdef CONFIG_SPARSEMEM_VMEMMAP
-static inline struct page *kmalloc_section_memmap(unsigned long pnum, int nid)
+static struct page *populate_section_memmap(unsigned long pfn,
+ unsigned long nr_pages, int nid)
{
- /* This will make the necessary allocations eventually. */
- return sparse_mem_map_populate(pnum, nid);
+ return __populate_section_memmap(pfn, nr_pages, nid);
}
-static void __kfree_section_memmap(struct page *memmap)
+
+static void depopulate_section_memmap(unsigned long pfn, unsigned long nr_pages)
{
- unsigned long start = (unsigned long)memmap;
- unsigned long end = (unsigned long)(memmap + PAGES_PER_SECTION);
+ unsigned long start = (unsigned long) pfn_to_page(pfn);
+ unsigned long end = start + nr_pages * sizeof(struct page);
vmemmap_free(start, end);
}
@@ -670,11 +675,18 @@ static void free_map_bootmem(struct page *memmap)
}
#endif /* CONFIG_MEMORY_HOTREMOVE */
#else
-static struct page *__kmalloc_section_memmap(void)
+struct page *populate_section_memmap(unsigned long pfn,
+ unsigned long nr_pages, int nid)
{
struct page *page, *ret;
unsigned long memmap_size = sizeof(struct page) * PAGES_PER_SECTION;
+ if ((pfn & ~PAGE_SECTION_MASK) || nr_pages != PAGES_PER_SECTION) {
+ WARN(1, "%s: called with section unaligned parameters\n",
+ __func__);
+ return NULL;
+ }
+
page = alloc_pages(GFP_KERNEL|__GFP_NOWARN, get_order(memmap_size));
if (page)
goto got_map_page;
@@ -691,13 +703,16 @@ static struct page *__kmalloc_section_memmap(void)
return ret;
}
-static inline struct page *kmalloc_section_memmap(unsigned long pnum, int nid)
+static void depopulate_section_memmap(unsigned long pfn, unsigned long nr_pages)
{
- return __kmalloc_section_memmap();
-}
+ struct page *memmap = pfn_to_page(pfn);
+
+ if ((pfn & ~PAGE_SECTION_MASK) || nr_pages != PAGES_PER_SECTION) {
+ WARN(1, "%s: called with section unaligned parameters\n",
+ __func__);
+ return;
+ }
-static void __kfree_section_memmap(struct page *memmap)
-{
if (is_vmalloc_addr(memmap))
vfree(memmap);
else
@@ -755,12 +770,13 @@ int __meminit sparse_add_one_section(struct zone *zone, unsigned long start_pfn)
ret = sparse_index_init(section_nr, pgdat->node_id);
if (ret < 0 && ret != -EEXIST)
return ret;
- memmap = kmalloc_section_memmap(section_nr, pgdat->node_id);
+ memmap = populate_section_memmap(start_pfn, PAGES_PER_SECTION,
+ pgdat->node_id);
if (!memmap)
return -ENOMEM;
usage = __alloc_section_usage();
if (!usage) {
- __kfree_section_memmap(memmap);
+ depopulate_section_memmap(start_pfn, PAGES_PER_SECTION);
return -ENOMEM;
}
@@ -782,7 +798,7 @@ int __meminit sparse_add_one_section(struct zone *zone, unsigned long start_pfn)
pgdat_resize_unlock(pgdat, &flags);
if (ret < 0 && ret != -EEXIST) {
kfree(usage);
- __kfree_section_memmap(memmap);
+ depopulate_section_memmap(start_pfn, PAGES_PER_SECTION);
return ret;
}
return 0;
@@ -811,7 +827,8 @@ static inline void clear_hwpoisoned_pages(struct page *memmap, int nr_pages)
#endif
static void free_section_usage(struct page *memmap,
- struct mem_section_usage *usage)
+ struct mem_section_usage *usage, unsigned long pfn,
+ unsigned long nr_pages)
{
struct page *usemap_page;
@@ -825,7 +842,7 @@ static void free_section_usage(struct page *memmap,
if (PageSlab(usemap_page) || PageCompound(usemap_page)) {
kfree(usage);
if (memmap)
- __kfree_section_memmap(memmap);
+ depopulate_section_memmap(pfn, nr_pages);
return;
}
@@ -858,7 +875,8 @@ void sparse_remove_one_section(struct zone *zone, struct mem_section *ms,
clear_hwpoisoned_pages(memmap + map_offset,
PAGES_PER_SECTION - map_offset);
- free_section_usage(memmap, usage);
+ free_section_usage(memmap, usage, section_nr_to_pfn(__section_nr(ms)),
+ PAGES_PER_SECTION);
}
#endif /* CONFIG_MEMORY_HOTREMOVE */
#endif /* CONFIG_MEMORY_HOTPLUG */
^ permalink raw reply related [flat|nested] 15+ messages in thread* [PATCH v3 09/12] mm: prepare for hot-{add, remove} of sub-section ranges
2017-01-19 22:06 [PATCH v3 00/12] mm: sub-section memory hotplug support Dan Williams
` (7 preceding siblings ...)
2017-01-19 22:07 ` [PATCH v3 08/12] mm: convert kmalloc_section_memmap() to populate_section_memmap() Dan Williams
@ 2017-01-19 22:07 ` Dan Williams
2017-01-19 22:07 ` [PATCH v3 10/12] mm: support section-unaligned ZONE_DEVICE memory ranges Dan Williams
` (2 subsequent siblings)
11 siblings, 0 replies; 15+ messages in thread
From: Dan Williams @ 2017-01-19 22:07 UTC (permalink / raw)
To: akpm
Cc: Michal Hocko, linux-nvdimm, Logan Gunthorpe, linux-kernel,
Stephen Bates, linux-mm, Johannes Weiner, Mel Gorman,
Vlastimil Babka
Prepare the memory hot-{add,remove} paths for handling sub-section
ranges by plumbing the starting page frame and number of pages being
handled through arch_{add,remove}_memory() to
sparse_{add,remove}_one_section().
This is simply plumbing, small cleanups, and some identifier renames. No
intended functional changes.
Cc: Michal Hocko <mhocko@suse.com>
Cc: Vlastimil Babka <vbabka@suse.cz>
Cc: Johannes Weiner <hannes@cmpxchg.org>
Cc: Logan Gunthorpe <logang@deltatee.com>
Cc: Mel Gorman <mgorman@techsingularity.net>
Cc: Andrew Morton <akpm@linux-foundation.org>
Cc: Stephen Bates <stephen.bates@microsemi.com>
Signed-off-by: Dan Williams <dan.j.williams@intel.com>
---
arch/x86/mm/init_64.c | 11 +++++
include/linux/memory_hotplug.h | 6 ++-
mm/memory_hotplug.c | 85 ++++++++++++++++++++++------------------
mm/sparse.c | 6 ++-
4 files changed, 65 insertions(+), 43 deletions(-)
diff --git a/arch/x86/mm/init_64.c b/arch/x86/mm/init_64.c
index 3d719d48f11d..843512f492e0 100644
--- a/arch/x86/mm/init_64.c
+++ b/arch/x86/mm/init_64.c
@@ -650,6 +650,17 @@ int arch_add_memory(int nid, u64 start, u64 size, bool for_device)
unsigned long nr_pages = size >> PAGE_SHIFT;
int ret;
+ /*
+ * Only allow partial section hotplug for ZONE_DEVICE ranges,
+ * since register_new_memory() requires section alignment, and
+ * CONFIG_SPARSEMEM_VMEMMAP=n requires sections to be fully
+ * populated.
+ */
+ if ((!IS_ENABLED(CONFIG_SPARSEMEM_VMEMMAP) || !for_device)
+ && ((start & ~PA_SECTION_MASK)
+ || (size & ~PA_SECTION_MASK)))
+ return -EINVAL;
+
init_memory_mapping(start, start + size);
ret = __add_pages(nid, zone, start_pfn, nr_pages);
diff --git a/include/linux/memory_hotplug.h b/include/linux/memory_hotplug.h
index 01033fadea47..a6ac3c975d5d 100644
--- a/include/linux/memory_hotplug.h
+++ b/include/linux/memory_hotplug.h
@@ -279,8 +279,10 @@ extern int arch_add_memory(int nid, u64 start, u64 size, bool for_device);
extern int offline_pages(unsigned long start_pfn, unsigned long nr_pages);
extern bool is_memblock_offlined(struct memory_block *mem);
extern void remove_memory(int nid, u64 start, u64 size);
-extern int sparse_add_one_section(struct zone *zone, unsigned long start_pfn);
-extern void sparse_remove_one_section(struct zone *zone, struct mem_section *ms,
+extern int sparse_add_section(struct zone *zone, unsigned long pfn,
+ unsigned long nr_pages);
+extern void sparse_remove_section(struct zone *zone, struct mem_section *ms,
+ unsigned long pfn, unsigned long nr_pages,
unsigned long map_offset);
extern struct page *sparse_decode_mem_map(unsigned long coded_mem_map,
unsigned long pnum);
diff --git a/mm/memory_hotplug.c b/mm/memory_hotplug.c
index 684addad9708..a203a795a46c 100644
--- a/mm/memory_hotplug.c
+++ b/mm/memory_hotplug.c
@@ -466,10 +466,10 @@ static void __meminit grow_pgdat_span(struct pglist_data *pgdat, unsigned long s
pgdat->node_start_pfn;
}
-static int __meminit __add_zone(struct zone *zone, unsigned long phys_start_pfn)
+static int __meminit __add_zone(struct zone *zone, unsigned long phys_start_pfn,
+ unsigned long nr_pages)
{
struct pglist_data *pgdat = zone->zone_pgdat;
- int nr_pages = PAGES_PER_SECTION;
int nid = pgdat->node_id;
int zone_type;
unsigned long flags, pfn;
@@ -499,24 +499,21 @@ static int __meminit __add_zone(struct zone *zone, unsigned long phys_start_pfn)
}
static int __meminit __add_section(int nid, struct zone *zone,
- unsigned long phys_start_pfn)
+ unsigned long pfn, unsigned long nr_pages)
{
int ret;
- if (pfn_valid(phys_start_pfn))
- return -EEXIST;
-
- ret = sparse_add_one_section(zone, phys_start_pfn);
+ ret = sparse_add_section(zone, pfn, nr_pages);
if (ret < 0)
return ret;
- ret = __add_zone(zone, phys_start_pfn);
+ ret = __add_zone(zone, pfn, nr_pages);
if (ret < 0)
return ret;
- return register_new_memory(zone, nid, __pfn_to_section(phys_start_pfn));
+ return register_new_memory(zone, nid, __pfn_to_section(pfn));
}
/*
@@ -525,7 +522,7 @@ static int __meminit __add_section(int nid, struct zone *zone,
* call this function after deciding the zone to which to
* add the new pages.
*/
-int __ref __add_pages(int nid, struct zone *zone, unsigned long phys_start_pfn,
+int __ref __add_pages(int nid, struct zone *zone, unsigned long pfn,
unsigned long nr_pages)
{
int err = 0, i, start_sec, end_sec;
@@ -533,16 +530,12 @@ int __ref __add_pages(int nid, struct zone *zone, unsigned long phys_start_pfn,
clear_zone_contiguous(zone);
- /* during initialize mem_map, align hot-added range to section */
- start_sec = pfn_to_section_nr(phys_start_pfn);
- end_sec = pfn_to_section_nr(phys_start_pfn + nr_pages - 1);
-
- altmap = to_vmem_altmap((unsigned long) pfn_to_page(phys_start_pfn));
+ altmap = to_vmem_altmap((unsigned long) pfn_to_page(pfn));
if (altmap) {
/*
* Validate altmap is within bounds of the total request
*/
- if (altmap->base_pfn != phys_start_pfn
+ if (altmap->base_pfn != pfn
|| vmem_altmap_offset(altmap) > nr_pages) {
pr_warn_once("memory add fail, invalid altmap\n");
err = -EINVAL;
@@ -551,8 +544,16 @@ int __ref __add_pages(int nid, struct zone *zone, unsigned long phys_start_pfn,
altmap->alloc = 0;
}
+ start_sec = pfn_to_section_nr(pfn);
+ end_sec = pfn_to_section_nr(pfn + nr_pages - 1);
for (i = start_sec; i <= end_sec; i++) {
- err = __add_section(nid, zone, section_nr_to_pfn(i));
+ unsigned long pfns;
+
+ pfns = min(nr_pages, PAGES_PER_SECTION
+ - (pfn & ~PAGE_SECTION_MASK));
+ err = __add_section(nid, zone, pfn, pfns);
+ pfn += pfns;
+ nr_pages -= pfns;
/*
* EEXIST is finally dealt with by ioresource collision
@@ -758,10 +759,10 @@ static void shrink_pgdat_span(struct pglist_data *pgdat,
pgdat->node_spanned_pages = 0;
}
-static void __remove_zone(struct zone *zone, unsigned long start_pfn)
+static void __remove_zone(struct zone *zone, unsigned long start_pfn,
+ unsigned long nr_pages)
{
struct pglist_data *pgdat = zone->zone_pgdat;
- int nr_pages = PAGES_PER_SECTION;
int zone_type;
unsigned long flags;
@@ -773,11 +774,10 @@ static void __remove_zone(struct zone *zone, unsigned long start_pfn)
pgdat_resize_unlock(zone->zone_pgdat, &flags);
}
-static int __remove_section(struct zone *zone, struct mem_section *ms,
- unsigned long map_offset)
+static int __remove_section(struct zone *zone, unsigned long pfn,
+ unsigned long nr_pages, unsigned long map_offset)
{
- unsigned long start_pfn;
- int scn_nr;
+ struct mem_section *ms = __nr_to_section(pfn_to_section_nr(pfn));
int ret = -EINVAL;
if (!valid_section(ms))
@@ -787,11 +787,9 @@ static int __remove_section(struct zone *zone, struct mem_section *ms,
if (ret)
return ret;
- scn_nr = __section_nr(ms);
- start_pfn = section_nr_to_pfn(scn_nr);
- __remove_zone(zone, start_pfn);
+ __remove_zone(zone, pfn, nr_pages);
- sparse_remove_one_section(zone, ms, map_offset);
+ sparse_remove_section(zone, ms, pfn, nr_pages, map_offset);
return 0;
}
@@ -806,16 +804,15 @@ static int __remove_section(struct zone *zone, struct mem_section *ms,
* sure that pages are marked reserved and zones are adjust properly by
* calling offline_pages().
*/
-int __remove_pages(struct zone *zone, unsigned long phys_start_pfn,
+int __remove_pages(struct zone *zone, unsigned long pfn,
unsigned long nr_pages)
{
- unsigned long i;
unsigned long map_offset = 0;
- int sections_to_remove, ret = 0;
+ int i, start_sec, end_sec, ret = 0;
/* In the ZONE_DEVICE case device driver owns the memory region */
if (is_dev_zone(zone)) {
- struct page *page = pfn_to_page(phys_start_pfn);
+ struct page *page = pfn_to_page(pfn);
struct vmem_altmap *altmap;
altmap = to_vmem_altmap((unsigned long) page);
@@ -824,7 +821,7 @@ int __remove_pages(struct zone *zone, unsigned long phys_start_pfn,
} else {
resource_size_t start, size;
- start = phys_start_pfn << PAGE_SHIFT;
+ start = pfn << PAGE_SHIFT;
size = nr_pages * PAGE_SIZE;
ret = release_mem_region_adjustable(&iomem_resource, start,
@@ -840,16 +837,26 @@ int __remove_pages(struct zone *zone, unsigned long phys_start_pfn,
clear_zone_contiguous(zone);
/*
- * We can only remove entire sections
+ * Only ZONE_DEVICE memory is enabled to remove
+ * section-unaligned ranges. See register_new_memory() which
+ * assumes section alignment and is skipped for ZONE_DEVICE
+ * ranges.
*/
- BUG_ON(phys_start_pfn & ~PAGE_SECTION_MASK);
- BUG_ON(nr_pages % PAGES_PER_SECTION);
+ if (!is_dev_zone(zone) && ((pfn | nr_pages) & ~PAGE_SECTION_MASK)) {
+ WARN(1, "section unaligned removal not supported\n");
+ return -EINVAL;
+ }
- sections_to_remove = nr_pages / PAGES_PER_SECTION;
- for (i = 0; i < sections_to_remove; i++) {
- unsigned long pfn = phys_start_pfn + i*PAGES_PER_SECTION;
+ start_sec = pfn_to_section_nr(pfn);
+ end_sec = pfn_to_section_nr(pfn + nr_pages - 1);
+ for (i = start_sec; i <= end_sec; i++) {
+ unsigned long pfns;
- ret = __remove_section(zone, __pfn_to_section(pfn), map_offset);
+ pfns = min(nr_pages, PAGES_PER_SECTION
+ - (pfn & ~PAGE_SECTION_MASK));
+ ret = __remove_section(zone, pfn, pfns, map_offset);
+ pfn += pfns;
+ nr_pages -= pfns;
map_offset = 0;
if (ret)
break;
diff --git a/mm/sparse.c b/mm/sparse.c
index 97f91770e3d0..a8358d15a90d 100644
--- a/mm/sparse.c
+++ b/mm/sparse.c
@@ -753,7 +753,8 @@ static void free_map_bootmem(struct page *memmap)
#endif /* CONFIG_MEMORY_HOTREMOVE */
#endif /* CONFIG_SPARSEMEM_VMEMMAP */
-int __meminit sparse_add_one_section(struct zone *zone, unsigned long start_pfn)
+int __meminit sparse_add_section(struct zone *zone, unsigned long start_pfn,
+ unsigned long nr_pages)
{
unsigned long section_nr = pfn_to_section_nr(start_pfn);
struct pglist_data *pgdat = zone->zone_pgdat;
@@ -855,7 +856,8 @@ static void free_section_usage(struct page *memmap,
free_map_bootmem(memmap);
}
-void sparse_remove_one_section(struct zone *zone, struct mem_section *ms,
+void sparse_remove_section(struct zone *zone, struct mem_section *ms,
+ unsigned long pfn, unsigned long nr_pages,
unsigned long map_offset)
{
unsigned long flags;
^ permalink raw reply related [flat|nested] 15+ messages in thread* [PATCH v3 10/12] mm: support section-unaligned ZONE_DEVICE memory ranges
2017-01-19 22:06 [PATCH v3 00/12] mm: sub-section memory hotplug support Dan Williams
` (8 preceding siblings ...)
2017-01-19 22:07 ` [PATCH v3 09/12] mm: prepare for hot-{add, remove} of sub-section ranges Dan Williams
@ 2017-01-19 22:07 ` Dan Williams
2017-01-19 22:07 ` [PATCH v3 11/12] mm: enable section-unaligned devm_memremap_pages() Dan Williams
2017-01-19 22:07 ` [PATCH v3 12/12] libnvdimm, pfn, dax: stop padding pmem namespaces to section alignment Dan Williams
11 siblings, 0 replies; 15+ messages in thread
From: Dan Williams @ 2017-01-19 22:07 UTC (permalink / raw)
To: akpm
Cc: Michal Hocko, linux-nvdimm, Logan Gunthorpe, linux-kernel,
Stephen Bates, linux-mm, Johannes Weiner, Mel Gorman,
Vlastimil Babka
The initial motivation for this change is persistent memory platforms
that, unfortunately, align the pmem range on a boundary less than a full
section (64M vs 128M), and may change the alignment from one boot to the
next. A secondary motivation is the arrival of prospective ZONE_DEVICE
users that want devm_memremap_pages() to map PCI-E device memory ranges
to enable peer-to-peer DMA.
Currently the nvdimm core injects padding when 'pfn' (struct page
mapping configuration) instances are created. However, not all users of
devm_memremap_pages() have the opportunity to inject such padding. Users
of the memmap=ss!nn kernel command line option can trigger the following
failure with unaligned parameters like "memmap=0xfc000000!8G":
WARNING: CPU: 0 PID: 558 at kernel/memremap.c:300 devm_memremap_pages+0x3b5/0x4c0
devm_memremap_pages attempted on mixed region [mem 0x200000000-0x2fbffffff flags 0x200]
[..]
Call Trace:
[<ffffffff814c0393>] dump_stack+0x86/0xc3
[<ffffffff810b173b>] __warn+0xcb/0xf0
[<ffffffff810b17bf>] warn_slowpath_fmt+0x5f/0x80
[<ffffffff811eb105>] devm_memremap_pages+0x3b5/0x4c0
[<ffffffffa006f308>] __wrap_devm_memremap_pages+0x58/0x70 [nfit_test_iomap]
[<ffffffffa00e231a>] pmem_attach_disk+0x19a/0x440 [nd_pmem]
Without this change a user could inadvertently lose access to nvdimm
namespaces by adding/removing other DIMMs in the platform leading to the
BIOS changing the base alignment of the namespace in an incompatible
fashion. With this support we can accommodate a BIOS changing the
namespace to any alignment provided it is >= SECTION_ACTIVE_SIZE.
Cc: Michal Hocko <mhocko@suse.com>
Cc: Vlastimil Babka <vbabka@suse.cz>
Cc: Johannes Weiner <hannes@cmpxchg.org>
Cc: Logan Gunthorpe <logang@deltatee.com>
Cc: Mel Gorman <mgorman@techsingularity.net>
Cc: Andrew Morton <akpm@linux-foundation.org>
Cc: Stephen Bates <stephen.bates@microsemi.com>
Signed-off-by: Dan Williams <dan.j.williams@intel.com>
---
mm/sparse.c | 272 ++++++++++++++++++++++++++++++++++++++++++++---------------
1 file changed, 204 insertions(+), 68 deletions(-)
diff --git a/mm/sparse.c b/mm/sparse.c
index a8358d15a90d..c89b3bbebce0 100644
--- a/mm/sparse.c
+++ b/mm/sparse.c
@@ -24,6 +24,7 @@
#ifdef CONFIG_SPARSEMEM_EXTREME
struct mem_section *mem_section[NR_SECTION_ROOTS]
____cacheline_internodealigned_in_smp;
+static DEFINE_SPINLOCK(mem_section_lock); /* atomically instantiate new entries */
#else
struct mem_section mem_section[NR_SECTION_ROOTS][SECTIONS_PER_ROOT]
____cacheline_internodealigned_in_smp;
@@ -89,7 +90,22 @@ static int __meminit sparse_index_init(unsigned long section_nr, int nid)
if (!section)
return -ENOMEM;
- mem_section[root] = section;
+ spin_lock(&mem_section_lock);
+ if (mem_section[root] == NULL) {
+ mem_section[root] = section;
+ section = NULL;
+ }
+ spin_unlock(&mem_section_lock);
+
+ /*
+ * The only time we expect adding a section may race is during
+ * post-meminit hotplug. So, there is no expectation that 'section'
+ * leaks in the !slab_is_available() case.
+ */
+ if (section && slab_is_available()) {
+ kfree(section);
+ return -EEXIST;
+ }
return 0;
}
@@ -288,6 +304,15 @@ static void __meminit sparse_init_one_section(struct mem_section *ms,
unsigned long pnum, struct page *mem_map,
struct mem_section_usage *usage)
{
+ /*
+ * Given that SPARSEMEM_VMEMMAP=y supports sub-section hotplug,
+ * ->section_mem_map can not be guaranteed to point to a full
+ * section's worth of memory. The field is only valid / used
+ * in the SPARSEMEM_VMEMMAP=n case.
+ */
+ if (IS_ENABLED(CONFIG_SPARSEMEM_VMEMMAP))
+ mem_map = NULL;
+
ms->section_mem_map &= ~SECTION_MAP_MASK;
ms->section_mem_map |= sparse_encode_mem_map(mem_map, pnum) |
SECTION_HAS_MEM_MAP;
@@ -753,12 +778,176 @@ static void free_map_bootmem(struct page *memmap)
#endif /* CONFIG_MEMORY_HOTREMOVE */
#endif /* CONFIG_SPARSEMEM_VMEMMAP */
+static bool is_early_section(struct mem_section *ms)
+{
+ struct page *usemap_page;
+
+ usemap_page = virt_to_page(ms->usage->pageblock_flags);
+ if (PageSlab(usemap_page) || PageCompound(usemap_page))
+ return false;
+ else
+ return true;
+
+}
+
+#ifndef CONFIG_MEMORY_HOTREMOVE
+static void free_map_bootmem(struct page *memmap)
+{
+}
+#endif
+
+static void section_deactivate(struct pglist_data *pgdat, unsigned long pfn,
+ unsigned long nr_pages)
+{
+ bool early_section;
+ struct page *memmap = NULL;
+ struct mem_section_usage *usage = NULL;
+ int section_nr = pfn_to_section_nr(pfn);
+ struct mem_section *ms = __nr_to_section(section_nr);
+ unsigned long mask = section_active_mask(pfn, nr_pages), flags;
+
+ pgdat_resize_lock(pgdat, &flags);
+ if (!ms->usage) {
+ mask = 0;
+ } else if ((ms->usage->map_active & mask) != mask) {
+ WARN(1, "section already deactivated active: %#lx mask: %#lx\n",
+ ms->usage->map_active, mask);
+ mask = 0;
+ } else {
+ early_section = is_early_section(ms);
+ ms->usage->map_active ^= mask;
+ if (ms->usage->map_active == 0) {
+ usage = ms->usage;
+ ms->usage = NULL;
+ memmap = sparse_decode_mem_map(ms->section_mem_map,
+ section_nr);
+ ms->section_mem_map = 0;
+ }
+ }
+ pgdat_resize_unlock(pgdat, &flags);
+
+ /*
+ * There are 3 cases to handle across two configurations
+ * (SPARSEMEM_VMEMMAP={y,n}):
+ *
+ * 1/ deactivation of a partial hot-added section (only possible
+ * in the SPARSEMEM_VMEMMAP=y case).
+ * a/ section was present at memory init
+ * b/ section was hot-added post memory init
+ * 2/ deactivation of a complete hot-added section
+ * 3/ deactivation of a complete section from memory init
+ *
+ * For 1/, when map_active does not go to zero we will not be
+ * freeing the usage map, but still need to free the vmemmap
+ * range.
+ *
+ * For 2/ and 3/ the SPARSEMEM_VMEMMAP={y,n} cases are unified
+ */
+ if (!mask)
+ return;
+ if (nr_pages < PAGES_PER_SECTION) {
+ if (!IS_ENABLED(CONFIG_SPARSEMEM_VMEMMAP)) {
+ WARN(1, "partial memory section removal not supported\n");
+ return;
+ }
+ if (!early_section)
+ depopulate_section_memmap(pfn, nr_pages);
+ memmap = 0;
+ }
+
+ if (usage) {
+ if (!early_section) {
+ /*
+ * 'memmap' may be zero in the SPARSEMEM_VMEMMAP=y case
+ * (see sparse_init_one_section()), so we can't rely on
+ * it to determine if we need to depopulate the memmap.
+ * Instead, we uncoditionally depopulate due to 'usage'
+ * being valid.
+ */
+ if (memmap || (nr_pages >= PAGES_PER_SECTION
+ && IS_ENABLED(CONFIG_SPARSEMEM_VMEMMAP)))
+ depopulate_section_memmap(pfn, nr_pages);
+ kfree(usage);
+ return;
+ }
+ }
+
+ /*
+ * The usemap came from bootmem. This is packed with other usemaps
+ * on the section which has pgdat at boot time. Just keep it as is now.
+ */
+ if (memmap)
+ free_map_bootmem(memmap);
+}
+
+static struct page * __meminit section_activate(struct pglist_data *pgdat,
+ unsigned long pfn, unsigned nr_pages)
+{
+ struct mem_section *ms = __nr_to_section(pfn_to_section_nr(pfn));
+ unsigned long mask = section_active_mask(pfn, nr_pages), flags;
+ struct mem_section_usage *usage;
+ bool early_section = false;
+ struct page *memmap;
+ int rc = 0;
+
+ usage = __alloc_section_usage();
+ if (!usage)
+ return ERR_PTR(-ENOMEM);
+
+ pgdat_resize_lock(pgdat, &flags);
+ if (!ms->usage) {
+ ms->usage = usage;
+ usage = NULL;
+ } else
+ early_section = is_early_section(ms);
+
+ if (!mask)
+ rc = -EINVAL;
+ else if (mask & ms->usage->map_active)
+ rc = -EBUSY;
+ else
+ ms->usage->map_active |= mask;
+ pgdat_resize_unlock(pgdat, &flags);
+
+ kfree(usage);
+
+ if (rc)
+ return ERR_PTR(rc);
+
+
+ /*
+ * The early init code does not consider partially populated
+ * initial sections, it simply assumes that memory will never be
+ * referenced. If we hot-add memory into such a section then we
+ * do not need to populate the memmap and can simply reuse what
+ * is already there.
+ */
+ if (nr_pages < PAGES_PER_SECTION && early_section)
+ return pfn_to_page(pfn);
+
+ memmap = populate_section_memmap(pfn, nr_pages, pgdat->node_id);
+ if (!memmap) {
+ section_deactivate(pgdat, pfn, nr_pages);
+ return ERR_PTR(-ENOMEM);
+ }
+
+ return memmap;
+}
+
+/**
+ * sparse_add_section() - create a new memmap section, or populate an
+ * existing one
+ * @zone: host zone for the new memory mapping
+ * @start_pfn: first pfn to add (section aligned if zone != ZONE_DEVICE)
+ * @nr_pages: number of new pages to add
+ *
+ * Returns 0 on success.
+ */
int __meminit sparse_add_section(struct zone *zone, unsigned long start_pfn,
unsigned long nr_pages)
{
unsigned long section_nr = pfn_to_section_nr(start_pfn);
struct pglist_data *pgdat = zone->zone_pgdat;
- static struct mem_section_usage *usage;
struct mem_section *ms;
struct page *memmap;
unsigned long flags;
@@ -771,37 +960,27 @@ int __meminit sparse_add_section(struct zone *zone, unsigned long start_pfn,
ret = sparse_index_init(section_nr, pgdat->node_id);
if (ret < 0 && ret != -EEXIST)
return ret;
- memmap = populate_section_memmap(start_pfn, PAGES_PER_SECTION,
- pgdat->node_id);
- if (!memmap)
- return -ENOMEM;
- usage = __alloc_section_usage();
- if (!usage) {
- depopulate_section_memmap(start_pfn, PAGES_PER_SECTION);
- return -ENOMEM;
- }
- pgdat_resize_lock(pgdat, &flags);
+ memmap = section_activate(pgdat, start_pfn, nr_pages);
+ if (IS_ERR(memmap))
+ return PTR_ERR(memmap);
+ pgdat_resize_lock(pgdat, &flags);
ms = __pfn_to_section(start_pfn);
- if (ms->section_mem_map & SECTION_MARKED_PRESENT) {
+ if (nr_pages == PAGES_PER_SECTION && (ms->section_mem_map
+ & SECTION_MARKED_PRESENT)) {
ret = -EBUSY;
goto out;
}
-
- memset(memmap, 0, sizeof(struct page) * PAGES_PER_SECTION);
-
ms->section_mem_map |= SECTION_MARKED_PRESENT;
-
- sparse_init_one_section(ms, section_nr, memmap, usage);
-
+ sparse_init_one_section(ms, section_nr, memmap, ms->usage);
out:
pgdat_resize_unlock(pgdat, &flags);
- if (ret < 0 && ret != -EEXIST) {
- kfree(usage);
- depopulate_section_memmap(start_pfn, PAGES_PER_SECTION);
+ if (nr_pages == PAGES_PER_SECTION && ret < 0 && ret != -EEXIST) {
+ section_deactivate(pgdat, start_pfn, nr_pages);
return ret;
}
+ memset(memmap, 0, sizeof(struct page) * nr_pages);
return 0;
}
@@ -827,58 +1006,15 @@ static inline void clear_hwpoisoned_pages(struct page *memmap, int nr_pages)
}
#endif
-static void free_section_usage(struct page *memmap,
- struct mem_section_usage *usage, unsigned long pfn,
- unsigned long nr_pages)
-{
- struct page *usemap_page;
-
- if (!usage)
- return;
-
- usemap_page = virt_to_page(usage->pageblock_flags);
- /*
- * Check to see if allocation came from hot-plug-add
- */
- if (PageSlab(usemap_page) || PageCompound(usemap_page)) {
- kfree(usage);
- if (memmap)
- depopulate_section_memmap(pfn, nr_pages);
- return;
- }
-
- /*
- * The usemap came from bootmem. This is packed with other usemaps
- * on the section which has pgdat at boot time. Just keep it as is now.
- */
-
- if (memmap)
- free_map_bootmem(memmap);
-}
-
void sparse_remove_section(struct zone *zone, struct mem_section *ms,
unsigned long pfn, unsigned long nr_pages,
unsigned long map_offset)
{
- unsigned long flags;
- struct page *memmap = NULL;
- struct mem_section_usage *usage = NULL;
struct pglist_data *pgdat = zone->zone_pgdat;
- pgdat_resize_lock(pgdat, &flags);
- if (ms->section_mem_map) {
- usage = ms->usage;
- memmap = sparse_decode_mem_map(ms->section_mem_map,
- __section_nr(ms));
- ms->section_mem_map = 0;
- ms->usage = NULL;
- }
- pgdat_resize_unlock(pgdat, &flags);
-
- clear_hwpoisoned_pages(memmap + map_offset,
- PAGES_PER_SECTION - map_offset);
- free_section_usage(memmap, usage, section_nr_to_pfn(__section_nr(ms)),
- PAGES_PER_SECTION);
+ clear_hwpoisoned_pages(pfn_to_page(pfn) + map_offset,
+ nr_pages - map_offset);
+ section_deactivate(pgdat, pfn, nr_pages);
}
#endif /* CONFIG_MEMORY_HOTREMOVE */
#endif /* CONFIG_MEMORY_HOTPLUG */
^ permalink raw reply related [flat|nested] 15+ messages in thread* [PATCH v3 11/12] mm: enable section-unaligned devm_memremap_pages()
2017-01-19 22:06 [PATCH v3 00/12] mm: sub-section memory hotplug support Dan Williams
` (9 preceding siblings ...)
2017-01-19 22:07 ` [PATCH v3 10/12] mm: support section-unaligned ZONE_DEVICE memory ranges Dan Williams
@ 2017-01-19 22:07 ` Dan Williams
2017-01-19 22:07 ` [PATCH v3 12/12] libnvdimm, pfn, dax: stop padding pmem namespaces to section alignment Dan Williams
11 siblings, 0 replies; 15+ messages in thread
From: Dan Williams @ 2017-01-19 22:07 UTC (permalink / raw)
To: akpm
Cc: Michal Hocko, Toshi Kani, linux-nvdimm, linux-kernel,
Stephen Bates, linux-mm, Logan Gunthorpe
Teach devm_memremap_pages() about the new sub-section capabilities of
arch_{add,remove}_memory().
Cc: Michal Hocko <mhocko@suse.com>
Cc: Toshi Kani <toshi.kani@hpe.com>
Cc: Andrew Morton <akpm@linux-foundation.org>
Cc: Logan Gunthorpe <logang@deltatee.com>
Cc: Stephen Bates <stephen.bates@microsemi.com>
Signed-off-by: Dan Williams <dan.j.williams@intel.com>
---
kernel/memremap.c | 22 +++++++---------------
1 file changed, 7 insertions(+), 15 deletions(-)
diff --git a/kernel/memremap.c b/kernel/memremap.c
index e58d7828ff9a..e6476a8e8b6a 100644
--- a/kernel/memremap.c
+++ b/kernel/memremap.c
@@ -256,7 +256,6 @@ static void devm_memremap_pages_release(struct device *dev, void *data)
{
struct page_map *page_map = data;
struct resource *res = &page_map->res;
- resource_size_t align_start, align_size;
struct dev_pagemap *pgmap = &page_map->pgmap;
if (percpu_ref_tryget_live(pgmap->ref)) {
@@ -265,12 +264,10 @@ static void devm_memremap_pages_release(struct device *dev, void *data)
}
/* pages are dead and unused, undo the arch mapping */
- align_start = res->start & PA_SECTION_MASK;
- align_size = ALIGN(resource_size(res), PA_SECTION_SIZE);
mem_hotplug_begin();
- arch_remove_memory(align_start, align_size);
+ arch_remove_memory(res->start, resource_size(res));
mem_hotplug_done();
- untrack_pfn(NULL, PHYS_PFN(align_start), align_size);
+ untrack_pfn(NULL, PHYS_PFN(res->start), resource_size(res));
pgmap_radix_release(res);
dev_WARN_ONCE(dev, pgmap->altmap && pgmap->altmap->alloc,
"%s: failed to free all reserved pages\n", __func__);
@@ -305,17 +302,13 @@ struct dev_pagemap *find_dev_pagemap(resource_size_t phys)
void *devm_memremap_pages(struct device *dev, struct resource *res,
struct percpu_ref *ref, struct vmem_altmap *altmap)
{
- resource_size_t align_start, align_size, align_end;
unsigned long pfn, pgoff, order;
pgprot_t pgprot = PAGE_KERNEL;
struct dev_pagemap *pgmap;
struct page_map *page_map;
int error, nid, is_ram;
- align_start = res->start & PA_SECTION_MASK;
- align_size = ALIGN(res->start + resource_size(res), PA_SECTION_SIZE)
- - align_start;
- is_ram = region_intersects(align_start, align_size,
+ is_ram = region_intersects(res->start, resource_size(res),
IORESOURCE_SYSTEM_RAM, IORES_DESC_NONE);
if (is_ram == REGION_MIXED) {
@@ -348,7 +341,6 @@ void *devm_memremap_pages(struct device *dev, struct resource *res,
mutex_lock(&pgmap_lock);
error = 0;
- align_end = align_start + align_size - 1;
foreach_order_pgoff(res, order, pgoff) {
struct dev_pagemap *dup;
@@ -377,13 +369,13 @@ void *devm_memremap_pages(struct device *dev, struct resource *res,
if (nid < 0)
nid = numa_mem_id();
- error = track_pfn_remap(NULL, &pgprot, PHYS_PFN(align_start), 0,
- align_size);
+ error = track_pfn_remap(NULL, &pgprot, PHYS_PFN(res->start), 0,
+ resource_size(res));
if (error)
goto err_pfn_remap;
mem_hotplug_begin();
- error = arch_add_memory(nid, align_start, align_size, true);
+ error = arch_add_memory(nid, res->start, resource_size(res), true);
mem_hotplug_done();
if (error)
goto err_add_memory;
@@ -404,7 +396,7 @@ void *devm_memremap_pages(struct device *dev, struct resource *res,
return __va(res->start);
err_add_memory:
- untrack_pfn(NULL, PHYS_PFN(align_start), align_size);
+ untrack_pfn(NULL, PHYS_PFN(res->start), resource_size(res));
err_pfn_remap:
err_radix:
pgmap_radix_release(res);
^ permalink raw reply related [flat|nested] 15+ messages in thread* [PATCH v3 12/12] libnvdimm, pfn, dax: stop padding pmem namespaces to section alignment
2017-01-19 22:06 [PATCH v3 00/12] mm: sub-section memory hotplug support Dan Williams
` (10 preceding siblings ...)
2017-01-19 22:07 ` [PATCH v3 11/12] mm: enable section-unaligned devm_memremap_pages() Dan Williams
@ 2017-01-19 22:07 ` Dan Williams
11 siblings, 0 replies; 15+ messages in thread
From: Dan Williams @ 2017-01-19 22:07 UTC (permalink / raw)
To: akpm; +Cc: linux-mm, linux-kernel, Toshi Kani, linux-nvdimm
Now that the mm core supports section-unaligned hotplug of ZONE_DEVICE
memory, we no longer need to add padding at pfn/dax device creation
time. The kernel will still honor padding established by older kernels.
Cc: Toshi Kani <toshi.kani@hpe.com>
Signed-off-by: Dan Williams <dan.j.williams@intel.com>
---
drivers/nvdimm/pfn_devs.c | 42 +++++++-----------------------------------
1 file changed, 7 insertions(+), 35 deletions(-)
diff --git a/drivers/nvdimm/pfn_devs.c b/drivers/nvdimm/pfn_devs.c
index a2ac9e641aa9..5d17710f7402 100644
--- a/drivers/nvdimm/pfn_devs.c
+++ b/drivers/nvdimm/pfn_devs.c
@@ -538,7 +538,7 @@ static struct vmem_altmap *__nvdimm_setup_pfn(struct nd_pfn *nd_pfn,
nd_pfn->npfns = le64_to_cpu(pfn_sb->npfns);
altmap = NULL;
} else if (nd_pfn->mode == PFN_MODE_PMEM) {
- nd_pfn->npfns = (resource_size(res) - offset) / PAGE_SIZE;
+ nd_pfn->npfns = PHYS_PFN((resource_size(res) - offset));
if (le64_to_cpu(nd_pfn->pfn_sb->npfns) > nd_pfn->npfns)
dev_info(&nd_pfn->dev,
"number of pfns truncated from %lld to %ld\n",
@@ -557,7 +557,6 @@ static int nd_pfn_init(struct nd_pfn *nd_pfn)
{
u32 dax_label_reserve = is_nd_dax(&nd_pfn->dev) ? SZ_128K : 0;
struct nd_namespace_common *ndns = nd_pfn->ndns;
- u32 start_pad = 0, end_trunc = 0;
resource_size_t start, size;
struct nd_namespace_io *nsio;
struct nd_region *nd_region;
@@ -590,42 +589,16 @@ static int nd_pfn_init(struct nd_pfn *nd_pfn)
return -ENXIO;
}
- memset(pfn_sb, 0, sizeof(*pfn_sb));
-
- /*
- * Check if pmem collides with 'System RAM' when section aligned and
- * trim it accordingly
- */
- nsio = to_nd_namespace_io(&ndns->dev);
- start = PHYS_SECTION_ALIGN_DOWN(nsio->res.start);
- size = resource_size(&nsio->res);
- if (region_intersects(start, size, IORESOURCE_SYSTEM_RAM,
- IORES_DESC_NONE) == REGION_MIXED) {
- start = nsio->res.start;
- start_pad = PHYS_SECTION_ALIGN_UP(start) - start;
- }
-
- start = nsio->res.start;
- size = PHYS_SECTION_ALIGN_UP(start + size) - start;
- if (region_intersects(start, size, IORESOURCE_SYSTEM_RAM,
- IORES_DESC_NONE) == REGION_MIXED) {
- size = resource_size(&nsio->res);
- end_trunc = start + size - PHYS_SECTION_ALIGN_DOWN(start + size);
- }
-
- if (start_pad + end_trunc)
- dev_info(&nd_pfn->dev, "%s section collision, truncate %d bytes\n",
- dev_name(&ndns->dev), start_pad + end_trunc);
-
/*
* Note, we use 64 here for the standard size of struct page,
* debugging options may cause it to be larger in which case the
* implementation will limit the pfns advertised through
* ->direct_access() to those that are included in the memmap.
*/
- start += start_pad;
+ nsio = to_nd_namespace_io(&ndns->dev);
+ start = nsio->res.start;
size = resource_size(&nsio->res);
- npfns = (size - start_pad - end_trunc - SZ_8K) / SZ_4K;
+ npfns = PHYS_PFN(size - SZ_8K);
if (nd_pfn->mode == PFN_MODE_PMEM) {
unsigned long memmap_size;
@@ -642,13 +615,14 @@ static int nd_pfn_init(struct nd_pfn *nd_pfn)
else
return -ENXIO;
- if (offset + start_pad + end_trunc >= size) {
+ if (offset >= size) {
dev_err(&nd_pfn->dev, "%s unable to satisfy requested alignment\n",
dev_name(&ndns->dev));
return -ENXIO;
}
- npfns = (size - offset - start_pad - end_trunc) / SZ_4K;
+ memset(pfn_sb, 0, sizeof(*pfn_sb));
+ npfns = PHYS_PFN(size - offset);
pfn_sb->mode = cpu_to_le32(nd_pfn->mode);
pfn_sb->dataoff = cpu_to_le64(offset);
pfn_sb->npfns = cpu_to_le64(npfns);
@@ -657,8 +631,6 @@ static int nd_pfn_init(struct nd_pfn *nd_pfn)
memcpy(pfn_sb->parent_uuid, nd_dev_to_uuid(&ndns->dev), 16);
pfn_sb->version_major = cpu_to_le16(1);
pfn_sb->version_minor = cpu_to_le16(2);
- pfn_sb->start_pad = cpu_to_le32(start_pad);
- pfn_sb->end_trunc = cpu_to_le32(end_trunc);
pfn_sb->align = cpu_to_le32(nd_pfn->align);
checksum = nd_sb_checksum((struct nd_gen_sb *) pfn_sb);
pfn_sb->checksum = cpu_to_le64(checksum);
^ permalink raw reply related [flat|nested] 15+ messages in thread