* [PATCH] mm, page_alloc: convert page_group_by_mobility_disable to static key
@ 2016-12-20 13:43 Vlastimil Babka
2016-12-29 12:00 ` Mel Gorman
2016-12-29 16:08 ` Rik van Riel
0 siblings, 2 replies; 3+ messages in thread
From: Vlastimil Babka @ 2016-12-20 13:43 UTC (permalink / raw)
To: Andrew Morton, Mel Gorman
Cc: linux-kernel, linux-mm, Joonsoo Kim, Kirill A. Shutemov,
Johannes Weiner, Rik van Riel, Peter Zijlstra, Vlastimil Babka
The flag is rarely enabled or even changed, so it's an ideal static key
candidate. Since it's being checked in the page allocator fastpath via
gfpflags_to_migratetype(), it may actually save some valuable cycles.
Here's a diff excerpt from __alloc_pages_nodemask() assembly:
-movl page_group_by_mobility_disabled(%rip), %ecx
+.byte 0x0f,0x1f,0x44,0x00,0
movl %r9d, %eax
shrl $3, %eax
andl $3, %eax
-testl %ecx, %ecx
-movl $0, %ecx
-cmovne %ecx, %eax
I.e. a NOP instead of test, conditional move and some assisting moves.
Signed-off-by: Vlastimil Babka <vbabka@suse.cz>
---
include/linux/gfp.h | 2 +-
include/linux/mmzone.h | 3 ++-
mm/page_alloc.c | 23 +++++++++++++----------
3 files changed, 16 insertions(+), 12 deletions(-)
diff --git a/include/linux/gfp.h b/include/linux/gfp.h
index f8041f9de31e..097609342608 100644
--- a/include/linux/gfp.h
+++ b/include/linux/gfp.h
@@ -270,7 +270,7 @@ static inline int gfpflags_to_migratetype(const gfp_t gfp_flags)
BUILD_BUG_ON((1UL << GFP_MOVABLE_SHIFT) != ___GFP_MOVABLE);
BUILD_BUG_ON((___GFP_MOVABLE >> GFP_MOVABLE_SHIFT) != MIGRATE_MOVABLE);
- if (unlikely(page_group_by_mobility_disabled))
+ if (static_branch_unlikely(&page_group_by_mobility_disabled))
return MIGRATE_UNMOVABLE;
/* Group based on mobility */
diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h
index 0f088f3a2fed..d1d440cff60e 100644
--- a/include/linux/mmzone.h
+++ b/include/linux/mmzone.h
@@ -17,6 +17,7 @@
#include <linux/pageblock-flags.h>
#include <linux/page-flags-layout.h>
#include <linux/atomic.h>
+#include <linux/jump_label.h>
#include <asm/page.h>
/* Free memory management - zoned buddy allocator. */
@@ -78,7 +79,7 @@ extern char * const migratetype_names[MIGRATE_TYPES];
for (order = 0; order < MAX_ORDER; order++) \
for (type = 0; type < MIGRATE_TYPES; type++)
-extern int page_group_by_mobility_disabled;
+extern struct static_key_false page_group_by_mobility_disabled;
#define NR_MIGRATETYPE_BITS (PB_migrate_end - PB_migrate + 1)
#define MIGRATETYPE_MASK ((1UL << NR_MIGRATETYPE_BITS) - 1)
diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index 6de9440e3ae2..655153ef8f2c 100644
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -281,7 +281,7 @@ EXPORT_SYMBOL(nr_node_ids);
EXPORT_SYMBOL(nr_online_nodes);
#endif
-int page_group_by_mobility_disabled __read_mostly;
+DEFINE_STATIC_KEY_FALSE(page_group_by_mobility_disabled);
#ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT
static inline void reset_deferred_meminit(pg_data_t *pgdat)
@@ -450,9 +450,10 @@ void set_pfnblock_flags_mask(struct page *page, unsigned long flags,
void set_pageblock_migratetype(struct page *page, int migratetype)
{
- if (unlikely(page_group_by_mobility_disabled &&
- migratetype < MIGRATE_PCPTYPES))
- migratetype = MIGRATE_UNMOVABLE;
+ if (static_branch_unlikely(&page_group_by_mobility_disabled)) {
+ if (migratetype < MIGRATE_PCPTYPES)
+ migratetype = MIGRATE_UNMOVABLE;
+ }
set_pageblock_flags_group(page, (unsigned long)migratetype,
PB_migrate, PB_migrate_end);
@@ -1945,8 +1946,10 @@ static bool can_steal_fallback(unsigned int order, int start_mt)
if (order >= pageblock_order / 2 ||
start_mt == MIGRATE_RECLAIMABLE ||
- start_mt == MIGRATE_UNMOVABLE ||
- page_group_by_mobility_disabled)
+ start_mt == MIGRATE_UNMOVABLE)
+ return true;
+
+ if (static_branch_unlikely(&page_group_by_mobility_disabled))
return true;
return false;
@@ -1975,7 +1978,7 @@ static void steal_suitable_fallback(struct zone *zone, struct page *page,
/* Claim the whole block if over half of it is free */
if (pages >= (1 << (pageblock_order-1)) ||
- page_group_by_mobility_disabled)
+ static_branch_unlikely(&page_group_by_mobility_disabled))
set_pageblock_migratetype(page, start_type);
}
@@ -4964,14 +4967,14 @@ void __ref build_all_zonelists(pg_data_t *pgdat, struct zone *zone)
* disabled and enable it later
*/
if (vm_total_pages < (pageblock_nr_pages * MIGRATE_TYPES))
- page_group_by_mobility_disabled = 1;
+ static_branch_enable(&page_group_by_mobility_disabled);
else
- page_group_by_mobility_disabled = 0;
+ static_branch_disable(&page_group_by_mobility_disabled);
pr_info("Built %i zonelists in %s order, mobility grouping %s. Total pages: %ld\n",
nr_online_nodes,
zonelist_order_name[current_zonelist_order],
- page_group_by_mobility_disabled ? "off" : "on",
+ static_key_enabled(&page_group_by_mobility_disabled) ? "off" : "on",
vm_total_pages);
#ifdef CONFIG_NUMA
pr_info("Policy zone: %s\n", zone_names[policy_zone]);
--
2.11.0
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>
^ permalink raw reply related [flat|nested] 3+ messages in thread
* Re: [PATCH] mm, page_alloc: convert page_group_by_mobility_disable to static key
2016-12-20 13:43 [PATCH] mm, page_alloc: convert page_group_by_mobility_disable to static key Vlastimil Babka
@ 2016-12-29 12:00 ` Mel Gorman
2016-12-29 16:08 ` Rik van Riel
1 sibling, 0 replies; 3+ messages in thread
From: Mel Gorman @ 2016-12-29 12:00 UTC (permalink / raw)
To: Vlastimil Babka
Cc: Andrew Morton, linux-kernel, linux-mm, Joonsoo Kim,
Kirill A. Shutemov, Johannes Weiner, Rik van Riel,
Peter Zijlstra
On Tue, Dec 20, 2016 at 02:43:12PM +0100, Vlastimil Babka wrote:
> The flag is rarely enabled or even changed, so it's an ideal static key
> candidate. Since it's being checked in the page allocator fastpath via
> gfpflags_to_migratetype(), it may actually save some valuable cycles.
>
> Here's a diff excerpt from __alloc_pages_nodemask() assembly:
>
> -movl page_group_by_mobility_disabled(%rip), %ecx
> +.byte 0x0f,0x1f,0x44,0x00,0
> movl %r9d, %eax
> shrl $3, %eax
> andl $3, %eax
> -testl %ecx, %ecx
> -movl $0, %ecx
> -cmovne %ecx, %eax
>
> I.e. a NOP instead of test, conditional move and some assisting moves.
>
> Signed-off-by: Vlastimil Babka <vbabka@suse.cz>
Acked-by: Mel Gorman <mgorman@techsingularity.net>
--
Mel Gorman
SUSE Labs
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>
^ permalink raw reply [flat|nested] 3+ messages in thread
* Re: [PATCH] mm, page_alloc: convert page_group_by_mobility_disable to static key
2016-12-20 13:43 [PATCH] mm, page_alloc: convert page_group_by_mobility_disable to static key Vlastimil Babka
2016-12-29 12:00 ` Mel Gorman
@ 2016-12-29 16:08 ` Rik van Riel
1 sibling, 0 replies; 3+ messages in thread
From: Rik van Riel @ 2016-12-29 16:08 UTC (permalink / raw)
To: Vlastimil Babka, Andrew Morton, Mel Gorman
Cc: linux-kernel, linux-mm, Joonsoo Kim, Kirill A. Shutemov,
Johannes Weiner, Peter Zijlstra
[-- Attachment #1: Type: text/plain, Size: 902 bytes --]
On Tue, 2016-12-20 at 14:43 +0100, Vlastimil Babka wrote:
> The flag is rarely enabled or even changed, so it's an ideal static
> key
> candidate. Since it's being checked in the page allocator fastpath
> via
> gfpflags_to_migratetype(), it may actually save some valuable cycles.
>
> Here's a diff excerpt from __alloc_pages_nodemask() assembly:
>
> -movl page_group_by_mobility_disabled(%rip), %ecx
> +.byte 0x0f,0x1f,0x44,0x00,0
> movl %r9d, %eax
> shrl $3, %eax
> andl $3, %eax
> -testl %ecx, %ecx
> -movl $0, %ecx
> -cmovne %ecx, %eax
>
> I.e. a NOP instead of test, conditional move and some assisting
> moves.
>
> Signed-off-by: Vlastimil Babka <vbabka@suse.cz>
Acked-by: Rik van Riel <riel@redhat.com>
--
All Rights Reversed.
[-- Attachment #2: This is a digitally signed message part --]
[-- Type: application/pgp-signature, Size: 473 bytes --]
^ permalink raw reply [flat|nested] 3+ messages in thread
end of thread, other threads:[~2016-12-29 16:08 UTC | newest]
Thread overview: 3+ messages (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2016-12-20 13:43 [PATCH] mm, page_alloc: convert page_group_by_mobility_disable to static key Vlastimil Babka
2016-12-29 12:00 ` Mel Gorman
2016-12-29 16:08 ` Rik van Riel
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).