From: Matthew Wilcox <willy@infradead.org>
To: linux-mm@kvack.org
Cc: "Matthew Wilcox" <mawilcox@microsoft.com>,
"Andrew Morton" <akpm@linux-foundation.org>,
"Kirill A . Shutemov" <kirill.shutemov@linux.intel.com>,
"Christoph Lameter" <cl@linux.com>,
"Lai Jiangshan" <jiangshanlai@gmail.com>,
"Pekka Enberg" <penberg@kernel.org>,
"Vlastimil Babka" <vbabka@suse.cz>,
"Dave Hansen" <dave.hansen@linux.intel.com>,
"Jérôme Glisse" <jglisse@redhat.com>
Subject: [PATCH v6 05/17] mm: Move 'private' union within struct page
Date: Fri, 18 May 2018 12:45:07 -0700 [thread overview]
Message-ID: <20180518194519.3820-6-willy@infradead.org> (raw)
In-Reply-To: <20180518194519.3820-1-willy@infradead.org>
From: Matthew Wilcox <mawilcox@microsoft.com>
By moving page->private to the fourth word of struct page, we can put
the SLUB counters in the same word as SLAB's s_mem and still do the
cmpxchg_double trick. Now the SLUB counters no longer overlap with the
mapcount or refcount so we can drop the call to page_mapcount_reset()
and simplify set_page_slub_counters() to a single line.
Signed-off-by: Matthew Wilcox <mawilcox@microsoft.com>
Acked-by: Vlastimil Babka <vbabka@suse.cz>
Acked-by: Kirill A. Shutemov <kirill.shutemov@linux.intel.com>
---
include/linux/mm_types.h | 56 ++++++++++++++++++----------------------
mm/slub.c | 20 ++------------
2 files changed, 27 insertions(+), 49 deletions(-)
diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h
index e97a310a6abe..23378a789af4 100644
--- a/include/linux/mm_types.h
+++ b/include/linux/mm_types.h
@@ -65,15 +65,9 @@ struct hmm;
*/
#ifdef CONFIG_HAVE_ALIGNED_STRUCT_PAGE
#define _struct_page_alignment __aligned(2 * sizeof(unsigned long))
-#if defined(CONFIG_HAVE_CMPXCHG_DOUBLE)
-#define _slub_counter_t unsigned long
#else
-#define _slub_counter_t unsigned int
-#endif
-#else /* !CONFIG_HAVE_ALIGNED_STRUCT_PAGE */
#define _struct_page_alignment
-#define _slub_counter_t unsigned int
-#endif /* !CONFIG_HAVE_ALIGNED_STRUCT_PAGE */
+#endif
struct page {
/* First double word block */
@@ -95,6 +89,30 @@ struct page {
/* page_deferred_list().prev -- second tail page */
};
+ union {
+ /*
+ * Mapping-private opaque data:
+ * Usually used for buffer_heads if PagePrivate
+ * Used for swp_entry_t if PageSwapCache
+ * Indicates order in the buddy system if PageBuddy
+ */
+ unsigned long private;
+#if USE_SPLIT_PTE_PTLOCKS
+#if ALLOC_SPLIT_PTLOCKS
+ spinlock_t *ptl;
+#else
+ spinlock_t ptl;
+#endif
+#endif
+ void *s_mem; /* slab first object */
+ unsigned long counters; /* SLUB */
+ struct { /* SLUB */
+ unsigned inuse:16;
+ unsigned objects:15;
+ unsigned frozen:1;
+ };
+ };
+
union {
/*
* If the page is neither PageSlab nor mappable to userspace,
@@ -104,13 +122,7 @@ struct page {
*/
unsigned int page_type;
- _slub_counter_t counters;
unsigned int active; /* SLAB */
- struct { /* SLUB */
- unsigned inuse:16;
- unsigned objects:15;
- unsigned frozen:1;
- };
int units; /* SLOB */
struct { /* Page cache */
@@ -179,24 +191,6 @@ struct page {
#endif
};
- union {
- /*
- * Mapping-private opaque data:
- * Usually used for buffer_heads if PagePrivate
- * Used for swp_entry_t if PageSwapCache
- * Indicates order in the buddy system if PageBuddy
- */
- unsigned long private;
-#if USE_SPLIT_PTE_PTLOCKS
-#if ALLOC_SPLIT_PTLOCKS
- spinlock_t *ptl;
-#else
- spinlock_t ptl;
-#endif
-#endif
- void *s_mem; /* slab first object */
- };
-
#ifdef CONFIG_MEMCG
struct mem_cgroup *mem_cgroup;
#endif
diff --git a/mm/slub.c b/mm/slub.c
index 7fc13c46e975..05ca612a5fe6 100644
--- a/mm/slub.c
+++ b/mm/slub.c
@@ -356,21 +356,6 @@ static __always_inline void slab_unlock(struct page *page)
__bit_spin_unlock(PG_locked, &page->flags);
}
-static inline void set_page_slub_counters(struct page *page, unsigned long counters_new)
-{
- struct page tmp;
- tmp.counters = counters_new;
- /*
- * page->counters can cover frozen/inuse/objects as well
- * as page->_refcount. If we assign to ->counters directly
- * we run the risk of losing updates to page->_refcount, so
- * be careful and only assign to the fields we need.
- */
- page->frozen = tmp.frozen;
- page->inuse = tmp.inuse;
- page->objects = tmp.objects;
-}
-
/* Interrupts must be disabled (for the fallback code to work right) */
static inline bool __cmpxchg_double_slab(struct kmem_cache *s, struct page *page,
void *freelist_old, unsigned long counters_old,
@@ -392,7 +377,7 @@ static inline bool __cmpxchg_double_slab(struct kmem_cache *s, struct page *page
if (page->freelist == freelist_old &&
page->counters == counters_old) {
page->freelist = freelist_new;
- set_page_slub_counters(page, counters_new);
+ page->counters = counters_new;
slab_unlock(page);
return true;
}
@@ -431,7 +416,7 @@ static inline bool cmpxchg_double_slab(struct kmem_cache *s, struct page *page,
if (page->freelist == freelist_old &&
page->counters == counters_old) {
page->freelist = freelist_new;
- set_page_slub_counters(page, counters_new);
+ page->counters = counters_new;
slab_unlock(page);
local_irq_restore(flags);
return true;
@@ -1689,7 +1674,6 @@ static void __free_slab(struct kmem_cache *s, struct page *page)
__ClearPageSlabPfmemalloc(page);
__ClearPageSlab(page);
- page_mapcount_reset(page);
page->mapping = NULL;
if (current->reclaim_state)
current->reclaim_state->reclaimed_slab += pages;
--
2.17.0
next prev parent reply other threads:[~2018-05-18 19:45 UTC|newest]
Thread overview: 32+ messages / expand[flat|nested] mbox.gz Atom feed top
2018-05-18 19:45 [PATCH v6 00/17] Rearrange struct page Matthew Wilcox
2018-05-18 19:45 ` [PATCH v6 01/17] s390: Use _refcount for pgtables Matthew Wilcox
2018-05-18 19:45 ` [PATCH v6 02/17] mm: Split page_type out from _mapcount Matthew Wilcox
2018-05-18 19:45 ` [PATCH v6 03/17] mm: Mark pages in use for page tables Matthew Wilcox
2018-05-18 19:45 ` [PATCH v6 04/17] mm: Switch s_mem and slab_cache in struct page Matthew Wilcox
2018-05-18 19:45 ` Matthew Wilcox [this message]
2018-05-18 19:45 ` [PATCH v6 06/17] mm: Move _refcount out of struct page union Matthew Wilcox
2018-05-18 19:45 ` [PATCH v6 07/17] mm: Combine first three unions in struct page Matthew Wilcox
2018-05-18 19:45 ` [PATCH v6 08/17] mm: Use page->deferred_list Matthew Wilcox
2018-05-18 19:45 ` [PATCH v6 09/17] mm: Move lru union within struct page Matthew Wilcox
2018-05-18 19:45 ` [PATCH v6 10/17] mm: Combine LRU and main union in " Matthew Wilcox
2018-05-18 19:45 ` [PATCH v6 11/17] mm: Improve struct page documentation Matthew Wilcox
2018-05-18 19:45 ` [PATCH v6 12/17] mm: Add pt_mm to struct page Matthew Wilcox
2018-05-18 19:45 ` [PATCH v6 13/17] mm: Add hmm_data " Matthew Wilcox
2018-05-18 19:45 ` [PATCH v6 14/17] slab,slub: Remove rcu_head size checks Matthew Wilcox
2018-05-18 19:45 ` [PATCH v6 15/17] slub: Remove kmem_cache->reserved Matthew Wilcox
2018-05-18 19:45 ` [PATCH v6 16/17] slub: Remove 'reserved' file from sysfs Matthew Wilcox
2018-05-18 19:45 ` [PATCH v6 17/17] mm: Distinguish VMalloc pages Matthew Wilcox
2018-05-22 16:10 ` Andrey Ryabinin
2018-05-22 17:58 ` Matthew Wilcox
2018-05-22 19:49 ` Andrew Morton
2018-05-22 19:57 ` Andrey Ryabinin
2018-05-22 20:19 ` Matthew Wilcox
2018-05-22 20:48 ` Andrew Morton
2018-05-22 21:45 ` Matthew Wilcox
2018-05-22 23:02 ` Andrew Morton
2018-05-23 6:36 ` Michal Hocko
2018-05-23 6:34 ` Michal Hocko
2018-05-23 9:14 ` Andrey Ryabinin
2018-05-23 9:25 ` Michal Hocko
2018-05-23 9:28 ` Andrey Ryabinin
2018-05-23 9:52 ` Michal Hocko
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20180518194519.3820-6-willy@infradead.org \
--to=willy@infradead.org \
--cc=akpm@linux-foundation.org \
--cc=cl@linux.com \
--cc=dave.hansen@linux.intel.com \
--cc=jglisse@redhat.com \
--cc=jiangshanlai@gmail.com \
--cc=kirill.shutemov@linux.intel.com \
--cc=linux-mm@kvack.org \
--cc=mawilcox@microsoft.com \
--cc=penberg@kernel.org \
--cc=vbabka@suse.cz \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).