* Common [01/20] [slob] Define page struct fields used in mm_types.h
2012-06-13 15:24 Common [00/20] Sl[auo]b: Common code rework V5 (for merge) Christoph Lameter
@ 2012-06-13 15:24 ` Christoph Lameter
2012-06-13 15:24 ` Common [03/20] [slob] Remove various small accessors Christoph Lameter
` (16 subsequent siblings)
17 siblings, 0 replies; 40+ messages in thread
From: Christoph Lameter @ 2012-06-13 15:24 UTC (permalink / raw)
To: Pekka Enberg
Cc: linux-mm, David Rientjes, Matt Mackall, Glauber Costa, Joonsoo Kim
[-- Attachment #1: slob_use_page_struct --]
[-- Type: text/plain, Size: 8697 bytes --]
Define the fields used by slob in mm_types.h and use struct page instead
of struct slob_page in slob. This cleans up numerous of typecasts in slob.c and
makes readers aware of slob's use of page struct fields.
[Also cleans up some bitrot in slob.c. The page struct field layout
in slob.c is an old layout and does not match the one in mm_types.h]
Reviewed-by: Glauber Costa <glommer@parallels.com>
Acked-by: David Rientjes <rientjes@google.com>
Reviewed-by: Joonsoo Kim <js1304@gmail.com>
Signed-off-by: Christoph Lameter <cl@linux.com>
---
include/linux/mm_types.h | 7 ++-
mm/slob.c | 95 ++++++++++++++++++-----------------------------
2 files changed, 42 insertions(+), 60 deletions(-)
Index: linux-2.6/mm/slob.c
===================================================================
--- linux-2.6.orig/mm/slob.c 2012-06-13 03:44:15.757477975 -0500
+++ linux-2.6/mm/slob.c 2012-06-13 03:46:12.173475563 -0500
@@ -92,33 +92,12 @@ struct slob_block {
typedef struct slob_block slob_t;
/*
- * We use struct page fields to manage some slob allocation aspects,
- * however to avoid the horrible mess in include/linux/mm_types.h, we'll
- * just define our own struct page type variant here.
- */
-struct slob_page {
- union {
- struct {
- unsigned long flags; /* mandatory */
- atomic_t _count; /* mandatory */
- slobidx_t units; /* free units left in page */
- unsigned long pad[2];
- slob_t *free; /* first free slob_t in page */
- struct list_head list; /* linked list of free pages */
- };
- struct page page;
- };
-};
-static inline void struct_slob_page_wrong_size(void)
-{ BUILD_BUG_ON(sizeof(struct slob_page) != sizeof(struct page)); }
-
-/*
* free_slob_page: call before a slob_page is returned to the page allocator.
*/
-static inline void free_slob_page(struct slob_page *sp)
+static inline void free_slob_page(struct page *sp)
{
- reset_page_mapcount(&sp->page);
- sp->page.mapping = NULL;
+ reset_page_mapcount(sp);
+ sp->mapping = NULL;
}
/*
@@ -133,44 +112,44 @@ static LIST_HEAD(free_slob_large);
/*
* is_slob_page: True for all slob pages (false for bigblock pages)
*/
-static inline int is_slob_page(struct slob_page *sp)
+static inline int is_slob_page(struct page *sp)
{
- return PageSlab((struct page *)sp);
+ return PageSlab(sp);
}
-static inline void set_slob_page(struct slob_page *sp)
+static inline void set_slob_page(struct page *sp)
{
- __SetPageSlab((struct page *)sp);
+ __SetPageSlab(sp);
}
-static inline void clear_slob_page(struct slob_page *sp)
+static inline void clear_slob_page(struct page *sp)
{
- __ClearPageSlab((struct page *)sp);
+ __ClearPageSlab(sp);
}
-static inline struct slob_page *slob_page(const void *addr)
+static inline struct page *slob_page(const void *addr)
{
- return (struct slob_page *)virt_to_page(addr);
+ return virt_to_page(addr);
}
/*
* slob_page_free: true for pages on free_slob_pages list.
*/
-static inline int slob_page_free(struct slob_page *sp)
+static inline int slob_page_free(struct page *sp)
{
- return PageSlobFree((struct page *)sp);
+ return PageSlobFree(sp);
}
-static void set_slob_page_free(struct slob_page *sp, struct list_head *list)
+static void set_slob_page_free(struct page *sp, struct list_head *list)
{
list_add(&sp->list, list);
- __SetPageSlobFree((struct page *)sp);
+ __SetPageSlobFree(sp);
}
-static inline void clear_slob_page_free(struct slob_page *sp)
+static inline void clear_slob_page_free(struct page *sp)
{
list_del(&sp->list);
- __ClearPageSlobFree((struct page *)sp);
+ __ClearPageSlobFree(sp);
}
#define SLOB_UNIT sizeof(slob_t)
@@ -267,12 +246,12 @@ static void slob_free_pages(void *b, int
/*
* Allocate a slob block within a given slob_page sp.
*/
-static void *slob_page_alloc(struct slob_page *sp, size_t size, int align)
+static void *slob_page_alloc(struct page *sp, size_t size, int align)
{
slob_t *prev, *cur, *aligned = NULL;
int delta = 0, units = SLOB_UNITS(size);
- for (prev = NULL, cur = sp->free; ; prev = cur, cur = slob_next(cur)) {
+ for (prev = NULL, cur = sp->freelist; ; prev = cur, cur = slob_next(cur)) {
slobidx_t avail = slob_units(cur);
if (align) {
@@ -296,12 +275,12 @@ static void *slob_page_alloc(struct slob
if (prev)
set_slob(prev, slob_units(prev), next);
else
- sp->free = next;
+ sp->freelist = next;
} else { /* fragment */
if (prev)
set_slob(prev, slob_units(prev), cur + units);
else
- sp->free = cur + units;
+ sp->freelist = cur + units;
set_slob(cur + units, avail - units, next);
}
@@ -320,7 +299,7 @@ static void *slob_page_alloc(struct slob
*/
static void *slob_alloc(size_t size, gfp_t gfp, int align, int node)
{
- struct slob_page *sp;
+ struct page *sp;
struct list_head *prev;
struct list_head *slob_list;
slob_t *b = NULL;
@@ -341,7 +320,7 @@ static void *slob_alloc(size_t size, gfp
* If there's a node specification, search for a partial
* page with a matching node id in the freelist.
*/
- if (node != -1 && page_to_nid(&sp->page) != node)
+ if (node != -1 && page_to_nid(sp) != node)
continue;
#endif
/* Enough room on this page? */
@@ -374,7 +353,7 @@ static void *slob_alloc(size_t size, gfp
spin_lock_irqsave(&slob_lock, flags);
sp->units = SLOB_UNITS(PAGE_SIZE);
- sp->free = b;
+ sp->freelist = b;
INIT_LIST_HEAD(&sp->list);
set_slob(b, SLOB_UNITS(PAGE_SIZE), b + SLOB_UNITS(PAGE_SIZE));
set_slob_page_free(sp, slob_list);
@@ -392,7 +371,7 @@ static void *slob_alloc(size_t size, gfp
*/
static void slob_free(void *block, int size)
{
- struct slob_page *sp;
+ struct page *sp;
slob_t *prev, *next, *b = (slob_t *)block;
slobidx_t units;
unsigned long flags;
@@ -421,7 +400,7 @@ static void slob_free(void *block, int s
if (!slob_page_free(sp)) {
/* This slob page is about to become partially free. Easy! */
sp->units = units;
- sp->free = b;
+ sp->freelist = b;
set_slob(b, units,
(void *)((unsigned long)(b +
SLOB_UNITS(PAGE_SIZE)) & PAGE_MASK));
@@ -441,15 +420,15 @@ static void slob_free(void *block, int s
*/
sp->units += units;
- if (b < sp->free) {
- if (b + units == sp->free) {
- units += slob_units(sp->free);
- sp->free = slob_next(sp->free);
+ if (b < (slob_t *)sp->freelist) {
+ if (b + units == sp->freelist) {
+ units += slob_units(sp->freelist);
+ sp->freelist = slob_next(sp->freelist);
}
- set_slob(b, units, sp->free);
- sp->free = b;
+ set_slob(b, units, sp->freelist);
+ sp->freelist = b;
} else {
- prev = sp->free;
+ prev = sp->freelist;
next = slob_next(prev);
while (b > next) {
prev = next;
@@ -522,7 +501,7 @@ EXPORT_SYMBOL(__kmalloc_node);
void kfree(const void *block)
{
- struct slob_page *sp;
+ struct page *sp;
trace_kfree(_RET_IP_, block);
@@ -536,14 +515,14 @@ void kfree(const void *block)
unsigned int *m = (unsigned int *)(block - align);
slob_free(m, *m + align);
} else
- put_page(&sp->page);
+ put_page(sp);
}
EXPORT_SYMBOL(kfree);
/* can't use ksize for kmem_cache_alloc memory, only kmalloc */
size_t ksize(const void *block)
{
- struct slob_page *sp;
+ struct page *sp;
BUG_ON(!block);
if (unlikely(block == ZERO_SIZE_PTR))
@@ -555,7 +534,7 @@ size_t ksize(const void *block)
unsigned int *m = (unsigned int *)(block - align);
return SLOB_UNITS(*m) * SLOB_UNIT;
} else
- return sp->page.private;
+ return sp->private;
}
EXPORT_SYMBOL(ksize);
Index: linux-2.6/include/linux/mm_types.h
===================================================================
--- linux-2.6.orig/include/linux/mm_types.h 2012-06-13 03:44:15.733477975 -0500
+++ linux-2.6/include/linux/mm_types.h 2012-06-13 03:46:06.129475688 -0500
@@ -53,7 +53,7 @@ struct page {
struct {
union {
pgoff_t index; /* Our offset within mapping. */
- void *freelist; /* slub first free object */
+ void *freelist; /* slub/slob first free object */
};
union {
@@ -81,11 +81,12 @@ struct page {
*/
atomic_t _mapcount;
- struct {
+ struct { /* SLUB */
unsigned inuse:16;
unsigned objects:15;
unsigned frozen:1;
};
+ int units; /* SLOB */
};
atomic_t _count; /* Usage count, see below. */
};
@@ -107,6 +108,8 @@ struct page {
short int pobjects;
#endif
};
+
+ struct list_head list; /* slobs list of pages */
};
/* Remainder is not double word aligned */
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>
^ permalink raw reply [flat|nested] 40+ messages in thread* Common [03/20] [slob] Remove various small accessors
2012-06-13 15:24 Common [00/20] Sl[auo]b: Common code rework V5 (for merge) Christoph Lameter
2012-06-13 15:24 ` Common [01/20] [slob] Define page struct fields used in mm_types.h Christoph Lameter
@ 2012-06-13 15:24 ` Christoph Lameter
2012-06-13 15:24 ` Common [04/20] [slab] Use page struct fields instead of casting Christoph Lameter
` (15 subsequent siblings)
17 siblings, 0 replies; 40+ messages in thread
From: Christoph Lameter @ 2012-06-13 15:24 UTC (permalink / raw)
To: Pekka Enberg
Cc: linux-mm, David Rientjes, Matt Mackall, Glauber Costa, Joonsoo Kim
[-- Attachment #1: slob_inline --]
[-- Type: text/plain, Size: 3265 bytes --]
Those have become so simple that they are no longer needed.
Reviewed-by: Joonsoo Kim <js1304@gmail.com>
Acked-by: David Rientjes <rientjes@google.com>
signed-off-by: Christoph Lameter <cl@linux.com>
---
mm/slob.c | 49 +++++++++----------------------------------------
1 file changed, 9 insertions(+), 40 deletions(-)
Index: linux-2.6/mm/slob.c
===================================================================
--- linux-2.6.orig/mm/slob.c 2012-06-13 03:44:40.465477463 -0500
+++ linux-2.6/mm/slob.c 2012-06-13 03:45:54.725475924 -0500
@@ -92,14 +92,6 @@ struct slob_block {
typedef struct slob_block slob_t;
/*
- * free_slob_page: call before a slob_page is returned to the page allocator.
- */
-static inline void free_slob_page(struct page *sp)
-{
- reset_page_mapcount(sp);
-}
-
-/*
* All partially free slob pages go on these lists.
*/
#define SLOB_BREAK1 256
@@ -109,29 +101,6 @@ static LIST_HEAD(free_slob_medium);
static LIST_HEAD(free_slob_large);
/*
- * is_slob_page: True for all slob pages (false for bigblock pages)
- */
-static inline int is_slob_page(struct page *sp)
-{
- return PageSlab(sp);
-}
-
-static inline void set_slob_page(struct page *sp)
-{
- __SetPageSlab(sp);
-}
-
-static inline void clear_slob_page(struct page *sp)
-{
- __ClearPageSlab(sp);
-}
-
-static inline struct page *slob_page(const void *addr)
-{
- return virt_to_page(addr);
-}
-
-/*
* slob_page_free: true for pages on free_slob_pages list.
*/
static inline int slob_page_free(struct page *sp)
@@ -347,8 +316,8 @@ static void *slob_alloc(size_t size, gfp
b = slob_new_pages(gfp & ~__GFP_ZERO, 0, node);
if (!b)
return NULL;
- sp = slob_page(b);
- set_slob_page(sp);
+ sp = virt_to_page(b);
+ __SetPageSlab(sp);
spin_lock_irqsave(&slob_lock, flags);
sp->units = SLOB_UNITS(PAGE_SIZE);
@@ -380,7 +349,7 @@ static void slob_free(void *block, int s
return;
BUG_ON(!size);
- sp = slob_page(block);
+ sp = virt_to_page(block);
units = SLOB_UNITS(size);
spin_lock_irqsave(&slob_lock, flags);
@@ -390,8 +359,8 @@ static void slob_free(void *block, int s
if (slob_page_free(sp))
clear_slob_page_free(sp);
spin_unlock_irqrestore(&slob_lock, flags);
- clear_slob_page(sp);
- free_slob_page(sp);
+ __ClearPageSlab(sp);
+ reset_page_mapcount(sp);
slob_free_pages(b, 0);
return;
}
@@ -508,8 +477,8 @@ void kfree(const void *block)
return;
kmemleak_free(block);
- sp = slob_page(block);
- if (is_slob_page(sp)) {
+ sp = virt_to_page(block);
+ if (PageSlab(sp)) {
int align = max(ARCH_KMALLOC_MINALIGN, ARCH_SLAB_MINALIGN);
unsigned int *m = (unsigned int *)(block - align);
slob_free(m, *m + align);
@@ -527,8 +496,8 @@ size_t ksize(const void *block)
if (unlikely(block == ZERO_SIZE_PTR))
return 0;
- sp = slob_page(block);
- if (is_slob_page(sp)) {
+ sp = virt_to_page(block);
+ if (PageSlab(sp)) {
int align = max(ARCH_KMALLOC_MINALIGN, ARCH_SLAB_MINALIGN);
unsigned int *m = (unsigned int *)(block - align);
return SLOB_UNITS(*m) * SLOB_UNIT;
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>
^ permalink raw reply [flat|nested] 40+ messages in thread* Common [04/20] [slab] Use page struct fields instead of casting
2012-06-13 15:24 Common [00/20] Sl[auo]b: Common code rework V5 (for merge) Christoph Lameter
2012-06-13 15:24 ` Common [01/20] [slob] Define page struct fields used in mm_types.h Christoph Lameter
2012-06-13 15:24 ` Common [03/20] [slob] Remove various small accessors Christoph Lameter
@ 2012-06-13 15:24 ` Christoph Lameter
2012-06-13 15:24 ` Common [05/20] [slab] Remove some accessors Christoph Lameter
` (14 subsequent siblings)
17 siblings, 0 replies; 40+ messages in thread
From: Christoph Lameter @ 2012-06-13 15:24 UTC (permalink / raw)
To: Pekka Enberg
Cc: linux-mm, David Rientjes, Matt Mackall, Glauber Costa, Joonsoo Kim
[-- Attachment #1: slab_page_struct_fields --]
[-- Type: text/plain, Size: 2187 bytes --]
Add fields to the page struct so that it is properly documented that
slab overlays the lru fields.
This cleans up some casts in slab.
Reviewed-by: Glauber Costa <glommer@parallels.com>
Reviewed-by: Joonsoo Kim <js1304@gmail.com>
Signed-off-by: Christoph Lameter <cl@linux.com>
---
include/linux/mm_types.h | 4 ++++
mm/slab.c | 8 ++++----
2 files changed, 8 insertions(+), 4 deletions(-)
Index: linux-2.6/mm/slab.c
===================================================================
--- linux-2.6.orig/mm/slab.c 2012-06-13 03:44:15.321477984 -0500
+++ linux-2.6/mm/slab.c 2012-06-13 03:45:57.341475870 -0500
@@ -496,25 +496,25 @@ static bool slab_max_order_set __initdat
*/
static inline void page_set_cache(struct page *page, struct kmem_cache *cache)
{
- page->lru.next = (struct list_head *)cache;
+ page->slab_cache = cache;
}
static inline struct kmem_cache *page_get_cache(struct page *page)
{
page = compound_head(page);
BUG_ON(!PageSlab(page));
- return (struct kmem_cache *)page->lru.next;
+ return page->slab_cache;
}
static inline void page_set_slab(struct page *page, struct slab *slab)
{
- page->lru.prev = (struct list_head *)slab;
+ page->slab_page = slab;
}
static inline struct slab *page_get_slab(struct page *page)
{
BUG_ON(!PageSlab(page));
- return (struct slab *)page->lru.prev;
+ return page->slab_page;
}
static inline struct kmem_cache *virt_to_cache(const void *obj)
Index: linux-2.6/include/linux/mm_types.h
===================================================================
--- linux-2.6.orig/include/linux/mm_types.h 2012-06-13 03:44:40.433477464 -0500
+++ linux-2.6/include/linux/mm_types.h 2012-06-13 03:44:40.525477462 -0500
@@ -110,6 +110,10 @@ struct page {
};
struct list_head list; /* slobs list of pages */
+ struct { /* slab fields */
+ struct kmem_cache *slab_cache;
+ struct slab *slab_page;
+ };
};
/* Remainder is not double word aligned */
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>
^ permalink raw reply [flat|nested] 40+ messages in thread* Common [05/20] [slab] Remove some accessors
2012-06-13 15:24 Common [00/20] Sl[auo]b: Common code rework V5 (for merge) Christoph Lameter
` (2 preceding siblings ...)
2012-06-13 15:24 ` Common [04/20] [slab] Use page struct fields instead of casting Christoph Lameter
@ 2012-06-13 15:24 ` Christoph Lameter
2012-06-13 15:24 ` Common [06/20] Extract common fields from struct kmem_cache Christoph Lameter
` (13 subsequent siblings)
17 siblings, 0 replies; 40+ messages in thread
From: Christoph Lameter @ 2012-06-13 15:24 UTC (permalink / raw)
To: Pekka Enberg
Cc: linux-mm, David Rientjes, Matt Mackall, Glauber Costa, Joonsoo Kim
[-- Attachment #1: slab_remove_accessors --]
[-- Type: text/plain, Size: 2903 bytes --]
Those are rather trivial now and its better to see inline what is
really going on.
Signed-off-by: Christoph Lameter <cl@linux.com>
---
mm/slab.c | 35 ++++++++---------------------------
1 file changed, 8 insertions(+), 27 deletions(-)
Index: linux-2.6/mm/slab.c
===================================================================
--- linux-2.6.orig/mm/slab.c 2012-06-13 03:44:40.525477462 -0500
+++ linux-2.6/mm/slab.c 2012-06-13 03:45:54.745475924 -0500
@@ -489,16 +489,6 @@ EXPORT_SYMBOL(slab_buffer_size);
static int slab_max_order = SLAB_MAX_ORDER_LO;
static bool slab_max_order_set __initdata;
-/*
- * Functions for storing/retrieving the cachep and or slab from the page
- * allocator. These are used to find the slab an obj belongs to. With kfree(),
- * these are used to find the cache which an obj belongs to.
- */
-static inline void page_set_cache(struct page *page, struct kmem_cache *cache)
-{
- page->slab_cache = cache;
-}
-
static inline struct kmem_cache *page_get_cache(struct page *page)
{
page = compound_head(page);
@@ -506,27 +496,18 @@ static inline struct kmem_cache *page_ge
return page->slab_cache;
}
-static inline void page_set_slab(struct page *page, struct slab *slab)
-{
- page->slab_page = slab;
-}
-
-static inline struct slab *page_get_slab(struct page *page)
-{
- BUG_ON(!PageSlab(page));
- return page->slab_page;
-}
-
static inline struct kmem_cache *virt_to_cache(const void *obj)
{
struct page *page = virt_to_head_page(obj);
- return page_get_cache(page);
+ return page->slab_cache;
}
static inline struct slab *virt_to_slab(const void *obj)
{
struct page *page = virt_to_head_page(obj);
- return page_get_slab(page);
+
+ VM_BUG_ON(!PageSlab(page));
+ return page->slab_page;
}
static inline void *index_to_obj(struct kmem_cache *cache, struct slab *slab,
@@ -2918,8 +2899,8 @@ static void slab_map_pages(struct kmem_c
nr_pages <<= cache->gfporder;
do {
- page_set_cache(page, cache);
- page_set_slab(page, slab);
+ page->slab_cache = cache;
+ page->slab_page = slab;
page++;
} while (--nr_pages);
}
@@ -3057,7 +3038,7 @@ static void *cache_free_debugcheck(struc
kfree_debugcheck(objp);
page = virt_to_head_page(objp);
- slabp = page_get_slab(page);
+ slabp = page->slab_page;
if (cachep->flags & SLAB_RED_ZONE) {
verify_redzone_free(cachep, objp);
@@ -3261,7 +3242,7 @@ static void *cache_alloc_debugcheck_afte
struct slab *slabp;
unsigned objnr;
- slabp = page_get_slab(virt_to_head_page(objp));
+ slabp = virt_to_head_page(objp)->slab_page;
objnr = (unsigned)(objp - slabp->s_mem) / cachep->buffer_size;
slab_bufctl(slabp)[objnr] = BUFCTL_ACTIVE;
}
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>
^ permalink raw reply [flat|nested] 40+ messages in thread* Common [06/20] Extract common fields from struct kmem_cache
2012-06-13 15:24 Common [00/20] Sl[auo]b: Common code rework V5 (for merge) Christoph Lameter
` (3 preceding siblings ...)
2012-06-13 15:24 ` Common [05/20] [slab] Remove some accessors Christoph Lameter
@ 2012-06-13 15:24 ` Christoph Lameter
2012-06-13 15:24 ` Common [07/20] [slab] Get rid of obj_size macro Christoph Lameter
` (12 subsequent siblings)
17 siblings, 0 replies; 40+ messages in thread
From: Christoph Lameter @ 2012-06-13 15:24 UTC (permalink / raw)
To: Pekka Enberg
Cc: linux-mm, David Rientjes, Matt Mackall, Glauber Costa, Joonsoo Kim
[-- Attachment #1: common_fields --]
[-- Type: text/plain, Size: 30961 bytes --]
Define a struct that describes common fields used in all slab allocators.
A slab allocator either uses the common definition (like SLOB) or is
required to provide members of kmem_cache with the definition given.
After that it will be possible to share code that
only operates on those fields of kmem_cache.
The patch basically takes the slob definition of kmem cache and
uses the field namees for the other allocators.
It also standardizes the names used for basic object lengths in
allocators:
object_size Struct size specified at kmem_cache_create. Basically
the payload expected to be used by the subsystem.
size The size of memory allocator for each object. This size
is larger than object_size and includes padding, alignment
and extra metadata for each object (f.e. for debugging
and rcu).
Signed-off-by: Christoph Lameter <cl@linux.com>
---
include/linux/slab.h | 24 +++++++++
include/linux/slab_def.h | 10 ++--
include/linux/slub_def.h | 2
mm/slab.c | 117 +++++++++++++++++++++++------------------------
mm/slob.c | 9 ---
mm/slub.c | 80 ++++++++++++++++----------------
6 files changed, 130 insertions(+), 112 deletions(-)
Index: linux-2.6/include/linux/slab.h
===================================================================
--- linux-2.6.orig/include/linux/slab.h 2012-06-13 03:44:14.921477992 -0500
+++ linux-2.6/include/linux/slab.h 2012-06-13 03:45:48.913476045 -0500
@@ -93,6 +93,30 @@
(unsigned long)ZERO_SIZE_PTR)
/*
+ * Common fields provided in kmem_cache by all slab allocators
+ * This struct is either used directly by the allocator (SLOB)
+ * or the allocator must include definitions for all fields
+ * provided in kmem_cache_common in their definition of kmem_cache.
+ *
+ * Once we can do anonymous structs (C11 standard) we could put a
+ * anonymous struct definition in these allocators so that the
+ * separate allocations in the kmem_cache structure of SLAB and
+ * SLUB is no longer needed.
+ */
+#ifdef CONFIG_SLOB
+struct kmem_cache {
+ unsigned int object_size;/* The original size of the object */
+ unsigned int size; /* The aligned/padded/added on size */
+ unsigned int align; /* Alignment as calculated */
+ unsigned long flags; /* Active flags on the slab */
+ const char *name; /* Slab name for sysfs */
+ int refcount; /* Use counter */
+ void (*ctor)(void *); /* Called on object slot creation */
+ struct list_head list; /* List of all slab caches on the system */
+};
+#endif
+
+/*
* struct kmem_cache related prototypes
*/
void __init kmem_cache_init(void);
Index: linux-2.6/mm/slob.c
===================================================================
--- linux-2.6.orig/mm/slob.c 2012-06-13 03:44:40.493477462 -0500
+++ linux-2.6/mm/slob.c 2012-06-13 03:45:48.937476044 -0500
@@ -506,13 +506,6 @@ size_t ksize(const void *block)
}
EXPORT_SYMBOL(ksize);
-struct kmem_cache {
- unsigned int size, align;
- unsigned long flags;
- const char *name;
- void (*ctor)(void *);
-};
-
struct kmem_cache *kmem_cache_create(const char *name, size_t size,
size_t align, unsigned long flags, void (*ctor)(void *))
{
@@ -523,7 +516,7 @@ struct kmem_cache *kmem_cache_create(con
if (c) {
c->name = name;
- c->size = size;
+ c->size = c->object_size;
if (flags & SLAB_DESTROY_BY_RCU) {
/* leave room for rcu footer at the end of object */
c->size += sizeof(struct slob_rcu);
Index: linux-2.6/mm/slab.c
===================================================================
--- linux-2.6.orig/mm/slab.c 2012-06-13 03:44:40.557477461 -0500
+++ linux-2.6/mm/slab.c 2012-06-13 03:45:51.877475983 -0500
@@ -424,8 +424,8 @@ static void kmem_list3_init(struct kmem_
* cachep->obj_offset - BYTES_PER_WORD .. cachep->obj_offset - 1:
* redzone word.
* cachep->obj_offset: The real object.
- * cachep->buffer_size - 2* BYTES_PER_WORD: redzone word [BYTES_PER_WORD long]
- * cachep->buffer_size - 1* BYTES_PER_WORD: last caller address
+ * cachep->size - 2* BYTES_PER_WORD: redzone word [BYTES_PER_WORD long]
+ * cachep->size - 1* BYTES_PER_WORD: last caller address
* [BYTES_PER_WORD long]
*/
static int obj_offset(struct kmem_cache *cachep)
@@ -435,7 +435,7 @@ static int obj_offset(struct kmem_cache
static int obj_size(struct kmem_cache *cachep)
{
- return cachep->obj_size;
+ return cachep->object_size;
}
static unsigned long long *dbg_redzone1(struct kmem_cache *cachep, void *objp)
@@ -449,23 +449,23 @@ static unsigned long long *dbg_redzone2(
{
BUG_ON(!(cachep->flags & SLAB_RED_ZONE));
if (cachep->flags & SLAB_STORE_USER)
- return (unsigned long long *)(objp + cachep->buffer_size -
+ return (unsigned long long *)(objp + cachep->size -
sizeof(unsigned long long) -
REDZONE_ALIGN);
- return (unsigned long long *) (objp + cachep->buffer_size -
+ return (unsigned long long *) (objp + cachep->size -
sizeof(unsigned long long));
}
static void **dbg_userword(struct kmem_cache *cachep, void *objp)
{
BUG_ON(!(cachep->flags & SLAB_STORE_USER));
- return (void **)(objp + cachep->buffer_size - BYTES_PER_WORD);
+ return (void **)(objp + cachep->size - BYTES_PER_WORD);
}
#else
#define obj_offset(x) 0
-#define obj_size(cachep) (cachep->buffer_size)
+#define obj_size(cachep) (cachep->size)
#define dbg_redzone1(cachep, objp) ({BUG(); (unsigned long long *)NULL;})
#define dbg_redzone2(cachep, objp) ({BUG(); (unsigned long long *)NULL;})
#define dbg_userword(cachep, objp) ({BUG(); (void **)NULL;})
@@ -475,7 +475,7 @@ static void **dbg_userword(struct kmem_c
#ifdef CONFIG_TRACING
size_t slab_buffer_size(struct kmem_cache *cachep)
{
- return cachep->buffer_size;
+ return cachep->size;
}
EXPORT_SYMBOL(slab_buffer_size);
#endif
@@ -513,13 +513,13 @@ static inline struct slab *virt_to_slab(
static inline void *index_to_obj(struct kmem_cache *cache, struct slab *slab,
unsigned int idx)
{
- return slab->s_mem + cache->buffer_size * idx;
+ return slab->s_mem + cache->size * idx;
}
/*
- * We want to avoid an expensive divide : (offset / cache->buffer_size)
- * Using the fact that buffer_size is a constant for a particular cache,
- * we can replace (offset / cache->buffer_size) by
+ * We want to avoid an expensive divide : (offset / cache->size)
+ * Using the fact that size is a constant for a particular cache,
+ * we can replace (offset / cache->size) by
* reciprocal_divide(offset, cache->reciprocal_buffer_size)
*/
static inline unsigned int obj_to_index(const struct kmem_cache *cache,
@@ -565,7 +565,7 @@ static struct kmem_cache cache_cache = {
.batchcount = 1,
.limit = BOOT_CPUCACHE_ENTRIES,
.shared = 1,
- .buffer_size = sizeof(struct kmem_cache),
+ .size = sizeof(struct kmem_cache),
.name = "kmem_cache",
};
@@ -1134,7 +1134,7 @@ static int init_cache_nodelists_node(int
struct kmem_list3 *l3;
const int memsize = sizeof(struct kmem_list3);
- list_for_each_entry(cachep, &cache_chain, next) {
+ list_for_each_entry(cachep, &cache_chain, list) {
/*
* Set up the size64 kmemlist for cpu before we can
* begin anything. Make sure some other cpu on this
@@ -1172,7 +1172,7 @@ static void __cpuinit cpuup_canceled(lon
int node = cpu_to_mem(cpu);
const struct cpumask *mask = cpumask_of_node(node);
- list_for_each_entry(cachep, &cache_chain, next) {
+ list_for_each_entry(cachep, &cache_chain, list) {
struct array_cache *nc;
struct array_cache *shared;
struct array_cache **alien;
@@ -1222,7 +1222,7 @@ free_array_cache:
* the respective cache's slabs, now we can go ahead and
* shrink each nodelist to its limit.
*/
- list_for_each_entry(cachep, &cache_chain, next) {
+ list_for_each_entry(cachep, &cache_chain, list) {
l3 = cachep->nodelists[node];
if (!l3)
continue;
@@ -1251,7 +1251,7 @@ static int __cpuinit cpuup_prepare(long
* Now we can go ahead with allocating the shared arrays and
* array caches
*/
- list_for_each_entry(cachep, &cache_chain, next) {
+ list_for_each_entry(cachep, &cache_chain, list) {
struct array_cache *nc;
struct array_cache *shared = NULL;
struct array_cache **alien = NULL;
@@ -1383,7 +1383,7 @@ static int __meminit drain_cache_nodelis
struct kmem_cache *cachep;
int ret = 0;
- list_for_each_entry(cachep, &cache_chain, next) {
+ list_for_each_entry(cachep, &cache_chain, list) {
struct kmem_list3 *l3;
l3 = cachep->nodelists[node];
@@ -1526,7 +1526,7 @@ void __init kmem_cache_init(void)
/* 1) create the cache_cache */
INIT_LIST_HEAD(&cache_chain);
- list_add(&cache_cache.next, &cache_chain);
+ list_add(&cache_cache.list, &cache_chain);
cache_cache.colour_off = cache_line_size();
cache_cache.array[smp_processor_id()] = &initarray_cache.cache;
cache_cache.nodelists[node] = &initkmem_list3[CACHE_CACHE + node];
@@ -1534,18 +1534,16 @@ void __init kmem_cache_init(void)
/*
* struct kmem_cache size depends on nr_node_ids & nr_cpu_ids
*/
- cache_cache.buffer_size = offsetof(struct kmem_cache, array[nr_cpu_ids]) +
+ cache_cache.size = offsetof(struct kmem_cache, array[nr_cpu_ids]) +
nr_node_ids * sizeof(struct kmem_list3 *);
-#if DEBUG
- cache_cache.obj_size = cache_cache.buffer_size;
-#endif
- cache_cache.buffer_size = ALIGN(cache_cache.buffer_size,
+ cache_cache.object_size = cache_cache.size;
+ cache_cache.size = ALIGN(cache_cache.size,
cache_line_size());
cache_cache.reciprocal_buffer_size =
- reciprocal_value(cache_cache.buffer_size);
+ reciprocal_value(cache_cache.size);
for (order = 0; order < MAX_ORDER; order++) {
- cache_estimate(order, cache_cache.buffer_size,
+ cache_estimate(order, cache_cache.size,
cache_line_size(), 0, &left_over, &cache_cache.num);
if (cache_cache.num)
break;
@@ -1671,7 +1669,7 @@ void __init kmem_cache_init_late(void)
/* 6) resize the head arrays to their final sizes */
mutex_lock(&cache_chain_mutex);
- list_for_each_entry(cachep, &cache_chain, next)
+ list_for_each_entry(cachep, &cache_chain, list)
if (enable_cpucache(cachep, GFP_NOWAIT))
BUG();
mutex_unlock(&cache_chain_mutex);
@@ -1724,7 +1722,7 @@ slab_out_of_memory(struct kmem_cache *ca
"SLAB: Unable to allocate memory on node %d (gfp=0x%x)\n",
nodeid, gfpflags);
printk(KERN_WARNING " cache: %s, object size: %d, order: %d\n",
- cachep->name, cachep->buffer_size, cachep->gfporder);
+ cachep->name, cachep->size, cachep->gfporder);
for_each_online_node(node) {
unsigned long active_objs = 0, num_objs = 0, free_objects = 0;
@@ -2028,10 +2026,10 @@ static void slab_destroy_debugcheck(stru
if (cachep->flags & SLAB_POISON) {
#ifdef CONFIG_DEBUG_PAGEALLOC
- if (cachep->buffer_size % PAGE_SIZE == 0 &&
+ if (cachep->size % PAGE_SIZE == 0 &&
OFF_SLAB(cachep))
kernel_map_pages(virt_to_page(objp),
- cachep->buffer_size / PAGE_SIZE, 1);
+ cachep->size / PAGE_SIZE, 1);
else
check_poison_obj(cachep, objp);
#else
@@ -2281,7 +2279,7 @@ kmem_cache_create (const char *name, siz
mutex_lock(&cache_chain_mutex);
}
- list_for_each_entry(pc, &cache_chain, next) {
+ list_for_each_entry(pc, &cache_chain, list) {
char tmp;
int res;
@@ -2294,7 +2292,7 @@ kmem_cache_create (const char *name, siz
if (res) {
printk(KERN_ERR
"SLAB: cache with size %d has lost its name\n",
- pc->buffer_size);
+ pc->size);
continue;
}
@@ -2399,8 +2397,9 @@ kmem_cache_create (const char *name, siz
goto oops;
cachep->nodelists = (struct kmem_list3 **)&cachep->array[nr_cpu_ids];
+ cachep->object_size = size;
+ cachep->align = align;
#if DEBUG
- cachep->obj_size = size;
/*
* Both debugging options require word-alignment which is calculated
@@ -2423,7 +2422,7 @@ kmem_cache_create (const char *name, siz
}
#if FORCED_DEBUG && defined(CONFIG_DEBUG_PAGEALLOC)
if (size >= malloc_sizes[INDEX_L3 + 1].cs_size
- && cachep->obj_size > cache_line_size() && ALIGN(size, align) < PAGE_SIZE) {
+ && cachep->object_size > cache_line_size() && ALIGN(size, align) < PAGE_SIZE) {
cachep->obj_offset += PAGE_SIZE - ALIGN(size, align);
size = PAGE_SIZE;
}
@@ -2492,7 +2491,7 @@ kmem_cache_create (const char *name, siz
cachep->gfpflags = 0;
if (CONFIG_ZONE_DMA_FLAG && (flags & SLAB_CACHE_DMA))
cachep->gfpflags |= GFP_DMA;
- cachep->buffer_size = size;
+ cachep->size = size;
cachep->reciprocal_buffer_size = reciprocal_value(size);
if (flags & CFLGS_OFF_SLAB) {
@@ -2526,7 +2525,7 @@ kmem_cache_create (const char *name, siz
}
/* cache setup completed, link it into the list */
- list_add(&cachep->next, &cache_chain);
+ list_add(&cachep->list, &cache_chain);
oops:
if (!cachep && (flags & SLAB_PANIC))
panic("kmem_cache_create(): failed to create slab `%s'\n",
@@ -2721,10 +2720,10 @@ void kmem_cache_destroy(struct kmem_cach
/*
* the chain is never empty, cache_cache is never destroyed
*/
- list_del(&cachep->next);
+ list_del(&cachep->list);
if (__cache_shrink(cachep)) {
slab_error(cachep, "Can't free all objects");
- list_add(&cachep->next, &cache_chain);
+ list_add(&cachep->list, &cache_chain);
mutex_unlock(&cache_chain_mutex);
put_online_cpus();
return;
@@ -2821,10 +2820,10 @@ static void cache_init_objs(struct kmem_
slab_error(cachep, "constructor overwrote the"
" start of an object");
}
- if ((cachep->buffer_size % PAGE_SIZE) == 0 &&
+ if ((cachep->size % PAGE_SIZE) == 0 &&
OFF_SLAB(cachep) && cachep->flags & SLAB_POISON)
kernel_map_pages(virt_to_page(objp),
- cachep->buffer_size / PAGE_SIZE, 0);
+ cachep->size / PAGE_SIZE, 0);
#else
if (cachep->ctor)
cachep->ctor(objp);
@@ -3058,10 +3057,10 @@ static void *cache_free_debugcheck(struc
#endif
if (cachep->flags & SLAB_POISON) {
#ifdef CONFIG_DEBUG_PAGEALLOC
- if ((cachep->buffer_size % PAGE_SIZE)==0 && OFF_SLAB(cachep)) {
+ if ((cachep->size % PAGE_SIZE)==0 && OFF_SLAB(cachep)) {
store_stackinfo(cachep, objp, (unsigned long)caller);
kernel_map_pages(virt_to_page(objp),
- cachep->buffer_size / PAGE_SIZE, 0);
+ cachep->size / PAGE_SIZE, 0);
} else {
poison_obj(cachep, objp, POISON_FREE);
}
@@ -3211,9 +3210,9 @@ static void *cache_alloc_debugcheck_afte
return objp;
if (cachep->flags & SLAB_POISON) {
#ifdef CONFIG_DEBUG_PAGEALLOC
- if ((cachep->buffer_size % PAGE_SIZE) == 0 && OFF_SLAB(cachep))
+ if ((cachep->size % PAGE_SIZE) == 0 && OFF_SLAB(cachep))
kernel_map_pages(virt_to_page(objp),
- cachep->buffer_size / PAGE_SIZE, 1);
+ cachep->size / PAGE_SIZE, 1);
else
check_poison_obj(cachep, objp);
#else
@@ -3243,7 +3242,7 @@ static void *cache_alloc_debugcheck_afte
unsigned objnr;
slabp = virt_to_head_page(objp)->slab_page;
- objnr = (unsigned)(objp - slabp->s_mem) / cachep->buffer_size;
+ objnr = (unsigned)(objp - slabp->s_mem) / cachep->size;
slab_bufctl(slabp)[objnr] = BUFCTL_ACTIVE;
}
#endif
@@ -3747,7 +3746,7 @@ void *kmem_cache_alloc(struct kmem_cache
void *ret = __cache_alloc(cachep, flags, __builtin_return_address(0));
trace_kmem_cache_alloc(_RET_IP_, ret,
- obj_size(cachep), cachep->buffer_size, flags);
+ obj_size(cachep), cachep->size, flags);
return ret;
}
@@ -3775,7 +3774,7 @@ void *kmem_cache_alloc_node(struct kmem_
__builtin_return_address(0));
trace_kmem_cache_alloc_node(_RET_IP_, ret,
- obj_size(cachep), cachep->buffer_size,
+ obj_size(cachep), cachep->size,
flags, nodeid);
return ret;
@@ -3857,7 +3856,7 @@ static __always_inline void *__do_kmallo
ret = __cache_alloc(cachep, flags, caller);
trace_kmalloc((unsigned long) caller, ret,
- size, cachep->buffer_size, flags);
+ size, cachep->size, flags);
return ret;
}
@@ -4011,7 +4010,7 @@ static int alloc_kmemlist(struct kmem_ca
return 0;
fail:
- if (!cachep->next.next) {
+ if (!cachep->list.next) {
/* Cache is not active yet. Roll back what we did */
node--;
while (node >= 0) {
@@ -4105,13 +4104,13 @@ static int enable_cpucache(struct kmem_c
* The numbers are guessed, we should auto-tune as described by
* Bonwick.
*/
- if (cachep->buffer_size > 131072)
+ if (cachep->size > 131072)
limit = 1;
- else if (cachep->buffer_size > PAGE_SIZE)
+ else if (cachep->size > PAGE_SIZE)
limit = 8;
- else if (cachep->buffer_size > 1024)
+ else if (cachep->size > 1024)
limit = 24;
- else if (cachep->buffer_size > 256)
+ else if (cachep->size > 256)
limit = 54;
else
limit = 120;
@@ -4126,7 +4125,7 @@ static int enable_cpucache(struct kmem_c
* to a larger limit. Thus disabled by default.
*/
shared = 0;
- if (cachep->buffer_size <= PAGE_SIZE && num_possible_cpus() > 1)
+ if (cachep->size <= PAGE_SIZE && num_possible_cpus() > 1)
shared = 8;
#if DEBUG
@@ -4196,7 +4195,7 @@ static void cache_reap(struct work_struc
/* Give up. Setup the next iteration. */
goto out;
- list_for_each_entry(searchp, &cache_chain, next) {
+ list_for_each_entry(searchp, &cache_chain, list) {
check_irq_on();
/*
@@ -4289,7 +4288,7 @@ static void s_stop(struct seq_file *m, v
static int s_show(struct seq_file *m, void *p)
{
- struct kmem_cache *cachep = list_entry(p, struct kmem_cache, next);
+ struct kmem_cache *cachep = list_entry(p, struct kmem_cache, list);
struct slab *slabp;
unsigned long active_objs;
unsigned long num_objs;
@@ -4345,7 +4344,7 @@ static int s_show(struct seq_file *m, vo
printk(KERN_ERR "slab: cache %s error: %s\n", name, error);
seq_printf(m, "%-17s %6lu %6lu %6u %4u %4d",
- name, active_objs, num_objs, cachep->buffer_size,
+ name, active_objs, num_objs, cachep->size,
cachep->num, (1 << cachep->gfporder));
seq_printf(m, " : tunables %4u %4u %4u",
cachep->limit, cachep->batchcount, cachep->shared);
@@ -4437,7 +4436,7 @@ static ssize_t slabinfo_write(struct fil
/* Find the cache in the chain of caches. */
mutex_lock(&cache_chain_mutex);
res = -EINVAL;
- list_for_each_entry(cachep, &cache_chain, next) {
+ list_for_each_entry(cachep, &cache_chain, list) {
if (!strcmp(cachep->name, kbuf)) {
if (limit < 1 || batchcount < 1 ||
batchcount > limit || shared < 0) {
@@ -4513,7 +4512,7 @@ static void handle_slab(unsigned long *n
int i;
if (n[0] == n[1])
return;
- for (i = 0, p = s->s_mem; i < c->num; i++, p += c->buffer_size) {
+ for (i = 0, p = s->s_mem; i < c->num; i++, p += c->size) {
if (slab_bufctl(s)[i] != BUFCTL_ACTIVE)
continue;
if (!add_caller(n, (unsigned long)*dbg_userword(c, p)))
Index: linux-2.6/include/linux/slab_def.h
===================================================================
--- linux-2.6.orig/include/linux/slab_def.h 2012-06-13 03:44:14.869477993 -0500
+++ linux-2.6/include/linux/slab_def.h 2012-06-13 03:44:40.593477460 -0500
@@ -27,7 +27,7 @@ struct kmem_cache {
unsigned int limit;
unsigned int shared;
- unsigned int buffer_size;
+ unsigned int size;
u32 reciprocal_buffer_size;
/* 2) touched by every alloc & free from the backend */
@@ -52,7 +52,10 @@ struct kmem_cache {
/* 4) cache creation/removal */
const char *name;
- struct list_head next;
+ struct list_head list;
+ int refcount;
+ int object_size;
+ int align;
/* 5) statistics */
#ifdef CONFIG_DEBUG_SLAB
@@ -73,12 +76,11 @@ struct kmem_cache {
/*
* If debugging is enabled, then the allocator can add additional
- * fields and/or padding to every object. buffer_size contains the total
+ * fields and/or padding to every object. size contains the total
* object size including these internal fields, the following two
* variables contain the offset to the user object and its size.
*/
int obj_offset;
- int obj_size;
#endif /* CONFIG_DEBUG_SLAB */
/* 6) per-cpu/per-node data, touched during every alloc/free */
Index: linux-2.6/include/linux/slub_def.h
===================================================================
--- linux-2.6.orig/include/linux/slub_def.h 2012-06-13 03:44:36.781477539 -0500
+++ linux-2.6/include/linux/slub_def.h 2012-06-13 03:44:40.593477460 -0500
@@ -82,7 +82,7 @@ struct kmem_cache {
unsigned long flags;
unsigned long min_partial;
int size; /* The size of an object including meta data */
- int objsize; /* The size of an object without meta data */
+ int object_size; /* The size of an object without meta data */
int offset; /* Free pointer offset. */
int cpu_partial; /* Number of per cpu partial objects to keep around */
struct kmem_cache_order_objects oo;
Index: linux-2.6/mm/slub.c
===================================================================
--- linux-2.6.orig/mm/slub.c 2012-06-13 03:44:37.389477527 -0500
+++ linux-2.6/mm/slub.c 2012-06-13 03:45:48.981476043 -0500
@@ -311,7 +311,7 @@ static inline size_t slab_ksize(const st
* and whatever may come after it.
*/
if (s->flags & (SLAB_RED_ZONE | SLAB_POISON))
- return s->objsize;
+ return s->object_size;
#endif
/*
@@ -609,11 +609,11 @@ static void print_trailer(struct kmem_ca
if (p > addr + 16)
print_section("Bytes b4 ", p - 16, 16);
- print_section("Object ", p, min_t(unsigned long, s->objsize,
+ print_section("Object ", p, min_t(unsigned long, s->object_size,
PAGE_SIZE));
if (s->flags & SLAB_RED_ZONE)
- print_section("Redzone ", p + s->objsize,
- s->inuse - s->objsize);
+ print_section("Redzone ", p + s->object_size,
+ s->inuse - s->object_size);
if (s->offset)
off = s->offset + sizeof(void *);
@@ -655,12 +655,12 @@ static void init_object(struct kmem_cach
u8 *p = object;
if (s->flags & __OBJECT_POISON) {
- memset(p, POISON_FREE, s->objsize - 1);
- p[s->objsize - 1] = POISON_END;
+ memset(p, POISON_FREE, s->object_size - 1);
+ p[s->object_size - 1] = POISON_END;
}
if (s->flags & SLAB_RED_ZONE)
- memset(p + s->objsize, val, s->inuse - s->objsize);
+ memset(p + s->object_size, val, s->inuse - s->object_size);
}
static void restore_bytes(struct kmem_cache *s, char *message, u8 data,
@@ -705,10 +705,10 @@ static int check_bytes_and_report(struct
* Poisoning uses 0x6b (POISON_FREE) and the last byte is
* 0xa5 (POISON_END)
*
- * object + s->objsize
+ * object + s->object_size
* Padding to reach word boundary. This is also used for Redzoning.
* Padding is extended by another word if Redzoning is enabled and
- * objsize == inuse.
+ * object_size == inuse.
*
* We fill with 0xbb (RED_INACTIVE) for inactive objects and with
* 0xcc (RED_ACTIVE) for objects in use.
@@ -727,7 +727,7 @@ static int check_bytes_and_report(struct
* object + s->size
* Nothing is used beyond s->size.
*
- * If slabcaches are merged then the objsize and inuse boundaries are mostly
+ * If slabcaches are merged then the object_size and inuse boundaries are mostly
* ignored. And therefore no slab options that rely on these boundaries
* may be used with merged slabcaches.
*/
@@ -787,25 +787,25 @@ static int check_object(struct kmem_cach
void *object, u8 val)
{
u8 *p = object;
- u8 *endobject = object + s->objsize;
+ u8 *endobject = object + s->object_size;
if (s->flags & SLAB_RED_ZONE) {
if (!check_bytes_and_report(s, page, object, "Redzone",
- endobject, val, s->inuse - s->objsize))
+ endobject, val, s->inuse - s->object_size))
return 0;
} else {
- if ((s->flags & SLAB_POISON) && s->objsize < s->inuse) {
+ if ((s->flags & SLAB_POISON) && s->object_size < s->inuse) {
check_bytes_and_report(s, page, p, "Alignment padding",
- endobject, POISON_INUSE, s->inuse - s->objsize);
+ endobject, POISON_INUSE, s->inuse - s->object_size);
}
}
if (s->flags & SLAB_POISON) {
if (val != SLUB_RED_ACTIVE && (s->flags & __OBJECT_POISON) &&
(!check_bytes_and_report(s, page, p, "Poison", p,
- POISON_FREE, s->objsize - 1) ||
+ POISON_FREE, s->object_size - 1) ||
!check_bytes_and_report(s, page, p, "Poison",
- p + s->objsize - 1, POISON_END, 1)))
+ p + s->object_size - 1, POISON_END, 1)))
return 0;
/*
* check_pad_bytes cleans up on its own.
@@ -926,7 +926,7 @@ static void trace(struct kmem_cache *s,
page->freelist);
if (!alloc)
- print_section("Object ", (void *)object, s->objsize);
+ print_section("Object ", (void *)object, s->object_size);
dump_stack();
}
@@ -942,14 +942,14 @@ static inline int slab_pre_alloc_hook(st
lockdep_trace_alloc(flags);
might_sleep_if(flags & __GFP_WAIT);
- return should_failslab(s->objsize, flags, s->flags);
+ return should_failslab(s->object_size, flags, s->flags);
}
static inline void slab_post_alloc_hook(struct kmem_cache *s, gfp_t flags, void *object)
{
flags &= gfp_allowed_mask;
kmemcheck_slab_alloc(s, flags, object, slab_ksize(s));
- kmemleak_alloc_recursive(object, s->objsize, 1, s->flags, flags);
+ kmemleak_alloc_recursive(object, s->object_size, 1, s->flags, flags);
}
static inline void slab_free_hook(struct kmem_cache *s, void *x)
@@ -966,13 +966,13 @@ static inline void slab_free_hook(struct
unsigned long flags;
local_irq_save(flags);
- kmemcheck_slab_free(s, x, s->objsize);
- debug_check_no_locks_freed(x, s->objsize);
+ kmemcheck_slab_free(s, x, s->object_size);
+ debug_check_no_locks_freed(x, s->object_size);
local_irq_restore(flags);
}
#endif
if (!(s->flags & SLAB_DEBUG_OBJECTS))
- debug_check_no_obj_freed(x, s->objsize);
+ debug_check_no_obj_freed(x, s->object_size);
}
/*
@@ -1207,7 +1207,7 @@ out:
__setup("slub_debug", setup_slub_debug);
-static unsigned long kmem_cache_flags(unsigned long objsize,
+static unsigned long kmem_cache_flags(unsigned long object_size,
unsigned long flags, const char *name,
void (*ctor)(void *))
{
@@ -1237,7 +1237,7 @@ static inline int check_object(struct km
static inline void add_full(struct kmem_cache *s, struct kmem_cache_node *n,
struct page *page) {}
static inline void remove_full(struct kmem_cache *s, struct page *page) {}
-static inline unsigned long kmem_cache_flags(unsigned long objsize,
+static inline unsigned long kmem_cache_flags(unsigned long object_size,
unsigned long flags, const char *name,
void (*ctor)(void *))
{
@@ -2098,10 +2098,10 @@ slab_out_of_memory(struct kmem_cache *s,
"SLUB: Unable to allocate memory on node %d (gfp=0x%x)\n",
nid, gfpflags);
printk(KERN_WARNING " cache: %s, object size: %d, buffer size: %d, "
- "default order: %d, min order: %d\n", s->name, s->objsize,
+ "default order: %d, min order: %d\n", s->name, s->object_size,
s->size, oo_order(s->oo), oo_order(s->min));
- if (oo_order(s->min) > get_order(s->objsize))
+ if (oo_order(s->min) > get_order(s->object_size))
printk(KERN_WARNING " %s debugging increased min order, use "
"slub_debug=O to disable.\n", s->name);
@@ -2374,7 +2374,7 @@ redo:
}
if (unlikely(gfpflags & __GFP_ZERO) && object)
- memset(object, 0, s->objsize);
+ memset(object, 0, s->object_size);
slab_post_alloc_hook(s, gfpflags, object);
@@ -2385,7 +2385,7 @@ void *kmem_cache_alloc(struct kmem_cache
{
void *ret = slab_alloc(s, gfpflags, NUMA_NO_NODE, _RET_IP_);
- trace_kmem_cache_alloc(_RET_IP_, ret, s->objsize, s->size, gfpflags);
+ trace_kmem_cache_alloc(_RET_IP_, ret, s->object_size, s->size, gfpflags);
return ret;
}
@@ -2415,7 +2415,7 @@ void *kmem_cache_alloc_node(struct kmem_
void *ret = slab_alloc(s, gfpflags, node, _RET_IP_);
trace_kmem_cache_alloc_node(_RET_IP_, ret,
- s->objsize, s->size, gfpflags, node);
+ s->object_size, s->size, gfpflags, node);
return ret;
}
@@ -2910,7 +2910,7 @@ static void set_min_partial(struct kmem_
static int calculate_sizes(struct kmem_cache *s, int forced_order)
{
unsigned long flags = s->flags;
- unsigned long size = s->objsize;
+ unsigned long size = s->object_size;
unsigned long align = s->align;
int order;
@@ -2939,7 +2939,7 @@ static int calculate_sizes(struct kmem_c
* end of the object and the free pointer. If not then add an
* additional word to have some bytes to store Redzone information.
*/
- if ((flags & SLAB_RED_ZONE) && size == s->objsize)
+ if ((flags & SLAB_RED_ZONE) && size == s->object_size)
size += sizeof(void *);
#endif
@@ -2987,7 +2987,7 @@ static int calculate_sizes(struct kmem_c
* user specified and the dynamic determination of cache line size
* on bootup.
*/
- align = calculate_alignment(flags, align, s->objsize);
+ align = calculate_alignment(flags, align, s->object_size);
s->align = align;
/*
@@ -3035,7 +3035,7 @@ static int kmem_cache_open(struct kmem_c
memset(s, 0, kmem_size);
s->name = name;
s->ctor = ctor;
- s->objsize = size;
+ s->object_size = size;
s->align = align;
s->flags = kmem_cache_flags(size, flags, name, ctor);
s->reserved = 0;
@@ -3050,7 +3050,7 @@ static int kmem_cache_open(struct kmem_c
* Disable debugging flags that store metadata if the min slab
* order increased.
*/
- if (get_order(s->size) > get_order(s->objsize)) {
+ if (get_order(s->size) > get_order(s->object_size)) {
s->flags &= ~DEBUG_METADATA_FLAGS;
s->offset = 0;
if (!calculate_sizes(s, -1))
@@ -3124,7 +3124,7 @@ error:
*/
unsigned int kmem_cache_size(struct kmem_cache *s)
{
- return s->objsize;
+ return s->object_size;
}
EXPORT_SYMBOL(kmem_cache_size);
@@ -3853,11 +3853,11 @@ void __init kmem_cache_init(void)
if (s && s->size) {
char *name = kasprintf(GFP_NOWAIT,
- "dma-kmalloc-%d", s->objsize);
+ "dma-kmalloc-%d", s->object_size);
BUG_ON(!name);
kmalloc_dma_caches[i] = create_kmalloc_cache(name,
- s->objsize, SLAB_CACHE_DMA);
+ s->object_size, SLAB_CACHE_DMA);
}
}
#endif
@@ -3951,7 +3951,7 @@ struct kmem_cache *kmem_cache_create(con
* Adjust the object sizes so that we clear
* the complete object on kzalloc.
*/
- s->objsize = max(s->objsize, (int)size);
+ s->object_size = max(s->object_size, (int)size);
s->inuse = max_t(int, s->inuse, ALIGN(size, sizeof(void *)));
if (sysfs_slab_alias(s, name)) {
@@ -4634,7 +4634,7 @@ SLAB_ATTR_RO(align);
static ssize_t object_size_show(struct kmem_cache *s, char *buf)
{
- return sprintf(buf, "%d\n", s->objsize);
+ return sprintf(buf, "%d\n", s->object_size);
}
SLAB_ATTR_RO(object_size);
@@ -5438,7 +5438,7 @@ __initcall(slab_sysfs_init);
static void print_slabinfo_header(struct seq_file *m)
{
seq_puts(m, "slabinfo - version: 2.1\n");
- seq_puts(m, "# name <active_objs> <num_objs> <objsize> "
+ seq_puts(m, "# name <active_objs> <num_objs> <object_size> "
"<objperslab> <pagesperslab>");
seq_puts(m, " : tunables <limit> <batchcount> <sharedfactor>");
seq_puts(m, " : slabdata <active_slabs> <num_slabs> <sharedavail>");
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>
^ permalink raw reply [flat|nested] 40+ messages in thread* Common [07/20] [slab] Get rid of obj_size macro
2012-06-13 15:24 Common [00/20] Sl[auo]b: Common code rework V5 (for merge) Christoph Lameter
` (4 preceding siblings ...)
2012-06-13 15:24 ` Common [06/20] Extract common fields from struct kmem_cache Christoph Lameter
@ 2012-06-13 15:24 ` Christoph Lameter
2012-06-13 15:24 ` Common [08/20] Extract common code for kmem_cache_create() Christoph Lameter
` (11 subsequent siblings)
17 siblings, 0 replies; 40+ messages in thread
From: Christoph Lameter @ 2012-06-13 15:24 UTC (permalink / raw)
To: Pekka Enberg
Cc: linux-mm, David Rientjes, Matt Mackall, Glauber Costa, Joonsoo Kim
[-- Attachment #1: get_rid_of_obj_size --]
[-- Type: text/plain, Size: 6425 bytes --]
The size of the slab object is frequently needed. Since we now
have a size field directly in the kmem_cache structure there is no
need anymore of the obj_size macro/function.
Signed-off-by: Christoph Lameter <cl@linux.com>
---
mm/slab.c | 47 +++++++++++++++++++++--------------------------
1 file changed, 21 insertions(+), 26 deletions(-)
Index: linux-2.6/mm/slab.c
===================================================================
--- linux-2.6.orig/mm/slab.c 2012-06-13 03:44:40.593477460 -0500
+++ linux-2.6/mm/slab.c 2012-06-13 03:45:48.961476044 -0500
@@ -433,11 +433,6 @@ static int obj_offset(struct kmem_cache
return cachep->obj_offset;
}
-static int obj_size(struct kmem_cache *cachep)
-{
- return cachep->object_size;
-}
-
static unsigned long long *dbg_redzone1(struct kmem_cache *cachep, void *objp)
{
BUG_ON(!(cachep->flags & SLAB_RED_ZONE));
@@ -465,7 +460,6 @@ static void **dbg_userword(struct kmem_c
#else
#define obj_offset(x) 0
-#define obj_size(cachep) (cachep->size)
#define dbg_redzone1(cachep, objp) ({BUG(); (unsigned long long *)NULL;})
#define dbg_redzone2(cachep, objp) ({BUG(); (unsigned long long *)NULL;})
#define dbg_userword(cachep, objp) ({BUG(); (void **)NULL;})
@@ -1853,7 +1847,7 @@ static void kmem_rcu_free(struct rcu_hea
static void store_stackinfo(struct kmem_cache *cachep, unsigned long *addr,
unsigned long caller)
{
- int size = obj_size(cachep);
+ int size = cachep->object_size;
addr = (unsigned long *)&((char *)addr)[obj_offset(cachep)];
@@ -1885,7 +1879,7 @@ static void store_stackinfo(struct kmem_
static void poison_obj(struct kmem_cache *cachep, void *addr, unsigned char val)
{
- int size = obj_size(cachep);
+ int size = cachep->object_size;
addr = &((char *)addr)[obj_offset(cachep)];
memset(addr, val, size);
@@ -1945,7 +1939,7 @@ static void print_objinfo(struct kmem_ca
printk("\n");
}
realobj = (char *)objp + obj_offset(cachep);
- size = obj_size(cachep);
+ size = cachep->object_size;
for (i = 0; i < size && lines; i += 16, lines--) {
int limit;
limit = 16;
@@ -1962,7 +1956,7 @@ static void check_poison_obj(struct kmem
int lines = 0;
realobj = (char *)objp + obj_offset(cachep);
- size = obj_size(cachep);
+ size = cachep->object_size;
for (i = 0; i < size; i++) {
char exp = POISON_FREE;
@@ -3265,7 +3259,7 @@ static bool slab_should_failslab(struct
if (cachep == &cache_cache)
return false;
- return should_failslab(obj_size(cachep), flags, cachep->flags);
+ return should_failslab(cachep->object_size, flags, cachep->flags);
}
static inline void *____cache_alloc(struct kmem_cache *cachep, gfp_t flags)
@@ -3525,14 +3519,14 @@ __cache_alloc_node(struct kmem_cache *ca
out:
local_irq_restore(save_flags);
ptr = cache_alloc_debugcheck_after(cachep, flags, ptr, caller);
- kmemleak_alloc_recursive(ptr, obj_size(cachep), 1, cachep->flags,
+ kmemleak_alloc_recursive(ptr, cachep->object_size, 1, cachep->flags,
flags);
if (likely(ptr))
- kmemcheck_slab_alloc(cachep, flags, ptr, obj_size(cachep));
+ kmemcheck_slab_alloc(cachep, flags, ptr, cachep->object_size);
if (unlikely((flags & __GFP_ZERO) && ptr))
- memset(ptr, 0, obj_size(cachep));
+ memset(ptr, 0, cachep->object_size);
return ptr;
}
@@ -3587,15 +3581,15 @@ __cache_alloc(struct kmem_cache *cachep,
objp = __do_cache_alloc(cachep, flags);
local_irq_restore(save_flags);
objp = cache_alloc_debugcheck_after(cachep, flags, objp, caller);
- kmemleak_alloc_recursive(objp, obj_size(cachep), 1, cachep->flags,
+ kmemleak_alloc_recursive(objp, cachep->object_size, 1, cachep->flags,
flags);
prefetchw(objp);
if (likely(objp))
- kmemcheck_slab_alloc(cachep, flags, objp, obj_size(cachep));
+ kmemcheck_slab_alloc(cachep, flags, objp, cachep->object_size);
if (unlikely((flags & __GFP_ZERO) && objp))
- memset(objp, 0, obj_size(cachep));
+ memset(objp, 0, cachep->object_size);
return objp;
}
@@ -3711,7 +3705,7 @@ static inline void __cache_free(struct k
kmemleak_free_recursive(objp, cachep->flags);
objp = cache_free_debugcheck(cachep, objp, caller);
- kmemcheck_slab_free(cachep, objp, obj_size(cachep));
+ kmemcheck_slab_free(cachep, objp, cachep->object_size);
/*
* Skip calling cache_free_alien() when the platform is not numa.
@@ -3746,7 +3740,7 @@ void *kmem_cache_alloc(struct kmem_cache
void *ret = __cache_alloc(cachep, flags, __builtin_return_address(0));
trace_kmem_cache_alloc(_RET_IP_, ret,
- obj_size(cachep), cachep->size, flags);
+ cachep->object_size, cachep->size, flags);
return ret;
}
@@ -3774,7 +3768,7 @@ void *kmem_cache_alloc_node(struct kmem_
__builtin_return_address(0));
trace_kmem_cache_alloc_node(_RET_IP_, ret,
- obj_size(cachep), cachep->size,
+ cachep->object_size, cachep->size,
flags, nodeid);
return ret;
@@ -3896,9 +3890,9 @@ void kmem_cache_free(struct kmem_cache *
unsigned long flags;
local_irq_save(flags);
- debug_check_no_locks_freed(objp, obj_size(cachep));
+ debug_check_no_locks_freed(objp, cachep->size);
if (!(cachep->flags & SLAB_DEBUG_OBJECTS))
- debug_check_no_obj_freed(objp, obj_size(cachep));
+ debug_check_no_obj_freed(objp, cachep->object_size);
__cache_free(cachep, objp, __builtin_return_address(0));
local_irq_restore(flags);
@@ -3927,8 +3921,9 @@ void kfree(const void *objp)
local_irq_save(flags);
kfree_debugcheck(objp);
c = virt_to_cache(objp);
- debug_check_no_locks_freed(objp, obj_size(c));
- debug_check_no_obj_freed(objp, obj_size(c));
+ debug_check_no_locks_freed(objp, c->object_size);
+
+ debug_check_no_obj_freed(objp, c->object_size);
__cache_free(c, (void *)objp, __builtin_return_address(0));
local_irq_restore(flags);
}
@@ -3936,7 +3931,7 @@ EXPORT_SYMBOL(kfree);
unsigned int kmem_cache_size(struct kmem_cache *cachep)
{
- return obj_size(cachep);
+ return cachep->object_size;
}
EXPORT_SYMBOL(kmem_cache_size);
@@ -4657,6 +4652,6 @@ size_t ksize(const void *objp)
if (unlikely(objp == ZERO_SIZE_PTR))
return 0;
- return obj_size(virt_to_cache(objp));
+ return virt_to_cache(objp)->object_size;
}
EXPORT_SYMBOL(ksize);
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>
^ permalink raw reply [flat|nested] 40+ messages in thread* Common [08/20] Extract common code for kmem_cache_create()
2012-06-13 15:24 Common [00/20] Sl[auo]b: Common code rework V5 (for merge) Christoph Lameter
` (5 preceding siblings ...)
2012-06-13 15:24 ` Common [07/20] [slab] Get rid of obj_size macro Christoph Lameter
@ 2012-06-13 15:24 ` Christoph Lameter
2012-06-14 8:15 ` Glauber Costa
2012-06-13 15:25 ` Common [09/20] Common definition for boot state of the slab allocators Christoph Lameter
` (10 subsequent siblings)
17 siblings, 1 reply; 40+ messages in thread
From: Christoph Lameter @ 2012-06-13 15:24 UTC (permalink / raw)
To: Pekka Enberg
Cc: linux-mm, David Rientjes, Matt Mackall, Glauber Costa, Joonsoo Kim
[-- Attachment #1: common_kmem_cache_checks --]
[-- Type: text/plain, Size: 9729 bytes --]
Kmem_cache_create() does a variety of sanity checks but those
vary depending on the allocator. Use the strictest tests and put them into
a slab_common file. Make the tests conditional on CONFIG_DEBUG_VM.
This patch has the effect of adding sanity checks for SLUB and SLOB
under CONFIG_DEBUG_VM and removes the checks in SLAB for !CONFIG_DEBUG_VM.
Signed-off-by: Christoph Lameter <cl@linux.com>
---
include/linux/slab.h | 4 +++
mm/Makefile | 3 +-
mm/slab.c | 24 ++++++------------
mm/slab_common.c | 67 +++++++++++++++++++++++++++++++++++++++++++++++++++
mm/slob.c | 8 ++----
mm/slub.c | 11 --------
6 files changed, 86 insertions(+), 31 deletions(-)
Index: linux-2.6/mm/slab.c
===================================================================
--- linux-2.6.orig/mm/slab.c 2012-06-13 03:44:40.629477460 -0500
+++ linux-2.6/mm/slab.c 2012-06-13 03:45:46.149476102 -0500
@@ -1558,7 +1558,7 @@ void __init kmem_cache_init(void)
* bug.
*/
- sizes[INDEX_AC].cs_cachep = kmem_cache_create(names[INDEX_AC].name,
+ sizes[INDEX_AC].cs_cachep = __kmem_cache_create(names[INDEX_AC].name,
sizes[INDEX_AC].cs_size,
ARCH_KMALLOC_MINALIGN,
ARCH_KMALLOC_FLAGS|SLAB_PANIC,
@@ -1566,7 +1566,7 @@ void __init kmem_cache_init(void)
if (INDEX_AC != INDEX_L3) {
sizes[INDEX_L3].cs_cachep =
- kmem_cache_create(names[INDEX_L3].name,
+ __kmem_cache_create(names[INDEX_L3].name,
sizes[INDEX_L3].cs_size,
ARCH_KMALLOC_MINALIGN,
ARCH_KMALLOC_FLAGS|SLAB_PANIC,
@@ -1584,14 +1584,14 @@ void __init kmem_cache_init(void)
* allow tighter packing of the smaller caches.
*/
if (!sizes->cs_cachep) {
- sizes->cs_cachep = kmem_cache_create(names->name,
+ sizes->cs_cachep = __kmem_cache_create(names->name,
sizes->cs_size,
ARCH_KMALLOC_MINALIGN,
ARCH_KMALLOC_FLAGS|SLAB_PANIC,
NULL);
}
#ifdef CONFIG_ZONE_DMA
- sizes->cs_dmacachep = kmem_cache_create(
+ sizes->cs_dmacachep = __kmem_cache_create(
names->name_dma,
sizes->cs_size,
ARCH_KMALLOC_MINALIGN,
@@ -2220,7 +2220,7 @@ static int __init_refok setup_cpu_cache(
}
/**
- * kmem_cache_create - Create a cache.
+ * __kmem_cache_create - Create a cache.
* @name: A string which is used in /proc/slabinfo to identify this cache.
* @size: The size of objects to be created in this cache.
* @align: The required alignment for the objects.
@@ -2247,7 +2247,7 @@ static int __init_refok setup_cpu_cache(
* as davem.
*/
struct kmem_cache *
-kmem_cache_create (const char *name, size_t size, size_t align,
+__kmem_cache_create (const char *name, size_t size, size_t align,
unsigned long flags, void (*ctor)(void *))
{
size_t left_over, slab_size, ralign;
@@ -2388,7 +2388,7 @@ kmem_cache_create (const char *name, siz
/* Get cache's description obj. */
cachep = kmem_cache_zalloc(&cache_cache, gfp);
if (!cachep)
- goto oops;
+ return NULL;
cachep->nodelists = (struct kmem_list3 **)&cachep->array[nr_cpu_ids];
cachep->object_size = size;
@@ -2445,8 +2445,7 @@ kmem_cache_create (const char *name, siz
printk(KERN_ERR
"kmem_cache_create: couldn't create cache %s.\n", name);
kmem_cache_free(&cache_cache, cachep);
- cachep = NULL;
- goto oops;
+ return NULL;
}
slab_size = ALIGN(cachep->num * sizeof(kmem_bufctl_t)
+ sizeof(struct slab), align);
@@ -2504,8 +2503,7 @@ kmem_cache_create (const char *name, siz
if (setup_cpu_cache(cachep, gfp)) {
__kmem_cache_destroy(cachep);
- cachep = NULL;
- goto oops;
+ return NULL;
}
if (flags & SLAB_DEBUG_OBJECTS) {
@@ -2521,16 +2519,12 @@ kmem_cache_create (const char *name, siz
/* cache setup completed, link it into the list */
list_add(&cachep->list, &cache_chain);
oops:
- if (!cachep && (flags & SLAB_PANIC))
- panic("kmem_cache_create(): failed to create slab `%s'\n",
- name);
if (slab_is_available()) {
mutex_unlock(&cache_chain_mutex);
put_online_cpus();
}
return cachep;
}
-EXPORT_SYMBOL(kmem_cache_create);
#if DEBUG
static void check_irq_off(void)
Index: linux-2.6/mm/slab_common.c
===================================================================
--- /dev/null 1970-01-01 00:00:00.000000000 +0000
+++ linux-2.6/mm/slab_common.c 2012-06-13 03:45:46.217476101 -0500
@@ -0,0 +1,67 @@
+/*
+ * Slab allocator functions that are independent of the allocator strategy
+ *
+ * (C) 2012 Christoph Lameter <cl@linux.com>
+ */
+#include <linux/slab.h>
+
+#include <linux/mm.h>
+#include <linux/poison.h>
+#include <linux/interrupt.h>
+#include <linux/memory.h>
+#include <linux/compiler.h>
+#include <linux/module.h>
+
+#include <asm/cacheflush.h>
+#include <asm/tlbflush.h>
+#include <asm/page.h>
+
+/*
+ * kmem_cache_create - Create a cache.
+ * @name: A string which is used in /proc/slabinfo to identify this cache.
+ * @size: The size of objects to be created in this cache.
+ * @align: The required alignment for the objects.
+ * @flags: SLAB flags
+ * @ctor: A constructor for the objects.
+ *
+ * Returns a ptr to the cache on success, NULL on failure.
+ * Cannot be called within a interrupt, but can be interrupted.
+ * The @ctor is run when new pages are allocated by the cache.
+ *
+ * The flags are
+ *
+ * %SLAB_POISON - Poison the slab with a known test pattern (a5a5a5a5)
+ * to catch references to uninitialised memory.
+ *
+ * %SLAB_RED_ZONE - Insert `Red' zones around the allocated memory to check
+ * for buffer overruns.
+ *
+ * %SLAB_HWCACHE_ALIGN - Align the objects in this cache to a hardware
+ * cacheline. This can be beneficial if you're counting cycles as closely
+ * as davem.
+ */
+
+struct kmem_cache *kmem_cache_create(const char *name, size_t size, size_t align,
+ unsigned long flags, void (*ctor)(void *))
+{
+ struct kmem_cache *s = NULL;
+
+#ifdef CONFIG_DEBUG_VM
+ if (!name || in_interrupt() || size < sizeof(void *) ||
+ size > KMALLOC_MAX_SIZE) {
+ printk(KERN_ERR "kmem_cache_create(%s) integrity check"
+ " failed\n", name);
+ goto out;
+ }
+#endif
+
+ s = __kmem_cache_create(name, size, align, flags, ctor);
+
+out:
+ if (!s && (flags & SLAB_PANIC))
+ panic("kmem_cache_create: Failed to create slab '%s'\n", name);
+
+ return s;
+}
+EXPORT_SYMBOL(kmem_cache_create);
+
Index: linux-2.6/mm/slub.c
===================================================================
--- linux-2.6.orig/mm/slub.c 2012-06-13 03:44:40.597477460 -0500
+++ linux-2.6/mm/slub.c 2012-06-13 03:45:46.177476101 -0500
@@ -3934,15 +3934,12 @@ static struct kmem_cache *find_mergeable
return NULL;
}
-struct kmem_cache *kmem_cache_create(const char *name, size_t size,
+struct kmem_cache *__kmem_cache_create(const char *name, size_t size,
size_t align, unsigned long flags, void (*ctor)(void *))
{
struct kmem_cache *s;
char *n;
- if (WARN_ON(!name))
- return NULL;
-
down_write(&slub_lock);
s = find_mergeable(size, align, flags, name, ctor);
if (s) {
@@ -3986,14 +3983,8 @@ struct kmem_cache *kmem_cache_create(con
kfree(n);
err:
up_write(&slub_lock);
-
- if (flags & SLAB_PANIC)
- panic("Cannot create slabcache %s\n", name);
- else
- s = NULL;
return s;
}
-EXPORT_SYMBOL(kmem_cache_create);
#ifdef CONFIG_SMP
/*
Index: linux-2.6/mm/slob.c
===================================================================
--- linux-2.6.orig/mm/slob.c 2012-06-13 03:44:40.589477460 -0500
+++ linux-2.6/mm/slob.c 2012-06-13 03:45:46.125476103 -0500
@@ -506,7 +506,7 @@ size_t ksize(const void *block)
}
EXPORT_SYMBOL(ksize);
-struct kmem_cache *kmem_cache_create(const char *name, size_t size,
+struct kmem_cache *__kmem_cache_create(const char *name, size_t size,
size_t align, unsigned long flags, void (*ctor)(void *))
{
struct kmem_cache *c;
@@ -529,13 +529,11 @@ struct kmem_cache *kmem_cache_create(con
c->align = ARCH_SLAB_MINALIGN;
if (c->align < align)
c->align = align;
- } else if (flags & SLAB_PANIC)
- panic("Cannot create slab cache %s\n", name);
- kmemleak_alloc(c, sizeof(struct kmem_cache), 1, GFP_KERNEL);
+ kmemleak_alloc(c, sizeof(struct kmem_cache), 1, GFP_KERNEL);
+ }
return c;
}
-EXPORT_SYMBOL(kmem_cache_create);
void kmem_cache_destroy(struct kmem_cache *c)
{
Index: linux-2.6/mm/Makefile
===================================================================
--- linux-2.6.orig/mm/Makefile 2012-06-13 03:44:14.489478001 -0500
+++ linux-2.6/mm/Makefile 2012-06-13 03:44:40.665477459 -0500
@@ -16,7 +16,8 @@ obj-y := filemap.o mempool.o oom_kill.
readahead.o swap.o truncate.o vmscan.o shmem.o \
prio_tree.o util.o mmzone.o vmstat.o backing-dev.o \
page_isolation.o mm_init.o mmu_context.o percpu.o \
- compaction.o $(mmu-y)
+ compaction.o slab_common.o $(mmu-y)
+
obj-y += init-mm.o
ifdef CONFIG_NO_BOOTMEM
Index: linux-2.6/include/linux/slab.h
===================================================================
--- linux-2.6.orig/include/linux/slab.h 2012-06-13 03:44:40.589477460 -0500
+++ linux-2.6/include/linux/slab.h 2012-06-13 03:45:46.101476103 -0500
@@ -130,6 +130,10 @@ int kmem_cache_shrink(struct kmem_cache
void kmem_cache_free(struct kmem_cache *, void *);
unsigned int kmem_cache_size(struct kmem_cache *);
+/* Slab internal function */
+struct kmem_cache *__kmem_cache_create(const char *, size_t, size_t,
+ unsigned long,
+ void (*)(void *));
/*
* Please use this macro to create slab caches. Simply specify the
* name of the structure and maybe some flags that are listed above.
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>
^ permalink raw reply [flat|nested] 40+ messages in thread* Re: Common [08/20] Extract common code for kmem_cache_create()
2012-06-13 15:24 ` Common [08/20] Extract common code for kmem_cache_create() Christoph Lameter
@ 2012-06-14 8:15 ` Glauber Costa
2012-06-14 14:18 ` Christoph Lameter
0 siblings, 1 reply; 40+ messages in thread
From: Glauber Costa @ 2012-06-14 8:15 UTC (permalink / raw)
To: Christoph Lameter
Cc: Pekka Enberg, linux-mm, David Rientjes, Matt Mackall, Joonsoo Kim
On 06/13/2012 07:24 PM, Christoph Lameter wrote:
> +struct kmem_cache *kmem_cache_create(const char *name, size_t size, size_t align,
> + unsigned long flags, void (*ctor)(void *))
> +{
> + struct kmem_cache *s = NULL;
> +
> +#ifdef CONFIG_DEBUG_VM
> + if (!name || in_interrupt() || size< sizeof(void *) ||
> + size> KMALLOC_MAX_SIZE) {
> + printk(KERN_ERR "kmem_cache_create(%s) integrity check"
> + " failed\n", name);
> + goto out;
> + }
> +#endif
Not really a BUG, but label out is not used if !CONFIG_DEBUG_VM. Suggest
testing for the slab panic flag here, and panicing if we need to.
> +
> + s = __kmem_cache_create(name, size, align, flags, ctor);
> +
> +out:
> + if (!s&& (flags& SLAB_PANIC))
> + panic("kmem_cache_create: Failed to create slab '%s'\n", name);
> +
> + return s;
> +}
> +EXPORT_SYMBOL(kmem_cache_create);
> +
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>
^ permalink raw reply [flat|nested] 40+ messages in thread* Re: Common [08/20] Extract common code for kmem_cache_create()
2012-06-14 8:15 ` Glauber Costa
@ 2012-06-14 14:18 ` Christoph Lameter
2012-06-14 14:20 ` Glauber Costa
0 siblings, 1 reply; 40+ messages in thread
From: Christoph Lameter @ 2012-06-14 14:18 UTC (permalink / raw)
To: Glauber Costa
Cc: Pekka Enberg, linux-mm, David Rientjes, Matt Mackall, Joonsoo Kim
On Thu, 14 Jun 2012, Glauber Costa wrote:
> On 06/13/2012 07:24 PM, Christoph Lameter wrote:
> > +struct kmem_cache *kmem_cache_create(const char *name, size_t size, size_t
> > align,
> > + unsigned long flags, void (*ctor)(void *))
> > +{
> > + struct kmem_cache *s = NULL;
> > +
> > +#ifdef CONFIG_DEBUG_VM
> > + if (!name || in_interrupt() || size< sizeof(void *) ||
> > + size> KMALLOC_MAX_SIZE) {
> > + printk(KERN_ERR "kmem_cache_create(%s) integrity check"
> > + " failed\n", name);
> > + goto out;
> > + }
> > +#endif
>
> Not really a BUG, but label out is not used if !CONFIG_DEBUG_VM. Suggest
> testing for the slab panic flag here, and panicing if we need to.
Hmmm.. That is quite sensitive. A change here will cause later patches in
the series to have issues. Maybe its best to put an #ifdef around the
label until a later patch that makes use of out: from code that is not
#ifdefed.
Subject: Add #ifdef to avoid warning about unused label
out: is only used if CONFIG_DEBUG_VM is enabled.
Signed-off-by: Christoph Lameter <cl@linux.com>
Index: linux-2.6/mm/slab_common.c
===================================================================
--- linux-2.6.orig/mm/slab_common.c 2012-06-14 03:16:06.778702087 -0500
+++ linux-2.6/mm/slab_common.c 2012-06-14 03:16:01.054702201 -0500
@@ -57,7 +57,9 @@ struct kmem_cache *kmem_cache_create(con
s = __kmem_cache_create(name, size, align, flags, ctor);
+#ifdef CONFIG_DEBUG_VM
out:
+#endif
if (!s && (flags & SLAB_PANIC))
panic("kmem_cache_create: Failed to create slab '%s'\n", name);
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>
^ permalink raw reply [flat|nested] 40+ messages in thread* Re: Common [08/20] Extract common code for kmem_cache_create()
2012-06-14 14:18 ` Christoph Lameter
@ 2012-06-14 14:20 ` Glauber Costa
2012-06-14 14:46 ` Christoph Lameter
0 siblings, 1 reply; 40+ messages in thread
From: Glauber Costa @ 2012-06-14 14:20 UTC (permalink / raw)
To: Christoph Lameter
Cc: Pekka Enberg, linux-mm, David Rientjes, Matt Mackall, Joonsoo Kim
On 06/14/2012 06:18 PM, Christoph Lameter wrote:
> On Thu, 14 Jun 2012, Glauber Costa wrote:
>
>> On 06/13/2012 07:24 PM, Christoph Lameter wrote:
>>> +struct kmem_cache *kmem_cache_create(const char *name, size_t size, size_t
>>> align,
>>> + unsigned long flags, void (*ctor)(void *))
>>> +{
>>> + struct kmem_cache *s = NULL;
>>> +
>>> +#ifdef CONFIG_DEBUG_VM
>>> + if (!name || in_interrupt() || size< sizeof(void *) ||
>>> + size> KMALLOC_MAX_SIZE) {
>>> + printk(KERN_ERR "kmem_cache_create(%s) integrity check"
>>> + " failed\n", name);
>>> + goto out;
>>> + }
>>> +#endif
>>
>> Not really a BUG, but label out is not used if !CONFIG_DEBUG_VM. Suggest
>> testing for the slab panic flag here, and panicing if we need to.
>
> Hmmm.. That is quite sensitive. A change here will cause later patches in
> the series to have issues. Maybe its best to put an #ifdef around the
> label until a later patch that makes use of out: from code that is not
> #ifdefed.
>
>
> Subject: Add #ifdef to avoid warning about unused label
>
> out: is only used if CONFIG_DEBUG_VM is enabled.
>
> Signed-off-by: Christoph Lameter<cl@linux.com>
>
> Index: linux-2.6/mm/slab_common.c
> ===================================================================
> --- linux-2.6.orig/mm/slab_common.c 2012-06-14 03:16:06.778702087 -0500
> +++ linux-2.6/mm/slab_common.c 2012-06-14 03:16:01.054702201 -0500
> @@ -57,7 +57,9 @@ struct kmem_cache *kmem_cache_create(con
>
> s = __kmem_cache_create(name, size, align, flags, ctor);
>
> +#ifdef CONFIG_DEBUG_VM
> out:
> +#endif
> if (!s&& (flags& SLAB_PANIC))
> panic("kmem_cache_create: Failed to create slab '%s'\n", name);
>
That's how my code reads:
#ifdef CONFIG_DEBUG_VM
if (!name || in_interrupt() || size < sizeof(void *) ||
size KMALLOC_MAX_SIZE) {
if ((flags & SLAB_PANIC))
panic("kmem_cache_create(%s) integrity check failed\n", name);
printk(KERN_ERR "kmem_cache_create(%s) integrity check failed\n",
name);
return NULL;
}
#endif
How can it put any patch later than this in trouble ?
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>
^ permalink raw reply [flat|nested] 40+ messages in thread* Re: Common [08/20] Extract common code for kmem_cache_create()
2012-06-14 14:20 ` Glauber Costa
@ 2012-06-14 14:46 ` Christoph Lameter
0 siblings, 0 replies; 40+ messages in thread
From: Christoph Lameter @ 2012-06-14 14:46 UTC (permalink / raw)
To: Glauber Costa
Cc: Pekka Enberg, linux-mm, David Rientjes, Matt Mackall, Joonsoo Kim
On Thu, 14 Jun 2012, Glauber Costa wrote:
> That's how my code reads:
>
> #ifdef CONFIG_DEBUG_VM
> if (!name || in_interrupt() || size < sizeof(void *) ||
> size KMALLOC_MAX_SIZE) {
>
> if ((flags & SLAB_PANIC))
> panic("kmem_cache_create(%s) integrity check failed\n", name);
> printk(KERN_ERR "kmem_cache_create(%s) integrity check failed\n",
> name);
> return NULL;
> }
> #endif
>
> How can it put any patch later than this in trouble ?
Well this is duplicating the exit handling which I would like to avoid.
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>
^ permalink raw reply [flat|nested] 40+ messages in thread
* Common [09/20] Common definition for boot state of the slab allocators
2012-06-13 15:24 Common [00/20] Sl[auo]b: Common code rework V5 (for merge) Christoph Lameter
` (6 preceding siblings ...)
2012-06-13 15:24 ` Common [08/20] Extract common code for kmem_cache_create() Christoph Lameter
@ 2012-06-13 15:25 ` Christoph Lameter
2012-06-13 15:25 ` Common [10/20] Use a common mutex definition Christoph Lameter
` (9 subsequent siblings)
17 siblings, 0 replies; 40+ messages in thread
From: Christoph Lameter @ 2012-06-13 15:25 UTC (permalink / raw)
To: Pekka Enberg
Cc: linux-mm, David Rientjes, Matt Mackall, Glauber Costa, Joonsoo Kim
[-- Attachment #1: slab_internal --]
[-- Type: text/plain, Size: 9080 bytes --]
All allocators have some sort of support for the bootstrap status.
Setup a common definition for the boot states and make all slab
allocators use that definition.
Reviewed-by: Glauber Costa <glommer@parallels.com>
Reviewed-by: Joonsoo Kim <js1304@gmail.com>
Signed-off-by: Christoph Lameter <cl@linux.com>
---
include/linux/slab.h | 4 ----
mm/slab.c | 42 +++++++++++-------------------------------
mm/slab.h | 30 ++++++++++++++++++++++++++++++
mm/slab_common.c | 9 +++++++++
mm/slob.c | 14 +++++---------
mm/slub.c | 21 +++++----------------
6 files changed, 60 insertions(+), 60 deletions(-)
Index: linux-2.6/mm/slab.c
===================================================================
--- linux-2.6.orig/mm/slab.c 2012-06-13 03:44:40.661477459 -0500
+++ linux-2.6/mm/slab.c 2012-06-13 03:45:43.373476160 -0500
@@ -87,6 +87,7 @@
*/
#include <linux/slab.h>
+#include "slab.h"
#include <linux/mm.h>
#include <linux/poison.h>
#include <linux/swap.h>
@@ -565,27 +566,6 @@ static struct kmem_cache cache_cache = {
#define BAD_ALIEN_MAGIC 0x01020304ul
-/*
- * chicken and egg problem: delay the per-cpu array allocation
- * until the general caches are up.
- */
-static enum {
- NONE,
- PARTIAL_AC,
- PARTIAL_L3,
- EARLY,
- LATE,
- FULL
-} g_cpucache_up;
-
-/*
- * used by boot code to determine if it can use slab based allocator
- */
-int slab_is_available(void)
-{
- return g_cpucache_up >= EARLY;
-}
-
#ifdef CONFIG_LOCKDEP
/*
@@ -651,7 +631,7 @@ static void init_node_lock_keys(int q)
{
struct cache_sizes *s = malloc_sizes;
- if (g_cpucache_up < LATE)
+ if (slab_state < UP)
return;
for (s = malloc_sizes; s->cs_size != ULONG_MAX; s++) {
@@ -1649,14 +1629,14 @@ void __init kmem_cache_init(void)
}
}
- g_cpucache_up = EARLY;
+ slab_state = UP;
}
void __init kmem_cache_init_late(void)
{
struct kmem_cache *cachep;
- g_cpucache_up = LATE;
+ slab_state = UP;
/* Annotate slab for lockdep -- annotate the malloc caches */
init_lock_keys();
@@ -1669,7 +1649,7 @@ void __init kmem_cache_init_late(void)
mutex_unlock(&cache_chain_mutex);
/* Done! */
- g_cpucache_up = FULL;
+ slab_state = FULL;
/*
* Register a cpu startup notifier callback that initializes
@@ -2167,10 +2147,10 @@ static size_t calculate_slab_order(struc
static int __init_refok setup_cpu_cache(struct kmem_cache *cachep, gfp_t gfp)
{
- if (g_cpucache_up == FULL)
+ if (slab_state == FULL)
return enable_cpucache(cachep, gfp);
- if (g_cpucache_up == NONE) {
+ if (slab_state == DOWN) {
/*
* Note: the first kmem_cache_create must create the cache
* that's used by kmalloc(24), otherwise the creation of
@@ -2185,16 +2165,16 @@ static int __init_refok setup_cpu_cache(
*/
set_up_list3s(cachep, SIZE_AC);
if (INDEX_AC == INDEX_L3)
- g_cpucache_up = PARTIAL_L3;
+ slab_state = PARTIAL_L3;
else
- g_cpucache_up = PARTIAL_AC;
+ slab_state = PARTIAL_ARRAYCACHE;
} else {
cachep->array[smp_processor_id()] =
kmalloc(sizeof(struct arraycache_init), gfp);
- if (g_cpucache_up == PARTIAL_AC) {
+ if (slab_state == PARTIAL_ARRAYCACHE) {
set_up_list3s(cachep, SIZE_L3);
- g_cpucache_up = PARTIAL_L3;
+ slab_state = PARTIAL_L3;
} else {
int node;
for_each_online_node(node) {
Index: linux-2.6/mm/slab.h
===================================================================
--- /dev/null 1970-01-01 00:00:00.000000000 +0000
+++ linux-2.6/mm/slab.h 2012-06-13 03:45:43.425476158 -0500
@@ -0,0 +1,30 @@
+#ifndef MM_SLAB_H
+#define MM_SLAB_H
+/*
+ * Internal slab definitions
+ */
+
+/*
+ * State of the slab allocator.
+ *
+ * This is used to describe the states of the allocator during bootup.
+ * Allocators use this to gradually bootstrap themselves. Most allocators
+ * have the problem that the structures used for managing slab caches are
+ * allocated from slab caches themselves.
+ */
+enum slab_state {
+ DOWN, /* No slab functionality yet */
+ PARTIAL, /* SLUB: kmem_cache_node available */
+ PARTIAL_ARRAYCACHE, /* SLAB: kmalloc size for arraycache available */
+ PARTIAL_L3, /* SLAB: kmalloc size for l3 struct available */
+ UP, /* Slab caches usable but not all extras yet */
+ FULL /* Everything is working */
+};
+
+extern enum slab_state slab_state;
+
+struct kmem_cache *__kmem_cache_create(const char *name, size_t size,
+ size_t align, unsigned long flags, void (*ctor)(void *));
+
+#endif
+
Index: linux-2.6/mm/slob.c
===================================================================
--- linux-2.6.orig/mm/slob.c 2012-06-13 03:44:40.665477459 -0500
+++ linux-2.6/mm/slob.c 2012-06-13 03:45:31.249476411 -0500
@@ -59,6 +59,8 @@
#include <linux/kernel.h>
#include <linux/slab.h>
+#include "slab.h"
+
#include <linux/mm.h>
#include <linux/swap.h> /* struct reclaim_state */
#include <linux/cache.h>
@@ -531,6 +533,7 @@ struct kmem_cache *__kmem_cache_create(c
c->align = align;
kmemleak_alloc(c, sizeof(struct kmem_cache), 1, GFP_KERNEL);
+ c->refcount = 1;
}
return c;
}
@@ -616,19 +619,12 @@ int kmem_cache_shrink(struct kmem_cache
}
EXPORT_SYMBOL(kmem_cache_shrink);
-static unsigned int slob_ready __read_mostly;
-
-int slab_is_available(void)
-{
- return slob_ready;
-}
-
void __init kmem_cache_init(void)
{
- slob_ready = 1;
+ slab_state = UP;
}
void __init kmem_cache_init_late(void)
{
- /* Nothing to do */
+ slab_state = FULL;
}
Index: linux-2.6/mm/slub.c
===================================================================
--- linux-2.6.orig/mm/slub.c 2012-06-13 03:44:40.665477459 -0500
+++ linux-2.6/mm/slub.c 2012-06-13 03:45:43.397476159 -0500
@@ -16,6 +16,7 @@
#include <linux/interrupt.h>
#include <linux/bitops.h>
#include <linux/slab.h>
+#include "slab.h"
#include <linux/proc_fs.h>
#include <linux/seq_file.h>
#include <linux/kmemcheck.h>
@@ -182,13 +183,6 @@ static int kmem_size = sizeof(struct kme
static struct notifier_block slab_notifier;
#endif
-static enum {
- DOWN, /* No slab functionality available */
- PARTIAL, /* Kmem_cache_node works */
- UP, /* Everything works but does not show up in sysfs */
- SYSFS /* Sysfs up */
-} slab_state = DOWN;
-
/* A list of all slab caches on the system */
static DECLARE_RWSEM(slub_lock);
static LIST_HEAD(slab_caches);
@@ -237,11 +231,6 @@ static inline void stat(const struct kme
* Core slab cache functions
*******************************************************************/
-int slab_is_available(void)
-{
- return slab_state >= UP;
-}
-
static inline struct kmem_cache_node *get_node(struct kmem_cache *s, int node)
{
return s->node[node];
@@ -5288,7 +5277,7 @@ static int sysfs_slab_add(struct kmem_ca
const char *name;
int unmergeable;
- if (slab_state < SYSFS)
+ if (slab_state < FULL)
/* Defer until later */
return 0;
@@ -5333,7 +5322,7 @@ static int sysfs_slab_add(struct kmem_ca
static void sysfs_slab_remove(struct kmem_cache *s)
{
- if (slab_state < SYSFS)
+ if (slab_state < FULL)
/*
* Sysfs has not been setup yet so no need to remove the
* cache from sysfs.
@@ -5361,7 +5350,7 @@ static int sysfs_slab_alias(struct kmem_
{
struct saved_alias *al;
- if (slab_state == SYSFS) {
+ if (slab_state == FULL) {
/*
* If we have a leftover link then remove it.
*/
@@ -5394,7 +5383,7 @@ static int __init slab_sysfs_init(void)
return -ENOSYS;
}
- slab_state = SYSFS;
+ slab_state = FULL;
list_for_each_entry(s, &slab_caches, list) {
err = sysfs_slab_add(s);
Index: linux-2.6/mm/slab_common.c
===================================================================
--- linux-2.6.orig/mm/slab_common.c 2012-06-13 03:44:40.661477459 -0500
+++ linux-2.6/mm/slab_common.c 2012-06-13 03:45:43.453476158 -0500
@@ -16,6 +16,10 @@
#include <asm/tlbflush.h>
#include <asm/page.h>
+#include "slab.h"
+
+enum slab_state slab_state;
+
/*
* kmem_cache_create - Create a cache.
* @name: A string which is used in /proc/slabinfo to identify this cache.
@@ -65,3 +69,8 @@ out:
}
EXPORT_SYMBOL(kmem_cache_create);
+int slab_is_available(void)
+{
+ return slab_state >= UP;
+}
+
Index: linux-2.6/include/linux/slab.h
===================================================================
--- linux-2.6.orig/include/linux/slab.h 2012-06-13 03:44:40.665477459 -0500
+++ linux-2.6/include/linux/slab.h 2012-06-13 03:44:40.705477458 -0500
@@ -130,10 +130,6 @@ int kmem_cache_shrink(struct kmem_cache
void kmem_cache_free(struct kmem_cache *, void *);
unsigned int kmem_cache_size(struct kmem_cache *);
-/* Slab internal function */
-struct kmem_cache *__kmem_cache_create(const char *, size_t, size_t,
- unsigned long,
- void (*)(void *));
/*
* Please use this macro to create slab caches. Simply specify the
* name of the structure and maybe some flags that are listed above.
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>
^ permalink raw reply [flat|nested] 40+ messages in thread* Common [10/20] Use a common mutex definition
2012-06-13 15:24 Common [00/20] Sl[auo]b: Common code rework V5 (for merge) Christoph Lameter
` (7 preceding siblings ...)
2012-06-13 15:25 ` Common [09/20] Common definition for boot state of the slab allocators Christoph Lameter
@ 2012-06-13 15:25 ` Christoph Lameter
2012-06-13 15:25 ` Common [11/20] Move kmem_cache_create mutex handling to common code Christoph Lameter
` (8 subsequent siblings)
17 siblings, 0 replies; 40+ messages in thread
From: Christoph Lameter @ 2012-06-13 15:25 UTC (permalink / raw)
To: Pekka Enberg
Cc: linux-mm, David Rientjes, Matt Mackall, Glauber Costa, Joonsoo Kim
[-- Attachment #1: common_mutex --]
[-- Type: text/plain, Size: 19007 bytes --]
Use the mutex definition from SLAB and make it the common way to take a sleeping lock.
This has the effect of using a mutex instead of a rw semaphore for SLUB.
SLOB gains the use of a mutex for kmem_cache_create serialization.
Not needed now but SLOB may acquire some more features later (like slabinfo
/ sysfs support) through the expansion of the common code that will
need this.
Reviewed-by: Glauber Costa <glommer@parallels.com>
Reviewed-by: Joonsoo Kim <js1304@gmail.com>
Signed-off-by: Christoph Lameter <cl@linux.com>
---
mm/slab.c | 108 +++++++++++++++++++++++++------------------------------
mm/slab.h | 4 ++
mm/slab_common.c | 2 +
mm/slub.c | 54 ++++++++++++---------------
4 files changed, 82 insertions(+), 86 deletions(-)
Index: linux-2.6/mm/slab.c
===================================================================
--- linux-2.6.orig/mm/slab.c 2012-06-13 03:44:40.701477458 -0500
+++ linux-2.6/mm/slab.c 2012-06-13 03:45:40.641476216 -0500
@@ -68,7 +68,7 @@
* Further notes from the original documentation:
*
* 11 April '97. Started multi-threading - markhe
- * The global cache-chain is protected by the mutex 'cache_chain_mutex'.
+ * The global cache-chain is protected by the mutex 'slab_mutex'.
* The sem is only needed when accessing/extending the cache-chain, which
* can never happen inside an interrupt (kmem_cache_create(),
* kmem_cache_shrink() and kmem_cache_reap()).
@@ -671,12 +671,6 @@ static void slab_set_debugobj_lock_class
}
#endif
-/*
- * Guard access to the cache-chain.
- */
-static DEFINE_MUTEX(cache_chain_mutex);
-static struct list_head cache_chain;
-
static DEFINE_PER_CPU(struct delayed_work, slab_reap_work);
static inline struct array_cache *cpu_cache_get(struct kmem_cache *cachep)
@@ -1100,7 +1094,7 @@ static inline int cache_free_alien(struc
* When hotplugging memory or a cpu, existing nodelists are not replaced if
* already in use.
*
- * Must hold cache_chain_mutex.
+ * Must hold slab_mutex.
*/
static int init_cache_nodelists_node(int node)
{
@@ -1108,7 +1102,7 @@ static int init_cache_nodelists_node(int
struct kmem_list3 *l3;
const int memsize = sizeof(struct kmem_list3);
- list_for_each_entry(cachep, &cache_chain, list) {
+ list_for_each_entry(cachep, &slab_caches, list) {
/*
* Set up the size64 kmemlist for cpu before we can
* begin anything. Make sure some other cpu on this
@@ -1124,7 +1118,7 @@ static int init_cache_nodelists_node(int
/*
* The l3s don't come and go as CPUs come and
- * go. cache_chain_mutex is sufficient
+ * go. slab_mutex is sufficient
* protection here.
*/
cachep->nodelists[node] = l3;
@@ -1146,7 +1140,7 @@ static void __cpuinit cpuup_canceled(lon
int node = cpu_to_mem(cpu);
const struct cpumask *mask = cpumask_of_node(node);
- list_for_each_entry(cachep, &cache_chain, list) {
+ list_for_each_entry(cachep, &slab_caches, list) {
struct array_cache *nc;
struct array_cache *shared;
struct array_cache **alien;
@@ -1196,7 +1190,7 @@ free_array_cache:
* the respective cache's slabs, now we can go ahead and
* shrink each nodelist to its limit.
*/
- list_for_each_entry(cachep, &cache_chain, list) {
+ list_for_each_entry(cachep, &slab_caches, list) {
l3 = cachep->nodelists[node];
if (!l3)
continue;
@@ -1225,7 +1219,7 @@ static int __cpuinit cpuup_prepare(long
* Now we can go ahead with allocating the shared arrays and
* array caches
*/
- list_for_each_entry(cachep, &cache_chain, list) {
+ list_for_each_entry(cachep, &slab_caches, list) {
struct array_cache *nc;
struct array_cache *shared = NULL;
struct array_cache **alien = NULL;
@@ -1293,9 +1287,9 @@ static int __cpuinit cpuup_callback(stru
switch (action) {
case CPU_UP_PREPARE:
case CPU_UP_PREPARE_FROZEN:
- mutex_lock(&cache_chain_mutex);
+ mutex_lock(&slab_mutex);
err = cpuup_prepare(cpu);
- mutex_unlock(&cache_chain_mutex);
+ mutex_unlock(&slab_mutex);
break;
case CPU_ONLINE:
case CPU_ONLINE_FROZEN:
@@ -1305,7 +1299,7 @@ static int __cpuinit cpuup_callback(stru
case CPU_DOWN_PREPARE:
case CPU_DOWN_PREPARE_FROZEN:
/*
- * Shutdown cache reaper. Note that the cache_chain_mutex is
+ * Shutdown cache reaper. Note that the slab_mutex is
* held so that if cache_reap() is invoked it cannot do
* anything expensive but will only modify reap_work
* and reschedule the timer.
@@ -1332,9 +1326,9 @@ static int __cpuinit cpuup_callback(stru
#endif
case CPU_UP_CANCELED:
case CPU_UP_CANCELED_FROZEN:
- mutex_lock(&cache_chain_mutex);
+ mutex_lock(&slab_mutex);
cpuup_canceled(cpu);
- mutex_unlock(&cache_chain_mutex);
+ mutex_unlock(&slab_mutex);
break;
}
return notifier_from_errno(err);
@@ -1350,14 +1344,14 @@ static struct notifier_block __cpuinitda
* Returns -EBUSY if all objects cannot be drained so that the node is not
* removed.
*
- * Must hold cache_chain_mutex.
+ * Must hold slab_mutex.
*/
static int __meminit drain_cache_nodelists_node(int node)
{
struct kmem_cache *cachep;
int ret = 0;
- list_for_each_entry(cachep, &cache_chain, list) {
+ list_for_each_entry(cachep, &slab_caches, list) {
struct kmem_list3 *l3;
l3 = cachep->nodelists[node];
@@ -1388,14 +1382,14 @@ static int __meminit slab_memory_callbac
switch (action) {
case MEM_GOING_ONLINE:
- mutex_lock(&cache_chain_mutex);
+ mutex_lock(&slab_mutex);
ret = init_cache_nodelists_node(nid);
- mutex_unlock(&cache_chain_mutex);
+ mutex_unlock(&slab_mutex);
break;
case MEM_GOING_OFFLINE:
- mutex_lock(&cache_chain_mutex);
+ mutex_lock(&slab_mutex);
ret = drain_cache_nodelists_node(nid);
- mutex_unlock(&cache_chain_mutex);
+ mutex_unlock(&slab_mutex);
break;
case MEM_ONLINE:
case MEM_OFFLINE:
@@ -1499,8 +1493,8 @@ void __init kmem_cache_init(void)
node = numa_mem_id();
/* 1) create the cache_cache */
- INIT_LIST_HEAD(&cache_chain);
- list_add(&cache_cache.list, &cache_chain);
+ INIT_LIST_HEAD(&slab_caches);
+ list_add(&cache_cache.list, &slab_caches);
cache_cache.colour_off = cache_line_size();
cache_cache.array[smp_processor_id()] = &initarray_cache.cache;
cache_cache.nodelists[node] = &initkmem_list3[CACHE_CACHE + node];
@@ -1642,11 +1636,11 @@ void __init kmem_cache_init_late(void)
init_lock_keys();
/* 6) resize the head arrays to their final sizes */
- mutex_lock(&cache_chain_mutex);
- list_for_each_entry(cachep, &cache_chain, list)
+ mutex_lock(&slab_mutex);
+ list_for_each_entry(cachep, &slab_caches, list)
if (enable_cpucache(cachep, GFP_NOWAIT))
BUG();
- mutex_unlock(&cache_chain_mutex);
+ mutex_unlock(&slab_mutex);
/* Done! */
slab_state = FULL;
@@ -2250,10 +2244,10 @@ __kmem_cache_create (const char *name, s
*/
if (slab_is_available()) {
get_online_cpus();
- mutex_lock(&cache_chain_mutex);
+ mutex_lock(&slab_mutex);
}
- list_for_each_entry(pc, &cache_chain, list) {
+ list_for_each_entry(pc, &slab_caches, list) {
char tmp;
int res;
@@ -2497,10 +2491,10 @@ __kmem_cache_create (const char *name, s
}
/* cache setup completed, link it into the list */
- list_add(&cachep->list, &cache_chain);
+ list_add(&cachep->list, &slab_caches);
oops:
if (slab_is_available()) {
- mutex_unlock(&cache_chain_mutex);
+ mutex_unlock(&slab_mutex);
put_online_cpus();
}
return cachep;
@@ -2619,7 +2613,7 @@ out:
return nr_freed;
}
-/* Called with cache_chain_mutex held to protect against cpu hotplug */
+/* Called with slab_mutex held to protect against cpu hotplug */
static int __cache_shrink(struct kmem_cache *cachep)
{
int ret = 0, i = 0;
@@ -2654,9 +2648,9 @@ int kmem_cache_shrink(struct kmem_cache
BUG_ON(!cachep || in_interrupt());
get_online_cpus();
- mutex_lock(&cache_chain_mutex);
+ mutex_lock(&slab_mutex);
ret = __cache_shrink(cachep);
- mutex_unlock(&cache_chain_mutex);
+ mutex_unlock(&slab_mutex);
put_online_cpus();
return ret;
}
@@ -2684,15 +2678,15 @@ void kmem_cache_destroy(struct kmem_cach
/* Find the cache in the chain of caches. */
get_online_cpus();
- mutex_lock(&cache_chain_mutex);
+ mutex_lock(&slab_mutex);
/*
* the chain is never empty, cache_cache is never destroyed
*/
list_del(&cachep->list);
if (__cache_shrink(cachep)) {
slab_error(cachep, "Can't free all objects");
- list_add(&cachep->list, &cache_chain);
- mutex_unlock(&cache_chain_mutex);
+ list_add(&cachep->list, &slab_caches);
+ mutex_unlock(&slab_mutex);
put_online_cpus();
return;
}
@@ -2701,7 +2695,7 @@ void kmem_cache_destroy(struct kmem_cach
rcu_barrier();
__kmem_cache_destroy(cachep);
- mutex_unlock(&cache_chain_mutex);
+ mutex_unlock(&slab_mutex);
put_online_cpus();
}
EXPORT_SYMBOL(kmem_cache_destroy);
@@ -4014,7 +4008,7 @@ static void do_ccupdate_local(void *info
new->new[smp_processor_id()] = old;
}
-/* Always called with the cache_chain_mutex held */
+/* Always called with the slab_mutex held */
static int do_tune_cpucache(struct kmem_cache *cachep, int limit,
int batchcount, int shared, gfp_t gfp)
{
@@ -4058,7 +4052,7 @@ static int do_tune_cpucache(struct kmem_
return alloc_kmemlist(cachep, gfp);
}
-/* Called with cache_chain_mutex held always */
+/* Called with slab_mutex held always */
static int enable_cpucache(struct kmem_cache *cachep, gfp_t gfp)
{
int err;
@@ -4160,11 +4154,11 @@ static void cache_reap(struct work_struc
int node = numa_mem_id();
struct delayed_work *work = to_delayed_work(w);
- if (!mutex_trylock(&cache_chain_mutex))
+ if (!mutex_trylock(&slab_mutex))
/* Give up. Setup the next iteration. */
goto out;
- list_for_each_entry(searchp, &cache_chain, list) {
+ list_for_each_entry(searchp, &slab_caches, list) {
check_irq_on();
/*
@@ -4202,7 +4196,7 @@ next:
cond_resched();
}
check_irq_on();
- mutex_unlock(&cache_chain_mutex);
+ mutex_unlock(&slab_mutex);
next_reap_node();
out:
/* Set up the next iteration */
@@ -4238,21 +4232,21 @@ static void *s_start(struct seq_file *m,
{
loff_t n = *pos;
- mutex_lock(&cache_chain_mutex);
+ mutex_lock(&slab_mutex);
if (!n)
print_slabinfo_header(m);
- return seq_list_start(&cache_chain, *pos);
+ return seq_list_start(&slab_caches, *pos);
}
static void *s_next(struct seq_file *m, void *p, loff_t *pos)
{
- return seq_list_next(p, &cache_chain, pos);
+ return seq_list_next(p, &slab_caches, pos);
}
static void s_stop(struct seq_file *m, void *p)
{
- mutex_unlock(&cache_chain_mutex);
+ mutex_unlock(&slab_mutex);
}
static int s_show(struct seq_file *m, void *p)
@@ -4403,9 +4397,9 @@ static ssize_t slabinfo_write(struct fil
return -EINVAL;
/* Find the cache in the chain of caches. */
- mutex_lock(&cache_chain_mutex);
+ mutex_lock(&slab_mutex);
res = -EINVAL;
- list_for_each_entry(cachep, &cache_chain, list) {
+ list_for_each_entry(cachep, &slab_caches, list) {
if (!strcmp(cachep->name, kbuf)) {
if (limit < 1 || batchcount < 1 ||
batchcount > limit || shared < 0) {
@@ -4418,7 +4412,7 @@ static ssize_t slabinfo_write(struct fil
break;
}
}
- mutex_unlock(&cache_chain_mutex);
+ mutex_unlock(&slab_mutex);
if (res >= 0)
res = count;
return res;
@@ -4441,8 +4435,8 @@ static const struct file_operations proc
static void *leaks_start(struct seq_file *m, loff_t *pos)
{
- mutex_lock(&cache_chain_mutex);
- return seq_list_start(&cache_chain, *pos);
+ mutex_lock(&slab_mutex);
+ return seq_list_start(&slab_caches, *pos);
}
static inline int add_caller(unsigned long *n, unsigned long v)
@@ -4541,17 +4535,17 @@ static int leaks_show(struct seq_file *m
name = cachep->name;
if (n[0] == n[1]) {
/* Increase the buffer size */
- mutex_unlock(&cache_chain_mutex);
+ mutex_unlock(&slab_mutex);
m->private = kzalloc(n[0] * 4 * sizeof(unsigned long), GFP_KERNEL);
if (!m->private) {
/* Too bad, we are really out */
m->private = n;
- mutex_lock(&cache_chain_mutex);
+ mutex_lock(&slab_mutex);
return -ENOMEM;
}
*(unsigned long *)m->private = n[0] * 2;
kfree(n);
- mutex_lock(&cache_chain_mutex);
+ mutex_lock(&slab_mutex);
/* Now make sure this entry will be retried */
m->count = m->size;
return 0;
Index: linux-2.6/mm/slab.h
===================================================================
--- linux-2.6.orig/mm/slab.h 2012-06-13 03:44:40.701477458 -0500
+++ linux-2.6/mm/slab.h 2012-06-13 03:45:31.309476410 -0500
@@ -23,6 +23,10 @@ enum slab_state {
extern enum slab_state slab_state;
+/* The slab cache mutex protects the management structures during changes */
+extern struct mutex slab_mutex;
+extern struct list_head slab_caches;
+
struct kmem_cache *__kmem_cache_create(const char *name, size_t size,
size_t align, unsigned long flags, void (*ctor)(void *));
Index: linux-2.6/mm/slub.c
===================================================================
--- linux-2.6.orig/mm/slub.c 2012-06-13 03:44:40.705477458 -0500
+++ linux-2.6/mm/slub.c 2012-06-13 03:45:40.661476216 -0500
@@ -36,13 +36,13 @@
/*
* Lock order:
- * 1. slub_lock (Global Semaphore)
+ * 1. slab_mutex (Global Mutex)
* 2. node->list_lock
* 3. slab_lock(page) (Only on some arches and for debugging)
*
- * slub_lock
+ * slab_mutex
*
- * The role of the slub_lock is to protect the list of all the slabs
+ * The role of the slab_mutex is to protect the list of all the slabs
* and to synchronize major metadata changes to slab cache structures.
*
* The slab_lock is only used for debugging and on arches that do not
@@ -183,10 +183,6 @@ static int kmem_size = sizeof(struct kme
static struct notifier_block slab_notifier;
#endif
-/* A list of all slab caches on the system */
-static DECLARE_RWSEM(slub_lock);
-static LIST_HEAD(slab_caches);
-
/*
* Tracking user of a slab.
*/
@@ -3191,11 +3187,11 @@ static inline int kmem_cache_close(struc
*/
void kmem_cache_destroy(struct kmem_cache *s)
{
- down_write(&slub_lock);
+ mutex_lock(&slab_mutex);
s->refcount--;
if (!s->refcount) {
list_del(&s->list);
- up_write(&slub_lock);
+ mutex_unlock(&slab_mutex);
if (kmem_cache_close(s)) {
printk(KERN_ERR "SLUB %s: %s called for cache that "
"still has objects.\n", s->name, __func__);
@@ -3205,7 +3201,7 @@ void kmem_cache_destroy(struct kmem_cach
rcu_barrier();
sysfs_slab_remove(s);
} else
- up_write(&slub_lock);
+ mutex_unlock(&slab_mutex);
}
EXPORT_SYMBOL(kmem_cache_destroy);
@@ -3267,7 +3263,7 @@ static struct kmem_cache *__init create_
/*
* This function is called with IRQs disabled during early-boot on
- * single CPU so there's no need to take slub_lock here.
+ * single CPU so there's no need to take slab_mutex here.
*/
if (!kmem_cache_open(s, name, size, ARCH_KMALLOC_MINALIGN,
flags, NULL))
@@ -3552,10 +3548,10 @@ static int slab_mem_going_offline_callba
{
struct kmem_cache *s;
- down_read(&slub_lock);
+ mutex_lock(&slab_mutex);
list_for_each_entry(s, &slab_caches, list)
kmem_cache_shrink(s);
- up_read(&slub_lock);
+ mutex_unlock(&slab_mutex);
return 0;
}
@@ -3576,7 +3572,7 @@ static void slab_mem_offline_callback(vo
if (offline_node < 0)
return;
- down_read(&slub_lock);
+ mutex_lock(&slab_mutex);
list_for_each_entry(s, &slab_caches, list) {
n = get_node(s, offline_node);
if (n) {
@@ -3592,7 +3588,7 @@ static void slab_mem_offline_callback(vo
kmem_cache_free(kmem_cache_node, n);
}
}
- up_read(&slub_lock);
+ mutex_unlock(&slab_mutex);
}
static int slab_mem_going_online_callback(void *arg)
@@ -3615,7 +3611,7 @@ static int slab_mem_going_online_callbac
* allocate a kmem_cache_node structure in order to bring the node
* online.
*/
- down_read(&slub_lock);
+ mutex_lock(&slab_mutex);
list_for_each_entry(s, &slab_caches, list) {
/*
* XXX: kmem_cache_alloc_node will fallback to other nodes
@@ -3631,7 +3627,7 @@ static int slab_mem_going_online_callbac
s->node[nid] = n;
}
out:
- up_read(&slub_lock);
+ mutex_unlock(&slab_mutex);
return ret;
}
@@ -3929,7 +3925,7 @@ struct kmem_cache *__kmem_cache_create(c
struct kmem_cache *s;
char *n;
- down_write(&slub_lock);
+ mutex_lock(&slab_mutex);
s = find_mergeable(size, align, flags, name, ctor);
if (s) {
s->refcount++;
@@ -3944,7 +3940,7 @@ struct kmem_cache *__kmem_cache_create(c
s->refcount--;
goto err;
}
- up_write(&slub_lock);
+ mutex_unlock(&slab_mutex);
return s;
}
@@ -3957,9 +3953,9 @@ struct kmem_cache *__kmem_cache_create(c
if (kmem_cache_open(s, n,
size, align, flags, ctor)) {
list_add(&s->list, &slab_caches);
- up_write(&slub_lock);
+ mutex_unlock(&slab_mutex);
if (sysfs_slab_add(s)) {
- down_write(&slub_lock);
+ mutex_lock(&slab_mutex);
list_del(&s->list);
kfree(n);
kfree(s);
@@ -3971,7 +3967,7 @@ struct kmem_cache *__kmem_cache_create(c
}
kfree(n);
err:
- up_write(&slub_lock);
+ mutex_unlock(&slab_mutex);
return s;
}
@@ -3992,13 +3988,13 @@ static int __cpuinit slab_cpuup_callback
case CPU_UP_CANCELED_FROZEN:
case CPU_DEAD:
case CPU_DEAD_FROZEN:
- down_read(&slub_lock);
+ mutex_lock(&slab_mutex);
list_for_each_entry(s, &slab_caches, list) {
local_irq_save(flags);
__flush_cpu_slab(s, cpu);
local_irq_restore(flags);
}
- up_read(&slub_lock);
+ mutex_unlock(&slab_mutex);
break;
default:
break;
@@ -5374,11 +5370,11 @@ static int __init slab_sysfs_init(void)
struct kmem_cache *s;
int err;
- down_write(&slub_lock);
+ mutex_lock(&slab_mutex);
slab_kset = kset_create_and_add("slab", &slab_uevent_ops, kernel_kobj);
if (!slab_kset) {
- up_write(&slub_lock);
+ mutex_unlock(&slab_mutex);
printk(KERN_ERR "Cannot register slab subsystem.\n");
return -ENOSYS;
}
@@ -5403,7 +5399,7 @@ static int __init slab_sysfs_init(void)
kfree(al);
}
- up_write(&slub_lock);
+ mutex_unlock(&slab_mutex);
resiliency_test();
return 0;
}
@@ -5429,7 +5425,7 @@ static void *s_start(struct seq_file *m,
{
loff_t n = *pos;
- down_read(&slub_lock);
+ mutex_lock(&slab_mutex);
if (!n)
print_slabinfo_header(m);
@@ -5443,7 +5439,7 @@ static void *s_next(struct seq_file *m,
static void s_stop(struct seq_file *m, void *p)
{
- up_read(&slub_lock);
+ mutex_unlock(&slab_mutex);
}
static int s_show(struct seq_file *m, void *p)
Index: linux-2.6/mm/slab_common.c
===================================================================
--- linux-2.6.orig/mm/slab_common.c 2012-06-13 03:44:40.705477458 -0500
+++ linux-2.6/mm/slab_common.c 2012-06-13 03:45:40.697476215 -0500
@@ -19,6 +19,8 @@
#include "slab.h"
enum slab_state slab_state;
+LIST_HEAD(slab_caches);
+DEFINE_MUTEX(slab_mutex);
/*
* kmem_cache_create - Create a cache.
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>
^ permalink raw reply [flat|nested] 40+ messages in thread* Common [11/20] Move kmem_cache_create mutex handling to common code
2012-06-13 15:24 Common [00/20] Sl[auo]b: Common code rework V5 (for merge) Christoph Lameter
` (8 preceding siblings ...)
2012-06-13 15:25 ` Common [10/20] Use a common mutex definition Christoph Lameter
@ 2012-06-13 15:25 ` Christoph Lameter
2012-06-13 15:25 ` Common [13/20] Extract a common function for kmem_cache_destroy Christoph Lameter
` (7 subsequent siblings)
17 siblings, 0 replies; 40+ messages in thread
From: Christoph Lameter @ 2012-06-13 15:25 UTC (permalink / raw)
To: Pekka Enberg
Cc: linux-mm, David Rientjes, Matt Mackall, Glauber Costa, Joonsoo Kim
[-- Attachment #1: move_mutex_to_common --]
[-- Type: text/plain, Size: 5558 bytes --]
Move the mutex handling into the common kmem_cache_create()
function.
Then we can also move more checks out of SLAB's kmem_cache_create()
into the common code.
Reviewed-by: Glauber Costa <glommer@parallels.com>
Signed-off-by: Christoph Lameter <cl@linux.com>
---
mm/slab.c | 52 +---------------------------------------------------
mm/slab_common.c | 41 ++++++++++++++++++++++++++++++++++++++++-
mm/slub.c | 28 +++++++++++++---------------
3 files changed, 54 insertions(+), 67 deletions(-)
Index: linux-2.6/mm/slab.c
===================================================================
--- linux-2.6.orig/mm/slab.c 2012-06-13 03:44:40.737477457 -0500
+++ linux-2.6/mm/slab.c 2012-06-13 03:45:37.497476281 -0500
@@ -2225,55 +2225,10 @@ __kmem_cache_create (const char *name, s
unsigned long flags, void (*ctor)(void *))
{
size_t left_over, slab_size, ralign;
- struct kmem_cache *cachep = NULL, *pc;
+ struct kmem_cache *cachep = NULL;
gfp_t gfp;
- /*
- * Sanity checks... these are all serious usage bugs.
- */
- if (!name || in_interrupt() || (size < BYTES_PER_WORD) ||
- size > KMALLOC_MAX_SIZE) {
- printk(KERN_ERR "%s: Early error in slab %s\n", __func__,
- name);
- BUG();
- }
-
- /*
- * We use cache_chain_mutex to ensure a consistent view of
- * cpu_online_mask as well. Please see cpuup_callback
- */
- if (slab_is_available()) {
- get_online_cpus();
- mutex_lock(&slab_mutex);
- }
-
- list_for_each_entry(pc, &slab_caches, list) {
- char tmp;
- int res;
-
- /*
- * This happens when the module gets unloaded and doesn't
- * destroy its slab cache and no-one else reuses the vmalloc
- * area of the module. Print a warning.
- */
- res = probe_kernel_address(pc->name, tmp);
- if (res) {
- printk(KERN_ERR
- "SLAB: cache with size %d has lost its name\n",
- pc->size);
- continue;
- }
-
- if (!strcmp(pc->name, name)) {
- printk(KERN_ERR
- "kmem_cache_create: duplicate cache %s\n", name);
- dump_stack();
- goto oops;
- }
- }
-
#if DEBUG
- WARN_ON(strchr(name, ' ')); /* It confuses parsers */
#if FORCED_DEBUG
/*
* Enable redzoning and last user accounting, except for caches with
@@ -2492,11 +2447,6 @@ __kmem_cache_create (const char *name, s
/* cache setup completed, link it into the list */
list_add(&cachep->list, &slab_caches);
-oops:
- if (slab_is_available()) {
- mutex_unlock(&slab_mutex);
- put_online_cpus();
- }
return cachep;
}
Index: linux-2.6/mm/slab_common.c
===================================================================
--- linux-2.6.orig/mm/slab_common.c 2012-06-13 03:44:40.741477457 -0500
+++ linux-2.6/mm/slab_common.c 2012-06-13 03:45:37.545476280 -0500
@@ -11,7 +11,8 @@
#include <linux/memory.h>
#include <linux/compiler.h>
#include <linux/module.h>
-
+#include <linux/cpu.h>
+#include <linux/uaccess.h>
#include <asm/cacheflush.h>
#include <asm/tlbflush.h>
#include <asm/page.h>
@@ -61,8 +62,46 @@ struct kmem_cache *kmem_cache_create(con
}
#endif
+ get_online_cpus();
+ mutex_lock(&slab_mutex);
+
+#ifdef CONFIG_DEBUG_VM
+ list_for_each_entry(s, &slab_caches, list) {
+ char tmp;
+ int res;
+
+ /*
+ * This happens when the module gets unloaded and doesn't
+ * destroy its slab cache and no-one else reuses the vmalloc
+ * area of the module. Print a warning.
+ */
+ res = probe_kernel_address(s->name, tmp);
+ if (res) {
+ printk(KERN_ERR
+ "Slab cache with size %d has lost its name\n",
+ s->object_size);
+ continue;
+ }
+
+ if (!strcmp(s->name, name)) {
+ printk(KERN_ERR "kmem_cache_create(%s): Cache name"
+ " already exists.\n",
+ name);
+ dump_stack();
+ s = NULL;
+ goto oops;
+ }
+ }
+
+ WARN_ON(strchr(name, ' ')); /* It confuses parsers */
+#endif
+
s = __kmem_cache_create(name, size, align, flags, ctor);
+oops:
+ mutex_unlock(&slab_mutex);
+ put_online_cpus();
+
out:
if (!s && (flags & SLAB_PANIC))
panic("kmem_cache_create: Failed to create slab '%s'\n", name);
Index: linux-2.6/mm/slub.c
===================================================================
--- linux-2.6.orig/mm/slub.c 2012-06-13 03:44:40.741477457 -0500
+++ linux-2.6/mm/slub.c 2012-06-13 03:45:37.525476281 -0500
@@ -3925,7 +3925,6 @@ struct kmem_cache *__kmem_cache_create(c
struct kmem_cache *s;
char *n;
- mutex_lock(&slab_mutex);
s = find_mergeable(size, align, flags, name, ctor);
if (s) {
s->refcount++;
@@ -3938,37 +3937,36 @@ struct kmem_cache *__kmem_cache_create(c
if (sysfs_slab_alias(s, name)) {
s->refcount--;
- goto err;
+ return NULL;
}
- mutex_unlock(&slab_mutex);
return s;
}
n = kstrdup(name, GFP_KERNEL);
if (!n)
- goto err;
+ return NULL;
s = kmalloc(kmem_size, GFP_KERNEL);
if (s) {
if (kmem_cache_open(s, n,
size, align, flags, ctor)) {
+ int r;
+
list_add(&s->list, &slab_caches);
mutex_unlock(&slab_mutex);
- if (sysfs_slab_add(s)) {
- mutex_lock(&slab_mutex);
- list_del(&s->list);
- kfree(n);
- kfree(s);
- goto err;
- }
- return s;
+ r = sysfs_slab_add(s);
+ mutex_lock(&slab_mutex);
+
+ if (!r)
+ return s;
+
+ list_del(&s->list);
+ kmem_cache_close(s);
}
kfree(s);
}
kfree(n);
-err:
- mutex_unlock(&slab_mutex);
- return s;
+ return NULL;
}
#ifdef CONFIG_SMP
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>
^ permalink raw reply [flat|nested] 40+ messages in thread* Common [13/20] Extract a common function for kmem_cache_destroy
2012-06-13 15:24 Common [00/20] Sl[auo]b: Common code rework V5 (for merge) Christoph Lameter
` (9 preceding siblings ...)
2012-06-13 15:25 ` Common [11/20] Move kmem_cache_create mutex handling to common code Christoph Lameter
@ 2012-06-13 15:25 ` Christoph Lameter
2012-06-13 15:25 ` Common [14/20] Always use the name "kmem_cache" for the slab cache with the kmem_cache structure Christoph Lameter
` (6 subsequent siblings)
17 siblings, 0 replies; 40+ messages in thread
From: Christoph Lameter @ 2012-06-13 15:25 UTC (permalink / raw)
To: Pekka Enberg
Cc: linux-mm, David Rientjes, Matt Mackall, Glauber Costa, Joonsoo Kim
[-- Attachment #1: kmem_cache_destroy --]
[-- Type: text/plain, Size: 6752 bytes --]
kmem_cache_destroy does basically the same in all allocators.
Extract common code which is easy since we already have common mutex handling.
Signed-off-by: Christoph Lameter <cl@linux.com>
---
mm/slab.c | 55 +++----------------------------------------------------
mm/slab.h | 4 +++-
mm/slab_common.c | 22 ++++++++++++++++++++++
mm/slob.c | 11 +++++++----
mm/slub.c | 29 ++++++++---------------------
5 files changed, 43 insertions(+), 78 deletions(-)
Index: linux-2.6/mm/slab_common.c
===================================================================
--- linux-2.6.orig/mm/slab_common.c 2012-06-13 03:44:40.805477456 -0500
+++ linux-2.6/mm/slab_common.c 2012-06-13 03:45:27.797476482 -0500
@@ -117,6 +117,28 @@ out:
}
EXPORT_SYMBOL(kmem_cache_create);
+void kmem_cache_destroy(struct kmem_cache *s)
+{
+ get_online_cpus();
+ mutex_lock(&slab_mutex);
+ list_del(&s->list);
+
+ if (!__kmem_cache_shutdown(s)) {
+ if (s->flags & SLAB_DESTROY_BY_RCU)
+ rcu_barrier();
+
+ __kmem_cache_destroy(s);
+ } else {
+ list_add(&s->list, &slab_caches);
+ printk(KERN_ERR "kmem_cache_destroy %s: Slab cache still has objects\n",
+ s->name);
+ dump_stack();
+ }
+ mutex_unlock(&slab_mutex);
+ put_online_cpus();
+}
+EXPORT_SYMBOL(kmem_cache_destroy);
+
int slab_is_available(void)
{
return slab_state >= UP;
Index: linux-2.6/mm/slab.c
===================================================================
--- linux-2.6.orig/mm/slab.c 2012-06-13 03:44:40.805477456 -0500
+++ linux-2.6/mm/slab.c 2012-06-13 03:45:27.749476483 -0500
@@ -779,16 +779,6 @@ static void cache_estimate(unsigned long
*left_over = slab_size - nr_objs*buffer_size - mgmt_size;
}
-#define slab_error(cachep, msg) __slab_error(__func__, cachep, msg)
-
-static void __slab_error(const char *function, struct kmem_cache *cachep,
- char *msg)
-{
- printk(KERN_ERR "slab error in %s(): cache `%s': %s\n",
- function, cachep->name, msg);
- dump_stack();
-}
-
/*
* By default on NUMA we use alien caches to stage the freeing of
* objects allocated from other nodes. This causes massive memory
@@ -2052,7 +2042,7 @@ static void slab_destroy(struct kmem_cac
}
}
-static void __kmem_cache_destroy(struct kmem_cache *cachep)
+void __kmem_cache_destroy(struct kmem_cache *cachep)
{
int i;
struct kmem_list3 *l3;
@@ -2609,49 +2599,10 @@ int kmem_cache_shrink(struct kmem_cache
}
EXPORT_SYMBOL(kmem_cache_shrink);
-/**
- * kmem_cache_destroy - delete a cache
- * @cachep: the cache to destroy
- *
- * Remove a &struct kmem_cache object from the slab cache.
- *
- * It is expected this function will be called by a module when it is
- * unloaded. This will remove the cache completely, and avoid a duplicate
- * cache being allocated each time a module is loaded and unloaded, if the
- * module doesn't have persistent in-kernel storage across loads and unloads.
- *
- * The cache must be empty before calling this function.
- *
- * The caller must guarantee that no one will allocate memory from the cache
- * during the kmem_cache_destroy().
- */
-void kmem_cache_destroy(struct kmem_cache *cachep)
+int __kmem_cache_shutdown(struct kmem_cache *cachep)
{
- BUG_ON(!cachep || in_interrupt());
-
- /* Find the cache in the chain of caches. */
- get_online_cpus();
- mutex_lock(&slab_mutex);
- /*
- * the chain is never empty, cache_cache is never destroyed
- */
- list_del(&cachep->list);
- if (__cache_shrink(cachep)) {
- slab_error(cachep, "Can't free all objects");
- list_add(&cachep->list, &slab_caches);
- mutex_unlock(&slab_mutex);
- put_online_cpus();
- return;
- }
-
- if (unlikely(cachep->flags & SLAB_DESTROY_BY_RCU))
- rcu_barrier();
-
- __kmem_cache_destroy(cachep);
- mutex_unlock(&slab_mutex);
- put_online_cpus();
+ return __cache_shrink(cachep);
}
-EXPORT_SYMBOL(kmem_cache_destroy);
/*
* Get the memory for a slab management obj.
Index: linux-2.6/mm/slab.h
===================================================================
--- linux-2.6.orig/mm/slab.h 2012-06-13 03:44:40.737477457 -0500
+++ linux-2.6/mm/slab.h 2012-06-13 03:45:27.781476483 -0500
@@ -30,5 +30,7 @@ extern struct list_head slab_caches;
struct kmem_cache *__kmem_cache_create(const char *name, size_t size,
size_t align, unsigned long flags, void (*ctor)(void *));
-#endif
+int __kmem_cache_shutdown(struct kmem_cache *);
+void __kmem_cache_destroy(struct kmem_cache *);
+#endif
Index: linux-2.6/mm/slob.c
===================================================================
--- linux-2.6.orig/mm/slob.c 2012-06-13 03:44:40.701477458 -0500
+++ linux-2.6/mm/slob.c 2012-06-13 03:45:27.729476484 -0500
@@ -538,14 +538,11 @@ struct kmem_cache *__kmem_cache_create(c
return c;
}
-void kmem_cache_destroy(struct kmem_cache *c)
+void __kmem_cache_destroy(struct kmem_cache *c)
{
kmemleak_free(c);
- if (c->flags & SLAB_DESTROY_BY_RCU)
- rcu_barrier();
slob_free(c, sizeof(struct kmem_cache));
}
-EXPORT_SYMBOL(kmem_cache_destroy);
void *kmem_cache_alloc_node(struct kmem_cache *c, gfp_t flags, int node)
{
@@ -613,6 +610,12 @@ unsigned int kmem_cache_size(struct kmem
}
EXPORT_SYMBOL(kmem_cache_size);
+int __kmem_cache_shutdown(struct kmem_cache *c)
+{
+ /* No way to check for remaining objects */
+ return 0;
+}
+
int kmem_cache_shrink(struct kmem_cache *d)
{
return 0;
Index: linux-2.6/mm/slub.c
===================================================================
--- linux-2.6.orig/mm/slub.c 2012-06-13 03:44:40.809477456 -0500
+++ linux-2.6/mm/slub.c 2012-06-13 03:45:27.765476483 -0500
@@ -3181,29 +3181,16 @@ static inline int kmem_cache_close(struc
return 0;
}
-/*
- * Close a cache and release the kmem_cache structure
- * (must be used for caches created using kmem_cache_create)
- */
-void kmem_cache_destroy(struct kmem_cache *s)
+int __kmem_cache_shutdown(struct kmem_cache *s)
{
- mutex_lock(&slab_mutex);
- s->refcount--;
- if (!s->refcount) {
- list_del(&s->list);
- mutex_unlock(&slab_mutex);
- if (kmem_cache_close(s)) {
- printk(KERN_ERR "SLUB %s: %s called for cache that "
- "still has objects.\n", s->name, __func__);
- dump_stack();
- }
- if (s->flags & SLAB_DESTROY_BY_RCU)
- rcu_barrier();
- sysfs_slab_remove(s);
- } else
- mutex_unlock(&slab_mutex);
+ return kmem_cache_close(s);
+}
+
+void __kmem_cache_destroy(struct kmem_cache *s)
+{
+ sysfs_slab_remove(s);
+ kfree(s);
}
-EXPORT_SYMBOL(kmem_cache_destroy);
/********************************************************************
* Kmalloc subsystem
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>
^ permalink raw reply [flat|nested] 40+ messages in thread* Common [14/20] Always use the name "kmem_cache" for the slab cache with the kmem_cache structure.
2012-06-13 15:24 Common [00/20] Sl[auo]b: Common code rework V5 (for merge) Christoph Lameter
` (10 preceding siblings ...)
2012-06-13 15:25 ` Common [13/20] Extract a common function for kmem_cache_destroy Christoph Lameter
@ 2012-06-13 15:25 ` Christoph Lameter
2012-06-14 8:16 ` Glauber Costa
2012-06-13 15:25 ` Common [16/20] Get rid of __kmem_cache_destroy Christoph Lameter
` (5 subsequent siblings)
17 siblings, 1 reply; 40+ messages in thread
From: Christoph Lameter @ 2012-06-13 15:25 UTC (permalink / raw)
To: Pekka Enberg
Cc: linux-mm, David Rientjes, Matt Mackall, Glauber Costa, Joonsoo Kim
[-- Attachment #1: common_kmem_cache_name --]
[-- Type: text/plain, Size: 9698 bytes --]
Make all allocators use the "kmem_cache" slabname for the "kmem_cache" structure.
Reviewed-by: Glauber Costa <glommer@parallels.com>
Reviewed-by: Joonsoo Kim <js1304@gmail.com>
Signed-off-by: Christoph Lameter <cl@linux.com>
---
mm/slab.c | 72 ++++++++++++++++++++++++++++---------------------------
mm/slab.h | 6 ++++
mm/slab_common.c | 1
mm/slob.c | 9 ++++++
mm/slub.c | 2 -
5 files changed, 52 insertions(+), 38 deletions(-)
Index: linux-2.6/mm/slab.c
===================================================================
--- linux-2.6.orig/mm/slab.c 2012-06-13 03:44:40.841477455 -0500
+++ linux-2.6/mm/slab.c 2012-06-13 03:45:24.793476545 -0500
@@ -554,9 +554,9 @@ static struct arraycache_init initarray_
{ {0, BOOT_CPUCACHE_ENTRIES, 1, 0} };
/* internal cache of cache description objs */
-static struct kmem_list3 *cache_cache_nodelists[MAX_NUMNODES];
-static struct kmem_cache cache_cache = {
- .nodelists = cache_cache_nodelists,
+static struct kmem_list3 *kmem_cache_nodelists[MAX_NUMNODES];
+static struct kmem_cache kmem_cache_boot = {
+ .nodelists = kmem_cache_nodelists,
.batchcount = 1,
.limit = BOOT_CPUCACHE_ENTRIES,
.shared = 1,
@@ -1442,15 +1442,17 @@ void __init kmem_cache_init(void)
int order;
int node;
+ kmem_cache = &kmem_cache_boot;
+
if (num_possible_nodes() == 1)
use_alien_caches = 0;
for (i = 0; i < NUM_INIT_LISTS; i++) {
kmem_list3_init(&initkmem_list3[i]);
if (i < MAX_NUMNODES)
- cache_cache.nodelists[i] = NULL;
+ kmem_cache->nodelists[i] = NULL;
}
- set_up_list3s(&cache_cache, CACHE_CACHE);
+ set_up_list3s(kmem_cache, CACHE_CACHE);
/*
* Fragmentation resistance on low memory - only use bigger
@@ -1462,9 +1464,9 @@ void __init kmem_cache_init(void)
/* Bootstrap is tricky, because several objects are allocated
* from caches that do not exist yet:
- * 1) initialize the cache_cache cache: it contains the struct
- * kmem_cache structures of all caches, except cache_cache itself:
- * cache_cache is statically allocated.
+ * 1) initialize the kmem_cache cache: it contains the struct
+ * kmem_cache structures of all caches, except kmem_cache itself:
+ * kmem_cache is statically allocated.
* Initially an __init data area is used for the head array and the
* kmem_list3 structures, it's replaced with a kmalloc allocated
* array at the end of the bootstrap.
@@ -1473,43 +1475,43 @@ void __init kmem_cache_init(void)
* An __init data area is used for the head array.
* 3) Create the remaining kmalloc caches, with minimally sized
* head arrays.
- * 4) Replace the __init data head arrays for cache_cache and the first
+ * 4) Replace the __init data head arrays for kmem_cache and the first
* kmalloc cache with kmalloc allocated arrays.
- * 5) Replace the __init data for kmem_list3 for cache_cache and
+ * 5) Replace the __init data for kmem_list3 for kmem_cache and
* the other cache's with kmalloc allocated memory.
* 6) Resize the head arrays of the kmalloc caches to their final sizes.
*/
node = numa_mem_id();
- /* 1) create the cache_cache */
+ /* 1) create the kmem_cache */
INIT_LIST_HEAD(&slab_caches);
- list_add(&cache_cache.list, &slab_caches);
- cache_cache.colour_off = cache_line_size();
- cache_cache.array[smp_processor_id()] = &initarray_cache.cache;
- cache_cache.nodelists[node] = &initkmem_list3[CACHE_CACHE + node];
+ list_add(&kmem_cache->list, &slab_caches);
+ kmem_cache->colour_off = cache_line_size();
+ kmem_cache->array[smp_processor_id()] = &initarray_cache.cache;
+ kmem_cache->nodelists[node] = &initkmem_list3[CACHE_CACHE + node];
/*
* struct kmem_cache size depends on nr_node_ids & nr_cpu_ids
*/
- cache_cache.size = offsetof(struct kmem_cache, array[nr_cpu_ids]) +
+ kmem_cache->size = offsetof(struct kmem_cache, array[nr_cpu_ids]) +
nr_node_ids * sizeof(struct kmem_list3 *);
- cache_cache.object_size = cache_cache.size;
- cache_cache.size = ALIGN(cache_cache.size,
+ kmem_cache->size = kmem_cache->size;
+ kmem_cache->size = ALIGN(kmem_cache->size,
cache_line_size());
- cache_cache.reciprocal_buffer_size =
- reciprocal_value(cache_cache.size);
+ kmem_cache->reciprocal_buffer_size =
+ reciprocal_value(kmem_cache->size);
for (order = 0; order < MAX_ORDER; order++) {
- cache_estimate(order, cache_cache.size,
- cache_line_size(), 0, &left_over, &cache_cache.num);
- if (cache_cache.num)
+ cache_estimate(order, kmem_cache->size,
+ cache_line_size(), 0, &left_over, &kmem_cache->num);
+ if (kmem_cache->num)
break;
}
- BUG_ON(!cache_cache.num);
- cache_cache.gfporder = order;
- cache_cache.colour = left_over / cache_cache.colour_off;
- cache_cache.slab_size = ALIGN(cache_cache.num * sizeof(kmem_bufctl_t) +
+ BUG_ON(!kmem_cache->num);
+ kmem_cache->gfporder = order;
+ kmem_cache->colour = left_over / kmem_cache->colour_off;
+ kmem_cache->slab_size = ALIGN(kmem_cache->num * sizeof(kmem_bufctl_t) +
sizeof(struct slab), cache_line_size());
/* 2+3) create the kmalloc caches */
@@ -1576,15 +1578,15 @@ void __init kmem_cache_init(void)
ptr = kmalloc(sizeof(struct arraycache_init), GFP_NOWAIT);
- BUG_ON(cpu_cache_get(&cache_cache) != &initarray_cache.cache);
- memcpy(ptr, cpu_cache_get(&cache_cache),
+ BUG_ON(cpu_cache_get(kmem_cache) != &initarray_cache.cache);
+ memcpy(ptr, cpu_cache_get(kmem_cache),
sizeof(struct arraycache_init));
/*
* Do not assume that spinlocks can be initialized via memcpy:
*/
spin_lock_init(&ptr->lock);
- cache_cache.array[smp_processor_id()] = ptr;
+ kmem_cache->array[smp_processor_id()] = ptr;
ptr = kmalloc(sizeof(struct arraycache_init), GFP_NOWAIT);
@@ -1605,7 +1607,7 @@ void __init kmem_cache_init(void)
int nid;
for_each_online_node(nid) {
- init_list(&cache_cache, &initkmem_list3[CACHE_CACHE + nid], nid);
+ init_list(kmem_cache, &initkmem_list3[CACHE_CACHE + nid], nid);
init_list(malloc_sizes[INDEX_AC].cs_cachep,
&initkmem_list3[SIZE_AC + nid], nid);
@@ -2059,7 +2061,7 @@ void __kmem_cache_destroy(struct kmem_ca
kfree(l3);
}
}
- kmem_cache_free(&cache_cache, cachep);
+ kmem_cache_free(kmem_cache, cachep);
}
@@ -2309,7 +2311,7 @@ __kmem_cache_create (const char *name, s
gfp = GFP_NOWAIT;
/* Get cache's description obj. */
- cachep = kmem_cache_zalloc(&cache_cache, gfp);
+ cachep = kmem_cache_zalloc(kmem_cache, gfp);
if (!cachep)
return NULL;
@@ -2367,7 +2369,7 @@ __kmem_cache_create (const char *name, s
if (!cachep->num) {
printk(KERN_ERR
"kmem_cache_create: couldn't create cache %s.\n", name);
- kmem_cache_free(&cache_cache, cachep);
+ kmem_cache_free(kmem_cache, cachep);
return NULL;
}
slab_size = ALIGN(cachep->num * sizeof(kmem_bufctl_t)
@@ -3128,7 +3130,7 @@ static void *cache_alloc_debugcheck_afte
static bool slab_should_failslab(struct kmem_cache *cachep, gfp_t flags)
{
- if (cachep == &cache_cache)
+ if (cachep == kmem_cache)
return false;
return should_failslab(cachep->object_size, flags, cachep->flags);
Index: linux-2.6/mm/slab.h
===================================================================
--- linux-2.6.orig/mm/slab.h 2012-06-13 03:44:40.841477455 -0500
+++ linux-2.6/mm/slab.h 2012-06-13 03:45:22.037476602 -0500
@@ -25,8 +25,14 @@ extern enum slab_state slab_state;
/* The slab cache mutex protects the management structures during changes */
extern struct mutex slab_mutex;
+
+/* The list of all slab caches on the system */
extern struct list_head slab_caches;
+/* The slab cache that manages slab cache information */
+extern struct kmem_cache *kmem_cache;
+
+/* Functions provided by the slab allocators */
struct kmem_cache *__kmem_cache_create(const char *name, size_t size,
size_t align, unsigned long flags, void (*ctor)(void *));
Index: linux-2.6/mm/slab_common.c
===================================================================
--- linux-2.6.orig/mm/slab_common.c 2012-06-13 03:44:40.837477455 -0500
+++ linux-2.6/mm/slab_common.c 2012-06-13 03:45:24.833476544 -0500
@@ -22,6 +22,7 @@
enum slab_state slab_state;
LIST_HEAD(slab_caches);
DEFINE_MUTEX(slab_mutex);
+struct kmem_cache *kmem_cache;
/*
* kmem_cache_create - Create a cache.
Index: linux-2.6/mm/slub.c
===================================================================
--- linux-2.6.orig/mm/slub.c 2012-06-13 03:44:40.845477455 -0500
+++ linux-2.6/mm/slub.c 2012-06-13 03:45:24.817476544 -0500
@@ -3199,8 +3199,6 @@ void __kmem_cache_destroy(struct kmem_ca
struct kmem_cache *kmalloc_caches[SLUB_PAGE_SHIFT];
EXPORT_SYMBOL(kmalloc_caches);
-static struct kmem_cache *kmem_cache;
-
#ifdef CONFIG_ZONE_DMA
static struct kmem_cache *kmalloc_dma_caches[SLUB_PAGE_SHIFT];
#endif
Index: linux-2.6/mm/slob.c
===================================================================
--- linux-2.6.orig/mm/slob.c 2012-06-13 03:44:40.841477455 -0500
+++ linux-2.6/mm/slob.c 2012-06-13 03:45:24.769476545 -0500
@@ -622,12 +622,19 @@ int kmem_cache_shrink(struct kmem_cache
}
EXPORT_SYMBOL(kmem_cache_shrink);
+struct kmem_cache kmem_cache_boot = {
+ .name = "kmem_cache",
+ .size = sizeof(struct kmem_cache),
+ .flags = SLAB_PANIC,
+ .align = ARCH_KMALLOC_MINALIGN,
+};
+
void __init kmem_cache_init(void)
{
+ kmem_cache = &kmem_cache_boot;
slab_state = UP;
}
void __init kmem_cache_init_late(void)
{
- slab_state = FULL;
}
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>
^ permalink raw reply [flat|nested] 40+ messages in thread* Re: Common [14/20] Always use the name "kmem_cache" for the slab cache with the kmem_cache structure.
2012-06-13 15:25 ` Common [14/20] Always use the name "kmem_cache" for the slab cache with the kmem_cache structure Christoph Lameter
@ 2012-06-14 8:16 ` Glauber Costa
2012-06-14 14:12 ` Christoph Lameter
0 siblings, 1 reply; 40+ messages in thread
From: Glauber Costa @ 2012-06-14 8:16 UTC (permalink / raw)
To: Christoph Lameter
Cc: Pekka Enberg, linux-mm, David Rientjes, Matt Mackall, Joonsoo Kim
On 06/13/2012 07:25 PM, Christoph Lameter wrote:
> - cache_cache.object_size = cache_cache.size;
> - cache_cache.size = ALIGN(cache_cache.size,
> + kmem_cache->size = kmem_cache->size;
You actually mean kmem_cache->object_size = kmem_cache->size.
Besides size = size making no sense, This had the effect for me to have
allocations that were supposed to be zeroed not being so particularly in
the edges of the objects.
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>
^ permalink raw reply [flat|nested] 40+ messages in thread
* Re: Common [14/20] Always use the name "kmem_cache" for the slab cache with the kmem_cache structure.
2012-06-14 8:16 ` Glauber Costa
@ 2012-06-14 14:12 ` Christoph Lameter
0 siblings, 0 replies; 40+ messages in thread
From: Christoph Lameter @ 2012-06-14 14:12 UTC (permalink / raw)
To: Glauber Costa
Cc: Pekka Enberg, linux-mm, David Rientjes, Matt Mackall, Joonsoo Kim
On Thu, 14 Jun 2012, Glauber Costa wrote:
> On 06/13/2012 07:25 PM, Christoph Lameter wrote:
> > - cache_cache.object_size = cache_cache.size;
> > - cache_cache.size = ALIGN(cache_cache.size,
> > + kmem_cache->size = kmem_cache->size;
>
> You actually mean kmem_cache->object_size = kmem_cache->size.
> Besides size = size making no sense, This had the effect for me to have
> allocations that were supposed to be zeroed not being so particularly in the
> edges of the objects.
Correct.
Subject: [slab] Provide correct reference to object size
Signed-off-by: Christoph Lameter <cl@linux.com>
Index: linux-2.6/mm/slab.c
===================================================================
--- linux-2.6.orig/mm/slab.c 2012-06-14 03:10:09.002709496 -0500
+++ linux-2.6/mm/slab.c 2012-06-14 03:10:02.478709623 -0500
@@ -1496,8 +1496,8 @@ void __init kmem_cache_init(void)
*/
kmem_cache->size = offsetof(struct kmem_cache, array[nr_cpu_ids]) +
nr_node_ids * sizeof(struct kmem_list3 *);
- kmem_cache->size = kmem_cache->size;
- kmem_cache->size = ALIGN(kmem_cache->size,
+ kmem_cache->object_size = kmem_cache->size;
+ kmem_cache->size = ALIGN(kmem_cache->object_size,
cache_line_size());
kmem_cache->reciprocal_buffer_size =
reciprocal_value(kmem_cache->size);
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>
^ permalink raw reply [flat|nested] 40+ messages in thread
* Common [16/20] Get rid of __kmem_cache_destroy
2012-06-13 15:24 Common [00/20] Sl[auo]b: Common code rework V5 (for merge) Christoph Lameter
` (11 preceding siblings ...)
2012-06-13 15:25 ` Common [14/20] Always use the name "kmem_cache" for the slab cache with the kmem_cache structure Christoph Lameter
@ 2012-06-13 15:25 ` Christoph Lameter
2012-06-13 15:25 ` Common [17/20] Move duping of slab name to slab_common.c Christoph Lameter
` (4 subsequent siblings)
17 siblings, 0 replies; 40+ messages in thread
From: Christoph Lameter @ 2012-06-13 15:25 UTC (permalink / raw)
To: Pekka Enberg
Cc: linux-mm, David Rientjes, Matt Mackall, Glauber Costa, Joonsoo Kim
[-- Attachment #1: no_slab_specific_kmem_cache_destroy --]
[-- Type: text/plain, Size: 4346 bytes --]
Actions done there can be done in __kmem_cache_shutdown.
This affects RCU handling somewhat. On rcu free all slab allocators
do not refer to other management structures than the kmem_cache structure.
Therefore these other structures can be freed before the rcu deferred
free to the page allocator occurs.
Reviewed-by: Joonsoo Kim <js1304@gmail.com>
Signed-off-by: Christoph Lameter <cl@linux.com>
---
mm/slab.c | 43 +++++++++++++++++++++----------------------
mm/slab.h | 1 -
mm/slab_common.c | 1 -
mm/slob.c | 4 ----
mm/slub.c | 10 +++++-----
5 files changed, 26 insertions(+), 33 deletions(-)
Index: linux-2.6/mm/slob.c
===================================================================
--- linux-2.6.orig/mm/slob.c 2012-06-13 03:44:40.909477454 -0500
+++ linux-2.6/mm/slob.c 2012-06-13 03:45:11.581476818 -0500
@@ -538,10 +538,6 @@ struct kmem_cache *__kmem_cache_create(c
return c;
}
-void __kmem_cache_destroy(struct kmem_cache *c)
-{
-}
-
void *kmem_cache_alloc_node(struct kmem_cache *c, gfp_t flags, int node)
{
void *b;
Index: linux-2.6/mm/slub.c
===================================================================
--- linux-2.6.orig/mm/slub.c 2012-06-13 03:44:40.913477454 -0500
+++ linux-2.6/mm/slub.c 2012-06-13 03:45:17.505476696 -0500
@@ -3183,12 +3183,12 @@ static inline int kmem_cache_close(struc
int __kmem_cache_shutdown(struct kmem_cache *s)
{
- return kmem_cache_close(s);
-}
+ int rc = kmem_cache_close(s);
-void __kmem_cache_destroy(struct kmem_cache *s)
-{
- sysfs_slab_remove(s);
+ if (!rc)
+ sysfs_slab_remove(s);
+
+ return rc;
}
/********************************************************************
Index: linux-2.6/mm/slab.c
===================================================================
--- linux-2.6.orig/mm/slab.c 2012-06-13 03:44:40.913477454 -0500
+++ linux-2.6/mm/slab.c 2012-06-13 03:45:11.605476818 -0500
@@ -2044,26 +2044,6 @@ static void slab_destroy(struct kmem_cac
}
}
-void __kmem_cache_destroy(struct kmem_cache *cachep)
-{
- int i;
- struct kmem_list3 *l3;
-
- for_each_online_cpu(i)
- kfree(cachep->array[i]);
-
- /* NUMA: free the list3 structures */
- for_each_online_node(i) {
- l3 = cachep->nodelists[i];
- if (l3) {
- kfree(l3->shared);
- free_alien_cache(l3->alien);
- kfree(l3);
- }
- }
-}
-
-
/**
* calculate_slab_order - calculate size (page order) of slabs
* @cachep: pointer to the cache that is being created
@@ -2427,7 +2407,7 @@ __kmem_cache_create (const char *name, s
cachep->refcount = 1;
if (setup_cpu_cache(cachep, gfp)) {
- __kmem_cache_destroy(cachep);
+ __kmem_cache_shutdown(cachep);
return NULL;
}
@@ -2602,7 +2582,26 @@ EXPORT_SYMBOL(kmem_cache_shrink);
int __kmem_cache_shutdown(struct kmem_cache *cachep)
{
- return __cache_shrink(cachep);
+ int i;
+ struct kmem_list3 *l3;
+ int rc = __cache_shrink(cachep);
+
+ if (rc)
+ return rc;
+
+ for_each_online_cpu(i)
+ kfree(cachep->array[i]);
+
+ /* NUMA: free the list3 structures */
+ for_each_online_node(i) {
+ l3 = cachep->nodelists[i];
+ if (l3) {
+ kfree(l3->shared);
+ free_alien_cache(l3->alien);
+ kfree(l3);
+ }
+ }
+ return 0;
}
/*
Index: linux-2.6/mm/slab.h
===================================================================
--- linux-2.6.orig/mm/slab.h 2012-06-13 03:44:40.873477455 -0500
+++ linux-2.6/mm/slab.h 2012-06-13 03:45:14.409476760 -0500
@@ -37,6 +37,5 @@ struct kmem_cache *__kmem_cache_create(c
size_t align, unsigned long flags, void (*ctor)(void *));
int __kmem_cache_shutdown(struct kmem_cache *);
-void __kmem_cache_destroy(struct kmem_cache *);
#endif
Index: linux-2.6/mm/slab_common.c
===================================================================
--- linux-2.6.orig/mm/slab_common.c 2012-06-13 03:44:40.909477454 -0500
+++ linux-2.6/mm/slab_common.c 2012-06-13 03:45:17.533476695 -0500
@@ -128,7 +128,6 @@ void kmem_cache_destroy(struct kmem_cach
if (s->flags & SLAB_DESTROY_BY_RCU)
rcu_barrier();
- __kmem_cache_destroy(s);
kmem_cache_free(kmem_cache, s);
} else {
list_add(&s->list, &slab_caches);
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>
^ permalink raw reply [flat|nested] 40+ messages in thread* Common [17/20] Move duping of slab name to slab_common.c
2012-06-13 15:24 Common [00/20] Sl[auo]b: Common code rework V5 (for merge) Christoph Lameter
` (12 preceding siblings ...)
2012-06-13 15:25 ` Common [16/20] Get rid of __kmem_cache_destroy Christoph Lameter
@ 2012-06-13 15:25 ` Christoph Lameter
2012-06-14 8:19 ` Glauber Costa
2012-06-13 15:25 ` Common [18/20] Do slab aliasing call from common code Christoph Lameter
` (3 subsequent siblings)
17 siblings, 1 reply; 40+ messages in thread
From: Christoph Lameter @ 2012-06-13 15:25 UTC (permalink / raw)
To: Pekka Enberg
Cc: linux-mm, David Rientjes, Matt Mackall, Glauber Costa, Joonsoo Kim
[-- Attachment #1: dup_name_in_common --]
[-- Type: text/plain, Size: 2986 bytes --]
Duping of the slabname has to be done by each slab. Moving this code
to slab_common avoids duplicate implementations.
With this patch we have common string handling for all slab allocators.
Strings passed to kmem_cache_create() are copied internally. Subsystems
can create temporary strings to create slab caches.
Slabs allocated in early states of bootstrap will never be freed (and those
can never be freed since they are essential to slab allocator operations).
During bootstrap we therefore do not have to worry about duping names.
Signed-off-by: Christoph Lameter <cl@linux.com>
---
mm/slab_common.c | 24 +++++++++++++++++-------
mm/slub.c | 5 -----
2 files changed, 17 insertions(+), 12 deletions(-)
Index: linux-2.6/mm/slab_common.c
===================================================================
--- linux-2.6.orig/mm/slab_common.c 2012-06-13 03:44:40.949477453 -0500
+++ linux-2.6/mm/slab_common.c 2012-06-13 03:45:14.429476759 -0500
@@ -53,6 +53,7 @@ struct kmem_cache *kmem_cache_create(con
unsigned long flags, void (*ctor)(void *))
{
struct kmem_cache *s = NULL;
+ char *n;
#ifdef CONFIG_DEBUG_VM
if (!name || in_interrupt() || size < sizeof(void *) ||
@@ -97,14 +98,22 @@ struct kmem_cache *kmem_cache_create(con
WARN_ON(strchr(name, ' ')); /* It confuses parsers */
#endif
- s = __kmem_cache_create(name, size, align, flags, ctor);
+ n = kstrdup(name, GFP_KERNEL);
+ if (!n)
+ goto oops;
- /*
- * Check if the slab has actually been created and if it was a
- * real instatiation. Aliases do not belong on the list
- */
- if (s && s->refcount == 1)
- list_add(&s->list, &slab_caches);
+ s = __kmem_cache_create(n, size, align, flags, ctor);
+
+ if (s) {
+ /*
+ * Check if the slab has actually been created and if it was a
+ * real instatiation. Aliases do not belong on the list
+ */
+ if (s->refcount == 1)
+ list_add(&s->list, &slab_caches);
+
+ } else
+ kfree(n);
oops:
mutex_unlock(&slab_mutex);
@@ -128,6 +137,7 @@ void kmem_cache_destroy(struct kmem_cach
if (s->flags & SLAB_DESTROY_BY_RCU)
rcu_barrier();
+ kfree(s->name);
kmem_cache_free(kmem_cache, s);
} else {
list_add(&s->list, &slab_caches);
Index: linux-2.6/mm/slub.c
===================================================================
--- linux-2.6.orig/mm/slub.c 2012-06-13 03:44:40.945477453 -0500
+++ linux-2.6/mm/slub.c 2012-06-13 03:45:14.393476760 -0500
@@ -3926,10 +3926,6 @@ struct kmem_cache *__kmem_cache_create(c
return s;
}
- n = kstrdup(name, GFP_KERNEL);
- if (!n)
- return NULL;
-
s = kmalloc(kmem_size, GFP_KERNEL);
if (s) {
if (kmem_cache_open(s, n,
@@ -3947,7 +3943,6 @@ struct kmem_cache *__kmem_cache_create(c
}
kfree(s);
}
- kfree(n);
return NULL;
}
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>
^ permalink raw reply [flat|nested] 40+ messages in thread* Re: Common [17/20] Move duping of slab name to slab_common.c
2012-06-13 15:25 ` Common [17/20] Move duping of slab name to slab_common.c Christoph Lameter
@ 2012-06-14 8:19 ` Glauber Costa
0 siblings, 0 replies; 40+ messages in thread
From: Glauber Costa @ 2012-06-14 8:19 UTC (permalink / raw)
To: Christoph Lameter
Cc: Pekka Enberg, linux-mm, David Rientjes, Matt Mackall, Joonsoo Kim
On 06/13/2012 07:25 PM, Christoph Lameter wrote:
> mutex_unlock(&slab_mutex);
> @@ -128,6 +137,7 @@ void kmem_cache_destroy(struct kmem_cach
> if (s->flags& SLAB_DESTROY_BY_RCU)
> rcu_barrier();
>
> + kfree(s->name);
> kmem_cache_free(kmem_cache, s);
> } else {
> list_add(&s->list,&slab_caches);
You forgot to remove the freeing of name in kmem_cache_release. This
kfree here then leads to a double free.
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>
^ permalink raw reply [flat|nested] 40+ messages in thread
* Common [18/20] Do slab aliasing call from common code
2012-06-13 15:24 Common [00/20] Sl[auo]b: Common code rework V5 (for merge) Christoph Lameter
` (13 preceding siblings ...)
2012-06-13 15:25 ` Common [17/20] Move duping of slab name to slab_common.c Christoph Lameter
@ 2012-06-13 15:25 ` Christoph Lameter
2012-06-13 15:25 ` Common [19/20] Allocate kmem_cache structure in slab_common.c Christoph Lameter
` (2 subsequent siblings)
17 siblings, 0 replies; 40+ messages in thread
From: Christoph Lameter @ 2012-06-13 15:25 UTC (permalink / raw)
To: Pekka Enberg
Cc: linux-mm, David Rientjes, Matt Mackall, Glauber Costa, Joonsoo Kim
[-- Attachment #1: slab_alias_common --]
[-- Type: text/plain, Size: 3569 bytes --]
The slab aliasing logic causes some strange contortions in
slub. So add a call to deal with aliases to slab_common.c
but disable it for other slab allocators by providng stubs
that fail to create aliases.
Full general support for aliases will require additional
cleanup passes and more standardization of fields in
kmem_cache.
Signed-off-by: Christoph Lameter <cl@linux.com>
---
mm/slab.h | 10 ++++++++++
mm/slab_common.c | 16 +++++++---------
mm/slub.c | 16 +++++++++++-----
3 files changed, 28 insertions(+), 14 deletions(-)
Index: linux-2.6/mm/slab.h
===================================================================
--- linux-2.6.orig/mm/slab.h 2012-06-13 03:44:40.949477453 -0500
+++ linux-2.6/mm/slab.h 2012-06-13 03:45:11.637476817 -0500
@@ -36,6 +36,16 @@ extern struct kmem_cache *kmem_cache;
struct kmem_cache *__kmem_cache_create(const char *name, size_t size,
size_t align, unsigned long flags, void (*ctor)(void *));
+#ifdef CONFIG_SLUB
+struct kmem_cache *__kmem_cache_alias(const char *name, size_t size,
+ size_t align, unsigned long flags, void (*ctor)(void *));
+#else
+static inline struct kmem_cache *__kmem_cache_alias(const char *name, size_t size,
+ size_t align, unsigned long flags, void (*ctor)(void *))
+{ return NULL; }
+#endif
+
+
int __kmem_cache_shutdown(struct kmem_cache *);
#endif
Index: linux-2.6/mm/slab_common.c
===================================================================
--- linux-2.6.orig/mm/slab_common.c 2012-06-13 03:44:40.977477452 -0500
+++ linux-2.6/mm/slab_common.c 2012-06-13 03:45:11.657476817 -0500
@@ -98,21 +98,19 @@ struct kmem_cache *kmem_cache_create(con
WARN_ON(strchr(name, ' ')); /* It confuses parsers */
#endif
+ s = __kmem_cache_alias(name, size, align, flags, ctor);
+ if (s)
+ goto oops;
+
n = kstrdup(name, GFP_KERNEL);
if (!n)
goto oops;
s = __kmem_cache_create(n, size, align, flags, ctor);
- if (s) {
- /*
- * Check if the slab has actually been created and if it was a
- * real instatiation. Aliases do not belong on the list
- */
- if (s->refcount == 1)
- list_add(&s->list, &slab_caches);
-
- } else
+ if (s)
+ list_add(&s->list, &slab_caches);
+ else
kfree(n);
oops:
Index: linux-2.6/mm/slub.c
===================================================================
--- linux-2.6.orig/mm/slub.c 2012-06-13 03:44:40.977477452 -0500
+++ linux-2.6/mm/slub.c 2012-06-13 03:45:11.621476817 -0500
@@ -3903,11 +3903,10 @@ static struct kmem_cache *find_mergeable
return NULL;
}
-struct kmem_cache *__kmem_cache_create(const char *name, size_t size,
+struct kmem_cache *__kmem_cache_alias(const char *name, size_t size,
size_t align, unsigned long flags, void (*ctor)(void *))
{
struct kmem_cache *s;
- char *n;
s = find_mergeable(size, align, flags, name, ctor);
if (s) {
@@ -3921,14 +3920,21 @@ struct kmem_cache *__kmem_cache_create(c
if (sysfs_slab_alias(s, name)) {
s->refcount--;
- return NULL;
+ s = NULL;
}
- return s;
}
+ return s;
+}
+
+struct kmem_cache *__kmem_cache_create(const char *name, size_t size,
+ size_t align, unsigned long flags, void (*ctor)(void *))
+{
+ struct kmem_cache *s;
+
s = kmalloc(kmem_size, GFP_KERNEL);
if (s) {
- if (kmem_cache_open(s, n,
+ if (kmem_cache_open(s, name,
size, align, flags, ctor)) {
int r;
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>
^ permalink raw reply [flat|nested] 40+ messages in thread* Common [19/20] Allocate kmem_cache structure in slab_common.c
2012-06-13 15:24 Common [00/20] Sl[auo]b: Common code rework V5 (for merge) Christoph Lameter
` (14 preceding siblings ...)
2012-06-13 15:25 ` Common [18/20] Do slab aliasing call from common code Christoph Lameter
@ 2012-06-13 15:25 ` Christoph Lameter
2012-06-14 10:16 ` Pekka Enberg
2012-06-13 15:25 ` Common [20/20] Common alignment code Christoph Lameter
2012-06-14 8:14 ` Common [00/20] Sl[auo]b: Common code rework V5 (for merge) Glauber Costa
17 siblings, 1 reply; 40+ messages in thread
From: Christoph Lameter @ 2012-06-13 15:25 UTC (permalink / raw)
To: Pekka Enberg
Cc: linux-mm, David Rientjes, Matt Mackall, Glauber Costa, Joonsoo Kim
[-- Attachment #1: move_kmem_alloc --]
[-- Type: text/plain, Size: 15052 bytes --]
Move kmem_cache memory allocation and the checks for success out of the
slab allocators into the common code.
Signed-off-by: Christoph Lameter <cl@linux.com>
---
mm/slab.c | 122 ++++++++++++++++++++-----------------------------------
mm/slab.h | 3 -
mm/slab_common.c | 25 +++++++++--
mm/slob.c | 39 ++++++-----------
mm/slub.c | 107 ++++++++++++++++++++++--------------------------
5 files changed, 134 insertions(+), 162 deletions(-)
Index: linux-2.6/mm/slab.h
===================================================================
--- linux-2.6.orig/mm/slab.h 2012-06-13 03:44:41.009477452 -0500
+++ linux-2.6/mm/slab.h 2012-06-13 03:45:07.781476897 -0500
@@ -33,8 +33,7 @@ extern struct list_head slab_caches;
extern struct kmem_cache *kmem_cache;
/* Functions provided by the slab allocators */
-struct kmem_cache *__kmem_cache_create(const char *name, size_t size,
- size_t align, unsigned long flags, void (*ctor)(void *));
+int __kmem_cache_create(struct kmem_cache *s);
#ifdef CONFIG_SLUB
struct kmem_cache *__kmem_cache_alias(const char *name, size_t size,
Index: linux-2.6/mm/slab_common.c
===================================================================
--- linux-2.6.orig/mm/slab_common.c 2012-06-13 03:44:41.009477452 -0500
+++ linux-2.6/mm/slab_common.c 2012-06-13 03:45:07.805476897 -0500
@@ -54,6 +54,7 @@ struct kmem_cache *kmem_cache_create(con
{
struct kmem_cache *s = NULL;
char *n;
+ int r;
#ifdef CONFIG_DEBUG_VM
if (!name || in_interrupt() || size < sizeof(void *) ||
@@ -106,12 +107,30 @@ struct kmem_cache *kmem_cache_create(con
if (!n)
goto oops;
- s = __kmem_cache_create(n, size, align, flags, ctor);
+ s = kmem_cache_zalloc(kmem_cache, GFP_KERNEL);
- if (s)
+ if (!s) {
+ kfree(n);
+ goto oops;
+ }
+
+ s->name = n;
+ s->size = s->object_size = size;
+ s->ctor = ctor;
+ s->flags = flags;
+ s->align = align;
+
+ r = __kmem_cache_create(s);
+
+ if (!r) {
+ s->refcount = 1;
list_add(&s->list, &slab_caches);
- else
+ }
+ else {
+ kmem_cache_free(kmem_cache, s);
kfree(n);
+ s = NULL;
+ }
oops:
mutex_unlock(&slab_mutex);
Index: linux-2.6/mm/slub.c
===================================================================
--- linux-2.6.orig/mm/slub.c 2012-06-13 03:44:41.013477452 -0500
+++ linux-2.6/mm/slub.c 2012-06-13 03:45:07.757476898 -0500
@@ -3012,24 +3012,17 @@ static int calculate_sizes(struct kmem_c
}
-static int kmem_cache_open(struct kmem_cache *s,
- const char *name, size_t size,
- size_t align, unsigned long flags,
- void (*ctor)(void *))
+static int kmem_cache_open(struct kmem_cache *s)
{
- memset(s, 0, kmem_size);
- s->name = name;
- s->ctor = ctor;
- s->object_size = size;
- s->align = align;
- s->flags = kmem_cache_flags(size, flags, name, ctor);
+ s->flags = kmem_cache_flags(s->size, s->flags, s->name, s->ctor);
s->reserved = 0;
if (need_reserve_slab_rcu && (s->flags & SLAB_DESTROY_BY_RCU))
s->reserved = sizeof(struct rcu_head);
if (!calculate_sizes(s, -1))
- goto error;
+ return -EINVAL;
+
if (disable_higher_order_debug) {
/*
* Disable debugging flags that store metadata if the min slab
@@ -3039,7 +3032,7 @@ static int kmem_cache_open(struct kmem_c
s->flags &= ~DEBUG_METADATA_FLAGS;
s->offset = 0;
if (!calculate_sizes(s, -1))
- goto error;
+ return -EINVAL;
}
}
@@ -3084,23 +3077,16 @@ static int kmem_cache_open(struct kmem_c
else
s->cpu_partial = 30;
- s->refcount = 1;
#ifdef CONFIG_NUMA
s->remote_node_defrag_ratio = 1000;
#endif
if (!init_kmem_cache_nodes(s))
- goto error;
-
- if (alloc_kmem_cache_cpus(s))
- return 1;
+ return -ENOMEM;
- free_kmem_cache_nodes(s);
-error:
- if (flags & SLAB_PANIC)
- panic("Cannot create slab %s size=%lu realsize=%u "
- "order=%u offset=%u flags=%lx\n",
- s->name, (unsigned long)size, s->size, oo_order(s->oo),
- s->offset, flags);
+ if (!alloc_kmem_cache_cpus(s)) {
+ free_kmem_cache_nodes(s);
+ return -ENOMEM;
+ }
return 0;
}
@@ -3242,23 +3228,25 @@ static struct kmem_cache *__init create_
int size, unsigned int flags)
{
struct kmem_cache *s;
+ int r;
- s = kmem_cache_alloc(kmem_cache, GFP_NOWAIT);
+ s = kmem_cache_zalloc(kmem_cache, GFP_NOWAIT);
+ s->name = name;
+ s->size = s->object_size = size;
+ s->align = ARCH_KMALLOC_MINALIGN;
+ s->flags = flags;
/*
* This function is called with IRQs disabled during early-boot on
* single CPU so there's no need to take slab_mutex here.
*/
- if (!kmem_cache_open(s, name, size, ARCH_KMALLOC_MINALIGN,
- flags, NULL))
- goto panic;
+ r = kmem_cache_open(s);
+ if (r)
+ panic("Creation of kmalloc slab %s size=%d failed. Code %d\n",
+ name, size, r);
list_add(&s->list, &slab_caches);
return s;
-
-panic:
- panic("Creation of kmalloc slab %s size=%d failed.\n", name, size);
- return NULL;
}
/*
@@ -3679,6 +3667,7 @@ static void __init kmem_cache_bootstrap_
void __init kmem_cache_init(void)
{
int i;
+ int r;
int caches = 0;
struct kmem_cache *temp_kmem_cache;
int order;
@@ -3702,10 +3691,13 @@ void __init kmem_cache_init(void)
* kmem_cache_open for slab_state == DOWN.
*/
kmem_cache_node = (void *)kmem_cache + kmalloc_size;
+ kmem_cache_node->name = "kmem_cache_node";
+ kmem_cache_node->size = kmem_cache_node->object_size = sizeof(struct kmem_cache_node);
+ kmem_cache_node->flags = SLAB_HWCACHE_ALIGN;
- kmem_cache_open(kmem_cache_node, "kmem_cache_node",
- sizeof(struct kmem_cache_node),
- 0, SLAB_HWCACHE_ALIGN | SLAB_PANIC, NULL);
+ r = kmem_cache_open(kmem_cache_node);
+ if (r)
+ goto panic;
hotplug_memory_notifier(slab_memory_callback, SLAB_CALLBACK_PRI);
@@ -3713,8 +3705,14 @@ void __init kmem_cache_init(void)
slab_state = PARTIAL;
temp_kmem_cache = kmem_cache;
- kmem_cache_open(kmem_cache, "kmem_cache", kmem_size,
- 0, SLAB_HWCACHE_ALIGN | SLAB_PANIC, NULL);
+ kmem_cache->name = "kmem_cache";
+ kmem_cache->size = kmem_cache->object_size = kmem_size;
+ kmem_cache->flags = SLAB_HWCACHE_ALIGN;
+
+ r = kmem_cache_open(kmem_cache);
+ if (r)
+ goto panic;
+
kmem_cache = kmem_cache_alloc(kmem_cache, GFP_NOWAIT);
memcpy(kmem_cache, temp_kmem_cache, kmem_size);
@@ -3836,6 +3834,10 @@ void __init kmem_cache_init(void)
caches, cache_line_size(),
slub_min_order, slub_max_order, slub_min_objects,
nr_cpu_ids, nr_node_ids);
+ return;
+
+panic:
+ panic("SLUB bootstrap failed. Code %d\n", r);
}
void __init kmem_cache_init_late(void)
@@ -3927,29 +3929,22 @@ struct kmem_cache *__kmem_cache_alias(co
return s;
}
-struct kmem_cache *__kmem_cache_create(const char *name, size_t size,
- size_t align, unsigned long flags, void (*ctor)(void *))
+int __kmem_cache_create(struct kmem_cache *s)
{
- struct kmem_cache *s;
+ int r = kmem_cache_open(s);
- s = kmalloc(kmem_size, GFP_KERNEL);
- if (s) {
- if (kmem_cache_open(s, name,
- size, align, flags, ctor)) {
- int r;
-
- mutex_unlock(&slab_mutex);
- r = sysfs_slab_add(s);
- mutex_lock(&slab_mutex);
+ if (r)
+ return r;
- if (!r)
- return s;
+ mutex_unlock(&slab_mutex);
+ r = sysfs_slab_add(s);
+ mutex_lock(&slab_mutex);
- kmem_cache_close(s);
- }
- kfree(s);
- }
- return NULL;
+ if (!r)
+ return 0;
+
+ kmem_cache_close(s);
+ return r;
}
#ifdef CONFIG_SMP
Index: linux-2.6/mm/slab.c
===================================================================
--- linux-2.6.orig/mm/slab.c 2012-06-13 03:44:40.949477453 -0500
+++ linux-2.6/mm/slab.c 2012-06-13 03:45:07.733476898 -0500
@@ -1429,6 +1429,30 @@ static void __init set_up_list3s(struct
}
}
+struct kmem_cache *create_kmalloc_cache(char *name, int size, int flags)
+{
+ struct kmem_cache *s = kmem_cache_zalloc(kmem_cache, GFP_KERNEL);
+ int r = -ENOMEM;
+
+ if (!s)
+ goto panic;
+
+ s->name = name;
+ s->size = s->object_size = size;
+ s->align = ARCH_KMALLOC_MINALIGN;
+ s->flags = flags | ARCH_KMALLOC_FLAGS;
+
+ r = __kmem_cache_create(s);
+
+ if (r)
+ goto panic;
+
+ list_add(&s->list, &slab_caches);
+ return s;
+panic:
+ panic("Failed to create kmalloc cache %s. Size=%d Code=%d\n", name, size, r);
+}
+
/*
* Initialisation. Called after the page allocator have been initialised and
* before smp_init().
@@ -1524,22 +1548,13 @@ void __init kmem_cache_init(void)
* bug.
*/
- sizes[INDEX_AC].cs_cachep = __kmem_cache_create(names[INDEX_AC].name,
- sizes[INDEX_AC].cs_size,
- ARCH_KMALLOC_MINALIGN,
- ARCH_KMALLOC_FLAGS|SLAB_PANIC,
- NULL);
+ sizes[INDEX_AC].cs_cachep = create_kmalloc_cache(names[INDEX_AC].name,
+ sizes[INDEX_AC].cs_size, 0);
- list_add(&sizes[INDEX_AC].cs_cachep->list, &slab_caches);
- if (INDEX_AC != INDEX_L3) {
+ if (INDEX_AC != INDEX_L3)
sizes[INDEX_L3].cs_cachep =
- __kmem_cache_create(names[INDEX_L3].name,
- sizes[INDEX_L3].cs_size,
- ARCH_KMALLOC_MINALIGN,
- ARCH_KMALLOC_FLAGS|SLAB_PANIC,
- NULL);
- list_add(&sizes[INDEX_L3].cs_cachep->list, &slab_caches);
- }
+ create_kmalloc_cache(names[INDEX_L3].name,
+ sizes[INDEX_L3].cs_size, 0);
slab_early_init = 0;
@@ -1551,23 +1566,13 @@ void __init kmem_cache_init(void)
* Note for systems short on memory removing the alignment will
* allow tighter packing of the smaller caches.
*/
- if (!sizes->cs_cachep) {
- sizes->cs_cachep = __kmem_cache_create(names->name,
- sizes->cs_size,
- ARCH_KMALLOC_MINALIGN,
- ARCH_KMALLOC_FLAGS|SLAB_PANIC,
- NULL);
- list_add(&sizes->cs_cachep->list, &slab_caches);
- }
+ if (!sizes->cs_cachep)
+ sizes->cs_cachep = create_kmalloc_cache(names->name,
+ sizes->cs_size, 0);
+
#ifdef CONFIG_ZONE_DMA
- sizes->cs_dmacachep = __kmem_cache_create(
- names->name_dma,
- sizes->cs_size,
- ARCH_KMALLOC_MINALIGN,
- ARCH_KMALLOC_FLAGS|SLAB_CACHE_DMA|
- SLAB_PANIC,
- NULL);
- list_add(&sizes->cs_dmacachep->list, &slab_caches);
+ sizes->cs_dmacachep = create_kmalloc_cache(
+ names->name_dma, sizes->cs_size, SLAB_CACHE_DMA);
#endif
sizes++;
names++;
@@ -2170,38 +2175,14 @@ static int __init_refok setup_cpu_cache(
/**
* __kmem_cache_create - Create a cache.
- * @name: A string which is used in /proc/slabinfo to identify this cache.
- * @size: The size of objects to be created in this cache.
- * @align: The required alignment for the objects.
- * @flags: SLAB flags
- * @ctor: A constructor for the objects.
- *
- * Returns a ptr to the cache on success, NULL on failure.
- * Cannot be called within a int, but can be interrupted.
- * The @ctor is run when new pages are allocated by the cache.
- *
- * @name must be valid until the cache is destroyed. This implies that
- * the module calling this has to destroy the cache before getting unloaded.
- *
- * The flags are
- *
- * %SLAB_POISON - Poison the slab with a known test pattern (a5a5a5a5)
- * to catch references to uninitialised memory.
- *
- * %SLAB_RED_ZONE - Insert `Red' zones around the allocated memory to check
- * for buffer overruns.
- *
- * %SLAB_HWCACHE_ALIGN - Align the objects in this cache to a hardware
- * cacheline. This can be beneficial if you're counting cycles as closely
- * as davem.
*/
-struct kmem_cache *
-__kmem_cache_create (const char *name, size_t size, size_t align,
- unsigned long flags, void (*ctor)(void *))
+int __kmem_cache_create(struct kmem_cache *cachep)
{
size_t left_over, slab_size, ralign;
- struct kmem_cache *cachep = NULL;
gfp_t gfp;
+ int flags = cachep->flags;
+ int size = cachep->object_size;
+ int align;
#if DEBUG
#if FORCED_DEBUG
@@ -2273,8 +2254,8 @@ __kmem_cache_create (const char *name, s
ralign = ARCH_SLAB_MINALIGN;
}
/* 3) caller mandated alignment */
- if (ralign < align) {
- ralign = align;
+ if (ralign < cachep->align) {
+ ralign = cachep->align;
}
/* disable debug if necessary */
if (ralign > __alignof__(unsigned long long))
@@ -2289,11 +2270,6 @@ __kmem_cache_create (const char *name, s
else
gfp = GFP_NOWAIT;
- /* Get cache's description obj. */
- cachep = kmem_cache_zalloc(kmem_cache, gfp);
- if (!cachep)
- return NULL;
-
cachep->nodelists = (struct kmem_list3 **)&cachep->array[nr_cpu_ids];
cachep->object_size = size;
cachep->align = align;
@@ -2345,12 +2321,9 @@ __kmem_cache_create (const char *name, s
left_over = calculate_slab_order(cachep, size, align, flags);
- if (!cachep->num) {
- printk(KERN_ERR
- "kmem_cache_create: couldn't create cache %s.\n", name);
- kmem_cache_free(kmem_cache, cachep);
- return NULL;
- }
+ if (!cachep->num)
+ return -EINVAL;
+
slab_size = ALIGN(cachep->num * sizeof(kmem_bufctl_t)
+ sizeof(struct slab), align);
@@ -2402,13 +2375,10 @@ __kmem_cache_create (const char *name, s
*/
BUG_ON(ZERO_OR_NULL_PTR(cachep->slabp_cache));
}
- cachep->ctor = ctor;
- cachep->name = name;
- cachep->refcount = 1;
if (setup_cpu_cache(cachep, gfp)) {
__kmem_cache_shutdown(cachep);
- return NULL;
+ return -ENOMEM;
}
if (flags & SLAB_DEBUG_OBJECTS) {
@@ -2421,7 +2391,7 @@ __kmem_cache_create (const char *name, s
slab_set_debugobj_lock_classes(cachep);
}
- return cachep;
+ return 0;
}
#if DEBUG
Index: linux-2.6/mm/slob.c
===================================================================
--- linux-2.6.orig/mm/slob.c 2012-06-13 03:44:40.945477453 -0500
+++ linux-2.6/mm/slob.c 2012-06-13 03:45:07.709476898 -0500
@@ -508,34 +508,23 @@ size_t ksize(const void *block)
}
EXPORT_SYMBOL(ksize);
-struct kmem_cache *__kmem_cache_create(const char *name, size_t size,
- size_t align, unsigned long flags, void (*ctor)(void *))
+int __kmem_cache_create(struct kmem_cache *c)
{
- struct kmem_cache *c;
+ int align = c->align;
- c = slob_alloc(sizeof(struct kmem_cache),
- GFP_KERNEL, ARCH_KMALLOC_MINALIGN, -1);
-
- if (c) {
- c->name = name;
- c->size = c->object_size;
- if (flags & SLAB_DESTROY_BY_RCU) {
- /* leave room for rcu footer at the end of object */
- c->size += sizeof(struct slob_rcu);
- }
- c->flags = flags;
- c->ctor = ctor;
- /* ignore alignment unless it's forced */
- c->align = (flags & SLAB_HWCACHE_ALIGN) ? SLOB_ALIGN : 0;
- if (c->align < ARCH_SLAB_MINALIGN)
- c->align = ARCH_SLAB_MINALIGN;
- if (c->align < align)
- c->align = align;
-
- kmemleak_alloc(c, sizeof(struct kmem_cache), 1, GFP_KERNEL);
- c->refcount = 1;
+ if (c->flags & SLAB_DESTROY_BY_RCU) {
+ /* leave room for rcu footer at the end of object */
+ c->size += sizeof(struct slob_rcu);
}
- return c;
+ /* ignore alignment unless it's forced */
+ c->align = (c->flags & SLAB_HWCACHE_ALIGN) ? SLOB_ALIGN : 0;
+ if (c->align < ARCH_SLAB_MINALIGN)
+ c->align = ARCH_SLAB_MINALIGN;
+ if (c->align < align)
+ c->align = align;
+
+ kmemleak_alloc(c, sizeof(struct kmem_cache), 1, GFP_KERNEL);
+ return 0;
}
void *kmem_cache_alloc_node(struct kmem_cache *c, gfp_t flags, int node)
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>
^ permalink raw reply [flat|nested] 40+ messages in thread* Re: Common [19/20] Allocate kmem_cache structure in slab_common.c
2012-06-13 15:25 ` Common [19/20] Allocate kmem_cache structure in slab_common.c Christoph Lameter
@ 2012-06-14 10:16 ` Pekka Enberg
2012-06-14 10:34 ` Glauber Costa
0 siblings, 1 reply; 40+ messages in thread
From: Pekka Enberg @ 2012-06-14 10:16 UTC (permalink / raw)
To: Christoph Lameter
Cc: linux-mm, David Rientjes, Matt Mackall, Glauber Costa, Joonsoo Kim
On Wed, Jun 13, 2012 at 6:25 PM, Christoph Lameter <cl@linux.com> wrote:
> Move kmem_cache memory allocation and the checks for success out of the
> slab allocators into the common code.
>
> Signed-off-by: Christoph Lameter <cl@linux.com>
This patch seems to cause hard lockup on my laptop on boot.
P.S. While bisecting this, I also saw some other oopses so I think
this series needs some more testing before we can put it in
linux-next...
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>
^ permalink raw reply [flat|nested] 40+ messages in thread
* Re: Common [19/20] Allocate kmem_cache structure in slab_common.c
2012-06-14 10:16 ` Pekka Enberg
@ 2012-06-14 10:34 ` Glauber Costa
2012-06-14 18:14 ` Christoph Lameter
0 siblings, 1 reply; 40+ messages in thread
From: Glauber Costa @ 2012-06-14 10:34 UTC (permalink / raw)
To: Pekka Enberg
Cc: Christoph Lameter, linux-mm, David Rientjes, Matt Mackall, Joonsoo Kim
[-- Attachment #1: Type: text/plain, Size: 703 bytes --]
On 06/14/2012 02:16 PM, Pekka Enberg wrote:
> On Wed, Jun 13, 2012 at 6:25 PM, Christoph Lameter<cl@linux.com> wrote:
>> Move kmem_cache memory allocation and the checks for success out of the
>> slab allocators into the common code.
>>
>> Signed-off-by: Christoph Lameter<cl@linux.com>
>
> This patch seems to cause hard lockup on my laptop on boot.
>
> P.S. While bisecting this, I also saw some other oopses so I think
> this series needs some more testing before we can put it in
> linux-next...
Hi Pekka
Could you please test the following patch ?
This made Cristoph's series a lot more stable for me, it will at least
allow us to know if this is the issue, or if you're seeing something else.
[-- Warning: decoded text below may be mangled, UTF-8 assumed --]
[-- Attachment #2: 0001-slab-fix-obj-size-calculation.patch --]
[-- Type: text/x-patch; name="0001-slab-fix-obj-size-calculation.patch", Size: 0 bytes --]
^ permalink raw reply [flat|nested] 40+ messages in thread
* Re: Common [19/20] Allocate kmem_cache structure in slab_common.c
2012-06-14 10:34 ` Glauber Costa
@ 2012-06-14 18:14 ` Christoph Lameter
2012-06-20 7:07 ` Pekka Enberg
0 siblings, 1 reply; 40+ messages in thread
From: Christoph Lameter @ 2012-06-14 18:14 UTC (permalink / raw)
To: Pekka Enberg
Cc: Glauber Costa, linux-mm, David Rientjes, Matt Mackall, Joonsoo Kim
Merge what you are comfortable with and I will have a look at the rest.
Just the initial cleanups for the allocators maybe?
I applied glommers' fix but my machine also will not boot with the whole
set.
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>
^ permalink raw reply [flat|nested] 40+ messages in thread
* Re: Common [19/20] Allocate kmem_cache structure in slab_common.c
2012-06-14 18:14 ` Christoph Lameter
@ 2012-06-20 7:07 ` Pekka Enberg
0 siblings, 0 replies; 40+ messages in thread
From: Pekka Enberg @ 2012-06-20 7:07 UTC (permalink / raw)
To: Christoph Lameter
Cc: Glauber Costa, linux-mm, David Rientjes, Matt Mackall, Joonsoo Kim
On Thu, Jun 14, 2012 at 9:14 PM, Christoph Lameter <cl@linux.com> wrote:
> Merge what you are comfortable with and I will have a look at the rest.
> Just the initial cleanups for the allocators maybe?
I merged patches up to "slab: Get rid of obj_size macro". If Glauber
already fixed any of the remaining patches, please just resend them.
Pekka
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>
^ permalink raw reply [flat|nested] 40+ messages in thread
* Common [20/20] Common alignment code
2012-06-13 15:24 Common [00/20] Sl[auo]b: Common code rework V5 (for merge) Christoph Lameter
` (15 preceding siblings ...)
2012-06-13 15:25 ` Common [19/20] Allocate kmem_cache structure in slab_common.c Christoph Lameter
@ 2012-06-13 15:25 ` Christoph Lameter
2012-06-14 8:14 ` Common [00/20] Sl[auo]b: Common code rework V5 (for merge) Glauber Costa
17 siblings, 0 replies; 40+ messages in thread
From: Christoph Lameter @ 2012-06-13 15:25 UTC (permalink / raw)
To: Pekka Enberg
Cc: linux-mm, David Rientjes, Matt Mackall, Glauber Costa, Joonsoo Kim
[-- Attachment #1: common_alignment --]
[-- Type: text/plain, Size: 8314 bytes --]
Extract the code to do object alignment from the allocators.
Signed-off-by: Christoph Lameter <cl@linux.com>
---
mm/slab.c | 22 +---------------------
mm/slab.h | 3 +++
mm/slab_common.c | 30 +++++++++++++++++++++++++++++-
mm/slob.c | 11 -----------
mm/slub.c | 45 ++++++++-------------------------------------
5 files changed, 41 insertions(+), 70 deletions(-)
Index: linux-2.6/mm/slab.c
===================================================================
--- linux-2.6.orig/mm/slab.c 2012-06-13 03:44:57.145477117 -0500
+++ linux-2.6/mm/slab.c 2012-06-13 03:45:03.621476983 -0500
@@ -1439,7 +1439,7 @@ struct kmem_cache *create_kmalloc_cache(
s->name = name;
s->size = s->object_size = size;
- s->align = ARCH_KMALLOC_MINALIGN;
+ s->align = calculate_alignment(flags, ARCH_KMALLOC_MINALIGN, size);
s->flags = flags | ARCH_KMALLOC_FLAGS;
r = __kmem_cache_create(s);
@@ -2217,22 +2217,6 @@ int __kmem_cache_create(struct kmem_cach
size &= ~(BYTES_PER_WORD - 1);
}
- /* calculate the final buffer alignment: */
-
- /* 1) arch recommendation: can be overridden for debug */
- if (flags & SLAB_HWCACHE_ALIGN) {
- /*
- * Default alignment: as specified by the arch code. Except if
- * an object is really small, then squeeze multiple objects into
- * one cacheline.
- */
- ralign = cache_line_size();
- while (size <= ralign / 2)
- ralign /= 2;
- } else {
- ralign = BYTES_PER_WORD;
- }
-
/*
* Redzoning and user store require word alignment or possibly larger.
* Note this will be overridden by architecture or caller mandated
@@ -2249,10 +2233,6 @@ int __kmem_cache_create(struct kmem_cach
size &= ~(REDZONE_ALIGN - 1);
}
- /* 2) arch mandated alignment */
- if (ralign < ARCH_SLAB_MINALIGN) {
- ralign = ARCH_SLAB_MINALIGN;
- }
/* 3) caller mandated alignment */
if (ralign < cachep->align) {
ralign = cachep->align;
Index: linux-2.6/mm/slab_common.c
===================================================================
--- linux-2.6.orig/mm/slab_common.c 2012-06-13 03:44:57.201477116 -0500
+++ linux-2.6/mm/slab_common.c 2012-06-13 03:45:03.621476983 -0500
@@ -25,6 +25,34 @@ DEFINE_MUTEX(slab_mutex);
struct kmem_cache *kmem_cache;
/*
+ * Figure out what the alignment of the objects will be given a set of
+ * flags, a user specified alignment and the size of the objects.
+ */
+unsigned long calculate_alignment(unsigned long flags,
+ unsigned long align, unsigned long size)
+{
+ /*
+ * If the user wants hardware cache aligned objects then follow that
+ * suggestion if the object is sufficiently large.
+ *
+ * The hardware cache alignment cannot override the specified
+ * alignment though. If that is greater then use it.
+ */
+ if (flags & SLAB_HWCACHE_ALIGN) {
+ unsigned long ralign = cache_line_size();
+ while (size <= ralign / 2)
+ ralign /= 2;
+ align = max(align, ralign);
+ }
+
+ if (align < ARCH_SLAB_MINALIGN)
+ align = ARCH_SLAB_MINALIGN;
+
+ return ALIGN(align, sizeof(void *));
+}
+
+
+/*
* kmem_cache_create - Create a cache.
* @name: A string which is used in /proc/slabinfo to identify this cache.
* @size: The size of objects to be created in this cache.
@@ -118,7 +146,7 @@ struct kmem_cache *kmem_cache_create(con
s->size = s->object_size = size;
s->ctor = ctor;
s->flags = flags;
- s->align = align;
+ s->align = calculate_alignment(flags, align, size);
r = __kmem_cache_create(s);
Index: linux-2.6/mm/slob.c
===================================================================
--- linux-2.6.orig/mm/slob.c 2012-06-13 03:44:57.125477118 -0500
+++ linux-2.6/mm/slob.c 2012-06-13 03:45:03.621476983 -0500
@@ -124,7 +124,6 @@ static inline void clear_slob_page_free(
#define SLOB_UNIT sizeof(slob_t)
#define SLOB_UNITS(size) (((size) + SLOB_UNIT - 1)/SLOB_UNIT)
-#define SLOB_ALIGN L1_CACHE_BYTES
/*
* struct slob_rcu is inserted at the tail of allocated slob blocks, which
@@ -510,20 +509,10 @@ EXPORT_SYMBOL(ksize);
int __kmem_cache_create(struct kmem_cache *c)
{
- int align = c->align;
-
if (c->flags & SLAB_DESTROY_BY_RCU) {
/* leave room for rcu footer at the end of object */
c->size += sizeof(struct slob_rcu);
}
- /* ignore alignment unless it's forced */
- c->align = (c->flags & SLAB_HWCACHE_ALIGN) ? SLOB_ALIGN : 0;
- if (c->align < ARCH_SLAB_MINALIGN)
- c->align = ARCH_SLAB_MINALIGN;
- if (c->align < align)
- c->align = align;
-
- kmemleak_alloc(c, sizeof(struct kmem_cache), 1, GFP_KERNEL);
return 0;
}
Index: linux-2.6/mm/slub.c
===================================================================
--- linux-2.6.orig/mm/slub.c 2012-06-13 03:44:57.161477117 -0500
+++ linux-2.6/mm/slub.c 2012-06-13 03:45:03.625476983 -0500
@@ -2737,32 +2737,6 @@ static inline int calculate_order(int si
return -ENOSYS;
}
-/*
- * Figure out what the alignment of the objects will be.
- */
-static unsigned long calculate_alignment(unsigned long flags,
- unsigned long align, unsigned long size)
-{
- /*
- * If the user wants hardware cache aligned objects then follow that
- * suggestion if the object is sufficiently large.
- *
- * The hardware cache alignment cannot override the specified
- * alignment though. If that is greater then use it.
- */
- if (flags & SLAB_HWCACHE_ALIGN) {
- unsigned long ralign = cache_line_size();
- while (size <= ralign / 2)
- ralign /= 2;
- align = max(align, ralign);
- }
-
- if (align < ARCH_SLAB_MINALIGN)
- align = ARCH_SLAB_MINALIGN;
-
- return ALIGN(align, sizeof(void *));
-}
-
static void
init_kmem_cache_node(struct kmem_cache_node *n)
{
@@ -2968,14 +2942,6 @@ static int calculate_sizes(struct kmem_c
#endif
/*
- * Determine the alignment based on various parameters that the
- * user specified and the dynamic determination of cache line size
- * on bootup.
- */
- align = calculate_alignment(flags, align, s->object_size);
- s->align = align;
-
- /*
* SLUB stores one object immediately after another beginning from
* offset 0. In order to align the objects we have to simply size
* each object to conform to the alignment.
@@ -3009,7 +2975,6 @@ static int calculate_sizes(struct kmem_c
s->max = s->oo;
return !!oo_objects(s->oo);
-
}
static int kmem_cache_open(struct kmem_cache *s)
@@ -3233,7 +3198,7 @@ static struct kmem_cache *__init create_
s = kmem_cache_zalloc(kmem_cache, GFP_NOWAIT);
s->name = name;
s->size = s->object_size = size;
- s->align = ARCH_KMALLOC_MINALIGN;
+ s->align = calculate_alignment(flags, ARCH_KMALLOC_MINALIGN, size);
s->flags = flags;
/*
@@ -3694,6 +3659,8 @@ void __init kmem_cache_init(void)
kmem_cache_node->name = "kmem_cache_node";
kmem_cache_node->size = kmem_cache_node->object_size = sizeof(struct kmem_cache_node);
kmem_cache_node->flags = SLAB_HWCACHE_ALIGN;
+ kmem_cache_node->align = calculate_alignment(SLAB_HWCACHE_ALIGN,
+ 0, sizeof(struct kmem_cache_node));
r = kmem_cache_open(kmem_cache_node);
if (r)
@@ -3708,6 +3675,8 @@ void __init kmem_cache_init(void)
kmem_cache->name = "kmem_cache";
kmem_cache->size = kmem_cache->object_size = kmem_size;
kmem_cache->flags = SLAB_HWCACHE_ALIGN;
+ kmem_cache->align = calculate_alignment(SLAB_HWCACHE_ALIGN,
+ 0, sizeof(struct kmem_cache));
r = kmem_cache_open(kmem_cache);
if (r)
@@ -3931,7 +3900,9 @@ struct kmem_cache *__kmem_cache_alias(co
int __kmem_cache_create(struct kmem_cache *s)
{
- int r = kmem_cache_open(s);
+ int r;
+
+ r = kmem_cache_open(s);
if (r)
return r;
Index: linux-2.6/mm/slab.h
===================================================================
--- linux-2.6.orig/mm/slab.h 2012-06-13 03:44:57.177477117 -0500
+++ linux-2.6/mm/slab.h 2012-06-13 03:45:03.625476983 -0500
@@ -32,6 +32,9 @@ extern struct list_head slab_caches;
/* The slab cache that manages slab cache information */
extern struct kmem_cache *kmem_cache;
+unsigned long calculate_alignment(unsigned long flags,
+ unsigned long align, unsigned long size);
+
/* Functions provided by the slab allocators */
int __kmem_cache_create(struct kmem_cache *s);
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>
^ permalink raw reply [flat|nested] 40+ messages in thread* Re: Common [00/20] Sl[auo]b: Common code rework V5 (for merge)
2012-06-13 15:24 Common [00/20] Sl[auo]b: Common code rework V5 (for merge) Christoph Lameter
` (16 preceding siblings ...)
2012-06-13 15:25 ` Common [20/20] Common alignment code Christoph Lameter
@ 2012-06-14 8:14 ` Glauber Costa
2012-06-14 14:03 ` Christoph Lameter
17 siblings, 1 reply; 40+ messages in thread
From: Glauber Costa @ 2012-06-14 8:14 UTC (permalink / raw)
To: Christoph Lameter
Cc: Pekka Enberg, linux-mm, David Rientjes, Matt Mackall, Joonsoo Kim
On 06/13/2012 07:24 PM, Christoph Lameter wrote:
> V4->V5
> - Rediff against current upstream + Pekka's cleanup branch.
>
> V3->V4:
> - Do not use the COMMON macro anymore.
> - Fixup various issues
> - No general sysfs support yet due to lockdep issues with
> keys in kmalloc'ed memory.
>
> V2->V3:
> - Incorporate more feedback from Joonsoo Kim and Glauber Costa
> - And a couple more patches to deal with slab duping and move
> more code to slab_common.c
>
> V1->V2:
> - Incorporate glommers feedback.
> - Add 2 more patches dealing with common code in kmem_cache_destroy
>
> This is a series of patches that extracts common functionality from
> slab allocators into a common code base. The intend is to standardize
> as much as possible of the allocator behavior while keeping the
> distinctive features of each allocator which are mostly due to their
> storage format and serialization approaches.
>
> This patchset makes a beginning by extracting common functionality in
> kmem_cache_create() and kmem_cache_destroy(). However, there are
> numerous other areas where such work could be beneficial:
>
Christoph,
I rebased my series on top of yours, and started testing. After some
debugging, some of the bugs were pinpointed to your code. I was going to
send patches for it in the belief the series was already in somewhere.
Since you are sending it again, I'll just point them here. If people
prefer, to avoid having you resend the series, I'll be happy to post mine.
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>
^ permalink raw reply [flat|nested] 40+ messages in thread* Re: Common [00/20] Sl[auo]b: Common code rework V5 (for merge)
2012-06-14 8:14 ` Common [00/20] Sl[auo]b: Common code rework V5 (for merge) Glauber Costa
@ 2012-06-14 14:03 ` Christoph Lameter
0 siblings, 0 replies; 40+ messages in thread
From: Christoph Lameter @ 2012-06-14 14:03 UTC (permalink / raw)
To: Glauber Costa
Cc: Pekka Enberg, linux-mm, David Rientjes, Matt Mackall, Joonsoo Kim
On Thu, 14 Jun 2012, Glauber Costa wrote:
> I rebased my series on top of yours, and started testing. After some
> debugging, some of the bugs were pinpointed to your code. I was going to send
> patches for it in the belief the series was already in somewhere.
>
> Since you are sending it again, I'll just point them here. If people prefer,
> to avoid having you resend the series, I'll be happy to post mine.
Yes, just post them here....
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>
^ permalink raw reply [flat|nested] 40+ messages in thread