From 9dfc6e68bfe6ee452efb1a4e9ca26a9007f2b864 Mon Sep 17 00:00:00 2001 From: Christoph Lameter Date: Fri, 18 Dec 2009 16:26:20 -0600 Subject: SLUB: Use this_cpu operations in slub Using per cpu allocations removes the needs for the per cpu arrays in the kmem_cache struct. These could get quite big if we have to support systems with thousands of cpus. The use of this_cpu_xx operations results in: 1. The size of kmem_cache for SMP configuration shrinks since we will only need 1 pointer instead of NR_CPUS. The same pointer can be used by all processors. Reduces cache footprint of the allocator. 2. We can dynamically size kmem_cache according to the actual nodes in the system meaning less memory overhead for configurations that may potentially support up to 1k NUMA nodes / 4k cpus. 3. We can remove the diddle widdle with allocating and releasing of kmem_cache_cpu structures when bringing up and shutting down cpus. The cpu alloc logic will do it all for us. Removes some portions of the cpu hotplug functionality. 4. Fastpath performance increases since per cpu pointer lookups and address calculations are avoided. V7-V8 - Convert missed get_cpu_slab() under CONFIG_SLUB_STATS Signed-off-by: Christoph Lameter Signed-off-by: Pekka Enberg --- include/linux/slub_def.h | 6 +- mm/slub.c | 202 +++++++++++------------------------------------ 2 files changed, 49 insertions(+), 159 deletions(-) diff --git a/include/linux/slub_def.h b/include/linux/slub_def.h index 1e14beb23f9b..17ebe0f89bf3 100644 --- a/include/linux/slub_def.h +++ b/include/linux/slub_def.h @@ -69,6 +69,7 @@ struct kmem_cache_order_objects { * Slab cache management. */ struct kmem_cache { + struct kmem_cache_cpu *cpu_slab; /* Used for retriving partial slabs etc */ unsigned long flags; int size; /* The size of an object including meta data */ @@ -104,11 +105,6 @@ struct kmem_cache { int remote_node_defrag_ratio; struct kmem_cache_node *node[MAX_NUMNODES]; #endif -#ifdef CONFIG_SMP - struct kmem_cache_cpu *cpu_slab[NR_CPUS]; -#else - struct kmem_cache_cpu cpu_slab; -#endif }; /* diff --git a/mm/slub.c b/mm/slub.c index 8d71aaf888d7..d6c9ecf629d5 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -242,15 +242,6 @@ static inline struct kmem_cache_node *get_node(struct kmem_cache *s, int node) #endif } -static inline struct kmem_cache_cpu *get_cpu_slab(struct kmem_cache *s, int cpu) -{ -#ifdef CONFIG_SMP - return s->cpu_slab[cpu]; -#else - return &s->cpu_slab; -#endif -} - /* Verify that a pointer has an address that is valid within a slab page */ static inline int check_valid_pointer(struct kmem_cache *s, struct page *page, const void *object) @@ -1124,7 +1115,7 @@ static struct page *allocate_slab(struct kmem_cache *s, gfp_t flags, int node) if (!page) return NULL; - stat(get_cpu_slab(s, raw_smp_processor_id()), ORDER_FALLBACK); + stat(this_cpu_ptr(s->cpu_slab), ORDER_FALLBACK); } if (kmemcheck_enabled @@ -1422,7 +1413,7 @@ static struct page *get_partial(struct kmem_cache *s, gfp_t flags, int node) static void unfreeze_slab(struct kmem_cache *s, struct page *page, int tail) { struct kmem_cache_node *n = get_node(s, page_to_nid(page)); - struct kmem_cache_cpu *c = get_cpu_slab(s, smp_processor_id()); + struct kmem_cache_cpu *c = this_cpu_ptr(s->cpu_slab); __ClearPageSlubFrozen(page); if (page->inuse) { @@ -1454,7 +1445,7 @@ static void unfreeze_slab(struct kmem_cache *s, struct page *page, int tail) slab_unlock(page); } else { slab_unlock(page); - stat(get_cpu_slab(s, raw_smp_processor_id()), FREE_SLAB); + stat(__this_cpu_ptr(s->cpu_slab), FREE_SLAB); discard_slab(s, page); } } @@ -1507,7 +1498,7 @@ static inline void flush_slab(struct kmem_cache *s, struct kmem_cache_cpu *c) */ static inline void __flush_cpu_slab(struct kmem_cache *s, int cpu) { - struct kmem_cache_cpu *c = get_cpu_slab(s, cpu); + struct kmem_cache_cpu *c = per_cpu_ptr(s->cpu_slab, cpu); if (likely(c && c->page)) flush_slab(s, c); @@ -1673,7 +1664,7 @@ new_slab: local_irq_disable(); if (new) { - c = get_cpu_slab(s, smp_processor_id()); + c = __this_cpu_ptr(s->cpu_slab); stat(c, ALLOC_SLAB); if (c->page) flush_slab(s, c); @@ -1711,7 +1702,7 @@ static __always_inline void *slab_alloc(struct kmem_cache *s, void **object; struct kmem_cache_cpu *c; unsigned long flags; - unsigned int objsize; + unsigned long objsize; gfpflags &= gfp_allowed_mask; @@ -1722,14 +1713,14 @@ static __always_inline void *slab_alloc(struct kmem_cache *s, return NULL; local_irq_save(flags); - c = get_cpu_slab(s, smp_processor_id()); + c = __this_cpu_ptr(s->cpu_slab); + object = c->freelist; objsize = c->objsize; - if (unlikely(!c->freelist || !node_match(c, node))) + if (unlikely(!object || !node_match(c, node))) object = __slab_alloc(s, gfpflags, node, addr, c); else { - object = c->freelist; c->freelist = object[c->offset]; stat(c, ALLOC_FASTPATH); } @@ -1800,7 +1791,7 @@ static void __slab_free(struct kmem_cache *s, struct page *page, void **object = (void *)x; struct kmem_cache_cpu *c; - c = get_cpu_slab(s, raw_smp_processor_id()); + c = __this_cpu_ptr(s->cpu_slab); stat(c, FREE_SLOWPATH); slab_lock(page); @@ -1872,7 +1863,7 @@ static __always_inline void slab_free(struct kmem_cache *s, kmemleak_free_recursive(x, s->flags); local_irq_save(flags); - c = get_cpu_slab(s, smp_processor_id()); + c = __this_cpu_ptr(s->cpu_slab); kmemcheck_slab_free(s, object, c->objsize); debug_check_no_locks_freed(object, c->objsize); if (!(s->flags & SLAB_DEBUG_OBJECTS)) @@ -2095,130 +2086,28 @@ init_kmem_cache_node(struct kmem_cache_node *n, struct kmem_cache *s) #endif } -#ifdef CONFIG_SMP -/* - * Per cpu array for per cpu structures. - * - * The per cpu array places all kmem_cache_cpu structures from one processor - * close together meaning that it becomes possible that multiple per cpu - * structures are contained in one cacheline. This may be particularly - * beneficial for the kmalloc caches. - * - * A desktop system typically has around 60-80 slabs. With 100 here we are - * likely able to get per cpu structures for all caches from the array defined - * here. We must be able to cover all kmalloc caches during bootstrap. - * - * If the per cpu array is exhausted then fall back to kmalloc - * of individual cachelines. No sharing is possible then. - */ -#define NR_KMEM_CACHE_CPU 100 - -static DEFINE_PER_CPU(struct kmem_cache_cpu [NR_KMEM_CACHE_CPU], - kmem_cache_cpu); - -static DEFINE_PER_CPU(struct kmem_cache_cpu *, kmem_cache_cpu_free); -static DECLARE_BITMAP(kmem_cach_cpu_free_init_once, CONFIG_NR_CPUS); - -static struct kmem_cache_cpu *alloc_kmem_cache_cpu(struct kmem_cache *s, - int cpu, gfp_t flags) -{ - struct kmem_cache_cpu *c = per_cpu(kmem_cache_cpu_free, cpu); - - if (c) - per_cpu(kmem_cache_cpu_free, cpu) = - (void *)c->freelist; - else { - /* Table overflow: So allocate ourselves */ - c = kmalloc_node( - ALIGN(sizeof(struct kmem_cache_cpu), cache_line_size()), - flags, cpu_to_node(cpu)); - if (!c) - return NULL; - } - - init_kmem_cache_cpu(s, c); - return c; -} - -static void free_kmem_cache_cpu(struct kmem_cache_cpu *c, int cpu) -{ - if (c < per_cpu(kmem_cache_cpu, cpu) || - c >= per_cpu(kmem_cache_cpu, cpu) + NR_KMEM_CACHE_CPU) { - kfree(c); - return; - } - c->freelist = (void *)per_cpu(kmem_cache_cpu_free, cpu); - per_cpu(kmem_cache_cpu_free, cpu) = c; -} - -static void free_kmem_cache_cpus(struct kmem_cache *s) -{ - int cpu; - - for_each_online_cpu(cpu) { - struct kmem_cache_cpu *c = get_cpu_slab(s, cpu); - - if (c) { - s->cpu_slab[cpu] = NULL; - free_kmem_cache_cpu(c, cpu); - } - } -} - -static int alloc_kmem_cache_cpus(struct kmem_cache *s, gfp_t flags) -{ - int cpu; - - for_each_online_cpu(cpu) { - struct kmem_cache_cpu *c = get_cpu_slab(s, cpu); +static DEFINE_PER_CPU(struct kmem_cache_cpu, kmalloc_percpu[SLUB_PAGE_SHIFT]); - if (c) - continue; - - c = alloc_kmem_cache_cpu(s, cpu, flags); - if (!c) { - free_kmem_cache_cpus(s); - return 0; - } - s->cpu_slab[cpu] = c; - } - return 1; -} - -/* - * Initialize the per cpu array. - */ -static void init_alloc_cpu_cpu(int cpu) -{ - int i; - - if (cpumask_test_cpu(cpu, to_cpumask(kmem_cach_cpu_free_init_once))) - return; - - for (i = NR_KMEM_CACHE_CPU - 1; i >= 0; i--) - free_kmem_cache_cpu(&per_cpu(kmem_cache_cpu, cpu)[i], cpu); - - cpumask_set_cpu(cpu, to_cpumask(kmem_cach_cpu_free_init_once)); -} - -static void __init init_alloc_cpu(void) +static inline int alloc_kmem_cache_cpus(struct kmem_cache *s, gfp_t flags) { int cpu; - for_each_online_cpu(cpu) - init_alloc_cpu_cpu(cpu); - } + if (s < kmalloc_caches + SLUB_PAGE_SHIFT && s >= kmalloc_caches) + /* + * Boot time creation of the kmalloc array. Use static per cpu data + * since the per cpu allocator is not available yet. + */ + s->cpu_slab = per_cpu_var(kmalloc_percpu) + (s - kmalloc_caches); + else + s->cpu_slab = alloc_percpu(struct kmem_cache_cpu); -#else -static inline void free_kmem_cache_cpus(struct kmem_cache *s) {} -static inline void init_alloc_cpu(void) {} + if (!s->cpu_slab) + return 0; -static inline int alloc_kmem_cache_cpus(struct kmem_cache *s, gfp_t flags) -{ - init_kmem_cache_cpu(s, &s->cpu_slab); + for_each_possible_cpu(cpu) + init_kmem_cache_cpu(s, per_cpu_ptr(s->cpu_slab, cpu)); return 1; } -#endif #ifdef CONFIG_NUMA /* @@ -2609,9 +2498,8 @@ static inline int kmem_cache_close(struct kmem_cache *s) int node; flush_all(s); - + free_percpu(s->cpu_slab); /* Attempt to free all objects */ - free_kmem_cache_cpus(s); for_each_node_state(node, N_NORMAL_MEMORY) { struct kmem_cache_node *n = get_node(s, node); @@ -2760,7 +2648,19 @@ static noinline struct kmem_cache *dma_kmalloc_cache(int index, gfp_t flags) realsize = kmalloc_caches[index].objsize; text = kasprintf(flags & ~SLUB_DMA, "kmalloc_dma-%d", (unsigned int)realsize); - s = kmalloc(kmem_size, flags & ~SLUB_DMA); + + if (flags & __GFP_WAIT) + s = kmalloc(kmem_size, flags & ~SLUB_DMA); + else { + int i; + + s = NULL; + for (i = 0; i < SLUB_PAGE_SHIFT; i++) + if (kmalloc_caches[i].size) { + s = kmalloc_caches + i; + break; + } + } /* * Must defer sysfs creation to a workqueue because we don't know @@ -3176,8 +3076,6 @@ void __init kmem_cache_init(void) int i; int caches = 0; - init_alloc_cpu(); - #ifdef CONFIG_NUMA /* * Must first have the slab cache available for the allocations of the @@ -3261,8 +3159,10 @@ void __init kmem_cache_init(void) #ifdef CONFIG_SMP register_cpu_notifier(&slab_notifier); - kmem_size = offsetof(struct kmem_cache, cpu_slab) + - nr_cpu_ids * sizeof(struct kmem_cache_cpu *); +#endif +#ifdef CONFIG_NUMA + kmem_size = offsetof(struct kmem_cache, node) + + nr_node_ids * sizeof(struct kmem_cache_node *); #else kmem_size = sizeof(struct kmem_cache); #endif @@ -3365,7 +3265,7 @@ struct kmem_cache *kmem_cache_create(const char *name, size_t size, * per cpu structures */ for_each_online_cpu(cpu) - get_cpu_slab(s, cpu)->objsize = s->objsize; + per_cpu_ptr(s->cpu_slab, cpu)->objsize = s->objsize; s->inuse = max_t(int, s->inuse, ALIGN(size, sizeof(void *))); up_write(&slub_lock); @@ -3422,11 +3322,9 @@ static int __cpuinit slab_cpuup_callback(struct notifier_block *nfb, switch (action) { case CPU_UP_PREPARE: case CPU_UP_PREPARE_FROZEN: - init_alloc_cpu_cpu(cpu); down_read(&slub_lock); list_for_each_entry(s, &slab_caches, list) - s->cpu_slab[cpu] = alloc_kmem_cache_cpu(s, cpu, - GFP_KERNEL); + init_kmem_cache_cpu(s, per_cpu_ptr(s->cpu_slab, cpu)); up_read(&slub_lock); break; @@ -3436,13 +3334,9 @@ static int __cpuinit slab_cpuup_callback(struct notifier_block *nfb, case CPU_DEAD_FROZEN: down_read(&slub_lock); list_for_each_entry(s, &slab_caches, list) { - struct kmem_cache_cpu *c = get_cpu_slab(s, cpu); - local_irq_save(flags); __flush_cpu_slab(s, cpu); local_irq_restore(flags); - free_kmem_cache_cpu(c, cpu); - s->cpu_slab[cpu] = NULL; } up_read(&slub_lock); break; @@ -3928,7 +3822,7 @@ static ssize_t show_slab_objects(struct kmem_cache *s, int cpu; for_each_possible_cpu(cpu) { - struct kmem_cache_cpu *c = get_cpu_slab(s, cpu); + struct kmem_cache_cpu *c = per_cpu_ptr(s->cpu_slab, cpu); if (!c || c->node < 0) continue; @@ -4353,7 +4247,7 @@ static int show_stat(struct kmem_cache *s, char *buf, enum stat_item si) return -ENOMEM; for_each_online_cpu(cpu) { - unsigned x = get_cpu_slab(s, cpu)->stat[si]; + unsigned x = per_cpu_ptr(s->cpu_slab, cpu)->stat[si]; data[cpu] = x; sum += x; @@ -4376,7 +4270,7 @@ static void clear_stat(struct kmem_cache *s, enum stat_item si) int cpu; for_each_online_cpu(cpu) - get_cpu_slab(s, cpu)->stat[si] = 0; + per_cpu_ptr(s->cpu_slab, cpu)->stat[si] = 0; } #define STAT_ATTR(si, text) \ -- cgit v1.2.3 From 756dee75872a2a764b478e18076360b8a4ec9045 Mon Sep 17 00:00:00 2001 From: Christoph Lameter Date: Fri, 18 Dec 2009 16:26:21 -0600 Subject: SLUB: Get rid of dynamic DMA kmalloc cache allocation Dynamic DMA kmalloc cache allocation is troublesome since the new percpu allocator does not support allocations in atomic contexts. Reserve some statically allocated kmalloc_cpu structures instead. Signed-off-by: Christoph Lameter Signed-off-by: Pekka Enberg --- include/linux/slub_def.h | 19 +++++++++++-------- mm/slub.c | 24 ++++++++++-------------- 2 files changed, 21 insertions(+), 22 deletions(-) diff --git a/include/linux/slub_def.h b/include/linux/slub_def.h index 17ebe0f89bf3..a78fb4ac2015 100644 --- a/include/linux/slub_def.h +++ b/include/linux/slub_def.h @@ -131,11 +131,21 @@ struct kmem_cache { #define SLUB_PAGE_SHIFT (PAGE_SHIFT + 2) +#ifdef CONFIG_ZONE_DMA +#define SLUB_DMA __GFP_DMA +/* Reserve extra caches for potential DMA use */ +#define KMALLOC_CACHES (2 * SLUB_PAGE_SHIFT - 6) +#else +/* Disable DMA functionality */ +#define SLUB_DMA (__force gfp_t)0 +#define KMALLOC_CACHES SLUB_PAGE_SHIFT +#endif + /* * We keep the general caches in an array of slab caches that are used for * 2^x bytes of allocations. */ -extern struct kmem_cache kmalloc_caches[SLUB_PAGE_SHIFT]; +extern struct kmem_cache kmalloc_caches[KMALLOC_CACHES]; /* * Sorry that the following has to be that ugly but some versions of GCC @@ -203,13 +213,6 @@ static __always_inline struct kmem_cache *kmalloc_slab(size_t size) return &kmalloc_caches[index]; } -#ifdef CONFIG_ZONE_DMA -#define SLUB_DMA __GFP_DMA -#else -/* Disable DMA functionality */ -#define SLUB_DMA (__force gfp_t)0 -#endif - void *kmem_cache_alloc(struct kmem_cache *, gfp_t); void *__kmalloc(size_t size, gfp_t flags); diff --git a/mm/slub.c b/mm/slub.c index d6c9ecf629d5..cdb7f0214af0 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -2092,7 +2092,7 @@ static inline int alloc_kmem_cache_cpus(struct kmem_cache *s, gfp_t flags) { int cpu; - if (s < kmalloc_caches + SLUB_PAGE_SHIFT && s >= kmalloc_caches) + if (s < kmalloc_caches + KMALLOC_CACHES && s >= kmalloc_caches) /* * Boot time creation of the kmalloc array. Use static per cpu data * since the per cpu allocator is not available yet. @@ -2539,7 +2539,7 @@ EXPORT_SYMBOL(kmem_cache_destroy); * Kmalloc subsystem *******************************************************************/ -struct kmem_cache kmalloc_caches[SLUB_PAGE_SHIFT] __cacheline_aligned; +struct kmem_cache kmalloc_caches[KMALLOC_CACHES] __cacheline_aligned; EXPORT_SYMBOL(kmalloc_caches); static int __init setup_slub_min_order(char *str) @@ -2629,6 +2629,7 @@ static noinline struct kmem_cache *dma_kmalloc_cache(int index, gfp_t flags) char *text; size_t realsize; unsigned long slabflags; + int i; s = kmalloc_caches_dma[index]; if (s) @@ -2649,18 +2650,13 @@ static noinline struct kmem_cache *dma_kmalloc_cache(int index, gfp_t flags) text = kasprintf(flags & ~SLUB_DMA, "kmalloc_dma-%d", (unsigned int)realsize); - if (flags & __GFP_WAIT) - s = kmalloc(kmem_size, flags & ~SLUB_DMA); - else { - int i; + s = NULL; + for (i = 0; i < KMALLOC_CACHES; i++) + if (!kmalloc_caches[i].size) + break; - s = NULL; - for (i = 0; i < SLUB_PAGE_SHIFT; i++) - if (kmalloc_caches[i].size) { - s = kmalloc_caches + i; - break; - } - } + BUG_ON(i >= KMALLOC_CACHES); + s = kmalloc_caches + i; /* * Must defer sysfs creation to a workqueue because we don't know @@ -2674,7 +2670,7 @@ static noinline struct kmem_cache *dma_kmalloc_cache(int index, gfp_t flags) if (!s || !text || !kmem_cache_open(s, flags, text, realsize, ARCH_KMALLOC_MINALIGN, slabflags, NULL)) { - kfree(s); + s->size = 0; kfree(text); goto unlock_out; } -- cgit v1.2.3 From ff12059ed14b0773d7bbef86f98218ada6c20770 Mon Sep 17 00:00:00 2001 From: Christoph Lameter Date: Fri, 18 Dec 2009 16:26:22 -0600 Subject: SLUB: this_cpu: Remove slub kmem_cache fields Remove the fields in struct kmem_cache_cpu that were used to cache data from struct kmem_cache when they were in different cachelines. The cacheline that holds the per cpu array pointer now also holds these values. We can cut down the struct kmem_cache_cpu size to almost half. The get_freepointer() and set_freepointer() functions that used to be only intended for the slow path now are also useful for the hot path since access to the size field does not require accessing an additional cacheline anymore. This results in consistent use of functions for setting the freepointer of objects throughout SLUB. Also we initialize all possible kmem_cache_cpu structures when a slab is created. No need to initialize them when a processor or node comes online. Signed-off-by: Christoph Lameter Signed-off-by: Pekka Enberg --- include/linux/slub_def.h | 2 -- mm/slub.c | 76 +++++++++++------------------------------------- 2 files changed, 17 insertions(+), 61 deletions(-) diff --git a/include/linux/slub_def.h b/include/linux/slub_def.h index a78fb4ac2015..0249d4175bac 100644 --- a/include/linux/slub_def.h +++ b/include/linux/slub_def.h @@ -38,8 +38,6 @@ struct kmem_cache_cpu { void **freelist; /* Pointer to first free per cpu object */ struct page *page; /* The slab from which we are allocating */ int node; /* The node of the page (or -1 for debug) */ - unsigned int offset; /* Freepointer offset (in word units) */ - unsigned int objsize; /* Size of an object (from kmem_cache) */ #ifdef CONFIG_SLUB_STATS unsigned stat[NR_SLUB_STAT_ITEMS]; #endif diff --git a/mm/slub.c b/mm/slub.c index cdb7f0214af0..30d2dde27563 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -260,13 +260,6 @@ static inline int check_valid_pointer(struct kmem_cache *s, return 1; } -/* - * Slow version of get and set free pointer. - * - * This version requires touching the cache lines of kmem_cache which - * we avoid to do in the fast alloc free paths. There we obtain the offset - * from the page struct. - */ static inline void *get_freepointer(struct kmem_cache *s, void *object) { return *(void **)(object + s->offset); @@ -1473,10 +1466,10 @@ static void deactivate_slab(struct kmem_cache *s, struct kmem_cache_cpu *c) /* Retrieve object from cpu_freelist */ object = c->freelist; - c->freelist = c->freelist[c->offset]; + c->freelist = get_freepointer(s, c->freelist); /* And put onto the regular freelist */ - object[c->offset] = page->freelist; + set_freepointer(s, object, page->freelist); page->freelist = object; page->inuse--; } @@ -1635,7 +1628,7 @@ load_freelist: if (unlikely(SLABDEBUG && PageSlubDebug(c->page))) goto debug; - c->freelist = object[c->offset]; + c->freelist = get_freepointer(s, object); c->page->inuse = c->page->objects; c->page->freelist = NULL; c->node = page_to_nid(c->page); @@ -1681,7 +1674,7 @@ debug: goto another_slab; c->page->inuse++; - c->page->freelist = object[c->offset]; + c->page->freelist = get_freepointer(s, object); c->node = -1; goto unlock_out; } @@ -1702,7 +1695,6 @@ static __always_inline void *slab_alloc(struct kmem_cache *s, void **object; struct kmem_cache_cpu *c; unsigned long flags; - unsigned long objsize; gfpflags &= gfp_allowed_mask; @@ -1715,22 +1707,21 @@ static __always_inline void *slab_alloc(struct kmem_cache *s, local_irq_save(flags); c = __this_cpu_ptr(s->cpu_slab); object = c->freelist; - objsize = c->objsize; if (unlikely(!object || !node_match(c, node))) object = __slab_alloc(s, gfpflags, node, addr, c); else { - c->freelist = object[c->offset]; + c->freelist = get_freepointer(s, object); stat(c, ALLOC_FASTPATH); } local_irq_restore(flags); if (unlikely(gfpflags & __GFP_ZERO) && object) - memset(object, 0, objsize); + memset(object, 0, s->objsize); - kmemcheck_slab_alloc(s, gfpflags, object, c->objsize); - kmemleak_alloc_recursive(object, objsize, 1, s->flags, gfpflags); + kmemcheck_slab_alloc(s, gfpflags, object, s->objsize); + kmemleak_alloc_recursive(object, s->objsize, 1, s->flags, gfpflags); return object; } @@ -1785,7 +1776,7 @@ EXPORT_SYMBOL(kmem_cache_alloc_node_notrace); * handling required then we can return immediately. */ static void __slab_free(struct kmem_cache *s, struct page *page, - void *x, unsigned long addr, unsigned int offset) + void *x, unsigned long addr) { void *prior; void **object = (void *)x; @@ -1799,7 +1790,8 @@ static void __slab_free(struct kmem_cache *s, struct page *page, goto debug; checks_ok: - prior = object[offset] = page->freelist; + prior = page->freelist; + set_freepointer(s, object, prior); page->freelist = object; page->inuse--; @@ -1864,16 +1856,16 @@ static __always_inline void slab_free(struct kmem_cache *s, kmemleak_free_recursive(x, s->flags); local_irq_save(flags); c = __this_cpu_ptr(s->cpu_slab); - kmemcheck_slab_free(s, object, c->objsize); - debug_check_no_locks_freed(object, c->objsize); + kmemcheck_slab_free(s, object, s->objsize); + debug_check_no_locks_freed(object, s->objsize); if (!(s->flags & SLAB_DEBUG_OBJECTS)) - debug_check_no_obj_freed(object, c->objsize); + debug_check_no_obj_freed(object, s->objsize); if (likely(page == c->page && c->node >= 0)) { - object[c->offset] = c->freelist; + set_freepointer(s, object, c->freelist); c->freelist = object; stat(c, FREE_FASTPATH); } else - __slab_free(s, page, x, addr, c->offset); + __slab_free(s, page, x, addr); local_irq_restore(flags); } @@ -2060,19 +2052,6 @@ static unsigned long calculate_alignment(unsigned long flags, return ALIGN(align, sizeof(void *)); } -static void init_kmem_cache_cpu(struct kmem_cache *s, - struct kmem_cache_cpu *c) -{ - c->page = NULL; - c->freelist = NULL; - c->node = 0; - c->offset = s->offset / sizeof(void *); - c->objsize = s->objsize; -#ifdef CONFIG_SLUB_STATS - memset(c->stat, 0, NR_SLUB_STAT_ITEMS * sizeof(unsigned)); -#endif -} - static void init_kmem_cache_node(struct kmem_cache_node *n, struct kmem_cache *s) { @@ -2090,8 +2069,6 @@ static DEFINE_PER_CPU(struct kmem_cache_cpu, kmalloc_percpu[SLUB_PAGE_SHIFT]); static inline int alloc_kmem_cache_cpus(struct kmem_cache *s, gfp_t flags) { - int cpu; - if (s < kmalloc_caches + KMALLOC_CACHES && s >= kmalloc_caches) /* * Boot time creation of the kmalloc array. Use static per cpu data @@ -2104,8 +2081,6 @@ static inline int alloc_kmem_cache_cpus(struct kmem_cache *s, gfp_t flags) if (!s->cpu_slab) return 0; - for_each_possible_cpu(cpu) - init_kmem_cache_cpu(s, per_cpu_ptr(s->cpu_slab, cpu)); return 1; } @@ -2391,6 +2366,7 @@ static int kmem_cache_open(struct kmem_cache *s, gfp_t gfpflags, if (alloc_kmem_cache_cpus(s, gfpflags & ~SLUB_DMA)) return 1; + free_kmem_cache_nodes(s); error: if (flags & SLAB_PANIC) @@ -3247,22 +3223,12 @@ struct kmem_cache *kmem_cache_create(const char *name, size_t size, down_write(&slub_lock); s = find_mergeable(size, align, flags, name, ctor); if (s) { - int cpu; - s->refcount++; /* * Adjust the object sizes so that we clear * the complete object on kzalloc. */ s->objsize = max(s->objsize, (int)size); - - /* - * And then we need to update the object size in the - * per cpu structures - */ - for_each_online_cpu(cpu) - per_cpu_ptr(s->cpu_slab, cpu)->objsize = s->objsize; - s->inuse = max_t(int, s->inuse, ALIGN(size, sizeof(void *))); up_write(&slub_lock); @@ -3316,14 +3282,6 @@ static int __cpuinit slab_cpuup_callback(struct notifier_block *nfb, unsigned long flags; switch (action) { - case CPU_UP_PREPARE: - case CPU_UP_PREPARE_FROZEN: - down_read(&slub_lock); - list_for_each_entry(s, &slab_caches, list) - init_kmem_cache_cpu(s, per_cpu_ptr(s->cpu_slab, cpu)); - up_read(&slub_lock); - break; - case CPU_UP_CANCELED: case CPU_UP_CANCELED_FROZEN: case CPU_DEAD: -- cgit v1.2.3 From 84e554e6865c4f4ae84d38800cf270b9a67901cc Mon Sep 17 00:00:00 2001 From: Christoph Lameter Date: Fri, 18 Dec 2009 16:26:23 -0600 Subject: SLUB: Make slub statistics use this_cpu_inc this_cpu_inc() translates into a single instruction on x86 and does not need any register. So use it in stat(). We also want to avoid the calculation of the per cpu kmem_cache_cpu structure pointer. So pass a kmem_cache pointer instead of a kmem_cache_cpu pointer. Signed-off-by: Christoph Lameter Signed-off-by: Pekka Enberg --- mm/slub.c | 43 ++++++++++++++++++++----------------------- 1 file changed, 20 insertions(+), 23 deletions(-) diff --git a/mm/slub.c b/mm/slub.c index 30d2dde27563..bddae72f6f49 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -217,10 +217,10 @@ static inline void sysfs_slab_remove(struct kmem_cache *s) #endif -static inline void stat(struct kmem_cache_cpu *c, enum stat_item si) +static inline void stat(struct kmem_cache *s, enum stat_item si) { #ifdef CONFIG_SLUB_STATS - c->stat[si]++; + __this_cpu_inc(s->cpu_slab->stat[si]); #endif } @@ -1108,7 +1108,7 @@ static struct page *allocate_slab(struct kmem_cache *s, gfp_t flags, int node) if (!page) return NULL; - stat(this_cpu_ptr(s->cpu_slab), ORDER_FALLBACK); + stat(s, ORDER_FALLBACK); } if (kmemcheck_enabled @@ -1406,23 +1406,22 @@ static struct page *get_partial(struct kmem_cache *s, gfp_t flags, int node) static void unfreeze_slab(struct kmem_cache *s, struct page *page, int tail) { struct kmem_cache_node *n = get_node(s, page_to_nid(page)); - struct kmem_cache_cpu *c = this_cpu_ptr(s->cpu_slab); __ClearPageSlubFrozen(page); if (page->inuse) { if (page->freelist) { add_partial(n, page, tail); - stat(c, tail ? DEACTIVATE_TO_TAIL : DEACTIVATE_TO_HEAD); + stat(s, tail ? DEACTIVATE_TO_TAIL : DEACTIVATE_TO_HEAD); } else { - stat(c, DEACTIVATE_FULL); + stat(s, DEACTIVATE_FULL); if (SLABDEBUG && PageSlubDebug(page) && (s->flags & SLAB_STORE_USER)) add_full(n, page); } slab_unlock(page); } else { - stat(c, DEACTIVATE_EMPTY); + stat(s, DEACTIVATE_EMPTY); if (n->nr_partial < s->min_partial) { /* * Adding an empty slab to the partial slabs in order @@ -1438,7 +1437,7 @@ static void unfreeze_slab(struct kmem_cache *s, struct page *page, int tail) slab_unlock(page); } else { slab_unlock(page); - stat(__this_cpu_ptr(s->cpu_slab), FREE_SLAB); + stat(s, FREE_SLAB); discard_slab(s, page); } } @@ -1453,7 +1452,7 @@ static void deactivate_slab(struct kmem_cache *s, struct kmem_cache_cpu *c) int tail = 1; if (page->freelist) - stat(c, DEACTIVATE_REMOTE_FREES); + stat(s, DEACTIVATE_REMOTE_FREES); /* * Merge cpu freelist into slab freelist. Typically we get here * because both freelists are empty. So this is unlikely @@ -1479,7 +1478,7 @@ static void deactivate_slab(struct kmem_cache *s, struct kmem_cache_cpu *c) static inline void flush_slab(struct kmem_cache *s, struct kmem_cache_cpu *c) { - stat(c, CPUSLAB_FLUSH); + stat(s, CPUSLAB_FLUSH); slab_lock(c->page); deactivate_slab(s, c); } @@ -1619,7 +1618,7 @@ static void *__slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node, if (unlikely(!node_match(c, node))) goto another_slab; - stat(c, ALLOC_REFILL); + stat(s, ALLOC_REFILL); load_freelist: object = c->page->freelist; @@ -1634,7 +1633,7 @@ load_freelist: c->node = page_to_nid(c->page); unlock_out: slab_unlock(c->page); - stat(c, ALLOC_SLOWPATH); + stat(s, ALLOC_SLOWPATH); return object; another_slab: @@ -1644,7 +1643,7 @@ new_slab: new = get_partial(s, gfpflags, node); if (new) { c->page = new; - stat(c, ALLOC_FROM_PARTIAL); + stat(s, ALLOC_FROM_PARTIAL); goto load_freelist; } @@ -1658,7 +1657,7 @@ new_slab: if (new) { c = __this_cpu_ptr(s->cpu_slab); - stat(c, ALLOC_SLAB); + stat(s, ALLOC_SLAB); if (c->page) flush_slab(s, c); slab_lock(new); @@ -1713,7 +1712,7 @@ static __always_inline void *slab_alloc(struct kmem_cache *s, else { c->freelist = get_freepointer(s, object); - stat(c, ALLOC_FASTPATH); + stat(s, ALLOC_FASTPATH); } local_irq_restore(flags); @@ -1780,10 +1779,8 @@ static void __slab_free(struct kmem_cache *s, struct page *page, { void *prior; void **object = (void *)x; - struct kmem_cache_cpu *c; - c = __this_cpu_ptr(s->cpu_slab); - stat(c, FREE_SLOWPATH); + stat(s, FREE_SLOWPATH); slab_lock(page); if (unlikely(SLABDEBUG && PageSlubDebug(page))) @@ -1796,7 +1793,7 @@ checks_ok: page->inuse--; if (unlikely(PageSlubFrozen(page))) { - stat(c, FREE_FROZEN); + stat(s, FREE_FROZEN); goto out_unlock; } @@ -1809,7 +1806,7 @@ checks_ok: */ if (unlikely(!prior)) { add_partial(get_node(s, page_to_nid(page)), page, 1); - stat(c, FREE_ADD_PARTIAL); + stat(s, FREE_ADD_PARTIAL); } out_unlock: @@ -1822,10 +1819,10 @@ slab_empty: * Slab still on the partial list. */ remove_partial(s, page); - stat(c, FREE_REMOVE_PARTIAL); + stat(s, FREE_REMOVE_PARTIAL); } slab_unlock(page); - stat(c, FREE_SLAB); + stat(s, FREE_SLAB); discard_slab(s, page); return; @@ -1863,7 +1860,7 @@ static __always_inline void slab_free(struct kmem_cache *s, if (likely(page == c->page && c->node >= 0)) { set_freepointer(s, object, c->freelist); c->freelist = object; - stat(c, FREE_FASTPATH); + stat(s, FREE_FASTPATH); } else __slab_free(s, page, x, addr); -- cgit v1.2.3 From f3186a9c51eabe75b2780153ed7f07778d78b16e Mon Sep 17 00:00:00 2001 From: Haicheng Li Date: Wed, 6 Jan 2010 15:25:23 +0800 Subject: slab: initialize unused alien cache entry as NULL at alloc_alien_cache(). Comparing with existing code, it's a simpler way to use kzalloc_node() to ensure that each unused alien cache entry is NULL. CC: Eric Dumazet Acked-by: Andi Kleen Acked-by: Christoph Lameter Acked-by: Matt Mackall Signed-off-by: Haicheng Li Signed-off-by: Pekka Enberg --- mm/slab.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/mm/slab.c b/mm/slab.c index 7d41f15b48d3..0c632a946ea1 100644 --- a/mm/slab.c +++ b/mm/slab.c @@ -983,13 +983,11 @@ static struct array_cache **alloc_alien_cache(int node, int limit, gfp_t gfp) if (limit > 1) limit = 12; - ac_ptr = kmalloc_node(memsize, gfp, node); + ac_ptr = kzalloc_node(memsize, gfp, node); if (ac_ptr) { for_each_node(i) { - if (i == node || !node_online(i)) { - ac_ptr[i] = NULL; + if (i == node || !node_online(i)) continue; - } ac_ptr[i] = alloc_arraycache(node, limit, 0xbaadf00d, gfp); if (!ac_ptr[i]) { for (i--; i >= 0; i--) -- cgit v1.2.3 From 7738dd9e8f2bc1c249e00c9c20e018448fac0084 Mon Sep 17 00:00:00 2001 From: David Rientjes Date: Fri, 15 Jan 2010 12:49:56 -0800 Subject: slub: remove impossible condition `s' cannot be NULL if kmalloc_caches is not NULL. This conditional would trigger a NULL pointer on `s', anyway, since it is immediately derefernced if true. Acked-by: Christoph Lameter Signed-off-by: David Rientjes Signed-off-by: Pekka Enberg --- mm/slub.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mm/slub.c b/mm/slub.c index bddae72f6f49..8fbb2fd70b64 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -2641,7 +2641,7 @@ static noinline struct kmem_cache *dma_kmalloc_cache(int index, gfp_t flags) if (slab_state >= SYSFS) slabflags |= __SYSFS_ADD_DEFERRED; - if (!s || !text || !kmem_cache_open(s, flags, text, + if (!text || !kmem_cache_open(s, flags, text, realsize, ARCH_KMALLOC_MINALIGN, slabflags, NULL)) { s->size = 0; kfree(text); -- cgit v1.2.3 From 91efd773c74bb26b5409c85ad755d536448e229c Mon Sep 17 00:00:00 2001 From: Christoph Lameter Date: Thu, 21 Jan 2010 17:43:35 -0600 Subject: dma kmalloc handling fixes 1. We need kmalloc_percpu for all of the now extended kmalloc caches array not just for each shift value. 2. init_kmem_cache_nodes() must assume node 0 locality for statically allocated dma kmem_cache structures even after boot is complete. Reported-and-tested-by: Alex Chiang Signed-off-by: Christoph Lameter Signed-off-by: Pekka Enberg --- mm/slub.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/mm/slub.c b/mm/slub.c index 8fbb2fd70b64..bd4a9e942ace 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -2062,7 +2062,7 @@ init_kmem_cache_node(struct kmem_cache_node *n, struct kmem_cache *s) #endif } -static DEFINE_PER_CPU(struct kmem_cache_cpu, kmalloc_percpu[SLUB_PAGE_SHIFT]); +static DEFINE_PER_CPU(struct kmem_cache_cpu, kmalloc_percpu[KMALLOC_CACHES]); static inline int alloc_kmem_cache_cpus(struct kmem_cache *s, gfp_t flags) { @@ -2148,7 +2148,8 @@ static int init_kmem_cache_nodes(struct kmem_cache *s, gfp_t gfpflags) int node; int local_node; - if (slab_state >= UP) + if (slab_state >= UP && (s < kmalloc_caches || + s > kmalloc_caches + KMALLOC_CACHES)) local_node = page_to_nid(virt_to_page(s)); else local_node = 0; -- cgit v1.2.3 From 44b57f1cc72a4a30b31f11b07a927d1534f1b93d Mon Sep 17 00:00:00 2001 From: Nick Piggin Date: Wed, 27 Jan 2010 22:27:40 +1100 Subject: slab: fix regression in touched logic When factoring common code into transfer_objects in commit 3ded175 ("slab: add transfer_objects() function"), the 'touched' logic got a bit broken. When refilling from the shared array (taking objects from the shared array), we are making use of the shared array so it should be marked as touched. Subsequently pulling an element from the cpu array and allocating it should also touch the cpu array, but that is taken care of after the alloc_done label. (So yes, the cpu array was getting touched = 1 twice). So revert this logic to how it worked in earlier kernels. This also affects the behaviour in __drain_alien_cache, which would previously 'touch' the shared array and now does not. I think it is more logical not to touch there, because we are pushing objects into the shared array rather than pulling them off. So there is no good reason to postpone reaping them -- if the shared array is getting utilized, then it will get 'touched' in the alloc path (where this patch now restores the touch). Acked-by: Christoph Lameter Signed-off-by: Nick Piggin Signed-off-by: Pekka Enberg --- mm/slab.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/mm/slab.c b/mm/slab.c index 7451bdacaf18..f9626d51a4b1 100644 --- a/mm/slab.c +++ b/mm/slab.c @@ -935,7 +935,6 @@ static int transfer_objects(struct array_cache *to, from->avail -= nr; to->avail += nr; - to->touched = 1; return nr; } @@ -2963,8 +2962,10 @@ retry: spin_lock(&l3->list_lock); /* See if we can refill from the shared array */ - if (l3->shared && transfer_objects(ac, l3->shared, batchcount)) + if (l3->shared && transfer_objects(ac, l3->shared, batchcount)) { + l3->shared->touched = 1; goto alloc_done; + } while (batchcount > 0) { struct list_head *entry; -- cgit v1.2.3 From c4dbf35e6c1528b02be627d4f95a8a6e13b8f627 Mon Sep 17 00:00:00 2001 From: Andi Kleen Date: Thu, 11 Feb 2010 21:54:00 +0100 Subject: SLAB: Handle node-not-up case in fallback_alloc() When fallback_alloc() runs the node of the CPU might not be initialized yet. Handle this case by allocating in another node. Acked-by: David Rientjes Signed-off-by: Andi Kleen Signed-off-by: Pekka Enberg --- mm/slab.c | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/mm/slab.c b/mm/slab.c index 7451bdacaf18..6a813cc5802f 100644 --- a/mm/slab.c +++ b/mm/slab.c @@ -3210,7 +3210,24 @@ retry: if (local_flags & __GFP_WAIT) local_irq_enable(); kmem_flagcheck(cache, flags); - obj = kmem_getpages(cache, local_flags, numa_node_id()); + + /* + * Node not set up yet? Try one that the cache has been set up + * for. + */ + nid = numa_node_id(); + if (cache->nodelists[nid] == NULL) { + for_each_zone_zonelist(zone, z, zonelist, high_zoneidx) { + nid = zone_to_nid(zone); + if (cache->nodelists[nid]) { + obj = kmem_getpages(cache, local_flags, nid); + if (obj) + break; + } + } + } else + obj = kmem_getpages(cache, local_flags, nid); + if (local_flags & __GFP_WAIT) local_irq_disable(); if (obj) { -- cgit v1.2.3 From f8fdd8f3388d887ee001030e1a8537fe511f8841 Mon Sep 17 00:00:00 2001 From: Andi Kleen Date: Thu, 11 Feb 2010 21:54:02 +0100 Subject: SLAB: Separate node initialization into separate function No functional changes. [ penberg@cs.helsinki.fi: Document slab_node_prepare locking ] Acked-by: Christoph Lameter Acked-by: David Rientjes Signed-off-by: Andi Kleen Signed-off-by: Pekka Enberg --- mm/slab.c | 35 ++++++++++++++++++++++------------- 1 file changed, 22 insertions(+), 13 deletions(-) diff --git a/mm/slab.c b/mm/slab.c index 6a813cc5802f..ef05b7d48014 100644 --- a/mm/slab.c +++ b/mm/slab.c @@ -1170,19 +1170,10 @@ free_array_cache: } } -static int __cpuinit cpuup_prepare(long cpu) +/* The caller needs to hold cache_chain_mutex. */ +static int slab_node_prepare(int node) { struct kmem_cache *cachep; - struct kmem_list3 *l3 = NULL; - int node = cpu_to_node(cpu); - const int memsize = sizeof(struct kmem_list3); - - /* - * We need to do this right in the beginning since - * alloc_arraycache's are going to use this list. - * kmalloc_node allows us to add the slab to the right - * kmem_list3 and not this cpu's kmem_list3 - */ list_for_each_entry(cachep, &cache_chain, next) { /* @@ -1191,9 +1182,10 @@ static int __cpuinit cpuup_prepare(long cpu) * node has not already allocated this */ if (!cachep->nodelists[node]) { - l3 = kmalloc_node(memsize, GFP_KERNEL, node); + struct kmem_list3 *l3; + l3 = kmalloc_node(sizeof(struct kmem_list3), GFP_KERNEL, node); if (!l3) - goto bad; + return -1; kmem_list3_init(l3); l3->next_reap = jiffies + REAPTIMEOUT_LIST3 + ((unsigned long)cachep) % REAPTIMEOUT_LIST3; @@ -1212,6 +1204,23 @@ static int __cpuinit cpuup_prepare(long cpu) cachep->batchcount + cachep->num; spin_unlock_irq(&cachep->nodelists[node]->list_lock); } + return 0; +} + +static int __cpuinit cpuup_prepare(long cpu) +{ + struct kmem_cache *cachep; + struct kmem_list3 *l3 = NULL; + int node = cpu_to_node(cpu); + + /* + * We need to do this right in the beginning since + * alloc_arraycache's are going to use this list. + * kmalloc_node allows us to add the slab to the right + * kmem_list3 and not this cpu's kmem_list3 + */ + if (slab_node_prepare(node) < 0) + goto bad; /* * Now we can go ahead with allocating the shared arrays and -- cgit v1.2.3 From 960fe74e955d738428842fc92edc02e02cf36026 Mon Sep 17 00:00:00 2001 From: Andi Kleen Date: Thu, 11 Feb 2010 21:54:03 +0100 Subject: SLAB: Set up the l3 lists for the memory of freshly added memory So kmalloc_node() works even if no CPU is up yet on the new node. [ penberg@cs.helsinki.fi: Fix build for !MEM_HOTPLUG ] Acked-by: David Rientjes Signed-off-by: Andi Kleen Signed-off-by: Pekka Enberg --- mm/slab.c | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/mm/slab.c b/mm/slab.c index ef05b7d48014..0631dd4bfd09 100644 --- a/mm/slab.c +++ b/mm/slab.c @@ -115,6 +115,7 @@ #include #include #include +#include #include #include @@ -1569,6 +1570,25 @@ void __init kmem_cache_init(void) g_cpucache_up = EARLY; } +#ifdef CONFIG_MEMORY_HOTPLUG +static int slab_memory_callback(struct notifier_block *self, + unsigned long action, void *arg) +{ + struct memory_notify *mn = (struct memory_notify *)arg; + + /* + * When a node goes online allocate l3s early. This way + * kmalloc_node() works for it. + */ + if (action == MEM_ONLINE && mn->status_change_nid >= 0) { + mutex_lock(&cache_chain_mutex); + slab_node_prepare(mn->status_change_nid); + mutex_unlock(&cache_chain_mutex); + } + return NOTIFY_OK; +} +#endif + void __init kmem_cache_init_late(void) { struct kmem_cache *cachep; @@ -1592,6 +1612,8 @@ void __init kmem_cache_init_late(void) */ register_cpu_notifier(&cpucache_notifier); + hotplug_memory_notifier(slab_memory_callback, SLAB_CALLBACK_PRI); + /* * The reap timers are started later, with a module init call: That part * of the kernel is not yet operational. -- cgit v1.2.3 From c2882a79eecc1d325c0e5af3970acfb9c2b429c3 Mon Sep 17 00:00:00 2001 From: Andi Kleen Date: Thu, 11 Feb 2010 21:54:04 +0100 Subject: SLAB: Fix node add timer race in cache_reap cache_reap can run before the node is set up and then reference a NULL l3 list. Check for this explicitely and just continue. The node will be eventually set up. Acked-by: Christoph Lameter Acked-by: David Rientjes Signed-off-by: Andi Kleen Signed-off-by: Pekka Enberg --- mm/slab.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/mm/slab.c b/mm/slab.c index 0631dd4bfd09..365162bf5bae 100644 --- a/mm/slab.c +++ b/mm/slab.c @@ -4118,6 +4118,9 @@ static void cache_reap(struct work_struct *w) * we can do some work if the lock was obtained. */ l3 = searchp->nodelists[node]; + /* Note node yet set up */ + if (!l3) + break; reap_alien(searchp, l3); -- cgit v1.2.3 From 81fc03909a80bead8f553287a2b749a1d29dca64 Mon Sep 17 00:00:00 2001 From: Catalin Marinas Date: Mon, 8 Feb 2010 11:16:24 +0000 Subject: kmemcheck: Test the full object in kmemcheck_is_obj_initialized() This is a fix for bug #14845 (bugzilla.kernel.org). The update_checksum() function in mm/kmemleak.c calls kmemcheck_is_obj_initialised() before scanning an object. When KMEMCHECK_PARTIAL_OK is enabled, this function returns true. However, the crc32_le() reads smaller intervals (32-bit) for which kmemleak_is_obj_initialised() may be false leading to a kmemcheck warning. Note that kmemcheck_is_obj_initialized() is currently only used by kmemleak before scanning a memory location. Signed-off-by: Catalin Marinas Cc: Andrew Morton Cc: Christian Casteyde Cc: Vegard Nossum Signed-off-by: Pekka Enberg --- arch/x86/mm/kmemcheck/kmemcheck.c | 2 +- arch/x86/mm/kmemcheck/shadow.c | 16 ++++++++++++++-- arch/x86/mm/kmemcheck/shadow.h | 2 ++ 3 files changed, 17 insertions(+), 3 deletions(-) diff --git a/arch/x86/mm/kmemcheck/kmemcheck.c b/arch/x86/mm/kmemcheck/kmemcheck.c index 8cc183344140..b3b531a4f8e5 100644 --- a/arch/x86/mm/kmemcheck/kmemcheck.c +++ b/arch/x86/mm/kmemcheck/kmemcheck.c @@ -337,7 +337,7 @@ bool kmemcheck_is_obj_initialized(unsigned long addr, size_t size) if (!shadow) return true; - status = kmemcheck_shadow_test(shadow, size); + status = kmemcheck_shadow_test_all(shadow, size); return status == KMEMCHECK_SHADOW_INITIALIZED; } diff --git a/arch/x86/mm/kmemcheck/shadow.c b/arch/x86/mm/kmemcheck/shadow.c index 3f66b82076a3..aec124214d97 100644 --- a/arch/x86/mm/kmemcheck/shadow.c +++ b/arch/x86/mm/kmemcheck/shadow.c @@ -125,12 +125,12 @@ void kmemcheck_mark_initialized_pages(struct page *p, unsigned int n) enum kmemcheck_shadow kmemcheck_shadow_test(void *shadow, unsigned int size) { +#ifdef CONFIG_KMEMCHECK_PARTIAL_OK uint8_t *x; unsigned int i; x = shadow; -#ifdef CONFIG_KMEMCHECK_PARTIAL_OK /* * Make sure _some_ bytes are initialized. Gcc frequently generates * code to access neighboring bytes. @@ -139,13 +139,25 @@ enum kmemcheck_shadow kmemcheck_shadow_test(void *shadow, unsigned int size) if (x[i] == KMEMCHECK_SHADOW_INITIALIZED) return x[i]; } + + return x[0]; #else + return kmemcheck_shadow_test_all(shadow, size); +#endif +} + +enum kmemcheck_shadow kmemcheck_shadow_test_all(void *shadow, unsigned int size) +{ + uint8_t *x; + unsigned int i; + + x = shadow; + /* All bytes must be initialized. */ for (i = 0; i < size; ++i) { if (x[i] != KMEMCHECK_SHADOW_INITIALIZED) return x[i]; } -#endif return x[0]; } diff --git a/arch/x86/mm/kmemcheck/shadow.h b/arch/x86/mm/kmemcheck/shadow.h index af46d9ab9d86..ff0b2f70fbcb 100644 --- a/arch/x86/mm/kmemcheck/shadow.h +++ b/arch/x86/mm/kmemcheck/shadow.h @@ -11,6 +11,8 @@ enum kmemcheck_shadow { void *kmemcheck_shadow_lookup(unsigned long address); enum kmemcheck_shadow kmemcheck_shadow_test(void *shadow, unsigned int size); +enum kmemcheck_shadow kmemcheck_shadow_test_all(void *shadow, + unsigned int size); void kmemcheck_shadow_set(void *shadow, unsigned int size); #endif -- cgit v1.2.3