#include <linux/cpu.h>
#include <linux/sysctl.h>
#include <linux/module.h>
+#include <linux/rcupdate.h>
#include <asm/uaccess.h>
#include <asm/cacheflush.h>
#include <asm/tlbflush.h>
+#include <asm/page.h>
/*
* DEBUG - 1 for kmem_cache_create() to honour; SLAB_DEBUG_INITIAL,
#endif
#ifndef ARCH_KMALLOC_MINALIGN
+/*
+ * Enforce a minimum alignment for the kmalloc caches.
+ * Usually, the kmalloc caches are cache_line_size() aligned, except when
+ * DEBUG and FORCED_DEBUG are enabled, then they are BYTES_PER_WORD aligned.
+ * Some archs want to perform DMA into kmalloc caches and need a guaranteed
+ * alignment larger than BYTES_PER_WORD. ARCH_KMALLOC_MINALIGN allows that.
+ * Note that this flag disables some debug features.
+ */
#define ARCH_KMALLOC_MINALIGN 0
#endif
+#ifndef ARCH_SLAB_MINALIGN
+/*
+ * Enforce a minimum alignment for all caches.
+ * Intended for archs that get misalignment faults even for BYTES_PER_WORD
+ * aligned buffers. Includes ARCH_KMALLOC_MINALIGN.
+ * If possible: Do not enable this flag for CONFIG_DEBUG_SLAB, it disables
+ * some debug features.
+ */
+#define ARCH_SLAB_MINALIGN 0
+#endif
+
+#ifndef ARCH_KMALLOC_FLAGS
+#define ARCH_KMALLOC_FLAGS SLAB_HWCACHE_ALIGN
+#endif
+
/* Legal flag mask for kmem_cache_create(). */
#if DEBUG
# define CREATE_MASK (SLAB_DEBUG_INITIAL | SLAB_RED_ZONE | \
SLAB_POISON | SLAB_HWCACHE_ALIGN | \
SLAB_NO_REAP | SLAB_CACHE_DMA | \
SLAB_MUST_HWCACHE_ALIGN | SLAB_STORE_USER | \
- SLAB_RECLAIM_ACCOUNT )
+ SLAB_RECLAIM_ACCOUNT | SLAB_PANIC | \
+ SLAB_DESTROY_BY_RCU)
#else
# define CREATE_MASK (SLAB_HWCACHE_ALIGN | SLAB_NO_REAP | \
SLAB_CACHE_DMA | SLAB_MUST_HWCACHE_ALIGN | \
- SLAB_RECLAIM_ACCOUNT)
+ SLAB_RECLAIM_ACCOUNT | SLAB_PANIC | \
+ SLAB_DESTROY_BY_RCU)
#endif
/*
* is less than 512 (PAGE_SIZE<<3), but greater than 256.
*/
-#define BUFCTL_END 0xffffFFFF
-#define BUFCTL_FREE 0xffffFFFE
-#define SLAB_LIMIT 0xffffFFFD
-typedef unsigned int kmem_bufctl_t;
+#define BUFCTL_END (((kmem_bufctl_t)(~0U))-0)
+#define BUFCTL_FREE (((kmem_bufctl_t)(~0U))-1)
+#define SLAB_LIMIT (((kmem_bufctl_t)(~0U))-2)
/* Max number of objs-per-slab for caches which use off-slab slabs.
* Needed to avoid a possible looping condition in cache_grow().
kmem_bufctl_t free;
};
+/*
+ * struct slab_rcu
+ *
+ * slab_destroy on a SLAB_DESTROY_BY_RCU cache uses this structure to
+ * arrange for kmem_freepages to be called via RCU. This is useful if
+ * we need to approach a kernel structure obliquely, from its address
+ * obtained without the usual locking. We can lock the structure to
+ * stabilize it and check it's still at the given address, only if we
+ * can be sure that the memory has not been meanwhile reused for some
+ * other kind of object (which our subsystem's lock might corrupt).
+ *
+ * rcu_read_lock before reading the address, then rcu_read_unlock after
+ * taking the spinlock within the structure expected at that address.
+ *
+ * We assume struct slab_rcu can overlay struct slab when destroying.
+ */
+struct slab_rcu {
+ struct rcu_head head;
+ kmem_cache_t *cachep;
+ void *addr;
+};
+
/*
* struct array_cache
*
unsigned long reaped;
unsigned long errors;
unsigned long max_freeable;
+ unsigned long node_allocs;
atomic_t allochit;
atomic_t allocmiss;
atomic_t freehit;
(x)->high_mark = (x)->num_active; \
} while (0)
#define STATS_INC_ERR(x) ((x)->errors++)
+#define STATS_INC_NODEALLOCS(x) ((x)->node_allocs++)
#define STATS_SET_FREEABLE(x, i) \
do { if ((x)->max_freeable < i) \
(x)->max_freeable = i; \
#define STATS_INC_REAPED(x) do { } while (0)
#define STATS_SET_HIGH(x) do { } while (0)
#define STATS_INC_ERR(x) do { } while (0)
+#define STATS_INC_NODEALLOCS(x) do { } while (0)
#define STATS_SET_FREEABLE(x, i) \
do { } while (0)
* cachep->objsize - 2* BYTES_PER_WORD: redzone word [BYTES_PER_WORD long]
* cachep->objsize - 1* BYTES_PER_WORD: last caller address [BYTES_PER_WORD long]
*/
-static inline int obj_dbghead(kmem_cache_t *cachep)
+static int obj_dbghead(kmem_cache_t *cachep)
{
return cachep->dbghead;
}
-static inline int obj_reallen(kmem_cache_t *cachep)
+static int obj_reallen(kmem_cache_t *cachep)
{
return cachep->reallen;
}
BUG_ON(!(cachep->flags & SLAB_STORE_USER));
return (void**)(objp+cachep->objsize-BYTES_PER_WORD);
}
+
#else
-static inline int obj_dbghead(kmem_cache_t *cachep)
-{
- return 0;
-}
-static inline int obj_reallen(kmem_cache_t *cachep)
-{
- return cachep->objsize;
-}
-static inline unsigned long *dbg_redzone1(kmem_cache_t *cachep, void *objp)
-{
- BUG();
- return 0;
-}
-static inline unsigned long *dbg_redzone2(kmem_cache_t *cachep, void *objp)
-{
- BUG();
- return 0;
-}
-static inline void **dbg_userword(kmem_cache_t *cachep, void *objp)
-{
- BUG();
- return 0;
-}
+
+#define obj_dbghead(x) 0
+#define obj_reallen(cachep) (cachep->objsize)
+#define dbg_redzone1(cachep, objp) ({BUG(); (unsigned long *)NULL;})
+#define dbg_redzone2(cachep, objp) ({BUG(); (unsigned long *)NULL;})
+#define dbg_userword(cachep, objp) ({BUG(); (void **)NULL;})
+
#endif
/*
EXPORT_SYMBOL(malloc_sizes);
/* Must match cache_sizes above. Out of line to keep cache footprint low. */
-static struct cache_names {
+struct cache_names {
char *name;
char *name_dma;
-} cache_names[] = {
+};
+
+static struct cache_names __initdata cache_names[] = {
#define CACHE(x) { .name = "size-" #x, .name_dma = "size-" #x "(DMA)" },
#include <linux/kmalloc_sizes.h>
- { 0, }
+ { NULL, }
#undef CACHE
};
-struct arraycache_init initarray_cache __initdata = { { 0, BOOT_CPUCACHE_ENTRIES, 1, 0} };
-struct arraycache_init initarray_generic __initdata = { { 0, BOOT_CPUCACHE_ENTRIES, 1, 0} };
+static struct arraycache_init initarray_cache __initdata =
+ { { 0, BOOT_CPUCACHE_ENTRIES, 1, 0} };
+static struct arraycache_init initarray_generic =
+ { { 0, BOOT_CPUCACHE_ENTRIES, 1, 0} };
/* internal cache of cache description objs */
static kmem_cache_t cache_cache = {
/* Guard access to the cache-chain. */
static struct semaphore cache_chain_sem;
-
-struct list_head cache_chain;
+static struct list_head cache_chain;
/*
* vm_enough_memory() looks at this to determine how many
* chicken and egg problem: delay the per-cpu array allocation
* until the general caches are up.
*/
-enum {
+static enum {
NONE,
PARTIAL,
FULL
} g_cpucache_up;
-static DEFINE_PER_CPU(struct timer_list, reap_timers);
+static DEFINE_PER_CPU(struct work_struct, reap_work);
-static void reap_timer_fnc(unsigned long data);
static void free_block(kmem_cache_t* cachep, void** objpp, int len);
static void enable_cpucache (kmem_cache_t *cachep);
+static void cache_reap (void *unused);
static inline void ** ac_entry(struct array_cache *ac)
{
return cachep->array[smp_processor_id()];
}
+static kmem_cache_t * kmem_find_general_cachep (size_t size, int gfpflags)
+{
+ struct cache_sizes *csizep = malloc_sizes;
+
+ /* This function could be moved to the header file, and
+ * made inline so consumers can quickly determine what
+ * cache pointer they require.
+ */
+ for ( ; csizep->cs_size; csizep++) {
+ if (size > csizep->cs_size)
+ continue;
+ break;
+ }
+ return (gfpflags & GFP_DMA) ? csizep->cs_dmacachep : csizep->cs_cachep;
+}
+
/* Cal the num objs, wastage, and bytes left over for a given slab size. */
static void cache_estimate (unsigned long gfporder, size_t size, size_t align,
int flags, size_t *left_over, unsigned int *num)
}
/*
- * Start the reap timer running on the target CPU. We run at around 1 to 2Hz.
- * Add the CPU number into the expiry time to minimize the possibility of the
- * CPUs getting into lockstep and contending for the global cache chain lock.
+ * Initiate the reap timer running on the target CPU. We run at around 1 to 2Hz
+ * via the workqueue/eventd.
+ * Add the CPU number into the expiration time to minimize the possibility of
+ * the CPUs getting into lockstep and contending for the global cache chain
+ * lock.
*/
static void __devinit start_cpu_timer(int cpu)
{
- struct timer_list *rt = &per_cpu(reap_timers, cpu);
+ struct work_struct *reap_work = &per_cpu(reap_work, cpu);
- if (rt->function == NULL) {
- init_timer(rt);
- rt->expires = jiffies + HZ + 3*cpu;
- rt->data = cpu;
- rt->function = reap_timer_fnc;
- add_timer_on(rt, cpu);
+ /*
+ * When this gets called from do_initcalls via cpucache_init(),
+ * init_workqueues() has already run, so keventd will be setup
+ * at that time.
+ */
+ if (keventd_up() && reap_work->func == NULL) {
+ INIT_WORK(reap_work, cache_reap, NULL);
+ schedule_delayed_work_on(cpu, reap_work, HZ + 3 * cpu);
}
}
-#ifdef CONFIG_HOTPLUG_CPU
-static void stop_cpu_timer(int cpu)
+static struct array_cache *alloc_arraycache(int cpu, int entries, int batchcount)
{
- struct timer_list *rt = &per_cpu(reap_timers, cpu);
+ int memsize = sizeof(void*)*entries+sizeof(struct array_cache);
+ struct array_cache *nc = NULL;
- if (rt->function) {
- del_timer_sync(rt);
- WARN_ON(timer_pending(rt));
- rt->function = NULL;
+ if (cpu != -1) {
+ nc = kmem_cache_alloc_node(kmem_find_general_cachep(memsize,
+ GFP_KERNEL), cpu_to_node(cpu));
}
+ if (!nc)
+ nc = kmalloc(memsize, GFP_KERNEL);
+ if (nc) {
+ nc->avail = 0;
+ nc->limit = entries;
+ nc->batchcount = batchcount;
+ nc->touched = 0;
+ }
+ return nc;
}
-#endif
static int __devinit cpuup_callback(struct notifier_block *nfb,
unsigned long action,
case CPU_UP_PREPARE:
down(&cache_chain_sem);
list_for_each_entry(cachep, &cache_chain, next) {
- int memsize;
struct array_cache *nc;
- memsize = sizeof(void*)*cachep->limit+sizeof(struct array_cache);
- nc = kmalloc(memsize, GFP_KERNEL);
+ nc = alloc_arraycache(cpu, cachep->limit, cachep->batchcount);
if (!nc)
goto bad;
- nc->avail = 0;
- nc->limit = cachep->limit;
- nc->batchcount = cachep->batchcount;
- nc->touched = 0;
spin_lock_irq(&cachep->spinlock);
cachep->array[cpu] = nc;
break;
#ifdef CONFIG_HOTPLUG_CPU
case CPU_DEAD:
- stop_cpu_timer(cpu);
/* fall thru */
case CPU_UP_CANCELED:
down(&cache_chain_sem);
* eliminates "false sharing".
* Note for systems short on memory removing the alignment will
* allow tighter packing of the smaller caches. */
- sizes->cs_cachep = kmem_cache_create(
- names->name, sizes->cs_size,
- ARCH_KMALLOC_MINALIGN, 0, NULL, NULL);
- if (!sizes->cs_cachep)
- BUG();
+ sizes->cs_cachep = kmem_cache_create(names->name,
+ sizes->cs_size, ARCH_KMALLOC_MINALIGN,
+ (ARCH_KMALLOC_FLAGS | SLAB_PANIC), NULL, NULL);
/* Inc off-slab bufctl limit until the ceiling is hit. */
if (!(OFF_SLAB(sizes->cs_cachep))) {
offslab_limit /= sizeof(kmem_bufctl_t);
}
- sizes->cs_dmacachep = kmem_cache_create(
- names->name_dma, sizes->cs_size,
- ARCH_KMALLOC_MINALIGN, SLAB_CACHE_DMA, NULL, NULL);
- if (!sizes->cs_dmacachep)
- BUG();
+ sizes->cs_dmacachep = kmem_cache_create(names->name_dma,
+ sizes->cs_size, ARCH_KMALLOC_MINALIGN,
+ (ARCH_KMALLOC_FLAGS | SLAB_CACHE_DMA | SLAB_PANIC),
+ NULL, NULL);
sizes++;
names++;
*/
}
-int __init cpucache_init(void)
+static int __init cpucache_init(void)
{
int cpu;
* did not request dmaable memory, we might get it, but that
* would be relatively rare and ignorable.
*/
-static inline void *kmem_getpages(kmem_cache_t *cachep, unsigned long flags)
+static void *kmem_getpages(kmem_cache_t *cachep, int flags, int nodeid)
{
+ struct page *page;
void *addr;
+ int i;
flags |= cachep->gfpflags;
- addr = (void*)__get_free_pages(flags, cachep->gfporder);
- if (addr) {
- int i = (1 << cachep->gfporder);
- struct page *page = virt_to_page(addr);
-
- if (cachep->flags & SLAB_RECLAIM_ACCOUNT)
- atomic_add(i, &slab_reclaim_pages);
- add_page_state(nr_slab, i);
- while (i--) {
- SetPageSlab(page);
- page++;
- }
+ if (likely(nodeid == -1)) {
+ page = alloc_pages(flags, cachep->gfporder);
+ } else {
+ page = alloc_pages_node(nodeid, flags, cachep->gfporder);
+ }
+ if (!page)
+ return NULL;
+ addr = page_address(page);
+
+ i = (1 << cachep->gfporder);
+ if (cachep->flags & SLAB_RECLAIM_ACCOUNT)
+ atomic_add(i, &slab_reclaim_pages);
+ add_page_state(nr_slab, i);
+ while (i--) {
+ SetPageSlab(page);
+ page++;
}
return addr;
}
/*
* Interface to system's page release.
*/
-static inline void kmem_freepages(kmem_cache_t *cachep, void *addr)
+static void kmem_freepages(kmem_cache_t *cachep, void *addr)
{
unsigned long i = (1<<cachep->gfporder);
struct page *page = virt_to_page(addr);
atomic_sub(1<<cachep->gfporder, &slab_reclaim_pages);
}
+static void kmem_rcu_free(struct rcu_head *head)
+{
+ struct slab_rcu *slab_rcu = (struct slab_rcu *) head;
+ kmem_cache_t *cachep = slab_rcu->cachep;
+
+ kmem_freepages(cachep, slab_rcu->addr);
+ if (OFF_SLAB(cachep))
+ kmem_cache_free(cachep->slabp_cache, slab_rcu);
+}
+
#if DEBUG
#ifdef CONFIG_DEBUG_PAGEALLOC
}
#endif
+#if DEBUG
+
static void print_objinfo(kmem_cache_t *cachep, void *objp, int lines)
{
-#if DEBUG
int i, size;
char *realobj;
}
if (cachep->flags & SLAB_STORE_USER) {
- printk(KERN_ERR "Last user: [<%p>]", *dbg_userword(cachep, objp));
- print_symbol("(%s)", (unsigned long)*dbg_userword(cachep, objp));
+ printk(KERN_ERR "Last user: [<%p>]",
+ *dbg_userword(cachep, objp));
+ print_symbol("(%s)",
+ (unsigned long)*dbg_userword(cachep, objp));
printk("\n");
}
realobj = (char*)objp+obj_dbghead(cachep);
limit = size-i;
dump_line(realobj, i, limit);
}
-#endif
}
-#if DEBUG
-
static void check_poison_obj(kmem_cache_t *cachep, void *objp)
{
char *realobj;
*/
static void slab_destroy (kmem_cache_t *cachep, struct slab *slabp)
{
+ void *addr = slabp->s_mem - slabp->colouroff;
+
#if DEBUG
int i;
for (i = 0; i < cachep->num; i++) {
}
}
#endif
-
- kmem_freepages(cachep, slabp->s_mem-slabp->colouroff);
- if (OFF_SLAB(cachep))
- kmem_cache_free(cachep->slabp_cache, slabp);
+
+ if (unlikely(cachep->flags & SLAB_DESTROY_BY_RCU)) {
+ struct slab_rcu *slab_rcu;
+
+ slab_rcu = (struct slab_rcu *) slabp;
+ slab_rcu->cachep = cachep;
+ slab_rcu->addr = addr;
+ call_rcu(&slab_rcu->head, kmem_rcu_free);
+ } else {
+ kmem_freepages(cachep, addr);
+ if (OFF_SLAB(cachep))
+ kmem_cache_free(cachep->slabp_cache, slabp);
+ }
}
/**
* %SLAB_NO_REAP - Don't automatically reap this cache when we're under
* memory pressure.
*
- * %SLAB_HWCACHE_ALIGN - This flag has no effect and will be removed soon.
- *
+ * %SLAB_HWCACHE_ALIGN - Align the objects in this cache to a hardware
+ * cacheline. This can be beneficial if you're counting cycles as closely
+ * as davem.
*/
kmem_cache_t *
kmem_cache_create (const char *name, size_t size, size_t align,
unsigned long flags, void (*ctor)(void*, kmem_cache_t *, unsigned long),
void (*dtor)(void*, kmem_cache_t *, unsigned long))
{
- size_t left_over, slab_size;
+ size_t left_over, slab_size, ralign;
kmem_cache_t *cachep = NULL;
/*
in_interrupt() ||
(size < BYTES_PER_WORD) ||
(size > (1<<MAX_OBJ_ORDER)*PAGE_SIZE) ||
- (dtor && !ctor) ||
- (align < 0)) {
+ (dtor && !ctor)) {
printk(KERN_ERR "%s: Early error in slab %s\n",
__FUNCTION__, name);
BUG();
*/
if ((size < 4096 || fls(size-1) == fls(size-1+3*BYTES_PER_WORD)))
flags |= SLAB_RED_ZONE|SLAB_STORE_USER;
- flags |= SLAB_POISON;
+ if (!(flags & SLAB_DESTROY_BY_RCU))
+ flags |= SLAB_POISON;
#endif
+ if (flags & SLAB_DESTROY_BY_RCU)
+ BUG_ON(flags & SLAB_POISON);
#endif
+ if (flags & SLAB_DESTROY_BY_RCU)
+ BUG_ON(dtor);
+
/*
* Always checks flags, a caller might be expecting debug
* support which isn't available.
if (flags & ~CREATE_MASK)
BUG();
- if (align) {
- /* combinations of forced alignment and advanced debugging is
- * not yet implemented.
+ /* Check that size is in terms of words. This is needed to avoid
+ * unaligned accesses for some archs when redzoning is used, and makes
+ * sure any on-slab bufctl's are also correctly aligned.
+ */
+ if (size & (BYTES_PER_WORD-1)) {
+ size += (BYTES_PER_WORD-1);
+ size &= ~(BYTES_PER_WORD-1);
+ }
+
+ /* calculate out the final buffer alignment: */
+ /* 1) arch recommendation: can be overridden for debug */
+ if (flags & SLAB_HWCACHE_ALIGN) {
+ /* Default alignment: as specified by the arch code.
+ * Except if an object is really small, then squeeze multiple
+ * objects into one cacheline.
*/
- flags &= ~(SLAB_RED_ZONE|SLAB_STORE_USER);
+ ralign = cache_line_size();
+ while (size <= ralign/2)
+ ralign /= 2;
} else {
- if (flags & SLAB_HWCACHE_ALIGN) {
- /* Default alignment: as specified by the arch code.
- * Except if an object is really small, then squeeze multiple
- * into one cacheline.
- */
- align = cache_line_size();
- while (size <= align/2)
- align /= 2;
- } else {
- align = BYTES_PER_WORD;
- }
- }
+ ralign = BYTES_PER_WORD;
+ }
+ /* 2) arch mandated alignment: disables debug if necessary */
+ if (ralign < ARCH_SLAB_MINALIGN) {
+ ralign = ARCH_SLAB_MINALIGN;
+ if (ralign > BYTES_PER_WORD)
+ flags &= ~(SLAB_RED_ZONE|SLAB_STORE_USER);
+ }
+ /* 3) caller mandated alignment: disables debug if necessary */
+ if (ralign < align) {
+ ralign = align;
+ if (ralign > BYTES_PER_WORD)
+ flags &= ~(SLAB_RED_ZONE|SLAB_STORE_USER);
+ }
+ /* 4) Store it. Note that the debug code below can reduce
+ * the alignment to BYTES_PER_WORD.
+ */
+ align = ralign;
/* Get cache's description obj. */
cachep = (kmem_cache_t *) kmem_cache_alloc(&cache_cache, SLAB_KERNEL);
goto opps;
memset(cachep, 0, sizeof(kmem_cache_t));
- /* Check that size is in terms of words. This is needed to avoid
- * unaligned accesses for some archs when redzoning is used, and makes
- * sure any on-slab bufctl's are also correctly aligned.
- */
- if (size & (BYTES_PER_WORD-1)) {
- size += (BYTES_PER_WORD-1);
- size &= ~(BYTES_PER_WORD-1);
- }
-
#if DEBUG
cachep->reallen = size;
up(&cache_chain_sem);
unlock_cpu_hotplug();
opps:
+ if (!cachep && (flags & SLAB_PANIC))
+ panic("kmem_cache_create(): failed to create slab `%s'\n",
+ name);
return cachep;
}
-
EXPORT_SYMBOL(kmem_cache_create);
-static inline void check_irq_off(void)
-{
#if DEBUG
+static void check_irq_off(void)
+{
BUG_ON(!irqs_disabled());
-#endif
}
-static inline void check_irq_on(void)
+static void check_irq_on(void)
{
-#if DEBUG
BUG_ON(irqs_disabled());
-#endif
}
-static inline void check_spinlock_acquired(kmem_cache_t *cachep)
+static void check_spinlock_acquired(kmem_cache_t *cachep)
{
#ifdef CONFIG_SMP
check_irq_off();
BUG_ON(spin_trylock(&cachep->spinlock));
#endif
}
+#else
+#define check_irq_off() do { } while(0)
+#define check_irq_on() do { } while(0)
+#define check_spinlock_acquired(x) do { } while(0)
+#endif
/*
* Waits for all CPUs to execute func().
return 1;
}
+ if (unlikely(cachep->flags & SLAB_DESTROY_BY_RCU))
+ synchronize_kernel();
+
/* no cpu_online check required here since we clear the percpu
* array on cpu offline and set this to NULL.
*/
EXPORT_SYMBOL(kmem_cache_destroy);
/* Get the memory for a slab management obj. */
-static inline struct slab* alloc_slabmgmt (kmem_cache_t *cachep,
+static struct slab* alloc_slabmgmt (kmem_cache_t *cachep,
void *objp, int colour_off, int local_flags)
{
struct slab *slabp;
}
}
+static void set_slab_attr(kmem_cache_t *cachep, struct slab *slabp, void *objp)
+{
+ int i;
+ struct page *page;
+
+ /* Nasty!!!!!! I hope this is OK. */
+ i = 1 << cachep->gfporder;
+ page = virt_to_page(objp);
+ do {
+ SET_PAGE_CACHE(page, cachep);
+ SET_PAGE_SLAB(page, slabp);
+ page++;
+ } while (--i);
+}
+
/*
* Grow (by 1) the number of slabs within a cache. This is called by
* kmem_cache_alloc() when there are no active objs left in a cache.
*/
-static int cache_grow (kmem_cache_t * cachep, int flags)
+static int cache_grow (kmem_cache_t * cachep, int flags, int nodeid)
{
struct slab *slabp;
- struct page *page;
void *objp;
size_t offset;
- unsigned int i, local_flags;
+ int local_flags;
unsigned long ctor_flags;
/* Be lazy and only check for valid flags here,
/* Get mem for the objs. */
- if (!(objp = kmem_getpages(cachep, flags)))
+ if (!(objp = kmem_getpages(cachep, flags, nodeid)))
goto failed;
/* Get slab management. */
if (!(slabp = alloc_slabmgmt(cachep, objp, offset, local_flags)))
goto opps1;
- /* Nasty!!!!!! I hope this is OK. */
- i = 1 << cachep->gfporder;
- page = virt_to_page(objp);
- do {
- SET_PAGE_CACHE(page, cachep);
- SET_PAGE_SLAB(page, slabp);
- page++;
- } while (--i);
+ set_slab_attr(cachep, slabp, objp);
cache_init_objs(cachep, slabp, ctor_flags);
return 0;
}
+#if DEBUG
+
/*
* Perform extra freeing checks:
* - detect bad pointers.
* - POISON/RED_ZONE checking
* - destructor calls, for caches with POISON+dtor
*/
-static inline void kfree_debugcheck(const void *objp)
+static void kfree_debugcheck(const void *objp)
{
-#if DEBUG
struct page *page;
if (!virt_addr_valid(objp)) {
printk(KERN_ERR "kfree_debugcheck: bad ptr %lxh.\n", (unsigned long)objp);
BUG();
}
-#endif
}
-static inline void *cache_free_debugcheck (kmem_cache_t * cachep, void * objp, void *caller)
+static void *cache_free_debugcheck (kmem_cache_t * cachep, void * objp, void *caller)
{
-#if DEBUG
struct page *page;
unsigned int objnr;
struct slab *slabp;
poison_obj(cachep, objp, POISON_FREE);
#endif
}
-#endif
return objp;
}
-static inline void check_slabp(kmem_cache_t *cachep, struct slab *slabp)
+static void check_slabp(kmem_cache_t *cachep, struct slab *slabp)
{
-#if DEBUG
int i;
int entries = 0;
printk("\n");
BUG();
}
-#endif
}
+#else
+#define kfree_debugcheck(x) do { } while(0)
+#define cache_free_debugcheck(x,objp,z) (objp)
+#define check_slabp(x,y) do { } while(0)
+#endif
static void* cache_alloc_refill(kmem_cache_t* cachep, int flags)
{
if (unlikely(!ac->avail)) {
int x;
- x = cache_grow(cachep, flags);
+ x = cache_grow(cachep, flags, -1);
// cache_grow can reenable interrupts, then ac could change.
ac = ac_data(cachep);
#endif
}
-static inline void *
+#if DEBUG
+static void *
cache_alloc_debugcheck_after(kmem_cache_t *cachep,
unsigned long flags, void *objp, void *caller)
{
-#if DEBUG
if (!objp)
return objp;
if (cachep->flags & SLAB_POISON) {
cachep->ctor(objp, cachep, ctor_flags);
}
-#endif
return objp;
}
+#else
+#define cache_alloc_debugcheck_after(a,b,objp,d) (objp)
+#endif
static inline void * __cache_alloc (kmem_cache_t *cachep, int flags)
return 0;
}
+#ifdef CONFIG_NUMA
+/**
+ * kmem_cache_alloc_node - Allocate an object on the specified node
+ * @cachep: The cache to allocate from.
+ * @flags: See kmalloc().
+ * @nodeid: node number of the target node.
+ *
+ * Identical to kmem_cache_alloc, except that this function is slow
+ * and can sleep. And it will allocate memory on the given node, which
+ * can improve the performance for cpu bound structures.
+ */
+void *kmem_cache_alloc_node(kmem_cache_t *cachep, int nodeid)
+{
+ int loop;
+ void *objp;
+ struct slab *slabp;
+ kmem_bufctl_t next;
+
+ for (loop = 0;;loop++) {
+ struct list_head *q;
+
+ objp = NULL;
+ check_irq_on();
+ spin_lock_irq(&cachep->spinlock);
+ /* walk through all partial and empty slab and find one
+ * from the right node */
+ list_for_each(q,&cachep->lists.slabs_partial) {
+ slabp = list_entry(q, struct slab, list);
+
+ if (page_to_nid(virt_to_page(slabp->s_mem)) == nodeid ||
+ loop > 2)
+ goto got_slabp;
+ }
+ list_for_each(q, &cachep->lists.slabs_free) {
+ slabp = list_entry(q, struct slab, list);
+
+ if (page_to_nid(virt_to_page(slabp->s_mem)) == nodeid ||
+ loop > 2)
+ goto got_slabp;
+ }
+ spin_unlock_irq(&cachep->spinlock);
+
+ local_irq_disable();
+ if (!cache_grow(cachep, GFP_KERNEL, nodeid)) {
+ local_irq_enable();
+ return NULL;
+ }
+ local_irq_enable();
+ }
+got_slabp:
+ /* found one: allocate object */
+ check_slabp(cachep, slabp);
+ check_spinlock_acquired(cachep);
+
+ STATS_INC_ALLOCED(cachep);
+ STATS_INC_ACTIVE(cachep);
+ STATS_SET_HIGH(cachep);
+ STATS_INC_NODEALLOCS(cachep);
+
+ objp = slabp->s_mem + slabp->free*cachep->objsize;
+
+ slabp->inuse++;
+ next = slab_bufctl(slabp)[slabp->free];
+#if DEBUG
+ slab_bufctl(slabp)[slabp->free] = BUFCTL_FREE;
+#endif
+ slabp->free = next;
+ check_slabp(cachep, slabp);
+
+ /* move slabp to correct slabp list: */
+ list_del(&slabp->list);
+ if (slabp->free == BUFCTL_END)
+ list_add(&slabp->list, &cachep->lists.slabs_full);
+ else
+ list_add(&slabp->list, &cachep->lists.slabs_partial);
+
+ list3_data(cachep)->free_objects--;
+ spin_unlock_irq(&cachep->spinlock);
+
+ objp = cache_alloc_debugcheck_after(cachep, GFP_KERNEL, objp,
+ __builtin_return_address(0));
+ return objp;
+}
+EXPORT_SYMBOL(kmem_cache_alloc_node);
+
+#endif
+
/**
* kmalloc - allocate memory
* @size: how many bytes of memory are required.
/**
* __alloc_percpu - allocate one copy of the object for every present
* cpu in the system, zeroing them.
- * Objects should be dereferenced using per_cpu_ptr/get_cpu_ptr
- * macros only.
+ * Objects should be dereferenced using the per_cpu_ptr macro only.
*
* @size: how many bytes of memory are required.
* @align: the alignment, which can't be greater than SMP_CACHE_BYTES.
for (i = 0; i < NR_CPUS; i++) {
if (!cpu_possible(i))
continue;
- pdata->ptrs[i] = kmalloc(size, GFP_KERNEL);
+ pdata->ptrs[i] = kmem_cache_alloc_node(
+ kmem_find_general_cachep(size, GFP_KERNEL),
+ cpu_to_node(i));
+
if (!pdata->ptrs[i])
goto unwind_oom;
memset(pdata->ptrs[i], 0, size);
EXPORT_SYMBOL(kmem_cache_free);
+/**
+ * kcalloc - allocate memory for an array. The memory is set to zero.
+ * @n: number of elements.
+ * @size: element size.
+ * @flags: the type of memory to allocate.
+ */
+void *kcalloc(size_t n, size_t size, int flags)
+{
+ void *ret = NULL;
+
+ if (n != 0 && size > INT_MAX / n)
+ return ret;
+
+ ret = kmalloc(n * size, flags);
+ if (ret)
+ memset(ret, 0, n * size);
+ return ret;
+}
+
+EXPORT_SYMBOL(kcalloc);
+
/**
* kfree - free previously allocated memory
* @objp: pointer returned by kmalloc.
continue;
kfree(p->ptrs[i]);
}
+ kfree(p);
}
EXPORT_SYMBOL(free_percpu);
EXPORT_SYMBOL(kmem_cache_size);
-kmem_cache_t * kmem_find_general_cachep (size_t size, int gfpflags)
-{
- struct cache_sizes *csizep = malloc_sizes;
-
- /* This function could be moved to the header file, and
- * made inline so consumers can quickly determine what
- * cache pointer they require.
- */
- for ( ; csizep->cs_size; csizep++) {
- if (size > csizep->cs_size)
- continue;
- break;
- }
- return (gfpflags & GFP_DMA) ? csizep->cs_dmacachep : csizep->cs_cachep;
-}
-
-EXPORT_SYMBOL(kmem_find_general_cachep);
-
struct ccupdate_struct {
kmem_cache_t *cachep;
struct array_cache *new[NR_CPUS];
memset(&new.new,0,sizeof(new.new));
for (i = 0; i < NR_CPUS; i++) {
- struct array_cache *ccnew;
-
- ccnew = kmalloc(sizeof(void*)*limit+
- sizeof(struct array_cache), GFP_KERNEL);
- if (!ccnew) {
- for (i--; i >= 0; i--) kfree(new.new[i]);
- return -ENOMEM;
+ if (cpu_online(i)) {
+ new.new[i] = alloc_arraycache(i, limit, batchcount);
+ if (!new.new[i]) {
+ for (i--; i >= 0; i--) kfree(new.new[i]);
+ return -ENOMEM;
+ }
+ } else {
+ new.new[i] = NULL;
}
- ccnew->avail = 0;
- ccnew->limit = limit;
- ccnew->batchcount = batchcount;
- ccnew->touched = 0;
- new.new[i] = ccnew;
}
new.cachep = cachep;
spin_unlock_irq(&cachep->spinlock);
kfree(ccold);
}
- new_shared = kmalloc(sizeof(void*)*batchcount*shared+
- sizeof(struct array_cache), GFP_KERNEL);
+ new_shared = alloc_arraycache(-1, batchcount*shared, 0xbaadf00d);
if (new_shared) {
struct array_cache *old;
- new_shared->avail = 0;
- new_shared->limit = batchcount*shared;
- new_shared->batchcount = 0xbaadf00d;
- new_shared->touched = 0;
spin_lock_irq(&cachep->spinlock);
old = cachep->lists.shared;
cachep->name, -err);
}
-static void drain_array(kmem_cache_t *cachep, struct array_cache *ac)
-{
- int tofree;
-
- check_irq_off();
- if (ac->touched) {
- ac->touched = 0;
- } else if (ac->avail) {
- tofree = (ac->limit+4)/5;
- if (tofree > ac->avail) {
- tofree = (ac->avail+1)/2;
- }
- spin_lock(&cachep->spinlock);
- free_block(cachep, ac_entry(ac), tofree);
- spin_unlock(&cachep->spinlock);
- ac->avail -= tofree;
- memmove(&ac_entry(ac)[0], &ac_entry(ac)[tofree],
- sizeof(void*)*ac->avail);
- }
-}
-
static void drain_array_locked(kmem_cache_t *cachep,
struct array_cache *ac, int force)
{
/**
* cache_reap - Reclaim memory from caches.
*
- * Called from a timer, every few seconds
+ * Called from workqueue/eventd every few seconds.
* Purpose:
* - clear the per-cpu caches for this CPU.
* - return freeable pages to the main free memory pool.
*
* If we cannot acquire the cache chain semaphore then just give up - we'll
- * try again next timer interrupt.
+ * try again on the next iteration.
*/
-static inline void cache_reap (void)
+static void cache_reap(void *unused)
{
struct list_head *walk;
-#if DEBUG
- BUG_ON(!in_interrupt());
- BUG_ON(in_irq());
-#endif
- if (down_trylock(&cache_chain_sem))
+ if (down_trylock(&cache_chain_sem)) {
+ /* Give up. Setup the next iteration. */
+ schedule_delayed_work(&__get_cpu_var(reap_work), REAPTIMEOUT_CPUC + smp_processor_id());
return;
+ }
list_for_each(walk, &cache_chain) {
kmem_cache_t *searchp;
goto next;
check_irq_on();
- local_irq_disable();
- drain_array(searchp, ac_data(searchp));
- if(time_after(searchp->lists.next_reap, jiffies))
- goto next_irqon;
+ spin_lock_irq(&searchp->spinlock);
- spin_lock(&searchp->spinlock);
- if(time_after(searchp->lists.next_reap, jiffies)) {
+ drain_array_locked(searchp, ac_data(searchp), 0);
+
+ if(time_after(searchp->lists.next_reap, jiffies))
goto next_unlock;
- }
+
searchp->lists.next_reap = jiffies + REAPTIMEOUT_LIST3;
if (searchp->lists.shared)
spin_lock_irq(&searchp->spinlock);
} while(--tofree > 0);
next_unlock:
- spin_unlock(&searchp->spinlock);
-next_irqon:
- local_irq_enable();
+ spin_unlock_irq(&searchp->spinlock);
next:
- ;
+ cond_resched();
}
check_irq_on();
up(&cache_chain_sem);
-}
-
-/*
- * This is a timer handler. There is one per CPU. It is called periodially
- * to shrink this CPU's caches. Otherwise there could be memory tied up
- * for long periods (or for ever) due to load changes.
- */
-static void reap_timer_fnc(unsigned long cpu)
-{
- struct timer_list *rt = &__get_cpu_var(reap_timers);
-
- /* CPU hotplug can drag us off cpu: don't run on wrong CPU */
- if (!cpu_is_offline(cpu)) {
- cache_reap();
- mod_timer(rt, jiffies + REAPTIMEOUT_CPUC + cpu);
- }
+ /* Setup the next iteration */
+ schedule_delayed_work(&__get_cpu_var(reap_work), REAPTIMEOUT_CPUC + smp_processor_id());
}
#ifdef CONFIG_PROC_FS
* without _too_ many complaints.
*/
#if STATS
- seq_puts(m, "slabinfo - version: 2.0 (statistics)\n");
+ seq_puts(m, "slabinfo - version: 2.1 (statistics)\n");
#else
- seq_puts(m, "slabinfo - version: 2.0\n");
+ seq_puts(m, "slabinfo - version: 2.1\n");
#endif
seq_puts(m, "# name <active_objs> <num_objs> <objsize> <objperslab> <pagesperslab>");
- seq_puts(m, " : tunables <batchcount> <limit> <sharedfactor>");
+ seq_puts(m, " : tunables <limit> <batchcount> <sharedfactor>");
seq_puts(m, " : slabdata <active_slabs> <num_slabs> <sharedavail>");
#if STATS
- seq_puts(m, " : globalstat <listallocs> <maxobjs> <grown> <reaped> <error> <maxfreeable> <freelimit>");
+ seq_puts(m, " : globalstat <listallocs> <maxobjs> <grown> <reaped>"
+ " <error> <maxfreeable> <freelimit> <nodeallocs>");
seq_puts(m, " : cpustat <allochit> <allocmiss> <freehit> <freemiss>");
#endif
seq_putc(m, '\n');
unsigned long num_slabs;
const char *name;
char *error = NULL;
- mm_segment_t old_fs;
- char tmp;
check_irq_on();
spin_lock_irq(&cachep->spinlock);
error = "free_objects accounting error";
name = cachep->name;
-
- /*
- * Check to see if `name' resides inside a module which has been
- * unloaded (someone forgot to destroy their cache)
- */
- old_fs = get_fs();
- set_fs(KERNEL_DS);
- if (__get_user(tmp, name))
- name = "broken";
- set_fs(old_fs);
-
if (error)
printk(KERN_ERR "slab: cache %s error: %s\n", name, error);
unsigned long errors = cachep->errors;
unsigned long max_freeable = cachep->max_freeable;
unsigned long free_limit = cachep->free_limit;
+ unsigned long node_allocs = cachep->node_allocs;
- seq_printf(m, " : globalstat %7lu %6lu %5lu %4lu %4lu %4lu %4lu",
+ seq_printf(m, " : globalstat %7lu %6lu %5lu %4lu %4lu %4lu %4lu %4lu",
allocs, high, grown, reaped, errors,
- max_freeable, free_limit);
+ max_freeable, free_limit, node_allocs);
}
/* cpu stats */
{
return size;
}
-
-void ptrinfo(unsigned long addr)
-{
- struct page *page;
-
- printk("Dumping data about address %p.\n", (void*)addr);
- if (!virt_addr_valid((void*)addr)) {
- printk("virt addr invalid.\n");
- return;
- }
-#ifdef CONFIG_MMU
- do {
- pgd_t *pgd = pgd_offset_k(addr);
- pmd_t *pmd;
- if (pgd_none(*pgd)) {
- printk("No pgd.\n");
- break;
- }
- pmd = pmd_offset(pgd, addr);
- if (pmd_none(*pmd)) {
- printk("No pmd.\n");
- break;
- }
-#ifdef CONFIG_X86
- if (pmd_large(*pmd)) {
- printk("Large page.\n");
- break;
- }
-#endif
- printk("normal page, pte_val 0x%llx\n",
- (unsigned long long)pte_val(*pte_offset_kernel(pmd, addr)));
- } while(0);
-#endif
-
- page = virt_to_page((void*)addr);
- printk("struct page at %p, flags %08lx\n",
- page, (unsigned long)page->flags);
- if (PageSlab(page)) {
- kmem_cache_t *c;
- struct slab *s;
- unsigned long flags;
- int objnr;
- void *objp;
-
- c = GET_PAGE_CACHE(page);
- printk("belongs to cache %s.\n",c->name);
-
- spin_lock_irqsave(&c->spinlock, flags);
- s = GET_PAGE_SLAB(page);
- printk("slabp %p with %d inuse objects (from %d).\n",
- s, s->inuse, c->num);
- check_slabp(c,s);
-
- objnr = (addr-(unsigned long)s->s_mem)/c->objsize;
- objp = s->s_mem+c->objsize*objnr;
- printk("points into object no %d, starting at %p, len %d.\n",
- objnr, objp, c->objsize);
- if (objnr >= c->num) {
- printk("Bad obj number.\n");
- } else {
- kernel_map_pages(virt_to_page(objp),
- c->objsize/PAGE_SIZE, 1);
-
- print_objinfo(c, objp, 2);
- }
- spin_unlock_irqrestore(&c->spinlock, flags);
-
- }
-}
git://git.onelab.eu / linux-2.6.git / blobdiff