*/
#define BUFCTL_END (((kmem_bufctl_t)(~0U))-0)
-#define BUFCTL_ALLOC (((kmem_bufctl_t)(~0U))-1)
+#define BUFCTL_FREE (((kmem_bufctl_t)(~0U))-1)
#define SLAB_LIMIT (((kmem_bufctl_t)(~0U))-2)
/* Max number of objs-per-slab for caches which use off-slab slabs.
unsigned long reaped;
unsigned long errors;
unsigned long max_freeable;
- unsigned long node_allocs;
atomic_t allochit;
atomic_t allocmiss;
atomic_t freehit;
#if DEBUG
int dbghead;
int reallen;
- unsigned long redzonetest;
#endif
};
*/
#define REAPTIMEOUT_CPUC (2*HZ)
#define REAPTIMEOUT_LIST3 (4*HZ)
-#define REDZONETIMEOUT (300*HZ)
#if STATS
#define STATS_INC_ACTIVE(x) ((x)->num_active++)
(x)->high_mark = (x)->num_active; \
} while (0)
#define STATS_INC_ERR(x) ((x)->errors++)
-#define STATS_INC_NODEALLOCS(x) ((x)->node_allocs++)
#define STATS_SET_FREEABLE(x, i) \
do { if ((x)->max_freeable < i) \
(x)->max_freeable = i; \
#define STATS_INC_REAPED(x) do { } while (0)
#define STATS_SET_HIGH(x) do { } while (0)
#define STATS_INC_ERR(x) do { } while (0)
-#define STATS_INC_NODEALLOCS(x) do { } while (0)
#define STATS_SET_FREEABLE(x, i) \
do { } while (0)
/* Mismatch ! */
/* Print header */
if (lines == 0) {
- printk(KERN_ERR "Slab corruption: (%s) start=%p, len=%d\n",
- print_tainted(), realobj, size);
+ printk(KERN_ERR "Slab corruption: start=%p, len=%d\n",
+ realobj, size);
print_objinfo(cachep, objp, 0);
}
/* Hexdump the affected line */
}
cachep->lists.next_reap = jiffies + REAPTIMEOUT_LIST3 +
- ((unsigned long)cachep/L1_CACHE_BYTES)%REAPTIMEOUT_LIST3;
-#if DEBUG
- cachep->redzonetest = jiffies + REDZONETIMEOUT +
- ((unsigned long)cachep/L1_CACHE_BYTES)%REDZONETIMEOUT;
-#endif
+ ((unsigned long)cachep)%REAPTIMEOUT_LIST3;
/* Need the semaphore to access the chain. */
down(&cache_chain_sem);
* Grow (by 1) the number of slabs within a cache. This is called by
* kmem_cache_alloc() when there are no active objs left in a cache.
*/
-static int cache_grow (kmem_cache_t * cachep, int flags, int nodeid)
+static int cache_grow (kmem_cache_t * cachep, int flags)
{
struct slab *slabp;
void *objp;
/* Get mem for the objs. */
- if (!(objp = kmem_getpages(cachep, flags, nodeid)))
+ if (!(objp = kmem_getpages(cachep, flags, -1)))
goto failed;
/* Get slab management. */
slabp->inuse++;
next = slab_bufctl(slabp)[slabp->free];
#if DEBUG
- slab_bufctl(slabp)[slabp->free] = BUFCTL_ALLOC;
+ slab_bufctl(slabp)[slabp->free] = BUFCTL_FREE;
#endif
slabp->free = next;
}
if (unlikely(!ac->avail)) {
int x;
- x = cache_grow(cachep, flags, -1);
+ x = cache_grow(cachep, flags);
// cache_grow can reenable interrupts, then ac could change.
ac = ac_data(cachep);
objnr = (objp - slabp->s_mem) / cachep->objsize;
check_slabp(cachep, slabp);
#if DEBUG
- if (slab_bufctl(slabp)[objnr] != BUFCTL_ALLOC) {
+ if (slab_bufctl(slabp)[objnr] != BUFCTL_FREE) {
printk(KERN_ERR "slab: double free detected in cache '%s', objp %p.\n",
cachep->name, objp);
BUG();
return 0;
}
-#ifdef CONFIG_NUMA
/**
* kmem_cache_alloc_node - Allocate an object on the specified node
* @cachep: The cache to allocate from.
*/
void *kmem_cache_alloc_node(kmem_cache_t *cachep, int nodeid)
{
- int loop;
+ size_t offset;
void *objp;
struct slab *slabp;
kmem_bufctl_t next;
- for (loop = 0;;loop++) {
- struct list_head *q;
+ /* The main algorithms are not node aware, thus we have to cheat:
+ * We bypass all caches and allocate a new slab.
+ * The following code is a streamlined copy of cache_grow().
+ */
- objp = NULL;
- check_irq_on();
- spin_lock_irq(&cachep->spinlock);
- /* walk through all partial and empty slab and find one
- * from the right node */
- list_for_each(q,&cachep->lists.slabs_partial) {
- slabp = list_entry(q, struct slab, list);
-
- if (page_to_nid(virt_to_page(slabp->s_mem)) == nodeid ||
- loop > 2)
- goto got_slabp;
- }
- list_for_each(q, &cachep->lists.slabs_free) {
- slabp = list_entry(q, struct slab, list);
+ /* Get colour for the slab, and update the next value. */
+ spin_lock_irq(&cachep->spinlock);
+ offset = cachep->colour_next;
+ cachep->colour_next++;
+ if (cachep->colour_next >= cachep->colour)
+ cachep->colour_next = 0;
+ offset *= cachep->colour_off;
+ spin_unlock_irq(&cachep->spinlock);
- if (page_to_nid(virt_to_page(slabp->s_mem)) == nodeid ||
- loop > 2)
- goto got_slabp;
- }
- spin_unlock_irq(&cachep->spinlock);
+ /* Get mem for the objs. */
+ if (!(objp = kmem_getpages(cachep, GFP_KERNEL, nodeid)))
+ goto failed;
- local_irq_disable();
- if (!cache_grow(cachep, GFP_KERNEL, nodeid)) {
- local_irq_enable();
- return NULL;
- }
- local_irq_enable();
- }
-got_slabp:
- /* found one: allocate object */
- check_slabp(cachep, slabp);
- check_spinlock_acquired(cachep);
+ /* Get slab management. */
+ if (!(slabp = alloc_slabmgmt(cachep, objp, offset, GFP_KERNEL)))
+ goto opps1;
- STATS_INC_ALLOCED(cachep);
- STATS_INC_ACTIVE(cachep);
- STATS_SET_HIGH(cachep);
- STATS_INC_NODEALLOCS(cachep);
+ set_slab_attr(cachep, slabp, objp);
+ cache_init_objs(cachep, slabp, SLAB_CTOR_CONSTRUCTOR);
+ /* The first object is ours: */
objp = slabp->s_mem + slabp->free*cachep->objsize;
-
slabp->inuse++;
next = slab_bufctl(slabp)[slabp->free];
#if DEBUG
- slab_bufctl(slabp)[slabp->free] = BUFCTL_ALLOC;
+ slab_bufctl(slabp)[slabp->free] = BUFCTL_FREE;
#endif
slabp->free = next;
- check_slabp(cachep, slabp);
- /* move slabp to correct slabp list: */
- list_del(&slabp->list);
- if (slabp->free == BUFCTL_END)
- list_add(&slabp->list, &cachep->lists.slabs_full);
- else
- list_add(&slabp->list, &cachep->lists.slabs_partial);
-
- list3_data(cachep)->free_objects--;
+ /* add the remaining objects into the cache */
+ spin_lock_irq(&cachep->spinlock);
+ check_slabp(cachep, slabp);
+ STATS_INC_GROWN(cachep);
+ /* Make slab active. */
+ if (slabp->free == BUFCTL_END) {
+ list_add_tail(&slabp->list, &(list3_data(cachep)->slabs_full));
+ } else {
+ list_add_tail(&slabp->list,
+ &(list3_data(cachep)->slabs_partial));
+ list3_data(cachep)->free_objects += cachep->num-1;
+ }
spin_unlock_irq(&cachep->spinlock);
-
objp = cache_alloc_debugcheck_after(cachep, GFP_KERNEL, objp,
__builtin_return_address(0));
return objp;
+opps1:
+ kmem_freepages(cachep, objp);
+failed:
+ return NULL;
+
}
EXPORT_SYMBOL(kmem_cache_alloc_node);
-#endif
-
/**
* kmalloc - allocate memory
* @size: how many bytes of memory are required.
continue;
kfree(p->ptrs[i]);
}
- kfree(p);
}
EXPORT_SYMBOL(free_percpu);
EXPORT_SYMBOL(kmem_cache_size);
-#if DEBUG
-static void check_slabuse(kmem_cache_t *cachep, struct slab *slabp)
-{
- int i;
-
- if (!(cachep->flags & SLAB_RED_ZONE))
- return; /* no redzone data to check */
-
- for (i=0;i<cachep->num;i++) {
- void *objp = slabp->s_mem + cachep->objsize * i;
- unsigned long red1, red2;
-
- red1 = *dbg_redzone1(cachep, objp);
- red2 = *dbg_redzone2(cachep, objp);
-
- /* simplest case: marked as inactive */
- if (red1 == RED_INACTIVE && red2 == RED_INACTIVE)
- continue;
-
- /* tricky case: if the bufctl value is BUFCTL_ALLOC, then
- * the object is either allocated or somewhere in a cpu
- * cache. The cpu caches are lockless and there might be
- * a concurrent alloc/free call, thus we must accept random
- * combinations of RED_ACTIVE and _INACTIVE
- */
- if (slab_bufctl(slabp)[i] == BUFCTL_ALLOC &&
- (red1 == RED_INACTIVE || red1 == RED_ACTIVE) &&
- (red2 == RED_INACTIVE || red2 == RED_ACTIVE))
- continue;
-
- printk(KERN_ERR "slab %s: redzone mismatch in slabp %p, objp %p, bufctl 0x%x\n",
- cachep->name, slabp, objp, slab_bufctl(slabp)[i]);
- print_objinfo(cachep, objp, 2);
- }
-}
-
-/*
- * Perform a self test on all slabs from a cache
- */
-static void check_redzone(kmem_cache_t *cachep)
-{
- struct list_head *q;
- struct slab *slabp;
-
- check_spinlock_acquired(cachep);
-
- list_for_each(q,&cachep->lists.slabs_full) {
- slabp = list_entry(q, struct slab, list);
-
- if (slabp->inuse != cachep->num) {
- printk(KERN_INFO "slab %s: wrong slabp found in full slab chain at %p (%d/%d).\n",
- cachep->name, slabp, slabp->inuse, cachep->num);
- }
- check_slabp(cachep, slabp);
- check_slabuse(cachep, slabp);
- }
- list_for_each(q,&cachep->lists.slabs_partial) {
- slabp = list_entry(q, struct slab, list);
-
- if (slabp->inuse == cachep->num || slabp->inuse == 0) {
- printk(KERN_INFO "slab %s: wrong slab found in partial chain at %p (%d/%d).\n",
- cachep->name, slabp, slabp->inuse, cachep->num);
- }
- check_slabp(cachep, slabp);
- check_slabuse(cachep, slabp);
- }
- list_for_each(q,&cachep->lists.slabs_free) {
- slabp = list_entry(q, struct slab, list);
-
- if (slabp->inuse != 0) {
- printk(KERN_INFO "slab %s: wrong slab found in free chain at %p (%d/%d).\n",
- cachep->name, slabp, slabp->inuse, cachep->num);
- }
- check_slabp(cachep, slabp);
- check_slabuse(cachep, slabp);
- }
-}
-
-#endif
-
struct ccupdate_struct {
kmem_cache_t *cachep;
struct array_cache *new[NR_CPUS];
drain_array_locked(searchp, ac_data(searchp), 0);
-#if DEBUG
- if(time_before(searchp->redzonetest, jiffies)) {
- searchp->redzonetest = jiffies + REDZONETIMEOUT;
- check_redzone(searchp);
- }
-#endif
if(time_after(searchp->lists.next_reap, jiffies))
goto next_unlock;
* without _too_ many complaints.
*/
#if STATS
- seq_puts(m, "slabinfo - version: 2.1 (statistics)\n");
+ seq_puts(m, "slabinfo - version: 2.0 (statistics)\n");
#else
- seq_puts(m, "slabinfo - version: 2.1\n");
+ seq_puts(m, "slabinfo - version: 2.0\n");
#endif
seq_puts(m, "# name <active_objs> <num_objs> <objsize> <objperslab> <pagesperslab>");
seq_puts(m, " : tunables <batchcount> <limit> <sharedfactor>");
seq_puts(m, " : slabdata <active_slabs> <num_slabs> <sharedavail>");
#if STATS
- seq_puts(m, " : globalstat <listallocs> <maxobjs> <grown> <reaped>"
- " <error> <maxfreeable> <freelimit> <nodeallocs>");
+ seq_puts(m, " : globalstat <listallocs> <maxobjs> <grown> <reaped> <error> <maxfreeable> <freelimit>");
seq_puts(m, " : cpustat <allochit> <allocmiss> <freehit> <freemiss>");
#endif
seq_putc(m, '\n');
unsigned long errors = cachep->errors;
unsigned long max_freeable = cachep->max_freeable;
unsigned long free_limit = cachep->free_limit;
- unsigned long node_allocs = cachep->node_allocs;
- seq_printf(m, " : globalstat %7lu %6lu %5lu %4lu %4lu %4lu %4lu %4lu",
+ seq_printf(m, " : globalstat %7lu %6lu %5lu %4lu %4lu %4lu %4lu",
allocs, high, grown, reaped, errors,
- max_freeable, free_limit, node_allocs);
+ max_freeable, free_limit);
}
/* cpu stats */
{
git://git.onelab.eu / linux-2.6.git / blobdiff