X-Git-Url: http://git.onelab.eu/?a=blobdiff_plain;f=mm%2Fmempool.c;h=ccd8cb8cd41f59fc19e2198bfb2862ad72d62ff9;hb=refs%2Fheads%2Fvserver;hp=8dec8779e28f0c60dfff8fc2afce56308b9c50d5;hpb=c7b5ebbddf7bcd3651947760f423e3783bbe6573;p=linux-2.6.git diff --git a/mm/mempool.c b/mm/mempool.c index 8dec8779e..ccd8cb8cd 100644 --- a/mm/mempool.c +++ b/mm/mempool.c @@ -51,16 +51,23 @@ static void free_pool(mempool_t *pool) * functions might sleep - as long as the mempool_alloc function is not called * from IRQ contexts. */ -mempool_t * mempool_create(int min_nr, mempool_alloc_t *alloc_fn, +mempool_t *mempool_create(int min_nr, mempool_alloc_t *alloc_fn, mempool_free_t *free_fn, void *pool_data) { - mempool_t *pool; + return mempool_create_node(min_nr,alloc_fn,free_fn, pool_data,-1); +} +EXPORT_SYMBOL(mempool_create); - pool = kmalloc(sizeof(*pool), GFP_KERNEL); +mempool_t *mempool_create_node(int min_nr, mempool_alloc_t *alloc_fn, + mempool_free_t *free_fn, void *pool_data, int node_id) +{ + mempool_t *pool; + pool = kmalloc_node(sizeof(*pool), GFP_KERNEL, node_id); if (!pool) return NULL; memset(pool, 0, sizeof(*pool)); - pool->elements = kmalloc(min_nr * sizeof(void *), GFP_KERNEL); + pool->elements = kmalloc_node(min_nr * sizeof(void *), + GFP_KERNEL, node_id); if (!pool->elements) { kfree(pool); return NULL; @@ -87,7 +94,7 @@ mempool_t * mempool_create(int min_nr, mempool_alloc_t *alloc_fn, } return pool; } -EXPORT_SYMBOL(mempool_create); +EXPORT_SYMBOL(mempool_create_node); /** * mempool_resize - resize an existing memory pool @@ -105,7 +112,7 @@ EXPORT_SYMBOL(mempool_create); * while this function is running. mempool_alloc() & mempool_free() * might be called (eg. from IRQ contexts) while this function executes. */ -int mempool_resize(mempool_t *pool, int new_min_nr, int gfp_mask) +int mempool_resize(mempool_t *pool, int new_min_nr, gfp_t gfp_mask) { void *element; void **new_elements; @@ -114,8 +121,8 @@ int mempool_resize(mempool_t *pool, int new_min_nr, int gfp_mask) BUG_ON(new_min_nr <= 0); spin_lock_irqsave(&pool->lock, flags); - if (new_min_nr < pool->min_nr) { - while (pool->curr_nr > new_min_nr) { + if (new_min_nr <= pool->min_nr) { + while (new_min_nr < pool->curr_nr) { element = remove_element(pool); spin_unlock_irqrestore(&pool->lock, flags); pool->free(element, pool->pool_data); @@ -132,6 +139,12 @@ int mempool_resize(mempool_t *pool, int new_min_nr, int gfp_mask) return -ENOMEM; spin_lock_irqsave(&pool->lock, flags); + if (unlikely(new_min_nr <= pool->min_nr)) { + /* Raced, other resize will do our work */ + spin_unlock_irqrestore(&pool->lock, flags); + kfree(new_elements); + goto out; + } memcpy(new_elements, pool->elements, pool->curr_nr * sizeof(*new_elements)); kfree(pool->elements); @@ -149,7 +162,7 @@ int mempool_resize(mempool_t *pool, int new_min_nr, int gfp_mask) } else { spin_unlock_irqrestore(&pool->lock, flags); pool->free(element, pool->pool_data); /* Raced */ - spin_lock_irqsave(&pool->lock, flags); + goto out; } } out_unlock: @@ -170,8 +183,8 @@ EXPORT_SYMBOL(mempool_resize); */ void mempool_destroy(mempool_t *pool) { - if (pool->curr_nr != pool->min_nr) - BUG(); /* There were outstanding elements */ + /* Check for outstanding elements */ + BUG_ON(pool->curr_nr != pool->min_nr); free_pool(pool); } EXPORT_SYMBOL(mempool_destroy); @@ -187,36 +200,27 @@ EXPORT_SYMBOL(mempool_destroy); * *never* fails when called from process contexts. (it might * fail if called from an IRQ context.) */ -void * mempool_alloc(mempool_t *pool, int gfp_mask) +void * mempool_alloc(mempool_t *pool, gfp_t gfp_mask) { void *element; unsigned long flags; - DEFINE_WAIT(wait); - int gfp_nowait = gfp_mask & ~(__GFP_WAIT | __GFP_IO); + wait_queue_t wait; + gfp_t gfp_temp; might_sleep_if(gfp_mask & __GFP_WAIT); + + gfp_mask |= __GFP_NOMEMALLOC; /* don't allocate emergency reserves */ + gfp_mask |= __GFP_NORETRY; /* don't loop in __alloc_pages */ + gfp_mask |= __GFP_NOWARN; /* failures are OK */ + + gfp_temp = gfp_mask & ~(__GFP_WAIT|__GFP_IO); + repeat_alloc: - element = pool->alloc(gfp_nowait|__GFP_NOWARN, pool->pool_data); + + element = pool->alloc(gfp_temp, pool->pool_data); if (likely(element != NULL)) return element; - /* - * If the pool is less than 50% full and we can perform effective - * page reclaim then try harder to allocate an element. - */ - mb(); - if ((gfp_mask & __GFP_FS) && (gfp_mask != gfp_nowait) && - (pool->curr_nr <= pool->min_nr/2)) { - element = pool->alloc(gfp_mask, pool->pool_data); - if (likely(element != NULL)) - return element; - } - - /* - * Kick the VM at this point. - */ - wakeup_bdflush(0); - spin_lock_irqsave(&pool->lock, flags); if (likely(pool->curr_nr)) { element = remove_element(pool); @@ -229,10 +233,18 @@ repeat_alloc: if (!(gfp_mask & __GFP_WAIT)) return NULL; + /* Now start performing page reclaim */ + gfp_temp = gfp_mask; + init_wait(&wait); prepare_to_wait(&pool->wait, &wait, TASK_UNINTERRUPTIBLE); - mb(); - if (!pool->curr_nr) - io_schedule(); + smp_mb(); + if (!pool->curr_nr) { + /* + * FIXME: this should be io_schedule(). The timeout is there + * as a workaround for some DM problems in 2.6.18. + */ + io_schedule_timeout(5*HZ); + } finish_wait(&pool->wait, &wait); goto repeat_alloc; @@ -251,7 +263,7 @@ void mempool_free(void *element, mempool_t *pool) { unsigned long flags; - mb(); + smp_mb(); if (pool->curr_nr < pool->min_nr) { spin_lock_irqsave(&pool->lock, flags); if (pool->curr_nr < pool->min_nr) { @@ -269,16 +281,58 @@ EXPORT_SYMBOL(mempool_free); /* * A commonly used alloc and free fn. */ -void *mempool_alloc_slab(int gfp_mask, void *pool_data) +void *mempool_alloc_slab(gfp_t gfp_mask, void *pool_data) { - kmem_cache_t *mem = (kmem_cache_t *) pool_data; + struct kmem_cache *mem = pool_data; return kmem_cache_alloc(mem, gfp_mask); } EXPORT_SYMBOL(mempool_alloc_slab); void mempool_free_slab(void *element, void *pool_data) { - kmem_cache_t *mem = (kmem_cache_t *) pool_data; + struct kmem_cache *mem = pool_data; kmem_cache_free(mem, element); } EXPORT_SYMBOL(mempool_free_slab); + +/* + * A commonly used alloc and free fn that kmalloc/kfrees the amount of memory + * specfied by pool_data + */ +void *mempool_kmalloc(gfp_t gfp_mask, void *pool_data) +{ + size_t size = (size_t)(long)pool_data; + return kmalloc(size, gfp_mask); +} +EXPORT_SYMBOL(mempool_kmalloc); + +void *mempool_kzalloc(gfp_t gfp_mask, void *pool_data) +{ + size_t size = (size_t) pool_data; + return kzalloc(size, gfp_mask); +} +EXPORT_SYMBOL(mempool_kzalloc); + +void mempool_kfree(void *element, void *pool_data) +{ + kfree(element); +} +EXPORT_SYMBOL(mempool_kfree); + +/* + * A simple mempool-backed page allocator that allocates pages + * of the order specified by pool_data. + */ +void *mempool_alloc_pages(gfp_t gfp_mask, void *pool_data) +{ + int order = (int)(long)pool_data; + return alloc_pages(gfp_mask, order); +} +EXPORT_SYMBOL(mempool_alloc_pages); + +void mempool_free_pages(void *element, void *pool_data) +{ + int order = (int)(long)pool_data; + __free_pages(element, order); +} +EXPORT_SYMBOL(mempool_free_pages);