From: Ben Pfaff Date: Tue, 11 Mar 2014 07:11:30 +0000 (-0700) Subject: util: New functions for allocating memory while avoiding false sharing. X-Git-Tag: sliver-openvswitch-2.2.90-1~6^2~81 X-Git-Url: http://git.onelab.eu/?a=commitdiff_plain;ds=sidebyside;h=2fec66dbb5493ee5815507d47796f84c35181dd0;p=sliver-openvswitch.git util: New functions for allocating memory while avoiding false sharing. This factors code out of fat-rwlock, making it easily usable by other code. Signed-off-by: Ben Pfaff Acked-by: Andy Zhou --- diff --git a/lib/fat-rwlock.c b/lib/fat-rwlock.c index 3866dda49..82dfbfe1c 100644 --- a/lib/fat-rwlock.c +++ b/lib/fat-rwlock.c @@ -57,16 +57,6 @@ struct fat_rwlock_slot { * Accessed only by the slot's own thread, so no synchronization is * needed. */ unsigned int depth; - - /* To prevent two of these structures from accidentally occupying the same - * cache line (causing "false sharing"), we cache-align each of these data - * structures. That requires malloc()ing extra space and throwing away - * some space at the beginning, which means that the pointer to this struct - * isn't necessarily the pointer to the beginning of the block, and so we - * need to retain the original pointer to free later. - * - * Accessed only by a single thread, so no synchronization is needed. */ - void *base; /* Pointer to pass to free() for this block. */ }; static void @@ -77,7 +67,7 @@ free_slot(struct fat_rwlock_slot *slot) } list_remove(&slot->list_node); - free(slot->base); + free_cacheline(slot); } static void @@ -124,7 +114,6 @@ static struct fat_rwlock_slot * fat_rwlock_get_slot__(struct fat_rwlock *rwlock) { struct fat_rwlock_slot *slot; - void *base; /* Fast path. */ slot = ovsthread_getspecific(rwlock->key); @@ -134,21 +123,7 @@ fat_rwlock_get_slot__(struct fat_rwlock *rwlock) /* Slow path: create a new slot for 'rwlock' in this thread. */ - /* Allocate room for: - * - * - Up to CACHE_LINE_SIZE - 1 bytes before the per-thread, so that - * the start of the slot doesn't potentially share a cache line. - * - * - The slot itself. - * - * - Space following the slot up to the end of the cache line, so - * that the end of the slot doesn't potentially share a cache - * line. */ - base = xmalloc((CACHE_LINE_SIZE - 1) - + ROUND_UP(sizeof *slot, CACHE_LINE_SIZE)); - slot = (void *) ROUND_UP((uintptr_t) base, CACHE_LINE_SIZE); - - slot->base = base; + slot = xmalloc_cacheline(sizeof *slot); slot->rwlock = rwlock; ovs_mutex_init(&slot->mutex); slot->depth = 0; diff --git a/lib/util.c b/lib/util.c index 6353e9c9e..0d1d9a52b 100644 --- a/lib/util.c +++ b/lib/util.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2008, 2009, 2010, 2011, 2012, 2013 Nicira, Inc. + * Copyright (c) 2008, 2009, 2010, 2011, 2012, 2013, 2014 Nicira, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -174,6 +174,64 @@ x2nrealloc(void *p, size_t *n, size_t s) return xrealloc(p, *n * s); } +/* The desired minimum alignment for an allocated block of memory. */ +#define MEM_ALIGN MAX(sizeof(void *), 8) +BUILD_ASSERT_DECL(IS_POW2(MEM_ALIGN)); +BUILD_ASSERT_DECL(CACHE_LINE_SIZE >= MEM_ALIGN); + +/* Allocates and returns 'size' bytes of memory in dedicated cache lines. That + * is, the memory block returned will not share a cache line with other data, + * avoiding "false sharing". (The memory returned will not be at the start of + * a cache line, though, so don't assume such alignment.) + * + * Use free_cacheline() to free the returned memory block. */ +void * +xmalloc_cacheline(size_t size) +{ + void **payload; + void *base; + + /* Allocate room for: + * + * - Up to CACHE_LINE_SIZE - 1 bytes before the payload, so that the + * start of the payload doesn't potentially share a cache line. + * + * - A payload consisting of a void *, followed by padding out to + * MEM_ALIGN bytes, followed by 'size' bytes of user data. + * + * - Space following the payload up to the end of the cache line, so + * that the end of the payload doesn't potentially share a cache line + * with some following block. */ + base = xmalloc((CACHE_LINE_SIZE - 1) + + ROUND_UP(MEM_ALIGN + size, CACHE_LINE_SIZE)); + + /* Locate the payload and store a pointer to the base at the beginning. */ + payload = (void **) ROUND_UP((uintptr_t) base, CACHE_LINE_SIZE); + *payload = base; + + return (char *) payload + MEM_ALIGN; +} + +/* Like xmalloc_cacheline() but clears the allocated memory to all zero + * bytes. */ +void * +xzalloc_cacheline(size_t size) +{ + void *p = xmalloc_cacheline(size); + memset(p, 0, size); + return p; +} + +/* Frees a memory block allocated with xmalloc_cacheline() or + * xzalloc_cacheline(). */ +void +free_cacheline(void *p) +{ + if (p) { + free(*(void **) ((uintptr_t) p - MEM_ALIGN)); + } +} + char * xasprintf(const char *format, ...) { diff --git a/lib/util.h b/lib/util.h index c1672e55e..3db005aae 100644 --- a/lib/util.h +++ b/lib/util.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2008, 2009, 2010, 2011, 2012, 2013 Nicira, Inc. + * Copyright (c) 2008, 2009, 2010, 2011, 2012, 2013, 2014 Nicira, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -264,6 +264,10 @@ char *xasprintf(const char *format, ...) PRINTF_FORMAT(1, 2) MALLOC_LIKE; char *xvasprintf(const char *format, va_list) PRINTF_FORMAT(1, 0) MALLOC_LIKE; void *x2nrealloc(void *p, size_t *n, size_t s); +void *xmalloc_cacheline(size_t) MALLOC_LIKE; +void *xzalloc_cacheline(size_t) MALLOC_LIKE; +void free_cacheline(void *); + void ovs_strlcpy(char *dst, const char *src, size_t size); void ovs_strzcpy(char *dst, const char *src, size_t size);