lib/ovs-atomic-gcc4+.h

   1 /*
   2  * Copyright (c) 2013 Nicira, Inc.
   3  *
   4  * Licensed under the Apache License, Version 2.0 (the "License");
   5  * you may not use this file except in compliance with the License.
   6  * You may obtain a copy of the License at:
   7  *
   8  *     http://www.apache.org/licenses/LICENSE-2.0
   9  *
  10  * Unless required by applicable law or agreed to in writing, software
  11  * distributed under the License is distributed on an "AS IS" BASIS,
  12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13  * See the License for the specific language governing permissions and
  14  * limitations under the License.
  15  */
  16
  17 /* This header implements atomic operation primitives on GCC 4.x. */
  18 #ifndef IN_OVS_ATOMIC_H
  19 #error "This header should only be included indirectly via ovs-atomic.h."
  20 #endif
  21
  22 #define OVS_ATOMIC_GCC4P_IMPL 1
  23
  24 #define DEFINE_LOCKLESS_ATOMIC(TYPE, NAME) typedef struct { TYPE value; } NAME
  25
  26 #define ATOMIC_BOOL_LOCK_FREE 2
  27 DEFINE_LOCKLESS_ATOMIC(bool, atomic_bool);
  28
  29 #define ATOMIC_CHAR_LOCK_FREE 2
  30 DEFINE_LOCKLESS_ATOMIC(char, atomic_char);
  31 DEFINE_LOCKLESS_ATOMIC(signed char, atomic_schar);
  32 DEFINE_LOCKLESS_ATOMIC(unsigned char, atomic_uchar);
  33
  34 #define ATOMIC_SHORT_LOCK_FREE 2
  35 DEFINE_LOCKLESS_ATOMIC(short, atomic_short);
  36 DEFINE_LOCKLESS_ATOMIC(unsigned short, atomic_ushort);
  37
  38 #define ATOMIC_INT_LOCK_FREE 2
  39 DEFINE_LOCKLESS_ATOMIC(int, atomic_int);
  40 DEFINE_LOCKLESS_ATOMIC(unsigned int, atomic_uint);
  41
  42 #if ULONG_MAX <= UINTPTR_MAX
  43     #define ATOMIC_LONG_LOCK_FREE 2
  44     DEFINE_LOCKLESS_ATOMIC(long, atomic_long);
  45     DEFINE_LOCKLESS_ATOMIC(unsigned long, atomic_ulong);
  46 #elif ULONG_MAX == UINT64_MAX
  47     #define ATOMIC_LONG_LOCK_FREE 0
  48     typedef struct locked_int64  atomic_long;
  49     typedef struct locked_uint64 atomic_ulong;
  50 #else
  51     #error "not implemented"
  52 #endif
  53
  54 #if ULLONG_MAX <= UINTPTR_MAX
  55     #define ATOMIC_LLONG_LOCK_FREE 2
  56     DEFINE_LOCKLESS_ATOMIC(long long, atomic_llong);
  57     DEFINE_LOCKLESS_ATOMIC(unsigned long long, atomic_ullong);
  58 #elif ULLONG_MAX == UINT64_MAX
  59     #define ATOMIC_LLONG_LOCK_FREE 0
  60     typedef struct locked_int64  atomic_llong;
  61     typedef struct locked_uint64 atomic_ullong;
  62 #else
  63     #error "not implemented"
  64 #endif
  65
  66 #if SIZE_MAX <= UINTPTR_MAX
  67     DEFINE_LOCKLESS_ATOMIC(size_t, atomic_size_t);
  68     DEFINE_LOCKLESS_ATOMIC(ptrdiff_t, atomic_ptrdiff_t);
  69 #elif SIZE_MAX == UINT64_MAX
  70     typedef struct locked_uint64 atomic_size_t;
  71     typedef struct locked_int64  atomic_ptrdiff_t;
  72 #else
  73     #error "not implemented"
  74 #endif
  75
  76 #if UINTMAX_MAX <= UINTPTR_MAX
  77     DEFINE_LOCKLESS_ATOMIC(intmax_t, atomic_intmax_t);
  78     DEFINE_LOCKLESS_ATOMIC(uintmax_t, atomic_uintmax_t);
  79 #elif UINTMAX_MAX == UINT64_MAX
  80     typedef struct locked_int64  atomic_intmax_t;
  81     typedef struct locked_uint64 atomic_uintmax_t;
  82 #else
  83     #error "not implemented"
  84 #endif
  85
  86 #define ATOMIC_POINTER_LOCK_FREE 2
  87 DEFINE_LOCKLESS_ATOMIC(intptr_t, atomic_intptr_t);
  88 DEFINE_LOCKLESS_ATOMIC(uintptr_t, atomic_uintptr_t);
  89
  90 /* Nonstandard atomic types. */
  91 DEFINE_LOCKLESS_ATOMIC(uint8_t,  atomic_uint8_t);
  92 DEFINE_LOCKLESS_ATOMIC(uint16_t, atomic_uint16_t);
  93 DEFINE_LOCKLESS_ATOMIC(uint32_t, atomic_uint32_t);
  94 DEFINE_LOCKLESS_ATOMIC(int8_t,   atomic_int8_t);
  95 DEFINE_LOCKLESS_ATOMIC(int16_t,  atomic_int16_t);
  96 DEFINE_LOCKLESS_ATOMIC(int32_t,  atomic_int32_t);
  97 #if UINT64_MAX <= UINTPTR_MAX
  98     DEFINE_LOCKLESS_ATOMIC(uint64_t, atomic_uint64_t);
  99     DEFINE_LOCKLESS_ATOMIC(int64_t,  atomic_int64_t);
 100 #else
 101     typedef struct locked_uint64 atomic_uint64_t;
 102     typedef struct locked_int64  atomic_int64_t;
 103 #endif
 104
 105 typedef enum {
 106     memory_order_relaxed,
 107     memory_order_consume,
 108     memory_order_acquire,
 109     memory_order_release,
 110     memory_order_acq_rel,
 111     memory_order_seq_cst
 112 } memory_order;
 113 \f
 114 /* locked_uint64. */
 115
 116 #define IF_LOCKED_UINT64(OBJECT, THEN, ELSE)                            \
 117     __builtin_choose_expr(                                              \
 118         __builtin_types_compatible_p(typeof(OBJECT), struct locked_uint64), \
 119         (THEN), (ELSE))
 120 #define AS_LOCKED_UINT64(OBJECT) ((struct locked_uint64 *) (OBJECT))
 121 #define AS_UINT64(OBJECT) ((uint64_t *) (OBJECT))
 122 struct locked_uint64 {
 123     uint64_t value;
 124 };
 125
 126 uint64_t locked_uint64_load(const struct locked_uint64 *);
 127 void locked_uint64_store(struct locked_uint64 *, uint64_t);
 128 uint64_t locked_uint64_add(struct locked_uint64 *, uint64_t arg);
 129 uint64_t locked_uint64_sub(struct locked_uint64 *, uint64_t arg);
 130 uint64_t locked_uint64_or(struct locked_uint64 *, uint64_t arg);
 131 uint64_t locked_uint64_xor(struct locked_uint64 *, uint64_t arg);
 132 uint64_t locked_uint64_and(struct locked_uint64 *, uint64_t arg);
 133 \f
 134 #define IF_LOCKED_INT64(OBJECT, THEN, ELSE)                             \
 135     __builtin_choose_expr(                                              \
 136         __builtin_types_compatible_p(typeof(OBJECT), struct locked_int64), \
 137         (THEN), (ELSE))
 138 #define AS_LOCKED_INT64(OBJECT) ((struct locked_int64 *) (OBJECT))
 139 #define AS_INT64(OBJECT) ((int64_t *) (OBJECT))
 140 struct locked_int64 {
 141     int64_t value;
 142 };
 143 int64_t locked_int64_load(const struct locked_int64 *);
 144 void locked_int64_store(struct locked_int64 *, int64_t);
 145 int64_t locked_int64_add(struct locked_int64 *, int64_t arg);
 146 int64_t locked_int64_sub(struct locked_int64 *, int64_t arg);
 147 int64_t locked_int64_or(struct locked_int64 *, int64_t arg);
 148 int64_t locked_int64_xor(struct locked_int64 *, int64_t arg);
 149 int64_t locked_int64_and(struct locked_int64 *, int64_t arg);
 150 \f
 151 #define ATOMIC_VAR_INIT(VALUE) { .value = (VALUE) }
 152 #define atomic_init(OBJECT, VALUE) ((OBJECT)->value = (VALUE), (void) 0)
 153
 154 static inline void
 155 atomic_thread_fence(memory_order order)
 156 {
 157     if (order != memory_order_relaxed) {
 158         __sync_synchronize();
 159     }
 160 }
 161
 162 static inline void
 163 atomic_thread_fence_if_seq_cst(memory_order order)
 164 {
 165     if (order == memory_order_seq_cst) {
 166         __sync_synchronize();
 167     }
 168 }
 169
 170 static inline void
 171 atomic_signal_fence(memory_order order OVS_UNUSED)
 172 {
 173     if (order != memory_order_relaxed) {
 174         asm volatile("" : : : "memory");
 175     }
 176 }
 177
 178 #define ATOMIC_SWITCH(OBJECT, LOCKLESS_CASE,                    \
 179                       LOCKED_UINT64_CASE, LOCKED_INT64_CASE)    \
 180     IF_LOCKED_UINT64(OBJECT, LOCKED_UINT64_CASE,                \
 181                      IF_LOCKED_INT64(OBJECT, LOCKED_INT64_CASE, \
 182                                      LOCKLESS_CASE))
 183
 184 #define atomic_is_lock_free(OBJ)                \
 185     ((void) (OBJ)->value,                       \
 186      ATOMIC_SWITCH(OBJ, true, false, false))
 187
 188 #define atomic_store(DST, SRC) \
 189     atomic_store_explicit(DST, SRC, memory_order_seq_cst)
 190 #define atomic_store_explicit(DST, SRC, ORDER)                          \
 191     (ATOMIC_SWITCH(DST,                                                 \
 192                    (atomic_thread_fence(ORDER),                         \
 193                     (DST)->value = (SRC),                               \
 194                     atomic_thread_fence_if_seq_cst(ORDER)),             \
 195                    locked_uint64_store(AS_LOCKED_UINT64(DST), SRC),     \
 196                    locked_int64_store(AS_LOCKED_INT64(DST), SRC)),      \
 197      (void) 0)
 198
 199 #define atomic_read(SRC, DST) \
 200     atomic_read_explicit(SRC, DST, memory_order_seq_cst)
 201 #define atomic_read_explicit(SRC, DST, ORDER)                           \
 202     (ATOMIC_SWITCH(SRC,                                                 \
 203                    (atomic_thread_fence_if_seq_cst(ORDER),              \
 204                     (*DST) = (SRC)->value,                              \
 205                     atomic_thread_fence(ORDER)),                        \
 206                    *(DST) = locked_uint64_load(AS_LOCKED_UINT64(SRC)),  \
 207                    *(DST) = locked_int64_load(AS_LOCKED_INT64(SRC))),   \
 208      (void) 0)
 209
 210 #define atomic_op__(RMW, OP, ARG, ORIG)                                 \
 211     (ATOMIC_SWITCH(RMW,                                                 \
 212                    *(ORIG) = __sync_fetch_and_##OP(&(RMW)->value, ARG), \
 213                    *(ORIG) = locked_uint64_##OP(AS_LOCKED_UINT64(RMW), ARG), \
 214                    *(ORIG) = locked_int64_##OP(AS_LOCKED_INT64(RMW), ARG)), \
 215      (void) 0)
 216
 217 #define atomic_add(RMW, ARG, ORIG) atomic_op__(RMW, add, ARG, ORIG)
 218 #define atomic_sub(RMW, ARG, ORIG) atomic_op__(RMW, sub, ARG, ORIG)
 219 #define atomic_or( RMW, ARG, ORIG) atomic_op__(RMW, or,  ARG, ORIG)
 220 #define atomic_xor(RMW, ARG, ORIG) atomic_op__(RMW, xor, ARG, ORIG)
 221 #define atomic_and(RMW, ARG, ORIG) atomic_op__(RMW, and, ARG, ORIG)
 222
 223 #define atomic_add_explicit(RMW, OPERAND, ORIG, ORDER)  \
 224     ((void) (ORDER), atomic_add(RMW, OPERAND, ORIG))
 225 #define atomic_sub_explicit(RMW, OPERAND, ORIG, ORDER)  \
 226     ((void) (ORDER), atomic_sub(RMW, OPERAND, ORIG))
 227 #define atomic_or_explicit(RMW, OPERAND, ORIG, ORDER)   \
 228     ((void) (ORDER), atomic_or(RMW, OPERAND, ORIG))
 229 #define atomic_xor_explicit(RMW, OPERAND, ORIG, ORDER)  \
 230     ((void) (ORDER), atomic_xor(RMW, OPERAND, ORIG))
 231 #define atomic_and_explicit(RMW, OPERAND, ORIG, ORDER)  \
 232     ((void) (ORDER), atomic_and(RMW, OPERAND, ORIG))
 233 \f
 234 /* atomic_flag */
 235
 236 typedef struct {
 237     int b;
 238 } atomic_flag;
 239 #define ATOMIC_FLAG_INIT { false }
 240
 241 static inline bool
 242 atomic_flag_test_and_set(volatile atomic_flag *object)
 243 {
 244     return __sync_lock_test_and_set(&object->b, 1);
 245 }
 246
 247 static inline bool
 248 atomic_flag_test_and_set_explicit(volatile atomic_flag *object,
 249                                   memory_order order OVS_UNUSED)
 250 {
 251     return atomic_flag_test_and_set(object);
 252 }
 253
 254 static inline void
 255 atomic_flag_clear(volatile atomic_flag *object)
 256 {
 257     __sync_lock_release(&object->b);
 258 }
 259
 260 static inline void
 261 atomic_flag_clear_explicit(volatile atomic_flag *object,
 262                            memory_order order OVS_UNUSED)
 263 {
 264     atomic_flag_clear(object);
 265 }