2 * PowerPC64 atomic bit operations.
3 * Dave Engebretsen, Todd Inglett, Don Reed, Pat McCarthy, Peter Bergner,
6 * Originally taken from the 32b PPC code. Modified to use 64b values for
7 * the various counters & memory references.
9 * Bitops are odd when viewed on big-endian systems. They were designed
10 * on little endian so the size of the bitset doesn't matter (low order bytes
11 * come first) as long as the bit in question is valid.
13 * Bits are "tested" often using the C expression (val & (1<<nr)) so we do
14 * our best to stay compatible with that. The assumption is that val will
15 * be unsigned long for such tests. As such, we assume the bits are stored
16 * as an array of unsigned long (the usual case is a single unsigned long,
17 * of course). Here's an example bitset with bit numbering:
19 * |63..........0|127........64|195.......128|255.......196|
21 * This leads to a problem. If an int, short or char is passed as a bitset
22 * it will be a bad memory reference since we want to store in chunks
23 * of unsigned long (64 bits here) size.
25 * This program is free software; you can redistribute it and/or
26 * modify it under the terms of the GNU General Public License
27 * as published by the Free Software Foundation; either version
28 * 2 of the License, or (at your option) any later version.
31 #ifndef _PPC64_BITOPS_H
32 #define _PPC64_BITOPS_H
36 #include <asm/memory.h>
39 * clear_bit doesn't imply a memory barrier
41 #define smp_mb__before_clear_bit() smp_mb()
42 #define smp_mb__after_clear_bit() smp_mb()
44 static __inline__ int test_bit(unsigned long nr, __const__ volatile unsigned long *addr)
46 return (1UL & (addr[nr >> 6] >> (nr & 63)));
49 static __inline__ void set_bit(unsigned long nr, volatile unsigned long *addr)
52 unsigned long mask = 1UL << (nr & 0x3f);
53 unsigned long *p = ((unsigned long *)addr) + (nr >> 6);
56 "1: ldarx %0,0,%3 # set_bit\n\
60 : "=&r" (old), "=m" (*p)
61 : "r" (mask), "r" (p), "m" (*p)
65 static __inline__ void clear_bit(unsigned long nr, volatile unsigned long *addr)
68 unsigned long mask = 1UL << (nr & 0x3f);
69 unsigned long *p = ((unsigned long *)addr) + (nr >> 6);
72 "1: ldarx %0,0,%3 # clear_bit\n\
76 : "=&r" (old), "=m" (*p)
77 : "r" (mask), "r" (p), "m" (*p)
81 static __inline__ void change_bit(unsigned long nr, volatile unsigned long *addr)
84 unsigned long mask = 1UL << (nr & 0x3f);
85 unsigned long *p = ((unsigned long *)addr) + (nr >> 6);
88 "1: ldarx %0,0,%3 # change_bit\n\
92 : "=&r" (old), "=m" (*p)
93 : "r" (mask), "r" (p), "m" (*p)
97 static __inline__ int test_and_set_bit(unsigned long nr, volatile unsigned long *addr)
100 unsigned long mask = 1UL << (nr & 0x3f);
101 unsigned long *p = ((unsigned long *)addr) + (nr >> 6);
103 __asm__ __volatile__(
105 "1: ldarx %0,0,%3 # test_and_set_bit\n\
110 : "=&r" (old), "=&r" (t)
111 : "r" (mask), "r" (p)
114 return (old & mask) != 0;
117 static __inline__ int test_and_clear_bit(unsigned long nr, volatile unsigned long *addr)
119 unsigned long old, t;
120 unsigned long mask = 1UL << (nr & 0x3f);
121 unsigned long *p = ((unsigned long *)addr) + (nr >> 6);
123 __asm__ __volatile__(
125 "1: ldarx %0,0,%3 # test_and_clear_bit\n\
130 : "=&r" (old), "=&r" (t)
131 : "r" (mask), "r" (p)
134 return (old & mask) != 0;
137 static __inline__ int test_and_change_bit(unsigned long nr, volatile unsigned long *addr)
139 unsigned long old, t;
140 unsigned long mask = 1UL << (nr & 0x3f);
141 unsigned long *p = ((unsigned long *)addr) + (nr >> 6);
143 __asm__ __volatile__(
145 "1: ldarx %0,0,%3 # test_and_change_bit\n\
150 : "=&r" (old), "=&r" (t)
151 : "r" (mask), "r" (p)
154 return (old & mask) != 0;
157 static __inline__ void set_bits(unsigned long mask, unsigned long *addr)
161 __asm__ __volatile__(
162 "1: ldarx %0,0,%3 # set_bit\n\
166 : "=&r" (old), "=m" (*addr)
167 : "r" (mask), "r" (addr), "m" (*addr)
172 * non-atomic versions
174 static __inline__ void __set_bit(unsigned long nr, volatile unsigned long *addr)
176 unsigned long mask = 1UL << (nr & 0x3f);
177 unsigned long *p = ((unsigned long *)addr) + (nr >> 6);
182 static __inline__ void __clear_bit(unsigned long nr, volatile unsigned long *addr)
184 unsigned long mask = 1UL << (nr & 0x3f);
185 unsigned long *p = ((unsigned long *)addr) + (nr >> 6);
190 static __inline__ void __change_bit(unsigned long nr, volatile unsigned long *addr)
192 unsigned long mask = 1UL << (nr & 0x3f);
193 unsigned long *p = ((unsigned long *)addr) + (nr >> 6);
198 static __inline__ int __test_and_set_bit(unsigned long nr, volatile unsigned long *addr)
200 unsigned long mask = 1UL << (nr & 0x3f);
201 unsigned long *p = ((unsigned long *)addr) + (nr >> 6);
202 unsigned long old = *p;
205 return (old & mask) != 0;
208 static __inline__ int __test_and_clear_bit(unsigned long nr, volatile unsigned long *addr)
210 unsigned long mask = 1UL << (nr & 0x3f);
211 unsigned long *p = ((unsigned long *)addr) + (nr >> 6);
212 unsigned long old = *p;
215 return (old & mask) != 0;
218 static __inline__ int __test_and_change_bit(unsigned long nr, volatile unsigned long *addr)
220 unsigned long mask = 1UL << (nr & 0x3f);
221 unsigned long *p = ((unsigned long *)addr) + (nr >> 6);
222 unsigned long old = *p;
225 return (old & mask) != 0;
229 * Return the zero-based bit position (from RIGHT TO LEFT, 63 -> 0) of the
230 * most significant (left-most) 1-bit in a double word.
232 static __inline__ int __ilog2(unsigned long x)
236 asm ("cntlzd %0,%1" : "=r" (lz) : "r" (x));
241 * Determines the bit position of the least significant (rightmost) 0 bit
242 * in the specified double word. The returned bit position will be zero-based,
243 * starting from the right side (63 - 0).
245 static __inline__ unsigned long ffz(unsigned long x)
247 /* no zero exists anywhere in the 8 byte area. */
252 * Calculate the bit position of the least signficant '1' bit in x
253 * (since x has been changed this will actually be the least signficant
254 * '0' bit in * the original x). Note: (x & -x) gives us a mask that
255 * is the least significant * (RIGHT-most) 1-bit of the value in x.
257 return __ilog2(x & -x);
260 static __inline__ int __ffs(unsigned long x)
262 return __ilog2(x & -x);
266 * ffs: find first bit set. This is defined the same way as
267 * the libc and compiler builtin ffs routines, therefore
268 * differs in spirit from the above ffz (man ffs).
270 static __inline__ int ffs(int x)
272 unsigned long i = (unsigned long)x;
273 return __ilog2(i & -i) + 1;
277 * fls: find last (most-significant) bit set.
278 * Note fls(0) = 0, fls(1) = 1, fls(0x80000000) = 32.
280 #define fls(x) generic_fls(x)
283 * hweightN: returns the hamming weight (i.e. the number
284 * of bits set) of a N-bit word
286 #define hweight64(x) generic_hweight64(x)
287 #define hweight32(x) generic_hweight32(x)
288 #define hweight16(x) generic_hweight16(x)
289 #define hweight8(x) generic_hweight8(x)
291 extern unsigned long find_next_zero_bit(unsigned long *addr, unsigned long size, unsigned long offset);
292 #define find_first_zero_bit(addr, size) \
293 find_next_zero_bit((addr), (size), 0)
295 extern unsigned long find_next_bit(unsigned long *addr, unsigned long size, unsigned long offset);
296 #define find_first_bit(addr, size) \
297 find_next_bit((addr), (size), 0)
299 extern unsigned long find_next_zero_le_bit(unsigned long *addr, unsigned long size, unsigned long offset);
300 #define find_first_zero_le_bit(addr, size) \
301 find_next_zero_le_bit((addr), (size), 0)
303 static __inline__ int test_le_bit(unsigned long nr, __const__ unsigned long * addr)
305 __const__ unsigned char *ADDR = (__const__ unsigned char *) addr;
306 return (ADDR[nr >> 3] >> (nr & 7)) & 1;
310 * non-atomic versions
312 static __inline__ void __set_le_bit(unsigned long nr, unsigned long *addr)
314 unsigned char *ADDR = (unsigned char *)addr;
317 *ADDR |= 1 << (nr & 0x07);
320 static __inline__ void __clear_le_bit(unsigned long nr, unsigned long *addr)
322 unsigned char *ADDR = (unsigned char *)addr;
325 *ADDR &= ~(1 << (nr & 0x07));
328 static __inline__ int __test_and_set_le_bit(unsigned long nr, unsigned long *addr)
331 unsigned char *ADDR = (unsigned char *)addr;
334 mask = 1 << (nr & 0x07);
335 retval = (mask & *ADDR) != 0;
340 static __inline__ int __test_and_clear_le_bit(unsigned long nr, unsigned long *addr)
343 unsigned char *ADDR = (unsigned char *)addr;
346 mask = 1 << (nr & 0x07);
347 retval = (mask & *ADDR) != 0;
352 #define ext2_set_bit(nr,addr) \
353 __test_and_set_le_bit((nr),(unsigned long*)addr)
354 #define ext2_clear_bit(nr, addr) \
355 __test_and_clear_le_bit((nr),(unsigned long*)addr)
357 #define ext2_set_bit_atomic(lock, nr, addr) \
361 ret = ext2_set_bit((nr), (addr)); \
366 #define ext2_clear_bit_atomic(lock, nr, addr) \
370 ret = ext2_clear_bit((nr), (addr)); \
375 #define ext2_test_bit(nr, addr) test_le_bit((nr),(unsigned long*)addr)
376 #define ext2_find_first_zero_bit(addr, size) \
377 find_first_zero_le_bit((unsigned long*)addr, size)
378 #define ext2_find_next_zero_bit(addr, size, off) \
379 find_next_zero_le_bit((unsigned long*)addr, size, off)
381 #define minix_test_and_set_bit(nr,addr) test_and_set_bit(nr,addr)
382 #define minix_set_bit(nr,addr) set_bit(nr,addr)
383 #define minix_test_and_clear_bit(nr,addr) test_and_clear_bit(nr,addr)
384 #define minix_test_bit(nr,addr) test_bit(nr,addr)
385 #define minix_find_first_zero_bit(addr,size) find_first_zero_bit(addr,size)
387 #endif /* __KERNEL__ */
388 #endif /* _PPC64_BITOPS_H */