2 * PowerPC64 atomic bit operations.
3 * Dave Engebretsen, Todd Inglett, Don Reed, Pat McCarthy, Peter Bergner,
6 * Originally taken from the 32b PPC code. Modified to use 64b values for
7 * the various counters & memory references.
9 * Bitops are odd when viewed on big-endian systems. They were designed
10 * on little endian so the size of the bitset doesn't matter (low order bytes
11 * come first) as long as the bit in question is valid.
13 * Bits are "tested" often using the C expression (val & (1<<nr)) so we do
14 * our best to stay compatible with that. The assumption is that val will
15 * be unsigned long for such tests. As such, we assume the bits are stored
16 * as an array of unsigned long (the usual case is a single unsigned long,
17 * of course). Here's an example bitset with bit numbering:
19 * |63..........0|127........64|195.......128|255.......196|
21 * This leads to a problem. If an int, short or char is passed as a bitset
22 * it will be a bad memory reference since we want to store in chunks
23 * of unsigned long (64 bits here) size.
25 * This program is free software; you can redistribute it and/or
26 * modify it under the terms of the GNU General Public License
27 * as published by the Free Software Foundation; either version
28 * 2 of the License, or (at your option) any later version.
31 #ifndef _PPC64_BITOPS_H
32 #define _PPC64_BITOPS_H
36 #include <asm/memory.h>
39 * clear_bit doesn't imply a memory barrier
41 #define smp_mb__before_clear_bit() smp_mb()
42 #define smp_mb__after_clear_bit() smp_mb()
44 static __inline__ int test_bit(unsigned long nr, __const__ volatile unsigned long *addr)
46 return (1UL & (addr[nr >> 6] >> (nr & 63)));
49 static __inline__ void set_bit(unsigned long nr, volatile unsigned long *addr)
52 unsigned long mask = 1UL << (nr & 0x3f);
53 unsigned long *p = ((unsigned long *)addr) + (nr >> 6);
56 "1: ldarx %0,0,%3 # set_bit\n\
60 : "=&r" (old), "=m" (*p)
61 : "r" (mask), "r" (p), "m" (*p)
65 static __inline__ void clear_bit(unsigned long nr, volatile unsigned long *addr)
68 unsigned long mask = 1UL << (nr & 0x3f);
69 unsigned long *p = ((unsigned long *)addr) + (nr >> 6);
72 "1: ldarx %0,0,%3 # clear_bit\n\
76 : "=&r" (old), "=m" (*p)
77 : "r" (mask), "r" (p), "m" (*p)
81 static __inline__ void change_bit(unsigned long nr, volatile unsigned long *addr)
84 unsigned long mask = 1UL << (nr & 0x3f);
85 unsigned long *p = ((unsigned long *)addr) + (nr >> 6);
88 "1: ldarx %0,0,%3 # change_bit\n\
92 : "=&r" (old), "=m" (*p)
93 : "r" (mask), "r" (p), "m" (*p)
97 static __inline__ int test_and_set_bit(unsigned long nr, volatile unsigned long *addr)
100 unsigned long mask = 1UL << (nr & 0x3f);
101 unsigned long *p = ((unsigned long *)addr) + (nr >> 6);
103 __asm__ __volatile__(
105 "1: ldarx %0,0,%3 # test_and_set_bit\n\
110 : "=&r" (old), "=&r" (t)
111 : "r" (mask), "r" (p)
114 return (old & mask) != 0;
117 static __inline__ int test_and_clear_bit(unsigned long nr, volatile unsigned long *addr)
119 unsigned long old, t;
120 unsigned long mask = 1UL << (nr & 0x3f);
121 unsigned long *p = ((unsigned long *)addr) + (nr >> 6);
123 __asm__ __volatile__(
125 "1: ldarx %0,0,%3 # test_and_clear_bit\n\
130 : "=&r" (old), "=&r" (t)
131 : "r" (mask), "r" (p)
134 return (old & mask) != 0;
137 static __inline__ int test_and_change_bit(unsigned long nr, volatile unsigned long *addr)
139 unsigned long old, t;
140 unsigned long mask = 1UL << (nr & 0x3f);
141 unsigned long *p = ((unsigned long *)addr) + (nr >> 6);
143 __asm__ __volatile__(
145 "1: ldarx %0,0,%3 # test_and_change_bit\n\
150 : "=&r" (old), "=&r" (t)
151 : "r" (mask), "r" (p)
154 return (old & mask) != 0;
158 * non-atomic versions
160 static __inline__ void __set_bit(unsigned long nr, volatile unsigned long *addr)
162 unsigned long mask = 1UL << (nr & 0x3f);
163 unsigned long *p = ((unsigned long *)addr) + (nr >> 6);
168 static __inline__ void __clear_bit(unsigned long nr, volatile unsigned long *addr)
170 unsigned long mask = 1UL << (nr & 0x3f);
171 unsigned long *p = ((unsigned long *)addr) + (nr >> 6);
176 static __inline__ void __change_bit(unsigned long nr, volatile unsigned long *addr)
178 unsigned long mask = 1UL << (nr & 0x3f);
179 unsigned long *p = ((unsigned long *)addr) + (nr >> 6);
184 static __inline__ int __test_and_set_bit(unsigned long nr, volatile unsigned long *addr)
186 unsigned long mask = 1UL << (nr & 0x3f);
187 unsigned long *p = ((unsigned long *)addr) + (nr >> 6);
188 unsigned long old = *p;
191 return (old & mask) != 0;
194 static __inline__ int __test_and_clear_bit(unsigned long nr, volatile unsigned long *addr)
196 unsigned long mask = 1UL << (nr & 0x3f);
197 unsigned long *p = ((unsigned long *)addr) + (nr >> 6);
198 unsigned long old = *p;
201 return (old & mask) != 0;
204 static __inline__ int __test_and_change_bit(unsigned long nr, volatile unsigned long *addr)
206 unsigned long mask = 1UL << (nr & 0x3f);
207 unsigned long *p = ((unsigned long *)addr) + (nr >> 6);
208 unsigned long old = *p;
211 return (old & mask) != 0;
215 * Return the zero-based bit position (from RIGHT TO LEFT, 63 -> 0) of the
216 * most significant (left-most) 1-bit in a double word.
218 static __inline__ int __ilog2(unsigned long x)
222 asm ("cntlzd %0,%1" : "=r" (lz) : "r" (x));
227 * Determines the bit position of the least significant (rightmost) 0 bit
228 * in the specified double word. The returned bit position will be zero-based,
229 * starting from the right side (63 - 0).
231 static __inline__ unsigned long ffz(unsigned long x)
233 /* no zero exists anywhere in the 8 byte area. */
238 * Calculate the bit position of the least signficant '1' bit in x
239 * (since x has been changed this will actually be the least signficant
240 * '0' bit in * the original x). Note: (x & -x) gives us a mask that
241 * is the least significant * (RIGHT-most) 1-bit of the value in x.
243 return __ilog2(x & -x);
246 static __inline__ int __ffs(unsigned long x)
248 return __ilog2(x & -x);
252 * ffs: find first bit set. This is defined the same way as
253 * the libc and compiler builtin ffs routines, therefore
254 * differs in spirit from the above ffz (man ffs).
256 static __inline__ int ffs(int x)
258 unsigned long i = (unsigned long)x;
259 return __ilog2(i & -i) + 1;
263 * fls: find last (most-significant) bit set.
264 * Note fls(0) = 0, fls(1) = 1, fls(0x80000000) = 32.
266 #define fls(x) generic_fls(x)
269 * hweightN: returns the hamming weight (i.e. the number
270 * of bits set) of a N-bit word
272 #define hweight64(x) generic_hweight64(x)
273 #define hweight32(x) generic_hweight32(x)
274 #define hweight16(x) generic_hweight16(x)
275 #define hweight8(x) generic_hweight8(x)
277 extern unsigned long find_next_zero_bit(unsigned long *addr, unsigned long size, unsigned long offset);
278 #define find_first_zero_bit(addr, size) \
279 find_next_zero_bit((addr), (size), 0)
281 extern unsigned long find_next_bit(unsigned long *addr, unsigned long size, unsigned long offset);
282 #define find_first_bit(addr, size) \
283 find_next_bit((addr), (size), 0)
285 extern unsigned long find_next_zero_le_bit(unsigned long *addr, unsigned long size, unsigned long offset);
286 #define find_first_zero_le_bit(addr, size) \
287 find_next_zero_le_bit((addr), (size), 0)
289 static __inline__ int test_le_bit(unsigned long nr, __const__ unsigned long * addr)
291 __const__ unsigned char *ADDR = (__const__ unsigned char *) addr;
292 return (ADDR[nr >> 3] >> (nr & 7)) & 1;
296 * non-atomic versions
298 static __inline__ void __set_le_bit(unsigned long nr, unsigned long *addr)
300 unsigned char *ADDR = (unsigned char *)addr;
303 *ADDR |= 1 << (nr & 0x07);
306 static __inline__ void __clear_le_bit(unsigned long nr, unsigned long *addr)
308 unsigned char *ADDR = (unsigned char *)addr;
311 *ADDR &= ~(1 << (nr & 0x07));
314 static __inline__ int __test_and_set_le_bit(unsigned long nr, unsigned long *addr)
317 unsigned char *ADDR = (unsigned char *)addr;
320 mask = 1 << (nr & 0x07);
321 retval = (mask & *ADDR) != 0;
326 static __inline__ int __test_and_clear_le_bit(unsigned long nr, unsigned long *addr)
329 unsigned char *ADDR = (unsigned char *)addr;
332 mask = 1 << (nr & 0x07);
333 retval = (mask & *ADDR) != 0;
338 #define ext2_set_bit(nr,addr) \
339 __test_and_set_le_bit((nr),(unsigned long*)addr)
340 #define ext2_clear_bit(nr, addr) \
341 __test_and_clear_le_bit((nr),(unsigned long*)addr)
343 #define ext2_set_bit_atomic(lock, nr, addr) \
347 ret = ext2_set_bit((nr), (addr)); \
352 #define ext2_clear_bit_atomic(lock, nr, addr) \
356 ret = ext2_clear_bit((nr), (addr)); \
361 #define ext2_test_bit(nr, addr) test_le_bit((nr),(unsigned long*)addr)
362 #define ext2_find_first_zero_bit(addr, size) \
363 find_first_zero_le_bit((unsigned long*)addr, size)
364 #define ext2_find_next_zero_bit(addr, size, off) \
365 find_next_zero_le_bit((unsigned long*)addr, size, off)
367 #define minix_test_and_set_bit(nr,addr) test_and_set_bit(nr,addr)
368 #define minix_set_bit(nr,addr) set_bit(nr,addr)
369 #define minix_test_and_clear_bit(nr,addr) test_and_clear_bit(nr,addr)
370 #define minix_test_bit(nr,addr) test_bit(nr,addr)
371 #define minix_find_first_zero_bit(addr,size) find_first_zero_bit(addr,size)
373 #endif /* __KERNEL__ */
374 #endif /* _PPC64_BITOPS_H */