* Copyright 1992, Linus Torvalds.
*/
-#include <linux/config.h>
+#include <asm/alternative.h>
-#ifdef CONFIG_SMP
-#define LOCK_PREFIX "lock ; "
+#if __GNUC__ < 4 || (__GNUC__ == 4 && __GNUC_MINOR__ < 1)
+/* Technically wrong, but this avoids compilation errors on some gcc
+ versions. */
+#define ADDR "=m" (*(volatile long *) addr)
#else
-#define LOCK_PREFIX ""
+#define ADDR "+m" (*(volatile long *) addr)
#endif
-#define ADDR (*(volatile long *) addr)
-
/**
* set_bit - Atomically set a bit in memory
* @nr: the bit to set
* Note that @nr may be almost arbitrarily large; this function is not
* restricted to acting on a single-word quantity.
*/
-static __inline__ void set_bit(long nr, volatile void * addr)
+static __inline__ void set_bit(int nr, volatile void * addr)
{
__asm__ __volatile__( LOCK_PREFIX
- "btsq %1,%0"
- :"=m" (ADDR)
+ "btsl %1,%0"
+ :ADDR
:"dIr" (nr) : "memory");
}
{
__asm__ volatile(
"btsl %1,%0"
- :"=m" (ADDR)
+ :ADDR
:"dIr" (nr) : "memory");
}
{
__asm__ __volatile__( LOCK_PREFIX
"btrl %1,%0"
- :"=m" (ADDR)
+ :ADDR
:"dIr" (nr));
}
{
__asm__ __volatile__(
"btrl %1,%0"
- :"=m" (ADDR)
+ :ADDR
:"dIr" (nr));
}
{
__asm__ __volatile__(
"btcl %1,%0"
- :"=m" (ADDR)
+ :ADDR
:"dIr" (nr));
}
{
__asm__ __volatile__( LOCK_PREFIX
"btcl %1,%0"
- :"=m" (ADDR)
+ :ADDR
:"dIr" (nr));
}
__asm__ __volatile__( LOCK_PREFIX
"btsl %2,%1\n\tsbbl %0,%0"
- :"=r" (oldbit),"=m" (ADDR)
+ :"=r" (oldbit),ADDR
:"dIr" (nr) : "memory");
return oldbit;
}
__asm__(
"btsl %2,%1\n\tsbbl %0,%0"
- :"=r" (oldbit),"=m" (ADDR)
+ :"=r" (oldbit),ADDR
:"dIr" (nr));
return oldbit;
}
__asm__ __volatile__( LOCK_PREFIX
"btrl %2,%1\n\tsbbl %0,%0"
- :"=r" (oldbit),"=m" (ADDR)
+ :"=r" (oldbit),ADDR
:"dIr" (nr) : "memory");
return oldbit;
}
__asm__(
"btrl %2,%1\n\tsbbl %0,%0"
- :"=r" (oldbit),"=m" (ADDR)
+ :"=r" (oldbit),ADDR
:"dIr" (nr));
return oldbit;
}
__asm__ __volatile__(
"btcl %2,%1\n\tsbbl %0,%0"
- :"=r" (oldbit),"=m" (ADDR)
+ :"=r" (oldbit),ADDR
:"dIr" (nr) : "memory");
return oldbit;
}
__asm__ __volatile__( LOCK_PREFIX
"btcl %2,%1\n\tsbbl %0,%0"
- :"=r" (oldbit),"=m" (ADDR)
+ :"=r" (oldbit),ADDR
:"dIr" (nr) : "memory");
return oldbit;
}
__asm__ __volatile__(
"btl %2,%1\n\tsbbl %0,%0"
:"=r" (oldbit)
- :"m" (ADDR),"dIr" (nr));
+ :"m" (*(volatile long *)addr),"dIr" (nr));
return oldbit;
}
#undef ADDR
-/**
- * find_first_zero_bit - find the first zero bit in a memory region
- * @addr: The address to start the search at
- * @size: The maximum size to search
- *
- * Returns the bit-number of the first zero bit, not the number of the byte
- * containing a bit.
- */
-static __inline__ int find_first_zero_bit(const unsigned long * addr, unsigned size)
-{
- int d0, d1, d2;
- int res;
-
- if (!size)
- return 0;
- __asm__ __volatile__(
- "movl $-1,%%eax\n\t"
- "xorl %%edx,%%edx\n\t"
- "repe; scasl\n\t"
- "je 1f\n\t"
- "xorl -4(%%rdi),%%eax\n\t"
- "subq $4,%%rdi\n\t"
- "bsfl %%eax,%%edx\n"
- "1:\tsubq %%rbx,%%rdi\n\t"
- "shlq $3,%%rdi\n\t"
- "addq %%rdi,%%rdx"
- :"=d" (res), "=&c" (d0), "=&D" (d1), "=&a" (d2)
- :"1" ((size + 31) >> 5), "2" (addr), "b" (addr) : "memory");
- return res;
-}
+extern long find_first_zero_bit(const unsigned long * addr, unsigned long size);
+extern long find_next_zero_bit (const unsigned long * addr, long size, long offset);
+extern long find_first_bit(const unsigned long * addr, unsigned long size);
+extern long find_next_bit(const unsigned long * addr, long size, long offset);
-/**
- * find_next_zero_bit - find the first zero bit in a memory region
- * @addr: The address to base the search on
- * @offset: The bitnumber to start searching at
- * @size: The maximum size to search
- */
-static __inline__ int find_next_zero_bit (const unsigned long * addr, int size, int offset)
+/* return index of first bet set in val or max when no bit is set */
+static inline unsigned long __scanbit(unsigned long val, unsigned long max)
{
- unsigned long * p = ((unsigned long *) addr) + (offset >> 6);
- unsigned long set = 0;
- unsigned long res, bit = offset&63;
-
- if (bit) {
- /*
- * Look for zero in first word
- */
- __asm__("bsfq %1,%0\n\t"
- "cmoveq %2,%0"
- : "=r" (set)
- : "r" (~(*p >> bit)), "r"(64L));
- if (set < (64 - bit))
- return set + offset;
- set = 64 - bit;
- p++;
- }
- /*
- * No zero yet, search remaining full words for a zero
- */
- res = find_first_zero_bit ((const unsigned long *)p, size - 64 * (p - (unsigned long *) addr));
- return (offset + set + res);
+ asm("bsfq %1,%0 ; cmovz %2,%0" : "=&r" (val) : "r" (val), "r" (max));
+ return val;
}
+#define find_first_bit(addr,size) \
+((__builtin_constant_p(size) && (size) <= BITS_PER_LONG ? \
+ (__scanbit(*(unsigned long *)addr,(size))) : \
+ find_first_bit(addr,size)))
-/**
- * find_first_bit - find the first set bit in a memory region
- * @addr: The address to start the search at
- * @size: The maximum size to search
- *
- * Returns the bit-number of the first set bit, not the number of the byte
- * containing a bit.
- */
-static __inline__ int find_first_bit(const unsigned long * addr, unsigned size)
-{
- int d0, d1;
- int res;
-
- /* This looks at memory. Mark it volatile to tell gcc not to move it around */
- __asm__ __volatile__(
- "xorl %%eax,%%eax\n\t"
- "repe; scasl\n\t"
- "jz 1f\n\t"
- "leaq -4(%%rdi),%%rdi\n\t"
- "bsfl (%%rdi),%%eax\n"
- "1:\tsubq %%rbx,%%rdi\n\t"
- "shll $3,%%edi\n\t"
- "addl %%edi,%%eax"
- :"=a" (res), "=&c" (d0), "=&D" (d1)
- :"1" ((size + 31) >> 5), "2" (addr), "b" (addr) : "memory");
- return res;
-}
+#define find_next_bit(addr,size,off) \
+((__builtin_constant_p(size) && (size) <= BITS_PER_LONG ? \
+ ((off) + (__scanbit((*(unsigned long *)addr) >> (off),(size)-(off)))) : \
+ find_next_bit(addr,size,off)))
-/**
- * find_next_bit - find the first set bit in a memory region
- * @addr: The address to base the search on
- * @offset: The bitnumber to start searching at
- * @size: The maximum size to search
- */
-static __inline__ int find_next_bit(const unsigned long * addr, int size, int offset)
-{
- const unsigned long * p = addr + (offset >> 6);
- unsigned long set = 0, bit = offset & 63, res;
+#define find_first_zero_bit(addr,size) \
+((__builtin_constant_p(size) && (size) <= BITS_PER_LONG ? \
+ (__scanbit(~*(unsigned long *)addr,(size))) : \
+ find_first_zero_bit(addr,size)))
- if (bit) {
- /*
- * Look for nonzero in the first 64 bits:
- */
- __asm__("bsfq %1,%0\n\t"
- "cmoveq %2,%0\n\t"
- : "=r" (set)
- : "r" (*p >> bit), "r" (64L));
- if (set < (64 - bit))
- return set + offset;
- set = 64 - bit;
- p++;
- }
- /*
- * No set bit yet, search remaining full words for a bit
- */
- res = find_first_bit (p, size - 64 * (p - addr));
- return (offset + set + res);
-}
+#define find_next_zero_bit(addr,size,off) \
+((__builtin_constant_p(size) && (size) <= BITS_PER_LONG ? \
+ ((off)+(__scanbit(~(((*(unsigned long *)addr)) >> (off)),(size)-(off)))) : \
+ find_next_zero_bit(addr,size,off)))
/*
* Find string of zero bits in a bitmap. -1 when not found.
return word;
}
-#ifdef __KERNEL__
-
-static inline int sched_find_first_bit(const unsigned long *b)
+/*
+ * __fls: find last bit set.
+ * @word: The word to search
+ *
+ * Undefined if no zero exists, so code should check against ~0UL first.
+ */
+static __inline__ unsigned long __fls(unsigned long word)
{
- if (b[0])
- return __ffs(b[0]);
- if (b[1])
- return __ffs(b[1]) + 64;
- if (b[2])
- return __ffs(b[2]) + 128;
+ __asm__("bsrq %1,%0"
+ :"=r" (word)
+ :"rm" (word));
+ return word;
}
+#ifdef __KERNEL__
+
+#include <asm-generic/bitops/sched.h>
+
/**
* ffs - find first bit set
* @x: the word to search
}
/**
- * hweightN - returns the hamming weight of a N-bit word
- * @x: the word to weigh
+ * fls64 - find last bit set in 64 bit word
+ * @x: the word to search
+ *
+ * This is defined the same way as fls.
+ */
+static __inline__ int fls64(__u64 x)
+{
+ if (x == 0)
+ return 0;
+ return __fls(x) + 1;
+}
+
+/**
+ * fls - find last bit set
+ * @x: the word to search
*
- * The Hamming Weight of a number is the total number of bits set in it.
+ * This is defined the same way as ffs.
*/
+static __inline__ int fls(int x)
+{
+ int r;
+
+ __asm__("bsrl %1,%0\n\t"
+ "cmovzl %2,%0"
+ : "=&r" (r) : "rm" (x), "rm" (-1));
+ return r+1;
+}
-#define hweight64(x) generic_hweight64(x)
-#define hweight32(x) generic_hweight32(x)
-#define hweight16(x) generic_hweight16(x)
-#define hweight8(x) generic_hweight8(x)
+#define ARCH_HAS_FAST_MULTIPLIER 1
+
+#include <asm-generic/bitops/hweight.h>
#endif /* __KERNEL__ */
#ifdef __KERNEL__
-#define ext2_set_bit(nr,addr) \
- __test_and_set_bit((nr),(unsigned long*)addr)
+#include <asm-generic/bitops/ext2-non-atomic.h>
+
#define ext2_set_bit_atomic(lock,nr,addr) \
test_and_set_bit((nr),(unsigned long*)addr)
-#define ext2_clear_bit(nr, addr) \
- __test_and_clear_bit((nr),(unsigned long*)addr)
#define ext2_clear_bit_atomic(lock,nr,addr) \
test_and_clear_bit((nr),(unsigned long*)addr)
-#define ext2_test_bit(nr, addr) test_bit((nr),(unsigned long*)addr)
-#define ext2_find_first_zero_bit(addr, size) \
- find_first_zero_bit((unsigned long*)addr, size)
-#define ext2_find_next_zero_bit(addr, size, off) \
- find_next_zero_bit((unsigned long*)addr, size, off)
-
-/* Bitmap functions for the minix filesystem. */
-#define minix_test_and_set_bit(nr,addr) __test_and_set_bit(nr,(void*)addr)
-#define minix_set_bit(nr,addr) __set_bit(nr,(void*)addr)
-#define minix_test_and_clear_bit(nr,addr) __test_and_clear_bit(nr,(void*)addr)
-#define minix_test_bit(nr,addr) test_bit(nr,(void*)addr)
-#define minix_find_first_zero_bit(addr,size) \
- find_first_zero_bit((void*)addr,size)
-
-/* find last set bit */
-#define fls(x) generic_fls(x)
-
-#define ARCH_HAS_ATOMIC_UNSIGNED 1
+
+#include <asm-generic/bitops/minix.h>
#endif /* __KERNEL__ */
git://git.onelab.eu / linux-2.6.git / blobdiff