X-Git-Url: http://git.onelab.eu/?a=blobdiff_plain;f=include%2Fasm-x86_64%2Fbitops.h;h=eb4df23e1e411c45c19d9de8d701b8fd30bcdfaa;hb=987b0145d94eecf292d8b301228356f44611ab7c;hp=c91f028db682ee9a36a7239fcbedff80520ec6cb;hpb=e686509282835a634ee3285c39a1b5b04ee2a88f;p=linux-2.6.git diff --git a/include/asm-x86_64/bitops.h b/include/asm-x86_64/bitops.h index c91f028db..eb4df23e1 100644 --- a/include/asm-x86_64/bitops.h +++ b/include/asm-x86_64/bitops.h @@ -25,11 +25,11 @@ * Note that @nr may be almost arbitrarily large; this function is not * restricted to acting on a single-word quantity. */ -static __inline__ void set_bit(long nr, volatile void * addr) +static __inline__ void set_bit(int nr, volatile void * addr) { __asm__ __volatile__( LOCK_PREFIX - "btsq %1,%0" - :"=m" (ADDR) + "btsl %1,%0" + :"+m" (ADDR) :"dIr" (nr) : "memory"); } @@ -46,7 +46,7 @@ static __inline__ void __set_bit(int nr, volatile void * addr) { __asm__ volatile( "btsl %1,%0" - :"=m" (ADDR) + :"+m" (ADDR) :"dIr" (nr) : "memory"); } @@ -64,7 +64,7 @@ static __inline__ void clear_bit(int nr, volatile void * addr) { __asm__ __volatile__( LOCK_PREFIX "btrl %1,%0" - :"=m" (ADDR) + :"+m" (ADDR) :"dIr" (nr)); } @@ -72,7 +72,7 @@ static __inline__ void __clear_bit(int nr, volatile void * addr) { __asm__ __volatile__( "btrl %1,%0" - :"=m" (ADDR) + :"+m" (ADDR) :"dIr" (nr)); } @@ -92,7 +92,7 @@ static __inline__ void __change_bit(int nr, volatile void * addr) { __asm__ __volatile__( "btcl %1,%0" - :"=m" (ADDR) + :"+m" (ADDR) :"dIr" (nr)); } @@ -109,7 +109,7 @@ static __inline__ void change_bit(int nr, volatile void * addr) { __asm__ __volatile__( LOCK_PREFIX "btcl %1,%0" - :"=m" (ADDR) + :"+m" (ADDR) :"dIr" (nr)); } @@ -127,7 +127,7 @@ static __inline__ int test_and_set_bit(int nr, volatile void * addr) __asm__ __volatile__( LOCK_PREFIX "btsl %2,%1\n\tsbbl %0,%0" - :"=r" (oldbit),"=m" (ADDR) + :"=r" (oldbit),"+m" (ADDR) :"dIr" (nr) : "memory"); return oldbit; } @@ -147,7 +147,7 @@ static __inline__ int __test_and_set_bit(int nr, volatile void * addr) __asm__( "btsl %2,%1\n\tsbbl %0,%0" - :"=r" (oldbit),"=m" (ADDR) + :"=r" (oldbit),"+m" (ADDR) :"dIr" (nr)); return oldbit; } @@ -166,7 +166,7 @@ static __inline__ int test_and_clear_bit(int nr, volatile void * addr) __asm__ __volatile__( LOCK_PREFIX "btrl %2,%1\n\tsbbl %0,%0" - :"=r" (oldbit),"=m" (ADDR) + :"=r" (oldbit),"+m" (ADDR) :"dIr" (nr) : "memory"); return oldbit; } @@ -186,7 +186,7 @@ static __inline__ int __test_and_clear_bit(int nr, volatile void * addr) __asm__( "btrl %2,%1\n\tsbbl %0,%0" - :"=r" (oldbit),"=m" (ADDR) + :"=r" (oldbit),"+m" (ADDR) :"dIr" (nr)); return oldbit; } @@ -198,7 +198,7 @@ static __inline__ int __test_and_change_bit(int nr, volatile void * addr) __asm__ __volatile__( "btcl %2,%1\n\tsbbl %0,%0" - :"=r" (oldbit),"=m" (ADDR) + :"=r" (oldbit),"+m" (ADDR) :"dIr" (nr) : "memory"); return oldbit; } @@ -217,7 +217,7 @@ static __inline__ int test_and_change_bit(int nr, volatile void * addr) __asm__ __volatile__( LOCK_PREFIX "btcl %2,%1\n\tsbbl %0,%0" - :"=r" (oldbit),"=m" (ADDR) + :"=r" (oldbit),"+m" (ADDR) :"dIr" (nr) : "memory"); return oldbit; } @@ -254,128 +254,37 @@ static __inline__ int variable_test_bit(int nr, volatile const void * addr) #undef ADDR -/** - * find_first_zero_bit - find the first zero bit in a memory region - * @addr: The address to start the search at - * @size: The maximum size to search - * - * Returns the bit-number of the first zero bit, not the number of the byte - * containing a bit. - */ -static __inline__ int find_first_zero_bit(const unsigned long * addr, unsigned size) -{ - int d0, d1, d2; - int res; - - if (!size) - return 0; - __asm__ __volatile__( - "movl $-1,%%eax\n\t" - "xorl %%edx,%%edx\n\t" - "repe; scasl\n\t" - "je 1f\n\t" - "xorl -4(%%rdi),%%eax\n\t" - "subq $4,%%rdi\n\t" - "bsfl %%eax,%%edx\n" - "1:\tsubq %%rbx,%%rdi\n\t" - "shlq $3,%%rdi\n\t" - "addq %%rdi,%%rdx" - :"=d" (res), "=&c" (d0), "=&D" (d1), "=&a" (d2) - :"1" ((size + 31) >> 5), "2" (addr), "b" (addr) : "memory"); - return res; -} +extern long find_first_zero_bit(const unsigned long * addr, unsigned long size); +extern long find_next_zero_bit (const unsigned long * addr, long size, long offset); +extern long find_first_bit(const unsigned long * addr, unsigned long size); +extern long find_next_bit(const unsigned long * addr, long size, long offset); -/** - * find_next_zero_bit - find the first zero bit in a memory region - * @addr: The address to base the search on - * @offset: The bitnumber to start searching at - * @size: The maximum size to search - */ -static __inline__ int find_next_zero_bit (const unsigned long * addr, int size, int offset) +/* return index of first bet set in val or max when no bit is set */ +static inline unsigned long __scanbit(unsigned long val, unsigned long max) { - unsigned long * p = ((unsigned long *) addr) + (offset >> 6); - unsigned long set = 0; - unsigned long res, bit = offset&63; - - if (bit) { - /* - * Look for zero in first word - */ - __asm__("bsfq %1,%0\n\t" - "cmoveq %2,%0" - : "=r" (set) - : "r" (~(*p >> bit)), "r"(64L)); - if (set < (64 - bit)) - return set + offset; - set = 64 - bit; - p++; - } - /* - * No zero yet, search remaining full words for a zero - */ - res = find_first_zero_bit ((const unsigned long *)p, size - 64 * (p - (unsigned long *) addr)); - return (offset + set + res); + asm("bsfq %1,%0 ; cmovz %2,%0" : "=&r" (val) : "r" (val), "r" (max)); + return val; } +#define find_first_bit(addr,size) \ +((__builtin_constant_p(size) && (size) <= BITS_PER_LONG ? \ + (__scanbit(*(unsigned long *)addr,(size))) : \ + find_first_bit(addr,size))) -/** - * find_first_bit - find the first set bit in a memory region - * @addr: The address to start the search at - * @size: The maximum size to search - * - * Returns the bit-number of the first set bit, not the number of the byte - * containing a bit. - */ -static __inline__ int find_first_bit(const unsigned long * addr, unsigned size) -{ - int d0, d1; - int res; - - /* This looks at memory. Mark it volatile to tell gcc not to move it around */ - __asm__ __volatile__( - "xorl %%eax,%%eax\n\t" - "repe; scasl\n\t" - "jz 1f\n\t" - "leaq -4(%%rdi),%%rdi\n\t" - "bsfl (%%rdi),%%eax\n" - "1:\tsubq %%rbx,%%rdi\n\t" - "shll $3,%%edi\n\t" - "addl %%edi,%%eax" - :"=a" (res), "=&c" (d0), "=&D" (d1) - :"1" ((size + 31) >> 5), "2" (addr), "b" (addr) : "memory"); - return res; -} +#define find_next_bit(addr,size,off) \ +((__builtin_constant_p(size) && (size) <= BITS_PER_LONG ? \ + ((off) + (__scanbit((*(unsigned long *)addr) >> (off),(size)-(off)))) : \ + find_next_bit(addr,size,off))) -/** - * find_next_bit - find the first set bit in a memory region - * @addr: The address to base the search on - * @offset: The bitnumber to start searching at - * @size: The maximum size to search - */ -static __inline__ int find_next_bit(const unsigned long * addr, int size, int offset) -{ - const unsigned long * p = addr + (offset >> 6); - unsigned long set = 0, bit = offset & 63, res; +#define find_first_zero_bit(addr,size) \ +((__builtin_constant_p(size) && (size) <= BITS_PER_LONG ? \ + (__scanbit(~*(unsigned long *)addr,(size))) : \ + find_first_zero_bit(addr,size))) - if (bit) { - /* - * Look for nonzero in the first 64 bits: - */ - __asm__("bsfq %1,%0\n\t" - "cmoveq %2,%0\n\t" - : "=r" (set) - : "r" (*p >> bit), "r" (64L)); - if (set < (64 - bit)) - return set + offset; - set = 64 - bit; - p++; - } - /* - * No set bit yet, search remaining full words for a bit - */ - res = find_first_bit (p, size - 64 * (p - addr)); - return (offset + set + res); -} +#define find_next_zero_bit(addr,size,off) \ +((__builtin_constant_p(size) && (size) <= BITS_PER_LONG ? \ + ((off)+(__scanbit(~(((*(unsigned long *)addr)) >> (off)),(size)-(off)))) : \ + find_next_zero_bit(addr,size,off))) /* * Find string of zero bits in a bitmap. -1 when not found. @@ -431,6 +340,20 @@ static __inline__ unsigned long __ffs(unsigned long word) return word; } +/* + * __fls: find last bit set. + * @word: The word to search + * + * Undefined if no zero exists, so code should check against ~0UL first. + */ +static __inline__ unsigned long __fls(unsigned long word) +{ + __asm__("bsrq %1,%0" + :"=r" (word) + :"rm" (word)); + return word; +} + #ifdef __KERNEL__ static inline int sched_find_first_bit(const unsigned long *b) @@ -439,8 +362,7 @@ static inline int sched_find_first_bit(const unsigned long *b) return __ffs(b[0]); if (b[1]) return __ffs(b[1]) + 64; - if (b[2]) - return __ffs(b[2]) + 128; + return __ffs(b[2]) + 128; } /** @@ -461,6 +383,35 @@ static __inline__ int ffs(int x) return r+1; } +/** + * fls64 - find last bit set in 64 bit word + * @x: the word to search + * + * This is defined the same way as fls. + */ +static __inline__ int fls64(__u64 x) +{ + if (x == 0) + return 0; + return __fls(x) + 1; +} + +/** + * fls - find last bit set + * @x: the word to search + * + * This is defined the same way as ffs. + */ +static __inline__ int fls(int x) +{ + int r; + + __asm__("bsrl %1,%0\n\t" + "cmovzl %2,%0" + : "=&r" (r) : "rm" (x), "rm" (-1)); + return r+1; +} + /** * hweightN - returns the hamming weight of a N-bit word * @x: the word to weigh @@ -499,11 +450,6 @@ static __inline__ int ffs(int x) #define minix_find_first_zero_bit(addr,size) \ find_first_zero_bit((void*)addr,size) -/* find last set bit */ -#define fls(x) generic_fls(x) - -#define ARCH_HAS_ATOMIC_UNSIGNED 1 - #endif /* __KERNEL__ */ #endif /* _X86_64_BITOPS_H */