1/* Modified by Broadcom Corp. Portions Copyright (c) Broadcom Corp, 2012. */ 2/* 3 * Copyright 1995, Russell King. 4 * Various bits and pieces copyrights include: 5 * Linus Torvalds (test_bit). 6 * Big endian support: Copyright 2001, Nicolas Pitre 7 * reworked by rmk. 8 * 9 * bit 0 is the LSB of an "unsigned long" quantity. 10 * 11 * Please note that the code in this file should never be included 12 * from user space. Many of these are not implemented in assembler 13 * since they would be too costly. Also, they require privileged 14 * instructions (which are not available from user mode) to ensure 15 * that they are atomic. 16 */ 17 18#ifndef __ASM_ARM_BITOPS_H 19#define __ASM_ARM_BITOPS_H 20 21#ifdef __KERNEL__ 22 23#ifndef _LINUX_BITOPS_H 24#error only <linux/bitops.h> can be included directly 25#endif 26 27#include <linux/compiler.h> 28#include <asm/system.h> 29 30#define smp_mb__before_clear_bit() mb() 31#define smp_mb__after_clear_bit() mb() 32 33#if defined(CONFIG_BUZZZ_FUNC) 34#ifndef __always_inline__ 35#define __always_inline__ inline __attribute__((always_inline)) __attribute__((no_instrument_function)) 36#endif 37#else /* !CONFIG_BUZZZ_FUNC */ 38#ifndef __always_inline__ 39#define __always_inline__ inline 40#endif 41#endif /* !CONFIG_BUZZZ_FUNC */ 42 43/* 44 * These functions are the basis of our bit ops. 45 * 46 * First, the atomic bitops. These use native endian. 47 */ 48static __always_inline__ void 49____atomic_set_bit(unsigned int bit, volatile unsigned long *p) 50{ 51 unsigned long flags; 52 unsigned long mask = 1UL << (bit & 31); 53 54 p += bit >> 5; 55 56 raw_local_irq_save(flags); 57 *p |= mask; 58 raw_local_irq_restore(flags); 59} 60 61static __always_inline__ void 62____atomic_clear_bit(unsigned int bit, volatile unsigned long *p) 63{ 64 unsigned long flags; 65 unsigned long mask = 1UL << (bit & 31); 66 67 p += bit >> 5; 68 69 raw_local_irq_save(flags); 70 *p &= ~mask; 71 raw_local_irq_restore(flags); 72} 73 74static __always_inline__ void 75____atomic_change_bit(unsigned int bit, volatile unsigned long *p) 76{ 77 unsigned long flags; 78 unsigned long mask = 1UL << (bit & 31); 79 80 p += bit >> 5; 81 82 raw_local_irq_save(flags); 83 *p ^= mask; 84 raw_local_irq_restore(flags); 85} 86 87static __always_inline__ int 88____atomic_test_and_set_bit(unsigned int bit, volatile unsigned long *p) 89{ 90 unsigned long flags; 91 unsigned int res; 92 unsigned long mask = 1UL << (bit & 31); 93 94 p += bit >> 5; 95 96 raw_local_irq_save(flags); 97 res = *p; 98 *p = res | mask; 99 raw_local_irq_restore(flags); 100 101 return (res & mask) != 0; 102} 103 104static __always_inline__ int 105____atomic_test_and_clear_bit(unsigned int bit, volatile unsigned long *p) 106{ 107 unsigned long flags; 108 unsigned int res; 109 unsigned long mask = 1UL << (bit & 31); 110 111 p += bit >> 5; 112 113 raw_local_irq_save(flags); 114 res = *p; 115 *p = res & ~mask; 116 raw_local_irq_restore(flags); 117 118 return (res & mask) != 0; 119} 120 121static __always_inline__ int 122____atomic_test_and_change_bit(unsigned int bit, volatile unsigned long *p) 123{ 124 unsigned long flags; 125 unsigned int res; 126 unsigned long mask = 1UL << (bit & 31); 127 128 p += bit >> 5; 129 130 raw_local_irq_save(flags); 131 res = *p; 132 *p = res ^ mask; 133 raw_local_irq_restore(flags); 134 135 return (res & mask) != 0; 136} 137 138#include <asm-generic/bitops/non-atomic.h> 139 140/* 141 * A note about Endian-ness. 142 * ------------------------- 143 * 144 * When the ARM is put into big endian mode via CR15, the processor 145 * merely swaps the order of bytes within words, thus: 146 * 147 * ------------ physical data bus bits ----------- 148 * D31 ... D24 D23 ... D16 D15 ... D8 D7 ... D0 149 * little byte 3 byte 2 byte 1 byte 0 150 * big byte 0 byte 1 byte 2 byte 3 151 * 152 * This means that reading a 32-bit word at address 0 returns the same 153 * value irrespective of the endian mode bit. 154 * 155 * Peripheral devices should be connected with the data bus reversed in 156 * "Big Endian" mode. ARM Application Note 61 is applicable, and is 157 * available from http://www.arm.com/. 158 * 159 * The following assumes that the data bus connectivity for big endian 160 * mode has been followed. 161 * 162 * Note that bit 0 is defined to be 32-bit word bit 0, not byte 0 bit 0. 163 */ 164 165/* 166 * Little endian assembly bitops. nr = 0 -> byte 0 bit 0. 167 */ 168extern void _set_bit_le(int nr, volatile unsigned long * p); 169extern void _clear_bit_le(int nr, volatile unsigned long * p); 170extern void _change_bit_le(int nr, volatile unsigned long * p); 171extern int _test_and_set_bit_le(int nr, volatile unsigned long * p); 172extern int _test_and_clear_bit_le(int nr, volatile unsigned long * p); 173extern int _test_and_change_bit_le(int nr, volatile unsigned long * p); 174extern int _find_first_zero_bit_le(const void * p, unsigned size); 175extern int _find_next_zero_bit_le(const void * p, int size, int offset); 176extern int _find_first_bit_le(const unsigned long *p, unsigned size); 177extern int _find_next_bit_le(const unsigned long *p, int size, int offset); 178 179/* 180 * Big endian assembly bitops. nr = 0 -> byte 3 bit 0. 181 */ 182extern void _set_bit_be(int nr, volatile unsigned long * p); 183extern void _clear_bit_be(int nr, volatile unsigned long * p); 184extern void _change_bit_be(int nr, volatile unsigned long * p); 185extern int _test_and_set_bit_be(int nr, volatile unsigned long * p); 186extern int _test_and_clear_bit_be(int nr, volatile unsigned long * p); 187extern int _test_and_change_bit_be(int nr, volatile unsigned long * p); 188extern int _find_first_zero_bit_be(const void * p, unsigned size); 189extern int _find_next_zero_bit_be(const void * p, int size, int offset); 190extern int _find_first_bit_be(const unsigned long *p, unsigned size); 191extern int _find_next_bit_be(const unsigned long *p, int size, int offset); 192 193#ifndef CONFIG_SMP 194/* 195 * The __* form of bitops are non-atomic and may be reordered. 196 */ 197#define ATOMIC_BITOP_LE(name,nr,p) \ 198 (__builtin_constant_p(nr) ? \ 199 ____atomic_##name(nr, p) : \ 200 _##name##_le(nr,p)) 201 202#define ATOMIC_BITOP_BE(name,nr,p) \ 203 (__builtin_constant_p(nr) ? \ 204 ____atomic_##name(nr, p) : \ 205 _##name##_be(nr,p)) 206#else 207#define ATOMIC_BITOP_LE(name,nr,p) _##name##_le(nr,p) 208#define ATOMIC_BITOP_BE(name,nr,p) _##name##_be(nr,p) 209#endif 210 211#define NONATOMIC_BITOP(name,nr,p) \ 212 (____nonatomic_##name(nr, p)) 213 214#ifndef __ARMEB__ 215/* 216 * These are the little endian, atomic definitions. 217 */ 218#define set_bit(nr,p) ATOMIC_BITOP_LE(set_bit,nr,p) 219#define clear_bit(nr,p) ATOMIC_BITOP_LE(clear_bit,nr,p) 220#define change_bit(nr,p) ATOMIC_BITOP_LE(change_bit,nr,p) 221#define test_and_set_bit(nr,p) ATOMIC_BITOP_LE(test_and_set_bit,nr,p) 222#define test_and_clear_bit(nr,p) ATOMIC_BITOP_LE(test_and_clear_bit,nr,p) 223#define test_and_change_bit(nr,p) ATOMIC_BITOP_LE(test_and_change_bit,nr,p) 224#define find_first_zero_bit(p,sz) _find_first_zero_bit_le(p,sz) 225#define find_next_zero_bit(p,sz,off) _find_next_zero_bit_le(p,sz,off) 226#define find_first_bit(p,sz) _find_first_bit_le(p,sz) 227#define find_next_bit(p,sz,off) _find_next_bit_le(p,sz,off) 228 229#define WORD_BITOFF_TO_LE(x) ((x)) 230 231#else 232 233/* 234 * These are the big endian, atomic definitions. 235 */ 236#define set_bit(nr,p) ATOMIC_BITOP_BE(set_bit,nr,p) 237#define clear_bit(nr,p) ATOMIC_BITOP_BE(clear_bit,nr,p) 238#define change_bit(nr,p) ATOMIC_BITOP_BE(change_bit,nr,p) 239#define test_and_set_bit(nr,p) ATOMIC_BITOP_BE(test_and_set_bit,nr,p) 240#define test_and_clear_bit(nr,p) ATOMIC_BITOP_BE(test_and_clear_bit,nr,p) 241#define test_and_change_bit(nr,p) ATOMIC_BITOP_BE(test_and_change_bit,nr,p) 242#define find_first_zero_bit(p,sz) _find_first_zero_bit_be(p,sz) 243#define find_next_zero_bit(p,sz,off) _find_next_zero_bit_be(p,sz,off) 244#define find_first_bit(p,sz) _find_first_bit_be(p,sz) 245#define find_next_bit(p,sz,off) _find_next_bit_be(p,sz,off) 246 247#define WORD_BITOFF_TO_LE(x) ((x) ^ 0x18) 248 249#endif 250 251#if __LINUX_ARM_ARCH__ < 5 252 253#include <asm-generic/bitops/ffz.h> 254#include <asm-generic/bitops/__fls.h> 255#include <asm-generic/bitops/__ffs.h> 256#include <asm-generic/bitops/fls.h> 257#include <asm-generic/bitops/ffs.h> 258 259#else 260 261static inline int constant_fls(int x) 262{ 263 int r = 32; 264 265 if (!x) 266 return 0; 267 if (!(x & 0xffff0000u)) { 268 x <<= 16; 269 r -= 16; 270 } 271 if (!(x & 0xff000000u)) { 272 x <<= 8; 273 r -= 8; 274 } 275 if (!(x & 0xf0000000u)) { 276 x <<= 4; 277 r -= 4; 278 } 279 if (!(x & 0xc0000000u)) { 280 x <<= 2; 281 r -= 2; 282 } 283 if (!(x & 0x80000000u)) { 284 x <<= 1; 285 r -= 1; 286 } 287 return r; 288} 289 290/* 291 * On ARMv5 and above those functions can be implemented around 292 * the clz instruction for much better code efficiency. 293 */ 294 295static inline int fls(int x) 296{ 297 int ret; 298 299 if (__builtin_constant_p(x)) 300 return constant_fls(x); 301 302 asm("clz\t%0, %1" : "=r" (ret) : "r" (x) : "cc"); 303 ret = 32 - ret; 304 return ret; 305} 306 307#define __fls(x) (fls(x) - 1) 308#define ffs(x) ({ unsigned long __t = (x); fls(__t & -__t); }) 309#define __ffs(x) (ffs(x) - 1) 310#define ffz(x) __ffs( ~(x) ) 311 312#endif 313 314#include <asm-generic/bitops/fls64.h> 315 316#include <asm-generic/bitops/sched.h> 317#include <asm-generic/bitops/hweight.h> 318#include <asm-generic/bitops/lock.h> 319 320/* 321 * Ext2 is defined to use little-endian byte ordering. 322 * These do not need to be atomic. 323 */ 324#define ext2_set_bit(nr,p) \ 325 __test_and_set_bit(WORD_BITOFF_TO_LE(nr), (unsigned long *)(p)) 326#define ext2_set_bit_atomic(lock,nr,p) \ 327 test_and_set_bit(WORD_BITOFF_TO_LE(nr), (unsigned long *)(p)) 328#define ext2_clear_bit(nr,p) \ 329 __test_and_clear_bit(WORD_BITOFF_TO_LE(nr), (unsigned long *)(p)) 330#define ext2_clear_bit_atomic(lock,nr,p) \ 331 test_and_clear_bit(WORD_BITOFF_TO_LE(nr), (unsigned long *)(p)) 332#define ext2_test_bit(nr,p) \ 333 test_bit(WORD_BITOFF_TO_LE(nr), (unsigned long *)(p)) 334#define ext2_find_first_zero_bit(p,sz) \ 335 _find_first_zero_bit_le(p,sz) 336#define ext2_find_next_zero_bit(p,sz,off) \ 337 _find_next_zero_bit_le(p,sz,off) 338#define ext2_find_next_bit(p, sz, off) \ 339 _find_next_bit_le(p, sz, off) 340 341/* 342 * Minix is defined to use little-endian byte ordering. 343 * These do not need to be atomic. 344 */ 345#define minix_set_bit(nr,p) \ 346 __set_bit(WORD_BITOFF_TO_LE(nr), (unsigned long *)(p)) 347#define minix_test_bit(nr,p) \ 348 test_bit(WORD_BITOFF_TO_LE(nr), (unsigned long *)(p)) 349#define minix_test_and_set_bit(nr,p) \ 350 __test_and_set_bit(WORD_BITOFF_TO_LE(nr), (unsigned long *)(p)) 351#define minix_test_and_clear_bit(nr,p) \ 352 __test_and_clear_bit(WORD_BITOFF_TO_LE(nr), (unsigned long *)(p)) 353#define minix_find_first_zero_bit(p,sz) \ 354 _find_first_zero_bit_le(p,sz) 355 356#endif /* __KERNEL__ */ 357 358#endif /* _ARM_BITOPS_H */ 359