1/* $NetBSD: atomic.h,v 1.1 2002/10/19 12:22:34 bsh Exp $ */ 2 3/*- 4 * Copyright (C) 2003-2004 Olivier Houchard 5 * Copyright (C) 1994-1997 Mark Brinicombe 6 * Copyright (C) 1994 Brini 7 * All rights reserved. 8 * 9 * This code is derived from software written for Brini by Mark Brinicombe 10 * 11 * Redistribution and use in source and binary forms, with or without 12 * modification, are permitted provided that the following conditions 13 * are met: 14 * 1. Redistributions of source code must retain the above copyright 15 * notice, this list of conditions and the following disclaimer. 16 * 2. Redistributions in binary form must reproduce the above copyright 17 * notice, this list of conditions and the following disclaimer in the 18 * documentation and/or other materials provided with the distribution. 19 * 3. All advertising materials mentioning features or use of this software 20 * must display the following acknowledgement: 21 * This product includes software developed by Brini. 22 * 4. The name of Brini may not be used to endorse or promote products 23 * derived from this software without specific prior written permission. 24 * 25 * THIS SOFTWARE IS PROVIDED BY BRINI ``AS IS'' AND ANY EXPRESS OR 26 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 27 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 28 * IN NO EVENT SHALL BRINI BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 29 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 30 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 31 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 32 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 33 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF 34 * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 35 * 36 * $FreeBSD$ 37 */ 38 39#ifndef _MACHINE_ATOMIC_V4_H_ 40#define _MACHINE_ATOMIC_V4_H_ 41 42#ifndef _MACHINE_ATOMIC_H_ 43#error Do not include this file directly, use <machine/atomic.h> 44#endif 45 46#if __ARM_ARCH <= 5 47#define isb() __asm __volatile("mcr p15, 0, %0, c7, c5, 4" : : "r" (0) : "memory") 48#define dsb() __asm __volatile("mcr p15, 0, %0, c7, c10, 4" : : "r" (0) : "memory") 49#define dmb() dsb() 50#else 51#error Only use this file with ARMv5 and earlier 52#endif 53 54#define mb() dmb() 55#define wmb() dmb() 56#define rmb() dmb() 57 58#define __with_interrupts_disabled(expr) \ 59 do { \ 60 u_int cpsr_save, tmp; \ 61 \ 62 __asm __volatile( \ 63 "mrs %0, cpsr;" \ 64 "orr %1, %0, %2;" \ 65 "msr cpsr_fsxc, %1;" \ 66 : "=r" (cpsr_save), "=r" (tmp) \ 67 : "I" (PSR_I | PSR_F) \ 68 : "cc" ); \ 69 (expr); \ 70 __asm __volatile( \ 71 "msr cpsr_fsxc, %0" \ 72 : /* no output */ \ 73 : "r" (cpsr_save) \ 74 : "cc" ); \ 75 } while(0) 76 77static __inline uint32_t 78__swp(uint32_t val, volatile uint32_t *ptr) 79{ 80 __asm __volatile("swp %0, %2, [%3]" 81 : "=&r" (val), "=m" (*ptr) 82 : "r" (val), "r" (ptr), "m" (*ptr) 83 : "memory"); 84 return (val); 85} 86 87 88#ifdef _KERNEL 89#define ARM_HAVE_ATOMIC64 90 91static __inline void 92atomic_add_32(volatile u_int32_t *p, u_int32_t val) 93{ 94 __with_interrupts_disabled(*p += val); 95} 96 97static __inline void 98atomic_add_64(volatile u_int64_t *p, u_int64_t val) 99{ 100 __with_interrupts_disabled(*p += val); 101} 102 103static __inline void 104atomic_clear_32(volatile uint32_t *address, uint32_t clearmask) 105{ 106 __with_interrupts_disabled(*address &= ~clearmask); 107} 108 109static __inline void 110atomic_clear_64(volatile uint64_t *address, uint64_t clearmask) 111{ 112 __with_interrupts_disabled(*address &= ~clearmask); 113} 114 115static __inline u_int32_t 116atomic_cmpset_32(volatile u_int32_t *p, volatile u_int32_t cmpval, volatile u_int32_t newval) 117{ 118 int ret; 119 120 __with_interrupts_disabled( 121 { 122 if (*p == cmpval) { 123 *p = newval; 124 ret = 1; 125 } else { 126 ret = 0; 127 } 128 }); 129 return (ret); 130} 131 132static __inline u_int64_t 133atomic_cmpset_64(volatile u_int64_t *p, volatile u_int64_t cmpval, volatile u_int64_t newval) 134{ 135 int ret; 136 137 __with_interrupts_disabled( 138 { 139 if (*p == cmpval) { 140 *p = newval; 141 ret = 1; 142 } else { 143 ret = 0; 144 } 145 }); 146 return (ret); 147} 148 149 150static __inline uint32_t 151atomic_fetchadd_32(volatile uint32_t *p, uint32_t v) 152{ 153 uint32_t value; 154 155 __with_interrupts_disabled( 156 { 157 value = *p; 158 *p += v; 159 }); 160 return (value); 161} 162 163static __inline uint64_t 164atomic_fetchadd_64(volatile uint64_t *p, uint64_t v) 165{ 166 uint64_t value; 167 168 __with_interrupts_disabled( 169 { 170 value = *p; 171 *p += v; 172 }); 173 return (value); 174} 175 176static __inline uint64_t 177atomic_load_64(volatile uint64_t *p) 178{ 179 uint64_t value; 180 181 __with_interrupts_disabled(value = *p); 182 return (value); 183} 184 185static __inline void 186atomic_set_32(volatile uint32_t *address, uint32_t setmask) 187{ 188 __with_interrupts_disabled(*address |= setmask); 189} 190 191static __inline void 192atomic_set_64(volatile uint64_t *address, uint64_t setmask) 193{ 194 __with_interrupts_disabled(*address |= setmask); 195} 196 197static __inline void 198atomic_store_64(volatile uint64_t *p, uint64_t value) 199{ 200 __with_interrupts_disabled(*p = value); 201} 202 203static __inline void 204atomic_subtract_32(volatile u_int32_t *p, u_int32_t val) 205{ 206 __with_interrupts_disabled(*p -= val); 207} 208 209static __inline void 210atomic_subtract_64(volatile u_int64_t *p, u_int64_t val) 211{ 212 __with_interrupts_disabled(*p -= val); 213} 214 215#else /* !_KERNEL */ 216 217static __inline void 218atomic_add_32(volatile u_int32_t *p, u_int32_t val) 219{ 220 int start, ras_start = ARM_RAS_START; 221 222 __asm __volatile("1:\n" 223 "adr %1, 1b\n" 224 "str %1, [%0]\n" 225 "adr %1, 2f\n" 226 "str %1, [%0, #4]\n" 227 "ldr %1, [%2]\n" 228 "add %1, %1, %3\n" 229 "str %1, [%2]\n" 230 "2:\n" 231 "mov %1, #0\n" 232 "str %1, [%0]\n" 233 "mov %1, #0xffffffff\n" 234 "str %1, [%0, #4]\n" 235 : "+r" (ras_start), "=r" (start), "+r" (p), "+r" (val) 236 : : "memory"); 237} 238 239static __inline void 240atomic_clear_32(volatile uint32_t *address, uint32_t clearmask) 241{ 242 int start, ras_start = ARM_RAS_START; 243 244 __asm __volatile("1:\n" 245 "adr %1, 1b\n" 246 "str %1, [%0]\n" 247 "adr %1, 2f\n" 248 "str %1, [%0, #4]\n" 249 "ldr %1, [%2]\n" 250 "bic %1, %1, %3\n" 251 "str %1, [%2]\n" 252 "2:\n" 253 "mov %1, #0\n" 254 "str %1, [%0]\n" 255 "mov %1, #0xffffffff\n" 256 "str %1, [%0, #4]\n" 257 : "+r" (ras_start), "=r" (start), "+r" (address), "+r" (clearmask) 258 : : "memory"); 259 260} 261 262static __inline u_int32_t 263atomic_cmpset_32(volatile u_int32_t *p, volatile u_int32_t cmpval, volatile u_int32_t newval) 264{ 265 register int done, ras_start = ARM_RAS_START; 266 267 __asm __volatile("1:\n" 268 "adr %1, 1b\n" 269 "str %1, [%0]\n" 270 "adr %1, 2f\n" 271 "str %1, [%0, #4]\n" 272 "ldr %1, [%2]\n" 273 "cmp %1, %3\n" 274 "streq %4, [%2]\n" 275 "2:\n" 276 "mov %1, #0\n" 277 "str %1, [%0]\n" 278 "mov %1, #0xffffffff\n" 279 "str %1, [%0, #4]\n" 280 "moveq %1, #1\n" 281 "movne %1, #0\n" 282 : "+r" (ras_start), "=r" (done) 283 ,"+r" (p), "+r" (cmpval), "+r" (newval) : : "cc", "memory"); 284 return (done); 285} 286 287static __inline uint32_t 288atomic_fetchadd_32(volatile uint32_t *p, uint32_t v) 289{ 290 uint32_t start, tmp, ras_start = ARM_RAS_START; 291 292 __asm __volatile("1:\n" 293 "adr %1, 1b\n" 294 "str %1, [%0]\n" 295 "adr %1, 2f\n" 296 "str %1, [%0, #4]\n" 297 "ldr %1, [%3]\n" 298 "mov %2, %1\n" 299 "add %2, %2, %4\n" 300 "str %2, [%3]\n" 301 "2:\n" 302 "mov %2, #0\n" 303 "str %2, [%0]\n" 304 "mov %2, #0xffffffff\n" 305 "str %2, [%0, #4]\n" 306 : "+r" (ras_start), "=r" (start), "=r" (tmp), "+r" (p), "+r" (v) 307 : : "memory"); 308 return (start); 309} 310 311static __inline void 312atomic_set_32(volatile uint32_t *address, uint32_t setmask) 313{ 314 int start, ras_start = ARM_RAS_START; 315 316 __asm __volatile("1:\n" 317 "adr %1, 1b\n" 318 "str %1, [%0]\n" 319 "adr %1, 2f\n" 320 "str %1, [%0, #4]\n" 321 "ldr %1, [%2]\n" 322 "orr %1, %1, %3\n" 323 "str %1, [%2]\n" 324 "2:\n" 325 "mov %1, #0\n" 326 "str %1, [%0]\n" 327 "mov %1, #0xffffffff\n" 328 "str %1, [%0, #4]\n" 329 330 : "+r" (ras_start), "=r" (start), "+r" (address), "+r" (setmask) 331 : : "memory"); 332} 333 334static __inline void 335atomic_subtract_32(volatile u_int32_t *p, u_int32_t val) 336{ 337 int start, ras_start = ARM_RAS_START; 338 339 __asm __volatile("1:\n" 340 "adr %1, 1b\n" 341 "str %1, [%0]\n" 342 "adr %1, 2f\n" 343 "str %1, [%0, #4]\n" 344 "ldr %1, [%2]\n" 345 "sub %1, %1, %3\n" 346 "str %1, [%2]\n" 347 "2:\n" 348 "mov %1, #0\n" 349 "str %1, [%0]\n" 350 "mov %1, #0xffffffff\n" 351 "str %1, [%0, #4]\n" 352 353 : "+r" (ras_start), "=r" (start), "+r" (p), "+r" (val) 354 : : "memory"); 355} 356 357#endif /* _KERNEL */ 358 359static __inline uint32_t 360atomic_readandclear_32(volatile u_int32_t *p) 361{ 362 363 return (__swp(0, p)); 364} 365 366static __inline uint32_t 367atomic_swap_32(volatile u_int32_t *p, u_int32_t v) 368{ 369 370 return (__swp(v, p)); 371} 372 373#define atomic_cmpset_rel_32 atomic_cmpset_32 374#define atomic_cmpset_acq_32 atomic_cmpset_32 375#define atomic_cmpset_rel_64 atomic_cmpset_64 376#define atomic_cmpset_acq_64 atomic_cmpset_64 377#define atomic_set_rel_32 atomic_set_32 378#define atomic_set_acq_32 atomic_set_32 379#define atomic_clear_rel_32 atomic_clear_32 380#define atomic_clear_acq_32 atomic_clear_32 381#define atomic_add_rel_32 atomic_add_32 382#define atomic_add_acq_32 atomic_add_32 383#define atomic_subtract_rel_32 atomic_subtract_32 384#define atomic_subtract_acq_32 atomic_subtract_32 385#define atomic_store_rel_32 atomic_store_32 386#define atomic_store_rel_long atomic_store_long 387#define atomic_load_acq_32 atomic_load_32 388#define atomic_load_acq_long atomic_load_long 389#define atomic_add_acq_long atomic_add_long 390#define atomic_add_rel_long atomic_add_long 391#define atomic_subtract_acq_long atomic_subtract_long 392#define atomic_subtract_rel_long atomic_subtract_long 393#define atomic_clear_acq_long atomic_clear_long 394#define atomic_clear_rel_long atomic_clear_long 395#define atomic_set_acq_long atomic_set_long 396#define atomic_set_rel_long atomic_set_long 397#define atomic_cmpset_acq_long atomic_cmpset_long 398#define atomic_cmpset_rel_long atomic_cmpset_long 399#define atomic_load_acq_long atomic_load_long 400#undef __with_interrupts_disabled 401 402static __inline void 403atomic_add_long(volatile u_long *p, u_long v) 404{ 405 406 atomic_add_32((volatile uint32_t *)p, v); 407} 408 409static __inline void 410atomic_clear_long(volatile u_long *p, u_long v) 411{ 412 413 atomic_clear_32((volatile uint32_t *)p, v); 414} 415 416static __inline int 417atomic_cmpset_long(volatile u_long *dst, u_long old, u_long newe) 418{ 419 420 return (atomic_cmpset_32((volatile uint32_t *)dst, old, newe)); 421} 422 423static __inline u_long 424atomic_fetchadd_long(volatile u_long *p, u_long v) 425{ 426 427 return (atomic_fetchadd_32((volatile uint32_t *)p, v)); 428} 429 430static __inline void 431atomic_readandclear_long(volatile u_long *p) 432{ 433 434 atomic_readandclear_32((volatile uint32_t *)p); 435} 436 437static __inline void 438atomic_set_long(volatile u_long *p, u_long v) 439{ 440 441 atomic_set_32((volatile uint32_t *)p, v); 442} 443 444static __inline void 445atomic_subtract_long(volatile u_long *p, u_long v) 446{ 447 448 atomic_subtract_32((volatile uint32_t *)p, v); 449} 450 451/* 452 * ARMv5 does not support SMP. For both kernel and user modes, only a 453 * compiler barrier is needed for fences, since CPU is always 454 * self-consistent. 455 */ 456static __inline void 457atomic_thread_fence_acq(void) 458{ 459 460 __compiler_membar(); 461} 462 463static __inline void 464atomic_thread_fence_rel(void) 465{ 466 467 __compiler_membar(); 468} 469 470static __inline void 471atomic_thread_fence_acq_rel(void) 472{ 473 474 __compiler_membar(); 475} 476 477static __inline void 478atomic_thread_fence_seq_cst(void) 479{ 480 481 __compiler_membar(); 482} 483 484#endif /* _MACHINE_ATOMIC_H_ */ 485