1/*- 2 * Copyright (c) 2013 Andrew Turner <andrew@freebsd.org> 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24 * SUCH DAMAGE. 25 * 26 * $FreeBSD$ 27 */ 28 29#ifndef _MACHINE_ATOMIC_H_ 30#define _MACHINE_ATOMIC_H_ 31 32#define isb() __asm __volatile("isb" : : : "memory") 33 34/* 35 * Options for DMB and DSB: 36 * oshld Outer Shareable, load 37 * oshst Outer Shareable, store 38 * osh Outer Shareable, all 39 * nshld Non-shareable, load 40 * nshst Non-shareable, store 41 * nsh Non-shareable, all 42 * ishld Inner Shareable, load 43 * ishst Inner Shareable, store 44 * ish Inner Shareable, all 45 * ld Full system, load 46 * st Full system, store 47 * sy Full system, all 48 */ 49#define dsb(opt) __asm __volatile("dsb " __STRING(opt) : : : "memory") 50#define dmb(opt) __asm __volatile("dmb " __STRING(opt) : : : "memory") 51 52#define mb() dmb(sy) /* Full system memory barrier all */ 53#define wmb() dmb(st) /* Full system memory barrier store */ 54#define rmb() dmb(ld) /* Full system memory barrier load */ 55 56#define ATOMIC_OP(op, asm_op, bar, a, l) \ 57static __inline void \ 58atomic_##op##_##bar##32(volatile uint32_t *p, uint32_t val) \ 59{ \ 60 uint32_t tmp; \ 61 int res; \ 62 \ 63 __asm __volatile( \ 64 "1: ld"#a"xr %w0, [%2] \n" \ 65 " "#asm_op" %w0, %w0, %w3 \n" \ 66 " st"#l"xr %w1, %w0, [%2] \n" \ 67 " cbnz %w1, 1b \n" \ 68 : "=&r"(tmp), "=&r"(res) \ 69 : "r" (p), "r" (val) \ 70 : "memory" \ 71 ); \ 72} \ 73 \ 74static __inline void \ 75atomic_##op##_##bar##64(volatile uint64_t *p, uint64_t val) \ 76{ \ 77 uint64_t tmp; \ 78 int res; \ 79 \ 80 __asm __volatile( \ 81 "1: ld"#a"xr %0, [%2] \n" \ 82 " "#asm_op" %0, %0, %3 \n" \ 83 " st"#l"xr %w1, %0, [%2] \n" \ 84 " cbnz %w1, 1b \n" \ 85 : "=&r"(tmp), "=&r"(res) \ 86 : "r" (p), "r" (val) \ 87 : "memory" \ 88 ); \ 89} 90 91#define ATOMIC(op, asm_op) \ 92 ATOMIC_OP(op, asm_op, , , ) \ 93 ATOMIC_OP(op, asm_op, acq_, a, ) \ 94 ATOMIC_OP(op, asm_op, rel_, , l) \ 95 96ATOMIC(add, add) 97ATOMIC(clear, bic) 98ATOMIC(set, orr) 99ATOMIC(subtract, sub) 100 101#define ATOMIC_CMPSET(bar, a, l) \ 102static __inline int \ 103atomic_cmpset_##bar##32(volatile uint32_t *p, uint32_t cmpval, \ 104 uint32_t newval) \ 105{ \ 106 uint32_t tmp; \ 107 int res; \ 108 \ 109 __asm __volatile( \ 110 "1: mov %w1, #1 \n" \ 111 " ld"#a"xr %w0, [%2] \n" \ 112 " cmp %w0, %w3 \n" \ 113 " b.ne 2f \n" \ 114 " st"#l"xr %w1, %w4, [%2] \n" \ 115 " cbnz %w1, 1b \n" \ 116 "2:" \ 117 : "=&r"(tmp), "=&r"(res) \ 118 : "r" (p), "r" (cmpval), "r" (newval) \ 119 : "cc", "memory" \ 120 ); \ 121 \ 122 return (!res); \ 123} \ 124 \ 125static __inline int \ 126atomic_cmpset_##bar##64(volatile uint64_t *p, uint64_t cmpval, \ 127 uint64_t newval) \ 128{ \ 129 uint64_t tmp; \ 130 int res; \ 131 \ 132 __asm __volatile( \ 133 "1: mov %w1, #1 \n" \ 134 " ld"#a"xr %0, [%2] \n" \ 135 " cmp %0, %3 \n" \ 136 " b.ne 2f \n" \ 137 " st"#l"xr %w1, %4, [%2] \n" \ 138 " cbnz %w1, 1b \n" \ 139 "2:" \ 140 : "=&r"(tmp), "=&r"(res) \ 141 : "r" (p), "r" (cmpval), "r" (newval) \ 142 : "cc", "memory" \ 143 ); \ 144 \ 145 return (!res); \ 146} 147 148ATOMIC_CMPSET( , , ) 149ATOMIC_CMPSET(acq_, a, ) 150ATOMIC_CMPSET(rel_, ,l) 151 152static __inline uint32_t 153atomic_fetchadd_32(volatile uint32_t *p, uint32_t val) 154{ 155 uint32_t tmp, ret; 156 int res; 157 158 __asm __volatile( 159 "1: ldxr %w2, [%3] \n" 160 " add %w0, %w2, %w4 \n" 161 " stxr %w1, %w0, [%3] \n" 162 " cbnz %w1, 1b \n" 163 : "=&r"(tmp), "=&r"(res), "=&r"(ret) 164 : "r" (p), "r" (val) 165 : "memory" 166 ); 167 168 return (ret); 169} 170 171static __inline uint64_t 172atomic_fetchadd_64(volatile uint64_t *p, uint64_t val) 173{ 174 uint64_t tmp, ret; 175 int res; 176 177 __asm __volatile( 178 "1: ldxr %2, [%3] \n" 179 " add %0, %2, %4 \n" 180 " stxr %w1, %0, [%3] \n" 181 " cbnz %w1, 1b \n" 182 : "=&r"(tmp), "=&r"(res), "=&r"(ret) 183 : "r" (p), "r" (val) 184 : "memory" 185 ); 186 187 return (ret); 188} 189 190static __inline uint32_t 191atomic_readandclear_32(volatile uint32_t *p) 192{ 193 uint32_t ret; 194 int res; 195 196 __asm __volatile( 197 "1: ldxr %w1, [%2] \n" 198 " stxr %w0, wzr, [%2] \n" 199 " cbnz %w0, 1b \n" 200 : "=&r"(res), "=&r"(ret) 201 : "r" (p) 202 : "memory" 203 ); 204 205 return (ret); 206} 207 208static __inline uint64_t 209atomic_readandclear_64(volatile uint64_t *p) 210{ 211 uint64_t ret; 212 int res; 213 214 __asm __volatile( 215 "1: ldxr %1, [%2] \n" 216 " stxr %w0, xzr, [%2] \n" 217 " cbnz %w0, 1b \n" 218 : "=&r"(res), "=&r"(ret) 219 : "r" (p) 220 : "memory" 221 ); 222 223 return (ret); 224} 225 226static __inline uint32_t 227atomic_swap_32(volatile uint32_t *p, uint32_t val) 228{ 229 uint32_t ret; 230 int res; 231 232 __asm __volatile( 233 "1: ldxr %w0, [%2] \n" 234 " stxr %w1, %w3, [%2] \n" 235 " cbnz %w1, 1b \n" 236 : "=&r"(ret), "=&r"(res) 237 : "r" (p), "r" (val) 238 : "memory" 239 ); 240 241 return (ret); 242} 243 244static __inline uint64_t 245atomic_swap_64(volatile uint64_t *p, uint64_t val) 246{ 247 uint64_t ret; 248 int res; 249 250 __asm __volatile( 251 "1: ldxr %0, [%2] \n" 252 " stxr %w1, %3, [%2] \n" 253 " cbnz %w1, 1b \n" 254 : "=&r"(ret), "=&r"(res) 255 : "r" (p), "r" (val) 256 : "memory" 257 ); 258 259 return (ret); 260} 261 262static __inline uint32_t 263atomic_load_acq_32(volatile uint32_t *p) 264{ 265 uint32_t ret; 266 267 __asm __volatile( 268 "ldar %w0, [%1] \n" 269 : "=&r" (ret) 270 : "r" (p) 271 : "memory"); 272 273 return (ret); 274} 275 276static __inline uint64_t 277atomic_load_acq_64(volatile uint64_t *p) 278{ 279 uint64_t ret; 280 281 __asm __volatile( 282 "ldar %0, [%1] \n" 283 : "=&r" (ret) 284 : "r" (p) 285 : "memory"); 286 287 return (ret); 288} 289 290static __inline void 291atomic_store_rel_32(volatile uint32_t *p, uint32_t val) 292{ 293 294 __asm __volatile( 295 "stlr %w0, [%1] \n" 296 : 297 : "r" (val), "r" (p) 298 : "memory"); 299} 300 301static __inline void 302atomic_store_rel_64(volatile uint64_t *p, uint64_t val) 303{ 304 305 __asm __volatile( 306 "stlr %0, [%1] \n" 307 : 308 : "r" (val), "r" (p) 309 : "memory"); 310} 311 312 313#define atomic_add_int atomic_add_32 314#define atomic_clear_int atomic_clear_32 315#define atomic_cmpset_int atomic_cmpset_32 316#define atomic_fetchadd_int atomic_fetchadd_32 317#define atomic_readandclear_int atomic_readandclear_32 318#define atomic_set_int atomic_set_32 319#define atomic_swap_int atomic_swap_32 320#define atomic_subtract_int atomic_subtract_32 321 322#define atomic_add_acq_int atomic_add_acq_32 323#define atomic_clear_acq_int atomic_clear_acq_32 324#define atomic_cmpset_acq_int atomic_cmpset_acq_32 325#define atomic_load_acq_int atomic_load_acq_32 326#define atomic_set_acq_int atomic_set_acq_32 327#define atomic_subtract_acq_int atomic_subtract_acq_32 328 329#define atomic_add_rel_int atomic_add_rel_32 330#define atomic_clear_rel_int atomic_add_rel_32 331#define atomic_cmpset_rel_int atomic_cmpset_rel_32 332#define atomic_set_rel_int atomic_set_rel_32 333#define atomic_subtract_rel_int atomic_subtract_rel_32 334#define atomic_store_rel_int atomic_store_rel_32 335 336#define atomic_add_long atomic_add_64 337#define atomic_clear_long atomic_clear_64 338#define atomic_cmpset_long atomic_cmpset_64 339#define atomic_fetchadd_long atomic_fetchadd_64 340#define atomic_readandclear_long atomic_readandclear_64 341#define atomic_set_long atomic_set_64 342#define atomic_swap_long atomic_swap_64 343#define atomic_subtract_long atomic_subtract_64 344 345#define atomic_add_ptr atomic_add_64 346#define atomic_clear_ptr atomic_clear_64 347#define atomic_cmpset_ptr atomic_cmpset_64 348#define atomic_fetchadd_ptr atomic_fetchadd_64 349#define atomic_readandclear_ptr atomic_readandclear_64 350#define atomic_set_ptr atomic_set_64 351#define atomic_swap_ptr atomic_swap_64 352#define atomic_subtract_ptr atomic_subtract_64 353 354#define atomic_add_acq_long atomic_add_acq_64 355#define atomic_clear_acq_long atomic_add_acq_64 356#define atomic_cmpset_acq_long atomic_cmpset_acq_64 357#define atomic_load_acq_long atomic_load_acq_64 358#define atomic_set_acq_long atomic_set_acq_64 359#define atomic_subtract_acq_long atomic_subtract_acq_64 360 361#define atomic_add_acq_ptr atomic_add_acq_64 362#define atomic_clear_acq_ptr atomic_add_acq_64 363#define atomic_cmpset_acq_ptr atomic_cmpset_acq_64 364#define atomic_load_acq_ptr atomic_load_acq_64 365#define atomic_set_acq_ptr atomic_set_acq_64 366#define atomic_subtract_acq_ptr atomic_subtract_acq_64 367 368#define atomic_add_rel_long atomic_add_rel_64 369#define atomic_clear_rel_long atomic_clear_rel_64 370#define atomic_cmpset_rel_long atomic_cmpset_rel_64 371#define atomic_set_rel_long atomic_set_rel_64 372#define atomic_subtract_rel_long atomic_subtract_rel_64 373#define atomic_store_rel_long atomic_store_rel_64 374 375#define atomic_add_rel_ptr atomic_add_rel_64 376#define atomic_clear_rel_ptr atomic_clear_rel_64 377#define atomic_cmpset_rel_ptr atomic_cmpset_rel_64 378#define atomic_set_rel_ptr atomic_set_rel_64 379#define atomic_subtract_rel_ptr atomic_subtract_rel_64 380#define atomic_store_rel_ptr atomic_store_rel_64 381 382static __inline void 383atomic_thread_fence_acq(void) 384{ 385 386 dmb(ld); 387} 388 389static __inline void 390atomic_thread_fence_rel(void) 391{ 392 393 dmb(sy); 394} 395 396static __inline void 397atomic_thread_fence_acq_rel(void) 398{ 399 400 dmb(sy); 401} 402 403static __inline void 404atomic_thread_fence_seq_cst(void) 405{ 406 407 dmb(sy); 408} 409 410#endif /* _MACHINE_ATOMIC_H_ */ 411 412