1/*- 2 * Copyright (c) 2015 Ruslan Bukin <br@bsdpad.com> 3 * All rights reserved. 4 * 5 * Portions of this software were developed by SRI International and the 6 * University of Cambridge Computer Laboratory under DARPA/AFRL contract 7 * FA8750-10-C-0237 ("CTSRD"), as part of the DARPA CRASH research programme. 8 * 9 * Portions of this software were developed by the University of Cambridge 10 * Computer Laboratory as part of the CTSRD Project, with support from the 11 * UK Higher Education Innovation Fund (HEIF). 12 * 13 * Redistribution and use in source and binary forms, with or without 14 * modification, are permitted provided that the following conditions 15 * are met: 16 * 1. Redistributions of source code must retain the above copyright 17 * notice, this list of conditions and the following disclaimer. 18 * 2. Redistributions in binary form must reproduce the above copyright 19 * notice, this list of conditions and the following disclaimer in the 20 * documentation and/or other materials provided with the distribution. 21 * 22 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 23 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 24 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 25 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 26 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 27 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 28 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 29 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 30 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 31 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 32 * SUCH DAMAGE. 33 * 34 * $FreeBSD: stable/11/sys/riscv/include/atomic.h 327195 2017-12-26 10:07:17Z kib $ 35 */ 36 37#ifndef _MACHINE_ATOMIC_H_ 38#define _MACHINE_ATOMIC_H_ 39 40#include <sys/atomic_common.h> 41 42#define fence() __asm __volatile("fence" ::: "memory"); 43#define mb() fence() 44#define rmb() fence() 45#define wmb() fence() 46 47#define ATOMIC_ACQ_REL(NAME, WIDTH) \ 48static __inline void \ 49atomic_##NAME##_acq_##WIDTH(__volatile uint##WIDTH##_t *p, uint##WIDTH##_t v)\ 50{ \ 51 atomic_##NAME##_##WIDTH(p, v); \ 52 fence(); \ 53} \ 54 \ 55static __inline void \ 56atomic_##NAME##_rel_##WIDTH(__volatile uint##WIDTH##_t *p, uint##WIDTH##_t v)\ 57{ \ 58 fence(); \ 59 atomic_##NAME##_##WIDTH(p, v); \ 60} 61 62static __inline void 63atomic_add_32(volatile uint32_t *p, uint32_t val) 64{ 65 66 __asm __volatile("amoadd.w zero, %1, %0" 67 : "+A" (*p) 68 : "r" (val) 69 : "memory"); 70} 71 72static __inline void 73atomic_subtract_32(volatile uint32_t *p, uint32_t val) 74{ 75 76 __asm __volatile("amoadd.w zero, %1, %0" 77 : "+A" (*p) 78 : "r" (-val) 79 : "memory"); 80} 81 82static __inline void 83atomic_set_32(volatile uint32_t *p, uint32_t val) 84{ 85 86 __asm __volatile("amoor.w zero, %1, %0" 87 : "+A" (*p) 88 : "r" (val) 89 : "memory"); 90} 91 92static __inline void 93atomic_clear_32(volatile uint32_t *p, uint32_t val) 94{ 95 96 __asm __volatile("amoand.w zero, %1, %0" 97 : "+A" (*p) 98 : "r" (~val) 99 : "memory"); 100} 101 102static __inline int 103atomic_cmpset_32(volatile uint32_t *p, uint32_t cmpval, uint32_t newval) 104{ 105 uint32_t tmp; 106 int res; 107 108 res = 0; 109 110 __asm __volatile( 111 "0:" 112 "li %1, 1\n" /* Preset to fail */ 113 "lr.w %0, %2\n" 114 "bne %0, %z3, 1f\n" 115 "sc.w %1, %z4, %2\n" 116 "bnez %1, 0b\n" 117 "1:" 118 : "=&r" (tmp), "=&r" (res), "+A" (*p) 119 : "rJ" (cmpval), "rJ" (newval) 120 : "memory"); 121 122 return (!res); 123} 124 125static __inline int 126atomic_fcmpset_32(volatile uint32_t *p, uint32_t *cmpval, uint32_t newval) 127{ 128 uint32_t tmp; 129 int res; 130 131 res = 0; 132 133 __asm __volatile( 134 "0:" 135 "li %1, 1\n" /* Preset to fail */ 136 "lr.w %0, %2\n" /* Load old value */ 137 "bne %0, %z4, 1f\n" /* Compare */ 138 "sc.w %1, %z5, %2\n" /* Try to store new value */ 139 "j 2f\n" 140 "1:" 141 "sw %0, %3\n" /* Save old value */ 142 "2:" 143 : "=&r" (tmp), "=&r" (res), "+A" (*p), "+A" (*cmpval) 144 : "rJ" (*cmpval), "rJ" (newval) 145 : "memory"); 146 147 return (!res); 148} 149 150static __inline uint32_t 151atomic_fetchadd_32(volatile uint32_t *p, uint32_t val) 152{ 153 uint32_t ret; 154 155 __asm __volatile("amoadd.w %0, %2, %1" 156 : "=&r" (ret), "+A" (*p) 157 : "r" (val) 158 : "memory"); 159 160 return (ret); 161} 162 163static __inline uint32_t 164atomic_readandclear_32(volatile uint32_t *p) 165{ 166 uint32_t ret; 167 uint32_t val; 168 169 val = 0; 170 171 __asm __volatile("amoswap.w %0, %2, %1" 172 : "=&r"(ret), "+A" (*p) 173 : "r" (val) 174 : "memory"); 175 176 return (ret); 177} 178 179#define atomic_add_int atomic_add_32 180#define atomic_clear_int atomic_clear_32 181#define atomic_cmpset_int atomic_cmpset_32 182#define atomic_fcmpset_int atomic_fcmpset_32 183#define atomic_fetchadd_int atomic_fetchadd_32 184#define atomic_readandclear_int atomic_readandclear_32 185#define atomic_set_int atomic_set_32 186#define atomic_subtract_int atomic_subtract_32 187 188ATOMIC_ACQ_REL(set, 32) 189ATOMIC_ACQ_REL(clear, 32) 190ATOMIC_ACQ_REL(add, 32) 191ATOMIC_ACQ_REL(subtract, 32) 192 193static __inline int 194atomic_cmpset_acq_32(volatile uint32_t *p, uint32_t cmpval, uint32_t newval) 195{ 196 int res; 197 198 res = atomic_cmpset_32(p, cmpval, newval); 199 200 fence(); 201 202 return (res); 203} 204 205static __inline int 206atomic_cmpset_rel_32(volatile uint32_t *p, uint32_t cmpval, uint32_t newval) 207{ 208 209 fence(); 210 211 return (atomic_cmpset_32(p, cmpval, newval)); 212} 213 214static __inline int 215atomic_fcmpset_acq_32(volatile uint32_t *p, uint32_t *cmpval, uint32_t newval) 216{ 217 int res; 218 219 res = atomic_fcmpset_32(p, cmpval, newval); 220 221 fence(); 222 223 return (res); 224} 225 226static __inline int 227atomic_fcmpset_rel_32(volatile uint32_t *p, uint32_t *cmpval, uint32_t newval) 228{ 229 230 fence(); 231 232 return (atomic_fcmpset_32(p, cmpval, newval)); 233} 234 235static __inline uint32_t 236atomic_load_acq_32(volatile uint32_t *p) 237{ 238 uint32_t ret; 239 240 ret = *p; 241 242 fence(); 243 244 return (ret); 245} 246 247static __inline void 248atomic_store_rel_32(volatile uint32_t *p, uint32_t val) 249{ 250 251 fence(); 252 253 *p = val; 254} 255 256#define atomic_add_acq_int atomic_add_acq_32 257#define atomic_clear_acq_int atomic_clear_acq_32 258#define atomic_cmpset_acq_int atomic_cmpset_acq_32 259#define atomic_fcmpset_acq_int atomic_fcmpset_acq_32 260#define atomic_load_acq_int atomic_load_acq_32 261#define atomic_set_acq_int atomic_set_acq_32 262#define atomic_subtract_acq_int atomic_subtract_acq_32 263 264#define atomic_add_rel_int atomic_add_rel_32 265#define atomic_clear_rel_int atomic_add_rel_32 266#define atomic_cmpset_rel_int atomic_cmpset_rel_32 267#define atomic_fcmpset_rel_int atomic_fcmpset_rel_32 268#define atomic_set_rel_int atomic_set_rel_32 269#define atomic_subtract_rel_int atomic_subtract_rel_32 270#define atomic_store_rel_int atomic_store_rel_32 271 272static __inline void 273atomic_add_64(volatile uint64_t *p, uint64_t val) 274{ 275 276 __asm __volatile("amoadd.d zero, %1, %0" 277 : "+A" (*p) 278 : "r" (val) 279 : "memory"); 280} 281 282static __inline void 283atomic_subtract_64(volatile uint64_t *p, uint64_t val) 284{ 285 286 __asm __volatile("amoadd.d zero, %1, %0" 287 : "+A" (*p) 288 : "r" (-val) 289 : "memory"); 290} 291 292static __inline void 293atomic_set_64(volatile uint64_t *p, uint64_t val) 294{ 295 296 __asm __volatile("amoor.d zero, %1, %0" 297 : "+A" (*p) 298 : "r" (val) 299 : "memory"); 300} 301 302static __inline void 303atomic_clear_64(volatile uint64_t *p, uint64_t val) 304{ 305 306 __asm __volatile("amoand.d zero, %1, %0" 307 : "+A" (*p) 308 : "r" (~val) 309 : "memory"); 310} 311 312static __inline int 313atomic_cmpset_64(volatile uint64_t *p, uint64_t cmpval, uint64_t newval) 314{ 315 uint64_t tmp; 316 int res; 317 318 res = 0; 319 320 __asm __volatile( 321 "0:" 322 "li %1, 1\n" /* Preset to fail */ 323 "lr.d %0, %2\n" 324 "bne %0, %z3, 1f\n" 325 "sc.d %1, %z4, %2\n" 326 "bnez %1, 0b\n" 327 "1:" 328 : "=&r" (tmp), "=&r" (res), "+A" (*p) 329 : "rJ" (cmpval), "rJ" (newval) 330 : "memory"); 331 332 return (!res); 333} 334 335static __inline int 336atomic_fcmpset_64(volatile uint64_t *p, uint64_t *cmpval, uint64_t newval) 337{ 338 uint64_t tmp; 339 int res; 340 341 res = 0; 342 343 __asm __volatile( 344 "0:" 345 "li %1, 1\n" /* Preset to fail */ 346 "lr.d %0, %2\n" /* Load old value */ 347 "bne %0, %z4, 1f\n" /* Compare */ 348 "sc.d %1, %z5, %2\n" /* Try to store new value */ 349 "j 2f\n" 350 "1:" 351 "sd %0, %3\n" /* Save old value */ 352 "2:" 353 : "=&r" (tmp), "=&r" (res), "+A" (*p), "+A" (*cmpval) 354 : "rJ" (*cmpval), "rJ" (newval) 355 : "memory"); 356 357 return (!res); 358} 359 360static __inline uint64_t 361atomic_fetchadd_64(volatile uint64_t *p, uint64_t val) 362{ 363 uint64_t ret; 364 365 __asm __volatile("amoadd.d %0, %2, %1" 366 : "=&r" (ret), "+A" (*p) 367 : "r" (val) 368 : "memory"); 369 370 return (ret); 371} 372 373static __inline uint64_t 374atomic_readandclear_64(volatile uint64_t *p) 375{ 376 uint64_t ret; 377 uint64_t val; 378 379 val = 0; 380 381 __asm __volatile("amoswap.d %0, %2, %1" 382 : "=&r"(ret), "+A" (*p) 383 : "r" (val) 384 : "memory"); 385 386 return (ret); 387} 388 389static __inline uint32_t 390atomic_swap_32(volatile uint32_t *p, uint32_t val) 391{ 392 uint32_t old; 393 394 __asm __volatile("amoswap.w %0, %2, %1" 395 : "=&r"(old), "+A" (*p) 396 : "r" (val) 397 : "memory"); 398 399 return (old); 400} 401 402static __inline uint64_t 403atomic_swap_64(volatile uint64_t *p, uint64_t val) 404{ 405 uint64_t old; 406 407 __asm __volatile("amoswap.d %0, %2, %1" 408 : "=&r"(old), "+A" (*p) 409 : "r" (val) 410 : "memory"); 411 412 return (old); 413} 414 415#define atomic_add_long atomic_add_64 416#define atomic_clear_long atomic_clear_64 417#define atomic_cmpset_long atomic_cmpset_64 418#define atomic_fcmpset_long atomic_fcmpset_64 419#define atomic_fetchadd_long atomic_fetchadd_64 420#define atomic_readandclear_long atomic_readandclear_64 421#define atomic_set_long atomic_set_64 422#define atomic_subtract_long atomic_subtract_64 423 424#define atomic_add_ptr atomic_add_64 425#define atomic_clear_ptr atomic_clear_64 426#define atomic_cmpset_ptr atomic_cmpset_64 427#define atomic_fcmpset_ptr atomic_fcmpset_64 428#define atomic_fetchadd_ptr atomic_fetchadd_64 429#define atomic_readandclear_ptr atomic_readandclear_64 430#define atomic_set_ptr atomic_set_64 431#define atomic_subtract_ptr atomic_subtract_64 432 433ATOMIC_ACQ_REL(set, 64) 434ATOMIC_ACQ_REL(clear, 64) 435ATOMIC_ACQ_REL(add, 64) 436ATOMIC_ACQ_REL(subtract, 64) 437 438static __inline int 439atomic_cmpset_acq_64(volatile uint64_t *p, uint64_t cmpval, uint64_t newval) 440{ 441 int res; 442 443 res = atomic_cmpset_64(p, cmpval, newval); 444 445 fence(); 446 447 return (res); 448} 449 450static __inline int 451atomic_cmpset_rel_64(volatile uint64_t *p, uint64_t cmpval, uint64_t newval) 452{ 453 454 fence(); 455 456 return (atomic_cmpset_64(p, cmpval, newval)); 457} 458 459static __inline int 460atomic_fcmpset_acq_64(volatile uint64_t *p, uint64_t *cmpval, uint64_t newval) 461{ 462 int res; 463 464 res = atomic_fcmpset_64(p, cmpval, newval); 465 466 fence(); 467 468 return (res); 469} 470 471static __inline int 472atomic_fcmpset_rel_64(volatile uint64_t *p, uint64_t *cmpval, uint64_t newval) 473{ 474 475 fence(); 476 477 return (atomic_fcmpset_64(p, cmpval, newval)); 478} 479 480static __inline uint64_t 481atomic_load_acq_64(volatile uint64_t *p) 482{ 483 uint64_t ret; 484 485 ret = *p; 486 487 fence(); 488 489 return (ret); 490} 491 492static __inline void 493atomic_store_rel_64(volatile uint64_t *p, uint64_t val) 494{ 495 496 fence(); 497 498 *p = val; 499} 500 501#define atomic_add_acq_long atomic_add_acq_64 502#define atomic_clear_acq_long atomic_add_acq_64 503#define atomic_cmpset_acq_long atomic_cmpset_acq_64 504#define atomic_fcmpset_acq_long atomic_fcmpset_acq_64 505#define atomic_load_acq_long atomic_load_acq_64 506#define atomic_set_acq_long atomic_set_acq_64 507#define atomic_subtract_acq_long atomic_subtract_acq_64 508 509#define atomic_add_acq_ptr atomic_add_acq_64 510#define atomic_clear_acq_ptr atomic_add_acq_64 511#define atomic_cmpset_acq_ptr atomic_cmpset_acq_64 512#define atomic_fcmpset_acq_ptr atomic_fcmpset_acq_64 513#define atomic_load_acq_ptr atomic_load_acq_64 514#define atomic_set_acq_ptr atomic_set_acq_64 515#define atomic_subtract_acq_ptr atomic_subtract_acq_64 516 517static __inline void 518atomic_thread_fence_acq(void) 519{ 520 521 fence(); 522} 523 524static __inline void 525atomic_thread_fence_rel(void) 526{ 527 528 fence(); 529} 530 531static __inline void 532atomic_thread_fence_acq_rel(void) 533{ 534 535 fence(); 536} 537 538static __inline void 539atomic_thread_fence_seq_cst(void) 540{ 541 542 fence(); 543} 544 545#define atomic_add_rel_long atomic_add_rel_64 546#define atomic_clear_rel_long atomic_clear_rel_64 547 548#define atomic_add_rel_long atomic_add_rel_64 549#define atomic_clear_rel_long atomic_clear_rel_64 550#define atomic_cmpset_rel_long atomic_cmpset_rel_64 551#define atomic_fcmpset_rel_long atomic_fcmpset_rel_64 552#define atomic_set_rel_long atomic_set_rel_64 553#define atomic_subtract_rel_long atomic_subtract_rel_64 554#define atomic_store_rel_long atomic_store_rel_64 555 556#define atomic_add_rel_ptr atomic_add_rel_64 557#define atomic_clear_rel_ptr atomic_clear_rel_64 558#define atomic_cmpset_rel_ptr atomic_cmpset_rel_64 559#define atomic_fcmpset_rel_ptr atomic_fcmpset_rel_64 560#define atomic_set_rel_ptr atomic_set_rel_64 561#define atomic_subtract_rel_ptr atomic_subtract_rel_64 562#define atomic_store_rel_ptr atomic_store_rel_64 563 564#endif /* _MACHINE_ATOMIC_H_ */ 565