stdatomic.c revision 331722
1/*- 2 * Copyright (c) 2013 Ed Schouten <ed@FreeBSD.org> 3 * All rights reserved. 4 * 5 * Copyright (c) 1998 Doug Rabson 6 * All rights reserved. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27 * SUCH DAMAGE. 28 */ 29 30#include <sys/cdefs.h> 31__FBSDID("$FreeBSD: stable/11/sys/mips/mips/stdatomic.c 331722 2018-03-29 02:50:57Z eadler $"); 32 33#include <sys/stdatomic.h> 34#include <sys/types.h> 35 36#ifndef _KERNEL 37#include <stdbool.h> 38#endif /* _KERNEL */ 39 40#if defined(__SYNC_ATOMICS) 41 42/* 43 * Memory barriers. 44 * 45 * It turns out __sync_synchronize() does not emit any code when used 46 * with GCC 4.2. Implement our own version that does work reliably. 47 * 48 * Although __sync_lock_test_and_set() should only perform an acquire 49 * barrier, make it do a full barrier like the other functions. This 50 * should make <stdatomic.h>'s atomic_exchange_explicit() work reliably. 51 */ 52 53static inline void 54do_sync(void) 55{ 56 57 __asm volatile ( 58#if !defined(_KERNEL) || defined(SMP) 59 ".set noreorder\n" 60 "\tsync\n" 61 "\tnop\n" 62 "\tnop\n" 63 "\tnop\n" 64 "\tnop\n" 65 "\tnop\n" 66 "\tnop\n" 67 "\tnop\n" 68 "\tnop\n" 69 ".set reorder\n" 70#else /* _KERNEL && !SMP */ 71 "" 72#endif /* !KERNEL || SMP */ 73 : : : "memory"); 74} 75 76typedef union { 77 uint8_t v8[4]; 78 uint32_t v32; 79} reg_t; 80 81/* 82 * Given a memory address pointing to an 8-bit or 16-bit integer, return 83 * the address of the 32-bit word containing it. 84 */ 85 86static inline uint32_t * 87round_to_word(void *ptr) 88{ 89 90 return ((uint32_t *)((intptr_t)ptr & ~3)); 91} 92 93/* 94 * Utility functions for loading and storing 8-bit and 16-bit integers 95 * in 32-bit words at an offset corresponding with the location of the 96 * atomic variable. 97 */ 98 99static inline void 100put_1(reg_t *r, const uint8_t *offset_ptr, uint8_t val) 101{ 102 size_t offset; 103 104 offset = (intptr_t)offset_ptr & 3; 105 r->v8[offset] = val; 106} 107 108static inline uint8_t 109get_1(const reg_t *r, const uint8_t *offset_ptr) 110{ 111 size_t offset; 112 113 offset = (intptr_t)offset_ptr & 3; 114 return (r->v8[offset]); 115} 116 117static inline void 118put_2(reg_t *r, const uint16_t *offset_ptr, uint16_t val) 119{ 120 size_t offset; 121 union { 122 uint16_t in; 123 uint8_t out[2]; 124 } bytes; 125 126 offset = (intptr_t)offset_ptr & 3; 127 bytes.in = val; 128 r->v8[offset] = bytes.out[0]; 129 r->v8[offset + 1] = bytes.out[1]; 130} 131 132static inline uint16_t 133get_2(const reg_t *r, const uint16_t *offset_ptr) 134{ 135 size_t offset; 136 union { 137 uint8_t in[2]; 138 uint16_t out; 139 } bytes; 140 141 offset = (intptr_t)offset_ptr & 3; 142 bytes.in[0] = r->v8[offset]; 143 bytes.in[1] = r->v8[offset + 1]; 144 return (bytes.out); 145} 146 147/* 148 * 8-bit and 16-bit routines. 149 * 150 * These operations are not natively supported by the CPU, so we use 151 * some shifting and bitmasking on top of the 32-bit instructions. 152 */ 153 154#define EMIT_LOCK_TEST_AND_SET_N(N, uintN_t) \ 155uintN_t \ 156__sync_lock_test_and_set_##N(uintN_t *mem, uintN_t val) \ 157{ \ 158 uint32_t *mem32; \ 159 reg_t val32, negmask, old; \ 160 uint32_t temp; \ 161 \ 162 mem32 = round_to_word(mem); \ 163 val32.v32 = 0x00000000; \ 164 put_##N(&val32, mem, val); \ 165 negmask.v32 = 0xffffffff; \ 166 put_##N(&negmask, mem, 0); \ 167 \ 168 do_sync(); \ 169 __asm volatile ( \ 170 "1:" \ 171 "\tll %0, %5\n" /* Load old value. */ \ 172 "\tand %2, %4, %0\n" /* Remove the old value. */ \ 173 "\tor %2, %3\n" /* Put in the new value. */ \ 174 "\tsc %2, %1\n" /* Attempt to store. */ \ 175 "\tbeqz %2, 1b\n" /* Spin if failed. */ \ 176 : "=&r" (old.v32), "=m" (*mem32), "=&r" (temp) \ 177 : "r" (val32.v32), "r" (negmask.v32), "m" (*mem32)); \ 178 return (get_##N(&old, mem)); \ 179} 180 181EMIT_LOCK_TEST_AND_SET_N(1, uint8_t) 182EMIT_LOCK_TEST_AND_SET_N(2, uint16_t) 183 184#define EMIT_VAL_COMPARE_AND_SWAP_N(N, uintN_t) \ 185uintN_t \ 186__sync_val_compare_and_swap_##N(uintN_t *mem, uintN_t expected, \ 187 uintN_t desired) \ 188{ \ 189 uint32_t *mem32; \ 190 reg_t expected32, desired32, posmask, old; \ 191 uint32_t negmask, temp; \ 192 \ 193 mem32 = round_to_word(mem); \ 194 expected32.v32 = 0x00000000; \ 195 put_##N(&expected32, mem, expected); \ 196 desired32.v32 = 0x00000000; \ 197 put_##N(&desired32, mem, desired); \ 198 posmask.v32 = 0x00000000; \ 199 put_##N(&posmask, mem, ~0); \ 200 negmask = ~posmask.v32; \ 201 \ 202 do_sync(); \ 203 __asm volatile ( \ 204 "1:" \ 205 "\tll %0, %7\n" /* Load old value. */ \ 206 "\tand %2, %5, %0\n" /* Isolate the old value. */ \ 207 "\tbne %2, %3, 2f\n" /* Compare to expected value. */\ 208 "\tand %2, %6, %0\n" /* Remove the old value. */ \ 209 "\tor %2, %4\n" /* Put in the new value. */ \ 210 "\tsc %2, %1\n" /* Attempt to store. */ \ 211 "\tbeqz %2, 1b\n" /* Spin if failed. */ \ 212 "2:" \ 213 : "=&r" (old), "=m" (*mem32), "=&r" (temp) \ 214 : "r" (expected32.v32), "r" (desired32.v32), \ 215 "r" (posmask.v32), "r" (negmask), "m" (*mem32)); \ 216 return (get_##N(&old, mem)); \ 217} 218 219EMIT_VAL_COMPARE_AND_SWAP_N(1, uint8_t) 220EMIT_VAL_COMPARE_AND_SWAP_N(2, uint16_t) 221 222#define EMIT_ARITHMETIC_FETCH_AND_OP_N(N, uintN_t, name, op) \ 223uintN_t \ 224__sync_##name##_##N(uintN_t *mem, uintN_t val) \ 225{ \ 226 uint32_t *mem32; \ 227 reg_t val32, posmask, old; \ 228 uint32_t negmask, temp1, temp2; \ 229 \ 230 mem32 = round_to_word(mem); \ 231 val32.v32 = 0x00000000; \ 232 put_##N(&val32, mem, val); \ 233 posmask.v32 = 0x00000000; \ 234 put_##N(&posmask, mem, ~0); \ 235 negmask = ~posmask.v32; \ 236 \ 237 do_sync(); \ 238 __asm volatile ( \ 239 "1:" \ 240 "\tll %0, %7\n" /* Load old value. */ \ 241 "\t"op" %2, %0, %4\n" /* Calculate new value. */ \ 242 "\tand %2, %5\n" /* Isolate the new value. */ \ 243 "\tand %3, %6, %0\n" /* Remove the old value. */ \ 244 "\tor %2, %3\n" /* Put in the new value. */ \ 245 "\tsc %2, %1\n" /* Attempt to store. */ \ 246 "\tbeqz %2, 1b\n" /* Spin if failed. */ \ 247 : "=&r" (old.v32), "=m" (*mem32), "=&r" (temp1), \ 248 "=&r" (temp2) \ 249 : "r" (val32.v32), "r" (posmask.v32), "r" (negmask), \ 250 "m" (*mem32)); \ 251 return (get_##N(&old, mem)); \ 252} 253 254EMIT_ARITHMETIC_FETCH_AND_OP_N(1, uint8_t, fetch_and_add, "addu") 255EMIT_ARITHMETIC_FETCH_AND_OP_N(1, uint8_t, fetch_and_sub, "subu") 256EMIT_ARITHMETIC_FETCH_AND_OP_N(2, uint16_t, fetch_and_add, "addu") 257EMIT_ARITHMETIC_FETCH_AND_OP_N(2, uint16_t, fetch_and_sub, "subu") 258 259#define EMIT_BITWISE_FETCH_AND_OP_N(N, uintN_t, name, op, idempotence) \ 260uintN_t \ 261__sync_##name##_##N(uintN_t *mem, uintN_t val) \ 262{ \ 263 uint32_t *mem32; \ 264 reg_t val32, old; \ 265 uint32_t temp; \ 266 \ 267 mem32 = round_to_word(mem); \ 268 val32.v32 = idempotence ? 0xffffffff : 0x00000000; \ 269 put_##N(&val32, mem, val); \ 270 \ 271 do_sync(); \ 272 __asm volatile ( \ 273 "1:" \ 274 "\tll %0, %4\n" /* Load old value. */ \ 275 "\t"op" %2, %3, %0\n" /* Calculate new value. */ \ 276 "\tsc %2, %1\n" /* Attempt to store. */ \ 277 "\tbeqz %2, 1b\n" /* Spin if failed. */ \ 278 : "=&r" (old.v32), "=m" (*mem32), "=&r" (temp) \ 279 : "r" (val32.v32), "m" (*mem32)); \ 280 return (get_##N(&old, mem)); \ 281} 282 283EMIT_BITWISE_FETCH_AND_OP_N(1, uint8_t, fetch_and_and, "and", 1) 284EMIT_BITWISE_FETCH_AND_OP_N(1, uint8_t, fetch_and_or, "or", 0) 285EMIT_BITWISE_FETCH_AND_OP_N(1, uint8_t, fetch_and_xor, "xor", 0) 286EMIT_BITWISE_FETCH_AND_OP_N(2, uint16_t, fetch_and_and, "and", 1) 287EMIT_BITWISE_FETCH_AND_OP_N(2, uint16_t, fetch_and_or, "or", 0) 288EMIT_BITWISE_FETCH_AND_OP_N(2, uint16_t, fetch_and_xor, "xor", 0) 289 290/* 291 * 32-bit routines. 292 */ 293 294static __inline uint32_t 295do_compare_and_swap_4(uint32_t *mem, uint32_t expected, 296 uint32_t desired) 297{ 298 uint32_t old, temp; 299 300 do_sync(); 301 __asm volatile ( 302 "1:" 303 "\tll %0, %5\n" /* Load old value. */ 304 "\tbne %0, %3, 2f\n" /* Compare to expected value. */ 305 "\tmove %2, %4\n" /* Value to store. */ 306 "\tsc %2, %1\n" /* Attempt to store. */ 307 "\tbeqz %2, 1b\n" /* Spin if failed. */ 308 "2:" 309 : "=&r" (old), "=m" (*mem), "=&r" (temp) 310 : "r" (expected), "r" (desired), "m" (*mem)); 311 return (old); 312} 313 314uint32_t 315__sync_val_compare_and_swap_4(uint32_t *mem, uint32_t expected, 316 uint32_t desired) 317{ 318 319 return (do_compare_and_swap_4(mem, expected, desired)); 320} 321 322bool 323__sync_bool_compare_and_swap_4(uint32_t *mem, uint32_t expected, 324 uint32_t desired) 325{ 326 327 return (do_compare_and_swap_4(mem, expected, desired) == 328 desired); 329} 330 331#define EMIT_FETCH_AND_OP_4(name, op) \ 332uint32_t \ 333__sync_##name##_4(uint32_t *mem, uint32_t val) \ 334{ \ 335 uint32_t old, temp; \ 336 \ 337 do_sync(); \ 338 __asm volatile ( \ 339 "1:" \ 340 "\tll %0, %4\n" /* Load old value. */ \ 341 "\t"op"\n" /* Calculate new value. */ \ 342 "\tsc %2, %1\n" /* Attempt to store. */ \ 343 "\tbeqz %2, 1b\n" /* Spin if failed. */ \ 344 : "=&r" (old), "=m" (*mem), "=&r" (temp) \ 345 : "r" (val), "m" (*mem)); \ 346 return (old); \ 347} 348 349EMIT_FETCH_AND_OP_4(lock_test_and_set, "move %2, %3") 350EMIT_FETCH_AND_OP_4(fetch_and_add, "addu %2, %0, %3") 351EMIT_FETCH_AND_OP_4(fetch_and_and, "and %2, %0, %3") 352EMIT_FETCH_AND_OP_4(fetch_and_or, "or %2, %0, %3") 353EMIT_FETCH_AND_OP_4(fetch_and_sub, "subu %2, %0, %3") 354EMIT_FETCH_AND_OP_4(fetch_and_xor, "xor %2, %0, %3") 355 356/* 357 * 64-bit routines. 358 * 359 * Note: All the 64-bit atomic operations are only atomic when running 360 * in 64-bit mode. It is assumed that code compiled for n32 and n64 fits 361 * into this definition and no further safeties are needed. 362 */ 363 364#if defined(__mips_n32) || defined(__mips_n64) 365 366uint64_t 367__sync_val_compare_and_swap_8(uint64_t *mem, uint64_t expected, 368 uint64_t desired) 369{ 370 uint64_t old, temp; 371 372 do_sync(); 373 __asm volatile ( 374 "1:" 375 "\tlld %0, %5\n" /* Load old value. */ 376 "\tbne %0, %3, 2f\n" /* Compare to expected value. */ 377 "\tmove %2, %4\n" /* Value to store. */ 378 "\tscd %2, %1\n" /* Attempt to store. */ 379 "\tbeqz %2, 1b\n" /* Spin if failed. */ 380 "2:" 381 : "=&r" (old), "=m" (*mem), "=&r" (temp) 382 : "r" (expected), "r" (desired), "m" (*mem)); 383 return (old); 384} 385 386#define EMIT_FETCH_AND_OP_8(name, op) \ 387uint64_t \ 388__sync_##name##_8(uint64_t *mem, uint64_t val) \ 389{ \ 390 uint64_t old, temp; \ 391 \ 392 do_sync(); \ 393 __asm volatile ( \ 394 "1:" \ 395 "\tlld %0, %4\n" /* Load old value. */ \ 396 "\t"op"\n" /* Calculate new value. */ \ 397 "\tscd %2, %1\n" /* Attempt to store. */ \ 398 "\tbeqz %2, 1b\n" /* Spin if failed. */ \ 399 : "=&r" (old), "=m" (*mem), "=&r" (temp) \ 400 : "r" (val), "m" (*mem)); \ 401 return (old); \ 402} 403 404EMIT_FETCH_AND_OP_8(lock_test_and_set, "move %2, %3") 405EMIT_FETCH_AND_OP_8(fetch_and_add, "daddu %2, %0, %3") 406EMIT_FETCH_AND_OP_8(fetch_and_and, "and %2, %0, %3") 407EMIT_FETCH_AND_OP_8(fetch_and_or, "or %2, %0, %3") 408EMIT_FETCH_AND_OP_8(fetch_and_sub, "dsubu %2, %0, %3") 409EMIT_FETCH_AND_OP_8(fetch_and_xor, "xor %2, %0, %3") 410 411#endif /* __mips_n32 || __mips_n64 */ 412 413#endif /* __SYNC_ATOMICS */ 414