stdatomic.c revision 251524
1/*- 2 * Copyright (c) 2013 Ed Schouten <ed@FreeBSD.org> 3 * All rights reserved. 4 * 5 * Copyright (c) 1998 Doug Rabson 6 * All rights reserved. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27 * SUCH DAMAGE. 28 */ 29 30#include <sys/cdefs.h> 31__FBSDID("$FreeBSD: head/sys/mips/mips/stdatomic.c 251524 2013-06-08 13:19:11Z ed $"); 32 33#include <sys/types.h> 34 35#ifdef _KERNEL 36#include "opt_global.h" 37#endif 38 39/* 40 * Memory barriers. 41 * 42 * It turns out __sync_synchronize() does not emit any code when used 43 * with GCC 4.2. Implement our own version that does work reliably. 44 * 45 * Although __sync_lock_test_and_set() should only perform an acquire 46 * barrier, make it do a full barrier like the other functions. This 47 * should make <stdatomic.h>'s atomic_exchange_explicit() work reliably. 48 */ 49 50static inline void 51mips_sync(void) 52{ 53 54 __asm volatile ( 55#if !defined(_KERNEL) || defined(SMP) 56 ".set noreorder\n" 57 "\tsync\n" 58 "\tnop\n" 59 "\tnop\n" 60 "\tnop\n" 61 "\tnop\n" 62 "\tnop\n" 63 "\tnop\n" 64 "\tnop\n" 65 "\tnop\n" 66 ".set reorder\n" 67#else /* _KERNEL && !SMP */ 68 "" 69#endif /* !KERNEL || SMP */ 70 : : : "memory"); 71} 72 73typedef union { 74 uint8_t v8[4]; 75 uint16_t v16[2]; 76 uint32_t v32; 77} reg_t; 78 79/* 80 * 8-bit routines. 81 */ 82 83uint8_t 84__sync_lock_test_and_set_1(uint8_t *mem8, uint8_t val8) 85{ 86 uint32_t *mem32; 87 reg_t val32, negmask32, old; 88 uint32_t temp; 89 90 mem32 = (uint32_t *)((intptr_t)mem8 & ~3); 91 val32.v32 = 0x00000000; 92 val32.v8[(intptr_t)mem8 & 3] = val8; 93 negmask32.v32 = 0xffffffff; 94 negmask32.v8[(intptr_t)mem8 & 3] = 0x00; 95 96 mips_sync(); 97 __asm volatile ( 98 "1:" 99 "\tll %0, %5\n" /* Load old value. */ 100 "\tand %2, %4, %0\n" /* Trim out affected part. */ 101 "\tor %2, %3\n" /* Put in the new value. */ 102 "\tsc %2, %1\n" /* Attempt to store. */ 103 "\tbeqz %2, 1b\n" /* Spin if failed. */ 104 : "=&r" (old.v32), "=m" (*mem32), "=&r" (temp) 105 : "r" (val32.v32), "r" (negmask32.v32), "m" (*mem32)); 106 return (old.v8[(intptr_t)mem8 & 3]); 107} 108 109uint8_t 110__sync_val_compare_and_swap_1(uint8_t *mem8, uint8_t expected, uint8_t desired) 111{ 112 uint32_t *mem32; 113 reg_t expected32, desired32, posmask32, negmask32, old; 114 uint32_t temp; 115 116 mem32 = (uint32_t *)((intptr_t)mem8 & ~3); 117 expected32.v32 = 0x00000000; 118 expected32.v8[(intptr_t)mem8 & 3] = expected; 119 desired32.v32 = 0x00000000; 120 desired32.v8[(intptr_t)mem8 & 3] = desired; 121 posmask32.v32 = 0x00000000; 122 posmask32.v8[(intptr_t)mem8 & 3] = 0xff; 123 negmask32.v32 = ~posmask32.v32; 124 125 mips_sync(); 126 __asm volatile ( 127 "1:" 128 "\tll %0, %7\n" /* Load old value. */ 129 "\tand %2, %5, %0\n" /* Isolate affected part. */ 130 "\tbne %2, %3, 2f\n" /* Compare to expected value. */ 131 "\tand %2, %6, %0\n" /* Trim out affected part. */ 132 "\tor %2, %4\n" /* Put in the new value. */ 133 "\tsc %2, %1\n" /* Attempt to store. */ 134 "\tbeqz %2, 1b\n" /* Spin if failed. */ 135 "2:" 136 : "=&r" (old), "=m" (*mem32), "=&r" (temp) 137 : "r" (expected32.v32), "r" (desired32.v32), 138 "r" (posmask32.v32), "r" (negmask32.v32), "m" (*mem32)); 139 return (old.v8[(intptr_t)mem8 & 3]); 140} 141 142#define EMIT_ARITHMETIC_FETCH_AND_OP_1(name, op) \ 143uint8_t \ 144__sync_##name##_1(uint8_t *mem8, uint8_t val8) \ 145{ \ 146 uint32_t *mem32; \ 147 reg_t val32, posmask32, negmask32, old; \ 148 uint32_t temp1, temp2; \ 149 \ 150 mem32 = (uint32_t *)((intptr_t)mem8 & ~3); \ 151 val32.v32 = 0x00000000; \ 152 val32.v8[(intptr_t)mem8 & 3] = val8; \ 153 posmask32.v32 = 0x00000000; \ 154 posmask32.v8[(intptr_t)mem8 & 3] = 0xff; \ 155 negmask32.v32 = ~posmask32.v32; \ 156 \ 157 mips_sync(); \ 158 __asm volatile ( \ 159 "1:" \ 160 "\tll %0, %7\n" /* Load old value. */ \ 161 "\t"op" %2, %0, %4\n" /* Calculate new value. */ \ 162 "\tand %2, %5\n" /* Isolate affected part. */ \ 163 "\tand %3, %6, %0\n" /* Trim out affected part. */ \ 164 "\tor %2, %3\n" /* Put in the new value. */ \ 165 "\tsc %2, %1\n" /* Attempt to store. */ \ 166 "\tbeqz %2, 1b\n" /* Spin if failed. */ \ 167 : "=&r" (old.v32), "=m" (*mem32), "=&r" (temp1), \ 168 "=&r" (temp2) \ 169 : "r" (val32.v32), "r" (posmask32.v32), \ 170 "r" (negmask32.v32), "m" (*mem32)); \ 171 return (old.v8[(intptr_t)mem8 & 3]); \ 172} 173 174EMIT_ARITHMETIC_FETCH_AND_OP_1(fetch_and_add, "addu") 175EMIT_ARITHMETIC_FETCH_AND_OP_1(fetch_and_sub, "subu") 176 177#define EMIT_BITWISE_FETCH_AND_OP_1(name, op, idempotence) \ 178uint8_t \ 179__sync_##name##_1(uint8_t *mem8, uint8_t val8) \ 180{ \ 181 uint32_t *mem32; \ 182 reg_t val32, old; \ 183 uint32_t temp; \ 184 \ 185 mem32 = (uint32_t *)((intptr_t)mem8 & ~3); \ 186 val32.v32 = idempotence ? 0xffffffff : 0x00000000; \ 187 val32.v8[(intptr_t)mem8 & 3] = val8; \ 188 \ 189 mips_sync(); \ 190 __asm volatile ( \ 191 "1:" \ 192 "\tll %0, %4\n" /* Load old value. */ \ 193 "\t"op" %2, %3, %0\n" /* Calculate new value. */ \ 194 "\tsc %2, %1\n" /* Attempt to store. */ \ 195 "\tbeqz %2, 1b\n" /* Spin if failed. */ \ 196 : "=&r" (old.v32), "=m" (*mem32), "=&r" (temp) \ 197 : "r" (val32.v32), "m" (*mem32)); \ 198 return (old.v8[(intptr_t)mem8 & 3]); \ 199} 200 201EMIT_BITWISE_FETCH_AND_OP_1(fetch_and_and, "and", 1) 202EMIT_BITWISE_FETCH_AND_OP_1(fetch_and_or, "or", 0) 203EMIT_BITWISE_FETCH_AND_OP_1(fetch_and_xor, "xor", 0) 204 205/* 206 * 16-bit routines. 207 */ 208 209uint16_t 210__sync_lock_test_and_set_2(uint16_t *mem16, uint16_t val16) 211{ 212 uint32_t *mem32; 213 reg_t val32, negmask32, old; 214 uint32_t temp; 215 216 mem32 = (uint32_t *)((intptr_t)mem16 & ~1); 217 val32.v32 = 0x00000000; 218 val32.v16[(intptr_t)mem16 & 1] = val16; 219 negmask32.v32 = 0xffffffff; 220 negmask32.v16[(intptr_t)mem16 & 1] = 0x0000; 221 222 mips_sync(); 223 __asm volatile ( 224 "1:" 225 "\tll %0, %5\n" /* Load old value. */ 226 "\tand %2, %4, %0\n" /* Trim out affected part. */ 227 "\tor %2, %3\n" /* Combine to new value. */ 228 "\tsc %2, %1\n" /* Attempt to store. */ 229 "\tbeqz %2, 1b\n" /* Spin if failed. */ 230 : "=&r" (old.v32), "=m" (*mem32), "=&r" (temp) 231 : "r" (val32.v32), "r" (negmask32.v32), "m" (*mem32)); 232 return (old.v16[(intptr_t)mem16 & 1]); 233} 234 235uint16_t 236__sync_val_compare_and_swap_2(uint16_t *mem16, uint16_t expected, 237 uint16_t desired) 238{ 239 uint32_t *mem32; 240 reg_t expected32, desired32, posmask32, negmask32, old; 241 uint32_t temp; 242 243 mem32 = (uint32_t *)((intptr_t)mem16 & ~1); 244 expected32.v32 = 0x00000000; 245 expected32.v16[(intptr_t)mem16 & 1] = expected; 246 desired32.v32 = 0x00000000; 247 desired32.v16[(intptr_t)mem16 & 1] = desired; 248 posmask32.v32 = 0x00000000; 249 posmask32.v16[(intptr_t)mem16 & 1] = 0xffff; 250 negmask32.v32 = ~posmask32.v32; 251 252 mips_sync(); 253 __asm volatile ( 254 "1:" 255 "\tll %0, %7\n" /* Load old value. */ 256 "\tand %2, %5, %0\n" /* Isolate affected part. */ 257 "\tbne %2, %3, 2f\n" /* Compare to expected value. */ 258 "\tand %2, %6, %0\n" /* Trim out affected part. */ 259 "\tor %2, %4\n" /* Put in the new value. */ 260 "\tsc %2, %1\n" /* Attempt to store. */ 261 "\tbeqz %2, 1b\n" /* Spin if failed. */ 262 "2:" 263 : "=&r" (old), "=m" (*mem32), "=&r" (temp) 264 : "r" (expected32.v32), "r" (desired32.v32), 265 "r" (posmask32.v32), "r" (negmask32.v32), "m" (*mem32)); 266 return (old.v16[(intptr_t)mem16 & 1]); 267} 268 269#define EMIT_ARITHMETIC_FETCH_AND_OP_2(name, op) \ 270uint16_t \ 271__sync_##name##_2(uint16_t *mem16, uint16_t val16) \ 272{ \ 273 uint32_t *mem32; \ 274 reg_t val32, posmask32, negmask32, old; \ 275 uint32_t temp1, temp2; \ 276 \ 277 mem32 = (uint32_t *)((intptr_t)mem16 & ~3); \ 278 val32.v32 = 0x00000000; \ 279 val32.v16[(intptr_t)mem16 & 1] = val16; \ 280 posmask32.v32 = 0x00000000; \ 281 posmask32.v16[(intptr_t)mem16 & 1] = 0xffff; \ 282 negmask32.v32 = ~posmask32.v32; \ 283 \ 284 mips_sync(); \ 285 __asm volatile ( \ 286 "1:" \ 287 "\tll %0, %7\n" /* Load old value. */ \ 288 "\t"op" %2, %0, %4\n" /* Calculate new value. */ \ 289 "\tand %2, %5\n" /* Isolate affected part. */ \ 290 "\tand %3, %6, %0\n" /* Trim out affected part. */ \ 291 "\tor %2, %3\n" /* Combine to new value. */ \ 292 "\tsc %2, %1\n" /* Attempt to store. */ \ 293 "\tbeqz %2, 1b\n" /* Spin if failed. */ \ 294 : "=&r" (old.v32), "=m" (*mem32), "=&r" (temp1), \ 295 "=&r" (temp2) \ 296 : "r" (val32.v32), "r" (posmask32.v32), \ 297 "r" (negmask32.v32), "m" (*mem32)); \ 298 return (old.v16[(intptr_t)mem16 & 1]); \ 299} 300 301EMIT_ARITHMETIC_FETCH_AND_OP_2(fetch_and_add, "addu") 302EMIT_ARITHMETIC_FETCH_AND_OP_2(fetch_and_sub, "subu") 303 304#define EMIT_BITWISE_FETCH_AND_OP_2(name, op, idempotence) \ 305uint16_t \ 306__sync_##name##_2(uint16_t *mem16, uint16_t val16) \ 307{ \ 308 uint32_t *mem32; \ 309 reg_t val32, old; \ 310 uint32_t temp; \ 311 \ 312 mem32 = (uint32_t *)((intptr_t)mem16 & ~1); \ 313 val32.v32 = idempotence ? 0xffffffff : 0x00000000; \ 314 val32.v16[(intptr_t)mem16 & 1] = val16; \ 315 \ 316 mips_sync(); \ 317 __asm volatile ( \ 318 "1:" \ 319 "\tll %0, %4\n" /* Load old value. */ \ 320 "\t"op" %2, %3, %0\n" /* Calculate new value. */ \ 321 "\tsc %2, %1\n" /* Attempt to store. */ \ 322 "\tbeqz %2, 1b\n" /* Spin if failed. */ \ 323 : "=&r" (old.v32), "=m" (*mem32), "=&r" (temp) \ 324 : "r" (val32.v32), "m" (*mem32)); \ 325 return (old.v16[(intptr_t)mem16 & 1]); \ 326} 327 328EMIT_BITWISE_FETCH_AND_OP_2(fetch_and_and, "and", 1) 329EMIT_BITWISE_FETCH_AND_OP_2(fetch_and_or, "or", 0) 330EMIT_BITWISE_FETCH_AND_OP_2(fetch_and_xor, "xor", 0) 331 332/* 333 * 32-bit routines. 334 */ 335 336uint32_t 337__sync_val_compare_and_swap_4(uint32_t *mem, uint32_t expected, 338 uint32_t desired) 339{ 340 uint32_t old, temp; 341 342 mips_sync(); 343 __asm volatile ( 344 "1:" 345 "\tll %0, %5\n" /* Load old value. */ 346 "\tbne %0, %3, 2f\n" /* Compare to expected value. */ 347 "\tmove %2, %4\n" /* Value to store. */ 348 "\tsc %2, %1\n" /* Attempt to store. */ 349 "\tbeqz %2, 1b\n" /* Spin if failed. */ 350 "2:" 351 : "=&r" (old), "=m" (*mem), "=&r" (temp) 352 : "r" (expected), "r" (desired), "m" (*mem)); 353 return (old); 354} 355 356#define EMIT_FETCH_AND_OP_4(name, op) \ 357uint32_t \ 358__sync_##name##_4(uint32_t *mem, uint32_t val) \ 359{ \ 360 uint32_t old, temp; \ 361 \ 362 mips_sync(); \ 363 __asm volatile ( \ 364 "1:" \ 365 "\tll %0, %4\n" /* Load old value. */ \ 366 "\t"op"\n" /* Calculate new value. */ \ 367 "\tsc %2, %1\n" /* Attempt to store. */ \ 368 "\tbeqz %2, 1b\n" /* Spin if failed. */ \ 369 : "=&r" (old), "=m" (*mem), "=&r" (temp) \ 370 : "r" (val), "m" (*mem)); \ 371 return (old); \ 372} 373 374EMIT_FETCH_AND_OP_4(lock_test_and_set, "move %2, %3") 375EMIT_FETCH_AND_OP_4(fetch_and_add, "addu %2, %0, %3") 376EMIT_FETCH_AND_OP_4(fetch_and_and, "and %2, %0, %3") 377EMIT_FETCH_AND_OP_4(fetch_and_or, "or %2, %0, %3") 378EMIT_FETCH_AND_OP_4(fetch_and_sub, "subu %2, %0, %3") 379EMIT_FETCH_AND_OP_4(fetch_and_xor, "xor %2, %0, %3") 380 381/* 382 * 64-bit routines. 383 * 384 * Note: All the 64-bit atomic operations are only atomic when running 385 * in 64-bit mode. It is assumed that code compiled for n32 and n64 fits 386 * into this definition and no further safeties are needed. 387 */ 388 389#if defined(__mips_n32) || defined(__mips_n64) 390 391uint64_t 392__sync_val_compare_and_swap_8(uint64_t *mem, uint64_t expected, 393 uint64_t desired) 394{ 395 uint64_t old, temp; 396 397 mips_sync(); 398 __asm volatile ( 399 "1:" 400 "\tlld %0, %5\n" /* Load old value. */ 401 "\tbne %0, %3, 2f\n" /* Compare to expected value. */ 402 "\tmove %2, %4\n" /* Value to store. */ 403 "\tscd %2, %1\n" /* Attempt to store. */ 404 "\tbeqz %2, 1b\n" /* Spin if failed. */ 405 "2:" 406 : "=&r" (old), "=m" (*mem), "=&r" (temp) 407 : "r" (expected), "r" (desired), "m" (*mem)); 408 return (old); 409} 410 411#define EMIT_FETCH_AND_OP_8(name, op) \ 412uint64_t \ 413__sync_##name##_8(uint64_t *mem, uint64_t val) \ 414{ \ 415 uint64_t old, temp; \ 416 \ 417 mips_sync(); \ 418 __asm volatile ( \ 419 "1:" \ 420 "\tlld %0, %4\n" /* Load old value. */ \ 421 "\t"op"\n" /* Calculate new value. */ \ 422 "\tscd %2, %1\n" /* Attempt to store. */ \ 423 "\tbeqz %2, 1b\n" /* Spin if failed. */ \ 424 : "=&r" (old), "=m" (*mem), "=&r" (temp) \ 425 : "r" (val), "m" (*mem)); \ 426 return (old); \ 427} 428 429EMIT_FETCH_AND_OP_8(lock_test_and_set, "move %2, %3") 430EMIT_FETCH_AND_OP_8(fetch_and_add, "daddu %2, %0, %3") 431EMIT_FETCH_AND_OP_8(fetch_and_and, "and %2, %0, %3") 432EMIT_FETCH_AND_OP_8(fetch_and_or, "or %2, %0, %3") 433EMIT_FETCH_AND_OP_8(fetch_and_sub, "dsubu %2, %0, %3") 434EMIT_FETCH_AND_OP_8(fetch_and_xor, "xor %2, %0, %3") 435 436#endif /* __mips_n32 || __mips_n64 */ 437