stdatomic.c revision 251524
189750Sdwmalone/*- 289750Sdwmalone * Copyright (c) 2013 Ed Schouten <ed@FreeBSD.org> 389750Sdwmalone * All rights reserved. 489750Sdwmalone * 589750Sdwmalone * Copyright (c) 1998 Doug Rabson 689750Sdwmalone * All rights reserved. 789750Sdwmalone * 889750Sdwmalone * Redistribution and use in source and binary forms, with or without 989750Sdwmalone * modification, are permitted provided that the following conditions 1089750Sdwmalone * are met: 1189750Sdwmalone * 1. Redistributions of source code must retain the above copyright 1289750Sdwmalone * notice, this list of conditions and the following disclaimer. 1389750Sdwmalone * 2. Redistributions in binary form must reproduce the above copyright 1489750Sdwmalone * notice, this list of conditions and the following disclaimer in the 1589750Sdwmalone * documentation and/or other materials provided with the distribution. 1689750Sdwmalone * 1789750Sdwmalone * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 1889750Sdwmalone * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 1989750Sdwmalone * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 2089750Sdwmalone * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 2124139Sjoerg * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 2289750Sdwmalone * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 2389750Sdwmalone * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 2424139Sjoerg * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 2589750Sdwmalone * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 2689750Sdwmalone * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 2789750Sdwmalone * SUCH DAMAGE. 2889750Sdwmalone */ 2989750Sdwmalone 3089750Sdwmalone#include <sys/cdefs.h> 3189750Sdwmalone__FBSDID("$FreeBSD: head/sys/mips/mips/stdatomic.c 251524 2013-06-08 13:19:11Z ed $"); 3289750Sdwmalone 3389750Sdwmalone#include <sys/types.h> 3489750Sdwmalone 3589750Sdwmalone#ifdef _KERNEL 3689750Sdwmalone#include "opt_global.h" 3789750Sdwmalone#endif 3889750Sdwmalone 3989750Sdwmalone/* 4089750Sdwmalone * Memory barriers. 4189750Sdwmalone * 4289750Sdwmalone * It turns out __sync_synchronize() does not emit any code when used 4324139Sjoerg * with GCC 4.2. Implement our own version that does work reliably. 4489750Sdwmalone * 4589750Sdwmalone * Although __sync_lock_test_and_set() should only perform an acquire 4689750Sdwmalone * barrier, make it do a full barrier like the other functions. This 4724139Sjoerg * should make <stdatomic.h>'s atomic_exchange_explicit() work reliably. 4889750Sdwmalone */ 4989750Sdwmalone 5089750Sdwmalonestatic inline void 5124139Sjoergmips_sync(void) 5289750Sdwmalone{ 5389750Sdwmalone 5489750Sdwmalone __asm volatile ( 5524139Sjoerg#if !defined(_KERNEL) || defined(SMP) 5689750Sdwmalone ".set noreorder\n" 5789750Sdwmalone "\tsync\n" 5889750Sdwmalone "\tnop\n" 5989750Sdwmalone "\tnop\n" 6024139Sjoerg "\tnop\n" 6189750Sdwmalone "\tnop\n" 6289750Sdwmalone "\tnop\n" 6389750Sdwmalone "\tnop\n" 6489750Sdwmalone "\tnop\n" 6524139Sjoerg "\tnop\n" 6689750Sdwmalone ".set reorder\n" 6789750Sdwmalone#else /* _KERNEL && !SMP */ 6889750Sdwmalone "" 6989750Sdwmalone#endif /* !KERNEL || SMP */ 7089750Sdwmalone : : : "memory"); 7189750Sdwmalone} 7289750Sdwmalone 7389750Sdwmalonetypedef union { 7489750Sdwmalone uint8_t v8[4]; 7524139Sjoerg uint16_t v16[2]; 7689750Sdwmalone uint32_t v32; 7789750Sdwmalone} reg_t; 7889750Sdwmalone 7989750Sdwmalone/* 8089750Sdwmalone * 8-bit routines. 8189750Sdwmalone */ 8289750Sdwmalone 8389750Sdwmaloneuint8_t 8489750Sdwmalone__sync_lock_test_and_set_1(uint8_t *mem8, uint8_t val8) 8589750Sdwmalone{ 8689750Sdwmalone uint32_t *mem32; 8789750Sdwmalone reg_t val32, negmask32, old; 8889750Sdwmalone uint32_t temp; 8989750Sdwmalone 9089750Sdwmalone mem32 = (uint32_t *)((intptr_t)mem8 & ~3); 9189750Sdwmalone val32.v32 = 0x00000000; 9224139Sjoerg val32.v8[(intptr_t)mem8 & 3] = val8; 9389750Sdwmalone negmask32.v32 = 0xffffffff; 9489750Sdwmalone negmask32.v8[(intptr_t)mem8 & 3] = 0x00; 9589750Sdwmalone 9689750Sdwmalone mips_sync(); 9789750Sdwmalone __asm volatile ( 9889750Sdwmalone "1:" 9989750Sdwmalone "\tll %0, %5\n" /* Load old value. */ 10089750Sdwmalone "\tand %2, %4, %0\n" /* Trim out affected part. */ 10189750Sdwmalone "\tor %2, %3\n" /* Put in the new value. */ 10289750Sdwmalone "\tsc %2, %1\n" /* Attempt to store. */ 10389750Sdwmalone "\tbeqz %2, 1b\n" /* Spin if failed. */ 10489750Sdwmalone : "=&r" (old.v32), "=m" (*mem32), "=&r" (temp) 10589750Sdwmalone : "r" (val32.v32), "r" (negmask32.v32), "m" (*mem32)); 10689750Sdwmalone return (old.v8[(intptr_t)mem8 & 3]); 10789750Sdwmalone} 10824139Sjoerg 10989750Sdwmaloneuint8_t 11089750Sdwmalone__sync_val_compare_and_swap_1(uint8_t *mem8, uint8_t expected, uint8_t desired) 11189750Sdwmalone{ 11289750Sdwmalone uint32_t *mem32; 11324139Sjoerg reg_t expected32, desired32, posmask32, negmask32, old; 11489750Sdwmalone uint32_t temp; 11589750Sdwmalone 11689750Sdwmalone mem32 = (uint32_t *)((intptr_t)mem8 & ~3); 11724139Sjoerg expected32.v32 = 0x00000000; 11889750Sdwmalone expected32.v8[(intptr_t)mem8 & 3] = expected; 119 desired32.v32 = 0x00000000; 120 desired32.v8[(intptr_t)mem8 & 3] = desired; 121 posmask32.v32 = 0x00000000; 122 posmask32.v8[(intptr_t)mem8 & 3] = 0xff; 123 negmask32.v32 = ~posmask32.v32; 124 125 mips_sync(); 126 __asm volatile ( 127 "1:" 128 "\tll %0, %7\n" /* Load old value. */ 129 "\tand %2, %5, %0\n" /* Isolate affected part. */ 130 "\tbne %2, %3, 2f\n" /* Compare to expected value. */ 131 "\tand %2, %6, %0\n" /* Trim out affected part. */ 132 "\tor %2, %4\n" /* Put in the new value. */ 133 "\tsc %2, %1\n" /* Attempt to store. */ 134 "\tbeqz %2, 1b\n" /* Spin if failed. */ 135 "2:" 136 : "=&r" (old), "=m" (*mem32), "=&r" (temp) 137 : "r" (expected32.v32), "r" (desired32.v32), 138 "r" (posmask32.v32), "r" (negmask32.v32), "m" (*mem32)); 139 return (old.v8[(intptr_t)mem8 & 3]); 140} 141 142#define EMIT_ARITHMETIC_FETCH_AND_OP_1(name, op) \ 143uint8_t \ 144__sync_##name##_1(uint8_t *mem8, uint8_t val8) \ 145{ \ 146 uint32_t *mem32; \ 147 reg_t val32, posmask32, negmask32, old; \ 148 uint32_t temp1, temp2; \ 149 \ 150 mem32 = (uint32_t *)((intptr_t)mem8 & ~3); \ 151 val32.v32 = 0x00000000; \ 152 val32.v8[(intptr_t)mem8 & 3] = val8; \ 153 posmask32.v32 = 0x00000000; \ 154 posmask32.v8[(intptr_t)mem8 & 3] = 0xff; \ 155 negmask32.v32 = ~posmask32.v32; \ 156 \ 157 mips_sync(); \ 158 __asm volatile ( \ 159 "1:" \ 160 "\tll %0, %7\n" /* Load old value. */ \ 161 "\t"op" %2, %0, %4\n" /* Calculate new value. */ \ 162 "\tand %2, %5\n" /* Isolate affected part. */ \ 163 "\tand %3, %6, %0\n" /* Trim out affected part. */ \ 164 "\tor %2, %3\n" /* Put in the new value. */ \ 165 "\tsc %2, %1\n" /* Attempt to store. */ \ 166 "\tbeqz %2, 1b\n" /* Spin if failed. */ \ 167 : "=&r" (old.v32), "=m" (*mem32), "=&r" (temp1), \ 168 "=&r" (temp2) \ 169 : "r" (val32.v32), "r" (posmask32.v32), \ 170 "r" (negmask32.v32), "m" (*mem32)); \ 171 return (old.v8[(intptr_t)mem8 & 3]); \ 172} 173 174EMIT_ARITHMETIC_FETCH_AND_OP_1(fetch_and_add, "addu") 175EMIT_ARITHMETIC_FETCH_AND_OP_1(fetch_and_sub, "subu") 176 177#define EMIT_BITWISE_FETCH_AND_OP_1(name, op, idempotence) \ 178uint8_t \ 179__sync_##name##_1(uint8_t *mem8, uint8_t val8) \ 180{ \ 181 uint32_t *mem32; \ 182 reg_t val32, old; \ 183 uint32_t temp; \ 184 \ 185 mem32 = (uint32_t *)((intptr_t)mem8 & ~3); \ 186 val32.v32 = idempotence ? 0xffffffff : 0x00000000; \ 187 val32.v8[(intptr_t)mem8 & 3] = val8; \ 188 \ 189 mips_sync(); \ 190 __asm volatile ( \ 191 "1:" \ 192 "\tll %0, %4\n" /* Load old value. */ \ 193 "\t"op" %2, %3, %0\n" /* Calculate new value. */ \ 194 "\tsc %2, %1\n" /* Attempt to store. */ \ 195 "\tbeqz %2, 1b\n" /* Spin if failed. */ \ 196 : "=&r" (old.v32), "=m" (*mem32), "=&r" (temp) \ 197 : "r" (val32.v32), "m" (*mem32)); \ 198 return (old.v8[(intptr_t)mem8 & 3]); \ 199} 200 201EMIT_BITWISE_FETCH_AND_OP_1(fetch_and_and, "and", 1) 202EMIT_BITWISE_FETCH_AND_OP_1(fetch_and_or, "or", 0) 203EMIT_BITWISE_FETCH_AND_OP_1(fetch_and_xor, "xor", 0) 204 205/* 206 * 16-bit routines. 207 */ 208 209uint16_t 210__sync_lock_test_and_set_2(uint16_t *mem16, uint16_t val16) 211{ 212 uint32_t *mem32; 213 reg_t val32, negmask32, old; 214 uint32_t temp; 215 216 mem32 = (uint32_t *)((intptr_t)mem16 & ~1); 217 val32.v32 = 0x00000000; 218 val32.v16[(intptr_t)mem16 & 1] = val16; 219 negmask32.v32 = 0xffffffff; 220 negmask32.v16[(intptr_t)mem16 & 1] = 0x0000; 221 222 mips_sync(); 223 __asm volatile ( 224 "1:" 225 "\tll %0, %5\n" /* Load old value. */ 226 "\tand %2, %4, %0\n" /* Trim out affected part. */ 227 "\tor %2, %3\n" /* Combine to new value. */ 228 "\tsc %2, %1\n" /* Attempt to store. */ 229 "\tbeqz %2, 1b\n" /* Spin if failed. */ 230 : "=&r" (old.v32), "=m" (*mem32), "=&r" (temp) 231 : "r" (val32.v32), "r" (negmask32.v32), "m" (*mem32)); 232 return (old.v16[(intptr_t)mem16 & 1]); 233} 234 235uint16_t 236__sync_val_compare_and_swap_2(uint16_t *mem16, uint16_t expected, 237 uint16_t desired) 238{ 239 uint32_t *mem32; 240 reg_t expected32, desired32, posmask32, negmask32, old; 241 uint32_t temp; 242 243 mem32 = (uint32_t *)((intptr_t)mem16 & ~1); 244 expected32.v32 = 0x00000000; 245 expected32.v16[(intptr_t)mem16 & 1] = expected; 246 desired32.v32 = 0x00000000; 247 desired32.v16[(intptr_t)mem16 & 1] = desired; 248 posmask32.v32 = 0x00000000; 249 posmask32.v16[(intptr_t)mem16 & 1] = 0xffff; 250 negmask32.v32 = ~posmask32.v32; 251 252 mips_sync(); 253 __asm volatile ( 254 "1:" 255 "\tll %0, %7\n" /* Load old value. */ 256 "\tand %2, %5, %0\n" /* Isolate affected part. */ 257 "\tbne %2, %3, 2f\n" /* Compare to expected value. */ 258 "\tand %2, %6, %0\n" /* Trim out affected part. */ 259 "\tor %2, %4\n" /* Put in the new value. */ 260 "\tsc %2, %1\n" /* Attempt to store. */ 261 "\tbeqz %2, 1b\n" /* Spin if failed. */ 262 "2:" 263 : "=&r" (old), "=m" (*mem32), "=&r" (temp) 264 : "r" (expected32.v32), "r" (desired32.v32), 265 "r" (posmask32.v32), "r" (negmask32.v32), "m" (*mem32)); 266 return (old.v16[(intptr_t)mem16 & 1]); 267} 268 269#define EMIT_ARITHMETIC_FETCH_AND_OP_2(name, op) \ 270uint16_t \ 271__sync_##name##_2(uint16_t *mem16, uint16_t val16) \ 272{ \ 273 uint32_t *mem32; \ 274 reg_t val32, posmask32, negmask32, old; \ 275 uint32_t temp1, temp2; \ 276 \ 277 mem32 = (uint32_t *)((intptr_t)mem16 & ~3); \ 278 val32.v32 = 0x00000000; \ 279 val32.v16[(intptr_t)mem16 & 1] = val16; \ 280 posmask32.v32 = 0x00000000; \ 281 posmask32.v16[(intptr_t)mem16 & 1] = 0xffff; \ 282 negmask32.v32 = ~posmask32.v32; \ 283 \ 284 mips_sync(); \ 285 __asm volatile ( \ 286 "1:" \ 287 "\tll %0, %7\n" /* Load old value. */ \ 288 "\t"op" %2, %0, %4\n" /* Calculate new value. */ \ 289 "\tand %2, %5\n" /* Isolate affected part. */ \ 290 "\tand %3, %6, %0\n" /* Trim out affected part. */ \ 291 "\tor %2, %3\n" /* Combine to new value. */ \ 292 "\tsc %2, %1\n" /* Attempt to store. */ \ 293 "\tbeqz %2, 1b\n" /* Spin if failed. */ \ 294 : "=&r" (old.v32), "=m" (*mem32), "=&r" (temp1), \ 295 "=&r" (temp2) \ 296 : "r" (val32.v32), "r" (posmask32.v32), \ 297 "r" (negmask32.v32), "m" (*mem32)); \ 298 return (old.v16[(intptr_t)mem16 & 1]); \ 299} 300 301EMIT_ARITHMETIC_FETCH_AND_OP_2(fetch_and_add, "addu") 302EMIT_ARITHMETIC_FETCH_AND_OP_2(fetch_and_sub, "subu") 303 304#define EMIT_BITWISE_FETCH_AND_OP_2(name, op, idempotence) \ 305uint16_t \ 306__sync_##name##_2(uint16_t *mem16, uint16_t val16) \ 307{ \ 308 uint32_t *mem32; \ 309 reg_t val32, old; \ 310 uint32_t temp; \ 311 \ 312 mem32 = (uint32_t *)((intptr_t)mem16 & ~1); \ 313 val32.v32 = idempotence ? 0xffffffff : 0x00000000; \ 314 val32.v16[(intptr_t)mem16 & 1] = val16; \ 315 \ 316 mips_sync(); \ 317 __asm volatile ( \ 318 "1:" \ 319 "\tll %0, %4\n" /* Load old value. */ \ 320 "\t"op" %2, %3, %0\n" /* Calculate new value. */ \ 321 "\tsc %2, %1\n" /* Attempt to store. */ \ 322 "\tbeqz %2, 1b\n" /* Spin if failed. */ \ 323 : "=&r" (old.v32), "=m" (*mem32), "=&r" (temp) \ 324 : "r" (val32.v32), "m" (*mem32)); \ 325 return (old.v16[(intptr_t)mem16 & 1]); \ 326} 327 328EMIT_BITWISE_FETCH_AND_OP_2(fetch_and_and, "and", 1) 329EMIT_BITWISE_FETCH_AND_OP_2(fetch_and_or, "or", 0) 330EMIT_BITWISE_FETCH_AND_OP_2(fetch_and_xor, "xor", 0) 331 332/* 333 * 32-bit routines. 334 */ 335 336uint32_t 337__sync_val_compare_and_swap_4(uint32_t *mem, uint32_t expected, 338 uint32_t desired) 339{ 340 uint32_t old, temp; 341 342 mips_sync(); 343 __asm volatile ( 344 "1:" 345 "\tll %0, %5\n" /* Load old value. */ 346 "\tbne %0, %3, 2f\n" /* Compare to expected value. */ 347 "\tmove %2, %4\n" /* Value to store. */ 348 "\tsc %2, %1\n" /* Attempt to store. */ 349 "\tbeqz %2, 1b\n" /* Spin if failed. */ 350 "2:" 351 : "=&r" (old), "=m" (*mem), "=&r" (temp) 352 : "r" (expected), "r" (desired), "m" (*mem)); 353 return (old); 354} 355 356#define EMIT_FETCH_AND_OP_4(name, op) \ 357uint32_t \ 358__sync_##name##_4(uint32_t *mem, uint32_t val) \ 359{ \ 360 uint32_t old, temp; \ 361 \ 362 mips_sync(); \ 363 __asm volatile ( \ 364 "1:" \ 365 "\tll %0, %4\n" /* Load old value. */ \ 366 "\t"op"\n" /* Calculate new value. */ \ 367 "\tsc %2, %1\n" /* Attempt to store. */ \ 368 "\tbeqz %2, 1b\n" /* Spin if failed. */ \ 369 : "=&r" (old), "=m" (*mem), "=&r" (temp) \ 370 : "r" (val), "m" (*mem)); \ 371 return (old); \ 372} 373 374EMIT_FETCH_AND_OP_4(lock_test_and_set, "move %2, %3") 375EMIT_FETCH_AND_OP_4(fetch_and_add, "addu %2, %0, %3") 376EMIT_FETCH_AND_OP_4(fetch_and_and, "and %2, %0, %3") 377EMIT_FETCH_AND_OP_4(fetch_and_or, "or %2, %0, %3") 378EMIT_FETCH_AND_OP_4(fetch_and_sub, "subu %2, %0, %3") 379EMIT_FETCH_AND_OP_4(fetch_and_xor, "xor %2, %0, %3") 380 381/* 382 * 64-bit routines. 383 * 384 * Note: All the 64-bit atomic operations are only atomic when running 385 * in 64-bit mode. It is assumed that code compiled for n32 and n64 fits 386 * into this definition and no further safeties are needed. 387 */ 388 389#if defined(__mips_n32) || defined(__mips_n64) 390 391uint64_t 392__sync_val_compare_and_swap_8(uint64_t *mem, uint64_t expected, 393 uint64_t desired) 394{ 395 uint64_t old, temp; 396 397 mips_sync(); 398 __asm volatile ( 399 "1:" 400 "\tlld %0, %5\n" /* Load old value. */ 401 "\tbne %0, %3, 2f\n" /* Compare to expected value. */ 402 "\tmove %2, %4\n" /* Value to store. */ 403 "\tscd %2, %1\n" /* Attempt to store. */ 404 "\tbeqz %2, 1b\n" /* Spin if failed. */ 405 "2:" 406 : "=&r" (old), "=m" (*mem), "=&r" (temp) 407 : "r" (expected), "r" (desired), "m" (*mem)); 408 return (old); 409} 410 411#define EMIT_FETCH_AND_OP_8(name, op) \ 412uint64_t \ 413__sync_##name##_8(uint64_t *mem, uint64_t val) \ 414{ \ 415 uint64_t old, temp; \ 416 \ 417 mips_sync(); \ 418 __asm volatile ( \ 419 "1:" \ 420 "\tlld %0, %4\n" /* Load old value. */ \ 421 "\t"op"\n" /* Calculate new value. */ \ 422 "\tscd %2, %1\n" /* Attempt to store. */ \ 423 "\tbeqz %2, 1b\n" /* Spin if failed. */ \ 424 : "=&r" (old), "=m" (*mem), "=&r" (temp) \ 425 : "r" (val), "m" (*mem)); \ 426 return (old); \ 427} 428 429EMIT_FETCH_AND_OP_8(lock_test_and_set, "move %2, %3") 430EMIT_FETCH_AND_OP_8(fetch_and_add, "daddu %2, %0, %3") 431EMIT_FETCH_AND_OP_8(fetch_and_and, "and %2, %0, %3") 432EMIT_FETCH_AND_OP_8(fetch_and_or, "or %2, %0, %3") 433EMIT_FETCH_AND_OP_8(fetch_and_sub, "dsubu %2, %0, %3") 434EMIT_FETCH_AND_OP_8(fetch_and_xor, "xor %2, %0, %3") 435 436#endif /* __mips_n32 || __mips_n64 */ 437