1/* 2 * Copyright 2009-2015 Samy Al Bahra. 3 * Copyright 2011 David Joseph. 4 * All rights reserved. 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25 * SUCH DAMAGE. 26 */ 27 28#ifndef CK_PR_H 29#define CK_PR_H 30 31#include <ck_cc.h> 32#include <ck_limits.h> 33#include <ck_md.h> 34#include <ck_stdint.h> 35#include <ck_stdbool.h> 36 37#ifndef CK_USE_CC_BUILTINS 38#if defined(__x86_64__) 39#include "gcc/x86_64/ck_pr.h" 40#elif defined(__x86__) 41#include "gcc/x86/ck_pr.h" 42#elif defined(__sparcv9__) 43#include "gcc/sparcv9/ck_pr.h" 44#elif defined(__ppc64__) 45#include "gcc/ppc64/ck_pr.h" 46#elif defined(__s390x__) 47#include "gcc/s390x/ck_pr.h" 48#elif defined(__ppc__) 49#include "gcc/ppc/ck_pr.h" 50#elif defined(__arm__) 51#if __ARM_ARCH >= 6 52#include "gcc/arm/ck_pr.h" 53#else 54#include "gcc/arm/ck_pr_armv4.h" 55#endif 56#elif defined(__aarch64__) 57#include "gcc/aarch64/ck_pr.h" 58#elif !defined(__GNUC__) 59#error Your platform is unsupported 60#endif 61#endif /* !CK_USE_CC_BUILTINS */ 62 63#if defined(__GNUC__) 64#include "gcc/ck_pr.h" 65#endif 66 67#define CK_PR_FENCE_EMIT(T) \ 68 CK_CC_INLINE static void \ 69 ck_pr_fence_##T(void) \ 70 { \ 71 ck_pr_fence_strict_##T(); \ 72 return; \ 73 } 74#define CK_PR_FENCE_NOOP(T) \ 75 CK_CC_INLINE static void \ 76 ck_pr_fence_##T(void) \ 77 { \ 78 ck_pr_barrier(); \ 79 return; \ 80 } 81 82/* 83 * None of the currently supported platforms allow for data-dependent 84 * load ordering. 85 */ 86CK_PR_FENCE_NOOP(load_depends) 87#define ck_pr_fence_strict_load_depends ck_pr_fence_load_depends 88 89/* 90 * In memory models where atomic operations do not have serializing 91 * effects, atomic read-modify-write operations are modeled as stores. 92 */ 93#if defined(CK_MD_RMO) 94/* 95 * Only stores to the same location have a global 96 * ordering. 97 */ 98CK_PR_FENCE_EMIT(atomic) 99CK_PR_FENCE_EMIT(atomic_load) 100CK_PR_FENCE_EMIT(atomic_store) 101CK_PR_FENCE_EMIT(store_atomic) 102CK_PR_FENCE_EMIT(load_atomic) 103CK_PR_FENCE_EMIT(load_store) 104CK_PR_FENCE_EMIT(store_load) 105CK_PR_FENCE_EMIT(load) 106CK_PR_FENCE_EMIT(store) 107CK_PR_FENCE_EMIT(memory) 108CK_PR_FENCE_EMIT(acquire) 109CK_PR_FENCE_EMIT(release) 110CK_PR_FENCE_EMIT(acqrel) 111CK_PR_FENCE_EMIT(lock) 112CK_PR_FENCE_EMIT(unlock) 113#elif defined(CK_MD_PSO) 114/* 115 * Anything can be re-ordered with respect to stores. 116 * Otherwise, loads are executed in-order. 117 */ 118CK_PR_FENCE_EMIT(atomic) 119CK_PR_FENCE_NOOP(atomic_load) 120CK_PR_FENCE_EMIT(atomic_store) 121CK_PR_FENCE_EMIT(store_atomic) 122CK_PR_FENCE_NOOP(load_atomic) 123CK_PR_FENCE_EMIT(load_store) 124CK_PR_FENCE_EMIT(store_load) 125CK_PR_FENCE_NOOP(load) 126CK_PR_FENCE_EMIT(store) 127CK_PR_FENCE_EMIT(memory) 128CK_PR_FENCE_EMIT(acquire) 129CK_PR_FENCE_EMIT(release) 130CK_PR_FENCE_EMIT(acqrel) 131CK_PR_FENCE_EMIT(lock) 132CK_PR_FENCE_EMIT(unlock) 133#elif defined(CK_MD_TSO) 134/* 135 * Only loads are re-ordered and only with respect to 136 * prior stores. Atomic operations are serializing. 137 */ 138CK_PR_FENCE_NOOP(atomic) 139CK_PR_FENCE_NOOP(atomic_load) 140CK_PR_FENCE_NOOP(atomic_store) 141CK_PR_FENCE_NOOP(store_atomic) 142CK_PR_FENCE_NOOP(load_atomic) 143CK_PR_FENCE_NOOP(load_store) 144CK_PR_FENCE_EMIT(store_load) 145CK_PR_FENCE_NOOP(load) 146CK_PR_FENCE_NOOP(store) 147CK_PR_FENCE_EMIT(memory) 148CK_PR_FENCE_NOOP(acquire) 149CK_PR_FENCE_NOOP(release) 150CK_PR_FENCE_NOOP(acqrel) 151CK_PR_FENCE_NOOP(lock) 152CK_PR_FENCE_NOOP(unlock) 153#else 154#error "No memory model has been defined." 155#endif /* CK_MD_TSO */ 156 157#undef CK_PR_FENCE_EMIT 158#undef CK_PR_FENCE_NOOP 159 160#ifndef CK_F_PR_RFO 161#define CK_F_PR_RFO 162CK_CC_INLINE static void 163ck_pr_rfo(const void *m) 164{ 165 166 (void)m; 167 return; 168} 169#endif /* CK_F_PR_RFO */ 170 171#define CK_PR_STORE_SAFE(DST, VAL, TYPE) \ 172 ck_pr_md_store_##TYPE( \ 173 ((void)sizeof(*(DST) = (VAL)), (DST)), \ 174 (VAL)) 175 176#define ck_pr_store_ptr(DST, VAL) CK_PR_STORE_SAFE((DST), (VAL), ptr) 177#define ck_pr_store_char(DST, VAL) CK_PR_STORE_SAFE((DST), (VAL), char) 178#ifndef CK_PR_DISABLE_DOUBLE 179#define ck_pr_store_double(DST, VAL) CK_PR_STORE_SAFE((DST), (VAL), double) 180#endif 181#define ck_pr_store_uint(DST, VAL) CK_PR_STORE_SAFE((DST), (VAL), uint) 182#define ck_pr_store_int(DST, VAL) CK_PR_STORE_SAFE((DST), (VAL), int) 183#define ck_pr_store_32(DST, VAL) CK_PR_STORE_SAFE((DST), (VAL), 32) 184#define ck_pr_store_16(DST, VAL) CK_PR_STORE_SAFE((DST), (VAL), 16) 185#define ck_pr_store_8(DST, VAL) CK_PR_STORE_SAFE((DST), (VAL), 8) 186 187#define ck_pr_store_ptr_unsafe(DST, VAL) ck_pr_md_store_ptr((DST), (VAL)) 188 189#ifdef CK_F_PR_LOAD_64 190#define ck_pr_store_64(DST, VAL) CK_PR_STORE_SAFE((DST), (VAL), 64) 191#endif /* CK_F_PR_LOAD_64 */ 192 193#define CK_PR_LOAD_PTR_SAFE(SRC) (CK_CC_TYPEOF(*(SRC), (void *)))ck_pr_md_load_ptr((SRC)) 194#define ck_pr_load_ptr(SRC) CK_PR_LOAD_PTR_SAFE((SRC)) 195 196#define CK_PR_LOAD_SAFE(SRC, TYPE) ck_pr_md_load_##TYPE((SRC)) 197#define ck_pr_load_char(SRC) CK_PR_LOAD_SAFE((SRC), char) 198#ifndef CK_PR_DISABLE_DOUBLE 199#define ck_pr_load_double(SRC) CK_PR_LOAD_SAFE((SRC), double) 200#endif 201#define ck_pr_load_uint(SRC) CK_PR_LOAD_SAFE((SRC), uint) 202#define ck_pr_load_int(SRC) CK_PR_LOAD_SAFE((SRC), int) 203#define ck_pr_load_32(SRC) CK_PR_LOAD_SAFE((SRC), 32) 204#define ck_pr_load_16(SRC) CK_PR_LOAD_SAFE((SRC), 16) 205#define ck_pr_load_8(SRC) CK_PR_LOAD_SAFE((SRC), 8) 206 207#ifdef CK_F_PR_LOAD_64 208#define ck_pr_load_64(SRC) CK_PR_LOAD_SAFE((SRC), 64) 209#endif /* CK_F_PR_LOAD_64 */ 210 211#define CK_PR_BIN(K, S, M, T, P, C) \ 212 CK_CC_INLINE static void \ 213 ck_pr_##K##_##S(M *target, T value) \ 214 { \ 215 T previous; \ 216 C punt; \ 217 punt = ck_pr_md_load_##S(target); \ 218 previous = (T)punt; \ 219 while (ck_pr_cas_##S##_value(target, \ 220 (C)previous, \ 221 (C)(previous P value), \ 222 &previous) == false) \ 223 ck_pr_stall(); \ 224 \ 225 return; \ 226 } 227 228#define CK_PR_BIN_S(K, S, T, P) CK_PR_BIN(K, S, T, T, P, T) 229 230#if defined(CK_F_PR_LOAD_CHAR) && defined(CK_F_PR_CAS_CHAR_VALUE) 231 232#ifndef CK_F_PR_ADD_CHAR 233#define CK_F_PR_ADD_CHAR 234CK_PR_BIN_S(add, char, char, +) 235#endif /* CK_F_PR_ADD_CHAR */ 236 237#ifndef CK_F_PR_SUB_CHAR 238#define CK_F_PR_SUB_CHAR 239CK_PR_BIN_S(sub, char, char, -) 240#endif /* CK_F_PR_SUB_CHAR */ 241 242#ifndef CK_F_PR_AND_CHAR 243#define CK_F_PR_AND_CHAR 244CK_PR_BIN_S(and, char, char, &) 245#endif /* CK_F_PR_AND_CHAR */ 246 247#ifndef CK_F_PR_XOR_CHAR 248#define CK_F_PR_XOR_CHAR 249CK_PR_BIN_S(xor, char, char, ^) 250#endif /* CK_F_PR_XOR_CHAR */ 251 252#ifndef CK_F_PR_OR_CHAR 253#define CK_F_PR_OR_CHAR 254CK_PR_BIN_S(or, char, char, |) 255#endif /* CK_F_PR_OR_CHAR */ 256 257#endif /* CK_F_PR_LOAD_CHAR && CK_F_PR_CAS_CHAR_VALUE */ 258 259#if defined(CK_F_PR_LOAD_INT) && defined(CK_F_PR_CAS_INT_VALUE) 260 261#ifndef CK_F_PR_ADD_INT 262#define CK_F_PR_ADD_INT 263CK_PR_BIN_S(add, int, int, +) 264#endif /* CK_F_PR_ADD_INT */ 265 266#ifndef CK_F_PR_SUB_INT 267#define CK_F_PR_SUB_INT 268CK_PR_BIN_S(sub, int, int, -) 269#endif /* CK_F_PR_SUB_INT */ 270 271#ifndef CK_F_PR_AND_INT 272#define CK_F_PR_AND_INT 273CK_PR_BIN_S(and, int, int, &) 274#endif /* CK_F_PR_AND_INT */ 275 276#ifndef CK_F_PR_XOR_INT 277#define CK_F_PR_XOR_INT 278CK_PR_BIN_S(xor, int, int, ^) 279#endif /* CK_F_PR_XOR_INT */ 280 281#ifndef CK_F_PR_OR_INT 282#define CK_F_PR_OR_INT 283CK_PR_BIN_S(or, int, int, |) 284#endif /* CK_F_PR_OR_INT */ 285 286#endif /* CK_F_PR_LOAD_INT && CK_F_PR_CAS_INT_VALUE */ 287 288#if defined(CK_F_PR_LOAD_DOUBLE) && defined(CK_F_PR_CAS_DOUBLE_VALUE) && \ 289 !defined(CK_PR_DISABLE_DOUBLE) 290 291#ifndef CK_F_PR_ADD_DOUBLE 292#define CK_F_PR_ADD_DOUBLE 293CK_PR_BIN_S(add, double, double, +) 294#endif /* CK_F_PR_ADD_DOUBLE */ 295 296#ifndef CK_F_PR_SUB_DOUBLE 297#define CK_F_PR_SUB_DOUBLE 298CK_PR_BIN_S(sub, double, double, -) 299#endif /* CK_F_PR_SUB_DOUBLE */ 300 301#endif /* CK_F_PR_LOAD_DOUBLE && CK_F_PR_CAS_DOUBLE_VALUE && !CK_PR_DISABLE_DOUBLE */ 302 303#if defined(CK_F_PR_LOAD_UINT) && defined(CK_F_PR_CAS_UINT_VALUE) 304 305#ifndef CK_F_PR_ADD_UINT 306#define CK_F_PR_ADD_UINT 307CK_PR_BIN_S(add, uint, unsigned int, +) 308#endif /* CK_F_PR_ADD_UINT */ 309 310#ifndef CK_F_PR_SUB_UINT 311#define CK_F_PR_SUB_UINT 312CK_PR_BIN_S(sub, uint, unsigned int, -) 313#endif /* CK_F_PR_SUB_UINT */ 314 315#ifndef CK_F_PR_AND_UINT 316#define CK_F_PR_AND_UINT 317CK_PR_BIN_S(and, uint, unsigned int, &) 318#endif /* CK_F_PR_AND_UINT */ 319 320#ifndef CK_F_PR_XOR_UINT 321#define CK_F_PR_XOR_UINT 322CK_PR_BIN_S(xor, uint, unsigned int, ^) 323#endif /* CK_F_PR_XOR_UINT */ 324 325#ifndef CK_F_PR_OR_UINT 326#define CK_F_PR_OR_UINT 327CK_PR_BIN_S(or, uint, unsigned int, |) 328#endif /* CK_F_PR_OR_UINT */ 329 330#endif /* CK_F_PR_LOAD_UINT && CK_F_PR_CAS_UINT_VALUE */ 331 332#if defined(CK_F_PR_LOAD_PTR) && defined(CK_F_PR_CAS_PTR_VALUE) 333 334#ifndef CK_F_PR_ADD_PTR 335#define CK_F_PR_ADD_PTR 336CK_PR_BIN(add, ptr, void, uintptr_t, +, void *) 337#endif /* CK_F_PR_ADD_PTR */ 338 339#ifndef CK_F_PR_SUB_PTR 340#define CK_F_PR_SUB_PTR 341CK_PR_BIN(sub, ptr, void, uintptr_t, -, void *) 342#endif /* CK_F_PR_SUB_PTR */ 343 344#ifndef CK_F_PR_AND_PTR 345#define CK_F_PR_AND_PTR 346CK_PR_BIN(and, ptr, void, uintptr_t, &, void *) 347#endif /* CK_F_PR_AND_PTR */ 348 349#ifndef CK_F_PR_XOR_PTR 350#define CK_F_PR_XOR_PTR 351CK_PR_BIN(xor, ptr, void, uintptr_t, ^, void *) 352#endif /* CK_F_PR_XOR_PTR */ 353 354#ifndef CK_F_PR_OR_PTR 355#define CK_F_PR_OR_PTR 356CK_PR_BIN(or, ptr, void, uintptr_t, |, void *) 357#endif /* CK_F_PR_OR_PTR */ 358 359#endif /* CK_F_PR_LOAD_PTR && CK_F_PR_CAS_PTR_VALUE */ 360 361#if defined(CK_F_PR_LOAD_64) && defined(CK_F_PR_CAS_64_VALUE) 362 363#ifndef CK_F_PR_ADD_64 364#define CK_F_PR_ADD_64 365CK_PR_BIN_S(add, 64, uint64_t, +) 366#endif /* CK_F_PR_ADD_64 */ 367 368#ifndef CK_F_PR_SUB_64 369#define CK_F_PR_SUB_64 370CK_PR_BIN_S(sub, 64, uint64_t, -) 371#endif /* CK_F_PR_SUB_64 */ 372 373#ifndef CK_F_PR_AND_64 374#define CK_F_PR_AND_64 375CK_PR_BIN_S(and, 64, uint64_t, &) 376#endif /* CK_F_PR_AND_64 */ 377 378#ifndef CK_F_PR_XOR_64 379#define CK_F_PR_XOR_64 380CK_PR_BIN_S(xor, 64, uint64_t, ^) 381#endif /* CK_F_PR_XOR_64 */ 382 383#ifndef CK_F_PR_OR_64 384#define CK_F_PR_OR_64 385CK_PR_BIN_S(or, 64, uint64_t, |) 386#endif /* CK_F_PR_OR_64 */ 387 388#endif /* CK_F_PR_LOAD_64 && CK_F_PR_CAS_64_VALUE */ 389 390#if defined(CK_F_PR_LOAD_32) && defined(CK_F_PR_CAS_32_VALUE) 391 392#ifndef CK_F_PR_ADD_32 393#define CK_F_PR_ADD_32 394CK_PR_BIN_S(add, 32, uint32_t, +) 395#endif /* CK_F_PR_ADD_32 */ 396 397#ifndef CK_F_PR_SUB_32 398#define CK_F_PR_SUB_32 399CK_PR_BIN_S(sub, 32, uint32_t, -) 400#endif /* CK_F_PR_SUB_32 */ 401 402#ifndef CK_F_PR_AND_32 403#define CK_F_PR_AND_32 404CK_PR_BIN_S(and, 32, uint32_t, &) 405#endif /* CK_F_PR_AND_32 */ 406 407#ifndef CK_F_PR_XOR_32 408#define CK_F_PR_XOR_32 409CK_PR_BIN_S(xor, 32, uint32_t, ^) 410#endif /* CK_F_PR_XOR_32 */ 411 412#ifndef CK_F_PR_OR_32 413#define CK_F_PR_OR_32 414CK_PR_BIN_S(or, 32, uint32_t, |) 415#endif /* CK_F_PR_OR_32 */ 416 417#endif /* CK_F_PR_LOAD_32 && CK_F_PR_CAS_32_VALUE */ 418 419#if defined(CK_F_PR_LOAD_16) && defined(CK_F_PR_CAS_16_VALUE) 420 421#ifndef CK_F_PR_ADD_16 422#define CK_F_PR_ADD_16 423CK_PR_BIN_S(add, 16, uint16_t, +) 424#endif /* CK_F_PR_ADD_16 */ 425 426#ifndef CK_F_PR_SUB_16 427#define CK_F_PR_SUB_16 428CK_PR_BIN_S(sub, 16, uint16_t, -) 429#endif /* CK_F_PR_SUB_16 */ 430 431#ifndef CK_F_PR_AND_16 432#define CK_F_PR_AND_16 433CK_PR_BIN_S(and, 16, uint16_t, &) 434#endif /* CK_F_PR_AND_16 */ 435 436#ifndef CK_F_PR_XOR_16 437#define CK_F_PR_XOR_16 438CK_PR_BIN_S(xor, 16, uint16_t, ^) 439#endif /* CK_F_PR_XOR_16 */ 440 441#ifndef CK_F_PR_OR_16 442#define CK_F_PR_OR_16 443CK_PR_BIN_S(or, 16, uint16_t, |) 444#endif /* CK_F_PR_OR_16 */ 445 446#endif /* CK_F_PR_LOAD_16 && CK_F_PR_CAS_16_VALUE */ 447 448#if defined(CK_F_PR_LOAD_8) && defined(CK_F_PR_CAS_8_VALUE) 449 450#ifndef CK_F_PR_ADD_8 451#define CK_F_PR_ADD_8 452CK_PR_BIN_S(add, 8, uint8_t, +) 453#endif /* CK_F_PR_ADD_8 */ 454 455#ifndef CK_F_PR_SUB_8 456#define CK_F_PR_SUB_8 457CK_PR_BIN_S(sub, 8, uint8_t, -) 458#endif /* CK_F_PR_SUB_8 */ 459 460#ifndef CK_F_PR_AND_8 461#define CK_F_PR_AND_8 462CK_PR_BIN_S(and, 8, uint8_t, &) 463#endif /* CK_F_PR_AND_8 */ 464 465#ifndef CK_F_PR_XOR_8 466#define CK_F_PR_XOR_8 467CK_PR_BIN_S(xor, 8, uint8_t, ^) 468#endif /* CK_F_PR_XOR_8 */ 469 470#ifndef CK_F_PR_OR_8 471#define CK_F_PR_OR_8 472CK_PR_BIN_S(or, 8, uint8_t, |) 473#endif /* CK_F_PR_OR_8 */ 474 475#endif /* CK_F_PR_LOAD_8 && CK_F_PR_CAS_8_VALUE */ 476 477#undef CK_PR_BIN_S 478#undef CK_PR_BIN 479 480#define CK_PR_BTX(K, S, M, T, P, C, R) \ 481 CK_CC_INLINE static bool \ 482 ck_pr_##K##_##S(M *target, unsigned int offset) \ 483 { \ 484 T previous; \ 485 C punt; \ 486 punt = ck_pr_md_load_##S(target); \ 487 previous = (T)punt; \ 488 while (ck_pr_cas_##S##_value(target, (C)previous, \ 489 (C)(previous P (R ((T)1 << offset))), &previous) == false) \ 490 ck_pr_stall(); \ 491 return ((previous >> offset) & 1); \ 492 } 493 494#define CK_PR_BTX_S(K, S, T, P, R) CK_PR_BTX(K, S, T, T, P, T, R) 495 496#if defined(CK_F_PR_LOAD_INT) && defined(CK_F_PR_CAS_INT_VALUE) 497 498#ifndef CK_F_PR_BTC_INT 499#define CK_F_PR_BTC_INT 500CK_PR_BTX_S(btc, int, int, ^,) 501#endif /* CK_F_PR_BTC_INT */ 502 503#ifndef CK_F_PR_BTR_INT 504#define CK_F_PR_BTR_INT 505CK_PR_BTX_S(btr, int, int, &, ~) 506#endif /* CK_F_PR_BTR_INT */ 507 508#ifndef CK_F_PR_BTS_INT 509#define CK_F_PR_BTS_INT 510CK_PR_BTX_S(bts, int, int, |,) 511#endif /* CK_F_PR_BTS_INT */ 512 513#endif /* CK_F_PR_LOAD_INT && CK_F_PR_CAS_INT_VALUE */ 514 515#if defined(CK_F_PR_LOAD_UINT) && defined(CK_F_PR_CAS_UINT_VALUE) 516 517#ifndef CK_F_PR_BTC_UINT 518#define CK_F_PR_BTC_UINT 519CK_PR_BTX_S(btc, uint, unsigned int, ^,) 520#endif /* CK_F_PR_BTC_UINT */ 521 522#ifndef CK_F_PR_BTR_UINT 523#define CK_F_PR_BTR_UINT 524CK_PR_BTX_S(btr, uint, unsigned int, &, ~) 525#endif /* CK_F_PR_BTR_UINT */ 526 527#ifndef CK_F_PR_BTS_UINT 528#define CK_F_PR_BTS_UINT 529CK_PR_BTX_S(bts, uint, unsigned int, |,) 530#endif /* CK_F_PR_BTS_UINT */ 531 532#endif /* CK_F_PR_LOAD_UINT && CK_F_PR_CAS_UINT_VALUE */ 533 534#if defined(CK_F_PR_LOAD_PTR) && defined(CK_F_PR_CAS_PTR_VALUE) 535 536#ifndef CK_F_PR_BTC_PTR 537#define CK_F_PR_BTC_PTR 538CK_PR_BTX(btc, ptr, void, uintptr_t, ^, void *,) 539#endif /* CK_F_PR_BTC_PTR */ 540 541#ifndef CK_F_PR_BTR_PTR 542#define CK_F_PR_BTR_PTR 543CK_PR_BTX(btr, ptr, void, uintptr_t, &, void *, ~) 544#endif /* CK_F_PR_BTR_PTR */ 545 546#ifndef CK_F_PR_BTS_PTR 547#define CK_F_PR_BTS_PTR 548CK_PR_BTX(bts, ptr, void, uintptr_t, |, void *,) 549#endif /* CK_F_PR_BTS_PTR */ 550 551#endif /* CK_F_PR_LOAD_PTR && CK_F_PR_CAS_PTR_VALUE */ 552 553#if defined(CK_F_PR_LOAD_64) && defined(CK_F_PR_CAS_64_VALUE) 554 555#ifndef CK_F_PR_BTC_64 556#define CK_F_PR_BTC_64 557CK_PR_BTX_S(btc, 64, uint64_t, ^,) 558#endif /* CK_F_PR_BTC_64 */ 559 560#ifndef CK_F_PR_BTR_64 561#define CK_F_PR_BTR_64 562CK_PR_BTX_S(btr, 64, uint64_t, &, ~) 563#endif /* CK_F_PR_BTR_64 */ 564 565#ifndef CK_F_PR_BTS_64 566#define CK_F_PR_BTS_64 567CK_PR_BTX_S(bts, 64, uint64_t, |,) 568#endif /* CK_F_PR_BTS_64 */ 569 570#endif /* CK_F_PR_LOAD_64 && CK_F_PR_CAS_64_VALUE */ 571 572#if defined(CK_F_PR_LOAD_32) && defined(CK_F_PR_CAS_32_VALUE) 573 574#ifndef CK_F_PR_BTC_32 575#define CK_F_PR_BTC_32 576CK_PR_BTX_S(btc, 32, uint32_t, ^,) 577#endif /* CK_F_PR_BTC_32 */ 578 579#ifndef CK_F_PR_BTR_32 580#define CK_F_PR_BTR_32 581CK_PR_BTX_S(btr, 32, uint32_t, &, ~) 582#endif /* CK_F_PR_BTR_32 */ 583 584#ifndef CK_F_PR_BTS_32 585#define CK_F_PR_BTS_32 586CK_PR_BTX_S(bts, 32, uint32_t, |,) 587#endif /* CK_F_PR_BTS_32 */ 588 589#endif /* CK_F_PR_LOAD_32 && CK_F_PR_CAS_32_VALUE */ 590 591#if defined(CK_F_PR_LOAD_16) && defined(CK_F_PR_CAS_16_VALUE) 592 593#ifndef CK_F_PR_BTC_16 594#define CK_F_PR_BTC_16 595CK_PR_BTX_S(btc, 16, uint16_t, ^,) 596#endif /* CK_F_PR_BTC_16 */ 597 598#ifndef CK_F_PR_BTR_16 599#define CK_F_PR_BTR_16 600CK_PR_BTX_S(btr, 16, uint16_t, &, ~) 601#endif /* CK_F_PR_BTR_16 */ 602 603#ifndef CK_F_PR_BTS_16 604#define CK_F_PR_BTS_16 605CK_PR_BTX_S(bts, 16, uint16_t, |,) 606#endif /* CK_F_PR_BTS_16 */ 607 608#endif /* CK_F_PR_LOAD_16 && CK_F_PR_CAS_16_VALUE */ 609 610#undef CK_PR_BTX_S 611#undef CK_PR_BTX 612 613#define CK_PR_UNARY(K, X, S, M, T) \ 614 CK_CC_INLINE static void \ 615 ck_pr_##K##_##S(M *target) \ 616 { \ 617 ck_pr_##X##_##S(target, (T)1); \ 618 return; \ 619 } 620 621#define CK_PR_UNARY_Z(K, S, M, T, P, C, Z) \ 622 CK_CC_INLINE static bool \ 623 ck_pr_##K##_##S##_is_zero(M *target) \ 624 { \ 625 T previous; \ 626 C punt; \ 627 punt = (C)ck_pr_md_load_##S(target); \ 628 previous = (T)punt; \ 629 while (ck_pr_cas_##S##_value(target, \ 630 (C)previous, \ 631 (C)(previous P 1), \ 632 &previous) == false) \ 633 ck_pr_stall(); \ 634 return previous == (T)Z; \ 635 } 636 637#define CK_PR_UNARY_Z_STUB(K, S, M) \ 638 CK_CC_INLINE static void \ 639 ck_pr_##K##_##S##_zero(M *target, bool *zero) \ 640 { \ 641 *zero = ck_pr_##K##_##S##_is_zero(target); \ 642 return; \ 643 } 644 645#define CK_PR_UNARY_S(K, X, S, M) CK_PR_UNARY(K, X, S, M, M) 646#define CK_PR_UNARY_Z_S(K, S, M, P, Z) \ 647 CK_PR_UNARY_Z(K, S, M, M, P, M, Z) \ 648 CK_PR_UNARY_Z_STUB(K, S, M) 649 650#if defined(CK_F_PR_LOAD_CHAR) && defined(CK_F_PR_CAS_CHAR_VALUE) 651 652#ifndef CK_F_PR_INC_CHAR 653#define CK_F_PR_INC_CHAR 654CK_PR_UNARY_S(inc, add, char, char) 655#endif /* CK_F_PR_INC_CHAR */ 656 657#ifndef CK_F_PR_INC_CHAR_ZERO 658#define CK_F_PR_INC_CHAR_ZERO 659CK_PR_UNARY_Z_S(inc, char, char, +, -1) 660#else 661CK_PR_UNARY_Z_STUB(inc, char, char) 662#endif /* CK_F_PR_INC_CHAR_ZERO */ 663 664#ifndef CK_F_PR_DEC_CHAR 665#define CK_F_PR_DEC_CHAR 666CK_PR_UNARY_S(dec, sub, char, char) 667#endif /* CK_F_PR_DEC_CHAR */ 668 669#ifndef CK_F_PR_DEC_CHAR_ZERO 670#define CK_F_PR_DEC_CHAR_ZERO 671CK_PR_UNARY_Z_S(dec, char, char, -, 1) 672#else 673CK_PR_UNARY_Z_STUB(dec, char, char) 674#endif /* CK_F_PR_DEC_CHAR_ZERO */ 675 676#endif /* CK_F_PR_LOAD_CHAR && CK_F_PR_CAS_CHAR_VALUE */ 677 678#if defined(CK_F_PR_LOAD_INT) && defined(CK_F_PR_CAS_INT_VALUE) 679 680#ifndef CK_F_PR_INC_INT 681#define CK_F_PR_INC_INT 682CK_PR_UNARY_S(inc, add, int, int) 683#endif /* CK_F_PR_INC_INT */ 684 685#ifndef CK_F_PR_INC_INT_ZERO 686#define CK_F_PR_INC_INT_ZERO 687CK_PR_UNARY_Z_S(inc, int, int, +, -1) 688#else 689CK_PR_UNARY_Z_STUB(inc, int, int) 690#endif /* CK_F_PR_INC_INT_ZERO */ 691 692#ifndef CK_F_PR_DEC_INT 693#define CK_F_PR_DEC_INT 694CK_PR_UNARY_S(dec, sub, int, int) 695#endif /* CK_F_PR_DEC_INT */ 696 697#ifndef CK_F_PR_DEC_INT_ZERO 698#define CK_F_PR_DEC_INT_ZERO 699CK_PR_UNARY_Z_S(dec, int, int, -, 1) 700#else 701CK_PR_UNARY_Z_STUB(dec, int, int) 702#endif /* CK_F_PR_DEC_INT_ZERO */ 703 704#endif /* CK_F_PR_LOAD_INT && CK_F_PR_CAS_INT_VALUE */ 705 706#if defined(CK_F_PR_LOAD_DOUBLE) && defined(CK_F_PR_CAS_DOUBLE_VALUE) && \ 707 !defined(CK_PR_DISABLE_DOUBLE) 708 709#ifndef CK_F_PR_INC_DOUBLE 710#define CK_F_PR_INC_DOUBLE 711CK_PR_UNARY_S(inc, add, double, double) 712#endif /* CK_F_PR_INC_DOUBLE */ 713 714#ifndef CK_F_PR_DEC_DOUBLE 715#define CK_F_PR_DEC_DOUBLE 716CK_PR_UNARY_S(dec, sub, double, double) 717#endif /* CK_F_PR_DEC_DOUBLE */ 718 719#endif /* CK_F_PR_LOAD_DOUBLE && CK_F_PR_CAS_DOUBLE_VALUE && !CK_PR_DISABLE_DOUBLE */ 720 721#if defined(CK_F_PR_LOAD_UINT) && defined(CK_F_PR_CAS_UINT_VALUE) 722 723#ifndef CK_F_PR_INC_UINT 724#define CK_F_PR_INC_UINT 725CK_PR_UNARY_S(inc, add, uint, unsigned int) 726#endif /* CK_F_PR_INC_UINT */ 727 728#ifndef CK_F_PR_INC_UINT_ZERO 729#define CK_F_PR_INC_UINT_ZERO 730CK_PR_UNARY_Z_S(inc, uint, unsigned int, +, UINT_MAX) 731#else 732CK_PR_UNARY_Z_STUB(inc, uint, unsigned int) 733#endif /* CK_F_PR_INC_UINT_ZERO */ 734 735#ifndef CK_F_PR_DEC_UINT 736#define CK_F_PR_DEC_UINT 737CK_PR_UNARY_S(dec, sub, uint, unsigned int) 738#endif /* CK_F_PR_DEC_UINT */ 739 740#ifndef CK_F_PR_DEC_UINT_ZERO 741#define CK_F_PR_DEC_UINT_ZERO 742CK_PR_UNARY_Z_S(dec, uint, unsigned int, -, 1) 743#else 744CK_PR_UNARY_Z_STUB(dec, uint, unsigned int) 745#endif /* CK_F_PR_DEC_UINT_ZERO */ 746 747#endif /* CK_F_PR_LOAD_UINT && CK_F_PR_CAS_UINT_VALUE */ 748 749#if defined(CK_F_PR_LOAD_PTR) && defined(CK_F_PR_CAS_PTR_VALUE) 750 751#ifndef CK_F_PR_INC_PTR 752#define CK_F_PR_INC_PTR 753CK_PR_UNARY(inc, add, ptr, void, uintptr_t) 754#endif /* CK_F_PR_INC_PTR */ 755 756#ifndef CK_F_PR_INC_PTR_ZERO 757#define CK_F_PR_INC_PTR_ZERO 758CK_PR_UNARY_Z(inc, ptr, void, uintptr_t, +, void *, UINT_MAX) 759#else 760CK_PR_UNARY_Z_STUB(inc, ptr, void) 761#endif /* CK_F_PR_INC_PTR_ZERO */ 762 763#ifndef CK_F_PR_DEC_PTR 764#define CK_F_PR_DEC_PTR 765CK_PR_UNARY(dec, sub, ptr, void, uintptr_t) 766#endif /* CK_F_PR_DEC_PTR */ 767 768#ifndef CK_F_PR_DEC_PTR_ZERO 769#define CK_F_PR_DEC_PTR_ZERO 770CK_PR_UNARY_Z(dec, ptr, void, uintptr_t, -, void *, 1) 771#else 772CK_PR_UNARY_Z_STUB(dec, ptr, void) 773#endif /* CK_F_PR_DEC_PTR_ZERO */ 774 775#endif /* CK_F_PR_LOAD_PTR && CK_F_PR_CAS_PTR_VALUE */ 776 777#if defined(CK_F_PR_LOAD_64) && defined(CK_F_PR_CAS_64_VALUE) 778 779#ifndef CK_F_PR_INC_64 780#define CK_F_PR_INC_64 781CK_PR_UNARY_S(inc, add, 64, uint64_t) 782#endif /* CK_F_PR_INC_64 */ 783 784#ifndef CK_F_PR_INC_64_ZERO 785#define CK_F_PR_INC_64_ZERO 786CK_PR_UNARY_Z_S(inc, 64, uint64_t, +, UINT64_MAX) 787#else 788CK_PR_UNARY_Z_STUB(inc, 64, uint64_t) 789#endif /* CK_F_PR_INC_64_ZERO */ 790 791#ifndef CK_F_PR_DEC_64 792#define CK_F_PR_DEC_64 793CK_PR_UNARY_S(dec, sub, 64, uint64_t) 794#endif /* CK_F_PR_DEC_64 */ 795 796#ifndef CK_F_PR_DEC_64_ZERO 797#define CK_F_PR_DEC_64_ZERO 798CK_PR_UNARY_Z_S(dec, 64, uint64_t, -, 1) 799#else 800CK_PR_UNARY_Z_STUB(dec, 64, uint64_t) 801#endif /* CK_F_PR_DEC_64_ZERO */ 802 803#endif /* CK_F_PR_LOAD_64 && CK_F_PR_CAS_64_VALUE */ 804 805#if defined(CK_F_PR_LOAD_32) && defined(CK_F_PR_CAS_32_VALUE) 806 807#ifndef CK_F_PR_INC_32 808#define CK_F_PR_INC_32 809CK_PR_UNARY_S(inc, add, 32, uint32_t) 810#endif /* CK_F_PR_INC_32 */ 811 812#ifndef CK_F_PR_INC_32_ZERO 813#define CK_F_PR_INC_32_ZERO 814CK_PR_UNARY_Z_S(inc, 32, uint32_t, +, UINT32_MAX) 815#else 816CK_PR_UNARY_Z_STUB(inc, 32, uint32_t) 817#endif /* CK_F_PR_INC_32_ZERO */ 818 819#ifndef CK_F_PR_DEC_32 820#define CK_F_PR_DEC_32 821CK_PR_UNARY_S(dec, sub, 32, uint32_t) 822#endif /* CK_F_PR_DEC_32 */ 823 824#ifndef CK_F_PR_DEC_32_ZERO 825#define CK_F_PR_DEC_32_ZERO 826CK_PR_UNARY_Z_S(dec, 32, uint32_t, -, 1) 827#else 828CK_PR_UNARY_Z_STUB(dec, 32, uint32_t) 829#endif /* CK_F_PR_DEC_32_ZERO */ 830 831#endif /* CK_F_PR_LOAD_32 && CK_F_PR_CAS_32_VALUE */ 832 833#if defined(CK_F_PR_LOAD_16) && defined(CK_F_PR_CAS_16_VALUE) 834 835#ifndef CK_F_PR_INC_16 836#define CK_F_PR_INC_16 837CK_PR_UNARY_S(inc, add, 16, uint16_t) 838#endif /* CK_F_PR_INC_16 */ 839 840#ifndef CK_F_PR_INC_16_ZERO 841#define CK_F_PR_INC_16_ZERO 842CK_PR_UNARY_Z_S(inc, 16, uint16_t, +, UINT16_MAX) 843#else 844CK_PR_UNARY_Z_STUB(inc, 16, uint16_t) 845#endif /* CK_F_PR_INC_16_ZERO */ 846 847#ifndef CK_F_PR_DEC_16 848#define CK_F_PR_DEC_16 849CK_PR_UNARY_S(dec, sub, 16, uint16_t) 850#endif /* CK_F_PR_DEC_16 */ 851 852#ifndef CK_F_PR_DEC_16_ZERO 853#define CK_F_PR_DEC_16_ZERO 854CK_PR_UNARY_Z_S(dec, 16, uint16_t, -, 1) 855#else 856CK_PR_UNARY_Z_STUB(dec, 16, uint16_t) 857#endif /* CK_F_PR_DEC_16_ZERO */ 858 859#endif /* CK_F_PR_LOAD_16 && CK_F_PR_CAS_16_VALUE */ 860 861#if defined(CK_F_PR_LOAD_8) && defined(CK_F_PR_CAS_8_VALUE) 862 863#ifndef CK_F_PR_INC_8 864#define CK_F_PR_INC_8 865CK_PR_UNARY_S(inc, add, 8, uint8_t) 866#endif /* CK_F_PR_INC_8 */ 867 868#ifndef CK_F_PR_INC_8_ZERO 869#define CK_F_PR_INC_8_ZERO 870CK_PR_UNARY_Z_S(inc, 8, uint8_t, +, UINT8_MAX) 871#else 872CK_PR_UNARY_Z_STUB(inc, 8, uint8_t) 873#endif /* CK_F_PR_INC_8_ZERO */ 874 875#ifndef CK_F_PR_DEC_8 876#define CK_F_PR_DEC_8 877CK_PR_UNARY_S(dec, sub, 8, uint8_t) 878#endif /* CK_F_PR_DEC_8 */ 879 880#ifndef CK_F_PR_DEC_8_ZERO 881#define CK_F_PR_DEC_8_ZERO 882CK_PR_UNARY_Z_S(dec, 8, uint8_t, -, 1) 883#else 884CK_PR_UNARY_Z_STUB(dec, 8, uint8_t) 885#endif /* CK_F_PR_DEC_8_ZERO */ 886 887#endif /* CK_F_PR_LOAD_8 && CK_F_PR_CAS_8_VALUE */ 888 889#undef CK_PR_UNARY_Z_S 890#undef CK_PR_UNARY_S 891#undef CK_PR_UNARY_Z 892#undef CK_PR_UNARY 893 894#define CK_PR_N(K, S, M, T, P, C) \ 895 CK_CC_INLINE static void \ 896 ck_pr_##K##_##S(M *target) \ 897 { \ 898 T previous; \ 899 C punt; \ 900 punt = (C)ck_pr_md_load_##S(target); \ 901 previous = (T)punt; \ 902 while (ck_pr_cas_##S##_value(target, \ 903 (C)previous, \ 904 (C)(P previous), \ 905 &previous) == false) \ 906 ck_pr_stall(); \ 907 \ 908 return; \ 909 } 910 911#define CK_PR_N_Z(S, M, T, C) \ 912 CK_CC_INLINE static void \ 913 ck_pr_neg_##S##_zero(M *target, bool *zero) \ 914 { \ 915 T previous; \ 916 C punt; \ 917 punt = (C)ck_pr_md_load_##S(target); \ 918 previous = (T)punt; \ 919 while (ck_pr_cas_##S##_value(target, \ 920 (C)previous, \ 921 (C)(-previous), \ 922 &previous) == false) \ 923 ck_pr_stall(); \ 924 \ 925 *zero = previous == 0; \ 926 return; \ 927 } 928 929#define CK_PR_N_S(K, S, M, P) CK_PR_N(K, S, M, M, P, M) 930#define CK_PR_N_Z_S(S, M) CK_PR_N_Z(S, M, M, M) 931 932#if defined(CK_F_PR_LOAD_CHAR) && defined(CK_F_PR_CAS_CHAR_VALUE) 933 934#ifndef CK_F_PR_NOT_CHAR 935#define CK_F_PR_NOT_CHAR 936CK_PR_N_S(not, char, char, ~) 937#endif /* CK_F_PR_NOT_CHAR */ 938 939#ifndef CK_F_PR_NEG_CHAR 940#define CK_F_PR_NEG_CHAR 941CK_PR_N_S(neg, char, char, -) 942#endif /* CK_F_PR_NEG_CHAR */ 943 944#ifndef CK_F_PR_NEG_CHAR_ZERO 945#define CK_F_PR_NEG_CHAR_ZERO 946CK_PR_N_Z_S(char, char) 947#endif /* CK_F_PR_NEG_CHAR_ZERO */ 948 949#endif /* CK_F_PR_LOAD_CHAR && CK_F_PR_CAS_CHAR_VALUE */ 950 951#if defined(CK_F_PR_LOAD_INT) && defined(CK_F_PR_CAS_INT_VALUE) 952 953#ifndef CK_F_PR_NOT_INT 954#define CK_F_PR_NOT_INT 955CK_PR_N_S(not, int, int, ~) 956#endif /* CK_F_PR_NOT_INT */ 957 958#ifndef CK_F_PR_NEG_INT 959#define CK_F_PR_NEG_INT 960CK_PR_N_S(neg, int, int, -) 961#endif /* CK_F_PR_NEG_INT */ 962 963#ifndef CK_F_PR_NEG_INT_ZERO 964#define CK_F_PR_NEG_INT_ZERO 965CK_PR_N_Z_S(int, int) 966#endif /* CK_F_PR_NEG_INT_ZERO */ 967 968#endif /* CK_F_PR_LOAD_INT && CK_F_PR_CAS_INT_VALUE */ 969 970#if defined(CK_F_PR_LOAD_DOUBLE) && defined(CK_F_PR_CAS_DOUBLE_VALUE) && \ 971 !defined(CK_PR_DISABLE_DOUBLE) 972 973#ifndef CK_F_PR_NEG_DOUBLE 974#define CK_F_PR_NEG_DOUBLE 975CK_PR_N_S(neg, double, double, -) 976#endif /* CK_F_PR_NEG_DOUBLE */ 977 978#endif /* CK_F_PR_LOAD_DOUBLE && CK_F_PR_CAS_DOUBLE_VALUE && !CK_PR_DISABLE_DOUBLE */ 979 980#if defined(CK_F_PR_LOAD_UINT) && defined(CK_F_PR_CAS_UINT_VALUE) 981 982#ifndef CK_F_PR_NOT_UINT 983#define CK_F_PR_NOT_UINT 984CK_PR_N_S(not, uint, unsigned int, ~) 985#endif /* CK_F_PR_NOT_UINT */ 986 987#ifndef CK_F_PR_NEG_UINT 988#define CK_F_PR_NEG_UINT 989CK_PR_N_S(neg, uint, unsigned int, -) 990#endif /* CK_F_PR_NEG_UINT */ 991 992#ifndef CK_F_PR_NEG_UINT_ZERO 993#define CK_F_PR_NEG_UINT_ZERO 994CK_PR_N_Z_S(uint, unsigned int) 995#endif /* CK_F_PR_NEG_UINT_ZERO */ 996 997#endif /* CK_F_PR_LOAD_UINT && CK_F_PR_CAS_UINT_VALUE */ 998 999#if defined(CK_F_PR_LOAD_PTR) && defined(CK_F_PR_CAS_PTR_VALUE) 1000 1001#ifndef CK_F_PR_NOT_PTR 1002#define CK_F_PR_NOT_PTR 1003CK_PR_N(not, ptr, void, uintptr_t, ~, void *) 1004#endif /* CK_F_PR_NOT_PTR */ 1005 1006#ifndef CK_F_PR_NEG_PTR 1007#define CK_F_PR_NEG_PTR 1008CK_PR_N(neg, ptr, void, uintptr_t, -, void *) 1009#endif /* CK_F_PR_NEG_PTR */ 1010 1011#ifndef CK_F_PR_NEG_PTR_ZERO 1012#define CK_F_PR_NEG_PTR_ZERO 1013CK_PR_N_Z(ptr, void, uintptr_t, void *) 1014#endif /* CK_F_PR_NEG_PTR_ZERO */ 1015 1016#endif /* CK_F_PR_LOAD_PTR && CK_F_PR_CAS_PTR_VALUE */ 1017 1018#if defined(CK_F_PR_LOAD_64) && defined(CK_F_PR_CAS_64_VALUE) 1019 1020#ifndef CK_F_PR_NOT_64 1021#define CK_F_PR_NOT_64 1022CK_PR_N_S(not, 64, uint64_t, ~) 1023#endif /* CK_F_PR_NOT_64 */ 1024 1025#ifndef CK_F_PR_NEG_64 1026#define CK_F_PR_NEG_64 1027CK_PR_N_S(neg, 64, uint64_t, -) 1028#endif /* CK_F_PR_NEG_64 */ 1029 1030#ifndef CK_F_PR_NEG_64_ZERO 1031#define CK_F_PR_NEG_64_ZERO 1032CK_PR_N_Z_S(64, uint64_t) 1033#endif /* CK_F_PR_NEG_64_ZERO */ 1034 1035#endif /* CK_F_PR_LOAD_64 && CK_F_PR_CAS_64_VALUE */ 1036 1037#if defined(CK_F_PR_LOAD_32) && defined(CK_F_PR_CAS_32_VALUE) 1038 1039#ifndef CK_F_PR_NOT_32 1040#define CK_F_PR_NOT_32 1041CK_PR_N_S(not, 32, uint32_t, ~) 1042#endif /* CK_F_PR_NOT_32 */ 1043 1044#ifndef CK_F_PR_NEG_32 1045#define CK_F_PR_NEG_32 1046CK_PR_N_S(neg, 32, uint32_t, -) 1047#endif /* CK_F_PR_NEG_32 */ 1048 1049#ifndef CK_F_PR_NEG_32_ZERO 1050#define CK_F_PR_NEG_32_ZERO 1051CK_PR_N_Z_S(32, uint32_t) 1052#endif /* CK_F_PR_NEG_32_ZERO */ 1053 1054#endif /* CK_F_PR_LOAD_32 && CK_F_PR_CAS_32_VALUE */ 1055 1056#if defined(CK_F_PR_LOAD_16) && defined(CK_F_PR_CAS_16_VALUE) 1057 1058#ifndef CK_F_PR_NOT_16 1059#define CK_F_PR_NOT_16 1060CK_PR_N_S(not, 16, uint16_t, ~) 1061#endif /* CK_F_PR_NOT_16 */ 1062 1063#ifndef CK_F_PR_NEG_16 1064#define CK_F_PR_NEG_16 1065CK_PR_N_S(neg, 16, uint16_t, -) 1066#endif /* CK_F_PR_NEG_16 */ 1067 1068#ifndef CK_F_PR_NEG_16_ZERO 1069#define CK_F_PR_NEG_16_ZERO 1070CK_PR_N_Z_S(16, uint16_t) 1071#endif /* CK_F_PR_NEG_16_ZERO */ 1072 1073#endif /* CK_F_PR_LOAD_16 && CK_F_PR_CAS_16_VALUE */ 1074 1075#if defined(CK_F_PR_LOAD_8) && defined(CK_F_PR_CAS_8_VALUE) 1076 1077#ifndef CK_F_PR_NOT_8 1078#define CK_F_PR_NOT_8 1079CK_PR_N_S(not, 8, uint8_t, ~) 1080#endif /* CK_F_PR_NOT_8 */ 1081 1082#ifndef CK_F_PR_NEG_8 1083#define CK_F_PR_NEG_8 1084CK_PR_N_S(neg, 8, uint8_t, -) 1085#endif /* CK_F_PR_NEG_8 */ 1086 1087#ifndef CK_F_PR_NEG_8_ZERO 1088#define CK_F_PR_NEG_8_ZERO 1089CK_PR_N_Z_S(8, uint8_t) 1090#endif /* CK_F_PR_NEG_8_ZERO */ 1091 1092#endif /* CK_F_PR_LOAD_8 && CK_F_PR_CAS_8_VALUE */ 1093 1094#undef CK_PR_N_Z_S 1095#undef CK_PR_N_S 1096#undef CK_PR_N_Z 1097#undef CK_PR_N 1098 1099#define CK_PR_FAA(S, M, T, C) \ 1100 CK_CC_INLINE static C \ 1101 ck_pr_faa_##S(M *target, T delta) \ 1102 { \ 1103 T previous; \ 1104 C punt; \ 1105 punt = (C)ck_pr_md_load_##S(target); \ 1106 previous = (T)punt; \ 1107 while (ck_pr_cas_##S##_value(target, \ 1108 (C)previous, \ 1109 (C)(previous + delta), \ 1110 &previous) == false) \ 1111 ck_pr_stall(); \ 1112 \ 1113 return ((C)previous); \ 1114 } 1115 1116#define CK_PR_FAS(S, M, C) \ 1117 CK_CC_INLINE static C \ 1118 ck_pr_fas_##S(M *target, C update) \ 1119 { \ 1120 C previous; \ 1121 previous = ck_pr_md_load_##S(target); \ 1122 while (ck_pr_cas_##S##_value(target, \ 1123 previous, \ 1124 update, \ 1125 &previous) == false) \ 1126 ck_pr_stall(); \ 1127 \ 1128 return (previous); \ 1129 } 1130 1131#define CK_PR_FAA_S(S, M) CK_PR_FAA(S, M, M, M) 1132#define CK_PR_FAS_S(S, M) CK_PR_FAS(S, M, M) 1133 1134#if defined(CK_F_PR_LOAD_CHAR) && defined(CK_F_PR_CAS_CHAR_VALUE) 1135 1136#ifndef CK_F_PR_FAA_CHAR 1137#define CK_F_PR_FAA_CHAR 1138CK_PR_FAA_S(char, char) 1139#endif /* CK_F_PR_FAA_CHAR */ 1140 1141#ifndef CK_F_PR_FAS_CHAR 1142#define CK_F_PR_FAS_CHAR 1143CK_PR_FAS_S(char, char) 1144#endif /* CK_F_PR_FAS_CHAR */ 1145 1146#endif /* CK_F_PR_LOAD_CHAR && CK_F_PR_CAS_CHAR_VALUE */ 1147 1148#if defined(CK_F_PR_LOAD_INT) && defined(CK_F_PR_CAS_INT_VALUE) 1149 1150#ifndef CK_F_PR_FAA_INT 1151#define CK_F_PR_FAA_INT 1152CK_PR_FAA_S(int, int) 1153#endif /* CK_F_PR_FAA_INT */ 1154 1155#ifndef CK_F_PR_FAS_INT 1156#define CK_F_PR_FAS_INT 1157CK_PR_FAS_S(int, int) 1158#endif /* CK_F_PR_FAS_INT */ 1159 1160#endif /* CK_F_PR_LOAD_INT && CK_F_PR_CAS_INT_VALUE */ 1161 1162#if defined(CK_F_PR_LOAD_DOUBLE) && defined(CK_F_PR_CAS_DOUBLE_VALUE) && \ 1163 !defined(CK_PR_DISABLE_DOUBLE) 1164 1165#ifndef CK_F_PR_FAA_DOUBLE 1166#define CK_F_PR_FAA_DOUBLE 1167CK_PR_FAA_S(double, double) 1168#endif /* CK_F_PR_FAA_DOUBLE */ 1169 1170#ifndef CK_F_PR_FAS_DOUBLE 1171#define CK_F_PR_FAS_DOUBLE 1172CK_PR_FAS_S(double, double) 1173#endif /* CK_F_PR_FAS_DOUBLE */ 1174 1175#endif /* CK_F_PR_LOAD_DOUBLE && CK_F_PR_CAS_DOUBLE_VALUE && !CK_PR_DISABLE_DOUBLE */ 1176 1177#if defined(CK_F_PR_LOAD_UINT) && defined(CK_F_PR_CAS_UINT_VALUE) 1178 1179#ifndef CK_F_PR_FAA_UINT 1180#define CK_F_PR_FAA_UINT 1181CK_PR_FAA_S(uint, unsigned int) 1182#endif /* CK_F_PR_FAA_UINT */ 1183 1184#ifndef CK_F_PR_FAS_UINT 1185#define CK_F_PR_FAS_UINT 1186CK_PR_FAS_S(uint, unsigned int) 1187#endif /* CK_F_PR_FAS_UINT */ 1188 1189#endif /* CK_F_PR_LOAD_UINT && CK_F_PR_CAS_UINT_VALUE */ 1190 1191#if defined(CK_F_PR_LOAD_PTR) && defined(CK_F_PR_CAS_PTR_VALUE) 1192 1193#ifndef CK_F_PR_FAA_PTR 1194#define CK_F_PR_FAA_PTR 1195CK_PR_FAA(ptr, void, uintptr_t, void *) 1196#endif /* CK_F_PR_FAA_PTR */ 1197 1198#ifndef CK_F_PR_FAS_PTR 1199#define CK_F_PR_FAS_PTR 1200CK_PR_FAS(ptr, void, void *) 1201#endif /* CK_F_PR_FAS_PTR */ 1202 1203#endif /* CK_F_PR_LOAD_PTR && CK_F_PR_CAS_PTR_VALUE */ 1204 1205#if defined(CK_F_PR_LOAD_64) && defined(CK_F_PR_CAS_64_VALUE) 1206 1207#ifndef CK_F_PR_FAA_64 1208#define CK_F_PR_FAA_64 1209CK_PR_FAA_S(64, uint64_t) 1210#endif /* CK_F_PR_FAA_64 */ 1211 1212#ifndef CK_F_PR_FAS_64 1213#define CK_F_PR_FAS_64 1214CK_PR_FAS_S(64, uint64_t) 1215#endif /* CK_F_PR_FAS_64 */ 1216 1217#endif /* CK_F_PR_LOAD_64 && CK_F_PR_CAS_64_VALUE */ 1218 1219#if defined(CK_F_PR_LOAD_32) && defined(CK_F_PR_CAS_32_VALUE) 1220 1221#ifndef CK_F_PR_FAA_32 1222#define CK_F_PR_FAA_32 1223CK_PR_FAA_S(32, uint32_t) 1224#endif /* CK_F_PR_FAA_32 */ 1225 1226#ifndef CK_F_PR_FAS_32 1227#define CK_F_PR_FAS_32 1228CK_PR_FAS_S(32, uint32_t) 1229#endif /* CK_F_PR_FAS_32 */ 1230 1231#endif /* CK_F_PR_LOAD_32 && CK_F_PR_CAS_32_VALUE */ 1232 1233#if defined(CK_F_PR_LOAD_16) && defined(CK_F_PR_CAS_16_VALUE) 1234 1235#ifndef CK_F_PR_FAA_16 1236#define CK_F_PR_FAA_16 1237CK_PR_FAA_S(16, uint16_t) 1238#endif /* CK_F_PR_FAA_16 */ 1239 1240#ifndef CK_F_PR_FAS_16 1241#define CK_F_PR_FAS_16 1242CK_PR_FAS_S(16, uint16_t) 1243#endif /* CK_F_PR_FAS_16 */ 1244 1245#endif /* CK_F_PR_LOAD_16 && CK_F_PR_CAS_16_VALUE */ 1246 1247#if defined(CK_F_PR_LOAD_8) && defined(CK_F_PR_CAS_8_VALUE) 1248 1249#ifndef CK_F_PR_FAA_8 1250#define CK_F_PR_FAA_8 1251CK_PR_FAA_S(8, uint8_t) 1252#endif /* CK_F_PR_FAA_8 */ 1253 1254#ifndef CK_F_PR_FAS_8 1255#define CK_F_PR_FAS_8 1256CK_PR_FAS_S(8, uint8_t) 1257#endif /* CK_F_PR_FAS_8 */ 1258 1259#endif /* CK_F_PR_LOAD_8 && CK_F_PR_CAS_8_VALUE */ 1260 1261#undef CK_PR_FAA_S 1262#undef CK_PR_FAS_S 1263#undef CK_PR_FAA 1264#undef CK_PR_FAS 1265 1266#endif /* CK_PR_H */ 1267