ck_pr.h revision 328515
1/* 2 * Copyright 2009-2015 Samy Al Bahra. 3 * Copyright 2011 David Joseph. 4 * All rights reserved. 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25 * SUCH DAMAGE. 26 */ 27 28#ifndef CK_PR_H 29#define CK_PR_H 30 31#include <ck_cc.h> 32#include <ck_limits.h> 33#include <ck_md.h> 34#include <ck_stdint.h> 35#include <ck_stdbool.h> 36 37#ifndef CK_USE_CC_BUILTINS 38#if defined(__x86_64__) 39#include "gcc/x86_64/ck_pr.h" 40#elif defined(__x86__) 41#include "gcc/x86/ck_pr.h" 42#elif defined(__sparcv9__) 43#include "gcc/sparcv9/ck_pr.h" 44#elif defined(__ppc64__) 45#include "gcc/ppc64/ck_pr.h" 46#elif defined(__ppc__) 47#include "gcc/ppc/ck_pr.h" 48#elif defined(__arm__) 49#if __ARM_ARCH >= 6 50#include "gcc/arm/ck_pr.h" 51#else 52#include "gcc/arm/ck_pr_armv4.h" 53#endif 54#elif defined(__aarch64__) 55#include "gcc/aarch64/ck_pr.h" 56#elif !defined(__GNUC__) 57#error Your platform is unsupported 58#endif 59#endif /* !CK_USE_CC_BUILTINS */ 60 61#if defined(__GNUC__) 62#include "gcc/ck_pr.h" 63#endif 64 65#define CK_PR_FENCE_EMIT(T) \ 66 CK_CC_INLINE static void \ 67 ck_pr_fence_##T(void) \ 68 { \ 69 ck_pr_fence_strict_##T(); \ 70 return; \ 71 } 72#define CK_PR_FENCE_NOOP(T) \ 73 CK_CC_INLINE static void \ 74 ck_pr_fence_##T(void) \ 75 { \ 76 ck_pr_barrier(); \ 77 return; \ 78 } 79 80/* 81 * None of the currently supported platforms allow for data-dependent 82 * load ordering. 83 */ 84CK_PR_FENCE_NOOP(load_depends) 85#define ck_pr_fence_strict_load_depends ck_pr_fence_load_depends 86 87/* 88 * In memory models where atomic operations do not have serializing 89 * effects, atomic read-modify-write operations are modeled as stores. 90 */ 91#if defined(CK_MD_RMO) 92/* 93 * Only stores to the same location have a global 94 * ordering. 95 */ 96CK_PR_FENCE_EMIT(atomic) 97CK_PR_FENCE_EMIT(atomic_load) 98CK_PR_FENCE_EMIT(atomic_store) 99CK_PR_FENCE_EMIT(store_atomic) 100CK_PR_FENCE_EMIT(load_atomic) 101CK_PR_FENCE_EMIT(load_store) 102CK_PR_FENCE_EMIT(store_load) 103CK_PR_FENCE_EMIT(load) 104CK_PR_FENCE_EMIT(store) 105CK_PR_FENCE_EMIT(memory) 106CK_PR_FENCE_EMIT(acquire) 107CK_PR_FENCE_EMIT(release) 108CK_PR_FENCE_EMIT(acqrel) 109CK_PR_FENCE_EMIT(lock) 110CK_PR_FENCE_EMIT(unlock) 111#elif defined(CK_MD_PSO) 112/* 113 * Anything can be re-ordered with respect to stores. 114 * Otherwise, loads are executed in-order. 115 */ 116CK_PR_FENCE_EMIT(atomic) 117CK_PR_FENCE_NOOP(atomic_load) 118CK_PR_FENCE_EMIT(atomic_store) 119CK_PR_FENCE_EMIT(store_atomic) 120CK_PR_FENCE_NOOP(load_atomic) 121CK_PR_FENCE_EMIT(load_store) 122CK_PR_FENCE_EMIT(store_load) 123CK_PR_FENCE_NOOP(load) 124CK_PR_FENCE_EMIT(store) 125CK_PR_FENCE_EMIT(memory) 126CK_PR_FENCE_EMIT(acquire) 127CK_PR_FENCE_EMIT(release) 128CK_PR_FENCE_EMIT(acqrel) 129CK_PR_FENCE_EMIT(lock) 130CK_PR_FENCE_EMIT(unlock) 131#elif defined(CK_MD_TSO) 132/* 133 * Only loads are re-ordered and only with respect to 134 * prior stores. Atomic operations are serializing. 135 */ 136CK_PR_FENCE_NOOP(atomic) 137CK_PR_FENCE_NOOP(atomic_load) 138CK_PR_FENCE_NOOP(atomic_store) 139CK_PR_FENCE_NOOP(store_atomic) 140CK_PR_FENCE_NOOP(load_atomic) 141CK_PR_FENCE_NOOP(load_store) 142CK_PR_FENCE_EMIT(store_load) 143CK_PR_FENCE_NOOP(load) 144CK_PR_FENCE_NOOP(store) 145CK_PR_FENCE_EMIT(memory) 146CK_PR_FENCE_NOOP(acquire) 147CK_PR_FENCE_NOOP(release) 148CK_PR_FENCE_NOOP(acqrel) 149CK_PR_FENCE_NOOP(lock) 150CK_PR_FENCE_NOOP(unlock) 151#else 152#error "No memory model has been defined." 153#endif /* CK_MD_TSO */ 154 155#undef CK_PR_FENCE_EMIT 156#undef CK_PR_FENCE_NOOP 157 158#ifndef CK_F_PR_RFO 159#define CK_F_PR_RFO 160CK_CC_INLINE static void 161ck_pr_rfo(const void *m) 162{ 163 164 (void)m; 165 return; 166} 167#endif /* CK_F_PR_RFO */ 168 169#define CK_PR_STORE_SAFE(DST, VAL, TYPE) \ 170 ck_pr_md_store_##TYPE( \ 171 ((void)sizeof(*(DST) = (VAL)), (DST)), \ 172 (VAL)) 173 174#define ck_pr_store_ptr(DST, VAL) CK_PR_STORE_SAFE((DST), (VAL), ptr) 175#define ck_pr_store_char(DST, VAL) CK_PR_STORE_SAFE((DST), (VAL), char) 176#ifndef CK_PR_DISABLE_DOUBLE 177#define ck_pr_store_double(DST, VAL) CK_PR_STORE_SAFE((DST), (VAL), double) 178#endif 179#define ck_pr_store_uint(DST, VAL) CK_PR_STORE_SAFE((DST), (VAL), uint) 180#define ck_pr_store_int(DST, VAL) CK_PR_STORE_SAFE((DST), (VAL), int) 181#define ck_pr_store_32(DST, VAL) CK_PR_STORE_SAFE((DST), (VAL), 32) 182#define ck_pr_store_16(DST, VAL) CK_PR_STORE_SAFE((DST), (VAL), 16) 183#define ck_pr_store_8(DST, VAL) CK_PR_STORE_SAFE((DST), (VAL), 8) 184 185#define ck_pr_store_ptr_unsafe(DST, VAL) ck_pr_md_store_ptr((DST), (VAL)) 186 187#ifdef CK_F_PR_LOAD_64 188#define ck_pr_store_64(DST, VAL) CK_PR_STORE_SAFE((DST), (VAL), 64) 189#endif /* CK_F_PR_LOAD_64 */ 190 191#define CK_PR_LOAD_PTR_SAFE(SRC) (CK_CC_TYPEOF(*(SRC), (void *)))ck_pr_md_load_ptr((SRC)) 192#define ck_pr_load_ptr(SRC) CK_PR_LOAD_PTR_SAFE((SRC)) 193 194#define CK_PR_LOAD_SAFE(SRC, TYPE) ck_pr_md_load_##TYPE((SRC)) 195#define ck_pr_load_char(SRC) CK_PR_LOAD_SAFE((SRC), char) 196#ifndef CK_PR_DISABLE_DOUBLE 197#define ck_pr_load_double(SRC) CK_PR_LOAD_SAFE((SRC), double) 198#endif 199#define ck_pr_load_uint(SRC) CK_PR_LOAD_SAFE((SRC), uint) 200#define ck_pr_load_int(SRC) CK_PR_LOAD_SAFE((SRC), int) 201#define ck_pr_load_32(SRC) CK_PR_LOAD_SAFE((SRC), 32) 202#define ck_pr_load_16(SRC) CK_PR_LOAD_SAFE((SRC), 16) 203#define ck_pr_load_8(SRC) CK_PR_LOAD_SAFE((SRC), 8) 204 205#ifdef CK_F_PR_LOAD_64 206#define ck_pr_load_64(SRC) CK_PR_LOAD_SAFE((SRC), 64) 207#endif /* CK_F_PR_LOAD_64 */ 208 209#define CK_PR_BIN(K, S, M, T, P, C) \ 210 CK_CC_INLINE static void \ 211 ck_pr_##K##_##S(M *target, T value) \ 212 { \ 213 T previous; \ 214 C punt; \ 215 punt = ck_pr_md_load_##S(target); \ 216 previous = (T)punt; \ 217 while (ck_pr_cas_##S##_value(target, \ 218 (C)previous, \ 219 (C)(previous P value), \ 220 &previous) == false) \ 221 ck_pr_stall(); \ 222 \ 223 return; \ 224 } 225 226#define CK_PR_BIN_S(K, S, T, P) CK_PR_BIN(K, S, T, T, P, T) 227 228#if defined(CK_F_PR_LOAD_CHAR) && defined(CK_F_PR_CAS_CHAR_VALUE) 229 230#ifndef CK_F_PR_ADD_CHAR 231#define CK_F_PR_ADD_CHAR 232CK_PR_BIN_S(add, char, char, +) 233#endif /* CK_F_PR_ADD_CHAR */ 234 235#ifndef CK_F_PR_SUB_CHAR 236#define CK_F_PR_SUB_CHAR 237CK_PR_BIN_S(sub, char, char, -) 238#endif /* CK_F_PR_SUB_CHAR */ 239 240#ifndef CK_F_PR_AND_CHAR 241#define CK_F_PR_AND_CHAR 242CK_PR_BIN_S(and, char, char, &) 243#endif /* CK_F_PR_AND_CHAR */ 244 245#ifndef CK_F_PR_XOR_CHAR 246#define CK_F_PR_XOR_CHAR 247CK_PR_BIN_S(xor, char, char, ^) 248#endif /* CK_F_PR_XOR_CHAR */ 249 250#ifndef CK_F_PR_OR_CHAR 251#define CK_F_PR_OR_CHAR 252CK_PR_BIN_S(or, char, char, |) 253#endif /* CK_F_PR_OR_CHAR */ 254 255#endif /* CK_F_PR_LOAD_CHAR && CK_F_PR_CAS_CHAR_VALUE */ 256 257#if defined(CK_F_PR_LOAD_INT) && defined(CK_F_PR_CAS_INT_VALUE) 258 259#ifndef CK_F_PR_ADD_INT 260#define CK_F_PR_ADD_INT 261CK_PR_BIN_S(add, int, int, +) 262#endif /* CK_F_PR_ADD_INT */ 263 264#ifndef CK_F_PR_SUB_INT 265#define CK_F_PR_SUB_INT 266CK_PR_BIN_S(sub, int, int, -) 267#endif /* CK_F_PR_SUB_INT */ 268 269#ifndef CK_F_PR_AND_INT 270#define CK_F_PR_AND_INT 271CK_PR_BIN_S(and, int, int, &) 272#endif /* CK_F_PR_AND_INT */ 273 274#ifndef CK_F_PR_XOR_INT 275#define CK_F_PR_XOR_INT 276CK_PR_BIN_S(xor, int, int, ^) 277#endif /* CK_F_PR_XOR_INT */ 278 279#ifndef CK_F_PR_OR_INT 280#define CK_F_PR_OR_INT 281CK_PR_BIN_S(or, int, int, |) 282#endif /* CK_F_PR_OR_INT */ 283 284#endif /* CK_F_PR_LOAD_INT && CK_F_PR_CAS_INT_VALUE */ 285 286#if defined(CK_F_PR_LOAD_DOUBLE) && defined(CK_F_PR_CAS_DOUBLE_VALUE) && \ 287 !defined(CK_PR_DISABLE_DOUBLE) 288 289#ifndef CK_F_PR_ADD_DOUBLE 290#define CK_F_PR_ADD_DOUBLE 291CK_PR_BIN_S(add, double, double, +) 292#endif /* CK_F_PR_ADD_DOUBLE */ 293 294#ifndef CK_F_PR_SUB_DOUBLE 295#define CK_F_PR_SUB_DOUBLE 296CK_PR_BIN_S(sub, double, double, -) 297#endif /* CK_F_PR_SUB_DOUBLE */ 298 299#endif /* CK_F_PR_LOAD_DOUBLE && CK_F_PR_CAS_DOUBLE_VALUE && !CK_PR_DISABLE_DOUBLE */ 300 301#if defined(CK_F_PR_LOAD_UINT) && defined(CK_F_PR_CAS_UINT_VALUE) 302 303#ifndef CK_F_PR_ADD_UINT 304#define CK_F_PR_ADD_UINT 305CK_PR_BIN_S(add, uint, unsigned int, +) 306#endif /* CK_F_PR_ADD_UINT */ 307 308#ifndef CK_F_PR_SUB_UINT 309#define CK_F_PR_SUB_UINT 310CK_PR_BIN_S(sub, uint, unsigned int, -) 311#endif /* CK_F_PR_SUB_UINT */ 312 313#ifndef CK_F_PR_AND_UINT 314#define CK_F_PR_AND_UINT 315CK_PR_BIN_S(and, uint, unsigned int, &) 316#endif /* CK_F_PR_AND_UINT */ 317 318#ifndef CK_F_PR_XOR_UINT 319#define CK_F_PR_XOR_UINT 320CK_PR_BIN_S(xor, uint, unsigned int, ^) 321#endif /* CK_F_PR_XOR_UINT */ 322 323#ifndef CK_F_PR_OR_UINT 324#define CK_F_PR_OR_UINT 325CK_PR_BIN_S(or, uint, unsigned int, |) 326#endif /* CK_F_PR_OR_UINT */ 327 328#endif /* CK_F_PR_LOAD_UINT && CK_F_PR_CAS_UINT_VALUE */ 329 330#if defined(CK_F_PR_LOAD_PTR) && defined(CK_F_PR_CAS_PTR_VALUE) 331 332#ifndef CK_F_PR_ADD_PTR 333#define CK_F_PR_ADD_PTR 334CK_PR_BIN(add, ptr, void, uintptr_t, +, void *) 335#endif /* CK_F_PR_ADD_PTR */ 336 337#ifndef CK_F_PR_SUB_PTR 338#define CK_F_PR_SUB_PTR 339CK_PR_BIN(sub, ptr, void, uintptr_t, -, void *) 340#endif /* CK_F_PR_SUB_PTR */ 341 342#ifndef CK_F_PR_AND_PTR 343#define CK_F_PR_AND_PTR 344CK_PR_BIN(and, ptr, void, uintptr_t, &, void *) 345#endif /* CK_F_PR_AND_PTR */ 346 347#ifndef CK_F_PR_XOR_PTR 348#define CK_F_PR_XOR_PTR 349CK_PR_BIN(xor, ptr, void, uintptr_t, ^, void *) 350#endif /* CK_F_PR_XOR_PTR */ 351 352#ifndef CK_F_PR_OR_PTR 353#define CK_F_PR_OR_PTR 354CK_PR_BIN(or, ptr, void, uintptr_t, |, void *) 355#endif /* CK_F_PR_OR_PTR */ 356 357#endif /* CK_F_PR_LOAD_PTR && CK_F_PR_CAS_PTR_VALUE */ 358 359#if defined(CK_F_PR_LOAD_64) && defined(CK_F_PR_CAS_64_VALUE) 360 361#ifndef CK_F_PR_ADD_64 362#define CK_F_PR_ADD_64 363CK_PR_BIN_S(add, 64, uint64_t, +) 364#endif /* CK_F_PR_ADD_64 */ 365 366#ifndef CK_F_PR_SUB_64 367#define CK_F_PR_SUB_64 368CK_PR_BIN_S(sub, 64, uint64_t, -) 369#endif /* CK_F_PR_SUB_64 */ 370 371#ifndef CK_F_PR_AND_64 372#define CK_F_PR_AND_64 373CK_PR_BIN_S(and, 64, uint64_t, &) 374#endif /* CK_F_PR_AND_64 */ 375 376#ifndef CK_F_PR_XOR_64 377#define CK_F_PR_XOR_64 378CK_PR_BIN_S(xor, 64, uint64_t, ^) 379#endif /* CK_F_PR_XOR_64 */ 380 381#ifndef CK_F_PR_OR_64 382#define CK_F_PR_OR_64 383CK_PR_BIN_S(or, 64, uint64_t, |) 384#endif /* CK_F_PR_OR_64 */ 385 386#endif /* CK_F_PR_LOAD_64 && CK_F_PR_CAS_64_VALUE */ 387 388#if defined(CK_F_PR_LOAD_32) && defined(CK_F_PR_CAS_32_VALUE) 389 390#ifndef CK_F_PR_ADD_32 391#define CK_F_PR_ADD_32 392CK_PR_BIN_S(add, 32, uint32_t, +) 393#endif /* CK_F_PR_ADD_32 */ 394 395#ifndef CK_F_PR_SUB_32 396#define CK_F_PR_SUB_32 397CK_PR_BIN_S(sub, 32, uint32_t, -) 398#endif /* CK_F_PR_SUB_32 */ 399 400#ifndef CK_F_PR_AND_32 401#define CK_F_PR_AND_32 402CK_PR_BIN_S(and, 32, uint32_t, &) 403#endif /* CK_F_PR_AND_32 */ 404 405#ifndef CK_F_PR_XOR_32 406#define CK_F_PR_XOR_32 407CK_PR_BIN_S(xor, 32, uint32_t, ^) 408#endif /* CK_F_PR_XOR_32 */ 409 410#ifndef CK_F_PR_OR_32 411#define CK_F_PR_OR_32 412CK_PR_BIN_S(or, 32, uint32_t, |) 413#endif /* CK_F_PR_OR_32 */ 414 415#endif /* CK_F_PR_LOAD_32 && CK_F_PR_CAS_32_VALUE */ 416 417#if defined(CK_F_PR_LOAD_16) && defined(CK_F_PR_CAS_16_VALUE) 418 419#ifndef CK_F_PR_ADD_16 420#define CK_F_PR_ADD_16 421CK_PR_BIN_S(add, 16, uint16_t, +) 422#endif /* CK_F_PR_ADD_16 */ 423 424#ifndef CK_F_PR_SUB_16 425#define CK_F_PR_SUB_16 426CK_PR_BIN_S(sub, 16, uint16_t, -) 427#endif /* CK_F_PR_SUB_16 */ 428 429#ifndef CK_F_PR_AND_16 430#define CK_F_PR_AND_16 431CK_PR_BIN_S(and, 16, uint16_t, &) 432#endif /* CK_F_PR_AND_16 */ 433 434#ifndef CK_F_PR_XOR_16 435#define CK_F_PR_XOR_16 436CK_PR_BIN_S(xor, 16, uint16_t, ^) 437#endif /* CK_F_PR_XOR_16 */ 438 439#ifndef CK_F_PR_OR_16 440#define CK_F_PR_OR_16 441CK_PR_BIN_S(or, 16, uint16_t, |) 442#endif /* CK_F_PR_OR_16 */ 443 444#endif /* CK_F_PR_LOAD_16 && CK_F_PR_CAS_16_VALUE */ 445 446#if defined(CK_F_PR_LOAD_8) && defined(CK_F_PR_CAS_8_VALUE) 447 448#ifndef CK_F_PR_ADD_8 449#define CK_F_PR_ADD_8 450CK_PR_BIN_S(add, 8, uint8_t, +) 451#endif /* CK_F_PR_ADD_8 */ 452 453#ifndef CK_F_PR_SUB_8 454#define CK_F_PR_SUB_8 455CK_PR_BIN_S(sub, 8, uint8_t, -) 456#endif /* CK_F_PR_SUB_8 */ 457 458#ifndef CK_F_PR_AND_8 459#define CK_F_PR_AND_8 460CK_PR_BIN_S(and, 8, uint8_t, &) 461#endif /* CK_F_PR_AND_8 */ 462 463#ifndef CK_F_PR_XOR_8 464#define CK_F_PR_XOR_8 465CK_PR_BIN_S(xor, 8, uint8_t, ^) 466#endif /* CK_F_PR_XOR_8 */ 467 468#ifndef CK_F_PR_OR_8 469#define CK_F_PR_OR_8 470CK_PR_BIN_S(or, 8, uint8_t, |) 471#endif /* CK_F_PR_OR_8 */ 472 473#endif /* CK_F_PR_LOAD_8 && CK_F_PR_CAS_8_VALUE */ 474 475#undef CK_PR_BIN_S 476#undef CK_PR_BIN 477 478#define CK_PR_BTX(K, S, M, T, P, C, R) \ 479 CK_CC_INLINE static bool \ 480 ck_pr_##K##_##S(M *target, unsigned int offset) \ 481 { \ 482 T previous; \ 483 C punt; \ 484 punt = ck_pr_md_load_##S(target); \ 485 previous = (T)punt; \ 486 while (ck_pr_cas_##S##_value(target, (C)previous, \ 487 (C)(previous P (R ((T)1 << offset))), &previous) == false) \ 488 ck_pr_stall(); \ 489 return ((previous >> offset) & 1); \ 490 } 491 492#define CK_PR_BTX_S(K, S, T, P, R) CK_PR_BTX(K, S, T, T, P, T, R) 493 494#if defined(CK_F_PR_LOAD_INT) && defined(CK_F_PR_CAS_INT_VALUE) 495 496#ifndef CK_F_PR_BTC_INT 497#define CK_F_PR_BTC_INT 498CK_PR_BTX_S(btc, int, int, ^,) 499#endif /* CK_F_PR_BTC_INT */ 500 501#ifndef CK_F_PR_BTR_INT 502#define CK_F_PR_BTR_INT 503CK_PR_BTX_S(btr, int, int, &, ~) 504#endif /* CK_F_PR_BTR_INT */ 505 506#ifndef CK_F_PR_BTS_INT 507#define CK_F_PR_BTS_INT 508CK_PR_BTX_S(bts, int, int, |,) 509#endif /* CK_F_PR_BTS_INT */ 510 511#endif /* CK_F_PR_LOAD_INT && CK_F_PR_CAS_INT_VALUE */ 512 513#if defined(CK_F_PR_LOAD_UINT) && defined(CK_F_PR_CAS_UINT_VALUE) 514 515#ifndef CK_F_PR_BTC_UINT 516#define CK_F_PR_BTC_UINT 517CK_PR_BTX_S(btc, uint, unsigned int, ^,) 518#endif /* CK_F_PR_BTC_UINT */ 519 520#ifndef CK_F_PR_BTR_UINT 521#define CK_F_PR_BTR_UINT 522CK_PR_BTX_S(btr, uint, unsigned int, &, ~) 523#endif /* CK_F_PR_BTR_UINT */ 524 525#ifndef CK_F_PR_BTS_UINT 526#define CK_F_PR_BTS_UINT 527CK_PR_BTX_S(bts, uint, unsigned int, |,) 528#endif /* CK_F_PR_BTS_UINT */ 529 530#endif /* CK_F_PR_LOAD_UINT && CK_F_PR_CAS_UINT_VALUE */ 531 532#if defined(CK_F_PR_LOAD_PTR) && defined(CK_F_PR_CAS_PTR_VALUE) 533 534#ifndef CK_F_PR_BTC_PTR 535#define CK_F_PR_BTC_PTR 536CK_PR_BTX(btc, ptr, void, uintptr_t, ^, void *,) 537#endif /* CK_F_PR_BTC_PTR */ 538 539#ifndef CK_F_PR_BTR_PTR 540#define CK_F_PR_BTR_PTR 541CK_PR_BTX(btr, ptr, void, uintptr_t, &, void *, ~) 542#endif /* CK_F_PR_BTR_PTR */ 543 544#ifndef CK_F_PR_BTS_PTR 545#define CK_F_PR_BTS_PTR 546CK_PR_BTX(bts, ptr, void, uintptr_t, |, void *,) 547#endif /* CK_F_PR_BTS_PTR */ 548 549#endif /* CK_F_PR_LOAD_PTR && CK_F_PR_CAS_PTR_VALUE */ 550 551#if defined(CK_F_PR_LOAD_64) && defined(CK_F_PR_CAS_64_VALUE) 552 553#ifndef CK_F_PR_BTC_64 554#define CK_F_PR_BTC_64 555CK_PR_BTX_S(btc, 64, uint64_t, ^,) 556#endif /* CK_F_PR_BTC_64 */ 557 558#ifndef CK_F_PR_BTR_64 559#define CK_F_PR_BTR_64 560CK_PR_BTX_S(btr, 64, uint64_t, &, ~) 561#endif /* CK_F_PR_BTR_64 */ 562 563#ifndef CK_F_PR_BTS_64 564#define CK_F_PR_BTS_64 565CK_PR_BTX_S(bts, 64, uint64_t, |,) 566#endif /* CK_F_PR_BTS_64 */ 567 568#endif /* CK_F_PR_LOAD_64 && CK_F_PR_CAS_64_VALUE */ 569 570#if defined(CK_F_PR_LOAD_32) && defined(CK_F_PR_CAS_32_VALUE) 571 572#ifndef CK_F_PR_BTC_32 573#define CK_F_PR_BTC_32 574CK_PR_BTX_S(btc, 32, uint32_t, ^,) 575#endif /* CK_F_PR_BTC_32 */ 576 577#ifndef CK_F_PR_BTR_32 578#define CK_F_PR_BTR_32 579CK_PR_BTX_S(btr, 32, uint32_t, &, ~) 580#endif /* CK_F_PR_BTR_32 */ 581 582#ifndef CK_F_PR_BTS_32 583#define CK_F_PR_BTS_32 584CK_PR_BTX_S(bts, 32, uint32_t, |,) 585#endif /* CK_F_PR_BTS_32 */ 586 587#endif /* CK_F_PR_LOAD_32 && CK_F_PR_CAS_32_VALUE */ 588 589#if defined(CK_F_PR_LOAD_16) && defined(CK_F_PR_CAS_16_VALUE) 590 591#ifndef CK_F_PR_BTC_16 592#define CK_F_PR_BTC_16 593CK_PR_BTX_S(btc, 16, uint16_t, ^,) 594#endif /* CK_F_PR_BTC_16 */ 595 596#ifndef CK_F_PR_BTR_16 597#define CK_F_PR_BTR_16 598CK_PR_BTX_S(btr, 16, uint16_t, &, ~) 599#endif /* CK_F_PR_BTR_16 */ 600 601#ifndef CK_F_PR_BTS_16 602#define CK_F_PR_BTS_16 603CK_PR_BTX_S(bts, 16, uint16_t, |,) 604#endif /* CK_F_PR_BTS_16 */ 605 606#endif /* CK_F_PR_LOAD_16 && CK_F_PR_CAS_16_VALUE */ 607 608#undef CK_PR_BTX_S 609#undef CK_PR_BTX 610 611#define CK_PR_UNARY(K, X, S, M, T) \ 612 CK_CC_INLINE static void \ 613 ck_pr_##K##_##S(M *target) \ 614 { \ 615 ck_pr_##X##_##S(target, (T)1); \ 616 return; \ 617 } 618 619#define CK_PR_UNARY_Z(K, S, M, T, P, C, Z) \ 620 CK_CC_INLINE static void \ 621 ck_pr_##K##_##S##_zero(M *target, bool *zero) \ 622 { \ 623 T previous; \ 624 C punt; \ 625 punt = (C)ck_pr_md_load_##S(target); \ 626 previous = (T)punt; \ 627 while (ck_pr_cas_##S##_value(target, \ 628 (C)previous, \ 629 (C)(previous P 1), \ 630 &previous) == false) \ 631 ck_pr_stall(); \ 632 *zero = previous == (T)Z; \ 633 return; \ 634 } 635 636#define CK_PR_UNARY_S(K, X, S, M) CK_PR_UNARY(K, X, S, M, M) 637#define CK_PR_UNARY_Z_S(K, S, M, P, Z) CK_PR_UNARY_Z(K, S, M, M, P, M, Z) 638 639#if defined(CK_F_PR_LOAD_CHAR) && defined(CK_F_PR_CAS_CHAR_VALUE) 640 641#ifndef CK_F_PR_INC_CHAR 642#define CK_F_PR_INC_CHAR 643CK_PR_UNARY_S(inc, add, char, char) 644#endif /* CK_F_PR_INC_CHAR */ 645 646#ifndef CK_F_PR_INC_CHAR_ZERO 647#define CK_F_PR_INC_CHAR_ZERO 648CK_PR_UNARY_Z_S(inc, char, char, +, -1) 649#endif /* CK_F_PR_INC_CHAR_ZERO */ 650 651#ifndef CK_F_PR_DEC_CHAR 652#define CK_F_PR_DEC_CHAR 653CK_PR_UNARY_S(dec, sub, char, char) 654#endif /* CK_F_PR_DEC_CHAR */ 655 656#ifndef CK_F_PR_DEC_CHAR_ZERO 657#define CK_F_PR_DEC_CHAR_ZERO 658CK_PR_UNARY_Z_S(dec, char, char, -, 1) 659#endif /* CK_F_PR_DEC_CHAR_ZERO */ 660 661#endif /* CK_F_PR_LOAD_CHAR && CK_F_PR_CAS_CHAR_VALUE */ 662 663#if defined(CK_F_PR_LOAD_INT) && defined(CK_F_PR_CAS_INT_VALUE) 664 665#ifndef CK_F_PR_INC_INT 666#define CK_F_PR_INC_INT 667CK_PR_UNARY_S(inc, add, int, int) 668#endif /* CK_F_PR_INC_INT */ 669 670#ifndef CK_F_PR_INC_INT_ZERO 671#define CK_F_PR_INC_INT_ZERO 672CK_PR_UNARY_Z_S(inc, int, int, +, -1) 673#endif /* CK_F_PR_INC_INT_ZERO */ 674 675#ifndef CK_F_PR_DEC_INT 676#define CK_F_PR_DEC_INT 677CK_PR_UNARY_S(dec, sub, int, int) 678#endif /* CK_F_PR_DEC_INT */ 679 680#ifndef CK_F_PR_DEC_INT_ZERO 681#define CK_F_PR_DEC_INT_ZERO 682CK_PR_UNARY_Z_S(dec, int, int, -, 1) 683#endif /* CK_F_PR_DEC_INT_ZERO */ 684 685#endif /* CK_F_PR_LOAD_INT && CK_F_PR_CAS_INT_VALUE */ 686 687#if defined(CK_F_PR_LOAD_DOUBLE) && defined(CK_F_PR_CAS_DOUBLE_VALUE) && \ 688 !defined(CK_PR_DISABLE_DOUBLE) 689 690#ifndef CK_F_PR_INC_DOUBLE 691#define CK_F_PR_INC_DOUBLE 692CK_PR_UNARY_S(inc, add, double, double) 693#endif /* CK_F_PR_INC_DOUBLE */ 694 695#ifndef CK_F_PR_DEC_DOUBLE 696#define CK_F_PR_DEC_DOUBLE 697CK_PR_UNARY_S(dec, sub, double, double) 698#endif /* CK_F_PR_DEC_DOUBLE */ 699 700#endif /* CK_F_PR_LOAD_DOUBLE && CK_F_PR_CAS_DOUBLE_VALUE && !CK_PR_DISABLE_DOUBLE */ 701 702#if defined(CK_F_PR_LOAD_UINT) && defined(CK_F_PR_CAS_UINT_VALUE) 703 704#ifndef CK_F_PR_INC_UINT 705#define CK_F_PR_INC_UINT 706CK_PR_UNARY_S(inc, add, uint, unsigned int) 707#endif /* CK_F_PR_INC_UINT */ 708 709#ifndef CK_F_PR_INC_UINT_ZERO 710#define CK_F_PR_INC_UINT_ZERO 711CK_PR_UNARY_Z_S(inc, uint, unsigned int, +, UINT_MAX) 712#endif /* CK_F_PR_INC_UINT_ZERO */ 713 714#ifndef CK_F_PR_DEC_UINT 715#define CK_F_PR_DEC_UINT 716CK_PR_UNARY_S(dec, sub, uint, unsigned int) 717#endif /* CK_F_PR_DEC_UINT */ 718 719#ifndef CK_F_PR_DEC_UINT_ZERO 720#define CK_F_PR_DEC_UINT_ZERO 721CK_PR_UNARY_Z_S(dec, uint, unsigned int, -, 1) 722#endif /* CK_F_PR_DEC_UINT_ZERO */ 723 724#endif /* CK_F_PR_LOAD_UINT && CK_F_PR_CAS_UINT_VALUE */ 725 726#if defined(CK_F_PR_LOAD_PTR) && defined(CK_F_PR_CAS_PTR_VALUE) 727 728#ifndef CK_F_PR_INC_PTR 729#define CK_F_PR_INC_PTR 730CK_PR_UNARY(inc, add, ptr, void, uintptr_t) 731#endif /* CK_F_PR_INC_PTR */ 732 733#ifndef CK_F_PR_INC_PTR_ZERO 734#define CK_F_PR_INC_PTR_ZERO 735CK_PR_UNARY_Z(inc, ptr, void, uintptr_t, +, void *, UINT_MAX) 736#endif /* CK_F_PR_INC_PTR_ZERO */ 737 738#ifndef CK_F_PR_DEC_PTR 739#define CK_F_PR_DEC_PTR 740CK_PR_UNARY(dec, sub, ptr, void, uintptr_t) 741#endif /* CK_F_PR_DEC_PTR */ 742 743#ifndef CK_F_PR_DEC_PTR_ZERO 744#define CK_F_PR_DEC_PTR_ZERO 745CK_PR_UNARY_Z(dec, ptr, void, uintptr_t, -, void *, 1) 746#endif /* CK_F_PR_DEC_PTR_ZERO */ 747 748#endif /* CK_F_PR_LOAD_PTR && CK_F_PR_CAS_PTR_VALUE */ 749 750#if defined(CK_F_PR_LOAD_64) && defined(CK_F_PR_CAS_64_VALUE) 751 752#ifndef CK_F_PR_INC_64 753#define CK_F_PR_INC_64 754CK_PR_UNARY_S(inc, add, 64, uint64_t) 755#endif /* CK_F_PR_INC_64 */ 756 757#ifndef CK_F_PR_INC_64_ZERO 758#define CK_F_PR_INC_64_ZERO 759CK_PR_UNARY_Z_S(inc, 64, uint64_t, +, UINT64_MAX) 760#endif /* CK_F_PR_INC_64_ZERO */ 761 762#ifndef CK_F_PR_DEC_64 763#define CK_F_PR_DEC_64 764CK_PR_UNARY_S(dec, sub, 64, uint64_t) 765#endif /* CK_F_PR_DEC_64 */ 766 767#ifndef CK_F_PR_DEC_64_ZERO 768#define CK_F_PR_DEC_64_ZERO 769CK_PR_UNARY_Z_S(dec, 64, uint64_t, -, 1) 770#endif /* CK_F_PR_DEC_64_ZERO */ 771 772#endif /* CK_F_PR_LOAD_64 && CK_F_PR_CAS_64_VALUE */ 773 774#if defined(CK_F_PR_LOAD_32) && defined(CK_F_PR_CAS_32_VALUE) 775 776#ifndef CK_F_PR_INC_32 777#define CK_F_PR_INC_32 778CK_PR_UNARY_S(inc, add, 32, uint32_t) 779#endif /* CK_F_PR_INC_32 */ 780 781#ifndef CK_F_PR_INC_32_ZERO 782#define CK_F_PR_INC_32_ZERO 783CK_PR_UNARY_Z_S(inc, 32, uint32_t, +, UINT32_MAX) 784#endif /* CK_F_PR_INC_32_ZERO */ 785 786#ifndef CK_F_PR_DEC_32 787#define CK_F_PR_DEC_32 788CK_PR_UNARY_S(dec, sub, 32, uint32_t) 789#endif /* CK_F_PR_DEC_32 */ 790 791#ifndef CK_F_PR_DEC_32_ZERO 792#define CK_F_PR_DEC_32_ZERO 793CK_PR_UNARY_Z_S(dec, 32, uint32_t, -, 1) 794#endif /* CK_F_PR_DEC_32_ZERO */ 795 796#endif /* CK_F_PR_LOAD_32 && CK_F_PR_CAS_32_VALUE */ 797 798#if defined(CK_F_PR_LOAD_16) && defined(CK_F_PR_CAS_16_VALUE) 799 800#ifndef CK_F_PR_INC_16 801#define CK_F_PR_INC_16 802CK_PR_UNARY_S(inc, add, 16, uint16_t) 803#endif /* CK_F_PR_INC_16 */ 804 805#ifndef CK_F_PR_INC_16_ZERO 806#define CK_F_PR_INC_16_ZERO 807CK_PR_UNARY_Z_S(inc, 16, uint16_t, +, UINT16_MAX) 808#endif /* CK_F_PR_INC_16_ZERO */ 809 810#ifndef CK_F_PR_DEC_16 811#define CK_F_PR_DEC_16 812CK_PR_UNARY_S(dec, sub, 16, uint16_t) 813#endif /* CK_F_PR_DEC_16 */ 814 815#ifndef CK_F_PR_DEC_16_ZERO 816#define CK_F_PR_DEC_16_ZERO 817CK_PR_UNARY_Z_S(dec, 16, uint16_t, -, 1) 818#endif /* CK_F_PR_DEC_16_ZERO */ 819 820#endif /* CK_F_PR_LOAD_16 && CK_F_PR_CAS_16_VALUE */ 821 822#if defined(CK_F_PR_LOAD_8) && defined(CK_F_PR_CAS_8_VALUE) 823 824#ifndef CK_F_PR_INC_8 825#define CK_F_PR_INC_8 826CK_PR_UNARY_S(inc, add, 8, uint8_t) 827#endif /* CK_F_PR_INC_8 */ 828 829#ifndef CK_F_PR_INC_8_ZERO 830#define CK_F_PR_INC_8_ZERO 831CK_PR_UNARY_Z_S(inc, 8, uint8_t, +, UINT8_MAX) 832#endif /* CK_F_PR_INC_8_ZERO */ 833 834#ifndef CK_F_PR_DEC_8 835#define CK_F_PR_DEC_8 836CK_PR_UNARY_S(dec, sub, 8, uint8_t) 837#endif /* CK_F_PR_DEC_8 */ 838 839#ifndef CK_F_PR_DEC_8_ZERO 840#define CK_F_PR_DEC_8_ZERO 841CK_PR_UNARY_Z_S(dec, 8, uint8_t, -, 1) 842#endif /* CK_F_PR_DEC_8_ZERO */ 843 844#endif /* CK_F_PR_LOAD_8 && CK_F_PR_CAS_8_VALUE */ 845 846#undef CK_PR_UNARY_Z_S 847#undef CK_PR_UNARY_S 848#undef CK_PR_UNARY_Z 849#undef CK_PR_UNARY 850 851#define CK_PR_N(K, S, M, T, P, C) \ 852 CK_CC_INLINE static void \ 853 ck_pr_##K##_##S(M *target) \ 854 { \ 855 T previous; \ 856 C punt; \ 857 punt = (C)ck_pr_md_load_##S(target); \ 858 previous = (T)punt; \ 859 while (ck_pr_cas_##S##_value(target, \ 860 (C)previous, \ 861 (C)(P previous), \ 862 &previous) == false) \ 863 ck_pr_stall(); \ 864 \ 865 return; \ 866 } 867 868#define CK_PR_N_Z(S, M, T, C) \ 869 CK_CC_INLINE static void \ 870 ck_pr_neg_##S##_zero(M *target, bool *zero) \ 871 { \ 872 T previous; \ 873 C punt; \ 874 punt = (C)ck_pr_md_load_##S(target); \ 875 previous = (T)punt; \ 876 while (ck_pr_cas_##S##_value(target, \ 877 (C)previous, \ 878 (C)(-previous), \ 879 &previous) == false) \ 880 ck_pr_stall(); \ 881 \ 882 *zero = previous == 0; \ 883 return; \ 884 } 885 886#define CK_PR_N_S(K, S, M, P) CK_PR_N(K, S, M, M, P, M) 887#define CK_PR_N_Z_S(S, M) CK_PR_N_Z(S, M, M, M) 888 889#if defined(CK_F_PR_LOAD_CHAR) && defined(CK_F_PR_CAS_CHAR_VALUE) 890 891#ifndef CK_F_PR_NOT_CHAR 892#define CK_F_PR_NOT_CHAR 893CK_PR_N_S(not, char, char, ~) 894#endif /* CK_F_PR_NOT_CHAR */ 895 896#ifndef CK_F_PR_NEG_CHAR 897#define CK_F_PR_NEG_CHAR 898CK_PR_N_S(neg, char, char, -) 899#endif /* CK_F_PR_NEG_CHAR */ 900 901#ifndef CK_F_PR_NEG_CHAR_ZERO 902#define CK_F_PR_NEG_CHAR_ZERO 903CK_PR_N_Z_S(char, char) 904#endif /* CK_F_PR_NEG_CHAR_ZERO */ 905 906#endif /* CK_F_PR_LOAD_CHAR && CK_F_PR_CAS_CHAR_VALUE */ 907 908#if defined(CK_F_PR_LOAD_INT) && defined(CK_F_PR_CAS_INT_VALUE) 909 910#ifndef CK_F_PR_NOT_INT 911#define CK_F_PR_NOT_INT 912CK_PR_N_S(not, int, int, ~) 913#endif /* CK_F_PR_NOT_INT */ 914 915#ifndef CK_F_PR_NEG_INT 916#define CK_F_PR_NEG_INT 917CK_PR_N_S(neg, int, int, -) 918#endif /* CK_F_PR_NEG_INT */ 919 920#ifndef CK_F_PR_NEG_INT_ZERO 921#define CK_F_PR_NEG_INT_ZERO 922CK_PR_N_Z_S(int, int) 923#endif /* CK_F_PR_NEG_INT_ZERO */ 924 925#endif /* CK_F_PR_LOAD_INT && CK_F_PR_CAS_INT_VALUE */ 926 927#if defined(CK_F_PR_LOAD_DOUBLE) && defined(CK_F_PR_CAS_DOUBLE_VALUE) && \ 928 !defined(CK_PR_DISABLE_DOUBLE) 929 930#ifndef CK_F_PR_NEG_DOUBLE 931#define CK_F_PR_NEG_DOUBLE 932CK_PR_N_S(neg, double, double, -) 933#endif /* CK_F_PR_NEG_DOUBLE */ 934 935#endif /* CK_F_PR_LOAD_DOUBLE && CK_F_PR_CAS_DOUBLE_VALUE && !CK_PR_DISABLE_DOUBLE */ 936 937#if defined(CK_F_PR_LOAD_UINT) && defined(CK_F_PR_CAS_UINT_VALUE) 938 939#ifndef CK_F_PR_NOT_UINT 940#define CK_F_PR_NOT_UINT 941CK_PR_N_S(not, uint, unsigned int, ~) 942#endif /* CK_F_PR_NOT_UINT */ 943 944#ifndef CK_F_PR_NEG_UINT 945#define CK_F_PR_NEG_UINT 946CK_PR_N_S(neg, uint, unsigned int, -) 947#endif /* CK_F_PR_NEG_UINT */ 948 949#ifndef CK_F_PR_NEG_UINT_ZERO 950#define CK_F_PR_NEG_UINT_ZERO 951CK_PR_N_Z_S(uint, unsigned int) 952#endif /* CK_F_PR_NEG_UINT_ZERO */ 953 954#endif /* CK_F_PR_LOAD_UINT && CK_F_PR_CAS_UINT_VALUE */ 955 956#if defined(CK_F_PR_LOAD_PTR) && defined(CK_F_PR_CAS_PTR_VALUE) 957 958#ifndef CK_F_PR_NOT_PTR 959#define CK_F_PR_NOT_PTR 960CK_PR_N(not, ptr, void, uintptr_t, ~, void *) 961#endif /* CK_F_PR_NOT_PTR */ 962 963#ifndef CK_F_PR_NEG_PTR 964#define CK_F_PR_NEG_PTR 965CK_PR_N(neg, ptr, void, uintptr_t, -, void *) 966#endif /* CK_F_PR_NEG_PTR */ 967 968#ifndef CK_F_PR_NEG_PTR_ZERO 969#define CK_F_PR_NEG_PTR_ZERO 970CK_PR_N_Z(ptr, void, uintptr_t, void *) 971#endif /* CK_F_PR_NEG_PTR_ZERO */ 972 973#endif /* CK_F_PR_LOAD_PTR && CK_F_PR_CAS_PTR_VALUE */ 974 975#if defined(CK_F_PR_LOAD_64) && defined(CK_F_PR_CAS_64_VALUE) 976 977#ifndef CK_F_PR_NOT_64 978#define CK_F_PR_NOT_64 979CK_PR_N_S(not, 64, uint64_t, ~) 980#endif /* CK_F_PR_NOT_64 */ 981 982#ifndef CK_F_PR_NEG_64 983#define CK_F_PR_NEG_64 984CK_PR_N_S(neg, 64, uint64_t, -) 985#endif /* CK_F_PR_NEG_64 */ 986 987#ifndef CK_F_PR_NEG_64_ZERO 988#define CK_F_PR_NEG_64_ZERO 989CK_PR_N_Z_S(64, uint64_t) 990#endif /* CK_F_PR_NEG_64_ZERO */ 991 992#endif /* CK_F_PR_LOAD_64 && CK_F_PR_CAS_64_VALUE */ 993 994#if defined(CK_F_PR_LOAD_32) && defined(CK_F_PR_CAS_32_VALUE) 995 996#ifndef CK_F_PR_NOT_32 997#define CK_F_PR_NOT_32 998CK_PR_N_S(not, 32, uint32_t, ~) 999#endif /* CK_F_PR_NOT_32 */ 1000 1001#ifndef CK_F_PR_NEG_32 1002#define CK_F_PR_NEG_32 1003CK_PR_N_S(neg, 32, uint32_t, -) 1004#endif /* CK_F_PR_NEG_32 */ 1005 1006#ifndef CK_F_PR_NEG_32_ZERO 1007#define CK_F_PR_NEG_32_ZERO 1008CK_PR_N_Z_S(32, uint32_t) 1009#endif /* CK_F_PR_NEG_32_ZERO */ 1010 1011#endif /* CK_F_PR_LOAD_32 && CK_F_PR_CAS_32_VALUE */ 1012 1013#if defined(CK_F_PR_LOAD_16) && defined(CK_F_PR_CAS_16_VALUE) 1014 1015#ifndef CK_F_PR_NOT_16 1016#define CK_F_PR_NOT_16 1017CK_PR_N_S(not, 16, uint16_t, ~) 1018#endif /* CK_F_PR_NOT_16 */ 1019 1020#ifndef CK_F_PR_NEG_16 1021#define CK_F_PR_NEG_16 1022CK_PR_N_S(neg, 16, uint16_t, -) 1023#endif /* CK_F_PR_NEG_16 */ 1024 1025#ifndef CK_F_PR_NEG_16_ZERO 1026#define CK_F_PR_NEG_16_ZERO 1027CK_PR_N_Z_S(16, uint16_t) 1028#endif /* CK_F_PR_NEG_16_ZERO */ 1029 1030#endif /* CK_F_PR_LOAD_16 && CK_F_PR_CAS_16_VALUE */ 1031 1032#if defined(CK_F_PR_LOAD_8) && defined(CK_F_PR_CAS_8_VALUE) 1033 1034#ifndef CK_F_PR_NOT_8 1035#define CK_F_PR_NOT_8 1036CK_PR_N_S(not, 8, uint8_t, ~) 1037#endif /* CK_F_PR_NOT_8 */ 1038 1039#ifndef CK_F_PR_NEG_8 1040#define CK_F_PR_NEG_8 1041CK_PR_N_S(neg, 8, uint8_t, -) 1042#endif /* CK_F_PR_NEG_8 */ 1043 1044#ifndef CK_F_PR_NEG_8_ZERO 1045#define CK_F_PR_NEG_8_ZERO 1046CK_PR_N_Z_S(8, uint8_t) 1047#endif /* CK_F_PR_NEG_8_ZERO */ 1048 1049#endif /* CK_F_PR_LOAD_8 && CK_F_PR_CAS_8_VALUE */ 1050 1051#undef CK_PR_N_Z_S 1052#undef CK_PR_N_S 1053#undef CK_PR_N_Z 1054#undef CK_PR_N 1055 1056#define CK_PR_FAA(S, M, T, C) \ 1057 CK_CC_INLINE static C \ 1058 ck_pr_faa_##S(M *target, T delta) \ 1059 { \ 1060 T previous; \ 1061 C punt; \ 1062 punt = (C)ck_pr_md_load_##S(target); \ 1063 previous = (T)punt; \ 1064 while (ck_pr_cas_##S##_value(target, \ 1065 (C)previous, \ 1066 (C)(previous + delta), \ 1067 &previous) == false) \ 1068 ck_pr_stall(); \ 1069 \ 1070 return ((C)previous); \ 1071 } 1072 1073#define CK_PR_FAS(S, M, C) \ 1074 CK_CC_INLINE static C \ 1075 ck_pr_fas_##S(M *target, C update) \ 1076 { \ 1077 C previous; \ 1078 previous = ck_pr_md_load_##S(target); \ 1079 while (ck_pr_cas_##S##_value(target, \ 1080 previous, \ 1081 update, \ 1082 &previous) == false) \ 1083 ck_pr_stall(); \ 1084 \ 1085 return (previous); \ 1086 } 1087 1088#define CK_PR_FAA_S(S, M) CK_PR_FAA(S, M, M, M) 1089#define CK_PR_FAS_S(S, M) CK_PR_FAS(S, M, M) 1090 1091#if defined(CK_F_PR_LOAD_CHAR) && defined(CK_F_PR_CAS_CHAR_VALUE) 1092 1093#ifndef CK_F_PR_FAA_CHAR 1094#define CK_F_PR_FAA_CHAR 1095CK_PR_FAA_S(char, char) 1096#endif /* CK_F_PR_FAA_CHAR */ 1097 1098#ifndef CK_F_PR_FAS_CHAR 1099#define CK_F_PR_FAS_CHAR 1100CK_PR_FAS_S(char, char) 1101#endif /* CK_F_PR_FAS_CHAR */ 1102 1103#endif /* CK_F_PR_LOAD_CHAR && CK_F_PR_CAS_CHAR_VALUE */ 1104 1105#if defined(CK_F_PR_LOAD_INT) && defined(CK_F_PR_CAS_INT_VALUE) 1106 1107#ifndef CK_F_PR_FAA_INT 1108#define CK_F_PR_FAA_INT 1109CK_PR_FAA_S(int, int) 1110#endif /* CK_F_PR_FAA_INT */ 1111 1112#ifndef CK_F_PR_FAS_INT 1113#define CK_F_PR_FAS_INT 1114CK_PR_FAS_S(int, int) 1115#endif /* CK_F_PR_FAS_INT */ 1116 1117#endif /* CK_F_PR_LOAD_INT && CK_F_PR_CAS_INT_VALUE */ 1118 1119#if defined(CK_F_PR_LOAD_DOUBLE) && defined(CK_F_PR_CAS_DOUBLE_VALUE) && \ 1120 !defined(CK_PR_DISABLE_DOUBLE) 1121 1122#ifndef CK_F_PR_FAA_DOUBLE 1123#define CK_F_PR_FAA_DOUBLE 1124CK_PR_FAA_S(double, double) 1125#endif /* CK_F_PR_FAA_DOUBLE */ 1126 1127#ifndef CK_F_PR_FAS_DOUBLE 1128#define CK_F_PR_FAS_DOUBLE 1129CK_PR_FAS_S(double, double) 1130#endif /* CK_F_PR_FAS_DOUBLE */ 1131 1132#endif /* CK_F_PR_LOAD_DOUBLE && CK_F_PR_CAS_DOUBLE_VALUE && !CK_PR_DISABLE_DOUBLE */ 1133 1134#if defined(CK_F_PR_LOAD_UINT) && defined(CK_F_PR_CAS_UINT_VALUE) 1135 1136#ifndef CK_F_PR_FAA_UINT 1137#define CK_F_PR_FAA_UINT 1138CK_PR_FAA_S(uint, unsigned int) 1139#endif /* CK_F_PR_FAA_UINT */ 1140 1141#ifndef CK_F_PR_FAS_UINT 1142#define CK_F_PR_FAS_UINT 1143CK_PR_FAS_S(uint, unsigned int) 1144#endif /* CK_F_PR_FAS_UINT */ 1145 1146#endif /* CK_F_PR_LOAD_UINT && CK_F_PR_CAS_UINT_VALUE */ 1147 1148#if defined(CK_F_PR_LOAD_PTR) && defined(CK_F_PR_CAS_PTR_VALUE) 1149 1150#ifndef CK_F_PR_FAA_PTR 1151#define CK_F_PR_FAA_PTR 1152CK_PR_FAA(ptr, void, uintptr_t, void *) 1153#endif /* CK_F_PR_FAA_PTR */ 1154 1155#ifndef CK_F_PR_FAS_PTR 1156#define CK_F_PR_FAS_PTR 1157CK_PR_FAS(ptr, void, void *) 1158#endif /* CK_F_PR_FAS_PTR */ 1159 1160#endif /* CK_F_PR_LOAD_PTR && CK_F_PR_CAS_PTR_VALUE */ 1161 1162#if defined(CK_F_PR_LOAD_64) && defined(CK_F_PR_CAS_64_VALUE) 1163 1164#ifndef CK_F_PR_FAA_64 1165#define CK_F_PR_FAA_64 1166CK_PR_FAA_S(64, uint64_t) 1167#endif /* CK_F_PR_FAA_64 */ 1168 1169#ifndef CK_F_PR_FAS_64 1170#define CK_F_PR_FAS_64 1171CK_PR_FAS_S(64, uint64_t) 1172#endif /* CK_F_PR_FAS_64 */ 1173 1174#endif /* CK_F_PR_LOAD_64 && CK_F_PR_CAS_64_VALUE */ 1175 1176#if defined(CK_F_PR_LOAD_32) && defined(CK_F_PR_CAS_32_VALUE) 1177 1178#ifndef CK_F_PR_FAA_32 1179#define CK_F_PR_FAA_32 1180CK_PR_FAA_S(32, uint32_t) 1181#endif /* CK_F_PR_FAA_32 */ 1182 1183#ifndef CK_F_PR_FAS_32 1184#define CK_F_PR_FAS_32 1185CK_PR_FAS_S(32, uint32_t) 1186#endif /* CK_F_PR_FAS_32 */ 1187 1188#endif /* CK_F_PR_LOAD_32 && CK_F_PR_CAS_32_VALUE */ 1189 1190#if defined(CK_F_PR_LOAD_16) && defined(CK_F_PR_CAS_16_VALUE) 1191 1192#ifndef CK_F_PR_FAA_16 1193#define CK_F_PR_FAA_16 1194CK_PR_FAA_S(16, uint16_t) 1195#endif /* CK_F_PR_FAA_16 */ 1196 1197#ifndef CK_F_PR_FAS_16 1198#define CK_F_PR_FAS_16 1199CK_PR_FAS_S(16, uint16_t) 1200#endif /* CK_F_PR_FAS_16 */ 1201 1202#endif /* CK_F_PR_LOAD_16 && CK_F_PR_CAS_16_VALUE */ 1203 1204#if defined(CK_F_PR_LOAD_8) && defined(CK_F_PR_CAS_8_VALUE) 1205 1206#ifndef CK_F_PR_FAA_8 1207#define CK_F_PR_FAA_8 1208CK_PR_FAA_S(8, uint8_t) 1209#endif /* CK_F_PR_FAA_8 */ 1210 1211#ifndef CK_F_PR_FAS_8 1212#define CK_F_PR_FAS_8 1213CK_PR_FAS_S(8, uint8_t) 1214#endif /* CK_F_PR_FAS_8 */ 1215 1216#endif /* CK_F_PR_LOAD_8 && CK_F_PR_CAS_8_VALUE */ 1217 1218#undef CK_PR_FAA_S 1219#undef CK_PR_FAS_S 1220#undef CK_PR_FAA 1221#undef CK_PR_FAS 1222 1223#endif /* CK_PR_H */ 1224