1/* 2 * Copyright 2009-2015 Samy Al Bahra. 3 * Copyright 2011 David Joseph. 4 * All rights reserved. 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25 * SUCH DAMAGE. 26 */ 27 28#ifndef CK_PR_H 29#define CK_PR_H 30 31#include <ck_cc.h> 32#include <ck_limits.h> 33#include <ck_md.h> 34#include <ck_stdint.h> 35#include <ck_stdbool.h> 36 37/* 38 * Default to using builtins for clang analyzer, coverity, and sparse: 39 * inline assembly is often too opaque for useful analysis. Override 40 * the defaults by defining CK_USE_CC_BUILTINS=0 or 1. 41 */ 42#if !defined(CK_USE_CC_BUILTINS) 43#if defined(__clang_analyzer__) || defined(__COVERITY__) || defined(__CHECKER__) 44#define CK_USE_CC_BUILTINS 1 45#else 46#define CK_USE_CC_BUILTINS 0 47#endif 48#endif 49 50#if !CK_USE_CC_BUILTINS 51#if defined(__x86_64__) 52#include "gcc/x86_64/ck_pr.h" 53#elif defined(__x86__) 54#include "gcc/x86/ck_pr.h" 55#elif defined(__sparcv9__) 56#include "gcc/sparcv9/ck_pr.h" 57#elif defined(__ppc64__) 58#include "gcc/ppc64/ck_pr.h" 59#elif defined(__s390x__) 60#include "gcc/s390x/ck_pr.h" 61#elif defined(__ppc__) 62#include "gcc/ppc/ck_pr.h" 63#elif defined(__arm__) 64#include "gcc/arm/ck_pr.h" 65#elif defined(__aarch64__) 66#include "gcc/aarch64/ck_pr.h" 67#elif !defined(__GNUC__) 68#error Your platform is unsupported 69#endif 70#endif /* !CK_USE_CC_BUILTINS */ 71 72#if defined(__GNUC__) 73#include "gcc/ck_pr.h" 74#endif 75 76#define CK_PR_FENCE_EMIT(T) \ 77 CK_CC_INLINE static void \ 78 ck_pr_fence_##T(void) \ 79 { \ 80 ck_pr_fence_strict_##T(); \ 81 return; \ 82 } 83#define CK_PR_FENCE_NOOP(T) \ 84 CK_CC_INLINE static void \ 85 ck_pr_fence_##T(void) \ 86 { \ 87 ck_pr_barrier(); \ 88 return; \ 89 } 90 91/* 92 * None of the currently supported platforms allow for data-dependent 93 * load ordering. 94 */ 95CK_PR_FENCE_NOOP(load_depends) 96#define ck_pr_fence_strict_load_depends ck_pr_fence_load_depends 97 98/* 99 * In memory models where atomic operations do not have serializing 100 * effects, atomic read-modify-write operations are modeled as stores. 101 */ 102#if defined(CK_MD_RMO) 103/* 104 * Only stores to the same location have a global 105 * ordering. 106 */ 107CK_PR_FENCE_EMIT(atomic) 108CK_PR_FENCE_EMIT(atomic_load) 109CK_PR_FENCE_EMIT(atomic_store) 110CK_PR_FENCE_EMIT(store_atomic) 111CK_PR_FENCE_EMIT(load_atomic) 112CK_PR_FENCE_EMIT(load_store) 113CK_PR_FENCE_EMIT(store_load) 114CK_PR_FENCE_EMIT(load) 115CK_PR_FENCE_EMIT(store) 116CK_PR_FENCE_EMIT(memory) 117CK_PR_FENCE_EMIT(acquire) 118CK_PR_FENCE_EMIT(release) 119CK_PR_FENCE_EMIT(acqrel) 120CK_PR_FENCE_EMIT(lock) 121CK_PR_FENCE_EMIT(unlock) 122#elif defined(CK_MD_PSO) 123/* 124 * Anything can be re-ordered with respect to stores. 125 * Otherwise, loads are executed in-order. 126 */ 127CK_PR_FENCE_EMIT(atomic) 128CK_PR_FENCE_NOOP(atomic_load) 129CK_PR_FENCE_EMIT(atomic_store) 130CK_PR_FENCE_EMIT(store_atomic) 131CK_PR_FENCE_NOOP(load_atomic) 132CK_PR_FENCE_EMIT(load_store) 133CK_PR_FENCE_EMIT(store_load) 134CK_PR_FENCE_NOOP(load) 135CK_PR_FENCE_EMIT(store) 136CK_PR_FENCE_EMIT(memory) 137CK_PR_FENCE_EMIT(acquire) 138CK_PR_FENCE_EMIT(release) 139CK_PR_FENCE_EMIT(acqrel) 140CK_PR_FENCE_EMIT(lock) 141CK_PR_FENCE_EMIT(unlock) 142#elif defined(CK_MD_TSO) 143/* 144 * Only loads are re-ordered and only with respect to 145 * prior stores. Atomic operations are serializing. 146 */ 147CK_PR_FENCE_NOOP(atomic) 148CK_PR_FENCE_NOOP(atomic_load) 149CK_PR_FENCE_NOOP(atomic_store) 150CK_PR_FENCE_NOOP(store_atomic) 151CK_PR_FENCE_NOOP(load_atomic) 152CK_PR_FENCE_NOOP(load_store) 153CK_PR_FENCE_EMIT(store_load) 154CK_PR_FENCE_NOOP(load) 155CK_PR_FENCE_NOOP(store) 156CK_PR_FENCE_EMIT(memory) 157CK_PR_FENCE_NOOP(acquire) 158CK_PR_FENCE_NOOP(release) 159CK_PR_FENCE_NOOP(acqrel) 160CK_PR_FENCE_NOOP(lock) 161CK_PR_FENCE_NOOP(unlock) 162#else 163#error "No memory model has been defined." 164#endif /* CK_MD_TSO */ 165 166#undef CK_PR_FENCE_EMIT 167#undef CK_PR_FENCE_NOOP 168 169#ifndef CK_F_PR_RFO 170#define CK_F_PR_RFO 171CK_CC_INLINE static void 172ck_pr_rfo(const void *m) 173{ 174 175 (void)m; 176 return; 177} 178#endif /* CK_F_PR_RFO */ 179 180#define CK_PR_STORE_SAFE(DST, VAL, TYPE) \ 181 ck_pr_md_store_##TYPE( \ 182 ((void)sizeof(*(DST) = (VAL)), (DST)), \ 183 (VAL)) 184 185#define ck_pr_store_ptr(DST, VAL) CK_PR_STORE_SAFE((DST), (VAL), ptr) 186#define ck_pr_store_char(DST, VAL) CK_PR_STORE_SAFE((DST), (VAL), char) 187#ifndef CK_PR_DISABLE_DOUBLE 188#define ck_pr_store_double(DST, VAL) CK_PR_STORE_SAFE((DST), (VAL), double) 189#endif 190#define ck_pr_store_uint(DST, VAL) CK_PR_STORE_SAFE((DST), (VAL), uint) 191#define ck_pr_store_int(DST, VAL) CK_PR_STORE_SAFE((DST), (VAL), int) 192#define ck_pr_store_32(DST, VAL) CK_PR_STORE_SAFE((DST), (VAL), 32) 193#define ck_pr_store_16(DST, VAL) CK_PR_STORE_SAFE((DST), (VAL), 16) 194#define ck_pr_store_8(DST, VAL) CK_PR_STORE_SAFE((DST), (VAL), 8) 195 196#define ck_pr_store_ptr_unsafe(DST, VAL) ck_pr_md_store_ptr((DST), (VAL)) 197 198#ifdef CK_F_PR_LOAD_64 199#define ck_pr_store_64(DST, VAL) CK_PR_STORE_SAFE((DST), (VAL), 64) 200#endif /* CK_F_PR_LOAD_64 */ 201 202#define CK_PR_LOAD_PTR_SAFE(SRC) (CK_CC_TYPEOF(*(SRC), (void *)))ck_pr_md_load_ptr((SRC)) 203#define ck_pr_load_ptr(SRC) CK_PR_LOAD_PTR_SAFE((SRC)) 204 205#define CK_PR_LOAD_SAFE(SRC, TYPE) ck_pr_md_load_##TYPE((SRC)) 206#define ck_pr_load_char(SRC) CK_PR_LOAD_SAFE((SRC), char) 207#ifndef CK_PR_DISABLE_DOUBLE 208#define ck_pr_load_double(SRC) CK_PR_LOAD_SAFE((SRC), double) 209#endif 210#define ck_pr_load_uint(SRC) CK_PR_LOAD_SAFE((SRC), uint) 211#define ck_pr_load_int(SRC) CK_PR_LOAD_SAFE((SRC), int) 212#define ck_pr_load_32(SRC) CK_PR_LOAD_SAFE((SRC), 32) 213#define ck_pr_load_16(SRC) CK_PR_LOAD_SAFE((SRC), 16) 214#define ck_pr_load_8(SRC) CK_PR_LOAD_SAFE((SRC), 8) 215 216#ifdef CK_F_PR_LOAD_64 217#define ck_pr_load_64(SRC) CK_PR_LOAD_SAFE((SRC), 64) 218#endif /* CK_F_PR_LOAD_64 */ 219 220#define CK_PR_BIN(K, S, M, T, P, C) \ 221 CK_CC_INLINE static void \ 222 ck_pr_##K##_##S(M *target, T value) \ 223 { \ 224 T previous; \ 225 C punt; \ 226 punt = ck_pr_md_load_##S(target); \ 227 previous = (T)punt; \ 228 while (ck_pr_cas_##S##_value(target, \ 229 (C)previous, \ 230 (C)(previous P value), \ 231 &previous) == false) \ 232 ck_pr_stall(); \ 233 \ 234 return; \ 235 } 236 237#define CK_PR_BIN_S(K, S, T, P) CK_PR_BIN(K, S, T, T, P, T) 238 239#if defined(CK_F_PR_LOAD_CHAR) && defined(CK_F_PR_CAS_CHAR_VALUE) 240 241#ifndef CK_F_PR_ADD_CHAR 242#define CK_F_PR_ADD_CHAR 243CK_PR_BIN_S(add, char, char, +) 244#endif /* CK_F_PR_ADD_CHAR */ 245 246#ifndef CK_F_PR_SUB_CHAR 247#define CK_F_PR_SUB_CHAR 248CK_PR_BIN_S(sub, char, char, -) 249#endif /* CK_F_PR_SUB_CHAR */ 250 251#ifndef CK_F_PR_AND_CHAR 252#define CK_F_PR_AND_CHAR 253CK_PR_BIN_S(and, char, char, &) 254#endif /* CK_F_PR_AND_CHAR */ 255 256#ifndef CK_F_PR_XOR_CHAR 257#define CK_F_PR_XOR_CHAR 258CK_PR_BIN_S(xor, char, char, ^) 259#endif /* CK_F_PR_XOR_CHAR */ 260 261#ifndef CK_F_PR_OR_CHAR 262#define CK_F_PR_OR_CHAR 263CK_PR_BIN_S(or, char, char, |) 264#endif /* CK_F_PR_OR_CHAR */ 265 266#endif /* CK_F_PR_LOAD_CHAR && CK_F_PR_CAS_CHAR_VALUE */ 267 268#if defined(CK_F_PR_LOAD_INT) && defined(CK_F_PR_CAS_INT_VALUE) 269 270#ifndef CK_F_PR_ADD_INT 271#define CK_F_PR_ADD_INT 272CK_PR_BIN_S(add, int, int, +) 273#endif /* CK_F_PR_ADD_INT */ 274 275#ifndef CK_F_PR_SUB_INT 276#define CK_F_PR_SUB_INT 277CK_PR_BIN_S(sub, int, int, -) 278#endif /* CK_F_PR_SUB_INT */ 279 280#ifndef CK_F_PR_AND_INT 281#define CK_F_PR_AND_INT 282CK_PR_BIN_S(and, int, int, &) 283#endif /* CK_F_PR_AND_INT */ 284 285#ifndef CK_F_PR_XOR_INT 286#define CK_F_PR_XOR_INT 287CK_PR_BIN_S(xor, int, int, ^) 288#endif /* CK_F_PR_XOR_INT */ 289 290#ifndef CK_F_PR_OR_INT 291#define CK_F_PR_OR_INT 292CK_PR_BIN_S(or, int, int, |) 293#endif /* CK_F_PR_OR_INT */ 294 295#endif /* CK_F_PR_LOAD_INT && CK_F_PR_CAS_INT_VALUE */ 296 297#if defined(CK_F_PR_LOAD_DOUBLE) && defined(CK_F_PR_CAS_DOUBLE_VALUE) && \ 298 !defined(CK_PR_DISABLE_DOUBLE) 299 300#ifndef CK_F_PR_ADD_DOUBLE 301#define CK_F_PR_ADD_DOUBLE 302CK_PR_BIN_S(add, double, double, +) 303#endif /* CK_F_PR_ADD_DOUBLE */ 304 305#ifndef CK_F_PR_SUB_DOUBLE 306#define CK_F_PR_SUB_DOUBLE 307CK_PR_BIN_S(sub, double, double, -) 308#endif /* CK_F_PR_SUB_DOUBLE */ 309 310#endif /* CK_F_PR_LOAD_DOUBLE && CK_F_PR_CAS_DOUBLE_VALUE && !CK_PR_DISABLE_DOUBLE */ 311 312#if defined(CK_F_PR_LOAD_UINT) && defined(CK_F_PR_CAS_UINT_VALUE) 313 314#ifndef CK_F_PR_ADD_UINT 315#define CK_F_PR_ADD_UINT 316CK_PR_BIN_S(add, uint, unsigned int, +) 317#endif /* CK_F_PR_ADD_UINT */ 318 319#ifndef CK_F_PR_SUB_UINT 320#define CK_F_PR_SUB_UINT 321CK_PR_BIN_S(sub, uint, unsigned int, -) 322#endif /* CK_F_PR_SUB_UINT */ 323 324#ifndef CK_F_PR_AND_UINT 325#define CK_F_PR_AND_UINT 326CK_PR_BIN_S(and, uint, unsigned int, &) 327#endif /* CK_F_PR_AND_UINT */ 328 329#ifndef CK_F_PR_XOR_UINT 330#define CK_F_PR_XOR_UINT 331CK_PR_BIN_S(xor, uint, unsigned int, ^) 332#endif /* CK_F_PR_XOR_UINT */ 333 334#ifndef CK_F_PR_OR_UINT 335#define CK_F_PR_OR_UINT 336CK_PR_BIN_S(or, uint, unsigned int, |) 337#endif /* CK_F_PR_OR_UINT */ 338 339#endif /* CK_F_PR_LOAD_UINT && CK_F_PR_CAS_UINT_VALUE */ 340 341#if defined(CK_F_PR_LOAD_PTR) && defined(CK_F_PR_CAS_PTR_VALUE) 342 343#ifndef CK_F_PR_ADD_PTR 344#define CK_F_PR_ADD_PTR 345CK_PR_BIN(add, ptr, void, uintptr_t, +, void *) 346#endif /* CK_F_PR_ADD_PTR */ 347 348#ifndef CK_F_PR_SUB_PTR 349#define CK_F_PR_SUB_PTR 350CK_PR_BIN(sub, ptr, void, uintptr_t, -, void *) 351#endif /* CK_F_PR_SUB_PTR */ 352 353#ifndef CK_F_PR_AND_PTR 354#define CK_F_PR_AND_PTR 355CK_PR_BIN(and, ptr, void, uintptr_t, &, void *) 356#endif /* CK_F_PR_AND_PTR */ 357 358#ifndef CK_F_PR_XOR_PTR 359#define CK_F_PR_XOR_PTR 360CK_PR_BIN(xor, ptr, void, uintptr_t, ^, void *) 361#endif /* CK_F_PR_XOR_PTR */ 362 363#ifndef CK_F_PR_OR_PTR 364#define CK_F_PR_OR_PTR 365CK_PR_BIN(or, ptr, void, uintptr_t, |, void *) 366#endif /* CK_F_PR_OR_PTR */ 367 368#endif /* CK_F_PR_LOAD_PTR && CK_F_PR_CAS_PTR_VALUE */ 369 370#if defined(CK_F_PR_LOAD_64) && defined(CK_F_PR_CAS_64_VALUE) 371 372#ifndef CK_F_PR_ADD_64 373#define CK_F_PR_ADD_64 374CK_PR_BIN_S(add, 64, uint64_t, +) 375#endif /* CK_F_PR_ADD_64 */ 376 377#ifndef CK_F_PR_SUB_64 378#define CK_F_PR_SUB_64 379CK_PR_BIN_S(sub, 64, uint64_t, -) 380#endif /* CK_F_PR_SUB_64 */ 381 382#ifndef CK_F_PR_AND_64 383#define CK_F_PR_AND_64 384CK_PR_BIN_S(and, 64, uint64_t, &) 385#endif /* CK_F_PR_AND_64 */ 386 387#ifndef CK_F_PR_XOR_64 388#define CK_F_PR_XOR_64 389CK_PR_BIN_S(xor, 64, uint64_t, ^) 390#endif /* CK_F_PR_XOR_64 */ 391 392#ifndef CK_F_PR_OR_64 393#define CK_F_PR_OR_64 394CK_PR_BIN_S(or, 64, uint64_t, |) 395#endif /* CK_F_PR_OR_64 */ 396 397#endif /* CK_F_PR_LOAD_64 && CK_F_PR_CAS_64_VALUE */ 398 399#if defined(CK_F_PR_LOAD_32) && defined(CK_F_PR_CAS_32_VALUE) 400 401#ifndef CK_F_PR_ADD_32 402#define CK_F_PR_ADD_32 403CK_PR_BIN_S(add, 32, uint32_t, +) 404#endif /* CK_F_PR_ADD_32 */ 405 406#ifndef CK_F_PR_SUB_32 407#define CK_F_PR_SUB_32 408CK_PR_BIN_S(sub, 32, uint32_t, -) 409#endif /* CK_F_PR_SUB_32 */ 410 411#ifndef CK_F_PR_AND_32 412#define CK_F_PR_AND_32 413CK_PR_BIN_S(and, 32, uint32_t, &) 414#endif /* CK_F_PR_AND_32 */ 415 416#ifndef CK_F_PR_XOR_32 417#define CK_F_PR_XOR_32 418CK_PR_BIN_S(xor, 32, uint32_t, ^) 419#endif /* CK_F_PR_XOR_32 */ 420 421#ifndef CK_F_PR_OR_32 422#define CK_F_PR_OR_32 423CK_PR_BIN_S(or, 32, uint32_t, |) 424#endif /* CK_F_PR_OR_32 */ 425 426#endif /* CK_F_PR_LOAD_32 && CK_F_PR_CAS_32_VALUE */ 427 428#if defined(CK_F_PR_LOAD_16) && defined(CK_F_PR_CAS_16_VALUE) 429 430#ifndef CK_F_PR_ADD_16 431#define CK_F_PR_ADD_16 432CK_PR_BIN_S(add, 16, uint16_t, +) 433#endif /* CK_F_PR_ADD_16 */ 434 435#ifndef CK_F_PR_SUB_16 436#define CK_F_PR_SUB_16 437CK_PR_BIN_S(sub, 16, uint16_t, -) 438#endif /* CK_F_PR_SUB_16 */ 439 440#ifndef CK_F_PR_AND_16 441#define CK_F_PR_AND_16 442CK_PR_BIN_S(and, 16, uint16_t, &) 443#endif /* CK_F_PR_AND_16 */ 444 445#ifndef CK_F_PR_XOR_16 446#define CK_F_PR_XOR_16 447CK_PR_BIN_S(xor, 16, uint16_t, ^) 448#endif /* CK_F_PR_XOR_16 */ 449 450#ifndef CK_F_PR_OR_16 451#define CK_F_PR_OR_16 452CK_PR_BIN_S(or, 16, uint16_t, |) 453#endif /* CK_F_PR_OR_16 */ 454 455#endif /* CK_F_PR_LOAD_16 && CK_F_PR_CAS_16_VALUE */ 456 457#if defined(CK_F_PR_LOAD_8) && defined(CK_F_PR_CAS_8_VALUE) 458 459#ifndef CK_F_PR_ADD_8 460#define CK_F_PR_ADD_8 461CK_PR_BIN_S(add, 8, uint8_t, +) 462#endif /* CK_F_PR_ADD_8 */ 463 464#ifndef CK_F_PR_SUB_8 465#define CK_F_PR_SUB_8 466CK_PR_BIN_S(sub, 8, uint8_t, -) 467#endif /* CK_F_PR_SUB_8 */ 468 469#ifndef CK_F_PR_AND_8 470#define CK_F_PR_AND_8 471CK_PR_BIN_S(and, 8, uint8_t, &) 472#endif /* CK_F_PR_AND_8 */ 473 474#ifndef CK_F_PR_XOR_8 475#define CK_F_PR_XOR_8 476CK_PR_BIN_S(xor, 8, uint8_t, ^) 477#endif /* CK_F_PR_XOR_8 */ 478 479#ifndef CK_F_PR_OR_8 480#define CK_F_PR_OR_8 481CK_PR_BIN_S(or, 8, uint8_t, |) 482#endif /* CK_F_PR_OR_8 */ 483 484#endif /* CK_F_PR_LOAD_8 && CK_F_PR_CAS_8_VALUE */ 485 486#undef CK_PR_BIN_S 487#undef CK_PR_BIN 488 489#define CK_PR_BTX(K, S, M, T, P, C, R) \ 490 CK_CC_INLINE static bool \ 491 ck_pr_##K##_##S(M *target, unsigned int offset) \ 492 { \ 493 T previous; \ 494 C punt; \ 495 punt = ck_pr_md_load_##S(target); \ 496 previous = (T)punt; \ 497 while (ck_pr_cas_##S##_value(target, (C)previous, \ 498 (C)(previous P (R ((T)1 << offset))), &previous) == false) \ 499 ck_pr_stall(); \ 500 return ((previous >> offset) & 1); \ 501 } 502 503#define CK_PR_BTX_S(K, S, T, P, R) CK_PR_BTX(K, S, T, T, P, T, R) 504 505#if defined(CK_F_PR_LOAD_INT) && defined(CK_F_PR_CAS_INT_VALUE) 506 507#ifndef CK_F_PR_BTC_INT 508#define CK_F_PR_BTC_INT 509CK_PR_BTX_S(btc, int, int, ^,) 510#endif /* CK_F_PR_BTC_INT */ 511 512#ifndef CK_F_PR_BTR_INT 513#define CK_F_PR_BTR_INT 514CK_PR_BTX_S(btr, int, int, &, ~) 515#endif /* CK_F_PR_BTR_INT */ 516 517#ifndef CK_F_PR_BTS_INT 518#define CK_F_PR_BTS_INT 519CK_PR_BTX_S(bts, int, int, |,) 520#endif /* CK_F_PR_BTS_INT */ 521 522#endif /* CK_F_PR_LOAD_INT && CK_F_PR_CAS_INT_VALUE */ 523 524#if defined(CK_F_PR_LOAD_UINT) && defined(CK_F_PR_CAS_UINT_VALUE) 525 526#ifndef CK_F_PR_BTC_UINT 527#define CK_F_PR_BTC_UINT 528CK_PR_BTX_S(btc, uint, unsigned int, ^,) 529#endif /* CK_F_PR_BTC_UINT */ 530 531#ifndef CK_F_PR_BTR_UINT 532#define CK_F_PR_BTR_UINT 533CK_PR_BTX_S(btr, uint, unsigned int, &, ~) 534#endif /* CK_F_PR_BTR_UINT */ 535 536#ifndef CK_F_PR_BTS_UINT 537#define CK_F_PR_BTS_UINT 538CK_PR_BTX_S(bts, uint, unsigned int, |,) 539#endif /* CK_F_PR_BTS_UINT */ 540 541#endif /* CK_F_PR_LOAD_UINT && CK_F_PR_CAS_UINT_VALUE */ 542 543#if defined(CK_F_PR_LOAD_PTR) && defined(CK_F_PR_CAS_PTR_VALUE) 544 545#ifndef CK_F_PR_BTC_PTR 546#define CK_F_PR_BTC_PTR 547CK_PR_BTX(btc, ptr, void, uintptr_t, ^, void *,) 548#endif /* CK_F_PR_BTC_PTR */ 549 550#ifndef CK_F_PR_BTR_PTR 551#define CK_F_PR_BTR_PTR 552CK_PR_BTX(btr, ptr, void, uintptr_t, &, void *, ~) 553#endif /* CK_F_PR_BTR_PTR */ 554 555#ifndef CK_F_PR_BTS_PTR 556#define CK_F_PR_BTS_PTR 557CK_PR_BTX(bts, ptr, void, uintptr_t, |, void *,) 558#endif /* CK_F_PR_BTS_PTR */ 559 560#endif /* CK_F_PR_LOAD_PTR && CK_F_PR_CAS_PTR_VALUE */ 561 562#if defined(CK_F_PR_LOAD_64) && defined(CK_F_PR_CAS_64_VALUE) 563 564#ifndef CK_F_PR_BTC_64 565#define CK_F_PR_BTC_64 566CK_PR_BTX_S(btc, 64, uint64_t, ^,) 567#endif /* CK_F_PR_BTC_64 */ 568 569#ifndef CK_F_PR_BTR_64 570#define CK_F_PR_BTR_64 571CK_PR_BTX_S(btr, 64, uint64_t, &, ~) 572#endif /* CK_F_PR_BTR_64 */ 573 574#ifndef CK_F_PR_BTS_64 575#define CK_F_PR_BTS_64 576CK_PR_BTX_S(bts, 64, uint64_t, |,) 577#endif /* CK_F_PR_BTS_64 */ 578 579#endif /* CK_F_PR_LOAD_64 && CK_F_PR_CAS_64_VALUE */ 580 581#if defined(CK_F_PR_LOAD_32) && defined(CK_F_PR_CAS_32_VALUE) 582 583#ifndef CK_F_PR_BTC_32 584#define CK_F_PR_BTC_32 585CK_PR_BTX_S(btc, 32, uint32_t, ^,) 586#endif /* CK_F_PR_BTC_32 */ 587 588#ifndef CK_F_PR_BTR_32 589#define CK_F_PR_BTR_32 590CK_PR_BTX_S(btr, 32, uint32_t, &, ~) 591#endif /* CK_F_PR_BTR_32 */ 592 593#ifndef CK_F_PR_BTS_32 594#define CK_F_PR_BTS_32 595CK_PR_BTX_S(bts, 32, uint32_t, |,) 596#endif /* CK_F_PR_BTS_32 */ 597 598#endif /* CK_F_PR_LOAD_32 && CK_F_PR_CAS_32_VALUE */ 599 600#if defined(CK_F_PR_LOAD_16) && defined(CK_F_PR_CAS_16_VALUE) 601 602#ifndef CK_F_PR_BTC_16 603#define CK_F_PR_BTC_16 604CK_PR_BTX_S(btc, 16, uint16_t, ^,) 605#endif /* CK_F_PR_BTC_16 */ 606 607#ifndef CK_F_PR_BTR_16 608#define CK_F_PR_BTR_16 609CK_PR_BTX_S(btr, 16, uint16_t, &, ~) 610#endif /* CK_F_PR_BTR_16 */ 611 612#ifndef CK_F_PR_BTS_16 613#define CK_F_PR_BTS_16 614CK_PR_BTX_S(bts, 16, uint16_t, |,) 615#endif /* CK_F_PR_BTS_16 */ 616 617#endif /* CK_F_PR_LOAD_16 && CK_F_PR_CAS_16_VALUE */ 618 619#undef CK_PR_BTX_S 620#undef CK_PR_BTX 621 622#define CK_PR_UNARY(K, X, S, M, T) \ 623 CK_CC_INLINE static void \ 624 ck_pr_##K##_##S(M *target) \ 625 { \ 626 ck_pr_##X##_##S(target, (T)1); \ 627 return; \ 628 } 629 630#define CK_PR_UNARY_Z(K, S, M, T, P, C, Z) \ 631 CK_CC_INLINE static bool \ 632 ck_pr_##K##_##S##_is_zero(M *target) \ 633 { \ 634 T previous; \ 635 C punt; \ 636 punt = (C)ck_pr_md_load_##S(target); \ 637 previous = (T)punt; \ 638 while (ck_pr_cas_##S##_value(target, \ 639 (C)previous, \ 640 (C)(previous P 1), \ 641 &previous) == false) \ 642 ck_pr_stall(); \ 643 return previous == (T)Z; \ 644 } 645 646#define CK_PR_UNARY_Z_STUB(K, S, M) \ 647 CK_CC_INLINE static void \ 648 ck_pr_##K##_##S##_zero(M *target, bool *zero) \ 649 { \ 650 *zero = ck_pr_##K##_##S##_is_zero(target); \ 651 return; \ 652 } 653 654#define CK_PR_UNARY_S(K, X, S, M) CK_PR_UNARY(K, X, S, M, M) 655#define CK_PR_UNARY_Z_S(K, S, M, P, Z) \ 656 CK_PR_UNARY_Z(K, S, M, M, P, M, Z) \ 657 CK_PR_UNARY_Z_STUB(K, S, M) 658 659#if defined(CK_F_PR_LOAD_CHAR) && defined(CK_F_PR_CAS_CHAR_VALUE) 660 661#ifndef CK_F_PR_INC_CHAR 662#define CK_F_PR_INC_CHAR 663CK_PR_UNARY_S(inc, add, char, char) 664#endif /* CK_F_PR_INC_CHAR */ 665 666#ifndef CK_F_PR_INC_CHAR_ZERO 667#define CK_F_PR_INC_CHAR_ZERO 668CK_PR_UNARY_Z_S(inc, char, char, +, -1) 669#else 670CK_PR_UNARY_Z_STUB(inc, char, char) 671#endif /* CK_F_PR_INC_CHAR_ZERO */ 672 673#ifndef CK_F_PR_DEC_CHAR 674#define CK_F_PR_DEC_CHAR 675CK_PR_UNARY_S(dec, sub, char, char) 676#endif /* CK_F_PR_DEC_CHAR */ 677 678#ifndef CK_F_PR_DEC_CHAR_ZERO 679#define CK_F_PR_DEC_CHAR_ZERO 680CK_PR_UNARY_Z_S(dec, char, char, -, 1) 681#else 682CK_PR_UNARY_Z_STUB(dec, char, char) 683#endif /* CK_F_PR_DEC_CHAR_ZERO */ 684 685#endif /* CK_F_PR_LOAD_CHAR && CK_F_PR_CAS_CHAR_VALUE */ 686 687#if defined(CK_F_PR_LOAD_INT) && defined(CK_F_PR_CAS_INT_VALUE) 688 689#ifndef CK_F_PR_INC_INT 690#define CK_F_PR_INC_INT 691CK_PR_UNARY_S(inc, add, int, int) 692#endif /* CK_F_PR_INC_INT */ 693 694#ifndef CK_F_PR_INC_INT_ZERO 695#define CK_F_PR_INC_INT_ZERO 696CK_PR_UNARY_Z_S(inc, int, int, +, -1) 697#else 698CK_PR_UNARY_Z_STUB(inc, int, int) 699#endif /* CK_F_PR_INC_INT_ZERO */ 700 701#ifndef CK_F_PR_DEC_INT 702#define CK_F_PR_DEC_INT 703CK_PR_UNARY_S(dec, sub, int, int) 704#endif /* CK_F_PR_DEC_INT */ 705 706#ifndef CK_F_PR_DEC_INT_ZERO 707#define CK_F_PR_DEC_INT_ZERO 708CK_PR_UNARY_Z_S(dec, int, int, -, 1) 709#else 710CK_PR_UNARY_Z_STUB(dec, int, int) 711#endif /* CK_F_PR_DEC_INT_ZERO */ 712 713#endif /* CK_F_PR_LOAD_INT && CK_F_PR_CAS_INT_VALUE */ 714 715#if defined(CK_F_PR_LOAD_DOUBLE) && defined(CK_F_PR_CAS_DOUBLE_VALUE) && \ 716 !defined(CK_PR_DISABLE_DOUBLE) 717 718#ifndef CK_F_PR_INC_DOUBLE 719#define CK_F_PR_INC_DOUBLE 720CK_PR_UNARY_S(inc, add, double, double) 721#endif /* CK_F_PR_INC_DOUBLE */ 722 723#ifndef CK_F_PR_DEC_DOUBLE 724#define CK_F_PR_DEC_DOUBLE 725CK_PR_UNARY_S(dec, sub, double, double) 726#endif /* CK_F_PR_DEC_DOUBLE */ 727 728#endif /* CK_F_PR_LOAD_DOUBLE && CK_F_PR_CAS_DOUBLE_VALUE && !CK_PR_DISABLE_DOUBLE */ 729 730#if defined(CK_F_PR_LOAD_UINT) && defined(CK_F_PR_CAS_UINT_VALUE) 731 732#ifndef CK_F_PR_INC_UINT 733#define CK_F_PR_INC_UINT 734CK_PR_UNARY_S(inc, add, uint, unsigned int) 735#endif /* CK_F_PR_INC_UINT */ 736 737#ifndef CK_F_PR_INC_UINT_ZERO 738#define CK_F_PR_INC_UINT_ZERO 739CK_PR_UNARY_Z_S(inc, uint, unsigned int, +, UINT_MAX) 740#else 741CK_PR_UNARY_Z_STUB(inc, uint, unsigned int) 742#endif /* CK_F_PR_INC_UINT_ZERO */ 743 744#ifndef CK_F_PR_DEC_UINT 745#define CK_F_PR_DEC_UINT 746CK_PR_UNARY_S(dec, sub, uint, unsigned int) 747#endif /* CK_F_PR_DEC_UINT */ 748 749#ifndef CK_F_PR_DEC_UINT_ZERO 750#define CK_F_PR_DEC_UINT_ZERO 751CK_PR_UNARY_Z_S(dec, uint, unsigned int, -, 1) 752#else 753CK_PR_UNARY_Z_STUB(dec, uint, unsigned int) 754#endif /* CK_F_PR_DEC_UINT_ZERO */ 755 756#endif /* CK_F_PR_LOAD_UINT && CK_F_PR_CAS_UINT_VALUE */ 757 758#if defined(CK_F_PR_LOAD_PTR) && defined(CK_F_PR_CAS_PTR_VALUE) 759 760#ifndef CK_F_PR_INC_PTR 761#define CK_F_PR_INC_PTR 762CK_PR_UNARY(inc, add, ptr, void, uintptr_t) 763#endif /* CK_F_PR_INC_PTR */ 764 765#ifndef CK_F_PR_INC_PTR_ZERO 766#define CK_F_PR_INC_PTR_ZERO 767CK_PR_UNARY_Z(inc, ptr, void, uintptr_t, +, void *, UINT_MAX) 768#else 769CK_PR_UNARY_Z_STUB(inc, ptr, void) 770#endif /* CK_F_PR_INC_PTR_ZERO */ 771 772#ifndef CK_F_PR_DEC_PTR 773#define CK_F_PR_DEC_PTR 774CK_PR_UNARY(dec, sub, ptr, void, uintptr_t) 775#endif /* CK_F_PR_DEC_PTR */ 776 777#ifndef CK_F_PR_DEC_PTR_ZERO 778#define CK_F_PR_DEC_PTR_ZERO 779CK_PR_UNARY_Z(dec, ptr, void, uintptr_t, -, void *, 1) 780#else 781CK_PR_UNARY_Z_STUB(dec, ptr, void) 782#endif /* CK_F_PR_DEC_PTR_ZERO */ 783 784#endif /* CK_F_PR_LOAD_PTR && CK_F_PR_CAS_PTR_VALUE */ 785 786#if defined(CK_F_PR_LOAD_64) && defined(CK_F_PR_CAS_64_VALUE) 787 788#ifndef CK_F_PR_INC_64 789#define CK_F_PR_INC_64 790CK_PR_UNARY_S(inc, add, 64, uint64_t) 791#endif /* CK_F_PR_INC_64 */ 792 793#ifndef CK_F_PR_INC_64_ZERO 794#define CK_F_PR_INC_64_ZERO 795CK_PR_UNARY_Z_S(inc, 64, uint64_t, +, UINT64_MAX) 796#else 797CK_PR_UNARY_Z_STUB(inc, 64, uint64_t) 798#endif /* CK_F_PR_INC_64_ZERO */ 799 800#ifndef CK_F_PR_DEC_64 801#define CK_F_PR_DEC_64 802CK_PR_UNARY_S(dec, sub, 64, uint64_t) 803#endif /* CK_F_PR_DEC_64 */ 804 805#ifndef CK_F_PR_DEC_64_ZERO 806#define CK_F_PR_DEC_64_ZERO 807CK_PR_UNARY_Z_S(dec, 64, uint64_t, -, 1) 808#else 809CK_PR_UNARY_Z_STUB(dec, 64, uint64_t) 810#endif /* CK_F_PR_DEC_64_ZERO */ 811 812#endif /* CK_F_PR_LOAD_64 && CK_F_PR_CAS_64_VALUE */ 813 814#if defined(CK_F_PR_LOAD_32) && defined(CK_F_PR_CAS_32_VALUE) 815 816#ifndef CK_F_PR_INC_32 817#define CK_F_PR_INC_32 818CK_PR_UNARY_S(inc, add, 32, uint32_t) 819#endif /* CK_F_PR_INC_32 */ 820 821#ifndef CK_F_PR_INC_32_ZERO 822#define CK_F_PR_INC_32_ZERO 823CK_PR_UNARY_Z_S(inc, 32, uint32_t, +, UINT32_MAX) 824#else 825CK_PR_UNARY_Z_STUB(inc, 32, uint32_t) 826#endif /* CK_F_PR_INC_32_ZERO */ 827 828#ifndef CK_F_PR_DEC_32 829#define CK_F_PR_DEC_32 830CK_PR_UNARY_S(dec, sub, 32, uint32_t) 831#endif /* CK_F_PR_DEC_32 */ 832 833#ifndef CK_F_PR_DEC_32_ZERO 834#define CK_F_PR_DEC_32_ZERO 835CK_PR_UNARY_Z_S(dec, 32, uint32_t, -, 1) 836#else 837CK_PR_UNARY_Z_STUB(dec, 32, uint32_t) 838#endif /* CK_F_PR_DEC_32_ZERO */ 839 840#endif /* CK_F_PR_LOAD_32 && CK_F_PR_CAS_32_VALUE */ 841 842#if defined(CK_F_PR_LOAD_16) && defined(CK_F_PR_CAS_16_VALUE) 843 844#ifndef CK_F_PR_INC_16 845#define CK_F_PR_INC_16 846CK_PR_UNARY_S(inc, add, 16, uint16_t) 847#endif /* CK_F_PR_INC_16 */ 848 849#ifndef CK_F_PR_INC_16_ZERO 850#define CK_F_PR_INC_16_ZERO 851CK_PR_UNARY_Z_S(inc, 16, uint16_t, +, UINT16_MAX) 852#else 853CK_PR_UNARY_Z_STUB(inc, 16, uint16_t) 854#endif /* CK_F_PR_INC_16_ZERO */ 855 856#ifndef CK_F_PR_DEC_16 857#define CK_F_PR_DEC_16 858CK_PR_UNARY_S(dec, sub, 16, uint16_t) 859#endif /* CK_F_PR_DEC_16 */ 860 861#ifndef CK_F_PR_DEC_16_ZERO 862#define CK_F_PR_DEC_16_ZERO 863CK_PR_UNARY_Z_S(dec, 16, uint16_t, -, 1) 864#else 865CK_PR_UNARY_Z_STUB(dec, 16, uint16_t) 866#endif /* CK_F_PR_DEC_16_ZERO */ 867 868#endif /* CK_F_PR_LOAD_16 && CK_F_PR_CAS_16_VALUE */ 869 870#if defined(CK_F_PR_LOAD_8) && defined(CK_F_PR_CAS_8_VALUE) 871 872#ifndef CK_F_PR_INC_8 873#define CK_F_PR_INC_8 874CK_PR_UNARY_S(inc, add, 8, uint8_t) 875#endif /* CK_F_PR_INC_8 */ 876 877#ifndef CK_F_PR_INC_8_ZERO 878#define CK_F_PR_INC_8_ZERO 879CK_PR_UNARY_Z_S(inc, 8, uint8_t, +, UINT8_MAX) 880#else 881CK_PR_UNARY_Z_STUB(inc, 8, uint8_t) 882#endif /* CK_F_PR_INC_8_ZERO */ 883 884#ifndef CK_F_PR_DEC_8 885#define CK_F_PR_DEC_8 886CK_PR_UNARY_S(dec, sub, 8, uint8_t) 887#endif /* CK_F_PR_DEC_8 */ 888 889#ifndef CK_F_PR_DEC_8_ZERO 890#define CK_F_PR_DEC_8_ZERO 891CK_PR_UNARY_Z_S(dec, 8, uint8_t, -, 1) 892#else 893CK_PR_UNARY_Z_STUB(dec, 8, uint8_t) 894#endif /* CK_F_PR_DEC_8_ZERO */ 895 896#endif /* CK_F_PR_LOAD_8 && CK_F_PR_CAS_8_VALUE */ 897 898#undef CK_PR_UNARY_Z_S 899#undef CK_PR_UNARY_S 900#undef CK_PR_UNARY_Z 901#undef CK_PR_UNARY 902 903#define CK_PR_N(K, S, M, T, P, C) \ 904 CK_CC_INLINE static void \ 905 ck_pr_##K##_##S(M *target) \ 906 { \ 907 T previous; \ 908 C punt; \ 909 punt = (C)ck_pr_md_load_##S(target); \ 910 previous = (T)punt; \ 911 while (ck_pr_cas_##S##_value(target, \ 912 (C)previous, \ 913 (C)(P previous), \ 914 &previous) == false) \ 915 ck_pr_stall(); \ 916 \ 917 return; \ 918 } 919 920#define CK_PR_N_Z(S, M, T, C) \ 921 CK_CC_INLINE static void \ 922 ck_pr_neg_##S##_zero(M *target, bool *zero) \ 923 { \ 924 T previous; \ 925 C punt; \ 926 punt = (C)ck_pr_md_load_##S(target); \ 927 previous = (T)punt; \ 928 while (ck_pr_cas_##S##_value(target, \ 929 (C)previous, \ 930 (C)(-previous), \ 931 &previous) == false) \ 932 ck_pr_stall(); \ 933 \ 934 *zero = previous == 0; \ 935 return; \ 936 } 937 938#define CK_PR_N_S(K, S, M, P) CK_PR_N(K, S, M, M, P, M) 939#define CK_PR_N_Z_S(S, M) CK_PR_N_Z(S, M, M, M) 940 941#if defined(CK_F_PR_LOAD_CHAR) && defined(CK_F_PR_CAS_CHAR_VALUE) 942 943#ifndef CK_F_PR_NOT_CHAR 944#define CK_F_PR_NOT_CHAR 945CK_PR_N_S(not, char, char, ~) 946#endif /* CK_F_PR_NOT_CHAR */ 947 948#ifndef CK_F_PR_NEG_CHAR 949#define CK_F_PR_NEG_CHAR 950CK_PR_N_S(neg, char, char, -) 951#endif /* CK_F_PR_NEG_CHAR */ 952 953#ifndef CK_F_PR_NEG_CHAR_ZERO 954#define CK_F_PR_NEG_CHAR_ZERO 955CK_PR_N_Z_S(char, char) 956#endif /* CK_F_PR_NEG_CHAR_ZERO */ 957 958#endif /* CK_F_PR_LOAD_CHAR && CK_F_PR_CAS_CHAR_VALUE */ 959 960#if defined(CK_F_PR_LOAD_INT) && defined(CK_F_PR_CAS_INT_VALUE) 961 962#ifndef CK_F_PR_NOT_INT 963#define CK_F_PR_NOT_INT 964CK_PR_N_S(not, int, int, ~) 965#endif /* CK_F_PR_NOT_INT */ 966 967#ifndef CK_F_PR_NEG_INT 968#define CK_F_PR_NEG_INT 969CK_PR_N_S(neg, int, int, -) 970#endif /* CK_F_PR_NEG_INT */ 971 972#ifndef CK_F_PR_NEG_INT_ZERO 973#define CK_F_PR_NEG_INT_ZERO 974CK_PR_N_Z_S(int, int) 975#endif /* CK_F_PR_NEG_INT_ZERO */ 976 977#endif /* CK_F_PR_LOAD_INT && CK_F_PR_CAS_INT_VALUE */ 978 979#if defined(CK_F_PR_LOAD_DOUBLE) && defined(CK_F_PR_CAS_DOUBLE_VALUE) && \ 980 !defined(CK_PR_DISABLE_DOUBLE) 981 982#ifndef CK_F_PR_NEG_DOUBLE 983#define CK_F_PR_NEG_DOUBLE 984CK_PR_N_S(neg, double, double, -) 985#endif /* CK_F_PR_NEG_DOUBLE */ 986 987#endif /* CK_F_PR_LOAD_DOUBLE && CK_F_PR_CAS_DOUBLE_VALUE && !CK_PR_DISABLE_DOUBLE */ 988 989#if defined(CK_F_PR_LOAD_UINT) && defined(CK_F_PR_CAS_UINT_VALUE) 990 991#ifndef CK_F_PR_NOT_UINT 992#define CK_F_PR_NOT_UINT 993CK_PR_N_S(not, uint, unsigned int, ~) 994#endif /* CK_F_PR_NOT_UINT */ 995 996#ifndef CK_F_PR_NEG_UINT 997#define CK_F_PR_NEG_UINT 998CK_PR_N_S(neg, uint, unsigned int, -) 999#endif /* CK_F_PR_NEG_UINT */ 1000 1001#ifndef CK_F_PR_NEG_UINT_ZERO 1002#define CK_F_PR_NEG_UINT_ZERO 1003CK_PR_N_Z_S(uint, unsigned int) 1004#endif /* CK_F_PR_NEG_UINT_ZERO */ 1005 1006#endif /* CK_F_PR_LOAD_UINT && CK_F_PR_CAS_UINT_VALUE */ 1007 1008#if defined(CK_F_PR_LOAD_PTR) && defined(CK_F_PR_CAS_PTR_VALUE) 1009 1010#ifndef CK_F_PR_NOT_PTR 1011#define CK_F_PR_NOT_PTR 1012CK_PR_N(not, ptr, void, uintptr_t, ~, void *) 1013#endif /* CK_F_PR_NOT_PTR */ 1014 1015#ifndef CK_F_PR_NEG_PTR 1016#define CK_F_PR_NEG_PTR 1017CK_PR_N(neg, ptr, void, uintptr_t, -, void *) 1018#endif /* CK_F_PR_NEG_PTR */ 1019 1020#ifndef CK_F_PR_NEG_PTR_ZERO 1021#define CK_F_PR_NEG_PTR_ZERO 1022CK_PR_N_Z(ptr, void, uintptr_t, void *) 1023#endif /* CK_F_PR_NEG_PTR_ZERO */ 1024 1025#endif /* CK_F_PR_LOAD_PTR && CK_F_PR_CAS_PTR_VALUE */ 1026 1027#if defined(CK_F_PR_LOAD_64) && defined(CK_F_PR_CAS_64_VALUE) 1028 1029#ifndef CK_F_PR_NOT_64 1030#define CK_F_PR_NOT_64 1031CK_PR_N_S(not, 64, uint64_t, ~) 1032#endif /* CK_F_PR_NOT_64 */ 1033 1034#ifndef CK_F_PR_NEG_64 1035#define CK_F_PR_NEG_64 1036CK_PR_N_S(neg, 64, uint64_t, -) 1037#endif /* CK_F_PR_NEG_64 */ 1038 1039#ifndef CK_F_PR_NEG_64_ZERO 1040#define CK_F_PR_NEG_64_ZERO 1041CK_PR_N_Z_S(64, uint64_t) 1042#endif /* CK_F_PR_NEG_64_ZERO */ 1043 1044#endif /* CK_F_PR_LOAD_64 && CK_F_PR_CAS_64_VALUE */ 1045 1046#if defined(CK_F_PR_LOAD_32) && defined(CK_F_PR_CAS_32_VALUE) 1047 1048#ifndef CK_F_PR_NOT_32 1049#define CK_F_PR_NOT_32 1050CK_PR_N_S(not, 32, uint32_t, ~) 1051#endif /* CK_F_PR_NOT_32 */ 1052 1053#ifndef CK_F_PR_NEG_32 1054#define CK_F_PR_NEG_32 1055CK_PR_N_S(neg, 32, uint32_t, -) 1056#endif /* CK_F_PR_NEG_32 */ 1057 1058#ifndef CK_F_PR_NEG_32_ZERO 1059#define CK_F_PR_NEG_32_ZERO 1060CK_PR_N_Z_S(32, uint32_t) 1061#endif /* CK_F_PR_NEG_32_ZERO */ 1062 1063#endif /* CK_F_PR_LOAD_32 && CK_F_PR_CAS_32_VALUE */ 1064 1065#if defined(CK_F_PR_LOAD_16) && defined(CK_F_PR_CAS_16_VALUE) 1066 1067#ifndef CK_F_PR_NOT_16 1068#define CK_F_PR_NOT_16 1069CK_PR_N_S(not, 16, uint16_t, ~) 1070#endif /* CK_F_PR_NOT_16 */ 1071 1072#ifndef CK_F_PR_NEG_16 1073#define CK_F_PR_NEG_16 1074CK_PR_N_S(neg, 16, uint16_t, -) 1075#endif /* CK_F_PR_NEG_16 */ 1076 1077#ifndef CK_F_PR_NEG_16_ZERO 1078#define CK_F_PR_NEG_16_ZERO 1079CK_PR_N_Z_S(16, uint16_t) 1080#endif /* CK_F_PR_NEG_16_ZERO */ 1081 1082#endif /* CK_F_PR_LOAD_16 && CK_F_PR_CAS_16_VALUE */ 1083 1084#if defined(CK_F_PR_LOAD_8) && defined(CK_F_PR_CAS_8_VALUE) 1085 1086#ifndef CK_F_PR_NOT_8 1087#define CK_F_PR_NOT_8 1088CK_PR_N_S(not, 8, uint8_t, ~) 1089#endif /* CK_F_PR_NOT_8 */ 1090 1091#ifndef CK_F_PR_NEG_8 1092#define CK_F_PR_NEG_8 1093CK_PR_N_S(neg, 8, uint8_t, -) 1094#endif /* CK_F_PR_NEG_8 */ 1095 1096#ifndef CK_F_PR_NEG_8_ZERO 1097#define CK_F_PR_NEG_8_ZERO 1098CK_PR_N_Z_S(8, uint8_t) 1099#endif /* CK_F_PR_NEG_8_ZERO */ 1100 1101#endif /* CK_F_PR_LOAD_8 && CK_F_PR_CAS_8_VALUE */ 1102 1103#undef CK_PR_N_Z_S 1104#undef CK_PR_N_S 1105#undef CK_PR_N_Z 1106#undef CK_PR_N 1107 1108#define CK_PR_FAA(S, M, T, C) \ 1109 CK_CC_INLINE static C \ 1110 ck_pr_faa_##S(M *target, T delta) \ 1111 { \ 1112 T previous; \ 1113 C punt; \ 1114 punt = (C)ck_pr_md_load_##S(target); \ 1115 previous = (T)punt; \ 1116 while (ck_pr_cas_##S##_value(target, \ 1117 (C)previous, \ 1118 (C)(previous + delta), \ 1119 &previous) == false) \ 1120 ck_pr_stall(); \ 1121 \ 1122 return ((C)previous); \ 1123 } 1124 1125#define CK_PR_FAS(S, M, C) \ 1126 CK_CC_INLINE static C \ 1127 ck_pr_fas_##S(M *target, C update) \ 1128 { \ 1129 C previous; \ 1130 previous = ck_pr_md_load_##S(target); \ 1131 while (ck_pr_cas_##S##_value(target, \ 1132 previous, \ 1133 update, \ 1134 &previous) == false) \ 1135 ck_pr_stall(); \ 1136 \ 1137 return (previous); \ 1138 } 1139 1140#define CK_PR_FAA_S(S, M) CK_PR_FAA(S, M, M, M) 1141#define CK_PR_FAS_S(S, M) CK_PR_FAS(S, M, M) 1142 1143#if defined(CK_F_PR_LOAD_CHAR) && defined(CK_F_PR_CAS_CHAR_VALUE) 1144 1145#ifndef CK_F_PR_FAA_CHAR 1146#define CK_F_PR_FAA_CHAR 1147CK_PR_FAA_S(char, char) 1148#endif /* CK_F_PR_FAA_CHAR */ 1149 1150#ifndef CK_F_PR_FAS_CHAR 1151#define CK_F_PR_FAS_CHAR 1152CK_PR_FAS_S(char, char) 1153#endif /* CK_F_PR_FAS_CHAR */ 1154 1155#endif /* CK_F_PR_LOAD_CHAR && CK_F_PR_CAS_CHAR_VALUE */ 1156 1157#if defined(CK_F_PR_LOAD_INT) && defined(CK_F_PR_CAS_INT_VALUE) 1158 1159#ifndef CK_F_PR_FAA_INT 1160#define CK_F_PR_FAA_INT 1161CK_PR_FAA_S(int, int) 1162#endif /* CK_F_PR_FAA_INT */ 1163 1164#ifndef CK_F_PR_FAS_INT 1165#define CK_F_PR_FAS_INT 1166CK_PR_FAS_S(int, int) 1167#endif /* CK_F_PR_FAS_INT */ 1168 1169#endif /* CK_F_PR_LOAD_INT && CK_F_PR_CAS_INT_VALUE */ 1170 1171#if defined(CK_F_PR_LOAD_DOUBLE) && defined(CK_F_PR_CAS_DOUBLE_VALUE) && \ 1172 !defined(CK_PR_DISABLE_DOUBLE) 1173 1174#ifndef CK_F_PR_FAA_DOUBLE 1175#define CK_F_PR_FAA_DOUBLE 1176CK_PR_FAA_S(double, double) 1177#endif /* CK_F_PR_FAA_DOUBLE */ 1178 1179#ifndef CK_F_PR_FAS_DOUBLE 1180#define CK_F_PR_FAS_DOUBLE 1181CK_PR_FAS_S(double, double) 1182#endif /* CK_F_PR_FAS_DOUBLE */ 1183 1184#endif /* CK_F_PR_LOAD_DOUBLE && CK_F_PR_CAS_DOUBLE_VALUE && !CK_PR_DISABLE_DOUBLE */ 1185 1186#if defined(CK_F_PR_LOAD_UINT) && defined(CK_F_PR_CAS_UINT_VALUE) 1187 1188#ifndef CK_F_PR_FAA_UINT 1189#define CK_F_PR_FAA_UINT 1190CK_PR_FAA_S(uint, unsigned int) 1191#endif /* CK_F_PR_FAA_UINT */ 1192 1193#ifndef CK_F_PR_FAS_UINT 1194#define CK_F_PR_FAS_UINT 1195CK_PR_FAS_S(uint, unsigned int) 1196#endif /* CK_F_PR_FAS_UINT */ 1197 1198#endif /* CK_F_PR_LOAD_UINT && CK_F_PR_CAS_UINT_VALUE */ 1199 1200#if defined(CK_F_PR_LOAD_PTR) && defined(CK_F_PR_CAS_PTR_VALUE) 1201 1202#ifndef CK_F_PR_FAA_PTR 1203#define CK_F_PR_FAA_PTR 1204CK_PR_FAA(ptr, void, uintptr_t, void *) 1205#endif /* CK_F_PR_FAA_PTR */ 1206 1207#ifndef CK_F_PR_FAS_PTR 1208#define CK_F_PR_FAS_PTR 1209CK_PR_FAS(ptr, void, void *) 1210#endif /* CK_F_PR_FAS_PTR */ 1211 1212#endif /* CK_F_PR_LOAD_PTR && CK_F_PR_CAS_PTR_VALUE */ 1213 1214#if defined(CK_F_PR_LOAD_64) && defined(CK_F_PR_CAS_64_VALUE) 1215 1216#ifndef CK_F_PR_FAA_64 1217#define CK_F_PR_FAA_64 1218CK_PR_FAA_S(64, uint64_t) 1219#endif /* CK_F_PR_FAA_64 */ 1220 1221#ifndef CK_F_PR_FAS_64 1222#define CK_F_PR_FAS_64 1223CK_PR_FAS_S(64, uint64_t) 1224#endif /* CK_F_PR_FAS_64 */ 1225 1226#endif /* CK_F_PR_LOAD_64 && CK_F_PR_CAS_64_VALUE */ 1227 1228#if defined(CK_F_PR_LOAD_32) && defined(CK_F_PR_CAS_32_VALUE) 1229 1230#ifndef CK_F_PR_FAA_32 1231#define CK_F_PR_FAA_32 1232CK_PR_FAA_S(32, uint32_t) 1233#endif /* CK_F_PR_FAA_32 */ 1234 1235#ifndef CK_F_PR_FAS_32 1236#define CK_F_PR_FAS_32 1237CK_PR_FAS_S(32, uint32_t) 1238#endif /* CK_F_PR_FAS_32 */ 1239 1240#endif /* CK_F_PR_LOAD_32 && CK_F_PR_CAS_32_VALUE */ 1241 1242#if defined(CK_F_PR_LOAD_16) && defined(CK_F_PR_CAS_16_VALUE) 1243 1244#ifndef CK_F_PR_FAA_16 1245#define CK_F_PR_FAA_16 1246CK_PR_FAA_S(16, uint16_t) 1247#endif /* CK_F_PR_FAA_16 */ 1248 1249#ifndef CK_F_PR_FAS_16 1250#define CK_F_PR_FAS_16 1251CK_PR_FAS_S(16, uint16_t) 1252#endif /* CK_F_PR_FAS_16 */ 1253 1254#endif /* CK_F_PR_LOAD_16 && CK_F_PR_CAS_16_VALUE */ 1255 1256#if defined(CK_F_PR_LOAD_8) && defined(CK_F_PR_CAS_8_VALUE) 1257 1258#ifndef CK_F_PR_FAA_8 1259#define CK_F_PR_FAA_8 1260CK_PR_FAA_S(8, uint8_t) 1261#endif /* CK_F_PR_FAA_8 */ 1262 1263#ifndef CK_F_PR_FAS_8 1264#define CK_F_PR_FAS_8 1265CK_PR_FAS_S(8, uint8_t) 1266#endif /* CK_F_PR_FAS_8 */ 1267 1268#endif /* CK_F_PR_LOAD_8 && CK_F_PR_CAS_8_VALUE */ 1269 1270#undef CK_PR_FAA_S 1271#undef CK_PR_FAS_S 1272#undef CK_PR_FAA 1273#undef CK_PR_FAS 1274 1275#endif /* CK_PR_H */ 1276