1/* 2 * Copyright 2009-2015 Samy Al Bahra. 3 * Copyright 2011 David Joseph. 4 * All rights reserved. 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25 * SUCH DAMAGE. 26 */ 27 28#ifndef CK_PR_H 29#define CK_PR_H 30 31#include <ck_cc.h> 32#include <ck_limits.h> 33#include <ck_md.h> 34#include <ck_stdint.h> 35#include <ck_stdbool.h> 36 37#ifndef CK_USE_CC_BUILTINS 38#if defined(__x86_64__) 39#include "gcc/x86_64/ck_pr.h" 40#elif defined(__x86__) 41#include "gcc/x86/ck_pr.h" 42#elif defined(__sparcv9__) 43#include "gcc/sparcv9/ck_pr.h" 44#elif defined(__ppc64__) 45#include "gcc/ppc64/ck_pr.h" 46#elif defined(__s390x__) 47#include "gcc/s390x/ck_pr.h" 48#elif defined(__ppc__) 49#include "gcc/ppc/ck_pr.h" 50#elif defined(__arm__) 51#include "gcc/arm/ck_pr.h" 52#elif defined(__aarch64__) 53#include "gcc/aarch64/ck_pr.h" 54#elif !defined(__GNUC__) 55#error Your platform is unsupported 56#endif 57#endif /* !CK_USE_CC_BUILTINS */ 58 59#if defined(__GNUC__) 60#include "gcc/ck_pr.h" 61#endif 62 63#define CK_PR_FENCE_EMIT(T) \ 64 CK_CC_INLINE static void \ 65 ck_pr_fence_##T(void) \ 66 { \ 67 ck_pr_fence_strict_##T(); \ 68 return; \ 69 } 70#define CK_PR_FENCE_NOOP(T) \ 71 CK_CC_INLINE static void \ 72 ck_pr_fence_##T(void) \ 73 { \ 74 ck_pr_barrier(); \ 75 return; \ 76 } 77 78/* 79 * None of the currently supported platforms allow for data-dependent 80 * load ordering. 81 */ 82CK_PR_FENCE_NOOP(load_depends) 83#define ck_pr_fence_strict_load_depends ck_pr_fence_load_depends 84 85/* 86 * In memory models where atomic operations do not have serializing 87 * effects, atomic read-modify-write operations are modeled as stores. 88 */ 89#if defined(CK_MD_RMO) 90/* 91 * Only stores to the same location have a global 92 * ordering. 93 */ 94CK_PR_FENCE_EMIT(atomic) 95CK_PR_FENCE_EMIT(atomic_load) 96CK_PR_FENCE_EMIT(atomic_store) 97CK_PR_FENCE_EMIT(store_atomic) 98CK_PR_FENCE_EMIT(load_atomic) 99CK_PR_FENCE_EMIT(load_store) 100CK_PR_FENCE_EMIT(store_load) 101CK_PR_FENCE_EMIT(load) 102CK_PR_FENCE_EMIT(store) 103CK_PR_FENCE_EMIT(memory) 104CK_PR_FENCE_EMIT(acquire) 105CK_PR_FENCE_EMIT(release) 106CK_PR_FENCE_EMIT(acqrel) 107CK_PR_FENCE_EMIT(lock) 108CK_PR_FENCE_EMIT(unlock) 109#elif defined(CK_MD_PSO) 110/* 111 * Anything can be re-ordered with respect to stores. 112 * Otherwise, loads are executed in-order. 113 */ 114CK_PR_FENCE_EMIT(atomic) 115CK_PR_FENCE_NOOP(atomic_load) 116CK_PR_FENCE_EMIT(atomic_store) 117CK_PR_FENCE_EMIT(store_atomic) 118CK_PR_FENCE_NOOP(load_atomic) 119CK_PR_FENCE_EMIT(load_store) 120CK_PR_FENCE_EMIT(store_load) 121CK_PR_FENCE_NOOP(load) 122CK_PR_FENCE_EMIT(store) 123CK_PR_FENCE_EMIT(memory) 124CK_PR_FENCE_EMIT(acquire) 125CK_PR_FENCE_EMIT(release) 126CK_PR_FENCE_EMIT(acqrel) 127CK_PR_FENCE_EMIT(lock) 128CK_PR_FENCE_EMIT(unlock) 129#elif defined(CK_MD_TSO) 130/* 131 * Only loads are re-ordered and only with respect to 132 * prior stores. Atomic operations are serializing. 133 */ 134CK_PR_FENCE_NOOP(atomic) 135CK_PR_FENCE_NOOP(atomic_load) 136CK_PR_FENCE_NOOP(atomic_store) 137CK_PR_FENCE_NOOP(store_atomic) 138CK_PR_FENCE_NOOP(load_atomic) 139CK_PR_FENCE_NOOP(load_store) 140CK_PR_FENCE_EMIT(store_load) 141CK_PR_FENCE_NOOP(load) 142CK_PR_FENCE_NOOP(store) 143CK_PR_FENCE_EMIT(memory) 144CK_PR_FENCE_NOOP(acquire) 145CK_PR_FENCE_NOOP(release) 146CK_PR_FENCE_NOOP(acqrel) 147CK_PR_FENCE_NOOP(lock) 148CK_PR_FENCE_NOOP(unlock) 149#else 150#error "No memory model has been defined." 151#endif /* CK_MD_TSO */ 152 153#undef CK_PR_FENCE_EMIT 154#undef CK_PR_FENCE_NOOP 155 156#ifndef CK_F_PR_RFO 157#define CK_F_PR_RFO 158CK_CC_INLINE static void 159ck_pr_rfo(const void *m) 160{ 161 162 (void)m; 163 return; 164} 165#endif /* CK_F_PR_RFO */ 166 167#define CK_PR_STORE_SAFE(DST, VAL, TYPE) \ 168 ck_pr_md_store_##TYPE( \ 169 ((void)sizeof(*(DST) = (VAL)), (DST)), \ 170 (VAL)) 171 172#define ck_pr_store_ptr(DST, VAL) CK_PR_STORE_SAFE((DST), (VAL), ptr) 173#define ck_pr_store_char(DST, VAL) CK_PR_STORE_SAFE((DST), (VAL), char) 174#ifndef CK_PR_DISABLE_DOUBLE 175#define ck_pr_store_double(DST, VAL) CK_PR_STORE_SAFE((DST), (VAL), double) 176#endif 177#define ck_pr_store_uint(DST, VAL) CK_PR_STORE_SAFE((DST), (VAL), uint) 178#define ck_pr_store_int(DST, VAL) CK_PR_STORE_SAFE((DST), (VAL), int) 179#define ck_pr_store_32(DST, VAL) CK_PR_STORE_SAFE((DST), (VAL), 32) 180#define ck_pr_store_16(DST, VAL) CK_PR_STORE_SAFE((DST), (VAL), 16) 181#define ck_pr_store_8(DST, VAL) CK_PR_STORE_SAFE((DST), (VAL), 8) 182 183#define ck_pr_store_ptr_unsafe(DST, VAL) ck_pr_md_store_ptr((DST), (VAL)) 184 185#ifdef CK_F_PR_LOAD_64 186#define ck_pr_store_64(DST, VAL) CK_PR_STORE_SAFE((DST), (VAL), 64) 187#endif /* CK_F_PR_LOAD_64 */ 188 189#define CK_PR_LOAD_PTR_SAFE(SRC) (CK_CC_TYPEOF(*(SRC), (void *)))ck_pr_md_load_ptr((SRC)) 190#define ck_pr_load_ptr(SRC) CK_PR_LOAD_PTR_SAFE((SRC)) 191 192#define CK_PR_LOAD_SAFE(SRC, TYPE) ck_pr_md_load_##TYPE((SRC)) 193#define ck_pr_load_char(SRC) CK_PR_LOAD_SAFE((SRC), char) 194#ifndef CK_PR_DISABLE_DOUBLE 195#define ck_pr_load_double(SRC) CK_PR_LOAD_SAFE((SRC), double) 196#endif 197#define ck_pr_load_uint(SRC) CK_PR_LOAD_SAFE((SRC), uint) 198#define ck_pr_load_int(SRC) CK_PR_LOAD_SAFE((SRC), int) 199#define ck_pr_load_32(SRC) CK_PR_LOAD_SAFE((SRC), 32) 200#define ck_pr_load_16(SRC) CK_PR_LOAD_SAFE((SRC), 16) 201#define ck_pr_load_8(SRC) CK_PR_LOAD_SAFE((SRC), 8) 202 203#ifdef CK_F_PR_LOAD_64 204#define ck_pr_load_64(SRC) CK_PR_LOAD_SAFE((SRC), 64) 205#endif /* CK_F_PR_LOAD_64 */ 206 207#define CK_PR_BIN(K, S, M, T, P, C) \ 208 CK_CC_INLINE static void \ 209 ck_pr_##K##_##S(M *target, T value) \ 210 { \ 211 T previous; \ 212 C punt; \ 213 punt = ck_pr_md_load_##S(target); \ 214 previous = (T)punt; \ 215 while (ck_pr_cas_##S##_value(target, \ 216 (C)previous, \ 217 (C)(previous P value), \ 218 &previous) == false) \ 219 ck_pr_stall(); \ 220 \ 221 return; \ 222 } 223 224#define CK_PR_BIN_S(K, S, T, P) CK_PR_BIN(K, S, T, T, P, T) 225 226#if defined(CK_F_PR_LOAD_CHAR) && defined(CK_F_PR_CAS_CHAR_VALUE) 227 228#ifndef CK_F_PR_ADD_CHAR 229#define CK_F_PR_ADD_CHAR 230CK_PR_BIN_S(add, char, char, +) 231#endif /* CK_F_PR_ADD_CHAR */ 232 233#ifndef CK_F_PR_SUB_CHAR 234#define CK_F_PR_SUB_CHAR 235CK_PR_BIN_S(sub, char, char, -) 236#endif /* CK_F_PR_SUB_CHAR */ 237 238#ifndef CK_F_PR_AND_CHAR 239#define CK_F_PR_AND_CHAR 240CK_PR_BIN_S(and, char, char, &) 241#endif /* CK_F_PR_AND_CHAR */ 242 243#ifndef CK_F_PR_XOR_CHAR 244#define CK_F_PR_XOR_CHAR 245CK_PR_BIN_S(xor, char, char, ^) 246#endif /* CK_F_PR_XOR_CHAR */ 247 248#ifndef CK_F_PR_OR_CHAR 249#define CK_F_PR_OR_CHAR 250CK_PR_BIN_S(or, char, char, |) 251#endif /* CK_F_PR_OR_CHAR */ 252 253#endif /* CK_F_PR_LOAD_CHAR && CK_F_PR_CAS_CHAR_VALUE */ 254 255#if defined(CK_F_PR_LOAD_INT) && defined(CK_F_PR_CAS_INT_VALUE) 256 257#ifndef CK_F_PR_ADD_INT 258#define CK_F_PR_ADD_INT 259CK_PR_BIN_S(add, int, int, +) 260#endif /* CK_F_PR_ADD_INT */ 261 262#ifndef CK_F_PR_SUB_INT 263#define CK_F_PR_SUB_INT 264CK_PR_BIN_S(sub, int, int, -) 265#endif /* CK_F_PR_SUB_INT */ 266 267#ifndef CK_F_PR_AND_INT 268#define CK_F_PR_AND_INT 269CK_PR_BIN_S(and, int, int, &) 270#endif /* CK_F_PR_AND_INT */ 271 272#ifndef CK_F_PR_XOR_INT 273#define CK_F_PR_XOR_INT 274CK_PR_BIN_S(xor, int, int, ^) 275#endif /* CK_F_PR_XOR_INT */ 276 277#ifndef CK_F_PR_OR_INT 278#define CK_F_PR_OR_INT 279CK_PR_BIN_S(or, int, int, |) 280#endif /* CK_F_PR_OR_INT */ 281 282#endif /* CK_F_PR_LOAD_INT && CK_F_PR_CAS_INT_VALUE */ 283 284#if defined(CK_F_PR_LOAD_DOUBLE) && defined(CK_F_PR_CAS_DOUBLE_VALUE) && \ 285 !defined(CK_PR_DISABLE_DOUBLE) 286 287#ifndef CK_F_PR_ADD_DOUBLE 288#define CK_F_PR_ADD_DOUBLE 289CK_PR_BIN_S(add, double, double, +) 290#endif /* CK_F_PR_ADD_DOUBLE */ 291 292#ifndef CK_F_PR_SUB_DOUBLE 293#define CK_F_PR_SUB_DOUBLE 294CK_PR_BIN_S(sub, double, double, -) 295#endif /* CK_F_PR_SUB_DOUBLE */ 296 297#endif /* CK_F_PR_LOAD_DOUBLE && CK_F_PR_CAS_DOUBLE_VALUE && !CK_PR_DISABLE_DOUBLE */ 298 299#if defined(CK_F_PR_LOAD_UINT) && defined(CK_F_PR_CAS_UINT_VALUE) 300 301#ifndef CK_F_PR_ADD_UINT 302#define CK_F_PR_ADD_UINT 303CK_PR_BIN_S(add, uint, unsigned int, +) 304#endif /* CK_F_PR_ADD_UINT */ 305 306#ifndef CK_F_PR_SUB_UINT 307#define CK_F_PR_SUB_UINT 308CK_PR_BIN_S(sub, uint, unsigned int, -) 309#endif /* CK_F_PR_SUB_UINT */ 310 311#ifndef CK_F_PR_AND_UINT 312#define CK_F_PR_AND_UINT 313CK_PR_BIN_S(and, uint, unsigned int, &) 314#endif /* CK_F_PR_AND_UINT */ 315 316#ifndef CK_F_PR_XOR_UINT 317#define CK_F_PR_XOR_UINT 318CK_PR_BIN_S(xor, uint, unsigned int, ^) 319#endif /* CK_F_PR_XOR_UINT */ 320 321#ifndef CK_F_PR_OR_UINT 322#define CK_F_PR_OR_UINT 323CK_PR_BIN_S(or, uint, unsigned int, |) 324#endif /* CK_F_PR_OR_UINT */ 325 326#endif /* CK_F_PR_LOAD_UINT && CK_F_PR_CAS_UINT_VALUE */ 327 328#if defined(CK_F_PR_LOAD_PTR) && defined(CK_F_PR_CAS_PTR_VALUE) 329 330#ifndef CK_F_PR_ADD_PTR 331#define CK_F_PR_ADD_PTR 332CK_PR_BIN(add, ptr, void, uintptr_t, +, void *) 333#endif /* CK_F_PR_ADD_PTR */ 334 335#ifndef CK_F_PR_SUB_PTR 336#define CK_F_PR_SUB_PTR 337CK_PR_BIN(sub, ptr, void, uintptr_t, -, void *) 338#endif /* CK_F_PR_SUB_PTR */ 339 340#ifndef CK_F_PR_AND_PTR 341#define CK_F_PR_AND_PTR 342CK_PR_BIN(and, ptr, void, uintptr_t, &, void *) 343#endif /* CK_F_PR_AND_PTR */ 344 345#ifndef CK_F_PR_XOR_PTR 346#define CK_F_PR_XOR_PTR 347CK_PR_BIN(xor, ptr, void, uintptr_t, ^, void *) 348#endif /* CK_F_PR_XOR_PTR */ 349 350#ifndef CK_F_PR_OR_PTR 351#define CK_F_PR_OR_PTR 352CK_PR_BIN(or, ptr, void, uintptr_t, |, void *) 353#endif /* CK_F_PR_OR_PTR */ 354 355#endif /* CK_F_PR_LOAD_PTR && CK_F_PR_CAS_PTR_VALUE */ 356 357#if defined(CK_F_PR_LOAD_64) && defined(CK_F_PR_CAS_64_VALUE) 358 359#ifndef CK_F_PR_ADD_64 360#define CK_F_PR_ADD_64 361CK_PR_BIN_S(add, 64, uint64_t, +) 362#endif /* CK_F_PR_ADD_64 */ 363 364#ifndef CK_F_PR_SUB_64 365#define CK_F_PR_SUB_64 366CK_PR_BIN_S(sub, 64, uint64_t, -) 367#endif /* CK_F_PR_SUB_64 */ 368 369#ifndef CK_F_PR_AND_64 370#define CK_F_PR_AND_64 371CK_PR_BIN_S(and, 64, uint64_t, &) 372#endif /* CK_F_PR_AND_64 */ 373 374#ifndef CK_F_PR_XOR_64 375#define CK_F_PR_XOR_64 376CK_PR_BIN_S(xor, 64, uint64_t, ^) 377#endif /* CK_F_PR_XOR_64 */ 378 379#ifndef CK_F_PR_OR_64 380#define CK_F_PR_OR_64 381CK_PR_BIN_S(or, 64, uint64_t, |) 382#endif /* CK_F_PR_OR_64 */ 383 384#endif /* CK_F_PR_LOAD_64 && CK_F_PR_CAS_64_VALUE */ 385 386#if defined(CK_F_PR_LOAD_32) && defined(CK_F_PR_CAS_32_VALUE) 387 388#ifndef CK_F_PR_ADD_32 389#define CK_F_PR_ADD_32 390CK_PR_BIN_S(add, 32, uint32_t, +) 391#endif /* CK_F_PR_ADD_32 */ 392 393#ifndef CK_F_PR_SUB_32 394#define CK_F_PR_SUB_32 395CK_PR_BIN_S(sub, 32, uint32_t, -) 396#endif /* CK_F_PR_SUB_32 */ 397 398#ifndef CK_F_PR_AND_32 399#define CK_F_PR_AND_32 400CK_PR_BIN_S(and, 32, uint32_t, &) 401#endif /* CK_F_PR_AND_32 */ 402 403#ifndef CK_F_PR_XOR_32 404#define CK_F_PR_XOR_32 405CK_PR_BIN_S(xor, 32, uint32_t, ^) 406#endif /* CK_F_PR_XOR_32 */ 407 408#ifndef CK_F_PR_OR_32 409#define CK_F_PR_OR_32 410CK_PR_BIN_S(or, 32, uint32_t, |) 411#endif /* CK_F_PR_OR_32 */ 412 413#endif /* CK_F_PR_LOAD_32 && CK_F_PR_CAS_32_VALUE */ 414 415#if defined(CK_F_PR_LOAD_16) && defined(CK_F_PR_CAS_16_VALUE) 416 417#ifndef CK_F_PR_ADD_16 418#define CK_F_PR_ADD_16 419CK_PR_BIN_S(add, 16, uint16_t, +) 420#endif /* CK_F_PR_ADD_16 */ 421 422#ifndef CK_F_PR_SUB_16 423#define CK_F_PR_SUB_16 424CK_PR_BIN_S(sub, 16, uint16_t, -) 425#endif /* CK_F_PR_SUB_16 */ 426 427#ifndef CK_F_PR_AND_16 428#define CK_F_PR_AND_16 429CK_PR_BIN_S(and, 16, uint16_t, &) 430#endif /* CK_F_PR_AND_16 */ 431 432#ifndef CK_F_PR_XOR_16 433#define CK_F_PR_XOR_16 434CK_PR_BIN_S(xor, 16, uint16_t, ^) 435#endif /* CK_F_PR_XOR_16 */ 436 437#ifndef CK_F_PR_OR_16 438#define CK_F_PR_OR_16 439CK_PR_BIN_S(or, 16, uint16_t, |) 440#endif /* CK_F_PR_OR_16 */ 441 442#endif /* CK_F_PR_LOAD_16 && CK_F_PR_CAS_16_VALUE */ 443 444#if defined(CK_F_PR_LOAD_8) && defined(CK_F_PR_CAS_8_VALUE) 445 446#ifndef CK_F_PR_ADD_8 447#define CK_F_PR_ADD_8 448CK_PR_BIN_S(add, 8, uint8_t, +) 449#endif /* CK_F_PR_ADD_8 */ 450 451#ifndef CK_F_PR_SUB_8 452#define CK_F_PR_SUB_8 453CK_PR_BIN_S(sub, 8, uint8_t, -) 454#endif /* CK_F_PR_SUB_8 */ 455 456#ifndef CK_F_PR_AND_8 457#define CK_F_PR_AND_8 458CK_PR_BIN_S(and, 8, uint8_t, &) 459#endif /* CK_F_PR_AND_8 */ 460 461#ifndef CK_F_PR_XOR_8 462#define CK_F_PR_XOR_8 463CK_PR_BIN_S(xor, 8, uint8_t, ^) 464#endif /* CK_F_PR_XOR_8 */ 465 466#ifndef CK_F_PR_OR_8 467#define CK_F_PR_OR_8 468CK_PR_BIN_S(or, 8, uint8_t, |) 469#endif /* CK_F_PR_OR_8 */ 470 471#endif /* CK_F_PR_LOAD_8 && CK_F_PR_CAS_8_VALUE */ 472 473#undef CK_PR_BIN_S 474#undef CK_PR_BIN 475 476#define CK_PR_BTX(K, S, M, T, P, C, R) \ 477 CK_CC_INLINE static bool \ 478 ck_pr_##K##_##S(M *target, unsigned int offset) \ 479 { \ 480 T previous; \ 481 C punt; \ 482 punt = ck_pr_md_load_##S(target); \ 483 previous = (T)punt; \ 484 while (ck_pr_cas_##S##_value(target, (C)previous, \ 485 (C)(previous P (R ((T)1 << offset))), &previous) == false) \ 486 ck_pr_stall(); \ 487 return ((previous >> offset) & 1); \ 488 } 489 490#define CK_PR_BTX_S(K, S, T, P, R) CK_PR_BTX(K, S, T, T, P, T, R) 491 492#if defined(CK_F_PR_LOAD_INT) && defined(CK_F_PR_CAS_INT_VALUE) 493 494#ifndef CK_F_PR_BTC_INT 495#define CK_F_PR_BTC_INT 496CK_PR_BTX_S(btc, int, int, ^,) 497#endif /* CK_F_PR_BTC_INT */ 498 499#ifndef CK_F_PR_BTR_INT 500#define CK_F_PR_BTR_INT 501CK_PR_BTX_S(btr, int, int, &, ~) 502#endif /* CK_F_PR_BTR_INT */ 503 504#ifndef CK_F_PR_BTS_INT 505#define CK_F_PR_BTS_INT 506CK_PR_BTX_S(bts, int, int, |,) 507#endif /* CK_F_PR_BTS_INT */ 508 509#endif /* CK_F_PR_LOAD_INT && CK_F_PR_CAS_INT_VALUE */ 510 511#if defined(CK_F_PR_LOAD_UINT) && defined(CK_F_PR_CAS_UINT_VALUE) 512 513#ifndef CK_F_PR_BTC_UINT 514#define CK_F_PR_BTC_UINT 515CK_PR_BTX_S(btc, uint, unsigned int, ^,) 516#endif /* CK_F_PR_BTC_UINT */ 517 518#ifndef CK_F_PR_BTR_UINT 519#define CK_F_PR_BTR_UINT 520CK_PR_BTX_S(btr, uint, unsigned int, &, ~) 521#endif /* CK_F_PR_BTR_UINT */ 522 523#ifndef CK_F_PR_BTS_UINT 524#define CK_F_PR_BTS_UINT 525CK_PR_BTX_S(bts, uint, unsigned int, |,) 526#endif /* CK_F_PR_BTS_UINT */ 527 528#endif /* CK_F_PR_LOAD_UINT && CK_F_PR_CAS_UINT_VALUE */ 529 530#if defined(CK_F_PR_LOAD_PTR) && defined(CK_F_PR_CAS_PTR_VALUE) 531 532#ifndef CK_F_PR_BTC_PTR 533#define CK_F_PR_BTC_PTR 534CK_PR_BTX(btc, ptr, void, uintptr_t, ^, void *,) 535#endif /* CK_F_PR_BTC_PTR */ 536 537#ifndef CK_F_PR_BTR_PTR 538#define CK_F_PR_BTR_PTR 539CK_PR_BTX(btr, ptr, void, uintptr_t, &, void *, ~) 540#endif /* CK_F_PR_BTR_PTR */ 541 542#ifndef CK_F_PR_BTS_PTR 543#define CK_F_PR_BTS_PTR 544CK_PR_BTX(bts, ptr, void, uintptr_t, |, void *,) 545#endif /* CK_F_PR_BTS_PTR */ 546 547#endif /* CK_F_PR_LOAD_PTR && CK_F_PR_CAS_PTR_VALUE */ 548 549#if defined(CK_F_PR_LOAD_64) && defined(CK_F_PR_CAS_64_VALUE) 550 551#ifndef CK_F_PR_BTC_64 552#define CK_F_PR_BTC_64 553CK_PR_BTX_S(btc, 64, uint64_t, ^,) 554#endif /* CK_F_PR_BTC_64 */ 555 556#ifndef CK_F_PR_BTR_64 557#define CK_F_PR_BTR_64 558CK_PR_BTX_S(btr, 64, uint64_t, &, ~) 559#endif /* CK_F_PR_BTR_64 */ 560 561#ifndef CK_F_PR_BTS_64 562#define CK_F_PR_BTS_64 563CK_PR_BTX_S(bts, 64, uint64_t, |,) 564#endif /* CK_F_PR_BTS_64 */ 565 566#endif /* CK_F_PR_LOAD_64 && CK_F_PR_CAS_64_VALUE */ 567 568#if defined(CK_F_PR_LOAD_32) && defined(CK_F_PR_CAS_32_VALUE) 569 570#ifndef CK_F_PR_BTC_32 571#define CK_F_PR_BTC_32 572CK_PR_BTX_S(btc, 32, uint32_t, ^,) 573#endif /* CK_F_PR_BTC_32 */ 574 575#ifndef CK_F_PR_BTR_32 576#define CK_F_PR_BTR_32 577CK_PR_BTX_S(btr, 32, uint32_t, &, ~) 578#endif /* CK_F_PR_BTR_32 */ 579 580#ifndef CK_F_PR_BTS_32 581#define CK_F_PR_BTS_32 582CK_PR_BTX_S(bts, 32, uint32_t, |,) 583#endif /* CK_F_PR_BTS_32 */ 584 585#endif /* CK_F_PR_LOAD_32 && CK_F_PR_CAS_32_VALUE */ 586 587#if defined(CK_F_PR_LOAD_16) && defined(CK_F_PR_CAS_16_VALUE) 588 589#ifndef CK_F_PR_BTC_16 590#define CK_F_PR_BTC_16 591CK_PR_BTX_S(btc, 16, uint16_t, ^,) 592#endif /* CK_F_PR_BTC_16 */ 593 594#ifndef CK_F_PR_BTR_16 595#define CK_F_PR_BTR_16 596CK_PR_BTX_S(btr, 16, uint16_t, &, ~) 597#endif /* CK_F_PR_BTR_16 */ 598 599#ifndef CK_F_PR_BTS_16 600#define CK_F_PR_BTS_16 601CK_PR_BTX_S(bts, 16, uint16_t, |,) 602#endif /* CK_F_PR_BTS_16 */ 603 604#endif /* CK_F_PR_LOAD_16 && CK_F_PR_CAS_16_VALUE */ 605 606#undef CK_PR_BTX_S 607#undef CK_PR_BTX 608 609#define CK_PR_UNARY(K, X, S, M, T) \ 610 CK_CC_INLINE static void \ 611 ck_pr_##K##_##S(M *target) \ 612 { \ 613 ck_pr_##X##_##S(target, (T)1); \ 614 return; \ 615 } 616 617#define CK_PR_UNARY_Z(K, S, M, T, P, C, Z) \ 618 CK_CC_INLINE static bool \ 619 ck_pr_##K##_##S##_is_zero(M *target) \ 620 { \ 621 T previous; \ 622 C punt; \ 623 punt = (C)ck_pr_md_load_##S(target); \ 624 previous = (T)punt; \ 625 while (ck_pr_cas_##S##_value(target, \ 626 (C)previous, \ 627 (C)(previous P 1), \ 628 &previous) == false) \ 629 ck_pr_stall(); \ 630 return previous == (T)Z; \ 631 } 632 633#define CK_PR_UNARY_Z_STUB(K, S, M) \ 634 CK_CC_INLINE static void \ 635 ck_pr_##K##_##S##_zero(M *target, bool *zero) \ 636 { \ 637 *zero = ck_pr_##K##_##S##_is_zero(target); \ 638 return; \ 639 } 640 641#define CK_PR_UNARY_S(K, X, S, M) CK_PR_UNARY(K, X, S, M, M) 642#define CK_PR_UNARY_Z_S(K, S, M, P, Z) \ 643 CK_PR_UNARY_Z(K, S, M, M, P, M, Z) \ 644 CK_PR_UNARY_Z_STUB(K, S, M) 645 646#if defined(CK_F_PR_LOAD_CHAR) && defined(CK_F_PR_CAS_CHAR_VALUE) 647 648#ifndef CK_F_PR_INC_CHAR 649#define CK_F_PR_INC_CHAR 650CK_PR_UNARY_S(inc, add, char, char) 651#endif /* CK_F_PR_INC_CHAR */ 652 653#ifndef CK_F_PR_INC_CHAR_ZERO 654#define CK_F_PR_INC_CHAR_ZERO 655CK_PR_UNARY_Z_S(inc, char, char, +, -1) 656#else 657CK_PR_UNARY_Z_STUB(inc, char, char) 658#endif /* CK_F_PR_INC_CHAR_ZERO */ 659 660#ifndef CK_F_PR_DEC_CHAR 661#define CK_F_PR_DEC_CHAR 662CK_PR_UNARY_S(dec, sub, char, char) 663#endif /* CK_F_PR_DEC_CHAR */ 664 665#ifndef CK_F_PR_DEC_CHAR_ZERO 666#define CK_F_PR_DEC_CHAR_ZERO 667CK_PR_UNARY_Z_S(dec, char, char, -, 1) 668#else 669CK_PR_UNARY_Z_STUB(dec, char, char) 670#endif /* CK_F_PR_DEC_CHAR_ZERO */ 671 672#endif /* CK_F_PR_LOAD_CHAR && CK_F_PR_CAS_CHAR_VALUE */ 673 674#if defined(CK_F_PR_LOAD_INT) && defined(CK_F_PR_CAS_INT_VALUE) 675 676#ifndef CK_F_PR_INC_INT 677#define CK_F_PR_INC_INT 678CK_PR_UNARY_S(inc, add, int, int) 679#endif /* CK_F_PR_INC_INT */ 680 681#ifndef CK_F_PR_INC_INT_ZERO 682#define CK_F_PR_INC_INT_ZERO 683CK_PR_UNARY_Z_S(inc, int, int, +, -1) 684#else 685CK_PR_UNARY_Z_STUB(inc, int, int) 686#endif /* CK_F_PR_INC_INT_ZERO */ 687 688#ifndef CK_F_PR_DEC_INT 689#define CK_F_PR_DEC_INT 690CK_PR_UNARY_S(dec, sub, int, int) 691#endif /* CK_F_PR_DEC_INT */ 692 693#ifndef CK_F_PR_DEC_INT_ZERO 694#define CK_F_PR_DEC_INT_ZERO 695CK_PR_UNARY_Z_S(dec, int, int, -, 1) 696#else 697CK_PR_UNARY_Z_STUB(dec, int, int) 698#endif /* CK_F_PR_DEC_INT_ZERO */ 699 700#endif /* CK_F_PR_LOAD_INT && CK_F_PR_CAS_INT_VALUE */ 701 702#if defined(CK_F_PR_LOAD_DOUBLE) && defined(CK_F_PR_CAS_DOUBLE_VALUE) && \ 703 !defined(CK_PR_DISABLE_DOUBLE) 704 705#ifndef CK_F_PR_INC_DOUBLE 706#define CK_F_PR_INC_DOUBLE 707CK_PR_UNARY_S(inc, add, double, double) 708#endif /* CK_F_PR_INC_DOUBLE */ 709 710#ifndef CK_F_PR_DEC_DOUBLE 711#define CK_F_PR_DEC_DOUBLE 712CK_PR_UNARY_S(dec, sub, double, double) 713#endif /* CK_F_PR_DEC_DOUBLE */ 714 715#endif /* CK_F_PR_LOAD_DOUBLE && CK_F_PR_CAS_DOUBLE_VALUE && !CK_PR_DISABLE_DOUBLE */ 716 717#if defined(CK_F_PR_LOAD_UINT) && defined(CK_F_PR_CAS_UINT_VALUE) 718 719#ifndef CK_F_PR_INC_UINT 720#define CK_F_PR_INC_UINT 721CK_PR_UNARY_S(inc, add, uint, unsigned int) 722#endif /* CK_F_PR_INC_UINT */ 723 724#ifndef CK_F_PR_INC_UINT_ZERO 725#define CK_F_PR_INC_UINT_ZERO 726CK_PR_UNARY_Z_S(inc, uint, unsigned int, +, UINT_MAX) 727#else 728CK_PR_UNARY_Z_STUB(inc, uint, unsigned int) 729#endif /* CK_F_PR_INC_UINT_ZERO */ 730 731#ifndef CK_F_PR_DEC_UINT 732#define CK_F_PR_DEC_UINT 733CK_PR_UNARY_S(dec, sub, uint, unsigned int) 734#endif /* CK_F_PR_DEC_UINT */ 735 736#ifndef CK_F_PR_DEC_UINT_ZERO 737#define CK_F_PR_DEC_UINT_ZERO 738CK_PR_UNARY_Z_S(dec, uint, unsigned int, -, 1) 739#else 740CK_PR_UNARY_Z_STUB(dec, uint, unsigned int) 741#endif /* CK_F_PR_DEC_UINT_ZERO */ 742 743#endif /* CK_F_PR_LOAD_UINT && CK_F_PR_CAS_UINT_VALUE */ 744 745#if defined(CK_F_PR_LOAD_PTR) && defined(CK_F_PR_CAS_PTR_VALUE) 746 747#ifndef CK_F_PR_INC_PTR 748#define CK_F_PR_INC_PTR 749CK_PR_UNARY(inc, add, ptr, void, uintptr_t) 750#endif /* CK_F_PR_INC_PTR */ 751 752#ifndef CK_F_PR_INC_PTR_ZERO 753#define CK_F_PR_INC_PTR_ZERO 754CK_PR_UNARY_Z(inc, ptr, void, uintptr_t, +, void *, UINT_MAX) 755#else 756CK_PR_UNARY_Z_STUB(inc, ptr, void) 757#endif /* CK_F_PR_INC_PTR_ZERO */ 758 759#ifndef CK_F_PR_DEC_PTR 760#define CK_F_PR_DEC_PTR 761CK_PR_UNARY(dec, sub, ptr, void, uintptr_t) 762#endif /* CK_F_PR_DEC_PTR */ 763 764#ifndef CK_F_PR_DEC_PTR_ZERO 765#define CK_F_PR_DEC_PTR_ZERO 766CK_PR_UNARY_Z(dec, ptr, void, uintptr_t, -, void *, 1) 767#else 768CK_PR_UNARY_Z_STUB(dec, ptr, void) 769#endif /* CK_F_PR_DEC_PTR_ZERO */ 770 771#endif /* CK_F_PR_LOAD_PTR && CK_F_PR_CAS_PTR_VALUE */ 772 773#if defined(CK_F_PR_LOAD_64) && defined(CK_F_PR_CAS_64_VALUE) 774 775#ifndef CK_F_PR_INC_64 776#define CK_F_PR_INC_64 777CK_PR_UNARY_S(inc, add, 64, uint64_t) 778#endif /* CK_F_PR_INC_64 */ 779 780#ifndef CK_F_PR_INC_64_ZERO 781#define CK_F_PR_INC_64_ZERO 782CK_PR_UNARY_Z_S(inc, 64, uint64_t, +, UINT64_MAX) 783#else 784CK_PR_UNARY_Z_STUB(inc, 64, uint64_t) 785#endif /* CK_F_PR_INC_64_ZERO */ 786 787#ifndef CK_F_PR_DEC_64 788#define CK_F_PR_DEC_64 789CK_PR_UNARY_S(dec, sub, 64, uint64_t) 790#endif /* CK_F_PR_DEC_64 */ 791 792#ifndef CK_F_PR_DEC_64_ZERO 793#define CK_F_PR_DEC_64_ZERO 794CK_PR_UNARY_Z_S(dec, 64, uint64_t, -, 1) 795#else 796CK_PR_UNARY_Z_STUB(dec, 64, uint64_t) 797#endif /* CK_F_PR_DEC_64_ZERO */ 798 799#endif /* CK_F_PR_LOAD_64 && CK_F_PR_CAS_64_VALUE */ 800 801#if defined(CK_F_PR_LOAD_32) && defined(CK_F_PR_CAS_32_VALUE) 802 803#ifndef CK_F_PR_INC_32 804#define CK_F_PR_INC_32 805CK_PR_UNARY_S(inc, add, 32, uint32_t) 806#endif /* CK_F_PR_INC_32 */ 807 808#ifndef CK_F_PR_INC_32_ZERO 809#define CK_F_PR_INC_32_ZERO 810CK_PR_UNARY_Z_S(inc, 32, uint32_t, +, UINT32_MAX) 811#else 812CK_PR_UNARY_Z_STUB(inc, 32, uint32_t) 813#endif /* CK_F_PR_INC_32_ZERO */ 814 815#ifndef CK_F_PR_DEC_32 816#define CK_F_PR_DEC_32 817CK_PR_UNARY_S(dec, sub, 32, uint32_t) 818#endif /* CK_F_PR_DEC_32 */ 819 820#ifndef CK_F_PR_DEC_32_ZERO 821#define CK_F_PR_DEC_32_ZERO 822CK_PR_UNARY_Z_S(dec, 32, uint32_t, -, 1) 823#else 824CK_PR_UNARY_Z_STUB(dec, 32, uint32_t) 825#endif /* CK_F_PR_DEC_32_ZERO */ 826 827#endif /* CK_F_PR_LOAD_32 && CK_F_PR_CAS_32_VALUE */ 828 829#if defined(CK_F_PR_LOAD_16) && defined(CK_F_PR_CAS_16_VALUE) 830 831#ifndef CK_F_PR_INC_16 832#define CK_F_PR_INC_16 833CK_PR_UNARY_S(inc, add, 16, uint16_t) 834#endif /* CK_F_PR_INC_16 */ 835 836#ifndef CK_F_PR_INC_16_ZERO 837#define CK_F_PR_INC_16_ZERO 838CK_PR_UNARY_Z_S(inc, 16, uint16_t, +, UINT16_MAX) 839#else 840CK_PR_UNARY_Z_STUB(inc, 16, uint16_t) 841#endif /* CK_F_PR_INC_16_ZERO */ 842 843#ifndef CK_F_PR_DEC_16 844#define CK_F_PR_DEC_16 845CK_PR_UNARY_S(dec, sub, 16, uint16_t) 846#endif /* CK_F_PR_DEC_16 */ 847 848#ifndef CK_F_PR_DEC_16_ZERO 849#define CK_F_PR_DEC_16_ZERO 850CK_PR_UNARY_Z_S(dec, 16, uint16_t, -, 1) 851#else 852CK_PR_UNARY_Z_STUB(dec, 16, uint16_t) 853#endif /* CK_F_PR_DEC_16_ZERO */ 854 855#endif /* CK_F_PR_LOAD_16 && CK_F_PR_CAS_16_VALUE */ 856 857#if defined(CK_F_PR_LOAD_8) && defined(CK_F_PR_CAS_8_VALUE) 858 859#ifndef CK_F_PR_INC_8 860#define CK_F_PR_INC_8 861CK_PR_UNARY_S(inc, add, 8, uint8_t) 862#endif /* CK_F_PR_INC_8 */ 863 864#ifndef CK_F_PR_INC_8_ZERO 865#define CK_F_PR_INC_8_ZERO 866CK_PR_UNARY_Z_S(inc, 8, uint8_t, +, UINT8_MAX) 867#else 868CK_PR_UNARY_Z_STUB(inc, 8, uint8_t) 869#endif /* CK_F_PR_INC_8_ZERO */ 870 871#ifndef CK_F_PR_DEC_8 872#define CK_F_PR_DEC_8 873CK_PR_UNARY_S(dec, sub, 8, uint8_t) 874#endif /* CK_F_PR_DEC_8 */ 875 876#ifndef CK_F_PR_DEC_8_ZERO 877#define CK_F_PR_DEC_8_ZERO 878CK_PR_UNARY_Z_S(dec, 8, uint8_t, -, 1) 879#else 880CK_PR_UNARY_Z_STUB(dec, 8, uint8_t) 881#endif /* CK_F_PR_DEC_8_ZERO */ 882 883#endif /* CK_F_PR_LOAD_8 && CK_F_PR_CAS_8_VALUE */ 884 885#undef CK_PR_UNARY_Z_S 886#undef CK_PR_UNARY_S 887#undef CK_PR_UNARY_Z 888#undef CK_PR_UNARY 889 890#define CK_PR_N(K, S, M, T, P, C) \ 891 CK_CC_INLINE static void \ 892 ck_pr_##K##_##S(M *target) \ 893 { \ 894 T previous; \ 895 C punt; \ 896 punt = (C)ck_pr_md_load_##S(target); \ 897 previous = (T)punt; \ 898 while (ck_pr_cas_##S##_value(target, \ 899 (C)previous, \ 900 (C)(P previous), \ 901 &previous) == false) \ 902 ck_pr_stall(); \ 903 \ 904 return; \ 905 } 906 907#define CK_PR_N_Z(S, M, T, C) \ 908 CK_CC_INLINE static void \ 909 ck_pr_neg_##S##_zero(M *target, bool *zero) \ 910 { \ 911 T previous; \ 912 C punt; \ 913 punt = (C)ck_pr_md_load_##S(target); \ 914 previous = (T)punt; \ 915 while (ck_pr_cas_##S##_value(target, \ 916 (C)previous, \ 917 (C)(-previous), \ 918 &previous) == false) \ 919 ck_pr_stall(); \ 920 \ 921 *zero = previous == 0; \ 922 return; \ 923 } 924 925#define CK_PR_N_S(K, S, M, P) CK_PR_N(K, S, M, M, P, M) 926#define CK_PR_N_Z_S(S, M) CK_PR_N_Z(S, M, M, M) 927 928#if defined(CK_F_PR_LOAD_CHAR) && defined(CK_F_PR_CAS_CHAR_VALUE) 929 930#ifndef CK_F_PR_NOT_CHAR 931#define CK_F_PR_NOT_CHAR 932CK_PR_N_S(not, char, char, ~) 933#endif /* CK_F_PR_NOT_CHAR */ 934 935#ifndef CK_F_PR_NEG_CHAR 936#define CK_F_PR_NEG_CHAR 937CK_PR_N_S(neg, char, char, -) 938#endif /* CK_F_PR_NEG_CHAR */ 939 940#ifndef CK_F_PR_NEG_CHAR_ZERO 941#define CK_F_PR_NEG_CHAR_ZERO 942CK_PR_N_Z_S(char, char) 943#endif /* CK_F_PR_NEG_CHAR_ZERO */ 944 945#endif /* CK_F_PR_LOAD_CHAR && CK_F_PR_CAS_CHAR_VALUE */ 946 947#if defined(CK_F_PR_LOAD_INT) && defined(CK_F_PR_CAS_INT_VALUE) 948 949#ifndef CK_F_PR_NOT_INT 950#define CK_F_PR_NOT_INT 951CK_PR_N_S(not, int, int, ~) 952#endif /* CK_F_PR_NOT_INT */ 953 954#ifndef CK_F_PR_NEG_INT 955#define CK_F_PR_NEG_INT 956CK_PR_N_S(neg, int, int, -) 957#endif /* CK_F_PR_NEG_INT */ 958 959#ifndef CK_F_PR_NEG_INT_ZERO 960#define CK_F_PR_NEG_INT_ZERO 961CK_PR_N_Z_S(int, int) 962#endif /* CK_F_PR_NEG_INT_ZERO */ 963 964#endif /* CK_F_PR_LOAD_INT && CK_F_PR_CAS_INT_VALUE */ 965 966#if defined(CK_F_PR_LOAD_DOUBLE) && defined(CK_F_PR_CAS_DOUBLE_VALUE) && \ 967 !defined(CK_PR_DISABLE_DOUBLE) 968 969#ifndef CK_F_PR_NEG_DOUBLE 970#define CK_F_PR_NEG_DOUBLE 971CK_PR_N_S(neg, double, double, -) 972#endif /* CK_F_PR_NEG_DOUBLE */ 973 974#endif /* CK_F_PR_LOAD_DOUBLE && CK_F_PR_CAS_DOUBLE_VALUE && !CK_PR_DISABLE_DOUBLE */ 975 976#if defined(CK_F_PR_LOAD_UINT) && defined(CK_F_PR_CAS_UINT_VALUE) 977 978#ifndef CK_F_PR_NOT_UINT 979#define CK_F_PR_NOT_UINT 980CK_PR_N_S(not, uint, unsigned int, ~) 981#endif /* CK_F_PR_NOT_UINT */ 982 983#ifndef CK_F_PR_NEG_UINT 984#define CK_F_PR_NEG_UINT 985CK_PR_N_S(neg, uint, unsigned int, -) 986#endif /* CK_F_PR_NEG_UINT */ 987 988#ifndef CK_F_PR_NEG_UINT_ZERO 989#define CK_F_PR_NEG_UINT_ZERO 990CK_PR_N_Z_S(uint, unsigned int) 991#endif /* CK_F_PR_NEG_UINT_ZERO */ 992 993#endif /* CK_F_PR_LOAD_UINT && CK_F_PR_CAS_UINT_VALUE */ 994 995#if defined(CK_F_PR_LOAD_PTR) && defined(CK_F_PR_CAS_PTR_VALUE) 996 997#ifndef CK_F_PR_NOT_PTR 998#define CK_F_PR_NOT_PTR 999CK_PR_N(not, ptr, void, uintptr_t, ~, void *) 1000#endif /* CK_F_PR_NOT_PTR */ 1001 1002#ifndef CK_F_PR_NEG_PTR 1003#define CK_F_PR_NEG_PTR 1004CK_PR_N(neg, ptr, void, uintptr_t, -, void *) 1005#endif /* CK_F_PR_NEG_PTR */ 1006 1007#ifndef CK_F_PR_NEG_PTR_ZERO 1008#define CK_F_PR_NEG_PTR_ZERO 1009CK_PR_N_Z(ptr, void, uintptr_t, void *) 1010#endif /* CK_F_PR_NEG_PTR_ZERO */ 1011 1012#endif /* CK_F_PR_LOAD_PTR && CK_F_PR_CAS_PTR_VALUE */ 1013 1014#if defined(CK_F_PR_LOAD_64) && defined(CK_F_PR_CAS_64_VALUE) 1015 1016#ifndef CK_F_PR_NOT_64 1017#define CK_F_PR_NOT_64 1018CK_PR_N_S(not, 64, uint64_t, ~) 1019#endif /* CK_F_PR_NOT_64 */ 1020 1021#ifndef CK_F_PR_NEG_64 1022#define CK_F_PR_NEG_64 1023CK_PR_N_S(neg, 64, uint64_t, -) 1024#endif /* CK_F_PR_NEG_64 */ 1025 1026#ifndef CK_F_PR_NEG_64_ZERO 1027#define CK_F_PR_NEG_64_ZERO 1028CK_PR_N_Z_S(64, uint64_t) 1029#endif /* CK_F_PR_NEG_64_ZERO */ 1030 1031#endif /* CK_F_PR_LOAD_64 && CK_F_PR_CAS_64_VALUE */ 1032 1033#if defined(CK_F_PR_LOAD_32) && defined(CK_F_PR_CAS_32_VALUE) 1034 1035#ifndef CK_F_PR_NOT_32 1036#define CK_F_PR_NOT_32 1037CK_PR_N_S(not, 32, uint32_t, ~) 1038#endif /* CK_F_PR_NOT_32 */ 1039 1040#ifndef CK_F_PR_NEG_32 1041#define CK_F_PR_NEG_32 1042CK_PR_N_S(neg, 32, uint32_t, -) 1043#endif /* CK_F_PR_NEG_32 */ 1044 1045#ifndef CK_F_PR_NEG_32_ZERO 1046#define CK_F_PR_NEG_32_ZERO 1047CK_PR_N_Z_S(32, uint32_t) 1048#endif /* CK_F_PR_NEG_32_ZERO */ 1049 1050#endif /* CK_F_PR_LOAD_32 && CK_F_PR_CAS_32_VALUE */ 1051 1052#if defined(CK_F_PR_LOAD_16) && defined(CK_F_PR_CAS_16_VALUE) 1053 1054#ifndef CK_F_PR_NOT_16 1055#define CK_F_PR_NOT_16 1056CK_PR_N_S(not, 16, uint16_t, ~) 1057#endif /* CK_F_PR_NOT_16 */ 1058 1059#ifndef CK_F_PR_NEG_16 1060#define CK_F_PR_NEG_16 1061CK_PR_N_S(neg, 16, uint16_t, -) 1062#endif /* CK_F_PR_NEG_16 */ 1063 1064#ifndef CK_F_PR_NEG_16_ZERO 1065#define CK_F_PR_NEG_16_ZERO 1066CK_PR_N_Z_S(16, uint16_t) 1067#endif /* CK_F_PR_NEG_16_ZERO */ 1068 1069#endif /* CK_F_PR_LOAD_16 && CK_F_PR_CAS_16_VALUE */ 1070 1071#if defined(CK_F_PR_LOAD_8) && defined(CK_F_PR_CAS_8_VALUE) 1072 1073#ifndef CK_F_PR_NOT_8 1074#define CK_F_PR_NOT_8 1075CK_PR_N_S(not, 8, uint8_t, ~) 1076#endif /* CK_F_PR_NOT_8 */ 1077 1078#ifndef CK_F_PR_NEG_8 1079#define CK_F_PR_NEG_8 1080CK_PR_N_S(neg, 8, uint8_t, -) 1081#endif /* CK_F_PR_NEG_8 */ 1082 1083#ifndef CK_F_PR_NEG_8_ZERO 1084#define CK_F_PR_NEG_8_ZERO 1085CK_PR_N_Z_S(8, uint8_t) 1086#endif /* CK_F_PR_NEG_8_ZERO */ 1087 1088#endif /* CK_F_PR_LOAD_8 && CK_F_PR_CAS_8_VALUE */ 1089 1090#undef CK_PR_N_Z_S 1091#undef CK_PR_N_S 1092#undef CK_PR_N_Z 1093#undef CK_PR_N 1094 1095#define CK_PR_FAA(S, M, T, C) \ 1096 CK_CC_INLINE static C \ 1097 ck_pr_faa_##S(M *target, T delta) \ 1098 { \ 1099 T previous; \ 1100 C punt; \ 1101 punt = (C)ck_pr_md_load_##S(target); \ 1102 previous = (T)punt; \ 1103 while (ck_pr_cas_##S##_value(target, \ 1104 (C)previous, \ 1105 (C)(previous + delta), \ 1106 &previous) == false) \ 1107 ck_pr_stall(); \ 1108 \ 1109 return ((C)previous); \ 1110 } 1111 1112#define CK_PR_FAS(S, M, C) \ 1113 CK_CC_INLINE static C \ 1114 ck_pr_fas_##S(M *target, C update) \ 1115 { \ 1116 C previous; \ 1117 previous = ck_pr_md_load_##S(target); \ 1118 while (ck_pr_cas_##S##_value(target, \ 1119 previous, \ 1120 update, \ 1121 &previous) == false) \ 1122 ck_pr_stall(); \ 1123 \ 1124 return (previous); \ 1125 } 1126 1127#define CK_PR_FAA_S(S, M) CK_PR_FAA(S, M, M, M) 1128#define CK_PR_FAS_S(S, M) CK_PR_FAS(S, M, M) 1129 1130#if defined(CK_F_PR_LOAD_CHAR) && defined(CK_F_PR_CAS_CHAR_VALUE) 1131 1132#ifndef CK_F_PR_FAA_CHAR 1133#define CK_F_PR_FAA_CHAR 1134CK_PR_FAA_S(char, char) 1135#endif /* CK_F_PR_FAA_CHAR */ 1136 1137#ifndef CK_F_PR_FAS_CHAR 1138#define CK_F_PR_FAS_CHAR 1139CK_PR_FAS_S(char, char) 1140#endif /* CK_F_PR_FAS_CHAR */ 1141 1142#endif /* CK_F_PR_LOAD_CHAR && CK_F_PR_CAS_CHAR_VALUE */ 1143 1144#if defined(CK_F_PR_LOAD_INT) && defined(CK_F_PR_CAS_INT_VALUE) 1145 1146#ifndef CK_F_PR_FAA_INT 1147#define CK_F_PR_FAA_INT 1148CK_PR_FAA_S(int, int) 1149#endif /* CK_F_PR_FAA_INT */ 1150 1151#ifndef CK_F_PR_FAS_INT 1152#define CK_F_PR_FAS_INT 1153CK_PR_FAS_S(int, int) 1154#endif /* CK_F_PR_FAS_INT */ 1155 1156#endif /* CK_F_PR_LOAD_INT && CK_F_PR_CAS_INT_VALUE */ 1157 1158#if defined(CK_F_PR_LOAD_DOUBLE) && defined(CK_F_PR_CAS_DOUBLE_VALUE) && \ 1159 !defined(CK_PR_DISABLE_DOUBLE) 1160 1161#ifndef CK_F_PR_FAA_DOUBLE 1162#define CK_F_PR_FAA_DOUBLE 1163CK_PR_FAA_S(double, double) 1164#endif /* CK_F_PR_FAA_DOUBLE */ 1165 1166#ifndef CK_F_PR_FAS_DOUBLE 1167#define CK_F_PR_FAS_DOUBLE 1168CK_PR_FAS_S(double, double) 1169#endif /* CK_F_PR_FAS_DOUBLE */ 1170 1171#endif /* CK_F_PR_LOAD_DOUBLE && CK_F_PR_CAS_DOUBLE_VALUE && !CK_PR_DISABLE_DOUBLE */ 1172 1173#if defined(CK_F_PR_LOAD_UINT) && defined(CK_F_PR_CAS_UINT_VALUE) 1174 1175#ifndef CK_F_PR_FAA_UINT 1176#define CK_F_PR_FAA_UINT 1177CK_PR_FAA_S(uint, unsigned int) 1178#endif /* CK_F_PR_FAA_UINT */ 1179 1180#ifndef CK_F_PR_FAS_UINT 1181#define CK_F_PR_FAS_UINT 1182CK_PR_FAS_S(uint, unsigned int) 1183#endif /* CK_F_PR_FAS_UINT */ 1184 1185#endif /* CK_F_PR_LOAD_UINT && CK_F_PR_CAS_UINT_VALUE */ 1186 1187#if defined(CK_F_PR_LOAD_PTR) && defined(CK_F_PR_CAS_PTR_VALUE) 1188 1189#ifndef CK_F_PR_FAA_PTR 1190#define CK_F_PR_FAA_PTR 1191CK_PR_FAA(ptr, void, uintptr_t, void *) 1192#endif /* CK_F_PR_FAA_PTR */ 1193 1194#ifndef CK_F_PR_FAS_PTR 1195#define CK_F_PR_FAS_PTR 1196CK_PR_FAS(ptr, void, void *) 1197#endif /* CK_F_PR_FAS_PTR */ 1198 1199#endif /* CK_F_PR_LOAD_PTR && CK_F_PR_CAS_PTR_VALUE */ 1200 1201#if defined(CK_F_PR_LOAD_64) && defined(CK_F_PR_CAS_64_VALUE) 1202 1203#ifndef CK_F_PR_FAA_64 1204#define CK_F_PR_FAA_64 1205CK_PR_FAA_S(64, uint64_t) 1206#endif /* CK_F_PR_FAA_64 */ 1207 1208#ifndef CK_F_PR_FAS_64 1209#define CK_F_PR_FAS_64 1210CK_PR_FAS_S(64, uint64_t) 1211#endif /* CK_F_PR_FAS_64 */ 1212 1213#endif /* CK_F_PR_LOAD_64 && CK_F_PR_CAS_64_VALUE */ 1214 1215#if defined(CK_F_PR_LOAD_32) && defined(CK_F_PR_CAS_32_VALUE) 1216 1217#ifndef CK_F_PR_FAA_32 1218#define CK_F_PR_FAA_32 1219CK_PR_FAA_S(32, uint32_t) 1220#endif /* CK_F_PR_FAA_32 */ 1221 1222#ifndef CK_F_PR_FAS_32 1223#define CK_F_PR_FAS_32 1224CK_PR_FAS_S(32, uint32_t) 1225#endif /* CK_F_PR_FAS_32 */ 1226 1227#endif /* CK_F_PR_LOAD_32 && CK_F_PR_CAS_32_VALUE */ 1228 1229#if defined(CK_F_PR_LOAD_16) && defined(CK_F_PR_CAS_16_VALUE) 1230 1231#ifndef CK_F_PR_FAA_16 1232#define CK_F_PR_FAA_16 1233CK_PR_FAA_S(16, uint16_t) 1234#endif /* CK_F_PR_FAA_16 */ 1235 1236#ifndef CK_F_PR_FAS_16 1237#define CK_F_PR_FAS_16 1238CK_PR_FAS_S(16, uint16_t) 1239#endif /* CK_F_PR_FAS_16 */ 1240 1241#endif /* CK_F_PR_LOAD_16 && CK_F_PR_CAS_16_VALUE */ 1242 1243#if defined(CK_F_PR_LOAD_8) && defined(CK_F_PR_CAS_8_VALUE) 1244 1245#ifndef CK_F_PR_FAA_8 1246#define CK_F_PR_FAA_8 1247CK_PR_FAA_S(8, uint8_t) 1248#endif /* CK_F_PR_FAA_8 */ 1249 1250#ifndef CK_F_PR_FAS_8 1251#define CK_F_PR_FAS_8 1252CK_PR_FAS_S(8, uint8_t) 1253#endif /* CK_F_PR_FAS_8 */ 1254 1255#endif /* CK_F_PR_LOAD_8 && CK_F_PR_CAS_8_VALUE */ 1256 1257#undef CK_PR_FAA_S 1258#undef CK_PR_FAS_S 1259#undef CK_PR_FAA 1260#undef CK_PR_FAS 1261 1262#endif /* CK_PR_H */ 1263