1345153Sdim/* 2345153Sdim * kmp_os.h -- KPTS runtime header file. 3345153Sdim */ 4345153Sdim 5345153Sdim//===----------------------------------------------------------------------===// 6345153Sdim// 7353358Sdim// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 8353358Sdim// See https://llvm.org/LICENSE.txt for license information. 9353358Sdim// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 10345153Sdim// 11345153Sdim//===----------------------------------------------------------------------===// 12345153Sdim 13345153Sdim#ifndef KMP_OS_H 14345153Sdim#define KMP_OS_H 15345153Sdim 16345153Sdim#include "kmp_config.h" 17365430Sdim#include <atomic> 18365430Sdim#include <stdarg.h> 19345153Sdim#include <stdlib.h> 20345153Sdim 21345153Sdim#define KMP_FTN_PLAIN 1 22345153Sdim#define KMP_FTN_APPEND 2 23345153Sdim#define KMP_FTN_UPPER 3 24345153Sdim/* 25345153Sdim#define KMP_FTN_PREPEND 4 26345153Sdim#define KMP_FTN_UAPPEND 5 27345153Sdim*/ 28345153Sdim 29345153Sdim#define KMP_PTR_SKIP (sizeof(void *)) 30345153Sdim 31345153Sdim/* -------------------------- Compiler variations ------------------------ */ 32345153Sdim 33345153Sdim#define KMP_OFF 0 34345153Sdim#define KMP_ON 1 35345153Sdim 36345153Sdim#define KMP_MEM_CONS_VOLATILE 0 37345153Sdim#define KMP_MEM_CONS_FENCE 1 38345153Sdim 39345153Sdim#ifndef KMP_MEM_CONS_MODEL 40345153Sdim#define KMP_MEM_CONS_MODEL KMP_MEM_CONS_VOLATILE 41345153Sdim#endif 42345153Sdim 43353358Sdim#ifndef __has_cpp_attribute 44353358Sdim#define __has_cpp_attribute(x) 0 45353358Sdim#endif 46353358Sdim 47353358Sdim#ifndef __has_attribute 48353358Sdim#define __has_attribute(x) 0 49353358Sdim#endif 50353358Sdim 51345153Sdim/* ------------------------- Compiler recognition ---------------------- */ 52345153Sdim#define KMP_COMPILER_ICC 0 53345153Sdim#define KMP_COMPILER_GCC 0 54345153Sdim#define KMP_COMPILER_CLANG 0 55345153Sdim#define KMP_COMPILER_MSVC 0 56345153Sdim 57345153Sdim#if defined(__INTEL_COMPILER) 58345153Sdim#undef KMP_COMPILER_ICC 59345153Sdim#define KMP_COMPILER_ICC 1 60345153Sdim#elif defined(__clang__) 61345153Sdim#undef KMP_COMPILER_CLANG 62345153Sdim#define KMP_COMPILER_CLANG 1 63345153Sdim#elif defined(__GNUC__) 64345153Sdim#undef KMP_COMPILER_GCC 65345153Sdim#define KMP_COMPILER_GCC 1 66345153Sdim#elif defined(_MSC_VER) 67345153Sdim#undef KMP_COMPILER_MSVC 68345153Sdim#define KMP_COMPILER_MSVC 1 69345153Sdim#else 70345153Sdim#error Unknown compiler 71345153Sdim#endif 72345153Sdim 73360784Sdim#if (KMP_OS_LINUX || KMP_OS_WINDOWS || KMP_OS_FREEBSD) && !KMP_OS_CNK 74345153Sdim#define KMP_AFFINITY_SUPPORTED 1 75345153Sdim#if KMP_OS_WINDOWS && KMP_ARCH_X86_64 76345153Sdim#define KMP_GROUP_AFFINITY 1 77345153Sdim#else 78345153Sdim#define KMP_GROUP_AFFINITY 0 79345153Sdim#endif 80345153Sdim#else 81345153Sdim#define KMP_AFFINITY_SUPPORTED 0 82345153Sdim#define KMP_GROUP_AFFINITY 0 83345153Sdim#endif 84345153Sdim 85345153Sdim/* Check for quad-precision extension. */ 86345153Sdim#define KMP_HAVE_QUAD 0 87345153Sdim#if KMP_ARCH_X86 || KMP_ARCH_X86_64 88345153Sdim#if KMP_COMPILER_ICC 89345153Sdim/* _Quad is already defined for icc */ 90345153Sdim#undef KMP_HAVE_QUAD 91345153Sdim#define KMP_HAVE_QUAD 1 92345153Sdim#elif KMP_COMPILER_CLANG 93345153Sdim/* Clang doesn't support a software-implemented 94345153Sdim 128-bit extended precision type yet */ 95345153Sdimtypedef long double _Quad; 96345153Sdim#elif KMP_COMPILER_GCC 97345153Sdim/* GCC on NetBSD lacks __multc3/__divtc3 builtins needed for quad */ 98345153Sdim#if !KMP_OS_NETBSD 99345153Sdimtypedef __float128 _Quad; 100345153Sdim#undef KMP_HAVE_QUAD 101345153Sdim#define KMP_HAVE_QUAD 1 102345153Sdim#endif 103345153Sdim#elif KMP_COMPILER_MSVC 104345153Sdimtypedef long double _Quad; 105345153Sdim#endif 106345153Sdim#else 107345153Sdim#if __LDBL_MAX_EXP__ >= 16384 && KMP_COMPILER_GCC 108345153Sdimtypedef long double _Quad; 109345153Sdim#undef KMP_HAVE_QUAD 110345153Sdim#define KMP_HAVE_QUAD 1 111345153Sdim#endif 112345153Sdim#endif /* KMP_ARCH_X86 || KMP_ARCH_X86_64 */ 113345153Sdim 114345153Sdim#define KMP_USE_X87CONTROL 0 115345153Sdim#if KMP_OS_WINDOWS 116345153Sdim#define KMP_END_OF_LINE "\r\n" 117345153Sdimtypedef char kmp_int8; 118345153Sdimtypedef unsigned char kmp_uint8; 119345153Sdimtypedef short kmp_int16; 120345153Sdimtypedef unsigned short kmp_uint16; 121345153Sdimtypedef int kmp_int32; 122345153Sdimtypedef unsigned int kmp_uint32; 123345153Sdim#define KMP_INT32_SPEC "d" 124345153Sdim#define KMP_UINT32_SPEC "u" 125345153Sdim#ifndef KMP_STRUCT64 126345153Sdimtypedef __int64 kmp_int64; 127345153Sdimtypedef unsigned __int64 kmp_uint64; 128345153Sdim#define KMP_INT64_SPEC "I64d" 129345153Sdim#define KMP_UINT64_SPEC "I64u" 130345153Sdim#else 131345153Sdimstruct kmp_struct64 { 132345153Sdim kmp_int32 a, b; 133345153Sdim}; 134345153Sdimtypedef struct kmp_struct64 kmp_int64; 135345153Sdimtypedef struct kmp_struct64 kmp_uint64; 136345153Sdim/* Not sure what to use for KMP_[U]INT64_SPEC here */ 137345153Sdim#endif 138345153Sdim#if KMP_ARCH_X86 && KMP_MSVC_COMPAT 139345153Sdim#undef KMP_USE_X87CONTROL 140345153Sdim#define KMP_USE_X87CONTROL 1 141345153Sdim#endif 142345153Sdim#if KMP_ARCH_X86_64 143345153Sdim#define KMP_INTPTR 1 144345153Sdimtypedef __int64 kmp_intptr_t; 145345153Sdimtypedef unsigned __int64 kmp_uintptr_t; 146345153Sdim#define KMP_INTPTR_SPEC "I64d" 147345153Sdim#define KMP_UINTPTR_SPEC "I64u" 148345153Sdim#endif 149345153Sdim#endif /* KMP_OS_WINDOWS */ 150345153Sdim 151345153Sdim#if KMP_OS_UNIX 152345153Sdim#define KMP_END_OF_LINE "\n" 153345153Sdimtypedef char kmp_int8; 154345153Sdimtypedef unsigned char kmp_uint8; 155345153Sdimtypedef short kmp_int16; 156345153Sdimtypedef unsigned short kmp_uint16; 157345153Sdimtypedef int kmp_int32; 158345153Sdimtypedef unsigned int kmp_uint32; 159345153Sdimtypedef long long kmp_int64; 160345153Sdimtypedef unsigned long long kmp_uint64; 161345153Sdim#define KMP_INT32_SPEC "d" 162345153Sdim#define KMP_UINT32_SPEC "u" 163345153Sdim#define KMP_INT64_SPEC "lld" 164345153Sdim#define KMP_UINT64_SPEC "llu" 165345153Sdim#endif /* KMP_OS_UNIX */ 166345153Sdim 167345153Sdim#if KMP_ARCH_X86 || KMP_ARCH_ARM || KMP_ARCH_MIPS 168345153Sdim#define KMP_SIZE_T_SPEC KMP_UINT32_SPEC 169360784Sdim#elif KMP_ARCH_X86_64 || KMP_ARCH_PPC64 || KMP_ARCH_AARCH64 || \ 170360784Sdim KMP_ARCH_MIPS64 || KMP_ARCH_RISCV64 171345153Sdim#define KMP_SIZE_T_SPEC KMP_UINT64_SPEC 172345153Sdim#else 173345153Sdim#error "Can't determine size_t printf format specifier." 174345153Sdim#endif 175345153Sdim 176345153Sdim#if KMP_ARCH_X86 177345153Sdim#define KMP_SIZE_T_MAX (0xFFFFFFFF) 178345153Sdim#else 179345153Sdim#define KMP_SIZE_T_MAX (0xFFFFFFFFFFFFFFFF) 180345153Sdim#endif 181345153Sdim 182345153Sdimtypedef size_t kmp_size_t; 183345153Sdimtypedef float kmp_real32; 184345153Sdimtypedef double kmp_real64; 185345153Sdim 186345153Sdim#ifndef KMP_INTPTR 187345153Sdim#define KMP_INTPTR 1 188345153Sdimtypedef long kmp_intptr_t; 189345153Sdimtypedef unsigned long kmp_uintptr_t; 190345153Sdim#define KMP_INTPTR_SPEC "ld" 191345153Sdim#define KMP_UINTPTR_SPEC "lu" 192345153Sdim#endif 193345153Sdim 194345153Sdim#ifdef BUILD_I8 195345153Sdimtypedef kmp_int64 kmp_int; 196345153Sdimtypedef kmp_uint64 kmp_uint; 197345153Sdim#else 198345153Sdimtypedef kmp_int32 kmp_int; 199345153Sdimtypedef kmp_uint32 kmp_uint; 200345153Sdim#endif /* BUILD_I8 */ 201345153Sdim#define KMP_INT_MAX ((kmp_int32)0x7FFFFFFF) 202345153Sdim#define KMP_INT_MIN ((kmp_int32)0x80000000) 203345153Sdim 204365427Sdim// stdarg handling 205365427Sdim#if (KMP_ARCH_ARM || KMP_ARCH_X86_64 || KMP_ARCH_AARCH64) && \ 206365427Sdim (KMP_OS_FREEBSD || KMP_OS_LINUX) 207365427Sdimtypedef va_list *kmp_va_list; 208365427Sdim#define kmp_va_deref(ap) (*(ap)) 209365427Sdim#define kmp_va_addr_of(ap) (&(ap)) 210365427Sdim#else 211365427Sdimtypedef va_list kmp_va_list; 212365427Sdim#define kmp_va_deref(ap) (ap) 213365427Sdim#define kmp_va_addr_of(ap) (ap) 214365427Sdim#endif 215365427Sdim 216345153Sdim#ifdef __cplusplus 217345153Sdim// macros to cast out qualifiers and to re-interpret types 218345153Sdim#define CCAST(type, var) const_cast<type>(var) 219345153Sdim#define RCAST(type, var) reinterpret_cast<type>(var) 220345153Sdim//------------------------------------------------------------------------- 221345153Sdim// template for debug prints specification ( d, u, lld, llu ), and to obtain 222345153Sdim// signed/unsigned flavors of a type 223345153Sdimtemplate <typename T> struct traits_t {}; 224345153Sdim// int 225345153Sdimtemplate <> struct traits_t<signed int> { 226345153Sdim typedef signed int signed_t; 227345153Sdim typedef unsigned int unsigned_t; 228345153Sdim typedef double floating_t; 229345153Sdim static char const *spec; 230345153Sdim static const signed_t max_value = 0x7fffffff; 231345153Sdim static const signed_t min_value = 0x80000000; 232345153Sdim static const int type_size = sizeof(signed_t); 233345153Sdim}; 234345153Sdim// unsigned int 235345153Sdimtemplate <> struct traits_t<unsigned int> { 236345153Sdim typedef signed int signed_t; 237345153Sdim typedef unsigned int unsigned_t; 238345153Sdim typedef double floating_t; 239345153Sdim static char const *spec; 240345153Sdim static const unsigned_t max_value = 0xffffffff; 241345153Sdim static const unsigned_t min_value = 0x00000000; 242345153Sdim static const int type_size = sizeof(unsigned_t); 243345153Sdim}; 244345153Sdim// long 245345153Sdimtemplate <> struct traits_t<signed long> { 246345153Sdim typedef signed long signed_t; 247345153Sdim typedef unsigned long unsigned_t; 248345153Sdim typedef long double floating_t; 249345153Sdim static char const *spec; 250345153Sdim static const int type_size = sizeof(signed_t); 251345153Sdim}; 252345153Sdim// long long 253345153Sdimtemplate <> struct traits_t<signed long long> { 254345153Sdim typedef signed long long signed_t; 255345153Sdim typedef unsigned long long unsigned_t; 256345153Sdim typedef long double floating_t; 257345153Sdim static char const *spec; 258345153Sdim static const signed_t max_value = 0x7fffffffffffffffLL; 259345153Sdim static const signed_t min_value = 0x8000000000000000LL; 260345153Sdim static const int type_size = sizeof(signed_t); 261345153Sdim}; 262345153Sdim// unsigned long long 263345153Sdimtemplate <> struct traits_t<unsigned long long> { 264345153Sdim typedef signed long long signed_t; 265345153Sdim typedef unsigned long long unsigned_t; 266345153Sdim typedef long double floating_t; 267345153Sdim static char const *spec; 268345153Sdim static const unsigned_t max_value = 0xffffffffffffffffLL; 269345153Sdim static const unsigned_t min_value = 0x0000000000000000LL; 270345153Sdim static const int type_size = sizeof(unsigned_t); 271345153Sdim}; 272345153Sdim//------------------------------------------------------------------------- 273345153Sdim#else 274345153Sdim#define CCAST(type, var) (type)(var) 275345153Sdim#define RCAST(type, var) (type)(var) 276345153Sdim#endif // __cplusplus 277345153Sdim 278345153Sdim#define KMP_EXPORT extern /* export declaration in guide libraries */ 279345153Sdim 280345153Sdim#if __GNUC__ >= 4 && !defined(__MINGW32__) 281345153Sdim#define __forceinline __inline 282345153Sdim#endif 283345153Sdim 284345153Sdim#if KMP_OS_WINDOWS 285345153Sdim#include <windows.h> 286345153Sdim 287345153Sdimstatic inline int KMP_GET_PAGE_SIZE(void) { 288345153Sdim SYSTEM_INFO si; 289345153Sdim GetSystemInfo(&si); 290345153Sdim return si.dwPageSize; 291345153Sdim} 292345153Sdim#else 293345153Sdim#define KMP_GET_PAGE_SIZE() getpagesize() 294345153Sdim#endif 295345153Sdim 296345153Sdim#define PAGE_ALIGNED(_addr) \ 297345153Sdim (!((size_t)_addr & (size_t)(KMP_GET_PAGE_SIZE() - 1))) 298345153Sdim#define ALIGN_TO_PAGE(x) \ 299345153Sdim (void *)(((size_t)(x)) & ~((size_t)(KMP_GET_PAGE_SIZE() - 1))) 300345153Sdim 301345153Sdim/* ---------- Support for cache alignment, padding, etc. ----------------*/ 302345153Sdim 303345153Sdim#ifdef __cplusplus 304345153Sdimextern "C" { 305345153Sdim#endif // __cplusplus 306345153Sdim 307345153Sdim#define INTERNODE_CACHE_LINE 4096 /* for multi-node systems */ 308345153Sdim 309345153Sdim/* Define the default size of the cache line */ 310345153Sdim#ifndef CACHE_LINE 311345153Sdim#define CACHE_LINE 128 /* cache line size in bytes */ 312345153Sdim#else 313345153Sdim#if (CACHE_LINE < 64) && !defined(KMP_OS_DARWIN) 314345153Sdim// 2006-02-13: This produces too many warnings on OS X*. Disable for now 315345153Sdim#warning CACHE_LINE is too small. 316345153Sdim#endif 317345153Sdim#endif /* CACHE_LINE */ 318345153Sdim 319345153Sdim#define KMP_CACHE_PREFETCH(ADDR) /* nothing */ 320345153Sdim 321353358Sdim// Define attribute that indicates that the fall through from the previous 322353358Sdim// case label is intentional and should not be diagnosed by a compiler 323353358Sdim// Code from libcxx/include/__config 324353358Sdim// Use a function like macro to imply that it must be followed by a semicolon 325353358Sdim#if __cplusplus > 201402L && __has_cpp_attribute(fallthrough) 326353358Sdim# define KMP_FALLTHROUGH() [[fallthrough]] 327353358Sdim#elif __has_cpp_attribute(clang::fallthrough) 328353358Sdim# define KMP_FALLTHROUGH() [[clang::fallthrough]] 329360784Sdim#elif __has_attribute(fallthrough) || __GNUC__ >= 7 330353358Sdim# define KMP_FALLTHROUGH() __attribute__((__fallthrough__)) 331353358Sdim#else 332353358Sdim# define KMP_FALLTHROUGH() ((void)0) 333353358Sdim#endif 334353358Sdim 335345153Sdim// Define attribute that indicates a function does not return 336345153Sdim#if __cplusplus >= 201103L 337345153Sdim#define KMP_NORETURN [[noreturn]] 338345153Sdim#elif KMP_OS_WINDOWS 339345153Sdim#define KMP_NORETURN __declspec(noreturn) 340345153Sdim#else 341345153Sdim#define KMP_NORETURN __attribute__((noreturn)) 342345153Sdim#endif 343345153Sdim 344345153Sdim#if KMP_OS_WINDOWS && KMP_MSVC_COMPAT 345345153Sdim#define KMP_ALIGN(bytes) __declspec(align(bytes)) 346345153Sdim#define KMP_THREAD_LOCAL __declspec(thread) 347345153Sdim#define KMP_ALIAS /* Nothing */ 348345153Sdim#else 349345153Sdim#define KMP_ALIGN(bytes) __attribute__((aligned(bytes))) 350345153Sdim#define KMP_THREAD_LOCAL __thread 351345153Sdim#define KMP_ALIAS(alias_of) __attribute__((alias(alias_of))) 352345153Sdim#endif 353345153Sdim 354345153Sdim#if KMP_HAVE_WEAK_ATTRIBUTE 355345153Sdim#define KMP_WEAK_ATTRIBUTE __attribute__((weak)) 356345153Sdim#else 357345153Sdim#define KMP_WEAK_ATTRIBUTE /* Nothing */ 358345153Sdim#endif 359345153Sdim 360345153Sdim// Define KMP_VERSION_SYMBOL and KMP_EXPAND_NAME 361345153Sdim#ifndef KMP_STR 362345153Sdim#define KMP_STR(x) _KMP_STR(x) 363345153Sdim#define _KMP_STR(x) #x 364345153Sdim#endif 365345153Sdim 366345153Sdim#ifdef KMP_USE_VERSION_SYMBOLS 367345153Sdim// If using versioned symbols, KMP_EXPAND_NAME prepends 368345153Sdim// __kmp_api_ to the real API name 369345153Sdim#define KMP_EXPAND_NAME(api_name) _KMP_EXPAND_NAME(api_name) 370345153Sdim#define _KMP_EXPAND_NAME(api_name) __kmp_api_##api_name 371345153Sdim#define KMP_VERSION_SYMBOL(api_name, ver_num, ver_str) \ 372345153Sdim _KMP_VERSION_SYMBOL(api_name, ver_num, ver_str, "VERSION") 373345153Sdim#define _KMP_VERSION_SYMBOL(api_name, ver_num, ver_str, default_ver) \ 374345153Sdim __typeof__(__kmp_api_##api_name) __kmp_api_##api_name##_##ver_num##_alias \ 375345153Sdim __attribute__((alias(KMP_STR(__kmp_api_##api_name)))); \ 376345153Sdim __asm__( \ 377345153Sdim ".symver " KMP_STR(__kmp_api_##api_name##_##ver_num##_alias) "," KMP_STR( \ 378345153Sdim api_name) "@" ver_str "\n\t"); \ 379345153Sdim __asm__(".symver " KMP_STR(__kmp_api_##api_name) "," KMP_STR( \ 380345153Sdim api_name) "@@" default_ver "\n\t") 381345153Sdim#else // KMP_USE_VERSION_SYMBOLS 382345153Sdim#define KMP_EXPAND_NAME(api_name) api_name 383345153Sdim#define KMP_VERSION_SYMBOL(api_name, ver_num, ver_str) /* Nothing */ 384345153Sdim#endif // KMP_USE_VERSION_SYMBOLS 385345153Sdim 386345153Sdim/* Temporary note: if performance testing of this passes, we can remove 387345153Sdim all references to KMP_DO_ALIGN and replace with KMP_ALIGN. */ 388345153Sdim#define KMP_DO_ALIGN(bytes) KMP_ALIGN(bytes) 389345153Sdim#define KMP_ALIGN_CACHE KMP_ALIGN(CACHE_LINE) 390345153Sdim#define KMP_ALIGN_CACHE_INTERNODE KMP_ALIGN(INTERNODE_CACHE_LINE) 391345153Sdim 392345153Sdim/* General purpose fence types for memory operations */ 393345153Sdimenum kmp_mem_fence_type { 394345153Sdim kmp_no_fence, /* No memory fence */ 395345153Sdim kmp_acquire_fence, /* Acquire (read) memory fence */ 396345153Sdim kmp_release_fence, /* Release (write) memory fence */ 397345153Sdim kmp_full_fence /* Full (read+write) memory fence */ 398345153Sdim}; 399345153Sdim 400345153Sdim// Synchronization primitives 401345153Sdim 402345153Sdim#if KMP_ASM_INTRINS && KMP_OS_WINDOWS 403345153Sdim 404345153Sdim#if KMP_MSVC_COMPAT && !KMP_COMPILER_CLANG 405345153Sdim#pragma intrinsic(InterlockedExchangeAdd) 406345153Sdim#pragma intrinsic(InterlockedCompareExchange) 407345153Sdim#pragma intrinsic(InterlockedExchange) 408345153Sdim#pragma intrinsic(InterlockedExchange64) 409345153Sdim#endif 410345153Sdim 411345153Sdim// Using InterlockedIncrement / InterlockedDecrement causes a library loading 412345153Sdim// ordering problem, so we use InterlockedExchangeAdd instead. 413345153Sdim#define KMP_TEST_THEN_INC32(p) InterlockedExchangeAdd((volatile long *)(p), 1) 414345153Sdim#define KMP_TEST_THEN_INC_ACQ32(p) \ 415345153Sdim InterlockedExchangeAdd((volatile long *)(p), 1) 416345153Sdim#define KMP_TEST_THEN_ADD4_32(p) InterlockedExchangeAdd((volatile long *)(p), 4) 417345153Sdim#define KMP_TEST_THEN_ADD4_ACQ32(p) \ 418345153Sdim InterlockedExchangeAdd((volatile long *)(p), 4) 419345153Sdim#define KMP_TEST_THEN_DEC32(p) InterlockedExchangeAdd((volatile long *)(p), -1) 420345153Sdim#define KMP_TEST_THEN_DEC_ACQ32(p) \ 421345153Sdim InterlockedExchangeAdd((volatile long *)(p), -1) 422345153Sdim#define KMP_TEST_THEN_ADD32(p, v) \ 423345153Sdim InterlockedExchangeAdd((volatile long *)(p), (v)) 424345153Sdim 425345153Sdim#define KMP_COMPARE_AND_STORE_RET32(p, cv, sv) \ 426345153Sdim InterlockedCompareExchange((volatile long *)(p), (long)(sv), (long)(cv)) 427345153Sdim 428345153Sdim#define KMP_XCHG_FIXED32(p, v) \ 429345153Sdim InterlockedExchange((volatile long *)(p), (long)(v)) 430345153Sdim#define KMP_XCHG_FIXED64(p, v) \ 431345153Sdim InterlockedExchange64((volatile kmp_int64 *)(p), (kmp_int64)(v)) 432345153Sdim 433345153Sdiminline kmp_real32 KMP_XCHG_REAL32(volatile kmp_real32 *p, kmp_real32 v) { 434345153Sdim kmp_int32 tmp = InterlockedExchange((volatile long *)p, *(long *)&v); 435345153Sdim return *(kmp_real32 *)&tmp; 436345153Sdim} 437345153Sdim 438345153Sdim// Routines that we still need to implement in assembly. 439345153Sdimextern kmp_int8 __kmp_test_then_add8(volatile kmp_int8 *p, kmp_int8 v); 440345153Sdimextern kmp_int8 __kmp_test_then_or8(volatile kmp_int8 *p, kmp_int8 v); 441345153Sdimextern kmp_int8 __kmp_test_then_and8(volatile kmp_int8 *p, kmp_int8 v); 442345153Sdimextern kmp_int32 __kmp_test_then_add32(volatile kmp_int32 *p, kmp_int32 v); 443345153Sdimextern kmp_uint32 __kmp_test_then_or32(volatile kmp_uint32 *p, kmp_uint32 v); 444345153Sdimextern kmp_uint32 __kmp_test_then_and32(volatile kmp_uint32 *p, kmp_uint32 v); 445345153Sdimextern kmp_int64 __kmp_test_then_add64(volatile kmp_int64 *p, kmp_int64 v); 446345153Sdimextern kmp_uint64 __kmp_test_then_or64(volatile kmp_uint64 *p, kmp_uint64 v); 447345153Sdimextern kmp_uint64 __kmp_test_then_and64(volatile kmp_uint64 *p, kmp_uint64 v); 448345153Sdim 449345153Sdimextern kmp_int8 __kmp_compare_and_store8(volatile kmp_int8 *p, kmp_int8 cv, 450345153Sdim kmp_int8 sv); 451345153Sdimextern kmp_int16 __kmp_compare_and_store16(volatile kmp_int16 *p, kmp_int16 cv, 452345153Sdim kmp_int16 sv); 453345153Sdimextern kmp_int32 __kmp_compare_and_store32(volatile kmp_int32 *p, kmp_int32 cv, 454345153Sdim kmp_int32 sv); 455345153Sdimextern kmp_int32 __kmp_compare_and_store64(volatile kmp_int64 *p, kmp_int64 cv, 456345153Sdim kmp_int64 sv); 457345153Sdimextern kmp_int8 __kmp_compare_and_store_ret8(volatile kmp_int8 *p, kmp_int8 cv, 458345153Sdim kmp_int8 sv); 459345153Sdimextern kmp_int16 __kmp_compare_and_store_ret16(volatile kmp_int16 *p, 460345153Sdim kmp_int16 cv, kmp_int16 sv); 461345153Sdimextern kmp_int32 __kmp_compare_and_store_ret32(volatile kmp_int32 *p, 462345153Sdim kmp_int32 cv, kmp_int32 sv); 463345153Sdimextern kmp_int64 __kmp_compare_and_store_ret64(volatile kmp_int64 *p, 464345153Sdim kmp_int64 cv, kmp_int64 sv); 465345153Sdim 466345153Sdimextern kmp_int8 __kmp_xchg_fixed8(volatile kmp_int8 *p, kmp_int8 v); 467345153Sdimextern kmp_int16 __kmp_xchg_fixed16(volatile kmp_int16 *p, kmp_int16 v); 468345153Sdimextern kmp_int32 __kmp_xchg_fixed32(volatile kmp_int32 *p, kmp_int32 v); 469345153Sdimextern kmp_int64 __kmp_xchg_fixed64(volatile kmp_int64 *p, kmp_int64 v); 470345153Sdimextern kmp_real32 __kmp_xchg_real32(volatile kmp_real32 *p, kmp_real32 v); 471345153Sdimextern kmp_real64 __kmp_xchg_real64(volatile kmp_real64 *p, kmp_real64 v); 472345153Sdim 473345153Sdim//#define KMP_TEST_THEN_INC32(p) __kmp_test_then_add32((p), 1) 474345153Sdim//#define KMP_TEST_THEN_INC_ACQ32(p) __kmp_test_then_add32((p), 1) 475345153Sdim#define KMP_TEST_THEN_INC64(p) __kmp_test_then_add64((p), 1LL) 476345153Sdim#define KMP_TEST_THEN_INC_ACQ64(p) __kmp_test_then_add64((p), 1LL) 477345153Sdim//#define KMP_TEST_THEN_ADD4_32(p) __kmp_test_then_add32((p), 4) 478345153Sdim//#define KMP_TEST_THEN_ADD4_ACQ32(p) __kmp_test_then_add32((p), 4) 479345153Sdim#define KMP_TEST_THEN_ADD4_64(p) __kmp_test_then_add64((p), 4LL) 480345153Sdim#define KMP_TEST_THEN_ADD4_ACQ64(p) __kmp_test_then_add64((p), 4LL) 481345153Sdim//#define KMP_TEST_THEN_DEC32(p) __kmp_test_then_add32((p), -1) 482345153Sdim//#define KMP_TEST_THEN_DEC_ACQ32(p) __kmp_test_then_add32((p), -1) 483345153Sdim#define KMP_TEST_THEN_DEC64(p) __kmp_test_then_add64((p), -1LL) 484345153Sdim#define KMP_TEST_THEN_DEC_ACQ64(p) __kmp_test_then_add64((p), -1LL) 485345153Sdim//#define KMP_TEST_THEN_ADD32(p, v) __kmp_test_then_add32((p), (v)) 486345153Sdim#define KMP_TEST_THEN_ADD8(p, v) __kmp_test_then_add8((p), (v)) 487345153Sdim#define KMP_TEST_THEN_ADD64(p, v) __kmp_test_then_add64((p), (v)) 488345153Sdim 489345153Sdim#define KMP_TEST_THEN_OR8(p, v) __kmp_test_then_or8((p), (v)) 490345153Sdim#define KMP_TEST_THEN_AND8(p, v) __kmp_test_then_and8((p), (v)) 491345153Sdim#define KMP_TEST_THEN_OR32(p, v) __kmp_test_then_or32((p), (v)) 492345153Sdim#define KMP_TEST_THEN_AND32(p, v) __kmp_test_then_and32((p), (v)) 493345153Sdim#define KMP_TEST_THEN_OR64(p, v) __kmp_test_then_or64((p), (v)) 494345153Sdim#define KMP_TEST_THEN_AND64(p, v) __kmp_test_then_and64((p), (v)) 495345153Sdim 496345153Sdim#define KMP_COMPARE_AND_STORE_ACQ8(p, cv, sv) \ 497345153Sdim __kmp_compare_and_store8((p), (cv), (sv)) 498345153Sdim#define KMP_COMPARE_AND_STORE_REL8(p, cv, sv) \ 499345153Sdim __kmp_compare_and_store8((p), (cv), (sv)) 500345153Sdim#define KMP_COMPARE_AND_STORE_ACQ16(p, cv, sv) \ 501345153Sdim __kmp_compare_and_store16((p), (cv), (sv)) 502345153Sdim#define KMP_COMPARE_AND_STORE_REL16(p, cv, sv) \ 503345153Sdim __kmp_compare_and_store16((p), (cv), (sv)) 504345153Sdim#define KMP_COMPARE_AND_STORE_ACQ32(p, cv, sv) \ 505345153Sdim __kmp_compare_and_store32((volatile kmp_int32 *)(p), (kmp_int32)(cv), \ 506345153Sdim (kmp_int32)(sv)) 507345153Sdim#define KMP_COMPARE_AND_STORE_REL32(p, cv, sv) \ 508345153Sdim __kmp_compare_and_store32((volatile kmp_int32 *)(p), (kmp_int32)(cv), \ 509345153Sdim (kmp_int32)(sv)) 510345153Sdim#define KMP_COMPARE_AND_STORE_ACQ64(p, cv, sv) \ 511345153Sdim __kmp_compare_and_store64((volatile kmp_int64 *)(p), (kmp_int64)(cv), \ 512345153Sdim (kmp_int64)(sv)) 513345153Sdim#define KMP_COMPARE_AND_STORE_REL64(p, cv, sv) \ 514345153Sdim __kmp_compare_and_store64((volatile kmp_int64 *)(p), (kmp_int64)(cv), \ 515345153Sdim (kmp_int64)(sv)) 516345153Sdim 517345153Sdim#if KMP_ARCH_X86 518345153Sdim#define KMP_COMPARE_AND_STORE_PTR(p, cv, sv) \ 519345153Sdim __kmp_compare_and_store32((volatile kmp_int32 *)(p), (kmp_int32)(cv), \ 520345153Sdim (kmp_int32)(sv)) 521345153Sdim#else /* 64 bit pointers */ 522345153Sdim#define KMP_COMPARE_AND_STORE_PTR(p, cv, sv) \ 523345153Sdim __kmp_compare_and_store64((volatile kmp_int64 *)(p), (kmp_int64)(cv), \ 524345153Sdim (kmp_int64)(sv)) 525345153Sdim#endif /* KMP_ARCH_X86 */ 526345153Sdim 527345153Sdim#define KMP_COMPARE_AND_STORE_RET8(p, cv, sv) \ 528345153Sdim __kmp_compare_and_store_ret8((p), (cv), (sv)) 529345153Sdim#define KMP_COMPARE_AND_STORE_RET16(p, cv, sv) \ 530345153Sdim __kmp_compare_and_store_ret16((p), (cv), (sv)) 531345153Sdim#define KMP_COMPARE_AND_STORE_RET64(p, cv, sv) \ 532345153Sdim __kmp_compare_and_store_ret64((volatile kmp_int64 *)(p), (kmp_int64)(cv), \ 533345153Sdim (kmp_int64)(sv)) 534345153Sdim 535345153Sdim#define KMP_XCHG_FIXED8(p, v) \ 536345153Sdim __kmp_xchg_fixed8((volatile kmp_int8 *)(p), (kmp_int8)(v)); 537345153Sdim#define KMP_XCHG_FIXED16(p, v) __kmp_xchg_fixed16((p), (v)); 538345153Sdim//#define KMP_XCHG_FIXED32(p, v) __kmp_xchg_fixed32((p), (v)); 539345153Sdim//#define KMP_XCHG_FIXED64(p, v) __kmp_xchg_fixed64((p), (v)); 540345153Sdim//#define KMP_XCHG_REAL32(p, v) __kmp_xchg_real32((p), (v)); 541345153Sdim#define KMP_XCHG_REAL64(p, v) __kmp_xchg_real64((p), (v)); 542345153Sdim 543345153Sdim#elif (KMP_ASM_INTRINS && KMP_OS_UNIX) || !(KMP_ARCH_X86 || KMP_ARCH_X86_64) 544345153Sdim 545345153Sdim/* cast p to correct type so that proper intrinsic will be used */ 546345153Sdim#define KMP_TEST_THEN_INC32(p) \ 547345153Sdim __sync_fetch_and_add((volatile kmp_int32 *)(p), 1) 548345153Sdim#define KMP_TEST_THEN_INC_ACQ32(p) \ 549345153Sdim __sync_fetch_and_add((volatile kmp_int32 *)(p), 1) 550353358Sdim#if KMP_ARCH_MIPS 551345153Sdim#define KMP_TEST_THEN_INC64(p) \ 552353358Sdim __atomic_fetch_add((volatile kmp_int64 *)(p), 1LL, __ATOMIC_SEQ_CST) 553353358Sdim#define KMP_TEST_THEN_INC_ACQ64(p) \ 554353358Sdim __atomic_fetch_add((volatile kmp_int64 *)(p), 1LL, __ATOMIC_SEQ_CST) 555353358Sdim#else 556353358Sdim#define KMP_TEST_THEN_INC64(p) \ 557345153Sdim __sync_fetch_and_add((volatile kmp_int64 *)(p), 1LL) 558345153Sdim#define KMP_TEST_THEN_INC_ACQ64(p) \ 559345153Sdim __sync_fetch_and_add((volatile kmp_int64 *)(p), 1LL) 560353358Sdim#endif 561345153Sdim#define KMP_TEST_THEN_ADD4_32(p) \ 562345153Sdim __sync_fetch_and_add((volatile kmp_int32 *)(p), 4) 563345153Sdim#define KMP_TEST_THEN_ADD4_ACQ32(p) \ 564345153Sdim __sync_fetch_and_add((volatile kmp_int32 *)(p), 4) 565353358Sdim#if KMP_ARCH_MIPS 566345153Sdim#define KMP_TEST_THEN_ADD4_64(p) \ 567353358Sdim __atomic_fetch_add((volatile kmp_int64 *)(p), 4LL, __ATOMIC_SEQ_CST) 568353358Sdim#define KMP_TEST_THEN_ADD4_ACQ64(p) \ 569353358Sdim __atomic_fetch_add((volatile kmp_int64 *)(p), 4LL, __ATOMIC_SEQ_CST) 570353358Sdim#define KMP_TEST_THEN_DEC64(p) \ 571353358Sdim __atomic_fetch_sub((volatile kmp_int64 *)(p), 1LL, __ATOMIC_SEQ_CST) 572353358Sdim#define KMP_TEST_THEN_DEC_ACQ64(p) \ 573353358Sdim __atomic_fetch_sub((volatile kmp_int64 *)(p), 1LL, __ATOMIC_SEQ_CST) 574353358Sdim#else 575353358Sdim#define KMP_TEST_THEN_ADD4_64(p) \ 576345153Sdim __sync_fetch_and_add((volatile kmp_int64 *)(p), 4LL) 577345153Sdim#define KMP_TEST_THEN_ADD4_ACQ64(p) \ 578345153Sdim __sync_fetch_and_add((volatile kmp_int64 *)(p), 4LL) 579353358Sdim#define KMP_TEST_THEN_DEC64(p) \ 580353358Sdim __sync_fetch_and_sub((volatile kmp_int64 *)(p), 1LL) 581353358Sdim#define KMP_TEST_THEN_DEC_ACQ64(p) \ 582353358Sdim __sync_fetch_and_sub((volatile kmp_int64 *)(p), 1LL) 583353358Sdim#endif 584345153Sdim#define KMP_TEST_THEN_DEC32(p) \ 585345153Sdim __sync_fetch_and_sub((volatile kmp_int32 *)(p), 1) 586345153Sdim#define KMP_TEST_THEN_DEC_ACQ32(p) \ 587345153Sdim __sync_fetch_and_sub((volatile kmp_int32 *)(p), 1) 588345153Sdim#define KMP_TEST_THEN_ADD8(p, v) \ 589345153Sdim __sync_fetch_and_add((volatile kmp_int8 *)(p), (kmp_int8)(v)) 590345153Sdim#define KMP_TEST_THEN_ADD32(p, v) \ 591345153Sdim __sync_fetch_and_add((volatile kmp_int32 *)(p), (kmp_int32)(v)) 592353358Sdim#if KMP_ARCH_MIPS 593345153Sdim#define KMP_TEST_THEN_ADD64(p, v) \ 594353358Sdim __atomic_fetch_add((volatile kmp_uint64 *)(p), (kmp_uint64)(v), \ 595353358Sdim __ATOMIC_SEQ_CST) 596353358Sdim#else 597353358Sdim#define KMP_TEST_THEN_ADD64(p, v) \ 598345153Sdim __sync_fetch_and_add((volatile kmp_int64 *)(p), (kmp_int64)(v)) 599353358Sdim#endif 600345153Sdim 601345153Sdim#define KMP_TEST_THEN_OR8(p, v) \ 602345153Sdim __sync_fetch_and_or((volatile kmp_int8 *)(p), (kmp_int8)(v)) 603345153Sdim#define KMP_TEST_THEN_AND8(p, v) \ 604345153Sdim __sync_fetch_and_and((volatile kmp_int8 *)(p), (kmp_int8)(v)) 605345153Sdim#define KMP_TEST_THEN_OR32(p, v) \ 606345153Sdim __sync_fetch_and_or((volatile kmp_uint32 *)(p), (kmp_uint32)(v)) 607345153Sdim#define KMP_TEST_THEN_AND32(p, v) \ 608345153Sdim __sync_fetch_and_and((volatile kmp_uint32 *)(p), (kmp_uint32)(v)) 609353358Sdim#if KMP_ARCH_MIPS 610345153Sdim#define KMP_TEST_THEN_OR64(p, v) \ 611353358Sdim __atomic_fetch_or((volatile kmp_uint64 *)(p), (kmp_uint64)(v), \ 612353358Sdim __ATOMIC_SEQ_CST) 613353358Sdim#define KMP_TEST_THEN_AND64(p, v) \ 614353358Sdim __atomic_fetch_and((volatile kmp_uint64 *)(p), (kmp_uint64)(v), \ 615353358Sdim __ATOMIC_SEQ_CST) 616353358Sdim#else 617353358Sdim#define KMP_TEST_THEN_OR64(p, v) \ 618345153Sdim __sync_fetch_and_or((volatile kmp_uint64 *)(p), (kmp_uint64)(v)) 619345153Sdim#define KMP_TEST_THEN_AND64(p, v) \ 620345153Sdim __sync_fetch_and_and((volatile kmp_uint64 *)(p), (kmp_uint64)(v)) 621353358Sdim#endif 622345153Sdim 623345153Sdim#define KMP_COMPARE_AND_STORE_ACQ8(p, cv, sv) \ 624345153Sdim __sync_bool_compare_and_swap((volatile kmp_uint8 *)(p), (kmp_uint8)(cv), \ 625345153Sdim (kmp_uint8)(sv)) 626345153Sdim#define KMP_COMPARE_AND_STORE_REL8(p, cv, sv) \ 627345153Sdim __sync_bool_compare_and_swap((volatile kmp_uint8 *)(p), (kmp_uint8)(cv), \ 628345153Sdim (kmp_uint8)(sv)) 629345153Sdim#define KMP_COMPARE_AND_STORE_ACQ16(p, cv, sv) \ 630345153Sdim __sync_bool_compare_and_swap((volatile kmp_uint16 *)(p), (kmp_uint16)(cv), \ 631345153Sdim (kmp_uint16)(sv)) 632345153Sdim#define KMP_COMPARE_AND_STORE_REL16(p, cv, sv) \ 633345153Sdim __sync_bool_compare_and_swap((volatile kmp_uint16 *)(p), (kmp_uint16)(cv), \ 634345153Sdim (kmp_uint16)(sv)) 635345153Sdim#define KMP_COMPARE_AND_STORE_ACQ32(p, cv, sv) \ 636345153Sdim __sync_bool_compare_and_swap((volatile kmp_uint32 *)(p), (kmp_uint32)(cv), \ 637345153Sdim (kmp_uint32)(sv)) 638345153Sdim#define KMP_COMPARE_AND_STORE_REL32(p, cv, sv) \ 639345153Sdim __sync_bool_compare_and_swap((volatile kmp_uint32 *)(p), (kmp_uint32)(cv), \ 640345153Sdim (kmp_uint32)(sv)) 641345153Sdim#define KMP_COMPARE_AND_STORE_PTR(p, cv, sv) \ 642345153Sdim __sync_bool_compare_and_swap((void *volatile *)(p), (void *)(cv), \ 643345153Sdim (void *)(sv)) 644345153Sdim 645345153Sdim#define KMP_COMPARE_AND_STORE_RET8(p, cv, sv) \ 646345153Sdim __sync_val_compare_and_swap((volatile kmp_uint8 *)(p), (kmp_uint8)(cv), \ 647345153Sdim (kmp_uint8)(sv)) 648345153Sdim#define KMP_COMPARE_AND_STORE_RET16(p, cv, sv) \ 649345153Sdim __sync_val_compare_and_swap((volatile kmp_uint16 *)(p), (kmp_uint16)(cv), \ 650345153Sdim (kmp_uint16)(sv)) 651345153Sdim#define KMP_COMPARE_AND_STORE_RET32(p, cv, sv) \ 652345153Sdim __sync_val_compare_and_swap((volatile kmp_uint32 *)(p), (kmp_uint32)(cv), \ 653345153Sdim (kmp_uint32)(sv)) 654353358Sdim#if KMP_ARCH_MIPS 655353358Sdimstatic inline bool mips_sync_bool_compare_and_swap( 656353358Sdim volatile kmp_uint64 *p, kmp_uint64 cv, kmp_uint64 sv) { 657353358Sdim return __atomic_compare_exchange(p, &cv, &sv, false, __ATOMIC_SEQ_CST, 658353358Sdim __ATOMIC_SEQ_CST); 659353358Sdim} 660353358Sdimstatic inline bool mips_sync_val_compare_and_swap( 661353358Sdim volatile kmp_uint64 *p, kmp_uint64 cv, kmp_uint64 sv) { 662353358Sdim __atomic_compare_exchange(p, &cv, &sv, false, __ATOMIC_SEQ_CST, 663353358Sdim __ATOMIC_SEQ_CST); 664353358Sdim return cv; 665353358Sdim} 666353358Sdim#define KMP_COMPARE_AND_STORE_ACQ64(p, cv, sv) \ 667353358Sdim mips_sync_bool_compare_and_swap((volatile kmp_uint64 *)(p), (kmp_uint64)(cv),\ 668353358Sdim (kmp_uint64)(sv)) 669353358Sdim#define KMP_COMPARE_AND_STORE_REL64(p, cv, sv) \ 670353358Sdim mips_sync_bool_compare_and_swap((volatile kmp_uint64 *)(p), (kmp_uint64)(cv),\ 671353358Sdim (kmp_uint64)(sv)) 672345153Sdim#define KMP_COMPARE_AND_STORE_RET64(p, cv, sv) \ 673353358Sdim mips_sync_val_compare_and_swap((volatile kmp_uint64 *)(p), (kmp_uint64)(cv), \ 674353358Sdim (kmp_uint64)(sv)) 675353358Sdim#else 676353358Sdim#define KMP_COMPARE_AND_STORE_ACQ64(p, cv, sv) \ 677353358Sdim __sync_bool_compare_and_swap((volatile kmp_uint64 *)(p), (kmp_uint64)(cv), \ 678353358Sdim (kmp_uint64)(sv)) 679353358Sdim#define KMP_COMPARE_AND_STORE_REL64(p, cv, sv) \ 680353358Sdim __sync_bool_compare_and_swap((volatile kmp_uint64 *)(p), (kmp_uint64)(cv), \ 681353358Sdim (kmp_uint64)(sv)) 682353358Sdim#define KMP_COMPARE_AND_STORE_RET64(p, cv, sv) \ 683345153Sdim __sync_val_compare_and_swap((volatile kmp_uint64 *)(p), (kmp_uint64)(cv), \ 684345153Sdim (kmp_uint64)(sv)) 685353358Sdim#endif 686345153Sdim 687345153Sdim#define KMP_XCHG_FIXED8(p, v) \ 688345153Sdim __sync_lock_test_and_set((volatile kmp_uint8 *)(p), (kmp_uint8)(v)) 689345153Sdim#define KMP_XCHG_FIXED16(p, v) \ 690345153Sdim __sync_lock_test_and_set((volatile kmp_uint16 *)(p), (kmp_uint16)(v)) 691345153Sdim#define KMP_XCHG_FIXED32(p, v) \ 692345153Sdim __sync_lock_test_and_set((volatile kmp_uint32 *)(p), (kmp_uint32)(v)) 693345153Sdim#define KMP_XCHG_FIXED64(p, v) \ 694345153Sdim __sync_lock_test_and_set((volatile kmp_uint64 *)(p), (kmp_uint64)(v)) 695345153Sdim 696345153Sdiminline kmp_real32 KMP_XCHG_REAL32(volatile kmp_real32 *p, kmp_real32 v) { 697345153Sdim kmp_int32 tmp = 698345153Sdim __sync_lock_test_and_set((volatile kmp_uint32 *)(p), *(kmp_uint32 *)&v); 699345153Sdim return *(kmp_real32 *)&tmp; 700345153Sdim} 701345153Sdim 702345153Sdiminline kmp_real64 KMP_XCHG_REAL64(volatile kmp_real64 *p, kmp_real64 v) { 703345153Sdim kmp_int64 tmp = 704345153Sdim __sync_lock_test_and_set((volatile kmp_uint64 *)(p), *(kmp_uint64 *)&v); 705345153Sdim return *(kmp_real64 *)&tmp; 706345153Sdim} 707345153Sdim 708345153Sdim#else 709345153Sdim 710345153Sdimextern kmp_int8 __kmp_test_then_add8(volatile kmp_int8 *p, kmp_int8 v); 711345153Sdimextern kmp_int8 __kmp_test_then_or8(volatile kmp_int8 *p, kmp_int8 v); 712345153Sdimextern kmp_int8 __kmp_test_then_and8(volatile kmp_int8 *p, kmp_int8 v); 713345153Sdimextern kmp_int32 __kmp_test_then_add32(volatile kmp_int32 *p, kmp_int32 v); 714345153Sdimextern kmp_uint32 __kmp_test_then_or32(volatile kmp_uint32 *p, kmp_uint32 v); 715345153Sdimextern kmp_uint32 __kmp_test_then_and32(volatile kmp_uint32 *p, kmp_uint32 v); 716345153Sdimextern kmp_int64 __kmp_test_then_add64(volatile kmp_int64 *p, kmp_int64 v); 717345153Sdimextern kmp_uint64 __kmp_test_then_or64(volatile kmp_uint64 *p, kmp_uint64 v); 718345153Sdimextern kmp_uint64 __kmp_test_then_and64(volatile kmp_uint64 *p, kmp_uint64 v); 719345153Sdim 720345153Sdimextern kmp_int8 __kmp_compare_and_store8(volatile kmp_int8 *p, kmp_int8 cv, 721345153Sdim kmp_int8 sv); 722345153Sdimextern kmp_int16 __kmp_compare_and_store16(volatile kmp_int16 *p, kmp_int16 cv, 723345153Sdim kmp_int16 sv); 724345153Sdimextern kmp_int32 __kmp_compare_and_store32(volatile kmp_int32 *p, kmp_int32 cv, 725345153Sdim kmp_int32 sv); 726345153Sdimextern kmp_int32 __kmp_compare_and_store64(volatile kmp_int64 *p, kmp_int64 cv, 727345153Sdim kmp_int64 sv); 728345153Sdimextern kmp_int8 __kmp_compare_and_store_ret8(volatile kmp_int8 *p, kmp_int8 cv, 729345153Sdim kmp_int8 sv); 730345153Sdimextern kmp_int16 __kmp_compare_and_store_ret16(volatile kmp_int16 *p, 731345153Sdim kmp_int16 cv, kmp_int16 sv); 732345153Sdimextern kmp_int32 __kmp_compare_and_store_ret32(volatile kmp_int32 *p, 733345153Sdim kmp_int32 cv, kmp_int32 sv); 734345153Sdimextern kmp_int64 __kmp_compare_and_store_ret64(volatile kmp_int64 *p, 735345153Sdim kmp_int64 cv, kmp_int64 sv); 736345153Sdim 737345153Sdimextern kmp_int8 __kmp_xchg_fixed8(volatile kmp_int8 *p, kmp_int8 v); 738345153Sdimextern kmp_int16 __kmp_xchg_fixed16(volatile kmp_int16 *p, kmp_int16 v); 739345153Sdimextern kmp_int32 __kmp_xchg_fixed32(volatile kmp_int32 *p, kmp_int32 v); 740345153Sdimextern kmp_int64 __kmp_xchg_fixed64(volatile kmp_int64 *p, kmp_int64 v); 741345153Sdimextern kmp_real32 __kmp_xchg_real32(volatile kmp_real32 *p, kmp_real32 v); 742345153Sdimextern kmp_real64 __kmp_xchg_real64(volatile kmp_real64 *p, kmp_real64 v); 743345153Sdim 744345153Sdim#define KMP_TEST_THEN_INC32(p) \ 745345153Sdim __kmp_test_then_add32((volatile kmp_int32 *)(p), 1) 746345153Sdim#define KMP_TEST_THEN_INC_ACQ32(p) \ 747345153Sdim __kmp_test_then_add32((volatile kmp_int32 *)(p), 1) 748345153Sdim#define KMP_TEST_THEN_INC64(p) \ 749345153Sdim __kmp_test_then_add64((volatile kmp_int64 *)(p), 1LL) 750345153Sdim#define KMP_TEST_THEN_INC_ACQ64(p) \ 751345153Sdim __kmp_test_then_add64((volatile kmp_int64 *)(p), 1LL) 752345153Sdim#define KMP_TEST_THEN_ADD4_32(p) \ 753345153Sdim __kmp_test_then_add32((volatile kmp_int32 *)(p), 4) 754345153Sdim#define KMP_TEST_THEN_ADD4_ACQ32(p) \ 755345153Sdim __kmp_test_then_add32((volatile kmp_int32 *)(p), 4) 756345153Sdim#define KMP_TEST_THEN_ADD4_64(p) \ 757345153Sdim __kmp_test_then_add64((volatile kmp_int64 *)(p), 4LL) 758345153Sdim#define KMP_TEST_THEN_ADD4_ACQ64(p) \ 759345153Sdim __kmp_test_then_add64((volatile kmp_int64 *)(p), 4LL) 760345153Sdim#define KMP_TEST_THEN_DEC32(p) \ 761345153Sdim __kmp_test_then_add32((volatile kmp_int32 *)(p), -1) 762345153Sdim#define KMP_TEST_THEN_DEC_ACQ32(p) \ 763345153Sdim __kmp_test_then_add32((volatile kmp_int32 *)(p), -1) 764345153Sdim#define KMP_TEST_THEN_DEC64(p) \ 765345153Sdim __kmp_test_then_add64((volatile kmp_int64 *)(p), -1LL) 766345153Sdim#define KMP_TEST_THEN_DEC_ACQ64(p) \ 767345153Sdim __kmp_test_then_add64((volatile kmp_int64 *)(p), -1LL) 768345153Sdim#define KMP_TEST_THEN_ADD8(p, v) \ 769345153Sdim __kmp_test_then_add8((volatile kmp_int8 *)(p), (kmp_int8)(v)) 770345153Sdim#define KMP_TEST_THEN_ADD32(p, v) \ 771345153Sdim __kmp_test_then_add32((volatile kmp_int32 *)(p), (kmp_int32)(v)) 772345153Sdim#define KMP_TEST_THEN_ADD64(p, v) \ 773345153Sdim __kmp_test_then_add64((volatile kmp_int64 *)(p), (kmp_int64)(v)) 774345153Sdim 775345153Sdim#define KMP_TEST_THEN_OR8(p, v) \ 776345153Sdim __kmp_test_then_or8((volatile kmp_int8 *)(p), (kmp_int8)(v)) 777345153Sdim#define KMP_TEST_THEN_AND8(p, v) \ 778345153Sdim __kmp_test_then_and8((volatile kmp_int8 *)(p), (kmp_int8)(v)) 779345153Sdim#define KMP_TEST_THEN_OR32(p, v) \ 780345153Sdim __kmp_test_then_or32((volatile kmp_uint32 *)(p), (kmp_uint32)(v)) 781345153Sdim#define KMP_TEST_THEN_AND32(p, v) \ 782345153Sdim __kmp_test_then_and32((volatile kmp_uint32 *)(p), (kmp_uint32)(v)) 783345153Sdim#define KMP_TEST_THEN_OR64(p, v) \ 784345153Sdim __kmp_test_then_or64((volatile kmp_uint64 *)(p), (kmp_uint64)(v)) 785345153Sdim#define KMP_TEST_THEN_AND64(p, v) \ 786345153Sdim __kmp_test_then_and64((volatile kmp_uint64 *)(p), (kmp_uint64)(v)) 787345153Sdim 788345153Sdim#define KMP_COMPARE_AND_STORE_ACQ8(p, cv, sv) \ 789345153Sdim __kmp_compare_and_store8((volatile kmp_int8 *)(p), (kmp_int8)(cv), \ 790345153Sdim (kmp_int8)(sv)) 791345153Sdim#define KMP_COMPARE_AND_STORE_REL8(p, cv, sv) \ 792345153Sdim __kmp_compare_and_store8((volatile kmp_int8 *)(p), (kmp_int8)(cv), \ 793345153Sdim (kmp_int8)(sv)) 794345153Sdim#define KMP_COMPARE_AND_STORE_ACQ16(p, cv, sv) \ 795345153Sdim __kmp_compare_and_store16((volatile kmp_int16 *)(p), (kmp_int16)(cv), \ 796345153Sdim (kmp_int16)(sv)) 797345153Sdim#define KMP_COMPARE_AND_STORE_REL16(p, cv, sv) \ 798345153Sdim __kmp_compare_and_store16((volatile kmp_int16 *)(p), (kmp_int16)(cv), \ 799345153Sdim (kmp_int16)(sv)) 800345153Sdim#define KMP_COMPARE_AND_STORE_ACQ32(p, cv, sv) \ 801345153Sdim __kmp_compare_and_store32((volatile kmp_int32 *)(p), (kmp_int32)(cv), \ 802345153Sdim (kmp_int32)(sv)) 803345153Sdim#define KMP_COMPARE_AND_STORE_REL32(p, cv, sv) \ 804345153Sdim __kmp_compare_and_store32((volatile kmp_int32 *)(p), (kmp_int32)(cv), \ 805345153Sdim (kmp_int32)(sv)) 806345153Sdim#define KMP_COMPARE_AND_STORE_ACQ64(p, cv, sv) \ 807345153Sdim __kmp_compare_and_store64((volatile kmp_int64 *)(p), (kmp_int64)(cv), \ 808345153Sdim (kmp_int64)(sv)) 809345153Sdim#define KMP_COMPARE_AND_STORE_REL64(p, cv, sv) \ 810345153Sdim __kmp_compare_and_store64((volatile kmp_int64 *)(p), (kmp_int64)(cv), \ 811345153Sdim (kmp_int64)(sv)) 812345153Sdim 813345153Sdim#if KMP_ARCH_X86 814345153Sdim#define KMP_COMPARE_AND_STORE_PTR(p, cv, sv) \ 815345153Sdim __kmp_compare_and_store32((volatile kmp_int32 *)(p), (kmp_int32)(cv), \ 816345153Sdim (kmp_int32)(sv)) 817345153Sdim#else /* 64 bit pointers */ 818345153Sdim#define KMP_COMPARE_AND_STORE_PTR(p, cv, sv) \ 819345153Sdim __kmp_compare_and_store64((volatile kmp_int64 *)(p), (kmp_int64)(cv), \ 820345153Sdim (kmp_int64)(sv)) 821345153Sdim#endif /* KMP_ARCH_X86 */ 822345153Sdim 823345153Sdim#define KMP_COMPARE_AND_STORE_RET8(p, cv, sv) \ 824345153Sdim __kmp_compare_and_store_ret8((p), (cv), (sv)) 825345153Sdim#define KMP_COMPARE_AND_STORE_RET16(p, cv, sv) \ 826345153Sdim __kmp_compare_and_store_ret16((p), (cv), (sv)) 827345153Sdim#define KMP_COMPARE_AND_STORE_RET32(p, cv, sv) \ 828345153Sdim __kmp_compare_and_store_ret32((volatile kmp_int32 *)(p), (kmp_int32)(cv), \ 829345153Sdim (kmp_int32)(sv)) 830345153Sdim#define KMP_COMPARE_AND_STORE_RET64(p, cv, sv) \ 831345153Sdim __kmp_compare_and_store_ret64((volatile kmp_int64 *)(p), (kmp_int64)(cv), \ 832345153Sdim (kmp_int64)(sv)) 833345153Sdim 834345153Sdim#define KMP_XCHG_FIXED8(p, v) \ 835345153Sdim __kmp_xchg_fixed8((volatile kmp_int8 *)(p), (kmp_int8)(v)); 836345153Sdim#define KMP_XCHG_FIXED16(p, v) __kmp_xchg_fixed16((p), (v)); 837345153Sdim#define KMP_XCHG_FIXED32(p, v) __kmp_xchg_fixed32((p), (v)); 838345153Sdim#define KMP_XCHG_FIXED64(p, v) __kmp_xchg_fixed64((p), (v)); 839345153Sdim#define KMP_XCHG_REAL32(p, v) __kmp_xchg_real32((p), (v)); 840345153Sdim#define KMP_XCHG_REAL64(p, v) __kmp_xchg_real64((p), (v)); 841345153Sdim 842345153Sdim#endif /* KMP_ASM_INTRINS */ 843345153Sdim 844345153Sdim/* ------------- relaxed consistency memory model stuff ------------------ */ 845345153Sdim 846345153Sdim#if KMP_OS_WINDOWS 847345153Sdim#ifdef __ABSOFT_WIN 848345153Sdim#define KMP_MB() asm("nop") 849345153Sdim#define KMP_IMB() asm("nop") 850345153Sdim#else 851345153Sdim#define KMP_MB() /* _asm{ nop } */ 852345153Sdim#define KMP_IMB() /* _asm{ nop } */ 853345153Sdim#endif 854345153Sdim#endif /* KMP_OS_WINDOWS */ 855345153Sdim 856345153Sdim#if KMP_ARCH_PPC64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64 || KMP_ARCH_MIPS || \ 857360784Sdim KMP_ARCH_MIPS64 || KMP_ARCH_RISCV64 858345153Sdim#define KMP_MB() __sync_synchronize() 859345153Sdim#endif 860345153Sdim 861345153Sdim#ifndef KMP_MB 862345153Sdim#define KMP_MB() /* nothing to do */ 863345153Sdim#endif 864345153Sdim 865345153Sdim#ifndef KMP_IMB 866345153Sdim#define KMP_IMB() /* nothing to do */ 867345153Sdim#endif 868345153Sdim 869345153Sdim#ifndef KMP_ST_REL32 870345153Sdim#define KMP_ST_REL32(A, D) (*(A) = (D)) 871345153Sdim#endif 872345153Sdim 873345153Sdim#ifndef KMP_ST_REL64 874345153Sdim#define KMP_ST_REL64(A, D) (*(A) = (D)) 875345153Sdim#endif 876345153Sdim 877345153Sdim#ifndef KMP_LD_ACQ32 878345153Sdim#define KMP_LD_ACQ32(A) (*(A)) 879345153Sdim#endif 880345153Sdim 881345153Sdim#ifndef KMP_LD_ACQ64 882345153Sdim#define KMP_LD_ACQ64(A) (*(A)) 883345153Sdim#endif 884345153Sdim 885345153Sdim/* ------------------------------------------------------------------------ */ 886345153Sdim// FIXME - maybe this should this be 887345153Sdim// 888345153Sdim// #define TCR_4(a) (*(volatile kmp_int32 *)(&a)) 889345153Sdim// #define TCW_4(a,b) (a) = (*(volatile kmp_int32 *)&(b)) 890345153Sdim// 891345153Sdim// #define TCR_8(a) (*(volatile kmp_int64 *)(a)) 892345153Sdim// #define TCW_8(a,b) (a) = (*(volatile kmp_int64 *)(&b)) 893345153Sdim// 894345153Sdim// I'm fairly certain this is the correct thing to do, but I'm afraid 895345153Sdim// of performance regressions. 896345153Sdim 897345153Sdim#define TCR_1(a) (a) 898345153Sdim#define TCW_1(a, b) (a) = (b) 899345153Sdim#define TCR_4(a) (a) 900345153Sdim#define TCW_4(a, b) (a) = (b) 901345153Sdim#define TCI_4(a) (++(a)) 902345153Sdim#define TCD_4(a) (--(a)) 903345153Sdim#define TCR_8(a) (a) 904345153Sdim#define TCW_8(a, b) (a) = (b) 905345153Sdim#define TCI_8(a) (++(a)) 906345153Sdim#define TCD_8(a) (--(a)) 907345153Sdim#define TCR_SYNC_4(a) (a) 908345153Sdim#define TCW_SYNC_4(a, b) (a) = (b) 909345153Sdim#define TCX_SYNC_4(a, b, c) \ 910345153Sdim KMP_COMPARE_AND_STORE_REL32((volatile kmp_int32 *)(volatile void *)&(a), \ 911345153Sdim (kmp_int32)(b), (kmp_int32)(c)) 912345153Sdim#define TCR_SYNC_8(a) (a) 913345153Sdim#define TCW_SYNC_8(a, b) (a) = (b) 914345153Sdim#define TCX_SYNC_8(a, b, c) \ 915345153Sdim KMP_COMPARE_AND_STORE_REL64((volatile kmp_int64 *)(volatile void *)&(a), \ 916345153Sdim (kmp_int64)(b), (kmp_int64)(c)) 917345153Sdim 918345153Sdim#if KMP_ARCH_X86 || KMP_ARCH_MIPS 919345153Sdim// What about ARM? 920345153Sdim#define TCR_PTR(a) ((void *)TCR_4(a)) 921345153Sdim#define TCW_PTR(a, b) TCW_4((a), (b)) 922345153Sdim#define TCR_SYNC_PTR(a) ((void *)TCR_SYNC_4(a)) 923345153Sdim#define TCW_SYNC_PTR(a, b) TCW_SYNC_4((a), (b)) 924345153Sdim#define TCX_SYNC_PTR(a, b, c) ((void *)TCX_SYNC_4((a), (b), (c))) 925345153Sdim 926345153Sdim#else /* 64 bit pointers */ 927345153Sdim 928345153Sdim#define TCR_PTR(a) ((void *)TCR_8(a)) 929345153Sdim#define TCW_PTR(a, b) TCW_8((a), (b)) 930345153Sdim#define TCR_SYNC_PTR(a) ((void *)TCR_SYNC_8(a)) 931345153Sdim#define TCW_SYNC_PTR(a, b) TCW_SYNC_8((a), (b)) 932345153Sdim#define TCX_SYNC_PTR(a, b, c) ((void *)TCX_SYNC_8((a), (b), (c))) 933345153Sdim 934345153Sdim#endif /* KMP_ARCH_X86 */ 935345153Sdim 936345153Sdim/* If these FTN_{TRUE,FALSE} values change, may need to change several places 937345153Sdim where they are used to check that language is Fortran, not C. */ 938345153Sdim 939345153Sdim#ifndef FTN_TRUE 940345153Sdim#define FTN_TRUE TRUE 941345153Sdim#endif 942345153Sdim 943345153Sdim#ifndef FTN_FALSE 944345153Sdim#define FTN_FALSE FALSE 945345153Sdim#endif 946345153Sdim 947345153Sdimtypedef void (*microtask_t)(int *gtid, int *npr, ...); 948345153Sdim 949345153Sdim#ifdef USE_VOLATILE_CAST 950345153Sdim#define VOLATILE_CAST(x) (volatile x) 951345153Sdim#else 952345153Sdim#define VOLATILE_CAST(x) (x) 953345153Sdim#endif 954345153Sdim 955353358Sdim#define KMP_WAIT __kmp_wait_4 956353358Sdim#define KMP_WAIT_PTR __kmp_wait_4_ptr 957345153Sdim#define KMP_EQ __kmp_eq_4 958345153Sdim#define KMP_NEQ __kmp_neq_4 959345153Sdim#define KMP_LT __kmp_lt_4 960345153Sdim#define KMP_GE __kmp_ge_4 961345153Sdim#define KMP_LE __kmp_le_4 962345153Sdim 963345153Sdim/* Workaround for Intel(R) 64 code gen bug when taking address of static array 964345153Sdim * (Intel(R) 64 Tracker #138) */ 965345153Sdim#if (KMP_ARCH_X86_64 || KMP_ARCH_PPC64) && KMP_OS_LINUX 966345153Sdim#define STATIC_EFI2_WORKAROUND 967345153Sdim#else 968345153Sdim#define STATIC_EFI2_WORKAROUND static 969345153Sdim#endif 970345153Sdim 971345153Sdim// Support of BGET usage 972345153Sdim#ifndef KMP_USE_BGET 973345153Sdim#define KMP_USE_BGET 1 974345153Sdim#endif 975345153Sdim 976345153Sdim// Switches for OSS builds 977345153Sdim#ifndef USE_CMPXCHG_FIX 978345153Sdim#define USE_CMPXCHG_FIX 1 979345153Sdim#endif 980345153Sdim 981345153Sdim// Enable dynamic user lock 982345153Sdim#define KMP_USE_DYNAMIC_LOCK 1 983345153Sdim 984345153Sdim// Enable Intel(R) Transactional Synchronization Extensions (Intel(R) TSX) if 985345153Sdim// dynamic user lock is turned on 986345153Sdim#if KMP_USE_DYNAMIC_LOCK 987345153Sdim// Visual studio can't handle the asm sections in this code 988345153Sdim#define KMP_USE_TSX (KMP_ARCH_X86 || KMP_ARCH_X86_64) && !KMP_COMPILER_MSVC 989345153Sdim#ifdef KMP_USE_ADAPTIVE_LOCKS 990345153Sdim#undef KMP_USE_ADAPTIVE_LOCKS 991345153Sdim#endif 992345153Sdim#define KMP_USE_ADAPTIVE_LOCKS KMP_USE_TSX 993345153Sdim#endif 994345153Sdim 995345153Sdim// Enable tick time conversion of ticks to seconds 996345153Sdim#if KMP_STATS_ENABLED 997345153Sdim#define KMP_HAVE_TICK_TIME \ 998345153Sdim (KMP_OS_LINUX && (KMP_MIC || KMP_ARCH_X86 || KMP_ARCH_X86_64)) 999345153Sdim#endif 1000345153Sdim 1001345153Sdim// Warning levels 1002345153Sdimenum kmp_warnings_level { 1003345153Sdim kmp_warnings_off = 0, /* No warnings */ 1004345153Sdim kmp_warnings_low, /* Minimal warnings (default) */ 1005345153Sdim kmp_warnings_explicit = 6, /* Explicitly set to ON - more warnings */ 1006345153Sdim kmp_warnings_verbose /* reserved */ 1007345153Sdim}; 1008345153Sdim 1009345153Sdim#ifdef __cplusplus 1010345153Sdim} // extern "C" 1011345153Sdim#endif // __cplusplus 1012345153Sdim 1013345153Sdim// Macros for C++11 atomic functions 1014345153Sdim#define KMP_ATOMIC_LD(p, order) (p)->load(std::memory_order_##order) 1015345153Sdim#define KMP_ATOMIC_OP(op, p, v, order) (p)->op(v, std::memory_order_##order) 1016345153Sdim 1017345153Sdim// For non-default load/store 1018345153Sdim#define KMP_ATOMIC_LD_ACQ(p) KMP_ATOMIC_LD(p, acquire) 1019345153Sdim#define KMP_ATOMIC_LD_RLX(p) KMP_ATOMIC_LD(p, relaxed) 1020345153Sdim#define KMP_ATOMIC_ST_REL(p, v) KMP_ATOMIC_OP(store, p, v, release) 1021345153Sdim#define KMP_ATOMIC_ST_RLX(p, v) KMP_ATOMIC_OP(store, p, v, relaxed) 1022345153Sdim 1023345153Sdim// For non-default fetch_<op> 1024345153Sdim#define KMP_ATOMIC_ADD(p, v) KMP_ATOMIC_OP(fetch_add, p, v, acq_rel) 1025345153Sdim#define KMP_ATOMIC_SUB(p, v) KMP_ATOMIC_OP(fetch_sub, p, v, acq_rel) 1026345153Sdim#define KMP_ATOMIC_AND(p, v) KMP_ATOMIC_OP(fetch_and, p, v, acq_rel) 1027345153Sdim#define KMP_ATOMIC_OR(p, v) KMP_ATOMIC_OP(fetch_or, p, v, acq_rel) 1028345153Sdim#define KMP_ATOMIC_INC(p) KMP_ATOMIC_OP(fetch_add, p, 1, acq_rel) 1029345153Sdim#define KMP_ATOMIC_DEC(p) KMP_ATOMIC_OP(fetch_sub, p, 1, acq_rel) 1030345153Sdim#define KMP_ATOMIC_ADD_RLX(p, v) KMP_ATOMIC_OP(fetch_add, p, v, relaxed) 1031345153Sdim#define KMP_ATOMIC_INC_RLX(p) KMP_ATOMIC_OP(fetch_add, p, 1, relaxed) 1032345153Sdim 1033345153Sdim// Callers of the following functions cannot see the side effect on "expected". 1034345153Sdimtemplate <typename T> 1035345153Sdimbool __kmp_atomic_compare_store(std::atomic<T> *p, T expected, T desired) { 1036345153Sdim return p->compare_exchange_strong( 1037345153Sdim expected, desired, std::memory_order_acq_rel, std::memory_order_relaxed); 1038345153Sdim} 1039345153Sdim 1040345153Sdimtemplate <typename T> 1041345153Sdimbool __kmp_atomic_compare_store_acq(std::atomic<T> *p, T expected, T desired) { 1042345153Sdim return p->compare_exchange_strong( 1043345153Sdim expected, desired, std::memory_order_acquire, std::memory_order_relaxed); 1044345153Sdim} 1045345153Sdim 1046345153Sdimtemplate <typename T> 1047345153Sdimbool __kmp_atomic_compare_store_rel(std::atomic<T> *p, T expected, T desired) { 1048345153Sdim return p->compare_exchange_strong( 1049345153Sdim expected, desired, std::memory_order_release, std::memory_order_relaxed); 1050345153Sdim} 1051345153Sdim 1052345153Sdim#endif /* KMP_OS_H */ 1053345153Sdim// Safe C API 1054345153Sdim#include "kmp_safe_c_api.h" 1055