intrin.h revision 344779
1/* ===-------- intrin.h ---------------------------------------------------=== 2 * 3 * Permission is hereby granted, free of charge, to any person obtaining a copy 4 * of this software and associated documentation files (the "Software"), to deal 5 * in the Software without restriction, including without limitation the rights 6 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 * copies of the Software, and to permit persons to whom the Software is 8 * furnished to do so, subject to the following conditions: 9 * 10 * The above copyright notice and this permission notice shall be included in 11 * all copies or substantial portions of the Software. 12 * 13 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 * THE SOFTWARE. 20 * 21 *===-----------------------------------------------------------------------=== 22 */ 23 24/* Only include this if we're compiling for the windows platform. */ 25#ifndef _MSC_VER 26#include_next <intrin.h> 27#else 28 29#ifndef __INTRIN_H 30#define __INTRIN_H 31 32/* First include the standard intrinsics. */ 33#if defined(__i386__) || defined(__x86_64__) 34#include <x86intrin.h> 35#endif 36 37#if defined(__arm__) 38#include <armintr.h> 39#endif 40 41#if defined(__aarch64__) 42#include <arm64intr.h> 43#endif 44 45/* For the definition of jmp_buf. */ 46#if __STDC_HOSTED__ 47#include <setjmp.h> 48#endif 49 50/* Define the default attributes for the functions in this file. */ 51#define __DEFAULT_FN_ATTRS __attribute__((__always_inline__, __nodebug__)) 52 53#ifdef __cplusplus 54extern "C" { 55#endif 56 57#if defined(__MMX__) 58/* And the random ones that aren't in those files. */ 59__m64 _m_from_float(float); 60float _m_to_float(__m64); 61#endif 62 63/* Other assorted instruction intrinsics. */ 64void __addfsbyte(unsigned long, unsigned char); 65void __addfsdword(unsigned long, unsigned long); 66void __addfsword(unsigned long, unsigned short); 67void __code_seg(const char *); 68static __inline__ 69void __cpuid(int[4], int); 70static __inline__ 71void __cpuidex(int[4], int, int); 72static __inline__ 73__int64 __emul(int, int); 74static __inline__ 75unsigned __int64 __emulu(unsigned int, unsigned int); 76unsigned int __getcallerseflags(void); 77static __inline__ 78void __halt(void); 79unsigned char __inbyte(unsigned short); 80void __inbytestring(unsigned short, unsigned char *, unsigned long); 81void __incfsbyte(unsigned long); 82void __incfsdword(unsigned long); 83void __incfsword(unsigned long); 84unsigned long __indword(unsigned short); 85void __indwordstring(unsigned short, unsigned long *, unsigned long); 86void __int2c(void); 87void __invlpg(void *); 88unsigned short __inword(unsigned short); 89void __inwordstring(unsigned short, unsigned short *, unsigned long); 90void __lidt(void *); 91unsigned __int64 __ll_lshift(unsigned __int64, int); 92__int64 __ll_rshift(__int64, int); 93static __inline__ 94void __movsb(unsigned char *, unsigned char const *, size_t); 95static __inline__ 96void __movsd(unsigned long *, unsigned long const *, size_t); 97static __inline__ 98void __movsw(unsigned short *, unsigned short const *, size_t); 99static __inline__ 100void __nop(void); 101void __nvreg_restore_fence(void); 102void __nvreg_save_fence(void); 103void __outbyte(unsigned short, unsigned char); 104void __outbytestring(unsigned short, unsigned char *, unsigned long); 105void __outdword(unsigned short, unsigned long); 106void __outdwordstring(unsigned short, unsigned long *, unsigned long); 107void __outword(unsigned short, unsigned short); 108void __outwordstring(unsigned short, unsigned short *, unsigned long); 109unsigned long __readcr0(void); 110unsigned long __readcr2(void); 111static __inline__ 112unsigned long __readcr3(void); 113unsigned long __readcr4(void); 114unsigned long __readcr8(void); 115unsigned int __readdr(unsigned int); 116#ifdef __i386__ 117static __inline__ 118unsigned char __readfsbyte(unsigned long); 119static __inline__ 120unsigned __int64 __readfsqword(unsigned long); 121static __inline__ 122unsigned short __readfsword(unsigned long); 123#endif 124static __inline__ 125unsigned __int64 __readmsr(unsigned long); 126unsigned __int64 __readpmc(unsigned long); 127unsigned long __segmentlimit(unsigned long); 128void __sidt(void *); 129static __inline__ 130void __stosb(unsigned char *, unsigned char, size_t); 131static __inline__ 132void __stosd(unsigned long *, unsigned long, size_t); 133static __inline__ 134void __stosw(unsigned short *, unsigned short, size_t); 135void __svm_clgi(void); 136void __svm_invlpga(void *, int); 137void __svm_skinit(int); 138void __svm_stgi(void); 139void __svm_vmload(size_t); 140void __svm_vmrun(size_t); 141void __svm_vmsave(size_t); 142void __ud2(void); 143unsigned __int64 __ull_rshift(unsigned __int64, int); 144void __vmx_off(void); 145void __vmx_vmptrst(unsigned __int64 *); 146void __wbinvd(void); 147void __writecr0(unsigned int); 148static __inline__ 149void __writecr3(unsigned int); 150void __writecr4(unsigned int); 151void __writecr8(unsigned int); 152void __writedr(unsigned int, unsigned int); 153void __writefsbyte(unsigned long, unsigned char); 154void __writefsdword(unsigned long, unsigned long); 155void __writefsqword(unsigned long, unsigned __int64); 156void __writefsword(unsigned long, unsigned short); 157void __writemsr(unsigned long, unsigned __int64); 158static __inline__ 159void *_AddressOfReturnAddress(void); 160static __inline__ 161unsigned char _BitScanForward(unsigned long *_Index, unsigned long _Mask); 162static __inline__ 163unsigned char _BitScanReverse(unsigned long *_Index, unsigned long _Mask); 164unsigned char _bittest(long const *, long); 165unsigned char _bittestandcomplement(long *, long); 166unsigned char _bittestandreset(long *, long); 167unsigned char _bittestandset(long *, long); 168void __cdecl _disable(void); 169void __cdecl _enable(void); 170long _InterlockedAddLargeStatistic(__int64 volatile *_Addend, long _Value); 171unsigned char _interlockedbittestandreset(long volatile *, long); 172unsigned char _interlockedbittestandset(long volatile *, long); 173void *_InterlockedCompareExchangePointer_HLEAcquire(void *volatile *, void *, 174 void *); 175void *_InterlockedCompareExchangePointer_HLERelease(void *volatile *, void *, 176 void *); 177long _InterlockedExchangeAdd_HLEAcquire(long volatile *, long); 178long _InterlockedExchangeAdd_HLERelease(long volatile *, long); 179__int64 _InterlockedExchangeAdd64_HLEAcquire(__int64 volatile *, __int64); 180__int64 _InterlockedExchangeAdd64_HLERelease(__int64 volatile *, __int64); 181void __cdecl _invpcid(unsigned int, void *); 182static __inline__ void 183__attribute__((__deprecated__("use other intrinsics or C++11 atomics instead"))) 184_ReadBarrier(void); 185static __inline__ void 186__attribute__((__deprecated__("use other intrinsics or C++11 atomics instead"))) 187_ReadWriteBarrier(void); 188unsigned int _rorx_u32(unsigned int, const unsigned int); 189int _sarx_i32(int, unsigned int); 190#if __STDC_HOSTED__ 191int __cdecl _setjmp(jmp_buf); 192#endif 193unsigned int _shlx_u32(unsigned int, unsigned int); 194unsigned int _shrx_u32(unsigned int, unsigned int); 195void _Store_HLERelease(long volatile *, long); 196void _Store64_HLERelease(__int64 volatile *, __int64); 197void _StorePointer_HLERelease(void *volatile *, void *); 198static __inline__ void 199__attribute__((__deprecated__("use other intrinsics or C++11 atomics instead"))) 200_WriteBarrier(void); 201unsigned __int32 xbegin(void); 202void _xend(void); 203static __inline__ 204#define _XCR_XFEATURE_ENABLED_MASK 0 205unsigned __int64 __cdecl _xgetbv(unsigned int); 206void __cdecl _xsetbv(unsigned int, unsigned __int64); 207 208/* These additional intrinsics are turned on in x64/amd64/x86_64 mode. */ 209#ifdef __x86_64__ 210void __addgsbyte(unsigned long, unsigned char); 211void __addgsdword(unsigned long, unsigned long); 212void __addgsqword(unsigned long, unsigned __int64); 213void __addgsword(unsigned long, unsigned short); 214static __inline__ 215void __faststorefence(void); 216void __incgsbyte(unsigned long); 217void __incgsdword(unsigned long); 218void __incgsqword(unsigned long); 219void __incgsword(unsigned long); 220static __inline__ 221void __movsq(unsigned long long *, unsigned long long const *, size_t); 222static __inline__ 223unsigned char __readgsbyte(unsigned long); 224static __inline__ 225unsigned long __readgsdword(unsigned long); 226static __inline__ 227unsigned __int64 __readgsqword(unsigned long); 228unsigned short __readgsword(unsigned long); 229unsigned __int64 __shiftleft128(unsigned __int64 _LowPart, 230 unsigned __int64 _HighPart, 231 unsigned char _Shift); 232unsigned __int64 __shiftright128(unsigned __int64 _LowPart, 233 unsigned __int64 _HighPart, 234 unsigned char _Shift); 235static __inline__ 236void __stosq(unsigned __int64 *, unsigned __int64, size_t); 237unsigned char __vmx_on(unsigned __int64 *); 238unsigned char __vmx_vmclear(unsigned __int64 *); 239unsigned char __vmx_vmlaunch(void); 240unsigned char __vmx_vmptrld(unsigned __int64 *); 241unsigned char __vmx_vmread(size_t, size_t *); 242unsigned char __vmx_vmresume(void); 243unsigned char __vmx_vmwrite(size_t, size_t); 244void __writegsbyte(unsigned long, unsigned char); 245void __writegsdword(unsigned long, unsigned long); 246void __writegsqword(unsigned long, unsigned __int64); 247void __writegsword(unsigned long, unsigned short); 248unsigned char _bittest64(__int64 const *, __int64); 249unsigned char _bittestandcomplement64(__int64 *, __int64); 250unsigned char _bittestandreset64(__int64 *, __int64); 251unsigned char _bittestandset64(__int64 *, __int64); 252long _InterlockedAnd_np(long volatile *_Value, long _Mask); 253short _InterlockedAnd16_np(short volatile *_Value, short _Mask); 254__int64 _InterlockedAnd64_np(__int64 volatile *_Value, __int64 _Mask); 255char _InterlockedAnd8_np(char volatile *_Value, char _Mask); 256unsigned char _interlockedbittestandreset64(__int64 volatile *, __int64); 257unsigned char _interlockedbittestandset64(__int64 volatile *, __int64); 258long _InterlockedCompareExchange_np(long volatile *_Destination, long _Exchange, 259 long _Comparand); 260unsigned char _InterlockedCompareExchange128(__int64 volatile *_Destination, 261 __int64 _ExchangeHigh, 262 __int64 _ExchangeLow, 263 __int64 *_CompareandResult); 264unsigned char _InterlockedCompareExchange128_np(__int64 volatile *_Destination, 265 __int64 _ExchangeHigh, 266 __int64 _ExchangeLow, 267 __int64 *_ComparandResult); 268short _InterlockedCompareExchange16_np(short volatile *_Destination, 269 short _Exchange, short _Comparand); 270__int64 _InterlockedCompareExchange64_np(__int64 volatile *_Destination, 271 __int64 _Exchange, __int64 _Comparand); 272void *_InterlockedCompareExchangePointer_np(void *volatile *_Destination, 273 void *_Exchange, void *_Comparand); 274long _InterlockedOr_np(long volatile *_Value, long _Mask); 275short _InterlockedOr16_np(short volatile *_Value, short _Mask); 276__int64 _InterlockedOr64_np(__int64 volatile *_Value, __int64 _Mask); 277char _InterlockedOr8_np(char volatile *_Value, char _Mask); 278long _InterlockedXor_np(long volatile *_Value, long _Mask); 279short _InterlockedXor16_np(short volatile *_Value, short _Mask); 280__int64 _InterlockedXor64_np(__int64 volatile *_Value, __int64 _Mask); 281char _InterlockedXor8_np(char volatile *_Value, char _Mask); 282unsigned __int64 _rorx_u64(unsigned __int64, const unsigned int); 283__int64 _sarx_i64(__int64, unsigned int); 284unsigned __int64 _shlx_u64(unsigned __int64, unsigned int); 285unsigned __int64 _shrx_u64(unsigned __int64, unsigned int); 286static __inline__ 287__int64 __mulh(__int64, __int64); 288static __inline__ 289unsigned __int64 __umulh(unsigned __int64, unsigned __int64); 290static __inline__ 291__int64 _mul128(__int64, __int64, __int64*); 292static __inline__ 293unsigned __int64 _umul128(unsigned __int64, 294 unsigned __int64, 295 unsigned __int64*); 296 297#endif /* __x86_64__ */ 298 299#if defined(__x86_64__) || defined(__arm__) || defined(__aarch64__) 300 301static __inline__ 302unsigned char _BitScanForward64(unsigned long *_Index, unsigned __int64 _Mask); 303static __inline__ 304unsigned char _BitScanReverse64(unsigned long *_Index, unsigned __int64 _Mask); 305 306static __inline__ 307__int64 _InterlockedDecrement64(__int64 volatile *_Addend); 308static __inline__ 309__int64 _InterlockedExchange64(__int64 volatile *_Target, __int64 _Value); 310static __inline__ 311__int64 _InterlockedExchangeAdd64(__int64 volatile *_Addend, __int64 _Value); 312static __inline__ 313__int64 _InterlockedExchangeSub64(__int64 volatile *_Subend, __int64 _Value); 314static __inline__ 315__int64 _InterlockedIncrement64(__int64 volatile *_Addend); 316static __inline__ 317__int64 _InterlockedOr64(__int64 volatile *_Value, __int64 _Mask); 318static __inline__ 319__int64 _InterlockedXor64(__int64 volatile *_Value, __int64 _Mask); 320static __inline__ 321__int64 _InterlockedAnd64(__int64 volatile *_Value, __int64 _Mask); 322 323#endif 324 325/*----------------------------------------------------------------------------*\ 326|* Interlocked Exchange Add 327\*----------------------------------------------------------------------------*/ 328#if defined(__arm__) || defined(__aarch64__) 329char _InterlockedExchangeAdd8_acq(char volatile *_Addend, char _Value); 330char _InterlockedExchangeAdd8_nf(char volatile *_Addend, char _Value); 331char _InterlockedExchangeAdd8_rel(char volatile *_Addend, char _Value); 332short _InterlockedExchangeAdd16_acq(short volatile *_Addend, short _Value); 333short _InterlockedExchangeAdd16_nf(short volatile *_Addend, short _Value); 334short _InterlockedExchangeAdd16_rel(short volatile *_Addend, short _Value); 335long _InterlockedExchangeAdd_acq(long volatile *_Addend, long _Value); 336long _InterlockedExchangeAdd_nf(long volatile *_Addend, long _Value); 337long _InterlockedExchangeAdd_rel(long volatile *_Addend, long _Value); 338__int64 _InterlockedExchangeAdd64_acq(__int64 volatile *_Addend, __int64 _Value); 339__int64 _InterlockedExchangeAdd64_nf(__int64 volatile *_Addend, __int64 _Value); 340__int64 _InterlockedExchangeAdd64_rel(__int64 volatile *_Addend, __int64 _Value); 341#endif 342/*----------------------------------------------------------------------------*\ 343|* Interlocked Increment 344\*----------------------------------------------------------------------------*/ 345#if defined(__arm__) || defined(__aarch64__) 346short _InterlockedIncrement16_acq(short volatile *_Value); 347short _InterlockedIncrement16_nf(short volatile *_Value); 348short _InterlockedIncrement16_rel(short volatile *_Value); 349long _InterlockedIncrement_acq(long volatile *_Value); 350long _InterlockedIncrement_nf(long volatile *_Value); 351long _InterlockedIncrement_rel(long volatile *_Value); 352__int64 _InterlockedIncrement64_acq(__int64 volatile *_Value); 353__int64 _InterlockedIncrement64_nf(__int64 volatile *_Value); 354__int64 _InterlockedIncrement64_rel(__int64 volatile *_Value); 355#endif 356/*----------------------------------------------------------------------------*\ 357|* Interlocked Decrement 358\*----------------------------------------------------------------------------*/ 359#if defined(__arm__) || defined(__aarch64__) 360short _InterlockedDecrement16_acq(short volatile *_Value); 361short _InterlockedDecrement16_nf(short volatile *_Value); 362short _InterlockedDecrement16_rel(short volatile *_Value); 363long _InterlockedDecrement_acq(long volatile *_Value); 364long _InterlockedDecrement_nf(long volatile *_Value); 365long _InterlockedDecrement_rel(long volatile *_Value); 366__int64 _InterlockedDecrement64_acq(__int64 volatile *_Value); 367__int64 _InterlockedDecrement64_nf(__int64 volatile *_Value); 368__int64 _InterlockedDecrement64_rel(__int64 volatile *_Value); 369#endif 370/*----------------------------------------------------------------------------*\ 371|* Interlocked And 372\*----------------------------------------------------------------------------*/ 373#if defined(__arm__) || defined(__aarch64__) 374char _InterlockedAnd8_acq(char volatile *_Value, char _Mask); 375char _InterlockedAnd8_nf(char volatile *_Value, char _Mask); 376char _InterlockedAnd8_rel(char volatile *_Value, char _Mask); 377short _InterlockedAnd16_acq(short volatile *_Value, short _Mask); 378short _InterlockedAnd16_nf(short volatile *_Value, short _Mask); 379short _InterlockedAnd16_rel(short volatile *_Value, short _Mask); 380long _InterlockedAnd_acq(long volatile *_Value, long _Mask); 381long _InterlockedAnd_nf(long volatile *_Value, long _Mask); 382long _InterlockedAnd_rel(long volatile *_Value, long _Mask); 383__int64 _InterlockedAnd64_acq(__int64 volatile *_Value, __int64 _Mask); 384__int64 _InterlockedAnd64_nf(__int64 volatile *_Value, __int64 _Mask); 385__int64 _InterlockedAnd64_rel(__int64 volatile *_Value, __int64 _Mask); 386#endif 387/*----------------------------------------------------------------------------*\ 388|* Bit Counting and Testing 389\*----------------------------------------------------------------------------*/ 390#if defined(__arm__) || defined(__aarch64__) 391unsigned char _interlockedbittestandset_acq(long volatile *_BitBase, 392 long _BitPos); 393unsigned char _interlockedbittestandset_nf(long volatile *_BitBase, 394 long _BitPos); 395unsigned char _interlockedbittestandset_rel(long volatile *_BitBase, 396 long _BitPos); 397unsigned char _interlockedbittestandreset_acq(long volatile *_BitBase, 398 long _BitPos); 399unsigned char _interlockedbittestandreset_nf(long volatile *_BitBase, 400 long _BitPos); 401unsigned char _interlockedbittestandreset_rel(long volatile *_BitBase, 402 long _BitPos); 403#endif 404/*----------------------------------------------------------------------------*\ 405|* Interlocked Or 406\*----------------------------------------------------------------------------*/ 407#if defined(__arm__) || defined(__aarch64__) 408char _InterlockedOr8_acq(char volatile *_Value, char _Mask); 409char _InterlockedOr8_nf(char volatile *_Value, char _Mask); 410char _InterlockedOr8_rel(char volatile *_Value, char _Mask); 411short _InterlockedOr16_acq(short volatile *_Value, short _Mask); 412short _InterlockedOr16_nf(short volatile *_Value, short _Mask); 413short _InterlockedOr16_rel(short volatile *_Value, short _Mask); 414long _InterlockedOr_acq(long volatile *_Value, long _Mask); 415long _InterlockedOr_nf(long volatile *_Value, long _Mask); 416long _InterlockedOr_rel(long volatile *_Value, long _Mask); 417__int64 _InterlockedOr64_acq(__int64 volatile *_Value, __int64 _Mask); 418__int64 _InterlockedOr64_nf(__int64 volatile *_Value, __int64 _Mask); 419__int64 _InterlockedOr64_rel(__int64 volatile *_Value, __int64 _Mask); 420#endif 421/*----------------------------------------------------------------------------*\ 422|* Interlocked Xor 423\*----------------------------------------------------------------------------*/ 424#if defined(__arm__) || defined(__aarch64__) 425char _InterlockedXor8_acq(char volatile *_Value, char _Mask); 426char _InterlockedXor8_nf(char volatile *_Value, char _Mask); 427char _InterlockedXor8_rel(char volatile *_Value, char _Mask); 428short _InterlockedXor16_acq(short volatile *_Value, short _Mask); 429short _InterlockedXor16_nf(short volatile *_Value, short _Mask); 430short _InterlockedXor16_rel(short volatile *_Value, short _Mask); 431long _InterlockedXor_acq(long volatile *_Value, long _Mask); 432long _InterlockedXor_nf(long volatile *_Value, long _Mask); 433long _InterlockedXor_rel(long volatile *_Value, long _Mask); 434__int64 _InterlockedXor64_acq(__int64 volatile *_Value, __int64 _Mask); 435__int64 _InterlockedXor64_nf(__int64 volatile *_Value, __int64 _Mask); 436__int64 _InterlockedXor64_rel(__int64 volatile *_Value, __int64 _Mask); 437#endif 438/*----------------------------------------------------------------------------*\ 439|* Interlocked Exchange 440\*----------------------------------------------------------------------------*/ 441#if defined(__arm__) || defined(__aarch64__) 442char _InterlockedExchange8_acq(char volatile *_Target, char _Value); 443char _InterlockedExchange8_nf(char volatile *_Target, char _Value); 444char _InterlockedExchange8_rel(char volatile *_Target, char _Value); 445short _InterlockedExchange16_acq(short volatile *_Target, short _Value); 446short _InterlockedExchange16_nf(short volatile *_Target, short _Value); 447short _InterlockedExchange16_rel(short volatile *_Target, short _Value); 448long _InterlockedExchange_acq(long volatile *_Target, long _Value); 449long _InterlockedExchange_nf(long volatile *_Target, long _Value); 450long _InterlockedExchange_rel(long volatile *_Target, long _Value); 451__int64 _InterlockedExchange64_acq(__int64 volatile *_Target, __int64 _Value); 452__int64 _InterlockedExchange64_nf(__int64 volatile *_Target, __int64 _Value); 453__int64 _InterlockedExchange64_rel(__int64 volatile *_Target, __int64 _Value); 454#endif 455/*----------------------------------------------------------------------------*\ 456|* Interlocked Compare Exchange 457\*----------------------------------------------------------------------------*/ 458#if defined(__arm__) || defined(__aarch64__) 459char _InterlockedCompareExchange8_acq(char volatile *_Destination, 460 char _Exchange, char _Comparand); 461char _InterlockedCompareExchange8_nf(char volatile *_Destination, 462 char _Exchange, char _Comparand); 463char _InterlockedCompareExchange8_rel(char volatile *_Destination, 464 char _Exchange, char _Comparand); 465short _InterlockedCompareExchange16_acq(short volatile *_Destination, 466 short _Exchange, short _Comparand); 467short _InterlockedCompareExchange16_nf(short volatile *_Destination, 468 short _Exchange, short _Comparand); 469short _InterlockedCompareExchange16_rel(short volatile *_Destination, 470 short _Exchange, short _Comparand); 471long _InterlockedCompareExchange_acq(long volatile *_Destination, 472 long _Exchange, long _Comparand); 473long _InterlockedCompareExchange_nf(long volatile *_Destination, 474 long _Exchange, long _Comparand); 475long _InterlockedCompareExchange_rel(long volatile *_Destination, 476 long _Exchange, long _Comparand); 477__int64 _InterlockedCompareExchange64_acq(__int64 volatile *_Destination, 478 __int64 _Exchange, __int64 _Comparand); 479__int64 _InterlockedCompareExchange64_nf(__int64 volatile *_Destination, 480 __int64 _Exchange, __int64 _Comparand); 481__int64 _InterlockedCompareExchange64_rel(__int64 volatile *_Destination, 482 __int64 _Exchange, __int64 _Comparand); 483#endif 484 485/*----------------------------------------------------------------------------*\ 486|* movs, stos 487\*----------------------------------------------------------------------------*/ 488#if defined(__i386__) || defined(__x86_64__) 489static __inline__ void __DEFAULT_FN_ATTRS 490__movsb(unsigned char *__dst, unsigned char const *__src, size_t __n) { 491 __asm__ __volatile__("rep movsb" : "+D"(__dst), "+S"(__src), "+c"(__n) 492 : : "memory"); 493} 494static __inline__ void __DEFAULT_FN_ATTRS 495__movsd(unsigned long *__dst, unsigned long const *__src, size_t __n) { 496 __asm__ __volatile__("rep movsl" : "+D"(__dst), "+S"(__src), "+c"(__n) 497 : : "memory"); 498} 499static __inline__ void __DEFAULT_FN_ATTRS 500__movsw(unsigned short *__dst, unsigned short const *__src, size_t __n) { 501 __asm__ __volatile__("rep movsw" : "+D"(__dst), "+S"(__src), "+c"(__n) 502 : : "memory"); 503} 504static __inline__ void __DEFAULT_FN_ATTRS 505__stosd(unsigned long *__dst, unsigned long __x, size_t __n) { 506 __asm__ __volatile__("rep stosl" : "+D"(__dst), "+c"(__n) : "a"(__x) 507 : "memory"); 508} 509static __inline__ void __DEFAULT_FN_ATTRS 510__stosw(unsigned short *__dst, unsigned short __x, size_t __n) { 511 __asm__ __volatile__("rep stosw" : "+D"(__dst), "+c"(__n) : "a"(__x) 512 : "memory"); 513} 514#endif 515#ifdef __x86_64__ 516static __inline__ void __DEFAULT_FN_ATTRS 517__movsq(unsigned long long *__dst, unsigned long long const *__src, size_t __n) { 518 __asm__ __volatile__("rep movsq" : "+D"(__dst), "+S"(__src), "+c"(__n) 519 : : "memory"); 520} 521static __inline__ void __DEFAULT_FN_ATTRS 522__stosq(unsigned __int64 *__dst, unsigned __int64 __x, size_t __n) { 523 __asm__ __volatile__("rep stosq" : "+D"(__dst), "+c"(__n) : "a"(__x) 524 : "memory"); 525} 526#endif 527 528/*----------------------------------------------------------------------------*\ 529|* Misc 530\*----------------------------------------------------------------------------*/ 531#if defined(__i386__) || defined(__x86_64__) 532static __inline__ void __DEFAULT_FN_ATTRS 533__cpuid(int __info[4], int __level) { 534 __asm__ ("cpuid" : "=a"(__info[0]), "=b" (__info[1]), "=c"(__info[2]), "=d"(__info[3]) 535 : "a"(__level), "c"(0)); 536} 537static __inline__ void __DEFAULT_FN_ATTRS 538__cpuidex(int __info[4], int __level, int __ecx) { 539 __asm__ ("cpuid" : "=a"(__info[0]), "=b" (__info[1]), "=c"(__info[2]), "=d"(__info[3]) 540 : "a"(__level), "c"(__ecx)); 541} 542static __inline__ unsigned __int64 __cdecl __DEFAULT_FN_ATTRS 543_xgetbv(unsigned int __xcr_no) { 544 unsigned int __eax, __edx; 545 __asm__ ("xgetbv" : "=a" (__eax), "=d" (__edx) : "c" (__xcr_no)); 546 return ((unsigned __int64)__edx << 32) | __eax; 547} 548static __inline__ void __DEFAULT_FN_ATTRS 549__halt(void) { 550 __asm__ volatile ("hlt"); 551} 552#endif 553 554#if defined(__i386__) || defined(__x86_64__) || defined(__aarch64__) 555static __inline__ void __DEFAULT_FN_ATTRS 556__nop(void) { 557 __asm__ volatile ("nop"); 558} 559#endif 560 561/*----------------------------------------------------------------------------*\ 562|* MS AArch64 specific 563\*----------------------------------------------------------------------------*/ 564#if defined(__aarch64__) 565unsigned __int64 __getReg(int); 566long _InterlockedAdd(long volatile *Addend, long Value); 567__int64 _ReadStatusReg(int); 568void _WriteStatusReg(int, __int64); 569 570static inline unsigned short _byteswap_ushort (unsigned short val) { 571 return __builtin_bswap16(val); 572} 573static inline unsigned long _byteswap_ulong (unsigned long val) { 574 return __builtin_bswap32(val); 575} 576static inline unsigned __int64 _byteswap_uint64 (unsigned __int64 val) { 577 return __builtin_bswap64(val); 578} 579#endif 580 581/*----------------------------------------------------------------------------*\ 582|* Privileged intrinsics 583\*----------------------------------------------------------------------------*/ 584#if defined(__i386__) || defined(__x86_64__) 585static __inline__ unsigned __int64 __DEFAULT_FN_ATTRS 586__readmsr(unsigned long __register) { 587 // Loads the contents of a 64-bit model specific register (MSR) specified in 588 // the ECX register into registers EDX:EAX. The EDX register is loaded with 589 // the high-order 32 bits of the MSR and the EAX register is loaded with the 590 // low-order 32 bits. If less than 64 bits are implemented in the MSR being 591 // read, the values returned to EDX:EAX in unimplemented bit locations are 592 // undefined. 593 unsigned long __edx; 594 unsigned long __eax; 595 __asm__ ("rdmsr" : "=d"(__edx), "=a"(__eax) : "c"(__register)); 596 return (((unsigned __int64)__edx) << 32) | (unsigned __int64)__eax; 597} 598 599static __inline__ unsigned long __DEFAULT_FN_ATTRS 600__readcr3(void) { 601 unsigned long __cr3_val; 602 __asm__ __volatile__ ("mov %%cr3, %0" : "=q"(__cr3_val) : : "memory"); 603 return __cr3_val; 604} 605 606static __inline__ void __DEFAULT_FN_ATTRS 607__writecr3(unsigned int __cr3_val) { 608 __asm__ ("mov %0, %%cr3" : : "q"(__cr3_val) : "memory"); 609} 610#endif 611 612#ifdef __cplusplus 613} 614#endif 615 616#undef __DEFAULT_FN_ATTRS 617 618#endif /* __INTRIN_H */ 619#endif /* _MSC_VER */ 620