intrin.h revision 360660
1/* ===-------- intrin.h ---------------------------------------------------=== 2 * 3 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 * See https://llvm.org/LICENSE.txt for license information. 5 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 * 7 *===-----------------------------------------------------------------------=== 8 */ 9 10/* Only include this if we're compiling for the windows platform. */ 11#ifndef _MSC_VER 12#include_next <intrin.h> 13#else 14 15#ifndef __INTRIN_H 16#define __INTRIN_H 17 18/* First include the standard intrinsics. */ 19#if defined(__i386__) || defined(__x86_64__) 20#include <x86intrin.h> 21#endif 22 23#if defined(__arm__) 24#include <armintr.h> 25#endif 26 27#if defined(__aarch64__) 28#include <arm64intr.h> 29#endif 30 31/* For the definition of jmp_buf. */ 32#if __STDC_HOSTED__ 33#include <setjmp.h> 34#endif 35 36/* Define the default attributes for the functions in this file. */ 37#define __DEFAULT_FN_ATTRS __attribute__((__always_inline__, __nodebug__)) 38 39#ifdef __cplusplus 40extern "C" { 41#endif 42 43#if defined(__MMX__) 44/* And the random ones that aren't in those files. */ 45__m64 _m_from_float(float); 46float _m_to_float(__m64); 47#endif 48 49/* Other assorted instruction intrinsics. */ 50void __addfsbyte(unsigned long, unsigned char); 51void __addfsdword(unsigned long, unsigned long); 52void __addfsword(unsigned long, unsigned short); 53void __code_seg(const char *); 54static __inline__ 55void __cpuid(int[4], int); 56static __inline__ 57void __cpuidex(int[4], int, int); 58static __inline__ 59__int64 __emul(int, int); 60static __inline__ 61unsigned __int64 __emulu(unsigned int, unsigned int); 62unsigned int __getcallerseflags(void); 63static __inline__ 64void __halt(void); 65unsigned char __inbyte(unsigned short); 66void __inbytestring(unsigned short, unsigned char *, unsigned long); 67void __incfsbyte(unsigned long); 68void __incfsdword(unsigned long); 69void __incfsword(unsigned long); 70unsigned long __indword(unsigned short); 71void __indwordstring(unsigned short, unsigned long *, unsigned long); 72void __int2c(void); 73void __invlpg(void *); 74unsigned short __inword(unsigned short); 75void __inwordstring(unsigned short, unsigned short *, unsigned long); 76void __lidt(void *); 77unsigned __int64 __ll_lshift(unsigned __int64, int); 78__int64 __ll_rshift(__int64, int); 79static __inline__ 80void __movsb(unsigned char *, unsigned char const *, size_t); 81static __inline__ 82void __movsd(unsigned long *, unsigned long const *, size_t); 83static __inline__ 84void __movsw(unsigned short *, unsigned short const *, size_t); 85static __inline__ 86void __nop(void); 87void __nvreg_restore_fence(void); 88void __nvreg_save_fence(void); 89void __outbyte(unsigned short, unsigned char); 90void __outbytestring(unsigned short, unsigned char *, unsigned long); 91void __outdword(unsigned short, unsigned long); 92void __outdwordstring(unsigned short, unsigned long *, unsigned long); 93void __outword(unsigned short, unsigned short); 94void __outwordstring(unsigned short, unsigned short *, unsigned long); 95unsigned long __readcr0(void); 96unsigned long __readcr2(void); 97static __inline__ 98unsigned long __readcr3(void); 99unsigned long __readcr4(void); 100unsigned long __readcr8(void); 101unsigned int __readdr(unsigned int); 102#ifdef __i386__ 103static __inline__ 104unsigned char __readfsbyte(unsigned long); 105static __inline__ 106unsigned __int64 __readfsqword(unsigned long); 107static __inline__ 108unsigned short __readfsword(unsigned long); 109#endif 110static __inline__ 111unsigned __int64 __readmsr(unsigned long); 112unsigned __int64 __readpmc(unsigned long); 113unsigned long __segmentlimit(unsigned long); 114void __sidt(void *); 115static __inline__ 116void __stosb(unsigned char *, unsigned char, size_t); 117static __inline__ 118void __stosd(unsigned long *, unsigned long, size_t); 119static __inline__ 120void __stosw(unsigned short *, unsigned short, size_t); 121void __svm_clgi(void); 122void __svm_invlpga(void *, int); 123void __svm_skinit(int); 124void __svm_stgi(void); 125void __svm_vmload(size_t); 126void __svm_vmrun(size_t); 127void __svm_vmsave(size_t); 128void __ud2(void); 129unsigned __int64 __ull_rshift(unsigned __int64, int); 130void __vmx_off(void); 131void __vmx_vmptrst(unsigned __int64 *); 132void __wbinvd(void); 133void __writecr0(unsigned int); 134static __inline__ 135void __writecr3(unsigned int); 136void __writecr4(unsigned int); 137void __writecr8(unsigned int); 138void __writedr(unsigned int, unsigned int); 139void __writefsbyte(unsigned long, unsigned char); 140void __writefsdword(unsigned long, unsigned long); 141void __writefsqword(unsigned long, unsigned __int64); 142void __writefsword(unsigned long, unsigned short); 143void __writemsr(unsigned long, unsigned __int64); 144static __inline__ 145void *_AddressOfReturnAddress(void); 146static __inline__ 147unsigned char _BitScanForward(unsigned long *_Index, unsigned long _Mask); 148static __inline__ 149unsigned char _BitScanReverse(unsigned long *_Index, unsigned long _Mask); 150unsigned char _bittest(long const *, long); 151unsigned char _bittestandcomplement(long *, long); 152unsigned char _bittestandreset(long *, long); 153unsigned char _bittestandset(long *, long); 154void __cdecl _disable(void); 155void __cdecl _enable(void); 156long _InterlockedAddLargeStatistic(__int64 volatile *_Addend, long _Value); 157unsigned char _interlockedbittestandreset(long volatile *, long); 158unsigned char _interlockedbittestandset(long volatile *, long); 159void *_InterlockedCompareExchangePointer_HLEAcquire(void *volatile *, void *, 160 void *); 161void *_InterlockedCompareExchangePointer_HLERelease(void *volatile *, void *, 162 void *); 163long _InterlockedExchangeAdd_HLEAcquire(long volatile *, long); 164long _InterlockedExchangeAdd_HLERelease(long volatile *, long); 165__int64 _InterlockedExchangeAdd64_HLEAcquire(__int64 volatile *, __int64); 166__int64 _InterlockedExchangeAdd64_HLERelease(__int64 volatile *, __int64); 167void __cdecl _invpcid(unsigned int, void *); 168static __inline__ void 169__attribute__((__deprecated__("use other intrinsics or C++11 atomics instead"))) 170_ReadBarrier(void); 171static __inline__ void 172__attribute__((__deprecated__("use other intrinsics or C++11 atomics instead"))) 173_ReadWriteBarrier(void); 174unsigned int _rorx_u32(unsigned int, const unsigned int); 175int _sarx_i32(int, unsigned int); 176#if __STDC_HOSTED__ 177int __cdecl _setjmp(jmp_buf); 178#endif 179unsigned int _shlx_u32(unsigned int, unsigned int); 180unsigned int _shrx_u32(unsigned int, unsigned int); 181void _Store_HLERelease(long volatile *, long); 182void _Store64_HLERelease(__int64 volatile *, __int64); 183void _StorePointer_HLERelease(void *volatile *, void *); 184static __inline__ void 185__attribute__((__deprecated__("use other intrinsics or C++11 atomics instead"))) 186_WriteBarrier(void); 187unsigned __int32 xbegin(void); 188void _xend(void); 189 190/* These additional intrinsics are turned on in x64/amd64/x86_64 mode. */ 191#ifdef __x86_64__ 192void __addgsbyte(unsigned long, unsigned char); 193void __addgsdword(unsigned long, unsigned long); 194void __addgsqword(unsigned long, unsigned __int64); 195void __addgsword(unsigned long, unsigned short); 196static __inline__ 197void __faststorefence(void); 198void __incgsbyte(unsigned long); 199void __incgsdword(unsigned long); 200void __incgsqword(unsigned long); 201void __incgsword(unsigned long); 202static __inline__ 203void __movsq(unsigned long long *, unsigned long long const *, size_t); 204static __inline__ 205unsigned char __readgsbyte(unsigned long); 206static __inline__ 207unsigned long __readgsdword(unsigned long); 208static __inline__ 209unsigned __int64 __readgsqword(unsigned long); 210unsigned short __readgsword(unsigned long); 211unsigned __int64 __shiftleft128(unsigned __int64 _LowPart, 212 unsigned __int64 _HighPart, 213 unsigned char _Shift); 214unsigned __int64 __shiftright128(unsigned __int64 _LowPart, 215 unsigned __int64 _HighPart, 216 unsigned char _Shift); 217static __inline__ 218void __stosq(unsigned __int64 *, unsigned __int64, size_t); 219unsigned char __vmx_on(unsigned __int64 *); 220unsigned char __vmx_vmclear(unsigned __int64 *); 221unsigned char __vmx_vmlaunch(void); 222unsigned char __vmx_vmptrld(unsigned __int64 *); 223unsigned char __vmx_vmread(size_t, size_t *); 224unsigned char __vmx_vmresume(void); 225unsigned char __vmx_vmwrite(size_t, size_t); 226void __writegsbyte(unsigned long, unsigned char); 227void __writegsdword(unsigned long, unsigned long); 228void __writegsqword(unsigned long, unsigned __int64); 229void __writegsword(unsigned long, unsigned short); 230unsigned char _bittest64(__int64 const *, __int64); 231unsigned char _bittestandcomplement64(__int64 *, __int64); 232unsigned char _bittestandreset64(__int64 *, __int64); 233unsigned char _bittestandset64(__int64 *, __int64); 234long _InterlockedAnd_np(long volatile *_Value, long _Mask); 235short _InterlockedAnd16_np(short volatile *_Value, short _Mask); 236__int64 _InterlockedAnd64_np(__int64 volatile *_Value, __int64 _Mask); 237char _InterlockedAnd8_np(char volatile *_Value, char _Mask); 238unsigned char _interlockedbittestandreset64(__int64 volatile *, __int64); 239unsigned char _interlockedbittestandset64(__int64 volatile *, __int64); 240long _InterlockedCompareExchange_np(long volatile *_Destination, long _Exchange, 241 long _Comparand); 242unsigned char _InterlockedCompareExchange128(__int64 volatile *_Destination, 243 __int64 _ExchangeHigh, 244 __int64 _ExchangeLow, 245 __int64 *_CompareandResult); 246unsigned char _InterlockedCompareExchange128_np(__int64 volatile *_Destination, 247 __int64 _ExchangeHigh, 248 __int64 _ExchangeLow, 249 __int64 *_ComparandResult); 250short _InterlockedCompareExchange16_np(short volatile *_Destination, 251 short _Exchange, short _Comparand); 252__int64 _InterlockedCompareExchange64_np(__int64 volatile *_Destination, 253 __int64 _Exchange, __int64 _Comparand); 254void *_InterlockedCompareExchangePointer_np(void *volatile *_Destination, 255 void *_Exchange, void *_Comparand); 256long _InterlockedOr_np(long volatile *_Value, long _Mask); 257short _InterlockedOr16_np(short volatile *_Value, short _Mask); 258__int64 _InterlockedOr64_np(__int64 volatile *_Value, __int64 _Mask); 259char _InterlockedOr8_np(char volatile *_Value, char _Mask); 260long _InterlockedXor_np(long volatile *_Value, long _Mask); 261short _InterlockedXor16_np(short volatile *_Value, short _Mask); 262__int64 _InterlockedXor64_np(__int64 volatile *_Value, __int64 _Mask); 263char _InterlockedXor8_np(char volatile *_Value, char _Mask); 264unsigned __int64 _rorx_u64(unsigned __int64, const unsigned int); 265__int64 _sarx_i64(__int64, unsigned int); 266unsigned __int64 _shlx_u64(unsigned __int64, unsigned int); 267unsigned __int64 _shrx_u64(unsigned __int64, unsigned int); 268static __inline__ 269__int64 __mulh(__int64, __int64); 270static __inline__ 271unsigned __int64 __umulh(unsigned __int64, unsigned __int64); 272static __inline__ 273__int64 _mul128(__int64, __int64, __int64*); 274static __inline__ 275unsigned __int64 _umul128(unsigned __int64, 276 unsigned __int64, 277 unsigned __int64*); 278 279#endif /* __x86_64__ */ 280 281#if defined(__x86_64__) || defined(__arm__) || defined(__aarch64__) 282 283static __inline__ 284unsigned char _BitScanForward64(unsigned long *_Index, unsigned __int64 _Mask); 285static __inline__ 286unsigned char _BitScanReverse64(unsigned long *_Index, unsigned __int64 _Mask); 287 288static __inline__ 289__int64 _InterlockedDecrement64(__int64 volatile *_Addend); 290static __inline__ 291__int64 _InterlockedExchange64(__int64 volatile *_Target, __int64 _Value); 292static __inline__ 293__int64 _InterlockedExchangeAdd64(__int64 volatile *_Addend, __int64 _Value); 294static __inline__ 295__int64 _InterlockedExchangeSub64(__int64 volatile *_Subend, __int64 _Value); 296static __inline__ 297__int64 _InterlockedIncrement64(__int64 volatile *_Addend); 298static __inline__ 299__int64 _InterlockedOr64(__int64 volatile *_Value, __int64 _Mask); 300static __inline__ 301__int64 _InterlockedXor64(__int64 volatile *_Value, __int64 _Mask); 302static __inline__ 303__int64 _InterlockedAnd64(__int64 volatile *_Value, __int64 _Mask); 304 305#endif 306 307/*----------------------------------------------------------------------------*\ 308|* Interlocked Exchange Add 309\*----------------------------------------------------------------------------*/ 310#if defined(__arm__) || defined(__aarch64__) 311char _InterlockedExchangeAdd8_acq(char volatile *_Addend, char _Value); 312char _InterlockedExchangeAdd8_nf(char volatile *_Addend, char _Value); 313char _InterlockedExchangeAdd8_rel(char volatile *_Addend, char _Value); 314short _InterlockedExchangeAdd16_acq(short volatile *_Addend, short _Value); 315short _InterlockedExchangeAdd16_nf(short volatile *_Addend, short _Value); 316short _InterlockedExchangeAdd16_rel(short volatile *_Addend, short _Value); 317long _InterlockedExchangeAdd_acq(long volatile *_Addend, long _Value); 318long _InterlockedExchangeAdd_nf(long volatile *_Addend, long _Value); 319long _InterlockedExchangeAdd_rel(long volatile *_Addend, long _Value); 320__int64 _InterlockedExchangeAdd64_acq(__int64 volatile *_Addend, __int64 _Value); 321__int64 _InterlockedExchangeAdd64_nf(__int64 volatile *_Addend, __int64 _Value); 322__int64 _InterlockedExchangeAdd64_rel(__int64 volatile *_Addend, __int64 _Value); 323#endif 324/*----------------------------------------------------------------------------*\ 325|* Interlocked Increment 326\*----------------------------------------------------------------------------*/ 327#if defined(__arm__) || defined(__aarch64__) 328short _InterlockedIncrement16_acq(short volatile *_Value); 329short _InterlockedIncrement16_nf(short volatile *_Value); 330short _InterlockedIncrement16_rel(short volatile *_Value); 331long _InterlockedIncrement_acq(long volatile *_Value); 332long _InterlockedIncrement_nf(long volatile *_Value); 333long _InterlockedIncrement_rel(long volatile *_Value); 334__int64 _InterlockedIncrement64_acq(__int64 volatile *_Value); 335__int64 _InterlockedIncrement64_nf(__int64 volatile *_Value); 336__int64 _InterlockedIncrement64_rel(__int64 volatile *_Value); 337#endif 338/*----------------------------------------------------------------------------*\ 339|* Interlocked Decrement 340\*----------------------------------------------------------------------------*/ 341#if defined(__arm__) || defined(__aarch64__) 342short _InterlockedDecrement16_acq(short volatile *_Value); 343short _InterlockedDecrement16_nf(short volatile *_Value); 344short _InterlockedDecrement16_rel(short volatile *_Value); 345long _InterlockedDecrement_acq(long volatile *_Value); 346long _InterlockedDecrement_nf(long volatile *_Value); 347long _InterlockedDecrement_rel(long volatile *_Value); 348__int64 _InterlockedDecrement64_acq(__int64 volatile *_Value); 349__int64 _InterlockedDecrement64_nf(__int64 volatile *_Value); 350__int64 _InterlockedDecrement64_rel(__int64 volatile *_Value); 351#endif 352/*----------------------------------------------------------------------------*\ 353|* Interlocked And 354\*----------------------------------------------------------------------------*/ 355#if defined(__arm__) || defined(__aarch64__) 356char _InterlockedAnd8_acq(char volatile *_Value, char _Mask); 357char _InterlockedAnd8_nf(char volatile *_Value, char _Mask); 358char _InterlockedAnd8_rel(char volatile *_Value, char _Mask); 359short _InterlockedAnd16_acq(short volatile *_Value, short _Mask); 360short _InterlockedAnd16_nf(short volatile *_Value, short _Mask); 361short _InterlockedAnd16_rel(short volatile *_Value, short _Mask); 362long _InterlockedAnd_acq(long volatile *_Value, long _Mask); 363long _InterlockedAnd_nf(long volatile *_Value, long _Mask); 364long _InterlockedAnd_rel(long volatile *_Value, long _Mask); 365__int64 _InterlockedAnd64_acq(__int64 volatile *_Value, __int64 _Mask); 366__int64 _InterlockedAnd64_nf(__int64 volatile *_Value, __int64 _Mask); 367__int64 _InterlockedAnd64_rel(__int64 volatile *_Value, __int64 _Mask); 368#endif 369/*----------------------------------------------------------------------------*\ 370|* Bit Counting and Testing 371\*----------------------------------------------------------------------------*/ 372#if defined(__arm__) || defined(__aarch64__) 373unsigned char _interlockedbittestandset_acq(long volatile *_BitBase, 374 long _BitPos); 375unsigned char _interlockedbittestandset_nf(long volatile *_BitBase, 376 long _BitPos); 377unsigned char _interlockedbittestandset_rel(long volatile *_BitBase, 378 long _BitPos); 379unsigned char _interlockedbittestandreset_acq(long volatile *_BitBase, 380 long _BitPos); 381unsigned char _interlockedbittestandreset_nf(long volatile *_BitBase, 382 long _BitPos); 383unsigned char _interlockedbittestandreset_rel(long volatile *_BitBase, 384 long _BitPos); 385#endif 386/*----------------------------------------------------------------------------*\ 387|* Interlocked Or 388\*----------------------------------------------------------------------------*/ 389#if defined(__arm__) || defined(__aarch64__) 390char _InterlockedOr8_acq(char volatile *_Value, char _Mask); 391char _InterlockedOr8_nf(char volatile *_Value, char _Mask); 392char _InterlockedOr8_rel(char volatile *_Value, char _Mask); 393short _InterlockedOr16_acq(short volatile *_Value, short _Mask); 394short _InterlockedOr16_nf(short volatile *_Value, short _Mask); 395short _InterlockedOr16_rel(short volatile *_Value, short _Mask); 396long _InterlockedOr_acq(long volatile *_Value, long _Mask); 397long _InterlockedOr_nf(long volatile *_Value, long _Mask); 398long _InterlockedOr_rel(long volatile *_Value, long _Mask); 399__int64 _InterlockedOr64_acq(__int64 volatile *_Value, __int64 _Mask); 400__int64 _InterlockedOr64_nf(__int64 volatile *_Value, __int64 _Mask); 401__int64 _InterlockedOr64_rel(__int64 volatile *_Value, __int64 _Mask); 402#endif 403/*----------------------------------------------------------------------------*\ 404|* Interlocked Xor 405\*----------------------------------------------------------------------------*/ 406#if defined(__arm__) || defined(__aarch64__) 407char _InterlockedXor8_acq(char volatile *_Value, char _Mask); 408char _InterlockedXor8_nf(char volatile *_Value, char _Mask); 409char _InterlockedXor8_rel(char volatile *_Value, char _Mask); 410short _InterlockedXor16_acq(short volatile *_Value, short _Mask); 411short _InterlockedXor16_nf(short volatile *_Value, short _Mask); 412short _InterlockedXor16_rel(short volatile *_Value, short _Mask); 413long _InterlockedXor_acq(long volatile *_Value, long _Mask); 414long _InterlockedXor_nf(long volatile *_Value, long _Mask); 415long _InterlockedXor_rel(long volatile *_Value, long _Mask); 416__int64 _InterlockedXor64_acq(__int64 volatile *_Value, __int64 _Mask); 417__int64 _InterlockedXor64_nf(__int64 volatile *_Value, __int64 _Mask); 418__int64 _InterlockedXor64_rel(__int64 volatile *_Value, __int64 _Mask); 419#endif 420/*----------------------------------------------------------------------------*\ 421|* Interlocked Exchange 422\*----------------------------------------------------------------------------*/ 423#if defined(__arm__) || defined(__aarch64__) 424char _InterlockedExchange8_acq(char volatile *_Target, char _Value); 425char _InterlockedExchange8_nf(char volatile *_Target, char _Value); 426char _InterlockedExchange8_rel(char volatile *_Target, char _Value); 427short _InterlockedExchange16_acq(short volatile *_Target, short _Value); 428short _InterlockedExchange16_nf(short volatile *_Target, short _Value); 429short _InterlockedExchange16_rel(short volatile *_Target, short _Value); 430long _InterlockedExchange_acq(long volatile *_Target, long _Value); 431long _InterlockedExchange_nf(long volatile *_Target, long _Value); 432long _InterlockedExchange_rel(long volatile *_Target, long _Value); 433__int64 _InterlockedExchange64_acq(__int64 volatile *_Target, __int64 _Value); 434__int64 _InterlockedExchange64_nf(__int64 volatile *_Target, __int64 _Value); 435__int64 _InterlockedExchange64_rel(__int64 volatile *_Target, __int64 _Value); 436#endif 437/*----------------------------------------------------------------------------*\ 438|* Interlocked Compare Exchange 439\*----------------------------------------------------------------------------*/ 440#if defined(__arm__) || defined(__aarch64__) 441char _InterlockedCompareExchange8_acq(char volatile *_Destination, 442 char _Exchange, char _Comparand); 443char _InterlockedCompareExchange8_nf(char volatile *_Destination, 444 char _Exchange, char _Comparand); 445char _InterlockedCompareExchange8_rel(char volatile *_Destination, 446 char _Exchange, char _Comparand); 447short _InterlockedCompareExchange16_acq(short volatile *_Destination, 448 short _Exchange, short _Comparand); 449short _InterlockedCompareExchange16_nf(short volatile *_Destination, 450 short _Exchange, short _Comparand); 451short _InterlockedCompareExchange16_rel(short volatile *_Destination, 452 short _Exchange, short _Comparand); 453long _InterlockedCompareExchange_acq(long volatile *_Destination, 454 long _Exchange, long _Comparand); 455long _InterlockedCompareExchange_nf(long volatile *_Destination, 456 long _Exchange, long _Comparand); 457long _InterlockedCompareExchange_rel(long volatile *_Destination, 458 long _Exchange, long _Comparand); 459__int64 _InterlockedCompareExchange64_acq(__int64 volatile *_Destination, 460 __int64 _Exchange, __int64 _Comparand); 461__int64 _InterlockedCompareExchange64_nf(__int64 volatile *_Destination, 462 __int64 _Exchange, __int64 _Comparand); 463__int64 _InterlockedCompareExchange64_rel(__int64 volatile *_Destination, 464 __int64 _Exchange, __int64 _Comparand); 465#endif 466 467/*----------------------------------------------------------------------------*\ 468|* movs, stos 469\*----------------------------------------------------------------------------*/ 470#if defined(__i386__) || defined(__x86_64__) 471static __inline__ void __DEFAULT_FN_ATTRS 472__movsb(unsigned char *__dst, unsigned char const *__src, size_t __n) { 473 __asm__ __volatile__("rep movsb" : "+D"(__dst), "+S"(__src), "+c"(__n) 474 : : "memory"); 475} 476static __inline__ void __DEFAULT_FN_ATTRS 477__movsd(unsigned long *__dst, unsigned long const *__src, size_t __n) { 478 __asm__ __volatile__("rep movsl" : "+D"(__dst), "+S"(__src), "+c"(__n) 479 : : "memory"); 480} 481static __inline__ void __DEFAULT_FN_ATTRS 482__movsw(unsigned short *__dst, unsigned short const *__src, size_t __n) { 483 __asm__ __volatile__("rep movsw" : "+D"(__dst), "+S"(__src), "+c"(__n) 484 : : "memory"); 485} 486static __inline__ void __DEFAULT_FN_ATTRS 487__stosd(unsigned long *__dst, unsigned long __x, size_t __n) { 488 __asm__ __volatile__("rep stosl" : "+D"(__dst), "+c"(__n) : "a"(__x) 489 : "memory"); 490} 491static __inline__ void __DEFAULT_FN_ATTRS 492__stosw(unsigned short *__dst, unsigned short __x, size_t __n) { 493 __asm__ __volatile__("rep stosw" : "+D"(__dst), "+c"(__n) : "a"(__x) 494 : "memory"); 495} 496#endif 497#ifdef __x86_64__ 498static __inline__ void __DEFAULT_FN_ATTRS 499__movsq(unsigned long long *__dst, unsigned long long const *__src, size_t __n) { 500 __asm__ __volatile__("rep movsq" : "+D"(__dst), "+S"(__src), "+c"(__n) 501 : : "memory"); 502} 503static __inline__ void __DEFAULT_FN_ATTRS 504__stosq(unsigned __int64 *__dst, unsigned __int64 __x, size_t __n) { 505 __asm__ __volatile__("rep stosq" : "+D"(__dst), "+c"(__n) : "a"(__x) 506 : "memory"); 507} 508#endif 509 510/*----------------------------------------------------------------------------*\ 511|* Misc 512\*----------------------------------------------------------------------------*/ 513#if defined(__i386__) || defined(__x86_64__) 514static __inline__ void __DEFAULT_FN_ATTRS 515__cpuid(int __info[4], int __level) { 516 __asm__ ("cpuid" : "=a"(__info[0]), "=b" (__info[1]), "=c"(__info[2]), "=d"(__info[3]) 517 : "a"(__level), "c"(0)); 518} 519static __inline__ void __DEFAULT_FN_ATTRS 520__cpuidex(int __info[4], int __level, int __ecx) { 521 __asm__ ("cpuid" : "=a"(__info[0]), "=b" (__info[1]), "=c"(__info[2]), "=d"(__info[3]) 522 : "a"(__level), "c"(__ecx)); 523} 524static __inline__ void __DEFAULT_FN_ATTRS 525__halt(void) { 526 __asm__ volatile ("hlt"); 527} 528#endif 529 530#if defined(__i386__) || defined(__x86_64__) || defined(__aarch64__) 531static __inline__ void __DEFAULT_FN_ATTRS 532__nop(void) { 533 __asm__ volatile ("nop"); 534} 535#endif 536 537/*----------------------------------------------------------------------------*\ 538|* MS AArch64 specific 539\*----------------------------------------------------------------------------*/ 540#if defined(__aarch64__) 541unsigned __int64 __getReg(int); 542long _InterlockedAdd(long volatile *Addend, long Value); 543__int64 _ReadStatusReg(int); 544void _WriteStatusReg(int, __int64); 545 546unsigned short __cdecl _byteswap_ushort(unsigned short val); 547unsigned long __cdecl _byteswap_ulong (unsigned long val); 548unsigned __int64 __cdecl _byteswap_uint64(unsigned __int64 val); 549#endif 550 551/*----------------------------------------------------------------------------*\ 552|* Privileged intrinsics 553\*----------------------------------------------------------------------------*/ 554#if defined(__i386__) || defined(__x86_64__) 555static __inline__ unsigned __int64 __DEFAULT_FN_ATTRS 556__readmsr(unsigned long __register) { 557 // Loads the contents of a 64-bit model specific register (MSR) specified in 558 // the ECX register into registers EDX:EAX. The EDX register is loaded with 559 // the high-order 32 bits of the MSR and the EAX register is loaded with the 560 // low-order 32 bits. If less than 64 bits are implemented in the MSR being 561 // read, the values returned to EDX:EAX in unimplemented bit locations are 562 // undefined. 563 unsigned long __edx; 564 unsigned long __eax; 565 __asm__ ("rdmsr" : "=d"(__edx), "=a"(__eax) : "c"(__register)); 566 return (((unsigned __int64)__edx) << 32) | (unsigned __int64)__eax; 567} 568 569static __inline__ unsigned long __DEFAULT_FN_ATTRS 570__readcr3(void) { 571 unsigned long __cr3_val; 572 __asm__ __volatile__ ("mov %%cr3, %0" : "=q"(__cr3_val) : : "memory"); 573 return __cr3_val; 574} 575 576static __inline__ void __DEFAULT_FN_ATTRS 577__writecr3(unsigned int __cr3_val) { 578 __asm__ ("mov %0, %%cr3" : : "q"(__cr3_val) : "memory"); 579} 580#endif 581 582#ifdef __cplusplus 583} 584#endif 585 586#undef __DEFAULT_FN_ATTRS 587 588#endif /* __INTRIN_H */ 589#endif /* _MSC_VER */ 590