avxintrin.h revision 309124
1/*===---- avxintrin.h - AVX intrinsics -------------------------------------=== 2 * 3 * Permission is hereby granted, free of charge, to any person obtaining a copy 4 * of this software and associated documentation files (the "Software"), to deal 5 * in the Software without restriction, including without limitation the rights 6 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 * copies of the Software, and to permit persons to whom the Software is 8 * furnished to do so, subject to the following conditions: 9 * 10 * The above copyright notice and this permission notice shall be included in 11 * all copies or substantial portions of the Software. 12 * 13 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 * THE SOFTWARE. 20 * 21 *===-----------------------------------------------------------------------=== 22 */ 23 24#ifndef __IMMINTRIN_H 25#error "Never use <avxintrin.h> directly; include <immintrin.h> instead." 26#endif 27 28#ifndef __AVXINTRIN_H 29#define __AVXINTRIN_H 30 31typedef double __v4df __attribute__ ((__vector_size__ (32))); 32typedef float __v8sf __attribute__ ((__vector_size__ (32))); 33typedef long long __v4di __attribute__ ((__vector_size__ (32))); 34typedef int __v8si __attribute__ ((__vector_size__ (32))); 35typedef short __v16hi __attribute__ ((__vector_size__ (32))); 36typedef char __v32qi __attribute__ ((__vector_size__ (32))); 37 38/* Unsigned types */ 39typedef unsigned long long __v4du __attribute__ ((__vector_size__ (32))); 40typedef unsigned int __v8su __attribute__ ((__vector_size__ (32))); 41typedef unsigned short __v16hu __attribute__ ((__vector_size__ (32))); 42typedef unsigned char __v32qu __attribute__ ((__vector_size__ (32))); 43 44/* We need an explicitly signed variant for char. Note that this shouldn't 45 * appear in the interface though. */ 46typedef signed char __v32qs __attribute__((__vector_size__(32))); 47 48typedef float __m256 __attribute__ ((__vector_size__ (32))); 49typedef double __m256d __attribute__((__vector_size__(32))); 50typedef long long __m256i __attribute__((__vector_size__(32))); 51 52/* Define the default attributes for the functions in this file. */ 53#define __DEFAULT_FN_ATTRS __attribute__((__always_inline__, __nodebug__, __target__("avx"))) 54 55/* Arithmetic */ 56/// \brief Adds two 256-bit vectors of [4 x double]. 57/// 58/// \headerfile <x86intrin.h> 59/// 60/// This intrinsic corresponds to the \c VADDPD / ADDPD instruction. 61/// 62/// \param __a 63/// A 256-bit vector of [4 x double] containing one of the source operands. 64/// \param __b 65/// A 256-bit vector of [4 x double] containing one of the source operands. 66/// \returns A 256-bit vector of [4 x double] containing the sums of both 67/// operands. 68static __inline __m256d __DEFAULT_FN_ATTRS 69_mm256_add_pd(__m256d __a, __m256d __b) 70{ 71 return (__m256d)((__v4df)__a+(__v4df)__b); 72} 73 74/// \brief Adds two 256-bit vectors of [8 x float]. 75/// 76/// \headerfile <x86intrin.h> 77/// 78/// This intrinsic corresponds to the \c VADDPS / ADDPS instruction. 79/// 80/// \param __a 81/// A 256-bit vector of [8 x float] containing one of the source operands. 82/// \param __b 83/// A 256-bit vector of [8 x float] containing one of the source operands. 84/// \returns A 256-bit vector of [8 x float] containing the sums of both 85/// operands. 86static __inline __m256 __DEFAULT_FN_ATTRS 87_mm256_add_ps(__m256 __a, __m256 __b) 88{ 89 return (__m256)((__v8sf)__a+(__v8sf)__b); 90} 91 92/// \brief Subtracts two 256-bit vectors of [4 x double]. 93/// 94/// \headerfile <x86intrin.h> 95/// 96/// This intrinsic corresponds to the \c VSUBPD / SUBPD instruction. 97/// 98/// \param __a 99/// A 256-bit vector of [4 x double] containing the minuend. 100/// \param __b 101/// A 256-bit vector of [4 x double] containing the subtrahend. 102/// \returns A 256-bit vector of [4 x double] containing the differences between 103/// both operands. 104static __inline __m256d __DEFAULT_FN_ATTRS 105_mm256_sub_pd(__m256d __a, __m256d __b) 106{ 107 return (__m256d)((__v4df)__a-(__v4df)__b); 108} 109 110/// \brief Subtracts two 256-bit vectors of [8 x float]. 111/// 112/// \headerfile <x86intrin.h> 113/// 114/// This intrinsic corresponds to the \c VSUBPS / SUBPS instruction. 115/// 116/// \param __a 117/// A 256-bit vector of [8 x float] containing the minuend. 118/// \param __b 119/// A 256-bit vector of [8 x float] containing the subtrahend. 120/// \returns A 256-bit vector of [8 x float] containing the differences between 121/// both operands. 122static __inline __m256 __DEFAULT_FN_ATTRS 123_mm256_sub_ps(__m256 __a, __m256 __b) 124{ 125 return (__m256)((__v8sf)__a-(__v8sf)__b); 126} 127 128/// \brief Adds the even-indexed values and subtracts the odd-indexed values of 129/// two 256-bit vectors of [4 x double]. 130/// 131/// \headerfile <x86intrin.h> 132/// 133/// This intrinsic corresponds to the \c VADDSUBPD / ADDSUBPD instruction. 134/// 135/// \param __a 136/// A 256-bit vector of [4 x double] containing the left source operand. 137/// \param __b 138/// A 256-bit vector of [4 x double] containing the right source operand. 139/// \returns A 256-bit vector of [4 x double] containing the alternating sums 140/// and differences between both operands. 141static __inline __m256d __DEFAULT_FN_ATTRS 142_mm256_addsub_pd(__m256d __a, __m256d __b) 143{ 144 return (__m256d)__builtin_ia32_addsubpd256((__v4df)__a, (__v4df)__b); 145} 146 147/// \brief Adds the even-indexed values and subtracts the odd-indexed values of 148/// two 256-bit vectors of [8 x float]. 149/// 150/// \headerfile <x86intrin.h> 151/// 152/// This intrinsic corresponds to the \c VADDSUBPS / ADDSUBPS instruction. 153/// 154/// \param __a 155/// A 256-bit vector of [8 x float] containing the left source operand. 156/// \param __b 157/// A 256-bit vector of [8 x float] containing the right source operand. 158/// \returns A 256-bit vector of [8 x float] containing the alternating sums and 159/// differences between both operands. 160static __inline __m256 __DEFAULT_FN_ATTRS 161_mm256_addsub_ps(__m256 __a, __m256 __b) 162{ 163 return (__m256)__builtin_ia32_addsubps256((__v8sf)__a, (__v8sf)__b); 164} 165 166/// \brief Divides two 256-bit vectors of [4 x double]. 167/// 168/// \headerfile <x86intrin.h> 169/// 170/// This intrinsic corresponds to the \c VDIVPD / DIVPD instruction. 171/// 172/// \param __a 173/// A 256-bit vector of [4 x double] containing the dividend. 174/// \param __b 175/// A 256-bit vector of [4 x double] containing the divisor. 176/// \returns A 256-bit vector of [4 x double] containing the quotients of both 177/// operands. 178static __inline __m256d __DEFAULT_FN_ATTRS 179_mm256_div_pd(__m256d __a, __m256d __b) 180{ 181 return (__m256d)((__v4df)__a/(__v4df)__b); 182} 183 184/// \brief Divides two 256-bit vectors of [8 x float]. 185/// 186/// \headerfile <x86intrin.h> 187/// 188/// This intrinsic corresponds to the \c VDIVPS / DIVPS instruction. 189/// 190/// \param __a 191/// A 256-bit vector of [8 x float] containing the dividend. 192/// \param __b 193/// A 256-bit vector of [8 x float] containing the divisor. 194/// \returns A 256-bit vector of [8 x float] containing the quotients of both 195/// operands. 196static __inline __m256 __DEFAULT_FN_ATTRS 197_mm256_div_ps(__m256 __a, __m256 __b) 198{ 199 return (__m256)((__v8sf)__a/(__v8sf)__b); 200} 201 202/// \brief Compares two 256-bit vectors of [4 x double] and returns the greater 203/// of each pair of values. 204/// 205/// \headerfile <x86intrin.h> 206/// 207/// This intrinsic corresponds to the \c VMAXPD / MAXPD instruction. 208/// 209/// \param __a 210/// A 256-bit vector of [4 x double] containing one of the operands. 211/// \param __b 212/// A 256-bit vector of [4 x double] containing one of the operands. 213/// \returns A 256-bit vector of [4 x double] containing the maximum values 214/// between both operands. 215static __inline __m256d __DEFAULT_FN_ATTRS 216_mm256_max_pd(__m256d __a, __m256d __b) 217{ 218 return (__m256d)__builtin_ia32_maxpd256((__v4df)__a, (__v4df)__b); 219} 220 221/// \brief Compares two 256-bit vectors of [8 x float] and returns the greater 222/// of each pair of values. 223/// 224/// \headerfile <x86intrin.h> 225/// 226/// This intrinsic corresponds to the \c VMAXPS / MAXPS instruction. 227/// 228/// \param __a 229/// A 256-bit vector of [8 x float] containing one of the operands. 230/// \param __b 231/// A 256-bit vector of [8 x float] containing one of the operands. 232/// \returns A 256-bit vector of [8 x float] containing the maximum values 233/// between both operands. 234static __inline __m256 __DEFAULT_FN_ATTRS 235_mm256_max_ps(__m256 __a, __m256 __b) 236{ 237 return (__m256)__builtin_ia32_maxps256((__v8sf)__a, (__v8sf)__b); 238} 239 240/// \brief Compares two 256-bit vectors of [4 x double] and returns the lesser 241/// of each pair of values. 242/// 243/// \headerfile <x86intrin.h> 244/// 245/// This intrinsic corresponds to the \c VMINPD / MINPD instruction. 246/// 247/// \param __a 248/// A 256-bit vector of [4 x double] containing one of the operands. 249/// \param __b 250/// A 256-bit vector of [4 x double] containing one of the operands. 251/// \returns A 256-bit vector of [4 x double] containing the minimum values 252/// between both operands. 253static __inline __m256d __DEFAULT_FN_ATTRS 254_mm256_min_pd(__m256d __a, __m256d __b) 255{ 256 return (__m256d)__builtin_ia32_minpd256((__v4df)__a, (__v4df)__b); 257} 258 259/// \brief Compares two 256-bit vectors of [8 x float] and returns the lesser 260/// of each pair of values. 261/// 262/// \headerfile <x86intrin.h> 263/// 264/// This intrinsic corresponds to the \c VMINPS / MINPS instruction. 265/// 266/// \param __a 267/// A 256-bit vector of [8 x float] containing one of the operands. 268/// \param __b 269/// A 256-bit vector of [8 x float] containing one of the operands. 270/// \returns A 256-bit vector of [8 x float] containing the minimum values 271/// between both operands. 272static __inline __m256 __DEFAULT_FN_ATTRS 273_mm256_min_ps(__m256 __a, __m256 __b) 274{ 275 return (__m256)__builtin_ia32_minps256((__v8sf)__a, (__v8sf)__b); 276} 277 278/// \brief Multiplies two 256-bit vectors of [4 x double]. 279/// 280/// \headerfile <x86intrin.h> 281/// 282/// This intrinsic corresponds to the \c VMULPD / MULPD instruction. 283/// 284/// \param __a 285/// A 256-bit vector of [4 x double] containing one of the operands. 286/// \param __b 287/// A 256-bit vector of [4 x double] containing one of the operands. 288/// \returns A 256-bit vector of [4 x double] containing the products of both 289/// operands. 290static __inline __m256d __DEFAULT_FN_ATTRS 291_mm256_mul_pd(__m256d __a, __m256d __b) 292{ 293 return (__m256d)((__v4df)__a * (__v4df)__b); 294} 295 296/// \brief Multiplies two 256-bit vectors of [8 x float]. 297/// 298/// \headerfile <x86intrin.h> 299/// 300/// This intrinsic corresponds to the \c VMULPS / MULPS instruction. 301/// 302/// \param __a 303/// A 256-bit vector of [8 x float] containing one of the operands. 304/// \param __b 305/// A 256-bit vector of [8 x float] containing one of the operands. 306/// \returns A 256-bit vector of [8 x float] containing the products of both 307/// operands. 308static __inline __m256 __DEFAULT_FN_ATTRS 309_mm256_mul_ps(__m256 __a, __m256 __b) 310{ 311 return (__m256)((__v8sf)__a * (__v8sf)__b); 312} 313 314/// \brief Calculates the square roots of the values in a 256-bit vector of 315/// [4 x double]. 316/// 317/// \headerfile <x86intrin.h> 318/// 319/// This intrinsic corresponds to the \c VSQRTPD / SQRTPD instruction. 320/// 321/// \param __a 322/// A 256-bit vector of [4 x double]. 323/// \returns A 256-bit vector of [4 x double] containing the square roots of the 324/// values in the operand. 325static __inline __m256d __DEFAULT_FN_ATTRS 326_mm256_sqrt_pd(__m256d __a) 327{ 328 return (__m256d)__builtin_ia32_sqrtpd256((__v4df)__a); 329} 330 331/// \brief Calculates the square roots of the values in a 256-bit vector of 332/// [8 x float]. 333/// 334/// \headerfile <x86intrin.h> 335/// 336/// This intrinsic corresponds to the \c VSQRTPS / SQRTPS instruction. 337/// 338/// \param __a 339/// A 256-bit vector of [8 x float]. 340/// \returns A 256-bit vector of [8 x float] containing the square roots of the 341/// values in the operand. 342static __inline __m256 __DEFAULT_FN_ATTRS 343_mm256_sqrt_ps(__m256 __a) 344{ 345 return (__m256)__builtin_ia32_sqrtps256((__v8sf)__a); 346} 347 348/// \brief Calculates the reciprocal square roots of the values in a 256-bit 349/// vector of [8 x float]. 350/// 351/// \headerfile <x86intrin.h> 352/// 353/// This intrinsic corresponds to the \c VRSQRTPS / RSQRTPS instruction. 354/// 355/// \param __a 356/// A 256-bit vector of [8 x float]. 357/// \returns A 256-bit vector of [8 x float] containing the reciprocal square 358/// roots of the values in the operand. 359static __inline __m256 __DEFAULT_FN_ATTRS 360_mm256_rsqrt_ps(__m256 __a) 361{ 362 return (__m256)__builtin_ia32_rsqrtps256((__v8sf)__a); 363} 364 365/// \brief Calculates the reciprocals of the values in a 256-bit vector of 366/// [8 x float]. 367/// 368/// \headerfile <x86intrin.h> 369/// 370/// This intrinsic corresponds to the \c VRCPPS / RCPPS instruction. 371/// 372/// \param __a 373/// A 256-bit vector of [8 x float]. 374/// \returns A 256-bit vector of [8 x float] containing the reciprocals of the 375/// values in the operand. 376static __inline __m256 __DEFAULT_FN_ATTRS 377_mm256_rcp_ps(__m256 __a) 378{ 379 return (__m256)__builtin_ia32_rcpps256((__v8sf)__a); 380} 381 382/// \brief Rounds the values in a 256-bit vector of [4 x double] as specified 383/// by the byte operand. The source values are rounded to integer values and 384/// returned as 64-bit double-precision floating-point values. 385/// 386/// \headerfile <x86intrin.h> 387/// 388/// \code 389/// __m256d _mm256_round_pd(__m256d V, const int M); 390/// \endcode 391/// 392/// This intrinsic corresponds to the \c VROUNDPD / ROUNDPD instruction. 393/// 394/// \param V 395/// A 256-bit vector of [4 x double]. 396/// \param M 397/// An integer value that specifies the rounding operation. 398/// Bits [7:4] are reserved. 399/// Bit [3] is a precision exception value: 400/// 0: A normal PE exception is used. 401/// 1: The PE field is not updated. 402/// Bit [2] is the rounding control source: 403/// 0: Use bits [1:0] of M. 404/// 1: Use the current MXCSR setting. 405/// Bits [1:0] contain the rounding control definition: 406/// 00: Nearest. 407/// 01: Downward (toward negative infinity). 408/// 10: Upward (toward positive infinity). 409/// 11: Truncated. 410/// \returns A 256-bit vector of [4 x double] containing the rounded values. 411#define _mm256_round_pd(V, M) __extension__ ({ \ 412 (__m256d)__builtin_ia32_roundpd256((__v4df)(__m256d)(V), (M)); }) 413 414/// \brief Rounds the values stored in a 256-bit vector of [8 x float] as 415/// specified by the byte operand. The source values are rounded to integer 416/// values and returned as floating-point values. 417/// 418/// \headerfile <x86intrin.h> 419/// 420/// \code 421/// __m256 _mm256_round_ps(__m256 V, const int M); 422/// \endcode 423/// 424/// This intrinsic corresponds to the \c VROUNDPS / ROUNDPS instruction. 425/// 426/// \param V 427/// A 256-bit vector of [8 x float]. 428/// \param M 429/// An integer value that specifies the rounding operation. 430/// Bits [7:4] are reserved. 431/// Bit [3] is a precision exception value: 432/// 0: A normal PE exception is used. 433/// 1: The PE field is not updated. 434/// Bit [2] is the rounding control source: 435/// 0: Use bits [1:0] of M. 436/// 1: Use the current MXCSR setting. 437/// Bits [1:0] contain the rounding control definition: 438/// 00: Nearest. 439/// 01: Downward (toward negative infinity). 440/// 10: Upward (toward positive infinity). 441/// 11: Truncated. 442/// \returns A 256-bit vector of [8 x float] containing the rounded values. 443#define _mm256_round_ps(V, M) __extension__ ({ \ 444 (__m256)__builtin_ia32_roundps256((__v8sf)(__m256)(V), (M)); }) 445 446/// \brief Rounds up the values stored in a 256-bit vector of [4 x double]. The 447/// source values are rounded up to integer values and returned as 64-bit 448/// double-precision floating-point values. 449/// 450/// \headerfile <x86intrin.h> 451/// 452/// \code 453/// __m256d _mm256_ceil_pd(__m256d V); 454/// \endcode 455/// 456/// This intrinsic corresponds to the \c VROUNDPD / ROUNDPD instruction. 457/// 458/// \param V 459/// A 256-bit vector of [4 x double]. 460/// \returns A 256-bit vector of [4 x double] containing the rounded up values. 461#define _mm256_ceil_pd(V) _mm256_round_pd((V), _MM_FROUND_CEIL) 462 463/// \brief Rounds down the values stored in a 256-bit vector of [4 x double]. 464/// The source values are rounded down to integer values and returned as 465/// 64-bit double-precision floating-point values. 466/// 467/// \headerfile <x86intrin.h> 468/// 469/// \code 470/// __m256d _mm256_floor_pd(__m256d V); 471/// \endcode 472/// 473/// This intrinsic corresponds to the \c VROUNDPD / ROUNDPD instruction. 474/// 475/// \param V 476/// A 256-bit vector of [4 x double]. 477/// \returns A 256-bit vector of [4 x double] containing the rounded down 478/// values. 479#define _mm256_floor_pd(V) _mm256_round_pd((V), _MM_FROUND_FLOOR) 480 481/// \brief Rounds up the values stored in a 256-bit vector of [8 x float]. The 482/// source values are rounded up to integer values and returned as 483/// floating-point values. 484/// 485/// \headerfile <x86intrin.h> 486/// 487/// \code 488/// __m256 _mm256_ceil_ps(__m256 V); 489/// \endcode 490/// 491/// This intrinsic corresponds to the \c VROUNDPS / ROUNDPS instruction. 492/// 493/// \param V 494/// A 256-bit vector of [8 x float]. 495/// \returns A 256-bit vector of [8 x float] containing the rounded up values. 496#define _mm256_ceil_ps(V) _mm256_round_ps((V), _MM_FROUND_CEIL) 497 498/// \brief Rounds down the values stored in a 256-bit vector of [8 x float]. The 499/// source values are rounded down to integer values and returned as 500/// floating-point values. 501/// 502/// \headerfile <x86intrin.h> 503/// 504/// \code 505/// __m256 _mm256_floor_ps(__m256 V); 506/// \endcode 507/// 508/// This intrinsic corresponds to the \c VROUNDPS / ROUNDPS instruction. 509/// 510/// \param V 511/// A 256-bit vector of [8 x float]. 512/// \returns A 256-bit vector of [8 x float] containing the rounded down values. 513#define _mm256_floor_ps(V) _mm256_round_ps((V), _MM_FROUND_FLOOR) 514 515/* Logical */ 516/// \brief Performs a bitwise AND of two 256-bit vectors of [4 x double]. 517/// 518/// \headerfile <x86intrin.h> 519/// 520/// This intrinsic corresponds to the \c VANDPD / ANDPD instruction. 521/// 522/// \param __a 523/// A 256-bit vector of [4 x double] containing one of the source operands. 524/// \param __b 525/// A 256-bit vector of [4 x double] containing one of the source operands. 526/// \returns A 256-bit vector of [4 x double] containing the bitwise AND of the 527/// values between both operands. 528static __inline __m256d __DEFAULT_FN_ATTRS 529_mm256_and_pd(__m256d __a, __m256d __b) 530{ 531 return (__m256d)((__v4du)__a & (__v4du)__b); 532} 533 534/// \brief Performs a bitwise AND of two 256-bit vectors of [8 x float]. 535/// 536/// \headerfile <x86intrin.h> 537/// 538/// This intrinsic corresponds to the \c VANDPS / ANDPS instruction. 539/// 540/// \param __a 541/// A 256-bit vector of [8 x float] containing one of the source operands. 542/// \param __b 543/// A 256-bit vector of [8 x float] containing one of the source operands. 544/// \returns A 256-bit vector of [8 x float] containing the bitwise AND of the 545/// values between both operands. 546static __inline __m256 __DEFAULT_FN_ATTRS 547_mm256_and_ps(__m256 __a, __m256 __b) 548{ 549 return (__m256)((__v8su)__a & (__v8su)__b); 550} 551 552/// \brief Performs a bitwise AND of two 256-bit vectors of [4 x double], using 553/// the one's complement of the values contained in the first source operand. 554/// 555/// \headerfile <x86intrin.h> 556/// 557/// This intrinsic corresponds to the \c VANDNPD / ANDNPD instruction. 558/// 559/// \param __a 560/// A 256-bit vector of [4 x double] containing the left source operand. The 561/// one's complement of this value is used in the bitwise AND. 562/// \param __b 563/// A 256-bit vector of [4 x double] containing the right source operand. 564/// \returns A 256-bit vector of [4 x double] containing the bitwise AND of the 565/// values of the second operand and the one's complement of the first 566/// operand. 567static __inline __m256d __DEFAULT_FN_ATTRS 568_mm256_andnot_pd(__m256d __a, __m256d __b) 569{ 570 return (__m256d)(~(__v4du)__a & (__v4du)__b); 571} 572 573/// \brief Performs a bitwise AND of two 256-bit vectors of [8 x float], using 574/// the one's complement of the values contained in the first source operand. 575/// 576/// \headerfile <x86intrin.h> 577/// 578/// This intrinsic corresponds to the \c VANDNPS / ANDNPS instruction. 579/// 580/// \param __a 581/// A 256-bit vector of [8 x float] containing the left source operand. The 582/// one's complement of this value is used in the bitwise AND. 583/// \param __b 584/// A 256-bit vector of [8 x float] containing the right source operand. 585/// \returns A 256-bit vector of [8 x float] containing the bitwise AND of the 586/// values of the second operand and the one's complement of the first 587/// operand. 588static __inline __m256 __DEFAULT_FN_ATTRS 589_mm256_andnot_ps(__m256 __a, __m256 __b) 590{ 591 return (__m256)(~(__v8su)__a & (__v8su)__b); 592} 593 594/// \brief Performs a bitwise OR of two 256-bit vectors of [4 x double]. 595/// 596/// \headerfile <x86intrin.h> 597/// 598/// This intrinsic corresponds to the \c VORPD / ORPD instruction. 599/// 600/// \param __a 601/// A 256-bit vector of [4 x double] containing one of the source operands. 602/// \param __b 603/// A 256-bit vector of [4 x double] containing one of the source operands. 604/// \returns A 256-bit vector of [4 x double] containing the bitwise OR of the 605/// values between both operands. 606static __inline __m256d __DEFAULT_FN_ATTRS 607_mm256_or_pd(__m256d __a, __m256d __b) 608{ 609 return (__m256d)((__v4du)__a | (__v4du)__b); 610} 611 612/// \brief Performs a bitwise OR of two 256-bit vectors of [8 x float]. 613/// 614/// \headerfile <x86intrin.h> 615/// 616/// This intrinsic corresponds to the \c VORPS / ORPS instruction. 617/// 618/// \param __a 619/// A 256-bit vector of [8 x float] containing one of the source operands. 620/// \param __b 621/// A 256-bit vector of [8 x float] containing one of the source operands. 622/// \returns A 256-bit vector of [8 x float] containing the bitwise OR of the 623/// values between both operands. 624static __inline __m256 __DEFAULT_FN_ATTRS 625_mm256_or_ps(__m256 __a, __m256 __b) 626{ 627 return (__m256)((__v8su)__a | (__v8su)__b); 628} 629 630/// \brief Performs a bitwise XOR of two 256-bit vectors of [4 x double]. 631/// 632/// \headerfile <x86intrin.h> 633/// 634/// This intrinsic corresponds to the \c VXORPD / XORPD instruction. 635/// 636/// \param __a 637/// A 256-bit vector of [4 x double] containing one of the source operands. 638/// \param __b 639/// A 256-bit vector of [4 x double] containing one of the source operands. 640/// \returns A 256-bit vector of [4 x double] containing the bitwise XOR of the 641/// values between both operands. 642static __inline __m256d __DEFAULT_FN_ATTRS 643_mm256_xor_pd(__m256d __a, __m256d __b) 644{ 645 return (__m256d)((__v4du)__a ^ (__v4du)__b); 646} 647 648/// \brief Performs a bitwise XOR of two 256-bit vectors of [8 x float]. 649/// 650/// \headerfile <x86intrin.h> 651/// 652/// This intrinsic corresponds to the \c VXORPS / XORPS instruction. 653/// 654/// \param __a 655/// A 256-bit vector of [8 x float] containing one of the source operands. 656/// \param __b 657/// A 256-bit vector of [8 x float] containing one of the source operands. 658/// \returns A 256-bit vector of [8 x float] containing the bitwise XOR of the 659/// values between both operands. 660static __inline __m256 __DEFAULT_FN_ATTRS 661_mm256_xor_ps(__m256 __a, __m256 __b) 662{ 663 return (__m256)((__v8su)__a ^ (__v8su)__b); 664} 665 666/* Horizontal arithmetic */ 667/// \brief Horizontally adds the adjacent pairs of values contained in two 668/// 256-bit vectors of [4 x double]. 669/// 670/// \headerfile <x86intrin.h> 671/// 672/// This intrinsic corresponds to the \c VHADDPD / HADDPD instruction. 673/// 674/// \param __a 675/// A 256-bit vector of [4 x double] containing one of the source operands. 676/// The horizontal sums of the values are returned in the even-indexed 677/// elements of a vector of [4 x double]. 678/// \param __b 679/// A 256-bit vector of [4 x double] containing one of the source operands. 680/// The horizontal sums of the values are returned in the odd-indexed 681/// elements of a vector of [4 x double]. 682/// \returns A 256-bit vector of [4 x double] containing the horizontal sums of 683/// both operands. 684static __inline __m256d __DEFAULT_FN_ATTRS 685_mm256_hadd_pd(__m256d __a, __m256d __b) 686{ 687 return (__m256d)__builtin_ia32_haddpd256((__v4df)__a, (__v4df)__b); 688} 689 690/// \brief Horizontally adds the adjacent pairs of values contained in two 691/// 256-bit vectors of [8 x float]. 692/// 693/// \headerfile <x86intrin.h> 694/// 695/// This intrinsic corresponds to the \c VHADDPS / HADDPS instruction. 696/// 697/// \param __a 698/// A 256-bit vector of [8 x float] containing one of the source operands. 699/// The horizontal sums of the values are returned in the elements with 700/// index 0, 1, 4, 5 of a vector of [8 x float]. 701/// \param __b 702/// A 256-bit vector of [8 x float] containing one of the source operands. 703/// The horizontal sums of the values are returned in the elements with 704/// index 2, 3, 6, 7 of a vector of [8 x float]. 705/// \returns A 256-bit vector of [8 x float] containing the horizontal sums of 706/// both operands. 707static __inline __m256 __DEFAULT_FN_ATTRS 708_mm256_hadd_ps(__m256 __a, __m256 __b) 709{ 710 return (__m256)__builtin_ia32_haddps256((__v8sf)__a, (__v8sf)__b); 711} 712 713/// \brief Horizontally subtracts the adjacent pairs of values contained in two 714/// 256-bit vectors of [4 x double]. 715/// 716/// \headerfile <x86intrin.h> 717/// 718/// This intrinsic corresponds to the \c VHSUBPD / HSUBPD instruction. 719/// 720/// \param __a 721/// A 256-bit vector of [4 x double] containing one of the source operands. 722/// The horizontal differences between the values are returned in the 723/// even-indexed elements of a vector of [4 x double]. 724/// \param __b 725/// A 256-bit vector of [4 x double] containing one of the source operands. 726/// The horizontal differences between the values are returned in the 727/// odd-indexed elements of a vector of [4 x double]. 728/// \returns A 256-bit vector of [4 x double] containing the horizontal 729/// differences of both operands. 730static __inline __m256d __DEFAULT_FN_ATTRS 731_mm256_hsub_pd(__m256d __a, __m256d __b) 732{ 733 return (__m256d)__builtin_ia32_hsubpd256((__v4df)__a, (__v4df)__b); 734} 735 736/// \brief Horizontally subtracts the adjacent pairs of values contained in two 737/// 256-bit vectors of [8 x float]. 738/// 739/// \headerfile <x86intrin.h> 740/// 741/// This intrinsic corresponds to the \c VHSUBPS / HSUBPS instruction. 742/// 743/// \param __a 744/// A 256-bit vector of [8 x float] containing one of the source operands. 745/// The horizontal differences between the values are returned in the 746/// elements with index 0, 1, 4, 5 of a vector of [8 x float]. 747/// \param __b 748/// A 256-bit vector of [8 x float] containing one of the source operands. 749/// The horizontal differences between the values are returned in the 750/// elements with index 2, 3, 6, 7 of a vector of [8 x float]. 751/// \returns A 256-bit vector of [8 x float] containing the horizontal 752/// differences of both operands. 753static __inline __m256 __DEFAULT_FN_ATTRS 754_mm256_hsub_ps(__m256 __a, __m256 __b) 755{ 756 return (__m256)__builtin_ia32_hsubps256((__v8sf)__a, (__v8sf)__b); 757} 758 759/* Vector permutations */ 760/// \brief Copies the values in a 128-bit vector of [2 x double] as specified 761/// by the 128-bit integer vector operand. 762/// 763/// \headerfile <x86intrin.h> 764/// 765/// This intrinsic corresponds to the \c VPERMILPD / PERMILPD instruction. 766/// 767/// \param __a 768/// A 128-bit vector of [2 x double]. 769/// \param __c 770/// A 128-bit integer vector operand specifying how the values are to be 771/// copied. 772/// Bit [1]: 773/// 0: Bits [63:0] of the source are copied to bits [63:0] of the 774/// returned vector. 775/// 1: Bits [127:64] of the source are copied to bits [63:0] of the 776/// returned vector. 777/// Bit [65]: 778/// 0: Bits [63:0] of the source are copied to bits [127:64] of the 779/// returned vector. 780/// 1: Bits [127:64] of the source are copied to bits [127:64] of the 781/// returned vector. 782/// \returns A 128-bit vector of [2 x double] containing the copied values. 783static __inline __m128d __DEFAULT_FN_ATTRS 784_mm_permutevar_pd(__m128d __a, __m128i __c) 785{ 786 return (__m128d)__builtin_ia32_vpermilvarpd((__v2df)__a, (__v2di)__c); 787} 788 789/// \brief Copies the values in a 256-bit vector of [4 x double] as 790/// specified by the 256-bit integer vector operand. 791/// 792/// \headerfile <x86intrin.h> 793/// 794/// This intrinsic corresponds to the \c VPERMILPD / PERMILPD instruction. 795/// 796/// \param __a 797/// A 256-bit vector of [4 x double]. 798/// \param __c 799/// A 256-bit integer vector operand specifying how the values are to be 800/// copied. 801/// Bit [1]: 802/// 0: Bits [63:0] of the source are copied to bits [63:0] of the 803/// returned vector. 804/// 1: Bits [127:64] of the source are copied to bits [63:0] of the 805/// returned vector. 806/// Bit [65]: 807/// 0: Bits [63:0] of the source are copied to bits [127:64] of the 808/// returned vector. 809/// 1: Bits [127:64] of the source are copied to bits [127:64] of the 810/// returned vector. 811/// Bit [129]: 812/// 0: Bits [191:128] of the source are copied to bits [191:128] of the 813/// returned vector. 814/// 1: Bits [255:192] of the source are copied to bits [191:128] of the 815/// returned vector. 816/// Bit [193]: 817/// 0: Bits [191:128] of the source are copied to bits [255:192] of the 818/// returned vector. 819/// 1: Bits [255:192] of the source are copied to bits [255:192] of the 820/// returned vector. 821/// \returns A 256-bit vector of [4 x double] containing the copied values. 822static __inline __m256d __DEFAULT_FN_ATTRS 823_mm256_permutevar_pd(__m256d __a, __m256i __c) 824{ 825 return (__m256d)__builtin_ia32_vpermilvarpd256((__v4df)__a, (__v4di)__c); 826} 827 828/// \brief Copies the values stored in a 128-bit vector of [4 x float] as 829/// specified by the 128-bit integer vector operand. 830/// 831/// \headerfile <x86intrin.h> 832/// 833/// This intrinsic corresponds to the \c VPERMILPS / PERMILPS instruction. 834/// 835/// \param __a 836/// A 128-bit vector of [4 x float]. 837/// \param __c 838/// A 128-bit integer vector operand specifying how the values are to be 839/// copied. 840/// Bits [1:0]: 841/// 00: Bits [31:0] of the source are copied to bits [31:0] of the 842/// returned vector. 843/// 01: Bits [63:32] of the source are copied to bits [31:0] of the 844/// returned vector. 845/// 10: Bits [95:64] of the source are copied to bits [31:0] of the 846/// returned vector. 847/// 11: Bits [127:96] of the source are copied to bits [31:0] of the 848/// returned vector. 849/// Bits [33:32]: 850/// 00: Bits [31:0] of the source are copied to bits [63:32] of the 851/// returned vector. 852/// 01: Bits [63:32] of the source are copied to bits [63:32] of the 853/// returned vector. 854/// 10: Bits [95:64] of the source are copied to bits [63:32] of the 855/// returned vector. 856/// 11: Bits [127:96] of the source are copied to bits [63:32] of the 857/// returned vector. 858/// Bits [65:64]: 859/// 00: Bits [31:0] of the source are copied to bits [95:64] of the 860/// returned vector. 861/// 01: Bits [63:32] of the source are copied to bits [95:64] of the 862/// returned vector. 863/// 10: Bits [95:64] of the source are copied to bits [95:64] of the 864/// returned vector. 865/// 11: Bits [127:96] of the source are copied to bits [95:64] of the 866/// returned vector. 867/// Bits [97:96]: 868/// 00: Bits [31:0] of the source are copied to bits [127:96] of the 869/// returned vector. 870/// 01: Bits [63:32] of the source are copied to bits [127:96] of the 871/// returned vector. 872/// 10: Bits [95:64] of the source are copied to bits [127:96] of the 873/// returned vector. 874/// 11: Bits [127:96] of the source are copied to bits [127:96] of the 875/// returned vector. 876/// \returns A 128-bit vector of [4 x float] containing the copied values. 877static __inline __m128 __DEFAULT_FN_ATTRS 878_mm_permutevar_ps(__m128 __a, __m128i __c) 879{ 880 return (__m128)__builtin_ia32_vpermilvarps((__v4sf)__a, (__v4si)__c); 881} 882 883/// \brief Copies the values stored in a 256-bit vector of [8 x float] as 884/// specified by the 256-bit integer vector operand. 885/// 886/// \headerfile <x86intrin.h> 887/// 888/// This intrinsic corresponds to the \c VPERMILPS / PERMILPS instruction. 889/// 890/// \param __a 891/// A 256-bit vector of [8 x float]. 892/// \param __c 893/// A 256-bit integer vector operand specifying how the values are to be 894/// copied. 895/// Bits [1:0]: 896/// 00: Bits [31:0] of the source are copied to bits [31:0] of the 897/// returned vector. 898/// 01: Bits [63:32] of the source are copied to bits [31:0] of the 899/// returned vector. 900/// 10: Bits [95:64] of the source are copied to bits [31:0] of the 901/// returned vector. 902/// 11: Bits [127:96] of the source are copied to bits [31:0] of the 903/// returned vector. 904/// Bits [33:32]: 905/// 00: Bits [31:0] of the source are copied to bits [63:32] of the 906/// returned vector. 907/// 01: Bits [63:32] of the source are copied to bits [63:32] of the 908/// returned vector. 909/// 10: Bits [95:64] of the source are copied to bits [63:32] of the 910/// returned vector. 911/// 11: Bits [127:96] of the source are copied to bits [63:32] of the 912/// returned vector. 913/// Bits [65:64]: 914/// 00: Bits [31:0] of the source are copied to bits [95:64] of the 915/// returned vector. 916/// 01: Bits [63:32] of the source are copied to bits [95:64] of the 917/// returned vector. 918/// 10: Bits [95:64] of the source are copied to bits [95:64] of the 919/// returned vector. 920/// 11: Bits [127:96] of the source are copied to bits [95:64] of the 921/// returned vector. 922/// Bits [97:96]: 923/// 00: Bits [31:0] of the source are copied to bits [127:96] of the 924/// returned vector. 925/// 01: Bits [63:32] of the source are copied to bits [127:96] of the 926/// returned vector. 927/// 10: Bits [95:64] of the source are copied to bits [127:96] of the 928/// returned vector. 929/// 11: Bits [127:96] of the source are copied to bits [127:96] of the 930/// returned vector. 931/// Bits [129:128]: 932/// 00: Bits [159:128] of the source are copied to bits [159:128] of the 933/// returned vector. 934/// 01: Bits [191:160] of the source are copied to bits [159:128] of the 935/// returned vector. 936/// 10: Bits [223:192] of the source are copied to bits [159:128] of the 937/// returned vector. 938/// 11: Bits [255:224] of the source are copied to bits [159:128] of the 939/// returned vector. 940/// Bits [161:160]: 941/// 00: Bits [159:128] of the source are copied to bits [191:160] of the 942/// returned vector. 943/// 01: Bits [191:160] of the source are copied to bits [191:160] of the 944/// returned vector. 945/// 10: Bits [223:192] of the source are copied to bits [191:160] of the 946/// returned vector. 947/// 11: Bits [255:224] of the source are copied to bits [191:160] of the 948/// returned vector. 949/// Bits [193:192]: 950/// 00: Bits [159:128] of the source are copied to bits [223:192] of the 951/// returned vector. 952/// 01: Bits [191:160] of the source are copied to bits [223:192] of the 953/// returned vector. 954/// 10: Bits [223:192] of the source are copied to bits [223:192] of the 955/// returned vector. 956/// 11: Bits [255:224] of the source are copied to bits [223:192] of the 957/// returned vector. 958/// Bits [225:224]: 959/// 00: Bits [159:128] of the source are copied to bits [255:224] of the 960/// returned vector. 961/// 01: Bits [191:160] of the source are copied to bits [255:224] of the 962/// returned vector. 963/// 10: Bits [223:192] of the source are copied to bits [255:224] of the 964/// returned vector. 965/// 11: Bits [255:224] of the source are copied to bits [255:224] of the 966/// returned vector. 967/// \returns A 256-bit vector of [8 x float] containing the copied values. 968static __inline __m256 __DEFAULT_FN_ATTRS 969_mm256_permutevar_ps(__m256 __a, __m256i __c) 970{ 971 return (__m256)__builtin_ia32_vpermilvarps256((__v8sf)__a, (__v8si)__c); 972} 973 974/// \brief Copies the values in a 128-bit vector of [2 x double] as 975/// specified by the immediate integer operand. 976/// 977/// \headerfile <x86intrin.h> 978/// 979/// \code 980/// __m128d _mm_permute_pd(__m128d A, const int C); 981/// \endcode 982/// 983/// This intrinsic corresponds to the \c VPERMILPD / PERMILPD instruction. 984/// 985/// \param A 986/// A 128-bit vector of [2 x double]. 987/// \param C 988/// An immediate integer operand specifying how the values are to be copied. 989/// Bit [0]: 990/// 0: Bits [63:0] of the source are copied to bits [63:0] of the 991/// returned vector. 992/// 1: Bits [127:64] of the source are copied to bits [63:0] of the 993/// returned vector. 994/// Bit [1]: 995/// 0: Bits [63:0] of the source are copied to bits [127:64] of the 996/// returned vector. 997/// 1: Bits [127:64] of the source are copied to bits [127:64] of the 998/// returned vector. 999/// \returns A 128-bit vector of [2 x double] containing the copied values. 1000#define _mm_permute_pd(A, C) __extension__ ({ \ 1001 (__m128d)__builtin_shufflevector((__v2df)(__m128d)(A), \ 1002 (__v2df)_mm_undefined_pd(), \ 1003 ((C) >> 0) & 0x1, ((C) >> 1) & 0x1); }) 1004 1005/// \brief Copies the values in a 256-bit vector of [4 x double] as 1006/// specified by the immediate integer operand. 1007/// 1008/// \headerfile <x86intrin.h> 1009/// 1010/// \code 1011/// __m256d _mm256_permute_pd(__m256d A, const int C); 1012/// \endcode 1013/// 1014/// This intrinsic corresponds to the \c VPERMILPD / PERMILPD instruction. 1015/// 1016/// \param A 1017/// A 256-bit vector of [4 x double]. 1018/// \param C 1019/// An immediate integer operand specifying how the values are to be copied. 1020/// Bit [0]: 1021/// 0: Bits [63:0] of the source are copied to bits [63:0] of the 1022/// returned vector. 1023/// 1: Bits [127:64] of the source are copied to bits [63:0] of the 1024/// returned vector. 1025/// Bit [1]: 1026/// 0: Bits [63:0] of the source are copied to bits [127:64] of the 1027/// returned vector. 1028/// 1: Bits [127:64] of the source are copied to bits [127:64] of the 1029/// returned vector. 1030/// Bit [2]: 1031/// 0: Bits [191:128] of the source are copied to bits [191:128] of the 1032/// returned vector. 1033/// 1: Bits [255:192] of the source are copied to bits [191:128] of the 1034/// returned vector. 1035/// Bit [3]: 1036/// 0: Bits [191:128] of the source are copied to bits [255:192] of the 1037/// returned vector. 1038/// 1: Bits [255:192] of the source are copied to bits [255:192] of the 1039/// returned vector. 1040/// \returns A 256-bit vector of [4 x double] containing the copied values. 1041#define _mm256_permute_pd(A, C) __extension__ ({ \ 1042 (__m256d)__builtin_shufflevector((__v4df)(__m256d)(A), \ 1043 (__v4df)_mm256_undefined_pd(), \ 1044 0 + (((C) >> 0) & 0x1), \ 1045 0 + (((C) >> 1) & 0x1), \ 1046 2 + (((C) >> 2) & 0x1), \ 1047 2 + (((C) >> 3) & 0x1)); }) 1048 1049/// \brief Copies the values in a 128-bit vector of [4 x float] as 1050/// specified by the immediate integer operand. 1051/// 1052/// \headerfile <x86intrin.h> 1053/// 1054/// \code 1055/// __m128 _mm_permute_ps(__m128 A, const int C); 1056/// \endcode 1057/// 1058/// This intrinsic corresponds to the \c VPERMILPS / PERMILPS instruction. 1059/// 1060/// \param A 1061/// A 128-bit vector of [4 x float]. 1062/// \param C 1063/// An immediate integer operand specifying how the values are to be copied. 1064/// Bits [1:0]: 1065/// 00: Bits [31:0] of the source are copied to bits [31:0] of the 1066/// returned vector. 1067/// 01: Bits [63:32] of the source are copied to bits [31:0] of the 1068/// returned vector. 1069/// 10: Bits [95:64] of the source are copied to bits [31:0] of the 1070/// returned vector. 1071/// 11: Bits [127:96] of the source are copied to bits [31:0] of the 1072/// returned vector. 1073/// Bits [3:2]: 1074/// 00: Bits [31:0] of the source are copied to bits [63:32] of the 1075/// returned vector. 1076/// 01: Bits [63:32] of the source are copied to bits [63:32] of the 1077/// returned vector. 1078/// 10: Bits [95:64] of the source are copied to bits [63:32] of the 1079/// returned vector. 1080/// 11: Bits [127:96] of the source are copied to bits [63:32] of the 1081/// returned vector. 1082/// Bits [5:4]: 1083/// 00: Bits [31:0] of the source are copied to bits [95:64] of the 1084/// returned vector. 1085/// 01: Bits [63:32] of the source are copied to bits [95:64] of the 1086/// returned vector. 1087/// 10: Bits [95:64] of the source are copied to bits [95:64] of the 1088/// returned vector. 1089/// 11: Bits [127:96] of the source are copied to bits [95:64] of the 1090/// returned vector. 1091/// Bits [7:6]: 1092/// 00: Bits [31:0] of the source are copied to bits [127:96] of the 1093/// returned vector. 1094/// 01: Bits [63:32] of the source are copied to bits [127:96] of the 1095/// returned vector. 1096/// 10: Bits [95:64] of the source are copied to bits [127:96] of the 1097/// returned vector. 1098/// 11: Bits [127:96] of the source are copied to bits [127:96] of the 1099/// returned vector. 1100/// \returns A 128-bit vector of [4 x float] containing the copied values. 1101#define _mm_permute_ps(A, C) __extension__ ({ \ 1102 (__m128)__builtin_shufflevector((__v4sf)(__m128)(A), \ 1103 (__v4sf)_mm_undefined_ps(), \ 1104 ((C) >> 0) & 0x3, ((C) >> 2) & 0x3, \ 1105 ((C) >> 4) & 0x3, ((C) >> 6) & 0x3); }) 1106 1107/// \brief Copies the values in a 256-bit vector of [8 x float] as 1108/// specified by the immediate integer operand. 1109/// 1110/// \headerfile <x86intrin.h> 1111/// 1112/// \code 1113/// __m256 _mm256_permute_ps(__m256 A, const int C); 1114/// \endcode 1115/// 1116/// This intrinsic corresponds to the \c VPERMILPS / PERMILPS instruction. 1117/// 1118/// \param A 1119/// A 256-bit vector of [8 x float]. 1120/// \param C 1121/// An immediate integer operand specifying how the values are to be copied. 1122/// Bits [1:0]: 1123/// 00: Bits [31:0] of the source are copied to bits [31:0] of the 1124/// returned vector. 1125/// 01: Bits [63:32] of the source are copied to bits [31:0] of the 1126/// returned vector. 1127/// 10: Bits [95:64] of the source are copied to bits [31:0] of the 1128/// returned vector. 1129/// 11: Bits [127:96] of the source are copied to bits [31:0] of the 1130/// returned vector. 1131/// Bits [3:2]: 1132/// 00: Bits [31:0] of the source are copied to bits [63:32] of the 1133/// returned vector. 1134/// 01: Bits [63:32] of the source are copied to bits [63:32] of the 1135/// returned vector. 1136/// 10: Bits [95:64] of the source are copied to bits [63:32] of the 1137/// returned vector. 1138/// 11: Bits [127:96] of the source are copied to bits [63:32] of the 1139/// returned vector. 1140/// Bits [5:4]: 1141/// 00: Bits [31:0] of the source are copied to bits [95:64] of the 1142/// returned vector. 1143/// 01: Bits [63:32] of the source are copied to bits [95:64] of the 1144/// returned vector. 1145/// 10: Bits [95:64] of the source are copied to bits [95:64] of the 1146/// returned vector. 1147/// 11: Bits [127:96] of the source are copied to bits [95:64] of the 1148/// returned vector. 1149/// Bits [7:6]: 1150/// 00: Bits [31:0] of the source are copied to bits [127:96] of the 1151/// returned vector. 1152/// 01: Bits [63:32] of the source are copied to bits [127:96] of the 1153/// returned vector. 1154/// 10: Bits [95:64] of the source are copied to bits [127:96] of the 1155/// returned vector. 1156/// 11: Bits [127:96] of the source are copied to bits [127:96] of the 1157/// returned vector. 1158/// Bits [1:0]: 1159/// 00: Bits [159:128] of the source are copied to bits [159:128] of the 1160/// returned vector. 1161/// 01: Bits [191:160] of the source are copied to bits [159:128] of the 1162/// returned vector. 1163/// 10: Bits [223:192] of the source are copied to bits [159:128] of the 1164/// returned vector. 1165/// 11: Bits [255:224] of the source are copied to bits [159:128] of the 1166/// returned vector. 1167/// Bits [3:2]: 1168/// 00: Bits [159:128] of the source are copied to bits [191:160] of the 1169/// returned vector. 1170/// 01: Bits [191:160] of the source are copied to bits [191:160] of the 1171/// returned vector. 1172/// 10: Bits [223:192] of the source are copied to bits [191:160] of the 1173/// returned vector. 1174/// 11: Bits [255:224] of the source are copied to bits [191:160] of the 1175/// returned vector. 1176/// Bits [5:4]: 1177/// 00: Bits [159:128] of the source are copied to bits [223:192] of the 1178/// returned vector. 1179/// 01: Bits [191:160] of the source are copied to bits [223:192] of the 1180/// returned vector. 1181/// 10: Bits [223:192] of the source are copied to bits [223:192] of the 1182/// returned vector. 1183/// 11: Bits [255:224] of the source are copied to bits [223:192] of the 1184/// returned vector. 1185/// Bits [7:6]: 1186/// 00: Bits [159:128] of the source are copied to bits [255:224] of the 1187/// returned vector. 1188/// 01: Bits [191:160] of the source are copied to bits [255:224] of the 1189/// returned vector. 1190/// 10: Bits [223:192] of the source are copied to bits [255:224] of the 1191/// returned vector. 1192/// 11: Bits [255:224] of the source are copied to bits [255:224] of the 1193/// returned vector. 1194/// \returns A 256-bit vector of [8 x float] containing the copied values. 1195#define _mm256_permute_ps(A, C) __extension__ ({ \ 1196 (__m256)__builtin_shufflevector((__v8sf)(__m256)(A), \ 1197 (__v8sf)_mm256_undefined_ps(), \ 1198 0 + (((C) >> 0) & 0x3), \ 1199 0 + (((C) >> 2) & 0x3), \ 1200 0 + (((C) >> 4) & 0x3), \ 1201 0 + (((C) >> 6) & 0x3), \ 1202 4 + (((C) >> 0) & 0x3), \ 1203 4 + (((C) >> 2) & 0x3), \ 1204 4 + (((C) >> 4) & 0x3), \ 1205 4 + (((C) >> 6) & 0x3)); }) 1206 1207/// \brief Permutes 128-bit data values stored in two 256-bit vectors of 1208/// [4 x double], as specified by the immediate integer operand. 1209/// 1210/// \headerfile <x86intrin.h> 1211/// 1212/// \code 1213/// __m256d _mm256_permute2f128_pd(__m256d V1, __m256d V2, const int M); 1214/// \endcode 1215/// 1216/// This intrinsic corresponds to the \c VPERM2F128 / PERM2F128 instruction. 1217/// 1218/// \param V1 1219/// A 256-bit vector of [4 x double]. 1220/// \param V2 1221/// A 256-bit vector of [4 x double. 1222/// \param M 1223/// An immediate integer operand specifying how the values are to be 1224/// permuted. 1225/// Bits [1:0]: 1226/// 00: Bits [127:0] of operand V1 are copied to bits [127:0] of the 1227/// destination. 1228/// 01: Bits [255:128] of operand V1 are copied to bits [127:0] of the 1229/// destination. 1230/// 10: Bits [127:0] of operand V2 are copied to bits [127:0] of the 1231/// destination. 1232/// 11: Bits [255:128] of operand V2 are copied to bits [127:0] of the 1233/// destination. 1234/// Bits [5:4]: 1235/// 00: Bits [127:0] of operand V1 are copied to bits [255:128] of the 1236/// destination. 1237/// 01: Bits [255:128] of operand V1 are copied to bits [255:128] of the 1238/// destination. 1239/// 10: Bits [127:0] of operand V2 are copied to bits [255:128] of the 1240/// destination. 1241/// 11: Bits [255:128] of operand V2 are copied to bits [255:128] of the 1242/// destination. 1243/// \returns A 256-bit vector of [4 x double] containing the copied values. 1244#define _mm256_permute2f128_pd(V1, V2, M) __extension__ ({ \ 1245 (__m256d)__builtin_ia32_vperm2f128_pd256((__v4df)(__m256d)(V1), \ 1246 (__v4df)(__m256d)(V2), (M)); }) 1247 1248/// \brief Permutes 128-bit data values stored in two 256-bit vectors of 1249/// [8 x float], as specified by the immediate integer operand. 1250/// 1251/// \headerfile <x86intrin.h> 1252/// 1253/// \code 1254/// __m256 _mm256_permute2f128_ps(__m256 V1, __m256 V2, const int M); 1255/// \endcode 1256/// 1257/// This intrinsic corresponds to the \c VPERM2F128 / PERM2F128 instruction. 1258/// 1259/// \param V1 1260/// A 256-bit vector of [8 x float]. 1261/// \param V2 1262/// A 256-bit vector of [8 x float]. 1263/// \param M 1264/// An immediate integer operand specifying how the values are to be 1265/// permuted. 1266/// Bits [1:0]: 1267/// 00: Bits [127:0] of operand V1 are copied to bits [127:0] of the 1268/// destination. 1269/// 01: Bits [255:128] of operand V1 are copied to bits [127:0] of the 1270/// destination. 1271/// 10: Bits [127:0] of operand V2 are copied to bits [127:0] of the 1272/// destination. 1273/// 11: Bits [255:128] of operand V2 are copied to bits [127:0] of the 1274/// destination. 1275/// Bits [5:4]: 1276/// 00: Bits [127:0] of operand V1 are copied to bits [255:128] of the 1277/// destination. 1278/// 01: Bits [255:128] of operand V1 are copied to bits [255:128] of the 1279/// destination. 1280/// 10: Bits [127:0] of operand V2 are copied to bits [255:128] of the 1281/// destination. 1282/// 11: Bits [255:128] of operand V2 are copied to bits [255:128] of the 1283/// destination. 1284/// \returns A 256-bit vector of [8 x float] containing the copied values. 1285#define _mm256_permute2f128_ps(V1, V2, M) __extension__ ({ \ 1286 (__m256)__builtin_ia32_vperm2f128_ps256((__v8sf)(__m256)(V1), \ 1287 (__v8sf)(__m256)(V2), (M)); }) 1288 1289/// \brief Permutes 128-bit data values stored in two 256-bit integer vectors, 1290/// as specified by the immediate integer operand. 1291/// 1292/// \headerfile <x86intrin.h> 1293/// 1294/// \code 1295/// __m256i _mm256_permute2f128_si256(__m256i V1, __m256i V2, const int M); 1296/// \endcode 1297/// 1298/// This intrinsic corresponds to the \c VPERM2F128 / PERM2F128 instruction. 1299/// 1300/// \param V1 1301/// A 256-bit integer vector. 1302/// \param V2 1303/// A 256-bit integer vector. 1304/// \param M 1305/// An immediate integer operand specifying how the values are to be copied. 1306/// Bits [1:0]: 1307/// 00: Bits [127:0] of operand V1 are copied to bits [127:0] of the 1308/// destination. 1309/// 01: Bits [255:128] of operand V1 are copied to bits [127:0] of the 1310/// destination. 1311/// 10: Bits [127:0] of operand V2 are copied to bits [127:0] of the 1312/// destination. 1313/// 11: Bits [255:128] of operand V2 are copied to bits [127:0] of the 1314/// destination. 1315/// Bits [5:4]: 1316/// 00: Bits [127:0] of operand V1 are copied to bits [255:128] of the 1317/// destination. 1318/// 01: Bits [255:128] of operand V1 are copied to bits [255:128] of the 1319/// destination. 1320/// 10: Bits [127:0] of operand V2 are copied to bits [255:128] of the 1321/// destination. 1322/// 11: Bits [255:128] of operand V2 are copied to bits [255:128] of the 1323/// destination. 1324/// \returns A 256-bit integer vector containing the copied values. 1325#define _mm256_permute2f128_si256(V1, V2, M) __extension__ ({ \ 1326 (__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(V1), \ 1327 (__v8si)(__m256i)(V2), (M)); }) 1328 1329/* Vector Blend */ 1330/// \brief Merges 64-bit double-precision data values stored in either of the 1331/// two 256-bit vectors of [4 x double], as specified by the immediate 1332/// integer operand. 1333/// 1334/// \headerfile <x86intrin.h> 1335/// 1336/// \code 1337/// __m256d _mm256_blend_pd(__m256d V1, __m256d V2, const int M); 1338/// \endcode 1339/// 1340/// This intrinsic corresponds to the \c VBLENDPD / BLENDPD instruction. 1341/// 1342/// \param V1 1343/// A 256-bit vector of [4 x double]. 1344/// \param V2 1345/// A 256-bit vector of [4 x double]. 1346/// \param M 1347/// An immediate integer operand, with mask bits [3:0] specifying how the 1348/// values are to be copied. The position of the mask bit corresponds to the 1349/// index of a copied value. When a mask bit is 0, the corresponding 64-bit 1350/// element in operand V1 is copied to the same position in the destination. 1351/// When a mask bit is 1, the corresponding 64-bit element in operand V2 is 1352/// copied to the same position in the destination. 1353/// \returns A 256-bit vector of [4 x double] containing the copied values. 1354#define _mm256_blend_pd(V1, V2, M) __extension__ ({ \ 1355 (__m256d)__builtin_shufflevector((__v4df)(__m256d)(V1), \ 1356 (__v4df)(__m256d)(V2), \ 1357 (((M) & 0x01) ? 4 : 0), \ 1358 (((M) & 0x02) ? 5 : 1), \ 1359 (((M) & 0x04) ? 6 : 2), \ 1360 (((M) & 0x08) ? 7 : 3)); }) 1361 1362/// \brief Merges 32-bit single-precision data values stored in either of the 1363/// two 256-bit vectors of [8 x float], as specified by the immediate 1364/// integer operand. 1365/// 1366/// \headerfile <x86intrin.h> 1367/// 1368/// \code 1369/// __m256 _mm256_blend_ps(__m256 V1, __m256 V2, const int M); 1370/// \endcode 1371/// 1372/// This intrinsic corresponds to the \c VBLENDPS / BLENDPS instruction. 1373/// 1374/// \param V1 1375/// A 256-bit vector of [8 x float]. 1376/// \param V2 1377/// A 256-bit vector of [8 x float]. 1378/// \param M 1379/// An immediate integer operand, with mask bits [7:0] specifying how the 1380/// values are to be copied. The position of the mask bit corresponds to the 1381/// index of a copied value. When a mask bit is 0, the corresponding 32-bit 1382/// element in operand V1 is copied to the same position in the destination. 1383/// When a mask bit is 1, the corresponding 32-bit element in operand V2 is 1384/// copied to the same position in the destination. 1385/// \returns A 256-bit vector of [8 x float] containing the copied values. 1386#define _mm256_blend_ps(V1, V2, M) __extension__ ({ \ 1387 (__m256)__builtin_shufflevector((__v8sf)(__m256)(V1), \ 1388 (__v8sf)(__m256)(V2), \ 1389 (((M) & 0x01) ? 8 : 0), \ 1390 (((M) & 0x02) ? 9 : 1), \ 1391 (((M) & 0x04) ? 10 : 2), \ 1392 (((M) & 0x08) ? 11 : 3), \ 1393 (((M) & 0x10) ? 12 : 4), \ 1394 (((M) & 0x20) ? 13 : 5), \ 1395 (((M) & 0x40) ? 14 : 6), \ 1396 (((M) & 0x80) ? 15 : 7)); }) 1397 1398/// \brief Merges 64-bit double-precision data values stored in either of the 1399/// two 256-bit vectors of [4 x double], as specified by the 256-bit vector 1400/// operand. 1401/// 1402/// \headerfile <x86intrin.h> 1403/// 1404/// This intrinsic corresponds to the \c VBLENDVPD / BLENDVPD instruction. 1405/// 1406/// \param __a 1407/// A 256-bit vector of [4 x double]. 1408/// \param __b 1409/// A 256-bit vector of [4 x double]. 1410/// \param __c 1411/// A 256-bit vector operand, with mask bits 255, 191, 127, and 63 specifying 1412/// how the values are to be copied. The position of the mask bit corresponds 1413/// to the most significant bit of a copied value. When a mask bit is 0, the 1414/// corresponding 64-bit element in operand __a is copied to the same 1415/// position in the destination. When a mask bit is 1, the corresponding 1416/// 64-bit element in operand __b is copied to the same position in the 1417/// destination. 1418/// \returns A 256-bit vector of [4 x double] containing the copied values. 1419static __inline __m256d __DEFAULT_FN_ATTRS 1420_mm256_blendv_pd(__m256d __a, __m256d __b, __m256d __c) 1421{ 1422 return (__m256d)__builtin_ia32_blendvpd256( 1423 (__v4df)__a, (__v4df)__b, (__v4df)__c); 1424} 1425 1426/// \brief Merges 32-bit single-precision data values stored in either of the 1427/// two 256-bit vectors of [8 x float], as specified by the 256-bit vector 1428/// operand. 1429/// 1430/// \headerfile <x86intrin.h> 1431/// 1432/// This intrinsic corresponds to the \c VBLENDVPS / BLENDVPS instruction. 1433/// 1434/// \param __a 1435/// A 256-bit vector of [8 x float]. 1436/// \param __b 1437/// A 256-bit vector of [8 x float]. 1438/// \param __c 1439/// A 256-bit vector operand, with mask bits 255, 223, 191, 159, 127, 95, 63, 1440/// and 31 specifying how the values are to be copied. The position of the 1441/// mask bit corresponds to the most significant bit of a copied value. When 1442/// a mask bit is 0, the corresponding 32-bit element in operand __a is 1443/// copied to the same position in the destination. When a mask bit is 1, the 1444/// corresponding 32-bit element in operand __b is copied to the same 1445/// position in the destination. 1446/// \returns A 256-bit vector of [8 x float] containing the copied values. 1447static __inline __m256 __DEFAULT_FN_ATTRS 1448_mm256_blendv_ps(__m256 __a, __m256 __b, __m256 __c) 1449{ 1450 return (__m256)__builtin_ia32_blendvps256( 1451 (__v8sf)__a, (__v8sf)__b, (__v8sf)__c); 1452} 1453 1454/* Vector Dot Product */ 1455/// \brief Computes two dot products in parallel, using the lower and upper 1456/// halves of two [8 x float] vectors as input to the two computations, and 1457/// returning the two dot products in the lower and upper halves of the 1458/// [8 x float] result. The immediate integer operand controls which 1459/// input elements will contribute to the dot product, and where the final 1460/// results are returned. In general, for each dot product, the four 1461/// corresponding elements of the input vectors are multiplied; the first 1462/// two and second two products are summed, then the two sums are added to 1463/// form the final result. 1464/// 1465/// \headerfile <x86intrin.h> 1466/// 1467/// \code 1468/// __m256 _mm256_dp_ps(__m256 V1, __m256 V2, const int M); 1469/// \endcode 1470/// 1471/// This intrinsic corresponds to the \c VDPPS / DPPS instruction. 1472/// 1473/// \param V1 1474/// A vector of [8 x float] values, treated as two [4 x float] vectors. 1475/// \param V2 1476/// A vector of [8 x float] values, treated as two [4 x float] vectors. 1477/// \param M 1478/// An immediate integer argument. Bits [7:4] determine which elements of 1479/// the input vectors are used, with bit [4] corresponding to the lowest 1480/// element and bit [7] corresponding to the highest element of each [4 x 1481/// float] subvector. If a bit is set, the corresponding elements from the 1482/// two input vectors are used as an input for dot product; otherwise that 1483/// input is treated as zero. Bits [3:0] determine which elements of the 1484/// result will receive a copy of the final dot product, with bit [0] 1485/// corresponding to the lowest element and bit [3] corresponding to the 1486/// highest element of each [4 x float] subvector. If a bit is set, the dot 1487/// product is returned in the corresponding element; otherwise that element 1488/// is set to zero. The bitmask is applied in the same way to each of the 1489/// two parallel dot product computations. 1490/// \returns A 256-bit vector of [8 x float] containing the two dot products. 1491#define _mm256_dp_ps(V1, V2, M) __extension__ ({ \ 1492 (__m256)__builtin_ia32_dpps256((__v8sf)(__m256)(V1), \ 1493 (__v8sf)(__m256)(V2), (M)); }) 1494 1495/* Vector shuffle */ 1496/// \brief Selects 8 float values from the 256-bit operands of [8 x float], as 1497/// specified by the immediate value operand. The four selected elements in 1498/// each operand are copied to the destination according to the bits 1499/// specified in the immediate operand. The selected elements from the first 1500/// 256-bit operand are copied to bits [63:0] and bits [191:128] of the 1501/// destination, and the selected elements from the second 256-bit operand 1502/// are copied to bits [127:64] and bits [255:192] of the destination. For 1503/// example, if bits [7:0] of the immediate operand contain a value of 0xFF, 1504/// the 256-bit destination vector would contain the following values: b[7], 1505/// b[7], a[7], a[7], b[3], b[3], a[3], a[3]. 1506/// 1507/// \headerfile <x86intrin.h> 1508/// 1509/// \code 1510/// __m256 _mm256_shuffle_ps(__m256 a, __m256 b, const int mask); 1511/// \endcode 1512/// 1513/// This intrinsic corresponds to the \c VSHUFPS / SHUFPS instruction. 1514/// 1515/// \param a 1516/// A 256-bit vector of [8 x float]. The four selected elements in this 1517/// operand are copied to bits [63:0] and bits [191:128] in the destination, 1518/// according to the bits specified in the immediate operand. 1519/// \param b 1520/// A 256-bit vector of [8 x float]. The four selected elements in this 1521/// operand are copied to bits [127:64] and bits [255:192] in the 1522/// destination, according to the bits specified in the immediate operand. 1523/// \param mask 1524/// An immediate value containing an 8-bit value specifying which elements to 1525/// copy from a and b. Bits [3:0] specify the values copied from operand a. 1526/// Bits [7:4] specify the values copied from operand b. 1527/// The destinations within the 256-bit destination are assigned values as 1528/// follows, according to the bit value assignments described below: 1529/// Bits [1:0] are used to assign values to bits [31:0] and [159:128] in the 1530/// destination. 1531/// Bits [3:2] are used to assign values to bits [63:32] and [191:160] in the 1532/// destination. 1533/// Bits [5:4] are used to assign values to bits [95:64] and [223:192] in the 1534/// destination. 1535/// Bits [7:6] are used to assign values to bits [127:96] and [255:224] in 1536/// the destination. 1537/// Bit value assignments: 1538/// 00: Bits [31:0] and [159:128] are copied from the selected operand. 1539/// 01: Bits [63:32] and [191:160] are copied from the selected operand. 1540/// 10: Bits [95:64] and [223:192] are copied from the selected operand. 1541/// 11: Bits [127:96] and [255:224] are copied from the selected operand. 1542/// \returns A 256-bit vector of [8 x float] containing the shuffled values. 1543#define _mm256_shuffle_ps(a, b, mask) __extension__ ({ \ 1544 (__m256)__builtin_shufflevector((__v8sf)(__m256)(a), \ 1545 (__v8sf)(__m256)(b), \ 1546 0 + (((mask) >> 0) & 0x3), \ 1547 0 + (((mask) >> 2) & 0x3), \ 1548 8 + (((mask) >> 4) & 0x3), \ 1549 8 + (((mask) >> 6) & 0x3), \ 1550 4 + (((mask) >> 0) & 0x3), \ 1551 4 + (((mask) >> 2) & 0x3), \ 1552 12 + (((mask) >> 4) & 0x3), \ 1553 12 + (((mask) >> 6) & 0x3)); }) 1554 1555/// \brief Selects four double-precision values from the 256-bit operands of 1556/// [4 x double], as specified by the immediate value operand. The selected 1557/// elements from the first 256-bit operand are copied to bits [63:0] and 1558/// bits [191:128] in the destination, and the selected elements from the 1559/// second 256-bit operand are copied to bits [127:64] and bits [255:192] in 1560/// the destination. For example, if bits [3:0] of the immediate operand 1561/// contain a value of 0xF, the 256-bit destination vector would contain the 1562/// following values: b[3], a[3], b[1], a[1]. 1563/// 1564/// \headerfile <x86intrin.h> 1565/// 1566/// \code 1567/// __m256d _mm256_shuffle_pd(__m256d a, __m256d b, const int mask); 1568/// \endcode 1569/// 1570/// This intrinsic corresponds to the \c VSHUFPD / SHUFPD instruction. 1571/// 1572/// \param a 1573/// A 256-bit vector of [4 x double]. 1574/// \param b 1575/// A 256-bit vector of [4 x double]. 1576/// \param mask 1577/// An immediate value containing 8-bit values specifying which elements to 1578/// copy from a and b: 1579/// Bit [0]=0: Bits [63:0] are copied from a to bits [63:0] of the 1580/// destination. 1581/// Bit [0]=1: Bits [127:64] are copied from a to bits [63:0] of the 1582/// destination. 1583/// Bit [1]=0: Bits [63:0] are copied from b to bits [127:64] of the 1584/// destination. 1585/// Bit [1]=1: Bits [127:64] are copied from b to bits [127:64] of the 1586/// destination. 1587/// Bit [2]=0: Bits [191:128] are copied from a to bits [191:128] of the 1588/// destination. 1589/// Bit [2]=1: Bits [255:192] are copied from a to bits [191:128] of the 1590/// destination. 1591/// Bit [3]=0: Bits [191:128] are copied from b to bits [255:192] of the 1592/// destination. 1593/// Bit [3]=1: Bits [255:192] are copied from b to bits [255:192] of the 1594/// destination. 1595/// \returns A 256-bit vector of [4 x double] containing the shuffled values. 1596#define _mm256_shuffle_pd(a, b, mask) __extension__ ({ \ 1597 (__m256d)__builtin_shufflevector((__v4df)(__m256d)(a), \ 1598 (__v4df)(__m256d)(b), \ 1599 0 + (((mask) >> 0) & 0x1), \ 1600 4 + (((mask) >> 1) & 0x1), \ 1601 2 + (((mask) >> 2) & 0x1), \ 1602 6 + (((mask) >> 3) & 0x1)); }) 1603 1604/* Compare */ 1605#define _CMP_EQ_OQ 0x00 /* Equal (ordered, non-signaling) */ 1606#define _CMP_LT_OS 0x01 /* Less-than (ordered, signaling) */ 1607#define _CMP_LE_OS 0x02 /* Less-than-or-equal (ordered, signaling) */ 1608#define _CMP_UNORD_Q 0x03 /* Unordered (non-signaling) */ 1609#define _CMP_NEQ_UQ 0x04 /* Not-equal (unordered, non-signaling) */ 1610#define _CMP_NLT_US 0x05 /* Not-less-than (unordered, signaling) */ 1611#define _CMP_NLE_US 0x06 /* Not-less-than-or-equal (unordered, signaling) */ 1612#define _CMP_ORD_Q 0x07 /* Ordered (nonsignaling) */ 1613#define _CMP_EQ_UQ 0x08 /* Equal (unordered, non-signaling) */ 1614#define _CMP_NGE_US 0x09 /* Not-greater-than-or-equal (unord, signaling) */ 1615#define _CMP_NGT_US 0x0a /* Not-greater-than (unordered, signaling) */ 1616#define _CMP_FALSE_OQ 0x0b /* False (ordered, non-signaling) */ 1617#define _CMP_NEQ_OQ 0x0c /* Not-equal (ordered, non-signaling) */ 1618#define _CMP_GE_OS 0x0d /* Greater-than-or-equal (ordered, signaling) */ 1619#define _CMP_GT_OS 0x0e /* Greater-than (ordered, signaling) */ 1620#define _CMP_TRUE_UQ 0x0f /* True (unordered, non-signaling) */ 1621#define _CMP_EQ_OS 0x10 /* Equal (ordered, signaling) */ 1622#define _CMP_LT_OQ 0x11 /* Less-than (ordered, non-signaling) */ 1623#define _CMP_LE_OQ 0x12 /* Less-than-or-equal (ordered, non-signaling) */ 1624#define _CMP_UNORD_S 0x13 /* Unordered (signaling) */ 1625#define _CMP_NEQ_US 0x14 /* Not-equal (unordered, signaling) */ 1626#define _CMP_NLT_UQ 0x15 /* Not-less-than (unordered, non-signaling) */ 1627#define _CMP_NLE_UQ 0x16 /* Not-less-than-or-equal (unord, non-signaling) */ 1628#define _CMP_ORD_S 0x17 /* Ordered (signaling) */ 1629#define _CMP_EQ_US 0x18 /* Equal (unordered, signaling) */ 1630#define _CMP_NGE_UQ 0x19 /* Not-greater-than-or-equal (unord, non-sign) */ 1631#define _CMP_NGT_UQ 0x1a /* Not-greater-than (unordered, non-signaling) */ 1632#define _CMP_FALSE_OS 0x1b /* False (ordered, signaling) */ 1633#define _CMP_NEQ_OS 0x1c /* Not-equal (ordered, signaling) */ 1634#define _CMP_GE_OQ 0x1d /* Greater-than-or-equal (ordered, non-signaling) */ 1635#define _CMP_GT_OQ 0x1e /* Greater-than (ordered, non-signaling) */ 1636#define _CMP_TRUE_US 0x1f /* True (unordered, signaling) */ 1637 1638/// \brief Compares each of the corresponding double-precision values of two 1639/// 128-bit vectors of [2 x double], using the operation specified by the 1640/// immediate integer operand. Returns a [2 x double] vector consisting of 1641/// two doubles corresponding to the two comparison results: zero if the 1642/// comparison is false, and all 1's if the comparison is true. 1643/// 1644/// \headerfile <x86intrin.h> 1645/// 1646/// \code 1647/// __m128d _mm_cmp_pd(__m128d a, __m128d b, const int c); 1648/// \endcode 1649/// 1650/// This intrinsic corresponds to the \c VCMPPD / CMPPD instruction. 1651/// 1652/// \param a 1653/// A 128-bit vector of [2 x double]. 1654/// \param b 1655/// A 128-bit vector of [2 x double]. 1656/// \param c 1657/// An immediate integer operand, with bits [4:0] specifying which comparison 1658/// operation to use: 1659/// 00h, 08h, 10h, 18h: Equal 1660/// 01h, 09h, 11h, 19h: Less than 1661/// 02h, 0Ah, 12h, 1Ah: Less than or equal / Greater than or equal (swapped 1662/// operands) 1663/// 03h, 0Bh, 13h, 1Bh: Unordered 1664/// 04h, 0Ch, 14h, 1Ch: Not equal 1665/// 05h, 0Dh, 15h, 1Dh: Not less than / Not greater than (swapped operands) 1666/// 06h, 0Eh, 16h, 1Eh: Not less than or equal / Not greater than or equal 1667/// (swapped operands) 1668/// 07h, 0Fh, 17h, 1Fh: Ordered 1669/// \returns A 128-bit vector of [2 x double] containing the comparison results. 1670#define _mm_cmp_pd(a, b, c) __extension__ ({ \ 1671 (__m128d)__builtin_ia32_cmppd((__v2df)(__m128d)(a), \ 1672 (__v2df)(__m128d)(b), (c)); }) 1673 1674/// \brief Compares each of the corresponding values of two 128-bit vectors of 1675/// [4 x float], using the operation specified by the immediate integer 1676/// operand. Returns a [4 x float] vector consisting of four floats 1677/// corresponding to the four comparison results: zero if the comparison is 1678/// false, and all 1's if the comparison is true. 1679/// 1680/// \headerfile <x86intrin.h> 1681/// 1682/// \code 1683/// __m128 _mm_cmp_ps(__m128 a, __m128 b, const int c); 1684/// \endcode 1685/// 1686/// This intrinsic corresponds to the \c VCMPPS / CMPPS instruction. 1687/// 1688/// \param a 1689/// A 128-bit vector of [4 x float]. 1690/// \param b 1691/// A 128-bit vector of [4 x float]. 1692/// \param c 1693/// An immediate integer operand, with bits [4:0] specifying which comparison 1694/// operation to use: 1695/// 00h, 08h, 10h, 18h: Equal 1696/// 01h, 09h, 11h, 19h: Less than 1697/// 02h, 0Ah, 12h, 1Ah: Less than or equal / Greater than or equal (swapped 1698/// operands) 1699/// 03h, 0Bh, 13h, 1Bh: Unordered 1700/// 04h, 0Ch, 14h, 1Ch: Not equal 1701/// 05h, 0Dh, 15h, 1Dh: Not less than / Not greater than (swapped operands) 1702/// 06h, 0Eh, 16h, 1Eh: Not less than or equal / Not greater than or equal 1703/// (swapped operands) 1704/// 07h, 0Fh, 17h, 1Fh: Ordered 1705/// \returns A 128-bit vector of [4 x float] containing the comparison results. 1706#define _mm_cmp_ps(a, b, c) __extension__ ({ \ 1707 (__m128)__builtin_ia32_cmpps((__v4sf)(__m128)(a), \ 1708 (__v4sf)(__m128)(b), (c)); }) 1709 1710/// \brief Compares each of the corresponding double-precision values of two 1711/// 256-bit vectors of [4 x double], using the operation specified by the 1712/// immediate integer operand. Returns a [4 x double] vector consisting of 1713/// four doubles corresponding to the four comparison results: zero if the 1714/// comparison is false, and all 1's if the comparison is true. 1715/// 1716/// \headerfile <x86intrin.h> 1717/// 1718/// \code 1719/// __m256d _mm256_cmp_pd(__m256d a, __m256d b, const int c); 1720/// \endcode 1721/// 1722/// This intrinsic corresponds to the \c VCMPPD / CMPPD instruction. 1723/// 1724/// \param a 1725/// A 256-bit vector of [4 x double]. 1726/// \param b 1727/// A 256-bit vector of [4 x double]. 1728/// \param c 1729/// An immediate integer operand, with bits [4:0] specifying which comparison 1730/// operation to use: 1731/// 00h, 08h, 10h, 18h: Equal 1732/// 01h, 09h, 11h, 19h: Less than 1733/// 02h, 0Ah, 12h, 1Ah: Less than or equal / Greater than or equal (swapped 1734/// operands) 1735/// 03h, 0Bh, 13h, 1Bh: Unordered 1736/// 04h, 0Ch, 14h, 1Ch: Not equal 1737/// 05h, 0Dh, 15h, 1Dh: Not less than / Not greater than (swapped operands) 1738/// 06h, 0Eh, 16h, 1Eh: Not less than or equal / Not greater than or equal 1739/// (swapped operands) 1740/// 07h, 0Fh, 17h, 1Fh: Ordered 1741/// \returns A 256-bit vector of [4 x double] containing the comparison results. 1742#define _mm256_cmp_pd(a, b, c) __extension__ ({ \ 1743 (__m256d)__builtin_ia32_cmppd256((__v4df)(__m256d)(a), \ 1744 (__v4df)(__m256d)(b), (c)); }) 1745 1746/// \brief Compares each of the corresponding values of two 256-bit vectors of 1747/// [8 x float], using the operation specified by the immediate integer 1748/// operand. Returns a [8 x float] vector consisting of eight floats 1749/// corresponding to the eight comparison results: zero if the comparison is 1750/// false, and all 1's if the comparison is true. 1751/// 1752/// \headerfile <x86intrin.h> 1753/// 1754/// \code 1755/// __m256 _mm256_cmp_ps(__m256 a, __m256 b, const int c); 1756/// \endcode 1757/// 1758/// This intrinsic corresponds to the \c VCMPPS / CMPPS instruction. 1759/// 1760/// \param a 1761/// A 256-bit vector of [8 x float]. 1762/// \param b 1763/// A 256-bit vector of [8 x float]. 1764/// \param c 1765/// An immediate integer operand, with bits [4:0] specifying which comparison 1766/// operation to use: 1767/// 00h, 08h, 10h, 18h: Equal 1768/// 01h, 09h, 11h, 19h: Less than 1769/// 02h, 0Ah, 12h, 1Ah: Less than or equal / Greater than or equal (swapped 1770/// operands) 1771/// 03h, 0Bh, 13h, 1Bh: Unordered 1772/// 04h, 0Ch, 14h, 1Ch: Not equal 1773/// 05h, 0Dh, 15h, 1Dh: Not less than / Not greater than (swapped operands) 1774/// 06h, 0Eh, 16h, 1Eh: Not less than or equal / Not greater than or equal 1775/// (swapped operands) 1776/// 07h, 0Fh, 17h, 1Fh: Ordered 1777/// \returns A 256-bit vector of [8 x float] containing the comparison results. 1778#define _mm256_cmp_ps(a, b, c) __extension__ ({ \ 1779 (__m256)__builtin_ia32_cmpps256((__v8sf)(__m256)(a), \ 1780 (__v8sf)(__m256)(b), (c)); }) 1781 1782/// \brief Compares each of the corresponding scalar double-precision values of 1783/// two 128-bit vectors of [2 x double], using the operation specified by the 1784/// immediate integer operand. If the result is true, all 64 bits of the 1785/// destination vector are set; otherwise they are cleared. 1786/// 1787/// \headerfile <x86intrin.h> 1788/// 1789/// \code 1790/// __m128d _mm_cmp_sd(__m128d a, __m128d b, const int c); 1791/// \endcode 1792/// 1793/// This intrinsic corresponds to the \c VCMPSD / CMPSD instruction. 1794/// 1795/// \param a 1796/// A 128-bit vector of [2 x double]. 1797/// \param b 1798/// A 128-bit vector of [2 x double]. 1799/// \param c 1800/// An immediate integer operand, with bits [4:0] specifying which comparison 1801/// operation to use: 1802/// 00h, 08h, 10h, 18h: Equal 1803/// 01h, 09h, 11h, 19h: Less than 1804/// 02h, 0Ah, 12h, 1Ah: Less than or equal / Greater than or equal (swapped 1805/// operands) 1806/// 03h, 0Bh, 13h, 1Bh: Unordered 1807/// 04h, 0Ch, 14h, 1Ch: Not equal 1808/// 05h, 0Dh, 15h, 1Dh: Not less than / Not greater than (swapped operands) 1809/// 06h, 0Eh, 16h, 1Eh: Not less than or equal / Not greater than or equal 1810/// (swapped operands) 1811/// 07h, 0Fh, 17h, 1Fh: Ordered 1812/// \returns A 128-bit vector of [2 x double] containing the comparison results. 1813#define _mm_cmp_sd(a, b, c) __extension__ ({ \ 1814 (__m128d)__builtin_ia32_cmpsd((__v2df)(__m128d)(a), \ 1815 (__v2df)(__m128d)(b), (c)); }) 1816 1817/// \brief Compares each of the corresponding scalar values of two 128-bit 1818/// vectors of [4 x float], using the operation specified by the immediate 1819/// integer operand. If the result is true, all 32 bits of the destination 1820/// vector are set; otherwise they are cleared. 1821/// 1822/// \headerfile <x86intrin.h> 1823/// 1824/// \code 1825/// __m128 _mm_cmp_ss(__m128 a, __m128 b, const int c); 1826/// \endcode 1827/// 1828/// This intrinsic corresponds to the \c VCMPSS / CMPSS instruction. 1829/// 1830/// \param a 1831/// A 128-bit vector of [4 x float]. 1832/// \param b 1833/// A 128-bit vector of [4 x float]. 1834/// \param c 1835/// An immediate integer operand, with bits [4:0] specifying which comparison 1836/// operation to use: 1837/// 00h, 08h, 10h, 18h: Equal 1838/// 01h, 09h, 11h, 19h: Less than 1839/// 02h, 0Ah, 12h, 1Ah: Less than or equal / Greater than or equal (swapped 1840/// operands) 1841/// 03h, 0Bh, 13h, 1Bh: Unordered 1842/// 04h, 0Ch, 14h, 1Ch: Not equal 1843/// 05h, 0Dh, 15h, 1Dh: Not less than / Not greater than (swapped operands) 1844/// 06h, 0Eh, 16h, 1Eh: Not less than or equal / Not greater than or equal 1845/// (swapped operands) 1846/// 07h, 0Fh, 17h, 1Fh: Ordered 1847/// \returns A 128-bit vector of [4 x float] containing the comparison results. 1848#define _mm_cmp_ss(a, b, c) __extension__ ({ \ 1849 (__m128)__builtin_ia32_cmpss((__v4sf)(__m128)(a), \ 1850 (__v4sf)(__m128)(b), (c)); }) 1851 1852/// \brief Takes a [8 x i32] vector and returns the vector element value 1853/// indexed by the immediate constant operand. 1854/// 1855/// \headerfile <x86intrin.h> 1856/// 1857/// This intrinsic corresponds to the \c VEXTRACTF128+COMPOSITE / 1858/// EXTRACTF128+COMPOSITE instruction. 1859/// 1860/// \param __a 1861/// A 256-bit vector of [8 x i32]. 1862/// \param __imm 1863/// An immediate integer operand with bits [2:0] determining which vector 1864/// element is extracted and returned. 1865/// \returns A 32-bit integer containing the extracted 32 bits of extended 1866/// packed data. 1867static __inline int __DEFAULT_FN_ATTRS 1868_mm256_extract_epi32(__m256i __a, const int __imm) 1869{ 1870 __v8si __b = (__v8si)__a; 1871 return __b[__imm & 7]; 1872} 1873 1874/// \brief Takes a [16 x i16] vector and returns the vector element value 1875/// indexed by the immediate constant operand. 1876/// 1877/// \headerfile <x86intrin.h> 1878/// 1879/// This intrinsic corresponds to the \c VEXTRACTF128+COMPOSITE / 1880/// EXTRACTF128+COMPOSITE instruction. 1881/// 1882/// \param __a 1883/// A 256-bit integer vector of [16 x i16]. 1884/// \param __imm 1885/// An immediate integer operand with bits [3:0] determining which vector 1886/// element is extracted and returned. 1887/// \returns A 32-bit integer containing the extracted 16 bits of zero extended 1888/// packed data. 1889static __inline int __DEFAULT_FN_ATTRS 1890_mm256_extract_epi16(__m256i __a, const int __imm) 1891{ 1892 __v16hi __b = (__v16hi)__a; 1893 return (unsigned short)__b[__imm & 15]; 1894} 1895 1896/// \brief Takes a [32 x i8] vector and returns the vector element value 1897/// indexed by the immediate constant operand. 1898/// 1899/// \headerfile <x86intrin.h> 1900/// 1901/// This intrinsic corresponds to the \c VEXTRACTF128+COMPOSITE / 1902/// EXTRACTF128+COMPOSITE instruction. 1903/// 1904/// \param __a 1905/// A 256-bit integer vector of [32 x i8]. 1906/// \param __imm 1907/// An immediate integer operand with bits [4:0] determining which vector 1908/// element is extracted and returned. 1909/// \returns A 32-bit integer containing the extracted 8 bits of zero extended 1910/// packed data. 1911static __inline int __DEFAULT_FN_ATTRS 1912_mm256_extract_epi8(__m256i __a, const int __imm) 1913{ 1914 __v32qi __b = (__v32qi)__a; 1915 return (unsigned char)__b[__imm & 31]; 1916} 1917 1918#ifdef __x86_64__ 1919/// \brief Takes a [4 x i64] vector and returns the vector element value 1920/// indexed by the immediate constant operand. 1921/// 1922/// \headerfile <x86intrin.h> 1923/// 1924/// This intrinsic corresponds to the \c VEXTRACTF128+COMPOSITE / 1925/// EXTRACTF128+COMPOSITE instruction. 1926/// 1927/// \param __a 1928/// A 256-bit integer vector of [4 x i64]. 1929/// \param __imm 1930/// An immediate integer operand with bits [1:0] determining which vector 1931/// element is extracted and returned. 1932/// \returns A 64-bit integer containing the extracted 64 bits of extended 1933/// packed data. 1934static __inline long long __DEFAULT_FN_ATTRS 1935_mm256_extract_epi64(__m256i __a, const int __imm) 1936{ 1937 __v4di __b = (__v4di)__a; 1938 return __b[__imm & 3]; 1939} 1940#endif 1941 1942/// \brief Takes a [8 x i32] vector and replaces the vector element value 1943/// indexed by the immediate constant operand by a new value. Returns the 1944/// modified vector. 1945/// 1946/// \headerfile <x86intrin.h> 1947/// 1948/// This intrinsic corresponds to the \c VINSERTF128+COMPOSITE / 1949/// INSERTF128+COMPOSITE instruction. 1950/// 1951/// \param __a 1952/// A vector of [8 x i32] to be used by the insert operation. 1953/// \param __b 1954/// An integer value. The replacement value for the insert operation. 1955/// \param __imm 1956/// An immediate integer specifying the index of the vector element to be 1957/// replaced. 1958/// \returns A copy of vector __a, after replacing its element indexed by __imm 1959/// with __b. 1960static __inline __m256i __DEFAULT_FN_ATTRS 1961_mm256_insert_epi32(__m256i __a, int __b, int const __imm) 1962{ 1963 __v8si __c = (__v8si)__a; 1964 __c[__imm & 7] = __b; 1965 return (__m256i)__c; 1966} 1967 1968 1969/// \brief Takes a [16 x i16] vector and replaces the vector element value 1970/// indexed by the immediate constant operand with a new value. Returns the 1971/// modified vector. 1972/// 1973/// \headerfile <x86intrin.h> 1974/// 1975/// This intrinsic corresponds to the \c VINSERTF128+COMPOSITE / 1976/// INSERTF128+COMPOSITE instruction. 1977/// 1978/// \param __a 1979/// A vector of [16 x i16] to be used by the insert operation. 1980/// \param __b 1981/// An i16 integer value. The replacement value for the insert operation. 1982/// \param __imm 1983/// An immediate integer specifying the index of the vector element to be 1984/// replaced. 1985/// \returns A copy of vector __a, after replacing its element indexed by __imm 1986/// with __b. 1987static __inline __m256i __DEFAULT_FN_ATTRS 1988_mm256_insert_epi16(__m256i __a, int __b, int const __imm) 1989{ 1990 __v16hi __c = (__v16hi)__a; 1991 __c[__imm & 15] = __b; 1992 return (__m256i)__c; 1993} 1994 1995/// \brief Takes a [32 x i8] vector and replaces the vector element value 1996/// indexed by the immediate constant operand with a new value. Returns the 1997/// modified vector. 1998/// 1999/// \headerfile <x86intrin.h> 2000/// 2001/// This intrinsic corresponds to the \c VINSERTF128+COMPOSITE / 2002/// INSERTF128+COMPOSITE instruction. 2003/// 2004/// \param __a 2005/// A vector of [32 x i8] to be used by the insert operation. 2006/// \param __b 2007/// An i8 integer value. The replacement value for the insert operation. 2008/// \param __imm 2009/// An immediate integer specifying the index of the vector element to be 2010/// replaced. 2011/// \returns A copy of vector __a, after replacing its element indexed by __imm 2012/// with __b. 2013static __inline __m256i __DEFAULT_FN_ATTRS 2014_mm256_insert_epi8(__m256i __a, int __b, int const __imm) 2015{ 2016 __v32qi __c = (__v32qi)__a; 2017 __c[__imm & 31] = __b; 2018 return (__m256i)__c; 2019} 2020 2021#ifdef __x86_64__ 2022/// \brief Takes a [4 x i64] vector and replaces the vector element value 2023/// indexed by the immediate constant operand with a new value. Returns the 2024/// modified vector. 2025/// 2026/// \headerfile <x86intrin.h> 2027/// 2028/// This intrinsic corresponds to the \c VINSERTF128+COMPOSITE / 2029/// INSERTF128+COMPOSITE instruction. 2030/// 2031/// \param __a 2032/// A vector of [4 x i64] to be used by the insert operation. 2033/// \param __b 2034/// A 64-bit integer value. The replacement value for the insert operation. 2035/// \param __imm 2036/// An immediate integer specifying the index of the vector element to be 2037/// replaced. 2038/// \returns A copy of vector __a, after replacing its element indexed by __imm 2039/// with __b. 2040static __inline __m256i __DEFAULT_FN_ATTRS 2041_mm256_insert_epi64(__m256i __a, long long __b, int const __imm) 2042{ 2043 __v4di __c = (__v4di)__a; 2044 __c[__imm & 3] = __b; 2045 return (__m256i)__c; 2046} 2047#endif 2048 2049/* Conversion */ 2050/// \brief Converts a vector of [4 x i32] into a vector of [4 x double]. 2051/// 2052/// \headerfile <x86intrin.h> 2053/// 2054/// This intrinsic corresponds to the \c VCVTDQ2PD / CVTDQ2PD instruction. 2055/// 2056/// \param __a 2057/// A 128-bit integer vector of [4 x i32]. 2058/// \returns A 256-bit vector of [4 x double] containing the converted values. 2059static __inline __m256d __DEFAULT_FN_ATTRS 2060_mm256_cvtepi32_pd(__m128i __a) 2061{ 2062 return (__m256d)__builtin_convertvector((__v4si)__a, __v4df); 2063} 2064 2065/// \brief Converts a vector of [8 x i32] into a vector of [8 x float]. 2066/// 2067/// \headerfile <x86intrin.h> 2068/// 2069/// This intrinsic corresponds to the \c VCVTDQ2PS / CVTDQ2PS instruction. 2070/// 2071/// \param __a 2072/// A 256-bit integer vector. 2073/// \returns A 256-bit vector of [8 x float] containing the converted values. 2074static __inline __m256 __DEFAULT_FN_ATTRS 2075_mm256_cvtepi32_ps(__m256i __a) 2076{ 2077 return (__m256)__builtin_ia32_cvtdq2ps256((__v8si) __a); 2078} 2079 2080/// \brief Converts a 256-bit vector of [4 x double] into a 128-bit vector of 2081/// [4 x float]. 2082/// 2083/// \headerfile <x86intrin.h> 2084/// 2085/// This intrinsic corresponds to the \c VCVTPD2PS / CVTPD2PS instruction. 2086/// 2087/// \param __a 2088/// A 256-bit vector of [4 x double]. 2089/// \returns A 128-bit vector of [4 x float] containing the converted values. 2090static __inline __m128 __DEFAULT_FN_ATTRS 2091_mm256_cvtpd_ps(__m256d __a) 2092{ 2093 return (__m128)__builtin_ia32_cvtpd2ps256((__v4df) __a); 2094} 2095 2096/// \brief Converts a vector of [8 x float] into a vector of [8 x i32]. 2097/// 2098/// \headerfile <x86intrin.h> 2099/// 2100/// This intrinsic corresponds to the \c VCVTPS2DQ / CVTPS2DQ instruction. 2101/// 2102/// \param __a 2103/// A 256-bit vector of [8 x float]. 2104/// \returns A 256-bit integer vector containing the converted values. 2105static __inline __m256i __DEFAULT_FN_ATTRS 2106_mm256_cvtps_epi32(__m256 __a) 2107{ 2108 return (__m256i)__builtin_ia32_cvtps2dq256((__v8sf) __a); 2109} 2110 2111static __inline __m256d __DEFAULT_FN_ATTRS 2112_mm256_cvtps_pd(__m128 __a) 2113{ 2114 return (__m256d)__builtin_convertvector((__v4sf)__a, __v4df); 2115} 2116 2117static __inline __m128i __DEFAULT_FN_ATTRS 2118_mm256_cvttpd_epi32(__m256d __a) 2119{ 2120 return (__m128i)__builtin_ia32_cvttpd2dq256((__v4df) __a); 2121} 2122 2123static __inline __m128i __DEFAULT_FN_ATTRS 2124_mm256_cvtpd_epi32(__m256d __a) 2125{ 2126 return (__m128i)__builtin_ia32_cvtpd2dq256((__v4df) __a); 2127} 2128 2129static __inline __m256i __DEFAULT_FN_ATTRS 2130_mm256_cvttps_epi32(__m256 __a) 2131{ 2132 return (__m256i)__builtin_ia32_cvttps2dq256((__v8sf) __a); 2133} 2134 2135static __inline double __DEFAULT_FN_ATTRS 2136_mm256_cvtsd_f64(__m256d __a) 2137{ 2138 return __a[0]; 2139} 2140 2141static __inline int __DEFAULT_FN_ATTRS 2142_mm256_cvtsi256_si32(__m256i __a) 2143{ 2144 __v8si __b = (__v8si)__a; 2145 return __b[0]; 2146} 2147 2148static __inline float __DEFAULT_FN_ATTRS 2149_mm256_cvtss_f32(__m256 __a) 2150{ 2151 return __a[0]; 2152} 2153 2154/* Vector replicate */ 2155static __inline __m256 __DEFAULT_FN_ATTRS 2156_mm256_movehdup_ps(__m256 __a) 2157{ 2158 return __builtin_shufflevector((__v8sf)__a, (__v8sf)__a, 1, 1, 3, 3, 5, 5, 7, 7); 2159} 2160 2161static __inline __m256 __DEFAULT_FN_ATTRS 2162_mm256_moveldup_ps(__m256 __a) 2163{ 2164 return __builtin_shufflevector((__v8sf)__a, (__v8sf)__a, 0, 0, 2, 2, 4, 4, 6, 6); 2165} 2166 2167static __inline __m256d __DEFAULT_FN_ATTRS 2168_mm256_movedup_pd(__m256d __a) 2169{ 2170 return __builtin_shufflevector((__v4df)__a, (__v4df)__a, 0, 0, 2, 2); 2171} 2172 2173/* Unpack and Interleave */ 2174static __inline __m256d __DEFAULT_FN_ATTRS 2175_mm256_unpackhi_pd(__m256d __a, __m256d __b) 2176{ 2177 return __builtin_shufflevector((__v4df)__a, (__v4df)__b, 1, 5, 1+2, 5+2); 2178} 2179 2180static __inline __m256d __DEFAULT_FN_ATTRS 2181_mm256_unpacklo_pd(__m256d __a, __m256d __b) 2182{ 2183 return __builtin_shufflevector((__v4df)__a, (__v4df)__b, 0, 4, 0+2, 4+2); 2184} 2185 2186static __inline __m256 __DEFAULT_FN_ATTRS 2187_mm256_unpackhi_ps(__m256 __a, __m256 __b) 2188{ 2189 return __builtin_shufflevector((__v8sf)__a, (__v8sf)__b, 2, 10, 2+1, 10+1, 6, 14, 6+1, 14+1); 2190} 2191 2192static __inline __m256 __DEFAULT_FN_ATTRS 2193_mm256_unpacklo_ps(__m256 __a, __m256 __b) 2194{ 2195 return __builtin_shufflevector((__v8sf)__a, (__v8sf)__b, 0, 8, 0+1, 8+1, 4, 12, 4+1, 12+1); 2196} 2197 2198/* Bit Test */ 2199static __inline int __DEFAULT_FN_ATTRS 2200_mm_testz_pd(__m128d __a, __m128d __b) 2201{ 2202 return __builtin_ia32_vtestzpd((__v2df)__a, (__v2df)__b); 2203} 2204 2205static __inline int __DEFAULT_FN_ATTRS 2206_mm_testc_pd(__m128d __a, __m128d __b) 2207{ 2208 return __builtin_ia32_vtestcpd((__v2df)__a, (__v2df)__b); 2209} 2210 2211static __inline int __DEFAULT_FN_ATTRS 2212_mm_testnzc_pd(__m128d __a, __m128d __b) 2213{ 2214 return __builtin_ia32_vtestnzcpd((__v2df)__a, (__v2df)__b); 2215} 2216 2217static __inline int __DEFAULT_FN_ATTRS 2218_mm_testz_ps(__m128 __a, __m128 __b) 2219{ 2220 return __builtin_ia32_vtestzps((__v4sf)__a, (__v4sf)__b); 2221} 2222 2223static __inline int __DEFAULT_FN_ATTRS 2224_mm_testc_ps(__m128 __a, __m128 __b) 2225{ 2226 return __builtin_ia32_vtestcps((__v4sf)__a, (__v4sf)__b); 2227} 2228 2229static __inline int __DEFAULT_FN_ATTRS 2230_mm_testnzc_ps(__m128 __a, __m128 __b) 2231{ 2232 return __builtin_ia32_vtestnzcps((__v4sf)__a, (__v4sf)__b); 2233} 2234 2235static __inline int __DEFAULT_FN_ATTRS 2236_mm256_testz_pd(__m256d __a, __m256d __b) 2237{ 2238 return __builtin_ia32_vtestzpd256((__v4df)__a, (__v4df)__b); 2239} 2240 2241static __inline int __DEFAULT_FN_ATTRS 2242_mm256_testc_pd(__m256d __a, __m256d __b) 2243{ 2244 return __builtin_ia32_vtestcpd256((__v4df)__a, (__v4df)__b); 2245} 2246 2247static __inline int __DEFAULT_FN_ATTRS 2248_mm256_testnzc_pd(__m256d __a, __m256d __b) 2249{ 2250 return __builtin_ia32_vtestnzcpd256((__v4df)__a, (__v4df)__b); 2251} 2252 2253static __inline int __DEFAULT_FN_ATTRS 2254_mm256_testz_ps(__m256 __a, __m256 __b) 2255{ 2256 return __builtin_ia32_vtestzps256((__v8sf)__a, (__v8sf)__b); 2257} 2258 2259static __inline int __DEFAULT_FN_ATTRS 2260_mm256_testc_ps(__m256 __a, __m256 __b) 2261{ 2262 return __builtin_ia32_vtestcps256((__v8sf)__a, (__v8sf)__b); 2263} 2264 2265static __inline int __DEFAULT_FN_ATTRS 2266_mm256_testnzc_ps(__m256 __a, __m256 __b) 2267{ 2268 return __builtin_ia32_vtestnzcps256((__v8sf)__a, (__v8sf)__b); 2269} 2270 2271static __inline int __DEFAULT_FN_ATTRS 2272_mm256_testz_si256(__m256i __a, __m256i __b) 2273{ 2274 return __builtin_ia32_ptestz256((__v4di)__a, (__v4di)__b); 2275} 2276 2277static __inline int __DEFAULT_FN_ATTRS 2278_mm256_testc_si256(__m256i __a, __m256i __b) 2279{ 2280 return __builtin_ia32_ptestc256((__v4di)__a, (__v4di)__b); 2281} 2282 2283static __inline int __DEFAULT_FN_ATTRS 2284_mm256_testnzc_si256(__m256i __a, __m256i __b) 2285{ 2286 return __builtin_ia32_ptestnzc256((__v4di)__a, (__v4di)__b); 2287} 2288 2289/* Vector extract sign mask */ 2290static __inline int __DEFAULT_FN_ATTRS 2291_mm256_movemask_pd(__m256d __a) 2292{ 2293 return __builtin_ia32_movmskpd256((__v4df)__a); 2294} 2295 2296static __inline int __DEFAULT_FN_ATTRS 2297_mm256_movemask_ps(__m256 __a) 2298{ 2299 return __builtin_ia32_movmskps256((__v8sf)__a); 2300} 2301 2302/* Vector __zero */ 2303static __inline void __DEFAULT_FN_ATTRS 2304_mm256_zeroall(void) 2305{ 2306 __builtin_ia32_vzeroall(); 2307} 2308 2309static __inline void __DEFAULT_FN_ATTRS 2310_mm256_zeroupper(void) 2311{ 2312 __builtin_ia32_vzeroupper(); 2313} 2314 2315/* Vector load with broadcast */ 2316static __inline __m128 __DEFAULT_FN_ATTRS 2317_mm_broadcast_ss(float const *__a) 2318{ 2319 float __f = *__a; 2320 return (__m128)(__v4sf){ __f, __f, __f, __f }; 2321} 2322 2323static __inline __m256d __DEFAULT_FN_ATTRS 2324_mm256_broadcast_sd(double const *__a) 2325{ 2326 double __d = *__a; 2327 return (__m256d)(__v4df){ __d, __d, __d, __d }; 2328} 2329 2330static __inline __m256 __DEFAULT_FN_ATTRS 2331_mm256_broadcast_ss(float const *__a) 2332{ 2333 float __f = *__a; 2334 return (__m256)(__v8sf){ __f, __f, __f, __f, __f, __f, __f, __f }; 2335} 2336 2337static __inline __m256d __DEFAULT_FN_ATTRS 2338_mm256_broadcast_pd(__m128d const *__a) 2339{ 2340 return (__m256d)__builtin_ia32_vbroadcastf128_pd256((__v2df const *)__a); 2341} 2342 2343static __inline __m256 __DEFAULT_FN_ATTRS 2344_mm256_broadcast_ps(__m128 const *__a) 2345{ 2346 return (__m256)__builtin_ia32_vbroadcastf128_ps256((__v4sf const *)__a); 2347} 2348 2349/* SIMD load ops */ 2350static __inline __m256d __DEFAULT_FN_ATTRS 2351_mm256_load_pd(double const *__p) 2352{ 2353 return *(__m256d *)__p; 2354} 2355 2356static __inline __m256 __DEFAULT_FN_ATTRS 2357_mm256_load_ps(float const *__p) 2358{ 2359 return *(__m256 *)__p; 2360} 2361 2362static __inline __m256d __DEFAULT_FN_ATTRS 2363_mm256_loadu_pd(double const *__p) 2364{ 2365 struct __loadu_pd { 2366 __m256d __v; 2367 } __attribute__((__packed__, __may_alias__)); 2368 return ((struct __loadu_pd*)__p)->__v; 2369} 2370 2371static __inline __m256 __DEFAULT_FN_ATTRS 2372_mm256_loadu_ps(float const *__p) 2373{ 2374 struct __loadu_ps { 2375 __m256 __v; 2376 } __attribute__((__packed__, __may_alias__)); 2377 return ((struct __loadu_ps*)__p)->__v; 2378} 2379 2380static __inline __m256i __DEFAULT_FN_ATTRS 2381_mm256_load_si256(__m256i const *__p) 2382{ 2383 return *__p; 2384} 2385 2386static __inline __m256i __DEFAULT_FN_ATTRS 2387_mm256_loadu_si256(__m256i const *__p) 2388{ 2389 struct __loadu_si256 { 2390 __m256i __v; 2391 } __attribute__((__packed__, __may_alias__)); 2392 return ((struct __loadu_si256*)__p)->__v; 2393} 2394 2395static __inline __m256i __DEFAULT_FN_ATTRS 2396_mm256_lddqu_si256(__m256i const *__p) 2397{ 2398 return (__m256i)__builtin_ia32_lddqu256((char const *)__p); 2399} 2400 2401/* SIMD store ops */ 2402static __inline void __DEFAULT_FN_ATTRS 2403_mm256_store_pd(double *__p, __m256d __a) 2404{ 2405 *(__m256d *)__p = __a; 2406} 2407 2408static __inline void __DEFAULT_FN_ATTRS 2409_mm256_store_ps(float *__p, __m256 __a) 2410{ 2411 *(__m256 *)__p = __a; 2412} 2413 2414static __inline void __DEFAULT_FN_ATTRS 2415_mm256_storeu_pd(double *__p, __m256d __a) 2416{ 2417 struct __storeu_pd { 2418 __m256d __v; 2419 } __attribute__((__packed__, __may_alias__)); 2420 ((struct __storeu_pd*)__p)->__v = __a; 2421} 2422 2423static __inline void __DEFAULT_FN_ATTRS 2424_mm256_storeu_ps(float *__p, __m256 __a) 2425{ 2426 struct __storeu_ps { 2427 __m256 __v; 2428 } __attribute__((__packed__, __may_alias__)); 2429 ((struct __storeu_ps*)__p)->__v = __a; 2430} 2431 2432static __inline void __DEFAULT_FN_ATTRS 2433_mm256_store_si256(__m256i *__p, __m256i __a) 2434{ 2435 *__p = __a; 2436} 2437 2438static __inline void __DEFAULT_FN_ATTRS 2439_mm256_storeu_si256(__m256i *__p, __m256i __a) 2440{ 2441 struct __storeu_si256 { 2442 __m256i __v; 2443 } __attribute__((__packed__, __may_alias__)); 2444 ((struct __storeu_si256*)__p)->__v = __a; 2445} 2446 2447/* Conditional load ops */ 2448static __inline __m128d __DEFAULT_FN_ATTRS 2449_mm_maskload_pd(double const *__p, __m128i __m) 2450{ 2451 return (__m128d)__builtin_ia32_maskloadpd((const __v2df *)__p, (__v2di)__m); 2452} 2453 2454static __inline __m256d __DEFAULT_FN_ATTRS 2455_mm256_maskload_pd(double const *__p, __m256i __m) 2456{ 2457 return (__m256d)__builtin_ia32_maskloadpd256((const __v4df *)__p, 2458 (__v4di)__m); 2459} 2460 2461static __inline __m128 __DEFAULT_FN_ATTRS 2462_mm_maskload_ps(float const *__p, __m128i __m) 2463{ 2464 return (__m128)__builtin_ia32_maskloadps((const __v4sf *)__p, (__v4si)__m); 2465} 2466 2467static __inline __m256 __DEFAULT_FN_ATTRS 2468_mm256_maskload_ps(float const *__p, __m256i __m) 2469{ 2470 return (__m256)__builtin_ia32_maskloadps256((const __v8sf *)__p, (__v8si)__m); 2471} 2472 2473/* Conditional store ops */ 2474static __inline void __DEFAULT_FN_ATTRS 2475_mm256_maskstore_ps(float *__p, __m256i __m, __m256 __a) 2476{ 2477 __builtin_ia32_maskstoreps256((__v8sf *)__p, (__v8si)__m, (__v8sf)__a); 2478} 2479 2480static __inline void __DEFAULT_FN_ATTRS 2481_mm_maskstore_pd(double *__p, __m128i __m, __m128d __a) 2482{ 2483 __builtin_ia32_maskstorepd((__v2df *)__p, (__v2di)__m, (__v2df)__a); 2484} 2485 2486static __inline void __DEFAULT_FN_ATTRS 2487_mm256_maskstore_pd(double *__p, __m256i __m, __m256d __a) 2488{ 2489 __builtin_ia32_maskstorepd256((__v4df *)__p, (__v4di)__m, (__v4df)__a); 2490} 2491 2492static __inline void __DEFAULT_FN_ATTRS 2493_mm_maskstore_ps(float *__p, __m128i __m, __m128 __a) 2494{ 2495 __builtin_ia32_maskstoreps((__v4sf *)__p, (__v4si)__m, (__v4sf)__a); 2496} 2497 2498/* Cacheability support ops */ 2499static __inline void __DEFAULT_FN_ATTRS 2500_mm256_stream_si256(__m256i *__a, __m256i __b) 2501{ 2502 __builtin_nontemporal_store((__v4di)__b, (__v4di*)__a); 2503} 2504 2505static __inline void __DEFAULT_FN_ATTRS 2506_mm256_stream_pd(double *__a, __m256d __b) 2507{ 2508 __builtin_nontemporal_store((__v4df)__b, (__v4df*)__a); 2509} 2510 2511static __inline void __DEFAULT_FN_ATTRS 2512_mm256_stream_ps(float *__p, __m256 __a) 2513{ 2514 __builtin_nontemporal_store((__v8sf)__a, (__v8sf*)__p); 2515} 2516 2517/* Create vectors */ 2518static __inline__ __m256d __DEFAULT_FN_ATTRS 2519_mm256_undefined_pd(void) 2520{ 2521 return (__m256d)__builtin_ia32_undef256(); 2522} 2523 2524static __inline__ __m256 __DEFAULT_FN_ATTRS 2525_mm256_undefined_ps(void) 2526{ 2527 return (__m256)__builtin_ia32_undef256(); 2528} 2529 2530static __inline__ __m256i __DEFAULT_FN_ATTRS 2531_mm256_undefined_si256(void) 2532{ 2533 return (__m256i)__builtin_ia32_undef256(); 2534} 2535 2536static __inline __m256d __DEFAULT_FN_ATTRS 2537_mm256_set_pd(double __a, double __b, double __c, double __d) 2538{ 2539 return (__m256d){ __d, __c, __b, __a }; 2540} 2541 2542static __inline __m256 __DEFAULT_FN_ATTRS 2543_mm256_set_ps(float __a, float __b, float __c, float __d, 2544 float __e, float __f, float __g, float __h) 2545{ 2546 return (__m256){ __h, __g, __f, __e, __d, __c, __b, __a }; 2547} 2548 2549static __inline __m256i __DEFAULT_FN_ATTRS 2550_mm256_set_epi32(int __i0, int __i1, int __i2, int __i3, 2551 int __i4, int __i5, int __i6, int __i7) 2552{ 2553 return (__m256i)(__v8si){ __i7, __i6, __i5, __i4, __i3, __i2, __i1, __i0 }; 2554} 2555 2556static __inline __m256i __DEFAULT_FN_ATTRS 2557_mm256_set_epi16(short __w15, short __w14, short __w13, short __w12, 2558 short __w11, short __w10, short __w09, short __w08, 2559 short __w07, short __w06, short __w05, short __w04, 2560 short __w03, short __w02, short __w01, short __w00) 2561{ 2562 return (__m256i)(__v16hi){ __w00, __w01, __w02, __w03, __w04, __w05, __w06, 2563 __w07, __w08, __w09, __w10, __w11, __w12, __w13, __w14, __w15 }; 2564} 2565 2566static __inline __m256i __DEFAULT_FN_ATTRS 2567_mm256_set_epi8(char __b31, char __b30, char __b29, char __b28, 2568 char __b27, char __b26, char __b25, char __b24, 2569 char __b23, char __b22, char __b21, char __b20, 2570 char __b19, char __b18, char __b17, char __b16, 2571 char __b15, char __b14, char __b13, char __b12, 2572 char __b11, char __b10, char __b09, char __b08, 2573 char __b07, char __b06, char __b05, char __b04, 2574 char __b03, char __b02, char __b01, char __b00) 2575{ 2576 return (__m256i)(__v32qi){ 2577 __b00, __b01, __b02, __b03, __b04, __b05, __b06, __b07, 2578 __b08, __b09, __b10, __b11, __b12, __b13, __b14, __b15, 2579 __b16, __b17, __b18, __b19, __b20, __b21, __b22, __b23, 2580 __b24, __b25, __b26, __b27, __b28, __b29, __b30, __b31 2581 }; 2582} 2583 2584static __inline __m256i __DEFAULT_FN_ATTRS 2585_mm256_set_epi64x(long long __a, long long __b, long long __c, long long __d) 2586{ 2587 return (__m256i)(__v4di){ __d, __c, __b, __a }; 2588} 2589 2590/* Create vectors with elements in reverse order */ 2591static __inline __m256d __DEFAULT_FN_ATTRS 2592_mm256_setr_pd(double __a, double __b, double __c, double __d) 2593{ 2594 return (__m256d){ __a, __b, __c, __d }; 2595} 2596 2597static __inline __m256 __DEFAULT_FN_ATTRS 2598_mm256_setr_ps(float __a, float __b, float __c, float __d, 2599 float __e, float __f, float __g, float __h) 2600{ 2601 return (__m256){ __a, __b, __c, __d, __e, __f, __g, __h }; 2602} 2603 2604static __inline __m256i __DEFAULT_FN_ATTRS 2605_mm256_setr_epi32(int __i0, int __i1, int __i2, int __i3, 2606 int __i4, int __i5, int __i6, int __i7) 2607{ 2608 return (__m256i)(__v8si){ __i0, __i1, __i2, __i3, __i4, __i5, __i6, __i7 }; 2609} 2610 2611static __inline __m256i __DEFAULT_FN_ATTRS 2612_mm256_setr_epi16(short __w15, short __w14, short __w13, short __w12, 2613 short __w11, short __w10, short __w09, short __w08, 2614 short __w07, short __w06, short __w05, short __w04, 2615 short __w03, short __w02, short __w01, short __w00) 2616{ 2617 return (__m256i)(__v16hi){ __w15, __w14, __w13, __w12, __w11, __w10, __w09, 2618 __w08, __w07, __w06, __w05, __w04, __w03, __w02, __w01, __w00 }; 2619} 2620 2621static __inline __m256i __DEFAULT_FN_ATTRS 2622_mm256_setr_epi8(char __b31, char __b30, char __b29, char __b28, 2623 char __b27, char __b26, char __b25, char __b24, 2624 char __b23, char __b22, char __b21, char __b20, 2625 char __b19, char __b18, char __b17, char __b16, 2626 char __b15, char __b14, char __b13, char __b12, 2627 char __b11, char __b10, char __b09, char __b08, 2628 char __b07, char __b06, char __b05, char __b04, 2629 char __b03, char __b02, char __b01, char __b00) 2630{ 2631 return (__m256i)(__v32qi){ 2632 __b31, __b30, __b29, __b28, __b27, __b26, __b25, __b24, 2633 __b23, __b22, __b21, __b20, __b19, __b18, __b17, __b16, 2634 __b15, __b14, __b13, __b12, __b11, __b10, __b09, __b08, 2635 __b07, __b06, __b05, __b04, __b03, __b02, __b01, __b00 }; 2636} 2637 2638static __inline __m256i __DEFAULT_FN_ATTRS 2639_mm256_setr_epi64x(long long __a, long long __b, long long __c, long long __d) 2640{ 2641 return (__m256i)(__v4di){ __a, __b, __c, __d }; 2642} 2643 2644/* Create vectors with repeated elements */ 2645static __inline __m256d __DEFAULT_FN_ATTRS 2646_mm256_set1_pd(double __w) 2647{ 2648 return (__m256d){ __w, __w, __w, __w }; 2649} 2650 2651static __inline __m256 __DEFAULT_FN_ATTRS 2652_mm256_set1_ps(float __w) 2653{ 2654 return (__m256){ __w, __w, __w, __w, __w, __w, __w, __w }; 2655} 2656 2657static __inline __m256i __DEFAULT_FN_ATTRS 2658_mm256_set1_epi32(int __i) 2659{ 2660 return (__m256i)(__v8si){ __i, __i, __i, __i, __i, __i, __i, __i }; 2661} 2662 2663static __inline __m256i __DEFAULT_FN_ATTRS 2664_mm256_set1_epi16(short __w) 2665{ 2666 return (__m256i)(__v16hi){ __w, __w, __w, __w, __w, __w, __w, __w, __w, __w, 2667 __w, __w, __w, __w, __w, __w }; 2668} 2669 2670static __inline __m256i __DEFAULT_FN_ATTRS 2671_mm256_set1_epi8(char __b) 2672{ 2673 return (__m256i)(__v32qi){ __b, __b, __b, __b, __b, __b, __b, __b, __b, __b, 2674 __b, __b, __b, __b, __b, __b, __b, __b, __b, __b, __b, __b, __b, __b, __b, 2675 __b, __b, __b, __b, __b, __b, __b }; 2676} 2677 2678static __inline __m256i __DEFAULT_FN_ATTRS 2679_mm256_set1_epi64x(long long __q) 2680{ 2681 return (__m256i)(__v4di){ __q, __q, __q, __q }; 2682} 2683 2684/* Create __zeroed vectors */ 2685static __inline __m256d __DEFAULT_FN_ATTRS 2686_mm256_setzero_pd(void) 2687{ 2688 return (__m256d){ 0, 0, 0, 0 }; 2689} 2690 2691static __inline __m256 __DEFAULT_FN_ATTRS 2692_mm256_setzero_ps(void) 2693{ 2694 return (__m256){ 0, 0, 0, 0, 0, 0, 0, 0 }; 2695} 2696 2697static __inline __m256i __DEFAULT_FN_ATTRS 2698_mm256_setzero_si256(void) 2699{ 2700 return (__m256i){ 0LL, 0LL, 0LL, 0LL }; 2701} 2702 2703/* Cast between vector types */ 2704static __inline __m256 __DEFAULT_FN_ATTRS 2705_mm256_castpd_ps(__m256d __a) 2706{ 2707 return (__m256)__a; 2708} 2709 2710static __inline __m256i __DEFAULT_FN_ATTRS 2711_mm256_castpd_si256(__m256d __a) 2712{ 2713 return (__m256i)__a; 2714} 2715 2716static __inline __m256d __DEFAULT_FN_ATTRS 2717_mm256_castps_pd(__m256 __a) 2718{ 2719 return (__m256d)__a; 2720} 2721 2722static __inline __m256i __DEFAULT_FN_ATTRS 2723_mm256_castps_si256(__m256 __a) 2724{ 2725 return (__m256i)__a; 2726} 2727 2728static __inline __m256 __DEFAULT_FN_ATTRS 2729_mm256_castsi256_ps(__m256i __a) 2730{ 2731 return (__m256)__a; 2732} 2733 2734static __inline __m256d __DEFAULT_FN_ATTRS 2735_mm256_castsi256_pd(__m256i __a) 2736{ 2737 return (__m256d)__a; 2738} 2739 2740static __inline __m128d __DEFAULT_FN_ATTRS 2741_mm256_castpd256_pd128(__m256d __a) 2742{ 2743 return __builtin_shufflevector((__v4df)__a, (__v4df)__a, 0, 1); 2744} 2745 2746static __inline __m128 __DEFAULT_FN_ATTRS 2747_mm256_castps256_ps128(__m256 __a) 2748{ 2749 return __builtin_shufflevector((__v8sf)__a, (__v8sf)__a, 0, 1, 2, 3); 2750} 2751 2752static __inline __m128i __DEFAULT_FN_ATTRS 2753_mm256_castsi256_si128(__m256i __a) 2754{ 2755 return __builtin_shufflevector((__v4di)__a, (__v4di)__a, 0, 1); 2756} 2757 2758static __inline __m256d __DEFAULT_FN_ATTRS 2759_mm256_castpd128_pd256(__m128d __a) 2760{ 2761 return __builtin_shufflevector((__v2df)__a, (__v2df)__a, 0, 1, -1, -1); 2762} 2763 2764static __inline __m256 __DEFAULT_FN_ATTRS 2765_mm256_castps128_ps256(__m128 __a) 2766{ 2767 return __builtin_shufflevector((__v4sf)__a, (__v4sf)__a, 0, 1, 2, 3, -1, -1, -1, -1); 2768} 2769 2770static __inline __m256i __DEFAULT_FN_ATTRS 2771_mm256_castsi128_si256(__m128i __a) 2772{ 2773 return __builtin_shufflevector((__v2di)__a, (__v2di)__a, 0, 1, -1, -1); 2774} 2775 2776/* 2777 Vector insert. 2778 We use macros rather than inlines because we only want to accept 2779 invocations where the immediate M is a constant expression. 2780*/ 2781#define _mm256_insertf128_ps(V1, V2, M) __extension__ ({ \ 2782 (__m256)__builtin_shufflevector( \ 2783 (__v8sf)(__m256)(V1), \ 2784 (__v8sf)_mm256_castps128_ps256((__m128)(V2)), \ 2785 (((M) & 1) ? 0 : 8), \ 2786 (((M) & 1) ? 1 : 9), \ 2787 (((M) & 1) ? 2 : 10), \ 2788 (((M) & 1) ? 3 : 11), \ 2789 (((M) & 1) ? 8 : 4), \ 2790 (((M) & 1) ? 9 : 5), \ 2791 (((M) & 1) ? 10 : 6), \ 2792 (((M) & 1) ? 11 : 7) );}) 2793 2794#define _mm256_insertf128_pd(V1, V2, M) __extension__ ({ \ 2795 (__m256d)__builtin_shufflevector( \ 2796 (__v4df)(__m256d)(V1), \ 2797 (__v4df)_mm256_castpd128_pd256((__m128d)(V2)), \ 2798 (((M) & 1) ? 0 : 4), \ 2799 (((M) & 1) ? 1 : 5), \ 2800 (((M) & 1) ? 4 : 2), \ 2801 (((M) & 1) ? 5 : 3) );}) 2802 2803#define _mm256_insertf128_si256(V1, V2, M) __extension__ ({ \ 2804 (__m256i)__builtin_shufflevector( \ 2805 (__v4di)(__m256i)(V1), \ 2806 (__v4di)_mm256_castsi128_si256((__m128i)(V2)), \ 2807 (((M) & 1) ? 0 : 4), \ 2808 (((M) & 1) ? 1 : 5), \ 2809 (((M) & 1) ? 4 : 2), \ 2810 (((M) & 1) ? 5 : 3) );}) 2811 2812/* 2813 Vector extract. 2814 We use macros rather than inlines because we only want to accept 2815 invocations where the immediate M is a constant expression. 2816*/ 2817#define _mm256_extractf128_ps(V, M) __extension__ ({ \ 2818 (__m128)__builtin_shufflevector( \ 2819 (__v8sf)(__m256)(V), \ 2820 (__v8sf)(_mm256_undefined_ps()), \ 2821 (((M) & 1) ? 4 : 0), \ 2822 (((M) & 1) ? 5 : 1), \ 2823 (((M) & 1) ? 6 : 2), \ 2824 (((M) & 1) ? 7 : 3) );}) 2825 2826#define _mm256_extractf128_pd(V, M) __extension__ ({ \ 2827 (__m128d)__builtin_shufflevector( \ 2828 (__v4df)(__m256d)(V), \ 2829 (__v4df)(_mm256_undefined_pd()), \ 2830 (((M) & 1) ? 2 : 0), \ 2831 (((M) & 1) ? 3 : 1) );}) 2832 2833#define _mm256_extractf128_si256(V, M) __extension__ ({ \ 2834 (__m128i)__builtin_shufflevector( \ 2835 (__v4di)(__m256i)(V), \ 2836 (__v4di)(_mm256_undefined_si256()), \ 2837 (((M) & 1) ? 2 : 0), \ 2838 (((M) & 1) ? 3 : 1) );}) 2839 2840/* SIMD load ops (unaligned) */ 2841static __inline __m256 __DEFAULT_FN_ATTRS 2842_mm256_loadu2_m128(float const *__addr_hi, float const *__addr_lo) 2843{ 2844 __m256 __v256 = _mm256_castps128_ps256(_mm_loadu_ps(__addr_lo)); 2845 return _mm256_insertf128_ps(__v256, _mm_loadu_ps(__addr_hi), 1); 2846} 2847 2848static __inline __m256d __DEFAULT_FN_ATTRS 2849_mm256_loadu2_m128d(double const *__addr_hi, double const *__addr_lo) 2850{ 2851 __m256d __v256 = _mm256_castpd128_pd256(_mm_loadu_pd(__addr_lo)); 2852 return _mm256_insertf128_pd(__v256, _mm_loadu_pd(__addr_hi), 1); 2853} 2854 2855static __inline __m256i __DEFAULT_FN_ATTRS 2856_mm256_loadu2_m128i(__m128i const *__addr_hi, __m128i const *__addr_lo) 2857{ 2858 __m256i __v256 = _mm256_castsi128_si256(_mm_loadu_si128(__addr_lo)); 2859 return _mm256_insertf128_si256(__v256, _mm_loadu_si128(__addr_hi), 1); 2860} 2861 2862/* SIMD store ops (unaligned) */ 2863static __inline void __DEFAULT_FN_ATTRS 2864_mm256_storeu2_m128(float *__addr_hi, float *__addr_lo, __m256 __a) 2865{ 2866 __m128 __v128; 2867 2868 __v128 = _mm256_castps256_ps128(__a); 2869 _mm_storeu_ps(__addr_lo, __v128); 2870 __v128 = _mm256_extractf128_ps(__a, 1); 2871 _mm_storeu_ps(__addr_hi, __v128); 2872} 2873 2874static __inline void __DEFAULT_FN_ATTRS 2875_mm256_storeu2_m128d(double *__addr_hi, double *__addr_lo, __m256d __a) 2876{ 2877 __m128d __v128; 2878 2879 __v128 = _mm256_castpd256_pd128(__a); 2880 _mm_storeu_pd(__addr_lo, __v128); 2881 __v128 = _mm256_extractf128_pd(__a, 1); 2882 _mm_storeu_pd(__addr_hi, __v128); 2883} 2884 2885static __inline void __DEFAULT_FN_ATTRS 2886_mm256_storeu2_m128i(__m128i *__addr_hi, __m128i *__addr_lo, __m256i __a) 2887{ 2888 __m128i __v128; 2889 2890 __v128 = _mm256_castsi256_si128(__a); 2891 _mm_storeu_si128(__addr_lo, __v128); 2892 __v128 = _mm256_extractf128_si256(__a, 1); 2893 _mm_storeu_si128(__addr_hi, __v128); 2894} 2895 2896static __inline __m256 __DEFAULT_FN_ATTRS 2897_mm256_set_m128 (__m128 __hi, __m128 __lo) { 2898 return (__m256) __builtin_shufflevector((__v4sf)__lo, (__v4sf)__hi, 0, 1, 2, 3, 4, 5, 6, 7); 2899} 2900 2901static __inline __m256d __DEFAULT_FN_ATTRS 2902_mm256_set_m128d (__m128d __hi, __m128d __lo) { 2903 return (__m256d)_mm256_set_m128((__m128)__hi, (__m128)__lo); 2904} 2905 2906static __inline __m256i __DEFAULT_FN_ATTRS 2907_mm256_set_m128i (__m128i __hi, __m128i __lo) { 2908 return (__m256i)_mm256_set_m128((__m128)__hi, (__m128)__lo); 2909} 2910 2911static __inline __m256 __DEFAULT_FN_ATTRS 2912_mm256_setr_m128 (__m128 __lo, __m128 __hi) { 2913 return _mm256_set_m128(__hi, __lo); 2914} 2915 2916static __inline __m256d __DEFAULT_FN_ATTRS 2917_mm256_setr_m128d (__m128d __lo, __m128d __hi) { 2918 return (__m256d)_mm256_set_m128((__m128)__hi, (__m128)__lo); 2919} 2920 2921static __inline __m256i __DEFAULT_FN_ATTRS 2922_mm256_setr_m128i (__m128i __lo, __m128i __hi) { 2923 return (__m256i)_mm256_set_m128((__m128)__hi, (__m128)__lo); 2924} 2925 2926#undef __DEFAULT_FN_ATTRS 2927 2928#endif /* __AVXINTRIN_H */ 2929