Deleted Added
sdiff udiff text old ( 117395 ) new ( 122180 )
full compact
xmmintrin.h (117395) xmmintrin.h (122180)
1/* Copyright (C) 2002 Free Software Foundation, Inc.
1/* Copyright (C) 2002, 2003 Free Software Foundation, Inc.
2
3 This file is part of GNU CC.
4
5 GNU CC is free software; you can redistribute it and/or modify
6 it under the terms of the GNU General Public License as published by
7 the Free Software Foundation; either version 2, or (at your option)
8 any later version.
9

--- 10 unchanged lines hidden (view full) ---

20/* As a special exception, if you include this header file into source
21 files compiled by GCC, this header file does not by itself cause
22 the resulting executable to be covered by the GNU General Public
23 License. This exception does not however invalidate any other
24 reasons why the executable file might be covered by the GNU General
25 Public License. */
26
27/* Implemented from the specification included in the Intel C++ Compiler
2
3 This file is part of GNU CC.
4
5 GNU CC is free software; you can redistribute it and/or modify
6 it under the terms of the GNU General Public License as published by
7 the Free Software Foundation; either version 2, or (at your option)
8 any later version.
9

--- 10 unchanged lines hidden (view full) ---

20/* As a special exception, if you include this header file into source
21 files compiled by GCC, this header file does not by itself cause
22 the resulting executable to be covered by the GNU General Public
23 License. This exception does not however invalidate any other
24 reasons why the executable file might be covered by the GNU General
25 Public License. */
26
27/* Implemented from the specification included in the Intel C++ Compiler
28 User Guide and Reference, version 5.0. */
28 User Guide and Reference, version 8.0. */
29
30#ifndef _XMMINTRIN_H_INCLUDED
31#define _XMMINTRIN_H_INCLUDED
32
33#ifndef __SSE__
34# error "SSE instruction set not enabled"
35#else
36

--- 433 unchanged lines hidden (view full) ---

470/* Convert the lower SPFP value to a 32-bit integer according to the current
471 rounding mode. */
472static __inline int
473_mm_cvtss_si32 (__m128 __A)
474{
475 return __builtin_ia32_cvtss2si ((__v4sf) __A);
476}
477
29
30#ifndef _XMMINTRIN_H_INCLUDED
31#define _XMMINTRIN_H_INCLUDED
32
33#ifndef __SSE__
34# error "SSE instruction set not enabled"
35#else
36

--- 433 unchanged lines hidden (view full) ---

470/* Convert the lower SPFP value to a 32-bit integer according to the current
471 rounding mode. */
472static __inline int
473_mm_cvtss_si32 (__m128 __A)
474{
475 return __builtin_ia32_cvtss2si ((__v4sf) __A);
476}
477
478static __inline int
479_mm_cvt_ss2si (__m128 __A)
480{
481 return _mm_cvtss_si32 (__A);
482}
483
478#ifdef __x86_64__
479/* Convert the lower SPFP value to a 32-bit integer according to the current
480 rounding mode. */
481static __inline long long
482_mm_cvtss_si64x (__m128 __A)
483{
484 return __builtin_ia32_cvtss2si64 ((__v4sf) __A);
485}
486#endif
487
488/* Convert the two lower SPFP values to 32-bit integers according to the
489 current rounding mode. Return the integers in packed form. */
490static __inline __m64
491_mm_cvtps_pi32 (__m128 __A)
492{
493 return (__m64) __builtin_ia32_cvtps2pi ((__v4sf) __A);
494}
495
484#ifdef __x86_64__
485/* Convert the lower SPFP value to a 32-bit integer according to the current
486 rounding mode. */
487static __inline long long
488_mm_cvtss_si64x (__m128 __A)
489{
490 return __builtin_ia32_cvtss2si64 ((__v4sf) __A);
491}
492#endif
493
494/* Convert the two lower SPFP values to 32-bit integers according to the
495 current rounding mode. Return the integers in packed form. */
496static __inline __m64
497_mm_cvtps_pi32 (__m128 __A)
498{
499 return (__m64) __builtin_ia32_cvtps2pi ((__v4sf) __A);
500}
501
502static __inline __m64
503_mm_cvt_ps2pi (__m128 __A)
504{
505 return _mm_cvtps_pi32 (__A);
506}
507
496/* Truncate the lower SPFP value to a 32-bit integer. */
497static __inline int
498_mm_cvttss_si32 (__m128 __A)
499{
500 return __builtin_ia32_cvttss2si ((__v4sf) __A);
501}
502
508/* Truncate the lower SPFP value to a 32-bit integer. */
509static __inline int
510_mm_cvttss_si32 (__m128 __A)
511{
512 return __builtin_ia32_cvttss2si ((__v4sf) __A);
513}
514
515static __inline int
516_mm_cvtt_ss2si (__m128 __A)
517{
518 return _mm_cvttss_si32 (__A);
519}
520
503#ifdef __x86_64__
504/* Truncate the lower SPFP value to a 32-bit integer. */
505static __inline long long
506_mm_cvttss_si64x (__m128 __A)
507{
508 return __builtin_ia32_cvttss2si64 ((__v4sf) __A);
509}
510#endif
511
512/* Truncate the two lower SPFP values to 32-bit integers. Return the
513 integers in packed form. */
514static __inline __m64
515_mm_cvttps_pi32 (__m128 __A)
516{
517 return (__m64) __builtin_ia32_cvttps2pi ((__v4sf) __A);
518}
519
521#ifdef __x86_64__
522/* Truncate the lower SPFP value to a 32-bit integer. */
523static __inline long long
524_mm_cvttss_si64x (__m128 __A)
525{
526 return __builtin_ia32_cvttss2si64 ((__v4sf) __A);
527}
528#endif
529
530/* Truncate the two lower SPFP values to 32-bit integers. Return the
531 integers in packed form. */
532static __inline __m64
533_mm_cvttps_pi32 (__m128 __A)
534{
535 return (__m64) __builtin_ia32_cvttps2pi ((__v4sf) __A);
536}
537
538static __inline __m64
539_mm_cvtt_ps2pi (__m128 __A)
540{
541 return _mm_cvttps_pi32 (__A);
542}
543
520/* Convert B to a SPFP value and insert it as element zero in A. */
521static __inline __m128
522_mm_cvtsi32_ss (__m128 __A, int __B)
523{
524 return (__m128) __builtin_ia32_cvtsi2ss ((__v4sf) __A, __B);
525}
526
544/* Convert B to a SPFP value and insert it as element zero in A. */
545static __inline __m128
546_mm_cvtsi32_ss (__m128 __A, int __B)
547{
548 return (__m128) __builtin_ia32_cvtsi2ss ((__v4sf) __A, __B);
549}
550
551static __inline __m128
552_mm_cvt_si2ss (__m128 __A, int __B)
553{
554 return _mm_cvtsi32_ss (__A, __B);
555}
556
527#ifdef __x86_64__
528/* Convert B to a SPFP value and insert it as element zero in A. */
529static __inline __m128
530_mm_cvtsi64x_ss (__m128 __A, long long __B)
531{
532 return (__m128) __builtin_ia32_cvtsi642ss ((__v4sf) __A, __B);
533}
534#endif
535
536/* Convert the two 32-bit values in B to SPFP form and insert them
537 as the two lower elements in A. */
538static __inline __m128
539_mm_cvtpi32_ps (__m128 __A, __m64 __B)
540{
541 return (__m128) __builtin_ia32_cvtpi2ps ((__v4sf) __A, (__v2si)__B);
542}
543
557#ifdef __x86_64__
558/* Convert B to a SPFP value and insert it as element zero in A. */
559static __inline __m128
560_mm_cvtsi64x_ss (__m128 __A, long long __B)
561{
562 return (__m128) __builtin_ia32_cvtsi642ss ((__v4sf) __A, __B);
563}
564#endif
565
566/* Convert the two 32-bit values in B to SPFP form and insert them
567 as the two lower elements in A. */
568static __inline __m128
569_mm_cvtpi32_ps (__m128 __A, __m64 __B)
570{
571 return (__m128) __builtin_ia32_cvtpi2ps ((__v4sf) __A, (__v2si)__B);
572}
573
574static __inline __m128
575_mm_cvt_pi2ps (__m128 __A, __m64 __B)
576{
577 return _mm_cvtpi32_ps (__A, __B);
578}
579
544/* Convert the four signed 16-bit values in A to SPFP form. */
545static __inline __m128
546_mm_cvtpi16_ps (__m64 __A)
547{
548 __v4hi __sign;
549 __v2si __hisi, __losi;
550 __v4sf __r;
551

--- 385 unchanged lines hidden (view full) ---

937
938/* Extracts one of the four words of A. The selector N must be immediate. */
939#if 0
940static __inline int
941_mm_extract_pi16 (__m64 __A, int __N)
942{
943 return __builtin_ia32_pextrw ((__v4hi)__A, __N);
944}
580/* Convert the four signed 16-bit values in A to SPFP form. */
581static __inline __m128
582_mm_cvtpi16_ps (__m64 __A)
583{
584 __v4hi __sign;
585 __v2si __hisi, __losi;
586 __v4sf __r;
587

--- 385 unchanged lines hidden (view full) ---

973
974/* Extracts one of the four words of A. The selector N must be immediate. */
975#if 0
976static __inline int
977_mm_extract_pi16 (__m64 __A, int __N)
978{
979 return __builtin_ia32_pextrw ((__v4hi)__A, __N);
980}
981
982static __inline int
983_m_pextrw (__m64 __A, int __N)
984{
985 return _mm_extract_pi16 (__A, __N);
986}
945#else
946#define _mm_extract_pi16(A, N) \
947 __builtin_ia32_pextrw ((__v4hi)(A), (N))
987#else
988#define _mm_extract_pi16(A, N) \
989 __builtin_ia32_pextrw ((__v4hi)(A), (N))
990#define _m_pextrw(A, N) _mm_extract_pi16((A), (N))
948#endif
949
950/* Inserts word D into one of four words of A. The selector N must be
951 immediate. */
952#if 0
953static __inline __m64
954_mm_insert_pi16 (__m64 __A, int __D, int __N)
955{
956 return (__m64)__builtin_ia32_pinsrw ((__v4hi)__A, __D, __N);
957}
991#endif
992
993/* Inserts word D into one of four words of A. The selector N must be
994 immediate. */
995#if 0
996static __inline __m64
997_mm_insert_pi16 (__m64 __A, int __D, int __N)
998{
999 return (__m64)__builtin_ia32_pinsrw ((__v4hi)__A, __D, __N);
1000}
1001
1002static __inline __m64
1003_m_pinsrw (__m64 __A, int __D, int __N)
1004{
1005 return _mm_insert_pi16 (__A, __D, __N);
1006}
958#else
959#define _mm_insert_pi16(A, D, N) \
960 ((__m64) __builtin_ia32_pinsrw ((__v4hi)(A), (D), (N)))
1007#else
1008#define _mm_insert_pi16(A, D, N) \
1009 ((__m64) __builtin_ia32_pinsrw ((__v4hi)(A), (D), (N)))
1010#define _m_pinsrw(A, D, N) _mm_insert_pi16((A), (D), (N))
961#endif
962
963/* Compute the element-wise maximum of signed 16-bit values. */
964static __inline __m64
965_mm_max_pi16 (__m64 __A, __m64 __B)
966{
967 return (__m64) __builtin_ia32_pmaxsw ((__v4hi)__A, (__v4hi)__B);
968}
969
1011#endif
1012
1013/* Compute the element-wise maximum of signed 16-bit values. */
1014static __inline __m64
1015_mm_max_pi16 (__m64 __A, __m64 __B)
1016{
1017 return (__m64) __builtin_ia32_pmaxsw ((__v4hi)__A, (__v4hi)__B);
1018}
1019
1020static __inline __m64
1021_m_pmaxsw (__m64 __A, __m64 __B)
1022{
1023 return _mm_max_pi16 (__A, __B);
1024}
1025
970/* Compute the element-wise maximum of unsigned 8-bit values. */
971static __inline __m64
972_mm_max_pu8 (__m64 __A, __m64 __B)
973{
974 return (__m64) __builtin_ia32_pmaxub ((__v8qi)__A, (__v8qi)__B);
975}
976
1026/* Compute the element-wise maximum of unsigned 8-bit values. */
1027static __inline __m64
1028_mm_max_pu8 (__m64 __A, __m64 __B)
1029{
1030 return (__m64) __builtin_ia32_pmaxub ((__v8qi)__A, (__v8qi)__B);
1031}
1032
1033static __inline __m64
1034_m_pmaxub (__m64 __A, __m64 __B)
1035{
1036 return _mm_max_pu8 (__A, __B);
1037}
1038
977/* Compute the element-wise minimum of signed 16-bit values. */
978static __inline __m64
979_mm_min_pi16 (__m64 __A, __m64 __B)
980{
981 return (__m64) __builtin_ia32_pminsw ((__v4hi)__A, (__v4hi)__B);
982}
983
1039/* Compute the element-wise minimum of signed 16-bit values. */
1040static __inline __m64
1041_mm_min_pi16 (__m64 __A, __m64 __B)
1042{
1043 return (__m64) __builtin_ia32_pminsw ((__v4hi)__A, (__v4hi)__B);
1044}
1045
1046static __inline __m64
1047_m_pminsw (__m64 __A, __m64 __B)
1048{
1049 return _mm_min_pi16 (__A, __B);
1050}
1051
984/* Compute the element-wise minimum of unsigned 8-bit values. */
985static __inline __m64
986_mm_min_pu8 (__m64 __A, __m64 __B)
987{
988 return (__m64) __builtin_ia32_pminub ((__v8qi)__A, (__v8qi)__B);
989}
990
1052/* Compute the element-wise minimum of unsigned 8-bit values. */
1053static __inline __m64
1054_mm_min_pu8 (__m64 __A, __m64 __B)
1055{
1056 return (__m64) __builtin_ia32_pminub ((__v8qi)__A, (__v8qi)__B);
1057}
1058
1059static __inline __m64
1060_m_pminub (__m64 __A, __m64 __B)
1061{
1062 return _mm_min_pu8 (__A, __B);
1063}
1064
991/* Create an 8-bit mask of the signs of 8-bit values. */
992static __inline int
993_mm_movemask_pi8 (__m64 __A)
994{
995 return __builtin_ia32_pmovmskb ((__v8qi)__A);
996}
997
1065/* Create an 8-bit mask of the signs of 8-bit values. */
1066static __inline int
1067_mm_movemask_pi8 (__m64 __A)
1068{
1069 return __builtin_ia32_pmovmskb ((__v8qi)__A);
1070}
1071
1072static __inline int
1073_m_pmovmskb (__m64 __A)
1074{
1075 return _mm_movemask_pi8 (__A);
1076}
1077
998/* Multiply four unsigned 16-bit values in A by four unsigned 16-bit values
999 in B and produce the high 16 bits of the 32-bit results. */
1000static __inline __m64
1001_mm_mulhi_pu16 (__m64 __A, __m64 __B)
1002{
1003 return (__m64) __builtin_ia32_pmulhuw ((__v4hi)__A, (__v4hi)__B);
1004}
1005
1078/* Multiply four unsigned 16-bit values in A by four unsigned 16-bit values
1079 in B and produce the high 16 bits of the 32-bit results. */
1080static __inline __m64
1081_mm_mulhi_pu16 (__m64 __A, __m64 __B)
1082{
1083 return (__m64) __builtin_ia32_pmulhuw ((__v4hi)__A, (__v4hi)__B);
1084}
1085
1086static __inline __m64
1087_m_pmulhuw (__m64 __A, __m64 __B)
1088{
1089 return _mm_mulhi_pu16 (__A, __B);
1090}
1091
1006/* Return a combination of the four 16-bit values in A. The selector
1007 must be an immediate. */
1008#if 0
1009static __inline __m64
1010_mm_shuffle_pi16 (__m64 __A, int __N)
1011{
1012 return (__m64) __builtin_ia32_pshufw ((__v4hi)__A, __N);
1013}
1092/* Return a combination of the four 16-bit values in A. The selector
1093 must be an immediate. */
1094#if 0
1095static __inline __m64
1096_mm_shuffle_pi16 (__m64 __A, int __N)
1097{
1098 return (__m64) __builtin_ia32_pshufw ((__v4hi)__A, __N);
1099}
1100
1101static __inline __m64
1102_m_pshufw (__m64 __A, int __N)
1103{
1104 return _mm_shuffle_pi16 (__A, __N);
1105}
1014#else
1015#define _mm_shuffle_pi16(A, N) \
1016 ((__m64) __builtin_ia32_pshufw ((__v4hi)(A), (N)))
1106#else
1107#define _mm_shuffle_pi16(A, N) \
1108 ((__m64) __builtin_ia32_pshufw ((__v4hi)(A), (N)))
1109#define _m_pshufw(A, N) _mm_shuffle_pi16 ((A), (N))
1017#endif
1018
1019/* Conditionally store byte elements of A into P. The high bit of each
1020 byte in the selector N determines whether the corresponding byte from
1021 A is stored. */
1022static __inline void
1023_mm_maskmove_si64 (__m64 __A, __m64 __N, char *__P)
1024{
1025 __builtin_ia32_maskmovq ((__v8qi)__A, (__v8qi)__N, __P);
1026}
1027
1110#endif
1111
1112/* Conditionally store byte elements of A into P. The high bit of each
1113 byte in the selector N determines whether the corresponding byte from
1114 A is stored. */
1115static __inline void
1116_mm_maskmove_si64 (__m64 __A, __m64 __N, char *__P)
1117{
1118 __builtin_ia32_maskmovq ((__v8qi)__A, (__v8qi)__N, __P);
1119}
1120
1121static __inline void
1122_m_maskmovq (__m64 __A, __m64 __N, char *__P)
1123{
1124 _mm_maskmove_si64 (__A, __N, __P);
1125}
1126
1028/* Compute the rounded averages of the unsigned 8-bit values in A and B. */
1029static __inline __m64
1030_mm_avg_pu8 (__m64 __A, __m64 __B)
1031{
1032 return (__m64) __builtin_ia32_pavgb ((__v8qi)__A, (__v8qi)__B);
1033}
1034
1127/* Compute the rounded averages of the unsigned 8-bit values in A and B. */
1128static __inline __m64
1129_mm_avg_pu8 (__m64 __A, __m64 __B)
1130{
1131 return (__m64) __builtin_ia32_pavgb ((__v8qi)__A, (__v8qi)__B);
1132}
1133
1134static __inline __m64
1135_m_pavgb (__m64 __A, __m64 __B)
1136{
1137 return _mm_avg_pu8 (__A, __B);
1138}
1139
1035/* Compute the rounded averages of the unsigned 16-bit values in A and B. */
1036static __inline __m64
1037_mm_avg_pu16 (__m64 __A, __m64 __B)
1038{
1039 return (__m64) __builtin_ia32_pavgw ((__v4hi)__A, (__v4hi)__B);
1040}
1041
1140/* Compute the rounded averages of the unsigned 16-bit values in A and B. */
1141static __inline __m64
1142_mm_avg_pu16 (__m64 __A, __m64 __B)
1143{
1144 return (__m64) __builtin_ia32_pavgw ((__v4hi)__A, (__v4hi)__B);
1145}
1146
1147static __inline __m64
1148_m_pavgw (__m64 __A, __m64 __B)
1149{
1150 return _mm_avg_pu16 (__A, __B);
1151}
1152
1042/* Compute the sum of the absolute differences of the unsigned 8-bit
1043 values in A and B. Return the value in the lower 16-bit word; the
1044 upper words are cleared. */
1045static __inline __m64
1046_mm_sad_pu8 (__m64 __A, __m64 __B)
1047{
1048 return (__m64) __builtin_ia32_psadbw ((__v8qi)__A, (__v8qi)__B);
1049}
1050
1153/* Compute the sum of the absolute differences of the unsigned 8-bit
1154 values in A and B. Return the value in the lower 16-bit word; the
1155 upper words are cleared. */
1156static __inline __m64
1157_mm_sad_pu8 (__m64 __A, __m64 __B)
1158{
1159 return (__m64) __builtin_ia32_psadbw ((__v8qi)__A, (__v8qi)__B);
1160}
1161
1162static __inline __m64
1163_m_psadbw (__m64 __A, __m64 __B)
1164{
1165 return _mm_sad_pu8 (__A, __B);
1166}
1167
1051/* Loads one cache line from address P to a location "closer" to the
1052 processor. The selector I specifies the type of prefetch operation. */
1053#if 0
1054static __inline void
1055_mm_prefetch (void *__P, enum _mm_hint __I)
1056{
1057 __builtin_prefetch (__P, 0, __I);
1058}

--- 42 unchanged lines hidden (view full) ---

1101 __v4sf __t1 = __builtin_ia32_shufps (__r2, __r3, 0x44); \
1102 __v4sf __t3 = __builtin_ia32_shufps (__r2, __r3, 0xEE); \
1103 (row0) = __builtin_ia32_shufps (__t0, __t1, 0x88); \
1104 (row1) = __builtin_ia32_shufps (__t0, __t1, 0xDD); \
1105 (row2) = __builtin_ia32_shufps (__t2, __t3, 0x88); \
1106 (row3) = __builtin_ia32_shufps (__t2, __t3, 0xDD); \
1107} while (0)
1108
1168/* Loads one cache line from address P to a location "closer" to the
1169 processor. The selector I specifies the type of prefetch operation. */
1170#if 0
1171static __inline void
1172_mm_prefetch (void *__P, enum _mm_hint __I)
1173{
1174 __builtin_prefetch (__P, 0, __I);
1175}

--- 42 unchanged lines hidden (view full) ---

1218 __v4sf __t1 = __builtin_ia32_shufps (__r2, __r3, 0x44); \
1219 __v4sf __t3 = __builtin_ia32_shufps (__r2, __r3, 0xEE); \
1220 (row0) = __builtin_ia32_shufps (__t0, __t1, 0x88); \
1221 (row1) = __builtin_ia32_shufps (__t0, __t1, 0xDD); \
1222 (row2) = __builtin_ia32_shufps (__t2, __t3, 0x88); \
1223 (row3) = __builtin_ia32_shufps (__t2, __t3, 0xDD); \
1224} while (0)
1225
1109#ifdef __SSE2__
1110/* SSE2 */
1111typedef int __v2df __attribute__ ((mode (V2DF)));
1112typedef int __v2di __attribute__ ((mode (V2DI)));
1113typedef int __v4si __attribute__ ((mode (V4SI)));
1114typedef int __v8hi __attribute__ ((mode (V8HI)));
1115typedef int __v16qi __attribute__ ((mode (V16QI)));
1226/* For backward source compatibility. */
1227#include <emmintrin.h>
1116
1228
1117/* Create a selector for use with the SHUFPD instruction. */
1118#define _MM_SHUFFLE2(fp1,fp0) \
1119 (((fp1) << 1) | (fp0))
1120
1121#define __m128i __v2di
1122#define __m128d __v2df
1123
1124/* Create a vector with element 0 as *P and the rest zero. */
1125static __inline __m128d
1126_mm_load_sd (double const *__P)
1127{
1128 return (__m128d) __builtin_ia32_loadsd (__P);
1129}
1130
1131/* Create a vector with all two elements equal to *P. */
1132static __inline __m128d
1133_mm_load1_pd (double const *__P)
1134{
1135 __v2df __tmp = __builtin_ia32_loadsd (__P);
1136 return (__m128d) __builtin_ia32_shufpd (__tmp, __tmp, _MM_SHUFFLE2 (0,0));
1137}
1138
1139static __inline __m128d
1140_mm_load_pd1 (double const *__P)
1141{
1142 return _mm_load1_pd (__P);
1143}
1144
1145/* Load two DPFP values from P. The addresd must be 16-byte aligned. */
1146static __inline __m128d
1147_mm_load_pd (double const *__P)
1148{
1149 return (__m128d) __builtin_ia32_loadapd (__P);
1150}
1151
1152/* Load two DPFP values from P. The addresd need not be 16-byte aligned. */
1153static __inline __m128d
1154_mm_loadu_pd (double const *__P)
1155{
1156 return (__m128d) __builtin_ia32_loadupd (__P);
1157}
1158
1159/* Load two DPFP values in reverse order. The addresd must be aligned. */
1160static __inline __m128d
1161_mm_loadr_pd (double const *__P)
1162{
1163 __v2df __tmp = __builtin_ia32_loadapd (__P);
1164 return (__m128d) __builtin_ia32_shufpd (__tmp, __tmp, _MM_SHUFFLE2 (0,1));
1165}
1166
1167/* Create a vector with element 0 as F and the rest zero. */
1168static __inline __m128d
1169_mm_set_sd (double __F)
1170{
1171 return (__m128d) __builtin_ia32_loadsd (&__F);
1172}
1173
1174/* Create a vector with all two elements equal to F. */
1175static __inline __m128d
1176_mm_set1_pd (double __F)
1177{
1178 __v2df __tmp = __builtin_ia32_loadsd (&__F);
1179 return (__m128d) __builtin_ia32_shufpd (__tmp, __tmp, _MM_SHUFFLE2 (0,0));
1180}
1181
1182static __inline __m128d
1183_mm_set_pd1 (double __F)
1184{
1185 return _mm_set1_pd (__F);
1186}
1187
1188/* Create the vector [Z Y]. */
1189static __inline __m128d
1190_mm_set_pd (double __Z, double __Y)
1191{
1192 union {
1193 double __a[2];
1194 __m128d __v;
1195 } __u;
1196
1197 __u.__a[0] = __Y;
1198 __u.__a[1] = __Z;
1199
1200 return __u.__v;
1201}
1202
1203/* Create the vector [Y Z]. */
1204static __inline __m128d
1205_mm_setr_pd (double __Z, double __Y)
1206{
1207 return _mm_set_pd (__Y, __Z);
1208}
1209
1210/* Create a vector of zeros. */
1211static __inline __m128d
1212_mm_setzero_pd (void)
1213{
1214 return (__m128d) __builtin_ia32_setzeropd ();
1215}
1216
1217/* Stores the lower DPFP value. */
1218static __inline void
1219_mm_store_sd (double *__P, __m128d __A)
1220{
1221 __builtin_ia32_storesd (__P, (__v2df)__A);
1222}
1223
1224/* Store the lower DPFP value acrosd two words. */
1225static __inline void
1226_mm_store1_pd (double *__P, __m128d __A)
1227{
1228 __v2df __va = (__v2df)__A;
1229 __v2df __tmp = __builtin_ia32_shufpd (__va, __va, _MM_SHUFFLE2 (0,0));
1230 __builtin_ia32_storeapd (__P, __tmp);
1231}
1232
1233static __inline void
1234_mm_store_pd1 (double *__P, __m128d __A)
1235{
1236 _mm_store1_pd (__P, __A);
1237}
1238
1239/* Store two DPFP values. The addresd must be 16-byte aligned. */
1240static __inline void
1241_mm_store_pd (double *__P, __m128d __A)
1242{
1243 __builtin_ia32_storeapd (__P, (__v2df)__A);
1244}
1245
1246/* Store two DPFP values. The addresd need not be 16-byte aligned. */
1247static __inline void
1248_mm_storeu_pd (double *__P, __m128d __A)
1249{
1250 __builtin_ia32_storeupd (__P, (__v2df)__A);
1251}
1252
1253/* Store two DPFP values in reverse order. The addresd must be aligned. */
1254static __inline void
1255_mm_storer_pd (double *__P, __m128d __A)
1256{
1257 __v2df __va = (__v2df)__A;
1258 __v2df __tmp = __builtin_ia32_shufpd (__va, __va, _MM_SHUFFLE2 (0,1));
1259 __builtin_ia32_storeapd (__P, __tmp);
1260}
1261
1262/* Sets the low DPFP value of A from the low value of B. */
1263static __inline __m128d
1264_mm_move_sd (__m128d __A, __m128d __B)
1265{
1266 return (__m128d) __builtin_ia32_movsd ((__v2df)__A, (__v2df)__B);
1267}
1268
1269
1270static __inline __m128d
1271_mm_add_pd (__m128d __A, __m128d __B)
1272{
1273 return (__m128d)__builtin_ia32_addpd ((__v2df)__A, (__v2df)__B);
1274}
1275
1276static __inline __m128d
1277_mm_add_sd (__m128d __A, __m128d __B)
1278{
1279 return (__m128d)__builtin_ia32_addsd ((__v2df)__A, (__v2df)__B);
1280}
1281
1282static __inline __m128d
1283_mm_sub_pd (__m128d __A, __m128d __B)
1284{
1285 return (__m128d)__builtin_ia32_subpd ((__v2df)__A, (__v2df)__B);
1286}
1287
1288static __inline __m128d
1289_mm_sub_sd (__m128d __A, __m128d __B)
1290{
1291 return (__m128d)__builtin_ia32_subsd ((__v2df)__A, (__v2df)__B);
1292}
1293
1294static __inline __m128d
1295_mm_mul_pd (__m128d __A, __m128d __B)
1296{
1297 return (__m128d)__builtin_ia32_mulpd ((__v2df)__A, (__v2df)__B);
1298}
1299
1300static __inline __m128d
1301_mm_mul_sd (__m128d __A, __m128d __B)
1302{
1303 return (__m128d)__builtin_ia32_mulsd ((__v2df)__A, (__v2df)__B);
1304}
1305
1306static __inline __m128d
1307_mm_div_pd (__m128d __A, __m128d __B)
1308{
1309 return (__m128d)__builtin_ia32_divpd ((__v2df)__A, (__v2df)__B);
1310}
1311
1312static __inline __m128d
1313_mm_div_sd (__m128d __A, __m128d __B)
1314{
1315 return (__m128d)__builtin_ia32_divsd ((__v2df)__A, (__v2df)__B);
1316}
1317
1318static __inline __m128d
1319_mm_sqrt_pd (__m128d __A)
1320{
1321 return (__m128d)__builtin_ia32_sqrtpd ((__v2df)__A);
1322}
1323
1324/* Return pair {sqrt (A[0), B[1]}. */
1325static __inline __m128d
1326_mm_sqrt_sd (__m128d __A, __m128d __B)
1327{
1328 __v2df __tmp = __builtin_ia32_movsd ((__v2df)__A, (__v2df)__B);
1329 return (__m128d)__builtin_ia32_sqrtsd ((__v2df)__tmp);
1330}
1331
1332static __inline __m128d
1333_mm_min_pd (__m128d __A, __m128d __B)
1334{
1335 return (__m128d)__builtin_ia32_minpd ((__v2df)__A, (__v2df)__B);
1336}
1337
1338static __inline __m128d
1339_mm_min_sd (__m128d __A, __m128d __B)
1340{
1341 return (__m128d)__builtin_ia32_minsd ((__v2df)__A, (__v2df)__B);
1342}
1343
1344static __inline __m128d
1345_mm_max_pd (__m128d __A, __m128d __B)
1346{
1347 return (__m128d)__builtin_ia32_maxpd ((__v2df)__A, (__v2df)__B);
1348}
1349
1350static __inline __m128d
1351_mm_max_sd (__m128d __A, __m128d __B)
1352{
1353 return (__m128d)__builtin_ia32_maxsd ((__v2df)__A, (__v2df)__B);
1354}
1355
1356static __inline __m128d
1357_mm_and_pd (__m128d __A, __m128d __B)
1358{
1359 return (__m128d)__builtin_ia32_andpd ((__v2df)__A, (__v2df)__B);
1360}
1361
1362static __inline __m128d
1363_mm_andnot_pd (__m128d __A, __m128d __B)
1364{
1365 return (__m128d)__builtin_ia32_andnpd ((__v2df)__A, (__v2df)__B);
1366}
1367
1368static __inline __m128d
1369_mm_or_pd (__m128d __A, __m128d __B)
1370{
1371 return (__m128d)__builtin_ia32_orpd ((__v2df)__A, (__v2df)__B);
1372}
1373
1374static __inline __m128d
1375_mm_xor_pd (__m128d __A, __m128d __B)
1376{
1377 return (__m128d)__builtin_ia32_xorpd ((__v2df)__A, (__v2df)__B);
1378}
1379
1380static __inline __m128d
1381_mm_cmpeq_pd (__m128d __A, __m128d __B)
1382{
1383 return (__m128d)__builtin_ia32_cmpeqpd ((__v2df)__A, (__v2df)__B);
1384}
1385
1386static __inline __m128d
1387_mm_cmplt_pd (__m128d __A, __m128d __B)
1388{
1389 return (__m128d)__builtin_ia32_cmpltpd ((__v2df)__A, (__v2df)__B);
1390}
1391
1392static __inline __m128d
1393_mm_cmple_pd (__m128d __A, __m128d __B)
1394{
1395 return (__m128d)__builtin_ia32_cmplepd ((__v2df)__A, (__v2df)__B);
1396}
1397
1398static __inline __m128d
1399_mm_cmpgt_pd (__m128d __A, __m128d __B)
1400{
1401 return (__m128d)__builtin_ia32_cmpgtpd ((__v2df)__A, (__v2df)__B);
1402}
1403
1404static __inline __m128d
1405_mm_cmpge_pd (__m128d __A, __m128d __B)
1406{
1407 return (__m128d)__builtin_ia32_cmpgepd ((__v2df)__A, (__v2df)__B);
1408}
1409
1410static __inline __m128d
1411_mm_cmpneq_pd (__m128d __A, __m128d __B)
1412{
1413 return (__m128d)__builtin_ia32_cmpneqpd ((__v2df)__A, (__v2df)__B);
1414}
1415
1416static __inline __m128d
1417_mm_cmpnlt_pd (__m128d __A, __m128d __B)
1418{
1419 return (__m128d)__builtin_ia32_cmpnltpd ((__v2df)__A, (__v2df)__B);
1420}
1421
1422static __inline __m128d
1423_mm_cmpnle_pd (__m128d __A, __m128d __B)
1424{
1425 return (__m128d)__builtin_ia32_cmpnlepd ((__v2df)__A, (__v2df)__B);
1426}
1427
1428static __inline __m128d
1429_mm_cmpngt_pd (__m128d __A, __m128d __B)
1430{
1431 return (__m128d)__builtin_ia32_cmpngtpd ((__v2df)__A, (__v2df)__B);
1432}
1433
1434static __inline __m128d
1435_mm_cmpnge_pd (__m128d __A, __m128d __B)
1436{
1437 return (__m128d)__builtin_ia32_cmpngepd ((__v2df)__A, (__v2df)__B);
1438}
1439
1440static __inline __m128d
1441_mm_cmpord_pd (__m128d __A, __m128d __B)
1442{
1443 return (__m128d)__builtin_ia32_cmpordpd ((__v2df)__A, (__v2df)__B);
1444}
1445
1446static __inline __m128d
1447_mm_cmpunord_pd (__m128d __A, __m128d __B)
1448{
1449 return (__m128d)__builtin_ia32_cmpunordpd ((__v2df)__A, (__v2df)__B);
1450}
1451
1452static __inline __m128d
1453_mm_cmpeq_sd (__m128d __A, __m128d __B)
1454{
1455 return (__m128d)__builtin_ia32_cmpeqsd ((__v2df)__A, (__v2df)__B);
1456}
1457
1458static __inline __m128d
1459_mm_cmplt_sd (__m128d __A, __m128d __B)
1460{
1461 return (__m128d)__builtin_ia32_cmpltsd ((__v2df)__A, (__v2df)__B);
1462}
1463
1464static __inline __m128d
1465_mm_cmple_sd (__m128d __A, __m128d __B)
1466{
1467 return (__m128d)__builtin_ia32_cmplesd ((__v2df)__A, (__v2df)__B);
1468}
1469
1470static __inline __m128d
1471_mm_cmpgt_sd (__m128d __A, __m128d __B)
1472{
1473 return (__m128d) __builtin_ia32_movsd ((__v2df) __A,
1474 (__v2df)
1475 __builtin_ia32_cmpltsd ((__v2df) __B,
1476 (__v2df)
1477 __A));
1478}
1479
1480static __inline __m128d
1481_mm_cmpge_sd (__m128d __A, __m128d __B)
1482{
1483 return (__m128d) __builtin_ia32_movsd ((__v2df) __A,
1484 (__v2df)
1485 __builtin_ia32_cmplesd ((__v2df) __B,
1486 (__v2df)
1487 __A));
1488}
1489
1490static __inline __m128d
1491_mm_cmpneq_sd (__m128d __A, __m128d __B)
1492{
1493 return (__m128d)__builtin_ia32_cmpneqsd ((__v2df)__A, (__v2df)__B);
1494}
1495
1496static __inline __m128d
1497_mm_cmpnlt_sd (__m128d __A, __m128d __B)
1498{
1499 return (__m128d)__builtin_ia32_cmpnltsd ((__v2df)__A, (__v2df)__B);
1500}
1501
1502static __inline __m128d
1503_mm_cmpnle_sd (__m128d __A, __m128d __B)
1504{
1505 return (__m128d)__builtin_ia32_cmpnlesd ((__v2df)__A, (__v2df)__B);
1506}
1507
1508static __inline __m128d
1509_mm_cmpngt_sd (__m128d __A, __m128d __B)
1510{
1511 return (__m128d) __builtin_ia32_movsd ((__v2df) __A,
1512 (__v2df)
1513 __builtin_ia32_cmpnltsd ((__v2df) __B,
1514 (__v2df)
1515 __A));
1516}
1517
1518static __inline __m128d
1519_mm_cmpnge_sd (__m128d __A, __m128d __B)
1520{
1521 return (__m128d) __builtin_ia32_movsd ((__v2df) __A,
1522 (__v2df)
1523 __builtin_ia32_cmpnlesd ((__v2df) __B,
1524 (__v2df)
1525 __A));
1526}
1527
1528static __inline __m128d
1529_mm_cmpord_sd (__m128d __A, __m128d __B)
1530{
1531 return (__m128d)__builtin_ia32_cmpordsd ((__v2df)__A, (__v2df)__B);
1532}
1533
1534static __inline __m128d
1535_mm_cmpunord_sd (__m128d __A, __m128d __B)
1536{
1537 return (__m128d)__builtin_ia32_cmpunordsd ((__v2df)__A, (__v2df)__B);
1538}
1539
1540static __inline int
1541_mm_comieq_sd (__m128d __A, __m128d __B)
1542{
1543 return __builtin_ia32_comisdeq ((__v2df)__A, (__v2df)__B);
1544}
1545
1546static __inline int
1547_mm_comilt_sd (__m128d __A, __m128d __B)
1548{
1549 return __builtin_ia32_comisdlt ((__v2df)__A, (__v2df)__B);
1550}
1551
1552static __inline int
1553_mm_comile_sd (__m128d __A, __m128d __B)
1554{
1555 return __builtin_ia32_comisdle ((__v2df)__A, (__v2df)__B);
1556}
1557
1558static __inline int
1559_mm_comigt_sd (__m128d __A, __m128d __B)
1560{
1561 return __builtin_ia32_comisdgt ((__v2df)__A, (__v2df)__B);
1562}
1563
1564static __inline int
1565_mm_comige_sd (__m128d __A, __m128d __B)
1566{
1567 return __builtin_ia32_comisdge ((__v2df)__A, (__v2df)__B);
1568}
1569
1570static __inline int
1571_mm_comineq_sd (__m128d __A, __m128d __B)
1572{
1573 return __builtin_ia32_comisdneq ((__v2df)__A, (__v2df)__B);
1574}
1575
1576static __inline int
1577_mm_ucomieq_sd (__m128d __A, __m128d __B)
1578{
1579 return __builtin_ia32_ucomisdeq ((__v2df)__A, (__v2df)__B);
1580}
1581
1582static __inline int
1583_mm_ucomilt_sd (__m128d __A, __m128d __B)
1584{
1585 return __builtin_ia32_ucomisdlt ((__v2df)__A, (__v2df)__B);
1586}
1587
1588static __inline int
1589_mm_ucomile_sd (__m128d __A, __m128d __B)
1590{
1591 return __builtin_ia32_ucomisdle ((__v2df)__A, (__v2df)__B);
1592}
1593
1594static __inline int
1595_mm_ucomigt_sd (__m128d __A, __m128d __B)
1596{
1597 return __builtin_ia32_ucomisdgt ((__v2df)__A, (__v2df)__B);
1598}
1599
1600static __inline int
1601_mm_ucomige_sd (__m128d __A, __m128d __B)
1602{
1603 return __builtin_ia32_ucomisdge ((__v2df)__A, (__v2df)__B);
1604}
1605
1606static __inline int
1607_mm_ucomineq_sd (__m128d __A, __m128d __B)
1608{
1609 return __builtin_ia32_ucomisdneq ((__v2df)__A, (__v2df)__B);
1610}
1611
1612/* Create a vector with element 0 as *P and the rest zero. */
1613
1614static __inline __m128i
1615_mm_load_si128 (__m128i const *__P)
1616{
1617 return (__m128i) __builtin_ia32_loaddqa ((char const *)__P);
1618}
1619
1620static __inline __m128i
1621_mm_loadu_si128 (__m128i const *__P)
1622{
1623 return (__m128i) __builtin_ia32_loaddqu ((char const *)__P);
1624}
1625
1626static __inline __m128i
1627_mm_loadl_epi64 (__m128i const *__P)
1628{
1629 return (__m128i) __builtin_ia32_movq2dq (*(unsigned long long *)__P);
1630}
1631
1632static __inline void
1633_mm_store_si128 (__m128i *__P, __m128i __B)
1634{
1635 __builtin_ia32_storedqa ((char *)__P, (__v16qi)__B);
1636}
1637
1638static __inline void
1639_mm_storeu_si128 (__m128i *__P, __m128i __B)
1640{
1641 __builtin_ia32_storedqu ((char *)__P, (__v16qi)__B);
1642}
1643
1644static __inline void
1645_mm_storel_epi64 (__m128i *__P, __m128i __B)
1646{
1647 *(long long *)__P = __builtin_ia32_movdq2q ((__v2di)__B);
1648}
1649
1650static __inline __m64
1651_mm_movepi64_pi64 (__m128i __B)
1652{
1653 return (__m64) __builtin_ia32_movdq2q ((__v2di)__B);
1654}
1655
1656static __inline __m128i
1657_mm_move_epi64 (__m128i __A)
1658{
1659 return (__m128i) __builtin_ia32_movq ((__v2di)__A);
1660}
1661
1662/* Create a vector of zeros. */
1663static __inline __m128i
1664_mm_setzero_si128 (void)
1665{
1666 return (__m128i) __builtin_ia32_setzero128 ();
1667}
1668
1669static __inline __m128i
1670_mm_set_epi64 (__m64 __A, __m64 __B)
1671{
1672 __v2di __tmp = (__v2di)__builtin_ia32_movq2dq ((unsigned long long)__A);
1673 __v2di __tmp2 = (__v2di)__builtin_ia32_movq2dq ((unsigned long long)__B);
1674 return (__m128i)__builtin_ia32_punpcklqdq128 (__tmp2, __tmp);
1675}
1676
1677/* Create the vector [Z Y X W]. */
1678static __inline __m128i
1679_mm_set_epi32 (int __Z, int __Y, int __X, int __W)
1680{
1681 union {
1682 int __a[4];
1683 __m128i __v;
1684 } __u;
1685
1686 __u.__a[0] = __W;
1687 __u.__a[1] = __X;
1688 __u.__a[2] = __Y;
1689 __u.__a[3] = __Z;
1690
1691 return __u.__v;
1692}
1693
1694#ifdef __x86_64__
1695/* Create the vector [Z Y]. */
1696static __inline __m128i
1697_mm_set_epi64x (long long __Z, long long __Y)
1698{
1699 union {
1700 long __a[2];
1701 __m128i __v;
1702 } __u;
1703
1704 __u.__a[0] = __Y;
1705 __u.__a[1] = __Z;
1706
1707 return __u.__v;
1708}
1709#endif
1710
1711/* Create the vector [S T U V Z Y X W]. */
1712static __inline __m128i
1713_mm_set_epi16 (short __Z, short __Y, short __X, short __W,
1714 short __V, short __U, short __T, short __S)
1715{
1716 union {
1717 short __a[8];
1718 __m128i __v;
1719 } __u;
1720
1721 __u.__a[0] = __S;
1722 __u.__a[1] = __T;
1723 __u.__a[2] = __U;
1724 __u.__a[3] = __V;
1725 __u.__a[4] = __W;
1726 __u.__a[5] = __X;
1727 __u.__a[6] = __Y;
1728 __u.__a[7] = __Z;
1729
1730 return __u.__v;
1731}
1732
1733/* Create the vector [S T U V Z Y X W]. */
1734static __inline __m128i
1735_mm_set_epi8 (char __Z, char __Y, char __X, char __W,
1736 char __V, char __U, char __T, char __S,
1737 char __Z1, char __Y1, char __X1, char __W1,
1738 char __V1, char __U1, char __T1, char __S1)
1739{
1740 union {
1741 char __a[16];
1742 __m128i __v;
1743 } __u;
1744
1745 __u.__a[0] = __S1;
1746 __u.__a[1] = __T1;
1747 __u.__a[2] = __U1;
1748 __u.__a[3] = __V1;
1749 __u.__a[4] = __W1;
1750 __u.__a[5] = __X1;
1751 __u.__a[6] = __Y1;
1752 __u.__a[7] = __Z1;
1753 __u.__a[8] = __S;
1754 __u.__a[9] = __T;
1755 __u.__a[10] = __U;
1756 __u.__a[11] = __V;
1757 __u.__a[12] = __W;
1758 __u.__a[13] = __X;
1759 __u.__a[14] = __Y;
1760 __u.__a[15] = __Z;
1761
1762 return __u.__v;
1763}
1764
1765static __inline __m128i
1766_mm_set1_epi64 (__m64 __A)
1767{
1768 __v2di __tmp = (__v2di)__builtin_ia32_movq2dq ((unsigned long long)__A);
1769 return (__m128i)__builtin_ia32_punpcklqdq128 (__tmp, __tmp);
1770}
1771
1772static __inline __m128i
1773_mm_set1_epi32 (int __A)
1774{
1775 __v4si __tmp = (__v4si)__builtin_ia32_loadd (&__A);
1776 return (__m128i) __builtin_ia32_pshufd ((__v4si)__tmp, _MM_SHUFFLE (0,0,0,0));
1777}
1778
1779#ifdef __x86_64__
1780static __inline __m128i
1781_mm_set1_epi64x (long long __A)
1782{
1783 __v2di __tmp = (__v2di)__builtin_ia32_movq2dq ((unsigned long long)__A);
1784 return (__m128i) __builtin_ia32_shufpd ((__v2df)__tmp, (__v2df)__tmp, _MM_SHUFFLE2 (0,0));
1785}
1786#endif
1787
1788static __inline __m128i
1789_mm_set1_epi16 (short __A)
1790{
1791 int __Acopy = (unsigned short)__A;
1792 __v4si __tmp = (__v4si)__builtin_ia32_loadd (&__Acopy);
1793 __tmp = (__v4si)__builtin_ia32_punpcklwd128 ((__v8hi)__tmp, (__v8hi)__tmp);
1794 return (__m128i) __builtin_ia32_pshufd ((__v4si)__tmp, _MM_SHUFFLE (0,0,0,0));
1795}
1796
1797static __inline __m128i
1798_mm_set1_epi8 (char __A)
1799{
1800 int __Acopy = (unsigned char)__A;
1801 __v4si __tmp = (__v4si)__builtin_ia32_loadd (&__Acopy);
1802 __tmp = (__v4si)__builtin_ia32_punpcklbw128 ((__v16qi)__tmp, (__v16qi)__tmp);
1803 __tmp = (__v4si)__builtin_ia32_punpcklbw128 ((__v16qi)__tmp, (__v16qi)__tmp);
1804 return (__m128i) __builtin_ia32_pshufd ((__v4si)__tmp, _MM_SHUFFLE (0,0,0,0));
1805}
1806
1807static __inline __m128i
1808_mm_setr_epi64 (__m64 __A, __m64 __B)
1809{
1810 __v2di __tmp = (__v2di)__builtin_ia32_movq2dq ((unsigned long long)__A);
1811 __v2di __tmp2 = (__v2di)__builtin_ia32_movq2dq ((unsigned long long)__B);
1812 return (__m128i)__builtin_ia32_punpcklqdq128 (__tmp, __tmp2);
1813}
1814
1815/* Create the vector [Z Y X W]. */
1816static __inline __m128i
1817_mm_setr_epi32 (int __W, int __X, int __Y, int __Z)
1818{
1819 union {
1820 int __a[4];
1821 __m128i __v;
1822 } __u;
1823
1824 __u.__a[0] = __W;
1825 __u.__a[1] = __X;
1826 __u.__a[2] = __Y;
1827 __u.__a[3] = __Z;
1828
1829 return __u.__v;
1830}
1831/* Create the vector [S T U V Z Y X W]. */
1832static __inline __m128i
1833_mm_setr_epi16 (short __S, short __T, short __U, short __V,
1834 short __W, short __X, short __Y, short __Z)
1835{
1836 union {
1837 short __a[8];
1838 __m128i __v;
1839 } __u;
1840
1841 __u.__a[0] = __S;
1842 __u.__a[1] = __T;
1843 __u.__a[2] = __U;
1844 __u.__a[3] = __V;
1845 __u.__a[4] = __W;
1846 __u.__a[5] = __X;
1847 __u.__a[6] = __Y;
1848 __u.__a[7] = __Z;
1849
1850 return __u.__v;
1851}
1852
1853/* Create the vector [S T U V Z Y X W]. */
1854static __inline __m128i
1855_mm_setr_epi8 (char __S1, char __T1, char __U1, char __V1,
1856 char __W1, char __X1, char __Y1, char __Z1,
1857 char __S, char __T, char __U, char __V,
1858 char __W, char __X, char __Y, char __Z)
1859{
1860 union {
1861 char __a[16];
1862 __m128i __v;
1863 } __u;
1864
1865 __u.__a[0] = __S1;
1866 __u.__a[1] = __T1;
1867 __u.__a[2] = __U1;
1868 __u.__a[3] = __V1;
1869 __u.__a[4] = __W1;
1870 __u.__a[5] = __X1;
1871 __u.__a[6] = __Y1;
1872 __u.__a[7] = __Z1;
1873 __u.__a[8] = __S;
1874 __u.__a[9] = __T;
1875 __u.__a[10] = __U;
1876 __u.__a[11] = __V;
1877 __u.__a[12] = __W;
1878 __u.__a[13] = __X;
1879 __u.__a[14] = __Y;
1880 __u.__a[15] = __Z;
1881
1882 return __u.__v;
1883}
1884
1885static __inline __m128d
1886_mm_cvtepi32_pd (__m128i __A)
1887{
1888 return (__m128d)__builtin_ia32_cvtdq2pd ((__v4si) __A);
1889}
1890
1891static __inline __m128
1892_mm_cvtepi32_ps (__m128i __A)
1893{
1894 return (__m128)__builtin_ia32_cvtdq2ps ((__v4si) __A);
1895}
1896
1897static __inline __m128i
1898_mm_cvtpd_epi32 (__m128d __A)
1899{
1900 return (__m128i)__builtin_ia32_cvtpd2dq ((__v2df) __A);
1901}
1902
1903static __inline __m64
1904_mm_cvtpd_pi32 (__m128d __A)
1905{
1906 return (__m64)__builtin_ia32_cvtpd2pi ((__v2df) __A);
1907}
1908
1909static __inline __m128
1910_mm_cvtpd_ps (__m128d __A)
1911{
1912 return (__m128)__builtin_ia32_cvtpd2ps ((__v2df) __A);
1913}
1914
1915static __inline __m128i
1916_mm_cvttpd_epi32 (__m128d __A)
1917{
1918 return (__m128i)__builtin_ia32_cvttpd2dq ((__v2df) __A);
1919}
1920
1921static __inline __m64
1922_mm_cvttpd_pi32 (__m128d __A)
1923{
1924 return (__m64)__builtin_ia32_cvttpd2pi ((__v2df) __A);
1925}
1926
1927static __inline __m128d
1928_mm_cvtpi32_pd (__m64 __A)
1929{
1930 return (__m128d)__builtin_ia32_cvtpi2pd ((__v2si) __A);
1931}
1932
1933static __inline __m128i
1934_mm_cvtps_epi32 (__m128 __A)
1935{
1936 return (__m128i)__builtin_ia32_cvtps2dq ((__v4sf) __A);
1937}
1938
1939static __inline __m128i
1940_mm_cvttps_epi32 (__m128 __A)
1941{
1942 return (__m128i)__builtin_ia32_cvttps2dq ((__v4sf) __A);
1943}
1944
1945static __inline __m128d
1946_mm_cvtps_pd (__m128 __A)
1947{
1948 return (__m128d)__builtin_ia32_cvtps2pd ((__v4sf) __A);
1949}
1950
1951static __inline int
1952_mm_cvtsd_si32 (__m128d __A)
1953{
1954 return __builtin_ia32_cvtsd2si ((__v2df) __A);
1955}
1956
1957#ifdef __x86_64__
1958static __inline long long
1959_mm_cvtsd_si64x (__m128d __A)
1960{
1961 return __builtin_ia32_cvtsd2si64 ((__v2df) __A);
1962}
1963#endif
1964
1965static __inline int
1966_mm_cvttsd_si32 (__m128d __A)
1967{
1968 return __builtin_ia32_cvttsd2si ((__v2df) __A);
1969}
1970
1971#ifdef __x86_64__
1972static __inline long long
1973_mm_cvttsd_si64x (__m128d __A)
1974{
1975 return __builtin_ia32_cvttsd2si64 ((__v2df) __A);
1976}
1977#endif
1978
1979static __inline __m128
1980_mm_cvtsd_ss (__m128 __A, __m128d __B)
1981{
1982 return (__m128)__builtin_ia32_cvtsd2ss ((__v4sf) __A, (__v2df) __B);
1983}
1984
1985static __inline __m128d
1986_mm_cvtsi32_sd (__m128d __A, int __B)
1987{
1988 return (__m128d)__builtin_ia32_cvtsi2sd ((__v2df) __A, __B);
1989}
1990
1991#ifdef __x86_64__
1992static __inline __m128d
1993_mm_cvtsi64x_sd (__m128d __A, long long __B)
1994{
1995 return (__m128d)__builtin_ia32_cvtsi642sd ((__v2df) __A, __B);
1996}
1997#endif
1998
1999static __inline __m128d
2000_mm_cvtss_sd (__m128d __A, __m128 __B)
2001{
2002 return (__m128d)__builtin_ia32_cvtss2sd ((__v2df) __A, (__v4sf)__B);
2003}
2004
2005#define _mm_shuffle_pd(__A, __B, __C) ((__m128d)__builtin_ia32_shufpd ((__v2df)__A, (__v2df)__B, (__C)))
2006
2007static __inline __m128d
2008_mm_unpackhi_pd (__m128d __A, __m128d __B)
2009{
2010 return (__m128d)__builtin_ia32_unpckhpd ((__v2df)__A, (__v2df)__B);
2011}
2012
2013static __inline __m128d
2014_mm_unpacklo_pd (__m128d __A, __m128d __B)
2015{
2016 return (__m128d)__builtin_ia32_unpcklpd ((__v2df)__A, (__v2df)__B);
2017}
2018
2019static __inline __m128d
2020_mm_loadh_pd (__m128d __A, double const *__B)
2021{
2022 return (__m128d)__builtin_ia32_loadhpd ((__v2df)__A, (__v2si *)__B);
2023}
2024
2025static __inline void
2026_mm_storeh_pd (double *__A, __m128d __B)
2027{
2028 __builtin_ia32_storehpd ((__v2si *)__A, (__v2df)__B);
2029}
2030
2031static __inline __m128d
2032_mm_loadl_pd (__m128d __A, double const *__B)
2033{
2034 return (__m128d)__builtin_ia32_loadlpd ((__v2df)__A, (__v2si *)__B);
2035}
2036
2037static __inline void
2038_mm_storel_pd (double *__A, __m128d __B)
2039{
2040 __builtin_ia32_storelpd ((__v2si *)__A, (__v2df)__B);
2041}
2042
2043static __inline int
2044_mm_movemask_pd (__m128d __A)
2045{
2046 return __builtin_ia32_movmskpd ((__v2df)__A);
2047}
2048
2049static __inline __m128i
2050_mm_packs_epi16 (__m128i __A, __m128i __B)
2051{
2052 return (__m128i)__builtin_ia32_packsswb128 ((__v8hi)__A, (__v8hi)__B);
2053}
2054
2055static __inline __m128i
2056_mm_packs_epi32 (__m128i __A, __m128i __B)
2057{
2058 return (__m128i)__builtin_ia32_packssdw128 ((__v4si)__A, (__v4si)__B);
2059}
2060
2061static __inline __m128i
2062_mm_packus_epi16 (__m128i __A, __m128i __B)
2063{
2064 return (__m128i)__builtin_ia32_packuswb128 ((__v8hi)__A, (__v8hi)__B);
2065}
2066
2067static __inline __m128i
2068_mm_unpackhi_epi8 (__m128i __A, __m128i __B)
2069{
2070 return (__m128i)__builtin_ia32_punpckhbw128 ((__v16qi)__A, (__v16qi)__B);
2071}
2072
2073static __inline __m128i
2074_mm_unpackhi_epi16 (__m128i __A, __m128i __B)
2075{
2076 return (__m128i)__builtin_ia32_punpckhwd128 ((__v8hi)__A, (__v8hi)__B);
2077}
2078
2079static __inline __m128i
2080_mm_unpackhi_epi32 (__m128i __A, __m128i __B)
2081{
2082 return (__m128i)__builtin_ia32_punpckhdq128 ((__v4si)__A, (__v4si)__B);
2083}
2084
2085static __inline __m128i
2086_mm_unpackhi_epi64 (__m128i __A, __m128i __B)
2087{
2088 return (__m128i)__builtin_ia32_punpckhqdq128 ((__v2di)__A, (__v2di)__B);
2089}
2090
2091static __inline __m128i
2092_mm_unpacklo_epi8 (__m128i __A, __m128i __B)
2093{
2094 return (__m128i)__builtin_ia32_punpcklbw128 ((__v16qi)__A, (__v16qi)__B);
2095}
2096
2097static __inline __m128i
2098_mm_unpacklo_epi16 (__m128i __A, __m128i __B)
2099{
2100 return (__m128i)__builtin_ia32_punpcklwd128 ((__v8hi)__A, (__v8hi)__B);
2101}
2102
2103static __inline __m128i
2104_mm_unpacklo_epi32 (__m128i __A, __m128i __B)
2105{
2106 return (__m128i)__builtin_ia32_punpckldq128 ((__v4si)__A, (__v4si)__B);
2107}
2108
2109static __inline __m128i
2110_mm_unpacklo_epi64 (__m128i __A, __m128i __B)
2111{
2112 return (__m128i)__builtin_ia32_punpcklqdq128 ((__v2di)__A, (__v2di)__B);
2113}
2114
2115static __inline __m128i
2116_mm_add_epi8 (__m128i __A, __m128i __B)
2117{
2118 return (__m128i)__builtin_ia32_paddb128 ((__v16qi)__A, (__v16qi)__B);
2119}
2120
2121static __inline __m128i
2122_mm_add_epi16 (__m128i __A, __m128i __B)
2123{
2124 return (__m128i)__builtin_ia32_paddw128 ((__v8hi)__A, (__v8hi)__B);
2125}
2126
2127static __inline __m128i
2128_mm_add_epi32 (__m128i __A, __m128i __B)
2129{
2130 return (__m128i)__builtin_ia32_paddd128 ((__v4si)__A, (__v4si)__B);
2131}
2132
2133static __inline __m128i
2134_mm_add_epi64 (__m128i __A, __m128i __B)
2135{
2136 return (__m128i)__builtin_ia32_paddq128 ((__v2di)__A, (__v2di)__B);
2137}
2138
2139static __inline __m128i
2140_mm_adds_epi8 (__m128i __A, __m128i __B)
2141{
2142 return (__m128i)__builtin_ia32_paddsb128 ((__v16qi)__A, (__v16qi)__B);
2143}
2144
2145static __inline __m128i
2146_mm_adds_epi16 (__m128i __A, __m128i __B)
2147{
2148 return (__m128i)__builtin_ia32_paddsw128 ((__v8hi)__A, (__v8hi)__B);
2149}
2150
2151static __inline __m128i
2152_mm_adds_epu8 (__m128i __A, __m128i __B)
2153{
2154 return (__m128i)__builtin_ia32_paddusb128 ((__v16qi)__A, (__v16qi)__B);
2155}
2156
2157static __inline __m128i
2158_mm_adds_epu16 (__m128i __A, __m128i __B)
2159{
2160 return (__m128i)__builtin_ia32_paddusw128 ((__v8hi)__A, (__v8hi)__B);
2161}
2162
2163static __inline __m128i
2164_mm_sub_epi8 (__m128i __A, __m128i __B)
2165{
2166 return (__m128i)__builtin_ia32_psubb128 ((__v16qi)__A, (__v16qi)__B);
2167}
2168
2169static __inline __m128i
2170_mm_sub_epi16 (__m128i __A, __m128i __B)
2171{
2172 return (__m128i)__builtin_ia32_psubw128 ((__v8hi)__A, (__v8hi)__B);
2173}
2174
2175static __inline __m128i
2176_mm_sub_epi32 (__m128i __A, __m128i __B)
2177{
2178 return (__m128i)__builtin_ia32_psubd128 ((__v4si)__A, (__v4si)__B);
2179}
2180
2181static __inline __m128i
2182_mm_sub_epi64 (__m128i __A, __m128i __B)
2183{
2184 return (__m128i)__builtin_ia32_psubq128 ((__v2di)__A, (__v2di)__B);
2185}
2186
2187static __inline __m128i
2188_mm_subs_epi8 (__m128i __A, __m128i __B)
2189{
2190 return (__m128i)__builtin_ia32_psubsb128 ((__v16qi)__A, (__v16qi)__B);
2191}
2192
2193static __inline __m128i
2194_mm_subs_epi16 (__m128i __A, __m128i __B)
2195{
2196 return (__m128i)__builtin_ia32_psubsw128 ((__v8hi)__A, (__v8hi)__B);
2197}
2198
2199static __inline __m128i
2200_mm_subs_epu8 (__m128i __A, __m128i __B)
2201{
2202 return (__m128i)__builtin_ia32_psubusb128 ((__v16qi)__A, (__v16qi)__B);
2203}
2204
2205static __inline __m128i
2206_mm_subs_epu16 (__m128i __A, __m128i __B)
2207{
2208 return (__m128i)__builtin_ia32_psubusw128 ((__v8hi)__A, (__v8hi)__B);
2209}
2210
2211static __inline __m128i
2212_mm_madd_epi16 (__m128i __A, __m128i __B)
2213{
2214 return (__m128i)__builtin_ia32_pmaddwd128 ((__v8hi)__A, (__v8hi)__B);
2215}
2216
2217static __inline __m128i
2218_mm_mulhi_epi16 (__m128i __A, __m128i __B)
2219{
2220 return (__m128i)__builtin_ia32_pmulhw128 ((__v8hi)__A, (__v8hi)__B);
2221}
2222
2223static __inline __m128i
2224_mm_mullo_epi16 (__m128i __A, __m128i __B)
2225{
2226 return (__m128i)__builtin_ia32_pmullw128 ((__v8hi)__A, (__v8hi)__B);
2227}
2228
2229static __inline __m64
2230_mm_mul_su32 (__m64 __A, __m64 __B)
2231{
2232 return (__m64)__builtin_ia32_pmuludq ((__v2si)__A, (__v2si)__B);
2233}
2234
2235static __inline __m128i
2236_mm_mul_epu32 (__m128i __A, __m128i __B)
2237{
2238 return (__m128i)__builtin_ia32_pmuludq128 ((__v4si)__A, (__v4si)__B);
2239}
2240
2241static __inline __m128i
2242_mm_sll_epi16 (__m128i __A, __m128i __B)
2243{
2244 return (__m128i)__builtin_ia32_psllw128 ((__v8hi)__A, (__v2di)__B);
2245}
2246
2247static __inline __m128i
2248_mm_sll_epi32 (__m128i __A, __m128i __B)
2249{
2250 return (__m128i)__builtin_ia32_pslld128 ((__v4si)__A, (__v2di)__B);
2251}
2252
2253static __inline __m128i
2254_mm_sll_epi64 (__m128i __A, __m128i __B)
2255{
2256 return (__m128i)__builtin_ia32_psllq128 ((__v2di)__A, (__v2di)__B);
2257}
2258
2259static __inline __m128i
2260_mm_sra_epi16 (__m128i __A, __m128i __B)
2261{
2262 return (__m128i)__builtin_ia32_psraw128 ((__v8hi)__A, (__v2di)__B);
2263}
2264
2265static __inline __m128i
2266_mm_sra_epi32 (__m128i __A, __m128i __B)
2267{
2268 return (__m128i)__builtin_ia32_psrad128 ((__v4si)__A, (__v2di)__B);
2269}
2270
2271static __inline __m128i
2272_mm_srl_epi16 (__m128i __A, __m128i __B)
2273{
2274 return (__m128i)__builtin_ia32_psrlw128 ((__v8hi)__A, (__v2di)__B);
2275}
2276
2277static __inline __m128i
2278_mm_srl_epi32 (__m128i __A, __m128i __B)
2279{
2280 return (__m128i)__builtin_ia32_psrld128 ((__v4si)__A, (__v2di)__B);
2281}
2282
2283static __inline __m128i
2284_mm_srl_epi64 (__m128i __A, __m128i __B)
2285{
2286 return (__m128i)__builtin_ia32_psrlq128 ((__v2di)__A, (__v2di)__B);
2287}
2288
2289static __inline __m128i
2290_mm_slli_epi16 (__m128i __A, int __B)
2291{
2292 return (__m128i)__builtin_ia32_psllwi128 ((__v8hi)__A, __B);
2293}
2294
2295static __inline __m128i
2296_mm_slli_epi32 (__m128i __A, int __B)
2297{
2298 return (__m128i)__builtin_ia32_pslldi128 ((__v4si)__A, __B);
2299}
2300
2301static __inline __m128i
2302_mm_slli_epi64 (__m128i __A, int __B)
2303{
2304 return (__m128i)__builtin_ia32_psllqi128 ((__v2di)__A, __B);
2305}
2306
2307static __inline __m128i
2308_mm_srai_epi16 (__m128i __A, int __B)
2309{
2310 return (__m128i)__builtin_ia32_psrawi128 ((__v8hi)__A, __B);
2311}
2312
2313static __inline __m128i
2314_mm_srai_epi32 (__m128i __A, int __B)
2315{
2316 return (__m128i)__builtin_ia32_psradi128 ((__v4si)__A, __B);
2317}
2318
2319#if 0
2320static __m128i __attribute__((__always_inline__))
2321_mm_srli_si128 (__m128i __A, const int __B)
2322{
2323 return ((__m128i)__builtin_ia32_psrldqi128 (__A, __B))
2324}
2325
2326static __m128i __attribute__((__always_inline__))
2327_mm_srli_si128 (__m128i __A, const int __B)
2328{
2329 return ((__m128i)__builtin_ia32_pslldqi128 (__A, __B))
2330}
2331#endif
2332#define _mm_srli_si128(__A, __B) ((__m128i)__builtin_ia32_psrldqi128 (__A, __B))
2333#define _mm_slli_si128(__A, __B) ((__m128i)__builtin_ia32_pslldqi128 (__A, __B))
2334
2335static __inline __m128i
2336_mm_srli_epi16 (__m128i __A, int __B)
2337{
2338 return (__m128i)__builtin_ia32_psrlwi128 ((__v8hi)__A, __B);
2339}
2340
2341static __inline __m128i
2342_mm_srli_epi32 (__m128i __A, int __B)
2343{
2344 return (__m128i)__builtin_ia32_psrldi128 ((__v4si)__A, __B);
2345}
2346
2347static __inline __m128i
2348_mm_srli_epi64 (__m128i __A, int __B)
2349{
2350 return (__m128i)__builtin_ia32_psrlqi128 ((__v2di)__A, __B);
2351}
2352
2353static __inline __m128i
2354_mm_and_si128 (__m128i __A, __m128i __B)
2355{
2356 return (__m128i)__builtin_ia32_pand128 ((__v2di)__A, (__v2di)__B);
2357}
2358
2359static __inline __m128i
2360_mm_andnot_si128 (__m128i __A, __m128i __B)
2361{
2362 return (__m128i)__builtin_ia32_pandn128 ((__v2di)__A, (__v2di)__B);
2363}
2364
2365static __inline __m128i
2366_mm_or_si128 (__m128i __A, __m128i __B)
2367{
2368 return (__m128i)__builtin_ia32_por128 ((__v2di)__A, (__v2di)__B);
2369}
2370
2371static __inline __m128i
2372_mm_xor_si128 (__m128i __A, __m128i __B)
2373{
2374 return (__m128i)__builtin_ia32_pxor128 ((__v2di)__A, (__v2di)__B);
2375}
2376
2377static __inline __m128i
2378_mm_cmpeq_epi8 (__m128i __A, __m128i __B)
2379{
2380 return (__m128i)__builtin_ia32_pcmpeqb128 ((__v16qi)__A, (__v16qi)__B);
2381}
2382
2383static __inline __m128i
2384_mm_cmpeq_epi16 (__m128i __A, __m128i __B)
2385{
2386 return (__m128i)__builtin_ia32_pcmpeqw128 ((__v8hi)__A, (__v8hi)__B);
2387}
2388
2389static __inline __m128i
2390_mm_cmpeq_epi32 (__m128i __A, __m128i __B)
2391{
2392 return (__m128i)__builtin_ia32_pcmpeqd128 ((__v4si)__A, (__v4si)__B);
2393}
2394
2395static __inline __m128i
2396_mm_cmplt_epi8 (__m128i __A, __m128i __B)
2397{
2398 return (__m128i)__builtin_ia32_pcmpgtb128 ((__v16qi)__B, (__v16qi)__A);
2399}
2400
2401static __inline __m128i
2402_mm_cmplt_epi16 (__m128i __A, __m128i __B)
2403{
2404 return (__m128i)__builtin_ia32_pcmpgtw128 ((__v8hi)__B, (__v8hi)__A);
2405}
2406
2407static __inline __m128i
2408_mm_cmplt_epi32 (__m128i __A, __m128i __B)
2409{
2410 return (__m128i)__builtin_ia32_pcmpgtd128 ((__v4si)__B, (__v4si)__A);
2411}
2412
2413static __inline __m128i
2414_mm_cmpgt_epi8 (__m128i __A, __m128i __B)
2415{
2416 return (__m128i)__builtin_ia32_pcmpgtb128 ((__v16qi)__A, (__v16qi)__B);
2417}
2418
2419static __inline __m128i
2420_mm_cmpgt_epi16 (__m128i __A, __m128i __B)
2421{
2422 return (__m128i)__builtin_ia32_pcmpgtw128 ((__v8hi)__A, (__v8hi)__B);
2423}
2424
2425static __inline __m128i
2426_mm_cmpgt_epi32 (__m128i __A, __m128i __B)
2427{
2428 return (__m128i)__builtin_ia32_pcmpgtd128 ((__v4si)__A, (__v4si)__B);
2429}
2430
2431#define _mm_extract_epi16(__A, __B) __builtin_ia32_pextrw128 ((__v8hi)__A, __B)
2432
2433#define _mm_insert_epi16(__A, __B, __C) ((__m128i)__builtin_ia32_pinsrw128 ((__v8hi)__A, __B, __C))
2434
2435static __inline __m128i
2436_mm_max_epi16 (__m128i __A, __m128i __B)
2437{
2438 return (__m128i)__builtin_ia32_pmaxsw128 ((__v8hi)__A, (__v8hi)__B);
2439}
2440
2441static __inline __m128i
2442_mm_max_epu8 (__m128i __A, __m128i __B)
2443{
2444 return (__m128i)__builtin_ia32_pmaxub128 ((__v16qi)__A, (__v16qi)__B);
2445}
2446
2447static __inline __m128i
2448_mm_min_epi16 (__m128i __A, __m128i __B)
2449{
2450 return (__m128i)__builtin_ia32_pminsw128 ((__v8hi)__A, (__v8hi)__B);
2451}
2452
2453static __inline __m128i
2454_mm_min_epu8 (__m128i __A, __m128i __B)
2455{
2456 return (__m128i)__builtin_ia32_pminub128 ((__v16qi)__A, (__v16qi)__B);
2457}
2458
2459static __inline int
2460_mm_movemask_epi8 (__m128i __A)
2461{
2462 return __builtin_ia32_pmovmskb128 ((__v16qi)__A);
2463}
2464
2465static __inline __m128i
2466_mm_mulhi_epu16 (__m128i __A, __m128i __B)
2467{
2468 return (__m128i)__builtin_ia32_pmulhuw128 ((__v8hi)__A, (__v8hi)__B);
2469}
2470
2471#define _mm_shufflehi_epi16(__A, __B) ((__m128i)__builtin_ia32_pshufhw ((__v8hi)__A, __B))
2472#define _mm_shufflelo_epi16(__A, __B) ((__m128i)__builtin_ia32_pshuflw ((__v8hi)__A, __B))
2473#define _mm_shuffle_epi32(__A, __B) ((__m128i)__builtin_ia32_pshufd ((__v4si)__A, __B))
2474
2475static __inline void
2476_mm_maskmoveu_si128 (__m128i __A, __m128i __B, char *__C)
2477{
2478 __builtin_ia32_maskmovdqu ((__v16qi)__A, (__v16qi)__B, __C);
2479}
2480
2481static __inline __m128i
2482_mm_avg_epu8 (__m128i __A, __m128i __B)
2483{
2484 return (__m128i)__builtin_ia32_pavgb128 ((__v16qi)__A, (__v16qi)__B);
2485}
2486
2487static __inline __m128i
2488_mm_avg_epu16 (__m128i __A, __m128i __B)
2489{
2490 return (__m128i)__builtin_ia32_pavgw128 ((__v8hi)__A, (__v8hi)__B);
2491}
2492
2493static __inline __m128i
2494_mm_sad_epu8 (__m128i __A, __m128i __B)
2495{
2496 return (__m128i)__builtin_ia32_psadbw128 ((__v16qi)__A, (__v16qi)__B);
2497}
2498
2499static __inline void
2500_mm_stream_si32 (int *__A, int __B)
2501{
2502 __builtin_ia32_movnti (__A, __B);
2503}
2504
2505static __inline void
2506_mm_stream_si128 (__m128i *__A, __m128i __B)
2507{
2508 __builtin_ia32_movntdq ((__v2di *)__A, (__v2di)__B);
2509}
2510
2511static __inline void
2512_mm_stream_pd (double *__A, __m128d __B)
2513{
2514 __builtin_ia32_movntpd (__A, (__v2df)__B);
2515}
2516
2517static __inline __m128i
2518_mm_movpi64_epi64 (__m64 __A)
2519{
2520 return (__m128i)__builtin_ia32_movq2dq ((unsigned long long)__A);
2521}
2522
2523static __inline void
2524_mm_clflush (void const *__A)
2525{
2526 return __builtin_ia32_clflush (__A);
2527}
2528
2529static __inline void
2530_mm_lfence (void)
2531{
2532 __builtin_ia32_lfence ();
2533}
2534
2535static __inline void
2536_mm_mfence (void)
2537{
2538 __builtin_ia32_mfence ();
2539}
2540
2541static __inline __m128i
2542_mm_cvtsi32_si128 (int __A)
2543{
2544 return (__m128i) __builtin_ia32_loadd (&__A);
2545}
2546
2547#ifdef __x86_64__
2548static __inline __m128i
2549_mm_cvtsi64x_si128 (long long __A)
2550{
2551 return (__m128i) __builtin_ia32_movq2dq (__A);
2552}
2553#endif
2554
2555static __inline int
2556_mm_cvtsi128_si32 (__m128i __A)
2557{
2558 int __tmp;
2559 __builtin_ia32_stored (&__tmp, (__v4si)__A);
2560 return __tmp;
2561}
2562
2563#ifdef __x86_64__
2564static __inline long long
2565_mm_cvtsi128_si64x (__m128i __A)
2566{
2567 return __builtin_ia32_movdq2q ((__v2di)__A);
2568}
2569#endif
2570
2571#endif /* __SSE2__ */
2572
2573#endif /* __SSE__ */
2574#endif /* _XMMINTRIN_H_INCLUDED */
1229#endif /* __SSE__ */
1230#endif /* _XMMINTRIN_H_INCLUDED */