1/*===---- mmintrin.h - Implementation of MMX intrinsics on PowerPC ---------===
2 *
3 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 * See https://llvm.org/LICENSE.txt for license information.
5 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 *
7 *===-----------------------------------------------------------------------===
8 */
9
10/* Implemented from the specification included in the Intel C++ Compiler
11   User Guide and Reference, version 9.0.  */
12
13#ifndef NO_WARN_X86_INTRINSICS
14/* This header file is to help porting code using Intel intrinsics
15   explicitly from x86_64 to powerpc64/powerpc64le.
16
17   Since PowerPC target doesn't support native 64-bit vector type, we
18   typedef __m64 to 64-bit unsigned long long in MMX intrinsics, which
19   works well for _si64 and some _pi32 operations.
20
21   For _pi16 and _pi8 operations, it's better to transfer __m64 into
22   128-bit PowerPC vector first. Power8 introduced direct register
23   move instructions which helps for more efficient implementation.
24
25   It's user's responsibility to determine if the results of such port
26   are acceptable or further changes are needed. Please note that much
27   code using Intel intrinsics CAN BE REWRITTEN in more portable and
28   efficient standard C or GNU C extensions with 64-bit scalar
29   operations, or 128-bit SSE/Altivec operations, which are more
30   recommended. */
31#error                                                                         \
32    "Please read comment above.  Use -DNO_WARN_X86_INTRINSICS to disable this error."
33#endif
34
35#ifndef _MMINTRIN_H_INCLUDED
36#define _MMINTRIN_H_INCLUDED
37
38#if defined(__powerpc64__) &&                                                  \
39    (defined(__linux__) || defined(__FreeBSD__) || defined(_AIX))
40
41#include <altivec.h>
42/* The Intel API is flexible enough that we must allow aliasing with other
43   vector types, and their scalar components.  */
44typedef __attribute__((__aligned__(8))) unsigned long long __m64;
45
46typedef __attribute__((__aligned__(8))) union {
47  __m64 as_m64;
48  char as_char[8];
49  signed char as_signed_char[8];
50  short as_short[4];
51  int as_int[2];
52  long long as_long_long;
53  float as_float[2];
54  double as_double;
55} __m64_union;
56
57/* Empty the multimedia state.  */
58extern __inline void
59    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
60    _mm_empty(void) {
61  /* nothing to do on PowerPC.  */
62}
63
64extern __inline void
65    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
66    _m_empty(void) {
67  /* nothing to do on PowerPC.  */
68}
69
70/* Convert I to a __m64 object.  The integer is zero-extended to 64-bits.  */
71extern __inline __m64
72    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
73    _mm_cvtsi32_si64(int __i) {
74  return (__m64)(unsigned int)__i;
75}
76
77extern __inline __m64
78    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
79    _m_from_int(int __i) {
80  return _mm_cvtsi32_si64(__i);
81}
82
83/* Convert the lower 32 bits of the __m64 object into an integer.  */
84extern __inline int
85    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
86    _mm_cvtsi64_si32(__m64 __i) {
87  return ((int)__i);
88}
89
90extern __inline int
91    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
92    _m_to_int(__m64 __i) {
93  return _mm_cvtsi64_si32(__i);
94}
95
96/* Convert I to a __m64 object.  */
97
98/* Intel intrinsic.  */
99extern __inline __m64
100    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
101    _m_from_int64(long long __i) {
102  return (__m64)__i;
103}
104
105extern __inline __m64
106    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
107    _mm_cvtsi64_m64(long long __i) {
108  return (__m64)__i;
109}
110
111/* Microsoft intrinsic.  */
112extern __inline __m64
113    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
114    _mm_cvtsi64x_si64(long long __i) {
115  return (__m64)__i;
116}
117
118extern __inline __m64
119    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
120    _mm_set_pi64x(long long __i) {
121  return (__m64)__i;
122}
123
124/* Convert the __m64 object to a 64bit integer.  */
125
126/* Intel intrinsic.  */
127extern __inline long long
128    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
129    _m_to_int64(__m64 __i) {
130  return (long long)__i;
131}
132
133extern __inline long long
134    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
135    _mm_cvtm64_si64(__m64 __i) {
136  return (long long)__i;
137}
138
139/* Microsoft intrinsic.  */
140extern __inline long long
141    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
142    _mm_cvtsi64_si64x(__m64 __i) {
143  return (long long)__i;
144}
145
146#ifdef _ARCH_PWR8
147/* Pack the four 16-bit values from M1 into the lower four 8-bit values of
148   the result, and the four 16-bit values from M2 into the upper four 8-bit
149   values of the result, all with signed saturation.  */
150extern __inline __m64
151    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
152    _mm_packs_pi16(__m64 __m1, __m64 __m2) {
153  __vector signed short __vm1;
154  __vector signed char __vresult;
155
156  __vm1 = (__vector signed short)(__vector unsigned long long)
157#ifdef __LITTLE_ENDIAN__
158      {__m1, __m2};
159#else
160      {__m2, __m1};
161#endif
162  __vresult = vec_packs(__vm1, __vm1);
163  return (__m64)((__vector long long)__vresult)[0];
164}
165
166extern __inline __m64
167    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
168    _m_packsswb(__m64 __m1, __m64 __m2) {
169  return _mm_packs_pi16(__m1, __m2);
170}
171
172/* Pack the two 32-bit values from M1 in to the lower two 16-bit values of
173   the result, and the two 32-bit values from M2 into the upper two 16-bit
174   values of the result, all with signed saturation.  */
175extern __inline __m64
176    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
177    _mm_packs_pi32(__m64 __m1, __m64 __m2) {
178  __vector signed int __vm1;
179  __vector signed short __vresult;
180
181  __vm1 = (__vector signed int)(__vector unsigned long long)
182#ifdef __LITTLE_ENDIAN__
183      {__m1, __m2};
184#else
185      {__m2, __m1};
186#endif
187  __vresult = vec_packs(__vm1, __vm1);
188  return (__m64)((__vector long long)__vresult)[0];
189}
190
191extern __inline __m64
192    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
193    _m_packssdw(__m64 __m1, __m64 __m2) {
194  return _mm_packs_pi32(__m1, __m2);
195}
196
197/* Pack the four 16-bit values from M1 into the lower four 8-bit values of
198   the result, and the four 16-bit values from M2 into the upper four 8-bit
199   values of the result, all with unsigned saturation.  */
200extern __inline __m64
201    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
202    _mm_packs_pu16(__m64 __m1, __m64 __m2) {
203  __vector unsigned char __r;
204  __vector signed short __vm1 = (__vector signed short)(__vector long long)
205#ifdef __LITTLE_ENDIAN__
206      {__m1, __m2};
207#else
208      {__m2, __m1};
209#endif
210  const __vector signed short __zero = {0};
211  __vector __bool short __select = vec_cmplt(__vm1, __zero);
212  __r =
213      vec_packs((__vector unsigned short)__vm1, (__vector unsigned short)__vm1);
214  __vector __bool char __packsel = vec_pack(__select, __select);
215  __r = vec_sel(__r, (const __vector unsigned char)__zero, __packsel);
216  return (__m64)((__vector long long)__r)[0];
217}
218
219extern __inline __m64
220    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
221    _m_packuswb(__m64 __m1, __m64 __m2) {
222  return _mm_packs_pu16(__m1, __m2);
223}
224#endif /* end ARCH_PWR8 */
225
226/* Interleave the four 8-bit values from the high half of M1 with the four
227   8-bit values from the high half of M2.  */
228extern __inline __m64
229    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
230    _mm_unpackhi_pi8(__m64 __m1, __m64 __m2) {
231#if _ARCH_PWR8
232  __vector unsigned char __a, __b, __c;
233
234  __a = (__vector unsigned char)vec_splats(__m1);
235  __b = (__vector unsigned char)vec_splats(__m2);
236  __c = vec_mergel(__a, __b);
237  return (__m64)((__vector long long)__c)[1];
238#else
239  __m64_union __mu1, __mu2, __res;
240
241  __mu1.as_m64 = __m1;
242  __mu2.as_m64 = __m2;
243
244  __res.as_char[0] = __mu1.as_char[4];
245  __res.as_char[1] = __mu2.as_char[4];
246  __res.as_char[2] = __mu1.as_char[5];
247  __res.as_char[3] = __mu2.as_char[5];
248  __res.as_char[4] = __mu1.as_char[6];
249  __res.as_char[5] = __mu2.as_char[6];
250  __res.as_char[6] = __mu1.as_char[7];
251  __res.as_char[7] = __mu2.as_char[7];
252
253  return (__m64)__res.as_m64;
254#endif
255}
256
257extern __inline __m64
258    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
259    _m_punpckhbw(__m64 __m1, __m64 __m2) {
260  return _mm_unpackhi_pi8(__m1, __m2);
261}
262
263/* Interleave the two 16-bit values from the high half of M1 with the two
264   16-bit values from the high half of M2.  */
265extern __inline __m64
266    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
267    _mm_unpackhi_pi16(__m64 __m1, __m64 __m2) {
268  __m64_union __mu1, __mu2, __res;
269
270  __mu1.as_m64 = __m1;
271  __mu2.as_m64 = __m2;
272
273  __res.as_short[0] = __mu1.as_short[2];
274  __res.as_short[1] = __mu2.as_short[2];
275  __res.as_short[2] = __mu1.as_short[3];
276  __res.as_short[3] = __mu2.as_short[3];
277
278  return (__m64)__res.as_m64;
279}
280
281extern __inline __m64
282    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
283    _m_punpckhwd(__m64 __m1, __m64 __m2) {
284  return _mm_unpackhi_pi16(__m1, __m2);
285}
286/* Interleave the 32-bit value from the high half of M1 with the 32-bit
287   value from the high half of M2.  */
288extern __inline __m64
289    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
290    _mm_unpackhi_pi32(__m64 __m1, __m64 __m2) {
291  __m64_union __mu1, __mu2, __res;
292
293  __mu1.as_m64 = __m1;
294  __mu2.as_m64 = __m2;
295
296  __res.as_int[0] = __mu1.as_int[1];
297  __res.as_int[1] = __mu2.as_int[1];
298
299  return (__m64)__res.as_m64;
300}
301
302extern __inline __m64
303    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
304    _m_punpckhdq(__m64 __m1, __m64 __m2) {
305  return _mm_unpackhi_pi32(__m1, __m2);
306}
307/* Interleave the four 8-bit values from the low half of M1 with the four
308   8-bit values from the low half of M2.  */
309extern __inline __m64
310    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
311    _mm_unpacklo_pi8(__m64 __m1, __m64 __m2) {
312#if _ARCH_PWR8
313  __vector unsigned char __a, __b, __c;
314
315  __a = (__vector unsigned char)vec_splats(__m1);
316  __b = (__vector unsigned char)vec_splats(__m2);
317  __c = vec_mergel(__a, __b);
318  return (__m64)((__vector long long)__c)[0];
319#else
320  __m64_union __mu1, __mu2, __res;
321
322  __mu1.as_m64 = __m1;
323  __mu2.as_m64 = __m2;
324
325  __res.as_char[0] = __mu1.as_char[0];
326  __res.as_char[1] = __mu2.as_char[0];
327  __res.as_char[2] = __mu1.as_char[1];
328  __res.as_char[3] = __mu2.as_char[1];
329  __res.as_char[4] = __mu1.as_char[2];
330  __res.as_char[5] = __mu2.as_char[2];
331  __res.as_char[6] = __mu1.as_char[3];
332  __res.as_char[7] = __mu2.as_char[3];
333
334  return (__m64)__res.as_m64;
335#endif
336}
337
338extern __inline __m64
339    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
340    _m_punpcklbw(__m64 __m1, __m64 __m2) {
341  return _mm_unpacklo_pi8(__m1, __m2);
342}
343/* Interleave the two 16-bit values from the low half of M1 with the two
344   16-bit values from the low half of M2.  */
345extern __inline __m64
346    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
347    _mm_unpacklo_pi16(__m64 __m1, __m64 __m2) {
348  __m64_union __mu1, __mu2, __res;
349
350  __mu1.as_m64 = __m1;
351  __mu2.as_m64 = __m2;
352
353  __res.as_short[0] = __mu1.as_short[0];
354  __res.as_short[1] = __mu2.as_short[0];
355  __res.as_short[2] = __mu1.as_short[1];
356  __res.as_short[3] = __mu2.as_short[1];
357
358  return (__m64)__res.as_m64;
359}
360
361extern __inline __m64
362    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
363    _m_punpcklwd(__m64 __m1, __m64 __m2) {
364  return _mm_unpacklo_pi16(__m1, __m2);
365}
366
367/* Interleave the 32-bit value from the low half of M1 with the 32-bit
368   value from the low half of M2.  */
369extern __inline __m64
370    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
371    _mm_unpacklo_pi32(__m64 __m1, __m64 __m2) {
372  __m64_union __mu1, __mu2, __res;
373
374  __mu1.as_m64 = __m1;
375  __mu2.as_m64 = __m2;
376
377  __res.as_int[0] = __mu1.as_int[0];
378  __res.as_int[1] = __mu2.as_int[0];
379
380  return (__m64)__res.as_m64;
381}
382
383extern __inline __m64
384    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
385    _m_punpckldq(__m64 __m1, __m64 __m2) {
386  return _mm_unpacklo_pi32(__m1, __m2);
387}
388
389/* Add the 8-bit values in M1 to the 8-bit values in M2.  */
390extern __inline __m64
391    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
392    _mm_add_pi8(__m64 __m1, __m64 __m2) {
393#if _ARCH_PWR8
394  __vector signed char __a, __b, __c;
395
396  __a = (__vector signed char)vec_splats(__m1);
397  __b = (__vector signed char)vec_splats(__m2);
398  __c = vec_add(__a, __b);
399  return (__m64)((__vector long long)__c)[0];
400#else
401  __m64_union __mu1, __mu2, __res;
402
403  __mu1.as_m64 = __m1;
404  __mu2.as_m64 = __m2;
405
406  __res.as_char[0] = __mu1.as_char[0] + __mu2.as_char[0];
407  __res.as_char[1] = __mu1.as_char[1] + __mu2.as_char[1];
408  __res.as_char[2] = __mu1.as_char[2] + __mu2.as_char[2];
409  __res.as_char[3] = __mu1.as_char[3] + __mu2.as_char[3];
410  __res.as_char[4] = __mu1.as_char[4] + __mu2.as_char[4];
411  __res.as_char[5] = __mu1.as_char[5] + __mu2.as_char[5];
412  __res.as_char[6] = __mu1.as_char[6] + __mu2.as_char[6];
413  __res.as_char[7] = __mu1.as_char[7] + __mu2.as_char[7];
414
415  return (__m64)__res.as_m64;
416#endif
417}
418
419extern __inline __m64
420    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
421    _m_paddb(__m64 __m1, __m64 __m2) {
422  return _mm_add_pi8(__m1, __m2);
423}
424
425/* Add the 16-bit values in M1 to the 16-bit values in M2.  */
426extern __inline __m64
427    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
428    _mm_add_pi16(__m64 __m1, __m64 __m2) {
429#if _ARCH_PWR8
430  __vector signed short __a, __b, __c;
431
432  __a = (__vector signed short)vec_splats(__m1);
433  __b = (__vector signed short)vec_splats(__m2);
434  __c = vec_add(__a, __b);
435  return (__m64)((__vector long long)__c)[0];
436#else
437  __m64_union __mu1, __mu2, __res;
438
439  __mu1.as_m64 = __m1;
440  __mu2.as_m64 = __m2;
441
442  __res.as_short[0] = __mu1.as_short[0] + __mu2.as_short[0];
443  __res.as_short[1] = __mu1.as_short[1] + __mu2.as_short[1];
444  __res.as_short[2] = __mu1.as_short[2] + __mu2.as_short[2];
445  __res.as_short[3] = __mu1.as_short[3] + __mu2.as_short[3];
446
447  return (__m64)__res.as_m64;
448#endif
449}
450
451extern __inline __m64
452    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
453    _m_paddw(__m64 __m1, __m64 __m2) {
454  return _mm_add_pi16(__m1, __m2);
455}
456
457/* Add the 32-bit values in M1 to the 32-bit values in M2.  */
458extern __inline __m64
459    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
460    _mm_add_pi32(__m64 __m1, __m64 __m2) {
461#if _ARCH_PWR9
462  __vector signed int __a, __b, __c;
463
464  __a = (__vector signed int)vec_splats(__m1);
465  __b = (__vector signed int)vec_splats(__m2);
466  __c = vec_add(__a, __b);
467  return (__m64)((__vector long long)__c)[0];
468#else
469  __m64_union __mu1, __mu2, __res;
470
471  __mu1.as_m64 = __m1;
472  __mu2.as_m64 = __m2;
473
474  __res.as_int[0] = __mu1.as_int[0] + __mu2.as_int[0];
475  __res.as_int[1] = __mu1.as_int[1] + __mu2.as_int[1];
476
477  return (__m64)__res.as_m64;
478#endif
479}
480
481extern __inline __m64
482    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
483    _m_paddd(__m64 __m1, __m64 __m2) {
484  return _mm_add_pi32(__m1, __m2);
485}
486
487/* Subtract the 8-bit values in M2 from the 8-bit values in M1.  */
488extern __inline __m64
489    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
490    _mm_sub_pi8(__m64 __m1, __m64 __m2) {
491#if _ARCH_PWR8
492  __vector signed char __a, __b, __c;
493
494  __a = (__vector signed char)vec_splats(__m1);
495  __b = (__vector signed char)vec_splats(__m2);
496  __c = vec_sub(__a, __b);
497  return (__m64)((__vector long long)__c)[0];
498#else
499  __m64_union __mu1, __mu2, __res;
500
501  __mu1.as_m64 = __m1;
502  __mu2.as_m64 = __m2;
503
504  __res.as_char[0] = __mu1.as_char[0] - __mu2.as_char[0];
505  __res.as_char[1] = __mu1.as_char[1] - __mu2.as_char[1];
506  __res.as_char[2] = __mu1.as_char[2] - __mu2.as_char[2];
507  __res.as_char[3] = __mu1.as_char[3] - __mu2.as_char[3];
508  __res.as_char[4] = __mu1.as_char[4] - __mu2.as_char[4];
509  __res.as_char[5] = __mu1.as_char[5] - __mu2.as_char[5];
510  __res.as_char[6] = __mu1.as_char[6] - __mu2.as_char[6];
511  __res.as_char[7] = __mu1.as_char[7] - __mu2.as_char[7];
512
513  return (__m64)__res.as_m64;
514#endif
515}
516
517extern __inline __m64
518    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
519    _m_psubb(__m64 __m1, __m64 __m2) {
520  return _mm_sub_pi8(__m1, __m2);
521}
522
523/* Subtract the 16-bit values in M2 from the 16-bit values in M1.  */
524extern __inline __m64
525    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
526    _mm_sub_pi16(__m64 __m1, __m64 __m2) {
527#if _ARCH_PWR8
528  __vector signed short __a, __b, __c;
529
530  __a = (__vector signed short)vec_splats(__m1);
531  __b = (__vector signed short)vec_splats(__m2);
532  __c = vec_sub(__a, __b);
533  return (__m64)((__vector long long)__c)[0];
534#else
535  __m64_union __mu1, __mu2, __res;
536
537  __mu1.as_m64 = __m1;
538  __mu2.as_m64 = __m2;
539
540  __res.as_short[0] = __mu1.as_short[0] - __mu2.as_short[0];
541  __res.as_short[1] = __mu1.as_short[1] - __mu2.as_short[1];
542  __res.as_short[2] = __mu1.as_short[2] - __mu2.as_short[2];
543  __res.as_short[3] = __mu1.as_short[3] - __mu2.as_short[3];
544
545  return (__m64)__res.as_m64;
546#endif
547}
548
549extern __inline __m64
550    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
551    _m_psubw(__m64 __m1, __m64 __m2) {
552  return _mm_sub_pi16(__m1, __m2);
553}
554
555/* Subtract the 32-bit values in M2 from the 32-bit values in M1.  */
556extern __inline __m64
557    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
558    _mm_sub_pi32(__m64 __m1, __m64 __m2) {
559#if _ARCH_PWR9
560  __vector signed int __a, __b, __c;
561
562  __a = (__vector signed int)vec_splats(__m1);
563  __b = (__vector signed int)vec_splats(__m2);
564  __c = vec_sub(__a, __b);
565  return (__m64)((__vector long long)__c)[0];
566#else
567  __m64_union __mu1, __mu2, __res;
568
569  __mu1.as_m64 = __m1;
570  __mu2.as_m64 = __m2;
571
572  __res.as_int[0] = __mu1.as_int[0] - __mu2.as_int[0];
573  __res.as_int[1] = __mu1.as_int[1] - __mu2.as_int[1];
574
575  return (__m64)__res.as_m64;
576#endif
577}
578
579extern __inline __m64
580    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
581    _m_psubd(__m64 __m1, __m64 __m2) {
582  return _mm_sub_pi32(__m1, __m2);
583}
584
585extern __inline __m64
586    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
587    _mm_add_si64(__m64 __m1, __m64 __m2) {
588  return (__m1 + __m2);
589}
590
591extern __inline __m64
592    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
593    _mm_sub_si64(__m64 __m1, __m64 __m2) {
594  return (__m1 - __m2);
595}
596
597/* Shift the 64-bit value in M left by COUNT.  */
598extern __inline __m64
599    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
600    _mm_sll_si64(__m64 __m, __m64 __count) {
601  return (__m << __count);
602}
603
604extern __inline __m64
605    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
606    _m_psllq(__m64 __m, __m64 __count) {
607  return _mm_sll_si64(__m, __count);
608}
609
610extern __inline __m64
611    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
612    _mm_slli_si64(__m64 __m, const int __count) {
613  return (__m << __count);
614}
615
616extern __inline __m64
617    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
618    _m_psllqi(__m64 __m, const int __count) {
619  return _mm_slli_si64(__m, __count);
620}
621
622/* Shift the 64-bit value in M left by COUNT; shift in zeros.  */
623extern __inline __m64
624    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
625    _mm_srl_si64(__m64 __m, __m64 __count) {
626  return (__m >> __count);
627}
628
629extern __inline __m64
630    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
631    _m_psrlq(__m64 __m, __m64 __count) {
632  return _mm_srl_si64(__m, __count);
633}
634
635extern __inline __m64
636    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
637    _mm_srli_si64(__m64 __m, const int __count) {
638  return (__m >> __count);
639}
640
641extern __inline __m64
642    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
643    _m_psrlqi(__m64 __m, const int __count) {
644  return _mm_srli_si64(__m, __count);
645}
646
647/* Bit-wise AND the 64-bit values in M1 and M2.  */
648extern __inline __m64
649    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
650    _mm_and_si64(__m64 __m1, __m64 __m2) {
651  return (__m1 & __m2);
652}
653
654extern __inline __m64
655    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
656    _m_pand(__m64 __m1, __m64 __m2) {
657  return _mm_and_si64(__m1, __m2);
658}
659
660/* Bit-wise complement the 64-bit value in M1 and bit-wise AND it with the
661   64-bit value in M2.  */
662extern __inline __m64
663    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
664    _mm_andnot_si64(__m64 __m1, __m64 __m2) {
665  return (~__m1 & __m2);
666}
667
668extern __inline __m64
669    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
670    _m_pandn(__m64 __m1, __m64 __m2) {
671  return _mm_andnot_si64(__m1, __m2);
672}
673
674/* Bit-wise inclusive OR the 64-bit values in M1 and M2.  */
675extern __inline __m64
676    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
677    _mm_or_si64(__m64 __m1, __m64 __m2) {
678  return (__m1 | __m2);
679}
680
681extern __inline __m64
682    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
683    _m_por(__m64 __m1, __m64 __m2) {
684  return _mm_or_si64(__m1, __m2);
685}
686
687/* Bit-wise exclusive OR the 64-bit values in M1 and M2.  */
688extern __inline __m64
689    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
690    _mm_xor_si64(__m64 __m1, __m64 __m2) {
691  return (__m1 ^ __m2);
692}
693
694extern __inline __m64
695    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
696    _m_pxor(__m64 __m1, __m64 __m2) {
697  return _mm_xor_si64(__m1, __m2);
698}
699
700/* Creates a 64-bit zero.  */
701extern __inline __m64
702    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
703    _mm_setzero_si64(void) {
704  return (__m64)0;
705}
706
707/* Compare eight 8-bit values.  The result of the comparison is 0xFF if the
708   test is true and zero if false.  */
709extern __inline __m64
710    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
711    _mm_cmpeq_pi8(__m64 __m1, __m64 __m2) {
712#if defined(_ARCH_PWR6) && defined(__powerpc64__)
713  __m64 __res;
714  __asm__("cmpb %0,%1,%2;\n" : "=r"(__res) : "r"(__m1), "r"(__m2) :);
715  return (__res);
716#else
717  __m64_union __mu1, __mu2, __res;
718
719  __mu1.as_m64 = __m1;
720  __mu2.as_m64 = __m2;
721
722  __res.as_char[0] = (__mu1.as_char[0] == __mu2.as_char[0]) ? -1 : 0;
723  __res.as_char[1] = (__mu1.as_char[1] == __mu2.as_char[1]) ? -1 : 0;
724  __res.as_char[2] = (__mu1.as_char[2] == __mu2.as_char[2]) ? -1 : 0;
725  __res.as_char[3] = (__mu1.as_char[3] == __mu2.as_char[3]) ? -1 : 0;
726  __res.as_char[4] = (__mu1.as_char[4] == __mu2.as_char[4]) ? -1 : 0;
727  __res.as_char[5] = (__mu1.as_char[5] == __mu2.as_char[5]) ? -1 : 0;
728  __res.as_char[6] = (__mu1.as_char[6] == __mu2.as_char[6]) ? -1 : 0;
729  __res.as_char[7] = (__mu1.as_char[7] == __mu2.as_char[7]) ? -1 : 0;
730
731  return (__m64)__res.as_m64;
732#endif
733}
734
735extern __inline __m64
736    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
737    _m_pcmpeqb(__m64 __m1, __m64 __m2) {
738  return _mm_cmpeq_pi8(__m1, __m2);
739}
740
741extern __inline __m64
742    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
743    _mm_cmpgt_pi8(__m64 __m1, __m64 __m2) {
744#if _ARCH_PWR8
745  __vector signed char __a, __b, __c;
746
747  __a = (__vector signed char)vec_splats(__m1);
748  __b = (__vector signed char)vec_splats(__m2);
749  __c = (__vector signed char)vec_cmpgt(__a, __b);
750  return (__m64)((__vector long long)__c)[0];
751#else
752  __m64_union __mu1, __mu2, __res;
753
754  __mu1.as_m64 = __m1;
755  __mu2.as_m64 = __m2;
756
757  __res.as_char[0] = (__mu1.as_char[0] > __mu2.as_char[0]) ? -1 : 0;
758  __res.as_char[1] = (__mu1.as_char[1] > __mu2.as_char[1]) ? -1 : 0;
759  __res.as_char[2] = (__mu1.as_char[2] > __mu2.as_char[2]) ? -1 : 0;
760  __res.as_char[3] = (__mu1.as_char[3] > __mu2.as_char[3]) ? -1 : 0;
761  __res.as_char[4] = (__mu1.as_char[4] > __mu2.as_char[4]) ? -1 : 0;
762  __res.as_char[5] = (__mu1.as_char[5] > __mu2.as_char[5]) ? -1 : 0;
763  __res.as_char[6] = (__mu1.as_char[6] > __mu2.as_char[6]) ? -1 : 0;
764  __res.as_char[7] = (__mu1.as_char[7] > __mu2.as_char[7]) ? -1 : 0;
765
766  return (__m64)__res.as_m64;
767#endif
768}
769
770extern __inline __m64
771    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
772    _m_pcmpgtb(__m64 __m1, __m64 __m2) {
773  return _mm_cmpgt_pi8(__m1, __m2);
774}
775
776/* Compare four 16-bit values.  The result of the comparison is 0xFFFF if
777   the test is true and zero if false.  */
778extern __inline __m64
779    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
780    _mm_cmpeq_pi16(__m64 __m1, __m64 __m2) {
781#if _ARCH_PWR8
782  __vector signed short __a, __b, __c;
783
784  __a = (__vector signed short)vec_splats(__m1);
785  __b = (__vector signed short)vec_splats(__m2);
786  __c = (__vector signed short)vec_cmpeq(__a, __b);
787  return (__m64)((__vector long long)__c)[0];
788#else
789  __m64_union __mu1, __mu2, __res;
790
791  __mu1.as_m64 = __m1;
792  __mu2.as_m64 = __m2;
793
794  __res.as_short[0] = (__mu1.as_short[0] == __mu2.as_short[0]) ? -1 : 0;
795  __res.as_short[1] = (__mu1.as_short[1] == __mu2.as_short[1]) ? -1 : 0;
796  __res.as_short[2] = (__mu1.as_short[2] == __mu2.as_short[2]) ? -1 : 0;
797  __res.as_short[3] = (__mu1.as_short[3] == __mu2.as_short[3]) ? -1 : 0;
798
799  return (__m64)__res.as_m64;
800#endif
801}
802
803extern __inline __m64
804    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
805    _m_pcmpeqw(__m64 __m1, __m64 __m2) {
806  return _mm_cmpeq_pi16(__m1, __m2);
807}
808
809extern __inline __m64
810    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
811    _mm_cmpgt_pi16(__m64 __m1, __m64 __m2) {
812#if _ARCH_PWR8
813  __vector signed short __a, __b, __c;
814
815  __a = (__vector signed short)vec_splats(__m1);
816  __b = (__vector signed short)vec_splats(__m2);
817  __c = (__vector signed short)vec_cmpgt(__a, __b);
818  return (__m64)((__vector long long)__c)[0];
819#else
820  __m64_union __mu1, __mu2, __res;
821
822  __mu1.as_m64 = __m1;
823  __mu2.as_m64 = __m2;
824
825  __res.as_short[0] = (__mu1.as_short[0] > __mu2.as_short[0]) ? -1 : 0;
826  __res.as_short[1] = (__mu1.as_short[1] > __mu2.as_short[1]) ? -1 : 0;
827  __res.as_short[2] = (__mu1.as_short[2] > __mu2.as_short[2]) ? -1 : 0;
828  __res.as_short[3] = (__mu1.as_short[3] > __mu2.as_short[3]) ? -1 : 0;
829
830  return (__m64)__res.as_m64;
831#endif
832}
833
834extern __inline __m64
835    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
836    _m_pcmpgtw(__m64 __m1, __m64 __m2) {
837  return _mm_cmpgt_pi16(__m1, __m2);
838}
839
840/* Compare two 32-bit values.  The result of the comparison is 0xFFFFFFFF if
841   the test is true and zero if false.  */
842extern __inline __m64
843    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
844    _mm_cmpeq_pi32(__m64 __m1, __m64 __m2) {
845#if _ARCH_PWR9
846  __vector signed int __a, __b, __c;
847
848  __a = (__vector signed int)vec_splats(__m1);
849  __b = (__vector signed int)vec_splats(__m2);
850  __c = (__vector signed int)vec_cmpeq(__a, __b);
851  return (__m64)((__vector long long)__c)[0];
852#else
853  __m64_union __mu1, __mu2, __res;
854
855  __mu1.as_m64 = __m1;
856  __mu2.as_m64 = __m2;
857
858  __res.as_int[0] = (__mu1.as_int[0] == __mu2.as_int[0]) ? -1 : 0;
859  __res.as_int[1] = (__mu1.as_int[1] == __mu2.as_int[1]) ? -1 : 0;
860
861  return (__m64)__res.as_m64;
862#endif
863}
864
865extern __inline __m64
866    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
867    _m_pcmpeqd(__m64 __m1, __m64 __m2) {
868  return _mm_cmpeq_pi32(__m1, __m2);
869}
870
871extern __inline __m64
872    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
873    _mm_cmpgt_pi32(__m64 __m1, __m64 __m2) {
874#if _ARCH_PWR9
875  __vector signed int __a, __b, __c;
876
877  __a = (__vector signed int)vec_splats(__m1);
878  __b = (__vector signed int)vec_splats(__m2);
879  __c = (__vector signed int)vec_cmpgt(__a, __b);
880  return (__m64)((__vector long long)__c)[0];
881#else
882  __m64_union __mu1, __mu2, __res;
883
884  __mu1.as_m64 = __m1;
885  __mu2.as_m64 = __m2;
886
887  __res.as_int[0] = (__mu1.as_int[0] > __mu2.as_int[0]) ? -1 : 0;
888  __res.as_int[1] = (__mu1.as_int[1] > __mu2.as_int[1]) ? -1 : 0;
889
890  return (__m64)__res.as_m64;
891#endif
892}
893
894extern __inline __m64
895    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
896    _m_pcmpgtd(__m64 __m1, __m64 __m2) {
897  return _mm_cmpgt_pi32(__m1, __m2);
898}
899
900#if _ARCH_PWR8
901/* Add the 8-bit values in M1 to the 8-bit values in M2 using signed
902   saturated arithmetic.  */
903extern __inline __m64
904    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
905    _mm_adds_pi8(__m64 __m1, __m64 __m2) {
906  __vector signed char __a, __b, __c;
907
908  __a = (__vector signed char)vec_splats(__m1);
909  __b = (__vector signed char)vec_splats(__m2);
910  __c = vec_adds(__a, __b);
911  return (__m64)((__vector long long)__c)[0];
912}
913
914extern __inline __m64
915    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
916    _m_paddsb(__m64 __m1, __m64 __m2) {
917  return _mm_adds_pi8(__m1, __m2);
918}
919/* Add the 16-bit values in M1 to the 16-bit values in M2 using signed
920   saturated arithmetic.  */
921extern __inline __m64
922    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
923    _mm_adds_pi16(__m64 __m1, __m64 __m2) {
924  __vector signed short __a, __b, __c;
925
926  __a = (__vector signed short)vec_splats(__m1);
927  __b = (__vector signed short)vec_splats(__m2);
928  __c = vec_adds(__a, __b);
929  return (__m64)((__vector long long)__c)[0];
930}
931
932extern __inline __m64
933    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
934    _m_paddsw(__m64 __m1, __m64 __m2) {
935  return _mm_adds_pi16(__m1, __m2);
936}
937/* Add the 8-bit values in M1 to the 8-bit values in M2 using unsigned
938   saturated arithmetic.  */
939extern __inline __m64
940    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
941    _mm_adds_pu8(__m64 __m1, __m64 __m2) {
942  __vector unsigned char __a, __b, __c;
943
944  __a = (__vector unsigned char)vec_splats(__m1);
945  __b = (__vector unsigned char)vec_splats(__m2);
946  __c = vec_adds(__a, __b);
947  return (__m64)((__vector long long)__c)[0];
948}
949
950extern __inline __m64
951    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
952    _m_paddusb(__m64 __m1, __m64 __m2) {
953  return _mm_adds_pu8(__m1, __m2);
954}
955
956/* Add the 16-bit values in M1 to the 16-bit values in M2 using unsigned
957   saturated arithmetic.  */
958extern __inline __m64
959    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
960    _mm_adds_pu16(__m64 __m1, __m64 __m2) {
961  __vector unsigned short __a, __b, __c;
962
963  __a = (__vector unsigned short)vec_splats(__m1);
964  __b = (__vector unsigned short)vec_splats(__m2);
965  __c = vec_adds(__a, __b);
966  return (__m64)((__vector long long)__c)[0];
967}
968
969extern __inline __m64
970    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
971    _m_paddusw(__m64 __m1, __m64 __m2) {
972  return _mm_adds_pu16(__m1, __m2);
973}
974
975/* Subtract the 8-bit values in M2 from the 8-bit values in M1 using signed
976   saturating arithmetic.  */
977extern __inline __m64
978    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
979    _mm_subs_pi8(__m64 __m1, __m64 __m2) {
980  __vector signed char __a, __b, __c;
981
982  __a = (__vector signed char)vec_splats(__m1);
983  __b = (__vector signed char)vec_splats(__m2);
984  __c = vec_subs(__a, __b);
985  return (__m64)((__vector long long)__c)[0];
986}
987
988extern __inline __m64
989    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
990    _m_psubsb(__m64 __m1, __m64 __m2) {
991  return _mm_subs_pi8(__m1, __m2);
992}
993
994/* Subtract the 16-bit values in M2 from the 16-bit values in M1 using
995   signed saturating arithmetic.  */
996extern __inline __m64
997    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
998    _mm_subs_pi16(__m64 __m1, __m64 __m2) {
999  __vector signed short __a, __b, __c;
1000
1001  __a = (__vector signed short)vec_splats(__m1);
1002  __b = (__vector signed short)vec_splats(__m2);
1003  __c = vec_subs(__a, __b);
1004  return (__m64)((__vector long long)__c)[0];
1005}
1006
1007extern __inline __m64
1008    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1009    _m_psubsw(__m64 __m1, __m64 __m2) {
1010  return _mm_subs_pi16(__m1, __m2);
1011}
1012
1013/* Subtract the 8-bit values in M2 from the 8-bit values in M1 using
1014   unsigned saturating arithmetic.  */
1015extern __inline __m64
1016    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1017    _mm_subs_pu8(__m64 __m1, __m64 __m2) {
1018  __vector unsigned char __a, __b, __c;
1019
1020  __a = (__vector unsigned char)vec_splats(__m1);
1021  __b = (__vector unsigned char)vec_splats(__m2);
1022  __c = vec_subs(__a, __b);
1023  return (__m64)((__vector long long)__c)[0];
1024}
1025
1026extern __inline __m64
1027    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1028    _m_psubusb(__m64 __m1, __m64 __m2) {
1029  return _mm_subs_pu8(__m1, __m2);
1030}
1031
1032/* Subtract the 16-bit values in M2 from the 16-bit values in M1 using
1033   unsigned saturating arithmetic.  */
1034extern __inline __m64
1035    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1036    _mm_subs_pu16(__m64 __m1, __m64 __m2) {
1037  __vector unsigned short __a, __b, __c;
1038
1039  __a = (__vector unsigned short)vec_splats(__m1);
1040  __b = (__vector unsigned short)vec_splats(__m2);
1041  __c = vec_subs(__a, __b);
1042  return (__m64)((__vector long long)__c)[0];
1043}
1044
1045extern __inline __m64
1046    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1047    _m_psubusw(__m64 __m1, __m64 __m2) {
1048  return _mm_subs_pu16(__m1, __m2);
1049}
1050
1051/* Multiply four 16-bit values in M1 by four 16-bit values in M2 producing
1052   four 32-bit intermediate results, which are then summed by pairs to
1053   produce two 32-bit results.  */
1054extern __inline __m64
1055    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1056    _mm_madd_pi16(__m64 __m1, __m64 __m2) {
1057  __vector signed short __a, __b;
1058  __vector signed int __c;
1059  __vector signed int __zero = {0, 0, 0, 0};
1060
1061  __a = (__vector signed short)vec_splats(__m1);
1062  __b = (__vector signed short)vec_splats(__m2);
1063  __c = vec_vmsumshm(__a, __b, __zero);
1064  return (__m64)((__vector long long)__c)[0];
1065}
1066
1067extern __inline __m64
1068    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1069    _m_pmaddwd(__m64 __m1, __m64 __m2) {
1070  return _mm_madd_pi16(__m1, __m2);
1071}
1072/* Multiply four signed 16-bit values in M1 by four signed 16-bit values in
1073   M2 and produce the high 16 bits of the 32-bit results.  */
1074extern __inline __m64
1075    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1076    _mm_mulhi_pi16(__m64 __m1, __m64 __m2) {
1077  __vector signed short __a, __b;
1078  __vector signed short __c;
1079  __vector signed int __w0, __w1;
1080  __vector unsigned char __xform1 = {
1081#ifdef __LITTLE_ENDIAN__
1082      0x02, 0x03, 0x12, 0x13, 0x06, 0x07, 0x16, 0x17, 0x0A,
1083      0x0B, 0x1A, 0x1B, 0x0E, 0x0F, 0x1E, 0x1F
1084#else
1085      0x00, 0x01, 0x10, 0x11, 0x04, 0x05, 0x14, 0x15, 0x00,
1086      0x01, 0x10, 0x11, 0x04, 0x05, 0x14, 0x15
1087#endif
1088  };
1089
1090  __a = (__vector signed short)vec_splats(__m1);
1091  __b = (__vector signed short)vec_splats(__m2);
1092
1093  __w0 = vec_vmulesh(__a, __b);
1094  __w1 = vec_vmulosh(__a, __b);
1095  __c = (__vector signed short)vec_perm(__w0, __w1, __xform1);
1096
1097  return (__m64)((__vector long long)__c)[0];
1098}
1099
1100extern __inline __m64
1101    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1102    _m_pmulhw(__m64 __m1, __m64 __m2) {
1103  return _mm_mulhi_pi16(__m1, __m2);
1104}
1105
1106/* Multiply four 16-bit values in M1 by four 16-bit values in M2 and produce
1107   the low 16 bits of the results.  */
1108extern __inline __m64
1109    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1110    _mm_mullo_pi16(__m64 __m1, __m64 __m2) {
1111  __vector signed short __a, __b, __c;
1112
1113  __a = (__vector signed short)vec_splats(__m1);
1114  __b = (__vector signed short)vec_splats(__m2);
1115  __c = __a * __b;
1116  return (__m64)((__vector long long)__c)[0];
1117}
1118
1119extern __inline __m64
1120    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1121    _m_pmullw(__m64 __m1, __m64 __m2) {
1122  return _mm_mullo_pi16(__m1, __m2);
1123}
1124
1125/* Shift four 16-bit values in M left by COUNT.  */
1126extern __inline __m64
1127    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1128    _mm_sll_pi16(__m64 __m, __m64 __count) {
1129  __vector signed short __r;
1130  __vector unsigned short __c;
1131
1132  if (__count <= 15) {
1133    __r = (__vector signed short)vec_splats(__m);
1134    __c = (__vector unsigned short)vec_splats((unsigned short)__count);
1135    __r = vec_sl(__r, (__vector unsigned short)__c);
1136    return (__m64)((__vector long long)__r)[0];
1137  } else
1138    return (0);
1139}
1140
1141extern __inline __m64
1142    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1143    _m_psllw(__m64 __m, __m64 __count) {
1144  return _mm_sll_pi16(__m, __count);
1145}
1146
1147extern __inline __m64
1148    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1149    _mm_slli_pi16(__m64 __m, int __count) {
1150  /* Promote int to long then invoke mm_sll_pi16.  */
1151  return _mm_sll_pi16(__m, __count);
1152}
1153
1154extern __inline __m64
1155    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1156    _m_psllwi(__m64 __m, int __count) {
1157  return _mm_slli_pi16(__m, __count);
1158}
1159
1160/* Shift two 32-bit values in M left by COUNT.  */
1161extern __inline __m64
1162    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1163    _mm_sll_pi32(__m64 __m, __m64 __count) {
1164  __m64_union __res;
1165
1166  __res.as_m64 = __m;
1167
1168  __res.as_int[0] = __res.as_int[0] << __count;
1169  __res.as_int[1] = __res.as_int[1] << __count;
1170  return (__res.as_m64);
1171}
1172
1173extern __inline __m64
1174    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1175    _m_pslld(__m64 __m, __m64 __count) {
1176  return _mm_sll_pi32(__m, __count);
1177}
1178
1179extern __inline __m64
1180    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1181    _mm_slli_pi32(__m64 __m, int __count) {
1182  /* Promote int to long then invoke mm_sll_pi32.  */
1183  return _mm_sll_pi32(__m, __count);
1184}
1185
1186extern __inline __m64
1187    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1188    _m_pslldi(__m64 __m, int __count) {
1189  return _mm_slli_pi32(__m, __count);
1190}
1191
1192/* Shift four 16-bit values in M right by COUNT; shift in the sign bit.  */
1193extern __inline __m64
1194    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1195    _mm_sra_pi16(__m64 __m, __m64 __count) {
1196  __vector signed short __r;
1197  __vector unsigned short __c;
1198
1199  if (__count <= 15) {
1200    __r = (__vector signed short)vec_splats(__m);
1201    __c = (__vector unsigned short)vec_splats((unsigned short)__count);
1202    __r = vec_sra(__r, (__vector unsigned short)__c);
1203    return (__m64)((__vector long long)__r)[0];
1204  } else
1205    return (0);
1206}
1207
1208extern __inline __m64
1209    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1210    _m_psraw(__m64 __m, __m64 __count) {
1211  return _mm_sra_pi16(__m, __count);
1212}
1213
1214extern __inline __m64
1215    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1216    _mm_srai_pi16(__m64 __m, int __count) {
1217  /* Promote int to long then invoke mm_sra_pi32.  */
1218  return _mm_sra_pi16(__m, __count);
1219}
1220
1221extern __inline __m64
1222    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1223    _m_psrawi(__m64 __m, int __count) {
1224  return _mm_srai_pi16(__m, __count);
1225}
1226
1227/* Shift two 32-bit values in M right by COUNT; shift in the sign bit.  */
1228extern __inline __m64
1229    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1230    _mm_sra_pi32(__m64 __m, __m64 __count) {
1231  __m64_union __res;
1232
1233  __res.as_m64 = __m;
1234
1235  __res.as_int[0] = __res.as_int[0] >> __count;
1236  __res.as_int[1] = __res.as_int[1] >> __count;
1237  return (__res.as_m64);
1238}
1239
1240extern __inline __m64
1241    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1242    _m_psrad(__m64 __m, __m64 __count) {
1243  return _mm_sra_pi32(__m, __count);
1244}
1245
1246extern __inline __m64
1247    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1248    _mm_srai_pi32(__m64 __m, int __count) {
1249  /* Promote int to long then invoke mm_sra_pi32.  */
1250  return _mm_sra_pi32(__m, __count);
1251}
1252
1253extern __inline __m64
1254    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1255    _m_psradi(__m64 __m, int __count) {
1256  return _mm_srai_pi32(__m, __count);
1257}
1258
1259/* Shift four 16-bit values in M right by COUNT; shift in zeros.  */
1260extern __inline __m64
1261    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1262    _mm_srl_pi16(__m64 __m, __m64 __count) {
1263  __vector unsigned short __r;
1264  __vector unsigned short __c;
1265
1266  if (__count <= 15) {
1267    __r = (__vector unsigned short)vec_splats(__m);
1268    __c = (__vector unsigned short)vec_splats((unsigned short)__count);
1269    __r = vec_sr(__r, (__vector unsigned short)__c);
1270    return (__m64)((__vector long long)__r)[0];
1271  } else
1272    return (0);
1273}
1274
1275extern __inline __m64
1276    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1277    _m_psrlw(__m64 __m, __m64 __count) {
1278  return _mm_srl_pi16(__m, __count);
1279}
1280
1281extern __inline __m64
1282    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1283    _mm_srli_pi16(__m64 __m, int __count) {
1284  /* Promote int to long then invoke mm_sra_pi32.  */
1285  return _mm_srl_pi16(__m, __count);
1286}
1287
1288extern __inline __m64
1289    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1290    _m_psrlwi(__m64 __m, int __count) {
1291  return _mm_srli_pi16(__m, __count);
1292}
1293
1294/* Shift two 32-bit values in M right by COUNT; shift in zeros.  */
1295extern __inline __m64
1296    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1297    _mm_srl_pi32(__m64 __m, __m64 __count) {
1298  __m64_union __res;
1299
1300  __res.as_m64 = __m;
1301
1302  __res.as_int[0] = (unsigned int)__res.as_int[0] >> __count;
1303  __res.as_int[1] = (unsigned int)__res.as_int[1] >> __count;
1304  return (__res.as_m64);
1305}
1306
1307extern __inline __m64
1308    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1309    _m_psrld(__m64 __m, __m64 __count) {
1310  return _mm_srl_pi32(__m, __count);
1311}
1312
1313extern __inline __m64
1314    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1315    _mm_srli_pi32(__m64 __m, int __count) {
1316  /* Promote int to long then invoke mm_srl_pi32.  */
1317  return _mm_srl_pi32(__m, __count);
1318}
1319
1320extern __inline __m64
1321    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1322    _m_psrldi(__m64 __m, int __count) {
1323  return _mm_srli_pi32(__m, __count);
1324}
1325#endif /* _ARCH_PWR8 */
1326
1327/* Creates a vector of two 32-bit values; I0 is least significant.  */
1328extern __inline __m64
1329    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1330    _mm_set_pi32(int __i1, int __i0) {
1331  __m64_union __res;
1332
1333  __res.as_int[0] = __i0;
1334  __res.as_int[1] = __i1;
1335  return (__res.as_m64);
1336}
1337
1338/* Creates a vector of four 16-bit values; W0 is least significant.  */
1339extern __inline __m64
1340    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1341    _mm_set_pi16(short __w3, short __w2, short __w1, short __w0) {
1342  __m64_union __res;
1343
1344  __res.as_short[0] = __w0;
1345  __res.as_short[1] = __w1;
1346  __res.as_short[2] = __w2;
1347  __res.as_short[3] = __w3;
1348  return (__res.as_m64);
1349}
1350
1351/* Creates a vector of eight 8-bit values; B0 is least significant.  */
1352extern __inline __m64
1353    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1354    _mm_set_pi8(char __b7, char __b6, char __b5, char __b4, char __b3,
1355                char __b2, char __b1, char __b0) {
1356  __m64_union __res;
1357
1358  __res.as_char[0] = __b0;
1359  __res.as_char[1] = __b1;
1360  __res.as_char[2] = __b2;
1361  __res.as_char[3] = __b3;
1362  __res.as_char[4] = __b4;
1363  __res.as_char[5] = __b5;
1364  __res.as_char[6] = __b6;
1365  __res.as_char[7] = __b7;
1366  return (__res.as_m64);
1367}
1368
1369/* Similar, but with the arguments in reverse order.  */
1370extern __inline __m64
1371    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1372    _mm_setr_pi32(int __i0, int __i1) {
1373  __m64_union __res;
1374
1375  __res.as_int[0] = __i0;
1376  __res.as_int[1] = __i1;
1377  return (__res.as_m64);
1378}
1379
1380extern __inline __m64
1381    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1382    _mm_setr_pi16(short __w0, short __w1, short __w2, short __w3) {
1383  return _mm_set_pi16(__w3, __w2, __w1, __w0);
1384}
1385
1386extern __inline __m64
1387    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1388    _mm_setr_pi8(char __b0, char __b1, char __b2, char __b3, char __b4,
1389                 char __b5, char __b6, char __b7) {
1390  return _mm_set_pi8(__b7, __b6, __b5, __b4, __b3, __b2, __b1, __b0);
1391}
1392
1393/* Creates a vector of two 32-bit values, both elements containing I.  */
1394extern __inline __m64
1395    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1396    _mm_set1_pi32(int __i) {
1397  __m64_union __res;
1398
1399  __res.as_int[0] = __i;
1400  __res.as_int[1] = __i;
1401  return (__res.as_m64);
1402}
1403
1404/* Creates a vector of four 16-bit values, all elements containing W.  */
1405extern __inline __m64
1406    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1407    _mm_set1_pi16(short __w) {
1408#if _ARCH_PWR9
1409  __vector signed short w;
1410
1411  w = (__vector signed short)vec_splats(__w);
1412  return (__m64)((__vector long long)w)[0];
1413#else
1414  __m64_union __res;
1415
1416  __res.as_short[0] = __w;
1417  __res.as_short[1] = __w;
1418  __res.as_short[2] = __w;
1419  __res.as_short[3] = __w;
1420  return (__res.as_m64);
1421#endif
1422}
1423
1424/* Creates a vector of eight 8-bit values, all elements containing B.  */
1425extern __inline __m64
1426    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
1427    _mm_set1_pi8(signed char __b) {
1428#if _ARCH_PWR8
1429  __vector signed char __res;
1430
1431  __res = (__vector signed char)vec_splats(__b);
1432  return (__m64)((__vector long long)__res)[0];
1433#else
1434  __m64_union __res;
1435
1436  __res.as_char[0] = __b;
1437  __res.as_char[1] = __b;
1438  __res.as_char[2] = __b;
1439  __res.as_char[3] = __b;
1440  __res.as_char[4] = __b;
1441  __res.as_char[5] = __b;
1442  __res.as_char[6] = __b;
1443  __res.as_char[7] = __b;
1444  return (__res.as_m64);
1445#endif
1446}
1447
1448#else
1449#include_next <mmintrin.h>
1450#endif /* defined(__powerpc64__) &&                                            \
1451        *   (defined(__linux__) || defined(__FreeBSD__) || defined(_AIX)) */
1452
1453#endif /* _MMINTRIN_H_INCLUDED */
1454