xmmintrin.h revision 117395
190075Sobrien/* Copyright (C) 2002 Free Software Foundation, Inc. 290075Sobrien 390075Sobrien This file is part of GNU CC. 490075Sobrien 590075Sobrien GNU CC is free software; you can redistribute it and/or modify 690075Sobrien it under the terms of the GNU General Public License as published by 790075Sobrien the Free Software Foundation; either version 2, or (at your option) 890075Sobrien any later version. 990075Sobrien 1090075Sobrien GNU CC is distributed in the hope that it will be useful, 1190075Sobrien but WITHOUT ANY WARRANTY; without even the implied warranty of 1290075Sobrien MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 1390075Sobrien GNU General Public License for more details. 1490075Sobrien 1590075Sobrien You should have received a copy of the GNU General Public License 1690075Sobrien along with GNU CC; see the file COPYING. If not, write to 1790075Sobrien the Free Software Foundation, 59 Temple Place - Suite 330, 1890075Sobrien Boston, MA 02111-1307, USA. */ 1990075Sobrien 2090075Sobrien/* As a special exception, if you include this header file into source 2190075Sobrien files compiled by GCC, this header file does not by itself cause 2290075Sobrien the resulting executable to be covered by the GNU General Public 2390075Sobrien License. This exception does not however invalidate any other 2490075Sobrien reasons why the executable file might be covered by the GNU General 2590075Sobrien Public License. */ 2690075Sobrien 2790075Sobrien/* Implemented from the specification included in the Intel C++ Compiler 2890075Sobrien User Guide and Reference, version 5.0. */ 2990075Sobrien 3090075Sobrien#ifndef _XMMINTRIN_H_INCLUDED 3190075Sobrien#define _XMMINTRIN_H_INCLUDED 3290075Sobrien 33117395Skan#ifndef __SSE__ 34117395Skan# error "SSE instruction set not enabled" 35117395Skan#else 36117395Skan 3790075Sobrien/* We need type definitions from the MMX header file. */ 3890075Sobrien#include <mmintrin.h> 3990075Sobrien 4090075Sobrien/* The data type indended for user use. */ 4190075Sobrientypedef int __m128 __attribute__ ((__mode__(__V4SF__))); 4290075Sobrien 4390075Sobrien/* Internal data types for implementing the instrinsics. */ 4490075Sobrientypedef int __v4sf __attribute__ ((__mode__(__V4SF__))); 4590075Sobrientypedef int __v4si __attribute__ ((__mode__(__V4SI__))); 4690075Sobrien 4790075Sobrien/* Create a selector for use with the SHUFPS instruction. */ 4890075Sobrien#define _MM_SHUFFLE(fp3,fp2,fp1,fp0) \ 4990075Sobrien (((fp3) << 6) | ((fp2) << 4) | ((fp1) << 2) | (fp0)) 5090075Sobrien 5190075Sobrien/* Constants for use with _mm_prefetch. */ 5290075Sobrienenum _mm_hint 5390075Sobrien{ 5490075Sobrien _MM_HINT_T0 = 3, 5590075Sobrien _MM_HINT_T1 = 2, 5690075Sobrien _MM_HINT_T2 = 1, 5790075Sobrien _MM_HINT_NTA = 0 5890075Sobrien}; 5990075Sobrien 6090075Sobrien/* Bits in the MXCSR. */ 6190075Sobrien#define _MM_EXCEPT_MASK 0x003f 6290075Sobrien#define _MM_EXCEPT_INVALID 0x0001 6390075Sobrien#define _MM_EXCEPT_DENORM 0x0002 6490075Sobrien#define _MM_EXCEPT_DIV_ZERO 0x0004 6590075Sobrien#define _MM_EXCEPT_OVERFLOW 0x0008 6690075Sobrien#define _MM_EXCEPT_UNDERFLOW 0x0010 6790075Sobrien#define _MM_EXCEPT_INEXACT 0x0020 6890075Sobrien 6990075Sobrien#define _MM_MASK_MASK 0x1f80 7090075Sobrien#define _MM_MASK_INVALID 0x0080 7190075Sobrien#define _MM_MASK_DENORM 0x0100 7290075Sobrien#define _MM_MASK_DIV_ZERO 0x0200 7390075Sobrien#define _MM_MASK_OVERFLOW 0x0400 7490075Sobrien#define _MM_MASK_UNDERFLOW 0x0800 7590075Sobrien#define _MM_MASK_INEXACT 0x1000 7690075Sobrien 7790075Sobrien#define _MM_ROUND_MASK 0x6000 7890075Sobrien#define _MM_ROUND_NEAREST 0x0000 7990075Sobrien#define _MM_ROUND_DOWN 0x2000 8090075Sobrien#define _MM_ROUND_UP 0x4000 8190075Sobrien#define _MM_ROUND_TOWARD_ZERO 0x6000 8290075Sobrien 8390075Sobrien#define _MM_FLUSH_ZERO_MASK 0x8000 8490075Sobrien#define _MM_FLUSH_ZERO_ON 0x8000 8590075Sobrien#define _MM_FLUSH_ZERO_OFF 0x0000 8690075Sobrien 8790075Sobrien/* Perform the respective operation on the lower SPFP (single-precision 8890075Sobrien floating-point) values of A and B; the upper three SPFP values are 8990075Sobrien passed through from A. */ 9090075Sobrien 9190075Sobrienstatic __inline __m128 9290075Sobrien_mm_add_ss (__m128 __A, __m128 __B) 9390075Sobrien{ 9490075Sobrien return (__m128) __builtin_ia32_addss ((__v4sf)__A, (__v4sf)__B); 9590075Sobrien} 9690075Sobrien 9790075Sobrienstatic __inline __m128 9890075Sobrien_mm_sub_ss (__m128 __A, __m128 __B) 9990075Sobrien{ 10090075Sobrien return (__m128) __builtin_ia32_subss ((__v4sf)__A, (__v4sf)__B); 10190075Sobrien} 10290075Sobrien 10390075Sobrienstatic __inline __m128 10490075Sobrien_mm_mul_ss (__m128 __A, __m128 __B) 10590075Sobrien{ 10690075Sobrien return (__m128) __builtin_ia32_mulss ((__v4sf)__A, (__v4sf)__B); 10790075Sobrien} 10890075Sobrien 10990075Sobrienstatic __inline __m128 11090075Sobrien_mm_div_ss (__m128 __A, __m128 __B) 11190075Sobrien{ 11290075Sobrien return (__m128) __builtin_ia32_divss ((__v4sf)__A, (__v4sf)__B); 11390075Sobrien} 11490075Sobrien 11590075Sobrienstatic __inline __m128 11690075Sobrien_mm_sqrt_ss (__m128 __A) 11790075Sobrien{ 11890075Sobrien return (__m128) __builtin_ia32_sqrtss ((__v4sf)__A); 11990075Sobrien} 12090075Sobrien 12190075Sobrienstatic __inline __m128 12290075Sobrien_mm_rcp_ss (__m128 __A) 12390075Sobrien{ 12490075Sobrien return (__m128) __builtin_ia32_rcpss ((__v4sf)__A); 12590075Sobrien} 12690075Sobrien 12790075Sobrienstatic __inline __m128 12890075Sobrien_mm_rsqrt_ss (__m128 __A) 12990075Sobrien{ 13090075Sobrien return (__m128) __builtin_ia32_rsqrtss ((__v4sf)__A); 13190075Sobrien} 13290075Sobrien 13390075Sobrienstatic __inline __m128 13490075Sobrien_mm_min_ss (__m128 __A, __m128 __B) 13590075Sobrien{ 13690075Sobrien return (__m128) __builtin_ia32_minss ((__v4sf)__A, (__v4sf)__B); 13790075Sobrien} 13890075Sobrien 13990075Sobrienstatic __inline __m128 14090075Sobrien_mm_max_ss (__m128 __A, __m128 __B) 14190075Sobrien{ 14290075Sobrien return (__m128) __builtin_ia32_maxss ((__v4sf)__A, (__v4sf)__B); 14390075Sobrien} 14490075Sobrien 14590075Sobrien/* Perform the respective operation on the four SPFP values in A and B. */ 14690075Sobrien 14790075Sobrienstatic __inline __m128 14890075Sobrien_mm_add_ps (__m128 __A, __m128 __B) 14990075Sobrien{ 15090075Sobrien return (__m128) __builtin_ia32_addps ((__v4sf)__A, (__v4sf)__B); 15190075Sobrien} 15290075Sobrien 15390075Sobrienstatic __inline __m128 15490075Sobrien_mm_sub_ps (__m128 __A, __m128 __B) 15590075Sobrien{ 15690075Sobrien return (__m128) __builtin_ia32_subps ((__v4sf)__A, (__v4sf)__B); 15790075Sobrien} 15890075Sobrien 15990075Sobrienstatic __inline __m128 16090075Sobrien_mm_mul_ps (__m128 __A, __m128 __B) 16190075Sobrien{ 16290075Sobrien return (__m128) __builtin_ia32_mulps ((__v4sf)__A, (__v4sf)__B); 16390075Sobrien} 16490075Sobrien 16590075Sobrienstatic __inline __m128 16690075Sobrien_mm_div_ps (__m128 __A, __m128 __B) 16790075Sobrien{ 16890075Sobrien return (__m128) __builtin_ia32_divps ((__v4sf)__A, (__v4sf)__B); 16990075Sobrien} 17090075Sobrien 17190075Sobrienstatic __inline __m128 17290075Sobrien_mm_sqrt_ps (__m128 __A) 17390075Sobrien{ 17490075Sobrien return (__m128) __builtin_ia32_sqrtps ((__v4sf)__A); 17590075Sobrien} 17690075Sobrien 17790075Sobrienstatic __inline __m128 17890075Sobrien_mm_rcp_ps (__m128 __A) 17990075Sobrien{ 18090075Sobrien return (__m128) __builtin_ia32_rcpps ((__v4sf)__A); 18190075Sobrien} 18290075Sobrien 18390075Sobrienstatic __inline __m128 18490075Sobrien_mm_rsqrt_ps (__m128 __A) 18590075Sobrien{ 18690075Sobrien return (__m128) __builtin_ia32_rsqrtps ((__v4sf)__A); 18790075Sobrien} 18890075Sobrien 18990075Sobrienstatic __inline __m128 19090075Sobrien_mm_min_ps (__m128 __A, __m128 __B) 19190075Sobrien{ 19290075Sobrien return (__m128) __builtin_ia32_minps ((__v4sf)__A, (__v4sf)__B); 19390075Sobrien} 19490075Sobrien 19590075Sobrienstatic __inline __m128 19690075Sobrien_mm_max_ps (__m128 __A, __m128 __B) 19790075Sobrien{ 19890075Sobrien return (__m128) __builtin_ia32_maxps ((__v4sf)__A, (__v4sf)__B); 19990075Sobrien} 20090075Sobrien 20190075Sobrien/* Perform logical bit-wise operations on 128-bit values. */ 20290075Sobrien 20390075Sobrienstatic __inline __m128 20490075Sobrien_mm_and_ps (__m128 __A, __m128 __B) 20590075Sobrien{ 20690075Sobrien return __builtin_ia32_andps (__A, __B); 20790075Sobrien} 20890075Sobrien 20990075Sobrienstatic __inline __m128 21090075Sobrien_mm_andnot_ps (__m128 __A, __m128 __B) 21190075Sobrien{ 21290075Sobrien return __builtin_ia32_andnps (__A, __B); 21390075Sobrien} 21490075Sobrien 21590075Sobrienstatic __inline __m128 21690075Sobrien_mm_or_ps (__m128 __A, __m128 __B) 21790075Sobrien{ 21890075Sobrien return __builtin_ia32_orps (__A, __B); 21990075Sobrien} 22090075Sobrien 22190075Sobrienstatic __inline __m128 22290075Sobrien_mm_xor_ps (__m128 __A, __m128 __B) 22390075Sobrien{ 22490075Sobrien return __builtin_ia32_xorps (__A, __B); 22590075Sobrien} 22690075Sobrien 22790075Sobrien/* Perform a comparison on the lower SPFP values of A and B. If the 22890075Sobrien comparison is true, place a mask of all ones in the result, otherwise a 22990075Sobrien mask of zeros. The upper three SPFP values are passed through from A. */ 23090075Sobrien 23190075Sobrienstatic __inline __m128 23290075Sobrien_mm_cmpeq_ss (__m128 __A, __m128 __B) 23390075Sobrien{ 23490075Sobrien return (__m128) __builtin_ia32_cmpeqss ((__v4sf)__A, (__v4sf)__B); 23590075Sobrien} 23690075Sobrien 23790075Sobrienstatic __inline __m128 23890075Sobrien_mm_cmplt_ss (__m128 __A, __m128 __B) 23990075Sobrien{ 24090075Sobrien return (__m128) __builtin_ia32_cmpltss ((__v4sf)__A, (__v4sf)__B); 24190075Sobrien} 24290075Sobrien 24390075Sobrienstatic __inline __m128 24490075Sobrien_mm_cmple_ss (__m128 __A, __m128 __B) 24590075Sobrien{ 24690075Sobrien return (__m128) __builtin_ia32_cmpless ((__v4sf)__A, (__v4sf)__B); 24790075Sobrien} 24890075Sobrien 24990075Sobrienstatic __inline __m128 25090075Sobrien_mm_cmpgt_ss (__m128 __A, __m128 __B) 25190075Sobrien{ 252107590Sobrien return (__m128) __builtin_ia32_movss ((__v4sf) __A, 253107590Sobrien (__v4sf) 254107590Sobrien __builtin_ia32_cmpltss ((__v4sf) __B, 255107590Sobrien (__v4sf) 256107590Sobrien __A)); 25790075Sobrien} 25890075Sobrien 25990075Sobrienstatic __inline __m128 26090075Sobrien_mm_cmpge_ss (__m128 __A, __m128 __B) 26190075Sobrien{ 262107590Sobrien return (__m128) __builtin_ia32_movss ((__v4sf) __A, 263107590Sobrien (__v4sf) 264107590Sobrien __builtin_ia32_cmpless ((__v4sf) __B, 265107590Sobrien (__v4sf) 266107590Sobrien __A)); 26790075Sobrien} 26890075Sobrien 26990075Sobrienstatic __inline __m128 27090075Sobrien_mm_cmpneq_ss (__m128 __A, __m128 __B) 27190075Sobrien{ 27290075Sobrien return (__m128) __builtin_ia32_cmpneqss ((__v4sf)__A, (__v4sf)__B); 27390075Sobrien} 27490075Sobrien 27590075Sobrienstatic __inline __m128 27690075Sobrien_mm_cmpnlt_ss (__m128 __A, __m128 __B) 27790075Sobrien{ 27890075Sobrien return (__m128) __builtin_ia32_cmpnltss ((__v4sf)__A, (__v4sf)__B); 27990075Sobrien} 28090075Sobrien 28190075Sobrienstatic __inline __m128 28290075Sobrien_mm_cmpnle_ss (__m128 __A, __m128 __B) 28390075Sobrien{ 28490075Sobrien return (__m128) __builtin_ia32_cmpnless ((__v4sf)__A, (__v4sf)__B); 28590075Sobrien} 28690075Sobrien 28790075Sobrienstatic __inline __m128 28890075Sobrien_mm_cmpngt_ss (__m128 __A, __m128 __B) 28990075Sobrien{ 290107590Sobrien return (__m128) __builtin_ia32_movss ((__v4sf) __A, 291107590Sobrien (__v4sf) 292107590Sobrien __builtin_ia32_cmpnltss ((__v4sf) __B, 293107590Sobrien (__v4sf) 294107590Sobrien __A)); 29590075Sobrien} 29690075Sobrien 29790075Sobrienstatic __inline __m128 29890075Sobrien_mm_cmpnge_ss (__m128 __A, __m128 __B) 29990075Sobrien{ 300107590Sobrien return (__m128) __builtin_ia32_movss ((__v4sf) __A, 301107590Sobrien (__v4sf) 302107590Sobrien __builtin_ia32_cmpnless ((__v4sf) __B, 303107590Sobrien (__v4sf) 304107590Sobrien __A)); 30590075Sobrien} 30690075Sobrien 30790075Sobrienstatic __inline __m128 30890075Sobrien_mm_cmpord_ss (__m128 __A, __m128 __B) 30990075Sobrien{ 31090075Sobrien return (__m128) __builtin_ia32_cmpordss ((__v4sf)__A, (__v4sf)__B); 31190075Sobrien} 31290075Sobrien 31390075Sobrienstatic __inline __m128 31490075Sobrien_mm_cmpunord_ss (__m128 __A, __m128 __B) 31590075Sobrien{ 31690075Sobrien return (__m128) __builtin_ia32_cmpunordss ((__v4sf)__A, (__v4sf)__B); 31790075Sobrien} 31890075Sobrien 31990075Sobrien/* Perform a comparison on the four SPFP values of A and B. For each 32090075Sobrien element, if the comparison is true, place a mask of all ones in the 32190075Sobrien result, otherwise a mask of zeros. */ 32290075Sobrien 32390075Sobrienstatic __inline __m128 32490075Sobrien_mm_cmpeq_ps (__m128 __A, __m128 __B) 32590075Sobrien{ 32690075Sobrien return (__m128) __builtin_ia32_cmpeqps ((__v4sf)__A, (__v4sf)__B); 32790075Sobrien} 32890075Sobrien 32990075Sobrienstatic __inline __m128 33090075Sobrien_mm_cmplt_ps (__m128 __A, __m128 __B) 33190075Sobrien{ 33290075Sobrien return (__m128) __builtin_ia32_cmpltps ((__v4sf)__A, (__v4sf)__B); 33390075Sobrien} 33490075Sobrien 33590075Sobrienstatic __inline __m128 33690075Sobrien_mm_cmple_ps (__m128 __A, __m128 __B) 33790075Sobrien{ 33890075Sobrien return (__m128) __builtin_ia32_cmpleps ((__v4sf)__A, (__v4sf)__B); 33990075Sobrien} 34090075Sobrien 34190075Sobrienstatic __inline __m128 34290075Sobrien_mm_cmpgt_ps (__m128 __A, __m128 __B) 34390075Sobrien{ 34490075Sobrien return (__m128) __builtin_ia32_cmpgtps ((__v4sf)__A, (__v4sf)__B); 34590075Sobrien} 34690075Sobrien 34790075Sobrienstatic __inline __m128 34890075Sobrien_mm_cmpge_ps (__m128 __A, __m128 __B) 34990075Sobrien{ 35090075Sobrien return (__m128) __builtin_ia32_cmpgeps ((__v4sf)__A, (__v4sf)__B); 35190075Sobrien} 35290075Sobrien 35390075Sobrienstatic __inline __m128 35490075Sobrien_mm_cmpneq_ps (__m128 __A, __m128 __B) 35590075Sobrien{ 35690075Sobrien return (__m128) __builtin_ia32_cmpneqps ((__v4sf)__A, (__v4sf)__B); 35790075Sobrien} 35890075Sobrien 35990075Sobrienstatic __inline __m128 36090075Sobrien_mm_cmpnlt_ps (__m128 __A, __m128 __B) 36190075Sobrien{ 36290075Sobrien return (__m128) __builtin_ia32_cmpnltps ((__v4sf)__A, (__v4sf)__B); 36390075Sobrien} 36490075Sobrien 36590075Sobrienstatic __inline __m128 36690075Sobrien_mm_cmpnle_ps (__m128 __A, __m128 __B) 36790075Sobrien{ 36890075Sobrien return (__m128) __builtin_ia32_cmpnleps ((__v4sf)__A, (__v4sf)__B); 36990075Sobrien} 37090075Sobrien 37190075Sobrienstatic __inline __m128 37290075Sobrien_mm_cmpngt_ps (__m128 __A, __m128 __B) 37390075Sobrien{ 37490075Sobrien return (__m128) __builtin_ia32_cmpngtps ((__v4sf)__A, (__v4sf)__B); 37590075Sobrien} 37690075Sobrien 37790075Sobrienstatic __inline __m128 37890075Sobrien_mm_cmpnge_ps (__m128 __A, __m128 __B) 37990075Sobrien{ 38090075Sobrien return (__m128) __builtin_ia32_cmpngeps ((__v4sf)__A, (__v4sf)__B); 38190075Sobrien} 38290075Sobrien 38390075Sobrienstatic __inline __m128 38490075Sobrien_mm_cmpord_ps (__m128 __A, __m128 __B) 38590075Sobrien{ 38690075Sobrien return (__m128) __builtin_ia32_cmpordps ((__v4sf)__A, (__v4sf)__B); 38790075Sobrien} 38890075Sobrien 38990075Sobrienstatic __inline __m128 39090075Sobrien_mm_cmpunord_ps (__m128 __A, __m128 __B) 39190075Sobrien{ 39290075Sobrien return (__m128) __builtin_ia32_cmpunordps ((__v4sf)__A, (__v4sf)__B); 39390075Sobrien} 39490075Sobrien 39590075Sobrien/* Compare the lower SPFP values of A and B and return 1 if true 39690075Sobrien and 0 if false. */ 39790075Sobrien 39890075Sobrienstatic __inline int 39990075Sobrien_mm_comieq_ss (__m128 __A, __m128 __B) 40090075Sobrien{ 40190075Sobrien return __builtin_ia32_comieq ((__v4sf)__A, (__v4sf)__B); 40290075Sobrien} 40390075Sobrien 40490075Sobrienstatic __inline int 40590075Sobrien_mm_comilt_ss (__m128 __A, __m128 __B) 40690075Sobrien{ 40790075Sobrien return __builtin_ia32_comilt ((__v4sf)__A, (__v4sf)__B); 40890075Sobrien} 40990075Sobrien 41090075Sobrienstatic __inline int 41190075Sobrien_mm_comile_ss (__m128 __A, __m128 __B) 41290075Sobrien{ 41390075Sobrien return __builtin_ia32_comile ((__v4sf)__A, (__v4sf)__B); 41490075Sobrien} 41590075Sobrien 41690075Sobrienstatic __inline int 41790075Sobrien_mm_comigt_ss (__m128 __A, __m128 __B) 41890075Sobrien{ 41990075Sobrien return __builtin_ia32_comigt ((__v4sf)__A, (__v4sf)__B); 42090075Sobrien} 42190075Sobrien 42290075Sobrienstatic __inline int 42390075Sobrien_mm_comige_ss (__m128 __A, __m128 __B) 42490075Sobrien{ 42590075Sobrien return __builtin_ia32_comige ((__v4sf)__A, (__v4sf)__B); 42690075Sobrien} 42790075Sobrien 42890075Sobrienstatic __inline int 42990075Sobrien_mm_comineq_ss (__m128 __A, __m128 __B) 43090075Sobrien{ 43190075Sobrien return __builtin_ia32_comineq ((__v4sf)__A, (__v4sf)__B); 43290075Sobrien} 43390075Sobrien 43490075Sobrienstatic __inline int 43590075Sobrien_mm_ucomieq_ss (__m128 __A, __m128 __B) 43690075Sobrien{ 43790075Sobrien return __builtin_ia32_ucomieq ((__v4sf)__A, (__v4sf)__B); 43890075Sobrien} 43990075Sobrien 44090075Sobrienstatic __inline int 44190075Sobrien_mm_ucomilt_ss (__m128 __A, __m128 __B) 44290075Sobrien{ 44390075Sobrien return __builtin_ia32_ucomilt ((__v4sf)__A, (__v4sf)__B); 44490075Sobrien} 44590075Sobrien 44690075Sobrienstatic __inline int 44790075Sobrien_mm_ucomile_ss (__m128 __A, __m128 __B) 44890075Sobrien{ 44990075Sobrien return __builtin_ia32_ucomile ((__v4sf)__A, (__v4sf)__B); 45090075Sobrien} 45190075Sobrien 45290075Sobrienstatic __inline int 45390075Sobrien_mm_ucomigt_ss (__m128 __A, __m128 __B) 45490075Sobrien{ 45590075Sobrien return __builtin_ia32_ucomigt ((__v4sf)__A, (__v4sf)__B); 45690075Sobrien} 45790075Sobrien 45890075Sobrienstatic __inline int 45990075Sobrien_mm_ucomige_ss (__m128 __A, __m128 __B) 46090075Sobrien{ 46190075Sobrien return __builtin_ia32_ucomige ((__v4sf)__A, (__v4sf)__B); 46290075Sobrien} 46390075Sobrien 46490075Sobrienstatic __inline int 46590075Sobrien_mm_ucomineq_ss (__m128 __A, __m128 __B) 46690075Sobrien{ 46790075Sobrien return __builtin_ia32_ucomineq ((__v4sf)__A, (__v4sf)__B); 46890075Sobrien} 46990075Sobrien 47090075Sobrien/* Convert the lower SPFP value to a 32-bit integer according to the current 47190075Sobrien rounding mode. */ 47290075Sobrienstatic __inline int 47390075Sobrien_mm_cvtss_si32 (__m128 __A) 47490075Sobrien{ 47590075Sobrien return __builtin_ia32_cvtss2si ((__v4sf) __A); 47690075Sobrien} 47790075Sobrien 478117395Skan#ifdef __x86_64__ 479117395Skan/* Convert the lower SPFP value to a 32-bit integer according to the current 480117395Skan rounding mode. */ 481117395Skanstatic __inline long long 482117395Skan_mm_cvtss_si64x (__m128 __A) 483117395Skan{ 484117395Skan return __builtin_ia32_cvtss2si64 ((__v4sf) __A); 485117395Skan} 486117395Skan#endif 487117395Skan 48890075Sobrien/* Convert the two lower SPFP values to 32-bit integers according to the 48990075Sobrien current rounding mode. Return the integers in packed form. */ 49090075Sobrienstatic __inline __m64 49190075Sobrien_mm_cvtps_pi32 (__m128 __A) 49290075Sobrien{ 49390075Sobrien return (__m64) __builtin_ia32_cvtps2pi ((__v4sf) __A); 49490075Sobrien} 49590075Sobrien 49690075Sobrien/* Truncate the lower SPFP value to a 32-bit integer. */ 49790075Sobrienstatic __inline int 49890075Sobrien_mm_cvttss_si32 (__m128 __A) 49990075Sobrien{ 50090075Sobrien return __builtin_ia32_cvttss2si ((__v4sf) __A); 50190075Sobrien} 50290075Sobrien 503117395Skan#ifdef __x86_64__ 504117395Skan/* Truncate the lower SPFP value to a 32-bit integer. */ 505117395Skanstatic __inline long long 506117395Skan_mm_cvttss_si64x (__m128 __A) 507117395Skan{ 508117395Skan return __builtin_ia32_cvttss2si64 ((__v4sf) __A); 509117395Skan} 510117395Skan#endif 511117395Skan 51290075Sobrien/* Truncate the two lower SPFP values to 32-bit integers. Return the 51390075Sobrien integers in packed form. */ 51490075Sobrienstatic __inline __m64 51590075Sobrien_mm_cvttps_pi32 (__m128 __A) 51690075Sobrien{ 51790075Sobrien return (__m64) __builtin_ia32_cvttps2pi ((__v4sf) __A); 51890075Sobrien} 51990075Sobrien 52090075Sobrien/* Convert B to a SPFP value and insert it as element zero in A. */ 52190075Sobrienstatic __inline __m128 52290075Sobrien_mm_cvtsi32_ss (__m128 __A, int __B) 52390075Sobrien{ 52490075Sobrien return (__m128) __builtin_ia32_cvtsi2ss ((__v4sf) __A, __B); 52590075Sobrien} 52690075Sobrien 527117395Skan#ifdef __x86_64__ 528117395Skan/* Convert B to a SPFP value and insert it as element zero in A. */ 529117395Skanstatic __inline __m128 530117395Skan_mm_cvtsi64x_ss (__m128 __A, long long __B) 531117395Skan{ 532117395Skan return (__m128) __builtin_ia32_cvtsi642ss ((__v4sf) __A, __B); 533117395Skan} 534117395Skan#endif 535117395Skan 53690075Sobrien/* Convert the two 32-bit values in B to SPFP form and insert them 53790075Sobrien as the two lower elements in A. */ 53890075Sobrienstatic __inline __m128 53990075Sobrien_mm_cvtpi32_ps (__m128 __A, __m64 __B) 54090075Sobrien{ 54190075Sobrien return (__m128) __builtin_ia32_cvtpi2ps ((__v4sf) __A, (__v2si)__B); 54290075Sobrien} 54390075Sobrien 54490075Sobrien/* Convert the four signed 16-bit values in A to SPFP form. */ 54590075Sobrienstatic __inline __m128 54690075Sobrien_mm_cvtpi16_ps (__m64 __A) 54790075Sobrien{ 54890075Sobrien __v4hi __sign; 54990075Sobrien __v2si __hisi, __losi; 55090075Sobrien __v4sf __r; 55190075Sobrien 55290075Sobrien /* This comparison against zero gives us a mask that can be used to 55390075Sobrien fill in the missing sign bits in the unpack operations below, so 55490075Sobrien that we get signed values after unpacking. */ 55590075Sobrien __sign = (__v4hi) __builtin_ia32_mmx_zero (); 55690075Sobrien __sign = __builtin_ia32_pcmpgtw (__sign, (__v4hi)__A); 55790075Sobrien 55890075Sobrien /* Convert the four words to doublewords. */ 55990075Sobrien __hisi = (__v2si) __builtin_ia32_punpckhwd ((__v4hi)__A, __sign); 56090075Sobrien __losi = (__v2si) __builtin_ia32_punpcklwd ((__v4hi)__A, __sign); 56190075Sobrien 56290075Sobrien /* Convert the doublewords to floating point two at a time. */ 56390075Sobrien __r = (__v4sf) __builtin_ia32_setzerops (); 56490075Sobrien __r = __builtin_ia32_cvtpi2ps (__r, __hisi); 56590075Sobrien __r = __builtin_ia32_movlhps (__r, __r); 56690075Sobrien __r = __builtin_ia32_cvtpi2ps (__r, __losi); 56790075Sobrien 56890075Sobrien return (__m128) __r; 56990075Sobrien} 57090075Sobrien 57190075Sobrien/* Convert the four unsigned 16-bit values in A to SPFP form. */ 57290075Sobrienstatic __inline __m128 57390075Sobrien_mm_cvtpu16_ps (__m64 __A) 57490075Sobrien{ 57590075Sobrien __v4hi __zero = (__v4hi) __builtin_ia32_mmx_zero (); 57690075Sobrien __v2si __hisi, __losi; 57790075Sobrien __v4sf __r; 57890075Sobrien 57990075Sobrien /* Convert the four words to doublewords. */ 58090075Sobrien __hisi = (__v2si) __builtin_ia32_punpckhwd ((__v4hi)__A, __zero); 58190075Sobrien __losi = (__v2si) __builtin_ia32_punpcklwd ((__v4hi)__A, __zero); 58290075Sobrien 58390075Sobrien /* Convert the doublewords to floating point two at a time. */ 58490075Sobrien __r = (__v4sf) __builtin_ia32_setzerops (); 58590075Sobrien __r = __builtin_ia32_cvtpi2ps (__r, __hisi); 58690075Sobrien __r = __builtin_ia32_movlhps (__r, __r); 58790075Sobrien __r = __builtin_ia32_cvtpi2ps (__r, __losi); 58890075Sobrien 58990075Sobrien return (__m128) __r; 59090075Sobrien} 59190075Sobrien 59290075Sobrien/* Convert the low four signed 8-bit values in A to SPFP form. */ 59390075Sobrienstatic __inline __m128 59490075Sobrien_mm_cvtpi8_ps (__m64 __A) 59590075Sobrien{ 59690075Sobrien __v8qi __sign; 59790075Sobrien 59890075Sobrien /* This comparison against zero gives us a mask that can be used to 59990075Sobrien fill in the missing sign bits in the unpack operations below, so 60090075Sobrien that we get signed values after unpacking. */ 60190075Sobrien __sign = (__v8qi) __builtin_ia32_mmx_zero (); 60290075Sobrien __sign = __builtin_ia32_pcmpgtb (__sign, (__v8qi)__A); 60390075Sobrien 60490075Sobrien /* Convert the four low bytes to words. */ 60590075Sobrien __A = (__m64) __builtin_ia32_punpcklbw ((__v8qi)__A, __sign); 60690075Sobrien 60790075Sobrien return _mm_cvtpi16_ps(__A); 60890075Sobrien} 60990075Sobrien 61090075Sobrien/* Convert the low four unsigned 8-bit values in A to SPFP form. */ 61190075Sobrienstatic __inline __m128 61290075Sobrien_mm_cvtpu8_ps(__m64 __A) 61390075Sobrien{ 61490075Sobrien __v8qi __zero = (__v8qi) __builtin_ia32_mmx_zero (); 61590075Sobrien __A = (__m64) __builtin_ia32_punpcklbw ((__v8qi)__A, __zero); 61690075Sobrien return _mm_cvtpu16_ps(__A); 61790075Sobrien} 61890075Sobrien 61990075Sobrien/* Convert the four signed 32-bit values in A and B to SPFP form. */ 62090075Sobrienstatic __inline __m128 62190075Sobrien_mm_cvtpi32x2_ps(__m64 __A, __m64 __B) 62290075Sobrien{ 62390075Sobrien __v4sf __zero = (__v4sf) __builtin_ia32_setzerops (); 62490075Sobrien __v4sf __sfa = __builtin_ia32_cvtpi2ps (__zero, (__v2si)__A); 62590075Sobrien __v4sf __sfb = __builtin_ia32_cvtpi2ps (__zero, (__v2si)__B); 62690075Sobrien return (__m128) __builtin_ia32_movlhps (__sfa, __sfb); 62790075Sobrien} 62890075Sobrien 62990075Sobrien/* Convert the four SPFP values in A to four signed 16-bit integers. */ 63090075Sobrienstatic __inline __m64 63190075Sobrien_mm_cvtps_pi16(__m128 __A) 63290075Sobrien{ 63390075Sobrien __v4sf __hisf = (__v4sf)__A; 63490075Sobrien __v4sf __losf = __builtin_ia32_movhlps (__hisf, __hisf); 63590075Sobrien __v2si __hisi = __builtin_ia32_cvtps2pi (__hisf); 63690075Sobrien __v2si __losi = __builtin_ia32_cvtps2pi (__losf); 637117395Skan return (__m64) __builtin_ia32_packssdw (__hisi, __losi); 63890075Sobrien} 63990075Sobrien 64090075Sobrien/* Convert the four SPFP values in A to four signed 8-bit integers. */ 64190075Sobrienstatic __inline __m64 64290075Sobrien_mm_cvtps_pi8(__m128 __A) 64390075Sobrien{ 64490075Sobrien __v4hi __tmp = (__v4hi) _mm_cvtps_pi16 (__A); 64590075Sobrien __v4hi __zero = (__v4hi) __builtin_ia32_mmx_zero (); 64690075Sobrien return (__m64) __builtin_ia32_packsswb (__tmp, __zero); 64790075Sobrien} 64890075Sobrien 64990075Sobrien/* Selects four specific SPFP values from A and B based on MASK. */ 65090075Sobrien#if 0 65190075Sobrienstatic __inline __m128 65290075Sobrien_mm_shuffle_ps (__m128 __A, __m128 __B, int __mask) 65390075Sobrien{ 65490075Sobrien return (__m128) __builtin_ia32_shufps ((__v4sf)__A, (__v4sf)__B, __mask); 65590075Sobrien} 65690075Sobrien#else 65790075Sobrien#define _mm_shuffle_ps(A, B, MASK) \ 65890075Sobrien ((__m128) __builtin_ia32_shufps ((__v4sf)(A), (__v4sf)(B), (MASK))) 65990075Sobrien#endif 66090075Sobrien 66190075Sobrien 66290075Sobrien/* Selects and interleaves the upper two SPFP values from A and B. */ 66390075Sobrienstatic __inline __m128 66490075Sobrien_mm_unpackhi_ps (__m128 __A, __m128 __B) 66590075Sobrien{ 66690075Sobrien return (__m128) __builtin_ia32_unpckhps ((__v4sf)__A, (__v4sf)__B); 66790075Sobrien} 66890075Sobrien 66990075Sobrien/* Selects and interleaves the lower two SPFP values from A and B. */ 67090075Sobrienstatic __inline __m128 67190075Sobrien_mm_unpacklo_ps (__m128 __A, __m128 __B) 67290075Sobrien{ 67390075Sobrien return (__m128) __builtin_ia32_unpcklps ((__v4sf)__A, (__v4sf)__B); 67490075Sobrien} 67590075Sobrien 67690075Sobrien/* Sets the upper two SPFP values with 64-bits of data loaded from P; 67790075Sobrien the lower two values are passed through from A. */ 67890075Sobrienstatic __inline __m128 679117395Skan_mm_loadh_pi (__m128 __A, __m64 const *__P) 68090075Sobrien{ 68190075Sobrien return (__m128) __builtin_ia32_loadhps ((__v4sf)__A, (__v2si *)__P); 68290075Sobrien} 68390075Sobrien 68490075Sobrien/* Stores the upper two SPFP values of A into P. */ 68590075Sobrienstatic __inline void 68690075Sobrien_mm_storeh_pi (__m64 *__P, __m128 __A) 68790075Sobrien{ 68890075Sobrien __builtin_ia32_storehps ((__v2si *)__P, (__v4sf)__A); 68990075Sobrien} 69090075Sobrien 69190075Sobrien/* Moves the upper two values of B into the lower two values of A. */ 69290075Sobrienstatic __inline __m128 69390075Sobrien_mm_movehl_ps (__m128 __A, __m128 __B) 69490075Sobrien{ 69590075Sobrien return (__m128) __builtin_ia32_movhlps ((__v4sf)__A, (__v4sf)__B); 69690075Sobrien} 69790075Sobrien 69890075Sobrien/* Moves the lower two values of B into the upper two values of A. */ 69990075Sobrienstatic __inline __m128 70090075Sobrien_mm_movelh_ps (__m128 __A, __m128 __B) 70190075Sobrien{ 70290075Sobrien return (__m128) __builtin_ia32_movlhps ((__v4sf)__A, (__v4sf)__B); 70390075Sobrien} 70490075Sobrien 70590075Sobrien/* Sets the lower two SPFP values with 64-bits of data loaded from P; 70690075Sobrien the upper two values are passed through from A. */ 70790075Sobrienstatic __inline __m128 708117395Skan_mm_loadl_pi (__m128 __A, __m64 const *__P) 70990075Sobrien{ 71090075Sobrien return (__m128) __builtin_ia32_loadlps ((__v4sf)__A, (__v2si *)__P); 71190075Sobrien} 71290075Sobrien 71390075Sobrien/* Stores the lower two SPFP values of A into P. */ 71490075Sobrienstatic __inline void 71590075Sobrien_mm_storel_pi (__m64 *__P, __m128 __A) 71690075Sobrien{ 71790075Sobrien __builtin_ia32_storelps ((__v2si *)__P, (__v4sf)__A); 71890075Sobrien} 71990075Sobrien 72090075Sobrien/* Creates a 4-bit mask from the most significant bits of the SPFP values. */ 72190075Sobrienstatic __inline int 72290075Sobrien_mm_movemask_ps (__m128 __A) 72390075Sobrien{ 72490075Sobrien return __builtin_ia32_movmskps ((__v4sf)__A); 72590075Sobrien} 72690075Sobrien 72790075Sobrien/* Return the contents of the control register. */ 72890075Sobrienstatic __inline unsigned int 72990075Sobrien_mm_getcsr (void) 73090075Sobrien{ 73190075Sobrien return __builtin_ia32_stmxcsr (); 73290075Sobrien} 73390075Sobrien 73490075Sobrien/* Read exception bits from the control register. */ 73590075Sobrienstatic __inline unsigned int 73690075Sobrien_MM_GET_EXCEPTION_STATE (void) 73790075Sobrien{ 73890075Sobrien return _mm_getcsr() & _MM_EXCEPT_MASK; 73990075Sobrien} 74090075Sobrien 74190075Sobrienstatic __inline unsigned int 74290075Sobrien_MM_GET_EXCEPTION_MASK (void) 74390075Sobrien{ 74490075Sobrien return _mm_getcsr() & _MM_MASK_MASK; 74590075Sobrien} 74690075Sobrien 74790075Sobrienstatic __inline unsigned int 74890075Sobrien_MM_GET_ROUNDING_MODE (void) 74990075Sobrien{ 75090075Sobrien return _mm_getcsr() & _MM_ROUND_MASK; 75190075Sobrien} 75290075Sobrien 75390075Sobrienstatic __inline unsigned int 75490075Sobrien_MM_GET_FLUSH_ZERO_MODE (void) 75590075Sobrien{ 75690075Sobrien return _mm_getcsr() & _MM_FLUSH_ZERO_MASK; 75790075Sobrien} 75890075Sobrien 75990075Sobrien/* Set the control register to I. */ 76090075Sobrienstatic __inline void 76190075Sobrien_mm_setcsr (unsigned int __I) 76290075Sobrien{ 76390075Sobrien __builtin_ia32_ldmxcsr (__I); 76490075Sobrien} 76590075Sobrien 76690075Sobrien/* Set exception bits in the control register. */ 76790075Sobrienstatic __inline void 76890075Sobrien_MM_SET_EXCEPTION_STATE(unsigned int __mask) 76990075Sobrien{ 77090075Sobrien _mm_setcsr((_mm_getcsr() & ~_MM_EXCEPT_MASK) | __mask); 77190075Sobrien} 77290075Sobrien 77390075Sobrienstatic __inline void 77490075Sobrien_MM_SET_EXCEPTION_MASK (unsigned int __mask) 77590075Sobrien{ 77690075Sobrien _mm_setcsr((_mm_getcsr() & ~_MM_MASK_MASK) | __mask); 77790075Sobrien} 77890075Sobrien 77990075Sobrienstatic __inline void 78090075Sobrien_MM_SET_ROUNDING_MODE (unsigned int __mode) 78190075Sobrien{ 78290075Sobrien _mm_setcsr((_mm_getcsr() & ~_MM_ROUND_MASK) | __mode); 78390075Sobrien} 78490075Sobrien 78590075Sobrienstatic __inline void 78690075Sobrien_MM_SET_FLUSH_ZERO_MODE (unsigned int __mode) 78790075Sobrien{ 78890075Sobrien _mm_setcsr((_mm_getcsr() & ~_MM_FLUSH_ZERO_MASK) | __mode); 78990075Sobrien} 79090075Sobrien 79190075Sobrien/* Create a vector with element 0 as *P and the rest zero. */ 79290075Sobrienstatic __inline __m128 793117395Skan_mm_load_ss (float const *__P) 79490075Sobrien{ 79590075Sobrien return (__m128) __builtin_ia32_loadss (__P); 79690075Sobrien} 79790075Sobrien 79890075Sobrien/* Create a vector with all four elements equal to *P. */ 79990075Sobrienstatic __inline __m128 800117395Skan_mm_load1_ps (float const *__P) 80190075Sobrien{ 80290075Sobrien __v4sf __tmp = __builtin_ia32_loadss (__P); 80390075Sobrien return (__m128) __builtin_ia32_shufps (__tmp, __tmp, _MM_SHUFFLE (0,0,0,0)); 80490075Sobrien} 80590075Sobrien 80690075Sobrienstatic __inline __m128 807117395Skan_mm_load_ps1 (float const *__P) 80890075Sobrien{ 80990075Sobrien return _mm_load1_ps (__P); 81090075Sobrien} 81190075Sobrien 81290075Sobrien/* Load four SPFP values from P. The address must be 16-byte aligned. */ 81390075Sobrienstatic __inline __m128 814117395Skan_mm_load_ps (float const *__P) 81590075Sobrien{ 81690075Sobrien return (__m128) __builtin_ia32_loadaps (__P); 81790075Sobrien} 81890075Sobrien 81990075Sobrien/* Load four SPFP values from P. The address need not be 16-byte aligned. */ 82090075Sobrienstatic __inline __m128 821117395Skan_mm_loadu_ps (float const *__P) 82290075Sobrien{ 82390075Sobrien return (__m128) __builtin_ia32_loadups (__P); 82490075Sobrien} 82590075Sobrien 82690075Sobrien/* Load four SPFP values in reverse order. The address must be aligned. */ 82790075Sobrienstatic __inline __m128 828117395Skan_mm_loadr_ps (float const *__P) 82990075Sobrien{ 83090075Sobrien __v4sf __tmp = __builtin_ia32_loadaps (__P); 83190075Sobrien return (__m128) __builtin_ia32_shufps (__tmp, __tmp, _MM_SHUFFLE (0,1,2,3)); 83290075Sobrien} 83390075Sobrien 83490075Sobrien/* Create a vector with element 0 as F and the rest zero. */ 83590075Sobrienstatic __inline __m128 83690075Sobrien_mm_set_ss (float __F) 83790075Sobrien{ 83890075Sobrien return (__m128) __builtin_ia32_loadss (&__F); 83990075Sobrien} 84090075Sobrien 84190075Sobrien/* Create a vector with all four elements equal to F. */ 84290075Sobrienstatic __inline __m128 84390075Sobrien_mm_set1_ps (float __F) 84490075Sobrien{ 84590075Sobrien __v4sf __tmp = __builtin_ia32_loadss (&__F); 84690075Sobrien return (__m128) __builtin_ia32_shufps (__tmp, __tmp, _MM_SHUFFLE (0,0,0,0)); 84790075Sobrien} 84890075Sobrien 84990075Sobrienstatic __inline __m128 85090075Sobrien_mm_set_ps1 (float __F) 85190075Sobrien{ 85290075Sobrien return _mm_set1_ps (__F); 85390075Sobrien} 85490075Sobrien 85590075Sobrien/* Create the vector [Z Y X W]. */ 85690075Sobrienstatic __inline __m128 85790075Sobrien_mm_set_ps (float __Z, float __Y, float __X, float __W) 85890075Sobrien{ 85990075Sobrien union { 86090075Sobrien float __a[4]; 86190075Sobrien __m128 __v; 86290075Sobrien } __u; 86390075Sobrien 86490075Sobrien __u.__a[0] = __W; 86590075Sobrien __u.__a[1] = __X; 86690075Sobrien __u.__a[2] = __Y; 86790075Sobrien __u.__a[3] = __Z; 86890075Sobrien 86990075Sobrien return __u.__v; 87090075Sobrien} 87190075Sobrien 87290075Sobrien/* Create the vector [W X Y Z]. */ 87390075Sobrienstatic __inline __m128 87490075Sobrien_mm_setr_ps (float __Z, float __Y, float __X, float __W) 87590075Sobrien{ 87690075Sobrien return _mm_set_ps (__W, __X, __Y, __Z); 87790075Sobrien} 87890075Sobrien 87990075Sobrien/* Create a vector of zeros. */ 88090075Sobrienstatic __inline __m128 88190075Sobrien_mm_setzero_ps (void) 88290075Sobrien{ 88390075Sobrien return (__m128) __builtin_ia32_setzerops (); 88490075Sobrien} 88590075Sobrien 88690075Sobrien/* Stores the lower SPFP value. */ 88790075Sobrienstatic __inline void 88890075Sobrien_mm_store_ss (float *__P, __m128 __A) 88990075Sobrien{ 89090075Sobrien __builtin_ia32_storess (__P, (__v4sf)__A); 89190075Sobrien} 89290075Sobrien 89390075Sobrien/* Store the lower SPFP value across four words. */ 89490075Sobrienstatic __inline void 89590075Sobrien_mm_store1_ps (float *__P, __m128 __A) 89690075Sobrien{ 89790075Sobrien __v4sf __va = (__v4sf)__A; 89890075Sobrien __v4sf __tmp = __builtin_ia32_shufps (__va, __va, _MM_SHUFFLE (0,0,0,0)); 89990075Sobrien __builtin_ia32_storeaps (__P, __tmp); 90090075Sobrien} 90190075Sobrien 90290075Sobrienstatic __inline void 90390075Sobrien_mm_store_ps1 (float *__P, __m128 __A) 90490075Sobrien{ 90590075Sobrien _mm_store1_ps (__P, __A); 90690075Sobrien} 90790075Sobrien 90890075Sobrien/* Store four SPFP values. The address must be 16-byte aligned. */ 90990075Sobrienstatic __inline void 91090075Sobrien_mm_store_ps (float *__P, __m128 __A) 91190075Sobrien{ 91290075Sobrien __builtin_ia32_storeaps (__P, (__v4sf)__A); 91390075Sobrien} 91490075Sobrien 91590075Sobrien/* Store four SPFP values. The address need not be 16-byte aligned. */ 91690075Sobrienstatic __inline void 91790075Sobrien_mm_storeu_ps (float *__P, __m128 __A) 91890075Sobrien{ 91990075Sobrien __builtin_ia32_storeups (__P, (__v4sf)__A); 92090075Sobrien} 92190075Sobrien 922117395Skan/* Store four SPFP values in reverse order. The address must be aligned. */ 92390075Sobrienstatic __inline void 92490075Sobrien_mm_storer_ps (float *__P, __m128 __A) 92590075Sobrien{ 92690075Sobrien __v4sf __va = (__v4sf)__A; 92790075Sobrien __v4sf __tmp = __builtin_ia32_shufps (__va, __va, _MM_SHUFFLE (0,1,2,3)); 92890075Sobrien __builtin_ia32_storeaps (__P, __tmp); 92990075Sobrien} 93090075Sobrien 93190075Sobrien/* Sets the low SPFP value of A from the low value of B. */ 93290075Sobrienstatic __inline __m128 93390075Sobrien_mm_move_ss (__m128 __A, __m128 __B) 93490075Sobrien{ 93590075Sobrien return (__m128) __builtin_ia32_movss ((__v4sf)__A, (__v4sf)__B); 93690075Sobrien} 93790075Sobrien 93890075Sobrien/* Extracts one of the four words of A. The selector N must be immediate. */ 93990075Sobrien#if 0 94090075Sobrienstatic __inline int 94190075Sobrien_mm_extract_pi16 (__m64 __A, int __N) 94290075Sobrien{ 94390075Sobrien return __builtin_ia32_pextrw ((__v4hi)__A, __N); 94490075Sobrien} 94590075Sobrien#else 94690075Sobrien#define _mm_extract_pi16(A, N) \ 94790075Sobrien __builtin_ia32_pextrw ((__v4hi)(A), (N)) 94890075Sobrien#endif 94990075Sobrien 95090075Sobrien/* Inserts word D into one of four words of A. The selector N must be 95190075Sobrien immediate. */ 95290075Sobrien#if 0 95390075Sobrienstatic __inline __m64 95490075Sobrien_mm_insert_pi16 (__m64 __A, int __D, int __N) 95590075Sobrien{ 95690075Sobrien return (__m64)__builtin_ia32_pinsrw ((__v4hi)__A, __D, __N); 95790075Sobrien} 95890075Sobrien#else 95990075Sobrien#define _mm_insert_pi16(A, D, N) \ 96090075Sobrien ((__m64) __builtin_ia32_pinsrw ((__v4hi)(A), (D), (N))) 96190075Sobrien#endif 96290075Sobrien 96390075Sobrien/* Compute the element-wise maximum of signed 16-bit values. */ 96490075Sobrienstatic __inline __m64 96590075Sobrien_mm_max_pi16 (__m64 __A, __m64 __B) 96690075Sobrien{ 96790075Sobrien return (__m64) __builtin_ia32_pmaxsw ((__v4hi)__A, (__v4hi)__B); 96890075Sobrien} 96990075Sobrien 97090075Sobrien/* Compute the element-wise maximum of unsigned 8-bit values. */ 97190075Sobrienstatic __inline __m64 97290075Sobrien_mm_max_pu8 (__m64 __A, __m64 __B) 97390075Sobrien{ 97490075Sobrien return (__m64) __builtin_ia32_pmaxub ((__v8qi)__A, (__v8qi)__B); 97590075Sobrien} 97690075Sobrien 97790075Sobrien/* Compute the element-wise minimum of signed 16-bit values. */ 97890075Sobrienstatic __inline __m64 97990075Sobrien_mm_min_pi16 (__m64 __A, __m64 __B) 98090075Sobrien{ 98190075Sobrien return (__m64) __builtin_ia32_pminsw ((__v4hi)__A, (__v4hi)__B); 98290075Sobrien} 98390075Sobrien 98490075Sobrien/* Compute the element-wise minimum of unsigned 8-bit values. */ 98590075Sobrienstatic __inline __m64 98690075Sobrien_mm_min_pu8 (__m64 __A, __m64 __B) 98790075Sobrien{ 98890075Sobrien return (__m64) __builtin_ia32_pminub ((__v8qi)__A, (__v8qi)__B); 98990075Sobrien} 99090075Sobrien 99190075Sobrien/* Create an 8-bit mask of the signs of 8-bit values. */ 99290075Sobrienstatic __inline int 99390075Sobrien_mm_movemask_pi8 (__m64 __A) 99490075Sobrien{ 99590075Sobrien return __builtin_ia32_pmovmskb ((__v8qi)__A); 99690075Sobrien} 99790075Sobrien 99890075Sobrien/* Multiply four unsigned 16-bit values in A by four unsigned 16-bit values 99990075Sobrien in B and produce the high 16 bits of the 32-bit results. */ 100090075Sobrienstatic __inline __m64 100190075Sobrien_mm_mulhi_pu16 (__m64 __A, __m64 __B) 100290075Sobrien{ 100390075Sobrien return (__m64) __builtin_ia32_pmulhuw ((__v4hi)__A, (__v4hi)__B); 100490075Sobrien} 100590075Sobrien 100690075Sobrien/* Return a combination of the four 16-bit values in A. The selector 100790075Sobrien must be an immediate. */ 100890075Sobrien#if 0 100990075Sobrienstatic __inline __m64 101090075Sobrien_mm_shuffle_pi16 (__m64 __A, int __N) 101190075Sobrien{ 101290075Sobrien return (__m64) __builtin_ia32_pshufw ((__v4hi)__A, __N); 101390075Sobrien} 101490075Sobrien#else 101590075Sobrien#define _mm_shuffle_pi16(A, N) \ 101690075Sobrien ((__m64) __builtin_ia32_pshufw ((__v4hi)(A), (N))) 101790075Sobrien#endif 101890075Sobrien 101990075Sobrien/* Conditionally store byte elements of A into P. The high bit of each 102090075Sobrien byte in the selector N determines whether the corresponding byte from 102190075Sobrien A is stored. */ 102290075Sobrienstatic __inline void 102390075Sobrien_mm_maskmove_si64 (__m64 __A, __m64 __N, char *__P) 102490075Sobrien{ 102590075Sobrien __builtin_ia32_maskmovq ((__v8qi)__A, (__v8qi)__N, __P); 102690075Sobrien} 102790075Sobrien 102890075Sobrien/* Compute the rounded averages of the unsigned 8-bit values in A and B. */ 102990075Sobrienstatic __inline __m64 103090075Sobrien_mm_avg_pu8 (__m64 __A, __m64 __B) 103190075Sobrien{ 103290075Sobrien return (__m64) __builtin_ia32_pavgb ((__v8qi)__A, (__v8qi)__B); 103390075Sobrien} 103490075Sobrien 103590075Sobrien/* Compute the rounded averages of the unsigned 16-bit values in A and B. */ 103690075Sobrienstatic __inline __m64 103790075Sobrien_mm_avg_pu16 (__m64 __A, __m64 __B) 103890075Sobrien{ 103990075Sobrien return (__m64) __builtin_ia32_pavgw ((__v4hi)__A, (__v4hi)__B); 104090075Sobrien} 104190075Sobrien 104290075Sobrien/* Compute the sum of the absolute differences of the unsigned 8-bit 104390075Sobrien values in A and B. Return the value in the lower 16-bit word; the 104490075Sobrien upper words are cleared. */ 104590075Sobrienstatic __inline __m64 104690075Sobrien_mm_sad_pu8 (__m64 __A, __m64 __B) 104790075Sobrien{ 104890075Sobrien return (__m64) __builtin_ia32_psadbw ((__v8qi)__A, (__v8qi)__B); 104990075Sobrien} 105090075Sobrien 105190075Sobrien/* Loads one cache line from address P to a location "closer" to the 105290075Sobrien processor. The selector I specifies the type of prefetch operation. */ 105390075Sobrien#if 0 105490075Sobrienstatic __inline void 105590075Sobrien_mm_prefetch (void *__P, enum _mm_hint __I) 105690075Sobrien{ 105790075Sobrien __builtin_prefetch (__P, 0, __I); 105890075Sobrien} 105990075Sobrien#else 106090075Sobrien#define _mm_prefetch(P, I) \ 106190075Sobrien __builtin_prefetch ((P), 0, (I)) 106290075Sobrien#endif 106390075Sobrien 106490075Sobrien/* Stores the data in A to the address P without polluting the caches. */ 106590075Sobrienstatic __inline void 106690075Sobrien_mm_stream_pi (__m64 *__P, __m64 __A) 106790075Sobrien{ 1068117395Skan __builtin_ia32_movntq ((unsigned long long *)__P, (unsigned long long)__A); 106990075Sobrien} 107090075Sobrien 107190075Sobrien/* Likewise. The address must be 16-byte aligned. */ 107290075Sobrienstatic __inline void 107390075Sobrien_mm_stream_ps (float *__P, __m128 __A) 107490075Sobrien{ 107590075Sobrien __builtin_ia32_movntps (__P, (__v4sf)__A); 107690075Sobrien} 107790075Sobrien 107890075Sobrien/* Guarantees that every preceeding store is globally visible before 107990075Sobrien any subsequent store. */ 108090075Sobrienstatic __inline void 108190075Sobrien_mm_sfence (void) 108290075Sobrien{ 108390075Sobrien __builtin_ia32_sfence (); 108490075Sobrien} 108590075Sobrien 108690075Sobrien/* The execution of the next instruction is delayed by an implementation 108790075Sobrien specific amount of time. The instruction does not modify the 108890075Sobrien architectural state. */ 108990075Sobrienstatic __inline void 109090075Sobrien_mm_pause (void) 109190075Sobrien{ 109290075Sobrien __asm__ __volatile__ ("rep; nop" : : ); 109390075Sobrien} 109490075Sobrien 109590075Sobrien/* Transpose the 4x4 matrix composed of row[0-3]. */ 109690075Sobrien#define _MM_TRANSPOSE4_PS(row0, row1, row2, row3) \ 109790075Sobriendo { \ 109890075Sobrien __v4sf __r0 = (row0), __r1 = (row1), __r2 = (row2), __r3 = (row3); \ 109990075Sobrien __v4sf __t0 = __builtin_ia32_shufps (__r0, __r1, 0x44); \ 1100107590Sobrien __v4sf __t2 = __builtin_ia32_shufps (__r0, __r1, 0xEE); \ 1101107590Sobrien __v4sf __t1 = __builtin_ia32_shufps (__r2, __r3, 0x44); \ 110290075Sobrien __v4sf __t3 = __builtin_ia32_shufps (__r2, __r3, 0xEE); \ 110390075Sobrien (row0) = __builtin_ia32_shufps (__t0, __t1, 0x88); \ 110490075Sobrien (row1) = __builtin_ia32_shufps (__t0, __t1, 0xDD); \ 110590075Sobrien (row2) = __builtin_ia32_shufps (__t2, __t3, 0x88); \ 110690075Sobrien (row3) = __builtin_ia32_shufps (__t2, __t3, 0xDD); \ 110790075Sobrien} while (0) 110890075Sobrien 1109117395Skan#ifdef __SSE2__ 1110117395Skan/* SSE2 */ 1111117395Skantypedef int __v2df __attribute__ ((mode (V2DF))); 1112117395Skantypedef int __v2di __attribute__ ((mode (V2DI))); 1113117395Skantypedef int __v4si __attribute__ ((mode (V4SI))); 1114117395Skantypedef int __v8hi __attribute__ ((mode (V8HI))); 1115117395Skantypedef int __v16qi __attribute__ ((mode (V16QI))); 1116117395Skan 1117117395Skan/* Create a selector for use with the SHUFPD instruction. */ 1118117395Skan#define _MM_SHUFFLE2(fp1,fp0) \ 1119117395Skan (((fp1) << 1) | (fp0)) 1120117395Skan 1121117395Skan#define __m128i __v2di 1122117395Skan#define __m128d __v2df 1123117395Skan 1124117395Skan/* Create a vector with element 0 as *P and the rest zero. */ 1125117395Skanstatic __inline __m128d 1126117395Skan_mm_load_sd (double const *__P) 1127117395Skan{ 1128117395Skan return (__m128d) __builtin_ia32_loadsd (__P); 1129117395Skan} 1130117395Skan 1131117395Skan/* Create a vector with all two elements equal to *P. */ 1132117395Skanstatic __inline __m128d 1133117395Skan_mm_load1_pd (double const *__P) 1134117395Skan{ 1135117395Skan __v2df __tmp = __builtin_ia32_loadsd (__P); 1136117395Skan return (__m128d) __builtin_ia32_shufpd (__tmp, __tmp, _MM_SHUFFLE2 (0,0)); 1137117395Skan} 1138117395Skan 1139117395Skanstatic __inline __m128d 1140117395Skan_mm_load_pd1 (double const *__P) 1141117395Skan{ 1142117395Skan return _mm_load1_pd (__P); 1143117395Skan} 1144117395Skan 1145117395Skan/* Load two DPFP values from P. The addresd must be 16-byte aligned. */ 1146117395Skanstatic __inline __m128d 1147117395Skan_mm_load_pd (double const *__P) 1148117395Skan{ 1149117395Skan return (__m128d) __builtin_ia32_loadapd (__P); 1150117395Skan} 1151117395Skan 1152117395Skan/* Load two DPFP values from P. The addresd need not be 16-byte aligned. */ 1153117395Skanstatic __inline __m128d 1154117395Skan_mm_loadu_pd (double const *__P) 1155117395Skan{ 1156117395Skan return (__m128d) __builtin_ia32_loadupd (__P); 1157117395Skan} 1158117395Skan 1159117395Skan/* Load two DPFP values in reverse order. The addresd must be aligned. */ 1160117395Skanstatic __inline __m128d 1161117395Skan_mm_loadr_pd (double const *__P) 1162117395Skan{ 1163117395Skan __v2df __tmp = __builtin_ia32_loadapd (__P); 1164117395Skan return (__m128d) __builtin_ia32_shufpd (__tmp, __tmp, _MM_SHUFFLE2 (0,1)); 1165117395Skan} 1166117395Skan 1167117395Skan/* Create a vector with element 0 as F and the rest zero. */ 1168117395Skanstatic __inline __m128d 1169117395Skan_mm_set_sd (double __F) 1170117395Skan{ 1171117395Skan return (__m128d) __builtin_ia32_loadsd (&__F); 1172117395Skan} 1173117395Skan 1174117395Skan/* Create a vector with all two elements equal to F. */ 1175117395Skanstatic __inline __m128d 1176117395Skan_mm_set1_pd (double __F) 1177117395Skan{ 1178117395Skan __v2df __tmp = __builtin_ia32_loadsd (&__F); 1179117395Skan return (__m128d) __builtin_ia32_shufpd (__tmp, __tmp, _MM_SHUFFLE2 (0,0)); 1180117395Skan} 1181117395Skan 1182117395Skanstatic __inline __m128d 1183117395Skan_mm_set_pd1 (double __F) 1184117395Skan{ 1185117395Skan return _mm_set1_pd (__F); 1186117395Skan} 1187117395Skan 1188117395Skan/* Create the vector [Z Y]. */ 1189117395Skanstatic __inline __m128d 1190117395Skan_mm_set_pd (double __Z, double __Y) 1191117395Skan{ 1192117395Skan union { 1193117395Skan double __a[2]; 1194117395Skan __m128d __v; 1195117395Skan } __u; 1196117395Skan 1197117395Skan __u.__a[0] = __Y; 1198117395Skan __u.__a[1] = __Z; 1199117395Skan 1200117395Skan return __u.__v; 1201117395Skan} 1202117395Skan 1203117395Skan/* Create the vector [Y Z]. */ 1204117395Skanstatic __inline __m128d 1205117395Skan_mm_setr_pd (double __Z, double __Y) 1206117395Skan{ 1207117395Skan return _mm_set_pd (__Y, __Z); 1208117395Skan} 1209117395Skan 1210117395Skan/* Create a vector of zeros. */ 1211117395Skanstatic __inline __m128d 1212117395Skan_mm_setzero_pd (void) 1213117395Skan{ 1214117395Skan return (__m128d) __builtin_ia32_setzeropd (); 1215117395Skan} 1216117395Skan 1217117395Skan/* Stores the lower DPFP value. */ 1218117395Skanstatic __inline void 1219117395Skan_mm_store_sd (double *__P, __m128d __A) 1220117395Skan{ 1221117395Skan __builtin_ia32_storesd (__P, (__v2df)__A); 1222117395Skan} 1223117395Skan 1224117395Skan/* Store the lower DPFP value acrosd two words. */ 1225117395Skanstatic __inline void 1226117395Skan_mm_store1_pd (double *__P, __m128d __A) 1227117395Skan{ 1228117395Skan __v2df __va = (__v2df)__A; 1229117395Skan __v2df __tmp = __builtin_ia32_shufpd (__va, __va, _MM_SHUFFLE2 (0,0)); 1230117395Skan __builtin_ia32_storeapd (__P, __tmp); 1231117395Skan} 1232117395Skan 1233117395Skanstatic __inline void 1234117395Skan_mm_store_pd1 (double *__P, __m128d __A) 1235117395Skan{ 1236117395Skan _mm_store1_pd (__P, __A); 1237117395Skan} 1238117395Skan 1239117395Skan/* Store two DPFP values. The addresd must be 16-byte aligned. */ 1240117395Skanstatic __inline void 1241117395Skan_mm_store_pd (double *__P, __m128d __A) 1242117395Skan{ 1243117395Skan __builtin_ia32_storeapd (__P, (__v2df)__A); 1244117395Skan} 1245117395Skan 1246117395Skan/* Store two DPFP values. The addresd need not be 16-byte aligned. */ 1247117395Skanstatic __inline void 1248117395Skan_mm_storeu_pd (double *__P, __m128d __A) 1249117395Skan{ 1250117395Skan __builtin_ia32_storeupd (__P, (__v2df)__A); 1251117395Skan} 1252117395Skan 1253117395Skan/* Store two DPFP values in reverse order. The addresd must be aligned. */ 1254117395Skanstatic __inline void 1255117395Skan_mm_storer_pd (double *__P, __m128d __A) 1256117395Skan{ 1257117395Skan __v2df __va = (__v2df)__A; 1258117395Skan __v2df __tmp = __builtin_ia32_shufpd (__va, __va, _MM_SHUFFLE2 (0,1)); 1259117395Skan __builtin_ia32_storeapd (__P, __tmp); 1260117395Skan} 1261117395Skan 1262117395Skan/* Sets the low DPFP value of A from the low value of B. */ 1263117395Skanstatic __inline __m128d 1264117395Skan_mm_move_sd (__m128d __A, __m128d __B) 1265117395Skan{ 1266117395Skan return (__m128d) __builtin_ia32_movsd ((__v2df)__A, (__v2df)__B); 1267117395Skan} 1268117395Skan 1269117395Skan 1270117395Skanstatic __inline __m128d 1271117395Skan_mm_add_pd (__m128d __A, __m128d __B) 1272117395Skan{ 1273117395Skan return (__m128d)__builtin_ia32_addpd ((__v2df)__A, (__v2df)__B); 1274117395Skan} 1275117395Skan 1276117395Skanstatic __inline __m128d 1277117395Skan_mm_add_sd (__m128d __A, __m128d __B) 1278117395Skan{ 1279117395Skan return (__m128d)__builtin_ia32_addsd ((__v2df)__A, (__v2df)__B); 1280117395Skan} 1281117395Skan 1282117395Skanstatic __inline __m128d 1283117395Skan_mm_sub_pd (__m128d __A, __m128d __B) 1284117395Skan{ 1285117395Skan return (__m128d)__builtin_ia32_subpd ((__v2df)__A, (__v2df)__B); 1286117395Skan} 1287117395Skan 1288117395Skanstatic __inline __m128d 1289117395Skan_mm_sub_sd (__m128d __A, __m128d __B) 1290117395Skan{ 1291117395Skan return (__m128d)__builtin_ia32_subsd ((__v2df)__A, (__v2df)__B); 1292117395Skan} 1293117395Skan 1294117395Skanstatic __inline __m128d 1295117395Skan_mm_mul_pd (__m128d __A, __m128d __B) 1296117395Skan{ 1297117395Skan return (__m128d)__builtin_ia32_mulpd ((__v2df)__A, (__v2df)__B); 1298117395Skan} 1299117395Skan 1300117395Skanstatic __inline __m128d 1301117395Skan_mm_mul_sd (__m128d __A, __m128d __B) 1302117395Skan{ 1303117395Skan return (__m128d)__builtin_ia32_mulsd ((__v2df)__A, (__v2df)__B); 1304117395Skan} 1305117395Skan 1306117395Skanstatic __inline __m128d 1307117395Skan_mm_div_pd (__m128d __A, __m128d __B) 1308117395Skan{ 1309117395Skan return (__m128d)__builtin_ia32_divpd ((__v2df)__A, (__v2df)__B); 1310117395Skan} 1311117395Skan 1312117395Skanstatic __inline __m128d 1313117395Skan_mm_div_sd (__m128d __A, __m128d __B) 1314117395Skan{ 1315117395Skan return (__m128d)__builtin_ia32_divsd ((__v2df)__A, (__v2df)__B); 1316117395Skan} 1317117395Skan 1318117395Skanstatic __inline __m128d 1319117395Skan_mm_sqrt_pd (__m128d __A) 1320117395Skan{ 1321117395Skan return (__m128d)__builtin_ia32_sqrtpd ((__v2df)__A); 1322117395Skan} 1323117395Skan 1324117395Skan/* Return pair {sqrt (A[0), B[1]}. */ 1325117395Skanstatic __inline __m128d 1326117395Skan_mm_sqrt_sd (__m128d __A, __m128d __B) 1327117395Skan{ 1328117395Skan __v2df __tmp = __builtin_ia32_movsd ((__v2df)__A, (__v2df)__B); 1329117395Skan return (__m128d)__builtin_ia32_sqrtsd ((__v2df)__tmp); 1330117395Skan} 1331117395Skan 1332117395Skanstatic __inline __m128d 1333117395Skan_mm_min_pd (__m128d __A, __m128d __B) 1334117395Skan{ 1335117395Skan return (__m128d)__builtin_ia32_minpd ((__v2df)__A, (__v2df)__B); 1336117395Skan} 1337117395Skan 1338117395Skanstatic __inline __m128d 1339117395Skan_mm_min_sd (__m128d __A, __m128d __B) 1340117395Skan{ 1341117395Skan return (__m128d)__builtin_ia32_minsd ((__v2df)__A, (__v2df)__B); 1342117395Skan} 1343117395Skan 1344117395Skanstatic __inline __m128d 1345117395Skan_mm_max_pd (__m128d __A, __m128d __B) 1346117395Skan{ 1347117395Skan return (__m128d)__builtin_ia32_maxpd ((__v2df)__A, (__v2df)__B); 1348117395Skan} 1349117395Skan 1350117395Skanstatic __inline __m128d 1351117395Skan_mm_max_sd (__m128d __A, __m128d __B) 1352117395Skan{ 1353117395Skan return (__m128d)__builtin_ia32_maxsd ((__v2df)__A, (__v2df)__B); 1354117395Skan} 1355117395Skan 1356117395Skanstatic __inline __m128d 1357117395Skan_mm_and_pd (__m128d __A, __m128d __B) 1358117395Skan{ 1359117395Skan return (__m128d)__builtin_ia32_andpd ((__v2df)__A, (__v2df)__B); 1360117395Skan} 1361117395Skan 1362117395Skanstatic __inline __m128d 1363117395Skan_mm_andnot_pd (__m128d __A, __m128d __B) 1364117395Skan{ 1365117395Skan return (__m128d)__builtin_ia32_andnpd ((__v2df)__A, (__v2df)__B); 1366117395Skan} 1367117395Skan 1368117395Skanstatic __inline __m128d 1369117395Skan_mm_or_pd (__m128d __A, __m128d __B) 1370117395Skan{ 1371117395Skan return (__m128d)__builtin_ia32_orpd ((__v2df)__A, (__v2df)__B); 1372117395Skan} 1373117395Skan 1374117395Skanstatic __inline __m128d 1375117395Skan_mm_xor_pd (__m128d __A, __m128d __B) 1376117395Skan{ 1377117395Skan return (__m128d)__builtin_ia32_xorpd ((__v2df)__A, (__v2df)__B); 1378117395Skan} 1379117395Skan 1380117395Skanstatic __inline __m128d 1381117395Skan_mm_cmpeq_pd (__m128d __A, __m128d __B) 1382117395Skan{ 1383117395Skan return (__m128d)__builtin_ia32_cmpeqpd ((__v2df)__A, (__v2df)__B); 1384117395Skan} 1385117395Skan 1386117395Skanstatic __inline __m128d 1387117395Skan_mm_cmplt_pd (__m128d __A, __m128d __B) 1388117395Skan{ 1389117395Skan return (__m128d)__builtin_ia32_cmpltpd ((__v2df)__A, (__v2df)__B); 1390117395Skan} 1391117395Skan 1392117395Skanstatic __inline __m128d 1393117395Skan_mm_cmple_pd (__m128d __A, __m128d __B) 1394117395Skan{ 1395117395Skan return (__m128d)__builtin_ia32_cmplepd ((__v2df)__A, (__v2df)__B); 1396117395Skan} 1397117395Skan 1398117395Skanstatic __inline __m128d 1399117395Skan_mm_cmpgt_pd (__m128d __A, __m128d __B) 1400117395Skan{ 1401117395Skan return (__m128d)__builtin_ia32_cmpgtpd ((__v2df)__A, (__v2df)__B); 1402117395Skan} 1403117395Skan 1404117395Skanstatic __inline __m128d 1405117395Skan_mm_cmpge_pd (__m128d __A, __m128d __B) 1406117395Skan{ 1407117395Skan return (__m128d)__builtin_ia32_cmpgepd ((__v2df)__A, (__v2df)__B); 1408117395Skan} 1409117395Skan 1410117395Skanstatic __inline __m128d 1411117395Skan_mm_cmpneq_pd (__m128d __A, __m128d __B) 1412117395Skan{ 1413117395Skan return (__m128d)__builtin_ia32_cmpneqpd ((__v2df)__A, (__v2df)__B); 1414117395Skan} 1415117395Skan 1416117395Skanstatic __inline __m128d 1417117395Skan_mm_cmpnlt_pd (__m128d __A, __m128d __B) 1418117395Skan{ 1419117395Skan return (__m128d)__builtin_ia32_cmpnltpd ((__v2df)__A, (__v2df)__B); 1420117395Skan} 1421117395Skan 1422117395Skanstatic __inline __m128d 1423117395Skan_mm_cmpnle_pd (__m128d __A, __m128d __B) 1424117395Skan{ 1425117395Skan return (__m128d)__builtin_ia32_cmpnlepd ((__v2df)__A, (__v2df)__B); 1426117395Skan} 1427117395Skan 1428117395Skanstatic __inline __m128d 1429117395Skan_mm_cmpngt_pd (__m128d __A, __m128d __B) 1430117395Skan{ 1431117395Skan return (__m128d)__builtin_ia32_cmpngtpd ((__v2df)__A, (__v2df)__B); 1432117395Skan} 1433117395Skan 1434117395Skanstatic __inline __m128d 1435117395Skan_mm_cmpnge_pd (__m128d __A, __m128d __B) 1436117395Skan{ 1437117395Skan return (__m128d)__builtin_ia32_cmpngepd ((__v2df)__A, (__v2df)__B); 1438117395Skan} 1439117395Skan 1440117395Skanstatic __inline __m128d 1441117395Skan_mm_cmpord_pd (__m128d __A, __m128d __B) 1442117395Skan{ 1443117395Skan return (__m128d)__builtin_ia32_cmpordpd ((__v2df)__A, (__v2df)__B); 1444117395Skan} 1445117395Skan 1446117395Skanstatic __inline __m128d 1447117395Skan_mm_cmpunord_pd (__m128d __A, __m128d __B) 1448117395Skan{ 1449117395Skan return (__m128d)__builtin_ia32_cmpunordpd ((__v2df)__A, (__v2df)__B); 1450117395Skan} 1451117395Skan 1452117395Skanstatic __inline __m128d 1453117395Skan_mm_cmpeq_sd (__m128d __A, __m128d __B) 1454117395Skan{ 1455117395Skan return (__m128d)__builtin_ia32_cmpeqsd ((__v2df)__A, (__v2df)__B); 1456117395Skan} 1457117395Skan 1458117395Skanstatic __inline __m128d 1459117395Skan_mm_cmplt_sd (__m128d __A, __m128d __B) 1460117395Skan{ 1461117395Skan return (__m128d)__builtin_ia32_cmpltsd ((__v2df)__A, (__v2df)__B); 1462117395Skan} 1463117395Skan 1464117395Skanstatic __inline __m128d 1465117395Skan_mm_cmple_sd (__m128d __A, __m128d __B) 1466117395Skan{ 1467117395Skan return (__m128d)__builtin_ia32_cmplesd ((__v2df)__A, (__v2df)__B); 1468117395Skan} 1469117395Skan 1470117395Skanstatic __inline __m128d 1471117395Skan_mm_cmpgt_sd (__m128d __A, __m128d __B) 1472117395Skan{ 1473117395Skan return (__m128d) __builtin_ia32_movsd ((__v2df) __A, 1474117395Skan (__v2df) 1475117395Skan __builtin_ia32_cmpltsd ((__v2df) __B, 1476117395Skan (__v2df) 1477117395Skan __A)); 1478117395Skan} 1479117395Skan 1480117395Skanstatic __inline __m128d 1481117395Skan_mm_cmpge_sd (__m128d __A, __m128d __B) 1482117395Skan{ 1483117395Skan return (__m128d) __builtin_ia32_movsd ((__v2df) __A, 1484117395Skan (__v2df) 1485117395Skan __builtin_ia32_cmplesd ((__v2df) __B, 1486117395Skan (__v2df) 1487117395Skan __A)); 1488117395Skan} 1489117395Skan 1490117395Skanstatic __inline __m128d 1491117395Skan_mm_cmpneq_sd (__m128d __A, __m128d __B) 1492117395Skan{ 1493117395Skan return (__m128d)__builtin_ia32_cmpneqsd ((__v2df)__A, (__v2df)__B); 1494117395Skan} 1495117395Skan 1496117395Skanstatic __inline __m128d 1497117395Skan_mm_cmpnlt_sd (__m128d __A, __m128d __B) 1498117395Skan{ 1499117395Skan return (__m128d)__builtin_ia32_cmpnltsd ((__v2df)__A, (__v2df)__B); 1500117395Skan} 1501117395Skan 1502117395Skanstatic __inline __m128d 1503117395Skan_mm_cmpnle_sd (__m128d __A, __m128d __B) 1504117395Skan{ 1505117395Skan return (__m128d)__builtin_ia32_cmpnlesd ((__v2df)__A, (__v2df)__B); 1506117395Skan} 1507117395Skan 1508117395Skanstatic __inline __m128d 1509117395Skan_mm_cmpngt_sd (__m128d __A, __m128d __B) 1510117395Skan{ 1511117395Skan return (__m128d) __builtin_ia32_movsd ((__v2df) __A, 1512117395Skan (__v2df) 1513117395Skan __builtin_ia32_cmpnltsd ((__v2df) __B, 1514117395Skan (__v2df) 1515117395Skan __A)); 1516117395Skan} 1517117395Skan 1518117395Skanstatic __inline __m128d 1519117395Skan_mm_cmpnge_sd (__m128d __A, __m128d __B) 1520117395Skan{ 1521117395Skan return (__m128d) __builtin_ia32_movsd ((__v2df) __A, 1522117395Skan (__v2df) 1523117395Skan __builtin_ia32_cmpnlesd ((__v2df) __B, 1524117395Skan (__v2df) 1525117395Skan __A)); 1526117395Skan} 1527117395Skan 1528117395Skanstatic __inline __m128d 1529117395Skan_mm_cmpord_sd (__m128d __A, __m128d __B) 1530117395Skan{ 1531117395Skan return (__m128d)__builtin_ia32_cmpordsd ((__v2df)__A, (__v2df)__B); 1532117395Skan} 1533117395Skan 1534117395Skanstatic __inline __m128d 1535117395Skan_mm_cmpunord_sd (__m128d __A, __m128d __B) 1536117395Skan{ 1537117395Skan return (__m128d)__builtin_ia32_cmpunordsd ((__v2df)__A, (__v2df)__B); 1538117395Skan} 1539117395Skan 1540117395Skanstatic __inline int 1541117395Skan_mm_comieq_sd (__m128d __A, __m128d __B) 1542117395Skan{ 1543117395Skan return __builtin_ia32_comisdeq ((__v2df)__A, (__v2df)__B); 1544117395Skan} 1545117395Skan 1546117395Skanstatic __inline int 1547117395Skan_mm_comilt_sd (__m128d __A, __m128d __B) 1548117395Skan{ 1549117395Skan return __builtin_ia32_comisdlt ((__v2df)__A, (__v2df)__B); 1550117395Skan} 1551117395Skan 1552117395Skanstatic __inline int 1553117395Skan_mm_comile_sd (__m128d __A, __m128d __B) 1554117395Skan{ 1555117395Skan return __builtin_ia32_comisdle ((__v2df)__A, (__v2df)__B); 1556117395Skan} 1557117395Skan 1558117395Skanstatic __inline int 1559117395Skan_mm_comigt_sd (__m128d __A, __m128d __B) 1560117395Skan{ 1561117395Skan return __builtin_ia32_comisdgt ((__v2df)__A, (__v2df)__B); 1562117395Skan} 1563117395Skan 1564117395Skanstatic __inline int 1565117395Skan_mm_comige_sd (__m128d __A, __m128d __B) 1566117395Skan{ 1567117395Skan return __builtin_ia32_comisdge ((__v2df)__A, (__v2df)__B); 1568117395Skan} 1569117395Skan 1570117395Skanstatic __inline int 1571117395Skan_mm_comineq_sd (__m128d __A, __m128d __B) 1572117395Skan{ 1573117395Skan return __builtin_ia32_comisdneq ((__v2df)__A, (__v2df)__B); 1574117395Skan} 1575117395Skan 1576117395Skanstatic __inline int 1577117395Skan_mm_ucomieq_sd (__m128d __A, __m128d __B) 1578117395Skan{ 1579117395Skan return __builtin_ia32_ucomisdeq ((__v2df)__A, (__v2df)__B); 1580117395Skan} 1581117395Skan 1582117395Skanstatic __inline int 1583117395Skan_mm_ucomilt_sd (__m128d __A, __m128d __B) 1584117395Skan{ 1585117395Skan return __builtin_ia32_ucomisdlt ((__v2df)__A, (__v2df)__B); 1586117395Skan} 1587117395Skan 1588117395Skanstatic __inline int 1589117395Skan_mm_ucomile_sd (__m128d __A, __m128d __B) 1590117395Skan{ 1591117395Skan return __builtin_ia32_ucomisdle ((__v2df)__A, (__v2df)__B); 1592117395Skan} 1593117395Skan 1594117395Skanstatic __inline int 1595117395Skan_mm_ucomigt_sd (__m128d __A, __m128d __B) 1596117395Skan{ 1597117395Skan return __builtin_ia32_ucomisdgt ((__v2df)__A, (__v2df)__B); 1598117395Skan} 1599117395Skan 1600117395Skanstatic __inline int 1601117395Skan_mm_ucomige_sd (__m128d __A, __m128d __B) 1602117395Skan{ 1603117395Skan return __builtin_ia32_ucomisdge ((__v2df)__A, (__v2df)__B); 1604117395Skan} 1605117395Skan 1606117395Skanstatic __inline int 1607117395Skan_mm_ucomineq_sd (__m128d __A, __m128d __B) 1608117395Skan{ 1609117395Skan return __builtin_ia32_ucomisdneq ((__v2df)__A, (__v2df)__B); 1610117395Skan} 1611117395Skan 1612117395Skan/* Create a vector with element 0 as *P and the rest zero. */ 1613117395Skan 1614117395Skanstatic __inline __m128i 1615117395Skan_mm_load_si128 (__m128i const *__P) 1616117395Skan{ 1617117395Skan return (__m128i) __builtin_ia32_loaddqa ((char const *)__P); 1618117395Skan} 1619117395Skan 1620117395Skanstatic __inline __m128i 1621117395Skan_mm_loadu_si128 (__m128i const *__P) 1622117395Skan{ 1623117395Skan return (__m128i) __builtin_ia32_loaddqu ((char const *)__P); 1624117395Skan} 1625117395Skan 1626117395Skanstatic __inline __m128i 1627117395Skan_mm_loadl_epi64 (__m128i const *__P) 1628117395Skan{ 1629117395Skan return (__m128i) __builtin_ia32_movq2dq (*(unsigned long long *)__P); 1630117395Skan} 1631117395Skan 1632117395Skanstatic __inline void 1633117395Skan_mm_store_si128 (__m128i *__P, __m128i __B) 1634117395Skan{ 1635117395Skan __builtin_ia32_storedqa ((char *)__P, (__v16qi)__B); 1636117395Skan} 1637117395Skan 1638117395Skanstatic __inline void 1639117395Skan_mm_storeu_si128 (__m128i *__P, __m128i __B) 1640117395Skan{ 1641117395Skan __builtin_ia32_storedqu ((char *)__P, (__v16qi)__B); 1642117395Skan} 1643117395Skan 1644117395Skanstatic __inline void 1645117395Skan_mm_storel_epi64 (__m128i *__P, __m128i __B) 1646117395Skan{ 1647117395Skan *(long long *)__P = __builtin_ia32_movdq2q ((__v2di)__B); 1648117395Skan} 1649117395Skan 1650117395Skanstatic __inline __m64 1651117395Skan_mm_movepi64_pi64 (__m128i __B) 1652117395Skan{ 1653117395Skan return (__m64) __builtin_ia32_movdq2q ((__v2di)__B); 1654117395Skan} 1655117395Skan 1656117395Skanstatic __inline __m128i 1657117395Skan_mm_move_epi64 (__m128i __A) 1658117395Skan{ 1659117395Skan return (__m128i) __builtin_ia32_movq ((__v2di)__A); 1660117395Skan} 1661117395Skan 1662117395Skan/* Create a vector of zeros. */ 1663117395Skanstatic __inline __m128i 1664117395Skan_mm_setzero_si128 (void) 1665117395Skan{ 1666117395Skan return (__m128i) __builtin_ia32_setzero128 (); 1667117395Skan} 1668117395Skan 1669117395Skanstatic __inline __m128i 1670117395Skan_mm_set_epi64 (__m64 __A, __m64 __B) 1671117395Skan{ 1672117395Skan __v2di __tmp = (__v2di)__builtin_ia32_movq2dq ((unsigned long long)__A); 1673117395Skan __v2di __tmp2 = (__v2di)__builtin_ia32_movq2dq ((unsigned long long)__B); 1674117395Skan return (__m128i)__builtin_ia32_punpcklqdq128 (__tmp2, __tmp); 1675117395Skan} 1676117395Skan 1677117395Skan/* Create the vector [Z Y X W]. */ 1678117395Skanstatic __inline __m128i 1679117395Skan_mm_set_epi32 (int __Z, int __Y, int __X, int __W) 1680117395Skan{ 1681117395Skan union { 1682117395Skan int __a[4]; 1683117395Skan __m128i __v; 1684117395Skan } __u; 1685117395Skan 1686117395Skan __u.__a[0] = __W; 1687117395Skan __u.__a[1] = __X; 1688117395Skan __u.__a[2] = __Y; 1689117395Skan __u.__a[3] = __Z; 1690117395Skan 1691117395Skan return __u.__v; 1692117395Skan} 1693117395Skan 1694117395Skan#ifdef __x86_64__ 1695117395Skan/* Create the vector [Z Y]. */ 1696117395Skanstatic __inline __m128i 1697117395Skan_mm_set_epi64x (long long __Z, long long __Y) 1698117395Skan{ 1699117395Skan union { 1700117395Skan long __a[2]; 1701117395Skan __m128i __v; 1702117395Skan } __u; 1703117395Skan 1704117395Skan __u.__a[0] = __Y; 1705117395Skan __u.__a[1] = __Z; 1706117395Skan 1707117395Skan return __u.__v; 1708117395Skan} 1709117395Skan#endif 1710117395Skan 1711117395Skan/* Create the vector [S T U V Z Y X W]. */ 1712117395Skanstatic __inline __m128i 1713117395Skan_mm_set_epi16 (short __Z, short __Y, short __X, short __W, 1714117395Skan short __V, short __U, short __T, short __S) 1715117395Skan{ 1716117395Skan union { 1717117395Skan short __a[8]; 1718117395Skan __m128i __v; 1719117395Skan } __u; 1720117395Skan 1721117395Skan __u.__a[0] = __S; 1722117395Skan __u.__a[1] = __T; 1723117395Skan __u.__a[2] = __U; 1724117395Skan __u.__a[3] = __V; 1725117395Skan __u.__a[4] = __W; 1726117395Skan __u.__a[5] = __X; 1727117395Skan __u.__a[6] = __Y; 1728117395Skan __u.__a[7] = __Z; 1729117395Skan 1730117395Skan return __u.__v; 1731117395Skan} 1732117395Skan 1733117395Skan/* Create the vector [S T U V Z Y X W]. */ 1734117395Skanstatic __inline __m128i 1735117395Skan_mm_set_epi8 (char __Z, char __Y, char __X, char __W, 1736117395Skan char __V, char __U, char __T, char __S, 1737117395Skan char __Z1, char __Y1, char __X1, char __W1, 1738117395Skan char __V1, char __U1, char __T1, char __S1) 1739117395Skan{ 1740117395Skan union { 1741117395Skan char __a[16]; 1742117395Skan __m128i __v; 1743117395Skan } __u; 1744117395Skan 1745117395Skan __u.__a[0] = __S1; 1746117395Skan __u.__a[1] = __T1; 1747117395Skan __u.__a[2] = __U1; 1748117395Skan __u.__a[3] = __V1; 1749117395Skan __u.__a[4] = __W1; 1750117395Skan __u.__a[5] = __X1; 1751117395Skan __u.__a[6] = __Y1; 1752117395Skan __u.__a[7] = __Z1; 1753117395Skan __u.__a[8] = __S; 1754117395Skan __u.__a[9] = __T; 1755117395Skan __u.__a[10] = __U; 1756117395Skan __u.__a[11] = __V; 1757117395Skan __u.__a[12] = __W; 1758117395Skan __u.__a[13] = __X; 1759117395Skan __u.__a[14] = __Y; 1760117395Skan __u.__a[15] = __Z; 1761117395Skan 1762117395Skan return __u.__v; 1763117395Skan} 1764117395Skan 1765117395Skanstatic __inline __m128i 1766117395Skan_mm_set1_epi64 (__m64 __A) 1767117395Skan{ 1768117395Skan __v2di __tmp = (__v2di)__builtin_ia32_movq2dq ((unsigned long long)__A); 1769117395Skan return (__m128i)__builtin_ia32_punpcklqdq128 (__tmp, __tmp); 1770117395Skan} 1771117395Skan 1772117395Skanstatic __inline __m128i 1773117395Skan_mm_set1_epi32 (int __A) 1774117395Skan{ 1775117395Skan __v4si __tmp = (__v4si)__builtin_ia32_loadd (&__A); 1776117395Skan return (__m128i) __builtin_ia32_pshufd ((__v4si)__tmp, _MM_SHUFFLE (0,0,0,0)); 1777117395Skan} 1778117395Skan 1779117395Skan#ifdef __x86_64__ 1780117395Skanstatic __inline __m128i 1781117395Skan_mm_set1_epi64x (long long __A) 1782117395Skan{ 1783117395Skan __v2di __tmp = (__v2di)__builtin_ia32_movq2dq ((unsigned long long)__A); 1784117395Skan return (__m128i) __builtin_ia32_shufpd ((__v2df)__tmp, (__v2df)__tmp, _MM_SHUFFLE2 (0,0)); 1785117395Skan} 1786117395Skan#endif 1787117395Skan 1788117395Skanstatic __inline __m128i 1789117395Skan_mm_set1_epi16 (short __A) 1790117395Skan{ 1791117395Skan int __Acopy = (unsigned short)__A; 1792117395Skan __v4si __tmp = (__v4si)__builtin_ia32_loadd (&__Acopy); 1793117395Skan __tmp = (__v4si)__builtin_ia32_punpcklwd128 ((__v8hi)__tmp, (__v8hi)__tmp); 1794117395Skan return (__m128i) __builtin_ia32_pshufd ((__v4si)__tmp, _MM_SHUFFLE (0,0,0,0)); 1795117395Skan} 1796117395Skan 1797117395Skanstatic __inline __m128i 1798117395Skan_mm_set1_epi8 (char __A) 1799117395Skan{ 1800117395Skan int __Acopy = (unsigned char)__A; 1801117395Skan __v4si __tmp = (__v4si)__builtin_ia32_loadd (&__Acopy); 1802117395Skan __tmp = (__v4si)__builtin_ia32_punpcklbw128 ((__v16qi)__tmp, (__v16qi)__tmp); 1803117395Skan __tmp = (__v4si)__builtin_ia32_punpcklbw128 ((__v16qi)__tmp, (__v16qi)__tmp); 1804117395Skan return (__m128i) __builtin_ia32_pshufd ((__v4si)__tmp, _MM_SHUFFLE (0,0,0,0)); 1805117395Skan} 1806117395Skan 1807117395Skanstatic __inline __m128i 1808117395Skan_mm_setr_epi64 (__m64 __A, __m64 __B) 1809117395Skan{ 1810117395Skan __v2di __tmp = (__v2di)__builtin_ia32_movq2dq ((unsigned long long)__A); 1811117395Skan __v2di __tmp2 = (__v2di)__builtin_ia32_movq2dq ((unsigned long long)__B); 1812117395Skan return (__m128i)__builtin_ia32_punpcklqdq128 (__tmp, __tmp2); 1813117395Skan} 1814117395Skan 1815117395Skan/* Create the vector [Z Y X W]. */ 1816117395Skanstatic __inline __m128i 1817117395Skan_mm_setr_epi32 (int __W, int __X, int __Y, int __Z) 1818117395Skan{ 1819117395Skan union { 1820117395Skan int __a[4]; 1821117395Skan __m128i __v; 1822117395Skan } __u; 1823117395Skan 1824117395Skan __u.__a[0] = __W; 1825117395Skan __u.__a[1] = __X; 1826117395Skan __u.__a[2] = __Y; 1827117395Skan __u.__a[3] = __Z; 1828117395Skan 1829117395Skan return __u.__v; 1830117395Skan} 1831117395Skan/* Create the vector [S T U V Z Y X W]. */ 1832117395Skanstatic __inline __m128i 1833117395Skan_mm_setr_epi16 (short __S, short __T, short __U, short __V, 1834117395Skan short __W, short __X, short __Y, short __Z) 1835117395Skan{ 1836117395Skan union { 1837117395Skan short __a[8]; 1838117395Skan __m128i __v; 1839117395Skan } __u; 1840117395Skan 1841117395Skan __u.__a[0] = __S; 1842117395Skan __u.__a[1] = __T; 1843117395Skan __u.__a[2] = __U; 1844117395Skan __u.__a[3] = __V; 1845117395Skan __u.__a[4] = __W; 1846117395Skan __u.__a[5] = __X; 1847117395Skan __u.__a[6] = __Y; 1848117395Skan __u.__a[7] = __Z; 1849117395Skan 1850117395Skan return __u.__v; 1851117395Skan} 1852117395Skan 1853117395Skan/* Create the vector [S T U V Z Y X W]. */ 1854117395Skanstatic __inline __m128i 1855117395Skan_mm_setr_epi8 (char __S1, char __T1, char __U1, char __V1, 1856117395Skan char __W1, char __X1, char __Y1, char __Z1, 1857117395Skan char __S, char __T, char __U, char __V, 1858117395Skan char __W, char __X, char __Y, char __Z) 1859117395Skan{ 1860117395Skan union { 1861117395Skan char __a[16]; 1862117395Skan __m128i __v; 1863117395Skan } __u; 1864117395Skan 1865117395Skan __u.__a[0] = __S1; 1866117395Skan __u.__a[1] = __T1; 1867117395Skan __u.__a[2] = __U1; 1868117395Skan __u.__a[3] = __V1; 1869117395Skan __u.__a[4] = __W1; 1870117395Skan __u.__a[5] = __X1; 1871117395Skan __u.__a[6] = __Y1; 1872117395Skan __u.__a[7] = __Z1; 1873117395Skan __u.__a[8] = __S; 1874117395Skan __u.__a[9] = __T; 1875117395Skan __u.__a[10] = __U; 1876117395Skan __u.__a[11] = __V; 1877117395Skan __u.__a[12] = __W; 1878117395Skan __u.__a[13] = __X; 1879117395Skan __u.__a[14] = __Y; 1880117395Skan __u.__a[15] = __Z; 1881117395Skan 1882117395Skan return __u.__v; 1883117395Skan} 1884117395Skan 1885117395Skanstatic __inline __m128d 1886117395Skan_mm_cvtepi32_pd (__m128i __A) 1887117395Skan{ 1888117395Skan return (__m128d)__builtin_ia32_cvtdq2pd ((__v4si) __A); 1889117395Skan} 1890117395Skan 1891117395Skanstatic __inline __m128 1892117395Skan_mm_cvtepi32_ps (__m128i __A) 1893117395Skan{ 1894117395Skan return (__m128)__builtin_ia32_cvtdq2ps ((__v4si) __A); 1895117395Skan} 1896117395Skan 1897117395Skanstatic __inline __m128i 1898117395Skan_mm_cvtpd_epi32 (__m128d __A) 1899117395Skan{ 1900117395Skan return (__m128i)__builtin_ia32_cvtpd2dq ((__v2df) __A); 1901117395Skan} 1902117395Skan 1903117395Skanstatic __inline __m64 1904117395Skan_mm_cvtpd_pi32 (__m128d __A) 1905117395Skan{ 1906117395Skan return (__m64)__builtin_ia32_cvtpd2pi ((__v2df) __A); 1907117395Skan} 1908117395Skan 1909117395Skanstatic __inline __m128 1910117395Skan_mm_cvtpd_ps (__m128d __A) 1911117395Skan{ 1912117395Skan return (__m128)__builtin_ia32_cvtpd2ps ((__v2df) __A); 1913117395Skan} 1914117395Skan 1915117395Skanstatic __inline __m128i 1916117395Skan_mm_cvttpd_epi32 (__m128d __A) 1917117395Skan{ 1918117395Skan return (__m128i)__builtin_ia32_cvttpd2dq ((__v2df) __A); 1919117395Skan} 1920117395Skan 1921117395Skanstatic __inline __m64 1922117395Skan_mm_cvttpd_pi32 (__m128d __A) 1923117395Skan{ 1924117395Skan return (__m64)__builtin_ia32_cvttpd2pi ((__v2df) __A); 1925117395Skan} 1926117395Skan 1927117395Skanstatic __inline __m128d 1928117395Skan_mm_cvtpi32_pd (__m64 __A) 1929117395Skan{ 1930117395Skan return (__m128d)__builtin_ia32_cvtpi2pd ((__v2si) __A); 1931117395Skan} 1932117395Skan 1933117395Skanstatic __inline __m128i 1934117395Skan_mm_cvtps_epi32 (__m128 __A) 1935117395Skan{ 1936117395Skan return (__m128i)__builtin_ia32_cvtps2dq ((__v4sf) __A); 1937117395Skan} 1938117395Skan 1939117395Skanstatic __inline __m128i 1940117395Skan_mm_cvttps_epi32 (__m128 __A) 1941117395Skan{ 1942117395Skan return (__m128i)__builtin_ia32_cvttps2dq ((__v4sf) __A); 1943117395Skan} 1944117395Skan 1945117395Skanstatic __inline __m128d 1946117395Skan_mm_cvtps_pd (__m128 __A) 1947117395Skan{ 1948117395Skan return (__m128d)__builtin_ia32_cvtps2pd ((__v4sf) __A); 1949117395Skan} 1950117395Skan 1951117395Skanstatic __inline int 1952117395Skan_mm_cvtsd_si32 (__m128d __A) 1953117395Skan{ 1954117395Skan return __builtin_ia32_cvtsd2si ((__v2df) __A); 1955117395Skan} 1956117395Skan 1957117395Skan#ifdef __x86_64__ 1958117395Skanstatic __inline long long 1959117395Skan_mm_cvtsd_si64x (__m128d __A) 1960117395Skan{ 1961117395Skan return __builtin_ia32_cvtsd2si64 ((__v2df) __A); 1962117395Skan} 1963117395Skan#endif 1964117395Skan 1965117395Skanstatic __inline int 1966117395Skan_mm_cvttsd_si32 (__m128d __A) 1967117395Skan{ 1968117395Skan return __builtin_ia32_cvttsd2si ((__v2df) __A); 1969117395Skan} 1970117395Skan 1971117395Skan#ifdef __x86_64__ 1972117395Skanstatic __inline long long 1973117395Skan_mm_cvttsd_si64x (__m128d __A) 1974117395Skan{ 1975117395Skan return __builtin_ia32_cvttsd2si64 ((__v2df) __A); 1976117395Skan} 1977117395Skan#endif 1978117395Skan 1979117395Skanstatic __inline __m128 1980117395Skan_mm_cvtsd_ss (__m128 __A, __m128d __B) 1981117395Skan{ 1982117395Skan return (__m128)__builtin_ia32_cvtsd2ss ((__v4sf) __A, (__v2df) __B); 1983117395Skan} 1984117395Skan 1985117395Skanstatic __inline __m128d 1986117395Skan_mm_cvtsi32_sd (__m128d __A, int __B) 1987117395Skan{ 1988117395Skan return (__m128d)__builtin_ia32_cvtsi2sd ((__v2df) __A, __B); 1989117395Skan} 1990117395Skan 1991117395Skan#ifdef __x86_64__ 1992117395Skanstatic __inline __m128d 1993117395Skan_mm_cvtsi64x_sd (__m128d __A, long long __B) 1994117395Skan{ 1995117395Skan return (__m128d)__builtin_ia32_cvtsi642sd ((__v2df) __A, __B); 1996117395Skan} 1997117395Skan#endif 1998117395Skan 1999117395Skanstatic __inline __m128d 2000117395Skan_mm_cvtss_sd (__m128d __A, __m128 __B) 2001117395Skan{ 2002117395Skan return (__m128d)__builtin_ia32_cvtss2sd ((__v2df) __A, (__v4sf)__B); 2003117395Skan} 2004117395Skan 2005117395Skan#define _mm_shuffle_pd(__A, __B, __C) ((__m128d)__builtin_ia32_shufpd ((__v2df)__A, (__v2df)__B, (__C))) 2006117395Skan 2007117395Skanstatic __inline __m128d 2008117395Skan_mm_unpackhi_pd (__m128d __A, __m128d __B) 2009117395Skan{ 2010117395Skan return (__m128d)__builtin_ia32_unpckhpd ((__v2df)__A, (__v2df)__B); 2011117395Skan} 2012117395Skan 2013117395Skanstatic __inline __m128d 2014117395Skan_mm_unpacklo_pd (__m128d __A, __m128d __B) 2015117395Skan{ 2016117395Skan return (__m128d)__builtin_ia32_unpcklpd ((__v2df)__A, (__v2df)__B); 2017117395Skan} 2018117395Skan 2019117395Skanstatic __inline __m128d 2020117395Skan_mm_loadh_pd (__m128d __A, double const *__B) 2021117395Skan{ 2022117395Skan return (__m128d)__builtin_ia32_loadhpd ((__v2df)__A, (__v2si *)__B); 2023117395Skan} 2024117395Skan 2025117395Skanstatic __inline void 2026117395Skan_mm_storeh_pd (double *__A, __m128d __B) 2027117395Skan{ 2028117395Skan __builtin_ia32_storehpd ((__v2si *)__A, (__v2df)__B); 2029117395Skan} 2030117395Skan 2031117395Skanstatic __inline __m128d 2032117395Skan_mm_loadl_pd (__m128d __A, double const *__B) 2033117395Skan{ 2034117395Skan return (__m128d)__builtin_ia32_loadlpd ((__v2df)__A, (__v2si *)__B); 2035117395Skan} 2036117395Skan 2037117395Skanstatic __inline void 2038117395Skan_mm_storel_pd (double *__A, __m128d __B) 2039117395Skan{ 2040117395Skan __builtin_ia32_storelpd ((__v2si *)__A, (__v2df)__B); 2041117395Skan} 2042117395Skan 2043117395Skanstatic __inline int 2044117395Skan_mm_movemask_pd (__m128d __A) 2045117395Skan{ 2046117395Skan return __builtin_ia32_movmskpd ((__v2df)__A); 2047117395Skan} 2048117395Skan 2049117395Skanstatic __inline __m128i 2050117395Skan_mm_packs_epi16 (__m128i __A, __m128i __B) 2051117395Skan{ 2052117395Skan return (__m128i)__builtin_ia32_packsswb128 ((__v8hi)__A, (__v8hi)__B); 2053117395Skan} 2054117395Skan 2055117395Skanstatic __inline __m128i 2056117395Skan_mm_packs_epi32 (__m128i __A, __m128i __B) 2057117395Skan{ 2058117395Skan return (__m128i)__builtin_ia32_packssdw128 ((__v4si)__A, (__v4si)__B); 2059117395Skan} 2060117395Skan 2061117395Skanstatic __inline __m128i 2062117395Skan_mm_packus_epi16 (__m128i __A, __m128i __B) 2063117395Skan{ 2064117395Skan return (__m128i)__builtin_ia32_packuswb128 ((__v8hi)__A, (__v8hi)__B); 2065117395Skan} 2066117395Skan 2067117395Skanstatic __inline __m128i 2068117395Skan_mm_unpackhi_epi8 (__m128i __A, __m128i __B) 2069117395Skan{ 2070117395Skan return (__m128i)__builtin_ia32_punpckhbw128 ((__v16qi)__A, (__v16qi)__B); 2071117395Skan} 2072117395Skan 2073117395Skanstatic __inline __m128i 2074117395Skan_mm_unpackhi_epi16 (__m128i __A, __m128i __B) 2075117395Skan{ 2076117395Skan return (__m128i)__builtin_ia32_punpckhwd128 ((__v8hi)__A, (__v8hi)__B); 2077117395Skan} 2078117395Skan 2079117395Skanstatic __inline __m128i 2080117395Skan_mm_unpackhi_epi32 (__m128i __A, __m128i __B) 2081117395Skan{ 2082117395Skan return (__m128i)__builtin_ia32_punpckhdq128 ((__v4si)__A, (__v4si)__B); 2083117395Skan} 2084117395Skan 2085117395Skanstatic __inline __m128i 2086117395Skan_mm_unpackhi_epi64 (__m128i __A, __m128i __B) 2087117395Skan{ 2088117395Skan return (__m128i)__builtin_ia32_punpckhqdq128 ((__v2di)__A, (__v2di)__B); 2089117395Skan} 2090117395Skan 2091117395Skanstatic __inline __m128i 2092117395Skan_mm_unpacklo_epi8 (__m128i __A, __m128i __B) 2093117395Skan{ 2094117395Skan return (__m128i)__builtin_ia32_punpcklbw128 ((__v16qi)__A, (__v16qi)__B); 2095117395Skan} 2096117395Skan 2097117395Skanstatic __inline __m128i 2098117395Skan_mm_unpacklo_epi16 (__m128i __A, __m128i __B) 2099117395Skan{ 2100117395Skan return (__m128i)__builtin_ia32_punpcklwd128 ((__v8hi)__A, (__v8hi)__B); 2101117395Skan} 2102117395Skan 2103117395Skanstatic __inline __m128i 2104117395Skan_mm_unpacklo_epi32 (__m128i __A, __m128i __B) 2105117395Skan{ 2106117395Skan return (__m128i)__builtin_ia32_punpckldq128 ((__v4si)__A, (__v4si)__B); 2107117395Skan} 2108117395Skan 2109117395Skanstatic __inline __m128i 2110117395Skan_mm_unpacklo_epi64 (__m128i __A, __m128i __B) 2111117395Skan{ 2112117395Skan return (__m128i)__builtin_ia32_punpcklqdq128 ((__v2di)__A, (__v2di)__B); 2113117395Skan} 2114117395Skan 2115117395Skanstatic __inline __m128i 2116117395Skan_mm_add_epi8 (__m128i __A, __m128i __B) 2117117395Skan{ 2118117395Skan return (__m128i)__builtin_ia32_paddb128 ((__v16qi)__A, (__v16qi)__B); 2119117395Skan} 2120117395Skan 2121117395Skanstatic __inline __m128i 2122117395Skan_mm_add_epi16 (__m128i __A, __m128i __B) 2123117395Skan{ 2124117395Skan return (__m128i)__builtin_ia32_paddw128 ((__v8hi)__A, (__v8hi)__B); 2125117395Skan} 2126117395Skan 2127117395Skanstatic __inline __m128i 2128117395Skan_mm_add_epi32 (__m128i __A, __m128i __B) 2129117395Skan{ 2130117395Skan return (__m128i)__builtin_ia32_paddd128 ((__v4si)__A, (__v4si)__B); 2131117395Skan} 2132117395Skan 2133117395Skanstatic __inline __m128i 2134117395Skan_mm_add_epi64 (__m128i __A, __m128i __B) 2135117395Skan{ 2136117395Skan return (__m128i)__builtin_ia32_paddq128 ((__v2di)__A, (__v2di)__B); 2137117395Skan} 2138117395Skan 2139117395Skanstatic __inline __m128i 2140117395Skan_mm_adds_epi8 (__m128i __A, __m128i __B) 2141117395Skan{ 2142117395Skan return (__m128i)__builtin_ia32_paddsb128 ((__v16qi)__A, (__v16qi)__B); 2143117395Skan} 2144117395Skan 2145117395Skanstatic __inline __m128i 2146117395Skan_mm_adds_epi16 (__m128i __A, __m128i __B) 2147117395Skan{ 2148117395Skan return (__m128i)__builtin_ia32_paddsw128 ((__v8hi)__A, (__v8hi)__B); 2149117395Skan} 2150117395Skan 2151117395Skanstatic __inline __m128i 2152117395Skan_mm_adds_epu8 (__m128i __A, __m128i __B) 2153117395Skan{ 2154117395Skan return (__m128i)__builtin_ia32_paddusb128 ((__v16qi)__A, (__v16qi)__B); 2155117395Skan} 2156117395Skan 2157117395Skanstatic __inline __m128i 2158117395Skan_mm_adds_epu16 (__m128i __A, __m128i __B) 2159117395Skan{ 2160117395Skan return (__m128i)__builtin_ia32_paddusw128 ((__v8hi)__A, (__v8hi)__B); 2161117395Skan} 2162117395Skan 2163117395Skanstatic __inline __m128i 2164117395Skan_mm_sub_epi8 (__m128i __A, __m128i __B) 2165117395Skan{ 2166117395Skan return (__m128i)__builtin_ia32_psubb128 ((__v16qi)__A, (__v16qi)__B); 2167117395Skan} 2168117395Skan 2169117395Skanstatic __inline __m128i 2170117395Skan_mm_sub_epi16 (__m128i __A, __m128i __B) 2171117395Skan{ 2172117395Skan return (__m128i)__builtin_ia32_psubw128 ((__v8hi)__A, (__v8hi)__B); 2173117395Skan} 2174117395Skan 2175117395Skanstatic __inline __m128i 2176117395Skan_mm_sub_epi32 (__m128i __A, __m128i __B) 2177117395Skan{ 2178117395Skan return (__m128i)__builtin_ia32_psubd128 ((__v4si)__A, (__v4si)__B); 2179117395Skan} 2180117395Skan 2181117395Skanstatic __inline __m128i 2182117395Skan_mm_sub_epi64 (__m128i __A, __m128i __B) 2183117395Skan{ 2184117395Skan return (__m128i)__builtin_ia32_psubq128 ((__v2di)__A, (__v2di)__B); 2185117395Skan} 2186117395Skan 2187117395Skanstatic __inline __m128i 2188117395Skan_mm_subs_epi8 (__m128i __A, __m128i __B) 2189117395Skan{ 2190117395Skan return (__m128i)__builtin_ia32_psubsb128 ((__v16qi)__A, (__v16qi)__B); 2191117395Skan} 2192117395Skan 2193117395Skanstatic __inline __m128i 2194117395Skan_mm_subs_epi16 (__m128i __A, __m128i __B) 2195117395Skan{ 2196117395Skan return (__m128i)__builtin_ia32_psubsw128 ((__v8hi)__A, (__v8hi)__B); 2197117395Skan} 2198117395Skan 2199117395Skanstatic __inline __m128i 2200117395Skan_mm_subs_epu8 (__m128i __A, __m128i __B) 2201117395Skan{ 2202117395Skan return (__m128i)__builtin_ia32_psubusb128 ((__v16qi)__A, (__v16qi)__B); 2203117395Skan} 2204117395Skan 2205117395Skanstatic __inline __m128i 2206117395Skan_mm_subs_epu16 (__m128i __A, __m128i __B) 2207117395Skan{ 2208117395Skan return (__m128i)__builtin_ia32_psubusw128 ((__v8hi)__A, (__v8hi)__B); 2209117395Skan} 2210117395Skan 2211117395Skanstatic __inline __m128i 2212117395Skan_mm_madd_epi16 (__m128i __A, __m128i __B) 2213117395Skan{ 2214117395Skan return (__m128i)__builtin_ia32_pmaddwd128 ((__v8hi)__A, (__v8hi)__B); 2215117395Skan} 2216117395Skan 2217117395Skanstatic __inline __m128i 2218117395Skan_mm_mulhi_epi16 (__m128i __A, __m128i __B) 2219117395Skan{ 2220117395Skan return (__m128i)__builtin_ia32_pmulhw128 ((__v8hi)__A, (__v8hi)__B); 2221117395Skan} 2222117395Skan 2223117395Skanstatic __inline __m128i 2224117395Skan_mm_mullo_epi16 (__m128i __A, __m128i __B) 2225117395Skan{ 2226117395Skan return (__m128i)__builtin_ia32_pmullw128 ((__v8hi)__A, (__v8hi)__B); 2227117395Skan} 2228117395Skan 2229117395Skanstatic __inline __m64 2230117395Skan_mm_mul_su32 (__m64 __A, __m64 __B) 2231117395Skan{ 2232117395Skan return (__m64)__builtin_ia32_pmuludq ((__v2si)__A, (__v2si)__B); 2233117395Skan} 2234117395Skan 2235117395Skanstatic __inline __m128i 2236117395Skan_mm_mul_epu32 (__m128i __A, __m128i __B) 2237117395Skan{ 2238117395Skan return (__m128i)__builtin_ia32_pmuludq128 ((__v4si)__A, (__v4si)__B); 2239117395Skan} 2240117395Skan 2241117395Skanstatic __inline __m128i 2242117395Skan_mm_sll_epi16 (__m128i __A, __m128i __B) 2243117395Skan{ 2244117395Skan return (__m128i)__builtin_ia32_psllw128 ((__v8hi)__A, (__v2di)__B); 2245117395Skan} 2246117395Skan 2247117395Skanstatic __inline __m128i 2248117395Skan_mm_sll_epi32 (__m128i __A, __m128i __B) 2249117395Skan{ 2250117395Skan return (__m128i)__builtin_ia32_pslld128 ((__v4si)__A, (__v2di)__B); 2251117395Skan} 2252117395Skan 2253117395Skanstatic __inline __m128i 2254117395Skan_mm_sll_epi64 (__m128i __A, __m128i __B) 2255117395Skan{ 2256117395Skan return (__m128i)__builtin_ia32_psllq128 ((__v2di)__A, (__v2di)__B); 2257117395Skan} 2258117395Skan 2259117395Skanstatic __inline __m128i 2260117395Skan_mm_sra_epi16 (__m128i __A, __m128i __B) 2261117395Skan{ 2262117395Skan return (__m128i)__builtin_ia32_psraw128 ((__v8hi)__A, (__v2di)__B); 2263117395Skan} 2264117395Skan 2265117395Skanstatic __inline __m128i 2266117395Skan_mm_sra_epi32 (__m128i __A, __m128i __B) 2267117395Skan{ 2268117395Skan return (__m128i)__builtin_ia32_psrad128 ((__v4si)__A, (__v2di)__B); 2269117395Skan} 2270117395Skan 2271117395Skanstatic __inline __m128i 2272117395Skan_mm_srl_epi16 (__m128i __A, __m128i __B) 2273117395Skan{ 2274117395Skan return (__m128i)__builtin_ia32_psrlw128 ((__v8hi)__A, (__v2di)__B); 2275117395Skan} 2276117395Skan 2277117395Skanstatic __inline __m128i 2278117395Skan_mm_srl_epi32 (__m128i __A, __m128i __B) 2279117395Skan{ 2280117395Skan return (__m128i)__builtin_ia32_psrld128 ((__v4si)__A, (__v2di)__B); 2281117395Skan} 2282117395Skan 2283117395Skanstatic __inline __m128i 2284117395Skan_mm_srl_epi64 (__m128i __A, __m128i __B) 2285117395Skan{ 2286117395Skan return (__m128i)__builtin_ia32_psrlq128 ((__v2di)__A, (__v2di)__B); 2287117395Skan} 2288117395Skan 2289117395Skanstatic __inline __m128i 2290117395Skan_mm_slli_epi16 (__m128i __A, int __B) 2291117395Skan{ 2292117395Skan return (__m128i)__builtin_ia32_psllwi128 ((__v8hi)__A, __B); 2293117395Skan} 2294117395Skan 2295117395Skanstatic __inline __m128i 2296117395Skan_mm_slli_epi32 (__m128i __A, int __B) 2297117395Skan{ 2298117395Skan return (__m128i)__builtin_ia32_pslldi128 ((__v4si)__A, __B); 2299117395Skan} 2300117395Skan 2301117395Skanstatic __inline __m128i 2302117395Skan_mm_slli_epi64 (__m128i __A, int __B) 2303117395Skan{ 2304117395Skan return (__m128i)__builtin_ia32_psllqi128 ((__v2di)__A, __B); 2305117395Skan} 2306117395Skan 2307117395Skanstatic __inline __m128i 2308117395Skan_mm_srai_epi16 (__m128i __A, int __B) 2309117395Skan{ 2310117395Skan return (__m128i)__builtin_ia32_psrawi128 ((__v8hi)__A, __B); 2311117395Skan} 2312117395Skan 2313117395Skanstatic __inline __m128i 2314117395Skan_mm_srai_epi32 (__m128i __A, int __B) 2315117395Skan{ 2316117395Skan return (__m128i)__builtin_ia32_psradi128 ((__v4si)__A, __B); 2317117395Skan} 2318117395Skan 2319117395Skan#if 0 2320117395Skanstatic __m128i __attribute__((__always_inline__)) 2321117395Skan_mm_srli_si128 (__m128i __A, const int __B) 2322117395Skan{ 2323117395Skan return ((__m128i)__builtin_ia32_psrldqi128 (__A, __B)) 2324117395Skan} 2325117395Skan 2326117395Skanstatic __m128i __attribute__((__always_inline__)) 2327117395Skan_mm_srli_si128 (__m128i __A, const int __B) 2328117395Skan{ 2329117395Skan return ((__m128i)__builtin_ia32_pslldqi128 (__A, __B)) 2330117395Skan} 2331117395Skan#endif 2332117395Skan#define _mm_srli_si128(__A, __B) ((__m128i)__builtin_ia32_psrldqi128 (__A, __B)) 2333117395Skan#define _mm_slli_si128(__A, __B) ((__m128i)__builtin_ia32_pslldqi128 (__A, __B)) 2334117395Skan 2335117395Skanstatic __inline __m128i 2336117395Skan_mm_srli_epi16 (__m128i __A, int __B) 2337117395Skan{ 2338117395Skan return (__m128i)__builtin_ia32_psrlwi128 ((__v8hi)__A, __B); 2339117395Skan} 2340117395Skan 2341117395Skanstatic __inline __m128i 2342117395Skan_mm_srli_epi32 (__m128i __A, int __B) 2343117395Skan{ 2344117395Skan return (__m128i)__builtin_ia32_psrldi128 ((__v4si)__A, __B); 2345117395Skan} 2346117395Skan 2347117395Skanstatic __inline __m128i 2348117395Skan_mm_srli_epi64 (__m128i __A, int __B) 2349117395Skan{ 2350117395Skan return (__m128i)__builtin_ia32_psrlqi128 ((__v2di)__A, __B); 2351117395Skan} 2352117395Skan 2353117395Skanstatic __inline __m128i 2354117395Skan_mm_and_si128 (__m128i __A, __m128i __B) 2355117395Skan{ 2356117395Skan return (__m128i)__builtin_ia32_pand128 ((__v2di)__A, (__v2di)__B); 2357117395Skan} 2358117395Skan 2359117395Skanstatic __inline __m128i 2360117395Skan_mm_andnot_si128 (__m128i __A, __m128i __B) 2361117395Skan{ 2362117395Skan return (__m128i)__builtin_ia32_pandn128 ((__v2di)__A, (__v2di)__B); 2363117395Skan} 2364117395Skan 2365117395Skanstatic __inline __m128i 2366117395Skan_mm_or_si128 (__m128i __A, __m128i __B) 2367117395Skan{ 2368117395Skan return (__m128i)__builtin_ia32_por128 ((__v2di)__A, (__v2di)__B); 2369117395Skan} 2370117395Skan 2371117395Skanstatic __inline __m128i 2372117395Skan_mm_xor_si128 (__m128i __A, __m128i __B) 2373117395Skan{ 2374117395Skan return (__m128i)__builtin_ia32_pxor128 ((__v2di)__A, (__v2di)__B); 2375117395Skan} 2376117395Skan 2377117395Skanstatic __inline __m128i 2378117395Skan_mm_cmpeq_epi8 (__m128i __A, __m128i __B) 2379117395Skan{ 2380117395Skan return (__m128i)__builtin_ia32_pcmpeqb128 ((__v16qi)__A, (__v16qi)__B); 2381117395Skan} 2382117395Skan 2383117395Skanstatic __inline __m128i 2384117395Skan_mm_cmpeq_epi16 (__m128i __A, __m128i __B) 2385117395Skan{ 2386117395Skan return (__m128i)__builtin_ia32_pcmpeqw128 ((__v8hi)__A, (__v8hi)__B); 2387117395Skan} 2388117395Skan 2389117395Skanstatic __inline __m128i 2390117395Skan_mm_cmpeq_epi32 (__m128i __A, __m128i __B) 2391117395Skan{ 2392117395Skan return (__m128i)__builtin_ia32_pcmpeqd128 ((__v4si)__A, (__v4si)__B); 2393117395Skan} 2394117395Skan 2395117395Skanstatic __inline __m128i 2396117395Skan_mm_cmplt_epi8 (__m128i __A, __m128i __B) 2397117395Skan{ 2398117395Skan return (__m128i)__builtin_ia32_pcmpgtb128 ((__v16qi)__B, (__v16qi)__A); 2399117395Skan} 2400117395Skan 2401117395Skanstatic __inline __m128i 2402117395Skan_mm_cmplt_epi16 (__m128i __A, __m128i __B) 2403117395Skan{ 2404117395Skan return (__m128i)__builtin_ia32_pcmpgtw128 ((__v8hi)__B, (__v8hi)__A); 2405117395Skan} 2406117395Skan 2407117395Skanstatic __inline __m128i 2408117395Skan_mm_cmplt_epi32 (__m128i __A, __m128i __B) 2409117395Skan{ 2410117395Skan return (__m128i)__builtin_ia32_pcmpgtd128 ((__v4si)__B, (__v4si)__A); 2411117395Skan} 2412117395Skan 2413117395Skanstatic __inline __m128i 2414117395Skan_mm_cmpgt_epi8 (__m128i __A, __m128i __B) 2415117395Skan{ 2416117395Skan return (__m128i)__builtin_ia32_pcmpgtb128 ((__v16qi)__A, (__v16qi)__B); 2417117395Skan} 2418117395Skan 2419117395Skanstatic __inline __m128i 2420117395Skan_mm_cmpgt_epi16 (__m128i __A, __m128i __B) 2421117395Skan{ 2422117395Skan return (__m128i)__builtin_ia32_pcmpgtw128 ((__v8hi)__A, (__v8hi)__B); 2423117395Skan} 2424117395Skan 2425117395Skanstatic __inline __m128i 2426117395Skan_mm_cmpgt_epi32 (__m128i __A, __m128i __B) 2427117395Skan{ 2428117395Skan return (__m128i)__builtin_ia32_pcmpgtd128 ((__v4si)__A, (__v4si)__B); 2429117395Skan} 2430117395Skan 2431117395Skan#define _mm_extract_epi16(__A, __B) __builtin_ia32_pextrw128 ((__v8hi)__A, __B) 2432117395Skan 2433117395Skan#define _mm_insert_epi16(__A, __B, __C) ((__m128i)__builtin_ia32_pinsrw128 ((__v8hi)__A, __B, __C)) 2434117395Skan 2435117395Skanstatic __inline __m128i 2436117395Skan_mm_max_epi16 (__m128i __A, __m128i __B) 2437117395Skan{ 2438117395Skan return (__m128i)__builtin_ia32_pmaxsw128 ((__v8hi)__A, (__v8hi)__B); 2439117395Skan} 2440117395Skan 2441117395Skanstatic __inline __m128i 2442117395Skan_mm_max_epu8 (__m128i __A, __m128i __B) 2443117395Skan{ 2444117395Skan return (__m128i)__builtin_ia32_pmaxub128 ((__v16qi)__A, (__v16qi)__B); 2445117395Skan} 2446117395Skan 2447117395Skanstatic __inline __m128i 2448117395Skan_mm_min_epi16 (__m128i __A, __m128i __B) 2449117395Skan{ 2450117395Skan return (__m128i)__builtin_ia32_pminsw128 ((__v8hi)__A, (__v8hi)__B); 2451117395Skan} 2452117395Skan 2453117395Skanstatic __inline __m128i 2454117395Skan_mm_min_epu8 (__m128i __A, __m128i __B) 2455117395Skan{ 2456117395Skan return (__m128i)__builtin_ia32_pminub128 ((__v16qi)__A, (__v16qi)__B); 2457117395Skan} 2458117395Skan 2459117395Skanstatic __inline int 2460117395Skan_mm_movemask_epi8 (__m128i __A) 2461117395Skan{ 2462117395Skan return __builtin_ia32_pmovmskb128 ((__v16qi)__A); 2463117395Skan} 2464117395Skan 2465117395Skanstatic __inline __m128i 2466117395Skan_mm_mulhi_epu16 (__m128i __A, __m128i __B) 2467117395Skan{ 2468117395Skan return (__m128i)__builtin_ia32_pmulhuw128 ((__v8hi)__A, (__v8hi)__B); 2469117395Skan} 2470117395Skan 2471117395Skan#define _mm_shufflehi_epi16(__A, __B) ((__m128i)__builtin_ia32_pshufhw ((__v8hi)__A, __B)) 2472117395Skan#define _mm_shufflelo_epi16(__A, __B) ((__m128i)__builtin_ia32_pshuflw ((__v8hi)__A, __B)) 2473117395Skan#define _mm_shuffle_epi32(__A, __B) ((__m128i)__builtin_ia32_pshufd ((__v4si)__A, __B)) 2474117395Skan 2475117395Skanstatic __inline void 2476117395Skan_mm_maskmoveu_si128 (__m128i __A, __m128i __B, char *__C) 2477117395Skan{ 2478117395Skan __builtin_ia32_maskmovdqu ((__v16qi)__A, (__v16qi)__B, __C); 2479117395Skan} 2480117395Skan 2481117395Skanstatic __inline __m128i 2482117395Skan_mm_avg_epu8 (__m128i __A, __m128i __B) 2483117395Skan{ 2484117395Skan return (__m128i)__builtin_ia32_pavgb128 ((__v16qi)__A, (__v16qi)__B); 2485117395Skan} 2486117395Skan 2487117395Skanstatic __inline __m128i 2488117395Skan_mm_avg_epu16 (__m128i __A, __m128i __B) 2489117395Skan{ 2490117395Skan return (__m128i)__builtin_ia32_pavgw128 ((__v8hi)__A, (__v8hi)__B); 2491117395Skan} 2492117395Skan 2493117395Skanstatic __inline __m128i 2494117395Skan_mm_sad_epu8 (__m128i __A, __m128i __B) 2495117395Skan{ 2496117395Skan return (__m128i)__builtin_ia32_psadbw128 ((__v16qi)__A, (__v16qi)__B); 2497117395Skan} 2498117395Skan 2499117395Skanstatic __inline void 2500117395Skan_mm_stream_si32 (int *__A, int __B) 2501117395Skan{ 2502117395Skan __builtin_ia32_movnti (__A, __B); 2503117395Skan} 2504117395Skan 2505117395Skanstatic __inline void 2506117395Skan_mm_stream_si128 (__m128i *__A, __m128i __B) 2507117395Skan{ 2508117395Skan __builtin_ia32_movntdq ((__v2di *)__A, (__v2di)__B); 2509117395Skan} 2510117395Skan 2511117395Skanstatic __inline void 2512117395Skan_mm_stream_pd (double *__A, __m128d __B) 2513117395Skan{ 2514117395Skan __builtin_ia32_movntpd (__A, (__v2df)__B); 2515117395Skan} 2516117395Skan 2517117395Skanstatic __inline __m128i 2518117395Skan_mm_movpi64_epi64 (__m64 __A) 2519117395Skan{ 2520117395Skan return (__m128i)__builtin_ia32_movq2dq ((unsigned long long)__A); 2521117395Skan} 2522117395Skan 2523117395Skanstatic __inline void 2524117395Skan_mm_clflush (void const *__A) 2525117395Skan{ 2526117395Skan return __builtin_ia32_clflush (__A); 2527117395Skan} 2528117395Skan 2529117395Skanstatic __inline void 2530117395Skan_mm_lfence (void) 2531117395Skan{ 2532117395Skan __builtin_ia32_lfence (); 2533117395Skan} 2534117395Skan 2535117395Skanstatic __inline void 2536117395Skan_mm_mfence (void) 2537117395Skan{ 2538117395Skan __builtin_ia32_mfence (); 2539117395Skan} 2540117395Skan 2541117395Skanstatic __inline __m128i 2542117395Skan_mm_cvtsi32_si128 (int __A) 2543117395Skan{ 2544117395Skan return (__m128i) __builtin_ia32_loadd (&__A); 2545117395Skan} 2546117395Skan 2547117395Skan#ifdef __x86_64__ 2548117395Skanstatic __inline __m128i 2549117395Skan_mm_cvtsi64x_si128 (long long __A) 2550117395Skan{ 2551117395Skan return (__m128i) __builtin_ia32_movq2dq (__A); 2552117395Skan} 2553117395Skan#endif 2554117395Skan 2555117395Skanstatic __inline int 2556117395Skan_mm_cvtsi128_si32 (__m128i __A) 2557117395Skan{ 2558117395Skan int __tmp; 2559117395Skan __builtin_ia32_stored (&__tmp, (__v4si)__A); 2560117395Skan return __tmp; 2561117395Skan} 2562117395Skan 2563117395Skan#ifdef __x86_64__ 2564117395Skanstatic __inline long long 2565117395Skan_mm_cvtsi128_si64x (__m128i __A) 2566117395Skan{ 2567117395Skan return __builtin_ia32_movdq2q ((__v2di)__A); 2568117395Skan} 2569117395Skan#endif 2570117395Skan 2571117395Skan#endif /* __SSE2__ */ 2572117395Skan 2573117395Skan#endif /* __SSE__ */ 257490075Sobrien#endif /* _XMMINTRIN_H_INCLUDED */ 2575