__clang_cuda_device_functions.h revision 336979
1/*===---- __clang_cuda_device_functions.h - CUDA runtime support -----------=== 2 * 3 * Permission is hereby granted, free of charge, to any person obtaining a copy 4 * of this software and associated documentation files (the "Software"), to deal 5 * in the Software without restriction, including without limitation the rights 6 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 * copies of the Software, and to permit persons to whom the Software is 8 * furnished to do so, subject to the following conditions: 9 * 10 * The above copyright notice and this permission notice shall be included in 11 * all copies or substantial portions of the Software. 12 * 13 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 * THE SOFTWARE. 20 * 21 *===-----------------------------------------------------------------------=== 22 */ 23 24#ifndef __CLANG_CUDA_DEVICE_FUNCTIONS_H__ 25#define __CLANG_CUDA_DEVICE_FUNCTIONS_H__ 26 27#if CUDA_VERSION < 9000 28#error This file is intended to be used with CUDA-9+ only. 29#endif 30 31// __DEVICE__ is a helper macro with common set of attributes for the wrappers 32// we implement in this file. We need static in order to avoid emitting unused 33// functions and __forceinline__ helps inlining these wrappers at -O1. 34#pragma push_macro("__DEVICE__") 35#define __DEVICE__ static __device__ __forceinline__ 36 37// libdevice provides fast low precision and slow full-recision implementations 38// for some functions. Which one gets selected depends on 39// __CLANG_CUDA_APPROX_TRANSCENDENTALS__ which gets defined by clang if 40// -ffast-math or -fcuda-approx-transcendentals are in effect. 41#pragma push_macro("__FAST_OR_SLOW") 42#if defined(__CLANG_CUDA_APPROX_TRANSCENDENTALS__) 43#define __FAST_OR_SLOW(fast, slow) fast 44#else 45#define __FAST_OR_SLOW(fast, slow) slow 46#endif 47 48__DEVICE__ int __all(int __a) { return __nvvm_vote_all(__a); } 49__DEVICE__ int __any(int __a) { return __nvvm_vote_any(__a); } 50__DEVICE__ unsigned int __ballot(int __a) { return __nvvm_vote_ballot(__a); } 51__DEVICE__ unsigned int __brev(unsigned int __a) { return __nv_brev(__a); } 52__DEVICE__ unsigned long long __brevll(unsigned long long __a) { 53 return __nv_brevll(__a); 54} 55__DEVICE__ void __brkpt() { asm volatile("brkpt;"); } 56__DEVICE__ void __brkpt(int __a) { __brkpt(); } 57__DEVICE__ unsigned int __byte_perm(unsigned int __a, unsigned int __b, 58 unsigned int __c) { 59 return __nv_byte_perm(__a, __b, __c); 60} 61__DEVICE__ int __clz(int __a) { return __nv_clz(__a); } 62__DEVICE__ int __clzll(long long __a) { return __nv_clzll(__a); } 63__DEVICE__ float __cosf(float __a) { return __nv_fast_cosf(__a); } 64__DEVICE__ double __dAtomicAdd(double *__p, double __v) { 65 return __nvvm_atom_add_gen_d(__p, __v); 66} 67__DEVICE__ double __dAtomicAdd_block(double *__p, double __v) { 68 return __nvvm_atom_cta_add_gen_d(__p, __v); 69} 70__DEVICE__ double __dAtomicAdd_system(double *__p, double __v) { 71 return __nvvm_atom_sys_add_gen_d(__p, __v); 72} 73__DEVICE__ double __dadd_rd(double __a, double __b) { 74 return __nv_dadd_rd(__a, __b); 75} 76__DEVICE__ double __dadd_rn(double __a, double __b) { 77 return __nv_dadd_rn(__a, __b); 78} 79__DEVICE__ double __dadd_ru(double __a, double __b) { 80 return __nv_dadd_ru(__a, __b); 81} 82__DEVICE__ double __dadd_rz(double __a, double __b) { 83 return __nv_dadd_rz(__a, __b); 84} 85__DEVICE__ double __ddiv_rd(double __a, double __b) { 86 return __nv_ddiv_rd(__a, __b); 87} 88__DEVICE__ double __ddiv_rn(double __a, double __b) { 89 return __nv_ddiv_rn(__a, __b); 90} 91__DEVICE__ double __ddiv_ru(double __a, double __b) { 92 return __nv_ddiv_ru(__a, __b); 93} 94__DEVICE__ double __ddiv_rz(double __a, double __b) { 95 return __nv_ddiv_rz(__a, __b); 96} 97__DEVICE__ double __dmul_rd(double __a, double __b) { 98 return __nv_dmul_rd(__a, __b); 99} 100__DEVICE__ double __dmul_rn(double __a, double __b) { 101 return __nv_dmul_rn(__a, __b); 102} 103__DEVICE__ double __dmul_ru(double __a, double __b) { 104 return __nv_dmul_ru(__a, __b); 105} 106__DEVICE__ double __dmul_rz(double __a, double __b) { 107 return __nv_dmul_rz(__a, __b); 108} 109__DEVICE__ float __double2float_rd(double __a) { 110 return __nv_double2float_rd(__a); 111} 112__DEVICE__ float __double2float_rn(double __a) { 113 return __nv_double2float_rn(__a); 114} 115__DEVICE__ float __double2float_ru(double __a) { 116 return __nv_double2float_ru(__a); 117} 118__DEVICE__ float __double2float_rz(double __a) { 119 return __nv_double2float_rz(__a); 120} 121__DEVICE__ int __double2hiint(double __a) { return __nv_double2hiint(__a); } 122__DEVICE__ int __double2int_rd(double __a) { return __nv_double2int_rd(__a); } 123__DEVICE__ int __double2int_rn(double __a) { return __nv_double2int_rn(__a); } 124__DEVICE__ int __double2int_ru(double __a) { return __nv_double2int_ru(__a); } 125__DEVICE__ int __double2int_rz(double __a) { return __nv_double2int_rz(__a); } 126__DEVICE__ long long __double2ll_rd(double __a) { 127 return __nv_double2ll_rd(__a); 128} 129__DEVICE__ long long __double2ll_rn(double __a) { 130 return __nv_double2ll_rn(__a); 131} 132__DEVICE__ long long __double2ll_ru(double __a) { 133 return __nv_double2ll_ru(__a); 134} 135__DEVICE__ long long __double2ll_rz(double __a) { 136 return __nv_double2ll_rz(__a); 137} 138__DEVICE__ int __double2loint(double __a) { return __nv_double2loint(__a); } 139__DEVICE__ unsigned int __double2uint_rd(double __a) { 140 return __nv_double2uint_rd(__a); 141} 142__DEVICE__ unsigned int __double2uint_rn(double __a) { 143 return __nv_double2uint_rn(__a); 144} 145__DEVICE__ unsigned int __double2uint_ru(double __a) { 146 return __nv_double2uint_ru(__a); 147} 148__DEVICE__ unsigned int __double2uint_rz(double __a) { 149 return __nv_double2uint_rz(__a); 150} 151__DEVICE__ unsigned long long __double2ull_rd(double __a) { 152 return __nv_double2ull_rd(__a); 153} 154__DEVICE__ unsigned long long __double2ull_rn(double __a) { 155 return __nv_double2ull_rn(__a); 156} 157__DEVICE__ unsigned long long __double2ull_ru(double __a) { 158 return __nv_double2ull_ru(__a); 159} 160__DEVICE__ unsigned long long __double2ull_rz(double __a) { 161 return __nv_double2ull_rz(__a); 162} 163__DEVICE__ long long __double_as_longlong(double __a) { 164 return __nv_double_as_longlong(__a); 165} 166__DEVICE__ double __drcp_rd(double __a) { return __nv_drcp_rd(__a); } 167__DEVICE__ double __drcp_rn(double __a) { return __nv_drcp_rn(__a); } 168__DEVICE__ double __drcp_ru(double __a) { return __nv_drcp_ru(__a); } 169__DEVICE__ double __drcp_rz(double __a) { return __nv_drcp_rz(__a); } 170__DEVICE__ double __dsqrt_rd(double __a) { return __nv_dsqrt_rd(__a); } 171__DEVICE__ double __dsqrt_rn(double __a) { return __nv_dsqrt_rn(__a); } 172__DEVICE__ double __dsqrt_ru(double __a) { return __nv_dsqrt_ru(__a); } 173__DEVICE__ double __dsqrt_rz(double __a) { return __nv_dsqrt_rz(__a); } 174__DEVICE__ double __dsub_rd(double __a, double __b) { 175 return __nv_dsub_rd(__a, __b); 176} 177__DEVICE__ double __dsub_rn(double __a, double __b) { 178 return __nv_dsub_rn(__a, __b); 179} 180__DEVICE__ double __dsub_ru(double __a, double __b) { 181 return __nv_dsub_ru(__a, __b); 182} 183__DEVICE__ double __dsub_rz(double __a, double __b) { 184 return __nv_dsub_rz(__a, __b); 185} 186__DEVICE__ float __exp10f(float __a) { return __nv_fast_exp10f(__a); } 187__DEVICE__ float __expf(float __a) { return __nv_fast_expf(__a); } 188__DEVICE__ float __fAtomicAdd(float *__p, float __v) { 189 return __nvvm_atom_add_gen_f(__p, __v); 190} 191__DEVICE__ float __fAtomicAdd_block(float *__p, float __v) { 192 return __nvvm_atom_cta_add_gen_f(__p, __v); 193} 194__DEVICE__ float __fAtomicAdd_system(float *__p, float __v) { 195 return __nvvm_atom_sys_add_gen_f(__p, __v); 196} 197__DEVICE__ float __fAtomicExch(float *__p, float __v) { 198 return __nv_int_as_float( 199 __nvvm_atom_xchg_gen_i((int *)__p, __nv_float_as_int(__v))); 200} 201__DEVICE__ float __fAtomicExch_block(float *__p, float __v) { 202 return __nv_int_as_float( 203 __nvvm_atom_cta_xchg_gen_i((int *)__p, __nv_float_as_int(__v))); 204} 205__DEVICE__ float __fAtomicExch_system(float *__p, float __v) { 206 return __nv_int_as_float( 207 __nvvm_atom_sys_xchg_gen_i((int *)__p, __nv_float_as_int(__v))); 208} 209__DEVICE__ float __fadd_rd(float __a, float __b) { 210 return __nv_fadd_rd(__a, __b); 211} 212__DEVICE__ float __fadd_rn(float __a, float __b) { 213 return __nv_fadd_rn(__a, __b); 214} 215__DEVICE__ float __fadd_ru(float __a, float __b) { 216 return __nv_fadd_ru(__a, __b); 217} 218__DEVICE__ float __fadd_rz(float __a, float __b) { 219 return __nv_fadd_rz(__a, __b); 220} 221__DEVICE__ float __fdiv_rd(float __a, float __b) { 222 return __nv_fdiv_rd(__a, __b); 223} 224__DEVICE__ float __fdiv_rn(float __a, float __b) { 225 return __nv_fdiv_rn(__a, __b); 226} 227__DEVICE__ float __fdiv_ru(float __a, float __b) { 228 return __nv_fdiv_ru(__a, __b); 229} 230__DEVICE__ float __fdiv_rz(float __a, float __b) { 231 return __nv_fdiv_rz(__a, __b); 232} 233__DEVICE__ float __fdividef(float __a, float __b) { 234 return __nv_fast_fdividef(__a, __b); 235} 236__DEVICE__ int __ffs(int __a) { return __nv_ffs(__a); } 237__DEVICE__ int __ffsll(long long __a) { return __nv_ffsll(__a); } 238__DEVICE__ int __finite(double __a) { return __nv_isfinited(__a); } 239__DEVICE__ int __finitef(float __a) { return __nv_finitef(__a); } 240__DEVICE__ int __float2int_rd(float __a) { return __nv_float2int_rd(__a); } 241__DEVICE__ int __float2int_rn(float __a) { return __nv_float2int_rn(__a); } 242__DEVICE__ int __float2int_ru(float __a) { return __nv_float2int_ru(__a); } 243__DEVICE__ int __float2int_rz(float __a) { return __nv_float2int_rz(__a); } 244__DEVICE__ long long __float2ll_rd(float __a) { return __nv_float2ll_rd(__a); } 245__DEVICE__ long long __float2ll_rn(float __a) { return __nv_float2ll_rn(__a); } 246__DEVICE__ long long __float2ll_ru(float __a) { return __nv_float2ll_ru(__a); } 247__DEVICE__ long long __float2ll_rz(float __a) { return __nv_float2ll_rz(__a); } 248__DEVICE__ unsigned int __float2uint_rd(float __a) { 249 return __nv_float2uint_rd(__a); 250} 251__DEVICE__ unsigned int __float2uint_rn(float __a) { 252 return __nv_float2uint_rn(__a); 253} 254__DEVICE__ unsigned int __float2uint_ru(float __a) { 255 return __nv_float2uint_ru(__a); 256} 257__DEVICE__ unsigned int __float2uint_rz(float __a) { 258 return __nv_float2uint_rz(__a); 259} 260__DEVICE__ unsigned long long __float2ull_rd(float __a) { 261 return __nv_float2ull_rd(__a); 262} 263__DEVICE__ unsigned long long __float2ull_rn(float __a) { 264 return __nv_float2ull_rn(__a); 265} 266__DEVICE__ unsigned long long __float2ull_ru(float __a) { 267 return __nv_float2ull_ru(__a); 268} 269__DEVICE__ unsigned long long __float2ull_rz(float __a) { 270 return __nv_float2ull_rz(__a); 271} 272__DEVICE__ int __float_as_int(float __a) { return __nv_float_as_int(__a); } 273__DEVICE__ unsigned int __float_as_uint(float __a) { 274 return __nv_float_as_uint(__a); 275} 276__DEVICE__ double __fma_rd(double __a, double __b, double __c) { 277 return __nv_fma_rd(__a, __b, __c); 278} 279__DEVICE__ double __fma_rn(double __a, double __b, double __c) { 280 return __nv_fma_rn(__a, __b, __c); 281} 282__DEVICE__ double __fma_ru(double __a, double __b, double __c) { 283 return __nv_fma_ru(__a, __b, __c); 284} 285__DEVICE__ double __fma_rz(double __a, double __b, double __c) { 286 return __nv_fma_rz(__a, __b, __c); 287} 288__DEVICE__ float __fmaf_ieee_rd(float __a, float __b, float __c) { 289 return __nv_fmaf_ieee_rd(__a, __b, __c); 290} 291__DEVICE__ float __fmaf_ieee_rn(float __a, float __b, float __c) { 292 return __nv_fmaf_ieee_rn(__a, __b, __c); 293} 294__DEVICE__ float __fmaf_ieee_ru(float __a, float __b, float __c) { 295 return __nv_fmaf_ieee_ru(__a, __b, __c); 296} 297__DEVICE__ float __fmaf_ieee_rz(float __a, float __b, float __c) { 298 return __nv_fmaf_ieee_rz(__a, __b, __c); 299} 300__DEVICE__ float __fmaf_rd(float __a, float __b, float __c) { 301 return __nv_fmaf_rd(__a, __b, __c); 302} 303__DEVICE__ float __fmaf_rn(float __a, float __b, float __c) { 304 return __nv_fmaf_rn(__a, __b, __c); 305} 306__DEVICE__ float __fmaf_ru(float __a, float __b, float __c) { 307 return __nv_fmaf_ru(__a, __b, __c); 308} 309__DEVICE__ float __fmaf_rz(float __a, float __b, float __c) { 310 return __nv_fmaf_rz(__a, __b, __c); 311} 312__DEVICE__ float __fmul_rd(float __a, float __b) { 313 return __nv_fmul_rd(__a, __b); 314} 315__DEVICE__ float __fmul_rn(float __a, float __b) { 316 return __nv_fmul_rn(__a, __b); 317} 318__DEVICE__ float __fmul_ru(float __a, float __b) { 319 return __nv_fmul_ru(__a, __b); 320} 321__DEVICE__ float __fmul_rz(float __a, float __b) { 322 return __nv_fmul_rz(__a, __b); 323} 324__DEVICE__ float __frcp_rd(float __a) { return __nv_frcp_rd(__a); } 325__DEVICE__ float __frcp_rn(float __a) { return __nv_frcp_rn(__a); } 326__DEVICE__ float __frcp_ru(float __a) { return __nv_frcp_ru(__a); } 327__DEVICE__ float __frcp_rz(float __a) { return __nv_frcp_rz(__a); } 328__DEVICE__ float __frsqrt_rn(float __a) { return __nv_frsqrt_rn(__a); } 329__DEVICE__ float __fsqrt_rd(float __a) { return __nv_fsqrt_rd(__a); } 330__DEVICE__ float __fsqrt_rn(float __a) { return __nv_fsqrt_rn(__a); } 331__DEVICE__ float __fsqrt_ru(float __a) { return __nv_fsqrt_ru(__a); } 332__DEVICE__ float __fsqrt_rz(float __a) { return __nv_fsqrt_rz(__a); } 333__DEVICE__ float __fsub_rd(float __a, float __b) { 334 return __nv_fsub_rd(__a, __b); 335} 336__DEVICE__ float __fsub_rn(float __a, float __b) { 337 return __nv_fsub_rn(__a, __b); 338} 339__DEVICE__ float __fsub_ru(float __a, float __b) { 340 return __nv_fsub_ru(__a, __b); 341} 342__DEVICE__ float __fsub_rz(float __a, float __b) { 343 return __nv_fsub_rz(__a, __b); 344} 345__DEVICE__ int __hadd(int __a, int __b) { return __nv_hadd(__a, __b); } 346__DEVICE__ double __hiloint2double(int __a, int __b) { 347 return __nv_hiloint2double(__a, __b); 348} 349__DEVICE__ int __iAtomicAdd(int *__p, int __v) { 350 return __nvvm_atom_add_gen_i(__p, __v); 351} 352__DEVICE__ int __iAtomicAdd_block(int *__p, int __v) { 353 __nvvm_atom_cta_add_gen_i(__p, __v); 354} 355__DEVICE__ int __iAtomicAdd_system(int *__p, int __v) { 356 __nvvm_atom_sys_add_gen_i(__p, __v); 357} 358__DEVICE__ int __iAtomicAnd(int *__p, int __v) { 359 return __nvvm_atom_and_gen_i(__p, __v); 360} 361__DEVICE__ int __iAtomicAnd_block(int *__p, int __v) { 362 return __nvvm_atom_cta_and_gen_i(__p, __v); 363} 364__DEVICE__ int __iAtomicAnd_system(int *__p, int __v) { 365 return __nvvm_atom_sys_and_gen_i(__p, __v); 366} 367__DEVICE__ int __iAtomicCAS(int *__p, int __cmp, int __v) { 368 return __nvvm_atom_cas_gen_i(__p, __cmp, __v); 369} 370__DEVICE__ int __iAtomicCAS_block(int *__p, int __cmp, int __v) { 371 return __nvvm_atom_cta_cas_gen_i(__p, __cmp, __v); 372} 373__DEVICE__ int __iAtomicCAS_system(int *__p, int __cmp, int __v) { 374 return __nvvm_atom_sys_cas_gen_i(__p, __cmp, __v); 375} 376__DEVICE__ int __iAtomicExch(int *__p, int __v) { 377 return __nvvm_atom_xchg_gen_i(__p, __v); 378} 379__DEVICE__ int __iAtomicExch_block(int *__p, int __v) { 380 return __nvvm_atom_cta_xchg_gen_i(__p, __v); 381} 382__DEVICE__ int __iAtomicExch_system(int *__p, int __v) { 383 return __nvvm_atom_sys_xchg_gen_i(__p, __v); 384} 385__DEVICE__ int __iAtomicMax(int *__p, int __v) { 386 return __nvvm_atom_max_gen_i(__p, __v); 387} 388__DEVICE__ int __iAtomicMax_block(int *__p, int __v) { 389 return __nvvm_atom_cta_max_gen_i(__p, __v); 390} 391__DEVICE__ int __iAtomicMax_system(int *__p, int __v) { 392 return __nvvm_atom_sys_max_gen_i(__p, __v); 393} 394__DEVICE__ int __iAtomicMin(int *__p, int __v) { 395 return __nvvm_atom_min_gen_i(__p, __v); 396} 397__DEVICE__ int __iAtomicMin_block(int *__p, int __v) { 398 return __nvvm_atom_cta_min_gen_i(__p, __v); 399} 400__DEVICE__ int __iAtomicMin_system(int *__p, int __v) { 401 return __nvvm_atom_sys_min_gen_i(__p, __v); 402} 403__DEVICE__ int __iAtomicOr(int *__p, int __v) { 404 return __nvvm_atom_or_gen_i(__p, __v); 405} 406__DEVICE__ int __iAtomicOr_block(int *__p, int __v) { 407 return __nvvm_atom_cta_or_gen_i(__p, __v); 408} 409__DEVICE__ int __iAtomicOr_system(int *__p, int __v) { 410 return __nvvm_atom_sys_or_gen_i(__p, __v); 411} 412__DEVICE__ int __iAtomicXor(int *__p, int __v) { 413 return __nvvm_atom_xor_gen_i(__p, __v); 414} 415__DEVICE__ int __iAtomicXor_block(int *__p, int __v) { 416 return __nvvm_atom_cta_xor_gen_i(__p, __v); 417} 418__DEVICE__ int __iAtomicXor_system(int *__p, int __v) { 419 return __nvvm_atom_sys_xor_gen_i(__p, __v); 420} 421__DEVICE__ long long __illAtomicMax(long long *__p, long long __v) { 422 return __nvvm_atom_max_gen_ll(__p, __v); 423} 424__DEVICE__ long long __illAtomicMax_block(long long *__p, long long __v) { 425 return __nvvm_atom_cta_max_gen_ll(__p, __v); 426} 427__DEVICE__ long long __illAtomicMax_system(long long *__p, long long __v) { 428 return __nvvm_atom_sys_max_gen_ll(__p, __v); 429} 430__DEVICE__ long long __illAtomicMin(long long *__p, long long __v) { 431 return __nvvm_atom_min_gen_ll(__p, __v); 432} 433__DEVICE__ long long __illAtomicMin_block(long long *__p, long long __v) { 434 return __nvvm_atom_cta_min_gen_ll(__p, __v); 435} 436__DEVICE__ long long __illAtomicMin_system(long long *__p, long long __v) { 437 return __nvvm_atom_sys_min_gen_ll(__p, __v); 438} 439__DEVICE__ double __int2double_rn(int __a) { return __nv_int2double_rn(__a); } 440__DEVICE__ float __int2float_rd(int __a) { return __nv_int2float_rd(__a); } 441__DEVICE__ float __int2float_rn(int __a) { return __nv_int2float_rn(__a); } 442__DEVICE__ float __int2float_ru(int __a) { return __nv_int2float_ru(__a); } 443__DEVICE__ float __int2float_rz(int __a) { return __nv_int2float_rz(__a); } 444__DEVICE__ float __int_as_float(int __a) { return __nv_int_as_float(__a); } 445__DEVICE__ int __isfinited(double __a) { return __nv_isfinited(__a); } 446__DEVICE__ int __isinf(double __a) { return __nv_isinfd(__a); } 447__DEVICE__ int __isinff(float __a) { return __nv_isinff(__a); } 448__DEVICE__ int __isnan(double __a) { return __nv_isnand(__a); } 449__DEVICE__ int __isnanf(float __a) { return __nv_isnanf(__a); } 450__DEVICE__ double __ll2double_rd(long long __a) { 451 return __nv_ll2double_rd(__a); 452} 453__DEVICE__ double __ll2double_rn(long long __a) { 454 return __nv_ll2double_rn(__a); 455} 456__DEVICE__ double __ll2double_ru(long long __a) { 457 return __nv_ll2double_ru(__a); 458} 459__DEVICE__ double __ll2double_rz(long long __a) { 460 return __nv_ll2double_rz(__a); 461} 462__DEVICE__ float __ll2float_rd(long long __a) { return __nv_ll2float_rd(__a); } 463__DEVICE__ float __ll2float_rn(long long __a) { return __nv_ll2float_rn(__a); } 464__DEVICE__ float __ll2float_ru(long long __a) { return __nv_ll2float_ru(__a); } 465__DEVICE__ float __ll2float_rz(long long __a) { return __nv_ll2float_rz(__a); } 466__DEVICE__ long long __llAtomicAnd(long long *__p, long long __v) { 467 return __nvvm_atom_and_gen_ll(__p, __v); 468} 469__DEVICE__ long long __llAtomicAnd_block(long long *__p, long long __v) { 470 return __nvvm_atom_cta_and_gen_ll(__p, __v); 471} 472__DEVICE__ long long __llAtomicAnd_system(long long *__p, long long __v) { 473 return __nvvm_atom_sys_and_gen_ll(__p, __v); 474} 475__DEVICE__ long long __llAtomicOr(long long *__p, long long __v) { 476 return __nvvm_atom_or_gen_ll(__p, __v); 477} 478__DEVICE__ long long __llAtomicOr_block(long long *__p, long long __v) { 479 return __nvvm_atom_cta_or_gen_ll(__p, __v); 480} 481__DEVICE__ long long __llAtomicOr_system(long long *__p, long long __v) { 482 return __nvvm_atom_sys_or_gen_ll(__p, __v); 483} 484__DEVICE__ long long __llAtomicXor(long long *__p, long long __v) { 485 return __nvvm_atom_xor_gen_ll(__p, __v); 486} 487__DEVICE__ long long __llAtomicXor_block(long long *__p, long long __v) { 488 return __nvvm_atom_cta_xor_gen_ll(__p, __v); 489} 490__DEVICE__ long long __llAtomicXor_system(long long *__p, long long __v) { 491 return __nvvm_atom_sys_xor_gen_ll(__p, __v); 492} 493__DEVICE__ float __log10f(float __a) { return __nv_fast_log10f(__a); } 494__DEVICE__ float __log2f(float __a) { return __nv_fast_log2f(__a); } 495__DEVICE__ float __logf(float __a) { return __nv_fast_logf(__a); } 496__DEVICE__ double __longlong_as_double(long long __a) { 497 return __nv_longlong_as_double(__a); 498} 499__DEVICE__ int __mul24(int __a, int __b) { return __nv_mul24(__a, __b); } 500__DEVICE__ long long __mul64hi(long long __a, long long __b) { 501 return __nv_mul64hi(__a, __b); 502} 503__DEVICE__ int __mulhi(int __a, int __b) { return __nv_mulhi(__a, __b); } 504__DEVICE__ unsigned int __pm0(void) { return __nvvm_read_ptx_sreg_pm0(); } 505__DEVICE__ unsigned int __pm1(void) { return __nvvm_read_ptx_sreg_pm1(); } 506__DEVICE__ unsigned int __pm2(void) { return __nvvm_read_ptx_sreg_pm2(); } 507__DEVICE__ unsigned int __pm3(void) { return __nvvm_read_ptx_sreg_pm3(); } 508__DEVICE__ int __popc(int __a) { return __nv_popc(__a); } 509__DEVICE__ int __popcll(long long __a) { return __nv_popcll(__a); } 510__DEVICE__ float __powf(float __a, float __b) { 511 return __nv_fast_powf(__a, __b); 512} 513 514// Parameter must have a known integer value. 515#define __prof_trigger(__a) asm __volatile__("pmevent \t%0;" ::"i"(__a)) 516__DEVICE__ int __rhadd(int __a, int __b) { return __nv_rhadd(__a, __b); } 517__DEVICE__ unsigned int __sad(int __a, int __b, unsigned int __c) { 518 return __nv_sad(__a, __b, __c); 519} 520__DEVICE__ float __saturatef(float __a) { return __nv_saturatef(__a); } 521__DEVICE__ int __signbitd(double __a) { return __nv_signbitd(__a); } 522__DEVICE__ int __signbitf(float __a) { return __nv_signbitf(__a); } 523__DEVICE__ void __sincosf(float __a, float *__sptr, float *__cptr) { 524 return __nv_fast_sincosf(__a, __sptr, __cptr); 525} 526__DEVICE__ float __sinf(float __a) { return __nv_fast_sinf(__a); } 527__DEVICE__ int __syncthreads_and(int __a) { return __nvvm_bar0_and(__a); } 528__DEVICE__ int __syncthreads_count(int __a) { return __nvvm_bar0_popc(__a); } 529__DEVICE__ int __syncthreads_or(int __a) { return __nvvm_bar0_or(__a); } 530__DEVICE__ float __tanf(float __a) { return __nv_fast_tanf(__a); } 531__DEVICE__ void __threadfence(void) { __nvvm_membar_gl(); } 532__DEVICE__ void __threadfence_block(void) { __nvvm_membar_cta(); }; 533__DEVICE__ void __threadfence_system(void) { __nvvm_membar_sys(); }; 534__DEVICE__ void __trap(void) { asm volatile("trap;"); } 535__DEVICE__ unsigned int __uAtomicAdd(unsigned int *__p, unsigned int __v) { 536 return __nvvm_atom_add_gen_i((int *)__p, __v); 537} 538__DEVICE__ unsigned int __uAtomicAdd_block(unsigned int *__p, 539 unsigned int __v) { 540 return __nvvm_atom_cta_add_gen_i((int *)__p, __v); 541} 542__DEVICE__ unsigned int __uAtomicAdd_system(unsigned int *__p, 543 unsigned int __v) { 544 return __nvvm_atom_sys_add_gen_i((int *)__p, __v); 545} 546__DEVICE__ unsigned int __uAtomicAnd(unsigned int *__p, unsigned int __v) { 547 return __nvvm_atom_and_gen_i((int *)__p, __v); 548} 549__DEVICE__ unsigned int __uAtomicAnd_block(unsigned int *__p, 550 unsigned int __v) { 551 return __nvvm_atom_cta_and_gen_i((int *)__p, __v); 552} 553__DEVICE__ unsigned int __uAtomicAnd_system(unsigned int *__p, 554 unsigned int __v) { 555 return __nvvm_atom_sys_and_gen_i((int *)__p, __v); 556} 557__DEVICE__ unsigned int __uAtomicCAS(unsigned int *__p, unsigned int __cmp, 558 unsigned int __v) { 559 return __nvvm_atom_cas_gen_i((int *)__p, __cmp, __v); 560} 561__DEVICE__ unsigned int 562__uAtomicCAS_block(unsigned int *__p, unsigned int __cmp, unsigned int __v) { 563 return __nvvm_atom_cta_cas_gen_i((int *)__p, __cmp, __v); 564} 565__DEVICE__ unsigned int 566__uAtomicCAS_system(unsigned int *__p, unsigned int __cmp, unsigned int __v) { 567 return __nvvm_atom_sys_cas_gen_i((int *)__p, __cmp, __v); 568} 569__DEVICE__ unsigned int __uAtomicDec(unsigned int *__p, unsigned int __v) { 570 return __nvvm_atom_dec_gen_ui(__p, __v); 571} 572__DEVICE__ unsigned int __uAtomicDec_block(unsigned int *__p, 573 unsigned int __v) { 574 return __nvvm_atom_cta_dec_gen_ui(__p, __v); 575} 576__DEVICE__ unsigned int __uAtomicDec_system(unsigned int *__p, 577 unsigned int __v) { 578 return __nvvm_atom_sys_dec_gen_ui(__p, __v); 579} 580__DEVICE__ unsigned int __uAtomicExch(unsigned int *__p, unsigned int __v) { 581 return __nvvm_atom_xchg_gen_i((int *)__p, __v); 582} 583__DEVICE__ unsigned int __uAtomicExch_block(unsigned int *__p, 584 unsigned int __v) { 585 return __nvvm_atom_cta_xchg_gen_i((int *)__p, __v); 586} 587__DEVICE__ unsigned int __uAtomicExch_system(unsigned int *__p, 588 unsigned int __v) { 589 return __nvvm_atom_sys_xchg_gen_i((int *)__p, __v); 590} 591__DEVICE__ unsigned int __uAtomicInc(unsigned int *__p, unsigned int __v) { 592 return __nvvm_atom_inc_gen_ui(__p, __v); 593} 594__DEVICE__ unsigned int __uAtomicInc_block(unsigned int *__p, 595 unsigned int __v) { 596 return __nvvm_atom_cta_inc_gen_ui(__p, __v); 597} 598__DEVICE__ unsigned int __uAtomicInc_system(unsigned int *__p, 599 unsigned int __v) { 600 return __nvvm_atom_sys_inc_gen_ui(__p, __v); 601} 602__DEVICE__ unsigned int __uAtomicMax(unsigned int *__p, unsigned int __v) { 603 return __nvvm_atom_max_gen_ui(__p, __v); 604} 605__DEVICE__ unsigned int __uAtomicMax_block(unsigned int *__p, 606 unsigned int __v) { 607 return __nvvm_atom_cta_max_gen_ui(__p, __v); 608} 609__DEVICE__ unsigned int __uAtomicMax_system(unsigned int *__p, 610 unsigned int __v) { 611 return __nvvm_atom_sys_max_gen_ui(__p, __v); 612} 613__DEVICE__ unsigned int __uAtomicMin(unsigned int *__p, unsigned int __v) { 614 return __nvvm_atom_min_gen_ui(__p, __v); 615} 616__DEVICE__ unsigned int __uAtomicMin_block(unsigned int *__p, 617 unsigned int __v) { 618 return __nvvm_atom_cta_min_gen_ui(__p, __v); 619} 620__DEVICE__ unsigned int __uAtomicMin_system(unsigned int *__p, 621 unsigned int __v) { 622 return __nvvm_atom_sys_min_gen_ui(__p, __v); 623} 624__DEVICE__ unsigned int __uAtomicOr(unsigned int *__p, unsigned int __v) { 625 return __nvvm_atom_or_gen_i((int *)__p, __v); 626} 627__DEVICE__ unsigned int __uAtomicOr_block(unsigned int *__p, unsigned int __v) { 628 return __nvvm_atom_cta_or_gen_i((int *)__p, __v); 629} 630__DEVICE__ unsigned int __uAtomicOr_system(unsigned int *__p, 631 unsigned int __v) { 632 return __nvvm_atom_sys_or_gen_i((int *)__p, __v); 633} 634__DEVICE__ unsigned int __uAtomicXor(unsigned int *__p, unsigned int __v) { 635 return __nvvm_atom_xor_gen_i((int *)__p, __v); 636} 637__DEVICE__ unsigned int __uAtomicXor_block(unsigned int *__p, 638 unsigned int __v) { 639 return __nvvm_atom_cta_xor_gen_i((int *)__p, __v); 640} 641__DEVICE__ unsigned int __uAtomicXor_system(unsigned int *__p, 642 unsigned int __v) { 643 return __nvvm_atom_sys_xor_gen_i((int *)__p, __v); 644} 645__DEVICE__ unsigned int __uhadd(unsigned int __a, unsigned int __b) { 646 return __nv_uhadd(__a, __b); 647} 648__DEVICE__ double __uint2double_rn(unsigned int __a) { 649 return __nv_uint2double_rn(__a); 650} 651__DEVICE__ float __uint2float_rd(unsigned int __a) { 652 return __nv_uint2float_rd(__a); 653} 654__DEVICE__ float __uint2float_rn(unsigned int __a) { 655 return __nv_uint2float_rn(__a); 656} 657__DEVICE__ float __uint2float_ru(unsigned int __a) { 658 return __nv_uint2float_ru(__a); 659} 660__DEVICE__ float __uint2float_rz(unsigned int __a) { 661 return __nv_uint2float_rz(__a); 662} 663__DEVICE__ float __uint_as_float(unsigned int __a) { 664 return __nv_uint_as_float(__a); 665} // 666__DEVICE__ double __ull2double_rd(unsigned long long __a) { 667 return __nv_ull2double_rd(__a); 668} 669__DEVICE__ double __ull2double_rn(unsigned long long __a) { 670 return __nv_ull2double_rn(__a); 671} 672__DEVICE__ double __ull2double_ru(unsigned long long __a) { 673 return __nv_ull2double_ru(__a); 674} 675__DEVICE__ double __ull2double_rz(unsigned long long __a) { 676 return __nv_ull2double_rz(__a); 677} 678__DEVICE__ float __ull2float_rd(unsigned long long __a) { 679 return __nv_ull2float_rd(__a); 680} 681__DEVICE__ float __ull2float_rn(unsigned long long __a) { 682 return __nv_ull2float_rn(__a); 683} 684__DEVICE__ float __ull2float_ru(unsigned long long __a) { 685 return __nv_ull2float_ru(__a); 686} 687__DEVICE__ float __ull2float_rz(unsigned long long __a) { 688 return __nv_ull2float_rz(__a); 689} 690__DEVICE__ unsigned long long __ullAtomicAdd(unsigned long long *__p, 691 unsigned long long __v) { 692 return __nvvm_atom_add_gen_ll((long long *)__p, __v); 693} 694__DEVICE__ unsigned long long __ullAtomicAdd_block(unsigned long long *__p, 695 unsigned long long __v) { 696 return __nvvm_atom_cta_add_gen_ll((long long *)__p, __v); 697} 698__DEVICE__ unsigned long long __ullAtomicAdd_system(unsigned long long *__p, 699 unsigned long long __v) { 700 return __nvvm_atom_sys_add_gen_ll((long long *)__p, __v); 701} 702__DEVICE__ unsigned long long __ullAtomicAnd(unsigned long long *__p, 703 unsigned long long __v) { 704 return __nvvm_atom_and_gen_ll((long long *)__p, __v); 705} 706__DEVICE__ unsigned long long __ullAtomicAnd_block(unsigned long long *__p, 707 unsigned long long __v) { 708 return __nvvm_atom_cta_and_gen_ll((long long *)__p, __v); 709} 710__DEVICE__ unsigned long long __ullAtomicAnd_system(unsigned long long *__p, 711 unsigned long long __v) { 712 return __nvvm_atom_sys_and_gen_ll((long long *)__p, __v); 713} 714__DEVICE__ unsigned long long __ullAtomicCAS(unsigned long long *__p, 715 unsigned long long __cmp, 716 unsigned long long __v) { 717 return __nvvm_atom_cas_gen_ll((long long *)__p, __cmp, __v); 718} 719__DEVICE__ unsigned long long __ullAtomicCAS_block(unsigned long long *__p, 720 unsigned long long __cmp, 721 unsigned long long __v) { 722 return __nvvm_atom_cta_cas_gen_ll((long long *)__p, __cmp, __v); 723} 724__DEVICE__ unsigned long long __ullAtomicCAS_system(unsigned long long *__p, 725 unsigned long long __cmp, 726 unsigned long long __v) { 727 return __nvvm_atom_sys_cas_gen_ll((long long *)__p, __cmp, __v); 728} 729__DEVICE__ unsigned long long __ullAtomicExch(unsigned long long *__p, 730 unsigned long long __v) { 731 return __nvvm_atom_xchg_gen_ll((long long *)__p, __v); 732} 733__DEVICE__ unsigned long long __ullAtomicExch_block(unsigned long long *__p, 734 unsigned long long __v) { 735 return __nvvm_atom_cta_xchg_gen_ll((long long *)__p, __v); 736} 737__DEVICE__ unsigned long long __ullAtomicExch_system(unsigned long long *__p, 738 unsigned long long __v) { 739 return __nvvm_atom_sys_xchg_gen_ll((long long *)__p, __v); 740} 741__DEVICE__ unsigned long long __ullAtomicMax(unsigned long long *__p, 742 unsigned long long __v) { 743 return __nvvm_atom_max_gen_ull(__p, __v); 744} 745__DEVICE__ unsigned long long __ullAtomicMax_block(unsigned long long *__p, 746 unsigned long long __v) { 747 return __nvvm_atom_cta_max_gen_ull(__p, __v); 748} 749__DEVICE__ unsigned long long __ullAtomicMax_system(unsigned long long *__p, 750 unsigned long long __v) { 751 return __nvvm_atom_sys_max_gen_ull(__p, __v); 752} 753__DEVICE__ unsigned long long __ullAtomicMin(unsigned long long *__p, 754 unsigned long long __v) { 755 return __nvvm_atom_min_gen_ull(__p, __v); 756} 757__DEVICE__ unsigned long long __ullAtomicMin_block(unsigned long long *__p, 758 unsigned long long __v) { 759 return __nvvm_atom_cta_min_gen_ull(__p, __v); 760} 761__DEVICE__ unsigned long long __ullAtomicMin_system(unsigned long long *__p, 762 unsigned long long __v) { 763 return __nvvm_atom_sys_min_gen_ull(__p, __v); 764} 765__DEVICE__ unsigned long long __ullAtomicOr(unsigned long long *__p, 766 unsigned long long __v) { 767 return __nvvm_atom_or_gen_ll((long long *)__p, __v); 768} 769__DEVICE__ unsigned long long __ullAtomicOr_block(unsigned long long *__p, 770 unsigned long long __v) { 771 return __nvvm_atom_cta_or_gen_ll((long long *)__p, __v); 772} 773__DEVICE__ unsigned long long __ullAtomicOr_system(unsigned long long *__p, 774 unsigned long long __v) { 775 return __nvvm_atom_sys_or_gen_ll((long long *)__p, __v); 776} 777__DEVICE__ unsigned long long __ullAtomicXor(unsigned long long *__p, 778 unsigned long long __v) { 779 return __nvvm_atom_xor_gen_ll((long long *)__p, __v); 780} 781__DEVICE__ unsigned long long __ullAtomicXor_block(unsigned long long *__p, 782 unsigned long long __v) { 783 return __nvvm_atom_cta_xor_gen_ll((long long *)__p, __v); 784} 785__DEVICE__ unsigned long long __ullAtomicXor_system(unsigned long long *__p, 786 unsigned long long __v) { 787 return __nvvm_atom_sys_xor_gen_ll((long long *)__p, __v); 788} 789__DEVICE__ unsigned int __umul24(unsigned int __a, unsigned int __b) { 790 return __nv_umul24(__a, __b); 791} 792__DEVICE__ unsigned long long __umul64hi(unsigned long long __a, 793 unsigned long long __b) { 794 return __nv_umul64hi(__a, __b); 795} 796__DEVICE__ unsigned int __umulhi(unsigned int __a, unsigned int __b) { 797 return __nv_umulhi(__a, __b); 798} 799__DEVICE__ unsigned int __urhadd(unsigned int __a, unsigned int __b) { 800 return __nv_urhadd(__a, __b); 801} 802__DEVICE__ unsigned int __usad(unsigned int __a, unsigned int __b, 803 unsigned int __c) { 804 return __nv_usad(__a, __b, __c); 805} 806 807#if CUDA_VERSION >= 9000 && CUDA_VERSION < 9020 808__DEVICE__ unsigned int __vabs2(unsigned int __a) { return __nv_vabs2(__a); } 809__DEVICE__ unsigned int __vabs4(unsigned int __a) { return __nv_vabs4(__a); } 810__DEVICE__ unsigned int __vabsdiffs2(unsigned int __a, unsigned int __b) { 811 return __nv_vabsdiffs2(__a, __b); 812} 813__DEVICE__ unsigned int __vabsdiffs4(unsigned int __a, unsigned int __b) { 814 return __nv_vabsdiffs4(__a, __b); 815} 816__DEVICE__ unsigned int __vabsdiffu2(unsigned int __a, unsigned int __b) { 817 return __nv_vabsdiffu2(__a, __b); 818} 819__DEVICE__ unsigned int __vabsdiffu4(unsigned int __a, unsigned int __b) { 820 return __nv_vabsdiffu4(__a, __b); 821} 822__DEVICE__ unsigned int __vabsss2(unsigned int __a) { 823 return __nv_vabsss2(__a); 824} 825__DEVICE__ unsigned int __vabsss4(unsigned int __a) { 826 return __nv_vabsss4(__a); 827} 828__DEVICE__ unsigned int __vadd2(unsigned int __a, unsigned int __b) { 829 return __nv_vadd2(__a, __b); 830} 831__DEVICE__ unsigned int __vadd4(unsigned int __a, unsigned int __b) { 832 return __nv_vadd4(__a, __b); 833} 834__DEVICE__ unsigned int __vaddss2(unsigned int __a, unsigned int __b) { 835 return __nv_vaddss2(__a, __b); 836} 837__DEVICE__ unsigned int __vaddss4(unsigned int __a, unsigned int __b) { 838 return __nv_vaddss4(__a, __b); 839} 840__DEVICE__ unsigned int __vaddus2(unsigned int __a, unsigned int __b) { 841 return __nv_vaddus2(__a, __b); 842} 843__DEVICE__ unsigned int __vaddus4(unsigned int __a, unsigned int __b) { 844 return __nv_vaddus4(__a, __b); 845} 846__DEVICE__ unsigned int __vavgs2(unsigned int __a, unsigned int __b) { 847 return __nv_vavgs2(__a, __b); 848} 849__DEVICE__ unsigned int __vavgs4(unsigned int __a, unsigned int __b) { 850 return __nv_vavgs4(__a, __b); 851} 852__DEVICE__ unsigned int __vavgu2(unsigned int __a, unsigned int __b) { 853 return __nv_vavgu2(__a, __b); 854} 855__DEVICE__ unsigned int __vavgu4(unsigned int __a, unsigned int __b) { 856 return __nv_vavgu4(__a, __b); 857} 858__DEVICE__ unsigned int __vcmpeq2(unsigned int __a, unsigned int __b) { 859 return __nv_vcmpeq2(__a, __b); 860} 861__DEVICE__ unsigned int __vcmpeq4(unsigned int __a, unsigned int __b) { 862 return __nv_vcmpeq4(__a, __b); 863} 864__DEVICE__ unsigned int __vcmpges2(unsigned int __a, unsigned int __b) { 865 return __nv_vcmpges2(__a, __b); 866} 867__DEVICE__ unsigned int __vcmpges4(unsigned int __a, unsigned int __b) { 868 return __nv_vcmpges4(__a, __b); 869} 870__DEVICE__ unsigned int __vcmpgeu2(unsigned int __a, unsigned int __b) { 871 return __nv_vcmpgeu2(__a, __b); 872} 873__DEVICE__ unsigned int __vcmpgeu4(unsigned int __a, unsigned int __b) { 874 return __nv_vcmpgeu4(__a, __b); 875} 876__DEVICE__ unsigned int __vcmpgts2(unsigned int __a, unsigned int __b) { 877 return __nv_vcmpgts2(__a, __b); 878} 879__DEVICE__ unsigned int __vcmpgts4(unsigned int __a, unsigned int __b) { 880 return __nv_vcmpgts4(__a, __b); 881} 882__DEVICE__ unsigned int __vcmpgtu2(unsigned int __a, unsigned int __b) { 883 return __nv_vcmpgtu2(__a, __b); 884} 885__DEVICE__ unsigned int __vcmpgtu4(unsigned int __a, unsigned int __b) { 886 return __nv_vcmpgtu4(__a, __b); 887} 888__DEVICE__ unsigned int __vcmples2(unsigned int __a, unsigned int __b) { 889 return __nv_vcmples2(__a, __b); 890} 891__DEVICE__ unsigned int __vcmples4(unsigned int __a, unsigned int __b) { 892 return __nv_vcmples4(__a, __b); 893} 894__DEVICE__ unsigned int __vcmpleu2(unsigned int __a, unsigned int __b) { 895 return __nv_vcmpleu2(__a, __b); 896} 897__DEVICE__ unsigned int __vcmpleu4(unsigned int __a, unsigned int __b) { 898 return __nv_vcmpleu4(__a, __b); 899} 900__DEVICE__ unsigned int __vcmplts2(unsigned int __a, unsigned int __b) { 901 return __nv_vcmplts2(__a, __b); 902} 903__DEVICE__ unsigned int __vcmplts4(unsigned int __a, unsigned int __b) { 904 return __nv_vcmplts4(__a, __b); 905} 906__DEVICE__ unsigned int __vcmpltu2(unsigned int __a, unsigned int __b) { 907 return __nv_vcmpltu2(__a, __b); 908} 909__DEVICE__ unsigned int __vcmpltu4(unsigned int __a, unsigned int __b) { 910 return __nv_vcmpltu4(__a, __b); 911} 912__DEVICE__ unsigned int __vcmpne2(unsigned int __a, unsigned int __b) { 913 return __nv_vcmpne2(__a, __b); 914} 915__DEVICE__ unsigned int __vcmpne4(unsigned int __a, unsigned int __b) { 916 return __nv_vcmpne4(__a, __b); 917} 918__DEVICE__ unsigned int __vhaddu2(unsigned int __a, unsigned int __b) { 919 return __nv_vhaddu2(__a, __b); 920} 921__DEVICE__ unsigned int __vhaddu4(unsigned int __a, unsigned int __b) { 922 return __nv_vhaddu4(__a, __b); 923} 924__DEVICE__ unsigned int __vmaxs2(unsigned int __a, unsigned int __b) { 925 return __nv_vmaxs2(__a, __b); 926} 927__DEVICE__ unsigned int __vmaxs4(unsigned int __a, unsigned int __b) { 928 return __nv_vmaxs4(__a, __b); 929} 930__DEVICE__ unsigned int __vmaxu2(unsigned int __a, unsigned int __b) { 931 return __nv_vmaxu2(__a, __b); 932} 933__DEVICE__ unsigned int __vmaxu4(unsigned int __a, unsigned int __b) { 934 return __nv_vmaxu4(__a, __b); 935} 936__DEVICE__ unsigned int __vmins2(unsigned int __a, unsigned int __b) { 937 return __nv_vmins2(__a, __b); 938} 939__DEVICE__ unsigned int __vmins4(unsigned int __a, unsigned int __b) { 940 return __nv_vmins4(__a, __b); 941} 942__DEVICE__ unsigned int __vminu2(unsigned int __a, unsigned int __b) { 943 return __nv_vminu2(__a, __b); 944} 945__DEVICE__ unsigned int __vminu4(unsigned int __a, unsigned int __b) { 946 return __nv_vminu4(__a, __b); 947} 948__DEVICE__ unsigned int __vneg2(unsigned int __a) { return __nv_vneg2(__a); } 949__DEVICE__ unsigned int __vneg4(unsigned int __a) { return __nv_vneg4(__a); } 950__DEVICE__ unsigned int __vnegss2(unsigned int __a) { 951 return __nv_vnegss2(__a); 952} 953__DEVICE__ unsigned int __vnegss4(unsigned int __a) { 954 return __nv_vnegss4(__a); 955} 956__DEVICE__ unsigned int __vsads2(unsigned int __a, unsigned int __b) { 957 return __nv_vsads2(__a, __b); 958} 959__DEVICE__ unsigned int __vsads4(unsigned int __a, unsigned int __b) { 960 return __nv_vsads4(__a, __b); 961} 962__DEVICE__ unsigned int __vsadu2(unsigned int __a, unsigned int __b) { 963 return __nv_vsadu2(__a, __b); 964} 965__DEVICE__ unsigned int __vsadu4(unsigned int __a, unsigned int __b) { 966 return __nv_vsadu4(__a, __b); 967} 968__DEVICE__ unsigned int __vseteq2(unsigned int __a, unsigned int __b) { 969 return __nv_vseteq2(__a, __b); 970} 971__DEVICE__ unsigned int __vseteq4(unsigned int __a, unsigned int __b) { 972 return __nv_vseteq4(__a, __b); 973} 974__DEVICE__ unsigned int __vsetges2(unsigned int __a, unsigned int __b) { 975 return __nv_vsetges2(__a, __b); 976} 977__DEVICE__ unsigned int __vsetges4(unsigned int __a, unsigned int __b) { 978 return __nv_vsetges4(__a, __b); 979} 980__DEVICE__ unsigned int __vsetgeu2(unsigned int __a, unsigned int __b) { 981 return __nv_vsetgeu2(__a, __b); 982} 983__DEVICE__ unsigned int __vsetgeu4(unsigned int __a, unsigned int __b) { 984 return __nv_vsetgeu4(__a, __b); 985} 986__DEVICE__ unsigned int __vsetgts2(unsigned int __a, unsigned int __b) { 987 return __nv_vsetgts2(__a, __b); 988} 989__DEVICE__ unsigned int __vsetgts4(unsigned int __a, unsigned int __b) { 990 return __nv_vsetgts4(__a, __b); 991} 992__DEVICE__ unsigned int __vsetgtu2(unsigned int __a, unsigned int __b) { 993 return __nv_vsetgtu2(__a, __b); 994} 995__DEVICE__ unsigned int __vsetgtu4(unsigned int __a, unsigned int __b) { 996 return __nv_vsetgtu4(__a, __b); 997} 998__DEVICE__ unsigned int __vsetles2(unsigned int __a, unsigned int __b) { 999 return __nv_vsetles2(__a, __b); 1000} 1001__DEVICE__ unsigned int __vsetles4(unsigned int __a, unsigned int __b) { 1002 return __nv_vsetles4(__a, __b); 1003} 1004__DEVICE__ unsigned int __vsetleu2(unsigned int __a, unsigned int __b) { 1005 return __nv_vsetleu2(__a, __b); 1006} 1007__DEVICE__ unsigned int __vsetleu4(unsigned int __a, unsigned int __b) { 1008 return __nv_vsetleu4(__a, __b); 1009} 1010__DEVICE__ unsigned int __vsetlts2(unsigned int __a, unsigned int __b) { 1011 return __nv_vsetlts2(__a, __b); 1012} 1013__DEVICE__ unsigned int __vsetlts4(unsigned int __a, unsigned int __b) { 1014 return __nv_vsetlts4(__a, __b); 1015} 1016__DEVICE__ unsigned int __vsetltu2(unsigned int __a, unsigned int __b) { 1017 return __nv_vsetltu2(__a, __b); 1018} 1019__DEVICE__ unsigned int __vsetltu4(unsigned int __a, unsigned int __b) { 1020 return __nv_vsetltu4(__a, __b); 1021} 1022__DEVICE__ unsigned int __vsetne2(unsigned int __a, unsigned int __b) { 1023 return __nv_vsetne2(__a, __b); 1024} 1025__DEVICE__ unsigned int __vsetne4(unsigned int __a, unsigned int __b) { 1026 return __nv_vsetne4(__a, __b); 1027} 1028__DEVICE__ unsigned int __vsub2(unsigned int __a, unsigned int __b) { 1029 return __nv_vsub2(__a, __b); 1030} 1031__DEVICE__ unsigned int __vsub4(unsigned int __a, unsigned int __b) { 1032 return __nv_vsub4(__a, __b); 1033} 1034__DEVICE__ unsigned int __vsubss2(unsigned int __a, unsigned int __b) { 1035 return __nv_vsubss2(__a, __b); 1036} 1037__DEVICE__ unsigned int __vsubss4(unsigned int __a, unsigned int __b) { 1038 return __nv_vsubss4(__a, __b); 1039} 1040__DEVICE__ unsigned int __vsubus2(unsigned int __a, unsigned int __b) { 1041 return __nv_vsubus2(__a, __b); 1042} 1043__DEVICE__ unsigned int __vsubus4(unsigned int __a, unsigned int __b) { 1044 return __nv_vsubus4(__a, __b); 1045} 1046#else // CUDA_VERSION >= 9020 1047// CUDA no longer provides inline assembly (or bitcode) implementation of these 1048// functions, so we have to reimplment them. The implementation is naive and is 1049// not optimized for performance. 1050 1051// Helper function to convert N-bit boolean subfields into all-0 or all-1. 1052// E.g. __bool2mask(0x01000100,8) -> 0xff00ff00 1053// __bool2mask(0x00010000,16) -> 0xffff0000 1054__DEVICE__ unsigned int __bool2mask(unsigned int __a, int shift) { 1055 return (__a << shift) - __a; 1056} 1057__DEVICE__ unsigned int __vabs2(unsigned int __a) { 1058 unsigned int r; 1059 asm("vabsdiff2.s32.s32.s32 %0,%1,%2,%3;" 1060 : "=r"(r) 1061 : "r"(__a), "r"(0), "r"(0)); 1062 return r; 1063} 1064__DEVICE__ unsigned int __vabs4(unsigned int __a) { 1065 unsigned int r; 1066 asm("vabsdiff4.s32.s32.s32 %0,%1,%2,%3;" 1067 : "=r"(r) 1068 : "r"(__a), "r"(0), "r"(0)); 1069 return r; 1070} 1071__DEVICE__ unsigned int __vabsdiffs2(unsigned int __a, unsigned int __b) { 1072 unsigned int r; 1073 asm("vabsdiff2.s32.s32.s32 %0,%1,%2,%3;" 1074 : "=r"(r) 1075 : "r"(__a), "r"(__b), "r"(0)); 1076 return r; 1077} 1078 1079__DEVICE__ unsigned int __vabsdiffs4(unsigned int __a, unsigned int __b) { 1080 unsigned int r; 1081 asm("vabsdiff4.s32.s32.s32 %0,%1,%2,%3;" 1082 : "=r"(r) 1083 : "r"(__a), "r"(__b), "r"(0)); 1084 return r; 1085} 1086__DEVICE__ unsigned int __vabsdiffu2(unsigned int __a, unsigned int __b) { 1087 unsigned int r; 1088 asm("vabsdiff2.u32.u32.u32 %0,%1,%2,%3;" 1089 : "=r"(r) 1090 : "r"(__a), "r"(__b), "r"(0)); 1091 return r; 1092} 1093__DEVICE__ unsigned int __vabsdiffu4(unsigned int __a, unsigned int __b) { 1094 unsigned int r; 1095 asm("vabsdiff4.u32.u32.u32 %0,%1,%2,%3;" 1096 : "=r"(r) 1097 : "r"(__a), "r"(__b), "r"(0)); 1098 return r; 1099} 1100__DEVICE__ unsigned int __vabsss2(unsigned int __a) { 1101 unsigned int r; 1102 asm("vabsdiff2.s32.s32.s32.sat %0,%1,%2,%3;" 1103 : "=r"(r) 1104 : "r"(__a), "r"(0), "r"(0)); 1105 return r; 1106} 1107__DEVICE__ unsigned int __vabsss4(unsigned int __a) { 1108 unsigned int r; 1109 asm("vabsdiff4.s32.s32.s32.sat %0,%1,%2,%3;" 1110 : "=r"(r) 1111 : "r"(__a), "r"(0), "r"(0)); 1112 return r; 1113} 1114__DEVICE__ unsigned int __vadd2(unsigned int __a, unsigned int __b) { 1115 unsigned int r; 1116 asm("vadd2.u32.u32.u32 %0,%1,%2,%3;" : "=r"(r) : "r"(__a), "r"(__b), "r"(0)); 1117 return r; 1118} 1119__DEVICE__ unsigned int __vadd4(unsigned int __a, unsigned int __b) { 1120 unsigned int r; 1121 asm("vadd4.u32.u32.u32 %0,%1,%2,%3;" : "=r"(r) : "r"(__a), "r"(__b), "r"(0)); 1122 return r; 1123} 1124__DEVICE__ unsigned int __vaddss2(unsigned int __a, unsigned int __b) { 1125 unsigned int r; 1126 asm("vadd2.s32.s32.s32.sat %0,%1,%2,%3;" 1127 : "=r"(r) 1128 : "r"(__a), "r"(__b), "r"(0)); 1129 return r; 1130} 1131__DEVICE__ unsigned int __vaddss4(unsigned int __a, unsigned int __b) { 1132 unsigned int r; 1133 asm("vadd4.s32.s32.s32.sat %0,%1,%2,%3;" 1134 : "=r"(r) 1135 : "r"(__a), "r"(__b), "r"(0)); 1136 return r; 1137} 1138__DEVICE__ unsigned int __vaddus2(unsigned int __a, unsigned int __b) { 1139 unsigned int r; 1140 asm("vadd2.u32.u32.u32.sat %0,%1,%2,%3;" 1141 : "=r"(r) 1142 : "r"(__a), "r"(__b), "r"(0)); 1143 return r; 1144} 1145__DEVICE__ unsigned int __vaddus4(unsigned int __a, unsigned int __b) { 1146 unsigned int r; 1147 asm("vadd4.u32.u32.u32.sat %0,%1,%2,%3;" 1148 : "=r"(r) 1149 : "r"(__a), "r"(__b), "r"(0)); 1150 return r; 1151} 1152__DEVICE__ unsigned int __vavgs2(unsigned int __a, unsigned int __b) { 1153 unsigned int r; 1154 asm("vavrg2.s32.s32.s32 %0,%1,%2,%3;" : "=r"(r) : "r"(__a), "r"(__b), "r"(0)); 1155 return r; 1156} 1157__DEVICE__ unsigned int __vavgs4(unsigned int __a, unsigned int __b) { 1158 unsigned int r; 1159 asm("vavrg4.s32.s32.s32 %0,%1,%2,%3;" : "=r"(r) : "r"(__a), "r"(__b), "r"(0)); 1160 return r; 1161} 1162__DEVICE__ unsigned int __vavgu2(unsigned int __a, unsigned int __b) { 1163 unsigned int r; 1164 asm("vavrg2.u32.u32.u32 %0,%1,%2,%3;" : "=r"(r) : "r"(__a), "r"(__b), "r"(0)); 1165 return r; 1166} 1167__DEVICE__ unsigned int __vavgu4(unsigned int __a, unsigned int __b) { 1168 unsigned int r; 1169 asm("vavrg4.u32.u32.u32 %0,%1,%2,%3;" : "=r"(r) : "r"(__a), "r"(__b), "r"(0)); 1170 return r; 1171} 1172__DEVICE__ unsigned int __vseteq2(unsigned int __a, unsigned int __b) { 1173 unsigned int r; 1174 asm("vset2.u32.u32.eq %0,%1,%2,%3;" : "=r"(r) : "r"(__a), "r"(__b), "r"(0)); 1175 return r; 1176} 1177__DEVICE__ unsigned int __vcmpeq2(unsigned int __a, unsigned int __b) { 1178 return __bool2mask(__vseteq2(__a, __b), 16); 1179} 1180__DEVICE__ unsigned int __vseteq4(unsigned int __a, unsigned int __b) { 1181 unsigned int r; 1182 asm("vset4.u32.u32.eq %0,%1,%2,%3;" : "=r"(r) : "r"(__a), "r"(__b), "r"(0)); 1183 return r; 1184} 1185__DEVICE__ unsigned int __vcmpeq4(unsigned int __a, unsigned int __b) { 1186 return __bool2mask(__vseteq4(__a, __b), 8); 1187} 1188__DEVICE__ unsigned int __vsetges2(unsigned int __a, unsigned int __b) { 1189 unsigned int r; 1190 asm("vset2.s32.s32.ge %0,%1,%2,%3;" : "=r"(r) : "r"(__a), "r"(__b), "r"(0)); 1191 return r; 1192} 1193__DEVICE__ unsigned int __vcmpges2(unsigned int __a, unsigned int __b) { 1194 return __bool2mask(__vsetges2(__a, __b), 16); 1195} 1196__DEVICE__ unsigned int __vsetges4(unsigned int __a, unsigned int __b) { 1197 unsigned int r; 1198 asm("vset4.s32.s32.ge %0,%1,%2,%3;" : "=r"(r) : "r"(__a), "r"(__b), "r"(0)); 1199 return r; 1200} 1201__DEVICE__ unsigned int __vcmpges4(unsigned int __a, unsigned int __b) { 1202 return __bool2mask(__vsetges4(__a, __b), 8); 1203} 1204__DEVICE__ unsigned int __vsetgeu2(unsigned int __a, unsigned int __b) { 1205 unsigned int r; 1206 asm("vset2.u32.u32.ge %0,%1,%2,%3;" : "=r"(r) : "r"(__a), "r"(__b), "r"(0)); 1207 return r; 1208} 1209__DEVICE__ unsigned int __vcmpgeu2(unsigned int __a, unsigned int __b) { 1210 return __bool2mask(__vsetgeu2(__a, __b), 16); 1211} 1212__DEVICE__ unsigned int __vsetgeu4(unsigned int __a, unsigned int __b) { 1213 unsigned int r; 1214 asm("vset4.u32.u32.ge %0,%1,%2,%3;" : "=r"(r) : "r"(__a), "r"(__b), "r"(0)); 1215 return r; 1216} 1217__DEVICE__ unsigned int __vcmpgeu4(unsigned int __a, unsigned int __b) { 1218 return __bool2mask(__vsetgeu4(__a, __b), 8); 1219} 1220__DEVICE__ unsigned int __vsetgts2(unsigned int __a, unsigned int __b) { 1221 unsigned int r; 1222 asm("vset2.s32.s32.gt %0,%1,%2,%3;" : "=r"(r) : "r"(__a), "r"(__b), "r"(0)); 1223 return r; 1224} 1225__DEVICE__ unsigned int __vcmpgts2(unsigned int __a, unsigned int __b) { 1226 return __bool2mask(__vsetgts2(__a, __b), 16); 1227} 1228__DEVICE__ unsigned int __vsetgts4(unsigned int __a, unsigned int __b) { 1229 unsigned int r; 1230 asm("vset4.s32.s32.gt %0,%1,%2,%3;" : "=r"(r) : "r"(__a), "r"(__b), "r"(0)); 1231 return r; 1232} 1233__DEVICE__ unsigned int __vcmpgts4(unsigned int __a, unsigned int __b) { 1234 return __bool2mask(__vsetgts4(__a, __b), 8); 1235} 1236__DEVICE__ unsigned int __vsetgtu2(unsigned int __a, unsigned int __b) { 1237 unsigned int r; 1238 asm("vset2.u32.u32.gt %0,%1,%2,%3;" : "=r"(r) : "r"(__a), "r"(__b), "r"(0)); 1239 return r; 1240} 1241__DEVICE__ unsigned int __vcmpgtu2(unsigned int __a, unsigned int __b) { 1242 return __bool2mask(__vsetgtu2(__a, __b), 16); 1243} 1244__DEVICE__ unsigned int __vsetgtu4(unsigned int __a, unsigned int __b) { 1245 unsigned int r; 1246 asm("vset4.u32.u32.gt %0,%1,%2,%3;" : "=r"(r) : "r"(__a), "r"(__b), "r"(0)); 1247 return r; 1248} 1249__DEVICE__ unsigned int __vcmpgtu4(unsigned int __a, unsigned int __b) { 1250 return __bool2mask(__vsetgtu4(__a, __b), 8); 1251} 1252__DEVICE__ unsigned int __vsetles2(unsigned int __a, unsigned int __b) { 1253 unsigned int r; 1254 asm("vset2.s32.s32.le %0,%1,%2,%3;" : "=r"(r) : "r"(__a), "r"(__b), "r"(0)); 1255 return r; 1256} 1257__DEVICE__ unsigned int __vcmples2(unsigned int __a, unsigned int __b) { 1258 return __bool2mask(__vsetles2(__a, __b), 16); 1259} 1260__DEVICE__ unsigned int __vsetles4(unsigned int __a, unsigned int __b) { 1261 unsigned int r; 1262 asm("vset4.s32.s32.le %0,%1,%2,%3;" : "=r"(r) : "r"(__a), "r"(__b), "r"(0)); 1263 return r; 1264} 1265__DEVICE__ unsigned int __vcmples4(unsigned int __a, unsigned int __b) { 1266 return __bool2mask(__vsetles4(__a, __b), 8); 1267} 1268__DEVICE__ unsigned int __vsetleu2(unsigned int __a, unsigned int __b) { 1269 unsigned int r; 1270 asm("vset2.u32.u32.le %0,%1,%2,%3;" : "=r"(r) : "r"(__a), "r"(__b), "r"(0)); 1271 return r; 1272} 1273__DEVICE__ unsigned int __vcmpleu2(unsigned int __a, unsigned int __b) { 1274 return __bool2mask(__vsetleu2(__a, __b), 16); 1275} 1276__DEVICE__ unsigned int __vsetleu4(unsigned int __a, unsigned int __b) { 1277 unsigned int r; 1278 asm("vset4.u32.u32.le %0,%1,%2,%3;" : "=r"(r) : "r"(__a), "r"(__b), "r"(0)); 1279 return r; 1280} 1281__DEVICE__ unsigned int __vcmpleu4(unsigned int __a, unsigned int __b) { 1282 return __bool2mask(__vsetleu4(__a, __b), 8); 1283} 1284__DEVICE__ unsigned int __vsetlts2(unsigned int __a, unsigned int __b) { 1285 unsigned int r; 1286 asm("vset2.s32.s32.lt %0,%1,%2,%3;" : "=r"(r) : "r"(__a), "r"(__b), "r"(0)); 1287 return r; 1288} 1289__DEVICE__ unsigned int __vcmplts2(unsigned int __a, unsigned int __b) { 1290 return __bool2mask(__vsetlts2(__a, __b), 16); 1291} 1292__DEVICE__ unsigned int __vsetlts4(unsigned int __a, unsigned int __b) { 1293 unsigned int r; 1294 asm("vset4.s32.s32.lt %0,%1,%2,%3;" : "=r"(r) : "r"(__a), "r"(__b), "r"(0)); 1295 return r; 1296} 1297__DEVICE__ unsigned int __vcmplts4(unsigned int __a, unsigned int __b) { 1298 return __bool2mask(__vsetlts4(__a, __b), 8); 1299} 1300__DEVICE__ unsigned int __vsetltu2(unsigned int __a, unsigned int __b) { 1301 unsigned int r; 1302 asm("vset2.u32.u32.lt %0,%1,%2,%3;" : "=r"(r) : "r"(__a), "r"(__b), "r"(0)); 1303 return r; 1304} 1305__DEVICE__ unsigned int __vcmpltu2(unsigned int __a, unsigned int __b) { 1306 return __bool2mask(__vsetltu2(__a, __b), 16); 1307} 1308__DEVICE__ unsigned int __vsetltu4(unsigned int __a, unsigned int __b) { 1309 unsigned int r; 1310 asm("vset4.u32.u32.lt %0,%1,%2,%3;" : "=r"(r) : "r"(__a), "r"(__b), "r"(0)); 1311 return r; 1312} 1313__DEVICE__ unsigned int __vcmpltu4(unsigned int __a, unsigned int __b) { 1314 return __bool2mask(__vsetltu4(__a, __b), 8); 1315} 1316__DEVICE__ unsigned int __vsetne2(unsigned int __a, unsigned int __b) { 1317 unsigned int r; 1318 asm("vset2.u32.u32.ne %0,%1,%2,%3;" : "=r"(r) : "r"(__a), "r"(__b), "r"(0)); 1319 return r; 1320} 1321__DEVICE__ unsigned int __vcmpne2(unsigned int __a, unsigned int __b) { 1322 return __bool2mask(__vsetne2(__a, __b), 16); 1323} 1324__DEVICE__ unsigned int __vsetne4(unsigned int __a, unsigned int __b) { 1325 unsigned int r; 1326 asm("vset4.u32.u32.ne %0,%1,%2,%3;" : "=r"(r) : "r"(__a), "r"(__b), "r"(0)); 1327 return r; 1328} 1329__DEVICE__ unsigned int __vcmpne4(unsigned int __a, unsigned int __b) { 1330 return __bool2mask(__vsetne4(__a, __b), 8); 1331} 1332 1333// Based on ITEM 23 in AIM-239: http://dspace.mit.edu/handle/1721.1/6086 1334// (a & b) + (a | b) = a + b = (a ^ b) + 2 * (a & b) => 1335// (a + b) / 2 = ((a ^ b) >> 1) + (a & b) 1336// To operate on multiple sub-elements we need to make sure to mask out bits 1337// that crossed over into adjacent elements during the shift. 1338__DEVICE__ unsigned int __vhaddu2(unsigned int __a, unsigned int __b) { 1339 return (((__a ^ __b) >> 1) & ~0x80008000u) + (__a & __b); 1340} 1341__DEVICE__ unsigned int __vhaddu4(unsigned int __a, unsigned int __b) { 1342 return (((__a ^ __b) >> 1) & ~0x80808080u) + (__a & __b); 1343} 1344 1345__DEVICE__ unsigned int __vmaxs2(unsigned int __a, unsigned int __b) { 1346 unsigned int r; 1347 if ((__a & 0x8000) && (__b & 0x8000)) { 1348 // Work around a bug in ptxas which produces invalid result if low element 1349 // is negative. 1350 unsigned mask = __vcmpgts2(__a, __b); 1351 r = (__a & mask) | (__b & ~mask); 1352 } else { 1353 asm("vmax2.s32.s32.s32 %0,%1,%2,%3;" 1354 : "=r"(r) 1355 : "r"(__a), "r"(__b), "r"(0)); 1356 } 1357 return r; 1358} 1359__DEVICE__ unsigned int __vmaxs4(unsigned int __a, unsigned int __b) { 1360 unsigned int r; 1361 asm("vmax4.s32.s32.s32 %0,%1,%2,%3;" : "=r"(r) : "r"(__a), "r"(__b), "r"(0)); 1362 return r; 1363} 1364__DEVICE__ unsigned int __vmaxu2(unsigned int __a, unsigned int __b) { 1365 unsigned int r; 1366 asm("vmax2.u32.u32.u32 %0,%1,%2,%3;" : "=r"(r) : "r"(__a), "r"(__b), "r"(0)); 1367 return r; 1368} 1369__DEVICE__ unsigned int __vmaxu4(unsigned int __a, unsigned int __b) { 1370 unsigned int r; 1371 asm("vmax4.u32.u32.u32 %0,%1,%2,%3;" : "=r"(r) : "r"(__a), "r"(__b), "r"(0)); 1372 return r; 1373} 1374__DEVICE__ unsigned int __vmins2(unsigned int __a, unsigned int __b) { 1375 unsigned int r; 1376 asm("vmin2.s32.s32.s32 %0,%1,%2,%3;" : "=r"(r) : "r"(__a), "r"(__b), "r"(0)); 1377 return r; 1378} 1379__DEVICE__ unsigned int __vmins4(unsigned int __a, unsigned int __b) { 1380 unsigned int r; 1381 asm("vmin4.s32.s32.s32 %0,%1,%2,%3;" : "=r"(r) : "r"(__a), "r"(__b), "r"(0)); 1382 return r; 1383} 1384__DEVICE__ unsigned int __vminu2(unsigned int __a, unsigned int __b) { 1385 unsigned int r; 1386 asm("vmin2.u32.u32.u32 %0,%1,%2,%3;" : "=r"(r) : "r"(__a), "r"(__b), "r"(0)); 1387 return r; 1388} 1389__DEVICE__ unsigned int __vminu4(unsigned int __a, unsigned int __b) { 1390 unsigned int r; 1391 asm("vmin4.u32.u32.u32 %0,%1,%2,%3;" : "=r"(r) : "r"(__a), "r"(__b), "r"(0)); 1392 return r; 1393} 1394__DEVICE__ unsigned int __vsads2(unsigned int __a, unsigned int __b) { 1395 unsigned int r; 1396 asm("vabsdiff2.s32.s32.s32.add %0,%1,%2,%3;" 1397 : "=r"(r) 1398 : "r"(__a), "r"(__b), "r"(0)); 1399 return r; 1400} 1401__DEVICE__ unsigned int __vsads4(unsigned int __a, unsigned int __b) { 1402 unsigned int r; 1403 asm("vabsdiff4.s32.s32.s32.add %0,%1,%2,%3;" 1404 : "=r"(r) 1405 : "r"(__a), "r"(__b), "r"(0)); 1406 return r; 1407} 1408__DEVICE__ unsigned int __vsadu2(unsigned int __a, unsigned int __b) { 1409 unsigned int r; 1410 asm("vabsdiff2.u32.u32.u32.add %0,%1,%2,%3;" 1411 : "=r"(r) 1412 : "r"(__a), "r"(__b), "r"(0)); 1413 return r; 1414} 1415__DEVICE__ unsigned int __vsadu4(unsigned int __a, unsigned int __b) { 1416 unsigned int r; 1417 asm("vabsdiff4.u32.u32.u32.add %0,%1,%2,%3;" 1418 : "=r"(r) 1419 : "r"(__a), "r"(__b), "r"(0)); 1420 return r; 1421} 1422 1423__DEVICE__ unsigned int __vsub2(unsigned int __a, unsigned int __b) { 1424 unsigned int r; 1425 asm("vsub2.u32.u32.u32 %0,%1,%2,%3;" : "=r"(r) : "r"(__a), "r"(__b), "r"(0)); 1426 return r; 1427} 1428__DEVICE__ unsigned int __vneg2(unsigned int __a) { return __vsub2(0, __a); } 1429 1430__DEVICE__ unsigned int __vsub4(unsigned int __a, unsigned int __b) { 1431 unsigned int r; 1432 asm("vsub4.u32.u32.u32 %0,%1,%2,%3;" : "=r"(r) : "r"(__a), "r"(__b), "r"(0)); 1433 return r; 1434} 1435__DEVICE__ unsigned int __vneg4(unsigned int __a) { return __vsub4(0, __a); } 1436__DEVICE__ unsigned int __vsubss2(unsigned int __a, unsigned int __b) { 1437 unsigned int r; 1438 asm("vsub2.s32.s32.s32.sat %0,%1,%2,%3;" 1439 : "=r"(r) 1440 : "r"(__a), "r"(__b), "r"(0)); 1441 return r; 1442} 1443__DEVICE__ unsigned int __vnegss2(unsigned int __a) { 1444 return __vsubss2(0, __a); 1445} 1446__DEVICE__ unsigned int __vsubss4(unsigned int __a, unsigned int __b) { 1447 unsigned int r; 1448 asm("vsub4.s32.s32.s32.sat %0,%1,%2,%3;" 1449 : "=r"(r) 1450 : "r"(__a), "r"(__b), "r"(0)); 1451 return r; 1452} 1453__DEVICE__ unsigned int __vnegss4(unsigned int __a) { 1454 return __vsubss4(0, __a); 1455} 1456__DEVICE__ unsigned int __vsubus2(unsigned int __a, unsigned int __b) { 1457 unsigned int r; 1458 asm("vsub2.u32.u32.u32.sat %0,%1,%2,%3;" 1459 : "=r"(r) 1460 : "r"(__a), "r"(__b), "r"(0)); 1461 return r; 1462} 1463__DEVICE__ unsigned int __vsubus4(unsigned int __a, unsigned int __b) { 1464 unsigned int r; 1465 asm("vsub4.u32.u32.u32.sat %0,%1,%2,%3;" 1466 : "=r"(r) 1467 : "r"(__a), "r"(__b), "r"(0)); 1468 return r; 1469} 1470#endif // CUDA_VERSION >= 9020 1471__DEVICE__ int abs(int __a) { return __nv_abs(__a); } 1472__DEVICE__ double acos(double __a) { return __nv_acos(__a); } 1473__DEVICE__ float acosf(float __a) { return __nv_acosf(__a); } 1474__DEVICE__ double acosh(double __a) { return __nv_acosh(__a); } 1475__DEVICE__ float acoshf(float __a) { return __nv_acoshf(__a); } 1476__DEVICE__ double asin(double __a) { return __nv_asin(__a); } 1477__DEVICE__ float asinf(float __a) { return __nv_asinf(__a); } 1478__DEVICE__ double asinh(double __a) { return __nv_asinh(__a); } 1479__DEVICE__ float asinhf(float __a) { return __nv_asinhf(__a); } 1480__DEVICE__ double atan(double __a) { return __nv_atan(__a); } 1481__DEVICE__ double atan2(double __a, double __b) { return __nv_atan2(__a, __b); } 1482__DEVICE__ float atan2f(float __a, float __b) { return __nv_atan2f(__a, __b); } 1483__DEVICE__ float atanf(float __a) { return __nv_atanf(__a); } 1484__DEVICE__ double atanh(double __a) { return __nv_atanh(__a); } 1485__DEVICE__ float atanhf(float __a) { return __nv_atanhf(__a); } 1486__DEVICE__ double cbrt(double __a) { return __nv_cbrt(__a); } 1487__DEVICE__ float cbrtf(float __a) { return __nv_cbrtf(__a); } 1488__DEVICE__ double ceil(double __a) { return __nv_ceil(__a); } 1489__DEVICE__ float ceilf(float __a) { return __nv_ceilf(__a); } 1490__DEVICE__ int clock() { return __nvvm_read_ptx_sreg_clock(); } 1491__DEVICE__ long long clock64() { return __nvvm_read_ptx_sreg_clock64(); } 1492__DEVICE__ double copysign(double __a, double __b) { 1493 return __nv_copysign(__a, __b); 1494} 1495__DEVICE__ float copysignf(float __a, float __b) { 1496 return __nv_copysignf(__a, __b); 1497} 1498__DEVICE__ double cos(double __a) { return __nv_cos(__a); } 1499__DEVICE__ float cosf(float __a) { 1500 return __FAST_OR_SLOW(__nv_fast_cosf, __nv_cosf)(__a); 1501} 1502__DEVICE__ double cosh(double __a) { return __nv_cosh(__a); } 1503__DEVICE__ float coshf(float __a) { return __nv_coshf(__a); } 1504__DEVICE__ double cospi(double __a) { return __nv_cospi(__a); } 1505__DEVICE__ float cospif(float __a) { return __nv_cospif(__a); } 1506__DEVICE__ double cyl_bessel_i0(double __a) { return __nv_cyl_bessel_i0(__a); } 1507__DEVICE__ float cyl_bessel_i0f(float __a) { return __nv_cyl_bessel_i0f(__a); } 1508__DEVICE__ double cyl_bessel_i1(double __a) { return __nv_cyl_bessel_i1(__a); } 1509__DEVICE__ float cyl_bessel_i1f(float __a) { return __nv_cyl_bessel_i1f(__a); } 1510__DEVICE__ double erf(double __a) { return __nv_erf(__a); } 1511__DEVICE__ double erfc(double __a) { return __nv_erfc(__a); } 1512__DEVICE__ float erfcf(float __a) { return __nv_erfcf(__a); } 1513__DEVICE__ double erfcinv(double __a) { return __nv_erfcinv(__a); } 1514__DEVICE__ float erfcinvf(float __a) { return __nv_erfcinvf(__a); } 1515__DEVICE__ double erfcx(double __a) { return __nv_erfcx(__a); } 1516__DEVICE__ float erfcxf(float __a) { return __nv_erfcxf(__a); } 1517__DEVICE__ float erff(float __a) { return __nv_erff(__a); } 1518__DEVICE__ double erfinv(double __a) { return __nv_erfinv(__a); } 1519__DEVICE__ float erfinvf(float __a) { return __nv_erfinvf(__a); } 1520__DEVICE__ double exp(double __a) { return __nv_exp(__a); } 1521__DEVICE__ double exp10(double __a) { return __nv_exp10(__a); } 1522__DEVICE__ float exp10f(float __a) { return __nv_exp10f(__a); } 1523__DEVICE__ double exp2(double __a) { return __nv_exp2(__a); } 1524__DEVICE__ float exp2f(float __a) { return __nv_exp2f(__a); } 1525__DEVICE__ float expf(float __a) { return __nv_expf(__a); } 1526__DEVICE__ double expm1(double __a) { return __nv_expm1(__a); } 1527__DEVICE__ float expm1f(float __a) { return __nv_expm1f(__a); } 1528__DEVICE__ double fabs(double __a) { return __nv_fabs(__a); } 1529__DEVICE__ float fabsf(float __a) { return __nv_fabsf(__a); } 1530__DEVICE__ double fdim(double __a, double __b) { return __nv_fdim(__a, __b); } 1531__DEVICE__ float fdimf(float __a, float __b) { return __nv_fdimf(__a, __b); } 1532__DEVICE__ double fdivide(double __a, double __b) { return __a / __b; } 1533__DEVICE__ float fdividef(float __a, float __b) { 1534#if __FAST_MATH__ && !__CUDA_PREC_DIV 1535 return __nv_fast_fdividef(__a, __b); 1536#else 1537 return __a / __b; 1538#endif 1539} 1540__DEVICE__ double floor(double __f) { return __nv_floor(__f); } 1541__DEVICE__ float floorf(float __f) { return __nv_floorf(__f); } 1542__DEVICE__ double fma(double __a, double __b, double __c) { 1543 return __nv_fma(__a, __b, __c); 1544} 1545__DEVICE__ float fmaf(float __a, float __b, float __c) { 1546 return __nv_fmaf(__a, __b, __c); 1547} 1548__DEVICE__ double fmax(double __a, double __b) { return __nv_fmax(__a, __b); } 1549__DEVICE__ float fmaxf(float __a, float __b) { return __nv_fmaxf(__a, __b); } 1550__DEVICE__ double fmin(double __a, double __b) { return __nv_fmin(__a, __b); } 1551__DEVICE__ float fminf(float __a, float __b) { return __nv_fminf(__a, __b); } 1552__DEVICE__ double fmod(double __a, double __b) { return __nv_fmod(__a, __b); } 1553__DEVICE__ float fmodf(float __a, float __b) { return __nv_fmodf(__a, __b); } 1554__DEVICE__ double frexp(double __a, int *__b) { return __nv_frexp(__a, __b); } 1555__DEVICE__ float frexpf(float __a, int *__b) { return __nv_frexpf(__a, __b); } 1556__DEVICE__ double hypot(double __a, double __b) { return __nv_hypot(__a, __b); } 1557__DEVICE__ float hypotf(float __a, float __b) { return __nv_hypotf(__a, __b); } 1558__DEVICE__ int ilogb(double __a) { return __nv_ilogb(__a); } 1559__DEVICE__ int ilogbf(float __a) { return __nv_ilogbf(__a); } 1560__DEVICE__ double j0(double __a) { return __nv_j0(__a); } 1561__DEVICE__ float j0f(float __a) { return __nv_j0f(__a); } 1562__DEVICE__ double j1(double __a) { return __nv_j1(__a); } 1563__DEVICE__ float j1f(float __a) { return __nv_j1f(__a); } 1564__DEVICE__ double jn(int __n, double __a) { return __nv_jn(__n, __a); } 1565__DEVICE__ float jnf(int __n, float __a) { return __nv_jnf(__n, __a); } 1566#if defined(__LP64__) 1567__DEVICE__ long labs(long __a) { return llabs(__a); }; 1568#else 1569__DEVICE__ long labs(long __a) { return __nv_abs(__a); }; 1570#endif 1571__DEVICE__ double ldexp(double __a, int __b) { return __nv_ldexp(__a, __b); } 1572__DEVICE__ float ldexpf(float __a, int __b) { return __nv_ldexpf(__a, __b); } 1573__DEVICE__ double lgamma(double __a) { return __nv_lgamma(__a); } 1574__DEVICE__ float lgammaf(float __a) { return __nv_lgammaf(__a); } 1575__DEVICE__ long long llabs(long long __a) { return __nv_llabs(__a); } 1576__DEVICE__ long long llmax(long long __a, long long __b) { 1577 return __nv_llmax(__a, __b); 1578} 1579__DEVICE__ long long llmin(long long __a, long long __b) { 1580 return __nv_llmin(__a, __b); 1581} 1582__DEVICE__ long long llrint(double __a) { return __nv_llrint(__a); } 1583__DEVICE__ long long llrintf(float __a) { return __nv_llrintf(__a); } 1584__DEVICE__ long long llround(double __a) { return __nv_llround(__a); } 1585__DEVICE__ long long llroundf(float __a) { return __nv_llroundf(__a); } 1586__DEVICE__ double log(double __a) { return __nv_log(__a); } 1587__DEVICE__ double log10(double __a) { return __nv_log10(__a); } 1588__DEVICE__ float log10f(float __a) { return __nv_log10f(__a); } 1589__DEVICE__ double log1p(double __a) { return __nv_log1p(__a); } 1590__DEVICE__ float log1pf(float __a) { return __nv_log1pf(__a); } 1591__DEVICE__ double log2(double __a) { return __nv_log2(__a); } 1592__DEVICE__ float log2f(float __a) { 1593 return __FAST_OR_SLOW(__nv_fast_log2f, __nv_log2f)(__a); 1594} 1595__DEVICE__ double logb(double __a) { return __nv_logb(__a); } 1596__DEVICE__ float logbf(float __a) { return __nv_logbf(__a); } 1597__DEVICE__ float logf(float __a) { 1598 return __FAST_OR_SLOW(__nv_fast_logf, __nv_logf)(__a); 1599} 1600#if defined(__LP64__) 1601__DEVICE__ long lrint(double __a) { return llrint(__a); } 1602__DEVICE__ long lrintf(float __a) { return __float2ll_rn(__a); } 1603__DEVICE__ long lround(double __a) { return llround(__a); } 1604__DEVICE__ long lroundf(float __a) { return llroundf(__a); } 1605#else 1606__DEVICE__ long lrint(double __a) { return (long)rint(__a); } 1607__DEVICE__ long lrintf(float __a) { return __float2int_rn(__a); } 1608__DEVICE__ long lround(double __a) { return round(__a); } 1609__DEVICE__ long lroundf(float __a) { return roundf(__a); } 1610#endif 1611__DEVICE__ int max(int __a, int __b) { return __nv_max(__a, __b); } 1612__DEVICE__ void *memcpy(void *__a, const void *__b, size_t __c) { 1613 return __builtin_memcpy(__a, __b, __c); 1614} 1615__DEVICE__ void *memset(void *__a, int __b, size_t __c) { 1616 return __builtin_memset(__a, __b, __c); 1617} 1618__DEVICE__ int min(int __a, int __b) { return __nv_min(__a, __b); } 1619__DEVICE__ double modf(double __a, double *__b) { return __nv_modf(__a, __b); } 1620__DEVICE__ float modff(float __a, float *__b) { return __nv_modff(__a, __b); } 1621__DEVICE__ double nearbyint(double __a) { return __nv_nearbyint(__a); } 1622__DEVICE__ float nearbyintf(float __a) { return __nv_nearbyintf(__a); } 1623__DEVICE__ double nextafter(double __a, double __b) { 1624 return __nv_nextafter(__a, __b); 1625} 1626__DEVICE__ float nextafterf(float __a, float __b) { 1627 return __nv_nextafterf(__a, __b); 1628} 1629__DEVICE__ double norm(int __dim, const double *__t) { 1630 return __nv_norm(__dim, __t); 1631} 1632__DEVICE__ double norm3d(double __a, double __b, double __c) { 1633 return __nv_norm3d(__a, __b, __c); 1634} 1635__DEVICE__ float norm3df(float __a, float __b, float __c) { 1636 return __nv_norm3df(__a, __b, __c); 1637} 1638__DEVICE__ double norm4d(double __a, double __b, double __c, double __d) { 1639 return __nv_norm4d(__a, __b, __c, __d); 1640} 1641__DEVICE__ float norm4df(float __a, float __b, float __c, float __d) { 1642 return __nv_norm4df(__a, __b, __c, __d); 1643} 1644__DEVICE__ double normcdf(double __a) { return __nv_normcdf(__a); } 1645__DEVICE__ float normcdff(float __a) { return __nv_normcdff(__a); } 1646__DEVICE__ double normcdfinv(double __a) { return __nv_normcdfinv(__a); } 1647__DEVICE__ float normcdfinvf(float __a) { return __nv_normcdfinvf(__a); } 1648__DEVICE__ float normf(int __dim, const float *__t) { 1649 return __nv_normf(__dim, __t); 1650} 1651__DEVICE__ double pow(double __a, double __b) { return __nv_pow(__a, __b); } 1652__DEVICE__ float powf(float __a, float __b) { return __nv_powf(__a, __b); } 1653__DEVICE__ double powi(double __a, int __b) { return __nv_powi(__a, __b); } 1654__DEVICE__ float powif(float __a, int __b) { return __nv_powif(__a, __b); } 1655__DEVICE__ double rcbrt(double __a) { return __nv_rcbrt(__a); } 1656__DEVICE__ float rcbrtf(float __a) { return __nv_rcbrtf(__a); } 1657__DEVICE__ double remainder(double __a, double __b) { 1658 return __nv_remainder(__a, __b); 1659} 1660__DEVICE__ float remainderf(float __a, float __b) { 1661 return __nv_remainderf(__a, __b); 1662} 1663__DEVICE__ double remquo(double __a, double __b, int *__c) { 1664 return __nv_remquo(__a, __b, __c); 1665} 1666__DEVICE__ float remquof(float __a, float __b, int *__c) { 1667 return __nv_remquof(__a, __b, __c); 1668} 1669__DEVICE__ double rhypot(double __a, double __b) { 1670 return __nv_rhypot(__a, __b); 1671} 1672__DEVICE__ float rhypotf(float __a, float __b) { 1673 return __nv_rhypotf(__a, __b); 1674} 1675__DEVICE__ double rint(double __a) { return __nv_rint(__a); } 1676__DEVICE__ float rintf(float __a) { return __nv_rintf(__a); } 1677__DEVICE__ double rnorm(int __a, const double *__b) { 1678 return __nv_rnorm(__a, __b); 1679} 1680__DEVICE__ double rnorm3d(double __a, double __b, double __c) { 1681 return __nv_rnorm3d(__a, __b, __c); 1682} 1683__DEVICE__ float rnorm3df(float __a, float __b, float __c) { 1684 return __nv_rnorm3df(__a, __b, __c); 1685} 1686__DEVICE__ double rnorm4d(double __a, double __b, double __c, double __d) { 1687 return __nv_rnorm4d(__a, __b, __c, __d); 1688} 1689__DEVICE__ float rnorm4df(float __a, float __b, float __c, float __d) { 1690 return __nv_rnorm4df(__a, __b, __c, __d); 1691} 1692__DEVICE__ float rnormf(int __dim, const float *__t) { 1693 return __nv_rnormf(__dim, __t); 1694} 1695__DEVICE__ double round(double __a) { return __nv_round(__a); } 1696__DEVICE__ float roundf(float __a) { return __nv_roundf(__a); } 1697__DEVICE__ double rsqrt(double __a) { return __nv_rsqrt(__a); } 1698__DEVICE__ float rsqrtf(float __a) { return __nv_rsqrtf(__a); } 1699__DEVICE__ double scalbn(double __a, int __b) { return __nv_scalbn(__a, __b); } 1700__DEVICE__ float scalbnf(float __a, int __b) { return __nv_scalbnf(__a, __b); } 1701__DEVICE__ double scalbln(double __a, long __b) { 1702 if (__b > INT_MAX) 1703 return __a > 0 ? HUGE_VAL : -HUGE_VAL; 1704 if (__b < INT_MIN) 1705 return __a > 0 ? 0.0 : -0.0; 1706 return scalbn(__a, (int)__b); 1707} 1708__DEVICE__ float scalblnf(float __a, long __b) { 1709 if (__b > INT_MAX) 1710 return __a > 0 ? HUGE_VALF : -HUGE_VALF; 1711 if (__b < INT_MIN) 1712 return __a > 0 ? 0.f : -0.f; 1713 return scalbnf(__a, (int)__b); 1714} 1715__DEVICE__ double sin(double __a) { return __nv_sin(__a); } 1716__DEVICE__ void sincos(double __a, double *__sptr, double *__cptr) { 1717 return __nv_sincos(__a, __sptr, __cptr); 1718} 1719__DEVICE__ void sincosf(float __a, float *__sptr, float *__cptr) { 1720 return __FAST_OR_SLOW(__nv_fast_sincosf, __nv_sincosf)(__a, __sptr, __cptr); 1721} 1722__DEVICE__ void sincospi(double __a, double *__sptr, double *__cptr) { 1723 return __nv_sincospi(__a, __sptr, __cptr); 1724} 1725__DEVICE__ void sincospif(float __a, float *__sptr, float *__cptr) { 1726 return __nv_sincospif(__a, __sptr, __cptr); 1727} 1728__DEVICE__ float sinf(float __a) { 1729 return __FAST_OR_SLOW(__nv_fast_sinf, __nv_sinf)(__a); 1730} 1731__DEVICE__ double sinh(double __a) { return __nv_sinh(__a); } 1732__DEVICE__ float sinhf(float __a) { return __nv_sinhf(__a); } 1733__DEVICE__ double sinpi(double __a) { return __nv_sinpi(__a); } 1734__DEVICE__ float sinpif(float __a) { return __nv_sinpif(__a); } 1735__DEVICE__ double sqrt(double __a) { return __nv_sqrt(__a); } 1736__DEVICE__ float sqrtf(float __a) { return __nv_sqrtf(__a); } 1737__DEVICE__ double tan(double __a) { return __nv_tan(__a); } 1738__DEVICE__ float tanf(float __a) { return __nv_tanf(__a); } 1739__DEVICE__ double tanh(double __a) { return __nv_tanh(__a); } 1740__DEVICE__ float tanhf(float __a) { return __nv_tanhf(__a); } 1741__DEVICE__ double tgamma(double __a) { return __nv_tgamma(__a); } 1742__DEVICE__ float tgammaf(float __a) { return __nv_tgammaf(__a); } 1743__DEVICE__ double trunc(double __a) { return __nv_trunc(__a); } 1744__DEVICE__ float truncf(float __a) { return __nv_truncf(__a); } 1745__DEVICE__ unsigned long long ullmax(unsigned long long __a, 1746 unsigned long long __b) { 1747 return __nv_ullmax(__a, __b); 1748} 1749__DEVICE__ unsigned long long ullmin(unsigned long long __a, 1750 unsigned long long __b) { 1751 return __nv_ullmin(__a, __b); 1752} 1753__DEVICE__ unsigned int umax(unsigned int __a, unsigned int __b) { 1754 return __nv_umax(__a, __b); 1755} 1756__DEVICE__ unsigned int umin(unsigned int __a, unsigned int __b) { 1757 return __nv_umin(__a, __b); 1758} 1759__DEVICE__ double y0(double __a) { return __nv_y0(__a); } 1760__DEVICE__ float y0f(float __a) { return __nv_y0f(__a); } 1761__DEVICE__ double y1(double __a) { return __nv_y1(__a); } 1762__DEVICE__ float y1f(float __a) { return __nv_y1f(__a); } 1763__DEVICE__ double yn(int __a, double __b) { return __nv_yn(__a, __b); } 1764__DEVICE__ float ynf(int __a, float __b) { return __nv_ynf(__a, __b); } 1765 1766#pragma pop_macro("__DEVICE__") 1767#pragma pop_macro("__FAST_OR_SLOW") 1768#endif // __CLANG_CUDA_DEVICE_FUNCTIONS_H__ 1769