1/*===---- arm_acle.h - ARM Non-Neon intrinsics -----------------------------===
2 *
3 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 * See https://llvm.org/LICENSE.txt for license information.
5 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 *
7 *===-----------------------------------------------------------------------===
8 */
9
10#ifndef __ARM_ACLE_H
11#define __ARM_ACLE_H
12
13#ifndef __ARM_ACLE
14#error "ACLE intrinsics support not enabled."
15#endif
16
17#include <stdint.h>
18
19#if defined(__cplusplus)
20extern "C" {
21#endif
22
23/* 8 SYNCHRONIZATION, BARRIER AND HINT INTRINSICS */
24/* 8.3 Memory barriers */
25#if !__has_builtin(__dmb)
26#define __dmb(i) __builtin_arm_dmb(i)
27#endif
28#if !__has_builtin(__dsb)
29#define __dsb(i) __builtin_arm_dsb(i)
30#endif
31#if !__has_builtin(__isb)
32#define __isb(i) __builtin_arm_isb(i)
33#endif
34
35/* 8.4 Hints */
36
37#if !__has_builtin(__wfi)
38static __inline__ void __attribute__((__always_inline__, __nodebug__)) __wfi(void) {
39  __builtin_arm_wfi();
40}
41#endif
42
43#if !__has_builtin(__wfe)
44static __inline__ void __attribute__((__always_inline__, __nodebug__)) __wfe(void) {
45  __builtin_arm_wfe();
46}
47#endif
48
49#if !__has_builtin(__sev)
50static __inline__ void __attribute__((__always_inline__, __nodebug__)) __sev(void) {
51  __builtin_arm_sev();
52}
53#endif
54
55#if !__has_builtin(__sevl)
56static __inline__ void __attribute__((__always_inline__, __nodebug__)) __sevl(void) {
57  __builtin_arm_sevl();
58}
59#endif
60
61#if !__has_builtin(__yield)
62static __inline__ void __attribute__((__always_inline__, __nodebug__)) __yield(void) {
63  __builtin_arm_yield();
64}
65#endif
66
67#if __ARM_32BIT_STATE
68#define __dbg(t) __builtin_arm_dbg(t)
69#endif
70
71/* 8.5 Swap */
72static __inline__ uint32_t __attribute__((__always_inline__, __nodebug__))
73__swp(uint32_t __x, volatile uint32_t *__p) {
74  uint32_t v;
75  do
76    v = __builtin_arm_ldrex(__p);
77  while (__builtin_arm_strex(__x, __p));
78  return v;
79}
80
81/* 8.6 Memory prefetch intrinsics */
82/* 8.6.1 Data prefetch */
83#define __pld(addr) __pldx(0, 0, 0, addr)
84
85#if __ARM_32BIT_STATE
86#define __pldx(access_kind, cache_level, retention_policy, addr) \
87  __builtin_arm_prefetch(addr, access_kind, 1)
88#else
89#define __pldx(access_kind, cache_level, retention_policy, addr) \
90  __builtin_arm_prefetch(addr, access_kind, cache_level, retention_policy, 1)
91#endif
92
93/* 8.6.2 Instruction prefetch */
94#define __pli(addr) __plix(0, 0, addr)
95
96#if __ARM_32BIT_STATE
97#define __plix(cache_level, retention_policy, addr) \
98  __builtin_arm_prefetch(addr, 0, 0)
99#else
100#define __plix(cache_level, retention_policy, addr) \
101  __builtin_arm_prefetch(addr, 0, cache_level, retention_policy, 0)
102#endif
103
104/* 8.7 NOP */
105#if !defined(_MSC_VER) || !defined(__aarch64__)
106static __inline__ void __attribute__((__always_inline__, __nodebug__)) __nop(void) {
107  __builtin_arm_nop();
108}
109#endif
110
111/* 9 DATA-PROCESSING INTRINSICS */
112/* 9.2 Miscellaneous data-processing intrinsics */
113/* ROR */
114static __inline__ uint32_t __attribute__((__always_inline__, __nodebug__))
115__ror(uint32_t __x, uint32_t __y) {
116  __y %= 32;
117  if (__y == 0)
118    return __x;
119  return (__x >> __y) | (__x << (32 - __y));
120}
121
122static __inline__ uint64_t __attribute__((__always_inline__, __nodebug__))
123__rorll(uint64_t __x, uint32_t __y) {
124  __y %= 64;
125  if (__y == 0)
126    return __x;
127  return (__x >> __y) | (__x << (64 - __y));
128}
129
130static __inline__ unsigned long __attribute__((__always_inline__, __nodebug__))
131__rorl(unsigned long __x, uint32_t __y) {
132#if __SIZEOF_LONG__ == 4
133  return __ror(__x, __y);
134#else
135  return __rorll(__x, __y);
136#endif
137}
138
139
140/* CLZ */
141static __inline__ uint32_t __attribute__((__always_inline__, __nodebug__))
142__clz(uint32_t __t) {
143  return __builtin_clz(__t);
144}
145
146static __inline__ unsigned long __attribute__((__always_inline__, __nodebug__))
147__clzl(unsigned long __t) {
148  return __builtin_clzl(__t);
149}
150
151static __inline__ uint64_t __attribute__((__always_inline__, __nodebug__))
152__clzll(uint64_t __t) {
153  return __builtin_clzll(__t);
154}
155
156/* CLS */
157static __inline__ uint32_t __attribute__((__always_inline__, __nodebug__))
158__cls(uint32_t __t) {
159  return __builtin_arm_cls(__t);
160}
161
162static __inline__ uint32_t __attribute__((__always_inline__, __nodebug__))
163__clsl(unsigned long __t) {
164#if __SIZEOF_LONG__ == 4
165  return __builtin_arm_cls(__t);
166#else
167  return __builtin_arm_cls64(__t);
168#endif
169}
170
171static __inline__ uint32_t __attribute__((__always_inline__, __nodebug__))
172__clsll(uint64_t __t) {
173  return __builtin_arm_cls64(__t);
174}
175
176/* REV */
177static __inline__ uint32_t __attribute__((__always_inline__, __nodebug__))
178__rev(uint32_t __t) {
179  return __builtin_bswap32(__t);
180}
181
182static __inline__ unsigned long __attribute__((__always_inline__, __nodebug__))
183__revl(unsigned long __t) {
184#if __SIZEOF_LONG__ == 4
185  return __builtin_bswap32(__t);
186#else
187  return __builtin_bswap64(__t);
188#endif
189}
190
191static __inline__ uint64_t __attribute__((__always_inline__, __nodebug__))
192__revll(uint64_t __t) {
193  return __builtin_bswap64(__t);
194}
195
196/* REV16 */
197static __inline__ uint32_t __attribute__((__always_inline__, __nodebug__))
198__rev16(uint32_t __t) {
199  return __ror(__rev(__t), 16);
200}
201
202static __inline__ uint64_t __attribute__((__always_inline__, __nodebug__))
203__rev16ll(uint64_t __t) {
204  return (((uint64_t)__rev16(__t >> 32)) << 32) | __rev16(__t);
205}
206
207static __inline__ unsigned long __attribute__((__always_inline__, __nodebug__))
208__rev16l(unsigned long __t) {
209#if __SIZEOF_LONG__ == 4
210    return __rev16(__t);
211#else
212    return __rev16ll(__t);
213#endif
214}
215
216/* REVSH */
217static __inline__ int16_t __attribute__((__always_inline__, __nodebug__))
218__revsh(int16_t __t) {
219  return __builtin_bswap16(__t);
220}
221
222/* RBIT */
223static __inline__ uint32_t __attribute__((__always_inline__, __nodebug__))
224__rbit(uint32_t __t) {
225  return __builtin_arm_rbit(__t);
226}
227
228static __inline__ uint64_t __attribute__((__always_inline__, __nodebug__))
229__rbitll(uint64_t __t) {
230#if __ARM_32BIT_STATE
231  return (((uint64_t)__builtin_arm_rbit(__t)) << 32) |
232         __builtin_arm_rbit(__t >> 32);
233#else
234  return __builtin_arm_rbit64(__t);
235#endif
236}
237
238static __inline__ unsigned long __attribute__((__always_inline__, __nodebug__))
239__rbitl(unsigned long __t) {
240#if __SIZEOF_LONG__ == 4
241  return __rbit(__t);
242#else
243  return __rbitll(__t);
244#endif
245}
246
247/*
248 * 9.3 16-bit multiplications
249 */
250#if __ARM_FEATURE_DSP
251static __inline__ int32_t __attribute__((__always_inline__,__nodebug__))
252__smulbb(int32_t __a, int32_t __b) {
253  return __builtin_arm_smulbb(__a, __b);
254}
255static __inline__ int32_t __attribute__((__always_inline__,__nodebug__))
256__smulbt(int32_t __a, int32_t __b) {
257  return __builtin_arm_smulbt(__a, __b);
258}
259static __inline__ int32_t __attribute__((__always_inline__,__nodebug__))
260__smultb(int32_t __a, int32_t __b) {
261  return __builtin_arm_smultb(__a, __b);
262}
263static __inline__ int32_t __attribute__((__always_inline__,__nodebug__))
264__smultt(int32_t __a, int32_t __b) {
265  return __builtin_arm_smultt(__a, __b);
266}
267static __inline__ int32_t __attribute__((__always_inline__,__nodebug__))
268__smulwb(int32_t __a, int32_t __b) {
269  return __builtin_arm_smulwb(__a, __b);
270}
271static __inline__ int32_t __attribute__((__always_inline__,__nodebug__))
272__smulwt(int32_t __a, int32_t __b) {
273  return __builtin_arm_smulwt(__a, __b);
274}
275#endif
276
277/*
278 * 9.4 Saturating intrinsics
279 *
280 * FIXME: Change guard to their corrosponding __ARM_FEATURE flag when Q flag
281 * intrinsics are implemented and the flag is enabled.
282 */
283/* 9.4.1 Width-specified saturation intrinsics */
284#if __ARM_FEATURE_SAT
285#define __ssat(x, y) __builtin_arm_ssat(x, y)
286#define __usat(x, y) __builtin_arm_usat(x, y)
287#endif
288
289/* 9.4.2 Saturating addition and subtraction intrinsics */
290#if __ARM_FEATURE_DSP
291static __inline__ int32_t __attribute__((__always_inline__, __nodebug__))
292__qadd(int32_t __t, int32_t __v) {
293  return __builtin_arm_qadd(__t, __v);
294}
295
296static __inline__ int32_t __attribute__((__always_inline__, __nodebug__))
297__qsub(int32_t __t, int32_t __v) {
298  return __builtin_arm_qsub(__t, __v);
299}
300
301static __inline__ int32_t __attribute__((__always_inline__, __nodebug__))
302__qdbl(int32_t __t) {
303  return __builtin_arm_qadd(__t, __t);
304}
305#endif
306
307/* 9.4.3 Accumultating multiplications */
308#if __ARM_FEATURE_DSP
309static __inline__ int32_t __attribute__((__always_inline__, __nodebug__))
310__smlabb(int32_t __a, int32_t __b, int32_t __c) {
311  return __builtin_arm_smlabb(__a, __b, __c);
312}
313static __inline__ int32_t __attribute__((__always_inline__, __nodebug__))
314__smlabt(int32_t __a, int32_t __b, int32_t __c) {
315  return __builtin_arm_smlabt(__a, __b, __c);
316}
317static __inline__ int32_t __attribute__((__always_inline__, __nodebug__))
318__smlatb(int32_t __a, int32_t __b, int32_t __c) {
319  return __builtin_arm_smlatb(__a, __b, __c);
320}
321static __inline__ int32_t __attribute__((__always_inline__, __nodebug__))
322__smlatt(int32_t __a, int32_t __b, int32_t __c) {
323  return __builtin_arm_smlatt(__a, __b, __c);
324}
325static __inline__ int32_t __attribute__((__always_inline__, __nodebug__))
326__smlawb(int32_t __a, int32_t __b, int32_t __c) {
327  return __builtin_arm_smlawb(__a, __b, __c);
328}
329static __inline__ int32_t __attribute__((__always_inline__, __nodebug__))
330__smlawt(int32_t __a, int32_t __b, int32_t __c) {
331  return __builtin_arm_smlawt(__a, __b, __c);
332}
333#endif
334
335
336/* 9.5.4 Parallel 16-bit saturation */
337#if __ARM_FEATURE_SIMD32
338#define __ssat16(x, y) __builtin_arm_ssat16(x, y)
339#define __usat16(x, y) __builtin_arm_usat16(x, y)
340#endif
341
342/* 9.5.5 Packing and unpacking */
343#if __ARM_FEATURE_SIMD32
344typedef int32_t int8x4_t;
345typedef int32_t int16x2_t;
346typedef uint32_t uint8x4_t;
347typedef uint32_t uint16x2_t;
348
349static __inline__ int16x2_t __attribute__((__always_inline__, __nodebug__))
350__sxtab16(int16x2_t __a, int8x4_t __b) {
351  return __builtin_arm_sxtab16(__a, __b);
352}
353static __inline__ int16x2_t __attribute__((__always_inline__, __nodebug__))
354__sxtb16(int8x4_t __a) {
355  return __builtin_arm_sxtb16(__a);
356}
357static __inline__ int16x2_t __attribute__((__always_inline__, __nodebug__))
358__uxtab16(int16x2_t __a, int8x4_t __b) {
359  return __builtin_arm_uxtab16(__a, __b);
360}
361static __inline__ int16x2_t __attribute__((__always_inline__, __nodebug__))
362__uxtb16(int8x4_t __a) {
363  return __builtin_arm_uxtb16(__a);
364}
365#endif
366
367/* 9.5.6 Parallel selection */
368#if __ARM_FEATURE_SIMD32
369static __inline__ uint8x4_t __attribute__((__always_inline__, __nodebug__))
370__sel(uint8x4_t __a, uint8x4_t __b) {
371  return __builtin_arm_sel(__a, __b);
372}
373#endif
374
375/* 9.5.7 Parallel 8-bit addition and subtraction */
376#if __ARM_FEATURE_SIMD32
377static __inline__ int8x4_t __attribute__((__always_inline__, __nodebug__))
378__qadd8(int8x4_t __a, int8x4_t __b) {
379  return __builtin_arm_qadd8(__a, __b);
380}
381static __inline__ int8x4_t __attribute__((__always_inline__, __nodebug__))
382__qsub8(int8x4_t __a, int8x4_t __b) {
383  return __builtin_arm_qsub8(__a, __b);
384}
385static __inline__ int8x4_t __attribute__((__always_inline__, __nodebug__))
386__sadd8(int8x4_t __a, int8x4_t __b) {
387  return __builtin_arm_sadd8(__a, __b);
388}
389static __inline__ int8x4_t __attribute__((__always_inline__, __nodebug__))
390__shadd8(int8x4_t __a, int8x4_t __b) {
391  return __builtin_arm_shadd8(__a, __b);
392}
393static __inline__ int8x4_t __attribute__((__always_inline__, __nodebug__))
394__shsub8(int8x4_t __a, int8x4_t __b) {
395  return __builtin_arm_shsub8(__a, __b);
396}
397static __inline__ int8x4_t __attribute__((__always_inline__, __nodebug__))
398__ssub8(int8x4_t __a, int8x4_t __b) {
399  return __builtin_arm_ssub8(__a, __b);
400}
401static __inline__ uint8x4_t __attribute__((__always_inline__, __nodebug__))
402__uadd8(uint8x4_t __a, uint8x4_t __b) {
403  return __builtin_arm_uadd8(__a, __b);
404}
405static __inline__ uint8x4_t __attribute__((__always_inline__, __nodebug__))
406__uhadd8(uint8x4_t __a, uint8x4_t __b) {
407  return __builtin_arm_uhadd8(__a, __b);
408}
409static __inline__ uint8x4_t __attribute__((__always_inline__, __nodebug__))
410__uhsub8(uint8x4_t __a, uint8x4_t __b) {
411  return __builtin_arm_uhsub8(__a, __b);
412}
413static __inline__ uint8x4_t __attribute__((__always_inline__, __nodebug__))
414__uqadd8(uint8x4_t __a, uint8x4_t __b) {
415  return __builtin_arm_uqadd8(__a, __b);
416}
417static __inline__ uint8x4_t __attribute__((__always_inline__, __nodebug__))
418__uqsub8(uint8x4_t __a, uint8x4_t __b) {
419  return __builtin_arm_uqsub8(__a, __b);
420}
421static __inline__ uint8x4_t __attribute__((__always_inline__, __nodebug__))
422__usub8(uint8x4_t __a, uint8x4_t __b) {
423  return __builtin_arm_usub8(__a, __b);
424}
425#endif
426
427/* 9.5.8 Sum of 8-bit absolute differences */
428#if __ARM_FEATURE_SIMD32
429static __inline__ uint32_t __attribute__((__always_inline__, __nodebug__))
430__usad8(uint8x4_t __a, uint8x4_t __b) {
431  return __builtin_arm_usad8(__a, __b);
432}
433static __inline__ uint32_t __attribute__((__always_inline__, __nodebug__))
434__usada8(uint8x4_t __a, uint8x4_t __b, uint32_t __c) {
435  return __builtin_arm_usada8(__a, __b, __c);
436}
437#endif
438
439/* 9.5.9 Parallel 16-bit addition and subtraction */
440#if __ARM_FEATURE_SIMD32
441static __inline__ int16x2_t __attribute__((__always_inline__, __nodebug__))
442__qadd16(int16x2_t __a, int16x2_t __b) {
443  return __builtin_arm_qadd16(__a, __b);
444}
445static __inline__ int16x2_t __attribute__((__always_inline__, __nodebug__))
446__qasx(int16x2_t __a, int16x2_t __b) {
447  return __builtin_arm_qasx(__a, __b);
448}
449static __inline__ int16x2_t __attribute__((__always_inline__, __nodebug__))
450__qsax(int16x2_t __a, int16x2_t __b) {
451  return __builtin_arm_qsax(__a, __b);
452}
453static __inline__ int16x2_t __attribute__((__always_inline__, __nodebug__))
454__qsub16(int16x2_t __a, int16x2_t __b) {
455  return __builtin_arm_qsub16(__a, __b);
456}
457static __inline__ int16x2_t __attribute__((__always_inline__, __nodebug__))
458__sadd16(int16x2_t __a, int16x2_t __b) {
459  return __builtin_arm_sadd16(__a, __b);
460}
461static __inline__ int16x2_t __attribute__((__always_inline__, __nodebug__))
462__sasx(int16x2_t __a, int16x2_t __b) {
463  return __builtin_arm_sasx(__a, __b);
464}
465static __inline__ int16x2_t __attribute__((__always_inline__, __nodebug__))
466__shadd16(int16x2_t __a, int16x2_t __b) {
467  return __builtin_arm_shadd16(__a, __b);
468}
469static __inline__ int16x2_t __attribute__((__always_inline__, __nodebug__))
470__shasx(int16x2_t __a, int16x2_t __b) {
471  return __builtin_arm_shasx(__a, __b);
472}
473static __inline__ int16x2_t __attribute__((__always_inline__, __nodebug__))
474__shsax(int16x2_t __a, int16x2_t __b) {
475  return __builtin_arm_shsax(__a, __b);
476}
477static __inline__ int16x2_t __attribute__((__always_inline__, __nodebug__))
478__shsub16(int16x2_t __a, int16x2_t __b) {
479  return __builtin_arm_shsub16(__a, __b);
480}
481static __inline__ int16x2_t __attribute__((__always_inline__, __nodebug__))
482__ssax(int16x2_t __a, int16x2_t __b) {
483  return __builtin_arm_ssax(__a, __b);
484}
485static __inline__ int16x2_t __attribute__((__always_inline__, __nodebug__))
486__ssub16(int16x2_t __a, int16x2_t __b) {
487  return __builtin_arm_ssub16(__a, __b);
488}
489static __inline__ uint16x2_t __attribute__((__always_inline__, __nodebug__))
490__uadd16(uint16x2_t __a, uint16x2_t __b) {
491  return __builtin_arm_uadd16(__a, __b);
492}
493static __inline__ uint16x2_t __attribute__((__always_inline__, __nodebug__))
494__uasx(uint16x2_t __a, uint16x2_t __b) {
495  return __builtin_arm_uasx(__a, __b);
496}
497static __inline__ uint16x2_t __attribute__((__always_inline__, __nodebug__))
498__uhadd16(uint16x2_t __a, uint16x2_t __b) {
499  return __builtin_arm_uhadd16(__a, __b);
500}
501static __inline__ uint16x2_t __attribute__((__always_inline__, __nodebug__))
502__uhasx(uint16x2_t __a, uint16x2_t __b) {
503  return __builtin_arm_uhasx(__a, __b);
504}
505static __inline__ uint16x2_t __attribute__((__always_inline__, __nodebug__))
506__uhsax(uint16x2_t __a, uint16x2_t __b) {
507  return __builtin_arm_uhsax(__a, __b);
508}
509static __inline__ uint16x2_t __attribute__((__always_inline__, __nodebug__))
510__uhsub16(uint16x2_t __a, uint16x2_t __b) {
511  return __builtin_arm_uhsub16(__a, __b);
512}
513static __inline__ uint16x2_t __attribute__((__always_inline__, __nodebug__))
514__uqadd16(uint16x2_t __a, uint16x2_t __b) {
515  return __builtin_arm_uqadd16(__a, __b);
516}
517static __inline__ uint16x2_t __attribute__((__always_inline__, __nodebug__))
518__uqasx(uint16x2_t __a, uint16x2_t __b) {
519  return __builtin_arm_uqasx(__a, __b);
520}
521static __inline__ uint16x2_t __attribute__((__always_inline__, __nodebug__))
522__uqsax(uint16x2_t __a, uint16x2_t __b) {
523  return __builtin_arm_uqsax(__a, __b);
524}
525static __inline__ uint16x2_t __attribute__((__always_inline__, __nodebug__))
526__uqsub16(uint16x2_t __a, uint16x2_t __b) {
527  return __builtin_arm_uqsub16(__a, __b);
528}
529static __inline__ uint16x2_t __attribute__((__always_inline__, __nodebug__))
530__usax(uint16x2_t __a, uint16x2_t __b) {
531  return __builtin_arm_usax(__a, __b);
532}
533static __inline__ uint16x2_t __attribute__((__always_inline__, __nodebug__))
534__usub16(uint16x2_t __a, uint16x2_t __b) {
535  return __builtin_arm_usub16(__a, __b);
536}
537#endif
538
539/* 9.5.10 Parallel 16-bit multiplications */
540#if __ARM_FEATURE_SIMD32
541static __inline__ int32_t __attribute__((__always_inline__, __nodebug__))
542__smlad(int16x2_t __a, int16x2_t __b, int32_t __c) {
543  return __builtin_arm_smlad(__a, __b, __c);
544}
545static __inline__ int32_t __attribute__((__always_inline__, __nodebug__))
546__smladx(int16x2_t __a, int16x2_t __b, int32_t __c) {
547  return __builtin_arm_smladx(__a, __b, __c);
548}
549static __inline__ int64_t __attribute__((__always_inline__, __nodebug__))
550__smlald(int16x2_t __a, int16x2_t __b, int64_t __c) {
551  return __builtin_arm_smlald(__a, __b, __c);
552}
553static __inline__ int64_t __attribute__((__always_inline__, __nodebug__))
554__smlaldx(int16x2_t __a, int16x2_t __b, int64_t __c) {
555  return __builtin_arm_smlaldx(__a, __b, __c);
556}
557static __inline__ int32_t __attribute__((__always_inline__, __nodebug__))
558__smlsd(int16x2_t __a, int16x2_t __b, int32_t __c) {
559  return __builtin_arm_smlsd(__a, __b, __c);
560}
561static __inline__ int32_t __attribute__((__always_inline__, __nodebug__))
562__smlsdx(int16x2_t __a, int16x2_t __b, int32_t __c) {
563  return __builtin_arm_smlsdx(__a, __b, __c);
564}
565static __inline__ int64_t __attribute__((__always_inline__, __nodebug__))
566__smlsld(int16x2_t __a, int16x2_t __b, int64_t __c) {
567  return __builtin_arm_smlsld(__a, __b, __c);
568}
569static __inline__ int64_t __attribute__((__always_inline__, __nodebug__))
570__smlsldx(int16x2_t __a, int16x2_t __b, int64_t __c) {
571  return __builtin_arm_smlsldx(__a, __b, __c);
572}
573static __inline__ int32_t __attribute__((__always_inline__, __nodebug__))
574__smuad(int16x2_t __a, int16x2_t __b) {
575  return __builtin_arm_smuad(__a, __b);
576}
577static __inline__ int32_t __attribute__((__always_inline__, __nodebug__))
578__smuadx(int16x2_t __a, int16x2_t __b) {
579  return __builtin_arm_smuadx(__a, __b);
580}
581static __inline__ int32_t __attribute__((__always_inline__, __nodebug__))
582__smusd(int16x2_t __a, int16x2_t __b) {
583  return __builtin_arm_smusd(__a, __b);
584}
585static __inline__ int32_t __attribute__((__always_inline__, __nodebug__))
586__smusdx(int16x2_t __a, int16x2_t __b) {
587  return __builtin_arm_smusdx(__a, __b);
588}
589#endif
590
591/* 9.7 CRC32 intrinsics */
592#if __ARM_FEATURE_CRC32
593static __inline__ uint32_t __attribute__((__always_inline__, __nodebug__))
594__crc32b(uint32_t __a, uint8_t __b) {
595  return __builtin_arm_crc32b(__a, __b);
596}
597
598static __inline__ uint32_t __attribute__((__always_inline__, __nodebug__))
599__crc32h(uint32_t __a, uint16_t __b) {
600  return __builtin_arm_crc32h(__a, __b);
601}
602
603static __inline__ uint32_t __attribute__((__always_inline__, __nodebug__))
604__crc32w(uint32_t __a, uint32_t __b) {
605  return __builtin_arm_crc32w(__a, __b);
606}
607
608static __inline__ uint32_t __attribute__((__always_inline__, __nodebug__))
609__crc32d(uint32_t __a, uint64_t __b) {
610  return __builtin_arm_crc32d(__a, __b);
611}
612
613static __inline__ uint32_t __attribute__((__always_inline__, __nodebug__))
614__crc32cb(uint32_t __a, uint8_t __b) {
615  return __builtin_arm_crc32cb(__a, __b);
616}
617
618static __inline__ uint32_t __attribute__((__always_inline__, __nodebug__))
619__crc32ch(uint32_t __a, uint16_t __b) {
620  return __builtin_arm_crc32ch(__a, __b);
621}
622
623static __inline__ uint32_t __attribute__((__always_inline__, __nodebug__))
624__crc32cw(uint32_t __a, uint32_t __b) {
625  return __builtin_arm_crc32cw(__a, __b);
626}
627
628static __inline__ uint32_t __attribute__((__always_inline__, __nodebug__))
629__crc32cd(uint32_t __a, uint64_t __b) {
630  return __builtin_arm_crc32cd(__a, __b);
631}
632#endif
633
634/* Armv8.3-A Javascript conversion intrinsic */
635#if __ARM_64BIT_STATE && defined(__ARM_FEATURE_JCVT)
636static __inline__ int32_t __attribute__((__always_inline__, __nodebug__))
637__jcvt(double __a) {
638  return __builtin_arm_jcvt(__a);
639}
640#endif
641
642/* Armv8.5-A FP rounding intrinsics */
643#if __ARM_64BIT_STATE && defined(__ARM_FEATURE_FRINT)
644static __inline__ float __attribute__((__always_inline__, __nodebug__))
645__frint32zf(float __a) {
646  return __builtin_arm_frint32zf(__a);
647}
648
649static __inline__ double __attribute__((__always_inline__, __nodebug__))
650__frint32z(double __a) {
651  return __builtin_arm_frint32z(__a);
652}
653
654static __inline__ float __attribute__((__always_inline__, __nodebug__))
655__frint64zf(float __a) {
656  return __builtin_arm_frint64zf(__a);
657}
658
659static __inline__ double __attribute__((__always_inline__, __nodebug__))
660__frint64z(double __a) {
661  return __builtin_arm_frint64z(__a);
662}
663
664static __inline__ float __attribute__((__always_inline__, __nodebug__))
665__frint32xf(float __a) {
666  return __builtin_arm_frint32xf(__a);
667}
668
669static __inline__ double __attribute__((__always_inline__, __nodebug__))
670__frint32x(double __a) {
671  return __builtin_arm_frint32x(__a);
672}
673
674static __inline__ float __attribute__((__always_inline__, __nodebug__))
675__frint64xf(float __a) {
676  return __builtin_arm_frint64xf(__a);
677}
678
679static __inline__ double __attribute__((__always_inline__, __nodebug__))
680__frint64x(double __a) {
681  return __builtin_arm_frint64x(__a);
682}
683#endif
684
685/* Armv8.7-A load/store 64-byte intrinsics */
686#if __ARM_64BIT_STATE && defined(__ARM_FEATURE_LS64)
687typedef struct {
688    uint64_t val[8];
689} data512_t;
690
691static __inline__ data512_t __attribute__((__always_inline__, __nodebug__))
692__arm_ld64b(const void *__addr) {
693    data512_t __value;
694    __builtin_arm_ld64b(__addr, __value.val);
695    return __value;
696}
697static __inline__ void __attribute__((__always_inline__, __nodebug__))
698__arm_st64b(void *__addr, data512_t __value) {
699    __builtin_arm_st64b(__addr, __value.val);
700}
701static __inline__ uint64_t __attribute__((__always_inline__, __nodebug__))
702__arm_st64bv(void *__addr, data512_t __value) {
703    return __builtin_arm_st64bv(__addr, __value.val);
704}
705static __inline__ uint64_t __attribute__((__always_inline__, __nodebug__))
706__arm_st64bv0(void *__addr, data512_t __value) {
707    return __builtin_arm_st64bv0(__addr, __value.val);
708}
709#endif
710
711/* 10.1 Special register intrinsics */
712#define __arm_rsr(sysreg) __builtin_arm_rsr(sysreg)
713#define __arm_rsr64(sysreg) __builtin_arm_rsr64(sysreg)
714#define __arm_rsrp(sysreg) __builtin_arm_rsrp(sysreg)
715#define __arm_rsrf(sysreg) __builtin_bit_cast(float, __arm_rsr(sysreg))
716#define __arm_rsrf64(sysreg) __builtin_bit_cast(double, __arm_rsr64(sysreg))
717#define __arm_wsr(sysreg, v) __builtin_arm_wsr(sysreg, v)
718#define __arm_wsr64(sysreg, v) __builtin_arm_wsr64(sysreg, v)
719#define __arm_wsrp(sysreg, v) __builtin_arm_wsrp(sysreg, v)
720#define __arm_wsrf(sysreg, v) __arm_wsr(sysreg, __builtin_bit_cast(uint32_t, v))
721#define __arm_wsrf64(sysreg, v) __arm_wsr64(sysreg, __builtin_bit_cast(uint64_t, v))
722
723/* Memory Tagging Extensions (MTE) Intrinsics */
724#if __ARM_FEATURE_MEMORY_TAGGING
725#define __arm_mte_create_random_tag(__ptr, __mask)  __builtin_arm_irg(__ptr, __mask)
726#define __arm_mte_increment_tag(__ptr, __tag_offset)  __builtin_arm_addg(__ptr, __tag_offset)
727#define __arm_mte_exclude_tag(__ptr, __excluded)  __builtin_arm_gmi(__ptr, __excluded)
728#define __arm_mte_get_tag(__ptr) __builtin_arm_ldg(__ptr)
729#define __arm_mte_set_tag(__ptr) __builtin_arm_stg(__ptr)
730#define __arm_mte_ptrdiff(__ptra, __ptrb) __builtin_arm_subp(__ptra, __ptrb)
731#endif
732
733/* Transactional Memory Extension (TME) Intrinsics */
734#if __ARM_FEATURE_TME
735
736#define _TMFAILURE_REASON  0x00007fffu
737#define _TMFAILURE_RTRY    0x00008000u
738#define _TMFAILURE_CNCL    0x00010000u
739#define _TMFAILURE_MEM     0x00020000u
740#define _TMFAILURE_IMP     0x00040000u
741#define _TMFAILURE_ERR     0x00080000u
742#define _TMFAILURE_SIZE    0x00100000u
743#define _TMFAILURE_NEST    0x00200000u
744#define _TMFAILURE_DBG     0x00400000u
745#define _TMFAILURE_INT     0x00800000u
746#define _TMFAILURE_TRIVIAL 0x01000000u
747
748#define __tstart()        __builtin_arm_tstart()
749#define __tcommit()       __builtin_arm_tcommit()
750#define __tcancel(__arg)  __builtin_arm_tcancel(__arg)
751#define __ttest()         __builtin_arm_ttest()
752
753#endif /* __ARM_FEATURE_TME */
754
755/* Armv8.5-A Random number generation intrinsics */
756#if __ARM_64BIT_STATE && defined(__ARM_FEATURE_RNG)
757static __inline__ int __attribute__((__always_inline__, __nodebug__))
758__rndr(uint64_t *__p) {
759  return __builtin_arm_rndr(__p);
760}
761static __inline__ int __attribute__((__always_inline__, __nodebug__))
762__rndrrs(uint64_t *__p) {
763  return __builtin_arm_rndrrs(__p);
764}
765#endif
766
767#if defined(__cplusplus)
768}
769#endif
770
771#endif /* __ARM_ACLE_H */
772