1//===-- sync-ops.h - --===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements outline macros for the __sync_fetch_and_*
10// operations. Different instantiations will generate appropriate assembly for
11// ARM and Thumb-2 versions of the functions.
12//
13//===----------------------------------------------------------------------===//
14
15#include "../assembly.h"
16
17#if __ARM_ARCH >= 7
18#define DMB dmb
19#elif __ARM_ARCH >= 6
20#define DMB mcr p15, #0, r0, c7, c10, #5
21#else
22#error DMB is only supported on ARMv6+
23#endif
24
25#define SYNC_OP_4(op)                                                          \
26  .p2align 2;                                                                  \
27  .syntax unified;                                                             \
28  DEFINE_COMPILERRT_FUNCTION(__sync_fetch_and_##op)                            \
29  DMB;                                                                         \
30  mov r12, r0;                                                                 \
31  LOCAL_LABEL(tryatomic_##op) : ldrex r0, [r12];                               \
32  op(r2, r0, r1);                                                              \
33  strex r3, r2, [r12];                                                         \
34  cmp r3, #0;                                                                  \
35  bne LOCAL_LABEL(tryatomic_##op);                                             \
36  DMB;                                                                         \
37  bx lr
38
39#define SYNC_OP_8(op)                                                          \
40  .p2align 2;                                                                  \
41  .syntax unified;                                                             \
42  DEFINE_COMPILERRT_FUNCTION(__sync_fetch_and_##op)                            \
43  push {r4, r5, r6, lr};                                                       \
44  DMB;                                                                         \
45  mov r12, r0;                                                                 \
46  LOCAL_LABEL(tryatomic_##op) : ldrexd r0, r1, [r12];                          \
47  op(r4, r5, r0, r1, r2, r3);                                                  \
48  strexd r6, r4, r5, [r12];                                                    \
49  cmp r6, #0;                                                                  \
50  bne LOCAL_LABEL(tryatomic_##op);                                             \
51  DMB;                                                                         \
52  pop { r4, r5, r6, pc }
53
54#define MINMAX_4(rD, rN, rM, cmp_kind)                                         \
55  cmp rN, rM;                                                                  \
56  mov rD, rM;                                                                  \
57  it cmp_kind;                                                                 \
58  mov##cmp_kind rD, rN
59
60#define MINMAX_8(rD_LO, rD_HI, rN_LO, rN_HI, rM_LO, rM_HI, cmp_kind)           \
61  cmp rN_LO, rM_LO;                                                            \
62  sbcs rN_HI, rM_HI;                                                           \
63  mov rD_LO, rM_LO;                                                            \
64  mov rD_HI, rM_HI;                                                            \
65  itt cmp_kind;                                                                \
66  mov##cmp_kind rD_LO, rN_LO;                                                  \
67  mov##cmp_kind rD_HI, rN_HI
68