1310843Scognet/* 2310843Scognet * Copyright 2009-2016 Samy Al Bahra. 3310843Scognet * Copyright 2013-2016 Olivier Houchard. 4310843Scognet * All rights reserved. 5310843Scognet * 6310843Scognet * Redistribution and use in source and binary forms, with or without 7310843Scognet * modification, are permitted provided that the following conditions 8310843Scognet * are met: 9310843Scognet * 1. Redistributions of source code must retain the above copyright 10310843Scognet * notice, this list of conditions and the following disclaimer. 11310843Scognet * 2. Redistributions in binary form must reproduce the above copyright 12310843Scognet * notice, this list of conditions and the following disclaimer in the 13310843Scognet * documentation and/or other materials provided with the distribution. 14310843Scognet * 15310843Scognet * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 16310843Scognet * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17310843Scognet * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18310843Scognet * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 19310843Scognet * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20310843Scognet * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21310843Scognet * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22310843Scognet * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23310843Scognet * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24310843Scognet * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25310843Scognet * SUCH DAMAGE. 26310843Scognet */ 27310843Scognet 28310843Scognet#ifndef CK_PR_AARCH64_LLSC_H 29310843Scognet#define CK_PR_AARCH64_LLSC_H 30310843Scognet 31310843Scognet#ifndef CK_PR_H 32310843Scognet#error Do not include this file directly, use ck_pr.h 33310843Scognet#endif 34310843Scognet 35310843ScognetCK_CC_INLINE static bool 36310843Scognetck_pr_cas_64_2_value(uint64_t target[2], uint64_t compare[2], uint64_t set[2], uint64_t value[2]) 37310843Scognet{ 38310843Scognet uint64_t tmp1, tmp2; 39310843Scognet 40310843Scognet __asm__ __volatile__("1:" 41310843Scognet "ldxp %0, %1, [%4];" 42310843Scognet "mov %2, %0;" 43310843Scognet "mov %3, %1;" 44310843Scognet "eor %0, %0, %5;" 45310843Scognet "eor %1, %1, %6;" 46310843Scognet "orr %1, %0, %1;" 47310843Scognet "mov %w0, #0;" 48310843Scognet "cbnz %1, 2f;" 49310843Scognet "stxp %w0, %7, %8, [%4];" 50310843Scognet "cbnz %w0, 1b;" 51310843Scognet "mov %w0, #1;" 52310843Scognet "2:" 53310843Scognet : "=&r" (tmp1), "=&r" (tmp2), "=&r" (value[0]), "=&r" (value[1]) 54310843Scognet : "r" (target), "r" (compare[0]), "r" (compare[1]), "r" (set[0]), "r" (set[1]) 55310843Scognet : "cc", "memory"); 56310843Scognet 57310843Scognet return (tmp1); 58310843Scognet} 59310843Scognet 60310843ScognetCK_CC_INLINE static bool 61310843Scognetck_pr_cas_ptr_2_value(void *target, void *compare, void *set, void *value) 62310843Scognet{ 63310843Scognet return (ck_pr_cas_64_2_value(CK_CPP_CAST(uint64_t *, target), 64310843Scognet CK_CPP_CAST(uint64_t *, compare), 65310843Scognet CK_CPP_CAST(uint64_t *, set), 66310843Scognet CK_CPP_CAST(uint64_t *, value))); 67310843Scognet} 68310843Scognet 69310843ScognetCK_CC_INLINE static bool 70310843Scognetck_pr_cas_64_2(uint64_t target[2], uint64_t compare[2], uint64_t set[2]) 71310843Scognet{ 72310843Scognet uint64_t tmp1, tmp2; 73310843Scognet 74310843Scognet __asm__ __volatile__("1:" 75310843Scognet "ldxp %0, %1, [%2];" 76310843Scognet "eor %0, %0, %3;" 77310843Scognet "eor %1, %1, %4;" 78310843Scognet "orr %1, %0, %1;" 79310843Scognet "mov %w0, #0;" 80310843Scognet "cbnz %1, 2f;" 81310843Scognet "stxp %w0, %5, %6, [%2];" 82310843Scognet "cbnz %w0, 1b;" 83310843Scognet "mov %w0, #1;" 84310843Scognet "2:" 85310843Scognet : "=&r" (tmp1), "=&r" (tmp2) 86310843Scognet : "r" (target), "r" (compare[0]), "r" (compare[1]), "r" (set[0]), "r" (set[1]) 87310843Scognet : "cc", "memory"); 88310843Scognet 89310843Scognet return (tmp1); 90310843Scognet} 91310843ScognetCK_CC_INLINE static bool 92310843Scognetck_pr_cas_ptr_2(void *target, void *compare, void *set) 93310843Scognet{ 94310843Scognet return (ck_pr_cas_64_2(CK_CPP_CAST(uint64_t *, target), 95310843Scognet CK_CPP_CAST(uint64_t *, compare), 96310843Scognet CK_CPP_CAST(uint64_t *, set))); 97310843Scognet} 98310843Scognet 99310843Scognet 100310843Scognet#define CK_PR_CAS(N, M, T, W, R) \ 101310843Scognet CK_CC_INLINE static bool \ 102310843Scognet ck_pr_cas_##N##_value(M *target, T compare, T set, M *value) \ 103310843Scognet { \ 104310843Scognet T previous; \ 105310843Scognet T tmp; \ 106310843Scognet __asm__ __volatile__("1:" \ 107310843Scognet "ldxr" W " %" R "0, [%2];" \ 108310843Scognet "cmp %" R "0, %" R "4;" \ 109310843Scognet "b.ne 2f;" \ 110310843Scognet "stxr" W " %w1, %" R "3, [%2];" \ 111310843Scognet "cbnz %w1, 1b;" \ 112310843Scognet "2:" \ 113310843Scognet : "=&r" (previous), \ 114310843Scognet "=&r" (tmp) \ 115310843Scognet : "r" (target), \ 116310843Scognet "r" (set), \ 117310843Scognet "r" (compare) \ 118310843Scognet : "memory", "cc"); \ 119310843Scognet *(T *)value = previous; \ 120310843Scognet return (previous == compare); \ 121310843Scognet } \ 122310843Scognet CK_CC_INLINE static bool \ 123310843Scognet ck_pr_cas_##N(M *target, T compare, T set) \ 124310843Scognet { \ 125310843Scognet T previous; \ 126310843Scognet T tmp; \ 127310843Scognet __asm__ __volatile__( \ 128310843Scognet "1:" \ 129310843Scognet "ldxr" W " %" R "0, [%2];" \ 130310843Scognet "cmp %" R "0, %" R "4;" \ 131310843Scognet "b.ne 2f;" \ 132310843Scognet "stxr" W " %w1, %" R "3, [%2];" \ 133310843Scognet "cbnz %w1, 1b;" \ 134310843Scognet "2:" \ 135310843Scognet : "=&r" (previous), \ 136310843Scognet "=&r" (tmp) \ 137310843Scognet : "r" (target), \ 138310843Scognet "r" (set), \ 139310843Scognet "r" (compare) \ 140310843Scognet : "memory", "cc"); \ 141310843Scognet return (previous == compare); \ 142310843Scognet } 143310843Scognet 144310843ScognetCK_PR_CAS(ptr, void, void *, "", "") 145310843Scognet 146310843Scognet#define CK_PR_CAS_S(N, M, W, R) CK_PR_CAS(N, M, M, W, R) 147310843ScognetCK_PR_CAS_S(64, uint64_t, "", "") 148310843Scognet#ifndef CK_PR_DISABLE_DOUBLE 149310843ScognetCK_PR_CAS_S(double, double, "", "") 150310843Scognet#endif 151310843ScognetCK_PR_CAS_S(32, uint32_t, "", "w") 152310843ScognetCK_PR_CAS_S(uint, unsigned int, "", "w") 153310843ScognetCK_PR_CAS_S(int, int, "", "w") 154310843ScognetCK_PR_CAS_S(16, uint16_t, "h", "w") 155310843ScognetCK_PR_CAS_S(8, uint8_t, "b", "w") 156310843ScognetCK_PR_CAS_S(short, short, "h", "w") 157310843ScognetCK_PR_CAS_S(char, char, "b", "w") 158310843Scognet 159310843Scognet 160310843Scognet#undef CK_PR_CAS_S 161310843Scognet#undef CK_PR_CAS 162310843Scognet 163310843Scognet#define CK_PR_FAS(N, M, T, W, R) \ 164310843Scognet CK_CC_INLINE static T \ 165310843Scognet ck_pr_fas_##N(M *target, T v) \ 166310843Scognet { \ 167310843Scognet T previous; \ 168310843Scognet T tmp; \ 169310843Scognet __asm__ __volatile__("1:" \ 170310843Scognet "ldxr" W " %" R "0, [%2];" \ 171310843Scognet "stxr" W " %w1, %" R "3, [%2];"\ 172310843Scognet "cbnz %w1, 1b;" \ 173310843Scognet : "=&r" (previous), \ 174310843Scognet "=&r" (tmp) \ 175310843Scognet : "r" (target), \ 176310843Scognet "r" (v) \ 177310843Scognet : "memory", "cc"); \ 178310843Scognet return (previous); \ 179310843Scognet } 180310843Scognet 181310843ScognetCK_PR_FAS(64, uint64_t, uint64_t, "", "") 182310843ScognetCK_PR_FAS(32, uint32_t, uint32_t, "", "w") 183310843ScognetCK_PR_FAS(ptr, void, void *, "", "") 184310843ScognetCK_PR_FAS(int, int, int, "", "w") 185310843ScognetCK_PR_FAS(uint, unsigned int, unsigned int, "", "w") 186310843ScognetCK_PR_FAS(16, uint16_t, uint16_t, "h", "w") 187310843ScognetCK_PR_FAS(8, uint8_t, uint8_t, "b", "w") 188310843ScognetCK_PR_FAS(short, short, short, "h", "w") 189310843ScognetCK_PR_FAS(char, char, char, "b", "w") 190310843Scognet 191310843Scognet 192310843Scognet#undef CK_PR_FAS 193310843Scognet 194310843Scognet#define CK_PR_UNARY(O, N, M, T, I, W, R) \ 195310843Scognet CK_CC_INLINE static void \ 196310843Scognet ck_pr_##O##_##N(M *target) \ 197310843Scognet { \ 198310843Scognet T previous = 0; \ 199310843Scognet T tmp = 0; \ 200310843Scognet __asm__ __volatile__("1:" \ 201310843Scognet "ldxr" W " %" R "0, [%2];" \ 202310843Scognet I ";" \ 203310843Scognet "stxr" W " %w1, %" R "0, [%2];" \ 204310843Scognet "cbnz %w1, 1b;" \ 205310843Scognet : "=&r" (previous), \ 206310843Scognet "=&r" (tmp) \ 207310843Scognet : "r" (target) \ 208310843Scognet : "memory", "cc"); \ 209310843Scognet return; \ 210310843Scognet } 211310843Scognet 212310843ScognetCK_PR_UNARY(inc, ptr, void, void *, "add %0, %0, #1", "", "") 213310843ScognetCK_PR_UNARY(dec, ptr, void, void *, "sub %0, %0, #1", "", "") 214310843ScognetCK_PR_UNARY(not, ptr, void, void *, "mvn %0, %0", "", "") 215310843ScognetCK_PR_UNARY(inc, 64, uint64_t, uint64_t, "add %0, %0, #1", "", "") 216310843ScognetCK_PR_UNARY(dec, 64, uint64_t, uint64_t, "sub %0, %0, #1", "", "") 217310843ScognetCK_PR_UNARY(not, 64, uint64_t, uint64_t, "mvn %0, %0", "", "") 218310843Scognet 219310843Scognet#define CK_PR_UNARY_S(S, T, W) \ 220310843Scognet CK_PR_UNARY(inc, S, T, T, "add %w0, %w0, #1", W, "w") \ 221310843Scognet CK_PR_UNARY(dec, S, T, T, "sub %w0, %w0, #1", W, "w") \ 222310843Scognet CK_PR_UNARY(not, S, T, T, "mvn %w0, %w0", W, "w") \ 223310843Scognet 224310843ScognetCK_PR_UNARY_S(32, uint32_t, "") 225310843ScognetCK_PR_UNARY_S(uint, unsigned int, "") 226310843ScognetCK_PR_UNARY_S(int, int, "") 227310843ScognetCK_PR_UNARY_S(16, uint16_t, "h") 228310843ScognetCK_PR_UNARY_S(8, uint8_t, "b") 229310843ScognetCK_PR_UNARY_S(short, short, "h") 230310843ScognetCK_PR_UNARY_S(char, char, "b") 231310843Scognet 232310843Scognet#undef CK_PR_UNARY_S 233310843Scognet#undef CK_PR_UNARY 234310843Scognet 235310843Scognet#define CK_PR_BINARY(O, N, M, T, I, W, R) \ 236310843Scognet CK_CC_INLINE static void \ 237310843Scognet ck_pr_##O##_##N(M *target, T delta) \ 238310843Scognet { \ 239310843Scognet T previous; \ 240310843Scognet T tmp; \ 241310843Scognet __asm__ __volatile__("1:" \ 242310843Scognet "ldxr" W " %" R "0, [%2];"\ 243310843Scognet I " %" R "0, %" R "0, %" R "3;" \ 244310843Scognet "stxr" W " %w1, %" R "0, [%2];" \ 245310843Scognet "cbnz %w1, 1b;" \ 246310843Scognet : "=&r" (previous), \ 247310843Scognet "=&r" (tmp) \ 248310843Scognet : "r" (target), \ 249310843Scognet "r" (delta) \ 250310843Scognet : "memory", "cc"); \ 251310843Scognet return; \ 252310843Scognet } 253310843Scognet 254310843ScognetCK_PR_BINARY(and, ptr, void, uintptr_t, "and", "", "") 255310843ScognetCK_PR_BINARY(add, ptr, void, uintptr_t, "add", "", "") 256310843ScognetCK_PR_BINARY(or, ptr, void, uintptr_t, "orr", "", "") 257310843ScognetCK_PR_BINARY(sub, ptr, void, uintptr_t, "sub", "", "") 258310843ScognetCK_PR_BINARY(xor, ptr, void, uintptr_t, "eor", "", "") 259310843ScognetCK_PR_BINARY(and, 64, uint64_t, uint64_t, "and", "", "") 260310843ScognetCK_PR_BINARY(add, 64, uint64_t, uint64_t, "add", "", "") 261310843ScognetCK_PR_BINARY(or, 64, uint64_t, uint64_t, "orr", "", "") 262310843ScognetCK_PR_BINARY(sub, 64, uint64_t, uint64_t, "sub", "", "") 263310843ScognetCK_PR_BINARY(xor, 64, uint64_t, uint64_t, "eor", "", "") 264310843Scognet 265310843Scognet#define CK_PR_BINARY_S(S, T, W) \ 266310843Scognet CK_PR_BINARY(and, S, T, T, "and", W, "w") \ 267310843Scognet CK_PR_BINARY(add, S, T, T, "add", W, "w") \ 268310843Scognet CK_PR_BINARY(or, S, T, T, "orr", W, "w") \ 269310843Scognet CK_PR_BINARY(sub, S, T, T, "sub", W, "w") \ 270310843Scognet CK_PR_BINARY(xor, S, T, T, "eor", W, "w") 271310843Scognet 272310843ScognetCK_PR_BINARY_S(32, uint32_t, "") 273310843ScognetCK_PR_BINARY_S(uint, unsigned int, "") 274310843ScognetCK_PR_BINARY_S(int, int, "") 275310843ScognetCK_PR_BINARY_S(16, uint16_t, "h") 276310843ScognetCK_PR_BINARY_S(8, uint8_t, "b") 277310843ScognetCK_PR_BINARY_S(short, short, "h") 278310843ScognetCK_PR_BINARY_S(char, char, "b") 279310843Scognet 280310843Scognet#undef CK_PR_BINARY_S 281310843Scognet#undef CK_PR_BINARY 282310843Scognet 283310843ScognetCK_CC_INLINE static void * 284310843Scognetck_pr_faa_ptr(void *target, uintptr_t delta) 285310843Scognet{ 286310843Scognet uintptr_t previous, r, tmp; 287310843Scognet 288310843Scognet __asm__ __volatile__("1:" 289310843Scognet "ldxr %0, [%3];" 290310843Scognet "add %1, %4, %0;" 291310843Scognet "stxr %w2, %1, [%3];" 292310843Scognet "cbnz %w2, 1b;" 293310843Scognet : "=&r" (previous), 294310843Scognet "=&r" (r), 295310843Scognet "=&r" (tmp) 296310843Scognet : "r" (target), 297310843Scognet "r" (delta) 298310843Scognet : "memory", "cc"); 299310843Scognet 300310843Scognet return (void *)(previous); 301310843Scognet} 302310843Scognet 303310843ScognetCK_CC_INLINE static uint64_t 304310843Scognetck_pr_faa_64(uint64_t *target, uint64_t delta) 305310843Scognet{ 306310843Scognet uint64_t previous, r, tmp; 307310843Scognet 308310843Scognet __asm__ __volatile__("1:" 309310843Scognet "ldxr %0, [%3];" 310310843Scognet "add %1, %4, %0;" 311310843Scognet "stxr %w2, %1, [%3];" 312310843Scognet "cbnz %w2, 1b;" 313310843Scognet : "=&r" (previous), 314310843Scognet "=&r" (r), 315310843Scognet "=&r" (tmp) 316310843Scognet : "r" (target), 317310843Scognet "r" (delta) 318310843Scognet : "memory", "cc"); 319310843Scognet 320310843Scognet return (previous); 321310843Scognet} 322310843Scognet 323310843Scognet#define CK_PR_FAA(S, T, W) \ 324310843Scognet CK_CC_INLINE static T \ 325310843Scognet ck_pr_faa_##S(T *target, T delta) \ 326310843Scognet { \ 327310843Scognet T previous, r, tmp; \ 328310843Scognet __asm__ __volatile__("1:" \ 329310843Scognet "ldxr" W " %w0, [%3];" \ 330310843Scognet "add %w1, %w4, %w0;" \ 331310843Scognet "stxr" W " %w2, %w1, [%3];" \ 332310843Scognet "cbnz %w2, 1b;" \ 333310843Scognet : "=&r" (previous), \ 334310843Scognet "=&r" (r), \ 335310843Scognet "=&r" (tmp) \ 336310843Scognet : "r" (target), \ 337310843Scognet "r" (delta) \ 338310843Scognet : "memory", "cc"); \ 339310843Scognet return (previous); \ 340310843Scognet } 341310843Scognet 342310843ScognetCK_PR_FAA(32, uint32_t, "") 343310843ScognetCK_PR_FAA(uint, unsigned int, "") 344310843ScognetCK_PR_FAA(int, int, "") 345310843ScognetCK_PR_FAA(16, uint16_t, "h") 346310843ScognetCK_PR_FAA(8, uint8_t, "b") 347310843ScognetCK_PR_FAA(short, short, "h") 348310843ScognetCK_PR_FAA(char, char, "b") 349310843Scognet 350310843Scognet#undef CK_PR_FAA 351310843Scognet 352310843Scognet#endif /* CK_PR_AARCH64_LLSC_H */ 353