1/* 2 * Copyright 2009-2015 Samy Al Bahra. 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24 * SUCH DAMAGE. 25 */ 26 27#ifndef CK_PR_PPC64_H 28#define CK_PR_PPC64_H 29 30#ifndef CK_PR_H 31#error Do not include this file directly, use ck_pr.h 32#endif 33 34#include <ck_cc.h> 35#include <ck_md.h> 36 37/* 38 * The following represent supported atomic operations. 39 * These operations may be emulated. 40 */ 41#include "ck_f_pr.h" 42 43/* 44 * Minimum interface requirement met. 45 */ 46#define CK_F_PR 47 48/* 49 * This bounces the hardware thread from low to medium 50 * priority. I am unsure of the benefits of this approach 51 * but it is used by the Linux kernel. 52 */ 53CK_CC_INLINE static void 54ck_pr_stall(void) 55{ 56 57 __asm__ __volatile__("or 1, 1, 1;" 58 "or 2, 2, 2;" ::: "memory"); 59 return; 60} 61 62#define CK_PR_FENCE(T, I) \ 63 CK_CC_INLINE static void \ 64 ck_pr_fence_strict_##T(void) \ 65 { \ 66 __asm__ __volatile__(I ::: "memory"); \ 67 } 68 69/* 70 * These are derived from: 71 * http://www.ibm.com/developerworks/systems/articles/powerpc.html 72 */ 73CK_PR_FENCE(atomic, "lwsync") 74CK_PR_FENCE(atomic_store, "lwsync") 75CK_PR_FENCE(atomic_load, "sync") 76CK_PR_FENCE(store_atomic, "lwsync") 77CK_PR_FENCE(load_atomic, "lwsync") 78CK_PR_FENCE(store, "lwsync") 79CK_PR_FENCE(store_load, "sync") 80CK_PR_FENCE(load, "lwsync") 81CK_PR_FENCE(load_store, "lwsync") 82CK_PR_FENCE(memory, "sync") 83CK_PR_FENCE(acquire, "lwsync") 84CK_PR_FENCE(release, "lwsync") 85CK_PR_FENCE(acqrel, "lwsync") 86CK_PR_FENCE(lock, "lwsync") 87CK_PR_FENCE(unlock, "lwsync") 88 89#undef CK_PR_FENCE 90 91#define CK_PR_LOAD(S, M, T, C, I) \ 92 CK_CC_INLINE static T \ 93 ck_pr_md_load_##S(const M *target) \ 94 { \ 95 T r; \ 96 __asm__ __volatile__(I "%U1%X1 %0, %1" \ 97 : "=r" (r) \ 98 : "m" (*(const C *)target) \ 99 : "memory"); \ 100 return (r); \ 101 } 102 103CK_PR_LOAD(ptr, void, void *, uint64_t, "ld") 104 105#define CK_PR_LOAD_S(S, T, I) CK_PR_LOAD(S, T, T, T, I) 106 107CK_PR_LOAD_S(64, uint64_t, "ld") 108CK_PR_LOAD_S(32, uint32_t, "lwz") 109CK_PR_LOAD_S(16, uint16_t, "lhz") 110CK_PR_LOAD_S(8, uint8_t, "lbz") 111CK_PR_LOAD_S(uint, unsigned int, "lwz") 112CK_PR_LOAD_S(int, int, "lwz") 113CK_PR_LOAD_S(short, short, "lhz") 114CK_PR_LOAD_S(char, char, "lbz") 115#ifndef CK_PR_DISABLE_DOUBLE 116CK_PR_LOAD_S(double, double, "ld") 117#endif 118 119#undef CK_PR_LOAD_S 120#undef CK_PR_LOAD 121 122#define CK_PR_STORE(S, M, T, C, I) \ 123 CK_CC_INLINE static void \ 124 ck_pr_md_store_##S(M *target, T v) \ 125 { \ 126 __asm__ __volatile__(I "%U0%X0 %1, %0" \ 127 : "=m" (*(C *)target) \ 128 : "r" (v) \ 129 : "memory"); \ 130 return; \ 131 } 132 133CK_PR_STORE(ptr, void, const void *, uint64_t, "std") 134 135#define CK_PR_STORE_S(S, T, I) CK_PR_STORE(S, T, T, T, I) 136 137CK_PR_STORE_S(64, uint64_t, "std") 138CK_PR_STORE_S(32, uint32_t, "stw") 139CK_PR_STORE_S(16, uint16_t, "sth") 140CK_PR_STORE_S(8, uint8_t, "stb") 141CK_PR_STORE_S(uint, unsigned int, "stw") 142CK_PR_STORE_S(int, int, "stw") 143CK_PR_STORE_S(short, short, "sth") 144CK_PR_STORE_S(char, char, "stb") 145#ifndef CK_PR_DISABLE_DOUBLE 146CK_PR_STORE_S(double, double, "std") 147#endif 148 149#undef CK_PR_STORE_S 150#undef CK_PR_STORE 151 152CK_CC_INLINE static bool 153ck_pr_cas_64_value(uint64_t *target, uint64_t compare, uint64_t set, uint64_t *value) 154{ 155 uint64_t previous; 156 157 __asm__ __volatile__("1:" 158 "ldarx %0, 0, %1;" 159 "cmpd 0, %0, %3;" 160 "bne- 2f;" 161 "stdcx. %2, 0, %1;" 162 "bne- 1b;" 163 "2:" 164 : "=&r" (previous) 165 : "r" (target), 166 "r" (set), 167 "r" (compare) 168 : "memory", "cc"); 169 170 *value = previous; 171 return (previous == compare); 172} 173 174CK_CC_INLINE static bool 175ck_pr_cas_ptr_value(void *target, void *compare, void *set, void *value) 176{ 177 void *previous; 178 179 __asm__ __volatile__("1:" 180 "ldarx %0, 0, %1;" 181 "cmpd 0, %0, %3;" 182 "bne- 2f;" 183 "stdcx. %2, 0, %1;" 184 "bne- 1b;" 185 "2:" 186 : "=&r" (previous) 187 : "r" (target), 188 "r" (set), 189 "r" (compare) 190 : "memory", "cc"); 191 192 ck_pr_md_store_ptr(value, previous); 193 return (previous == compare); 194} 195 196CK_CC_INLINE static bool 197ck_pr_cas_64(uint64_t *target, uint64_t compare, uint64_t set) 198{ 199 uint64_t previous; 200 201 __asm__ __volatile__("1:" 202 "ldarx %0, 0, %1;" 203 "cmpd 0, %0, %3;" 204 "bne- 2f;" 205 "stdcx. %2, 0, %1;" 206 "bne- 1b;" 207 "2:" 208 : "=&r" (previous) 209 : "r" (target), 210 "r" (set), 211 "r" (compare) 212 : "memory", "cc"); 213 214 return (previous == compare); 215} 216 217CK_CC_INLINE static bool 218ck_pr_cas_ptr(void *target, void *compare, void *set) 219{ 220 void *previous; 221 222 __asm__ __volatile__("1:" 223 "ldarx %0, 0, %1;" 224 "cmpd 0, %0, %3;" 225 "bne- 2f;" 226 "stdcx. %2, 0, %1;" 227 "bne- 1b;" 228 "2:" 229 : "=&r" (previous) 230 : "r" (target), 231 "r" (set), 232 "r" (compare) 233 : "memory", "cc"); 234 235 return (previous == compare); 236} 237 238#define CK_PR_CAS(N, T) \ 239 CK_CC_INLINE static bool \ 240 ck_pr_cas_##N##_value(T *target, T compare, T set, T *value) \ 241 { \ 242 T previous; \ 243 __asm__ __volatile__("1:" \ 244 "lwarx %0, 0, %1;" \ 245 "cmpw 0, %0, %3;" \ 246 "bne- 2f;" \ 247 "stwcx. %2, 0, %1;" \ 248 "bne- 1b;" \ 249 "2:" \ 250 : "=&r" (previous) \ 251 : "r" (target), \ 252 "r" (set), \ 253 "r" (compare) \ 254 : "memory", "cc"); \ 255 *value = previous; \ 256 return (previous == compare); \ 257 } \ 258 CK_CC_INLINE static bool \ 259 ck_pr_cas_##N(T *target, T compare, T set) \ 260 { \ 261 T previous; \ 262 __asm__ __volatile__("1:" \ 263 "lwarx %0, 0, %1;" \ 264 "cmpw 0, %0, %3;" \ 265 "bne- 2f;" \ 266 "stwcx. %2, 0, %1;" \ 267 "bne- 1b;" \ 268 "2:" \ 269 : "=&r" (previous) \ 270 : "r" (target), \ 271 "r" (set), \ 272 "r" (compare) \ 273 : "memory", "cc"); \ 274 return (previous == compare); \ 275 } 276 277CK_PR_CAS(32, uint32_t) 278CK_PR_CAS(uint, unsigned int) 279CK_PR_CAS(int, int) 280 281#undef CK_PR_CAS 282 283#define CK_PR_FAS(N, M, T, W) \ 284 CK_CC_INLINE static T \ 285 ck_pr_fas_##N(M *target, T v) \ 286 { \ 287 T previous; \ 288 __asm__ __volatile__("1:" \ 289 "l" W "arx %0, 0, %1;" \ 290 "st" W "cx. %2, 0, %1;" \ 291 "bne- 1b;" \ 292 : "=&r" (previous) \ 293 : "r" (target), \ 294 "r" (v) \ 295 : "memory", "cc"); \ 296 return (previous); \ 297 } 298 299CK_PR_FAS(64, uint64_t, uint64_t, "d") 300CK_PR_FAS(32, uint32_t, uint32_t, "w") 301#ifndef CK_PR_DISABLE_DOUBLE 302CK_PR_FAS(double, double, double, "d") 303#endif 304CK_PR_FAS(ptr, void, void *, "d") 305CK_PR_FAS(int, int, int, "w") 306CK_PR_FAS(uint, unsigned int, unsigned int, "w") 307 308#undef CK_PR_FAS 309 310#define CK_PR_UNARY(O, N, M, T, I, W) \ 311 CK_CC_INLINE static void \ 312 ck_pr_##O##_##N(M *target) \ 313 { \ 314 T previous; \ 315 __asm__ __volatile__("1:" \ 316 "l" W "arx %0, 0, %1;" \ 317 I ";" \ 318 "st" W "cx. %0, 0, %1;" \ 319 "bne- 1b;" \ 320 : "=&r" (previous) \ 321 : "r" (target) \ 322 : "memory", "cc"); \ 323 return; \ 324 } 325 326CK_PR_UNARY(inc, ptr, void, void *, "addic %0, %0, 1", "d") 327CK_PR_UNARY(dec, ptr, void, void *, "addic %0, %0, -1", "d") 328CK_PR_UNARY(not, ptr, void, void *, "not %0, %0", "d") 329CK_PR_UNARY(neg, ptr, void, void *, "neg %0, %0", "d") 330 331#define CK_PR_UNARY_S(S, T, W) \ 332 CK_PR_UNARY(inc, S, T, T, "addic %0, %0, 1", W) \ 333 CK_PR_UNARY(dec, S, T, T, "addic %0, %0, -1", W) \ 334 CK_PR_UNARY(not, S, T, T, "not %0, %0", W) \ 335 CK_PR_UNARY(neg, S, T, T, "neg %0, %0", W) 336 337CK_PR_UNARY_S(64, uint64_t, "d") 338CK_PR_UNARY_S(32, uint32_t, "w") 339CK_PR_UNARY_S(uint, unsigned int, "w") 340CK_PR_UNARY_S(int, int, "w") 341 342#undef CK_PR_UNARY_S 343#undef CK_PR_UNARY 344 345#define CK_PR_BINARY(O, N, M, T, I, W) \ 346 CK_CC_INLINE static void \ 347 ck_pr_##O##_##N(M *target, T delta) \ 348 { \ 349 T previous; \ 350 __asm__ __volatile__("1:" \ 351 "l" W "arx %0, 0, %1;" \ 352 I " %0, %2, %0;" \ 353 "st" W "cx. %0, 0, %1;" \ 354 "bne- 1b;" \ 355 : "=&r" (previous) \ 356 : "r" (target), \ 357 "r" (delta) \ 358 : "memory", "cc"); \ 359 return; \ 360 } 361 362CK_PR_BINARY(and, ptr, void, uintptr_t, "and", "d") 363CK_PR_BINARY(add, ptr, void, uintptr_t, "add", "d") 364CK_PR_BINARY(or, ptr, void, uintptr_t, "or", "d") 365CK_PR_BINARY(sub, ptr, void, uintptr_t, "sub", "d") 366CK_PR_BINARY(xor, ptr, void, uintptr_t, "xor", "d") 367 368#define CK_PR_BINARY_S(S, T, W) \ 369 CK_PR_BINARY(and, S, T, T, "and", W) \ 370 CK_PR_BINARY(add, S, T, T, "add", W) \ 371 CK_PR_BINARY(or, S, T, T, "or", W) \ 372 CK_PR_BINARY(sub, S, T, T, "subf", W) \ 373 CK_PR_BINARY(xor, S, T, T, "xor", W) 374 375CK_PR_BINARY_S(64, uint64_t, "d") 376CK_PR_BINARY_S(32, uint32_t, "w") 377CK_PR_BINARY_S(uint, unsigned int, "w") 378CK_PR_BINARY_S(int, int, "w") 379 380#undef CK_PR_BINARY_S 381#undef CK_PR_BINARY 382 383CK_CC_INLINE static void * 384ck_pr_faa_ptr(void *target, uintptr_t delta) 385{ 386 uintptr_t previous, r; 387 388 __asm__ __volatile__("1:" 389 "ldarx %0, 0, %2;" 390 "add %1, %3, %0;" 391 "stdcx. %1, 0, %2;" 392 "bne- 1b;" 393 : "=&r" (previous), 394 "=&r" (r) 395 : "r" (target), 396 "r" (delta) 397 : "memory", "cc"); 398 399 return (void *)(previous); 400} 401 402#define CK_PR_FAA(S, T, W) \ 403 CK_CC_INLINE static T \ 404 ck_pr_faa_##S(T *target, T delta) \ 405 { \ 406 T previous, r; \ 407 __asm__ __volatile__("1:" \ 408 "l" W "arx %0, 0, %2;" \ 409 "add %1, %3, %0;" \ 410 "st" W "cx. %1, 0, %2;" \ 411 "bne- 1b;" \ 412 : "=&r" (previous), \ 413 "=&r" (r) \ 414 : "r" (target), \ 415 "r" (delta) \ 416 : "memory", "cc"); \ 417 return (previous); \ 418 } 419 420CK_PR_FAA(64, uint64_t, "d") 421CK_PR_FAA(32, uint32_t, "w") 422CK_PR_FAA(uint, unsigned int, "w") 423CK_PR_FAA(int, int, "w") 424 425#undef CK_PR_FAA 426 427#endif /* CK_PR_PPC64_H */ 428