1309260Scognet/* 2309260Scognet * Copyright 2009-2015 Samy Al Bahra. 3309260Scognet * All rights reserved. 4309260Scognet * 5309260Scognet * Redistribution and use in source and binary forms, with or without 6309260Scognet * modification, are permitted provided that the following conditions 7309260Scognet * are met: 8309260Scognet * 1. Redistributions of source code must retain the above copyright 9309260Scognet * notice, this list of conditions and the following disclaimer. 10309260Scognet * 2. Redistributions in binary form must reproduce the above copyright 11309260Scognet * notice, this list of conditions and the following disclaimer in the 12309260Scognet * documentation and/or other materials provided with the distribution. 13309260Scognet * 14309260Scognet * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15309260Scognet * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16309260Scognet * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17309260Scognet * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18309260Scognet * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19309260Scognet * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20309260Scognet * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21309260Scognet * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22309260Scognet * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23309260Scognet * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24309260Scognet * SUCH DAMAGE. 25309260Scognet */ 26309260Scognet 27309260Scognet#ifndef CK_PR_PPC64_H 28309260Scognet#define CK_PR_PPC64_H 29309260Scognet 30309260Scognet#ifndef CK_PR_H 31309260Scognet#error Do not include this file directly, use ck_pr.h 32309260Scognet#endif 33309260Scognet 34309260Scognet#include <ck_cc.h> 35309260Scognet#include <ck_md.h> 36309260Scognet 37309260Scognet/* 38309260Scognet * The following represent supported atomic operations. 39309260Scognet * These operations may be emulated. 40309260Scognet */ 41309260Scognet#include "ck_f_pr.h" 42309260Scognet 43309260Scognet/* 44309260Scognet * Minimum interface requirement met. 45309260Scognet */ 46309260Scognet#define CK_F_PR 47309260Scognet 48309260Scognet/* 49309260Scognet * This bounces the hardware thread from low to medium 50309260Scognet * priority. I am unsure of the benefits of this approach 51309260Scognet * but it is used by the Linux kernel. 52309260Scognet */ 53309260ScognetCK_CC_INLINE static void 54309260Scognetck_pr_stall(void) 55309260Scognet{ 56309260Scognet 57309260Scognet __asm__ __volatile__("or 1, 1, 1;" 58309260Scognet "or 2, 2, 2;" ::: "memory"); 59309260Scognet return; 60309260Scognet} 61309260Scognet 62309260Scognet#define CK_PR_FENCE(T, I) \ 63309260Scognet CK_CC_INLINE static void \ 64309260Scognet ck_pr_fence_strict_##T(void) \ 65309260Scognet { \ 66309260Scognet __asm__ __volatile__(I ::: "memory"); \ 67309260Scognet } 68309260Scognet 69309260Scognet/* 70309260Scognet * These are derived from: 71309260Scognet * http://www.ibm.com/developerworks/systems/articles/powerpc.html 72309260Scognet */ 73309260ScognetCK_PR_FENCE(atomic, "lwsync") 74309260ScognetCK_PR_FENCE(atomic_store, "lwsync") 75309260ScognetCK_PR_FENCE(atomic_load, "sync") 76309260ScognetCK_PR_FENCE(store_atomic, "lwsync") 77309260ScognetCK_PR_FENCE(load_atomic, "lwsync") 78309260ScognetCK_PR_FENCE(store, "lwsync") 79309260ScognetCK_PR_FENCE(store_load, "sync") 80309260ScognetCK_PR_FENCE(load, "lwsync") 81309260ScognetCK_PR_FENCE(load_store, "lwsync") 82309260ScognetCK_PR_FENCE(memory, "sync") 83309260ScognetCK_PR_FENCE(acquire, "lwsync") 84309260ScognetCK_PR_FENCE(release, "lwsync") 85309260ScognetCK_PR_FENCE(acqrel, "lwsync") 86309260ScognetCK_PR_FENCE(lock, "lwsync") 87309260ScognetCK_PR_FENCE(unlock, "lwsync") 88309260Scognet 89309260Scognet#undef CK_PR_FENCE 90309260Scognet 91309260Scognet#define CK_PR_LOAD(S, M, T, C, I) \ 92309260Scognet CK_CC_INLINE static T \ 93309260Scognet ck_pr_md_load_##S(const M *target) \ 94309260Scognet { \ 95309260Scognet T r; \ 96309260Scognet __asm__ __volatile__(I "%U1%X1 %0, %1" \ 97309260Scognet : "=r" (r) \ 98309260Scognet : "m" (*(const C *)target) \ 99309260Scognet : "memory"); \ 100309260Scognet return (r); \ 101309260Scognet } 102309260Scognet 103309260ScognetCK_PR_LOAD(ptr, void, void *, uint64_t, "ld") 104309260Scognet 105309260Scognet#define CK_PR_LOAD_S(S, T, I) CK_PR_LOAD(S, T, T, T, I) 106309260Scognet 107309260ScognetCK_PR_LOAD_S(64, uint64_t, "ld") 108309260ScognetCK_PR_LOAD_S(32, uint32_t, "lwz") 109309260ScognetCK_PR_LOAD_S(16, uint16_t, "lhz") 110309260ScognetCK_PR_LOAD_S(8, uint8_t, "lbz") 111309260ScognetCK_PR_LOAD_S(uint, unsigned int, "lwz") 112309260ScognetCK_PR_LOAD_S(int, int, "lwz") 113309260ScognetCK_PR_LOAD_S(short, short, "lhz") 114309260ScognetCK_PR_LOAD_S(char, char, "lbz") 115328515Scognet#ifndef CK_PR_DISABLE_DOUBLE 116309260ScognetCK_PR_LOAD_S(double, double, "ld") 117328515Scognet#endif 118309260Scognet 119309260Scognet#undef CK_PR_LOAD_S 120309260Scognet#undef CK_PR_LOAD 121309260Scognet 122309260Scognet#define CK_PR_STORE(S, M, T, C, I) \ 123309260Scognet CK_CC_INLINE static void \ 124309260Scognet ck_pr_md_store_##S(M *target, T v) \ 125309260Scognet { \ 126309260Scognet __asm__ __volatile__(I "%U0%X0 %1, %0" \ 127309260Scognet : "=m" (*(C *)target) \ 128309260Scognet : "r" (v) \ 129309260Scognet : "memory"); \ 130309260Scognet return; \ 131309260Scognet } 132309260Scognet 133309260ScognetCK_PR_STORE(ptr, void, const void *, uint64_t, "std") 134309260Scognet 135309260Scognet#define CK_PR_STORE_S(S, T, I) CK_PR_STORE(S, T, T, T, I) 136309260Scognet 137309260ScognetCK_PR_STORE_S(64, uint64_t, "std") 138309260ScognetCK_PR_STORE_S(32, uint32_t, "stw") 139309260ScognetCK_PR_STORE_S(16, uint16_t, "sth") 140309260ScognetCK_PR_STORE_S(8, uint8_t, "stb") 141309260ScognetCK_PR_STORE_S(uint, unsigned int, "stw") 142309260ScognetCK_PR_STORE_S(int, int, "stw") 143309260ScognetCK_PR_STORE_S(short, short, "sth") 144309260ScognetCK_PR_STORE_S(char, char, "stb") 145328515Scognet#ifndef CK_PR_DISABLE_DOUBLE 146309260ScognetCK_PR_STORE_S(double, double, "std") 147328515Scognet#endif 148309260Scognet 149309260Scognet#undef CK_PR_STORE_S 150309260Scognet#undef CK_PR_STORE 151309260Scognet 152309260ScognetCK_CC_INLINE static bool 153309260Scognetck_pr_cas_64_value(uint64_t *target, uint64_t compare, uint64_t set, uint64_t *value) 154309260Scognet{ 155309260Scognet uint64_t previous; 156309260Scognet 157309260Scognet __asm__ __volatile__("1:" 158309260Scognet "ldarx %0, 0, %1;" 159309260Scognet "cmpd 0, %0, %3;" 160309260Scognet "bne- 2f;" 161309260Scognet "stdcx. %2, 0, %1;" 162309260Scognet "bne- 1b;" 163309260Scognet "2:" 164309260Scognet : "=&r" (previous) 165309260Scognet : "r" (target), 166309260Scognet "r" (set), 167309260Scognet "r" (compare) 168309260Scognet : "memory", "cc"); 169309260Scognet 170309260Scognet *value = previous; 171309260Scognet return (previous == compare); 172309260Scognet} 173309260Scognet 174309260ScognetCK_CC_INLINE static bool 175309260Scognetck_pr_cas_ptr_value(void *target, void *compare, void *set, void *value) 176309260Scognet{ 177309260Scognet void *previous; 178309260Scognet 179309260Scognet __asm__ __volatile__("1:" 180309260Scognet "ldarx %0, 0, %1;" 181309260Scognet "cmpd 0, %0, %3;" 182309260Scognet "bne- 2f;" 183309260Scognet "stdcx. %2, 0, %1;" 184309260Scognet "bne- 1b;" 185309260Scognet "2:" 186309260Scognet : "=&r" (previous) 187309260Scognet : "r" (target), 188309260Scognet "r" (set), 189309260Scognet "r" (compare) 190309260Scognet : "memory", "cc"); 191309260Scognet 192309260Scognet ck_pr_md_store_ptr(value, previous); 193309260Scognet return (previous == compare); 194309260Scognet} 195309260Scognet 196309260ScognetCK_CC_INLINE static bool 197309260Scognetck_pr_cas_64(uint64_t *target, uint64_t compare, uint64_t set) 198309260Scognet{ 199309260Scognet uint64_t previous; 200309260Scognet 201309260Scognet __asm__ __volatile__("1:" 202309260Scognet "ldarx %0, 0, %1;" 203309260Scognet "cmpd 0, %0, %3;" 204309260Scognet "bne- 2f;" 205309260Scognet "stdcx. %2, 0, %1;" 206309260Scognet "bne- 1b;" 207309260Scognet "2:" 208309260Scognet : "=&r" (previous) 209309260Scognet : "r" (target), 210309260Scognet "r" (set), 211309260Scognet "r" (compare) 212309260Scognet : "memory", "cc"); 213309260Scognet 214309260Scognet return (previous == compare); 215309260Scognet} 216309260Scognet 217309260ScognetCK_CC_INLINE static bool 218309260Scognetck_pr_cas_ptr(void *target, void *compare, void *set) 219309260Scognet{ 220309260Scognet void *previous; 221309260Scognet 222309260Scognet __asm__ __volatile__("1:" 223309260Scognet "ldarx %0, 0, %1;" 224309260Scognet "cmpd 0, %0, %3;" 225309260Scognet "bne- 2f;" 226309260Scognet "stdcx. %2, 0, %1;" 227309260Scognet "bne- 1b;" 228309260Scognet "2:" 229309260Scognet : "=&r" (previous) 230309260Scognet : "r" (target), 231309260Scognet "r" (set), 232309260Scognet "r" (compare) 233309260Scognet : "memory", "cc"); 234309260Scognet 235309260Scognet return (previous == compare); 236309260Scognet} 237309260Scognet 238309260Scognet#define CK_PR_CAS(N, T) \ 239309260Scognet CK_CC_INLINE static bool \ 240309260Scognet ck_pr_cas_##N##_value(T *target, T compare, T set, T *value) \ 241309260Scognet { \ 242309260Scognet T previous; \ 243309260Scognet __asm__ __volatile__("1:" \ 244309260Scognet "lwarx %0, 0, %1;" \ 245309260Scognet "cmpw 0, %0, %3;" \ 246309260Scognet "bne- 2f;" \ 247309260Scognet "stwcx. %2, 0, %1;" \ 248309260Scognet "bne- 1b;" \ 249309260Scognet "2:" \ 250309260Scognet : "=&r" (previous) \ 251309260Scognet : "r" (target), \ 252309260Scognet "r" (set), \ 253309260Scognet "r" (compare) \ 254309260Scognet : "memory", "cc"); \ 255309260Scognet *value = previous; \ 256309260Scognet return (previous == compare); \ 257309260Scognet } \ 258309260Scognet CK_CC_INLINE static bool \ 259309260Scognet ck_pr_cas_##N(T *target, T compare, T set) \ 260309260Scognet { \ 261309260Scognet T previous; \ 262309260Scognet __asm__ __volatile__("1:" \ 263309260Scognet "lwarx %0, 0, %1;" \ 264309260Scognet "cmpw 0, %0, %3;" \ 265309260Scognet "bne- 2f;" \ 266309260Scognet "stwcx. %2, 0, %1;" \ 267309260Scognet "bne- 1b;" \ 268309260Scognet "2:" \ 269309260Scognet : "=&r" (previous) \ 270309260Scognet : "r" (target), \ 271309260Scognet "r" (set), \ 272309260Scognet "r" (compare) \ 273309260Scognet : "memory", "cc"); \ 274309260Scognet return (previous == compare); \ 275309260Scognet } 276309260Scognet 277309260ScognetCK_PR_CAS(32, uint32_t) 278309260ScognetCK_PR_CAS(uint, unsigned int) 279309260ScognetCK_PR_CAS(int, int) 280309260Scognet 281309260Scognet#undef CK_PR_CAS 282309260Scognet 283309260Scognet#define CK_PR_FAS(N, M, T, W) \ 284309260Scognet CK_CC_INLINE static T \ 285309260Scognet ck_pr_fas_##N(M *target, T v) \ 286309260Scognet { \ 287309260Scognet T previous; \ 288309260Scognet __asm__ __volatile__("1:" \ 289309260Scognet "l" W "arx %0, 0, %1;" \ 290309260Scognet "st" W "cx. %2, 0, %1;" \ 291309260Scognet "bne- 1b;" \ 292309260Scognet : "=&r" (previous) \ 293309260Scognet : "r" (target), \ 294309260Scognet "r" (v) \ 295309260Scognet : "memory", "cc"); \ 296309260Scognet return (previous); \ 297309260Scognet } 298309260Scognet 299309260ScognetCK_PR_FAS(64, uint64_t, uint64_t, "d") 300309260ScognetCK_PR_FAS(32, uint32_t, uint32_t, "w") 301328515Scognet#ifndef CK_PR_DISABLE_DOUBLE 302309260ScognetCK_PR_FAS(double, double, double, "d") 303328515Scognet#endif 304309260ScognetCK_PR_FAS(ptr, void, void *, "d") 305309260ScognetCK_PR_FAS(int, int, int, "w") 306309260ScognetCK_PR_FAS(uint, unsigned int, unsigned int, "w") 307309260Scognet 308309260Scognet#undef CK_PR_FAS 309309260Scognet 310309260Scognet#define CK_PR_UNARY(O, N, M, T, I, W) \ 311309260Scognet CK_CC_INLINE static void \ 312309260Scognet ck_pr_##O##_##N(M *target) \ 313309260Scognet { \ 314309260Scognet T previous; \ 315309260Scognet __asm__ __volatile__("1:" \ 316309260Scognet "l" W "arx %0, 0, %1;" \ 317309260Scognet I ";" \ 318309260Scognet "st" W "cx. %0, 0, %1;" \ 319309260Scognet "bne- 1b;" \ 320309260Scognet : "=&r" (previous) \ 321309260Scognet : "r" (target) \ 322309260Scognet : "memory", "cc"); \ 323309260Scognet return; \ 324309260Scognet } 325309260Scognet 326309260ScognetCK_PR_UNARY(inc, ptr, void, void *, "addic %0, %0, 1", "d") 327309260ScognetCK_PR_UNARY(dec, ptr, void, void *, "addic %0, %0, -1", "d") 328309260ScognetCK_PR_UNARY(not, ptr, void, void *, "not %0, %0", "d") 329309260ScognetCK_PR_UNARY(neg, ptr, void, void *, "neg %0, %0", "d") 330309260Scognet 331309260Scognet#define CK_PR_UNARY_S(S, T, W) \ 332309260Scognet CK_PR_UNARY(inc, S, T, T, "addic %0, %0, 1", W) \ 333309260Scognet CK_PR_UNARY(dec, S, T, T, "addic %0, %0, -1", W) \ 334309260Scognet CK_PR_UNARY(not, S, T, T, "not %0, %0", W) \ 335309260Scognet CK_PR_UNARY(neg, S, T, T, "neg %0, %0", W) 336309260Scognet 337309260ScognetCK_PR_UNARY_S(64, uint64_t, "d") 338309260ScognetCK_PR_UNARY_S(32, uint32_t, "w") 339309260ScognetCK_PR_UNARY_S(uint, unsigned int, "w") 340309260ScognetCK_PR_UNARY_S(int, int, "w") 341309260Scognet 342309260Scognet#undef CK_PR_UNARY_S 343309260Scognet#undef CK_PR_UNARY 344309260Scognet 345309260Scognet#define CK_PR_BINARY(O, N, M, T, I, W) \ 346309260Scognet CK_CC_INLINE static void \ 347309260Scognet ck_pr_##O##_##N(M *target, T delta) \ 348309260Scognet { \ 349309260Scognet T previous; \ 350309260Scognet __asm__ __volatile__("1:" \ 351309260Scognet "l" W "arx %0, 0, %1;" \ 352309260Scognet I " %0, %2, %0;" \ 353309260Scognet "st" W "cx. %0, 0, %1;" \ 354309260Scognet "bne- 1b;" \ 355309260Scognet : "=&r" (previous) \ 356309260Scognet : "r" (target), \ 357309260Scognet "r" (delta) \ 358309260Scognet : "memory", "cc"); \ 359309260Scognet return; \ 360309260Scognet } 361309260Scognet 362309260ScognetCK_PR_BINARY(and, ptr, void, uintptr_t, "and", "d") 363309260ScognetCK_PR_BINARY(add, ptr, void, uintptr_t, "add", "d") 364309260ScognetCK_PR_BINARY(or, ptr, void, uintptr_t, "or", "d") 365309260ScognetCK_PR_BINARY(sub, ptr, void, uintptr_t, "sub", "d") 366309260ScognetCK_PR_BINARY(xor, ptr, void, uintptr_t, "xor", "d") 367309260Scognet 368309260Scognet#define CK_PR_BINARY_S(S, T, W) \ 369309260Scognet CK_PR_BINARY(and, S, T, T, "and", W) \ 370309260Scognet CK_PR_BINARY(add, S, T, T, "add", W) \ 371309260Scognet CK_PR_BINARY(or, S, T, T, "or", W) \ 372309260Scognet CK_PR_BINARY(sub, S, T, T, "subf", W) \ 373309260Scognet CK_PR_BINARY(xor, S, T, T, "xor", W) 374309260Scognet 375309260ScognetCK_PR_BINARY_S(64, uint64_t, "d") 376309260ScognetCK_PR_BINARY_S(32, uint32_t, "w") 377309260ScognetCK_PR_BINARY_S(uint, unsigned int, "w") 378309260ScognetCK_PR_BINARY_S(int, int, "w") 379309260Scognet 380309260Scognet#undef CK_PR_BINARY_S 381309260Scognet#undef CK_PR_BINARY 382309260Scognet 383309260ScognetCK_CC_INLINE static void * 384309260Scognetck_pr_faa_ptr(void *target, uintptr_t delta) 385309260Scognet{ 386309260Scognet uintptr_t previous, r; 387309260Scognet 388309260Scognet __asm__ __volatile__("1:" 389309260Scognet "ldarx %0, 0, %2;" 390309260Scognet "add %1, %3, %0;" 391309260Scognet "stdcx. %1, 0, %2;" 392309260Scognet "bne- 1b;" 393309260Scognet : "=&r" (previous), 394309260Scognet "=&r" (r) 395309260Scognet : "r" (target), 396309260Scognet "r" (delta) 397309260Scognet : "memory", "cc"); 398309260Scognet 399309260Scognet return (void *)(previous); 400309260Scognet} 401309260Scognet 402309260Scognet#define CK_PR_FAA(S, T, W) \ 403309260Scognet CK_CC_INLINE static T \ 404309260Scognet ck_pr_faa_##S(T *target, T delta) \ 405309260Scognet { \ 406309260Scognet T previous, r; \ 407309260Scognet __asm__ __volatile__("1:" \ 408309260Scognet "l" W "arx %0, 0, %2;" \ 409309260Scognet "add %1, %3, %0;" \ 410309260Scognet "st" W "cx. %1, 0, %2;" \ 411309260Scognet "bne- 1b;" \ 412309260Scognet : "=&r" (previous), \ 413309260Scognet "=&r" (r) \ 414309260Scognet : "r" (target), \ 415309260Scognet "r" (delta) \ 416309260Scognet : "memory", "cc"); \ 417309260Scognet return (previous); \ 418309260Scognet } 419309260Scognet 420309260ScognetCK_PR_FAA(64, uint64_t, "d") 421309260ScognetCK_PR_FAA(32, uint32_t, "w") 422309260ScognetCK_PR_FAA(uint, unsigned int, "w") 423309260ScognetCK_PR_FAA(int, int, "w") 424309260Scognet 425309260Scognet#undef CK_PR_FAA 426309260Scognet 427309260Scognet#endif /* CK_PR_PPC64_H */ 428