1309260Scognet/* 2309260Scognet * Copyright 2009-2015 Samy Al Bahra. 3309260Scognet * Copyright 2012 Jo��o Fernandes. 4309260Scognet * All rights reserved. 5309260Scognet * 6309260Scognet * Redistribution and use in source and binary forms, with or without 7309260Scognet * modification, are permitted provided that the following conditions 8309260Scognet * are met: 9309260Scognet * 1. Redistributions of source code must retain the above copyright 10309260Scognet * notice, this list of conditions and the following disclaimer. 11309260Scognet * 2. Redistributions in binary form must reproduce the above copyright 12309260Scognet * notice, this list of conditions and the following disclaimer in the 13309260Scognet * documentation and/or other materials provided with the distribution. 14309260Scognet * 15309260Scognet * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 16309260Scognet * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17309260Scognet * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18309260Scognet * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 19309260Scognet * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20309260Scognet * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21309260Scognet * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22309260Scognet * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23309260Scognet * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24309260Scognet * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25309260Scognet * SUCH DAMAGE. 26309260Scognet */ 27309260Scognet 28309260Scognet#ifndef CK_PR_PPC_H 29309260Scognet#define CK_PR_PPC_H 30309260Scognet 31309260Scognet#ifndef CK_PR_H 32309260Scognet#error Do not include this file directly, use ck_pr.h 33309260Scognet#endif 34309260Scognet 35309260Scognet#include <ck_cc.h> 36309260Scognet#include <ck_md.h> 37309260Scognet 38309260Scognet/* 39309260Scognet * The following represent supported atomic operations. 40309260Scognet * These operations may be emulated. 41309260Scognet */ 42309260Scognet#include "ck_f_pr.h" 43309260Scognet 44309260Scognet/* 45309260Scognet * Minimum interface requirement met. 46309260Scognet */ 47309260Scognet#define CK_F_PR 48309260Scognet 49309260Scognet/* 50309260Scognet * This bounces the hardware thread from low to medium 51309260Scognet * priority. I am unsure of the benefits of this approach 52309260Scognet * but it is used by the Linux kernel. 53309260Scognet */ 54309260ScognetCK_CC_INLINE static void 55309260Scognetck_pr_stall(void) 56309260Scognet{ 57309260Scognet 58309260Scognet __asm__ __volatile__("or 1, 1, 1;" 59309260Scognet "or 2, 2, 2;" ::: "memory"); 60309260Scognet return; 61309260Scognet} 62309260Scognet 63309260Scognet#define CK_PR_FENCE(T, I) \ 64309260Scognet CK_CC_INLINE static void \ 65309260Scognet ck_pr_fence_strict_##T(void) \ 66309260Scognet { \ 67309260Scognet __asm__ __volatile__(I ::: "memory"); \ 68309260Scognet } 69309260Scognet 70343494Smarius#ifdef CK_MD_PPC32_LWSYNC 71343494Smarius#define CK_PR_LWSYNCOP "lwsync" 72343494Smarius#else /* CK_MD_PPC32_LWSYNC_DISABLE */ 73343494Smarius#define CK_PR_LWSYNCOP "sync" 74343494Smarius#endif 75343494Smarius 76343494SmariusCK_PR_FENCE(atomic, CK_PR_LWSYNCOP) 77343494SmariusCK_PR_FENCE(atomic_store, CK_PR_LWSYNCOP) 78309260ScognetCK_PR_FENCE(atomic_load, "sync") 79343494SmariusCK_PR_FENCE(store_atomic, CK_PR_LWSYNCOP) 80343494SmariusCK_PR_FENCE(load_atomic, CK_PR_LWSYNCOP) 81343494SmariusCK_PR_FENCE(store, CK_PR_LWSYNCOP) 82309260ScognetCK_PR_FENCE(store_load, "sync") 83343494SmariusCK_PR_FENCE(load, CK_PR_LWSYNCOP) 84343494SmariusCK_PR_FENCE(load_store, CK_PR_LWSYNCOP) 85309260ScognetCK_PR_FENCE(memory, "sync") 86343494SmariusCK_PR_FENCE(acquire, CK_PR_LWSYNCOP) 87343494SmariusCK_PR_FENCE(release, CK_PR_LWSYNCOP) 88343494SmariusCK_PR_FENCE(acqrel, CK_PR_LWSYNCOP) 89343494SmariusCK_PR_FENCE(lock, CK_PR_LWSYNCOP) 90343494SmariusCK_PR_FENCE(unlock, CK_PR_LWSYNCOP) 91309260Scognet 92343494Smarius#undef CK_PR_LWSYNCOP 93343494Smarius 94309260Scognet#undef CK_PR_FENCE 95309260Scognet 96309260Scognet#define CK_PR_LOAD(S, M, T, C, I) \ 97309260Scognet CK_CC_INLINE static T \ 98309260Scognet ck_pr_md_load_##S(const M *target) \ 99309260Scognet { \ 100309260Scognet T r; \ 101309260Scognet __asm__ __volatile__(I "%U1%X1 %0, %1" \ 102309260Scognet : "=r" (r) \ 103309260Scognet : "m" (*(const C *)target) \ 104309260Scognet : "memory"); \ 105309260Scognet return (r); \ 106309260Scognet } 107309260Scognet 108309260ScognetCK_PR_LOAD(ptr, void, void *, uint32_t, "lwz") 109309260Scognet 110309260Scognet#define CK_PR_LOAD_S(S, T, I) CK_PR_LOAD(S, T, T, T, I) 111309260Scognet 112309260ScognetCK_PR_LOAD_S(32, uint32_t, "lwz") 113309260ScognetCK_PR_LOAD_S(16, uint16_t, "lhz") 114309260ScognetCK_PR_LOAD_S(8, uint8_t, "lbz") 115309260ScognetCK_PR_LOAD_S(uint, unsigned int, "lwz") 116309260ScognetCK_PR_LOAD_S(int, int, "lwz") 117309260ScognetCK_PR_LOAD_S(short, short, "lhz") 118309260ScognetCK_PR_LOAD_S(char, char, "lbz") 119309260Scognet 120309260Scognet#undef CK_PR_LOAD_S 121309260Scognet#undef CK_PR_LOAD 122309260Scognet 123309260Scognet#define CK_PR_STORE(S, M, T, C, I) \ 124309260Scognet CK_CC_INLINE static void \ 125309260Scognet ck_pr_md_store_##S(M *target, T v) \ 126309260Scognet { \ 127309260Scognet __asm__ __volatile__(I "%U0%X0 %1, %0" \ 128309260Scognet : "=m" (*(C *)target) \ 129309260Scognet : "r" (v) \ 130309260Scognet : "memory"); \ 131309260Scognet return; \ 132309260Scognet } 133309260Scognet 134309260ScognetCK_PR_STORE(ptr, void, const void *, uint32_t, "stw") 135309260Scognet 136309260Scognet#define CK_PR_STORE_S(S, T, I) CK_PR_STORE(S, T, T, T, I) 137309260Scognet 138309260ScognetCK_PR_STORE_S(32, uint32_t, "stw") 139309260ScognetCK_PR_STORE_S(16, uint16_t, "sth") 140309260ScognetCK_PR_STORE_S(8, uint8_t, "stb") 141309260ScognetCK_PR_STORE_S(uint, unsigned int, "stw") 142309260ScognetCK_PR_STORE_S(int, int, "stw") 143309260ScognetCK_PR_STORE_S(short, short, "sth") 144309260ScognetCK_PR_STORE_S(char, char, "stb") 145309260Scognet 146309260Scognet#undef CK_PR_STORE_S 147309260Scognet#undef CK_PR_STORE 148309260Scognet 149309260Scognet#define CK_PR_CAS(N, T, M) \ 150309260Scognet CK_CC_INLINE static bool \ 151309260Scognet ck_pr_cas_##N##_value(M *target, T compare, T set, M *value) \ 152309260Scognet { \ 153309260Scognet T previous; \ 154309260Scognet __asm__ __volatile__("1:" \ 155309260Scognet "lwarx %0, 0, %1;" \ 156309260Scognet "cmpw 0, %0, %3;" \ 157309260Scognet "bne- 2f;" \ 158309260Scognet "stwcx. %2, 0, %1;" \ 159309260Scognet "bne- 1b;" \ 160309260Scognet "2:" \ 161309260Scognet : "=&r" (previous) \ 162309260Scognet : "r" (target), \ 163309260Scognet "r" (set), \ 164309260Scognet "r" (compare) \ 165309260Scognet : "memory", "cc"); \ 166309260Scognet *(T *)value = previous; \ 167309260Scognet return (previous == compare); \ 168309260Scognet } \ 169309260Scognet CK_CC_INLINE static bool \ 170309260Scognet ck_pr_cas_##N(M *target, T compare, T set) \ 171309260Scognet { \ 172309260Scognet T previous; \ 173309260Scognet __asm__ __volatile__("1:" \ 174309260Scognet "lwarx %0, 0, %1;" \ 175309260Scognet "cmpw 0, %0, %3;" \ 176309260Scognet "bne- 2f;" \ 177309260Scognet "stwcx. %2, 0, %1;" \ 178309260Scognet "bne- 1b;" \ 179309260Scognet "2:" \ 180309260Scognet : "=&r" (previous) \ 181309260Scognet : "r" (target), \ 182309260Scognet "r" (set), \ 183309260Scognet "r" (compare) \ 184309260Scognet : "memory", "cc"); \ 185309260Scognet return (previous == compare); \ 186309260Scognet } 187309260Scognet 188309260ScognetCK_PR_CAS(ptr, void *, void) 189309260Scognet#define CK_PR_CAS_S(a, b) CK_PR_CAS(a, b, b) 190309260ScognetCK_PR_CAS_S(32, uint32_t) 191309260ScognetCK_PR_CAS_S(uint, unsigned int) 192309260ScognetCK_PR_CAS_S(int, int) 193309260Scognet 194309260Scognet#undef CK_PR_CAS_S 195309260Scognet#undef CK_PR_CAS 196309260Scognet 197309260Scognet#define CK_PR_FAS(N, M, T, W) \ 198309260Scognet CK_CC_INLINE static T \ 199309260Scognet ck_pr_fas_##N(M *target, T v) \ 200309260Scognet { \ 201309260Scognet T previous; \ 202309260Scognet __asm__ __volatile__("1:" \ 203309260Scognet "l" W "arx %0, 0, %1;" \ 204309260Scognet "st" W "cx. %2, 0, %1;" \ 205309260Scognet "bne- 1b;" \ 206309260Scognet : "=&r" (previous) \ 207309260Scognet : "r" (target), \ 208309260Scognet "r" (v) \ 209309260Scognet : "memory", "cc"); \ 210309260Scognet return (previous); \ 211309260Scognet } 212309260Scognet 213309260ScognetCK_PR_FAS(32, uint32_t, uint32_t, "w") 214309260ScognetCK_PR_FAS(ptr, void, void *, "w") 215309260ScognetCK_PR_FAS(int, int, int, "w") 216309260ScognetCK_PR_FAS(uint, unsigned int, unsigned int, "w") 217309260Scognet 218309260Scognet#undef CK_PR_FAS 219309260Scognet 220309260Scognet#define CK_PR_UNARY(O, N, M, T, I, W) \ 221309260Scognet CK_CC_INLINE static void \ 222309260Scognet ck_pr_##O##_##N(M *target) \ 223309260Scognet { \ 224309260Scognet T previous; \ 225309260Scognet __asm__ __volatile__("1:" \ 226309260Scognet "l" W "arx %0, 0, %1;" \ 227309260Scognet I ";" \ 228309260Scognet "st" W "cx. %0, 0, %1;" \ 229309260Scognet "bne- 1b;" \ 230309260Scognet : "=&r" (previous) \ 231309260Scognet : "r" (target) \ 232309260Scognet : "memory", "cc"); \ 233309260Scognet return; \ 234309260Scognet } 235309260Scognet 236309260ScognetCK_PR_UNARY(inc, ptr, void, void *, "addic %0, %0, 1", "w") 237309260ScognetCK_PR_UNARY(dec, ptr, void, void *, "addic %0, %0, -1", "w") 238309260ScognetCK_PR_UNARY(not, ptr, void, void *, "not %0, %0", "w") 239309260ScognetCK_PR_UNARY(neg, ptr, void, void *, "neg %0, %0", "w") 240309260Scognet 241309260Scognet#define CK_PR_UNARY_S(S, T, W) \ 242309260Scognet CK_PR_UNARY(inc, S, T, T, "addic %0, %0, 1", W) \ 243309260Scognet CK_PR_UNARY(dec, S, T, T, "addic %0, %0, -1", W) \ 244309260Scognet CK_PR_UNARY(not, S, T, T, "not %0, %0", W) \ 245309260Scognet CK_PR_UNARY(neg, S, T, T, "neg %0, %0", W) 246309260Scognet 247309260ScognetCK_PR_UNARY_S(32, uint32_t, "w") 248309260ScognetCK_PR_UNARY_S(uint, unsigned int, "w") 249309260ScognetCK_PR_UNARY_S(int, int, "w") 250309260Scognet 251309260Scognet#undef CK_PR_UNARY_S 252309260Scognet#undef CK_PR_UNARY 253309260Scognet 254309260Scognet#define CK_PR_BINARY(O, N, M, T, I, W) \ 255309260Scognet CK_CC_INLINE static void \ 256309260Scognet ck_pr_##O##_##N(M *target, T delta) \ 257309260Scognet { \ 258309260Scognet T previous; \ 259309260Scognet __asm__ __volatile__("1:" \ 260309260Scognet "l" W "arx %0, 0, %1;" \ 261309260Scognet I " %0, %2, %0;" \ 262309260Scognet "st" W "cx. %0, 0, %1;" \ 263309260Scognet "bne- 1b;" \ 264309260Scognet : "=&r" (previous) \ 265309260Scognet : "r" (target), \ 266309260Scognet "r" (delta) \ 267309260Scognet : "memory", "cc"); \ 268309260Scognet return; \ 269309260Scognet } 270309260Scognet 271309260ScognetCK_PR_BINARY(and, ptr, void, uintptr_t, "and", "w") 272309260ScognetCK_PR_BINARY(add, ptr, void, uintptr_t, "add", "w") 273309260ScognetCK_PR_BINARY(or, ptr, void, uintptr_t, "or", "w") 274309260ScognetCK_PR_BINARY(sub, ptr, void, uintptr_t, "sub", "w") 275309260ScognetCK_PR_BINARY(xor, ptr, void, uintptr_t, "xor", "w") 276309260Scognet 277309260Scognet#define CK_PR_BINARY_S(S, T, W) \ 278309260Scognet CK_PR_BINARY(and, S, T, T, "and", W) \ 279309260Scognet CK_PR_BINARY(add, S, T, T, "add", W) \ 280309260Scognet CK_PR_BINARY(or, S, T, T, "or", W) \ 281309260Scognet CK_PR_BINARY(sub, S, T, T, "subf", W) \ 282309260Scognet CK_PR_BINARY(xor, S, T, T, "xor", W) 283309260Scognet 284309260ScognetCK_PR_BINARY_S(32, uint32_t, "w") 285309260ScognetCK_PR_BINARY_S(uint, unsigned int, "w") 286309260ScognetCK_PR_BINARY_S(int, int, "w") 287309260Scognet 288309260Scognet#undef CK_PR_BINARY_S 289309260Scognet#undef CK_PR_BINARY 290309260Scognet 291309260ScognetCK_CC_INLINE static void * 292309260Scognetck_pr_faa_ptr(void *target, uintptr_t delta) 293309260Scognet{ 294309260Scognet uintptr_t previous, r; 295309260Scognet 296309260Scognet __asm__ __volatile__("1:" 297309260Scognet "lwarx %0, 0, %2;" 298309260Scognet "add %1, %3, %0;" 299309260Scognet "stwcx. %1, 0, %2;" 300309260Scognet "bne- 1b;" 301309260Scognet : "=&r" (previous), 302309260Scognet "=&r" (r) 303309260Scognet : "r" (target), 304309260Scognet "r" (delta) 305309260Scognet : "memory", "cc"); 306309260Scognet 307309260Scognet return (void *)(previous); 308309260Scognet} 309309260Scognet 310309260Scognet#define CK_PR_FAA(S, T, W) \ 311309260Scognet CK_CC_INLINE static T \ 312309260Scognet ck_pr_faa_##S(T *target, T delta) \ 313309260Scognet { \ 314309260Scognet T previous, r; \ 315309260Scognet __asm__ __volatile__("1:" \ 316309260Scognet "l" W "arx %0, 0, %2;" \ 317309260Scognet "add %1, %3, %0;" \ 318309260Scognet "st" W "cx. %1, 0, %2;" \ 319309260Scognet "bne- 1b;" \ 320309260Scognet : "=&r" (previous), \ 321309260Scognet "=&r" (r) \ 322309260Scognet : "r" (target), \ 323309260Scognet "r" (delta) \ 324309260Scognet : "memory", "cc"); \ 325309260Scognet return (previous); \ 326309260Scognet } 327309260Scognet 328309260ScognetCK_PR_FAA(32, uint32_t, "w") 329309260ScognetCK_PR_FAA(uint, unsigned int, "w") 330309260ScognetCK_PR_FAA(int, int, "w") 331309260Scognet 332309260Scognet#undef CK_PR_FAA 333309260Scognet 334309260Scognet#endif /* CK_PR_PPC_H */ 335309260Scognet 336