1/* Linux-specific atomic operations for PA Linux. 2 Copyright (C) 2008-2022 Free Software Foundation, Inc. 3 Based on code contributed by CodeSourcery for ARM EABI Linux. 4 Modifications for PA Linux by Helge Deller <deller@gmx.de> 5 6This file is part of GCC. 7 8GCC is free software; you can redistribute it and/or modify it under 9the terms of the GNU General Public License as published by the Free 10Software Foundation; either version 3, or (at your option) any later 11version. 12 13GCC is distributed in the hope that it will be useful, but WITHOUT ANY 14WARRANTY; without even the implied warranty of MERCHANTABILITY or 15FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 16for more details. 17 18Under Section 7 of GPL version 3, you are granted additional 19permissions described in the GCC Runtime Library Exception, version 203.1, as published by the Free Software Foundation. 21 22You should have received a copy of the GNU General Public License and 23a copy of the GCC Runtime Library Exception along with this program; 24see the files COPYING3 and COPYING.RUNTIME respectively. If not, see 25<http://www.gnu.org/licenses/>. */ 26 27#define EFAULT 14 28#define EBUSY 16 29#define ENOSYS 251 30 31#define _ASM_EFAULT "-14" 32 33typedef unsigned char u8; 34typedef short unsigned int u16; 35#ifdef __LP64__ 36typedef long unsigned int u64; 37#else 38typedef long long unsigned int u64; 39#endif 40 41/* PA-RISC 2.0 supports out-of-order execution for loads and stores. 42 Thus, we need to synchonize memory accesses. For more info, see: 43 "Advanced Performance Features of the 64-bit PA-8000" by Doug Hunt. 44 45 We implement byte, short and int versions of each atomic operation 46 using the kernel helper defined below. There is no support for 47 64-bit operations yet. */ 48 49/* Determine kernel LWS function call (0=32-bit, 1=64-bit userspace). */ 50#define LWS_CAS (sizeof(long) == 4 ? 0 : 1) 51 52/* Kernel helper for compare-and-exchange a 32-bit value. */ 53static inline long 54__kernel_cmpxchg (volatile void *mem, int oldval, int newval) 55{ 56 register unsigned long lws_mem asm("r26") = (unsigned long) (mem); 57 register int lws_old asm("r25") = oldval; 58 register int lws_new asm("r24") = newval; 59 register long lws_ret asm("r28"); 60 register long lws_errno asm("r21"); 61 asm volatile ( "ble 0xb0(%%sr2, %%r0) \n\t" 62 "ldi %2, %%r20 \n\t" 63 "cmpiclr,<> " _ASM_EFAULT ", %%r21, %%r0\n\t" 64 "iitlbp %%r0,(%%sr0, %%r0) \n\t" 65 : "=r" (lws_ret), "=r" (lws_errno) 66 : "i" (LWS_CAS), "r" (lws_mem), "r" (lws_old), "r" (lws_new) 67 : "r1", "r20", "r22", "r23", "r29", "r31", "memory" 68 ); 69 70 /* If the kernel LWS call succeeded (lws_errno == 0), lws_ret contains 71 the old value from memory. If this value is equal to OLDVAL, the 72 new value was written to memory. If not, return -EBUSY. */ 73 if (!lws_errno && lws_ret != oldval) 74 return -EBUSY; 75 76 return lws_errno; 77} 78 79static inline long 80__kernel_cmpxchg2 (volatile void *mem, const void *oldval, const void *newval, 81 int val_size) 82{ 83 register unsigned long lws_mem asm("r26") = (unsigned long) (mem); 84 register unsigned long lws_old asm("r25") = (unsigned long) oldval; 85 register unsigned long lws_new asm("r24") = (unsigned long) newval; 86 register int lws_size asm("r23") = val_size; 87 register long lws_ret asm("r28"); 88 register long lws_errno asm("r21"); 89 asm volatile ( "ble 0xb0(%%sr2, %%r0) \n\t" 90 "ldi %6, %%r20 \n\t" 91 "cmpiclr,<> " _ASM_EFAULT ", %%r21, %%r0\n\t" 92 "iitlbp %%r0,(%%sr0, %%r0) \n\t" 93 : "=r" (lws_ret), "=r" (lws_errno), "+r" (lws_mem), 94 "+r" (lws_old), "+r" (lws_new), "+r" (lws_size) 95 : "i" (2) 96 : "r1", "r20", "r22", "r29", "r31", "fr4", "memory" 97 ); 98 99 /* If the kernel LWS call is successful, lws_ret contains 0. */ 100 if (__builtin_expect (lws_ret == 0, 1)) 101 return 0; 102 103 /* If the kernel LWS call fails with no error, return -EBUSY */ 104 if (__builtin_expect (!lws_errno, 0)) 105 return -EBUSY; 106 107 return lws_errno; 108} 109#define HIDDEN __attribute__ ((visibility ("hidden"))) 110 111/* Big endian masks */ 112#define INVERT_MASK_1 24 113#define INVERT_MASK_2 16 114 115#define MASK_1 0xffu 116#define MASK_2 0xffffu 117 118#define FETCH_AND_OP_2(OP, PFX_OP, INF_OP, TYPE, WIDTH, INDEX) \ 119 TYPE HIDDEN \ 120 __sync_fetch_and_##OP##_##WIDTH (volatile void *ptr, TYPE val) \ 121 { \ 122 TYPE tmp, newval; \ 123 long failure; \ 124 \ 125 do { \ 126 tmp = __atomic_load_n ((volatile TYPE *)ptr, __ATOMIC_RELAXED); \ 127 newval = PFX_OP (tmp INF_OP val); \ 128 failure = __kernel_cmpxchg2 (ptr, &tmp, &newval, INDEX); \ 129 } while (failure != 0); \ 130 \ 131 return tmp; \ 132 } 133 134FETCH_AND_OP_2 (add, , +, u64, 8, 3) 135FETCH_AND_OP_2 (sub, , -, u64, 8, 3) 136FETCH_AND_OP_2 (or, , |, u64, 8, 3) 137FETCH_AND_OP_2 (and, , &, u64, 8, 3) 138FETCH_AND_OP_2 (xor, , ^, u64, 8, 3) 139FETCH_AND_OP_2 (nand, ~, &, u64, 8, 3) 140 141FETCH_AND_OP_2 (add, , +, u16, 2, 1) 142FETCH_AND_OP_2 (sub, , -, u16, 2, 1) 143FETCH_AND_OP_2 (or, , |, u16, 2, 1) 144FETCH_AND_OP_2 (and, , &, u16, 2, 1) 145FETCH_AND_OP_2 (xor, , ^, u16, 2, 1) 146FETCH_AND_OP_2 (nand, ~, &, u16, 2, 1) 147 148FETCH_AND_OP_2 (add, , +, u8, 1, 0) 149FETCH_AND_OP_2 (sub, , -, u8, 1, 0) 150FETCH_AND_OP_2 (or, , |, u8, 1, 0) 151FETCH_AND_OP_2 (and, , &, u8, 1, 0) 152FETCH_AND_OP_2 (xor, , ^, u8, 1, 0) 153FETCH_AND_OP_2 (nand, ~, &, u8, 1, 0) 154 155#define OP_AND_FETCH_2(OP, PFX_OP, INF_OP, TYPE, WIDTH, INDEX) \ 156 TYPE HIDDEN \ 157 __sync_##OP##_and_fetch_##WIDTH (volatile void *ptr, TYPE val) \ 158 { \ 159 TYPE tmp, newval; \ 160 long failure; \ 161 \ 162 do { \ 163 tmp = __atomic_load_n ((volatile TYPE *)ptr, __ATOMIC_RELAXED); \ 164 newval = PFX_OP (tmp INF_OP val); \ 165 failure = __kernel_cmpxchg2 (ptr, &tmp, &newval, INDEX); \ 166 } while (failure != 0); \ 167 \ 168 return PFX_OP (tmp INF_OP val); \ 169 } 170 171OP_AND_FETCH_2 (add, , +, u64, 8, 3) 172OP_AND_FETCH_2 (sub, , -, u64, 8, 3) 173OP_AND_FETCH_2 (or, , |, u64, 8, 3) 174OP_AND_FETCH_2 (and, , &, u64, 8, 3) 175OP_AND_FETCH_2 (xor, , ^, u64, 8, 3) 176OP_AND_FETCH_2 (nand, ~, &, u64, 8, 3) 177 178OP_AND_FETCH_2 (add, , +, u16, 2, 1) 179OP_AND_FETCH_2 (sub, , -, u16, 2, 1) 180OP_AND_FETCH_2 (or, , |, u16, 2, 1) 181OP_AND_FETCH_2 (and, , &, u16, 2, 1) 182OP_AND_FETCH_2 (xor, , ^, u16, 2, 1) 183OP_AND_FETCH_2 (nand, ~, &, u16, 2, 1) 184 185OP_AND_FETCH_2 (add, , +, u8, 1, 0) 186OP_AND_FETCH_2 (sub, , -, u8, 1, 0) 187OP_AND_FETCH_2 (or, , |, u8, 1, 0) 188OP_AND_FETCH_2 (and, , &, u8, 1, 0) 189OP_AND_FETCH_2 (xor, , ^, u8, 1, 0) 190OP_AND_FETCH_2 (nand, ~, &, u8, 1, 0) 191 192#define FETCH_AND_OP_WORD(OP, PFX_OP, INF_OP) \ 193 unsigned int HIDDEN \ 194 __sync_fetch_and_##OP##_4 (volatile void *ptr, unsigned int val) \ 195 { \ 196 unsigned int tmp; \ 197 long failure; \ 198 \ 199 do { \ 200 tmp = __atomic_load_n ((volatile unsigned int *)ptr, \ 201 __ATOMIC_RELAXED); \ 202 failure = __kernel_cmpxchg (ptr, tmp, PFX_OP (tmp INF_OP val)); \ 203 } while (failure != 0); \ 204 \ 205 return tmp; \ 206 } 207 208FETCH_AND_OP_WORD (add, , +) 209FETCH_AND_OP_WORD (sub, , -) 210FETCH_AND_OP_WORD (or, , |) 211FETCH_AND_OP_WORD (and, , &) 212FETCH_AND_OP_WORD (xor, , ^) 213FETCH_AND_OP_WORD (nand, ~, &) 214 215#define OP_AND_FETCH_WORD(OP, PFX_OP, INF_OP) \ 216 unsigned int HIDDEN \ 217 __sync_##OP##_and_fetch_4 (volatile void *ptr, unsigned int val) \ 218 { \ 219 unsigned int tmp; \ 220 long failure; \ 221 \ 222 do { \ 223 tmp = __atomic_load_n ((volatile unsigned int *)ptr, \ 224 __ATOMIC_RELAXED); \ 225 failure = __kernel_cmpxchg (ptr, tmp, PFX_OP (tmp INF_OP val)); \ 226 } while (failure != 0); \ 227 \ 228 return PFX_OP (tmp INF_OP val); \ 229 } 230 231OP_AND_FETCH_WORD (add, , +) 232OP_AND_FETCH_WORD (sub, , -) 233OP_AND_FETCH_WORD (or, , |) 234OP_AND_FETCH_WORD (and, , &) 235OP_AND_FETCH_WORD (xor, , ^) 236OP_AND_FETCH_WORD (nand, ~, &) 237 238typedef unsigned char bool; 239 240#define COMPARE_AND_SWAP_2(TYPE, WIDTH, INDEX) \ 241 TYPE HIDDEN \ 242 __sync_val_compare_and_swap_##WIDTH (volatile void *ptr, TYPE oldval, \ 243 TYPE newval) \ 244 { \ 245 TYPE actual_oldval; \ 246 long fail; \ 247 \ 248 while (1) \ 249 { \ 250 actual_oldval = __atomic_load_n ((volatile TYPE *)ptr, \ 251 __ATOMIC_RELAXED); \ 252 \ 253 if (__builtin_expect (oldval != actual_oldval, 0)) \ 254 return actual_oldval; \ 255 \ 256 fail = __kernel_cmpxchg2 (ptr, &actual_oldval, &newval, INDEX); \ 257 \ 258 if (__builtin_expect (!fail, 1)) \ 259 return actual_oldval; \ 260 } \ 261 } \ 262 \ 263 _Bool HIDDEN \ 264 __sync_bool_compare_and_swap_##WIDTH (volatile void *ptr, \ 265 TYPE oldval, TYPE newval) \ 266 { \ 267 long failure = __kernel_cmpxchg2 (ptr, &oldval, &newval, INDEX); \ 268 return (failure == 0); \ 269 } 270 271COMPARE_AND_SWAP_2 (u64, 8, 3) 272COMPARE_AND_SWAP_2 (u16, 2, 1) 273COMPARE_AND_SWAP_2 (u8, 1, 0) 274 275unsigned int HIDDEN 276__sync_val_compare_and_swap_4 (volatile void *ptr, unsigned int oldval, 277 unsigned int newval) 278{ 279 long fail; 280 unsigned int actual_oldval; 281 282 while (1) 283 { 284 actual_oldval = __atomic_load_n ((volatile unsigned int *)ptr, 285 __ATOMIC_RELAXED); 286 287 if (__builtin_expect (oldval != actual_oldval, 0)) 288 return actual_oldval; 289 290 fail = __kernel_cmpxchg (ptr, actual_oldval, newval); 291 292 if (__builtin_expect (!fail, 1)) 293 return actual_oldval; 294 } 295} 296 297_Bool HIDDEN 298__sync_bool_compare_and_swap_4 (volatile void *ptr, unsigned int oldval, 299 unsigned int newval) 300{ 301 long failure = __kernel_cmpxchg (ptr, oldval, newval); 302 return (failure == 0); 303} 304 305#define SYNC_LOCK_TEST_AND_SET_2(TYPE, WIDTH, INDEX) \ 306TYPE HIDDEN \ 307 __sync_lock_test_and_set_##WIDTH (volatile void *ptr, TYPE val) \ 308 { \ 309 TYPE oldval; \ 310 long failure; \ 311 \ 312 do { \ 313 oldval = __atomic_load_n ((volatile TYPE *)ptr, \ 314 __ATOMIC_RELAXED); \ 315 failure = __kernel_cmpxchg2 (ptr, &oldval, &val, INDEX); \ 316 } while (failure != 0); \ 317 \ 318 return oldval; \ 319 } 320 321SYNC_LOCK_TEST_AND_SET_2 (u64, 8, 3) 322SYNC_LOCK_TEST_AND_SET_2 (u16, 2, 1) 323SYNC_LOCK_TEST_AND_SET_2 (u8, 1, 0) 324 325unsigned int HIDDEN 326__sync_lock_test_and_set_4 (volatile void *ptr, unsigned int val) 327{ 328 long failure; 329 unsigned int oldval; 330 331 do { 332 oldval = __atomic_load_n ((volatile unsigned int *)ptr, __ATOMIC_RELAXED); 333 failure = __kernel_cmpxchg (ptr, oldval, val); 334 } while (failure != 0); 335 336 return oldval; 337} 338 339#define SYNC_LOCK_RELEASE_1(TYPE, WIDTH, INDEX) \ 340 void HIDDEN \ 341 __sync_lock_release_##WIDTH (volatile void *ptr) \ 342 { \ 343 TYPE oldval, val = 0; \ 344 long failure; \ 345 \ 346 do { \ 347 oldval = __atomic_load_n ((volatile TYPE *)ptr, \ 348 __ATOMIC_RELAXED); \ 349 failure = __kernel_cmpxchg2 (ptr, &oldval, &val, INDEX); \ 350 } while (failure != 0); \ 351 } 352 353SYNC_LOCK_RELEASE_1 (u64, 8, 3) 354SYNC_LOCK_RELEASE_1 (u16, 2, 1) 355SYNC_LOCK_RELEASE_1 (u8, 1, 0) 356 357void HIDDEN 358__sync_lock_release_4 (volatile void *ptr) 359{ 360 long failure; 361 unsigned int oldval; 362 363 do { 364 oldval = __atomic_load_n ((volatile unsigned int *)ptr, __ATOMIC_RELAXED); 365 failure = __kernel_cmpxchg (ptr, oldval, 0); 366 } while (failure != 0); 367} 368