1/* SPDX-License-Identifier: LGPL-2.1 OR MIT */ 2/* 3 * rseq-ppc-bits.h 4 * 5 * (C) Copyright 2016-2018 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com> 6 * (C) Copyright 2016-2018 - Boqun Feng <boqun.feng@gmail.com> 7 */ 8 9#include "rseq-bits-template.h" 10 11#if defined(RSEQ_TEMPLATE_MO_RELAXED) && \ 12 (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) 13 14static inline __attribute__((always_inline)) 15int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_storev)(intptr_t *v, intptr_t expect, intptr_t newv, int cpu) 16{ 17 RSEQ_INJECT_C(9) 18 19 __asm__ __volatile__ goto ( 20 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 21 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 22#ifdef RSEQ_COMPARE_TWICE 23 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 24 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 25#endif 26 /* Start rseq by storing table entry pointer into rseq_cs. */ 27 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) 28 /* cmp cpuid */ 29 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 30 RSEQ_INJECT_ASM(3) 31 /* cmp @v equal to @expect */ 32 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail]) 33 RSEQ_INJECT_ASM(4) 34#ifdef RSEQ_COMPARE_TWICE 35 /* cmp cpuid */ 36 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 37 /* cmp @v equal to @expect */ 38 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2]) 39#endif 40 /* final store */ 41 RSEQ_ASM_OP_FINAL_STORE(newv, v, 2) 42 RSEQ_INJECT_ASM(5) 43 RSEQ_ASM_DEFINE_ABORT(4, abort) 44 : /* gcc asm goto does not allow outputs */ 45 : [cpu_id] "r" (cpu), 46 [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD), 47 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), 48 [v] "m" (*v), 49 [expect] "r" (expect), 50 [newv] "r" (newv) 51 RSEQ_INJECT_INPUT 52 : "memory", "cc", "r17" 53 RSEQ_INJECT_CLOBBER 54 : abort, cmpfail 55#ifdef RSEQ_COMPARE_TWICE 56 , error1, error2 57#endif 58 ); 59 rseq_after_asm_goto(); 60 return 0; 61abort: 62 rseq_after_asm_goto(); 63 RSEQ_INJECT_FAILED 64 return -1; 65cmpfail: 66 rseq_after_asm_goto(); 67 return 1; 68#ifdef RSEQ_COMPARE_TWICE 69error1: 70 rseq_after_asm_goto(); 71 rseq_bug("cpu_id comparison failed"); 72error2: 73 rseq_after_asm_goto(); 74 rseq_bug("expected value comparison failed"); 75#endif 76} 77 78static inline __attribute__((always_inline)) 79int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpnev_storeoffp_load)(intptr_t *v, intptr_t expectnot, 80 long voffp, intptr_t *load, int cpu) 81{ 82 RSEQ_INJECT_C(9) 83 84 __asm__ __volatile__ goto ( 85 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 86 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 87#ifdef RSEQ_COMPARE_TWICE 88 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 89 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 90#endif 91 /* Start rseq by storing table entry pointer into rseq_cs. */ 92 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) 93 /* cmp cpuid */ 94 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 95 RSEQ_INJECT_ASM(3) 96 /* cmp @v not equal to @expectnot */ 97 RSEQ_ASM_OP_CMPNE(v, expectnot, %l[cmpfail]) 98 RSEQ_INJECT_ASM(4) 99#ifdef RSEQ_COMPARE_TWICE 100 /* cmp cpuid */ 101 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 102 /* cmp @v not equal to @expectnot */ 103 RSEQ_ASM_OP_CMPNE(v, expectnot, %l[error2]) 104#endif 105 /* load the value of @v */ 106 RSEQ_ASM_OP_R_LOAD(v) 107 /* store it in @load */ 108 RSEQ_ASM_OP_R_STORE(load) 109 /* dereference voffp(v) */ 110 RSEQ_ASM_OP_R_LOADX(voffp) 111 /* final store the value at voffp(v) */ 112 RSEQ_ASM_OP_R_FINAL_STORE(v, 2) 113 RSEQ_INJECT_ASM(5) 114 RSEQ_ASM_DEFINE_ABORT(4, abort) 115 : /* gcc asm goto does not allow outputs */ 116 : [cpu_id] "r" (cpu), 117 [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD), 118 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), 119 /* final store input */ 120 [v] "m" (*v), 121 [expectnot] "r" (expectnot), 122 [voffp] "b" (voffp), 123 [load] "m" (*load) 124 RSEQ_INJECT_INPUT 125 : "memory", "cc", "r17" 126 RSEQ_INJECT_CLOBBER 127 : abort, cmpfail 128#ifdef RSEQ_COMPARE_TWICE 129 , error1, error2 130#endif 131 ); 132 rseq_after_asm_goto(); 133 return 0; 134abort: 135 rseq_after_asm_goto(); 136 RSEQ_INJECT_FAILED 137 return -1; 138cmpfail: 139 rseq_after_asm_goto(); 140 return 1; 141#ifdef RSEQ_COMPARE_TWICE 142error1: 143 rseq_after_asm_goto(); 144 rseq_bug("cpu_id comparison failed"); 145error2: 146 rseq_after_asm_goto(); 147 rseq_bug("expected value comparison failed"); 148#endif 149} 150 151static inline __attribute__((always_inline)) 152int RSEQ_TEMPLATE_IDENTIFIER(rseq_addv)(intptr_t *v, intptr_t count, int cpu) 153{ 154 RSEQ_INJECT_C(9) 155 156 __asm__ __volatile__ goto ( 157 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 158#ifdef RSEQ_COMPARE_TWICE 159 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 160#endif 161 /* Start rseq by storing table entry pointer into rseq_cs. */ 162 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) 163 /* cmp cpuid */ 164 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 165 RSEQ_INJECT_ASM(3) 166#ifdef RSEQ_COMPARE_TWICE 167 /* cmp cpuid */ 168 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 169#endif 170 /* load the value of @v */ 171 RSEQ_ASM_OP_R_LOAD(v) 172 /* add @count to it */ 173 RSEQ_ASM_OP_R_ADD(count) 174 /* final store */ 175 RSEQ_ASM_OP_R_FINAL_STORE(v, 2) 176 RSEQ_INJECT_ASM(4) 177 RSEQ_ASM_DEFINE_ABORT(4, abort) 178 : /* gcc asm goto does not allow outputs */ 179 : [cpu_id] "r" (cpu), 180 [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD), 181 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), 182 /* final store input */ 183 [v] "m" (*v), 184 [count] "r" (count) 185 RSEQ_INJECT_INPUT 186 : "memory", "cc", "r17" 187 RSEQ_INJECT_CLOBBER 188 : abort 189#ifdef RSEQ_COMPARE_TWICE 190 , error1 191#endif 192 ); 193 rseq_after_asm_goto(); 194 return 0; 195abort: 196 rseq_after_asm_goto(); 197 RSEQ_INJECT_FAILED 198 return -1; 199#ifdef RSEQ_COMPARE_TWICE 200error1: 201 rseq_after_asm_goto(); 202 rseq_bug("cpu_id comparison failed"); 203#endif 204} 205 206static inline __attribute__((always_inline)) 207int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_cmpeqv_storev)(intptr_t *v, intptr_t expect, 208 intptr_t *v2, intptr_t expect2, 209 intptr_t newv, int cpu) 210{ 211 RSEQ_INJECT_C(9) 212 213 __asm__ __volatile__ goto ( 214 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 215 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 216#ifdef RSEQ_COMPARE_TWICE 217 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 218 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 219 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3]) 220#endif 221 /* Start rseq by storing table entry pointer into rseq_cs. */ 222 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) 223 /* cmp cpuid */ 224 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 225 RSEQ_INJECT_ASM(3) 226 /* cmp @v equal to @expect */ 227 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail]) 228 RSEQ_INJECT_ASM(4) 229 /* cmp @v2 equal to @expct2 */ 230 RSEQ_ASM_OP_CMPEQ(v2, expect2, %l[cmpfail]) 231 RSEQ_INJECT_ASM(5) 232#ifdef RSEQ_COMPARE_TWICE 233 /* cmp cpuid */ 234 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 235 /* cmp @v equal to @expect */ 236 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2]) 237 /* cmp @v2 equal to @expct2 */ 238 RSEQ_ASM_OP_CMPEQ(v2, expect2, %l[error3]) 239#endif 240 /* final store */ 241 RSEQ_ASM_OP_FINAL_STORE(newv, v, 2) 242 RSEQ_INJECT_ASM(6) 243 RSEQ_ASM_DEFINE_ABORT(4, abort) 244 : /* gcc asm goto does not allow outputs */ 245 : [cpu_id] "r" (cpu), 246 [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD), 247 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), 248 /* cmp2 input */ 249 [v2] "m" (*v2), 250 [expect2] "r" (expect2), 251 /* final store input */ 252 [v] "m" (*v), 253 [expect] "r" (expect), 254 [newv] "r" (newv) 255 RSEQ_INJECT_INPUT 256 : "memory", "cc", "r17" 257 RSEQ_INJECT_CLOBBER 258 : abort, cmpfail 259#ifdef RSEQ_COMPARE_TWICE 260 , error1, error2, error3 261#endif 262 ); 263 rseq_after_asm_goto(); 264 return 0; 265abort: 266 rseq_after_asm_goto(); 267 RSEQ_INJECT_FAILED 268 return -1; 269cmpfail: 270 rseq_after_asm_goto(); 271 return 1; 272#ifdef RSEQ_COMPARE_TWICE 273error1: 274 rseq_after_asm_goto(); 275 rseq_bug("cpu_id comparison failed"); 276error2: 277 rseq_after_asm_goto(); 278 rseq_bug("1st expected value comparison failed"); 279error3: 280 rseq_after_asm_goto(); 281 rseq_bug("2nd expected value comparison failed"); 282#endif 283} 284 285#endif /* #if defined(RSEQ_TEMPLATE_MO_RELAXED) && 286 (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) */ 287 288#if (defined(RSEQ_TEMPLATE_MO_RELAXED) || defined(RSEQ_TEMPLATE_MO_RELEASE)) && \ 289 (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) 290 291static inline __attribute__((always_inline)) 292int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trystorev_storev)(intptr_t *v, intptr_t expect, 293 intptr_t *v2, intptr_t newv2, 294 intptr_t newv, int cpu) 295{ 296 RSEQ_INJECT_C(9) 297 298 __asm__ __volatile__ goto ( 299 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 300 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 301#ifdef RSEQ_COMPARE_TWICE 302 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 303 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 304#endif 305 /* Start rseq by storing table entry pointer into rseq_cs. */ 306 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) 307 /* cmp cpuid */ 308 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 309 RSEQ_INJECT_ASM(3) 310 /* cmp @v equal to @expect */ 311 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail]) 312 RSEQ_INJECT_ASM(4) 313#ifdef RSEQ_COMPARE_TWICE 314 /* cmp cpuid */ 315 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 316 /* cmp @v equal to @expect */ 317 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2]) 318#endif 319 /* try store */ 320 RSEQ_ASM_OP_STORE(newv2, v2) 321 RSEQ_INJECT_ASM(5) 322#ifdef RSEQ_TEMPLATE_MO_RELEASE 323 /* for 'release' */ 324 "lwsync\n\t" 325#endif 326 /* final store */ 327 RSEQ_ASM_OP_FINAL_STORE(newv, v, 2) 328 RSEQ_INJECT_ASM(6) 329 RSEQ_ASM_DEFINE_ABORT(4, abort) 330 : /* gcc asm goto does not allow outputs */ 331 : [cpu_id] "r" (cpu), 332 [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD), 333 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), 334 /* try store input */ 335 [v2] "m" (*v2), 336 [newv2] "r" (newv2), 337 /* final store input */ 338 [v] "m" (*v), 339 [expect] "r" (expect), 340 [newv] "r" (newv) 341 RSEQ_INJECT_INPUT 342 : "memory", "cc", "r17" 343 RSEQ_INJECT_CLOBBER 344 : abort, cmpfail 345#ifdef RSEQ_COMPARE_TWICE 346 , error1, error2 347#endif 348 ); 349 rseq_after_asm_goto(); 350 return 0; 351abort: 352 rseq_after_asm_goto(); 353 RSEQ_INJECT_FAILED 354 return -1; 355cmpfail: 356 rseq_after_asm_goto(); 357 return 1; 358#ifdef RSEQ_COMPARE_TWICE 359error1: 360 rseq_after_asm_goto(); 361 rseq_bug("cpu_id comparison failed"); 362error2: 363 rseq_after_asm_goto(); 364 rseq_bug("expected value comparison failed"); 365#endif 366} 367 368static inline __attribute__((always_inline)) 369int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trymemcpy_storev)(intptr_t *v, intptr_t expect, 370 void *dst, void *src, size_t len, 371 intptr_t newv, int cpu) 372{ 373 RSEQ_INJECT_C(9) 374 375 __asm__ __volatile__ goto ( 376 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 377 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 378#ifdef RSEQ_COMPARE_TWICE 379 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 380 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 381#endif 382 /* setup for mempcy */ 383 "mr %%r19, %[len]\n\t" 384 "mr %%r20, %[src]\n\t" 385 "mr %%r21, %[dst]\n\t" 386 /* Start rseq by storing table entry pointer into rseq_cs. */ 387 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) 388 /* cmp cpuid */ 389 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 390 RSEQ_INJECT_ASM(3) 391 /* cmp @v equal to @expect */ 392 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail]) 393 RSEQ_INJECT_ASM(4) 394#ifdef RSEQ_COMPARE_TWICE 395 /* cmp cpuid */ 396 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 397 /* cmp @v equal to @expect */ 398 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2]) 399#endif 400 /* try memcpy */ 401 RSEQ_ASM_OP_R_MEMCPY() 402 RSEQ_INJECT_ASM(5) 403#ifdef RSEQ_TEMPLATE_MO_RELEASE 404 /* for 'release' */ 405 "lwsync\n\t" 406#endif 407 /* final store */ 408 RSEQ_ASM_OP_FINAL_STORE(newv, v, 2) 409 RSEQ_INJECT_ASM(6) 410 /* teardown */ 411 RSEQ_ASM_DEFINE_ABORT(4, abort) 412 : /* gcc asm goto does not allow outputs */ 413 : [cpu_id] "r" (cpu), 414 [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD), 415 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), 416 /* final store input */ 417 [v] "m" (*v), 418 [expect] "r" (expect), 419 [newv] "r" (newv), 420 /* try memcpy input */ 421 [dst] "r" (dst), 422 [src] "r" (src), 423 [len] "r" (len) 424 RSEQ_INJECT_INPUT 425 : "memory", "cc", "r17", "r18", "r19", "r20", "r21" 426 RSEQ_INJECT_CLOBBER 427 : abort, cmpfail 428#ifdef RSEQ_COMPARE_TWICE 429 , error1, error2 430#endif 431 ); 432 rseq_after_asm_goto(); 433 return 0; 434abort: 435 rseq_after_asm_goto(); 436 RSEQ_INJECT_FAILED 437 return -1; 438cmpfail: 439 rseq_after_asm_goto(); 440 return 1; 441#ifdef RSEQ_COMPARE_TWICE 442error1: 443 rseq_after_asm_goto(); 444 rseq_bug("cpu_id comparison failed"); 445error2: 446 rseq_after_asm_goto(); 447 rseq_bug("expected value comparison failed"); 448#endif 449} 450 451#endif /* #if (defined(RSEQ_TEMPLATE_MO_RELAXED) || defined(RSEQ_TEMPLATE_MO_RELEASE)) && 452 (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) */ 453 454#include "rseq-bits-reset.h" 455