1/* SPDX-License-Identifier: LGPL-2.1 OR MIT */ 2 3#include "rseq-bits-template.h" 4 5#if defined(RSEQ_TEMPLATE_MO_RELAXED) && \ 6 (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) 7 8static inline __attribute__((always_inline)) 9int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_storev)(intptr_t *v, intptr_t expect, intptr_t newv, int cpu) 10{ 11 RSEQ_INJECT_C(9) 12 13 __asm__ __volatile__ goto ( 14 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 15 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 16#ifdef RSEQ_COMPARE_TWICE 17 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 18 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 19#endif 20 /* Start rseq by storing table entry pointer into rseq_cs. */ 21 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) 22 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 23 RSEQ_INJECT_ASM(3) 24 LONG_CMP " %[expect], %[v]\n\t" 25 "jnz %l[cmpfail]\n\t" 26 RSEQ_INJECT_ASM(4) 27#ifdef RSEQ_COMPARE_TWICE 28 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 29 LONG_CMP " %[expect], %[v]\n\t" 30 "jnz %l[error2]\n\t" 31#endif 32 /* final store */ 33 LONG_S " %[newv], %[v]\n\t" 34 "2:\n\t" 35 RSEQ_INJECT_ASM(5) 36 RSEQ_ASM_DEFINE_ABORT(4, "", abort) 37 : /* gcc asm goto does not allow outputs */ 38 : [cpu_id] "r" (cpu), 39 [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD), 40 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), 41 [v] "m" (*v), 42 [expect] "r" (expect), 43 [newv] "r" (newv) 44 RSEQ_INJECT_INPUT 45 : "memory", "cc", "r0" 46 RSEQ_INJECT_CLOBBER 47 : abort, cmpfail 48#ifdef RSEQ_COMPARE_TWICE 49 , error1, error2 50#endif 51 ); 52 rseq_after_asm_goto(); 53 return 0; 54abort: 55 rseq_after_asm_goto(); 56 RSEQ_INJECT_FAILED 57 return -1; 58cmpfail: 59 rseq_after_asm_goto(); 60 return 1; 61#ifdef RSEQ_COMPARE_TWICE 62error1: 63 rseq_after_asm_goto(); 64 rseq_bug("cpu_id comparison failed"); 65error2: 66 rseq_after_asm_goto(); 67 rseq_bug("expected value comparison failed"); 68#endif 69} 70 71/* 72 * Compare @v against @expectnot. When it does _not_ match, load @v 73 * into @load, and store the content of *@v + voffp into @v. 74 */ 75static inline __attribute__((always_inline)) 76int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpnev_storeoffp_load)(intptr_t *v, intptr_t expectnot, 77 long voffp, intptr_t *load, int cpu) 78{ 79 RSEQ_INJECT_C(9) 80 81 __asm__ __volatile__ goto ( 82 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 83 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 84#ifdef RSEQ_COMPARE_TWICE 85 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 86 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 87#endif 88 /* Start rseq by storing table entry pointer into rseq_cs. */ 89 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) 90 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 91 RSEQ_INJECT_ASM(3) 92 LONG_L " %%r1, %[v]\n\t" 93 LONG_CMP_R " %%r1, %[expectnot]\n\t" 94 "je %l[cmpfail]\n\t" 95 RSEQ_INJECT_ASM(4) 96#ifdef RSEQ_COMPARE_TWICE 97 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 98 LONG_L " %%r1, %[v]\n\t" 99 LONG_CMP_R " %%r1, %[expectnot]\n\t" 100 "je %l[error2]\n\t" 101#endif 102 LONG_S " %%r1, %[load]\n\t" 103 LONG_ADD_R " %%r1, %[voffp]\n\t" 104 LONG_L " %%r1, 0(%%r1)\n\t" 105 /* final store */ 106 LONG_S " %%r1, %[v]\n\t" 107 "2:\n\t" 108 RSEQ_INJECT_ASM(5) 109 RSEQ_ASM_DEFINE_ABORT(4, "", abort) 110 : /* gcc asm goto does not allow outputs */ 111 : [cpu_id] "r" (cpu), 112 [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD), 113 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), 114 /* final store input */ 115 [v] "m" (*v), 116 [expectnot] "r" (expectnot), 117 [voffp] "r" (voffp), 118 [load] "m" (*load) 119 RSEQ_INJECT_INPUT 120 : "memory", "cc", "r0", "r1" 121 RSEQ_INJECT_CLOBBER 122 : abort, cmpfail 123#ifdef RSEQ_COMPARE_TWICE 124 , error1, error2 125#endif 126 ); 127 rseq_after_asm_goto(); 128 return 0; 129abort: 130 rseq_after_asm_goto(); 131 RSEQ_INJECT_FAILED 132 return -1; 133cmpfail: 134 rseq_after_asm_goto(); 135 return 1; 136#ifdef RSEQ_COMPARE_TWICE 137error1: 138 rseq_after_asm_goto(); 139 rseq_bug("cpu_id comparison failed"); 140error2: 141 rseq_after_asm_goto(); 142 rseq_bug("expected value comparison failed"); 143#endif 144} 145 146static inline __attribute__((always_inline)) 147int RSEQ_TEMPLATE_IDENTIFIER(rseq_addv)(intptr_t *v, intptr_t count, int cpu) 148{ 149 RSEQ_INJECT_C(9) 150 151 __asm__ __volatile__ goto ( 152 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 153#ifdef RSEQ_COMPARE_TWICE 154 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 155#endif 156 /* Start rseq by storing table entry pointer into rseq_cs. */ 157 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) 158 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 159 RSEQ_INJECT_ASM(3) 160#ifdef RSEQ_COMPARE_TWICE 161 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 162#endif 163 LONG_L " %%r0, %[v]\n\t" 164 LONG_ADD_R " %%r0, %[count]\n\t" 165 /* final store */ 166 LONG_S " %%r0, %[v]\n\t" 167 "2:\n\t" 168 RSEQ_INJECT_ASM(4) 169 RSEQ_ASM_DEFINE_ABORT(4, "", abort) 170 : /* gcc asm goto does not allow outputs */ 171 : [cpu_id] "r" (cpu), 172 [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD), 173 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), 174 /* final store input */ 175 [v] "m" (*v), 176 [count] "r" (count) 177 RSEQ_INJECT_INPUT 178 : "memory", "cc", "r0" 179 RSEQ_INJECT_CLOBBER 180 : abort 181#ifdef RSEQ_COMPARE_TWICE 182 , error1 183#endif 184 ); 185 rseq_after_asm_goto(); 186 return 0; 187abort: 188 rseq_after_asm_goto(); 189 RSEQ_INJECT_FAILED 190 return -1; 191#ifdef RSEQ_COMPARE_TWICE 192error1: 193 rseq_after_asm_goto(); 194 rseq_bug("cpu_id comparison failed"); 195#endif 196} 197 198static inline __attribute__((always_inline)) 199int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_cmpeqv_storev)(intptr_t *v, intptr_t expect, 200 intptr_t *v2, intptr_t expect2, 201 intptr_t newv, int cpu) 202{ 203 RSEQ_INJECT_C(9) 204 205 __asm__ __volatile__ goto ( 206 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 207 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 208#ifdef RSEQ_COMPARE_TWICE 209 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 210 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 211 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3]) 212#endif 213 /* Start rseq by storing table entry pointer into rseq_cs. */ 214 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) 215 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 216 RSEQ_INJECT_ASM(3) 217 LONG_CMP " %[expect], %[v]\n\t" 218 "jnz %l[cmpfail]\n\t" 219 RSEQ_INJECT_ASM(4) 220 LONG_CMP " %[expect2], %[v2]\n\t" 221 "jnz %l[cmpfail]\n\t" 222 RSEQ_INJECT_ASM(5) 223#ifdef RSEQ_COMPARE_TWICE 224 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 225 LONG_CMP " %[expect], %[v]\n\t" 226 "jnz %l[error2]\n\t" 227 LONG_CMP " %[expect2], %[v2]\n\t" 228 "jnz %l[error3]\n\t" 229#endif 230 /* final store */ 231 LONG_S " %[newv], %[v]\n\t" 232 "2:\n\t" 233 RSEQ_INJECT_ASM(6) 234 RSEQ_ASM_DEFINE_ABORT(4, "", abort) 235 : /* gcc asm goto does not allow outputs */ 236 : [cpu_id] "r" (cpu), 237 [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD), 238 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), 239 /* cmp2 input */ 240 [v2] "m" (*v2), 241 [expect2] "r" (expect2), 242 /* final store input */ 243 [v] "m" (*v), 244 [expect] "r" (expect), 245 [newv] "r" (newv) 246 RSEQ_INJECT_INPUT 247 : "memory", "cc", "r0" 248 RSEQ_INJECT_CLOBBER 249 : abort, cmpfail 250#ifdef RSEQ_COMPARE_TWICE 251 , error1, error2, error3 252#endif 253 ); 254 rseq_after_asm_goto(); 255 return 0; 256abort: 257 rseq_after_asm_goto(); 258 RSEQ_INJECT_FAILED 259 return -1; 260cmpfail: 261 rseq_after_asm_goto(); 262 return 1; 263#ifdef RSEQ_COMPARE_TWICE 264error1: 265 rseq_after_asm_goto(); 266 rseq_bug("cpu_id comparison failed"); 267error2: 268 rseq_after_asm_goto(); 269 rseq_bug("1st expected value comparison failed"); 270error3: 271 rseq_after_asm_goto(); 272 rseq_bug("2nd expected value comparison failed"); 273#endif 274} 275 276#endif /* #if defined(RSEQ_TEMPLATE_MO_RELAXED) && 277 (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) */ 278 279#if (defined(RSEQ_TEMPLATE_MO_RELAXED) || defined(RSEQ_TEMPLATE_MO_RELEASE)) && \ 280 (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) 281 282/* s390 is TSO. */ 283static inline __attribute__((always_inline)) 284int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trystorev_storev)(intptr_t *v, intptr_t expect, 285 intptr_t *v2, intptr_t newv2, 286 intptr_t newv, int cpu) 287{ 288 RSEQ_INJECT_C(9) 289 290 __asm__ __volatile__ goto ( 291 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 292 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 293#ifdef RSEQ_COMPARE_TWICE 294 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 295 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 296#endif 297 /* Start rseq by storing table entry pointer into rseq_cs. */ 298 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) 299 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 300 RSEQ_INJECT_ASM(3) 301 LONG_CMP " %[expect], %[v]\n\t" 302 "jnz %l[cmpfail]\n\t" 303 RSEQ_INJECT_ASM(4) 304#ifdef RSEQ_COMPARE_TWICE 305 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 306 LONG_CMP " %[expect], %[v]\n\t" 307 "jnz %l[error2]\n\t" 308#endif 309 /* try store */ 310 LONG_S " %[newv2], %[v2]\n\t" 311 RSEQ_INJECT_ASM(5) 312 /* final store */ 313 LONG_S " %[newv], %[v]\n\t" 314 "2:\n\t" 315 RSEQ_INJECT_ASM(6) 316 RSEQ_ASM_DEFINE_ABORT(4, "", abort) 317 : /* gcc asm goto does not allow outputs */ 318 : [cpu_id] "r" (cpu), 319 [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD), 320 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), 321 /* try store input */ 322 [v2] "m" (*v2), 323 [newv2] "r" (newv2), 324 /* final store input */ 325 [v] "m" (*v), 326 [expect] "r" (expect), 327 [newv] "r" (newv) 328 RSEQ_INJECT_INPUT 329 : "memory", "cc", "r0" 330 RSEQ_INJECT_CLOBBER 331 : abort, cmpfail 332#ifdef RSEQ_COMPARE_TWICE 333 , error1, error2 334#endif 335 ); 336 rseq_after_asm_goto(); 337 return 0; 338abort: 339 rseq_after_asm_goto(); 340 RSEQ_INJECT_FAILED 341 return -1; 342cmpfail: 343 rseq_after_asm_goto(); 344 return 1; 345#ifdef RSEQ_COMPARE_TWICE 346error1: 347 rseq_after_asm_goto(); 348 rseq_bug("cpu_id comparison failed"); 349error2: 350 rseq_after_asm_goto(); 351 rseq_bug("expected value comparison failed"); 352#endif 353} 354 355/* s390 is TSO. */ 356static inline __attribute__((always_inline)) 357int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trymemcpy_storev)(intptr_t *v, intptr_t expect, 358 void *dst, void *src, size_t len, 359 intptr_t newv, int cpu) 360{ 361 uint64_t rseq_scratch[3]; 362 363 RSEQ_INJECT_C(9) 364 365 __asm__ __volatile__ goto ( 366 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 367 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 368#ifdef RSEQ_COMPARE_TWICE 369 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 370 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 371#endif 372 LONG_S " %[src], %[rseq_scratch0]\n\t" 373 LONG_S " %[dst], %[rseq_scratch1]\n\t" 374 LONG_S " %[len], %[rseq_scratch2]\n\t" 375 /* Start rseq by storing table entry pointer into rseq_cs. */ 376 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) 377 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 378 RSEQ_INJECT_ASM(3) 379 LONG_CMP " %[expect], %[v]\n\t" 380 "jnz 5f\n\t" 381 RSEQ_INJECT_ASM(4) 382#ifdef RSEQ_COMPARE_TWICE 383 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 6f) 384 LONG_CMP " %[expect], %[v]\n\t" 385 "jnz 7f\n\t" 386#endif 387 /* try memcpy */ 388 LONG_LT_R " %[len], %[len]\n\t" 389 "jz 333f\n\t" 390 "222:\n\t" 391 "ic %%r0,0(%[src])\n\t" 392 "stc %%r0,0(%[dst])\n\t" 393 LONG_ADDI " %[src], 1\n\t" 394 LONG_ADDI " %[dst], 1\n\t" 395 LONG_ADDI " %[len], -1\n\t" 396 "jnz 222b\n\t" 397 "333:\n\t" 398 RSEQ_INJECT_ASM(5) 399 /* final store */ 400 LONG_S " %[newv], %[v]\n\t" 401 "2:\n\t" 402 RSEQ_INJECT_ASM(6) 403 /* teardown */ 404 LONG_L " %[len], %[rseq_scratch2]\n\t" 405 LONG_L " %[dst], %[rseq_scratch1]\n\t" 406 LONG_L " %[src], %[rseq_scratch0]\n\t" 407 RSEQ_ASM_DEFINE_ABORT(4, 408 LONG_L " %[len], %[rseq_scratch2]\n\t" 409 LONG_L " %[dst], %[rseq_scratch1]\n\t" 410 LONG_L " %[src], %[rseq_scratch0]\n\t", 411 abort) 412 RSEQ_ASM_DEFINE_CMPFAIL(5, 413 LONG_L " %[len], %[rseq_scratch2]\n\t" 414 LONG_L " %[dst], %[rseq_scratch1]\n\t" 415 LONG_L " %[src], %[rseq_scratch0]\n\t", 416 cmpfail) 417#ifdef RSEQ_COMPARE_TWICE 418 RSEQ_ASM_DEFINE_CMPFAIL(6, 419 LONG_L " %[len], %[rseq_scratch2]\n\t" 420 LONG_L " %[dst], %[rseq_scratch1]\n\t" 421 LONG_L " %[src], %[rseq_scratch0]\n\t", 422 error1) 423 RSEQ_ASM_DEFINE_CMPFAIL(7, 424 LONG_L " %[len], %[rseq_scratch2]\n\t" 425 LONG_L " %[dst], %[rseq_scratch1]\n\t" 426 LONG_L " %[src], %[rseq_scratch0]\n\t", 427 error2) 428#endif 429 : /* gcc asm goto does not allow outputs */ 430 : [cpu_id] "r" (cpu), 431 [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD), 432 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), 433 /* final store input */ 434 [v] "m" (*v), 435 [expect] "r" (expect), 436 [newv] "r" (newv), 437 /* try memcpy input */ 438 [dst] "r" (dst), 439 [src] "r" (src), 440 [len] "r" (len), 441 [rseq_scratch0] "m" (rseq_scratch[0]), 442 [rseq_scratch1] "m" (rseq_scratch[1]), 443 [rseq_scratch2] "m" (rseq_scratch[2]) 444 RSEQ_INJECT_INPUT 445 : "memory", "cc", "r0" 446 RSEQ_INJECT_CLOBBER 447 : abort, cmpfail 448#ifdef RSEQ_COMPARE_TWICE 449 , error1, error2 450#endif 451 ); 452 rseq_after_asm_goto(); 453 return 0; 454abort: 455 rseq_after_asm_goto(); 456 RSEQ_INJECT_FAILED 457 return -1; 458cmpfail: 459 rseq_after_asm_goto(); 460 return 1; 461#ifdef RSEQ_COMPARE_TWICE 462error1: 463 rseq_after_asm_goto(); 464 rseq_bug("cpu_id comparison failed"); 465error2: 466 rseq_after_asm_goto(); 467 rseq_bug("expected value comparison failed"); 468#endif 469} 470 471#endif /* #if (defined(RSEQ_TEMPLATE_MO_RELAXED) || defined(RSEQ_TEMPLATE_MO_RELEASE)) && 472 (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) */ 473 474#include "rseq-bits-reset.h" 475