1/* SPDX-License-Identifier: LGPL-2.1 OR MIT */ 2/* 3 * rseq-arm-bits.h 4 * 5 * (C) Copyright 2016-2022 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com> 6 */ 7 8#include "rseq-bits-template.h" 9 10#if defined(RSEQ_TEMPLATE_MO_RELAXED) && \ 11 (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) 12 13static inline __attribute__((always_inline)) 14int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_storev)(intptr_t *v, intptr_t expect, intptr_t newv, int cpu) 15{ 16 RSEQ_INJECT_C(9) 17 18 __asm__ __volatile__ goto ( 19 RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */ 20 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 21#ifdef RSEQ_COMPARE_TWICE 22 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 23 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 24#endif 25 /* Start rseq by storing table entry pointer into rseq_cs. */ 26 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) 27 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 28 RSEQ_INJECT_ASM(3) 29 "ldr r0, %[v]\n\t" 30 "cmp %[expect], r0\n\t" 31 "bne %l[cmpfail]\n\t" 32 RSEQ_INJECT_ASM(4) 33#ifdef RSEQ_COMPARE_TWICE 34 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 35 "ldr r0, %[v]\n\t" 36 "cmp %[expect], r0\n\t" 37 "bne %l[error2]\n\t" 38#endif 39 /* final store */ 40 "str %[newv], %[v]\n\t" 41 "2:\n\t" 42 RSEQ_INJECT_ASM(5) 43 "b 5f\n\t" 44 RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f) 45 "5:\n\t" 46 : /* gcc asm goto does not allow outputs */ 47 : [cpu_id] "r" (cpu), 48 [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD), 49 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), 50 [v] "m" (*v), 51 [expect] "r" (expect), 52 [newv] "r" (newv) 53 RSEQ_INJECT_INPUT 54 : "r0", "memory", "cc" 55 RSEQ_INJECT_CLOBBER 56 : abort, cmpfail 57#ifdef RSEQ_COMPARE_TWICE 58 , error1, error2 59#endif 60 ); 61 rseq_after_asm_goto(); 62 return 0; 63abort: 64 rseq_after_asm_goto(); 65 RSEQ_INJECT_FAILED 66 return -1; 67cmpfail: 68 rseq_after_asm_goto(); 69 return 1; 70#ifdef RSEQ_COMPARE_TWICE 71error1: 72 rseq_after_asm_goto(); 73 rseq_bug("cpu_id comparison failed"); 74error2: 75 rseq_after_asm_goto(); 76 rseq_bug("expected value comparison failed"); 77#endif 78} 79 80static inline __attribute__((always_inline)) 81int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpnev_storeoffp_load)(intptr_t *v, intptr_t expectnot, 82 long voffp, intptr_t *load, int cpu) 83{ 84 RSEQ_INJECT_C(9) 85 86 __asm__ __volatile__ goto ( 87 RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */ 88 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 89#ifdef RSEQ_COMPARE_TWICE 90 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 91 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 92#endif 93 /* Start rseq by storing table entry pointer into rseq_cs. */ 94 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) 95 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 96 RSEQ_INJECT_ASM(3) 97 "ldr r0, %[v]\n\t" 98 "cmp %[expectnot], r0\n\t" 99 "beq %l[cmpfail]\n\t" 100 RSEQ_INJECT_ASM(4) 101#ifdef RSEQ_COMPARE_TWICE 102 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 103 "ldr r0, %[v]\n\t" 104 "cmp %[expectnot], r0\n\t" 105 "beq %l[error2]\n\t" 106#endif 107 "str r0, %[load]\n\t" 108 "add r0, %[voffp]\n\t" 109 "ldr r0, [r0]\n\t" 110 /* final store */ 111 "str r0, %[v]\n\t" 112 "2:\n\t" 113 RSEQ_INJECT_ASM(5) 114 "b 5f\n\t" 115 RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f) 116 "5:\n\t" 117 : /* gcc asm goto does not allow outputs */ 118 : [cpu_id] "r" (cpu), 119 [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD), 120 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), 121 /* final store input */ 122 [v] "m" (*v), 123 [expectnot] "r" (expectnot), 124 [voffp] "Ir" (voffp), 125 [load] "m" (*load) 126 RSEQ_INJECT_INPUT 127 : "r0", "memory", "cc" 128 RSEQ_INJECT_CLOBBER 129 : abort, cmpfail 130#ifdef RSEQ_COMPARE_TWICE 131 , error1, error2 132#endif 133 ); 134 rseq_after_asm_goto(); 135 return 0; 136abort: 137 rseq_after_asm_goto(); 138 RSEQ_INJECT_FAILED 139 return -1; 140cmpfail: 141 rseq_after_asm_goto(); 142 return 1; 143#ifdef RSEQ_COMPARE_TWICE 144error1: 145 rseq_after_asm_goto(); 146 rseq_bug("cpu_id comparison failed"); 147error2: 148 rseq_after_asm_goto(); 149 rseq_bug("expected value comparison failed"); 150#endif 151} 152 153static inline __attribute__((always_inline)) 154int RSEQ_TEMPLATE_IDENTIFIER(rseq_addv)(intptr_t *v, intptr_t count, int cpu) 155{ 156 RSEQ_INJECT_C(9) 157 158 __asm__ __volatile__ goto ( 159 RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */ 160#ifdef RSEQ_COMPARE_TWICE 161 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 162#endif 163 /* Start rseq by storing table entry pointer into rseq_cs. */ 164 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) 165 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 166 RSEQ_INJECT_ASM(3) 167#ifdef RSEQ_COMPARE_TWICE 168 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 169#endif 170 "ldr r0, %[v]\n\t" 171 "add r0, %[count]\n\t" 172 /* final store */ 173 "str r0, %[v]\n\t" 174 "2:\n\t" 175 RSEQ_INJECT_ASM(4) 176 "b 5f\n\t" 177 RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f) 178 "5:\n\t" 179 : /* gcc asm goto does not allow outputs */ 180 : [cpu_id] "r" (cpu), 181 [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD), 182 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), 183 [v] "m" (*v), 184 [count] "Ir" (count) 185 RSEQ_INJECT_INPUT 186 : "r0", "memory", "cc" 187 RSEQ_INJECT_CLOBBER 188 : abort 189#ifdef RSEQ_COMPARE_TWICE 190 , error1 191#endif 192 ); 193 rseq_after_asm_goto(); 194 return 0; 195abort: 196 rseq_after_asm_goto(); 197 RSEQ_INJECT_FAILED 198 return -1; 199#ifdef RSEQ_COMPARE_TWICE 200error1: 201 rseq_after_asm_goto(); 202 rseq_bug("cpu_id comparison failed"); 203#endif 204} 205 206static inline __attribute__((always_inline)) 207int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_cmpeqv_storev)(intptr_t *v, intptr_t expect, 208 intptr_t *v2, intptr_t expect2, 209 intptr_t newv, int cpu) 210{ 211 RSEQ_INJECT_C(9) 212 213 __asm__ __volatile__ goto ( 214 RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */ 215 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 216#ifdef RSEQ_COMPARE_TWICE 217 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 218 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 219 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3]) 220#endif 221 /* Start rseq by storing table entry pointer into rseq_cs. */ 222 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) 223 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 224 RSEQ_INJECT_ASM(3) 225 "ldr r0, %[v]\n\t" 226 "cmp %[expect], r0\n\t" 227 "bne %l[cmpfail]\n\t" 228 RSEQ_INJECT_ASM(4) 229 "ldr r0, %[v2]\n\t" 230 "cmp %[expect2], r0\n\t" 231 "bne %l[cmpfail]\n\t" 232 RSEQ_INJECT_ASM(5) 233#ifdef RSEQ_COMPARE_TWICE 234 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 235 "ldr r0, %[v]\n\t" 236 "cmp %[expect], r0\n\t" 237 "bne %l[error2]\n\t" 238 "ldr r0, %[v2]\n\t" 239 "cmp %[expect2], r0\n\t" 240 "bne %l[error3]\n\t" 241#endif 242 /* final store */ 243 "str %[newv], %[v]\n\t" 244 "2:\n\t" 245 RSEQ_INJECT_ASM(6) 246 "b 5f\n\t" 247 RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f) 248 "5:\n\t" 249 : /* gcc asm goto does not allow outputs */ 250 : [cpu_id] "r" (cpu), 251 [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD), 252 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), 253 /* cmp2 input */ 254 [v2] "m" (*v2), 255 [expect2] "r" (expect2), 256 /* final store input */ 257 [v] "m" (*v), 258 [expect] "r" (expect), 259 [newv] "r" (newv) 260 RSEQ_INJECT_INPUT 261 : "r0", "memory", "cc" 262 RSEQ_INJECT_CLOBBER 263 : abort, cmpfail 264#ifdef RSEQ_COMPARE_TWICE 265 , error1, error2, error3 266#endif 267 ); 268 rseq_after_asm_goto(); 269 return 0; 270abort: 271 rseq_after_asm_goto(); 272 RSEQ_INJECT_FAILED 273 return -1; 274cmpfail: 275 rseq_after_asm_goto(); 276 return 1; 277#ifdef RSEQ_COMPARE_TWICE 278error1: 279 rseq_after_asm_goto(); 280 rseq_bug("cpu_id comparison failed"); 281error2: 282 rseq_after_asm_goto(); 283 rseq_bug("1st expected value comparison failed"); 284error3: 285 rseq_after_asm_goto(); 286 rseq_bug("2nd expected value comparison failed"); 287#endif 288} 289 290#endif /* #if defined(RSEQ_TEMPLATE_MO_RELAXED) && 291 (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) */ 292 293#if (defined(RSEQ_TEMPLATE_MO_RELAXED) || defined(RSEQ_TEMPLATE_MO_RELEASE)) && \ 294 (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) 295 296static inline __attribute__((always_inline)) 297int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trystorev_storev)(intptr_t *v, intptr_t expect, 298 intptr_t *v2, intptr_t newv2, 299 intptr_t newv, int cpu) 300{ 301 RSEQ_INJECT_C(9) 302 303 __asm__ __volatile__ goto ( 304 RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */ 305 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 306#ifdef RSEQ_COMPARE_TWICE 307 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 308 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 309#endif 310 /* Start rseq by storing table entry pointer into rseq_cs. */ 311 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) 312 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 313 RSEQ_INJECT_ASM(3) 314 "ldr r0, %[v]\n\t" 315 "cmp %[expect], r0\n\t" 316 "bne %l[cmpfail]\n\t" 317 RSEQ_INJECT_ASM(4) 318#ifdef RSEQ_COMPARE_TWICE 319 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 320 "ldr r0, %[v]\n\t" 321 "cmp %[expect], r0\n\t" 322 "bne %l[error2]\n\t" 323#endif 324 /* try store */ 325 "str %[newv2], %[v2]\n\t" 326 RSEQ_INJECT_ASM(5) 327#ifdef RSEQ_TEMPLATE_MO_RELEASE 328 "dmb\n\t" /* full mb provides store-release */ 329#endif 330 /* final store */ 331 "str %[newv], %[v]\n\t" 332 "2:\n\t" 333 RSEQ_INJECT_ASM(6) 334 "b 5f\n\t" 335 RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f) 336 "5:\n\t" 337 : /* gcc asm goto does not allow outputs */ 338 : [cpu_id] "r" (cpu), 339 [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD), 340 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), 341 /* try store input */ 342 [v2] "m" (*v2), 343 [newv2] "r" (newv2), 344 /* final store input */ 345 [v] "m" (*v), 346 [expect] "r" (expect), 347 [newv] "r" (newv) 348 RSEQ_INJECT_INPUT 349 : "r0", "memory", "cc" 350 RSEQ_INJECT_CLOBBER 351 : abort, cmpfail 352#ifdef RSEQ_COMPARE_TWICE 353 , error1, error2 354#endif 355 ); 356 rseq_after_asm_goto(); 357 return 0; 358abort: 359 rseq_after_asm_goto(); 360 RSEQ_INJECT_FAILED 361 return -1; 362cmpfail: 363 rseq_after_asm_goto(); 364 return 1; 365#ifdef RSEQ_COMPARE_TWICE 366error1: 367 rseq_after_asm_goto(); 368 rseq_bug("cpu_id comparison failed"); 369error2: 370 rseq_after_asm_goto(); 371 rseq_bug("expected value comparison failed"); 372#endif 373} 374 375 376static inline __attribute__((always_inline)) 377int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trymemcpy_storev)(intptr_t *v, intptr_t expect, 378 void *dst, void *src, size_t len, 379 intptr_t newv, int cpu) 380{ 381 uint32_t rseq_scratch[3]; 382 383 RSEQ_INJECT_C(9) 384 385 __asm__ __volatile__ goto ( 386 RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */ 387 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 388#ifdef RSEQ_COMPARE_TWICE 389 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 390 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 391#endif 392 "str %[src], %[rseq_scratch0]\n\t" 393 "str %[dst], %[rseq_scratch1]\n\t" 394 "str %[len], %[rseq_scratch2]\n\t" 395 /* Start rseq by storing table entry pointer into rseq_cs. */ 396 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) 397 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 398 RSEQ_INJECT_ASM(3) 399 "ldr r0, %[v]\n\t" 400 "cmp %[expect], r0\n\t" 401 "bne 5f\n\t" 402 RSEQ_INJECT_ASM(4) 403#ifdef RSEQ_COMPARE_TWICE 404 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 6f) 405 "ldr r0, %[v]\n\t" 406 "cmp %[expect], r0\n\t" 407 "bne 7f\n\t" 408#endif 409 /* try memcpy */ 410 "cmp %[len], #0\n\t" \ 411 "beq 333f\n\t" \ 412 "222:\n\t" \ 413 "ldrb %%r0, [%[src]]\n\t" \ 414 "strb %%r0, [%[dst]]\n\t" \ 415 "adds %[src], #1\n\t" \ 416 "adds %[dst], #1\n\t" \ 417 "subs %[len], #1\n\t" \ 418 "bne 222b\n\t" \ 419 "333:\n\t" \ 420 RSEQ_INJECT_ASM(5) 421#ifdef RSEQ_TEMPLATE_MO_RELEASE 422 "dmb\n\t" /* full mb provides store-release */ 423#endif 424 /* final store */ 425 "str %[newv], %[v]\n\t" 426 "2:\n\t" 427 RSEQ_INJECT_ASM(6) 428 /* teardown */ 429 "ldr %[len], %[rseq_scratch2]\n\t" 430 "ldr %[dst], %[rseq_scratch1]\n\t" 431 "ldr %[src], %[rseq_scratch0]\n\t" 432 "b 8f\n\t" 433 RSEQ_ASM_DEFINE_ABORT(3, 4, 434 /* teardown */ 435 "ldr %[len], %[rseq_scratch2]\n\t" 436 "ldr %[dst], %[rseq_scratch1]\n\t" 437 "ldr %[src], %[rseq_scratch0]\n\t", 438 abort, 1b, 2b, 4f) 439 RSEQ_ASM_DEFINE_CMPFAIL(5, 440 /* teardown */ 441 "ldr %[len], %[rseq_scratch2]\n\t" 442 "ldr %[dst], %[rseq_scratch1]\n\t" 443 "ldr %[src], %[rseq_scratch0]\n\t", 444 cmpfail) 445#ifdef RSEQ_COMPARE_TWICE 446 RSEQ_ASM_DEFINE_CMPFAIL(6, 447 /* teardown */ 448 "ldr %[len], %[rseq_scratch2]\n\t" 449 "ldr %[dst], %[rseq_scratch1]\n\t" 450 "ldr %[src], %[rseq_scratch0]\n\t", 451 error1) 452 RSEQ_ASM_DEFINE_CMPFAIL(7, 453 /* teardown */ 454 "ldr %[len], %[rseq_scratch2]\n\t" 455 "ldr %[dst], %[rseq_scratch1]\n\t" 456 "ldr %[src], %[rseq_scratch0]\n\t", 457 error2) 458#endif 459 "8:\n\t" 460 : /* gcc asm goto does not allow outputs */ 461 : [cpu_id] "r" (cpu), 462 [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD), 463 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), 464 /* final store input */ 465 [v] "m" (*v), 466 [expect] "r" (expect), 467 [newv] "r" (newv), 468 /* try memcpy input */ 469 [dst] "r" (dst), 470 [src] "r" (src), 471 [len] "r" (len), 472 [rseq_scratch0] "m" (rseq_scratch[0]), 473 [rseq_scratch1] "m" (rseq_scratch[1]), 474 [rseq_scratch2] "m" (rseq_scratch[2]) 475 RSEQ_INJECT_INPUT 476 : "r0", "memory", "cc" 477 RSEQ_INJECT_CLOBBER 478 : abort, cmpfail 479#ifdef RSEQ_COMPARE_TWICE 480 , error1, error2 481#endif 482 ); 483 rseq_after_asm_goto(); 484 return 0; 485abort: 486 rseq_after_asm_goto(); 487 RSEQ_INJECT_FAILED 488 return -1; 489cmpfail: 490 rseq_after_asm_goto(); 491 return 1; 492#ifdef RSEQ_COMPARE_TWICE 493error1: 494 rseq_after_asm_goto(); 495 rseq_bug("cpu_id comparison failed"); 496error2: 497 rseq_after_asm_goto(); 498 rseq_bug("expected value comparison failed"); 499#endif 500} 501 502#endif /* #if (defined(RSEQ_TEMPLATE_MO_RELAXED) || defined(RSEQ_TEMPLATE_MO_RELEASE)) && 503 (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) */ 504 505#include "rseq-bits-reset.h" 506