1/* SPDX-License-Identifier: LGPL-2.1 OR MIT */ 2 3#include "rseq-bits-template.h" 4 5#if defined(RSEQ_TEMPLATE_MO_RELAXED) && \ 6 (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) 7 8static inline __always_inline 9int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_storev)(intptr_t *v, intptr_t expect, intptr_t newv, int cpu) 10{ 11 RSEQ_INJECT_C(9) 12 13 __asm__ __volatile__ goto(RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f) 14 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[cmpfail]") 15#ifdef RSEQ_COMPARE_TWICE 16 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error1]") 17 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error2]") 18#endif 19 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs) 20 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 21 RSEQ_INJECT_ASM(3) 22 RSEQ_ASM_OP_CMPEQ(v, expect, "%l[cmpfail]") 23 RSEQ_INJECT_ASM(4) 24#ifdef RSEQ_COMPARE_TWICE 25 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, "%l[error1]") 26 RSEQ_ASM_OP_CMPEQ(v, expect, "%l[error2]") 27#endif 28 RSEQ_ASM_OP_FINAL_STORE(newv, v, 3) 29 RSEQ_INJECT_ASM(5) 30 RSEQ_ASM_DEFINE_ABORT(4, abort) 31 : /* gcc asm goto does not allow outputs */ 32 : [cpu_id] "r" (cpu), 33 [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD), 34 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), 35 [v] "m" (*v), 36 [expect] "r" (expect), 37 [newv] "r" (newv) 38 RSEQ_INJECT_INPUT 39 : "memory", RSEQ_ASM_TMP_REG_1 40 RSEQ_INJECT_CLOBBER 41 : abort, cmpfail 42#ifdef RSEQ_COMPARE_TWICE 43 , error1, error2 44#endif 45 ); 46 47 return 0; 48abort: 49 RSEQ_INJECT_FAILED 50 return -1; 51cmpfail: 52 return 1; 53#ifdef RSEQ_COMPARE_TWICE 54error1: 55 rseq_bug("cpu_id comparison failed"); 56error2: 57 rseq_bug("expected value comparison failed"); 58#endif 59} 60 61static inline __always_inline 62int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpnev_storeoffp_load)(intptr_t *v, intptr_t expectnot, 63 off_t voffp, intptr_t *load, int cpu) 64{ 65 RSEQ_INJECT_C(9) 66 67 __asm__ __volatile__ goto(RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f) 68 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[cmpfail]") 69#ifdef RSEQ_COMPARE_TWICE 70 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error1]") 71 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error2]") 72#endif 73 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs) 74 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 75 RSEQ_INJECT_ASM(3) 76 RSEQ_ASM_OP_CMPNE(v, expectnot, "%l[cmpfail]") 77 RSEQ_INJECT_ASM(4) 78#ifdef RSEQ_COMPARE_TWICE 79 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, "%l[error1]") 80 RSEQ_ASM_OP_CMPNE(v, expectnot, "%l[error2]") 81#endif 82 RSEQ_ASM_OP_R_LOAD(v) 83 RSEQ_ASM_OP_R_STORE(load) 84 RSEQ_ASM_OP_R_LOAD_OFF(voffp) 85 RSEQ_ASM_OP_R_FINAL_STORE(v, 3) 86 RSEQ_INJECT_ASM(5) 87 RSEQ_ASM_DEFINE_ABORT(4, abort) 88 : /* gcc asm goto does not allow outputs */ 89 : [cpu_id] "r" (cpu), 90 [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD), 91 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), 92 [v] "m" (*v), 93 [expectnot] "r" (expectnot), 94 [load] "m" (*load), 95 [voffp] "r" (voffp) 96 RSEQ_INJECT_INPUT 97 : "memory", RSEQ_ASM_TMP_REG_1 98 RSEQ_INJECT_CLOBBER 99 : abort, cmpfail 100#ifdef RSEQ_COMPARE_TWICE 101 , error1, error2 102#endif 103 ); 104 return 0; 105abort: 106 RSEQ_INJECT_FAILED 107 return -1; 108cmpfail: 109 return 1; 110#ifdef RSEQ_COMPARE_TWICE 111error1: 112 rseq_bug("cpu_id comparison failed"); 113error2: 114 rseq_bug("expected value comparison failed"); 115#endif 116} 117 118static inline __always_inline 119int RSEQ_TEMPLATE_IDENTIFIER(rseq_addv)(intptr_t *v, intptr_t count, int cpu) 120{ 121 RSEQ_INJECT_C(9) 122 123 __asm__ __volatile__ goto(RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f) 124#ifdef RSEQ_COMPARE_TWICE 125 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error1]") 126#endif 127 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs) 128 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 129 RSEQ_INJECT_ASM(3) 130#ifdef RSEQ_COMPARE_TWICE 131 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, "%l[error1]") 132#endif 133 RSEQ_ASM_OP_R_LOAD(v) 134 RSEQ_ASM_OP_R_ADD(count) 135 RSEQ_ASM_OP_R_FINAL_STORE(v, 3) 136 RSEQ_INJECT_ASM(4) 137 RSEQ_ASM_DEFINE_ABORT(4, abort) 138 : /* gcc asm goto does not allow outputs */ 139 : [cpu_id] "r" (cpu), 140 [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD), 141 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), 142 [v] "m" (*v), 143 [count] "r" (count) 144 RSEQ_INJECT_INPUT 145 : "memory", RSEQ_ASM_TMP_REG_1 146 RSEQ_INJECT_CLOBBER 147 : abort 148#ifdef RSEQ_COMPARE_TWICE 149 , error1 150#endif 151 ); 152 return 0; 153abort: 154 RSEQ_INJECT_FAILED 155 return -1; 156#ifdef RSEQ_COMPARE_TWICE 157error1: 158 rseq_bug("cpu_id comparison failed"); 159#endif 160} 161 162static inline __always_inline 163int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_cmpeqv_storev)(intptr_t *v, intptr_t expect, 164 intptr_t *v2, intptr_t expect2, 165 intptr_t newv, int cpu) 166{ 167 RSEQ_INJECT_C(9) 168 169 __asm__ __volatile__ goto(RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f) 170 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[cmpfail]") 171#ifdef RSEQ_COMPARE_TWICE 172 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error1]") 173 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error2]") 174 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error3]") 175#endif 176 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs) 177 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 178 RSEQ_INJECT_ASM(3) 179 RSEQ_ASM_OP_CMPEQ(v, expect, "%l[cmpfail]") 180 RSEQ_INJECT_ASM(4) 181 RSEQ_ASM_OP_CMPEQ(v2, expect2, "%l[cmpfail]") 182 RSEQ_INJECT_ASM(5) 183#ifdef RSEQ_COMPARE_TWICE 184 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, "%l[error1]") 185 RSEQ_ASM_OP_CMPEQ(v, expect, "%l[error2]") 186 RSEQ_ASM_OP_CMPEQ(v2, expect2, "%l[error3]") 187#endif 188 RSEQ_ASM_OP_FINAL_STORE(newv, v, 3) 189 RSEQ_INJECT_ASM(6) 190 RSEQ_ASM_DEFINE_ABORT(4, abort) 191 : /* gcc asm goto does not allow outputs */ 192 : [cpu_id] "r" (cpu), 193 [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD), 194 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), 195 [v] "m" (*v), 196 [expect] "r" (expect), 197 [v2] "m" (*v2), 198 [expect2] "r" (expect2), 199 [newv] "r" (newv) 200 RSEQ_INJECT_INPUT 201 : "memory", RSEQ_ASM_TMP_REG_1 202 RSEQ_INJECT_CLOBBER 203 : abort, cmpfail 204#ifdef RSEQ_COMPARE_TWICE 205 , error1, error2, error3 206#endif 207 ); 208 209 return 0; 210abort: 211 RSEQ_INJECT_FAILED 212 return -1; 213cmpfail: 214 return 1; 215#ifdef RSEQ_COMPARE_TWICE 216error1: 217 rseq_bug("cpu_id comparison failed"); 218error2: 219 rseq_bug("expected value comparison failed"); 220error3: 221 rseq_bug("2nd expected value comparison failed"); 222#endif 223} 224 225#define RSEQ_ARCH_HAS_OFFSET_DEREF_ADDV 226 227/* 228 * pval = *(ptr+off) 229 * *pval += inc; 230 */ 231static inline __always_inline 232int RSEQ_TEMPLATE_IDENTIFIER(rseq_offset_deref_addv)(intptr_t *ptr, off_t off, intptr_t inc, int cpu) 233{ 234 RSEQ_INJECT_C(9) 235 236 __asm__ __volatile__ goto(RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f) 237#ifdef RSEQ_COMPARE_TWICE 238 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error1]") 239#endif 240 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs) 241 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 242 RSEQ_INJECT_ASM(3) 243#ifdef RSEQ_COMPARE_TWICE 244 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, "%l[error1]") 245#endif 246 RSEQ_ASM_OP_R_DEREF_ADDV(ptr, off, 3) 247 RSEQ_INJECT_ASM(4) 248 RSEQ_ASM_DEFINE_ABORT(4, abort) 249 : /* gcc asm goto does not allow outputs */ 250 : [cpu_id] "r" (cpu), 251 [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD), 252 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), 253 [ptr] "r" (ptr), 254 [off] "er" (off), 255 [inc] "er" (inc) 256 RSEQ_INJECT_INPUT 257 : "memory", RSEQ_ASM_TMP_REG_1 258 RSEQ_INJECT_CLOBBER 259 : abort 260#ifdef RSEQ_COMPARE_TWICE 261 , error1 262#endif 263 ); 264 return 0; 265abort: 266 RSEQ_INJECT_FAILED 267 return -1; 268#ifdef RSEQ_COMPARE_TWICE 269error1: 270 rseq_bug("cpu_id comparison failed"); 271#endif 272} 273 274#endif /* #if defined(RSEQ_TEMPLATE_MO_RELAXED) && 275 (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) */ 276 277#if (defined(RSEQ_TEMPLATE_MO_RELAXED) || defined(RSEQ_TEMPLATE_MO_RELEASE)) && \ 278 (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) 279 280static inline __always_inline 281int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trystorev_storev)(intptr_t *v, intptr_t expect, 282 intptr_t *v2, intptr_t newv2, 283 intptr_t newv, int cpu) 284{ 285 RSEQ_INJECT_C(9) 286 287 __asm__ __volatile__ goto(RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f) 288 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[cmpfail]") 289#ifdef RSEQ_COMPARE_TWICE 290 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error1]") 291 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error2]") 292#endif 293 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs) 294 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 295 RSEQ_INJECT_ASM(3) 296 RSEQ_ASM_OP_CMPEQ(v, expect, "%l[cmpfail]") 297 RSEQ_INJECT_ASM(4) 298#ifdef RSEQ_COMPARE_TWICE 299 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, "%l[error1]") 300 RSEQ_ASM_OP_CMPEQ(v, expect, "%l[error2]") 301#endif 302 RSEQ_ASM_OP_STORE(newv2, v2) 303 RSEQ_INJECT_ASM(5) 304#ifdef RSEQ_TEMPLATE_MO_RELEASE 305 RSEQ_ASM_OP_FINAL_STORE_RELEASE(newv, v, 3) 306#else 307 RSEQ_ASM_OP_FINAL_STORE(newv, v, 3) 308#endif 309 RSEQ_INJECT_ASM(6) 310 RSEQ_ASM_DEFINE_ABORT(4, abort) 311 : /* gcc asm goto does not allow outputs */ 312 : [cpu_id] "r" (cpu), 313 [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD), 314 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), 315 [expect] "r" (expect), 316 [v] "m" (*v), 317 [newv] "r" (newv), 318 [v2] "m" (*v2), 319 [newv2] "r" (newv2) 320 RSEQ_INJECT_INPUT 321 : "memory", RSEQ_ASM_TMP_REG_1 322 RSEQ_INJECT_CLOBBER 323 : abort, cmpfail 324#ifdef RSEQ_COMPARE_TWICE 325 , error1, error2 326#endif 327 ); 328 329 return 0; 330abort: 331 RSEQ_INJECT_FAILED 332 return -1; 333cmpfail: 334 return 1; 335#ifdef RSEQ_COMPARE_TWICE 336error1: 337 rseq_bug("cpu_id comparison failed"); 338error2: 339 rseq_bug("expected value comparison failed"); 340#endif 341} 342 343static inline __always_inline 344int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trymemcpy_storev)(intptr_t *v, intptr_t expect, 345 void *dst, void *src, size_t len, 346 intptr_t newv, int cpu) 347{ 348 RSEQ_INJECT_C(9) 349 __asm__ __volatile__ goto(RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f) 350 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[cmpfail]") 351#ifdef RSEQ_COMPARE_TWICE 352 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error1]") 353 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error2]") 354#endif 355 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs) 356 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 357 RSEQ_INJECT_ASM(3) 358 RSEQ_ASM_OP_CMPEQ(v, expect, "%l[cmpfail]") 359 RSEQ_INJECT_ASM(4) 360#ifdef RSEQ_COMPARE_TWICE 361 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, "%l[error1]") 362 RSEQ_ASM_OP_CMPEQ(v, expect, "%l[error2]") 363#endif 364 RSEQ_ASM_OP_R_BAD_MEMCPY(dst, src, len) 365 RSEQ_INJECT_ASM(5) 366#ifdef RSEQ_TEMPLATE_MO_RELEASE 367 RSEQ_ASM_OP_FINAL_STORE_RELEASE(newv, v, 3) 368#else 369 RSEQ_ASM_OP_FINAL_STORE(newv, v, 3) 370#endif 371 RSEQ_INJECT_ASM(6) 372 RSEQ_ASM_DEFINE_ABORT(4, abort) 373 : /* gcc asm goto does not allow outputs */ 374 : [cpu_id] "r" (cpu), 375 [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD), 376 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), 377 [expect] "r" (expect), 378 [v] "m" (*v), 379 [newv] "r" (newv), 380 [dst] "r" (dst), 381 [src] "r" (src), 382 [len] "r" (len) 383 RSEQ_INJECT_INPUT 384 : "memory", RSEQ_ASM_TMP_REG_1, RSEQ_ASM_TMP_REG_2, 385 RSEQ_ASM_TMP_REG_3, RSEQ_ASM_TMP_REG_4 386 RSEQ_INJECT_CLOBBER 387 : abort, cmpfail 388#ifdef RSEQ_COMPARE_TWICE 389 , error1, error2 390#endif 391 ); 392 393 return 0; 394abort: 395 RSEQ_INJECT_FAILED 396 return -1; 397cmpfail: 398 return 1; 399#ifdef RSEQ_COMPARE_TWICE 400error1: 401 rseq_bug("cpu_id comparison failed"); 402error2: 403 rseq_bug("expected value comparison failed"); 404#endif 405} 406 407#endif /* #if (defined(RSEQ_TEMPLATE_MO_RELAXED) || defined(RSEQ_TEMPLATE_MO_RELEASE)) && 408 (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) */ 409 410#include "rseq-bits-reset.h" 411