1/* SPDX-License-Identifier: LGPL-2.1 OR MIT */ 2/* 3 * rseq-x86-bits.h 4 * 5 * (C) Copyright 2016-2022 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com> 6 */ 7 8#include "rseq-bits-template.h" 9 10#ifdef __x86_64__ 11 12#if defined(RSEQ_TEMPLATE_MO_RELAXED) && \ 13 (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) 14 15static inline __attribute__((always_inline)) 16int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_storev)(intptr_t *v, intptr_t expect, intptr_t newv, int cpu) 17{ 18 RSEQ_INJECT_C(9) 19 20 __asm__ __volatile__ goto ( 21 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 22 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 23#ifdef RSEQ_COMPARE_TWICE 24 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 25 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 26#endif 27 /* Start rseq by storing table entry pointer into rseq_cs. */ 28 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset])) 29 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), 4f) 30 RSEQ_INJECT_ASM(3) 31 "cmpq %[v], %[expect]\n\t" 32 "jnz %l[cmpfail]\n\t" 33 RSEQ_INJECT_ASM(4) 34#ifdef RSEQ_COMPARE_TWICE 35 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), %l[error1]) 36 "cmpq %[v], %[expect]\n\t" 37 "jnz %l[error2]\n\t" 38#endif 39 /* final store */ 40 "movq %[newv], %[v]\n\t" 41 "2:\n\t" 42 RSEQ_INJECT_ASM(5) 43 RSEQ_ASM_DEFINE_ABORT(4, "", abort) 44 : /* gcc asm goto does not allow outputs */ 45 : [cpu_id] "r" (cpu), 46 [rseq_offset] "r" (rseq_offset), 47 [v] "m" (*v), 48 [expect] "r" (expect), 49 [newv] "r" (newv) 50 : "memory", "cc", "rax" 51 RSEQ_INJECT_CLOBBER 52 : abort, cmpfail 53#ifdef RSEQ_COMPARE_TWICE 54 , error1, error2 55#endif 56 ); 57 rseq_after_asm_goto(); 58 return 0; 59abort: 60 rseq_after_asm_goto(); 61 RSEQ_INJECT_FAILED 62 return -1; 63cmpfail: 64 rseq_after_asm_goto(); 65 return 1; 66#ifdef RSEQ_COMPARE_TWICE 67error1: 68 rseq_after_asm_goto(); 69 rseq_bug("cpu_id comparison failed"); 70error2: 71 rseq_after_asm_goto(); 72 rseq_bug("expected value comparison failed"); 73#endif 74} 75 76/* 77 * Compare @v against @expectnot. When it does _not_ match, load @v 78 * into @load, and store the content of *@v + voffp into @v. 79 */ 80static inline __attribute__((always_inline)) 81int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpnev_storeoffp_load)(intptr_t *v, intptr_t expectnot, 82 long voffp, intptr_t *load, int cpu) 83{ 84 RSEQ_INJECT_C(9) 85 86 __asm__ __volatile__ goto ( 87 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 88 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 89#ifdef RSEQ_COMPARE_TWICE 90 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 91 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 92#endif 93 /* Start rseq by storing table entry pointer into rseq_cs. */ 94 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset])) 95 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), 4f) 96 RSEQ_INJECT_ASM(3) 97 "movq %[v], %%rbx\n\t" 98 "cmpq %%rbx, %[expectnot]\n\t" 99 "je %l[cmpfail]\n\t" 100 RSEQ_INJECT_ASM(4) 101#ifdef RSEQ_COMPARE_TWICE 102 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), %l[error1]) 103 "movq %[v], %%rbx\n\t" 104 "cmpq %%rbx, %[expectnot]\n\t" 105 "je %l[error2]\n\t" 106#endif 107 "movq %%rbx, %[load]\n\t" 108 "addq %[voffp], %%rbx\n\t" 109 "movq (%%rbx), %%rbx\n\t" 110 /* final store */ 111 "movq %%rbx, %[v]\n\t" 112 "2:\n\t" 113 RSEQ_INJECT_ASM(5) 114 RSEQ_ASM_DEFINE_ABORT(4, "", abort) 115 : /* gcc asm goto does not allow outputs */ 116 : [cpu_id] "r" (cpu), 117 [rseq_offset] "r" (rseq_offset), 118 /* final store input */ 119 [v] "m" (*v), 120 [expectnot] "r" (expectnot), 121 [voffp] "er" (voffp), 122 [load] "m" (*load) 123 : "memory", "cc", "rax", "rbx" 124 RSEQ_INJECT_CLOBBER 125 : abort, cmpfail 126#ifdef RSEQ_COMPARE_TWICE 127 , error1, error2 128#endif 129 ); 130 rseq_after_asm_goto(); 131 return 0; 132abort: 133 rseq_after_asm_goto(); 134 RSEQ_INJECT_FAILED 135 return -1; 136cmpfail: 137 rseq_after_asm_goto(); 138 return 1; 139#ifdef RSEQ_COMPARE_TWICE 140error1: 141 rseq_after_asm_goto(); 142 rseq_bug("cpu_id comparison failed"); 143error2: 144 rseq_after_asm_goto(); 145 rseq_bug("expected value comparison failed"); 146#endif 147} 148 149static inline __attribute__((always_inline)) 150int RSEQ_TEMPLATE_IDENTIFIER(rseq_addv)(intptr_t *v, intptr_t count, int cpu) 151{ 152 RSEQ_INJECT_C(9) 153 154 __asm__ __volatile__ goto ( 155 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 156#ifdef RSEQ_COMPARE_TWICE 157 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 158#endif 159 /* Start rseq by storing table entry pointer into rseq_cs. */ 160 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset])) 161 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), 4f) 162 RSEQ_INJECT_ASM(3) 163#ifdef RSEQ_COMPARE_TWICE 164 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), %l[error1]) 165#endif 166 /* final store */ 167 "addq %[count], %[v]\n\t" 168 "2:\n\t" 169 RSEQ_INJECT_ASM(4) 170 RSEQ_ASM_DEFINE_ABORT(4, "", abort) 171 : /* gcc asm goto does not allow outputs */ 172 : [cpu_id] "r" (cpu), 173 [rseq_offset] "r" (rseq_offset), 174 /* final store input */ 175 [v] "m" (*v), 176 [count] "er" (count) 177 : "memory", "cc", "rax" 178 RSEQ_INJECT_CLOBBER 179 : abort 180#ifdef RSEQ_COMPARE_TWICE 181 , error1 182#endif 183 ); 184 rseq_after_asm_goto(); 185 return 0; 186abort: 187 rseq_after_asm_goto(); 188 RSEQ_INJECT_FAILED 189 return -1; 190#ifdef RSEQ_COMPARE_TWICE 191error1: 192 rseq_after_asm_goto(); 193 rseq_bug("cpu_id comparison failed"); 194#endif 195} 196 197#define RSEQ_ARCH_HAS_OFFSET_DEREF_ADDV 198 199/* 200 * pval = *(ptr+off) 201 * *pval += inc; 202 */ 203static inline __attribute__((always_inline)) 204int RSEQ_TEMPLATE_IDENTIFIER(rseq_offset_deref_addv)(intptr_t *ptr, long off, intptr_t inc, int cpu) 205{ 206 RSEQ_INJECT_C(9) 207 208 __asm__ __volatile__ goto ( 209 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 210#ifdef RSEQ_COMPARE_TWICE 211 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 212#endif 213 /* Start rseq by storing table entry pointer into rseq_cs. */ 214 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset])) 215 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), 4f) 216 RSEQ_INJECT_ASM(3) 217#ifdef RSEQ_COMPARE_TWICE 218 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), %l[error1]) 219#endif 220 /* get p+v */ 221 "movq %[ptr], %%rbx\n\t" 222 "addq %[off], %%rbx\n\t" 223 /* get pv */ 224 "movq (%%rbx), %%rcx\n\t" 225 /* *pv += inc */ 226 "addq %[inc], (%%rcx)\n\t" 227 "2:\n\t" 228 RSEQ_INJECT_ASM(4) 229 RSEQ_ASM_DEFINE_ABORT(4, "", abort) 230 : /* gcc asm goto does not allow outputs */ 231 : [cpu_id] "r" (cpu), 232 [rseq_offset] "r" (rseq_offset), 233 /* final store input */ 234 [ptr] "m" (*ptr), 235 [off] "er" (off), 236 [inc] "er" (inc) 237 : "memory", "cc", "rax", "rbx", "rcx" 238 RSEQ_INJECT_CLOBBER 239 : abort 240#ifdef RSEQ_COMPARE_TWICE 241 , error1 242#endif 243 ); 244 return 0; 245abort: 246 RSEQ_INJECT_FAILED 247 return -1; 248#ifdef RSEQ_COMPARE_TWICE 249error1: 250 rseq_bug("cpu_id comparison failed"); 251#endif 252} 253 254static inline __attribute__((always_inline)) 255int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_cmpeqv_storev)(intptr_t *v, intptr_t expect, 256 intptr_t *v2, intptr_t expect2, 257 intptr_t newv, int cpu) 258{ 259 RSEQ_INJECT_C(9) 260 261 __asm__ __volatile__ goto ( 262 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 263 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 264#ifdef RSEQ_COMPARE_TWICE 265 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 266 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 267 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3]) 268#endif 269 /* Start rseq by storing table entry pointer into rseq_cs. */ 270 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset])) 271 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), 4f) 272 RSEQ_INJECT_ASM(3) 273 "cmpq %[v], %[expect]\n\t" 274 "jnz %l[cmpfail]\n\t" 275 RSEQ_INJECT_ASM(4) 276 "cmpq %[v2], %[expect2]\n\t" 277 "jnz %l[cmpfail]\n\t" 278 RSEQ_INJECT_ASM(5) 279#ifdef RSEQ_COMPARE_TWICE 280 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), %l[error1]) 281 "cmpq %[v], %[expect]\n\t" 282 "jnz %l[error2]\n\t" 283 "cmpq %[v2], %[expect2]\n\t" 284 "jnz %l[error3]\n\t" 285#endif 286 /* final store */ 287 "movq %[newv], %[v]\n\t" 288 "2:\n\t" 289 RSEQ_INJECT_ASM(6) 290 RSEQ_ASM_DEFINE_ABORT(4, "", abort) 291 : /* gcc asm goto does not allow outputs */ 292 : [cpu_id] "r" (cpu), 293 [rseq_offset] "r" (rseq_offset), 294 /* cmp2 input */ 295 [v2] "m" (*v2), 296 [expect2] "r" (expect2), 297 /* final store input */ 298 [v] "m" (*v), 299 [expect] "r" (expect), 300 [newv] "r" (newv) 301 : "memory", "cc", "rax" 302 RSEQ_INJECT_CLOBBER 303 : abort, cmpfail 304#ifdef RSEQ_COMPARE_TWICE 305 , error1, error2, error3 306#endif 307 ); 308 rseq_after_asm_goto(); 309 return 0; 310abort: 311 rseq_after_asm_goto(); 312 RSEQ_INJECT_FAILED 313 return -1; 314cmpfail: 315 rseq_after_asm_goto(); 316 return 1; 317#ifdef RSEQ_COMPARE_TWICE 318error1: 319 rseq_after_asm_goto(); 320 rseq_bug("cpu_id comparison failed"); 321error2: 322 rseq_after_asm_goto(); 323 rseq_bug("1st expected value comparison failed"); 324error3: 325 rseq_after_asm_goto(); 326 rseq_bug("2nd expected value comparison failed"); 327#endif 328} 329 330#endif /* #if defined(RSEQ_TEMPLATE_MO_RELAXED) && 331 (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) */ 332 333#if (defined(RSEQ_TEMPLATE_MO_RELAXED) || defined(RSEQ_TEMPLATE_MO_RELEASE)) && \ 334 (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) 335 336static inline __attribute__((always_inline)) 337int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trystorev_storev)(intptr_t *v, intptr_t expect, 338 intptr_t *v2, intptr_t newv2, 339 intptr_t newv, int cpu) 340{ 341 RSEQ_INJECT_C(9) 342 343 __asm__ __volatile__ goto ( 344 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 345 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 346#ifdef RSEQ_COMPARE_TWICE 347 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 348 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 349#endif 350 /* Start rseq by storing table entry pointer into rseq_cs. */ 351 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset])) 352 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), 4f) 353 RSEQ_INJECT_ASM(3) 354 "cmpq %[v], %[expect]\n\t" 355 "jnz %l[cmpfail]\n\t" 356 RSEQ_INJECT_ASM(4) 357#ifdef RSEQ_COMPARE_TWICE 358 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), %l[error1]) 359 "cmpq %[v], %[expect]\n\t" 360 "jnz %l[error2]\n\t" 361#endif 362 /* try store */ 363 "movq %[newv2], %[v2]\n\t" 364 RSEQ_INJECT_ASM(5) 365 /* final store */ 366 "movq %[newv], %[v]\n\t" 367 "2:\n\t" 368 RSEQ_INJECT_ASM(6) 369 RSEQ_ASM_DEFINE_ABORT(4, "", abort) 370 : /* gcc asm goto does not allow outputs */ 371 : [cpu_id] "r" (cpu), 372 [rseq_offset] "r" (rseq_offset), 373 /* try store input */ 374 [v2] "m" (*v2), 375 [newv2] "r" (newv2), 376 /* final store input */ 377 [v] "m" (*v), 378 [expect] "r" (expect), 379 [newv] "r" (newv) 380 : "memory", "cc", "rax" 381 RSEQ_INJECT_CLOBBER 382 : abort, cmpfail 383#ifdef RSEQ_COMPARE_TWICE 384 , error1, error2 385#endif 386 ); 387 rseq_after_asm_goto(); 388 return 0; 389abort: 390 rseq_after_asm_goto(); 391 RSEQ_INJECT_FAILED 392 return -1; 393cmpfail: 394 rseq_after_asm_goto(); 395 return 1; 396#ifdef RSEQ_COMPARE_TWICE 397error1: 398 rseq_after_asm_goto(); 399 rseq_bug("cpu_id comparison failed"); 400error2: 401 rseq_after_asm_goto(); 402 rseq_bug("expected value comparison failed"); 403#endif 404} 405 406static inline __attribute__((always_inline)) 407int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trymemcpy_storev)(intptr_t *v, intptr_t expect, 408 void *dst, void *src, size_t len, 409 intptr_t newv, int cpu) 410{ 411 uint64_t rseq_scratch[3]; 412 413 RSEQ_INJECT_C(9) 414 415 __asm__ __volatile__ goto ( 416 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 417 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 418#ifdef RSEQ_COMPARE_TWICE 419 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 420 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 421#endif 422 "movq %[src], %[rseq_scratch0]\n\t" 423 "movq %[dst], %[rseq_scratch1]\n\t" 424 "movq %[len], %[rseq_scratch2]\n\t" 425 /* Start rseq by storing table entry pointer into rseq_cs. */ 426 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset])) 427 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), 4f) 428 RSEQ_INJECT_ASM(3) 429 "cmpq %[v], %[expect]\n\t" 430 "jnz 5f\n\t" 431 RSEQ_INJECT_ASM(4) 432#ifdef RSEQ_COMPARE_TWICE 433 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), 6f) 434 "cmpq %[v], %[expect]\n\t" 435 "jnz 7f\n\t" 436#endif 437 /* try memcpy */ 438 "test %[len], %[len]\n\t" \ 439 "jz 333f\n\t" \ 440 "222:\n\t" \ 441 "movb (%[src]), %%al\n\t" \ 442 "movb %%al, (%[dst])\n\t" \ 443 "inc %[src]\n\t" \ 444 "inc %[dst]\n\t" \ 445 "dec %[len]\n\t" \ 446 "jnz 222b\n\t" \ 447 "333:\n\t" \ 448 RSEQ_INJECT_ASM(5) 449 /* final store */ 450 "movq %[newv], %[v]\n\t" 451 "2:\n\t" 452 RSEQ_INJECT_ASM(6) 453 /* teardown */ 454 "movq %[rseq_scratch2], %[len]\n\t" 455 "movq %[rseq_scratch1], %[dst]\n\t" 456 "movq %[rseq_scratch0], %[src]\n\t" 457 RSEQ_ASM_DEFINE_ABORT(4, 458 "movq %[rseq_scratch2], %[len]\n\t" 459 "movq %[rseq_scratch1], %[dst]\n\t" 460 "movq %[rseq_scratch0], %[src]\n\t", 461 abort) 462 RSEQ_ASM_DEFINE_CMPFAIL(5, 463 "movq %[rseq_scratch2], %[len]\n\t" 464 "movq %[rseq_scratch1], %[dst]\n\t" 465 "movq %[rseq_scratch0], %[src]\n\t", 466 cmpfail) 467#ifdef RSEQ_COMPARE_TWICE 468 RSEQ_ASM_DEFINE_CMPFAIL(6, 469 "movq %[rseq_scratch2], %[len]\n\t" 470 "movq %[rseq_scratch1], %[dst]\n\t" 471 "movq %[rseq_scratch0], %[src]\n\t", 472 error1) 473 RSEQ_ASM_DEFINE_CMPFAIL(7, 474 "movq %[rseq_scratch2], %[len]\n\t" 475 "movq %[rseq_scratch1], %[dst]\n\t" 476 "movq %[rseq_scratch0], %[src]\n\t", 477 error2) 478#endif 479 : /* gcc asm goto does not allow outputs */ 480 : [cpu_id] "r" (cpu), 481 [rseq_offset] "r" (rseq_offset), 482 /* final store input */ 483 [v] "m" (*v), 484 [expect] "r" (expect), 485 [newv] "r" (newv), 486 /* try memcpy input */ 487 [dst] "r" (dst), 488 [src] "r" (src), 489 [len] "r" (len), 490 [rseq_scratch0] "m" (rseq_scratch[0]), 491 [rseq_scratch1] "m" (rseq_scratch[1]), 492 [rseq_scratch2] "m" (rseq_scratch[2]) 493 : "memory", "cc", "rax" 494 RSEQ_INJECT_CLOBBER 495 : abort, cmpfail 496#ifdef RSEQ_COMPARE_TWICE 497 , error1, error2 498#endif 499 ); 500 rseq_after_asm_goto(); 501 return 0; 502abort: 503 rseq_after_asm_goto(); 504 RSEQ_INJECT_FAILED 505 return -1; 506cmpfail: 507 rseq_after_asm_goto(); 508 return 1; 509#ifdef RSEQ_COMPARE_TWICE 510error1: 511 rseq_after_asm_goto(); 512 rseq_bug("cpu_id comparison failed"); 513error2: 514 rseq_after_asm_goto(); 515 rseq_bug("expected value comparison failed"); 516#endif 517} 518 519#endif /* #if (defined(RSEQ_TEMPLATE_MO_RELAXED) || defined(RSEQ_TEMPLATE_MO_RELEASE)) && 520 (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) */ 521 522#elif defined(__i386__) 523 524#if defined(RSEQ_TEMPLATE_MO_RELAXED) && \ 525 (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) 526 527static inline __attribute__((always_inline)) 528int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_storev)(intptr_t *v, intptr_t expect, intptr_t newv, int cpu) 529{ 530 RSEQ_INJECT_C(9) 531 532 __asm__ __volatile__ goto ( 533 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 534 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 535#ifdef RSEQ_COMPARE_TWICE 536 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 537 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 538#endif 539 /* Start rseq by storing table entry pointer into rseq_cs. */ 540 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset])) 541 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), 4f) 542 RSEQ_INJECT_ASM(3) 543 "cmpl %[v], %[expect]\n\t" 544 "jnz %l[cmpfail]\n\t" 545 RSEQ_INJECT_ASM(4) 546#ifdef RSEQ_COMPARE_TWICE 547 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), %l[error1]) 548 "cmpl %[v], %[expect]\n\t" 549 "jnz %l[error2]\n\t" 550#endif 551 /* final store */ 552 "movl %[newv], %[v]\n\t" 553 "2:\n\t" 554 RSEQ_INJECT_ASM(5) 555 RSEQ_ASM_DEFINE_ABORT(4, "", abort) 556 : /* gcc asm goto does not allow outputs */ 557 : [cpu_id] "r" (cpu), 558 [rseq_offset] "r" (rseq_offset), 559 [v] "m" (*v), 560 [expect] "r" (expect), 561 [newv] "r" (newv) 562 : "memory", "cc", "eax" 563 RSEQ_INJECT_CLOBBER 564 : abort, cmpfail 565#ifdef RSEQ_COMPARE_TWICE 566 , error1, error2 567#endif 568 ); 569 rseq_after_asm_goto(); 570 return 0; 571abort: 572 rseq_after_asm_goto(); 573 RSEQ_INJECT_FAILED 574 return -1; 575cmpfail: 576 rseq_after_asm_goto(); 577 return 1; 578#ifdef RSEQ_COMPARE_TWICE 579error1: 580 rseq_after_asm_goto(); 581 rseq_bug("cpu_id comparison failed"); 582error2: 583 rseq_after_asm_goto(); 584 rseq_bug("expected value comparison failed"); 585#endif 586} 587 588/* 589 * Compare @v against @expectnot. When it does _not_ match, load @v 590 * into @load, and store the content of *@v + voffp into @v. 591 */ 592static inline __attribute__((always_inline)) 593int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpnev_storeoffp_load)(intptr_t *v, intptr_t expectnot, 594 long voffp, intptr_t *load, int cpu) 595{ 596 RSEQ_INJECT_C(9) 597 598 __asm__ __volatile__ goto ( 599 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 600 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 601#ifdef RSEQ_COMPARE_TWICE 602 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 603 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 604#endif 605 /* Start rseq by storing table entry pointer into rseq_cs. */ 606 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset])) 607 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), 4f) 608 RSEQ_INJECT_ASM(3) 609 "movl %[v], %%ebx\n\t" 610 "cmpl %%ebx, %[expectnot]\n\t" 611 "je %l[cmpfail]\n\t" 612 RSEQ_INJECT_ASM(4) 613#ifdef RSEQ_COMPARE_TWICE 614 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), %l[error1]) 615 "movl %[v], %%ebx\n\t" 616 "cmpl %%ebx, %[expectnot]\n\t" 617 "je %l[error2]\n\t" 618#endif 619 "movl %%ebx, %[load]\n\t" 620 "addl %[voffp], %%ebx\n\t" 621 "movl (%%ebx), %%ebx\n\t" 622 /* final store */ 623 "movl %%ebx, %[v]\n\t" 624 "2:\n\t" 625 RSEQ_INJECT_ASM(5) 626 RSEQ_ASM_DEFINE_ABORT(4, "", abort) 627 : /* gcc asm goto does not allow outputs */ 628 : [cpu_id] "r" (cpu), 629 [rseq_offset] "r" (rseq_offset), 630 /* final store input */ 631 [v] "m" (*v), 632 [expectnot] "r" (expectnot), 633 [voffp] "ir" (voffp), 634 [load] "m" (*load) 635 : "memory", "cc", "eax", "ebx" 636 RSEQ_INJECT_CLOBBER 637 : abort, cmpfail 638#ifdef RSEQ_COMPARE_TWICE 639 , error1, error2 640#endif 641 ); 642 rseq_after_asm_goto(); 643 return 0; 644abort: 645 rseq_after_asm_goto(); 646 RSEQ_INJECT_FAILED 647 return -1; 648cmpfail: 649 rseq_after_asm_goto(); 650 return 1; 651#ifdef RSEQ_COMPARE_TWICE 652error1: 653 rseq_after_asm_goto(); 654 rseq_bug("cpu_id comparison failed"); 655error2: 656 rseq_after_asm_goto(); 657 rseq_bug("expected value comparison failed"); 658#endif 659} 660 661static inline __attribute__((always_inline)) 662int RSEQ_TEMPLATE_IDENTIFIER(rseq_addv)(intptr_t *v, intptr_t count, int cpu) 663{ 664 RSEQ_INJECT_C(9) 665 666 __asm__ __volatile__ goto ( 667 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 668#ifdef RSEQ_COMPARE_TWICE 669 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 670#endif 671 /* Start rseq by storing table entry pointer into rseq_cs. */ 672 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset])) 673 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), 4f) 674 RSEQ_INJECT_ASM(3) 675#ifdef RSEQ_COMPARE_TWICE 676 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), %l[error1]) 677#endif 678 /* final store */ 679 "addl %[count], %[v]\n\t" 680 "2:\n\t" 681 RSEQ_INJECT_ASM(4) 682 RSEQ_ASM_DEFINE_ABORT(4, "", abort) 683 : /* gcc asm goto does not allow outputs */ 684 : [cpu_id] "r" (cpu), 685 [rseq_offset] "r" (rseq_offset), 686 /* final store input */ 687 [v] "m" (*v), 688 [count] "ir" (count) 689 : "memory", "cc", "eax" 690 RSEQ_INJECT_CLOBBER 691 : abort 692#ifdef RSEQ_COMPARE_TWICE 693 , error1 694#endif 695 ); 696 rseq_after_asm_goto(); 697 return 0; 698abort: 699 rseq_after_asm_goto(); 700 RSEQ_INJECT_FAILED 701 return -1; 702#ifdef RSEQ_COMPARE_TWICE 703error1: 704 rseq_after_asm_goto(); 705 rseq_bug("cpu_id comparison failed"); 706#endif 707} 708 709static inline __attribute__((always_inline)) 710int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_cmpeqv_storev)(intptr_t *v, intptr_t expect, 711 intptr_t *v2, intptr_t expect2, 712 intptr_t newv, int cpu) 713{ 714 RSEQ_INJECT_C(9) 715 716 __asm__ __volatile__ goto ( 717 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 718 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 719#ifdef RSEQ_COMPARE_TWICE 720 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 721 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 722 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3]) 723#endif 724 /* Start rseq by storing table entry pointer into rseq_cs. */ 725 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset])) 726 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), 4f) 727 RSEQ_INJECT_ASM(3) 728 "cmpl %[v], %[expect]\n\t" 729 "jnz %l[cmpfail]\n\t" 730 RSEQ_INJECT_ASM(4) 731 "cmpl %[expect2], %[v2]\n\t" 732 "jnz %l[cmpfail]\n\t" 733 RSEQ_INJECT_ASM(5) 734#ifdef RSEQ_COMPARE_TWICE 735 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), %l[error1]) 736 "cmpl %[v], %[expect]\n\t" 737 "jnz %l[error2]\n\t" 738 "cmpl %[expect2], %[v2]\n\t" 739 "jnz %l[error3]\n\t" 740#endif 741 "movl %[newv], %%eax\n\t" 742 /* final store */ 743 "movl %%eax, %[v]\n\t" 744 "2:\n\t" 745 RSEQ_INJECT_ASM(6) 746 RSEQ_ASM_DEFINE_ABORT(4, "", abort) 747 : /* gcc asm goto does not allow outputs */ 748 : [cpu_id] "r" (cpu), 749 [rseq_offset] "r" (rseq_offset), 750 /* cmp2 input */ 751 [v2] "m" (*v2), 752 [expect2] "r" (expect2), 753 /* final store input */ 754 [v] "m" (*v), 755 [expect] "r" (expect), 756 [newv] "m" (newv) 757 : "memory", "cc", "eax" 758 RSEQ_INJECT_CLOBBER 759 : abort, cmpfail 760#ifdef RSEQ_COMPARE_TWICE 761 , error1, error2, error3 762#endif 763 ); 764 rseq_after_asm_goto(); 765 return 0; 766abort: 767 rseq_after_asm_goto(); 768 RSEQ_INJECT_FAILED 769 return -1; 770cmpfail: 771 rseq_after_asm_goto(); 772 return 1; 773#ifdef RSEQ_COMPARE_TWICE 774error1: 775 rseq_after_asm_goto(); 776 rseq_bug("cpu_id comparison failed"); 777error2: 778 rseq_after_asm_goto(); 779 rseq_bug("1st expected value comparison failed"); 780error3: 781 rseq_after_asm_goto(); 782 rseq_bug("2nd expected value comparison failed"); 783#endif 784} 785 786#endif /* #if defined(RSEQ_TEMPLATE_MO_RELAXED) && 787 (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) */ 788 789#if (defined(RSEQ_TEMPLATE_MO_RELAXED) || defined(RSEQ_TEMPLATE_MO_RELEASE)) && \ 790 (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) 791 792static inline __attribute__((always_inline)) 793int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trystorev_storev)(intptr_t *v, intptr_t expect, 794 intptr_t *v2, intptr_t newv2, 795 intptr_t newv, int cpu) 796{ 797 RSEQ_INJECT_C(9) 798 799 __asm__ __volatile__ goto ( 800 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 801 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 802#ifdef RSEQ_COMPARE_TWICE 803 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 804 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 805#endif 806 /* Start rseq by storing table entry pointer into rseq_cs. */ 807 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset])) 808 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), 4f) 809 RSEQ_INJECT_ASM(3) 810 "movl %[expect], %%eax\n\t" 811 "cmpl %[v], %%eax\n\t" 812 "jnz %l[cmpfail]\n\t" 813 RSEQ_INJECT_ASM(4) 814#ifdef RSEQ_COMPARE_TWICE 815 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), %l[error1]) 816 "movl %[expect], %%eax\n\t" 817 "cmpl %[v], %%eax\n\t" 818 "jnz %l[error2]\n\t" 819#endif 820 /* try store */ 821 "movl %[newv2], %[v2]\n\t" 822 RSEQ_INJECT_ASM(5) 823#ifdef RSEQ_TEMPLATE_MO_RELEASE 824 "lock; addl $0,-128(%%esp)\n\t" 825#endif 826 /* final store */ 827 "movl %[newv], %[v]\n\t" 828 "2:\n\t" 829 RSEQ_INJECT_ASM(6) 830 RSEQ_ASM_DEFINE_ABORT(4, "", abort) 831 : /* gcc asm goto does not allow outputs */ 832 : [cpu_id] "r" (cpu), 833 [rseq_offset] "r" (rseq_offset), 834 /* try store input */ 835 [v2] "m" (*v2), 836 [newv2] "r" (newv2), 837 /* final store input */ 838 [v] "m" (*v), 839 [expect] "m" (expect), 840 [newv] "r" (newv) 841 : "memory", "cc", "eax" 842 RSEQ_INJECT_CLOBBER 843 : abort, cmpfail 844#ifdef RSEQ_COMPARE_TWICE 845 , error1, error2 846#endif 847 ); 848 rseq_after_asm_goto(); 849 return 0; 850abort: 851 rseq_after_asm_goto(); 852 RSEQ_INJECT_FAILED 853 return -1; 854cmpfail: 855 rseq_after_asm_goto(); 856 return 1; 857#ifdef RSEQ_COMPARE_TWICE 858error1: 859 rseq_after_asm_goto(); 860 rseq_bug("cpu_id comparison failed"); 861error2: 862 rseq_after_asm_goto(); 863 rseq_bug("expected value comparison failed"); 864#endif 865 866} 867 868/* TODO: implement a faster memcpy. */ 869static inline __attribute__((always_inline)) 870int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trymemcpy_storev)(intptr_t *v, intptr_t expect, 871 void *dst, void *src, size_t len, 872 intptr_t newv, int cpu) 873{ 874 uint32_t rseq_scratch[3]; 875 876 RSEQ_INJECT_C(9) 877 878 __asm__ __volatile__ goto ( 879 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 880 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 881#ifdef RSEQ_COMPARE_TWICE 882 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 883 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 884#endif 885 "movl %[src], %[rseq_scratch0]\n\t" 886 "movl %[dst], %[rseq_scratch1]\n\t" 887 "movl %[len], %[rseq_scratch2]\n\t" 888 /* Start rseq by storing table entry pointer into rseq_cs. */ 889 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset])) 890 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), 4f) 891 RSEQ_INJECT_ASM(3) 892 "movl %[expect], %%eax\n\t" 893 "cmpl %%eax, %[v]\n\t" 894 "jnz 5f\n\t" 895 RSEQ_INJECT_ASM(4) 896#ifdef RSEQ_COMPARE_TWICE 897 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), 6f) 898 "movl %[expect], %%eax\n\t" 899 "cmpl %%eax, %[v]\n\t" 900 "jnz 7f\n\t" 901#endif 902 /* try memcpy */ 903 "test %[len], %[len]\n\t" \ 904 "jz 333f\n\t" \ 905 "222:\n\t" \ 906 "movb (%[src]), %%al\n\t" \ 907 "movb %%al, (%[dst])\n\t" \ 908 "inc %[src]\n\t" \ 909 "inc %[dst]\n\t" \ 910 "dec %[len]\n\t" \ 911 "jnz 222b\n\t" \ 912 "333:\n\t" \ 913 RSEQ_INJECT_ASM(5) 914#ifdef RSEQ_TEMPLATE_MO_RELEASE 915 "lock; addl $0,-128(%%esp)\n\t" 916#endif 917 "movl %[newv], %%eax\n\t" 918 /* final store */ 919 "movl %%eax, %[v]\n\t" 920 "2:\n\t" 921 RSEQ_INJECT_ASM(6) 922 /* teardown */ 923 "movl %[rseq_scratch2], %[len]\n\t" 924 "movl %[rseq_scratch1], %[dst]\n\t" 925 "movl %[rseq_scratch0], %[src]\n\t" 926 RSEQ_ASM_DEFINE_ABORT(4, 927 "movl %[rseq_scratch2], %[len]\n\t" 928 "movl %[rseq_scratch1], %[dst]\n\t" 929 "movl %[rseq_scratch0], %[src]\n\t", 930 abort) 931 RSEQ_ASM_DEFINE_CMPFAIL(5, 932 "movl %[rseq_scratch2], %[len]\n\t" 933 "movl %[rseq_scratch1], %[dst]\n\t" 934 "movl %[rseq_scratch0], %[src]\n\t", 935 cmpfail) 936#ifdef RSEQ_COMPARE_TWICE 937 RSEQ_ASM_DEFINE_CMPFAIL(6, 938 "movl %[rseq_scratch2], %[len]\n\t" 939 "movl %[rseq_scratch1], %[dst]\n\t" 940 "movl %[rseq_scratch0], %[src]\n\t", 941 error1) 942 RSEQ_ASM_DEFINE_CMPFAIL(7, 943 "movl %[rseq_scratch2], %[len]\n\t" 944 "movl %[rseq_scratch1], %[dst]\n\t" 945 "movl %[rseq_scratch0], %[src]\n\t", 946 error2) 947#endif 948 : /* gcc asm goto does not allow outputs */ 949 : [cpu_id] "r" (cpu), 950 [rseq_offset] "r" (rseq_offset), 951 /* final store input */ 952 [v] "m" (*v), 953 [expect] "m" (expect), 954 [newv] "m" (newv), 955 /* try memcpy input */ 956 [dst] "r" (dst), 957 [src] "r" (src), 958 [len] "r" (len), 959 [rseq_scratch0] "m" (rseq_scratch[0]), 960 [rseq_scratch1] "m" (rseq_scratch[1]), 961 [rseq_scratch2] "m" (rseq_scratch[2]) 962 : "memory", "cc", "eax" 963 RSEQ_INJECT_CLOBBER 964 : abort, cmpfail 965#ifdef RSEQ_COMPARE_TWICE 966 , error1, error2 967#endif 968 ); 969 rseq_after_asm_goto(); 970 return 0; 971abort: 972 rseq_after_asm_goto(); 973 RSEQ_INJECT_FAILED 974 return -1; 975cmpfail: 976 rseq_after_asm_goto(); 977 return 1; 978#ifdef RSEQ_COMPARE_TWICE 979error1: 980 rseq_after_asm_goto(); 981 rseq_bug("cpu_id comparison failed"); 982error2: 983 rseq_after_asm_goto(); 984 rseq_bug("expected value comparison failed"); 985#endif 986} 987 988#endif /* #if (defined(RSEQ_TEMPLATE_MO_RELAXED) || defined(RSEQ_TEMPLATE_MO_RELEASE)) && 989 (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) */ 990 991#endif 992 993#include "rseq-bits-reset.h" 994