1/* SPDX-License-Identifier: LGPL-2.1 OR MIT */ 2/* 3 * Author: Paul Burton <paul.burton@mips.com> 4 * (C) Copyright 2018 MIPS Tech LLC 5 * (C) Copyright 2016-2022 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com> 6 */ 7 8#include "rseq-bits-template.h" 9 10#if defined(RSEQ_TEMPLATE_MO_RELAXED) && \ 11 (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) 12 13static inline __attribute__((always_inline)) 14int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_storev)(intptr_t *v, intptr_t expect, intptr_t newv, int cpu) 15{ 16 RSEQ_INJECT_C(9) 17 18 __asm__ __volatile__ goto ( 19 RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */ 20 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 21#ifdef RSEQ_COMPARE_TWICE 22 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 23 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 24#endif 25 /* Start rseq by storing table entry pointer into rseq_cs. */ 26 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) 27 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 28 RSEQ_INJECT_ASM(3) 29 LONG_L " $4, %[v]\n\t" 30 "bne $4, %[expect], %l[cmpfail]\n\t" 31 RSEQ_INJECT_ASM(4) 32#ifdef RSEQ_COMPARE_TWICE 33 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 34 LONG_L " $4, %[v]\n\t" 35 "bne $4, %[expect], %l[error2]\n\t" 36#endif 37 /* final store */ 38 LONG_S " %[newv], %[v]\n\t" 39 "2:\n\t" 40 RSEQ_INJECT_ASM(5) 41 "b 5f\n\t" 42 RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f) 43 "5:\n\t" 44 : /* gcc asm goto does not allow outputs */ 45 : [cpu_id] "r" (cpu), 46 [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD), 47 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), 48 [v] "m" (*v), 49 [expect] "r" (expect), 50 [newv] "r" (newv) 51 RSEQ_INJECT_INPUT 52 : "$4", "memory" 53 RSEQ_INJECT_CLOBBER 54 : abort, cmpfail 55#ifdef RSEQ_COMPARE_TWICE 56 , error1, error2 57#endif 58 ); 59 return 0; 60abort: 61 RSEQ_INJECT_FAILED 62 return -1; 63cmpfail: 64 return 1; 65#ifdef RSEQ_COMPARE_TWICE 66error1: 67 rseq_bug("cpu_id comparison failed"); 68error2: 69 rseq_bug("expected value comparison failed"); 70#endif 71} 72 73static inline __attribute__((always_inline)) 74int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpnev_storeoffp_load)(intptr_t *v, intptr_t expectnot, 75 long voffp, intptr_t *load, int cpu) 76{ 77 RSEQ_INJECT_C(9) 78 79 __asm__ __volatile__ goto ( 80 RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */ 81 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 82#ifdef RSEQ_COMPARE_TWICE 83 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 84 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 85#endif 86 /* Start rseq by storing table entry pointer into rseq_cs. */ 87 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) 88 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 89 RSEQ_INJECT_ASM(3) 90 LONG_L " $4, %[v]\n\t" 91 "beq $4, %[expectnot], %l[cmpfail]\n\t" 92 RSEQ_INJECT_ASM(4) 93#ifdef RSEQ_COMPARE_TWICE 94 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 95 LONG_L " $4, %[v]\n\t" 96 "beq $4, %[expectnot], %l[error2]\n\t" 97#endif 98 LONG_S " $4, %[load]\n\t" 99 LONG_ADDI " $4, %[voffp]\n\t" 100 LONG_L " $4, 0($4)\n\t" 101 /* final store */ 102 LONG_S " $4, %[v]\n\t" 103 "2:\n\t" 104 RSEQ_INJECT_ASM(5) 105 "b 5f\n\t" 106 RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f) 107 "5:\n\t" 108 : /* gcc asm goto does not allow outputs */ 109 : [cpu_id] "r" (cpu), 110 [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD), 111 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), 112 /* final store input */ 113 [v] "m" (*v), 114 [expectnot] "r" (expectnot), 115 [voffp] "Ir" (voffp), 116 [load] "m" (*load) 117 RSEQ_INJECT_INPUT 118 : "$4", "memory" 119 RSEQ_INJECT_CLOBBER 120 : abort, cmpfail 121#ifdef RSEQ_COMPARE_TWICE 122 , error1, error2 123#endif 124 ); 125 return 0; 126abort: 127 RSEQ_INJECT_FAILED 128 return -1; 129cmpfail: 130 return 1; 131#ifdef RSEQ_COMPARE_TWICE 132error1: 133 rseq_bug("cpu_id comparison failed"); 134error2: 135 rseq_bug("expected value comparison failed"); 136#endif 137} 138 139static inline __attribute__((always_inline)) 140int RSEQ_TEMPLATE_IDENTIFIER(rseq_addv)(intptr_t *v, intptr_t count, int cpu) 141{ 142 RSEQ_INJECT_C(9) 143 144 __asm__ __volatile__ goto ( 145 RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */ 146#ifdef RSEQ_COMPARE_TWICE 147 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 148#endif 149 /* Start rseq by storing table entry pointer into rseq_cs. */ 150 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) 151 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 152 RSEQ_INJECT_ASM(3) 153#ifdef RSEQ_COMPARE_TWICE 154 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 155#endif 156 LONG_L " $4, %[v]\n\t" 157 LONG_ADDI " $4, %[count]\n\t" 158 /* final store */ 159 LONG_S " $4, %[v]\n\t" 160 "2:\n\t" 161 RSEQ_INJECT_ASM(4) 162 "b 5f\n\t" 163 RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f) 164 "5:\n\t" 165 : /* gcc asm goto does not allow outputs */ 166 : [cpu_id] "r" (cpu), 167 [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD), 168 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), 169 [v] "m" (*v), 170 [count] "Ir" (count) 171 RSEQ_INJECT_INPUT 172 : "$4", "memory" 173 RSEQ_INJECT_CLOBBER 174 : abort 175#ifdef RSEQ_COMPARE_TWICE 176 , error1 177#endif 178 ); 179 return 0; 180abort: 181 RSEQ_INJECT_FAILED 182 return -1; 183#ifdef RSEQ_COMPARE_TWICE 184error1: 185 rseq_bug("cpu_id comparison failed"); 186#endif 187} 188 189static inline __attribute__((always_inline)) 190int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_cmpeqv_storev)(intptr_t *v, intptr_t expect, 191 intptr_t *v2, intptr_t expect2, 192 intptr_t newv, int cpu) 193{ 194 RSEQ_INJECT_C(9) 195 196 __asm__ __volatile__ goto ( 197 RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */ 198 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 199#ifdef RSEQ_COMPARE_TWICE 200 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 201 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 202 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3]) 203#endif 204 /* Start rseq by storing table entry pointer into rseq_cs. */ 205 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) 206 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 207 RSEQ_INJECT_ASM(3) 208 LONG_L " $4, %[v]\n\t" 209 "bne $4, %[expect], %l[cmpfail]\n\t" 210 RSEQ_INJECT_ASM(4) 211 LONG_L " $4, %[v2]\n\t" 212 "bne $4, %[expect2], %l[cmpfail]\n\t" 213 RSEQ_INJECT_ASM(5) 214#ifdef RSEQ_COMPARE_TWICE 215 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 216 LONG_L " $4, %[v]\n\t" 217 "bne $4, %[expect], %l[error2]\n\t" 218 LONG_L " $4, %[v2]\n\t" 219 "bne $4, %[expect2], %l[error3]\n\t" 220#endif 221 /* final store */ 222 LONG_S " %[newv], %[v]\n\t" 223 "2:\n\t" 224 RSEQ_INJECT_ASM(6) 225 "b 5f\n\t" 226 RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f) 227 "5:\n\t" 228 : /* gcc asm goto does not allow outputs */ 229 : [cpu_id] "r" (cpu), 230 [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD), 231 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), 232 /* cmp2 input */ 233 [v2] "m" (*v2), 234 [expect2] "r" (expect2), 235 /* final store input */ 236 [v] "m" (*v), 237 [expect] "r" (expect), 238 [newv] "r" (newv) 239 RSEQ_INJECT_INPUT 240 : "$4", "memory" 241 RSEQ_INJECT_CLOBBER 242 : abort, cmpfail 243#ifdef RSEQ_COMPARE_TWICE 244 , error1, error2, error3 245#endif 246 ); 247 return 0; 248abort: 249 RSEQ_INJECT_FAILED 250 return -1; 251cmpfail: 252 return 1; 253#ifdef RSEQ_COMPARE_TWICE 254error1: 255 rseq_bug("cpu_id comparison failed"); 256error2: 257 rseq_bug("1st expected value comparison failed"); 258error3: 259 rseq_bug("2nd expected value comparison failed"); 260#endif 261} 262 263#endif /* #if defined(RSEQ_TEMPLATE_MO_RELAXED) && 264 (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) */ 265 266#if (defined(RSEQ_TEMPLATE_MO_RELAXED) || defined(RSEQ_TEMPLATE_MO_RELEASE)) && \ 267 (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) 268 269static inline __attribute__((always_inline)) 270int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trystorev_storev)(intptr_t *v, intptr_t expect, 271 intptr_t *v2, intptr_t newv2, 272 intptr_t newv, int cpu) 273{ 274 RSEQ_INJECT_C(9) 275 276 __asm__ __volatile__ goto ( 277 RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */ 278 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 279#ifdef RSEQ_COMPARE_TWICE 280 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 281 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 282#endif 283 /* Start rseq by storing table entry pointer into rseq_cs. */ 284 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) 285 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 286 RSEQ_INJECT_ASM(3) 287 LONG_L " $4, %[v]\n\t" 288 "bne $4, %[expect], %l[cmpfail]\n\t" 289 RSEQ_INJECT_ASM(4) 290#ifdef RSEQ_COMPARE_TWICE 291 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 292 LONG_L " $4, %[v]\n\t" 293 "bne $4, %[expect], %l[error2]\n\t" 294#endif 295 /* try store */ 296 LONG_S " %[newv2], %[v2]\n\t" 297 RSEQ_INJECT_ASM(5) 298#ifdef RSEQ_TEMPLATE_MO_RELEASE 299 "sync\n\t" /* full sync provides store-release */ 300#endif 301 /* final store */ 302 LONG_S " %[newv], %[v]\n\t" 303 "2:\n\t" 304 RSEQ_INJECT_ASM(6) 305 "b 5f\n\t" 306 RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f) 307 "5:\n\t" 308 : /* gcc asm goto does not allow outputs */ 309 : [cpu_id] "r" (cpu), 310 [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD), 311 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), 312 /* try store input */ 313 [v2] "m" (*v2), 314 [newv2] "r" (newv2), 315 /* final store input */ 316 [v] "m" (*v), 317 [expect] "r" (expect), 318 [newv] "r" (newv) 319 RSEQ_INJECT_INPUT 320 : "$4", "memory" 321 RSEQ_INJECT_CLOBBER 322 : abort, cmpfail 323#ifdef RSEQ_COMPARE_TWICE 324 , error1, error2 325#endif 326 ); 327 return 0; 328abort: 329 RSEQ_INJECT_FAILED 330 return -1; 331cmpfail: 332 return 1; 333#ifdef RSEQ_COMPARE_TWICE 334error1: 335 rseq_bug("cpu_id comparison failed"); 336error2: 337 rseq_bug("expected value comparison failed"); 338#endif 339} 340 341static inline __attribute__((always_inline)) 342int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trymemcpy_storev)(intptr_t *v, intptr_t expect, 343 void *dst, void *src, size_t len, 344 intptr_t newv, int cpu) 345{ 346 uintptr_t rseq_scratch[3]; 347 348 RSEQ_INJECT_C(9) 349 350 __asm__ __volatile__ goto ( 351 RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */ 352 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 353#ifdef RSEQ_COMPARE_TWICE 354 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 355 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 356#endif 357 LONG_S " %[src], %[rseq_scratch0]\n\t" 358 LONG_S " %[dst], %[rseq_scratch1]\n\t" 359 LONG_S " %[len], %[rseq_scratch2]\n\t" 360 /* Start rseq by storing table entry pointer into rseq_cs. */ 361 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) 362 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 363 RSEQ_INJECT_ASM(3) 364 LONG_L " $4, %[v]\n\t" 365 "bne $4, %[expect], 5f\n\t" 366 RSEQ_INJECT_ASM(4) 367#ifdef RSEQ_COMPARE_TWICE 368 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 6f) 369 LONG_L " $4, %[v]\n\t" 370 "bne $4, %[expect], 7f\n\t" 371#endif 372 /* try memcpy */ 373 "beqz %[len], 333f\n\t" \ 374 "222:\n\t" \ 375 "lb $4, 0(%[src])\n\t" \ 376 "sb $4, 0(%[dst])\n\t" \ 377 LONG_ADDI " %[src], 1\n\t" \ 378 LONG_ADDI " %[dst], 1\n\t" \ 379 LONG_ADDI " %[len], -1\n\t" \ 380 "bnez %[len], 222b\n\t" \ 381 "333:\n\t" \ 382 RSEQ_INJECT_ASM(5) 383#ifdef RSEQ_TEMPLATE_MO_RELEASE 384 "sync\n\t" /* full sync provides store-release */ 385#endif 386 /* final store */ 387 LONG_S " %[newv], %[v]\n\t" 388 "2:\n\t" 389 RSEQ_INJECT_ASM(6) 390 /* teardown */ 391 LONG_L " %[len], %[rseq_scratch2]\n\t" 392 LONG_L " %[dst], %[rseq_scratch1]\n\t" 393 LONG_L " %[src], %[rseq_scratch0]\n\t" 394 "b 8f\n\t" 395 RSEQ_ASM_DEFINE_ABORT(3, 4, 396 /* teardown */ 397 LONG_L " %[len], %[rseq_scratch2]\n\t" 398 LONG_L " %[dst], %[rseq_scratch1]\n\t" 399 LONG_L " %[src], %[rseq_scratch0]\n\t", 400 abort, 1b, 2b, 4f) 401 RSEQ_ASM_DEFINE_CMPFAIL(5, 402 /* teardown */ 403 LONG_L " %[len], %[rseq_scratch2]\n\t" 404 LONG_L " %[dst], %[rseq_scratch1]\n\t" 405 LONG_L " %[src], %[rseq_scratch0]\n\t", 406 cmpfail) 407#ifdef RSEQ_COMPARE_TWICE 408 RSEQ_ASM_DEFINE_CMPFAIL(6, 409 /* teardown */ 410 LONG_L " %[len], %[rseq_scratch2]\n\t" 411 LONG_L " %[dst], %[rseq_scratch1]\n\t" 412 LONG_L " %[src], %[rseq_scratch0]\n\t", 413 error1) 414 RSEQ_ASM_DEFINE_CMPFAIL(7, 415 /* teardown */ 416 LONG_L " %[len], %[rseq_scratch2]\n\t" 417 LONG_L " %[dst], %[rseq_scratch1]\n\t" 418 LONG_L " %[src], %[rseq_scratch0]\n\t", 419 error2) 420#endif 421 "8:\n\t" 422 : /* gcc asm goto does not allow outputs */ 423 : [cpu_id] "r" (cpu), 424 [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD), 425 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), 426 /* final store input */ 427 [v] "m" (*v), 428 [expect] "r" (expect), 429 [newv] "r" (newv), 430 /* try memcpy input */ 431 [dst] "r" (dst), 432 [src] "r" (src), 433 [len] "r" (len), 434 [rseq_scratch0] "m" (rseq_scratch[0]), 435 [rseq_scratch1] "m" (rseq_scratch[1]), 436 [rseq_scratch2] "m" (rseq_scratch[2]) 437 RSEQ_INJECT_INPUT 438 : "$4", "memory" 439 RSEQ_INJECT_CLOBBER 440 : abort, cmpfail 441#ifdef RSEQ_COMPARE_TWICE 442 , error1, error2 443#endif 444 ); 445 return 0; 446abort: 447 RSEQ_INJECT_FAILED 448 return -1; 449cmpfail: 450 return 1; 451#ifdef RSEQ_COMPARE_TWICE 452error1: 453 rseq_bug("cpu_id comparison failed"); 454error2: 455 rseq_bug("expected value comparison failed"); 456#endif 457} 458 459#endif /* #if (defined(RSEQ_TEMPLATE_MO_RELAXED) || defined(RSEQ_TEMPLATE_MO_RELEASE)) && 460 (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) */ 461 462#include "rseq-bits-reset.h" 463