1/* SPDX-License-Identifier: LGPL-2.1 OR MIT */ 2/* 3 * rseq-x86.h 4 * 5 * (C) Copyright 2016-2022 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com> 6 */ 7 8#ifndef RSEQ_H 9#error "Never use <rseq-x86.h> directly; include <rseq.h> instead." 10#endif 11 12#include <stdint.h> 13 14/* 15 * RSEQ_SIG is used with the following reserved undefined instructions, which 16 * trap in user-space: 17 * 18 * x86-32: 0f b9 3d 53 30 05 53 ud1 0x53053053,%edi 19 * x86-64: 0f b9 3d 53 30 05 53 ud1 0x53053053(%rip),%edi 20 */ 21#define RSEQ_SIG 0x53053053 22 23/* 24 * Due to a compiler optimization bug in gcc-8 with asm goto and TLS asm input 25 * operands, we cannot use "m" input operands, and rather pass the __rseq_abi 26 * address through a "r" input operand. 27 */ 28 29/* Offset of cpu_id, rseq_cs, and mm_cid fields in struct rseq. */ 30#define RSEQ_CPU_ID_OFFSET 4 31#define RSEQ_CS_OFFSET 8 32#define RSEQ_MM_CID_OFFSET 24 33 34#ifdef __x86_64__ 35 36#define RSEQ_ASM_TP_SEGMENT %%fs 37 38#define rseq_smp_mb() \ 39 __asm__ __volatile__ ("lock; addl $0,-128(%%rsp)" ::: "memory", "cc") 40#define rseq_smp_rmb() rseq_barrier() 41#define rseq_smp_wmb() rseq_barrier() 42 43#define rseq_smp_load_acquire(p) \ 44__extension__ ({ \ 45 rseq_unqual_scalar_typeof(*(p)) ____p1 = RSEQ_READ_ONCE(*(p)); \ 46 rseq_barrier(); \ 47 ____p1; \ 48}) 49 50#define rseq_smp_acquire__after_ctrl_dep() rseq_smp_rmb() 51 52#define rseq_smp_store_release(p, v) \ 53do { \ 54 rseq_barrier(); \ 55 RSEQ_WRITE_ONCE(*(p), v); \ 56} while (0) 57 58#define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, \ 59 start_ip, post_commit_offset, abort_ip) \ 60 ".pushsection __rseq_cs, \"aw\"\n\t" \ 61 ".balign 32\n\t" \ 62 __rseq_str(label) ":\n\t" \ 63 ".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \ 64 ".quad " __rseq_str(start_ip) ", " __rseq_str(post_commit_offset) ", " __rseq_str(abort_ip) "\n\t" \ 65 ".popsection\n\t" \ 66 ".pushsection __rseq_cs_ptr_array, \"aw\"\n\t" \ 67 ".quad " __rseq_str(label) "b\n\t" \ 68 ".popsection\n\t" 69 70 71#define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \ 72 __RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip, \ 73 (post_commit_ip - start_ip), abort_ip) 74 75/* 76 * Exit points of a rseq critical section consist of all instructions outside 77 * of the critical section where a critical section can either branch to or 78 * reach through the normal course of its execution. The abort IP and the 79 * post-commit IP are already part of the __rseq_cs section and should not be 80 * explicitly defined as additional exit points. Knowing all exit points is 81 * useful to assist debuggers stepping over the critical section. 82 */ 83#define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip) \ 84 ".pushsection __rseq_exit_point_array, \"aw\"\n\t" \ 85 ".quad " __rseq_str(start_ip) ", " __rseq_str(exit_ip) "\n\t" \ 86 ".popsection\n\t" 87 88#define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \ 89 RSEQ_INJECT_ASM(1) \ 90 "leaq " __rseq_str(cs_label) "(%%rip), %%rax\n\t" \ 91 "movq %%rax, " __rseq_str(rseq_cs) "\n\t" \ 92 __rseq_str(label) ":\n\t" 93 94#define RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, label) \ 95 RSEQ_INJECT_ASM(2) \ 96 "cmpl %[" __rseq_str(cpu_id) "], " __rseq_str(current_cpu_id) "\n\t" \ 97 "jnz " __rseq_str(label) "\n\t" 98 99#define RSEQ_ASM_DEFINE_ABORT(label, teardown, abort_label) \ 100 ".pushsection __rseq_failure, \"ax\"\n\t" \ 101 /* Disassembler-friendly signature: ud1 <sig>(%rip),%edi. */ \ 102 ".byte 0x0f, 0xb9, 0x3d\n\t" \ 103 ".long " __rseq_str(RSEQ_SIG) "\n\t" \ 104 __rseq_str(label) ":\n\t" \ 105 teardown \ 106 "jmp %l[" __rseq_str(abort_label) "]\n\t" \ 107 ".popsection\n\t" 108 109#define RSEQ_ASM_DEFINE_CMPFAIL(label, teardown, cmpfail_label) \ 110 ".pushsection __rseq_failure, \"ax\"\n\t" \ 111 __rseq_str(label) ":\n\t" \ 112 teardown \ 113 "jmp %l[" __rseq_str(cmpfail_label) "]\n\t" \ 114 ".popsection\n\t" 115 116#elif defined(__i386__) 117 118#define RSEQ_ASM_TP_SEGMENT %%gs 119 120#define rseq_smp_mb() \ 121 __asm__ __volatile__ ("lock; addl $0,-128(%%esp)" ::: "memory", "cc") 122#define rseq_smp_rmb() \ 123 __asm__ __volatile__ ("lock; addl $0,-128(%%esp)" ::: "memory", "cc") 124#define rseq_smp_wmb() \ 125 __asm__ __volatile__ ("lock; addl $0,-128(%%esp)" ::: "memory", "cc") 126 127#define rseq_smp_load_acquire(p) \ 128__extension__ ({ \ 129 __typeof(*p) ____p1 = RSEQ_READ_ONCE(*p); \ 130 rseq_smp_mb(); \ 131 ____p1; \ 132}) 133 134#define rseq_smp_acquire__after_ctrl_dep() rseq_smp_rmb() 135 136#define rseq_smp_store_release(p, v) \ 137do { \ 138 rseq_smp_mb(); \ 139 RSEQ_WRITE_ONCE(*p, v); \ 140} while (0) 141 142/* 143 * Use eax as scratch register and take memory operands as input to 144 * lessen register pressure. Especially needed when compiling in O0. 145 */ 146#define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, \ 147 start_ip, post_commit_offset, abort_ip) \ 148 ".pushsection __rseq_cs, \"aw\"\n\t" \ 149 ".balign 32\n\t" \ 150 __rseq_str(label) ":\n\t" \ 151 ".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \ 152 ".long " __rseq_str(start_ip) ", 0x0, " __rseq_str(post_commit_offset) ", 0x0, " __rseq_str(abort_ip) ", 0x0\n\t" \ 153 ".popsection\n\t" \ 154 ".pushsection __rseq_cs_ptr_array, \"aw\"\n\t" \ 155 ".long " __rseq_str(label) "b, 0x0\n\t" \ 156 ".popsection\n\t" 157 158#define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \ 159 __RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip, \ 160 (post_commit_ip - start_ip), abort_ip) 161 162/* 163 * Exit points of a rseq critical section consist of all instructions outside 164 * of the critical section where a critical section can either branch to or 165 * reach through the normal course of its execution. The abort IP and the 166 * post-commit IP are already part of the __rseq_cs section and should not be 167 * explicitly defined as additional exit points. Knowing all exit points is 168 * useful to assist debuggers stepping over the critical section. 169 */ 170#define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip) \ 171 ".pushsection __rseq_exit_point_array, \"aw\"\n\t" \ 172 ".long " __rseq_str(start_ip) ", 0x0, " __rseq_str(exit_ip) ", 0x0\n\t" \ 173 ".popsection\n\t" 174 175#define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \ 176 RSEQ_INJECT_ASM(1) \ 177 "movl $" __rseq_str(cs_label) ", " __rseq_str(rseq_cs) "\n\t" \ 178 __rseq_str(label) ":\n\t" 179 180#define RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, label) \ 181 RSEQ_INJECT_ASM(2) \ 182 "cmpl %[" __rseq_str(cpu_id) "], " __rseq_str(current_cpu_id) "\n\t" \ 183 "jnz " __rseq_str(label) "\n\t" 184 185#define RSEQ_ASM_DEFINE_ABORT(label, teardown, abort_label) \ 186 ".pushsection __rseq_failure, \"ax\"\n\t" \ 187 /* Disassembler-friendly signature: ud1 <sig>,%edi. */ \ 188 ".byte 0x0f, 0xb9, 0x3d\n\t" \ 189 ".long " __rseq_str(RSEQ_SIG) "\n\t" \ 190 __rseq_str(label) ":\n\t" \ 191 teardown \ 192 "jmp %l[" __rseq_str(abort_label) "]\n\t" \ 193 ".popsection\n\t" 194 195#define RSEQ_ASM_DEFINE_CMPFAIL(label, teardown, cmpfail_label) \ 196 ".pushsection __rseq_failure, \"ax\"\n\t" \ 197 __rseq_str(label) ":\n\t" \ 198 teardown \ 199 "jmp %l[" __rseq_str(cmpfail_label) "]\n\t" \ 200 ".popsection\n\t" 201 202#endif 203 204/* Per-cpu-id indexing. */ 205 206#define RSEQ_TEMPLATE_CPU_ID 207#define RSEQ_TEMPLATE_MO_RELAXED 208#include "rseq-x86-bits.h" 209#undef RSEQ_TEMPLATE_MO_RELAXED 210 211#define RSEQ_TEMPLATE_MO_RELEASE 212#include "rseq-x86-bits.h" 213#undef RSEQ_TEMPLATE_MO_RELEASE 214#undef RSEQ_TEMPLATE_CPU_ID 215 216/* Per-mm-cid indexing. */ 217 218#define RSEQ_TEMPLATE_MM_CID 219#define RSEQ_TEMPLATE_MO_RELAXED 220#include "rseq-x86-bits.h" 221#undef RSEQ_TEMPLATE_MO_RELAXED 222 223#define RSEQ_TEMPLATE_MO_RELEASE 224#include "rseq-x86-bits.h" 225#undef RSEQ_TEMPLATE_MO_RELEASE 226#undef RSEQ_TEMPLATE_MM_CID 227 228/* APIs which are not based on cpu ids. */ 229 230#define RSEQ_TEMPLATE_CPU_ID_NONE 231#define RSEQ_TEMPLATE_MO_RELAXED 232#include "rseq-x86-bits.h" 233#undef RSEQ_TEMPLATE_MO_RELAXED 234#undef RSEQ_TEMPLATE_CPU_ID_NONE 235