1// Copyright 2016 The Fuchsia Authors 2// 3// Use of this source code is governed by a MIT-style 4// license that can be found in the LICENSE file or at 5// https://opensource.org/licenses/MIT 6 7#include <arch/debugger.h> 8#include <arch/x86.h> 9#include <arch/x86/feature.h> 10#include <arch/x86/registers.h> 11#include <err.h> 12#include <kernel/lockdep.h> 13#include <kernel/thread.h> 14#include <kernel/thread_lock.h> 15#include <string.h> 16#include <sys/types.h> 17#include <zircon/syscalls/debug.h> 18#include <zircon/types.h> 19 20// Note on locking: The below functions need to read and write the register state and make sure that 21// nothing happens with respect to scheduling that thread while this is happening. As a result they 22// use ThreadLock. In most cases this will not be necessary but there are relatively few 23// guarantees so we lock the scheduler. Since these functions are used mostly for debugging, this 24// shouldn't be too significant a performance penalty. 25 26namespace { 27 28#define SYSCALL_OFFSETS_EQUAL(reg) \ 29 (__offsetof(zx_thread_state_general_regs_t, reg) == \ 30 __offsetof(x86_syscall_general_regs_t, reg)) 31 32static_assert(SYSCALL_OFFSETS_EQUAL(rax), ""); 33static_assert(SYSCALL_OFFSETS_EQUAL(rbx), ""); 34static_assert(SYSCALL_OFFSETS_EQUAL(rcx), ""); 35static_assert(SYSCALL_OFFSETS_EQUAL(rdx), ""); 36static_assert(SYSCALL_OFFSETS_EQUAL(rsi), ""); 37static_assert(SYSCALL_OFFSETS_EQUAL(rdi), ""); 38static_assert(SYSCALL_OFFSETS_EQUAL(rbp), ""); 39static_assert(SYSCALL_OFFSETS_EQUAL(rsp), ""); 40static_assert(SYSCALL_OFFSETS_EQUAL(r8), ""); 41static_assert(SYSCALL_OFFSETS_EQUAL(r9), ""); 42static_assert(SYSCALL_OFFSETS_EQUAL(r10), ""); 43static_assert(SYSCALL_OFFSETS_EQUAL(r11), ""); 44static_assert(SYSCALL_OFFSETS_EQUAL(r12), ""); 45static_assert(SYSCALL_OFFSETS_EQUAL(r13), ""); 46static_assert(SYSCALL_OFFSETS_EQUAL(r14), ""); 47static_assert(SYSCALL_OFFSETS_EQUAL(r15), ""); 48static_assert(sizeof(zx_thread_state_general_regs_t) == sizeof(x86_syscall_general_regs_t), ""); 49 50void x86_fill_in_gregs_from_syscall(zx_thread_state_general_regs_t* out, 51 const x86_syscall_general_regs_t* in) { 52 memcpy(out, in, sizeof(*in)); 53} 54 55void x86_fill_in_syscall_from_gregs(x86_syscall_general_regs_t* out, 56 const zx_thread_state_general_regs_t* in) { 57 // Don't allow overriding privileged fields of rflags, and ignore writes 58 // to reserved fields. 59 const uint64_t orig_rflags = out->rflags; 60 memcpy(out, in, sizeof(*in)); 61 out->rflags = orig_rflags & ~X86_FLAGS_USER; 62 out->rflags |= in->rflags & X86_FLAGS_USER; 63} 64 65#define COPY_REG(out, in, reg) (out)->reg = (in)->reg 66#define COPY_COMMON_IFRAME_REGS(out, in) \ 67 do { \ 68 COPY_REG(out, in, rax); \ 69 COPY_REG(out, in, rbx); \ 70 COPY_REG(out, in, rcx); \ 71 COPY_REG(out, in, rdx); \ 72 COPY_REG(out, in, rsi); \ 73 COPY_REG(out, in, rdi); \ 74 COPY_REG(out, in, rbp); \ 75 COPY_REG(out, in, r8); \ 76 COPY_REG(out, in, r9); \ 77 COPY_REG(out, in, r10); \ 78 COPY_REG(out, in, r11); \ 79 COPY_REG(out, in, r12); \ 80 COPY_REG(out, in, r13); \ 81 COPY_REG(out, in, r14); \ 82 COPY_REG(out, in, r15); \ 83 } while (0) 84 85void x86_fill_in_gregs_from_iframe(zx_thread_state_general_regs_t* out, 86 const x86_iframe_t* in) { 87 COPY_COMMON_IFRAME_REGS(out, in); 88 out->rsp = in->user_sp; 89 out->rip = in->ip; 90 out->rflags = in->flags; 91} 92 93void x86_fill_in_iframe_from_gregs(x86_iframe_t* out, 94 const zx_thread_state_general_regs_t* in) { 95 COPY_COMMON_IFRAME_REGS(out, in); 96 out->user_sp = in->rsp; 97 out->ip = in->rip; 98 // Don't allow overriding privileged fields of rflags, and ignore writes 99 // to reserved fields. 100 out->flags &= ~X86_FLAGS_USER; 101 out->flags |= in->rflags & X86_FLAGS_USER; 102} 103 104// Whether an operation gets thread state or sets it. 105enum class RegAccess { kGet, 106 kSet }; 107 108// Backend for arch_get_vector_regs and arch_set_vector_regs. This does a read or write of the 109// thread to or from the regs structure. 110zx_status_t x86_get_set_vector_regs(struct thread* thread, zx_thread_state_vector_regs* regs, 111 RegAccess access) { 112 // Function to copy memory in the correct direction. Write the code using this function as if it 113 // was "memcpy" in "get" mode, and it will be reversed in "set" mode. 114 auto get_set_memcpy = (access == RegAccess::kGet) ? [](void* regs, void* thread, size_t size) { memcpy(regs, thread, size); } : // Get mode. 115 [](void* regs, void* thread, size_t size) { memcpy(thread, regs, size); }; // Set mode. 116 117 if (access == RegAccess::kGet) { 118 // Not all parts will be filled in in all cases so zero out first. 119 memset(regs, 0, sizeof(zx_thread_state_vector_regs)); 120 } 121 122 // Whether to force the components to be marked present in the xsave area. 123 bool mark_present = access == RegAccess::kSet; 124 125 Guard<spin_lock_t, IrqSave> thread_lock_guard{ThreadLock::Get()}; 126 if (thread->state == THREAD_RUNNING) { 127 return ZX_ERR_BAD_STATE; 128 } 129 130 constexpr int kNumSSERegs = 16; 131 132 // The low 128 bits of registers 0-15 come from the legacy area and are always present. 133 constexpr int kXmmRegSize = 16; // Each XMM register is 128 bits / 16 bytes. 134 uint32_t comp_size = 0; 135 x86_xsave_legacy_area* save = static_cast<x86_xsave_legacy_area*>( 136 x86_get_extended_register_state_component(thread->arch.extended_register_state, 137 X86_XSAVE_STATE_INDEX_SSE, mark_present, 138 &comp_size)); 139 DEBUG_ASSERT(save); // Legacy getter should always succeed. 140 for (int i = 0; i < kNumSSERegs; i++) { 141 get_set_memcpy(®s->zmm[i].v[0], &save->xmm[i], kXmmRegSize); 142 } 143 144 // MXCSR (always present): 32-bit status word. 145 get_set_memcpy(®s->mxcsr, &save->mxcsr, 4); 146 147 // AVX grows the registers to 256 bits each. Optional. 148 constexpr int kYmmHighSize = 16; // Additional bytes in each register. 149 uint8_t* ymm_highbits = static_cast<uint8_t*>( 150 x86_get_extended_register_state_component(thread->arch.extended_register_state, 151 X86_XSAVE_STATE_INDEX_AVX, mark_present, 152 &comp_size)); 153 if (ymm_highbits) { 154 DEBUG_ASSERT(comp_size == kYmmHighSize * kNumSSERegs); 155 for (int i = 0; i < kNumSSERegs; i++) { 156 get_set_memcpy(®s->zmm[i].v[2], &ymm_highbits[i * kYmmHighSize], kYmmHighSize); 157 } 158 } 159 160 // AVX-512 opmask registers (8 64-bit registers). Optional. 161 constexpr int kNumOpmaskRegs = 8; 162 uint64_t* opmask = static_cast<uint64_t*>( 163 x86_get_extended_register_state_component(thread->arch.extended_register_state, 164 X86_XSAVE_STATE_INDEX_AVX512_OPMASK, mark_present, 165 &comp_size)); 166 if (opmask) { 167 DEBUG_ASSERT(comp_size == kNumOpmaskRegs * sizeof(uint64_t)); 168 for (int i = 0; i < kNumOpmaskRegs; i++) { 169 get_set_memcpy(®s->opmask[i], &opmask[i], sizeof(uint64_t)); 170 } 171 } 172 173 // AVX-512 high bits (256 bits extra each) for ZMM0-15. 174 constexpr int kZmmHighSize = 32; // Additional bytes in each register. 175 uint8_t* zmm_highbits = static_cast<uint8_t*>( 176 x86_get_extended_register_state_component(thread->arch.extended_register_state, 177 X86_XSAVE_STATE_INDEX_AVX512_LOWERZMM_HIGH, 178 mark_present, &comp_size)); 179 if (zmm_highbits) { 180 DEBUG_ASSERT(comp_size == kZmmHighSize * kNumSSERegs); 181 for (int i = 0; i < kNumSSERegs; i++) { 182 get_set_memcpy(®s->zmm[i].v[4], &zmm_highbits[i * kZmmHighSize], kZmmHighSize); 183 } 184 } 185 186 // AVX-512 registers 16-31 (512 bits each) are in component 7. 187 constexpr int kNumZmmHighRegs = 16; // Extra registers added over xmm/ymm. 188 constexpr int kZmmRegSize = 64; // Total register size. 189 uint8_t* zmm_highregs = static_cast<uint8_t*>( 190 x86_get_extended_register_state_component(thread->arch.extended_register_state, 191 X86_XSAVE_STATE_INDEX_AVX512_HIGHERZMM, 192 mark_present, &comp_size)); 193 if (zmm_highregs) { 194 DEBUG_ASSERT(comp_size == kNumZmmHighRegs * kZmmRegSize); 195 for (int i = 0; i < kNumZmmHighRegs; i++) { 196 get_set_memcpy(®s->zmm[i + kNumSSERegs], &zmm_highregs[i * kZmmRegSize], 197 kZmmRegSize); 198 } 199 } 200 201 return ZX_OK; 202} 203 204} // namespace 205 206zx_status_t arch_get_general_regs(struct thread* thread, zx_thread_state_general_regs_t* out) { 207 Guard<spin_lock_t, IrqSave> thread_lock_guard{ThreadLock::Get()}; 208 209 // Punt if registers aren't available. E.g., 210 // ZX-563 (registers aren't available in synthetic exceptions) 211 if (thread->arch.suspended_general_regs.gregs == nullptr) 212 return ZX_ERR_NOT_SUPPORTED; 213 214 DEBUG_ASSERT(thread->arch.suspended_general_regs.gregs); 215 switch (thread->arch.general_regs_source) { 216 case X86_GENERAL_REGS_SYSCALL: 217 x86_fill_in_gregs_from_syscall(out, thread->arch.suspended_general_regs.syscall); 218 break; 219 case X86_GENERAL_REGS_IFRAME: 220 x86_fill_in_gregs_from_iframe(out, thread->arch.suspended_general_regs.iframe); 221 break; 222 default: 223 DEBUG_ASSERT(false); 224 return ZX_ERR_BAD_STATE; 225 } 226 227 return ZX_OK; 228} 229 230zx_status_t arch_set_general_regs(struct thread* thread, const zx_thread_state_general_regs_t* in) { 231 Guard<spin_lock_t, IrqSave> thread_lock_guard{ThreadLock::Get()}; 232 233 // Punt if registers aren't available. E.g., 234 // ZX-563 (registers aren't available in synthetic exceptions) 235 if (thread->arch.suspended_general_regs.gregs == nullptr) 236 return ZX_ERR_NOT_SUPPORTED; 237 238 DEBUG_ASSERT(thread->arch.suspended_general_regs.gregs); 239 switch (thread->arch.general_regs_source) { 240 case X86_GENERAL_REGS_SYSCALL: { 241 // Disallow setting RIP to a non-canonical address, to prevent 242 // returning to such addresses using the SYSRET instruction. 243 // See docs/sysret_problem.md. Note that this check also 244 // disallows canonical top-bit-set addresses, but allowing such 245 // addresses is not useful and it is simpler to disallow them. 246 uint8_t addr_width = x86_linear_address_width(); 247 uint64_t noncanonical_addr = ((uint64_t)1) << (addr_width - 1); 248 if (in->rip >= noncanonical_addr) 249 return ZX_ERR_INVALID_ARGS; 250 x86_fill_in_syscall_from_gregs(thread->arch.suspended_general_regs.syscall, in); 251 break; 252 } 253 case X86_GENERAL_REGS_IFRAME: 254 x86_fill_in_iframe_from_gregs(thread->arch.suspended_general_regs.iframe, in); 255 break; 256 default: 257 DEBUG_ASSERT(false); 258 return ZX_ERR_BAD_STATE; 259 } 260 261 return ZX_OK; 262} 263 264zx_status_t arch_get_single_step(struct thread* thread, bool* single_step) { 265 Guard<spin_lock_t, IrqSave> thread_lock_guard{ThreadLock::Get()}; 266 267 // Punt if registers aren't available. E.g., 268 // ZX-563 (registers aren't available in synthetic exceptions) 269 if (thread->arch.suspended_general_regs.gregs == nullptr) 270 return ZX_ERR_NOT_SUPPORTED; 271 272 uint64_t* flags = nullptr; 273 switch (thread->arch.general_regs_source) { 274 case X86_GENERAL_REGS_SYSCALL: 275 flags = &thread->arch.suspended_general_regs.syscall->rflags; 276 break; 277 case X86_GENERAL_REGS_IFRAME: 278 flags = &thread->arch.suspended_general_regs.iframe->flags; 279 break; 280 default: 281 DEBUG_ASSERT(false); 282 return ZX_ERR_BAD_STATE; 283 } 284 285 *single_step = !!(*flags & X86_FLAGS_TF); 286 return ZX_OK; 287} 288 289zx_status_t arch_set_single_step(struct thread* thread, bool single_step) { 290 Guard<spin_lock_t, IrqSave> thread_lock_guard{ThreadLock::Get()}; 291 292 // Punt if registers aren't available. E.g., 293 // ZX-563 (registers aren't available in synthetic exceptions) 294 if (thread->arch.suspended_general_regs.gregs == nullptr) 295 return ZX_ERR_NOT_SUPPORTED; 296 297 uint64_t* flags = nullptr; 298 switch (thread->arch.general_regs_source) { 299 case X86_GENERAL_REGS_SYSCALL: 300 flags = &thread->arch.suspended_general_regs.syscall->rflags; 301 break; 302 case X86_GENERAL_REGS_IFRAME: 303 flags = &thread->arch.suspended_general_regs.iframe->flags; 304 break; 305 default: 306 DEBUG_ASSERT(false); 307 return ZX_ERR_BAD_STATE; 308 } 309 310 if (single_step) { 311 *flags |= X86_FLAGS_TF; 312 } else { 313 *flags &= ~X86_FLAGS_TF; 314 } 315 return ZX_OK; 316} 317 318zx_status_t arch_get_fp_regs(struct thread* thread, zx_thread_state_fp_regs* out) { 319 // Don't leak any reserved fields. 320 memset(out, 0, sizeof(zx_thread_state_fp_regs)); 321 322 Guard<spin_lock_t, IrqSave> thread_lock_guard{ThreadLock::Get()}; 323 if (thread->state == THREAD_RUNNING) { 324 return ZX_ERR_BAD_STATE; 325 } 326 327 uint32_t comp_size = 0; 328 x86_xsave_legacy_area* save = static_cast<x86_xsave_legacy_area*>( 329 x86_get_extended_register_state_component(thread->arch.extended_register_state, 330 X86_XSAVE_STATE_INDEX_X87, false, &comp_size)); 331 DEBUG_ASSERT(save); // Legacy getter should always succeed. 332 333 out->fcw = save->fcw; 334 out->fsw = save->fsw; 335 out->ftw = save->ftw; 336 out->fop = save->fop; 337 out->fip = save->fip; 338 out->fdp = save->fdp; 339 memcpy(&out->st[0], &save->st[0], sizeof(out->st)); 340 341 return ZX_OK; 342} 343 344zx_status_t arch_set_fp_regs(struct thread* thread, const zx_thread_state_fp_regs* in) { 345 Guard<spin_lock_t, IrqSave> thread_lock_guard{ThreadLock::Get()}; 346 if (thread->state == THREAD_RUNNING) { 347 return ZX_ERR_BAD_STATE; 348 } 349 350 uint32_t comp_size = 0; 351 x86_xsave_legacy_area* save = static_cast<x86_xsave_legacy_area*>( 352 x86_get_extended_register_state_component(thread->arch.extended_register_state, 353 X86_XSAVE_STATE_INDEX_X87, true, &comp_size)); 354 DEBUG_ASSERT(save); // Legacy getter should always succeed. 355 356 save->fcw = in->fcw; 357 save->fsw = in->fsw; 358 save->ftw = in->ftw; 359 save->fop = in->fop; 360 save->fip = in->fip; 361 save->fdp = in->fdp; 362 memcpy(&save->st[0], &in->st[0], sizeof(in->st)); 363 364 return ZX_OK; 365} 366 367zx_status_t arch_get_vector_regs(struct thread* thread, zx_thread_state_vector_regs* out) { 368 return x86_get_set_vector_regs(thread, out, RegAccess::kGet); 369} 370 371zx_status_t arch_set_vector_regs(struct thread* thread, const zx_thread_state_vector_regs* in) { 372 // The get_set function won't write in "kSet" mode so the const_cast is safe. 373 return x86_get_set_vector_regs(thread, const_cast<zx_thread_state_vector_regs*>(in), 374 RegAccess::kSet); 375} 376 377zx_status_t arch_get_x86_register_fs(struct thread* thread, uint64_t* out) { 378 Guard<spin_lock_t, IrqSave> thread_lock_guard{ThreadLock::Get()}; 379 if (thread->state == THREAD_RUNNING) { 380 return ZX_ERR_BAD_STATE; 381 } 382 383 *out = thread->arch.fs_base; 384 return ZX_OK; 385} 386 387zx_status_t arch_set_x86_register_fs(struct thread* thread, const uint64_t* in) { 388 Guard<spin_lock_t, IrqSave> thread_lock_guard{ThreadLock::Get()}; 389 if (thread->state == THREAD_RUNNING) { 390 return ZX_ERR_BAD_STATE; 391 } 392 393 thread->arch.fs_base = *in; 394 return ZX_OK; 395} 396 397zx_status_t arch_get_x86_register_gs(struct thread* thread, uint64_t* out) { 398 Guard<spin_lock_t, IrqSave> thread_lock_guard{ThreadLock::Get()}; 399 if (thread->state == THREAD_RUNNING) { 400 return ZX_ERR_BAD_STATE; 401 } 402 403 *out = thread->arch.gs_base; 404 return ZX_OK; 405} 406 407zx_status_t arch_set_x86_register_gs(struct thread* thread, const uint64_t* in) { 408 Guard<spin_lock_t, IrqSave> thread_lock_guard{ThreadLock::Get()}; 409 if (thread->state == THREAD_RUNNING) { 410 return ZX_ERR_BAD_STATE; 411 } 412 413 thread->arch.gs_base = *in; 414 return ZX_OK; 415} 416