1// Copyright 2016 The Fuchsia Authors
2//
3// Use of this source code is governed by a MIT-style
4// license that can be found in the LICENSE file or at
5// https://opensource.org/licenses/MIT
6
7#include <arch/debugger.h>
8#include <arch/x86.h>
9#include <arch/x86/feature.h>
10#include <arch/x86/registers.h>
11#include <err.h>
12#include <kernel/lockdep.h>
13#include <kernel/thread.h>
14#include <kernel/thread_lock.h>
15#include <string.h>
16#include <sys/types.h>
17#include <zircon/syscalls/debug.h>
18#include <zircon/types.h>
19
20// Note on locking: The below functions need to read and write the register state and make sure that
21// nothing happens with respect to scheduling that thread while this is happening. As a result they
22// use ThreadLock. In most cases this will not be necessary but there are relatively few
23// guarantees so we lock the scheduler. Since these functions are used mostly for debugging, this
24// shouldn't be too significant a performance penalty.
25
26namespace {
27
28#define SYSCALL_OFFSETS_EQUAL(reg)                      \
29    (__offsetof(zx_thread_state_general_regs_t, reg) == \
30     __offsetof(x86_syscall_general_regs_t, reg))
31
32static_assert(SYSCALL_OFFSETS_EQUAL(rax), "");
33static_assert(SYSCALL_OFFSETS_EQUAL(rbx), "");
34static_assert(SYSCALL_OFFSETS_EQUAL(rcx), "");
35static_assert(SYSCALL_OFFSETS_EQUAL(rdx), "");
36static_assert(SYSCALL_OFFSETS_EQUAL(rsi), "");
37static_assert(SYSCALL_OFFSETS_EQUAL(rdi), "");
38static_assert(SYSCALL_OFFSETS_EQUAL(rbp), "");
39static_assert(SYSCALL_OFFSETS_EQUAL(rsp), "");
40static_assert(SYSCALL_OFFSETS_EQUAL(r8), "");
41static_assert(SYSCALL_OFFSETS_EQUAL(r9), "");
42static_assert(SYSCALL_OFFSETS_EQUAL(r10), "");
43static_assert(SYSCALL_OFFSETS_EQUAL(r11), "");
44static_assert(SYSCALL_OFFSETS_EQUAL(r12), "");
45static_assert(SYSCALL_OFFSETS_EQUAL(r13), "");
46static_assert(SYSCALL_OFFSETS_EQUAL(r14), "");
47static_assert(SYSCALL_OFFSETS_EQUAL(r15), "");
48static_assert(sizeof(zx_thread_state_general_regs_t) == sizeof(x86_syscall_general_regs_t), "");
49
50void x86_fill_in_gregs_from_syscall(zx_thread_state_general_regs_t* out,
51                                    const x86_syscall_general_regs_t* in) {
52    memcpy(out, in, sizeof(*in));
53}
54
55void x86_fill_in_syscall_from_gregs(x86_syscall_general_regs_t* out,
56                                    const zx_thread_state_general_regs_t* in) {
57    // Don't allow overriding privileged fields of rflags, and ignore writes
58    // to reserved fields.
59    const uint64_t orig_rflags = out->rflags;
60    memcpy(out, in, sizeof(*in));
61    out->rflags = orig_rflags & ~X86_FLAGS_USER;
62    out->rflags |= in->rflags & X86_FLAGS_USER;
63}
64
65#define COPY_REG(out, in, reg) (out)->reg = (in)->reg
66#define COPY_COMMON_IFRAME_REGS(out, in) \
67    do {                                 \
68        COPY_REG(out, in, rax);          \
69        COPY_REG(out, in, rbx);          \
70        COPY_REG(out, in, rcx);          \
71        COPY_REG(out, in, rdx);          \
72        COPY_REG(out, in, rsi);          \
73        COPY_REG(out, in, rdi);          \
74        COPY_REG(out, in, rbp);          \
75        COPY_REG(out, in, r8);           \
76        COPY_REG(out, in, r9);           \
77        COPY_REG(out, in, r10);          \
78        COPY_REG(out, in, r11);          \
79        COPY_REG(out, in, r12);          \
80        COPY_REG(out, in, r13);          \
81        COPY_REG(out, in, r14);          \
82        COPY_REG(out, in, r15);          \
83    } while (0)
84
85void x86_fill_in_gregs_from_iframe(zx_thread_state_general_regs_t* out,
86                                   const x86_iframe_t* in) {
87    COPY_COMMON_IFRAME_REGS(out, in);
88    out->rsp = in->user_sp;
89    out->rip = in->ip;
90    out->rflags = in->flags;
91}
92
93void x86_fill_in_iframe_from_gregs(x86_iframe_t* out,
94                                   const zx_thread_state_general_regs_t* in) {
95    COPY_COMMON_IFRAME_REGS(out, in);
96    out->user_sp = in->rsp;
97    out->ip = in->rip;
98    // Don't allow overriding privileged fields of rflags, and ignore writes
99    // to reserved fields.
100    out->flags &= ~X86_FLAGS_USER;
101    out->flags |= in->rflags & X86_FLAGS_USER;
102}
103
104// Whether an operation gets thread state or sets it.
105enum class RegAccess { kGet,
106                       kSet };
107
108// Backend for arch_get_vector_regs and arch_set_vector_regs. This does a read or write of the
109// thread to or from the regs structure.
110zx_status_t x86_get_set_vector_regs(struct thread* thread, zx_thread_state_vector_regs* regs,
111                                    RegAccess access) {
112    // Function to copy memory in the correct direction. Write the code using this function as if it
113    // was "memcpy" in "get" mode, and it will be reversed in "set" mode.
114    auto get_set_memcpy = (access == RegAccess::kGet) ? [](void* regs, void* thread, size_t size) { memcpy(regs, thread, size); } : // Get mode.
115                              [](void* regs, void* thread, size_t size) { memcpy(thread, regs, size); };                            // Set mode.
116
117    if (access == RegAccess::kGet) {
118        // Not all parts will be filled in in all cases so zero out first.
119        memset(regs, 0, sizeof(zx_thread_state_vector_regs));
120    }
121
122    // Whether to force the components to be marked present in the xsave area.
123    bool mark_present = access == RegAccess::kSet;
124
125    Guard<spin_lock_t, IrqSave> thread_lock_guard{ThreadLock::Get()};
126    if (thread->state == THREAD_RUNNING) {
127        return ZX_ERR_BAD_STATE;
128    }
129
130    constexpr int kNumSSERegs = 16;
131
132    // The low 128 bits of registers 0-15 come from the legacy area and are always present.
133    constexpr int kXmmRegSize = 16; // Each XMM register is 128 bits / 16 bytes.
134    uint32_t comp_size = 0;
135    x86_xsave_legacy_area* save = static_cast<x86_xsave_legacy_area*>(
136        x86_get_extended_register_state_component(thread->arch.extended_register_state,
137                                                  X86_XSAVE_STATE_INDEX_SSE, mark_present,
138                                                  &comp_size));
139    DEBUG_ASSERT(save); // Legacy getter should always succeed.
140    for (int i = 0; i < kNumSSERegs; i++) {
141        get_set_memcpy(&regs->zmm[i].v[0], &save->xmm[i], kXmmRegSize);
142    }
143
144    // MXCSR (always present): 32-bit status word.
145    get_set_memcpy(&regs->mxcsr, &save->mxcsr, 4);
146
147    // AVX grows the registers to 256 bits each. Optional.
148    constexpr int kYmmHighSize = 16; // Additional bytes in each register.
149    uint8_t* ymm_highbits = static_cast<uint8_t*>(
150        x86_get_extended_register_state_component(thread->arch.extended_register_state,
151                                                  X86_XSAVE_STATE_INDEX_AVX, mark_present,
152                                                  &comp_size));
153    if (ymm_highbits) {
154        DEBUG_ASSERT(comp_size == kYmmHighSize * kNumSSERegs);
155        for (int i = 0; i < kNumSSERegs; i++) {
156            get_set_memcpy(&regs->zmm[i].v[2], &ymm_highbits[i * kYmmHighSize], kYmmHighSize);
157        }
158    }
159
160    // AVX-512 opmask registers (8 64-bit registers). Optional.
161    constexpr int kNumOpmaskRegs = 8;
162    uint64_t* opmask = static_cast<uint64_t*>(
163        x86_get_extended_register_state_component(thread->arch.extended_register_state,
164                                                  X86_XSAVE_STATE_INDEX_AVX512_OPMASK, mark_present,
165                                                  &comp_size));
166    if (opmask) {
167        DEBUG_ASSERT(comp_size == kNumOpmaskRegs * sizeof(uint64_t));
168        for (int i = 0; i < kNumOpmaskRegs; i++) {
169            get_set_memcpy(&regs->opmask[i], &opmask[i], sizeof(uint64_t));
170        }
171    }
172
173    // AVX-512 high bits (256 bits extra each) for ZMM0-15.
174    constexpr int kZmmHighSize = 32; // Additional bytes in each register.
175    uint8_t* zmm_highbits = static_cast<uint8_t*>(
176        x86_get_extended_register_state_component(thread->arch.extended_register_state,
177                                                  X86_XSAVE_STATE_INDEX_AVX512_LOWERZMM_HIGH,
178                                                  mark_present, &comp_size));
179    if (zmm_highbits) {
180        DEBUG_ASSERT(comp_size == kZmmHighSize * kNumSSERegs);
181        for (int i = 0; i < kNumSSERegs; i++) {
182            get_set_memcpy(&regs->zmm[i].v[4], &zmm_highbits[i * kZmmHighSize], kZmmHighSize);
183        }
184    }
185
186    // AVX-512 registers 16-31 (512 bits each) are in component 7.
187    constexpr int kNumZmmHighRegs = 16; // Extra registers added over xmm/ymm.
188    constexpr int kZmmRegSize = 64;     // Total register size.
189    uint8_t* zmm_highregs = static_cast<uint8_t*>(
190        x86_get_extended_register_state_component(thread->arch.extended_register_state,
191                                                  X86_XSAVE_STATE_INDEX_AVX512_HIGHERZMM,
192                                                  mark_present, &comp_size));
193    if (zmm_highregs) {
194        DEBUG_ASSERT(comp_size == kNumZmmHighRegs * kZmmRegSize);
195        for (int i = 0; i < kNumZmmHighRegs; i++) {
196            get_set_memcpy(&regs->zmm[i + kNumSSERegs], &zmm_highregs[i * kZmmRegSize],
197                           kZmmRegSize);
198        }
199    }
200
201    return ZX_OK;
202}
203
204} // namespace
205
206zx_status_t arch_get_general_regs(struct thread* thread, zx_thread_state_general_regs_t* out) {
207    Guard<spin_lock_t, IrqSave> thread_lock_guard{ThreadLock::Get()};
208
209    // Punt if registers aren't available. E.g.,
210    // ZX-563 (registers aren't available in synthetic exceptions)
211    if (thread->arch.suspended_general_regs.gregs == nullptr)
212        return ZX_ERR_NOT_SUPPORTED;
213
214    DEBUG_ASSERT(thread->arch.suspended_general_regs.gregs);
215    switch (thread->arch.general_regs_source) {
216    case X86_GENERAL_REGS_SYSCALL:
217        x86_fill_in_gregs_from_syscall(out, thread->arch.suspended_general_regs.syscall);
218        break;
219    case X86_GENERAL_REGS_IFRAME:
220        x86_fill_in_gregs_from_iframe(out, thread->arch.suspended_general_regs.iframe);
221        break;
222    default:
223        DEBUG_ASSERT(false);
224        return ZX_ERR_BAD_STATE;
225    }
226
227    return ZX_OK;
228}
229
230zx_status_t arch_set_general_regs(struct thread* thread, const zx_thread_state_general_regs_t* in) {
231    Guard<spin_lock_t, IrqSave> thread_lock_guard{ThreadLock::Get()};
232
233    // Punt if registers aren't available. E.g.,
234    // ZX-563 (registers aren't available in synthetic exceptions)
235    if (thread->arch.suspended_general_regs.gregs == nullptr)
236        return ZX_ERR_NOT_SUPPORTED;
237
238    DEBUG_ASSERT(thread->arch.suspended_general_regs.gregs);
239    switch (thread->arch.general_regs_source) {
240    case X86_GENERAL_REGS_SYSCALL: {
241        // Disallow setting RIP to a non-canonical address, to prevent
242        // returning to such addresses using the SYSRET instruction.
243        // See docs/sysret_problem.md.  Note that this check also
244        // disallows canonical top-bit-set addresses, but allowing such
245        // addresses is not useful and it is simpler to disallow them.
246        uint8_t addr_width = x86_linear_address_width();
247        uint64_t noncanonical_addr = ((uint64_t)1) << (addr_width - 1);
248        if (in->rip >= noncanonical_addr)
249            return ZX_ERR_INVALID_ARGS;
250        x86_fill_in_syscall_from_gregs(thread->arch.suspended_general_regs.syscall, in);
251        break;
252    }
253    case X86_GENERAL_REGS_IFRAME:
254        x86_fill_in_iframe_from_gregs(thread->arch.suspended_general_regs.iframe, in);
255        break;
256    default:
257        DEBUG_ASSERT(false);
258        return ZX_ERR_BAD_STATE;
259    }
260
261    return ZX_OK;
262}
263
264zx_status_t arch_get_single_step(struct thread* thread, bool* single_step) {
265    Guard<spin_lock_t, IrqSave> thread_lock_guard{ThreadLock::Get()};
266
267    // Punt if registers aren't available. E.g.,
268    // ZX-563 (registers aren't available in synthetic exceptions)
269    if (thread->arch.suspended_general_regs.gregs == nullptr)
270        return ZX_ERR_NOT_SUPPORTED;
271
272    uint64_t* flags = nullptr;
273    switch (thread->arch.general_regs_source) {
274    case X86_GENERAL_REGS_SYSCALL:
275        flags = &thread->arch.suspended_general_regs.syscall->rflags;
276        break;
277    case X86_GENERAL_REGS_IFRAME:
278        flags = &thread->arch.suspended_general_regs.iframe->flags;
279        break;
280    default:
281        DEBUG_ASSERT(false);
282        return ZX_ERR_BAD_STATE;
283    }
284
285    *single_step = !!(*flags & X86_FLAGS_TF);
286    return ZX_OK;
287}
288
289zx_status_t arch_set_single_step(struct thread* thread, bool single_step) {
290    Guard<spin_lock_t, IrqSave> thread_lock_guard{ThreadLock::Get()};
291
292    // Punt if registers aren't available. E.g.,
293    // ZX-563 (registers aren't available in synthetic exceptions)
294    if (thread->arch.suspended_general_regs.gregs == nullptr)
295        return ZX_ERR_NOT_SUPPORTED;
296
297    uint64_t* flags = nullptr;
298    switch (thread->arch.general_regs_source) {
299    case X86_GENERAL_REGS_SYSCALL:
300        flags = &thread->arch.suspended_general_regs.syscall->rflags;
301        break;
302    case X86_GENERAL_REGS_IFRAME:
303        flags = &thread->arch.suspended_general_regs.iframe->flags;
304        break;
305    default:
306        DEBUG_ASSERT(false);
307        return ZX_ERR_BAD_STATE;
308    }
309
310    if (single_step) {
311        *flags |= X86_FLAGS_TF;
312    } else {
313        *flags &= ~X86_FLAGS_TF;
314    }
315    return ZX_OK;
316}
317
318zx_status_t arch_get_fp_regs(struct thread* thread, zx_thread_state_fp_regs* out) {
319    // Don't leak any reserved fields.
320    memset(out, 0, sizeof(zx_thread_state_fp_regs));
321
322    Guard<spin_lock_t, IrqSave> thread_lock_guard{ThreadLock::Get()};
323    if (thread->state == THREAD_RUNNING) {
324        return ZX_ERR_BAD_STATE;
325    }
326
327    uint32_t comp_size = 0;
328    x86_xsave_legacy_area* save = static_cast<x86_xsave_legacy_area*>(
329        x86_get_extended_register_state_component(thread->arch.extended_register_state,
330                                                  X86_XSAVE_STATE_INDEX_X87, false, &comp_size));
331    DEBUG_ASSERT(save); // Legacy getter should always succeed.
332
333    out->fcw = save->fcw;
334    out->fsw = save->fsw;
335    out->ftw = save->ftw;
336    out->fop = save->fop;
337    out->fip = save->fip;
338    out->fdp = save->fdp;
339    memcpy(&out->st[0], &save->st[0], sizeof(out->st));
340
341    return ZX_OK;
342}
343
344zx_status_t arch_set_fp_regs(struct thread* thread, const zx_thread_state_fp_regs* in) {
345    Guard<spin_lock_t, IrqSave> thread_lock_guard{ThreadLock::Get()};
346    if (thread->state == THREAD_RUNNING) {
347        return ZX_ERR_BAD_STATE;
348    }
349
350    uint32_t comp_size = 0;
351    x86_xsave_legacy_area* save = static_cast<x86_xsave_legacy_area*>(
352        x86_get_extended_register_state_component(thread->arch.extended_register_state,
353                                                  X86_XSAVE_STATE_INDEX_X87, true, &comp_size));
354    DEBUG_ASSERT(save); // Legacy getter should always succeed.
355
356    save->fcw = in->fcw;
357    save->fsw = in->fsw;
358    save->ftw = in->ftw;
359    save->fop = in->fop;
360    save->fip = in->fip;
361    save->fdp = in->fdp;
362    memcpy(&save->st[0], &in->st[0], sizeof(in->st));
363
364    return ZX_OK;
365}
366
367zx_status_t arch_get_vector_regs(struct thread* thread, zx_thread_state_vector_regs* out) {
368    return x86_get_set_vector_regs(thread, out, RegAccess::kGet);
369}
370
371zx_status_t arch_set_vector_regs(struct thread* thread, const zx_thread_state_vector_regs* in) {
372    // The get_set function won't write in "kSet" mode so the const_cast is safe.
373    return x86_get_set_vector_regs(thread, const_cast<zx_thread_state_vector_regs*>(in),
374                                   RegAccess::kSet);
375}
376
377zx_status_t arch_get_x86_register_fs(struct thread* thread, uint64_t* out) {
378    Guard<spin_lock_t, IrqSave> thread_lock_guard{ThreadLock::Get()};
379    if (thread->state == THREAD_RUNNING) {
380        return ZX_ERR_BAD_STATE;
381    }
382
383    *out = thread->arch.fs_base;
384    return ZX_OK;
385}
386
387zx_status_t arch_set_x86_register_fs(struct thread* thread, const uint64_t* in) {
388    Guard<spin_lock_t, IrqSave> thread_lock_guard{ThreadLock::Get()};
389    if (thread->state == THREAD_RUNNING) {
390        return ZX_ERR_BAD_STATE;
391    }
392
393    thread->arch.fs_base = *in;
394    return ZX_OK;
395}
396
397zx_status_t arch_get_x86_register_gs(struct thread* thread, uint64_t* out) {
398    Guard<spin_lock_t, IrqSave> thread_lock_guard{ThreadLock::Get()};
399    if (thread->state == THREAD_RUNNING) {
400        return ZX_ERR_BAD_STATE;
401    }
402
403    *out = thread->arch.gs_base;
404    return ZX_OK;
405}
406
407zx_status_t arch_set_x86_register_gs(struct thread* thread, const uint64_t* in) {
408    Guard<spin_lock_t, IrqSave> thread_lock_guard{ThreadLock::Get()};
409    if (thread->state == THREAD_RUNNING) {
410        return ZX_ERR_BAD_STATE;
411    }
412
413    thread->arch.gs_base = *in;
414    return ZX_OK;
415}
416