1// Copyright 2017 The Fuchsia Authors
2//
3// Use of this source code is governed by a MIT-style
4// license that can be found in the LICENSE file or at
5// https://opensource.org/licenses/MIT
6
7#include <platform.h>
8
9#include <arch/hypervisor.h>
10#include <arch/ops.h>
11#include <dev/interrupt/arm_gic_hw_interface.h>
12#include <fbl/auto_call.h>
13#include <hypervisor/cpu.h>
14#include <hypervisor/guest_physical_address_space.h>
15#include <hypervisor/ktrace.h>
16#include <kernel/event.h>
17#include <kernel/mp.h>
18#include <lib/ktrace.h>
19#include <platform/timer.h>
20#include <vm/physmap.h>
21#include <vm/pmm.h>
22#include <zircon/errors.h>
23#include <zircon/syscalls/hypervisor.h>
24
25#include "el2_cpu_state_priv.h"
26#include "vmexit_priv.h"
27
28static constexpr uint32_t kGichHcrEn = 1u << 0;
29static constexpr uint32_t kGichHcrUie = 1u << 1;
30static constexpr uint32_t kGichMisrU = 1u << 1;
31static constexpr uint32_t kSpsrDaif = 0b1111 << 6;
32static constexpr uint32_t kSpsrEl1h = 0b0101;
33static constexpr uint32_t kSpsrNzcv = 0b1111 << 28;
34
35static uint64_t vmpidr_of(uint8_t vpid, uint64_t mpidr) {
36    return (vpid - 1) | (mpidr & 0xffffff00fe000000);
37}
38
39static bool gich_maybe_interrupt(GichState* gich_state) {
40    uint64_t elrsr = gich_state->elrsr;
41    if (elrsr == 0) {
42        // All list registers are in use, therefore return and indicate that we
43        // should raise an IRQ.
44        return true;
45    }
46    zx_status_t status;
47    uint32_t pending = 0;
48    uint32_t vector = kTimerVector;
49    if (gich_state->interrupt_tracker.TryPop(vector)) {
50        // We give timer interrupts precedence over all others. If we find a
51        // timer interrupt is pending, process it first.
52        goto has_timer;
53    }
54    while (elrsr != 0) {
55        status = gich_state->interrupt_tracker.Pop(&vector);
56        if (status != ZX_OK) {
57            // There are no more pending interrupts.
58            break;
59        }
60    has_timer:
61        pending++;
62        if (gich_state->active_interrupts.GetOne(vector)) {
63            // Skip an interrupt if it was already active.
64            continue;
65        }
66        uint32_t lr_index = __builtin_ctzl(elrsr);
67        uint64_t lr = gic_get_lr_from_vector(vector);
68        gich_state->lr[lr_index] = lr;
69        elrsr &= ~(1u << lr_index);
70    }
71    // If there are pending interrupts, indicate that we should raise an IRQ.
72    return pending > 0;
73}
74
75static bool gich_active_interrupts(GichState* gich_state) {
76    gich_state->active_interrupts.ClearAll();
77    const uint32_t lr_limit = __builtin_ctzl(gich_state->elrsr);
78    for (uint32_t i = 0; i < lr_limit; i++) {
79        uint32_t vector = gic_get_vector_from_lr(gich_state->lr[i]);
80        gich_state->active_interrupts.SetOne(vector);
81    }
82    return lr_limit > 0;
83}
84
85static VcpuExit vmexit_interrupt_ktrace_meta(uint32_t misr) {
86    if ((misr & kGichMisrU) != 0) {
87        return VCPU_UNDERFLOW_MAINTENANCE_INTERRUPT;
88    }
89    return VCPU_PHYSICAL_INTERRUPT;
90}
91
92AutoGich::AutoGich(GichState* gich_state)
93    : gich_state_(gich_state) {
94    DEBUG_ASSERT(!arch_ints_disabled());
95    arch_disable_ints();
96
97    // Load
98    gic_write_gich_vmcr(gich_state_->vmcr);
99    gic_write_gich_apr(gich_state_->apr);
100    for (uint32_t i = 0; i < gich_state_->num_lrs; i++) {
101        gic_write_gich_lr(i, gich_state->lr[i]);
102    }
103}
104
105AutoGich::~AutoGich() {
106    DEBUG_ASSERT(arch_ints_disabled());
107
108    // Save
109    gich_state_->vmcr = gic_read_gich_vmcr();
110    gich_state_->elrsr = gic_read_gich_elrsr();
111    gich_state_->apr = gic_read_gich_apr();
112    for (uint32_t i = 0; i < gich_state_->num_lrs; i++) {
113        gich_state_->lr[i] = gic_read_gich_lr(i);
114    }
115    arch_enable_ints();
116}
117
118zx_status_t El2StatePtr::Alloc() {
119    zx_status_t status = page_.Alloc(0);
120    if (status != ZX_OK) {
121        return status;
122    }
123    state_ = page_.VirtualAddress<El2State>();
124    return ZX_OK;
125}
126
127// static
128zx_status_t Vcpu::Create(Guest* guest, zx_vaddr_t entry, fbl::unique_ptr<Vcpu>* out) {
129    hypervisor::GuestPhysicalAddressSpace* gpas = guest->AddressSpace();
130    if (entry >= gpas->size()) {
131        return ZX_ERR_INVALID_ARGS;
132    }
133
134    uint8_t vpid;
135    zx_status_t status = guest->AllocVpid(&vpid);
136    if (status != ZX_OK) {
137        return status;
138    }
139    auto auto_call = fbl::MakeAutoCall([guest, vpid]() { guest->FreeVpid(vpid); });
140
141    // For efficiency, we pin the thread to the CPU.
142    thread_t* thread = hypervisor::pin_thread(vpid);
143
144    fbl::AllocChecker ac;
145    fbl::unique_ptr<Vcpu> vcpu(new (&ac) Vcpu(guest, vpid, thread));
146    if (!ac.check()) {
147        return ZX_ERR_NO_MEMORY;
148    }
149    auto_call.cancel();
150
151    timer_init(&vcpu->gich_state_.timer);
152    status = vcpu->gich_state_.interrupt_tracker.Init();
153    if (status != ZX_OK) {
154        return status;
155    }
156
157    status = vcpu->el2_state_.Alloc();
158    if (status != ZX_OK) {
159        return status;
160    }
161
162    gic_write_gich_hcr(kGichHcrEn);
163    vcpu->gich_state_.active_interrupts.Reset(kNumInterrupts);
164    vcpu->gich_state_.num_lrs = gic_get_num_lrs();
165    vcpu->gich_state_.vmcr = gic_default_gich_vmcr();
166    vcpu->gich_state_.elrsr = gic_read_gich_elrsr();
167    vcpu->gich_state_.apr = 0;
168    vcpu->el2_state_->guest_state.system_state.elr_el2 = entry;
169    vcpu->el2_state_->guest_state.system_state.spsr_el2 = kSpsrDaif | kSpsrEl1h;
170    uint64_t mpidr = ARM64_READ_SYSREG(mpidr_el1);
171    vcpu->el2_state_->guest_state.system_state.vmpidr_el2 = vmpidr_of(vpid, mpidr);
172    vcpu->el2_state_->host_state.system_state.vmpidr_el2 = mpidr;
173    vcpu->hcr_ = HCR_EL2_VM | HCR_EL2_PTW | HCR_EL2_IMO | HCR_EL2_DC | HCR_EL2_TWI | HCR_EL2_TWE |
174                 HCR_EL2_TSC | HCR_EL2_TVM | HCR_EL2_RW;
175
176    *out = fbl::move(vcpu);
177    return ZX_OK;
178}
179
180Vcpu::Vcpu(Guest* guest, uint8_t vpid, const thread_t* thread)
181    : guest_(guest), vpid_(vpid), thread_(thread), running_(false) {
182    (void)thread_;
183}
184
185Vcpu::~Vcpu() {
186    timer_cancel(&gich_state_.timer);
187    __UNUSED zx_status_t status = guest_->FreeVpid(vpid_);
188    DEBUG_ASSERT(status == ZX_OK);
189}
190
191zx_status_t Vcpu::Resume(zx_port_packet_t* packet) {
192    if (!hypervisor::check_pinned_cpu_invariant(vpid_, thread_))
193        return ZX_ERR_BAD_STATE;
194    const ArchVmAspace& aspace = *guest_->AddressSpace()->arch_aspace();
195    zx_paddr_t vttbr = arm64_vttbr(aspace.arch_asid(), aspace.arch_table_phys());
196    GuestState* guest_state = &el2_state_->guest_state;
197    bool force_virtual_interrupt = false;
198    zx_status_t status;
199    do {
200        uint64_t curr_hcr = hcr_;
201        uint32_t misr = 0;
202        if (gich_maybe_interrupt(&gich_state_) || force_virtual_interrupt) {
203            curr_hcr |= HCR_EL2_VI;
204            force_virtual_interrupt = false;
205        }
206        {
207            AutoGich auto_gich(&gich_state_);
208
209            // Underflow maintenance interrupt is signalled if there is one or no free LRs.
210            // We use it in case when there is not enough free LRs to inject all pending
211            // interrupts, so when guest finishes processing most of them, a maintenance
212            // interrupt will cause VM exit and will give us a chance to inject the remaining
213            // interrupts. The point of this is to reduce latency when processing interrupts.
214            uint32_t gich_hcr = 0;
215            if (gich_state_.interrupt_tracker.Pending() && gich_state_.num_lrs > 1) {
216                gich_hcr = gic_read_gich_hcr() | kGichHcrUie;
217                gic_write_gich_hcr(gich_hcr);
218            }
219
220            ktrace(TAG_VCPU_ENTER, 0, 0, 0, 0);
221            running_.store(true);
222            status = arm64_el2_resume(vttbr, el2_state_.PhysicalAddress(), curr_hcr);
223            running_.store(false);
224
225            // If we enabled underflow interrupt before we entered the guest we disable it
226            // to deassert it in case it is signalled. For details please refer to ARM Generic
227            // Interrupt Controller, Architecture Specification, 5.3.4 Maintenance Interrupt
228            // Status Register, GICH_MISR. Description of U bit in GICH_MISR.
229            if ((gich_hcr & kGichHcrUie) != 0) {
230                misr = gic_read_gich_misr();
231                gic_write_gich_hcr(gich_hcr & ~kGichHcrUie);
232            }
233        }
234        bool has_active_interrupt = gich_active_interrupts(&gich_state_);
235        if (status == ZX_ERR_NEXT) {
236            // We received a physical interrupt. If it was due to the thread
237            // being killed, then we should exit with an error, otherwise return
238            // to the guest.
239            ktrace_vcpu_exit(vmexit_interrupt_ktrace_meta(misr),
240                             guest_state->system_state.elr_el2);
241            status = thread_->signals & THREAD_SIGNAL_KILL ? ZX_ERR_CANCELED : ZX_OK;
242            // If there were active interrupts when the physical interrupt
243            // occurred, raise a virtual interrupt when we re-enter the guest.
244            force_virtual_interrupt = has_active_interrupt;
245        } else if (status == ZX_OK) {
246            status = vmexit_handler(&hcr_, guest_state, &gich_state_, guest_->AddressSpace(),
247                                    guest_->Traps(), packet);
248        } else {
249            ktrace_vcpu_exit(VCPU_FAILURE, guest_state->system_state.elr_el2);
250            dprintf(INFO, "VCPU resume failed: %d\n", status);
251        }
252    } while (status == ZX_OK);
253    return status == ZX_ERR_NEXT ? ZX_OK : status;
254}
255
256zx_status_t Vcpu::Interrupt(uint32_t vector) {
257    bool signaled = false;
258    zx_status_t status = gich_state_.interrupt_tracker.Interrupt(vector, &signaled);
259    if (status != ZX_OK) {
260        return status;
261    } else if (!signaled && running_.load()) {
262        mp_interrupt(MP_IPI_TARGET_MASK, cpu_num_to_mask(hypervisor::cpu_of(vpid_)));
263    }
264    return ZX_OK;
265}
266
267zx_status_t Vcpu::ReadState(uint32_t kind, void* buf, size_t len) const {
268    if (!hypervisor::check_pinned_cpu_invariant(vpid_, thread_)) {
269        return ZX_ERR_BAD_STATE;
270    } else if (kind != ZX_VCPU_STATE || len != sizeof(zx_vcpu_state_t)) {
271        return ZX_ERR_INVALID_ARGS;
272    }
273
274    auto state = static_cast<zx_vcpu_state_t*>(buf);
275    memcpy(state->x, el2_state_->guest_state.x, sizeof(uint64_t) * GS_NUM_REGS);
276    state->sp = el2_state_->guest_state.system_state.sp_el1;
277    state->cpsr = el2_state_->guest_state.system_state.spsr_el2 & kSpsrNzcv;
278    return ZX_OK;
279}
280
281zx_status_t Vcpu::WriteState(uint32_t kind, const void* buf, size_t len) {
282    if (!hypervisor::check_pinned_cpu_invariant(vpid_, thread_)) {
283        return ZX_ERR_BAD_STATE;
284    } else if (kind != ZX_VCPU_STATE || len != sizeof(zx_vcpu_state_t)) {
285        return ZX_ERR_INVALID_ARGS;
286    }
287
288    auto state = static_cast<const zx_vcpu_state_t*>(buf);
289    memcpy(el2_state_->guest_state.x, state->x, sizeof(uint64_t) * GS_NUM_REGS);
290    el2_state_->guest_state.system_state.sp_el1 = state->sp;
291    el2_state_->guest_state.system_state.spsr_el2 |= state->cpsr & kSpsrNzcv;
292    return ZX_OK;
293}
294