1// Copyright 2017 The Fuchsia Authors 2// 3// Use of this source code is governed by a MIT-style 4// license that can be found in the LICENSE file or at 5// https://opensource.org/licenses/MIT 6 7#include <platform.h> 8 9#include <arch/hypervisor.h> 10#include <arch/ops.h> 11#include <dev/interrupt/arm_gic_hw_interface.h> 12#include <fbl/auto_call.h> 13#include <hypervisor/cpu.h> 14#include <hypervisor/guest_physical_address_space.h> 15#include <hypervisor/ktrace.h> 16#include <kernel/event.h> 17#include <kernel/mp.h> 18#include <lib/ktrace.h> 19#include <platform/timer.h> 20#include <vm/physmap.h> 21#include <vm/pmm.h> 22#include <zircon/errors.h> 23#include <zircon/syscalls/hypervisor.h> 24 25#include "el2_cpu_state_priv.h" 26#include "vmexit_priv.h" 27 28static constexpr uint32_t kGichHcrEn = 1u << 0; 29static constexpr uint32_t kGichHcrUie = 1u << 1; 30static constexpr uint32_t kGichMisrU = 1u << 1; 31static constexpr uint32_t kSpsrDaif = 0b1111 << 6; 32static constexpr uint32_t kSpsrEl1h = 0b0101; 33static constexpr uint32_t kSpsrNzcv = 0b1111 << 28; 34 35static uint64_t vmpidr_of(uint8_t vpid, uint64_t mpidr) { 36 return (vpid - 1) | (mpidr & 0xffffff00fe000000); 37} 38 39static bool gich_maybe_interrupt(GichState* gich_state) { 40 uint64_t elrsr = gich_state->elrsr; 41 if (elrsr == 0) { 42 // All list registers are in use, therefore return and indicate that we 43 // should raise an IRQ. 44 return true; 45 } 46 zx_status_t status; 47 uint32_t pending = 0; 48 uint32_t vector = kTimerVector; 49 if (gich_state->interrupt_tracker.TryPop(vector)) { 50 // We give timer interrupts precedence over all others. If we find a 51 // timer interrupt is pending, process it first. 52 goto has_timer; 53 } 54 while (elrsr != 0) { 55 status = gich_state->interrupt_tracker.Pop(&vector); 56 if (status != ZX_OK) { 57 // There are no more pending interrupts. 58 break; 59 } 60 has_timer: 61 pending++; 62 if (gich_state->active_interrupts.GetOne(vector)) { 63 // Skip an interrupt if it was already active. 64 continue; 65 } 66 uint32_t lr_index = __builtin_ctzl(elrsr); 67 uint64_t lr = gic_get_lr_from_vector(vector); 68 gich_state->lr[lr_index] = lr; 69 elrsr &= ~(1u << lr_index); 70 } 71 // If there are pending interrupts, indicate that we should raise an IRQ. 72 return pending > 0; 73} 74 75static bool gich_active_interrupts(GichState* gich_state) { 76 gich_state->active_interrupts.ClearAll(); 77 const uint32_t lr_limit = __builtin_ctzl(gich_state->elrsr); 78 for (uint32_t i = 0; i < lr_limit; i++) { 79 uint32_t vector = gic_get_vector_from_lr(gich_state->lr[i]); 80 gich_state->active_interrupts.SetOne(vector); 81 } 82 return lr_limit > 0; 83} 84 85static VcpuExit vmexit_interrupt_ktrace_meta(uint32_t misr) { 86 if ((misr & kGichMisrU) != 0) { 87 return VCPU_UNDERFLOW_MAINTENANCE_INTERRUPT; 88 } 89 return VCPU_PHYSICAL_INTERRUPT; 90} 91 92AutoGich::AutoGich(GichState* gich_state) 93 : gich_state_(gich_state) { 94 DEBUG_ASSERT(!arch_ints_disabled()); 95 arch_disable_ints(); 96 97 // Load 98 gic_write_gich_vmcr(gich_state_->vmcr); 99 gic_write_gich_apr(gich_state_->apr); 100 for (uint32_t i = 0; i < gich_state_->num_lrs; i++) { 101 gic_write_gich_lr(i, gich_state->lr[i]); 102 } 103} 104 105AutoGich::~AutoGich() { 106 DEBUG_ASSERT(arch_ints_disabled()); 107 108 // Save 109 gich_state_->vmcr = gic_read_gich_vmcr(); 110 gich_state_->elrsr = gic_read_gich_elrsr(); 111 gich_state_->apr = gic_read_gich_apr(); 112 for (uint32_t i = 0; i < gich_state_->num_lrs; i++) { 113 gich_state_->lr[i] = gic_read_gich_lr(i); 114 } 115 arch_enable_ints(); 116} 117 118zx_status_t El2StatePtr::Alloc() { 119 zx_status_t status = page_.Alloc(0); 120 if (status != ZX_OK) { 121 return status; 122 } 123 state_ = page_.VirtualAddress<El2State>(); 124 return ZX_OK; 125} 126 127// static 128zx_status_t Vcpu::Create(Guest* guest, zx_vaddr_t entry, fbl::unique_ptr<Vcpu>* out) { 129 hypervisor::GuestPhysicalAddressSpace* gpas = guest->AddressSpace(); 130 if (entry >= gpas->size()) { 131 return ZX_ERR_INVALID_ARGS; 132 } 133 134 uint8_t vpid; 135 zx_status_t status = guest->AllocVpid(&vpid); 136 if (status != ZX_OK) { 137 return status; 138 } 139 auto auto_call = fbl::MakeAutoCall([guest, vpid]() { guest->FreeVpid(vpid); }); 140 141 // For efficiency, we pin the thread to the CPU. 142 thread_t* thread = hypervisor::pin_thread(vpid); 143 144 fbl::AllocChecker ac; 145 fbl::unique_ptr<Vcpu> vcpu(new (&ac) Vcpu(guest, vpid, thread)); 146 if (!ac.check()) { 147 return ZX_ERR_NO_MEMORY; 148 } 149 auto_call.cancel(); 150 151 timer_init(&vcpu->gich_state_.timer); 152 status = vcpu->gich_state_.interrupt_tracker.Init(); 153 if (status != ZX_OK) { 154 return status; 155 } 156 157 status = vcpu->el2_state_.Alloc(); 158 if (status != ZX_OK) { 159 return status; 160 } 161 162 gic_write_gich_hcr(kGichHcrEn); 163 vcpu->gich_state_.active_interrupts.Reset(kNumInterrupts); 164 vcpu->gich_state_.num_lrs = gic_get_num_lrs(); 165 vcpu->gich_state_.vmcr = gic_default_gich_vmcr(); 166 vcpu->gich_state_.elrsr = gic_read_gich_elrsr(); 167 vcpu->gich_state_.apr = 0; 168 vcpu->el2_state_->guest_state.system_state.elr_el2 = entry; 169 vcpu->el2_state_->guest_state.system_state.spsr_el2 = kSpsrDaif | kSpsrEl1h; 170 uint64_t mpidr = ARM64_READ_SYSREG(mpidr_el1); 171 vcpu->el2_state_->guest_state.system_state.vmpidr_el2 = vmpidr_of(vpid, mpidr); 172 vcpu->el2_state_->host_state.system_state.vmpidr_el2 = mpidr; 173 vcpu->hcr_ = HCR_EL2_VM | HCR_EL2_PTW | HCR_EL2_IMO | HCR_EL2_DC | HCR_EL2_TWI | HCR_EL2_TWE | 174 HCR_EL2_TSC | HCR_EL2_TVM | HCR_EL2_RW; 175 176 *out = fbl::move(vcpu); 177 return ZX_OK; 178} 179 180Vcpu::Vcpu(Guest* guest, uint8_t vpid, const thread_t* thread) 181 : guest_(guest), vpid_(vpid), thread_(thread), running_(false) { 182 (void)thread_; 183} 184 185Vcpu::~Vcpu() { 186 timer_cancel(&gich_state_.timer); 187 __UNUSED zx_status_t status = guest_->FreeVpid(vpid_); 188 DEBUG_ASSERT(status == ZX_OK); 189} 190 191zx_status_t Vcpu::Resume(zx_port_packet_t* packet) { 192 if (!hypervisor::check_pinned_cpu_invariant(vpid_, thread_)) 193 return ZX_ERR_BAD_STATE; 194 const ArchVmAspace& aspace = *guest_->AddressSpace()->arch_aspace(); 195 zx_paddr_t vttbr = arm64_vttbr(aspace.arch_asid(), aspace.arch_table_phys()); 196 GuestState* guest_state = &el2_state_->guest_state; 197 bool force_virtual_interrupt = false; 198 zx_status_t status; 199 do { 200 uint64_t curr_hcr = hcr_; 201 uint32_t misr = 0; 202 if (gich_maybe_interrupt(&gich_state_) || force_virtual_interrupt) { 203 curr_hcr |= HCR_EL2_VI; 204 force_virtual_interrupt = false; 205 } 206 { 207 AutoGich auto_gich(&gich_state_); 208 209 // Underflow maintenance interrupt is signalled if there is one or no free LRs. 210 // We use it in case when there is not enough free LRs to inject all pending 211 // interrupts, so when guest finishes processing most of them, a maintenance 212 // interrupt will cause VM exit and will give us a chance to inject the remaining 213 // interrupts. The point of this is to reduce latency when processing interrupts. 214 uint32_t gich_hcr = 0; 215 if (gich_state_.interrupt_tracker.Pending() && gich_state_.num_lrs > 1) { 216 gich_hcr = gic_read_gich_hcr() | kGichHcrUie; 217 gic_write_gich_hcr(gich_hcr); 218 } 219 220 ktrace(TAG_VCPU_ENTER, 0, 0, 0, 0); 221 running_.store(true); 222 status = arm64_el2_resume(vttbr, el2_state_.PhysicalAddress(), curr_hcr); 223 running_.store(false); 224 225 // If we enabled underflow interrupt before we entered the guest we disable it 226 // to deassert it in case it is signalled. For details please refer to ARM Generic 227 // Interrupt Controller, Architecture Specification, 5.3.4 Maintenance Interrupt 228 // Status Register, GICH_MISR. Description of U bit in GICH_MISR. 229 if ((gich_hcr & kGichHcrUie) != 0) { 230 misr = gic_read_gich_misr(); 231 gic_write_gich_hcr(gich_hcr & ~kGichHcrUie); 232 } 233 } 234 bool has_active_interrupt = gich_active_interrupts(&gich_state_); 235 if (status == ZX_ERR_NEXT) { 236 // We received a physical interrupt. If it was due to the thread 237 // being killed, then we should exit with an error, otherwise return 238 // to the guest. 239 ktrace_vcpu_exit(vmexit_interrupt_ktrace_meta(misr), 240 guest_state->system_state.elr_el2); 241 status = thread_->signals & THREAD_SIGNAL_KILL ? ZX_ERR_CANCELED : ZX_OK; 242 // If there were active interrupts when the physical interrupt 243 // occurred, raise a virtual interrupt when we re-enter the guest. 244 force_virtual_interrupt = has_active_interrupt; 245 } else if (status == ZX_OK) { 246 status = vmexit_handler(&hcr_, guest_state, &gich_state_, guest_->AddressSpace(), 247 guest_->Traps(), packet); 248 } else { 249 ktrace_vcpu_exit(VCPU_FAILURE, guest_state->system_state.elr_el2); 250 dprintf(INFO, "VCPU resume failed: %d\n", status); 251 } 252 } while (status == ZX_OK); 253 return status == ZX_ERR_NEXT ? ZX_OK : status; 254} 255 256zx_status_t Vcpu::Interrupt(uint32_t vector) { 257 bool signaled = false; 258 zx_status_t status = gich_state_.interrupt_tracker.Interrupt(vector, &signaled); 259 if (status != ZX_OK) { 260 return status; 261 } else if (!signaled && running_.load()) { 262 mp_interrupt(MP_IPI_TARGET_MASK, cpu_num_to_mask(hypervisor::cpu_of(vpid_))); 263 } 264 return ZX_OK; 265} 266 267zx_status_t Vcpu::ReadState(uint32_t kind, void* buf, size_t len) const { 268 if (!hypervisor::check_pinned_cpu_invariant(vpid_, thread_)) { 269 return ZX_ERR_BAD_STATE; 270 } else if (kind != ZX_VCPU_STATE || len != sizeof(zx_vcpu_state_t)) { 271 return ZX_ERR_INVALID_ARGS; 272 } 273 274 auto state = static_cast<zx_vcpu_state_t*>(buf); 275 memcpy(state->x, el2_state_->guest_state.x, sizeof(uint64_t) * GS_NUM_REGS); 276 state->sp = el2_state_->guest_state.system_state.sp_el1; 277 state->cpsr = el2_state_->guest_state.system_state.spsr_el2 & kSpsrNzcv; 278 return ZX_OK; 279} 280 281zx_status_t Vcpu::WriteState(uint32_t kind, const void* buf, size_t len) { 282 if (!hypervisor::check_pinned_cpu_invariant(vpid_, thread_)) { 283 return ZX_ERR_BAD_STATE; 284 } else if (kind != ZX_VCPU_STATE || len != sizeof(zx_vcpu_state_t)) { 285 return ZX_ERR_INVALID_ARGS; 286 } 287 288 auto state = static_cast<const zx_vcpu_state_t*>(buf); 289 memcpy(el2_state_->guest_state.x, state->x, sizeof(uint64_t) * GS_NUM_REGS); 290 el2_state_->guest_state.system_state.sp_el1 = state->sp; 291 el2_state_->guest_state.system_state.spsr_el2 |= state->cpsr & kSpsrNzcv; 292 return ZX_OK; 293} 294