vmm.c revision 245021
1221828Sgrehan/*- 2221828Sgrehan * Copyright (c) 2011 NetApp, Inc. 3221828Sgrehan * All rights reserved. 4221828Sgrehan * 5221828Sgrehan * Redistribution and use in source and binary forms, with or without 6221828Sgrehan * modification, are permitted provided that the following conditions 7221828Sgrehan * are met: 8221828Sgrehan * 1. Redistributions of source code must retain the above copyright 9221828Sgrehan * notice, this list of conditions and the following disclaimer. 10221828Sgrehan * 2. Redistributions in binary form must reproduce the above copyright 11221828Sgrehan * notice, this list of conditions and the following disclaimer in the 12221828Sgrehan * documentation and/or other materials provided with the distribution. 13221828Sgrehan * 14221828Sgrehan * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND 15221828Sgrehan * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16221828Sgrehan * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17221828Sgrehan * ARE DISCLAIMED. IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE 18221828Sgrehan * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19221828Sgrehan * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20221828Sgrehan * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21221828Sgrehan * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22221828Sgrehan * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23221828Sgrehan * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24221828Sgrehan * SUCH DAMAGE. 25221828Sgrehan * 26221828Sgrehan * $FreeBSD$ 27221828Sgrehan */ 28221828Sgrehan 29221828Sgrehan#include <sys/cdefs.h> 30221828Sgrehan__FBSDID("$FreeBSD$"); 31221828Sgrehan 32221828Sgrehan#include <sys/param.h> 33234695Sgrehan#include <sys/systm.h> 34221828Sgrehan#include <sys/kernel.h> 35221828Sgrehan#include <sys/module.h> 36221828Sgrehan#include <sys/sysctl.h> 37221828Sgrehan#include <sys/malloc.h> 38221828Sgrehan#include <sys/pcpu.h> 39221828Sgrehan#include <sys/lock.h> 40221828Sgrehan#include <sys/mutex.h> 41221828Sgrehan#include <sys/proc.h> 42221828Sgrehan#include <sys/sched.h> 43221828Sgrehan#include <sys/smp.h> 44221828Sgrehan#include <sys/systm.h> 45221828Sgrehan 46221828Sgrehan#include <vm/vm.h> 47221828Sgrehan 48221828Sgrehan#include <machine/vm.h> 49221828Sgrehan#include <machine/pcb.h> 50241489Sneel#include <machine/smp.h> 51221914Sjhb#include <x86/apicreg.h> 52221828Sgrehan 53221828Sgrehan#include <machine/vmm.h> 54242275Sneel#include "vmm_host.h" 55221828Sgrehan#include "vmm_mem.h" 56221828Sgrehan#include "vmm_util.h" 57221828Sgrehan#include <machine/vmm_dev.h> 58221828Sgrehan#include "vlapic.h" 59221828Sgrehan#include "vmm_msr.h" 60221828Sgrehan#include "vmm_ipi.h" 61221828Sgrehan#include "vmm_stat.h" 62242065Sneel#include "vmm_lapic.h" 63221828Sgrehan 64221828Sgrehan#include "io/ppt.h" 65221828Sgrehan#include "io/iommu.h" 66221828Sgrehan 67221828Sgrehanstruct vlapic; 68221828Sgrehan 69221828Sgrehanstruct vcpu { 70221828Sgrehan int flags; 71241489Sneel enum vcpu_state state; 72241489Sneel struct mtx mtx; 73221828Sgrehan int pincpu; /* host cpuid this vcpu is bound to */ 74221828Sgrehan int hostcpu; /* host cpuid this vcpu last ran on */ 75221828Sgrehan uint64_t guest_msrs[VMM_MSR_NUM]; 76221828Sgrehan struct vlapic *vlapic; 77221828Sgrehan int vcpuid; 78234695Sgrehan struct savefpu *guestfpu; /* guest fpu state */ 79221828Sgrehan void *stats; 80240894Sneel struct vm_exit exitinfo; 81240922Sneel enum x2apic_state x2apic_state; 82241982Sneel int nmi_pending; 83221828Sgrehan}; 84221828Sgrehan#define VCPU_F_PINNED 0x0001 85221828Sgrehan 86221828Sgrehan#define VCPU_PINCPU(vm, vcpuid) \ 87221828Sgrehan ((vm->vcpu[vcpuid].flags & VCPU_F_PINNED) ? vm->vcpu[vcpuid].pincpu : -1) 88221828Sgrehan 89221828Sgrehan#define VCPU_UNPIN(vm, vcpuid) (vm->vcpu[vcpuid].flags &= ~VCPU_F_PINNED) 90221828Sgrehan 91221828Sgrehan#define VCPU_PIN(vm, vcpuid, host_cpuid) \ 92221828Sgrehando { \ 93221828Sgrehan vm->vcpu[vcpuid].flags |= VCPU_F_PINNED; \ 94221828Sgrehan vm->vcpu[vcpuid].pincpu = host_cpuid; \ 95221828Sgrehan} while(0) 96221828Sgrehan 97242065Sneel#define vcpu_lock_init(v) mtx_init(&((v)->mtx), "vcpu lock", 0, MTX_SPIN) 98242065Sneel#define vcpu_lock(v) mtx_lock_spin(&((v)->mtx)) 99242065Sneel#define vcpu_unlock(v) mtx_unlock_spin(&((v)->mtx)) 100241489Sneel 101221828Sgrehan#define VM_MAX_MEMORY_SEGMENTS 2 102221828Sgrehan 103221828Sgrehanstruct vm { 104221828Sgrehan void *cookie; /* processor-specific data */ 105221828Sgrehan void *iommu; /* iommu-specific data */ 106221828Sgrehan struct vcpu vcpu[VM_MAXCPU]; 107221828Sgrehan int num_mem_segs; 108221828Sgrehan struct vm_memory_segment mem_segs[VM_MAX_MEMORY_SEGMENTS]; 109221828Sgrehan char name[VM_MAX_NAMELEN]; 110221828Sgrehan 111221828Sgrehan /* 112223621Sgrehan * Set of active vcpus. 113221828Sgrehan * An active vcpu is one that has been started implicitly (BSP) or 114221828Sgrehan * explicitly (AP) by sending it a startup ipi. 115221828Sgrehan */ 116223621Sgrehan cpuset_t active_cpus; 117221828Sgrehan}; 118221828Sgrehan 119221828Sgrehanstatic struct vmm_ops *ops; 120221828Sgrehan#define VMM_INIT() (ops != NULL ? (*ops->init)() : 0) 121221828Sgrehan#define VMM_CLEANUP() (ops != NULL ? (*ops->cleanup)() : 0) 122221828Sgrehan 123221828Sgrehan#define VMINIT(vm) (ops != NULL ? (*ops->vminit)(vm): NULL) 124240894Sneel#define VMRUN(vmi, vcpu, rip) \ 125240894Sneel (ops != NULL ? (*ops->vmrun)(vmi, vcpu, rip) : ENXIO) 126221828Sgrehan#define VMCLEANUP(vmi) (ops != NULL ? (*ops->vmcleanup)(vmi) : NULL) 127241147Sneel#define VMMMAP_SET(vmi, gpa, hpa, len, attr, prot, spm) \ 128241147Sneel (ops != NULL ? \ 129241147Sneel (*ops->vmmmap_set)(vmi, gpa, hpa, len, attr, prot, spm) : \ 130241147Sneel ENXIO) 131241147Sneel#define VMMMAP_GET(vmi, gpa) \ 132241147Sneel (ops != NULL ? (*ops->vmmmap_get)(vmi, gpa) : ENXIO) 133221828Sgrehan#define VMGETREG(vmi, vcpu, num, retval) \ 134221828Sgrehan (ops != NULL ? (*ops->vmgetreg)(vmi, vcpu, num, retval) : ENXIO) 135221828Sgrehan#define VMSETREG(vmi, vcpu, num, val) \ 136221828Sgrehan (ops != NULL ? (*ops->vmsetreg)(vmi, vcpu, num, val) : ENXIO) 137221828Sgrehan#define VMGETDESC(vmi, vcpu, num, desc) \ 138221828Sgrehan (ops != NULL ? (*ops->vmgetdesc)(vmi, vcpu, num, desc) : ENXIO) 139221828Sgrehan#define VMSETDESC(vmi, vcpu, num, desc) \ 140221828Sgrehan (ops != NULL ? (*ops->vmsetdesc)(vmi, vcpu, num, desc) : ENXIO) 141221828Sgrehan#define VMINJECT(vmi, vcpu, type, vec, ec, ecv) \ 142221828Sgrehan (ops != NULL ? (*ops->vminject)(vmi, vcpu, type, vec, ec, ecv) : ENXIO) 143221828Sgrehan#define VMGETCAP(vmi, vcpu, num, retval) \ 144221828Sgrehan (ops != NULL ? (*ops->vmgetcap)(vmi, vcpu, num, retval) : ENXIO) 145221828Sgrehan#define VMSETCAP(vmi, vcpu, num, val) \ 146221828Sgrehan (ops != NULL ? (*ops->vmsetcap)(vmi, vcpu, num, val) : ENXIO) 147221828Sgrehan 148245021Sneel#define fpu_start_emulating() load_cr0(rcr0() | CR0_TS) 149245021Sneel#define fpu_stop_emulating() clts() 150221828Sgrehan 151221828Sgrehanstatic MALLOC_DEFINE(M_VM, "vm", "vm"); 152221828SgrehanCTASSERT(VMM_MSR_NUM <= 64); /* msr_mask can keep track of up to 64 msrs */ 153221828Sgrehan 154221828Sgrehan/* statistics */ 155221828Sgrehanstatic VMM_STAT_DEFINE(VCPU_TOTAL_RUNTIME, "vcpu total runtime"); 156221828Sgrehan 157221828Sgrehanstatic void 158221828Sgrehanvcpu_cleanup(struct vcpu *vcpu) 159221828Sgrehan{ 160221828Sgrehan vlapic_cleanup(vcpu->vlapic); 161234695Sgrehan vmm_stat_free(vcpu->stats); 162234695Sgrehan fpu_save_area_free(vcpu->guestfpu); 163221828Sgrehan} 164221828Sgrehan 165221828Sgrehanstatic void 166221828Sgrehanvcpu_init(struct vm *vm, uint32_t vcpu_id) 167221828Sgrehan{ 168221828Sgrehan struct vcpu *vcpu; 169221828Sgrehan 170221828Sgrehan vcpu = &vm->vcpu[vcpu_id]; 171221828Sgrehan 172241489Sneel vcpu_lock_init(vcpu); 173241489Sneel vcpu->hostcpu = NOCPU; 174221828Sgrehan vcpu->vcpuid = vcpu_id; 175221828Sgrehan vcpu->vlapic = vlapic_init(vm, vcpu_id); 176240943Sneel vm_set_x2apic_state(vm, vcpu_id, X2APIC_ENABLED); 177234695Sgrehan vcpu->guestfpu = fpu_save_area_alloc(); 178234695Sgrehan fpu_save_area_reset(vcpu->guestfpu); 179221828Sgrehan vcpu->stats = vmm_stat_alloc(); 180221828Sgrehan} 181221828Sgrehan 182240894Sneelstruct vm_exit * 183240894Sneelvm_exitinfo(struct vm *vm, int cpuid) 184240894Sneel{ 185240894Sneel struct vcpu *vcpu; 186240894Sneel 187240894Sneel if (cpuid < 0 || cpuid >= VM_MAXCPU) 188240894Sneel panic("vm_exitinfo: invalid cpuid %d", cpuid); 189240894Sneel 190240894Sneel vcpu = &vm->vcpu[cpuid]; 191240894Sneel 192240894Sneel return (&vcpu->exitinfo); 193240894Sneel} 194240894Sneel 195221828Sgrehanstatic int 196221828Sgrehanvmm_init(void) 197221828Sgrehan{ 198221828Sgrehan int error; 199221828Sgrehan 200242275Sneel vmm_host_state_init(); 201221828Sgrehan vmm_ipi_init(); 202221828Sgrehan 203221828Sgrehan error = vmm_mem_init(); 204221828Sgrehan if (error) 205221828Sgrehan return (error); 206221828Sgrehan 207221828Sgrehan if (vmm_is_intel()) 208221828Sgrehan ops = &vmm_ops_intel; 209221828Sgrehan else if (vmm_is_amd()) 210221828Sgrehan ops = &vmm_ops_amd; 211221828Sgrehan else 212221828Sgrehan return (ENXIO); 213221828Sgrehan 214221828Sgrehan vmm_msr_init(); 215221828Sgrehan 216221828Sgrehan return (VMM_INIT()); 217221828Sgrehan} 218221828Sgrehan 219221828Sgrehanstatic int 220221828Sgrehanvmm_handler(module_t mod, int what, void *arg) 221221828Sgrehan{ 222221828Sgrehan int error; 223221828Sgrehan 224221828Sgrehan switch (what) { 225221828Sgrehan case MOD_LOAD: 226221828Sgrehan vmmdev_init(); 227221828Sgrehan iommu_init(); 228221828Sgrehan error = vmm_init(); 229221828Sgrehan break; 230221828Sgrehan case MOD_UNLOAD: 231241454Sneel error = vmmdev_cleanup(); 232241454Sneel if (error == 0) { 233241454Sneel iommu_cleanup(); 234241454Sneel vmm_ipi_cleanup(); 235241454Sneel error = VMM_CLEANUP(); 236241454Sneel } 237221828Sgrehan break; 238221828Sgrehan default: 239221828Sgrehan error = 0; 240221828Sgrehan break; 241221828Sgrehan } 242221828Sgrehan return (error); 243221828Sgrehan} 244221828Sgrehan 245221828Sgrehanstatic moduledata_t vmm_kmod = { 246221828Sgrehan "vmm", 247221828Sgrehan vmm_handler, 248221828Sgrehan NULL 249221828Sgrehan}; 250221828Sgrehan 251221828Sgrehan/* 252221828Sgrehan * Execute the module load handler after the pci passthru driver has had 253221828Sgrehan * a chance to claim devices. We need this information at the time we do 254221828Sgrehan * iommu initialization. 255221828Sgrehan */ 256221828SgrehanDECLARE_MODULE(vmm, vmm_kmod, SI_SUB_CONFIGURE + 1, SI_ORDER_ANY); 257221828SgrehanMODULE_VERSION(vmm, 1); 258221828Sgrehan 259221828SgrehanSYSCTL_NODE(_hw, OID_AUTO, vmm, CTLFLAG_RW, NULL, NULL); 260221828Sgrehan 261221828Sgrehanstruct vm * 262221828Sgrehanvm_create(const char *name) 263221828Sgrehan{ 264221828Sgrehan int i; 265221828Sgrehan struct vm *vm; 266221828Sgrehan vm_paddr_t maxaddr; 267221828Sgrehan 268221828Sgrehan const int BSP = 0; 269221828Sgrehan 270221828Sgrehan if (name == NULL || strlen(name) >= VM_MAX_NAMELEN) 271221828Sgrehan return (NULL); 272221828Sgrehan 273221828Sgrehan vm = malloc(sizeof(struct vm), M_VM, M_WAITOK | M_ZERO); 274221828Sgrehan strcpy(vm->name, name); 275221828Sgrehan vm->cookie = VMINIT(vm); 276221828Sgrehan 277221828Sgrehan for (i = 0; i < VM_MAXCPU; i++) { 278221828Sgrehan vcpu_init(vm, i); 279221828Sgrehan guest_msrs_init(vm, i); 280221828Sgrehan } 281221828Sgrehan 282221828Sgrehan maxaddr = vmm_mem_maxaddr(); 283221828Sgrehan vm->iommu = iommu_create_domain(maxaddr); 284221828Sgrehan vm_activate_cpu(vm, BSP); 285221828Sgrehan 286221828Sgrehan return (vm); 287221828Sgrehan} 288221828Sgrehan 289241178Sneelstatic void 290241178Sneelvm_free_mem_seg(struct vm *vm, struct vm_memory_segment *seg) 291241178Sneel{ 292241178Sneel size_t len; 293241178Sneel vm_paddr_t hpa; 294241362Sneel void *host_domain; 295241178Sneel 296241362Sneel host_domain = iommu_host_domain(); 297241362Sneel 298241178Sneel len = 0; 299241178Sneel while (len < seg->len) { 300241178Sneel hpa = vm_gpa2hpa(vm, seg->gpa + len, PAGE_SIZE); 301241178Sneel if (hpa == (vm_paddr_t)-1) { 302241178Sneel panic("vm_free_mem_segs: cannot free hpa " 303241178Sneel "associated with gpa 0x%016lx", seg->gpa + len); 304241178Sneel } 305241178Sneel 306241362Sneel /* 307241362Sneel * Remove the 'gpa' to 'hpa' mapping in VMs domain. 308241362Sneel * And resurrect the 1:1 mapping for 'hpa' in 'host_domain'. 309241362Sneel */ 310241362Sneel iommu_remove_mapping(vm->iommu, seg->gpa + len, PAGE_SIZE); 311241362Sneel iommu_create_mapping(host_domain, hpa, hpa, PAGE_SIZE); 312241362Sneel 313241178Sneel vmm_mem_free(hpa, PAGE_SIZE); 314241178Sneel 315241178Sneel len += PAGE_SIZE; 316241178Sneel } 317241178Sneel 318241362Sneel /* 319241362Sneel * Invalidate cached translations associated with 'vm->iommu' since 320241362Sneel * we have now moved some pages from it. 321241362Sneel */ 322241362Sneel iommu_invalidate_tlb(vm->iommu); 323241362Sneel 324241178Sneel bzero(seg, sizeof(struct vm_memory_segment)); 325241178Sneel} 326241178Sneel 327221828Sgrehanvoid 328221828Sgrehanvm_destroy(struct vm *vm) 329221828Sgrehan{ 330221828Sgrehan int i; 331221828Sgrehan 332221828Sgrehan ppt_unassign_all(vm); 333221828Sgrehan 334221828Sgrehan for (i = 0; i < vm->num_mem_segs; i++) 335241178Sneel vm_free_mem_seg(vm, &vm->mem_segs[i]); 336221828Sgrehan 337241178Sneel vm->num_mem_segs = 0; 338241178Sneel 339221828Sgrehan for (i = 0; i < VM_MAXCPU; i++) 340221828Sgrehan vcpu_cleanup(&vm->vcpu[i]); 341221828Sgrehan 342221828Sgrehan iommu_destroy_domain(vm->iommu); 343221828Sgrehan 344221828Sgrehan VMCLEANUP(vm->cookie); 345221828Sgrehan 346221828Sgrehan free(vm, M_VM); 347221828Sgrehan} 348221828Sgrehan 349221828Sgrehanconst char * 350221828Sgrehanvm_name(struct vm *vm) 351221828Sgrehan{ 352221828Sgrehan return (vm->name); 353221828Sgrehan} 354221828Sgrehan 355221828Sgrehanint 356221828Sgrehanvm_map_mmio(struct vm *vm, vm_paddr_t gpa, size_t len, vm_paddr_t hpa) 357221828Sgrehan{ 358221828Sgrehan const boolean_t spok = TRUE; /* superpage mappings are ok */ 359221828Sgrehan 360241147Sneel return (VMMMAP_SET(vm->cookie, gpa, hpa, len, VM_MEMATTR_UNCACHEABLE, 361241147Sneel VM_PROT_RW, spok)); 362221828Sgrehan} 363221828Sgrehan 364221828Sgrehanint 365221828Sgrehanvm_unmap_mmio(struct vm *vm, vm_paddr_t gpa, size_t len) 366221828Sgrehan{ 367221828Sgrehan const boolean_t spok = TRUE; /* superpage mappings are ok */ 368221828Sgrehan 369241147Sneel return (VMMMAP_SET(vm->cookie, gpa, 0, len, 0, 370241147Sneel VM_PROT_NONE, spok)); 371221828Sgrehan} 372221828Sgrehan 373241041Sneel/* 374241041Sneel * Returns TRUE if 'gpa' is available for allocation and FALSE otherwise 375241041Sneel */ 376241041Sneelstatic boolean_t 377241041Sneelvm_gpa_available(struct vm *vm, vm_paddr_t gpa) 378241041Sneel{ 379241041Sneel int i; 380241041Sneel vm_paddr_t gpabase, gpalimit; 381241041Sneel 382241041Sneel if (gpa & PAGE_MASK) 383241041Sneel panic("vm_gpa_available: gpa (0x%016lx) not page aligned", gpa); 384241041Sneel 385241041Sneel for (i = 0; i < vm->num_mem_segs; i++) { 386241041Sneel gpabase = vm->mem_segs[i].gpa; 387241041Sneel gpalimit = gpabase + vm->mem_segs[i].len; 388241041Sneel if (gpa >= gpabase && gpa < gpalimit) 389241041Sneel return (FALSE); 390241041Sneel } 391241041Sneel 392241041Sneel return (TRUE); 393241041Sneel} 394241041Sneel 395221828Sgrehanint 396241041Sneelvm_malloc(struct vm *vm, vm_paddr_t gpa, size_t len) 397221828Sgrehan{ 398241041Sneel int error, available, allocated; 399241178Sneel struct vm_memory_segment *seg; 400241041Sneel vm_paddr_t g, hpa; 401241362Sneel void *host_domain; 402221828Sgrehan 403221828Sgrehan const boolean_t spok = TRUE; /* superpage mappings are ok */ 404241041Sneel 405241041Sneel if ((gpa & PAGE_MASK) || (len & PAGE_MASK) || len == 0) 406241041Sneel return (EINVAL); 407221828Sgrehan 408241041Sneel available = allocated = 0; 409241041Sneel g = gpa; 410241041Sneel while (g < gpa + len) { 411241041Sneel if (vm_gpa_available(vm, g)) 412241041Sneel available++; 413241041Sneel else 414241041Sneel allocated++; 415241041Sneel 416241041Sneel g += PAGE_SIZE; 417241041Sneel } 418241041Sneel 419221828Sgrehan /* 420241041Sneel * If there are some allocated and some available pages in the address 421241041Sneel * range then it is an error. 422221828Sgrehan */ 423241041Sneel if (allocated && available) 424241041Sneel return (EINVAL); 425221828Sgrehan 426241041Sneel /* 427241041Sneel * If the entire address range being requested has already been 428241041Sneel * allocated then there isn't anything more to do. 429241041Sneel */ 430241041Sneel if (allocated && available == 0) 431241041Sneel return (0); 432241041Sneel 433221828Sgrehan if (vm->num_mem_segs >= VM_MAX_MEMORY_SEGMENTS) 434221828Sgrehan return (E2BIG); 435221828Sgrehan 436241362Sneel host_domain = iommu_host_domain(); 437241362Sneel 438241178Sneel seg = &vm->mem_segs[vm->num_mem_segs]; 439221828Sgrehan 440241362Sneel error = 0; 441241178Sneel seg->gpa = gpa; 442241178Sneel seg->len = 0; 443241178Sneel while (seg->len < len) { 444241178Sneel hpa = vmm_mem_alloc(PAGE_SIZE); 445241178Sneel if (hpa == 0) { 446241178Sneel error = ENOMEM; 447241178Sneel break; 448241178Sneel } 449241178Sneel 450241178Sneel error = VMMMAP_SET(vm->cookie, gpa + seg->len, hpa, PAGE_SIZE, 451241178Sneel VM_MEMATTR_WRITE_BACK, VM_PROT_ALL, spok); 452241178Sneel if (error) 453241178Sneel break; 454241178Sneel 455241362Sneel /* 456241362Sneel * Remove the 1:1 mapping for 'hpa' from the 'host_domain'. 457241362Sneel * Add mapping for 'gpa + seg->len' to 'hpa' in the VMs domain. 458241362Sneel */ 459241362Sneel iommu_remove_mapping(host_domain, hpa, PAGE_SIZE); 460241178Sneel iommu_create_mapping(vm->iommu, gpa + seg->len, hpa, PAGE_SIZE); 461241178Sneel 462241178Sneel seg->len += PAGE_SIZE; 463241178Sneel } 464241178Sneel 465241362Sneel if (error) { 466241178Sneel vm_free_mem_seg(vm, seg); 467221828Sgrehan return (error); 468221828Sgrehan } 469221828Sgrehan 470241362Sneel /* 471241362Sneel * Invalidate cached translations associated with 'host_domain' since 472241362Sneel * we have now moved some pages from it. 473241362Sneel */ 474241362Sneel iommu_invalidate_tlb(host_domain); 475241362Sneel 476221828Sgrehan vm->num_mem_segs++; 477241041Sneel 478221828Sgrehan return (0); 479221828Sgrehan} 480221828Sgrehan 481221828Sgrehanvm_paddr_t 482221828Sgrehanvm_gpa2hpa(struct vm *vm, vm_paddr_t gpa, size_t len) 483221828Sgrehan{ 484241148Sneel vm_paddr_t nextpage; 485221828Sgrehan 486241148Sneel nextpage = rounddown(gpa + PAGE_SIZE, PAGE_SIZE); 487241148Sneel if (len > nextpage - gpa) 488241148Sneel panic("vm_gpa2hpa: invalid gpa/len: 0x%016lx/%lu", gpa, len); 489241148Sneel 490241147Sneel return (VMMMAP_GET(vm->cookie, gpa)); 491221828Sgrehan} 492221828Sgrehan 493221828Sgrehanint 494221828Sgrehanvm_gpabase2memseg(struct vm *vm, vm_paddr_t gpabase, 495221828Sgrehan struct vm_memory_segment *seg) 496221828Sgrehan{ 497221828Sgrehan int i; 498221828Sgrehan 499221828Sgrehan for (i = 0; i < vm->num_mem_segs; i++) { 500221828Sgrehan if (gpabase == vm->mem_segs[i].gpa) { 501221828Sgrehan *seg = vm->mem_segs[i]; 502221828Sgrehan return (0); 503221828Sgrehan } 504221828Sgrehan } 505221828Sgrehan return (-1); 506221828Sgrehan} 507221828Sgrehan 508221828Sgrehanint 509221828Sgrehanvm_get_register(struct vm *vm, int vcpu, int reg, uint64_t *retval) 510221828Sgrehan{ 511221828Sgrehan 512221828Sgrehan if (vcpu < 0 || vcpu >= VM_MAXCPU) 513221828Sgrehan return (EINVAL); 514221828Sgrehan 515221828Sgrehan if (reg >= VM_REG_LAST) 516221828Sgrehan return (EINVAL); 517221828Sgrehan 518221828Sgrehan return (VMGETREG(vm->cookie, vcpu, reg, retval)); 519221828Sgrehan} 520221828Sgrehan 521221828Sgrehanint 522221828Sgrehanvm_set_register(struct vm *vm, int vcpu, int reg, uint64_t val) 523221828Sgrehan{ 524221828Sgrehan 525221828Sgrehan if (vcpu < 0 || vcpu >= VM_MAXCPU) 526221828Sgrehan return (EINVAL); 527221828Sgrehan 528221828Sgrehan if (reg >= VM_REG_LAST) 529221828Sgrehan return (EINVAL); 530221828Sgrehan 531221828Sgrehan return (VMSETREG(vm->cookie, vcpu, reg, val)); 532221828Sgrehan} 533221828Sgrehan 534221828Sgrehanstatic boolean_t 535221828Sgrehanis_descriptor_table(int reg) 536221828Sgrehan{ 537221828Sgrehan 538221828Sgrehan switch (reg) { 539221828Sgrehan case VM_REG_GUEST_IDTR: 540221828Sgrehan case VM_REG_GUEST_GDTR: 541221828Sgrehan return (TRUE); 542221828Sgrehan default: 543221828Sgrehan return (FALSE); 544221828Sgrehan } 545221828Sgrehan} 546221828Sgrehan 547221828Sgrehanstatic boolean_t 548221828Sgrehanis_segment_register(int reg) 549221828Sgrehan{ 550221828Sgrehan 551221828Sgrehan switch (reg) { 552221828Sgrehan case VM_REG_GUEST_ES: 553221828Sgrehan case VM_REG_GUEST_CS: 554221828Sgrehan case VM_REG_GUEST_SS: 555221828Sgrehan case VM_REG_GUEST_DS: 556221828Sgrehan case VM_REG_GUEST_FS: 557221828Sgrehan case VM_REG_GUEST_GS: 558221828Sgrehan case VM_REG_GUEST_TR: 559221828Sgrehan case VM_REG_GUEST_LDTR: 560221828Sgrehan return (TRUE); 561221828Sgrehan default: 562221828Sgrehan return (FALSE); 563221828Sgrehan } 564221828Sgrehan} 565221828Sgrehan 566221828Sgrehanint 567221828Sgrehanvm_get_seg_desc(struct vm *vm, int vcpu, int reg, 568221828Sgrehan struct seg_desc *desc) 569221828Sgrehan{ 570221828Sgrehan 571221828Sgrehan if (vcpu < 0 || vcpu >= VM_MAXCPU) 572221828Sgrehan return (EINVAL); 573221828Sgrehan 574221828Sgrehan if (!is_segment_register(reg) && !is_descriptor_table(reg)) 575221828Sgrehan return (EINVAL); 576221828Sgrehan 577221828Sgrehan return (VMGETDESC(vm->cookie, vcpu, reg, desc)); 578221828Sgrehan} 579221828Sgrehan 580221828Sgrehanint 581221828Sgrehanvm_set_seg_desc(struct vm *vm, int vcpu, int reg, 582221828Sgrehan struct seg_desc *desc) 583221828Sgrehan{ 584221828Sgrehan if (vcpu < 0 || vcpu >= VM_MAXCPU) 585221828Sgrehan return (EINVAL); 586221828Sgrehan 587221828Sgrehan if (!is_segment_register(reg) && !is_descriptor_table(reg)) 588221828Sgrehan return (EINVAL); 589221828Sgrehan 590221828Sgrehan return (VMSETDESC(vm->cookie, vcpu, reg, desc)); 591221828Sgrehan} 592221828Sgrehan 593221828Sgrehanint 594221828Sgrehanvm_get_pinning(struct vm *vm, int vcpuid, int *cpuid) 595221828Sgrehan{ 596221828Sgrehan 597221828Sgrehan if (vcpuid < 0 || vcpuid >= VM_MAXCPU) 598221828Sgrehan return (EINVAL); 599221828Sgrehan 600221828Sgrehan *cpuid = VCPU_PINCPU(vm, vcpuid); 601221828Sgrehan 602221828Sgrehan return (0); 603221828Sgrehan} 604221828Sgrehan 605221828Sgrehanint 606221828Sgrehanvm_set_pinning(struct vm *vm, int vcpuid, int host_cpuid) 607221828Sgrehan{ 608221828Sgrehan struct thread *td; 609221828Sgrehan 610221828Sgrehan if (vcpuid < 0 || vcpuid >= VM_MAXCPU) 611221828Sgrehan return (EINVAL); 612221828Sgrehan 613221828Sgrehan td = curthread; /* XXXSMP only safe when muxing vcpus */ 614221828Sgrehan 615221828Sgrehan /* unpin */ 616221828Sgrehan if (host_cpuid < 0) { 617221828Sgrehan VCPU_UNPIN(vm, vcpuid); 618221828Sgrehan thread_lock(td); 619221828Sgrehan sched_unbind(td); 620221828Sgrehan thread_unlock(td); 621221828Sgrehan return (0); 622221828Sgrehan } 623221828Sgrehan 624221828Sgrehan if (CPU_ABSENT(host_cpuid)) 625221828Sgrehan return (EINVAL); 626221828Sgrehan 627221828Sgrehan /* 628221828Sgrehan * XXX we should check that 'host_cpuid' has not already been pinned 629221828Sgrehan * by another vm. 630221828Sgrehan */ 631221828Sgrehan thread_lock(td); 632221828Sgrehan sched_bind(td, host_cpuid); 633221828Sgrehan thread_unlock(td); 634221828Sgrehan VCPU_PIN(vm, vcpuid, host_cpuid); 635221828Sgrehan 636221828Sgrehan return (0); 637221828Sgrehan} 638221828Sgrehan 639221828Sgrehanstatic void 640221828Sgrehanrestore_guest_fpustate(struct vcpu *vcpu) 641221828Sgrehan{ 642221828Sgrehan 643234695Sgrehan /* flush host state to the pcb */ 644234695Sgrehan fpuexit(curthread); 645242122Sneel 646242122Sneel /* restore guest FPU state */ 647221828Sgrehan fpu_stop_emulating(); 648234695Sgrehan fpurestore(vcpu->guestfpu); 649242122Sneel 650242122Sneel /* 651242122Sneel * The FPU is now "dirty" with the guest's state so turn on emulation 652242122Sneel * to trap any access to the FPU by the host. 653242122Sneel */ 654242122Sneel fpu_start_emulating(); 655221828Sgrehan} 656221828Sgrehan 657221828Sgrehanstatic void 658221828Sgrehansave_guest_fpustate(struct vcpu *vcpu) 659221828Sgrehan{ 660221828Sgrehan 661242122Sneel if ((rcr0() & CR0_TS) == 0) 662242122Sneel panic("fpu emulation not enabled in host!"); 663242122Sneel 664242122Sneel /* save guest FPU state */ 665242122Sneel fpu_stop_emulating(); 666234695Sgrehan fpusave(vcpu->guestfpu); 667221828Sgrehan fpu_start_emulating(); 668221828Sgrehan} 669221828Sgrehan 670242065Sneelstatic VMM_STAT_DEFINE(VCPU_IDLE_TICKS, "number of ticks vcpu was idle"); 671242065Sneel 672221828Sgrehanint 673221828Sgrehanvm_run(struct vm *vm, struct vm_run *vmrun) 674221828Sgrehan{ 675242065Sneel int error, vcpuid, sleepticks, t; 676221828Sgrehan struct vcpu *vcpu; 677221828Sgrehan struct pcb *pcb; 678242065Sneel uint64_t tscval, rip; 679242065Sneel struct vm_exit *vme; 680221828Sgrehan 681221828Sgrehan vcpuid = vmrun->cpuid; 682221828Sgrehan 683221828Sgrehan if (vcpuid < 0 || vcpuid >= VM_MAXCPU) 684221828Sgrehan return (EINVAL); 685221828Sgrehan 686221828Sgrehan vcpu = &vm->vcpu[vcpuid]; 687242065Sneel vme = &vmrun->vm_exit; 688242065Sneel rip = vmrun->rip; 689242065Sneelrestart: 690221828Sgrehan critical_enter(); 691221828Sgrehan 692221828Sgrehan tscval = rdtsc(); 693221828Sgrehan 694221828Sgrehan pcb = PCPU_GET(curpcb); 695221914Sjhb set_pcb_flags(pcb, PCB_FULL_IRET); 696221828Sgrehan 697234695Sgrehan restore_guest_msrs(vm, vcpuid); 698221828Sgrehan restore_guest_fpustate(vcpu); 699241489Sneel 700241489Sneel vcpu->hostcpu = curcpu; 701242065Sneel error = VMRUN(vm->cookie, vcpuid, rip); 702241489Sneel vcpu->hostcpu = NOCPU; 703241489Sneel 704221828Sgrehan save_guest_fpustate(vcpu); 705221828Sgrehan restore_host_msrs(vm, vcpuid); 706221828Sgrehan 707221828Sgrehan vmm_stat_incr(vm, vcpuid, VCPU_TOTAL_RUNTIME, rdtsc() - tscval); 708221828Sgrehan 709240894Sneel /* copy the exit information */ 710242065Sneel bcopy(&vcpu->exitinfo, vme, sizeof(struct vm_exit)); 711240894Sneel 712221828Sgrehan critical_exit(); 713221828Sgrehan 714242065Sneel /* 715242065Sneel * Oblige the guest's desire to 'hlt' by sleeping until the vcpu 716242065Sneel * is ready to run. 717242065Sneel */ 718242065Sneel if (error == 0 && vme->exitcode == VM_EXITCODE_HLT) { 719242065Sneel vcpu_lock(vcpu); 720242065Sneel 721242065Sneel /* 722242065Sneel * Figure out the number of host ticks until the next apic 723242065Sneel * timer interrupt in the guest. 724242065Sneel */ 725242065Sneel sleepticks = lapic_timer_tick(vm, vcpuid); 726242065Sneel 727242065Sneel /* 728242065Sneel * If the guest local apic timer is disabled then sleep for 729242065Sneel * a long time but not forever. 730242065Sneel */ 731242065Sneel if (sleepticks < 0) 732242065Sneel sleepticks = hz; 733242065Sneel 734242065Sneel /* 735242065Sneel * Do a final check for pending NMI or interrupts before 736242065Sneel * really putting this thread to sleep. 737242065Sneel * 738242065Sneel * These interrupts could have happened any time after we 739242065Sneel * returned from VMRUN() and before we grabbed the vcpu lock. 740242065Sneel */ 741242065Sneel if (!vm_nmi_pending(vm, vcpuid) && 742242065Sneel lapic_pending_intr(vm, vcpuid) < 0) { 743242065Sneel if (sleepticks <= 0) 744242065Sneel panic("invalid sleepticks %d", sleepticks); 745242065Sneel t = ticks; 746242065Sneel msleep_spin(vcpu, &vcpu->mtx, "vmidle", sleepticks); 747242065Sneel vmm_stat_incr(vm, vcpuid, VCPU_IDLE_TICKS, ticks - t); 748242065Sneel } 749242065Sneel 750242065Sneel vcpu_unlock(vcpu); 751242065Sneel 752242065Sneel rip = vme->rip + vme->inst_length; 753242065Sneel goto restart; 754242065Sneel } 755242065Sneel 756221828Sgrehan return (error); 757221828Sgrehan} 758221828Sgrehan 759221828Sgrehanint 760221828Sgrehanvm_inject_event(struct vm *vm, int vcpuid, int type, 761221828Sgrehan int vector, uint32_t code, int code_valid) 762221828Sgrehan{ 763221828Sgrehan if (vcpuid < 0 || vcpuid >= VM_MAXCPU) 764221828Sgrehan return (EINVAL); 765221828Sgrehan 766221828Sgrehan if ((type > VM_EVENT_NONE && type < VM_EVENT_MAX) == 0) 767221828Sgrehan return (EINVAL); 768221828Sgrehan 769221828Sgrehan if (vector < 0 || vector > 255) 770221828Sgrehan return (EINVAL); 771221828Sgrehan 772221828Sgrehan return (VMINJECT(vm->cookie, vcpuid, type, vector, code, code_valid)); 773221828Sgrehan} 774221828Sgrehan 775242065Sneelstatic VMM_STAT_DEFINE(VCPU_NMI_COUNT, "number of NMIs delivered to vcpu"); 776241982Sneel 777221828Sgrehanint 778241982Sneelvm_inject_nmi(struct vm *vm, int vcpuid) 779221828Sgrehan{ 780241982Sneel struct vcpu *vcpu; 781221828Sgrehan 782241982Sneel if (vcpuid < 0 || vcpuid >= VM_MAXCPU) 783221828Sgrehan return (EINVAL); 784221828Sgrehan 785241982Sneel vcpu = &vm->vcpu[vcpuid]; 786241982Sneel 787241982Sneel vcpu->nmi_pending = 1; 788241982Sneel vm_interrupt_hostcpu(vm, vcpuid); 789241982Sneel return (0); 790221828Sgrehan} 791221828Sgrehan 792221828Sgrehanint 793241982Sneelvm_nmi_pending(struct vm *vm, int vcpuid) 794241982Sneel{ 795241982Sneel struct vcpu *vcpu; 796241982Sneel 797241982Sneel if (vcpuid < 0 || vcpuid >= VM_MAXCPU) 798241982Sneel panic("vm_nmi_pending: invalid vcpuid %d", vcpuid); 799241982Sneel 800241982Sneel vcpu = &vm->vcpu[vcpuid]; 801241982Sneel 802241982Sneel return (vcpu->nmi_pending); 803241982Sneel} 804241982Sneel 805241982Sneelvoid 806241982Sneelvm_nmi_clear(struct vm *vm, int vcpuid) 807241982Sneel{ 808241982Sneel struct vcpu *vcpu; 809241982Sneel 810241982Sneel if (vcpuid < 0 || vcpuid >= VM_MAXCPU) 811241982Sneel panic("vm_nmi_pending: invalid vcpuid %d", vcpuid); 812241982Sneel 813241982Sneel vcpu = &vm->vcpu[vcpuid]; 814241982Sneel 815241982Sneel if (vcpu->nmi_pending == 0) 816241982Sneel panic("vm_nmi_clear: inconsistent nmi_pending state"); 817241982Sneel 818241982Sneel vcpu->nmi_pending = 0; 819241982Sneel vmm_stat_incr(vm, vcpuid, VCPU_NMI_COUNT, 1); 820241982Sneel} 821241982Sneel 822241982Sneelint 823221828Sgrehanvm_get_capability(struct vm *vm, int vcpu, int type, int *retval) 824221828Sgrehan{ 825221828Sgrehan if (vcpu < 0 || vcpu >= VM_MAXCPU) 826221828Sgrehan return (EINVAL); 827221828Sgrehan 828221828Sgrehan if (type < 0 || type >= VM_CAP_MAX) 829221828Sgrehan return (EINVAL); 830221828Sgrehan 831221828Sgrehan return (VMGETCAP(vm->cookie, vcpu, type, retval)); 832221828Sgrehan} 833221828Sgrehan 834221828Sgrehanint 835221828Sgrehanvm_set_capability(struct vm *vm, int vcpu, int type, int val) 836221828Sgrehan{ 837221828Sgrehan if (vcpu < 0 || vcpu >= VM_MAXCPU) 838221828Sgrehan return (EINVAL); 839221828Sgrehan 840221828Sgrehan if (type < 0 || type >= VM_CAP_MAX) 841221828Sgrehan return (EINVAL); 842221828Sgrehan 843221828Sgrehan return (VMSETCAP(vm->cookie, vcpu, type, val)); 844221828Sgrehan} 845221828Sgrehan 846221828Sgrehanuint64_t * 847221828Sgrehanvm_guest_msrs(struct vm *vm, int cpu) 848221828Sgrehan{ 849221828Sgrehan return (vm->vcpu[cpu].guest_msrs); 850221828Sgrehan} 851221828Sgrehan 852221828Sgrehanstruct vlapic * 853221828Sgrehanvm_lapic(struct vm *vm, int cpu) 854221828Sgrehan{ 855221828Sgrehan return (vm->vcpu[cpu].vlapic); 856221828Sgrehan} 857221828Sgrehan 858221828Sgrehanboolean_t 859221828Sgrehanvmm_is_pptdev(int bus, int slot, int func) 860221828Sgrehan{ 861221828Sgrehan int found, b, s, f, n; 862221828Sgrehan char *val, *cp, *cp2; 863221828Sgrehan 864221828Sgrehan /* 865221828Sgrehan * setenv pptdevs "1/2/3 4/5/6 7/8/9 10/11/12" 866221828Sgrehan */ 867221828Sgrehan found = 0; 868221828Sgrehan cp = val = getenv("pptdevs"); 869221828Sgrehan while (cp != NULL && *cp != '\0') { 870221828Sgrehan if ((cp2 = strchr(cp, ' ')) != NULL) 871221828Sgrehan *cp2 = '\0'; 872221828Sgrehan 873221828Sgrehan n = sscanf(cp, "%d/%d/%d", &b, &s, &f); 874221828Sgrehan if (n == 3 && bus == b && slot == s && func == f) { 875221828Sgrehan found = 1; 876221828Sgrehan break; 877221828Sgrehan } 878221828Sgrehan 879221828Sgrehan if (cp2 != NULL) 880221828Sgrehan *cp2++ = ' '; 881221828Sgrehan 882221828Sgrehan cp = cp2; 883221828Sgrehan } 884221828Sgrehan freeenv(val); 885221828Sgrehan return (found); 886221828Sgrehan} 887221828Sgrehan 888221828Sgrehanvoid * 889221828Sgrehanvm_iommu_domain(struct vm *vm) 890221828Sgrehan{ 891221828Sgrehan 892221828Sgrehan return (vm->iommu); 893221828Sgrehan} 894221828Sgrehan 895241489Sneelint 896241489Sneelvcpu_set_state(struct vm *vm, int vcpuid, enum vcpu_state state) 897221828Sgrehan{ 898241489Sneel int error; 899221828Sgrehan struct vcpu *vcpu; 900221828Sgrehan 901221828Sgrehan if (vcpuid < 0 || vcpuid >= VM_MAXCPU) 902221828Sgrehan panic("vm_set_run_state: invalid vcpuid %d", vcpuid); 903221828Sgrehan 904221828Sgrehan vcpu = &vm->vcpu[vcpuid]; 905221828Sgrehan 906241489Sneel vcpu_lock(vcpu); 907241489Sneel 908241489Sneel /* 909241489Sneel * The following state transitions are allowed: 910241489Sneel * IDLE -> RUNNING -> IDLE 911241489Sneel * IDLE -> CANNOT_RUN -> IDLE 912241489Sneel */ 913241489Sneel if ((vcpu->state == VCPU_IDLE && state != VCPU_IDLE) || 914241489Sneel (vcpu->state != VCPU_IDLE && state == VCPU_IDLE)) { 915241489Sneel error = 0; 916241489Sneel vcpu->state = state; 917221828Sgrehan } else { 918241489Sneel error = EBUSY; 919221828Sgrehan } 920241489Sneel 921241489Sneel vcpu_unlock(vcpu); 922241489Sneel 923241489Sneel return (error); 924221828Sgrehan} 925221828Sgrehan 926241489Sneelenum vcpu_state 927241489Sneelvcpu_get_state(struct vm *vm, int vcpuid) 928221828Sgrehan{ 929221828Sgrehan struct vcpu *vcpu; 930241489Sneel enum vcpu_state state; 931221828Sgrehan 932221828Sgrehan if (vcpuid < 0 || vcpuid >= VM_MAXCPU) 933221828Sgrehan panic("vm_get_run_state: invalid vcpuid %d", vcpuid); 934221828Sgrehan 935221828Sgrehan vcpu = &vm->vcpu[vcpuid]; 936221828Sgrehan 937241489Sneel vcpu_lock(vcpu); 938241489Sneel state = vcpu->state; 939241489Sneel vcpu_unlock(vcpu); 940221828Sgrehan 941241489Sneel return (state); 942221828Sgrehan} 943221828Sgrehan 944221828Sgrehanvoid 945221828Sgrehanvm_activate_cpu(struct vm *vm, int vcpuid) 946221828Sgrehan{ 947221828Sgrehan 948221828Sgrehan if (vcpuid >= 0 && vcpuid < VM_MAXCPU) 949223621Sgrehan CPU_SET(vcpuid, &vm->active_cpus); 950221828Sgrehan} 951221828Sgrehan 952223621Sgrehancpuset_t 953221828Sgrehanvm_active_cpus(struct vm *vm) 954221828Sgrehan{ 955221828Sgrehan 956221828Sgrehan return (vm->active_cpus); 957221828Sgrehan} 958221828Sgrehan 959221828Sgrehanvoid * 960221828Sgrehanvcpu_stats(struct vm *vm, int vcpuid) 961221828Sgrehan{ 962221828Sgrehan 963221828Sgrehan return (vm->vcpu[vcpuid].stats); 964221828Sgrehan} 965240922Sneel 966240922Sneelint 967240922Sneelvm_get_x2apic_state(struct vm *vm, int vcpuid, enum x2apic_state *state) 968240922Sneel{ 969240922Sneel if (vcpuid < 0 || vcpuid >= VM_MAXCPU) 970240922Sneel return (EINVAL); 971240922Sneel 972240922Sneel *state = vm->vcpu[vcpuid].x2apic_state; 973240922Sneel 974240922Sneel return (0); 975240922Sneel} 976240922Sneel 977240922Sneelint 978240922Sneelvm_set_x2apic_state(struct vm *vm, int vcpuid, enum x2apic_state state) 979240922Sneel{ 980240922Sneel if (vcpuid < 0 || vcpuid >= VM_MAXCPU) 981240922Sneel return (EINVAL); 982240922Sneel 983240922Sneel if (state < 0 || state >= X2APIC_STATE_LAST) 984240922Sneel return (EINVAL); 985240922Sneel 986240922Sneel vm->vcpu[vcpuid].x2apic_state = state; 987240922Sneel 988240943Sneel vlapic_set_x2apic_state(vm, vcpuid, state); 989240943Sneel 990240922Sneel return (0); 991240922Sneel} 992241489Sneel 993241489Sneelvoid 994241489Sneelvm_interrupt_hostcpu(struct vm *vm, int vcpuid) 995241489Sneel{ 996241489Sneel int hostcpu; 997241489Sneel struct vcpu *vcpu; 998241489Sneel 999241489Sneel vcpu = &vm->vcpu[vcpuid]; 1000241489Sneel 1001242065Sneel vcpu_lock(vcpu); 1002241489Sneel hostcpu = vcpu->hostcpu; 1003242065Sneel if (hostcpu == NOCPU) { 1004242065Sneel /* 1005242065Sneel * If the vcpu is 'RUNNING' but without a valid 'hostcpu' then 1006242065Sneel * the host thread must be sleeping waiting for an event to 1007242065Sneel * kick the vcpu out of 'hlt'. 1008242065Sneel * 1009242065Sneel * XXX this is racy because the condition exists right before 1010242065Sneel * and after calling VMRUN() in vm_run(). The wakeup() is 1011242065Sneel * benign in this case. 1012242065Sneel */ 1013242065Sneel if (vcpu->state == VCPU_RUNNING) 1014242065Sneel wakeup_one(vcpu); 1015242065Sneel } else { 1016242065Sneel if (vcpu->state != VCPU_RUNNING) 1017242065Sneel panic("invalid vcpu state %d", vcpu->state); 1018242065Sneel if (hostcpu != curcpu) 1019242065Sneel ipi_cpu(hostcpu, vmm_ipinum); 1020242065Sneel } 1021242065Sneel vcpu_unlock(vcpu); 1022241489Sneel} 1023