vmm.c revision 242122
1221828Sgrehan/*- 2221828Sgrehan * Copyright (c) 2011 NetApp, Inc. 3221828Sgrehan * All rights reserved. 4221828Sgrehan * 5221828Sgrehan * Redistribution and use in source and binary forms, with or without 6221828Sgrehan * modification, are permitted provided that the following conditions 7221828Sgrehan * are met: 8221828Sgrehan * 1. Redistributions of source code must retain the above copyright 9221828Sgrehan * notice, this list of conditions and the following disclaimer. 10221828Sgrehan * 2. Redistributions in binary form must reproduce the above copyright 11221828Sgrehan * notice, this list of conditions and the following disclaimer in the 12221828Sgrehan * documentation and/or other materials provided with the distribution. 13221828Sgrehan * 14221828Sgrehan * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND 15221828Sgrehan * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16221828Sgrehan * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17221828Sgrehan * ARE DISCLAIMED. IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE 18221828Sgrehan * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19221828Sgrehan * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20221828Sgrehan * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21221828Sgrehan * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22221828Sgrehan * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23221828Sgrehan * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24221828Sgrehan * SUCH DAMAGE. 25221828Sgrehan * 26221828Sgrehan * $FreeBSD$ 27221828Sgrehan */ 28221828Sgrehan 29221828Sgrehan#include <sys/cdefs.h> 30221828Sgrehan__FBSDID("$FreeBSD$"); 31221828Sgrehan 32221828Sgrehan#include <sys/param.h> 33234695Sgrehan#include <sys/systm.h> 34221828Sgrehan#include <sys/kernel.h> 35221828Sgrehan#include <sys/module.h> 36221828Sgrehan#include <sys/sysctl.h> 37221828Sgrehan#include <sys/malloc.h> 38221828Sgrehan#include <sys/pcpu.h> 39221828Sgrehan#include <sys/lock.h> 40221828Sgrehan#include <sys/mutex.h> 41221828Sgrehan#include <sys/proc.h> 42221828Sgrehan#include <sys/sched.h> 43221828Sgrehan#include <sys/smp.h> 44221828Sgrehan#include <sys/systm.h> 45221828Sgrehan 46221828Sgrehan#include <vm/vm.h> 47221828Sgrehan 48221828Sgrehan#include <machine/vm.h> 49221828Sgrehan#include <machine/pcb.h> 50241489Sneel#include <machine/smp.h> 51221914Sjhb#include <x86/apicreg.h> 52221828Sgrehan 53221828Sgrehan#include <machine/vmm.h> 54221828Sgrehan#include "vmm_mem.h" 55221828Sgrehan#include "vmm_util.h" 56221828Sgrehan#include <machine/vmm_dev.h> 57221828Sgrehan#include "vlapic.h" 58221828Sgrehan#include "vmm_msr.h" 59221828Sgrehan#include "vmm_ipi.h" 60221828Sgrehan#include "vmm_stat.h" 61242065Sneel#include "vmm_lapic.h" 62221828Sgrehan 63221828Sgrehan#include "io/ppt.h" 64221828Sgrehan#include "io/iommu.h" 65221828Sgrehan 66221828Sgrehanstruct vlapic; 67221828Sgrehan 68221828Sgrehanstruct vcpu { 69221828Sgrehan int flags; 70241489Sneel enum vcpu_state state; 71241489Sneel struct mtx mtx; 72221828Sgrehan int pincpu; /* host cpuid this vcpu is bound to */ 73221828Sgrehan int hostcpu; /* host cpuid this vcpu last ran on */ 74221828Sgrehan uint64_t guest_msrs[VMM_MSR_NUM]; 75221828Sgrehan struct vlapic *vlapic; 76221828Sgrehan int vcpuid; 77234695Sgrehan struct savefpu *guestfpu; /* guest fpu state */ 78221828Sgrehan void *stats; 79240894Sneel struct vm_exit exitinfo; 80240922Sneel enum x2apic_state x2apic_state; 81241982Sneel int nmi_pending; 82221828Sgrehan}; 83221828Sgrehan#define VCPU_F_PINNED 0x0001 84221828Sgrehan 85221828Sgrehan#define VCPU_PINCPU(vm, vcpuid) \ 86221828Sgrehan ((vm->vcpu[vcpuid].flags & VCPU_F_PINNED) ? vm->vcpu[vcpuid].pincpu : -1) 87221828Sgrehan 88221828Sgrehan#define VCPU_UNPIN(vm, vcpuid) (vm->vcpu[vcpuid].flags &= ~VCPU_F_PINNED) 89221828Sgrehan 90221828Sgrehan#define VCPU_PIN(vm, vcpuid, host_cpuid) \ 91221828Sgrehando { \ 92221828Sgrehan vm->vcpu[vcpuid].flags |= VCPU_F_PINNED; \ 93221828Sgrehan vm->vcpu[vcpuid].pincpu = host_cpuid; \ 94221828Sgrehan} while(0) 95221828Sgrehan 96242065Sneel#define vcpu_lock_init(v) mtx_init(&((v)->mtx), "vcpu lock", 0, MTX_SPIN) 97242065Sneel#define vcpu_lock(v) mtx_lock_spin(&((v)->mtx)) 98242065Sneel#define vcpu_unlock(v) mtx_unlock_spin(&((v)->mtx)) 99241489Sneel 100221828Sgrehan#define VM_MAX_MEMORY_SEGMENTS 2 101221828Sgrehan 102221828Sgrehanstruct vm { 103221828Sgrehan void *cookie; /* processor-specific data */ 104221828Sgrehan void *iommu; /* iommu-specific data */ 105221828Sgrehan struct vcpu vcpu[VM_MAXCPU]; 106221828Sgrehan int num_mem_segs; 107221828Sgrehan struct vm_memory_segment mem_segs[VM_MAX_MEMORY_SEGMENTS]; 108221828Sgrehan char name[VM_MAX_NAMELEN]; 109221828Sgrehan 110221828Sgrehan /* 111223621Sgrehan * Set of active vcpus. 112221828Sgrehan * An active vcpu is one that has been started implicitly (BSP) or 113221828Sgrehan * explicitly (AP) by sending it a startup ipi. 114221828Sgrehan */ 115223621Sgrehan cpuset_t active_cpus; 116221828Sgrehan}; 117221828Sgrehan 118221828Sgrehanstatic struct vmm_ops *ops; 119221828Sgrehan#define VMM_INIT() (ops != NULL ? (*ops->init)() : 0) 120221828Sgrehan#define VMM_CLEANUP() (ops != NULL ? (*ops->cleanup)() : 0) 121221828Sgrehan 122221828Sgrehan#define VMINIT(vm) (ops != NULL ? (*ops->vminit)(vm): NULL) 123240894Sneel#define VMRUN(vmi, vcpu, rip) \ 124240894Sneel (ops != NULL ? (*ops->vmrun)(vmi, vcpu, rip) : ENXIO) 125221828Sgrehan#define VMCLEANUP(vmi) (ops != NULL ? (*ops->vmcleanup)(vmi) : NULL) 126241147Sneel#define VMMMAP_SET(vmi, gpa, hpa, len, attr, prot, spm) \ 127241147Sneel (ops != NULL ? \ 128241147Sneel (*ops->vmmmap_set)(vmi, gpa, hpa, len, attr, prot, spm) : \ 129241147Sneel ENXIO) 130241147Sneel#define VMMMAP_GET(vmi, gpa) \ 131241147Sneel (ops != NULL ? (*ops->vmmmap_get)(vmi, gpa) : ENXIO) 132221828Sgrehan#define VMGETREG(vmi, vcpu, num, retval) \ 133221828Sgrehan (ops != NULL ? (*ops->vmgetreg)(vmi, vcpu, num, retval) : ENXIO) 134221828Sgrehan#define VMSETREG(vmi, vcpu, num, val) \ 135221828Sgrehan (ops != NULL ? (*ops->vmsetreg)(vmi, vcpu, num, val) : ENXIO) 136221828Sgrehan#define VMGETDESC(vmi, vcpu, num, desc) \ 137221828Sgrehan (ops != NULL ? (*ops->vmgetdesc)(vmi, vcpu, num, desc) : ENXIO) 138221828Sgrehan#define VMSETDESC(vmi, vcpu, num, desc) \ 139221828Sgrehan (ops != NULL ? (*ops->vmsetdesc)(vmi, vcpu, num, desc) : ENXIO) 140221828Sgrehan#define VMINJECT(vmi, vcpu, type, vec, ec, ecv) \ 141221828Sgrehan (ops != NULL ? (*ops->vminject)(vmi, vcpu, type, vec, ec, ecv) : ENXIO) 142221828Sgrehan#define VMGETCAP(vmi, vcpu, num, retval) \ 143221828Sgrehan (ops != NULL ? (*ops->vmgetcap)(vmi, vcpu, num, retval) : ENXIO) 144221828Sgrehan#define VMSETCAP(vmi, vcpu, num, val) \ 145221828Sgrehan (ops != NULL ? (*ops->vmsetcap)(vmi, vcpu, num, val) : ENXIO) 146221828Sgrehan 147234695Sgrehan#define fpu_start_emulating() start_emulating() 148234695Sgrehan#define fpu_stop_emulating() stop_emulating() 149221828Sgrehan 150221828Sgrehanstatic MALLOC_DEFINE(M_VM, "vm", "vm"); 151221828SgrehanCTASSERT(VMM_MSR_NUM <= 64); /* msr_mask can keep track of up to 64 msrs */ 152221828Sgrehan 153221828Sgrehan/* statistics */ 154221828Sgrehanstatic VMM_STAT_DEFINE(VCPU_TOTAL_RUNTIME, "vcpu total runtime"); 155221828Sgrehan 156221828Sgrehanstatic void 157221828Sgrehanvcpu_cleanup(struct vcpu *vcpu) 158221828Sgrehan{ 159221828Sgrehan vlapic_cleanup(vcpu->vlapic); 160234695Sgrehan vmm_stat_free(vcpu->stats); 161234695Sgrehan fpu_save_area_free(vcpu->guestfpu); 162221828Sgrehan} 163221828Sgrehan 164221828Sgrehanstatic void 165221828Sgrehanvcpu_init(struct vm *vm, uint32_t vcpu_id) 166221828Sgrehan{ 167221828Sgrehan struct vcpu *vcpu; 168221828Sgrehan 169221828Sgrehan vcpu = &vm->vcpu[vcpu_id]; 170221828Sgrehan 171241489Sneel vcpu_lock_init(vcpu); 172241489Sneel vcpu->hostcpu = NOCPU; 173221828Sgrehan vcpu->vcpuid = vcpu_id; 174221828Sgrehan vcpu->vlapic = vlapic_init(vm, vcpu_id); 175240943Sneel vm_set_x2apic_state(vm, vcpu_id, X2APIC_ENABLED); 176234695Sgrehan vcpu->guestfpu = fpu_save_area_alloc(); 177234695Sgrehan fpu_save_area_reset(vcpu->guestfpu); 178221828Sgrehan vcpu->stats = vmm_stat_alloc(); 179221828Sgrehan} 180221828Sgrehan 181240894Sneelstruct vm_exit * 182240894Sneelvm_exitinfo(struct vm *vm, int cpuid) 183240894Sneel{ 184240894Sneel struct vcpu *vcpu; 185240894Sneel 186240894Sneel if (cpuid < 0 || cpuid >= VM_MAXCPU) 187240894Sneel panic("vm_exitinfo: invalid cpuid %d", cpuid); 188240894Sneel 189240894Sneel vcpu = &vm->vcpu[cpuid]; 190240894Sneel 191240894Sneel return (&vcpu->exitinfo); 192240894Sneel} 193240894Sneel 194221828Sgrehanstatic int 195221828Sgrehanvmm_init(void) 196221828Sgrehan{ 197221828Sgrehan int error; 198221828Sgrehan 199221828Sgrehan vmm_ipi_init(); 200221828Sgrehan 201221828Sgrehan error = vmm_mem_init(); 202221828Sgrehan if (error) 203221828Sgrehan return (error); 204221828Sgrehan 205221828Sgrehan if (vmm_is_intel()) 206221828Sgrehan ops = &vmm_ops_intel; 207221828Sgrehan else if (vmm_is_amd()) 208221828Sgrehan ops = &vmm_ops_amd; 209221828Sgrehan else 210221828Sgrehan return (ENXIO); 211221828Sgrehan 212221828Sgrehan vmm_msr_init(); 213221828Sgrehan 214221828Sgrehan return (VMM_INIT()); 215221828Sgrehan} 216221828Sgrehan 217221828Sgrehanstatic int 218221828Sgrehanvmm_handler(module_t mod, int what, void *arg) 219221828Sgrehan{ 220221828Sgrehan int error; 221221828Sgrehan 222221828Sgrehan switch (what) { 223221828Sgrehan case MOD_LOAD: 224221828Sgrehan vmmdev_init(); 225221828Sgrehan iommu_init(); 226221828Sgrehan error = vmm_init(); 227221828Sgrehan break; 228221828Sgrehan case MOD_UNLOAD: 229241454Sneel error = vmmdev_cleanup(); 230241454Sneel if (error == 0) { 231241454Sneel iommu_cleanup(); 232241454Sneel vmm_ipi_cleanup(); 233241454Sneel error = VMM_CLEANUP(); 234241454Sneel } 235221828Sgrehan break; 236221828Sgrehan default: 237221828Sgrehan error = 0; 238221828Sgrehan break; 239221828Sgrehan } 240221828Sgrehan return (error); 241221828Sgrehan} 242221828Sgrehan 243221828Sgrehanstatic moduledata_t vmm_kmod = { 244221828Sgrehan "vmm", 245221828Sgrehan vmm_handler, 246221828Sgrehan NULL 247221828Sgrehan}; 248221828Sgrehan 249221828Sgrehan/* 250221828Sgrehan * Execute the module load handler after the pci passthru driver has had 251221828Sgrehan * a chance to claim devices. We need this information at the time we do 252221828Sgrehan * iommu initialization. 253221828Sgrehan */ 254221828SgrehanDECLARE_MODULE(vmm, vmm_kmod, SI_SUB_CONFIGURE + 1, SI_ORDER_ANY); 255221828SgrehanMODULE_VERSION(vmm, 1); 256221828Sgrehan 257221828SgrehanSYSCTL_NODE(_hw, OID_AUTO, vmm, CTLFLAG_RW, NULL, NULL); 258221828Sgrehan 259221828Sgrehanstruct vm * 260221828Sgrehanvm_create(const char *name) 261221828Sgrehan{ 262221828Sgrehan int i; 263221828Sgrehan struct vm *vm; 264221828Sgrehan vm_paddr_t maxaddr; 265221828Sgrehan 266221828Sgrehan const int BSP = 0; 267221828Sgrehan 268221828Sgrehan if (name == NULL || strlen(name) >= VM_MAX_NAMELEN) 269221828Sgrehan return (NULL); 270221828Sgrehan 271221828Sgrehan vm = malloc(sizeof(struct vm), M_VM, M_WAITOK | M_ZERO); 272221828Sgrehan strcpy(vm->name, name); 273221828Sgrehan vm->cookie = VMINIT(vm); 274221828Sgrehan 275221828Sgrehan for (i = 0; i < VM_MAXCPU; i++) { 276221828Sgrehan vcpu_init(vm, i); 277221828Sgrehan guest_msrs_init(vm, i); 278221828Sgrehan } 279221828Sgrehan 280221828Sgrehan maxaddr = vmm_mem_maxaddr(); 281221828Sgrehan vm->iommu = iommu_create_domain(maxaddr); 282221828Sgrehan vm_activate_cpu(vm, BSP); 283221828Sgrehan 284221828Sgrehan return (vm); 285221828Sgrehan} 286221828Sgrehan 287241178Sneelstatic void 288241178Sneelvm_free_mem_seg(struct vm *vm, struct vm_memory_segment *seg) 289241178Sneel{ 290241178Sneel size_t len; 291241178Sneel vm_paddr_t hpa; 292241362Sneel void *host_domain; 293241178Sneel 294241362Sneel host_domain = iommu_host_domain(); 295241362Sneel 296241178Sneel len = 0; 297241178Sneel while (len < seg->len) { 298241178Sneel hpa = vm_gpa2hpa(vm, seg->gpa + len, PAGE_SIZE); 299241178Sneel if (hpa == (vm_paddr_t)-1) { 300241178Sneel panic("vm_free_mem_segs: cannot free hpa " 301241178Sneel "associated with gpa 0x%016lx", seg->gpa + len); 302241178Sneel } 303241178Sneel 304241362Sneel /* 305241362Sneel * Remove the 'gpa' to 'hpa' mapping in VMs domain. 306241362Sneel * And resurrect the 1:1 mapping for 'hpa' in 'host_domain'. 307241362Sneel */ 308241362Sneel iommu_remove_mapping(vm->iommu, seg->gpa + len, PAGE_SIZE); 309241362Sneel iommu_create_mapping(host_domain, hpa, hpa, PAGE_SIZE); 310241362Sneel 311241178Sneel vmm_mem_free(hpa, PAGE_SIZE); 312241178Sneel 313241178Sneel len += PAGE_SIZE; 314241178Sneel } 315241178Sneel 316241362Sneel /* 317241362Sneel * Invalidate cached translations associated with 'vm->iommu' since 318241362Sneel * we have now moved some pages from it. 319241362Sneel */ 320241362Sneel iommu_invalidate_tlb(vm->iommu); 321241362Sneel 322241178Sneel bzero(seg, sizeof(struct vm_memory_segment)); 323241178Sneel} 324241178Sneel 325221828Sgrehanvoid 326221828Sgrehanvm_destroy(struct vm *vm) 327221828Sgrehan{ 328221828Sgrehan int i; 329221828Sgrehan 330221828Sgrehan ppt_unassign_all(vm); 331221828Sgrehan 332221828Sgrehan for (i = 0; i < vm->num_mem_segs; i++) 333241178Sneel vm_free_mem_seg(vm, &vm->mem_segs[i]); 334221828Sgrehan 335241178Sneel vm->num_mem_segs = 0; 336241178Sneel 337221828Sgrehan for (i = 0; i < VM_MAXCPU; i++) 338221828Sgrehan vcpu_cleanup(&vm->vcpu[i]); 339221828Sgrehan 340221828Sgrehan iommu_destroy_domain(vm->iommu); 341221828Sgrehan 342221828Sgrehan VMCLEANUP(vm->cookie); 343221828Sgrehan 344221828Sgrehan free(vm, M_VM); 345221828Sgrehan} 346221828Sgrehan 347221828Sgrehanconst char * 348221828Sgrehanvm_name(struct vm *vm) 349221828Sgrehan{ 350221828Sgrehan return (vm->name); 351221828Sgrehan} 352221828Sgrehan 353221828Sgrehanint 354221828Sgrehanvm_map_mmio(struct vm *vm, vm_paddr_t gpa, size_t len, vm_paddr_t hpa) 355221828Sgrehan{ 356221828Sgrehan const boolean_t spok = TRUE; /* superpage mappings are ok */ 357221828Sgrehan 358241147Sneel return (VMMMAP_SET(vm->cookie, gpa, hpa, len, VM_MEMATTR_UNCACHEABLE, 359241147Sneel VM_PROT_RW, spok)); 360221828Sgrehan} 361221828Sgrehan 362221828Sgrehanint 363221828Sgrehanvm_unmap_mmio(struct vm *vm, vm_paddr_t gpa, size_t len) 364221828Sgrehan{ 365221828Sgrehan const boolean_t spok = TRUE; /* superpage mappings are ok */ 366221828Sgrehan 367241147Sneel return (VMMMAP_SET(vm->cookie, gpa, 0, len, 0, 368241147Sneel VM_PROT_NONE, spok)); 369221828Sgrehan} 370221828Sgrehan 371241041Sneel/* 372241041Sneel * Returns TRUE if 'gpa' is available for allocation and FALSE otherwise 373241041Sneel */ 374241041Sneelstatic boolean_t 375241041Sneelvm_gpa_available(struct vm *vm, vm_paddr_t gpa) 376241041Sneel{ 377241041Sneel int i; 378241041Sneel vm_paddr_t gpabase, gpalimit; 379241041Sneel 380241041Sneel if (gpa & PAGE_MASK) 381241041Sneel panic("vm_gpa_available: gpa (0x%016lx) not page aligned", gpa); 382241041Sneel 383241041Sneel for (i = 0; i < vm->num_mem_segs; i++) { 384241041Sneel gpabase = vm->mem_segs[i].gpa; 385241041Sneel gpalimit = gpabase + vm->mem_segs[i].len; 386241041Sneel if (gpa >= gpabase && gpa < gpalimit) 387241041Sneel return (FALSE); 388241041Sneel } 389241041Sneel 390241041Sneel return (TRUE); 391241041Sneel} 392241041Sneel 393221828Sgrehanint 394241041Sneelvm_malloc(struct vm *vm, vm_paddr_t gpa, size_t len) 395221828Sgrehan{ 396241041Sneel int error, available, allocated; 397241178Sneel struct vm_memory_segment *seg; 398241041Sneel vm_paddr_t g, hpa; 399241362Sneel void *host_domain; 400221828Sgrehan 401221828Sgrehan const boolean_t spok = TRUE; /* superpage mappings are ok */ 402241041Sneel 403241041Sneel if ((gpa & PAGE_MASK) || (len & PAGE_MASK) || len == 0) 404241041Sneel return (EINVAL); 405221828Sgrehan 406241041Sneel available = allocated = 0; 407241041Sneel g = gpa; 408241041Sneel while (g < gpa + len) { 409241041Sneel if (vm_gpa_available(vm, g)) 410241041Sneel available++; 411241041Sneel else 412241041Sneel allocated++; 413241041Sneel 414241041Sneel g += PAGE_SIZE; 415241041Sneel } 416241041Sneel 417221828Sgrehan /* 418241041Sneel * If there are some allocated and some available pages in the address 419241041Sneel * range then it is an error. 420221828Sgrehan */ 421241041Sneel if (allocated && available) 422241041Sneel return (EINVAL); 423221828Sgrehan 424241041Sneel /* 425241041Sneel * If the entire address range being requested has already been 426241041Sneel * allocated then there isn't anything more to do. 427241041Sneel */ 428241041Sneel if (allocated && available == 0) 429241041Sneel return (0); 430241041Sneel 431221828Sgrehan if (vm->num_mem_segs >= VM_MAX_MEMORY_SEGMENTS) 432221828Sgrehan return (E2BIG); 433221828Sgrehan 434241362Sneel host_domain = iommu_host_domain(); 435241362Sneel 436241178Sneel seg = &vm->mem_segs[vm->num_mem_segs]; 437221828Sgrehan 438241362Sneel error = 0; 439241178Sneel seg->gpa = gpa; 440241178Sneel seg->len = 0; 441241178Sneel while (seg->len < len) { 442241178Sneel hpa = vmm_mem_alloc(PAGE_SIZE); 443241178Sneel if (hpa == 0) { 444241178Sneel error = ENOMEM; 445241178Sneel break; 446241178Sneel } 447241178Sneel 448241178Sneel error = VMMMAP_SET(vm->cookie, gpa + seg->len, hpa, PAGE_SIZE, 449241178Sneel VM_MEMATTR_WRITE_BACK, VM_PROT_ALL, spok); 450241178Sneel if (error) 451241178Sneel break; 452241178Sneel 453241362Sneel /* 454241362Sneel * Remove the 1:1 mapping for 'hpa' from the 'host_domain'. 455241362Sneel * Add mapping for 'gpa + seg->len' to 'hpa' in the VMs domain. 456241362Sneel */ 457241362Sneel iommu_remove_mapping(host_domain, hpa, PAGE_SIZE); 458241178Sneel iommu_create_mapping(vm->iommu, gpa + seg->len, hpa, PAGE_SIZE); 459241178Sneel 460241178Sneel seg->len += PAGE_SIZE; 461241178Sneel } 462241178Sneel 463241362Sneel if (error) { 464241178Sneel vm_free_mem_seg(vm, seg); 465221828Sgrehan return (error); 466221828Sgrehan } 467221828Sgrehan 468241362Sneel /* 469241362Sneel * Invalidate cached translations associated with 'host_domain' since 470241362Sneel * we have now moved some pages from it. 471241362Sneel */ 472241362Sneel iommu_invalidate_tlb(host_domain); 473241362Sneel 474221828Sgrehan vm->num_mem_segs++; 475241041Sneel 476221828Sgrehan return (0); 477221828Sgrehan} 478221828Sgrehan 479221828Sgrehanvm_paddr_t 480221828Sgrehanvm_gpa2hpa(struct vm *vm, vm_paddr_t gpa, size_t len) 481221828Sgrehan{ 482241148Sneel vm_paddr_t nextpage; 483221828Sgrehan 484241148Sneel nextpage = rounddown(gpa + PAGE_SIZE, PAGE_SIZE); 485241148Sneel if (len > nextpage - gpa) 486241148Sneel panic("vm_gpa2hpa: invalid gpa/len: 0x%016lx/%lu", gpa, len); 487241148Sneel 488241147Sneel return (VMMMAP_GET(vm->cookie, gpa)); 489221828Sgrehan} 490221828Sgrehan 491221828Sgrehanint 492221828Sgrehanvm_gpabase2memseg(struct vm *vm, vm_paddr_t gpabase, 493221828Sgrehan struct vm_memory_segment *seg) 494221828Sgrehan{ 495221828Sgrehan int i; 496221828Sgrehan 497221828Sgrehan for (i = 0; i < vm->num_mem_segs; i++) { 498221828Sgrehan if (gpabase == vm->mem_segs[i].gpa) { 499221828Sgrehan *seg = vm->mem_segs[i]; 500221828Sgrehan return (0); 501221828Sgrehan } 502221828Sgrehan } 503221828Sgrehan return (-1); 504221828Sgrehan} 505221828Sgrehan 506221828Sgrehanint 507221828Sgrehanvm_get_register(struct vm *vm, int vcpu, int reg, uint64_t *retval) 508221828Sgrehan{ 509221828Sgrehan 510221828Sgrehan if (vcpu < 0 || vcpu >= VM_MAXCPU) 511221828Sgrehan return (EINVAL); 512221828Sgrehan 513221828Sgrehan if (reg >= VM_REG_LAST) 514221828Sgrehan return (EINVAL); 515221828Sgrehan 516221828Sgrehan return (VMGETREG(vm->cookie, vcpu, reg, retval)); 517221828Sgrehan} 518221828Sgrehan 519221828Sgrehanint 520221828Sgrehanvm_set_register(struct vm *vm, int vcpu, int reg, uint64_t val) 521221828Sgrehan{ 522221828Sgrehan 523221828Sgrehan if (vcpu < 0 || vcpu >= VM_MAXCPU) 524221828Sgrehan return (EINVAL); 525221828Sgrehan 526221828Sgrehan if (reg >= VM_REG_LAST) 527221828Sgrehan return (EINVAL); 528221828Sgrehan 529221828Sgrehan return (VMSETREG(vm->cookie, vcpu, reg, val)); 530221828Sgrehan} 531221828Sgrehan 532221828Sgrehanstatic boolean_t 533221828Sgrehanis_descriptor_table(int reg) 534221828Sgrehan{ 535221828Sgrehan 536221828Sgrehan switch (reg) { 537221828Sgrehan case VM_REG_GUEST_IDTR: 538221828Sgrehan case VM_REG_GUEST_GDTR: 539221828Sgrehan return (TRUE); 540221828Sgrehan default: 541221828Sgrehan return (FALSE); 542221828Sgrehan } 543221828Sgrehan} 544221828Sgrehan 545221828Sgrehanstatic boolean_t 546221828Sgrehanis_segment_register(int reg) 547221828Sgrehan{ 548221828Sgrehan 549221828Sgrehan switch (reg) { 550221828Sgrehan case VM_REG_GUEST_ES: 551221828Sgrehan case VM_REG_GUEST_CS: 552221828Sgrehan case VM_REG_GUEST_SS: 553221828Sgrehan case VM_REG_GUEST_DS: 554221828Sgrehan case VM_REG_GUEST_FS: 555221828Sgrehan case VM_REG_GUEST_GS: 556221828Sgrehan case VM_REG_GUEST_TR: 557221828Sgrehan case VM_REG_GUEST_LDTR: 558221828Sgrehan return (TRUE); 559221828Sgrehan default: 560221828Sgrehan return (FALSE); 561221828Sgrehan } 562221828Sgrehan} 563221828Sgrehan 564221828Sgrehanint 565221828Sgrehanvm_get_seg_desc(struct vm *vm, int vcpu, int reg, 566221828Sgrehan struct seg_desc *desc) 567221828Sgrehan{ 568221828Sgrehan 569221828Sgrehan if (vcpu < 0 || vcpu >= VM_MAXCPU) 570221828Sgrehan return (EINVAL); 571221828Sgrehan 572221828Sgrehan if (!is_segment_register(reg) && !is_descriptor_table(reg)) 573221828Sgrehan return (EINVAL); 574221828Sgrehan 575221828Sgrehan return (VMGETDESC(vm->cookie, vcpu, reg, desc)); 576221828Sgrehan} 577221828Sgrehan 578221828Sgrehanint 579221828Sgrehanvm_set_seg_desc(struct vm *vm, int vcpu, int reg, 580221828Sgrehan struct seg_desc *desc) 581221828Sgrehan{ 582221828Sgrehan if (vcpu < 0 || vcpu >= VM_MAXCPU) 583221828Sgrehan return (EINVAL); 584221828Sgrehan 585221828Sgrehan if (!is_segment_register(reg) && !is_descriptor_table(reg)) 586221828Sgrehan return (EINVAL); 587221828Sgrehan 588221828Sgrehan return (VMSETDESC(vm->cookie, vcpu, reg, desc)); 589221828Sgrehan} 590221828Sgrehan 591221828Sgrehanint 592221828Sgrehanvm_get_pinning(struct vm *vm, int vcpuid, int *cpuid) 593221828Sgrehan{ 594221828Sgrehan 595221828Sgrehan if (vcpuid < 0 || vcpuid >= VM_MAXCPU) 596221828Sgrehan return (EINVAL); 597221828Sgrehan 598221828Sgrehan *cpuid = VCPU_PINCPU(vm, vcpuid); 599221828Sgrehan 600221828Sgrehan return (0); 601221828Sgrehan} 602221828Sgrehan 603221828Sgrehanint 604221828Sgrehanvm_set_pinning(struct vm *vm, int vcpuid, int host_cpuid) 605221828Sgrehan{ 606221828Sgrehan struct thread *td; 607221828Sgrehan 608221828Sgrehan if (vcpuid < 0 || vcpuid >= VM_MAXCPU) 609221828Sgrehan return (EINVAL); 610221828Sgrehan 611221828Sgrehan td = curthread; /* XXXSMP only safe when muxing vcpus */ 612221828Sgrehan 613221828Sgrehan /* unpin */ 614221828Sgrehan if (host_cpuid < 0) { 615221828Sgrehan VCPU_UNPIN(vm, vcpuid); 616221828Sgrehan thread_lock(td); 617221828Sgrehan sched_unbind(td); 618221828Sgrehan thread_unlock(td); 619221828Sgrehan return (0); 620221828Sgrehan } 621221828Sgrehan 622221828Sgrehan if (CPU_ABSENT(host_cpuid)) 623221828Sgrehan return (EINVAL); 624221828Sgrehan 625221828Sgrehan /* 626221828Sgrehan * XXX we should check that 'host_cpuid' has not already been pinned 627221828Sgrehan * by another vm. 628221828Sgrehan */ 629221828Sgrehan thread_lock(td); 630221828Sgrehan sched_bind(td, host_cpuid); 631221828Sgrehan thread_unlock(td); 632221828Sgrehan VCPU_PIN(vm, vcpuid, host_cpuid); 633221828Sgrehan 634221828Sgrehan return (0); 635221828Sgrehan} 636221828Sgrehan 637221828Sgrehanstatic void 638221828Sgrehanrestore_guest_fpustate(struct vcpu *vcpu) 639221828Sgrehan{ 640221828Sgrehan 641234695Sgrehan /* flush host state to the pcb */ 642234695Sgrehan fpuexit(curthread); 643242122Sneel 644242122Sneel /* restore guest FPU state */ 645221828Sgrehan fpu_stop_emulating(); 646234695Sgrehan fpurestore(vcpu->guestfpu); 647242122Sneel 648242122Sneel /* 649242122Sneel * The FPU is now "dirty" with the guest's state so turn on emulation 650242122Sneel * to trap any access to the FPU by the host. 651242122Sneel */ 652242122Sneel fpu_start_emulating(); 653221828Sgrehan} 654221828Sgrehan 655221828Sgrehanstatic void 656221828Sgrehansave_guest_fpustate(struct vcpu *vcpu) 657221828Sgrehan{ 658221828Sgrehan 659242122Sneel if ((rcr0() & CR0_TS) == 0) 660242122Sneel panic("fpu emulation not enabled in host!"); 661242122Sneel 662242122Sneel /* save guest FPU state */ 663242122Sneel fpu_stop_emulating(); 664234695Sgrehan fpusave(vcpu->guestfpu); 665221828Sgrehan fpu_start_emulating(); 666221828Sgrehan} 667221828Sgrehan 668242065Sneelstatic VMM_STAT_DEFINE(VCPU_IDLE_TICKS, "number of ticks vcpu was idle"); 669242065Sneel 670221828Sgrehanint 671221828Sgrehanvm_run(struct vm *vm, struct vm_run *vmrun) 672221828Sgrehan{ 673242065Sneel int error, vcpuid, sleepticks, t; 674221828Sgrehan struct vcpu *vcpu; 675221828Sgrehan struct pcb *pcb; 676242065Sneel uint64_t tscval, rip; 677242065Sneel struct vm_exit *vme; 678221828Sgrehan 679221828Sgrehan vcpuid = vmrun->cpuid; 680221828Sgrehan 681221828Sgrehan if (vcpuid < 0 || vcpuid >= VM_MAXCPU) 682221828Sgrehan return (EINVAL); 683221828Sgrehan 684221828Sgrehan vcpu = &vm->vcpu[vcpuid]; 685242065Sneel vme = &vmrun->vm_exit; 686242065Sneel rip = vmrun->rip; 687242065Sneelrestart: 688221828Sgrehan critical_enter(); 689221828Sgrehan 690221828Sgrehan tscval = rdtsc(); 691221828Sgrehan 692221828Sgrehan pcb = PCPU_GET(curpcb); 693221914Sjhb set_pcb_flags(pcb, PCB_FULL_IRET); 694221828Sgrehan 695234695Sgrehan restore_guest_msrs(vm, vcpuid); 696221828Sgrehan restore_guest_fpustate(vcpu); 697241489Sneel 698241489Sneel vcpu->hostcpu = curcpu; 699242065Sneel error = VMRUN(vm->cookie, vcpuid, rip); 700241489Sneel vcpu->hostcpu = NOCPU; 701241489Sneel 702221828Sgrehan save_guest_fpustate(vcpu); 703221828Sgrehan restore_host_msrs(vm, vcpuid); 704221828Sgrehan 705221828Sgrehan vmm_stat_incr(vm, vcpuid, VCPU_TOTAL_RUNTIME, rdtsc() - tscval); 706221828Sgrehan 707240894Sneel /* copy the exit information */ 708242065Sneel bcopy(&vcpu->exitinfo, vme, sizeof(struct vm_exit)); 709240894Sneel 710221828Sgrehan critical_exit(); 711221828Sgrehan 712242065Sneel /* 713242065Sneel * Oblige the guest's desire to 'hlt' by sleeping until the vcpu 714242065Sneel * is ready to run. 715242065Sneel */ 716242065Sneel if (error == 0 && vme->exitcode == VM_EXITCODE_HLT) { 717242065Sneel vcpu_lock(vcpu); 718242065Sneel 719242065Sneel /* 720242065Sneel * Figure out the number of host ticks until the next apic 721242065Sneel * timer interrupt in the guest. 722242065Sneel */ 723242065Sneel sleepticks = lapic_timer_tick(vm, vcpuid); 724242065Sneel 725242065Sneel /* 726242065Sneel * If the guest local apic timer is disabled then sleep for 727242065Sneel * a long time but not forever. 728242065Sneel */ 729242065Sneel if (sleepticks < 0) 730242065Sneel sleepticks = hz; 731242065Sneel 732242065Sneel /* 733242065Sneel * Do a final check for pending NMI or interrupts before 734242065Sneel * really putting this thread to sleep. 735242065Sneel * 736242065Sneel * These interrupts could have happened any time after we 737242065Sneel * returned from VMRUN() and before we grabbed the vcpu lock. 738242065Sneel */ 739242065Sneel if (!vm_nmi_pending(vm, vcpuid) && 740242065Sneel lapic_pending_intr(vm, vcpuid) < 0) { 741242065Sneel if (sleepticks <= 0) 742242065Sneel panic("invalid sleepticks %d", sleepticks); 743242065Sneel t = ticks; 744242065Sneel msleep_spin(vcpu, &vcpu->mtx, "vmidle", sleepticks); 745242065Sneel vmm_stat_incr(vm, vcpuid, VCPU_IDLE_TICKS, ticks - t); 746242065Sneel } 747242065Sneel 748242065Sneel vcpu_unlock(vcpu); 749242065Sneel 750242065Sneel rip = vme->rip + vme->inst_length; 751242065Sneel goto restart; 752242065Sneel } 753242065Sneel 754221828Sgrehan return (error); 755221828Sgrehan} 756221828Sgrehan 757221828Sgrehanint 758221828Sgrehanvm_inject_event(struct vm *vm, int vcpuid, int type, 759221828Sgrehan int vector, uint32_t code, int code_valid) 760221828Sgrehan{ 761221828Sgrehan if (vcpuid < 0 || vcpuid >= VM_MAXCPU) 762221828Sgrehan return (EINVAL); 763221828Sgrehan 764221828Sgrehan if ((type > VM_EVENT_NONE && type < VM_EVENT_MAX) == 0) 765221828Sgrehan return (EINVAL); 766221828Sgrehan 767221828Sgrehan if (vector < 0 || vector > 255) 768221828Sgrehan return (EINVAL); 769221828Sgrehan 770221828Sgrehan return (VMINJECT(vm->cookie, vcpuid, type, vector, code, code_valid)); 771221828Sgrehan} 772221828Sgrehan 773242065Sneelstatic VMM_STAT_DEFINE(VCPU_NMI_COUNT, "number of NMIs delivered to vcpu"); 774241982Sneel 775221828Sgrehanint 776241982Sneelvm_inject_nmi(struct vm *vm, int vcpuid) 777221828Sgrehan{ 778241982Sneel struct vcpu *vcpu; 779221828Sgrehan 780241982Sneel if (vcpuid < 0 || vcpuid >= VM_MAXCPU) 781221828Sgrehan return (EINVAL); 782221828Sgrehan 783241982Sneel vcpu = &vm->vcpu[vcpuid]; 784241982Sneel 785241982Sneel vcpu->nmi_pending = 1; 786241982Sneel vm_interrupt_hostcpu(vm, vcpuid); 787241982Sneel return (0); 788221828Sgrehan} 789221828Sgrehan 790221828Sgrehanint 791241982Sneelvm_nmi_pending(struct vm *vm, int vcpuid) 792241982Sneel{ 793241982Sneel struct vcpu *vcpu; 794241982Sneel 795241982Sneel if (vcpuid < 0 || vcpuid >= VM_MAXCPU) 796241982Sneel panic("vm_nmi_pending: invalid vcpuid %d", vcpuid); 797241982Sneel 798241982Sneel vcpu = &vm->vcpu[vcpuid]; 799241982Sneel 800241982Sneel return (vcpu->nmi_pending); 801241982Sneel} 802241982Sneel 803241982Sneelvoid 804241982Sneelvm_nmi_clear(struct vm *vm, int vcpuid) 805241982Sneel{ 806241982Sneel struct vcpu *vcpu; 807241982Sneel 808241982Sneel if (vcpuid < 0 || vcpuid >= VM_MAXCPU) 809241982Sneel panic("vm_nmi_pending: invalid vcpuid %d", vcpuid); 810241982Sneel 811241982Sneel vcpu = &vm->vcpu[vcpuid]; 812241982Sneel 813241982Sneel if (vcpu->nmi_pending == 0) 814241982Sneel panic("vm_nmi_clear: inconsistent nmi_pending state"); 815241982Sneel 816241982Sneel vcpu->nmi_pending = 0; 817241982Sneel vmm_stat_incr(vm, vcpuid, VCPU_NMI_COUNT, 1); 818241982Sneel} 819241982Sneel 820241982Sneelint 821221828Sgrehanvm_get_capability(struct vm *vm, int vcpu, int type, int *retval) 822221828Sgrehan{ 823221828Sgrehan if (vcpu < 0 || vcpu >= VM_MAXCPU) 824221828Sgrehan return (EINVAL); 825221828Sgrehan 826221828Sgrehan if (type < 0 || type >= VM_CAP_MAX) 827221828Sgrehan return (EINVAL); 828221828Sgrehan 829221828Sgrehan return (VMGETCAP(vm->cookie, vcpu, type, retval)); 830221828Sgrehan} 831221828Sgrehan 832221828Sgrehanint 833221828Sgrehanvm_set_capability(struct vm *vm, int vcpu, int type, int val) 834221828Sgrehan{ 835221828Sgrehan if (vcpu < 0 || vcpu >= VM_MAXCPU) 836221828Sgrehan return (EINVAL); 837221828Sgrehan 838221828Sgrehan if (type < 0 || type >= VM_CAP_MAX) 839221828Sgrehan return (EINVAL); 840221828Sgrehan 841221828Sgrehan return (VMSETCAP(vm->cookie, vcpu, type, val)); 842221828Sgrehan} 843221828Sgrehan 844221828Sgrehanuint64_t * 845221828Sgrehanvm_guest_msrs(struct vm *vm, int cpu) 846221828Sgrehan{ 847221828Sgrehan return (vm->vcpu[cpu].guest_msrs); 848221828Sgrehan} 849221828Sgrehan 850221828Sgrehanstruct vlapic * 851221828Sgrehanvm_lapic(struct vm *vm, int cpu) 852221828Sgrehan{ 853221828Sgrehan return (vm->vcpu[cpu].vlapic); 854221828Sgrehan} 855221828Sgrehan 856221828Sgrehanboolean_t 857221828Sgrehanvmm_is_pptdev(int bus, int slot, int func) 858221828Sgrehan{ 859221828Sgrehan int found, b, s, f, n; 860221828Sgrehan char *val, *cp, *cp2; 861221828Sgrehan 862221828Sgrehan /* 863221828Sgrehan * setenv pptdevs "1/2/3 4/5/6 7/8/9 10/11/12" 864221828Sgrehan */ 865221828Sgrehan found = 0; 866221828Sgrehan cp = val = getenv("pptdevs"); 867221828Sgrehan while (cp != NULL && *cp != '\0') { 868221828Sgrehan if ((cp2 = strchr(cp, ' ')) != NULL) 869221828Sgrehan *cp2 = '\0'; 870221828Sgrehan 871221828Sgrehan n = sscanf(cp, "%d/%d/%d", &b, &s, &f); 872221828Sgrehan if (n == 3 && bus == b && slot == s && func == f) { 873221828Sgrehan found = 1; 874221828Sgrehan break; 875221828Sgrehan } 876221828Sgrehan 877221828Sgrehan if (cp2 != NULL) 878221828Sgrehan *cp2++ = ' '; 879221828Sgrehan 880221828Sgrehan cp = cp2; 881221828Sgrehan } 882221828Sgrehan freeenv(val); 883221828Sgrehan return (found); 884221828Sgrehan} 885221828Sgrehan 886221828Sgrehanvoid * 887221828Sgrehanvm_iommu_domain(struct vm *vm) 888221828Sgrehan{ 889221828Sgrehan 890221828Sgrehan return (vm->iommu); 891221828Sgrehan} 892221828Sgrehan 893241489Sneelint 894241489Sneelvcpu_set_state(struct vm *vm, int vcpuid, enum vcpu_state state) 895221828Sgrehan{ 896241489Sneel int error; 897221828Sgrehan struct vcpu *vcpu; 898221828Sgrehan 899221828Sgrehan if (vcpuid < 0 || vcpuid >= VM_MAXCPU) 900221828Sgrehan panic("vm_set_run_state: invalid vcpuid %d", vcpuid); 901221828Sgrehan 902221828Sgrehan vcpu = &vm->vcpu[vcpuid]; 903221828Sgrehan 904241489Sneel vcpu_lock(vcpu); 905241489Sneel 906241489Sneel /* 907241489Sneel * The following state transitions are allowed: 908241489Sneel * IDLE -> RUNNING -> IDLE 909241489Sneel * IDLE -> CANNOT_RUN -> IDLE 910241489Sneel */ 911241489Sneel if ((vcpu->state == VCPU_IDLE && state != VCPU_IDLE) || 912241489Sneel (vcpu->state != VCPU_IDLE && state == VCPU_IDLE)) { 913241489Sneel error = 0; 914241489Sneel vcpu->state = state; 915221828Sgrehan } else { 916241489Sneel error = EBUSY; 917221828Sgrehan } 918241489Sneel 919241489Sneel vcpu_unlock(vcpu); 920241489Sneel 921241489Sneel return (error); 922221828Sgrehan} 923221828Sgrehan 924241489Sneelenum vcpu_state 925241489Sneelvcpu_get_state(struct vm *vm, int vcpuid) 926221828Sgrehan{ 927221828Sgrehan struct vcpu *vcpu; 928241489Sneel enum vcpu_state state; 929221828Sgrehan 930221828Sgrehan if (vcpuid < 0 || vcpuid >= VM_MAXCPU) 931221828Sgrehan panic("vm_get_run_state: invalid vcpuid %d", vcpuid); 932221828Sgrehan 933221828Sgrehan vcpu = &vm->vcpu[vcpuid]; 934221828Sgrehan 935241489Sneel vcpu_lock(vcpu); 936241489Sneel state = vcpu->state; 937241489Sneel vcpu_unlock(vcpu); 938221828Sgrehan 939241489Sneel return (state); 940221828Sgrehan} 941221828Sgrehan 942221828Sgrehanvoid 943221828Sgrehanvm_activate_cpu(struct vm *vm, int vcpuid) 944221828Sgrehan{ 945221828Sgrehan 946221828Sgrehan if (vcpuid >= 0 && vcpuid < VM_MAXCPU) 947223621Sgrehan CPU_SET(vcpuid, &vm->active_cpus); 948221828Sgrehan} 949221828Sgrehan 950223621Sgrehancpuset_t 951221828Sgrehanvm_active_cpus(struct vm *vm) 952221828Sgrehan{ 953221828Sgrehan 954221828Sgrehan return (vm->active_cpus); 955221828Sgrehan} 956221828Sgrehan 957221828Sgrehanvoid * 958221828Sgrehanvcpu_stats(struct vm *vm, int vcpuid) 959221828Sgrehan{ 960221828Sgrehan 961221828Sgrehan return (vm->vcpu[vcpuid].stats); 962221828Sgrehan} 963240922Sneel 964240922Sneelint 965240922Sneelvm_get_x2apic_state(struct vm *vm, int vcpuid, enum x2apic_state *state) 966240922Sneel{ 967240922Sneel if (vcpuid < 0 || vcpuid >= VM_MAXCPU) 968240922Sneel return (EINVAL); 969240922Sneel 970240922Sneel *state = vm->vcpu[vcpuid].x2apic_state; 971240922Sneel 972240922Sneel return (0); 973240922Sneel} 974240922Sneel 975240922Sneelint 976240922Sneelvm_set_x2apic_state(struct vm *vm, int vcpuid, enum x2apic_state state) 977240922Sneel{ 978240922Sneel if (vcpuid < 0 || vcpuid >= VM_MAXCPU) 979240922Sneel return (EINVAL); 980240922Sneel 981240922Sneel if (state < 0 || state >= X2APIC_STATE_LAST) 982240922Sneel return (EINVAL); 983240922Sneel 984240922Sneel vm->vcpu[vcpuid].x2apic_state = state; 985240922Sneel 986240943Sneel vlapic_set_x2apic_state(vm, vcpuid, state); 987240943Sneel 988240922Sneel return (0); 989240922Sneel} 990241489Sneel 991241489Sneelvoid 992241489Sneelvm_interrupt_hostcpu(struct vm *vm, int vcpuid) 993241489Sneel{ 994241489Sneel int hostcpu; 995241489Sneel struct vcpu *vcpu; 996241489Sneel 997241489Sneel vcpu = &vm->vcpu[vcpuid]; 998241489Sneel 999242065Sneel vcpu_lock(vcpu); 1000241489Sneel hostcpu = vcpu->hostcpu; 1001242065Sneel if (hostcpu == NOCPU) { 1002242065Sneel /* 1003242065Sneel * If the vcpu is 'RUNNING' but without a valid 'hostcpu' then 1004242065Sneel * the host thread must be sleeping waiting for an event to 1005242065Sneel * kick the vcpu out of 'hlt'. 1006242065Sneel * 1007242065Sneel * XXX this is racy because the condition exists right before 1008242065Sneel * and after calling VMRUN() in vm_run(). The wakeup() is 1009242065Sneel * benign in this case. 1010242065Sneel */ 1011242065Sneel if (vcpu->state == VCPU_RUNNING) 1012242065Sneel wakeup_one(vcpu); 1013242065Sneel } else { 1014242065Sneel if (vcpu->state != VCPU_RUNNING) 1015242065Sneel panic("invalid vcpu state %d", vcpu->state); 1016242065Sneel if (hostcpu != curcpu) 1017242065Sneel ipi_cpu(hostcpu, vmm_ipinum); 1018242065Sneel } 1019242065Sneel vcpu_unlock(vcpu); 1020241489Sneel} 1021