vmm.c revision 256281
119304Speter/*- 219304Speter * Copyright (c) 2011 NetApp, Inc. 319304Speter * All rights reserved. 419304Speter * 519304Speter * Redistribution and use in source and binary forms, with or without 619304Speter * modification, are permitted provided that the following conditions 719304Speter * are met: 819304Speter * 1. Redistributions of source code must retain the above copyright 919304Speter * notice, this list of conditions and the following disclaimer. 1019304Speter * 2. Redistributions in binary form must reproduce the above copyright 1119304Speter * notice, this list of conditions and the following disclaimer in the 1219304Speter * documentation and/or other materials provided with the distribution. 13254225Speter * 1419304Speter * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND 1519304Speter * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 1619304Speter * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 1719304Speter * ARE DISCLAIMED. IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE 18254225Speter * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 1919304Speter * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 2019304Speter * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 2119304Speter * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 2219304Speter * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 2319304Speter * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 2419304Speter * SUCH DAMAGE. 2519304Speter * 2619304Speter * $FreeBSD: stable/10/sys/amd64/vmm/vmm.c 256072 2013-10-05 21:22:35Z neel $ 2719304Speter */ 2819304Speter 2919304Speter#include <sys/cdefs.h> 3019304Speter__FBSDID("$FreeBSD: stable/10/sys/amd64/vmm/vmm.c 256072 2013-10-05 21:22:35Z neel $"); 3119304Speter 3219304Speter#include <sys/param.h> 3319304Speter#include <sys/systm.h> 34254225Speter#include <sys/kernel.h> 3519304Speter#include <sys/module.h> 3619304Speter#include <sys/sysctl.h> 3719304Speter#include <sys/malloc.h> 3819304Speter#include <sys/pcpu.h> 3919304Speter#include <sys/lock.h> 4019304Speter#include <sys/mutex.h> 41254225Speter#include <sys/proc.h> 42254225Speter#include <sys/rwlock.h> 43254225Speter#include <sys/sched.h> 44254225Speter#include <sys/smp.h> 45254225Speter#include <sys/systm.h> 46254225Speter 47254225Speter#include <vm/vm.h> 4819304Speter#include <vm/vm_object.h> 4919304Speter#include <vm/vm_page.h> 5019304Speter#include <vm/pmap.h> 51254225Speter#include <vm/vm_map.h> 5219304Speter#include <vm/vm_extern.h> 5319304Speter#include <vm/vm_param.h> 5419304Speter 5519304Speter#include <machine/vm.h> 5619304Speter#include <machine/pcb.h> 5719304Speter#include <machine/smp.h> 5819304Speter#include <x86/apicreg.h> 5919304Speter#include <machine/pmap.h> 6019304Speter#include <machine/vmparam.h> 6119304Speter 6219304Speter#include <machine/vmm.h> 6319304Speter#include "vmm_ktr.h" 6419304Speter#include "vmm_host.h" 6519304Speter#include "vmm_mem.h" 6619304Speter#include "vmm_util.h" 6719304Speter#include <machine/vmm_dev.h> 6819304Speter#include "vlapic.h" 6919304Speter#include "vmm_msr.h" 7019304Speter#include "vmm_ipi.h" 7119304Speter#include "vmm_stat.h" 7219304Speter#include "vmm_lapic.h" 7319304Speter 7419304Speter#include "io/ppt.h" 7519304Speter#include "io/iommu.h" 7619304Speter 7719304Speterstruct vlapic; 7819304Speter 7919304Speterstruct vcpu { 8019304Speter int flags; 8119304Speter enum vcpu_state state; 8219304Speter struct mtx mtx; 8319304Speter int hostcpu; /* host cpuid this vcpu last ran on */ 8419304Speter uint64_t guest_msrs[VMM_MSR_NUM]; 8519304Speter struct vlapic *vlapic; 8619304Speter int vcpuid; 8719304Speter struct savefpu *guestfpu; /* guest fpu state */ 8819304Speter void *stats; 8919304Speter struct vm_exit exitinfo; 9019304Speter enum x2apic_state x2apic_state; 9119304Speter int nmi_pending; 9219304Speter}; 9319304Speter 9419304Speter#define vcpu_lock_init(v) mtx_init(&((v)->mtx), "vcpu lock", 0, MTX_SPIN) 9519304Speter#define vcpu_lock(v) mtx_lock_spin(&((v)->mtx)) 9619304Speter#define vcpu_unlock(v) mtx_unlock_spin(&((v)->mtx)) 9719304Speter#define vcpu_assert_locked(v) mtx_assert(&((v)->mtx), MA_OWNED) 9819304Speter 9919304Speterstruct mem_seg { 10019304Speter vm_paddr_t gpa; 10119304Speter size_t len; 10219304Speter boolean_t wired; 10319304Speter vm_object_t object; 10419304Speter}; 10519304Speter#define VM_MAX_MEMORY_SEGMENTS 2 10619304Speter 10719304Speterstruct vm { 10819304Speter void *cookie; /* processor-specific data */ 10919304Speter void *iommu; /* iommu-specific data */ 11019304Speter struct vmspace *vmspace; /* guest's address space */ 11119304Speter struct vcpu vcpu[VM_MAXCPU]; 11219304Speter int num_mem_segs; 11319304Speter struct mem_seg mem_segs[VM_MAX_MEMORY_SEGMENTS]; 11419304Speter char name[VM_MAX_NAMELEN]; 11519304Speter 11619304Speter /* 11719304Speter * Set of active vcpus. 11819304Speter * An active vcpu is one that has been started implicitly (BSP) or 11919304Speter * explicitly (AP) by sending it a startup ipi. 12019304Speter */ 12119304Speter cpuset_t active_cpus; 12219304Speter}; 12319304Speter 12419304Speterstatic int vmm_initialized; 12519304Speter 12619304Speterstatic struct vmm_ops *ops; 12719304Speter#define VMM_INIT() (ops != NULL ? (*ops->init)() : 0) 12819304Speter#define VMM_CLEANUP() (ops != NULL ? (*ops->cleanup)() : 0) 12919304Speter 13019304Speter#define VMINIT(vm, pmap) (ops != NULL ? (*ops->vminit)(vm, pmap): NULL) 13119304Speter#define VMRUN(vmi, vcpu, rip, pmap) \ 13219304Speter (ops != NULL ? (*ops->vmrun)(vmi, vcpu, rip, pmap) : ENXIO) 13319304Speter#define VMCLEANUP(vmi) (ops != NULL ? (*ops->vmcleanup)(vmi) : NULL) 13419304Speter#define VMSPACE_ALLOC(min, max) \ 13519304Speter (ops != NULL ? (*ops->vmspace_alloc)(min, max) : NULL) 13619304Speter#define VMSPACE_FREE(vmspace) \ 13719304Speter (ops != NULL ? (*ops->vmspace_free)(vmspace) : ENXIO) 13819304Speter#define VMGETREG(vmi, vcpu, num, retval) \ 13919304Speter (ops != NULL ? (*ops->vmgetreg)(vmi, vcpu, num, retval) : ENXIO) 14019304Speter#define VMSETREG(vmi, vcpu, num, val) \ 14119304Speter (ops != NULL ? (*ops->vmsetreg)(vmi, vcpu, num, val) : ENXIO) 14219304Speter#define VMGETDESC(vmi, vcpu, num, desc) \ 14319304Speter (ops != NULL ? (*ops->vmgetdesc)(vmi, vcpu, num, desc) : ENXIO) 14419304Speter#define VMSETDESC(vmi, vcpu, num, desc) \ 14519304Speter (ops != NULL ? (*ops->vmsetdesc)(vmi, vcpu, num, desc) : ENXIO) 146254225Speter#define VMINJECT(vmi, vcpu, type, vec, ec, ecv) \ 14719304Speter (ops != NULL ? (*ops->vminject)(vmi, vcpu, type, vec, ec, ecv) : ENXIO) 14819304Speter#define VMGETCAP(vmi, vcpu, num, retval) \ 149254225Speter (ops != NULL ? (*ops->vmgetcap)(vmi, vcpu, num, retval) : ENXIO) 150254225Speter#define VMSETCAP(vmi, vcpu, num, val) \ 151254225Speter (ops != NULL ? (*ops->vmsetcap)(vmi, vcpu, num, val) : ENXIO) 152254225Speter 153254225Speter#define fpu_start_emulating() load_cr0(rcr0() | CR0_TS) 154254225Speter#define fpu_stop_emulating() clts() 155254225Speter 156254225Speterstatic MALLOC_DEFINE(M_VM, "vm", "vm"); 15719304SpeterCTASSERT(VMM_MSR_NUM <= 64); /* msr_mask can keep track of up to 64 msrs */ 15819304Speter 15919304Speter/* statistics */ 16019304Speterstatic VMM_STAT(VCPU_TOTAL_RUNTIME, "vcpu total runtime"); 16119304Speter 16219304Speterstatic void 163254225Spetervcpu_cleanup(struct vcpu *vcpu) 16419304Speter{ 16519304Speter vlapic_cleanup(vcpu->vlapic); 16619304Speter vmm_stat_free(vcpu->stats); 16719304Speter fpu_save_area_free(vcpu->guestfpu); 16819304Speter} 16919304Speter 17019304Speterstatic void 17119304Spetervcpu_init(struct vm *vm, uint32_t vcpu_id) 17219304Speter{ 17319304Speter struct vcpu *vcpu; 17419304Speter 17519304Speter vcpu = &vm->vcpu[vcpu_id]; 17619304Speter 17719304Speter vcpu_lock_init(vcpu); 17819304Speter vcpu->hostcpu = NOCPU; 17919304Speter vcpu->vcpuid = vcpu_id; 18019304Speter vcpu->vlapic = vlapic_init(vm, vcpu_id); 18119304Speter vm_set_x2apic_state(vm, vcpu_id, X2APIC_ENABLED); 18219304Speter vcpu->guestfpu = fpu_save_area_alloc(); 18319304Speter fpu_save_area_reset(vcpu->guestfpu); 18419304Speter vcpu->stats = vmm_stat_alloc(); 18519304Speter} 18619304Speter 18719304Speterstruct vm_exit * 18819304Spetervm_exitinfo(struct vm *vm, int cpuid) 18919304Speter{ 19019304Speter struct vcpu *vcpu; 19119304Speter 19219304Speter if (cpuid < 0 || cpuid >= VM_MAXCPU) 19319304Speter panic("vm_exitinfo: invalid cpuid %d", cpuid); 19419304Speter 19519304Speter vcpu = &vm->vcpu[cpuid]; 19619304Speter 19719304Speter return (&vcpu->exitinfo); 19819304Speter} 19919304Speter 20019304Speterstatic int 20119304Spetervmm_init(void) 20219304Speter{ 20319304Speter int error; 20419304Speter 20519304Speter vmm_host_state_init(); 20619304Speter vmm_ipi_init(); 20719304Speter 20819304Speter error = vmm_mem_init(); 20919304Speter if (error) 21019304Speter return (error); 21119304Speter 21219304Speter if (vmm_is_intel()) 21319304Speter ops = &vmm_ops_intel; 21419304Speter else if (vmm_is_amd()) 21519304Speter ops = &vmm_ops_amd; 21619304Speter else 217254225Speter return (ENXIO); 21819304Speter 21919304Speter vmm_msr_init(); 22019304Speter 22119304Speter return (VMM_INIT()); 22219304Speter} 22319304Speter 22419304Speterstatic int 22519304Spetervmm_handler(module_t mod, int what, void *arg) 22619304Speter{ 22719304Speter int error; 22819304Speter 22919304Speter switch (what) { 23019304Speter case MOD_LOAD: 23119304Speter vmmdev_init(); 23219304Speter iommu_init(); 23319304Speter error = vmm_init(); 23419304Speter if (error == 0) 23519304Speter vmm_initialized = 1; 23619304Speter break; 23719304Speter case MOD_UNLOAD: 23819304Speter error = vmmdev_cleanup(); 23919304Speter if (error == 0) { 24019304Speter iommu_cleanup(); 24119304Speter vmm_ipi_cleanup(); 24219304Speter error = VMM_CLEANUP(); 24319304Speter /* 24419304Speter * Something bad happened - prevent new 24519304Speter * VMs from being created 24619304Speter */ 24719304Speter if (error) 24819304Speter vmm_initialized = 0; 24919304Speter } 25019304Speter break; 25119304Speter default: 25219304Speter error = 0; 25319304Speter break; 25419304Speter } 25519304Speter return (error); 25619304Speter} 25719304Speter 25819304Speterstatic moduledata_t vmm_kmod = { 25919304Speter "vmm", 26019304Speter vmm_handler, 26119304Speter NULL 26219304Speter}; 26319304Speter 26419304Speter/* 26519304Speter * vmm initialization has the following dependencies: 26619304Speter * 26719304Speter * - iommu initialization must happen after the pci passthru driver has had 26819304Speter * a chance to attach to any passthru devices (after SI_SUB_CONFIGURE). 26919304Speter * 27019304Speter * - VT-x initialization requires smp_rendezvous() and therefore must happen 27119304Speter * after SMP is fully functional (after SI_SUB_SMP). 27219304Speter */ 27319304SpeterDECLARE_MODULE(vmm, vmm_kmod, SI_SUB_SMP + 1, SI_ORDER_ANY); 27419304SpeterMODULE_VERSION(vmm, 1); 27519304Speter 27619304SpeterSYSCTL_NODE(_hw, OID_AUTO, vmm, CTLFLAG_RW, NULL, NULL); 27719304Speter 27819304Speterint 27919304Spetervm_create(const char *name, struct vm **retvm) 28019304Speter{ 28119304Speter int i; 28219304Speter struct vm *vm; 28319304Speter struct vmspace *vmspace; 28419304Speter 28519304Speter const int BSP = 0; 28619304Speter 28719304Speter /* 28819304Speter * If vmm.ko could not be successfully initialized then don't attempt 28919304Speter * to create the virtual machine. 29019304Speter */ 29119304Speter if (!vmm_initialized) 292254225Speter return (ENXIO); 29319304Speter 29419304Speter if (name == NULL || strlen(name) >= VM_MAX_NAMELEN) 295254225Speter return (EINVAL); 296254225Speter 297254225Speter vmspace = VMSPACE_ALLOC(VM_MIN_ADDRESS, VM_MAXUSER_ADDRESS); 298254225Speter if (vmspace == NULL) 299254225Speter return (ENOMEM); 300254225Speter 301254225Speter vm = malloc(sizeof(struct vm), M_VM, M_WAITOK | M_ZERO); 302254225Speter strcpy(vm->name, name); 30319304Speter vm->cookie = VMINIT(vm, vmspace_pmap(vmspace)); 30419304Speter 30519304Speter for (i = 0; i < VM_MAXCPU; i++) { 30619304Speter vcpu_init(vm, i); 30719304Speter guest_msrs_init(vm, i); 30819304Speter } 309254225Speter 31019304Speter vm_activate_cpu(vm, BSP); 31119304Speter vm->vmspace = vmspace; 31219304Speter 31319304Speter *retvm = vm; 31419304Speter return (0); 31519304Speter} 31619304Speter 31719304Speterstatic void 31819304Spetervm_free_mem_seg(struct vm *vm, struct mem_seg *seg) 31919304Speter{ 32019304Speter 32119304Speter if (seg->object != NULL) 32219304Speter vmm_mem_free(vm->vmspace, seg->gpa, seg->len); 32319304Speter 32419304Speter bzero(seg, sizeof(*seg)); 32519304Speter} 32619304Speter 32719304Spetervoid 32819304Spetervm_destroy(struct vm *vm) 32919304Speter{ 33019304Speter int i; 33119304Speter 33219304Speter ppt_unassign_all(vm); 33319304Speter 33419304Speter if (vm->iommu != NULL) 33519304Speter iommu_destroy_domain(vm->iommu); 33619304Speter 33719304Speter for (i = 0; i < vm->num_mem_segs; i++) 33819304Speter vm_free_mem_seg(vm, &vm->mem_segs[i]); 33919304Speter 34019304Speter vm->num_mem_segs = 0; 34119304Speter 34219304Speter for (i = 0; i < VM_MAXCPU; i++) 34319304Speter vcpu_cleanup(&vm->vcpu[i]); 34419304Speter 34519304Speter VMSPACE_FREE(vm->vmspace); 34619304Speter 34719304Speter VMCLEANUP(vm->cookie); 34819304Speter 34919304Speter free(vm, M_VM); 35019304Speter} 351254225Speter 35219304Speterconst char * 35319304Spetervm_name(struct vm *vm) 35419304Speter{ 35519304Speter return (vm->name); 35619304Speter} 35719304Speter 35819304Speterint 35919304Spetervm_map_mmio(struct vm *vm, vm_paddr_t gpa, size_t len, vm_paddr_t hpa) 36019304Speter{ 36119304Speter vm_object_t obj; 36219304Speter 36319304Speter if ((obj = vmm_mmio_alloc(vm->vmspace, gpa, len, hpa)) == NULL) 36419304Speter return (ENOMEM); 36519304Speter else 36619304Speter return (0); 36719304Speter} 36819304Speter 36919304Speterint 37019304Spetervm_unmap_mmio(struct vm *vm, vm_paddr_t gpa, size_t len) 37119304Speter{ 37219304Speter 37319304Speter vmm_mmio_free(vm->vmspace, gpa, len); 37419304Speter return (0); 37519304Speter} 37619304Speter 37719304Speterboolean_t 37819304Spetervm_mem_allocated(struct vm *vm, vm_paddr_t gpa) 37919304Speter{ 38019304Speter int i; 38119304Speter vm_paddr_t gpabase, gpalimit; 38219304Speter 38319304Speter for (i = 0; i < vm->num_mem_segs; i++) { 38419304Speter gpabase = vm->mem_segs[i].gpa; 38519304Speter gpalimit = gpabase + vm->mem_segs[i].len; 38619304Speter if (gpa >= gpabase && gpa < gpalimit) 38719304Speter return (TRUE); /* 'gpa' is regular memory */ 38819304Speter } 38919304Speter 39019304Speter if (ppt_is_mmio(vm, gpa)) 39119304Speter return (TRUE); /* 'gpa' is pci passthru mmio */ 39219304Speter 39319304Speter return (FALSE); 39419304Speter} 39519304Speter 39619304Speterint 39719304Spetervm_malloc(struct vm *vm, vm_paddr_t gpa, size_t len) 39819304Speter{ 39919304Speter int available, allocated; 40019304Speter struct mem_seg *seg; 40119304Speter vm_object_t object; 40219304Speter vm_paddr_t g; 40319304Speter 40419304Speter if ((gpa & PAGE_MASK) || (len & PAGE_MASK) || len == 0) 40519304Speter return (EINVAL); 40619304Speter 40719304Speter available = allocated = 0; 40819304Speter g = gpa; 40919304Speter while (g < gpa + len) { 41019304Speter if (vm_mem_allocated(vm, g)) 41119304Speter allocated++; 41219304Speter else 41319304Speter available++; 41419304Speter 41519304Speter g += PAGE_SIZE; 41619304Speter } 41719304Speter 41819304Speter /* 41919304Speter * If there are some allocated and some available pages in the address 42019304Speter * range then it is an error. 42119304Speter */ 42219304Speter if (allocated && available) 42319304Speter return (EINVAL); 42419304Speter 42519304Speter /* 42619304Speter * If the entire address range being requested has already been 42719304Speter * allocated then there isn't anything more to do. 42819304Speter */ 42919304Speter if (allocated && available == 0) 43019304Speter return (0); 43119304Speter 43219304Speter if (vm->num_mem_segs >= VM_MAX_MEMORY_SEGMENTS) 43319304Speter return (E2BIG); 43419304Speter 43519304Speter seg = &vm->mem_segs[vm->num_mem_segs]; 43619304Speter 43719304Speter if ((object = vmm_mem_alloc(vm->vmspace, gpa, len)) == NULL) 43819304Speter return (ENOMEM); 43919304Speter 44019304Speter seg->gpa = gpa; 44119304Speter seg->len = len; 44219304Speter seg->object = object; 44319304Speter seg->wired = FALSE; 44419304Speter 44519304Speter vm->num_mem_segs++; 44619304Speter 44719304Speter return (0); 44819304Speter} 44919304Speter 45019304Speterstatic void 45119304Spetervm_gpa_unwire(struct vm *vm) 45219304Speter{ 45319304Speter int i, rv; 45419304Speter struct mem_seg *seg; 45519304Speter 456254225Speter for (i = 0; i < vm->num_mem_segs; i++) { 457254225Speter seg = &vm->mem_segs[i]; 458254225Speter if (!seg->wired) 45919304Speter continue; 46019304Speter 46119304Speter rv = vm_map_unwire(&vm->vmspace->vm_map, 46219304Speter seg->gpa, seg->gpa + seg->len, 46319304Speter VM_MAP_WIRE_USER | VM_MAP_WIRE_NOHOLES); 46419304Speter KASSERT(rv == KERN_SUCCESS, ("vm(%s) memory segment " 46519304Speter "%#lx/%ld could not be unwired: %d", 46619304Speter vm_name(vm), seg->gpa, seg->len, rv)); 46719304Speter 46819304Speter seg->wired = FALSE; 46919304Speter } 47019304Speter} 47119304Speter 47219304Speterstatic int 47319304Spetervm_gpa_wire(struct vm *vm) 47419304Speter{ 47519304Speter int i, rv; 47619304Speter struct mem_seg *seg; 47719304Speter 47819304Speter for (i = 0; i < vm->num_mem_segs; i++) { 47919304Speter seg = &vm->mem_segs[i]; 48019304Speter if (seg->wired) 48119304Speter continue; 48219304Speter 48319304Speter /* XXX rlimits? */ 48419304Speter rv = vm_map_wire(&vm->vmspace->vm_map, 48519304Speter seg->gpa, seg->gpa + seg->len, 48619304Speter VM_MAP_WIRE_USER | VM_MAP_WIRE_NOHOLES); 48719304Speter if (rv != KERN_SUCCESS) 48819304Speter break; 48919304Speter 49019304Speter seg->wired = TRUE; 49119304Speter } 49219304Speter 493254225Speter if (i < vm->num_mem_segs) { 494254225Speter /* 495254225Speter * Undo the wiring before returning an error. 49619304Speter */ 49719304Speter vm_gpa_unwire(vm); 49819304Speter return (EAGAIN); 499 } 500 501 return (0); 502} 503 504static void 505vm_iommu_modify(struct vm *vm, boolean_t map) 506{ 507 int i, sz; 508 vm_paddr_t gpa, hpa; 509 struct mem_seg *seg; 510 void *vp, *cookie, *host_domain; 511 512 sz = PAGE_SIZE; 513 host_domain = iommu_host_domain(); 514 515 for (i = 0; i < vm->num_mem_segs; i++) { 516 seg = &vm->mem_segs[i]; 517 KASSERT(seg->wired, ("vm(%s) memory segment %#lx/%ld not wired", 518 vm_name(vm), seg->gpa, seg->len)); 519 520 gpa = seg->gpa; 521 while (gpa < seg->gpa + seg->len) { 522 vp = vm_gpa_hold(vm, gpa, PAGE_SIZE, VM_PROT_WRITE, 523 &cookie); 524 KASSERT(vp != NULL, ("vm(%s) could not map gpa %#lx", 525 vm_name(vm), gpa)); 526 527 vm_gpa_release(cookie); 528 529 hpa = DMAP_TO_PHYS((uintptr_t)vp); 530 if (map) { 531 iommu_create_mapping(vm->iommu, gpa, hpa, sz); 532 iommu_remove_mapping(host_domain, hpa, sz); 533 } else { 534 iommu_remove_mapping(vm->iommu, gpa, sz); 535 iommu_create_mapping(host_domain, hpa, hpa, sz); 536 } 537 538 gpa += PAGE_SIZE; 539 } 540 } 541 542 /* 543 * Invalidate the cached translations associated with the domain 544 * from which pages were removed. 545 */ 546 if (map) 547 iommu_invalidate_tlb(host_domain); 548 else 549 iommu_invalidate_tlb(vm->iommu); 550} 551 552#define vm_iommu_unmap(vm) vm_iommu_modify((vm), FALSE) 553#define vm_iommu_map(vm) vm_iommu_modify((vm), TRUE) 554 555int 556vm_unassign_pptdev(struct vm *vm, int bus, int slot, int func) 557{ 558 int error; 559 560 error = ppt_unassign_device(vm, bus, slot, func); 561 if (error) 562 return (error); 563 564 if (ppt_num_devices(vm) == 0) { 565 vm_iommu_unmap(vm); 566 vm_gpa_unwire(vm); 567 } 568 return (0); 569} 570 571int 572vm_assign_pptdev(struct vm *vm, int bus, int slot, int func) 573{ 574 int error; 575 vm_paddr_t maxaddr; 576 577 /* 578 * Virtual machines with pci passthru devices get special treatment: 579 * - the guest physical memory is wired 580 * - the iommu is programmed to do the 'gpa' to 'hpa' translation 581 * 582 * We need to do this before the first pci passthru device is attached. 583 */ 584 if (ppt_num_devices(vm) == 0) { 585 KASSERT(vm->iommu == NULL, 586 ("vm_assign_pptdev: iommu must be NULL")); 587 maxaddr = vmm_mem_maxaddr(); 588 vm->iommu = iommu_create_domain(maxaddr); 589 590 error = vm_gpa_wire(vm); 591 if (error) 592 return (error); 593 594 vm_iommu_map(vm); 595 } 596 597 error = ppt_assign_device(vm, bus, slot, func); 598 return (error); 599} 600 601void * 602vm_gpa_hold(struct vm *vm, vm_paddr_t gpa, size_t len, int reqprot, 603 void **cookie) 604{ 605 int count, pageoff; 606 vm_page_t m; 607 608 pageoff = gpa & PAGE_MASK; 609 if (len > PAGE_SIZE - pageoff) 610 panic("vm_gpa_hold: invalid gpa/len: 0x%016lx/%lu", gpa, len); 611 612 count = vm_fault_quick_hold_pages(&vm->vmspace->vm_map, 613 trunc_page(gpa), PAGE_SIZE, reqprot, &m, 1); 614 615 if (count == 1) { 616 *cookie = m; 617 return ((void *)(PHYS_TO_DMAP(VM_PAGE_TO_PHYS(m)) + pageoff)); 618 } else { 619 *cookie = NULL; 620 return (NULL); 621 } 622} 623 624void 625vm_gpa_release(void *cookie) 626{ 627 vm_page_t m = cookie; 628 629 vm_page_lock(m); 630 vm_page_unhold(m); 631 vm_page_unlock(m); 632} 633 634int 635vm_gpabase2memseg(struct vm *vm, vm_paddr_t gpabase, 636 struct vm_memory_segment *seg) 637{ 638 int i; 639 640 for (i = 0; i < vm->num_mem_segs; i++) { 641 if (gpabase == vm->mem_segs[i].gpa) { 642 seg->gpa = vm->mem_segs[i].gpa; 643 seg->len = vm->mem_segs[i].len; 644 seg->wired = vm->mem_segs[i].wired; 645 return (0); 646 } 647 } 648 return (-1); 649} 650 651int 652vm_get_memobj(struct vm *vm, vm_paddr_t gpa, size_t len, 653 vm_offset_t *offset, struct vm_object **object) 654{ 655 int i; 656 size_t seg_len; 657 vm_paddr_t seg_gpa; 658 vm_object_t seg_obj; 659 660 for (i = 0; i < vm->num_mem_segs; i++) { 661 if ((seg_obj = vm->mem_segs[i].object) == NULL) 662 continue; 663 664 seg_gpa = vm->mem_segs[i].gpa; 665 seg_len = vm->mem_segs[i].len; 666 667 if (gpa >= seg_gpa && gpa < seg_gpa + seg_len) { 668 *offset = gpa - seg_gpa; 669 *object = seg_obj; 670 vm_object_reference(seg_obj); 671 return (0); 672 } 673 } 674 675 return (EINVAL); 676} 677 678int 679vm_get_register(struct vm *vm, int vcpu, int reg, uint64_t *retval) 680{ 681 682 if (vcpu < 0 || vcpu >= VM_MAXCPU) 683 return (EINVAL); 684 685 if (reg >= VM_REG_LAST) 686 return (EINVAL); 687 688 return (VMGETREG(vm->cookie, vcpu, reg, retval)); 689} 690 691int 692vm_set_register(struct vm *vm, int vcpu, int reg, uint64_t val) 693{ 694 695 if (vcpu < 0 || vcpu >= VM_MAXCPU) 696 return (EINVAL); 697 698 if (reg >= VM_REG_LAST) 699 return (EINVAL); 700 701 return (VMSETREG(vm->cookie, vcpu, reg, val)); 702} 703 704static boolean_t 705is_descriptor_table(int reg) 706{ 707 708 switch (reg) { 709 case VM_REG_GUEST_IDTR: 710 case VM_REG_GUEST_GDTR: 711 return (TRUE); 712 default: 713 return (FALSE); 714 } 715} 716 717static boolean_t 718is_segment_register(int reg) 719{ 720 721 switch (reg) { 722 case VM_REG_GUEST_ES: 723 case VM_REG_GUEST_CS: 724 case VM_REG_GUEST_SS: 725 case VM_REG_GUEST_DS: 726 case VM_REG_GUEST_FS: 727 case VM_REG_GUEST_GS: 728 case VM_REG_GUEST_TR: 729 case VM_REG_GUEST_LDTR: 730 return (TRUE); 731 default: 732 return (FALSE); 733 } 734} 735 736int 737vm_get_seg_desc(struct vm *vm, int vcpu, int reg, 738 struct seg_desc *desc) 739{ 740 741 if (vcpu < 0 || vcpu >= VM_MAXCPU) 742 return (EINVAL); 743 744 if (!is_segment_register(reg) && !is_descriptor_table(reg)) 745 return (EINVAL); 746 747 return (VMGETDESC(vm->cookie, vcpu, reg, desc)); 748} 749 750int 751vm_set_seg_desc(struct vm *vm, int vcpu, int reg, 752 struct seg_desc *desc) 753{ 754 if (vcpu < 0 || vcpu >= VM_MAXCPU) 755 return (EINVAL); 756 757 if (!is_segment_register(reg) && !is_descriptor_table(reg)) 758 return (EINVAL); 759 760 return (VMSETDESC(vm->cookie, vcpu, reg, desc)); 761} 762 763static void 764restore_guest_fpustate(struct vcpu *vcpu) 765{ 766 767 /* flush host state to the pcb */ 768 fpuexit(curthread); 769 770 /* restore guest FPU state */ 771 fpu_stop_emulating(); 772 fpurestore(vcpu->guestfpu); 773 774 /* 775 * The FPU is now "dirty" with the guest's state so turn on emulation 776 * to trap any access to the FPU by the host. 777 */ 778 fpu_start_emulating(); 779} 780 781static void 782save_guest_fpustate(struct vcpu *vcpu) 783{ 784 785 if ((rcr0() & CR0_TS) == 0) 786 panic("fpu emulation not enabled in host!"); 787 788 /* save guest FPU state */ 789 fpu_stop_emulating(); 790 fpusave(vcpu->guestfpu); 791 fpu_start_emulating(); 792} 793 794static VMM_STAT(VCPU_IDLE_TICKS, "number of ticks vcpu was idle"); 795 796static int 797vcpu_set_state_locked(struct vcpu *vcpu, enum vcpu_state newstate) 798{ 799 int error; 800 801 vcpu_assert_locked(vcpu); 802 803 /* 804 * The following state transitions are allowed: 805 * IDLE -> FROZEN -> IDLE 806 * FROZEN -> RUNNING -> FROZEN 807 * FROZEN -> SLEEPING -> FROZEN 808 */ 809 switch (vcpu->state) { 810 case VCPU_IDLE: 811 case VCPU_RUNNING: 812 case VCPU_SLEEPING: 813 error = (newstate != VCPU_FROZEN); 814 break; 815 case VCPU_FROZEN: 816 error = (newstate == VCPU_FROZEN); 817 break; 818 default: 819 error = 1; 820 break; 821 } 822 823 if (error == 0) 824 vcpu->state = newstate; 825 else 826 error = EBUSY; 827 828 return (error); 829} 830 831static void 832vcpu_require_state(struct vm *vm, int vcpuid, enum vcpu_state newstate) 833{ 834 int error; 835 836 if ((error = vcpu_set_state(vm, vcpuid, newstate)) != 0) 837 panic("Error %d setting state to %d\n", error, newstate); 838} 839 840static void 841vcpu_require_state_locked(struct vcpu *vcpu, enum vcpu_state newstate) 842{ 843 int error; 844 845 if ((error = vcpu_set_state_locked(vcpu, newstate)) != 0) 846 panic("Error %d setting state to %d", error, newstate); 847} 848 849/* 850 * Emulate a guest 'hlt' by sleeping until the vcpu is ready to run. 851 */ 852static int 853vm_handle_hlt(struct vm *vm, int vcpuid, boolean_t *retu) 854{ 855 struct vcpu *vcpu; 856 int sleepticks, t; 857 858 vcpu = &vm->vcpu[vcpuid]; 859 860 vcpu_lock(vcpu); 861 862 /* 863 * Figure out the number of host ticks until the next apic 864 * timer interrupt in the guest. 865 */ 866 sleepticks = lapic_timer_tick(vm, vcpuid); 867 868 /* 869 * If the guest local apic timer is disabled then sleep for 870 * a long time but not forever. 871 */ 872 if (sleepticks < 0) 873 sleepticks = hz; 874 875 /* 876 * Do a final check for pending NMI or interrupts before 877 * really putting this thread to sleep. 878 * 879 * These interrupts could have happened any time after we 880 * returned from VMRUN() and before we grabbed the vcpu lock. 881 */ 882 if (!vm_nmi_pending(vm, vcpuid) && lapic_pending_intr(vm, vcpuid) < 0) { 883 if (sleepticks <= 0) 884 panic("invalid sleepticks %d", sleepticks); 885 t = ticks; 886 vcpu_require_state_locked(vcpu, VCPU_SLEEPING); 887 msleep_spin(vcpu, &vcpu->mtx, "vmidle", sleepticks); 888 vcpu_require_state_locked(vcpu, VCPU_FROZEN); 889 vmm_stat_incr(vm, vcpuid, VCPU_IDLE_TICKS, ticks - t); 890 } 891 vcpu_unlock(vcpu); 892 893 return (0); 894} 895 896static int 897vm_handle_paging(struct vm *vm, int vcpuid, boolean_t *retu) 898{ 899 int rv, ftype; 900 struct vm_map *map; 901 struct vcpu *vcpu; 902 struct vm_exit *vme; 903 904 vcpu = &vm->vcpu[vcpuid]; 905 vme = &vcpu->exitinfo; 906 907 ftype = vme->u.paging.fault_type; 908 KASSERT(ftype == VM_PROT_READ || 909 ftype == VM_PROT_WRITE || ftype == VM_PROT_EXECUTE, 910 ("vm_handle_paging: invalid fault_type %d", ftype)); 911 912 if (ftype == VM_PROT_READ || ftype == VM_PROT_WRITE) { 913 rv = pmap_emulate_accessed_dirty(vmspace_pmap(vm->vmspace), 914 vme->u.paging.gpa, ftype); 915 if (rv == 0) 916 goto done; 917 } 918 919 map = &vm->vmspace->vm_map; 920 rv = vm_fault(map, vme->u.paging.gpa, ftype, VM_FAULT_NORMAL); 921 922 VMM_CTR3(vm, vcpuid, "vm_handle_paging rv = %d, gpa = %#lx, ftype = %d", 923 rv, vme->u.paging.gpa, ftype); 924 925 if (rv != KERN_SUCCESS) 926 return (EFAULT); 927done: 928 /* restart execution at the faulting instruction */ 929 vme->inst_length = 0; 930 931 return (0); 932} 933 934static int 935vm_handle_inst_emul(struct vm *vm, int vcpuid, boolean_t *retu) 936{ 937 struct vie *vie; 938 struct vcpu *vcpu; 939 struct vm_exit *vme; 940 int error, inst_length; 941 uint64_t rip, gla, gpa, cr3; 942 943 vcpu = &vm->vcpu[vcpuid]; 944 vme = &vcpu->exitinfo; 945 946 rip = vme->rip; 947 inst_length = vme->inst_length; 948 949 gla = vme->u.inst_emul.gla; 950 gpa = vme->u.inst_emul.gpa; 951 cr3 = vme->u.inst_emul.cr3; 952 vie = &vme->u.inst_emul.vie; 953 954 vie_init(vie); 955 956 /* Fetch, decode and emulate the faulting instruction */ 957 if (vmm_fetch_instruction(vm, vcpuid, rip, inst_length, cr3, vie) != 0) 958 return (EFAULT); 959 960 if (vmm_decode_instruction(vm, vcpuid, gla, vie) != 0) 961 return (EFAULT); 962 963 /* return to userland unless this is a local apic access */ 964 if (gpa < DEFAULT_APIC_BASE || gpa >= DEFAULT_APIC_BASE + PAGE_SIZE) { 965 *retu = TRUE; 966 return (0); 967 } 968 969 error = vmm_emulate_instruction(vm, vcpuid, gpa, vie, 970 lapic_mmio_read, lapic_mmio_write, 0); 971 972 /* return to userland to spin up the AP */ 973 if (error == 0 && vme->exitcode == VM_EXITCODE_SPINUP_AP) 974 *retu = TRUE; 975 976 return (error); 977} 978 979int 980vm_run(struct vm *vm, struct vm_run *vmrun) 981{ 982 int error, vcpuid; 983 struct vcpu *vcpu; 984 struct pcb *pcb; 985 uint64_t tscval, rip; 986 struct vm_exit *vme; 987 boolean_t retu; 988 pmap_t pmap; 989 990 vcpuid = vmrun->cpuid; 991 992 if (vcpuid < 0 || vcpuid >= VM_MAXCPU) 993 return (EINVAL); 994 995 pmap = vmspace_pmap(vm->vmspace); 996 vcpu = &vm->vcpu[vcpuid]; 997 vme = &vcpu->exitinfo; 998 rip = vmrun->rip; 999restart: 1000 critical_enter(); 1001 1002 KASSERT(!CPU_ISSET(curcpu, &pmap->pm_active), 1003 ("vm_run: absurd pm_active")); 1004 1005 tscval = rdtsc(); 1006 1007 pcb = PCPU_GET(curpcb); 1008 set_pcb_flags(pcb, PCB_FULL_IRET); 1009 1010 restore_guest_msrs(vm, vcpuid); 1011 restore_guest_fpustate(vcpu); 1012 1013 vcpu_require_state(vm, vcpuid, VCPU_RUNNING); 1014 vcpu->hostcpu = curcpu; 1015 error = VMRUN(vm->cookie, vcpuid, rip, pmap); 1016 vcpu->hostcpu = NOCPU; 1017 vcpu_require_state(vm, vcpuid, VCPU_FROZEN); 1018 1019 save_guest_fpustate(vcpu); 1020 restore_host_msrs(vm, vcpuid); 1021 1022 vmm_stat_incr(vm, vcpuid, VCPU_TOTAL_RUNTIME, rdtsc() - tscval); 1023 1024 critical_exit(); 1025 1026 if (error == 0) { 1027 retu = FALSE; 1028 switch (vme->exitcode) { 1029 case VM_EXITCODE_HLT: 1030 error = vm_handle_hlt(vm, vcpuid, &retu); 1031 break; 1032 case VM_EXITCODE_PAGING: 1033 error = vm_handle_paging(vm, vcpuid, &retu); 1034 break; 1035 case VM_EXITCODE_INST_EMUL: 1036 error = vm_handle_inst_emul(vm, vcpuid, &retu); 1037 break; 1038 default: 1039 retu = TRUE; /* handled in userland */ 1040 break; 1041 } 1042 } 1043 1044 if (error == 0 && retu == FALSE) { 1045 rip = vme->rip + vme->inst_length; 1046 goto restart; 1047 } 1048 1049 /* copy the exit information */ 1050 bcopy(vme, &vmrun->vm_exit, sizeof(struct vm_exit)); 1051 return (error); 1052} 1053 1054int 1055vm_inject_event(struct vm *vm, int vcpuid, int type, 1056 int vector, uint32_t code, int code_valid) 1057{ 1058 if (vcpuid < 0 || vcpuid >= VM_MAXCPU) 1059 return (EINVAL); 1060 1061 if ((type > VM_EVENT_NONE && type < VM_EVENT_MAX) == 0) 1062 return (EINVAL); 1063 1064 if (vector < 0 || vector > 255) 1065 return (EINVAL); 1066 1067 return (VMINJECT(vm->cookie, vcpuid, type, vector, code, code_valid)); 1068} 1069 1070static VMM_STAT(VCPU_NMI_COUNT, "number of NMIs delivered to vcpu"); 1071 1072int 1073vm_inject_nmi(struct vm *vm, int vcpuid) 1074{ 1075 struct vcpu *vcpu; 1076 1077 if (vcpuid < 0 || vcpuid >= VM_MAXCPU) 1078 return (EINVAL); 1079 1080 vcpu = &vm->vcpu[vcpuid]; 1081 1082 vcpu->nmi_pending = 1; 1083 vm_interrupt_hostcpu(vm, vcpuid); 1084 return (0); 1085} 1086 1087int 1088vm_nmi_pending(struct vm *vm, int vcpuid) 1089{ 1090 struct vcpu *vcpu; 1091 1092 if (vcpuid < 0 || vcpuid >= VM_MAXCPU) 1093 panic("vm_nmi_pending: invalid vcpuid %d", vcpuid); 1094 1095 vcpu = &vm->vcpu[vcpuid]; 1096 1097 return (vcpu->nmi_pending); 1098} 1099 1100void 1101vm_nmi_clear(struct vm *vm, int vcpuid) 1102{ 1103 struct vcpu *vcpu; 1104 1105 if (vcpuid < 0 || vcpuid >= VM_MAXCPU) 1106 panic("vm_nmi_pending: invalid vcpuid %d", vcpuid); 1107 1108 vcpu = &vm->vcpu[vcpuid]; 1109 1110 if (vcpu->nmi_pending == 0) 1111 panic("vm_nmi_clear: inconsistent nmi_pending state"); 1112 1113 vcpu->nmi_pending = 0; 1114 vmm_stat_incr(vm, vcpuid, VCPU_NMI_COUNT, 1); 1115} 1116 1117int 1118vm_get_capability(struct vm *vm, int vcpu, int type, int *retval) 1119{ 1120 if (vcpu < 0 || vcpu >= VM_MAXCPU) 1121 return (EINVAL); 1122 1123 if (type < 0 || type >= VM_CAP_MAX) 1124 return (EINVAL); 1125 1126 return (VMGETCAP(vm->cookie, vcpu, type, retval)); 1127} 1128 1129int 1130vm_set_capability(struct vm *vm, int vcpu, int type, int val) 1131{ 1132 if (vcpu < 0 || vcpu >= VM_MAXCPU) 1133 return (EINVAL); 1134 1135 if (type < 0 || type >= VM_CAP_MAX) 1136 return (EINVAL); 1137 1138 return (VMSETCAP(vm->cookie, vcpu, type, val)); 1139} 1140 1141uint64_t * 1142vm_guest_msrs(struct vm *vm, int cpu) 1143{ 1144 return (vm->vcpu[cpu].guest_msrs); 1145} 1146 1147struct vlapic * 1148vm_lapic(struct vm *vm, int cpu) 1149{ 1150 return (vm->vcpu[cpu].vlapic); 1151} 1152 1153boolean_t 1154vmm_is_pptdev(int bus, int slot, int func) 1155{ 1156 int found, i, n; 1157 int b, s, f; 1158 char *val, *cp, *cp2; 1159 1160 /* 1161 * XXX 1162 * The length of an environment variable is limited to 128 bytes which 1163 * puts an upper limit on the number of passthru devices that may be 1164 * specified using a single environment variable. 1165 * 1166 * Work around this by scanning multiple environment variable 1167 * names instead of a single one - yuck! 1168 */ 1169 const char *names[] = { "pptdevs", "pptdevs2", "pptdevs3", NULL }; 1170 1171 /* set pptdevs="1/2/3 4/5/6 7/8/9 10/11/12" */ 1172 found = 0; 1173 for (i = 0; names[i] != NULL && !found; i++) { 1174 cp = val = getenv(names[i]); 1175 while (cp != NULL && *cp != '\0') { 1176 if ((cp2 = strchr(cp, ' ')) != NULL) 1177 *cp2 = '\0'; 1178 1179 n = sscanf(cp, "%d/%d/%d", &b, &s, &f); 1180 if (n == 3 && bus == b && slot == s && func == f) { 1181 found = 1; 1182 break; 1183 } 1184 1185 if (cp2 != NULL) 1186 *cp2++ = ' '; 1187 1188 cp = cp2; 1189 } 1190 freeenv(val); 1191 } 1192 return (found); 1193} 1194 1195void * 1196vm_iommu_domain(struct vm *vm) 1197{ 1198 1199 return (vm->iommu); 1200} 1201 1202int 1203vcpu_set_state(struct vm *vm, int vcpuid, enum vcpu_state newstate) 1204{ 1205 int error; 1206 struct vcpu *vcpu; 1207 1208 if (vcpuid < 0 || vcpuid >= VM_MAXCPU) 1209 panic("vm_set_run_state: invalid vcpuid %d", vcpuid); 1210 1211 vcpu = &vm->vcpu[vcpuid]; 1212 1213 vcpu_lock(vcpu); 1214 error = vcpu_set_state_locked(vcpu, newstate); 1215 vcpu_unlock(vcpu); 1216 1217 return (error); 1218} 1219 1220enum vcpu_state 1221vcpu_get_state(struct vm *vm, int vcpuid, int *hostcpu) 1222{ 1223 struct vcpu *vcpu; 1224 enum vcpu_state state; 1225 1226 if (vcpuid < 0 || vcpuid >= VM_MAXCPU) 1227 panic("vm_get_run_state: invalid vcpuid %d", vcpuid); 1228 1229 vcpu = &vm->vcpu[vcpuid]; 1230 1231 vcpu_lock(vcpu); 1232 state = vcpu->state; 1233 if (hostcpu != NULL) 1234 *hostcpu = vcpu->hostcpu; 1235 vcpu_unlock(vcpu); 1236 1237 return (state); 1238} 1239 1240void 1241vm_activate_cpu(struct vm *vm, int vcpuid) 1242{ 1243 1244 if (vcpuid >= 0 && vcpuid < VM_MAXCPU) 1245 CPU_SET(vcpuid, &vm->active_cpus); 1246} 1247 1248cpuset_t 1249vm_active_cpus(struct vm *vm) 1250{ 1251 1252 return (vm->active_cpus); 1253} 1254 1255void * 1256vcpu_stats(struct vm *vm, int vcpuid) 1257{ 1258 1259 return (vm->vcpu[vcpuid].stats); 1260} 1261 1262int 1263vm_get_x2apic_state(struct vm *vm, int vcpuid, enum x2apic_state *state) 1264{ 1265 if (vcpuid < 0 || vcpuid >= VM_MAXCPU) 1266 return (EINVAL); 1267 1268 *state = vm->vcpu[vcpuid].x2apic_state; 1269 1270 return (0); 1271} 1272 1273int 1274vm_set_x2apic_state(struct vm *vm, int vcpuid, enum x2apic_state state) 1275{ 1276 if (vcpuid < 0 || vcpuid >= VM_MAXCPU) 1277 return (EINVAL); 1278 1279 if (state >= X2APIC_STATE_LAST) 1280 return (EINVAL); 1281 1282 vm->vcpu[vcpuid].x2apic_state = state; 1283 1284 vlapic_set_x2apic_state(vm, vcpuid, state); 1285 1286 return (0); 1287} 1288 1289void 1290vm_interrupt_hostcpu(struct vm *vm, int vcpuid) 1291{ 1292 int hostcpu; 1293 struct vcpu *vcpu; 1294 1295 vcpu = &vm->vcpu[vcpuid]; 1296 1297 vcpu_lock(vcpu); 1298 hostcpu = vcpu->hostcpu; 1299 if (hostcpu == NOCPU) { 1300 if (vcpu->state == VCPU_SLEEPING) 1301 wakeup_one(vcpu); 1302 } else { 1303 if (vcpu->state != VCPU_RUNNING) 1304 panic("invalid vcpu state %d", vcpu->state); 1305 if (hostcpu != curcpu) 1306 ipi_cpu(hostcpu, vmm_ipinum); 1307 } 1308 vcpu_unlock(vcpu); 1309} 1310 1311struct vmspace * 1312vm_get_vmspace(struct vm *vm) 1313{ 1314 1315 return (vm->vmspace); 1316} 1317