vmm.c revision 261088
1/*- 2 * Copyright (c) 2011 NetApp, Inc. 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND 15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17 * ARE DISCLAIMED. IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE 18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24 * SUCH DAMAGE. 25 * 26 * $FreeBSD: stable/10/sys/amd64/vmm/vmm.c 261088 2014-01-23 20:21:39Z jhb $ 27 */ 28 29#include <sys/cdefs.h> 30__FBSDID("$FreeBSD: stable/10/sys/amd64/vmm/vmm.c 261088 2014-01-23 20:21:39Z jhb $"); 31 32#include <sys/param.h> 33#include <sys/systm.h> 34#include <sys/kernel.h> 35#include <sys/module.h> 36#include <sys/sysctl.h> 37#include <sys/malloc.h> 38#include <sys/pcpu.h> 39#include <sys/lock.h> 40#include <sys/mutex.h> 41#include <sys/proc.h> 42#include <sys/rwlock.h> 43#include <sys/sched.h> 44#include <sys/smp.h> 45#include <sys/systm.h> 46 47#include <vm/vm.h> 48#include <vm/vm_object.h> 49#include <vm/vm_page.h> 50#include <vm/pmap.h> 51#include <vm/vm_map.h> 52#include <vm/vm_extern.h> 53#include <vm/vm_param.h> 54 55#include <machine/vm.h> 56#include <machine/pcb.h> 57#include <machine/smp.h> 58#include <x86/apicreg.h> 59#include <machine/pmap.h> 60#include <machine/vmparam.h> 61 62#include <machine/vmm.h> 63#include <machine/vmm_dev.h> 64 65#include "vmm_ktr.h" 66#include "vmm_host.h" 67#include "vmm_mem.h" 68#include "vmm_util.h" 69#include "vhpet.h" 70#include "vioapic.h" 71#include "vlapic.h" 72#include "vmm_msr.h" 73#include "vmm_ipi.h" 74#include "vmm_stat.h" 75#include "vmm_lapic.h" 76 77#include "io/ppt.h" 78#include "io/iommu.h" 79 80struct vlapic; 81 82struct vcpu { 83 int flags; 84 enum vcpu_state state; 85 struct mtx mtx; 86 int hostcpu; /* host cpuid this vcpu last ran on */ 87 uint64_t guest_msrs[VMM_MSR_NUM]; 88 struct vlapic *vlapic; 89 int vcpuid; 90 struct savefpu *guestfpu; /* guest fpu state */ 91 void *stats; 92 struct vm_exit exitinfo; 93 enum x2apic_state x2apic_state; 94 int nmi_pending; 95}; 96 97#define vcpu_lock_init(v) mtx_init(&((v)->mtx), "vcpu lock", 0, MTX_SPIN) 98#define vcpu_lock(v) mtx_lock_spin(&((v)->mtx)) 99#define vcpu_unlock(v) mtx_unlock_spin(&((v)->mtx)) 100#define vcpu_assert_locked(v) mtx_assert(&((v)->mtx), MA_OWNED) 101 102struct mem_seg { 103 vm_paddr_t gpa; 104 size_t len; 105 boolean_t wired; 106 vm_object_t object; 107}; 108#define VM_MAX_MEMORY_SEGMENTS 2 109 110struct vm { 111 void *cookie; /* processor-specific data */ 112 void *iommu; /* iommu-specific data */ 113 struct vhpet *vhpet; /* virtual HPET */ 114 struct vioapic *vioapic; /* virtual ioapic */ 115 struct vmspace *vmspace; /* guest's address space */ 116 struct vcpu vcpu[VM_MAXCPU]; 117 int num_mem_segs; 118 struct mem_seg mem_segs[VM_MAX_MEMORY_SEGMENTS]; 119 char name[VM_MAX_NAMELEN]; 120 121 /* 122 * Set of active vcpus. 123 * An active vcpu is one that has been started implicitly (BSP) or 124 * explicitly (AP) by sending it a startup ipi. 125 */ 126 cpuset_t active_cpus; 127}; 128 129static int vmm_initialized; 130 131static struct vmm_ops *ops; 132#define VMM_INIT() (ops != NULL ? (*ops->init)() : 0) 133#define VMM_CLEANUP() (ops != NULL ? (*ops->cleanup)() : 0) 134 135#define VMINIT(vm, pmap) (ops != NULL ? (*ops->vminit)(vm, pmap): NULL) 136#define VMRUN(vmi, vcpu, rip, pmap) \ 137 (ops != NULL ? (*ops->vmrun)(vmi, vcpu, rip, pmap) : ENXIO) 138#define VMCLEANUP(vmi) (ops != NULL ? (*ops->vmcleanup)(vmi) : NULL) 139#define VMSPACE_ALLOC(min, max) \ 140 (ops != NULL ? (*ops->vmspace_alloc)(min, max) : NULL) 141#define VMSPACE_FREE(vmspace) \ 142 (ops != NULL ? (*ops->vmspace_free)(vmspace) : ENXIO) 143#define VMGETREG(vmi, vcpu, num, retval) \ 144 (ops != NULL ? (*ops->vmgetreg)(vmi, vcpu, num, retval) : ENXIO) 145#define VMSETREG(vmi, vcpu, num, val) \ 146 (ops != NULL ? (*ops->vmsetreg)(vmi, vcpu, num, val) : ENXIO) 147#define VMGETDESC(vmi, vcpu, num, desc) \ 148 (ops != NULL ? (*ops->vmgetdesc)(vmi, vcpu, num, desc) : ENXIO) 149#define VMSETDESC(vmi, vcpu, num, desc) \ 150 (ops != NULL ? (*ops->vmsetdesc)(vmi, vcpu, num, desc) : ENXIO) 151#define VMINJECT(vmi, vcpu, type, vec, ec, ecv) \ 152 (ops != NULL ? (*ops->vminject)(vmi, vcpu, type, vec, ec, ecv) : ENXIO) 153#define VMGETCAP(vmi, vcpu, num, retval) \ 154 (ops != NULL ? (*ops->vmgetcap)(vmi, vcpu, num, retval) : ENXIO) 155#define VMSETCAP(vmi, vcpu, num, val) \ 156 (ops != NULL ? (*ops->vmsetcap)(vmi, vcpu, num, val) : ENXIO) 157 158#define fpu_start_emulating() load_cr0(rcr0() | CR0_TS) 159#define fpu_stop_emulating() clts() 160 161static MALLOC_DEFINE(M_VM, "vm", "vm"); 162CTASSERT(VMM_MSR_NUM <= 64); /* msr_mask can keep track of up to 64 msrs */ 163 164/* statistics */ 165static VMM_STAT(VCPU_TOTAL_RUNTIME, "vcpu total runtime"); 166 167static void 168vcpu_cleanup(struct vcpu *vcpu) 169{ 170 vlapic_cleanup(vcpu->vlapic); 171 vmm_stat_free(vcpu->stats); 172 fpu_save_area_free(vcpu->guestfpu); 173} 174 175static void 176vcpu_init(struct vm *vm, uint32_t vcpu_id) 177{ 178 struct vcpu *vcpu; 179 180 vcpu = &vm->vcpu[vcpu_id]; 181 182 vcpu_lock_init(vcpu); 183 vcpu->hostcpu = NOCPU; 184 vcpu->vcpuid = vcpu_id; 185 vcpu->vlapic = vlapic_init(vm, vcpu_id); 186 vm_set_x2apic_state(vm, vcpu_id, X2APIC_ENABLED); 187 vcpu->guestfpu = fpu_save_area_alloc(); 188 fpu_save_area_reset(vcpu->guestfpu); 189 vcpu->stats = vmm_stat_alloc(); 190} 191 192struct vm_exit * 193vm_exitinfo(struct vm *vm, int cpuid) 194{ 195 struct vcpu *vcpu; 196 197 if (cpuid < 0 || cpuid >= VM_MAXCPU) 198 panic("vm_exitinfo: invalid cpuid %d", cpuid); 199 200 vcpu = &vm->vcpu[cpuid]; 201 202 return (&vcpu->exitinfo); 203} 204 205static int 206vmm_init(void) 207{ 208 int error; 209 210 vmm_host_state_init(); 211 vmm_ipi_init(); 212 213 error = vmm_mem_init(); 214 if (error) 215 return (error); 216 217 if (vmm_is_intel()) 218 ops = &vmm_ops_intel; 219 else if (vmm_is_amd()) 220 ops = &vmm_ops_amd; 221 else 222 return (ENXIO); 223 224 vmm_msr_init(); 225 226 return (VMM_INIT()); 227} 228 229static int 230vmm_handler(module_t mod, int what, void *arg) 231{ 232 int error; 233 234 switch (what) { 235 case MOD_LOAD: 236 vmmdev_init(); 237 iommu_init(); 238 error = vmm_init(); 239 if (error == 0) 240 vmm_initialized = 1; 241 break; 242 case MOD_UNLOAD: 243 error = vmmdev_cleanup(); 244 if (error == 0) { 245 iommu_cleanup(); 246 vmm_ipi_cleanup(); 247 error = VMM_CLEANUP(); 248 /* 249 * Something bad happened - prevent new 250 * VMs from being created 251 */ 252 if (error) 253 vmm_initialized = 0; 254 } 255 break; 256 default: 257 error = 0; 258 break; 259 } 260 return (error); 261} 262 263static moduledata_t vmm_kmod = { 264 "vmm", 265 vmm_handler, 266 NULL 267}; 268 269/* 270 * vmm initialization has the following dependencies: 271 * 272 * - iommu initialization must happen after the pci passthru driver has had 273 * a chance to attach to any passthru devices (after SI_SUB_CONFIGURE). 274 * 275 * - VT-x initialization requires smp_rendezvous() and therefore must happen 276 * after SMP is fully functional (after SI_SUB_SMP). 277 */ 278DECLARE_MODULE(vmm, vmm_kmod, SI_SUB_SMP + 1, SI_ORDER_ANY); 279MODULE_VERSION(vmm, 1); 280 281SYSCTL_NODE(_hw, OID_AUTO, vmm, CTLFLAG_RW, NULL, NULL); 282 283int 284vm_create(const char *name, struct vm **retvm) 285{ 286 int i; 287 struct vm *vm; 288 struct vmspace *vmspace; 289 290 const int BSP = 0; 291 292 /* 293 * If vmm.ko could not be successfully initialized then don't attempt 294 * to create the virtual machine. 295 */ 296 if (!vmm_initialized) 297 return (ENXIO); 298 299 if (name == NULL || strlen(name) >= VM_MAX_NAMELEN) 300 return (EINVAL); 301 302 vmspace = VMSPACE_ALLOC(VM_MIN_ADDRESS, VM_MAXUSER_ADDRESS); 303 if (vmspace == NULL) 304 return (ENOMEM); 305 306 vm = malloc(sizeof(struct vm), M_VM, M_WAITOK | M_ZERO); 307 strcpy(vm->name, name); 308 vm->cookie = VMINIT(vm, vmspace_pmap(vmspace)); 309 vm->vioapic = vioapic_init(vm); 310 vm->vhpet = vhpet_init(vm); 311 312 for (i = 0; i < VM_MAXCPU; i++) { 313 vcpu_init(vm, i); 314 guest_msrs_init(vm, i); 315 } 316 317 vm_activate_cpu(vm, BSP); 318 vm->vmspace = vmspace; 319 320 *retvm = vm; 321 return (0); 322} 323 324static void 325vm_free_mem_seg(struct vm *vm, struct mem_seg *seg) 326{ 327 328 if (seg->object != NULL) 329 vmm_mem_free(vm->vmspace, seg->gpa, seg->len); 330 331 bzero(seg, sizeof(*seg)); 332} 333 334void 335vm_destroy(struct vm *vm) 336{ 337 int i; 338 339 ppt_unassign_all(vm); 340 341 if (vm->iommu != NULL) 342 iommu_destroy_domain(vm->iommu); 343 344 vhpet_cleanup(vm->vhpet); 345 vioapic_cleanup(vm->vioapic); 346 347 for (i = 0; i < vm->num_mem_segs; i++) 348 vm_free_mem_seg(vm, &vm->mem_segs[i]); 349 350 vm->num_mem_segs = 0; 351 352 for (i = 0; i < VM_MAXCPU; i++) 353 vcpu_cleanup(&vm->vcpu[i]); 354 355 VMSPACE_FREE(vm->vmspace); 356 357 VMCLEANUP(vm->cookie); 358 359 free(vm, M_VM); 360} 361 362const char * 363vm_name(struct vm *vm) 364{ 365 return (vm->name); 366} 367 368int 369vm_map_mmio(struct vm *vm, vm_paddr_t gpa, size_t len, vm_paddr_t hpa) 370{ 371 vm_object_t obj; 372 373 if ((obj = vmm_mmio_alloc(vm->vmspace, gpa, len, hpa)) == NULL) 374 return (ENOMEM); 375 else 376 return (0); 377} 378 379int 380vm_unmap_mmio(struct vm *vm, vm_paddr_t gpa, size_t len) 381{ 382 383 vmm_mmio_free(vm->vmspace, gpa, len); 384 return (0); 385} 386 387boolean_t 388vm_mem_allocated(struct vm *vm, vm_paddr_t gpa) 389{ 390 int i; 391 vm_paddr_t gpabase, gpalimit; 392 393 for (i = 0; i < vm->num_mem_segs; i++) { 394 gpabase = vm->mem_segs[i].gpa; 395 gpalimit = gpabase + vm->mem_segs[i].len; 396 if (gpa >= gpabase && gpa < gpalimit) 397 return (TRUE); /* 'gpa' is regular memory */ 398 } 399 400 if (ppt_is_mmio(vm, gpa)) 401 return (TRUE); /* 'gpa' is pci passthru mmio */ 402 403 return (FALSE); 404} 405 406int 407vm_malloc(struct vm *vm, vm_paddr_t gpa, size_t len) 408{ 409 int available, allocated; 410 struct mem_seg *seg; 411 vm_object_t object; 412 vm_paddr_t g; 413 414 if ((gpa & PAGE_MASK) || (len & PAGE_MASK) || len == 0) 415 return (EINVAL); 416 417 available = allocated = 0; 418 g = gpa; 419 while (g < gpa + len) { 420 if (vm_mem_allocated(vm, g)) 421 allocated++; 422 else 423 available++; 424 425 g += PAGE_SIZE; 426 } 427 428 /* 429 * If there are some allocated and some available pages in the address 430 * range then it is an error. 431 */ 432 if (allocated && available) 433 return (EINVAL); 434 435 /* 436 * If the entire address range being requested has already been 437 * allocated then there isn't anything more to do. 438 */ 439 if (allocated && available == 0) 440 return (0); 441 442 if (vm->num_mem_segs >= VM_MAX_MEMORY_SEGMENTS) 443 return (E2BIG); 444 445 seg = &vm->mem_segs[vm->num_mem_segs]; 446 447 if ((object = vmm_mem_alloc(vm->vmspace, gpa, len)) == NULL) 448 return (ENOMEM); 449 450 seg->gpa = gpa; 451 seg->len = len; 452 seg->object = object; 453 seg->wired = FALSE; 454 455 vm->num_mem_segs++; 456 457 return (0); 458} 459 460static void 461vm_gpa_unwire(struct vm *vm) 462{ 463 int i, rv; 464 struct mem_seg *seg; 465 466 for (i = 0; i < vm->num_mem_segs; i++) { 467 seg = &vm->mem_segs[i]; 468 if (!seg->wired) 469 continue; 470 471 rv = vm_map_unwire(&vm->vmspace->vm_map, 472 seg->gpa, seg->gpa + seg->len, 473 VM_MAP_WIRE_USER | VM_MAP_WIRE_NOHOLES); 474 KASSERT(rv == KERN_SUCCESS, ("vm(%s) memory segment " 475 "%#lx/%ld could not be unwired: %d", 476 vm_name(vm), seg->gpa, seg->len, rv)); 477 478 seg->wired = FALSE; 479 } 480} 481 482static int 483vm_gpa_wire(struct vm *vm) 484{ 485 int i, rv; 486 struct mem_seg *seg; 487 488 for (i = 0; i < vm->num_mem_segs; i++) { 489 seg = &vm->mem_segs[i]; 490 if (seg->wired) 491 continue; 492 493 /* XXX rlimits? */ 494 rv = vm_map_wire(&vm->vmspace->vm_map, 495 seg->gpa, seg->gpa + seg->len, 496 VM_MAP_WIRE_USER | VM_MAP_WIRE_NOHOLES); 497 if (rv != KERN_SUCCESS) 498 break; 499 500 seg->wired = TRUE; 501 } 502 503 if (i < vm->num_mem_segs) { 504 /* 505 * Undo the wiring before returning an error. 506 */ 507 vm_gpa_unwire(vm); 508 return (EAGAIN); 509 } 510 511 return (0); 512} 513 514static void 515vm_iommu_modify(struct vm *vm, boolean_t map) 516{ 517 int i, sz; 518 vm_paddr_t gpa, hpa; 519 struct mem_seg *seg; 520 void *vp, *cookie, *host_domain; 521 522 sz = PAGE_SIZE; 523 host_domain = iommu_host_domain(); 524 525 for (i = 0; i < vm->num_mem_segs; i++) { 526 seg = &vm->mem_segs[i]; 527 KASSERT(seg->wired, ("vm(%s) memory segment %#lx/%ld not wired", 528 vm_name(vm), seg->gpa, seg->len)); 529 530 gpa = seg->gpa; 531 while (gpa < seg->gpa + seg->len) { 532 vp = vm_gpa_hold(vm, gpa, PAGE_SIZE, VM_PROT_WRITE, 533 &cookie); 534 KASSERT(vp != NULL, ("vm(%s) could not map gpa %#lx", 535 vm_name(vm), gpa)); 536 537 vm_gpa_release(cookie); 538 539 hpa = DMAP_TO_PHYS((uintptr_t)vp); 540 if (map) { 541 iommu_create_mapping(vm->iommu, gpa, hpa, sz); 542 iommu_remove_mapping(host_domain, hpa, sz); 543 } else { 544 iommu_remove_mapping(vm->iommu, gpa, sz); 545 iommu_create_mapping(host_domain, hpa, hpa, sz); 546 } 547 548 gpa += PAGE_SIZE; 549 } 550 } 551 552 /* 553 * Invalidate the cached translations associated with the domain 554 * from which pages were removed. 555 */ 556 if (map) 557 iommu_invalidate_tlb(host_domain); 558 else 559 iommu_invalidate_tlb(vm->iommu); 560} 561 562#define vm_iommu_unmap(vm) vm_iommu_modify((vm), FALSE) 563#define vm_iommu_map(vm) vm_iommu_modify((vm), TRUE) 564 565int 566vm_unassign_pptdev(struct vm *vm, int bus, int slot, int func) 567{ 568 int error; 569 570 error = ppt_unassign_device(vm, bus, slot, func); 571 if (error) 572 return (error); 573 574 if (ppt_num_devices(vm) == 0) { 575 vm_iommu_unmap(vm); 576 vm_gpa_unwire(vm); 577 } 578 return (0); 579} 580 581int 582vm_assign_pptdev(struct vm *vm, int bus, int slot, int func) 583{ 584 int error; 585 vm_paddr_t maxaddr; 586 587 /* 588 * Virtual machines with pci passthru devices get special treatment: 589 * - the guest physical memory is wired 590 * - the iommu is programmed to do the 'gpa' to 'hpa' translation 591 * 592 * We need to do this before the first pci passthru device is attached. 593 */ 594 if (ppt_num_devices(vm) == 0) { 595 KASSERT(vm->iommu == NULL, 596 ("vm_assign_pptdev: iommu must be NULL")); 597 maxaddr = vmm_mem_maxaddr(); 598 vm->iommu = iommu_create_domain(maxaddr); 599 600 error = vm_gpa_wire(vm); 601 if (error) 602 return (error); 603 604 vm_iommu_map(vm); 605 } 606 607 error = ppt_assign_device(vm, bus, slot, func); 608 return (error); 609} 610 611void * 612vm_gpa_hold(struct vm *vm, vm_paddr_t gpa, size_t len, int reqprot, 613 void **cookie) 614{ 615 int count, pageoff; 616 vm_page_t m; 617 618 pageoff = gpa & PAGE_MASK; 619 if (len > PAGE_SIZE - pageoff) 620 panic("vm_gpa_hold: invalid gpa/len: 0x%016lx/%lu", gpa, len); 621 622 count = vm_fault_quick_hold_pages(&vm->vmspace->vm_map, 623 trunc_page(gpa), PAGE_SIZE, reqprot, &m, 1); 624 625 if (count == 1) { 626 *cookie = m; 627 return ((void *)(PHYS_TO_DMAP(VM_PAGE_TO_PHYS(m)) + pageoff)); 628 } else { 629 *cookie = NULL; 630 return (NULL); 631 } 632} 633 634void 635vm_gpa_release(void *cookie) 636{ 637 vm_page_t m = cookie; 638 639 vm_page_lock(m); 640 vm_page_unhold(m); 641 vm_page_unlock(m); 642} 643 644int 645vm_gpabase2memseg(struct vm *vm, vm_paddr_t gpabase, 646 struct vm_memory_segment *seg) 647{ 648 int i; 649 650 for (i = 0; i < vm->num_mem_segs; i++) { 651 if (gpabase == vm->mem_segs[i].gpa) { 652 seg->gpa = vm->mem_segs[i].gpa; 653 seg->len = vm->mem_segs[i].len; 654 seg->wired = vm->mem_segs[i].wired; 655 return (0); 656 } 657 } 658 return (-1); 659} 660 661int 662vm_get_memobj(struct vm *vm, vm_paddr_t gpa, size_t len, 663 vm_offset_t *offset, struct vm_object **object) 664{ 665 int i; 666 size_t seg_len; 667 vm_paddr_t seg_gpa; 668 vm_object_t seg_obj; 669 670 for (i = 0; i < vm->num_mem_segs; i++) { 671 if ((seg_obj = vm->mem_segs[i].object) == NULL) 672 continue; 673 674 seg_gpa = vm->mem_segs[i].gpa; 675 seg_len = vm->mem_segs[i].len; 676 677 if (gpa >= seg_gpa && gpa < seg_gpa + seg_len) { 678 *offset = gpa - seg_gpa; 679 *object = seg_obj; 680 vm_object_reference(seg_obj); 681 return (0); 682 } 683 } 684 685 return (EINVAL); 686} 687 688int 689vm_get_register(struct vm *vm, int vcpu, int reg, uint64_t *retval) 690{ 691 692 if (vcpu < 0 || vcpu >= VM_MAXCPU) 693 return (EINVAL); 694 695 if (reg >= VM_REG_LAST) 696 return (EINVAL); 697 698 return (VMGETREG(vm->cookie, vcpu, reg, retval)); 699} 700 701int 702vm_set_register(struct vm *vm, int vcpu, int reg, uint64_t val) 703{ 704 705 if (vcpu < 0 || vcpu >= VM_MAXCPU) 706 return (EINVAL); 707 708 if (reg >= VM_REG_LAST) 709 return (EINVAL); 710 711 return (VMSETREG(vm->cookie, vcpu, reg, val)); 712} 713 714static boolean_t 715is_descriptor_table(int reg) 716{ 717 718 switch (reg) { 719 case VM_REG_GUEST_IDTR: 720 case VM_REG_GUEST_GDTR: 721 return (TRUE); 722 default: 723 return (FALSE); 724 } 725} 726 727static boolean_t 728is_segment_register(int reg) 729{ 730 731 switch (reg) { 732 case VM_REG_GUEST_ES: 733 case VM_REG_GUEST_CS: 734 case VM_REG_GUEST_SS: 735 case VM_REG_GUEST_DS: 736 case VM_REG_GUEST_FS: 737 case VM_REG_GUEST_GS: 738 case VM_REG_GUEST_TR: 739 case VM_REG_GUEST_LDTR: 740 return (TRUE); 741 default: 742 return (FALSE); 743 } 744} 745 746int 747vm_get_seg_desc(struct vm *vm, int vcpu, int reg, 748 struct seg_desc *desc) 749{ 750 751 if (vcpu < 0 || vcpu >= VM_MAXCPU) 752 return (EINVAL); 753 754 if (!is_segment_register(reg) && !is_descriptor_table(reg)) 755 return (EINVAL); 756 757 return (VMGETDESC(vm->cookie, vcpu, reg, desc)); 758} 759 760int 761vm_set_seg_desc(struct vm *vm, int vcpu, int reg, 762 struct seg_desc *desc) 763{ 764 if (vcpu < 0 || vcpu >= VM_MAXCPU) 765 return (EINVAL); 766 767 if (!is_segment_register(reg) && !is_descriptor_table(reg)) 768 return (EINVAL); 769 770 return (VMSETDESC(vm->cookie, vcpu, reg, desc)); 771} 772 773static void 774restore_guest_fpustate(struct vcpu *vcpu) 775{ 776 777 /* flush host state to the pcb */ 778 fpuexit(curthread); 779 780 /* restore guest FPU state */ 781 fpu_stop_emulating(); 782 fpurestore(vcpu->guestfpu); 783 784 /* 785 * The FPU is now "dirty" with the guest's state so turn on emulation 786 * to trap any access to the FPU by the host. 787 */ 788 fpu_start_emulating(); 789} 790 791static void 792save_guest_fpustate(struct vcpu *vcpu) 793{ 794 795 if ((rcr0() & CR0_TS) == 0) 796 panic("fpu emulation not enabled in host!"); 797 798 /* save guest FPU state */ 799 fpu_stop_emulating(); 800 fpusave(vcpu->guestfpu); 801 fpu_start_emulating(); 802} 803 804static VMM_STAT(VCPU_IDLE_TICKS, "number of ticks vcpu was idle"); 805 806static int 807vcpu_set_state_locked(struct vcpu *vcpu, enum vcpu_state newstate) 808{ 809 int error; 810 811 vcpu_assert_locked(vcpu); 812 813 /* 814 * The following state transitions are allowed: 815 * IDLE -> FROZEN -> IDLE 816 * FROZEN -> RUNNING -> FROZEN 817 * FROZEN -> SLEEPING -> FROZEN 818 */ 819 switch (vcpu->state) { 820 case VCPU_IDLE: 821 case VCPU_RUNNING: 822 case VCPU_SLEEPING: 823 error = (newstate != VCPU_FROZEN); 824 break; 825 case VCPU_FROZEN: 826 error = (newstate == VCPU_FROZEN); 827 break; 828 default: 829 error = 1; 830 break; 831 } 832 833 if (error == 0) 834 vcpu->state = newstate; 835 else 836 error = EBUSY; 837 838 return (error); 839} 840 841static void 842vcpu_require_state(struct vm *vm, int vcpuid, enum vcpu_state newstate) 843{ 844 int error; 845 846 if ((error = vcpu_set_state(vm, vcpuid, newstate)) != 0) 847 panic("Error %d setting state to %d\n", error, newstate); 848} 849 850static void 851vcpu_require_state_locked(struct vcpu *vcpu, enum vcpu_state newstate) 852{ 853 int error; 854 855 if ((error = vcpu_set_state_locked(vcpu, newstate)) != 0) 856 panic("Error %d setting state to %d", error, newstate); 857} 858 859/* 860 * Emulate a guest 'hlt' by sleeping until the vcpu is ready to run. 861 */ 862static int 863vm_handle_hlt(struct vm *vm, int vcpuid, boolean_t *retu) 864{ 865 struct vcpu *vcpu; 866 int sleepticks, t; 867 868 vcpu = &vm->vcpu[vcpuid]; 869 870 vcpu_lock(vcpu); 871 872 /* 873 * Figure out the number of host ticks until the next apic 874 * timer interrupt in the guest. 875 */ 876 sleepticks = lapic_timer_tick(vm, vcpuid); 877 878 /* 879 * If the guest local apic timer is disabled then sleep for 880 * a long time but not forever. 881 */ 882 if (sleepticks < 0) 883 sleepticks = hz; 884 885 /* 886 * Do a final check for pending NMI or interrupts before 887 * really putting this thread to sleep. 888 * 889 * These interrupts could have happened any time after we 890 * returned from VMRUN() and before we grabbed the vcpu lock. 891 */ 892 if (!vm_nmi_pending(vm, vcpuid) && lapic_pending_intr(vm, vcpuid) < 0) { 893 if (sleepticks <= 0) 894 panic("invalid sleepticks %d", sleepticks); 895 t = ticks; 896 vcpu_require_state_locked(vcpu, VCPU_SLEEPING); 897 msleep_spin(vcpu, &vcpu->mtx, "vmidle", sleepticks); 898 vcpu_require_state_locked(vcpu, VCPU_FROZEN); 899 vmm_stat_incr(vm, vcpuid, VCPU_IDLE_TICKS, ticks - t); 900 } 901 vcpu_unlock(vcpu); 902 903 return (0); 904} 905 906static int 907vm_handle_paging(struct vm *vm, int vcpuid, boolean_t *retu) 908{ 909 int rv, ftype; 910 struct vm_map *map; 911 struct vcpu *vcpu; 912 struct vm_exit *vme; 913 914 vcpu = &vm->vcpu[vcpuid]; 915 vme = &vcpu->exitinfo; 916 917 ftype = vme->u.paging.fault_type; 918 KASSERT(ftype == VM_PROT_READ || 919 ftype == VM_PROT_WRITE || ftype == VM_PROT_EXECUTE, 920 ("vm_handle_paging: invalid fault_type %d", ftype)); 921 922 if (ftype == VM_PROT_READ || ftype == VM_PROT_WRITE) { 923 rv = pmap_emulate_accessed_dirty(vmspace_pmap(vm->vmspace), 924 vme->u.paging.gpa, ftype); 925 if (rv == 0) 926 goto done; 927 } 928 929 map = &vm->vmspace->vm_map; 930 rv = vm_fault(map, vme->u.paging.gpa, ftype, VM_FAULT_NORMAL); 931 932 VCPU_CTR3(vm, vcpuid, "vm_handle_paging rv = %d, gpa = %#lx, " 933 "ftype = %d", rv, vme->u.paging.gpa, ftype); 934 935 if (rv != KERN_SUCCESS) 936 return (EFAULT); 937done: 938 /* restart execution at the faulting instruction */ 939 vme->inst_length = 0; 940 941 return (0); 942} 943 944static int 945vm_handle_inst_emul(struct vm *vm, int vcpuid, boolean_t *retu) 946{ 947 struct vie *vie; 948 struct vcpu *vcpu; 949 struct vm_exit *vme; 950 int error, inst_length; 951 uint64_t rip, gla, gpa, cr3; 952 mem_region_read_t mread; 953 mem_region_write_t mwrite; 954 955 vcpu = &vm->vcpu[vcpuid]; 956 vme = &vcpu->exitinfo; 957 958 rip = vme->rip; 959 inst_length = vme->inst_length; 960 961 gla = vme->u.inst_emul.gla; 962 gpa = vme->u.inst_emul.gpa; 963 cr3 = vme->u.inst_emul.cr3; 964 vie = &vme->u.inst_emul.vie; 965 966 vie_init(vie); 967 968 /* Fetch, decode and emulate the faulting instruction */ 969 if (vmm_fetch_instruction(vm, vcpuid, rip, inst_length, cr3, vie) != 0) 970 return (EFAULT); 971 972 if (vmm_decode_instruction(vm, vcpuid, gla, vie) != 0) 973 return (EFAULT); 974 975 /* return to userland unless this is an in-kernel emulated device */ 976 if (gpa >= DEFAULT_APIC_BASE && gpa < DEFAULT_APIC_BASE + PAGE_SIZE) { 977 mread = lapic_mmio_read; 978 mwrite = lapic_mmio_write; 979 } else if (gpa >= VIOAPIC_BASE && gpa < VIOAPIC_BASE + VIOAPIC_SIZE) { 980 mread = vioapic_mmio_read; 981 mwrite = vioapic_mmio_write; 982 } else if (gpa >= VHPET_BASE && gpa < VHPET_BASE + VHPET_SIZE) { 983 mread = vhpet_mmio_read; 984 mwrite = vhpet_mmio_write; 985 } else { 986 *retu = TRUE; 987 return (0); 988 } 989 990 error = vmm_emulate_instruction(vm, vcpuid, gpa, vie, mread, mwrite, 0); 991 992 /* return to userland to spin up the AP */ 993 if (error == 0 && vme->exitcode == VM_EXITCODE_SPINUP_AP) 994 *retu = TRUE; 995 996 return (error); 997} 998 999int 1000vm_run(struct vm *vm, struct vm_run *vmrun) 1001{ 1002 int error, vcpuid; 1003 struct vcpu *vcpu; 1004 struct pcb *pcb; 1005 uint64_t tscval, rip; 1006 struct vm_exit *vme; 1007 boolean_t retu; 1008 pmap_t pmap; 1009 1010 vcpuid = vmrun->cpuid; 1011 1012 if (vcpuid < 0 || vcpuid >= VM_MAXCPU) 1013 return (EINVAL); 1014 1015 pmap = vmspace_pmap(vm->vmspace); 1016 vcpu = &vm->vcpu[vcpuid]; 1017 vme = &vcpu->exitinfo; 1018 rip = vmrun->rip; 1019restart: 1020 critical_enter(); 1021 1022 KASSERT(!CPU_ISSET(curcpu, &pmap->pm_active), 1023 ("vm_run: absurd pm_active")); 1024 1025 tscval = rdtsc(); 1026 1027 pcb = PCPU_GET(curpcb); 1028 set_pcb_flags(pcb, PCB_FULL_IRET); 1029 1030 restore_guest_msrs(vm, vcpuid); 1031 restore_guest_fpustate(vcpu); 1032 1033 vcpu_require_state(vm, vcpuid, VCPU_RUNNING); 1034 vcpu->hostcpu = curcpu; 1035 error = VMRUN(vm->cookie, vcpuid, rip, pmap); 1036 vcpu->hostcpu = NOCPU; 1037 vcpu_require_state(vm, vcpuid, VCPU_FROZEN); 1038 1039 save_guest_fpustate(vcpu); 1040 restore_host_msrs(vm, vcpuid); 1041 1042 vmm_stat_incr(vm, vcpuid, VCPU_TOTAL_RUNTIME, rdtsc() - tscval); 1043 1044 critical_exit(); 1045 1046 if (error == 0) { 1047 retu = FALSE; 1048 switch (vme->exitcode) { 1049 case VM_EXITCODE_HLT: 1050 error = vm_handle_hlt(vm, vcpuid, &retu); 1051 break; 1052 case VM_EXITCODE_PAGING: 1053 error = vm_handle_paging(vm, vcpuid, &retu); 1054 break; 1055 case VM_EXITCODE_INST_EMUL: 1056 error = vm_handle_inst_emul(vm, vcpuid, &retu); 1057 break; 1058 default: 1059 retu = TRUE; /* handled in userland */ 1060 break; 1061 } 1062 } 1063 1064 if (error == 0 && retu == FALSE) { 1065 rip = vme->rip + vme->inst_length; 1066 goto restart; 1067 } 1068 1069 /* copy the exit information */ 1070 bcopy(vme, &vmrun->vm_exit, sizeof(struct vm_exit)); 1071 return (error); 1072} 1073 1074int 1075vm_inject_event(struct vm *vm, int vcpuid, int type, 1076 int vector, uint32_t code, int code_valid) 1077{ 1078 if (vcpuid < 0 || vcpuid >= VM_MAXCPU) 1079 return (EINVAL); 1080 1081 if ((type > VM_EVENT_NONE && type < VM_EVENT_MAX) == 0) 1082 return (EINVAL); 1083 1084 if (vector < 0 || vector > 255) 1085 return (EINVAL); 1086 1087 return (VMINJECT(vm->cookie, vcpuid, type, vector, code, code_valid)); 1088} 1089 1090static VMM_STAT(VCPU_NMI_COUNT, "number of NMIs delivered to vcpu"); 1091 1092int 1093vm_inject_nmi(struct vm *vm, int vcpuid) 1094{ 1095 struct vcpu *vcpu; 1096 1097 if (vcpuid < 0 || vcpuid >= VM_MAXCPU) 1098 return (EINVAL); 1099 1100 vcpu = &vm->vcpu[vcpuid]; 1101 1102 vcpu->nmi_pending = 1; 1103 vm_interrupt_hostcpu(vm, vcpuid); 1104 return (0); 1105} 1106 1107int 1108vm_nmi_pending(struct vm *vm, int vcpuid) 1109{ 1110 struct vcpu *vcpu; 1111 1112 if (vcpuid < 0 || vcpuid >= VM_MAXCPU) 1113 panic("vm_nmi_pending: invalid vcpuid %d", vcpuid); 1114 1115 vcpu = &vm->vcpu[vcpuid]; 1116 1117 return (vcpu->nmi_pending); 1118} 1119 1120void 1121vm_nmi_clear(struct vm *vm, int vcpuid) 1122{ 1123 struct vcpu *vcpu; 1124 1125 if (vcpuid < 0 || vcpuid >= VM_MAXCPU) 1126 panic("vm_nmi_pending: invalid vcpuid %d", vcpuid); 1127 1128 vcpu = &vm->vcpu[vcpuid]; 1129 1130 if (vcpu->nmi_pending == 0) 1131 panic("vm_nmi_clear: inconsistent nmi_pending state"); 1132 1133 vcpu->nmi_pending = 0; 1134 vmm_stat_incr(vm, vcpuid, VCPU_NMI_COUNT, 1); 1135} 1136 1137int 1138vm_get_capability(struct vm *vm, int vcpu, int type, int *retval) 1139{ 1140 if (vcpu < 0 || vcpu >= VM_MAXCPU) 1141 return (EINVAL); 1142 1143 if (type < 0 || type >= VM_CAP_MAX) 1144 return (EINVAL); 1145 1146 return (VMGETCAP(vm->cookie, vcpu, type, retval)); 1147} 1148 1149int 1150vm_set_capability(struct vm *vm, int vcpu, int type, int val) 1151{ 1152 if (vcpu < 0 || vcpu >= VM_MAXCPU) 1153 return (EINVAL); 1154 1155 if (type < 0 || type >= VM_CAP_MAX) 1156 return (EINVAL); 1157 1158 return (VMSETCAP(vm->cookie, vcpu, type, val)); 1159} 1160 1161uint64_t * 1162vm_guest_msrs(struct vm *vm, int cpu) 1163{ 1164 return (vm->vcpu[cpu].guest_msrs); 1165} 1166 1167struct vlapic * 1168vm_lapic(struct vm *vm, int cpu) 1169{ 1170 return (vm->vcpu[cpu].vlapic); 1171} 1172 1173struct vioapic * 1174vm_ioapic(struct vm *vm) 1175{ 1176 1177 return (vm->vioapic); 1178} 1179 1180struct vhpet * 1181vm_hpet(struct vm *vm) 1182{ 1183 1184 return (vm->vhpet); 1185} 1186 1187boolean_t 1188vmm_is_pptdev(int bus, int slot, int func) 1189{ 1190 int found, i, n; 1191 int b, s, f; 1192 char *val, *cp, *cp2; 1193 1194 /* 1195 * XXX 1196 * The length of an environment variable is limited to 128 bytes which 1197 * puts an upper limit on the number of passthru devices that may be 1198 * specified using a single environment variable. 1199 * 1200 * Work around this by scanning multiple environment variable 1201 * names instead of a single one - yuck! 1202 */ 1203 const char *names[] = { "pptdevs", "pptdevs2", "pptdevs3", NULL }; 1204 1205 /* set pptdevs="1/2/3 4/5/6 7/8/9 10/11/12" */ 1206 found = 0; 1207 for (i = 0; names[i] != NULL && !found; i++) { 1208 cp = val = getenv(names[i]); 1209 while (cp != NULL && *cp != '\0') { 1210 if ((cp2 = strchr(cp, ' ')) != NULL) 1211 *cp2 = '\0'; 1212 1213 n = sscanf(cp, "%d/%d/%d", &b, &s, &f); 1214 if (n == 3 && bus == b && slot == s && func == f) { 1215 found = 1; 1216 break; 1217 } 1218 1219 if (cp2 != NULL) 1220 *cp2++ = ' '; 1221 1222 cp = cp2; 1223 } 1224 freeenv(val); 1225 } 1226 return (found); 1227} 1228 1229void * 1230vm_iommu_domain(struct vm *vm) 1231{ 1232 1233 return (vm->iommu); 1234} 1235 1236int 1237vcpu_set_state(struct vm *vm, int vcpuid, enum vcpu_state newstate) 1238{ 1239 int error; 1240 struct vcpu *vcpu; 1241 1242 if (vcpuid < 0 || vcpuid >= VM_MAXCPU) 1243 panic("vm_set_run_state: invalid vcpuid %d", vcpuid); 1244 1245 vcpu = &vm->vcpu[vcpuid]; 1246 1247 vcpu_lock(vcpu); 1248 error = vcpu_set_state_locked(vcpu, newstate); 1249 vcpu_unlock(vcpu); 1250 1251 return (error); 1252} 1253 1254enum vcpu_state 1255vcpu_get_state(struct vm *vm, int vcpuid, int *hostcpu) 1256{ 1257 struct vcpu *vcpu; 1258 enum vcpu_state state; 1259 1260 if (vcpuid < 0 || vcpuid >= VM_MAXCPU) 1261 panic("vm_get_run_state: invalid vcpuid %d", vcpuid); 1262 1263 vcpu = &vm->vcpu[vcpuid]; 1264 1265 vcpu_lock(vcpu); 1266 state = vcpu->state; 1267 if (hostcpu != NULL) 1268 *hostcpu = vcpu->hostcpu; 1269 vcpu_unlock(vcpu); 1270 1271 return (state); 1272} 1273 1274void 1275vm_activate_cpu(struct vm *vm, int vcpuid) 1276{ 1277 1278 if (vcpuid >= 0 && vcpuid < VM_MAXCPU) 1279 CPU_SET(vcpuid, &vm->active_cpus); 1280} 1281 1282cpuset_t 1283vm_active_cpus(struct vm *vm) 1284{ 1285 1286 return (vm->active_cpus); 1287} 1288 1289void * 1290vcpu_stats(struct vm *vm, int vcpuid) 1291{ 1292 1293 return (vm->vcpu[vcpuid].stats); 1294} 1295 1296int 1297vm_get_x2apic_state(struct vm *vm, int vcpuid, enum x2apic_state *state) 1298{ 1299 if (vcpuid < 0 || vcpuid >= VM_MAXCPU) 1300 return (EINVAL); 1301 1302 *state = vm->vcpu[vcpuid].x2apic_state; 1303 1304 return (0); 1305} 1306 1307int 1308vm_set_x2apic_state(struct vm *vm, int vcpuid, enum x2apic_state state) 1309{ 1310 if (vcpuid < 0 || vcpuid >= VM_MAXCPU) 1311 return (EINVAL); 1312 1313 if (state >= X2APIC_STATE_LAST) 1314 return (EINVAL); 1315 1316 vm->vcpu[vcpuid].x2apic_state = state; 1317 1318 vlapic_set_x2apic_state(vm, vcpuid, state); 1319 1320 return (0); 1321} 1322 1323void 1324vm_interrupt_hostcpu(struct vm *vm, int vcpuid) 1325{ 1326 int hostcpu; 1327 struct vcpu *vcpu; 1328 1329 vcpu = &vm->vcpu[vcpuid]; 1330 1331 vcpu_lock(vcpu); 1332 hostcpu = vcpu->hostcpu; 1333 if (hostcpu == NOCPU) { 1334 if (vcpu->state == VCPU_SLEEPING) 1335 wakeup_one(vcpu); 1336 } else { 1337 if (vcpu->state != VCPU_RUNNING) 1338 panic("invalid vcpu state %d", vcpu->state); 1339 if (hostcpu != curcpu) 1340 ipi_cpu(hostcpu, vmm_ipinum); 1341 } 1342 vcpu_unlock(vcpu); 1343} 1344 1345struct vmspace * 1346vm_get_vmspace(struct vm *vm) 1347{ 1348 1349 return (vm->vmspace); 1350} 1351 1352int 1353vm_apicid2vcpuid(struct vm *vm, int apicid) 1354{ 1355 /* 1356 * XXX apic id is assumed to be numerically identical to vcpu id 1357 */ 1358 return (apicid); 1359} 1360