31
32#include <sys/param.h>
33#include <sys/systm.h>
34#include <sys/kernel.h>
35#include <sys/module.h>
36#include <sys/sysctl.h>
37#include <sys/malloc.h>
38#include <sys/pcpu.h>
39#include <sys/lock.h>
40#include <sys/mutex.h>
41#include <sys/proc.h>
42#include <sys/rwlock.h>
43#include <sys/sched.h>
44#include <sys/smp.h>
45#include <sys/systm.h>
46
47#include <vm/vm.h>
48#include <vm/vm_object.h>
49#include <vm/vm_page.h>
50#include <vm/pmap.h>
51#include <vm/vm_map.h>
52#include <vm/vm_extern.h>
53#include <vm/vm_param.h>
54
55#include <machine/cpu.h>
56#include <machine/vm.h>
57#include <machine/pcb.h>
58#include <machine/smp.h>
59#include <x86/psl.h>
60#include <x86/apicreg.h>
61#include <machine/vmparam.h>
62
63#include <machine/vmm.h>
64#include <machine/vmm_dev.h>
65#include <machine/vmm_instruction_emul.h>
66
67#include "vmm_ioport.h"
68#include "vmm_ktr.h"
69#include "vmm_host.h"
70#include "vmm_mem.h"
71#include "vmm_util.h"
72#include "vatpic.h"
73#include "vatpit.h"
74#include "vhpet.h"
75#include "vioapic.h"
76#include "vlapic.h"
77#include "vmm_ipi.h"
78#include "vmm_stat.h"
79#include "vmm_lapic.h"
80
81#include "io/ppt.h"
82#include "io/iommu.h"
83
84struct vlapic;
85
86/*
87 * Initialization:
88 * (a) allocated when vcpu is created
89 * (i) initialized when vcpu is created and when it is reinitialized
90 * (o) initialized the first time the vcpu is created
91 * (x) initialized before use
92 */
93struct vcpu {
94 struct mtx mtx; /* (o) protects 'state' and 'hostcpu' */
95 enum vcpu_state state; /* (o) vcpu state */
96 int hostcpu; /* (o) vcpu's host cpu */
97 struct vlapic *vlapic; /* (i) APIC device model */
98 enum x2apic_state x2apic_state; /* (i) APIC mode */
99 uint64_t exitintinfo; /* (i) events pending at VM exit */
100 int nmi_pending; /* (i) NMI pending */
101 int extint_pending; /* (i) INTR pending */
102 struct vm_exception exception; /* (x) exception collateral */
103 int exception_pending; /* (i) exception pending */
104 struct savefpu *guestfpu; /* (a,i) guest fpu state */
105 uint64_t guest_xcr0; /* (i) guest %xcr0 register */
106 void *stats; /* (a,i) statistics */
107 struct vm_exit exitinfo; /* (x) exit reason and collateral */
108};
109
110#define vcpu_lock_initialized(v) mtx_initialized(&((v)->mtx))
111#define vcpu_lock_init(v) mtx_init(&((v)->mtx), "vcpu lock", 0, MTX_SPIN)
112#define vcpu_lock(v) mtx_lock_spin(&((v)->mtx))
113#define vcpu_unlock(v) mtx_unlock_spin(&((v)->mtx))
114#define vcpu_assert_locked(v) mtx_assert(&((v)->mtx), MA_OWNED)
115
116struct mem_seg {
117 vm_paddr_t gpa;
118 size_t len;
119 boolean_t wired;
120 vm_object_t object;
121};
122#define VM_MAX_MEMORY_SEGMENTS 2
123
124/*
125 * Initialization:
126 * (o) initialized the first time the VM is created
127 * (i) initialized when VM is created and when it is reinitialized
128 * (x) initialized before use
129 */
130struct vm {
131 void *cookie; /* (i) cpu-specific data */
132 void *iommu; /* (x) iommu-specific data */
133 struct vhpet *vhpet; /* (i) virtual HPET */
134 struct vioapic *vioapic; /* (i) virtual ioapic */
135 struct vatpic *vatpic; /* (i) virtual atpic */
136 struct vatpit *vatpit; /* (i) virtual atpit */
137 volatile cpuset_t active_cpus; /* (i) active vcpus */
138 int suspend; /* (i) stop VM execution */
139 volatile cpuset_t suspended_cpus; /* (i) suspended vcpus */
140 volatile cpuset_t halted_cpus; /* (x) cpus in a hard halt */
141 cpuset_t rendezvous_req_cpus; /* (x) rendezvous requested */
142 cpuset_t rendezvous_done_cpus; /* (x) rendezvous finished */
143 void *rendezvous_arg; /* (x) rendezvous func/arg */
144 vm_rendezvous_func_t rendezvous_func;
145 struct mtx rendezvous_mtx; /* (o) rendezvous lock */
146 int num_mem_segs; /* (o) guest memory segments */
147 struct mem_seg mem_segs[VM_MAX_MEMORY_SEGMENTS];
148 struct vmspace *vmspace; /* (o) guest's address space */
149 char name[VM_MAX_NAMELEN]; /* (o) virtual machine name */
150 struct vcpu vcpu[VM_MAXCPU]; /* (i) guest vcpus */
151};
152
153static int vmm_initialized;
154
155static struct vmm_ops *ops;
156#define VMM_INIT(num) (ops != NULL ? (*ops->init)(num) : 0)
157#define VMM_CLEANUP() (ops != NULL ? (*ops->cleanup)() : 0)
158#define VMM_RESUME() (ops != NULL ? (*ops->resume)() : 0)
159
160#define VMINIT(vm, pmap) (ops != NULL ? (*ops->vminit)(vm, pmap): NULL)
161#define VMRUN(vmi, vcpu, rip, pmap, rptr, sptr) \
162 (ops != NULL ? (*ops->vmrun)(vmi, vcpu, rip, pmap, rptr, sptr) : ENXIO)
163#define VMCLEANUP(vmi) (ops != NULL ? (*ops->vmcleanup)(vmi) : NULL)
164#define VMSPACE_ALLOC(min, max) \
165 (ops != NULL ? (*ops->vmspace_alloc)(min, max) : NULL)
166#define VMSPACE_FREE(vmspace) \
167 (ops != NULL ? (*ops->vmspace_free)(vmspace) : ENXIO)
168#define VMGETREG(vmi, vcpu, num, retval) \
169 (ops != NULL ? (*ops->vmgetreg)(vmi, vcpu, num, retval) : ENXIO)
170#define VMSETREG(vmi, vcpu, num, val) \
171 (ops != NULL ? (*ops->vmsetreg)(vmi, vcpu, num, val) : ENXIO)
172#define VMGETDESC(vmi, vcpu, num, desc) \
173 (ops != NULL ? (*ops->vmgetdesc)(vmi, vcpu, num, desc) : ENXIO)
174#define VMSETDESC(vmi, vcpu, num, desc) \
175 (ops != NULL ? (*ops->vmsetdesc)(vmi, vcpu, num, desc) : ENXIO)
176#define VMGETCAP(vmi, vcpu, num, retval) \
177 (ops != NULL ? (*ops->vmgetcap)(vmi, vcpu, num, retval) : ENXIO)
178#define VMSETCAP(vmi, vcpu, num, val) \
179 (ops != NULL ? (*ops->vmsetcap)(vmi, vcpu, num, val) : ENXIO)
180#define VLAPIC_INIT(vmi, vcpu) \
181 (ops != NULL ? (*ops->vlapic_init)(vmi, vcpu) : NULL)
182#define VLAPIC_CLEANUP(vmi, vlapic) \
183 (ops != NULL ? (*ops->vlapic_cleanup)(vmi, vlapic) : NULL)
184
185#define fpu_start_emulating() load_cr0(rcr0() | CR0_TS)
186#define fpu_stop_emulating() clts()
187
188static MALLOC_DEFINE(M_VM, "vm", "vm");
189
190/* statistics */
191static VMM_STAT(VCPU_TOTAL_RUNTIME, "vcpu total runtime");
192
193SYSCTL_NODE(_hw, OID_AUTO, vmm, CTLFLAG_RW, NULL, NULL);
194
195/*
196 * Halt the guest if all vcpus are executing a HLT instruction with
197 * interrupts disabled.
198 */
199static int halt_detection_enabled = 1;
200SYSCTL_INT(_hw_vmm, OID_AUTO, halt_detection, CTLFLAG_RDTUN,
201 &halt_detection_enabled, 0,
202 "Halt VM if all vcpus execute HLT with interrupts disabled");
203
204static int vmm_ipinum;
205SYSCTL_INT(_hw_vmm, OID_AUTO, ipinum, CTLFLAG_RD, &vmm_ipinum, 0,
206 "IPI vector used for vcpu notifications");
207
208static void
209vcpu_cleanup(struct vm *vm, int i, bool destroy)
210{
211 struct vcpu *vcpu = &vm->vcpu[i];
212
213 VLAPIC_CLEANUP(vm->cookie, vcpu->vlapic);
214 if (destroy) {
215 vmm_stat_free(vcpu->stats);
216 fpu_save_area_free(vcpu->guestfpu);
217 }
218}
219
220static void
221vcpu_init(struct vm *vm, int vcpu_id, bool create)
222{
223 struct vcpu *vcpu;
224
225 KASSERT(vcpu_id >= 0 && vcpu_id < VM_MAXCPU,
226 ("vcpu_init: invalid vcpu %d", vcpu_id));
227
228 vcpu = &vm->vcpu[vcpu_id];
229
230 if (create) {
231 KASSERT(!vcpu_lock_initialized(vcpu), ("vcpu %d already "
232 "initialized", vcpu_id));
233 vcpu_lock_init(vcpu);
234 vcpu->state = VCPU_IDLE;
235 vcpu->hostcpu = NOCPU;
236 vcpu->guestfpu = fpu_save_area_alloc();
237 vcpu->stats = vmm_stat_alloc();
238 }
239
240 vcpu->vlapic = VLAPIC_INIT(vm->cookie, vcpu_id);
241 vm_set_x2apic_state(vm, vcpu_id, X2APIC_DISABLED);
242 vcpu->exitintinfo = 0;
243 vcpu->nmi_pending = 0;
244 vcpu->extint_pending = 0;
245 vcpu->exception_pending = 0;
246 vcpu->guest_xcr0 = XFEATURE_ENABLED_X87;
247 fpu_save_area_reset(vcpu->guestfpu);
248 vmm_stat_init(vcpu->stats);
249}
250
251struct vm_exit *
252vm_exitinfo(struct vm *vm, int cpuid)
253{
254 struct vcpu *vcpu;
255
256 if (cpuid < 0 || cpuid >= VM_MAXCPU)
257 panic("vm_exitinfo: invalid cpuid %d", cpuid);
258
259 vcpu = &vm->vcpu[cpuid];
260
261 return (&vcpu->exitinfo);
262}
263
264static void
265vmm_resume(void)
266{
267 VMM_RESUME();
268}
269
270static int
271vmm_init(void)
272{
273 int error;
274
275 vmm_host_state_init();
276
277 vmm_ipinum = vmm_ipi_alloc();
278 if (vmm_ipinum == 0)
279 vmm_ipinum = IPI_AST;
280
281 error = vmm_mem_init();
282 if (error)
283 return (error);
284
285 if (vmm_is_intel())
286 ops = &vmm_ops_intel;
287 else if (vmm_is_amd())
288 ops = &vmm_ops_amd;
289 else
290 return (ENXIO);
291
292 vmm_resume_p = vmm_resume;
293
294 return (VMM_INIT(vmm_ipinum));
295}
296
297static int
298vmm_handler(module_t mod, int what, void *arg)
299{
300 int error;
301
302 switch (what) {
303 case MOD_LOAD:
304 vmmdev_init();
305 if (ppt_avail_devices() > 0)
306 iommu_init();
307 error = vmm_init();
308 if (error == 0)
309 vmm_initialized = 1;
310 break;
311 case MOD_UNLOAD:
312 error = vmmdev_cleanup();
313 if (error == 0) {
314 vmm_resume_p = NULL;
315 iommu_cleanup();
316 if (vmm_ipinum != IPI_AST)
317 vmm_ipi_free(vmm_ipinum);
318 error = VMM_CLEANUP();
319 /*
320 * Something bad happened - prevent new
321 * VMs from being created
322 */
323 if (error)
324 vmm_initialized = 0;
325 }
326 break;
327 default:
328 error = 0;
329 break;
330 }
331 return (error);
332}
333
334static moduledata_t vmm_kmod = {
335 "vmm",
336 vmm_handler,
337 NULL
338};
339
340/*
341 * vmm initialization has the following dependencies:
342 *
343 * - iommu initialization must happen after the pci passthru driver has had
344 * a chance to attach to any passthru devices (after SI_SUB_CONFIGURE).
345 *
346 * - VT-x initialization requires smp_rendezvous() and therefore must happen
347 * after SMP is fully functional (after SI_SUB_SMP).
348 */
349DECLARE_MODULE(vmm, vmm_kmod, SI_SUB_SMP + 1, SI_ORDER_ANY);
350MODULE_VERSION(vmm, 1);
351
352static void
353vm_init(struct vm *vm, bool create)
354{
355 int i;
356
357 vm->cookie = VMINIT(vm, vmspace_pmap(vm->vmspace));
358 vm->iommu = NULL;
359 vm->vioapic = vioapic_init(vm);
360 vm->vhpet = vhpet_init(vm);
361 vm->vatpic = vatpic_init(vm);
362 vm->vatpit = vatpit_init(vm);
363
364 CPU_ZERO(&vm->active_cpus);
365
366 vm->suspend = 0;
367 CPU_ZERO(&vm->suspended_cpus);
368
369 for (i = 0; i < VM_MAXCPU; i++)
370 vcpu_init(vm, i, create);
371}
372
373int
374vm_create(const char *name, struct vm **retvm)
375{
376 struct vm *vm;
377 struct vmspace *vmspace;
378
379 /*
380 * If vmm.ko could not be successfully initialized then don't attempt
381 * to create the virtual machine.
382 */
383 if (!vmm_initialized)
384 return (ENXIO);
385
386 if (name == NULL || strlen(name) >= VM_MAX_NAMELEN)
387 return (EINVAL);
388
389 vmspace = VMSPACE_ALLOC(VM_MIN_ADDRESS, VM_MAXUSER_ADDRESS);
390 if (vmspace == NULL)
391 return (ENOMEM);
392
393 vm = malloc(sizeof(struct vm), M_VM, M_WAITOK | M_ZERO);
394 strcpy(vm->name, name);
395 vm->num_mem_segs = 0;
396 vm->vmspace = vmspace;
397 mtx_init(&vm->rendezvous_mtx, "vm rendezvous lock", 0, MTX_DEF);
398
399 vm_init(vm, true);
400
401 *retvm = vm;
402 return (0);
403}
404
405static void
406vm_free_mem_seg(struct vm *vm, struct mem_seg *seg)
407{
408
409 if (seg->object != NULL)
410 vmm_mem_free(vm->vmspace, seg->gpa, seg->len);
411
412 bzero(seg, sizeof(*seg));
413}
414
415static void
416vm_cleanup(struct vm *vm, bool destroy)
417{
418 int i;
419
420 ppt_unassign_all(vm);
421
422 if (vm->iommu != NULL)
423 iommu_destroy_domain(vm->iommu);
424
425 vatpit_cleanup(vm->vatpit);
426 vhpet_cleanup(vm->vhpet);
427 vatpic_cleanup(vm->vatpic);
428 vioapic_cleanup(vm->vioapic);
429
430 for (i = 0; i < VM_MAXCPU; i++)
431 vcpu_cleanup(vm, i, destroy);
432
433 VMCLEANUP(vm->cookie);
434
435 if (destroy) {
436 for (i = 0; i < vm->num_mem_segs; i++)
437 vm_free_mem_seg(vm, &vm->mem_segs[i]);
438
439 vm->num_mem_segs = 0;
440
441 VMSPACE_FREE(vm->vmspace);
442 vm->vmspace = NULL;
443 }
444}
445
446void
447vm_destroy(struct vm *vm)
448{
449 vm_cleanup(vm, true);
450 free(vm, M_VM);
451}
452
453int
454vm_reinit(struct vm *vm)
455{
456 int error;
457
458 /*
459 * A virtual machine can be reset only if all vcpus are suspended.
460 */
461 if (CPU_CMP(&vm->suspended_cpus, &vm->active_cpus) == 0) {
462 vm_cleanup(vm, false);
463 vm_init(vm, false);
464 error = 0;
465 } else {
466 error = EBUSY;
467 }
468
469 return (error);
470}
471
472const char *
473vm_name(struct vm *vm)
474{
475 return (vm->name);
476}
477
478int
479vm_map_mmio(struct vm *vm, vm_paddr_t gpa, size_t len, vm_paddr_t hpa)
480{
481 vm_object_t obj;
482
483 if ((obj = vmm_mmio_alloc(vm->vmspace, gpa, len, hpa)) == NULL)
484 return (ENOMEM);
485 else
486 return (0);
487}
488
489int
490vm_unmap_mmio(struct vm *vm, vm_paddr_t gpa, size_t len)
491{
492
493 vmm_mmio_free(vm->vmspace, gpa, len);
494 return (0);
495}
496
497boolean_t
498vm_mem_allocated(struct vm *vm, vm_paddr_t gpa)
499{
500 int i;
501 vm_paddr_t gpabase, gpalimit;
502
503 for (i = 0; i < vm->num_mem_segs; i++) {
504 gpabase = vm->mem_segs[i].gpa;
505 gpalimit = gpabase + vm->mem_segs[i].len;
506 if (gpa >= gpabase && gpa < gpalimit)
507 return (TRUE); /* 'gpa' is regular memory */
508 }
509
510 if (ppt_is_mmio(vm, gpa))
511 return (TRUE); /* 'gpa' is pci passthru mmio */
512
513 return (FALSE);
514}
515
516int
517vm_malloc(struct vm *vm, vm_paddr_t gpa, size_t len)
518{
519 int available, allocated;
520 struct mem_seg *seg;
521 vm_object_t object;
522 vm_paddr_t g;
523
524 if ((gpa & PAGE_MASK) || (len & PAGE_MASK) || len == 0)
525 return (EINVAL);
526
527 available = allocated = 0;
528 g = gpa;
529 while (g < gpa + len) {
530 if (vm_mem_allocated(vm, g))
531 allocated++;
532 else
533 available++;
534
535 g += PAGE_SIZE;
536 }
537
538 /*
539 * If there are some allocated and some available pages in the address
540 * range then it is an error.
541 */
542 if (allocated && available)
543 return (EINVAL);
544
545 /*
546 * If the entire address range being requested has already been
547 * allocated then there isn't anything more to do.
548 */
549 if (allocated && available == 0)
550 return (0);
551
552 if (vm->num_mem_segs >= VM_MAX_MEMORY_SEGMENTS)
553 return (E2BIG);
554
555 seg = &vm->mem_segs[vm->num_mem_segs];
556
557 if ((object = vmm_mem_alloc(vm->vmspace, gpa, len)) == NULL)
558 return (ENOMEM);
559
560 seg->gpa = gpa;
561 seg->len = len;
562 seg->object = object;
563 seg->wired = FALSE;
564
565 vm->num_mem_segs++;
566
567 return (0);
568}
569
570static vm_paddr_t
571vm_maxmem(struct vm *vm)
572{
573 int i;
574 vm_paddr_t gpa, maxmem;
575
576 maxmem = 0;
577 for (i = 0; i < vm->num_mem_segs; i++) {
578 gpa = vm->mem_segs[i].gpa + vm->mem_segs[i].len;
579 if (gpa > maxmem)
580 maxmem = gpa;
581 }
582 return (maxmem);
583}
584
585static void
586vm_gpa_unwire(struct vm *vm)
587{
588 int i, rv;
589 struct mem_seg *seg;
590
591 for (i = 0; i < vm->num_mem_segs; i++) {
592 seg = &vm->mem_segs[i];
593 if (!seg->wired)
594 continue;
595
596 rv = vm_map_unwire(&vm->vmspace->vm_map,
597 seg->gpa, seg->gpa + seg->len,
598 VM_MAP_WIRE_USER | VM_MAP_WIRE_NOHOLES);
599 KASSERT(rv == KERN_SUCCESS, ("vm(%s) memory segment "
600 "%#lx/%ld could not be unwired: %d",
601 vm_name(vm), seg->gpa, seg->len, rv));
602
603 seg->wired = FALSE;
604 }
605}
606
607static int
608vm_gpa_wire(struct vm *vm)
609{
610 int i, rv;
611 struct mem_seg *seg;
612
613 for (i = 0; i < vm->num_mem_segs; i++) {
614 seg = &vm->mem_segs[i];
615 if (seg->wired)
616 continue;
617
618 /* XXX rlimits? */
619 rv = vm_map_wire(&vm->vmspace->vm_map,
620 seg->gpa, seg->gpa + seg->len,
621 VM_MAP_WIRE_USER | VM_MAP_WIRE_NOHOLES);
622 if (rv != KERN_SUCCESS)
623 break;
624
625 seg->wired = TRUE;
626 }
627
628 if (i < vm->num_mem_segs) {
629 /*
630 * Undo the wiring before returning an error.
631 */
632 vm_gpa_unwire(vm);
633 return (EAGAIN);
634 }
635
636 return (0);
637}
638
639static void
640vm_iommu_modify(struct vm *vm, boolean_t map)
641{
642 int i, sz;
643 vm_paddr_t gpa, hpa;
644 struct mem_seg *seg;
645 void *vp, *cookie, *host_domain;
646
647 sz = PAGE_SIZE;
648 host_domain = iommu_host_domain();
649
650 for (i = 0; i < vm->num_mem_segs; i++) {
651 seg = &vm->mem_segs[i];
652 KASSERT(seg->wired, ("vm(%s) memory segment %#lx/%ld not wired",
653 vm_name(vm), seg->gpa, seg->len));
654
655 gpa = seg->gpa;
656 while (gpa < seg->gpa + seg->len) {
657 vp = vm_gpa_hold(vm, gpa, PAGE_SIZE, VM_PROT_WRITE,
658 &cookie);
659 KASSERT(vp != NULL, ("vm(%s) could not map gpa %#lx",
660 vm_name(vm), gpa));
661
662 vm_gpa_release(cookie);
663
664 hpa = DMAP_TO_PHYS((uintptr_t)vp);
665 if (map) {
666 iommu_create_mapping(vm->iommu, gpa, hpa, sz);
667 iommu_remove_mapping(host_domain, hpa, sz);
668 } else {
669 iommu_remove_mapping(vm->iommu, gpa, sz);
670 iommu_create_mapping(host_domain, hpa, hpa, sz);
671 }
672
673 gpa += PAGE_SIZE;
674 }
675 }
676
677 /*
678 * Invalidate the cached translations associated with the domain
679 * from which pages were removed.
680 */
681 if (map)
682 iommu_invalidate_tlb(host_domain);
683 else
684 iommu_invalidate_tlb(vm->iommu);
685}
686
687#define vm_iommu_unmap(vm) vm_iommu_modify((vm), FALSE)
688#define vm_iommu_map(vm) vm_iommu_modify((vm), TRUE)
689
690int
691vm_unassign_pptdev(struct vm *vm, int bus, int slot, int func)
692{
693 int error;
694
695 error = ppt_unassign_device(vm, bus, slot, func);
696 if (error)
697 return (error);
698
699 if (ppt_assigned_devices(vm) == 0) {
700 vm_iommu_unmap(vm);
701 vm_gpa_unwire(vm);
702 }
703 return (0);
704}
705
706int
707vm_assign_pptdev(struct vm *vm, int bus, int slot, int func)
708{
709 int error;
710 vm_paddr_t maxaddr;
711
712 /*
713 * Virtual machines with pci passthru devices get special treatment:
714 * - the guest physical memory is wired
715 * - the iommu is programmed to do the 'gpa' to 'hpa' translation
716 *
717 * We need to do this before the first pci passthru device is attached.
718 */
719 if (ppt_assigned_devices(vm) == 0) {
720 KASSERT(vm->iommu == NULL,
721 ("vm_assign_pptdev: iommu must be NULL"));
722 maxaddr = vm_maxmem(vm);
723 vm->iommu = iommu_create_domain(maxaddr);
724
725 error = vm_gpa_wire(vm);
726 if (error)
727 return (error);
728
729 vm_iommu_map(vm);
730 }
731
732 error = ppt_assign_device(vm, bus, slot, func);
733 return (error);
734}
735
736void *
737vm_gpa_hold(struct vm *vm, vm_paddr_t gpa, size_t len, int reqprot,
738 void **cookie)
739{
740 int count, pageoff;
741 vm_page_t m;
742
743 pageoff = gpa & PAGE_MASK;
744 if (len > PAGE_SIZE - pageoff)
745 panic("vm_gpa_hold: invalid gpa/len: 0x%016lx/%lu", gpa, len);
746
747 count = vm_fault_quick_hold_pages(&vm->vmspace->vm_map,
748 trunc_page(gpa), PAGE_SIZE, reqprot, &m, 1);
749
750 if (count == 1) {
751 *cookie = m;
752 return ((void *)(PHYS_TO_DMAP(VM_PAGE_TO_PHYS(m)) + pageoff));
753 } else {
754 *cookie = NULL;
755 return (NULL);
756 }
757}
758
759void
760vm_gpa_release(void *cookie)
761{
762 vm_page_t m = cookie;
763
764 vm_page_lock(m);
765 vm_page_unhold(m);
766 vm_page_unlock(m);
767}
768
769int
770vm_gpabase2memseg(struct vm *vm, vm_paddr_t gpabase,
771 struct vm_memory_segment *seg)
772{
773 int i;
774
775 for (i = 0; i < vm->num_mem_segs; i++) {
776 if (gpabase == vm->mem_segs[i].gpa) {
777 seg->gpa = vm->mem_segs[i].gpa;
778 seg->len = vm->mem_segs[i].len;
779 seg->wired = vm->mem_segs[i].wired;
780 return (0);
781 }
782 }
783 return (-1);
784}
785
786int
787vm_get_memobj(struct vm *vm, vm_paddr_t gpa, size_t len,
788 vm_offset_t *offset, struct vm_object **object)
789{
790 int i;
791 size_t seg_len;
792 vm_paddr_t seg_gpa;
793 vm_object_t seg_obj;
794
795 for (i = 0; i < vm->num_mem_segs; i++) {
796 if ((seg_obj = vm->mem_segs[i].object) == NULL)
797 continue;
798
799 seg_gpa = vm->mem_segs[i].gpa;
800 seg_len = vm->mem_segs[i].len;
801
802 if (gpa >= seg_gpa && gpa < seg_gpa + seg_len) {
803 *offset = gpa - seg_gpa;
804 *object = seg_obj;
805 vm_object_reference(seg_obj);
806 return (0);
807 }
808 }
809
810 return (EINVAL);
811}
812
813int
814vm_get_register(struct vm *vm, int vcpu, int reg, uint64_t *retval)
815{
816
817 if (vcpu < 0 || vcpu >= VM_MAXCPU)
818 return (EINVAL);
819
820 if (reg >= VM_REG_LAST)
821 return (EINVAL);
822
823 return (VMGETREG(vm->cookie, vcpu, reg, retval));
824}
825
826int
827vm_set_register(struct vm *vm, int vcpu, int reg, uint64_t val)
828{
829
830 if (vcpu < 0 || vcpu >= VM_MAXCPU)
831 return (EINVAL);
832
833 if (reg >= VM_REG_LAST)
834 return (EINVAL);
835
836 return (VMSETREG(vm->cookie, vcpu, reg, val));
837}
838
839static boolean_t
840is_descriptor_table(int reg)
841{
842
843 switch (reg) {
844 case VM_REG_GUEST_IDTR:
845 case VM_REG_GUEST_GDTR:
846 return (TRUE);
847 default:
848 return (FALSE);
849 }
850}
851
852static boolean_t
853is_segment_register(int reg)
854{
855
856 switch (reg) {
857 case VM_REG_GUEST_ES:
858 case VM_REG_GUEST_CS:
859 case VM_REG_GUEST_SS:
860 case VM_REG_GUEST_DS:
861 case VM_REG_GUEST_FS:
862 case VM_REG_GUEST_GS:
863 case VM_REG_GUEST_TR:
864 case VM_REG_GUEST_LDTR:
865 return (TRUE);
866 default:
867 return (FALSE);
868 }
869}
870
871int
872vm_get_seg_desc(struct vm *vm, int vcpu, int reg,
873 struct seg_desc *desc)
874{
875
876 if (vcpu < 0 || vcpu >= VM_MAXCPU)
877 return (EINVAL);
878
879 if (!is_segment_register(reg) && !is_descriptor_table(reg))
880 return (EINVAL);
881
882 return (VMGETDESC(vm->cookie, vcpu, reg, desc));
883}
884
885int
886vm_set_seg_desc(struct vm *vm, int vcpu, int reg,
887 struct seg_desc *desc)
888{
889 if (vcpu < 0 || vcpu >= VM_MAXCPU)
890 return (EINVAL);
891
892 if (!is_segment_register(reg) && !is_descriptor_table(reg))
893 return (EINVAL);
894
895 return (VMSETDESC(vm->cookie, vcpu, reg, desc));
896}
897
898static void
899restore_guest_fpustate(struct vcpu *vcpu)
900{
901
902 /* flush host state to the pcb */
903 fpuexit(curthread);
904
905 /* restore guest FPU state */
906 fpu_stop_emulating();
907 fpurestore(vcpu->guestfpu);
908
909 /* restore guest XCR0 if XSAVE is enabled in the host */
910 if (rcr4() & CR4_XSAVE)
911 load_xcr(0, vcpu->guest_xcr0);
912
913 /*
914 * The FPU is now "dirty" with the guest's state so turn on emulation
915 * to trap any access to the FPU by the host.
916 */
917 fpu_start_emulating();
918}
919
920static void
921save_guest_fpustate(struct vcpu *vcpu)
922{
923
924 if ((rcr0() & CR0_TS) == 0)
925 panic("fpu emulation not enabled in host!");
926
927 /* save guest XCR0 and restore host XCR0 */
928 if (rcr4() & CR4_XSAVE) {
929 vcpu->guest_xcr0 = rxcr(0);
930 load_xcr(0, vmm_get_host_xcr0());
931 }
932
933 /* save guest FPU state */
934 fpu_stop_emulating();
935 fpusave(vcpu->guestfpu);
936 fpu_start_emulating();
937}
938
939static VMM_STAT(VCPU_IDLE_TICKS, "number of ticks vcpu was idle");
940
941static int
942vcpu_set_state_locked(struct vcpu *vcpu, enum vcpu_state newstate,
943 bool from_idle)
944{
945 int error;
946
947 vcpu_assert_locked(vcpu);
948
949 /*
950 * State transitions from the vmmdev_ioctl() must always begin from
951 * the VCPU_IDLE state. This guarantees that there is only a single
952 * ioctl() operating on a vcpu at any point.
953 */
954 if (from_idle) {
955 while (vcpu->state != VCPU_IDLE)
956 msleep_spin(&vcpu->state, &vcpu->mtx, "vmstat", hz);
957 } else {
958 KASSERT(vcpu->state != VCPU_IDLE, ("invalid transition from "
959 "vcpu idle state"));
960 }
961
962 if (vcpu->state == VCPU_RUNNING) {
963 KASSERT(vcpu->hostcpu == curcpu, ("curcpu %d and hostcpu %d "
964 "mismatch for running vcpu", curcpu, vcpu->hostcpu));
965 } else {
966 KASSERT(vcpu->hostcpu == NOCPU, ("Invalid hostcpu %d for a "
967 "vcpu that is not running", vcpu->hostcpu));
968 }
969
970 /*
971 * The following state transitions are allowed:
972 * IDLE -> FROZEN -> IDLE
973 * FROZEN -> RUNNING -> FROZEN
974 * FROZEN -> SLEEPING -> FROZEN
975 */
976 switch (vcpu->state) {
977 case VCPU_IDLE:
978 case VCPU_RUNNING:
979 case VCPU_SLEEPING:
980 error = (newstate != VCPU_FROZEN);
981 break;
982 case VCPU_FROZEN:
983 error = (newstate == VCPU_FROZEN);
984 break;
985 default:
986 error = 1;
987 break;
988 }
989
990 if (error)
991 return (EBUSY);
992
993 vcpu->state = newstate;
994 if (newstate == VCPU_RUNNING)
995 vcpu->hostcpu = curcpu;
996 else
997 vcpu->hostcpu = NOCPU;
998
999 if (newstate == VCPU_IDLE)
1000 wakeup(&vcpu->state);
1001
1002 return (0);
1003}
1004
1005static void
1006vcpu_require_state(struct vm *vm, int vcpuid, enum vcpu_state newstate)
1007{
1008 int error;
1009
1010 if ((error = vcpu_set_state(vm, vcpuid, newstate, false)) != 0)
1011 panic("Error %d setting state to %d\n", error, newstate);
1012}
1013
1014static void
1015vcpu_require_state_locked(struct vcpu *vcpu, enum vcpu_state newstate)
1016{
1017 int error;
1018
1019 if ((error = vcpu_set_state_locked(vcpu, newstate, false)) != 0)
1020 panic("Error %d setting state to %d", error, newstate);
1021}
1022
1023static void
1024vm_set_rendezvous_func(struct vm *vm, vm_rendezvous_func_t func)
1025{
1026
1027 KASSERT(mtx_owned(&vm->rendezvous_mtx), ("rendezvous_mtx not locked"));
1028
1029 /*
1030 * Update 'rendezvous_func' and execute a write memory barrier to
1031 * ensure that it is visible across all host cpus. This is not needed
1032 * for correctness but it does ensure that all the vcpus will notice
1033 * that the rendezvous is requested immediately.
1034 */
1035 vm->rendezvous_func = func;
1036 wmb();
1037}
1038
1039#define RENDEZVOUS_CTR0(vm, vcpuid, fmt) \
1040 do { \
1041 if (vcpuid >= 0) \
1042 VCPU_CTR0(vm, vcpuid, fmt); \
1043 else \
1044 VM_CTR0(vm, fmt); \
1045 } while (0)
1046
1047static void
1048vm_handle_rendezvous(struct vm *vm, int vcpuid)
1049{
1050
1051 KASSERT(vcpuid == -1 || (vcpuid >= 0 && vcpuid < VM_MAXCPU),
1052 ("vm_handle_rendezvous: invalid vcpuid %d", vcpuid));
1053
1054 mtx_lock(&vm->rendezvous_mtx);
1055 while (vm->rendezvous_func != NULL) {
1056 /* 'rendezvous_req_cpus' must be a subset of 'active_cpus' */
1057 CPU_AND(&vm->rendezvous_req_cpus, &vm->active_cpus);
1058
1059 if (vcpuid != -1 &&
1060 CPU_ISSET(vcpuid, &vm->rendezvous_req_cpus) &&
1061 !CPU_ISSET(vcpuid, &vm->rendezvous_done_cpus)) {
1062 VCPU_CTR0(vm, vcpuid, "Calling rendezvous func");
1063 (*vm->rendezvous_func)(vm, vcpuid, vm->rendezvous_arg);
1064 CPU_SET(vcpuid, &vm->rendezvous_done_cpus);
1065 }
1066 if (CPU_CMP(&vm->rendezvous_req_cpus,
1067 &vm->rendezvous_done_cpus) == 0) {
1068 VCPU_CTR0(vm, vcpuid, "Rendezvous completed");
1069 vm_set_rendezvous_func(vm, NULL);
1070 wakeup(&vm->rendezvous_func);
1071 break;
1072 }
1073 RENDEZVOUS_CTR0(vm, vcpuid, "Wait for rendezvous completion");
1074 mtx_sleep(&vm->rendezvous_func, &vm->rendezvous_mtx, 0,
1075 "vmrndv", 0);
1076 }
1077 mtx_unlock(&vm->rendezvous_mtx);
1078}
1079
1080/*
1081 * Emulate a guest 'hlt' by sleeping until the vcpu is ready to run.
1082 */
1083static int
1084vm_handle_hlt(struct vm *vm, int vcpuid, bool intr_disabled, bool *retu)
1085{
1086 struct vcpu *vcpu;
1087 const char *wmesg;
1088 int error, t, vcpu_halted, vm_halted;
1089
1090 KASSERT(!CPU_ISSET(vcpuid, &vm->halted_cpus), ("vcpu already halted"));
1091
1092 vcpu = &vm->vcpu[vcpuid];
1093 vcpu_halted = 0;
1094 vm_halted = 0;
1095
1096 /*
1097 * The typical way to halt a cpu is to execute: "sti; hlt"
1098 *
1099 * STI sets RFLAGS.IF to enable interrupts. However, the processor
1100 * remains in an "interrupt shadow" for an additional instruction
1101 * following the STI. This guarantees that "sti; hlt" sequence is
1102 * atomic and a pending interrupt will be recognized after the HLT.
1103 *
1104 * After the HLT emulation is done the vcpu is no longer in an
1105 * interrupt shadow and a pending interrupt can be injected on
1106 * the next entry into the guest.
1107 */
1108 error = vm_set_register(vm, vcpuid, VM_REG_GUEST_INTR_SHADOW, 0);
1109 KASSERT(error == 0, ("%s: error %d clearing interrupt shadow",
1110 __func__, error));
1111
1112 vcpu_lock(vcpu);
1113 while (1) {
1114 /*
1115 * Do a final check for pending NMI or interrupts before
1116 * really putting this thread to sleep. Also check for
1117 * software events that would cause this vcpu to wakeup.
1118 *
1119 * These interrupts/events could have happened after the
1120 * vcpu returned from VMRUN() and before it acquired the
1121 * vcpu lock above.
1122 */
1123 if (vm->rendezvous_func != NULL || vm->suspend)
1124 break;
1125 if (vm_nmi_pending(vm, vcpuid))
1126 break;
1127 if (!intr_disabled) {
1128 if (vm_extint_pending(vm, vcpuid) ||
1129 vlapic_pending_intr(vcpu->vlapic, NULL)) {
1130 break;
1131 }
1132 }
1133
1134 /* Don't go to sleep if the vcpu thread needs to yield */
1135 if (vcpu_should_yield(vm, vcpuid))
1136 break;
1137
1138 /*
1139 * Some Linux guests implement "halt" by having all vcpus
1140 * execute HLT with interrupts disabled. 'halted_cpus' keeps
1141 * track of the vcpus that have entered this state. When all
1142 * vcpus enter the halted state the virtual machine is halted.
1143 */
1144 if (intr_disabled) {
1145 wmesg = "vmhalt";
1146 VCPU_CTR0(vm, vcpuid, "Halted");
1147 if (!vcpu_halted && halt_detection_enabled) {
1148 vcpu_halted = 1;
1149 CPU_SET_ATOMIC(vcpuid, &vm->halted_cpus);
1150 }
1151 if (CPU_CMP(&vm->halted_cpus, &vm->active_cpus) == 0) {
1152 vm_halted = 1;
1153 break;
1154 }
1155 } else {
1156 wmesg = "vmidle";
1157 }
1158
1159 t = ticks;
1160 vcpu_require_state_locked(vcpu, VCPU_SLEEPING);
1161 /*
1162 * XXX msleep_spin() cannot be interrupted by signals so
1163 * wake up periodically to check pending signals.
1164 */
1165 msleep_spin(vcpu, &vcpu->mtx, wmesg, hz);
1166 vcpu_require_state_locked(vcpu, VCPU_FROZEN);
1167 vmm_stat_incr(vm, vcpuid, VCPU_IDLE_TICKS, ticks - t);
1168 }
1169
1170 if (vcpu_halted)
1171 CPU_CLR_ATOMIC(vcpuid, &vm->halted_cpus);
1172
1173 vcpu_unlock(vcpu);
1174
1175 if (vm_halted)
1176 vm_suspend(vm, VM_SUSPEND_HALT);
1177
1178 return (0);
1179}
1180
1181static int
1182vm_handle_paging(struct vm *vm, int vcpuid, bool *retu)
1183{
1184 int rv, ftype;
1185 struct vm_map *map;
1186 struct vcpu *vcpu;
1187 struct vm_exit *vme;
1188
1189 vcpu = &vm->vcpu[vcpuid];
1190 vme = &vcpu->exitinfo;
1191
1192 ftype = vme->u.paging.fault_type;
1193 KASSERT(ftype == VM_PROT_READ ||
1194 ftype == VM_PROT_WRITE || ftype == VM_PROT_EXECUTE,
1195 ("vm_handle_paging: invalid fault_type %d", ftype));
1196
1197 if (ftype == VM_PROT_READ || ftype == VM_PROT_WRITE) {
1198 rv = pmap_emulate_accessed_dirty(vmspace_pmap(vm->vmspace),
1199 vme->u.paging.gpa, ftype);
1200 if (rv == 0) {
1201 VCPU_CTR2(vm, vcpuid, "%s bit emulation for gpa %#lx",
1202 ftype == VM_PROT_READ ? "accessed" : "dirty",
1203 vme->u.paging.gpa);
1204 goto done;
1205 }
1206 }
1207
1208 map = &vm->vmspace->vm_map;
1209 rv = vm_fault(map, vme->u.paging.gpa, ftype, VM_FAULT_NORMAL);
1210
1211 VCPU_CTR3(vm, vcpuid, "vm_handle_paging rv = %d, gpa = %#lx, "
1212 "ftype = %d", rv, vme->u.paging.gpa, ftype);
1213
1214 if (rv != KERN_SUCCESS)
1215 return (EFAULT);
1216done:
1217 /* restart execution at the faulting instruction */
1218 vme->inst_length = 0;
1219
1220 return (0);
1221}
1222
1223static int
1224vm_handle_inst_emul(struct vm *vm, int vcpuid, bool *retu)
1225{
1226 struct vie *vie;
1227 struct vcpu *vcpu;
1228 struct vm_exit *vme;
1229 uint64_t gla, gpa;
1230 struct vm_guest_paging *paging;
1231 mem_region_read_t mread;
1232 mem_region_write_t mwrite;
1233 enum vm_cpu_mode cpu_mode;
1234 int cs_d, error;
1235
1236 vcpu = &vm->vcpu[vcpuid];
1237 vme = &vcpu->exitinfo;
1238
1239 gla = vme->u.inst_emul.gla;
1240 gpa = vme->u.inst_emul.gpa;
1241 cs_d = vme->u.inst_emul.cs_d;
1242 vie = &vme->u.inst_emul.vie;
1243 paging = &vme->u.inst_emul.paging;
1244 cpu_mode = paging->cpu_mode;
1245
1246 VCPU_CTR1(vm, vcpuid, "inst_emul fault accessing gpa %#lx", gpa);
1247
1248 vie_init(vie);
1249
1250 /* Fetch, decode and emulate the faulting instruction */
1251 error = vmm_fetch_instruction(vm, vcpuid, paging, vme->rip,
1252 vme->inst_length, vie);
1253 if (error == 1)
1254 return (0); /* Resume guest to handle page fault */
1255 else if (error == -1)
1256 return (EFAULT);
1257 else if (error != 0)
1258 panic("%s: vmm_fetch_instruction error %d", __func__, error);
1259
1260 if (vmm_decode_instruction(vm, vcpuid, gla, cpu_mode, cs_d, vie) != 0)
1261 return (EFAULT);
1262
1263 /* return to userland unless this is an in-kernel emulated device */
1264 if (gpa >= DEFAULT_APIC_BASE && gpa < DEFAULT_APIC_BASE + PAGE_SIZE) {
1265 mread = lapic_mmio_read;
1266 mwrite = lapic_mmio_write;
1267 } else if (gpa >= VIOAPIC_BASE && gpa < VIOAPIC_BASE + VIOAPIC_SIZE) {
1268 mread = vioapic_mmio_read;
1269 mwrite = vioapic_mmio_write;
1270 } else if (gpa >= VHPET_BASE && gpa < VHPET_BASE + VHPET_SIZE) {
1271 mread = vhpet_mmio_read;
1272 mwrite = vhpet_mmio_write;
1273 } else {
1274 *retu = true;
1275 return (0);
1276 }
1277
1278 error = vmm_emulate_instruction(vm, vcpuid, gpa, vie, paging,
1279 mread, mwrite, retu);
1280
1281 return (error);
1282}
1283
1284static int
1285vm_handle_suspend(struct vm *vm, int vcpuid, bool *retu)
1286{
1287 int i, done;
1288 struct vcpu *vcpu;
1289
1290 done = 0;
1291 vcpu = &vm->vcpu[vcpuid];
1292
1293 CPU_SET_ATOMIC(vcpuid, &vm->suspended_cpus);
1294
1295 /*
1296 * Wait until all 'active_cpus' have suspended themselves.
1297 *
1298 * Since a VM may be suspended at any time including when one or
1299 * more vcpus are doing a rendezvous we need to call the rendezvous
1300 * handler while we are waiting to prevent a deadlock.
1301 */
1302 vcpu_lock(vcpu);
1303 while (1) {
1304 if (CPU_CMP(&vm->suspended_cpus, &vm->active_cpus) == 0) {
1305 VCPU_CTR0(vm, vcpuid, "All vcpus suspended");
1306 break;
1307 }
1308
1309 if (vm->rendezvous_func == NULL) {
1310 VCPU_CTR0(vm, vcpuid, "Sleeping during suspend");
1311 vcpu_require_state_locked(vcpu, VCPU_SLEEPING);
1312 msleep_spin(vcpu, &vcpu->mtx, "vmsusp", hz);
1313 vcpu_require_state_locked(vcpu, VCPU_FROZEN);
1314 } else {
1315 VCPU_CTR0(vm, vcpuid, "Rendezvous during suspend");
1316 vcpu_unlock(vcpu);
1317 vm_handle_rendezvous(vm, vcpuid);
1318 vcpu_lock(vcpu);
1319 }
1320 }
1321 vcpu_unlock(vcpu);
1322
1323 /*
1324 * Wakeup the other sleeping vcpus and return to userspace.
1325 */
1326 for (i = 0; i < VM_MAXCPU; i++) {
1327 if (CPU_ISSET(i, &vm->suspended_cpus)) {
1328 vcpu_notify_event(vm, i, false);
1329 }
1330 }
1331
1332 *retu = true;
1333 return (0);
1334}
1335
1336int
1337vm_suspend(struct vm *vm, enum vm_suspend_how how)
1338{
1339 int i;
1340
1341 if (how <= VM_SUSPEND_NONE || how >= VM_SUSPEND_LAST)
1342 return (EINVAL);
1343
1344 if (atomic_cmpset_int(&vm->suspend, 0, how) == 0) {
1345 VM_CTR2(vm, "virtual machine already suspended %d/%d",
1346 vm->suspend, how);
1347 return (EALREADY);
1348 }
1349
1350 VM_CTR1(vm, "virtual machine successfully suspended %d", how);
1351
1352 /*
1353 * Notify all active vcpus that they are now suspended.
1354 */
1355 for (i = 0; i < VM_MAXCPU; i++) {
1356 if (CPU_ISSET(i, &vm->active_cpus))
1357 vcpu_notify_event(vm, i, false);
1358 }
1359
1360 return (0);
1361}
1362
1363void
1364vm_exit_suspended(struct vm *vm, int vcpuid, uint64_t rip)
1365{
1366 struct vm_exit *vmexit;
1367
1368 KASSERT(vm->suspend > VM_SUSPEND_NONE && vm->suspend < VM_SUSPEND_LAST,
1369 ("vm_exit_suspended: invalid suspend type %d", vm->suspend));
1370
1371 vmexit = vm_exitinfo(vm, vcpuid);
1372 vmexit->rip = rip;
1373 vmexit->inst_length = 0;
1374 vmexit->exitcode = VM_EXITCODE_SUSPENDED;
1375 vmexit->u.suspended.how = vm->suspend;
1376}
1377
1378void
1379vm_exit_rendezvous(struct vm *vm, int vcpuid, uint64_t rip)
1380{
1381 struct vm_exit *vmexit;
1382
1383 KASSERT(vm->rendezvous_func != NULL, ("rendezvous not in progress"));
1384
1385 vmexit = vm_exitinfo(vm, vcpuid);
1386 vmexit->rip = rip;
1387 vmexit->inst_length = 0;
1388 vmexit->exitcode = VM_EXITCODE_RENDEZVOUS;
1389 vmm_stat_incr(vm, vcpuid, VMEXIT_RENDEZVOUS, 1);
1390}
1391
1392void
1393vm_exit_astpending(struct vm *vm, int vcpuid, uint64_t rip)
1394{
1395 struct vm_exit *vmexit;
1396
1397 vmexit = vm_exitinfo(vm, vcpuid);
1398 vmexit->rip = rip;
1399 vmexit->inst_length = 0;
1400 vmexit->exitcode = VM_EXITCODE_BOGUS;
1401 vmm_stat_incr(vm, vcpuid, VMEXIT_ASTPENDING, 1);
1402}
1403
1404int
1405vm_run(struct vm *vm, struct vm_run *vmrun)
1406{
1407 int error, vcpuid;
1408 struct vcpu *vcpu;
1409 struct pcb *pcb;
1410 uint64_t tscval, rip;
1411 struct vm_exit *vme;
1412 bool retu, intr_disabled;
1413 pmap_t pmap;
1414 void *rptr, *sptr;
1415
1416 vcpuid = vmrun->cpuid;
1417
1418 if (vcpuid < 0 || vcpuid >= VM_MAXCPU)
1419 return (EINVAL);
1420
1421 if (!CPU_ISSET(vcpuid, &vm->active_cpus))
1422 return (EINVAL);
1423
1424 if (CPU_ISSET(vcpuid, &vm->suspended_cpus))
1425 return (EINVAL);
1426
1427 rptr = &vm->rendezvous_func;
1428 sptr = &vm->suspend;
1429 pmap = vmspace_pmap(vm->vmspace);
1430 vcpu = &vm->vcpu[vcpuid];
1431 vme = &vcpu->exitinfo;
1432 rip = vmrun->rip;
1433restart:
1434 critical_enter();
1435
1436 KASSERT(!CPU_ISSET(curcpu, &pmap->pm_active),
1437 ("vm_run: absurd pm_active"));
1438
1439 tscval = rdtsc();
1440
1441 pcb = PCPU_GET(curpcb);
1442 set_pcb_flags(pcb, PCB_FULL_IRET);
1443
1444 restore_guest_fpustate(vcpu);
1445
1446 vcpu_require_state(vm, vcpuid, VCPU_RUNNING);
1447 error = VMRUN(vm->cookie, vcpuid, rip, pmap, rptr, sptr);
1448 vcpu_require_state(vm, vcpuid, VCPU_FROZEN);
1449
1450 save_guest_fpustate(vcpu);
1451
1452 vmm_stat_incr(vm, vcpuid, VCPU_TOTAL_RUNTIME, rdtsc() - tscval);
1453
1454 critical_exit();
1455
1456 if (error == 0) {
1457 retu = false;
1458 switch (vme->exitcode) {
1459 case VM_EXITCODE_SUSPENDED:
1460 error = vm_handle_suspend(vm, vcpuid, &retu);
1461 break;
1462 case VM_EXITCODE_IOAPIC_EOI:
1463 vioapic_process_eoi(vm, vcpuid,
1464 vme->u.ioapic_eoi.vector);
1465 break;
1466 case VM_EXITCODE_RENDEZVOUS:
1467 vm_handle_rendezvous(vm, vcpuid);
1468 error = 0;
1469 break;
1470 case VM_EXITCODE_HLT:
1471 intr_disabled = ((vme->u.hlt.rflags & PSL_I) == 0);
1472 error = vm_handle_hlt(vm, vcpuid, intr_disabled, &retu);
1473 break;
1474 case VM_EXITCODE_PAGING:
1475 error = vm_handle_paging(vm, vcpuid, &retu);
1476 break;
1477 case VM_EXITCODE_INST_EMUL:
1478 error = vm_handle_inst_emul(vm, vcpuid, &retu);
1479 break;
1480 case VM_EXITCODE_INOUT:
1481 case VM_EXITCODE_INOUT_STR:
1482 error = vm_handle_inout(vm, vcpuid, vme, &retu);
1483 break;
1484 case VM_EXITCODE_MONITOR:
1485 case VM_EXITCODE_MWAIT:
1486 vm_inject_ud(vm, vcpuid);
1487 break;
1488 default:
1489 retu = true; /* handled in userland */
1490 break;
1491 }
1492 }
1493
1494 if (error == 0 && retu == false) {
1495 rip = vme->rip + vme->inst_length;
1496 goto restart;
1497 }
1498
1499 /* copy the exit information */
1500 bcopy(vme, &vmrun->vm_exit, sizeof(struct vm_exit));
1501 return (error);
1502}
1503
1504int
1505vm_exit_intinfo(struct vm *vm, int vcpuid, uint64_t info)
1506{
1507 struct vcpu *vcpu;
1508 int type, vector;
1509
1510 if (vcpuid < 0 || vcpuid >= VM_MAXCPU)
1511 return (EINVAL);
1512
1513 vcpu = &vm->vcpu[vcpuid];
1514
1515 if (info & VM_INTINFO_VALID) {
1516 type = info & VM_INTINFO_TYPE;
1517 vector = info & 0xff;
1518 if (type == VM_INTINFO_NMI && vector != IDT_NMI)
1519 return (EINVAL);
1520 if (type == VM_INTINFO_HWEXCEPTION && vector >= 32)
1521 return (EINVAL);
1522 if (info & VM_INTINFO_RSVD)
1523 return (EINVAL);
1524 } else {
1525 info = 0;
1526 }
1527 VCPU_CTR2(vm, vcpuid, "%s: info1(%#lx)", __func__, info);
1528 vcpu->exitintinfo = info;
1529 return (0);
1530}
1531
1532enum exc_class {
1533 EXC_BENIGN,
1534 EXC_CONTRIBUTORY,
1535 EXC_PAGEFAULT
1536};
1537
1538#define IDT_VE 20 /* Virtualization Exception (Intel specific) */
1539
1540static enum exc_class
1541exception_class(uint64_t info)
1542{
1543 int type, vector;
1544
1545 KASSERT(info & VM_INTINFO_VALID, ("intinfo must be valid: %#lx", info));
1546 type = info & VM_INTINFO_TYPE;
1547 vector = info & 0xff;
1548
1549 /* Table 6-4, "Interrupt and Exception Classes", Intel SDM, Vol 3 */
1550 switch (type) {
1551 case VM_INTINFO_HWINTR:
1552 case VM_INTINFO_SWINTR:
1553 case VM_INTINFO_NMI:
1554 return (EXC_BENIGN);
1555 default:
1556 /*
1557 * Hardware exception.
1558 *
1559 * SVM and VT-x use identical type values to represent NMI,
1560 * hardware interrupt and software interrupt.
1561 *
1562 * SVM uses type '3' for all exceptions. VT-x uses type '3'
1563 * for exceptions except #BP and #OF. #BP and #OF use a type
1564 * value of '5' or '6'. Therefore we don't check for explicit
1565 * values of 'type' to classify 'intinfo' into a hardware
1566 * exception.
1567 */
1568 break;
1569 }
1570
1571 switch (vector) {
1572 case IDT_PF:
1573 case IDT_VE:
1574 return (EXC_PAGEFAULT);
1575 case IDT_DE:
1576 case IDT_TS:
1577 case IDT_NP:
1578 case IDT_SS:
1579 case IDT_GP:
1580 return (EXC_CONTRIBUTORY);
1581 default:
1582 return (EXC_BENIGN);
1583 }
1584}
1585
1586static int
1587nested_fault(struct vm *vm, int vcpuid, uint64_t info1, uint64_t info2,
1588 uint64_t *retinfo)
1589{
1590 enum exc_class exc1, exc2;
1591 int type1, vector1;
1592
1593 KASSERT(info1 & VM_INTINFO_VALID, ("info1 %#lx is not valid", info1));
1594 KASSERT(info2 & VM_INTINFO_VALID, ("info2 %#lx is not valid", info2));
1595
1596 /*
1597 * If an exception occurs while attempting to call the double-fault
1598 * handler the processor enters shutdown mode (aka triple fault).
1599 */
1600 type1 = info1 & VM_INTINFO_TYPE;
1601 vector1 = info1 & 0xff;
1602 if (type1 == VM_INTINFO_HWEXCEPTION && vector1 == IDT_DF) {
1603 VCPU_CTR2(vm, vcpuid, "triple fault: info1(%#lx), info2(%#lx)",
1604 info1, info2);
1605 vm_suspend(vm, VM_SUSPEND_TRIPLEFAULT);
1606 *retinfo = 0;
1607 return (0);
1608 }
1609
1610 /*
1611 * Table 6-5 "Conditions for Generating a Double Fault", Intel SDM, Vol3
1612 */
1613 exc1 = exception_class(info1);
1614 exc2 = exception_class(info2);
1615 if ((exc1 == EXC_CONTRIBUTORY && exc2 == EXC_CONTRIBUTORY) ||
1616 (exc1 == EXC_PAGEFAULT && exc2 != EXC_BENIGN)) {
1617 /* Convert nested fault into a double fault. */
1618 *retinfo = IDT_DF;
1619 *retinfo |= VM_INTINFO_VALID | VM_INTINFO_HWEXCEPTION;
1620 *retinfo |= VM_INTINFO_DEL_ERRCODE;
1621 } else {
1622 /* Handle exceptions serially */
1623 *retinfo = info2;
1624 }
1625 return (1);
1626}
1627
1628static uint64_t
1629vcpu_exception_intinfo(struct vcpu *vcpu)
1630{
1631 uint64_t info = 0;
1632
1633 if (vcpu->exception_pending) {
1634 info = vcpu->exception.vector & 0xff;
1635 info |= VM_INTINFO_VALID | VM_INTINFO_HWEXCEPTION;
1636 if (vcpu->exception.error_code_valid) {
1637 info |= VM_INTINFO_DEL_ERRCODE;
1638 info |= (uint64_t)vcpu->exception.error_code << 32;
1639 }
1640 }
1641 return (info);
1642}
1643
1644int
1645vm_entry_intinfo(struct vm *vm, int vcpuid, uint64_t *retinfo)
1646{
1647 struct vcpu *vcpu;
1648 uint64_t info1, info2;
1649 int valid;
1650
1651 KASSERT(vcpuid >= 0 && vcpuid < VM_MAXCPU, ("invalid vcpu %d", vcpuid));
1652
1653 vcpu = &vm->vcpu[vcpuid];
1654
1655 info1 = vcpu->exitintinfo;
1656 vcpu->exitintinfo = 0;
1657
1658 info2 = 0;
1659 if (vcpu->exception_pending) {
1660 info2 = vcpu_exception_intinfo(vcpu);
1661 vcpu->exception_pending = 0;
1662 VCPU_CTR2(vm, vcpuid, "Exception %d delivered: %#lx",
1663 vcpu->exception.vector, info2);
1664 }
1665
1666 if ((info1 & VM_INTINFO_VALID) && (info2 & VM_INTINFO_VALID)) {
1667 valid = nested_fault(vm, vcpuid, info1, info2, retinfo);
1668 } else if (info1 & VM_INTINFO_VALID) {
1669 *retinfo = info1;
1670 valid = 1;
1671 } else if (info2 & VM_INTINFO_VALID) {
1672 *retinfo = info2;
1673 valid = 1;
1674 } else {
1675 valid = 0;
1676 }
1677
1678 if (valid) {
1679 VCPU_CTR4(vm, vcpuid, "%s: info1(%#lx), info2(%#lx), "
1680 "retinfo(%#lx)", __func__, info1, info2, *retinfo);
1681 }
1682
1683 return (valid);
1684}
1685
1686int
1687vm_get_intinfo(struct vm *vm, int vcpuid, uint64_t *info1, uint64_t *info2)
1688{
1689 struct vcpu *vcpu;
1690
1691 if (vcpuid < 0 || vcpuid >= VM_MAXCPU)
1692 return (EINVAL);
1693
1694 vcpu = &vm->vcpu[vcpuid];
1695 *info1 = vcpu->exitintinfo;
1696 *info2 = vcpu_exception_intinfo(vcpu);
1697 return (0);
1698}
1699
1700int
1701vm_inject_exception(struct vm *vm, int vcpuid, struct vm_exception *exception)
1702{
1703 struct vcpu *vcpu;
1704
1705 if (vcpuid < 0 || vcpuid >= VM_MAXCPU)
1706 return (EINVAL);
1707
1708 if (exception->vector < 0 || exception->vector >= 32)
1709 return (EINVAL);
1710
1711 /*
1712 * A double fault exception should never be injected directly into
1713 * the guest. It is a derived exception that results from specific
1714 * combinations of nested faults.
1715 */
1716 if (exception->vector == IDT_DF)
1717 return (EINVAL);
1718
1719 vcpu = &vm->vcpu[vcpuid];
1720
1721 if (vcpu->exception_pending) {
1722 VCPU_CTR2(vm, vcpuid, "Unable to inject exception %d due to "
1723 "pending exception %d", exception->vector,
1724 vcpu->exception.vector);
1725 return (EBUSY);
1726 }
1727
1728 vcpu->exception_pending = 1;
1729 vcpu->exception = *exception;
1730 VCPU_CTR1(vm, vcpuid, "Exception %d pending", exception->vector);
1731 return (0);
1732}
1733
1734void
1735vm_inject_fault(void *vmarg, int vcpuid, int vector, int errcode_valid,
1736 int errcode)
1737{
1738 struct vm_exception exception;
1739 struct vm_exit *vmexit;
1740 struct vm *vm;
1741 int error;
1742
1743 vm = vmarg;
1744
1745 exception.vector = vector;
1746 exception.error_code = errcode;
1747 exception.error_code_valid = errcode_valid;
1748 error = vm_inject_exception(vm, vcpuid, &exception);
1749 KASSERT(error == 0, ("vm_inject_exception error %d", error));
1750
1751 /*
1752 * A fault-like exception allows the instruction to be restarted
1753 * after the exception handler returns.
1754 *
1755 * By setting the inst_length to 0 we ensure that the instruction
1756 * pointer remains at the faulting instruction.
1757 */
1758 vmexit = vm_exitinfo(vm, vcpuid);
1759 vmexit->inst_length = 0;
1760}
1761
1762void
1763vm_inject_pf(void *vmarg, int vcpuid, int error_code, uint64_t cr2)
1764{
1765 struct vm *vm;
1766 int error;
1767
1768 vm = vmarg;
1769 VCPU_CTR2(vm, vcpuid, "Injecting page fault: error_code %#x, cr2 %#lx",
1770 error_code, cr2);
1771
1772 error = vm_set_register(vm, vcpuid, VM_REG_GUEST_CR2, cr2);
1773 KASSERT(error == 0, ("vm_set_register(cr2) error %d", error));
1774
1775 vm_inject_fault(vm, vcpuid, IDT_PF, 1, error_code);
1776}
1777
1778static VMM_STAT(VCPU_NMI_COUNT, "number of NMIs delivered to vcpu");
1779
1780int
1781vm_inject_nmi(struct vm *vm, int vcpuid)
1782{
1783 struct vcpu *vcpu;
1784
1785 if (vcpuid < 0 || vcpuid >= VM_MAXCPU)
1786 return (EINVAL);
1787
1788 vcpu = &vm->vcpu[vcpuid];
1789
1790 vcpu->nmi_pending = 1;
1791 vcpu_notify_event(vm, vcpuid, false);
1792 return (0);
1793}
1794
1795int
1796vm_nmi_pending(struct vm *vm, int vcpuid)
1797{
1798 struct vcpu *vcpu;
1799
1800 if (vcpuid < 0 || vcpuid >= VM_MAXCPU)
1801 panic("vm_nmi_pending: invalid vcpuid %d", vcpuid);
1802
1803 vcpu = &vm->vcpu[vcpuid];
1804
1805 return (vcpu->nmi_pending);
1806}
1807
1808void
1809vm_nmi_clear(struct vm *vm, int vcpuid)
1810{
1811 struct vcpu *vcpu;
1812
1813 if (vcpuid < 0 || vcpuid >= VM_MAXCPU)
1814 panic("vm_nmi_pending: invalid vcpuid %d", vcpuid);
1815
1816 vcpu = &vm->vcpu[vcpuid];
1817
1818 if (vcpu->nmi_pending == 0)
1819 panic("vm_nmi_clear: inconsistent nmi_pending state");
1820
1821 vcpu->nmi_pending = 0;
1822 vmm_stat_incr(vm, vcpuid, VCPU_NMI_COUNT, 1);
1823}
1824
1825static VMM_STAT(VCPU_EXTINT_COUNT, "number of ExtINTs delivered to vcpu");
1826
1827int
1828vm_inject_extint(struct vm *vm, int vcpuid)
1829{
1830 struct vcpu *vcpu;
1831
1832 if (vcpuid < 0 || vcpuid >= VM_MAXCPU)
1833 return (EINVAL);
1834
1835 vcpu = &vm->vcpu[vcpuid];
1836
1837 vcpu->extint_pending = 1;
1838 vcpu_notify_event(vm, vcpuid, false);
1839 return (0);
1840}
1841
1842int
1843vm_extint_pending(struct vm *vm, int vcpuid)
1844{
1845 struct vcpu *vcpu;
1846
1847 if (vcpuid < 0 || vcpuid >= VM_MAXCPU)
1848 panic("vm_extint_pending: invalid vcpuid %d", vcpuid);
1849
1850 vcpu = &vm->vcpu[vcpuid];
1851
1852 return (vcpu->extint_pending);
1853}
1854
1855void
1856vm_extint_clear(struct vm *vm, int vcpuid)
1857{
1858 struct vcpu *vcpu;
1859
1860 if (vcpuid < 0 || vcpuid >= VM_MAXCPU)
1861 panic("vm_extint_pending: invalid vcpuid %d", vcpuid);
1862
1863 vcpu = &vm->vcpu[vcpuid];
1864
1865 if (vcpu->extint_pending == 0)
1866 panic("vm_extint_clear: inconsistent extint_pending state");
1867
1868 vcpu->extint_pending = 0;
1869 vmm_stat_incr(vm, vcpuid, VCPU_EXTINT_COUNT, 1);
1870}
1871
1872int
1873vm_get_capability(struct vm *vm, int vcpu, int type, int *retval)
1874{
1875 if (vcpu < 0 || vcpu >= VM_MAXCPU)
1876 return (EINVAL);
1877
1878 if (type < 0 || type >= VM_CAP_MAX)
1879 return (EINVAL);
1880
1881 return (VMGETCAP(vm->cookie, vcpu, type, retval));
1882}
1883
1884int
1885vm_set_capability(struct vm *vm, int vcpu, int type, int val)
1886{
1887 if (vcpu < 0 || vcpu >= VM_MAXCPU)
1888 return (EINVAL);
1889
1890 if (type < 0 || type >= VM_CAP_MAX)
1891 return (EINVAL);
1892
1893 return (VMSETCAP(vm->cookie, vcpu, type, val));
1894}
1895
1896struct vlapic *
1897vm_lapic(struct vm *vm, int cpu)
1898{
1899 return (vm->vcpu[cpu].vlapic);
1900}
1901
1902struct vioapic *
1903vm_ioapic(struct vm *vm)
1904{
1905
1906 return (vm->vioapic);
1907}
1908
1909struct vhpet *
1910vm_hpet(struct vm *vm)
1911{
1912
1913 return (vm->vhpet);
1914}
1915
1916boolean_t
1917vmm_is_pptdev(int bus, int slot, int func)
1918{
1919 int found, i, n;
1920 int b, s, f;
1921 char *val, *cp, *cp2;
1922
1923 /*
1924 * XXX
1925 * The length of an environment variable is limited to 128 bytes which
1926 * puts an upper limit on the number of passthru devices that may be
1927 * specified using a single environment variable.
1928 *
1929 * Work around this by scanning multiple environment variable
1930 * names instead of a single one - yuck!
1931 */
1932 const char *names[] = { "pptdevs", "pptdevs2", "pptdevs3", NULL };
1933
1934 /* set pptdevs="1/2/3 4/5/6 7/8/9 10/11/12" */
1935 found = 0;
1936 for (i = 0; names[i] != NULL && !found; i++) {
| 31
32#include <sys/param.h>
33#include <sys/systm.h>
34#include <sys/kernel.h>
35#include <sys/module.h>
36#include <sys/sysctl.h>
37#include <sys/malloc.h>
38#include <sys/pcpu.h>
39#include <sys/lock.h>
40#include <sys/mutex.h>
41#include <sys/proc.h>
42#include <sys/rwlock.h>
43#include <sys/sched.h>
44#include <sys/smp.h>
45#include <sys/systm.h>
46
47#include <vm/vm.h>
48#include <vm/vm_object.h>
49#include <vm/vm_page.h>
50#include <vm/pmap.h>
51#include <vm/vm_map.h>
52#include <vm/vm_extern.h>
53#include <vm/vm_param.h>
54
55#include <machine/cpu.h>
56#include <machine/vm.h>
57#include <machine/pcb.h>
58#include <machine/smp.h>
59#include <x86/psl.h>
60#include <x86/apicreg.h>
61#include <machine/vmparam.h>
62
63#include <machine/vmm.h>
64#include <machine/vmm_dev.h>
65#include <machine/vmm_instruction_emul.h>
66
67#include "vmm_ioport.h"
68#include "vmm_ktr.h"
69#include "vmm_host.h"
70#include "vmm_mem.h"
71#include "vmm_util.h"
72#include "vatpic.h"
73#include "vatpit.h"
74#include "vhpet.h"
75#include "vioapic.h"
76#include "vlapic.h"
77#include "vmm_ipi.h"
78#include "vmm_stat.h"
79#include "vmm_lapic.h"
80
81#include "io/ppt.h"
82#include "io/iommu.h"
83
84struct vlapic;
85
86/*
87 * Initialization:
88 * (a) allocated when vcpu is created
89 * (i) initialized when vcpu is created and when it is reinitialized
90 * (o) initialized the first time the vcpu is created
91 * (x) initialized before use
92 */
93struct vcpu {
94 struct mtx mtx; /* (o) protects 'state' and 'hostcpu' */
95 enum vcpu_state state; /* (o) vcpu state */
96 int hostcpu; /* (o) vcpu's host cpu */
97 struct vlapic *vlapic; /* (i) APIC device model */
98 enum x2apic_state x2apic_state; /* (i) APIC mode */
99 uint64_t exitintinfo; /* (i) events pending at VM exit */
100 int nmi_pending; /* (i) NMI pending */
101 int extint_pending; /* (i) INTR pending */
102 struct vm_exception exception; /* (x) exception collateral */
103 int exception_pending; /* (i) exception pending */
104 struct savefpu *guestfpu; /* (a,i) guest fpu state */
105 uint64_t guest_xcr0; /* (i) guest %xcr0 register */
106 void *stats; /* (a,i) statistics */
107 struct vm_exit exitinfo; /* (x) exit reason and collateral */
108};
109
110#define vcpu_lock_initialized(v) mtx_initialized(&((v)->mtx))
111#define vcpu_lock_init(v) mtx_init(&((v)->mtx), "vcpu lock", 0, MTX_SPIN)
112#define vcpu_lock(v) mtx_lock_spin(&((v)->mtx))
113#define vcpu_unlock(v) mtx_unlock_spin(&((v)->mtx))
114#define vcpu_assert_locked(v) mtx_assert(&((v)->mtx), MA_OWNED)
115
116struct mem_seg {
117 vm_paddr_t gpa;
118 size_t len;
119 boolean_t wired;
120 vm_object_t object;
121};
122#define VM_MAX_MEMORY_SEGMENTS 2
123
124/*
125 * Initialization:
126 * (o) initialized the first time the VM is created
127 * (i) initialized when VM is created and when it is reinitialized
128 * (x) initialized before use
129 */
130struct vm {
131 void *cookie; /* (i) cpu-specific data */
132 void *iommu; /* (x) iommu-specific data */
133 struct vhpet *vhpet; /* (i) virtual HPET */
134 struct vioapic *vioapic; /* (i) virtual ioapic */
135 struct vatpic *vatpic; /* (i) virtual atpic */
136 struct vatpit *vatpit; /* (i) virtual atpit */
137 volatile cpuset_t active_cpus; /* (i) active vcpus */
138 int suspend; /* (i) stop VM execution */
139 volatile cpuset_t suspended_cpus; /* (i) suspended vcpus */
140 volatile cpuset_t halted_cpus; /* (x) cpus in a hard halt */
141 cpuset_t rendezvous_req_cpus; /* (x) rendezvous requested */
142 cpuset_t rendezvous_done_cpus; /* (x) rendezvous finished */
143 void *rendezvous_arg; /* (x) rendezvous func/arg */
144 vm_rendezvous_func_t rendezvous_func;
145 struct mtx rendezvous_mtx; /* (o) rendezvous lock */
146 int num_mem_segs; /* (o) guest memory segments */
147 struct mem_seg mem_segs[VM_MAX_MEMORY_SEGMENTS];
148 struct vmspace *vmspace; /* (o) guest's address space */
149 char name[VM_MAX_NAMELEN]; /* (o) virtual machine name */
150 struct vcpu vcpu[VM_MAXCPU]; /* (i) guest vcpus */
151};
152
153static int vmm_initialized;
154
155static struct vmm_ops *ops;
156#define VMM_INIT(num) (ops != NULL ? (*ops->init)(num) : 0)
157#define VMM_CLEANUP() (ops != NULL ? (*ops->cleanup)() : 0)
158#define VMM_RESUME() (ops != NULL ? (*ops->resume)() : 0)
159
160#define VMINIT(vm, pmap) (ops != NULL ? (*ops->vminit)(vm, pmap): NULL)
161#define VMRUN(vmi, vcpu, rip, pmap, rptr, sptr) \
162 (ops != NULL ? (*ops->vmrun)(vmi, vcpu, rip, pmap, rptr, sptr) : ENXIO)
163#define VMCLEANUP(vmi) (ops != NULL ? (*ops->vmcleanup)(vmi) : NULL)
164#define VMSPACE_ALLOC(min, max) \
165 (ops != NULL ? (*ops->vmspace_alloc)(min, max) : NULL)
166#define VMSPACE_FREE(vmspace) \
167 (ops != NULL ? (*ops->vmspace_free)(vmspace) : ENXIO)
168#define VMGETREG(vmi, vcpu, num, retval) \
169 (ops != NULL ? (*ops->vmgetreg)(vmi, vcpu, num, retval) : ENXIO)
170#define VMSETREG(vmi, vcpu, num, val) \
171 (ops != NULL ? (*ops->vmsetreg)(vmi, vcpu, num, val) : ENXIO)
172#define VMGETDESC(vmi, vcpu, num, desc) \
173 (ops != NULL ? (*ops->vmgetdesc)(vmi, vcpu, num, desc) : ENXIO)
174#define VMSETDESC(vmi, vcpu, num, desc) \
175 (ops != NULL ? (*ops->vmsetdesc)(vmi, vcpu, num, desc) : ENXIO)
176#define VMGETCAP(vmi, vcpu, num, retval) \
177 (ops != NULL ? (*ops->vmgetcap)(vmi, vcpu, num, retval) : ENXIO)
178#define VMSETCAP(vmi, vcpu, num, val) \
179 (ops != NULL ? (*ops->vmsetcap)(vmi, vcpu, num, val) : ENXIO)
180#define VLAPIC_INIT(vmi, vcpu) \
181 (ops != NULL ? (*ops->vlapic_init)(vmi, vcpu) : NULL)
182#define VLAPIC_CLEANUP(vmi, vlapic) \
183 (ops != NULL ? (*ops->vlapic_cleanup)(vmi, vlapic) : NULL)
184
185#define fpu_start_emulating() load_cr0(rcr0() | CR0_TS)
186#define fpu_stop_emulating() clts()
187
188static MALLOC_DEFINE(M_VM, "vm", "vm");
189
190/* statistics */
191static VMM_STAT(VCPU_TOTAL_RUNTIME, "vcpu total runtime");
192
193SYSCTL_NODE(_hw, OID_AUTO, vmm, CTLFLAG_RW, NULL, NULL);
194
195/*
196 * Halt the guest if all vcpus are executing a HLT instruction with
197 * interrupts disabled.
198 */
199static int halt_detection_enabled = 1;
200SYSCTL_INT(_hw_vmm, OID_AUTO, halt_detection, CTLFLAG_RDTUN,
201 &halt_detection_enabled, 0,
202 "Halt VM if all vcpus execute HLT with interrupts disabled");
203
204static int vmm_ipinum;
205SYSCTL_INT(_hw_vmm, OID_AUTO, ipinum, CTLFLAG_RD, &vmm_ipinum, 0,
206 "IPI vector used for vcpu notifications");
207
208static void
209vcpu_cleanup(struct vm *vm, int i, bool destroy)
210{
211 struct vcpu *vcpu = &vm->vcpu[i];
212
213 VLAPIC_CLEANUP(vm->cookie, vcpu->vlapic);
214 if (destroy) {
215 vmm_stat_free(vcpu->stats);
216 fpu_save_area_free(vcpu->guestfpu);
217 }
218}
219
220static void
221vcpu_init(struct vm *vm, int vcpu_id, bool create)
222{
223 struct vcpu *vcpu;
224
225 KASSERT(vcpu_id >= 0 && vcpu_id < VM_MAXCPU,
226 ("vcpu_init: invalid vcpu %d", vcpu_id));
227
228 vcpu = &vm->vcpu[vcpu_id];
229
230 if (create) {
231 KASSERT(!vcpu_lock_initialized(vcpu), ("vcpu %d already "
232 "initialized", vcpu_id));
233 vcpu_lock_init(vcpu);
234 vcpu->state = VCPU_IDLE;
235 vcpu->hostcpu = NOCPU;
236 vcpu->guestfpu = fpu_save_area_alloc();
237 vcpu->stats = vmm_stat_alloc();
238 }
239
240 vcpu->vlapic = VLAPIC_INIT(vm->cookie, vcpu_id);
241 vm_set_x2apic_state(vm, vcpu_id, X2APIC_DISABLED);
242 vcpu->exitintinfo = 0;
243 vcpu->nmi_pending = 0;
244 vcpu->extint_pending = 0;
245 vcpu->exception_pending = 0;
246 vcpu->guest_xcr0 = XFEATURE_ENABLED_X87;
247 fpu_save_area_reset(vcpu->guestfpu);
248 vmm_stat_init(vcpu->stats);
249}
250
251struct vm_exit *
252vm_exitinfo(struct vm *vm, int cpuid)
253{
254 struct vcpu *vcpu;
255
256 if (cpuid < 0 || cpuid >= VM_MAXCPU)
257 panic("vm_exitinfo: invalid cpuid %d", cpuid);
258
259 vcpu = &vm->vcpu[cpuid];
260
261 return (&vcpu->exitinfo);
262}
263
264static void
265vmm_resume(void)
266{
267 VMM_RESUME();
268}
269
270static int
271vmm_init(void)
272{
273 int error;
274
275 vmm_host_state_init();
276
277 vmm_ipinum = vmm_ipi_alloc();
278 if (vmm_ipinum == 0)
279 vmm_ipinum = IPI_AST;
280
281 error = vmm_mem_init();
282 if (error)
283 return (error);
284
285 if (vmm_is_intel())
286 ops = &vmm_ops_intel;
287 else if (vmm_is_amd())
288 ops = &vmm_ops_amd;
289 else
290 return (ENXIO);
291
292 vmm_resume_p = vmm_resume;
293
294 return (VMM_INIT(vmm_ipinum));
295}
296
297static int
298vmm_handler(module_t mod, int what, void *arg)
299{
300 int error;
301
302 switch (what) {
303 case MOD_LOAD:
304 vmmdev_init();
305 if (ppt_avail_devices() > 0)
306 iommu_init();
307 error = vmm_init();
308 if (error == 0)
309 vmm_initialized = 1;
310 break;
311 case MOD_UNLOAD:
312 error = vmmdev_cleanup();
313 if (error == 0) {
314 vmm_resume_p = NULL;
315 iommu_cleanup();
316 if (vmm_ipinum != IPI_AST)
317 vmm_ipi_free(vmm_ipinum);
318 error = VMM_CLEANUP();
319 /*
320 * Something bad happened - prevent new
321 * VMs from being created
322 */
323 if (error)
324 vmm_initialized = 0;
325 }
326 break;
327 default:
328 error = 0;
329 break;
330 }
331 return (error);
332}
333
334static moduledata_t vmm_kmod = {
335 "vmm",
336 vmm_handler,
337 NULL
338};
339
340/*
341 * vmm initialization has the following dependencies:
342 *
343 * - iommu initialization must happen after the pci passthru driver has had
344 * a chance to attach to any passthru devices (after SI_SUB_CONFIGURE).
345 *
346 * - VT-x initialization requires smp_rendezvous() and therefore must happen
347 * after SMP is fully functional (after SI_SUB_SMP).
348 */
349DECLARE_MODULE(vmm, vmm_kmod, SI_SUB_SMP + 1, SI_ORDER_ANY);
350MODULE_VERSION(vmm, 1);
351
352static void
353vm_init(struct vm *vm, bool create)
354{
355 int i;
356
357 vm->cookie = VMINIT(vm, vmspace_pmap(vm->vmspace));
358 vm->iommu = NULL;
359 vm->vioapic = vioapic_init(vm);
360 vm->vhpet = vhpet_init(vm);
361 vm->vatpic = vatpic_init(vm);
362 vm->vatpit = vatpit_init(vm);
363
364 CPU_ZERO(&vm->active_cpus);
365
366 vm->suspend = 0;
367 CPU_ZERO(&vm->suspended_cpus);
368
369 for (i = 0; i < VM_MAXCPU; i++)
370 vcpu_init(vm, i, create);
371}
372
373int
374vm_create(const char *name, struct vm **retvm)
375{
376 struct vm *vm;
377 struct vmspace *vmspace;
378
379 /*
380 * If vmm.ko could not be successfully initialized then don't attempt
381 * to create the virtual machine.
382 */
383 if (!vmm_initialized)
384 return (ENXIO);
385
386 if (name == NULL || strlen(name) >= VM_MAX_NAMELEN)
387 return (EINVAL);
388
389 vmspace = VMSPACE_ALLOC(VM_MIN_ADDRESS, VM_MAXUSER_ADDRESS);
390 if (vmspace == NULL)
391 return (ENOMEM);
392
393 vm = malloc(sizeof(struct vm), M_VM, M_WAITOK | M_ZERO);
394 strcpy(vm->name, name);
395 vm->num_mem_segs = 0;
396 vm->vmspace = vmspace;
397 mtx_init(&vm->rendezvous_mtx, "vm rendezvous lock", 0, MTX_DEF);
398
399 vm_init(vm, true);
400
401 *retvm = vm;
402 return (0);
403}
404
405static void
406vm_free_mem_seg(struct vm *vm, struct mem_seg *seg)
407{
408
409 if (seg->object != NULL)
410 vmm_mem_free(vm->vmspace, seg->gpa, seg->len);
411
412 bzero(seg, sizeof(*seg));
413}
414
415static void
416vm_cleanup(struct vm *vm, bool destroy)
417{
418 int i;
419
420 ppt_unassign_all(vm);
421
422 if (vm->iommu != NULL)
423 iommu_destroy_domain(vm->iommu);
424
425 vatpit_cleanup(vm->vatpit);
426 vhpet_cleanup(vm->vhpet);
427 vatpic_cleanup(vm->vatpic);
428 vioapic_cleanup(vm->vioapic);
429
430 for (i = 0; i < VM_MAXCPU; i++)
431 vcpu_cleanup(vm, i, destroy);
432
433 VMCLEANUP(vm->cookie);
434
435 if (destroy) {
436 for (i = 0; i < vm->num_mem_segs; i++)
437 vm_free_mem_seg(vm, &vm->mem_segs[i]);
438
439 vm->num_mem_segs = 0;
440
441 VMSPACE_FREE(vm->vmspace);
442 vm->vmspace = NULL;
443 }
444}
445
446void
447vm_destroy(struct vm *vm)
448{
449 vm_cleanup(vm, true);
450 free(vm, M_VM);
451}
452
453int
454vm_reinit(struct vm *vm)
455{
456 int error;
457
458 /*
459 * A virtual machine can be reset only if all vcpus are suspended.
460 */
461 if (CPU_CMP(&vm->suspended_cpus, &vm->active_cpus) == 0) {
462 vm_cleanup(vm, false);
463 vm_init(vm, false);
464 error = 0;
465 } else {
466 error = EBUSY;
467 }
468
469 return (error);
470}
471
472const char *
473vm_name(struct vm *vm)
474{
475 return (vm->name);
476}
477
478int
479vm_map_mmio(struct vm *vm, vm_paddr_t gpa, size_t len, vm_paddr_t hpa)
480{
481 vm_object_t obj;
482
483 if ((obj = vmm_mmio_alloc(vm->vmspace, gpa, len, hpa)) == NULL)
484 return (ENOMEM);
485 else
486 return (0);
487}
488
489int
490vm_unmap_mmio(struct vm *vm, vm_paddr_t gpa, size_t len)
491{
492
493 vmm_mmio_free(vm->vmspace, gpa, len);
494 return (0);
495}
496
497boolean_t
498vm_mem_allocated(struct vm *vm, vm_paddr_t gpa)
499{
500 int i;
501 vm_paddr_t gpabase, gpalimit;
502
503 for (i = 0; i < vm->num_mem_segs; i++) {
504 gpabase = vm->mem_segs[i].gpa;
505 gpalimit = gpabase + vm->mem_segs[i].len;
506 if (gpa >= gpabase && gpa < gpalimit)
507 return (TRUE); /* 'gpa' is regular memory */
508 }
509
510 if (ppt_is_mmio(vm, gpa))
511 return (TRUE); /* 'gpa' is pci passthru mmio */
512
513 return (FALSE);
514}
515
516int
517vm_malloc(struct vm *vm, vm_paddr_t gpa, size_t len)
518{
519 int available, allocated;
520 struct mem_seg *seg;
521 vm_object_t object;
522 vm_paddr_t g;
523
524 if ((gpa & PAGE_MASK) || (len & PAGE_MASK) || len == 0)
525 return (EINVAL);
526
527 available = allocated = 0;
528 g = gpa;
529 while (g < gpa + len) {
530 if (vm_mem_allocated(vm, g))
531 allocated++;
532 else
533 available++;
534
535 g += PAGE_SIZE;
536 }
537
538 /*
539 * If there are some allocated and some available pages in the address
540 * range then it is an error.
541 */
542 if (allocated && available)
543 return (EINVAL);
544
545 /*
546 * If the entire address range being requested has already been
547 * allocated then there isn't anything more to do.
548 */
549 if (allocated && available == 0)
550 return (0);
551
552 if (vm->num_mem_segs >= VM_MAX_MEMORY_SEGMENTS)
553 return (E2BIG);
554
555 seg = &vm->mem_segs[vm->num_mem_segs];
556
557 if ((object = vmm_mem_alloc(vm->vmspace, gpa, len)) == NULL)
558 return (ENOMEM);
559
560 seg->gpa = gpa;
561 seg->len = len;
562 seg->object = object;
563 seg->wired = FALSE;
564
565 vm->num_mem_segs++;
566
567 return (0);
568}
569
570static vm_paddr_t
571vm_maxmem(struct vm *vm)
572{
573 int i;
574 vm_paddr_t gpa, maxmem;
575
576 maxmem = 0;
577 for (i = 0; i < vm->num_mem_segs; i++) {
578 gpa = vm->mem_segs[i].gpa + vm->mem_segs[i].len;
579 if (gpa > maxmem)
580 maxmem = gpa;
581 }
582 return (maxmem);
583}
584
585static void
586vm_gpa_unwire(struct vm *vm)
587{
588 int i, rv;
589 struct mem_seg *seg;
590
591 for (i = 0; i < vm->num_mem_segs; i++) {
592 seg = &vm->mem_segs[i];
593 if (!seg->wired)
594 continue;
595
596 rv = vm_map_unwire(&vm->vmspace->vm_map,
597 seg->gpa, seg->gpa + seg->len,
598 VM_MAP_WIRE_USER | VM_MAP_WIRE_NOHOLES);
599 KASSERT(rv == KERN_SUCCESS, ("vm(%s) memory segment "
600 "%#lx/%ld could not be unwired: %d",
601 vm_name(vm), seg->gpa, seg->len, rv));
602
603 seg->wired = FALSE;
604 }
605}
606
607static int
608vm_gpa_wire(struct vm *vm)
609{
610 int i, rv;
611 struct mem_seg *seg;
612
613 for (i = 0; i < vm->num_mem_segs; i++) {
614 seg = &vm->mem_segs[i];
615 if (seg->wired)
616 continue;
617
618 /* XXX rlimits? */
619 rv = vm_map_wire(&vm->vmspace->vm_map,
620 seg->gpa, seg->gpa + seg->len,
621 VM_MAP_WIRE_USER | VM_MAP_WIRE_NOHOLES);
622 if (rv != KERN_SUCCESS)
623 break;
624
625 seg->wired = TRUE;
626 }
627
628 if (i < vm->num_mem_segs) {
629 /*
630 * Undo the wiring before returning an error.
631 */
632 vm_gpa_unwire(vm);
633 return (EAGAIN);
634 }
635
636 return (0);
637}
638
639static void
640vm_iommu_modify(struct vm *vm, boolean_t map)
641{
642 int i, sz;
643 vm_paddr_t gpa, hpa;
644 struct mem_seg *seg;
645 void *vp, *cookie, *host_domain;
646
647 sz = PAGE_SIZE;
648 host_domain = iommu_host_domain();
649
650 for (i = 0; i < vm->num_mem_segs; i++) {
651 seg = &vm->mem_segs[i];
652 KASSERT(seg->wired, ("vm(%s) memory segment %#lx/%ld not wired",
653 vm_name(vm), seg->gpa, seg->len));
654
655 gpa = seg->gpa;
656 while (gpa < seg->gpa + seg->len) {
657 vp = vm_gpa_hold(vm, gpa, PAGE_SIZE, VM_PROT_WRITE,
658 &cookie);
659 KASSERT(vp != NULL, ("vm(%s) could not map gpa %#lx",
660 vm_name(vm), gpa));
661
662 vm_gpa_release(cookie);
663
664 hpa = DMAP_TO_PHYS((uintptr_t)vp);
665 if (map) {
666 iommu_create_mapping(vm->iommu, gpa, hpa, sz);
667 iommu_remove_mapping(host_domain, hpa, sz);
668 } else {
669 iommu_remove_mapping(vm->iommu, gpa, sz);
670 iommu_create_mapping(host_domain, hpa, hpa, sz);
671 }
672
673 gpa += PAGE_SIZE;
674 }
675 }
676
677 /*
678 * Invalidate the cached translations associated with the domain
679 * from which pages were removed.
680 */
681 if (map)
682 iommu_invalidate_tlb(host_domain);
683 else
684 iommu_invalidate_tlb(vm->iommu);
685}
686
687#define vm_iommu_unmap(vm) vm_iommu_modify((vm), FALSE)
688#define vm_iommu_map(vm) vm_iommu_modify((vm), TRUE)
689
690int
691vm_unassign_pptdev(struct vm *vm, int bus, int slot, int func)
692{
693 int error;
694
695 error = ppt_unassign_device(vm, bus, slot, func);
696 if (error)
697 return (error);
698
699 if (ppt_assigned_devices(vm) == 0) {
700 vm_iommu_unmap(vm);
701 vm_gpa_unwire(vm);
702 }
703 return (0);
704}
705
706int
707vm_assign_pptdev(struct vm *vm, int bus, int slot, int func)
708{
709 int error;
710 vm_paddr_t maxaddr;
711
712 /*
713 * Virtual machines with pci passthru devices get special treatment:
714 * - the guest physical memory is wired
715 * - the iommu is programmed to do the 'gpa' to 'hpa' translation
716 *
717 * We need to do this before the first pci passthru device is attached.
718 */
719 if (ppt_assigned_devices(vm) == 0) {
720 KASSERT(vm->iommu == NULL,
721 ("vm_assign_pptdev: iommu must be NULL"));
722 maxaddr = vm_maxmem(vm);
723 vm->iommu = iommu_create_domain(maxaddr);
724
725 error = vm_gpa_wire(vm);
726 if (error)
727 return (error);
728
729 vm_iommu_map(vm);
730 }
731
732 error = ppt_assign_device(vm, bus, slot, func);
733 return (error);
734}
735
736void *
737vm_gpa_hold(struct vm *vm, vm_paddr_t gpa, size_t len, int reqprot,
738 void **cookie)
739{
740 int count, pageoff;
741 vm_page_t m;
742
743 pageoff = gpa & PAGE_MASK;
744 if (len > PAGE_SIZE - pageoff)
745 panic("vm_gpa_hold: invalid gpa/len: 0x%016lx/%lu", gpa, len);
746
747 count = vm_fault_quick_hold_pages(&vm->vmspace->vm_map,
748 trunc_page(gpa), PAGE_SIZE, reqprot, &m, 1);
749
750 if (count == 1) {
751 *cookie = m;
752 return ((void *)(PHYS_TO_DMAP(VM_PAGE_TO_PHYS(m)) + pageoff));
753 } else {
754 *cookie = NULL;
755 return (NULL);
756 }
757}
758
759void
760vm_gpa_release(void *cookie)
761{
762 vm_page_t m = cookie;
763
764 vm_page_lock(m);
765 vm_page_unhold(m);
766 vm_page_unlock(m);
767}
768
769int
770vm_gpabase2memseg(struct vm *vm, vm_paddr_t gpabase,
771 struct vm_memory_segment *seg)
772{
773 int i;
774
775 for (i = 0; i < vm->num_mem_segs; i++) {
776 if (gpabase == vm->mem_segs[i].gpa) {
777 seg->gpa = vm->mem_segs[i].gpa;
778 seg->len = vm->mem_segs[i].len;
779 seg->wired = vm->mem_segs[i].wired;
780 return (0);
781 }
782 }
783 return (-1);
784}
785
786int
787vm_get_memobj(struct vm *vm, vm_paddr_t gpa, size_t len,
788 vm_offset_t *offset, struct vm_object **object)
789{
790 int i;
791 size_t seg_len;
792 vm_paddr_t seg_gpa;
793 vm_object_t seg_obj;
794
795 for (i = 0; i < vm->num_mem_segs; i++) {
796 if ((seg_obj = vm->mem_segs[i].object) == NULL)
797 continue;
798
799 seg_gpa = vm->mem_segs[i].gpa;
800 seg_len = vm->mem_segs[i].len;
801
802 if (gpa >= seg_gpa && gpa < seg_gpa + seg_len) {
803 *offset = gpa - seg_gpa;
804 *object = seg_obj;
805 vm_object_reference(seg_obj);
806 return (0);
807 }
808 }
809
810 return (EINVAL);
811}
812
813int
814vm_get_register(struct vm *vm, int vcpu, int reg, uint64_t *retval)
815{
816
817 if (vcpu < 0 || vcpu >= VM_MAXCPU)
818 return (EINVAL);
819
820 if (reg >= VM_REG_LAST)
821 return (EINVAL);
822
823 return (VMGETREG(vm->cookie, vcpu, reg, retval));
824}
825
826int
827vm_set_register(struct vm *vm, int vcpu, int reg, uint64_t val)
828{
829
830 if (vcpu < 0 || vcpu >= VM_MAXCPU)
831 return (EINVAL);
832
833 if (reg >= VM_REG_LAST)
834 return (EINVAL);
835
836 return (VMSETREG(vm->cookie, vcpu, reg, val));
837}
838
839static boolean_t
840is_descriptor_table(int reg)
841{
842
843 switch (reg) {
844 case VM_REG_GUEST_IDTR:
845 case VM_REG_GUEST_GDTR:
846 return (TRUE);
847 default:
848 return (FALSE);
849 }
850}
851
852static boolean_t
853is_segment_register(int reg)
854{
855
856 switch (reg) {
857 case VM_REG_GUEST_ES:
858 case VM_REG_GUEST_CS:
859 case VM_REG_GUEST_SS:
860 case VM_REG_GUEST_DS:
861 case VM_REG_GUEST_FS:
862 case VM_REG_GUEST_GS:
863 case VM_REG_GUEST_TR:
864 case VM_REG_GUEST_LDTR:
865 return (TRUE);
866 default:
867 return (FALSE);
868 }
869}
870
871int
872vm_get_seg_desc(struct vm *vm, int vcpu, int reg,
873 struct seg_desc *desc)
874{
875
876 if (vcpu < 0 || vcpu >= VM_MAXCPU)
877 return (EINVAL);
878
879 if (!is_segment_register(reg) && !is_descriptor_table(reg))
880 return (EINVAL);
881
882 return (VMGETDESC(vm->cookie, vcpu, reg, desc));
883}
884
885int
886vm_set_seg_desc(struct vm *vm, int vcpu, int reg,
887 struct seg_desc *desc)
888{
889 if (vcpu < 0 || vcpu >= VM_MAXCPU)
890 return (EINVAL);
891
892 if (!is_segment_register(reg) && !is_descriptor_table(reg))
893 return (EINVAL);
894
895 return (VMSETDESC(vm->cookie, vcpu, reg, desc));
896}
897
898static void
899restore_guest_fpustate(struct vcpu *vcpu)
900{
901
902 /* flush host state to the pcb */
903 fpuexit(curthread);
904
905 /* restore guest FPU state */
906 fpu_stop_emulating();
907 fpurestore(vcpu->guestfpu);
908
909 /* restore guest XCR0 if XSAVE is enabled in the host */
910 if (rcr4() & CR4_XSAVE)
911 load_xcr(0, vcpu->guest_xcr0);
912
913 /*
914 * The FPU is now "dirty" with the guest's state so turn on emulation
915 * to trap any access to the FPU by the host.
916 */
917 fpu_start_emulating();
918}
919
920static void
921save_guest_fpustate(struct vcpu *vcpu)
922{
923
924 if ((rcr0() & CR0_TS) == 0)
925 panic("fpu emulation not enabled in host!");
926
927 /* save guest XCR0 and restore host XCR0 */
928 if (rcr4() & CR4_XSAVE) {
929 vcpu->guest_xcr0 = rxcr(0);
930 load_xcr(0, vmm_get_host_xcr0());
931 }
932
933 /* save guest FPU state */
934 fpu_stop_emulating();
935 fpusave(vcpu->guestfpu);
936 fpu_start_emulating();
937}
938
939static VMM_STAT(VCPU_IDLE_TICKS, "number of ticks vcpu was idle");
940
941static int
942vcpu_set_state_locked(struct vcpu *vcpu, enum vcpu_state newstate,
943 bool from_idle)
944{
945 int error;
946
947 vcpu_assert_locked(vcpu);
948
949 /*
950 * State transitions from the vmmdev_ioctl() must always begin from
951 * the VCPU_IDLE state. This guarantees that there is only a single
952 * ioctl() operating on a vcpu at any point.
953 */
954 if (from_idle) {
955 while (vcpu->state != VCPU_IDLE)
956 msleep_spin(&vcpu->state, &vcpu->mtx, "vmstat", hz);
957 } else {
958 KASSERT(vcpu->state != VCPU_IDLE, ("invalid transition from "
959 "vcpu idle state"));
960 }
961
962 if (vcpu->state == VCPU_RUNNING) {
963 KASSERT(vcpu->hostcpu == curcpu, ("curcpu %d and hostcpu %d "
964 "mismatch for running vcpu", curcpu, vcpu->hostcpu));
965 } else {
966 KASSERT(vcpu->hostcpu == NOCPU, ("Invalid hostcpu %d for a "
967 "vcpu that is not running", vcpu->hostcpu));
968 }
969
970 /*
971 * The following state transitions are allowed:
972 * IDLE -> FROZEN -> IDLE
973 * FROZEN -> RUNNING -> FROZEN
974 * FROZEN -> SLEEPING -> FROZEN
975 */
976 switch (vcpu->state) {
977 case VCPU_IDLE:
978 case VCPU_RUNNING:
979 case VCPU_SLEEPING:
980 error = (newstate != VCPU_FROZEN);
981 break;
982 case VCPU_FROZEN:
983 error = (newstate == VCPU_FROZEN);
984 break;
985 default:
986 error = 1;
987 break;
988 }
989
990 if (error)
991 return (EBUSY);
992
993 vcpu->state = newstate;
994 if (newstate == VCPU_RUNNING)
995 vcpu->hostcpu = curcpu;
996 else
997 vcpu->hostcpu = NOCPU;
998
999 if (newstate == VCPU_IDLE)
1000 wakeup(&vcpu->state);
1001
1002 return (0);
1003}
1004
1005static void
1006vcpu_require_state(struct vm *vm, int vcpuid, enum vcpu_state newstate)
1007{
1008 int error;
1009
1010 if ((error = vcpu_set_state(vm, vcpuid, newstate, false)) != 0)
1011 panic("Error %d setting state to %d\n", error, newstate);
1012}
1013
1014static void
1015vcpu_require_state_locked(struct vcpu *vcpu, enum vcpu_state newstate)
1016{
1017 int error;
1018
1019 if ((error = vcpu_set_state_locked(vcpu, newstate, false)) != 0)
1020 panic("Error %d setting state to %d", error, newstate);
1021}
1022
1023static void
1024vm_set_rendezvous_func(struct vm *vm, vm_rendezvous_func_t func)
1025{
1026
1027 KASSERT(mtx_owned(&vm->rendezvous_mtx), ("rendezvous_mtx not locked"));
1028
1029 /*
1030 * Update 'rendezvous_func' and execute a write memory barrier to
1031 * ensure that it is visible across all host cpus. This is not needed
1032 * for correctness but it does ensure that all the vcpus will notice
1033 * that the rendezvous is requested immediately.
1034 */
1035 vm->rendezvous_func = func;
1036 wmb();
1037}
1038
1039#define RENDEZVOUS_CTR0(vm, vcpuid, fmt) \
1040 do { \
1041 if (vcpuid >= 0) \
1042 VCPU_CTR0(vm, vcpuid, fmt); \
1043 else \
1044 VM_CTR0(vm, fmt); \
1045 } while (0)
1046
1047static void
1048vm_handle_rendezvous(struct vm *vm, int vcpuid)
1049{
1050
1051 KASSERT(vcpuid == -1 || (vcpuid >= 0 && vcpuid < VM_MAXCPU),
1052 ("vm_handle_rendezvous: invalid vcpuid %d", vcpuid));
1053
1054 mtx_lock(&vm->rendezvous_mtx);
1055 while (vm->rendezvous_func != NULL) {
1056 /* 'rendezvous_req_cpus' must be a subset of 'active_cpus' */
1057 CPU_AND(&vm->rendezvous_req_cpus, &vm->active_cpus);
1058
1059 if (vcpuid != -1 &&
1060 CPU_ISSET(vcpuid, &vm->rendezvous_req_cpus) &&
1061 !CPU_ISSET(vcpuid, &vm->rendezvous_done_cpus)) {
1062 VCPU_CTR0(vm, vcpuid, "Calling rendezvous func");
1063 (*vm->rendezvous_func)(vm, vcpuid, vm->rendezvous_arg);
1064 CPU_SET(vcpuid, &vm->rendezvous_done_cpus);
1065 }
1066 if (CPU_CMP(&vm->rendezvous_req_cpus,
1067 &vm->rendezvous_done_cpus) == 0) {
1068 VCPU_CTR0(vm, vcpuid, "Rendezvous completed");
1069 vm_set_rendezvous_func(vm, NULL);
1070 wakeup(&vm->rendezvous_func);
1071 break;
1072 }
1073 RENDEZVOUS_CTR0(vm, vcpuid, "Wait for rendezvous completion");
1074 mtx_sleep(&vm->rendezvous_func, &vm->rendezvous_mtx, 0,
1075 "vmrndv", 0);
1076 }
1077 mtx_unlock(&vm->rendezvous_mtx);
1078}
1079
1080/*
1081 * Emulate a guest 'hlt' by sleeping until the vcpu is ready to run.
1082 */
1083static int
1084vm_handle_hlt(struct vm *vm, int vcpuid, bool intr_disabled, bool *retu)
1085{
1086 struct vcpu *vcpu;
1087 const char *wmesg;
1088 int error, t, vcpu_halted, vm_halted;
1089
1090 KASSERT(!CPU_ISSET(vcpuid, &vm->halted_cpus), ("vcpu already halted"));
1091
1092 vcpu = &vm->vcpu[vcpuid];
1093 vcpu_halted = 0;
1094 vm_halted = 0;
1095
1096 /*
1097 * The typical way to halt a cpu is to execute: "sti; hlt"
1098 *
1099 * STI sets RFLAGS.IF to enable interrupts. However, the processor
1100 * remains in an "interrupt shadow" for an additional instruction
1101 * following the STI. This guarantees that "sti; hlt" sequence is
1102 * atomic and a pending interrupt will be recognized after the HLT.
1103 *
1104 * After the HLT emulation is done the vcpu is no longer in an
1105 * interrupt shadow and a pending interrupt can be injected on
1106 * the next entry into the guest.
1107 */
1108 error = vm_set_register(vm, vcpuid, VM_REG_GUEST_INTR_SHADOW, 0);
1109 KASSERT(error == 0, ("%s: error %d clearing interrupt shadow",
1110 __func__, error));
1111
1112 vcpu_lock(vcpu);
1113 while (1) {
1114 /*
1115 * Do a final check for pending NMI or interrupts before
1116 * really putting this thread to sleep. Also check for
1117 * software events that would cause this vcpu to wakeup.
1118 *
1119 * These interrupts/events could have happened after the
1120 * vcpu returned from VMRUN() and before it acquired the
1121 * vcpu lock above.
1122 */
1123 if (vm->rendezvous_func != NULL || vm->suspend)
1124 break;
1125 if (vm_nmi_pending(vm, vcpuid))
1126 break;
1127 if (!intr_disabled) {
1128 if (vm_extint_pending(vm, vcpuid) ||
1129 vlapic_pending_intr(vcpu->vlapic, NULL)) {
1130 break;
1131 }
1132 }
1133
1134 /* Don't go to sleep if the vcpu thread needs to yield */
1135 if (vcpu_should_yield(vm, vcpuid))
1136 break;
1137
1138 /*
1139 * Some Linux guests implement "halt" by having all vcpus
1140 * execute HLT with interrupts disabled. 'halted_cpus' keeps
1141 * track of the vcpus that have entered this state. When all
1142 * vcpus enter the halted state the virtual machine is halted.
1143 */
1144 if (intr_disabled) {
1145 wmesg = "vmhalt";
1146 VCPU_CTR0(vm, vcpuid, "Halted");
1147 if (!vcpu_halted && halt_detection_enabled) {
1148 vcpu_halted = 1;
1149 CPU_SET_ATOMIC(vcpuid, &vm->halted_cpus);
1150 }
1151 if (CPU_CMP(&vm->halted_cpus, &vm->active_cpus) == 0) {
1152 vm_halted = 1;
1153 break;
1154 }
1155 } else {
1156 wmesg = "vmidle";
1157 }
1158
1159 t = ticks;
1160 vcpu_require_state_locked(vcpu, VCPU_SLEEPING);
1161 /*
1162 * XXX msleep_spin() cannot be interrupted by signals so
1163 * wake up periodically to check pending signals.
1164 */
1165 msleep_spin(vcpu, &vcpu->mtx, wmesg, hz);
1166 vcpu_require_state_locked(vcpu, VCPU_FROZEN);
1167 vmm_stat_incr(vm, vcpuid, VCPU_IDLE_TICKS, ticks - t);
1168 }
1169
1170 if (vcpu_halted)
1171 CPU_CLR_ATOMIC(vcpuid, &vm->halted_cpus);
1172
1173 vcpu_unlock(vcpu);
1174
1175 if (vm_halted)
1176 vm_suspend(vm, VM_SUSPEND_HALT);
1177
1178 return (0);
1179}
1180
1181static int
1182vm_handle_paging(struct vm *vm, int vcpuid, bool *retu)
1183{
1184 int rv, ftype;
1185 struct vm_map *map;
1186 struct vcpu *vcpu;
1187 struct vm_exit *vme;
1188
1189 vcpu = &vm->vcpu[vcpuid];
1190 vme = &vcpu->exitinfo;
1191
1192 ftype = vme->u.paging.fault_type;
1193 KASSERT(ftype == VM_PROT_READ ||
1194 ftype == VM_PROT_WRITE || ftype == VM_PROT_EXECUTE,
1195 ("vm_handle_paging: invalid fault_type %d", ftype));
1196
1197 if (ftype == VM_PROT_READ || ftype == VM_PROT_WRITE) {
1198 rv = pmap_emulate_accessed_dirty(vmspace_pmap(vm->vmspace),
1199 vme->u.paging.gpa, ftype);
1200 if (rv == 0) {
1201 VCPU_CTR2(vm, vcpuid, "%s bit emulation for gpa %#lx",
1202 ftype == VM_PROT_READ ? "accessed" : "dirty",
1203 vme->u.paging.gpa);
1204 goto done;
1205 }
1206 }
1207
1208 map = &vm->vmspace->vm_map;
1209 rv = vm_fault(map, vme->u.paging.gpa, ftype, VM_FAULT_NORMAL);
1210
1211 VCPU_CTR3(vm, vcpuid, "vm_handle_paging rv = %d, gpa = %#lx, "
1212 "ftype = %d", rv, vme->u.paging.gpa, ftype);
1213
1214 if (rv != KERN_SUCCESS)
1215 return (EFAULT);
1216done:
1217 /* restart execution at the faulting instruction */
1218 vme->inst_length = 0;
1219
1220 return (0);
1221}
1222
1223static int
1224vm_handle_inst_emul(struct vm *vm, int vcpuid, bool *retu)
1225{
1226 struct vie *vie;
1227 struct vcpu *vcpu;
1228 struct vm_exit *vme;
1229 uint64_t gla, gpa;
1230 struct vm_guest_paging *paging;
1231 mem_region_read_t mread;
1232 mem_region_write_t mwrite;
1233 enum vm_cpu_mode cpu_mode;
1234 int cs_d, error;
1235
1236 vcpu = &vm->vcpu[vcpuid];
1237 vme = &vcpu->exitinfo;
1238
1239 gla = vme->u.inst_emul.gla;
1240 gpa = vme->u.inst_emul.gpa;
1241 cs_d = vme->u.inst_emul.cs_d;
1242 vie = &vme->u.inst_emul.vie;
1243 paging = &vme->u.inst_emul.paging;
1244 cpu_mode = paging->cpu_mode;
1245
1246 VCPU_CTR1(vm, vcpuid, "inst_emul fault accessing gpa %#lx", gpa);
1247
1248 vie_init(vie);
1249
1250 /* Fetch, decode and emulate the faulting instruction */
1251 error = vmm_fetch_instruction(vm, vcpuid, paging, vme->rip,
1252 vme->inst_length, vie);
1253 if (error == 1)
1254 return (0); /* Resume guest to handle page fault */
1255 else if (error == -1)
1256 return (EFAULT);
1257 else if (error != 0)
1258 panic("%s: vmm_fetch_instruction error %d", __func__, error);
1259
1260 if (vmm_decode_instruction(vm, vcpuid, gla, cpu_mode, cs_d, vie) != 0)
1261 return (EFAULT);
1262
1263 /* return to userland unless this is an in-kernel emulated device */
1264 if (gpa >= DEFAULT_APIC_BASE && gpa < DEFAULT_APIC_BASE + PAGE_SIZE) {
1265 mread = lapic_mmio_read;
1266 mwrite = lapic_mmio_write;
1267 } else if (gpa >= VIOAPIC_BASE && gpa < VIOAPIC_BASE + VIOAPIC_SIZE) {
1268 mread = vioapic_mmio_read;
1269 mwrite = vioapic_mmio_write;
1270 } else if (gpa >= VHPET_BASE && gpa < VHPET_BASE + VHPET_SIZE) {
1271 mread = vhpet_mmio_read;
1272 mwrite = vhpet_mmio_write;
1273 } else {
1274 *retu = true;
1275 return (0);
1276 }
1277
1278 error = vmm_emulate_instruction(vm, vcpuid, gpa, vie, paging,
1279 mread, mwrite, retu);
1280
1281 return (error);
1282}
1283
1284static int
1285vm_handle_suspend(struct vm *vm, int vcpuid, bool *retu)
1286{
1287 int i, done;
1288 struct vcpu *vcpu;
1289
1290 done = 0;
1291 vcpu = &vm->vcpu[vcpuid];
1292
1293 CPU_SET_ATOMIC(vcpuid, &vm->suspended_cpus);
1294
1295 /*
1296 * Wait until all 'active_cpus' have suspended themselves.
1297 *
1298 * Since a VM may be suspended at any time including when one or
1299 * more vcpus are doing a rendezvous we need to call the rendezvous
1300 * handler while we are waiting to prevent a deadlock.
1301 */
1302 vcpu_lock(vcpu);
1303 while (1) {
1304 if (CPU_CMP(&vm->suspended_cpus, &vm->active_cpus) == 0) {
1305 VCPU_CTR0(vm, vcpuid, "All vcpus suspended");
1306 break;
1307 }
1308
1309 if (vm->rendezvous_func == NULL) {
1310 VCPU_CTR0(vm, vcpuid, "Sleeping during suspend");
1311 vcpu_require_state_locked(vcpu, VCPU_SLEEPING);
1312 msleep_spin(vcpu, &vcpu->mtx, "vmsusp", hz);
1313 vcpu_require_state_locked(vcpu, VCPU_FROZEN);
1314 } else {
1315 VCPU_CTR0(vm, vcpuid, "Rendezvous during suspend");
1316 vcpu_unlock(vcpu);
1317 vm_handle_rendezvous(vm, vcpuid);
1318 vcpu_lock(vcpu);
1319 }
1320 }
1321 vcpu_unlock(vcpu);
1322
1323 /*
1324 * Wakeup the other sleeping vcpus and return to userspace.
1325 */
1326 for (i = 0; i < VM_MAXCPU; i++) {
1327 if (CPU_ISSET(i, &vm->suspended_cpus)) {
1328 vcpu_notify_event(vm, i, false);
1329 }
1330 }
1331
1332 *retu = true;
1333 return (0);
1334}
1335
1336int
1337vm_suspend(struct vm *vm, enum vm_suspend_how how)
1338{
1339 int i;
1340
1341 if (how <= VM_SUSPEND_NONE || how >= VM_SUSPEND_LAST)
1342 return (EINVAL);
1343
1344 if (atomic_cmpset_int(&vm->suspend, 0, how) == 0) {
1345 VM_CTR2(vm, "virtual machine already suspended %d/%d",
1346 vm->suspend, how);
1347 return (EALREADY);
1348 }
1349
1350 VM_CTR1(vm, "virtual machine successfully suspended %d", how);
1351
1352 /*
1353 * Notify all active vcpus that they are now suspended.
1354 */
1355 for (i = 0; i < VM_MAXCPU; i++) {
1356 if (CPU_ISSET(i, &vm->active_cpus))
1357 vcpu_notify_event(vm, i, false);
1358 }
1359
1360 return (0);
1361}
1362
1363void
1364vm_exit_suspended(struct vm *vm, int vcpuid, uint64_t rip)
1365{
1366 struct vm_exit *vmexit;
1367
1368 KASSERT(vm->suspend > VM_SUSPEND_NONE && vm->suspend < VM_SUSPEND_LAST,
1369 ("vm_exit_suspended: invalid suspend type %d", vm->suspend));
1370
1371 vmexit = vm_exitinfo(vm, vcpuid);
1372 vmexit->rip = rip;
1373 vmexit->inst_length = 0;
1374 vmexit->exitcode = VM_EXITCODE_SUSPENDED;
1375 vmexit->u.suspended.how = vm->suspend;
1376}
1377
1378void
1379vm_exit_rendezvous(struct vm *vm, int vcpuid, uint64_t rip)
1380{
1381 struct vm_exit *vmexit;
1382
1383 KASSERT(vm->rendezvous_func != NULL, ("rendezvous not in progress"));
1384
1385 vmexit = vm_exitinfo(vm, vcpuid);
1386 vmexit->rip = rip;
1387 vmexit->inst_length = 0;
1388 vmexit->exitcode = VM_EXITCODE_RENDEZVOUS;
1389 vmm_stat_incr(vm, vcpuid, VMEXIT_RENDEZVOUS, 1);
1390}
1391
1392void
1393vm_exit_astpending(struct vm *vm, int vcpuid, uint64_t rip)
1394{
1395 struct vm_exit *vmexit;
1396
1397 vmexit = vm_exitinfo(vm, vcpuid);
1398 vmexit->rip = rip;
1399 vmexit->inst_length = 0;
1400 vmexit->exitcode = VM_EXITCODE_BOGUS;
1401 vmm_stat_incr(vm, vcpuid, VMEXIT_ASTPENDING, 1);
1402}
1403
1404int
1405vm_run(struct vm *vm, struct vm_run *vmrun)
1406{
1407 int error, vcpuid;
1408 struct vcpu *vcpu;
1409 struct pcb *pcb;
1410 uint64_t tscval, rip;
1411 struct vm_exit *vme;
1412 bool retu, intr_disabled;
1413 pmap_t pmap;
1414 void *rptr, *sptr;
1415
1416 vcpuid = vmrun->cpuid;
1417
1418 if (vcpuid < 0 || vcpuid >= VM_MAXCPU)
1419 return (EINVAL);
1420
1421 if (!CPU_ISSET(vcpuid, &vm->active_cpus))
1422 return (EINVAL);
1423
1424 if (CPU_ISSET(vcpuid, &vm->suspended_cpus))
1425 return (EINVAL);
1426
1427 rptr = &vm->rendezvous_func;
1428 sptr = &vm->suspend;
1429 pmap = vmspace_pmap(vm->vmspace);
1430 vcpu = &vm->vcpu[vcpuid];
1431 vme = &vcpu->exitinfo;
1432 rip = vmrun->rip;
1433restart:
1434 critical_enter();
1435
1436 KASSERT(!CPU_ISSET(curcpu, &pmap->pm_active),
1437 ("vm_run: absurd pm_active"));
1438
1439 tscval = rdtsc();
1440
1441 pcb = PCPU_GET(curpcb);
1442 set_pcb_flags(pcb, PCB_FULL_IRET);
1443
1444 restore_guest_fpustate(vcpu);
1445
1446 vcpu_require_state(vm, vcpuid, VCPU_RUNNING);
1447 error = VMRUN(vm->cookie, vcpuid, rip, pmap, rptr, sptr);
1448 vcpu_require_state(vm, vcpuid, VCPU_FROZEN);
1449
1450 save_guest_fpustate(vcpu);
1451
1452 vmm_stat_incr(vm, vcpuid, VCPU_TOTAL_RUNTIME, rdtsc() - tscval);
1453
1454 critical_exit();
1455
1456 if (error == 0) {
1457 retu = false;
1458 switch (vme->exitcode) {
1459 case VM_EXITCODE_SUSPENDED:
1460 error = vm_handle_suspend(vm, vcpuid, &retu);
1461 break;
1462 case VM_EXITCODE_IOAPIC_EOI:
1463 vioapic_process_eoi(vm, vcpuid,
1464 vme->u.ioapic_eoi.vector);
1465 break;
1466 case VM_EXITCODE_RENDEZVOUS:
1467 vm_handle_rendezvous(vm, vcpuid);
1468 error = 0;
1469 break;
1470 case VM_EXITCODE_HLT:
1471 intr_disabled = ((vme->u.hlt.rflags & PSL_I) == 0);
1472 error = vm_handle_hlt(vm, vcpuid, intr_disabled, &retu);
1473 break;
1474 case VM_EXITCODE_PAGING:
1475 error = vm_handle_paging(vm, vcpuid, &retu);
1476 break;
1477 case VM_EXITCODE_INST_EMUL:
1478 error = vm_handle_inst_emul(vm, vcpuid, &retu);
1479 break;
1480 case VM_EXITCODE_INOUT:
1481 case VM_EXITCODE_INOUT_STR:
1482 error = vm_handle_inout(vm, vcpuid, vme, &retu);
1483 break;
1484 case VM_EXITCODE_MONITOR:
1485 case VM_EXITCODE_MWAIT:
1486 vm_inject_ud(vm, vcpuid);
1487 break;
1488 default:
1489 retu = true; /* handled in userland */
1490 break;
1491 }
1492 }
1493
1494 if (error == 0 && retu == false) {
1495 rip = vme->rip + vme->inst_length;
1496 goto restart;
1497 }
1498
1499 /* copy the exit information */
1500 bcopy(vme, &vmrun->vm_exit, sizeof(struct vm_exit));
1501 return (error);
1502}
1503
1504int
1505vm_exit_intinfo(struct vm *vm, int vcpuid, uint64_t info)
1506{
1507 struct vcpu *vcpu;
1508 int type, vector;
1509
1510 if (vcpuid < 0 || vcpuid >= VM_MAXCPU)
1511 return (EINVAL);
1512
1513 vcpu = &vm->vcpu[vcpuid];
1514
1515 if (info & VM_INTINFO_VALID) {
1516 type = info & VM_INTINFO_TYPE;
1517 vector = info & 0xff;
1518 if (type == VM_INTINFO_NMI && vector != IDT_NMI)
1519 return (EINVAL);
1520 if (type == VM_INTINFO_HWEXCEPTION && vector >= 32)
1521 return (EINVAL);
1522 if (info & VM_INTINFO_RSVD)
1523 return (EINVAL);
1524 } else {
1525 info = 0;
1526 }
1527 VCPU_CTR2(vm, vcpuid, "%s: info1(%#lx)", __func__, info);
1528 vcpu->exitintinfo = info;
1529 return (0);
1530}
1531
1532enum exc_class {
1533 EXC_BENIGN,
1534 EXC_CONTRIBUTORY,
1535 EXC_PAGEFAULT
1536};
1537
1538#define IDT_VE 20 /* Virtualization Exception (Intel specific) */
1539
1540static enum exc_class
1541exception_class(uint64_t info)
1542{
1543 int type, vector;
1544
1545 KASSERT(info & VM_INTINFO_VALID, ("intinfo must be valid: %#lx", info));
1546 type = info & VM_INTINFO_TYPE;
1547 vector = info & 0xff;
1548
1549 /* Table 6-4, "Interrupt and Exception Classes", Intel SDM, Vol 3 */
1550 switch (type) {
1551 case VM_INTINFO_HWINTR:
1552 case VM_INTINFO_SWINTR:
1553 case VM_INTINFO_NMI:
1554 return (EXC_BENIGN);
1555 default:
1556 /*
1557 * Hardware exception.
1558 *
1559 * SVM and VT-x use identical type values to represent NMI,
1560 * hardware interrupt and software interrupt.
1561 *
1562 * SVM uses type '3' for all exceptions. VT-x uses type '3'
1563 * for exceptions except #BP and #OF. #BP and #OF use a type
1564 * value of '5' or '6'. Therefore we don't check for explicit
1565 * values of 'type' to classify 'intinfo' into a hardware
1566 * exception.
1567 */
1568 break;
1569 }
1570
1571 switch (vector) {
1572 case IDT_PF:
1573 case IDT_VE:
1574 return (EXC_PAGEFAULT);
1575 case IDT_DE:
1576 case IDT_TS:
1577 case IDT_NP:
1578 case IDT_SS:
1579 case IDT_GP:
1580 return (EXC_CONTRIBUTORY);
1581 default:
1582 return (EXC_BENIGN);
1583 }
1584}
1585
1586static int
1587nested_fault(struct vm *vm, int vcpuid, uint64_t info1, uint64_t info2,
1588 uint64_t *retinfo)
1589{
1590 enum exc_class exc1, exc2;
1591 int type1, vector1;
1592
1593 KASSERT(info1 & VM_INTINFO_VALID, ("info1 %#lx is not valid", info1));
1594 KASSERT(info2 & VM_INTINFO_VALID, ("info2 %#lx is not valid", info2));
1595
1596 /*
1597 * If an exception occurs while attempting to call the double-fault
1598 * handler the processor enters shutdown mode (aka triple fault).
1599 */
1600 type1 = info1 & VM_INTINFO_TYPE;
1601 vector1 = info1 & 0xff;
1602 if (type1 == VM_INTINFO_HWEXCEPTION && vector1 == IDT_DF) {
1603 VCPU_CTR2(vm, vcpuid, "triple fault: info1(%#lx), info2(%#lx)",
1604 info1, info2);
1605 vm_suspend(vm, VM_SUSPEND_TRIPLEFAULT);
1606 *retinfo = 0;
1607 return (0);
1608 }
1609
1610 /*
1611 * Table 6-5 "Conditions for Generating a Double Fault", Intel SDM, Vol3
1612 */
1613 exc1 = exception_class(info1);
1614 exc2 = exception_class(info2);
1615 if ((exc1 == EXC_CONTRIBUTORY && exc2 == EXC_CONTRIBUTORY) ||
1616 (exc1 == EXC_PAGEFAULT && exc2 != EXC_BENIGN)) {
1617 /* Convert nested fault into a double fault. */
1618 *retinfo = IDT_DF;
1619 *retinfo |= VM_INTINFO_VALID | VM_INTINFO_HWEXCEPTION;
1620 *retinfo |= VM_INTINFO_DEL_ERRCODE;
1621 } else {
1622 /* Handle exceptions serially */
1623 *retinfo = info2;
1624 }
1625 return (1);
1626}
1627
1628static uint64_t
1629vcpu_exception_intinfo(struct vcpu *vcpu)
1630{
1631 uint64_t info = 0;
1632
1633 if (vcpu->exception_pending) {
1634 info = vcpu->exception.vector & 0xff;
1635 info |= VM_INTINFO_VALID | VM_INTINFO_HWEXCEPTION;
1636 if (vcpu->exception.error_code_valid) {
1637 info |= VM_INTINFO_DEL_ERRCODE;
1638 info |= (uint64_t)vcpu->exception.error_code << 32;
1639 }
1640 }
1641 return (info);
1642}
1643
1644int
1645vm_entry_intinfo(struct vm *vm, int vcpuid, uint64_t *retinfo)
1646{
1647 struct vcpu *vcpu;
1648 uint64_t info1, info2;
1649 int valid;
1650
1651 KASSERT(vcpuid >= 0 && vcpuid < VM_MAXCPU, ("invalid vcpu %d", vcpuid));
1652
1653 vcpu = &vm->vcpu[vcpuid];
1654
1655 info1 = vcpu->exitintinfo;
1656 vcpu->exitintinfo = 0;
1657
1658 info2 = 0;
1659 if (vcpu->exception_pending) {
1660 info2 = vcpu_exception_intinfo(vcpu);
1661 vcpu->exception_pending = 0;
1662 VCPU_CTR2(vm, vcpuid, "Exception %d delivered: %#lx",
1663 vcpu->exception.vector, info2);
1664 }
1665
1666 if ((info1 & VM_INTINFO_VALID) && (info2 & VM_INTINFO_VALID)) {
1667 valid = nested_fault(vm, vcpuid, info1, info2, retinfo);
1668 } else if (info1 & VM_INTINFO_VALID) {
1669 *retinfo = info1;
1670 valid = 1;
1671 } else if (info2 & VM_INTINFO_VALID) {
1672 *retinfo = info2;
1673 valid = 1;
1674 } else {
1675 valid = 0;
1676 }
1677
1678 if (valid) {
1679 VCPU_CTR4(vm, vcpuid, "%s: info1(%#lx), info2(%#lx), "
1680 "retinfo(%#lx)", __func__, info1, info2, *retinfo);
1681 }
1682
1683 return (valid);
1684}
1685
1686int
1687vm_get_intinfo(struct vm *vm, int vcpuid, uint64_t *info1, uint64_t *info2)
1688{
1689 struct vcpu *vcpu;
1690
1691 if (vcpuid < 0 || vcpuid >= VM_MAXCPU)
1692 return (EINVAL);
1693
1694 vcpu = &vm->vcpu[vcpuid];
1695 *info1 = vcpu->exitintinfo;
1696 *info2 = vcpu_exception_intinfo(vcpu);
1697 return (0);
1698}
1699
1700int
1701vm_inject_exception(struct vm *vm, int vcpuid, struct vm_exception *exception)
1702{
1703 struct vcpu *vcpu;
1704
1705 if (vcpuid < 0 || vcpuid >= VM_MAXCPU)
1706 return (EINVAL);
1707
1708 if (exception->vector < 0 || exception->vector >= 32)
1709 return (EINVAL);
1710
1711 /*
1712 * A double fault exception should never be injected directly into
1713 * the guest. It is a derived exception that results from specific
1714 * combinations of nested faults.
1715 */
1716 if (exception->vector == IDT_DF)
1717 return (EINVAL);
1718
1719 vcpu = &vm->vcpu[vcpuid];
1720
1721 if (vcpu->exception_pending) {
1722 VCPU_CTR2(vm, vcpuid, "Unable to inject exception %d due to "
1723 "pending exception %d", exception->vector,
1724 vcpu->exception.vector);
1725 return (EBUSY);
1726 }
1727
1728 vcpu->exception_pending = 1;
1729 vcpu->exception = *exception;
1730 VCPU_CTR1(vm, vcpuid, "Exception %d pending", exception->vector);
1731 return (0);
1732}
1733
1734void
1735vm_inject_fault(void *vmarg, int vcpuid, int vector, int errcode_valid,
1736 int errcode)
1737{
1738 struct vm_exception exception;
1739 struct vm_exit *vmexit;
1740 struct vm *vm;
1741 int error;
1742
1743 vm = vmarg;
1744
1745 exception.vector = vector;
1746 exception.error_code = errcode;
1747 exception.error_code_valid = errcode_valid;
1748 error = vm_inject_exception(vm, vcpuid, &exception);
1749 KASSERT(error == 0, ("vm_inject_exception error %d", error));
1750
1751 /*
1752 * A fault-like exception allows the instruction to be restarted
1753 * after the exception handler returns.
1754 *
1755 * By setting the inst_length to 0 we ensure that the instruction
1756 * pointer remains at the faulting instruction.
1757 */
1758 vmexit = vm_exitinfo(vm, vcpuid);
1759 vmexit->inst_length = 0;
1760}
1761
1762void
1763vm_inject_pf(void *vmarg, int vcpuid, int error_code, uint64_t cr2)
1764{
1765 struct vm *vm;
1766 int error;
1767
1768 vm = vmarg;
1769 VCPU_CTR2(vm, vcpuid, "Injecting page fault: error_code %#x, cr2 %#lx",
1770 error_code, cr2);
1771
1772 error = vm_set_register(vm, vcpuid, VM_REG_GUEST_CR2, cr2);
1773 KASSERT(error == 0, ("vm_set_register(cr2) error %d", error));
1774
1775 vm_inject_fault(vm, vcpuid, IDT_PF, 1, error_code);
1776}
1777
1778static VMM_STAT(VCPU_NMI_COUNT, "number of NMIs delivered to vcpu");
1779
1780int
1781vm_inject_nmi(struct vm *vm, int vcpuid)
1782{
1783 struct vcpu *vcpu;
1784
1785 if (vcpuid < 0 || vcpuid >= VM_MAXCPU)
1786 return (EINVAL);
1787
1788 vcpu = &vm->vcpu[vcpuid];
1789
1790 vcpu->nmi_pending = 1;
1791 vcpu_notify_event(vm, vcpuid, false);
1792 return (0);
1793}
1794
1795int
1796vm_nmi_pending(struct vm *vm, int vcpuid)
1797{
1798 struct vcpu *vcpu;
1799
1800 if (vcpuid < 0 || vcpuid >= VM_MAXCPU)
1801 panic("vm_nmi_pending: invalid vcpuid %d", vcpuid);
1802
1803 vcpu = &vm->vcpu[vcpuid];
1804
1805 return (vcpu->nmi_pending);
1806}
1807
1808void
1809vm_nmi_clear(struct vm *vm, int vcpuid)
1810{
1811 struct vcpu *vcpu;
1812
1813 if (vcpuid < 0 || vcpuid >= VM_MAXCPU)
1814 panic("vm_nmi_pending: invalid vcpuid %d", vcpuid);
1815
1816 vcpu = &vm->vcpu[vcpuid];
1817
1818 if (vcpu->nmi_pending == 0)
1819 panic("vm_nmi_clear: inconsistent nmi_pending state");
1820
1821 vcpu->nmi_pending = 0;
1822 vmm_stat_incr(vm, vcpuid, VCPU_NMI_COUNT, 1);
1823}
1824
1825static VMM_STAT(VCPU_EXTINT_COUNT, "number of ExtINTs delivered to vcpu");
1826
1827int
1828vm_inject_extint(struct vm *vm, int vcpuid)
1829{
1830 struct vcpu *vcpu;
1831
1832 if (vcpuid < 0 || vcpuid >= VM_MAXCPU)
1833 return (EINVAL);
1834
1835 vcpu = &vm->vcpu[vcpuid];
1836
1837 vcpu->extint_pending = 1;
1838 vcpu_notify_event(vm, vcpuid, false);
1839 return (0);
1840}
1841
1842int
1843vm_extint_pending(struct vm *vm, int vcpuid)
1844{
1845 struct vcpu *vcpu;
1846
1847 if (vcpuid < 0 || vcpuid >= VM_MAXCPU)
1848 panic("vm_extint_pending: invalid vcpuid %d", vcpuid);
1849
1850 vcpu = &vm->vcpu[vcpuid];
1851
1852 return (vcpu->extint_pending);
1853}
1854
1855void
1856vm_extint_clear(struct vm *vm, int vcpuid)
1857{
1858 struct vcpu *vcpu;
1859
1860 if (vcpuid < 0 || vcpuid >= VM_MAXCPU)
1861 panic("vm_extint_pending: invalid vcpuid %d", vcpuid);
1862
1863 vcpu = &vm->vcpu[vcpuid];
1864
1865 if (vcpu->extint_pending == 0)
1866 panic("vm_extint_clear: inconsistent extint_pending state");
1867
1868 vcpu->extint_pending = 0;
1869 vmm_stat_incr(vm, vcpuid, VCPU_EXTINT_COUNT, 1);
1870}
1871
1872int
1873vm_get_capability(struct vm *vm, int vcpu, int type, int *retval)
1874{
1875 if (vcpu < 0 || vcpu >= VM_MAXCPU)
1876 return (EINVAL);
1877
1878 if (type < 0 || type >= VM_CAP_MAX)
1879 return (EINVAL);
1880
1881 return (VMGETCAP(vm->cookie, vcpu, type, retval));
1882}
1883
1884int
1885vm_set_capability(struct vm *vm, int vcpu, int type, int val)
1886{
1887 if (vcpu < 0 || vcpu >= VM_MAXCPU)
1888 return (EINVAL);
1889
1890 if (type < 0 || type >= VM_CAP_MAX)
1891 return (EINVAL);
1892
1893 return (VMSETCAP(vm->cookie, vcpu, type, val));
1894}
1895
1896struct vlapic *
1897vm_lapic(struct vm *vm, int cpu)
1898{
1899 return (vm->vcpu[cpu].vlapic);
1900}
1901
1902struct vioapic *
1903vm_ioapic(struct vm *vm)
1904{
1905
1906 return (vm->vioapic);
1907}
1908
1909struct vhpet *
1910vm_hpet(struct vm *vm)
1911{
1912
1913 return (vm->vhpet);
1914}
1915
1916boolean_t
1917vmm_is_pptdev(int bus, int slot, int func)
1918{
1919 int found, i, n;
1920 int b, s, f;
1921 char *val, *cp, *cp2;
1922
1923 /*
1924 * XXX
1925 * The length of an environment variable is limited to 128 bytes which
1926 * puts an upper limit on the number of passthru devices that may be
1927 * specified using a single environment variable.
1928 *
1929 * Work around this by scanning multiple environment variable
1930 * names instead of a single one - yuck!
1931 */
1932 const char *names[] = { "pptdevs", "pptdevs2", "pptdevs3", NULL };
1933
1934 /* set pptdevs="1/2/3 4/5/6 7/8/9 10/11/12" */
1935 found = 0;
1936 for (i = 0; names[i] != NULL && !found; i++) {
|
1938 while (cp != NULL && *cp != '\0') {
1939 if ((cp2 = strchr(cp, ' ')) != NULL)
1940 *cp2 = '\0';
1941
1942 n = sscanf(cp, "%d/%d/%d", &b, &s, &f);
1943 if (n == 3 && bus == b && slot == s && func == f) {
1944 found = 1;
1945 break;
1946 }
1947
1948 if (cp2 != NULL)
1949 *cp2++ = ' ';
1950
1951 cp = cp2;
1952 }
1953 freeenv(val);
1954 }
1955 return (found);
1956}
1957
1958void *
1959vm_iommu_domain(struct vm *vm)
1960{
1961
1962 return (vm->iommu);
1963}
1964
1965int
1966vcpu_set_state(struct vm *vm, int vcpuid, enum vcpu_state newstate,
1967 bool from_idle)
1968{
1969 int error;
1970 struct vcpu *vcpu;
1971
1972 if (vcpuid < 0 || vcpuid >= VM_MAXCPU)
1973 panic("vm_set_run_state: invalid vcpuid %d", vcpuid);
1974
1975 vcpu = &vm->vcpu[vcpuid];
1976
1977 vcpu_lock(vcpu);
1978 error = vcpu_set_state_locked(vcpu, newstate, from_idle);
1979 vcpu_unlock(vcpu);
1980
1981 return (error);
1982}
1983
1984enum vcpu_state
1985vcpu_get_state(struct vm *vm, int vcpuid, int *hostcpu)
1986{
1987 struct vcpu *vcpu;
1988 enum vcpu_state state;
1989
1990 if (vcpuid < 0 || vcpuid >= VM_MAXCPU)
1991 panic("vm_get_run_state: invalid vcpuid %d", vcpuid);
1992
1993 vcpu = &vm->vcpu[vcpuid];
1994
1995 vcpu_lock(vcpu);
1996 state = vcpu->state;
1997 if (hostcpu != NULL)
1998 *hostcpu = vcpu->hostcpu;
1999 vcpu_unlock(vcpu);
2000
2001 return (state);
2002}
2003
2004int
2005vm_activate_cpu(struct vm *vm, int vcpuid)
2006{
2007
2008 if (vcpuid < 0 || vcpuid >= VM_MAXCPU)
2009 return (EINVAL);
2010
2011 if (CPU_ISSET(vcpuid, &vm->active_cpus))
2012 return (EBUSY);
2013
2014 VCPU_CTR0(vm, vcpuid, "activated");
2015 CPU_SET_ATOMIC(vcpuid, &vm->active_cpus);
2016 return (0);
2017}
2018
2019cpuset_t
2020vm_active_cpus(struct vm *vm)
2021{
2022
2023 return (vm->active_cpus);
2024}
2025
2026cpuset_t
2027vm_suspended_cpus(struct vm *vm)
2028{
2029
2030 return (vm->suspended_cpus);
2031}
2032
2033void *
2034vcpu_stats(struct vm *vm, int vcpuid)
2035{
2036
2037 return (vm->vcpu[vcpuid].stats);
2038}
2039
2040int
2041vm_get_x2apic_state(struct vm *vm, int vcpuid, enum x2apic_state *state)
2042{
2043 if (vcpuid < 0 || vcpuid >= VM_MAXCPU)
2044 return (EINVAL);
2045
2046 *state = vm->vcpu[vcpuid].x2apic_state;
2047
2048 return (0);
2049}
2050
2051int
2052vm_set_x2apic_state(struct vm *vm, int vcpuid, enum x2apic_state state)
2053{
2054 if (vcpuid < 0 || vcpuid >= VM_MAXCPU)
2055 return (EINVAL);
2056
2057 if (state >= X2APIC_STATE_LAST)
2058 return (EINVAL);
2059
2060 vm->vcpu[vcpuid].x2apic_state = state;
2061
2062 vlapic_set_x2apic_state(vm, vcpuid, state);
2063
2064 return (0);
2065}
2066
2067/*
2068 * This function is called to ensure that a vcpu "sees" a pending event
2069 * as soon as possible:
2070 * - If the vcpu thread is sleeping then it is woken up.
2071 * - If the vcpu is running on a different host_cpu then an IPI will be directed
2072 * to the host_cpu to cause the vcpu to trap into the hypervisor.
2073 */
2074void
2075vcpu_notify_event(struct vm *vm, int vcpuid, bool lapic_intr)
2076{
2077 int hostcpu;
2078 struct vcpu *vcpu;
2079
2080 vcpu = &vm->vcpu[vcpuid];
2081
2082 vcpu_lock(vcpu);
2083 hostcpu = vcpu->hostcpu;
2084 if (vcpu->state == VCPU_RUNNING) {
2085 KASSERT(hostcpu != NOCPU, ("vcpu running on invalid hostcpu"));
2086 if (hostcpu != curcpu) {
2087 if (lapic_intr) {
2088 vlapic_post_intr(vcpu->vlapic, hostcpu,
2089 vmm_ipinum);
2090 } else {
2091 ipi_cpu(hostcpu, vmm_ipinum);
2092 }
2093 } else {
2094 /*
2095 * If the 'vcpu' is running on 'curcpu' then it must
2096 * be sending a notification to itself (e.g. SELF_IPI).
2097 * The pending event will be picked up when the vcpu
2098 * transitions back to guest context.
2099 */
2100 }
2101 } else {
2102 KASSERT(hostcpu == NOCPU, ("vcpu state %d not consistent "
2103 "with hostcpu %d", vcpu->state, hostcpu));
2104 if (vcpu->state == VCPU_SLEEPING)
2105 wakeup_one(vcpu);
2106 }
2107 vcpu_unlock(vcpu);
2108}
2109
2110struct vmspace *
2111vm_get_vmspace(struct vm *vm)
2112{
2113
2114 return (vm->vmspace);
2115}
2116
2117int
2118vm_apicid2vcpuid(struct vm *vm, int apicid)
2119{
2120 /*
2121 * XXX apic id is assumed to be numerically identical to vcpu id
2122 */
2123 return (apicid);
2124}
2125
2126void
2127vm_smp_rendezvous(struct vm *vm, int vcpuid, cpuset_t dest,
2128 vm_rendezvous_func_t func, void *arg)
2129{
2130 int i;
2131
2132 /*
2133 * Enforce that this function is called without any locks
2134 */
2135 WITNESS_WARN(WARN_PANIC, NULL, "vm_smp_rendezvous");
2136 KASSERT(vcpuid == -1 || (vcpuid >= 0 && vcpuid < VM_MAXCPU),
2137 ("vm_smp_rendezvous: invalid vcpuid %d", vcpuid));
2138
2139restart:
2140 mtx_lock(&vm->rendezvous_mtx);
2141 if (vm->rendezvous_func != NULL) {
2142 /*
2143 * If a rendezvous is already in progress then we need to
2144 * call the rendezvous handler in case this 'vcpuid' is one
2145 * of the targets of the rendezvous.
2146 */
2147 RENDEZVOUS_CTR0(vm, vcpuid, "Rendezvous already in progress");
2148 mtx_unlock(&vm->rendezvous_mtx);
2149 vm_handle_rendezvous(vm, vcpuid);
2150 goto restart;
2151 }
2152 KASSERT(vm->rendezvous_func == NULL, ("vm_smp_rendezvous: previous "
2153 "rendezvous is still in progress"));
2154
2155 RENDEZVOUS_CTR0(vm, vcpuid, "Initiating rendezvous");
2156 vm->rendezvous_req_cpus = dest;
2157 CPU_ZERO(&vm->rendezvous_done_cpus);
2158 vm->rendezvous_arg = arg;
2159 vm_set_rendezvous_func(vm, func);
2160 mtx_unlock(&vm->rendezvous_mtx);
2161
2162 /*
2163 * Wake up any sleeping vcpus and trigger a VM-exit in any running
2164 * vcpus so they handle the rendezvous as soon as possible.
2165 */
2166 for (i = 0; i < VM_MAXCPU; i++) {
2167 if (CPU_ISSET(i, &dest))
2168 vcpu_notify_event(vm, i, false);
2169 }
2170
2171 vm_handle_rendezvous(vm, vcpuid);
2172}
2173
2174struct vatpic *
2175vm_atpic(struct vm *vm)
2176{
2177 return (vm->vatpic);
2178}
2179
2180struct vatpit *
2181vm_atpit(struct vm *vm)
2182{
2183 return (vm->vatpit);
2184}
2185
2186enum vm_reg_name
2187vm_segment_name(int seg)
2188{
2189 static enum vm_reg_name seg_names[] = {
2190 VM_REG_GUEST_ES,
2191 VM_REG_GUEST_CS,
2192 VM_REG_GUEST_SS,
2193 VM_REG_GUEST_DS,
2194 VM_REG_GUEST_FS,
2195 VM_REG_GUEST_GS
2196 };
2197
2198 KASSERT(seg >= 0 && seg < nitems(seg_names),
2199 ("%s: invalid segment encoding %d", __func__, seg));
2200 return (seg_names[seg]);
2201}
2202
2203void
2204vm_copy_teardown(struct vm *vm, int vcpuid, struct vm_copyinfo *copyinfo,
2205 int num_copyinfo)
2206{
2207 int idx;
2208
2209 for (idx = 0; idx < num_copyinfo; idx++) {
2210 if (copyinfo[idx].cookie != NULL)
2211 vm_gpa_release(copyinfo[idx].cookie);
2212 }
2213 bzero(copyinfo, num_copyinfo * sizeof(struct vm_copyinfo));
2214}
2215
2216int
2217vm_copy_setup(struct vm *vm, int vcpuid, struct vm_guest_paging *paging,
2218 uint64_t gla, size_t len, int prot, struct vm_copyinfo *copyinfo,
2219 int num_copyinfo)
2220{
2221 int error, idx, nused;
2222 size_t n, off, remaining;
2223 void *hva, *cookie;
2224 uint64_t gpa;
2225
2226 bzero(copyinfo, sizeof(struct vm_copyinfo) * num_copyinfo);
2227
2228 nused = 0;
2229 remaining = len;
2230 while (remaining > 0) {
2231 KASSERT(nused < num_copyinfo, ("insufficient vm_copyinfo"));
2232 error = vmm_gla2gpa(vm, vcpuid, paging, gla, prot, &gpa);
2233 if (error)
2234 return (error);
2235 off = gpa & PAGE_MASK;
2236 n = min(remaining, PAGE_SIZE - off);
2237 copyinfo[nused].gpa = gpa;
2238 copyinfo[nused].len = n;
2239 remaining -= n;
2240 gla += n;
2241 nused++;
2242 }
2243
2244 for (idx = 0; idx < nused; idx++) {
2245 hva = vm_gpa_hold(vm, copyinfo[idx].gpa, copyinfo[idx].len,
2246 prot, &cookie);
2247 if (hva == NULL)
2248 break;
2249 copyinfo[idx].hva = hva;
2250 copyinfo[idx].cookie = cookie;
2251 }
2252
2253 if (idx != nused) {
2254 vm_copy_teardown(vm, vcpuid, copyinfo, num_copyinfo);
2255 return (-1);
2256 } else {
2257 return (0);
2258 }
2259}
2260
2261void
2262vm_copyin(struct vm *vm, int vcpuid, struct vm_copyinfo *copyinfo, void *kaddr,
2263 size_t len)
2264{
2265 char *dst;
2266 int idx;
2267
2268 dst = kaddr;
2269 idx = 0;
2270 while (len > 0) {
2271 bcopy(copyinfo[idx].hva, dst, copyinfo[idx].len);
2272 len -= copyinfo[idx].len;
2273 dst += copyinfo[idx].len;
2274 idx++;
2275 }
2276}
2277
2278void
2279vm_copyout(struct vm *vm, int vcpuid, const void *kaddr,
2280 struct vm_copyinfo *copyinfo, size_t len)
2281{
2282 const char *src;
2283 int idx;
2284
2285 src = kaddr;
2286 idx = 0;
2287 while (len > 0) {
2288 bcopy(src, copyinfo[idx].hva, copyinfo[idx].len);
2289 len -= copyinfo[idx].len;
2290 src += copyinfo[idx].len;
2291 idx++;
2292 }
2293}
2294
2295/*
2296 * Return the amount of in-use and wired memory for the VM. Since
2297 * these are global stats, only return the values with for vCPU 0
2298 */
2299VMM_STAT_DECLARE(VMM_MEM_RESIDENT);
2300VMM_STAT_DECLARE(VMM_MEM_WIRED);
2301
2302static void
2303vm_get_rescnt(struct vm *vm, int vcpu, struct vmm_stat_type *stat)
2304{
2305
2306 if (vcpu == 0) {
2307 vmm_stat_set(vm, vcpu, VMM_MEM_RESIDENT,
2308 PAGE_SIZE * vmspace_resident_count(vm->vmspace));
2309 }
2310}
2311
2312static void
2313vm_get_wiredcnt(struct vm *vm, int vcpu, struct vmm_stat_type *stat)
2314{
2315
2316 if (vcpu == 0) {
2317 vmm_stat_set(vm, vcpu, VMM_MEM_WIRED,
2318 PAGE_SIZE * pmap_wired_count(vmspace_pmap(vm->vmspace)));
2319 }
2320}
2321
2322VMM_STAT_FUNC(VMM_MEM_RESIDENT, "Resident memory", vm_get_rescnt);
2323VMM_STAT_FUNC(VMM_MEM_WIRED, "Wired memory", vm_get_wiredcnt);
| 1938 while (cp != NULL && *cp != '\0') {
1939 if ((cp2 = strchr(cp, ' ')) != NULL)
1940 *cp2 = '\0';
1941
1942 n = sscanf(cp, "%d/%d/%d", &b, &s, &f);
1943 if (n == 3 && bus == b && slot == s && func == f) {
1944 found = 1;
1945 break;
1946 }
1947
1948 if (cp2 != NULL)
1949 *cp2++ = ' ';
1950
1951 cp = cp2;
1952 }
1953 freeenv(val);
1954 }
1955 return (found);
1956}
1957
1958void *
1959vm_iommu_domain(struct vm *vm)
1960{
1961
1962 return (vm->iommu);
1963}
1964
1965int
1966vcpu_set_state(struct vm *vm, int vcpuid, enum vcpu_state newstate,
1967 bool from_idle)
1968{
1969 int error;
1970 struct vcpu *vcpu;
1971
1972 if (vcpuid < 0 || vcpuid >= VM_MAXCPU)
1973 panic("vm_set_run_state: invalid vcpuid %d", vcpuid);
1974
1975 vcpu = &vm->vcpu[vcpuid];
1976
1977 vcpu_lock(vcpu);
1978 error = vcpu_set_state_locked(vcpu, newstate, from_idle);
1979 vcpu_unlock(vcpu);
1980
1981 return (error);
1982}
1983
1984enum vcpu_state
1985vcpu_get_state(struct vm *vm, int vcpuid, int *hostcpu)
1986{
1987 struct vcpu *vcpu;
1988 enum vcpu_state state;
1989
1990 if (vcpuid < 0 || vcpuid >= VM_MAXCPU)
1991 panic("vm_get_run_state: invalid vcpuid %d", vcpuid);
1992
1993 vcpu = &vm->vcpu[vcpuid];
1994
1995 vcpu_lock(vcpu);
1996 state = vcpu->state;
1997 if (hostcpu != NULL)
1998 *hostcpu = vcpu->hostcpu;
1999 vcpu_unlock(vcpu);
2000
2001 return (state);
2002}
2003
2004int
2005vm_activate_cpu(struct vm *vm, int vcpuid)
2006{
2007
2008 if (vcpuid < 0 || vcpuid >= VM_MAXCPU)
2009 return (EINVAL);
2010
2011 if (CPU_ISSET(vcpuid, &vm->active_cpus))
2012 return (EBUSY);
2013
2014 VCPU_CTR0(vm, vcpuid, "activated");
2015 CPU_SET_ATOMIC(vcpuid, &vm->active_cpus);
2016 return (0);
2017}
2018
2019cpuset_t
2020vm_active_cpus(struct vm *vm)
2021{
2022
2023 return (vm->active_cpus);
2024}
2025
2026cpuset_t
2027vm_suspended_cpus(struct vm *vm)
2028{
2029
2030 return (vm->suspended_cpus);
2031}
2032
2033void *
2034vcpu_stats(struct vm *vm, int vcpuid)
2035{
2036
2037 return (vm->vcpu[vcpuid].stats);
2038}
2039
2040int
2041vm_get_x2apic_state(struct vm *vm, int vcpuid, enum x2apic_state *state)
2042{
2043 if (vcpuid < 0 || vcpuid >= VM_MAXCPU)
2044 return (EINVAL);
2045
2046 *state = vm->vcpu[vcpuid].x2apic_state;
2047
2048 return (0);
2049}
2050
2051int
2052vm_set_x2apic_state(struct vm *vm, int vcpuid, enum x2apic_state state)
2053{
2054 if (vcpuid < 0 || vcpuid >= VM_MAXCPU)
2055 return (EINVAL);
2056
2057 if (state >= X2APIC_STATE_LAST)
2058 return (EINVAL);
2059
2060 vm->vcpu[vcpuid].x2apic_state = state;
2061
2062 vlapic_set_x2apic_state(vm, vcpuid, state);
2063
2064 return (0);
2065}
2066
2067/*
2068 * This function is called to ensure that a vcpu "sees" a pending event
2069 * as soon as possible:
2070 * - If the vcpu thread is sleeping then it is woken up.
2071 * - If the vcpu is running on a different host_cpu then an IPI will be directed
2072 * to the host_cpu to cause the vcpu to trap into the hypervisor.
2073 */
2074void
2075vcpu_notify_event(struct vm *vm, int vcpuid, bool lapic_intr)
2076{
2077 int hostcpu;
2078 struct vcpu *vcpu;
2079
2080 vcpu = &vm->vcpu[vcpuid];
2081
2082 vcpu_lock(vcpu);
2083 hostcpu = vcpu->hostcpu;
2084 if (vcpu->state == VCPU_RUNNING) {
2085 KASSERT(hostcpu != NOCPU, ("vcpu running on invalid hostcpu"));
2086 if (hostcpu != curcpu) {
2087 if (lapic_intr) {
2088 vlapic_post_intr(vcpu->vlapic, hostcpu,
2089 vmm_ipinum);
2090 } else {
2091 ipi_cpu(hostcpu, vmm_ipinum);
2092 }
2093 } else {
2094 /*
2095 * If the 'vcpu' is running on 'curcpu' then it must
2096 * be sending a notification to itself (e.g. SELF_IPI).
2097 * The pending event will be picked up when the vcpu
2098 * transitions back to guest context.
2099 */
2100 }
2101 } else {
2102 KASSERT(hostcpu == NOCPU, ("vcpu state %d not consistent "
2103 "with hostcpu %d", vcpu->state, hostcpu));
2104 if (vcpu->state == VCPU_SLEEPING)
2105 wakeup_one(vcpu);
2106 }
2107 vcpu_unlock(vcpu);
2108}
2109
2110struct vmspace *
2111vm_get_vmspace(struct vm *vm)
2112{
2113
2114 return (vm->vmspace);
2115}
2116
2117int
2118vm_apicid2vcpuid(struct vm *vm, int apicid)
2119{
2120 /*
2121 * XXX apic id is assumed to be numerically identical to vcpu id
2122 */
2123 return (apicid);
2124}
2125
2126void
2127vm_smp_rendezvous(struct vm *vm, int vcpuid, cpuset_t dest,
2128 vm_rendezvous_func_t func, void *arg)
2129{
2130 int i;
2131
2132 /*
2133 * Enforce that this function is called without any locks
2134 */
2135 WITNESS_WARN(WARN_PANIC, NULL, "vm_smp_rendezvous");
2136 KASSERT(vcpuid == -1 || (vcpuid >= 0 && vcpuid < VM_MAXCPU),
2137 ("vm_smp_rendezvous: invalid vcpuid %d", vcpuid));
2138
2139restart:
2140 mtx_lock(&vm->rendezvous_mtx);
2141 if (vm->rendezvous_func != NULL) {
2142 /*
2143 * If a rendezvous is already in progress then we need to
2144 * call the rendezvous handler in case this 'vcpuid' is one
2145 * of the targets of the rendezvous.
2146 */
2147 RENDEZVOUS_CTR0(vm, vcpuid, "Rendezvous already in progress");
2148 mtx_unlock(&vm->rendezvous_mtx);
2149 vm_handle_rendezvous(vm, vcpuid);
2150 goto restart;
2151 }
2152 KASSERT(vm->rendezvous_func == NULL, ("vm_smp_rendezvous: previous "
2153 "rendezvous is still in progress"));
2154
2155 RENDEZVOUS_CTR0(vm, vcpuid, "Initiating rendezvous");
2156 vm->rendezvous_req_cpus = dest;
2157 CPU_ZERO(&vm->rendezvous_done_cpus);
2158 vm->rendezvous_arg = arg;
2159 vm_set_rendezvous_func(vm, func);
2160 mtx_unlock(&vm->rendezvous_mtx);
2161
2162 /*
2163 * Wake up any sleeping vcpus and trigger a VM-exit in any running
2164 * vcpus so they handle the rendezvous as soon as possible.
2165 */
2166 for (i = 0; i < VM_MAXCPU; i++) {
2167 if (CPU_ISSET(i, &dest))
2168 vcpu_notify_event(vm, i, false);
2169 }
2170
2171 vm_handle_rendezvous(vm, vcpuid);
2172}
2173
2174struct vatpic *
2175vm_atpic(struct vm *vm)
2176{
2177 return (vm->vatpic);
2178}
2179
2180struct vatpit *
2181vm_atpit(struct vm *vm)
2182{
2183 return (vm->vatpit);
2184}
2185
2186enum vm_reg_name
2187vm_segment_name(int seg)
2188{
2189 static enum vm_reg_name seg_names[] = {
2190 VM_REG_GUEST_ES,
2191 VM_REG_GUEST_CS,
2192 VM_REG_GUEST_SS,
2193 VM_REG_GUEST_DS,
2194 VM_REG_GUEST_FS,
2195 VM_REG_GUEST_GS
2196 };
2197
2198 KASSERT(seg >= 0 && seg < nitems(seg_names),
2199 ("%s: invalid segment encoding %d", __func__, seg));
2200 return (seg_names[seg]);
2201}
2202
2203void
2204vm_copy_teardown(struct vm *vm, int vcpuid, struct vm_copyinfo *copyinfo,
2205 int num_copyinfo)
2206{
2207 int idx;
2208
2209 for (idx = 0; idx < num_copyinfo; idx++) {
2210 if (copyinfo[idx].cookie != NULL)
2211 vm_gpa_release(copyinfo[idx].cookie);
2212 }
2213 bzero(copyinfo, num_copyinfo * sizeof(struct vm_copyinfo));
2214}
2215
2216int
2217vm_copy_setup(struct vm *vm, int vcpuid, struct vm_guest_paging *paging,
2218 uint64_t gla, size_t len, int prot, struct vm_copyinfo *copyinfo,
2219 int num_copyinfo)
2220{
2221 int error, idx, nused;
2222 size_t n, off, remaining;
2223 void *hva, *cookie;
2224 uint64_t gpa;
2225
2226 bzero(copyinfo, sizeof(struct vm_copyinfo) * num_copyinfo);
2227
2228 nused = 0;
2229 remaining = len;
2230 while (remaining > 0) {
2231 KASSERT(nused < num_copyinfo, ("insufficient vm_copyinfo"));
2232 error = vmm_gla2gpa(vm, vcpuid, paging, gla, prot, &gpa);
2233 if (error)
2234 return (error);
2235 off = gpa & PAGE_MASK;
2236 n = min(remaining, PAGE_SIZE - off);
2237 copyinfo[nused].gpa = gpa;
2238 copyinfo[nused].len = n;
2239 remaining -= n;
2240 gla += n;
2241 nused++;
2242 }
2243
2244 for (idx = 0; idx < nused; idx++) {
2245 hva = vm_gpa_hold(vm, copyinfo[idx].gpa, copyinfo[idx].len,
2246 prot, &cookie);
2247 if (hva == NULL)
2248 break;
2249 copyinfo[idx].hva = hva;
2250 copyinfo[idx].cookie = cookie;
2251 }
2252
2253 if (idx != nused) {
2254 vm_copy_teardown(vm, vcpuid, copyinfo, num_copyinfo);
2255 return (-1);
2256 } else {
2257 return (0);
2258 }
2259}
2260
2261void
2262vm_copyin(struct vm *vm, int vcpuid, struct vm_copyinfo *copyinfo, void *kaddr,
2263 size_t len)
2264{
2265 char *dst;
2266 int idx;
2267
2268 dst = kaddr;
2269 idx = 0;
2270 while (len > 0) {
2271 bcopy(copyinfo[idx].hva, dst, copyinfo[idx].len);
2272 len -= copyinfo[idx].len;
2273 dst += copyinfo[idx].len;
2274 idx++;
2275 }
2276}
2277
2278void
2279vm_copyout(struct vm *vm, int vcpuid, const void *kaddr,
2280 struct vm_copyinfo *copyinfo, size_t len)
2281{
2282 const char *src;
2283 int idx;
2284
2285 src = kaddr;
2286 idx = 0;
2287 while (len > 0) {
2288 bcopy(src, copyinfo[idx].hva, copyinfo[idx].len);
2289 len -= copyinfo[idx].len;
2290 src += copyinfo[idx].len;
2291 idx++;
2292 }
2293}
2294
2295/*
2296 * Return the amount of in-use and wired memory for the VM. Since
2297 * these are global stats, only return the values with for vCPU 0
2298 */
2299VMM_STAT_DECLARE(VMM_MEM_RESIDENT);
2300VMM_STAT_DECLARE(VMM_MEM_WIRED);
2301
2302static void
2303vm_get_rescnt(struct vm *vm, int vcpu, struct vmm_stat_type *stat)
2304{
2305
2306 if (vcpu == 0) {
2307 vmm_stat_set(vm, vcpu, VMM_MEM_RESIDENT,
2308 PAGE_SIZE * vmspace_resident_count(vm->vmspace));
2309 }
2310}
2311
2312static void
2313vm_get_wiredcnt(struct vm *vm, int vcpu, struct vmm_stat_type *stat)
2314{
2315
2316 if (vcpu == 0) {
2317 vmm_stat_set(vm, vcpu, VMM_MEM_WIRED,
2318 PAGE_SIZE * pmap_wired_count(vmspace_pmap(vm->vmspace)));
2319 }
2320}
2321
2322VMM_STAT_FUNC(VMM_MEM_RESIDENT, "Resident memory", vm_get_rescnt);
2323VMM_STAT_FUNC(VMM_MEM_WIRED, "Wired memory", vm_get_wiredcnt);
|