Deleted Added
sdiff udiff text old ( 271152 ) new ( 271203 )
full compact
svm.c (271152) svm.c (271203)
1/*-
2 * Copyright (c) 2013, Anish Gupta (akgupt3@gmail.com)
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright

--- 11 unchanged lines hidden (view full) ---

20 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
21 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
22 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
23 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
24 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
25 */
26
27#include <sys/cdefs.h>
1/*-
2 * Copyright (c) 2013, Anish Gupta (akgupt3@gmail.com)
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright

--- 11 unchanged lines hidden (view full) ---

20 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
21 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
22 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
23 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
24 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
25 */
26
27#include <sys/cdefs.h>
28__FBSDID("$FreeBSD: projects/bhyve_svm/sys/amd64/vmm/amd/svm.c 271152 2014-09-05 03:33:16Z neel $");
28__FBSDID("$FreeBSD: projects/bhyve_svm/sys/amd64/vmm/amd/svm.c 271203 2014-09-06 19:02:52Z neel $");
29
30#include <sys/param.h>
31#include <sys/systm.h>
32#include <sys/smp.h>
33#include <sys/kernel.h>
34#include <sys/malloc.h>
35#include <sys/pcpu.h>
36#include <sys/proc.h>

--- 33 unchanged lines hidden (view full) ---

70 * SVM CPUID function 0x8000_000A, edx bit decoding.
71 */
72#define AMD_CPUID_SVM_NP BIT(0) /* Nested paging or RVI */
73#define AMD_CPUID_SVM_LBR BIT(1) /* Last branch virtualization */
74#define AMD_CPUID_SVM_SVML BIT(2) /* SVM lock */
75#define AMD_CPUID_SVM_NRIP_SAVE BIT(3) /* Next RIP is saved */
76#define AMD_CPUID_SVM_TSC_RATE BIT(4) /* TSC rate control. */
77#define AMD_CPUID_SVM_VMCB_CLEAN BIT(5) /* VMCB state caching */
29
30#include <sys/param.h>
31#include <sys/systm.h>
32#include <sys/smp.h>
33#include <sys/kernel.h>
34#include <sys/malloc.h>
35#include <sys/pcpu.h>
36#include <sys/proc.h>

--- 33 unchanged lines hidden (view full) ---

70 * SVM CPUID function 0x8000_000A, edx bit decoding.
71 */
72#define AMD_CPUID_SVM_NP BIT(0) /* Nested paging or RVI */
73#define AMD_CPUID_SVM_LBR BIT(1) /* Last branch virtualization */
74#define AMD_CPUID_SVM_SVML BIT(2) /* SVM lock */
75#define AMD_CPUID_SVM_NRIP_SAVE BIT(3) /* Next RIP is saved */
76#define AMD_CPUID_SVM_TSC_RATE BIT(4) /* TSC rate control. */
77#define AMD_CPUID_SVM_VMCB_CLEAN BIT(5) /* VMCB state caching */
78#define AMD_CPUID_SVM_ASID_FLUSH BIT(6) /* Flush by ASID */
78#define AMD_CPUID_SVM_FLUSH_BY_ASID BIT(6) /* Flush by ASID */
79#define AMD_CPUID_SVM_DECODE_ASSIST BIT(7) /* Decode assist */
80#define AMD_CPUID_SVM_PAUSE_INC BIT(10) /* Pause intercept filter. */
81#define AMD_CPUID_SVM_PAUSE_FTH BIT(12) /* Pause filter threshold */
82
79#define AMD_CPUID_SVM_DECODE_ASSIST BIT(7) /* Decode assist */
80#define AMD_CPUID_SVM_PAUSE_INC BIT(10) /* Pause intercept filter. */
81#define AMD_CPUID_SVM_PAUSE_FTH BIT(12) /* Pause filter threshold */
82
83#define VMCB_CACHE_DEFAULT \
84 (VMCB_CACHE_ASID | VMCB_CACHE_IOPM | VMCB_CACHE_NP)
85
83MALLOC_DEFINE(M_SVM, "svm", "svm");
84MALLOC_DEFINE(M_SVM_VLAPIC, "svm-vlapic", "svm-vlapic");
85
86/* Per-CPU context area. */
87extern struct pcpu __pcpu[];
88
89static bool svm_vmexit(struct svm_softc *svm_sc, int vcpu,
90 struct vm_exit *vmexit);
91static int svm_msr_rw_ok(uint8_t *btmap, uint64_t msr);
92static int svm_msr_rd_ok(uint8_t *btmap, uint64_t msr);
93static int svm_msr_index(uint64_t msr, int *index, int *bit);
94static int svm_getdesc(void *arg, int vcpu, int type, struct seg_desc *desc);
95
86MALLOC_DEFINE(M_SVM, "svm", "svm");
87MALLOC_DEFINE(M_SVM_VLAPIC, "svm-vlapic", "svm-vlapic");
88
89/* Per-CPU context area. */
90extern struct pcpu __pcpu[];
91
92static bool svm_vmexit(struct svm_softc *svm_sc, int vcpu,
93 struct vm_exit *vmexit);
94static int svm_msr_rw_ok(uint8_t *btmap, uint64_t msr);
95static int svm_msr_rd_ok(uint8_t *btmap, uint64_t msr);
96static int svm_msr_index(uint64_t msr, int *index, int *bit);
97static int svm_getdesc(void *arg, int vcpu, int type, struct seg_desc *desc);
98
96static uint32_t svm_feature; /* AMD SVM features. */
99static uint32_t svm_feature; /* AMD SVM features. */
97
100
98/*
99 * Starting guest ASID, 0 is reserved for host.
100 * Each guest will have its own unique ASID.
101 */
102static uint32_t guest_asid = 1;
101/* Maximum ASIDs supported by the processor */
102static uint32_t nasid;
103
103
104/*
105 * Max ASID processor can support.
106 * This limit the maximum number of virtual machines that can be created.
107 */
108static int max_asid;
104/* Current ASID generation for each host cpu */
105static struct asid asid[MAXCPU];
109
110/*
111 * SVM host state saved area of size 4KB for each core.
112 */
113static uint8_t hsave[MAXCPU][PAGE_SIZE] __aligned(PAGE_SIZE);
114
115/*
116 * S/w saved host context.

--- 52 unchanged lines hidden (view full) ---

169{
170 u_int regs[4];
171
172 /* CPUID Fn8000_000A is for SVM */
173 do_cpuid(0x8000000A, regs);
174 svm_feature = regs[3];
175
176 printf("SVM rev: 0x%x NASID:0x%x\n", regs[0] & 0xFF, regs[1]);
106
107/*
108 * SVM host state saved area of size 4KB for each core.
109 */
110static uint8_t hsave[MAXCPU][PAGE_SIZE] __aligned(PAGE_SIZE);
111
112/*
113 * S/w saved host context.

--- 52 unchanged lines hidden (view full) ---

166{
167 u_int regs[4];
168
169 /* CPUID Fn8000_000A is for SVM */
170 do_cpuid(0x8000000A, regs);
171 svm_feature = regs[3];
172
173 printf("SVM rev: 0x%x NASID:0x%x\n", regs[0] & 0xFF, regs[1]);
177 max_asid = regs[1];
178
174 nasid = regs[1];
175 KASSERT(nasid > 1, ("Insufficient ASIDs for guests: %#x", nasid));
176
179 printf("SVM Features:0x%b\n", svm_feature,
180 "\020"
181 "\001NP" /* Nested paging */
182 "\002LbrVirt" /* LBR virtualization */
183 "\003SVML" /* SVM lock */
184 "\004NRIPS" /* NRIP save */
185 "\005TscRateMsr" /* MSR based TSC rate control */
186 "\006VmcbClean" /* VMCB clean bits */

--- 22 unchanged lines hidden (view full) ---

209 }
210
211 if (svm_feature & AMD_CPUID_SVM_NRIP_SAVE)
212 return (0);
213
214 return (EIO);
215}
216
177 printf("SVM Features:0x%b\n", svm_feature,
178 "\020"
179 "\001NP" /* Nested paging */
180 "\002LbrVirt" /* LBR virtualization */
181 "\003SVML" /* SVM lock */
182 "\004NRIPS" /* NRIP save */
183 "\005TscRateMsr" /* MSR based TSC rate control */
184 "\006VmcbClean" /* VMCB clean bits */

--- 22 unchanged lines hidden (view full) ---

207 }
208
209 if (svm_feature & AMD_CPUID_SVM_NRIP_SAVE)
210 return (0);
211
212 return (EIO);
213}
214
215static __inline int
216flush_by_asid(void)
217{
218 return (svm_feature & AMD_CPUID_SVM_FLUSH_BY_ASID);
219}
220
217/*
218 * Enable SVM for a CPU.
219 */
220static void
221svm_enable(void *arg __unused)
222{
223 uint64_t hsave_pa;
224

--- 32 unchanged lines hidden (view full) ---

257}
258
259/*
260 * Enable SVM on CPU and initialize nested page table h/w.
261 */
262static int
263svm_init(int ipinum)
264{
221/*
222 * Enable SVM for a CPU.
223 */
224static void
225svm_enable(void *arg __unused)
226{
227 uint64_t hsave_pa;
228

--- 32 unchanged lines hidden (view full) ---

261}
262
263/*
264 * Enable SVM on CPU and initialize nested page table h/w.
265 */
266static int
267svm_init(int ipinum)
268{
265 int err;
269 int err, cpu;
266
267 err = is_svm_enabled();
268 if (err)
269 return (err);
270
270
271 err = is_svm_enabled();
272 if (err)
273 return (err);
274
275 for (cpu = 0; cpu < MAXCPU; cpu++) {
276 /*
277 * Initialize the host ASIDs to their "highest" valid values.
278 *
279 * The next ASID allocation will rollover both 'gen' and 'num'
280 * and start off the sequence at {1,1}.
281 */
282 asid[cpu].gen = ~0UL;
283 asid[cpu].num = nasid - 1;
284 }
271
272 svm_npt_init(ipinum);
273
274 /* Start SVM on all CPUs */
275 smp_rendezvous(NULL, svm_enable, NULL, NULL);
285
286 svm_npt_init(ipinum);
287
288 /* Start SVM on all CPUs */
289 smp_rendezvous(NULL, svm_enable, NULL, NULL);
276
290
277 return (0);
278}
279
280static void
281svm_restore(void)
282{
283 svm_enable(NULL);
284}

--- 84 unchanged lines hidden (view full) ---

369}
370
371static int
372svm_msr_rd_ok(uint8_t *perm_bitmap, uint64_t msr)
373{
374 return svm_msr_perm(perm_bitmap, msr, true, false);
375}
376
291 return (0);
292}
293
294static void
295svm_restore(void)
296{
297 svm_enable(NULL);
298}

--- 84 unchanged lines hidden (view full) ---

383}
384
385static int
386svm_msr_rd_ok(uint8_t *perm_bitmap, uint64_t msr)
387{
388 return svm_msr_perm(perm_bitmap, msr, true, false);
389}
390
391static __inline void
392vcpu_set_dirty(struct svm_vcpu *vcpustate, uint32_t dirtybits)
393{
394 vcpustate->dirty |= dirtybits;
395}
396
377/*
378 * Initialise a virtual machine.
379 */
380static void *
381svm_vminit(struct vm *vm, pmap_t pmap)
382{
383 struct svm_softc *svm_sc;
384 struct svm_vcpu *vcpu;
385 vm_paddr_t msrpm_pa, iopm_pa, pml4_pa;
397/*
398 * Initialise a virtual machine.
399 */
400static void *
401svm_vminit(struct vm *vm, pmap_t pmap)
402{
403 struct svm_softc *svm_sc;
404 struct svm_vcpu *vcpu;
405 vm_paddr_t msrpm_pa, iopm_pa, pml4_pa;
386 int i, error;
406 int i;
387
407
388 if (guest_asid >= max_asid) {
389 ERR("Host support max ASID:%d, can't create more guests.\n",
390 max_asid);
391 return (NULL);
392 }
393
394 svm_sc = (struct svm_softc *)malloc(sizeof (struct svm_softc),
395 M_SVM, M_WAITOK | M_ZERO);
396
397 svm_sc->vm = vm;
398 svm_sc->svm_feature = svm_feature;
399 svm_sc->vcpu_cnt = VM_MAXCPU;
400 svm_sc->nptp = (vm_offset_t)vtophys(pmap->pm_pml4);
408 svm_sc = (struct svm_softc *)malloc(sizeof (struct svm_softc),
409 M_SVM, M_WAITOK | M_ZERO);
410
411 svm_sc->vm = vm;
412 svm_sc->svm_feature = svm_feature;
413 svm_sc->vcpu_cnt = VM_MAXCPU;
414 svm_sc->nptp = (vm_offset_t)vtophys(pmap->pm_pml4);
415
401 /*
416 /*
402 * Each guest has its own unique ASID.
403 * ASID(Address Space Identifier) is used by TLB entry.
404 */
405 svm_sc->asid = guest_asid++;
406
407 /*
408 * Intercept MSR access to all MSRs except GSBASE, FSBASE,... etc.
409 */
410 memset(svm_sc->msr_bitmap, 0xFF, sizeof(svm_sc->msr_bitmap));
411
412 /*
413 * Following MSR can be completely controlled by virtual machines
414 * since access to following are translated to access to VMCB.
415 */

--- 21 unchanged lines hidden (view full) ---

437 iopm_pa = vtophys(svm_sc->iopm_bitmap);
438 msrpm_pa = vtophys(svm_sc->msr_bitmap);
439 pml4_pa = svm_sc->nptp;
440
441 for (i = 0; i < svm_sc->vcpu_cnt; i++) {
442 vcpu = svm_get_vcpu(svm_sc, i);
443 vcpu->lastcpu = NOCPU;
444 vcpu->vmcb_pa = vtophys(&vcpu->vmcb);
417 * Intercept MSR access to all MSRs except GSBASE, FSBASE,... etc.
418 */
419 memset(svm_sc->msr_bitmap, 0xFF, sizeof(svm_sc->msr_bitmap));
420
421 /*
422 * Following MSR can be completely controlled by virtual machines
423 * since access to following are translated to access to VMCB.
424 */

--- 21 unchanged lines hidden (view full) ---

446 iopm_pa = vtophys(svm_sc->iopm_bitmap);
447 msrpm_pa = vtophys(svm_sc->msr_bitmap);
448 pml4_pa = svm_sc->nptp;
449
450 for (i = 0; i < svm_sc->vcpu_cnt; i++) {
451 vcpu = svm_get_vcpu(svm_sc, i);
452 vcpu->lastcpu = NOCPU;
453 vcpu->vmcb_pa = vtophys(&vcpu->vmcb);
445 error = svm_init_vmcb(&vcpu->vmcb, iopm_pa, msrpm_pa, pml4_pa,
446 svm_sc->asid);
447 if (error)
448 goto cleanup;
454 svm_init_vmcb(&vcpu->vmcb, iopm_pa, msrpm_pa, pml4_pa);
449 }
455 }
450
451 return (svm_sc);
456 return (svm_sc);
452
453cleanup:
454 free(svm_sc, M_SVM);
455 return (NULL);
456}
457
458static int
459svm_cpl(struct vmcb_state *state)
460{
461
462 /*
463 * From APMv2:

--- 616 unchanged lines hidden (view full) ---

1080 /*
1081 * XXX need to recheck exting_pending ala VT-x
1082 */
1083 }
1084
1085 VCPU_CTR1(svm_sc->vm, vcpu, "SVM:event injected,vector=%d.\n", vector);
1086}
1087
457}
458
459static int
460svm_cpl(struct vmcb_state *state)
461{
462
463 /*
464 * From APMv2:

--- 616 unchanged lines hidden (view full) ---

1081 /*
1082 * XXX need to recheck exting_pending ala VT-x
1083 */
1084 }
1085
1086 VCPU_CTR1(svm_sc->vm, vcpu, "SVM:event injected,vector=%d.\n", vector);
1087}
1088
1088static void
1089static __inline void
1089restore_host_tss(void)
1090{
1091 struct system_segment_descriptor *tss_sd;
1092
1093 /*
1094 * The TSS descriptor was in use prior to launching the guest so it
1095 * has been marked busy.
1096 *
1097 * 'ltr' requires the descriptor to be marked available so change the
1098 * type to "64-bit available TSS".
1099 */
1100 tss_sd = PCPU_GET(tss);
1101 tss_sd->sd_type = SDT_SYSTSS;
1102 ltr(GSEL(GPROC0_SEL, SEL_KPL));
1103}
1104
1090restore_host_tss(void)
1091{
1092 struct system_segment_descriptor *tss_sd;
1093
1094 /*
1095 * The TSS descriptor was in use prior to launching the guest so it
1096 * has been marked busy.
1097 *
1098 * 'ltr' requires the descriptor to be marked available so change the
1099 * type to "64-bit available TSS".
1100 */
1101 tss_sd = PCPU_GET(tss);
1102 tss_sd->sd_type = SDT_SYSTSS;
1103 ltr(GSEL(GPROC0_SEL, SEL_KPL));
1104}
1105
1106static void
1107check_asid(struct svm_softc *sc, int vcpuid, pmap_t pmap, u_int thiscpu)
1108{
1109 struct svm_vcpu *vcpustate;
1110 struct vmcb_ctrl *ctrl;
1111 long eptgen;
1112 bool alloc_asid;
1113
1114 KASSERT(CPU_ISSET(thiscpu, &pmap->pm_active), ("%s: nested pmap not "
1115 "active on cpu %u", __func__, thiscpu));
1116
1117 vcpustate = svm_get_vcpu(sc, vcpuid);
1118 ctrl = svm_get_vmcb_ctrl(sc, vcpuid);
1119
1120 /*
1121 * The TLB entries associated with the vcpu's ASID are not valid
1122 * if either of the following conditions is true:
1123 *
1124 * 1. The vcpu's ASID generation is different than the host cpu's
1125 * ASID generation. This happens when the vcpu migrates to a new
1126 * host cpu. It can also happen when the number of vcpus executing
1127 * on a host cpu is greater than the number of ASIDs available.
1128 *
1129 * 2. The pmap generation number is different than the value cached in
1130 * the 'vcpustate'. This happens when the host invalidates pages
1131 * belonging to the guest.
1132 *
1133 * asidgen eptgen Action
1134 * mismatch mismatch
1135 * 0 0 (a)
1136 * 0 1 (b1) or (b2)
1137 * 1 0 (c)
1138 * 1 1 (d)
1139 *
1140 * (a) There is no mismatch in eptgen or ASID generation and therefore
1141 * no further action is needed.
1142 *
1143 * (b1) If the cpu supports FlushByAsid then the vcpu's ASID is
1144 * retained and the TLB entries associated with this ASID
1145 * are flushed by VMRUN.
1146 *
1147 * (b2) If the cpu does not support FlushByAsid then a new ASID is
1148 * allocated.
1149 *
1150 * (c) A new ASID is allocated.
1151 *
1152 * (d) A new ASID is allocated.
1153 */
1154
1155 alloc_asid = false;
1156 eptgen = pmap->pm_eptgen;
1157 ctrl->tlb_ctrl = VMCB_TLB_FLUSH_NOTHING;
1158
1159 if (vcpustate->asid.gen != asid[thiscpu].gen) {
1160 alloc_asid = true; /* (c) and (d) */
1161 } else if (vcpustate->eptgen != eptgen) {
1162 if (flush_by_asid())
1163 ctrl->tlb_ctrl = VMCB_TLB_FLUSH_GUEST; /* (b1) */
1164 else
1165 alloc_asid = true; /* (b2) */
1166 } else {
1167 /*
1168 * This is the common case (a).
1169 */
1170 KASSERT(!alloc_asid, ("ASID allocation not necessary"));
1171 KASSERT(ctrl->tlb_ctrl == VMCB_TLB_FLUSH_NOTHING,
1172 ("Invalid VMCB tlb_ctrl: %#x", ctrl->tlb_ctrl));
1173 }
1174
1175 if (alloc_asid) {
1176 if (++asid[thiscpu].num >= nasid) {
1177 asid[thiscpu].num = 1;
1178 if (++asid[thiscpu].gen == 0)
1179 asid[thiscpu].gen = 1;
1180 /*
1181 * If this cpu does not support "flush-by-asid"
1182 * then flush the entire TLB on a generation
1183 * bump. Subsequent ASID allocation in this
1184 * generation can be done without a TLB flush.
1185 */
1186 if (!flush_by_asid())
1187 ctrl->tlb_ctrl = VMCB_TLB_FLUSH_ALL;
1188 }
1189 vcpustate->asid.gen = asid[thiscpu].gen;
1190 vcpustate->asid.num = asid[thiscpu].num;
1191
1192 ctrl->asid = vcpustate->asid.num;
1193 vcpu_set_dirty(vcpustate, VMCB_CACHE_ASID);
1194 /*
1195 * If this cpu supports "flush-by-asid" then the TLB
1196 * was not flushed after the generation bump. The TLB
1197 * is flushed selectively after every new ASID allocation.
1198 */
1199 if (flush_by_asid())
1200 ctrl->tlb_ctrl = VMCB_TLB_FLUSH_GUEST;
1201 }
1202 vcpustate->eptgen = eptgen;
1203
1204 KASSERT(ctrl->asid != 0, ("Guest ASID must be non-zero"));
1205 KASSERT(ctrl->asid == vcpustate->asid.num,
1206 ("ASID mismatch: %u/%u", ctrl->asid, vcpustate->asid.num));
1207}
1208
1105/*
1106 * Start vcpu with specified RIP.
1107 */
1108static int
1109svm_vmrun(void *arg, int vcpu, register_t rip, pmap_t pmap,
1110 void *rend_cookie, void *suspended_cookie)
1111{
1112 struct svm_regctx *hctx, *gctx;
1113 struct svm_softc *svm_sc;
1114 struct svm_vcpu *vcpustate;
1115 struct vmcb_state *state;
1116 struct vmcb_ctrl *ctrl;
1117 struct vm_exit *vmexit;
1118 struct vlapic *vlapic;
1119 struct vm *vm;
1120 uint64_t vmcb_pa;
1209/*
1210 * Start vcpu with specified RIP.
1211 */
1212static int
1213svm_vmrun(void *arg, int vcpu, register_t rip, pmap_t pmap,
1214 void *rend_cookie, void *suspended_cookie)
1215{
1216 struct svm_regctx *hctx, *gctx;
1217 struct svm_softc *svm_sc;
1218 struct svm_vcpu *vcpustate;
1219 struct vmcb_state *state;
1220 struct vmcb_ctrl *ctrl;
1221 struct vm_exit *vmexit;
1222 struct vlapic *vlapic;
1223 struct vm *vm;
1224 uint64_t vmcb_pa;
1225 u_int thiscpu;
1121 bool loop; /* Continue vcpu execution loop. */
1122
1123 loop = true;
1124 svm_sc = arg;
1125 vm = svm_sc->vm;
1126
1127 vcpustate = svm_get_vcpu(svm_sc, vcpu);
1128 state = svm_get_vmcb_state(svm_sc, vcpu);
1129 ctrl = svm_get_vmcb_ctrl(svm_sc, vcpu);
1130 vmexit = vm_exitinfo(vm, vcpu);
1131 vlapic = vm_lapic(vm, vcpu);
1132
1226 bool loop; /* Continue vcpu execution loop. */
1227
1228 loop = true;
1229 svm_sc = arg;
1230 vm = svm_sc->vm;
1231
1232 vcpustate = svm_get_vcpu(svm_sc, vcpu);
1233 state = svm_get_vmcb_state(svm_sc, vcpu);
1234 ctrl = svm_get_vmcb_ctrl(svm_sc, vcpu);
1235 vmexit = vm_exitinfo(vm, vcpu);
1236 vlapic = vm_lapic(vm, vcpu);
1237
1238 /*
1239 * Stash 'curcpu' on the stack as 'thiscpu'.
1240 *
1241 * The per-cpu data area is not accessible until MSR_GSBASE is restored
1242 * after the #VMEXIT. Since VMRUN is executed inside a critical section
1243 * 'curcpu' and 'thiscpu' are guaranteed to identical.
1244 */
1245 thiscpu = curcpu;
1246
1133 gctx = svm_get_guest_regctx(svm_sc, vcpu);
1247 gctx = svm_get_guest_regctx(svm_sc, vcpu);
1134 hctx = &host_ctx[curcpu];
1248 hctx = &host_ctx[thiscpu];
1135 vmcb_pa = svm_sc->vcpu[vcpu].vmcb_pa;
1136
1249 vmcb_pa = svm_sc->vcpu[vcpu].vmcb_pa;
1250
1137 if (vcpustate->lastcpu != curcpu) {
1138 /* Virtual CPU is running on a diiferent CPU now.*/
1139 vmm_stat_incr(vm, vcpu, VCPU_MIGRATIONS, 1);
1140
1251 if (vcpustate->lastcpu != thiscpu) {
1141 /*
1252 /*
1142 * Flush all TLB mappings for this guest on this CPU,
1143 * it might have stale entries since vcpu has migrated
1144 * or vmm is restarted.
1253 * Force new ASID allocation by invalidating the generation.
1145 */
1254 */
1146 ctrl->tlb_ctrl = VMCB_TLB_FLUSH_GUEST;
1255 vcpustate->asid.gen = 0;
1147
1256
1148 /* Can't use any cached VMCB state by cpu.*/
1149 ctrl->vmcb_clean = VMCB_CACHE_NONE;
1150 } else {
1151 /*
1257 /*
1152 * XXX: Using same ASID for all vcpus of a VM will cause TLB
1153 * corruption. This can easily be produced by muxing two vcpus
1154 * on same core.
1155 * For now, flush guest TLB for every vmrun.
1258 * Invalidate the VMCB state cache by marking all fields dirty.
1156 */
1259 */
1157 ctrl->tlb_ctrl = VMCB_TLB_FLUSH_GUEST;
1158
1159 /*
1160 * This is the same cpu on which vcpu last ran so don't
1161 * need to reload all VMCB state.
1162 * ASID is unique for a guest.
1163 * IOPM is unchanged.
1164 * RVI/EPT is unchanged.
1260 vcpu_set_dirty(vcpustate, 0xffffffff);
1261
1262 /*
1263 * XXX
1264 * Setting 'vcpustate->lastcpu' here is bit premature because
1265 * we may return from this function without actually executing
1266 * the VMRUN instruction. This could happen if a rendezvous
1267 * or an AST is pending on the first time through the loop.
1165 *
1268 *
1269 * This works for now but any new side-effects of vcpu
1270 * migration should take this case into account.
1166 */
1271 */
1167 ctrl->vmcb_clean = VMCB_CACHE_ASID |
1168 VMCB_CACHE_IOPM |
1169 VMCB_CACHE_NP;
1272 vcpustate->lastcpu = thiscpu;
1273 vmm_stat_incr(vm, vcpu, VCPU_MIGRATIONS, 1);
1170 }
1171
1274 }
1275
1172 vcpustate->lastcpu = curcpu;
1173 VCPU_CTR3(vm, vcpu, "SVM:Enter vmrun RIP:0x%lx"
1174 " inst len=%d/%d\n",
1175 rip, vmexit->inst_length,
1176 vmexit->u.inst_emul.vie.num_valid);
1177 /* Update Guest RIP */
1178 state->rip = rip;
1276 VCPU_CTR3(vm, vcpu, "SVM:Enter vmrun RIP:0x%lx"
1277 " inst len=%d/%d\n",
1278 rip, vmexit->inst_length,
1279 vmexit->u.inst_emul.vie.num_valid);
1280 /* Update Guest RIP */
1281 state->rip = rip;
1179
1282
1180 do {
1181 vmexit->inst_length = 0;
1182
1183 /*
1184 * Disable global interrupts to guarantee atomicity during
1185 * loading of guest state. This includes not only the state
1186 * loaded by the "vmrun" instruction but also software state
1187 * maintained by the hypervisor: suspended and rendezvous

--- 26 unchanged lines hidden (view full) ---

1214 VCPU_CTR1(vm, vcpu,
1215 "SVM: ASTPENDING, RIP:0x%lx\n", state->rip);
1216 vmexit->rip = state->rip;
1217 break;
1218 }
1219
1220 svm_inj_interrupts(svm_sc, vcpu, vlapic);
1221
1283 do {
1284 vmexit->inst_length = 0;
1285
1286 /*
1287 * Disable global interrupts to guarantee atomicity during
1288 * loading of guest state. This includes not only the state
1289 * loaded by the "vmrun" instruction but also software state
1290 * maintained by the hypervisor: suspended and rendezvous

--- 26 unchanged lines hidden (view full) ---

1317 VCPU_CTR1(vm, vcpu,
1318 "SVM: ASTPENDING, RIP:0x%lx\n", state->rip);
1319 vmexit->rip = state->rip;
1320 break;
1321 }
1322
1323 svm_inj_interrupts(svm_sc, vcpu, vlapic);
1324
1325 /* Activate the nested pmap on 'thiscpu' */
1326 CPU_SET_ATOMIC_ACQ(thiscpu, &pmap->pm_active);
1327
1328 /*
1329 * Check the pmap generation and the ASID generation to
1330 * ensure that the vcpu does not use stale TLB mappings.
1331 */
1332 check_asid(svm_sc, vcpu, pmap, thiscpu);
1333
1334 ctrl->vmcb_clean = VMCB_CACHE_DEFAULT & ~vcpustate->dirty;
1335 vcpustate->dirty = 0;
1336
1222 /* Launch Virtual Machine. */
1223 svm_launch(vmcb_pa, gctx, hctx);
1224
1337 /* Launch Virtual Machine. */
1338 svm_launch(vmcb_pa, gctx, hctx);
1339
1340 CPU_CLR_ATOMIC(thiscpu, &pmap->pm_active);
1341
1225 /*
1226 * Restore MSR_GSBASE to point to the pcpu data area.
1227 *
1228 * Note that accesses done via PCPU_GET/PCPU_SET will work
1229 * only after MSR_GSBASE is restored.
1230 *
1231 * Also note that we don't bother restoring MSR_KGSBASE
1232 * since it is not used in the kernel and will be restored
1233 * when the VMRUN ioctl returns to userspace.
1234 */
1342 /*
1343 * Restore MSR_GSBASE to point to the pcpu data area.
1344 *
1345 * Note that accesses done via PCPU_GET/PCPU_SET will work
1346 * only after MSR_GSBASE is restored.
1347 *
1348 * Also note that we don't bother restoring MSR_KGSBASE
1349 * since it is not used in the kernel and will be restored
1350 * when the VMRUN ioctl returns to userspace.
1351 */
1235 wrmsr(MSR_GSBASE, (uint64_t)&__pcpu[vcpustate->lastcpu]);
1352 wrmsr(MSR_GSBASE, (uint64_t)&__pcpu[thiscpu]);
1353 KASSERT(curcpu == thiscpu, ("thiscpu/curcpu (%u/%u) mismatch",
1354 thiscpu, curcpu));
1236
1237 /*
1238 * The host GDTR and IDTR is saved by VMRUN and restored
1239 * automatically on #VMEXIT. However, the host TSS needs
1240 * to be restored explicitly.
1241 */
1242 restore_host_tss();
1243

--- 114 unchanged lines hidden (view full) ---

1358 KASSERT(vcpu < svm_sc->vcpu_cnt, ("Guest doesn't have VCPU%d", vcpu));
1359
1360 vmcb = svm_get_vmcb(svm_sc, vcpu);
1361 if (vmcb_write(vmcb, ident, val) == 0) {
1362 return (0);
1363 }
1364
1365 reg = swctx_regptr(svm_get_guest_regctx(svm_sc, vcpu), ident);
1355
1356 /*
1357 * The host GDTR and IDTR is saved by VMRUN and restored
1358 * automatically on #VMEXIT. However, the host TSS needs
1359 * to be restored explicitly.
1360 */
1361 restore_host_tss();
1362

--- 114 unchanged lines hidden (view full) ---

1477 KASSERT(vcpu < svm_sc->vcpu_cnt, ("Guest doesn't have VCPU%d", vcpu));
1478
1479 vmcb = svm_get_vmcb(svm_sc, vcpu);
1480 if (vmcb_write(vmcb, ident, val) == 0) {
1481 return (0);
1482 }
1483
1484 reg = swctx_regptr(svm_get_guest_regctx(svm_sc, vcpu), ident);
1366
1485
1367 if (reg != NULL) {
1368 *reg = val;
1369 return (0);
1370 }
1371
1486 if (reg != NULL) {
1487 *reg = val;
1488 return (0);
1489 }
1490
1491 /*
1492 * XXX deal with CR3 and invalidate TLB entries tagged with the
1493 * vcpu's ASID. This needs to be treated differently depending on
1494 * whether 'running' is true/false.
1495 */
1496
1372 ERR("SVM_ERR:reg type %x is not saved in VMCB.\n", ident);
1373 return (EINVAL);
1374}
1375
1376
1377/*
1378 * Inteface to set various descriptors.
1379 */

--- 216 unchanged lines hidden --- 		1497 ERR("SVM_ERR:reg type %x is not saved in VMCB.\n", ident);
1498 return (EINVAL);
1499}
1500
1501
1502/*
1503 * Inteface to set various descriptors.
1504 */

--- 216 unchanged lines hidden ---