svm.c (271152) | svm.c (271203) |
---|---|
1/*- 2 * Copyright (c) 2013, Anish Gupta (akgupt3@gmail.com) 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright --- 11 unchanged lines hidden (view full) --- 20 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 21 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 22 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 23 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 24 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 25 */ 26 27#include <sys/cdefs.h> | 1/*- 2 * Copyright (c) 2013, Anish Gupta (akgupt3@gmail.com) 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright --- 11 unchanged lines hidden (view full) --- 20 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 21 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 22 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 23 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 24 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 25 */ 26 27#include <sys/cdefs.h> |
28__FBSDID("$FreeBSD: projects/bhyve_svm/sys/amd64/vmm/amd/svm.c 271152 2014-09-05 03:33:16Z neel $"); | 28__FBSDID("$FreeBSD: projects/bhyve_svm/sys/amd64/vmm/amd/svm.c 271203 2014-09-06 19:02:52Z neel $"); |
29 30#include <sys/param.h> 31#include <sys/systm.h> 32#include <sys/smp.h> 33#include <sys/kernel.h> 34#include <sys/malloc.h> 35#include <sys/pcpu.h> 36#include <sys/proc.h> --- 33 unchanged lines hidden (view full) --- 70 * SVM CPUID function 0x8000_000A, edx bit decoding. 71 */ 72#define AMD_CPUID_SVM_NP BIT(0) /* Nested paging or RVI */ 73#define AMD_CPUID_SVM_LBR BIT(1) /* Last branch virtualization */ 74#define AMD_CPUID_SVM_SVML BIT(2) /* SVM lock */ 75#define AMD_CPUID_SVM_NRIP_SAVE BIT(3) /* Next RIP is saved */ 76#define AMD_CPUID_SVM_TSC_RATE BIT(4) /* TSC rate control. */ 77#define AMD_CPUID_SVM_VMCB_CLEAN BIT(5) /* VMCB state caching */ | 29 30#include <sys/param.h> 31#include <sys/systm.h> 32#include <sys/smp.h> 33#include <sys/kernel.h> 34#include <sys/malloc.h> 35#include <sys/pcpu.h> 36#include <sys/proc.h> --- 33 unchanged lines hidden (view full) --- 70 * SVM CPUID function 0x8000_000A, edx bit decoding. 71 */ 72#define AMD_CPUID_SVM_NP BIT(0) /* Nested paging or RVI */ 73#define AMD_CPUID_SVM_LBR BIT(1) /* Last branch virtualization */ 74#define AMD_CPUID_SVM_SVML BIT(2) /* SVM lock */ 75#define AMD_CPUID_SVM_NRIP_SAVE BIT(3) /* Next RIP is saved */ 76#define AMD_CPUID_SVM_TSC_RATE BIT(4) /* TSC rate control. */ 77#define AMD_CPUID_SVM_VMCB_CLEAN BIT(5) /* VMCB state caching */ |
78#define AMD_CPUID_SVM_ASID_FLUSH BIT(6) /* Flush by ASID */ | 78#define AMD_CPUID_SVM_FLUSH_BY_ASID BIT(6) /* Flush by ASID */ |
79#define AMD_CPUID_SVM_DECODE_ASSIST BIT(7) /* Decode assist */ 80#define AMD_CPUID_SVM_PAUSE_INC BIT(10) /* Pause intercept filter. */ 81#define AMD_CPUID_SVM_PAUSE_FTH BIT(12) /* Pause filter threshold */ 82 | 79#define AMD_CPUID_SVM_DECODE_ASSIST BIT(7) /* Decode assist */ 80#define AMD_CPUID_SVM_PAUSE_INC BIT(10) /* Pause intercept filter. */ 81#define AMD_CPUID_SVM_PAUSE_FTH BIT(12) /* Pause filter threshold */ 82 |
83#define VMCB_CACHE_DEFAULT \ 84 (VMCB_CACHE_ASID | VMCB_CACHE_IOPM | VMCB_CACHE_NP) 85 |
|
83MALLOC_DEFINE(M_SVM, "svm", "svm"); 84MALLOC_DEFINE(M_SVM_VLAPIC, "svm-vlapic", "svm-vlapic"); 85 86/* Per-CPU context area. */ 87extern struct pcpu __pcpu[]; 88 89static bool svm_vmexit(struct svm_softc *svm_sc, int vcpu, 90 struct vm_exit *vmexit); 91static int svm_msr_rw_ok(uint8_t *btmap, uint64_t msr); 92static int svm_msr_rd_ok(uint8_t *btmap, uint64_t msr); 93static int svm_msr_index(uint64_t msr, int *index, int *bit); 94static int svm_getdesc(void *arg, int vcpu, int type, struct seg_desc *desc); 95 | 86MALLOC_DEFINE(M_SVM, "svm", "svm"); 87MALLOC_DEFINE(M_SVM_VLAPIC, "svm-vlapic", "svm-vlapic"); 88 89/* Per-CPU context area. */ 90extern struct pcpu __pcpu[]; 91 92static bool svm_vmexit(struct svm_softc *svm_sc, int vcpu, 93 struct vm_exit *vmexit); 94static int svm_msr_rw_ok(uint8_t *btmap, uint64_t msr); 95static int svm_msr_rd_ok(uint8_t *btmap, uint64_t msr); 96static int svm_msr_index(uint64_t msr, int *index, int *bit); 97static int svm_getdesc(void *arg, int vcpu, int type, struct seg_desc *desc); 98 |
96static uint32_t svm_feature; /* AMD SVM features. */ | 99static uint32_t svm_feature; /* AMD SVM features. */ |
97 | 100 |
98/* 99 * Starting guest ASID, 0 is reserved for host. 100 * Each guest will have its own unique ASID. 101 */ 102static uint32_t guest_asid = 1; | 101/* Maximum ASIDs supported by the processor */ 102static uint32_t nasid; |
103 | 103 |
104/* 105 * Max ASID processor can support. 106 * This limit the maximum number of virtual machines that can be created. 107 */ 108static int max_asid; | 104/* Current ASID generation for each host cpu */ 105static struct asid asid[MAXCPU]; |
109 110/* 111 * SVM host state saved area of size 4KB for each core. 112 */ 113static uint8_t hsave[MAXCPU][PAGE_SIZE] __aligned(PAGE_SIZE); 114 115/* 116 * S/w saved host context. --- 52 unchanged lines hidden (view full) --- 169{ 170 u_int regs[4]; 171 172 /* CPUID Fn8000_000A is for SVM */ 173 do_cpuid(0x8000000A, regs); 174 svm_feature = regs[3]; 175 176 printf("SVM rev: 0x%x NASID:0x%x\n", regs[0] & 0xFF, regs[1]); | 106 107/* 108 * SVM host state saved area of size 4KB for each core. 109 */ 110static uint8_t hsave[MAXCPU][PAGE_SIZE] __aligned(PAGE_SIZE); 111 112/* 113 * S/w saved host context. --- 52 unchanged lines hidden (view full) --- 166{ 167 u_int regs[4]; 168 169 /* CPUID Fn8000_000A is for SVM */ 170 do_cpuid(0x8000000A, regs); 171 svm_feature = regs[3]; 172 173 printf("SVM rev: 0x%x NASID:0x%x\n", regs[0] & 0xFF, regs[1]); |
177 max_asid = regs[1]; 178 | 174 nasid = regs[1]; 175 KASSERT(nasid > 1, ("Insufficient ASIDs for guests: %#x", nasid)); 176 |
179 printf("SVM Features:0x%b\n", svm_feature, 180 "\020" 181 "\001NP" /* Nested paging */ 182 "\002LbrVirt" /* LBR virtualization */ 183 "\003SVML" /* SVM lock */ 184 "\004NRIPS" /* NRIP save */ 185 "\005TscRateMsr" /* MSR based TSC rate control */ 186 "\006VmcbClean" /* VMCB clean bits */ --- 22 unchanged lines hidden (view full) --- 209 } 210 211 if (svm_feature & AMD_CPUID_SVM_NRIP_SAVE) 212 return (0); 213 214 return (EIO); 215} 216 | 177 printf("SVM Features:0x%b\n", svm_feature, 178 "\020" 179 "\001NP" /* Nested paging */ 180 "\002LbrVirt" /* LBR virtualization */ 181 "\003SVML" /* SVM lock */ 182 "\004NRIPS" /* NRIP save */ 183 "\005TscRateMsr" /* MSR based TSC rate control */ 184 "\006VmcbClean" /* VMCB clean bits */ --- 22 unchanged lines hidden (view full) --- 207 } 208 209 if (svm_feature & AMD_CPUID_SVM_NRIP_SAVE) 210 return (0); 211 212 return (EIO); 213} 214 |
215static __inline int 216flush_by_asid(void) 217{ 218 return (svm_feature & AMD_CPUID_SVM_FLUSH_BY_ASID); 219} 220 |
|
217/* 218 * Enable SVM for a CPU. 219 */ 220static void 221svm_enable(void *arg __unused) 222{ 223 uint64_t hsave_pa; 224 --- 32 unchanged lines hidden (view full) --- 257} 258 259/* 260 * Enable SVM on CPU and initialize nested page table h/w. 261 */ 262static int 263svm_init(int ipinum) 264{ | 221/* 222 * Enable SVM for a CPU. 223 */ 224static void 225svm_enable(void *arg __unused) 226{ 227 uint64_t hsave_pa; 228 --- 32 unchanged lines hidden (view full) --- 261} 262 263/* 264 * Enable SVM on CPU and initialize nested page table h/w. 265 */ 266static int 267svm_init(int ipinum) 268{ |
265 int err; | 269 int err, cpu; |
266 267 err = is_svm_enabled(); 268 if (err) 269 return (err); 270 | 270 271 err = is_svm_enabled(); 272 if (err) 273 return (err); 274 |
275 for (cpu = 0; cpu < MAXCPU; cpu++) { 276 /* 277 * Initialize the host ASIDs to their "highest" valid values. 278 * 279 * The next ASID allocation will rollover both 'gen' and 'num' 280 * and start off the sequence at {1,1}. 281 */ 282 asid[cpu].gen = ~0UL; 283 asid[cpu].num = nasid - 1; 284 } |
|
271 272 svm_npt_init(ipinum); 273 274 /* Start SVM on all CPUs */ 275 smp_rendezvous(NULL, svm_enable, NULL, NULL); | 285 286 svm_npt_init(ipinum); 287 288 /* Start SVM on all CPUs */ 289 smp_rendezvous(NULL, svm_enable, NULL, NULL); |
276 | 290 |
277 return (0); 278} 279 280static void 281svm_restore(void) 282{ 283 svm_enable(NULL); 284} --- 84 unchanged lines hidden (view full) --- 369} 370 371static int 372svm_msr_rd_ok(uint8_t *perm_bitmap, uint64_t msr) 373{ 374 return svm_msr_perm(perm_bitmap, msr, true, false); 375} 376 | 291 return (0); 292} 293 294static void 295svm_restore(void) 296{ 297 svm_enable(NULL); 298} --- 84 unchanged lines hidden (view full) --- 383} 384 385static int 386svm_msr_rd_ok(uint8_t *perm_bitmap, uint64_t msr) 387{ 388 return svm_msr_perm(perm_bitmap, msr, true, false); 389} 390 |
391static __inline void 392vcpu_set_dirty(struct svm_vcpu *vcpustate, uint32_t dirtybits) 393{ 394 vcpustate->dirty |= dirtybits; 395} 396 |
|
377/* 378 * Initialise a virtual machine. 379 */ 380static void * 381svm_vminit(struct vm *vm, pmap_t pmap) 382{ 383 struct svm_softc *svm_sc; 384 struct svm_vcpu *vcpu; 385 vm_paddr_t msrpm_pa, iopm_pa, pml4_pa; | 397/* 398 * Initialise a virtual machine. 399 */ 400static void * 401svm_vminit(struct vm *vm, pmap_t pmap) 402{ 403 struct svm_softc *svm_sc; 404 struct svm_vcpu *vcpu; 405 vm_paddr_t msrpm_pa, iopm_pa, pml4_pa; |
386 int i, error; | 406 int i; |
387 | 407 |
388 if (guest_asid >= max_asid) { 389 ERR("Host support max ASID:%d, can't create more guests.\n", 390 max_asid); 391 return (NULL); 392 } 393 | |
394 svm_sc = (struct svm_softc *)malloc(sizeof (struct svm_softc), 395 M_SVM, M_WAITOK | M_ZERO); 396 397 svm_sc->vm = vm; 398 svm_sc->svm_feature = svm_feature; 399 svm_sc->vcpu_cnt = VM_MAXCPU; 400 svm_sc->nptp = (vm_offset_t)vtophys(pmap->pm_pml4); | 408 svm_sc = (struct svm_softc *)malloc(sizeof (struct svm_softc), 409 M_SVM, M_WAITOK | M_ZERO); 410 411 svm_sc->vm = vm; 412 svm_sc->svm_feature = svm_feature; 413 svm_sc->vcpu_cnt = VM_MAXCPU; 414 svm_sc->nptp = (vm_offset_t)vtophys(pmap->pm_pml4); |
415 |
|
401 /* | 416 /* |
402 * Each guest has its own unique ASID. 403 * ASID(Address Space Identifier) is used by TLB entry. 404 */ 405 svm_sc->asid = guest_asid++; 406 407 /* | |
408 * Intercept MSR access to all MSRs except GSBASE, FSBASE,... etc. 409 */ 410 memset(svm_sc->msr_bitmap, 0xFF, sizeof(svm_sc->msr_bitmap)); 411 412 /* 413 * Following MSR can be completely controlled by virtual machines 414 * since access to following are translated to access to VMCB. 415 */ --- 21 unchanged lines hidden (view full) --- 437 iopm_pa = vtophys(svm_sc->iopm_bitmap); 438 msrpm_pa = vtophys(svm_sc->msr_bitmap); 439 pml4_pa = svm_sc->nptp; 440 441 for (i = 0; i < svm_sc->vcpu_cnt; i++) { 442 vcpu = svm_get_vcpu(svm_sc, i); 443 vcpu->lastcpu = NOCPU; 444 vcpu->vmcb_pa = vtophys(&vcpu->vmcb); | 417 * Intercept MSR access to all MSRs except GSBASE, FSBASE,... etc. 418 */ 419 memset(svm_sc->msr_bitmap, 0xFF, sizeof(svm_sc->msr_bitmap)); 420 421 /* 422 * Following MSR can be completely controlled by virtual machines 423 * since access to following are translated to access to VMCB. 424 */ --- 21 unchanged lines hidden (view full) --- 446 iopm_pa = vtophys(svm_sc->iopm_bitmap); 447 msrpm_pa = vtophys(svm_sc->msr_bitmap); 448 pml4_pa = svm_sc->nptp; 449 450 for (i = 0; i < svm_sc->vcpu_cnt; i++) { 451 vcpu = svm_get_vcpu(svm_sc, i); 452 vcpu->lastcpu = NOCPU; 453 vcpu->vmcb_pa = vtophys(&vcpu->vmcb); |
445 error = svm_init_vmcb(&vcpu->vmcb, iopm_pa, msrpm_pa, pml4_pa, 446 svm_sc->asid); 447 if (error) 448 goto cleanup; | 454 svm_init_vmcb(&vcpu->vmcb, iopm_pa, msrpm_pa, pml4_pa); |
449 } | 455 } |
450 | |
451 return (svm_sc); | 456 return (svm_sc); |
452 453cleanup: 454 free(svm_sc, M_SVM); 455 return (NULL); | |
456} 457 458static int 459svm_cpl(struct vmcb_state *state) 460{ 461 462 /* 463 * From APMv2: --- 616 unchanged lines hidden (view full) --- 1080 /* 1081 * XXX need to recheck exting_pending ala VT-x 1082 */ 1083 } 1084 1085 VCPU_CTR1(svm_sc->vm, vcpu, "SVM:event injected,vector=%d.\n", vector); 1086} 1087 | 457} 458 459static int 460svm_cpl(struct vmcb_state *state) 461{ 462 463 /* 464 * From APMv2: --- 616 unchanged lines hidden (view full) --- 1081 /* 1082 * XXX need to recheck exting_pending ala VT-x 1083 */ 1084 } 1085 1086 VCPU_CTR1(svm_sc->vm, vcpu, "SVM:event injected,vector=%d.\n", vector); 1087} 1088 |
1088static void | 1089static __inline void |
1089restore_host_tss(void) 1090{ 1091 struct system_segment_descriptor *tss_sd; 1092 1093 /* 1094 * The TSS descriptor was in use prior to launching the guest so it 1095 * has been marked busy. 1096 * 1097 * 'ltr' requires the descriptor to be marked available so change the 1098 * type to "64-bit available TSS". 1099 */ 1100 tss_sd = PCPU_GET(tss); 1101 tss_sd->sd_type = SDT_SYSTSS; 1102 ltr(GSEL(GPROC0_SEL, SEL_KPL)); 1103} 1104 | 1090restore_host_tss(void) 1091{ 1092 struct system_segment_descriptor *tss_sd; 1093 1094 /* 1095 * The TSS descriptor was in use prior to launching the guest so it 1096 * has been marked busy. 1097 * 1098 * 'ltr' requires the descriptor to be marked available so change the 1099 * type to "64-bit available TSS". 1100 */ 1101 tss_sd = PCPU_GET(tss); 1102 tss_sd->sd_type = SDT_SYSTSS; 1103 ltr(GSEL(GPROC0_SEL, SEL_KPL)); 1104} 1105 |
1106static void 1107check_asid(struct svm_softc *sc, int vcpuid, pmap_t pmap, u_int thiscpu) 1108{ 1109 struct svm_vcpu *vcpustate; 1110 struct vmcb_ctrl *ctrl; 1111 long eptgen; 1112 bool alloc_asid; 1113 1114 KASSERT(CPU_ISSET(thiscpu, &pmap->pm_active), ("%s: nested pmap not " 1115 "active on cpu %u", __func__, thiscpu)); 1116 1117 vcpustate = svm_get_vcpu(sc, vcpuid); 1118 ctrl = svm_get_vmcb_ctrl(sc, vcpuid); 1119 1120 /* 1121 * The TLB entries associated with the vcpu's ASID are not valid 1122 * if either of the following conditions is true: 1123 * 1124 * 1. The vcpu's ASID generation is different than the host cpu's 1125 * ASID generation. This happens when the vcpu migrates to a new 1126 * host cpu. It can also happen when the number of vcpus executing 1127 * on a host cpu is greater than the number of ASIDs available. 1128 * 1129 * 2. The pmap generation number is different than the value cached in 1130 * the 'vcpustate'. This happens when the host invalidates pages 1131 * belonging to the guest. 1132 * 1133 * asidgen eptgen Action 1134 * mismatch mismatch 1135 * 0 0 (a) 1136 * 0 1 (b1) or (b2) 1137 * 1 0 (c) 1138 * 1 1 (d) 1139 * 1140 * (a) There is no mismatch in eptgen or ASID generation and therefore 1141 * no further action is needed. 1142 * 1143 * (b1) If the cpu supports FlushByAsid then the vcpu's ASID is 1144 * retained and the TLB entries associated with this ASID 1145 * are flushed by VMRUN. 1146 * 1147 * (b2) If the cpu does not support FlushByAsid then a new ASID is 1148 * allocated. 1149 * 1150 * (c) A new ASID is allocated. 1151 * 1152 * (d) A new ASID is allocated. 1153 */ 1154 1155 alloc_asid = false; 1156 eptgen = pmap->pm_eptgen; 1157 ctrl->tlb_ctrl = VMCB_TLB_FLUSH_NOTHING; 1158 1159 if (vcpustate->asid.gen != asid[thiscpu].gen) { 1160 alloc_asid = true; /* (c) and (d) */ 1161 } else if (vcpustate->eptgen != eptgen) { 1162 if (flush_by_asid()) 1163 ctrl->tlb_ctrl = VMCB_TLB_FLUSH_GUEST; /* (b1) */ 1164 else 1165 alloc_asid = true; /* (b2) */ 1166 } else { 1167 /* 1168 * This is the common case (a). 1169 */ 1170 KASSERT(!alloc_asid, ("ASID allocation not necessary")); 1171 KASSERT(ctrl->tlb_ctrl == VMCB_TLB_FLUSH_NOTHING, 1172 ("Invalid VMCB tlb_ctrl: %#x", ctrl->tlb_ctrl)); 1173 } 1174 1175 if (alloc_asid) { 1176 if (++asid[thiscpu].num >= nasid) { 1177 asid[thiscpu].num = 1; 1178 if (++asid[thiscpu].gen == 0) 1179 asid[thiscpu].gen = 1; 1180 /* 1181 * If this cpu does not support "flush-by-asid" 1182 * then flush the entire TLB on a generation 1183 * bump. Subsequent ASID allocation in this 1184 * generation can be done without a TLB flush. 1185 */ 1186 if (!flush_by_asid()) 1187 ctrl->tlb_ctrl = VMCB_TLB_FLUSH_ALL; 1188 } 1189 vcpustate->asid.gen = asid[thiscpu].gen; 1190 vcpustate->asid.num = asid[thiscpu].num; 1191 1192 ctrl->asid = vcpustate->asid.num; 1193 vcpu_set_dirty(vcpustate, VMCB_CACHE_ASID); 1194 /* 1195 * If this cpu supports "flush-by-asid" then the TLB 1196 * was not flushed after the generation bump. The TLB 1197 * is flushed selectively after every new ASID allocation. 1198 */ 1199 if (flush_by_asid()) 1200 ctrl->tlb_ctrl = VMCB_TLB_FLUSH_GUEST; 1201 } 1202 vcpustate->eptgen = eptgen; 1203 1204 KASSERT(ctrl->asid != 0, ("Guest ASID must be non-zero")); 1205 KASSERT(ctrl->asid == vcpustate->asid.num, 1206 ("ASID mismatch: %u/%u", ctrl->asid, vcpustate->asid.num)); 1207} 1208 |
|
1105/* 1106 * Start vcpu with specified RIP. 1107 */ 1108static int 1109svm_vmrun(void *arg, int vcpu, register_t rip, pmap_t pmap, 1110 void *rend_cookie, void *suspended_cookie) 1111{ 1112 struct svm_regctx *hctx, *gctx; 1113 struct svm_softc *svm_sc; 1114 struct svm_vcpu *vcpustate; 1115 struct vmcb_state *state; 1116 struct vmcb_ctrl *ctrl; 1117 struct vm_exit *vmexit; 1118 struct vlapic *vlapic; 1119 struct vm *vm; 1120 uint64_t vmcb_pa; | 1209/* 1210 * Start vcpu with specified RIP. 1211 */ 1212static int 1213svm_vmrun(void *arg, int vcpu, register_t rip, pmap_t pmap, 1214 void *rend_cookie, void *suspended_cookie) 1215{ 1216 struct svm_regctx *hctx, *gctx; 1217 struct svm_softc *svm_sc; 1218 struct svm_vcpu *vcpustate; 1219 struct vmcb_state *state; 1220 struct vmcb_ctrl *ctrl; 1221 struct vm_exit *vmexit; 1222 struct vlapic *vlapic; 1223 struct vm *vm; 1224 uint64_t vmcb_pa; |
1225 u_int thiscpu; |
|
1121 bool loop; /* Continue vcpu execution loop. */ 1122 1123 loop = true; 1124 svm_sc = arg; 1125 vm = svm_sc->vm; 1126 1127 vcpustate = svm_get_vcpu(svm_sc, vcpu); 1128 state = svm_get_vmcb_state(svm_sc, vcpu); 1129 ctrl = svm_get_vmcb_ctrl(svm_sc, vcpu); 1130 vmexit = vm_exitinfo(vm, vcpu); 1131 vlapic = vm_lapic(vm, vcpu); 1132 | 1226 bool loop; /* Continue vcpu execution loop. */ 1227 1228 loop = true; 1229 svm_sc = arg; 1230 vm = svm_sc->vm; 1231 1232 vcpustate = svm_get_vcpu(svm_sc, vcpu); 1233 state = svm_get_vmcb_state(svm_sc, vcpu); 1234 ctrl = svm_get_vmcb_ctrl(svm_sc, vcpu); 1235 vmexit = vm_exitinfo(vm, vcpu); 1236 vlapic = vm_lapic(vm, vcpu); 1237 |
1238 /* 1239 * Stash 'curcpu' on the stack as 'thiscpu'. 1240 * 1241 * The per-cpu data area is not accessible until MSR_GSBASE is restored 1242 * after the #VMEXIT. Since VMRUN is executed inside a critical section 1243 * 'curcpu' and 'thiscpu' are guaranteed to identical. 1244 */ 1245 thiscpu = curcpu; 1246 |
|
1133 gctx = svm_get_guest_regctx(svm_sc, vcpu); | 1247 gctx = svm_get_guest_regctx(svm_sc, vcpu); |
1134 hctx = &host_ctx[curcpu]; | 1248 hctx = &host_ctx[thiscpu]; |
1135 vmcb_pa = svm_sc->vcpu[vcpu].vmcb_pa; 1136 | 1249 vmcb_pa = svm_sc->vcpu[vcpu].vmcb_pa; 1250 |
1137 if (vcpustate->lastcpu != curcpu) { 1138 /* Virtual CPU is running on a diiferent CPU now.*/ 1139 vmm_stat_incr(vm, vcpu, VCPU_MIGRATIONS, 1); 1140 | 1251 if (vcpustate->lastcpu != thiscpu) { |
1141 /* | 1252 /* |
1142 * Flush all TLB mappings for this guest on this CPU, 1143 * it might have stale entries since vcpu has migrated 1144 * or vmm is restarted. | 1253 * Force new ASID allocation by invalidating the generation. |
1145 */ | 1254 */ |
1146 ctrl->tlb_ctrl = VMCB_TLB_FLUSH_GUEST; | 1255 vcpustate->asid.gen = 0; |
1147 | 1256 |
1148 /* Can't use any cached VMCB state by cpu.*/ 1149 ctrl->vmcb_clean = VMCB_CACHE_NONE; 1150 } else { | |
1151 /* | 1257 /* |
1152 * XXX: Using same ASID for all vcpus of a VM will cause TLB 1153 * corruption. This can easily be produced by muxing two vcpus 1154 * on same core. 1155 * For now, flush guest TLB for every vmrun. | 1258 * Invalidate the VMCB state cache by marking all fields dirty. |
1156 */ | 1259 */ |
1157 ctrl->tlb_ctrl = VMCB_TLB_FLUSH_GUEST; 1158 1159 /* 1160 * This is the same cpu on which vcpu last ran so don't 1161 * need to reload all VMCB state. 1162 * ASID is unique for a guest. 1163 * IOPM is unchanged. 1164 * RVI/EPT is unchanged. | 1260 vcpu_set_dirty(vcpustate, 0xffffffff); 1261 1262 /* 1263 * XXX 1264 * Setting 'vcpustate->lastcpu' here is bit premature because 1265 * we may return from this function without actually executing 1266 * the VMRUN instruction. This could happen if a rendezvous 1267 * or an AST is pending on the first time through the loop. |
1165 * | 1268 * |
1269 * This works for now but any new side-effects of vcpu 1270 * migration should take this case into account. |
|
1166 */ | 1271 */ |
1167 ctrl->vmcb_clean = VMCB_CACHE_ASID | 1168 VMCB_CACHE_IOPM | 1169 VMCB_CACHE_NP; | 1272 vcpustate->lastcpu = thiscpu; 1273 vmm_stat_incr(vm, vcpu, VCPU_MIGRATIONS, 1); |
1170 } 1171 | 1274 } 1275 |
1172 vcpustate->lastcpu = curcpu; | |
1173 VCPU_CTR3(vm, vcpu, "SVM:Enter vmrun RIP:0x%lx" 1174 " inst len=%d/%d\n", 1175 rip, vmexit->inst_length, 1176 vmexit->u.inst_emul.vie.num_valid); 1177 /* Update Guest RIP */ 1178 state->rip = rip; | 1276 VCPU_CTR3(vm, vcpu, "SVM:Enter vmrun RIP:0x%lx" 1277 " inst len=%d/%d\n", 1278 rip, vmexit->inst_length, 1279 vmexit->u.inst_emul.vie.num_valid); 1280 /* Update Guest RIP */ 1281 state->rip = rip; |
1179 | 1282 |
1180 do { 1181 vmexit->inst_length = 0; 1182 1183 /* 1184 * Disable global interrupts to guarantee atomicity during 1185 * loading of guest state. This includes not only the state 1186 * loaded by the "vmrun" instruction but also software state 1187 * maintained by the hypervisor: suspended and rendezvous --- 26 unchanged lines hidden (view full) --- 1214 VCPU_CTR1(vm, vcpu, 1215 "SVM: ASTPENDING, RIP:0x%lx\n", state->rip); 1216 vmexit->rip = state->rip; 1217 break; 1218 } 1219 1220 svm_inj_interrupts(svm_sc, vcpu, vlapic); 1221 | 1283 do { 1284 vmexit->inst_length = 0; 1285 1286 /* 1287 * Disable global interrupts to guarantee atomicity during 1288 * loading of guest state. This includes not only the state 1289 * loaded by the "vmrun" instruction but also software state 1290 * maintained by the hypervisor: suspended and rendezvous --- 26 unchanged lines hidden (view full) --- 1317 VCPU_CTR1(vm, vcpu, 1318 "SVM: ASTPENDING, RIP:0x%lx\n", state->rip); 1319 vmexit->rip = state->rip; 1320 break; 1321 } 1322 1323 svm_inj_interrupts(svm_sc, vcpu, vlapic); 1324 |
1325 /* Activate the nested pmap on 'thiscpu' */ 1326 CPU_SET_ATOMIC_ACQ(thiscpu, &pmap->pm_active); 1327 1328 /* 1329 * Check the pmap generation and the ASID generation to 1330 * ensure that the vcpu does not use stale TLB mappings. 1331 */ 1332 check_asid(svm_sc, vcpu, pmap, thiscpu); 1333 1334 ctrl->vmcb_clean = VMCB_CACHE_DEFAULT & ~vcpustate->dirty; 1335 vcpustate->dirty = 0; 1336 |
|
1222 /* Launch Virtual Machine. */ 1223 svm_launch(vmcb_pa, gctx, hctx); 1224 | 1337 /* Launch Virtual Machine. */ 1338 svm_launch(vmcb_pa, gctx, hctx); 1339 |
1340 CPU_CLR_ATOMIC(thiscpu, &pmap->pm_active); 1341 |
|
1225 /* 1226 * Restore MSR_GSBASE to point to the pcpu data area. 1227 * 1228 * Note that accesses done via PCPU_GET/PCPU_SET will work 1229 * only after MSR_GSBASE is restored. 1230 * 1231 * Also note that we don't bother restoring MSR_KGSBASE 1232 * since it is not used in the kernel and will be restored 1233 * when the VMRUN ioctl returns to userspace. 1234 */ | 1342 /* 1343 * Restore MSR_GSBASE to point to the pcpu data area. 1344 * 1345 * Note that accesses done via PCPU_GET/PCPU_SET will work 1346 * only after MSR_GSBASE is restored. 1347 * 1348 * Also note that we don't bother restoring MSR_KGSBASE 1349 * since it is not used in the kernel and will be restored 1350 * when the VMRUN ioctl returns to userspace. 1351 */ |
1235 wrmsr(MSR_GSBASE, (uint64_t)&__pcpu[vcpustate->lastcpu]); | 1352 wrmsr(MSR_GSBASE, (uint64_t)&__pcpu[thiscpu]); 1353 KASSERT(curcpu == thiscpu, ("thiscpu/curcpu (%u/%u) mismatch", 1354 thiscpu, curcpu)); |
1236 1237 /* 1238 * The host GDTR and IDTR is saved by VMRUN and restored 1239 * automatically on #VMEXIT. However, the host TSS needs 1240 * to be restored explicitly. 1241 */ 1242 restore_host_tss(); 1243 --- 114 unchanged lines hidden (view full) --- 1358 KASSERT(vcpu < svm_sc->vcpu_cnt, ("Guest doesn't have VCPU%d", vcpu)); 1359 1360 vmcb = svm_get_vmcb(svm_sc, vcpu); 1361 if (vmcb_write(vmcb, ident, val) == 0) { 1362 return (0); 1363 } 1364 1365 reg = swctx_regptr(svm_get_guest_regctx(svm_sc, vcpu), ident); | 1355 1356 /* 1357 * The host GDTR and IDTR is saved by VMRUN and restored 1358 * automatically on #VMEXIT. However, the host TSS needs 1359 * to be restored explicitly. 1360 */ 1361 restore_host_tss(); 1362 --- 114 unchanged lines hidden (view full) --- 1477 KASSERT(vcpu < svm_sc->vcpu_cnt, ("Guest doesn't have VCPU%d", vcpu)); 1478 1479 vmcb = svm_get_vmcb(svm_sc, vcpu); 1480 if (vmcb_write(vmcb, ident, val) == 0) { 1481 return (0); 1482 } 1483 1484 reg = swctx_regptr(svm_get_guest_regctx(svm_sc, vcpu), ident); |
1366 | 1485 |
1367 if (reg != NULL) { 1368 *reg = val; 1369 return (0); 1370 } 1371 | 1486 if (reg != NULL) { 1487 *reg = val; 1488 return (0); 1489 } 1490 |
1491 /* 1492 * XXX deal with CR3 and invalidate TLB entries tagged with the 1493 * vcpu's ASID. This needs to be treated differently depending on 1494 * whether 'running' is true/false. 1495 */ 1496 |
|
1372 ERR("SVM_ERR:reg type %x is not saved in VMCB.\n", ident); 1373 return (EINVAL); 1374} 1375 1376 1377/* 1378 * Inteface to set various descriptors. 1379 */ --- 216 unchanged lines hidden --- | 1497 ERR("SVM_ERR:reg type %x is not saved in VMCB.\n", ident); 1498 return (EINVAL); 1499} 1500 1501 1502/* 1503 * Inteface to set various descriptors. 1504 */ --- 216 unchanged lines hidden --- |