Lines Matching refs:cpudata
671 struct svm_cpudata *cpudata = vcpu->cpudata;
672 struct vmcb *vmcb = cpudata->vmcb;
676 cpudata->nmi_window_exit = true;
681 cpudata->int_window_exit = true;
690 struct svm_cpudata *cpudata = vcpu->cpudata;
691 struct vmcb *vmcb = cpudata->vmcb;
695 cpudata->nmi_window_exit = false;
700 cpudata->int_window_exit = false;
742 struct svm_cpudata *cpudata = vcpu->cpudata;
743 struct vmcb *vmcb = cpudata->vmcb;
785 cpudata->evt_pending = true;
848 svm_inkernel_exec_cpuid(struct svm_cpudata *cpudata, uint64_t eax, uint64_t ecx)
853 cpudata->vmcb->state.rax = descs[0];
854 cpudata->gprs[NVMM_X64_GPR_RBX] = descs[1];
855 cpudata->gprs[NVMM_X64_GPR_RCX] = descs[2];
856 cpudata->gprs[NVMM_X64_GPR_RDX] = descs[3];
862 struct svm_cpudata *cpudata = vcpu->cpudata;
868 svm_inkernel_exec_cpuid(cpudata, eax, ecx);
873 svm_inkernel_exec_cpuid(cpudata, eax, ecx);
878 svm_inkernel_exec_cpuid(cpudata, eax, ecx);
884 cpudata->vmcb->state.rax = svm_cpuid_max_basic;
887 cpudata->vmcb->state.rax &= nvmm_cpuid_00000001.eax;
889 cpudata->gprs[NVMM_X64_GPR_RBX] &= ~CPUID_LOCAL_APIC_ID;
890 cpudata->gprs[NVMM_X64_GPR_RBX] |= __SHIFTIN(vcpu->cpuid,
893 cpudata->gprs[NVMM_X64_GPR_RCX] &= nvmm_cpuid_00000001.ecx;
894 cpudata->gprs[NVMM_X64_GPR_RCX] |= CPUID2_RAZ;
896 cpudata->gprs[NVMM_X64_GPR_RDX] &= nvmm_cpuid_00000001.edx;
899 cr4 = cpudata->vmcb->state.cr4;
901 cpudata->gprs[NVMM_X64_GPR_RCX] &= ~CPUID2_OSXSAVE;
909 cpudata->vmcb->state.rax = 0;
910 cpudata->gprs[NVMM_X64_GPR_RBX] = 0;
911 cpudata->gprs[NVMM_X64_GPR_RCX] = 0;
912 cpudata->gprs[NVMM_X64_GPR_RDX] = 0;
917 cpudata->vmcb->state.rax = 0;
918 cpudata->gprs[NVMM_X64_GPR_RBX] &= nvmm_cpuid_00000007.ebx;
919 cpudata->gprs[NVMM_X64_GPR_RCX] &= nvmm_cpuid_00000007.ecx;
920 cpudata->gprs[NVMM_X64_GPR_RDX] &= nvmm_cpuid_00000007.edx;
923 cpudata->vmcb->state.rax = 0;
924 cpudata->gprs[NVMM_X64_GPR_RBX] = 0;
925 cpudata->gprs[NVMM_X64_GPR_RCX] = 0;
926 cpudata->gprs[NVMM_X64_GPR_RDX] = 0;
935 cpudata->vmcb->state.rax = 0;
936 cpudata->gprs[NVMM_X64_GPR_RBX] = 0;
937 cpudata->gprs[NVMM_X64_GPR_RCX] = 0;
938 cpudata->gprs[NVMM_X64_GPR_RDX] = 0;
946 cpudata->vmcb->state.rax = svm_xcr0_mask & 0xFFFFFFFF;
947 if (cpudata->gxcr0 & XCR0_SSE) {
948 cpudata->gprs[NVMM_X64_GPR_RBX] = sizeof(struct fxsave);
950 cpudata->gprs[NVMM_X64_GPR_RBX] = sizeof(struct save87);
952 cpudata->gprs[NVMM_X64_GPR_RBX] += 64; /* XSAVE header */
953 cpudata->gprs[NVMM_X64_GPR_RCX] = sizeof(struct fxsave) + 64;
954 cpudata->gprs[NVMM_X64_GPR_RDX] = svm_xcr0_mask >> 32;
957 cpudata->vmcb->state.rax &=
960 cpudata->gprs[NVMM_X64_GPR_RBX] = 0;
961 cpudata->gprs[NVMM_X64_GPR_RCX] = 0;
962 cpudata->gprs[NVMM_X64_GPR_RDX] = 0;
965 cpudata->vmcb->state.rax = 0;
966 cpudata->gprs[NVMM_X64_GPR_RBX] = 0;
967 cpudata->gprs[NVMM_X64_GPR_RCX] = 0;
968 cpudata->gprs[NVMM_X64_GPR_RDX] = 0;
974 cpudata->vmcb->state.rax = SVM_CPUID_MAX_HYPERVISOR;
975 cpudata->gprs[NVMM_X64_GPR_RBX] = 0;
976 cpudata->gprs[NVMM_X64_GPR_RCX] = 0;
977 cpudata->gprs[NVMM_X64_GPR_RDX] = 0;
978 memcpy(&cpudata->gprs[NVMM_X64_GPR_RBX], "___ ", 4);
979 memcpy(&cpudata->gprs[NVMM_X64_GPR_RCX], "NVMM", 4);
980 memcpy(&cpudata->gprs[NVMM_X64_GPR_RDX], " ___", 4);
984 cpudata->vmcb->state.rax = svm_cpuid_max_extended;
987 cpudata->vmcb->state.rax &= nvmm_cpuid_80000001.eax;
988 cpudata->gprs[NVMM_X64_GPR_RBX] &= nvmm_cpuid_80000001.ebx;
989 cpudata->gprs[NVMM_X64_GPR_RCX] &= nvmm_cpuid_80000001.ecx;
990 cpudata->gprs[NVMM_X64_GPR_RDX] &= nvmm_cpuid_80000001.edx;
999 cpudata->vmcb->state.rax &= nvmm_cpuid_80000007.eax;
1000 cpudata->gprs[NVMM_X64_GPR_RBX] &= nvmm_cpuid_80000007.ebx;
1001 cpudata->gprs[NVMM_X64_GPR_RCX] &= nvmm_cpuid_80000007.ecx;
1002 cpudata->gprs[NVMM_X64_GPR_RDX] &= nvmm_cpuid_80000007.edx;
1005 cpudata->vmcb->state.rax &= nvmm_cpuid_80000008.eax;
1006 cpudata->gprs[NVMM_X64_GPR_RBX] &= nvmm_cpuid_80000008.ebx;
1007 cpudata->gprs[NVMM_X64_GPR_RCX] &= nvmm_cpuid_80000008.ecx;
1008 cpudata->gprs[NVMM_X64_GPR_RDX] &= nvmm_cpuid_80000008.edx;
1026 cpudata->vmcb->state.rax = 0;
1027 cpudata->gprs[NVMM_X64_GPR_RBX] = 0;
1028 cpudata->gprs[NVMM_X64_GPR_RCX] = 0;
1029 cpudata->gprs[NVMM_X64_GPR_RDX] = 0;
1036 cpudata->vmcb->state.rax = 0;
1037 cpudata->gprs[NVMM_X64_GPR_RBX] = 0;
1038 cpudata->gprs[NVMM_X64_GPR_RCX] = 0;
1039 cpudata->gprs[NVMM_X64_GPR_RDX] = 0;
1045 cpudata->vmcb->state.rax = 0;
1046 cpudata->gprs[NVMM_X64_GPR_RBX] = 0;
1047 cpudata->gprs[NVMM_X64_GPR_RCX] = 0;
1048 cpudata->gprs[NVMM_X64_GPR_RDX] = 0;
1067 struct svm_cpudata *cpudata = vcpu->cpudata;
1072 eax = cpudata->vmcb->state.rax;
1073 ecx = cpudata->gprs[NVMM_X64_GPR_RCX];
1074 svm_inkernel_exec_cpuid(cpudata, eax, ecx);
1078 if (!cpudata->cpuidpresent[i]) {
1081 cpuid = &cpudata->cpuid[i];
1087 svm_exit_insn(cpudata->vmcb, exit, NVMM_VCPU_EXIT_CPUID);
1093 cpudata->vmcb->state.rax &= ~cpuid->u.mask.del.eax;
1094 cpudata->gprs[NVMM_X64_GPR_RBX] &= ~cpuid->u.mask.del.ebx;
1095 cpudata->gprs[NVMM_X64_GPR_RCX] &= ~cpuid->u.mask.del.ecx;
1096 cpudata->gprs[NVMM_X64_GPR_RDX] &= ~cpuid->u.mask.del.edx;
1099 cpudata->vmcb->state.rax |= cpuid->u.mask.set.eax;
1100 cpudata->gprs[NVMM_X64_GPR_RBX] |= cpuid->u.mask.set.ebx;
1101 cpudata->gprs[NVMM_X64_GPR_RCX] |= cpuid->u.mask.set.ecx;
1102 cpudata->gprs[NVMM_X64_GPR_RDX] |= cpuid->u.mask.set.edx;
1107 svm_inkernel_advance(cpudata->vmcb);
1115 struct svm_cpudata *cpudata = vcpu->cpudata;
1116 struct vmcb *vmcb = cpudata->vmcb;
1118 if (cpudata->int_window_exit && (vmcb->state.rflags & PSL_I)) {
1122 svm_inkernel_advance(cpudata->vmcb);
1142 struct svm_cpudata *cpudata = vcpu->cpudata;
1143 uint64_t info = cpudata->vmcb->ctrl.exitinfo1;
1144 uint64_t nextpc = cpudata->vmcb->ctrl.exitinfo2;
1194 struct svm_cpudata *cpudata = vcpu->cpudata;
1195 struct vmcb *vmcb = cpudata->vmcb;
1203 cpudata->gprs[NVMM_X64_GPR_RDX] = (val >> 32);
1209 cpudata->gprs[NVMM_X64_GPR_RDX] = (val >> 32);
1217 cpudata->gprs[NVMM_X64_GPR_RDX] = (val >> 32);
1227 cpudata->gtlb_want_flush = true;
1234 cpudata->gtsc = exit->u.wrmsr.val;
1235 cpudata->gtsc_want_update = true;
1248 svm_inkernel_advance(cpudata->vmcb);
1260 struct svm_cpudata *cpudata = vcpu->cpudata;
1263 exit->u.rdmsr.msr = (cpudata->gprs[NVMM_X64_GPR_RCX] & 0xFFFFFFFF);
1264 exit->u.rdmsr.npc = cpudata->vmcb->ctrl.nrip;
1278 struct svm_cpudata *cpudata = vcpu->cpudata;
1281 rdx = cpudata->gprs[NVMM_X64_GPR_RDX];
1282 rax = cpudata->vmcb->state.rax;
1285 exit->u.wrmsr.msr = (cpudata->gprs[NVMM_X64_GPR_RCX] & 0xFFFFFFFF);
1287 exit->u.wrmsr.npc = cpudata->vmcb->ctrl.nrip;
1301 struct svm_cpudata *cpudata = vcpu->cpudata;
1302 uint64_t info = cpudata->vmcb->ctrl.exitinfo1;
1315 struct svm_cpudata *cpudata = vcpu->cpudata;
1316 gpaddr_t gpa = cpudata->vmcb->ctrl.exitinfo2;
1319 if (cpudata->vmcb->ctrl.exitinfo1 & PGEX_W)
1321 else if (cpudata->vmcb->ctrl.exitinfo1 & PGEX_I)
1326 exit->u.mem.inst_len = cpudata->vmcb->ctrl.inst_len;
1327 memcpy(exit->u.mem.inst_bytes, cpudata->vmcb->ctrl.inst_bytes,
1339 struct svm_cpudata *cpudata = vcpu->cpudata;
1340 struct vmcb *vmcb = cpudata->vmcb;
1345 val = (cpudata->gprs[NVMM_X64_GPR_RDX] << 32) |
1348 if (__predict_false(cpudata->gprs[NVMM_X64_GPR_RCX] != 0)) {
1358 cpudata->gxcr0 = val;
1360 svm_inkernel_advance(cpudata->vmcb);
1379 struct svm_cpudata *cpudata = vcpu->cpudata;
1383 fpu_area_restore(&cpudata->gfpu, svm_xcr0_mask, false);
1386 cpudata->hxcr0 = rdxcr(0);
1387 wrxcr(0, cpudata->gxcr0);
1394 struct svm_cpudata *cpudata = vcpu->cpudata;
1397 cpudata->gxcr0 = rdxcr(0);
1398 wrxcr(0, cpudata->hxcr0);
1402 fpu_area_save(&cpudata->gfpu, svm_xcr0_mask, false);
1409 struct svm_cpudata *cpudata = vcpu->cpudata;
1415 ldr0(cpudata->drs[NVMM_X64_DR_DR0]);
1416 ldr1(cpudata->drs[NVMM_X64_DR_DR1]);
1417 ldr2(cpudata->drs[NVMM_X64_DR_DR2]);
1418 ldr3(cpudata->drs[NVMM_X64_DR_DR3]);
1424 struct svm_cpudata *cpudata = vcpu->cpudata;
1426 cpudata->drs[NVMM_X64_DR_DR0] = rdr0();
1427 cpudata->drs[NVMM_X64_DR_DR1] = rdr1();
1428 cpudata->drs[NVMM_X64_DR_DR2] = rdr2();
1429 cpudata->drs[NVMM_X64_DR_DR3] = rdr3();
1437 struct svm_cpudata *cpudata = vcpu->cpudata;
1439 cpudata->fsbase = rdmsr(MSR_FSBASE);
1440 cpudata->kernelgsbase = rdmsr(MSR_KERNELGSBASE);
1446 struct svm_cpudata *cpudata = vcpu->cpudata;
1448 wrmsr(MSR_STAR, cpudata->star);
1449 wrmsr(MSR_LSTAR, cpudata->lstar);
1450 wrmsr(MSR_CSTAR, cpudata->cstar);
1451 wrmsr(MSR_SFMASK, cpudata->sfmask);
1452 wrmsr(MSR_FSBASE, cpudata->fsbase);
1453 wrmsr(MSR_KERNELGSBASE, cpudata->kernelgsbase);
1461 struct svm_cpudata *cpudata = vcpu->cpudata;
1463 if (vcpu->hcpu_last != hcpu || cpudata->shared_asid) {
1464 cpudata->gtlb_want_flush = true;
1480 svm_htlb_flush(struct svm_machdata *machdata, struct svm_cpudata *cpudata)
1482 struct vmcb *vmcb = cpudata->vmcb;
1486 if (__predict_true(machgen == cpudata->vcpu_htlb_gen)) {
1495 svm_htlb_flush_ack(struct svm_cpudata *cpudata, uint64_t machgen)
1497 struct vmcb *vmcb = cpudata->vmcb;
1500 cpudata->vcpu_htlb_gen = machgen;
1505 svm_exit_evt(struct svm_cpudata *cpudata, struct vmcb *vmcb)
1507 cpudata->evt_pending = false;
1511 cpudata->evt_pending = true;
1521 struct svm_cpudata *cpudata = vcpu->cpudata;
1522 struct vmcb *vmcb = cpudata->vmcb;
1541 cpudata->gtsc_want_update = true;
1548 if (cpudata->gtlb_want_flush) {
1554 if (__predict_false(cpudata->gtsc_want_update)) {
1555 vmcb->ctrl.tsc_offset = cpudata->gtsc - rdtsc();
1561 machgen = svm_htlb_flush(machdata, cpudata);
1562 svm_vmrun(cpudata->vmcb_pa, cpudata->gprs);
1563 svm_htlb_flush_ack(cpudata, machgen);
1570 cpudata->gtlb_want_flush = false;
1571 cpudata->gtsc_want_update = false;
1574 svm_exit_evt(cpudata, vmcb);
1651 cpudata->gtsc = rdtsc() + vmcb->ctrl.tsc_offset;
1662 exit->exitstate.int_window_exiting = cpudata->int_window_exit;
1663 exit->exitstate.nmi_window_exiting = cpudata->nmi_window_exit;
1664 exit->exitstate.evt_pending = cpudata->evt_pending;
1832 struct svm_cpudata *cpudata = vcpu->cpudata;
1833 struct vmcb *vmcb = cpudata->vmcb;
1840 cpudata->gtlb_want_flush = true;
1868 CTASSERT(sizeof(cpudata->gprs) == sizeof(state->gprs));
1870 memcpy(cpudata->gprs, state->gprs, sizeof(state->gprs));
1891 cpudata->gxcr0 = state->crs[NVMM_X64_CR_XCR0];
1892 cpudata->gxcr0 &= svm_xcr0_mask;
1893 cpudata->gxcr0 |= XCR0_X87;
1897 CTASSERT(sizeof(cpudata->drs) == sizeof(state->drs));
1899 memcpy(cpudata->drs, state->drs, sizeof(state->drs));
1924 cpudata->gtsc = state->msrs[NVMM_X64_MSR_TSC];
1925 cpudata->gtsc_want_update = true;
1948 CTASSERT(sizeof(cpudata->gfpu.xsh_fxsave) == sizeof(state->fpu));
1950 memcpy(cpudata->gfpu.xsh_fxsave, &state->fpu,
1953 fpustate = (struct fxsave *)cpudata->gfpu.xsh_fxsave;
1959 cpudata->gfpu.xsh_xstate_bv = svm_xcr0_mask;
1974 struct svm_cpudata *cpudata = vcpu->cpudata;
1975 struct vmcb *vmcb = cpudata->vmcb;
2005 CTASSERT(sizeof(cpudata->gprs) == sizeof(state->gprs));
2007 memcpy(state->gprs, cpudata->gprs, sizeof(state->gprs));
2022 state->crs[NVMM_X64_CR_XCR0] = cpudata->gxcr0;
2025 CTASSERT(sizeof(cpudata->drs) == sizeof(state->drs));
2027 memcpy(state->drs, cpudata->drs, sizeof(state->drs));
2048 state->msrs[NVMM_X64_MSR_TSC] = cpudata->gtsc;
2057 state->intr.int_window_exiting = cpudata->int_window_exit;
2058 state->intr.nmi_window_exiting = cpudata->nmi_window_exit;
2059 state->intr.evt_pending = cpudata->evt_pending;
2062 CTASSERT(sizeof(cpudata->gfpu.xsh_fxsave) == sizeof(state->fpu));
2064 memcpy(&state->fpu, cpudata->gfpu.xsh_fxsave,
2092 struct svm_cpudata *cpudata = vcpu->cpudata;
2093 struct vmcb *vmcb = cpudata->vmcb;
2116 cpudata->shared_asid = true;
2124 struct svm_cpudata *cpudata = vcpu->cpudata;
2125 struct vmcb *vmcb = cpudata->vmcb;
2128 if (cpudata->shared_asid) {
2143 struct svm_cpudata *cpudata = vcpu->cpudata;
2144 struct vmcb *vmcb = cpudata->vmcb;
2226 memset(cpudata->iobm, 0xFF, IOBM_SIZE);
2227 vmcb->ctrl.iopm_base_pa = cpudata->iobm_pa;
2230 memset(cpudata->msrbm, 0xFF, MSRBM_SIZE);
2231 svm_vcpu_msr_allow(cpudata->msrbm, MSR_STAR, true, true);
2232 svm_vcpu_msr_allow(cpudata->msrbm, MSR_LSTAR, true, true);
2233 svm_vcpu_msr_allow(cpudata->msrbm, MSR_CSTAR, true, true);
2234 svm_vcpu_msr_allow(cpudata->msrbm, MSR_SFMASK, true, true);
2235 svm_vcpu_msr_allow(cpudata->msrbm, MSR_KERNELGSBASE, true, true);
2236 svm_vcpu_msr_allow(cpudata->msrbm, MSR_SYSENTER_CS, true, true);
2237 svm_vcpu_msr_allow(cpudata->msrbm, MSR_SYSENTER_ESP, true, true);
2238 svm_vcpu_msr_allow(cpudata->msrbm, MSR_SYSENTER_EIP, true, true);
2239 svm_vcpu_msr_allow(cpudata->msrbm, MSR_FSBASE, true, true);
2240 svm_vcpu_msr_allow(cpudata->msrbm, MSR_GSBASE, true, true);
2241 svm_vcpu_msr_allow(cpudata->msrbm, MSR_CR_PAT, true, true);
2242 svm_vcpu_msr_allow(cpudata->msrbm, MSR_TSC, true, false);
2243 vmcb->ctrl.msrpm_base_pa = cpudata->msrbm_pa;
2256 cpudata->gfpu.xsh_xstate_bv = svm_xcr0_mask;
2257 cpudata->gfpu.xsh_xcomp_bv = 0;
2260 cpudata->star = rdmsr(MSR_STAR);
2261 cpudata->lstar = rdmsr(MSR_LSTAR);
2262 cpudata->cstar = rdmsr(MSR_CSTAR);
2263 cpudata->sfmask = rdmsr(MSR_SFMASK);
2276 struct svm_cpudata *cpudata;
2279 /* Allocate the SVM cpudata. */
2280 cpudata = (struct svm_cpudata *)uvm_km_alloc(kernel_map,
2281 roundup(sizeof(*cpudata), PAGE_SIZE), 0,
2283 vcpu->cpudata = cpudata;
2286 error = svm_memalloc(&cpudata->vmcb_pa, (vaddr_t *)&cpudata->vmcb,
2292 error = svm_memalloc(&cpudata->iobm_pa, (vaddr_t *)&cpudata->iobm,
2298 error = svm_memalloc(&cpudata->msrbm_pa, (vaddr_t *)&cpudata->msrbm,
2309 if (cpudata->vmcb_pa) {
2310 svm_memfree(cpudata->vmcb_pa, (vaddr_t)cpudata->vmcb,
2313 if (cpudata->iobm_pa) {
2314 svm_memfree(cpudata->iobm_pa, (vaddr_t)cpudata->iobm,
2317 if (cpudata->msrbm_pa) {
2318 svm_memfree(cpudata->msrbm_pa, (vaddr_t)cpudata->msrbm,
2321 uvm_km_free(kernel_map, (vaddr_t)cpudata,
2322 roundup(sizeof(*cpudata), PAGE_SIZE), UVM_KMF_WIRED);
2329 struct svm_cpudata *cpudata = vcpu->cpudata;
2333 svm_memfree(cpudata->vmcb_pa, (vaddr_t)cpudata->vmcb, VMCB_NPAGES);
2334 svm_memfree(cpudata->iobm_pa, (vaddr_t)cpudata->iobm, IOBM_NPAGES);
2335 svm_memfree(cpudata->msrbm_pa, (vaddr_t)cpudata->msrbm, MSRBM_NPAGES);
2337 uvm_km_free(kernel_map, (vaddr_t)cpudata,
2338 roundup(sizeof(*cpudata), PAGE_SIZE), UVM_KMF_WIRED);
2344 svm_vcpu_configure_cpuid(struct svm_cpudata *cpudata, void *data)
2363 if (!cpudata->cpuidpresent[i]) {
2366 if (cpudata->cpuid[i].leaf == cpuid->leaf) {
2367 cpudata->cpuidpresent[i] = false;
2375 if (!cpudata->cpuidpresent[i]) {
2378 if (cpudata->cpuid[i].leaf == cpuid->leaf) {
2379 memcpy(&cpudata->cpuid[i], cpuid,
2387 if (!cpudata->cpuidpresent[i]) {
2388 cpudata->cpuidpresent[i] = true;
2389 memcpy(&cpudata->cpuid[i], cpuid,
2401 struct svm_cpudata *cpudata = vcpu->cpudata;
2405 return svm_vcpu_configure_cpuid(cpudata, data);