nvmm_x86_svm.c revision 1.82
1/* $NetBSD: nvmm_x86_svm.c,v 1.82 2020/10/24 07:14:30 mgorny Exp $ */ 2 3/* 4 * Copyright (c) 2018-2020 Maxime Villard, m00nbsd.net 5 * All rights reserved. 6 * 7 * This code is part of the NVMM hypervisor. 8 * 9 * Redistribution and use in source and binary forms, with or without 10 * modification, are permitted provided that the following conditions 11 * are met: 12 * 1. Redistributions of source code must retain the above copyright 13 * notice, this list of conditions and the following disclaimer. 14 * 2. Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in the 16 * documentation and/or other materials provided with the distribution. 17 * 18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 19 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 20 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 21 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 22 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 23 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 24 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED 25 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 26 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 28 * SUCH DAMAGE. 29 */ 30 31#include <sys/cdefs.h> 32__KERNEL_RCSID(0, "$NetBSD: nvmm_x86_svm.c,v 1.82 2020/10/24 07:14:30 mgorny Exp $"); 33 34#include <sys/param.h> 35#include <sys/systm.h> 36#include <sys/kernel.h> 37#include <sys/kmem.h> 38#include <sys/cpu.h> 39#include <sys/xcall.h> 40#include <sys/mman.h> 41 42#include <uvm/uvm_extern.h> 43#include <uvm/uvm_page.h> 44 45#include <x86/cputypes.h> 46#include <x86/specialreg.h> 47#include <x86/dbregs.h> 48#include <x86/cpu_counter.h> 49 50#include <machine/cpuvar.h> 51 52#include <dev/nvmm/nvmm.h> 53#include <dev/nvmm/nvmm_internal.h> 54#include <dev/nvmm/x86/nvmm_x86.h> 55 56int svm_vmrun(paddr_t, uint64_t *); 57 58static inline void 59svm_clgi(void) 60{ 61 asm volatile ("clgi" ::: "memory"); 62} 63 64static inline void 65svm_stgi(void) 66{ 67 asm volatile ("stgi" ::: "memory"); 68} 69 70#define MSR_VM_HSAVE_PA 0xC0010117 71 72/* -------------------------------------------------------------------------- */ 73 74#define VMCB_EXITCODE_CR0_READ 0x0000 75#define VMCB_EXITCODE_CR1_READ 0x0001 76#define VMCB_EXITCODE_CR2_READ 0x0002 77#define VMCB_EXITCODE_CR3_READ 0x0003 78#define VMCB_EXITCODE_CR4_READ 0x0004 79#define VMCB_EXITCODE_CR5_READ 0x0005 80#define VMCB_EXITCODE_CR6_READ 0x0006 81#define VMCB_EXITCODE_CR7_READ 0x0007 82#define VMCB_EXITCODE_CR8_READ 0x0008 83#define VMCB_EXITCODE_CR9_READ 0x0009 84#define VMCB_EXITCODE_CR10_READ 0x000A 85#define VMCB_EXITCODE_CR11_READ 0x000B 86#define VMCB_EXITCODE_CR12_READ 0x000C 87#define VMCB_EXITCODE_CR13_READ 0x000D 88#define VMCB_EXITCODE_CR14_READ 0x000E 89#define VMCB_EXITCODE_CR15_READ 0x000F 90#define VMCB_EXITCODE_CR0_WRITE 0x0010 91#define VMCB_EXITCODE_CR1_WRITE 0x0011 92#define VMCB_EXITCODE_CR2_WRITE 0x0012 93#define VMCB_EXITCODE_CR3_WRITE 0x0013 94#define VMCB_EXITCODE_CR4_WRITE 0x0014 95#define VMCB_EXITCODE_CR5_WRITE 0x0015 96#define VMCB_EXITCODE_CR6_WRITE 0x0016 97#define VMCB_EXITCODE_CR7_WRITE 0x0017 98#define VMCB_EXITCODE_CR8_WRITE 0x0018 99#define VMCB_EXITCODE_CR9_WRITE 0x0019 100#define VMCB_EXITCODE_CR10_WRITE 0x001A 101#define VMCB_EXITCODE_CR11_WRITE 0x001B 102#define VMCB_EXITCODE_CR12_WRITE 0x001C 103#define VMCB_EXITCODE_CR13_WRITE 0x001D 104#define VMCB_EXITCODE_CR14_WRITE 0x001E 105#define VMCB_EXITCODE_CR15_WRITE 0x001F 106#define VMCB_EXITCODE_DR0_READ 0x0020 107#define VMCB_EXITCODE_DR1_READ 0x0021 108#define VMCB_EXITCODE_DR2_READ 0x0022 109#define VMCB_EXITCODE_DR3_READ 0x0023 110#define VMCB_EXITCODE_DR4_READ 0x0024 111#define VMCB_EXITCODE_DR5_READ 0x0025 112#define VMCB_EXITCODE_DR6_READ 0x0026 113#define VMCB_EXITCODE_DR7_READ 0x0027 114#define VMCB_EXITCODE_DR8_READ 0x0028 115#define VMCB_EXITCODE_DR9_READ 0x0029 116#define VMCB_EXITCODE_DR10_READ 0x002A 117#define VMCB_EXITCODE_DR11_READ 0x002B 118#define VMCB_EXITCODE_DR12_READ 0x002C 119#define VMCB_EXITCODE_DR13_READ 0x002D 120#define VMCB_EXITCODE_DR14_READ 0x002E 121#define VMCB_EXITCODE_DR15_READ 0x002F 122#define VMCB_EXITCODE_DR0_WRITE 0x0030 123#define VMCB_EXITCODE_DR1_WRITE 0x0031 124#define VMCB_EXITCODE_DR2_WRITE 0x0032 125#define VMCB_EXITCODE_DR3_WRITE 0x0033 126#define VMCB_EXITCODE_DR4_WRITE 0x0034 127#define VMCB_EXITCODE_DR5_WRITE 0x0035 128#define VMCB_EXITCODE_DR6_WRITE 0x0036 129#define VMCB_EXITCODE_DR7_WRITE 0x0037 130#define VMCB_EXITCODE_DR8_WRITE 0x0038 131#define VMCB_EXITCODE_DR9_WRITE 0x0039 132#define VMCB_EXITCODE_DR10_WRITE 0x003A 133#define VMCB_EXITCODE_DR11_WRITE 0x003B 134#define VMCB_EXITCODE_DR12_WRITE 0x003C 135#define VMCB_EXITCODE_DR13_WRITE 0x003D 136#define VMCB_EXITCODE_DR14_WRITE 0x003E 137#define VMCB_EXITCODE_DR15_WRITE 0x003F 138#define VMCB_EXITCODE_EXCP0 0x0040 139#define VMCB_EXITCODE_EXCP1 0x0041 140#define VMCB_EXITCODE_EXCP2 0x0042 141#define VMCB_EXITCODE_EXCP3 0x0043 142#define VMCB_EXITCODE_EXCP4 0x0044 143#define VMCB_EXITCODE_EXCP5 0x0045 144#define VMCB_EXITCODE_EXCP6 0x0046 145#define VMCB_EXITCODE_EXCP7 0x0047 146#define VMCB_EXITCODE_EXCP8 0x0048 147#define VMCB_EXITCODE_EXCP9 0x0049 148#define VMCB_EXITCODE_EXCP10 0x004A 149#define VMCB_EXITCODE_EXCP11 0x004B 150#define VMCB_EXITCODE_EXCP12 0x004C 151#define VMCB_EXITCODE_EXCP13 0x004D 152#define VMCB_EXITCODE_EXCP14 0x004E 153#define VMCB_EXITCODE_EXCP15 0x004F 154#define VMCB_EXITCODE_EXCP16 0x0050 155#define VMCB_EXITCODE_EXCP17 0x0051 156#define VMCB_EXITCODE_EXCP18 0x0052 157#define VMCB_EXITCODE_EXCP19 0x0053 158#define VMCB_EXITCODE_EXCP20 0x0054 159#define VMCB_EXITCODE_EXCP21 0x0055 160#define VMCB_EXITCODE_EXCP22 0x0056 161#define VMCB_EXITCODE_EXCP23 0x0057 162#define VMCB_EXITCODE_EXCP24 0x0058 163#define VMCB_EXITCODE_EXCP25 0x0059 164#define VMCB_EXITCODE_EXCP26 0x005A 165#define VMCB_EXITCODE_EXCP27 0x005B 166#define VMCB_EXITCODE_EXCP28 0x005C 167#define VMCB_EXITCODE_EXCP29 0x005D 168#define VMCB_EXITCODE_EXCP30 0x005E 169#define VMCB_EXITCODE_EXCP31 0x005F 170#define VMCB_EXITCODE_INTR 0x0060 171#define VMCB_EXITCODE_NMI 0x0061 172#define VMCB_EXITCODE_SMI 0x0062 173#define VMCB_EXITCODE_INIT 0x0063 174#define VMCB_EXITCODE_VINTR 0x0064 175#define VMCB_EXITCODE_CR0_SEL_WRITE 0x0065 176#define VMCB_EXITCODE_IDTR_READ 0x0066 177#define VMCB_EXITCODE_GDTR_READ 0x0067 178#define VMCB_EXITCODE_LDTR_READ 0x0068 179#define VMCB_EXITCODE_TR_READ 0x0069 180#define VMCB_EXITCODE_IDTR_WRITE 0x006A 181#define VMCB_EXITCODE_GDTR_WRITE 0x006B 182#define VMCB_EXITCODE_LDTR_WRITE 0x006C 183#define VMCB_EXITCODE_TR_WRITE 0x006D 184#define VMCB_EXITCODE_RDTSC 0x006E 185#define VMCB_EXITCODE_RDPMC 0x006F 186#define VMCB_EXITCODE_PUSHF 0x0070 187#define VMCB_EXITCODE_POPF 0x0071 188#define VMCB_EXITCODE_CPUID 0x0072 189#define VMCB_EXITCODE_RSM 0x0073 190#define VMCB_EXITCODE_IRET 0x0074 191#define VMCB_EXITCODE_SWINT 0x0075 192#define VMCB_EXITCODE_INVD 0x0076 193#define VMCB_EXITCODE_PAUSE 0x0077 194#define VMCB_EXITCODE_HLT 0x0078 195#define VMCB_EXITCODE_INVLPG 0x0079 196#define VMCB_EXITCODE_INVLPGA 0x007A 197#define VMCB_EXITCODE_IOIO 0x007B 198#define VMCB_EXITCODE_MSR 0x007C 199#define VMCB_EXITCODE_TASK_SWITCH 0x007D 200#define VMCB_EXITCODE_FERR_FREEZE 0x007E 201#define VMCB_EXITCODE_SHUTDOWN 0x007F 202#define VMCB_EXITCODE_VMRUN 0x0080 203#define VMCB_EXITCODE_VMMCALL 0x0081 204#define VMCB_EXITCODE_VMLOAD 0x0082 205#define VMCB_EXITCODE_VMSAVE 0x0083 206#define VMCB_EXITCODE_STGI 0x0084 207#define VMCB_EXITCODE_CLGI 0x0085 208#define VMCB_EXITCODE_SKINIT 0x0086 209#define VMCB_EXITCODE_RDTSCP 0x0087 210#define VMCB_EXITCODE_ICEBP 0x0088 211#define VMCB_EXITCODE_WBINVD 0x0089 212#define VMCB_EXITCODE_MONITOR 0x008A 213#define VMCB_EXITCODE_MWAIT 0x008B 214#define VMCB_EXITCODE_MWAIT_CONDITIONAL 0x008C 215#define VMCB_EXITCODE_XSETBV 0x008D 216#define VMCB_EXITCODE_RDPRU 0x008E 217#define VMCB_EXITCODE_EFER_WRITE_TRAP 0x008F 218#define VMCB_EXITCODE_CR0_WRITE_TRAP 0x0090 219#define VMCB_EXITCODE_CR1_WRITE_TRAP 0x0091 220#define VMCB_EXITCODE_CR2_WRITE_TRAP 0x0092 221#define VMCB_EXITCODE_CR3_WRITE_TRAP 0x0093 222#define VMCB_EXITCODE_CR4_WRITE_TRAP 0x0094 223#define VMCB_EXITCODE_CR5_WRITE_TRAP 0x0095 224#define VMCB_EXITCODE_CR6_WRITE_TRAP 0x0096 225#define VMCB_EXITCODE_CR7_WRITE_TRAP 0x0097 226#define VMCB_EXITCODE_CR8_WRITE_TRAP 0x0098 227#define VMCB_EXITCODE_CR9_WRITE_TRAP 0x0099 228#define VMCB_EXITCODE_CR10_WRITE_TRAP 0x009A 229#define VMCB_EXITCODE_CR11_WRITE_TRAP 0x009B 230#define VMCB_EXITCODE_CR12_WRITE_TRAP 0x009C 231#define VMCB_EXITCODE_CR13_WRITE_TRAP 0x009D 232#define VMCB_EXITCODE_CR14_WRITE_TRAP 0x009E 233#define VMCB_EXITCODE_CR15_WRITE_TRAP 0x009F 234#define VMCB_EXITCODE_INVLPGB 0x00A0 235#define VMCB_EXITCODE_INVLPGB_ILLEGAL 0x00A1 236#define VMCB_EXITCODE_INVPCID 0x00A2 237#define VMCB_EXITCODE_MCOMMIT 0x00A3 238#define VMCB_EXITCODE_TLBSYNC 0x00A4 239#define VMCB_EXITCODE_NPF 0x0400 240#define VMCB_EXITCODE_AVIC_INCOMP_IPI 0x0401 241#define VMCB_EXITCODE_AVIC_NOACCEL 0x0402 242#define VMCB_EXITCODE_VMGEXIT 0x0403 243#define VMCB_EXITCODE_BUSY -2ULL 244#define VMCB_EXITCODE_INVALID -1ULL 245 246/* -------------------------------------------------------------------------- */ 247 248struct vmcb_ctrl { 249 uint32_t intercept_cr; 250#define VMCB_CTRL_INTERCEPT_RCR(x) __BIT( 0 + x) 251#define VMCB_CTRL_INTERCEPT_WCR(x) __BIT(16 + x) 252 253 uint32_t intercept_dr; 254#define VMCB_CTRL_INTERCEPT_RDR(x) __BIT( 0 + x) 255#define VMCB_CTRL_INTERCEPT_WDR(x) __BIT(16 + x) 256 257 uint32_t intercept_vec; 258#define VMCB_CTRL_INTERCEPT_VEC(x) __BIT(x) 259 260 uint32_t intercept_misc1; 261#define VMCB_CTRL_INTERCEPT_INTR __BIT(0) 262#define VMCB_CTRL_INTERCEPT_NMI __BIT(1) 263#define VMCB_CTRL_INTERCEPT_SMI __BIT(2) 264#define VMCB_CTRL_INTERCEPT_INIT __BIT(3) 265#define VMCB_CTRL_INTERCEPT_VINTR __BIT(4) 266#define VMCB_CTRL_INTERCEPT_CR0_SPEC __BIT(5) 267#define VMCB_CTRL_INTERCEPT_RIDTR __BIT(6) 268#define VMCB_CTRL_INTERCEPT_RGDTR __BIT(7) 269#define VMCB_CTRL_INTERCEPT_RLDTR __BIT(8) 270#define VMCB_CTRL_INTERCEPT_RTR __BIT(9) 271#define VMCB_CTRL_INTERCEPT_WIDTR __BIT(10) 272#define VMCB_CTRL_INTERCEPT_WGDTR __BIT(11) 273#define VMCB_CTRL_INTERCEPT_WLDTR __BIT(12) 274#define VMCB_CTRL_INTERCEPT_WTR __BIT(13) 275#define VMCB_CTRL_INTERCEPT_RDTSC __BIT(14) 276#define VMCB_CTRL_INTERCEPT_RDPMC __BIT(15) 277#define VMCB_CTRL_INTERCEPT_PUSHF __BIT(16) 278#define VMCB_CTRL_INTERCEPT_POPF __BIT(17) 279#define VMCB_CTRL_INTERCEPT_CPUID __BIT(18) 280#define VMCB_CTRL_INTERCEPT_RSM __BIT(19) 281#define VMCB_CTRL_INTERCEPT_IRET __BIT(20) 282#define VMCB_CTRL_INTERCEPT_INTN __BIT(21) 283#define VMCB_CTRL_INTERCEPT_INVD __BIT(22) 284#define VMCB_CTRL_INTERCEPT_PAUSE __BIT(23) 285#define VMCB_CTRL_INTERCEPT_HLT __BIT(24) 286#define VMCB_CTRL_INTERCEPT_INVLPG __BIT(25) 287#define VMCB_CTRL_INTERCEPT_INVLPGA __BIT(26) 288#define VMCB_CTRL_INTERCEPT_IOIO_PROT __BIT(27) 289#define VMCB_CTRL_INTERCEPT_MSR_PROT __BIT(28) 290#define VMCB_CTRL_INTERCEPT_TASKSW __BIT(29) 291#define VMCB_CTRL_INTERCEPT_FERR_FREEZE __BIT(30) 292#define VMCB_CTRL_INTERCEPT_SHUTDOWN __BIT(31) 293 294 uint32_t intercept_misc2; 295#define VMCB_CTRL_INTERCEPT_VMRUN __BIT(0) 296#define VMCB_CTRL_INTERCEPT_VMMCALL __BIT(1) 297#define VMCB_CTRL_INTERCEPT_VMLOAD __BIT(2) 298#define VMCB_CTRL_INTERCEPT_VMSAVE __BIT(3) 299#define VMCB_CTRL_INTERCEPT_STGI __BIT(4) 300#define VMCB_CTRL_INTERCEPT_CLGI __BIT(5) 301#define VMCB_CTRL_INTERCEPT_SKINIT __BIT(6) 302#define VMCB_CTRL_INTERCEPT_RDTSCP __BIT(7) 303#define VMCB_CTRL_INTERCEPT_ICEBP __BIT(8) 304#define VMCB_CTRL_INTERCEPT_WBINVD __BIT(9) 305#define VMCB_CTRL_INTERCEPT_MONITOR __BIT(10) 306#define VMCB_CTRL_INTERCEPT_MWAIT __BIT(11) 307#define VMCB_CTRL_INTERCEPT_MWAIT_ARMED __BIT(12) 308#define VMCB_CTRL_INTERCEPT_XSETBV __BIT(13) 309#define VMCB_CTRL_INTERCEPT_RDPRU __BIT(14) 310#define VMCB_CTRL_INTERCEPT_EFER_SPEC __BIT(15) 311#define VMCB_CTRL_INTERCEPT_WCR_SPEC(x) __BIT(16 + x) 312 313 uint32_t intercept_misc3; 314#define VMCB_CTRL_INTERCEPT_INVLPGB_ALL __BIT(0) 315#define VMCB_CTRL_INTERCEPT_INVLPGB_ILL __BIT(1) 316#define VMCB_CTRL_INTERCEPT_PCID __BIT(2) 317#define VMCB_CTRL_INTERCEPT_MCOMMIT __BIT(3) 318#define VMCB_CTRL_INTERCEPT_TLBSYNC __BIT(4) 319 320 uint8_t rsvd1[36]; 321 uint16_t pause_filt_thresh; 322 uint16_t pause_filt_cnt; 323 uint64_t iopm_base_pa; 324 uint64_t msrpm_base_pa; 325 uint64_t tsc_offset; 326 uint32_t guest_asid; 327 328 uint32_t tlb_ctrl; 329#define VMCB_CTRL_TLB_CTRL_FLUSH_ALL 0x01 330#define VMCB_CTRL_TLB_CTRL_FLUSH_GUEST 0x03 331#define VMCB_CTRL_TLB_CTRL_FLUSH_GUEST_NONGLOBAL 0x07 332 333 uint64_t v; 334#define VMCB_CTRL_V_TPR __BITS(3,0) 335#define VMCB_CTRL_V_IRQ __BIT(8) 336#define VMCB_CTRL_V_VGIF __BIT(9) 337#define VMCB_CTRL_V_INTR_PRIO __BITS(19,16) 338#define VMCB_CTRL_V_IGN_TPR __BIT(20) 339#define VMCB_CTRL_V_INTR_MASKING __BIT(24) 340#define VMCB_CTRL_V_GUEST_VGIF __BIT(25) 341#define VMCB_CTRL_V_AVIC_EN __BIT(31) 342#define VMCB_CTRL_V_INTR_VECTOR __BITS(39,32) 343 344 uint64_t intr; 345#define VMCB_CTRL_INTR_SHADOW __BIT(0) 346#define VMCB_CTRL_INTR_MASK __BIT(1) 347 348 uint64_t exitcode; 349 uint64_t exitinfo1; 350 uint64_t exitinfo2; 351 352 uint64_t exitintinfo; 353#define VMCB_CTRL_EXITINTINFO_VECTOR __BITS(7,0) 354#define VMCB_CTRL_EXITINTINFO_TYPE __BITS(10,8) 355#define VMCB_CTRL_EXITINTINFO_EV __BIT(11) 356#define VMCB_CTRL_EXITINTINFO_V __BIT(31) 357#define VMCB_CTRL_EXITINTINFO_ERRORCODE __BITS(63,32) 358 359 uint64_t enable1; 360#define VMCB_CTRL_ENABLE_NP __BIT(0) 361#define VMCB_CTRL_ENABLE_SEV __BIT(1) 362#define VMCB_CTRL_ENABLE_ES_SEV __BIT(2) 363#define VMCB_CTRL_ENABLE_GMET __BIT(3) 364#define VMCB_CTRL_ENABLE_VTE __BIT(5) 365 366 uint64_t avic; 367#define VMCB_CTRL_AVIC_APIC_BAR __BITS(51,0) 368 369 uint64_t ghcb; 370 371 uint64_t eventinj; 372#define VMCB_CTRL_EVENTINJ_VECTOR __BITS(7,0) 373#define VMCB_CTRL_EVENTINJ_TYPE __BITS(10,8) 374#define VMCB_CTRL_EVENTINJ_EV __BIT(11) 375#define VMCB_CTRL_EVENTINJ_V __BIT(31) 376#define VMCB_CTRL_EVENTINJ_ERRORCODE __BITS(63,32) 377 378 uint64_t n_cr3; 379 380 uint64_t enable2; 381#define VMCB_CTRL_ENABLE_LBR __BIT(0) 382#define VMCB_CTRL_ENABLE_VVMSAVE __BIT(1) 383 384 uint32_t vmcb_clean; 385#define VMCB_CTRL_VMCB_CLEAN_I __BIT(0) 386#define VMCB_CTRL_VMCB_CLEAN_IOPM __BIT(1) 387#define VMCB_CTRL_VMCB_CLEAN_ASID __BIT(2) 388#define VMCB_CTRL_VMCB_CLEAN_TPR __BIT(3) 389#define VMCB_CTRL_VMCB_CLEAN_NP __BIT(4) 390#define VMCB_CTRL_VMCB_CLEAN_CR __BIT(5) 391#define VMCB_CTRL_VMCB_CLEAN_DR __BIT(6) 392#define VMCB_CTRL_VMCB_CLEAN_DT __BIT(7) 393#define VMCB_CTRL_VMCB_CLEAN_SEG __BIT(8) 394#define VMCB_CTRL_VMCB_CLEAN_CR2 __BIT(9) 395#define VMCB_CTRL_VMCB_CLEAN_LBR __BIT(10) 396#define VMCB_CTRL_VMCB_CLEAN_AVIC __BIT(11) 397 398 uint32_t rsvd2; 399 uint64_t nrip; 400 uint8_t inst_len; 401 uint8_t inst_bytes[15]; 402 uint64_t avic_abpp; 403 uint64_t rsvd3; 404 uint64_t avic_ltp; 405 406 uint64_t avic_phys; 407#define VMCB_CTRL_AVIC_PHYS_TABLE_PTR __BITS(51,12) 408#define VMCB_CTRL_AVIC_PHYS_MAX_INDEX __BITS(7,0) 409 410 uint64_t rsvd4; 411 uint64_t vmsa_ptr; 412 413 uint8_t pad[752]; 414} __packed; 415 416CTASSERT(sizeof(struct vmcb_ctrl) == 1024); 417 418struct vmcb_segment { 419 uint16_t selector; 420 uint16_t attrib; /* hidden */ 421 uint32_t limit; /* hidden */ 422 uint64_t base; /* hidden */ 423} __packed; 424 425CTASSERT(sizeof(struct vmcb_segment) == 16); 426 427struct vmcb_state { 428 struct vmcb_segment es; 429 struct vmcb_segment cs; 430 struct vmcb_segment ss; 431 struct vmcb_segment ds; 432 struct vmcb_segment fs; 433 struct vmcb_segment gs; 434 struct vmcb_segment gdt; 435 struct vmcb_segment ldt; 436 struct vmcb_segment idt; 437 struct vmcb_segment tr; 438 uint8_t rsvd1[43]; 439 uint8_t cpl; 440 uint8_t rsvd2[4]; 441 uint64_t efer; 442 uint8_t rsvd3[112]; 443 uint64_t cr4; 444 uint64_t cr3; 445 uint64_t cr0; 446 uint64_t dr7; 447 uint64_t dr6; 448 uint64_t rflags; 449 uint64_t rip; 450 uint8_t rsvd4[88]; 451 uint64_t rsp; 452 uint8_t rsvd5[24]; 453 uint64_t rax; 454 uint64_t star; 455 uint64_t lstar; 456 uint64_t cstar; 457 uint64_t sfmask; 458 uint64_t kernelgsbase; 459 uint64_t sysenter_cs; 460 uint64_t sysenter_esp; 461 uint64_t sysenter_eip; 462 uint64_t cr2; 463 uint8_t rsvd6[32]; 464 uint64_t g_pat; 465 uint64_t dbgctl; 466 uint64_t br_from; 467 uint64_t br_to; 468 uint64_t int_from; 469 uint64_t int_to; 470 uint8_t pad[2408]; 471} __packed; 472 473CTASSERT(sizeof(struct vmcb_state) == 0xC00); 474 475struct vmcb { 476 struct vmcb_ctrl ctrl; 477 struct vmcb_state state; 478} __packed; 479 480CTASSERT(sizeof(struct vmcb) == PAGE_SIZE); 481CTASSERT(offsetof(struct vmcb, state) == 0x400); 482 483/* -------------------------------------------------------------------------- */ 484 485static void svm_vcpu_state_provide(struct nvmm_cpu *, uint64_t); 486static void svm_vcpu_state_commit(struct nvmm_cpu *); 487 488struct svm_hsave { 489 paddr_t pa; 490}; 491 492static struct svm_hsave hsave[MAXCPUS]; 493 494static uint8_t *svm_asidmap __read_mostly; 495static uint32_t svm_maxasid __read_mostly; 496static kmutex_t svm_asidlock __cacheline_aligned; 497 498static bool svm_decode_assist __read_mostly; 499static uint32_t svm_ctrl_tlb_flush __read_mostly; 500 501#define SVM_XCR0_MASK_DEFAULT (XCR0_X87|XCR0_SSE) 502static uint64_t svm_xcr0_mask __read_mostly; 503 504#define SVM_NCPUIDS 32 505 506#define VMCB_NPAGES 1 507 508#define MSRBM_NPAGES 2 509#define MSRBM_SIZE (MSRBM_NPAGES * PAGE_SIZE) 510 511#define IOBM_NPAGES 3 512#define IOBM_SIZE (IOBM_NPAGES * PAGE_SIZE) 513 514/* Does not include EFER_LMSLE. */ 515#define EFER_VALID \ 516 (EFER_SCE|EFER_LME|EFER_LMA|EFER_NXE|EFER_SVME|EFER_FFXSR|EFER_TCE) 517 518#define EFER_TLB_FLUSH \ 519 (EFER_NXE|EFER_LMA|EFER_LME) 520#define CR0_TLB_FLUSH \ 521 (CR0_PG|CR0_WP|CR0_CD|CR0_NW) 522#define CR4_TLB_FLUSH \ 523 (CR4_PSE|CR4_PAE|CR4_PGE|CR4_PCIDE|CR4_SMEP) 524 525/* -------------------------------------------------------------------------- */ 526 527struct svm_machdata { 528 volatile uint64_t mach_htlb_gen; 529}; 530 531static const size_t svm_vcpu_conf_sizes[NVMM_X86_VCPU_NCONF] = { 532 [NVMM_VCPU_CONF_MD(NVMM_VCPU_CONF_CPUID)] = 533 sizeof(struct nvmm_vcpu_conf_cpuid), 534 [NVMM_VCPU_CONF_MD(NVMM_VCPU_CONF_TPR)] = 535 sizeof(struct nvmm_vcpu_conf_tpr) 536}; 537 538struct svm_cpudata { 539 /* General */ 540 bool shared_asid; 541 bool gtlb_want_flush; 542 bool gtsc_want_update; 543 uint64_t vcpu_htlb_gen; 544 545 /* VMCB */ 546 struct vmcb *vmcb; 547 paddr_t vmcb_pa; 548 549 /* I/O bitmap */ 550 uint8_t *iobm; 551 paddr_t iobm_pa; 552 553 /* MSR bitmap */ 554 uint8_t *msrbm; 555 paddr_t msrbm_pa; 556 557 /* Host state */ 558 uint64_t hxcr0; 559 uint64_t star; 560 uint64_t lstar; 561 uint64_t cstar; 562 uint64_t sfmask; 563 uint64_t fsbase; 564 uint64_t kernelgsbase; 565 566 /* Intr state */ 567 bool int_window_exit; 568 bool nmi_window_exit; 569 bool evt_pending; 570 571 /* Guest state */ 572 uint64_t gxcr0; 573 uint64_t gprs[NVMM_X64_NGPR]; 574 uint64_t drs[NVMM_X64_NDR]; 575 uint64_t gtsc; 576 struct xsave_header gfpu __aligned(64); 577 578 /* VCPU configuration. */ 579 bool cpuidpresent[SVM_NCPUIDS]; 580 struct nvmm_vcpu_conf_cpuid cpuid[SVM_NCPUIDS]; 581}; 582 583static void 584svm_vmcb_cache_default(struct vmcb *vmcb) 585{ 586 vmcb->ctrl.vmcb_clean = 587 VMCB_CTRL_VMCB_CLEAN_I | 588 VMCB_CTRL_VMCB_CLEAN_IOPM | 589 VMCB_CTRL_VMCB_CLEAN_ASID | 590 VMCB_CTRL_VMCB_CLEAN_TPR | 591 VMCB_CTRL_VMCB_CLEAN_NP | 592 VMCB_CTRL_VMCB_CLEAN_CR | 593 VMCB_CTRL_VMCB_CLEAN_DR | 594 VMCB_CTRL_VMCB_CLEAN_DT | 595 VMCB_CTRL_VMCB_CLEAN_SEG | 596 VMCB_CTRL_VMCB_CLEAN_CR2 | 597 VMCB_CTRL_VMCB_CLEAN_LBR | 598 VMCB_CTRL_VMCB_CLEAN_AVIC; 599} 600 601static void 602svm_vmcb_cache_update(struct vmcb *vmcb, uint64_t flags) 603{ 604 if (flags & NVMM_X64_STATE_SEGS) { 605 vmcb->ctrl.vmcb_clean &= 606 ~(VMCB_CTRL_VMCB_CLEAN_SEG | VMCB_CTRL_VMCB_CLEAN_DT); 607 } 608 if (flags & NVMM_X64_STATE_CRS) { 609 vmcb->ctrl.vmcb_clean &= 610 ~(VMCB_CTRL_VMCB_CLEAN_CR | VMCB_CTRL_VMCB_CLEAN_CR2 | 611 VMCB_CTRL_VMCB_CLEAN_TPR); 612 } 613 if (flags & NVMM_X64_STATE_DRS) { 614 vmcb->ctrl.vmcb_clean &= ~VMCB_CTRL_VMCB_CLEAN_DR; 615 } 616 if (flags & NVMM_X64_STATE_MSRS) { 617 /* CR for EFER, NP for PAT. */ 618 vmcb->ctrl.vmcb_clean &= 619 ~(VMCB_CTRL_VMCB_CLEAN_CR | VMCB_CTRL_VMCB_CLEAN_NP); 620 } 621} 622 623static inline void 624svm_vmcb_cache_flush(struct vmcb *vmcb, uint64_t flags) 625{ 626 vmcb->ctrl.vmcb_clean &= ~flags; 627} 628 629static inline void 630svm_vmcb_cache_flush_all(struct vmcb *vmcb) 631{ 632 vmcb->ctrl.vmcb_clean = 0; 633} 634 635#define SVM_EVENT_TYPE_HW_INT 0 636#define SVM_EVENT_TYPE_NMI 2 637#define SVM_EVENT_TYPE_EXC 3 638#define SVM_EVENT_TYPE_SW_INT 4 639 640static void 641svm_event_waitexit_enable(struct nvmm_cpu *vcpu, bool nmi) 642{ 643 struct svm_cpudata *cpudata = vcpu->cpudata; 644 struct vmcb *vmcb = cpudata->vmcb; 645 646 if (nmi) { 647 vmcb->ctrl.intercept_misc1 |= VMCB_CTRL_INTERCEPT_IRET; 648 cpudata->nmi_window_exit = true; 649 } else { 650 vmcb->ctrl.intercept_misc1 |= VMCB_CTRL_INTERCEPT_VINTR; 651 vmcb->ctrl.v |= (VMCB_CTRL_V_IRQ | VMCB_CTRL_V_IGN_TPR); 652 svm_vmcb_cache_flush(vmcb, VMCB_CTRL_VMCB_CLEAN_TPR); 653 cpudata->int_window_exit = true; 654 } 655 656 svm_vmcb_cache_flush(vmcb, VMCB_CTRL_VMCB_CLEAN_I); 657} 658 659static void 660svm_event_waitexit_disable(struct nvmm_cpu *vcpu, bool nmi) 661{ 662 struct svm_cpudata *cpudata = vcpu->cpudata; 663 struct vmcb *vmcb = cpudata->vmcb; 664 665 if (nmi) { 666 vmcb->ctrl.intercept_misc1 &= ~VMCB_CTRL_INTERCEPT_IRET; 667 cpudata->nmi_window_exit = false; 668 } else { 669 vmcb->ctrl.intercept_misc1 &= ~VMCB_CTRL_INTERCEPT_VINTR; 670 vmcb->ctrl.v &= ~(VMCB_CTRL_V_IRQ | VMCB_CTRL_V_IGN_TPR); 671 svm_vmcb_cache_flush(vmcb, VMCB_CTRL_VMCB_CLEAN_TPR); 672 cpudata->int_window_exit = false; 673 } 674 675 svm_vmcb_cache_flush(vmcb, VMCB_CTRL_VMCB_CLEAN_I); 676} 677 678static inline bool 679svm_excp_has_rf(uint8_t vector) 680{ 681 switch (vector) { 682 case 1: /* #DB */ 683 case 4: /* #OF */ 684 case 8: /* #DF */ 685 case 18: /* #MC */ 686 return false; 687 default: 688 return true; 689 } 690} 691 692static inline int 693svm_excp_has_error(uint8_t vector) 694{ 695 switch (vector) { 696 case 8: /* #DF */ 697 case 10: /* #TS */ 698 case 11: /* #NP */ 699 case 12: /* #SS */ 700 case 13: /* #GP */ 701 case 14: /* #PF */ 702 case 17: /* #AC */ 703 case 30: /* #SX */ 704 return 1; 705 default: 706 return 0; 707 } 708} 709 710static int 711svm_vcpu_inject(struct nvmm_cpu *vcpu) 712{ 713 struct nvmm_comm_page *comm = vcpu->comm; 714 struct svm_cpudata *cpudata = vcpu->cpudata; 715 struct vmcb *vmcb = cpudata->vmcb; 716 u_int evtype; 717 uint8_t vector; 718 uint64_t error; 719 int type = 0, err = 0; 720 721 evtype = comm->event.type; 722 vector = comm->event.vector; 723 error = comm->event.u.excp.error; 724 __insn_barrier(); 725 726 switch (evtype) { 727 case NVMM_VCPU_EVENT_EXCP: 728 type = SVM_EVENT_TYPE_EXC; 729 if (vector == 2 || vector >= 32) 730 return EINVAL; 731 if (vector == 3 || vector == 0) 732 return EINVAL; 733 if (svm_excp_has_rf(vector)) { 734 vmcb->state.rflags |= PSL_RF; 735 } 736 err = svm_excp_has_error(vector); 737 break; 738 case NVMM_VCPU_EVENT_INTR: 739 type = SVM_EVENT_TYPE_HW_INT; 740 if (vector == 2) { 741 type = SVM_EVENT_TYPE_NMI; 742 svm_event_waitexit_enable(vcpu, true); 743 } 744 err = 0; 745 break; 746 default: 747 return EINVAL; 748 } 749 750 vmcb->ctrl.eventinj = 751 __SHIFTIN((uint64_t)vector, VMCB_CTRL_EVENTINJ_VECTOR) | 752 __SHIFTIN((uint64_t)type, VMCB_CTRL_EVENTINJ_TYPE) | 753 __SHIFTIN((uint64_t)err, VMCB_CTRL_EVENTINJ_EV) | 754 __SHIFTIN((uint64_t)1, VMCB_CTRL_EVENTINJ_V) | 755 __SHIFTIN((uint64_t)error, VMCB_CTRL_EVENTINJ_ERRORCODE); 756 757 cpudata->evt_pending = true; 758 759 return 0; 760} 761 762static void 763svm_inject_ud(struct nvmm_cpu *vcpu) 764{ 765 struct nvmm_comm_page *comm = vcpu->comm; 766 int ret __diagused; 767 768 comm->event.type = NVMM_VCPU_EVENT_EXCP; 769 comm->event.vector = 6; 770 comm->event.u.excp.error = 0; 771 772 ret = svm_vcpu_inject(vcpu); 773 KASSERT(ret == 0); 774} 775 776static void 777svm_inject_gp(struct nvmm_cpu *vcpu) 778{ 779 struct nvmm_comm_page *comm = vcpu->comm; 780 int ret __diagused; 781 782 comm->event.type = NVMM_VCPU_EVENT_EXCP; 783 comm->event.vector = 13; 784 comm->event.u.excp.error = 0; 785 786 ret = svm_vcpu_inject(vcpu); 787 KASSERT(ret == 0); 788} 789 790static inline int 791svm_vcpu_event_commit(struct nvmm_cpu *vcpu) 792{ 793 if (__predict_true(!vcpu->comm->event_commit)) { 794 return 0; 795 } 796 vcpu->comm->event_commit = false; 797 return svm_vcpu_inject(vcpu); 798} 799 800static inline void 801svm_inkernel_advance(struct vmcb *vmcb) 802{ 803 /* 804 * Maybe we should also apply single-stepping and debug exceptions. 805 * Matters for guest-ring3, because it can execute 'cpuid' under a 806 * debugger. 807 */ 808 vmcb->state.rip = vmcb->ctrl.nrip; 809 vmcb->state.rflags &= ~PSL_RF; 810 vmcb->ctrl.intr &= ~VMCB_CTRL_INTR_SHADOW; 811} 812 813#define SVM_CPUID_MAX_BASIC 0xD 814#define SVM_CPUID_MAX_HYPERVISOR 0x40000000 815#define SVM_CPUID_MAX_EXTENDED 0x8000001F 816static uint32_t svm_cpuid_max_basic __read_mostly; 817static uint32_t svm_cpuid_max_extended __read_mostly; 818 819static void 820svm_inkernel_exec_cpuid(struct svm_cpudata *cpudata, uint64_t eax, uint64_t ecx) 821{ 822 u_int descs[4]; 823 824 x86_cpuid2(eax, ecx, descs); 825 cpudata->vmcb->state.rax = descs[0]; 826 cpudata->gprs[NVMM_X64_GPR_RBX] = descs[1]; 827 cpudata->gprs[NVMM_X64_GPR_RCX] = descs[2]; 828 cpudata->gprs[NVMM_X64_GPR_RDX] = descs[3]; 829} 830 831static void 832svm_inkernel_handle_cpuid(struct nvmm_cpu *vcpu, uint64_t eax, uint64_t ecx) 833{ 834 struct svm_cpudata *cpudata = vcpu->cpudata; 835 uint64_t cr4; 836 837 if (eax < 0x40000000) { 838 if (__predict_false(eax > svm_cpuid_max_basic)) { 839 eax = svm_cpuid_max_basic; 840 svm_inkernel_exec_cpuid(cpudata, eax, ecx); 841 } 842 } else if (eax < 0x80000000) { 843 if (__predict_false(eax > SVM_CPUID_MAX_HYPERVISOR)) { 844 eax = svm_cpuid_max_basic; 845 svm_inkernel_exec_cpuid(cpudata, eax, ecx); 846 } 847 } else { 848 if (__predict_false(eax > svm_cpuid_max_extended)) { 849 eax = svm_cpuid_max_basic; 850 svm_inkernel_exec_cpuid(cpudata, eax, ecx); 851 } 852 } 853 854 switch (eax) { 855 case 0x00000000: 856 cpudata->vmcb->state.rax = svm_cpuid_max_basic; 857 break; 858 case 0x00000001: 859 cpudata->vmcb->state.rax &= nvmm_cpuid_00000001.eax; 860 861 cpudata->gprs[NVMM_X64_GPR_RBX] &= ~CPUID_LOCAL_APIC_ID; 862 cpudata->gprs[NVMM_X64_GPR_RBX] |= __SHIFTIN(vcpu->cpuid, 863 CPUID_LOCAL_APIC_ID); 864 865 cpudata->gprs[NVMM_X64_GPR_RCX] &= nvmm_cpuid_00000001.ecx; 866 cpudata->gprs[NVMM_X64_GPR_RCX] |= CPUID2_RAZ; 867 868 cpudata->gprs[NVMM_X64_GPR_RDX] &= nvmm_cpuid_00000001.edx; 869 870 /* CPUID2_OSXSAVE depends on CR4. */ 871 cr4 = cpudata->vmcb->state.cr4; 872 if (!(cr4 & CR4_OSXSAVE)) { 873 cpudata->gprs[NVMM_X64_GPR_RCX] &= ~CPUID2_OSXSAVE; 874 } 875 break; 876 case 0x00000002: /* Empty */ 877 case 0x00000003: /* Empty */ 878 case 0x00000004: /* Empty */ 879 case 0x00000005: /* Monitor/MWait */ 880 case 0x00000006: /* Power Management Related Features */ 881 cpudata->vmcb->state.rax = 0; 882 cpudata->gprs[NVMM_X64_GPR_RBX] = 0; 883 cpudata->gprs[NVMM_X64_GPR_RCX] = 0; 884 cpudata->gprs[NVMM_X64_GPR_RDX] = 0; 885 break; 886 case 0x00000007: /* Structured Extended Features */ 887 switch (ecx) { 888 case 0: 889 cpudata->vmcb->state.rax = 0; 890 cpudata->gprs[NVMM_X64_GPR_RBX] &= nvmm_cpuid_00000007.ebx; 891 cpudata->gprs[NVMM_X64_GPR_RCX] &= nvmm_cpuid_00000007.ecx; 892 cpudata->gprs[NVMM_X64_GPR_RDX] &= nvmm_cpuid_00000007.edx; 893 break; 894 default: 895 cpudata->vmcb->state.rax = 0; 896 cpudata->gprs[NVMM_X64_GPR_RBX] = 0; 897 cpudata->gprs[NVMM_X64_GPR_RCX] = 0; 898 cpudata->gprs[NVMM_X64_GPR_RDX] = 0; 899 break; 900 } 901 break; 902 case 0x00000008: /* Empty */ 903 case 0x00000009: /* Empty */ 904 case 0x0000000A: /* Empty */ 905 case 0x0000000B: /* Empty */ 906 case 0x0000000C: /* Empty */ 907 cpudata->vmcb->state.rax = 0; 908 cpudata->gprs[NVMM_X64_GPR_RBX] = 0; 909 cpudata->gprs[NVMM_X64_GPR_RCX] = 0; 910 cpudata->gprs[NVMM_X64_GPR_RDX] = 0; 911 break; 912 case 0x0000000D: /* Processor Extended State Enumeration */ 913 if (svm_xcr0_mask == 0) { 914 break; 915 } 916 switch (ecx) { 917 case 0: 918 cpudata->vmcb->state.rax = svm_xcr0_mask & 0xFFFFFFFF; 919 if (cpudata->gxcr0 & XCR0_SSE) { 920 cpudata->gprs[NVMM_X64_GPR_RBX] = sizeof(struct fxsave); 921 } else { 922 cpudata->gprs[NVMM_X64_GPR_RBX] = sizeof(struct save87); 923 } 924 cpudata->gprs[NVMM_X64_GPR_RBX] += 64; /* XSAVE header */ 925 cpudata->gprs[NVMM_X64_GPR_RCX] = sizeof(struct fxsave) + 64; 926 cpudata->gprs[NVMM_X64_GPR_RDX] = svm_xcr0_mask >> 32; 927 break; 928 case 1: 929 cpudata->vmcb->state.rax &= 930 (CPUID_PES1_XSAVEOPT | CPUID_PES1_XSAVEC | 931 CPUID_PES1_XGETBV); 932 cpudata->gprs[NVMM_X64_GPR_RBX] = 0; 933 cpudata->gprs[NVMM_X64_GPR_RCX] = 0; 934 cpudata->gprs[NVMM_X64_GPR_RDX] = 0; 935 break; 936 default: 937 cpudata->vmcb->state.rax = 0; 938 cpudata->gprs[NVMM_X64_GPR_RBX] = 0; 939 cpudata->gprs[NVMM_X64_GPR_RCX] = 0; 940 cpudata->gprs[NVMM_X64_GPR_RDX] = 0; 941 break; 942 } 943 break; 944 945 case 0x40000000: /* Hypervisor Information */ 946 cpudata->vmcb->state.rax = SVM_CPUID_MAX_HYPERVISOR; 947 cpudata->gprs[NVMM_X64_GPR_RBX] = 0; 948 cpudata->gprs[NVMM_X64_GPR_RCX] = 0; 949 cpudata->gprs[NVMM_X64_GPR_RDX] = 0; 950 memcpy(&cpudata->gprs[NVMM_X64_GPR_RBX], "___ ", 4); 951 memcpy(&cpudata->gprs[NVMM_X64_GPR_RCX], "NVMM", 4); 952 memcpy(&cpudata->gprs[NVMM_X64_GPR_RDX], " ___", 4); 953 break; 954 955 case 0x80000000: 956 cpudata->vmcb->state.rax = svm_cpuid_max_extended; 957 break; 958 case 0x80000001: 959 cpudata->vmcb->state.rax &= nvmm_cpuid_80000001.eax; 960 cpudata->gprs[NVMM_X64_GPR_RBX] &= nvmm_cpuid_80000001.ebx; 961 cpudata->gprs[NVMM_X64_GPR_RCX] &= nvmm_cpuid_80000001.ecx; 962 cpudata->gprs[NVMM_X64_GPR_RDX] &= nvmm_cpuid_80000001.edx; 963 break; 964 case 0x80000002: /* Extended Processor Name String */ 965 case 0x80000003: /* Extended Processor Name String */ 966 case 0x80000004: /* Extended Processor Name String */ 967 case 0x80000005: /* L1 Cache and TLB Information */ 968 case 0x80000006: /* L2 Cache and TLB and L3 Cache Information */ 969 break; 970 case 0x80000007: /* Processor Power Management and RAS Capabilities */ 971 cpudata->vmcb->state.rax &= nvmm_cpuid_80000007.eax; 972 cpudata->gprs[NVMM_X64_GPR_RBX] &= nvmm_cpuid_80000007.ebx; 973 cpudata->gprs[NVMM_X64_GPR_RCX] &= nvmm_cpuid_80000007.ecx; 974 cpudata->gprs[NVMM_X64_GPR_RDX] &= nvmm_cpuid_80000007.edx; 975 break; 976 case 0x80000008: /* Processor Capacity Parameters and Ext Feat Ident */ 977 cpudata->vmcb->state.rax &= nvmm_cpuid_80000008.eax; 978 cpudata->gprs[NVMM_X64_GPR_RBX] &= nvmm_cpuid_80000008.ebx; 979 cpudata->gprs[NVMM_X64_GPR_RCX] &= nvmm_cpuid_80000008.ecx; 980 cpudata->gprs[NVMM_X64_GPR_RDX] &= nvmm_cpuid_80000008.edx; 981 break; 982 case 0x80000009: /* Empty */ 983 case 0x8000000A: /* SVM Features */ 984 case 0x8000000B: /* Empty */ 985 case 0x8000000C: /* Empty */ 986 case 0x8000000D: /* Empty */ 987 case 0x8000000E: /* Empty */ 988 case 0x8000000F: /* Empty */ 989 case 0x80000010: /* Empty */ 990 case 0x80000011: /* Empty */ 991 case 0x80000012: /* Empty */ 992 case 0x80000013: /* Empty */ 993 case 0x80000014: /* Empty */ 994 case 0x80000015: /* Empty */ 995 case 0x80000016: /* Empty */ 996 case 0x80000017: /* Empty */ 997 case 0x80000018: /* Empty */ 998 cpudata->vmcb->state.rax = 0; 999 cpudata->gprs[NVMM_X64_GPR_RBX] = 0; 1000 cpudata->gprs[NVMM_X64_GPR_RCX] = 0; 1001 cpudata->gprs[NVMM_X64_GPR_RDX] = 0; 1002 break; 1003 case 0x80000019: /* TLB Characteristics for 1GB pages */ 1004 case 0x8000001A: /* Instruction Optimizations */ 1005 break; 1006 case 0x8000001B: /* Instruction-Based Sampling Capabilities */ 1007 case 0x8000001C: /* Lightweight Profiling Capabilities */ 1008 cpudata->vmcb->state.rax = 0; 1009 cpudata->gprs[NVMM_X64_GPR_RBX] = 0; 1010 cpudata->gprs[NVMM_X64_GPR_RCX] = 0; 1011 cpudata->gprs[NVMM_X64_GPR_RDX] = 0; 1012 break; 1013 case 0x8000001D: /* Cache Topology Information */ 1014 case 0x8000001E: /* Processor Topology Information */ 1015 break; /* TODO? */ 1016 case 0x8000001F: /* Encrypted Memory Capabilities */ 1017 cpudata->vmcb->state.rax = 0; 1018 cpudata->gprs[NVMM_X64_GPR_RBX] = 0; 1019 cpudata->gprs[NVMM_X64_GPR_RCX] = 0; 1020 cpudata->gprs[NVMM_X64_GPR_RDX] = 0; 1021 break; 1022 1023 default: 1024 break; 1025 } 1026} 1027 1028static void 1029svm_exit_insn(struct vmcb *vmcb, struct nvmm_vcpu_exit *exit, uint64_t reason) 1030{ 1031 exit->u.insn.npc = vmcb->ctrl.nrip; 1032 exit->reason = reason; 1033} 1034 1035static void 1036svm_exit_cpuid(struct nvmm_machine *mach, struct nvmm_cpu *vcpu, 1037 struct nvmm_vcpu_exit *exit) 1038{ 1039 struct svm_cpudata *cpudata = vcpu->cpudata; 1040 struct nvmm_vcpu_conf_cpuid *cpuid; 1041 uint64_t eax, ecx; 1042 size_t i; 1043 1044 eax = cpudata->vmcb->state.rax; 1045 ecx = cpudata->gprs[NVMM_X64_GPR_RCX]; 1046 svm_inkernel_exec_cpuid(cpudata, eax, ecx); 1047 svm_inkernel_handle_cpuid(vcpu, eax, ecx); 1048 1049 for (i = 0; i < SVM_NCPUIDS; i++) { 1050 if (!cpudata->cpuidpresent[i]) { 1051 continue; 1052 } 1053 cpuid = &cpudata->cpuid[i]; 1054 if (cpuid->leaf != eax) { 1055 continue; 1056 } 1057 1058 if (cpuid->exit) { 1059 svm_exit_insn(cpudata->vmcb, exit, NVMM_VCPU_EXIT_CPUID); 1060 return; 1061 } 1062 KASSERT(cpuid->mask); 1063 1064 /* del */ 1065 cpudata->vmcb->state.rax &= ~cpuid->u.mask.del.eax; 1066 cpudata->gprs[NVMM_X64_GPR_RBX] &= ~cpuid->u.mask.del.ebx; 1067 cpudata->gprs[NVMM_X64_GPR_RCX] &= ~cpuid->u.mask.del.ecx; 1068 cpudata->gprs[NVMM_X64_GPR_RDX] &= ~cpuid->u.mask.del.edx; 1069 1070 /* set */ 1071 cpudata->vmcb->state.rax |= cpuid->u.mask.set.eax; 1072 cpudata->gprs[NVMM_X64_GPR_RBX] |= cpuid->u.mask.set.ebx; 1073 cpudata->gprs[NVMM_X64_GPR_RCX] |= cpuid->u.mask.set.ecx; 1074 cpudata->gprs[NVMM_X64_GPR_RDX] |= cpuid->u.mask.set.edx; 1075 1076 break; 1077 } 1078 1079 svm_inkernel_advance(cpudata->vmcb); 1080 exit->reason = NVMM_VCPU_EXIT_NONE; 1081} 1082 1083static void 1084svm_exit_hlt(struct nvmm_machine *mach, struct nvmm_cpu *vcpu, 1085 struct nvmm_vcpu_exit *exit) 1086{ 1087 struct svm_cpudata *cpudata = vcpu->cpudata; 1088 struct vmcb *vmcb = cpudata->vmcb; 1089 1090 if (cpudata->int_window_exit && (vmcb->state.rflags & PSL_I)) { 1091 svm_event_waitexit_disable(vcpu, false); 1092 } 1093 1094 svm_inkernel_advance(cpudata->vmcb); 1095 exit->reason = NVMM_VCPU_EXIT_HALTED; 1096} 1097 1098#define SVM_EXIT_IO_PORT __BITS(31,16) 1099#define SVM_EXIT_IO_SEG __BITS(12,10) 1100#define SVM_EXIT_IO_A64 __BIT(9) 1101#define SVM_EXIT_IO_A32 __BIT(8) 1102#define SVM_EXIT_IO_A16 __BIT(7) 1103#define SVM_EXIT_IO_SZ32 __BIT(6) 1104#define SVM_EXIT_IO_SZ16 __BIT(5) 1105#define SVM_EXIT_IO_SZ8 __BIT(4) 1106#define SVM_EXIT_IO_REP __BIT(3) 1107#define SVM_EXIT_IO_STR __BIT(2) 1108#define SVM_EXIT_IO_IN __BIT(0) 1109 1110static void 1111svm_exit_io(struct nvmm_machine *mach, struct nvmm_cpu *vcpu, 1112 struct nvmm_vcpu_exit *exit) 1113{ 1114 struct svm_cpudata *cpudata = vcpu->cpudata; 1115 uint64_t info = cpudata->vmcb->ctrl.exitinfo1; 1116 uint64_t nextpc = cpudata->vmcb->ctrl.exitinfo2; 1117 1118 exit->reason = NVMM_VCPU_EXIT_IO; 1119 1120 exit->u.io.in = (info & SVM_EXIT_IO_IN) != 0; 1121 exit->u.io.port = __SHIFTOUT(info, SVM_EXIT_IO_PORT); 1122 1123 if (svm_decode_assist) { 1124 KASSERT(__SHIFTOUT(info, SVM_EXIT_IO_SEG) < 6); 1125 exit->u.io.seg = __SHIFTOUT(info, SVM_EXIT_IO_SEG); 1126 } else { 1127 exit->u.io.seg = -1; 1128 } 1129 1130 if (info & SVM_EXIT_IO_A64) { 1131 exit->u.io.address_size = 8; 1132 } else if (info & SVM_EXIT_IO_A32) { 1133 exit->u.io.address_size = 4; 1134 } else if (info & SVM_EXIT_IO_A16) { 1135 exit->u.io.address_size = 2; 1136 } 1137 1138 if (info & SVM_EXIT_IO_SZ32) { 1139 exit->u.io.operand_size = 4; 1140 } else if (info & SVM_EXIT_IO_SZ16) { 1141 exit->u.io.operand_size = 2; 1142 } else if (info & SVM_EXIT_IO_SZ8) { 1143 exit->u.io.operand_size = 1; 1144 } 1145 1146 exit->u.io.rep = (info & SVM_EXIT_IO_REP) != 0; 1147 exit->u.io.str = (info & SVM_EXIT_IO_STR) != 0; 1148 exit->u.io.npc = nextpc; 1149 1150 svm_vcpu_state_provide(vcpu, 1151 NVMM_X64_STATE_GPRS | NVMM_X64_STATE_SEGS | 1152 NVMM_X64_STATE_CRS | NVMM_X64_STATE_MSRS); 1153} 1154 1155static const uint64_t msr_ignore_list[] = { 1156 0xc0010055, /* MSR_CMPHALT */ 1157 MSR_DE_CFG, 1158 MSR_IC_CFG, 1159 MSR_UCODE_AMD_PATCHLEVEL 1160}; 1161 1162static bool 1163svm_inkernel_handle_msr(struct nvmm_machine *mach, struct nvmm_cpu *vcpu, 1164 struct nvmm_vcpu_exit *exit) 1165{ 1166 struct svm_cpudata *cpudata = vcpu->cpudata; 1167 struct vmcb *vmcb = cpudata->vmcb; 1168 uint64_t val; 1169 size_t i; 1170 1171 if (exit->reason == NVMM_VCPU_EXIT_RDMSR) { 1172 if (exit->u.rdmsr.msr == MSR_EFER) { 1173 val = vmcb->state.efer & ~EFER_SVME; 1174 vmcb->state.rax = (val & 0xFFFFFFFF); 1175 cpudata->gprs[NVMM_X64_GPR_RDX] = (val >> 32); 1176 goto handled; 1177 } 1178 if (exit->u.rdmsr.msr == MSR_NB_CFG) { 1179 val = NB_CFG_INITAPICCPUIDLO; 1180 vmcb->state.rax = (val & 0xFFFFFFFF); 1181 cpudata->gprs[NVMM_X64_GPR_RDX] = (val >> 32); 1182 goto handled; 1183 } 1184 for (i = 0; i < __arraycount(msr_ignore_list); i++) { 1185 if (msr_ignore_list[i] != exit->u.rdmsr.msr) 1186 continue; 1187 val = 0; 1188 vmcb->state.rax = (val & 0xFFFFFFFF); 1189 cpudata->gprs[NVMM_X64_GPR_RDX] = (val >> 32); 1190 goto handled; 1191 } 1192 } else { 1193 if (exit->u.wrmsr.msr == MSR_EFER) { 1194 if (__predict_false(exit->u.wrmsr.val & ~EFER_VALID)) { 1195 goto error; 1196 } 1197 if ((vmcb->state.efer ^ exit->u.wrmsr.val) & 1198 EFER_TLB_FLUSH) { 1199 cpudata->gtlb_want_flush = true; 1200 } 1201 vmcb->state.efer = exit->u.wrmsr.val | EFER_SVME; 1202 svm_vmcb_cache_flush(vmcb, VMCB_CTRL_VMCB_CLEAN_CR); 1203 goto handled; 1204 } 1205 if (exit->u.wrmsr.msr == MSR_TSC) { 1206 cpudata->gtsc = exit->u.wrmsr.val; 1207 cpudata->gtsc_want_update = true; 1208 goto handled; 1209 } 1210 for (i = 0; i < __arraycount(msr_ignore_list); i++) { 1211 if (msr_ignore_list[i] != exit->u.wrmsr.msr) 1212 continue; 1213 goto handled; 1214 } 1215 } 1216 1217 return false; 1218 1219handled: 1220 svm_inkernel_advance(cpudata->vmcb); 1221 return true; 1222 1223error: 1224 svm_inject_gp(vcpu); 1225 return true; 1226} 1227 1228static inline void 1229svm_exit_rdmsr(struct nvmm_machine *mach, struct nvmm_cpu *vcpu, 1230 struct nvmm_vcpu_exit *exit) 1231{ 1232 struct svm_cpudata *cpudata = vcpu->cpudata; 1233 1234 exit->reason = NVMM_VCPU_EXIT_RDMSR; 1235 exit->u.rdmsr.msr = (cpudata->gprs[NVMM_X64_GPR_RCX] & 0xFFFFFFFF); 1236 exit->u.rdmsr.npc = cpudata->vmcb->ctrl.nrip; 1237 1238 if (svm_inkernel_handle_msr(mach, vcpu, exit)) { 1239 exit->reason = NVMM_VCPU_EXIT_NONE; 1240 return; 1241 } 1242 1243 svm_vcpu_state_provide(vcpu, NVMM_X64_STATE_GPRS); 1244} 1245 1246static inline void 1247svm_exit_wrmsr(struct nvmm_machine *mach, struct nvmm_cpu *vcpu, 1248 struct nvmm_vcpu_exit *exit) 1249{ 1250 struct svm_cpudata *cpudata = vcpu->cpudata; 1251 uint64_t rdx, rax; 1252 1253 rdx = cpudata->gprs[NVMM_X64_GPR_RDX]; 1254 rax = cpudata->vmcb->state.rax; 1255 1256 exit->reason = NVMM_VCPU_EXIT_WRMSR; 1257 exit->u.wrmsr.msr = (cpudata->gprs[NVMM_X64_GPR_RCX] & 0xFFFFFFFF); 1258 exit->u.wrmsr.val = (rdx << 32) | (rax & 0xFFFFFFFF); 1259 exit->u.wrmsr.npc = cpudata->vmcb->ctrl.nrip; 1260 1261 if (svm_inkernel_handle_msr(mach, vcpu, exit)) { 1262 exit->reason = NVMM_VCPU_EXIT_NONE; 1263 return; 1264 } 1265 1266 svm_vcpu_state_provide(vcpu, NVMM_X64_STATE_GPRS); 1267} 1268 1269static void 1270svm_exit_msr(struct nvmm_machine *mach, struct nvmm_cpu *vcpu, 1271 struct nvmm_vcpu_exit *exit) 1272{ 1273 struct svm_cpudata *cpudata = vcpu->cpudata; 1274 uint64_t info = cpudata->vmcb->ctrl.exitinfo1; 1275 1276 if (info == 0) { 1277 svm_exit_rdmsr(mach, vcpu, exit); 1278 } else { 1279 svm_exit_wrmsr(mach, vcpu, exit); 1280 } 1281} 1282 1283static void 1284svm_exit_npf(struct nvmm_machine *mach, struct nvmm_cpu *vcpu, 1285 struct nvmm_vcpu_exit *exit) 1286{ 1287 struct svm_cpudata *cpudata = vcpu->cpudata; 1288 gpaddr_t gpa = cpudata->vmcb->ctrl.exitinfo2; 1289 1290 exit->reason = NVMM_VCPU_EXIT_MEMORY; 1291 if (cpudata->vmcb->ctrl.exitinfo1 & PGEX_W) 1292 exit->u.mem.prot = PROT_WRITE; 1293 else if (cpudata->vmcb->ctrl.exitinfo1 & PGEX_I) 1294 exit->u.mem.prot = PROT_EXEC; 1295 else 1296 exit->u.mem.prot = PROT_READ; 1297 exit->u.mem.gpa = gpa; 1298 exit->u.mem.inst_len = cpudata->vmcb->ctrl.inst_len; 1299 memcpy(exit->u.mem.inst_bytes, cpudata->vmcb->ctrl.inst_bytes, 1300 sizeof(exit->u.mem.inst_bytes)); 1301 1302 svm_vcpu_state_provide(vcpu, 1303 NVMM_X64_STATE_GPRS | NVMM_X64_STATE_SEGS | 1304 NVMM_X64_STATE_CRS | NVMM_X64_STATE_MSRS); 1305} 1306 1307static void 1308svm_exit_xsetbv(struct nvmm_machine *mach, struct nvmm_cpu *vcpu, 1309 struct nvmm_vcpu_exit *exit) 1310{ 1311 struct svm_cpudata *cpudata = vcpu->cpudata; 1312 struct vmcb *vmcb = cpudata->vmcb; 1313 uint64_t val; 1314 1315 exit->reason = NVMM_VCPU_EXIT_NONE; 1316 1317 val = (cpudata->gprs[NVMM_X64_GPR_RDX] << 32) | 1318 (vmcb->state.rax & 0xFFFFFFFF); 1319 1320 if (__predict_false(cpudata->gprs[NVMM_X64_GPR_RCX] != 0)) { 1321 goto error; 1322 } else if (__predict_false(vmcb->state.cpl != 0)) { 1323 goto error; 1324 } else if (__predict_false((val & ~svm_xcr0_mask) != 0)) { 1325 goto error; 1326 } else if (__predict_false((val & XCR0_X87) == 0)) { 1327 goto error; 1328 } 1329 1330 cpudata->gxcr0 = val; 1331 1332 svm_inkernel_advance(cpudata->vmcb); 1333 return; 1334 1335error: 1336 svm_inject_gp(vcpu); 1337} 1338 1339static void 1340svm_exit_invalid(struct nvmm_vcpu_exit *exit, uint64_t code) 1341{ 1342 exit->u.inv.hwcode = code; 1343 exit->reason = NVMM_VCPU_EXIT_INVALID; 1344} 1345 1346/* -------------------------------------------------------------------------- */ 1347 1348static void 1349svm_vcpu_guest_fpu_enter(struct nvmm_cpu *vcpu) 1350{ 1351 struct svm_cpudata *cpudata = vcpu->cpudata; 1352 1353 fpu_kern_enter(); 1354 /* TODO: should we use *XSAVE64 here? */ 1355 fpu_area_restore(&cpudata->gfpu, svm_xcr0_mask, false); 1356 1357 if (svm_xcr0_mask != 0) { 1358 cpudata->hxcr0 = rdxcr(0); 1359 wrxcr(0, cpudata->gxcr0); 1360 } 1361} 1362 1363static void 1364svm_vcpu_guest_fpu_leave(struct nvmm_cpu *vcpu) 1365{ 1366 struct svm_cpudata *cpudata = vcpu->cpudata; 1367 1368 if (svm_xcr0_mask != 0) { 1369 cpudata->gxcr0 = rdxcr(0); 1370 wrxcr(0, cpudata->hxcr0); 1371 } 1372 1373 /* TODO: should we use *XSAVE64 here? */ 1374 fpu_area_save(&cpudata->gfpu, svm_xcr0_mask, false); 1375 fpu_kern_leave(); 1376} 1377 1378static void 1379svm_vcpu_guest_dbregs_enter(struct nvmm_cpu *vcpu) 1380{ 1381 struct svm_cpudata *cpudata = vcpu->cpudata; 1382 1383 x86_dbregs_save(curlwp); 1384 1385 ldr7(0); 1386 1387 ldr0(cpudata->drs[NVMM_X64_DR_DR0]); 1388 ldr1(cpudata->drs[NVMM_X64_DR_DR1]); 1389 ldr2(cpudata->drs[NVMM_X64_DR_DR2]); 1390 ldr3(cpudata->drs[NVMM_X64_DR_DR3]); 1391} 1392 1393static void 1394svm_vcpu_guest_dbregs_leave(struct nvmm_cpu *vcpu) 1395{ 1396 struct svm_cpudata *cpudata = vcpu->cpudata; 1397 1398 cpudata->drs[NVMM_X64_DR_DR0] = rdr0(); 1399 cpudata->drs[NVMM_X64_DR_DR1] = rdr1(); 1400 cpudata->drs[NVMM_X64_DR_DR2] = rdr2(); 1401 cpudata->drs[NVMM_X64_DR_DR3] = rdr3(); 1402 1403 x86_dbregs_restore(curlwp); 1404} 1405 1406static void 1407svm_vcpu_guest_misc_enter(struct nvmm_cpu *vcpu) 1408{ 1409 struct svm_cpudata *cpudata = vcpu->cpudata; 1410 1411 cpudata->fsbase = rdmsr(MSR_FSBASE); 1412 cpudata->kernelgsbase = rdmsr(MSR_KERNELGSBASE); 1413} 1414 1415static void 1416svm_vcpu_guest_misc_leave(struct nvmm_cpu *vcpu) 1417{ 1418 struct svm_cpudata *cpudata = vcpu->cpudata; 1419 1420 wrmsr(MSR_STAR, cpudata->star); 1421 wrmsr(MSR_LSTAR, cpudata->lstar); 1422 wrmsr(MSR_CSTAR, cpudata->cstar); 1423 wrmsr(MSR_SFMASK, cpudata->sfmask); 1424 wrmsr(MSR_FSBASE, cpudata->fsbase); 1425 wrmsr(MSR_KERNELGSBASE, cpudata->kernelgsbase); 1426} 1427 1428/* -------------------------------------------------------------------------- */ 1429 1430static inline void 1431svm_gtlb_catchup(struct nvmm_cpu *vcpu, int hcpu) 1432{ 1433 struct svm_cpudata *cpudata = vcpu->cpudata; 1434 1435 if (vcpu->hcpu_last != hcpu || cpudata->shared_asid) { 1436 cpudata->gtlb_want_flush = true; 1437 } 1438} 1439 1440static inline void 1441svm_htlb_catchup(struct nvmm_cpu *vcpu, int hcpu) 1442{ 1443 /* 1444 * Nothing to do. If an hTLB flush was needed, either the VCPU was 1445 * executing on this hCPU and the hTLB already got flushed, or it 1446 * was executing on another hCPU in which case the catchup is done 1447 * in svm_gtlb_catchup(). 1448 */ 1449} 1450 1451static inline uint64_t 1452svm_htlb_flush(struct svm_machdata *machdata, struct svm_cpudata *cpudata) 1453{ 1454 struct vmcb *vmcb = cpudata->vmcb; 1455 uint64_t machgen; 1456 1457 machgen = machdata->mach_htlb_gen; 1458 if (__predict_true(machgen == cpudata->vcpu_htlb_gen)) { 1459 return machgen; 1460 } 1461 1462 vmcb->ctrl.tlb_ctrl = svm_ctrl_tlb_flush; 1463 return machgen; 1464} 1465 1466static inline void 1467svm_htlb_flush_ack(struct svm_cpudata *cpudata, uint64_t machgen) 1468{ 1469 struct vmcb *vmcb = cpudata->vmcb; 1470 1471 if (__predict_true(vmcb->ctrl.exitcode != VMCB_EXITCODE_INVALID)) { 1472 cpudata->vcpu_htlb_gen = machgen; 1473 } 1474} 1475 1476static inline void 1477svm_exit_evt(struct svm_cpudata *cpudata, struct vmcb *vmcb) 1478{ 1479 cpudata->evt_pending = false; 1480 1481 if (__predict_false(vmcb->ctrl.exitintinfo & VMCB_CTRL_EXITINTINFO_V)) { 1482 vmcb->ctrl.eventinj = vmcb->ctrl.exitintinfo; 1483 cpudata->evt_pending = true; 1484 } 1485} 1486 1487static int 1488svm_vcpu_run(struct nvmm_machine *mach, struct nvmm_cpu *vcpu, 1489 struct nvmm_vcpu_exit *exit) 1490{ 1491 struct nvmm_comm_page *comm = vcpu->comm; 1492 struct svm_machdata *machdata = mach->machdata; 1493 struct svm_cpudata *cpudata = vcpu->cpudata; 1494 struct vmcb *vmcb = cpudata->vmcb; 1495 uint64_t machgen; 1496 int hcpu; 1497 1498 svm_vcpu_state_commit(vcpu); 1499 comm->state_cached = 0; 1500 1501 if (__predict_false(svm_vcpu_event_commit(vcpu) != 0)) { 1502 return EINVAL; 1503 } 1504 1505 kpreempt_disable(); 1506 hcpu = cpu_number(); 1507 1508 svm_gtlb_catchup(vcpu, hcpu); 1509 svm_htlb_catchup(vcpu, hcpu); 1510 1511 if (vcpu->hcpu_last != hcpu) { 1512 svm_vmcb_cache_flush_all(vmcb); 1513 cpudata->gtsc_want_update = true; 1514 } 1515 1516 svm_vcpu_guest_dbregs_enter(vcpu); 1517 svm_vcpu_guest_misc_enter(vcpu); 1518 1519 while (1) { 1520 if (cpudata->gtlb_want_flush) { 1521 vmcb->ctrl.tlb_ctrl = svm_ctrl_tlb_flush; 1522 } else { 1523 vmcb->ctrl.tlb_ctrl = 0; 1524 } 1525 1526 if (__predict_false(cpudata->gtsc_want_update)) { 1527 vmcb->ctrl.tsc_offset = cpudata->gtsc - rdtsc(); 1528 svm_vmcb_cache_flush(vmcb, VMCB_CTRL_VMCB_CLEAN_I); 1529 } 1530 1531 svm_vcpu_guest_fpu_enter(vcpu); 1532 svm_clgi(); 1533 machgen = svm_htlb_flush(machdata, cpudata); 1534 svm_vmrun(cpudata->vmcb_pa, cpudata->gprs); 1535 svm_htlb_flush_ack(cpudata, machgen); 1536 svm_stgi(); 1537 svm_vcpu_guest_fpu_leave(vcpu); 1538 1539 svm_vmcb_cache_default(vmcb); 1540 1541 if (vmcb->ctrl.exitcode != VMCB_EXITCODE_INVALID) { 1542 cpudata->gtlb_want_flush = false; 1543 cpudata->gtsc_want_update = false; 1544 vcpu->hcpu_last = hcpu; 1545 } 1546 svm_exit_evt(cpudata, vmcb); 1547 1548 switch (vmcb->ctrl.exitcode) { 1549 case VMCB_EXITCODE_INTR: 1550 case VMCB_EXITCODE_NMI: 1551 exit->reason = NVMM_VCPU_EXIT_NONE; 1552 break; 1553 case VMCB_EXITCODE_VINTR: 1554 svm_event_waitexit_disable(vcpu, false); 1555 exit->reason = NVMM_VCPU_EXIT_INT_READY; 1556 break; 1557 case VMCB_EXITCODE_IRET: 1558 svm_event_waitexit_disable(vcpu, true); 1559 exit->reason = NVMM_VCPU_EXIT_NMI_READY; 1560 break; 1561 case VMCB_EXITCODE_CPUID: 1562 svm_exit_cpuid(mach, vcpu, exit); 1563 break; 1564 case VMCB_EXITCODE_HLT: 1565 svm_exit_hlt(mach, vcpu, exit); 1566 break; 1567 case VMCB_EXITCODE_IOIO: 1568 svm_exit_io(mach, vcpu, exit); 1569 break; 1570 case VMCB_EXITCODE_MSR: 1571 svm_exit_msr(mach, vcpu, exit); 1572 break; 1573 case VMCB_EXITCODE_SHUTDOWN: 1574 exit->reason = NVMM_VCPU_EXIT_SHUTDOWN; 1575 break; 1576 case VMCB_EXITCODE_RDPMC: 1577 case VMCB_EXITCODE_RSM: 1578 case VMCB_EXITCODE_INVLPGA: 1579 case VMCB_EXITCODE_VMRUN: 1580 case VMCB_EXITCODE_VMMCALL: 1581 case VMCB_EXITCODE_VMLOAD: 1582 case VMCB_EXITCODE_VMSAVE: 1583 case VMCB_EXITCODE_STGI: 1584 case VMCB_EXITCODE_CLGI: 1585 case VMCB_EXITCODE_SKINIT: 1586 case VMCB_EXITCODE_RDTSCP: 1587 case VMCB_EXITCODE_RDPRU: 1588 case VMCB_EXITCODE_INVLPGB: 1589 case VMCB_EXITCODE_INVPCID: 1590 case VMCB_EXITCODE_MCOMMIT: 1591 case VMCB_EXITCODE_TLBSYNC: 1592 svm_inject_ud(vcpu); 1593 exit->reason = NVMM_VCPU_EXIT_NONE; 1594 break; 1595 case VMCB_EXITCODE_MONITOR: 1596 svm_exit_insn(vmcb, exit, NVMM_VCPU_EXIT_MONITOR); 1597 break; 1598 case VMCB_EXITCODE_MWAIT: 1599 case VMCB_EXITCODE_MWAIT_CONDITIONAL: 1600 svm_exit_insn(vmcb, exit, NVMM_VCPU_EXIT_MWAIT); 1601 break; 1602 case VMCB_EXITCODE_XSETBV: 1603 svm_exit_xsetbv(mach, vcpu, exit); 1604 break; 1605 case VMCB_EXITCODE_NPF: 1606 svm_exit_npf(mach, vcpu, exit); 1607 break; 1608 case VMCB_EXITCODE_FERR_FREEZE: /* ? */ 1609 default: 1610 svm_exit_invalid(exit, vmcb->ctrl.exitcode); 1611 break; 1612 } 1613 1614 /* If no reason to return to userland, keep rolling. */ 1615 if (nvmm_return_needed()) { 1616 break; 1617 } 1618 if (exit->reason != NVMM_VCPU_EXIT_NONE) { 1619 break; 1620 } 1621 } 1622 1623 cpudata->gtsc = rdtsc() + vmcb->ctrl.tsc_offset; 1624 1625 svm_vcpu_guest_misc_leave(vcpu); 1626 svm_vcpu_guest_dbregs_leave(vcpu); 1627 1628 kpreempt_enable(); 1629 1630 exit->exitstate.rflags = vmcb->state.rflags; 1631 exit->exitstate.cr8 = __SHIFTOUT(vmcb->ctrl.v, VMCB_CTRL_V_TPR); 1632 exit->exitstate.int_shadow = 1633 ((vmcb->ctrl.intr & VMCB_CTRL_INTR_SHADOW) != 0); 1634 exit->exitstate.int_window_exiting = cpudata->int_window_exit; 1635 exit->exitstate.nmi_window_exiting = cpudata->nmi_window_exit; 1636 exit->exitstate.evt_pending = cpudata->evt_pending; 1637 1638 return 0; 1639} 1640 1641/* -------------------------------------------------------------------------- */ 1642 1643static int 1644svm_memalloc(paddr_t *pa, vaddr_t *va, size_t npages) 1645{ 1646 struct pglist pglist; 1647 paddr_t _pa; 1648 vaddr_t _va; 1649 size_t i; 1650 int ret; 1651 1652 ret = uvm_pglistalloc(npages * PAGE_SIZE, 0, ~0UL, PAGE_SIZE, 0, 1653 &pglist, 1, 0); 1654 if (ret != 0) 1655 return ENOMEM; 1656 _pa = VM_PAGE_TO_PHYS(TAILQ_FIRST(&pglist)); 1657 _va = uvm_km_alloc(kernel_map, npages * PAGE_SIZE, 0, 1658 UVM_KMF_VAONLY | UVM_KMF_NOWAIT); 1659 if (_va == 0) 1660 goto error; 1661 1662 for (i = 0; i < npages; i++) { 1663 pmap_kenter_pa(_va + i * PAGE_SIZE, _pa + i * PAGE_SIZE, 1664 VM_PROT_READ | VM_PROT_WRITE, PMAP_WRITE_BACK); 1665 } 1666 pmap_update(pmap_kernel()); 1667 1668 memset((void *)_va, 0, npages * PAGE_SIZE); 1669 1670 *pa = _pa; 1671 *va = _va; 1672 return 0; 1673 1674error: 1675 for (i = 0; i < npages; i++) { 1676 uvm_pagefree(PHYS_TO_VM_PAGE(_pa + i * PAGE_SIZE)); 1677 } 1678 return ENOMEM; 1679} 1680 1681static void 1682svm_memfree(paddr_t pa, vaddr_t va, size_t npages) 1683{ 1684 size_t i; 1685 1686 pmap_kremove(va, npages * PAGE_SIZE); 1687 pmap_update(pmap_kernel()); 1688 uvm_km_free(kernel_map, va, npages * PAGE_SIZE, UVM_KMF_VAONLY); 1689 for (i = 0; i < npages; i++) { 1690 uvm_pagefree(PHYS_TO_VM_PAGE(pa + i * PAGE_SIZE)); 1691 } 1692} 1693 1694/* -------------------------------------------------------------------------- */ 1695 1696#define SVM_MSRBM_READ __BIT(0) 1697#define SVM_MSRBM_WRITE __BIT(1) 1698 1699static void 1700svm_vcpu_msr_allow(uint8_t *bitmap, uint64_t msr, bool read, bool write) 1701{ 1702 uint64_t byte; 1703 uint8_t bitoff; 1704 1705 if (msr < 0x00002000) { 1706 /* Range 1 */ 1707 byte = ((msr - 0x00000000) >> 2UL) + 0x0000; 1708 } else if (msr >= 0xC0000000 && msr < 0xC0002000) { 1709 /* Range 2 */ 1710 byte = ((msr - 0xC0000000) >> 2UL) + 0x0800; 1711 } else if (msr >= 0xC0010000 && msr < 0xC0012000) { 1712 /* Range 3 */ 1713 byte = ((msr - 0xC0010000) >> 2UL) + 0x1000; 1714 } else { 1715 panic("%s: wrong range", __func__); 1716 } 1717 1718 bitoff = (msr & 0x3) << 1; 1719 1720 if (read) { 1721 bitmap[byte] &= ~(SVM_MSRBM_READ << bitoff); 1722 } 1723 if (write) { 1724 bitmap[byte] &= ~(SVM_MSRBM_WRITE << bitoff); 1725 } 1726} 1727 1728#define SVM_SEG_ATTRIB_TYPE __BITS(3,0) 1729#define SVM_SEG_ATTRIB_S __BIT(4) 1730#define SVM_SEG_ATTRIB_DPL __BITS(6,5) 1731#define SVM_SEG_ATTRIB_P __BIT(7) 1732#define SVM_SEG_ATTRIB_AVL __BIT(8) 1733#define SVM_SEG_ATTRIB_L __BIT(9) 1734#define SVM_SEG_ATTRIB_DEF __BIT(10) 1735#define SVM_SEG_ATTRIB_G __BIT(11) 1736 1737static void 1738svm_vcpu_setstate_seg(const struct nvmm_x64_state_seg *seg, 1739 struct vmcb_segment *vseg) 1740{ 1741 vseg->selector = seg->selector; 1742 vseg->attrib = 1743 __SHIFTIN(seg->attrib.type, SVM_SEG_ATTRIB_TYPE) | 1744 __SHIFTIN(seg->attrib.s, SVM_SEG_ATTRIB_S) | 1745 __SHIFTIN(seg->attrib.dpl, SVM_SEG_ATTRIB_DPL) | 1746 __SHIFTIN(seg->attrib.p, SVM_SEG_ATTRIB_P) | 1747 __SHIFTIN(seg->attrib.avl, SVM_SEG_ATTRIB_AVL) | 1748 __SHIFTIN(seg->attrib.l, SVM_SEG_ATTRIB_L) | 1749 __SHIFTIN(seg->attrib.def, SVM_SEG_ATTRIB_DEF) | 1750 __SHIFTIN(seg->attrib.g, SVM_SEG_ATTRIB_G); 1751 vseg->limit = seg->limit; 1752 vseg->base = seg->base; 1753} 1754 1755static void 1756svm_vcpu_getstate_seg(struct nvmm_x64_state_seg *seg, struct vmcb_segment *vseg) 1757{ 1758 seg->selector = vseg->selector; 1759 seg->attrib.type = __SHIFTOUT(vseg->attrib, SVM_SEG_ATTRIB_TYPE); 1760 seg->attrib.s = __SHIFTOUT(vseg->attrib, SVM_SEG_ATTRIB_S); 1761 seg->attrib.dpl = __SHIFTOUT(vseg->attrib, SVM_SEG_ATTRIB_DPL); 1762 seg->attrib.p = __SHIFTOUT(vseg->attrib, SVM_SEG_ATTRIB_P); 1763 seg->attrib.avl = __SHIFTOUT(vseg->attrib, SVM_SEG_ATTRIB_AVL); 1764 seg->attrib.l = __SHIFTOUT(vseg->attrib, SVM_SEG_ATTRIB_L); 1765 seg->attrib.def = __SHIFTOUT(vseg->attrib, SVM_SEG_ATTRIB_DEF); 1766 seg->attrib.g = __SHIFTOUT(vseg->attrib, SVM_SEG_ATTRIB_G); 1767 seg->limit = vseg->limit; 1768 seg->base = vseg->base; 1769} 1770 1771static inline bool 1772svm_state_tlb_flush(const struct vmcb *vmcb, const struct nvmm_x64_state *state, 1773 uint64_t flags) 1774{ 1775 if (flags & NVMM_X64_STATE_CRS) { 1776 if ((vmcb->state.cr0 ^ 1777 state->crs[NVMM_X64_CR_CR0]) & CR0_TLB_FLUSH) { 1778 return true; 1779 } 1780 if (vmcb->state.cr3 != state->crs[NVMM_X64_CR_CR3]) { 1781 return true; 1782 } 1783 if ((vmcb->state.cr4 ^ 1784 state->crs[NVMM_X64_CR_CR4]) & CR4_TLB_FLUSH) { 1785 return true; 1786 } 1787 } 1788 1789 if (flags & NVMM_X64_STATE_MSRS) { 1790 if ((vmcb->state.efer ^ 1791 state->msrs[NVMM_X64_MSR_EFER]) & EFER_TLB_FLUSH) { 1792 return true; 1793 } 1794 } 1795 1796 return false; 1797} 1798 1799static void 1800svm_vcpu_setstate(struct nvmm_cpu *vcpu) 1801{ 1802 struct nvmm_comm_page *comm = vcpu->comm; 1803 const struct nvmm_x64_state *state = &comm->state; 1804 struct svm_cpudata *cpudata = vcpu->cpudata; 1805 struct vmcb *vmcb = cpudata->vmcb; 1806 struct fxsave *fpustate; 1807 uint64_t flags; 1808 1809 flags = comm->state_wanted; 1810 1811 if (svm_state_tlb_flush(vmcb, state, flags)) { 1812 cpudata->gtlb_want_flush = true; 1813 } 1814 1815 if (flags & NVMM_X64_STATE_SEGS) { 1816 svm_vcpu_setstate_seg(&state->segs[NVMM_X64_SEG_CS], 1817 &vmcb->state.cs); 1818 svm_vcpu_setstate_seg(&state->segs[NVMM_X64_SEG_DS], 1819 &vmcb->state.ds); 1820 svm_vcpu_setstate_seg(&state->segs[NVMM_X64_SEG_ES], 1821 &vmcb->state.es); 1822 svm_vcpu_setstate_seg(&state->segs[NVMM_X64_SEG_FS], 1823 &vmcb->state.fs); 1824 svm_vcpu_setstate_seg(&state->segs[NVMM_X64_SEG_GS], 1825 &vmcb->state.gs); 1826 svm_vcpu_setstate_seg(&state->segs[NVMM_X64_SEG_SS], 1827 &vmcb->state.ss); 1828 svm_vcpu_setstate_seg(&state->segs[NVMM_X64_SEG_GDT], 1829 &vmcb->state.gdt); 1830 svm_vcpu_setstate_seg(&state->segs[NVMM_X64_SEG_IDT], 1831 &vmcb->state.idt); 1832 svm_vcpu_setstate_seg(&state->segs[NVMM_X64_SEG_LDT], 1833 &vmcb->state.ldt); 1834 svm_vcpu_setstate_seg(&state->segs[NVMM_X64_SEG_TR], 1835 &vmcb->state.tr); 1836 1837 vmcb->state.cpl = state->segs[NVMM_X64_SEG_SS].attrib.dpl; 1838 } 1839 1840 CTASSERT(sizeof(cpudata->gprs) == sizeof(state->gprs)); 1841 if (flags & NVMM_X64_STATE_GPRS) { 1842 memcpy(cpudata->gprs, state->gprs, sizeof(state->gprs)); 1843 1844 vmcb->state.rip = state->gprs[NVMM_X64_GPR_RIP]; 1845 vmcb->state.rsp = state->gprs[NVMM_X64_GPR_RSP]; 1846 vmcb->state.rax = state->gprs[NVMM_X64_GPR_RAX]; 1847 vmcb->state.rflags = state->gprs[NVMM_X64_GPR_RFLAGS]; 1848 } 1849 1850 if (flags & NVMM_X64_STATE_CRS) { 1851 vmcb->state.cr0 = state->crs[NVMM_X64_CR_CR0]; 1852 vmcb->state.cr2 = state->crs[NVMM_X64_CR_CR2]; 1853 vmcb->state.cr3 = state->crs[NVMM_X64_CR_CR3]; 1854 vmcb->state.cr4 = state->crs[NVMM_X64_CR_CR4]; 1855 1856 vmcb->ctrl.v &= ~VMCB_CTRL_V_TPR; 1857 vmcb->ctrl.v |= __SHIFTIN(state->crs[NVMM_X64_CR_CR8], 1858 VMCB_CTRL_V_TPR); 1859 1860 if (svm_xcr0_mask != 0) { 1861 /* Clear illegal XCR0 bits, set mandatory X87 bit. */ 1862 cpudata->gxcr0 = state->crs[NVMM_X64_CR_XCR0]; 1863 cpudata->gxcr0 &= svm_xcr0_mask; 1864 cpudata->gxcr0 |= XCR0_X87; 1865 } 1866 } 1867 1868 CTASSERT(sizeof(cpudata->drs) == sizeof(state->drs)); 1869 if (flags & NVMM_X64_STATE_DRS) { 1870 memcpy(cpudata->drs, state->drs, sizeof(state->drs)); 1871 1872 vmcb->state.dr6 = state->drs[NVMM_X64_DR_DR6]; 1873 vmcb->state.dr7 = state->drs[NVMM_X64_DR_DR7]; 1874 } 1875 1876 if (flags & NVMM_X64_STATE_MSRS) { 1877 /* 1878 * EFER_SVME is mandatory. 1879 */ 1880 vmcb->state.efer = state->msrs[NVMM_X64_MSR_EFER] | EFER_SVME; 1881 vmcb->state.star = state->msrs[NVMM_X64_MSR_STAR]; 1882 vmcb->state.lstar = state->msrs[NVMM_X64_MSR_LSTAR]; 1883 vmcb->state.cstar = state->msrs[NVMM_X64_MSR_CSTAR]; 1884 vmcb->state.sfmask = state->msrs[NVMM_X64_MSR_SFMASK]; 1885 vmcb->state.kernelgsbase = 1886 state->msrs[NVMM_X64_MSR_KERNELGSBASE]; 1887 vmcb->state.sysenter_cs = 1888 state->msrs[NVMM_X64_MSR_SYSENTER_CS]; 1889 vmcb->state.sysenter_esp = 1890 state->msrs[NVMM_X64_MSR_SYSENTER_ESP]; 1891 vmcb->state.sysenter_eip = 1892 state->msrs[NVMM_X64_MSR_SYSENTER_EIP]; 1893 vmcb->state.g_pat = state->msrs[NVMM_X64_MSR_PAT]; 1894 1895 cpudata->gtsc = state->msrs[NVMM_X64_MSR_TSC]; 1896 cpudata->gtsc_want_update = true; 1897 } 1898 1899 if (flags & NVMM_X64_STATE_INTR) { 1900 if (state->intr.int_shadow) { 1901 vmcb->ctrl.intr |= VMCB_CTRL_INTR_SHADOW; 1902 } else { 1903 vmcb->ctrl.intr &= ~VMCB_CTRL_INTR_SHADOW; 1904 } 1905 1906 if (state->intr.int_window_exiting) { 1907 svm_event_waitexit_enable(vcpu, false); 1908 } else { 1909 svm_event_waitexit_disable(vcpu, false); 1910 } 1911 1912 if (state->intr.nmi_window_exiting) { 1913 svm_event_waitexit_enable(vcpu, true); 1914 } else { 1915 svm_event_waitexit_disable(vcpu, true); 1916 } 1917 } 1918 1919 CTASSERT(sizeof(cpudata->gfpu.xsh_fxsave) == sizeof(state->fpu)); 1920 if (flags & NVMM_X64_STATE_FPU) { 1921 memcpy(cpudata->gfpu.xsh_fxsave, &state->fpu, 1922 sizeof(state->fpu)); 1923 1924 fpustate = (struct fxsave *)cpudata->gfpu.xsh_fxsave; 1925 fpustate->fx_mxcsr_mask &= x86_fpu_mxcsr_mask; 1926 fpustate->fx_mxcsr &= fpustate->fx_mxcsr_mask; 1927 1928 if (svm_xcr0_mask != 0) { 1929 /* Reset XSTATE_BV, to force a reload. */ 1930 cpudata->gfpu.xsh_xstate_bv = svm_xcr0_mask; 1931 } 1932 } 1933 1934 svm_vmcb_cache_update(vmcb, flags); 1935 1936 comm->state_wanted = 0; 1937 comm->state_cached |= flags; 1938} 1939 1940static void 1941svm_vcpu_getstate(struct nvmm_cpu *vcpu) 1942{ 1943 struct nvmm_comm_page *comm = vcpu->comm; 1944 struct nvmm_x64_state *state = &comm->state; 1945 struct svm_cpudata *cpudata = vcpu->cpudata; 1946 struct vmcb *vmcb = cpudata->vmcb; 1947 uint64_t flags; 1948 1949 flags = comm->state_wanted; 1950 1951 if (flags & NVMM_X64_STATE_SEGS) { 1952 svm_vcpu_getstate_seg(&state->segs[NVMM_X64_SEG_CS], 1953 &vmcb->state.cs); 1954 svm_vcpu_getstate_seg(&state->segs[NVMM_X64_SEG_DS], 1955 &vmcb->state.ds); 1956 svm_vcpu_getstate_seg(&state->segs[NVMM_X64_SEG_ES], 1957 &vmcb->state.es); 1958 svm_vcpu_getstate_seg(&state->segs[NVMM_X64_SEG_FS], 1959 &vmcb->state.fs); 1960 svm_vcpu_getstate_seg(&state->segs[NVMM_X64_SEG_GS], 1961 &vmcb->state.gs); 1962 svm_vcpu_getstate_seg(&state->segs[NVMM_X64_SEG_SS], 1963 &vmcb->state.ss); 1964 svm_vcpu_getstate_seg(&state->segs[NVMM_X64_SEG_GDT], 1965 &vmcb->state.gdt); 1966 svm_vcpu_getstate_seg(&state->segs[NVMM_X64_SEG_IDT], 1967 &vmcb->state.idt); 1968 svm_vcpu_getstate_seg(&state->segs[NVMM_X64_SEG_LDT], 1969 &vmcb->state.ldt); 1970 svm_vcpu_getstate_seg(&state->segs[NVMM_X64_SEG_TR], 1971 &vmcb->state.tr); 1972 1973 state->segs[NVMM_X64_SEG_SS].attrib.dpl = vmcb->state.cpl; 1974 } 1975 1976 CTASSERT(sizeof(cpudata->gprs) == sizeof(state->gprs)); 1977 if (flags & NVMM_X64_STATE_GPRS) { 1978 memcpy(state->gprs, cpudata->gprs, sizeof(state->gprs)); 1979 1980 state->gprs[NVMM_X64_GPR_RIP] = vmcb->state.rip; 1981 state->gprs[NVMM_X64_GPR_RSP] = vmcb->state.rsp; 1982 state->gprs[NVMM_X64_GPR_RAX] = vmcb->state.rax; 1983 state->gprs[NVMM_X64_GPR_RFLAGS] = vmcb->state.rflags; 1984 } 1985 1986 if (flags & NVMM_X64_STATE_CRS) { 1987 state->crs[NVMM_X64_CR_CR0] = vmcb->state.cr0; 1988 state->crs[NVMM_X64_CR_CR2] = vmcb->state.cr2; 1989 state->crs[NVMM_X64_CR_CR3] = vmcb->state.cr3; 1990 state->crs[NVMM_X64_CR_CR4] = vmcb->state.cr4; 1991 state->crs[NVMM_X64_CR_CR8] = __SHIFTOUT(vmcb->ctrl.v, 1992 VMCB_CTRL_V_TPR); 1993 state->crs[NVMM_X64_CR_XCR0] = cpudata->gxcr0; 1994 } 1995 1996 CTASSERT(sizeof(cpudata->drs) == sizeof(state->drs)); 1997 if (flags & NVMM_X64_STATE_DRS) { 1998 memcpy(state->drs, cpudata->drs, sizeof(state->drs)); 1999 2000 state->drs[NVMM_X64_DR_DR6] = vmcb->state.dr6; 2001 state->drs[NVMM_X64_DR_DR7] = vmcb->state.dr7; 2002 } 2003 2004 if (flags & NVMM_X64_STATE_MSRS) { 2005 state->msrs[NVMM_X64_MSR_EFER] = vmcb->state.efer; 2006 state->msrs[NVMM_X64_MSR_STAR] = vmcb->state.star; 2007 state->msrs[NVMM_X64_MSR_LSTAR] = vmcb->state.lstar; 2008 state->msrs[NVMM_X64_MSR_CSTAR] = vmcb->state.cstar; 2009 state->msrs[NVMM_X64_MSR_SFMASK] = vmcb->state.sfmask; 2010 state->msrs[NVMM_X64_MSR_KERNELGSBASE] = 2011 vmcb->state.kernelgsbase; 2012 state->msrs[NVMM_X64_MSR_SYSENTER_CS] = 2013 vmcb->state.sysenter_cs; 2014 state->msrs[NVMM_X64_MSR_SYSENTER_ESP] = 2015 vmcb->state.sysenter_esp; 2016 state->msrs[NVMM_X64_MSR_SYSENTER_EIP] = 2017 vmcb->state.sysenter_eip; 2018 state->msrs[NVMM_X64_MSR_PAT] = vmcb->state.g_pat; 2019 state->msrs[NVMM_X64_MSR_TSC] = cpudata->gtsc; 2020 2021 /* Hide SVME. */ 2022 state->msrs[NVMM_X64_MSR_EFER] &= ~EFER_SVME; 2023 } 2024 2025 if (flags & NVMM_X64_STATE_INTR) { 2026 state->intr.int_shadow = 2027 (vmcb->ctrl.intr & VMCB_CTRL_INTR_SHADOW) != 0; 2028 state->intr.int_window_exiting = cpudata->int_window_exit; 2029 state->intr.nmi_window_exiting = cpudata->nmi_window_exit; 2030 state->intr.evt_pending = cpudata->evt_pending; 2031 } 2032 2033 CTASSERT(sizeof(cpudata->gfpu.xsh_fxsave) == sizeof(state->fpu)); 2034 if (flags & NVMM_X64_STATE_FPU) { 2035 memcpy(&state->fpu, cpudata->gfpu.xsh_fxsave, 2036 sizeof(state->fpu)); 2037 } 2038 2039 comm->state_wanted = 0; 2040 comm->state_cached |= flags; 2041} 2042 2043static void 2044svm_vcpu_state_provide(struct nvmm_cpu *vcpu, uint64_t flags) 2045{ 2046 vcpu->comm->state_wanted = flags; 2047 svm_vcpu_getstate(vcpu); 2048} 2049 2050static void 2051svm_vcpu_state_commit(struct nvmm_cpu *vcpu) 2052{ 2053 vcpu->comm->state_wanted = vcpu->comm->state_commit; 2054 vcpu->comm->state_commit = 0; 2055 svm_vcpu_setstate(vcpu); 2056} 2057 2058/* -------------------------------------------------------------------------- */ 2059 2060static void 2061svm_asid_alloc(struct nvmm_cpu *vcpu) 2062{ 2063 struct svm_cpudata *cpudata = vcpu->cpudata; 2064 struct vmcb *vmcb = cpudata->vmcb; 2065 size_t i, oct, bit; 2066 2067 mutex_enter(&svm_asidlock); 2068 2069 for (i = 0; i < svm_maxasid; i++) { 2070 oct = i / 8; 2071 bit = i % 8; 2072 2073 if (svm_asidmap[oct] & __BIT(bit)) { 2074 continue; 2075 } 2076 2077 svm_asidmap[oct] |= __BIT(bit); 2078 vmcb->ctrl.guest_asid = i; 2079 mutex_exit(&svm_asidlock); 2080 return; 2081 } 2082 2083 /* 2084 * No free ASID. Use the last one, which is shared and requires 2085 * special TLB handling. 2086 */ 2087 cpudata->shared_asid = true; 2088 vmcb->ctrl.guest_asid = svm_maxasid - 1; 2089 mutex_exit(&svm_asidlock); 2090} 2091 2092static void 2093svm_asid_free(struct nvmm_cpu *vcpu) 2094{ 2095 struct svm_cpudata *cpudata = vcpu->cpudata; 2096 struct vmcb *vmcb = cpudata->vmcb; 2097 size_t oct, bit; 2098 2099 if (cpudata->shared_asid) { 2100 return; 2101 } 2102 2103 oct = vmcb->ctrl.guest_asid / 8; 2104 bit = vmcb->ctrl.guest_asid % 8; 2105 2106 mutex_enter(&svm_asidlock); 2107 svm_asidmap[oct] &= ~__BIT(bit); 2108 mutex_exit(&svm_asidlock); 2109} 2110 2111static void 2112svm_vcpu_init(struct nvmm_machine *mach, struct nvmm_cpu *vcpu) 2113{ 2114 struct svm_cpudata *cpudata = vcpu->cpudata; 2115 struct vmcb *vmcb = cpudata->vmcb; 2116 2117 /* Allow reads/writes of Control Registers. */ 2118 vmcb->ctrl.intercept_cr = 0; 2119 2120 /* Allow reads/writes of Debug Registers. */ 2121 vmcb->ctrl.intercept_dr = 0; 2122 2123 /* Allow exceptions 0 to 31. */ 2124 vmcb->ctrl.intercept_vec = 0; 2125 2126 /* 2127 * Allow: 2128 * - SMI [smm interrupts] 2129 * - VINTR [virtual interrupts] 2130 * - CR0_SPEC [CR0 writes changing other fields than CR0.TS or CR0.MP] 2131 * - RIDTR [reads of IDTR] 2132 * - RGDTR [reads of GDTR] 2133 * - RLDTR [reads of LDTR] 2134 * - RTR [reads of TR] 2135 * - WIDTR [writes of IDTR] 2136 * - WGDTR [writes of GDTR] 2137 * - WLDTR [writes of LDTR] 2138 * - WTR [writes of TR] 2139 * - RDTSC [rdtsc instruction] 2140 * - PUSHF [pushf instruction] 2141 * - POPF [popf instruction] 2142 * - IRET [iret instruction] 2143 * - INTN [int $n instructions] 2144 * - PAUSE [pause instruction] 2145 * - INVLPG [invplg instruction] 2146 * - TASKSW [task switches] 2147 * 2148 * Intercept the rest below. 2149 */ 2150 vmcb->ctrl.intercept_misc1 = 2151 VMCB_CTRL_INTERCEPT_INTR | 2152 VMCB_CTRL_INTERCEPT_NMI | 2153 VMCB_CTRL_INTERCEPT_INIT | 2154 VMCB_CTRL_INTERCEPT_RDPMC | 2155 VMCB_CTRL_INTERCEPT_CPUID | 2156 VMCB_CTRL_INTERCEPT_RSM | 2157 VMCB_CTRL_INTERCEPT_INVD | 2158 VMCB_CTRL_INTERCEPT_HLT | 2159 VMCB_CTRL_INTERCEPT_INVLPGA | 2160 VMCB_CTRL_INTERCEPT_IOIO_PROT | 2161 VMCB_CTRL_INTERCEPT_MSR_PROT | 2162 VMCB_CTRL_INTERCEPT_FERR_FREEZE | 2163 VMCB_CTRL_INTERCEPT_SHUTDOWN; 2164 2165 /* 2166 * Allow: 2167 * - ICEBP [icebp instruction] 2168 * - WBINVD [wbinvd instruction] 2169 * - WCR_SPEC(0..15) [writes of CR0-15, received after instruction] 2170 * 2171 * Intercept the rest below. 2172 */ 2173 vmcb->ctrl.intercept_misc2 = 2174 VMCB_CTRL_INTERCEPT_VMRUN | 2175 VMCB_CTRL_INTERCEPT_VMMCALL | 2176 VMCB_CTRL_INTERCEPT_VMLOAD | 2177 VMCB_CTRL_INTERCEPT_VMSAVE | 2178 VMCB_CTRL_INTERCEPT_STGI | 2179 VMCB_CTRL_INTERCEPT_CLGI | 2180 VMCB_CTRL_INTERCEPT_SKINIT | 2181 VMCB_CTRL_INTERCEPT_RDTSCP | 2182 VMCB_CTRL_INTERCEPT_MONITOR | 2183 VMCB_CTRL_INTERCEPT_MWAIT | 2184 VMCB_CTRL_INTERCEPT_XSETBV | 2185 VMCB_CTRL_INTERCEPT_RDPRU; 2186 2187 /* 2188 * Intercept everything. 2189 */ 2190 vmcb->ctrl.intercept_misc3 = 2191 VMCB_CTRL_INTERCEPT_INVLPGB_ALL | 2192 VMCB_CTRL_INTERCEPT_PCID | 2193 VMCB_CTRL_INTERCEPT_MCOMMIT | 2194 VMCB_CTRL_INTERCEPT_TLBSYNC; 2195 2196 /* Intercept all I/O accesses. */ 2197 memset(cpudata->iobm, 0xFF, IOBM_SIZE); 2198 vmcb->ctrl.iopm_base_pa = cpudata->iobm_pa; 2199 2200 /* Allow direct access to certain MSRs. */ 2201 memset(cpudata->msrbm, 0xFF, MSRBM_SIZE); 2202 svm_vcpu_msr_allow(cpudata->msrbm, MSR_STAR, true, true); 2203 svm_vcpu_msr_allow(cpudata->msrbm, MSR_LSTAR, true, true); 2204 svm_vcpu_msr_allow(cpudata->msrbm, MSR_CSTAR, true, true); 2205 svm_vcpu_msr_allow(cpudata->msrbm, MSR_SFMASK, true, true); 2206 svm_vcpu_msr_allow(cpudata->msrbm, MSR_KERNELGSBASE, true, true); 2207 svm_vcpu_msr_allow(cpudata->msrbm, MSR_SYSENTER_CS, true, true); 2208 svm_vcpu_msr_allow(cpudata->msrbm, MSR_SYSENTER_ESP, true, true); 2209 svm_vcpu_msr_allow(cpudata->msrbm, MSR_SYSENTER_EIP, true, true); 2210 svm_vcpu_msr_allow(cpudata->msrbm, MSR_FSBASE, true, true); 2211 svm_vcpu_msr_allow(cpudata->msrbm, MSR_GSBASE, true, true); 2212 svm_vcpu_msr_allow(cpudata->msrbm, MSR_CR_PAT, true, true); 2213 svm_vcpu_msr_allow(cpudata->msrbm, MSR_TSC, true, false); 2214 vmcb->ctrl.msrpm_base_pa = cpudata->msrbm_pa; 2215 2216 /* Generate ASID. */ 2217 svm_asid_alloc(vcpu); 2218 2219 /* Virtual TPR. */ 2220 vmcb->ctrl.v = VMCB_CTRL_V_INTR_MASKING; 2221 2222 /* Enable Nested Paging. */ 2223 vmcb->ctrl.enable1 = VMCB_CTRL_ENABLE_NP; 2224 vmcb->ctrl.n_cr3 = mach->vm->vm_map.pmap->pm_pdirpa[0]; 2225 2226 /* Init XSAVE header. */ 2227 cpudata->gfpu.xsh_xstate_bv = svm_xcr0_mask; 2228 cpudata->gfpu.xsh_xcomp_bv = 0; 2229 2230 /* These MSRs are static. */ 2231 cpudata->star = rdmsr(MSR_STAR); 2232 cpudata->lstar = rdmsr(MSR_LSTAR); 2233 cpudata->cstar = rdmsr(MSR_CSTAR); 2234 cpudata->sfmask = rdmsr(MSR_SFMASK); 2235 2236 /* Install the RESET state. */ 2237 memcpy(&vcpu->comm->state, &nvmm_x86_reset_state, 2238 sizeof(nvmm_x86_reset_state)); 2239 vcpu->comm->state_wanted = NVMM_X64_STATE_ALL; 2240 vcpu->comm->state_cached = 0; 2241 svm_vcpu_setstate(vcpu); 2242} 2243 2244static int 2245svm_vcpu_create(struct nvmm_machine *mach, struct nvmm_cpu *vcpu) 2246{ 2247 struct svm_cpudata *cpudata; 2248 int error; 2249 2250 /* Allocate the SVM cpudata. */ 2251 cpudata = (struct svm_cpudata *)uvm_km_alloc(kernel_map, 2252 roundup(sizeof(*cpudata), PAGE_SIZE), 0, 2253 UVM_KMF_WIRED|UVM_KMF_ZERO); 2254 vcpu->cpudata = cpudata; 2255 2256 /* VMCB */ 2257 error = svm_memalloc(&cpudata->vmcb_pa, (vaddr_t *)&cpudata->vmcb, 2258 VMCB_NPAGES); 2259 if (error) 2260 goto error; 2261 2262 /* I/O Bitmap */ 2263 error = svm_memalloc(&cpudata->iobm_pa, (vaddr_t *)&cpudata->iobm, 2264 IOBM_NPAGES); 2265 if (error) 2266 goto error; 2267 2268 /* MSR Bitmap */ 2269 error = svm_memalloc(&cpudata->msrbm_pa, (vaddr_t *)&cpudata->msrbm, 2270 MSRBM_NPAGES); 2271 if (error) 2272 goto error; 2273 2274 /* Init the VCPU info. */ 2275 svm_vcpu_init(mach, vcpu); 2276 2277 return 0; 2278 2279error: 2280 if (cpudata->vmcb_pa) { 2281 svm_memfree(cpudata->vmcb_pa, (vaddr_t)cpudata->vmcb, 2282 VMCB_NPAGES); 2283 } 2284 if (cpudata->iobm_pa) { 2285 svm_memfree(cpudata->iobm_pa, (vaddr_t)cpudata->iobm, 2286 IOBM_NPAGES); 2287 } 2288 if (cpudata->msrbm_pa) { 2289 svm_memfree(cpudata->msrbm_pa, (vaddr_t)cpudata->msrbm, 2290 MSRBM_NPAGES); 2291 } 2292 uvm_km_free(kernel_map, (vaddr_t)cpudata, 2293 roundup(sizeof(*cpudata), PAGE_SIZE), UVM_KMF_WIRED); 2294 return error; 2295} 2296 2297static void 2298svm_vcpu_destroy(struct nvmm_machine *mach, struct nvmm_cpu *vcpu) 2299{ 2300 struct svm_cpudata *cpudata = vcpu->cpudata; 2301 2302 svm_asid_free(vcpu); 2303 2304 svm_memfree(cpudata->vmcb_pa, (vaddr_t)cpudata->vmcb, VMCB_NPAGES); 2305 svm_memfree(cpudata->iobm_pa, (vaddr_t)cpudata->iobm, IOBM_NPAGES); 2306 svm_memfree(cpudata->msrbm_pa, (vaddr_t)cpudata->msrbm, MSRBM_NPAGES); 2307 2308 uvm_km_free(kernel_map, (vaddr_t)cpudata, 2309 roundup(sizeof(*cpudata), PAGE_SIZE), UVM_KMF_WIRED); 2310} 2311 2312/* -------------------------------------------------------------------------- */ 2313 2314static int 2315svm_vcpu_configure_cpuid(struct svm_cpudata *cpudata, void *data) 2316{ 2317 struct nvmm_vcpu_conf_cpuid *cpuid = data; 2318 size_t i; 2319 2320 if (__predict_false(cpuid->mask && cpuid->exit)) { 2321 return EINVAL; 2322 } 2323 if (__predict_false(cpuid->mask && 2324 ((cpuid->u.mask.set.eax & cpuid->u.mask.del.eax) || 2325 (cpuid->u.mask.set.ebx & cpuid->u.mask.del.ebx) || 2326 (cpuid->u.mask.set.ecx & cpuid->u.mask.del.ecx) || 2327 (cpuid->u.mask.set.edx & cpuid->u.mask.del.edx)))) { 2328 return EINVAL; 2329 } 2330 2331 /* If unset, delete, to restore the default behavior. */ 2332 if (!cpuid->mask && !cpuid->exit) { 2333 for (i = 0; i < SVM_NCPUIDS; i++) { 2334 if (!cpudata->cpuidpresent[i]) { 2335 continue; 2336 } 2337 if (cpudata->cpuid[i].leaf == cpuid->leaf) { 2338 cpudata->cpuidpresent[i] = false; 2339 } 2340 } 2341 return 0; 2342 } 2343 2344 /* If already here, replace. */ 2345 for (i = 0; i < SVM_NCPUIDS; i++) { 2346 if (!cpudata->cpuidpresent[i]) { 2347 continue; 2348 } 2349 if (cpudata->cpuid[i].leaf == cpuid->leaf) { 2350 memcpy(&cpudata->cpuid[i], cpuid, 2351 sizeof(struct nvmm_vcpu_conf_cpuid)); 2352 return 0; 2353 } 2354 } 2355 2356 /* Not here, insert. */ 2357 for (i = 0; i < SVM_NCPUIDS; i++) { 2358 if (!cpudata->cpuidpresent[i]) { 2359 cpudata->cpuidpresent[i] = true; 2360 memcpy(&cpudata->cpuid[i], cpuid, 2361 sizeof(struct nvmm_vcpu_conf_cpuid)); 2362 return 0; 2363 } 2364 } 2365 2366 return ENOBUFS; 2367} 2368 2369static int 2370svm_vcpu_configure(struct nvmm_cpu *vcpu, uint64_t op, void *data) 2371{ 2372 struct svm_cpudata *cpudata = vcpu->cpudata; 2373 2374 switch (op) { 2375 case NVMM_VCPU_CONF_MD(NVMM_VCPU_CONF_CPUID): 2376 return svm_vcpu_configure_cpuid(cpudata, data); 2377 default: 2378 return EINVAL; 2379 } 2380} 2381 2382/* -------------------------------------------------------------------------- */ 2383 2384static void 2385svm_tlb_flush(struct pmap *pm) 2386{ 2387 struct nvmm_machine *mach = pm->pm_data; 2388 struct svm_machdata *machdata = mach->machdata; 2389 2390 atomic_inc_64(&machdata->mach_htlb_gen); 2391 2392 /* Generates IPIs, which cause #VMEXITs. */ 2393 pmap_tlb_shootdown(pmap_kernel(), -1, PTE_G, TLBSHOOT_NVMM); 2394} 2395 2396static void 2397svm_machine_create(struct nvmm_machine *mach) 2398{ 2399 struct svm_machdata *machdata; 2400 2401 /* Fill in pmap info. */ 2402 mach->vm->vm_map.pmap->pm_data = (void *)mach; 2403 mach->vm->vm_map.pmap->pm_tlb_flush = svm_tlb_flush; 2404 2405 machdata = kmem_zalloc(sizeof(struct svm_machdata), KM_SLEEP); 2406 mach->machdata = machdata; 2407 2408 /* Start with an hTLB flush everywhere. */ 2409 machdata->mach_htlb_gen = 1; 2410} 2411 2412static void 2413svm_machine_destroy(struct nvmm_machine *mach) 2414{ 2415 kmem_free(mach->machdata, sizeof(struct svm_machdata)); 2416} 2417 2418static int 2419svm_machine_configure(struct nvmm_machine *mach, uint64_t op, void *data) 2420{ 2421 panic("%s: impossible", __func__); 2422} 2423 2424/* -------------------------------------------------------------------------- */ 2425 2426static bool 2427svm_ident(void) 2428{ 2429 u_int descs[4]; 2430 uint64_t msr; 2431 2432 if (cpu_vendor != CPUVENDOR_AMD) { 2433 return false; 2434 } 2435 if (!(cpu_feature[3] & CPUID_SVM)) { 2436 printf("NVMM: SVM not supported\n"); 2437 return false; 2438 } 2439 2440 if (curcpu()->ci_max_ext_cpuid < 0x8000000a) { 2441 printf("NVMM: CPUID leaf not available\n"); 2442 return false; 2443 } 2444 x86_cpuid(0x8000000a, descs); 2445 2446 /* Expect revision 1. */ 2447 if (__SHIFTOUT(descs[0], CPUID_AMD_SVM_REV) != 1) { 2448 printf("NVMM: SVM revision not supported\n"); 2449 return false; 2450 } 2451 2452 /* Want Nested Paging. */ 2453 if (!(descs[3] & CPUID_AMD_SVM_NP)) { 2454 printf("NVMM: SVM-NP not supported\n"); 2455 return false; 2456 } 2457 2458 /* Want nRIP. */ 2459 if (!(descs[3] & CPUID_AMD_SVM_NRIPS)) { 2460 printf("NVMM: SVM-NRIPS not supported\n"); 2461 return false; 2462 } 2463 2464 svm_decode_assist = (descs[3] & CPUID_AMD_SVM_DecodeAssist) != 0; 2465 2466 msr = rdmsr(MSR_VMCR); 2467 if ((msr & VMCR_SVMED) && (msr & VMCR_LOCK)) { 2468 printf("NVMM: SVM disabled in BIOS\n"); 2469 return false; 2470 } 2471 2472 return true; 2473} 2474 2475static void 2476svm_init_asid(uint32_t maxasid) 2477{ 2478 size_t i, j, allocsz; 2479 2480 mutex_init(&svm_asidlock, MUTEX_DEFAULT, IPL_NONE); 2481 2482 /* Arbitrarily limit. */ 2483 maxasid = uimin(maxasid, 8192); 2484 2485 svm_maxasid = maxasid; 2486 allocsz = roundup(maxasid, 8) / 8; 2487 svm_asidmap = kmem_zalloc(allocsz, KM_SLEEP); 2488 2489 /* ASID 0 is reserved for the host. */ 2490 svm_asidmap[0] |= __BIT(0); 2491 2492 /* ASID n-1 is special, we share it. */ 2493 i = (maxasid - 1) / 8; 2494 j = (maxasid - 1) % 8; 2495 svm_asidmap[i] |= __BIT(j); 2496} 2497 2498static void 2499svm_change_cpu(void *arg1, void *arg2) 2500{ 2501 bool enable = arg1 != NULL; 2502 uint64_t msr; 2503 2504 msr = rdmsr(MSR_VMCR); 2505 if (msr & VMCR_SVMED) { 2506 wrmsr(MSR_VMCR, msr & ~VMCR_SVMED); 2507 } 2508 2509 if (!enable) { 2510 wrmsr(MSR_VM_HSAVE_PA, 0); 2511 } 2512 2513 msr = rdmsr(MSR_EFER); 2514 if (enable) { 2515 msr |= EFER_SVME; 2516 } else { 2517 msr &= ~EFER_SVME; 2518 } 2519 wrmsr(MSR_EFER, msr); 2520 2521 if (enable) { 2522 wrmsr(MSR_VM_HSAVE_PA, hsave[cpu_index(curcpu())].pa); 2523 } 2524} 2525 2526static void 2527svm_init(void) 2528{ 2529 CPU_INFO_ITERATOR cii; 2530 struct cpu_info *ci; 2531 struct vm_page *pg; 2532 u_int descs[4]; 2533 uint64_t xc; 2534 2535 x86_cpuid(0x8000000a, descs); 2536 2537 /* The guest TLB flush command. */ 2538 if (descs[3] & CPUID_AMD_SVM_FlushByASID) { 2539 svm_ctrl_tlb_flush = VMCB_CTRL_TLB_CTRL_FLUSH_GUEST; 2540 } else { 2541 svm_ctrl_tlb_flush = VMCB_CTRL_TLB_CTRL_FLUSH_ALL; 2542 } 2543 2544 /* Init the ASID. */ 2545 svm_init_asid(descs[1]); 2546 2547 /* Init the XCR0 mask. */ 2548 svm_xcr0_mask = SVM_XCR0_MASK_DEFAULT & x86_xsave_features; 2549 2550 /* Init the max basic CPUID leaf. */ 2551 svm_cpuid_max_basic = uimin(cpuid_level, SVM_CPUID_MAX_BASIC); 2552 2553 /* Init the max extended CPUID leaf. */ 2554 x86_cpuid(0x80000000, descs); 2555 svm_cpuid_max_extended = uimin(descs[0], SVM_CPUID_MAX_EXTENDED); 2556 2557 memset(hsave, 0, sizeof(hsave)); 2558 for (CPU_INFO_FOREACH(cii, ci)) { 2559 pg = uvm_pagealloc(NULL, 0, NULL, UVM_PGA_ZERO); 2560 hsave[cpu_index(ci)].pa = VM_PAGE_TO_PHYS(pg); 2561 } 2562 2563 xc = xc_broadcast(0, svm_change_cpu, (void *)true, NULL); 2564 xc_wait(xc); 2565} 2566 2567static void 2568svm_fini_asid(void) 2569{ 2570 size_t allocsz; 2571 2572 allocsz = roundup(svm_maxasid, 8) / 8; 2573 kmem_free(svm_asidmap, allocsz); 2574 2575 mutex_destroy(&svm_asidlock); 2576} 2577 2578static void 2579svm_fini(void) 2580{ 2581 uint64_t xc; 2582 size_t i; 2583 2584 xc = xc_broadcast(0, svm_change_cpu, (void *)false, NULL); 2585 xc_wait(xc); 2586 2587 for (i = 0; i < MAXCPUS; i++) { 2588 if (hsave[i].pa != 0) 2589 uvm_pagefree(PHYS_TO_VM_PAGE(hsave[i].pa)); 2590 } 2591 2592 svm_fini_asid(); 2593} 2594 2595static void 2596svm_capability(struct nvmm_capability *cap) 2597{ 2598 cap->arch.mach_conf_support = 0; 2599 cap->arch.vcpu_conf_support = 2600 NVMM_CAP_ARCH_VCPU_CONF_CPUID; 2601 cap->arch.xcr0_mask = svm_xcr0_mask; 2602 cap->arch.mxcsr_mask = x86_fpu_mxcsr_mask; 2603 cap->arch.conf_cpuid_maxops = SVM_NCPUIDS; 2604} 2605 2606const struct nvmm_impl nvmm_x86_svm = { 2607 .name = "x86-svm", 2608 .ident = svm_ident, 2609 .init = svm_init, 2610 .fini = svm_fini, 2611 .capability = svm_capability, 2612 .mach_conf_max = NVMM_X86_MACH_NCONF, 2613 .mach_conf_sizes = NULL, 2614 .vcpu_conf_max = NVMM_X86_VCPU_NCONF, 2615 .vcpu_conf_sizes = svm_vcpu_conf_sizes, 2616 .state_size = sizeof(struct nvmm_x64_state), 2617 .machine_create = svm_machine_create, 2618 .machine_destroy = svm_machine_destroy, 2619 .machine_configure = svm_machine_configure, 2620 .vcpu_create = svm_vcpu_create, 2621 .vcpu_destroy = svm_vcpu_destroy, 2622 .vcpu_configure = svm_vcpu_configure, 2623 .vcpu_setstate = svm_vcpu_setstate, 2624 .vcpu_getstate = svm_vcpu_getstate, 2625 .vcpu_inject = svm_vcpu_inject, 2626 .vcpu_run = svm_vcpu_run 2627}; 2628