nvmm_x86_svm.c revision 1.23
1/* $NetBSD: nvmm_x86_svm.c,v 1.23 2019/02/14 14:30:20 maxv Exp $ */ 2 3/* 4 * Copyright (c) 2018 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Maxime Villard. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32#include <sys/cdefs.h> 33__KERNEL_RCSID(0, "$NetBSD: nvmm_x86_svm.c,v 1.23 2019/02/14 14:30:20 maxv Exp $"); 34 35#include <sys/param.h> 36#include <sys/systm.h> 37#include <sys/kernel.h> 38#include <sys/kmem.h> 39#include <sys/cpu.h> 40#include <sys/xcall.h> 41 42#include <uvm/uvm.h> 43#include <uvm/uvm_page.h> 44 45#include <x86/cputypes.h> 46#include <x86/specialreg.h> 47#include <x86/pmap.h> 48#include <x86/dbregs.h> 49#include <machine/cpuvar.h> 50 51#include <dev/nvmm/nvmm.h> 52#include <dev/nvmm/nvmm_internal.h> 53#include <dev/nvmm/x86/nvmm_x86.h> 54 55int svm_vmrun(paddr_t, uint64_t *); 56 57#define MSR_VM_HSAVE_PA 0xC0010117 58 59/* -------------------------------------------------------------------------- */ 60 61#define VMCB_EXITCODE_CR0_READ 0x0000 62#define VMCB_EXITCODE_CR1_READ 0x0001 63#define VMCB_EXITCODE_CR2_READ 0x0002 64#define VMCB_EXITCODE_CR3_READ 0x0003 65#define VMCB_EXITCODE_CR4_READ 0x0004 66#define VMCB_EXITCODE_CR5_READ 0x0005 67#define VMCB_EXITCODE_CR6_READ 0x0006 68#define VMCB_EXITCODE_CR7_READ 0x0007 69#define VMCB_EXITCODE_CR8_READ 0x0008 70#define VMCB_EXITCODE_CR9_READ 0x0009 71#define VMCB_EXITCODE_CR10_READ 0x000A 72#define VMCB_EXITCODE_CR11_READ 0x000B 73#define VMCB_EXITCODE_CR12_READ 0x000C 74#define VMCB_EXITCODE_CR13_READ 0x000D 75#define VMCB_EXITCODE_CR14_READ 0x000E 76#define VMCB_EXITCODE_CR15_READ 0x000F 77#define VMCB_EXITCODE_CR0_WRITE 0x0010 78#define VMCB_EXITCODE_CR1_WRITE 0x0011 79#define VMCB_EXITCODE_CR2_WRITE 0x0012 80#define VMCB_EXITCODE_CR3_WRITE 0x0013 81#define VMCB_EXITCODE_CR4_WRITE 0x0014 82#define VMCB_EXITCODE_CR5_WRITE 0x0015 83#define VMCB_EXITCODE_CR6_WRITE 0x0016 84#define VMCB_EXITCODE_CR7_WRITE 0x0017 85#define VMCB_EXITCODE_CR8_WRITE 0x0018 86#define VMCB_EXITCODE_CR9_WRITE 0x0019 87#define VMCB_EXITCODE_CR10_WRITE 0x001A 88#define VMCB_EXITCODE_CR11_WRITE 0x001B 89#define VMCB_EXITCODE_CR12_WRITE 0x001C 90#define VMCB_EXITCODE_CR13_WRITE 0x001D 91#define VMCB_EXITCODE_CR14_WRITE 0x001E 92#define VMCB_EXITCODE_CR15_WRITE 0x001F 93#define VMCB_EXITCODE_DR0_READ 0x0020 94#define VMCB_EXITCODE_DR1_READ 0x0021 95#define VMCB_EXITCODE_DR2_READ 0x0022 96#define VMCB_EXITCODE_DR3_READ 0x0023 97#define VMCB_EXITCODE_DR4_READ 0x0024 98#define VMCB_EXITCODE_DR5_READ 0x0025 99#define VMCB_EXITCODE_DR6_READ 0x0026 100#define VMCB_EXITCODE_DR7_READ 0x0027 101#define VMCB_EXITCODE_DR8_READ 0x0028 102#define VMCB_EXITCODE_DR9_READ 0x0029 103#define VMCB_EXITCODE_DR10_READ 0x002A 104#define VMCB_EXITCODE_DR11_READ 0x002B 105#define VMCB_EXITCODE_DR12_READ 0x002C 106#define VMCB_EXITCODE_DR13_READ 0x002D 107#define VMCB_EXITCODE_DR14_READ 0x002E 108#define VMCB_EXITCODE_DR15_READ 0x002F 109#define VMCB_EXITCODE_DR0_WRITE 0x0030 110#define VMCB_EXITCODE_DR1_WRITE 0x0031 111#define VMCB_EXITCODE_DR2_WRITE 0x0032 112#define VMCB_EXITCODE_DR3_WRITE 0x0033 113#define VMCB_EXITCODE_DR4_WRITE 0x0034 114#define VMCB_EXITCODE_DR5_WRITE 0x0035 115#define VMCB_EXITCODE_DR6_WRITE 0x0036 116#define VMCB_EXITCODE_DR7_WRITE 0x0037 117#define VMCB_EXITCODE_DR8_WRITE 0x0038 118#define VMCB_EXITCODE_DR9_WRITE 0x0039 119#define VMCB_EXITCODE_DR10_WRITE 0x003A 120#define VMCB_EXITCODE_DR11_WRITE 0x003B 121#define VMCB_EXITCODE_DR12_WRITE 0x003C 122#define VMCB_EXITCODE_DR13_WRITE 0x003D 123#define VMCB_EXITCODE_DR14_WRITE 0x003E 124#define VMCB_EXITCODE_DR15_WRITE 0x003F 125#define VMCB_EXITCODE_EXCP0 0x0040 126#define VMCB_EXITCODE_EXCP1 0x0041 127#define VMCB_EXITCODE_EXCP2 0x0042 128#define VMCB_EXITCODE_EXCP3 0x0043 129#define VMCB_EXITCODE_EXCP4 0x0044 130#define VMCB_EXITCODE_EXCP5 0x0045 131#define VMCB_EXITCODE_EXCP6 0x0046 132#define VMCB_EXITCODE_EXCP7 0x0047 133#define VMCB_EXITCODE_EXCP8 0x0048 134#define VMCB_EXITCODE_EXCP9 0x0049 135#define VMCB_EXITCODE_EXCP10 0x004A 136#define VMCB_EXITCODE_EXCP11 0x004B 137#define VMCB_EXITCODE_EXCP12 0x004C 138#define VMCB_EXITCODE_EXCP13 0x004D 139#define VMCB_EXITCODE_EXCP14 0x004E 140#define VMCB_EXITCODE_EXCP15 0x004F 141#define VMCB_EXITCODE_EXCP16 0x0050 142#define VMCB_EXITCODE_EXCP17 0x0051 143#define VMCB_EXITCODE_EXCP18 0x0052 144#define VMCB_EXITCODE_EXCP19 0x0053 145#define VMCB_EXITCODE_EXCP20 0x0054 146#define VMCB_EXITCODE_EXCP21 0x0055 147#define VMCB_EXITCODE_EXCP22 0x0056 148#define VMCB_EXITCODE_EXCP23 0x0057 149#define VMCB_EXITCODE_EXCP24 0x0058 150#define VMCB_EXITCODE_EXCP25 0x0059 151#define VMCB_EXITCODE_EXCP26 0x005A 152#define VMCB_EXITCODE_EXCP27 0x005B 153#define VMCB_EXITCODE_EXCP28 0x005C 154#define VMCB_EXITCODE_EXCP29 0x005D 155#define VMCB_EXITCODE_EXCP30 0x005E 156#define VMCB_EXITCODE_EXCP31 0x005F 157#define VMCB_EXITCODE_INTR 0x0060 158#define VMCB_EXITCODE_NMI 0x0061 159#define VMCB_EXITCODE_SMI 0x0062 160#define VMCB_EXITCODE_INIT 0x0063 161#define VMCB_EXITCODE_VINTR 0x0064 162#define VMCB_EXITCODE_CR0_SEL_WRITE 0x0065 163#define VMCB_EXITCODE_IDTR_READ 0x0066 164#define VMCB_EXITCODE_GDTR_READ 0x0067 165#define VMCB_EXITCODE_LDTR_READ 0x0068 166#define VMCB_EXITCODE_TR_READ 0x0069 167#define VMCB_EXITCODE_IDTR_WRITE 0x006A 168#define VMCB_EXITCODE_GDTR_WRITE 0x006B 169#define VMCB_EXITCODE_LDTR_WRITE 0x006C 170#define VMCB_EXITCODE_TR_WRITE 0x006D 171#define VMCB_EXITCODE_RDTSC 0x006E 172#define VMCB_EXITCODE_RDPMC 0x006F 173#define VMCB_EXITCODE_PUSHF 0x0070 174#define VMCB_EXITCODE_POPF 0x0071 175#define VMCB_EXITCODE_CPUID 0x0072 176#define VMCB_EXITCODE_RSM 0x0073 177#define VMCB_EXITCODE_IRET 0x0074 178#define VMCB_EXITCODE_SWINT 0x0075 179#define VMCB_EXITCODE_INVD 0x0076 180#define VMCB_EXITCODE_PAUSE 0x0077 181#define VMCB_EXITCODE_HLT 0x0078 182#define VMCB_EXITCODE_INVLPG 0x0079 183#define VMCB_EXITCODE_INVLPGA 0x007A 184#define VMCB_EXITCODE_IOIO 0x007B 185#define VMCB_EXITCODE_MSR 0x007C 186#define VMCB_EXITCODE_TASK_SWITCH 0x007D 187#define VMCB_EXITCODE_FERR_FREEZE 0x007E 188#define VMCB_EXITCODE_SHUTDOWN 0x007F 189#define VMCB_EXITCODE_VMRUN 0x0080 190#define VMCB_EXITCODE_VMMCALL 0x0081 191#define VMCB_EXITCODE_VMLOAD 0x0082 192#define VMCB_EXITCODE_VMSAVE 0x0083 193#define VMCB_EXITCODE_STGI 0x0084 194#define VMCB_EXITCODE_CLGI 0x0085 195#define VMCB_EXITCODE_SKINIT 0x0086 196#define VMCB_EXITCODE_RDTSCP 0x0087 197#define VMCB_EXITCODE_ICEBP 0x0088 198#define VMCB_EXITCODE_WBINVD 0x0089 199#define VMCB_EXITCODE_MONITOR 0x008A 200#define VMCB_EXITCODE_MWAIT 0x008B 201#define VMCB_EXITCODE_MWAIT_CONDITIONAL 0x008C 202#define VMCB_EXITCODE_XSETBV 0x008D 203#define VMCB_EXITCODE_EFER_WRITE_TRAP 0x008F 204#define VMCB_EXITCODE_CR0_WRITE_TRAP 0x0090 205#define VMCB_EXITCODE_CR1_WRITE_TRAP 0x0091 206#define VMCB_EXITCODE_CR2_WRITE_TRAP 0x0092 207#define VMCB_EXITCODE_CR3_WRITE_TRAP 0x0093 208#define VMCB_EXITCODE_CR4_WRITE_TRAP 0x0094 209#define VMCB_EXITCODE_CR5_WRITE_TRAP 0x0095 210#define VMCB_EXITCODE_CR6_WRITE_TRAP 0x0096 211#define VMCB_EXITCODE_CR7_WRITE_TRAP 0x0097 212#define VMCB_EXITCODE_CR8_WRITE_TRAP 0x0098 213#define VMCB_EXITCODE_CR9_WRITE_TRAP 0x0099 214#define VMCB_EXITCODE_CR10_WRITE_TRAP 0x009A 215#define VMCB_EXITCODE_CR11_WRITE_TRAP 0x009B 216#define VMCB_EXITCODE_CR12_WRITE_TRAP 0x009C 217#define VMCB_EXITCODE_CR13_WRITE_TRAP 0x009D 218#define VMCB_EXITCODE_CR14_WRITE_TRAP 0x009E 219#define VMCB_EXITCODE_CR15_WRITE_TRAP 0x009F 220#define VMCB_EXITCODE_NPF 0x0400 221#define VMCB_EXITCODE_AVIC_INCOMP_IPI 0x0401 222#define VMCB_EXITCODE_AVIC_NOACCEL 0x0402 223#define VMCB_EXITCODE_VMGEXIT 0x0403 224#define VMCB_EXITCODE_INVALID -1 225 226/* -------------------------------------------------------------------------- */ 227 228struct vmcb_ctrl { 229 uint32_t intercept_cr; 230#define VMCB_CTRL_INTERCEPT_RCR(x) __BIT( 0 + x) 231#define VMCB_CTRL_INTERCEPT_WCR(x) __BIT(16 + x) 232 233 uint32_t intercept_dr; 234#define VMCB_CTRL_INTERCEPT_RDR(x) __BIT( 0 + x) 235#define VMCB_CTRL_INTERCEPT_WDR(x) __BIT(16 + x) 236 237 uint32_t intercept_vec; 238#define VMCB_CTRL_INTERCEPT_VEC(x) __BIT(x) 239 240 uint32_t intercept_misc1; 241#define VMCB_CTRL_INTERCEPT_INTR __BIT(0) 242#define VMCB_CTRL_INTERCEPT_NMI __BIT(1) 243#define VMCB_CTRL_INTERCEPT_SMI __BIT(2) 244#define VMCB_CTRL_INTERCEPT_INIT __BIT(3) 245#define VMCB_CTRL_INTERCEPT_VINTR __BIT(4) 246#define VMCB_CTRL_INTERCEPT_CR0_SPEC __BIT(5) 247#define VMCB_CTRL_INTERCEPT_RIDTR __BIT(6) 248#define VMCB_CTRL_INTERCEPT_RGDTR __BIT(7) 249#define VMCB_CTRL_INTERCEPT_RLDTR __BIT(8) 250#define VMCB_CTRL_INTERCEPT_RTR __BIT(9) 251#define VMCB_CTRL_INTERCEPT_WIDTR __BIT(10) 252#define VMCB_CTRL_INTERCEPT_WGDTR __BIT(11) 253#define VMCB_CTRL_INTERCEPT_WLDTR __BIT(12) 254#define VMCB_CTRL_INTERCEPT_WTR __BIT(13) 255#define VMCB_CTRL_INTERCEPT_RDTSC __BIT(14) 256#define VMCB_CTRL_INTERCEPT_RDPMC __BIT(15) 257#define VMCB_CTRL_INTERCEPT_PUSHF __BIT(16) 258#define VMCB_CTRL_INTERCEPT_POPF __BIT(17) 259#define VMCB_CTRL_INTERCEPT_CPUID __BIT(18) 260#define VMCB_CTRL_INTERCEPT_RSM __BIT(19) 261#define VMCB_CTRL_INTERCEPT_IRET __BIT(20) 262#define VMCB_CTRL_INTERCEPT_INTN __BIT(21) 263#define VMCB_CTRL_INTERCEPT_INVD __BIT(22) 264#define VMCB_CTRL_INTERCEPT_PAUSE __BIT(23) 265#define VMCB_CTRL_INTERCEPT_HLT __BIT(24) 266#define VMCB_CTRL_INTERCEPT_INVLPG __BIT(25) 267#define VMCB_CTRL_INTERCEPT_INVLPGA __BIT(26) 268#define VMCB_CTRL_INTERCEPT_IOIO_PROT __BIT(27) 269#define VMCB_CTRL_INTERCEPT_MSR_PROT __BIT(28) 270#define VMCB_CTRL_INTERCEPT_TASKSW __BIT(29) 271#define VMCB_CTRL_INTERCEPT_FERR_FREEZE __BIT(30) 272#define VMCB_CTRL_INTERCEPT_SHUTDOWN __BIT(31) 273 274 uint32_t intercept_misc2; 275#define VMCB_CTRL_INTERCEPT_VMRUN __BIT(0) 276#define VMCB_CTRL_INTERCEPT_VMMCALL __BIT(1) 277#define VMCB_CTRL_INTERCEPT_VMLOAD __BIT(2) 278#define VMCB_CTRL_INTERCEPT_VMSAVE __BIT(3) 279#define VMCB_CTRL_INTERCEPT_STGI __BIT(4) 280#define VMCB_CTRL_INTERCEPT_CLGI __BIT(5) 281#define VMCB_CTRL_INTERCEPT_SKINIT __BIT(6) 282#define VMCB_CTRL_INTERCEPT_RDTSCP __BIT(7) 283#define VMCB_CTRL_INTERCEPT_ICEBP __BIT(8) 284#define VMCB_CTRL_INTERCEPT_WBINVD __BIT(9) 285#define VMCB_CTRL_INTERCEPT_MONITOR __BIT(10) 286#define VMCB_CTRL_INTERCEPT_MWAIT __BIT(12) 287#define VMCB_CTRL_INTERCEPT_XSETBV __BIT(13) 288#define VMCB_CTRL_INTERCEPT_EFER_SPEC __BIT(15) 289#define VMCB_CTRL_INTERCEPT_WCR_SPEC(x) __BIT(16 + x) 290 291 uint8_t rsvd1[40]; 292 uint16_t pause_filt_thresh; 293 uint16_t pause_filt_cnt; 294 uint64_t iopm_base_pa; 295 uint64_t msrpm_base_pa; 296 uint64_t tsc_offset; 297 uint32_t guest_asid; 298 299 uint32_t tlb_ctrl; 300#define VMCB_CTRL_TLB_CTRL_FLUSH_ALL 0x01 301#define VMCB_CTRL_TLB_CTRL_FLUSH_GUEST 0x03 302#define VMCB_CTRL_TLB_CTRL_FLUSH_GUEST_NONGLOBAL 0x07 303 304 uint64_t v; 305#define VMCB_CTRL_V_TPR __BITS(7,0) 306#define VMCB_CTRL_V_IRQ __BIT(8) 307#define VMCB_CTRL_V_VGIF __BIT(9) 308#define VMCB_CTRL_V_INTR_PRIO __BITS(19,16) 309#define VMCB_CTRL_V_IGN_TPR __BIT(20) 310#define VMCB_CTRL_V_INTR_MASKING __BIT(24) 311#define VMCB_CTRL_V_GUEST_VGIF __BIT(25) 312#define VMCB_CTRL_V_AVIC_EN __BIT(31) 313#define VMCB_CTRL_V_INTR_VECTOR __BITS(39,32) 314 315 uint64_t intr; 316#define VMCB_CTRL_INTR_SHADOW __BIT(0) 317 318 uint64_t exitcode; 319 uint64_t exitinfo1; 320 uint64_t exitinfo2; 321 322 uint64_t exitintinfo; 323#define VMCB_CTRL_EXITINTINFO_VECTOR __BITS(7,0) 324#define VMCB_CTRL_EXITINTINFO_TYPE __BITS(10,8) 325#define VMCB_CTRL_EXITINTINFO_EV __BIT(11) 326#define VMCB_CTRL_EXITINTINFO_V __BIT(31) 327#define VMCB_CTRL_EXITINTINFO_ERRORCODE __BITS(63,32) 328 329 uint64_t enable1; 330#define VMCB_CTRL_ENABLE_NP __BIT(0) 331#define VMCB_CTRL_ENABLE_SEV __BIT(1) 332#define VMCB_CTRL_ENABLE_ES_SEV __BIT(2) 333 334 uint64_t avic; 335#define VMCB_CTRL_AVIC_APIC_BAR __BITS(51,0) 336 337 uint64_t ghcb; 338 339 uint64_t eventinj; 340#define VMCB_CTRL_EVENTINJ_VECTOR __BITS(7,0) 341#define VMCB_CTRL_EVENTINJ_TYPE __BITS(10,8) 342#define VMCB_CTRL_EVENTINJ_EV __BIT(11) 343#define VMCB_CTRL_EVENTINJ_V __BIT(31) 344#define VMCB_CTRL_EVENTINJ_ERRORCODE __BITS(63,32) 345 346 uint64_t n_cr3; 347 348 uint64_t enable2; 349#define VMCB_CTRL_ENABLE_LBR __BIT(0) 350#define VMCB_CTRL_ENABLE_VVMSAVE __BIT(1) 351 352 uint32_t vmcb_clean; 353#define VMCB_CTRL_VMCB_CLEAN_I __BIT(0) 354#define VMCB_CTRL_VMCB_CLEAN_IOPM __BIT(1) 355#define VMCB_CTRL_VMCB_CLEAN_ASID __BIT(2) 356#define VMCB_CTRL_VMCB_CLEAN_TPR __BIT(3) 357#define VMCB_CTRL_VMCB_CLEAN_NP __BIT(4) 358#define VMCB_CTRL_VMCB_CLEAN_CR __BIT(5) 359#define VMCB_CTRL_VMCB_CLEAN_DR __BIT(6) 360#define VMCB_CTRL_VMCB_CLEAN_DT __BIT(7) 361#define VMCB_CTRL_VMCB_CLEAN_SEG __BIT(8) 362#define VMCB_CTRL_VMCB_CLEAN_CR2 __BIT(9) 363#define VMCB_CTRL_VMCB_CLEAN_LBR __BIT(10) 364#define VMCB_CTRL_VMCB_CLEAN_AVIC __BIT(11) 365 366 uint32_t rsvd2; 367 uint64_t nrip; 368 uint8_t inst_len; 369 uint8_t inst_bytes[15]; 370 uint64_t avic_abpp; 371 uint64_t rsvd3; 372 uint64_t avic_ltp; 373 374 uint64_t avic_phys; 375#define VMCB_CTRL_AVIC_PHYS_TABLE_PTR __BITS(51,12) 376#define VMCB_CTRL_AVIC_PHYS_MAX_INDEX __BITS(7,0) 377 378 uint64_t rsvd4; 379 uint64_t vmcb_ptr; 380 381 uint8_t pad[752]; 382} __packed; 383 384CTASSERT(sizeof(struct vmcb_ctrl) == 1024); 385 386struct vmcb_segment { 387 uint16_t selector; 388 uint16_t attrib; /* hidden */ 389 uint32_t limit; /* hidden */ 390 uint64_t base; /* hidden */ 391} __packed; 392 393CTASSERT(sizeof(struct vmcb_segment) == 16); 394 395struct vmcb_state { 396 struct vmcb_segment es; 397 struct vmcb_segment cs; 398 struct vmcb_segment ss; 399 struct vmcb_segment ds; 400 struct vmcb_segment fs; 401 struct vmcb_segment gs; 402 struct vmcb_segment gdt; 403 struct vmcb_segment ldt; 404 struct vmcb_segment idt; 405 struct vmcb_segment tr; 406 uint8_t rsvd1[43]; 407 uint8_t cpl; 408 uint8_t rsvd2[4]; 409 uint64_t efer; 410 uint8_t rsvd3[112]; 411 uint64_t cr4; 412 uint64_t cr3; 413 uint64_t cr0; 414 uint64_t dr7; 415 uint64_t dr6; 416 uint64_t rflags; 417 uint64_t rip; 418 uint8_t rsvd4[88]; 419 uint64_t rsp; 420 uint8_t rsvd5[24]; 421 uint64_t rax; 422 uint64_t star; 423 uint64_t lstar; 424 uint64_t cstar; 425 uint64_t sfmask; 426 uint64_t kernelgsbase; 427 uint64_t sysenter_cs; 428 uint64_t sysenter_esp; 429 uint64_t sysenter_eip; 430 uint64_t cr2; 431 uint8_t rsvd6[32]; 432 uint64_t g_pat; 433 uint64_t dbgctl; 434 uint64_t br_from; 435 uint64_t br_to; 436 uint64_t int_from; 437 uint64_t int_to; 438 uint8_t pad[2408]; 439} __packed; 440 441CTASSERT(sizeof(struct vmcb_state) == 0xC00); 442 443struct vmcb { 444 struct vmcb_ctrl ctrl; 445 struct vmcb_state state; 446} __packed; 447 448CTASSERT(sizeof(struct vmcb) == PAGE_SIZE); 449CTASSERT(offsetof(struct vmcb, state) == 0x400); 450 451/* -------------------------------------------------------------------------- */ 452 453struct svm_hsave { 454 paddr_t pa; 455}; 456 457static struct svm_hsave hsave[MAXCPUS]; 458 459static uint8_t *svm_asidmap __read_mostly; 460static uint32_t svm_maxasid __read_mostly; 461static kmutex_t svm_asidlock __cacheline_aligned; 462 463static bool svm_decode_assist __read_mostly; 464static uint32_t svm_ctrl_tlb_flush __read_mostly; 465 466#define SVM_XCR0_MASK_DEFAULT (XCR0_X87|XCR0_SSE) 467static uint64_t svm_xcr0_mask __read_mostly; 468 469#define SVM_NCPUIDS 32 470 471#define VMCB_NPAGES 1 472 473#define MSRBM_NPAGES 2 474#define MSRBM_SIZE (MSRBM_NPAGES * PAGE_SIZE) 475 476#define IOBM_NPAGES 3 477#define IOBM_SIZE (IOBM_NPAGES * PAGE_SIZE) 478 479/* Does not include EFER_LMSLE. */ 480#define EFER_VALID \ 481 (EFER_SCE|EFER_LME|EFER_LMA|EFER_NXE|EFER_SVME|EFER_FFXSR|EFER_TCE) 482 483#define EFER_TLB_FLUSH \ 484 (EFER_NXE|EFER_LMA|EFER_LME) 485#define CR0_TLB_FLUSH \ 486 (CR0_PG|CR0_WP|CR0_CD|CR0_NW) 487#define CR4_TLB_FLUSH \ 488 (CR4_PGE|CR4_PAE|CR4_PSE) 489 490/* -------------------------------------------------------------------------- */ 491 492struct svm_machdata { 493 bool cpuidpresent[SVM_NCPUIDS]; 494 struct nvmm_x86_conf_cpuid cpuid[SVM_NCPUIDS]; 495}; 496 497static const size_t svm_conf_sizes[NVMM_X86_NCONF] = { 498 [NVMM_X86_CONF_CPUID] = sizeof(struct nvmm_x86_conf_cpuid) 499}; 500 501struct svm_cpudata { 502 /* General */ 503 bool shared_asid; 504 bool tlb_want_flush; 505 506 /* VMCB */ 507 struct vmcb *vmcb; 508 paddr_t vmcb_pa; 509 510 /* I/O bitmap */ 511 uint8_t *iobm; 512 paddr_t iobm_pa; 513 514 /* MSR bitmap */ 515 uint8_t *msrbm; 516 paddr_t msrbm_pa; 517 518 /* Host state */ 519 uint64_t hxcr0; 520 uint64_t star; 521 uint64_t lstar; 522 uint64_t cstar; 523 uint64_t sfmask; 524 uint64_t fsbase; 525 uint64_t kernelgsbase; 526 bool ts_set; 527 struct xsave_header hfpu __aligned(64); 528 529 /* Event state */ 530 bool int_window_exit; 531 bool nmi_window_exit; 532 533 /* Guest state */ 534 uint64_t gxcr0; 535 uint64_t gprs[NVMM_X64_NGPR]; 536 uint64_t drs[NVMM_X64_NDR]; 537 uint64_t tsc_offset; 538 struct xsave_header gfpu __aligned(64); 539}; 540 541static void 542svm_vmcb_cache_default(struct vmcb *vmcb) 543{ 544 vmcb->ctrl.vmcb_clean = 545 VMCB_CTRL_VMCB_CLEAN_I | 546 VMCB_CTRL_VMCB_CLEAN_IOPM | 547 VMCB_CTRL_VMCB_CLEAN_ASID | 548 VMCB_CTRL_VMCB_CLEAN_TPR | 549 VMCB_CTRL_VMCB_CLEAN_NP | 550 VMCB_CTRL_VMCB_CLEAN_CR | 551 VMCB_CTRL_VMCB_CLEAN_DR | 552 VMCB_CTRL_VMCB_CLEAN_DT | 553 VMCB_CTRL_VMCB_CLEAN_SEG | 554 VMCB_CTRL_VMCB_CLEAN_CR2 | 555 VMCB_CTRL_VMCB_CLEAN_LBR | 556 VMCB_CTRL_VMCB_CLEAN_AVIC; 557} 558 559static void 560svm_vmcb_cache_update(struct vmcb *vmcb, uint64_t flags) 561{ 562 if (flags & NVMM_X64_STATE_SEGS) { 563 vmcb->ctrl.vmcb_clean &= 564 ~(VMCB_CTRL_VMCB_CLEAN_SEG | VMCB_CTRL_VMCB_CLEAN_DT); 565 } 566 if (flags & NVMM_X64_STATE_CRS) { 567 vmcb->ctrl.vmcb_clean &= 568 ~(VMCB_CTRL_VMCB_CLEAN_CR | VMCB_CTRL_VMCB_CLEAN_CR2 | 569 VMCB_CTRL_VMCB_CLEAN_TPR); 570 } 571 if (flags & NVMM_X64_STATE_DRS) { 572 vmcb->ctrl.vmcb_clean &= ~VMCB_CTRL_VMCB_CLEAN_DR; 573 } 574 if (flags & NVMM_X64_STATE_MSRS) { 575 /* CR for EFER, NP for PAT. */ 576 vmcb->ctrl.vmcb_clean &= 577 ~(VMCB_CTRL_VMCB_CLEAN_CR | VMCB_CTRL_VMCB_CLEAN_NP); 578 } 579} 580 581static inline void 582svm_vmcb_cache_flush(struct vmcb *vmcb, uint64_t flags) 583{ 584 vmcb->ctrl.vmcb_clean &= ~flags; 585} 586 587static inline void 588svm_vmcb_cache_flush_all(struct vmcb *vmcb) 589{ 590 vmcb->ctrl.vmcb_clean = 0; 591} 592 593#define SVM_EVENT_TYPE_HW_INT 0 594#define SVM_EVENT_TYPE_NMI 2 595#define SVM_EVENT_TYPE_EXC 3 596#define SVM_EVENT_TYPE_SW_INT 4 597 598static void 599svm_event_waitexit_enable(struct nvmm_cpu *vcpu, bool nmi) 600{ 601 struct svm_cpudata *cpudata = vcpu->cpudata; 602 struct vmcb *vmcb = cpudata->vmcb; 603 604 if (nmi) { 605 vmcb->ctrl.intercept_misc1 |= VMCB_CTRL_INTERCEPT_IRET; 606 cpudata->nmi_window_exit = true; 607 } else { 608 vmcb->ctrl.intercept_misc1 |= VMCB_CTRL_INTERCEPT_VINTR; 609 vmcb->ctrl.v |= (VMCB_CTRL_V_IRQ | VMCB_CTRL_V_IGN_TPR); 610 svm_vmcb_cache_flush(vmcb, VMCB_CTRL_VMCB_CLEAN_TPR); 611 cpudata->int_window_exit = true; 612 } 613 614 svm_vmcb_cache_flush(vmcb, VMCB_CTRL_VMCB_CLEAN_I); 615} 616 617static void 618svm_event_waitexit_disable(struct nvmm_cpu *vcpu, bool nmi) 619{ 620 struct svm_cpudata *cpudata = vcpu->cpudata; 621 struct vmcb *vmcb = cpudata->vmcb; 622 623 if (nmi) { 624 vmcb->ctrl.intercept_misc1 &= ~VMCB_CTRL_INTERCEPT_IRET; 625 cpudata->nmi_window_exit = false; 626 } else { 627 vmcb->ctrl.intercept_misc1 &= ~VMCB_CTRL_INTERCEPT_VINTR; 628 vmcb->ctrl.v &= ~(VMCB_CTRL_V_IRQ | VMCB_CTRL_V_IGN_TPR); 629 svm_vmcb_cache_flush(vmcb, VMCB_CTRL_VMCB_CLEAN_TPR); 630 cpudata->int_window_exit = false; 631 } 632 633 svm_vmcb_cache_flush(vmcb, VMCB_CTRL_VMCB_CLEAN_I); 634} 635 636static inline int 637svm_event_has_error(uint64_t vector) 638{ 639 switch (vector) { 640 case 8: /* #DF */ 641 case 10: /* #TS */ 642 case 11: /* #NP */ 643 case 12: /* #SS */ 644 case 13: /* #GP */ 645 case 14: /* #PF */ 646 case 17: /* #AC */ 647 case 30: /* #SX */ 648 return 1; 649 default: 650 return 0; 651 } 652} 653 654static int 655svm_vcpu_inject(struct nvmm_machine *mach, struct nvmm_cpu *vcpu, 656 struct nvmm_event *event) 657{ 658 struct svm_cpudata *cpudata = vcpu->cpudata; 659 struct vmcb *vmcb = cpudata->vmcb; 660 int type = 0, err = 0; 661 662 if (event->vector >= 256) { 663 return EINVAL; 664 } 665 666 switch (event->type) { 667 case NVMM_EVENT_INTERRUPT_HW: 668 type = SVM_EVENT_TYPE_HW_INT; 669 if (event->vector == 2) { 670 type = SVM_EVENT_TYPE_NMI; 671 } 672 if (type == SVM_EVENT_TYPE_NMI) { 673 if (cpudata->nmi_window_exit) { 674 return EAGAIN; 675 } 676 svm_event_waitexit_enable(vcpu, true); 677 } else { 678 if (((vmcb->state.rflags & PSL_I) == 0) || 679 ((vmcb->ctrl.intr & VMCB_CTRL_INTR_SHADOW) != 0)) { 680 svm_event_waitexit_enable(vcpu, false); 681 return EAGAIN; 682 } 683 } 684 err = 0; 685 break; 686 case NVMM_EVENT_INTERRUPT_SW: 687 return EINVAL; 688 case NVMM_EVENT_EXCEPTION: 689 type = SVM_EVENT_TYPE_EXC; 690 if (event->vector == 2 || event->vector >= 32) 691 return EINVAL; 692 if (event->vector == 3 || event->vector == 0) 693 return EINVAL; 694 err = svm_event_has_error(event->vector); 695 break; 696 default: 697 return EINVAL; 698 } 699 700 vmcb->ctrl.eventinj = 701 __SHIFTIN(event->vector, VMCB_CTRL_EVENTINJ_VECTOR) | 702 __SHIFTIN(type, VMCB_CTRL_EVENTINJ_TYPE) | 703 __SHIFTIN(err, VMCB_CTRL_EVENTINJ_EV) | 704 __SHIFTIN(1, VMCB_CTRL_EVENTINJ_V) | 705 __SHIFTIN(event->u.error, VMCB_CTRL_EVENTINJ_ERRORCODE); 706 707 return 0; 708} 709 710static void 711svm_inject_ud(struct nvmm_machine *mach, struct nvmm_cpu *vcpu) 712{ 713 struct nvmm_event event; 714 int ret __diagused; 715 716 event.type = NVMM_EVENT_EXCEPTION; 717 event.vector = 6; 718 event.u.error = 0; 719 720 ret = svm_vcpu_inject(mach, vcpu, &event); 721 KASSERT(ret == 0); 722} 723 724static void 725svm_inject_gp(struct nvmm_machine *mach, struct nvmm_cpu *vcpu) 726{ 727 struct nvmm_event event; 728 int ret __diagused; 729 730 event.type = NVMM_EVENT_EXCEPTION; 731 event.vector = 13; 732 event.u.error = 0; 733 734 ret = svm_vcpu_inject(mach, vcpu, &event); 735 KASSERT(ret == 0); 736} 737 738static inline void 739svm_inkernel_advance(struct vmcb *vmcb) 740{ 741 /* 742 * Maybe we should also apply single-stepping and debug exceptions. 743 * Matters for guest-ring3, because it can execute 'cpuid' under a 744 * debugger. 745 */ 746 vmcb->state.rip = vmcb->ctrl.nrip; 747 vmcb->ctrl.intr &= ~VMCB_CTRL_INTR_SHADOW; 748} 749 750static void 751svm_inkernel_handle_cpuid(struct nvmm_cpu *vcpu, uint64_t eax, uint64_t ecx) 752{ 753 struct svm_cpudata *cpudata = vcpu->cpudata; 754 755 switch (eax) { 756 case 0x00000001: /* APIC number in RBX. The rest is tunable. */ 757 cpudata->gprs[NVMM_X64_GPR_RBX] &= ~CPUID_LOCAL_APIC_ID; 758 cpudata->gprs[NVMM_X64_GPR_RBX] |= __SHIFTIN(vcpu->cpuid, 759 CPUID_LOCAL_APIC_ID); 760 break; 761 case 0x0000000D: /* FPU description. Not tunable. */ 762 if (ecx != 0 || svm_xcr0_mask == 0) { 763 break; 764 } 765 cpudata->vmcb->state.rax = svm_xcr0_mask & 0xFFFFFFFF; 766 if (cpudata->gxcr0 & XCR0_SSE) { 767 cpudata->gprs[NVMM_X64_GPR_RBX] = sizeof(struct fxsave); 768 } else { 769 cpudata->gprs[NVMM_X64_GPR_RBX] = sizeof(struct save87); 770 } 771 cpudata->gprs[NVMM_X64_GPR_RBX] += 64; /* XSAVE header */ 772 cpudata->gprs[NVMM_X64_GPR_RCX] = sizeof(struct fxsave); 773 cpudata->gprs[NVMM_X64_GPR_RDX] = svm_xcr0_mask >> 32; 774 break; 775 case 0x40000000: 776 cpudata->gprs[NVMM_X64_GPR_RBX] = 0; 777 cpudata->gprs[NVMM_X64_GPR_RCX] = 0; 778 cpudata->gprs[NVMM_X64_GPR_RDX] = 0; 779 memcpy(&cpudata->gprs[NVMM_X64_GPR_RBX], "___ ", 4); 780 memcpy(&cpudata->gprs[NVMM_X64_GPR_RCX], "NVMM", 4); 781 memcpy(&cpudata->gprs[NVMM_X64_GPR_RDX], " ___", 4); 782 break; 783 case 0x80000001: /* No SVM, no RDTSCP. The rest is tunable. */ 784 cpudata->gprs[NVMM_X64_GPR_RCX] &= ~CPUID_SVM; 785 cpudata->gprs[NVMM_X64_GPR_RDX] &= ~CPUID_RDTSCP; 786 break; 787 default: 788 break; 789 } 790} 791 792static void 793svm_exit_cpuid(struct nvmm_machine *mach, struct nvmm_cpu *vcpu, 794 struct nvmm_exit *exit) 795{ 796 struct svm_machdata *machdata = mach->machdata; 797 struct svm_cpudata *cpudata = vcpu->cpudata; 798 struct nvmm_x86_conf_cpuid *cpuid; 799 uint64_t eax, ecx; 800 u_int descs[4]; 801 size_t i; 802 803 eax = cpudata->vmcb->state.rax; 804 ecx = cpudata->gprs[NVMM_X64_GPR_RCX]; 805 x86_cpuid2(eax, ecx, descs); 806 807 cpudata->vmcb->state.rax = descs[0]; 808 cpudata->gprs[NVMM_X64_GPR_RBX] = descs[1]; 809 cpudata->gprs[NVMM_X64_GPR_RCX] = descs[2]; 810 cpudata->gprs[NVMM_X64_GPR_RDX] = descs[3]; 811 812 for (i = 0; i < SVM_NCPUIDS; i++) { 813 cpuid = &machdata->cpuid[i]; 814 if (!machdata->cpuidpresent[i]) { 815 continue; 816 } 817 if (cpuid->leaf != eax) { 818 continue; 819 } 820 821 /* del */ 822 cpudata->vmcb->state.rax &= ~cpuid->del.eax; 823 cpudata->gprs[NVMM_X64_GPR_RBX] &= ~cpuid->del.ebx; 824 cpudata->gprs[NVMM_X64_GPR_RCX] &= ~cpuid->del.ecx; 825 cpudata->gprs[NVMM_X64_GPR_RDX] &= ~cpuid->del.edx; 826 827 /* set */ 828 cpudata->vmcb->state.rax |= cpuid->set.eax; 829 cpudata->gprs[NVMM_X64_GPR_RBX] |= cpuid->set.ebx; 830 cpudata->gprs[NVMM_X64_GPR_RCX] |= cpuid->set.ecx; 831 cpudata->gprs[NVMM_X64_GPR_RDX] |= cpuid->set.edx; 832 833 break; 834 } 835 836 /* Overwrite non-tunable leaves. */ 837 svm_inkernel_handle_cpuid(vcpu, eax, ecx); 838 839 svm_inkernel_advance(cpudata->vmcb); 840 exit->reason = NVMM_EXIT_NONE; 841} 842 843static void 844svm_exit_hlt(struct nvmm_machine *mach, struct nvmm_cpu *vcpu, 845 struct nvmm_exit *exit) 846{ 847 struct svm_cpudata *cpudata = vcpu->cpudata; 848 struct vmcb *vmcb = cpudata->vmcb; 849 850 if (cpudata->int_window_exit && (vmcb->state.rflags & PSL_I)) { 851 svm_event_waitexit_disable(vcpu, false); 852 } 853 854 svm_inkernel_advance(cpudata->vmcb); 855 exit->reason = NVMM_EXIT_HALTED; 856} 857 858#define SVM_EXIT_IO_PORT __BITS(31,16) 859#define SVM_EXIT_IO_SEG __BITS(12,10) 860#define SVM_EXIT_IO_A64 __BIT(9) 861#define SVM_EXIT_IO_A32 __BIT(8) 862#define SVM_EXIT_IO_A16 __BIT(7) 863#define SVM_EXIT_IO_SZ32 __BIT(6) 864#define SVM_EXIT_IO_SZ16 __BIT(5) 865#define SVM_EXIT_IO_SZ8 __BIT(4) 866#define SVM_EXIT_IO_REP __BIT(3) 867#define SVM_EXIT_IO_STR __BIT(2) 868#define SVM_EXIT_IO_IN __BIT(0) 869 870static const int seg_to_nvmm[] = { 871 [0] = NVMM_X64_SEG_ES, 872 [1] = NVMM_X64_SEG_CS, 873 [2] = NVMM_X64_SEG_SS, 874 [3] = NVMM_X64_SEG_DS, 875 [4] = NVMM_X64_SEG_FS, 876 [5] = NVMM_X64_SEG_GS 877}; 878 879static void 880svm_exit_io(struct nvmm_machine *mach, struct nvmm_cpu *vcpu, 881 struct nvmm_exit *exit) 882{ 883 struct svm_cpudata *cpudata = vcpu->cpudata; 884 uint64_t info = cpudata->vmcb->ctrl.exitinfo1; 885 uint64_t nextpc = cpudata->vmcb->ctrl.exitinfo2; 886 887 exit->reason = NVMM_EXIT_IO; 888 889 if (info & SVM_EXIT_IO_IN) { 890 exit->u.io.type = NVMM_EXIT_IO_IN; 891 } else { 892 exit->u.io.type = NVMM_EXIT_IO_OUT; 893 } 894 895 exit->u.io.port = __SHIFTOUT(info, SVM_EXIT_IO_PORT); 896 897 if (svm_decode_assist) { 898 KASSERT(__SHIFTOUT(info, SVM_EXIT_IO_SEG) < 6); 899 exit->u.io.seg = seg_to_nvmm[__SHIFTOUT(info, SVM_EXIT_IO_SEG)]; 900 } else { 901 exit->u.io.seg = -1; 902 } 903 904 if (info & SVM_EXIT_IO_A64) { 905 exit->u.io.address_size = 8; 906 } else if (info & SVM_EXIT_IO_A32) { 907 exit->u.io.address_size = 4; 908 } else if (info & SVM_EXIT_IO_A16) { 909 exit->u.io.address_size = 2; 910 } 911 912 if (info & SVM_EXIT_IO_SZ32) { 913 exit->u.io.operand_size = 4; 914 } else if (info & SVM_EXIT_IO_SZ16) { 915 exit->u.io.operand_size = 2; 916 } else if (info & SVM_EXIT_IO_SZ8) { 917 exit->u.io.operand_size = 1; 918 } 919 920 exit->u.io.rep = (info & SVM_EXIT_IO_REP) != 0; 921 exit->u.io.str = (info & SVM_EXIT_IO_STR) != 0; 922 exit->u.io.npc = nextpc; 923} 924 925static const uint64_t msr_ignore_list[] = { 926 0xc0010055, /* MSR_CMPHALT */ 927 MSR_DE_CFG, 928 MSR_IC_CFG, 929 MSR_UCODE_AMD_PATCHLEVEL 930}; 931 932static bool 933svm_inkernel_handle_msr(struct nvmm_machine *mach, struct nvmm_cpu *vcpu, 934 struct nvmm_exit *exit) 935{ 936 struct svm_cpudata *cpudata = vcpu->cpudata; 937 struct vmcb *vmcb = cpudata->vmcb; 938 uint64_t val; 939 size_t i; 940 941 switch (exit->u.msr.type) { 942 case NVMM_EXIT_MSR_RDMSR: 943 if (exit->u.msr.msr == MSR_NB_CFG) { 944 val = NB_CFG_INITAPICCPUIDLO; 945 vmcb->state.rax = (val & 0xFFFFFFFF); 946 cpudata->gprs[NVMM_X64_GPR_RDX] = (val >> 32); 947 goto handled; 948 } 949 for (i = 0; i < __arraycount(msr_ignore_list); i++) { 950 if (msr_ignore_list[i] != exit->u.msr.msr) 951 continue; 952 val = 0; 953 vmcb->state.rax = (val & 0xFFFFFFFF); 954 cpudata->gprs[NVMM_X64_GPR_RDX] = (val >> 32); 955 goto handled; 956 } 957 break; 958 case NVMM_EXIT_MSR_WRMSR: 959 if (exit->u.msr.msr == MSR_EFER) { 960 if (__predict_false(exit->u.msr.val & ~EFER_VALID)) { 961 goto error; 962 } 963 if ((vmcb->state.efer ^ exit->u.msr.val) & 964 EFER_TLB_FLUSH) { 965 cpudata->tlb_want_flush = true; 966 } 967 vmcb->state.efer = exit->u.msr.val | EFER_SVME; 968 vmcb->ctrl.vmcb_clean &= ~VMCB_CTRL_VMCB_CLEAN_CR; 969 goto handled; 970 } 971 for (i = 0; i < __arraycount(msr_ignore_list); i++) { 972 if (msr_ignore_list[i] != exit->u.msr.msr) 973 continue; 974 goto handled; 975 } 976 break; 977 } 978 979 return false; 980 981handled: 982 svm_inkernel_advance(cpudata->vmcb); 983 return true; 984 985error: 986 svm_inject_gp(mach, vcpu); 987 return true; 988} 989 990static void 991svm_exit_msr(struct nvmm_machine *mach, struct nvmm_cpu *vcpu, 992 struct nvmm_exit *exit) 993{ 994 struct svm_cpudata *cpudata = vcpu->cpudata; 995 uint64_t info = cpudata->vmcb->ctrl.exitinfo1; 996 997 if (info == 0) { 998 exit->u.msr.type = NVMM_EXIT_MSR_RDMSR; 999 } else { 1000 exit->u.msr.type = NVMM_EXIT_MSR_WRMSR; 1001 } 1002 1003 exit->u.msr.msr = (cpudata->gprs[NVMM_X64_GPR_RCX] & 0xFFFFFFFF); 1004 1005 if (info == 1) { 1006 uint64_t rdx, rax; 1007 rdx = cpudata->gprs[NVMM_X64_GPR_RDX]; 1008 rax = cpudata->vmcb->state.rax; 1009 exit->u.msr.val = (rdx << 32) | (rax & 0xFFFFFFFF); 1010 } else { 1011 exit->u.msr.val = 0; 1012 } 1013 1014 if (svm_inkernel_handle_msr(mach, vcpu, exit)) { 1015 exit->reason = NVMM_EXIT_NONE; 1016 return; 1017 } 1018 1019 exit->reason = NVMM_EXIT_MSR; 1020 exit->u.msr.npc = cpudata->vmcb->ctrl.nrip; 1021} 1022 1023static void 1024svm_exit_npf(struct nvmm_machine *mach, struct nvmm_cpu *vcpu, 1025 struct nvmm_exit *exit) 1026{ 1027 struct svm_cpudata *cpudata = vcpu->cpudata; 1028 gpaddr_t gpa = cpudata->vmcb->ctrl.exitinfo2; 1029 int error; 1030 1031 error = uvm_fault(&mach->vm->vm_map, gpa, VM_PROT_ALL); 1032 1033 if (error) { 1034 exit->reason = NVMM_EXIT_MEMORY; 1035 if (cpudata->vmcb->ctrl.exitinfo1 & PGEX_W) 1036 exit->u.mem.perm = NVMM_EXIT_MEMORY_WRITE; 1037 else if (cpudata->vmcb->ctrl.exitinfo1 & PGEX_X) 1038 exit->u.mem.perm = NVMM_EXIT_MEMORY_EXEC; 1039 else 1040 exit->u.mem.perm = NVMM_EXIT_MEMORY_READ; 1041 exit->u.mem.gpa = gpa; 1042 exit->u.mem.inst_len = cpudata->vmcb->ctrl.inst_len; 1043 memcpy(exit->u.mem.inst_bytes, cpudata->vmcb->ctrl.inst_bytes, 1044 sizeof(exit->u.mem.inst_bytes)); 1045 } else { 1046 exit->reason = NVMM_EXIT_NONE; 1047 } 1048} 1049 1050static void 1051svm_exit_insn(struct vmcb *vmcb, struct nvmm_exit *exit, uint64_t reason) 1052{ 1053 exit->u.insn.npc = vmcb->ctrl.nrip; 1054 exit->reason = reason; 1055} 1056 1057static void 1058svm_exit_xsetbv(struct nvmm_machine *mach, struct nvmm_cpu *vcpu, 1059 struct nvmm_exit *exit) 1060{ 1061 struct svm_cpudata *cpudata = vcpu->cpudata; 1062 struct vmcb *vmcb = cpudata->vmcb; 1063 uint64_t val; 1064 1065 exit->reason = NVMM_EXIT_NONE; 1066 1067 val = (cpudata->gprs[NVMM_X64_GPR_RDX] << 32) | 1068 (vmcb->state.rax & 0xFFFFFFFF); 1069 1070 if (__predict_false(cpudata->gprs[NVMM_X64_GPR_RCX] != 0)) { 1071 goto error; 1072 } else if (__predict_false(vmcb->state.cpl != 0)) { 1073 goto error; 1074 } else if (__predict_false((val & ~svm_xcr0_mask) != 0)) { 1075 goto error; 1076 } else if (__predict_false((val & XCR0_X87) == 0)) { 1077 goto error; 1078 } 1079 1080 cpudata->gxcr0 = val; 1081 1082 svm_inkernel_advance(cpudata->vmcb); 1083 return; 1084 1085error: 1086 svm_inject_gp(mach, vcpu); 1087} 1088 1089static void 1090svm_vcpu_guest_fpu_enter(struct nvmm_cpu *vcpu) 1091{ 1092 struct svm_cpudata *cpudata = vcpu->cpudata; 1093 1094 cpudata->ts_set = (rcr0() & CR0_TS) != 0; 1095 1096 fpu_area_save(&cpudata->hfpu, svm_xcr0_mask); 1097 fpu_area_restore(&cpudata->gfpu, svm_xcr0_mask); 1098 1099 if (svm_xcr0_mask != 0) { 1100 cpudata->hxcr0 = rdxcr(0); 1101 wrxcr(0, cpudata->gxcr0); 1102 } 1103} 1104 1105static void 1106svm_vcpu_guest_fpu_leave(struct nvmm_cpu *vcpu) 1107{ 1108 struct svm_cpudata *cpudata = vcpu->cpudata; 1109 1110 if (svm_xcr0_mask != 0) { 1111 cpudata->gxcr0 = rdxcr(0); 1112 wrxcr(0, cpudata->hxcr0); 1113 } 1114 1115 fpu_area_save(&cpudata->gfpu, svm_xcr0_mask); 1116 fpu_area_restore(&cpudata->hfpu, svm_xcr0_mask); 1117 1118 if (cpudata->ts_set) { 1119 stts(); 1120 } 1121} 1122 1123static void 1124svm_vcpu_guest_dbregs_enter(struct nvmm_cpu *vcpu) 1125{ 1126 struct svm_cpudata *cpudata = vcpu->cpudata; 1127 1128 x86_dbregs_save(curlwp); 1129 1130 ldr7(0); 1131 1132 ldr0(cpudata->drs[NVMM_X64_DR_DR0]); 1133 ldr1(cpudata->drs[NVMM_X64_DR_DR1]); 1134 ldr2(cpudata->drs[NVMM_X64_DR_DR2]); 1135 ldr3(cpudata->drs[NVMM_X64_DR_DR3]); 1136} 1137 1138static void 1139svm_vcpu_guest_dbregs_leave(struct nvmm_cpu *vcpu) 1140{ 1141 struct svm_cpudata *cpudata = vcpu->cpudata; 1142 1143 cpudata->drs[NVMM_X64_DR_DR0] = rdr0(); 1144 cpudata->drs[NVMM_X64_DR_DR1] = rdr1(); 1145 cpudata->drs[NVMM_X64_DR_DR2] = rdr2(); 1146 cpudata->drs[NVMM_X64_DR_DR3] = rdr3(); 1147 1148 x86_dbregs_restore(curlwp); 1149} 1150 1151static void 1152svm_vcpu_guest_misc_enter(struct nvmm_cpu *vcpu) 1153{ 1154 struct svm_cpudata *cpudata = vcpu->cpudata; 1155 1156 cpudata->fsbase = rdmsr(MSR_FSBASE); 1157 cpudata->kernelgsbase = rdmsr(MSR_KERNELGSBASE); 1158} 1159 1160static void 1161svm_vcpu_guest_misc_leave(struct nvmm_cpu *vcpu) 1162{ 1163 struct svm_cpudata *cpudata = vcpu->cpudata; 1164 1165 wrmsr(MSR_STAR, cpudata->star); 1166 wrmsr(MSR_LSTAR, cpudata->lstar); 1167 wrmsr(MSR_CSTAR, cpudata->cstar); 1168 wrmsr(MSR_SFMASK, cpudata->sfmask); 1169 wrmsr(MSR_FSBASE, cpudata->fsbase); 1170 wrmsr(MSR_KERNELGSBASE, cpudata->kernelgsbase); 1171} 1172 1173static int 1174svm_vcpu_run(struct nvmm_machine *mach, struct nvmm_cpu *vcpu, 1175 struct nvmm_exit *exit) 1176{ 1177 struct svm_cpudata *cpudata = vcpu->cpudata; 1178 struct vmcb *vmcb = cpudata->vmcb; 1179 bool tlb_need_flush = false; 1180 int hcpu, s; 1181 1182 kpreempt_disable(); 1183 hcpu = cpu_number(); 1184 1185 if (vcpu->hcpu_last != hcpu || cpudata->shared_asid) { 1186 tlb_need_flush = true; 1187 } 1188 1189 if (vcpu->hcpu_last != hcpu) { 1190 vmcb->ctrl.tsc_offset = cpudata->tsc_offset + 1191 curcpu()->ci_data.cpu_cc_skew; 1192 svm_vmcb_cache_flush_all(vmcb); 1193 } 1194 1195 svm_vcpu_guest_dbregs_enter(vcpu); 1196 svm_vcpu_guest_misc_enter(vcpu); 1197 1198 while (1) { 1199 if (cpudata->tlb_want_flush || tlb_need_flush) { 1200 vmcb->ctrl.tlb_ctrl = svm_ctrl_tlb_flush; 1201 } else { 1202 vmcb->ctrl.tlb_ctrl = 0; 1203 } 1204 1205 s = splhigh(); 1206 svm_vcpu_guest_fpu_enter(vcpu); 1207 svm_vmrun(cpudata->vmcb_pa, cpudata->gprs); 1208 svm_vcpu_guest_fpu_leave(vcpu); 1209 splx(s); 1210 1211 svm_vmcb_cache_default(vmcb); 1212 1213 if (vmcb->ctrl.exitcode != VMCB_EXITCODE_INVALID) { 1214 cpudata->tlb_want_flush = false; 1215 tlb_need_flush = false; 1216 vcpu->hcpu_last = hcpu; 1217 } 1218 1219 switch (vmcb->ctrl.exitcode) { 1220 case VMCB_EXITCODE_INTR: 1221 case VMCB_EXITCODE_NMI: 1222 exit->reason = NVMM_EXIT_NONE; 1223 break; 1224 case VMCB_EXITCODE_VINTR: 1225 svm_event_waitexit_disable(vcpu, false); 1226 exit->reason = NVMM_EXIT_INT_READY; 1227 break; 1228 case VMCB_EXITCODE_IRET: 1229 svm_event_waitexit_disable(vcpu, true); 1230 exit->reason = NVMM_EXIT_NMI_READY; 1231 break; 1232 case VMCB_EXITCODE_CPUID: 1233 svm_exit_cpuid(mach, vcpu, exit); 1234 break; 1235 case VMCB_EXITCODE_HLT: 1236 svm_exit_hlt(mach, vcpu, exit); 1237 break; 1238 case VMCB_EXITCODE_IOIO: 1239 svm_exit_io(mach, vcpu, exit); 1240 break; 1241 case VMCB_EXITCODE_MSR: 1242 svm_exit_msr(mach, vcpu, exit); 1243 break; 1244 case VMCB_EXITCODE_SHUTDOWN: 1245 exit->reason = NVMM_EXIT_SHUTDOWN; 1246 break; 1247 case VMCB_EXITCODE_RDPMC: 1248 case VMCB_EXITCODE_RSM: 1249 case VMCB_EXITCODE_INVLPGA: 1250 case VMCB_EXITCODE_VMRUN: 1251 case VMCB_EXITCODE_VMMCALL: 1252 case VMCB_EXITCODE_VMLOAD: 1253 case VMCB_EXITCODE_VMSAVE: 1254 case VMCB_EXITCODE_STGI: 1255 case VMCB_EXITCODE_CLGI: 1256 case VMCB_EXITCODE_SKINIT: 1257 case VMCB_EXITCODE_RDTSCP: 1258 svm_inject_ud(mach, vcpu); 1259 exit->reason = NVMM_EXIT_NONE; 1260 break; 1261 case VMCB_EXITCODE_MONITOR: 1262 svm_exit_insn(vmcb, exit, NVMM_EXIT_MONITOR); 1263 break; 1264 case VMCB_EXITCODE_MWAIT: 1265 svm_exit_insn(vmcb, exit, NVMM_EXIT_MWAIT); 1266 break; 1267 case VMCB_EXITCODE_MWAIT_CONDITIONAL: 1268 svm_exit_insn(vmcb, exit, NVMM_EXIT_MWAIT_COND); 1269 break; 1270 case VMCB_EXITCODE_XSETBV: 1271 svm_exit_xsetbv(mach, vcpu, exit); 1272 break; 1273 case VMCB_EXITCODE_NPF: 1274 svm_exit_npf(mach, vcpu, exit); 1275 break; 1276 case VMCB_EXITCODE_FERR_FREEZE: /* ? */ 1277 default: 1278 exit->reason = NVMM_EXIT_INVALID; 1279 break; 1280 } 1281 1282 /* If no reason to return to userland, keep rolling. */ 1283 if (curcpu()->ci_schedstate.spc_flags & SPCF_SHOULDYIELD) { 1284 break; 1285 } 1286 if (curcpu()->ci_data.cpu_softints != 0) { 1287 break; 1288 } 1289 if (curlwp->l_flag & LW_USERRET) { 1290 break; 1291 } 1292 if (exit->reason != NVMM_EXIT_NONE) { 1293 break; 1294 } 1295 } 1296 1297 svm_vcpu_guest_misc_leave(vcpu); 1298 svm_vcpu_guest_dbregs_leave(vcpu); 1299 1300 kpreempt_enable(); 1301 1302 exit->exitstate[NVMM_X64_EXITSTATE_CR8] = __SHIFTOUT(vmcb->ctrl.v, 1303 VMCB_CTRL_V_TPR); 1304 exit->exitstate[NVMM_X64_EXITSTATE_RFLAGS] = vmcb->state.rflags; 1305 1306 exit->exitstate[NVMM_X64_EXITSTATE_INT_SHADOW] = 1307 ((vmcb->ctrl.intr & VMCB_CTRL_INTR_SHADOW) != 0); 1308 exit->exitstate[NVMM_X64_EXITSTATE_INT_WINDOW_EXIT] = 1309 cpudata->int_window_exit; 1310 exit->exitstate[NVMM_X64_EXITSTATE_NMI_WINDOW_EXIT] = 1311 cpudata->nmi_window_exit; 1312 1313 return 0; 1314} 1315 1316/* -------------------------------------------------------------------------- */ 1317 1318static int 1319svm_memalloc(paddr_t *pa, vaddr_t *va, size_t npages) 1320{ 1321 struct pglist pglist; 1322 paddr_t _pa; 1323 vaddr_t _va; 1324 size_t i; 1325 int ret; 1326 1327 ret = uvm_pglistalloc(npages * PAGE_SIZE, 0, ~0UL, PAGE_SIZE, 0, 1328 &pglist, 1, 0); 1329 if (ret != 0) 1330 return ENOMEM; 1331 _pa = TAILQ_FIRST(&pglist)->phys_addr; 1332 _va = uvm_km_alloc(kernel_map, npages * PAGE_SIZE, 0, 1333 UVM_KMF_VAONLY | UVM_KMF_NOWAIT); 1334 if (_va == 0) 1335 goto error; 1336 1337 for (i = 0; i < npages; i++) { 1338 pmap_kenter_pa(_va + i * PAGE_SIZE, _pa + i * PAGE_SIZE, 1339 VM_PROT_READ | VM_PROT_WRITE, PMAP_WRITE_BACK); 1340 } 1341 pmap_update(pmap_kernel()); 1342 1343 memset((void *)_va, 0, npages * PAGE_SIZE); 1344 1345 *pa = _pa; 1346 *va = _va; 1347 return 0; 1348 1349error: 1350 for (i = 0; i < npages; i++) { 1351 uvm_pagefree(PHYS_TO_VM_PAGE(_pa + i * PAGE_SIZE)); 1352 } 1353 return ENOMEM; 1354} 1355 1356static void 1357svm_memfree(paddr_t pa, vaddr_t va, size_t npages) 1358{ 1359 size_t i; 1360 1361 pmap_kremove(va, npages * PAGE_SIZE); 1362 pmap_update(pmap_kernel()); 1363 uvm_km_free(kernel_map, va, npages * PAGE_SIZE, UVM_KMF_VAONLY); 1364 for (i = 0; i < npages; i++) { 1365 uvm_pagefree(PHYS_TO_VM_PAGE(pa + i * PAGE_SIZE)); 1366 } 1367} 1368 1369/* -------------------------------------------------------------------------- */ 1370 1371#define SVM_MSRBM_READ __BIT(0) 1372#define SVM_MSRBM_WRITE __BIT(1) 1373 1374static void 1375svm_vcpu_msr_allow(uint8_t *bitmap, uint64_t msr, bool read, bool write) 1376{ 1377 uint64_t byte; 1378 uint8_t bitoff; 1379 1380 if (msr < 0x00002000) { 1381 /* Range 1 */ 1382 byte = ((msr - 0x00000000) >> 2UL) + 0x0000; 1383 } else if (msr >= 0xC0000000 && msr < 0xC0002000) { 1384 /* Range 2 */ 1385 byte = ((msr - 0xC0000000) >> 2UL) + 0x0800; 1386 } else if (msr >= 0xC0010000 && msr < 0xC0012000) { 1387 /* Range 3 */ 1388 byte = ((msr - 0xC0010000) >> 2UL) + 0x1000; 1389 } else { 1390 panic("%s: wrong range", __func__); 1391 } 1392 1393 bitoff = (msr & 0x3) << 1; 1394 1395 if (read) { 1396 bitmap[byte] &= ~(SVM_MSRBM_READ << bitoff); 1397 } 1398 if (write) { 1399 bitmap[byte] &= ~(SVM_MSRBM_WRITE << bitoff); 1400 } 1401} 1402 1403static void 1404svm_asid_alloc(struct nvmm_cpu *vcpu) 1405{ 1406 struct svm_cpudata *cpudata = vcpu->cpudata; 1407 struct vmcb *vmcb = cpudata->vmcb; 1408 size_t i, oct, bit; 1409 1410 mutex_enter(&svm_asidlock); 1411 1412 for (i = 0; i < svm_maxasid; i++) { 1413 oct = i / 8; 1414 bit = i % 8; 1415 1416 if (svm_asidmap[oct] & __BIT(bit)) { 1417 continue; 1418 } 1419 1420 svm_asidmap[oct] |= __BIT(bit); 1421 vmcb->ctrl.guest_asid = i; 1422 mutex_exit(&svm_asidlock); 1423 return; 1424 } 1425 1426 /* 1427 * No free ASID. Use the last one, which is shared and requires 1428 * special TLB handling. 1429 */ 1430 cpudata->shared_asid = true; 1431 vmcb->ctrl.guest_asid = svm_maxasid - 1; 1432 mutex_exit(&svm_asidlock); 1433} 1434 1435static void 1436svm_asid_free(struct nvmm_cpu *vcpu) 1437{ 1438 struct svm_cpudata *cpudata = vcpu->cpudata; 1439 struct vmcb *vmcb = cpudata->vmcb; 1440 size_t oct, bit; 1441 1442 if (cpudata->shared_asid) { 1443 return; 1444 } 1445 1446 oct = vmcb->ctrl.guest_asid / 8; 1447 bit = vmcb->ctrl.guest_asid % 8; 1448 1449 mutex_enter(&svm_asidlock); 1450 svm_asidmap[oct] &= ~__BIT(bit); 1451 mutex_exit(&svm_asidlock); 1452} 1453 1454static void 1455svm_vcpu_init(struct nvmm_machine *mach, struct nvmm_cpu *vcpu) 1456{ 1457 struct svm_cpudata *cpudata = vcpu->cpudata; 1458 struct vmcb *vmcb = cpudata->vmcb; 1459 1460 /* Allow reads/writes of Control Registers. */ 1461 vmcb->ctrl.intercept_cr = 0; 1462 1463 /* Allow reads/writes of Debug Registers. */ 1464 vmcb->ctrl.intercept_dr = 0; 1465 1466 /* Allow exceptions 0 to 31. */ 1467 vmcb->ctrl.intercept_vec = 0; 1468 1469 /* 1470 * Allow: 1471 * - SMI [smm interrupts] 1472 * - VINTR [virtual interrupts] 1473 * - CR0_SPEC [CR0 writes changing other fields than CR0.TS or CR0.MP] 1474 * - RIDTR [reads of IDTR] 1475 * - RGDTR [reads of GDTR] 1476 * - RLDTR [reads of LDTR] 1477 * - RTR [reads of TR] 1478 * - WIDTR [writes of IDTR] 1479 * - WGDTR [writes of GDTR] 1480 * - WLDTR [writes of LDTR] 1481 * - WTR [writes of TR] 1482 * - RDTSC [rdtsc instruction] 1483 * - PUSHF [pushf instruction] 1484 * - POPF [popf instruction] 1485 * - IRET [iret instruction] 1486 * - INTN [int $n instructions] 1487 * - INVD [invd instruction] 1488 * - PAUSE [pause instruction] 1489 * - INVLPG [invplg instruction] 1490 * - TASKSW [task switches] 1491 * 1492 * Intercept the rest below. 1493 */ 1494 vmcb->ctrl.intercept_misc1 = 1495 VMCB_CTRL_INTERCEPT_INTR | 1496 VMCB_CTRL_INTERCEPT_NMI | 1497 VMCB_CTRL_INTERCEPT_INIT | 1498 VMCB_CTRL_INTERCEPT_RDPMC | 1499 VMCB_CTRL_INTERCEPT_CPUID | 1500 VMCB_CTRL_INTERCEPT_RSM | 1501 VMCB_CTRL_INTERCEPT_HLT | 1502 VMCB_CTRL_INTERCEPT_INVLPGA | 1503 VMCB_CTRL_INTERCEPT_IOIO_PROT | 1504 VMCB_CTRL_INTERCEPT_MSR_PROT | 1505 VMCB_CTRL_INTERCEPT_FERR_FREEZE | 1506 VMCB_CTRL_INTERCEPT_SHUTDOWN; 1507 1508 /* 1509 * Allow: 1510 * - ICEBP [icebp instruction] 1511 * - WBINVD [wbinvd instruction] 1512 * - WCR_SPEC(0..15) [writes of CR0-15, received after instruction] 1513 * 1514 * Intercept the rest below. 1515 */ 1516 vmcb->ctrl.intercept_misc2 = 1517 VMCB_CTRL_INTERCEPT_VMRUN | 1518 VMCB_CTRL_INTERCEPT_VMMCALL | 1519 VMCB_CTRL_INTERCEPT_VMLOAD | 1520 VMCB_CTRL_INTERCEPT_VMSAVE | 1521 VMCB_CTRL_INTERCEPT_STGI | 1522 VMCB_CTRL_INTERCEPT_CLGI | 1523 VMCB_CTRL_INTERCEPT_SKINIT | 1524 VMCB_CTRL_INTERCEPT_RDTSCP | 1525 VMCB_CTRL_INTERCEPT_MONITOR | 1526 VMCB_CTRL_INTERCEPT_MWAIT | 1527 VMCB_CTRL_INTERCEPT_XSETBV; 1528 1529 /* Intercept all I/O accesses. */ 1530 memset(cpudata->iobm, 0xFF, IOBM_SIZE); 1531 vmcb->ctrl.iopm_base_pa = cpudata->iobm_pa; 1532 1533 /* 1534 * Allow: 1535 * - EFER [read] 1536 * - STAR [read, write] 1537 * - LSTAR [read, write] 1538 * - CSTAR [read, write] 1539 * - SFMASK [read, write] 1540 * - KERNELGSBASE [read, write] 1541 * - SYSENTER_CS [read, write] 1542 * - SYSENTER_ESP [read, write] 1543 * - SYSENTER_EIP [read, write] 1544 * - FSBASE [read, write] 1545 * - GSBASE [read, write] 1546 * - PAT [read, write] 1547 * - TSC [read] 1548 * 1549 * Intercept the rest. 1550 */ 1551 memset(cpudata->msrbm, 0xFF, MSRBM_SIZE); 1552 svm_vcpu_msr_allow(cpudata->msrbm, MSR_EFER, true, false); 1553 svm_vcpu_msr_allow(cpudata->msrbm, MSR_STAR, true, true); 1554 svm_vcpu_msr_allow(cpudata->msrbm, MSR_LSTAR, true, true); 1555 svm_vcpu_msr_allow(cpudata->msrbm, MSR_CSTAR, true, true); 1556 svm_vcpu_msr_allow(cpudata->msrbm, MSR_SFMASK, true, true); 1557 svm_vcpu_msr_allow(cpudata->msrbm, MSR_KERNELGSBASE, true, true); 1558 svm_vcpu_msr_allow(cpudata->msrbm, MSR_SYSENTER_CS, true, true); 1559 svm_vcpu_msr_allow(cpudata->msrbm, MSR_SYSENTER_ESP, true, true); 1560 svm_vcpu_msr_allow(cpudata->msrbm, MSR_SYSENTER_EIP, true, true); 1561 svm_vcpu_msr_allow(cpudata->msrbm, MSR_FSBASE, true, true); 1562 svm_vcpu_msr_allow(cpudata->msrbm, MSR_GSBASE, true, true); 1563 svm_vcpu_msr_allow(cpudata->msrbm, MSR_CR_PAT, true, true); 1564 svm_vcpu_msr_allow(cpudata->msrbm, MSR_TSC, true, false); 1565 vmcb->ctrl.msrpm_base_pa = cpudata->msrbm_pa; 1566 1567 /* Generate ASID. */ 1568 svm_asid_alloc(vcpu); 1569 1570 /* Virtual TPR. */ 1571 vmcb->ctrl.v = VMCB_CTRL_V_INTR_MASKING; 1572 1573 /* Enable Nested Paging. */ 1574 vmcb->ctrl.enable1 = VMCB_CTRL_ENABLE_NP; 1575 vmcb->ctrl.n_cr3 = mach->vm->vm_map.pmap->pm_pdirpa[0]; 1576 1577 /* Must always be set. */ 1578 vmcb->state.efer = EFER_SVME; 1579 cpudata->gxcr0 = XCR0_X87; 1580 1581 /* Init XSAVE header. */ 1582 cpudata->gfpu.xsh_xstate_bv = svm_xcr0_mask; 1583 cpudata->gfpu.xsh_xcomp_bv = 0; 1584 1585 /* Bluntly hide the host TSC. */ 1586 cpudata->tsc_offset = rdtsc(); 1587 1588 /* These MSRs are static. */ 1589 cpudata->star = rdmsr(MSR_STAR); 1590 cpudata->lstar = rdmsr(MSR_LSTAR); 1591 cpudata->cstar = rdmsr(MSR_CSTAR); 1592 cpudata->sfmask = rdmsr(MSR_SFMASK); 1593} 1594 1595static int 1596svm_vcpu_create(struct nvmm_machine *mach, struct nvmm_cpu *vcpu) 1597{ 1598 struct svm_cpudata *cpudata; 1599 int error; 1600 1601 /* Allocate the SVM cpudata. */ 1602 cpudata = (struct svm_cpudata *)uvm_km_alloc(kernel_map, 1603 roundup(sizeof(*cpudata), PAGE_SIZE), 0, 1604 UVM_KMF_WIRED|UVM_KMF_ZERO); 1605 vcpu->cpudata = cpudata; 1606 1607 /* VMCB */ 1608 error = svm_memalloc(&cpudata->vmcb_pa, (vaddr_t *)&cpudata->vmcb, 1609 VMCB_NPAGES); 1610 if (error) 1611 goto error; 1612 1613 /* I/O Bitmap */ 1614 error = svm_memalloc(&cpudata->iobm_pa, (vaddr_t *)&cpudata->iobm, 1615 IOBM_NPAGES); 1616 if (error) 1617 goto error; 1618 1619 /* MSR Bitmap */ 1620 error = svm_memalloc(&cpudata->msrbm_pa, (vaddr_t *)&cpudata->msrbm, 1621 MSRBM_NPAGES); 1622 if (error) 1623 goto error; 1624 1625 /* Init the VCPU info. */ 1626 svm_vcpu_init(mach, vcpu); 1627 1628 return 0; 1629 1630error: 1631 if (cpudata->vmcb_pa) { 1632 svm_memfree(cpudata->vmcb_pa, (vaddr_t)cpudata->vmcb, 1633 VMCB_NPAGES); 1634 } 1635 if (cpudata->iobm_pa) { 1636 svm_memfree(cpudata->iobm_pa, (vaddr_t)cpudata->iobm, 1637 IOBM_NPAGES); 1638 } 1639 if (cpudata->msrbm_pa) { 1640 svm_memfree(cpudata->msrbm_pa, (vaddr_t)cpudata->msrbm, 1641 MSRBM_NPAGES); 1642 } 1643 uvm_km_free(kernel_map, (vaddr_t)cpudata, 1644 roundup(sizeof(*cpudata), PAGE_SIZE), UVM_KMF_WIRED); 1645 return error; 1646} 1647 1648static void 1649svm_vcpu_destroy(struct nvmm_machine *mach, struct nvmm_cpu *vcpu) 1650{ 1651 struct svm_cpudata *cpudata = vcpu->cpudata; 1652 1653 svm_asid_free(vcpu); 1654 1655 svm_memfree(cpudata->vmcb_pa, (vaddr_t)cpudata->vmcb, VMCB_NPAGES); 1656 svm_memfree(cpudata->iobm_pa, (vaddr_t)cpudata->iobm, IOBM_NPAGES); 1657 svm_memfree(cpudata->msrbm_pa, (vaddr_t)cpudata->msrbm, MSRBM_NPAGES); 1658 1659 uvm_km_free(kernel_map, (vaddr_t)cpudata, 1660 roundup(sizeof(*cpudata), PAGE_SIZE), UVM_KMF_WIRED); 1661} 1662 1663#define SVM_SEG_ATTRIB_TYPE __BITS(4,0) 1664#define SVM_SEG_ATTRIB_DPL __BITS(6,5) 1665#define SVM_SEG_ATTRIB_P __BIT(7) 1666#define SVM_SEG_ATTRIB_AVL __BIT(8) 1667#define SVM_SEG_ATTRIB_LONG __BIT(9) 1668#define SVM_SEG_ATTRIB_DEF32 __BIT(10) 1669#define SVM_SEG_ATTRIB_GRAN __BIT(11) 1670 1671static void 1672svm_vcpu_setstate_seg(struct nvmm_x64_state_seg *seg, struct vmcb_segment *vseg) 1673{ 1674 vseg->selector = seg->selector; 1675 vseg->attrib = 1676 __SHIFTIN(seg->attrib.type, SVM_SEG_ATTRIB_TYPE) | 1677 __SHIFTIN(seg->attrib.dpl, SVM_SEG_ATTRIB_DPL) | 1678 __SHIFTIN(seg->attrib.p, SVM_SEG_ATTRIB_P) | 1679 __SHIFTIN(seg->attrib.avl, SVM_SEG_ATTRIB_AVL) | 1680 __SHIFTIN(seg->attrib.lng, SVM_SEG_ATTRIB_LONG) | 1681 __SHIFTIN(seg->attrib.def32, SVM_SEG_ATTRIB_DEF32) | 1682 __SHIFTIN(seg->attrib.gran, SVM_SEG_ATTRIB_GRAN); 1683 vseg->limit = seg->limit; 1684 vseg->base = seg->base; 1685} 1686 1687static void 1688svm_vcpu_getstate_seg(struct nvmm_x64_state_seg *seg, struct vmcb_segment *vseg) 1689{ 1690 seg->selector = vseg->selector; 1691 seg->attrib.type = __SHIFTOUT(vseg->attrib, SVM_SEG_ATTRIB_TYPE); 1692 seg->attrib.dpl = __SHIFTOUT(vseg->attrib, SVM_SEG_ATTRIB_DPL); 1693 seg->attrib.p = __SHIFTOUT(vseg->attrib, SVM_SEG_ATTRIB_P); 1694 seg->attrib.avl = __SHIFTOUT(vseg->attrib, SVM_SEG_ATTRIB_AVL); 1695 seg->attrib.lng = __SHIFTOUT(vseg->attrib, SVM_SEG_ATTRIB_LONG); 1696 seg->attrib.def32 = __SHIFTOUT(vseg->attrib, SVM_SEG_ATTRIB_DEF32); 1697 seg->attrib.gran = __SHIFTOUT(vseg->attrib, SVM_SEG_ATTRIB_GRAN); 1698 seg->limit = vseg->limit; 1699 seg->base = vseg->base; 1700} 1701 1702static inline bool 1703svm_state_tlb_flush(struct vmcb *vmcb, struct nvmm_x64_state *state, 1704 uint64_t flags) 1705{ 1706 if (flags & NVMM_X64_STATE_CRS) { 1707 if ((vmcb->state.cr0 ^ 1708 state->crs[NVMM_X64_CR_CR0]) & CR0_TLB_FLUSH) { 1709 return true; 1710 } 1711 if (vmcb->state.cr3 != state->crs[NVMM_X64_CR_CR3]) { 1712 return true; 1713 } 1714 if ((vmcb->state.cr4 ^ 1715 state->crs[NVMM_X64_CR_CR4]) & CR4_TLB_FLUSH) { 1716 return true; 1717 } 1718 } 1719 1720 if (flags & NVMM_X64_STATE_MSRS) { 1721 if ((vmcb->state.efer ^ 1722 state->msrs[NVMM_X64_MSR_EFER]) & EFER_TLB_FLUSH) { 1723 return true; 1724 } 1725 } 1726 1727 return false; 1728} 1729 1730static void 1731svm_vcpu_setstate(struct nvmm_cpu *vcpu, void *data, uint64_t flags) 1732{ 1733 struct nvmm_x64_state *state = (struct nvmm_x64_state *)data; 1734 struct svm_cpudata *cpudata = vcpu->cpudata; 1735 struct vmcb *vmcb = cpudata->vmcb; 1736 struct fxsave *fpustate; 1737 1738 if (svm_state_tlb_flush(vmcb, state, flags)) { 1739 cpudata->tlb_want_flush = true; 1740 } 1741 1742 if (flags & NVMM_X64_STATE_SEGS) { 1743 svm_vcpu_setstate_seg(&state->segs[NVMM_X64_SEG_CS], 1744 &vmcb->state.cs); 1745 svm_vcpu_setstate_seg(&state->segs[NVMM_X64_SEG_DS], 1746 &vmcb->state.ds); 1747 svm_vcpu_setstate_seg(&state->segs[NVMM_X64_SEG_ES], 1748 &vmcb->state.es); 1749 svm_vcpu_setstate_seg(&state->segs[NVMM_X64_SEG_FS], 1750 &vmcb->state.fs); 1751 svm_vcpu_setstate_seg(&state->segs[NVMM_X64_SEG_GS], 1752 &vmcb->state.gs); 1753 svm_vcpu_setstate_seg(&state->segs[NVMM_X64_SEG_SS], 1754 &vmcb->state.ss); 1755 svm_vcpu_setstate_seg(&state->segs[NVMM_X64_SEG_GDT], 1756 &vmcb->state.gdt); 1757 svm_vcpu_setstate_seg(&state->segs[NVMM_X64_SEG_IDT], 1758 &vmcb->state.idt); 1759 svm_vcpu_setstate_seg(&state->segs[NVMM_X64_SEG_LDT], 1760 &vmcb->state.ldt); 1761 svm_vcpu_setstate_seg(&state->segs[NVMM_X64_SEG_TR], 1762 &vmcb->state.tr); 1763 1764 vmcb->state.cpl = state->segs[NVMM_X64_SEG_SS].attrib.dpl; 1765 } 1766 1767 CTASSERT(sizeof(cpudata->gprs) == sizeof(state->gprs)); 1768 if (flags & NVMM_X64_STATE_GPRS) { 1769 memcpy(cpudata->gprs, state->gprs, sizeof(state->gprs)); 1770 1771 vmcb->state.rip = state->gprs[NVMM_X64_GPR_RIP]; 1772 vmcb->state.rsp = state->gprs[NVMM_X64_GPR_RSP]; 1773 vmcb->state.rax = state->gprs[NVMM_X64_GPR_RAX]; 1774 vmcb->state.rflags = state->gprs[NVMM_X64_GPR_RFLAGS]; 1775 } 1776 1777 if (flags & NVMM_X64_STATE_CRS) { 1778 vmcb->state.cr0 = state->crs[NVMM_X64_CR_CR0]; 1779 vmcb->state.cr2 = state->crs[NVMM_X64_CR_CR2]; 1780 vmcb->state.cr3 = state->crs[NVMM_X64_CR_CR3]; 1781 vmcb->state.cr4 = state->crs[NVMM_X64_CR_CR4]; 1782 1783 vmcb->ctrl.v &= ~VMCB_CTRL_V_TPR; 1784 vmcb->ctrl.v |= __SHIFTIN(state->crs[NVMM_X64_CR_CR8], 1785 VMCB_CTRL_V_TPR); 1786 1787 if (svm_xcr0_mask != 0) { 1788 /* Clear illegal XCR0 bits, set mandatory X87 bit. */ 1789 cpudata->gxcr0 = state->crs[NVMM_X64_CR_XCR0]; 1790 cpudata->gxcr0 &= svm_xcr0_mask; 1791 cpudata->gxcr0 |= XCR0_X87; 1792 } 1793 } 1794 1795 CTASSERT(sizeof(cpudata->drs) == sizeof(state->drs)); 1796 if (flags & NVMM_X64_STATE_DRS) { 1797 memcpy(cpudata->drs, state->drs, sizeof(state->drs)); 1798 1799 vmcb->state.dr6 = state->drs[NVMM_X64_DR_DR6]; 1800 vmcb->state.dr7 = state->drs[NVMM_X64_DR_DR7]; 1801 } 1802 1803 if (flags & NVMM_X64_STATE_MSRS) { 1804 /* Bit EFER_SVME is mandatory. */ 1805 vmcb->state.efer = state->msrs[NVMM_X64_MSR_EFER] | EFER_SVME; 1806 1807 vmcb->state.star = state->msrs[NVMM_X64_MSR_STAR]; 1808 vmcb->state.lstar = state->msrs[NVMM_X64_MSR_LSTAR]; 1809 vmcb->state.cstar = state->msrs[NVMM_X64_MSR_CSTAR]; 1810 vmcb->state.sfmask = state->msrs[NVMM_X64_MSR_SFMASK]; 1811 vmcb->state.kernelgsbase = 1812 state->msrs[NVMM_X64_MSR_KERNELGSBASE]; 1813 vmcb->state.sysenter_cs = 1814 state->msrs[NVMM_X64_MSR_SYSENTER_CS]; 1815 vmcb->state.sysenter_esp = 1816 state->msrs[NVMM_X64_MSR_SYSENTER_ESP]; 1817 vmcb->state.sysenter_eip = 1818 state->msrs[NVMM_X64_MSR_SYSENTER_EIP]; 1819 vmcb->state.g_pat = state->msrs[NVMM_X64_MSR_PAT]; 1820 } 1821 1822 if (flags & NVMM_X64_STATE_MISC) { 1823 if (state->misc[NVMM_X64_MISC_INT_SHADOW]) { 1824 vmcb->ctrl.intr |= VMCB_CTRL_INTR_SHADOW; 1825 } else { 1826 vmcb->ctrl.intr &= ~VMCB_CTRL_INTR_SHADOW; 1827 } 1828 1829 if (state->misc[NVMM_X64_MISC_INT_WINDOW_EXIT]) { 1830 svm_event_waitexit_enable(vcpu, false); 1831 } else { 1832 svm_event_waitexit_disable(vcpu, false); 1833 } 1834 1835 if (state->misc[NVMM_X64_MISC_NMI_WINDOW_EXIT]) { 1836 svm_event_waitexit_enable(vcpu, true); 1837 } else { 1838 svm_event_waitexit_disable(vcpu, true); 1839 } 1840 } 1841 1842 CTASSERT(sizeof(cpudata->gfpu.xsh_fxsave) == sizeof(state->fpu)); 1843 if (flags & NVMM_X64_STATE_FPU) { 1844 memcpy(cpudata->gfpu.xsh_fxsave, &state->fpu, 1845 sizeof(state->fpu)); 1846 1847 fpustate = (struct fxsave *)cpudata->gfpu.xsh_fxsave; 1848 fpustate->fx_mxcsr_mask &= x86_fpu_mxcsr_mask; 1849 fpustate->fx_mxcsr &= fpustate->fx_mxcsr_mask; 1850 1851 if (svm_xcr0_mask != 0) { 1852 /* Reset XSTATE_BV, to force a reload. */ 1853 cpudata->gfpu.xsh_xstate_bv = svm_xcr0_mask; 1854 } 1855 } 1856 1857 svm_vmcb_cache_update(vmcb, flags); 1858} 1859 1860static void 1861svm_vcpu_getstate(struct nvmm_cpu *vcpu, void *data, uint64_t flags) 1862{ 1863 struct nvmm_x64_state *state = (struct nvmm_x64_state *)data; 1864 struct svm_cpudata *cpudata = vcpu->cpudata; 1865 struct vmcb *vmcb = cpudata->vmcb; 1866 1867 if (flags & NVMM_X64_STATE_SEGS) { 1868 svm_vcpu_getstate_seg(&state->segs[NVMM_X64_SEG_CS], 1869 &vmcb->state.cs); 1870 svm_vcpu_getstate_seg(&state->segs[NVMM_X64_SEG_DS], 1871 &vmcb->state.ds); 1872 svm_vcpu_getstate_seg(&state->segs[NVMM_X64_SEG_ES], 1873 &vmcb->state.es); 1874 svm_vcpu_getstate_seg(&state->segs[NVMM_X64_SEG_FS], 1875 &vmcb->state.fs); 1876 svm_vcpu_getstate_seg(&state->segs[NVMM_X64_SEG_GS], 1877 &vmcb->state.gs); 1878 svm_vcpu_getstate_seg(&state->segs[NVMM_X64_SEG_SS], 1879 &vmcb->state.ss); 1880 svm_vcpu_getstate_seg(&state->segs[NVMM_X64_SEG_GDT], 1881 &vmcb->state.gdt); 1882 svm_vcpu_getstate_seg(&state->segs[NVMM_X64_SEG_IDT], 1883 &vmcb->state.idt); 1884 svm_vcpu_getstate_seg(&state->segs[NVMM_X64_SEG_LDT], 1885 &vmcb->state.ldt); 1886 svm_vcpu_getstate_seg(&state->segs[NVMM_X64_SEG_TR], 1887 &vmcb->state.tr); 1888 1889 state->segs[NVMM_X64_SEG_SS].attrib.dpl = vmcb->state.cpl; 1890 } 1891 1892 CTASSERT(sizeof(cpudata->gprs) == sizeof(state->gprs)); 1893 if (flags & NVMM_X64_STATE_GPRS) { 1894 memcpy(state->gprs, cpudata->gprs, sizeof(state->gprs)); 1895 1896 state->gprs[NVMM_X64_GPR_RIP] = vmcb->state.rip; 1897 state->gprs[NVMM_X64_GPR_RSP] = vmcb->state.rsp; 1898 state->gprs[NVMM_X64_GPR_RAX] = vmcb->state.rax; 1899 state->gprs[NVMM_X64_GPR_RFLAGS] = vmcb->state.rflags; 1900 } 1901 1902 if (flags & NVMM_X64_STATE_CRS) { 1903 state->crs[NVMM_X64_CR_CR0] = vmcb->state.cr0; 1904 state->crs[NVMM_X64_CR_CR2] = vmcb->state.cr2; 1905 state->crs[NVMM_X64_CR_CR3] = vmcb->state.cr3; 1906 state->crs[NVMM_X64_CR_CR4] = vmcb->state.cr4; 1907 state->crs[NVMM_X64_CR_CR8] = __SHIFTOUT(vmcb->ctrl.v, 1908 VMCB_CTRL_V_TPR); 1909 state->crs[NVMM_X64_CR_XCR0] = cpudata->gxcr0; 1910 } 1911 1912 CTASSERT(sizeof(cpudata->drs) == sizeof(state->drs)); 1913 if (flags & NVMM_X64_STATE_DRS) { 1914 memcpy(state->drs, cpudata->drs, sizeof(state->drs)); 1915 1916 state->drs[NVMM_X64_DR_DR6] = vmcb->state.dr6; 1917 state->drs[NVMM_X64_DR_DR7] = vmcb->state.dr7; 1918 } 1919 1920 if (flags & NVMM_X64_STATE_MSRS) { 1921 state->msrs[NVMM_X64_MSR_EFER] = vmcb->state.efer; 1922 state->msrs[NVMM_X64_MSR_STAR] = vmcb->state.star; 1923 state->msrs[NVMM_X64_MSR_LSTAR] = vmcb->state.lstar; 1924 state->msrs[NVMM_X64_MSR_CSTAR] = vmcb->state.cstar; 1925 state->msrs[NVMM_X64_MSR_SFMASK] = vmcb->state.sfmask; 1926 state->msrs[NVMM_X64_MSR_KERNELGSBASE] = 1927 vmcb->state.kernelgsbase; 1928 state->msrs[NVMM_X64_MSR_SYSENTER_CS] = 1929 vmcb->state.sysenter_cs; 1930 state->msrs[NVMM_X64_MSR_SYSENTER_ESP] = 1931 vmcb->state.sysenter_esp; 1932 state->msrs[NVMM_X64_MSR_SYSENTER_EIP] = 1933 vmcb->state.sysenter_eip; 1934 state->msrs[NVMM_X64_MSR_PAT] = vmcb->state.g_pat; 1935 1936 /* Hide SVME. */ 1937 state->msrs[NVMM_X64_MSR_EFER] &= ~EFER_SVME; 1938 } 1939 1940 if (flags & NVMM_X64_STATE_MISC) { 1941 state->misc[NVMM_X64_MISC_INT_SHADOW] = 1942 (vmcb->ctrl.intr & VMCB_CTRL_INTR_SHADOW) != 0; 1943 state->misc[NVMM_X64_MISC_INT_WINDOW_EXIT] = 1944 cpudata->int_window_exit; 1945 state->misc[NVMM_X64_MISC_NMI_WINDOW_EXIT] = 1946 cpudata->nmi_window_exit; 1947 } 1948 1949 CTASSERT(sizeof(cpudata->gfpu.xsh_fxsave) == sizeof(state->fpu)); 1950 if (flags & NVMM_X64_STATE_FPU) { 1951 memcpy(&state->fpu, cpudata->gfpu.xsh_fxsave, 1952 sizeof(state->fpu)); 1953 } 1954} 1955 1956/* -------------------------------------------------------------------------- */ 1957 1958static void 1959svm_tlb_flush(struct pmap *pm) 1960{ 1961 struct nvmm_machine *mach = pm->pm_data; 1962 struct svm_cpudata *cpudata; 1963 struct nvmm_cpu *vcpu; 1964 int error; 1965 size_t i; 1966 1967 /* Request TLB flushes. */ 1968 for (i = 0; i < NVMM_MAX_VCPUS; i++) { 1969 error = nvmm_vcpu_get(mach, i, &vcpu); 1970 if (error) 1971 continue; 1972 cpudata = vcpu->cpudata; 1973 cpudata->tlb_want_flush = true; 1974 nvmm_vcpu_put(vcpu); 1975 } 1976} 1977 1978static void 1979svm_machine_create(struct nvmm_machine *mach) 1980{ 1981 /* Fill in pmap info. */ 1982 mach->vm->vm_map.pmap->pm_data = (void *)mach; 1983 mach->vm->vm_map.pmap->pm_tlb_flush = svm_tlb_flush; 1984 1985 mach->machdata = kmem_zalloc(sizeof(struct svm_machdata), KM_SLEEP); 1986} 1987 1988static void 1989svm_machine_destroy(struct nvmm_machine *mach) 1990{ 1991 kmem_free(mach->machdata, sizeof(struct svm_machdata)); 1992} 1993 1994static int 1995svm_machine_configure(struct nvmm_machine *mach, uint64_t op, void *data) 1996{ 1997 struct nvmm_x86_conf_cpuid *cpuid = data; 1998 struct svm_machdata *machdata = (struct svm_machdata *)mach->machdata; 1999 size_t i; 2000 2001 if (__predict_false(op != NVMM_X86_CONF_CPUID)) { 2002 return EINVAL; 2003 } 2004 2005 if (__predict_false((cpuid->set.eax & cpuid->del.eax) || 2006 (cpuid->set.ebx & cpuid->del.ebx) || 2007 (cpuid->set.ecx & cpuid->del.ecx) || 2008 (cpuid->set.edx & cpuid->del.edx))) { 2009 return EINVAL; 2010 } 2011 2012 /* If already here, replace. */ 2013 for (i = 0; i < SVM_NCPUIDS; i++) { 2014 if (!machdata->cpuidpresent[i]) { 2015 continue; 2016 } 2017 if (machdata->cpuid[i].leaf == cpuid->leaf) { 2018 memcpy(&machdata->cpuid[i], cpuid, 2019 sizeof(struct nvmm_x86_conf_cpuid)); 2020 return 0; 2021 } 2022 } 2023 2024 /* Not here, insert. */ 2025 for (i = 0; i < SVM_NCPUIDS; i++) { 2026 if (!machdata->cpuidpresent[i]) { 2027 machdata->cpuidpresent[i] = true; 2028 memcpy(&machdata->cpuid[i], cpuid, 2029 sizeof(struct nvmm_x86_conf_cpuid)); 2030 return 0; 2031 } 2032 } 2033 2034 return ENOBUFS; 2035} 2036 2037/* -------------------------------------------------------------------------- */ 2038 2039static bool 2040svm_ident(void) 2041{ 2042 u_int descs[4]; 2043 uint64_t msr; 2044 2045 if (cpu_vendor != CPUVENDOR_AMD) { 2046 return false; 2047 } 2048 if (!(cpu_feature[3] & CPUID_SVM)) { 2049 return false; 2050 } 2051 2052 if (curcpu()->ci_max_ext_cpuid < 0x8000000a) { 2053 return false; 2054 } 2055 x86_cpuid(0x8000000a, descs); 2056 2057 /* Want Nested Paging. */ 2058 if (!(descs[3] & CPUID_AMD_SVM_NP)) { 2059 return false; 2060 } 2061 2062 /* Want nRIP. */ 2063 if (!(descs[3] & CPUID_AMD_SVM_NRIPS)) { 2064 return false; 2065 } 2066 2067 svm_decode_assist = (descs[3] & CPUID_AMD_SVM_DecodeAssist) != 0; 2068 2069 msr = rdmsr(MSR_VMCR); 2070 if ((msr & VMCR_SVMED) && (msr & VMCR_LOCK)) { 2071 return false; 2072 } 2073 2074 return true; 2075} 2076 2077static void 2078svm_init_asid(uint32_t maxasid) 2079{ 2080 size_t i, j, allocsz; 2081 2082 mutex_init(&svm_asidlock, MUTEX_DEFAULT, IPL_NONE); 2083 2084 /* Arbitrarily limit. */ 2085 maxasid = uimin(maxasid, 8192); 2086 2087 svm_maxasid = maxasid; 2088 allocsz = roundup(maxasid, 8) / 8; 2089 svm_asidmap = kmem_zalloc(allocsz, KM_SLEEP); 2090 2091 /* ASID 0 is reserved for the host. */ 2092 svm_asidmap[0] |= __BIT(0); 2093 2094 /* ASID n-1 is special, we share it. */ 2095 i = (maxasid - 1) / 8; 2096 j = (maxasid - 1) % 8; 2097 svm_asidmap[i] |= __BIT(j); 2098} 2099 2100static void 2101svm_change_cpu(void *arg1, void *arg2) 2102{ 2103 bool enable = (bool)arg1; 2104 uint64_t msr; 2105 2106 msr = rdmsr(MSR_VMCR); 2107 if (msr & VMCR_SVMED) { 2108 wrmsr(MSR_VMCR, msr & ~VMCR_SVMED); 2109 } 2110 2111 if (!enable) { 2112 wrmsr(MSR_VM_HSAVE_PA, 0); 2113 } 2114 2115 msr = rdmsr(MSR_EFER); 2116 if (enable) { 2117 msr |= EFER_SVME; 2118 } else { 2119 msr &= ~EFER_SVME; 2120 } 2121 wrmsr(MSR_EFER, msr); 2122 2123 if (enable) { 2124 wrmsr(MSR_VM_HSAVE_PA, hsave[cpu_index(curcpu())].pa); 2125 } 2126} 2127 2128static void 2129svm_init(void) 2130{ 2131 CPU_INFO_ITERATOR cii; 2132 struct cpu_info *ci; 2133 struct vm_page *pg; 2134 u_int descs[4]; 2135 uint64_t xc; 2136 2137 x86_cpuid(0x8000000a, descs); 2138 2139 /* The guest TLB flush command. */ 2140 if (descs[3] & CPUID_AMD_SVM_FlushByASID) { 2141 svm_ctrl_tlb_flush = VMCB_CTRL_TLB_CTRL_FLUSH_GUEST; 2142 } else { 2143 svm_ctrl_tlb_flush = VMCB_CTRL_TLB_CTRL_FLUSH_ALL; 2144 } 2145 2146 /* Init the ASID. */ 2147 svm_init_asid(descs[1]); 2148 2149 /* Init the XCR0 mask. */ 2150 svm_xcr0_mask = SVM_XCR0_MASK_DEFAULT & x86_xsave_features; 2151 2152 memset(hsave, 0, sizeof(hsave)); 2153 for (CPU_INFO_FOREACH(cii, ci)) { 2154 pg = uvm_pagealloc(NULL, 0, NULL, UVM_PGA_ZERO); 2155 hsave[cpu_index(ci)].pa = VM_PAGE_TO_PHYS(pg); 2156 } 2157 2158 xc = xc_broadcast(0, svm_change_cpu, (void *)true, NULL); 2159 xc_wait(xc); 2160} 2161 2162static void 2163svm_fini_asid(void) 2164{ 2165 size_t allocsz; 2166 2167 allocsz = roundup(svm_maxasid, 8) / 8; 2168 kmem_free(svm_asidmap, allocsz); 2169 2170 mutex_destroy(&svm_asidlock); 2171} 2172 2173static void 2174svm_fini(void) 2175{ 2176 uint64_t xc; 2177 size_t i; 2178 2179 xc = xc_broadcast(0, svm_change_cpu, (void *)false, NULL); 2180 xc_wait(xc); 2181 2182 for (i = 0; i < MAXCPUS; i++) { 2183 if (hsave[i].pa != 0) 2184 uvm_pagefree(PHYS_TO_VM_PAGE(hsave[i].pa)); 2185 } 2186 2187 svm_fini_asid(); 2188} 2189 2190static void 2191svm_capability(struct nvmm_capability *cap) 2192{ 2193 cap->u.x86.xcr0_mask = svm_xcr0_mask; 2194 cap->u.x86.mxcsr_mask = x86_fpu_mxcsr_mask; 2195 cap->u.x86.conf_cpuid_maxops = SVM_NCPUIDS; 2196} 2197 2198const struct nvmm_impl nvmm_x86_svm = { 2199 .ident = svm_ident, 2200 .init = svm_init, 2201 .fini = svm_fini, 2202 .capability = svm_capability, 2203 .conf_max = NVMM_X86_NCONF, 2204 .conf_sizes = svm_conf_sizes, 2205 .state_size = sizeof(struct nvmm_x64_state), 2206 .machine_create = svm_machine_create, 2207 .machine_destroy = svm_machine_destroy, 2208 .machine_configure = svm_machine_configure, 2209 .vcpu_create = svm_vcpu_create, 2210 .vcpu_destroy = svm_vcpu_destroy, 2211 .vcpu_setstate = svm_vcpu_setstate, 2212 .vcpu_getstate = svm_vcpu_getstate, 2213 .vcpu_inject = svm_vcpu_inject, 2214 .vcpu_run = svm_vcpu_run 2215}; 2216