ept.c revision 266339
1221828Sgrehan/*- 2221828Sgrehan * Copyright (c) 2011 NetApp, Inc. 3221828Sgrehan * All rights reserved. 4221828Sgrehan * 5221828Sgrehan * Redistribution and use in source and binary forms, with or without 6221828Sgrehan * modification, are permitted provided that the following conditions 7221828Sgrehan * are met: 8221828Sgrehan * 1. Redistributions of source code must retain the above copyright 9221828Sgrehan * notice, this list of conditions and the following disclaimer. 10221828Sgrehan * 2. Redistributions in binary form must reproduce the above copyright 11221828Sgrehan * notice, this list of conditions and the following disclaimer in the 12221828Sgrehan * documentation and/or other materials provided with the distribution. 13221828Sgrehan * 14221828Sgrehan * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND 15221828Sgrehan * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16221828Sgrehan * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17221828Sgrehan * ARE DISCLAIMED. IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE 18221828Sgrehan * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19221828Sgrehan * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20221828Sgrehan * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21221828Sgrehan * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22221828Sgrehan * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23221828Sgrehan * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24221828Sgrehan * SUCH DAMAGE. 25221828Sgrehan * 26221828Sgrehan * $FreeBSD: stable/10/sys/amd64/vmm/intel/ept.c 266339 2014-05-17 19:11:08Z jhb $ 27221828Sgrehan */ 28221828Sgrehan 29221828Sgrehan#include <sys/cdefs.h> 30221828Sgrehan__FBSDID("$FreeBSD: stable/10/sys/amd64/vmm/intel/ept.c 266339 2014-05-17 19:11:08Z jhb $"); 31221828Sgrehan 32256072Sneel#include <sys/param.h> 33256072Sneel#include <sys/kernel.h> 34221828Sgrehan#include <sys/types.h> 35221828Sgrehan#include <sys/systm.h> 36221828Sgrehan#include <sys/smp.h> 37256072Sneel#include <sys/sysctl.h> 38221828Sgrehan 39221828Sgrehan#include <vm/vm.h> 40221828Sgrehan#include <vm/pmap.h> 41256072Sneel#include <vm/vm_extern.h> 42221828Sgrehan 43256072Sneel#include <machine/vmm.h> 44221828Sgrehan 45221828Sgrehan#include "vmx_cpufunc.h" 46266339Sjhb#include "vmm_ipi.h" 47221828Sgrehan#include "vmx_msr.h" 48221828Sgrehan#include "ept.h" 49221828Sgrehan 50256072Sneel#define EPT_SUPPORTS_EXEC_ONLY(cap) ((cap) & (1UL << 0)) 51221828Sgrehan#define EPT_PWL4(cap) ((cap) & (1UL << 6)) 52221828Sgrehan#define EPT_MEMORY_TYPE_WB(cap) ((cap) & (1UL << 14)) 53221828Sgrehan#define EPT_PDE_SUPERPAGE(cap) ((cap) & (1UL << 16)) /* 2MB pages */ 54221828Sgrehan#define EPT_PDPTE_SUPERPAGE(cap) ((cap) & (1UL << 17)) /* 1GB pages */ 55256072Sneel#define INVEPT_SUPPORTED(cap) ((cap) & (1UL << 20)) 56256072Sneel#define AD_BITS_SUPPORTED(cap) ((cap) & (1UL << 21)) 57221828Sgrehan#define INVVPID_SUPPORTED(cap) ((cap) & (1UL << 32)) 58221828Sgrehan 59221828Sgrehan#define INVVPID_ALL_TYPES_MASK 0xF0000000000UL 60221828Sgrehan#define INVVPID_ALL_TYPES_SUPPORTED(cap) \ 61221828Sgrehan (((cap) & INVVPID_ALL_TYPES_MASK) == INVVPID_ALL_TYPES_MASK) 62221828Sgrehan 63221828Sgrehan#define INVEPT_ALL_TYPES_MASK 0x6000000UL 64221828Sgrehan#define INVEPT_ALL_TYPES_SUPPORTED(cap) \ 65221828Sgrehan (((cap) & INVEPT_ALL_TYPES_MASK) == INVEPT_ALL_TYPES_MASK) 66221828Sgrehan 67256072Sneel#define EPT_PWLEVELS 4 /* page walk levels */ 68256072Sneel#define EPT_ENABLE_AD_BITS (1 << 6) 69221828Sgrehan 70256072SneelSYSCTL_DECL(_hw_vmm); 71256072SneelSYSCTL_NODE(_hw_vmm, OID_AUTO, ept, CTLFLAG_RW, NULL, NULL); 72221828Sgrehan 73256072Sneelstatic int ept_enable_ad_bits; 74221828Sgrehan 75256072Sneelstatic int ept_pmap_flags; 76256072SneelSYSCTL_INT(_hw_vmm_ept, OID_AUTO, pmap_flags, CTLFLAG_RD, 77256072Sneel &ept_pmap_flags, 0, NULL); 78221828Sgrehan 79221828Sgrehanint 80266339Sjhbept_init(int ipinum) 81221828Sgrehan{ 82256072Sneel int use_hw_ad_bits, use_superpages, use_exec_only; 83221828Sgrehan uint64_t cap; 84221828Sgrehan 85221828Sgrehan cap = rdmsr(MSR_VMX_EPT_VPID_CAP); 86221828Sgrehan 87221828Sgrehan /* 88221828Sgrehan * Verify that: 89221828Sgrehan * - page walk length is 4 steps 90221828Sgrehan * - extended page tables can be laid out in write-back memory 91221828Sgrehan * - invvpid instruction with all possible types is supported 92221828Sgrehan * - invept instruction with all possible types is supported 93221828Sgrehan */ 94221828Sgrehan if (!EPT_PWL4(cap) || 95221828Sgrehan !EPT_MEMORY_TYPE_WB(cap) || 96221828Sgrehan !INVVPID_SUPPORTED(cap) || 97221828Sgrehan !INVVPID_ALL_TYPES_SUPPORTED(cap) || 98221828Sgrehan !INVEPT_SUPPORTED(cap) || 99221828Sgrehan !INVEPT_ALL_TYPES_SUPPORTED(cap)) 100221828Sgrehan return (EINVAL); 101221828Sgrehan 102266339Sjhb ept_pmap_flags = ipinum & PMAP_NESTED_IPIMASK; 103266339Sjhb 104256072Sneel use_superpages = 1; 105256072Sneel TUNABLE_INT_FETCH("hw.vmm.ept.use_superpages", &use_superpages); 106256072Sneel if (use_superpages && EPT_PDE_SUPERPAGE(cap)) 107256072Sneel ept_pmap_flags |= PMAP_PDE_SUPERPAGE; /* 2MB superpage */ 108221828Sgrehan 109256072Sneel use_hw_ad_bits = 1; 110256072Sneel TUNABLE_INT_FETCH("hw.vmm.ept.use_hw_ad_bits", &use_hw_ad_bits); 111256072Sneel if (use_hw_ad_bits && AD_BITS_SUPPORTED(cap)) 112256072Sneel ept_enable_ad_bits = 1; 113256072Sneel else 114256072Sneel ept_pmap_flags |= PMAP_EMULATE_AD_BITS; 115221828Sgrehan 116256072Sneel use_exec_only = 1; 117256072Sneel TUNABLE_INT_FETCH("hw.vmm.ept.use_exec_only", &use_exec_only); 118256072Sneel if (use_exec_only && EPT_SUPPORTS_EXEC_ONLY(cap)) 119256072Sneel ept_pmap_flags |= PMAP_SUPPORTS_EXEC_ONLY; 120221828Sgrehan 121221828Sgrehan return (0); 122221828Sgrehan} 123221828Sgrehan 124241147Sneel#if 0 125241147Sneelstatic void 126241147Sneelept_dump(uint64_t *ptp, int nlevels) 127241147Sneel{ 128241147Sneel int i, t, tabs; 129241147Sneel uint64_t *ptpnext, ptpval; 130241147Sneel 131241147Sneel if (--nlevels < 0) 132241147Sneel return; 133241147Sneel 134241147Sneel tabs = 3 - nlevels; 135241147Sneel for (t = 0; t < tabs; t++) 136241147Sneel printf("\t"); 137241147Sneel printf("PTP = %p\n", ptp); 138241147Sneel 139241147Sneel for (i = 0; i < 512; i++) { 140241147Sneel ptpval = ptp[i]; 141241147Sneel 142241147Sneel if (ptpval == 0) 143241147Sneel continue; 144241147Sneel 145241147Sneel for (t = 0; t < tabs; t++) 146241147Sneel printf("\t"); 147241147Sneel printf("%3d 0x%016lx\n", i, ptpval); 148241147Sneel 149241147Sneel if (nlevels != 0 && (ptpval & EPT_PG_SUPERPAGE) == 0) { 150241147Sneel ptpnext = (uint64_t *) 151241147Sneel PHYS_TO_DMAP(ptpval & EPT_ADDR_MASK); 152241147Sneel ept_dump(ptpnext, nlevels); 153241147Sneel } 154241147Sneel } 155241147Sneel} 156241147Sneel#endif 157241147Sneel 158221828Sgrehanstatic void 159256072Sneelinvept_single_context(void *arg) 160221828Sgrehan{ 161256072Sneel struct invept_desc desc = *(struct invept_desc *)arg; 162221828Sgrehan 163256072Sneel invept(INVEPT_TYPE_SINGLE_CONTEXT, desc); 164221828Sgrehan} 165221828Sgrehan 166256072Sneelvoid 167256072Sneelept_invalidate_mappings(u_long eptp) 168221828Sgrehan{ 169256072Sneel struct invept_desc invept_desc = { 0 }; 170221828Sgrehan 171256072Sneel invept_desc.eptp = eptp; 172221828Sgrehan 173256072Sneel smp_rendezvous(NULL, invept_single_context, NULL, &invept_desc); 174221828Sgrehan} 175221828Sgrehan 176256072Sneelstatic int 177256072Sneelept_pinit(pmap_t pmap) 178221828Sgrehan{ 179221828Sgrehan 180256072Sneel return (pmap_pinit_type(pmap, PT_EPT, ept_pmap_flags)); 181221828Sgrehan} 182221828Sgrehan 183256072Sneelstruct vmspace * 184256072Sneelept_vmspace_alloc(vm_offset_t min, vm_offset_t max) 185221828Sgrehan{ 186221828Sgrehan 187256072Sneel return (vmspace_alloc(min, max, ept_pinit)); 188221828Sgrehan} 189221828Sgrehan 190221828Sgrehanvoid 191256072Sneelept_vmspace_free(struct vmspace *vmspace) 192221828Sgrehan{ 193221828Sgrehan 194256072Sneel vmspace_free(vmspace); 195221828Sgrehan} 196221828Sgrehan 197256072Sneeluint64_t 198256072Sneeleptp(uint64_t pml4) 199221828Sgrehan{ 200256072Sneel uint64_t eptp_val; 201221828Sgrehan 202256072Sneel eptp_val = pml4 | (EPT_PWLEVELS - 1) << 3 | PAT_WRITE_BACK; 203256072Sneel if (ept_enable_ad_bits) 204256072Sneel eptp_val |= EPT_ENABLE_AD_BITS; 205221828Sgrehan 206256072Sneel return (eptp_val); 207221828Sgrehan} 208