1221828Sgrehan/*- 2221828Sgrehan * Copyright (c) 2011 NetApp, Inc. 3221828Sgrehan * All rights reserved. 4221828Sgrehan * 5221828Sgrehan * Redistribution and use in source and binary forms, with or without 6221828Sgrehan * modification, are permitted provided that the following conditions 7221828Sgrehan * are met: 8221828Sgrehan * 1. Redistributions of source code must retain the above copyright 9221828Sgrehan * notice, this list of conditions and the following disclaimer. 10221828Sgrehan * 2. Redistributions in binary form must reproduce the above copyright 11221828Sgrehan * notice, this list of conditions and the following disclaimer in the 12221828Sgrehan * documentation and/or other materials provided with the distribution. 13221828Sgrehan * 14221828Sgrehan * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND 15221828Sgrehan * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16221828Sgrehan * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17221828Sgrehan * ARE DISCLAIMED. IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE 18221828Sgrehan * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19221828Sgrehan * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20221828Sgrehan * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21221828Sgrehan * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22221828Sgrehan * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23221828Sgrehan * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24221828Sgrehan * SUCH DAMAGE. 25221828Sgrehan * 26221828Sgrehan * $FreeBSD: releng/10.2/sys/amd64/vmm/intel/ept.c 276349 2014-12-28 21:27:13Z neel $ 27221828Sgrehan */ 28221828Sgrehan 29221828Sgrehan#include <sys/cdefs.h> 30221828Sgrehan__FBSDID("$FreeBSD: releng/10.2/sys/amd64/vmm/intel/ept.c 276349 2014-12-28 21:27:13Z neel $"); 31221828Sgrehan 32256072Sneel#include <sys/param.h> 33256072Sneel#include <sys/kernel.h> 34221828Sgrehan#include <sys/types.h> 35221828Sgrehan#include <sys/systm.h> 36221828Sgrehan#include <sys/smp.h> 37256072Sneel#include <sys/sysctl.h> 38221828Sgrehan 39221828Sgrehan#include <vm/vm.h> 40221828Sgrehan#include <vm/pmap.h> 41256072Sneel#include <vm/vm_extern.h> 42221828Sgrehan 43256072Sneel#include <machine/vmm.h> 44221828Sgrehan 45221828Sgrehan#include "vmx_cpufunc.h" 46266339Sjhb#include "vmm_ipi.h" 47221828Sgrehan#include "ept.h" 48221828Sgrehan 49256072Sneel#define EPT_SUPPORTS_EXEC_ONLY(cap) ((cap) & (1UL << 0)) 50221828Sgrehan#define EPT_PWL4(cap) ((cap) & (1UL << 6)) 51221828Sgrehan#define EPT_MEMORY_TYPE_WB(cap) ((cap) & (1UL << 14)) 52221828Sgrehan#define EPT_PDE_SUPERPAGE(cap) ((cap) & (1UL << 16)) /* 2MB pages */ 53221828Sgrehan#define EPT_PDPTE_SUPERPAGE(cap) ((cap) & (1UL << 17)) /* 1GB pages */ 54256072Sneel#define INVEPT_SUPPORTED(cap) ((cap) & (1UL << 20)) 55256072Sneel#define AD_BITS_SUPPORTED(cap) ((cap) & (1UL << 21)) 56221828Sgrehan#define INVVPID_SUPPORTED(cap) ((cap) & (1UL << 32)) 57221828Sgrehan 58221828Sgrehan#define INVVPID_ALL_TYPES_MASK 0xF0000000000UL 59221828Sgrehan#define INVVPID_ALL_TYPES_SUPPORTED(cap) \ 60221828Sgrehan (((cap) & INVVPID_ALL_TYPES_MASK) == INVVPID_ALL_TYPES_MASK) 61221828Sgrehan 62221828Sgrehan#define INVEPT_ALL_TYPES_MASK 0x6000000UL 63221828Sgrehan#define INVEPT_ALL_TYPES_SUPPORTED(cap) \ 64221828Sgrehan (((cap) & INVEPT_ALL_TYPES_MASK) == INVEPT_ALL_TYPES_MASK) 65221828Sgrehan 66256072Sneel#define EPT_PWLEVELS 4 /* page walk levels */ 67256072Sneel#define EPT_ENABLE_AD_BITS (1 << 6) 68221828Sgrehan 69256072SneelSYSCTL_DECL(_hw_vmm); 70256072SneelSYSCTL_NODE(_hw_vmm, OID_AUTO, ept, CTLFLAG_RW, NULL, NULL); 71221828Sgrehan 72256072Sneelstatic int ept_enable_ad_bits; 73221828Sgrehan 74256072Sneelstatic int ept_pmap_flags; 75256072SneelSYSCTL_INT(_hw_vmm_ept, OID_AUTO, pmap_flags, CTLFLAG_RD, 76256072Sneel &ept_pmap_flags, 0, NULL); 77221828Sgrehan 78221828Sgrehanint 79266339Sjhbept_init(int ipinum) 80221828Sgrehan{ 81256072Sneel int use_hw_ad_bits, use_superpages, use_exec_only; 82221828Sgrehan uint64_t cap; 83221828Sgrehan 84221828Sgrehan cap = rdmsr(MSR_VMX_EPT_VPID_CAP); 85221828Sgrehan 86221828Sgrehan /* 87221828Sgrehan * Verify that: 88221828Sgrehan * - page walk length is 4 steps 89221828Sgrehan * - extended page tables can be laid out in write-back memory 90221828Sgrehan * - invvpid instruction with all possible types is supported 91221828Sgrehan * - invept instruction with all possible types is supported 92221828Sgrehan */ 93221828Sgrehan if (!EPT_PWL4(cap) || 94221828Sgrehan !EPT_MEMORY_TYPE_WB(cap) || 95221828Sgrehan !INVVPID_SUPPORTED(cap) || 96221828Sgrehan !INVVPID_ALL_TYPES_SUPPORTED(cap) || 97221828Sgrehan !INVEPT_SUPPORTED(cap) || 98221828Sgrehan !INVEPT_ALL_TYPES_SUPPORTED(cap)) 99221828Sgrehan return (EINVAL); 100221828Sgrehan 101266339Sjhb ept_pmap_flags = ipinum & PMAP_NESTED_IPIMASK; 102266339Sjhb 103256072Sneel use_superpages = 1; 104256072Sneel TUNABLE_INT_FETCH("hw.vmm.ept.use_superpages", &use_superpages); 105256072Sneel if (use_superpages && EPT_PDE_SUPERPAGE(cap)) 106256072Sneel ept_pmap_flags |= PMAP_PDE_SUPERPAGE; /* 2MB superpage */ 107221828Sgrehan 108256072Sneel use_hw_ad_bits = 1; 109256072Sneel TUNABLE_INT_FETCH("hw.vmm.ept.use_hw_ad_bits", &use_hw_ad_bits); 110256072Sneel if (use_hw_ad_bits && AD_BITS_SUPPORTED(cap)) 111256072Sneel ept_enable_ad_bits = 1; 112256072Sneel else 113256072Sneel ept_pmap_flags |= PMAP_EMULATE_AD_BITS; 114221828Sgrehan 115256072Sneel use_exec_only = 1; 116256072Sneel TUNABLE_INT_FETCH("hw.vmm.ept.use_exec_only", &use_exec_only); 117256072Sneel if (use_exec_only && EPT_SUPPORTS_EXEC_ONLY(cap)) 118256072Sneel ept_pmap_flags |= PMAP_SUPPORTS_EXEC_ONLY; 119221828Sgrehan 120221828Sgrehan return (0); 121221828Sgrehan} 122221828Sgrehan 123241147Sneel#if 0 124241147Sneelstatic void 125241147Sneelept_dump(uint64_t *ptp, int nlevels) 126241147Sneel{ 127241147Sneel int i, t, tabs; 128241147Sneel uint64_t *ptpnext, ptpval; 129241147Sneel 130241147Sneel if (--nlevels < 0) 131241147Sneel return; 132241147Sneel 133241147Sneel tabs = 3 - nlevels; 134241147Sneel for (t = 0; t < tabs; t++) 135241147Sneel printf("\t"); 136241147Sneel printf("PTP = %p\n", ptp); 137241147Sneel 138241147Sneel for (i = 0; i < 512; i++) { 139241147Sneel ptpval = ptp[i]; 140241147Sneel 141241147Sneel if (ptpval == 0) 142241147Sneel continue; 143241147Sneel 144241147Sneel for (t = 0; t < tabs; t++) 145241147Sneel printf("\t"); 146241147Sneel printf("%3d 0x%016lx\n", i, ptpval); 147241147Sneel 148241147Sneel if (nlevels != 0 && (ptpval & EPT_PG_SUPERPAGE) == 0) { 149241147Sneel ptpnext = (uint64_t *) 150241147Sneel PHYS_TO_DMAP(ptpval & EPT_ADDR_MASK); 151241147Sneel ept_dump(ptpnext, nlevels); 152241147Sneel } 153241147Sneel } 154241147Sneel} 155241147Sneel#endif 156241147Sneel 157221828Sgrehanstatic void 158256072Sneelinvept_single_context(void *arg) 159221828Sgrehan{ 160256072Sneel struct invept_desc desc = *(struct invept_desc *)arg; 161221828Sgrehan 162256072Sneel invept(INVEPT_TYPE_SINGLE_CONTEXT, desc); 163221828Sgrehan} 164221828Sgrehan 165256072Sneelvoid 166256072Sneelept_invalidate_mappings(u_long eptp) 167221828Sgrehan{ 168256072Sneel struct invept_desc invept_desc = { 0 }; 169221828Sgrehan 170256072Sneel invept_desc.eptp = eptp; 171221828Sgrehan 172256072Sneel smp_rendezvous(NULL, invept_single_context, NULL, &invept_desc); 173221828Sgrehan} 174221828Sgrehan 175256072Sneelstatic int 176256072Sneelept_pinit(pmap_t pmap) 177221828Sgrehan{ 178221828Sgrehan 179256072Sneel return (pmap_pinit_type(pmap, PT_EPT, ept_pmap_flags)); 180221828Sgrehan} 181221828Sgrehan 182256072Sneelstruct vmspace * 183256072Sneelept_vmspace_alloc(vm_offset_t min, vm_offset_t max) 184221828Sgrehan{ 185221828Sgrehan 186256072Sneel return (vmspace_alloc(min, max, ept_pinit)); 187221828Sgrehan} 188221828Sgrehan 189221828Sgrehanvoid 190256072Sneelept_vmspace_free(struct vmspace *vmspace) 191221828Sgrehan{ 192221828Sgrehan 193256072Sneel vmspace_free(vmspace); 194221828Sgrehan} 195221828Sgrehan 196256072Sneeluint64_t 197256072Sneeleptp(uint64_t pml4) 198221828Sgrehan{ 199256072Sneel uint64_t eptp_val; 200221828Sgrehan 201256072Sneel eptp_val = pml4 | (EPT_PWLEVELS - 1) << 3 | PAT_WRITE_BACK; 202256072Sneel if (ept_enable_ad_bits) 203256072Sneel eptp_val |= EPT_ENABLE_AD_BITS; 204221828Sgrehan 205256072Sneel return (eptp_val); 206221828Sgrehan} 207