1221828Sgrehan/*- 2221828Sgrehan * Copyright (c) 2011 NetApp, Inc. 3221828Sgrehan * All rights reserved. 4221828Sgrehan * 5221828Sgrehan * Redistribution and use in source and binary forms, with or without 6221828Sgrehan * modification, are permitted provided that the following conditions 7221828Sgrehan * are met: 8221828Sgrehan * 1. Redistributions of source code must retain the above copyright 9221828Sgrehan * notice, this list of conditions and the following disclaimer. 10221828Sgrehan * 2. Redistributions in binary form must reproduce the above copyright 11221828Sgrehan * notice, this list of conditions and the following disclaimer in the 12221828Sgrehan * documentation and/or other materials provided with the distribution. 13221828Sgrehan * 14221828Sgrehan * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND 15221828Sgrehan * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16221828Sgrehan * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17221828Sgrehan * ARE DISCLAIMED. IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE 18221828Sgrehan * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19221828Sgrehan * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20221828Sgrehan * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21221828Sgrehan * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22221828Sgrehan * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23221828Sgrehan * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24221828Sgrehan * SUCH DAMAGE. 25221828Sgrehan * 26221828Sgrehan * $FreeBSD$ 27221828Sgrehan */ 28221828Sgrehan 29221828Sgrehan#include <sys/cdefs.h> 30221828Sgrehan__FBSDID("$FreeBSD$"); 31221828Sgrehan 32256072Sneel#include <sys/param.h> 33256072Sneel#include <sys/kernel.h> 34221828Sgrehan#include <sys/types.h> 35221828Sgrehan#include <sys/systm.h> 36221828Sgrehan#include <sys/smp.h> 37256072Sneel#include <sys/sysctl.h> 38221828Sgrehan 39221828Sgrehan#include <vm/vm.h> 40221828Sgrehan#include <vm/pmap.h> 41256072Sneel#include <vm/vm_extern.h> 42221828Sgrehan 43256072Sneel#include <machine/vmm.h> 44221828Sgrehan 45221828Sgrehan#include "vmx_cpufunc.h" 46221828Sgrehan#include "ept.h" 47221828Sgrehan 48256072Sneel#define EPT_SUPPORTS_EXEC_ONLY(cap) ((cap) & (1UL << 0)) 49221828Sgrehan#define EPT_PWL4(cap) ((cap) & (1UL << 6)) 50221828Sgrehan#define EPT_MEMORY_TYPE_WB(cap) ((cap) & (1UL << 14)) 51221828Sgrehan#define EPT_PDE_SUPERPAGE(cap) ((cap) & (1UL << 16)) /* 2MB pages */ 52221828Sgrehan#define EPT_PDPTE_SUPERPAGE(cap) ((cap) & (1UL << 17)) /* 1GB pages */ 53256072Sneel#define INVEPT_SUPPORTED(cap) ((cap) & (1UL << 20)) 54256072Sneel#define AD_BITS_SUPPORTED(cap) ((cap) & (1UL << 21)) 55221828Sgrehan#define INVVPID_SUPPORTED(cap) ((cap) & (1UL << 32)) 56221828Sgrehan 57221828Sgrehan#define INVVPID_ALL_TYPES_MASK 0xF0000000000UL 58221828Sgrehan#define INVVPID_ALL_TYPES_SUPPORTED(cap) \ 59221828Sgrehan (((cap) & INVVPID_ALL_TYPES_MASK) == INVVPID_ALL_TYPES_MASK) 60221828Sgrehan 61221828Sgrehan#define INVEPT_ALL_TYPES_MASK 0x6000000UL 62221828Sgrehan#define INVEPT_ALL_TYPES_SUPPORTED(cap) \ 63221828Sgrehan (((cap) & INVEPT_ALL_TYPES_MASK) == INVEPT_ALL_TYPES_MASK) 64221828Sgrehan 65256072Sneel#define EPT_PWLEVELS 4 /* page walk levels */ 66256072Sneel#define EPT_ENABLE_AD_BITS (1 << 6) 67221828Sgrehan 68256072SneelSYSCTL_DECL(_hw_vmm); 69256072SneelSYSCTL_NODE(_hw_vmm, OID_AUTO, ept, CTLFLAG_RW, NULL, NULL); 70221828Sgrehan 71256072Sneelstatic int ept_enable_ad_bits; 72221828Sgrehan 73256072Sneelstatic int ept_pmap_flags; 74256072SneelSYSCTL_INT(_hw_vmm_ept, OID_AUTO, pmap_flags, CTLFLAG_RD, 75256072Sneel &ept_pmap_flags, 0, NULL); 76221828Sgrehan 77221828Sgrehanint 78260466Sneelept_init(int ipinum) 79221828Sgrehan{ 80256072Sneel int use_hw_ad_bits, use_superpages, use_exec_only; 81221828Sgrehan uint64_t cap; 82221828Sgrehan 83221828Sgrehan cap = rdmsr(MSR_VMX_EPT_VPID_CAP); 84221828Sgrehan 85221828Sgrehan /* 86221828Sgrehan * Verify that: 87221828Sgrehan * - page walk length is 4 steps 88221828Sgrehan * - extended page tables can be laid out in write-back memory 89221828Sgrehan * - invvpid instruction with all possible types is supported 90221828Sgrehan * - invept instruction with all possible types is supported 91221828Sgrehan */ 92221828Sgrehan if (!EPT_PWL4(cap) || 93221828Sgrehan !EPT_MEMORY_TYPE_WB(cap) || 94221828Sgrehan !INVVPID_SUPPORTED(cap) || 95221828Sgrehan !INVVPID_ALL_TYPES_SUPPORTED(cap) || 96221828Sgrehan !INVEPT_SUPPORTED(cap) || 97221828Sgrehan !INVEPT_ALL_TYPES_SUPPORTED(cap)) 98221828Sgrehan return (EINVAL); 99221828Sgrehan 100260466Sneel ept_pmap_flags = ipinum & PMAP_NESTED_IPIMASK; 101259641Sneel 102256072Sneel use_superpages = 1; 103256072Sneel TUNABLE_INT_FETCH("hw.vmm.ept.use_superpages", &use_superpages); 104256072Sneel if (use_superpages && EPT_PDE_SUPERPAGE(cap)) 105256072Sneel ept_pmap_flags |= PMAP_PDE_SUPERPAGE; /* 2MB superpage */ 106221828Sgrehan 107256072Sneel use_hw_ad_bits = 1; 108256072Sneel TUNABLE_INT_FETCH("hw.vmm.ept.use_hw_ad_bits", &use_hw_ad_bits); 109256072Sneel if (use_hw_ad_bits && AD_BITS_SUPPORTED(cap)) 110256072Sneel ept_enable_ad_bits = 1; 111256072Sneel else 112256072Sneel ept_pmap_flags |= PMAP_EMULATE_AD_BITS; 113221828Sgrehan 114256072Sneel use_exec_only = 1; 115256072Sneel TUNABLE_INT_FETCH("hw.vmm.ept.use_exec_only", &use_exec_only); 116256072Sneel if (use_exec_only && EPT_SUPPORTS_EXEC_ONLY(cap)) 117256072Sneel ept_pmap_flags |= PMAP_SUPPORTS_EXEC_ONLY; 118221828Sgrehan 119221828Sgrehan return (0); 120221828Sgrehan} 121221828Sgrehan 122241147Sneel#if 0 123241147Sneelstatic void 124241147Sneelept_dump(uint64_t *ptp, int nlevels) 125241147Sneel{ 126241147Sneel int i, t, tabs; 127241147Sneel uint64_t *ptpnext, ptpval; 128241147Sneel 129241147Sneel if (--nlevels < 0) 130241147Sneel return; 131241147Sneel 132241147Sneel tabs = 3 - nlevels; 133241147Sneel for (t = 0; t < tabs; t++) 134241147Sneel printf("\t"); 135241147Sneel printf("PTP = %p\n", ptp); 136241147Sneel 137241147Sneel for (i = 0; i < 512; i++) { 138241147Sneel ptpval = ptp[i]; 139241147Sneel 140241147Sneel if (ptpval == 0) 141241147Sneel continue; 142241147Sneel 143241147Sneel for (t = 0; t < tabs; t++) 144241147Sneel printf("\t"); 145241147Sneel printf("%3d 0x%016lx\n", i, ptpval); 146241147Sneel 147241147Sneel if (nlevels != 0 && (ptpval & EPT_PG_SUPERPAGE) == 0) { 148241147Sneel ptpnext = (uint64_t *) 149241147Sneel PHYS_TO_DMAP(ptpval & EPT_ADDR_MASK); 150241147Sneel ept_dump(ptpnext, nlevels); 151241147Sneel } 152241147Sneel } 153241147Sneel} 154241147Sneel#endif 155241147Sneel 156221828Sgrehanstatic void 157256072Sneelinvept_single_context(void *arg) 158221828Sgrehan{ 159256072Sneel struct invept_desc desc = *(struct invept_desc *)arg; 160221828Sgrehan 161256072Sneel invept(INVEPT_TYPE_SINGLE_CONTEXT, desc); 162221828Sgrehan} 163221828Sgrehan 164256072Sneelvoid 165256072Sneelept_invalidate_mappings(u_long eptp) 166221828Sgrehan{ 167256072Sneel struct invept_desc invept_desc = { 0 }; 168221828Sgrehan 169256072Sneel invept_desc.eptp = eptp; 170221828Sgrehan 171256072Sneel smp_rendezvous(NULL, invept_single_context, NULL, &invept_desc); 172221828Sgrehan} 173221828Sgrehan 174256072Sneelstatic int 175256072Sneelept_pinit(pmap_t pmap) 176221828Sgrehan{ 177221828Sgrehan 178256072Sneel return (pmap_pinit_type(pmap, PT_EPT, ept_pmap_flags)); 179221828Sgrehan} 180221828Sgrehan 181256072Sneelstruct vmspace * 182256072Sneelept_vmspace_alloc(vm_offset_t min, vm_offset_t max) 183221828Sgrehan{ 184221828Sgrehan 185256072Sneel return (vmspace_alloc(min, max, ept_pinit)); 186221828Sgrehan} 187221828Sgrehan 188221828Sgrehanvoid 189256072Sneelept_vmspace_free(struct vmspace *vmspace) 190221828Sgrehan{ 191221828Sgrehan 192256072Sneel vmspace_free(vmspace); 193221828Sgrehan} 194221828Sgrehan 195256072Sneeluint64_t 196256072Sneeleptp(uint64_t pml4) 197221828Sgrehan{ 198256072Sneel uint64_t eptp_val; 199221828Sgrehan 200256072Sneel eptp_val = pml4 | (EPT_PWLEVELS - 1) << 3 | PAT_WRITE_BACK; 201256072Sneel if (ept_enable_ad_bits) 202256072Sneel eptp_val |= EPT_ENABLE_AD_BITS; 203221828Sgrehan 204256072Sneel return (eptp_val); 205221828Sgrehan} 206