| pmap.c (351449) | pmap.c (354651) |
|---|---|
| 1/*- 2 * Copyright (c) 1991 Regents of the University of California. 3 * All rights reserved. 4 * Copyright (c) 1994 John S. Dyson 5 * All rights reserved. 6 * Copyright (c) 1994 David Greenman 7 * All rights reserved. 8 * Copyright (c) 2003 Peter Wemm --- 71 unchanged lines hidden (view full) --- 80 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 81 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 82 * SUCH DAMAGE. 83 */ 84 85#define AMD64_NPT_AWARE 86 87#include <sys/cdefs.h> | 1/*- 2 * Copyright (c) 1991 Regents of the University of California. 3 * All rights reserved. 4 * Copyright (c) 1994 John S. Dyson 5 * All rights reserved. 6 * Copyright (c) 1994 David Greenman 7 * All rights reserved. 8 * Copyright (c) 2003 Peter Wemm --- 71 unchanged lines hidden (view full) --- 80 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 81 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 82 * SUCH DAMAGE. 83 */ 84 85#define AMD64_NPT_AWARE 86 87#include <sys/cdefs.h> |
| 88__FBSDID("$FreeBSD: stable/11/sys/amd64/amd64/pmap.c 351449 2019-08-24 00:35:59Z jhb $"); | 88__FBSDID("$FreeBSD: stable/11/sys/amd64/amd64/pmap.c 354651 2019-11-12 18:04:28Z kib $"); |
| 89 90/* 91 * Manages physical address maps. 92 * 93 * Since the information managed by this module is 94 * also stored by the logical address mapping module, 95 * this module may throw away valid virtual-to-physical 96 * mappings at almost any time. However, invalidations --- 1124 unchanged lines hidden (view full) --- 1221void 1222pmap_page_init(vm_page_t m) 1223{ 1224 1225 TAILQ_INIT(&m->md.pv_list); 1226 m->md.pat_mode = PAT_WRITE_BACK; 1227} 1228 | 89 90/* 91 * Manages physical address maps. 92 * 93 * Since the information managed by this module is 94 * also stored by the logical address mapping module, 95 * this module may throw away valid virtual-to-physical 96 * mappings at almost any time. However, invalidations --- 1124 unchanged lines hidden (view full) --- 1221void 1222pmap_page_init(vm_page_t m) 1223{ 1224 1225 TAILQ_INIT(&m->md.pv_list); 1226 m->md.pat_mode = PAT_WRITE_BACK; 1227} 1228 |
| 1229static int pmap_allow_2m_x_ept; 1230SYSCTL_INT(_vm_pmap, OID_AUTO, allow_2m_x_ept, CTLFLAG_RWTUN | CTLFLAG_NOFETCH, 1231 &pmap_allow_2m_x_ept, 0, 1232 "Allow executable superpage mappings in EPT"); 1233 1234void 1235pmap_allow_2m_x_ept_recalculate(void) 1236{ 1237 /* 1238 * SKL002, SKL012S. Since the EPT format is only used by 1239 * Intel CPUs, the vendor check is merely a formality. 1240 */ 1241 if (!(cpu_vendor_id != CPU_VENDOR_INTEL || 1242 (cpu_ia32_arch_caps & IA32_ARCH_CAP_IF_PSCHANGE_MC_NO) != 0 || 1243 (CPUID_TO_FAMILY(cpu_id) == 0x6 && 1244 (CPUID_TO_MODEL(cpu_id) == 0x26 || /* Atoms */ 1245 CPUID_TO_MODEL(cpu_id) == 0x27 || 1246 CPUID_TO_MODEL(cpu_id) == 0x35 || 1247 CPUID_TO_MODEL(cpu_id) == 0x36 || 1248 CPUID_TO_MODEL(cpu_id) == 0x37 || 1249 CPUID_TO_MODEL(cpu_id) == 0x86 || 1250 CPUID_TO_MODEL(cpu_id) == 0x1c || 1251 CPUID_TO_MODEL(cpu_id) == 0x4a || 1252 CPUID_TO_MODEL(cpu_id) == 0x4c || 1253 CPUID_TO_MODEL(cpu_id) == 0x4d || 1254 CPUID_TO_MODEL(cpu_id) == 0x5a || 1255 CPUID_TO_MODEL(cpu_id) == 0x5c || 1256 CPUID_TO_MODEL(cpu_id) == 0x5d || 1257 CPUID_TO_MODEL(cpu_id) == 0x5f || 1258 CPUID_TO_MODEL(cpu_id) == 0x6e || 1259 CPUID_TO_MODEL(cpu_id) == 0x7a || 1260 CPUID_TO_MODEL(cpu_id) == 0x57 || /* Knights */ 1261 CPUID_TO_MODEL(cpu_id) == 0x85)))) 1262 pmap_allow_2m_x_ept = 1; 1263 TUNABLE_INT_FETCH("hw.allow_2m_x_ept", &pmap_allow_2m_x_ept); 1264} 1265 1266static bool 1267pmap_allow_2m_x_page(pmap_t pmap, bool executable) 1268{ 1269 1270 return (pmap->pm_type != PT_EPT || !executable || 1271 !pmap_allow_2m_x_ept); 1272} 1273 |
|
| 1229/* 1230 * Initialize the pmap module. 1231 * Called by vm_init, to initialize any structures that the pmap 1232 * system needs to map virtual memory. 1233 */ 1234void 1235pmap_init(void) 1236{ --- 28 unchanged lines hidden (view full) --- 1265 ptoa(i), FALSE); 1266 if (!ret && bootverbose) 1267 printf("page at %#lx already used\n", 1268 0x40000000 + ptoa(i)); 1269 } 1270 } 1271 } 1272 | 1274/* 1275 * Initialize the pmap module. 1276 * Called by vm_init, to initialize any structures that the pmap 1277 * system needs to map virtual memory. 1278 */ 1279void 1280pmap_init(void) 1281{ --- 28 unchanged lines hidden (view full) --- 1310 ptoa(i), FALSE); 1311 if (!ret && bootverbose) 1312 printf("page at %#lx already used\n", 1313 0x40000000 + ptoa(i)); 1314 } 1315 } 1316 } 1317 |
| 1318 /* IFU */ 1319 pmap_allow_2m_x_ept_recalculate(); 1320 |
|
| 1273 /* 1274 * Initialize the vm page array entries for the kernel pmap's 1275 * page table pages. 1276 */ 1277 PMAP_LOCK(kernel_pmap); 1278 for (i = 0; i < nkpt; i++) { 1279 mpte = PHYS_TO_VM_PAGE(KPTphys + (i << PAGE_SHIFT)); 1280 KASSERT(mpte >= vm_page_array && --- 3264 unchanged lines hidden (view full) --- 4545 } 4546 } 4547 if (anychanged) 4548 pmap_invalidate_all(pmap); 4549 PMAP_UNLOCK(pmap); 4550} 4551 4552#if VM_NRESERVLEVEL > 0 | 1321 /* 1322 * Initialize the vm page array entries for the kernel pmap's 1323 * page table pages. 1324 */ 1325 PMAP_LOCK(kernel_pmap); 1326 for (i = 0; i < nkpt; i++) { 1327 mpte = PHYS_TO_VM_PAGE(KPTphys + (i << PAGE_SHIFT)); 1328 KASSERT(mpte >= vm_page_array && --- 3264 unchanged lines hidden (view full) --- 4593 } 4594 } 4595 if (anychanged) 4596 pmap_invalidate_all(pmap); 4597 PMAP_UNLOCK(pmap); 4598} 4599 4600#if VM_NRESERVLEVEL > 0 |
| 4601static bool 4602pmap_pde_ept_executable(pmap_t pmap, pd_entry_t pde) 4603{ 4604 4605 if (pmap->pm_type != PT_EPT) 4606 return (false); 4607 return ((pde & EPT_PG_EXECUTE) != 0); 4608} 4609 |
|
| 4553/* 4554 * Tries to promote the 512, contiguous 4KB page mappings that are within a 4555 * single page table page (PTP) to a single 2MB page mapping. For promotion 4556 * to occur, two conditions must be met: (1) the 4KB page mappings must map 4557 * aligned, contiguous physical memory and (2) the 4KB page mappings must have 4558 * identical characteristics. 4559 */ 4560static void --- 18 unchanged lines hidden (view full) --- 4579 /* 4580 * Examine the first PTE in the specified PTP. Abort if this PTE is 4581 * either invalid, unused, or does not map the first 4KB physical page 4582 * within a 2MB page. 4583 */ 4584 firstpte = (pt_entry_t *)PHYS_TO_DMAP(*pde & PG_FRAME); 4585setpde: 4586 newpde = *firstpte; | 4610/* 4611 * Tries to promote the 512, contiguous 4KB page mappings that are within a 4612 * single page table page (PTP) to a single 2MB page mapping. For promotion 4613 * to occur, two conditions must be met: (1) the 4KB page mappings must map 4614 * aligned, contiguous physical memory and (2) the 4KB page mappings must have 4615 * identical characteristics. 4616 */ 4617static void --- 18 unchanged lines hidden (view full) --- 4636 /* 4637 * Examine the first PTE in the specified PTP. Abort if this PTE is 4638 * either invalid, unused, or does not map the first 4KB physical page 4639 * within a 2MB page. 4640 */ 4641 firstpte = (pt_entry_t *)PHYS_TO_DMAP(*pde & PG_FRAME); 4642setpde: 4643 newpde = *firstpte; |
| 4587 if ((newpde & ((PG_FRAME & PDRMASK) | PG_A | PG_V)) != (PG_A | PG_V)) { | 4644 if ((newpde & ((PG_FRAME & PDRMASK) | PG_A | PG_V)) != (PG_A | PG_V) || 4645 !pmap_allow_2m_x_page(pmap, pmap_pde_ept_executable(pmap, 4646 newpde))) { |
| 4588 atomic_add_long(&pmap_pde_p_failures, 1); 4589 CTR2(KTR_PMAP, "pmap_promote_pde: failure for va %#lx" 4590 " in pmap %p", va, pmap); 4591 return; 4592 } 4593 if ((newpde & (PG_M | PG_RW)) == PG_RW) { 4594 /* 4595 * When PG_M is already clear, PG_RW can be cleared without --- 409 unchanged lines hidden (view full) --- 5005 5006 PG_G = pmap_global_bit(pmap); 5007 PG_RW = pmap_rw_bit(pmap); 5008 KASSERT((newpde & (pmap_modified_bit(pmap) | PG_RW)) != PG_RW, 5009 ("pmap_enter_pde: newpde is missing PG_M")); 5010 PG_V = pmap_valid_bit(pmap); 5011 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 5012 | 4647 atomic_add_long(&pmap_pde_p_failures, 1); 4648 CTR2(KTR_PMAP, "pmap_promote_pde: failure for va %#lx" 4649 " in pmap %p", va, pmap); 4650 return; 4651 } 4652 if ((newpde & (PG_M | PG_RW)) == PG_RW) { 4653 /* 4654 * When PG_M is already clear, PG_RW can be cleared without --- 409 unchanged lines hidden (view full) --- 5064 5065 PG_G = pmap_global_bit(pmap); 5066 PG_RW = pmap_rw_bit(pmap); 5067 KASSERT((newpde & (pmap_modified_bit(pmap) | PG_RW)) != PG_RW, 5068 ("pmap_enter_pde: newpde is missing PG_M")); 5069 PG_V = pmap_valid_bit(pmap); 5070 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 5071 |
| 5072 if (!pmap_allow_2m_x_page(pmap, pmap_pde_ept_executable(pmap, 5073 newpde))) { 5074 CTR2(KTR_PMAP, "pmap_enter_pde: 2m x blocked for va %#lx" 5075 " in pmap %p", va, pmap); 5076 return (KERN_FAILURE); 5077 } |
|
| 5013 if ((pdpg = pmap_allocpde(pmap, va, (flags & PMAP_ENTER_NOSLEEP) != 0 ? 5014 NULL : lockp)) == NULL) { 5015 CTR2(KTR_PMAP, "pmap_enter_pde: failure for va %#lx" 5016 " in pmap %p", va, pmap); 5017 return (KERN_RESOURCE_SHORTAGE); 5018 } 5019 pde = (pd_entry_t *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(pdpg)); 5020 pde = &pde[pmap_pde_index(va)]; --- 113 unchanged lines hidden (view full) --- 5134 mpte = NULL; 5135 m = m_start; 5136 lock = NULL; 5137 PMAP_LOCK(pmap); 5138 while (m != NULL && (diff = m->pindex - m_start->pindex) < psize) { 5139 va = start + ptoa(diff); 5140 if ((va & PDRMASK) == 0 && va + NBPDR <= end && 5141 m->psind == 1 && pmap_ps_enabled(pmap) && | 5078 if ((pdpg = pmap_allocpde(pmap, va, (flags & PMAP_ENTER_NOSLEEP) != 0 ? 5079 NULL : lockp)) == NULL) { 5080 CTR2(KTR_PMAP, "pmap_enter_pde: failure for va %#lx" 5081 " in pmap %p", va, pmap); 5082 return (KERN_RESOURCE_SHORTAGE); 5083 } 5084 pde = (pd_entry_t *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(pdpg)); 5085 pde = &pde[pmap_pde_index(va)]; --- 113 unchanged lines hidden (view full) --- 5199 mpte = NULL; 5200 m = m_start; 5201 lock = NULL; 5202 PMAP_LOCK(pmap); 5203 while (m != NULL && (diff = m->pindex - m_start->pindex) < psize) { 5204 va = start + ptoa(diff); 5205 if ((va & PDRMASK) == 0 && va + NBPDR <= end && 5206 m->psind == 1 && pmap_ps_enabled(pmap) && |
| 5207 pmap_allow_2m_x_page(pmap, (prot & VM_PROT_EXECUTE) != 0) && |
|
| 5142 pmap_enter_2mpage(pmap, va, m, prot, &lock)) 5143 m = &m[NBPDR / PAGE_SIZE - 1]; 5144 else 5145 mpte = pmap_enter_quick_locked(pmap, va, m, prot, 5146 mpte, &lock); 5147 m = TAILQ_NEXT(m, listq); 5148 } 5149 if (lock != NULL) --- 3072 unchanged lines hidden --- | 5208 pmap_enter_2mpage(pmap, va, m, prot, &lock)) 5209 m = &m[NBPDR / PAGE_SIZE - 1]; 5210 else 5211 mpte = pmap_enter_quick_locked(pmap, va, m, prot, 5212 mpte, &lock); 5213 m = TAILQ_NEXT(m, listq); 5214 } 5215 if (lock != NULL) --- 3072 unchanged lines hidden --- |