pmap.c revision 295425
1281494Sandrew/*- 2281494Sandrew * Copyright (c) 1991 Regents of the University of California. 3281494Sandrew * All rights reserved. 4281494Sandrew * Copyright (c) 1994 John S. Dyson 5281494Sandrew * All rights reserved. 6281494Sandrew * Copyright (c) 1994 David Greenman 7281494Sandrew * All rights reserved. 8281494Sandrew * Copyright (c) 2003 Peter Wemm 9281494Sandrew * All rights reserved. 10281494Sandrew * Copyright (c) 2005-2010 Alan L. Cox <alc@cs.rice.edu> 11281494Sandrew * All rights reserved. 12281494Sandrew * Copyright (c) 2014 Andrew Turner 13281494Sandrew * All rights reserved. 14281494Sandrew * Copyright (c) 2014 The FreeBSD Foundation 15281494Sandrew * All rights reserved. 16281494Sandrew * 17281494Sandrew * This code is derived from software contributed to Berkeley by 18281494Sandrew * the Systems Programming Group of the University of Utah Computer 19281494Sandrew * Science Department and William Jolitz of UUNET Technologies Inc. 20281494Sandrew * 21281494Sandrew * This software was developed by Andrew Turner under sponsorship from 22281494Sandrew * the FreeBSD Foundation. 23281494Sandrew * 24281494Sandrew * Redistribution and use in source and binary forms, with or without 25281494Sandrew * modification, are permitted provided that the following conditions 26281494Sandrew * are met: 27281494Sandrew * 1. Redistributions of source code must retain the above copyright 28281494Sandrew * notice, this list of conditions and the following disclaimer. 29281494Sandrew * 2. Redistributions in binary form must reproduce the above copyright 30281494Sandrew * notice, this list of conditions and the following disclaimer in the 31281494Sandrew * documentation and/or other materials provided with the distribution. 32281494Sandrew * 3. All advertising materials mentioning features or use of this software 33281494Sandrew * must display the following acknowledgement: 34281494Sandrew * This product includes software developed by the University of 35281494Sandrew * California, Berkeley and its contributors. 36281494Sandrew * 4. Neither the name of the University nor the names of its contributors 37281494Sandrew * may be used to endorse or promote products derived from this software 38281494Sandrew * without specific prior written permission. 39281494Sandrew * 40281494Sandrew * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 41281494Sandrew * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 42281494Sandrew * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 43281494Sandrew * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 44281494Sandrew * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 45281494Sandrew * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 46281494Sandrew * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 47281494Sandrew * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 48281494Sandrew * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 49281494Sandrew * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 50281494Sandrew * SUCH DAMAGE. 51281494Sandrew * 52281494Sandrew * from: @(#)pmap.c 7.7 (Berkeley) 5/12/91 53281494Sandrew */ 54281494Sandrew/*- 55281494Sandrew * Copyright (c) 2003 Networks Associates Technology, Inc. 56281494Sandrew * All rights reserved. 57281494Sandrew * 58281494Sandrew * This software was developed for the FreeBSD Project by Jake Burkholder, 59281494Sandrew * Safeport Network Services, and Network Associates Laboratories, the 60281494Sandrew * Security Research Division of Network Associates, Inc. under 61281494Sandrew * DARPA/SPAWAR contract N66001-01-C-8035 ("CBOSS"), as part of the DARPA 62281494Sandrew * CHATS research program. 63281494Sandrew * 64281494Sandrew * Redistribution and use in source and binary forms, with or without 65281494Sandrew * modification, are permitted provided that the following conditions 66281494Sandrew * are met: 67281494Sandrew * 1. Redistributions of source code must retain the above copyright 68281494Sandrew * notice, this list of conditions and the following disclaimer. 69281494Sandrew * 2. Redistributions in binary form must reproduce the above copyright 70281494Sandrew * notice, this list of conditions and the following disclaimer in the 71281494Sandrew * documentation and/or other materials provided with the distribution. 72281494Sandrew * 73281494Sandrew * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 74281494Sandrew * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 75281494Sandrew * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 76281494Sandrew * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 77281494Sandrew * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 78281494Sandrew * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 79281494Sandrew * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 80281494Sandrew * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 81281494Sandrew * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 82281494Sandrew * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 83281494Sandrew * SUCH DAMAGE. 84281494Sandrew */ 85281494Sandrew 86281494Sandrew#include <sys/cdefs.h> 87281494Sandrew__FBSDID("$FreeBSD: head/sys/arm64/arm64/pmap.c 295425 2016-02-09 06:26:27Z wma $"); 88281494Sandrew 89281494Sandrew/* 90281494Sandrew * Manages physical address maps. 91281494Sandrew * 92281494Sandrew * Since the information managed by this module is 93281494Sandrew * also stored by the logical address mapping module, 94281494Sandrew * this module may throw away valid virtual-to-physical 95281494Sandrew * mappings at almost any time. However, invalidations 96281494Sandrew * of virtual-to-physical mappings must be done as 97281494Sandrew * requested. 98281494Sandrew * 99281494Sandrew * In order to cope with hardware architectures which 100281494Sandrew * make virtual-to-physical map invalidates expensive, 101281494Sandrew * this module may delay invalidate or reduced protection 102281494Sandrew * operations until such time as they are actually 103281494Sandrew * necessary. This module is given full information as 104281494Sandrew * to which processors are currently using which maps, 105281494Sandrew * and to when physical maps must be made correct. 106281494Sandrew */ 107281494Sandrew 108281494Sandrew#include <sys/param.h> 109281494Sandrew#include <sys/bus.h> 110281494Sandrew#include <sys/systm.h> 111281494Sandrew#include <sys/kernel.h> 112281494Sandrew#include <sys/ktr.h> 113281494Sandrew#include <sys/lock.h> 114281494Sandrew#include <sys/malloc.h> 115281494Sandrew#include <sys/mman.h> 116281494Sandrew#include <sys/msgbuf.h> 117281494Sandrew#include <sys/mutex.h> 118281494Sandrew#include <sys/proc.h> 119281494Sandrew#include <sys/rwlock.h> 120281494Sandrew#include <sys/sx.h> 121281494Sandrew#include <sys/vmem.h> 122281494Sandrew#include <sys/vmmeter.h> 123281494Sandrew#include <sys/sched.h> 124281494Sandrew#include <sys/sysctl.h> 125281494Sandrew#include <sys/_unrhdr.h> 126281494Sandrew#include <sys/smp.h> 127281494Sandrew 128281494Sandrew#include <vm/vm.h> 129281494Sandrew#include <vm/vm_param.h> 130281494Sandrew#include <vm/vm_kern.h> 131281494Sandrew#include <vm/vm_page.h> 132281494Sandrew#include <vm/vm_map.h> 133281494Sandrew#include <vm/vm_object.h> 134281494Sandrew#include <vm/vm_extern.h> 135281494Sandrew#include <vm/vm_pageout.h> 136281494Sandrew#include <vm/vm_pager.h> 137281494Sandrew#include <vm/vm_radix.h> 138281494Sandrew#include <vm/vm_reserv.h> 139281494Sandrew#include <vm/uma.h> 140281494Sandrew 141281494Sandrew#include <machine/machdep.h> 142281494Sandrew#include <machine/md_var.h> 143281494Sandrew#include <machine/pcb.h> 144281494Sandrew 145281494Sandrew#define NPDEPG (PAGE_SIZE/(sizeof (pd_entry_t))) 146281494Sandrew#define NUPDE (NPDEPG * NPDEPG) 147281494Sandrew#define NUSERPGTBLS (NUPDE + NPDEPG) 148281494Sandrew 149281494Sandrew#if !defined(DIAGNOSTIC) 150281494Sandrew#ifdef __GNUC_GNU_INLINE__ 151281494Sandrew#define PMAP_INLINE __attribute__((__gnu_inline__)) inline 152281494Sandrew#else 153281494Sandrew#define PMAP_INLINE extern inline 154281494Sandrew#endif 155281494Sandrew#else 156281494Sandrew#define PMAP_INLINE 157281494Sandrew#endif 158281494Sandrew 159281494Sandrew/* 160281494Sandrew * These are configured by the mair_el1 register. This is set up in locore.S 161281494Sandrew */ 162281494Sandrew#define DEVICE_MEMORY 0 163281494Sandrew#define UNCACHED_MEMORY 1 164281494Sandrew#define CACHED_MEMORY 2 165281494Sandrew 166281494Sandrew 167281494Sandrew#ifdef PV_STATS 168281494Sandrew#define PV_STAT(x) do { x ; } while (0) 169281494Sandrew#else 170281494Sandrew#define PV_STAT(x) do { } while (0) 171281494Sandrew#endif 172281494Sandrew 173281494Sandrew#define pmap_l2_pindex(v) ((v) >> L2_SHIFT) 174281494Sandrew 175281494Sandrew#define NPV_LIST_LOCKS MAXCPU 176281494Sandrew 177281494Sandrew#define PHYS_TO_PV_LIST_LOCK(pa) \ 178281494Sandrew (&pv_list_locks[pa_index(pa) % NPV_LIST_LOCKS]) 179281494Sandrew 180281494Sandrew#define CHANGE_PV_LIST_LOCK_TO_PHYS(lockp, pa) do { \ 181281494Sandrew struct rwlock **_lockp = (lockp); \ 182281494Sandrew struct rwlock *_new_lock; \ 183281494Sandrew \ 184281494Sandrew _new_lock = PHYS_TO_PV_LIST_LOCK(pa); \ 185281494Sandrew if (_new_lock != *_lockp) { \ 186281494Sandrew if (*_lockp != NULL) \ 187281494Sandrew rw_wunlock(*_lockp); \ 188281494Sandrew *_lockp = _new_lock; \ 189281494Sandrew rw_wlock(*_lockp); \ 190281494Sandrew } \ 191281494Sandrew} while (0) 192281494Sandrew 193281494Sandrew#define CHANGE_PV_LIST_LOCK_TO_VM_PAGE(lockp, m) \ 194281494Sandrew CHANGE_PV_LIST_LOCK_TO_PHYS(lockp, VM_PAGE_TO_PHYS(m)) 195281494Sandrew 196281494Sandrew#define RELEASE_PV_LIST_LOCK(lockp) do { \ 197281494Sandrew struct rwlock **_lockp = (lockp); \ 198281494Sandrew \ 199281494Sandrew if (*_lockp != NULL) { \ 200281494Sandrew rw_wunlock(*_lockp); \ 201281494Sandrew *_lockp = NULL; \ 202281494Sandrew } \ 203281494Sandrew} while (0) 204281494Sandrew 205281494Sandrew#define VM_PAGE_TO_PV_LIST_LOCK(m) \ 206281494Sandrew PHYS_TO_PV_LIST_LOCK(VM_PAGE_TO_PHYS(m)) 207281494Sandrew 208281494Sandrewstruct pmap kernel_pmap_store; 209281494Sandrew 210281494Sandrewvm_offset_t virtual_avail; /* VA of first avail page (after kernel bss) */ 211281494Sandrewvm_offset_t virtual_end; /* VA of last avail page (end of kernel AS) */ 212281494Sandrewvm_offset_t kernel_vm_end = 0; 213281494Sandrew 214281494Sandrewstruct msgbuf *msgbufp = NULL; 215281494Sandrew 216281494Sandrewstatic struct rwlock_padalign pvh_global_lock; 217281494Sandrew 218291246Sandrewvm_paddr_t dmap_phys_base; /* The start of the dmap region */ 219291246Sandrew 220281494Sandrew/* 221281494Sandrew * Data for the pv entry allocation mechanism 222281494Sandrew */ 223281494Sandrewstatic TAILQ_HEAD(pch, pv_chunk) pv_chunks = TAILQ_HEAD_INITIALIZER(pv_chunks); 224281494Sandrewstatic struct mtx pv_chunks_mutex; 225281494Sandrewstatic struct rwlock pv_list_locks[NPV_LIST_LOCKS]; 226281494Sandrew 227281494Sandrewstatic void free_pv_chunk(struct pv_chunk *pc); 228281494Sandrewstatic void free_pv_entry(pmap_t pmap, pv_entry_t pv); 229281494Sandrewstatic pv_entry_t get_pv_entry(pmap_t pmap, struct rwlock **lockp); 230281494Sandrewstatic vm_page_t reclaim_pv_chunk(pmap_t locked_pmap, struct rwlock **lockp); 231281494Sandrewstatic void pmap_pvh_free(struct md_page *pvh, pmap_t pmap, vm_offset_t va); 232281494Sandrewstatic pv_entry_t pmap_pvh_remove(struct md_page *pvh, pmap_t pmap, 233281494Sandrew vm_offset_t va); 234281494Sandrewstatic vm_page_t pmap_enter_quick_locked(pmap_t pmap, vm_offset_t va, 235281494Sandrew vm_page_t m, vm_prot_t prot, vm_page_t mpte, struct rwlock **lockp); 236281494Sandrewstatic int pmap_remove_l3(pmap_t pmap, pt_entry_t *l3, vm_offset_t sva, 237281494Sandrew pd_entry_t ptepde, struct spglist *free, struct rwlock **lockp); 238281494Sandrewstatic boolean_t pmap_try_insert_pv_entry(pmap_t pmap, vm_offset_t va, 239281494Sandrew vm_page_t m, struct rwlock **lockp); 240281494Sandrew 241281494Sandrewstatic vm_page_t _pmap_alloc_l3(pmap_t pmap, vm_pindex_t ptepindex, 242281494Sandrew struct rwlock **lockp); 243281494Sandrew 244281494Sandrewstatic void _pmap_unwire_l3(pmap_t pmap, vm_offset_t va, vm_page_t m, 245281494Sandrew struct spglist *free); 246281494Sandrewstatic int pmap_unuse_l3(pmap_t, vm_offset_t, pd_entry_t, struct spglist *); 247281494Sandrew 248288445Sandrew/* 249288445Sandrew * These load the old table data and store the new value. 250288445Sandrew * They need to be atomic as the System MMU may write to the table at 251288445Sandrew * the same time as the CPU. 252288445Sandrew */ 253288445Sandrew#define pmap_load_store(table, entry) atomic_swap_64(table, entry) 254288445Sandrew#define pmap_set(table, mask) atomic_set_64(table, mask) 255288445Sandrew#define pmap_load_clear(table) atomic_swap_64(table, 0) 256288445Sandrew#define pmap_load(table) (*table) 257288445Sandrew 258281494Sandrew/********************/ 259281494Sandrew/* Inline functions */ 260281494Sandrew/********************/ 261281494Sandrew 262281494Sandrewstatic __inline void 263281494Sandrewpagecopy(void *s, void *d) 264281494Sandrew{ 265281494Sandrew 266281494Sandrew memcpy(d, s, PAGE_SIZE); 267281494Sandrew} 268281494Sandrew 269281494Sandrewstatic __inline void 270281494Sandrewpagezero(void *p) 271281494Sandrew{ 272281494Sandrew 273281494Sandrew bzero(p, PAGE_SIZE); 274281494Sandrew} 275281494Sandrew 276281494Sandrew#define pmap_l1_index(va) (((va) >> L1_SHIFT) & Ln_ADDR_MASK) 277281494Sandrew#define pmap_l2_index(va) (((va) >> L2_SHIFT) & Ln_ADDR_MASK) 278281494Sandrew#define pmap_l3_index(va) (((va) >> L3_SHIFT) & Ln_ADDR_MASK) 279281494Sandrew 280281494Sandrewstatic __inline pd_entry_t * 281281494Sandrewpmap_l1(pmap_t pmap, vm_offset_t va) 282281494Sandrew{ 283281494Sandrew 284281494Sandrew return (&pmap->pm_l1[pmap_l1_index(va)]); 285281494Sandrew} 286281494Sandrew 287281494Sandrewstatic __inline pd_entry_t * 288281494Sandrewpmap_l1_to_l2(pd_entry_t *l1, vm_offset_t va) 289281494Sandrew{ 290281494Sandrew pd_entry_t *l2; 291281494Sandrew 292288445Sandrew l2 = (pd_entry_t *)PHYS_TO_DMAP(pmap_load(l1) & ~ATTR_MASK); 293281494Sandrew return (&l2[pmap_l2_index(va)]); 294281494Sandrew} 295281494Sandrew 296281494Sandrewstatic __inline pd_entry_t * 297281494Sandrewpmap_l2(pmap_t pmap, vm_offset_t va) 298281494Sandrew{ 299281494Sandrew pd_entry_t *l1; 300281494Sandrew 301281494Sandrew l1 = pmap_l1(pmap, va); 302288445Sandrew if ((pmap_load(l1) & ATTR_DESCR_MASK) != L1_TABLE) 303281494Sandrew return (NULL); 304281494Sandrew 305281494Sandrew return (pmap_l1_to_l2(l1, va)); 306281494Sandrew} 307281494Sandrew 308281494Sandrewstatic __inline pt_entry_t * 309281494Sandrewpmap_l2_to_l3(pd_entry_t *l2, vm_offset_t va) 310281494Sandrew{ 311281494Sandrew pt_entry_t *l3; 312281494Sandrew 313288445Sandrew l3 = (pd_entry_t *)PHYS_TO_DMAP(pmap_load(l2) & ~ATTR_MASK); 314281494Sandrew return (&l3[pmap_l3_index(va)]); 315281494Sandrew} 316281494Sandrew 317281494Sandrewstatic __inline pt_entry_t * 318281494Sandrewpmap_l3(pmap_t pmap, vm_offset_t va) 319281494Sandrew{ 320281494Sandrew pd_entry_t *l2; 321281494Sandrew 322281494Sandrew l2 = pmap_l2(pmap, va); 323288445Sandrew if (l2 == NULL || (pmap_load(l2) & ATTR_DESCR_MASK) != L2_TABLE) 324281494Sandrew return (NULL); 325281494Sandrew 326281494Sandrew return (pmap_l2_to_l3(l2, va)); 327281494Sandrew} 328281494Sandrew 329286956Sandrewbool 330286956Sandrewpmap_get_tables(pmap_t pmap, vm_offset_t va, pd_entry_t **l1, pd_entry_t **l2, 331286956Sandrew pt_entry_t **l3) 332286956Sandrew{ 333286956Sandrew pd_entry_t *l1p, *l2p; 334286956Sandrew 335286956Sandrew if (pmap->pm_l1 == NULL) 336286956Sandrew return (false); 337286956Sandrew 338286956Sandrew l1p = pmap_l1(pmap, va); 339286956Sandrew *l1 = l1p; 340286956Sandrew 341288445Sandrew if ((pmap_load(l1p) & ATTR_DESCR_MASK) == L1_BLOCK) { 342286956Sandrew *l2 = NULL; 343286956Sandrew *l3 = NULL; 344286956Sandrew return (true); 345286956Sandrew } 346286956Sandrew 347288445Sandrew if ((pmap_load(l1p) & ATTR_DESCR_MASK) != L1_TABLE) 348286956Sandrew return (false); 349286956Sandrew 350286956Sandrew l2p = pmap_l1_to_l2(l1p, va); 351286956Sandrew *l2 = l2p; 352286956Sandrew 353288445Sandrew if ((pmap_load(l2p) & ATTR_DESCR_MASK) == L2_BLOCK) { 354286956Sandrew *l3 = NULL; 355286956Sandrew return (true); 356286956Sandrew } 357286956Sandrew 358286956Sandrew *l3 = pmap_l2_to_l3(l2p, va); 359286956Sandrew 360286956Sandrew return (true); 361286956Sandrew} 362286956Sandrew 363281494Sandrewstatic __inline int 364281494Sandrewpmap_is_current(pmap_t pmap) 365281494Sandrew{ 366281494Sandrew 367281494Sandrew return ((pmap == pmap_kernel()) || 368281494Sandrew (pmap == curthread->td_proc->p_vmspace->vm_map.pmap)); 369281494Sandrew} 370281494Sandrew 371281494Sandrewstatic __inline int 372281494Sandrewpmap_l3_valid(pt_entry_t l3) 373281494Sandrew{ 374281494Sandrew 375281494Sandrew return ((l3 & ATTR_DESCR_MASK) == L3_PAGE); 376281494Sandrew} 377281494Sandrew 378281494Sandrewstatic __inline int 379281494Sandrewpmap_l3_valid_cacheable(pt_entry_t l3) 380281494Sandrew{ 381281494Sandrew 382281494Sandrew return (((l3 & ATTR_DESCR_MASK) == L3_PAGE) && 383281494Sandrew ((l3 & ATTR_IDX_MASK) == ATTR_IDX(CACHED_MEMORY))); 384281494Sandrew} 385281494Sandrew 386281494Sandrew#define PTE_SYNC(pte) cpu_dcache_wb_range((vm_offset_t)pte, sizeof(*pte)) 387281494Sandrew 388281494Sandrew/* 389281494Sandrew * Checks if the page is dirty. We currently lack proper tracking of this on 390281494Sandrew * arm64 so for now assume is a page mapped as rw was accessed it is. 391281494Sandrew */ 392281494Sandrewstatic inline int 393281494Sandrewpmap_page_dirty(pt_entry_t pte) 394281494Sandrew{ 395281494Sandrew 396281494Sandrew return ((pte & (ATTR_AF | ATTR_AP_RW_BIT)) == 397281494Sandrew (ATTR_AF | ATTR_AP(ATTR_AP_RW))); 398281494Sandrew} 399281494Sandrew 400281494Sandrewstatic __inline void 401281494Sandrewpmap_resident_count_inc(pmap_t pmap, int count) 402281494Sandrew{ 403281494Sandrew 404281494Sandrew PMAP_LOCK_ASSERT(pmap, MA_OWNED); 405281494Sandrew pmap->pm_stats.resident_count += count; 406281494Sandrew} 407281494Sandrew 408281494Sandrewstatic __inline void 409281494Sandrewpmap_resident_count_dec(pmap_t pmap, int count) 410281494Sandrew{ 411281494Sandrew 412281494Sandrew PMAP_LOCK_ASSERT(pmap, MA_OWNED); 413281494Sandrew KASSERT(pmap->pm_stats.resident_count >= count, 414281494Sandrew ("pmap %p resident count underflow %ld %d", pmap, 415281494Sandrew pmap->pm_stats.resident_count, count)); 416281494Sandrew pmap->pm_stats.resident_count -= count; 417281494Sandrew} 418281494Sandrew 419281494Sandrewstatic pt_entry_t * 420281494Sandrewpmap_early_page_idx(vm_offset_t l1pt, vm_offset_t va, u_int *l1_slot, 421281494Sandrew u_int *l2_slot) 422281494Sandrew{ 423281494Sandrew pt_entry_t *l2; 424281494Sandrew pd_entry_t *l1; 425281494Sandrew 426281494Sandrew l1 = (pd_entry_t *)l1pt; 427281494Sandrew *l1_slot = (va >> L1_SHIFT) & Ln_ADDR_MASK; 428281494Sandrew 429281494Sandrew /* Check locore has used a table L1 map */ 430281494Sandrew KASSERT((l1[*l1_slot] & ATTR_DESCR_MASK) == L1_TABLE, 431281494Sandrew ("Invalid bootstrap L1 table")); 432281494Sandrew /* Find the address of the L2 table */ 433281494Sandrew l2 = (pt_entry_t *)init_pt_va; 434281494Sandrew *l2_slot = pmap_l2_index(va); 435281494Sandrew 436281494Sandrew return (l2); 437281494Sandrew} 438281494Sandrew 439281494Sandrewstatic vm_paddr_t 440281494Sandrewpmap_early_vtophys(vm_offset_t l1pt, vm_offset_t va) 441281494Sandrew{ 442281494Sandrew u_int l1_slot, l2_slot; 443281494Sandrew pt_entry_t *l2; 444281494Sandrew 445281494Sandrew l2 = pmap_early_page_idx(l1pt, va, &l1_slot, &l2_slot); 446281494Sandrew 447281494Sandrew return ((l2[l2_slot] & ~ATTR_MASK) + (va & L2_OFFSET)); 448281494Sandrew} 449281494Sandrew 450281494Sandrewstatic void 451291246Sandrewpmap_bootstrap_dmap(vm_offset_t l1pt, vm_paddr_t kernstart) 452281494Sandrew{ 453281494Sandrew vm_offset_t va; 454281494Sandrew vm_paddr_t pa; 455281494Sandrew pd_entry_t *l1; 456281494Sandrew u_int l1_slot; 457281494Sandrew 458291246Sandrew pa = dmap_phys_base = kernstart & ~L1_OFFSET; 459281494Sandrew va = DMAP_MIN_ADDRESS; 460281494Sandrew l1 = (pd_entry_t *)l1pt; 461281494Sandrew l1_slot = pmap_l1_index(DMAP_MIN_ADDRESS); 462281494Sandrew 463291246Sandrew for (; va < DMAP_MAX_ADDRESS; 464281494Sandrew pa += L1_SIZE, va += L1_SIZE, l1_slot++) { 465281494Sandrew KASSERT(l1_slot < Ln_ENTRIES, ("Invalid L1 index")); 466281494Sandrew 467281494Sandrew pmap_load_store(&l1[l1_slot], 468285537Sandrew (pa & ~L1_OFFSET) | ATTR_DEFAULT | 469285537Sandrew ATTR_IDX(CACHED_MEMORY) | L1_BLOCK); 470281494Sandrew } 471281494Sandrew 472281494Sandrew cpu_dcache_wb_range((vm_offset_t)l1, PAGE_SIZE); 473281494Sandrew cpu_tlb_flushID(); 474281494Sandrew} 475281494Sandrew 476281494Sandrewstatic vm_offset_t 477281494Sandrewpmap_bootstrap_l2(vm_offset_t l1pt, vm_offset_t va, vm_offset_t l2_start) 478281494Sandrew{ 479281494Sandrew vm_offset_t l2pt; 480281494Sandrew vm_paddr_t pa; 481281494Sandrew pd_entry_t *l1; 482281494Sandrew u_int l1_slot; 483281494Sandrew 484281494Sandrew KASSERT((va & L1_OFFSET) == 0, ("Invalid virtual address")); 485281494Sandrew 486281494Sandrew l1 = (pd_entry_t *)l1pt; 487281494Sandrew l1_slot = pmap_l1_index(va); 488281494Sandrew l2pt = l2_start; 489281494Sandrew 490281494Sandrew for (; va < VM_MAX_KERNEL_ADDRESS; l1_slot++, va += L1_SIZE) { 491281494Sandrew KASSERT(l1_slot < Ln_ENTRIES, ("Invalid L1 index")); 492281494Sandrew 493281494Sandrew pa = pmap_early_vtophys(l1pt, l2pt); 494281494Sandrew pmap_load_store(&l1[l1_slot], 495281494Sandrew (pa & ~Ln_TABLE_MASK) | L1_TABLE); 496281494Sandrew l2pt += PAGE_SIZE; 497281494Sandrew } 498281494Sandrew 499281494Sandrew /* Clean the L2 page table */ 500281494Sandrew memset((void *)l2_start, 0, l2pt - l2_start); 501281494Sandrew cpu_dcache_wb_range(l2_start, l2pt - l2_start); 502281494Sandrew 503281494Sandrew /* Flush the l1 table to ram */ 504281494Sandrew cpu_dcache_wb_range((vm_offset_t)l1, PAGE_SIZE); 505281494Sandrew 506281494Sandrew return l2pt; 507281494Sandrew} 508281494Sandrew 509281494Sandrewstatic vm_offset_t 510281494Sandrewpmap_bootstrap_l3(vm_offset_t l1pt, vm_offset_t va, vm_offset_t l3_start) 511281494Sandrew{ 512281494Sandrew vm_offset_t l2pt, l3pt; 513281494Sandrew vm_paddr_t pa; 514281494Sandrew pd_entry_t *l2; 515281494Sandrew u_int l2_slot; 516281494Sandrew 517281494Sandrew KASSERT((va & L2_OFFSET) == 0, ("Invalid virtual address")); 518281494Sandrew 519281494Sandrew l2 = pmap_l2(kernel_pmap, va); 520281494Sandrew l2 = (pd_entry_t *)((uintptr_t)l2 & ~(PAGE_SIZE - 1)); 521281494Sandrew l2pt = (vm_offset_t)l2; 522281494Sandrew l2_slot = pmap_l2_index(va); 523281494Sandrew l3pt = l3_start; 524281494Sandrew 525281494Sandrew for (; va < VM_MAX_KERNEL_ADDRESS; l2_slot++, va += L2_SIZE) { 526281494Sandrew KASSERT(l2_slot < Ln_ENTRIES, ("Invalid L2 index")); 527281494Sandrew 528281494Sandrew pa = pmap_early_vtophys(l1pt, l3pt); 529281494Sandrew pmap_load_store(&l2[l2_slot], 530281494Sandrew (pa & ~Ln_TABLE_MASK) | L2_TABLE); 531281494Sandrew l3pt += PAGE_SIZE; 532281494Sandrew } 533281494Sandrew 534281494Sandrew /* Clean the L2 page table */ 535281494Sandrew memset((void *)l3_start, 0, l3pt - l3_start); 536281494Sandrew cpu_dcache_wb_range(l3_start, l3pt - l3_start); 537281494Sandrew 538281494Sandrew cpu_dcache_wb_range((vm_offset_t)l2, PAGE_SIZE); 539281494Sandrew 540281494Sandrew return l3pt; 541281494Sandrew} 542281494Sandrew 543281494Sandrew/* 544281494Sandrew * Bootstrap the system enough to run with virtual memory. 545281494Sandrew */ 546281494Sandrewvoid 547281494Sandrewpmap_bootstrap(vm_offset_t l1pt, vm_paddr_t kernstart, vm_size_t kernlen) 548281494Sandrew{ 549281494Sandrew u_int l1_slot, l2_slot, avail_slot, map_slot, used_map_slot; 550281494Sandrew uint64_t kern_delta; 551281494Sandrew pt_entry_t *l2; 552281494Sandrew vm_offset_t va, freemempos; 553281494Sandrew vm_offset_t dpcpu, msgbufpv; 554291246Sandrew vm_paddr_t pa, min_pa; 555291246Sandrew int i; 556281494Sandrew 557281494Sandrew kern_delta = KERNBASE - kernstart; 558281494Sandrew physmem = 0; 559281494Sandrew 560281494Sandrew printf("pmap_bootstrap %lx %lx %lx\n", l1pt, kernstart, kernlen); 561281494Sandrew printf("%lx\n", l1pt); 562281494Sandrew printf("%lx\n", (KERNBASE >> L1_SHIFT) & Ln_ADDR_MASK); 563281494Sandrew 564281494Sandrew /* Set this early so we can use the pagetable walking functions */ 565281494Sandrew kernel_pmap_store.pm_l1 = (pd_entry_t *)l1pt; 566281494Sandrew PMAP_LOCK_INIT(kernel_pmap); 567281494Sandrew 568281494Sandrew /* 569281494Sandrew * Initialize the global pv list lock. 570281494Sandrew */ 571281494Sandrew rw_init(&pvh_global_lock, "pmap pv global"); 572281494Sandrew 573291246Sandrew /* Assume the address we were loaded to is a valid physical address */ 574291246Sandrew min_pa = KERNBASE - kern_delta; 575291246Sandrew 576291246Sandrew /* 577291246Sandrew * Find the minimum physical address. physmap is sorted, 578291246Sandrew * but may contain empty ranges. 579291246Sandrew */ 580291246Sandrew for (i = 0; i < (physmap_idx * 2); i += 2) { 581291246Sandrew if (physmap[i] == physmap[i + 1]) 582291246Sandrew continue; 583291246Sandrew if (physmap[i] <= min_pa) 584291246Sandrew min_pa = physmap[i]; 585291246Sandrew break; 586291246Sandrew } 587291246Sandrew 588281494Sandrew /* Create a direct map region early so we can use it for pa -> va */ 589291246Sandrew pmap_bootstrap_dmap(l1pt, min_pa); 590281494Sandrew 591281494Sandrew va = KERNBASE; 592281494Sandrew pa = KERNBASE - kern_delta; 593281494Sandrew 594281494Sandrew /* 595281494Sandrew * Start to initialise phys_avail by copying from physmap 596281494Sandrew * up to the physical address KERNBASE points at. 597281494Sandrew */ 598281494Sandrew map_slot = avail_slot = 0; 599295157Sandrew for (; map_slot < (physmap_idx * 2) && 600295157Sandrew avail_slot < (PHYS_AVAIL_SIZE - 2); map_slot += 2) { 601281494Sandrew if (physmap[map_slot] == physmap[map_slot + 1]) 602281494Sandrew continue; 603281494Sandrew 604281494Sandrew if (physmap[map_slot] <= pa && 605281494Sandrew physmap[map_slot + 1] > pa) 606281494Sandrew break; 607281494Sandrew 608281494Sandrew phys_avail[avail_slot] = physmap[map_slot]; 609281494Sandrew phys_avail[avail_slot + 1] = physmap[map_slot + 1]; 610281494Sandrew physmem += (phys_avail[avail_slot + 1] - 611281494Sandrew phys_avail[avail_slot]) >> PAGE_SHIFT; 612281494Sandrew avail_slot += 2; 613281494Sandrew } 614281494Sandrew 615281494Sandrew /* Add the memory before the kernel */ 616295157Sandrew if (physmap[avail_slot] < pa && avail_slot < (PHYS_AVAIL_SIZE - 2)) { 617281494Sandrew phys_avail[avail_slot] = physmap[map_slot]; 618281494Sandrew phys_avail[avail_slot + 1] = pa; 619281494Sandrew physmem += (phys_avail[avail_slot + 1] - 620281494Sandrew phys_avail[avail_slot]) >> PAGE_SHIFT; 621281494Sandrew avail_slot += 2; 622281494Sandrew } 623281494Sandrew used_map_slot = map_slot; 624281494Sandrew 625281494Sandrew /* 626281494Sandrew * Read the page table to find out what is already mapped. 627281494Sandrew * This assumes we have mapped a block of memory from KERNBASE 628281494Sandrew * using a single L1 entry. 629281494Sandrew */ 630281494Sandrew l2 = pmap_early_page_idx(l1pt, KERNBASE, &l1_slot, &l2_slot); 631281494Sandrew 632281494Sandrew /* Sanity check the index, KERNBASE should be the first VA */ 633281494Sandrew KASSERT(l2_slot == 0, ("The L2 index is non-zero")); 634281494Sandrew 635281494Sandrew /* Find how many pages we have mapped */ 636281494Sandrew for (; l2_slot < Ln_ENTRIES; l2_slot++) { 637281494Sandrew if ((l2[l2_slot] & ATTR_DESCR_MASK) == 0) 638281494Sandrew break; 639281494Sandrew 640281494Sandrew /* Check locore used L2 blocks */ 641281494Sandrew KASSERT((l2[l2_slot] & ATTR_DESCR_MASK) == L2_BLOCK, 642281494Sandrew ("Invalid bootstrap L2 table")); 643281494Sandrew KASSERT((l2[l2_slot] & ~ATTR_MASK) == pa, 644281494Sandrew ("Incorrect PA in L2 table")); 645281494Sandrew 646281494Sandrew va += L2_SIZE; 647281494Sandrew pa += L2_SIZE; 648281494Sandrew } 649281494Sandrew 650281494Sandrew va = roundup2(va, L1_SIZE); 651281494Sandrew 652281494Sandrew freemempos = KERNBASE + kernlen; 653281494Sandrew freemempos = roundup2(freemempos, PAGE_SIZE); 654281494Sandrew /* Create the l2 tables up to VM_MAX_KERNEL_ADDRESS */ 655281494Sandrew freemempos = pmap_bootstrap_l2(l1pt, va, freemempos); 656281494Sandrew /* And the l3 tables for the early devmap */ 657281494Sandrew freemempos = pmap_bootstrap_l3(l1pt, 658281494Sandrew VM_MAX_KERNEL_ADDRESS - L2_SIZE, freemempos); 659281494Sandrew 660281494Sandrew cpu_tlb_flushID(); 661281494Sandrew 662281494Sandrew#define alloc_pages(var, np) \ 663281494Sandrew (var) = freemempos; \ 664281494Sandrew freemempos += (np * PAGE_SIZE); \ 665281494Sandrew memset((char *)(var), 0, ((np) * PAGE_SIZE)); 666281494Sandrew 667281494Sandrew /* Allocate dynamic per-cpu area. */ 668281494Sandrew alloc_pages(dpcpu, DPCPU_SIZE / PAGE_SIZE); 669281494Sandrew dpcpu_init((void *)dpcpu, 0); 670281494Sandrew 671281494Sandrew /* Allocate memory for the msgbuf, e.g. for /sbin/dmesg */ 672281494Sandrew alloc_pages(msgbufpv, round_page(msgbufsize) / PAGE_SIZE); 673281494Sandrew msgbufp = (void *)msgbufpv; 674281494Sandrew 675281494Sandrew virtual_avail = roundup2(freemempos, L1_SIZE); 676281494Sandrew virtual_end = VM_MAX_KERNEL_ADDRESS - L2_SIZE; 677281494Sandrew kernel_vm_end = virtual_avail; 678281494Sandrew 679281494Sandrew pa = pmap_early_vtophys(l1pt, freemempos); 680281494Sandrew 681281494Sandrew /* Finish initialising physmap */ 682281494Sandrew map_slot = used_map_slot; 683281494Sandrew for (; avail_slot < (PHYS_AVAIL_SIZE - 2) && 684281494Sandrew map_slot < (physmap_idx * 2); map_slot += 2) { 685281494Sandrew if (physmap[map_slot] == physmap[map_slot + 1]) 686281494Sandrew continue; 687281494Sandrew 688281494Sandrew /* Have we used the current range? */ 689281494Sandrew if (physmap[map_slot + 1] <= pa) 690281494Sandrew continue; 691281494Sandrew 692281494Sandrew /* Do we need to split the entry? */ 693281494Sandrew if (physmap[map_slot] < pa) { 694281494Sandrew phys_avail[avail_slot] = pa; 695281494Sandrew phys_avail[avail_slot + 1] = physmap[map_slot + 1]; 696281494Sandrew } else { 697281494Sandrew phys_avail[avail_slot] = physmap[map_slot]; 698281494Sandrew phys_avail[avail_slot + 1] = physmap[map_slot + 1]; 699281494Sandrew } 700281494Sandrew physmem += (phys_avail[avail_slot + 1] - 701281494Sandrew phys_avail[avail_slot]) >> PAGE_SHIFT; 702281494Sandrew 703281494Sandrew avail_slot += 2; 704281494Sandrew } 705281494Sandrew phys_avail[avail_slot] = 0; 706281494Sandrew phys_avail[avail_slot + 1] = 0; 707281494Sandrew 708281494Sandrew /* 709281494Sandrew * Maxmem isn't the "maximum memory", it's one larger than the 710281494Sandrew * highest page of the physical address space. It should be 711281494Sandrew * called something like "Maxphyspage". 712281494Sandrew */ 713281494Sandrew Maxmem = atop(phys_avail[avail_slot - 1]); 714281494Sandrew 715281494Sandrew cpu_tlb_flushID(); 716281494Sandrew} 717281494Sandrew 718281494Sandrew/* 719281494Sandrew * Initialize a vm_page's machine-dependent fields. 720281494Sandrew */ 721281494Sandrewvoid 722281494Sandrewpmap_page_init(vm_page_t m) 723281494Sandrew{ 724281494Sandrew 725281494Sandrew TAILQ_INIT(&m->md.pv_list); 726281494Sandrew m->md.pv_memattr = VM_MEMATTR_WRITE_BACK; 727281494Sandrew} 728281494Sandrew 729281494Sandrew/* 730281494Sandrew * Initialize the pmap module. 731281494Sandrew * Called by vm_init, to initialize any structures that the pmap 732281494Sandrew * system needs to map virtual memory. 733281494Sandrew */ 734281494Sandrewvoid 735281494Sandrewpmap_init(void) 736281494Sandrew{ 737281494Sandrew int i; 738281494Sandrew 739281494Sandrew /* 740281494Sandrew * Initialize the pv chunk list mutex. 741281494Sandrew */ 742281494Sandrew mtx_init(&pv_chunks_mutex, "pmap pv chunk list", NULL, MTX_DEF); 743281494Sandrew 744281494Sandrew /* 745281494Sandrew * Initialize the pool of pv list locks. 746281494Sandrew */ 747281494Sandrew for (i = 0; i < NPV_LIST_LOCKS; i++) 748281494Sandrew rw_init(&pv_list_locks[i], "pmap pv list"); 749281494Sandrew} 750281494Sandrew 751281494Sandrew/* 752281494Sandrew * Normal, non-SMP, invalidation functions. 753281494Sandrew * We inline these within pmap.c for speed. 754281494Sandrew */ 755281494SandrewPMAP_INLINE void 756281494Sandrewpmap_invalidate_page(pmap_t pmap, vm_offset_t va) 757281494Sandrew{ 758281494Sandrew 759281494Sandrew sched_pin(); 760281494Sandrew __asm __volatile( 761281494Sandrew "dsb sy \n" 762281494Sandrew "tlbi vaae1is, %0 \n" 763281494Sandrew "dsb sy \n" 764281494Sandrew "isb \n" 765281494Sandrew : : "r"(va >> PAGE_SHIFT)); 766281494Sandrew sched_unpin(); 767281494Sandrew} 768281494Sandrew 769281494SandrewPMAP_INLINE void 770281494Sandrewpmap_invalidate_range(pmap_t pmap, vm_offset_t sva, vm_offset_t eva) 771281494Sandrew{ 772281494Sandrew vm_offset_t addr; 773281494Sandrew 774281494Sandrew sched_pin(); 775281494Sandrew sva >>= PAGE_SHIFT; 776281494Sandrew eva >>= PAGE_SHIFT; 777281494Sandrew __asm __volatile("dsb sy"); 778281494Sandrew for (addr = sva; addr < eva; addr++) { 779281494Sandrew __asm __volatile( 780281494Sandrew "tlbi vaae1is, %0" : : "r"(addr)); 781281494Sandrew } 782281494Sandrew __asm __volatile( 783281494Sandrew "dsb sy \n" 784281494Sandrew "isb \n"); 785281494Sandrew sched_unpin(); 786281494Sandrew} 787281494Sandrew 788281494SandrewPMAP_INLINE void 789281494Sandrewpmap_invalidate_all(pmap_t pmap) 790281494Sandrew{ 791281494Sandrew 792281494Sandrew sched_pin(); 793281494Sandrew __asm __volatile( 794281494Sandrew "dsb sy \n" 795281494Sandrew "tlbi vmalle1is \n" 796281494Sandrew "dsb sy \n" 797281494Sandrew "isb \n"); 798281494Sandrew sched_unpin(); 799281494Sandrew} 800281494Sandrew 801281494Sandrew/* 802281494Sandrew * Routine: pmap_extract 803281494Sandrew * Function: 804281494Sandrew * Extract the physical page address associated 805281494Sandrew * with the given map/virtual_address pair. 806281494Sandrew */ 807281494Sandrewvm_paddr_t 808281494Sandrewpmap_extract(pmap_t pmap, vm_offset_t va) 809281494Sandrew{ 810281494Sandrew pd_entry_t *l2p, l2; 811281494Sandrew pt_entry_t *l3p, l3; 812281494Sandrew vm_paddr_t pa; 813281494Sandrew 814281494Sandrew pa = 0; 815281494Sandrew PMAP_LOCK(pmap); 816281494Sandrew /* 817281494Sandrew * Start with the l2 tabel. We are unable to allocate 818281494Sandrew * pages in the l1 table. 819281494Sandrew */ 820281494Sandrew l2p = pmap_l2(pmap, va); 821281494Sandrew if (l2p != NULL) { 822288445Sandrew l2 = pmap_load(l2p); 823281494Sandrew if ((l2 & ATTR_DESCR_MASK) == L2_TABLE) { 824281494Sandrew l3p = pmap_l2_to_l3(l2p, va); 825281494Sandrew if (l3p != NULL) { 826288445Sandrew l3 = pmap_load(l3p); 827281494Sandrew 828281494Sandrew if ((l3 & ATTR_DESCR_MASK) == L3_PAGE) 829281494Sandrew pa = (l3 & ~ATTR_MASK) | 830281494Sandrew (va & L3_OFFSET); 831281494Sandrew } 832281494Sandrew } else if ((l2 & ATTR_DESCR_MASK) == L2_BLOCK) 833281494Sandrew pa = (l2 & ~ATTR_MASK) | (va & L2_OFFSET); 834281494Sandrew } 835281494Sandrew PMAP_UNLOCK(pmap); 836281494Sandrew return (pa); 837281494Sandrew} 838281494Sandrew 839281494Sandrew/* 840281494Sandrew * Routine: pmap_extract_and_hold 841281494Sandrew * Function: 842281494Sandrew * Atomically extract and hold the physical page 843281494Sandrew * with the given pmap and virtual address pair 844281494Sandrew * if that mapping permits the given protection. 845281494Sandrew */ 846281494Sandrewvm_page_t 847281494Sandrewpmap_extract_and_hold(pmap_t pmap, vm_offset_t va, vm_prot_t prot) 848281494Sandrew{ 849281494Sandrew pt_entry_t *l3p, l3; 850281494Sandrew vm_paddr_t pa; 851281494Sandrew vm_page_t m; 852281494Sandrew 853281494Sandrew pa = 0; 854281494Sandrew m = NULL; 855281494Sandrew PMAP_LOCK(pmap); 856281494Sandrewretry: 857281494Sandrew l3p = pmap_l3(pmap, va); 858285045Sandrew if (l3p != NULL && (l3 = pmap_load(l3p)) != 0) { 859281494Sandrew if (((l3 & ATTR_AP_RW_BIT) == ATTR_AP(ATTR_AP_RW)) || 860281494Sandrew ((prot & VM_PROT_WRITE) == 0)) { 861281494Sandrew if (vm_page_pa_tryrelock(pmap, l3 & ~ATTR_MASK, &pa)) 862281494Sandrew goto retry; 863281494Sandrew m = PHYS_TO_VM_PAGE(l3 & ~ATTR_MASK); 864281494Sandrew vm_page_hold(m); 865281494Sandrew } 866281494Sandrew } 867281494Sandrew PA_UNLOCK_COND(pa); 868281494Sandrew PMAP_UNLOCK(pmap); 869281494Sandrew return (m); 870281494Sandrew} 871281494Sandrew 872281494Sandrewvm_paddr_t 873281494Sandrewpmap_kextract(vm_offset_t va) 874281494Sandrew{ 875288445Sandrew pd_entry_t *l2p, l2; 876281494Sandrew pt_entry_t *l3; 877281494Sandrew vm_paddr_t pa; 878281494Sandrew 879281494Sandrew if (va >= DMAP_MIN_ADDRESS && va < DMAP_MAX_ADDRESS) { 880281494Sandrew pa = DMAP_TO_PHYS(va); 881281494Sandrew } else { 882288445Sandrew l2p = pmap_l2(kernel_pmap, va); 883288445Sandrew if (l2p == NULL) 884281494Sandrew panic("pmap_kextract: No l2"); 885288445Sandrew l2 = pmap_load(l2p); 886288445Sandrew if ((l2 & ATTR_DESCR_MASK) == L2_BLOCK) 887288445Sandrew return ((l2 & ~ATTR_MASK) | 888288445Sandrew (va & L2_OFFSET)); 889281494Sandrew 890288445Sandrew l3 = pmap_l2_to_l3(l2p, va); 891281494Sandrew if (l3 == NULL) 892281494Sandrew panic("pmap_kextract: No l3..."); 893288445Sandrew pa = (pmap_load(l3) & ~ATTR_MASK) | (va & PAGE_MASK); 894281494Sandrew } 895281494Sandrew return (pa); 896281494Sandrew} 897281494Sandrew 898281494Sandrew/*************************************************** 899281494Sandrew * Low level mapping routines..... 900281494Sandrew ***************************************************/ 901281494Sandrew 902281494Sandrewvoid 903285212Sandrewpmap_kenter_device(vm_offset_t sva, vm_size_t size, vm_paddr_t pa) 904281494Sandrew{ 905281494Sandrew pt_entry_t *l3; 906285212Sandrew vm_offset_t va; 907281494Sandrew 908281494Sandrew KASSERT((pa & L3_OFFSET) == 0, 909281494Sandrew ("pmap_kenter_device: Invalid physical address")); 910285212Sandrew KASSERT((sva & L3_OFFSET) == 0, 911281494Sandrew ("pmap_kenter_device: Invalid virtual address")); 912281494Sandrew KASSERT((size & PAGE_MASK) == 0, 913281494Sandrew ("pmap_kenter_device: Mapping is not page-sized")); 914281494Sandrew 915285212Sandrew va = sva; 916281494Sandrew while (size != 0) { 917281494Sandrew l3 = pmap_l3(kernel_pmap, va); 918281494Sandrew KASSERT(l3 != NULL, ("Invalid page table, va: 0x%lx", va)); 919285537Sandrew pmap_load_store(l3, (pa & ~L3_OFFSET) | ATTR_DEFAULT | 920285537Sandrew ATTR_IDX(DEVICE_MEMORY) | L3_PAGE); 921281494Sandrew PTE_SYNC(l3); 922281494Sandrew 923281494Sandrew va += PAGE_SIZE; 924281494Sandrew pa += PAGE_SIZE; 925281494Sandrew size -= PAGE_SIZE; 926281494Sandrew } 927285212Sandrew pmap_invalidate_range(kernel_pmap, sva, va); 928281494Sandrew} 929281494Sandrew 930281494Sandrew/* 931281494Sandrew * Remove a page from the kernel pagetables. 932281494Sandrew * Note: not SMP coherent. 933281494Sandrew */ 934281494SandrewPMAP_INLINE void 935281494Sandrewpmap_kremove(vm_offset_t va) 936281494Sandrew{ 937281494Sandrew pt_entry_t *l3; 938281494Sandrew 939281494Sandrew l3 = pmap_l3(kernel_pmap, va); 940281494Sandrew KASSERT(l3 != NULL, ("pmap_kremove: Invalid address")); 941281494Sandrew 942281494Sandrew if (pmap_l3_valid_cacheable(pmap_load(l3))) 943281494Sandrew cpu_dcache_wb_range(va, L3_SIZE); 944281494Sandrew pmap_load_clear(l3); 945281494Sandrew PTE_SYNC(l3); 946285212Sandrew pmap_invalidate_page(kernel_pmap, va); 947281494Sandrew} 948281494Sandrew 949281494Sandrewvoid 950285212Sandrewpmap_kremove_device(vm_offset_t sva, vm_size_t size) 951281494Sandrew{ 952281494Sandrew pt_entry_t *l3; 953285212Sandrew vm_offset_t va; 954281494Sandrew 955285212Sandrew KASSERT((sva & L3_OFFSET) == 0, 956281494Sandrew ("pmap_kremove_device: Invalid virtual address")); 957281494Sandrew KASSERT((size & PAGE_MASK) == 0, 958281494Sandrew ("pmap_kremove_device: Mapping is not page-sized")); 959281494Sandrew 960285212Sandrew va = sva; 961281494Sandrew while (size != 0) { 962281494Sandrew l3 = pmap_l3(kernel_pmap, va); 963281494Sandrew KASSERT(l3 != NULL, ("Invalid page table, va: 0x%lx", va)); 964281494Sandrew pmap_load_clear(l3); 965281494Sandrew PTE_SYNC(l3); 966281494Sandrew 967281494Sandrew va += PAGE_SIZE; 968281494Sandrew size -= PAGE_SIZE; 969281494Sandrew } 970285212Sandrew pmap_invalidate_range(kernel_pmap, sva, va); 971281494Sandrew} 972281494Sandrew 973281494Sandrew/* 974281494Sandrew * Used to map a range of physical addresses into kernel 975281494Sandrew * virtual address space. 976281494Sandrew * 977281494Sandrew * The value passed in '*virt' is a suggested virtual address for 978281494Sandrew * the mapping. Architectures which can support a direct-mapped 979281494Sandrew * physical to virtual region can return the appropriate address 980281494Sandrew * within that region, leaving '*virt' unchanged. Other 981281494Sandrew * architectures should map the pages starting at '*virt' and 982281494Sandrew * update '*virt' with the first usable address after the mapped 983281494Sandrew * region. 984281494Sandrew */ 985281494Sandrewvm_offset_t 986281494Sandrewpmap_map(vm_offset_t *virt, vm_paddr_t start, vm_paddr_t end, int prot) 987281494Sandrew{ 988281494Sandrew return PHYS_TO_DMAP(start); 989281494Sandrew} 990281494Sandrew 991281494Sandrew 992281494Sandrew/* 993281494Sandrew * Add a list of wired pages to the kva 994281494Sandrew * this routine is only used for temporary 995281494Sandrew * kernel mappings that do not need to have 996281494Sandrew * page modification or references recorded. 997281494Sandrew * Note that old mappings are simply written 998281494Sandrew * over. The page *must* be wired. 999281494Sandrew * Note: SMP coherent. Uses a ranged shootdown IPI. 1000281494Sandrew */ 1001281494Sandrewvoid 1002281494Sandrewpmap_qenter(vm_offset_t sva, vm_page_t *ma, int count) 1003281494Sandrew{ 1004281494Sandrew pt_entry_t *l3, pa; 1005281494Sandrew vm_offset_t va; 1006281494Sandrew vm_page_t m; 1007281494Sandrew int i; 1008281494Sandrew 1009281494Sandrew va = sva; 1010281494Sandrew for (i = 0; i < count; i++) { 1011281494Sandrew m = ma[i]; 1012285537Sandrew pa = VM_PAGE_TO_PHYS(m) | ATTR_DEFAULT | ATTR_AP(ATTR_AP_RW) | 1013285537Sandrew ATTR_IDX(m->md.pv_memattr) | L3_PAGE; 1014281494Sandrew l3 = pmap_l3(kernel_pmap, va); 1015281494Sandrew pmap_load_store(l3, pa); 1016281494Sandrew PTE_SYNC(l3); 1017281494Sandrew 1018281494Sandrew va += L3_SIZE; 1019281494Sandrew } 1020285212Sandrew pmap_invalidate_range(kernel_pmap, sva, va); 1021281494Sandrew} 1022281494Sandrew 1023281494Sandrew/* 1024281494Sandrew * This routine tears out page mappings from the 1025281494Sandrew * kernel -- it is meant only for temporary mappings. 1026281494Sandrew * Note: SMP coherent. Uses a ranged shootdown IPI. 1027281494Sandrew */ 1028281494Sandrewvoid 1029281494Sandrewpmap_qremove(vm_offset_t sva, int count) 1030281494Sandrew{ 1031285212Sandrew pt_entry_t *l3; 1032281494Sandrew vm_offset_t va; 1033281494Sandrew 1034285212Sandrew KASSERT(sva >= VM_MIN_KERNEL_ADDRESS, ("usermode va %lx", sva)); 1035285212Sandrew 1036281494Sandrew va = sva; 1037281494Sandrew while (count-- > 0) { 1038285212Sandrew l3 = pmap_l3(kernel_pmap, va); 1039285212Sandrew KASSERT(l3 != NULL, ("pmap_kremove: Invalid address")); 1040285212Sandrew 1041285212Sandrew if (pmap_l3_valid_cacheable(pmap_load(l3))) 1042285212Sandrew cpu_dcache_wb_range(va, L3_SIZE); 1043285212Sandrew pmap_load_clear(l3); 1044285212Sandrew PTE_SYNC(l3); 1045285212Sandrew 1046281494Sandrew va += PAGE_SIZE; 1047281494Sandrew } 1048281494Sandrew pmap_invalidate_range(kernel_pmap, sva, va); 1049281494Sandrew} 1050281494Sandrew 1051281494Sandrew/*************************************************** 1052281494Sandrew * Page table page management routines..... 1053281494Sandrew ***************************************************/ 1054281494Sandrewstatic __inline void 1055281494Sandrewpmap_free_zero_pages(struct spglist *free) 1056281494Sandrew{ 1057281494Sandrew vm_page_t m; 1058281494Sandrew 1059281494Sandrew while ((m = SLIST_FIRST(free)) != NULL) { 1060281494Sandrew SLIST_REMOVE_HEAD(free, plinks.s.ss); 1061281494Sandrew /* Preserve the page's PG_ZERO setting. */ 1062281494Sandrew vm_page_free_toq(m); 1063281494Sandrew } 1064281494Sandrew} 1065281494Sandrew 1066281494Sandrew/* 1067281494Sandrew * Schedule the specified unused page table page to be freed. Specifically, 1068281494Sandrew * add the page to the specified list of pages that will be released to the 1069281494Sandrew * physical memory manager after the TLB has been updated. 1070281494Sandrew */ 1071281494Sandrewstatic __inline void 1072281494Sandrewpmap_add_delayed_free_list(vm_page_t m, struct spglist *free, 1073281494Sandrew boolean_t set_PG_ZERO) 1074281494Sandrew{ 1075281494Sandrew 1076281494Sandrew if (set_PG_ZERO) 1077281494Sandrew m->flags |= PG_ZERO; 1078281494Sandrew else 1079281494Sandrew m->flags &= ~PG_ZERO; 1080281494Sandrew SLIST_INSERT_HEAD(free, m, plinks.s.ss); 1081281494Sandrew} 1082281494Sandrew 1083281494Sandrew/* 1084281494Sandrew * Decrements a page table page's wire count, which is used to record the 1085281494Sandrew * number of valid page table entries within the page. If the wire count 1086281494Sandrew * drops to zero, then the page table page is unmapped. Returns TRUE if the 1087281494Sandrew * page table page was unmapped and FALSE otherwise. 1088281494Sandrew */ 1089281494Sandrewstatic inline boolean_t 1090281494Sandrewpmap_unwire_l3(pmap_t pmap, vm_offset_t va, vm_page_t m, struct spglist *free) 1091281494Sandrew{ 1092281494Sandrew 1093281494Sandrew --m->wire_count; 1094281494Sandrew if (m->wire_count == 0) { 1095281494Sandrew _pmap_unwire_l3(pmap, va, m, free); 1096281494Sandrew return (TRUE); 1097281494Sandrew } else 1098281494Sandrew return (FALSE); 1099281494Sandrew} 1100281494Sandrew 1101281494Sandrewstatic void 1102281494Sandrew_pmap_unwire_l3(pmap_t pmap, vm_offset_t va, vm_page_t m, struct spglist *free) 1103281494Sandrew{ 1104281494Sandrew 1105281494Sandrew PMAP_LOCK_ASSERT(pmap, MA_OWNED); 1106281494Sandrew /* 1107281494Sandrew * unmap the page table page 1108281494Sandrew */ 1109281494Sandrew if (m->pindex >= NUPDE) { 1110281494Sandrew /* PD page */ 1111281494Sandrew pd_entry_t *l1; 1112281494Sandrew l1 = pmap_l1(pmap, va); 1113281494Sandrew pmap_load_clear(l1); 1114281494Sandrew PTE_SYNC(l1); 1115281494Sandrew } else { 1116281494Sandrew /* PTE page */ 1117281494Sandrew pd_entry_t *l2; 1118281494Sandrew l2 = pmap_l2(pmap, va); 1119281494Sandrew pmap_load_clear(l2); 1120281494Sandrew PTE_SYNC(l2); 1121281494Sandrew } 1122281494Sandrew pmap_resident_count_dec(pmap, 1); 1123281494Sandrew if (m->pindex < NUPDE) { 1124281494Sandrew /* We just released a PT, unhold the matching PD */ 1125281494Sandrew vm_page_t pdpg; 1126281494Sandrew 1127281494Sandrew pdpg = PHYS_TO_VM_PAGE(*pmap_l1(pmap, va) & ~ATTR_MASK); 1128281494Sandrew pmap_unwire_l3(pmap, va, pdpg, free); 1129281494Sandrew } 1130285212Sandrew pmap_invalidate_page(pmap, va); 1131281494Sandrew 1132281494Sandrew /* 1133281494Sandrew * This is a release store so that the ordinary store unmapping 1134281494Sandrew * the page table page is globally performed before TLB shoot- 1135281494Sandrew * down is begun. 1136281494Sandrew */ 1137281494Sandrew atomic_subtract_rel_int(&vm_cnt.v_wire_count, 1); 1138281494Sandrew 1139281494Sandrew /* 1140281494Sandrew * Put page on a list so that it is released after 1141281494Sandrew * *ALL* TLB shootdown is done 1142281494Sandrew */ 1143281494Sandrew pmap_add_delayed_free_list(m, free, TRUE); 1144281494Sandrew} 1145281494Sandrew 1146281494Sandrew/* 1147281494Sandrew * After removing an l3 entry, this routine is used to 1148281494Sandrew * conditionally free the page, and manage the hold/wire counts. 1149281494Sandrew */ 1150281494Sandrewstatic int 1151281494Sandrewpmap_unuse_l3(pmap_t pmap, vm_offset_t va, pd_entry_t ptepde, 1152281494Sandrew struct spglist *free) 1153281494Sandrew{ 1154281494Sandrew vm_page_t mpte; 1155281494Sandrew 1156281494Sandrew if (va >= VM_MAXUSER_ADDRESS) 1157281494Sandrew return (0); 1158281494Sandrew KASSERT(ptepde != 0, ("pmap_unuse_pt: ptepde != 0")); 1159281494Sandrew mpte = PHYS_TO_VM_PAGE(ptepde & ~ATTR_MASK); 1160281494Sandrew return (pmap_unwire_l3(pmap, va, mpte, free)); 1161281494Sandrew} 1162281494Sandrew 1163281494Sandrewvoid 1164281494Sandrewpmap_pinit0(pmap_t pmap) 1165281494Sandrew{ 1166281494Sandrew 1167281494Sandrew PMAP_LOCK_INIT(pmap); 1168281494Sandrew bzero(&pmap->pm_stats, sizeof(pmap->pm_stats)); 1169281494Sandrew pmap->pm_l1 = kernel_pmap->pm_l1; 1170281494Sandrew} 1171281494Sandrew 1172281494Sandrewint 1173281494Sandrewpmap_pinit(pmap_t pmap) 1174281494Sandrew{ 1175281494Sandrew vm_paddr_t l1phys; 1176281494Sandrew vm_page_t l1pt; 1177281494Sandrew 1178281494Sandrew /* 1179281494Sandrew * allocate the l1 page 1180281494Sandrew */ 1181281494Sandrew while ((l1pt = vm_page_alloc(NULL, 0xdeadbeef, VM_ALLOC_NORMAL | 1182281494Sandrew VM_ALLOC_NOOBJ | VM_ALLOC_WIRED | VM_ALLOC_ZERO)) == NULL) 1183281494Sandrew VM_WAIT; 1184281494Sandrew 1185281494Sandrew l1phys = VM_PAGE_TO_PHYS(l1pt); 1186281494Sandrew pmap->pm_l1 = (pd_entry_t *)PHYS_TO_DMAP(l1phys); 1187281494Sandrew 1188281494Sandrew if ((l1pt->flags & PG_ZERO) == 0) 1189281494Sandrew pagezero(pmap->pm_l1); 1190281494Sandrew 1191281494Sandrew bzero(&pmap->pm_stats, sizeof(pmap->pm_stats)); 1192281494Sandrew 1193281494Sandrew return (1); 1194281494Sandrew} 1195281494Sandrew 1196281494Sandrew/* 1197281494Sandrew * This routine is called if the desired page table page does not exist. 1198281494Sandrew * 1199281494Sandrew * If page table page allocation fails, this routine may sleep before 1200281494Sandrew * returning NULL. It sleeps only if a lock pointer was given. 1201281494Sandrew * 1202281494Sandrew * Note: If a page allocation fails at page table level two or three, 1203281494Sandrew * one or two pages may be held during the wait, only to be released 1204281494Sandrew * afterwards. This conservative approach is easily argued to avoid 1205281494Sandrew * race conditions. 1206281494Sandrew */ 1207281494Sandrewstatic vm_page_t 1208281494Sandrew_pmap_alloc_l3(pmap_t pmap, vm_pindex_t ptepindex, struct rwlock **lockp) 1209281494Sandrew{ 1210281494Sandrew vm_page_t m, /*pdppg, */pdpg; 1211281494Sandrew 1212281494Sandrew PMAP_LOCK_ASSERT(pmap, MA_OWNED); 1213281494Sandrew 1214281494Sandrew /* 1215281494Sandrew * Allocate a page table page. 1216281494Sandrew */ 1217281494Sandrew if ((m = vm_page_alloc(NULL, ptepindex, VM_ALLOC_NOOBJ | 1218281494Sandrew VM_ALLOC_WIRED | VM_ALLOC_ZERO)) == NULL) { 1219281494Sandrew if (lockp != NULL) { 1220281494Sandrew RELEASE_PV_LIST_LOCK(lockp); 1221281494Sandrew PMAP_UNLOCK(pmap); 1222281494Sandrew rw_runlock(&pvh_global_lock); 1223281494Sandrew VM_WAIT; 1224281494Sandrew rw_rlock(&pvh_global_lock); 1225281494Sandrew PMAP_LOCK(pmap); 1226281494Sandrew } 1227281494Sandrew 1228281494Sandrew /* 1229281494Sandrew * Indicate the need to retry. While waiting, the page table 1230281494Sandrew * page may have been allocated. 1231281494Sandrew */ 1232281494Sandrew return (NULL); 1233281494Sandrew } 1234281494Sandrew if ((m->flags & PG_ZERO) == 0) 1235281494Sandrew pmap_zero_page(m); 1236281494Sandrew 1237281494Sandrew /* 1238281494Sandrew * Map the pagetable page into the process address space, if 1239281494Sandrew * it isn't already there. 1240281494Sandrew */ 1241281494Sandrew 1242281494Sandrew if (ptepindex >= NUPDE) { 1243281494Sandrew pd_entry_t *l1; 1244281494Sandrew vm_pindex_t l1index; 1245281494Sandrew 1246281494Sandrew l1index = ptepindex - NUPDE; 1247281494Sandrew l1 = &pmap->pm_l1[l1index]; 1248281494Sandrew pmap_load_store(l1, VM_PAGE_TO_PHYS(m) | L1_TABLE); 1249281494Sandrew PTE_SYNC(l1); 1250281494Sandrew 1251281494Sandrew } else { 1252281494Sandrew vm_pindex_t l1index; 1253281494Sandrew pd_entry_t *l1, *l2; 1254281494Sandrew 1255281494Sandrew l1index = ptepindex >> (L1_SHIFT - L2_SHIFT); 1256281494Sandrew l1 = &pmap->pm_l1[l1index]; 1257285045Sandrew if (pmap_load(l1) == 0) { 1258281494Sandrew /* recurse for allocating page dir */ 1259281494Sandrew if (_pmap_alloc_l3(pmap, NUPDE + l1index, 1260281494Sandrew lockp) == NULL) { 1261281494Sandrew --m->wire_count; 1262281494Sandrew atomic_subtract_int(&vm_cnt.v_wire_count, 1); 1263281494Sandrew vm_page_free_zero(m); 1264281494Sandrew return (NULL); 1265281494Sandrew } 1266281494Sandrew } else { 1267288445Sandrew pdpg = PHYS_TO_VM_PAGE(pmap_load(l1) & ~ATTR_MASK); 1268281494Sandrew pdpg->wire_count++; 1269281494Sandrew } 1270281494Sandrew 1271288445Sandrew l2 = (pd_entry_t *)PHYS_TO_DMAP(pmap_load(l1) & ~ATTR_MASK); 1272281494Sandrew l2 = &l2[ptepindex & Ln_ADDR_MASK]; 1273285537Sandrew pmap_load_store(l2, VM_PAGE_TO_PHYS(m) | L2_TABLE); 1274281494Sandrew PTE_SYNC(l2); 1275281494Sandrew } 1276281494Sandrew 1277281494Sandrew pmap_resident_count_inc(pmap, 1); 1278281494Sandrew 1279281494Sandrew return (m); 1280281494Sandrew} 1281281494Sandrew 1282281494Sandrewstatic vm_page_t 1283281494Sandrewpmap_alloc_l3(pmap_t pmap, vm_offset_t va, struct rwlock **lockp) 1284281494Sandrew{ 1285281494Sandrew vm_pindex_t ptepindex; 1286281494Sandrew pd_entry_t *l2; 1287281494Sandrew vm_page_t m; 1288281494Sandrew 1289281494Sandrew /* 1290281494Sandrew * Calculate pagetable page index 1291281494Sandrew */ 1292281494Sandrew ptepindex = pmap_l2_pindex(va); 1293281494Sandrewretry: 1294281494Sandrew /* 1295281494Sandrew * Get the page directory entry 1296281494Sandrew */ 1297281494Sandrew l2 = pmap_l2(pmap, va); 1298281494Sandrew 1299281494Sandrew /* 1300281494Sandrew * If the page table page is mapped, we just increment the 1301281494Sandrew * hold count, and activate it. 1302281494Sandrew */ 1303285045Sandrew if (l2 != NULL && pmap_load(l2) != 0) { 1304285045Sandrew m = PHYS_TO_VM_PAGE(pmap_load(l2) & ~ATTR_MASK); 1305281494Sandrew m->wire_count++; 1306281494Sandrew } else { 1307281494Sandrew /* 1308281494Sandrew * Here if the pte page isn't mapped, or if it has been 1309281494Sandrew * deallocated. 1310281494Sandrew */ 1311281494Sandrew m = _pmap_alloc_l3(pmap, ptepindex, lockp); 1312281494Sandrew if (m == NULL && lockp != NULL) 1313281494Sandrew goto retry; 1314281494Sandrew } 1315281494Sandrew return (m); 1316281494Sandrew} 1317281494Sandrew 1318281494Sandrew 1319281494Sandrew/*************************************************** 1320281494Sandrew * Pmap allocation/deallocation routines. 1321281494Sandrew ***************************************************/ 1322281494Sandrew 1323281494Sandrew/* 1324281494Sandrew * Release any resources held by the given physical map. 1325281494Sandrew * Called when a pmap initialized by pmap_pinit is being released. 1326281494Sandrew * Should only be called if the map contains no valid mappings. 1327281494Sandrew */ 1328281494Sandrewvoid 1329281494Sandrewpmap_release(pmap_t pmap) 1330281494Sandrew{ 1331281494Sandrew vm_page_t m; 1332281494Sandrew 1333281494Sandrew KASSERT(pmap->pm_stats.resident_count == 0, 1334281494Sandrew ("pmap_release: pmap resident count %ld != 0", 1335281494Sandrew pmap->pm_stats.resident_count)); 1336281494Sandrew 1337281494Sandrew m = PHYS_TO_VM_PAGE(DMAP_TO_PHYS((vm_offset_t)pmap->pm_l1)); 1338281494Sandrew 1339281494Sandrew m->wire_count--; 1340281494Sandrew atomic_subtract_int(&vm_cnt.v_wire_count, 1); 1341281494Sandrew vm_page_free_zero(m); 1342281494Sandrew} 1343281494Sandrew 1344281494Sandrew#if 0 1345281494Sandrewstatic int 1346281494Sandrewkvm_size(SYSCTL_HANDLER_ARGS) 1347281494Sandrew{ 1348281494Sandrew unsigned long ksize = VM_MAX_KERNEL_ADDRESS - VM_MIN_KERNEL_ADDRESS; 1349281494Sandrew 1350281494Sandrew return sysctl_handle_long(oidp, &ksize, 0, req); 1351281494Sandrew} 1352281494SandrewSYSCTL_PROC(_vm, OID_AUTO, kvm_size, CTLTYPE_LONG|CTLFLAG_RD, 1353281494Sandrew 0, 0, kvm_size, "LU", "Size of KVM"); 1354281494Sandrew 1355281494Sandrewstatic int 1356281494Sandrewkvm_free(SYSCTL_HANDLER_ARGS) 1357281494Sandrew{ 1358281494Sandrew unsigned long kfree = VM_MAX_KERNEL_ADDRESS - kernel_vm_end; 1359281494Sandrew 1360281494Sandrew return sysctl_handle_long(oidp, &kfree, 0, req); 1361281494Sandrew} 1362281494SandrewSYSCTL_PROC(_vm, OID_AUTO, kvm_free, CTLTYPE_LONG|CTLFLAG_RD, 1363281494Sandrew 0, 0, kvm_free, "LU", "Amount of KVM free"); 1364281494Sandrew#endif /* 0 */ 1365281494Sandrew 1366281494Sandrew/* 1367281494Sandrew * grow the number of kernel page table entries, if needed 1368281494Sandrew */ 1369281494Sandrewvoid 1370281494Sandrewpmap_growkernel(vm_offset_t addr) 1371281494Sandrew{ 1372281494Sandrew vm_paddr_t paddr; 1373281494Sandrew vm_page_t nkpg; 1374281494Sandrew pd_entry_t *l1, *l2; 1375281494Sandrew 1376281494Sandrew mtx_assert(&kernel_map->system_mtx, MA_OWNED); 1377281494Sandrew 1378281494Sandrew addr = roundup2(addr, L2_SIZE); 1379281494Sandrew if (addr - 1 >= kernel_map->max_offset) 1380281494Sandrew addr = kernel_map->max_offset; 1381281494Sandrew while (kernel_vm_end < addr) { 1382281494Sandrew l1 = pmap_l1(kernel_pmap, kernel_vm_end); 1383285045Sandrew if (pmap_load(l1) == 0) { 1384281494Sandrew /* We need a new PDP entry */ 1385281494Sandrew nkpg = vm_page_alloc(NULL, kernel_vm_end >> L1_SHIFT, 1386281494Sandrew VM_ALLOC_INTERRUPT | VM_ALLOC_NOOBJ | 1387281494Sandrew VM_ALLOC_WIRED | VM_ALLOC_ZERO); 1388281494Sandrew if (nkpg == NULL) 1389281494Sandrew panic("pmap_growkernel: no memory to grow kernel"); 1390281494Sandrew if ((nkpg->flags & PG_ZERO) == 0) 1391281494Sandrew pmap_zero_page(nkpg); 1392281494Sandrew paddr = VM_PAGE_TO_PHYS(nkpg); 1393281494Sandrew pmap_load_store(l1, paddr | L1_TABLE); 1394281494Sandrew PTE_SYNC(l1); 1395281494Sandrew continue; /* try again */ 1396281494Sandrew } 1397281494Sandrew l2 = pmap_l1_to_l2(l1, kernel_vm_end); 1398285045Sandrew if ((pmap_load(l2) & ATTR_AF) != 0) { 1399281494Sandrew kernel_vm_end = (kernel_vm_end + L2_SIZE) & ~L2_OFFSET; 1400281494Sandrew if (kernel_vm_end - 1 >= kernel_map->max_offset) { 1401281494Sandrew kernel_vm_end = kernel_map->max_offset; 1402281494Sandrew break; 1403281494Sandrew } 1404281494Sandrew continue; 1405281494Sandrew } 1406281494Sandrew 1407281494Sandrew nkpg = vm_page_alloc(NULL, kernel_vm_end >> L2_SHIFT, 1408281494Sandrew VM_ALLOC_INTERRUPT | VM_ALLOC_NOOBJ | VM_ALLOC_WIRED | 1409281494Sandrew VM_ALLOC_ZERO); 1410281494Sandrew if (nkpg == NULL) 1411281494Sandrew panic("pmap_growkernel: no memory to grow kernel"); 1412281494Sandrew if ((nkpg->flags & PG_ZERO) == 0) 1413281494Sandrew pmap_zero_page(nkpg); 1414281494Sandrew paddr = VM_PAGE_TO_PHYS(nkpg); 1415281494Sandrew pmap_load_store(l2, paddr | L2_TABLE); 1416281494Sandrew PTE_SYNC(l2); 1417285212Sandrew pmap_invalidate_page(kernel_pmap, kernel_vm_end); 1418281494Sandrew 1419281494Sandrew kernel_vm_end = (kernel_vm_end + L2_SIZE) & ~L2_OFFSET; 1420281494Sandrew if (kernel_vm_end - 1 >= kernel_map->max_offset) { 1421281494Sandrew kernel_vm_end = kernel_map->max_offset; 1422281494Sandrew break; 1423281494Sandrew } 1424281494Sandrew } 1425281494Sandrew} 1426281494Sandrew 1427281494Sandrew 1428281494Sandrew/*************************************************** 1429281494Sandrew * page management routines. 1430281494Sandrew ***************************************************/ 1431281494Sandrew 1432281494SandrewCTASSERT(sizeof(struct pv_chunk) == PAGE_SIZE); 1433281494SandrewCTASSERT(_NPCM == 3); 1434281494SandrewCTASSERT(_NPCPV == 168); 1435281494Sandrew 1436281494Sandrewstatic __inline struct pv_chunk * 1437281494Sandrewpv_to_chunk(pv_entry_t pv) 1438281494Sandrew{ 1439281494Sandrew 1440281494Sandrew return ((struct pv_chunk *)((uintptr_t)pv & ~(uintptr_t)PAGE_MASK)); 1441281494Sandrew} 1442281494Sandrew 1443281494Sandrew#define PV_PMAP(pv) (pv_to_chunk(pv)->pc_pmap) 1444281494Sandrew 1445281494Sandrew#define PC_FREE0 0xfffffffffffffffful 1446281494Sandrew#define PC_FREE1 0xfffffffffffffffful 1447281494Sandrew#define PC_FREE2 0x000000fffffffffful 1448281494Sandrew 1449281494Sandrewstatic const uint64_t pc_freemask[_NPCM] = { PC_FREE0, PC_FREE1, PC_FREE2 }; 1450281494Sandrew 1451281494Sandrew#if 0 1452281494Sandrew#ifdef PV_STATS 1453281494Sandrewstatic int pc_chunk_count, pc_chunk_allocs, pc_chunk_frees, pc_chunk_tryfail; 1454281494Sandrew 1455281494SandrewSYSCTL_INT(_vm_pmap, OID_AUTO, pc_chunk_count, CTLFLAG_RD, &pc_chunk_count, 0, 1456281494Sandrew "Current number of pv entry chunks"); 1457281494SandrewSYSCTL_INT(_vm_pmap, OID_AUTO, pc_chunk_allocs, CTLFLAG_RD, &pc_chunk_allocs, 0, 1458281494Sandrew "Current number of pv entry chunks allocated"); 1459281494SandrewSYSCTL_INT(_vm_pmap, OID_AUTO, pc_chunk_frees, CTLFLAG_RD, &pc_chunk_frees, 0, 1460281494Sandrew "Current number of pv entry chunks frees"); 1461281494SandrewSYSCTL_INT(_vm_pmap, OID_AUTO, pc_chunk_tryfail, CTLFLAG_RD, &pc_chunk_tryfail, 0, 1462281494Sandrew "Number of times tried to get a chunk page but failed."); 1463281494Sandrew 1464281494Sandrewstatic long pv_entry_frees, pv_entry_allocs, pv_entry_count; 1465281494Sandrewstatic int pv_entry_spare; 1466281494Sandrew 1467281494SandrewSYSCTL_LONG(_vm_pmap, OID_AUTO, pv_entry_frees, CTLFLAG_RD, &pv_entry_frees, 0, 1468281494Sandrew "Current number of pv entry frees"); 1469281494SandrewSYSCTL_LONG(_vm_pmap, OID_AUTO, pv_entry_allocs, CTLFLAG_RD, &pv_entry_allocs, 0, 1470281494Sandrew "Current number of pv entry allocs"); 1471281494SandrewSYSCTL_LONG(_vm_pmap, OID_AUTO, pv_entry_count, CTLFLAG_RD, &pv_entry_count, 0, 1472281494Sandrew "Current number of pv entries"); 1473281494SandrewSYSCTL_INT(_vm_pmap, OID_AUTO, pv_entry_spare, CTLFLAG_RD, &pv_entry_spare, 0, 1474281494Sandrew "Current number of spare pv entries"); 1475281494Sandrew#endif 1476281494Sandrew#endif /* 0 */ 1477281494Sandrew 1478281494Sandrew/* 1479281494Sandrew * We are in a serious low memory condition. Resort to 1480281494Sandrew * drastic measures to free some pages so we can allocate 1481281494Sandrew * another pv entry chunk. 1482281494Sandrew * 1483281494Sandrew * Returns NULL if PV entries were reclaimed from the specified pmap. 1484281494Sandrew * 1485281494Sandrew * We do not, however, unmap 2mpages because subsequent accesses will 1486281494Sandrew * allocate per-page pv entries until repromotion occurs, thereby 1487281494Sandrew * exacerbating the shortage of free pv entries. 1488281494Sandrew */ 1489281494Sandrewstatic vm_page_t 1490281494Sandrewreclaim_pv_chunk(pmap_t locked_pmap, struct rwlock **lockp) 1491281494Sandrew{ 1492281494Sandrew 1493286073Semaste panic("ARM64TODO: reclaim_pv_chunk"); 1494281494Sandrew} 1495281494Sandrew 1496281494Sandrew/* 1497281494Sandrew * free the pv_entry back to the free list 1498281494Sandrew */ 1499281494Sandrewstatic void 1500281494Sandrewfree_pv_entry(pmap_t pmap, pv_entry_t pv) 1501281494Sandrew{ 1502281494Sandrew struct pv_chunk *pc; 1503281494Sandrew int idx, field, bit; 1504281494Sandrew 1505281494Sandrew rw_assert(&pvh_global_lock, RA_LOCKED); 1506281494Sandrew PMAP_LOCK_ASSERT(pmap, MA_OWNED); 1507281494Sandrew PV_STAT(atomic_add_long(&pv_entry_frees, 1)); 1508281494Sandrew PV_STAT(atomic_add_int(&pv_entry_spare, 1)); 1509281494Sandrew PV_STAT(atomic_subtract_long(&pv_entry_count, 1)); 1510281494Sandrew pc = pv_to_chunk(pv); 1511281494Sandrew idx = pv - &pc->pc_pventry[0]; 1512281494Sandrew field = idx / 64; 1513281494Sandrew bit = idx % 64; 1514281494Sandrew pc->pc_map[field] |= 1ul << bit; 1515281494Sandrew if (pc->pc_map[0] != PC_FREE0 || pc->pc_map[1] != PC_FREE1 || 1516281494Sandrew pc->pc_map[2] != PC_FREE2) { 1517281494Sandrew /* 98% of the time, pc is already at the head of the list. */ 1518281494Sandrew if (__predict_false(pc != TAILQ_FIRST(&pmap->pm_pvchunk))) { 1519281494Sandrew TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list); 1520281494Sandrew TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc, pc_list); 1521281494Sandrew } 1522281494Sandrew return; 1523281494Sandrew } 1524281494Sandrew TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list); 1525281494Sandrew free_pv_chunk(pc); 1526281494Sandrew} 1527281494Sandrew 1528281494Sandrewstatic void 1529281494Sandrewfree_pv_chunk(struct pv_chunk *pc) 1530281494Sandrew{ 1531281494Sandrew vm_page_t m; 1532281494Sandrew 1533281494Sandrew mtx_lock(&pv_chunks_mutex); 1534281494Sandrew TAILQ_REMOVE(&pv_chunks, pc, pc_lru); 1535281494Sandrew mtx_unlock(&pv_chunks_mutex); 1536281494Sandrew PV_STAT(atomic_subtract_int(&pv_entry_spare, _NPCPV)); 1537281494Sandrew PV_STAT(atomic_subtract_int(&pc_chunk_count, 1)); 1538281494Sandrew PV_STAT(atomic_add_int(&pc_chunk_frees, 1)); 1539281494Sandrew /* entire chunk is free, return it */ 1540281494Sandrew m = PHYS_TO_VM_PAGE(DMAP_TO_PHYS((vm_offset_t)pc)); 1541281494Sandrew dump_drop_page(m->phys_addr); 1542288256Salc vm_page_unwire(m, PQ_NONE); 1543281494Sandrew vm_page_free(m); 1544281494Sandrew} 1545281494Sandrew 1546281494Sandrew/* 1547281494Sandrew * Returns a new PV entry, allocating a new PV chunk from the system when 1548281494Sandrew * needed. If this PV chunk allocation fails and a PV list lock pointer was 1549281494Sandrew * given, a PV chunk is reclaimed from an arbitrary pmap. Otherwise, NULL is 1550281494Sandrew * returned. 1551281494Sandrew * 1552281494Sandrew * The given PV list lock may be released. 1553281494Sandrew */ 1554281494Sandrewstatic pv_entry_t 1555281494Sandrewget_pv_entry(pmap_t pmap, struct rwlock **lockp) 1556281494Sandrew{ 1557281494Sandrew int bit, field; 1558281494Sandrew pv_entry_t pv; 1559281494Sandrew struct pv_chunk *pc; 1560281494Sandrew vm_page_t m; 1561281494Sandrew 1562281494Sandrew rw_assert(&pvh_global_lock, RA_LOCKED); 1563281494Sandrew PMAP_LOCK_ASSERT(pmap, MA_OWNED); 1564281494Sandrew PV_STAT(atomic_add_long(&pv_entry_allocs, 1)); 1565281494Sandrewretry: 1566281494Sandrew pc = TAILQ_FIRST(&pmap->pm_pvchunk); 1567281494Sandrew if (pc != NULL) { 1568281494Sandrew for (field = 0; field < _NPCM; field++) { 1569281494Sandrew if (pc->pc_map[field]) { 1570281494Sandrew bit = ffsl(pc->pc_map[field]) - 1; 1571281494Sandrew break; 1572281494Sandrew } 1573281494Sandrew } 1574281494Sandrew if (field < _NPCM) { 1575281494Sandrew pv = &pc->pc_pventry[field * 64 + bit]; 1576281494Sandrew pc->pc_map[field] &= ~(1ul << bit); 1577281494Sandrew /* If this was the last item, move it to tail */ 1578281494Sandrew if (pc->pc_map[0] == 0 && pc->pc_map[1] == 0 && 1579281494Sandrew pc->pc_map[2] == 0) { 1580281494Sandrew TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list); 1581281494Sandrew TAILQ_INSERT_TAIL(&pmap->pm_pvchunk, pc, 1582281494Sandrew pc_list); 1583281494Sandrew } 1584281494Sandrew PV_STAT(atomic_add_long(&pv_entry_count, 1)); 1585281494Sandrew PV_STAT(atomic_subtract_int(&pv_entry_spare, 1)); 1586281494Sandrew return (pv); 1587281494Sandrew } 1588281494Sandrew } 1589281494Sandrew /* No free items, allocate another chunk */ 1590281494Sandrew m = vm_page_alloc(NULL, 0, VM_ALLOC_NORMAL | VM_ALLOC_NOOBJ | 1591281494Sandrew VM_ALLOC_WIRED); 1592281494Sandrew if (m == NULL) { 1593281494Sandrew if (lockp == NULL) { 1594281494Sandrew PV_STAT(pc_chunk_tryfail++); 1595281494Sandrew return (NULL); 1596281494Sandrew } 1597281494Sandrew m = reclaim_pv_chunk(pmap, lockp); 1598281494Sandrew if (m == NULL) 1599281494Sandrew goto retry; 1600281494Sandrew } 1601281494Sandrew PV_STAT(atomic_add_int(&pc_chunk_count, 1)); 1602281494Sandrew PV_STAT(atomic_add_int(&pc_chunk_allocs, 1)); 1603281494Sandrew dump_add_page(m->phys_addr); 1604281494Sandrew pc = (void *)PHYS_TO_DMAP(m->phys_addr); 1605281494Sandrew pc->pc_pmap = pmap; 1606281494Sandrew pc->pc_map[0] = PC_FREE0 & ~1ul; /* preallocated bit 0 */ 1607281494Sandrew pc->pc_map[1] = PC_FREE1; 1608281494Sandrew pc->pc_map[2] = PC_FREE2; 1609281494Sandrew mtx_lock(&pv_chunks_mutex); 1610281494Sandrew TAILQ_INSERT_TAIL(&pv_chunks, pc, pc_lru); 1611281494Sandrew mtx_unlock(&pv_chunks_mutex); 1612281494Sandrew pv = &pc->pc_pventry[0]; 1613281494Sandrew TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc, pc_list); 1614281494Sandrew PV_STAT(atomic_add_long(&pv_entry_count, 1)); 1615281494Sandrew PV_STAT(atomic_add_int(&pv_entry_spare, _NPCPV - 1)); 1616281494Sandrew return (pv); 1617281494Sandrew} 1618281494Sandrew 1619281494Sandrew/* 1620281494Sandrew * First find and then remove the pv entry for the specified pmap and virtual 1621281494Sandrew * address from the specified pv list. Returns the pv entry if found and NULL 1622281494Sandrew * otherwise. This operation can be performed on pv lists for either 4KB or 1623281494Sandrew * 2MB page mappings. 1624281494Sandrew */ 1625281494Sandrewstatic __inline pv_entry_t 1626281494Sandrewpmap_pvh_remove(struct md_page *pvh, pmap_t pmap, vm_offset_t va) 1627281494Sandrew{ 1628281494Sandrew pv_entry_t pv; 1629281494Sandrew 1630281494Sandrew rw_assert(&pvh_global_lock, RA_LOCKED); 1631281494Sandrew TAILQ_FOREACH(pv, &pvh->pv_list, pv_next) { 1632281494Sandrew if (pmap == PV_PMAP(pv) && va == pv->pv_va) { 1633281494Sandrew TAILQ_REMOVE(&pvh->pv_list, pv, pv_next); 1634281494Sandrew pvh->pv_gen++; 1635281494Sandrew break; 1636281494Sandrew } 1637281494Sandrew } 1638281494Sandrew return (pv); 1639281494Sandrew} 1640281494Sandrew 1641281494Sandrew/* 1642281494Sandrew * First find and then destroy the pv entry for the specified pmap and virtual 1643281494Sandrew * address. This operation can be performed on pv lists for either 4KB or 2MB 1644281494Sandrew * page mappings. 1645281494Sandrew */ 1646281494Sandrewstatic void 1647281494Sandrewpmap_pvh_free(struct md_page *pvh, pmap_t pmap, vm_offset_t va) 1648281494Sandrew{ 1649281494Sandrew pv_entry_t pv; 1650281494Sandrew 1651281494Sandrew pv = pmap_pvh_remove(pvh, pmap, va); 1652281494Sandrew KASSERT(pv != NULL, ("pmap_pvh_free: pv not found")); 1653281494Sandrew free_pv_entry(pmap, pv); 1654281494Sandrew} 1655281494Sandrew 1656281494Sandrew/* 1657281494Sandrew * Conditionally create the PV entry for a 4KB page mapping if the required 1658281494Sandrew * memory can be allocated without resorting to reclamation. 1659281494Sandrew */ 1660281494Sandrewstatic boolean_t 1661281494Sandrewpmap_try_insert_pv_entry(pmap_t pmap, vm_offset_t va, vm_page_t m, 1662281494Sandrew struct rwlock **lockp) 1663281494Sandrew{ 1664281494Sandrew pv_entry_t pv; 1665281494Sandrew 1666281494Sandrew rw_assert(&pvh_global_lock, RA_LOCKED); 1667281494Sandrew PMAP_LOCK_ASSERT(pmap, MA_OWNED); 1668281494Sandrew /* Pass NULL instead of the lock pointer to disable reclamation. */ 1669281494Sandrew if ((pv = get_pv_entry(pmap, NULL)) != NULL) { 1670281494Sandrew pv->pv_va = va; 1671281494Sandrew CHANGE_PV_LIST_LOCK_TO_VM_PAGE(lockp, m); 1672281494Sandrew TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_next); 1673281494Sandrew m->md.pv_gen++; 1674281494Sandrew return (TRUE); 1675281494Sandrew } else 1676281494Sandrew return (FALSE); 1677281494Sandrew} 1678281494Sandrew 1679281494Sandrew/* 1680281494Sandrew * pmap_remove_l3: do the things to unmap a page in a process 1681281494Sandrew */ 1682281494Sandrewstatic int 1683281494Sandrewpmap_remove_l3(pmap_t pmap, pt_entry_t *l3, vm_offset_t va, 1684281494Sandrew pd_entry_t l2e, struct spglist *free, struct rwlock **lockp) 1685281494Sandrew{ 1686281494Sandrew pt_entry_t old_l3; 1687281494Sandrew vm_page_t m; 1688281494Sandrew 1689281494Sandrew PMAP_LOCK_ASSERT(pmap, MA_OWNED); 1690281494Sandrew if (pmap_is_current(pmap) && pmap_l3_valid_cacheable(pmap_load(l3))) 1691281494Sandrew cpu_dcache_wb_range(va, L3_SIZE); 1692281494Sandrew old_l3 = pmap_load_clear(l3); 1693281494Sandrew PTE_SYNC(l3); 1694285212Sandrew pmap_invalidate_page(pmap, va); 1695281494Sandrew if (old_l3 & ATTR_SW_WIRED) 1696281494Sandrew pmap->pm_stats.wired_count -= 1; 1697281494Sandrew pmap_resident_count_dec(pmap, 1); 1698281494Sandrew if (old_l3 & ATTR_SW_MANAGED) { 1699281494Sandrew m = PHYS_TO_VM_PAGE(old_l3 & ~ATTR_MASK); 1700281494Sandrew if (pmap_page_dirty(old_l3)) 1701281494Sandrew vm_page_dirty(m); 1702281494Sandrew if (old_l3 & ATTR_AF) 1703281494Sandrew vm_page_aflag_set(m, PGA_REFERENCED); 1704281494Sandrew CHANGE_PV_LIST_LOCK_TO_VM_PAGE(lockp, m); 1705281494Sandrew pmap_pvh_free(&m->md, pmap, va); 1706281494Sandrew } 1707281494Sandrew return (pmap_unuse_l3(pmap, va, l2e, free)); 1708281494Sandrew} 1709281494Sandrew 1710281494Sandrew/* 1711281494Sandrew * Remove the given range of addresses from the specified map. 1712281494Sandrew * 1713281494Sandrew * It is assumed that the start and end are properly 1714281494Sandrew * rounded to the page size. 1715281494Sandrew */ 1716281494Sandrewvoid 1717281494Sandrewpmap_remove(pmap_t pmap, vm_offset_t sva, vm_offset_t eva) 1718281494Sandrew{ 1719281494Sandrew struct rwlock *lock; 1720281494Sandrew vm_offset_t va, va_next; 1721281494Sandrew pd_entry_t *l1, *l2; 1722281494Sandrew pt_entry_t l3_paddr, *l3; 1723281494Sandrew struct spglist free; 1724281494Sandrew int anyvalid; 1725281494Sandrew 1726281494Sandrew /* 1727281494Sandrew * Perform an unsynchronized read. This is, however, safe. 1728281494Sandrew */ 1729281494Sandrew if (pmap->pm_stats.resident_count == 0) 1730281494Sandrew return; 1731281494Sandrew 1732281494Sandrew anyvalid = 0; 1733281494Sandrew SLIST_INIT(&free); 1734281494Sandrew 1735281494Sandrew rw_rlock(&pvh_global_lock); 1736281494Sandrew PMAP_LOCK(pmap); 1737281494Sandrew 1738281494Sandrew lock = NULL; 1739281494Sandrew for (; sva < eva; sva = va_next) { 1740281494Sandrew 1741281494Sandrew if (pmap->pm_stats.resident_count == 0) 1742281494Sandrew break; 1743281494Sandrew 1744281494Sandrew l1 = pmap_l1(pmap, sva); 1745285045Sandrew if (pmap_load(l1) == 0) { 1746281494Sandrew va_next = (sva + L1_SIZE) & ~L1_OFFSET; 1747281494Sandrew if (va_next < sva) 1748281494Sandrew va_next = eva; 1749281494Sandrew continue; 1750281494Sandrew } 1751281494Sandrew 1752281494Sandrew /* 1753281494Sandrew * Calculate index for next page table. 1754281494Sandrew */ 1755281494Sandrew va_next = (sva + L2_SIZE) & ~L2_OFFSET; 1756281494Sandrew if (va_next < sva) 1757281494Sandrew va_next = eva; 1758281494Sandrew 1759281494Sandrew l2 = pmap_l1_to_l2(l1, sva); 1760281494Sandrew if (l2 == NULL) 1761281494Sandrew continue; 1762281494Sandrew 1763288445Sandrew l3_paddr = pmap_load(l2); 1764281494Sandrew 1765281494Sandrew /* 1766281494Sandrew * Weed out invalid mappings. 1767281494Sandrew */ 1768281494Sandrew if ((l3_paddr & ATTR_DESCR_MASK) != L2_TABLE) 1769281494Sandrew continue; 1770281494Sandrew 1771281494Sandrew /* 1772281494Sandrew * Limit our scan to either the end of the va represented 1773281494Sandrew * by the current page table page, or to the end of the 1774281494Sandrew * range being removed. 1775281494Sandrew */ 1776281494Sandrew if (va_next > eva) 1777281494Sandrew va_next = eva; 1778281494Sandrew 1779281494Sandrew va = va_next; 1780281494Sandrew for (l3 = pmap_l2_to_l3(l2, sva); sva != va_next; l3++, 1781281494Sandrew sva += L3_SIZE) { 1782281494Sandrew if (l3 == NULL) 1783281494Sandrew panic("l3 == NULL"); 1784285045Sandrew if (pmap_load(l3) == 0) { 1785281494Sandrew if (va != va_next) { 1786281494Sandrew pmap_invalidate_range(pmap, va, sva); 1787281494Sandrew va = va_next; 1788281494Sandrew } 1789281494Sandrew continue; 1790281494Sandrew } 1791281494Sandrew if (va == va_next) 1792281494Sandrew va = sva; 1793281494Sandrew if (pmap_remove_l3(pmap, l3, sva, l3_paddr, &free, 1794281494Sandrew &lock)) { 1795281494Sandrew sva += L3_SIZE; 1796281494Sandrew break; 1797281494Sandrew } 1798281494Sandrew } 1799281494Sandrew if (va != va_next) 1800281494Sandrew pmap_invalidate_range(pmap, va, sva); 1801281494Sandrew } 1802281494Sandrew if (lock != NULL) 1803281494Sandrew rw_wunlock(lock); 1804281494Sandrew if (anyvalid) 1805281494Sandrew pmap_invalidate_all(pmap); 1806281494Sandrew rw_runlock(&pvh_global_lock); 1807281494Sandrew PMAP_UNLOCK(pmap); 1808281494Sandrew pmap_free_zero_pages(&free); 1809281494Sandrew} 1810281494Sandrew 1811281494Sandrew/* 1812281494Sandrew * Routine: pmap_remove_all 1813281494Sandrew * Function: 1814281494Sandrew * Removes this physical page from 1815281494Sandrew * all physical maps in which it resides. 1816281494Sandrew * Reflects back modify bits to the pager. 1817281494Sandrew * 1818281494Sandrew * Notes: 1819281494Sandrew * Original versions of this routine were very 1820281494Sandrew * inefficient because they iteratively called 1821281494Sandrew * pmap_remove (slow...) 1822281494Sandrew */ 1823281494Sandrew 1824281494Sandrewvoid 1825281494Sandrewpmap_remove_all(vm_page_t m) 1826281494Sandrew{ 1827281494Sandrew pv_entry_t pv; 1828281494Sandrew pmap_t pmap; 1829281494Sandrew pt_entry_t *l3, tl3; 1830288445Sandrew pd_entry_t *l2, tl2; 1831281494Sandrew struct spglist free; 1832281494Sandrew 1833281494Sandrew KASSERT((m->oflags & VPO_UNMANAGED) == 0, 1834281494Sandrew ("pmap_remove_all: page %p is not managed", m)); 1835281494Sandrew SLIST_INIT(&free); 1836281494Sandrew rw_wlock(&pvh_global_lock); 1837281494Sandrew while ((pv = TAILQ_FIRST(&m->md.pv_list)) != NULL) { 1838281494Sandrew pmap = PV_PMAP(pv); 1839281494Sandrew PMAP_LOCK(pmap); 1840281494Sandrew pmap_resident_count_dec(pmap, 1); 1841281494Sandrew l2 = pmap_l2(pmap, pv->pv_va); 1842288445Sandrew KASSERT(l2 != NULL, ("pmap_remove_all: no l2 table found")); 1843288445Sandrew tl2 = pmap_load(l2); 1844288445Sandrew KASSERT((tl2 & ATTR_DESCR_MASK) == L2_TABLE, 1845281494Sandrew ("pmap_remove_all: found a table when expecting " 1846281494Sandrew "a block in %p's pv list", m)); 1847281494Sandrew l3 = pmap_l2_to_l3(l2, pv->pv_va); 1848281494Sandrew if (pmap_is_current(pmap) && 1849281494Sandrew pmap_l3_valid_cacheable(pmap_load(l3))) 1850281494Sandrew cpu_dcache_wb_range(pv->pv_va, L3_SIZE); 1851281494Sandrew tl3 = pmap_load_clear(l3); 1852281494Sandrew PTE_SYNC(l3); 1853285212Sandrew pmap_invalidate_page(pmap, pv->pv_va); 1854281494Sandrew if (tl3 & ATTR_SW_WIRED) 1855281494Sandrew pmap->pm_stats.wired_count--; 1856281494Sandrew if ((tl3 & ATTR_AF) != 0) 1857281494Sandrew vm_page_aflag_set(m, PGA_REFERENCED); 1858281494Sandrew 1859281494Sandrew /* 1860281494Sandrew * Update the vm_page_t clean and reference bits. 1861281494Sandrew */ 1862281494Sandrew if (pmap_page_dirty(tl3)) 1863281494Sandrew vm_page_dirty(m); 1864288445Sandrew pmap_unuse_l3(pmap, pv->pv_va, tl2, &free); 1865281494Sandrew TAILQ_REMOVE(&m->md.pv_list, pv, pv_next); 1866281494Sandrew m->md.pv_gen++; 1867281494Sandrew free_pv_entry(pmap, pv); 1868281494Sandrew PMAP_UNLOCK(pmap); 1869281494Sandrew } 1870281494Sandrew vm_page_aflag_clear(m, PGA_WRITEABLE); 1871281494Sandrew rw_wunlock(&pvh_global_lock); 1872281494Sandrew pmap_free_zero_pages(&free); 1873281494Sandrew} 1874281494Sandrew 1875281494Sandrew/* 1876281494Sandrew * Set the physical protection on the 1877281494Sandrew * specified range of this map as requested. 1878281494Sandrew */ 1879281494Sandrewvoid 1880281494Sandrewpmap_protect(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, vm_prot_t prot) 1881281494Sandrew{ 1882281494Sandrew vm_offset_t va, va_next; 1883281494Sandrew pd_entry_t *l1, *l2; 1884281494Sandrew pt_entry_t *l3p, l3; 1885281494Sandrew 1886281494Sandrew if ((prot & VM_PROT_READ) == VM_PROT_NONE) { 1887281494Sandrew pmap_remove(pmap, sva, eva); 1888281494Sandrew return; 1889281494Sandrew } 1890281494Sandrew 1891281494Sandrew if ((prot & VM_PROT_WRITE) == VM_PROT_WRITE) 1892281494Sandrew return; 1893281494Sandrew 1894281494Sandrew PMAP_LOCK(pmap); 1895281494Sandrew for (; sva < eva; sva = va_next) { 1896281494Sandrew 1897281494Sandrew l1 = pmap_l1(pmap, sva); 1898285045Sandrew if (pmap_load(l1) == 0) { 1899281494Sandrew va_next = (sva + L1_SIZE) & ~L1_OFFSET; 1900281494Sandrew if (va_next < sva) 1901281494Sandrew va_next = eva; 1902281494Sandrew continue; 1903281494Sandrew } 1904281494Sandrew 1905281494Sandrew va_next = (sva + L2_SIZE) & ~L2_OFFSET; 1906281494Sandrew if (va_next < sva) 1907281494Sandrew va_next = eva; 1908281494Sandrew 1909281494Sandrew l2 = pmap_l1_to_l2(l1, sva); 1910288445Sandrew if (l2 == NULL || (pmap_load(l2) & ATTR_DESCR_MASK) != L2_TABLE) 1911281494Sandrew continue; 1912281494Sandrew 1913281494Sandrew if (va_next > eva) 1914281494Sandrew va_next = eva; 1915281494Sandrew 1916281494Sandrew va = va_next; 1917281494Sandrew for (l3p = pmap_l2_to_l3(l2, sva); sva != va_next; l3p++, 1918281494Sandrew sva += L3_SIZE) { 1919281494Sandrew l3 = pmap_load(l3p); 1920281494Sandrew if (pmap_l3_valid(l3)) { 1921281494Sandrew pmap_set(l3p, ATTR_AP(ATTR_AP_RO)); 1922281494Sandrew PTE_SYNC(l3p); 1923285212Sandrew /* XXX: Use pmap_invalidate_range */ 1924285212Sandrew pmap_invalidate_page(pmap, va); 1925281494Sandrew } 1926281494Sandrew } 1927281494Sandrew } 1928281494Sandrew PMAP_UNLOCK(pmap); 1929281494Sandrew 1930281494Sandrew /* TODO: Only invalidate entries we are touching */ 1931281494Sandrew pmap_invalidate_all(pmap); 1932281494Sandrew} 1933281494Sandrew 1934281494Sandrew/* 1935281494Sandrew * Insert the given physical page (p) at 1936281494Sandrew * the specified virtual address (v) in the 1937281494Sandrew * target physical map with the protection requested. 1938281494Sandrew * 1939281494Sandrew * If specified, the page will be wired down, meaning 1940281494Sandrew * that the related pte can not be reclaimed. 1941281494Sandrew * 1942281494Sandrew * NB: This is the only routine which MAY NOT lazy-evaluate 1943281494Sandrew * or lose information. That is, this routine must actually 1944281494Sandrew * insert this page into the given map NOW. 1945281494Sandrew */ 1946281494Sandrewint 1947281494Sandrewpmap_enter(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot, 1948281494Sandrew u_int flags, int8_t psind __unused) 1949281494Sandrew{ 1950281494Sandrew struct rwlock *lock; 1951281494Sandrew pd_entry_t *l1, *l2; 1952281494Sandrew pt_entry_t new_l3, orig_l3; 1953281494Sandrew pt_entry_t *l3; 1954281494Sandrew pv_entry_t pv; 1955281494Sandrew vm_paddr_t opa, pa, l2_pa, l3_pa; 1956281494Sandrew vm_page_t mpte, om, l2_m, l3_m; 1957281494Sandrew boolean_t nosleep; 1958281494Sandrew 1959281494Sandrew va = trunc_page(va); 1960281494Sandrew if ((m->oflags & VPO_UNMANAGED) == 0 && !vm_page_xbusied(m)) 1961281494Sandrew VM_OBJECT_ASSERT_LOCKED(m->object); 1962281494Sandrew pa = VM_PAGE_TO_PHYS(m); 1963285537Sandrew new_l3 = (pt_entry_t)(pa | ATTR_DEFAULT | ATTR_IDX(m->md.pv_memattr) | 1964285537Sandrew L3_PAGE); 1965281494Sandrew if ((prot & VM_PROT_WRITE) == 0) 1966281494Sandrew new_l3 |= ATTR_AP(ATTR_AP_RO); 1967281494Sandrew if ((flags & PMAP_ENTER_WIRED) != 0) 1968281494Sandrew new_l3 |= ATTR_SW_WIRED; 1969281494Sandrew if ((va >> 63) == 0) 1970281494Sandrew new_l3 |= ATTR_AP(ATTR_AP_USER); 1971281494Sandrew 1972285212Sandrew CTR2(KTR_PMAP, "pmap_enter: %.16lx -> %.16lx", va, pa); 1973285212Sandrew 1974281494Sandrew mpte = NULL; 1975281494Sandrew 1976281494Sandrew lock = NULL; 1977281494Sandrew rw_rlock(&pvh_global_lock); 1978281494Sandrew PMAP_LOCK(pmap); 1979281494Sandrew 1980281494Sandrew if (va < VM_MAXUSER_ADDRESS) { 1981281494Sandrew nosleep = (flags & PMAP_ENTER_NOSLEEP) != 0; 1982281494Sandrew mpte = pmap_alloc_l3(pmap, va, nosleep ? NULL : &lock); 1983281494Sandrew if (mpte == NULL && nosleep) { 1984285212Sandrew CTR0(KTR_PMAP, "pmap_enter: mpte == NULL"); 1985281494Sandrew if (lock != NULL) 1986281494Sandrew rw_wunlock(lock); 1987281494Sandrew rw_runlock(&pvh_global_lock); 1988281494Sandrew PMAP_UNLOCK(pmap); 1989281494Sandrew return (KERN_RESOURCE_SHORTAGE); 1990281494Sandrew } 1991281494Sandrew l3 = pmap_l3(pmap, va); 1992281494Sandrew } else { 1993281494Sandrew l3 = pmap_l3(pmap, va); 1994281494Sandrew /* TODO: This is not optimal, but should mostly work */ 1995281494Sandrew if (l3 == NULL) { 1996281494Sandrew l2 = pmap_l2(pmap, va); 1997281494Sandrew 1998281494Sandrew if (l2 == NULL) { 1999281494Sandrew l2_m = vm_page_alloc(NULL, 0, VM_ALLOC_NORMAL | 2000281494Sandrew VM_ALLOC_NOOBJ | VM_ALLOC_WIRED | 2001281494Sandrew VM_ALLOC_ZERO); 2002281494Sandrew if (l2_m == NULL) 2003281494Sandrew panic("pmap_enter: l2 pte_m == NULL"); 2004281494Sandrew if ((l2_m->flags & PG_ZERO) == 0) 2005281494Sandrew pmap_zero_page(l2_m); 2006281494Sandrew 2007281494Sandrew l2_pa = VM_PAGE_TO_PHYS(l2_m); 2008281494Sandrew l1 = pmap_l1(pmap, va); 2009281494Sandrew pmap_load_store(l1, l2_pa | L1_TABLE); 2010281494Sandrew PTE_SYNC(l1); 2011281494Sandrew l2 = pmap_l1_to_l2(l1, va); 2012281494Sandrew } 2013281494Sandrew 2014281494Sandrew KASSERT(l2 != NULL, 2015281494Sandrew ("No l2 table after allocating one")); 2016281494Sandrew 2017281494Sandrew l3_m = vm_page_alloc(NULL, 0, VM_ALLOC_NORMAL | 2018281494Sandrew VM_ALLOC_NOOBJ | VM_ALLOC_WIRED | VM_ALLOC_ZERO); 2019281494Sandrew if (l3_m == NULL) 2020281494Sandrew panic("pmap_enter: l3 pte_m == NULL"); 2021281494Sandrew if ((l3_m->flags & PG_ZERO) == 0) 2022281494Sandrew pmap_zero_page(l3_m); 2023281494Sandrew 2024281494Sandrew l3_pa = VM_PAGE_TO_PHYS(l3_m); 2025281494Sandrew pmap_load_store(l2, l3_pa | L2_TABLE); 2026281494Sandrew PTE_SYNC(l2); 2027281494Sandrew l3 = pmap_l2_to_l3(l2, va); 2028281494Sandrew } 2029285212Sandrew pmap_invalidate_page(pmap, va); 2030281494Sandrew } 2031281494Sandrew 2032281494Sandrew om = NULL; 2033281494Sandrew orig_l3 = pmap_load(l3); 2034281494Sandrew opa = orig_l3 & ~ATTR_MASK; 2035281494Sandrew 2036281494Sandrew /* 2037281494Sandrew * Is the specified virtual address already mapped? 2038281494Sandrew */ 2039281494Sandrew if (pmap_l3_valid(orig_l3)) { 2040281494Sandrew /* 2041281494Sandrew * Wiring change, just update stats. We don't worry about 2042281494Sandrew * wiring PT pages as they remain resident as long as there 2043281494Sandrew * are valid mappings in them. Hence, if a user page is wired, 2044281494Sandrew * the PT page will be also. 2045281494Sandrew */ 2046281494Sandrew if ((flags & PMAP_ENTER_WIRED) != 0 && 2047281494Sandrew (orig_l3 & ATTR_SW_WIRED) == 0) 2048281494Sandrew pmap->pm_stats.wired_count++; 2049281494Sandrew else if ((flags & PMAP_ENTER_WIRED) == 0 && 2050281494Sandrew (orig_l3 & ATTR_SW_WIRED) != 0) 2051281494Sandrew pmap->pm_stats.wired_count--; 2052281494Sandrew 2053281494Sandrew /* 2054281494Sandrew * Remove the extra PT page reference. 2055281494Sandrew */ 2056281494Sandrew if (mpte != NULL) { 2057281494Sandrew mpte->wire_count--; 2058281494Sandrew KASSERT(mpte->wire_count > 0, 2059281494Sandrew ("pmap_enter: missing reference to page table page," 2060281494Sandrew " va: 0x%lx", va)); 2061281494Sandrew } 2062281494Sandrew 2063281494Sandrew /* 2064281494Sandrew * Has the physical page changed? 2065281494Sandrew */ 2066281494Sandrew if (opa == pa) { 2067281494Sandrew /* 2068281494Sandrew * No, might be a protection or wiring change. 2069281494Sandrew */ 2070281494Sandrew if ((orig_l3 & ATTR_SW_MANAGED) != 0) { 2071281494Sandrew new_l3 |= ATTR_SW_MANAGED; 2072281494Sandrew if ((new_l3 & ATTR_AP(ATTR_AP_RW)) == 2073281494Sandrew ATTR_AP(ATTR_AP_RW)) { 2074281494Sandrew vm_page_aflag_set(m, PGA_WRITEABLE); 2075281494Sandrew } 2076281494Sandrew } 2077281494Sandrew goto validate; 2078281494Sandrew } 2079281494Sandrew 2080281494Sandrew /* Flush the cache, there might be uncommitted data in it */ 2081281494Sandrew if (pmap_is_current(pmap) && pmap_l3_valid_cacheable(orig_l3)) 2082281494Sandrew cpu_dcache_wb_range(va, L3_SIZE); 2083281494Sandrew } else { 2084281494Sandrew /* 2085281494Sandrew * Increment the counters. 2086281494Sandrew */ 2087281494Sandrew if ((new_l3 & ATTR_SW_WIRED) != 0) 2088281494Sandrew pmap->pm_stats.wired_count++; 2089281494Sandrew pmap_resident_count_inc(pmap, 1); 2090281494Sandrew } 2091281494Sandrew /* 2092281494Sandrew * Enter on the PV list if part of our managed memory. 2093281494Sandrew */ 2094281494Sandrew if ((m->oflags & VPO_UNMANAGED) == 0) { 2095281494Sandrew new_l3 |= ATTR_SW_MANAGED; 2096281494Sandrew pv = get_pv_entry(pmap, &lock); 2097281494Sandrew pv->pv_va = va; 2098281494Sandrew CHANGE_PV_LIST_LOCK_TO_PHYS(&lock, pa); 2099281494Sandrew TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_next); 2100281494Sandrew m->md.pv_gen++; 2101281494Sandrew if ((new_l3 & ATTR_AP_RW_BIT) == ATTR_AP(ATTR_AP_RW)) 2102281494Sandrew vm_page_aflag_set(m, PGA_WRITEABLE); 2103281494Sandrew } 2104281494Sandrew 2105281494Sandrew /* 2106281494Sandrew * Update the L3 entry. 2107281494Sandrew */ 2108281494Sandrew if (orig_l3 != 0) { 2109281494Sandrewvalidate: 2110281494Sandrew orig_l3 = pmap_load_store(l3, new_l3); 2111281494Sandrew PTE_SYNC(l3); 2112281494Sandrew opa = orig_l3 & ~ATTR_MASK; 2113281494Sandrew 2114281494Sandrew if (opa != pa) { 2115281494Sandrew if ((orig_l3 & ATTR_SW_MANAGED) != 0) { 2116281494Sandrew om = PHYS_TO_VM_PAGE(opa); 2117281494Sandrew if (pmap_page_dirty(orig_l3)) 2118281494Sandrew vm_page_dirty(om); 2119281494Sandrew if ((orig_l3 & ATTR_AF) != 0) 2120281494Sandrew vm_page_aflag_set(om, PGA_REFERENCED); 2121281494Sandrew CHANGE_PV_LIST_LOCK_TO_PHYS(&lock, opa); 2122281494Sandrew pmap_pvh_free(&om->md, pmap, va); 2123281494Sandrew } 2124281494Sandrew } else if (pmap_page_dirty(orig_l3)) { 2125281494Sandrew if ((orig_l3 & ATTR_SW_MANAGED) != 0) 2126281494Sandrew vm_page_dirty(m); 2127281494Sandrew } 2128281494Sandrew } else { 2129281494Sandrew pmap_load_store(l3, new_l3); 2130281494Sandrew PTE_SYNC(l3); 2131281494Sandrew } 2132285212Sandrew pmap_invalidate_page(pmap, va); 2133281494Sandrew if ((pmap != pmap_kernel()) && (pmap == &curproc->p_vmspace->vm_pmap)) 2134281494Sandrew cpu_icache_sync_range(va, PAGE_SIZE); 2135281494Sandrew 2136281494Sandrew if (lock != NULL) 2137281494Sandrew rw_wunlock(lock); 2138281494Sandrew rw_runlock(&pvh_global_lock); 2139281494Sandrew PMAP_UNLOCK(pmap); 2140281494Sandrew return (KERN_SUCCESS); 2141281494Sandrew} 2142281494Sandrew 2143281494Sandrew/* 2144281494Sandrew * Maps a sequence of resident pages belonging to the same object. 2145281494Sandrew * The sequence begins with the given page m_start. This page is 2146281494Sandrew * mapped at the given virtual address start. Each subsequent page is 2147281494Sandrew * mapped at a virtual address that is offset from start by the same 2148281494Sandrew * amount as the page is offset from m_start within the object. The 2149281494Sandrew * last page in the sequence is the page with the largest offset from 2150281494Sandrew * m_start that can be mapped at a virtual address less than the given 2151281494Sandrew * virtual address end. Not every virtual page between start and end 2152281494Sandrew * is mapped; only those for which a resident page exists with the 2153281494Sandrew * corresponding offset from m_start are mapped. 2154281494Sandrew */ 2155281494Sandrewvoid 2156281494Sandrewpmap_enter_object(pmap_t pmap, vm_offset_t start, vm_offset_t end, 2157281494Sandrew vm_page_t m_start, vm_prot_t prot) 2158281494Sandrew{ 2159281494Sandrew struct rwlock *lock; 2160281494Sandrew vm_offset_t va; 2161281494Sandrew vm_page_t m, mpte; 2162281494Sandrew vm_pindex_t diff, psize; 2163281494Sandrew 2164281494Sandrew VM_OBJECT_ASSERT_LOCKED(m_start->object); 2165281494Sandrew 2166281494Sandrew psize = atop(end - start); 2167281494Sandrew mpte = NULL; 2168281494Sandrew m = m_start; 2169281494Sandrew lock = NULL; 2170281494Sandrew rw_rlock(&pvh_global_lock); 2171281494Sandrew PMAP_LOCK(pmap); 2172281494Sandrew while (m != NULL && (diff = m->pindex - m_start->pindex) < psize) { 2173281494Sandrew va = start + ptoa(diff); 2174281494Sandrew mpte = pmap_enter_quick_locked(pmap, va, m, prot, mpte, &lock); 2175281494Sandrew m = TAILQ_NEXT(m, listq); 2176281494Sandrew } 2177281494Sandrew if (lock != NULL) 2178281494Sandrew rw_wunlock(lock); 2179281494Sandrew rw_runlock(&pvh_global_lock); 2180281494Sandrew PMAP_UNLOCK(pmap); 2181281494Sandrew} 2182281494Sandrew 2183281494Sandrew/* 2184281494Sandrew * this code makes some *MAJOR* assumptions: 2185281494Sandrew * 1. Current pmap & pmap exists. 2186281494Sandrew * 2. Not wired. 2187281494Sandrew * 3. Read access. 2188281494Sandrew * 4. No page table pages. 2189281494Sandrew * but is *MUCH* faster than pmap_enter... 2190281494Sandrew */ 2191281494Sandrew 2192281494Sandrewvoid 2193281494Sandrewpmap_enter_quick(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot) 2194281494Sandrew{ 2195281494Sandrew struct rwlock *lock; 2196281494Sandrew 2197281494Sandrew lock = NULL; 2198281494Sandrew rw_rlock(&pvh_global_lock); 2199281494Sandrew PMAP_LOCK(pmap); 2200281494Sandrew (void)pmap_enter_quick_locked(pmap, va, m, prot, NULL, &lock); 2201281494Sandrew if (lock != NULL) 2202281494Sandrew rw_wunlock(lock); 2203281494Sandrew rw_runlock(&pvh_global_lock); 2204281494Sandrew PMAP_UNLOCK(pmap); 2205281494Sandrew} 2206281494Sandrew 2207281494Sandrewstatic vm_page_t 2208281494Sandrewpmap_enter_quick_locked(pmap_t pmap, vm_offset_t va, vm_page_t m, 2209281494Sandrew vm_prot_t prot, vm_page_t mpte, struct rwlock **lockp) 2210281494Sandrew{ 2211281494Sandrew struct spglist free; 2212281494Sandrew pd_entry_t *l2; 2213281494Sandrew pt_entry_t *l3; 2214281494Sandrew vm_paddr_t pa; 2215281494Sandrew 2216281494Sandrew KASSERT(va < kmi.clean_sva || va >= kmi.clean_eva || 2217281494Sandrew (m->oflags & VPO_UNMANAGED) != 0, 2218281494Sandrew ("pmap_enter_quick_locked: managed mapping within the clean submap")); 2219281494Sandrew rw_assert(&pvh_global_lock, RA_LOCKED); 2220281494Sandrew PMAP_LOCK_ASSERT(pmap, MA_OWNED); 2221281494Sandrew 2222285212Sandrew CTR2(KTR_PMAP, "pmap_enter_quick_locked: %p %lx", pmap, va); 2223281494Sandrew /* 2224281494Sandrew * In the case that a page table page is not 2225281494Sandrew * resident, we are creating it here. 2226281494Sandrew */ 2227281494Sandrew if (va < VM_MAXUSER_ADDRESS) { 2228281494Sandrew vm_pindex_t l2pindex; 2229281494Sandrew 2230281494Sandrew /* 2231281494Sandrew * Calculate pagetable page index 2232281494Sandrew */ 2233281494Sandrew l2pindex = pmap_l2_pindex(va); 2234281494Sandrew if (mpte && (mpte->pindex == l2pindex)) { 2235281494Sandrew mpte->wire_count++; 2236281494Sandrew } else { 2237281494Sandrew /* 2238281494Sandrew * Get the l2 entry 2239281494Sandrew */ 2240281494Sandrew l2 = pmap_l2(pmap, va); 2241281494Sandrew 2242281494Sandrew /* 2243281494Sandrew * If the page table page is mapped, we just increment 2244281494Sandrew * the hold count, and activate it. Otherwise, we 2245281494Sandrew * attempt to allocate a page table page. If this 2246281494Sandrew * attempt fails, we don't retry. Instead, we give up. 2247281494Sandrew */ 2248285045Sandrew if (l2 != NULL && pmap_load(l2) != 0) { 2249285045Sandrew mpte = 2250285045Sandrew PHYS_TO_VM_PAGE(pmap_load(l2) & ~ATTR_MASK); 2251281494Sandrew mpte->wire_count++; 2252281494Sandrew } else { 2253281494Sandrew /* 2254281494Sandrew * Pass NULL instead of the PV list lock 2255281494Sandrew * pointer, because we don't intend to sleep. 2256281494Sandrew */ 2257281494Sandrew mpte = _pmap_alloc_l3(pmap, l2pindex, NULL); 2258281494Sandrew if (mpte == NULL) 2259281494Sandrew return (mpte); 2260281494Sandrew } 2261281494Sandrew } 2262281494Sandrew l3 = (pt_entry_t *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(mpte)); 2263281494Sandrew l3 = &l3[pmap_l3_index(va)]; 2264281494Sandrew } else { 2265281494Sandrew mpte = NULL; 2266281494Sandrew l3 = pmap_l3(kernel_pmap, va); 2267281494Sandrew } 2268281494Sandrew if (l3 == NULL) 2269281494Sandrew panic("pmap_enter_quick_locked: No l3"); 2270285212Sandrew if (pmap_load(l3) != 0) { 2271281494Sandrew if (mpte != NULL) { 2272281494Sandrew mpte->wire_count--; 2273281494Sandrew mpte = NULL; 2274281494Sandrew } 2275281494Sandrew return (mpte); 2276281494Sandrew } 2277281494Sandrew 2278281494Sandrew /* 2279281494Sandrew * Enter on the PV list if part of our managed memory. 2280281494Sandrew */ 2281281494Sandrew if ((m->oflags & VPO_UNMANAGED) == 0 && 2282281494Sandrew !pmap_try_insert_pv_entry(pmap, va, m, lockp)) { 2283281494Sandrew if (mpte != NULL) { 2284281494Sandrew SLIST_INIT(&free); 2285281494Sandrew if (pmap_unwire_l3(pmap, va, mpte, &free)) { 2286281494Sandrew pmap_invalidate_page(pmap, va); 2287281494Sandrew pmap_free_zero_pages(&free); 2288281494Sandrew } 2289281494Sandrew mpte = NULL; 2290281494Sandrew } 2291281494Sandrew return (mpte); 2292281494Sandrew } 2293281494Sandrew 2294281494Sandrew /* 2295281494Sandrew * Increment counters 2296281494Sandrew */ 2297281494Sandrew pmap_resident_count_inc(pmap, 1); 2298281494Sandrew 2299285537Sandrew pa = VM_PAGE_TO_PHYS(m) | ATTR_DEFAULT | ATTR_IDX(m->md.pv_memattr) | 2300281494Sandrew ATTR_AP(ATTR_AP_RW) | L3_PAGE; 2301281494Sandrew 2302281494Sandrew /* 2303281494Sandrew * Now validate mapping with RO protection 2304281494Sandrew */ 2305281494Sandrew if ((m->oflags & VPO_UNMANAGED) == 0) 2306281494Sandrew pa |= ATTR_SW_MANAGED; 2307281494Sandrew pmap_load_store(l3, pa); 2308281494Sandrew PTE_SYNC(l3); 2309281494Sandrew pmap_invalidate_page(pmap, va); 2310281494Sandrew return (mpte); 2311281494Sandrew} 2312281494Sandrew 2313281494Sandrew/* 2314281494Sandrew * This code maps large physical mmap regions into the 2315281494Sandrew * processor address space. Note that some shortcuts 2316281494Sandrew * are taken, but the code works. 2317281494Sandrew */ 2318281494Sandrewvoid 2319281494Sandrewpmap_object_init_pt(pmap_t pmap, vm_offset_t addr, vm_object_t object, 2320281494Sandrew vm_pindex_t pindex, vm_size_t size) 2321281494Sandrew{ 2322281494Sandrew 2323281846Sandrew VM_OBJECT_ASSERT_WLOCKED(object); 2324281846Sandrew KASSERT(object->type == OBJT_DEVICE || object->type == OBJT_SG, 2325281846Sandrew ("pmap_object_init_pt: non-device object")); 2326281494Sandrew} 2327281494Sandrew 2328281494Sandrew/* 2329281494Sandrew * Clear the wired attribute from the mappings for the specified range of 2330281494Sandrew * addresses in the given pmap. Every valid mapping within that range 2331281494Sandrew * must have the wired attribute set. In contrast, invalid mappings 2332281494Sandrew * cannot have the wired attribute set, so they are ignored. 2333281494Sandrew * 2334281494Sandrew * The wired attribute of the page table entry is not a hardware feature, 2335281494Sandrew * so there is no need to invalidate any TLB entries. 2336281494Sandrew */ 2337281494Sandrewvoid 2338281494Sandrewpmap_unwire(pmap_t pmap, vm_offset_t sva, vm_offset_t eva) 2339281494Sandrew{ 2340281494Sandrew vm_offset_t va_next; 2341281494Sandrew pd_entry_t *l1, *l2; 2342281494Sandrew pt_entry_t *l3; 2343281494Sandrew boolean_t pv_lists_locked; 2344281494Sandrew 2345281494Sandrew pv_lists_locked = FALSE; 2346281494Sandrew PMAP_LOCK(pmap); 2347281494Sandrew for (; sva < eva; sva = va_next) { 2348281494Sandrew l1 = pmap_l1(pmap, sva); 2349285045Sandrew if (pmap_load(l1) == 0) { 2350281494Sandrew va_next = (sva + L1_SIZE) & ~L1_OFFSET; 2351281494Sandrew if (va_next < sva) 2352281494Sandrew va_next = eva; 2353281494Sandrew continue; 2354281494Sandrew } 2355281494Sandrew 2356281494Sandrew va_next = (sva + L2_SIZE) & ~L2_OFFSET; 2357281494Sandrew if (va_next < sva) 2358281494Sandrew va_next = eva; 2359281494Sandrew 2360281494Sandrew l2 = pmap_l1_to_l2(l1, sva); 2361285045Sandrew if (pmap_load(l2) == 0) 2362281494Sandrew continue; 2363281494Sandrew 2364281494Sandrew if (va_next > eva) 2365281494Sandrew va_next = eva; 2366281494Sandrew for (l3 = pmap_l2_to_l3(l2, sva); sva != va_next; l3++, 2367281494Sandrew sva += L3_SIZE) { 2368285045Sandrew if (pmap_load(l3) == 0) 2369281494Sandrew continue; 2370285045Sandrew if ((pmap_load(l3) & ATTR_SW_WIRED) == 0) 2371281494Sandrew panic("pmap_unwire: l3 %#jx is missing " 2372288445Sandrew "ATTR_SW_WIRED", (uintmax_t)pmap_load(l3)); 2373281494Sandrew 2374281494Sandrew /* 2375281494Sandrew * PG_W must be cleared atomically. Although the pmap 2376281494Sandrew * lock synchronizes access to PG_W, another processor 2377281494Sandrew * could be setting PG_M and/or PG_A concurrently. 2378281494Sandrew */ 2379281494Sandrew atomic_clear_long(l3, ATTR_SW_WIRED); 2380281494Sandrew pmap->pm_stats.wired_count--; 2381281494Sandrew } 2382281494Sandrew } 2383281494Sandrew if (pv_lists_locked) 2384281494Sandrew rw_runlock(&pvh_global_lock); 2385281494Sandrew PMAP_UNLOCK(pmap); 2386281494Sandrew} 2387281494Sandrew 2388281494Sandrew/* 2389281494Sandrew * Copy the range specified by src_addr/len 2390281494Sandrew * from the source map to the range dst_addr/len 2391281494Sandrew * in the destination map. 2392281494Sandrew * 2393281494Sandrew * This routine is only advisory and need not do anything. 2394281494Sandrew */ 2395281494Sandrew 2396281494Sandrewvoid 2397281494Sandrewpmap_copy(pmap_t dst_pmap, pmap_t src_pmap, vm_offset_t dst_addr, vm_size_t len, 2398281494Sandrew vm_offset_t src_addr) 2399281494Sandrew{ 2400281494Sandrew} 2401281494Sandrew 2402281494Sandrew/* 2403281494Sandrew * pmap_zero_page zeros the specified hardware page by mapping 2404281494Sandrew * the page into KVM and using bzero to clear its contents. 2405281494Sandrew */ 2406281494Sandrewvoid 2407281494Sandrewpmap_zero_page(vm_page_t m) 2408281494Sandrew{ 2409281494Sandrew vm_offset_t va = PHYS_TO_DMAP(VM_PAGE_TO_PHYS(m)); 2410281494Sandrew 2411281494Sandrew pagezero((void *)va); 2412281494Sandrew} 2413281494Sandrew 2414281494Sandrew/* 2415281494Sandrew * pmap_zero_page_area zeros the specified hardware page by mapping 2416281494Sandrew * the page into KVM and using bzero to clear its contents. 2417281494Sandrew * 2418281494Sandrew * off and size may not cover an area beyond a single hardware page. 2419281494Sandrew */ 2420281494Sandrewvoid 2421281494Sandrewpmap_zero_page_area(vm_page_t m, int off, int size) 2422281494Sandrew{ 2423281494Sandrew vm_offset_t va = PHYS_TO_DMAP(VM_PAGE_TO_PHYS(m)); 2424281494Sandrew 2425281494Sandrew if (off == 0 && size == PAGE_SIZE) 2426281494Sandrew pagezero((void *)va); 2427281494Sandrew else 2428281494Sandrew bzero((char *)va + off, size); 2429281494Sandrew} 2430281494Sandrew 2431281494Sandrew/* 2432281494Sandrew * pmap_zero_page_idle zeros the specified hardware page by mapping 2433281494Sandrew * the page into KVM and using bzero to clear its contents. This 2434281494Sandrew * is intended to be called from the vm_pagezero process only and 2435281494Sandrew * outside of Giant. 2436281494Sandrew */ 2437281494Sandrewvoid 2438281494Sandrewpmap_zero_page_idle(vm_page_t m) 2439281494Sandrew{ 2440281494Sandrew vm_offset_t va = PHYS_TO_DMAP(VM_PAGE_TO_PHYS(m)); 2441281494Sandrew 2442281494Sandrew pagezero((void *)va); 2443281494Sandrew} 2444281494Sandrew 2445281494Sandrew/* 2446281494Sandrew * pmap_copy_page copies the specified (machine independent) 2447281494Sandrew * page by mapping the page into virtual memory and using 2448281494Sandrew * bcopy to copy the page, one machine dependent page at a 2449281494Sandrew * time. 2450281494Sandrew */ 2451281494Sandrewvoid 2452281494Sandrewpmap_copy_page(vm_page_t msrc, vm_page_t mdst) 2453281494Sandrew{ 2454281494Sandrew vm_offset_t src = PHYS_TO_DMAP(VM_PAGE_TO_PHYS(msrc)); 2455281494Sandrew vm_offset_t dst = PHYS_TO_DMAP(VM_PAGE_TO_PHYS(mdst)); 2456281494Sandrew 2457281494Sandrew pagecopy((void *)src, (void *)dst); 2458281494Sandrew} 2459281494Sandrew 2460281494Sandrewint unmapped_buf_allowed = 1; 2461281494Sandrew 2462281494Sandrewvoid 2463281494Sandrewpmap_copy_pages(vm_page_t ma[], vm_offset_t a_offset, vm_page_t mb[], 2464281494Sandrew vm_offset_t b_offset, int xfersize) 2465281494Sandrew{ 2466281494Sandrew void *a_cp, *b_cp; 2467281494Sandrew vm_page_t m_a, m_b; 2468281494Sandrew vm_paddr_t p_a, p_b; 2469281494Sandrew vm_offset_t a_pg_offset, b_pg_offset; 2470281494Sandrew int cnt; 2471281494Sandrew 2472281494Sandrew while (xfersize > 0) { 2473281494Sandrew a_pg_offset = a_offset & PAGE_MASK; 2474281494Sandrew m_a = ma[a_offset >> PAGE_SHIFT]; 2475281494Sandrew p_a = m_a->phys_addr; 2476281494Sandrew b_pg_offset = b_offset & PAGE_MASK; 2477281494Sandrew m_b = mb[b_offset >> PAGE_SHIFT]; 2478281494Sandrew p_b = m_b->phys_addr; 2479281494Sandrew cnt = min(xfersize, PAGE_SIZE - a_pg_offset); 2480281494Sandrew cnt = min(cnt, PAGE_SIZE - b_pg_offset); 2481281494Sandrew if (__predict_false(!PHYS_IN_DMAP(p_a))) { 2482281494Sandrew panic("!DMAP a %lx", p_a); 2483281494Sandrew } else { 2484281494Sandrew a_cp = (char *)PHYS_TO_DMAP(p_a) + a_pg_offset; 2485281494Sandrew } 2486281494Sandrew if (__predict_false(!PHYS_IN_DMAP(p_b))) { 2487281494Sandrew panic("!DMAP b %lx", p_b); 2488281494Sandrew } else { 2489281494Sandrew b_cp = (char *)PHYS_TO_DMAP(p_b) + b_pg_offset; 2490281494Sandrew } 2491281494Sandrew bcopy(a_cp, b_cp, cnt); 2492281494Sandrew a_offset += cnt; 2493281494Sandrew b_offset += cnt; 2494281494Sandrew xfersize -= cnt; 2495281494Sandrew } 2496281494Sandrew} 2497281494Sandrew 2498286296Sjahvm_offset_t 2499286296Sjahpmap_quick_enter_page(vm_page_t m) 2500286296Sjah{ 2501286296Sjah 2502286296Sjah return (PHYS_TO_DMAP(VM_PAGE_TO_PHYS(m))); 2503286296Sjah} 2504286296Sjah 2505286296Sjahvoid 2506286296Sjahpmap_quick_remove_page(vm_offset_t addr) 2507286296Sjah{ 2508286296Sjah} 2509286296Sjah 2510281494Sandrew/* 2511281494Sandrew * Returns true if the pmap's pv is one of the first 2512281494Sandrew * 16 pvs linked to from this page. This count may 2513281494Sandrew * be changed upwards or downwards in the future; it 2514281494Sandrew * is only necessary that true be returned for a small 2515281494Sandrew * subset of pmaps for proper page aging. 2516281494Sandrew */ 2517281494Sandrewboolean_t 2518281494Sandrewpmap_page_exists_quick(pmap_t pmap, vm_page_t m) 2519281494Sandrew{ 2520281494Sandrew struct rwlock *lock; 2521281494Sandrew pv_entry_t pv; 2522281494Sandrew int loops = 0; 2523281494Sandrew boolean_t rv; 2524281494Sandrew 2525281494Sandrew KASSERT((m->oflags & VPO_UNMANAGED) == 0, 2526281494Sandrew ("pmap_page_exists_quick: page %p is not managed", m)); 2527281494Sandrew rv = FALSE; 2528281494Sandrew rw_rlock(&pvh_global_lock); 2529281494Sandrew lock = VM_PAGE_TO_PV_LIST_LOCK(m); 2530281494Sandrew rw_rlock(lock); 2531281494Sandrew TAILQ_FOREACH(pv, &m->md.pv_list, pv_next) { 2532281494Sandrew if (PV_PMAP(pv) == pmap) { 2533281494Sandrew rv = TRUE; 2534281494Sandrew break; 2535281494Sandrew } 2536281494Sandrew loops++; 2537281494Sandrew if (loops >= 16) 2538281494Sandrew break; 2539281494Sandrew } 2540281494Sandrew rw_runlock(lock); 2541281494Sandrew rw_runlock(&pvh_global_lock); 2542281494Sandrew return (rv); 2543281494Sandrew} 2544281494Sandrew 2545281494Sandrew/* 2546281494Sandrew * pmap_page_wired_mappings: 2547281494Sandrew * 2548281494Sandrew * Return the number of managed mappings to the given physical page 2549281494Sandrew * that are wired. 2550281494Sandrew */ 2551281494Sandrewint 2552281494Sandrewpmap_page_wired_mappings(vm_page_t m) 2553281494Sandrew{ 2554281494Sandrew struct rwlock *lock; 2555281494Sandrew pmap_t pmap; 2556281494Sandrew pt_entry_t *l3; 2557281494Sandrew pv_entry_t pv; 2558281494Sandrew int count, md_gen; 2559281494Sandrew 2560281494Sandrew if ((m->oflags & VPO_UNMANAGED) != 0) 2561281494Sandrew return (0); 2562281494Sandrew rw_rlock(&pvh_global_lock); 2563281494Sandrew lock = VM_PAGE_TO_PV_LIST_LOCK(m); 2564281494Sandrew rw_rlock(lock); 2565281494Sandrewrestart: 2566281494Sandrew count = 0; 2567281494Sandrew TAILQ_FOREACH(pv, &m->md.pv_list, pv_next) { 2568281494Sandrew pmap = PV_PMAP(pv); 2569281494Sandrew if (!PMAP_TRYLOCK(pmap)) { 2570281494Sandrew md_gen = m->md.pv_gen; 2571281494Sandrew rw_runlock(lock); 2572281494Sandrew PMAP_LOCK(pmap); 2573281494Sandrew rw_rlock(lock); 2574281494Sandrew if (md_gen != m->md.pv_gen) { 2575281494Sandrew PMAP_UNLOCK(pmap); 2576281494Sandrew goto restart; 2577281494Sandrew } 2578281494Sandrew } 2579281494Sandrew l3 = pmap_l3(pmap, pv->pv_va); 2580285045Sandrew if (l3 != NULL && (pmap_load(l3) & ATTR_SW_WIRED) != 0) 2581281494Sandrew count++; 2582281494Sandrew PMAP_UNLOCK(pmap); 2583281494Sandrew } 2584281494Sandrew rw_runlock(lock); 2585281494Sandrew rw_runlock(&pvh_global_lock); 2586281494Sandrew return (count); 2587281494Sandrew} 2588281494Sandrew 2589281494Sandrew/* 2590281494Sandrew * Destroy all managed, non-wired mappings in the given user-space 2591281494Sandrew * pmap. This pmap cannot be active on any processor besides the 2592281494Sandrew * caller. 2593281494Sandrew * 2594281494Sandrew * This function cannot be applied to the kernel pmap. Moreover, it 2595281494Sandrew * is not intended for general use. It is only to be used during 2596281494Sandrew * process termination. Consequently, it can be implemented in ways 2597281494Sandrew * that make it faster than pmap_remove(). First, it can more quickly 2598281494Sandrew * destroy mappings by iterating over the pmap's collection of PV 2599281494Sandrew * entries, rather than searching the page table. Second, it doesn't 2600281494Sandrew * have to test and clear the page table entries atomically, because 2601281494Sandrew * no processor is currently accessing the user address space. In 2602281494Sandrew * particular, a page table entry's dirty bit won't change state once 2603281494Sandrew * this function starts. 2604281494Sandrew */ 2605281494Sandrewvoid 2606281494Sandrewpmap_remove_pages(pmap_t pmap) 2607281494Sandrew{ 2608281494Sandrew pd_entry_t ptepde, *l2; 2609281494Sandrew pt_entry_t *l3, tl3; 2610281494Sandrew struct spglist free; 2611281494Sandrew vm_page_t m; 2612281494Sandrew pv_entry_t pv; 2613281494Sandrew struct pv_chunk *pc, *npc; 2614281494Sandrew struct rwlock *lock; 2615281494Sandrew int64_t bit; 2616281494Sandrew uint64_t inuse, bitmask; 2617281494Sandrew int allfree, field, freed, idx; 2618281494Sandrew vm_paddr_t pa; 2619281494Sandrew 2620281494Sandrew lock = NULL; 2621281494Sandrew 2622281494Sandrew SLIST_INIT(&free); 2623281494Sandrew rw_rlock(&pvh_global_lock); 2624281494Sandrew PMAP_LOCK(pmap); 2625281494Sandrew TAILQ_FOREACH_SAFE(pc, &pmap->pm_pvchunk, pc_list, npc) { 2626281494Sandrew allfree = 1; 2627281494Sandrew freed = 0; 2628281494Sandrew for (field = 0; field < _NPCM; field++) { 2629281494Sandrew inuse = ~pc->pc_map[field] & pc_freemask[field]; 2630281494Sandrew while (inuse != 0) { 2631281494Sandrew bit = ffsl(inuse) - 1; 2632281494Sandrew bitmask = 1UL << bit; 2633281494Sandrew idx = field * 64 + bit; 2634281494Sandrew pv = &pc->pc_pventry[idx]; 2635281494Sandrew inuse &= ~bitmask; 2636281494Sandrew 2637281494Sandrew l2 = pmap_l2(pmap, pv->pv_va); 2638281494Sandrew ptepde = pmap_load(l2); 2639281494Sandrew l3 = pmap_l2_to_l3(l2, pv->pv_va); 2640281494Sandrew tl3 = pmap_load(l3); 2641281494Sandrew 2642281494Sandrew/* 2643281494Sandrew * We cannot remove wired pages from a process' mapping at this time 2644281494Sandrew */ 2645281494Sandrew if (tl3 & ATTR_SW_WIRED) { 2646281494Sandrew allfree = 0; 2647281494Sandrew continue; 2648281494Sandrew } 2649281494Sandrew 2650281494Sandrew pa = tl3 & ~ATTR_MASK; 2651281494Sandrew 2652281494Sandrew m = PHYS_TO_VM_PAGE(pa); 2653281494Sandrew KASSERT(m->phys_addr == pa, 2654281494Sandrew ("vm_page_t %p phys_addr mismatch %016jx %016jx", 2655281494Sandrew m, (uintmax_t)m->phys_addr, 2656281494Sandrew (uintmax_t)tl3)); 2657281494Sandrew 2658281494Sandrew KASSERT((m->flags & PG_FICTITIOUS) != 0 || 2659281494Sandrew m < &vm_page_array[vm_page_array_size], 2660281494Sandrew ("pmap_remove_pages: bad l3 %#jx", 2661281494Sandrew (uintmax_t)tl3)); 2662281494Sandrew 2663281494Sandrew if (pmap_is_current(pmap) && 2664281494Sandrew pmap_l3_valid_cacheable(pmap_load(l3))) 2665281494Sandrew cpu_dcache_wb_range(pv->pv_va, L3_SIZE); 2666281494Sandrew pmap_load_clear(l3); 2667281494Sandrew PTE_SYNC(l3); 2668285212Sandrew pmap_invalidate_page(pmap, pv->pv_va); 2669281494Sandrew 2670281494Sandrew /* 2671281494Sandrew * Update the vm_page_t clean/reference bits. 2672281494Sandrew */ 2673281494Sandrew if ((tl3 & ATTR_AP_RW_BIT) == 2674281494Sandrew ATTR_AP(ATTR_AP_RW)) 2675281494Sandrew vm_page_dirty(m); 2676281494Sandrew 2677281494Sandrew CHANGE_PV_LIST_LOCK_TO_VM_PAGE(&lock, m); 2678281494Sandrew 2679281494Sandrew /* Mark free */ 2680281494Sandrew pc->pc_map[field] |= bitmask; 2681281494Sandrew 2682281494Sandrew pmap_resident_count_dec(pmap, 1); 2683281494Sandrew TAILQ_REMOVE(&m->md.pv_list, pv, pv_next); 2684281494Sandrew m->md.pv_gen++; 2685281494Sandrew 2686281494Sandrew pmap_unuse_l3(pmap, pv->pv_va, ptepde, &free); 2687281494Sandrew freed++; 2688281494Sandrew } 2689281494Sandrew } 2690281494Sandrew PV_STAT(atomic_add_long(&pv_entry_frees, freed)); 2691281494Sandrew PV_STAT(atomic_add_int(&pv_entry_spare, freed)); 2692281494Sandrew PV_STAT(atomic_subtract_long(&pv_entry_count, freed)); 2693281494Sandrew if (allfree) { 2694281494Sandrew TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list); 2695281494Sandrew free_pv_chunk(pc); 2696281494Sandrew } 2697281494Sandrew } 2698281494Sandrew pmap_invalidate_all(pmap); 2699281494Sandrew if (lock != NULL) 2700281494Sandrew rw_wunlock(lock); 2701281494Sandrew rw_runlock(&pvh_global_lock); 2702281494Sandrew PMAP_UNLOCK(pmap); 2703281494Sandrew pmap_free_zero_pages(&free); 2704281494Sandrew} 2705281494Sandrew 2706281494Sandrew/* 2707281494Sandrew * This is used to check if a page has been accessed or modified. As we 2708281494Sandrew * don't have a bit to see if it has been modified we have to assume it 2709281494Sandrew * has been if the page is read/write. 2710281494Sandrew */ 2711281494Sandrewstatic boolean_t 2712281494Sandrewpmap_page_test_mappings(vm_page_t m, boolean_t accessed, boolean_t modified) 2713281494Sandrew{ 2714281494Sandrew struct rwlock *lock; 2715281494Sandrew pv_entry_t pv; 2716281494Sandrew pt_entry_t *l3, mask, value; 2717281494Sandrew pmap_t pmap; 2718281494Sandrew int md_gen; 2719281494Sandrew boolean_t rv; 2720281494Sandrew 2721281494Sandrew rv = FALSE; 2722281494Sandrew rw_rlock(&pvh_global_lock); 2723281494Sandrew lock = VM_PAGE_TO_PV_LIST_LOCK(m); 2724281494Sandrew rw_rlock(lock); 2725281494Sandrewrestart: 2726281494Sandrew TAILQ_FOREACH(pv, &m->md.pv_list, pv_next) { 2727281494Sandrew pmap = PV_PMAP(pv); 2728281494Sandrew if (!PMAP_TRYLOCK(pmap)) { 2729281494Sandrew md_gen = m->md.pv_gen; 2730281494Sandrew rw_runlock(lock); 2731281494Sandrew PMAP_LOCK(pmap); 2732281494Sandrew rw_rlock(lock); 2733281494Sandrew if (md_gen != m->md.pv_gen) { 2734281494Sandrew PMAP_UNLOCK(pmap); 2735281494Sandrew goto restart; 2736281494Sandrew } 2737281494Sandrew } 2738281494Sandrew l3 = pmap_l3(pmap, pv->pv_va); 2739281494Sandrew mask = 0; 2740281494Sandrew value = 0; 2741281494Sandrew if (modified) { 2742281494Sandrew mask |= ATTR_AP_RW_BIT; 2743281494Sandrew value |= ATTR_AP(ATTR_AP_RW); 2744281494Sandrew } 2745281494Sandrew if (accessed) { 2746281494Sandrew mask |= ATTR_AF | ATTR_DESCR_MASK; 2747281494Sandrew value |= ATTR_AF | L3_PAGE; 2748281494Sandrew } 2749281494Sandrew rv = (pmap_load(l3) & mask) == value; 2750281494Sandrew PMAP_UNLOCK(pmap); 2751281494Sandrew if (rv) 2752281494Sandrew goto out; 2753281494Sandrew } 2754281494Sandrewout: 2755281494Sandrew rw_runlock(lock); 2756281494Sandrew rw_runlock(&pvh_global_lock); 2757281494Sandrew return (rv); 2758281494Sandrew} 2759281494Sandrew 2760281494Sandrew/* 2761281494Sandrew * pmap_is_modified: 2762281494Sandrew * 2763281494Sandrew * Return whether or not the specified physical page was modified 2764281494Sandrew * in any physical maps. 2765281494Sandrew */ 2766281494Sandrewboolean_t 2767281494Sandrewpmap_is_modified(vm_page_t m) 2768281494Sandrew{ 2769281494Sandrew 2770281494Sandrew KASSERT((m->oflags & VPO_UNMANAGED) == 0, 2771281494Sandrew ("pmap_is_modified: page %p is not managed", m)); 2772281494Sandrew 2773281494Sandrew /* 2774281494Sandrew * If the page is not exclusive busied, then PGA_WRITEABLE cannot be 2775281494Sandrew * concurrently set while the object is locked. Thus, if PGA_WRITEABLE 2776281494Sandrew * is clear, no PTEs can have PG_M set. 2777281494Sandrew */ 2778281494Sandrew VM_OBJECT_ASSERT_WLOCKED(m->object); 2779281494Sandrew if (!vm_page_xbusied(m) && (m->aflags & PGA_WRITEABLE) == 0) 2780281494Sandrew return (FALSE); 2781281494Sandrew return (pmap_page_test_mappings(m, FALSE, TRUE)); 2782281494Sandrew} 2783281494Sandrew 2784281494Sandrew/* 2785281494Sandrew * pmap_is_prefaultable: 2786281494Sandrew * 2787281494Sandrew * Return whether or not the specified virtual address is eligible 2788281494Sandrew * for prefault. 2789281494Sandrew */ 2790281494Sandrewboolean_t 2791281494Sandrewpmap_is_prefaultable(pmap_t pmap, vm_offset_t addr) 2792281494Sandrew{ 2793281494Sandrew pt_entry_t *l3; 2794281494Sandrew boolean_t rv; 2795281494Sandrew 2796281494Sandrew rv = FALSE; 2797281494Sandrew PMAP_LOCK(pmap); 2798281494Sandrew l3 = pmap_l3(pmap, addr); 2799285045Sandrew if (l3 != NULL && pmap_load(l3) != 0) { 2800281494Sandrew rv = TRUE; 2801281494Sandrew } 2802281494Sandrew PMAP_UNLOCK(pmap); 2803281494Sandrew return (rv); 2804281494Sandrew} 2805281494Sandrew 2806281494Sandrew/* 2807281494Sandrew * pmap_is_referenced: 2808281494Sandrew * 2809281494Sandrew * Return whether or not the specified physical page was referenced 2810281494Sandrew * in any physical maps. 2811281494Sandrew */ 2812281494Sandrewboolean_t 2813281494Sandrewpmap_is_referenced(vm_page_t m) 2814281494Sandrew{ 2815281494Sandrew 2816281494Sandrew KASSERT((m->oflags & VPO_UNMANAGED) == 0, 2817281494Sandrew ("pmap_is_referenced: page %p is not managed", m)); 2818281494Sandrew return (pmap_page_test_mappings(m, TRUE, FALSE)); 2819281494Sandrew} 2820281494Sandrew 2821281494Sandrew/* 2822281494Sandrew * Clear the write and modified bits in each of the given page's mappings. 2823281494Sandrew */ 2824281494Sandrewvoid 2825281494Sandrewpmap_remove_write(vm_page_t m) 2826281494Sandrew{ 2827281494Sandrew pmap_t pmap; 2828281494Sandrew struct rwlock *lock; 2829281494Sandrew pv_entry_t pv; 2830281494Sandrew pt_entry_t *l3, oldl3; 2831281494Sandrew int md_gen; 2832281494Sandrew 2833281494Sandrew KASSERT((m->oflags & VPO_UNMANAGED) == 0, 2834281494Sandrew ("pmap_remove_write: page %p is not managed", m)); 2835281494Sandrew 2836281494Sandrew /* 2837281494Sandrew * If the page is not exclusive busied, then PGA_WRITEABLE cannot be 2838281494Sandrew * set by another thread while the object is locked. Thus, 2839281494Sandrew * if PGA_WRITEABLE is clear, no page table entries need updating. 2840281494Sandrew */ 2841281494Sandrew VM_OBJECT_ASSERT_WLOCKED(m->object); 2842281494Sandrew if (!vm_page_xbusied(m) && (m->aflags & PGA_WRITEABLE) == 0) 2843281494Sandrew return; 2844281494Sandrew rw_rlock(&pvh_global_lock); 2845281494Sandrew lock = VM_PAGE_TO_PV_LIST_LOCK(m); 2846281494Sandrewretry_pv_loop: 2847281494Sandrew rw_wlock(lock); 2848281494Sandrew TAILQ_FOREACH(pv, &m->md.pv_list, pv_next) { 2849281494Sandrew pmap = PV_PMAP(pv); 2850281494Sandrew if (!PMAP_TRYLOCK(pmap)) { 2851281494Sandrew md_gen = m->md.pv_gen; 2852281494Sandrew rw_wunlock(lock); 2853281494Sandrew PMAP_LOCK(pmap); 2854281494Sandrew rw_wlock(lock); 2855281494Sandrew if (md_gen != m->md.pv_gen) { 2856281494Sandrew PMAP_UNLOCK(pmap); 2857281494Sandrew rw_wunlock(lock); 2858281494Sandrew goto retry_pv_loop; 2859281494Sandrew } 2860281494Sandrew } 2861281494Sandrew l3 = pmap_l3(pmap, pv->pv_va); 2862281494Sandrewretry: 2863288445Sandrew oldl3 = pmap_load(l3); 2864281494Sandrew if ((oldl3 & ATTR_AP_RW_BIT) == ATTR_AP(ATTR_AP_RW)) { 2865281494Sandrew if (!atomic_cmpset_long(l3, oldl3, 2866281494Sandrew oldl3 | ATTR_AP(ATTR_AP_RO))) 2867281494Sandrew goto retry; 2868281494Sandrew if ((oldl3 & ATTR_AF) != 0) 2869281494Sandrew vm_page_dirty(m); 2870281494Sandrew pmap_invalidate_page(pmap, pv->pv_va); 2871281494Sandrew } 2872281494Sandrew PMAP_UNLOCK(pmap); 2873281494Sandrew } 2874281494Sandrew rw_wunlock(lock); 2875281494Sandrew vm_page_aflag_clear(m, PGA_WRITEABLE); 2876281494Sandrew rw_runlock(&pvh_global_lock); 2877281494Sandrew} 2878281494Sandrew 2879281494Sandrewstatic __inline boolean_t 2880281494Sandrewsafe_to_clear_referenced(pmap_t pmap, pt_entry_t pte) 2881281494Sandrew{ 2882281494Sandrew 2883281494Sandrew return (FALSE); 2884281494Sandrew} 2885281494Sandrew 2886281494Sandrew#define PMAP_TS_REFERENCED_MAX 5 2887281494Sandrew 2888281494Sandrew/* 2889281494Sandrew * pmap_ts_referenced: 2890281494Sandrew * 2891281494Sandrew * Return a count of reference bits for a page, clearing those bits. 2892281494Sandrew * It is not necessary for every reference bit to be cleared, but it 2893281494Sandrew * is necessary that 0 only be returned when there are truly no 2894281494Sandrew * reference bits set. 2895281494Sandrew * 2896281494Sandrew * XXX: The exact number of bits to check and clear is a matter that 2897281494Sandrew * should be tested and standardized at some point in the future for 2898281494Sandrew * optimal aging of shared pages. 2899281494Sandrew */ 2900281494Sandrewint 2901281494Sandrewpmap_ts_referenced(vm_page_t m) 2902281494Sandrew{ 2903281494Sandrew pv_entry_t pv, pvf; 2904281494Sandrew pmap_t pmap; 2905281494Sandrew struct rwlock *lock; 2906288445Sandrew pd_entry_t *l2p, l2; 2907281494Sandrew pt_entry_t *l3; 2908281494Sandrew vm_paddr_t pa; 2909281494Sandrew int cleared, md_gen, not_cleared; 2910281494Sandrew struct spglist free; 2911281494Sandrew 2912281494Sandrew KASSERT((m->oflags & VPO_UNMANAGED) == 0, 2913281494Sandrew ("pmap_ts_referenced: page %p is not managed", m)); 2914281494Sandrew SLIST_INIT(&free); 2915281494Sandrew cleared = 0; 2916281494Sandrew pa = VM_PAGE_TO_PHYS(m); 2917281494Sandrew lock = PHYS_TO_PV_LIST_LOCK(pa); 2918281494Sandrew rw_rlock(&pvh_global_lock); 2919281494Sandrew rw_wlock(lock); 2920281494Sandrewretry: 2921281494Sandrew not_cleared = 0; 2922281494Sandrew if ((pvf = TAILQ_FIRST(&m->md.pv_list)) == NULL) 2923281494Sandrew goto out; 2924281494Sandrew pv = pvf; 2925281494Sandrew do { 2926281494Sandrew if (pvf == NULL) 2927281494Sandrew pvf = pv; 2928281494Sandrew pmap = PV_PMAP(pv); 2929281494Sandrew if (!PMAP_TRYLOCK(pmap)) { 2930281494Sandrew md_gen = m->md.pv_gen; 2931281494Sandrew rw_wunlock(lock); 2932281494Sandrew PMAP_LOCK(pmap); 2933281494Sandrew rw_wlock(lock); 2934281494Sandrew if (md_gen != m->md.pv_gen) { 2935281494Sandrew PMAP_UNLOCK(pmap); 2936281494Sandrew goto retry; 2937281494Sandrew } 2938281494Sandrew } 2939288445Sandrew l2p = pmap_l2(pmap, pv->pv_va); 2940288445Sandrew KASSERT(l2p != NULL, ("pmap_ts_referenced: no l2 table found")); 2941288445Sandrew l2 = pmap_load(l2p); 2942288445Sandrew KASSERT((l2 & ATTR_DESCR_MASK) == L2_TABLE, 2943281494Sandrew ("pmap_ts_referenced: found an invalid l2 table")); 2944288445Sandrew l3 = pmap_l2_to_l3(l2p, pv->pv_va); 2945285045Sandrew if ((pmap_load(l3) & ATTR_AF) != 0) { 2946288445Sandrew if (safe_to_clear_referenced(pmap, pmap_load(l3))) { 2947281494Sandrew /* 2948281494Sandrew * TODO: We don't handle the access flag 2949281494Sandrew * at all. We need to be able to set it in 2950281494Sandrew * the exception handler. 2951281494Sandrew */ 2952286073Semaste panic("ARM64TODO: safe_to_clear_referenced\n"); 2953285045Sandrew } else if ((pmap_load(l3) & ATTR_SW_WIRED) == 0) { 2954281494Sandrew /* 2955281494Sandrew * Wired pages cannot be paged out so 2956281494Sandrew * doing accessed bit emulation for 2957281494Sandrew * them is wasted effort. We do the 2958281494Sandrew * hard work for unwired pages only. 2959281494Sandrew */ 2960288445Sandrew pmap_remove_l3(pmap, l3, pv->pv_va, l2, 2961288445Sandrew &free, &lock); 2962281494Sandrew pmap_invalidate_page(pmap, pv->pv_va); 2963281494Sandrew cleared++; 2964281494Sandrew if (pvf == pv) 2965281494Sandrew pvf = NULL; 2966281494Sandrew pv = NULL; 2967281494Sandrew KASSERT(lock == VM_PAGE_TO_PV_LIST_LOCK(m), 2968281494Sandrew ("inconsistent pv lock %p %p for page %p", 2969281494Sandrew lock, VM_PAGE_TO_PV_LIST_LOCK(m), m)); 2970281494Sandrew } else 2971281494Sandrew not_cleared++; 2972281494Sandrew } 2973281494Sandrew PMAP_UNLOCK(pmap); 2974281494Sandrew /* Rotate the PV list if it has more than one entry. */ 2975281494Sandrew if (pv != NULL && TAILQ_NEXT(pv, pv_next) != NULL) { 2976281494Sandrew TAILQ_REMOVE(&m->md.pv_list, pv, pv_next); 2977281494Sandrew TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_next); 2978281494Sandrew m->md.pv_gen++; 2979281494Sandrew } 2980281494Sandrew } while ((pv = TAILQ_FIRST(&m->md.pv_list)) != pvf && cleared + 2981281494Sandrew not_cleared < PMAP_TS_REFERENCED_MAX); 2982281494Sandrewout: 2983281494Sandrew rw_wunlock(lock); 2984281494Sandrew rw_runlock(&pvh_global_lock); 2985281494Sandrew pmap_free_zero_pages(&free); 2986281494Sandrew return (cleared + not_cleared); 2987281494Sandrew} 2988281494Sandrew 2989281494Sandrew/* 2990281494Sandrew * Apply the given advice to the specified range of addresses within the 2991281494Sandrew * given pmap. Depending on the advice, clear the referenced and/or 2992281494Sandrew * modified flags in each mapping and set the mapped page's dirty field. 2993281494Sandrew */ 2994281494Sandrewvoid 2995281494Sandrewpmap_advise(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, int advice) 2996281494Sandrew{ 2997281494Sandrew} 2998281494Sandrew 2999281494Sandrew/* 3000281494Sandrew * Clear the modify bits on the specified physical page. 3001281494Sandrew */ 3002281494Sandrewvoid 3003281494Sandrewpmap_clear_modify(vm_page_t m) 3004281494Sandrew{ 3005281494Sandrew 3006281494Sandrew KASSERT((m->oflags & VPO_UNMANAGED) == 0, 3007281494Sandrew ("pmap_clear_modify: page %p is not managed", m)); 3008281494Sandrew VM_OBJECT_ASSERT_WLOCKED(m->object); 3009281494Sandrew KASSERT(!vm_page_xbusied(m), 3010281494Sandrew ("pmap_clear_modify: page %p is exclusive busied", m)); 3011281494Sandrew 3012281494Sandrew /* 3013281494Sandrew * If the page is not PGA_WRITEABLE, then no PTEs can have PG_M set. 3014281494Sandrew * If the object containing the page is locked and the page is not 3015281494Sandrew * exclusive busied, then PGA_WRITEABLE cannot be concurrently set. 3016281494Sandrew */ 3017281494Sandrew if ((m->aflags & PGA_WRITEABLE) == 0) 3018281494Sandrew return; 3019281846Sandrew 3020286073Semaste /* ARM64TODO: We lack support for tracking if a page is modified */ 3021281494Sandrew} 3022281494Sandrew 3023282221Sandrewvoid * 3024282221Sandrewpmap_mapbios(vm_paddr_t pa, vm_size_t size) 3025282221Sandrew{ 3026282221Sandrew 3027282221Sandrew return ((void *)PHYS_TO_DMAP(pa)); 3028282221Sandrew} 3029282221Sandrew 3030282221Sandrewvoid 3031282221Sandrewpmap_unmapbios(vm_paddr_t pa, vm_size_t size) 3032282221Sandrew{ 3033282221Sandrew} 3034282221Sandrew 3035281494Sandrew/* 3036281494Sandrew * Sets the memory attribute for the specified page. 3037281494Sandrew */ 3038281494Sandrewvoid 3039281494Sandrewpmap_page_set_memattr(vm_page_t m, vm_memattr_t ma) 3040281494Sandrew{ 3041281494Sandrew 3042286080Sandrew m->md.pv_memattr = ma; 3043286080Sandrew 3044286080Sandrew /* 3045286080Sandrew * ARM64TODO: Implement the below (from the amd64 pmap) 3046286080Sandrew * If "m" is a normal page, update its direct mapping. This update 3047286080Sandrew * can be relied upon to perform any cache operations that are 3048286080Sandrew * required for data coherence. 3049286080Sandrew */ 3050286080Sandrew if ((m->flags & PG_FICTITIOUS) == 0 && 3051286080Sandrew PHYS_IN_DMAP(VM_PAGE_TO_PHYS(m))) 3052286080Sandrew panic("ARM64TODO: pmap_page_set_memattr"); 3053281494Sandrew} 3054281494Sandrew 3055281494Sandrew/* 3056281494Sandrew * perform the pmap work for mincore 3057281494Sandrew */ 3058281494Sandrewint 3059281494Sandrewpmap_mincore(pmap_t pmap, vm_offset_t addr, vm_paddr_t *locked_pa) 3060281494Sandrew{ 3061287570Sandrew pd_entry_t *l1p, l1; 3062287570Sandrew pd_entry_t *l2p, l2; 3063287570Sandrew pt_entry_t *l3p, l3; 3064287570Sandrew vm_paddr_t pa; 3065287570Sandrew bool managed; 3066287570Sandrew int val; 3067281494Sandrew 3068287570Sandrew PMAP_LOCK(pmap); 3069287570Sandrewretry: 3070287570Sandrew pa = 0; 3071287570Sandrew val = 0; 3072287570Sandrew managed = false; 3073287570Sandrew 3074287570Sandrew l1p = pmap_l1(pmap, addr); 3075287570Sandrew if (l1p == NULL) /* No l1 */ 3076287570Sandrew goto done; 3077295425Swma 3078287570Sandrew l1 = pmap_load(l1p); 3079295425Swma if ((l1 & ATTR_DESCR_MASK) == L1_INVAL) 3080295425Swma goto done; 3081295425Swma 3082287570Sandrew if ((l1 & ATTR_DESCR_MASK) == L1_BLOCK) { 3083287570Sandrew pa = (l1 & ~ATTR_MASK) | (addr & L1_OFFSET); 3084287570Sandrew managed = (l1 & ATTR_SW_MANAGED) == ATTR_SW_MANAGED; 3085287570Sandrew val = MINCORE_SUPER | MINCORE_INCORE; 3086287570Sandrew if (pmap_page_dirty(l1)) 3087287570Sandrew val |= MINCORE_MODIFIED | MINCORE_MODIFIED_OTHER; 3088287570Sandrew if ((l1 & ATTR_AF) == ATTR_AF) 3089287570Sandrew val |= MINCORE_REFERENCED | MINCORE_REFERENCED_OTHER; 3090287570Sandrew goto done; 3091287570Sandrew } 3092287570Sandrew 3093287570Sandrew l2p = pmap_l1_to_l2(l1p, addr); 3094287570Sandrew if (l2p == NULL) /* No l2 */ 3095287570Sandrew goto done; 3096295425Swma 3097287570Sandrew l2 = pmap_load(l2p); 3098295425Swma if ((l2 & ATTR_DESCR_MASK) == L2_INVAL) 3099295425Swma goto done; 3100295425Swma 3101287570Sandrew if ((l2 & ATTR_DESCR_MASK) == L2_BLOCK) { 3102287570Sandrew pa = (l2 & ~ATTR_MASK) | (addr & L2_OFFSET); 3103287570Sandrew managed = (l2 & ATTR_SW_MANAGED) == ATTR_SW_MANAGED; 3104287570Sandrew val = MINCORE_SUPER | MINCORE_INCORE; 3105287570Sandrew if (pmap_page_dirty(l2)) 3106287570Sandrew val |= MINCORE_MODIFIED | MINCORE_MODIFIED_OTHER; 3107287570Sandrew if ((l2 & ATTR_AF) == ATTR_AF) 3108287570Sandrew val |= MINCORE_REFERENCED | MINCORE_REFERENCED_OTHER; 3109287570Sandrew goto done; 3110287570Sandrew } 3111287570Sandrew 3112287570Sandrew l3p = pmap_l2_to_l3(l2p, addr); 3113287570Sandrew if (l3p == NULL) /* No l3 */ 3114287570Sandrew goto done; 3115295425Swma 3116287570Sandrew l3 = pmap_load(l2p); 3117295425Swma if ((l3 & ATTR_DESCR_MASK) == L3_INVAL) 3118295425Swma goto done; 3119295425Swma 3120287570Sandrew if ((l3 & ATTR_DESCR_MASK) == L3_PAGE) { 3121287570Sandrew pa = (l3 & ~ATTR_MASK) | (addr & L3_OFFSET); 3122287570Sandrew managed = (l3 & ATTR_SW_MANAGED) == ATTR_SW_MANAGED; 3123287570Sandrew val = MINCORE_INCORE; 3124287570Sandrew if (pmap_page_dirty(l3)) 3125287570Sandrew val |= MINCORE_MODIFIED | MINCORE_MODIFIED_OTHER; 3126287570Sandrew if ((l3 & ATTR_AF) == ATTR_AF) 3127287570Sandrew val |= MINCORE_REFERENCED | MINCORE_REFERENCED_OTHER; 3128287570Sandrew } 3129287570Sandrew 3130287570Sandrewdone: 3131287570Sandrew if ((val & (MINCORE_MODIFIED_OTHER | MINCORE_REFERENCED_OTHER)) != 3132287570Sandrew (MINCORE_MODIFIED_OTHER | MINCORE_REFERENCED_OTHER) && managed) { 3133287570Sandrew /* Ensure that "PHYS_TO_VM_PAGE(pa)->object" doesn't change. */ 3134287570Sandrew if (vm_page_pa_tryrelock(pmap, pa, locked_pa)) 3135287570Sandrew goto retry; 3136287570Sandrew } else 3137287570Sandrew PA_UNLOCK_COND(*locked_pa); 3138287570Sandrew PMAP_UNLOCK(pmap); 3139287570Sandrew 3140287570Sandrew return (val); 3141281494Sandrew} 3142281494Sandrew 3143281494Sandrewvoid 3144281494Sandrewpmap_activate(struct thread *td) 3145281494Sandrew{ 3146281494Sandrew pmap_t pmap; 3147281494Sandrew 3148281494Sandrew critical_enter(); 3149281494Sandrew pmap = vmspace_pmap(td->td_proc->p_vmspace); 3150281494Sandrew td->td_pcb->pcb_l1addr = vtophys(pmap->pm_l1); 3151281494Sandrew __asm __volatile("msr ttbr0_el1, %0" : : "r"(td->td_pcb->pcb_l1addr)); 3152285212Sandrew pmap_invalidate_all(pmap); 3153281494Sandrew critical_exit(); 3154281494Sandrew} 3155281494Sandrew 3156281494Sandrewvoid 3157287105Sandrewpmap_sync_icache(pmap_t pmap, vm_offset_t va, vm_size_t sz) 3158281494Sandrew{ 3159281494Sandrew 3160287105Sandrew if (va >= VM_MIN_KERNEL_ADDRESS) { 3161287105Sandrew cpu_icache_sync_range(va, sz); 3162287105Sandrew } else { 3163287105Sandrew u_int len, offset; 3164287105Sandrew vm_paddr_t pa; 3165287105Sandrew 3166287105Sandrew /* Find the length of data in this page to flush */ 3167287105Sandrew offset = va & PAGE_MASK; 3168287105Sandrew len = imin(PAGE_SIZE - offset, sz); 3169287105Sandrew 3170287105Sandrew while (sz != 0) { 3171287105Sandrew /* Extract the physical address & find it in the DMAP */ 3172287105Sandrew pa = pmap_extract(pmap, va); 3173287105Sandrew if (pa != 0) 3174287105Sandrew cpu_icache_sync_range(PHYS_TO_DMAP(pa), len); 3175287105Sandrew 3176287105Sandrew /* Move to the next page */ 3177287105Sandrew sz -= len; 3178287105Sandrew va += len; 3179287105Sandrew /* Set the length for the next iteration */ 3180287105Sandrew len = imin(PAGE_SIZE, sz); 3181287105Sandrew } 3182287105Sandrew } 3183281494Sandrew} 3184281494Sandrew 3185281494Sandrew/* 3186281494Sandrew * Increase the starting virtual address of the given mapping if a 3187281494Sandrew * different alignment might result in more superpage mappings. 3188281494Sandrew */ 3189281494Sandrewvoid 3190281494Sandrewpmap_align_superpage(vm_object_t object, vm_ooffset_t offset, 3191281494Sandrew vm_offset_t *addr, vm_size_t size) 3192281494Sandrew{ 3193281494Sandrew} 3194281494Sandrew 3195281494Sandrew/** 3196281494Sandrew * Get the kernel virtual address of a set of physical pages. If there are 3197281494Sandrew * physical addresses not covered by the DMAP perform a transient mapping 3198281494Sandrew * that will be removed when calling pmap_unmap_io_transient. 3199281494Sandrew * 3200281494Sandrew * \param page The pages the caller wishes to obtain the virtual 3201281494Sandrew * address on the kernel memory map. 3202281494Sandrew * \param vaddr On return contains the kernel virtual memory address 3203281494Sandrew * of the pages passed in the page parameter. 3204281494Sandrew * \param count Number of pages passed in. 3205281494Sandrew * \param can_fault TRUE if the thread using the mapped pages can take 3206281494Sandrew * page faults, FALSE otherwise. 3207281494Sandrew * 3208281494Sandrew * \returns TRUE if the caller must call pmap_unmap_io_transient when 3209281494Sandrew * finished or FALSE otherwise. 3210281494Sandrew * 3211281494Sandrew */ 3212281494Sandrewboolean_t 3213281494Sandrewpmap_map_io_transient(vm_page_t page[], vm_offset_t vaddr[], int count, 3214281494Sandrew boolean_t can_fault) 3215281494Sandrew{ 3216281494Sandrew vm_paddr_t paddr; 3217281494Sandrew boolean_t needs_mapping; 3218281494Sandrew int error, i; 3219281494Sandrew 3220281494Sandrew /* 3221281494Sandrew * Allocate any KVA space that we need, this is done in a separate 3222281494Sandrew * loop to prevent calling vmem_alloc while pinned. 3223281494Sandrew */ 3224281494Sandrew needs_mapping = FALSE; 3225281494Sandrew for (i = 0; i < count; i++) { 3226281494Sandrew paddr = VM_PAGE_TO_PHYS(page[i]); 3227281494Sandrew if (__predict_false(paddr >= DMAP_MAX_PHYSADDR)) { 3228281494Sandrew error = vmem_alloc(kernel_arena, PAGE_SIZE, 3229281494Sandrew M_BESTFIT | M_WAITOK, &vaddr[i]); 3230281494Sandrew KASSERT(error == 0, ("vmem_alloc failed: %d", error)); 3231281494Sandrew needs_mapping = TRUE; 3232281494Sandrew } else { 3233281494Sandrew vaddr[i] = PHYS_TO_DMAP(paddr); 3234281494Sandrew } 3235281494Sandrew } 3236281494Sandrew 3237281494Sandrew /* Exit early if everything is covered by the DMAP */ 3238281494Sandrew if (!needs_mapping) 3239281494Sandrew return (FALSE); 3240281494Sandrew 3241281494Sandrew if (!can_fault) 3242281494Sandrew sched_pin(); 3243281494Sandrew for (i = 0; i < count; i++) { 3244281494Sandrew paddr = VM_PAGE_TO_PHYS(page[i]); 3245281494Sandrew if (paddr >= DMAP_MAX_PHYSADDR) { 3246281494Sandrew panic( 3247281494Sandrew "pmap_map_io_transient: TODO: Map out of DMAP data"); 3248281494Sandrew } 3249281494Sandrew } 3250281494Sandrew 3251281494Sandrew return (needs_mapping); 3252281494Sandrew} 3253281494Sandrew 3254281494Sandrewvoid 3255281494Sandrewpmap_unmap_io_transient(vm_page_t page[], vm_offset_t vaddr[], int count, 3256281494Sandrew boolean_t can_fault) 3257281494Sandrew{ 3258281494Sandrew vm_paddr_t paddr; 3259281494Sandrew int i; 3260281494Sandrew 3261281494Sandrew if (!can_fault) 3262281494Sandrew sched_unpin(); 3263281494Sandrew for (i = 0; i < count; i++) { 3264281494Sandrew paddr = VM_PAGE_TO_PHYS(page[i]); 3265281494Sandrew if (paddr >= DMAP_MAX_PHYSADDR) { 3266286073Semaste panic("ARM64TODO: pmap_unmap_io_transient: Unmap data"); 3267281494Sandrew } 3268281494Sandrew } 3269281494Sandrew} 3270