pmap.c revision 15340
118316Swollman/* 218316Swollman * Copyright (c) 1991 Regents of the University of California. 318316Swollman * All rights reserved. 418316Swollman * Copyright (c) 1994 John S. Dyson 518316Swollman * All rights reserved. 618316Swollman * Copyright (c) 1994 David Greenman 718316Swollman * All rights reserved. 818316Swollman * 918316Swollman * This code is derived from software contributed to Berkeley by 1018316Swollman * the Systems Programming Group of the University of Utah Computer 1118316Swollman * Science Department and William Jolitz of UUNET Technologies Inc. 1218316Swollman * 1318316Swollman * Redistribution and use in source and binary forms, with or without 1418316Swollman * modification, are permitted provided that the following conditions 1518316Swollman * are met: 1618316Swollman * 1. Redistributions of source code must retain the above copyright 1718316Swollman * notice, this list of conditions and the following disclaimer. 1818316Swollman * 2. Redistributions in binary form must reproduce the above copyright 1918316Swollman * notice, this list of conditions and the following disclaimer in the 2018316Swollman * documentation and/or other materials provided with the distribution. 2118316Swollman * 3. All advertising materials mentioning features or use of this software 2218316Swollman * must display the following acknowledgement: 2318316Swollman * This product includes software developed by the University of 2418316Swollman * California, Berkeley and its contributors. 2518316Swollman * 4. Neither the name of the University nor the names of its contributors 2618316Swollman * may be used to endorse or promote products derived from this software 2718316Swollman * without specific prior written permission. 2818316Swollman * 2918316Swollman * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 3018316Swollman * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 3118316Swollman * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 3218316Swollman * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 3318316Swollman * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 3418316Swollman * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 3518316Swollman * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 3618316Swollman * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 3718316Swollman * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 3818316Swollman * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 3918316Swollman * SUCH DAMAGE. 4018316Swollman * 4118316Swollman * from: @(#)pmap.c 7.7 (Berkeley) 5/12/91 4218316Swollman * $Id: pmap.c,v 1.85 1996/04/07 02:23:05 dyson Exp $ 4318316Swollman */ 4418316Swollman 4518316Swollman/* 4618316Swollman * Derived from hp300 version by Mike Hibler, this version by William 4718316Swollman * Jolitz uses a recursive map [a pde points to the page directory] to 4818316Swollman * map the page tables using the pagetables themselves. This is done to 4918316Swollman * reduce the impact on kernel virtual memory for lots of sparse address 5018316Swollman * space, and to reduce the cost of memory to each process. 5118316Swollman * 5218316Swollman * Derived from: hp300/@(#)pmap.c 7.1 (Berkeley) 12/5/90 5318316Swollman */ 5418316Swollman 5518316Swollman/* 5618316Swollman * Manages physical address maps. 5718316Swollman * 5818316Swollman * In addition to hardware address maps, this 5918316Swollman * module is called upon to provide software-use-only 6018316Swollman * maps which may or may not be stored in the same 6118316Swollman * form as hardware maps. These pseudo-maps are 6218316Swollman * used to store intermediate results from copy 6318316Swollman * operations to and from address spaces. 6418316Swollman * 6518316Swollman * Since the information managed by this module is 6618316Swollman * also stored by the logical address mapping module, 6718316Swollman * this module may throw away valid virtual-to-physical 6818316Swollman * mappings at almost any time. However, invalidations 6918316Swollman * of virtual-to-physical mappings must be done as 7018316Swollman * requested. 7118316Swollman * 7218316Swollman * In order to cope with hardware architectures which 7318316Swollman * make virtual-to-physical map invalidates expensive, 7418316Swollman * this module may delay invalidate or reduced protection 7518316Swollman * operations until such time as they are actually 7618316Swollman * necessary. This module is given full information as 7718316Swollman * to which processors are currently using which maps, 7818316Swollman * and to when physical maps must be made correct. 7918316Swollman */ 8018316Swollman 8118316Swollman#include <sys/param.h> 8218316Swollman#include <sys/systm.h> 8318316Swollman#include <sys/proc.h> 8418316Swollman#include <sys/malloc.h> 8518316Swollman#include <sys/msgbuf.h> 8618316Swollman#include <sys/queue.h> 8718316Swollman#include <sys/vmmeter.h> 8818316Swollman 8918316Swollman#include <vm/vm.h> 9018316Swollman#include <vm/vm_param.h> 9118316Swollman#include <vm/vm_prot.h> 9218316Swollman#include <vm/lock.h> 9318316Swollman#include <vm/vm_kern.h> 9418316Swollman#include <vm/vm_page.h> 9518316Swollman#include <vm/vm_map.h> 9618316Swollman#include <vm/vm_object.h> 9718316Swollman#include <vm/vm_extern.h> 9818316Swollman 9918316Swollman#include <machine/pcb.h> 10018316Swollman#include <machine/cputypes.h> 10118316Swollman#include <machine/md_var.h> 10218316Swollman 10318316Swollman#include <i386/isa/isa.h> 10418316Swollman 10518316Swollman#define PMAP_KEEP_PDIRS 10618316Swollman 10718316Swollman#if defined(DIAGNOSTIC) 10818316Swollman#define PMAP_DIAGNOSTIC 10918316Swollman#endif 11018316Swollman 11118316Swollmanstatic void init_pv_entries __P((int)); 11218316Swollman 11318316Swollman/* 11418316Swollman * Get PDEs and PTEs for user/kernel address space 11518316Swollman */ 11618316Swollman#define pmap_pde(m, v) (&((m)->pm_pdir[((vm_offset_t)(v) >> PD_SHIFT)&1023])) 11718316Swollman#define pdir_pde(m, v) (m[((vm_offset_t)(v) >> PD_SHIFT)&1023]) 11818316Swollman 11918316Swollman#define pmap_pte_pa(pte) (*(int *)(pte) & PG_FRAME) 12018316Swollman 12118316Swollman#define pmap_pde_v(pte) ((*(int *)pte & PG_V) != 0) 12218316Swollman#define pmap_pte_w(pte) ((*(int *)pte & PG_W) != 0) 12318316Swollman#define pmap_pte_m(pte) ((*(int *)pte & PG_M) != 0) 12418316Swollman#define pmap_pte_u(pte) ((*(int *)pte & PG_U) != 0) 12518316Swollman#define pmap_pte_v(pte) ((*(int *)pte & PG_V) != 0) 12618316Swollman 12718316Swollman#define pmap_pte_set_w(pte, v) ((v)?(*(int *)pte |= PG_W):(*(int *)pte &= ~PG_W)) 12818316Swollman#define pmap_pte_set_prot(pte, v) ((*(int *)pte &= ~PG_PROT), (*(int *)pte |= (v))) 12918316Swollman 13018316Swollman/* 13118316Swollman * Given a map and a machine independent protection code, 13218316Swollman * convert to a vax protection code. 13318316Swollman */ 13418316Swollman#define pte_prot(m, p) (protection_codes[p]) 13518316Swollmanstatic int protection_codes[8]; 13618316Swollman 13718316Swollmanstatic struct pmap kernel_pmap_store; 13818316Swollmanpmap_t kernel_pmap; 13918316Swollman 14018316Swollmanvm_offset_t avail_start; /* PA of first available physical page */ 14118316Swollmanvm_offset_t avail_end; /* PA of last available physical page */ 14218316Swollmanvm_offset_t virtual_avail; /* VA of first avail page (after kernel bss) */ 14318316Swollmanvm_offset_t virtual_end; /* VA of last avail page (end of kernel AS) */ 14418316Swollmanstatic boolean_t pmap_initialized = FALSE; /* Has pmap_init completed? */ 14518316Swollmanstatic vm_offset_t vm_first_phys; 14618316Swollman 14718316Swollmanstatic int nkpt; 14818316Swollman 14918316Swollmanextern vm_offset_t clean_sva, clean_eva; 15018316Swollmanextern int cpu_class; 15118316Swollman 15218316Swollman/* 15318316Swollman * All those kernel PT submaps that BSD is so fond of 15418316Swollman */ 15518316Swollmanpt_entry_t *CMAP1; 15618316Swollmanstatic pt_entry_t *CMAP2, *ptmmap; 15718316Swollmanstatic pv_entry_t pv_table; 15818316Swollmancaddr_t CADDR1, ptvmmap; 15918316Swollmanstatic caddr_t CADDR2; 16018316Swollmanstatic pt_entry_t *msgbufmap; 16118316Swollmanstruct msgbuf *msgbufp; 16218316Swollman 16318316Swollmanstatic void free_pv_entry __P((pv_entry_t pv)); 16418316Swollmanpt_entry_t * 16518316Swollman get_ptbase __P((pmap_t pmap)); 16618316Swollmanstatic pv_entry_t 16718316Swollman get_pv_entry __P((void)); 16818316Swollmanstatic void i386_protection_init __P((void)); 16918316Swollmanstatic void pmap_alloc_pv_entry __P((void)); 17018316Swollmanstatic void pmap_changebit __P((vm_offset_t pa, int bit, boolean_t setem)); 17118316Swollmanstatic void pmap_enter_quick __P((pmap_t pmap, vm_offset_t va, 17218316Swollman vm_offset_t pa)); 17318316Swollmanstatic int pmap_is_managed __P((vm_offset_t pa)); 17418316Swollmanstatic void pmap_remove_all __P((vm_offset_t pa)); 17518316Swollmanstatic void pmap_remove_page __P((struct pmap *pmap, vm_offset_t va)); 17618316Swollmanstatic __inline void pmap_remove_entry __P((struct pmap *pmap, pv_entry_t pv, 17718316Swollman vm_offset_t va)); 17818316Swollmanstatic void pmap_remove_pte __P((struct pmap *pmap, pt_entry_t *ptq, 17918316Swollman vm_offset_t sva)); 18018316Swollmanstatic vm_page_t 18118316Swollman pmap_pte_vm_page __P((pmap_t pmap, vm_offset_t pt)); 18218316Swollmanstatic boolean_t 18318316Swollman pmap_testbit __P((vm_offset_t pa, int bit)); 18418316Swollmanstatic void * pmap_getpdir __P((void)); 18518316Swollman 18618316Swollman 18718316Swollman#if defined(PMAP_DIAGNOSTIC) 18818316Swollman 18918316Swollman/* 19018316Swollman * This code checks for non-writeable/modified pages. 19118316Swollman * This should be an invalid condition. 19218316Swollman */ 19318316Swollmanstatic int 19418316Swollmanpmap_nw_modified(pt_entry_t ptea) { 19518316Swollman int pte; 19618316Swollman 19718316Swollman pte = (int) ptea; 19818316Swollman 19918316Swollman if ((pte & (PG_M|PG_RW)) == PG_M) 20018316Swollman return 1; 20118316Swollman else 20218316Swollman return 0; 20318316Swollman} 20418316Swollman#endif 20518316Swollman 20618316Swollman/* 20718316Swollman * The below are finer grained pmap_update routines. These eliminate 20818316Swollman * the gratuitious tlb flushes on non-i386 architectures. 20918316Swollman */ 21018316Swollmanstatic __inline void 21118316Swollmanpmap_update_1pg( vm_offset_t va) { 21218316Swollman#if defined(I386_CPU) 21318316Swollman if (cpu_class == CPUCLASS_386) 21418316Swollman pmap_update(); 21518316Swollman else 21618316Swollman#endif 21718316Swollman __asm __volatile(".byte 0xf,0x1,0x38": :"a" (va)); 21818316Swollman} 21918316Swollman 22018316Swollmanstatic __inline void 22118316Swollmanpmap_update_2pg( vm_offset_t va1, vm_offset_t va2) { 22218316Swollman#if defined(I386_CPU) 22318316Swollman if (cpu_class == CPUCLASS_386) { 22418316Swollman pmap_update(); 22518316Swollman } else 22618316Swollman#endif 22718316Swollman { 22818316Swollman __asm __volatile(".byte 0xf,0x1,0x38": :"a" (va1)); 22918316Swollman __asm __volatile(".byte 0xf,0x1,0x38": :"a" (va2)); 23018316Swollman } 23118316Swollman} 23218316Swollman 23318316Swollman/* 23418316Swollman * Routine: pmap_pte 23518316Swollman * Function: 23618316Swollman * Extract the page table entry associated 23718316Swollman * with the given map/virtual_address pair. 23818316Swollman * [ what about induced faults -wfj] 23918316Swollman */ 24018316Swollman 24118316Swollman__inline pt_entry_t * __pure 24218316Swollmanpmap_pte(pmap, va) 24318316Swollman register pmap_t pmap; 24418316Swollman vm_offset_t va; 24518316Swollman{ 24618316Swollman 24718316Swollman if (pmap && *pmap_pde(pmap, va)) { 24818316Swollman vm_offset_t frame = (int) pmap->pm_pdir[PTDPTDI] & PG_FRAME; 24918316Swollman 25018316Swollman /* are we current address space or kernel? */ 25118316Swollman if ((pmap == kernel_pmap) || (frame == ((int) PTDpde & PG_FRAME))) 25218316Swollman return ((pt_entry_t *) vtopte(va)); 25318316Swollman /* otherwise, we are alternate address space */ 25418316Swollman else { 25518316Swollman if (frame != ((int) APTDpde & PG_FRAME)) { 25618316Swollman APTDpde = pmap->pm_pdir[PTDPTDI]; 25718316Swollman pmap_update(); 25818316Swollman } 25918316Swollman return ((pt_entry_t *) avtopte(va)); 26018316Swollman } 26118316Swollman } 26218316Swollman return (0); 26318316Swollman} 26418316Swollman 26518316Swollman/* 26618316Swollman * Routine: pmap_extract 26718316Swollman * Function: 26818316Swollman * Extract the physical page address associated 26918316Swollman * with the given map/virtual_address pair. 27018316Swollman */ 27118316Swollman 27218316Swollmanvm_offset_t 27318316Swollmanpmap_extract(pmap, va) 27418316Swollman register pmap_t pmap; 27518316Swollman vm_offset_t va; 27618316Swollman{ 27718316Swollman vm_offset_t pa; 27818316Swollman 27918316Swollman if (pmap && *pmap_pde(pmap, va)) { 28018316Swollman vm_offset_t frame = (int) pmap->pm_pdir[PTDPTDI] & PG_FRAME; 28118316Swollman 28218316Swollman /* are we current address space or kernel? */ 28318316Swollman if ((pmap == kernel_pmap) 28418316Swollman || (frame == ((int) PTDpde & PG_FRAME))) { 28518316Swollman pa = *(int *) vtopte(va); 28618316Swollman /* otherwise, we are alternate address space */ 28718316Swollman } else { 28818316Swollman if (frame != ((int) APTDpde & PG_FRAME)) { 28918316Swollman APTDpde = pmap->pm_pdir[PTDPTDI]; 29018316Swollman pmap_update(); 29118316Swollman } 29218316Swollman pa = *(int *) avtopte(va); 29318316Swollman } 29418316Swollman return ((pa & PG_FRAME) | (va & ~PG_FRAME)); 29518316Swollman } 29618316Swollman return 0; 29718316Swollman 29818316Swollman} 29918316Swollman 30018316Swollman/* 30118316Swollman * determine if a page is managed (memory vs. device) 30218316Swollman */ 30318316Swollmanstatic __inline int 30418316Swollmanpmap_is_managed(pa) 30518316Swollman vm_offset_t pa; 30618316Swollman{ 30718316Swollman int i; 30818316Swollman 30918316Swollman if (!pmap_initialized) 31018316Swollman return 0; 31118316Swollman 31218316Swollman for (i = 0; phys_avail[i + 1]; i += 2) { 31318316Swollman if (pa < phys_avail[i + 1] && pa >= phys_avail[i]) 31418316Swollman return 1; 31518316Swollman } 31618316Swollman return 0; 31718316Swollman} 31818316Swollman 31918316Swollmanvm_page_t 32018316Swollmanpmap_use_pt(pmap, va) 32118316Swollman pmap_t pmap; 32218316Swollman vm_offset_t va; 32318316Swollman{ 32418316Swollman vm_offset_t ptepa; 32518316Swollman vm_page_t mpte; 32618316Swollman 32718316Swollman if (va >= UPT_MIN_ADDRESS) 32818316Swollman return NULL; 32918316Swollman 33018316Swollman ptepa = ((vm_offset_t) *pmap_pde(pmap, va)) & PG_FRAME; 33118316Swollman#if defined(PMAP_DIAGNOSTIC) 33218316Swollman if (!ptepa) 33318316Swollman panic("pmap_use_pt: pagetable page missing, va: 0x%x", va); 33418316Swollman#endif 33518316Swollman 33618316Swollman mpte = PHYS_TO_VM_PAGE(ptepa); 33718316Swollman ++mpte->hold_count; 33818316Swollman return mpte; 33918316Swollman} 34018316Swollman 34118316Swollman#if !defined(PMAP_DIAGNOSTIC) 34218316Swollman__inline 34318316Swollman#endif 34418316Swollmanvoid 34518316Swollmanpmap_unuse_pt(pmap, va, mpte) 34618316Swollman pmap_t pmap; 34718316Swollman vm_offset_t va; 34818316Swollman vm_page_t mpte; 34918316Swollman{ 35018316Swollman if (va >= UPT_MIN_ADDRESS) 35118316Swollman return; 35218316Swollman 35318316Swollman if (mpte == NULL) { 35418316Swollman vm_offset_t ptepa; 35518316Swollman ptepa = ((vm_offset_t) *pmap_pde(pmap, va)) & PG_FRAME; 35618316Swollman#if defined(PMAP_DIAGNOSTIC) 35718316Swollman if (!ptepa) 35818316Swollman panic("pmap_unuse_pt: pagetable page missing, va: 0x%x", va); 35918316Swollman#endif 36018316Swollman mpte = PHYS_TO_VM_PAGE(ptepa); 36118316Swollman } 36218316Swollman 36318316Swollman#if defined(PMAP_DIAGNOSTIC) 36418316Swollman if (mpte->hold_count == 0) { 36518316Swollman panic("pmap_unuse_pt: hold count < 0, va: 0x%x", va); 36618316Swollman } 36718316Swollman#endif 36818316Swollman 36918316Swollman vm_page_unhold(mpte); 37018316Swollman 37118316Swollman if ((mpte->hold_count == 0) && 37218316Swollman (mpte->wire_count == 0) && 37318316Swollman (pmap != kernel_pmap) && 37418316Swollman (va < KPT_MIN_ADDRESS)) { 37518316Swollman/* 37618316Swollman * We don't free page-table-pages anymore because it can have a negative 37718316Swollman * impact on perf at times. Now we just deactivate, and it'll get cleaned 37818316Swollman * up if needed... Also, if the page ends up getting used, it will fault 37918316Swollman * back into the process address space and be reactivated. 38018316Swollman */ 38118316Swollman#if defined(PMAP_FREE_OLD_PTES) 38218316Swollman pmap_page_protect(VM_PAGE_TO_PHYS(mpte), VM_PROT_NONE); 38318316Swollman vm_page_free(mpte); 38418316Swollman#else 38518316Swollman mpte->dirty = 0; 38618316Swollman vm_page_deactivate(mpte); 38718316Swollman#endif 38818316Swollman } 38918316Swollman} 39018316Swollman 39118316Swollman/* 39218316Swollman * Bootstrap the system enough to run with virtual memory. 39318316Swollman * 39418316Swollman * On the i386 this is called after mapping has already been enabled 39518316Swollman * and just syncs the pmap module with what has already been done. 39618316Swollman * [We can't call it easily with mapping off since the kernel is not 39718316Swollman * mapped with PA == VA, hence we would have to relocate every address 39818316Swollman * from the linked base (virtual) address "KERNBASE" to the actual 39918316Swollman * (physical) address starting relative to 0] 40018316Swollman */ 40118316Swollmanvoid 40218316Swollmanpmap_bootstrap(firstaddr, loadaddr) 40318316Swollman vm_offset_t firstaddr; 40418316Swollman vm_offset_t loadaddr; 40518316Swollman{ 40618316Swollman vm_offset_t va; 40718316Swollman pt_entry_t *pte; 40818316Swollman 40918316Swollman avail_start = firstaddr; 41018316Swollman 41118316Swollman /* 41218316Swollman * XXX The calculation of virtual_avail is wrong. It's NKPT*PAGE_SIZE too 41318316Swollman * large. It should instead be correctly calculated in locore.s and 41418316Swollman * not based on 'first' (which is a physical address, not a virtual 41518316Swollman * address, for the start of unused physical memory). The kernel 41618316Swollman * page tables are NOT double mapped and thus should not be included 41718316Swollman * in this calculation. 41818316Swollman */ 41918316Swollman virtual_avail = (vm_offset_t) KERNBASE + firstaddr; 42018316Swollman virtual_end = VM_MAX_KERNEL_ADDRESS; 42118316Swollman 42218316Swollman /* 42318316Swollman * Initialize protection array. 42418316Swollman */ 42518316Swollman i386_protection_init(); 42618316Swollman 42718316Swollman /* 42818316Swollman * The kernel's pmap is statically allocated so we don't have to use 42918316Swollman * pmap_create, which is unlikely to work correctly at this part of 43018316Swollman * the boot sequence (XXX and which no longer exists). 43118316Swollman */ 43218316Swollman kernel_pmap = &kernel_pmap_store; 43318316Swollman 43418316Swollman kernel_pmap->pm_pdir = (pd_entry_t *) (KERNBASE + IdlePTD); 43518316Swollman 43618316Swollman kernel_pmap->pm_count = 1; 43718316Swollman nkpt = NKPT; 43818316Swollman 43918316Swollman /* 44018316Swollman * Reserve some special page table entries/VA space for temporary 44118316Swollman * mapping of pages. 44218316Swollman */ 44318316Swollman#define SYSMAP(c, p, v, n) \ 44418316Swollman v = (c)va; va += ((n)*PAGE_SIZE); p = pte; pte += (n); 44518316Swollman 44618316Swollman va = virtual_avail; 44718316Swollman pte = pmap_pte(kernel_pmap, va); 44818316Swollman 44918316Swollman /* 45018316Swollman * CMAP1/CMAP2 are used for zeroing and copying pages. 45118316Swollman */ 45218316Swollman SYSMAP(caddr_t, CMAP1, CADDR1, 1) 45318316Swollman SYSMAP(caddr_t, CMAP2, CADDR2, 1) 45418316Swollman 45518316Swollman /* 45618316Swollman * ptmmap is used for reading arbitrary physical pages via /dev/mem. 45718316Swollman */ 45818316Swollman SYSMAP(caddr_t, ptmmap, ptvmmap, 1) 45918316Swollman 46018316Swollman /* 46118316Swollman * msgbufmap is used to map the system message buffer. 46218316Swollman */ 46318316Swollman SYSMAP(struct msgbuf *, msgbufmap, msgbufp, 1) 46418316Swollman 46518316Swollman virtual_avail = va; 46618316Swollman 46718316Swollman *(int *) CMAP1 = *(int *) CMAP2 = *(int *) PTD = 0; 46818316Swollman pmap_update(); 46918316Swollman} 47018316Swollman 47118316Swollman/* 47218316Swollman * Initialize the pmap module. 47318316Swollman * Called by vm_init, to initialize any structures that the pmap 47418316Swollman * system needs to map virtual memory. 47518316Swollman * pmap_init has been enhanced to support in a fairly consistant 47618316Swollman * way, discontiguous physical memory. 47718316Swollman */ 47818316Swollmanvoid 47918316Swollmanpmap_init(phys_start, phys_end) 48018316Swollman vm_offset_t phys_start, phys_end; 48118316Swollman{ 48218316Swollman vm_offset_t addr; 48318316Swollman vm_size_t npg, s; 48418316Swollman int i; 48518316Swollman 48618316Swollman /* 48718316Swollman * calculate the number of pv_entries needed 48818316Swollman */ 48918316Swollman vm_first_phys = phys_avail[0]; 49018316Swollman for (i = 0; phys_avail[i + 1]; i += 2); 49118316Swollman npg = (phys_avail[(i - 2) + 1] - vm_first_phys) / PAGE_SIZE; 49218316Swollman 49318316Swollman /* 49418316Swollman * Allocate memory for random pmap data structures. Includes the 49518316Swollman * pv_head_table. 49618316Swollman */ 49718316Swollman s = (vm_size_t) (sizeof(struct pv_entry) * npg); 49818316Swollman s = round_page(s); 49918316Swollman addr = (vm_offset_t) kmem_alloc(kernel_map, s); 50018316Swollman pv_table = (pv_entry_t) addr; 50118316Swollman 50218316Swollman /* 50318316Swollman * init the pv free list 50418316Swollman */ 50518316Swollman init_pv_entries(npg); 50618316Swollman /* 50718316Swollman * Now it is safe to enable pv_table recording. 50818316Swollman */ 50918316Swollman pmap_initialized = TRUE; 51018316Swollman} 51118316Swollman 51218316Swollman/* 51318316Swollman * Used to map a range of physical addresses into kernel 51418316Swollman * virtual address space. 51518316Swollman * 51618316Swollman * For now, VM is already on, we only need to map the 51718316Swollman * specified memory. 51818316Swollman */ 51918316Swollmanvm_offset_t 52018316Swollmanpmap_map(virt, start, end, prot) 52118316Swollman vm_offset_t virt; 52218316Swollman vm_offset_t start; 52318316Swollman vm_offset_t end; 52418316Swollman int prot; 52518316Swollman{ 52618316Swollman while (start < end) { 52718316Swollman pmap_enter(kernel_pmap, virt, start, prot, FALSE); 52818316Swollman virt += PAGE_SIZE; 52918316Swollman start += PAGE_SIZE; 53018316Swollman } 53118316Swollman return (virt); 53218316Swollman} 53318316Swollman 53418316Swollman#if defined(PMAP_KEEP_PDIRS) 53518316Swollmanint nfreepdir; 53618316Swollmancaddr_t *pdirlist; 53718316Swollman#define NFREEPDIR 3 53818316Swollman 53918316Swollmanstatic void * 54018316Swollmanpmap_getpdir() { 54118316Swollman caddr_t *pdir; 54218316Swollman if (pdirlist) { 54318316Swollman --nfreepdir; 54418316Swollman pdir = pdirlist; 54518316Swollman pdirlist = (caddr_t *) *pdir; 54618316Swollman *pdir = 0; 54718316Swollman#if 0 /* Not needed anymore */ 54818316Swollman bzero( (caddr_t) pdir, PAGE_SIZE); 54918316Swollman#endif 55018316Swollman } else { 55118316Swollman pdir = (caddr_t *) kmem_alloc(kernel_map, PAGE_SIZE); 55218316Swollman } 55318316Swollman 55418316Swollman return (void *) pdir; 55518316Swollman} 55618316Swollman 55718316Swollmanstatic void 55818316Swollmanpmap_freepdir(void *pdir) { 55918316Swollman if (nfreepdir > NFREEPDIR) { 56018316Swollman kmem_free(kernel_map, (vm_offset_t) pdir, PAGE_SIZE); 56118316Swollman } else { 56218316Swollman int i; 56318316Swollman pt_entry_t *s; 56418316Swollman s = (pt_entry_t *) pdir; 56518316Swollman 56618316Swollman /* 56718316Swollman * remove wired in kernel mappings 56818316Swollman */ 56918316Swollman bzero(s + KPTDI, nkpt * PTESIZE); 57018316Swollman s[APTDPTDI] = 0; 57118316Swollman s[PTDPTDI] = 0; 57218316Swollman 57318316Swollman#if defined(PMAP_DIAGNOSTIC) 57418316Swollman for(i=0;i<PAGE_SIZE/4;i++,s++) { 57518316Swollman if (*s) { 57618316Swollman printf("pmap_freepdir: index %d not zero: %lx\n", i, *s); 57718316Swollman } 57818316Swollman } 57918316Swollman#endif 58018316Swollman * (caddr_t *) pdir = (caddr_t) pdirlist; 58118316Swollman pdirlist = (caddr_t *) pdir; 58218316Swollman ++nfreepdir; 58318316Swollman } 58418316Swollman} 58518316Swollman#endif 58618316Swollman 58718316Swollman/* 58818316Swollman * Initialize a preallocated and zeroed pmap structure, 58918316Swollman * such as one in a vmspace structure. 59018316Swollman */ 59118316Swollmanvoid 59218316Swollmanpmap_pinit(pmap) 59318316Swollman register struct pmap *pmap; 59418316Swollman{ 59518316Swollman /* 59618316Swollman * No need to allocate page table space yet but we do need a valid 59718316Swollman * page directory table. 59818316Swollman */ 59918316Swollman 60018316Swollman#if defined(PMAP_KEEP_PDIRS) 60118316Swollman pmap->pm_pdir = pmap_getpdir(); 60218316Swollman#else 60318316Swollman pmap->pm_pdir = (pd_entry_t *) kmem_alloc(kernel_map, PAGE_SIZE); 60418316Swollman#endif 60518316Swollman 60618316Swollman /* wire in kernel global address entries */ 60718316Swollman bcopy(PTD + KPTDI, pmap->pm_pdir + KPTDI, nkpt * PTESIZE); 60818316Swollman 60918316Swollman /* install self-referential address mapping entry */ 61018316Swollman *(int *) (pmap->pm_pdir + PTDPTDI) = 61118316Swollman ((int) pmap_kextract((vm_offset_t) pmap->pm_pdir)) | PG_V | PG_KW; 61218316Swollman 61318316Swollman pmap->pm_count = 1; 61418316Swollman} 61518316Swollman 61618316Swollman/* 61718316Swollman * grow the number of kernel page table entries, if needed 61818316Swollman */ 61918316Swollman 62018316Swollmanstatic vm_page_t nkpg; 62118316Swollmanvm_offset_t kernel_vm_end; 62218316Swollman 62318316Swollmanvoid 62418316Swollmanpmap_growkernel(vm_offset_t addr) 62518316Swollman{ 62618316Swollman struct proc *p; 62718316Swollman struct pmap *pmap; 62818316Swollman int s; 62918316Swollman 63018316Swollman s = splhigh(); 63118316Swollman if (kernel_vm_end == 0) { 63218316Swollman kernel_vm_end = KERNBASE; 63318316Swollman nkpt = 0; 63418316Swollman while (pdir_pde(PTD, kernel_vm_end)) { 63518316Swollman kernel_vm_end = (kernel_vm_end + PAGE_SIZE * NPTEPG) & ~(PAGE_SIZE * NPTEPG - 1); 63618316Swollman ++nkpt; 63718316Swollman } 63818316Swollman } 63918316Swollman addr = (addr + PAGE_SIZE * NPTEPG) & ~(PAGE_SIZE * NPTEPG - 1); 64018316Swollman while (kernel_vm_end < addr) { 64118316Swollman if (pdir_pde(PTD, kernel_vm_end)) { 64218316Swollman kernel_vm_end = (kernel_vm_end + PAGE_SIZE * NPTEPG) & ~(PAGE_SIZE * NPTEPG - 1); 64318316Swollman continue; 64418316Swollman } 64518316Swollman ++nkpt; 64618316Swollman if (!nkpg) { 64718316Swollman nkpg = vm_page_alloc(kernel_object, 0, VM_ALLOC_SYSTEM); 64818316Swollman if (!nkpg) 64918316Swollman panic("pmap_growkernel: no memory to grow kernel"); 65018316Swollman vm_page_wire(nkpg); 65118316Swollman vm_page_remove(nkpg); 65218316Swollman pmap_zero_page(VM_PAGE_TO_PHYS(nkpg)); 65318316Swollman } 65418316Swollman pdir_pde(PTD, kernel_vm_end) = (pd_entry_t) (VM_PAGE_TO_PHYS(nkpg) | PG_V | PG_KW); 65518316Swollman nkpg = NULL; 65618316Swollman 65718316Swollman for (p = allproc.lh_first; p != 0; p = p->p_list.le_next) { 65818316Swollman if (p->p_vmspace) { 65918316Swollman pmap = &p->p_vmspace->vm_pmap; 66018316Swollman *pmap_pde(pmap, kernel_vm_end) = pdir_pde(PTD, kernel_vm_end); 66118316Swollman } 66218316Swollman } 66318316Swollman *pmap_pde(kernel_pmap, kernel_vm_end) = pdir_pde(PTD, kernel_vm_end); 66418316Swollman kernel_vm_end = (kernel_vm_end + PAGE_SIZE * NPTEPG) & ~(PAGE_SIZE * NPTEPG - 1); 66518316Swollman } 66618316Swollman splx(s); 66718316Swollman} 66818316Swollman 66918316Swollman/* 67018316Swollman * Retire the given physical map from service. 67118316Swollman * Should only be called if the map contains 67218316Swollman * no valid mappings. 67318316Swollman */ 67418316Swollmanvoid 67518316Swollmanpmap_destroy(pmap) 67618316Swollman register pmap_t pmap; 67718316Swollman{ 67818316Swollman int count; 67918316Swollman 68018316Swollman if (pmap == NULL) 68118316Swollman return; 68218316Swollman 68318316Swollman count = --pmap->pm_count; 68418316Swollman if (count == 0) { 68518316Swollman pmap_release(pmap); 68618316Swollman free((caddr_t) pmap, M_VMPMAP); 68718316Swollman } 68818316Swollman} 68918316Swollman 69018316Swollman/* 69118316Swollman * Release any resources held by the given physical map. 69218316Swollman * Called when a pmap initialized by pmap_pinit is being released. 69318316Swollman * Should only be called if the map contains no valid mappings. 69418316Swollman */ 69518316Swollmanvoid 69618316Swollmanpmap_release(pmap) 69718316Swollman register struct pmap *pmap; 69818316Swollman{ 69918316Swollman#if defined(PMAP_KEEP_PDIRS) 70018316Swollman pmap_freepdir( (void *)pmap->pm_pdir); 70118316Swollman#else 70218316Swollman kmem_free(kernel_map, (vm_offset_t) pmap->pm_pdir, PAGE_SIZE); 70318316Swollman#endif 70418316Swollman} 70518316Swollman 70618316Swollman/* 70718316Swollman * Add a reference to the specified pmap. 70818316Swollman */ 70918316Swollmanvoid 71018316Swollmanpmap_reference(pmap) 71118316Swollman pmap_t pmap; 71218316Swollman{ 71318316Swollman if (pmap != NULL) { 71418316Swollman pmap->pm_count++; 71518316Swollman } 71618316Swollman} 71718316Swollman 71818316Swollman#define PV_FREELIST_MIN ((PAGE_SIZE / sizeof (struct pv_entry)) / 2) 71918316Swollman 72018316Swollman/* 72118316Swollman * Data for the pv entry allocation mechanism 72218316Swollman */ 72318316Swollmanstatic int pv_freelistcnt; 72418316Swollmanstatic pv_entry_t pv_freelist; 72518316Swollmanstatic vm_offset_t pvva; 72618316Swollmanstatic int npvvapg; 72718316Swollman 72818316Swollman/* 72918316Swollman * free the pv_entry back to the free list 73018316Swollman */ 73118316Swollmanstatic __inline void 73218316Swollmanfree_pv_entry(pv) 73318316Swollman pv_entry_t pv; 73418316Swollman{ 73518316Swollman if (!pv) 73618316Swollman return; 73718316Swollman ++pv_freelistcnt; 73818316Swollman pv->pv_next = pv_freelist; 73918316Swollman pv_freelist = pv; 74018316Swollman} 74118316Swollman 74218316Swollman/* 74318316Swollman * get a new pv_entry, allocating a block from the system 74418316Swollman * when needed. 74518316Swollman * the memory allocation is performed bypassing the malloc code 74618316Swollman * because of the possibility of allocations at interrupt time. 74718316Swollman */ 74818316Swollmanstatic __inline pv_entry_t 74918316Swollmanget_pv_entry() 75018316Swollman{ 75118316Swollman pv_entry_t tmp; 75218316Swollman 75318316Swollman /* 75418316Swollman * get more pv_entry pages if needed 75518316Swollman */ 75618316Swollman if (pv_freelistcnt < PV_FREELIST_MIN || pv_freelist == 0) { 75718316Swollman pmap_alloc_pv_entry(); 75818316Swollman } 75918316Swollman /* 76018316Swollman * get a pv_entry off of the free list 76118316Swollman */ 76218316Swollman --pv_freelistcnt; 76318316Swollman tmp = pv_freelist; 76418316Swollman pv_freelist = tmp->pv_next; 76518316Swollman return tmp; 76618316Swollman} 76718316Swollman 76818316Swollman/* 76918316Swollman * this *strange* allocation routine *statistically* eliminates the 77018316Swollman * *possibility* of a malloc failure (*FATAL*) for a pv_entry_t data structure. 77118316Swollman * also -- this code is MUCH MUCH faster than the malloc equiv... 77218316Swollman */ 77318316Swollmanstatic void 77418316Swollmanpmap_alloc_pv_entry() 77518316Swollman{ 77618316Swollman /* 77718316Swollman * do we have any pre-allocated map-pages left? 77818316Swollman */ 77918316Swollman if (npvvapg) { 78018316Swollman vm_page_t m; 78118316Swollman 78218316Swollman /* 78318316Swollman * we do this to keep recursion away 78418316Swollman */ 78518316Swollman pv_freelistcnt += PV_FREELIST_MIN; 78618316Swollman /* 78718316Swollman * allocate a physical page out of the vm system 78818316Swollman */ 78918316Swollman m = vm_page_alloc(kernel_object, 79018316Swollman OFF_TO_IDX(pvva - vm_map_min(kernel_map)), 79118316Swollman VM_ALLOC_INTERRUPT); 79218316Swollman if (m) { 79318316Swollman int newentries; 79418316Swollman int i; 79518316Swollman pv_entry_t entry; 79618316Swollman 79718316Swollman newentries = (PAGE_SIZE / sizeof(struct pv_entry)); 79818316Swollman /* 79918316Swollman * wire the page 80018316Swollman */ 80118316Swollman vm_page_wire(m); 80218316Swollman m->flags &= ~PG_BUSY; 80318316Swollman /* 80418316Swollman * let the kernel see it 80518316Swollman */ 80618316Swollman pmap_kenter(pvva, VM_PAGE_TO_PHYS(m)); 80718316Swollman 80818316Swollman entry = (pv_entry_t) pvva; 80918316Swollman /* 81018316Swollman * update the allocation pointers 81118316Swollman */ 81218316Swollman pvva += PAGE_SIZE; 81318316Swollman --npvvapg; 81418316Swollman 81518316Swollman /* 81618316Swollman * free the entries into the free list 81718316Swollman */ 81818316Swollman for (i = 0; i < newentries; i++) { 81918316Swollman free_pv_entry(entry); 82018316Swollman entry++; 82118316Swollman } 82218316Swollman } 82318316Swollman pv_freelistcnt -= PV_FREELIST_MIN; 82418316Swollman } 82518316Swollman if (!pv_freelist) 82618316Swollman panic("get_pv_entry: cannot get a pv_entry_t"); 82718316Swollman} 82818316Swollman 82918316Swollman 83018316Swollman 83118316Swollman/* 83218316Swollman * init the pv_entry allocation system 83318316Swollman */ 83418316Swollman#define PVSPERPAGE 64 83518316Swollmanvoid 83618316Swollmaninit_pv_entries(npg) 83718316Swollman int npg; 83818316Swollman{ 83918316Swollman /* 84018316Swollman * allocate enough kvm space for PVSPERPAGE entries per page (lots) 84118316Swollman * kvm space is fairly cheap, be generous!!! (the system can panic if 84218316Swollman * this is too small.) 84318316Swollman */ 84418316Swollman npvvapg = ((npg * PVSPERPAGE) * sizeof(struct pv_entry) 84518316Swollman + PAGE_SIZE - 1) / PAGE_SIZE; 84618316Swollman pvva = kmem_alloc_pageable(kernel_map, npvvapg * PAGE_SIZE); 84718316Swollman /* 84818316Swollman * get the first batch of entries 84918316Swollman */ 85018316Swollman free_pv_entry(get_pv_entry()); 85118316Swollman} 85218316Swollman 85318316Swollman__inline pt_entry_t * 85418316Swollmanget_ptbase(pmap) 85518316Swollman pmap_t pmap; 85618316Swollman{ 85718316Swollman vm_offset_t frame = (int) pmap->pm_pdir[PTDPTDI] & PG_FRAME; 85818316Swollman 85918316Swollman /* are we current address space or kernel? */ 86018316Swollman if (pmap == kernel_pmap || frame == ((int) PTDpde & PG_FRAME)) { 86118316Swollman return PTmap; 86218316Swollman } 86318316Swollman /* otherwise, we are alternate address space */ 86418316Swollman if (frame != ((int) APTDpde & PG_FRAME)) { 86518316Swollman APTDpde = pmap->pm_pdir[PTDPTDI]; 86618316Swollman pmap_update(); 86718316Swollman } 86818316Swollman return APTmap; 86918316Swollman} 87018316Swollman 87118316Swollman/* 87218316Swollman * If it is the first entry on the list, it is actually 87318316Swollman * in the header and we must copy the following entry up 87418316Swollman * to the header. Otherwise we must search the list for 87518316Swollman * the entry. In either case we free the now unused entry. 87618316Swollman */ 87718316Swollmanstatic __inline void 87818316Swollmanpmap_remove_entry(pmap, pv, va) 87918316Swollman struct pmap *pmap; 88018316Swollman pv_entry_t pv; 88118316Swollman vm_offset_t va; 88218316Swollman{ 88318316Swollman pv_entry_t npv; 88418316Swollman int s; 88518316Swollman s = splhigh(); 88618316Swollman if (pmap == pv->pv_pmap && va == pv->pv_va) { 88718316Swollman pmap_unuse_pt(pmap, va, pv->pv_ptem); 88818316Swollman npv = pv->pv_next; 88918316Swollman if (npv) { 89018316Swollman *pv = *npv; 89118316Swollman free_pv_entry(npv); 89218316Swollman } else { 89318316Swollman pv->pv_pmap = NULL; 89418316Swollman } 89518316Swollman } else { 89618316Swollman for (npv = pv->pv_next; npv; (pv = npv, npv = pv->pv_next)) { 89718316Swollman if (pmap == npv->pv_pmap && va == npv->pv_va) { 89818316Swollman pmap_unuse_pt(pmap, va, npv->pv_ptem); 89918316Swollman pv->pv_next = npv->pv_next; 90018316Swollman free_pv_entry(npv); 90118316Swollman break; 90218316Swollman } 90318316Swollman } 90418316Swollman } 90518316Swollman splx(s); 90618316Swollman} 90718316Swollman 90818316Swollman/* 90918316Swollman * pmap_remove_pte: do the things to unmap a page in a process 91018316Swollman */ 91118316Swollmanstatic void 91218316Swollmanpmap_remove_pte(pmap, ptq, sva) 91318316Swollman struct pmap *pmap; 91418316Swollman pt_entry_t *ptq; 91518316Swollman vm_offset_t sva; 91618316Swollman{ 91718316Swollman pt_entry_t oldpte; 91818316Swollman vm_offset_t pa; 91918316Swollman pv_entry_t pv; 92018316Swollman 92118316Swollman oldpte = *ptq; 92218316Swollman if (((int)oldpte) & PG_W) 92318316Swollman pmap->pm_stats.wired_count--; 92418316Swollman pmap->pm_stats.resident_count--; 92518316Swollman 92618316Swollman pa = ((vm_offset_t)oldpte) & PG_FRAME; 92718316Swollman if (pmap_is_managed(pa)) { 92818316Swollman if ((int) oldpte & PG_M) { 92918316Swollman#if defined(PMAP_DIAGNOSTIC) 93018316Swollman if (pmap_nw_modified(oldpte)) { 93118316Swollman printf("pmap_remove: modified page not writable: va: 0x%lx, pte: 0x%lx\n", sva, (int) oldpte); 93218316Swollman } 93318316Swollman#endif 93418316Swollman 93518316Swollman if (sva < USRSTACK + (UPAGES * PAGE_SIZE) || 93618316Swollman (sva >= KERNBASE && (sva < clean_sva || sva >= clean_eva))) { 93718316Swollman PHYS_TO_VM_PAGE(pa)->dirty = VM_PAGE_BITS_ALL; 93818316Swollman } 93918316Swollman } 94018316Swollman pv = pa_to_pvh(pa); 94118316Swollman pmap_remove_entry(pmap, pv, sva); 94218316Swollman } else { 94318316Swollman pmap_unuse_pt(pmap, sva, NULL); 94418316Swollman } 94518316Swollman 94618316Swollman *ptq = 0; 94718316Swollman return; 94818316Swollman} 94918316Swollman 95018316Swollman/* 95118316Swollman * Remove a single page from a process address space 95218316Swollman */ 95318316Swollmanstatic __inline void 95418316Swollmanpmap_remove_page(pmap, va) 95518316Swollman struct pmap *pmap; 95618316Swollman register vm_offset_t va; 95718316Swollman{ 95818316Swollman register pt_entry_t *ptbase, *ptq; 95918316Swollman /* 96018316Swollman * if there is no pte for this address, just skip it!!! 96118316Swollman */ 96218316Swollman if (*pmap_pde(pmap, va) == 0) 96318316Swollman return; 96418316Swollman /* 96518316Swollman * get a local va for mappings for this pmap. 96618316Swollman */ 96718316Swollman ptbase = get_ptbase(pmap); 96818316Swollman ptq = ptbase + i386_btop(va); 96918316Swollman if (*ptq) { 97018316Swollman pmap_remove_pte(pmap, ptq, va); 97118316Swollman pmap_update_1pg(va); 97218316Swollman } 97318316Swollman return; 97418316Swollman} 97518316Swollman 97618316Swollman/* 97718316Swollman * Remove the given range of addresses from the specified map. 97818316Swollman * 97918316Swollman * It is assumed that the start and end are properly 98018316Swollman * rounded to the page size. 98118316Swollman */ 98218316Swollmanvoid 98318316Swollmanpmap_remove(pmap, sva, eva) 98418316Swollman struct pmap *pmap; 98518316Swollman register vm_offset_t sva; 98618316Swollman register vm_offset_t eva; 98718316Swollman{ 98818316Swollman register pt_entry_t *ptbase; 98918316Swollman vm_offset_t va; 99018316Swollman vm_offset_t pdnxt; 99118316Swollman vm_offset_t ptpaddr; 99218316Swollman vm_offset_t sindex, eindex; 99318316Swollman vm_page_t mpte; 99418316Swollman 99518316Swollman if (pmap == NULL) 99618316Swollman return; 99718316Swollman 99818316Swollman /* 99918316Swollman * special handling of removing one page. a very 100018316Swollman * common operation and easy to short circuit some 100118316Swollman * code. 100218316Swollman */ 100318316Swollman if ((sva + PAGE_SIZE) == eva) { 100418316Swollman pmap_remove_page(pmap, sva); 100518316Swollman return; 100618316Swollman } 100718316Swollman 100818316Swollman /* 100918316Swollman * Get a local virtual address for the mappings that are being 101018316Swollman * worked with. 101118316Swollman */ 101218316Swollman ptbase = get_ptbase(pmap); 101318316Swollman 101418316Swollman sindex = i386_btop(sva); 101518316Swollman eindex = i386_btop(eva); 101618316Swollman 101718316Swollman for (; sindex < eindex; sindex = pdnxt) { 101818316Swollman 101918316Swollman /* 102018316Swollman * Calculate index for next page table. 102118316Swollman */ 102218316Swollman pdnxt = ((sindex + NPTEPG) & ~(NPTEPG - 1)); 102318316Swollman ptpaddr = (vm_offset_t) *pmap_pde(pmap, i386_ptob(sindex)); 102418316Swollman 102518316Swollman /* 102618316Swollman * Weed out invalid mappings. Note: we assume that the page 102718316Swollman * directory table is always allocated, and in kernel virtual. 102818316Swollman */ 102918316Swollman if (ptpaddr == 0) 103018316Swollman continue; 103118316Swollman 103218316Swollman /* 103318316Swollman * get the vm_page_t for the page table page 103418316Swollman */ 103518316Swollman mpte = PHYS_TO_VM_PAGE(ptpaddr); 103618316Swollman 103718316Swollman /* 103818316Swollman * if the pte isn't wired or held, just skip it. 103918316Swollman */ 104018316Swollman if ((mpte->hold_count == 0) && (mpte->wire_count == 0)) 104118316Swollman continue; 104218316Swollman 104318316Swollman /* 104418316Swollman * Limit our scan to either the end of the va represented 104518316Swollman * by the current page table page, or to the end of the 104618316Swollman * range being removed. 104718316Swollman */ 104818316Swollman if (pdnxt > eindex) { 104918316Swollman pdnxt = eindex; 105018316Swollman } 105118316Swollman 105218316Swollman for ( ;sindex != pdnxt; sindex++) { 105318316Swollman if (ptbase[sindex] == 0) 105418316Swollman continue; 105518316Swollman pmap_remove_pte(pmap, ptbase + sindex, i386_ptob(sindex)); 105618316Swollman if (mpte->hold_count == 0 && mpte->wire_count == 0) 105718316Swollman break; 105818316Swollman } 105918316Swollman } 106018316Swollman pmap_update(); 106118316Swollman} 106218316Swollman 106318316Swollman/* 106418316Swollman * Routine: pmap_remove_all 106518316Swollman * Function: 106618316Swollman * Removes this physical page from 106718316Swollman * all physical maps in which it resides. 106818316Swollman * Reflects back modify bits to the pager. 106918316Swollman * 107018316Swollman * Notes: 107118316Swollman * Original versions of this routine were very 107218316Swollman * inefficient because they iteratively called 107318316Swollman * pmap_remove (slow...) 107418316Swollman */ 107518316Swollmanstatic void 107618316Swollmanpmap_remove_all(pa) 107718316Swollman vm_offset_t pa; 107818316Swollman{ 107918316Swollman register pv_entry_t pv, opv, npv; 108018316Swollman register pt_entry_t *pte, *ptbase; 108118316Swollman vm_offset_t va; 108218316Swollman struct pmap *pmap; 108318316Swollman vm_page_t m; 108418316Swollman int s; 108518316Swollman int anyvalid = 0; 108618316Swollman 108718316Swollman#if defined(PMAP_DIAGNOSTIC) 108818316Swollman /* 108918316Swollman * XXX this makes pmap_page_protect(NONE) illegal for non-managed 109018316Swollman * pages! 109118316Swollman */ 109218316Swollman if (!pmap_is_managed(pa)) { 109318316Swollman panic("pmap_page_protect: illegal for unmanaged page, va: 0x%lx", pa); 109418316Swollman } 109518316Swollman#endif 109618316Swollman 109718316Swollman pa = pa & PG_FRAME; 109818316Swollman opv = pa_to_pvh(pa); 109918316Swollman if (opv->pv_pmap == NULL) 110018316Swollman return; 110118316Swollman 110218316Swollman m = PHYS_TO_VM_PAGE(pa); 110318316Swollman s = splhigh(); 110418316Swollman pv = opv; 110518316Swollman while (pv && ((pmap = pv->pv_pmap) != NULL)) { 110618316Swollman int tpte; 110718316Swollman ptbase = get_ptbase(pmap); 110818316Swollman va = pv->pv_va; 110918316Swollman pte = ptbase + i386_btop(va); 111018316Swollman if (tpte = ((int) *pte)) { 111118316Swollman *pte = 0; 111218316Swollman if (tpte & PG_W) 111318316Swollman pmap->pm_stats.wired_count--; 111418316Swollman pmap->pm_stats.resident_count--; 111518316Swollman anyvalid = 1; 111618316Swollman 111718316Swollman /* 111818316Swollman * Update the vm_page_t clean and reference bits. 111918316Swollman */ 112018316Swollman if ((tpte & PG_M) != 0) { 112118316Swollman#if defined(PMAP_DIAGNOSTIC) 112218316Swollman if (pmap_nw_modified((pt_entry_t) tpte)) { 112318316Swollman printf("pmap_remove_all: modified page not writable: va: 0x%lx, pte: 0x%lx\n", va, tpte); 112418316Swollman } 112518316Swollman#endif 112618316Swollman if (va < USRSTACK + (UPAGES * PAGE_SIZE) || 112718316Swollman (va >= KERNBASE && (va < clean_sva || va >= clean_eva))) { 112818316Swollman m->dirty = VM_PAGE_BITS_ALL; 112918316Swollman } 113018316Swollman } 113118316Swollman } 113218316Swollman pv = pv->pv_next; 113318316Swollman } 113418316Swollman 113518316Swollman if (opv->pv_pmap != NULL) { 113618316Swollman pmap_unuse_pt(opv->pv_pmap, opv->pv_va, opv->pv_ptem); 113718316Swollman for (pv = opv->pv_next; pv; pv = npv) { 113818316Swollman npv = pv->pv_next; 113918316Swollman pmap_unuse_pt(pv->pv_pmap, pv->pv_va, pv->pv_ptem); 114018316Swollman free_pv_entry(pv); 114118316Swollman } 114218316Swollman } 114318316Swollman 114418316Swollman opv->pv_pmap = NULL; 114518316Swollman opv->pv_next = NULL; 114618316Swollman 114718316Swollman splx(s); 114818316Swollman if (anyvalid) 114918316Swollman pmap_update(); 115018316Swollman} 115118316Swollman 115218316Swollman 115318316Swollman/* 115418316Swollman * Set the physical protection on the 115518316Swollman * specified range of this map as requested. 115618316Swollman */ 115718316Swollmanvoid 115818316Swollmanpmap_protect(pmap, sva, eva, prot) 115918316Swollman register pmap_t pmap; 116018316Swollman vm_offset_t sva, eva; 116118316Swollman vm_prot_t prot; 116218316Swollman{ 116318316Swollman register pt_entry_t *pte; 116418316Swollman register vm_offset_t va; 116518316Swollman register pt_entry_t *ptbase; 116618316Swollman vm_offset_t pdnxt; 116718316Swollman vm_offset_t ptpaddr; 116818316Swollman vm_offset_t sindex, eindex; 116918316Swollman vm_page_t mpte; 117018316Swollman int anychanged; 117118316Swollman 117218316Swollman 117318316Swollman if (pmap == NULL) 117418316Swollman return; 117518316Swollman 117618316Swollman if ((prot & VM_PROT_READ) == VM_PROT_NONE) { 117718316Swollman pmap_remove(pmap, sva, eva); 117818316Swollman return; 117918316Swollman } 118018316Swollman if (prot & VM_PROT_WRITE) 118118316Swollman return; 118218316Swollman 118318316Swollman anychanged = 0; 118418316Swollman 118518316Swollman ptbase = get_ptbase(pmap); 118618316Swollman 118718316Swollman sindex = i386_btop(sva); 118818316Swollman eindex = i386_btop(eva); 118918316Swollman 119018316Swollman for (; sindex < eindex; sindex = pdnxt) { 119118316Swollman int pprot; 119218316Swollman int pbits; 119318316Swollman 119418316Swollman pdnxt = ((sindex + NPTEPG) & ~(NPTEPG - 1)); 119518316Swollman ptpaddr = (vm_offset_t) *pmap_pde(pmap, i386_ptob(sindex)); 119618316Swollman 119718316Swollman /* 119818316Swollman * Weed out invalid mappings. Note: we assume that the page 119918316Swollman * directory table is always allocated, and in kernel virtual. 120018316Swollman */ 120118316Swollman if (ptpaddr == 0) 120218316Swollman continue; 120318316Swollman 120418316Swollman mpte = PHYS_TO_VM_PAGE(ptpaddr); 120518316Swollman 120618316Swollman if ((mpte->hold_count == 0) && (mpte->wire_count == 0)) 120718316Swollman continue; 120818316Swollman 120918316Swollman if (pdnxt > eindex) { 121018316Swollman pdnxt = eindex; 121118316Swollman } 121218316Swollman 121318316Swollman for (; sindex != pdnxt; sindex++) { 121418316Swollman if (ptbase[sindex] == 0) 121518316Swollman continue; 121618316Swollman pte = ptbase + sindex; 121718316Swollman pbits = *(int *)pte; 121818316Swollman if (pbits & PG_RW) { 121918316Swollman if (pbits & PG_M) { 122018316Swollman vm_page_t m; 122118316Swollman vm_offset_t pa = pbits & PG_FRAME; 122218316Swollman m = PHYS_TO_VM_PAGE(pa); 122318316Swollman m->dirty = VM_PAGE_BITS_ALL; 122418316Swollman } 122518316Swollman *(int *)pte &= ~(PG_M|PG_RW); 122618316Swollman anychanged=1; 122718316Swollman } 122818316Swollman } 122918316Swollman } 123018316Swollman if (anychanged) 123118316Swollman pmap_update(); 123218316Swollman} 123318316Swollman 123418316Swollman/* 123518316Swollman * Insert the given physical page (p) at 123618316Swollman * the specified virtual address (v) in the 123718316Swollman * target physical map with the protection requested. 123818316Swollman * 123918316Swollman * If specified, the page will be wired down, meaning 124018316Swollman * that the related pte can not be reclaimed. 124118316Swollman * 124218316Swollman * NB: This is the only routine which MAY NOT lazy-evaluate 124318316Swollman * or lose information. That is, this routine must actually 124418316Swollman * insert this page into the given map NOW. 124518316Swollman */ 124618316Swollmanvoid 124718316Swollmanpmap_enter(pmap, va, pa, prot, wired) 124818316Swollman register pmap_t pmap; 124918316Swollman vm_offset_t va; 125018316Swollman register vm_offset_t pa; 125118316Swollman vm_prot_t prot; 125218316Swollman boolean_t wired; 125318316Swollman{ 125418316Swollman register pt_entry_t *pte; 125518316Swollman vm_offset_t opa; 125618316Swollman register pv_entry_t pv, npv; 125718316Swollman int ptevalid; 125818316Swollman vm_offset_t origpte, newpte; 125918316Swollman 126018316Swollman if (pmap == NULL) 126118316Swollman return; 126218316Swollman 126318316Swollman pv = NULL; 126418316Swollman 126518316Swollman va = va & PG_FRAME; 126618316Swollman if (va > VM_MAX_KERNEL_ADDRESS) 126718316Swollman panic("pmap_enter: toobig"); 126818316Swollman 126918316Swollman /* 127018316Swollman * In the case that a page table page is not 127118316Swollman * resident, we are creating it here. 127218316Swollman */ 127318316Swollman if ((va < VM_MIN_KERNEL_ADDRESS) && 127418316Swollman (curproc != NULL) && 127518316Swollman (pmap->pm_map->pmap == pmap)) { 127618316Swollman vm_offset_t v; 127718316Swollman 127818316Swollman v = (vm_offset_t) vtopte(va); 127918316Swollman /* Fault the pte only if needed: */ 128018316Swollman if (*((int *)vtopte(v)) == 0) 128118316Swollman (void) vm_fault(pmap->pm_map, 128218316Swollman trunc_page(v), VM_PROT_WRITE, FALSE); 128318316Swollman } 128418316Swollman 128518316Swollman /* 128618316Swollman * Page Directory table entry not valid, we need a new PT page 128718316Swollman */ 128818316Swollman pte = pmap_pte(pmap, va); 128918316Swollman if (pte == NULL) { 129018316Swollman printf("kernel page directory invalid pdir=%p, va=0x%lx\n", 129118316Swollman pmap->pm_pdir[PTDPTDI], va); 129218316Swollman panic("invalid kernel page directory"); 129318316Swollman } 129418316Swollman 129518316Swollman origpte = *(vm_offset_t *)pte; 129618316Swollman opa = origpte & PG_FRAME; 129718316Swollman 129818316Swollman pa = pa & PG_FRAME; 129918316Swollman 130018316Swollman /* 130118316Swollman * Mapping has not changed, must be protection or wiring change. 130218316Swollman */ 130318316Swollman if (opa == pa) { 130418316Swollman /* 130518316Swollman * Wiring change, just update stats. We don't worry about 130618316Swollman * wiring PT pages as they remain resident as long as there 130718316Swollman * are valid mappings in them. Hence, if a user page is wired, 130818316Swollman * the PT page will be also. 130918316Swollman */ 131018316Swollman if (wired && ((origpte & PG_W) == 0)) 131118316Swollman pmap->pm_stats.wired_count++; 131218316Swollman else if (!wired && (origpte & PG_W)) 131318316Swollman pmap->pm_stats.wired_count--; 131418316Swollman 131518316Swollman#if defined(PMAP_DIAGNOSTIC) 131618316Swollman if (pmap_nw_modified((pt_entry_t) origpte)) { 131718316Swollman printf("pmap_enter: modified page not writable: va: 0x%lx, pte: 0x%lx\n", va, origpte); 131818316Swollman } 131918316Swollman#endif 132018316Swollman 132118316Swollman /* 132218316Swollman * We might be turning off write access to the page, 132318316Swollman * so we go ahead and sense modify status. 132418316Swollman */ 132518316Swollman if (origpte & PG_M) { 132618316Swollman vm_page_t m; 132718316Swollman m = PHYS_TO_VM_PAGE(pa); 132818316Swollman m->dirty = VM_PAGE_BITS_ALL; 132918316Swollman } 133018316Swollman goto validate; 133118316Swollman } 133218316Swollman /* 133318316Swollman * Mapping has changed, invalidate old range and fall through to 133418316Swollman * handle validating new mapping. 133518316Swollman */ 133618316Swollman if (opa) { 133718316Swollman pmap_remove_page(pmap, va); 133818316Swollman opa = 0; 133918316Swollman origpte = 0; 134018316Swollman } 134118316Swollman /* 134218316Swollman * Enter on the PV list if part of our managed memory Note that we 134318316Swollman * raise IPL while manipulating pv_table since pmap_enter can be 134418316Swollman * called at interrupt time. 134518316Swollman */ 134618316Swollman if (pmap_is_managed(pa)) { 134718316Swollman int s; 134818316Swollman 134918316Swollman pv = pa_to_pvh(pa); 135018316Swollman s = splhigh(); 135118316Swollman /* 135218316Swollman * No entries yet, use header as the first entry 135318316Swollman */ 135418316Swollman if (pv->pv_pmap == NULL) { 135518316Swollman pv->pv_va = va; 135618316Swollman pv->pv_pmap = pmap; 135718316Swollman pv->pv_next = NULL; 135818316Swollman pv->pv_ptem = NULL; 135918316Swollman } 136018316Swollman /* 136118316Swollman * There is at least one other VA mapping this page. Place 136218316Swollman * this entry after the header. 136318316Swollman */ 136418316Swollman else { 136518316Swollman npv = get_pv_entry(); 136618316Swollman npv->pv_va = va; 136718316Swollman npv->pv_pmap = pmap; 136818316Swollman npv->pv_next = pv->pv_next; 136918316Swollman pv->pv_next = npv; 137018316Swollman pv = npv; 137118316Swollman pv->pv_ptem = NULL; 137218316Swollman } 137318316Swollman splx(s); 137418316Swollman } 137518316Swollman 137618316Swollman /* 137718316Swollman * Increment counters 137818316Swollman */ 137918316Swollman pmap->pm_stats.resident_count++; 138018316Swollman if (wired) 138118316Swollman pmap->pm_stats.wired_count++; 138218316Swollman 138318316Swollmanvalidate: 138418316Swollman /* 138518316Swollman * Now validate mapping with desired protection/wiring. 138618316Swollman */ 138718316Swollman newpte = (vm_offset_t) (pa | pte_prot(pmap, prot) | PG_V); 138818316Swollman 138918316Swollman if (wired) 139018316Swollman newpte |= PG_W; 139118316Swollman if (va < UPT_MIN_ADDRESS) 139218316Swollman newpte |= PG_u; 139318316Swollman else if (va < UPT_MAX_ADDRESS) 139418316Swollman newpte |= PG_u | PG_RW; 139518316Swollman 139618316Swollman /* 139718316Swollman * if the mapping or permission bits are different, we need 139818316Swollman * to update the pte. 139918316Swollman */ 140018316Swollman if ((origpte & ~(PG_M|PG_U)) != newpte) { 140118316Swollman *pte = (pt_entry_t) newpte; 140218316Swollman if (origpte) 140318316Swollman pmap_update_1pg(va); 140418316Swollman } 140518316Swollman 140618316Swollman if (origpte == 0) { 140718316Swollman vm_page_t mpte; 140818316Swollman mpte = pmap_use_pt(pmap, va); 140918316Swollman if (pv) 141018316Swollman pv->pv_ptem = mpte; 141118316Swollman } 141218316Swollman} 141318316Swollman 141418316Swollman/* 141518316Swollman * Add a list of wired pages to the kva 141618316Swollman * this routine is only used for temporary 141718316Swollman * kernel mappings that do not need to have 141818316Swollman * page modification or references recorded. 141918316Swollman * Note that old mappings are simply written 142018316Swollman * over. The page *must* be wired. 142118316Swollman */ 142218316Swollmanvoid 142318316Swollmanpmap_qenter(va, m, count) 142418316Swollman vm_offset_t va; 142518316Swollman vm_page_t *m; 142618316Swollman int count; 142718316Swollman{ 142818316Swollman int i; 142918316Swollman int anyvalid = 0; 143018316Swollman register pt_entry_t *pte; 143118316Swollman 143218316Swollman for (i = 0; i < count; i++) { 143318316Swollman vm_offset_t tva = va + i * PAGE_SIZE; 143418316Swollman pt_entry_t npte = (pt_entry_t) ((int) (VM_PAGE_TO_PHYS(m[i]) | PG_RW | PG_V)); 143518316Swollman pt_entry_t opte; 143618316Swollman pte = vtopte(tva); 143718316Swollman opte = *pte; 143818316Swollman *pte = npte; 143918316Swollman if (opte) pmap_update_1pg(tva); 144018316Swollman } 144118316Swollman} 144218316Swollman/* 144318316Swollman * this routine jerks page mappings from the 144418316Swollman * kernel -- it is meant only for temporary mappings. 144518316Swollman */ 144618316Swollmanvoid 144718316Swollmanpmap_qremove(va, count) 144818316Swollman vm_offset_t va; 144918316Swollman int count; 145018316Swollman{ 145118316Swollman int i; 145218316Swollman register pt_entry_t *pte; 145318316Swollman 145418316Swollman for (i = 0; i < count; i++) { 145518316Swollman vm_offset_t tva = va + i * PAGE_SIZE; 145618316Swollman pte = vtopte(tva); 145718316Swollman *pte = 0; 145818316Swollman pmap_update_1pg(tva); 145918316Swollman } 146018316Swollman} 146118316Swollman 146218316Swollman/* 146318316Swollman * add a wired page to the kva 146418316Swollman * note that in order for the mapping to take effect -- you 146518316Swollman * should do a pmap_update after doing the pmap_kenter... 146618316Swollman */ 146718316Swollmanvoid 146818316Swollmanpmap_kenter(va, pa) 146918316Swollman vm_offset_t va; 147018316Swollman register vm_offset_t pa; 147118316Swollman{ 147218316Swollman register pt_entry_t *pte; 147318316Swollman pt_entry_t npte, opte; 147418316Swollman 147518316Swollman npte = (pt_entry_t) ((int) (pa | PG_RW | PG_V)); 147618316Swollman pte = vtopte(va); 147718316Swollman opte = *pte; 147818316Swollman *pte = npte; 147918316Swollman if (opte) pmap_update_1pg(va); 148018316Swollman} 148118316Swollman 148218316Swollman/* 148318316Swollman * remove a page from the kernel pagetables 148418316Swollman */ 148518316Swollmanvoid 148618316Swollmanpmap_kremove(va) 148718316Swollman vm_offset_t va; 148818316Swollman{ 148918316Swollman register pt_entry_t *pte; 149018316Swollman 149118316Swollman pte = vtopte(va); 149218316Swollman *pte = (pt_entry_t) 0; 149318316Swollman pmap_update_1pg(va); 149418316Swollman} 149518316Swollman 149618316Swollman/* 149718316Swollman * this code makes some *MAJOR* assumptions: 149818316Swollman * 1. Current pmap & pmap exists. 149918316Swollman * 2. Not wired. 150018316Swollman * 3. Read access. 150118316Swollman * 4. No page table pages. 150218316Swollman * 5. Tlbflush is deferred to calling procedure. 150318316Swollman * 6. Page IS managed. 150418316Swollman * but is *MUCH* faster than pmap_enter... 150518316Swollman */ 150618316Swollman 150718316Swollmanstatic void 150818316Swollmanpmap_enter_quick(pmap, va, pa) 150918316Swollman register pmap_t pmap; 151018316Swollman vm_offset_t va; 151118316Swollman register vm_offset_t pa; 151218316Swollman{ 151318316Swollman register pt_entry_t *pte; 151418316Swollman register pv_entry_t pv, npv; 151518316Swollman int s; 151618316Swollman 151718316Swollman /* 151818316Swollman * Enter on the PV list if part of our managed memory Note that we 151918316Swollman * raise IPL while manipulating pv_table since pmap_enter can be 152018316Swollman * called at interrupt time. 152118316Swollman */ 152218316Swollman 152318316Swollman pte = vtopte(va); 152418316Swollman /* a fault on the page table might occur here */ 152518316Swollman if (*pte) { 152618316Swollman pmap_remove_page(pmap, va); 152718316Swollman } 152818316Swollman 152918316Swollman pv = pa_to_pvh(pa); 153018316Swollman s = splhigh(); 153118316Swollman /* 153218316Swollman * No entries yet, use header as the first entry 153318316Swollman */ 153418316Swollman if (pv->pv_pmap == NULL) { 153518316Swollman pv->pv_pmap = pmap; 153618316Swollman pv->pv_va = va; 153718316Swollman pv->pv_next = NULL; 153818316Swollman } 153918316Swollman /* 154018316Swollman * There is at least one other VA mapping this page. Place this entry 154118316Swollman * after the header. 154218316Swollman */ 154318316Swollman else { 154418316Swollman npv = get_pv_entry(); 154518316Swollman npv->pv_va = va; 154618316Swollman npv->pv_pmap = pmap; 154718316Swollman npv->pv_next = pv->pv_next; 154818316Swollman pv->pv_next = npv; 154918316Swollman pv = npv; 155018316Swollman } 155118316Swollman splx(s); 155218316Swollman pv->pv_ptem = pmap_use_pt(pmap, va); 155318316Swollman 155418316Swollman /* 155518316Swollman * Increment counters 155618316Swollman */ 155718316Swollman pmap->pm_stats.resident_count++; 155818316Swollman 155918316Swollman /* 156018316Swollman * Now validate mapping with RO protection 156118316Swollman */ 156218316Swollman *pte = (pt_entry_t) ((int) (pa | PG_V | PG_u)); 156318316Swollman 156418316Swollman return; 156518316Swollman} 156618316Swollman 156718316Swollman#define MAX_INIT_PT (96) 156818316Swollman/* 156918316Swollman * pmap_object_init_pt preloads the ptes for a given object 157018316Swollman * into the specified pmap. This eliminates the blast of soft 157118316Swollman * faults on process startup and immediately after an mmap. 157218316Swollman */ 157318316Swollmanvoid 157418316Swollmanpmap_object_init_pt(pmap, addr, object, pindex, size) 157518316Swollman pmap_t pmap; 157618316Swollman vm_offset_t addr; 157718316Swollman vm_object_t object; 157818316Swollman vm_pindex_t pindex; 157918316Swollman vm_size_t size; 158018316Swollman{ 158118316Swollman vm_offset_t tmpidx; 158218316Swollman int psize; 158318316Swollman vm_page_t p; 158418316Swollman int objpgs; 158518316Swollman 158618316Swollman psize = (size >> PAGE_SHIFT); 158718316Swollman 158818316Swollman if (!pmap || (object->type != OBJT_VNODE) || 158918316Swollman ((psize > MAX_INIT_PT) && 159018316Swollman (object->resident_page_count > MAX_INIT_PT))) { 159118316Swollman return; 159218316Swollman } 159318316Swollman 159418316Swollman /* 159518316Swollman * remove any already used mappings 159618316Swollman */ 159718316Swollman pmap_remove( pmap, trunc_page(addr), round_page(addr + size)); 159818316Swollman 159918316Swollman /* 160018316Swollman * if we are processing a major portion of the object, then scan the 160118316Swollman * entire thing. 160218316Swollman */ 160318316Swollman if (psize > (object->size >> 2)) { 160418316Swollman objpgs = psize; 160518316Swollman 160618316Swollman for (p = object->memq.tqh_first; 160718316Swollman ((objpgs > 0) && (p != NULL)); 160818316Swollman p = p->listq.tqe_next) { 160918316Swollman 161018316Swollman tmpidx = p->pindex; 161118316Swollman if (tmpidx < pindex) { 161218316Swollman continue; 161318316Swollman } 161418316Swollman tmpidx -= pindex; 161518316Swollman if (tmpidx >= psize) { 161618316Swollman continue; 161718316Swollman } 161818316Swollman if (((p->valid & VM_PAGE_BITS_ALL) == VM_PAGE_BITS_ALL) && 161918316Swollman (p->busy == 0) && 162018316Swollman (p->flags & (PG_BUSY | PG_FICTITIOUS)) == 0) { 162118316Swollman if (p->queue == PQ_CACHE) 162218316Swollman vm_page_deactivate(p); 162318316Swollman vm_page_hold(p); 162418316Swollman p->flags |= PG_MAPPED; 162518316Swollman pmap_enter_quick(pmap, 162618316Swollman addr + (tmpidx << PAGE_SHIFT), 162718316Swollman VM_PAGE_TO_PHYS(p)); 162818316Swollman vm_page_unhold(p); 162918316Swollman } 163018316Swollman objpgs -= 1; 163118316Swollman } 163218316Swollman } else { 163318316Swollman /* 163418316Swollman * else lookup the pages one-by-one. 163518316Swollman */ 163618316Swollman for (tmpidx = 0; tmpidx < psize; tmpidx += 1) { 163718316Swollman p = vm_page_lookup(object, tmpidx + pindex); 163818316Swollman if (p && (p->busy == 0) && 163918316Swollman ((p->valid & VM_PAGE_BITS_ALL) == VM_PAGE_BITS_ALL) && 164018316Swollman (p->flags & (PG_BUSY | PG_FICTITIOUS)) == 0) { 164118316Swollman if (p->queue == PQ_CACHE) 164218316Swollman vm_page_deactivate(p); 164318316Swollman vm_page_hold(p); 164418316Swollman p->flags |= PG_MAPPED; 164518316Swollman pmap_enter_quick(pmap, 164618316Swollman addr + (tmpidx << PAGE_SHIFT), 164718316Swollman VM_PAGE_TO_PHYS(p)); 164818316Swollman vm_page_unhold(p); 164918316Swollman } 165018316Swollman } 165118316Swollman } 165218316Swollman return; 165318316Swollman} 165418316Swollman 165518316Swollman/* 165618316Swollman * pmap_prefault provides a quick way of clustering 165718316Swollman * pagefaults into a processes address space. It is a "cousin" 165818316Swollman * of pmap_object_init_pt, except it runs at page fault time instead 165918316Swollman * of mmap time. 166018316Swollman */ 166118316Swollman#define PFBAK 2 166218316Swollman#define PFFOR 2 166318316Swollman#define PAGEORDER_SIZE (PFBAK+PFFOR) 166418316Swollman 166518316Swollmanstatic int pmap_prefault_pageorder[] = { 166618316Swollman -NBPG, NBPG, -2 * NBPG, 2 * NBPG 166718316Swollman}; 166818316Swollman 166918316Swollmanvoid 167018316Swollmanpmap_prefault(pmap, addra, entry, object) 167118316Swollman pmap_t pmap; 167218316Swollman vm_offset_t addra; 167318316Swollman vm_map_entry_t entry; 167418316Swollman vm_object_t object; 167518316Swollman{ 167618316Swollman int i; 167718316Swollman vm_offset_t starta; 167818316Swollman vm_offset_t addr; 167918316Swollman vm_pindex_t pindex; 168018316Swollman vm_page_t m; 168118316Swollman int pageorder_index; 168218316Swollman 168318316Swollman if (entry->object.vm_object != object) 168418316Swollman return; 168518316Swollman 168618316Swollman if (!curproc || (pmap != &curproc->p_vmspace->vm_pmap)) 168718316Swollman return; 168818316Swollman 168918316Swollman starta = addra - PFBAK * PAGE_SIZE; 169018316Swollman if (starta < entry->start) { 169118316Swollman starta = entry->start; 169218316Swollman } else if (starta > addra) { 169318316Swollman starta = 0; 169418316Swollman } 169518316Swollman 169618316Swollman for (i = 0; i < PAGEORDER_SIZE; i++) { 169718316Swollman vm_object_t lobject; 169818316Swollman pt_entry_t *pte; 169918316Swollman 170018316Swollman addr = addra + pmap_prefault_pageorder[i]; 170118316Swollman if (addr < starta || addr >= entry->end) 170218316Swollman continue; 170318316Swollman 170418316Swollman pte = vtopte(addr); 170518316Swollman if (*pte) 170618316Swollman continue; 170718316Swollman 170818316Swollman pindex = ((addr - entry->start) + entry->offset) >> PAGE_SHIFT; 170918316Swollman lobject = object; 171018316Swollman for (m = vm_page_lookup(lobject, pindex); 171118316Swollman (!m && (lobject->type == OBJT_DEFAULT) && (lobject->backing_object)); 171218316Swollman lobject = lobject->backing_object) { 171318316Swollman if (lobject->backing_object_offset & PAGE_MASK) 171418316Swollman break; 171518316Swollman pindex += (lobject->backing_object_offset >> PAGE_SHIFT); 171618316Swollman m = vm_page_lookup(lobject->backing_object, pindex); 171718316Swollman } 171818316Swollman 171918316Swollman /* 172018316Swollman * give-up when a page is not in memory 172118316Swollman */ 172218316Swollman if (m == NULL) 172318316Swollman break; 172418316Swollman 172518316Swollman if (((m->valid & VM_PAGE_BITS_ALL) == VM_PAGE_BITS_ALL) && 172618316Swollman (m->busy == 0) && 172718316Swollman (m->flags & (PG_BUSY | PG_FICTITIOUS)) == 0) { 172818316Swollman 172918316Swollman if (m->queue == PQ_CACHE) { 173018316Swollman if ((cnt.v_free_count + cnt.v_cache_count) < 173118316Swollman cnt.v_free_min) 173218316Swollman break; 173318316Swollman vm_page_deactivate(m); 173418316Swollman } 173518316Swollman vm_page_hold(m); 173618316Swollman m->flags |= PG_MAPPED; 173718316Swollman pmap_enter_quick(pmap, addr, VM_PAGE_TO_PHYS(m)); 173818316Swollman vm_page_unhold(m); 173918316Swollman 174018316Swollman } 174118316Swollman } 174218316Swollman} 174318316Swollman 174418316Swollman/* 174518316Swollman * Routine: pmap_change_wiring 174618316Swollman * Function: Change the wiring attribute for a map/virtual-address 174718316Swollman * pair. 174818316Swollman * In/out conditions: 174918316Swollman * The mapping must already exist in the pmap. 175018316Swollman */ 175118316Swollmanvoid 175218316Swollmanpmap_change_wiring(pmap, va, wired) 175318316Swollman register pmap_t pmap; 175418316Swollman vm_offset_t va; 175518316Swollman boolean_t wired; 175618316Swollman{ 175718316Swollman register pt_entry_t *pte; 175818316Swollman 175918316Swollman if (pmap == NULL) 176018316Swollman return; 176118316Swollman 176218316Swollman pte = pmap_pte(pmap, va); 176318316Swollman 176418316Swollman if (wired && !pmap_pte_w(pte)) 176518316Swollman pmap->pm_stats.wired_count++; 176618316Swollman else if (!wired && pmap_pte_w(pte)) 176718316Swollman pmap->pm_stats.wired_count--; 176818316Swollman 176918316Swollman /* 177018316Swollman * Wiring is not a hardware characteristic so there is no need to 177118316Swollman * invalidate TLB. 177218316Swollman */ 177318316Swollman pmap_pte_set_w(pte, wired); 177418316Swollman} 177518316Swollman 177618316Swollman 177718316Swollman 177818316Swollman/* 177918316Swollman * Copy the range specified by src_addr/len 178018316Swollman * from the source map to the range dst_addr/len 178118316Swollman * in the destination map. 178218316Swollman * 178318316Swollman * This routine is only advisory and need not do anything. 178418316Swollman */ 178518316Swollmanvoid 178618316Swollmanpmap_copy(dst_pmap, src_pmap, dst_addr, len, src_addr) 178718316Swollman pmap_t dst_pmap, src_pmap; 178818316Swollman vm_offset_t dst_addr; 178918316Swollman vm_size_t len; 179018316Swollman vm_offset_t src_addr; 179118316Swollman{ 179218316Swollman} 179318316Swollman 179418316Swollman/* 179518316Swollman * Routine: pmap_kernel 179618316Swollman * Function: 179718316Swollman * Returns the physical map handle for the kernel. 179818316Swollman */ 179918316Swollmanpmap_t 180018316Swollmanpmap_kernel() 180118316Swollman{ 180218316Swollman return (kernel_pmap); 180318316Swollman} 180418316Swollman 180518316Swollman/* 180618316Swollman * pmap_zero_page zeros the specified (machine independent) 180718316Swollman * page by mapping the page into virtual memory and using 180818316Swollman * bzero to clear its contents, one machine dependent page 180918316Swollman * at a time. 181018316Swollman */ 181118316Swollmanvoid 181218316Swollmanpmap_zero_page(phys) 181318316Swollman vm_offset_t phys; 181418316Swollman{ 181518316Swollman if (*(int *) CMAP2) 181618316Swollman panic("pmap_zero_page: CMAP busy"); 181718316Swollman 181818316Swollman *(int *) CMAP2 = PG_V | PG_KW | (phys & PG_FRAME); 181918316Swollman bzero(CADDR2, PAGE_SIZE); 182018316Swollman 182118316Swollman *(int *) CMAP2 = 0; 182218316Swollman pmap_update_1pg((vm_offset_t) CADDR2); 182318316Swollman} 182418316Swollman 182518316Swollman/* 182618316Swollman * pmap_copy_page copies the specified (machine independent) 182718316Swollman * page by mapping the page into virtual memory and using 182818316Swollman * bcopy to copy the page, one machine dependent page at a 182918316Swollman * time. 183018316Swollman */ 183118316Swollmanvoid 183218316Swollmanpmap_copy_page(src, dst) 183318316Swollman vm_offset_t src; 183418316Swollman vm_offset_t dst; 183518316Swollman{ 183618316Swollman if (*(int *) CMAP1 || *(int *) CMAP2) 183718316Swollman panic("pmap_copy_page: CMAP busy"); 183818316Swollman 183918316Swollman *(int *) CMAP1 = PG_V | PG_KW | (src & PG_FRAME); 184018316Swollman *(int *) CMAP2 = PG_V | PG_KW | (dst & PG_FRAME); 184118316Swollman 184218316Swollman#if __GNUC__ > 1 184318316Swollman memcpy(CADDR2, CADDR1, PAGE_SIZE); 184418316Swollman#else 184518316Swollman bcopy(CADDR1, CADDR2, PAGE_SIZE); 184618316Swollman#endif 184718316Swollman *(int *) CMAP1 = 0; 184818316Swollman *(int *) CMAP2 = 0; 184918316Swollman pmap_update_2pg( (vm_offset_t) CADDR1, (vm_offset_t) CADDR2); 185018316Swollman} 185118316Swollman 185218316Swollman 185318316Swollman/* 185418316Swollman * Routine: pmap_pageable 185518316Swollman * Function: 185618316Swollman * Make the specified pages (by pmap, offset) 185718316Swollman * pageable (or not) as requested. 185818316Swollman * 185918316Swollman * A page which is not pageable may not take 186018316Swollman * a fault; therefore, its page table entry 186118316Swollman * must remain valid for the duration. 186218316Swollman * 186318316Swollman * This routine is merely advisory; pmap_enter 186418316Swollman * will specify that these pages are to be wired 186518316Swollman * down (or not) as appropriate. 186618316Swollman */ 186718316Swollmanvoid 186818316Swollmanpmap_pageable(pmap, sva, eva, pageable) 186918316Swollman pmap_t pmap; 187018316Swollman vm_offset_t sva, eva; 187118316Swollman boolean_t pageable; 187218316Swollman{ 187318316Swollman} 187418316Swollman 187518316Swollman/* 187618316Swollman * this routine returns true if a physical page resides 187718316Swollman * in the given pmap. 187818316Swollman */ 187918316Swollmanboolean_t 188018316Swollmanpmap_page_exists(pmap, pa) 188118316Swollman pmap_t pmap; 188218316Swollman vm_offset_t pa; 188318316Swollman{ 188418316Swollman register pv_entry_t pv; 188518316Swollman int s; 188618316Swollman 188718316Swollman if (!pmap_is_managed(pa)) 188818316Swollman return FALSE; 188918316Swollman 189018316Swollman pv = pa_to_pvh(pa); 189118316Swollman s = splhigh(); 189218316Swollman 189318316Swollman /* 189418316Swollman * Not found, check current mappings returning immediately if found. 189518316Swollman */ 189618316Swollman if (pv->pv_pmap != NULL) { 189718316Swollman for (; pv; pv = pv->pv_next) { 189818316Swollman if (pv->pv_pmap == pmap) { 189918316Swollman splx(s); 190018316Swollman return TRUE; 190118316Swollman } 190218316Swollman } 190318316Swollman } 190418316Swollman splx(s); 190518316Swollman return (FALSE); 190618316Swollman} 190718316Swollman 190818316Swollman/* 190918316Swollman * pmap_testbit tests bits in pte's 191018316Swollman * note that the testbit/changebit routines are inline, 191118316Swollman * and a lot of things compile-time evaluate. 191218316Swollman */ 191318316Swollmanstatic __inline boolean_t 191418316Swollmanpmap_testbit(pa, bit) 191518316Swollman register vm_offset_t pa; 191618316Swollman int bit; 191718316Swollman{ 191818316Swollman register pv_entry_t pv; 191918316Swollman pt_entry_t *pte; 192018316Swollman int s; 192118316Swollman 192218316Swollman if (!pmap_is_managed(pa)) 192318316Swollman return FALSE; 192418316Swollman 192518316Swollman pv = pa_to_pvh(pa); 192618316Swollman s = splhigh(); 192718316Swollman 192818316Swollman /* 192918316Swollman * Not found, check current mappings returning immediately if found. 193018316Swollman */ 193118316Swollman if (pv->pv_pmap != NULL) { 193218316Swollman for (; pv; pv = pv->pv_next) { 193318316Swollman /* 193418316Swollman * if the bit being tested is the modified bit, then 193518316Swollman * mark UPAGES as always modified, and ptes as never 193618316Swollman * modified. 193718316Swollman */ 193818316Swollman if (bit & (PG_U|PG_M)) { 193918316Swollman if ((pv->pv_va >= clean_sva) && (pv->pv_va < clean_eva)) { 194018316Swollman continue; 194118316Swollman } 194218316Swollman } 194318316Swollman if (!pv->pv_pmap) { 194418316Swollman#if defined(PMAP_DIAGNOSTIC) 194518316Swollman printf("Null pmap (tb) at va: 0x%lx\n", pv->pv_va); 194618316Swollman#endif 194718316Swollman continue; 194818316Swollman } 194918316Swollman pte = pmap_pte(pv->pv_pmap, pv->pv_va); 195018316Swollman if ((int) *pte & bit) { 195118316Swollman splx(s); 195218316Swollman return TRUE; 195318316Swollman } 195418316Swollman } 195518316Swollman } 195618316Swollman splx(s); 195718316Swollman return (FALSE); 195818316Swollman} 195918316Swollman 196018316Swollman/* 196118316Swollman * this routine is used to modify bits in ptes 196218316Swollman */ 196318316Swollmanstatic __inline void 196418316Swollmanpmap_changebit(pa, bit, setem) 196518316Swollman vm_offset_t pa; 196618316Swollman int bit; 196718316Swollman boolean_t setem; 196818316Swollman{ 196918316Swollman register pv_entry_t pv; 197018316Swollman register pt_entry_t *pte, npte; 1971 vm_offset_t va; 1972 int changed; 1973 int s; 1974 1975 if (!pmap_is_managed(pa)) 1976 return; 1977 1978 pv = pa_to_pvh(pa); 1979 s = splhigh(); 1980 1981 /* 1982 * Loop over all current mappings setting/clearing as appropos If 1983 * setting RO do we need to clear the VAC? 1984 */ 1985 if (pv->pv_pmap != NULL) { 1986 for (; pv; pv = pv->pv_next) { 1987 va = pv->pv_va; 1988 1989 /* 1990 * don't write protect pager mappings 1991 */ 1992 if (!setem && (bit == PG_RW)) { 1993 if (va >= clean_sva && va < clean_eva) 1994 continue; 1995 } 1996 if (!pv->pv_pmap) { 1997#if defined(PMAP_DIAGNOSTIC) 1998 printf("Null pmap (cb) at va: 0x%lx\n", va); 1999#endif 2000 continue; 2001 } 2002 2003 pte = pmap_pte(pv->pv_pmap, va); 2004 if (setem) { 2005 *(int *)pte |= bit; 2006 } else { 2007 if (bit == PG_RW) { 2008 vm_offset_t pbits = *(vm_offset_t *)pte; 2009 if (pbits & PG_M) { 2010 vm_page_t m; 2011 vm_offset_t pa = pbits & PG_FRAME; 2012 m = PHYS_TO_VM_PAGE(pa); 2013 m->dirty = VM_PAGE_BITS_ALL; 2014 } 2015 *(int *)pte &= ~(PG_M|PG_RW); 2016 } else { 2017 *(int *)pte &= ~bit; 2018 } 2019 } 2020 } 2021 } 2022 splx(s); 2023 pmap_update(); 2024} 2025 2026/* 2027 * pmap_page_protect: 2028 * 2029 * Lower the permission for all mappings to a given page. 2030 */ 2031void 2032pmap_page_protect(phys, prot) 2033 vm_offset_t phys; 2034 vm_prot_t prot; 2035{ 2036 if ((prot & VM_PROT_WRITE) == 0) { 2037 if (prot & (VM_PROT_READ | VM_PROT_EXECUTE)) 2038 pmap_changebit(phys, PG_RW, FALSE); 2039 else 2040 pmap_remove_all(phys); 2041 } 2042} 2043 2044vm_offset_t 2045pmap_phys_address(ppn) 2046 int ppn; 2047{ 2048 return (i386_ptob(ppn)); 2049} 2050 2051/* 2052 * pmap_is_referenced: 2053 * 2054 * Return whether or not the specified physical page was referenced 2055 * by any physical maps. 2056 */ 2057boolean_t 2058pmap_is_referenced(vm_offset_t pa) 2059{ 2060 return pmap_testbit((pa), PG_U); 2061} 2062 2063/* 2064 * pmap_is_modified: 2065 * 2066 * Return whether or not the specified physical page was modified 2067 * in any physical maps. 2068 */ 2069boolean_t 2070pmap_is_modified(vm_offset_t pa) 2071{ 2072 return pmap_testbit((pa), PG_M); 2073} 2074 2075/* 2076 * Clear the modify bits on the specified physical page. 2077 */ 2078void 2079pmap_clear_modify(vm_offset_t pa) 2080{ 2081 pmap_changebit((pa), PG_M, FALSE); 2082} 2083 2084/* 2085 * pmap_clear_reference: 2086 * 2087 * Clear the reference bit on the specified physical page. 2088 */ 2089void 2090pmap_clear_reference(vm_offset_t pa) 2091{ 2092 pmap_changebit((pa), PG_U, FALSE); 2093} 2094 2095/* 2096 * Miscellaneous support routines follow 2097 */ 2098 2099static void 2100i386_protection_init() 2101{ 2102 register int *kp, prot; 2103 2104 kp = protection_codes; 2105 for (prot = 0; prot < 8; prot++) { 2106 switch (prot) { 2107 case VM_PROT_NONE | VM_PROT_NONE | VM_PROT_NONE: 2108 /* 2109 * Read access is also 0. There isn't any execute bit, 2110 * so just make it readable. 2111 */ 2112 case VM_PROT_READ | VM_PROT_NONE | VM_PROT_NONE: 2113 case VM_PROT_READ | VM_PROT_NONE | VM_PROT_EXECUTE: 2114 case VM_PROT_NONE | VM_PROT_NONE | VM_PROT_EXECUTE: 2115 *kp++ = 0; 2116 break; 2117 case VM_PROT_NONE | VM_PROT_WRITE | VM_PROT_NONE: 2118 case VM_PROT_NONE | VM_PROT_WRITE | VM_PROT_EXECUTE: 2119 case VM_PROT_READ | VM_PROT_WRITE | VM_PROT_NONE: 2120 case VM_PROT_READ | VM_PROT_WRITE | VM_PROT_EXECUTE: 2121 *kp++ = PG_RW; 2122 break; 2123 } 2124 } 2125} 2126 2127/* 2128 * Map a set of physical memory pages into the kernel virtual 2129 * address space. Return a pointer to where it is mapped. This 2130 * routine is intended to be used for mapping device memory, 2131 * NOT real memory. The non-cacheable bits are set on each 2132 * mapped page. 2133 */ 2134void * 2135pmap_mapdev(pa, size) 2136 vm_offset_t pa; 2137 vm_size_t size; 2138{ 2139 vm_offset_t va, tmpva; 2140 pt_entry_t *pte; 2141 2142 size = roundup(size, PAGE_SIZE); 2143 2144 va = kmem_alloc_pageable(kernel_map, size); 2145 if (!va) 2146 panic("pmap_mapdev: Couldn't alloc kernel virtual memory"); 2147 2148 pa = pa & PG_FRAME; 2149 for (tmpva = va; size > 0;) { 2150 pte = vtopte(tmpva); 2151 *pte = (pt_entry_t) ((int) (pa | PG_RW | PG_V | PG_N)); 2152 size -= PAGE_SIZE; 2153 tmpva += PAGE_SIZE; 2154 pa += PAGE_SIZE; 2155 } 2156 pmap_update(); 2157 2158 return ((void *) va); 2159} 2160 2161#if defined(PMAP_DEBUG) 2162pmap_pid_dump(int pid) { 2163 pmap_t pmap; 2164 struct proc *p; 2165 int npte = 0; 2166 int index; 2167 for (p = allproc.lh_first; p != NULL; p = p->p_list.le_next) { 2168 if (p->p_pid != pid) 2169 continue; 2170 2171 if (p->p_vmspace) { 2172 int i,j; 2173 index = 0; 2174 pmap = &p->p_vmspace->vm_pmap; 2175 for(i=0;i<1024;i++) { 2176 pd_entry_t *pde; 2177 pt_entry_t *pte; 2178 unsigned base = i << PD_SHIFT; 2179 2180 pde = &pmap->pm_pdir[i]; 2181 if (pde && pmap_pde_v(pde)) { 2182 for(j=0;j<1024;j++) { 2183 unsigned va = base + (j << PG_SHIFT); 2184 if (va >= (vm_offset_t) VM_MIN_KERNEL_ADDRESS) { 2185 if (index) { 2186 index = 0; 2187 printf("\n"); 2188 } 2189 return npte; 2190 } 2191 pte = pmap_pte( pmap, va); 2192 if (pte && pmap_pte_v(pte)) { 2193 vm_offset_t pa; 2194 vm_page_t m; 2195 pa = *(int *)pte; 2196 m = PHYS_TO_VM_PAGE((pa & PG_FRAME)); 2197 printf("va: 0x%x, pt: 0x%x, h: %d, w: %d, f: 0x%x", 2198 va, pa, m->hold_count, m->wire_count, m->flags); 2199 npte++; 2200 index++; 2201 if (index >= 2) { 2202 index = 0; 2203 printf("\n"); 2204 } else { 2205 printf(" "); 2206 } 2207 } 2208 } 2209 } 2210 } 2211 } 2212 } 2213 return npte; 2214} 2215#endif 2216 2217#if defined(DEBUG) 2218 2219static void pads __P((pmap_t pm)); 2220static void pmap_pvdump __P((vm_offset_t pa)); 2221 2222/* print address space of pmap*/ 2223static void 2224pads(pm) 2225 pmap_t pm; 2226{ 2227 unsigned va, i, j; 2228 pt_entry_t *ptep; 2229 2230 if (pm == kernel_pmap) 2231 return; 2232 for (i = 0; i < 1024; i++) 2233 if (pm->pm_pdir[i]) 2234 for (j = 0; j < 1024; j++) { 2235 va = (i << PD_SHIFT) + (j << PG_SHIFT); 2236 if (pm == kernel_pmap && va < KERNBASE) 2237 continue; 2238 if (pm != kernel_pmap && va > UPT_MAX_ADDRESS) 2239 continue; 2240 ptep = pmap_pte(pm, va); 2241 if (pmap_pte_v(ptep)) 2242 printf("%x:%x ", va, *(int *) ptep); 2243 }; 2244 2245} 2246 2247static void 2248pmap_pvdump(pa) 2249 vm_offset_t pa; 2250{ 2251 register pv_entry_t pv; 2252 2253 printf("pa %x", pa); 2254 for (pv = pa_to_pvh(pa); pv; pv = pv->pv_next) { 2255#ifdef used_to_be 2256 printf(" -> pmap %x, va %x, flags %x", 2257 pv->pv_pmap, pv->pv_va, pv->pv_flags); 2258#endif 2259 printf(" -> pmap %x, va %x", 2260 pv->pv_pmap, pv->pv_va); 2261 pads(pv->pv_pmap); 2262 } 2263 printf(" "); 2264} 2265#endif 2266