1178172Simp/* 2178172Simp * Copyright (c) 1991 Regents of the University of California. 3178172Simp * All rights reserved. 4178172Simp * Copyright (c) 1994 John S. Dyson 5178172Simp * All rights reserved. 6178172Simp * Copyright (c) 1994 David Greenman 7178172Simp * All rights reserved. 8178172Simp * 9178172Simp * This code is derived from software contributed to Berkeley by 10178172Simp * the Systems Programming Group of the University of Utah Computer 11178172Simp * Science Department and William Jolitz of UUNET Technologies Inc. 12178172Simp * 13178172Simp * Redistribution and use in source and binary forms, with or without 14178172Simp * modification, are permitted provided that the following conditions 15178172Simp * are met: 16178172Simp * 1. Redistributions of source code must retain the above copyright 17178172Simp * notice, this list of conditions and the following disclaimer. 18178172Simp * 2. Redistributions in binary form must reproduce the above copyright 19178172Simp * notice, this list of conditions and the following disclaimer in the 20178172Simp * documentation and/or other materials provided with the distribution. 21178172Simp * 4. Neither the name of the University nor the names of its contributors 22178172Simp * may be used to endorse or promote products derived from this software 23178172Simp * without specific prior written permission. 24178172Simp * 25178172Simp * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 26178172Simp * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 27178172Simp * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 28178172Simp * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 29178172Simp * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 30178172Simp * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 31178172Simp * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 32178172Simp * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 33178172Simp * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 34178172Simp * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 35178172Simp * SUCH DAMAGE. 36178172Simp * 37178172Simp * from: @(#)pmap.c 7.7 (Berkeley) 5/12/91 38178172Simp * from: src/sys/i386/i386/pmap.c,v 1.250.2.8 2000/11/21 00:09:14 ps 39178172Simp * JNPR: pmap.c,v 1.11.2.1 2007/08/16 11:51:06 girish 40178172Simp */ 41178172Simp 42178172Simp/* 43178172Simp * Manages physical address maps. 44178172Simp * 45178172Simp * In addition to hardware address maps, this 46178172Simp * module is called upon to provide software-use-only 47178172Simp * maps which may or may not be stored in the same 48178172Simp * form as hardware maps. These pseudo-maps are 49178172Simp * used to store intermediate results from copy 50178172Simp * operations to and from address spaces. 51178172Simp * 52178172Simp * Since the information managed by this module is 53178172Simp * also stored by the logical address mapping module, 54178172Simp * this module may throw away valid virtual-to-physical 55178172Simp * mappings at almost any time. However, invalidations 56178172Simp * of virtual-to-physical mappings must be done as 57178172Simp * requested. 58178172Simp * 59178172Simp * In order to cope with hardware architectures which 60178172Simp * make virtual-to-physical map invalidates expensive, 61178172Simp * this module may delay invalidate or reduced protection 62178172Simp * operations until such time as they are actually 63178172Simp * necessary. This module is given full information as 64178172Simp * to which processors are currently using which maps, 65178172Simp * and to when physical maps must be made correct. 66178172Simp */ 67178172Simp 68178172Simp#include <sys/cdefs.h> 69178172Simp__FBSDID("$FreeBSD$"); 70178172Simp 71210846Sjchandra#include "opt_ddb.h" 72210846Sjchandra 73178172Simp#include <sys/param.h> 74178172Simp#include <sys/systm.h> 75178172Simp#include <sys/proc.h> 76178172Simp#include <sys/msgbuf.h> 77178172Simp#include <sys/vmmeter.h> 78178172Simp#include <sys/mman.h> 79205064Sneel#include <sys/smp.h> 80210846Sjchandra#ifdef DDB 81210846Sjchandra#include <ddb/ddb.h> 82210846Sjchandra#endif 83178172Simp 84178172Simp#include <vm/vm.h> 85178172Simp#include <vm/vm_param.h> 86208165Srrs#include <vm/vm_phys.h> 87178172Simp#include <sys/lock.h> 88178172Simp#include <sys/mutex.h> 89178172Simp#include <vm/vm_kern.h> 90178172Simp#include <vm/vm_page.h> 91178172Simp#include <vm/vm_map.h> 92178172Simp#include <vm/vm_object.h> 93178172Simp#include <vm/vm_extern.h> 94178172Simp#include <vm/vm_pageout.h> 95178172Simp#include <vm/vm_pager.h> 96178172Simp#include <vm/uma.h> 97178172Simp#include <sys/pcpu.h> 98178172Simp#include <sys/sched.h> 99178172Simp#ifdef SMP 100178172Simp#include <sys/smp.h> 101178172Simp#endif 102178172Simp 103178172Simp#include <machine/cache.h> 104178172Simp#include <machine/md_var.h> 105209243Sjchandra#include <machine/tlb.h> 106178172Simp 107187301Sgonzo#undef PMAP_DEBUG 108187301Sgonzo 109178172Simp#ifndef PMAP_SHPGPERPROC 110178172Simp#define PMAP_SHPGPERPROC 200 111178172Simp#endif 112178172Simp 113211958Sjchandra#if !defined(DIAGNOSTIC) 114178172Simp#define PMAP_INLINE __inline 115178172Simp#else 116178172Simp#define PMAP_INLINE 117178172Simp#endif 118178172Simp 119178172Simp/* 120178172Simp * Get PDEs and PTEs for user/kernel address space 121178172Simp */ 122210846Sjchandra#define pmap_seg_index(v) (((v) >> SEGSHIFT) & (NPDEPG - 1)) 123210846Sjchandra#define pmap_pde_index(v) (((v) >> PDRSHIFT) & (NPDEPG - 1)) 124210846Sjchandra#define pmap_pte_index(v) (((v) >> PAGE_SHIFT) & (NPTEPG - 1)) 125210846Sjchandra#define pmap_pde_pindex(v) ((v) >> PDRSHIFT) 126178172Simp 127210846Sjchandra#ifdef __mips_n64 128210846Sjchandra#define NUPDE (NPDEPG * NPDEPG) 129210846Sjchandra#define NUSERPGTBLS (NUPDE + NPDEPG) 130209930Sjchandra#else 131210846Sjchandra#define NUPDE (NPDEPG) 132210846Sjchandra#define NUSERPGTBLS (NUPDE) 133209930Sjchandra#endif 134210846Sjchandra 135178172Simp#define is_kernel_pmap(x) ((x) == kernel_pmap) 136178172Simp 137191735Salcstruct pmap kernel_pmap_store; 138178172Simppd_entry_t *kernel_segmap; 139178172Simp 140178172Simpvm_offset_t virtual_avail; /* VA of first avail page (after kernel bss) */ 141178172Simpvm_offset_t virtual_end; /* VA of last avail page (end of kernel AS) */ 142178172Simp 143178172Simpstatic int nkpt; 144178172Simpunsigned pmap_max_asid; /* max ASID supported by the system */ 145178172Simp 146178172Simp#define PMAP_ASID_RESERVED 0 147178172Simp 148210846Sjchandravm_offset_t kernel_vm_end = VM_MIN_KERNEL_ADDRESS; 149178172Simp 150178172Simpstatic void pmap_asid_alloc(pmap_t pmap); 151178172Simp 152178172Simp/* 153178172Simp * Data for the pv entry allocation mechanism 154178172Simp */ 155178172Simpstatic uma_zone_t pvzone; 156178172Simpstatic struct vm_object pvzone_obj; 157178172Simpstatic int pv_entry_count = 0, pv_entry_max = 0, pv_entry_high_water = 0; 158178172Simp 159178172Simpstatic PMAP_INLINE void free_pv_entry(pv_entry_t pv); 160188507Simpstatic pv_entry_t get_pv_entry(pmap_t locked_pmap); 161208665Salcstatic void pmap_pvh_free(struct md_page *pvh, pmap_t pmap, vm_offset_t va); 162208665Salcstatic pv_entry_t pmap_pvh_remove(struct md_page *pvh, pmap_t pmap, 163208665Salc vm_offset_t va); 164178172Simpstatic __inline void pmap_changebit(vm_page_t m, int bit, boolean_t setem); 165191300Salcstatic vm_page_t pmap_enter_quick_locked(pmap_t pmap, vm_offset_t va, 166191300Salc vm_page_t m, vm_prot_t prot, vm_page_t mpte); 167178172Simpstatic int pmap_remove_pte(struct pmap *pmap, pt_entry_t *ptq, vm_offset_t va); 168178172Simpstatic void pmap_remove_page(struct pmap *pmap, vm_offset_t va); 169178172Simpstatic void pmap_remove_entry(struct pmap *pmap, vm_page_t m, vm_offset_t va); 170191300Salcstatic boolean_t pmap_try_insert_pv_entry(pmap_t pmap, vm_page_t mpte, 171191300Salc vm_offset_t va, vm_page_t m); 172211217Sjchandrastatic void pmap_update_page(pmap_t pmap, vm_offset_t va, pt_entry_t pte); 173211215Sjchandrastatic void pmap_invalidate_all(pmap_t pmap); 174211215Sjchandrastatic void pmap_invalidate_page(pmap_t pmap, vm_offset_t va); 175240151Skibstatic void _pmap_unwire_ptp(pmap_t pmap, vm_offset_t va, vm_page_t m); 176178172Simp 177178172Simpstatic vm_page_t pmap_allocpte(pmap_t pmap, vm_offset_t va, int flags); 178178172Simpstatic vm_page_t _pmap_allocpte(pmap_t pmap, unsigned ptepindex, int flags); 179178172Simpstatic int pmap_unuse_pt(pmap_t, vm_offset_t, vm_page_t); 180217345Sjchandrastatic pt_entry_t init_pte_prot(vm_offset_t va, vm_page_t m, vm_prot_t prot); 181178172Simp 182178172Simp#ifdef SMP 183178172Simpstatic void pmap_invalidate_page_action(void *arg); 184178172Simpstatic void pmap_invalidate_all_action(void *arg); 185178172Simpstatic void pmap_update_page_action(void *arg); 186208165Srrs#endif 187178172Simp 188211453Sjchandra#ifndef __mips_n64 189211453Sjchandra/* 190216157Sjchandra * This structure is for high memory (memory above 512Meg in 32 bit) support. 191216157Sjchandra * The highmem area does not have a KSEG0 mapping, and we need a mechanism to 192216157Sjchandra * do temporary per-CPU mappings for pmap_zero_page, pmap_copy_page etc. 193211453Sjchandra * 194216157Sjchandra * At bootup, we reserve 2 virtual pages per CPU for mapping highmem pages. To 195216157Sjchandra * access a highmem physical address on a CPU, we map the physical address to 196216157Sjchandra * the reserved virtual address for the CPU in the kernel pagetable. This is 197216157Sjchandra * done with interrupts disabled(although a spinlock and sched_pin would be 198216157Sjchandra * sufficient). 199211453Sjchandra */ 200178172Simpstruct local_sysmaps { 201211453Sjchandra vm_offset_t base; 202211453Sjchandra uint32_t saved_intr; 203211453Sjchandra uint16_t valid1, valid2; 204178172Simp}; 205178172Simpstatic struct local_sysmaps sysmap_lmem[MAXCPU]; 206178172Simp 207211453Sjchandrastatic __inline void 208211453Sjchandrapmap_alloc_lmem_map(void) 209211453Sjchandra{ 210211453Sjchandra int i; 211206717Sjmallett 212211453Sjchandra for (i = 0; i < MAXCPU; i++) { 213211453Sjchandra sysmap_lmem[i].base = virtual_avail; 214211453Sjchandra virtual_avail += PAGE_SIZE * 2; 215211453Sjchandra sysmap_lmem[i].valid1 = sysmap_lmem[i].valid2 = 0; 216211453Sjchandra } 217211453Sjchandra} 218211453Sjchandra 219211453Sjchandrastatic __inline vm_offset_t 220211453Sjchandrapmap_lmem_map1(vm_paddr_t phys) 221211453Sjchandra{ 222211453Sjchandra struct local_sysmaps *sysm; 223211453Sjchandra pt_entry_t *pte, npte; 224211453Sjchandra vm_offset_t va; 225211453Sjchandra uint32_t intr; 226211453Sjchandra int cpu; 227211453Sjchandra 228211453Sjchandra intr = intr_disable(); 229211453Sjchandra cpu = PCPU_GET(cpuid); 230211453Sjchandra sysm = &sysmap_lmem[cpu]; 231211453Sjchandra sysm->saved_intr = intr; 232211453Sjchandra va = sysm->base; 233211453Sjchandra npte = TLBLO_PA_TO_PFN(phys) | 234211453Sjchandra PTE_D | PTE_V | PTE_G | PTE_W | PTE_C_CACHE; 235211453Sjchandra pte = pmap_pte(kernel_pmap, va); 236211453Sjchandra *pte = npte; 237211453Sjchandra sysm->valid1 = 1; 238211453Sjchandra return (va); 239211453Sjchandra} 240211453Sjchandra 241211453Sjchandrastatic __inline vm_offset_t 242211453Sjchandrapmap_lmem_map2(vm_paddr_t phys1, vm_paddr_t phys2) 243211453Sjchandra{ 244211453Sjchandra struct local_sysmaps *sysm; 245211453Sjchandra pt_entry_t *pte, npte; 246211453Sjchandra vm_offset_t va1, va2; 247211453Sjchandra uint32_t intr; 248211453Sjchandra int cpu; 249211453Sjchandra 250211453Sjchandra intr = intr_disable(); 251211453Sjchandra cpu = PCPU_GET(cpuid); 252211453Sjchandra sysm = &sysmap_lmem[cpu]; 253211453Sjchandra sysm->saved_intr = intr; 254211453Sjchandra va1 = sysm->base; 255211453Sjchandra va2 = sysm->base + PAGE_SIZE; 256211453Sjchandra npte = TLBLO_PA_TO_PFN(phys1) | 257211453Sjchandra PTE_D | PTE_V | PTE_G | PTE_W | PTE_C_CACHE; 258211453Sjchandra pte = pmap_pte(kernel_pmap, va1); 259211453Sjchandra *pte = npte; 260211453Sjchandra npte = TLBLO_PA_TO_PFN(phys2) | 261211453Sjchandra PTE_D | PTE_V | PTE_G | PTE_W | PTE_C_CACHE; 262211453Sjchandra pte = pmap_pte(kernel_pmap, va2); 263211453Sjchandra *pte = npte; 264211453Sjchandra sysm->valid1 = 1; 265206717Sjmallett sysm->valid2 = 1; 266211453Sjchandra return (va1); 267211453Sjchandra} 268206717Sjmallett 269211453Sjchandrastatic __inline void 270211453Sjchandrapmap_lmem_unmap(void) 271211453Sjchandra{ 272211453Sjchandra struct local_sysmaps *sysm; 273211453Sjchandra pt_entry_t *pte; 274211453Sjchandra int cpu; 275206717Sjmallett 276211453Sjchandra cpu = PCPU_GET(cpuid); 277211453Sjchandra sysm = &sysmap_lmem[cpu]; 278211453Sjchandra pte = pmap_pte(kernel_pmap, sysm->base); 279211453Sjchandra *pte = PTE_G; 280211453Sjchandra tlb_invalidate_address(kernel_pmap, sysm->base); 281211453Sjchandra sysm->valid1 = 0; 282211453Sjchandra if (sysm->valid2) { 283211453Sjchandra pte = pmap_pte(kernel_pmap, sysm->base + PAGE_SIZE); 284211453Sjchandra *pte = PTE_G; 285211453Sjchandra tlb_invalidate_address(kernel_pmap, sysm->base + PAGE_SIZE); 286211453Sjchandra sysm->valid2 = 0; 287211453Sjchandra } 288211453Sjchandra intr_restore(sysm->saved_intr); 289211453Sjchandra} 290211453Sjchandra#else /* __mips_n64 */ 291211453Sjchandra 292211453Sjchandrastatic __inline void 293211453Sjchandrapmap_alloc_lmem_map(void) 294211453Sjchandra{ 295211453Sjchandra} 296211453Sjchandra 297211453Sjchandrastatic __inline vm_offset_t 298211453Sjchandrapmap_lmem_map1(vm_paddr_t phys) 299211453Sjchandra{ 300211453Sjchandra 301211453Sjchandra return (0); 302211453Sjchandra} 303211453Sjchandra 304211453Sjchandrastatic __inline vm_offset_t 305211453Sjchandrapmap_lmem_map2(vm_paddr_t phys1, vm_paddr_t phys2) 306211453Sjchandra{ 307211453Sjchandra 308211453Sjchandra return (0); 309211453Sjchandra} 310211453Sjchandra 311211453Sjchandrastatic __inline vm_offset_t 312211453Sjchandrapmap_lmem_unmap(void) 313211453Sjchandra{ 314211453Sjchandra 315211453Sjchandra return (0); 316211453Sjchandra} 317211453Sjchandra#endif /* !__mips_n64 */ 318211453Sjchandra 319210846Sjchandra/* 320210846Sjchandra * Page table entry lookup routines. 321210846Sjchandra */ 322210846Sjchandrastatic __inline pd_entry_t * 323178172Simppmap_segmap(pmap_t pmap, vm_offset_t va) 324178172Simp{ 325211445Sjchandra 326210846Sjchandra return (&pmap->pm_segtab[pmap_seg_index(va)]); 327210846Sjchandra} 328210846Sjchandra 329210846Sjchandra#ifdef __mips_n64 330210846Sjchandrastatic __inline pd_entry_t * 331210846Sjchandrapmap_pdpe_to_pde(pd_entry_t *pdpe, vm_offset_t va) 332210846Sjchandra{ 333210846Sjchandra pd_entry_t *pde; 334210846Sjchandra 335210846Sjchandra pde = (pd_entry_t *)*pdpe; 336210846Sjchandra return (&pde[pmap_pde_index(va)]); 337210846Sjchandra} 338210846Sjchandra 339210846Sjchandrastatic __inline pd_entry_t * 340210846Sjchandrapmap_pde(pmap_t pmap, vm_offset_t va) 341210846Sjchandra{ 342210846Sjchandra pd_entry_t *pdpe; 343210846Sjchandra 344210846Sjchandra pdpe = pmap_segmap(pmap, va); 345210846Sjchandra if (pdpe == NULL || *pdpe == NULL) 346209805Sjchandra return (NULL); 347210846Sjchandra 348210846Sjchandra return (pmap_pdpe_to_pde(pdpe, va)); 349178172Simp} 350210846Sjchandra#else 351210846Sjchandrastatic __inline pd_entry_t * 352210846Sjchandrapmap_pdpe_to_pde(pd_entry_t *pdpe, vm_offset_t va) 353210846Sjchandra{ 354211445Sjchandra 355211445Sjchandra return (pdpe); 356210846Sjchandra} 357178172Simp 358210846Sjchandrastatic __inline 359210846Sjchandrapd_entry_t *pmap_pde(pmap_t pmap, vm_offset_t va) 360210846Sjchandra{ 361211445Sjchandra 362211445Sjchandra return (pmap_segmap(pmap, va)); 363210846Sjchandra} 364210846Sjchandra#endif 365210846Sjchandra 366210846Sjchandrastatic __inline pt_entry_t * 367210846Sjchandrapmap_pde_to_pte(pd_entry_t *pde, vm_offset_t va) 368210846Sjchandra{ 369210846Sjchandra pt_entry_t *pte; 370210846Sjchandra 371210846Sjchandra pte = (pt_entry_t *)*pde; 372210846Sjchandra return (&pte[pmap_pte_index(va)]); 373210846Sjchandra} 374210846Sjchandra 375178172Simppt_entry_t * 376178172Simppmap_pte(pmap_t pmap, vm_offset_t va) 377178172Simp{ 378210846Sjchandra pd_entry_t *pde; 379178172Simp 380210846Sjchandra pde = pmap_pde(pmap, va); 381210846Sjchandra if (pde == NULL || *pde == NULL) 382210846Sjchandra return (NULL); 383210846Sjchandra 384210846Sjchandra return (pmap_pde_to_pte(pde, va)); 385178172Simp} 386178172Simp 387178172Simpvm_offset_t 388178172Simppmap_steal_memory(vm_size_t size) 389178172Simp{ 390219106Sjchandra vm_paddr_t bank_size, pa; 391219106Sjchandra vm_offset_t va; 392178172Simp 393178172Simp size = round_page(size); 394178172Simp bank_size = phys_avail[1] - phys_avail[0]; 395178172Simp while (size > bank_size) { 396178172Simp int i; 397178172Simp 398178172Simp for (i = 0; phys_avail[i + 2]; i += 2) { 399178172Simp phys_avail[i] = phys_avail[i + 2]; 400178172Simp phys_avail[i + 1] = phys_avail[i + 3]; 401178172Simp } 402178172Simp phys_avail[i] = 0; 403178172Simp phys_avail[i + 1] = 0; 404178172Simp if (!phys_avail[0]) 405178172Simp panic("pmap_steal_memory: out of memory"); 406178172Simp bank_size = phys_avail[1] - phys_avail[0]; 407178172Simp } 408178172Simp 409178172Simp pa = phys_avail[0]; 410178172Simp phys_avail[0] += size; 411211453Sjchandra if (MIPS_DIRECT_MAPPABLE(pa) == 0) 412178172Simp panic("Out of memory below 512Meg?"); 413211453Sjchandra va = MIPS_PHYS_TO_DIRECT(pa); 414178172Simp bzero((caddr_t)va, size); 415211445Sjchandra return (va); 416178172Simp} 417178172Simp 418178172Simp/* 419209930Sjchandra * Bootstrap the system enough to run with virtual memory. This 420178172Simp * assumes that the phys_avail array has been initialized. 421178172Simp */ 422210846Sjchandrastatic void 423210846Sjchandrapmap_create_kernel_pagetable(void) 424210846Sjchandra{ 425210846Sjchandra int i, j; 426210846Sjchandra vm_offset_t ptaddr; 427210846Sjchandra pt_entry_t *pte; 428210846Sjchandra#ifdef __mips_n64 429210846Sjchandra pd_entry_t *pde; 430210846Sjchandra vm_offset_t pdaddr; 431210846Sjchandra int npt, npde; 432210846Sjchandra#endif 433210846Sjchandra 434210846Sjchandra /* 435210846Sjchandra * Allocate segment table for the kernel 436210846Sjchandra */ 437210846Sjchandra kernel_segmap = (pd_entry_t *)pmap_steal_memory(PAGE_SIZE); 438210846Sjchandra 439210846Sjchandra /* 440210846Sjchandra * Allocate second level page tables for the kernel 441210846Sjchandra */ 442210846Sjchandra#ifdef __mips_n64 443210846Sjchandra npde = howmany(NKPT, NPDEPG); 444210846Sjchandra pdaddr = pmap_steal_memory(PAGE_SIZE * npde); 445210846Sjchandra#endif 446210846Sjchandra nkpt = NKPT; 447210846Sjchandra ptaddr = pmap_steal_memory(PAGE_SIZE * nkpt); 448210846Sjchandra 449210846Sjchandra /* 450210846Sjchandra * The R[4-7]?00 stores only one copy of the Global bit in the 451210846Sjchandra * translation lookaside buffer for each 2 page entry. Thus invalid 452210846Sjchandra * entrys must have the Global bit set so when Entry LO and Entry HI 453210846Sjchandra * G bits are anded together they will produce a global bit to store 454210846Sjchandra * in the tlb. 455210846Sjchandra */ 456210846Sjchandra for (i = 0, pte = (pt_entry_t *)ptaddr; i < (nkpt * NPTEPG); i++, pte++) 457210846Sjchandra *pte = PTE_G; 458210846Sjchandra 459210846Sjchandra#ifdef __mips_n64 460210846Sjchandra for (i = 0, npt = nkpt; npt > 0; i++) { 461210846Sjchandra kernel_segmap[i] = (pd_entry_t)(pdaddr + i * PAGE_SIZE); 462210846Sjchandra pde = (pd_entry_t *)kernel_segmap[i]; 463210846Sjchandra 464210846Sjchandra for (j = 0; j < NPDEPG && npt > 0; j++, npt--) 465210846Sjchandra pde[j] = (pd_entry_t)(ptaddr + (i * NPDEPG + j) * PAGE_SIZE); 466210846Sjchandra } 467210846Sjchandra#else 468210846Sjchandra for (i = 0, j = pmap_seg_index(VM_MIN_KERNEL_ADDRESS); i < nkpt; i++, j++) 469210846Sjchandra kernel_segmap[j] = (pd_entry_t)(ptaddr + (i * PAGE_SIZE)); 470210846Sjchandra#endif 471210846Sjchandra 472210846Sjchandra PMAP_LOCK_INIT(kernel_pmap); 473210846Sjchandra kernel_pmap->pm_segtab = kernel_segmap; 474222813Sattilio CPU_FILL(&kernel_pmap->pm_active); 475210846Sjchandra TAILQ_INIT(&kernel_pmap->pm_pvlist); 476210846Sjchandra kernel_pmap->pm_asid[0].asid = PMAP_ASID_RESERVED; 477210846Sjchandra kernel_pmap->pm_asid[0].gen = 0; 478210846Sjchandra kernel_vm_end += nkpt * NPTEPG * PAGE_SIZE; 479210846Sjchandra} 480210846Sjchandra 481178172Simpvoid 482178172Simppmap_bootstrap(void) 483178172Simp{ 484210846Sjchandra int i; 485211453Sjchandra int need_local_mappings = 0; 486178172Simp 487178172Simp /* Sort. */ 488178172Simpagain: 489178172Simp for (i = 0; phys_avail[i + 1] != 0; i += 2) { 490202046Simp /* 491202046Simp * Keep the memory aligned on page boundary. 492202046Simp */ 493202046Simp phys_avail[i] = round_page(phys_avail[i]); 494202046Simp phys_avail[i + 1] = trunc_page(phys_avail[i + 1]); 495202046Simp 496178172Simp if (i < 2) 497178172Simp continue; 498178172Simp if (phys_avail[i - 2] > phys_avail[i]) { 499178172Simp vm_paddr_t ptemp[2]; 500178172Simp 501178172Simp ptemp[0] = phys_avail[i + 0]; 502178172Simp ptemp[1] = phys_avail[i + 1]; 503178172Simp 504178172Simp phys_avail[i + 0] = phys_avail[i - 2]; 505178172Simp phys_avail[i + 1] = phys_avail[i - 1]; 506178172Simp 507178172Simp phys_avail[i - 2] = ptemp[0]; 508178172Simp phys_avail[i - 1] = ptemp[1]; 509178172Simp goto again; 510178172Simp } 511178172Simp } 512178172Simp 513211453Sjchandra /* 514216157Sjchandra * In 32 bit, we may have memory which cannot be mapped directly. 515216157Sjchandra * This memory will need temporary mapping before it can be 516211453Sjchandra * accessed. 517211453Sjchandra */ 518216157Sjchandra if (!MIPS_DIRECT_MAPPABLE(phys_avail[i - 1] - 1)) 519211453Sjchandra need_local_mappings = 1; 520209930Sjchandra 521202046Simp /* 522202046Simp * Copy the phys_avail[] array before we start stealing memory from it. 523202046Simp */ 524202046Simp for (i = 0; phys_avail[i + 1] != 0; i += 2) { 525202046Simp physmem_desc[i] = phys_avail[i]; 526202046Simp physmem_desc[i + 1] = phys_avail[i + 1]; 527202046Simp } 528202046Simp 529202046Simp Maxmem = atop(phys_avail[i - 1]); 530202046Simp 531178172Simp if (bootverbose) { 532178172Simp printf("Physical memory chunk(s):\n"); 533178172Simp for (i = 0; phys_avail[i + 1] != 0; i += 2) { 534178172Simp vm_paddr_t size; 535178172Simp 536178172Simp size = phys_avail[i + 1] - phys_avail[i]; 537178172Simp printf("%#08jx - %#08jx, %ju bytes (%ju pages)\n", 538178172Simp (uintmax_t) phys_avail[i], 539178172Simp (uintmax_t) phys_avail[i + 1] - 1, 540178172Simp (uintmax_t) size, (uintmax_t) size / PAGE_SIZE); 541178172Simp } 542219106Sjchandra printf("Maxmem is 0x%0jx\n", ptoa((uintmax_t)Maxmem)); 543178172Simp } 544178172Simp /* 545178172Simp * Steal the message buffer from the beginning of memory. 546178172Simp */ 547217688Spluknet msgbufp = (struct msgbuf *)pmap_steal_memory(msgbufsize); 548217688Spluknet msgbufinit(msgbufp, msgbufsize); 549178172Simp 550178172Simp /* 551178172Simp * Steal thread0 kstack. 552178172Simp */ 553178172Simp kstack0 = pmap_steal_memory(KSTACK_PAGES << PAGE_SHIFT); 554178172Simp 555206716Sjmallett virtual_avail = VM_MIN_KERNEL_ADDRESS; 556178172Simp virtual_end = VM_MAX_KERNEL_ADDRESS; 557178172Simp 558203180Sneel#ifdef SMP 559178172Simp /* 560203180Sneel * Steal some virtual address space to map the pcpu area. 561203180Sneel */ 562203180Sneel virtual_avail = roundup2(virtual_avail, PAGE_SIZE * 2); 563203180Sneel pcpup = (struct pcpu *)virtual_avail; 564203180Sneel virtual_avail += PAGE_SIZE * 2; 565203697Sneel 566203697Sneel /* 567203697Sneel * Initialize the wired TLB entry mapping the pcpu region for 568203697Sneel * the BSP at 'pcpup'. Up until this point we were operating 569203697Sneel * with the 'pcpup' for the BSP pointing to a virtual address 570203697Sneel * in KSEG0 so there was no need for a TLB mapping. 571203697Sneel */ 572203697Sneel mips_pcpu_tlb_init(PCPU_ADDR(0)); 573203697Sneel 574203180Sneel if (bootverbose) 575203180Sneel printf("pcpu is available at virtual address %p.\n", pcpup); 576203180Sneel#endif 577203180Sneel 578211453Sjchandra if (need_local_mappings) 579211453Sjchandra pmap_alloc_lmem_map(); 580210846Sjchandra pmap_create_kernel_pagetable(); 581178172Simp pmap_max_asid = VMNUM_PIDS; 582209243Sjchandra mips_wr_entryhi(0); 583210846Sjchandra mips_wr_pagemask(0); 584178172Simp} 585178172Simp 586178172Simp/* 587178172Simp * Initialize a vm_page's machine-dependent fields. 588178172Simp */ 589178172Simpvoid 590178172Simppmap_page_init(vm_page_t m) 591178172Simp{ 592178172Simp 593178172Simp TAILQ_INIT(&m->md.pv_list); 594178172Simp m->md.pv_list_count = 0; 595178172Simp m->md.pv_flags = 0; 596178172Simp} 597178172Simp 598178172Simp/* 599178172Simp * Initialize the pmap module. 600178172Simp * Called by vm_init, to initialize any structures that the pmap 601178172Simp * system needs to map virtual memory. 602178172Simp * pmap_init has been enhanced to support in a fairly consistant 603178172Simp * way, discontiguous physical memory. 604178172Simp */ 605178172Simpvoid 606178172Simppmap_init(void) 607178172Simp{ 608178172Simp 609178172Simp /* 610178172Simp * Initialize the address space (zone) for the pv entries. Set a 611178172Simp * high water mark so that the system can recover from excessive 612178172Simp * numbers of pv entries. 613178172Simp */ 614178172Simp pvzone = uma_zcreate("PV ENTRY", sizeof(struct pv_entry), NULL, NULL, 615178172Simp NULL, NULL, UMA_ALIGN_PTR, UMA_ZONE_VM | UMA_ZONE_NOFREE); 616178172Simp pv_entry_max = PMAP_SHPGPERPROC * maxproc + cnt.v_page_count; 617178172Simp pv_entry_high_water = 9 * (pv_entry_max / 10); 618178172Simp uma_zone_set_obj(pvzone, &pvzone_obj, pv_entry_max); 619178172Simp} 620178172Simp 621178172Simp/*************************************************** 622178172Simp * Low level helper routines..... 623178172Simp ***************************************************/ 624178172Simp 625211215Sjchandrastatic __inline void 626211215Sjchandrapmap_invalidate_all_local(pmap_t pmap) 627211215Sjchandra{ 628223758Sattilio u_int cpuid; 629211215Sjchandra 630223758Sattilio cpuid = PCPU_GET(cpuid); 631223758Sattilio 632211215Sjchandra if (pmap == kernel_pmap) { 633211215Sjchandra tlb_invalidate_all(); 634211215Sjchandra return; 635211215Sjchandra } 636223758Sattilio if (CPU_ISSET(cpuid, &pmap->pm_active)) 637211215Sjchandra tlb_invalidate_all_user(pmap); 638223758Sattilio else 639223758Sattilio pmap->pm_asid[cpuid].gen = 0; 640211215Sjchandra} 641211215Sjchandra 642211215Sjchandra#ifdef SMP 643178172Simpstatic void 644178172Simppmap_invalidate_all(pmap_t pmap) 645178172Simp{ 646211215Sjchandra 647211215Sjchandra smp_rendezvous(0, pmap_invalidate_all_action, 0, pmap); 648178172Simp} 649178172Simp 650178172Simpstatic void 651178172Simppmap_invalidate_all_action(void *arg) 652178172Simp{ 653178172Simp 654211215Sjchandra pmap_invalidate_all_local((pmap_t)arg); 655211215Sjchandra} 656211215Sjchandra#else 657211215Sjchandrastatic void 658211215Sjchandrapmap_invalidate_all(pmap_t pmap) 659211215Sjchandra{ 660211215Sjchandra 661211215Sjchandra pmap_invalidate_all_local(pmap); 662211215Sjchandra} 663178172Simp#endif 664178172Simp 665211215Sjchandrastatic __inline void 666211215Sjchandrapmap_invalidate_page_local(pmap_t pmap, vm_offset_t va) 667211215Sjchandra{ 668223758Sattilio u_int cpuid; 669211215Sjchandra 670223758Sattilio cpuid = PCPU_GET(cpuid); 671223758Sattilio 672211215Sjchandra if (is_kernel_pmap(pmap)) { 673211215Sjchandra tlb_invalidate_address(pmap, va); 674209243Sjchandra return; 675209243Sjchandra } 676223758Sattilio if (pmap->pm_asid[cpuid].gen != PCPU_GET(asid_generation)) 677211215Sjchandra return; 678223758Sattilio else if (!CPU_ISSET(cpuid, &pmap->pm_active)) { 679223758Sattilio pmap->pm_asid[cpuid].gen = 0; 680211215Sjchandra return; 681211215Sjchandra } 682211215Sjchandra tlb_invalidate_address(pmap, va); 683178172Simp} 684178172Simp 685211215Sjchandra#ifdef SMP 686178172Simpstruct pmap_invalidate_page_arg { 687178172Simp pmap_t pmap; 688178172Simp vm_offset_t va; 689178172Simp}; 690178172Simp 691211215Sjchandrastatic void 692178172Simppmap_invalidate_page(pmap_t pmap, vm_offset_t va) 693178172Simp{ 694178172Simp struct pmap_invalidate_page_arg arg; 695178172Simp 696178172Simp arg.pmap = pmap; 697178172Simp arg.va = va; 698211215Sjchandra smp_rendezvous(0, pmap_invalidate_page_action, 0, &arg); 699178172Simp} 700178172Simp 701178172Simpstatic void 702178172Simppmap_invalidate_page_action(void *arg) 703178172Simp{ 704211215Sjchandra struct pmap_invalidate_page_arg *p = arg; 705178172Simp 706211215Sjchandra pmap_invalidate_page_local(p->pmap, p->va); 707211215Sjchandra} 708211215Sjchandra#else 709211215Sjchandrastatic void 710211215Sjchandrapmap_invalidate_page(pmap_t pmap, vm_offset_t va) 711211215Sjchandra{ 712211215Sjchandra 713211215Sjchandra pmap_invalidate_page_local(pmap, va); 714211215Sjchandra} 715178172Simp#endif 716178172Simp 717211215Sjchandrastatic __inline void 718211215Sjchandrapmap_update_page_local(pmap_t pmap, vm_offset_t va, pt_entry_t pte) 719211215Sjchandra{ 720223758Sattilio u_int cpuid; 721211215Sjchandra 722223758Sattilio cpuid = PCPU_GET(cpuid); 723223758Sattilio 724178172Simp if (is_kernel_pmap(pmap)) { 725211215Sjchandra tlb_update(pmap, va, pte); 726178172Simp return; 727178172Simp } 728223758Sattilio if (pmap->pm_asid[cpuid].gen != PCPU_GET(asid_generation)) 729178172Simp return; 730223758Sattilio else if (!CPU_ISSET(cpuid, &pmap->pm_active)) { 731223758Sattilio pmap->pm_asid[cpuid].gen = 0; 732178172Simp return; 733178172Simp } 734211215Sjchandra tlb_update(pmap, va, pte); 735178172Simp} 736178172Simp 737211215Sjchandra#ifdef SMP 738178172Simpstruct pmap_update_page_arg { 739178172Simp pmap_t pmap; 740178172Simp vm_offset_t va; 741178172Simp pt_entry_t pte; 742178172Simp}; 743178172Simp 744211217Sjchandrastatic void 745178172Simppmap_update_page(pmap_t pmap, vm_offset_t va, pt_entry_t pte) 746178172Simp{ 747178172Simp struct pmap_update_page_arg arg; 748178172Simp 749178172Simp arg.pmap = pmap; 750178172Simp arg.va = va; 751178172Simp arg.pte = pte; 752211215Sjchandra smp_rendezvous(0, pmap_update_page_action, 0, &arg); 753178172Simp} 754178172Simp 755178172Simpstatic void 756178172Simppmap_update_page_action(void *arg) 757178172Simp{ 758211215Sjchandra struct pmap_update_page_arg *p = arg; 759178172Simp 760211215Sjchandra pmap_update_page_local(p->pmap, p->va, p->pte); 761178172Simp} 762211215Sjchandra#else 763211217Sjchandrastatic void 764211215Sjchandrapmap_update_page(pmap_t pmap, vm_offset_t va, pt_entry_t pte) 765211215Sjchandra{ 766178172Simp 767211215Sjchandra pmap_update_page_local(pmap, va, pte); 768211215Sjchandra} 769211215Sjchandra#endif 770211215Sjchandra 771178172Simp/* 772178172Simp * Routine: pmap_extract 773178172Simp * Function: 774178172Simp * Extract the physical page address associated 775178172Simp * with the given map/virtual_address pair. 776178172Simp */ 777178172Simpvm_paddr_t 778178172Simppmap_extract(pmap_t pmap, vm_offset_t va) 779178172Simp{ 780178172Simp pt_entry_t *pte; 781178172Simp vm_offset_t retval = 0; 782178172Simp 783178172Simp PMAP_LOCK(pmap); 784178172Simp pte = pmap_pte(pmap, va); 785178172Simp if (pte) { 786209243Sjchandra retval = TLBLO_PTE_TO_PA(*pte) | (va & PAGE_MASK); 787178172Simp } 788178172Simp PMAP_UNLOCK(pmap); 789211445Sjchandra return (retval); 790178172Simp} 791178172Simp 792178172Simp/* 793178172Simp * Routine: pmap_extract_and_hold 794178172Simp * Function: 795178172Simp * Atomically extract and hold the physical page 796178172Simp * with the given pmap and virtual address pair 797178172Simp * if that mapping permits the given protection. 798178172Simp */ 799178172Simpvm_page_t 800178172Simppmap_extract_and_hold(pmap_t pmap, vm_offset_t va, vm_prot_t prot) 801178172Simp{ 802178172Simp pt_entry_t pte; 803178172Simp vm_page_t m; 804207410Skmacy vm_paddr_t pa; 805178172Simp 806178172Simp m = NULL; 807207410Skmacy pa = 0; 808178172Simp PMAP_LOCK(pmap); 809207410Skmacyretry: 810178172Simp pte = *pmap_pte(pmap, va); 811209482Sjchandra if (pte != 0 && pte_test(&pte, PTE_V) && 812209482Sjchandra (pte_test(&pte, PTE_D) || (prot & VM_PROT_WRITE) == 0)) { 813209243Sjchandra if (vm_page_pa_tryrelock(pmap, TLBLO_PTE_TO_PA(pte), &pa)) 814207410Skmacy goto retry; 815207410Skmacy 816209243Sjchandra m = PHYS_TO_VM_PAGE(TLBLO_PTE_TO_PA(pte)); 817178172Simp vm_page_hold(m); 818178172Simp } 819207410Skmacy PA_UNLOCK_COND(pa); 820178172Simp PMAP_UNLOCK(pmap); 821178172Simp return (m); 822178172Simp} 823178172Simp 824178172Simp/*************************************************** 825178172Simp * Low level mapping routines..... 826178172Simp ***************************************************/ 827178172Simp 828178172Simp/* 829178172Simp * add a wired page to the kva 830178172Simp */ 831212989Sneelvoid 832212589Sneelpmap_kenter_attr(vm_offset_t va, vm_paddr_t pa, int attr) 833178172Simp{ 834209482Sjchandra pt_entry_t *pte; 835209482Sjchandra pt_entry_t opte, npte; 836178172Simp 837187301Sgonzo#ifdef PMAP_DEBUG 838209482Sjchandra printf("pmap_kenter: va: %p -> pa: %p\n", (void *)va, (void *)pa); 839187301Sgonzo#endif 840178172Simp 841178172Simp pte = pmap_pte(kernel_pmap, va); 842178172Simp opte = *pte; 843240754Salc npte = TLBLO_PA_TO_PFN(pa) | attr | PTE_D | PTE_V | PTE_G; 844178172Simp *pte = npte; 845211216Sjchandra if (pte_test(&opte, PTE_V) && opte != npte) 846211216Sjchandra pmap_update_page(kernel_pmap, va, npte); 847178172Simp} 848178172Simp 849212589Sneelvoid 850212589Sneelpmap_kenter(vm_offset_t va, vm_paddr_t pa) 851212589Sneel{ 852212589Sneel 853212989Sneel KASSERT(is_cacheable_mem(pa), 854212989Sneel ("pmap_kenter: memory at 0x%lx is not cacheable", (u_long)pa)); 855212589Sneel 856212989Sneel pmap_kenter_attr(va, pa, PTE_C_CACHE); 857212589Sneel} 858212589Sneel 859178172Simp/* 860178172Simp * remove a page from the kernel pagetables 861178172Simp */ 862178172Simp /* PMAP_INLINE */ void 863178172Simppmap_kremove(vm_offset_t va) 864178172Simp{ 865209482Sjchandra pt_entry_t *pte; 866178172Simp 867202046Simp /* 868202046Simp * Write back all caches from the page being destroyed 869202046Simp */ 870206746Sjmallett mips_dcache_wbinv_range_index(va, PAGE_SIZE); 871202046Simp 872178172Simp pte = pmap_pte(kernel_pmap, va); 873178172Simp *pte = PTE_G; 874178172Simp pmap_invalidate_page(kernel_pmap, va); 875178172Simp} 876178172Simp 877178172Simp/* 878178172Simp * Used to map a range of physical addresses into kernel 879178172Simp * virtual address space. 880178172Simp * 881178172Simp * The value passed in '*virt' is a suggested virtual address for 882178172Simp * the mapping. Architectures which can support a direct-mapped 883178172Simp * physical to virtual region can return the appropriate address 884178172Simp * within that region, leaving '*virt' unchanged. Other 885178172Simp * architectures should map the pages starting at '*virt' and 886178172Simp * update '*virt' with the first usable address after the mapped 887178172Simp * region. 888209930Sjchandra * 889209930Sjchandra * Use XKPHYS for 64 bit, and KSEG0 where possible for 32 bit. 890178172Simp */ 891178172Simpvm_offset_t 892217345Sjchandrapmap_map(vm_offset_t *virt, vm_paddr_t start, vm_paddr_t end, int prot) 893178172Simp{ 894178172Simp vm_offset_t va, sva; 895178172Simp 896216157Sjchandra if (MIPS_DIRECT_MAPPABLE(end - 1)) 897211453Sjchandra return (MIPS_PHYS_TO_DIRECT(start)); 898209930Sjchandra 899178172Simp va = sva = *virt; 900178172Simp while (start < end) { 901178172Simp pmap_kenter(va, start); 902178172Simp va += PAGE_SIZE; 903178172Simp start += PAGE_SIZE; 904178172Simp } 905178172Simp *virt = va; 906178172Simp return (sva); 907178172Simp} 908178172Simp 909178172Simp/* 910178172Simp * Add a list of wired pages to the kva 911178172Simp * this routine is only used for temporary 912178172Simp * kernel mappings that do not need to have 913178172Simp * page modification or references recorded. 914178172Simp * Note that old mappings are simply written 915178172Simp * over. The page *must* be wired. 916178172Simp */ 917178172Simpvoid 918178172Simppmap_qenter(vm_offset_t va, vm_page_t *m, int count) 919178172Simp{ 920178172Simp int i; 921202046Simp vm_offset_t origva = va; 922178172Simp 923178172Simp for (i = 0; i < count; i++) { 924202046Simp pmap_flush_pvcache(m[i]); 925178172Simp pmap_kenter(va, VM_PAGE_TO_PHYS(m[i])); 926178172Simp va += PAGE_SIZE; 927178172Simp } 928202046Simp 929202046Simp mips_dcache_wbinv_range_index(origva, PAGE_SIZE*count); 930178172Simp} 931178172Simp 932178172Simp/* 933178172Simp * this routine jerks page mappings from the 934178172Simp * kernel -- it is meant only for temporary mappings. 935178172Simp */ 936178172Simpvoid 937178172Simppmap_qremove(vm_offset_t va, int count) 938178172Simp{ 939202046Simp /* 940202046Simp * No need to wb/inv caches here, 941202046Simp * pmap_kremove will do it for us 942202046Simp */ 943202046Simp 944178172Simp while (count-- > 0) { 945178172Simp pmap_kremove(va); 946178172Simp va += PAGE_SIZE; 947178172Simp } 948178172Simp} 949178172Simp 950178172Simp/*************************************************** 951178172Simp * Page table page management routines..... 952178172Simp ***************************************************/ 953178172Simp 954178172Simp/* 955240151Skib * Decrements a page table page's wire count, which is used to record the 956240151Skib * number of valid page table entries within the page. If the wire count 957240151Skib * drops to zero, then the page table page is unmapped. Returns TRUE if the 958240151Skib * page table page was unmapped and FALSE otherwise. 959178172Simp */ 960240151Skibstatic PMAP_INLINE boolean_t 961240151Skibpmap_unwire_ptp(pmap_t pmap, vm_offset_t va, vm_page_t m) 962210846Sjchandra{ 963240151Skib 964210846Sjchandra --m->wire_count; 965240151Skib if (m->wire_count == 0) { 966240151Skib _pmap_unwire_ptp(pmap, va, m); 967240151Skib return (TRUE); 968240151Skib } else 969240151Skib return (FALSE); 970210846Sjchandra} 971210846Sjchandra 972240151Skibstatic void 973240151Skib_pmap_unwire_ptp(pmap_t pmap, vm_offset_t va, vm_page_t m) 974178172Simp{ 975210846Sjchandra pd_entry_t *pde; 976178172Simp 977210846Sjchandra PMAP_LOCK_ASSERT(pmap, MA_OWNED); 978178172Simp /* 979178172Simp * unmap the page table page 980178172Simp */ 981210846Sjchandra#ifdef __mips_n64 982210846Sjchandra if (m->pindex < NUPDE) 983210846Sjchandra pde = pmap_pde(pmap, va); 984210846Sjchandra else 985210846Sjchandra pde = pmap_segmap(pmap, va); 986210846Sjchandra#else 987210846Sjchandra pde = pmap_pde(pmap, va); 988210846Sjchandra#endif 989210846Sjchandra *pde = 0; 990210846Sjchandra pmap->pm_stats.resident_count--; 991178172Simp 992210846Sjchandra#ifdef __mips_n64 993210846Sjchandra if (m->pindex < NUPDE) { 994210846Sjchandra pd_entry_t *pdp; 995210846Sjchandra vm_page_t pdpg; 996210846Sjchandra 997210846Sjchandra /* 998210846Sjchandra * Recursively decrement next level pagetable refcount 999210846Sjchandra */ 1000210846Sjchandra pdp = (pd_entry_t *)*pmap_segmap(pmap, va); 1001211453Sjchandra pdpg = PHYS_TO_VM_PAGE(MIPS_DIRECT_TO_PHYS(pdp)); 1002240151Skib pmap_unwire_ptp(pmap, va, pdpg); 1003210846Sjchandra } 1004210846Sjchandra#endif 1005178172Simp if (pmap->pm_ptphint == m) 1006178172Simp pmap->pm_ptphint = NULL; 1007178172Simp 1008178172Simp /* 1009178172Simp * If the page is finally unwired, simply free it. 1010178172Simp */ 1011210327Sjchandra vm_page_free_zero(m); 1012208616Sjchandra atomic_subtract_int(&cnt.v_wire_count, 1); 1013178172Simp} 1014178172Simp 1015178172Simp/* 1016178172Simp * After removing a page table entry, this routine is used to 1017178172Simp * conditionally free the page, and manage the hold/wire counts. 1018178172Simp */ 1019178172Simpstatic int 1020178172Simppmap_unuse_pt(pmap_t pmap, vm_offset_t va, vm_page_t mpte) 1021178172Simp{ 1022178172Simp unsigned ptepindex; 1023178172Simp pd_entry_t pteva; 1024178172Simp 1025178172Simp if (va >= VM_MAXUSER_ADDRESS) 1026178172Simp return (0); 1027178172Simp 1028178172Simp if (mpte == NULL) { 1029210846Sjchandra ptepindex = pmap_pde_pindex(va); 1030178172Simp if (pmap->pm_ptphint && 1031178172Simp (pmap->pm_ptphint->pindex == ptepindex)) { 1032178172Simp mpte = pmap->pm_ptphint; 1033178172Simp } else { 1034210846Sjchandra pteva = *pmap_pde(pmap, va); 1035211453Sjchandra mpte = PHYS_TO_VM_PAGE(MIPS_DIRECT_TO_PHYS(pteva)); 1036178172Simp pmap->pm_ptphint = mpte; 1037178172Simp } 1038178172Simp } 1039240151Skib return (pmap_unwire_ptp(pmap, va, mpte)); 1040178172Simp} 1041178172Simp 1042178172Simpvoid 1043178172Simppmap_pinit0(pmap_t pmap) 1044178172Simp{ 1045178172Simp int i; 1046178172Simp 1047178172Simp PMAP_LOCK_INIT(pmap); 1048178172Simp pmap->pm_segtab = kernel_segmap; 1049222813Sattilio CPU_ZERO(&pmap->pm_active); 1050178172Simp pmap->pm_ptphint = NULL; 1051178172Simp for (i = 0; i < MAXCPU; i++) { 1052178172Simp pmap->pm_asid[i].asid = PMAP_ASID_RESERVED; 1053178172Simp pmap->pm_asid[i].gen = 0; 1054178172Simp } 1055178172Simp PCPU_SET(curpmap, pmap); 1056178172Simp TAILQ_INIT(&pmap->pm_pvlist); 1057178172Simp bzero(&pmap->pm_stats, sizeof pmap->pm_stats); 1058178172Simp} 1059178172Simp 1060216315Sjchandravoid 1061216315Sjchandrapmap_grow_direct_page_cache() 1062208422Sneel{ 1063208422Sneel 1064211453Sjchandra#ifdef __mips_n64 1065211453Sjchandra vm_contig_grow_cache(3, 0, MIPS_XKPHYS_LARGEST_PHYS); 1066211453Sjchandra#else 1067210327Sjchandra vm_contig_grow_cache(3, 0, MIPS_KSEG0_LARGEST_PHYS); 1068211453Sjchandra#endif 1069208422Sneel} 1070208422Sneel 1071216315Sjchandravm_page_t 1072216315Sjchandrapmap_alloc_direct_page(unsigned int index, int req) 1073208165Srrs{ 1074208165Srrs vm_page_t m; 1075208165Srrs 1076262933Sdumbbell m = vm_page_alloc_freelist(VM_FREELIST_DIRECT, req | VM_ALLOC_WIRED | 1077262933Sdumbbell VM_ALLOC_ZERO); 1078210327Sjchandra if (m == NULL) 1079208165Srrs return (NULL); 1080208165Srrs 1081210327Sjchandra if ((m->flags & PG_ZERO) == 0) 1082210327Sjchandra pmap_zero_page(m); 1083210327Sjchandra 1084208165Srrs m->pindex = index; 1085208165Srrs return (m); 1086208165Srrs} 1087208165Srrs 1088178172Simp/* 1089178172Simp * Initialize a preallocated and zeroed pmap structure, 1090178172Simp * such as one in a vmspace structure. 1091178172Simp */ 1092178172Simpint 1093178172Simppmap_pinit(pmap_t pmap) 1094178172Simp{ 1095205360Sneel vm_offset_t ptdva; 1096178172Simp vm_page_t ptdpg; 1097178172Simp int i; 1098178172Simp 1099178172Simp PMAP_LOCK_INIT(pmap); 1100178172Simp 1101178172Simp /* 1102178172Simp * allocate the page directory page 1103178172Simp */ 1104216315Sjchandra while ((ptdpg = pmap_alloc_direct_page(NUSERPGTBLS, VM_ALLOC_NORMAL)) == NULL) 1105216315Sjchandra pmap_grow_direct_page_cache(); 1106208589Sjchandra 1107211453Sjchandra ptdva = MIPS_PHYS_TO_DIRECT(VM_PAGE_TO_PHYS(ptdpg)); 1108205360Sneel pmap->pm_segtab = (pd_entry_t *)ptdva; 1109222813Sattilio CPU_ZERO(&pmap->pm_active); 1110178172Simp pmap->pm_ptphint = NULL; 1111178172Simp for (i = 0; i < MAXCPU; i++) { 1112178172Simp pmap->pm_asid[i].asid = PMAP_ASID_RESERVED; 1113178172Simp pmap->pm_asid[i].gen = 0; 1114178172Simp } 1115178172Simp TAILQ_INIT(&pmap->pm_pvlist); 1116178172Simp bzero(&pmap->pm_stats, sizeof pmap->pm_stats); 1117178172Simp 1118178172Simp return (1); 1119178172Simp} 1120178172Simp 1121178172Simp/* 1122178172Simp * this routine is called if the page table page is not 1123178172Simp * mapped correctly. 1124178172Simp */ 1125178172Simpstatic vm_page_t 1126178172Simp_pmap_allocpte(pmap_t pmap, unsigned ptepindex, int flags) 1127178172Simp{ 1128210846Sjchandra vm_offset_t pageva; 1129178172Simp vm_page_t m; 1130178172Simp 1131178172Simp KASSERT((flags & (M_NOWAIT | M_WAITOK)) == M_NOWAIT || 1132178172Simp (flags & (M_NOWAIT | M_WAITOK)) == M_WAITOK, 1133178172Simp ("_pmap_allocpte: flags is neither M_NOWAIT nor M_WAITOK")); 1134178172Simp 1135178172Simp /* 1136178172Simp * Find or fabricate a new pagetable page 1137178172Simp */ 1138216315Sjchandra if ((m = pmap_alloc_direct_page(ptepindex, VM_ALLOC_NORMAL)) == NULL) { 1139210327Sjchandra if (flags & M_WAITOK) { 1140210327Sjchandra PMAP_UNLOCK(pmap); 1141210327Sjchandra vm_page_unlock_queues(); 1142216315Sjchandra pmap_grow_direct_page_cache(); 1143210327Sjchandra vm_page_lock_queues(); 1144210327Sjchandra PMAP_LOCK(pmap); 1145210327Sjchandra } 1146210327Sjchandra 1147210327Sjchandra /* 1148210327Sjchandra * Indicate the need to retry. While waiting, the page 1149210327Sjchandra * table page may have been allocated. 1150210327Sjchandra */ 1151178172Simp return (NULL); 1152210327Sjchandra } 1153178172Simp 1154178172Simp /* 1155178172Simp * Map the pagetable page into the process address space, if it 1156178172Simp * isn't already there. 1157178172Simp */ 1158211453Sjchandra pageva = MIPS_PHYS_TO_DIRECT(VM_PAGE_TO_PHYS(m)); 1159178172Simp 1160210846Sjchandra#ifdef __mips_n64 1161210846Sjchandra if (ptepindex >= NUPDE) { 1162210846Sjchandra pmap->pm_segtab[ptepindex - NUPDE] = (pd_entry_t)pageva; 1163210846Sjchandra } else { 1164210846Sjchandra pd_entry_t *pdep, *pde; 1165210846Sjchandra int segindex = ptepindex >> (SEGSHIFT - PDRSHIFT); 1166210846Sjchandra int pdeindex = ptepindex & (NPDEPG - 1); 1167210846Sjchandra vm_page_t pg; 1168210846Sjchandra 1169210846Sjchandra pdep = &pmap->pm_segtab[segindex]; 1170210846Sjchandra if (*pdep == NULL) { 1171210846Sjchandra /* recurse for allocating page dir */ 1172210846Sjchandra if (_pmap_allocpte(pmap, NUPDE + segindex, 1173210846Sjchandra flags) == NULL) { 1174210846Sjchandra /* alloc failed, release current */ 1175210846Sjchandra --m->wire_count; 1176210846Sjchandra atomic_subtract_int(&cnt.v_wire_count, 1); 1177210846Sjchandra vm_page_free_zero(m); 1178210846Sjchandra return (NULL); 1179210846Sjchandra } 1180210846Sjchandra } else { 1181211453Sjchandra pg = PHYS_TO_VM_PAGE(MIPS_DIRECT_TO_PHYS(*pdep)); 1182210846Sjchandra pg->wire_count++; 1183210846Sjchandra } 1184210846Sjchandra /* Next level entry */ 1185210846Sjchandra pde = (pd_entry_t *)*pdep; 1186210846Sjchandra pde[pdeindex] = (pd_entry_t)pageva; 1187210846Sjchandra pmap->pm_ptphint = m; 1188210846Sjchandra } 1189210846Sjchandra#else 1190210846Sjchandra pmap->pm_segtab[ptepindex] = (pd_entry_t)pageva; 1191210846Sjchandra#endif 1192178172Simp pmap->pm_stats.resident_count++; 1193178172Simp 1194178172Simp /* 1195178172Simp * Set the page table hint 1196178172Simp */ 1197178172Simp pmap->pm_ptphint = m; 1198178172Simp return (m); 1199178172Simp} 1200178172Simp 1201178172Simpstatic vm_page_t 1202178172Simppmap_allocpte(pmap_t pmap, vm_offset_t va, int flags) 1203178172Simp{ 1204178172Simp unsigned ptepindex; 1205210846Sjchandra pd_entry_t *pde; 1206178172Simp vm_page_t m; 1207178172Simp 1208178172Simp KASSERT((flags & (M_NOWAIT | M_WAITOK)) == M_NOWAIT || 1209178172Simp (flags & (M_NOWAIT | M_WAITOK)) == M_WAITOK, 1210178172Simp ("pmap_allocpte: flags is neither M_NOWAIT nor M_WAITOK")); 1211178172Simp 1212178172Simp /* 1213178172Simp * Calculate pagetable page index 1214178172Simp */ 1215210846Sjchandra ptepindex = pmap_pde_pindex(va); 1216178172Simpretry: 1217178172Simp /* 1218178172Simp * Get the page directory entry 1219178172Simp */ 1220210846Sjchandra pde = pmap_pde(pmap, va); 1221178172Simp 1222178172Simp /* 1223178172Simp * If the page table page is mapped, we just increment the hold 1224178172Simp * count, and activate it. 1225178172Simp */ 1226210846Sjchandra if (pde != NULL && *pde != NULL) { 1227178172Simp /* 1228178172Simp * In order to get the page table page, try the hint first. 1229178172Simp */ 1230178172Simp if (pmap->pm_ptphint && 1231178172Simp (pmap->pm_ptphint->pindex == ptepindex)) { 1232178172Simp m = pmap->pm_ptphint; 1233178172Simp } else { 1234211453Sjchandra m = PHYS_TO_VM_PAGE(MIPS_DIRECT_TO_PHYS(*pde)); 1235178172Simp pmap->pm_ptphint = m; 1236178172Simp } 1237178172Simp m->wire_count++; 1238178172Simp } else { 1239178172Simp /* 1240178172Simp * Here if the pte page isn't mapped, or if it has been 1241178172Simp * deallocated. 1242178172Simp */ 1243178172Simp m = _pmap_allocpte(pmap, ptepindex, flags); 1244178172Simp if (m == NULL && (flags & M_WAITOK)) 1245178172Simp goto retry; 1246178172Simp } 1247210846Sjchandra return (m); 1248178172Simp} 1249178172Simp 1250178172Simp 1251178172Simp/*************************************************** 1252178172Simp* Pmap allocation/deallocation routines. 1253178172Simp ***************************************************/ 1254178172Simp/* 1255178172Simp * Revision 1.397 1256178172Simp * - Merged pmap_release and pmap_release_free_page. When pmap_release is 1257178172Simp * called only the page directory page(s) can be left in the pmap pte 1258178172Simp * object, since all page table pages will have been freed by 1259178172Simp * pmap_remove_pages and pmap_remove. In addition, there can only be one 1260178172Simp * reference to the pmap and the page directory is wired, so the page(s) 1261178172Simp * can never be busy. So all there is to do is clear the magic mappings 1262178172Simp * from the page directory and free the page(s). 1263178172Simp */ 1264178172Simp 1265178172Simp 1266178172Simp/* 1267178172Simp * Release any resources held by the given physical map. 1268178172Simp * Called when a pmap initialized by pmap_pinit is being released. 1269178172Simp * Should only be called if the map contains no valid mappings. 1270178172Simp */ 1271178172Simpvoid 1272178172Simppmap_release(pmap_t pmap) 1273178172Simp{ 1274205360Sneel vm_offset_t ptdva; 1275178172Simp vm_page_t ptdpg; 1276178172Simp 1277178172Simp KASSERT(pmap->pm_stats.resident_count == 0, 1278178172Simp ("pmap_release: pmap resident count %ld != 0", 1279178172Simp pmap->pm_stats.resident_count)); 1280178172Simp 1281205360Sneel ptdva = (vm_offset_t)pmap->pm_segtab; 1282211453Sjchandra ptdpg = PHYS_TO_VM_PAGE(MIPS_DIRECT_TO_PHYS(ptdva)); 1283205360Sneel 1284178172Simp ptdpg->wire_count--; 1285178172Simp atomic_subtract_int(&cnt.v_wire_count, 1); 1286210327Sjchandra vm_page_free_zero(ptdpg); 1287206584Sneel PMAP_LOCK_DESTROY(pmap); 1288178172Simp} 1289178172Simp 1290178172Simp/* 1291178172Simp * grow the number of kernel page table entries, if needed 1292178172Simp */ 1293178172Simpvoid 1294178172Simppmap_growkernel(vm_offset_t addr) 1295178172Simp{ 1296178172Simp vm_page_t nkpg; 1297210846Sjchandra pd_entry_t *pde, *pdpe; 1298178172Simp pt_entry_t *pte; 1299208165Srrs int i; 1300178172Simp 1301183510Simp mtx_assert(&kernel_map->system_mtx, MA_OWNED); 1302210846Sjchandra addr = roundup2(addr, NBSEG); 1303178172Simp if (addr - 1 >= kernel_map->max_offset) 1304178172Simp addr = kernel_map->max_offset; 1305178172Simp while (kernel_vm_end < addr) { 1306210846Sjchandra pdpe = pmap_segmap(kernel_pmap, kernel_vm_end); 1307210846Sjchandra#ifdef __mips_n64 1308210846Sjchandra if (*pdpe == 0) { 1309210846Sjchandra /* new intermediate page table entry */ 1310216315Sjchandra nkpg = pmap_alloc_direct_page(nkpt, VM_ALLOC_INTERRUPT); 1311210846Sjchandra if (nkpg == NULL) 1312210846Sjchandra panic("pmap_growkernel: no memory to grow kernel"); 1313211453Sjchandra *pdpe = (pd_entry_t)MIPS_PHYS_TO_DIRECT(VM_PAGE_TO_PHYS(nkpg)); 1314210846Sjchandra continue; /* try again */ 1315210846Sjchandra } 1316210846Sjchandra#endif 1317210846Sjchandra pde = pmap_pdpe_to_pde(pdpe, kernel_vm_end); 1318210846Sjchandra if (*pde != 0) { 1319210846Sjchandra kernel_vm_end = (kernel_vm_end + NBPDR) & ~PDRMASK; 1320178172Simp if (kernel_vm_end - 1 >= kernel_map->max_offset) { 1321178172Simp kernel_vm_end = kernel_map->max_offset; 1322178172Simp break; 1323178172Simp } 1324178172Simp continue; 1325178172Simp } 1326210846Sjchandra 1327178172Simp /* 1328178172Simp * This index is bogus, but out of the way 1329178172Simp */ 1330216315Sjchandra nkpg = pmap_alloc_direct_page(nkpt, VM_ALLOC_INTERRUPT); 1331178172Simp if (!nkpg) 1332178172Simp panic("pmap_growkernel: no memory to grow kernel"); 1333178172Simp nkpt++; 1334211453Sjchandra *pde = (pd_entry_t)MIPS_PHYS_TO_DIRECT(VM_PAGE_TO_PHYS(nkpg)); 1335178172Simp 1336178172Simp /* 1337178172Simp * The R[4-7]?00 stores only one copy of the Global bit in 1338178172Simp * the translation lookaside buffer for each 2 page entry. 1339178172Simp * Thus invalid entrys must have the Global bit set so when 1340178172Simp * Entry LO and Entry HI G bits are anded together they will 1341178172Simp * produce a global bit to store in the tlb. 1342178172Simp */ 1343210846Sjchandra pte = (pt_entry_t *)*pde; 1344210846Sjchandra for (i = 0; i < NPTEPG; i++) 1345210846Sjchandra pte[i] = PTE_G; 1346178172Simp 1347210846Sjchandra kernel_vm_end = (kernel_vm_end + NBPDR) & ~PDRMASK; 1348178172Simp if (kernel_vm_end - 1 >= kernel_map->max_offset) { 1349178172Simp kernel_vm_end = kernel_map->max_offset; 1350178172Simp break; 1351178172Simp } 1352178172Simp } 1353178172Simp} 1354178172Simp 1355178172Simp/*************************************************** 1356178172Simp* page management routines. 1357178172Simp ***************************************************/ 1358178172Simp 1359178172Simp/* 1360178172Simp * free the pv_entry back to the free list 1361178172Simp */ 1362178172Simpstatic PMAP_INLINE void 1363178172Simpfree_pv_entry(pv_entry_t pv) 1364178172Simp{ 1365178172Simp 1366178172Simp pv_entry_count--; 1367178172Simp uma_zfree(pvzone, pv); 1368178172Simp} 1369178172Simp 1370178172Simp/* 1371178172Simp * get a new pv_entry, allocating a block from the system 1372178172Simp * when needed. 1373178172Simp * the memory allocation is performed bypassing the malloc code 1374178172Simp * because of the possibility of allocations at interrupt time. 1375178172Simp */ 1376178172Simpstatic pv_entry_t 1377188507Simpget_pv_entry(pmap_t locked_pmap) 1378178172Simp{ 1379188507Simp static const struct timeval printinterval = { 60, 0 }; 1380188507Simp static struct timeval lastprint; 1381188507Simp struct vpgqueues *vpq; 1382188507Simp pt_entry_t *pte, oldpte; 1383188507Simp pmap_t pmap; 1384188507Simp pv_entry_t allocated_pv, next_pv, pv; 1385188507Simp vm_offset_t va; 1386188507Simp vm_page_t m; 1387178172Simp 1388188507Simp PMAP_LOCK_ASSERT(locked_pmap, MA_OWNED); 1389188507Simp mtx_assert(&vm_page_queue_mtx, MA_OWNED); 1390188507Simp allocated_pv = uma_zalloc(pvzone, M_NOWAIT); 1391188507Simp if (allocated_pv != NULL) { 1392188507Simp pv_entry_count++; 1393188507Simp if (pv_entry_count > pv_entry_high_water) 1394188507Simp pagedaemon_wakeup(); 1395188507Simp else 1396188507Simp return (allocated_pv); 1397178172Simp } 1398188507Simp /* 1399188507Simp * Reclaim pv entries: At first, destroy mappings to inactive 1400188507Simp * pages. After that, if a pv entry is still needed, destroy 1401188507Simp * mappings to active pages. 1402188507Simp */ 1403188507Simp if (ratecheck(&lastprint, &printinterval)) 1404188507Simp printf("Approaching the limit on PV entries, " 1405188507Simp "increase the vm.pmap.shpgperproc tunable.\n"); 1406188507Simp vpq = &vm_page_queues[PQ_INACTIVE]; 1407188507Simpretry: 1408188507Simp TAILQ_FOREACH(m, &vpq->pl, pageq) { 1409223732Salc if ((m->flags & PG_MARKER) != 0 || m->hold_count || m->busy) 1410188507Simp continue; 1411188507Simp TAILQ_FOREACH_SAFE(pv, &m->md.pv_list, pv_list, next_pv) { 1412188507Simp va = pv->pv_va; 1413188507Simp pmap = pv->pv_pmap; 1414188507Simp /* Avoid deadlock and lock recursion. */ 1415188507Simp if (pmap > locked_pmap) 1416188507Simp PMAP_LOCK(pmap); 1417188507Simp else if (pmap != locked_pmap && !PMAP_TRYLOCK(pmap)) 1418188507Simp continue; 1419188507Simp pmap->pm_stats.resident_count--; 1420188507Simp pte = pmap_pte(pmap, va); 1421188507Simp KASSERT(pte != NULL, ("pte")); 1422211068Sjchandra oldpte = *pte; 1423188507Simp if (is_kernel_pmap(pmap)) 1424188507Simp *pte = PTE_G; 1425211068Sjchandra else 1426211068Sjchandra *pte = 0; 1427209482Sjchandra KASSERT(!pte_test(&oldpte, PTE_W), 1428188507Simp ("wired pte for unwired page")); 1429188507Simp if (m->md.pv_flags & PV_TABLE_REF) 1430225418Skib vm_page_aflag_set(m, PGA_REFERENCED); 1431209482Sjchandra if (pte_test(&oldpte, PTE_D)) 1432188507Simp vm_page_dirty(m); 1433188507Simp pmap_invalidate_page(pmap, va); 1434188507Simp TAILQ_REMOVE(&pmap->pm_pvlist, pv, pv_plist); 1435188507Simp m->md.pv_list_count--; 1436188507Simp TAILQ_REMOVE(&m->md.pv_list, pv, pv_list); 1437188507Simp pmap_unuse_pt(pmap, va, pv->pv_ptem); 1438188507Simp if (pmap != locked_pmap) 1439188507Simp PMAP_UNLOCK(pmap); 1440188507Simp if (allocated_pv == NULL) 1441188507Simp allocated_pv = pv; 1442188507Simp else 1443188507Simp free_pv_entry(pv); 1444188507Simp } 1445208659Salc if (TAILQ_EMPTY(&m->md.pv_list)) { 1446225418Skib vm_page_aflag_clear(m, PGA_WRITEABLE); 1447208659Salc m->md.pv_flags &= ~(PV_TABLE_REF | PV_TABLE_MOD); 1448208659Salc } 1449188507Simp } 1450188507Simp if (allocated_pv == NULL) { 1451188507Simp if (vpq == &vm_page_queues[PQ_INACTIVE]) { 1452188507Simp vpq = &vm_page_queues[PQ_ACTIVE]; 1453188507Simp goto retry; 1454188507Simp } 1455188507Simp panic("get_pv_entry: increase the vm.pmap.shpgperproc tunable"); 1456188507Simp } 1457188507Simp return (allocated_pv); 1458178172Simp} 1459178172Simp 1460178172Simp/* 1461178172Simp * Revision 1.370 1462178172Simp * 1463178172Simp * Move pmap_collect() out of the machine-dependent code, rename it 1464178172Simp * to reflect its new location, and add page queue and flag locking. 1465178172Simp * 1466178172Simp * Notes: (1) alpha, i386, and ia64 had identical implementations 1467178172Simp * of pmap_collect() in terms of machine-independent interfaces; 1468178172Simp * (2) sparc64 doesn't require it; (3) powerpc had it as a TODO. 1469178172Simp * 1470178172Simp * MIPS implementation was identical to alpha [Junos 8.2] 1471178172Simp */ 1472178172Simp 1473178172Simp/* 1474178172Simp * If it is the first entry on the list, it is actually 1475178172Simp * in the header and we must copy the following entry up 1476178172Simp * to the header. Otherwise we must search the list for 1477178172Simp * the entry. In either case we free the now unused entry. 1478178172Simp */ 1479178172Simp 1480208665Salcstatic pv_entry_t 1481208665Salcpmap_pvh_remove(struct md_page *pvh, pmap_t pmap, vm_offset_t va) 1482178172Simp{ 1483178172Simp pv_entry_t pv; 1484178172Simp 1485178172Simp PMAP_LOCK_ASSERT(pmap, MA_OWNED); 1486178172Simp mtx_assert(&vm_page_queue_mtx, MA_OWNED); 1487208665Salc if (pvh->pv_list_count < pmap->pm_stats.resident_count) { 1488208665Salc TAILQ_FOREACH(pv, &pvh->pv_list, pv_list) { 1489178172Simp if (pmap == pv->pv_pmap && va == pv->pv_va) 1490178172Simp break; 1491178172Simp } 1492178172Simp } else { 1493178172Simp TAILQ_FOREACH(pv, &pmap->pm_pvlist, pv_plist) { 1494178172Simp if (va == pv->pv_va) 1495178172Simp break; 1496178172Simp } 1497178172Simp } 1498208665Salc if (pv != NULL) { 1499208665Salc TAILQ_REMOVE(&pvh->pv_list, pv, pv_list); 1500208665Salc pvh->pv_list_count--; 1501208665Salc TAILQ_REMOVE(&pmap->pm_pvlist, pv, pv_plist); 1502208665Salc } 1503208665Salc return (pv); 1504208665Salc} 1505178172Simp 1506208665Salcstatic void 1507208665Salcpmap_pvh_free(struct md_page *pvh, pmap_t pmap, vm_offset_t va) 1508208665Salc{ 1509208665Salc pv_entry_t pv; 1510208665Salc 1511208665Salc pv = pmap_pvh_remove(pvh, pmap, va); 1512208665Salc KASSERT(pv != NULL, ("pmap_pvh_free: pv not found, pa %lx va %lx", 1513208686Salc (u_long)VM_PAGE_TO_PHYS(member2struct(vm_page, md, pvh)), 1514208686Salc (u_long)va)); 1515178172Simp free_pv_entry(pv); 1516178172Simp} 1517178172Simp 1518178172Simpstatic void 1519208665Salcpmap_remove_entry(pmap_t pmap, vm_page_t m, vm_offset_t va) 1520178172Simp{ 1521178172Simp 1522178172Simp mtx_assert(&vm_page_queue_mtx, MA_OWNED); 1523208665Salc pmap_pvh_free(&m->md, pmap, va); 1524208665Salc if (TAILQ_EMPTY(&m->md.pv_list)) 1525225418Skib vm_page_aflag_clear(m, PGA_WRITEABLE); 1526178172Simp} 1527178172Simp 1528178172Simp/* 1529191300Salc * Conditionally create a pv entry. 1530191300Salc */ 1531191300Salcstatic boolean_t 1532191300Salcpmap_try_insert_pv_entry(pmap_t pmap, vm_page_t mpte, vm_offset_t va, 1533191300Salc vm_page_t m) 1534191300Salc{ 1535191300Salc pv_entry_t pv; 1536191300Salc 1537191300Salc PMAP_LOCK_ASSERT(pmap, MA_OWNED); 1538191300Salc mtx_assert(&vm_page_queue_mtx, MA_OWNED); 1539191300Salc if (pv_entry_count < pv_entry_high_water && 1540191300Salc (pv = uma_zalloc(pvzone, M_NOWAIT)) != NULL) { 1541191300Salc pv_entry_count++; 1542191300Salc pv->pv_va = va; 1543191300Salc pv->pv_pmap = pmap; 1544191300Salc pv->pv_ptem = mpte; 1545191300Salc TAILQ_INSERT_TAIL(&pmap->pm_pvlist, pv, pv_plist); 1546191300Salc TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_list); 1547191300Salc m->md.pv_list_count++; 1548191300Salc return (TRUE); 1549191300Salc } else 1550191300Salc return (FALSE); 1551191300Salc} 1552191300Salc 1553191300Salc/* 1554178172Simp * pmap_remove_pte: do the things to unmap a page in a process 1555178172Simp */ 1556178172Simpstatic int 1557178172Simppmap_remove_pte(struct pmap *pmap, pt_entry_t *ptq, vm_offset_t va) 1558178172Simp{ 1559178172Simp pt_entry_t oldpte; 1560178172Simp vm_page_t m; 1561217345Sjchandra vm_paddr_t pa; 1562178172Simp 1563178172Simp mtx_assert(&vm_page_queue_mtx, MA_OWNED); 1564178172Simp PMAP_LOCK_ASSERT(pmap, MA_OWNED); 1565178172Simp 1566211068Sjchandra oldpte = *ptq; 1567178172Simp if (is_kernel_pmap(pmap)) 1568178172Simp *ptq = PTE_G; 1569211068Sjchandra else 1570211068Sjchandra *ptq = 0; 1571178172Simp 1572209482Sjchandra if (pte_test(&oldpte, PTE_W)) 1573178172Simp pmap->pm_stats.wired_count -= 1; 1574178172Simp 1575178172Simp pmap->pm_stats.resident_count -= 1; 1576209243Sjchandra pa = TLBLO_PTE_TO_PA(oldpte); 1577178172Simp 1578178172Simp if (page_is_managed(pa)) { 1579178172Simp m = PHYS_TO_VM_PAGE(pa); 1580209482Sjchandra if (pte_test(&oldpte, PTE_D)) { 1581211958Sjchandra KASSERT(!pte_test(&oldpte, PTE_RO), 1582217345Sjchandra ("%s: modified page not writable: va: %p, pte: %#jx", 1583217345Sjchandra __func__, (void *)va, (uintmax_t)oldpte)); 1584187319Sgonzo vm_page_dirty(m); 1585178172Simp } 1586178172Simp if (m->md.pv_flags & PV_TABLE_REF) 1587225418Skib vm_page_aflag_set(m, PGA_REFERENCED); 1588178172Simp m->md.pv_flags &= ~(PV_TABLE_REF | PV_TABLE_MOD); 1589178172Simp 1590178172Simp pmap_remove_entry(pmap, m, va); 1591178172Simp } 1592211445Sjchandra return (pmap_unuse_pt(pmap, va, NULL)); 1593178172Simp} 1594178172Simp 1595178172Simp/* 1596178172Simp * Remove a single page from a process address space 1597178172Simp */ 1598178172Simpstatic void 1599178172Simppmap_remove_page(struct pmap *pmap, vm_offset_t va) 1600178172Simp{ 1601209482Sjchandra pt_entry_t *ptq; 1602178172Simp 1603178172Simp mtx_assert(&vm_page_queue_mtx, MA_OWNED); 1604178172Simp PMAP_LOCK_ASSERT(pmap, MA_OWNED); 1605178172Simp ptq = pmap_pte(pmap, va); 1606178172Simp 1607178172Simp /* 1608178172Simp * if there is no pte for this address, just skip it!!! 1609178172Simp */ 1610209482Sjchandra if (!ptq || !pte_test(ptq, PTE_V)) { 1611178172Simp return; 1612178172Simp } 1613202046Simp 1614178172Simp /* 1615202046Simp * Write back all caches from the page being destroyed 1616202046Simp */ 1617206746Sjmallett mips_dcache_wbinv_range_index(va, PAGE_SIZE); 1618202046Simp 1619202046Simp /* 1620178172Simp * get a local va for mappings for this pmap. 1621178172Simp */ 1622178172Simp (void)pmap_remove_pte(pmap, ptq, va); 1623178172Simp pmap_invalidate_page(pmap, va); 1624178172Simp 1625178172Simp return; 1626178172Simp} 1627178172Simp 1628178172Simp/* 1629178172Simp * Remove the given range of addresses from the specified map. 1630178172Simp * 1631178172Simp * It is assumed that the start and end are properly 1632178172Simp * rounded to the page size. 1633178172Simp */ 1634178172Simpvoid 1635178172Simppmap_remove(struct pmap *pmap, vm_offset_t sva, vm_offset_t eva) 1636178172Simp{ 1637210846Sjchandra vm_offset_t va_next; 1638210846Sjchandra pd_entry_t *pde, *pdpe; 1639210846Sjchandra pt_entry_t *pte; 1640178172Simp 1641178172Simp if (pmap == NULL) 1642178172Simp return; 1643178172Simp 1644178172Simp if (pmap->pm_stats.resident_count == 0) 1645178172Simp return; 1646178172Simp 1647178172Simp vm_page_lock_queues(); 1648178172Simp PMAP_LOCK(pmap); 1649178172Simp 1650178172Simp /* 1651178172Simp * special handling of removing one page. a very common operation 1652178172Simp * and easy to short circuit some code. 1653178172Simp */ 1654178172Simp if ((sva + PAGE_SIZE) == eva) { 1655178172Simp pmap_remove_page(pmap, sva); 1656178172Simp goto out; 1657178172Simp } 1658210846Sjchandra for (; sva < eva; sva = va_next) { 1659210846Sjchandra pdpe = pmap_segmap(pmap, sva); 1660210846Sjchandra#ifdef __mips_n64 1661210846Sjchandra if (*pdpe == 0) { 1662210846Sjchandra va_next = (sva + NBSEG) & ~SEGMASK; 1663210846Sjchandra if (va_next < sva) 1664210846Sjchandra va_next = eva; 1665178172Simp continue; 1666178172Simp } 1667210846Sjchandra#endif 1668210846Sjchandra va_next = (sva + NBPDR) & ~PDRMASK; 1669210846Sjchandra if (va_next < sva) 1670210846Sjchandra va_next = eva; 1671210846Sjchandra 1672210846Sjchandra pde = pmap_pdpe_to_pde(pdpe, sva); 1673210846Sjchandra if (*pde == 0) 1674210846Sjchandra continue; 1675210846Sjchandra if (va_next > eva) 1676210846Sjchandra va_next = eva; 1677210846Sjchandra for (pte = pmap_pde_to_pte(pde, sva); sva != va_next; 1678210846Sjchandra pte++, sva += PAGE_SIZE) { 1679210846Sjchandra pmap_remove_page(pmap, sva); 1680210846Sjchandra } 1681178172Simp } 1682178172Simpout: 1683178172Simp vm_page_unlock_queues(); 1684178172Simp PMAP_UNLOCK(pmap); 1685178172Simp} 1686178172Simp 1687178172Simp/* 1688178172Simp * Routine: pmap_remove_all 1689178172Simp * Function: 1690178172Simp * Removes this physical page from 1691178172Simp * all physical maps in which it resides. 1692178172Simp * Reflects back modify bits to the pager. 1693178172Simp * 1694178172Simp * Notes: 1695178172Simp * Original versions of this routine were very 1696178172Simp * inefficient because they iteratively called 1697178172Simp * pmap_remove (slow...) 1698178172Simp */ 1699178172Simp 1700178172Simpvoid 1701178172Simppmap_remove_all(vm_page_t m) 1702178172Simp{ 1703209482Sjchandra pv_entry_t pv; 1704209482Sjchandra pt_entry_t *pte, tpte; 1705178172Simp 1706224746Skib KASSERT((m->oflags & VPO_UNMANAGED) == 0, 1707223677Salc ("pmap_remove_all: page %p is not managed", m)); 1708207796Salc vm_page_lock_queues(); 1709178172Simp 1710178172Simp if (m->md.pv_flags & PV_TABLE_REF) 1711225418Skib vm_page_aflag_set(m, PGA_REFERENCED); 1712178172Simp 1713178172Simp while ((pv = TAILQ_FIRST(&m->md.pv_list)) != NULL) { 1714178172Simp PMAP_LOCK(pv->pv_pmap); 1715202046Simp 1716202046Simp /* 1717202046Simp * If it's last mapping writeback all caches from 1718202046Simp * the page being destroyed 1719202046Simp */ 1720202046Simp if (m->md.pv_list_count == 1) 1721206746Sjmallett mips_dcache_wbinv_range_index(pv->pv_va, PAGE_SIZE); 1722202046Simp 1723178172Simp pv->pv_pmap->pm_stats.resident_count--; 1724178172Simp 1725178172Simp pte = pmap_pte(pv->pv_pmap, pv->pv_va); 1726178172Simp 1727211068Sjchandra tpte = *pte; 1728178172Simp if (is_kernel_pmap(pv->pv_pmap)) 1729178172Simp *pte = PTE_G; 1730211068Sjchandra else 1731211068Sjchandra *pte = 0; 1732178172Simp 1733209482Sjchandra if (pte_test(&tpte, PTE_W)) 1734178172Simp pv->pv_pmap->pm_stats.wired_count--; 1735178172Simp 1736178172Simp /* 1737178172Simp * Update the vm_page_t clean and reference bits. 1738178172Simp */ 1739209482Sjchandra if (pte_test(&tpte, PTE_D)) { 1740211958Sjchandra KASSERT(!pte_test(&tpte, PTE_RO), 1741217345Sjchandra ("%s: modified page not writable: va: %p, pte: %#jx", 1742217345Sjchandra __func__, (void *)pv->pv_va, (uintmax_t)tpte)); 1743178606Salc vm_page_dirty(m); 1744178172Simp } 1745178172Simp pmap_invalidate_page(pv->pv_pmap, pv->pv_va); 1746178172Simp 1747178172Simp TAILQ_REMOVE(&pv->pv_pmap->pm_pvlist, pv, pv_plist); 1748178172Simp TAILQ_REMOVE(&m->md.pv_list, pv, pv_list); 1749178172Simp m->md.pv_list_count--; 1750178172Simp pmap_unuse_pt(pv->pv_pmap, pv->pv_va, pv->pv_ptem); 1751178172Simp PMAP_UNLOCK(pv->pv_pmap); 1752178172Simp free_pv_entry(pv); 1753178172Simp } 1754178172Simp 1755225418Skib vm_page_aflag_clear(m, PGA_WRITEABLE); 1756178172Simp m->md.pv_flags &= ~(PV_TABLE_REF | PV_TABLE_MOD); 1757207796Salc vm_page_unlock_queues(); 1758178172Simp} 1759178172Simp 1760178172Simp/* 1761178172Simp * Set the physical protection on the 1762178172Simp * specified range of this map as requested. 1763178172Simp */ 1764178172Simpvoid 1765178172Simppmap_protect(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, vm_prot_t prot) 1766178172Simp{ 1767178172Simp pt_entry_t *pte; 1768210846Sjchandra pd_entry_t *pde, *pdpe; 1769210846Sjchandra vm_offset_t va_next; 1770178172Simp 1771178172Simp if (pmap == NULL) 1772178172Simp return; 1773178172Simp 1774178172Simp if ((prot & VM_PROT_READ) == VM_PROT_NONE) { 1775178172Simp pmap_remove(pmap, sva, eva); 1776178172Simp return; 1777178172Simp } 1778178172Simp if (prot & VM_PROT_WRITE) 1779178172Simp return; 1780178172Simp 1781178172Simp vm_page_lock_queues(); 1782178172Simp PMAP_LOCK(pmap); 1783210846Sjchandra for (; sva < eva; sva = va_next) { 1784210922Sjchandra pt_entry_t pbits; 1785178172Simp vm_page_t m; 1786210846Sjchandra vm_paddr_t pa; 1787178172Simp 1788210846Sjchandra pdpe = pmap_segmap(pmap, sva); 1789210846Sjchandra#ifdef __mips_n64 1790210846Sjchandra if (*pdpe == 0) { 1791210846Sjchandra va_next = (sva + NBSEG) & ~SEGMASK; 1792210846Sjchandra if (va_next < sva) 1793210846Sjchandra va_next = eva; 1794178172Simp continue; 1795178172Simp } 1796210846Sjchandra#endif 1797210846Sjchandra va_next = (sva + NBPDR) & ~PDRMASK; 1798210846Sjchandra if (va_next < sva) 1799210846Sjchandra va_next = eva; 1800210846Sjchandra 1801210846Sjchandra pde = pmap_pdpe_to_pde(pdpe, sva); 1802210846Sjchandra if (pde == NULL || *pde == NULL) 1803178172Simp continue; 1804210846Sjchandra if (va_next > eva) 1805210846Sjchandra va_next = eva; 1806178172Simp 1807210846Sjchandra for (pte = pmap_pde_to_pte(pde, sva); sva != va_next; pte++, 1808210846Sjchandra sva += PAGE_SIZE) { 1809178172Simp 1810210846Sjchandra /* Skip invalid PTEs */ 1811210846Sjchandra if (!pte_test(pte, PTE_V)) 1812210846Sjchandra continue; 1813210922Sjchandra pbits = *pte; 1814210846Sjchandra pa = TLBLO_PTE_TO_PA(pbits); 1815210846Sjchandra if (page_is_managed(pa) && pte_test(&pbits, PTE_D)) { 1816210846Sjchandra m = PHYS_TO_VM_PAGE(pa); 1817210846Sjchandra vm_page_dirty(m); 1818210846Sjchandra m->md.pv_flags &= ~PV_TABLE_MOD; 1819210846Sjchandra } 1820210846Sjchandra pte_clear(&pbits, PTE_D); 1821210846Sjchandra pte_set(&pbits, PTE_RO); 1822210846Sjchandra 1823210846Sjchandra if (pbits != *pte) { 1824210922Sjchandra *pte = pbits; 1825210846Sjchandra pmap_update_page(pmap, sva, pbits); 1826210846Sjchandra } 1827178172Simp } 1828178172Simp } 1829178172Simp vm_page_unlock_queues(); 1830178172Simp PMAP_UNLOCK(pmap); 1831178172Simp} 1832178172Simp 1833178172Simp/* 1834178172Simp * Insert the given physical page (p) at 1835178172Simp * the specified virtual address (v) in the 1836178172Simp * target physical map with the protection requested. 1837178172Simp * 1838178172Simp * If specified, the page will be wired down, meaning 1839178172Simp * that the related pte can not be reclaimed. 1840178172Simp * 1841178172Simp * NB: This is the only routine which MAY NOT lazy-evaluate 1842178172Simp * or lose information. That is, this routine must actually 1843178172Simp * insert this page into the given map NOW. 1844178172Simp */ 1845178172Simpvoid 1846192659Salcpmap_enter(pmap_t pmap, vm_offset_t va, vm_prot_t access, vm_page_t m, 1847192659Salc vm_prot_t prot, boolean_t wired) 1848178172Simp{ 1849217345Sjchandra vm_paddr_t pa, opa; 1850209482Sjchandra pt_entry_t *pte; 1851178172Simp pt_entry_t origpte, newpte; 1852208665Salc pv_entry_t pv; 1853178172Simp vm_page_t mpte, om; 1854217345Sjchandra pt_entry_t rw = 0; 1855178172Simp 1856178172Simp if (pmap == NULL) 1857178172Simp return; 1858178172Simp 1859178172Simp va &= ~PAGE_MASK; 1860208175Salc KASSERT(va <= VM_MAX_KERNEL_ADDRESS, ("pmap_enter: toobig")); 1861224746Skib KASSERT((m->oflags & (VPO_UNMANAGED | VPO_BUSY)) != 0, 1862208175Salc ("pmap_enter: page %p is not busy", m)); 1863178172Simp 1864178172Simp mpte = NULL; 1865178172Simp 1866178172Simp vm_page_lock_queues(); 1867178172Simp PMAP_LOCK(pmap); 1868178172Simp 1869178172Simp /* 1870178172Simp * In the case that a page table page is not resident, we are 1871178172Simp * creating it here. 1872178172Simp */ 1873178172Simp if (va < VM_MAXUSER_ADDRESS) { 1874178172Simp mpte = pmap_allocpte(pmap, va, M_WAITOK); 1875178172Simp } 1876178172Simp pte = pmap_pte(pmap, va); 1877178172Simp 1878178172Simp /* 1879178172Simp * Page Directory table entry not valid, we need a new PT page 1880178172Simp */ 1881178172Simp if (pte == NULL) { 1882211958Sjchandra panic("pmap_enter: invalid page directory, pdir=%p, va=%p", 1883202046Simp (void *)pmap->pm_segtab, (void *)va); 1884178172Simp } 1885178172Simp pa = VM_PAGE_TO_PHYS(m); 1886178172Simp om = NULL; 1887178172Simp origpte = *pte; 1888209243Sjchandra opa = TLBLO_PTE_TO_PA(origpte); 1889178172Simp 1890178172Simp /* 1891178172Simp * Mapping has not changed, must be protection or wiring change. 1892178172Simp */ 1893209482Sjchandra if (pte_test(&origpte, PTE_V) && opa == pa) { 1894178172Simp /* 1895178172Simp * Wiring change, just update stats. We don't worry about 1896178172Simp * wiring PT pages as they remain resident as long as there 1897178172Simp * are valid mappings in them. Hence, if a user page is 1898178172Simp * wired, the PT page will be also. 1899178172Simp */ 1900209482Sjchandra if (wired && !pte_test(&origpte, PTE_W)) 1901178172Simp pmap->pm_stats.wired_count++; 1902209482Sjchandra else if (!wired && pte_test(&origpte, PTE_W)) 1903178172Simp pmap->pm_stats.wired_count--; 1904178172Simp 1905211958Sjchandra KASSERT(!pte_test(&origpte, PTE_D | PTE_RO), 1906217345Sjchandra ("%s: modified page not writable: va: %p, pte: %#jx", 1907217345Sjchandra __func__, (void *)va, (uintmax_t)origpte)); 1908178172Simp 1909178172Simp /* 1910178172Simp * Remove extra pte reference 1911178172Simp */ 1912178172Simp if (mpte) 1913178172Simp mpte->wire_count--; 1914178172Simp 1915178172Simp if (page_is_managed(opa)) { 1916178172Simp om = m; 1917178172Simp } 1918178172Simp goto validate; 1919178172Simp } 1920208665Salc 1921208665Salc pv = NULL; 1922208665Salc 1923178172Simp /* 1924178172Simp * Mapping has changed, invalidate old range and fall through to 1925178172Simp * handle validating new mapping. 1926178172Simp */ 1927178172Simp if (opa) { 1928209482Sjchandra if (pte_test(&origpte, PTE_W)) 1929178172Simp pmap->pm_stats.wired_count--; 1930178172Simp 1931178172Simp if (page_is_managed(opa)) { 1932178172Simp om = PHYS_TO_VM_PAGE(opa); 1933208665Salc pv = pmap_pvh_remove(&om->md, pmap, va); 1934178172Simp } 1935178172Simp if (mpte != NULL) { 1936178172Simp mpte->wire_count--; 1937178172Simp KASSERT(mpte->wire_count > 0, 1938178172Simp ("pmap_enter: missing reference to page table page," 1939202046Simp " va: %p", (void *)va)); 1940178172Simp } 1941178172Simp } else 1942178172Simp pmap->pm_stats.resident_count++; 1943178172Simp 1944178172Simp /* 1945178172Simp * Enter on the PV list if part of our managed memory. Note that we 1946178172Simp * raise IPL while manipulating pv_table since pmap_enter can be 1947178172Simp * called at interrupt time. 1948178172Simp */ 1949224746Skib if ((m->oflags & VPO_UNMANAGED) == 0) { 1950178606Salc KASSERT(va < kmi.clean_sva || va >= kmi.clean_eva, 1951178606Salc ("pmap_enter: managed mapping within the clean submap")); 1952208665Salc if (pv == NULL) 1953208665Salc pv = get_pv_entry(pmap); 1954208665Salc pv->pv_va = va; 1955208665Salc pv->pv_pmap = pmap; 1956208665Salc pv->pv_ptem = mpte; 1957208665Salc TAILQ_INSERT_TAIL(&pmap->pm_pvlist, pv, pv_plist); 1958208665Salc TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_list); 1959208665Salc m->md.pv_list_count++; 1960208665Salc } else if (pv != NULL) 1961208665Salc free_pv_entry(pv); 1962208665Salc 1963178172Simp /* 1964178172Simp * Increment counters 1965178172Simp */ 1966178172Simp if (wired) 1967178172Simp pmap->pm_stats.wired_count++; 1968178172Simp 1969178172Simpvalidate: 1970192628Salc if ((access & VM_PROT_WRITE) != 0) 1971192628Salc m->md.pv_flags |= PV_TABLE_MOD | PV_TABLE_REF; 1972178172Simp rw = init_pte_prot(va, m, prot); 1973178172Simp 1974187301Sgonzo#ifdef PMAP_DEBUG 1975209482Sjchandra printf("pmap_enter: va: %p -> pa: %p\n", (void *)va, (void *)pa); 1976187301Sgonzo#endif 1977178172Simp /* 1978178172Simp * Now validate mapping with desired protection/wiring. 1979178172Simp */ 1980209243Sjchandra newpte = TLBLO_PA_TO_PFN(pa) | rw | PTE_V; 1981178172Simp 1982178172Simp if (is_cacheable_mem(pa)) 1983209482Sjchandra newpte |= PTE_C_CACHE; 1984178172Simp else 1985209482Sjchandra newpte |= PTE_C_UNCACHED; 1986178172Simp 1987178172Simp if (wired) 1988178172Simp newpte |= PTE_W; 1989178172Simp 1990209482Sjchandra if (is_kernel_pmap(pmap)) 1991178172Simp newpte |= PTE_G; 1992178172Simp 1993178172Simp /* 1994178172Simp * if the mapping or permission bits are different, we need to 1995178172Simp * update the pte. 1996178172Simp */ 1997178172Simp if (origpte != newpte) { 1998209482Sjchandra if (pte_test(&origpte, PTE_V)) { 1999178172Simp *pte = newpte; 2000178172Simp if (page_is_managed(opa) && (opa != pa)) { 2001178172Simp if (om->md.pv_flags & PV_TABLE_REF) 2002225418Skib vm_page_aflag_set(om, PGA_REFERENCED); 2003178172Simp om->md.pv_flags &= 2004178172Simp ~(PV_TABLE_REF | PV_TABLE_MOD); 2005178172Simp } 2006209482Sjchandra if (pte_test(&origpte, PTE_D)) { 2007209482Sjchandra KASSERT(!pte_test(&origpte, PTE_RO), 2008178172Simp ("pmap_enter: modified page not writable:" 2009217345Sjchandra " va: %p, pte: %#jx", (void *)va, (uintmax_t)origpte)); 2010178606Salc if (page_is_managed(opa)) 2011178172Simp vm_page_dirty(om); 2012178172Simp } 2013208665Salc if (page_is_managed(opa) && 2014208665Salc TAILQ_EMPTY(&om->md.pv_list)) 2015225418Skib vm_page_aflag_clear(om, PGA_WRITEABLE); 2016178172Simp } else { 2017178172Simp *pte = newpte; 2018178172Simp } 2019178172Simp } 2020178172Simp pmap_update_page(pmap, va, newpte); 2021178172Simp 2022178172Simp /* 2023218909Sbrucec * Sync I & D caches for executable pages. Do this only if the 2024178172Simp * target pmap belongs to the current process. Otherwise, an 2025178172Simp * unresolvable TLB miss may occur. 2026178172Simp */ 2027178172Simp if (!is_kernel_pmap(pmap) && (pmap == &curproc->p_vmspace->vm_pmap) && 2028178172Simp (prot & VM_PROT_EXECUTE)) { 2029206746Sjmallett mips_icache_sync_range(va, PAGE_SIZE); 2030206746Sjmallett mips_dcache_wbinv_range(va, PAGE_SIZE); 2031178172Simp } 2032178172Simp vm_page_unlock_queues(); 2033178172Simp PMAP_UNLOCK(pmap); 2034178172Simp} 2035178172Simp 2036178172Simp/* 2037178172Simp * this code makes some *MAJOR* assumptions: 2038178172Simp * 1. Current pmap & pmap exists. 2039178172Simp * 2. Not wired. 2040178172Simp * 3. Read access. 2041178172Simp * 4. No page table pages. 2042178172Simp * but is *MUCH* faster than pmap_enter... 2043178172Simp */ 2044178172Simp 2045178172Simpvoid 2046178172Simppmap_enter_quick(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot) 2047178172Simp{ 2048191300Salc 2049207796Salc vm_page_lock_queues(); 2050191300Salc PMAP_LOCK(pmap); 2051191300Salc (void)pmap_enter_quick_locked(pmap, va, m, prot, NULL); 2052207796Salc vm_page_unlock_queues(); 2053191300Salc PMAP_UNLOCK(pmap); 2054191300Salc} 2055191300Salc 2056191300Salcstatic vm_page_t 2057191300Salcpmap_enter_quick_locked(pmap_t pmap, vm_offset_t va, vm_page_t m, 2058191300Salc vm_prot_t prot, vm_page_t mpte) 2059191300Salc{ 2060178172Simp pt_entry_t *pte; 2061217345Sjchandra vm_paddr_t pa; 2062178172Simp 2063178606Salc KASSERT(va < kmi.clean_sva || va >= kmi.clean_eva || 2064224746Skib (m->oflags & VPO_UNMANAGED) != 0, 2065191300Salc ("pmap_enter_quick_locked: managed mapping within the clean submap")); 2066178172Simp mtx_assert(&vm_page_queue_mtx, MA_OWNED); 2067191300Salc PMAP_LOCK_ASSERT(pmap, MA_OWNED); 2068191300Salc 2069178172Simp /* 2070178172Simp * In the case that a page table page is not resident, we are 2071178172Simp * creating it here. 2072178172Simp */ 2073178172Simp if (va < VM_MAXUSER_ADDRESS) { 2074210846Sjchandra pd_entry_t *pde; 2075178172Simp unsigned ptepindex; 2076178172Simp 2077178172Simp /* 2078178172Simp * Calculate pagetable page index 2079178172Simp */ 2080210846Sjchandra ptepindex = pmap_pde_pindex(va); 2081178172Simp if (mpte && (mpte->pindex == ptepindex)) { 2082178172Simp mpte->wire_count++; 2083178172Simp } else { 2084178172Simp /* 2085178172Simp * Get the page directory entry 2086178172Simp */ 2087210846Sjchandra pde = pmap_pde(pmap, va); 2088178172Simp 2089178172Simp /* 2090178172Simp * If the page table page is mapped, we just 2091178172Simp * increment the hold count, and activate it. 2092178172Simp */ 2093210846Sjchandra if (pde && *pde != 0) { 2094178172Simp if (pmap->pm_ptphint && 2095178172Simp (pmap->pm_ptphint->pindex == ptepindex)) { 2096178172Simp mpte = pmap->pm_ptphint; 2097178172Simp } else { 2098208422Sneel mpte = PHYS_TO_VM_PAGE( 2099211453Sjchandra MIPS_DIRECT_TO_PHYS(*pde)); 2100178172Simp pmap->pm_ptphint = mpte; 2101178172Simp } 2102178172Simp mpte->wire_count++; 2103178172Simp } else { 2104191300Salc mpte = _pmap_allocpte(pmap, ptepindex, 2105191300Salc M_NOWAIT); 2106191300Salc if (mpte == NULL) 2107191300Salc return (mpte); 2108178172Simp } 2109178172Simp } 2110178172Simp } else { 2111178172Simp mpte = NULL; 2112178172Simp } 2113178172Simp 2114178172Simp pte = pmap_pte(pmap, va); 2115209482Sjchandra if (pte_test(pte, PTE_V)) { 2116191300Salc if (mpte != NULL) { 2117191300Salc mpte->wire_count--; 2118191300Salc mpte = NULL; 2119191300Salc } 2120191300Salc return (mpte); 2121178172Simp } 2122191300Salc 2123178172Simp /* 2124191300Salc * Enter on the PV list if part of our managed memory. 2125178172Simp */ 2126224746Skib if ((m->oflags & VPO_UNMANAGED) == 0 && 2127191300Salc !pmap_try_insert_pv_entry(pmap, mpte, va, m)) { 2128191300Salc if (mpte != NULL) { 2129240151Skib pmap_unwire_ptp(pmap, va, mpte); 2130191300Salc mpte = NULL; 2131191300Salc } 2132191300Salc return (mpte); 2133191300Salc } 2134178172Simp 2135178172Simp /* 2136178172Simp * Increment counters 2137178172Simp */ 2138178172Simp pmap->pm_stats.resident_count++; 2139178172Simp 2140178172Simp pa = VM_PAGE_TO_PHYS(m); 2141178172Simp 2142178172Simp /* 2143178172Simp * Now validate mapping with RO protection 2144178172Simp */ 2145209243Sjchandra *pte = TLBLO_PA_TO_PFN(pa) | PTE_V; 2146178172Simp 2147178172Simp if (is_cacheable_mem(pa)) 2148209482Sjchandra *pte |= PTE_C_CACHE; 2149178172Simp else 2150209482Sjchandra *pte |= PTE_C_UNCACHED; 2151178172Simp 2152178172Simp if (is_kernel_pmap(pmap)) 2153178172Simp *pte |= PTE_G; 2154178172Simp else { 2155178172Simp *pte |= PTE_RO; 2156178172Simp /* 2157218909Sbrucec * Sync I & D caches. Do this only if the target pmap 2158178172Simp * belongs to the current process. Otherwise, an 2159178172Simp * unresolvable TLB miss may occur. */ 2160178172Simp if (pmap == &curproc->p_vmspace->vm_pmap) { 2161178172Simp va &= ~PAGE_MASK; 2162206746Sjmallett mips_icache_sync_range(va, PAGE_SIZE); 2163206746Sjmallett mips_dcache_wbinv_range(va, PAGE_SIZE); 2164178172Simp } 2165178172Simp } 2166191300Salc return (mpte); 2167178172Simp} 2168178172Simp 2169178172Simp/* 2170178172Simp * Make a temporary mapping for a physical address. This is only intended 2171178172Simp * to be used for panic dumps. 2172209930Sjchandra * 2173209930Sjchandra * Use XKPHYS for 64 bit, and KSEG0 where possible for 32 bit. 2174178172Simp */ 2175178172Simpvoid * 2176178172Simppmap_kenter_temporary(vm_paddr_t pa, int i) 2177178172Simp{ 2178178172Simp vm_offset_t va; 2179211453Sjchandra 2180178172Simp if (i != 0) 2181178172Simp printf("%s: ERROR!!! More than one page of virtual address mapping not supported\n", 2182178172Simp __func__); 2183178172Simp 2184211453Sjchandra if (MIPS_DIRECT_MAPPABLE(pa)) { 2185211453Sjchandra va = MIPS_PHYS_TO_DIRECT(pa); 2186178172Simp } else { 2187211453Sjchandra#ifndef __mips_n64 /* XXX : to be converted to new style */ 2188178172Simp int cpu; 2189211453Sjchandra register_t intr; 2190178172Simp struct local_sysmaps *sysm; 2191206717Sjmallett pt_entry_t *pte, npte; 2192206717Sjmallett 2193203151Srrs /* If this is used other than for dumps, we may need to leave 2194203151Srrs * interrupts disasbled on return. If crash dumps don't work when 2195203151Srrs * we get to this point, we might want to consider this (leaving things 2196203151Srrs * disabled as a starting point ;-) 2197203151Srrs */ 2198206717Sjmallett intr = intr_disable(); 2199178172Simp cpu = PCPU_GET(cpuid); 2200178172Simp sysm = &sysmap_lmem[cpu]; 2201178172Simp /* Since this is for the debugger, no locks or any other fun */ 2202209482Sjchandra npte = TLBLO_PA_TO_PFN(pa) | PTE_D | PTE_V | PTE_G | PTE_W | PTE_C_CACHE; 2203206717Sjmallett pte = pmap_pte(kernel_pmap, sysm->base); 2204206717Sjmallett *pte = npte; 2205203151Srrs sysm->valid1 = 1; 2206206717Sjmallett pmap_update_page(kernel_pmap, sysm->base, npte); 2207206717Sjmallett va = sysm->base; 2208206717Sjmallett intr_restore(intr); 2209211453Sjchandra#endif 2210178172Simp } 2211178172Simp return ((void *)va); 2212178172Simp} 2213178172Simp 2214178172Simpvoid 2215178172Simppmap_kenter_temporary_free(vm_paddr_t pa) 2216178172Simp{ 2217211453Sjchandra#ifndef __mips_n64 /* XXX : to be converted to new style */ 2218178172Simp int cpu; 2219206717Sjmallett register_t intr; 2220178172Simp struct local_sysmaps *sysm; 2221211453Sjchandra#endif 2222178172Simp 2223211453Sjchandra if (MIPS_DIRECT_MAPPABLE(pa)) { 2224178172Simp /* nothing to do for this case */ 2225178172Simp return; 2226178172Simp } 2227211453Sjchandra#ifndef __mips_n64 /* XXX : to be converted to new style */ 2228178172Simp cpu = PCPU_GET(cpuid); 2229178172Simp sysm = &sysmap_lmem[cpu]; 2230178172Simp if (sysm->valid1) { 2231206717Sjmallett pt_entry_t *pte; 2232206717Sjmallett 2233206717Sjmallett intr = intr_disable(); 2234206717Sjmallett pte = pmap_pte(kernel_pmap, sysm->base); 2235206717Sjmallett *pte = PTE_G; 2236206717Sjmallett pmap_invalidate_page(kernel_pmap, sysm->base); 2237206717Sjmallett intr_restore(intr); 2238178172Simp sysm->valid1 = 0; 2239178172Simp } 2240211453Sjchandra#endif 2241178172Simp} 2242178172Simp 2243178172Simp/* 2244178172Simp * Moved the code to Machine Independent 2245178172Simp * vm_map_pmap_enter() 2246178172Simp */ 2247178172Simp 2248178172Simp/* 2249178172Simp * Maps a sequence of resident pages belonging to the same object. 2250178172Simp * The sequence begins with the given page m_start. This page is 2251178172Simp * mapped at the given virtual address start. Each subsequent page is 2252178172Simp * mapped at a virtual address that is offset from start by the same 2253178172Simp * amount as the page is offset from m_start within the object. The 2254178172Simp * last page in the sequence is the page with the largest offset from 2255178172Simp * m_start that can be mapped at a virtual address less than the given 2256178172Simp * virtual address end. Not every virtual page between start and end 2257178172Simp * is mapped; only those for which a resident page exists with the 2258178172Simp * corresponding offset from m_start are mapped. 2259178172Simp */ 2260178172Simpvoid 2261178172Simppmap_enter_object(pmap_t pmap, vm_offset_t start, vm_offset_t end, 2262178172Simp vm_page_t m_start, vm_prot_t prot) 2263178172Simp{ 2264191300Salc vm_page_t m, mpte; 2265178172Simp vm_pindex_t diff, psize; 2266178172Simp 2267191300Salc VM_OBJECT_LOCK_ASSERT(m_start->object, MA_OWNED); 2268178172Simp psize = atop(end - start); 2269191300Salc mpte = NULL; 2270178172Simp m = m_start; 2271208574Salc vm_page_lock_queues(); 2272191300Salc PMAP_LOCK(pmap); 2273178172Simp while (m != NULL && (diff = m->pindex - m_start->pindex) < psize) { 2274191300Salc mpte = pmap_enter_quick_locked(pmap, start + ptoa(diff), m, 2275191300Salc prot, mpte); 2276178172Simp m = TAILQ_NEXT(m, listq); 2277178172Simp } 2278208574Salc vm_page_unlock_queues(); 2279191300Salc PMAP_UNLOCK(pmap); 2280178172Simp} 2281178172Simp 2282178172Simp/* 2283178172Simp * pmap_object_init_pt preloads the ptes for a given object 2284178172Simp * into the specified pmap. This eliminates the blast of soft 2285178172Simp * faults on process startup and immediately after an mmap. 2286178172Simp */ 2287178172Simpvoid 2288178172Simppmap_object_init_pt(pmap_t pmap, vm_offset_t addr, 2289178172Simp vm_object_t object, vm_pindex_t pindex, vm_size_t size) 2290178172Simp{ 2291178172Simp VM_OBJECT_LOCK_ASSERT(object, MA_OWNED); 2292195840Sjhb KASSERT(object->type == OBJT_DEVICE || object->type == OBJT_SG, 2293178172Simp ("pmap_object_init_pt: non-device object")); 2294178172Simp} 2295178172Simp 2296178172Simp/* 2297178172Simp * Routine: pmap_change_wiring 2298178172Simp * Function: Change the wiring attribute for a map/virtual-address 2299178172Simp * pair. 2300178172Simp * In/out conditions: 2301178172Simp * The mapping must already exist in the pmap. 2302178172Simp */ 2303178172Simpvoid 2304178172Simppmap_change_wiring(pmap_t pmap, vm_offset_t va, boolean_t wired) 2305178172Simp{ 2306209482Sjchandra pt_entry_t *pte; 2307178172Simp 2308178172Simp if (pmap == NULL) 2309178172Simp return; 2310178172Simp 2311178172Simp PMAP_LOCK(pmap); 2312178172Simp pte = pmap_pte(pmap, va); 2313178172Simp 2314209482Sjchandra if (wired && !pte_test(pte, PTE_W)) 2315178172Simp pmap->pm_stats.wired_count++; 2316209482Sjchandra else if (!wired && pte_test(pte, PTE_W)) 2317178172Simp pmap->pm_stats.wired_count--; 2318178172Simp 2319178172Simp /* 2320178172Simp * Wiring is not a hardware characteristic so there is no need to 2321178172Simp * invalidate TLB. 2322178172Simp */ 2323209482Sjchandra if (wired) 2324209482Sjchandra pte_set(pte, PTE_W); 2325209482Sjchandra else 2326209482Sjchandra pte_clear(pte, PTE_W); 2327178172Simp PMAP_UNLOCK(pmap); 2328178172Simp} 2329178172Simp 2330178172Simp/* 2331178172Simp * Copy the range specified by src_addr/len 2332178172Simp * from the source map to the range dst_addr/len 2333178172Simp * in the destination map. 2334178172Simp * 2335178172Simp * This routine is only advisory and need not do anything. 2336178172Simp */ 2337178172Simp 2338178172Simpvoid 2339178172Simppmap_copy(pmap_t dst_pmap, pmap_t src_pmap, vm_offset_t dst_addr, 2340178172Simp vm_size_t len, vm_offset_t src_addr) 2341178172Simp{ 2342178172Simp} 2343178172Simp 2344178172Simp/* 2345178172Simp * pmap_zero_page zeros the specified hardware page by mapping 2346178172Simp * the page into KVM and using bzero to clear its contents. 2347209930Sjchandra * 2348209930Sjchandra * Use XKPHYS for 64 bit, and KSEG0 where possible for 32 bit. 2349178172Simp */ 2350178172Simpvoid 2351178172Simppmap_zero_page(vm_page_t m) 2352178172Simp{ 2353178172Simp vm_offset_t va; 2354178172Simp vm_paddr_t phys = VM_PAGE_TO_PHYS(m); 2355209930Sjchandra 2356211453Sjchandra if (MIPS_DIRECT_MAPPABLE(phys)) { 2357211453Sjchandra va = MIPS_PHYS_TO_DIRECT(phys); 2358178172Simp bzero((caddr_t)va, PAGE_SIZE); 2359187301Sgonzo mips_dcache_wbinv_range(va, PAGE_SIZE); 2360178172Simp } else { 2361211453Sjchandra va = pmap_lmem_map1(phys); 2362206717Sjmallett bzero((caddr_t)va, PAGE_SIZE); 2363206717Sjmallett mips_dcache_wbinv_range(va, PAGE_SIZE); 2364211453Sjchandra pmap_lmem_unmap(); 2365178172Simp } 2366178172Simp} 2367211453Sjchandra 2368178172Simp/* 2369178172Simp * pmap_zero_page_area zeros the specified hardware page by mapping 2370178172Simp * the page into KVM and using bzero to clear its contents. 2371178172Simp * 2372178172Simp * off and size may not cover an area beyond a single hardware page. 2373178172Simp */ 2374178172Simpvoid 2375178172Simppmap_zero_page_area(vm_page_t m, int off, int size) 2376178172Simp{ 2377178172Simp vm_offset_t va; 2378178172Simp vm_paddr_t phys = VM_PAGE_TO_PHYS(m); 2379209930Sjchandra 2380211453Sjchandra if (MIPS_DIRECT_MAPPABLE(phys)) { 2381211453Sjchandra va = MIPS_PHYS_TO_DIRECT(phys); 2382178172Simp bzero((char *)(caddr_t)va + off, size); 2383187301Sgonzo mips_dcache_wbinv_range(va + off, size); 2384178172Simp } else { 2385211453Sjchandra va = pmap_lmem_map1(phys); 2386206717Sjmallett bzero((char *)va + off, size); 2387206717Sjmallett mips_dcache_wbinv_range(va + off, size); 2388211453Sjchandra pmap_lmem_unmap(); 2389178172Simp } 2390178172Simp} 2391178172Simp 2392178172Simpvoid 2393178172Simppmap_zero_page_idle(vm_page_t m) 2394178172Simp{ 2395178172Simp vm_offset_t va; 2396178172Simp vm_paddr_t phys = VM_PAGE_TO_PHYS(m); 2397209930Sjchandra 2398211453Sjchandra if (MIPS_DIRECT_MAPPABLE(phys)) { 2399211453Sjchandra va = MIPS_PHYS_TO_DIRECT(phys); 2400178172Simp bzero((caddr_t)va, PAGE_SIZE); 2401187301Sgonzo mips_dcache_wbinv_range(va, PAGE_SIZE); 2402178172Simp } else { 2403211453Sjchandra va = pmap_lmem_map1(phys); 2404206717Sjmallett bzero((caddr_t)va, PAGE_SIZE); 2405206717Sjmallett mips_dcache_wbinv_range(va, PAGE_SIZE); 2406211453Sjchandra pmap_lmem_unmap(); 2407178172Simp } 2408178172Simp} 2409178172Simp 2410178172Simp/* 2411178172Simp * pmap_copy_page copies the specified (machine independent) 2412178172Simp * page by mapping the page into virtual memory and using 2413178172Simp * bcopy to copy the page, one machine dependent page at a 2414178172Simp * time. 2415209930Sjchandra * 2416209930Sjchandra * Use XKPHYS for 64 bit, and KSEG0 where possible for 32 bit. 2417178172Simp */ 2418178172Simpvoid 2419178172Simppmap_copy_page(vm_page_t src, vm_page_t dst) 2420178172Simp{ 2421178172Simp vm_offset_t va_src, va_dst; 2422211453Sjchandra vm_paddr_t phys_src = VM_PAGE_TO_PHYS(src); 2423211453Sjchandra vm_paddr_t phys_dst = VM_PAGE_TO_PHYS(dst); 2424209930Sjchandra 2425211453Sjchandra if (MIPS_DIRECT_MAPPABLE(phys_src) && MIPS_DIRECT_MAPPABLE(phys_dst)) { 2426206716Sjmallett /* easy case, all can be accessed via KSEG0 */ 2427206716Sjmallett /* 2428206716Sjmallett * Flush all caches for VA that are mapped to this page 2429206716Sjmallett * to make sure that data in SDRAM is up to date 2430206716Sjmallett */ 2431206716Sjmallett pmap_flush_pvcache(src); 2432206716Sjmallett mips_dcache_wbinv_range_index( 2433211453Sjchandra MIPS_PHYS_TO_DIRECT(phys_dst), PAGE_SIZE); 2434211453Sjchandra va_src = MIPS_PHYS_TO_DIRECT(phys_src); 2435211453Sjchandra va_dst = MIPS_PHYS_TO_DIRECT(phys_dst); 2436178172Simp bcopy((caddr_t)va_src, (caddr_t)va_dst, PAGE_SIZE); 2437206716Sjmallett mips_dcache_wbinv_range(va_dst, PAGE_SIZE); 2438206716Sjmallett } else { 2439211453Sjchandra va_src = pmap_lmem_map2(phys_src, phys_dst); 2440211453Sjchandra va_dst = va_src + PAGE_SIZE; 2441206716Sjmallett bcopy((void *)va_src, (void *)va_dst, PAGE_SIZE); 2442206717Sjmallett mips_dcache_wbinv_range(va_dst, PAGE_SIZE); 2443211453Sjchandra pmap_lmem_unmap(); 2444178172Simp } 2445178172Simp} 2446178172Simp 2447251897Sscottlint unmapped_buf_allowed; 2448251897Sscottl 2449248814Skibvoid 2450248814Skibpmap_copy_pages(vm_page_t ma[], vm_offset_t a_offset, vm_page_t mb[], 2451248814Skib vm_offset_t b_offset, int xfersize) 2452248814Skib{ 2453248814Skib char *a_cp, *b_cp; 2454248814Skib vm_page_t a_m, b_m; 2455248814Skib vm_offset_t a_pg_offset, b_pg_offset; 2456248814Skib vm_paddr_t a_phys, b_phys; 2457248814Skib int cnt; 2458248814Skib 2459248814Skib while (xfersize > 0) { 2460248814Skib a_pg_offset = a_offset & PAGE_MASK; 2461248814Skib cnt = min(xfersize, PAGE_SIZE - a_pg_offset); 2462248814Skib a_m = ma[a_offset >> PAGE_SHIFT]; 2463248814Skib a_phys = VM_PAGE_TO_PHYS(a_m); 2464248814Skib b_pg_offset = b_offset & PAGE_MASK; 2465248814Skib cnt = min(cnt, PAGE_SIZE - b_pg_offset); 2466248814Skib b_m = mb[b_offset >> PAGE_SHIFT]; 2467248814Skib b_phys = VM_PAGE_TO_PHYS(b_m); 2468248814Skib if (MIPS_DIRECT_MAPPABLE(a_phys) && 2469248814Skib MIPS_DIRECT_MAPPABLE(b_phys)) { 2470248814Skib pmap_flush_pvcache(a_m); 2471248814Skib mips_dcache_wbinv_range_index( 2472248814Skib MIPS_PHYS_TO_DIRECT(b_phys), PAGE_SIZE); 2473248814Skib a_cp = (char *)MIPS_PHYS_TO_DIRECT(a_phys) + 2474248814Skib a_pg_offset; 2475248814Skib b_cp = (char *)MIPS_PHYS_TO_DIRECT(b_phys) + 2476248814Skib b_pg_offset; 2477248814Skib bcopy(a_cp, b_cp, cnt); 2478248814Skib mips_dcache_wbinv_range((vm_offset_t)b_cp, cnt); 2479248814Skib } else { 2480248814Skib a_cp = (char *)pmap_lmem_map2(a_phys, b_phys); 2481248814Skib b_cp = (char *)a_cp + PAGE_SIZE; 2482248814Skib a_cp += a_pg_offset; 2483248814Skib b_cp += b_pg_offset; 2484248814Skib bcopy(a_cp, b_cp, cnt); 2485248814Skib mips_dcache_wbinv_range((vm_offset_t)b_cp, cnt); 2486248814Skib pmap_lmem_unmap(); 2487248814Skib } 2488248814Skib a_offset += cnt; 2489248814Skib b_offset += cnt; 2490248814Skib xfersize -= cnt; 2491248814Skib } 2492248814Skib} 2493248814Skib 2494178172Simp/* 2495178172Simp * Returns true if the pmap's pv is one of the first 2496178172Simp * 16 pvs linked to from this page. This count may 2497178172Simp * be changed upwards or downwards in the future; it 2498178172Simp * is only necessary that true be returned for a small 2499178172Simp * subset of pmaps for proper page aging. 2500178172Simp */ 2501178172Simpboolean_t 2502178172Simppmap_page_exists_quick(pmap_t pmap, vm_page_t m) 2503178172Simp{ 2504178172Simp pv_entry_t pv; 2505178172Simp int loops = 0; 2506208990Salc boolean_t rv; 2507178172Simp 2508224746Skib KASSERT((m->oflags & VPO_UNMANAGED) == 0, 2509208990Salc ("pmap_page_exists_quick: page %p is not managed", m)); 2510208990Salc rv = FALSE; 2511208990Salc vm_page_lock_queues(); 2512178172Simp TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) { 2513178172Simp if (pv->pv_pmap == pmap) { 2514208990Salc rv = TRUE; 2515208990Salc break; 2516178172Simp } 2517178172Simp loops++; 2518178172Simp if (loops >= 16) 2519178172Simp break; 2520178172Simp } 2521208990Salc vm_page_unlock_queues(); 2522208990Salc return (rv); 2523178172Simp} 2524178172Simp 2525178172Simp/* 2526178172Simp * Remove all pages from specified address space 2527178172Simp * this aids process exit speeds. Also, this code 2528178172Simp * is special cased for current process only, but 2529178172Simp * can have the more generic (and slightly slower) 2530178172Simp * mode enabled. This is much faster than pmap_remove 2531178172Simp * in the case of running down an entire address space. 2532178172Simp */ 2533178172Simpvoid 2534178172Simppmap_remove_pages(pmap_t pmap) 2535178172Simp{ 2536178172Simp pt_entry_t *pte, tpte; 2537178172Simp pv_entry_t pv, npv; 2538178172Simp vm_page_t m; 2539178172Simp 2540178172Simp if (pmap != vmspace_pmap(curthread->td_proc->p_vmspace)) { 2541178172Simp printf("warning: pmap_remove_pages called with non-current pmap\n"); 2542178172Simp return; 2543178172Simp } 2544178172Simp vm_page_lock_queues(); 2545178172Simp PMAP_LOCK(pmap); 2546211958Sjchandra for (pv = TAILQ_FIRST(&pmap->pm_pvlist); pv != NULL; pv = npv) { 2547178172Simp 2548178172Simp pte = pmap_pte(pv->pv_pmap, pv->pv_va); 2549209482Sjchandra if (!pte_test(pte, PTE_V)) 2550211958Sjchandra panic("pmap_remove_pages: page on pm_pvlist has no pte"); 2551178172Simp tpte = *pte; 2552178172Simp 2553178172Simp/* 2554178172Simp * We cannot remove wired pages from a process' mapping at this time 2555178172Simp */ 2556209482Sjchandra if (pte_test(&tpte, PTE_W)) { 2557178172Simp npv = TAILQ_NEXT(pv, pv_plist); 2558178172Simp continue; 2559178172Simp } 2560178172Simp *pte = is_kernel_pmap(pmap) ? PTE_G : 0; 2561178172Simp 2562209243Sjchandra m = PHYS_TO_VM_PAGE(TLBLO_PTE_TO_PA(tpte)); 2563207139Sjmallett KASSERT(m != NULL, 2564217345Sjchandra ("pmap_remove_pages: bad tpte %#jx", (uintmax_t)tpte)); 2565178172Simp 2566178172Simp pv->pv_pmap->pm_stats.resident_count--; 2567178172Simp 2568178172Simp /* 2569178172Simp * Update the vm_page_t clean and reference bits. 2570178172Simp */ 2571209482Sjchandra if (pte_test(&tpte, PTE_D)) { 2572178172Simp vm_page_dirty(m); 2573178172Simp } 2574178172Simp npv = TAILQ_NEXT(pv, pv_plist); 2575178172Simp TAILQ_REMOVE(&pv->pv_pmap->pm_pvlist, pv, pv_plist); 2576178172Simp 2577178172Simp m->md.pv_list_count--; 2578178172Simp TAILQ_REMOVE(&m->md.pv_list, pv, pv_list); 2579178172Simp if (TAILQ_FIRST(&m->md.pv_list) == NULL) { 2580225418Skib vm_page_aflag_clear(m, PGA_WRITEABLE); 2581178172Simp } 2582178172Simp pmap_unuse_pt(pv->pv_pmap, pv->pv_va, pv->pv_ptem); 2583178172Simp free_pv_entry(pv); 2584178172Simp } 2585178172Simp pmap_invalidate_all(pmap); 2586178172Simp PMAP_UNLOCK(pmap); 2587178172Simp vm_page_unlock_queues(); 2588178172Simp} 2589178172Simp 2590178172Simp/* 2591178172Simp * pmap_testbit tests bits in pte's 2592178172Simp * note that the testbit/changebit routines are inline, 2593178172Simp * and a lot of things compile-time evaluate. 2594178172Simp */ 2595178172Simpstatic boolean_t 2596178172Simppmap_testbit(vm_page_t m, int bit) 2597178172Simp{ 2598178172Simp pv_entry_t pv; 2599178172Simp pt_entry_t *pte; 2600178172Simp boolean_t rv = FALSE; 2601178172Simp 2602224746Skib if (m->oflags & VPO_UNMANAGED) 2603211445Sjchandra return (rv); 2604178172Simp 2605178172Simp if (TAILQ_FIRST(&m->md.pv_list) == NULL) 2606211445Sjchandra return (rv); 2607178172Simp 2608178172Simp mtx_assert(&vm_page_queue_mtx, MA_OWNED); 2609178172Simp TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) { 2610178172Simp PMAP_LOCK(pv->pv_pmap); 2611178172Simp pte = pmap_pte(pv->pv_pmap, pv->pv_va); 2612209482Sjchandra rv = pte_test(pte, bit); 2613178172Simp PMAP_UNLOCK(pv->pv_pmap); 2614178172Simp if (rv) 2615178172Simp break; 2616178172Simp } 2617178172Simp return (rv); 2618178172Simp} 2619178172Simp 2620178172Simp/* 2621209482Sjchandra * this routine is used to clear dirty bits in ptes 2622178172Simp */ 2623178172Simpstatic __inline void 2624178172Simppmap_changebit(vm_page_t m, int bit, boolean_t setem) 2625178172Simp{ 2626209482Sjchandra pv_entry_t pv; 2627209482Sjchandra pt_entry_t *pte; 2628178172Simp 2629224746Skib if (m->oflags & VPO_UNMANAGED) 2630178172Simp return; 2631178172Simp 2632178172Simp mtx_assert(&vm_page_queue_mtx, MA_OWNED); 2633178172Simp /* 2634178172Simp * Loop over all current mappings setting/clearing as appropos If 2635178172Simp * setting RO do we need to clear the VAC? 2636178172Simp */ 2637178172Simp TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) { 2638178172Simp PMAP_LOCK(pv->pv_pmap); 2639178172Simp pte = pmap_pte(pv->pv_pmap, pv->pv_va); 2640178172Simp if (setem) { 2641210846Sjchandra *pte |= bit; 2642178172Simp pmap_update_page(pv->pv_pmap, pv->pv_va, *pte); 2643178172Simp } else { 2644210846Sjchandra pt_entry_t pbits = *pte; 2645178172Simp 2646178172Simp if (pbits & bit) { 2647209482Sjchandra if (bit == PTE_D) { 2648210846Sjchandra if (pbits & PTE_D) 2649178172Simp vm_page_dirty(m); 2650210846Sjchandra *pte = (pbits & ~PTE_D) | PTE_RO; 2651178172Simp } else { 2652210846Sjchandra *pte = pbits & ~bit; 2653178172Simp } 2654178172Simp pmap_update_page(pv->pv_pmap, pv->pv_va, *pte); 2655178172Simp } 2656178172Simp } 2657178172Simp PMAP_UNLOCK(pv->pv_pmap); 2658178172Simp } 2659209482Sjchandra if (!setem && bit == PTE_D) 2660225418Skib vm_page_aflag_clear(m, PGA_WRITEABLE); 2661178172Simp} 2662178172Simp 2663178172Simp/* 2664178172Simp * pmap_page_wired_mappings: 2665178172Simp * 2666178172Simp * Return the number of managed mappings to the given physical page 2667178172Simp * that are wired. 2668178172Simp */ 2669178172Simpint 2670178172Simppmap_page_wired_mappings(vm_page_t m) 2671178172Simp{ 2672178172Simp pv_entry_t pv; 2673210914Sjchandra pmap_t pmap; 2674210914Sjchandra pt_entry_t *pte; 2675178172Simp int count; 2676178172Simp 2677178172Simp count = 0; 2678224746Skib if ((m->oflags & VPO_UNMANAGED) != 0) 2679178172Simp return (count); 2680207796Salc vm_page_lock_queues(); 2681210914Sjchandra TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) { 2682210914Sjchandra pmap = pv->pv_pmap; 2683210914Sjchandra PMAP_LOCK(pmap); 2684210914Sjchandra pte = pmap_pte(pmap, pv->pv_va); 2685210914Sjchandra if (pte_test(pte, PTE_W)) 2686210914Sjchandra count++; 2687210914Sjchandra PMAP_UNLOCK(pmap); 2688210914Sjchandra } 2689207796Salc vm_page_unlock_queues(); 2690178172Simp return (count); 2691178172Simp} 2692178172Simp 2693178172Simp/* 2694178172Simp * Clear the write and modified bits in each of the given page's mappings. 2695178172Simp */ 2696178172Simpvoid 2697178172Simppmap_remove_write(vm_page_t m) 2698178172Simp{ 2699178172Simp pv_entry_t pv, npv; 2700178172Simp vm_offset_t va; 2701178172Simp pt_entry_t *pte; 2702178172Simp 2703224746Skib KASSERT((m->oflags & VPO_UNMANAGED) == 0, 2704208175Salc ("pmap_remove_write: page %p is not managed", m)); 2705208175Salc 2706208175Salc /* 2707225418Skib * If the page is not VPO_BUSY, then PGA_WRITEABLE cannot be set by 2708225418Skib * another thread while the object is locked. Thus, if PGA_WRITEABLE 2709208175Salc * is clear, no page table entries need updating. 2710208175Salc */ 2711208175Salc VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED); 2712208175Salc if ((m->oflags & VPO_BUSY) == 0 && 2713225418Skib (m->aflags & PGA_WRITEABLE) == 0) 2714178172Simp return; 2715178172Simp 2716178172Simp /* 2717178172Simp * Loop over all current mappings setting/clearing as appropos. 2718178172Simp */ 2719207796Salc vm_page_lock_queues(); 2720178172Simp for (pv = TAILQ_FIRST(&m->md.pv_list); pv; pv = npv) { 2721178172Simp npv = TAILQ_NEXT(pv, pv_plist); 2722178172Simp pte = pmap_pte(pv->pv_pmap, pv->pv_va); 2723209482Sjchandra if (pte == NULL || !pte_test(pte, PTE_V)) 2724211958Sjchandra panic("page on pm_pvlist has no pte"); 2725178172Simp 2726178172Simp va = pv->pv_va; 2727178172Simp pmap_protect(pv->pv_pmap, va, va + PAGE_SIZE, 2728178172Simp VM_PROT_READ | VM_PROT_EXECUTE); 2729178172Simp } 2730225418Skib vm_page_aflag_clear(m, PGA_WRITEABLE); 2731207796Salc vm_page_unlock_queues(); 2732178172Simp} 2733178172Simp 2734178172Simp/* 2735178172Simp * pmap_ts_referenced: 2736178172Simp * 2737178172Simp * Return the count of reference bits for a page, clearing all of them. 2738178172Simp */ 2739178172Simpint 2740178172Simppmap_ts_referenced(vm_page_t m) 2741178172Simp{ 2742178172Simp 2743224746Skib KASSERT((m->oflags & VPO_UNMANAGED) == 0, 2744208990Salc ("pmap_ts_referenced: page %p is not managed", m)); 2745178172Simp if (m->md.pv_flags & PV_TABLE_REF) { 2746208990Salc vm_page_lock_queues(); 2747178172Simp m->md.pv_flags &= ~PV_TABLE_REF; 2748208990Salc vm_page_unlock_queues(); 2749208990Salc return (1); 2750178172Simp } 2751208990Salc return (0); 2752178172Simp} 2753178172Simp 2754178172Simp/* 2755178172Simp * pmap_is_modified: 2756178172Simp * 2757178172Simp * Return whether or not the specified physical page was modified 2758178172Simp * in any physical maps. 2759178172Simp */ 2760178172Simpboolean_t 2761178172Simppmap_is_modified(vm_page_t m) 2762178172Simp{ 2763208504Salc boolean_t rv; 2764178172Simp 2765224746Skib KASSERT((m->oflags & VPO_UNMANAGED) == 0, 2766208504Salc ("pmap_is_modified: page %p is not managed", m)); 2767208504Salc 2768208504Salc /* 2769225418Skib * If the page is not VPO_BUSY, then PGA_WRITEABLE cannot be 2770225418Skib * concurrently set while the object is locked. Thus, if PGA_WRITEABLE 2771209482Sjchandra * is clear, no PTEs can have PTE_D set. 2772208504Salc */ 2773208504Salc VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED); 2774208504Salc if ((m->oflags & VPO_BUSY) == 0 && 2775225418Skib (m->aflags & PGA_WRITEABLE) == 0) 2776208504Salc return (FALSE); 2777208504Salc vm_page_lock_queues(); 2778178172Simp if (m->md.pv_flags & PV_TABLE_MOD) 2779208504Salc rv = TRUE; 2780178172Simp else 2781209482Sjchandra rv = pmap_testbit(m, PTE_D); 2782208504Salc vm_page_unlock_queues(); 2783208504Salc return (rv); 2784178172Simp} 2785178172Simp 2786178172Simp/* N/C */ 2787178172Simp 2788178172Simp/* 2789178172Simp * pmap_is_prefaultable: 2790178172Simp * 2791178172Simp * Return whether or not the specified virtual address is elgible 2792178172Simp * for prefault. 2793178172Simp */ 2794178172Simpboolean_t 2795178172Simppmap_is_prefaultable(pmap_t pmap, vm_offset_t addr) 2796178172Simp{ 2797210846Sjchandra pd_entry_t *pde; 2798178172Simp pt_entry_t *pte; 2799178172Simp boolean_t rv; 2800178172Simp 2801178172Simp rv = FALSE; 2802178172Simp PMAP_LOCK(pmap); 2803210846Sjchandra pde = pmap_pde(pmap, addr); 2804210846Sjchandra if (pde != NULL && *pde != 0) { 2805210846Sjchandra pte = pmap_pde_to_pte(pde, addr); 2806178172Simp rv = (*pte == 0); 2807178172Simp } 2808178172Simp PMAP_UNLOCK(pmap); 2809178172Simp return (rv); 2810178172Simp} 2811178172Simp 2812178172Simp/* 2813178172Simp * Clear the modify bits on the specified physical page. 2814178172Simp */ 2815178172Simpvoid 2816178172Simppmap_clear_modify(vm_page_t m) 2817178172Simp{ 2818208504Salc 2819224746Skib KASSERT((m->oflags & VPO_UNMANAGED) == 0, 2820208504Salc ("pmap_clear_modify: page %p is not managed", m)); 2821208504Salc VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED); 2822208504Salc KASSERT((m->oflags & VPO_BUSY) == 0, 2823208504Salc ("pmap_clear_modify: page %p is busy", m)); 2824208504Salc 2825208504Salc /* 2826225418Skib * If the page is not PGA_WRITEABLE, then no PTEs can have PTE_D set. 2827208504Salc * If the object containing the page is locked and the page is not 2828225418Skib * VPO_BUSY, then PGA_WRITEABLE cannot be concurrently set. 2829208504Salc */ 2830225418Skib if ((m->aflags & PGA_WRITEABLE) == 0) 2831178172Simp return; 2832208504Salc vm_page_lock_queues(); 2833178172Simp if (m->md.pv_flags & PV_TABLE_MOD) { 2834209482Sjchandra pmap_changebit(m, PTE_D, FALSE); 2835178172Simp m->md.pv_flags &= ~PV_TABLE_MOD; 2836178172Simp } 2837208504Salc vm_page_unlock_queues(); 2838178172Simp} 2839178172Simp 2840178172Simp/* 2841207155Salc * pmap_is_referenced: 2842207155Salc * 2843207155Salc * Return whether or not the specified physical page was referenced 2844207155Salc * in any physical maps. 2845207155Salc */ 2846207155Salcboolean_t 2847207155Salcpmap_is_referenced(vm_page_t m) 2848207155Salc{ 2849207155Salc 2850224746Skib KASSERT((m->oflags & VPO_UNMANAGED) == 0, 2851208574Salc ("pmap_is_referenced: page %p is not managed", m)); 2852208574Salc return ((m->md.pv_flags & PV_TABLE_REF) != 0); 2853207155Salc} 2854207155Salc 2855207155Salc/* 2856178172Simp * pmap_clear_reference: 2857178172Simp * 2858178172Simp * Clear the reference bit on the specified physical page. 2859178172Simp */ 2860178172Simpvoid 2861178172Simppmap_clear_reference(vm_page_t m) 2862178172Simp{ 2863178172Simp 2864224746Skib KASSERT((m->oflags & VPO_UNMANAGED) == 0, 2865208504Salc ("pmap_clear_reference: page %p is not managed", m)); 2866208504Salc vm_page_lock_queues(); 2867178172Simp if (m->md.pv_flags & PV_TABLE_REF) { 2868178172Simp m->md.pv_flags &= ~PV_TABLE_REF; 2869178172Simp } 2870208504Salc vm_page_unlock_queues(); 2871178172Simp} 2872178172Simp 2873178172Simp/* 2874178172Simp * Miscellaneous support routines follow 2875178172Simp */ 2876178172Simp 2877178172Simp/* 2878178172Simp * Map a set of physical memory pages into the kernel virtual 2879178172Simp * address space. Return a pointer to where it is mapped. This 2880178172Simp * routine is intended to be used for mapping device memory, 2881178172Simp * NOT real memory. 2882178172Simp */ 2883178172Simp 2884178172Simp/* 2885178172Simp * Map a set of physical memory pages into the kernel virtual 2886178172Simp * address space. Return a pointer to where it is mapped. This 2887178172Simp * routine is intended to be used for mapping device memory, 2888178172Simp * NOT real memory. 2889209930Sjchandra * 2890209930Sjchandra * Use XKPHYS uncached for 64 bit, and KSEG1 where possible for 32 bit. 2891178172Simp */ 2892178172Simpvoid * 2893217345Sjchandrapmap_mapdev(vm_paddr_t pa, vm_size_t size) 2894178172Simp{ 2895178172Simp vm_offset_t va, tmpva, offset; 2896178172Simp 2897178172Simp /* 2898178172Simp * KSEG1 maps only first 512M of phys address space. For 2899178172Simp * pa > 0x20000000 we should make proper mapping * using pmap_kenter. 2900178172Simp */ 2901211453Sjchandra if (MIPS_DIRECT_MAPPABLE(pa + size - 1)) 2902211453Sjchandra return ((void *)MIPS_PHYS_TO_DIRECT_UNCACHED(pa)); 2903178172Simp else { 2904178172Simp offset = pa & PAGE_MASK; 2905202046Simp size = roundup(size + offset, PAGE_SIZE); 2906178172Simp 2907178172Simp va = kmem_alloc_nofault(kernel_map, size); 2908178172Simp if (!va) 2909178172Simp panic("pmap_mapdev: Couldn't alloc kernel virtual memory"); 2910202046Simp pa = trunc_page(pa); 2911178172Simp for (tmpva = va; size > 0;) { 2912212589Sneel pmap_kenter_attr(tmpva, pa, PTE_C_UNCACHED); 2913178172Simp size -= PAGE_SIZE; 2914178172Simp tmpva += PAGE_SIZE; 2915178172Simp pa += PAGE_SIZE; 2916178172Simp } 2917178172Simp } 2918178172Simp 2919178172Simp return ((void *)(va + offset)); 2920178172Simp} 2921178172Simp 2922178172Simpvoid 2923178172Simppmap_unmapdev(vm_offset_t va, vm_size_t size) 2924178172Simp{ 2925211453Sjchandra#ifndef __mips_n64 2926240754Salc vm_offset_t base, offset; 2927202046Simp 2928202046Simp /* If the address is within KSEG1 then there is nothing to do */ 2929202046Simp if (va >= MIPS_KSEG1_START && va <= MIPS_KSEG1_END) 2930202046Simp return; 2931202046Simp 2932202046Simp base = trunc_page(va); 2933202046Simp offset = va & PAGE_MASK; 2934202046Simp size = roundup(size + offset, PAGE_SIZE); 2935202046Simp kmem_free(kernel_map, base, size); 2936211453Sjchandra#endif 2937178172Simp} 2938178172Simp 2939178172Simp/* 2940178172Simp * perform the pmap work for mincore 2941178172Simp */ 2942178172Simpint 2943208504Salcpmap_mincore(pmap_t pmap, vm_offset_t addr, vm_paddr_t *locked_pa) 2944178172Simp{ 2945178172Simp pt_entry_t *ptep, pte; 2946217345Sjchandra vm_paddr_t pa; 2947208532Sneel vm_page_t m; 2948208504Salc int val; 2949208504Salc boolean_t managed; 2950178172Simp 2951178172Simp PMAP_LOCK(pmap); 2952208504Salcretry: 2953178172Simp ptep = pmap_pte(pmap, addr); 2954178172Simp pte = (ptep != NULL) ? *ptep : 0; 2955209482Sjchandra if (!pte_test(&pte, PTE_V)) { 2956208504Salc val = 0; 2957208504Salc goto out; 2958208504Salc } 2959208504Salc val = MINCORE_INCORE; 2960209482Sjchandra if (pte_test(&pte, PTE_D)) 2961208504Salc val |= MINCORE_MODIFIED | MINCORE_MODIFIED_OTHER; 2962209243Sjchandra pa = TLBLO_PTE_TO_PA(pte); 2963208504Salc managed = page_is_managed(pa); 2964208504Salc if (managed) { 2965178172Simp /* 2966208504Salc * This may falsely report the given address as 2967208504Salc * MINCORE_REFERENCED. Unfortunately, due to the lack of 2968208504Salc * per-PTE reference information, it is impossible to 2969208504Salc * determine if the address is MINCORE_REFERENCED. 2970178172Simp */ 2971208504Salc m = PHYS_TO_VM_PAGE(pa); 2972225418Skib if ((m->aflags & PGA_REFERENCED) != 0) 2973178172Simp val |= MINCORE_REFERENCED | MINCORE_REFERENCED_OTHER; 2974178172Simp } 2975208504Salc if ((val & (MINCORE_MODIFIED_OTHER | MINCORE_REFERENCED_OTHER)) != 2976208504Salc (MINCORE_MODIFIED_OTHER | MINCORE_REFERENCED_OTHER) && managed) { 2977208504Salc /* Ensure that "PHYS_TO_VM_PAGE(pa)->object" doesn't change. */ 2978208504Salc if (vm_page_pa_tryrelock(pmap, pa, locked_pa)) 2979208504Salc goto retry; 2980208504Salc } else 2981208504Salcout: 2982208504Salc PA_UNLOCK_COND(*locked_pa); 2983208504Salc PMAP_UNLOCK(pmap); 2984208504Salc return (val); 2985178172Simp} 2986178172Simp 2987178172Simpvoid 2988178172Simppmap_activate(struct thread *td) 2989178172Simp{ 2990178172Simp pmap_t pmap, oldpmap; 2991178172Simp struct proc *p = td->td_proc; 2992223758Sattilio u_int cpuid; 2993178172Simp 2994178172Simp critical_enter(); 2995178172Simp 2996178172Simp pmap = vmspace_pmap(p->p_vmspace); 2997178172Simp oldpmap = PCPU_GET(curpmap); 2998223758Sattilio cpuid = PCPU_GET(cpuid); 2999178172Simp 3000178172Simp if (oldpmap) 3001223758Sattilio CPU_CLR_ATOMIC(cpuid, &oldpmap->pm_active); 3002223758Sattilio CPU_SET_ATOMIC(cpuid, &pmap->pm_active); 3003178172Simp pmap_asid_alloc(pmap); 3004178172Simp if (td == curthread) { 3005178172Simp PCPU_SET(segbase, pmap->pm_segtab); 3006223758Sattilio mips_wr_entryhi(pmap->pm_asid[cpuid].asid); 3007178172Simp } 3008202046Simp 3009178172Simp PCPU_SET(curpmap, pmap); 3010178172Simp critical_exit(); 3011178172Simp} 3012178172Simp 3013198341Smarcelvoid 3014198341Smarcelpmap_sync_icache(pmap_t pm, vm_offset_t va, vm_size_t sz) 3015198341Smarcel{ 3016198341Smarcel} 3017198341Smarcel 3018178893Salc/* 3019178893Salc * Increase the starting virtual address of the given mapping if a 3020178893Salc * different alignment might result in more superpage mappings. 3021178893Salc */ 3022178893Salcvoid 3023178893Salcpmap_align_superpage(vm_object_t object, vm_ooffset_t offset, 3024178893Salc vm_offset_t *addr, vm_size_t size) 3025178893Salc{ 3026179081Salc vm_offset_t superpage_offset; 3027179081Salc 3028179081Salc if (size < NBSEG) 3029179081Salc return; 3030179081Salc if (object != NULL && (object->flags & OBJ_COLORED) != 0) 3031179081Salc offset += ptoa(object->pg_color); 3032210627Sjchandra superpage_offset = offset & SEGMASK; 3033210627Sjchandra if (size - ((NBSEG - superpage_offset) & SEGMASK) < NBSEG || 3034210627Sjchandra (*addr & SEGMASK) == superpage_offset) 3035179081Salc return; 3036210627Sjchandra if ((*addr & SEGMASK) < superpage_offset) 3037210627Sjchandra *addr = (*addr & ~SEGMASK) + superpage_offset; 3038179081Salc else 3039210627Sjchandra *addr = ((*addr + SEGMASK) & ~SEGMASK) + superpage_offset; 3040178893Salc} 3041178893Salc 3042206819Sjmallett/* 3043206819Sjmallett * Increase the starting virtual address of the given mapping so 3044206819Sjmallett * that it is aligned to not be the second page in a TLB entry. 3045206819Sjmallett * This routine assumes that the length is appropriately-sized so 3046206819Sjmallett * that the allocation does not share a TLB entry at all if required. 3047206819Sjmallett */ 3048206819Sjmallettvoid 3049206819Sjmallettpmap_align_tlb(vm_offset_t *addr) 3050206819Sjmallett{ 3051206819Sjmallett if ((*addr & PAGE_SIZE) == 0) 3052206819Sjmallett return; 3053206819Sjmallett *addr += PAGE_SIZE; 3054206819Sjmallett return; 3055206819Sjmallett} 3056206819Sjmallett 3057211167Sjchandra#ifdef DDB 3058210846SjchandraDB_SHOW_COMMAND(ptable, ddb_pid_dump) 3059178172Simp{ 3060178172Simp pmap_t pmap; 3061210846Sjchandra struct thread *td = NULL; 3062178172Simp struct proc *p; 3063210846Sjchandra int i, j, k; 3064210846Sjchandra vm_paddr_t pa; 3065210846Sjchandra vm_offset_t va; 3066178172Simp 3067210846Sjchandra if (have_addr) { 3068210846Sjchandra td = db_lookup_thread(addr, TRUE); 3069210846Sjchandra if (td == NULL) { 3070210846Sjchandra db_printf("Invalid pid or tid"); 3071210846Sjchandra return; 3072210846Sjchandra } 3073210846Sjchandra p = td->td_proc; 3074210846Sjchandra if (p->p_vmspace == NULL) { 3075210846Sjchandra db_printf("No vmspace for process"); 3076210846Sjchandra return; 3077210846Sjchandra } 3078210846Sjchandra pmap = vmspace_pmap(p->p_vmspace); 3079210846Sjchandra } else 3080210846Sjchandra pmap = kernel_pmap; 3081178172Simp 3082210846Sjchandra db_printf("pmap:%p segtab:%p asid:%x generation:%x\n", 3083211167Sjchandra pmap, pmap->pm_segtab, pmap->pm_asid[0].asid, 3084211167Sjchandra pmap->pm_asid[0].gen); 3085210846Sjchandra for (i = 0; i < NPDEPG; i++) { 3086210846Sjchandra pd_entry_t *pdpe; 3087210846Sjchandra pt_entry_t *pde; 3088210846Sjchandra pt_entry_t pte; 3089178172Simp 3090210846Sjchandra pdpe = (pd_entry_t *)pmap->pm_segtab[i]; 3091210846Sjchandra if (pdpe == NULL) 3092210846Sjchandra continue; 3093210846Sjchandra db_printf("[%4d] %p\n", i, pdpe); 3094210846Sjchandra#ifdef __mips_n64 3095210846Sjchandra for (j = 0; j < NPDEPG; j++) { 3096210846Sjchandra pde = (pt_entry_t *)pdpe[j]; 3097210846Sjchandra if (pde == NULL) 3098210846Sjchandra continue; 3099210846Sjchandra db_printf("\t[%4d] %p\n", j, pde); 3100210846Sjchandra#else 3101210846Sjchandra { 3102210846Sjchandra j = 0; 3103210846Sjchandra pde = (pt_entry_t *)pdpe; 3104210846Sjchandra#endif 3105210846Sjchandra for (k = 0; k < NPTEPG; k++) { 3106210846Sjchandra pte = pde[k]; 3107210846Sjchandra if (pte == 0 || !pte_test(&pte, PTE_V)) 3108210846Sjchandra continue; 3109210846Sjchandra pa = TLBLO_PTE_TO_PA(pte); 3110210846Sjchandra va = ((u_long)i << SEGSHIFT) | (j << PDRSHIFT) | (k << PAGE_SHIFT); 3111217345Sjchandra db_printf("\t\t[%04d] va: %p pte: %8jx pa:%jx\n", 3112217345Sjchandra k, (void *)va, (uintmax_t)pte, (uintmax_t)pa); 3113178172Simp } 3114178172Simp } 3115178172Simp } 3116178172Simp} 3117211167Sjchandra#endif 3118178172Simp 3119178172Simp#if defined(DEBUG) 3120178172Simp 3121178172Simpstatic void pads(pmap_t pm); 3122178172Simpvoid pmap_pvdump(vm_offset_t pa); 3123178172Simp 3124178172Simp/* print address space of pmap*/ 3125178172Simpstatic void 3126178172Simppads(pmap_t pm) 3127178172Simp{ 3128178172Simp unsigned va, i, j; 3129178172Simp pt_entry_t *ptep; 3130178172Simp 3131178172Simp if (pm == kernel_pmap) 3132178172Simp return; 3133178172Simp for (i = 0; i < NPTEPG; i++) 3134178172Simp if (pm->pm_segtab[i]) 3135178172Simp for (j = 0; j < NPTEPG; j++) { 3136178172Simp va = (i << SEGSHIFT) + (j << PAGE_SHIFT); 3137178172Simp if (pm == kernel_pmap && va < KERNBASE) 3138178172Simp continue; 3139178172Simp if (pm != kernel_pmap && 3140178172Simp va >= VM_MAXUSER_ADDRESS) 3141178172Simp continue; 3142178172Simp ptep = pmap_pte(pm, va); 3143216324Sjchandra if (pte_test(ptep, PTE_V)) 3144178172Simp printf("%x:%x ", va, *(int *)ptep); 3145178172Simp } 3146178172Simp 3147178172Simp} 3148178172Simp 3149178172Simpvoid 3150178172Simppmap_pvdump(vm_offset_t pa) 3151178172Simp{ 3152178172Simp register pv_entry_t pv; 3153178172Simp vm_page_t m; 3154178172Simp 3155178172Simp printf("pa %x", pa); 3156178172Simp m = PHYS_TO_VM_PAGE(pa); 3157178172Simp for (pv = TAILQ_FIRST(&m->md.pv_list); pv; 3158178172Simp pv = TAILQ_NEXT(pv, pv_list)) { 3159178172Simp printf(" -> pmap %p, va %x", (void *)pv->pv_pmap, pv->pv_va); 3160178172Simp pads(pv->pv_pmap); 3161178172Simp } 3162178172Simp printf(" "); 3163178172Simp} 3164178172Simp 3165178172Simp/* N/C */ 3166178172Simp#endif 3167178172Simp 3168178172Simp 3169178172Simp/* 3170178172Simp * Allocate TLB address space tag (called ASID or TLBPID) and return it. 3171178172Simp * It takes almost as much or more time to search the TLB for a 3172178172Simp * specific ASID and flush those entries as it does to flush the entire TLB. 3173178172Simp * Therefore, when we allocate a new ASID, we just take the next number. When 3174178172Simp * we run out of numbers, we flush the TLB, increment the generation count 3175178172Simp * and start over. ASID zero is reserved for kernel use. 3176178172Simp */ 3177178172Simpstatic void 3178178172Simppmap_asid_alloc(pmap) 3179178172Simp pmap_t pmap; 3180178172Simp{ 3181178172Simp if (pmap->pm_asid[PCPU_GET(cpuid)].asid != PMAP_ASID_RESERVED && 3182178172Simp pmap->pm_asid[PCPU_GET(cpuid)].gen == PCPU_GET(asid_generation)); 3183178172Simp else { 3184178172Simp if (PCPU_GET(next_asid) == pmap_max_asid) { 3185209243Sjchandra tlb_invalidate_all_user(NULL); 3186178172Simp PCPU_SET(asid_generation, 3187178172Simp (PCPU_GET(asid_generation) + 1) & ASIDGEN_MASK); 3188178172Simp if (PCPU_GET(asid_generation) == 0) { 3189178172Simp PCPU_SET(asid_generation, 1); 3190178172Simp } 3191178172Simp PCPU_SET(next_asid, 1); /* 0 means invalid */ 3192178172Simp } 3193178172Simp pmap->pm_asid[PCPU_GET(cpuid)].asid = PCPU_GET(next_asid); 3194178172Simp pmap->pm_asid[PCPU_GET(cpuid)].gen = PCPU_GET(asid_generation); 3195178172Simp PCPU_SET(next_asid, PCPU_GET(next_asid) + 1); 3196178172Simp } 3197178172Simp} 3198178172Simp 3199178172Simpint 3200217345Sjchandrapage_is_managed(vm_paddr_t pa) 3201178172Simp{ 3202217354Sjchandra vm_offset_t pgnum = atop(pa); 3203178172Simp 3204207139Sjmallett if (pgnum >= first_page) { 3205178172Simp vm_page_t m; 3206178172Simp 3207178172Simp m = PHYS_TO_VM_PAGE(pa); 3208207139Sjmallett if (m == NULL) 3209211445Sjchandra return (0); 3210224746Skib if ((m->oflags & VPO_UNMANAGED) == 0) 3211211445Sjchandra return (1); 3212178172Simp } 3213211445Sjchandra return (0); 3214178172Simp} 3215178172Simp 3216217345Sjchandrastatic pt_entry_t 3217178172Simpinit_pte_prot(vm_offset_t va, vm_page_t m, vm_prot_t prot) 3218178172Simp{ 3219217345Sjchandra pt_entry_t rw; 3220178172Simp 3221178172Simp if (!(prot & VM_PROT_WRITE)) 3222209482Sjchandra rw = PTE_V | PTE_RO | PTE_C_CACHE; 3223224746Skib else if ((m->oflags & VPO_UNMANAGED) == 0) { 3224208866Salc if ((m->md.pv_flags & PV_TABLE_MOD) != 0) 3225209482Sjchandra rw = PTE_V | PTE_D | PTE_C_CACHE; 3226178172Simp else 3227209482Sjchandra rw = PTE_V | PTE_C_CACHE; 3228225418Skib vm_page_aflag_set(m, PGA_WRITEABLE); 3229208866Salc } else 3230208866Salc /* Needn't emulate a modified bit for unmanaged pages. */ 3231209482Sjchandra rw = PTE_V | PTE_D | PTE_C_CACHE; 3232208866Salc return (rw); 3233178172Simp} 3234178172Simp 3235178172Simp/* 3236211217Sjchandra * pmap_emulate_modified : do dirty bit emulation 3237178172Simp * 3238211217Sjchandra * On SMP, update just the local TLB, other CPUs will update their 3239211217Sjchandra * TLBs from PTE lazily, if they get the exception. 3240211217Sjchandra * Returns 0 in case of sucess, 1 if the page is read only and we 3241211217Sjchandra * need to fault. 3242178172Simp */ 3243211217Sjchandraint 3244211217Sjchandrapmap_emulate_modified(pmap_t pmap, vm_offset_t va) 3245178172Simp{ 3246211217Sjchandra vm_page_t m; 3247211217Sjchandra pt_entry_t *pte; 3248217345Sjchandra vm_paddr_t pa; 3249178172Simp 3250211217Sjchandra PMAP_LOCK(pmap); 3251211217Sjchandra pte = pmap_pte(pmap, va); 3252211217Sjchandra if (pte == NULL) 3253211217Sjchandra panic("pmap_emulate_modified: can't find PTE"); 3254211217Sjchandra#ifdef SMP 3255211217Sjchandra /* It is possible that some other CPU changed m-bit */ 3256211217Sjchandra if (!pte_test(pte, PTE_V) || pte_test(pte, PTE_D)) { 3257211217Sjchandra pmap_update_page_local(pmap, va, *pte); 3258211217Sjchandra PMAP_UNLOCK(pmap); 3259211217Sjchandra return (0); 3260211217Sjchandra } 3261211217Sjchandra#else 3262211217Sjchandra if (!pte_test(pte, PTE_V) || pte_test(pte, PTE_D)) 3263211217Sjchandra panic("pmap_emulate_modified: invalid pte"); 3264211217Sjchandra#endif 3265211217Sjchandra if (pte_test(pte, PTE_RO)) { 3266211217Sjchandra /* write to read only page in the kernel */ 3267211217Sjchandra PMAP_UNLOCK(pmap); 3268211217Sjchandra return (1); 3269211217Sjchandra } 3270211217Sjchandra pte_set(pte, PTE_D); 3271211217Sjchandra pmap_update_page_local(pmap, va, *pte); 3272211217Sjchandra pa = TLBLO_PTE_TO_PA(*pte); 3273211217Sjchandra if (!page_is_managed(pa)) 3274211217Sjchandra panic("pmap_emulate_modified: unmanaged page"); 3275211217Sjchandra m = PHYS_TO_VM_PAGE(pa); 3276211217Sjchandra m->md.pv_flags |= (PV_TABLE_REF | PV_TABLE_MOD); 3277211217Sjchandra PMAP_UNLOCK(pmap); 3278211217Sjchandra return (0); 3279178172Simp} 3280178172Simp 3281178172Simp/* 3282178172Simp * Routine: pmap_kextract 3283178172Simp * Function: 3284178172Simp * Extract the physical page address associated 3285178172Simp * virtual address. 3286178172Simp */ 3287178172Simp /* PMAP_INLINE */ vm_offset_t 3288178172Simppmap_kextract(vm_offset_t va) 3289178172Simp{ 3290209930Sjchandra int mapped; 3291178172Simp 3292209930Sjchandra /* 3293209930Sjchandra * First, the direct-mapped regions. 3294209930Sjchandra */ 3295209930Sjchandra#if defined(__mips_n64) 3296209930Sjchandra if (va >= MIPS_XKPHYS_START && va < MIPS_XKPHYS_END) 3297209930Sjchandra return (MIPS_XKPHYS_TO_PHYS(va)); 3298209930Sjchandra#endif 3299209930Sjchandra if (va >= MIPS_KSEG0_START && va < MIPS_KSEG0_END) 3300209930Sjchandra return (MIPS_KSEG0_TO_PHYS(va)); 3301209930Sjchandra 3302209930Sjchandra if (va >= MIPS_KSEG1_START && va < MIPS_KSEG1_END) 3303209930Sjchandra return (MIPS_KSEG1_TO_PHYS(va)); 3304209930Sjchandra 3305209930Sjchandra /* 3306209930Sjchandra * User virtual addresses. 3307209930Sjchandra */ 3308209930Sjchandra if (va < VM_MAXUSER_ADDRESS) { 3309178172Simp pt_entry_t *ptep; 3310178172Simp 3311178172Simp if (curproc && curproc->p_vmspace) { 3312178172Simp ptep = pmap_pte(&curproc->p_vmspace->vm_pmap, va); 3313209930Sjchandra if (ptep) { 3314209930Sjchandra return (TLBLO_PTE_TO_PA(*ptep) | 3315209930Sjchandra (va & PAGE_MASK)); 3316209930Sjchandra } 3317209930Sjchandra return (0); 3318178172Simp } 3319209930Sjchandra } 3320209930Sjchandra 3321209930Sjchandra /* 3322209930Sjchandra * Should be kernel virtual here, otherwise fail 3323209930Sjchandra */ 3324209930Sjchandra mapped = (va >= MIPS_KSEG2_START || va < MIPS_KSEG2_END); 3325209930Sjchandra#if defined(__mips_n64) 3326209930Sjchandra mapped = mapped || (va >= MIPS_XKSEG_START || va < MIPS_XKSEG_END); 3327209930Sjchandra#endif 3328209930Sjchandra /* 3329209930Sjchandra * Kernel virtual. 3330209930Sjchandra */ 3331209930Sjchandra 3332209930Sjchandra if (mapped) { 3333178172Simp pt_entry_t *ptep; 3334178172Simp 3335191735Salc /* Is the kernel pmap initialized? */ 3336222813Sattilio if (!CPU_EMPTY(&kernel_pmap->pm_active)) { 3337209930Sjchandra /* It's inside the virtual address range */ 3338206717Sjmallett ptep = pmap_pte(kernel_pmap, va); 3339209243Sjchandra if (ptep) { 3340209243Sjchandra return (TLBLO_PTE_TO_PA(*ptep) | 3341209243Sjchandra (va & PAGE_MASK)); 3342209243Sjchandra } 3343178172Simp } 3344209930Sjchandra return (0); 3345178172Simp } 3346209930Sjchandra 3347209930Sjchandra panic("%s for unknown address space %p.", __func__, (void *)va); 3348178172Simp} 3349202046Simp 3350209930Sjchandra 3351202046Simpvoid 3352202046Simppmap_flush_pvcache(vm_page_t m) 3353202046Simp{ 3354202046Simp pv_entry_t pv; 3355202046Simp 3356202046Simp if (m != NULL) { 3357202046Simp for (pv = TAILQ_FIRST(&m->md.pv_list); pv; 3358210846Sjchandra pv = TAILQ_NEXT(pv, pv_list)) { 3359206746Sjmallett mips_dcache_wbinv_range_index(pv->pv_va, PAGE_SIZE); 3360202046Simp } 3361202046Simp } 3362202046Simp} 3363