pmap.c revision 238561
1169689Skan/* 2169689Skan * Copyright (c) 1991 Regents of the University of California. 3169689Skan * All rights reserved. 4169689Skan * Copyright (c) 1994 John S. Dyson 5169689Skan * All rights reserved. 6169689Skan * Copyright (c) 1994 David Greenman 7169689Skan * All rights reserved. 8169689Skan * 9169689Skan * This code is derived from software contributed to Berkeley by 10169689Skan * the Systems Programming Group of the University of Utah Computer 11169689Skan * Science Department and William Jolitz of UUNET Technologies Inc. 12169689Skan * 13169689Skan * Redistribution and use in source and binary forms, with or without 14169689Skan * modification, are permitted provided that the following conditions 15169689Skan * are met: 16169689Skan * 1. Redistributions of source code must retain the above copyright 17169689Skan * notice, this list of conditions and the following disclaimer. 18169689Skan * 2. Redistributions in binary form must reproduce the above copyright 19169689Skan * notice, this list of conditions and the following disclaimer in the 20169689Skan * documentation and/or other materials provided with the distribution. 21169689Skan * 4. Neither the name of the University nor the names of its contributors 22169689Skan * may be used to endorse or promote products derived from this software 23169689Skan * without specific prior written permission. 24169689Skan * 25169689Skan * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 26169689Skan * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 27169689Skan * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 28169689Skan * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 29169689Skan * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 30169689Skan * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 31169689Skan * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 32169689Skan * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 33169689Skan * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 34169689Skan * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 35169689Skan * SUCH DAMAGE. 36169689Skan * 37169689Skan * from: @(#)pmap.c 7.7 (Berkeley) 5/12/91 38169689Skan * from: src/sys/i386/i386/pmap.c,v 1.250.2.8 2000/11/21 00:09:14 ps 39169689Skan * JNPR: pmap.c,v 1.11.2.1 2007/08/16 11:51:06 girish 40169689Skan */ 41169689Skan 42169689Skan/* 43169689Skan * Manages physical address maps. 44169689Skan * 45169689Skan * In addition to hardware address maps, this 46169689Skan * module is called upon to provide software-use-only 47169689Skan * maps which may or may not be stored in the same 48169689Skan * form as hardware maps. These pseudo-maps are 49169689Skan * used to store intermediate results from copy 50169689Skan * operations to and from address spaces. 51169689Skan * 52169689Skan * Since the information managed by this module is 53169689Skan * also stored by the logical address mapping module, 54169689Skan * this module may throw away valid virtual-to-physical 55169689Skan * mappings at almost any time. However, invalidations 56169689Skan * of virtual-to-physical mappings must be done as 57169689Skan * requested. 58169689Skan * 59169689Skan * In order to cope with hardware architectures which 60169689Skan * make virtual-to-physical map invalidates expensive, 61169689Skan * this module may delay invalidate or reduced protection 62169689Skan * operations until such time as they are actually 63169689Skan * necessary. This module is given full information as 64169689Skan * to which processors are currently using which maps, 65169689Skan * and to when physical maps must be made correct. 66169689Skan */ 67169689Skan 68169689Skan#include <sys/cdefs.h> 69169689Skan__FBSDID("$FreeBSD: head/sys/mips/mips/pmap.c 238561 2012-07-18 05:21:34Z alc $"); 70169689Skan 71169689Skan#include "opt_ddb.h" 72169689Skan 73169689Skan#include <sys/param.h> 74169689Skan#include <sys/systm.h> 75169689Skan#include <sys/proc.h> 76169689Skan#include <sys/msgbuf.h> 77169689Skan#include <sys/vmmeter.h> 78169689Skan#include <sys/mman.h> 79169689Skan#include <sys/smp.h> 80169689Skan#ifdef DDB 81169689Skan#include <ddb/ddb.h> 82169689Skan#endif 83169689Skan 84169689Skan#include <vm/vm.h> 85169689Skan#include <vm/vm_param.h> 86169689Skan#include <vm/vm_phys.h> 87169689Skan#include <sys/lock.h> 88169689Skan#include <sys/mutex.h> 89169689Skan#include <vm/vm_kern.h> 90169689Skan#include <vm/vm_page.h> 91169689Skan#include <vm/vm_map.h> 92169689Skan#include <vm/vm_object.h> 93169689Skan#include <vm/vm_extern.h> 94169689Skan#include <vm/vm_pageout.h> 95169689Skan#include <vm/vm_pager.h> 96169689Skan#include <vm/uma.h> 97169689Skan#include <sys/pcpu.h> 98169689Skan#include <sys/sched.h> 99169689Skan#ifdef SMP 100169689Skan#include <sys/smp.h> 101169689Skan#endif 102169689Skan 103169689Skan#include <machine/cache.h> 104169689Skan#include <machine/md_var.h> 105169689Skan#include <machine/tlb.h> 106169689Skan 107169689Skan#undef PMAP_DEBUG 108169689Skan 109169689Skan#ifndef PMAP_SHPGPERPROC 110169689Skan#define PMAP_SHPGPERPROC 200 111169689Skan#endif 112169689Skan 113169689Skan#if !defined(DIAGNOSTIC) 114169689Skan#define PMAP_INLINE __inline 115169689Skan#else 116169689Skan#define PMAP_INLINE 117169689Skan#endif 118169689Skan 119169689Skan/* 120169689Skan * Get PDEs and PTEs for user/kernel address space 121169689Skan */ 122169689Skan#define pmap_seg_index(v) (((v) >> SEGSHIFT) & (NPDEPG - 1)) 123169689Skan#define pmap_pde_index(v) (((v) >> PDRSHIFT) & (NPDEPG - 1)) 124169689Skan#define pmap_pte_index(v) (((v) >> PAGE_SHIFT) & (NPTEPG - 1)) 125169689Skan#define pmap_pde_pindex(v) ((v) >> PDRSHIFT) 126169689Skan 127169689Skan#ifdef __mips_n64 128169689Skan#define NUPDE (NPDEPG * NPDEPG) 129169689Skan#define NUSERPGTBLS (NUPDE + NPDEPG) 130169689Skan#else 131169689Skan#define NUPDE (NPDEPG) 132169689Skan#define NUSERPGTBLS (NUPDE) 133169689Skan#endif 134169689Skan 135169689Skan#define is_kernel_pmap(x) ((x) == kernel_pmap) 136169689Skan 137169689Skanstruct pmap kernel_pmap_store; 138169689Skanpd_entry_t *kernel_segmap; 139169689Skan 140169689Skanvm_offset_t virtual_avail; /* VA of first avail page (after kernel bss) */ 141169689Skanvm_offset_t virtual_end; /* VA of last avail page (end of kernel AS) */ 142169689Skan 143169689Skanstatic int nkpt; 144169689Skanunsigned pmap_max_asid; /* max ASID supported by the system */ 145169689Skan 146169689Skan#define PMAP_ASID_RESERVED 0 147169689Skan 148169689Skanvm_offset_t kernel_vm_end = VM_MIN_KERNEL_ADDRESS; 149169689Skan 150169689Skanstatic void pmap_asid_alloc(pmap_t pmap); 151169689Skan 152169689Skan/* 153169689Skan * Data for the pv entry allocation mechanism 154169689Skan */ 155169689Skanstatic uma_zone_t pvzone; 156169689Skanstatic struct vm_object pvzone_obj; 157169689Skanstatic int pv_entry_count = 0, pv_entry_max = 0, pv_entry_high_water = 0; 158169689Skan 159169689Skanstatic PMAP_INLINE void free_pv_entry(pv_entry_t pv); 160169689Skanstatic pv_entry_t get_pv_entry(pmap_t locked_pmap); 161169689Skanstatic void pmap_pvh_free(struct md_page *pvh, pmap_t pmap, vm_offset_t va); 162169689Skanstatic pv_entry_t pmap_pvh_remove(struct md_page *pvh, pmap_t pmap, 163169689Skan vm_offset_t va); 164169689Skanstatic __inline void pmap_changebit(vm_page_t m, int bit, boolean_t setem); 165169689Skanstatic vm_page_t pmap_enter_quick_locked(pmap_t pmap, vm_offset_t va, 166169689Skan vm_page_t m, vm_prot_t prot, vm_page_t mpte); 167169689Skanstatic int pmap_remove_pte(struct pmap *pmap, pt_entry_t *ptq, vm_offset_t va); 168169689Skanstatic void pmap_remove_page(struct pmap *pmap, vm_offset_t va); 169169689Skanstatic void pmap_remove_entry(struct pmap *pmap, vm_page_t m, vm_offset_t va); 170169689Skanstatic boolean_t pmap_try_insert_pv_entry(pmap_t pmap, vm_page_t mpte, 171169689Skan vm_offset_t va, vm_page_t m); 172169689Skanstatic void pmap_update_page(pmap_t pmap, vm_offset_t va, pt_entry_t pte); 173169689Skanstatic void pmap_invalidate_all(pmap_t pmap); 174169689Skanstatic void pmap_invalidate_page(pmap_t pmap, vm_offset_t va); 175169689Skanstatic int _pmap_unwire_pte_hold(pmap_t pmap, vm_offset_t va, vm_page_t m); 176169689Skan 177169689Skanstatic vm_page_t pmap_allocpte(pmap_t pmap, vm_offset_t va, int flags); 178169689Skanstatic vm_page_t _pmap_allocpte(pmap_t pmap, unsigned ptepindex, int flags); 179169689Skanstatic int pmap_unuse_pt(pmap_t, vm_offset_t, vm_page_t); 180169689Skanstatic pt_entry_t init_pte_prot(vm_offset_t va, vm_page_t m, vm_prot_t prot); 181169689Skan 182169689Skan#ifdef SMP 183169689Skanstatic void pmap_invalidate_page_action(void *arg); 184169689Skanstatic void pmap_update_page_action(void *arg); 185169689Skan#endif 186169689Skan 187169689Skan#ifndef __mips_n64 188169689Skan/* 189169689Skan * This structure is for high memory (memory above 512Meg in 32 bit) support. 190169689Skan * The highmem area does not have a KSEG0 mapping, and we need a mechanism to 191169689Skan * do temporary per-CPU mappings for pmap_zero_page, pmap_copy_page etc. 192169689Skan * 193169689Skan * At bootup, we reserve 2 virtual pages per CPU for mapping highmem pages. To 194169689Skan * access a highmem physical address on a CPU, we map the physical address to 195169689Skan * the reserved virtual address for the CPU in the kernel pagetable. This is 196169689Skan * done with interrupts disabled(although a spinlock and sched_pin would be 197169689Skan * sufficient). 198169689Skan */ 199169689Skanstruct local_sysmaps { 200169689Skan vm_offset_t base; 201169689Skan uint32_t saved_intr; 202169689Skan uint16_t valid1, valid2; 203169689Skan}; 204169689Skanstatic struct local_sysmaps sysmap_lmem[MAXCPU]; 205169689Skan 206169689Skanstatic __inline void 207169689Skanpmap_alloc_lmem_map(void) 208169689Skan{ 209169689Skan int i; 210169689Skan 211169689Skan for (i = 0; i < MAXCPU; i++) { 212169689Skan sysmap_lmem[i].base = virtual_avail; 213169689Skan virtual_avail += PAGE_SIZE * 2; 214169689Skan sysmap_lmem[i].valid1 = sysmap_lmem[i].valid2 = 0; 215169689Skan } 216169689Skan} 217169689Skan 218169689Skanstatic __inline vm_offset_t 219169689Skanpmap_lmem_map1(vm_paddr_t phys) 220169689Skan{ 221169689Skan struct local_sysmaps *sysm; 222169689Skan pt_entry_t *pte, npte; 223169689Skan vm_offset_t va; 224169689Skan uint32_t intr; 225169689Skan int cpu; 226169689Skan 227169689Skan intr = intr_disable(); 228169689Skan cpu = PCPU_GET(cpuid); 229169689Skan sysm = &sysmap_lmem[cpu]; 230169689Skan sysm->saved_intr = intr; 231169689Skan va = sysm->base; 232169689Skan npte = TLBLO_PA_TO_PFN(phys) | 233169689Skan PTE_D | PTE_V | PTE_G | PTE_W | PTE_C_CACHE; 234169689Skan pte = pmap_pte(kernel_pmap, va); 235169689Skan *pte = npte; 236169689Skan sysm->valid1 = 1; 237169689Skan return (va); 238169689Skan} 239169689Skan 240169689Skanstatic __inline vm_offset_t 241169689Skanpmap_lmem_map2(vm_paddr_t phys1, vm_paddr_t phys2) 242169689Skan{ 243169689Skan struct local_sysmaps *sysm; 244169689Skan pt_entry_t *pte, npte; 245169689Skan vm_offset_t va1, va2; 246169689Skan uint32_t intr; 247169689Skan int cpu; 248169689Skan 249169689Skan intr = intr_disable(); 250169689Skan cpu = PCPU_GET(cpuid); 251169689Skan sysm = &sysmap_lmem[cpu]; 252169689Skan sysm->saved_intr = intr; 253169689Skan va1 = sysm->base; 254169689Skan va2 = sysm->base + PAGE_SIZE; 255169689Skan npte = TLBLO_PA_TO_PFN(phys1) | 256169689Skan PTE_D | PTE_V | PTE_G | PTE_W | PTE_C_CACHE; 257169689Skan pte = pmap_pte(kernel_pmap, va1); 258169689Skan *pte = npte; 259169689Skan npte = TLBLO_PA_TO_PFN(phys2) | 260169689Skan PTE_D | PTE_V | PTE_G | PTE_W | PTE_C_CACHE; 261169689Skan pte = pmap_pte(kernel_pmap, va2); 262169689Skan *pte = npte; 263169689Skan sysm->valid1 = 1; 264169689Skan sysm->valid2 = 1; 265169689Skan return (va1); 266169689Skan} 267169689Skan 268169689Skanstatic __inline void 269169689Skanpmap_lmem_unmap(void) 270169689Skan{ 271169689Skan struct local_sysmaps *sysm; 272169689Skan pt_entry_t *pte; 273169689Skan int cpu; 274169689Skan 275169689Skan cpu = PCPU_GET(cpuid); 276169689Skan sysm = &sysmap_lmem[cpu]; 277169689Skan pte = pmap_pte(kernel_pmap, sysm->base); 278169689Skan *pte = PTE_G; 279169689Skan tlb_invalidate_address(kernel_pmap, sysm->base); 280169689Skan sysm->valid1 = 0; 281169689Skan if (sysm->valid2) { 282169689Skan pte = pmap_pte(kernel_pmap, sysm->base + PAGE_SIZE); 283169689Skan *pte = PTE_G; 284169689Skan tlb_invalidate_address(kernel_pmap, sysm->base + PAGE_SIZE); 285169689Skan sysm->valid2 = 0; 286169689Skan } 287169689Skan intr_restore(sysm->saved_intr); 288169689Skan} 289169689Skan#else /* __mips_n64 */ 290169689Skan 291169689Skanstatic __inline void 292169689Skanpmap_alloc_lmem_map(void) 293169689Skan{ 294169689Skan} 295169689Skan 296169689Skanstatic __inline vm_offset_t 297169689Skanpmap_lmem_map1(vm_paddr_t phys) 298169689Skan{ 299169689Skan 300169689Skan return (0); 301169689Skan} 302169689Skan 303169689Skanstatic __inline vm_offset_t 304169689Skanpmap_lmem_map2(vm_paddr_t phys1, vm_paddr_t phys2) 305169689Skan{ 306169689Skan 307169689Skan return (0); 308169689Skan} 309169689Skan 310169689Skanstatic __inline vm_offset_t 311169689Skanpmap_lmem_unmap(void) 312{ 313 314 return (0); 315} 316#endif /* !__mips_n64 */ 317 318/* 319 * Page table entry lookup routines. 320 */ 321static __inline pd_entry_t * 322pmap_segmap(pmap_t pmap, vm_offset_t va) 323{ 324 325 return (&pmap->pm_segtab[pmap_seg_index(va)]); 326} 327 328#ifdef __mips_n64 329static __inline pd_entry_t * 330pmap_pdpe_to_pde(pd_entry_t *pdpe, vm_offset_t va) 331{ 332 pd_entry_t *pde; 333 334 pde = (pd_entry_t *)*pdpe; 335 return (&pde[pmap_pde_index(va)]); 336} 337 338static __inline pd_entry_t * 339pmap_pde(pmap_t pmap, vm_offset_t va) 340{ 341 pd_entry_t *pdpe; 342 343 pdpe = pmap_segmap(pmap, va); 344 if (pdpe == NULL || *pdpe == NULL) 345 return (NULL); 346 347 return (pmap_pdpe_to_pde(pdpe, va)); 348} 349#else 350static __inline pd_entry_t * 351pmap_pdpe_to_pde(pd_entry_t *pdpe, vm_offset_t va) 352{ 353 354 return (pdpe); 355} 356 357static __inline 358pd_entry_t *pmap_pde(pmap_t pmap, vm_offset_t va) 359{ 360 361 return (pmap_segmap(pmap, va)); 362} 363#endif 364 365static __inline pt_entry_t * 366pmap_pde_to_pte(pd_entry_t *pde, vm_offset_t va) 367{ 368 pt_entry_t *pte; 369 370 pte = (pt_entry_t *)*pde; 371 return (&pte[pmap_pte_index(va)]); 372} 373 374pt_entry_t * 375pmap_pte(pmap_t pmap, vm_offset_t va) 376{ 377 pd_entry_t *pde; 378 379 pde = pmap_pde(pmap, va); 380 if (pde == NULL || *pde == NULL) 381 return (NULL); 382 383 return (pmap_pde_to_pte(pde, va)); 384} 385 386vm_offset_t 387pmap_steal_memory(vm_size_t size) 388{ 389 vm_paddr_t bank_size, pa; 390 vm_offset_t va; 391 392 size = round_page(size); 393 bank_size = phys_avail[1] - phys_avail[0]; 394 while (size > bank_size) { 395 int i; 396 397 for (i = 0; phys_avail[i + 2]; i += 2) { 398 phys_avail[i] = phys_avail[i + 2]; 399 phys_avail[i + 1] = phys_avail[i + 3]; 400 } 401 phys_avail[i] = 0; 402 phys_avail[i + 1] = 0; 403 if (!phys_avail[0]) 404 panic("pmap_steal_memory: out of memory"); 405 bank_size = phys_avail[1] - phys_avail[0]; 406 } 407 408 pa = phys_avail[0]; 409 phys_avail[0] += size; 410 if (MIPS_DIRECT_MAPPABLE(pa) == 0) 411 panic("Out of memory below 512Meg?"); 412 va = MIPS_PHYS_TO_DIRECT(pa); 413 bzero((caddr_t)va, size); 414 return (va); 415} 416 417/* 418 * Bootstrap the system enough to run with virtual memory. This 419 * assumes that the phys_avail array has been initialized. 420 */ 421static void 422pmap_create_kernel_pagetable(void) 423{ 424 int i, j; 425 vm_offset_t ptaddr; 426 pt_entry_t *pte; 427#ifdef __mips_n64 428 pd_entry_t *pde; 429 vm_offset_t pdaddr; 430 int npt, npde; 431#endif 432 433 /* 434 * Allocate segment table for the kernel 435 */ 436 kernel_segmap = (pd_entry_t *)pmap_steal_memory(PAGE_SIZE); 437 438 /* 439 * Allocate second level page tables for the kernel 440 */ 441#ifdef __mips_n64 442 npde = howmany(NKPT, NPDEPG); 443 pdaddr = pmap_steal_memory(PAGE_SIZE * npde); 444#endif 445 nkpt = NKPT; 446 ptaddr = pmap_steal_memory(PAGE_SIZE * nkpt); 447 448 /* 449 * The R[4-7]?00 stores only one copy of the Global bit in the 450 * translation lookaside buffer for each 2 page entry. Thus invalid 451 * entrys must have the Global bit set so when Entry LO and Entry HI 452 * G bits are anded together they will produce a global bit to store 453 * in the tlb. 454 */ 455 for (i = 0, pte = (pt_entry_t *)ptaddr; i < (nkpt * NPTEPG); i++, pte++) 456 *pte = PTE_G; 457 458#ifdef __mips_n64 459 for (i = 0, npt = nkpt; npt > 0; i++) { 460 kernel_segmap[i] = (pd_entry_t)(pdaddr + i * PAGE_SIZE); 461 pde = (pd_entry_t *)kernel_segmap[i]; 462 463 for (j = 0; j < NPDEPG && npt > 0; j++, npt--) 464 pde[j] = (pd_entry_t)(ptaddr + (i * NPDEPG + j) * PAGE_SIZE); 465 } 466#else 467 for (i = 0, j = pmap_seg_index(VM_MIN_KERNEL_ADDRESS); i < nkpt; i++, j++) 468 kernel_segmap[j] = (pd_entry_t)(ptaddr + (i * PAGE_SIZE)); 469#endif 470 471 PMAP_LOCK_INIT(kernel_pmap); 472 kernel_pmap->pm_segtab = kernel_segmap; 473 CPU_FILL(&kernel_pmap->pm_active); 474 TAILQ_INIT(&kernel_pmap->pm_pvlist); 475 kernel_pmap->pm_asid[0].asid = PMAP_ASID_RESERVED; 476 kernel_pmap->pm_asid[0].gen = 0; 477 kernel_vm_end += nkpt * NPTEPG * PAGE_SIZE; 478} 479 480void 481pmap_bootstrap(void) 482{ 483 int i; 484 int need_local_mappings = 0; 485 486 /* Sort. */ 487again: 488 for (i = 0; phys_avail[i + 1] != 0; i += 2) { 489 /* 490 * Keep the memory aligned on page boundary. 491 */ 492 phys_avail[i] = round_page(phys_avail[i]); 493 phys_avail[i + 1] = trunc_page(phys_avail[i + 1]); 494 495 if (i < 2) 496 continue; 497 if (phys_avail[i - 2] > phys_avail[i]) { 498 vm_paddr_t ptemp[2]; 499 500 ptemp[0] = phys_avail[i + 0]; 501 ptemp[1] = phys_avail[i + 1]; 502 503 phys_avail[i + 0] = phys_avail[i - 2]; 504 phys_avail[i + 1] = phys_avail[i - 1]; 505 506 phys_avail[i - 2] = ptemp[0]; 507 phys_avail[i - 1] = ptemp[1]; 508 goto again; 509 } 510 } 511 512 /* 513 * In 32 bit, we may have memory which cannot be mapped directly. 514 * This memory will need temporary mapping before it can be 515 * accessed. 516 */ 517 if (!MIPS_DIRECT_MAPPABLE(phys_avail[i - 1] - 1)) 518 need_local_mappings = 1; 519 520 /* 521 * Copy the phys_avail[] array before we start stealing memory from it. 522 */ 523 for (i = 0; phys_avail[i + 1] != 0; i += 2) { 524 physmem_desc[i] = phys_avail[i]; 525 physmem_desc[i + 1] = phys_avail[i + 1]; 526 } 527 528 Maxmem = atop(phys_avail[i - 1]); 529 530 if (bootverbose) { 531 printf("Physical memory chunk(s):\n"); 532 for (i = 0; phys_avail[i + 1] != 0; i += 2) { 533 vm_paddr_t size; 534 535 size = phys_avail[i + 1] - phys_avail[i]; 536 printf("%#08jx - %#08jx, %ju bytes (%ju pages)\n", 537 (uintmax_t) phys_avail[i], 538 (uintmax_t) phys_avail[i + 1] - 1, 539 (uintmax_t) size, (uintmax_t) size / PAGE_SIZE); 540 } 541 printf("Maxmem is 0x%0jx\n", ptoa((uintmax_t)Maxmem)); 542 } 543 /* 544 * Steal the message buffer from the beginning of memory. 545 */ 546 msgbufp = (struct msgbuf *)pmap_steal_memory(msgbufsize); 547 msgbufinit(msgbufp, msgbufsize); 548 549 /* 550 * Steal thread0 kstack. 551 */ 552 kstack0 = pmap_steal_memory(KSTACK_PAGES << PAGE_SHIFT); 553 554 virtual_avail = VM_MIN_KERNEL_ADDRESS; 555 virtual_end = VM_MAX_KERNEL_ADDRESS; 556 557#ifdef SMP 558 /* 559 * Steal some virtual address space to map the pcpu area. 560 */ 561 virtual_avail = roundup2(virtual_avail, PAGE_SIZE * 2); 562 pcpup = (struct pcpu *)virtual_avail; 563 virtual_avail += PAGE_SIZE * 2; 564 565 /* 566 * Initialize the wired TLB entry mapping the pcpu region for 567 * the BSP at 'pcpup'. Up until this point we were operating 568 * with the 'pcpup' for the BSP pointing to a virtual address 569 * in KSEG0 so there was no need for a TLB mapping. 570 */ 571 mips_pcpu_tlb_init(PCPU_ADDR(0)); 572 573 if (bootverbose) 574 printf("pcpu is available at virtual address %p.\n", pcpup); 575#endif 576 577 if (need_local_mappings) 578 pmap_alloc_lmem_map(); 579 pmap_create_kernel_pagetable(); 580 pmap_max_asid = VMNUM_PIDS; 581 mips_wr_entryhi(0); 582 mips_wr_pagemask(0); 583} 584 585/* 586 * Initialize a vm_page's machine-dependent fields. 587 */ 588void 589pmap_page_init(vm_page_t m) 590{ 591 592 TAILQ_INIT(&m->md.pv_list); 593 m->md.pv_list_count = 0; 594 m->md.pv_flags = 0; 595} 596 597/* 598 * Initialize the pmap module. 599 * Called by vm_init, to initialize any structures that the pmap 600 * system needs to map virtual memory. 601 * pmap_init has been enhanced to support in a fairly consistant 602 * way, discontiguous physical memory. 603 */ 604void 605pmap_init(void) 606{ 607 608 /* 609 * Initialize the address space (zone) for the pv entries. Set a 610 * high water mark so that the system can recover from excessive 611 * numbers of pv entries. 612 */ 613 pvzone = uma_zcreate("PV ENTRY", sizeof(struct pv_entry), NULL, NULL, 614 NULL, NULL, UMA_ALIGN_PTR, UMA_ZONE_VM | UMA_ZONE_NOFREE); 615 pv_entry_max = PMAP_SHPGPERPROC * maxproc + cnt.v_page_count; 616 pv_entry_high_water = 9 * (pv_entry_max / 10); 617 uma_zone_set_obj(pvzone, &pvzone_obj, pv_entry_max); 618} 619 620/*************************************************** 621 * Low level helper routines..... 622 ***************************************************/ 623 624#ifdef SMP 625static __inline void 626pmap_call_on_active_cpus(pmap_t pmap, void (*fn)(void *), void *arg) 627{ 628 int cpuid, cpu, self; 629 cpuset_t active_cpus; 630 631 sched_pin(); 632 if (is_kernel_pmap(pmap)) { 633 smp_rendezvous(NULL, fn, NULL, arg); 634 goto out; 635 } 636 /* Force ASID update on inactive CPUs */ 637 CPU_FOREACH(cpu) { 638 if (!CPU_ISSET(cpu, &pmap->pm_active)) 639 pmap->pm_asid[cpu].gen = 0; 640 } 641 cpuid = PCPU_GET(cpuid); 642 /* 643 * XXX: barrier/locking for active? 644 * 645 * Take a snapshot of active here, any further changes are ignored. 646 * tlb update/invalidate should be harmless on inactive CPUs 647 */ 648 active_cpus = pmap->pm_active; 649 self = CPU_ISSET(cpuid, &active_cpus); 650 CPU_CLR(cpuid, &active_cpus); 651 /* Optimize for the case where this cpu is the only active one */ 652 if (CPU_EMPTY(&active_cpus)) { 653 if (self) 654 fn(arg); 655 } else { 656 if (self) 657 CPU_SET(cpuid, &active_cpus); 658 smp_rendezvous_cpus(active_cpus, NULL, fn, NULL, arg); 659 } 660out: 661 sched_unpin(); 662} 663#else /* !SMP */ 664static __inline void 665pmap_call_on_active_cpus(pmap_t pmap, void (*fn)(void *), void *arg) 666{ 667 int cpuid; 668 669 if (is_kernel_pmap(pmap)) { 670 fn(arg); 671 return; 672 } 673 cpuid = PCPU_GET(cpuid); 674 if (!CPU_ISSET(cpuid, &pmap->pm_active)) 675 pmap->pm_asid[cpuid].gen = 0; 676 else 677 fn(arg); 678} 679#endif /* SMP */ 680 681static void 682pmap_invalidate_all(pmap_t pmap) 683{ 684 685 pmap_call_on_active_cpus(pmap, 686 (void (*)(void *))tlb_invalidate_all_user, pmap); 687} 688 689struct pmap_invalidate_page_arg { 690 pmap_t pmap; 691 vm_offset_t va; 692}; 693 694static void 695pmap_invalidate_page_action(void *arg) 696{ 697 struct pmap_invalidate_page_arg *p = arg; 698 699 tlb_invalidate_address(p->pmap, p->va); 700} 701 702static void 703pmap_invalidate_page(pmap_t pmap, vm_offset_t va) 704{ 705 struct pmap_invalidate_page_arg arg; 706 707 arg.pmap = pmap; 708 arg.va = va; 709 pmap_call_on_active_cpus(pmap, pmap_invalidate_page_action, &arg); 710} 711 712struct pmap_update_page_arg { 713 pmap_t pmap; 714 vm_offset_t va; 715 pt_entry_t pte; 716}; 717 718static void 719pmap_update_page_action(void *arg) 720{ 721 struct pmap_update_page_arg *p = arg; 722 723 tlb_update(p->pmap, p->va, p->pte); 724} 725 726static void 727pmap_update_page(pmap_t pmap, vm_offset_t va, pt_entry_t pte) 728{ 729 struct pmap_update_page_arg arg; 730 731 arg.pmap = pmap; 732 arg.va = va; 733 arg.pte = pte; 734 pmap_call_on_active_cpus(pmap, pmap_update_page_action, &arg); 735} 736 737/* 738 * Routine: pmap_extract 739 * Function: 740 * Extract the physical page address associated 741 * with the given map/virtual_address pair. 742 */ 743vm_paddr_t 744pmap_extract(pmap_t pmap, vm_offset_t va) 745{ 746 pt_entry_t *pte; 747 vm_offset_t retval = 0; 748 749 PMAP_LOCK(pmap); 750 pte = pmap_pte(pmap, va); 751 if (pte) { 752 retval = TLBLO_PTE_TO_PA(*pte) | (va & PAGE_MASK); 753 } 754 PMAP_UNLOCK(pmap); 755 return (retval); 756} 757 758/* 759 * Routine: pmap_extract_and_hold 760 * Function: 761 * Atomically extract and hold the physical page 762 * with the given pmap and virtual address pair 763 * if that mapping permits the given protection. 764 */ 765vm_page_t 766pmap_extract_and_hold(pmap_t pmap, vm_offset_t va, vm_prot_t prot) 767{ 768 pt_entry_t *ptep; 769 pt_entry_t pte; 770 vm_page_t m; 771 vm_paddr_t pa; 772 773 m = NULL; 774 pa = 0; 775 PMAP_LOCK(pmap); 776retry: 777 ptep = pmap_pte(pmap, va); 778 if ((ptep != NULL) && ((pte = *ptep) != 0) && 779 pte_test(&pte, PTE_V) && 780 (pte_test(&pte, PTE_D) || (prot & VM_PROT_WRITE) == 0)) { 781 if (vm_page_pa_tryrelock(pmap, TLBLO_PTE_TO_PA(pte), &pa)) 782 goto retry; 783 784 m = PHYS_TO_VM_PAGE(TLBLO_PTE_TO_PA(pte)); 785 vm_page_hold(m); 786 } 787 PA_UNLOCK_COND(pa); 788 PMAP_UNLOCK(pmap); 789 return (m); 790} 791 792/*************************************************** 793 * Low level mapping routines..... 794 ***************************************************/ 795 796/* 797 * add a wired page to the kva 798 */ 799void 800pmap_kenter_attr(vm_offset_t va, vm_paddr_t pa, int attr) 801{ 802 pt_entry_t *pte; 803 pt_entry_t opte, npte; 804 805#ifdef PMAP_DEBUG 806 printf("pmap_kenter: va: %p -> pa: %p\n", (void *)va, (void *)pa); 807#endif 808 npte = TLBLO_PA_TO_PFN(pa) | PTE_D | PTE_V | PTE_G | PTE_W | attr; 809 810 pte = pmap_pte(kernel_pmap, va); 811 opte = *pte; 812 *pte = npte; 813 if (pte_test(&opte, PTE_V) && opte != npte) 814 pmap_update_page(kernel_pmap, va, npte); 815} 816 817void 818pmap_kenter(vm_offset_t va, vm_paddr_t pa) 819{ 820 821 KASSERT(is_cacheable_mem(pa), 822 ("pmap_kenter: memory at 0x%lx is not cacheable", (u_long)pa)); 823 824 pmap_kenter_attr(va, pa, PTE_C_CACHE); 825} 826 827/* 828 * remove a page from the kernel pagetables 829 */ 830 /* PMAP_INLINE */ void 831pmap_kremove(vm_offset_t va) 832{ 833 pt_entry_t *pte; 834 835 /* 836 * Write back all caches from the page being destroyed 837 */ 838 mips_dcache_wbinv_range_index(va, PAGE_SIZE); 839 840 pte = pmap_pte(kernel_pmap, va); 841 *pte = PTE_G; 842 pmap_invalidate_page(kernel_pmap, va); 843} 844 845/* 846 * Used to map a range of physical addresses into kernel 847 * virtual address space. 848 * 849 * The value passed in '*virt' is a suggested virtual address for 850 * the mapping. Architectures which can support a direct-mapped 851 * physical to virtual region can return the appropriate address 852 * within that region, leaving '*virt' unchanged. Other 853 * architectures should map the pages starting at '*virt' and 854 * update '*virt' with the first usable address after the mapped 855 * region. 856 * 857 * Use XKPHYS for 64 bit, and KSEG0 where possible for 32 bit. 858 */ 859vm_offset_t 860pmap_map(vm_offset_t *virt, vm_paddr_t start, vm_paddr_t end, int prot) 861{ 862 vm_offset_t va, sva; 863 864 if (MIPS_DIRECT_MAPPABLE(end - 1)) 865 return (MIPS_PHYS_TO_DIRECT(start)); 866 867 va = sva = *virt; 868 while (start < end) { 869 pmap_kenter(va, start); 870 va += PAGE_SIZE; 871 start += PAGE_SIZE; 872 } 873 *virt = va; 874 return (sva); 875} 876 877/* 878 * Add a list of wired pages to the kva 879 * this routine is only used for temporary 880 * kernel mappings that do not need to have 881 * page modification or references recorded. 882 * Note that old mappings are simply written 883 * over. The page *must* be wired. 884 */ 885void 886pmap_qenter(vm_offset_t va, vm_page_t *m, int count) 887{ 888 int i; 889 vm_offset_t origva = va; 890 891 for (i = 0; i < count; i++) { 892 pmap_flush_pvcache(m[i]); 893 pmap_kenter(va, VM_PAGE_TO_PHYS(m[i])); 894 va += PAGE_SIZE; 895 } 896 897 mips_dcache_wbinv_range_index(origva, PAGE_SIZE*count); 898} 899 900/* 901 * this routine jerks page mappings from the 902 * kernel -- it is meant only for temporary mappings. 903 */ 904void 905pmap_qremove(vm_offset_t va, int count) 906{ 907 /* 908 * No need to wb/inv caches here, 909 * pmap_kremove will do it for us 910 */ 911 912 while (count-- > 0) { 913 pmap_kremove(va); 914 va += PAGE_SIZE; 915 } 916} 917 918/*************************************************** 919 * Page table page management routines..... 920 ***************************************************/ 921 922/* Revision 1.507 923 * 924 * Simplify the reference counting of page table pages. Specifically, use 925 * the page table page's wired count rather than its hold count to contain 926 * the reference count. 927 */ 928 929/* 930 * This routine unholds page table pages, and if the hold count 931 * drops to zero, then it decrements the wire count. 932 */ 933static PMAP_INLINE int 934pmap_unwire_pte_hold(pmap_t pmap, vm_offset_t va, vm_page_t m) 935{ 936 --m->wire_count; 937 if (m->wire_count == 0) 938 return (_pmap_unwire_pte_hold(pmap, va, m)); 939 else 940 return (0); 941} 942 943static int 944_pmap_unwire_pte_hold(pmap_t pmap, vm_offset_t va, vm_page_t m) 945{ 946 pd_entry_t *pde; 947 948 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 949 /* 950 * unmap the page table page 951 */ 952#ifdef __mips_n64 953 if (m->pindex < NUPDE) 954 pde = pmap_pde(pmap, va); 955 else 956 pde = pmap_segmap(pmap, va); 957#else 958 pde = pmap_pde(pmap, va); 959#endif 960 *pde = 0; 961 pmap->pm_stats.resident_count--; 962 963#ifdef __mips_n64 964 if (m->pindex < NUPDE) { 965 pd_entry_t *pdp; 966 vm_page_t pdpg; 967 968 /* 969 * Recursively decrement next level pagetable refcount 970 */ 971 pdp = (pd_entry_t *)*pmap_segmap(pmap, va); 972 pdpg = PHYS_TO_VM_PAGE(MIPS_DIRECT_TO_PHYS(pdp)); 973 pmap_unwire_pte_hold(pmap, va, pdpg); 974 } 975#endif 976 if (pmap->pm_ptphint == m) 977 pmap->pm_ptphint = NULL; 978 979 /* 980 * If the page is finally unwired, simply free it. 981 */ 982 vm_page_free_zero(m); 983 atomic_subtract_int(&cnt.v_wire_count, 1); 984 return (1); 985} 986 987/* 988 * After removing a page table entry, this routine is used to 989 * conditionally free the page, and manage the hold/wire counts. 990 */ 991static int 992pmap_unuse_pt(pmap_t pmap, vm_offset_t va, vm_page_t mpte) 993{ 994 unsigned ptepindex; 995 pd_entry_t pteva; 996 997 if (va >= VM_MAXUSER_ADDRESS) 998 return (0); 999 1000 if (mpte == NULL) { 1001 ptepindex = pmap_pde_pindex(va); 1002 if (pmap->pm_ptphint && 1003 (pmap->pm_ptphint->pindex == ptepindex)) { 1004 mpte = pmap->pm_ptphint; 1005 } else { 1006 pteva = *pmap_pde(pmap, va); 1007 mpte = PHYS_TO_VM_PAGE(MIPS_DIRECT_TO_PHYS(pteva)); 1008 pmap->pm_ptphint = mpte; 1009 } 1010 } 1011 return (pmap_unwire_pte_hold(pmap, va, mpte)); 1012} 1013 1014void 1015pmap_pinit0(pmap_t pmap) 1016{ 1017 int i; 1018 1019 PMAP_LOCK_INIT(pmap); 1020 pmap->pm_segtab = kernel_segmap; 1021 CPU_ZERO(&pmap->pm_active); 1022 pmap->pm_ptphint = NULL; 1023 for (i = 0; i < MAXCPU; i++) { 1024 pmap->pm_asid[i].asid = PMAP_ASID_RESERVED; 1025 pmap->pm_asid[i].gen = 0; 1026 } 1027 PCPU_SET(curpmap, pmap); 1028 TAILQ_INIT(&pmap->pm_pvlist); 1029 bzero(&pmap->pm_stats, sizeof pmap->pm_stats); 1030} 1031 1032void 1033pmap_grow_direct_page_cache() 1034{ 1035 1036#ifdef __mips_n64 1037 vm_pageout_grow_cache(3, 0, MIPS_XKPHYS_LARGEST_PHYS); 1038#else 1039 vm_pageout_grow_cache(3, 0, MIPS_KSEG0_LARGEST_PHYS); 1040#endif 1041} 1042 1043vm_page_t 1044pmap_alloc_direct_page(unsigned int index, int req) 1045{ 1046 vm_page_t m; 1047 1048 m = vm_page_alloc_freelist(VM_FREELIST_DIRECT, req | VM_ALLOC_WIRED | 1049 VM_ALLOC_ZERO); 1050 if (m == NULL) 1051 return (NULL); 1052 1053 if ((m->flags & PG_ZERO) == 0) 1054 pmap_zero_page(m); 1055 1056 m->pindex = index; 1057 return (m); 1058} 1059 1060/* 1061 * Initialize a preallocated and zeroed pmap structure, 1062 * such as one in a vmspace structure. 1063 */ 1064int 1065pmap_pinit(pmap_t pmap) 1066{ 1067 vm_offset_t ptdva; 1068 vm_page_t ptdpg; 1069 int i; 1070 1071 PMAP_LOCK_INIT(pmap); 1072 1073 /* 1074 * allocate the page directory page 1075 */ 1076 while ((ptdpg = pmap_alloc_direct_page(NUSERPGTBLS, VM_ALLOC_NORMAL)) == NULL) 1077 pmap_grow_direct_page_cache(); 1078 1079 ptdva = MIPS_PHYS_TO_DIRECT(VM_PAGE_TO_PHYS(ptdpg)); 1080 pmap->pm_segtab = (pd_entry_t *)ptdva; 1081 CPU_ZERO(&pmap->pm_active); 1082 pmap->pm_ptphint = NULL; 1083 for (i = 0; i < MAXCPU; i++) { 1084 pmap->pm_asid[i].asid = PMAP_ASID_RESERVED; 1085 pmap->pm_asid[i].gen = 0; 1086 } 1087 TAILQ_INIT(&pmap->pm_pvlist); 1088 bzero(&pmap->pm_stats, sizeof pmap->pm_stats); 1089 1090 return (1); 1091} 1092 1093/* 1094 * this routine is called if the page table page is not 1095 * mapped correctly. 1096 */ 1097static vm_page_t 1098_pmap_allocpte(pmap_t pmap, unsigned ptepindex, int flags) 1099{ 1100 vm_offset_t pageva; 1101 vm_page_t m; 1102 1103 KASSERT((flags & (M_NOWAIT | M_WAITOK)) == M_NOWAIT || 1104 (flags & (M_NOWAIT | M_WAITOK)) == M_WAITOK, 1105 ("_pmap_allocpte: flags is neither M_NOWAIT nor M_WAITOK")); 1106 1107 /* 1108 * Find or fabricate a new pagetable page 1109 */ 1110 if ((m = pmap_alloc_direct_page(ptepindex, VM_ALLOC_NORMAL)) == NULL) { 1111 if (flags & M_WAITOK) { 1112 PMAP_UNLOCK(pmap); 1113 vm_page_unlock_queues(); 1114 pmap_grow_direct_page_cache(); 1115 vm_page_lock_queues(); 1116 PMAP_LOCK(pmap); 1117 } 1118 1119 /* 1120 * Indicate the need to retry. While waiting, the page 1121 * table page may have been allocated. 1122 */ 1123 return (NULL); 1124 } 1125 1126 /* 1127 * Map the pagetable page into the process address space, if it 1128 * isn't already there. 1129 */ 1130 pageva = MIPS_PHYS_TO_DIRECT(VM_PAGE_TO_PHYS(m)); 1131 1132#ifdef __mips_n64 1133 if (ptepindex >= NUPDE) { 1134 pmap->pm_segtab[ptepindex - NUPDE] = (pd_entry_t)pageva; 1135 } else { 1136 pd_entry_t *pdep, *pde; 1137 int segindex = ptepindex >> (SEGSHIFT - PDRSHIFT); 1138 int pdeindex = ptepindex & (NPDEPG - 1); 1139 vm_page_t pg; 1140 1141 pdep = &pmap->pm_segtab[segindex]; 1142 if (*pdep == NULL) { 1143 /* recurse for allocating page dir */ 1144 if (_pmap_allocpte(pmap, NUPDE + segindex, 1145 flags) == NULL) { 1146 /* alloc failed, release current */ 1147 --m->wire_count; 1148 atomic_subtract_int(&cnt.v_wire_count, 1); 1149 vm_page_free_zero(m); 1150 return (NULL); 1151 } 1152 } else { 1153 pg = PHYS_TO_VM_PAGE(MIPS_DIRECT_TO_PHYS(*pdep)); 1154 pg->wire_count++; 1155 } 1156 /* Next level entry */ 1157 pde = (pd_entry_t *)*pdep; 1158 pde[pdeindex] = (pd_entry_t)pageva; 1159 pmap->pm_ptphint = m; 1160 } 1161#else 1162 pmap->pm_segtab[ptepindex] = (pd_entry_t)pageva; 1163#endif 1164 pmap->pm_stats.resident_count++; 1165 1166 /* 1167 * Set the page table hint 1168 */ 1169 pmap->pm_ptphint = m; 1170 return (m); 1171} 1172 1173static vm_page_t 1174pmap_allocpte(pmap_t pmap, vm_offset_t va, int flags) 1175{ 1176 unsigned ptepindex; 1177 pd_entry_t *pde; 1178 vm_page_t m; 1179 1180 KASSERT((flags & (M_NOWAIT | M_WAITOK)) == M_NOWAIT || 1181 (flags & (M_NOWAIT | M_WAITOK)) == M_WAITOK, 1182 ("pmap_allocpte: flags is neither M_NOWAIT nor M_WAITOK")); 1183 1184 /* 1185 * Calculate pagetable page index 1186 */ 1187 ptepindex = pmap_pde_pindex(va); 1188retry: 1189 /* 1190 * Get the page directory entry 1191 */ 1192 pde = pmap_pde(pmap, va); 1193 1194 /* 1195 * If the page table page is mapped, we just increment the hold 1196 * count, and activate it. 1197 */ 1198 if (pde != NULL && *pde != NULL) { 1199 /* 1200 * In order to get the page table page, try the hint first. 1201 */ 1202 if (pmap->pm_ptphint && 1203 (pmap->pm_ptphint->pindex == ptepindex)) { 1204 m = pmap->pm_ptphint; 1205 } else { 1206 m = PHYS_TO_VM_PAGE(MIPS_DIRECT_TO_PHYS(*pde)); 1207 pmap->pm_ptphint = m; 1208 } 1209 m->wire_count++; 1210 } else { 1211 /* 1212 * Here if the pte page isn't mapped, or if it has been 1213 * deallocated. 1214 */ 1215 m = _pmap_allocpte(pmap, ptepindex, flags); 1216 if (m == NULL && (flags & M_WAITOK)) 1217 goto retry; 1218 } 1219 return (m); 1220} 1221 1222 1223/*************************************************** 1224* Pmap allocation/deallocation routines. 1225 ***************************************************/ 1226/* 1227 * Revision 1.397 1228 * - Merged pmap_release and pmap_release_free_page. When pmap_release is 1229 * called only the page directory page(s) can be left in the pmap pte 1230 * object, since all page table pages will have been freed by 1231 * pmap_remove_pages and pmap_remove. In addition, there can only be one 1232 * reference to the pmap and the page directory is wired, so the page(s) 1233 * can never be busy. So all there is to do is clear the magic mappings 1234 * from the page directory and free the page(s). 1235 */ 1236 1237 1238/* 1239 * Release any resources held by the given physical map. 1240 * Called when a pmap initialized by pmap_pinit is being released. 1241 * Should only be called if the map contains no valid mappings. 1242 */ 1243void 1244pmap_release(pmap_t pmap) 1245{ 1246 vm_offset_t ptdva; 1247 vm_page_t ptdpg; 1248 1249 KASSERT(pmap->pm_stats.resident_count == 0, 1250 ("pmap_release: pmap resident count %ld != 0", 1251 pmap->pm_stats.resident_count)); 1252 1253 ptdva = (vm_offset_t)pmap->pm_segtab; 1254 ptdpg = PHYS_TO_VM_PAGE(MIPS_DIRECT_TO_PHYS(ptdva)); 1255 1256 ptdpg->wire_count--; 1257 atomic_subtract_int(&cnt.v_wire_count, 1); 1258 vm_page_free_zero(ptdpg); 1259 PMAP_LOCK_DESTROY(pmap); 1260} 1261 1262/* 1263 * grow the number of kernel page table entries, if needed 1264 */ 1265void 1266pmap_growkernel(vm_offset_t addr) 1267{ 1268 vm_page_t nkpg; 1269 pd_entry_t *pde, *pdpe; 1270 pt_entry_t *pte; 1271 int i; 1272 1273 mtx_assert(&kernel_map->system_mtx, MA_OWNED); 1274 addr = roundup2(addr, NBSEG); 1275 if (addr - 1 >= kernel_map->max_offset) 1276 addr = kernel_map->max_offset; 1277 while (kernel_vm_end < addr) { 1278 pdpe = pmap_segmap(kernel_pmap, kernel_vm_end); 1279#ifdef __mips_n64 1280 if (*pdpe == 0) { 1281 /* new intermediate page table entry */ 1282 nkpg = pmap_alloc_direct_page(nkpt, VM_ALLOC_INTERRUPT); 1283 if (nkpg == NULL) 1284 panic("pmap_growkernel: no memory to grow kernel"); 1285 *pdpe = (pd_entry_t)MIPS_PHYS_TO_DIRECT(VM_PAGE_TO_PHYS(nkpg)); 1286 continue; /* try again */ 1287 } 1288#endif 1289 pde = pmap_pdpe_to_pde(pdpe, kernel_vm_end); 1290 if (*pde != 0) { 1291 kernel_vm_end = (kernel_vm_end + NBPDR) & ~PDRMASK; 1292 if (kernel_vm_end - 1 >= kernel_map->max_offset) { 1293 kernel_vm_end = kernel_map->max_offset; 1294 break; 1295 } 1296 continue; 1297 } 1298 1299 /* 1300 * This index is bogus, but out of the way 1301 */ 1302 nkpg = pmap_alloc_direct_page(nkpt, VM_ALLOC_INTERRUPT); 1303 if (!nkpg) 1304 panic("pmap_growkernel: no memory to grow kernel"); 1305 nkpt++; 1306 *pde = (pd_entry_t)MIPS_PHYS_TO_DIRECT(VM_PAGE_TO_PHYS(nkpg)); 1307 1308 /* 1309 * The R[4-7]?00 stores only one copy of the Global bit in 1310 * the translation lookaside buffer for each 2 page entry. 1311 * Thus invalid entrys must have the Global bit set so when 1312 * Entry LO and Entry HI G bits are anded together they will 1313 * produce a global bit to store in the tlb. 1314 */ 1315 pte = (pt_entry_t *)*pde; 1316 for (i = 0; i < NPTEPG; i++) 1317 pte[i] = PTE_G; 1318 1319 kernel_vm_end = (kernel_vm_end + NBPDR) & ~PDRMASK; 1320 if (kernel_vm_end - 1 >= kernel_map->max_offset) { 1321 kernel_vm_end = kernel_map->max_offset; 1322 break; 1323 } 1324 } 1325} 1326 1327/*************************************************** 1328* page management routines. 1329 ***************************************************/ 1330 1331/* 1332 * free the pv_entry back to the free list 1333 */ 1334static PMAP_INLINE void 1335free_pv_entry(pv_entry_t pv) 1336{ 1337 1338 pv_entry_count--; 1339 uma_zfree(pvzone, pv); 1340} 1341 1342/* 1343 * get a new pv_entry, allocating a block from the system 1344 * when needed. 1345 * the memory allocation is performed bypassing the malloc code 1346 * because of the possibility of allocations at interrupt time. 1347 */ 1348static pv_entry_t 1349get_pv_entry(pmap_t locked_pmap) 1350{ 1351 static const struct timeval printinterval = { 60, 0 }; 1352 static struct timeval lastprint; 1353 struct vpgqueues *vpq; 1354 pt_entry_t *pte, oldpte; 1355 pmap_t pmap; 1356 pv_entry_t allocated_pv, next_pv, pv; 1357 vm_offset_t va; 1358 vm_page_t m; 1359 1360 PMAP_LOCK_ASSERT(locked_pmap, MA_OWNED); 1361 mtx_assert(&vm_page_queue_mtx, MA_OWNED); 1362 allocated_pv = uma_zalloc(pvzone, M_NOWAIT); 1363 if (allocated_pv != NULL) { 1364 pv_entry_count++; 1365 if (pv_entry_count > pv_entry_high_water) 1366 pagedaemon_wakeup(); 1367 else 1368 return (allocated_pv); 1369 } 1370 /* 1371 * Reclaim pv entries: At first, destroy mappings to inactive 1372 * pages. After that, if a pv entry is still needed, destroy 1373 * mappings to active pages. 1374 */ 1375 if (ratecheck(&lastprint, &printinterval)) 1376 printf("Approaching the limit on PV entries, " 1377 "increase the vm.pmap.shpgperproc tunable.\n"); 1378 vpq = &vm_page_queues[PQ_INACTIVE]; 1379retry: 1380 TAILQ_FOREACH(m, &vpq->pl, pageq) { 1381 if ((m->flags & PG_MARKER) != 0 || m->hold_count || m->busy) 1382 continue; 1383 TAILQ_FOREACH_SAFE(pv, &m->md.pv_list, pv_list, next_pv) { 1384 va = pv->pv_va; 1385 pmap = pv->pv_pmap; 1386 /* Avoid deadlock and lock recursion. */ 1387 if (pmap > locked_pmap) 1388 PMAP_LOCK(pmap); 1389 else if (pmap != locked_pmap && !PMAP_TRYLOCK(pmap)) 1390 continue; 1391 pmap->pm_stats.resident_count--; 1392 pte = pmap_pte(pmap, va); 1393 KASSERT(pte != NULL, ("pte")); 1394 oldpte = *pte; 1395 if (is_kernel_pmap(pmap)) 1396 *pte = PTE_G; 1397 else 1398 *pte = 0; 1399 KASSERT(!pte_test(&oldpte, PTE_W), 1400 ("wired pte for unwired page")); 1401 if (m->md.pv_flags & PV_TABLE_REF) 1402 vm_page_aflag_set(m, PGA_REFERENCED); 1403 if (pte_test(&oldpte, PTE_D)) 1404 vm_page_dirty(m); 1405 pmap_invalidate_page(pmap, va); 1406 TAILQ_REMOVE(&pmap->pm_pvlist, pv, pv_plist); 1407 m->md.pv_list_count--; 1408 TAILQ_REMOVE(&m->md.pv_list, pv, pv_list); 1409 pmap_unuse_pt(pmap, va, pv->pv_ptem); 1410 if (pmap != locked_pmap) 1411 PMAP_UNLOCK(pmap); 1412 if (allocated_pv == NULL) 1413 allocated_pv = pv; 1414 else 1415 free_pv_entry(pv); 1416 } 1417 if (TAILQ_EMPTY(&m->md.pv_list)) { 1418 vm_page_aflag_clear(m, PGA_WRITEABLE); 1419 m->md.pv_flags &= ~(PV_TABLE_REF | PV_TABLE_MOD); 1420 } 1421 } 1422 if (allocated_pv == NULL) { 1423 if (vpq == &vm_page_queues[PQ_INACTIVE]) { 1424 vpq = &vm_page_queues[PQ_ACTIVE]; 1425 goto retry; 1426 } 1427 panic("get_pv_entry: increase the vm.pmap.shpgperproc tunable"); 1428 } 1429 return (allocated_pv); 1430} 1431 1432/* 1433 * Revision 1.370 1434 * 1435 * Move pmap_collect() out of the machine-dependent code, rename it 1436 * to reflect its new location, and add page queue and flag locking. 1437 * 1438 * Notes: (1) alpha, i386, and ia64 had identical implementations 1439 * of pmap_collect() in terms of machine-independent interfaces; 1440 * (2) sparc64 doesn't require it; (3) powerpc had it as a TODO. 1441 * 1442 * MIPS implementation was identical to alpha [Junos 8.2] 1443 */ 1444 1445/* 1446 * If it is the first entry on the list, it is actually 1447 * in the header and we must copy the following entry up 1448 * to the header. Otherwise we must search the list for 1449 * the entry. In either case we free the now unused entry. 1450 */ 1451 1452static pv_entry_t 1453pmap_pvh_remove(struct md_page *pvh, pmap_t pmap, vm_offset_t va) 1454{ 1455 pv_entry_t pv; 1456 1457 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 1458 mtx_assert(&vm_page_queue_mtx, MA_OWNED); 1459 if (pvh->pv_list_count < pmap->pm_stats.resident_count) { 1460 TAILQ_FOREACH(pv, &pvh->pv_list, pv_list) { 1461 if (pmap == pv->pv_pmap && va == pv->pv_va) 1462 break; 1463 } 1464 } else { 1465 TAILQ_FOREACH(pv, &pmap->pm_pvlist, pv_plist) { 1466 if (va == pv->pv_va) 1467 break; 1468 } 1469 } 1470 if (pv != NULL) { 1471 TAILQ_REMOVE(&pvh->pv_list, pv, pv_list); 1472 pvh->pv_list_count--; 1473 TAILQ_REMOVE(&pmap->pm_pvlist, pv, pv_plist); 1474 } 1475 return (pv); 1476} 1477 1478static void 1479pmap_pvh_free(struct md_page *pvh, pmap_t pmap, vm_offset_t va) 1480{ 1481 pv_entry_t pv; 1482 1483 pv = pmap_pvh_remove(pvh, pmap, va); 1484 KASSERT(pv != NULL, ("pmap_pvh_free: pv not found, pa %lx va %lx", 1485 (u_long)VM_PAGE_TO_PHYS(member2struct(vm_page, md, pvh)), 1486 (u_long)va)); 1487 free_pv_entry(pv); 1488} 1489 1490static void 1491pmap_remove_entry(pmap_t pmap, vm_page_t m, vm_offset_t va) 1492{ 1493 1494 mtx_assert(&vm_page_queue_mtx, MA_OWNED); 1495 pmap_pvh_free(&m->md, pmap, va); 1496 if (TAILQ_EMPTY(&m->md.pv_list)) 1497 vm_page_aflag_clear(m, PGA_WRITEABLE); 1498} 1499 1500/* 1501 * Conditionally create a pv entry. 1502 */ 1503static boolean_t 1504pmap_try_insert_pv_entry(pmap_t pmap, vm_page_t mpte, vm_offset_t va, 1505 vm_page_t m) 1506{ 1507 pv_entry_t pv; 1508 1509 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 1510 mtx_assert(&vm_page_queue_mtx, MA_OWNED); 1511 if (pv_entry_count < pv_entry_high_water && 1512 (pv = uma_zalloc(pvzone, M_NOWAIT)) != NULL) { 1513 pv_entry_count++; 1514 pv->pv_va = va; 1515 pv->pv_pmap = pmap; 1516 pv->pv_ptem = mpte; 1517 TAILQ_INSERT_TAIL(&pmap->pm_pvlist, pv, pv_plist); 1518 TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_list); 1519 m->md.pv_list_count++; 1520 return (TRUE); 1521 } else 1522 return (FALSE); 1523} 1524 1525/* 1526 * pmap_remove_pte: do the things to unmap a page in a process 1527 */ 1528static int 1529pmap_remove_pte(struct pmap *pmap, pt_entry_t *ptq, vm_offset_t va) 1530{ 1531 pt_entry_t oldpte; 1532 vm_page_t m; 1533 vm_paddr_t pa; 1534 1535 mtx_assert(&vm_page_queue_mtx, MA_OWNED); 1536 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 1537 1538 oldpte = *ptq; 1539 if (is_kernel_pmap(pmap)) 1540 *ptq = PTE_G; 1541 else 1542 *ptq = 0; 1543 1544 if (pte_test(&oldpte, PTE_W)) 1545 pmap->pm_stats.wired_count -= 1; 1546 1547 pmap->pm_stats.resident_count -= 1; 1548 pa = TLBLO_PTE_TO_PA(oldpte); 1549 1550 if (page_is_managed(pa)) { 1551 m = PHYS_TO_VM_PAGE(pa); 1552 if (pte_test(&oldpte, PTE_D)) { 1553 KASSERT(!pte_test(&oldpte, PTE_RO), 1554 ("%s: modified page not writable: va: %p, pte: %#jx", 1555 __func__, (void *)va, (uintmax_t)oldpte)); 1556 vm_page_dirty(m); 1557 } 1558 if (m->md.pv_flags & PV_TABLE_REF) 1559 vm_page_aflag_set(m, PGA_REFERENCED); 1560 m->md.pv_flags &= ~(PV_TABLE_REF | PV_TABLE_MOD); 1561 1562 pmap_remove_entry(pmap, m, va); 1563 } 1564 return (pmap_unuse_pt(pmap, va, NULL)); 1565} 1566 1567/* 1568 * Remove a single page from a process address space 1569 */ 1570static void 1571pmap_remove_page(struct pmap *pmap, vm_offset_t va) 1572{ 1573 pt_entry_t *ptq; 1574 1575 mtx_assert(&vm_page_queue_mtx, MA_OWNED); 1576 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 1577 ptq = pmap_pte(pmap, va); 1578 1579 /* 1580 * if there is no pte for this address, just skip it!!! 1581 */ 1582 if (!ptq || !pte_test(ptq, PTE_V)) { 1583 return; 1584 } 1585 1586 /* 1587 * Write back all caches from the page being destroyed 1588 */ 1589 mips_dcache_wbinv_range_index(va, PAGE_SIZE); 1590 1591 /* 1592 * get a local va for mappings for this pmap. 1593 */ 1594 (void)pmap_remove_pte(pmap, ptq, va); 1595 pmap_invalidate_page(pmap, va); 1596 1597 return; 1598} 1599 1600/* 1601 * Remove the given range of addresses from the specified map. 1602 * 1603 * It is assumed that the start and end are properly 1604 * rounded to the page size. 1605 */ 1606void 1607pmap_remove(struct pmap *pmap, vm_offset_t sva, vm_offset_t eva) 1608{ 1609 vm_offset_t va_next; 1610 pd_entry_t *pde, *pdpe; 1611 pt_entry_t *pte; 1612 1613 if (pmap == NULL) 1614 return; 1615 1616 if (pmap->pm_stats.resident_count == 0) 1617 return; 1618 1619 vm_page_lock_queues(); 1620 PMAP_LOCK(pmap); 1621 1622 /* 1623 * special handling of removing one page. a very common operation 1624 * and easy to short circuit some code. 1625 */ 1626 if ((sva + PAGE_SIZE) == eva) { 1627 pmap_remove_page(pmap, sva); 1628 goto out; 1629 } 1630 for (; sva < eva; sva = va_next) { 1631 pdpe = pmap_segmap(pmap, sva); 1632#ifdef __mips_n64 1633 if (*pdpe == 0) { 1634 va_next = (sva + NBSEG) & ~SEGMASK; 1635 if (va_next < sva) 1636 va_next = eva; 1637 continue; 1638 } 1639#endif 1640 va_next = (sva + NBPDR) & ~PDRMASK; 1641 if (va_next < sva) 1642 va_next = eva; 1643 1644 pde = pmap_pdpe_to_pde(pdpe, sva); 1645 if (*pde == 0) 1646 continue; 1647 if (va_next > eva) 1648 va_next = eva; 1649 for (pte = pmap_pde_to_pte(pde, sva); sva != va_next; 1650 pte++, sva += PAGE_SIZE) { 1651 pmap_remove_page(pmap, sva); 1652 } 1653 } 1654out: 1655 vm_page_unlock_queues(); 1656 PMAP_UNLOCK(pmap); 1657} 1658 1659/* 1660 * Routine: pmap_remove_all 1661 * Function: 1662 * Removes this physical page from 1663 * all physical maps in which it resides. 1664 * Reflects back modify bits to the pager. 1665 * 1666 * Notes: 1667 * Original versions of this routine were very 1668 * inefficient because they iteratively called 1669 * pmap_remove (slow...) 1670 */ 1671 1672void 1673pmap_remove_all(vm_page_t m) 1674{ 1675 pv_entry_t pv; 1676 pt_entry_t *pte, tpte; 1677 1678 KASSERT((m->oflags & VPO_UNMANAGED) == 0, 1679 ("pmap_remove_all: page %p is not managed", m)); 1680 vm_page_lock_queues(); 1681 1682 if (m->md.pv_flags & PV_TABLE_REF) 1683 vm_page_aflag_set(m, PGA_REFERENCED); 1684 1685 while ((pv = TAILQ_FIRST(&m->md.pv_list)) != NULL) { 1686 PMAP_LOCK(pv->pv_pmap); 1687 1688 /* 1689 * If it's last mapping writeback all caches from 1690 * the page being destroyed 1691 */ 1692 if (m->md.pv_list_count == 1) 1693 mips_dcache_wbinv_range_index(pv->pv_va, PAGE_SIZE); 1694 1695 pv->pv_pmap->pm_stats.resident_count--; 1696 1697 pte = pmap_pte(pv->pv_pmap, pv->pv_va); 1698 1699 tpte = *pte; 1700 if (is_kernel_pmap(pv->pv_pmap)) 1701 *pte = PTE_G; 1702 else 1703 *pte = 0; 1704 1705 if (pte_test(&tpte, PTE_W)) 1706 pv->pv_pmap->pm_stats.wired_count--; 1707 1708 /* 1709 * Update the vm_page_t clean and reference bits. 1710 */ 1711 if (pte_test(&tpte, PTE_D)) { 1712 KASSERT(!pte_test(&tpte, PTE_RO), 1713 ("%s: modified page not writable: va: %p, pte: %#jx", 1714 __func__, (void *)pv->pv_va, (uintmax_t)tpte)); 1715 vm_page_dirty(m); 1716 } 1717 pmap_invalidate_page(pv->pv_pmap, pv->pv_va); 1718 1719 TAILQ_REMOVE(&pv->pv_pmap->pm_pvlist, pv, pv_plist); 1720 TAILQ_REMOVE(&m->md.pv_list, pv, pv_list); 1721 m->md.pv_list_count--; 1722 pmap_unuse_pt(pv->pv_pmap, pv->pv_va, pv->pv_ptem); 1723 PMAP_UNLOCK(pv->pv_pmap); 1724 free_pv_entry(pv); 1725 } 1726 1727 vm_page_aflag_clear(m, PGA_WRITEABLE); 1728 m->md.pv_flags &= ~(PV_TABLE_REF | PV_TABLE_MOD); 1729 vm_page_unlock_queues(); 1730} 1731 1732/* 1733 * Set the physical protection on the 1734 * specified range of this map as requested. 1735 */ 1736void 1737pmap_protect(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, vm_prot_t prot) 1738{ 1739 pt_entry_t *pte; 1740 pd_entry_t *pde, *pdpe; 1741 vm_offset_t va_next; 1742 1743 if (pmap == NULL) 1744 return; 1745 1746 if ((prot & VM_PROT_READ) == VM_PROT_NONE) { 1747 pmap_remove(pmap, sva, eva); 1748 return; 1749 } 1750 if (prot & VM_PROT_WRITE) 1751 return; 1752 1753 vm_page_lock_queues(); 1754 PMAP_LOCK(pmap); 1755 for (; sva < eva; sva = va_next) { 1756 pt_entry_t pbits; 1757 vm_page_t m; 1758 vm_paddr_t pa; 1759 1760 pdpe = pmap_segmap(pmap, sva); 1761#ifdef __mips_n64 1762 if (*pdpe == 0) { 1763 va_next = (sva + NBSEG) & ~SEGMASK; 1764 if (va_next < sva) 1765 va_next = eva; 1766 continue; 1767 } 1768#endif 1769 va_next = (sva + NBPDR) & ~PDRMASK; 1770 if (va_next < sva) 1771 va_next = eva; 1772 1773 pde = pmap_pdpe_to_pde(pdpe, sva); 1774 if (pde == NULL || *pde == NULL) 1775 continue; 1776 if (va_next > eva) 1777 va_next = eva; 1778 1779 for (pte = pmap_pde_to_pte(pde, sva); sva != va_next; pte++, 1780 sva += PAGE_SIZE) { 1781 1782 /* Skip invalid PTEs */ 1783 if (!pte_test(pte, PTE_V)) 1784 continue; 1785 pbits = *pte; 1786 pa = TLBLO_PTE_TO_PA(pbits); 1787 if (page_is_managed(pa) && pte_test(&pbits, PTE_D)) { 1788 m = PHYS_TO_VM_PAGE(pa); 1789 vm_page_dirty(m); 1790 m->md.pv_flags &= ~PV_TABLE_MOD; 1791 } 1792 pte_clear(&pbits, PTE_D); 1793 pte_set(&pbits, PTE_RO); 1794 1795 if (pbits != *pte) { 1796 *pte = pbits; 1797 pmap_update_page(pmap, sva, pbits); 1798 } 1799 } 1800 } 1801 vm_page_unlock_queues(); 1802 PMAP_UNLOCK(pmap); 1803} 1804 1805/* 1806 * Insert the given physical page (p) at 1807 * the specified virtual address (v) in the 1808 * target physical map with the protection requested. 1809 * 1810 * If specified, the page will be wired down, meaning 1811 * that the related pte can not be reclaimed. 1812 * 1813 * NB: This is the only routine which MAY NOT lazy-evaluate 1814 * or lose information. That is, this routine must actually 1815 * insert this page into the given map NOW. 1816 */ 1817void 1818pmap_enter(pmap_t pmap, vm_offset_t va, vm_prot_t access, vm_page_t m, 1819 vm_prot_t prot, boolean_t wired) 1820{ 1821 vm_paddr_t pa, opa; 1822 pt_entry_t *pte; 1823 pt_entry_t origpte, newpte; 1824 pv_entry_t pv; 1825 vm_page_t mpte, om; 1826 pt_entry_t rw = 0; 1827 1828 if (pmap == NULL) 1829 return; 1830 1831 va &= ~PAGE_MASK; 1832 KASSERT(va <= VM_MAX_KERNEL_ADDRESS, ("pmap_enter: toobig")); 1833 KASSERT((m->oflags & (VPO_UNMANAGED | VPO_BUSY)) != 0, 1834 ("pmap_enter: page %p is not busy", m)); 1835 1836 mpte = NULL; 1837 1838 vm_page_lock_queues(); 1839 PMAP_LOCK(pmap); 1840 1841 /* 1842 * In the case that a page table page is not resident, we are 1843 * creating it here. 1844 */ 1845 if (va < VM_MAXUSER_ADDRESS) { 1846 mpte = pmap_allocpte(pmap, va, M_WAITOK); 1847 } 1848 pte = pmap_pte(pmap, va); 1849 1850 /* 1851 * Page Directory table entry not valid, we need a new PT page 1852 */ 1853 if (pte == NULL) { 1854 panic("pmap_enter: invalid page directory, pdir=%p, va=%p", 1855 (void *)pmap->pm_segtab, (void *)va); 1856 } 1857 pa = VM_PAGE_TO_PHYS(m); 1858 om = NULL; 1859 origpte = *pte; 1860 opa = TLBLO_PTE_TO_PA(origpte); 1861 1862 /* 1863 * Mapping has not changed, must be protection or wiring change. 1864 */ 1865 if (pte_test(&origpte, PTE_V) && opa == pa) { 1866 /* 1867 * Wiring change, just update stats. We don't worry about 1868 * wiring PT pages as they remain resident as long as there 1869 * are valid mappings in them. Hence, if a user page is 1870 * wired, the PT page will be also. 1871 */ 1872 if (wired && !pte_test(&origpte, PTE_W)) 1873 pmap->pm_stats.wired_count++; 1874 else if (!wired && pte_test(&origpte, PTE_W)) 1875 pmap->pm_stats.wired_count--; 1876 1877 KASSERT(!pte_test(&origpte, PTE_D | PTE_RO), 1878 ("%s: modified page not writable: va: %p, pte: %#jx", 1879 __func__, (void *)va, (uintmax_t)origpte)); 1880 1881 /* 1882 * Remove extra pte reference 1883 */ 1884 if (mpte) 1885 mpte->wire_count--; 1886 1887 if (page_is_managed(opa)) { 1888 om = m; 1889 } 1890 goto validate; 1891 } 1892 1893 pv = NULL; 1894 1895 /* 1896 * Mapping has changed, invalidate old range and fall through to 1897 * handle validating new mapping. 1898 */ 1899 if (opa) { 1900 if (pte_test(&origpte, PTE_W)) 1901 pmap->pm_stats.wired_count--; 1902 1903 if (page_is_managed(opa)) { 1904 om = PHYS_TO_VM_PAGE(opa); 1905 pv = pmap_pvh_remove(&om->md, pmap, va); 1906 } 1907 if (mpte != NULL) { 1908 mpte->wire_count--; 1909 KASSERT(mpte->wire_count > 0, 1910 ("pmap_enter: missing reference to page table page," 1911 " va: %p", (void *)va)); 1912 } 1913 } else 1914 pmap->pm_stats.resident_count++; 1915 1916 /* 1917 * Enter on the PV list if part of our managed memory. Note that we 1918 * raise IPL while manipulating pv_table since pmap_enter can be 1919 * called at interrupt time. 1920 */ 1921 if ((m->oflags & VPO_UNMANAGED) == 0) { 1922 KASSERT(va < kmi.clean_sva || va >= kmi.clean_eva, 1923 ("pmap_enter: managed mapping within the clean submap")); 1924 if (pv == NULL) 1925 pv = get_pv_entry(pmap); 1926 pv->pv_va = va; 1927 pv->pv_pmap = pmap; 1928 pv->pv_ptem = mpte; 1929 TAILQ_INSERT_TAIL(&pmap->pm_pvlist, pv, pv_plist); 1930 TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_list); 1931 m->md.pv_list_count++; 1932 } else if (pv != NULL) 1933 free_pv_entry(pv); 1934 1935 /* 1936 * Increment counters 1937 */ 1938 if (wired) 1939 pmap->pm_stats.wired_count++; 1940 1941validate: 1942 if ((access & VM_PROT_WRITE) != 0) 1943 m->md.pv_flags |= PV_TABLE_MOD | PV_TABLE_REF; 1944 rw = init_pte_prot(va, m, prot); 1945 1946#ifdef PMAP_DEBUG 1947 printf("pmap_enter: va: %p -> pa: %p\n", (void *)va, (void *)pa); 1948#endif 1949 /* 1950 * Now validate mapping with desired protection/wiring. 1951 */ 1952 newpte = TLBLO_PA_TO_PFN(pa) | rw | PTE_V; 1953 1954 if (is_cacheable_mem(pa)) 1955 newpte |= PTE_C_CACHE; 1956 else 1957 newpte |= PTE_C_UNCACHED; 1958 1959 if (wired) 1960 newpte |= PTE_W; 1961 1962 if (is_kernel_pmap(pmap)) 1963 newpte |= PTE_G; 1964 1965 /* 1966 * if the mapping or permission bits are different, we need to 1967 * update the pte. 1968 */ 1969 if (origpte != newpte) { 1970 if (pte_test(&origpte, PTE_V)) { 1971 *pte = newpte; 1972 if (page_is_managed(opa) && (opa != pa)) { 1973 if (om->md.pv_flags & PV_TABLE_REF) 1974 vm_page_aflag_set(om, PGA_REFERENCED); 1975 om->md.pv_flags &= 1976 ~(PV_TABLE_REF | PV_TABLE_MOD); 1977 } 1978 if (pte_test(&origpte, PTE_D)) { 1979 KASSERT(!pte_test(&origpte, PTE_RO), 1980 ("pmap_enter: modified page not writable:" 1981 " va: %p, pte: %#jx", (void *)va, (uintmax_t)origpte)); 1982 if (page_is_managed(opa)) 1983 vm_page_dirty(om); 1984 } 1985 if (page_is_managed(opa) && 1986 TAILQ_EMPTY(&om->md.pv_list)) 1987 vm_page_aflag_clear(om, PGA_WRITEABLE); 1988 } else { 1989 *pte = newpte; 1990 } 1991 } 1992 pmap_update_page(pmap, va, newpte); 1993 1994 /* 1995 * Sync I & D caches for executable pages. Do this only if the 1996 * target pmap belongs to the current process. Otherwise, an 1997 * unresolvable TLB miss may occur. 1998 */ 1999 if (!is_kernel_pmap(pmap) && (pmap == &curproc->p_vmspace->vm_pmap) && 2000 (prot & VM_PROT_EXECUTE)) { 2001 mips_icache_sync_range(va, PAGE_SIZE); 2002 mips_dcache_wbinv_range(va, PAGE_SIZE); 2003 } 2004 vm_page_unlock_queues(); 2005 PMAP_UNLOCK(pmap); 2006} 2007 2008/* 2009 * this code makes some *MAJOR* assumptions: 2010 * 1. Current pmap & pmap exists. 2011 * 2. Not wired. 2012 * 3. Read access. 2013 * 4. No page table pages. 2014 * but is *MUCH* faster than pmap_enter... 2015 */ 2016 2017void 2018pmap_enter_quick(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot) 2019{ 2020 2021 vm_page_lock_queues(); 2022 PMAP_LOCK(pmap); 2023 (void)pmap_enter_quick_locked(pmap, va, m, prot, NULL); 2024 vm_page_unlock_queues(); 2025 PMAP_UNLOCK(pmap); 2026} 2027 2028static vm_page_t 2029pmap_enter_quick_locked(pmap_t pmap, vm_offset_t va, vm_page_t m, 2030 vm_prot_t prot, vm_page_t mpte) 2031{ 2032 pt_entry_t *pte; 2033 vm_paddr_t pa; 2034 2035 KASSERT(va < kmi.clean_sva || va >= kmi.clean_eva || 2036 (m->oflags & VPO_UNMANAGED) != 0, 2037 ("pmap_enter_quick_locked: managed mapping within the clean submap")); 2038 mtx_assert(&vm_page_queue_mtx, MA_OWNED); 2039 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 2040 2041 /* 2042 * In the case that a page table page is not resident, we are 2043 * creating it here. 2044 */ 2045 if (va < VM_MAXUSER_ADDRESS) { 2046 pd_entry_t *pde; 2047 unsigned ptepindex; 2048 2049 /* 2050 * Calculate pagetable page index 2051 */ 2052 ptepindex = pmap_pde_pindex(va); 2053 if (mpte && (mpte->pindex == ptepindex)) { 2054 mpte->wire_count++; 2055 } else { 2056 /* 2057 * Get the page directory entry 2058 */ 2059 pde = pmap_pde(pmap, va); 2060 2061 /* 2062 * If the page table page is mapped, we just 2063 * increment the hold count, and activate it. 2064 */ 2065 if (pde && *pde != 0) { 2066 if (pmap->pm_ptphint && 2067 (pmap->pm_ptphint->pindex == ptepindex)) { 2068 mpte = pmap->pm_ptphint; 2069 } else { 2070 mpte = PHYS_TO_VM_PAGE( 2071 MIPS_DIRECT_TO_PHYS(*pde)); 2072 pmap->pm_ptphint = mpte; 2073 } 2074 mpte->wire_count++; 2075 } else { 2076 mpte = _pmap_allocpte(pmap, ptepindex, 2077 M_NOWAIT); 2078 if (mpte == NULL) 2079 return (mpte); 2080 } 2081 } 2082 } else { 2083 mpte = NULL; 2084 } 2085 2086 pte = pmap_pte(pmap, va); 2087 if (pte_test(pte, PTE_V)) { 2088 if (mpte != NULL) { 2089 mpte->wire_count--; 2090 mpte = NULL; 2091 } 2092 return (mpte); 2093 } 2094 2095 /* 2096 * Enter on the PV list if part of our managed memory. 2097 */ 2098 if ((m->oflags & VPO_UNMANAGED) == 0 && 2099 !pmap_try_insert_pv_entry(pmap, mpte, va, m)) { 2100 if (mpte != NULL) { 2101 pmap_unwire_pte_hold(pmap, va, mpte); 2102 mpte = NULL; 2103 } 2104 return (mpte); 2105 } 2106 2107 /* 2108 * Increment counters 2109 */ 2110 pmap->pm_stats.resident_count++; 2111 2112 pa = VM_PAGE_TO_PHYS(m); 2113 2114 /* 2115 * Now validate mapping with RO protection 2116 */ 2117 *pte = TLBLO_PA_TO_PFN(pa) | PTE_V; 2118 2119 if (is_cacheable_mem(pa)) 2120 *pte |= PTE_C_CACHE; 2121 else 2122 *pte |= PTE_C_UNCACHED; 2123 2124 if (is_kernel_pmap(pmap)) 2125 *pte |= PTE_G; 2126 else { 2127 *pte |= PTE_RO; 2128 /* 2129 * Sync I & D caches. Do this only if the target pmap 2130 * belongs to the current process. Otherwise, an 2131 * unresolvable TLB miss may occur. */ 2132 if (pmap == &curproc->p_vmspace->vm_pmap) { 2133 va &= ~PAGE_MASK; 2134 mips_icache_sync_range(va, PAGE_SIZE); 2135 mips_dcache_wbinv_range(va, PAGE_SIZE); 2136 } 2137 } 2138 return (mpte); 2139} 2140 2141/* 2142 * Make a temporary mapping for a physical address. This is only intended 2143 * to be used for panic dumps. 2144 * 2145 * Use XKPHYS for 64 bit, and KSEG0 where possible for 32 bit. 2146 */ 2147void * 2148pmap_kenter_temporary(vm_paddr_t pa, int i) 2149{ 2150 vm_offset_t va; 2151 2152 if (i != 0) 2153 printf("%s: ERROR!!! More than one page of virtual address mapping not supported\n", 2154 __func__); 2155 2156 if (MIPS_DIRECT_MAPPABLE(pa)) { 2157 va = MIPS_PHYS_TO_DIRECT(pa); 2158 } else { 2159#ifndef __mips_n64 /* XXX : to be converted to new style */ 2160 int cpu; 2161 register_t intr; 2162 struct local_sysmaps *sysm; 2163 pt_entry_t *pte, npte; 2164 2165 /* If this is used other than for dumps, we may need to leave 2166 * interrupts disasbled on return. If crash dumps don't work when 2167 * we get to this point, we might want to consider this (leaving things 2168 * disabled as a starting point ;-) 2169 */ 2170 intr = intr_disable(); 2171 cpu = PCPU_GET(cpuid); 2172 sysm = &sysmap_lmem[cpu]; 2173 /* Since this is for the debugger, no locks or any other fun */ 2174 npte = TLBLO_PA_TO_PFN(pa) | PTE_D | PTE_V | PTE_G | PTE_W | PTE_C_CACHE; 2175 pte = pmap_pte(kernel_pmap, sysm->base); 2176 *pte = npte; 2177 sysm->valid1 = 1; 2178 pmap_update_page(kernel_pmap, sysm->base, npte); 2179 va = sysm->base; 2180 intr_restore(intr); 2181#endif 2182 } 2183 return ((void *)va); 2184} 2185 2186void 2187pmap_kenter_temporary_free(vm_paddr_t pa) 2188{ 2189#ifndef __mips_n64 /* XXX : to be converted to new style */ 2190 int cpu; 2191 register_t intr; 2192 struct local_sysmaps *sysm; 2193#endif 2194 2195 if (MIPS_DIRECT_MAPPABLE(pa)) { 2196 /* nothing to do for this case */ 2197 return; 2198 } 2199#ifndef __mips_n64 /* XXX : to be converted to new style */ 2200 cpu = PCPU_GET(cpuid); 2201 sysm = &sysmap_lmem[cpu]; 2202 if (sysm->valid1) { 2203 pt_entry_t *pte; 2204 2205 intr = intr_disable(); 2206 pte = pmap_pte(kernel_pmap, sysm->base); 2207 *pte = PTE_G; 2208 pmap_invalidate_page(kernel_pmap, sysm->base); 2209 intr_restore(intr); 2210 sysm->valid1 = 0; 2211 } 2212#endif 2213} 2214 2215/* 2216 * Moved the code to Machine Independent 2217 * vm_map_pmap_enter() 2218 */ 2219 2220/* 2221 * Maps a sequence of resident pages belonging to the same object. 2222 * The sequence begins with the given page m_start. This page is 2223 * mapped at the given virtual address start. Each subsequent page is 2224 * mapped at a virtual address that is offset from start by the same 2225 * amount as the page is offset from m_start within the object. The 2226 * last page in the sequence is the page with the largest offset from 2227 * m_start that can be mapped at a virtual address less than the given 2228 * virtual address end. Not every virtual page between start and end 2229 * is mapped; only those for which a resident page exists with the 2230 * corresponding offset from m_start are mapped. 2231 */ 2232void 2233pmap_enter_object(pmap_t pmap, vm_offset_t start, vm_offset_t end, 2234 vm_page_t m_start, vm_prot_t prot) 2235{ 2236 vm_page_t m, mpte; 2237 vm_pindex_t diff, psize; 2238 2239 VM_OBJECT_LOCK_ASSERT(m_start->object, MA_OWNED); 2240 psize = atop(end - start); 2241 mpte = NULL; 2242 m = m_start; 2243 vm_page_lock_queues(); 2244 PMAP_LOCK(pmap); 2245 while (m != NULL && (diff = m->pindex - m_start->pindex) < psize) { 2246 mpte = pmap_enter_quick_locked(pmap, start + ptoa(diff), m, 2247 prot, mpte); 2248 m = TAILQ_NEXT(m, listq); 2249 } 2250 vm_page_unlock_queues(); 2251 PMAP_UNLOCK(pmap); 2252} 2253 2254/* 2255 * pmap_object_init_pt preloads the ptes for a given object 2256 * into the specified pmap. This eliminates the blast of soft 2257 * faults on process startup and immediately after an mmap. 2258 */ 2259void 2260pmap_object_init_pt(pmap_t pmap, vm_offset_t addr, 2261 vm_object_t object, vm_pindex_t pindex, vm_size_t size) 2262{ 2263 VM_OBJECT_LOCK_ASSERT(object, MA_OWNED); 2264 KASSERT(object->type == OBJT_DEVICE || object->type == OBJT_SG, 2265 ("pmap_object_init_pt: non-device object")); 2266} 2267 2268/* 2269 * Routine: pmap_change_wiring 2270 * Function: Change the wiring attribute for a map/virtual-address 2271 * pair. 2272 * In/out conditions: 2273 * The mapping must already exist in the pmap. 2274 */ 2275void 2276pmap_change_wiring(pmap_t pmap, vm_offset_t va, boolean_t wired) 2277{ 2278 pt_entry_t *pte; 2279 2280 if (pmap == NULL) 2281 return; 2282 2283 PMAP_LOCK(pmap); 2284 pte = pmap_pte(pmap, va); 2285 2286 if (wired && !pte_test(pte, PTE_W)) 2287 pmap->pm_stats.wired_count++; 2288 else if (!wired && pte_test(pte, PTE_W)) 2289 pmap->pm_stats.wired_count--; 2290 2291 /* 2292 * Wiring is not a hardware characteristic so there is no need to 2293 * invalidate TLB. 2294 */ 2295 if (wired) 2296 pte_set(pte, PTE_W); 2297 else 2298 pte_clear(pte, PTE_W); 2299 PMAP_UNLOCK(pmap); 2300} 2301 2302/* 2303 * Copy the range specified by src_addr/len 2304 * from the source map to the range dst_addr/len 2305 * in the destination map. 2306 * 2307 * This routine is only advisory and need not do anything. 2308 */ 2309 2310void 2311pmap_copy(pmap_t dst_pmap, pmap_t src_pmap, vm_offset_t dst_addr, 2312 vm_size_t len, vm_offset_t src_addr) 2313{ 2314} 2315 2316/* 2317 * pmap_zero_page zeros the specified hardware page by mapping 2318 * the page into KVM and using bzero to clear its contents. 2319 * 2320 * Use XKPHYS for 64 bit, and KSEG0 where possible for 32 bit. 2321 */ 2322void 2323pmap_zero_page(vm_page_t m) 2324{ 2325 vm_offset_t va; 2326 vm_paddr_t phys = VM_PAGE_TO_PHYS(m); 2327 2328 if (MIPS_DIRECT_MAPPABLE(phys)) { 2329 va = MIPS_PHYS_TO_DIRECT(phys); 2330 bzero((caddr_t)va, PAGE_SIZE); 2331 mips_dcache_wbinv_range(va, PAGE_SIZE); 2332 } else { 2333 va = pmap_lmem_map1(phys); 2334 bzero((caddr_t)va, PAGE_SIZE); 2335 mips_dcache_wbinv_range(va, PAGE_SIZE); 2336 pmap_lmem_unmap(); 2337 } 2338} 2339 2340/* 2341 * pmap_zero_page_area zeros the specified hardware page by mapping 2342 * the page into KVM and using bzero to clear its contents. 2343 * 2344 * off and size may not cover an area beyond a single hardware page. 2345 */ 2346void 2347pmap_zero_page_area(vm_page_t m, int off, int size) 2348{ 2349 vm_offset_t va; 2350 vm_paddr_t phys = VM_PAGE_TO_PHYS(m); 2351 2352 if (MIPS_DIRECT_MAPPABLE(phys)) { 2353 va = MIPS_PHYS_TO_DIRECT(phys); 2354 bzero((char *)(caddr_t)va + off, size); 2355 mips_dcache_wbinv_range(va + off, size); 2356 } else { 2357 va = pmap_lmem_map1(phys); 2358 bzero((char *)va + off, size); 2359 mips_dcache_wbinv_range(va + off, size); 2360 pmap_lmem_unmap(); 2361 } 2362} 2363 2364void 2365pmap_zero_page_idle(vm_page_t m) 2366{ 2367 vm_offset_t va; 2368 vm_paddr_t phys = VM_PAGE_TO_PHYS(m); 2369 2370 if (MIPS_DIRECT_MAPPABLE(phys)) { 2371 va = MIPS_PHYS_TO_DIRECT(phys); 2372 bzero((caddr_t)va, PAGE_SIZE); 2373 mips_dcache_wbinv_range(va, PAGE_SIZE); 2374 } else { 2375 va = pmap_lmem_map1(phys); 2376 bzero((caddr_t)va, PAGE_SIZE); 2377 mips_dcache_wbinv_range(va, PAGE_SIZE); 2378 pmap_lmem_unmap(); 2379 } 2380} 2381 2382/* 2383 * pmap_copy_page copies the specified (machine independent) 2384 * page by mapping the page into virtual memory and using 2385 * bcopy to copy the page, one machine dependent page at a 2386 * time. 2387 * 2388 * Use XKPHYS for 64 bit, and KSEG0 where possible for 32 bit. 2389 */ 2390void 2391pmap_copy_page(vm_page_t src, vm_page_t dst) 2392{ 2393 vm_offset_t va_src, va_dst; 2394 vm_paddr_t phys_src = VM_PAGE_TO_PHYS(src); 2395 vm_paddr_t phys_dst = VM_PAGE_TO_PHYS(dst); 2396 2397 if (MIPS_DIRECT_MAPPABLE(phys_src) && MIPS_DIRECT_MAPPABLE(phys_dst)) { 2398 /* easy case, all can be accessed via KSEG0 */ 2399 /* 2400 * Flush all caches for VA that are mapped to this page 2401 * to make sure that data in SDRAM is up to date 2402 */ 2403 pmap_flush_pvcache(src); 2404 mips_dcache_wbinv_range_index( 2405 MIPS_PHYS_TO_DIRECT(phys_dst), PAGE_SIZE); 2406 va_src = MIPS_PHYS_TO_DIRECT(phys_src); 2407 va_dst = MIPS_PHYS_TO_DIRECT(phys_dst); 2408 bcopy((caddr_t)va_src, (caddr_t)va_dst, PAGE_SIZE); 2409 mips_dcache_wbinv_range(va_dst, PAGE_SIZE); 2410 } else { 2411 va_src = pmap_lmem_map2(phys_src, phys_dst); 2412 va_dst = va_src + PAGE_SIZE; 2413 bcopy((void *)va_src, (void *)va_dst, PAGE_SIZE); 2414 mips_dcache_wbinv_range(va_dst, PAGE_SIZE); 2415 pmap_lmem_unmap(); 2416 } 2417} 2418 2419/* 2420 * Returns true if the pmap's pv is one of the first 2421 * 16 pvs linked to from this page. This count may 2422 * be changed upwards or downwards in the future; it 2423 * is only necessary that true be returned for a small 2424 * subset of pmaps for proper page aging. 2425 */ 2426boolean_t 2427pmap_page_exists_quick(pmap_t pmap, vm_page_t m) 2428{ 2429 pv_entry_t pv; 2430 int loops = 0; 2431 boolean_t rv; 2432 2433 KASSERT((m->oflags & VPO_UNMANAGED) == 0, 2434 ("pmap_page_exists_quick: page %p is not managed", m)); 2435 rv = FALSE; 2436 vm_page_lock_queues(); 2437 TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) { 2438 if (pv->pv_pmap == pmap) { 2439 rv = TRUE; 2440 break; 2441 } 2442 loops++; 2443 if (loops >= 16) 2444 break; 2445 } 2446 vm_page_unlock_queues(); 2447 return (rv); 2448} 2449 2450/* 2451 * Remove all pages from specified address space 2452 * this aids process exit speeds. Also, this code 2453 * is special cased for current process only, but 2454 * can have the more generic (and slightly slower) 2455 * mode enabled. This is much faster than pmap_remove 2456 * in the case of running down an entire address space. 2457 */ 2458void 2459pmap_remove_pages(pmap_t pmap) 2460{ 2461 pt_entry_t *pte, tpte; 2462 pv_entry_t pv, npv; 2463 vm_page_t m; 2464 2465 if (pmap != vmspace_pmap(curthread->td_proc->p_vmspace)) { 2466 printf("warning: pmap_remove_pages called with non-current pmap\n"); 2467 return; 2468 } 2469 vm_page_lock_queues(); 2470 PMAP_LOCK(pmap); 2471 for (pv = TAILQ_FIRST(&pmap->pm_pvlist); pv != NULL; pv = npv) { 2472 2473 pte = pmap_pte(pv->pv_pmap, pv->pv_va); 2474 if (!pte_test(pte, PTE_V)) 2475 panic("pmap_remove_pages: page on pm_pvlist has no pte"); 2476 tpte = *pte; 2477 2478/* 2479 * We cannot remove wired pages from a process' mapping at this time 2480 */ 2481 if (pte_test(&tpte, PTE_W)) { 2482 npv = TAILQ_NEXT(pv, pv_plist); 2483 continue; 2484 } 2485 *pte = is_kernel_pmap(pmap) ? PTE_G : 0; 2486 2487 m = PHYS_TO_VM_PAGE(TLBLO_PTE_TO_PA(tpte)); 2488 KASSERT(m != NULL, 2489 ("pmap_remove_pages: bad tpte %#jx", (uintmax_t)tpte)); 2490 2491 pv->pv_pmap->pm_stats.resident_count--; 2492 2493 /* 2494 * Update the vm_page_t clean and reference bits. 2495 */ 2496 if (pte_test(&tpte, PTE_D)) { 2497 vm_page_dirty(m); 2498 } 2499 npv = TAILQ_NEXT(pv, pv_plist); 2500 TAILQ_REMOVE(&pv->pv_pmap->pm_pvlist, pv, pv_plist); 2501 2502 m->md.pv_list_count--; 2503 TAILQ_REMOVE(&m->md.pv_list, pv, pv_list); 2504 if (TAILQ_FIRST(&m->md.pv_list) == NULL) { 2505 vm_page_aflag_clear(m, PGA_WRITEABLE); 2506 } 2507 pmap_unuse_pt(pv->pv_pmap, pv->pv_va, pv->pv_ptem); 2508 free_pv_entry(pv); 2509 } 2510 pmap_invalidate_all(pmap); 2511 PMAP_UNLOCK(pmap); 2512 vm_page_unlock_queues(); 2513} 2514 2515/* 2516 * pmap_testbit tests bits in pte's 2517 * note that the testbit/changebit routines are inline, 2518 * and a lot of things compile-time evaluate. 2519 */ 2520static boolean_t 2521pmap_testbit(vm_page_t m, int bit) 2522{ 2523 pv_entry_t pv; 2524 pt_entry_t *pte; 2525 boolean_t rv = FALSE; 2526 2527 if (m->oflags & VPO_UNMANAGED) 2528 return (rv); 2529 2530 if (TAILQ_FIRST(&m->md.pv_list) == NULL) 2531 return (rv); 2532 2533 mtx_assert(&vm_page_queue_mtx, MA_OWNED); 2534 TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) { 2535 PMAP_LOCK(pv->pv_pmap); 2536 pte = pmap_pte(pv->pv_pmap, pv->pv_va); 2537 rv = pte_test(pte, bit); 2538 PMAP_UNLOCK(pv->pv_pmap); 2539 if (rv) 2540 break; 2541 } 2542 return (rv); 2543} 2544 2545/* 2546 * this routine is used to clear dirty bits in ptes 2547 */ 2548static __inline void 2549pmap_changebit(vm_page_t m, int bit, boolean_t setem) 2550{ 2551 pv_entry_t pv; 2552 pt_entry_t *pte; 2553 2554 if (m->oflags & VPO_UNMANAGED) 2555 return; 2556 2557 mtx_assert(&vm_page_queue_mtx, MA_OWNED); 2558 /* 2559 * Loop over all current mappings setting/clearing as appropos If 2560 * setting RO do we need to clear the VAC? 2561 */ 2562 TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) { 2563 PMAP_LOCK(pv->pv_pmap); 2564 pte = pmap_pte(pv->pv_pmap, pv->pv_va); 2565 if (setem) { 2566 *pte |= bit; 2567 pmap_update_page(pv->pv_pmap, pv->pv_va, *pte); 2568 } else { 2569 pt_entry_t pbits = *pte; 2570 2571 if (pbits & bit) { 2572 if (bit == PTE_D) { 2573 if (pbits & PTE_D) 2574 vm_page_dirty(m); 2575 *pte = (pbits & ~PTE_D) | PTE_RO; 2576 } else { 2577 *pte = pbits & ~bit; 2578 } 2579 pmap_update_page(pv->pv_pmap, pv->pv_va, *pte); 2580 } 2581 } 2582 PMAP_UNLOCK(pv->pv_pmap); 2583 } 2584 if (!setem && bit == PTE_D) 2585 vm_page_aflag_clear(m, PGA_WRITEABLE); 2586} 2587 2588/* 2589 * pmap_page_wired_mappings: 2590 * 2591 * Return the number of managed mappings to the given physical page 2592 * that are wired. 2593 */ 2594int 2595pmap_page_wired_mappings(vm_page_t m) 2596{ 2597 pv_entry_t pv; 2598 pmap_t pmap; 2599 pt_entry_t *pte; 2600 int count; 2601 2602 count = 0; 2603 if ((m->oflags & VPO_UNMANAGED) != 0) 2604 return (count); 2605 vm_page_lock_queues(); 2606 TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) { 2607 pmap = pv->pv_pmap; 2608 PMAP_LOCK(pmap); 2609 pte = pmap_pte(pmap, pv->pv_va); 2610 if (pte_test(pte, PTE_W)) 2611 count++; 2612 PMAP_UNLOCK(pmap); 2613 } 2614 vm_page_unlock_queues(); 2615 return (count); 2616} 2617 2618/* 2619 * Clear the write and modified bits in each of the given page's mappings. 2620 */ 2621void 2622pmap_remove_write(vm_page_t m) 2623{ 2624 pv_entry_t pv, npv; 2625 vm_offset_t va; 2626 pt_entry_t *pte; 2627 2628 KASSERT((m->oflags & VPO_UNMANAGED) == 0, 2629 ("pmap_remove_write: page %p is not managed", m)); 2630 2631 /* 2632 * If the page is not VPO_BUSY, then PGA_WRITEABLE cannot be set by 2633 * another thread while the object is locked. Thus, if PGA_WRITEABLE 2634 * is clear, no page table entries need updating. 2635 */ 2636 VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED); 2637 if ((m->oflags & VPO_BUSY) == 0 && 2638 (m->aflags & PGA_WRITEABLE) == 0) 2639 return; 2640 2641 /* 2642 * Loop over all current mappings setting/clearing as appropos. 2643 */ 2644 vm_page_lock_queues(); 2645 for (pv = TAILQ_FIRST(&m->md.pv_list); pv; pv = npv) { 2646 npv = TAILQ_NEXT(pv, pv_plist); 2647 pte = pmap_pte(pv->pv_pmap, pv->pv_va); 2648 if (pte == NULL || !pte_test(pte, PTE_V)) 2649 panic("page on pm_pvlist has no pte"); 2650 2651 va = pv->pv_va; 2652 pmap_protect(pv->pv_pmap, va, va + PAGE_SIZE, 2653 VM_PROT_READ | VM_PROT_EXECUTE); 2654 } 2655 vm_page_aflag_clear(m, PGA_WRITEABLE); 2656 vm_page_unlock_queues(); 2657} 2658 2659/* 2660 * pmap_ts_referenced: 2661 * 2662 * Return the count of reference bits for a page, clearing all of them. 2663 */ 2664int 2665pmap_ts_referenced(vm_page_t m) 2666{ 2667 2668 KASSERT((m->oflags & VPO_UNMANAGED) == 0, 2669 ("pmap_ts_referenced: page %p is not managed", m)); 2670 if (m->md.pv_flags & PV_TABLE_REF) { 2671 vm_page_lock_queues(); 2672 m->md.pv_flags &= ~PV_TABLE_REF; 2673 vm_page_unlock_queues(); 2674 return (1); 2675 } 2676 return (0); 2677} 2678 2679/* 2680 * pmap_is_modified: 2681 * 2682 * Return whether or not the specified physical page was modified 2683 * in any physical maps. 2684 */ 2685boolean_t 2686pmap_is_modified(vm_page_t m) 2687{ 2688 boolean_t rv; 2689 2690 KASSERT((m->oflags & VPO_UNMANAGED) == 0, 2691 ("pmap_is_modified: page %p is not managed", m)); 2692 2693 /* 2694 * If the page is not VPO_BUSY, then PGA_WRITEABLE cannot be 2695 * concurrently set while the object is locked. Thus, if PGA_WRITEABLE 2696 * is clear, no PTEs can have PTE_D set. 2697 */ 2698 VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED); 2699 if ((m->oflags & VPO_BUSY) == 0 && 2700 (m->aflags & PGA_WRITEABLE) == 0) 2701 return (FALSE); 2702 vm_page_lock_queues(); 2703 if (m->md.pv_flags & PV_TABLE_MOD) 2704 rv = TRUE; 2705 else 2706 rv = pmap_testbit(m, PTE_D); 2707 vm_page_unlock_queues(); 2708 return (rv); 2709} 2710 2711/* N/C */ 2712 2713/* 2714 * pmap_is_prefaultable: 2715 * 2716 * Return whether or not the specified virtual address is elgible 2717 * for prefault. 2718 */ 2719boolean_t 2720pmap_is_prefaultable(pmap_t pmap, vm_offset_t addr) 2721{ 2722 pd_entry_t *pde; 2723 pt_entry_t *pte; 2724 boolean_t rv; 2725 2726 rv = FALSE; 2727 PMAP_LOCK(pmap); 2728 pde = pmap_pde(pmap, addr); 2729 if (pde != NULL && *pde != 0) { 2730 pte = pmap_pde_to_pte(pde, addr); 2731 rv = (*pte == 0); 2732 } 2733 PMAP_UNLOCK(pmap); 2734 return (rv); 2735} 2736 2737/* 2738 * Clear the modify bits on the specified physical page. 2739 */ 2740void 2741pmap_clear_modify(vm_page_t m) 2742{ 2743 2744 KASSERT((m->oflags & VPO_UNMANAGED) == 0, 2745 ("pmap_clear_modify: page %p is not managed", m)); 2746 VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED); 2747 KASSERT((m->oflags & VPO_BUSY) == 0, 2748 ("pmap_clear_modify: page %p is busy", m)); 2749 2750 /* 2751 * If the page is not PGA_WRITEABLE, then no PTEs can have PTE_D set. 2752 * If the object containing the page is locked and the page is not 2753 * VPO_BUSY, then PGA_WRITEABLE cannot be concurrently set. 2754 */ 2755 if ((m->aflags & PGA_WRITEABLE) == 0) 2756 return; 2757 vm_page_lock_queues(); 2758 if (m->md.pv_flags & PV_TABLE_MOD) { 2759 pmap_changebit(m, PTE_D, FALSE); 2760 m->md.pv_flags &= ~PV_TABLE_MOD; 2761 } 2762 vm_page_unlock_queues(); 2763} 2764 2765/* 2766 * pmap_is_referenced: 2767 * 2768 * Return whether or not the specified physical page was referenced 2769 * in any physical maps. 2770 */ 2771boolean_t 2772pmap_is_referenced(vm_page_t m) 2773{ 2774 2775 KASSERT((m->oflags & VPO_UNMANAGED) == 0, 2776 ("pmap_is_referenced: page %p is not managed", m)); 2777 return ((m->md.pv_flags & PV_TABLE_REF) != 0); 2778} 2779 2780/* 2781 * pmap_clear_reference: 2782 * 2783 * Clear the reference bit on the specified physical page. 2784 */ 2785void 2786pmap_clear_reference(vm_page_t m) 2787{ 2788 2789 KASSERT((m->oflags & VPO_UNMANAGED) == 0, 2790 ("pmap_clear_reference: page %p is not managed", m)); 2791 vm_page_lock_queues(); 2792 if (m->md.pv_flags & PV_TABLE_REF) { 2793 m->md.pv_flags &= ~PV_TABLE_REF; 2794 } 2795 vm_page_unlock_queues(); 2796} 2797 2798/* 2799 * Miscellaneous support routines follow 2800 */ 2801 2802/* 2803 * Map a set of physical memory pages into the kernel virtual 2804 * address space. Return a pointer to where it is mapped. This 2805 * routine is intended to be used for mapping device memory, 2806 * NOT real memory. 2807 */ 2808 2809/* 2810 * Map a set of physical memory pages into the kernel virtual 2811 * address space. Return a pointer to where it is mapped. This 2812 * routine is intended to be used for mapping device memory, 2813 * NOT real memory. 2814 * 2815 * Use XKPHYS uncached for 64 bit, and KSEG1 where possible for 32 bit. 2816 */ 2817void * 2818pmap_mapdev(vm_paddr_t pa, vm_size_t size) 2819{ 2820 vm_offset_t va, tmpva, offset; 2821 2822 /* 2823 * KSEG1 maps only first 512M of phys address space. For 2824 * pa > 0x20000000 we should make proper mapping * using pmap_kenter. 2825 */ 2826 if (MIPS_DIRECT_MAPPABLE(pa + size - 1)) 2827 return ((void *)MIPS_PHYS_TO_DIRECT_UNCACHED(pa)); 2828 else { 2829 offset = pa & PAGE_MASK; 2830 size = roundup(size + offset, PAGE_SIZE); 2831 2832 va = kmem_alloc_nofault(kernel_map, size); 2833 if (!va) 2834 panic("pmap_mapdev: Couldn't alloc kernel virtual memory"); 2835 pa = trunc_page(pa); 2836 for (tmpva = va; size > 0;) { 2837 pmap_kenter_attr(tmpva, pa, PTE_C_UNCACHED); 2838 size -= PAGE_SIZE; 2839 tmpva += PAGE_SIZE; 2840 pa += PAGE_SIZE; 2841 } 2842 } 2843 2844 return ((void *)(va + offset)); 2845} 2846 2847void 2848pmap_unmapdev(vm_offset_t va, vm_size_t size) 2849{ 2850#ifndef __mips_n64 2851 vm_offset_t base, offset, tmpva; 2852 2853 /* If the address is within KSEG1 then there is nothing to do */ 2854 if (va >= MIPS_KSEG1_START && va <= MIPS_KSEG1_END) 2855 return; 2856 2857 base = trunc_page(va); 2858 offset = va & PAGE_MASK; 2859 size = roundup(size + offset, PAGE_SIZE); 2860 for (tmpva = base; tmpva < base + size; tmpva += PAGE_SIZE) 2861 pmap_kremove(tmpva); 2862 kmem_free(kernel_map, base, size); 2863#endif 2864} 2865 2866/* 2867 * perform the pmap work for mincore 2868 */ 2869int 2870pmap_mincore(pmap_t pmap, vm_offset_t addr, vm_paddr_t *locked_pa) 2871{ 2872 pt_entry_t *ptep, pte; 2873 vm_paddr_t pa; 2874 vm_page_t m; 2875 int val; 2876 boolean_t managed; 2877 2878 PMAP_LOCK(pmap); 2879retry: 2880 ptep = pmap_pte(pmap, addr); 2881 pte = (ptep != NULL) ? *ptep : 0; 2882 if (!pte_test(&pte, PTE_V)) { 2883 val = 0; 2884 goto out; 2885 } 2886 val = MINCORE_INCORE; 2887 if (pte_test(&pte, PTE_D)) 2888 val |= MINCORE_MODIFIED | MINCORE_MODIFIED_OTHER; 2889 pa = TLBLO_PTE_TO_PA(pte); 2890 managed = page_is_managed(pa); 2891 if (managed) { 2892 /* 2893 * This may falsely report the given address as 2894 * MINCORE_REFERENCED. Unfortunately, due to the lack of 2895 * per-PTE reference information, it is impossible to 2896 * determine if the address is MINCORE_REFERENCED. 2897 */ 2898 m = PHYS_TO_VM_PAGE(pa); 2899 if ((m->aflags & PGA_REFERENCED) != 0) 2900 val |= MINCORE_REFERENCED | MINCORE_REFERENCED_OTHER; 2901 } 2902 if ((val & (MINCORE_MODIFIED_OTHER | MINCORE_REFERENCED_OTHER)) != 2903 (MINCORE_MODIFIED_OTHER | MINCORE_REFERENCED_OTHER) && managed) { 2904 /* Ensure that "PHYS_TO_VM_PAGE(pa)->object" doesn't change. */ 2905 if (vm_page_pa_tryrelock(pmap, pa, locked_pa)) 2906 goto retry; 2907 } else 2908out: 2909 PA_UNLOCK_COND(*locked_pa); 2910 PMAP_UNLOCK(pmap); 2911 return (val); 2912} 2913 2914void 2915pmap_activate(struct thread *td) 2916{ 2917 pmap_t pmap, oldpmap; 2918 struct proc *p = td->td_proc; 2919 u_int cpuid; 2920 2921 critical_enter(); 2922 2923 pmap = vmspace_pmap(p->p_vmspace); 2924 oldpmap = PCPU_GET(curpmap); 2925 cpuid = PCPU_GET(cpuid); 2926 2927 if (oldpmap) 2928 CPU_CLR_ATOMIC(cpuid, &oldpmap->pm_active); 2929 CPU_SET_ATOMIC(cpuid, &pmap->pm_active); 2930 pmap_asid_alloc(pmap); 2931 if (td == curthread) { 2932 PCPU_SET(segbase, pmap->pm_segtab); 2933 mips_wr_entryhi(pmap->pm_asid[cpuid].asid); 2934 } 2935 2936 PCPU_SET(curpmap, pmap); 2937 critical_exit(); 2938} 2939 2940void 2941pmap_sync_icache(pmap_t pm, vm_offset_t va, vm_size_t sz) 2942{ 2943} 2944 2945/* 2946 * Increase the starting virtual address of the given mapping if a 2947 * different alignment might result in more superpage mappings. 2948 */ 2949void 2950pmap_align_superpage(vm_object_t object, vm_ooffset_t offset, 2951 vm_offset_t *addr, vm_size_t size) 2952{ 2953 vm_offset_t superpage_offset; 2954 2955 if (size < NBSEG) 2956 return; 2957 if (object != NULL && (object->flags & OBJ_COLORED) != 0) 2958 offset += ptoa(object->pg_color); 2959 superpage_offset = offset & SEGMASK; 2960 if (size - ((NBSEG - superpage_offset) & SEGMASK) < NBSEG || 2961 (*addr & SEGMASK) == superpage_offset) 2962 return; 2963 if ((*addr & SEGMASK) < superpage_offset) 2964 *addr = (*addr & ~SEGMASK) + superpage_offset; 2965 else 2966 *addr = ((*addr + SEGMASK) & ~SEGMASK) + superpage_offset; 2967} 2968 2969/* 2970 * Increase the starting virtual address of the given mapping so 2971 * that it is aligned to not be the second page in a TLB entry. 2972 * This routine assumes that the length is appropriately-sized so 2973 * that the allocation does not share a TLB entry at all if required. 2974 */ 2975void 2976pmap_align_tlb(vm_offset_t *addr) 2977{ 2978 if ((*addr & PAGE_SIZE) == 0) 2979 return; 2980 *addr += PAGE_SIZE; 2981 return; 2982} 2983 2984#ifdef DDB 2985DB_SHOW_COMMAND(ptable, ddb_pid_dump) 2986{ 2987 pmap_t pmap; 2988 struct thread *td = NULL; 2989 struct proc *p; 2990 int i, j, k; 2991 vm_paddr_t pa; 2992 vm_offset_t va; 2993 2994 if (have_addr) { 2995 td = db_lookup_thread(addr, TRUE); 2996 if (td == NULL) { 2997 db_printf("Invalid pid or tid"); 2998 return; 2999 } 3000 p = td->td_proc; 3001 if (p->p_vmspace == NULL) { 3002 db_printf("No vmspace for process"); 3003 return; 3004 } 3005 pmap = vmspace_pmap(p->p_vmspace); 3006 } else 3007 pmap = kernel_pmap; 3008 3009 db_printf("pmap:%p segtab:%p asid:%x generation:%x\n", 3010 pmap, pmap->pm_segtab, pmap->pm_asid[0].asid, 3011 pmap->pm_asid[0].gen); 3012 for (i = 0; i < NPDEPG; i++) { 3013 pd_entry_t *pdpe; 3014 pt_entry_t *pde; 3015 pt_entry_t pte; 3016 3017 pdpe = (pd_entry_t *)pmap->pm_segtab[i]; 3018 if (pdpe == NULL) 3019 continue; 3020 db_printf("[%4d] %p\n", i, pdpe); 3021#ifdef __mips_n64 3022 for (j = 0; j < NPDEPG; j++) { 3023 pde = (pt_entry_t *)pdpe[j]; 3024 if (pde == NULL) 3025 continue; 3026 db_printf("\t[%4d] %p\n", j, pde); 3027#else 3028 { 3029 j = 0; 3030 pde = (pt_entry_t *)pdpe; 3031#endif 3032 for (k = 0; k < NPTEPG; k++) { 3033 pte = pde[k]; 3034 if (pte == 0 || !pte_test(&pte, PTE_V)) 3035 continue; 3036 pa = TLBLO_PTE_TO_PA(pte); 3037 va = ((u_long)i << SEGSHIFT) | (j << PDRSHIFT) | (k << PAGE_SHIFT); 3038 db_printf("\t\t[%04d] va: %p pte: %8jx pa:%jx\n", 3039 k, (void *)va, (uintmax_t)pte, (uintmax_t)pa); 3040 } 3041 } 3042 } 3043} 3044#endif 3045 3046#if defined(DEBUG) 3047 3048static void pads(pmap_t pm); 3049void pmap_pvdump(vm_offset_t pa); 3050 3051/* print address space of pmap*/ 3052static void 3053pads(pmap_t pm) 3054{ 3055 unsigned va, i, j; 3056 pt_entry_t *ptep; 3057 3058 if (pm == kernel_pmap) 3059 return; 3060 for (i = 0; i < NPTEPG; i++) 3061 if (pm->pm_segtab[i]) 3062 for (j = 0; j < NPTEPG; j++) { 3063 va = (i << SEGSHIFT) + (j << PAGE_SHIFT); 3064 if (pm == kernel_pmap && va < KERNBASE) 3065 continue; 3066 if (pm != kernel_pmap && 3067 va >= VM_MAXUSER_ADDRESS) 3068 continue; 3069 ptep = pmap_pte(pm, va); 3070 if (pte_test(ptep, PTE_V)) 3071 printf("%x:%x ", va, *(int *)ptep); 3072 } 3073 3074} 3075 3076void 3077pmap_pvdump(vm_offset_t pa) 3078{ 3079 register pv_entry_t pv; 3080 vm_page_t m; 3081 3082 printf("pa %x", pa); 3083 m = PHYS_TO_VM_PAGE(pa); 3084 for (pv = TAILQ_FIRST(&m->md.pv_list); pv; 3085 pv = TAILQ_NEXT(pv, pv_list)) { 3086 printf(" -> pmap %p, va %x", (void *)pv->pv_pmap, pv->pv_va); 3087 pads(pv->pv_pmap); 3088 } 3089 printf(" "); 3090} 3091 3092/* N/C */ 3093#endif 3094 3095 3096/* 3097 * Allocate TLB address space tag (called ASID or TLBPID) and return it. 3098 * It takes almost as much or more time to search the TLB for a 3099 * specific ASID and flush those entries as it does to flush the entire TLB. 3100 * Therefore, when we allocate a new ASID, we just take the next number. When 3101 * we run out of numbers, we flush the TLB, increment the generation count 3102 * and start over. ASID zero is reserved for kernel use. 3103 */ 3104static void 3105pmap_asid_alloc(pmap) 3106 pmap_t pmap; 3107{ 3108 if (pmap->pm_asid[PCPU_GET(cpuid)].asid != PMAP_ASID_RESERVED && 3109 pmap->pm_asid[PCPU_GET(cpuid)].gen == PCPU_GET(asid_generation)); 3110 else { 3111 if (PCPU_GET(next_asid) == pmap_max_asid) { 3112 tlb_invalidate_all_user(NULL); 3113 PCPU_SET(asid_generation, 3114 (PCPU_GET(asid_generation) + 1) & ASIDGEN_MASK); 3115 if (PCPU_GET(asid_generation) == 0) { 3116 PCPU_SET(asid_generation, 1); 3117 } 3118 PCPU_SET(next_asid, 1); /* 0 means invalid */ 3119 } 3120 pmap->pm_asid[PCPU_GET(cpuid)].asid = PCPU_GET(next_asid); 3121 pmap->pm_asid[PCPU_GET(cpuid)].gen = PCPU_GET(asid_generation); 3122 PCPU_SET(next_asid, PCPU_GET(next_asid) + 1); 3123 } 3124} 3125 3126int 3127page_is_managed(vm_paddr_t pa) 3128{ 3129 vm_offset_t pgnum = atop(pa); 3130 3131 if (pgnum >= first_page) { 3132 vm_page_t m; 3133 3134 m = PHYS_TO_VM_PAGE(pa); 3135 if (m == NULL) 3136 return (0); 3137 if ((m->oflags & VPO_UNMANAGED) == 0) 3138 return (1); 3139 } 3140 return (0); 3141} 3142 3143static pt_entry_t 3144init_pte_prot(vm_offset_t va, vm_page_t m, vm_prot_t prot) 3145{ 3146 pt_entry_t rw; 3147 3148 if (!(prot & VM_PROT_WRITE)) 3149 rw = PTE_V | PTE_RO | PTE_C_CACHE; 3150 else if ((m->oflags & VPO_UNMANAGED) == 0) { 3151 if ((m->md.pv_flags & PV_TABLE_MOD) != 0) 3152 rw = PTE_V | PTE_D | PTE_C_CACHE; 3153 else 3154 rw = PTE_V | PTE_C_CACHE; 3155 vm_page_aflag_set(m, PGA_WRITEABLE); 3156 } else 3157 /* Needn't emulate a modified bit for unmanaged pages. */ 3158 rw = PTE_V | PTE_D | PTE_C_CACHE; 3159 return (rw); 3160} 3161 3162/* 3163 * pmap_emulate_modified : do dirty bit emulation 3164 * 3165 * On SMP, update just the local TLB, other CPUs will update their 3166 * TLBs from PTE lazily, if they get the exception. 3167 * Returns 0 in case of sucess, 1 if the page is read only and we 3168 * need to fault. 3169 */ 3170int 3171pmap_emulate_modified(pmap_t pmap, vm_offset_t va) 3172{ 3173 vm_page_t m; 3174 pt_entry_t *pte; 3175 vm_paddr_t pa; 3176 3177 PMAP_LOCK(pmap); 3178 pte = pmap_pte(pmap, va); 3179 if (pte == NULL) 3180 panic("pmap_emulate_modified: can't find PTE"); 3181#ifdef SMP 3182 /* It is possible that some other CPU changed m-bit */ 3183 if (!pte_test(pte, PTE_V) || pte_test(pte, PTE_D)) { 3184 tlb_update(pmap, va, *pte); 3185 PMAP_UNLOCK(pmap); 3186 return (0); 3187 } 3188#else 3189 if (!pte_test(pte, PTE_V) || pte_test(pte, PTE_D)) 3190 panic("pmap_emulate_modified: invalid pte"); 3191#endif 3192 if (pte_test(pte, PTE_RO)) { 3193 /* write to read only page in the kernel */ 3194 PMAP_UNLOCK(pmap); 3195 return (1); 3196 } 3197 pte_set(pte, PTE_D); 3198 tlb_update(pmap, va, *pte); 3199 pa = TLBLO_PTE_TO_PA(*pte); 3200 if (!page_is_managed(pa)) 3201 panic("pmap_emulate_modified: unmanaged page"); 3202 m = PHYS_TO_VM_PAGE(pa); 3203 m->md.pv_flags |= (PV_TABLE_REF | PV_TABLE_MOD); 3204 PMAP_UNLOCK(pmap); 3205 return (0); 3206} 3207 3208/* 3209 * Routine: pmap_kextract 3210 * Function: 3211 * Extract the physical page address associated 3212 * virtual address. 3213 */ 3214vm_paddr_t 3215pmap_kextract(vm_offset_t va) 3216{ 3217 int mapped; 3218 3219 /* 3220 * First, the direct-mapped regions. 3221 */ 3222#if defined(__mips_n64) 3223 if (va >= MIPS_XKPHYS_START && va < MIPS_XKPHYS_END) 3224 return (MIPS_XKPHYS_TO_PHYS(va)); 3225#endif 3226 if (va >= MIPS_KSEG0_START && va < MIPS_KSEG0_END) 3227 return (MIPS_KSEG0_TO_PHYS(va)); 3228 3229 if (va >= MIPS_KSEG1_START && va < MIPS_KSEG1_END) 3230 return (MIPS_KSEG1_TO_PHYS(va)); 3231 3232 /* 3233 * User virtual addresses. 3234 */ 3235 if (va < VM_MAXUSER_ADDRESS) { 3236 pt_entry_t *ptep; 3237 3238 if (curproc && curproc->p_vmspace) { 3239 ptep = pmap_pte(&curproc->p_vmspace->vm_pmap, va); 3240 if (ptep) { 3241 return (TLBLO_PTE_TO_PA(*ptep) | 3242 (va & PAGE_MASK)); 3243 } 3244 return (0); 3245 } 3246 } 3247 3248 /* 3249 * Should be kernel virtual here, otherwise fail 3250 */ 3251 mapped = (va >= MIPS_KSEG2_START || va < MIPS_KSEG2_END); 3252#if defined(__mips_n64) 3253 mapped = mapped || (va >= MIPS_XKSEG_START || va < MIPS_XKSEG_END); 3254#endif 3255 /* 3256 * Kernel virtual. 3257 */ 3258 3259 if (mapped) { 3260 pt_entry_t *ptep; 3261 3262 /* Is the kernel pmap initialized? */ 3263 if (!CPU_EMPTY(&kernel_pmap->pm_active)) { 3264 /* It's inside the virtual address range */ 3265 ptep = pmap_pte(kernel_pmap, va); 3266 if (ptep) { 3267 return (TLBLO_PTE_TO_PA(*ptep) | 3268 (va & PAGE_MASK)); 3269 } 3270 } 3271 return (0); 3272 } 3273 3274 panic("%s for unknown address space %p.", __func__, (void *)va); 3275} 3276 3277 3278void 3279pmap_flush_pvcache(vm_page_t m) 3280{ 3281 pv_entry_t pv; 3282 3283 if (m != NULL) { 3284 for (pv = TAILQ_FIRST(&m->md.pv_list); pv; 3285 pv = TAILQ_NEXT(pv, pv_list)) { 3286 mips_dcache_wbinv_range_index(pv->pv_va, PAGE_SIZE); 3287 } 3288 } 3289} 3290