pmap.c revision 211453
1/* 2 * Copyright (c) 1991 Regents of the University of California. 3 * All rights reserved. 4 * Copyright (c) 1994 John S. Dyson 5 * All rights reserved. 6 * Copyright (c) 1994 David Greenman 7 * All rights reserved. 8 * 9 * This code is derived from software contributed to Berkeley by 10 * the Systems Programming Group of the University of Utah Computer 11 * Science Department and William Jolitz of UUNET Technologies Inc. 12 * 13 * Redistribution and use in source and binary forms, with or without 14 * modification, are permitted provided that the following conditions 15 * are met: 16 * 1. Redistributions of source code must retain the above copyright 17 * notice, this list of conditions and the following disclaimer. 18 * 2. Redistributions in binary form must reproduce the above copyright 19 * notice, this list of conditions and the following disclaimer in the 20 * documentation and/or other materials provided with the distribution. 21 * 4. Neither the name of the University nor the names of its contributors 22 * may be used to endorse or promote products derived from this software 23 * without specific prior written permission. 24 * 25 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 26 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 27 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 28 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 29 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 30 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 31 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 32 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 33 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 34 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 35 * SUCH DAMAGE. 36 * 37 * from: @(#)pmap.c 7.7 (Berkeley) 5/12/91 38 * from: src/sys/i386/i386/pmap.c,v 1.250.2.8 2000/11/21 00:09:14 ps 39 * JNPR: pmap.c,v 1.11.2.1 2007/08/16 11:51:06 girish 40 */ 41 42/* 43 * Manages physical address maps. 44 * 45 * In addition to hardware address maps, this 46 * module is called upon to provide software-use-only 47 * maps which may or may not be stored in the same 48 * form as hardware maps. These pseudo-maps are 49 * used to store intermediate results from copy 50 * operations to and from address spaces. 51 * 52 * Since the information managed by this module is 53 * also stored by the logical address mapping module, 54 * this module may throw away valid virtual-to-physical 55 * mappings at almost any time. However, invalidations 56 * of virtual-to-physical mappings must be done as 57 * requested. 58 * 59 * In order to cope with hardware architectures which 60 * make virtual-to-physical map invalidates expensive, 61 * this module may delay invalidate or reduced protection 62 * operations until such time as they are actually 63 * necessary. This module is given full information as 64 * to which processors are currently using which maps, 65 * and to when physical maps must be made correct. 66 */ 67 68#include <sys/cdefs.h> 69__FBSDID("$FreeBSD: head/sys/mips/mips/pmap.c 211453 2010-08-18 12:52:21Z jchandra $"); 70 71#include "opt_msgbuf.h" 72#include "opt_ddb.h" 73 74#include <sys/param.h> 75#include <sys/systm.h> 76#include <sys/proc.h> 77#include <sys/msgbuf.h> 78#include <sys/vmmeter.h> 79#include <sys/mman.h> 80#include <sys/smp.h> 81#ifdef DDB 82#include <ddb/ddb.h> 83#endif 84 85#include <vm/vm.h> 86#include <vm/vm_param.h> 87#include <vm/vm_phys.h> 88#include <sys/lock.h> 89#include <sys/mutex.h> 90#include <vm/vm_kern.h> 91#include <vm/vm_page.h> 92#include <vm/vm_map.h> 93#include <vm/vm_object.h> 94#include <vm/vm_extern.h> 95#include <vm/vm_pageout.h> 96#include <vm/vm_pager.h> 97#include <vm/uma.h> 98#include <sys/pcpu.h> 99#include <sys/sched.h> 100#ifdef SMP 101#include <sys/smp.h> 102#endif 103 104#include <machine/cache.h> 105#include <machine/md_var.h> 106#include <machine/tlb.h> 107 108#if defined(DIAGNOSTIC) 109#define PMAP_DIAGNOSTIC 110#endif 111 112#undef PMAP_DEBUG 113 114#ifndef PMAP_SHPGPERPROC 115#define PMAP_SHPGPERPROC 200 116#endif 117 118#if !defined(PMAP_DIAGNOSTIC) 119#define PMAP_INLINE __inline 120#else 121#define PMAP_INLINE 122#endif 123 124/* 125 * Get PDEs and PTEs for user/kernel address space 126 * 127 * XXX The & for pmap_segshift() is wrong, as is the fact that it doesn't 128 * trim off gratuitous bits of the address space. By having the & 129 * there, we break defining NUSERPGTBLS below because the address space 130 * is defined such that it ends immediately after NPDEPG*NPTEPG*PAGE_SIZE, 131 * so we end up getting NUSERPGTBLS of 0. 132 */ 133#define pmap_seg_index(v) (((v) >> SEGSHIFT) & (NPDEPG - 1)) 134#define pmap_pde_index(v) (((v) >> PDRSHIFT) & (NPDEPG - 1)) 135#define pmap_pte_index(v) (((v) >> PAGE_SHIFT) & (NPTEPG - 1)) 136#define pmap_pde_pindex(v) ((v) >> PDRSHIFT) 137 138#ifdef __mips_n64 139#define NUPDE (NPDEPG * NPDEPG) 140#define NUSERPGTBLS (NUPDE + NPDEPG) 141#else 142#define NUPDE (NPDEPG) 143#define NUSERPGTBLS (NUPDE) 144#endif 145 146#define is_kernel_pmap(x) ((x) == kernel_pmap) 147 148struct pmap kernel_pmap_store; 149pd_entry_t *kernel_segmap; 150 151vm_offset_t virtual_avail; /* VA of first avail page (after kernel bss) */ 152vm_offset_t virtual_end; /* VA of last avail page (end of kernel AS) */ 153 154static int nkpt; 155unsigned pmap_max_asid; /* max ASID supported by the system */ 156 157#define PMAP_ASID_RESERVED 0 158 159vm_offset_t kernel_vm_end = VM_MIN_KERNEL_ADDRESS; 160 161static void pmap_asid_alloc(pmap_t pmap); 162 163/* 164 * Data for the pv entry allocation mechanism 165 */ 166static uma_zone_t pvzone; 167static struct vm_object pvzone_obj; 168static int pv_entry_count = 0, pv_entry_max = 0, pv_entry_high_water = 0; 169 170static PMAP_INLINE void free_pv_entry(pv_entry_t pv); 171static pv_entry_t get_pv_entry(pmap_t locked_pmap); 172static void pmap_pvh_free(struct md_page *pvh, pmap_t pmap, vm_offset_t va); 173static pv_entry_t pmap_pvh_remove(struct md_page *pvh, pmap_t pmap, 174 vm_offset_t va); 175static __inline void pmap_changebit(vm_page_t m, int bit, boolean_t setem); 176static vm_page_t pmap_enter_quick_locked(pmap_t pmap, vm_offset_t va, 177 vm_page_t m, vm_prot_t prot, vm_page_t mpte); 178static int pmap_remove_pte(struct pmap *pmap, pt_entry_t *ptq, vm_offset_t va); 179static void pmap_remove_page(struct pmap *pmap, vm_offset_t va); 180static void pmap_remove_entry(struct pmap *pmap, vm_page_t m, vm_offset_t va); 181static boolean_t pmap_try_insert_pv_entry(pmap_t pmap, vm_page_t mpte, 182 vm_offset_t va, vm_page_t m); 183static void pmap_update_page(pmap_t pmap, vm_offset_t va, pt_entry_t pte); 184static void pmap_invalidate_all(pmap_t pmap); 185static void pmap_invalidate_page(pmap_t pmap, vm_offset_t va); 186static int _pmap_unwire_pte_hold(pmap_t pmap, vm_offset_t va, vm_page_t m); 187 188static vm_page_t pmap_allocpte(pmap_t pmap, vm_offset_t va, int flags); 189static vm_page_t _pmap_allocpte(pmap_t pmap, unsigned ptepindex, int flags); 190static int pmap_unuse_pt(pmap_t, vm_offset_t, vm_page_t); 191static int init_pte_prot(vm_offset_t va, vm_page_t m, vm_prot_t prot); 192static vm_page_t pmap_alloc_pte_page(unsigned int index, int req); 193static void pmap_grow_pte_page_cache(void); 194 195#ifdef SMP 196static void pmap_invalidate_page_action(void *arg); 197static void pmap_invalidate_all_action(void *arg); 198static void pmap_update_page_action(void *arg); 199#endif 200 201#ifndef __mips_n64 202/* 203 * This structure is for high memory (memory above 512Meg in 32 bit) 204 * This memory area does not have direct mapping, so we a mechanism to do 205 * temporary per-CPU mapping to access these addresses. 206 * 207 * At bootup we reserve 2 virtual pages per CPU for mapping highmem pages, to 208 * access a highmem physical address on a CPU, we will disable interrupts and 209 * add the mapping from the reserved virtual address for the CPU to the physical 210 * address in the kernel pagetable. 211 */ 212struct local_sysmaps { 213 vm_offset_t base; 214 uint32_t saved_intr; 215 uint16_t valid1, valid2; 216}; 217static struct local_sysmaps sysmap_lmem[MAXCPU]; 218 219static __inline void 220pmap_alloc_lmem_map(void) 221{ 222 int i; 223 224 for (i = 0; i < MAXCPU; i++) { 225 sysmap_lmem[i].base = virtual_avail; 226 virtual_avail += PAGE_SIZE * 2; 227 sysmap_lmem[i].valid1 = sysmap_lmem[i].valid2 = 0; 228 } 229} 230 231static __inline vm_offset_t 232pmap_lmem_map1(vm_paddr_t phys) 233{ 234 struct local_sysmaps *sysm; 235 pt_entry_t *pte, npte; 236 vm_offset_t va; 237 uint32_t intr; 238 int cpu; 239 240 intr = intr_disable(); 241 cpu = PCPU_GET(cpuid); 242 sysm = &sysmap_lmem[cpu]; 243 sysm->saved_intr = intr; 244 va = sysm->base; 245 npte = TLBLO_PA_TO_PFN(phys) | 246 PTE_D | PTE_V | PTE_G | PTE_W | PTE_C_CACHE; 247 pte = pmap_pte(kernel_pmap, va); 248 *pte = npte; 249 sysm->valid1 = 1; 250 return (va); 251} 252 253static __inline vm_offset_t 254pmap_lmem_map2(vm_paddr_t phys1, vm_paddr_t phys2) 255{ 256 struct local_sysmaps *sysm; 257 pt_entry_t *pte, npte; 258 vm_offset_t va1, va2; 259 uint32_t intr; 260 int cpu; 261 262 intr = intr_disable(); 263 cpu = PCPU_GET(cpuid); 264 sysm = &sysmap_lmem[cpu]; 265 sysm->saved_intr = intr; 266 va1 = sysm->base; 267 va2 = sysm->base + PAGE_SIZE; 268 npte = TLBLO_PA_TO_PFN(phys1) | 269 PTE_D | PTE_V | PTE_G | PTE_W | PTE_C_CACHE; 270 pte = pmap_pte(kernel_pmap, va1); 271 *pte = npte; 272 npte = TLBLO_PA_TO_PFN(phys2) | 273 PTE_D | PTE_V | PTE_G | PTE_W | PTE_C_CACHE; 274 pte = pmap_pte(kernel_pmap, va2); 275 *pte = npte; 276 sysm->valid1 = 1; 277 sysm->valid2 = 1; 278 return (va1); 279} 280 281static __inline void 282pmap_lmem_unmap(void) 283{ 284 struct local_sysmaps *sysm; 285 pt_entry_t *pte; 286 int cpu; 287 288 cpu = PCPU_GET(cpuid); 289 sysm = &sysmap_lmem[cpu]; 290 pte = pmap_pte(kernel_pmap, sysm->base); 291 *pte = PTE_G; 292 tlb_invalidate_address(kernel_pmap, sysm->base); 293 sysm->valid1 = 0; 294 if (sysm->valid2) { 295 pte = pmap_pte(kernel_pmap, sysm->base + PAGE_SIZE); 296 *pte = PTE_G; 297 tlb_invalidate_address(kernel_pmap, sysm->base + PAGE_SIZE); 298 sysm->valid2 = 0; 299 } 300 intr_restore(sysm->saved_intr); 301} 302#else /* __mips_n64 */ 303 304static __inline void 305pmap_alloc_lmem_map(void) 306{ 307} 308 309static __inline vm_offset_t 310pmap_lmem_map1(vm_paddr_t phys) 311{ 312 313 return (0); 314} 315 316static __inline vm_offset_t 317pmap_lmem_map2(vm_paddr_t phys1, vm_paddr_t phys2) 318{ 319 320 return (0); 321} 322 323static __inline vm_offset_t 324pmap_lmem_unmap(void) 325{ 326 327 return (0); 328} 329#endif /* !__mips_n64 */ 330 331/* 332 * Page table entry lookup routines. 333 */ 334static __inline pd_entry_t * 335pmap_segmap(pmap_t pmap, vm_offset_t va) 336{ 337 338 return (&pmap->pm_segtab[pmap_seg_index(va)]); 339} 340 341#ifdef __mips_n64 342static __inline pd_entry_t * 343pmap_pdpe_to_pde(pd_entry_t *pdpe, vm_offset_t va) 344{ 345 pd_entry_t *pde; 346 347 pde = (pd_entry_t *)*pdpe; 348 return (&pde[pmap_pde_index(va)]); 349} 350 351static __inline pd_entry_t * 352pmap_pde(pmap_t pmap, vm_offset_t va) 353{ 354 pd_entry_t *pdpe; 355 356 pdpe = pmap_segmap(pmap, va); 357 if (pdpe == NULL || *pdpe == NULL) 358 return (NULL); 359 360 return (pmap_pdpe_to_pde(pdpe, va)); 361} 362#else 363static __inline pd_entry_t * 364pmap_pdpe_to_pde(pd_entry_t *pdpe, vm_offset_t va) 365{ 366 367 return (pdpe); 368} 369 370static __inline 371pd_entry_t *pmap_pde(pmap_t pmap, vm_offset_t va) 372{ 373 374 return (pmap_segmap(pmap, va)); 375} 376#endif 377 378static __inline pt_entry_t * 379pmap_pde_to_pte(pd_entry_t *pde, vm_offset_t va) 380{ 381 pt_entry_t *pte; 382 383 pte = (pt_entry_t *)*pde; 384 return (&pte[pmap_pte_index(va)]); 385} 386 387pt_entry_t * 388pmap_pte(pmap_t pmap, vm_offset_t va) 389{ 390 pd_entry_t *pde; 391 392 pde = pmap_pde(pmap, va); 393 if (pde == NULL || *pde == NULL) 394 return (NULL); 395 396 return (pmap_pde_to_pte(pde, va)); 397} 398 399vm_offset_t 400pmap_steal_memory(vm_size_t size) 401{ 402 vm_size_t bank_size; 403 vm_offset_t pa, va; 404 405 size = round_page(size); 406 407 bank_size = phys_avail[1] - phys_avail[0]; 408 while (size > bank_size) { 409 int i; 410 411 for (i = 0; phys_avail[i + 2]; i += 2) { 412 phys_avail[i] = phys_avail[i + 2]; 413 phys_avail[i + 1] = phys_avail[i + 3]; 414 } 415 phys_avail[i] = 0; 416 phys_avail[i + 1] = 0; 417 if (!phys_avail[0]) 418 panic("pmap_steal_memory: out of memory"); 419 bank_size = phys_avail[1] - phys_avail[0]; 420 } 421 422 pa = phys_avail[0]; 423 phys_avail[0] += size; 424 if (MIPS_DIRECT_MAPPABLE(pa) == 0) 425 panic("Out of memory below 512Meg?"); 426 va = MIPS_PHYS_TO_DIRECT(pa); 427 bzero((caddr_t)va, size); 428 return (va); 429} 430 431/* 432 * Bootstrap the system enough to run with virtual memory. This 433 * assumes that the phys_avail array has been initialized. 434 */ 435static void 436pmap_create_kernel_pagetable(void) 437{ 438 int i, j; 439 vm_offset_t ptaddr; 440 pt_entry_t *pte; 441#ifdef __mips_n64 442 pd_entry_t *pde; 443 vm_offset_t pdaddr; 444 int npt, npde; 445#endif 446 447 /* 448 * Allocate segment table for the kernel 449 */ 450 kernel_segmap = (pd_entry_t *)pmap_steal_memory(PAGE_SIZE); 451 452 /* 453 * Allocate second level page tables for the kernel 454 */ 455#ifdef __mips_n64 456 npde = howmany(NKPT, NPDEPG); 457 pdaddr = pmap_steal_memory(PAGE_SIZE * npde); 458#endif 459 nkpt = NKPT; 460 ptaddr = pmap_steal_memory(PAGE_SIZE * nkpt); 461 462 /* 463 * The R[4-7]?00 stores only one copy of the Global bit in the 464 * translation lookaside buffer for each 2 page entry. Thus invalid 465 * entrys must have the Global bit set so when Entry LO and Entry HI 466 * G bits are anded together they will produce a global bit to store 467 * in the tlb. 468 */ 469 for (i = 0, pte = (pt_entry_t *)ptaddr; i < (nkpt * NPTEPG); i++, pte++) 470 *pte = PTE_G; 471 472#ifdef __mips_n64 473 for (i = 0, npt = nkpt; npt > 0; i++) { 474 kernel_segmap[i] = (pd_entry_t)(pdaddr + i * PAGE_SIZE); 475 pde = (pd_entry_t *)kernel_segmap[i]; 476 477 for (j = 0; j < NPDEPG && npt > 0; j++, npt--) 478 pde[j] = (pd_entry_t)(ptaddr + (i * NPDEPG + j) * PAGE_SIZE); 479 } 480#else 481 for (i = 0, j = pmap_seg_index(VM_MIN_KERNEL_ADDRESS); i < nkpt; i++, j++) 482 kernel_segmap[j] = (pd_entry_t)(ptaddr + (i * PAGE_SIZE)); 483#endif 484 485 PMAP_LOCK_INIT(kernel_pmap); 486 kernel_pmap->pm_segtab = kernel_segmap; 487 kernel_pmap->pm_active = ~0; 488 TAILQ_INIT(&kernel_pmap->pm_pvlist); 489 kernel_pmap->pm_asid[0].asid = PMAP_ASID_RESERVED; 490 kernel_pmap->pm_asid[0].gen = 0; 491 kernel_vm_end += nkpt * NPTEPG * PAGE_SIZE; 492} 493 494void 495pmap_bootstrap(void) 496{ 497 int i; 498 int need_local_mappings = 0; 499 500 /* Sort. */ 501again: 502 for (i = 0; phys_avail[i + 1] != 0; i += 2) { 503 /* 504 * Keep the memory aligned on page boundary. 505 */ 506 phys_avail[i] = round_page(phys_avail[i]); 507 phys_avail[i + 1] = trunc_page(phys_avail[i + 1]); 508 509 if (i < 2) 510 continue; 511 if (phys_avail[i - 2] > phys_avail[i]) { 512 vm_paddr_t ptemp[2]; 513 514 ptemp[0] = phys_avail[i + 0]; 515 ptemp[1] = phys_avail[i + 1]; 516 517 phys_avail[i + 0] = phys_avail[i - 2]; 518 phys_avail[i + 1] = phys_avail[i - 1]; 519 520 phys_avail[i - 2] = ptemp[0]; 521 phys_avail[i - 1] = ptemp[1]; 522 goto again; 523 } 524 } 525 526 /* 527 * In 32 bit, we may have memory which cannot be mapped directly 528 * this memory will need temporary mapping before it can be 529 * accessed. 530 */ 531 if (!MIPS_DIRECT_MAPPABLE(phys_avail[i - 1])) 532 need_local_mappings = 1; 533 534 /* 535 * Copy the phys_avail[] array before we start stealing memory from it. 536 */ 537 for (i = 0; phys_avail[i + 1] != 0; i += 2) { 538 physmem_desc[i] = phys_avail[i]; 539 physmem_desc[i + 1] = phys_avail[i + 1]; 540 } 541 542 Maxmem = atop(phys_avail[i - 1]); 543 544 if (bootverbose) { 545 printf("Physical memory chunk(s):\n"); 546 for (i = 0; phys_avail[i + 1] != 0; i += 2) { 547 vm_paddr_t size; 548 549 size = phys_avail[i + 1] - phys_avail[i]; 550 printf("%#08jx - %#08jx, %ju bytes (%ju pages)\n", 551 (uintmax_t) phys_avail[i], 552 (uintmax_t) phys_avail[i + 1] - 1, 553 (uintmax_t) size, (uintmax_t) size / PAGE_SIZE); 554 } 555 printf("Maxmem is 0x%0lx\n", ptoa(Maxmem)); 556 } 557 /* 558 * Steal the message buffer from the beginning of memory. 559 */ 560 msgbufp = (struct msgbuf *)pmap_steal_memory(MSGBUF_SIZE); 561 msgbufinit(msgbufp, MSGBUF_SIZE); 562 563 /* 564 * Steal thread0 kstack. 565 */ 566 kstack0 = pmap_steal_memory(KSTACK_PAGES << PAGE_SHIFT); 567 568 virtual_avail = VM_MIN_KERNEL_ADDRESS; 569 virtual_end = VM_MAX_KERNEL_ADDRESS; 570 571#ifdef SMP 572 /* 573 * Steal some virtual address space to map the pcpu area. 574 */ 575 virtual_avail = roundup2(virtual_avail, PAGE_SIZE * 2); 576 pcpup = (struct pcpu *)virtual_avail; 577 virtual_avail += PAGE_SIZE * 2; 578 579 /* 580 * Initialize the wired TLB entry mapping the pcpu region for 581 * the BSP at 'pcpup'. Up until this point we were operating 582 * with the 'pcpup' for the BSP pointing to a virtual address 583 * in KSEG0 so there was no need for a TLB mapping. 584 */ 585 mips_pcpu_tlb_init(PCPU_ADDR(0)); 586 587 if (bootverbose) 588 printf("pcpu is available at virtual address %p.\n", pcpup); 589#endif 590 591 if (need_local_mappings) 592 pmap_alloc_lmem_map(); 593 pmap_create_kernel_pagetable(); 594 pmap_max_asid = VMNUM_PIDS; 595 mips_wr_entryhi(0); 596 mips_wr_pagemask(0); 597} 598 599/* 600 * Initialize a vm_page's machine-dependent fields. 601 */ 602void 603pmap_page_init(vm_page_t m) 604{ 605 606 TAILQ_INIT(&m->md.pv_list); 607 m->md.pv_list_count = 0; 608 m->md.pv_flags = 0; 609} 610 611/* 612 * Initialize the pmap module. 613 * Called by vm_init, to initialize any structures that the pmap 614 * system needs to map virtual memory. 615 * pmap_init has been enhanced to support in a fairly consistant 616 * way, discontiguous physical memory. 617 */ 618void 619pmap_init(void) 620{ 621 622 /* 623 * Initialize the address space (zone) for the pv entries. Set a 624 * high water mark so that the system can recover from excessive 625 * numbers of pv entries. 626 */ 627 pvzone = uma_zcreate("PV ENTRY", sizeof(struct pv_entry), NULL, NULL, 628 NULL, NULL, UMA_ALIGN_PTR, UMA_ZONE_VM | UMA_ZONE_NOFREE); 629 pv_entry_max = PMAP_SHPGPERPROC * maxproc + cnt.v_page_count; 630 pv_entry_high_water = 9 * (pv_entry_max / 10); 631 uma_zone_set_obj(pvzone, &pvzone_obj, pv_entry_max); 632} 633 634/*************************************************** 635 * Low level helper routines..... 636 ***************************************************/ 637 638#if defined(PMAP_DIAGNOSTIC) 639 640/* 641 * This code checks for non-writeable/modified pages. 642 * This should be an invalid condition. 643 */ 644static int 645pmap_nw_modified(pt_entry_t pte) 646{ 647 if ((pte & (PTE_D | PTE_RO)) == (PTE_D | PTE_RO)) 648 return (1); 649 else 650 return (0); 651} 652#endif 653 654static __inline void 655pmap_invalidate_all_local(pmap_t pmap) 656{ 657 658 if (pmap == kernel_pmap) { 659 tlb_invalidate_all(); 660 return; 661 } 662 if (pmap->pm_active & PCPU_GET(cpumask)) 663 tlb_invalidate_all_user(pmap); 664 else 665 pmap->pm_asid[PCPU_GET(cpuid)].gen = 0; 666} 667 668#ifdef SMP 669static void 670pmap_invalidate_all(pmap_t pmap) 671{ 672 673 smp_rendezvous(0, pmap_invalidate_all_action, 0, pmap); 674} 675 676static void 677pmap_invalidate_all_action(void *arg) 678{ 679 680 pmap_invalidate_all_local((pmap_t)arg); 681} 682#else 683static void 684pmap_invalidate_all(pmap_t pmap) 685{ 686 687 pmap_invalidate_all_local(pmap); 688} 689#endif 690 691static __inline void 692pmap_invalidate_page_local(pmap_t pmap, vm_offset_t va) 693{ 694 695 if (is_kernel_pmap(pmap)) { 696 tlb_invalidate_address(pmap, va); 697 return; 698 } 699 if (pmap->pm_asid[PCPU_GET(cpuid)].gen != PCPU_GET(asid_generation)) 700 return; 701 else if (!(pmap->pm_active & PCPU_GET(cpumask))) { 702 pmap->pm_asid[PCPU_GET(cpuid)].gen = 0; 703 return; 704 } 705 tlb_invalidate_address(pmap, va); 706} 707 708#ifdef SMP 709struct pmap_invalidate_page_arg { 710 pmap_t pmap; 711 vm_offset_t va; 712}; 713 714static void 715pmap_invalidate_page(pmap_t pmap, vm_offset_t va) 716{ 717 struct pmap_invalidate_page_arg arg; 718 719 arg.pmap = pmap; 720 arg.va = va; 721 smp_rendezvous(0, pmap_invalidate_page_action, 0, &arg); 722} 723 724static void 725pmap_invalidate_page_action(void *arg) 726{ 727 struct pmap_invalidate_page_arg *p = arg; 728 729 pmap_invalidate_page_local(p->pmap, p->va); 730} 731#else 732static void 733pmap_invalidate_page(pmap_t pmap, vm_offset_t va) 734{ 735 736 pmap_invalidate_page_local(pmap, va); 737} 738#endif 739 740static __inline void 741pmap_update_page_local(pmap_t pmap, vm_offset_t va, pt_entry_t pte) 742{ 743 744 if (is_kernel_pmap(pmap)) { 745 tlb_update(pmap, va, pte); 746 return; 747 } 748 if (pmap->pm_asid[PCPU_GET(cpuid)].gen != PCPU_GET(asid_generation)) 749 return; 750 else if (!(pmap->pm_active & PCPU_GET(cpumask))) { 751 pmap->pm_asid[PCPU_GET(cpuid)].gen = 0; 752 return; 753 } 754 tlb_update(pmap, va, pte); 755} 756 757#ifdef SMP 758struct pmap_update_page_arg { 759 pmap_t pmap; 760 vm_offset_t va; 761 pt_entry_t pte; 762}; 763 764static void 765pmap_update_page(pmap_t pmap, vm_offset_t va, pt_entry_t pte) 766{ 767 struct pmap_update_page_arg arg; 768 769 arg.pmap = pmap; 770 arg.va = va; 771 arg.pte = pte; 772 smp_rendezvous(0, pmap_update_page_action, 0, &arg); 773} 774 775static void 776pmap_update_page_action(void *arg) 777{ 778 struct pmap_update_page_arg *p = arg; 779 780 pmap_update_page_local(p->pmap, p->va, p->pte); 781} 782#else 783static void 784pmap_update_page(pmap_t pmap, vm_offset_t va, pt_entry_t pte) 785{ 786 787 pmap_update_page_local(pmap, va, pte); 788} 789#endif 790 791/* 792 * Routine: pmap_extract 793 * Function: 794 * Extract the physical page address associated 795 * with the given map/virtual_address pair. 796 */ 797vm_paddr_t 798pmap_extract(pmap_t pmap, vm_offset_t va) 799{ 800 pt_entry_t *pte; 801 vm_offset_t retval = 0; 802 803 PMAP_LOCK(pmap); 804 pte = pmap_pte(pmap, va); 805 if (pte) { 806 retval = TLBLO_PTE_TO_PA(*pte) | (va & PAGE_MASK); 807 } 808 PMAP_UNLOCK(pmap); 809 return (retval); 810} 811 812/* 813 * Routine: pmap_extract_and_hold 814 * Function: 815 * Atomically extract and hold the physical page 816 * with the given pmap and virtual address pair 817 * if that mapping permits the given protection. 818 */ 819vm_page_t 820pmap_extract_and_hold(pmap_t pmap, vm_offset_t va, vm_prot_t prot) 821{ 822 pt_entry_t pte; 823 vm_page_t m; 824 vm_paddr_t pa; 825 826 m = NULL; 827 pa = 0; 828 PMAP_LOCK(pmap); 829retry: 830 pte = *pmap_pte(pmap, va); 831 if (pte != 0 && pte_test(&pte, PTE_V) && 832 (pte_test(&pte, PTE_D) || (prot & VM_PROT_WRITE) == 0)) { 833 if (vm_page_pa_tryrelock(pmap, TLBLO_PTE_TO_PA(pte), &pa)) 834 goto retry; 835 836 m = PHYS_TO_VM_PAGE(TLBLO_PTE_TO_PA(pte)); 837 vm_page_hold(m); 838 } 839 PA_UNLOCK_COND(pa); 840 PMAP_UNLOCK(pmap); 841 return (m); 842} 843 844/*************************************************** 845 * Low level mapping routines..... 846 ***************************************************/ 847 848/* 849 * add a wired page to the kva 850 */ 851 /* PMAP_INLINE */ void 852pmap_kenter(vm_offset_t va, vm_paddr_t pa) 853{ 854 pt_entry_t *pte; 855 pt_entry_t opte, npte; 856 857#ifdef PMAP_DEBUG 858 printf("pmap_kenter: va: %p -> pa: %p\n", (void *)va, (void *)pa); 859#endif 860 npte = TLBLO_PA_TO_PFN(pa) | PTE_D | PTE_V | PTE_G | PTE_W; 861 862 if (is_cacheable_mem(pa)) 863 npte |= PTE_C_CACHE; 864 else 865 npte |= PTE_C_UNCACHED; 866 867 pte = pmap_pte(kernel_pmap, va); 868 opte = *pte; 869 *pte = npte; 870 if (pte_test(&opte, PTE_V) && opte != npte) 871 pmap_update_page(kernel_pmap, va, npte); 872} 873 874/* 875 * remove a page from the kernel pagetables 876 */ 877 /* PMAP_INLINE */ void 878pmap_kremove(vm_offset_t va) 879{ 880 pt_entry_t *pte; 881 882 /* 883 * Write back all caches from the page being destroyed 884 */ 885 mips_dcache_wbinv_range_index(va, PAGE_SIZE); 886 887 pte = pmap_pte(kernel_pmap, va); 888 *pte = PTE_G; 889 pmap_invalidate_page(kernel_pmap, va); 890} 891 892/* 893 * Used to map a range of physical addresses into kernel 894 * virtual address space. 895 * 896 * The value passed in '*virt' is a suggested virtual address for 897 * the mapping. Architectures which can support a direct-mapped 898 * physical to virtual region can return the appropriate address 899 * within that region, leaving '*virt' unchanged. Other 900 * architectures should map the pages starting at '*virt' and 901 * update '*virt' with the first usable address after the mapped 902 * region. 903 * 904 * Use XKPHYS for 64 bit, and KSEG0 where possible for 32 bit. 905 */ 906vm_offset_t 907pmap_map(vm_offset_t *virt, vm_offset_t start, vm_offset_t end, int prot) 908{ 909 vm_offset_t va, sva; 910 911 if (MIPS_DIRECT_MAPPABLE(end)) 912 return (MIPS_PHYS_TO_DIRECT(start)); 913 914 va = sva = *virt; 915 while (start < end) { 916 pmap_kenter(va, start); 917 va += PAGE_SIZE; 918 start += PAGE_SIZE; 919 } 920 *virt = va; 921 return (sva); 922} 923 924/* 925 * Add a list of wired pages to the kva 926 * this routine is only used for temporary 927 * kernel mappings that do not need to have 928 * page modification or references recorded. 929 * Note that old mappings are simply written 930 * over. The page *must* be wired. 931 */ 932void 933pmap_qenter(vm_offset_t va, vm_page_t *m, int count) 934{ 935 int i; 936 vm_offset_t origva = va; 937 938 for (i = 0; i < count; i++) { 939 pmap_flush_pvcache(m[i]); 940 pmap_kenter(va, VM_PAGE_TO_PHYS(m[i])); 941 va += PAGE_SIZE; 942 } 943 944 mips_dcache_wbinv_range_index(origva, PAGE_SIZE*count); 945} 946 947/* 948 * this routine jerks page mappings from the 949 * kernel -- it is meant only for temporary mappings. 950 */ 951void 952pmap_qremove(vm_offset_t va, int count) 953{ 954 /* 955 * No need to wb/inv caches here, 956 * pmap_kremove will do it for us 957 */ 958 959 while (count-- > 0) { 960 pmap_kremove(va); 961 va += PAGE_SIZE; 962 } 963} 964 965/*************************************************** 966 * Page table page management routines..... 967 ***************************************************/ 968 969/* Revision 1.507 970 * 971 * Simplify the reference counting of page table pages. Specifically, use 972 * the page table page's wired count rather than its hold count to contain 973 * the reference count. 974 */ 975 976/* 977 * This routine unholds page table pages, and if the hold count 978 * drops to zero, then it decrements the wire count. 979 */ 980static PMAP_INLINE int 981pmap_unwire_pte_hold(pmap_t pmap, vm_offset_t va, vm_page_t m) 982{ 983 --m->wire_count; 984 if (m->wire_count == 0) 985 return (_pmap_unwire_pte_hold(pmap, va, m)); 986 else 987 return (0); 988} 989 990static int 991_pmap_unwire_pte_hold(pmap_t pmap, vm_offset_t va, vm_page_t m) 992{ 993 pd_entry_t *pde; 994 995 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 996 /* 997 * unmap the page table page 998 */ 999#ifdef __mips_n64 1000 if (m->pindex < NUPDE) 1001 pde = pmap_pde(pmap, va); 1002 else 1003 pde = pmap_segmap(pmap, va); 1004#else 1005 pde = pmap_pde(pmap, va); 1006#endif 1007 *pde = 0; 1008 pmap->pm_stats.resident_count--; 1009 1010#ifdef __mips_n64 1011 if (m->pindex < NUPDE) { 1012 pd_entry_t *pdp; 1013 vm_page_t pdpg; 1014 1015 /* 1016 * Recursively decrement next level pagetable refcount 1017 */ 1018 pdp = (pd_entry_t *)*pmap_segmap(pmap, va); 1019 pdpg = PHYS_TO_VM_PAGE(MIPS_DIRECT_TO_PHYS(pdp)); 1020 pmap_unwire_pte_hold(pmap, va, pdpg); 1021 } 1022#endif 1023 if (pmap->pm_ptphint == m) 1024 pmap->pm_ptphint = NULL; 1025 1026 /* 1027 * If the page is finally unwired, simply free it. 1028 */ 1029 vm_page_free_zero(m); 1030 atomic_subtract_int(&cnt.v_wire_count, 1); 1031 return (1); 1032} 1033 1034/* 1035 * After removing a page table entry, this routine is used to 1036 * conditionally free the page, and manage the hold/wire counts. 1037 */ 1038static int 1039pmap_unuse_pt(pmap_t pmap, vm_offset_t va, vm_page_t mpte) 1040{ 1041 unsigned ptepindex; 1042 pd_entry_t pteva; 1043 1044 if (va >= VM_MAXUSER_ADDRESS) 1045 return (0); 1046 1047 if (mpte == NULL) { 1048 ptepindex = pmap_pde_pindex(va); 1049 if (pmap->pm_ptphint && 1050 (pmap->pm_ptphint->pindex == ptepindex)) { 1051 mpte = pmap->pm_ptphint; 1052 } else { 1053 pteva = *pmap_pde(pmap, va); 1054 mpte = PHYS_TO_VM_PAGE(MIPS_DIRECT_TO_PHYS(pteva)); 1055 pmap->pm_ptphint = mpte; 1056 } 1057 } 1058 return (pmap_unwire_pte_hold(pmap, va, mpte)); 1059} 1060 1061void 1062pmap_pinit0(pmap_t pmap) 1063{ 1064 int i; 1065 1066 PMAP_LOCK_INIT(pmap); 1067 pmap->pm_segtab = kernel_segmap; 1068 pmap->pm_active = 0; 1069 pmap->pm_ptphint = NULL; 1070 for (i = 0; i < MAXCPU; i++) { 1071 pmap->pm_asid[i].asid = PMAP_ASID_RESERVED; 1072 pmap->pm_asid[i].gen = 0; 1073 } 1074 PCPU_SET(curpmap, pmap); 1075 TAILQ_INIT(&pmap->pm_pvlist); 1076 bzero(&pmap->pm_stats, sizeof pmap->pm_stats); 1077} 1078 1079static void 1080pmap_grow_pte_page_cache() 1081{ 1082 1083#ifdef __mips_n64 1084 vm_contig_grow_cache(3, 0, MIPS_XKPHYS_LARGEST_PHYS); 1085#else 1086 vm_contig_grow_cache(3, 0, MIPS_KSEG0_LARGEST_PHYS); 1087#endif 1088} 1089 1090static vm_page_t 1091pmap_alloc_pte_page(unsigned int index, int req) 1092{ 1093 vm_page_t m; 1094 1095 m = vm_page_alloc_freelist(VM_FREELIST_DIRECT, 0, req); 1096 if (m == NULL) 1097 return (NULL); 1098 1099 if ((m->flags & PG_ZERO) == 0) 1100 pmap_zero_page(m); 1101 1102 m->pindex = index; 1103 atomic_add_int(&cnt.v_wire_count, 1); 1104 m->wire_count = 1; 1105 return (m); 1106} 1107 1108/* 1109 * Initialize a preallocated and zeroed pmap structure, 1110 * such as one in a vmspace structure. 1111 */ 1112int 1113pmap_pinit(pmap_t pmap) 1114{ 1115 vm_offset_t ptdva; 1116 vm_page_t ptdpg; 1117 int i; 1118 1119 PMAP_LOCK_INIT(pmap); 1120 1121 /* 1122 * allocate the page directory page 1123 */ 1124 while ((ptdpg = pmap_alloc_pte_page(NUSERPGTBLS, VM_ALLOC_NORMAL)) == NULL) 1125 pmap_grow_pte_page_cache(); 1126 1127 ptdva = MIPS_PHYS_TO_DIRECT(VM_PAGE_TO_PHYS(ptdpg)); 1128 pmap->pm_segtab = (pd_entry_t *)ptdva; 1129 pmap->pm_active = 0; 1130 pmap->pm_ptphint = NULL; 1131 for (i = 0; i < MAXCPU; i++) { 1132 pmap->pm_asid[i].asid = PMAP_ASID_RESERVED; 1133 pmap->pm_asid[i].gen = 0; 1134 } 1135 TAILQ_INIT(&pmap->pm_pvlist); 1136 bzero(&pmap->pm_stats, sizeof pmap->pm_stats); 1137 1138 return (1); 1139} 1140 1141/* 1142 * this routine is called if the page table page is not 1143 * mapped correctly. 1144 */ 1145static vm_page_t 1146_pmap_allocpte(pmap_t pmap, unsigned ptepindex, int flags) 1147{ 1148 vm_offset_t pageva; 1149 vm_page_t m; 1150 1151 KASSERT((flags & (M_NOWAIT | M_WAITOK)) == M_NOWAIT || 1152 (flags & (M_NOWAIT | M_WAITOK)) == M_WAITOK, 1153 ("_pmap_allocpte: flags is neither M_NOWAIT nor M_WAITOK")); 1154 1155 /* 1156 * Find or fabricate a new pagetable page 1157 */ 1158 if ((m = pmap_alloc_pte_page(ptepindex, VM_ALLOC_NORMAL)) == NULL) { 1159 if (flags & M_WAITOK) { 1160 PMAP_UNLOCK(pmap); 1161 vm_page_unlock_queues(); 1162 pmap_grow_pte_page_cache(); 1163 vm_page_lock_queues(); 1164 PMAP_LOCK(pmap); 1165 } 1166 1167 /* 1168 * Indicate the need to retry. While waiting, the page 1169 * table page may have been allocated. 1170 */ 1171 return (NULL); 1172 } 1173 1174 /* 1175 * Map the pagetable page into the process address space, if it 1176 * isn't already there. 1177 */ 1178 pageva = MIPS_PHYS_TO_DIRECT(VM_PAGE_TO_PHYS(m)); 1179 1180#ifdef __mips_n64 1181 if (ptepindex >= NUPDE) { 1182 pmap->pm_segtab[ptepindex - NUPDE] = (pd_entry_t)pageva; 1183 } else { 1184 pd_entry_t *pdep, *pde; 1185 int segindex = ptepindex >> (SEGSHIFT - PDRSHIFT); 1186 int pdeindex = ptepindex & (NPDEPG - 1); 1187 vm_page_t pg; 1188 1189 pdep = &pmap->pm_segtab[segindex]; 1190 if (*pdep == NULL) { 1191 /* recurse for allocating page dir */ 1192 if (_pmap_allocpte(pmap, NUPDE + segindex, 1193 flags) == NULL) { 1194 /* alloc failed, release current */ 1195 --m->wire_count; 1196 atomic_subtract_int(&cnt.v_wire_count, 1); 1197 vm_page_free_zero(m); 1198 return (NULL); 1199 } 1200 } else { 1201 pg = PHYS_TO_VM_PAGE(MIPS_DIRECT_TO_PHYS(*pdep)); 1202 pg->wire_count++; 1203 } 1204 /* Next level entry */ 1205 pde = (pd_entry_t *)*pdep; 1206 pde[pdeindex] = (pd_entry_t)pageva; 1207 pmap->pm_ptphint = m; 1208 } 1209#else 1210 pmap->pm_segtab[ptepindex] = (pd_entry_t)pageva; 1211#endif 1212 pmap->pm_stats.resident_count++; 1213 1214 /* 1215 * Set the page table hint 1216 */ 1217 pmap->pm_ptphint = m; 1218 return (m); 1219} 1220 1221static vm_page_t 1222pmap_allocpte(pmap_t pmap, vm_offset_t va, int flags) 1223{ 1224 unsigned ptepindex; 1225 pd_entry_t *pde; 1226 vm_page_t m; 1227 1228 KASSERT((flags & (M_NOWAIT | M_WAITOK)) == M_NOWAIT || 1229 (flags & (M_NOWAIT | M_WAITOK)) == M_WAITOK, 1230 ("pmap_allocpte: flags is neither M_NOWAIT nor M_WAITOK")); 1231 1232 /* 1233 * Calculate pagetable page index 1234 */ 1235 ptepindex = pmap_pde_pindex(va); 1236retry: 1237 /* 1238 * Get the page directory entry 1239 */ 1240 pde = pmap_pde(pmap, va); 1241 1242 /* 1243 * If the page table page is mapped, we just increment the hold 1244 * count, and activate it. 1245 */ 1246 if (pde != NULL && *pde != NULL) { 1247 /* 1248 * In order to get the page table page, try the hint first. 1249 */ 1250 if (pmap->pm_ptphint && 1251 (pmap->pm_ptphint->pindex == ptepindex)) { 1252 m = pmap->pm_ptphint; 1253 } else { 1254 m = PHYS_TO_VM_PAGE(MIPS_DIRECT_TO_PHYS(*pde)); 1255 pmap->pm_ptphint = m; 1256 } 1257 m->wire_count++; 1258 } else { 1259 /* 1260 * Here if the pte page isn't mapped, or if it has been 1261 * deallocated. 1262 */ 1263 m = _pmap_allocpte(pmap, ptepindex, flags); 1264 if (m == NULL && (flags & M_WAITOK)) 1265 goto retry; 1266 } 1267 return (m); 1268} 1269 1270 1271/*************************************************** 1272* Pmap allocation/deallocation routines. 1273 ***************************************************/ 1274/* 1275 * Revision 1.397 1276 * - Merged pmap_release and pmap_release_free_page. When pmap_release is 1277 * called only the page directory page(s) can be left in the pmap pte 1278 * object, since all page table pages will have been freed by 1279 * pmap_remove_pages and pmap_remove. In addition, there can only be one 1280 * reference to the pmap and the page directory is wired, so the page(s) 1281 * can never be busy. So all there is to do is clear the magic mappings 1282 * from the page directory and free the page(s). 1283 */ 1284 1285 1286/* 1287 * Release any resources held by the given physical map. 1288 * Called when a pmap initialized by pmap_pinit is being released. 1289 * Should only be called if the map contains no valid mappings. 1290 */ 1291void 1292pmap_release(pmap_t pmap) 1293{ 1294 vm_offset_t ptdva; 1295 vm_page_t ptdpg; 1296 1297 KASSERT(pmap->pm_stats.resident_count == 0, 1298 ("pmap_release: pmap resident count %ld != 0", 1299 pmap->pm_stats.resident_count)); 1300 1301 ptdva = (vm_offset_t)pmap->pm_segtab; 1302 ptdpg = PHYS_TO_VM_PAGE(MIPS_DIRECT_TO_PHYS(ptdva)); 1303 1304 ptdpg->wire_count--; 1305 atomic_subtract_int(&cnt.v_wire_count, 1); 1306 vm_page_free_zero(ptdpg); 1307 PMAP_LOCK_DESTROY(pmap); 1308} 1309 1310/* 1311 * grow the number of kernel page table entries, if needed 1312 */ 1313void 1314pmap_growkernel(vm_offset_t addr) 1315{ 1316 vm_page_t nkpg; 1317 pd_entry_t *pde, *pdpe; 1318 pt_entry_t *pte; 1319 int i; 1320 1321 mtx_assert(&kernel_map->system_mtx, MA_OWNED); 1322 addr = roundup2(addr, NBSEG); 1323 if (addr - 1 >= kernel_map->max_offset) 1324 addr = kernel_map->max_offset; 1325 while (kernel_vm_end < addr) { 1326 pdpe = pmap_segmap(kernel_pmap, kernel_vm_end); 1327#ifdef __mips_n64 1328 if (*pdpe == 0) { 1329 /* new intermediate page table entry */ 1330 nkpg = pmap_alloc_pte_page(nkpt, VM_ALLOC_INTERRUPT); 1331 if (nkpg == NULL) 1332 panic("pmap_growkernel: no memory to grow kernel"); 1333 *pdpe = (pd_entry_t)MIPS_PHYS_TO_DIRECT(VM_PAGE_TO_PHYS(nkpg)); 1334 continue; /* try again */ 1335 } 1336#endif 1337 pde = pmap_pdpe_to_pde(pdpe, kernel_vm_end); 1338 if (*pde != 0) { 1339 kernel_vm_end = (kernel_vm_end + NBPDR) & ~PDRMASK; 1340 if (kernel_vm_end - 1 >= kernel_map->max_offset) { 1341 kernel_vm_end = kernel_map->max_offset; 1342 break; 1343 } 1344 continue; 1345 } 1346 1347 /* 1348 * This index is bogus, but out of the way 1349 */ 1350 nkpg = pmap_alloc_pte_page(nkpt, VM_ALLOC_INTERRUPT); 1351 if (!nkpg) 1352 panic("pmap_growkernel: no memory to grow kernel"); 1353 nkpt++; 1354 *pde = (pd_entry_t)MIPS_PHYS_TO_DIRECT(VM_PAGE_TO_PHYS(nkpg)); 1355 1356 /* 1357 * The R[4-7]?00 stores only one copy of the Global bit in 1358 * the translation lookaside buffer for each 2 page entry. 1359 * Thus invalid entrys must have the Global bit set so when 1360 * Entry LO and Entry HI G bits are anded together they will 1361 * produce a global bit to store in the tlb. 1362 */ 1363 pte = (pt_entry_t *)*pde; 1364 for (i = 0; i < NPTEPG; i++) 1365 pte[i] = PTE_G; 1366 1367 kernel_vm_end = (kernel_vm_end + NBPDR) & ~PDRMASK; 1368 if (kernel_vm_end - 1 >= kernel_map->max_offset) { 1369 kernel_vm_end = kernel_map->max_offset; 1370 break; 1371 } 1372 } 1373} 1374 1375/*************************************************** 1376* page management routines. 1377 ***************************************************/ 1378 1379/* 1380 * free the pv_entry back to the free list 1381 */ 1382static PMAP_INLINE void 1383free_pv_entry(pv_entry_t pv) 1384{ 1385 1386 pv_entry_count--; 1387 uma_zfree(pvzone, pv); 1388} 1389 1390/* 1391 * get a new pv_entry, allocating a block from the system 1392 * when needed. 1393 * the memory allocation is performed bypassing the malloc code 1394 * because of the possibility of allocations at interrupt time. 1395 */ 1396static pv_entry_t 1397get_pv_entry(pmap_t locked_pmap) 1398{ 1399 static const struct timeval printinterval = { 60, 0 }; 1400 static struct timeval lastprint; 1401 struct vpgqueues *vpq; 1402 pt_entry_t *pte, oldpte; 1403 pmap_t pmap; 1404 pv_entry_t allocated_pv, next_pv, pv; 1405 vm_offset_t va; 1406 vm_page_t m; 1407 1408 PMAP_LOCK_ASSERT(locked_pmap, MA_OWNED); 1409 mtx_assert(&vm_page_queue_mtx, MA_OWNED); 1410 allocated_pv = uma_zalloc(pvzone, M_NOWAIT); 1411 if (allocated_pv != NULL) { 1412 pv_entry_count++; 1413 if (pv_entry_count > pv_entry_high_water) 1414 pagedaemon_wakeup(); 1415 else 1416 return (allocated_pv); 1417 } 1418 /* 1419 * Reclaim pv entries: At first, destroy mappings to inactive 1420 * pages. After that, if a pv entry is still needed, destroy 1421 * mappings to active pages. 1422 */ 1423 if (ratecheck(&lastprint, &printinterval)) 1424 printf("Approaching the limit on PV entries, " 1425 "increase the vm.pmap.shpgperproc tunable.\n"); 1426 vpq = &vm_page_queues[PQ_INACTIVE]; 1427retry: 1428 TAILQ_FOREACH(m, &vpq->pl, pageq) { 1429 if (m->hold_count || m->busy) 1430 continue; 1431 TAILQ_FOREACH_SAFE(pv, &m->md.pv_list, pv_list, next_pv) { 1432 va = pv->pv_va; 1433 pmap = pv->pv_pmap; 1434 /* Avoid deadlock and lock recursion. */ 1435 if (pmap > locked_pmap) 1436 PMAP_LOCK(pmap); 1437 else if (pmap != locked_pmap && !PMAP_TRYLOCK(pmap)) 1438 continue; 1439 pmap->pm_stats.resident_count--; 1440 pte = pmap_pte(pmap, va); 1441 KASSERT(pte != NULL, ("pte")); 1442 oldpte = *pte; 1443 if (is_kernel_pmap(pmap)) 1444 *pte = PTE_G; 1445 else 1446 *pte = 0; 1447 KASSERT(!pte_test(&oldpte, PTE_W), 1448 ("wired pte for unwired page")); 1449 if (m->md.pv_flags & PV_TABLE_REF) 1450 vm_page_flag_set(m, PG_REFERENCED); 1451 if (pte_test(&oldpte, PTE_D)) 1452 vm_page_dirty(m); 1453 pmap_invalidate_page(pmap, va); 1454 TAILQ_REMOVE(&pmap->pm_pvlist, pv, pv_plist); 1455 m->md.pv_list_count--; 1456 TAILQ_REMOVE(&m->md.pv_list, pv, pv_list); 1457 pmap_unuse_pt(pmap, va, pv->pv_ptem); 1458 if (pmap != locked_pmap) 1459 PMAP_UNLOCK(pmap); 1460 if (allocated_pv == NULL) 1461 allocated_pv = pv; 1462 else 1463 free_pv_entry(pv); 1464 } 1465 if (TAILQ_EMPTY(&m->md.pv_list)) { 1466 vm_page_flag_clear(m, PG_WRITEABLE); 1467 m->md.pv_flags &= ~(PV_TABLE_REF | PV_TABLE_MOD); 1468 } 1469 } 1470 if (allocated_pv == NULL) { 1471 if (vpq == &vm_page_queues[PQ_INACTIVE]) { 1472 vpq = &vm_page_queues[PQ_ACTIVE]; 1473 goto retry; 1474 } 1475 panic("get_pv_entry: increase the vm.pmap.shpgperproc tunable"); 1476 } 1477 return (allocated_pv); 1478} 1479 1480/* 1481 * Revision 1.370 1482 * 1483 * Move pmap_collect() out of the machine-dependent code, rename it 1484 * to reflect its new location, and add page queue and flag locking. 1485 * 1486 * Notes: (1) alpha, i386, and ia64 had identical implementations 1487 * of pmap_collect() in terms of machine-independent interfaces; 1488 * (2) sparc64 doesn't require it; (3) powerpc had it as a TODO. 1489 * 1490 * MIPS implementation was identical to alpha [Junos 8.2] 1491 */ 1492 1493/* 1494 * If it is the first entry on the list, it is actually 1495 * in the header and we must copy the following entry up 1496 * to the header. Otherwise we must search the list for 1497 * the entry. In either case we free the now unused entry. 1498 */ 1499 1500static pv_entry_t 1501pmap_pvh_remove(struct md_page *pvh, pmap_t pmap, vm_offset_t va) 1502{ 1503 pv_entry_t pv; 1504 1505 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 1506 mtx_assert(&vm_page_queue_mtx, MA_OWNED); 1507 if (pvh->pv_list_count < pmap->pm_stats.resident_count) { 1508 TAILQ_FOREACH(pv, &pvh->pv_list, pv_list) { 1509 if (pmap == pv->pv_pmap && va == pv->pv_va) 1510 break; 1511 } 1512 } else { 1513 TAILQ_FOREACH(pv, &pmap->pm_pvlist, pv_plist) { 1514 if (va == pv->pv_va) 1515 break; 1516 } 1517 } 1518 if (pv != NULL) { 1519 TAILQ_REMOVE(&pvh->pv_list, pv, pv_list); 1520 pvh->pv_list_count--; 1521 TAILQ_REMOVE(&pmap->pm_pvlist, pv, pv_plist); 1522 } 1523 return (pv); 1524} 1525 1526static void 1527pmap_pvh_free(struct md_page *pvh, pmap_t pmap, vm_offset_t va) 1528{ 1529 pv_entry_t pv; 1530 1531 pv = pmap_pvh_remove(pvh, pmap, va); 1532 KASSERT(pv != NULL, ("pmap_pvh_free: pv not found, pa %lx va %lx", 1533 (u_long)VM_PAGE_TO_PHYS(member2struct(vm_page, md, pvh)), 1534 (u_long)va)); 1535 free_pv_entry(pv); 1536} 1537 1538static void 1539pmap_remove_entry(pmap_t pmap, vm_page_t m, vm_offset_t va) 1540{ 1541 1542 mtx_assert(&vm_page_queue_mtx, MA_OWNED); 1543 pmap_pvh_free(&m->md, pmap, va); 1544 if (TAILQ_EMPTY(&m->md.pv_list)) 1545 vm_page_flag_clear(m, PG_WRITEABLE); 1546} 1547 1548/* 1549 * Conditionally create a pv entry. 1550 */ 1551static boolean_t 1552pmap_try_insert_pv_entry(pmap_t pmap, vm_page_t mpte, vm_offset_t va, 1553 vm_page_t m) 1554{ 1555 pv_entry_t pv; 1556 1557 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 1558 mtx_assert(&vm_page_queue_mtx, MA_OWNED); 1559 if (pv_entry_count < pv_entry_high_water && 1560 (pv = uma_zalloc(pvzone, M_NOWAIT)) != NULL) { 1561 pv_entry_count++; 1562 pv->pv_va = va; 1563 pv->pv_pmap = pmap; 1564 pv->pv_ptem = mpte; 1565 TAILQ_INSERT_TAIL(&pmap->pm_pvlist, pv, pv_plist); 1566 TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_list); 1567 m->md.pv_list_count++; 1568 return (TRUE); 1569 } else 1570 return (FALSE); 1571} 1572 1573/* 1574 * pmap_remove_pte: do the things to unmap a page in a process 1575 */ 1576static int 1577pmap_remove_pte(struct pmap *pmap, pt_entry_t *ptq, vm_offset_t va) 1578{ 1579 pt_entry_t oldpte; 1580 vm_page_t m; 1581 vm_offset_t pa; 1582 1583 mtx_assert(&vm_page_queue_mtx, MA_OWNED); 1584 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 1585 1586 oldpte = *ptq; 1587 if (is_kernel_pmap(pmap)) 1588 *ptq = PTE_G; 1589 else 1590 *ptq = 0; 1591 1592 if (pte_test(&oldpte, PTE_W)) 1593 pmap->pm_stats.wired_count -= 1; 1594 1595 pmap->pm_stats.resident_count -= 1; 1596 pa = TLBLO_PTE_TO_PA(oldpte); 1597 1598 if (page_is_managed(pa)) { 1599 m = PHYS_TO_VM_PAGE(pa); 1600 if (pte_test(&oldpte, PTE_D)) { 1601#if defined(PMAP_DIAGNOSTIC) 1602 if (pmap_nw_modified(oldpte)) { 1603 printf( 1604 "pmap_remove: modified page not writable: va: 0x%x, pte: 0x%x\n", 1605 va, oldpte); 1606 } 1607#endif 1608 vm_page_dirty(m); 1609 } 1610 if (m->md.pv_flags & PV_TABLE_REF) 1611 vm_page_flag_set(m, PG_REFERENCED); 1612 m->md.pv_flags &= ~(PV_TABLE_REF | PV_TABLE_MOD); 1613 1614 pmap_remove_entry(pmap, m, va); 1615 } 1616 return (pmap_unuse_pt(pmap, va, NULL)); 1617} 1618 1619/* 1620 * Remove a single page from a process address space 1621 */ 1622static void 1623pmap_remove_page(struct pmap *pmap, vm_offset_t va) 1624{ 1625 pt_entry_t *ptq; 1626 1627 mtx_assert(&vm_page_queue_mtx, MA_OWNED); 1628 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 1629 ptq = pmap_pte(pmap, va); 1630 1631 /* 1632 * if there is no pte for this address, just skip it!!! 1633 */ 1634 if (!ptq || !pte_test(ptq, PTE_V)) { 1635 return; 1636 } 1637 1638 /* 1639 * Write back all caches from the page being destroyed 1640 */ 1641 mips_dcache_wbinv_range_index(va, PAGE_SIZE); 1642 1643 /* 1644 * get a local va for mappings for this pmap. 1645 */ 1646 (void)pmap_remove_pte(pmap, ptq, va); 1647 pmap_invalidate_page(pmap, va); 1648 1649 return; 1650} 1651 1652/* 1653 * Remove the given range of addresses from the specified map. 1654 * 1655 * It is assumed that the start and end are properly 1656 * rounded to the page size. 1657 */ 1658void 1659pmap_remove(struct pmap *pmap, vm_offset_t sva, vm_offset_t eva) 1660{ 1661 vm_offset_t va_next; 1662 pd_entry_t *pde, *pdpe; 1663 pt_entry_t *pte; 1664 1665 if (pmap == NULL) 1666 return; 1667 1668 if (pmap->pm_stats.resident_count == 0) 1669 return; 1670 1671 vm_page_lock_queues(); 1672 PMAP_LOCK(pmap); 1673 1674 /* 1675 * special handling of removing one page. a very common operation 1676 * and easy to short circuit some code. 1677 */ 1678 if ((sva + PAGE_SIZE) == eva) { 1679 pmap_remove_page(pmap, sva); 1680 goto out; 1681 } 1682 for (; sva < eva; sva = va_next) { 1683 pdpe = pmap_segmap(pmap, sva); 1684#ifdef __mips_n64 1685 if (*pdpe == 0) { 1686 va_next = (sva + NBSEG) & ~SEGMASK; 1687 if (va_next < sva) 1688 va_next = eva; 1689 continue; 1690 } 1691#endif 1692 va_next = (sva + NBPDR) & ~PDRMASK; 1693 if (va_next < sva) 1694 va_next = eva; 1695 1696 pde = pmap_pdpe_to_pde(pdpe, sva); 1697 if (*pde == 0) 1698 continue; 1699 if (va_next > eva) 1700 va_next = eva; 1701 for (pte = pmap_pde_to_pte(pde, sva); sva != va_next; 1702 pte++, sva += PAGE_SIZE) { 1703 pmap_remove_page(pmap, sva); 1704 } 1705 } 1706out: 1707 vm_page_unlock_queues(); 1708 PMAP_UNLOCK(pmap); 1709} 1710 1711/* 1712 * Routine: pmap_remove_all 1713 * Function: 1714 * Removes this physical page from 1715 * all physical maps in which it resides. 1716 * Reflects back modify bits to the pager. 1717 * 1718 * Notes: 1719 * Original versions of this routine were very 1720 * inefficient because they iteratively called 1721 * pmap_remove (slow...) 1722 */ 1723 1724void 1725pmap_remove_all(vm_page_t m) 1726{ 1727 pv_entry_t pv; 1728 pt_entry_t *pte, tpte; 1729 1730 KASSERT((m->flags & PG_FICTITIOUS) == 0, 1731 ("pmap_remove_all: page %p is fictitious", m)); 1732 vm_page_lock_queues(); 1733 1734 if (m->md.pv_flags & PV_TABLE_REF) 1735 vm_page_flag_set(m, PG_REFERENCED); 1736 1737 while ((pv = TAILQ_FIRST(&m->md.pv_list)) != NULL) { 1738 PMAP_LOCK(pv->pv_pmap); 1739 1740 /* 1741 * If it's last mapping writeback all caches from 1742 * the page being destroyed 1743 */ 1744 if (m->md.pv_list_count == 1) 1745 mips_dcache_wbinv_range_index(pv->pv_va, PAGE_SIZE); 1746 1747 pv->pv_pmap->pm_stats.resident_count--; 1748 1749 pte = pmap_pte(pv->pv_pmap, pv->pv_va); 1750 1751 tpte = *pte; 1752 if (is_kernel_pmap(pv->pv_pmap)) 1753 *pte = PTE_G; 1754 else 1755 *pte = 0; 1756 1757 if (pte_test(&tpte, PTE_W)) 1758 pv->pv_pmap->pm_stats.wired_count--; 1759 1760 /* 1761 * Update the vm_page_t clean and reference bits. 1762 */ 1763 if (pte_test(&tpte, PTE_D)) { 1764#if defined(PMAP_DIAGNOSTIC) 1765 if (pmap_nw_modified(tpte)) { 1766 printf( 1767 "pmap_remove_all: modified page not writable: va: 0x%x, pte: 0x%x\n", 1768 pv->pv_va, tpte); 1769 } 1770#endif 1771 vm_page_dirty(m); 1772 } 1773 pmap_invalidate_page(pv->pv_pmap, pv->pv_va); 1774 1775 TAILQ_REMOVE(&pv->pv_pmap->pm_pvlist, pv, pv_plist); 1776 TAILQ_REMOVE(&m->md.pv_list, pv, pv_list); 1777 m->md.pv_list_count--; 1778 pmap_unuse_pt(pv->pv_pmap, pv->pv_va, pv->pv_ptem); 1779 PMAP_UNLOCK(pv->pv_pmap); 1780 free_pv_entry(pv); 1781 } 1782 1783 vm_page_flag_clear(m, PG_WRITEABLE); 1784 m->md.pv_flags &= ~(PV_TABLE_REF | PV_TABLE_MOD); 1785 vm_page_unlock_queues(); 1786} 1787 1788/* 1789 * Set the physical protection on the 1790 * specified range of this map as requested. 1791 */ 1792void 1793pmap_protect(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, vm_prot_t prot) 1794{ 1795 pt_entry_t *pte; 1796 pd_entry_t *pde, *pdpe; 1797 vm_offset_t va_next; 1798 1799 if (pmap == NULL) 1800 return; 1801 1802 if ((prot & VM_PROT_READ) == VM_PROT_NONE) { 1803 pmap_remove(pmap, sva, eva); 1804 return; 1805 } 1806 if (prot & VM_PROT_WRITE) 1807 return; 1808 1809 vm_page_lock_queues(); 1810 PMAP_LOCK(pmap); 1811 for (; sva < eva; sva = va_next) { 1812 pt_entry_t pbits; 1813 vm_page_t m; 1814 vm_paddr_t pa; 1815 1816 pdpe = pmap_segmap(pmap, sva); 1817#ifdef __mips_n64 1818 if (*pdpe == 0) { 1819 va_next = (sva + NBSEG) & ~SEGMASK; 1820 if (va_next < sva) 1821 va_next = eva; 1822 continue; 1823 } 1824#endif 1825 va_next = (sva + NBPDR) & ~PDRMASK; 1826 if (va_next < sva) 1827 va_next = eva; 1828 1829 pde = pmap_pdpe_to_pde(pdpe, sva); 1830 if (pde == NULL || *pde == NULL) 1831 continue; 1832 if (va_next > eva) 1833 va_next = eva; 1834 1835 for (pte = pmap_pde_to_pte(pde, sva); sva != va_next; pte++, 1836 sva += PAGE_SIZE) { 1837 1838 /* Skip invalid PTEs */ 1839 if (!pte_test(pte, PTE_V)) 1840 continue; 1841 pbits = *pte; 1842 pa = TLBLO_PTE_TO_PA(pbits); 1843 if (page_is_managed(pa) && pte_test(&pbits, PTE_D)) { 1844 m = PHYS_TO_VM_PAGE(pa); 1845 vm_page_dirty(m); 1846 m->md.pv_flags &= ~PV_TABLE_MOD; 1847 } 1848 pte_clear(&pbits, PTE_D); 1849 pte_set(&pbits, PTE_RO); 1850 1851 if (pbits != *pte) { 1852 *pte = pbits; 1853 pmap_update_page(pmap, sva, pbits); 1854 } 1855 } 1856 } 1857 vm_page_unlock_queues(); 1858 PMAP_UNLOCK(pmap); 1859} 1860 1861/* 1862 * Insert the given physical page (p) at 1863 * the specified virtual address (v) in the 1864 * target physical map with the protection requested. 1865 * 1866 * If specified, the page will be wired down, meaning 1867 * that the related pte can not be reclaimed. 1868 * 1869 * NB: This is the only routine which MAY NOT lazy-evaluate 1870 * or lose information. That is, this routine must actually 1871 * insert this page into the given map NOW. 1872 */ 1873void 1874pmap_enter(pmap_t pmap, vm_offset_t va, vm_prot_t access, vm_page_t m, 1875 vm_prot_t prot, boolean_t wired) 1876{ 1877 vm_offset_t pa, opa; 1878 pt_entry_t *pte; 1879 pt_entry_t origpte, newpte; 1880 pv_entry_t pv; 1881 vm_page_t mpte, om; 1882 int rw = 0; 1883 1884 if (pmap == NULL) 1885 return; 1886 1887 va &= ~PAGE_MASK; 1888 KASSERT(va <= VM_MAX_KERNEL_ADDRESS, ("pmap_enter: toobig")); 1889 KASSERT((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) != 0 || 1890 (m->oflags & VPO_BUSY) != 0, 1891 ("pmap_enter: page %p is not busy", m)); 1892 1893 mpte = NULL; 1894 1895 vm_page_lock_queues(); 1896 PMAP_LOCK(pmap); 1897 1898 /* 1899 * In the case that a page table page is not resident, we are 1900 * creating it here. 1901 */ 1902 if (va < VM_MAXUSER_ADDRESS) { 1903 mpte = pmap_allocpte(pmap, va, M_WAITOK); 1904 } 1905 pte = pmap_pte(pmap, va); 1906 1907 /* 1908 * Page Directory table entry not valid, we need a new PT page 1909 */ 1910 if (pte == NULL) { 1911 panic("pmap_enter: invalid page directory, pdir=%p, va=%p\n", 1912 (void *)pmap->pm_segtab, (void *)va); 1913 } 1914 pa = VM_PAGE_TO_PHYS(m); 1915 om = NULL; 1916 origpte = *pte; 1917 opa = TLBLO_PTE_TO_PA(origpte); 1918 1919 /* 1920 * Mapping has not changed, must be protection or wiring change. 1921 */ 1922 if (pte_test(&origpte, PTE_V) && opa == pa) { 1923 /* 1924 * Wiring change, just update stats. We don't worry about 1925 * wiring PT pages as they remain resident as long as there 1926 * are valid mappings in them. Hence, if a user page is 1927 * wired, the PT page will be also. 1928 */ 1929 if (wired && !pte_test(&origpte, PTE_W)) 1930 pmap->pm_stats.wired_count++; 1931 else if (!wired && pte_test(&origpte, PTE_W)) 1932 pmap->pm_stats.wired_count--; 1933 1934#if defined(PMAP_DIAGNOSTIC) 1935 if (pmap_nw_modified(origpte)) { 1936 printf( 1937 "pmap_enter: modified page not writable: va: 0x%x, pte: 0x%x\n", 1938 va, origpte); 1939 } 1940#endif 1941 1942 /* 1943 * Remove extra pte reference 1944 */ 1945 if (mpte) 1946 mpte->wire_count--; 1947 1948 if (page_is_managed(opa)) { 1949 om = m; 1950 } 1951 goto validate; 1952 } 1953 1954 pv = NULL; 1955 1956 /* 1957 * Mapping has changed, invalidate old range and fall through to 1958 * handle validating new mapping. 1959 */ 1960 if (opa) { 1961 if (pte_test(&origpte, PTE_W)) 1962 pmap->pm_stats.wired_count--; 1963 1964 if (page_is_managed(opa)) { 1965 om = PHYS_TO_VM_PAGE(opa); 1966 pv = pmap_pvh_remove(&om->md, pmap, va); 1967 } 1968 if (mpte != NULL) { 1969 mpte->wire_count--; 1970 KASSERT(mpte->wire_count > 0, 1971 ("pmap_enter: missing reference to page table page," 1972 " va: %p", (void *)va)); 1973 } 1974 } else 1975 pmap->pm_stats.resident_count++; 1976 1977 /* 1978 * Enter on the PV list if part of our managed memory. Note that we 1979 * raise IPL while manipulating pv_table since pmap_enter can be 1980 * called at interrupt time. 1981 */ 1982 if ((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) == 0) { 1983 KASSERT(va < kmi.clean_sva || va >= kmi.clean_eva, 1984 ("pmap_enter: managed mapping within the clean submap")); 1985 if (pv == NULL) 1986 pv = get_pv_entry(pmap); 1987 pv->pv_va = va; 1988 pv->pv_pmap = pmap; 1989 pv->pv_ptem = mpte; 1990 TAILQ_INSERT_TAIL(&pmap->pm_pvlist, pv, pv_plist); 1991 TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_list); 1992 m->md.pv_list_count++; 1993 } else if (pv != NULL) 1994 free_pv_entry(pv); 1995 1996 /* 1997 * Increment counters 1998 */ 1999 if (wired) 2000 pmap->pm_stats.wired_count++; 2001 2002validate: 2003 if ((access & VM_PROT_WRITE) != 0) 2004 m->md.pv_flags |= PV_TABLE_MOD | PV_TABLE_REF; 2005 rw = init_pte_prot(va, m, prot); 2006 2007#ifdef PMAP_DEBUG 2008 printf("pmap_enter: va: %p -> pa: %p\n", (void *)va, (void *)pa); 2009#endif 2010 /* 2011 * Now validate mapping with desired protection/wiring. 2012 */ 2013 newpte = TLBLO_PA_TO_PFN(pa) | rw | PTE_V; 2014 2015 if (is_cacheable_mem(pa)) 2016 newpte |= PTE_C_CACHE; 2017 else 2018 newpte |= PTE_C_UNCACHED; 2019 2020 if (wired) 2021 newpte |= PTE_W; 2022 2023 if (is_kernel_pmap(pmap)) 2024 newpte |= PTE_G; 2025 2026 /* 2027 * if the mapping or permission bits are different, we need to 2028 * update the pte. 2029 */ 2030 if (origpte != newpte) { 2031 if (pte_test(&origpte, PTE_V)) { 2032 *pte = newpte; 2033 if (page_is_managed(opa) && (opa != pa)) { 2034 if (om->md.pv_flags & PV_TABLE_REF) 2035 vm_page_flag_set(om, PG_REFERENCED); 2036 om->md.pv_flags &= 2037 ~(PV_TABLE_REF | PV_TABLE_MOD); 2038 } 2039 if (pte_test(&origpte, PTE_D)) { 2040 KASSERT(!pte_test(&origpte, PTE_RO), 2041 ("pmap_enter: modified page not writable:" 2042 " va: %p, pte: 0x%x", (void *)va, origpte)); 2043 if (page_is_managed(opa)) 2044 vm_page_dirty(om); 2045 } 2046 if (page_is_managed(opa) && 2047 TAILQ_EMPTY(&om->md.pv_list)) 2048 vm_page_flag_clear(om, PG_WRITEABLE); 2049 } else { 2050 *pte = newpte; 2051 } 2052 } 2053 pmap_update_page(pmap, va, newpte); 2054 2055 /* 2056 * Sync I & D caches for executable pages. Do this only if the the 2057 * target pmap belongs to the current process. Otherwise, an 2058 * unresolvable TLB miss may occur. 2059 */ 2060 if (!is_kernel_pmap(pmap) && (pmap == &curproc->p_vmspace->vm_pmap) && 2061 (prot & VM_PROT_EXECUTE)) { 2062 mips_icache_sync_range(va, PAGE_SIZE); 2063 mips_dcache_wbinv_range(va, PAGE_SIZE); 2064 } 2065 vm_page_unlock_queues(); 2066 PMAP_UNLOCK(pmap); 2067} 2068 2069/* 2070 * this code makes some *MAJOR* assumptions: 2071 * 1. Current pmap & pmap exists. 2072 * 2. Not wired. 2073 * 3. Read access. 2074 * 4. No page table pages. 2075 * but is *MUCH* faster than pmap_enter... 2076 */ 2077 2078void 2079pmap_enter_quick(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot) 2080{ 2081 2082 vm_page_lock_queues(); 2083 PMAP_LOCK(pmap); 2084 (void)pmap_enter_quick_locked(pmap, va, m, prot, NULL); 2085 vm_page_unlock_queues(); 2086 PMAP_UNLOCK(pmap); 2087} 2088 2089static vm_page_t 2090pmap_enter_quick_locked(pmap_t pmap, vm_offset_t va, vm_page_t m, 2091 vm_prot_t prot, vm_page_t mpte) 2092{ 2093 pt_entry_t *pte; 2094 vm_offset_t pa; 2095 2096 KASSERT(va < kmi.clean_sva || va >= kmi.clean_eva || 2097 (m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) != 0, 2098 ("pmap_enter_quick_locked: managed mapping within the clean submap")); 2099 mtx_assert(&vm_page_queue_mtx, MA_OWNED); 2100 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 2101 2102 /* 2103 * In the case that a page table page is not resident, we are 2104 * creating it here. 2105 */ 2106 if (va < VM_MAXUSER_ADDRESS) { 2107 pd_entry_t *pde; 2108 unsigned ptepindex; 2109 2110 /* 2111 * Calculate pagetable page index 2112 */ 2113 ptepindex = pmap_pde_pindex(va); 2114 if (mpte && (mpte->pindex == ptepindex)) { 2115 mpte->wire_count++; 2116 } else { 2117 /* 2118 * Get the page directory entry 2119 */ 2120 pde = pmap_pde(pmap, va); 2121 2122 /* 2123 * If the page table page is mapped, we just 2124 * increment the hold count, and activate it. 2125 */ 2126 if (pde && *pde != 0) { 2127 if (pmap->pm_ptphint && 2128 (pmap->pm_ptphint->pindex == ptepindex)) { 2129 mpte = pmap->pm_ptphint; 2130 } else { 2131 mpte = PHYS_TO_VM_PAGE( 2132 MIPS_DIRECT_TO_PHYS(*pde)); 2133 pmap->pm_ptphint = mpte; 2134 } 2135 mpte->wire_count++; 2136 } else { 2137 mpte = _pmap_allocpte(pmap, ptepindex, 2138 M_NOWAIT); 2139 if (mpte == NULL) 2140 return (mpte); 2141 } 2142 } 2143 } else { 2144 mpte = NULL; 2145 } 2146 2147 pte = pmap_pte(pmap, va); 2148 if (pte_test(pte, PTE_V)) { 2149 if (mpte != NULL) { 2150 mpte->wire_count--; 2151 mpte = NULL; 2152 } 2153 return (mpte); 2154 } 2155 2156 /* 2157 * Enter on the PV list if part of our managed memory. 2158 */ 2159 if ((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) == 0 && 2160 !pmap_try_insert_pv_entry(pmap, mpte, va, m)) { 2161 if (mpte != NULL) { 2162 pmap_unwire_pte_hold(pmap, va, mpte); 2163 mpte = NULL; 2164 } 2165 return (mpte); 2166 } 2167 2168 /* 2169 * Increment counters 2170 */ 2171 pmap->pm_stats.resident_count++; 2172 2173 pa = VM_PAGE_TO_PHYS(m); 2174 2175 /* 2176 * Now validate mapping with RO protection 2177 */ 2178 *pte = TLBLO_PA_TO_PFN(pa) | PTE_V; 2179 2180 if (is_cacheable_mem(pa)) 2181 *pte |= PTE_C_CACHE; 2182 else 2183 *pte |= PTE_C_UNCACHED; 2184 2185 if (is_kernel_pmap(pmap)) 2186 *pte |= PTE_G; 2187 else { 2188 *pte |= PTE_RO; 2189 /* 2190 * Sync I & D caches. Do this only if the the target pmap 2191 * belongs to the current process. Otherwise, an 2192 * unresolvable TLB miss may occur. */ 2193 if (pmap == &curproc->p_vmspace->vm_pmap) { 2194 va &= ~PAGE_MASK; 2195 mips_icache_sync_range(va, PAGE_SIZE); 2196 mips_dcache_wbinv_range(va, PAGE_SIZE); 2197 } 2198 } 2199 return (mpte); 2200} 2201 2202/* 2203 * Make a temporary mapping for a physical address. This is only intended 2204 * to be used for panic dumps. 2205 * 2206 * Use XKPHYS for 64 bit, and KSEG0 where possible for 32 bit. 2207 */ 2208void * 2209pmap_kenter_temporary(vm_paddr_t pa, int i) 2210{ 2211 vm_offset_t va; 2212 2213 if (i != 0) 2214 printf("%s: ERROR!!! More than one page of virtual address mapping not supported\n", 2215 __func__); 2216 2217 if (MIPS_DIRECT_MAPPABLE(pa)) { 2218 va = MIPS_PHYS_TO_DIRECT(pa); 2219 } else { 2220#ifndef __mips_n64 /* XXX : to be converted to new style */ 2221 int cpu; 2222 register_t intr; 2223 struct local_sysmaps *sysm; 2224 pt_entry_t *pte, npte; 2225 2226 /* If this is used other than for dumps, we may need to leave 2227 * interrupts disasbled on return. If crash dumps don't work when 2228 * we get to this point, we might want to consider this (leaving things 2229 * disabled as a starting point ;-) 2230 */ 2231 intr = intr_disable(); 2232 cpu = PCPU_GET(cpuid); 2233 sysm = &sysmap_lmem[cpu]; 2234 /* Since this is for the debugger, no locks or any other fun */ 2235 npte = TLBLO_PA_TO_PFN(pa) | PTE_D | PTE_V | PTE_G | PTE_W | PTE_C_CACHE; 2236 pte = pmap_pte(kernel_pmap, sysm->base); 2237 *pte = npte; 2238 sysm->valid1 = 1; 2239 pmap_update_page(kernel_pmap, sysm->base, npte); 2240 va = sysm->base; 2241 intr_restore(intr); 2242#endif 2243 } 2244 return ((void *)va); 2245} 2246 2247void 2248pmap_kenter_temporary_free(vm_paddr_t pa) 2249{ 2250#ifndef __mips_n64 /* XXX : to be converted to new style */ 2251 int cpu; 2252 register_t intr; 2253 struct local_sysmaps *sysm; 2254#endif 2255 2256 if (MIPS_DIRECT_MAPPABLE(pa)) { 2257 /* nothing to do for this case */ 2258 return; 2259 } 2260#ifndef __mips_n64 /* XXX : to be converted to new style */ 2261 cpu = PCPU_GET(cpuid); 2262 sysm = &sysmap_lmem[cpu]; 2263 if (sysm->valid1) { 2264 pt_entry_t *pte; 2265 2266 intr = intr_disable(); 2267 pte = pmap_pte(kernel_pmap, sysm->base); 2268 *pte = PTE_G; 2269 pmap_invalidate_page(kernel_pmap, sysm->base); 2270 intr_restore(intr); 2271 sysm->valid1 = 0; 2272 } 2273#endif 2274} 2275 2276/* 2277 * Moved the code to Machine Independent 2278 * vm_map_pmap_enter() 2279 */ 2280 2281/* 2282 * Maps a sequence of resident pages belonging to the same object. 2283 * The sequence begins with the given page m_start. This page is 2284 * mapped at the given virtual address start. Each subsequent page is 2285 * mapped at a virtual address that is offset from start by the same 2286 * amount as the page is offset from m_start within the object. The 2287 * last page in the sequence is the page with the largest offset from 2288 * m_start that can be mapped at a virtual address less than the given 2289 * virtual address end. Not every virtual page between start and end 2290 * is mapped; only those for which a resident page exists with the 2291 * corresponding offset from m_start are mapped. 2292 */ 2293void 2294pmap_enter_object(pmap_t pmap, vm_offset_t start, vm_offset_t end, 2295 vm_page_t m_start, vm_prot_t prot) 2296{ 2297 vm_page_t m, mpte; 2298 vm_pindex_t diff, psize; 2299 2300 VM_OBJECT_LOCK_ASSERT(m_start->object, MA_OWNED); 2301 psize = atop(end - start); 2302 mpte = NULL; 2303 m = m_start; 2304 vm_page_lock_queues(); 2305 PMAP_LOCK(pmap); 2306 while (m != NULL && (diff = m->pindex - m_start->pindex) < psize) { 2307 mpte = pmap_enter_quick_locked(pmap, start + ptoa(diff), m, 2308 prot, mpte); 2309 m = TAILQ_NEXT(m, listq); 2310 } 2311 vm_page_unlock_queues(); 2312 PMAP_UNLOCK(pmap); 2313} 2314 2315/* 2316 * pmap_object_init_pt preloads the ptes for a given object 2317 * into the specified pmap. This eliminates the blast of soft 2318 * faults on process startup and immediately after an mmap. 2319 */ 2320void 2321pmap_object_init_pt(pmap_t pmap, vm_offset_t addr, 2322 vm_object_t object, vm_pindex_t pindex, vm_size_t size) 2323{ 2324 VM_OBJECT_LOCK_ASSERT(object, MA_OWNED); 2325 KASSERT(object->type == OBJT_DEVICE || object->type == OBJT_SG, 2326 ("pmap_object_init_pt: non-device object")); 2327} 2328 2329/* 2330 * Routine: pmap_change_wiring 2331 * Function: Change the wiring attribute for a map/virtual-address 2332 * pair. 2333 * In/out conditions: 2334 * The mapping must already exist in the pmap. 2335 */ 2336void 2337pmap_change_wiring(pmap_t pmap, vm_offset_t va, boolean_t wired) 2338{ 2339 pt_entry_t *pte; 2340 2341 if (pmap == NULL) 2342 return; 2343 2344 PMAP_LOCK(pmap); 2345 pte = pmap_pte(pmap, va); 2346 2347 if (wired && !pte_test(pte, PTE_W)) 2348 pmap->pm_stats.wired_count++; 2349 else if (!wired && pte_test(pte, PTE_W)) 2350 pmap->pm_stats.wired_count--; 2351 2352 /* 2353 * Wiring is not a hardware characteristic so there is no need to 2354 * invalidate TLB. 2355 */ 2356 if (wired) 2357 pte_set(pte, PTE_W); 2358 else 2359 pte_clear(pte, PTE_W); 2360 PMAP_UNLOCK(pmap); 2361} 2362 2363/* 2364 * Copy the range specified by src_addr/len 2365 * from the source map to the range dst_addr/len 2366 * in the destination map. 2367 * 2368 * This routine is only advisory and need not do anything. 2369 */ 2370 2371void 2372pmap_copy(pmap_t dst_pmap, pmap_t src_pmap, vm_offset_t dst_addr, 2373 vm_size_t len, vm_offset_t src_addr) 2374{ 2375} 2376 2377/* 2378 * pmap_zero_page zeros the specified hardware page by mapping 2379 * the page into KVM and using bzero to clear its contents. 2380 * 2381 * Use XKPHYS for 64 bit, and KSEG0 where possible for 32 bit. 2382 */ 2383void 2384pmap_zero_page(vm_page_t m) 2385{ 2386 vm_offset_t va; 2387 vm_paddr_t phys = VM_PAGE_TO_PHYS(m); 2388 2389 if (MIPS_DIRECT_MAPPABLE(phys)) { 2390 va = MIPS_PHYS_TO_DIRECT(phys); 2391 bzero((caddr_t)va, PAGE_SIZE); 2392 mips_dcache_wbinv_range(va, PAGE_SIZE); 2393 } else { 2394 va = pmap_lmem_map1(phys); 2395 bzero((caddr_t)va, PAGE_SIZE); 2396 mips_dcache_wbinv_range(va, PAGE_SIZE); 2397 pmap_lmem_unmap(); 2398 } 2399} 2400 2401/* 2402 * pmap_zero_page_area zeros the specified hardware page by mapping 2403 * the page into KVM and using bzero to clear its contents. 2404 * 2405 * off and size may not cover an area beyond a single hardware page. 2406 */ 2407void 2408pmap_zero_page_area(vm_page_t m, int off, int size) 2409{ 2410 vm_offset_t va; 2411 vm_paddr_t phys = VM_PAGE_TO_PHYS(m); 2412 2413 if (MIPS_DIRECT_MAPPABLE(phys)) { 2414 va = MIPS_PHYS_TO_DIRECT(phys); 2415 bzero((char *)(caddr_t)va + off, size); 2416 mips_dcache_wbinv_range(va + off, size); 2417 } else { 2418 va = pmap_lmem_map1(phys); 2419 bzero((char *)va + off, size); 2420 mips_dcache_wbinv_range(va + off, size); 2421 pmap_lmem_unmap(); 2422 } 2423} 2424 2425void 2426pmap_zero_page_idle(vm_page_t m) 2427{ 2428 vm_offset_t va; 2429 vm_paddr_t phys = VM_PAGE_TO_PHYS(m); 2430 2431 if (MIPS_DIRECT_MAPPABLE(phys)) { 2432 va = MIPS_PHYS_TO_DIRECT(phys); 2433 bzero((caddr_t)va, PAGE_SIZE); 2434 mips_dcache_wbinv_range(va, PAGE_SIZE); 2435 } else { 2436 va = pmap_lmem_map1(phys); 2437 bzero((caddr_t)va, PAGE_SIZE); 2438 mips_dcache_wbinv_range(va, PAGE_SIZE); 2439 pmap_lmem_unmap(); 2440 } 2441} 2442 2443/* 2444 * pmap_copy_page copies the specified (machine independent) 2445 * page by mapping the page into virtual memory and using 2446 * bcopy to copy the page, one machine dependent page at a 2447 * time. 2448 * 2449 * Use XKPHYS for 64 bit, and KSEG0 where possible for 32 bit. 2450 */ 2451void 2452pmap_copy_page(vm_page_t src, vm_page_t dst) 2453{ 2454 vm_offset_t va_src, va_dst; 2455 vm_paddr_t phys_src = VM_PAGE_TO_PHYS(src); 2456 vm_paddr_t phys_dst = VM_PAGE_TO_PHYS(dst); 2457 2458 if (MIPS_DIRECT_MAPPABLE(phys_src) && MIPS_DIRECT_MAPPABLE(phys_dst)) { 2459 /* easy case, all can be accessed via KSEG0 */ 2460 /* 2461 * Flush all caches for VA that are mapped to this page 2462 * to make sure that data in SDRAM is up to date 2463 */ 2464 pmap_flush_pvcache(src); 2465 mips_dcache_wbinv_range_index( 2466 MIPS_PHYS_TO_DIRECT(phys_dst), PAGE_SIZE); 2467 va_src = MIPS_PHYS_TO_DIRECT(phys_src); 2468 va_dst = MIPS_PHYS_TO_DIRECT(phys_dst); 2469 bcopy((caddr_t)va_src, (caddr_t)va_dst, PAGE_SIZE); 2470 mips_dcache_wbinv_range(va_dst, PAGE_SIZE); 2471 } else { 2472 va_src = pmap_lmem_map2(phys_src, phys_dst); 2473 va_dst = va_src + PAGE_SIZE; 2474 bcopy((void *)va_src, (void *)va_dst, PAGE_SIZE); 2475 mips_dcache_wbinv_range(va_dst, PAGE_SIZE); 2476 pmap_lmem_unmap(); 2477 } 2478} 2479 2480/* 2481 * Returns true if the pmap's pv is one of the first 2482 * 16 pvs linked to from this page. This count may 2483 * be changed upwards or downwards in the future; it 2484 * is only necessary that true be returned for a small 2485 * subset of pmaps for proper page aging. 2486 */ 2487boolean_t 2488pmap_page_exists_quick(pmap_t pmap, vm_page_t m) 2489{ 2490 pv_entry_t pv; 2491 int loops = 0; 2492 boolean_t rv; 2493 2494 KASSERT((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) == 0, 2495 ("pmap_page_exists_quick: page %p is not managed", m)); 2496 rv = FALSE; 2497 vm_page_lock_queues(); 2498 TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) { 2499 if (pv->pv_pmap == pmap) { 2500 rv = TRUE; 2501 break; 2502 } 2503 loops++; 2504 if (loops >= 16) 2505 break; 2506 } 2507 vm_page_unlock_queues(); 2508 return (rv); 2509} 2510 2511/* 2512 * Remove all pages from specified address space 2513 * this aids process exit speeds. Also, this code 2514 * is special cased for current process only, but 2515 * can have the more generic (and slightly slower) 2516 * mode enabled. This is much faster than pmap_remove 2517 * in the case of running down an entire address space. 2518 */ 2519void 2520pmap_remove_pages(pmap_t pmap) 2521{ 2522 pt_entry_t *pte, tpte; 2523 pv_entry_t pv, npv; 2524 vm_page_t m; 2525 2526 if (pmap != vmspace_pmap(curthread->td_proc->p_vmspace)) { 2527 printf("warning: pmap_remove_pages called with non-current pmap\n"); 2528 return; 2529 } 2530 vm_page_lock_queues(); 2531 PMAP_LOCK(pmap); 2532 sched_pin(); 2533 //XXX need to be TAILQ_FOREACH_SAFE ? 2534 for (pv = TAILQ_FIRST(&pmap->pm_pvlist); pv; pv = npv) { 2535 2536 pte = pmap_pte(pv->pv_pmap, pv->pv_va); 2537 if (!pte_test(pte, PTE_V)) 2538 panic("pmap_remove_pages: page on pm_pvlist has no pte\n"); 2539 tpte = *pte; 2540 2541/* 2542 * We cannot remove wired pages from a process' mapping at this time 2543 */ 2544 if (pte_test(&tpte, PTE_W)) { 2545 npv = TAILQ_NEXT(pv, pv_plist); 2546 continue; 2547 } 2548 *pte = is_kernel_pmap(pmap) ? PTE_G : 0; 2549 2550 m = PHYS_TO_VM_PAGE(TLBLO_PTE_TO_PA(tpte)); 2551 KASSERT(m != NULL, 2552 ("pmap_remove_pages: bad tpte %x", tpte)); 2553 2554 pv->pv_pmap->pm_stats.resident_count--; 2555 2556 /* 2557 * Update the vm_page_t clean and reference bits. 2558 */ 2559 if (pte_test(&tpte, PTE_D)) { 2560 vm_page_dirty(m); 2561 } 2562 npv = TAILQ_NEXT(pv, pv_plist); 2563 TAILQ_REMOVE(&pv->pv_pmap->pm_pvlist, pv, pv_plist); 2564 2565 m->md.pv_list_count--; 2566 TAILQ_REMOVE(&m->md.pv_list, pv, pv_list); 2567 if (TAILQ_FIRST(&m->md.pv_list) == NULL) { 2568 vm_page_flag_clear(m, PG_WRITEABLE); 2569 } 2570 pmap_unuse_pt(pv->pv_pmap, pv->pv_va, pv->pv_ptem); 2571 free_pv_entry(pv); 2572 } 2573 sched_unpin(); 2574 pmap_invalidate_all(pmap); 2575 PMAP_UNLOCK(pmap); 2576 vm_page_unlock_queues(); 2577} 2578 2579/* 2580 * pmap_testbit tests bits in pte's 2581 * note that the testbit/changebit routines are inline, 2582 * and a lot of things compile-time evaluate. 2583 */ 2584static boolean_t 2585pmap_testbit(vm_page_t m, int bit) 2586{ 2587 pv_entry_t pv; 2588 pt_entry_t *pte; 2589 boolean_t rv = FALSE; 2590 2591 if (m->flags & PG_FICTITIOUS) 2592 return (rv); 2593 2594 if (TAILQ_FIRST(&m->md.pv_list) == NULL) 2595 return (rv); 2596 2597 mtx_assert(&vm_page_queue_mtx, MA_OWNED); 2598 TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) { 2599#if defined(PMAP_DIAGNOSTIC) 2600 if (!pv->pv_pmap) { 2601 printf("Null pmap (tb) at va: 0x%x\n", pv->pv_va); 2602 continue; 2603 } 2604#endif 2605 PMAP_LOCK(pv->pv_pmap); 2606 pte = pmap_pte(pv->pv_pmap, pv->pv_va); 2607 rv = pte_test(pte, bit); 2608 PMAP_UNLOCK(pv->pv_pmap); 2609 if (rv) 2610 break; 2611 } 2612 return (rv); 2613} 2614 2615/* 2616 * this routine is used to clear dirty bits in ptes 2617 */ 2618static __inline void 2619pmap_changebit(vm_page_t m, int bit, boolean_t setem) 2620{ 2621 pv_entry_t pv; 2622 pt_entry_t *pte; 2623 2624 if (m->flags & PG_FICTITIOUS) 2625 return; 2626 2627 mtx_assert(&vm_page_queue_mtx, MA_OWNED); 2628 /* 2629 * Loop over all current mappings setting/clearing as appropos If 2630 * setting RO do we need to clear the VAC? 2631 */ 2632 TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) { 2633#if defined(PMAP_DIAGNOSTIC) 2634 if (!pv->pv_pmap) { 2635 printf("Null pmap (cb) at va: 0x%x\n", pv->pv_va); 2636 continue; 2637 } 2638#endif 2639 2640 PMAP_LOCK(pv->pv_pmap); 2641 pte = pmap_pte(pv->pv_pmap, pv->pv_va); 2642 if (setem) { 2643 *pte |= bit; 2644 pmap_update_page(pv->pv_pmap, pv->pv_va, *pte); 2645 } else { 2646 pt_entry_t pbits = *pte; 2647 2648 if (pbits & bit) { 2649 if (bit == PTE_D) { 2650 if (pbits & PTE_D) 2651 vm_page_dirty(m); 2652 *pte = (pbits & ~PTE_D) | PTE_RO; 2653 } else { 2654 *pte = pbits & ~bit; 2655 } 2656 pmap_update_page(pv->pv_pmap, pv->pv_va, *pte); 2657 } 2658 } 2659 PMAP_UNLOCK(pv->pv_pmap); 2660 } 2661 if (!setem && bit == PTE_D) 2662 vm_page_flag_clear(m, PG_WRITEABLE); 2663} 2664 2665/* 2666 * pmap_page_wired_mappings: 2667 * 2668 * Return the number of managed mappings to the given physical page 2669 * that are wired. 2670 */ 2671int 2672pmap_page_wired_mappings(vm_page_t m) 2673{ 2674 pv_entry_t pv; 2675 pmap_t pmap; 2676 pt_entry_t *pte; 2677 int count; 2678 2679 count = 0; 2680 if ((m->flags & PG_FICTITIOUS) != 0) 2681 return (count); 2682 vm_page_lock_queues(); 2683 TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) { 2684 pmap = pv->pv_pmap; 2685 PMAP_LOCK(pmap); 2686 pte = pmap_pte(pmap, pv->pv_va); 2687 if (pte_test(pte, PTE_W)) 2688 count++; 2689 PMAP_UNLOCK(pmap); 2690 } 2691 vm_page_unlock_queues(); 2692 return (count); 2693} 2694 2695/* 2696 * Clear the write and modified bits in each of the given page's mappings. 2697 */ 2698void 2699pmap_remove_write(vm_page_t m) 2700{ 2701 pv_entry_t pv, npv; 2702 vm_offset_t va; 2703 pt_entry_t *pte; 2704 2705 KASSERT((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) == 0, 2706 ("pmap_remove_write: page %p is not managed", m)); 2707 2708 /* 2709 * If the page is not VPO_BUSY, then PG_WRITEABLE cannot be set by 2710 * another thread while the object is locked. Thus, if PG_WRITEABLE 2711 * is clear, no page table entries need updating. 2712 */ 2713 VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED); 2714 if ((m->oflags & VPO_BUSY) == 0 && 2715 (m->flags & PG_WRITEABLE) == 0) 2716 return; 2717 2718 /* 2719 * Loop over all current mappings setting/clearing as appropos. 2720 */ 2721 vm_page_lock_queues(); 2722 for (pv = TAILQ_FIRST(&m->md.pv_list); pv; pv = npv) { 2723 npv = TAILQ_NEXT(pv, pv_plist); 2724 pte = pmap_pte(pv->pv_pmap, pv->pv_va); 2725 if (pte == NULL || !pte_test(pte, PTE_V)) 2726 panic("page on pm_pvlist has no pte\n"); 2727 2728 va = pv->pv_va; 2729 pmap_protect(pv->pv_pmap, va, va + PAGE_SIZE, 2730 VM_PROT_READ | VM_PROT_EXECUTE); 2731 } 2732 vm_page_flag_clear(m, PG_WRITEABLE); 2733 vm_page_unlock_queues(); 2734} 2735 2736/* 2737 * pmap_ts_referenced: 2738 * 2739 * Return the count of reference bits for a page, clearing all of them. 2740 */ 2741int 2742pmap_ts_referenced(vm_page_t m) 2743{ 2744 2745 KASSERT((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) == 0, 2746 ("pmap_ts_referenced: page %p is not managed", m)); 2747 if (m->md.pv_flags & PV_TABLE_REF) { 2748 vm_page_lock_queues(); 2749 m->md.pv_flags &= ~PV_TABLE_REF; 2750 vm_page_unlock_queues(); 2751 return (1); 2752 } 2753 return (0); 2754} 2755 2756/* 2757 * pmap_is_modified: 2758 * 2759 * Return whether or not the specified physical page was modified 2760 * in any physical maps. 2761 */ 2762boolean_t 2763pmap_is_modified(vm_page_t m) 2764{ 2765 boolean_t rv; 2766 2767 KASSERT((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) == 0, 2768 ("pmap_is_modified: page %p is not managed", m)); 2769 2770 /* 2771 * If the page is not VPO_BUSY, then PG_WRITEABLE cannot be 2772 * concurrently set while the object is locked. Thus, if PG_WRITEABLE 2773 * is clear, no PTEs can have PTE_D set. 2774 */ 2775 VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED); 2776 if ((m->oflags & VPO_BUSY) == 0 && 2777 (m->flags & PG_WRITEABLE) == 0) 2778 return (FALSE); 2779 vm_page_lock_queues(); 2780 if (m->md.pv_flags & PV_TABLE_MOD) 2781 rv = TRUE; 2782 else 2783 rv = pmap_testbit(m, PTE_D); 2784 vm_page_unlock_queues(); 2785 return (rv); 2786} 2787 2788/* N/C */ 2789 2790/* 2791 * pmap_is_prefaultable: 2792 * 2793 * Return whether or not the specified virtual address is elgible 2794 * for prefault. 2795 */ 2796boolean_t 2797pmap_is_prefaultable(pmap_t pmap, vm_offset_t addr) 2798{ 2799 pd_entry_t *pde; 2800 pt_entry_t *pte; 2801 boolean_t rv; 2802 2803 rv = FALSE; 2804 PMAP_LOCK(pmap); 2805 pde = pmap_pde(pmap, addr); 2806 if (pde != NULL && *pde != 0) { 2807 pte = pmap_pde_to_pte(pde, addr); 2808 rv = (*pte == 0); 2809 } 2810 PMAP_UNLOCK(pmap); 2811 return (rv); 2812} 2813 2814/* 2815 * Clear the modify bits on the specified physical page. 2816 */ 2817void 2818pmap_clear_modify(vm_page_t m) 2819{ 2820 2821 KASSERT((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) == 0, 2822 ("pmap_clear_modify: page %p is not managed", m)); 2823 VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED); 2824 KASSERT((m->oflags & VPO_BUSY) == 0, 2825 ("pmap_clear_modify: page %p is busy", m)); 2826 2827 /* 2828 * If the page is not PG_WRITEABLE, then no PTEs can have PTE_D set. 2829 * If the object containing the page is locked and the page is not 2830 * VPO_BUSY, then PG_WRITEABLE cannot be concurrently set. 2831 */ 2832 if ((m->flags & PG_WRITEABLE) == 0) 2833 return; 2834 vm_page_lock_queues(); 2835 if (m->md.pv_flags & PV_TABLE_MOD) { 2836 pmap_changebit(m, PTE_D, FALSE); 2837 m->md.pv_flags &= ~PV_TABLE_MOD; 2838 } 2839 vm_page_unlock_queues(); 2840} 2841 2842/* 2843 * pmap_is_referenced: 2844 * 2845 * Return whether or not the specified physical page was referenced 2846 * in any physical maps. 2847 */ 2848boolean_t 2849pmap_is_referenced(vm_page_t m) 2850{ 2851 2852 KASSERT((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) == 0, 2853 ("pmap_is_referenced: page %p is not managed", m)); 2854 return ((m->md.pv_flags & PV_TABLE_REF) != 0); 2855} 2856 2857/* 2858 * pmap_clear_reference: 2859 * 2860 * Clear the reference bit on the specified physical page. 2861 */ 2862void 2863pmap_clear_reference(vm_page_t m) 2864{ 2865 2866 KASSERT((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) == 0, 2867 ("pmap_clear_reference: page %p is not managed", m)); 2868 vm_page_lock_queues(); 2869 if (m->md.pv_flags & PV_TABLE_REF) { 2870 m->md.pv_flags &= ~PV_TABLE_REF; 2871 } 2872 vm_page_unlock_queues(); 2873} 2874 2875/* 2876 * Miscellaneous support routines follow 2877 */ 2878 2879/* 2880 * Map a set of physical memory pages into the kernel virtual 2881 * address space. Return a pointer to where it is mapped. This 2882 * routine is intended to be used for mapping device memory, 2883 * NOT real memory. 2884 */ 2885 2886/* 2887 * Map a set of physical memory pages into the kernel virtual 2888 * address space. Return a pointer to where it is mapped. This 2889 * routine is intended to be used for mapping device memory, 2890 * NOT real memory. 2891 * 2892 * Use XKPHYS uncached for 64 bit, and KSEG1 where possible for 32 bit. 2893 */ 2894void * 2895pmap_mapdev(vm_offset_t pa, vm_size_t size) 2896{ 2897 vm_offset_t va, tmpva, offset; 2898 2899 /* 2900 * KSEG1 maps only first 512M of phys address space. For 2901 * pa > 0x20000000 we should make proper mapping * using pmap_kenter. 2902 */ 2903 if (MIPS_DIRECT_MAPPABLE(pa + size - 1)) 2904 return ((void *)MIPS_PHYS_TO_DIRECT_UNCACHED(pa)); 2905 else { 2906 offset = pa & PAGE_MASK; 2907 size = roundup(size + offset, PAGE_SIZE); 2908 2909 va = kmem_alloc_nofault(kernel_map, size); 2910 if (!va) 2911 panic("pmap_mapdev: Couldn't alloc kernel virtual memory"); 2912 pa = trunc_page(pa); 2913 for (tmpva = va; size > 0;) { 2914 pmap_kenter(tmpva, pa); 2915 size -= PAGE_SIZE; 2916 tmpva += PAGE_SIZE; 2917 pa += PAGE_SIZE; 2918 } 2919 } 2920 2921 return ((void *)(va + offset)); 2922} 2923 2924void 2925pmap_unmapdev(vm_offset_t va, vm_size_t size) 2926{ 2927#ifndef __mips_n64 2928 vm_offset_t base, offset, tmpva; 2929 2930 /* If the address is within KSEG1 then there is nothing to do */ 2931 if (va >= MIPS_KSEG1_START && va <= MIPS_KSEG1_END) 2932 return; 2933 2934 base = trunc_page(va); 2935 offset = va & PAGE_MASK; 2936 size = roundup(size + offset, PAGE_SIZE); 2937 for (tmpva = base; tmpva < base + size; tmpva += PAGE_SIZE) 2938 pmap_kremove(tmpva); 2939 kmem_free(kernel_map, base, size); 2940#endif 2941} 2942 2943/* 2944 * perform the pmap work for mincore 2945 */ 2946int 2947pmap_mincore(pmap_t pmap, vm_offset_t addr, vm_paddr_t *locked_pa) 2948{ 2949 pt_entry_t *ptep, pte; 2950 vm_offset_t pa; 2951 vm_page_t m; 2952 int val; 2953 boolean_t managed; 2954 2955 PMAP_LOCK(pmap); 2956retry: 2957 ptep = pmap_pte(pmap, addr); 2958 pte = (ptep != NULL) ? *ptep : 0; 2959 if (!pte_test(&pte, PTE_V)) { 2960 val = 0; 2961 goto out; 2962 } 2963 val = MINCORE_INCORE; 2964 if (pte_test(&pte, PTE_D)) 2965 val |= MINCORE_MODIFIED | MINCORE_MODIFIED_OTHER; 2966 pa = TLBLO_PTE_TO_PA(pte); 2967 managed = page_is_managed(pa); 2968 if (managed) { 2969 /* 2970 * This may falsely report the given address as 2971 * MINCORE_REFERENCED. Unfortunately, due to the lack of 2972 * per-PTE reference information, it is impossible to 2973 * determine if the address is MINCORE_REFERENCED. 2974 */ 2975 m = PHYS_TO_VM_PAGE(pa); 2976 if ((m->flags & PG_REFERENCED) != 0) 2977 val |= MINCORE_REFERENCED | MINCORE_REFERENCED_OTHER; 2978 } 2979 if ((val & (MINCORE_MODIFIED_OTHER | MINCORE_REFERENCED_OTHER)) != 2980 (MINCORE_MODIFIED_OTHER | MINCORE_REFERENCED_OTHER) && managed) { 2981 /* Ensure that "PHYS_TO_VM_PAGE(pa)->object" doesn't change. */ 2982 if (vm_page_pa_tryrelock(pmap, pa, locked_pa)) 2983 goto retry; 2984 } else 2985out: 2986 PA_UNLOCK_COND(*locked_pa); 2987 PMAP_UNLOCK(pmap); 2988 return (val); 2989} 2990 2991void 2992pmap_activate(struct thread *td) 2993{ 2994 pmap_t pmap, oldpmap; 2995 struct proc *p = td->td_proc; 2996 2997 critical_enter(); 2998 2999 pmap = vmspace_pmap(p->p_vmspace); 3000 oldpmap = PCPU_GET(curpmap); 3001 3002 if (oldpmap) 3003 atomic_clear_32(&oldpmap->pm_active, PCPU_GET(cpumask)); 3004 atomic_set_32(&pmap->pm_active, PCPU_GET(cpumask)); 3005 pmap_asid_alloc(pmap); 3006 if (td == curthread) { 3007 PCPU_SET(segbase, pmap->pm_segtab); 3008 mips_wr_entryhi(pmap->pm_asid[PCPU_GET(cpuid)].asid); 3009 } 3010 3011 PCPU_SET(curpmap, pmap); 3012 critical_exit(); 3013} 3014 3015void 3016pmap_sync_icache(pmap_t pm, vm_offset_t va, vm_size_t sz) 3017{ 3018} 3019 3020/* 3021 * Increase the starting virtual address of the given mapping if a 3022 * different alignment might result in more superpage mappings. 3023 */ 3024void 3025pmap_align_superpage(vm_object_t object, vm_ooffset_t offset, 3026 vm_offset_t *addr, vm_size_t size) 3027{ 3028 vm_offset_t superpage_offset; 3029 3030 if (size < NBSEG) 3031 return; 3032 if (object != NULL && (object->flags & OBJ_COLORED) != 0) 3033 offset += ptoa(object->pg_color); 3034 superpage_offset = offset & SEGMASK; 3035 if (size - ((NBSEG - superpage_offset) & SEGMASK) < NBSEG || 3036 (*addr & SEGMASK) == superpage_offset) 3037 return; 3038 if ((*addr & SEGMASK) < superpage_offset) 3039 *addr = (*addr & ~SEGMASK) + superpage_offset; 3040 else 3041 *addr = ((*addr + SEGMASK) & ~SEGMASK) + superpage_offset; 3042} 3043 3044/* 3045 * Increase the starting virtual address of the given mapping so 3046 * that it is aligned to not be the second page in a TLB entry. 3047 * This routine assumes that the length is appropriately-sized so 3048 * that the allocation does not share a TLB entry at all if required. 3049 */ 3050void 3051pmap_align_tlb(vm_offset_t *addr) 3052{ 3053 if ((*addr & PAGE_SIZE) == 0) 3054 return; 3055 *addr += PAGE_SIZE; 3056 return; 3057} 3058 3059#ifdef DDB 3060DB_SHOW_COMMAND(ptable, ddb_pid_dump) 3061{ 3062 pmap_t pmap; 3063 struct thread *td = NULL; 3064 struct proc *p; 3065 int i, j, k; 3066 vm_paddr_t pa; 3067 vm_offset_t va; 3068 3069 if (have_addr) { 3070 td = db_lookup_thread(addr, TRUE); 3071 if (td == NULL) { 3072 db_printf("Invalid pid or tid"); 3073 return; 3074 } 3075 p = td->td_proc; 3076 if (p->p_vmspace == NULL) { 3077 db_printf("No vmspace for process"); 3078 return; 3079 } 3080 pmap = vmspace_pmap(p->p_vmspace); 3081 } else 3082 pmap = kernel_pmap; 3083 3084 db_printf("pmap:%p segtab:%p asid:%x generation:%x\n", 3085 pmap, pmap->pm_segtab, pmap->pm_asid[0].asid, 3086 pmap->pm_asid[0].gen); 3087 for (i = 0; i < NPDEPG; i++) { 3088 pd_entry_t *pdpe; 3089 pt_entry_t *pde; 3090 pt_entry_t pte; 3091 3092 pdpe = (pd_entry_t *)pmap->pm_segtab[i]; 3093 if (pdpe == NULL) 3094 continue; 3095 db_printf("[%4d] %p\n", i, pdpe); 3096#ifdef __mips_n64 3097 for (j = 0; j < NPDEPG; j++) { 3098 pde = (pt_entry_t *)pdpe[j]; 3099 if (pde == NULL) 3100 continue; 3101 db_printf("\t[%4d] %p\n", j, pde); 3102#else 3103 { 3104 j = 0; 3105 pde = (pt_entry_t *)pdpe; 3106#endif 3107 for (k = 0; k < NPTEPG; k++) { 3108 pte = pde[k]; 3109 if (pte == 0 || !pte_test(&pte, PTE_V)) 3110 continue; 3111 pa = TLBLO_PTE_TO_PA(pte); 3112 va = ((u_long)i << SEGSHIFT) | (j << PDRSHIFT) | (k << PAGE_SHIFT); 3113 db_printf("\t\t[%04d] va: %p pte: %8x pa:%lx\n", 3114 k, (void *)va, pte, (u_long)pa); 3115 } 3116 } 3117 } 3118} 3119#endif 3120 3121#if defined(DEBUG) 3122 3123static void pads(pmap_t pm); 3124void pmap_pvdump(vm_offset_t pa); 3125 3126/* print address space of pmap*/ 3127static void 3128pads(pmap_t pm) 3129{ 3130 unsigned va, i, j; 3131 pt_entry_t *ptep; 3132 3133 if (pm == kernel_pmap) 3134 return; 3135 for (i = 0; i < NPTEPG; i++) 3136 if (pm->pm_segtab[i]) 3137 for (j = 0; j < NPTEPG; j++) { 3138 va = (i << SEGSHIFT) + (j << PAGE_SHIFT); 3139 if (pm == kernel_pmap && va < KERNBASE) 3140 continue; 3141 if (pm != kernel_pmap && 3142 va >= VM_MAXUSER_ADDRESS) 3143 continue; 3144 ptep = pmap_pte(pm, va); 3145 if (pmap_pte_v(ptep)) 3146 printf("%x:%x ", va, *(int *)ptep); 3147 } 3148 3149} 3150 3151void 3152pmap_pvdump(vm_offset_t pa) 3153{ 3154 register pv_entry_t pv; 3155 vm_page_t m; 3156 3157 printf("pa %x", pa); 3158 m = PHYS_TO_VM_PAGE(pa); 3159 for (pv = TAILQ_FIRST(&m->md.pv_list); pv; 3160 pv = TAILQ_NEXT(pv, pv_list)) { 3161 printf(" -> pmap %p, va %x", (void *)pv->pv_pmap, pv->pv_va); 3162 pads(pv->pv_pmap); 3163 } 3164 printf(" "); 3165} 3166 3167/* N/C */ 3168#endif 3169 3170 3171/* 3172 * Allocate TLB address space tag (called ASID or TLBPID) and return it. 3173 * It takes almost as much or more time to search the TLB for a 3174 * specific ASID and flush those entries as it does to flush the entire TLB. 3175 * Therefore, when we allocate a new ASID, we just take the next number. When 3176 * we run out of numbers, we flush the TLB, increment the generation count 3177 * and start over. ASID zero is reserved for kernel use. 3178 */ 3179static void 3180pmap_asid_alloc(pmap) 3181 pmap_t pmap; 3182{ 3183 if (pmap->pm_asid[PCPU_GET(cpuid)].asid != PMAP_ASID_RESERVED && 3184 pmap->pm_asid[PCPU_GET(cpuid)].gen == PCPU_GET(asid_generation)); 3185 else { 3186 if (PCPU_GET(next_asid) == pmap_max_asid) { 3187 tlb_invalidate_all_user(NULL); 3188 PCPU_SET(asid_generation, 3189 (PCPU_GET(asid_generation) + 1) & ASIDGEN_MASK); 3190 if (PCPU_GET(asid_generation) == 0) { 3191 PCPU_SET(asid_generation, 1); 3192 } 3193 PCPU_SET(next_asid, 1); /* 0 means invalid */ 3194 } 3195 pmap->pm_asid[PCPU_GET(cpuid)].asid = PCPU_GET(next_asid); 3196 pmap->pm_asid[PCPU_GET(cpuid)].gen = PCPU_GET(asid_generation); 3197 PCPU_SET(next_asid, PCPU_GET(next_asid) + 1); 3198 } 3199} 3200 3201int 3202page_is_managed(vm_offset_t pa) 3203{ 3204 vm_offset_t pgnum = mips_btop(pa); 3205 3206 if (pgnum >= first_page) { 3207 vm_page_t m; 3208 3209 m = PHYS_TO_VM_PAGE(pa); 3210 if (m == NULL) 3211 return (0); 3212 if ((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) == 0) 3213 return (1); 3214 } 3215 return (0); 3216} 3217 3218static int 3219init_pte_prot(vm_offset_t va, vm_page_t m, vm_prot_t prot) 3220{ 3221 int rw; 3222 3223 if (!(prot & VM_PROT_WRITE)) 3224 rw = PTE_V | PTE_RO | PTE_C_CACHE; 3225 else if ((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) == 0) { 3226 if ((m->md.pv_flags & PV_TABLE_MOD) != 0) 3227 rw = PTE_V | PTE_D | PTE_C_CACHE; 3228 else 3229 rw = PTE_V | PTE_C_CACHE; 3230 vm_page_flag_set(m, PG_WRITEABLE); 3231 } else 3232 /* Needn't emulate a modified bit for unmanaged pages. */ 3233 rw = PTE_V | PTE_D | PTE_C_CACHE; 3234 return (rw); 3235} 3236 3237/* 3238 * pmap_emulate_modified : do dirty bit emulation 3239 * 3240 * On SMP, update just the local TLB, other CPUs will update their 3241 * TLBs from PTE lazily, if they get the exception. 3242 * Returns 0 in case of sucess, 1 if the page is read only and we 3243 * need to fault. 3244 */ 3245int 3246pmap_emulate_modified(pmap_t pmap, vm_offset_t va) 3247{ 3248 vm_page_t m; 3249 pt_entry_t *pte; 3250 vm_offset_t pa; 3251 3252 PMAP_LOCK(pmap); 3253 pte = pmap_pte(pmap, va); 3254 if (pte == NULL) 3255 panic("pmap_emulate_modified: can't find PTE"); 3256#ifdef SMP 3257 /* It is possible that some other CPU changed m-bit */ 3258 if (!pte_test(pte, PTE_V) || pte_test(pte, PTE_D)) { 3259 pmap_update_page_local(pmap, va, *pte); 3260 PMAP_UNLOCK(pmap); 3261 return (0); 3262 } 3263#else 3264 if (!pte_test(pte, PTE_V) || pte_test(pte, PTE_D)) 3265 panic("pmap_emulate_modified: invalid pte"); 3266#endif 3267 if (pte_test(pte, PTE_RO)) { 3268 /* write to read only page in the kernel */ 3269 PMAP_UNLOCK(pmap); 3270 return (1); 3271 } 3272 pte_set(pte, PTE_D); 3273 pmap_update_page_local(pmap, va, *pte); 3274 pa = TLBLO_PTE_TO_PA(*pte); 3275 if (!page_is_managed(pa)) 3276 panic("pmap_emulate_modified: unmanaged page"); 3277 m = PHYS_TO_VM_PAGE(pa); 3278 m->md.pv_flags |= (PV_TABLE_REF | PV_TABLE_MOD); 3279 PMAP_UNLOCK(pmap); 3280 return (0); 3281} 3282 3283/* 3284 * Routine: pmap_kextract 3285 * Function: 3286 * Extract the physical page address associated 3287 * virtual address. 3288 */ 3289 /* PMAP_INLINE */ vm_offset_t 3290pmap_kextract(vm_offset_t va) 3291{ 3292 int mapped; 3293 3294 /* 3295 * First, the direct-mapped regions. 3296 */ 3297#if defined(__mips_n64) 3298 if (va >= MIPS_XKPHYS_START && va < MIPS_XKPHYS_END) 3299 return (MIPS_XKPHYS_TO_PHYS(va)); 3300#endif 3301 if (va >= MIPS_KSEG0_START && va < MIPS_KSEG0_END) 3302 return (MIPS_KSEG0_TO_PHYS(va)); 3303 3304 if (va >= MIPS_KSEG1_START && va < MIPS_KSEG1_END) 3305 return (MIPS_KSEG1_TO_PHYS(va)); 3306 3307 /* 3308 * User virtual addresses. 3309 */ 3310 if (va < VM_MAXUSER_ADDRESS) { 3311 pt_entry_t *ptep; 3312 3313 if (curproc && curproc->p_vmspace) { 3314 ptep = pmap_pte(&curproc->p_vmspace->vm_pmap, va); 3315 if (ptep) { 3316 return (TLBLO_PTE_TO_PA(*ptep) | 3317 (va & PAGE_MASK)); 3318 } 3319 return (0); 3320 } 3321 } 3322 3323 /* 3324 * Should be kernel virtual here, otherwise fail 3325 */ 3326 mapped = (va >= MIPS_KSEG2_START || va < MIPS_KSEG2_END); 3327#if defined(__mips_n64) 3328 mapped = mapped || (va >= MIPS_XKSEG_START || va < MIPS_XKSEG_END); 3329#endif 3330 /* 3331 * Kernel virtual. 3332 */ 3333 3334 if (mapped) { 3335 pt_entry_t *ptep; 3336 3337 /* Is the kernel pmap initialized? */ 3338 if (kernel_pmap->pm_active) { 3339 /* It's inside the virtual address range */ 3340 ptep = pmap_pte(kernel_pmap, va); 3341 if (ptep) { 3342 return (TLBLO_PTE_TO_PA(*ptep) | 3343 (va & PAGE_MASK)); 3344 } 3345 } 3346 return (0); 3347 } 3348 3349 panic("%s for unknown address space %p.", __func__, (void *)va); 3350} 3351 3352 3353void 3354pmap_flush_pvcache(vm_page_t m) 3355{ 3356 pv_entry_t pv; 3357 3358 if (m != NULL) { 3359 for (pv = TAILQ_FIRST(&m->md.pv_list); pv; 3360 pv = TAILQ_NEXT(pv, pv_list)) { 3361 mips_dcache_wbinv_range_index(pv->pv_va, PAGE_SIZE); 3362 } 3363 } 3364} 3365