pmap.c revision 202175
1/* 2 * Copyright (c) 1991 Regents of the University of California. 3 * All rights reserved. 4 * Copyright (c) 1994 John S. Dyson 5 * All rights reserved. 6 * Copyright (c) 1994 David Greenman 7 * All rights reserved. 8 * 9 * This code is derived from software contributed to Berkeley by 10 * the Systems Programming Group of the University of Utah Computer 11 * Science Department and William Jolitz of UUNET Technologies Inc. 12 * 13 * Redistribution and use in source and binary forms, with or without 14 * modification, are permitted provided that the following conditions 15 * are met: 16 * 1. Redistributions of source code must retain the above copyright 17 * notice, this list of conditions and the following disclaimer. 18 * 2. Redistributions in binary form must reproduce the above copyright 19 * notice, this list of conditions and the following disclaimer in the 20 * documentation and/or other materials provided with the distribution. 21 * 4. Neither the name of the University nor the names of its contributors 22 * may be used to endorse or promote products derived from this software 23 * without specific prior written permission. 24 * 25 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 26 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 27 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 28 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 29 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 30 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 31 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 32 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 33 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 34 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 35 * SUCH DAMAGE. 36 * 37 * from: @(#)pmap.c 7.7 (Berkeley) 5/12/91 38 * from: src/sys/i386/i386/pmap.c,v 1.250.2.8 2000/11/21 00:09:14 ps 39 * JNPR: pmap.c,v 1.11.2.1 2007/08/16 11:51:06 girish 40 */ 41 42/* 43 * Manages physical address maps. 44 * 45 * In addition to hardware address maps, this 46 * module is called upon to provide software-use-only 47 * maps which may or may not be stored in the same 48 * form as hardware maps. These pseudo-maps are 49 * used to store intermediate results from copy 50 * operations to and from address spaces. 51 * 52 * Since the information managed by this module is 53 * also stored by the logical address mapping module, 54 * this module may throw away valid virtual-to-physical 55 * mappings at almost any time. However, invalidations 56 * of virtual-to-physical mappings must be done as 57 * requested. 58 * 59 * In order to cope with hardware architectures which 60 * make virtual-to-physical map invalidates expensive, 61 * this module may delay invalidate or reduced protection 62 * operations until such time as they are actually 63 * necessary. This module is given full information as 64 * to which processors are currently using which maps, 65 * and to when physical maps must be made correct. 66 */ 67 68#include <sys/cdefs.h> 69__FBSDID("$FreeBSD: head/sys/mips/mips/pmap.c 202175 2010-01-12 21:36:08Z imp $"); 70 71#include "opt_ddb.h" 72#include "opt_msgbuf.h" 73#include <sys/param.h> 74#include <sys/systm.h> 75#include <sys/proc.h> 76#include <sys/msgbuf.h> 77#include <sys/vmmeter.h> 78#include <sys/mman.h> 79 80#include <vm/vm.h> 81#include <vm/vm_param.h> 82#include <sys/lock.h> 83#include <sys/mutex.h> 84#include <vm/vm_kern.h> 85#include <vm/vm_page.h> 86#include <vm/vm_map.h> 87#include <vm/vm_object.h> 88#include <vm/vm_extern.h> 89#include <vm/vm_pageout.h> 90#include <vm/vm_pager.h> 91#include <vm/uma.h> 92#include <sys/pcpu.h> 93#include <sys/sched.h> 94#ifdef SMP 95#include <sys/smp.h> 96#endif 97 98#include <machine/cache.h> 99#include <machine/md_var.h> 100 101#if defined(DIAGNOSTIC) 102#define PMAP_DIAGNOSTIC 103#endif 104 105#undef PMAP_DEBUG 106 107#ifndef PMAP_SHPGPERPROC 108#define PMAP_SHPGPERPROC 200 109#endif 110 111#if !defined(PMAP_DIAGNOSTIC) 112#define PMAP_INLINE __inline 113#else 114#define PMAP_INLINE 115#endif 116 117/* 118 * Get PDEs and PTEs for user/kernel address space 119 */ 120#define pmap_pde(m, v) (&((m)->pm_segtab[(vm_offset_t)(v) >> SEGSHIFT])) 121#define segtab_pde(m, v) (m[(vm_offset_t)(v) >> SEGSHIFT]) 122 123#define pmap_pte_w(pte) ((*(int *)pte & PTE_W) != 0) 124#define pmap_pde_v(pte) ((*(int *)pte) != 0) 125#define pmap_pte_m(pte) ((*(int *)pte & PTE_M) != 0) 126#define pmap_pte_v(pte) ((*(int *)pte & PTE_V) != 0) 127 128#define pmap_pte_set_w(pte, v) ((v)?(*(int *)pte |= PTE_W):(*(int *)pte &= ~PTE_W)) 129#define pmap_pte_set_prot(pte, v) ((*(int *)pte &= ~PG_PROT), (*(int *)pte |= (v))) 130 131#define MIPS_SEGSIZE (1L << SEGSHIFT) 132#define mips_segtrunc(va) ((va) & ~(MIPS_SEGSIZE-1)) 133#define pmap_TLB_invalidate_all() MIPS_TBIAP() 134#define pmap_va_asid(pmap, va) ((va) | ((pmap)->pm_asid[PCPU_GET(cpuid)].asid << VMTLB_PID_SHIFT)) 135#define is_kernel_pmap(x) ((x) == kernel_pmap) 136 137struct pmap kernel_pmap_store; 138pd_entry_t *kernel_segmap; 139 140vm_offset_t virtual_avail; /* VA of first avail page (after kernel bss) */ 141vm_offset_t virtual_end; /* VA of last avail page (end of kernel AS) */ 142 143static int nkpt; 144unsigned pmap_max_asid; /* max ASID supported by the system */ 145 146 147#define PMAP_ASID_RESERVED 0 148 149 150vm_offset_t kernel_vm_end; 151 152static void pmap_asid_alloc(pmap_t pmap); 153 154/* 155 * Data for the pv entry allocation mechanism 156 */ 157static uma_zone_t pvzone; 158static struct vm_object pvzone_obj; 159static int pv_entry_count = 0, pv_entry_max = 0, pv_entry_high_water = 0; 160 161struct fpage fpages_shared[FPAGES_SHARED]; 162 163struct sysmaps sysmaps_pcpu[MAXCPU]; 164 165static PMAP_INLINE void free_pv_entry(pv_entry_t pv); 166static pv_entry_t get_pv_entry(pmap_t locked_pmap); 167static __inline void pmap_changebit(vm_page_t m, int bit, boolean_t setem); 168 169static vm_page_t pmap_enter_quick_locked(pmap_t pmap, vm_offset_t va, 170 vm_page_t m, vm_prot_t prot, vm_page_t mpte); 171static int pmap_remove_pte(struct pmap *pmap, pt_entry_t *ptq, vm_offset_t va); 172static void pmap_remove_page(struct pmap *pmap, vm_offset_t va); 173static void pmap_remove_entry(struct pmap *pmap, vm_page_t m, vm_offset_t va); 174static boolean_t pmap_testbit(vm_page_t m, int bit); 175static void 176pmap_insert_entry(pmap_t pmap, vm_offset_t va, vm_page_t mpte, 177 vm_page_t m, boolean_t wired); 178static boolean_t pmap_try_insert_pv_entry(pmap_t pmap, vm_page_t mpte, 179 vm_offset_t va, vm_page_t m); 180 181static vm_page_t pmap_allocpte(pmap_t pmap, vm_offset_t va, int flags); 182 183static vm_page_t _pmap_allocpte(pmap_t pmap, unsigned ptepindex, int flags); 184static int pmap_unuse_pt(pmap_t, vm_offset_t, vm_page_t); 185static int init_pte_prot(vm_offset_t va, vm_page_t m, vm_prot_t prot); 186static void pmap_TLB_invalidate_kernel(vm_offset_t); 187static void pmap_TLB_update_kernel(vm_offset_t, pt_entry_t); 188static void pmap_init_fpage(void); 189 190#ifdef SMP 191static void pmap_invalidate_page_action(void *arg); 192static void pmap_invalidate_all_action(void *arg); 193static void pmap_update_page_action(void *arg); 194 195#endif 196 197struct local_sysmaps { 198 struct mtx lock; 199 pt_entry_t CMAP1; 200 pt_entry_t CMAP2; 201 caddr_t CADDR1; 202 caddr_t CADDR2; 203 uint16_t valid1, valid2; 204}; 205 206/* This structure is for large memory 207 * above 512Meg. We can't (in 32 bit mode) 208 * just use the direct mapped MIPS_CACHED_TO_PHYS() 209 * macros since we can't see the memory and must 210 * map it in when we need to access it. In 64 211 * bit mode this goes away. 212 */ 213static struct local_sysmaps sysmap_lmem[MAXCPU]; 214caddr_t virtual_sys_start = (caddr_t)0; 215 216pd_entry_t 217pmap_segmap(pmap_t pmap, vm_offset_t va) 218{ 219 if (pmap->pm_segtab) 220 return (pmap->pm_segtab[((vm_offset_t)(va) >> SEGSHIFT)]); 221 else 222 return ((pd_entry_t)0); 223} 224 225/* 226 * Routine: pmap_pte 227 * Function: 228 * Extract the page table entry associated 229 * with the given map/virtual_address pair. 230 */ 231pt_entry_t * 232pmap_pte(pmap_t pmap, vm_offset_t va) 233{ 234 pt_entry_t *pdeaddr; 235 236 if (pmap) { 237 pdeaddr = (pt_entry_t *)pmap_segmap(pmap, va); 238 if (pdeaddr) { 239 return pdeaddr + vad_to_pte_offset(va); 240 } 241 } 242 return ((pt_entry_t *)0); 243} 244 245 246vm_offset_t 247pmap_steal_memory(vm_size_t size) 248{ 249 vm_size_t bank_size; 250 vm_offset_t pa, va; 251 252 size = round_page(size); 253 254 bank_size = phys_avail[1] - phys_avail[0]; 255 while (size > bank_size) { 256 int i; 257 258 for (i = 0; phys_avail[i + 2]; i += 2) { 259 phys_avail[i] = phys_avail[i + 2]; 260 phys_avail[i + 1] = phys_avail[i + 3]; 261 } 262 phys_avail[i] = 0; 263 phys_avail[i + 1] = 0; 264 if (!phys_avail[0]) 265 panic("pmap_steal_memory: out of memory"); 266 bank_size = phys_avail[1] - phys_avail[0]; 267 } 268 269 pa = phys_avail[0]; 270 phys_avail[0] += size; 271 if (pa >= MIPS_KSEG0_LARGEST_PHYS) { 272 panic("Out of memory below 512Meg?"); 273 } 274 va = MIPS_PHYS_TO_CACHED(pa); 275 bzero((caddr_t)va, size); 276 return va; 277} 278 279/* 280 * Bootstrap the system enough to run with virtual memory. This 281 * assumes that the phys_avail array has been initialized. 282 */ 283void 284pmap_bootstrap(void) 285{ 286 pt_entry_t *pgtab; 287 pt_entry_t *pte; 288 int i, j; 289 int memory_larger_than_512meg = 0; 290 291 /* Sort. */ 292again: 293 for (i = 0; phys_avail[i + 1] != 0; i += 2) { 294 /* 295 * Keep the memory aligned on page boundary. 296 */ 297 phys_avail[i] = round_page(phys_avail[i]); 298 phys_avail[i + 1] = trunc_page(phys_avail[i + 1]); 299 300 if (phys_avail[i + 1] >= MIPS_KSEG0_LARGEST_PHYS) 301 memory_larger_than_512meg++; 302 if (i < 2) 303 continue; 304 if (phys_avail[i - 2] > phys_avail[i]) { 305 vm_paddr_t ptemp[2]; 306 307 308 ptemp[0] = phys_avail[i + 0]; 309 ptemp[1] = phys_avail[i + 1]; 310 311 phys_avail[i + 0] = phys_avail[i - 2]; 312 phys_avail[i + 1] = phys_avail[i - 1]; 313 314 phys_avail[i - 2] = ptemp[0]; 315 phys_avail[i - 1] = ptemp[1]; 316 goto again; 317 } 318 } 319 320 /* 321 * Copy the phys_avail[] array before we start stealing memory from it. 322 */ 323 for (i = 0; phys_avail[i + 1] != 0; i += 2) { 324 physmem_desc[i] = phys_avail[i]; 325 physmem_desc[i + 1] = phys_avail[i + 1]; 326 } 327 328 Maxmem = atop(phys_avail[i - 1]); 329 330 if (bootverbose) { 331 printf("Physical memory chunk(s):\n"); 332 for (i = 0; phys_avail[i + 1] != 0; i += 2) { 333 vm_paddr_t size; 334 335 size = phys_avail[i + 1] - phys_avail[i]; 336 printf("%#08jx - %#08jx, %ju bytes (%ju pages)\n", 337 (uintmax_t) phys_avail[i], 338 (uintmax_t) phys_avail[i + 1] - 1, 339 (uintmax_t) size, (uintmax_t) size / PAGE_SIZE); 340 } 341 printf("Maxmem is 0x%0lx\n", ptoa(Maxmem)); 342 } 343 /* 344 * Steal the message buffer from the beginning of memory. 345 */ 346 msgbufp = (struct msgbuf *)pmap_steal_memory(MSGBUF_SIZE); 347 msgbufinit(msgbufp, MSGBUF_SIZE); 348 349 /* 350 * Steal thread0 kstack. 351 */ 352 kstack0 = pmap_steal_memory(KSTACK_PAGES << PAGE_SHIFT); 353 354 355 virtual_avail = VM_MIN_KERNEL_ADDRESS + VM_KERNEL_ALLOC_OFFSET; 356 virtual_end = VM_MAX_KERNEL_ADDRESS; 357 358 /* 359 * Steal some virtual space that will not be in kernel_segmap. This 360 * va memory space will be used to map in kernel pages that are 361 * outside the 512Meg region. Note that we only do this steal when 362 * we do have memory in this region, that way for systems with 363 * smaller memory we don't "steal" any va ranges :-) 364 */ 365 if (memory_larger_than_512meg) { 366 for (i = 0; i < MAXCPU; i++) { 367 sysmap_lmem[i].CMAP1 = PTE_G; 368 sysmap_lmem[i].CMAP2 = PTE_G; 369 sysmap_lmem[i].CADDR1 = (caddr_t)virtual_avail; 370 virtual_avail += PAGE_SIZE; 371 sysmap_lmem[i].CADDR2 = (caddr_t)virtual_avail; 372 virtual_avail += PAGE_SIZE; 373 sysmap_lmem[i].valid1 = sysmap_lmem[i].valid2 = 0; 374 PMAP_LGMEM_LOCK_INIT(&sysmap_lmem[i]); 375 } 376 } 377 virtual_sys_start = (caddr_t)virtual_avail; 378 /* 379 * Allocate segment table for the kernel 380 */ 381 kernel_segmap = (pd_entry_t *)pmap_steal_memory(PAGE_SIZE); 382 383 /* 384 * Allocate second level page tables for the kernel 385 */ 386 nkpt = NKPT; 387 if (memory_larger_than_512meg) { 388 /* 389 * If we have a large memory system we CANNOT afford to hit 390 * pmap_growkernel() and allocate memory. Since we MAY end 391 * up with a page that is NOT mappable. For that reason we 392 * up front grab more. Normall NKPT is 120 (YMMV see pmap.h) 393 * this gives us 480meg of kernel virtual addresses at the 394 * cost of 120 pages (each page gets us 4 Meg). Since the 395 * kernel starts at virtual_avail, we can use this to 396 * calculate how many entris are left from there to the end 397 * of the segmap, we want to allocate all of it, which would 398 * be somewhere above 0xC0000000 - 0xFFFFFFFF which results 399 * in about 256 entries or so instead of the 120. 400 */ 401 nkpt = (PAGE_SIZE / sizeof(pd_entry_t)) - (virtual_avail >> SEGSHIFT); 402 } 403 pgtab = (pt_entry_t *)pmap_steal_memory(PAGE_SIZE * nkpt); 404 405 /* 406 * The R[4-7]?00 stores only one copy of the Global bit in the 407 * translation lookaside buffer for each 2 page entry. Thus invalid 408 * entrys must have the Global bit set so when Entry LO and Entry HI 409 * G bits are anded together they will produce a global bit to store 410 * in the tlb. 411 */ 412 for (i = 0, pte = pgtab; i < (nkpt * NPTEPG); i++, pte++) 413 *pte = PTE_G; 414 415 /* 416 * The segment table contains the KVA of the pages in the second 417 * level page table. 418 */ 419 for (i = 0, j = (virtual_avail >> SEGSHIFT); i < nkpt; i++, j++) 420 kernel_segmap[j] = (pd_entry_t)(pgtab + (i * NPTEPG)); 421 422 /* 423 * The kernel's pmap is statically allocated so we don't have to use 424 * pmap_create, which is unlikely to work correctly at this part of 425 * the boot sequence (XXX and which no longer exists). 426 */ 427 PMAP_LOCK_INIT(kernel_pmap); 428 kernel_pmap->pm_segtab = kernel_segmap; 429 kernel_pmap->pm_active = ~0; 430 TAILQ_INIT(&kernel_pmap->pm_pvlist); 431 kernel_pmap->pm_asid[PCPU_GET(cpuid)].asid = PMAP_ASID_RESERVED; 432 kernel_pmap->pm_asid[PCPU_GET(cpuid)].gen = 0; 433 pmap_max_asid = VMNUM_PIDS; 434 MachSetPID(0); 435} 436 437/* 438 * Initialize a vm_page's machine-dependent fields. 439 */ 440void 441pmap_page_init(vm_page_t m) 442{ 443 444 TAILQ_INIT(&m->md.pv_list); 445 m->md.pv_list_count = 0; 446 m->md.pv_flags = 0; 447} 448 449/* 450 * Initialize the pmap module. 451 * Called by vm_init, to initialize any structures that the pmap 452 * system needs to map virtual memory. 453 * pmap_init has been enhanced to support in a fairly consistant 454 * way, discontiguous physical memory. 455 */ 456void 457pmap_init(void) 458{ 459 460 if (need_wired_tlb_page_pool) 461 pmap_init_fpage(); 462 /* 463 * Initialize the address space (zone) for the pv entries. Set a 464 * high water mark so that the system can recover from excessive 465 * numbers of pv entries. 466 */ 467 pvzone = uma_zcreate("PV ENTRY", sizeof(struct pv_entry), NULL, NULL, 468 NULL, NULL, UMA_ALIGN_PTR, UMA_ZONE_VM | UMA_ZONE_NOFREE); 469 pv_entry_max = PMAP_SHPGPERPROC * maxproc + cnt.v_page_count; 470 pv_entry_high_water = 9 * (pv_entry_max / 10); 471 uma_zone_set_obj(pvzone, &pvzone_obj, pv_entry_max); 472} 473 474/*************************************************** 475 * Low level helper routines..... 476 ***************************************************/ 477 478#if defined(PMAP_DIAGNOSTIC) 479 480/* 481 * This code checks for non-writeable/modified pages. 482 * This should be an invalid condition. 483 */ 484static int 485pmap_nw_modified(pt_entry_t pte) 486{ 487 if ((pte & (PTE_M | PTE_RO)) == (PTE_M | PTE_RO)) 488 return (1); 489 else 490 return (0); 491} 492 493#endif 494 495static void 496pmap_invalidate_all(pmap_t pmap) 497{ 498#ifdef SMP 499 smp_rendezvous(0, pmap_invalidate_all_action, 0, (void *)pmap); 500} 501 502static void 503pmap_invalidate_all_action(void *arg) 504{ 505 pmap_t pmap = (pmap_t)arg; 506 507#endif 508 509 if (pmap->pm_active & PCPU_GET(cpumask)) { 510 pmap_TLB_invalidate_all(); 511 } else 512 pmap->pm_asid[PCPU_GET(cpuid)].gen = 0; 513} 514 515struct pmap_invalidate_page_arg { 516 pmap_t pmap; 517 vm_offset_t va; 518}; 519 520static __inline void 521pmap_invalidate_page(pmap_t pmap, vm_offset_t va) 522{ 523#ifdef SMP 524 struct pmap_invalidate_page_arg arg; 525 526 arg.pmap = pmap; 527 arg.va = va; 528 529 smp_rendezvous(0, pmap_invalidate_page_action, 0, (void *)&arg); 530} 531 532static void 533pmap_invalidate_page_action(void *arg) 534{ 535 pmap_t pmap = ((struct pmap_invalidate_page_arg *)arg)->pmap; 536 vm_offset_t va = ((struct pmap_invalidate_page_arg *)arg)->va; 537 538#endif 539 540 if (is_kernel_pmap(pmap)) { 541 pmap_TLB_invalidate_kernel(va); 542 return; 543 } 544 if (pmap->pm_asid[PCPU_GET(cpuid)].gen != PCPU_GET(asid_generation)) 545 return; 546 else if (!(pmap->pm_active & PCPU_GET(cpumask))) { 547 pmap->pm_asid[PCPU_GET(cpuid)].gen = 0; 548 return; 549 } 550 va = pmap_va_asid(pmap, (va & ~PGOFSET)); 551 mips_TBIS(va); 552} 553 554static void 555pmap_TLB_invalidate_kernel(vm_offset_t va) 556{ 557 u_int32_t pid; 558 559 MachTLBGetPID(pid); 560 va = va | (pid << VMTLB_PID_SHIFT); 561 mips_TBIS(va); 562} 563 564struct pmap_update_page_arg { 565 pmap_t pmap; 566 vm_offset_t va; 567 pt_entry_t pte; 568}; 569 570void 571pmap_update_page(pmap_t pmap, vm_offset_t va, pt_entry_t pte) 572{ 573#ifdef SMP 574 struct pmap_update_page_arg arg; 575 576 arg.pmap = pmap; 577 arg.va = va; 578 arg.pte = pte; 579 580 smp_rendezvous(0, pmap_update_page_action, 0, (void *)&arg); 581} 582 583static void 584pmap_update_page_action(void *arg) 585{ 586 pmap_t pmap = ((struct pmap_update_page_arg *)arg)->pmap; 587 vm_offset_t va = ((struct pmap_update_page_arg *)arg)->va; 588 pt_entry_t pte = ((struct pmap_update_page_arg *)arg)->pte; 589 590#endif 591 if (is_kernel_pmap(pmap)) { 592 pmap_TLB_update_kernel(va, pte); 593 return; 594 } 595 if (pmap->pm_asid[PCPU_GET(cpuid)].gen != PCPU_GET(asid_generation)) 596 return; 597 else if (!(pmap->pm_active & PCPU_GET(cpumask))) { 598 pmap->pm_asid[PCPU_GET(cpuid)].gen = 0; 599 return; 600 } 601 va = pmap_va_asid(pmap, va); 602 MachTLBUpdate(va, pte); 603} 604 605static void 606pmap_TLB_update_kernel(vm_offset_t va, pt_entry_t pte) 607{ 608 u_int32_t pid; 609 610 MachTLBGetPID(pid); 611 va = va | (pid << VMTLB_PID_SHIFT); 612 613 MachTLBUpdate(va, pte); 614} 615 616/* 617 * Routine: pmap_extract 618 * Function: 619 * Extract the physical page address associated 620 * with the given map/virtual_address pair. 621 */ 622vm_paddr_t 623pmap_extract(pmap_t pmap, vm_offset_t va) 624{ 625 pt_entry_t *pte; 626 vm_offset_t retval = 0; 627 628 PMAP_LOCK(pmap); 629 pte = pmap_pte(pmap, va); 630 if (pte) { 631 retval = mips_tlbpfn_to_paddr(*pte) | (va & PAGE_MASK); 632 } 633 PMAP_UNLOCK(pmap); 634 return retval; 635} 636 637/* 638 * Routine: pmap_extract_and_hold 639 * Function: 640 * Atomically extract and hold the physical page 641 * with the given pmap and virtual address pair 642 * if that mapping permits the given protection. 643 */ 644vm_page_t 645pmap_extract_and_hold(pmap_t pmap, vm_offset_t va, vm_prot_t prot) 646{ 647 pt_entry_t pte; 648 vm_page_t m; 649 650 m = NULL; 651 vm_page_lock_queues(); 652 PMAP_LOCK(pmap); 653 654 pte = *pmap_pte(pmap, va); 655 if (pte != 0 && pmap_pte_v(&pte) && 656 ((pte & PTE_RW) || (prot & VM_PROT_WRITE) == 0)) { 657 m = PHYS_TO_VM_PAGE(mips_tlbpfn_to_paddr(pte)); 658 vm_page_hold(m); 659 } 660 vm_page_unlock_queues(); 661 PMAP_UNLOCK(pmap); 662 return (m); 663} 664 665/*************************************************** 666 * Low level mapping routines..... 667 ***************************************************/ 668 669/* 670 * add a wired page to the kva 671 */ 672 /* PMAP_INLINE */ void 673pmap_kenter(vm_offset_t va, vm_paddr_t pa) 674{ 675 register pt_entry_t *pte; 676 pt_entry_t npte, opte; 677 678#ifdef PMAP_DEBUG 679 printf("pmap_kenter: va: 0x%08x -> pa: 0x%08x\n", va, pa); 680#endif 681 npte = mips_paddr_to_tlbpfn(pa) | PTE_RW | PTE_V | PTE_G | PTE_W; 682 683 if (is_cacheable_mem(pa)) 684 npte |= PTE_CACHE; 685 else 686 npte |= PTE_UNCACHED; 687 688 pte = pmap_pte(kernel_pmap, va); 689 opte = *pte; 690 *pte = npte; 691 692 pmap_update_page(kernel_pmap, va, npte); 693} 694 695/* 696 * remove a page from the kernel pagetables 697 */ 698 /* PMAP_INLINE */ void 699pmap_kremove(vm_offset_t va) 700{ 701 register pt_entry_t *pte; 702 703 /* 704 * Write back all caches from the page being destroyed 705 */ 706 mips_dcache_wbinv_range_index(va, NBPG); 707 708 pte = pmap_pte(kernel_pmap, va); 709 *pte = PTE_G; 710 pmap_invalidate_page(kernel_pmap, va); 711} 712 713/* 714 * Used to map a range of physical addresses into kernel 715 * virtual address space. 716 * 717 * The value passed in '*virt' is a suggested virtual address for 718 * the mapping. Architectures which can support a direct-mapped 719 * physical to virtual region can return the appropriate address 720 * within that region, leaving '*virt' unchanged. Other 721 * architectures should map the pages starting at '*virt' and 722 * update '*virt' with the first usable address after the mapped 723 * region. 724 */ 725vm_offset_t 726pmap_map(vm_offset_t *virt, vm_offset_t start, vm_offset_t end, int prot) 727{ 728 vm_offset_t va, sva; 729 730 va = sva = *virt; 731 while (start < end) { 732 pmap_kenter(va, start); 733 va += PAGE_SIZE; 734 start += PAGE_SIZE; 735 } 736 *virt = va; 737 return (sva); 738} 739 740/* 741 * Add a list of wired pages to the kva 742 * this routine is only used for temporary 743 * kernel mappings that do not need to have 744 * page modification or references recorded. 745 * Note that old mappings are simply written 746 * over. The page *must* be wired. 747 */ 748void 749pmap_qenter(vm_offset_t va, vm_page_t *m, int count) 750{ 751 int i; 752 vm_offset_t origva = va; 753 754 for (i = 0; i < count; i++) { 755 pmap_flush_pvcache(m[i]); 756 pmap_kenter(va, VM_PAGE_TO_PHYS(m[i])); 757 va += PAGE_SIZE; 758 } 759 760 mips_dcache_wbinv_range_index(origva, PAGE_SIZE*count); 761} 762 763/* 764 * this routine jerks page mappings from the 765 * kernel -- it is meant only for temporary mappings. 766 */ 767void 768pmap_qremove(vm_offset_t va, int count) 769{ 770 /* 771 * No need to wb/inv caches here, 772 * pmap_kremove will do it for us 773 */ 774 775 while (count-- > 0) { 776 pmap_kremove(va); 777 va += PAGE_SIZE; 778 } 779} 780 781/*************************************************** 782 * Page table page management routines..... 783 ***************************************************/ 784 785/* 786 * floating pages (FPAGES) management routines 787 * 788 * FPAGES are the reserved virtual memory areas which can be 789 * mapped to any physical memory. This gets used typically 790 * in the following functions: 791 * 792 * pmap_zero_page 793 * pmap_copy_page 794 */ 795 796/* 797 * Create the floating pages, aka FPAGES! 798 */ 799static void 800pmap_init_fpage() 801{ 802 vm_offset_t kva; 803 int i, j; 804 struct sysmaps *sysmaps; 805 806 /* 807 * We allocate a total of (FPAGES*MAXCPU + FPAGES_SHARED + 1) pages 808 * at first. FPAGES & FPAGES_SHARED should be EVEN Then we'll adjust 809 * 'kva' to be even-page aligned so that the fpage area can be wired 810 * in the TLB with a single TLB entry. 811 */ 812 kva = kmem_alloc_nofault(kernel_map, 813 (FPAGES * MAXCPU + 1 + FPAGES_SHARED) * PAGE_SIZE); 814 if ((void *)kva == NULL) 815 panic("pmap_init_fpage: fpage allocation failed"); 816 817 /* 818 * Make up start at an even page number so we can wire down the 819 * fpage area in the tlb with a single tlb entry. 820 */ 821 if ((((vm_offset_t)kva) >> PGSHIFT) & 1) { 822 /* 823 * 'kva' is not even-page aligned. Adjust it and free the 824 * first page which is unused. 825 */ 826 kmem_free(kernel_map, (vm_offset_t)kva, NBPG); 827 kva = ((vm_offset_t)kva) + NBPG; 828 } else { 829 /* 830 * 'kva' is even page aligned. We don't need the last page, 831 * free it. 832 */ 833 kmem_free(kernel_map, ((vm_offset_t)kva) + FSPACE, NBPG); 834 } 835 836 for (i = 0; i < MAXCPU; i++) { 837 sysmaps = &sysmaps_pcpu[i]; 838 mtx_init(&sysmaps->lock, "SYSMAPS", NULL, MTX_DEF); 839 840 /* Assign FPAGES pages to the CPU */ 841 for (j = 0; j < FPAGES; j++) 842 sysmaps->fp[j].kva = kva + (j) * PAGE_SIZE; 843 kva = ((vm_offset_t)kva) + (FPAGES * PAGE_SIZE); 844 } 845 846 /* 847 * An additional 2 pages are needed, one for pmap_zero_page_idle() 848 * and one for coredump. These pages are shared by all cpu's 849 */ 850 fpages_shared[PMAP_FPAGE3].kva = kva; 851 fpages_shared[PMAP_FPAGE_KENTER_TEMP].kva = kva + PAGE_SIZE; 852} 853 854/* 855 * Map the page to the fpage virtual address as specified thru' fpage id 856 */ 857vm_offset_t 858pmap_map_fpage(vm_paddr_t pa, struct fpage *fp, boolean_t check_unmaped) 859{ 860 vm_offset_t kva; 861 register pt_entry_t *pte; 862 pt_entry_t npte; 863 864 KASSERT(curthread->td_pinned > 0, ("curthread not pinned")); 865 /* 866 * Check if the fpage is free 867 */ 868 if (fp->state) { 869 if (check_unmaped == TRUE) 870 pmap_unmap_fpage(pa, fp); 871 else 872 panic("pmap_map_fpage: fpage is busy"); 873 } 874 fp->state = TRUE; 875 kva = fp->kva; 876 877 npte = mips_paddr_to_tlbpfn(pa) | PTE_RW | PTE_V | PTE_G | PTE_W | PTE_CACHE; 878 pte = pmap_pte(kernel_pmap, kva); 879 *pte = npte; 880 881 pmap_TLB_update_kernel(kva, npte); 882 883 return (kva); 884} 885 886/* 887 * Unmap the page from the fpage virtual address as specified thru' fpage id 888 */ 889void 890pmap_unmap_fpage(vm_paddr_t pa, struct fpage *fp) 891{ 892 vm_offset_t kva; 893 register pt_entry_t *pte; 894 895 KASSERT(curthread->td_pinned > 0, ("curthread not pinned")); 896 /* 897 * Check if the fpage is busy 898 */ 899 if (!(fp->state)) { 900 panic("pmap_unmap_fpage: fpage is free"); 901 } 902 kva = fp->kva; 903 904 pte = pmap_pte(kernel_pmap, kva); 905 *pte = PTE_G; 906 pmap_TLB_invalidate_kernel(kva); 907 908 fp->state = FALSE; 909 910 /* 911 * Should there be any flush operation at the end? 912 */ 913} 914 915/* Revision 1.507 916 * 917 * Simplify the reference counting of page table pages. Specifically, use 918 * the page table page's wired count rather than its hold count to contain 919 * the reference count. 920 */ 921 922/* 923 * This routine unholds page table pages, and if the hold count 924 * drops to zero, then it decrements the wire count. 925 */ 926static int 927_pmap_unwire_pte_hold(pmap_t pmap, vm_page_t m) 928{ 929 930 /* 931 * unmap the page table page 932 */ 933 pmap->pm_segtab[m->pindex] = 0; 934 --pmap->pm_stats.resident_count; 935 936 if (pmap->pm_ptphint == m) 937 pmap->pm_ptphint = NULL; 938 939 /* 940 * If the page is finally unwired, simply free it. 941 */ 942 vm_page_free_zero(m); 943 atomic_subtract_int(&cnt.v_wire_count, 1); 944 return (1); 945} 946 947static PMAP_INLINE int 948pmap_unwire_pte_hold(pmap_t pmap, vm_page_t m) 949{ 950 --m->wire_count; 951 if (m->wire_count == 0) 952 return (_pmap_unwire_pte_hold(pmap, m)); 953 else 954 return (0); 955} 956 957/* 958 * After removing a page table entry, this routine is used to 959 * conditionally free the page, and manage the hold/wire counts. 960 */ 961static int 962pmap_unuse_pt(pmap_t pmap, vm_offset_t va, vm_page_t mpte) 963{ 964 unsigned ptepindex; 965 pd_entry_t pteva; 966 967 if (va >= VM_MAXUSER_ADDRESS) 968 return (0); 969 970 if (mpte == NULL) { 971 ptepindex = (va >> SEGSHIFT); 972 if (pmap->pm_ptphint && 973 (pmap->pm_ptphint->pindex == ptepindex)) { 974 mpte = pmap->pm_ptphint; 975 } else { 976 pteva = *pmap_pde(pmap, va); 977 mpte = PHYS_TO_VM_PAGE(MIPS_CACHED_TO_PHYS(pteva)); 978 pmap->pm_ptphint = mpte; 979 } 980 } 981 return pmap_unwire_pte_hold(pmap, mpte); 982} 983 984void 985pmap_pinit0(pmap_t pmap) 986{ 987 int i; 988 989 PMAP_LOCK_INIT(pmap); 990 pmap->pm_segtab = kernel_segmap; 991 pmap->pm_active = 0; 992 pmap->pm_ptphint = NULL; 993 for (i = 0; i < MAXCPU; i++) { 994 pmap->pm_asid[i].asid = PMAP_ASID_RESERVED; 995 pmap->pm_asid[i].gen = 0; 996 } 997 PCPU_SET(curpmap, pmap); 998 TAILQ_INIT(&pmap->pm_pvlist); 999 bzero(&pmap->pm_stats, sizeof pmap->pm_stats); 1000} 1001 1002/* 1003 * Initialize a preallocated and zeroed pmap structure, 1004 * such as one in a vmspace structure. 1005 */ 1006int 1007pmap_pinit(pmap_t pmap) 1008{ 1009 vm_page_t ptdpg; 1010 int i; 1011 int req; 1012 1013 PMAP_LOCK_INIT(pmap); 1014 1015 req = VM_ALLOC_NOOBJ | VM_ALLOC_NORMAL | VM_ALLOC_WIRED | 1016 VM_ALLOC_ZERO; 1017 1018#ifdef VM_ALLOC_WIRED_TLB_PG_POOL 1019 if (need_wired_tlb_page_pool) 1020 req |= VM_ALLOC_WIRED_TLB_PG_POOL; 1021#endif 1022 /* 1023 * allocate the page directory page 1024 */ 1025 while ((ptdpg = vm_page_alloc(NULL, NUSERPGTBLS, req)) == NULL) 1026 VM_WAIT; 1027 1028 ptdpg->valid = VM_PAGE_BITS_ALL; 1029 1030 pmap->pm_segtab = (pd_entry_t *) 1031 MIPS_PHYS_TO_CACHED(VM_PAGE_TO_PHYS(ptdpg)); 1032 if ((ptdpg->flags & PG_ZERO) == 0) 1033 bzero(pmap->pm_segtab, PAGE_SIZE); 1034 1035 pmap->pm_active = 0; 1036 pmap->pm_ptphint = NULL; 1037 for (i = 0; i < MAXCPU; i++) { 1038 pmap->pm_asid[i].asid = PMAP_ASID_RESERVED; 1039 pmap->pm_asid[i].gen = 0; 1040 } 1041 TAILQ_INIT(&pmap->pm_pvlist); 1042 bzero(&pmap->pm_stats, sizeof pmap->pm_stats); 1043 1044 return (1); 1045} 1046 1047/* 1048 * this routine is called if the page table page is not 1049 * mapped correctly. 1050 */ 1051static vm_page_t 1052_pmap_allocpte(pmap_t pmap, unsigned ptepindex, int flags) 1053{ 1054 vm_offset_t pteva, ptepa; 1055 vm_page_t m; 1056 int req; 1057 1058 KASSERT((flags & (M_NOWAIT | M_WAITOK)) == M_NOWAIT || 1059 (flags & (M_NOWAIT | M_WAITOK)) == M_WAITOK, 1060 ("_pmap_allocpte: flags is neither M_NOWAIT nor M_WAITOK")); 1061 1062 req = VM_ALLOC_WIRED | VM_ALLOC_ZERO | VM_ALLOC_NOOBJ; 1063#ifdef VM_ALLOC_WIRED_TLB_PG_POOL 1064 if (need_wired_tlb_page_pool) 1065 req |= VM_ALLOC_WIRED_TLB_PG_POOL; 1066#endif 1067 /* 1068 * Find or fabricate a new pagetable page 1069 */ 1070 if ((m = vm_page_alloc(NULL, ptepindex, req)) == NULL) { 1071 if (flags & M_WAITOK) { 1072 PMAP_UNLOCK(pmap); 1073 vm_page_unlock_queues(); 1074 VM_WAIT; 1075 vm_page_lock_queues(); 1076 PMAP_LOCK(pmap); 1077 } 1078 /* 1079 * Indicate the need to retry. While waiting, the page 1080 * table page may have been allocated. 1081 */ 1082 return (NULL); 1083 } 1084 if ((m->flags & PG_ZERO) == 0) 1085 pmap_zero_page(m); 1086 1087 KASSERT(m->queue == PQ_NONE, 1088 ("_pmap_allocpte: %p->queue != PQ_NONE", m)); 1089 1090 /* 1091 * Map the pagetable page into the process address space, if it 1092 * isn't already there. 1093 */ 1094 1095 pmap->pm_stats.resident_count++; 1096 1097 ptepa = VM_PAGE_TO_PHYS(m); 1098 pteva = MIPS_PHYS_TO_CACHED(ptepa); 1099 pmap->pm_segtab[ptepindex] = (pd_entry_t)pteva; 1100 1101 /* 1102 * Set the page table hint 1103 */ 1104 pmap->pm_ptphint = m; 1105 1106 /* 1107 * Kernel page tables are allocated in pmap_bootstrap() or 1108 * pmap_growkernel(). 1109 */ 1110 if (is_kernel_pmap(pmap)) 1111 panic("_pmap_allocpte() called for kernel pmap\n"); 1112 1113 m->valid = VM_PAGE_BITS_ALL; 1114 vm_page_flag_clear(m, PG_ZERO); 1115 1116 return (m); 1117} 1118 1119static vm_page_t 1120pmap_allocpte(pmap_t pmap, vm_offset_t va, int flags) 1121{ 1122 unsigned ptepindex; 1123 vm_offset_t pteva; 1124 vm_page_t m; 1125 1126 KASSERT((flags & (M_NOWAIT | M_WAITOK)) == M_NOWAIT || 1127 (flags & (M_NOWAIT | M_WAITOK)) == M_WAITOK, 1128 ("pmap_allocpte: flags is neither M_NOWAIT nor M_WAITOK")); 1129 1130 /* 1131 * Calculate pagetable page index 1132 */ 1133 ptepindex = va >> SEGSHIFT; 1134retry: 1135 /* 1136 * Get the page directory entry 1137 */ 1138 pteva = (vm_offset_t)pmap->pm_segtab[ptepindex]; 1139 1140 /* 1141 * If the page table page is mapped, we just increment the hold 1142 * count, and activate it. 1143 */ 1144 if (pteva) { 1145 /* 1146 * In order to get the page table page, try the hint first. 1147 */ 1148 if (pmap->pm_ptphint && 1149 (pmap->pm_ptphint->pindex == ptepindex)) { 1150 m = pmap->pm_ptphint; 1151 } else { 1152 m = PHYS_TO_VM_PAGE(MIPS_CACHED_TO_PHYS(pteva)); 1153 pmap->pm_ptphint = m; 1154 } 1155 m->wire_count++; 1156 } else { 1157 /* 1158 * Here if the pte page isn't mapped, or if it has been 1159 * deallocated. 1160 */ 1161 m = _pmap_allocpte(pmap, ptepindex, flags); 1162 if (m == NULL && (flags & M_WAITOK)) 1163 goto retry; 1164 } 1165 return m; 1166} 1167 1168 1169/*************************************************** 1170* Pmap allocation/deallocation routines. 1171 ***************************************************/ 1172/* 1173 * Revision 1.397 1174 * - Merged pmap_release and pmap_release_free_page. When pmap_release is 1175 * called only the page directory page(s) can be left in the pmap pte 1176 * object, since all page table pages will have been freed by 1177 * pmap_remove_pages and pmap_remove. In addition, there can only be one 1178 * reference to the pmap and the page directory is wired, so the page(s) 1179 * can never be busy. So all there is to do is clear the magic mappings 1180 * from the page directory and free the page(s). 1181 */ 1182 1183 1184/* 1185 * Release any resources held by the given physical map. 1186 * Called when a pmap initialized by pmap_pinit is being released. 1187 * Should only be called if the map contains no valid mappings. 1188 */ 1189void 1190pmap_release(pmap_t pmap) 1191{ 1192 vm_page_t ptdpg; 1193 1194 KASSERT(pmap->pm_stats.resident_count == 0, 1195 ("pmap_release: pmap resident count %ld != 0", 1196 pmap->pm_stats.resident_count)); 1197 1198 ptdpg = PHYS_TO_VM_PAGE(MIPS_CACHED_TO_PHYS(pmap->pm_segtab)); 1199 ptdpg->wire_count--; 1200 atomic_subtract_int(&cnt.v_wire_count, 1); 1201 vm_page_free_zero(ptdpg); 1202} 1203 1204/* 1205 * grow the number of kernel page table entries, if needed 1206 */ 1207void 1208pmap_growkernel(vm_offset_t addr) 1209{ 1210 vm_offset_t ptppaddr; 1211 vm_page_t nkpg; 1212 pt_entry_t *pte; 1213 int i, req; 1214 1215 mtx_assert(&kernel_map->system_mtx, MA_OWNED); 1216 if (kernel_vm_end == 0) { 1217 kernel_vm_end = VM_MIN_KERNEL_ADDRESS + VM_KERNEL_ALLOC_OFFSET; 1218 nkpt = 0; 1219 while (segtab_pde(kernel_segmap, kernel_vm_end)) { 1220 kernel_vm_end = (kernel_vm_end + PAGE_SIZE * NPTEPG) & 1221 ~(PAGE_SIZE * NPTEPG - 1); 1222 nkpt++; 1223 if (kernel_vm_end - 1 >= kernel_map->max_offset) { 1224 kernel_vm_end = kernel_map->max_offset; 1225 break; 1226 } 1227 } 1228 } 1229 addr = (addr + PAGE_SIZE * NPTEPG) & ~(PAGE_SIZE * NPTEPG - 1); 1230 if (addr - 1 >= kernel_map->max_offset) 1231 addr = kernel_map->max_offset; 1232 while (kernel_vm_end < addr) { 1233 if (segtab_pde(kernel_segmap, kernel_vm_end)) { 1234 kernel_vm_end = (kernel_vm_end + PAGE_SIZE * NPTEPG) & 1235 ~(PAGE_SIZE * NPTEPG - 1); 1236 if (kernel_vm_end - 1 >= kernel_map->max_offset) { 1237 kernel_vm_end = kernel_map->max_offset; 1238 break; 1239 } 1240 continue; 1241 } 1242 /* 1243 * This index is bogus, but out of the way 1244 */ 1245 req = VM_ALLOC_INTERRUPT | VM_ALLOC_WIRED | VM_ALLOC_NOOBJ; 1246#ifdef VM_ALLOC_WIRED_TLB_PG_POOL 1247 if (need_wired_tlb_page_pool) 1248 req |= VM_ALLOC_WIRED_TLB_PG_POOL; 1249#endif 1250 nkpg = vm_page_alloc(NULL, nkpt, req); 1251 if (!nkpg) 1252 panic("pmap_growkernel: no memory to grow kernel"); 1253 1254 nkpt++; 1255 1256 ptppaddr = VM_PAGE_TO_PHYS(nkpg); 1257 if (ptppaddr >= MIPS_KSEG0_LARGEST_PHYS) { 1258 /* 1259 * We need to do something here, but I am not sure 1260 * what. We can access anything in the 0 - 512Meg 1261 * region, but if we get a page to go in the kernel 1262 * segmap that is outside of of that we really need 1263 * to have another mapping beyond the temporary ones 1264 * I have. Not sure how to do this yet. FIXME FIXME. 1265 */ 1266 panic("Gak, can't handle a k-page table outside of lower 512Meg"); 1267 } 1268 pte = (pt_entry_t *)MIPS_PHYS_TO_CACHED(ptppaddr); 1269 segtab_pde(kernel_segmap, kernel_vm_end) = (pd_entry_t)pte; 1270 1271 /* 1272 * The R[4-7]?00 stores only one copy of the Global bit in 1273 * the translation lookaside buffer for each 2 page entry. 1274 * Thus invalid entrys must have the Global bit set so when 1275 * Entry LO and Entry HI G bits are anded together they will 1276 * produce a global bit to store in the tlb. 1277 */ 1278 for (i = 0; i < NPTEPG; i++, pte++) 1279 *pte = PTE_G; 1280 1281 kernel_vm_end = (kernel_vm_end + PAGE_SIZE * NPTEPG) & 1282 ~(PAGE_SIZE * NPTEPG - 1); 1283 if (kernel_vm_end - 1 >= kernel_map->max_offset) { 1284 kernel_vm_end = kernel_map->max_offset; 1285 break; 1286 } 1287 } 1288} 1289 1290/*************************************************** 1291* page management routines. 1292 ***************************************************/ 1293 1294/* 1295 * free the pv_entry back to the free list 1296 */ 1297static PMAP_INLINE void 1298free_pv_entry(pv_entry_t pv) 1299{ 1300 1301 pv_entry_count--; 1302 uma_zfree(pvzone, pv); 1303} 1304 1305/* 1306 * get a new pv_entry, allocating a block from the system 1307 * when needed. 1308 * the memory allocation is performed bypassing the malloc code 1309 * because of the possibility of allocations at interrupt time. 1310 */ 1311static pv_entry_t 1312get_pv_entry(pmap_t locked_pmap) 1313{ 1314 static const struct timeval printinterval = { 60, 0 }; 1315 static struct timeval lastprint; 1316 struct vpgqueues *vpq; 1317 pt_entry_t *pte, oldpte; 1318 pmap_t pmap; 1319 pv_entry_t allocated_pv, next_pv, pv; 1320 vm_offset_t va; 1321 vm_page_t m; 1322 1323 PMAP_LOCK_ASSERT(locked_pmap, MA_OWNED); 1324 mtx_assert(&vm_page_queue_mtx, MA_OWNED); 1325 allocated_pv = uma_zalloc(pvzone, M_NOWAIT); 1326 if (allocated_pv != NULL) { 1327 pv_entry_count++; 1328 if (pv_entry_count > pv_entry_high_water) 1329 pagedaemon_wakeup(); 1330 else 1331 return (allocated_pv); 1332 } 1333 /* 1334 * Reclaim pv entries: At first, destroy mappings to inactive 1335 * pages. After that, if a pv entry is still needed, destroy 1336 * mappings to active pages. 1337 */ 1338 if (ratecheck(&lastprint, &printinterval)) 1339 printf("Approaching the limit on PV entries, " 1340 "increase the vm.pmap.shpgperproc tunable.\n"); 1341 vpq = &vm_page_queues[PQ_INACTIVE]; 1342retry: 1343 TAILQ_FOREACH(m, &vpq->pl, pageq) { 1344 if (m->hold_count || m->busy) 1345 continue; 1346 TAILQ_FOREACH_SAFE(pv, &m->md.pv_list, pv_list, next_pv) { 1347 va = pv->pv_va; 1348 pmap = pv->pv_pmap; 1349 /* Avoid deadlock and lock recursion. */ 1350 if (pmap > locked_pmap) 1351 PMAP_LOCK(pmap); 1352 else if (pmap != locked_pmap && !PMAP_TRYLOCK(pmap)) 1353 continue; 1354 pmap->pm_stats.resident_count--; 1355 pte = pmap_pte(pmap, va); 1356 KASSERT(pte != NULL, ("pte")); 1357 oldpte = loadandclear((u_int *)pte); 1358 if (is_kernel_pmap(pmap)) 1359 *pte = PTE_G; 1360 KASSERT((oldpte & PTE_W) == 0, 1361 ("wired pte for unwired page")); 1362 if (m->md.pv_flags & PV_TABLE_REF) 1363 vm_page_flag_set(m, PG_REFERENCED); 1364 if (oldpte & PTE_M) 1365 vm_page_dirty(m); 1366 pmap_invalidate_page(pmap, va); 1367 TAILQ_REMOVE(&pmap->pm_pvlist, pv, pv_plist); 1368 m->md.pv_list_count--; 1369 TAILQ_REMOVE(&m->md.pv_list, pv, pv_list); 1370 if (TAILQ_EMPTY(&m->md.pv_list)) { 1371 vm_page_flag_clear(m, PG_WRITEABLE); 1372 m->md.pv_flags &= ~(PV_TABLE_REF | PV_TABLE_MOD); 1373 } 1374 pmap_unuse_pt(pmap, va, pv->pv_ptem); 1375 if (pmap != locked_pmap) 1376 PMAP_UNLOCK(pmap); 1377 if (allocated_pv == NULL) 1378 allocated_pv = pv; 1379 else 1380 free_pv_entry(pv); 1381 } 1382 } 1383 if (allocated_pv == NULL) { 1384 if (vpq == &vm_page_queues[PQ_INACTIVE]) { 1385 vpq = &vm_page_queues[PQ_ACTIVE]; 1386 goto retry; 1387 } 1388 panic("get_pv_entry: increase the vm.pmap.shpgperproc tunable"); 1389 } 1390 return (allocated_pv); 1391} 1392 1393/* 1394 * Revision 1.370 1395 * 1396 * Move pmap_collect() out of the machine-dependent code, rename it 1397 * to reflect its new location, and add page queue and flag locking. 1398 * 1399 * Notes: (1) alpha, i386, and ia64 had identical implementations 1400 * of pmap_collect() in terms of machine-independent interfaces; 1401 * (2) sparc64 doesn't require it; (3) powerpc had it as a TODO. 1402 * 1403 * MIPS implementation was identical to alpha [Junos 8.2] 1404 */ 1405 1406/* 1407 * If it is the first entry on the list, it is actually 1408 * in the header and we must copy the following entry up 1409 * to the header. Otherwise we must search the list for 1410 * the entry. In either case we free the now unused entry. 1411 */ 1412 1413static void 1414pmap_remove_entry(struct pmap *pmap, vm_page_t m, vm_offset_t va) 1415{ 1416 pv_entry_t pv; 1417 1418 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 1419 mtx_assert(&vm_page_queue_mtx, MA_OWNED); 1420 if (m->md.pv_list_count < pmap->pm_stats.resident_count) { 1421 TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) { 1422 if (pmap == pv->pv_pmap && va == pv->pv_va) 1423 break; 1424 } 1425 } else { 1426 TAILQ_FOREACH(pv, &pmap->pm_pvlist, pv_plist) { 1427 if (va == pv->pv_va) 1428 break; 1429 } 1430 } 1431 1432 KASSERT(pv != NULL, ("pmap_remove_entry: pv not found")); 1433 TAILQ_REMOVE(&m->md.pv_list, pv, pv_list); 1434 m->md.pv_list_count--; 1435 if (TAILQ_FIRST(&m->md.pv_list) == NULL) 1436 vm_page_flag_clear(m, PG_WRITEABLE); 1437 1438 TAILQ_REMOVE(&pmap->pm_pvlist, pv, pv_plist); 1439 free_pv_entry(pv); 1440} 1441 1442/* 1443 * Create a pv entry for page at pa for 1444 * (pmap, va). 1445 */ 1446static void 1447pmap_insert_entry(pmap_t pmap, vm_offset_t va, vm_page_t mpte, vm_page_t m, 1448 boolean_t wired) 1449{ 1450 pv_entry_t pv; 1451 1452 pv = get_pv_entry(pmap); 1453 pv->pv_va = va; 1454 pv->pv_pmap = pmap; 1455 pv->pv_ptem = mpte; 1456 pv->pv_wired = wired; 1457 1458 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 1459 mtx_assert(&vm_page_queue_mtx, MA_OWNED); 1460 TAILQ_INSERT_TAIL(&pmap->pm_pvlist, pv, pv_plist); 1461 TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_list); 1462 m->md.pv_list_count++; 1463} 1464 1465/* 1466 * Conditionally create a pv entry. 1467 */ 1468static boolean_t 1469pmap_try_insert_pv_entry(pmap_t pmap, vm_page_t mpte, vm_offset_t va, 1470 vm_page_t m) 1471{ 1472 pv_entry_t pv; 1473 1474 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 1475 mtx_assert(&vm_page_queue_mtx, MA_OWNED); 1476 if (pv_entry_count < pv_entry_high_water && 1477 (pv = uma_zalloc(pvzone, M_NOWAIT)) != NULL) { 1478 pv_entry_count++; 1479 pv->pv_va = va; 1480 pv->pv_pmap = pmap; 1481 pv->pv_ptem = mpte; 1482 pv->pv_wired = FALSE; 1483 TAILQ_INSERT_TAIL(&pmap->pm_pvlist, pv, pv_plist); 1484 TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_list); 1485 m->md.pv_list_count++; 1486 return (TRUE); 1487 } else 1488 return (FALSE); 1489} 1490 1491/* 1492 * pmap_remove_pte: do the things to unmap a page in a process 1493 */ 1494static int 1495pmap_remove_pte(struct pmap *pmap, pt_entry_t *ptq, vm_offset_t va) 1496{ 1497 pt_entry_t oldpte; 1498 vm_page_t m; 1499 vm_offset_t pa; 1500 1501 mtx_assert(&vm_page_queue_mtx, MA_OWNED); 1502 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 1503 1504 oldpte = loadandclear((u_int *)ptq); 1505 if (is_kernel_pmap(pmap)) 1506 *ptq = PTE_G; 1507 1508 if (oldpte & PTE_W) 1509 pmap->pm_stats.wired_count -= 1; 1510 1511 pmap->pm_stats.resident_count -= 1; 1512 pa = mips_tlbpfn_to_paddr(oldpte); 1513 1514 if (page_is_managed(pa)) { 1515 m = PHYS_TO_VM_PAGE(pa); 1516 if (oldpte & PTE_M) { 1517#if defined(PMAP_DIAGNOSTIC) 1518 if (pmap_nw_modified(oldpte)) { 1519 printf( 1520 "pmap_remove: modified page not writable: va: 0x%x, pte: 0x%x\n", 1521 va, oldpte); 1522 } 1523#endif 1524 vm_page_dirty(m); 1525 } 1526 if (m->md.pv_flags & PV_TABLE_REF) 1527 vm_page_flag_set(m, PG_REFERENCED); 1528 m->md.pv_flags &= ~(PV_TABLE_REF | PV_TABLE_MOD); 1529 1530 pmap_remove_entry(pmap, m, va); 1531 } 1532 return pmap_unuse_pt(pmap, va, NULL); 1533} 1534 1535/* 1536 * Remove a single page from a process address space 1537 */ 1538static void 1539pmap_remove_page(struct pmap *pmap, vm_offset_t va) 1540{ 1541 register pt_entry_t *ptq; 1542 1543 mtx_assert(&vm_page_queue_mtx, MA_OWNED); 1544 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 1545 ptq = pmap_pte(pmap, va); 1546 1547 /* 1548 * if there is no pte for this address, just skip it!!! 1549 */ 1550 if (!ptq || !pmap_pte_v(ptq)) { 1551 return; 1552 } 1553 1554 /* 1555 * Write back all caches from the page being destroyed 1556 */ 1557 mips_dcache_wbinv_range_index(va, NBPG); 1558 1559 /* 1560 * get a local va for mappings for this pmap. 1561 */ 1562 (void)pmap_remove_pte(pmap, ptq, va); 1563 pmap_invalidate_page(pmap, va); 1564 1565 return; 1566} 1567 1568/* 1569 * Remove the given range of addresses from the specified map. 1570 * 1571 * It is assumed that the start and end are properly 1572 * rounded to the page size. 1573 */ 1574void 1575pmap_remove(struct pmap *pmap, vm_offset_t sva, vm_offset_t eva) 1576{ 1577 vm_offset_t va, nva; 1578 1579 if (pmap == NULL) 1580 return; 1581 1582 if (pmap->pm_stats.resident_count == 0) 1583 return; 1584 1585 vm_page_lock_queues(); 1586 PMAP_LOCK(pmap); 1587 1588 /* 1589 * special handling of removing one page. a very common operation 1590 * and easy to short circuit some code. 1591 */ 1592 if ((sva + PAGE_SIZE) == eva) { 1593 pmap_remove_page(pmap, sva); 1594 goto out; 1595 } 1596 for (va = sva; va < eva; va = nva) { 1597 if (!*pmap_pde(pmap, va)) { 1598 nva = mips_segtrunc(va + MIPS_SEGSIZE); 1599 continue; 1600 } 1601 pmap_remove_page(pmap, va); 1602 nva = va + PAGE_SIZE; 1603 } 1604 1605out: 1606 vm_page_unlock_queues(); 1607 PMAP_UNLOCK(pmap); 1608} 1609 1610/* 1611 * Routine: pmap_remove_all 1612 * Function: 1613 * Removes this physical page from 1614 * all physical maps in which it resides. 1615 * Reflects back modify bits to the pager. 1616 * 1617 * Notes: 1618 * Original versions of this routine were very 1619 * inefficient because they iteratively called 1620 * pmap_remove (slow...) 1621 */ 1622 1623void 1624pmap_remove_all(vm_page_t m) 1625{ 1626 register pv_entry_t pv; 1627 register pt_entry_t *pte, tpte; 1628 1629 KASSERT((m->flags & PG_FICTITIOUS) == 0, 1630 ("pmap_remove_all: page %p is fictitious", m)); 1631 mtx_assert(&vm_page_queue_mtx, MA_OWNED); 1632 1633 if (m->md.pv_flags & PV_TABLE_REF) 1634 vm_page_flag_set(m, PG_REFERENCED); 1635 1636 while ((pv = TAILQ_FIRST(&m->md.pv_list)) != NULL) { 1637 PMAP_LOCK(pv->pv_pmap); 1638 1639 /* 1640 * If it's last mapping writeback all caches from 1641 * the page being destroyed 1642 */ 1643 if (m->md.pv_list_count == 1) 1644 mips_dcache_wbinv_range_index(pv->pv_va, NBPG); 1645 1646 pv->pv_pmap->pm_stats.resident_count--; 1647 1648 pte = pmap_pte(pv->pv_pmap, pv->pv_va); 1649 1650 tpte = loadandclear((u_int *)pte); 1651 if (is_kernel_pmap(pv->pv_pmap)) 1652 *pte = PTE_G; 1653 1654 if (tpte & PTE_W) 1655 pv->pv_pmap->pm_stats.wired_count--; 1656 1657 /* 1658 * Update the vm_page_t clean and reference bits. 1659 */ 1660 if (tpte & PTE_M) { 1661#if defined(PMAP_DIAGNOSTIC) 1662 if (pmap_nw_modified(tpte)) { 1663 printf( 1664 "pmap_remove_all: modified page not writable: va: 0x%x, pte: 0x%x\n", 1665 pv->pv_va, tpte); 1666 } 1667#endif 1668 vm_page_dirty(m); 1669 } 1670 pmap_invalidate_page(pv->pv_pmap, pv->pv_va); 1671 1672 TAILQ_REMOVE(&pv->pv_pmap->pm_pvlist, pv, pv_plist); 1673 TAILQ_REMOVE(&m->md.pv_list, pv, pv_list); 1674 m->md.pv_list_count--; 1675 pmap_unuse_pt(pv->pv_pmap, pv->pv_va, pv->pv_ptem); 1676 PMAP_UNLOCK(pv->pv_pmap); 1677 free_pv_entry(pv); 1678 } 1679 1680 vm_page_flag_clear(m, PG_WRITEABLE); 1681 m->md.pv_flags &= ~(PV_TABLE_REF | PV_TABLE_MOD); 1682} 1683 1684/* 1685 * Set the physical protection on the 1686 * specified range of this map as requested. 1687 */ 1688void 1689pmap_protect(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, vm_prot_t prot) 1690{ 1691 pt_entry_t *pte; 1692 1693 if (pmap == NULL) 1694 return; 1695 1696 if ((prot & VM_PROT_READ) == VM_PROT_NONE) { 1697 pmap_remove(pmap, sva, eva); 1698 return; 1699 } 1700 if (prot & VM_PROT_WRITE) 1701 return; 1702 1703 vm_page_lock_queues(); 1704 PMAP_LOCK(pmap); 1705 while (sva < eva) { 1706 pt_entry_t pbits, obits; 1707 vm_page_t m; 1708 vm_offset_t pa; 1709 1710 /* 1711 * If segment table entry is empty, skip this segment. 1712 */ 1713 if (!*pmap_pde(pmap, sva)) { 1714 sva = mips_segtrunc(sva + MIPS_SEGSIZE); 1715 continue; 1716 } 1717 /* 1718 * If pte is invalid, skip this page 1719 */ 1720 pte = pmap_pte(pmap, sva); 1721 if (!pmap_pte_v(pte)) { 1722 sva += PAGE_SIZE; 1723 continue; 1724 } 1725retry: 1726 obits = pbits = *pte; 1727 pa = mips_tlbpfn_to_paddr(pbits); 1728 1729 if (page_is_managed(pa)) { 1730 m = PHYS_TO_VM_PAGE(pa); 1731 if (m->md.pv_flags & PV_TABLE_REF) { 1732 vm_page_flag_set(m, PG_REFERENCED); 1733 m->md.pv_flags &= ~PV_TABLE_REF; 1734 } 1735 if (pbits & PTE_M) { 1736 vm_page_dirty(m); 1737 m->md.pv_flags &= ~PV_TABLE_MOD; 1738 } 1739 } 1740 pbits = (pbits & ~PTE_M) | PTE_RO; 1741 1742 if (pbits != *pte) { 1743 if (!atomic_cmpset_int((u_int *)pte, obits, pbits)) 1744 goto retry; 1745 pmap_update_page(pmap, sva, pbits); 1746 } 1747 sva += PAGE_SIZE; 1748 } 1749 vm_page_unlock_queues(); 1750 PMAP_UNLOCK(pmap); 1751} 1752 1753/* 1754 * Insert the given physical page (p) at 1755 * the specified virtual address (v) in the 1756 * target physical map with the protection requested. 1757 * 1758 * If specified, the page will be wired down, meaning 1759 * that the related pte can not be reclaimed. 1760 * 1761 * NB: This is the only routine which MAY NOT lazy-evaluate 1762 * or lose information. That is, this routine must actually 1763 * insert this page into the given map NOW. 1764 */ 1765void 1766pmap_enter(pmap_t pmap, vm_offset_t va, vm_prot_t access, vm_page_t m, 1767 vm_prot_t prot, boolean_t wired) 1768{ 1769 vm_offset_t pa, opa; 1770 register pt_entry_t *pte; 1771 pt_entry_t origpte, newpte; 1772 vm_page_t mpte, om; 1773 int rw = 0; 1774 1775 if (pmap == NULL) 1776 return; 1777 1778 va &= ~PAGE_MASK; 1779#ifdef PMAP_DIAGNOSTIC 1780 if (va > VM_MAX_KERNEL_ADDRESS) 1781 panic("pmap_enter: toobig"); 1782#endif 1783 1784 mpte = NULL; 1785 1786 vm_page_lock_queues(); 1787 PMAP_LOCK(pmap); 1788 1789 /* 1790 * In the case that a page table page is not resident, we are 1791 * creating it here. 1792 */ 1793 if (va < VM_MAXUSER_ADDRESS) { 1794 mpte = pmap_allocpte(pmap, va, M_WAITOK); 1795 } 1796 pte = pmap_pte(pmap, va); 1797 1798 /* 1799 * Page Directory table entry not valid, we need a new PT page 1800 */ 1801 if (pte == NULL) { 1802 panic("pmap_enter: invalid page directory, pdir=%p, va=%p\n", 1803 (void *)pmap->pm_segtab, (void *)va); 1804 } 1805 pa = VM_PAGE_TO_PHYS(m); 1806 om = NULL; 1807 origpte = *pte; 1808 opa = mips_tlbpfn_to_paddr(origpte); 1809 1810 /* 1811 * Mapping has not changed, must be protection or wiring change. 1812 */ 1813 if ((origpte & PTE_V) && (opa == pa)) { 1814 /* 1815 * Wiring change, just update stats. We don't worry about 1816 * wiring PT pages as they remain resident as long as there 1817 * are valid mappings in them. Hence, if a user page is 1818 * wired, the PT page will be also. 1819 */ 1820 if (wired && ((origpte & PTE_W) == 0)) 1821 pmap->pm_stats.wired_count++; 1822 else if (!wired && (origpte & PTE_W)) 1823 pmap->pm_stats.wired_count--; 1824 1825#if defined(PMAP_DIAGNOSTIC) 1826 if (pmap_nw_modified(origpte)) { 1827 printf( 1828 "pmap_enter: modified page not writable: va: 0x%x, pte: 0x%x\n", 1829 va, origpte); 1830 } 1831#endif 1832 1833 /* 1834 * Remove extra pte reference 1835 */ 1836 if (mpte) 1837 mpte->wire_count--; 1838 1839 /* 1840 * We might be turning off write access to the page, so we 1841 * go ahead and sense modify status. 1842 */ 1843 if (page_is_managed(opa)) { 1844 om = m; 1845 } 1846 goto validate; 1847 } 1848 /* 1849 * Mapping has changed, invalidate old range and fall through to 1850 * handle validating new mapping. 1851 */ 1852 if (opa) { 1853 if (origpte & PTE_W) 1854 pmap->pm_stats.wired_count--; 1855 1856 if (page_is_managed(opa)) { 1857 om = PHYS_TO_VM_PAGE(opa); 1858 pmap_remove_entry(pmap, om, va); 1859 } 1860 if (mpte != NULL) { 1861 mpte->wire_count--; 1862 KASSERT(mpte->wire_count > 0, 1863 ("pmap_enter: missing reference to page table page," 1864 " va: %p", (void *)va)); 1865 } 1866 } else 1867 pmap->pm_stats.resident_count++; 1868 1869 /* 1870 * Enter on the PV list if part of our managed memory. Note that we 1871 * raise IPL while manipulating pv_table since pmap_enter can be 1872 * called at interrupt time. 1873 */ 1874 if ((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) == 0) { 1875 KASSERT(va < kmi.clean_sva || va >= kmi.clean_eva, 1876 ("pmap_enter: managed mapping within the clean submap")); 1877 pmap_insert_entry(pmap, va, mpte, m, wired); 1878 } 1879 /* 1880 * Increment counters 1881 */ 1882 if (wired) 1883 pmap->pm_stats.wired_count++; 1884 1885validate: 1886 if ((access & VM_PROT_WRITE) != 0) 1887 m->md.pv_flags |= PV_TABLE_MOD | PV_TABLE_REF; 1888 rw = init_pte_prot(va, m, prot); 1889 1890#ifdef PMAP_DEBUG 1891 printf("pmap_enter: va: 0x%08x -> pa: 0x%08x\n", va, pa); 1892#endif 1893 /* 1894 * Now validate mapping with desired protection/wiring. 1895 */ 1896 newpte = mips_paddr_to_tlbpfn(pa) | rw | PTE_V; 1897 1898 if (is_cacheable_mem(pa)) 1899 newpte |= PTE_CACHE; 1900 else 1901 newpte |= PTE_UNCACHED; 1902 1903 if (wired) 1904 newpte |= PTE_W; 1905 1906 if (is_kernel_pmap(pmap)) { 1907 newpte |= PTE_G; 1908 } 1909 1910 /* 1911 * if the mapping or permission bits are different, we need to 1912 * update the pte. 1913 */ 1914 if (origpte != newpte) { 1915 if (origpte & PTE_V) { 1916 *pte = newpte; 1917 if (page_is_managed(opa) && (opa != pa)) { 1918 if (om->md.pv_flags & PV_TABLE_REF) 1919 vm_page_flag_set(om, PG_REFERENCED); 1920 om->md.pv_flags &= 1921 ~(PV_TABLE_REF | PV_TABLE_MOD); 1922 } 1923 if (origpte & PTE_M) { 1924 KASSERT((origpte & PTE_RW), 1925 ("pmap_enter: modified page not writable:" 1926 " va: %p, pte: 0x%lx", (void *)va, origpte)); 1927 if (page_is_managed(opa)) 1928 vm_page_dirty(om); 1929 } 1930 } else { 1931 *pte = newpte; 1932 } 1933 } 1934 pmap_update_page(pmap, va, newpte); 1935 1936 /* 1937 * Sync I & D caches for executable pages. Do this only if the the 1938 * target pmap belongs to the current process. Otherwise, an 1939 * unresolvable TLB miss may occur. 1940 */ 1941 if (!is_kernel_pmap(pmap) && (pmap == &curproc->p_vmspace->vm_pmap) && 1942 (prot & VM_PROT_EXECUTE)) { 1943 mips_icache_sync_range(va, NBPG); 1944 mips_dcache_wbinv_range(va, NBPG); 1945 } 1946 vm_page_unlock_queues(); 1947 PMAP_UNLOCK(pmap); 1948} 1949 1950/* 1951 * this code makes some *MAJOR* assumptions: 1952 * 1. Current pmap & pmap exists. 1953 * 2. Not wired. 1954 * 3. Read access. 1955 * 4. No page table pages. 1956 * but is *MUCH* faster than pmap_enter... 1957 */ 1958 1959void 1960pmap_enter_quick(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot) 1961{ 1962 1963 PMAP_LOCK(pmap); 1964 (void)pmap_enter_quick_locked(pmap, va, m, prot, NULL); 1965 PMAP_UNLOCK(pmap); 1966} 1967 1968static vm_page_t 1969pmap_enter_quick_locked(pmap_t pmap, vm_offset_t va, vm_page_t m, 1970 vm_prot_t prot, vm_page_t mpte) 1971{ 1972 pt_entry_t *pte; 1973 vm_offset_t pa; 1974 1975 KASSERT(va < kmi.clean_sva || va >= kmi.clean_eva || 1976 (m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) != 0, 1977 ("pmap_enter_quick_locked: managed mapping within the clean submap")); 1978 mtx_assert(&vm_page_queue_mtx, MA_OWNED); 1979 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 1980 1981 /* 1982 * In the case that a page table page is not resident, we are 1983 * creating it here. 1984 */ 1985 if (va < VM_MAXUSER_ADDRESS) { 1986 unsigned ptepindex; 1987 vm_offset_t pteva; 1988 1989 /* 1990 * Calculate pagetable page index 1991 */ 1992 ptepindex = va >> SEGSHIFT; 1993 if (mpte && (mpte->pindex == ptepindex)) { 1994 mpte->wire_count++; 1995 } else { 1996 /* 1997 * Get the page directory entry 1998 */ 1999 pteva = (vm_offset_t)pmap->pm_segtab[ptepindex]; 2000 2001 /* 2002 * If the page table page is mapped, we just 2003 * increment the hold count, and activate it. 2004 */ 2005 if (pteva) { 2006 if (pmap->pm_ptphint && 2007 (pmap->pm_ptphint->pindex == ptepindex)) { 2008 mpte = pmap->pm_ptphint; 2009 } else { 2010 mpte = PHYS_TO_VM_PAGE(MIPS_CACHED_TO_PHYS(pteva)); 2011 pmap->pm_ptphint = mpte; 2012 } 2013 mpte->wire_count++; 2014 } else { 2015 mpte = _pmap_allocpte(pmap, ptepindex, 2016 M_NOWAIT); 2017 if (mpte == NULL) 2018 return (mpte); 2019 } 2020 } 2021 } else { 2022 mpte = NULL; 2023 } 2024 2025 pte = pmap_pte(pmap, va); 2026 if (pmap_pte_v(pte)) { 2027 if (mpte != NULL) { 2028 mpte->wire_count--; 2029 mpte = NULL; 2030 } 2031 return (mpte); 2032 } 2033 2034 /* 2035 * Enter on the PV list if part of our managed memory. 2036 */ 2037 if ((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) == 0 && 2038 !pmap_try_insert_pv_entry(pmap, mpte, va, m)) { 2039 if (mpte != NULL) { 2040 pmap_unwire_pte_hold(pmap, mpte); 2041 mpte = NULL; 2042 } 2043 return (mpte); 2044 } 2045 2046 /* 2047 * Increment counters 2048 */ 2049 pmap->pm_stats.resident_count++; 2050 2051 pa = VM_PAGE_TO_PHYS(m); 2052 2053 /* 2054 * Now validate mapping with RO protection 2055 */ 2056 *pte = mips_paddr_to_tlbpfn(pa) | PTE_V; 2057 2058 if (is_cacheable_mem(pa)) 2059 *pte |= PTE_CACHE; 2060 else 2061 *pte |= PTE_UNCACHED; 2062 2063 if (is_kernel_pmap(pmap)) 2064 *pte |= PTE_G; 2065 else { 2066 *pte |= PTE_RO; 2067 /* 2068 * Sync I & D caches. Do this only if the the target pmap 2069 * belongs to the current process. Otherwise, an 2070 * unresolvable TLB miss may occur. */ 2071 if (pmap == &curproc->p_vmspace->vm_pmap) { 2072 va &= ~PAGE_MASK; 2073 mips_icache_sync_range(va, NBPG); 2074 mips_dcache_wbinv_range(va, NBPG); 2075 } 2076 } 2077 return (mpte); 2078} 2079 2080/* 2081 * Make a temporary mapping for a physical address. This is only intended 2082 * to be used for panic dumps. 2083 */ 2084void * 2085pmap_kenter_temporary(vm_paddr_t pa, int i) 2086{ 2087 vm_offset_t va; 2088 2089 if (i != 0) 2090 printf("%s: ERROR!!! More than one page of virtual address mapping not supported\n", 2091 __func__); 2092 2093#ifdef VM_ALLOC_WIRED_TLB_PG_POOL 2094 if (need_wired_tlb_page_pool) { 2095 va = pmap_map_fpage(pa, &fpages_shared[PMAP_FPAGE_KENTER_TEMP], 2096 TRUE); 2097 } else 2098#endif 2099 if (pa < MIPS_KSEG0_LARGEST_PHYS) { 2100 va = MIPS_PHYS_TO_CACHED(pa); 2101 } else { 2102 int cpu; 2103 struct local_sysmaps *sysm; 2104 2105 cpu = PCPU_GET(cpuid); 2106 sysm = &sysmap_lmem[cpu]; 2107 /* Since this is for the debugger, no locks or any other fun */ 2108 sysm->CMAP1 = mips_paddr_to_tlbpfn(pa) | PTE_RW | PTE_V | PTE_G | PTE_W | PTE_CACHE; 2109 pmap_TLB_update_kernel((vm_offset_t)sysm->CADDR1, sysm->CMAP1); 2110 sysm->valid1 = 1; 2111 va = (vm_offset_t)sysm->CADDR1; 2112 } 2113 return ((void *)va); 2114} 2115 2116void 2117pmap_kenter_temporary_free(vm_paddr_t pa) 2118{ 2119 int cpu; 2120 struct local_sysmaps *sysm; 2121 2122 if (pa < MIPS_KSEG0_LARGEST_PHYS) { 2123 /* nothing to do for this case */ 2124 return; 2125 } 2126 cpu = PCPU_GET(cpuid); 2127 sysm = &sysmap_lmem[cpu]; 2128 if (sysm->valid1) { 2129 pmap_TLB_invalidate_kernel((vm_offset_t)sysm->CADDR1); 2130 sysm->CMAP1 = 0; 2131 sysm->valid1 = 0; 2132 } 2133} 2134 2135/* 2136 * Moved the code to Machine Independent 2137 * vm_map_pmap_enter() 2138 */ 2139 2140/* 2141 * Maps a sequence of resident pages belonging to the same object. 2142 * The sequence begins with the given page m_start. This page is 2143 * mapped at the given virtual address start. Each subsequent page is 2144 * mapped at a virtual address that is offset from start by the same 2145 * amount as the page is offset from m_start within the object. The 2146 * last page in the sequence is the page with the largest offset from 2147 * m_start that can be mapped at a virtual address less than the given 2148 * virtual address end. Not every virtual page between start and end 2149 * is mapped; only those for which a resident page exists with the 2150 * corresponding offset from m_start are mapped. 2151 */ 2152void 2153pmap_enter_object(pmap_t pmap, vm_offset_t start, vm_offset_t end, 2154 vm_page_t m_start, vm_prot_t prot) 2155{ 2156 vm_page_t m, mpte; 2157 vm_pindex_t diff, psize; 2158 2159 VM_OBJECT_LOCK_ASSERT(m_start->object, MA_OWNED); 2160 psize = atop(end - start); 2161 mpte = NULL; 2162 m = m_start; 2163 PMAP_LOCK(pmap); 2164 while (m != NULL && (diff = m->pindex - m_start->pindex) < psize) { 2165 mpte = pmap_enter_quick_locked(pmap, start + ptoa(diff), m, 2166 prot, mpte); 2167 m = TAILQ_NEXT(m, listq); 2168 } 2169 PMAP_UNLOCK(pmap); 2170} 2171 2172/* 2173 * pmap_object_init_pt preloads the ptes for a given object 2174 * into the specified pmap. This eliminates the blast of soft 2175 * faults on process startup and immediately after an mmap. 2176 */ 2177void 2178pmap_object_init_pt(pmap_t pmap, vm_offset_t addr, 2179 vm_object_t object, vm_pindex_t pindex, vm_size_t size) 2180{ 2181 VM_OBJECT_LOCK_ASSERT(object, MA_OWNED); 2182 KASSERT(object->type == OBJT_DEVICE || object->type == OBJT_SG, 2183 ("pmap_object_init_pt: non-device object")); 2184} 2185 2186/* 2187 * Routine: pmap_change_wiring 2188 * Function: Change the wiring attribute for a map/virtual-address 2189 * pair. 2190 * In/out conditions: 2191 * The mapping must already exist in the pmap. 2192 */ 2193void 2194pmap_change_wiring(pmap_t pmap, vm_offset_t va, boolean_t wired) 2195{ 2196 register pt_entry_t *pte; 2197 2198 if (pmap == NULL) 2199 return; 2200 2201 PMAP_LOCK(pmap); 2202 pte = pmap_pte(pmap, va); 2203 2204 if (wired && !pmap_pte_w(pte)) 2205 pmap->pm_stats.wired_count++; 2206 else if (!wired && pmap_pte_w(pte)) 2207 pmap->pm_stats.wired_count--; 2208 2209 /* 2210 * Wiring is not a hardware characteristic so there is no need to 2211 * invalidate TLB. 2212 */ 2213 pmap_pte_set_w(pte, wired); 2214 PMAP_UNLOCK(pmap); 2215} 2216 2217/* 2218 * Copy the range specified by src_addr/len 2219 * from the source map to the range dst_addr/len 2220 * in the destination map. 2221 * 2222 * This routine is only advisory and need not do anything. 2223 */ 2224 2225void 2226pmap_copy(pmap_t dst_pmap, pmap_t src_pmap, vm_offset_t dst_addr, 2227 vm_size_t len, vm_offset_t src_addr) 2228{ 2229} 2230 2231/* 2232 * pmap_zero_page zeros the specified hardware page by mapping 2233 * the page into KVM and using bzero to clear its contents. 2234 */ 2235void 2236pmap_zero_page(vm_page_t m) 2237{ 2238 vm_offset_t va; 2239 vm_paddr_t phys = VM_PAGE_TO_PHYS(m); 2240 2241#ifdef VM_ALLOC_WIRED_TLB_PG_POOL 2242 if (need_wired_tlb_page_pool) { 2243 struct fpage *fp1; 2244 struct sysmaps *sysmaps; 2245 2246 sysmaps = &sysmaps_pcpu[PCPU_GET(cpuid)]; 2247 mtx_lock(&sysmaps->lock); 2248 sched_pin(); 2249 2250 fp1 = &sysmaps->fp[PMAP_FPAGE1]; 2251 va = pmap_map_fpage(phys, fp1, FALSE); 2252 bzero((caddr_t)va, PAGE_SIZE); 2253 pmap_unmap_fpage(phys, fp1); 2254 sched_unpin(); 2255 mtx_unlock(&sysmaps->lock); 2256 /* 2257 * Should you do cache flush? 2258 */ 2259 } else 2260#endif 2261 if (phys < MIPS_KSEG0_LARGEST_PHYS) { 2262 2263 va = MIPS_PHYS_TO_CACHED(phys); 2264 2265 bzero((caddr_t)va, PAGE_SIZE); 2266 mips_dcache_wbinv_range(va, PAGE_SIZE); 2267 } else { 2268 int cpu; 2269 struct local_sysmaps *sysm; 2270 2271 cpu = PCPU_GET(cpuid); 2272 sysm = &sysmap_lmem[cpu]; 2273 PMAP_LGMEM_LOCK(sysm); 2274 sched_pin(); 2275 sysm->CMAP1 = mips_paddr_to_tlbpfn(phys) | PTE_RW | PTE_V | PTE_G | PTE_W | PTE_CACHE; 2276 pmap_TLB_update_kernel((vm_offset_t)sysm->CADDR1, sysm->CMAP1); 2277 sysm->valid1 = 1; 2278 bzero(sysm->CADDR1, PAGE_SIZE); 2279 pmap_TLB_invalidate_kernel((vm_offset_t)sysm->CADDR1); 2280 sysm->CMAP1 = 0; 2281 sysm->valid1 = 0; 2282 sched_unpin(); 2283 PMAP_LGMEM_UNLOCK(sysm); 2284 } 2285 2286} 2287 2288/* 2289 * pmap_zero_page_area zeros the specified hardware page by mapping 2290 * the page into KVM and using bzero to clear its contents. 2291 * 2292 * off and size may not cover an area beyond a single hardware page. 2293 */ 2294void 2295pmap_zero_page_area(vm_page_t m, int off, int size) 2296{ 2297 vm_offset_t va; 2298 vm_paddr_t phys = VM_PAGE_TO_PHYS(m); 2299 2300#ifdef VM_ALLOC_WIRED_TLB_PG_POOL 2301 if (need_wired_tlb_page_pool) { 2302 struct fpage *fp1; 2303 struct sysmaps *sysmaps; 2304 2305 sysmaps = &sysmaps_pcpu[PCPU_GET(cpuid)]; 2306 mtx_lock(&sysmaps->lock); 2307 sched_pin(); 2308 2309 fp1 = &sysmaps->fp[PMAP_FPAGE1]; 2310 va = pmap_map_fpage(phys, fp1, FALSE); 2311 bzero((caddr_t)va + off, size); 2312 pmap_unmap_fpage(phys, fp1); 2313 2314 sched_unpin(); 2315 mtx_unlock(&sysmaps->lock); 2316 } else 2317#endif 2318 if (phys < MIPS_KSEG0_LARGEST_PHYS) { 2319 va = MIPS_PHYS_TO_CACHED(phys); 2320 bzero((char *)(caddr_t)va + off, size); 2321 mips_dcache_wbinv_range(va + off, size); 2322 } else { 2323 int cpu; 2324 struct local_sysmaps *sysm; 2325 2326 cpu = PCPU_GET(cpuid); 2327 sysm = &sysmap_lmem[cpu]; 2328 PMAP_LGMEM_LOCK(sysm); 2329 sched_pin(); 2330 sysm->CMAP1 = mips_paddr_to_tlbpfn(phys) | PTE_RW | PTE_V | PTE_G | PTE_W | PTE_CACHE; 2331 pmap_TLB_update_kernel((vm_offset_t)sysm->CADDR1, sysm->CMAP1); 2332 sysm->valid1 = 1; 2333 bzero((char *)sysm->CADDR1 + off, size); 2334 pmap_TLB_invalidate_kernel((vm_offset_t)sysm->CADDR1); 2335 sysm->CMAP1 = 0; 2336 sysm->valid1 = 0; 2337 sched_unpin(); 2338 PMAP_LGMEM_UNLOCK(sysm); 2339 } 2340} 2341 2342void 2343pmap_zero_page_idle(vm_page_t m) 2344{ 2345 vm_offset_t va; 2346 vm_paddr_t phys = VM_PAGE_TO_PHYS(m); 2347 2348#ifdef VM_ALLOC_WIRED_TLB_PG_POOL 2349 if (need_wired_tlb_page_pool) { 2350 sched_pin(); 2351 va = pmap_map_fpage(phys, &fpages_shared[PMAP_FPAGE3], FALSE); 2352 bzero((caddr_t)va, PAGE_SIZE); 2353 pmap_unmap_fpage(phys, &fpages_shared[PMAP_FPAGE3]); 2354 sched_unpin(); 2355 } else 2356#endif 2357 if (phys < MIPS_KSEG0_LARGEST_PHYS) { 2358 va = MIPS_PHYS_TO_CACHED(phys); 2359 bzero((caddr_t)va, PAGE_SIZE); 2360 mips_dcache_wbinv_range(va, PAGE_SIZE); 2361 } else { 2362 int cpu; 2363 struct local_sysmaps *sysm; 2364 2365 cpu = PCPU_GET(cpuid); 2366 sysm = &sysmap_lmem[cpu]; 2367 PMAP_LGMEM_LOCK(sysm); 2368 sched_pin(); 2369 sysm->CMAP1 = mips_paddr_to_tlbpfn(phys) | PTE_RW | PTE_V | PTE_G | PTE_W | PTE_CACHE; 2370 pmap_TLB_update_kernel((vm_offset_t)sysm->CADDR1, sysm->CMAP1); 2371 sysm->valid1 = 1; 2372 bzero(sysm->CADDR1, PAGE_SIZE); 2373 pmap_TLB_invalidate_kernel((vm_offset_t)sysm->CADDR1); 2374 sysm->CMAP1 = 0; 2375 sysm->valid1 = 0; 2376 sched_unpin(); 2377 PMAP_LGMEM_UNLOCK(sysm); 2378 } 2379 2380} 2381 2382/* 2383 * pmap_copy_page copies the specified (machine independent) 2384 * page by mapping the page into virtual memory and using 2385 * bcopy to copy the page, one machine dependent page at a 2386 * time. 2387 */ 2388void 2389pmap_copy_page(vm_page_t src, vm_page_t dst) 2390{ 2391 vm_offset_t va_src, va_dst; 2392 vm_paddr_t phy_src = VM_PAGE_TO_PHYS(src); 2393 vm_paddr_t phy_dst = VM_PAGE_TO_PHYS(dst); 2394 2395#ifdef VM_ALLOC_WIRED_TLB_PG_POOL 2396 if (need_wired_tlb_page_pool) { 2397 struct fpage *fp1, *fp2; 2398 struct sysmaps *sysmaps; 2399 2400 sysmaps = &sysmaps_pcpu[PCPU_GET(cpuid)]; 2401 mtx_lock(&sysmaps->lock); 2402 sched_pin(); 2403 2404 fp1 = &sysmaps->fp[PMAP_FPAGE1]; 2405 fp2 = &sysmaps->fp[PMAP_FPAGE2]; 2406 2407 va_src = pmap_map_fpage(phy_src, fp1, FALSE); 2408 va_dst = pmap_map_fpage(phy_dst, fp2, FALSE); 2409 2410 bcopy((caddr_t)va_src, (caddr_t)va_dst, PAGE_SIZE); 2411 2412 pmap_unmap_fpage(phy_src, fp1); 2413 pmap_unmap_fpage(phy_dst, fp2); 2414 sched_unpin(); 2415 mtx_unlock(&sysmaps->lock); 2416 2417 /* 2418 * Should you flush the cache? 2419 */ 2420 } else 2421#endif 2422 { 2423 if ((phy_src < MIPS_KSEG0_LARGEST_PHYS) && (phy_dst < MIPS_KSEG0_LARGEST_PHYS)) { 2424 /* easy case, all can be accessed via KSEG0 */ 2425 /* 2426 * Flush all caches for VA that are mapped to this page 2427 * to make sure that data in SDRAM is up to date 2428 */ 2429 pmap_flush_pvcache(src); 2430 mips_dcache_wbinv_range_index( 2431 MIPS_PHYS_TO_CACHED(phy_dst), NBPG); 2432 va_src = MIPS_PHYS_TO_CACHED(phy_src); 2433 va_dst = MIPS_PHYS_TO_CACHED(phy_dst); 2434 bcopy((caddr_t)va_src, (caddr_t)va_dst, PAGE_SIZE); 2435 mips_dcache_wbinv_range(va_dst, PAGE_SIZE); 2436 } else { 2437 int cpu; 2438 struct local_sysmaps *sysm; 2439 2440 cpu = PCPU_GET(cpuid); 2441 sysm = &sysmap_lmem[cpu]; 2442 PMAP_LGMEM_LOCK(sysm); 2443 sched_pin(); 2444 if (phy_src < MIPS_KSEG0_LARGEST_PHYS) { 2445 /* one side needs mapping - dest */ 2446 va_src = MIPS_PHYS_TO_CACHED(phy_src); 2447 sysm->CMAP2 = mips_paddr_to_tlbpfn(phy_dst) | PTE_RW | PTE_V | PTE_G | PTE_W | PTE_CACHE; 2448 pmap_TLB_update_kernel((vm_offset_t)sysm->CADDR2, sysm->CMAP2); 2449 sysm->valid2 = 2; 2450 va_dst = (vm_offset_t)sysm->CADDR2; 2451 } else if (phy_dst < MIPS_KSEG0_LARGEST_PHYS) { 2452 /* one side needs mapping - src */ 2453 va_dst = MIPS_PHYS_TO_CACHED(phy_dst); 2454 sysm->CMAP1 = mips_paddr_to_tlbpfn(phy_src) | PTE_RW | PTE_V | PTE_G | PTE_W | PTE_CACHE; 2455 pmap_TLB_update_kernel((vm_offset_t)sysm->CADDR1, sysm->CMAP1); 2456 va_src = (vm_offset_t)sysm->CADDR1; 2457 sysm->valid1 = 1; 2458 } else { 2459 /* all need mapping */ 2460 sysm->CMAP1 = mips_paddr_to_tlbpfn(phy_src) | PTE_RW | PTE_V | PTE_G | PTE_W | PTE_CACHE; 2461 sysm->CMAP2 = mips_paddr_to_tlbpfn(phy_dst) | PTE_RW | PTE_V | PTE_G | PTE_W | PTE_CACHE; 2462 pmap_TLB_update_kernel((vm_offset_t)sysm->CADDR1, sysm->CMAP1); 2463 pmap_TLB_update_kernel((vm_offset_t)sysm->CADDR2, sysm->CMAP2); 2464 sysm->valid1 = sysm->valid2 = 1; 2465 va_src = (vm_offset_t)sysm->CADDR1; 2466 va_dst = (vm_offset_t)sysm->CADDR2; 2467 } 2468 bcopy((void *)va_src, (void *)va_dst, PAGE_SIZE); 2469 if (sysm->valid1) { 2470 pmap_TLB_invalidate_kernel((vm_offset_t)sysm->CADDR1); 2471 sysm->CMAP1 = 0; 2472 sysm->valid1 = 0; 2473 } 2474 if (sysm->valid2) { 2475 pmap_TLB_invalidate_kernel((vm_offset_t)sysm->CADDR2); 2476 sysm->CMAP2 = 0; 2477 sysm->valid2 = 0; 2478 } 2479 sched_unpin(); 2480 PMAP_LGMEM_UNLOCK(sysm); 2481 } 2482 } 2483} 2484 2485/* 2486 * Returns true if the pmap's pv is one of the first 2487 * 16 pvs linked to from this page. This count may 2488 * be changed upwards or downwards in the future; it 2489 * is only necessary that true be returned for a small 2490 * subset of pmaps for proper page aging. 2491 */ 2492boolean_t 2493pmap_page_exists_quick(pmap_t pmap, vm_page_t m) 2494{ 2495 pv_entry_t pv; 2496 int loops = 0; 2497 2498 if (m->flags & PG_FICTITIOUS) 2499 return FALSE; 2500 2501 mtx_assert(&vm_page_queue_mtx, MA_OWNED); 2502 TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) { 2503 if (pv->pv_pmap == pmap) { 2504 return TRUE; 2505 } 2506 loops++; 2507 if (loops >= 16) 2508 break; 2509 } 2510 return (FALSE); 2511} 2512 2513/* 2514 * Remove all pages from specified address space 2515 * this aids process exit speeds. Also, this code 2516 * is special cased for current process only, but 2517 * can have the more generic (and slightly slower) 2518 * mode enabled. This is much faster than pmap_remove 2519 * in the case of running down an entire address space. 2520 */ 2521void 2522pmap_remove_pages(pmap_t pmap) 2523{ 2524 pt_entry_t *pte, tpte; 2525 pv_entry_t pv, npv; 2526 vm_page_t m; 2527 2528 if (pmap != vmspace_pmap(curthread->td_proc->p_vmspace)) { 2529 printf("warning: pmap_remove_pages called with non-current pmap\n"); 2530 return; 2531 } 2532 vm_page_lock_queues(); 2533 PMAP_LOCK(pmap); 2534 sched_pin(); 2535 //XXX need to be TAILQ_FOREACH_SAFE ? 2536 for (pv = TAILQ_FIRST(&pmap->pm_pvlist); pv; pv = npv) { 2537 2538 pte = pmap_pte(pv->pv_pmap, pv->pv_va); 2539 if (!pmap_pte_v(pte)) 2540 panic("pmap_remove_pages: page on pm_pvlist has no pte\n"); 2541 tpte = *pte; 2542 2543/* 2544 * We cannot remove wired pages from a process' mapping at this time 2545 */ 2546 if (tpte & PTE_W) { 2547 npv = TAILQ_NEXT(pv, pv_plist); 2548 continue; 2549 } 2550 *pte = is_kernel_pmap(pmap) ? PTE_G : 0; 2551 2552 m = PHYS_TO_VM_PAGE(mips_tlbpfn_to_paddr(tpte)); 2553 2554 KASSERT(m < &vm_page_array[vm_page_array_size], 2555 ("pmap_remove_pages: bad tpte %lx", tpte)); 2556 2557 pv->pv_pmap->pm_stats.resident_count--; 2558 2559 /* 2560 * Update the vm_page_t clean and reference bits. 2561 */ 2562 if (tpte & PTE_M) { 2563 vm_page_dirty(m); 2564 } 2565 npv = TAILQ_NEXT(pv, pv_plist); 2566 TAILQ_REMOVE(&pv->pv_pmap->pm_pvlist, pv, pv_plist); 2567 2568 m->md.pv_list_count--; 2569 TAILQ_REMOVE(&m->md.pv_list, pv, pv_list); 2570 if (TAILQ_FIRST(&m->md.pv_list) == NULL) { 2571 vm_page_flag_clear(m, PG_WRITEABLE); 2572 } 2573 pmap_unuse_pt(pv->pv_pmap, pv->pv_va, pv->pv_ptem); 2574 free_pv_entry(pv); 2575 } 2576 sched_unpin(); 2577 pmap_invalidate_all(pmap); 2578 PMAP_UNLOCK(pmap); 2579 vm_page_unlock_queues(); 2580} 2581 2582/* 2583 * pmap_testbit tests bits in pte's 2584 * note that the testbit/changebit routines are inline, 2585 * and a lot of things compile-time evaluate. 2586 */ 2587static boolean_t 2588pmap_testbit(vm_page_t m, int bit) 2589{ 2590 pv_entry_t pv; 2591 pt_entry_t *pte; 2592 boolean_t rv = FALSE; 2593 2594 if (m->flags & PG_FICTITIOUS) 2595 return rv; 2596 2597 if (TAILQ_FIRST(&m->md.pv_list) == NULL) 2598 return rv; 2599 2600 mtx_assert(&vm_page_queue_mtx, MA_OWNED); 2601 TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) { 2602#if defined(PMAP_DIAGNOSTIC) 2603 if (!pv->pv_pmap) { 2604 printf("Null pmap (tb) at va: 0x%x\n", pv->pv_va); 2605 continue; 2606 } 2607#endif 2608 PMAP_LOCK(pv->pv_pmap); 2609 pte = pmap_pte(pv->pv_pmap, pv->pv_va); 2610 rv = (*pte & bit) != 0; 2611 PMAP_UNLOCK(pv->pv_pmap); 2612 if (rv) 2613 break; 2614 } 2615 return (rv); 2616} 2617 2618/* 2619 * this routine is used to modify bits in ptes 2620 */ 2621static __inline void 2622pmap_changebit(vm_page_t m, int bit, boolean_t setem) 2623{ 2624 register pv_entry_t pv; 2625 register pt_entry_t *pte; 2626 2627 if (m->flags & PG_FICTITIOUS) 2628 return; 2629 2630 mtx_assert(&vm_page_queue_mtx, MA_OWNED); 2631 /* 2632 * Loop over all current mappings setting/clearing as appropos If 2633 * setting RO do we need to clear the VAC? 2634 */ 2635 TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) { 2636#if defined(PMAP_DIAGNOSTIC) 2637 if (!pv->pv_pmap) { 2638 printf("Null pmap (cb) at va: 0x%x\n", pv->pv_va); 2639 continue; 2640 } 2641#endif 2642 2643 PMAP_LOCK(pv->pv_pmap); 2644 pte = pmap_pte(pv->pv_pmap, pv->pv_va); 2645 2646 if (setem) { 2647 *(int *)pte |= bit; 2648 pmap_update_page(pv->pv_pmap, pv->pv_va, *pte); 2649 } else { 2650 vm_offset_t pbits = *(vm_offset_t *)pte; 2651 2652 if (pbits & bit) { 2653 if (bit == PTE_RW) { 2654 if (pbits & PTE_M) { 2655 vm_page_dirty(m); 2656 } 2657 *(int *)pte = (pbits & ~(PTE_M | PTE_RW)) | 2658 PTE_RO; 2659 } else { 2660 *(int *)pte = pbits & ~bit; 2661 } 2662 pmap_update_page(pv->pv_pmap, pv->pv_va, *pte); 2663 } 2664 } 2665 PMAP_UNLOCK(pv->pv_pmap); 2666 } 2667 if (!setem && bit == PTE_RW) 2668 vm_page_flag_clear(m, PG_WRITEABLE); 2669} 2670 2671/* 2672 * pmap_page_wired_mappings: 2673 * 2674 * Return the number of managed mappings to the given physical page 2675 * that are wired. 2676 */ 2677int 2678pmap_page_wired_mappings(vm_page_t m) 2679{ 2680 pv_entry_t pv; 2681 int count; 2682 2683 count = 0; 2684 if ((m->flags & PG_FICTITIOUS) != 0) 2685 return (count); 2686 mtx_assert(&vm_page_queue_mtx, MA_OWNED); 2687 TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) 2688 if (pv->pv_wired) 2689 count++; 2690 return (count); 2691} 2692 2693/* 2694 * Clear the write and modified bits in each of the given page's mappings. 2695 */ 2696void 2697pmap_remove_write(vm_page_t m) 2698{ 2699 pv_entry_t pv, npv; 2700 vm_offset_t va; 2701 pt_entry_t *pte; 2702 2703 if ((m->flags & PG_WRITEABLE) == 0) 2704 return; 2705 2706 /* 2707 * Loop over all current mappings setting/clearing as appropos. 2708 */ 2709 for (pv = TAILQ_FIRST(&m->md.pv_list); pv; pv = npv) { 2710 npv = TAILQ_NEXT(pv, pv_plist); 2711 pte = pmap_pte(pv->pv_pmap, pv->pv_va); 2712 2713 if ((pte == NULL) || !mips_pg_v(*pte)) 2714 panic("page on pm_pvlist has no pte\n"); 2715 2716 va = pv->pv_va; 2717 pmap_protect(pv->pv_pmap, va, va + PAGE_SIZE, 2718 VM_PROT_READ | VM_PROT_EXECUTE); 2719 } 2720 vm_page_flag_clear(m, PG_WRITEABLE); 2721} 2722 2723/* 2724 * pmap_ts_referenced: 2725 * 2726 * Return the count of reference bits for a page, clearing all of them. 2727 */ 2728int 2729pmap_ts_referenced(vm_page_t m) 2730{ 2731 if (m->flags & PG_FICTITIOUS) 2732 return (0); 2733 2734 if (m->md.pv_flags & PV_TABLE_REF) { 2735 m->md.pv_flags &= ~PV_TABLE_REF; 2736 return 1; 2737 } 2738 return 0; 2739} 2740 2741/* 2742 * pmap_is_modified: 2743 * 2744 * Return whether or not the specified physical page was modified 2745 * in any physical maps. 2746 */ 2747boolean_t 2748pmap_is_modified(vm_page_t m) 2749{ 2750 if (m->flags & PG_FICTITIOUS) 2751 return FALSE; 2752 2753 if (m->md.pv_flags & PV_TABLE_MOD) 2754 return TRUE; 2755 else 2756 return pmap_testbit(m, PTE_M); 2757} 2758 2759/* N/C */ 2760 2761/* 2762 * pmap_is_prefaultable: 2763 * 2764 * Return whether or not the specified virtual address is elgible 2765 * for prefault. 2766 */ 2767boolean_t 2768pmap_is_prefaultable(pmap_t pmap, vm_offset_t addr) 2769{ 2770 pt_entry_t *pte; 2771 boolean_t rv; 2772 2773 rv = FALSE; 2774 PMAP_LOCK(pmap); 2775 if (*pmap_pde(pmap, addr)) { 2776 pte = pmap_pte(pmap, addr); 2777 rv = (*pte == 0); 2778 } 2779 PMAP_UNLOCK(pmap); 2780 return (rv); 2781} 2782 2783/* 2784 * Clear the modify bits on the specified physical page. 2785 */ 2786void 2787pmap_clear_modify(vm_page_t m) 2788{ 2789 if (m->flags & PG_FICTITIOUS) 2790 return; 2791 mtx_assert(&vm_page_queue_mtx, MA_OWNED); 2792 if (m->md.pv_flags & PV_TABLE_MOD) { 2793 pmap_changebit(m, PTE_M, FALSE); 2794 m->md.pv_flags &= ~PV_TABLE_MOD; 2795 } 2796} 2797 2798/* 2799 * pmap_clear_reference: 2800 * 2801 * Clear the reference bit on the specified physical page. 2802 */ 2803void 2804pmap_clear_reference(vm_page_t m) 2805{ 2806 if (m->flags & PG_FICTITIOUS) 2807 return; 2808 2809 mtx_assert(&vm_page_queue_mtx, MA_OWNED); 2810 if (m->md.pv_flags & PV_TABLE_REF) { 2811 m->md.pv_flags &= ~PV_TABLE_REF; 2812 } 2813} 2814 2815/* 2816 * Miscellaneous support routines follow 2817 */ 2818 2819/* 2820 * Map a set of physical memory pages into the kernel virtual 2821 * address space. Return a pointer to where it is mapped. This 2822 * routine is intended to be used for mapping device memory, 2823 * NOT real memory. 2824 */ 2825 2826/* 2827 * Map a set of physical memory pages into the kernel virtual 2828 * address space. Return a pointer to where it is mapped. This 2829 * routine is intended to be used for mapping device memory, 2830 * NOT real memory. 2831 */ 2832void * 2833pmap_mapdev(vm_offset_t pa, vm_size_t size) 2834{ 2835 vm_offset_t va, tmpva, offset; 2836 2837 /* 2838 * KSEG1 maps only first 512M of phys address space. For 2839 * pa > 0x20000000 we should make proper mapping * using pmap_kenter. 2840 */ 2841 if ((pa + size - 1) < MIPS_KSEG0_LARGEST_PHYS) 2842 return (void *)MIPS_PHYS_TO_KSEG1(pa); 2843 else { 2844 offset = pa & PAGE_MASK; 2845 size = roundup(size + offset, PAGE_SIZE); 2846 2847 va = kmem_alloc_nofault(kernel_map, size); 2848 if (!va) 2849 panic("pmap_mapdev: Couldn't alloc kernel virtual memory"); 2850 pa = trunc_page(pa); 2851 for (tmpva = va; size > 0;) { 2852 pmap_kenter(tmpva, pa); 2853 size -= PAGE_SIZE; 2854 tmpva += PAGE_SIZE; 2855 pa += PAGE_SIZE; 2856 } 2857 } 2858 2859 return ((void *)(va + offset)); 2860} 2861 2862void 2863pmap_unmapdev(vm_offset_t va, vm_size_t size) 2864{ 2865 vm_offset_t base, offset, tmpva; 2866 2867 /* If the address is within KSEG1 then there is nothing to do */ 2868 if (va >= MIPS_KSEG1_START && va <= MIPS_KSEG1_END) 2869 return; 2870 2871 base = trunc_page(va); 2872 offset = va & PAGE_MASK; 2873 size = roundup(size + offset, PAGE_SIZE); 2874 for (tmpva = base; tmpva < base + size; tmpva += PAGE_SIZE) 2875 pmap_kremove(tmpva); 2876 kmem_free(kernel_map, base, size); 2877} 2878 2879/* 2880 * perform the pmap work for mincore 2881 */ 2882int 2883pmap_mincore(pmap_t pmap, vm_offset_t addr) 2884{ 2885 2886 pt_entry_t *ptep, pte; 2887 vm_page_t m; 2888 int val = 0; 2889 2890 PMAP_LOCK(pmap); 2891 ptep = pmap_pte(pmap, addr); 2892 pte = (ptep != NULL) ? *ptep : 0; 2893 PMAP_UNLOCK(pmap); 2894 2895 if (mips_pg_v(pte)) { 2896 vm_offset_t pa; 2897 2898 val = MINCORE_INCORE; 2899 pa = mips_tlbpfn_to_paddr(pte); 2900 if (!page_is_managed(pa)) 2901 return val; 2902 2903 m = PHYS_TO_VM_PAGE(pa); 2904 2905 /* 2906 * Modified by us 2907 */ 2908 if (pte & PTE_M) 2909 val |= MINCORE_MODIFIED | MINCORE_MODIFIED_OTHER; 2910 /* 2911 * Modified by someone 2912 */ 2913 else { 2914 vm_page_lock_queues(); 2915 if (m->dirty || pmap_is_modified(m)) 2916 val |= MINCORE_MODIFIED_OTHER; 2917 vm_page_unlock_queues(); 2918 } 2919 /* 2920 * Referenced by us or someone 2921 */ 2922 vm_page_lock_queues(); 2923 if ((m->flags & PG_REFERENCED) || pmap_ts_referenced(m)) { 2924 val |= MINCORE_REFERENCED | MINCORE_REFERENCED_OTHER; 2925 vm_page_flag_set(m, PG_REFERENCED); 2926 } 2927 vm_page_unlock_queues(); 2928 } 2929 return val; 2930} 2931 2932void 2933pmap_activate(struct thread *td) 2934{ 2935 pmap_t pmap, oldpmap; 2936 struct proc *p = td->td_proc; 2937 2938 critical_enter(); 2939 2940 pmap = vmspace_pmap(p->p_vmspace); 2941 oldpmap = PCPU_GET(curpmap); 2942 2943 if (oldpmap) 2944 atomic_clear_32(&oldpmap->pm_active, PCPU_GET(cpumask)); 2945 atomic_set_32(&pmap->pm_active, PCPU_GET(cpumask)); 2946 pmap_asid_alloc(pmap); 2947 if (td == curthread) { 2948 PCPU_SET(segbase, pmap->pm_segtab); 2949 MachSetPID(pmap->pm_asid[PCPU_GET(cpuid)].asid); 2950 } 2951 2952 PCPU_SET(curpmap, pmap); 2953 critical_exit(); 2954} 2955 2956void 2957pmap_sync_icache(pmap_t pm, vm_offset_t va, vm_size_t sz) 2958{ 2959} 2960 2961/* 2962 * Increase the starting virtual address of the given mapping if a 2963 * different alignment might result in more superpage mappings. 2964 */ 2965void 2966pmap_align_superpage(vm_object_t object, vm_ooffset_t offset, 2967 vm_offset_t *addr, vm_size_t size) 2968{ 2969 vm_offset_t superpage_offset; 2970 2971 if (size < NBSEG) 2972 return; 2973 if (object != NULL && (object->flags & OBJ_COLORED) != 0) 2974 offset += ptoa(object->pg_color); 2975 superpage_offset = offset & SEGOFSET; 2976 if (size - ((NBSEG - superpage_offset) & SEGOFSET) < NBSEG || 2977 (*addr & SEGOFSET) == superpage_offset) 2978 return; 2979 if ((*addr & SEGOFSET) < superpage_offset) 2980 *addr = (*addr & ~SEGOFSET) + superpage_offset; 2981 else 2982 *addr = ((*addr + SEGOFSET) & ~SEGOFSET) + superpage_offset; 2983} 2984 2985int pmap_pid_dump(int pid); 2986 2987int 2988pmap_pid_dump(int pid) 2989{ 2990 pmap_t pmap; 2991 struct proc *p; 2992 int npte = 0; 2993 int index; 2994 2995 sx_slock(&allproc_lock); 2996 LIST_FOREACH(p, &allproc, p_list) { 2997 if (p->p_pid != pid) 2998 continue; 2999 3000 if (p->p_vmspace) { 3001 int i, j; 3002 3003 printf("vmspace is %p\n", 3004 p->p_vmspace); 3005 index = 0; 3006 pmap = vmspace_pmap(p->p_vmspace); 3007 printf("pmap asid:%x generation:%x\n", 3008 pmap->pm_asid[0].asid, 3009 pmap->pm_asid[0].gen); 3010 for (i = 0; i < NUSERPGTBLS; i++) { 3011 pd_entry_t *pde; 3012 pt_entry_t *pte; 3013 unsigned base = i << SEGSHIFT; 3014 3015 pde = &pmap->pm_segtab[i]; 3016 if (pde && pmap_pde_v(pde)) { 3017 for (j = 0; j < 1024; j++) { 3018 vm_offset_t va = base + 3019 (j << PAGE_SHIFT); 3020 3021 pte = pmap_pte(pmap, va); 3022 if (pte && pmap_pte_v(pte)) { 3023 vm_offset_t pa; 3024 vm_page_t m; 3025 3026 pa = mips_tlbpfn_to_paddr(*pte); 3027 m = PHYS_TO_VM_PAGE(pa); 3028 printf("va: %p, pt: %p, h: %d, w: %d, f: 0x%x", 3029 (void *)va, 3030 (void *)pa, 3031 m->hold_count, 3032 m->wire_count, 3033 m->flags); 3034 npte++; 3035 index++; 3036 if (index >= 2) { 3037 index = 0; 3038 printf("\n"); 3039 } else { 3040 printf(" "); 3041 } 3042 } 3043 } 3044 } 3045 } 3046 } else { 3047 printf("Process pid:%d has no vm_space\n", pid); 3048 } 3049 break; 3050 } 3051 sx_sunlock(&allproc_lock); 3052 return npte; 3053} 3054 3055 3056#if defined(DEBUG) 3057 3058static void pads(pmap_t pm); 3059void pmap_pvdump(vm_offset_t pa); 3060 3061/* print address space of pmap*/ 3062static void 3063pads(pmap_t pm) 3064{ 3065 unsigned va, i, j; 3066 pt_entry_t *ptep; 3067 3068 if (pm == kernel_pmap) 3069 return; 3070 for (i = 0; i < NPTEPG; i++) 3071 if (pm->pm_segtab[i]) 3072 for (j = 0; j < NPTEPG; j++) { 3073 va = (i << SEGSHIFT) + (j << PAGE_SHIFT); 3074 if (pm == kernel_pmap && va < KERNBASE) 3075 continue; 3076 if (pm != kernel_pmap && 3077 va >= VM_MAXUSER_ADDRESS) 3078 continue; 3079 ptep = pmap_pte(pm, va); 3080 if (pmap_pte_v(ptep)) 3081 printf("%x:%x ", va, *(int *)ptep); 3082 } 3083 3084} 3085 3086void 3087pmap_pvdump(vm_offset_t pa) 3088{ 3089 register pv_entry_t pv; 3090 vm_page_t m; 3091 3092 printf("pa %x", pa); 3093 m = PHYS_TO_VM_PAGE(pa); 3094 for (pv = TAILQ_FIRST(&m->md.pv_list); pv; 3095 pv = TAILQ_NEXT(pv, pv_list)) { 3096 printf(" -> pmap %p, va %x", (void *)pv->pv_pmap, pv->pv_va); 3097 pads(pv->pv_pmap); 3098 } 3099 printf(" "); 3100} 3101 3102/* N/C */ 3103#endif 3104 3105 3106/* 3107 * Allocate TLB address space tag (called ASID or TLBPID) and return it. 3108 * It takes almost as much or more time to search the TLB for a 3109 * specific ASID and flush those entries as it does to flush the entire TLB. 3110 * Therefore, when we allocate a new ASID, we just take the next number. When 3111 * we run out of numbers, we flush the TLB, increment the generation count 3112 * and start over. ASID zero is reserved for kernel use. 3113 */ 3114static void 3115pmap_asid_alloc(pmap) 3116 pmap_t pmap; 3117{ 3118 if (pmap->pm_asid[PCPU_GET(cpuid)].asid != PMAP_ASID_RESERVED && 3119 pmap->pm_asid[PCPU_GET(cpuid)].gen == PCPU_GET(asid_generation)); 3120 else { 3121 if (PCPU_GET(next_asid) == pmap_max_asid) { 3122 MIPS_TBIAP(); 3123 PCPU_SET(asid_generation, 3124 (PCPU_GET(asid_generation) + 1) & ASIDGEN_MASK); 3125 if (PCPU_GET(asid_generation) == 0) { 3126 PCPU_SET(asid_generation, 1); 3127 } 3128 PCPU_SET(next_asid, 1); /* 0 means invalid */ 3129 } 3130 pmap->pm_asid[PCPU_GET(cpuid)].asid = PCPU_GET(next_asid); 3131 pmap->pm_asid[PCPU_GET(cpuid)].gen = PCPU_GET(asid_generation); 3132 PCPU_SET(next_asid, PCPU_GET(next_asid) + 1); 3133 } 3134 3135#ifdef DEBUG 3136 if (pmapdebug & (PDB_FOLLOW | PDB_TLBPID)) { 3137 if (curproc) 3138 printf("pmap_asid_alloc: curproc %d '%s' ", 3139 curproc->p_pid, curproc->p_comm); 3140 else 3141 printf("pmap_asid_alloc: curproc <none> "); 3142 printf("segtab %p asid %d\n", pmap->pm_segtab, 3143 pmap->pm_asid[PCPU_GET(cpuid)].asid); 3144 } 3145#endif 3146} 3147 3148int 3149page_is_managed(vm_offset_t pa) 3150{ 3151 vm_offset_t pgnum = mips_btop(pa); 3152 3153 if (pgnum >= first_page && (pgnum < (first_page + vm_page_array_size))) { 3154 vm_page_t m; 3155 3156 m = PHYS_TO_VM_PAGE(pa); 3157 if ((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) == 0) 3158 return 1; 3159 } 3160 return 0; 3161} 3162 3163static int 3164init_pte_prot(vm_offset_t va, vm_page_t m, vm_prot_t prot) 3165{ 3166 int rw = 0; 3167 3168 if (!(prot & VM_PROT_WRITE)) 3169 rw = PTE_ROPAGE; 3170 else { 3171 if (va >= VM_MIN_KERNEL_ADDRESS) { 3172 /* 3173 * Don't bother to trap on kernel writes, just 3174 * record page as dirty. 3175 */ 3176 rw = PTE_RWPAGE; 3177 vm_page_dirty(m); 3178 } else if ((m->md.pv_flags & PV_TABLE_MOD) || 3179 m->dirty == VM_PAGE_BITS_ALL) 3180 rw = PTE_RWPAGE; 3181 else 3182 rw = PTE_CWPAGE; 3183 vm_page_flag_set(m, PG_WRITEABLE); 3184 } 3185 return rw; 3186} 3187 3188/* 3189 * pmap_page_is_free: 3190 * 3191 * Called when a page is freed to allow pmap to clean up 3192 * any extra state associated with the page. In this case 3193 * clear modified/referenced bits. 3194 */ 3195void 3196pmap_page_is_free(vm_page_t m) 3197{ 3198 3199 m->md.pv_flags = 0; 3200} 3201 3202/* 3203 * pmap_set_modified: 3204 * 3205 * Sets the page modified and reference bits for the specified page. 3206 */ 3207void 3208pmap_set_modified(vm_offset_t pa) 3209{ 3210 3211 PHYS_TO_VM_PAGE(pa)->md.pv_flags |= (PV_TABLE_REF | PV_TABLE_MOD); 3212} 3213 3214#include <machine/db_machdep.h> 3215 3216/* 3217 * Dump the translation buffer (TLB) in readable form. 3218 */ 3219 3220void 3221db_dump_tlb(int first, int last) 3222{ 3223 struct tlb tlb; 3224 int tlbno; 3225 3226 tlbno = first; 3227 3228 while (tlbno <= last) { 3229 MachTLBRead(tlbno, &tlb); 3230 if (tlb.tlb_lo0 & PTE_V || tlb.tlb_lo1 & PTE_V) { 3231 printf("TLB %2d vad 0x%08x ", tlbno, (tlb.tlb_hi & 0xffffff00)); 3232 } else { 3233 printf("TLB*%2d vad 0x%08x ", tlbno, (tlb.tlb_hi & 0xffffff00)); 3234 } 3235 printf("0=0x%08x ", pfn_to_vad(tlb.tlb_lo0)); 3236 printf("%c", tlb.tlb_lo0 & PTE_M ? 'M' : ' '); 3237 printf("%c", tlb.tlb_lo0 & PTE_G ? 'G' : ' '); 3238 printf(" atr %x ", (tlb.tlb_lo0 >> 3) & 7); 3239 printf("1=0x%08x ", pfn_to_vad(tlb.tlb_lo1)); 3240 printf("%c", tlb.tlb_lo1 & PTE_M ? 'M' : ' '); 3241 printf("%c", tlb.tlb_lo1 & PTE_G ? 'G' : ' '); 3242 printf(" atr %x ", (tlb.tlb_lo1 >> 3) & 7); 3243 printf(" sz=%x pid=%x\n", tlb.tlb_mask, 3244 (tlb.tlb_hi & 0x000000ff) 3245 ); 3246 tlbno++; 3247 } 3248} 3249 3250#ifdef DDB 3251#include <sys/kernel.h> 3252#include <ddb/ddb.h> 3253 3254DB_SHOW_COMMAND(tlb, ddb_dump_tlb) 3255{ 3256 db_dump_tlb(0, num_tlbentries - 1); 3257} 3258 3259#endif 3260 3261/* 3262 * Routine: pmap_kextract 3263 * Function: 3264 * Extract the physical page address associated 3265 * virtual address. 3266 */ 3267 /* PMAP_INLINE */ vm_offset_t 3268pmap_kextract(vm_offset_t va) 3269{ 3270 vm_offset_t pa = 0; 3271 3272 if (va < MIPS_CACHED_MEMORY_ADDR) { 3273 /* user virtual address */ 3274 pt_entry_t *ptep; 3275 3276 if (curproc && curproc->p_vmspace) { 3277 ptep = pmap_pte(&curproc->p_vmspace->vm_pmap, va); 3278 if (ptep) 3279 pa = mips_tlbpfn_to_paddr(*ptep) | 3280 (va & PAGE_MASK); 3281 } 3282 } else if (va >= MIPS_CACHED_MEMORY_ADDR && 3283 va < MIPS_UNCACHED_MEMORY_ADDR) 3284 pa = MIPS_CACHED_TO_PHYS(va); 3285 else if (va >= MIPS_UNCACHED_MEMORY_ADDR && 3286 va < MIPS_KSEG2_START) 3287 pa = MIPS_UNCACHED_TO_PHYS(va); 3288#ifdef VM_ALLOC_WIRED_TLB_PG_POOL 3289 else if (need_wired_tlb_page_pool && ((va >= VM_MIN_KERNEL_ADDRESS) && 3290 (va < (VM_MIN_KERNEL_ADDRESS + VM_KERNEL_ALLOC_OFFSET)))) 3291 pa = MIPS_CACHED_TO_PHYS(va); 3292#endif 3293 else if (va >= MIPS_KSEG2_START && va < VM_MAX_KERNEL_ADDRESS) { 3294 pt_entry_t *ptep; 3295 3296 /* Is the kernel pmap initialized? */ 3297 if (kernel_pmap->pm_active) { 3298 if (va >= (vm_offset_t)virtual_sys_start) { 3299 /* Its inside the virtual address range */ 3300 ptep = pmap_pte(kernel_pmap, va); 3301 if (ptep) 3302 pa = mips_tlbpfn_to_paddr(*ptep) | 3303 (va & PAGE_MASK); 3304 } else { 3305 int i; 3306 3307 /* 3308 * its inside the special mapping area, I 3309 * don't think this should happen, but if it 3310 * does I want it toa all work right :-) 3311 * Note if it does happen, we assume the 3312 * caller has the lock? FIXME, this needs to 3313 * be checked FIXEM - RRS. 3314 */ 3315 for (i = 0; i < MAXCPU; i++) { 3316 if ((sysmap_lmem[i].valid1) && ((vm_offset_t)sysmap_lmem[i].CADDR1 == va)) { 3317 pa = mips_tlbpfn_to_paddr(sysmap_lmem[i].CMAP1); 3318 break; 3319 } 3320 if ((sysmap_lmem[i].valid2) && ((vm_offset_t)sysmap_lmem[i].CADDR2 == va)) { 3321 pa = mips_tlbpfn_to_paddr(sysmap_lmem[i].CMAP2); 3322 break; 3323 } 3324 } 3325 } 3326 } 3327 } 3328 return pa; 3329} 3330 3331void 3332pmap_flush_pvcache(vm_page_t m) 3333{ 3334 pv_entry_t pv; 3335 3336 if (m != NULL) { 3337 for (pv = TAILQ_FIRST(&m->md.pv_list); pv; 3338 pv = TAILQ_NEXT(pv, pv_list)) { 3339 mips_dcache_wbinv_range_index(pv->pv_va, NBPG); 3340 } 3341 } 3342} 3343