pmap.c revision 240241
1/* 2 * Copyright (c) 1991 Regents of the University of California. 3 * All rights reserved. 4 * Copyright (c) 1994 John S. Dyson 5 * All rights reserved. 6 * Copyright (c) 1994 David Greenman 7 * All rights reserved. 8 * 9 * This code is derived from software contributed to Berkeley by 10 * the Systems Programming Group of the University of Utah Computer 11 * Science Department and William Jolitz of UUNET Technologies Inc. 12 * 13 * Redistribution and use in source and binary forms, with or without 14 * modification, are permitted provided that the following conditions 15 * are met: 16 * 1. Redistributions of source code must retain the above copyright 17 * notice, this list of conditions and the following disclaimer. 18 * 2. Redistributions in binary form must reproduce the above copyright 19 * notice, this list of conditions and the following disclaimer in the 20 * documentation and/or other materials provided with the distribution. 21 * 4. Neither the name of the University nor the names of its contributors 22 * may be used to endorse or promote products derived from this software 23 * without specific prior written permission. 24 * 25 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 26 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 27 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 28 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 29 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 30 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 31 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 32 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 33 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 34 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 35 * SUCH DAMAGE. 36 * 37 * from: @(#)pmap.c 7.7 (Berkeley) 5/12/91 38 * from: src/sys/i386/i386/pmap.c,v 1.250.2.8 2000/11/21 00:09:14 ps 39 * JNPR: pmap.c,v 1.11.2.1 2007/08/16 11:51:06 girish 40 */ 41 42/* 43 * Manages physical address maps. 44 * 45 * In addition to hardware address maps, this 46 * module is called upon to provide software-use-only 47 * maps which may or may not be stored in the same 48 * form as hardware maps. These pseudo-maps are 49 * used to store intermediate results from copy 50 * operations to and from address spaces. 51 * 52 * Since the information managed by this module is 53 * also stored by the logical address mapping module, 54 * this module may throw away valid virtual-to-physical 55 * mappings at almost any time. However, invalidations 56 * of virtual-to-physical mappings must be done as 57 * requested. 58 * 59 * In order to cope with hardware architectures which 60 * make virtual-to-physical map invalidates expensive, 61 * this module may delay invalidate or reduced protection 62 * operations until such time as they are actually 63 * necessary. This module is given full information as 64 * to which processors are currently using which maps, 65 * and to when physical maps must be made correct. 66 */ 67 68#include <sys/cdefs.h> 69__FBSDID("$FreeBSD: head/sys/mips/mips/pmap.c 240241 2012-09-08 17:43:20Z alc $"); 70 71#include "opt_ddb.h" 72#include "opt_pmap.h" 73 74#include <sys/param.h> 75#include <sys/systm.h> 76#include <sys/lock.h> 77#include <sys/mman.h> 78#include <sys/msgbuf.h> 79#include <sys/mutex.h> 80#include <sys/pcpu.h> 81#include <sys/proc.h> 82#include <sys/rwlock.h> 83#include <sys/sched.h> 84#ifdef SMP 85#include <sys/smp.h> 86#else 87#include <sys/cpuset.h> 88#endif 89#include <sys/sysctl.h> 90#include <sys/vmmeter.h> 91 92#ifdef DDB 93#include <ddb/ddb.h> 94#endif 95 96#include <vm/vm.h> 97#include <vm/vm_param.h> 98#include <vm/vm_kern.h> 99#include <vm/vm_page.h> 100#include <vm/vm_map.h> 101#include <vm/vm_object.h> 102#include <vm/vm_extern.h> 103#include <vm/vm_pageout.h> 104#include <vm/vm_pager.h> 105#include <vm/uma.h> 106 107#include <machine/cache.h> 108#include <machine/md_var.h> 109#include <machine/tlb.h> 110 111#undef PMAP_DEBUG 112 113#if !defined(DIAGNOSTIC) 114#define PMAP_INLINE __inline 115#else 116#define PMAP_INLINE 117#endif 118 119#ifdef PV_STATS 120#define PV_STAT(x) do { x ; } while (0) 121#else 122#define PV_STAT(x) do { } while (0) 123#endif 124 125/* 126 * Get PDEs and PTEs for user/kernel address space 127 */ 128#define pmap_seg_index(v) (((v) >> SEGSHIFT) & (NPDEPG - 1)) 129#define pmap_pde_index(v) (((v) >> PDRSHIFT) & (NPDEPG - 1)) 130#define pmap_pte_index(v) (((v) >> PAGE_SHIFT) & (NPTEPG - 1)) 131#define pmap_pde_pindex(v) ((v) >> PDRSHIFT) 132 133#ifdef __mips_n64 134#define NUPDE (NPDEPG * NPDEPG) 135#define NUSERPGTBLS (NUPDE + NPDEPG) 136#else 137#define NUPDE (NPDEPG) 138#define NUSERPGTBLS (NUPDE) 139#endif 140 141#define is_kernel_pmap(x) ((x) == kernel_pmap) 142 143struct pmap kernel_pmap_store; 144pd_entry_t *kernel_segmap; 145 146vm_offset_t virtual_avail; /* VA of first avail page (after kernel bss) */ 147vm_offset_t virtual_end; /* VA of last avail page (end of kernel AS) */ 148 149static int nkpt; 150unsigned pmap_max_asid; /* max ASID supported by the system */ 151 152#define PMAP_ASID_RESERVED 0 153 154vm_offset_t kernel_vm_end = VM_MIN_KERNEL_ADDRESS; 155 156static void pmap_asid_alloc(pmap_t pmap); 157 158/* 159 * Isolate the global pv list lock from data and other locks to prevent false 160 * sharing within the cache. 161 */ 162static struct { 163 struct rwlock lock; 164 char padding[CACHE_LINE_SIZE - sizeof(struct rwlock)]; 165} pvh_global __aligned(CACHE_LINE_SIZE); 166 167#define pvh_global_lock pvh_global.lock 168 169/* 170 * Data for the pv entry allocation mechanism 171 */ 172static TAILQ_HEAD(pch, pv_chunk) pv_chunks = TAILQ_HEAD_INITIALIZER(pv_chunks); 173static int pv_entry_count; 174 175static void free_pv_chunk(struct pv_chunk *pc); 176static void free_pv_entry(pmap_t pmap, pv_entry_t pv); 177static pv_entry_t get_pv_entry(pmap_t pmap, boolean_t try); 178static vm_page_t pmap_pv_reclaim(pmap_t locked_pmap); 179static void pmap_pvh_free(struct md_page *pvh, pmap_t pmap, vm_offset_t va); 180static pv_entry_t pmap_pvh_remove(struct md_page *pvh, pmap_t pmap, 181 vm_offset_t va); 182static vm_page_t pmap_enter_quick_locked(pmap_t pmap, vm_offset_t va, 183 vm_page_t m, vm_prot_t prot, vm_page_t mpte); 184static int pmap_remove_pte(struct pmap *pmap, pt_entry_t *ptq, vm_offset_t va, 185 pd_entry_t pde); 186static void pmap_remove_page(struct pmap *pmap, vm_offset_t va); 187static void pmap_remove_entry(struct pmap *pmap, vm_page_t m, vm_offset_t va); 188static boolean_t pmap_try_insert_pv_entry(pmap_t pmap, vm_page_t mpte, 189 vm_offset_t va, vm_page_t m); 190static void pmap_update_page(pmap_t pmap, vm_offset_t va, pt_entry_t pte); 191static void pmap_invalidate_all(pmap_t pmap); 192static void pmap_invalidate_page(pmap_t pmap, vm_offset_t va); 193static void _pmap_unwire_ptp(pmap_t pmap, vm_offset_t va, vm_page_t m); 194 195static vm_page_t pmap_allocpte(pmap_t pmap, vm_offset_t va, int flags); 196static vm_page_t _pmap_allocpte(pmap_t pmap, unsigned ptepindex, int flags); 197static int pmap_unuse_pt(pmap_t, vm_offset_t, pd_entry_t); 198static pt_entry_t init_pte_prot(vm_page_t m, vm_prot_t access, vm_prot_t prot); 199 200#ifdef SMP 201static void pmap_invalidate_page_action(void *arg); 202static void pmap_update_page_action(void *arg); 203#endif 204 205#ifndef __mips_n64 206/* 207 * This structure is for high memory (memory above 512Meg in 32 bit) support. 208 * The highmem area does not have a KSEG0 mapping, and we need a mechanism to 209 * do temporary per-CPU mappings for pmap_zero_page, pmap_copy_page etc. 210 * 211 * At bootup, we reserve 2 virtual pages per CPU for mapping highmem pages. To 212 * access a highmem physical address on a CPU, we map the physical address to 213 * the reserved virtual address for the CPU in the kernel pagetable. This is 214 * done with interrupts disabled(although a spinlock and sched_pin would be 215 * sufficient). 216 */ 217struct local_sysmaps { 218 vm_offset_t base; 219 uint32_t saved_intr; 220 uint16_t valid1, valid2; 221}; 222static struct local_sysmaps sysmap_lmem[MAXCPU]; 223 224static __inline void 225pmap_alloc_lmem_map(void) 226{ 227 int i; 228 229 for (i = 0; i < MAXCPU; i++) { 230 sysmap_lmem[i].base = virtual_avail; 231 virtual_avail += PAGE_SIZE * 2; 232 sysmap_lmem[i].valid1 = sysmap_lmem[i].valid2 = 0; 233 } 234} 235 236static __inline vm_offset_t 237pmap_lmem_map1(vm_paddr_t phys) 238{ 239 struct local_sysmaps *sysm; 240 pt_entry_t *pte, npte; 241 vm_offset_t va; 242 uint32_t intr; 243 int cpu; 244 245 intr = intr_disable(); 246 cpu = PCPU_GET(cpuid); 247 sysm = &sysmap_lmem[cpu]; 248 sysm->saved_intr = intr; 249 va = sysm->base; 250 npte = TLBLO_PA_TO_PFN(phys) | 251 PTE_D | PTE_V | PTE_G | PTE_W | PTE_C_CACHE; 252 pte = pmap_pte(kernel_pmap, va); 253 *pte = npte; 254 sysm->valid1 = 1; 255 return (va); 256} 257 258static __inline vm_offset_t 259pmap_lmem_map2(vm_paddr_t phys1, vm_paddr_t phys2) 260{ 261 struct local_sysmaps *sysm; 262 pt_entry_t *pte, npte; 263 vm_offset_t va1, va2; 264 uint32_t intr; 265 int cpu; 266 267 intr = intr_disable(); 268 cpu = PCPU_GET(cpuid); 269 sysm = &sysmap_lmem[cpu]; 270 sysm->saved_intr = intr; 271 va1 = sysm->base; 272 va2 = sysm->base + PAGE_SIZE; 273 npte = TLBLO_PA_TO_PFN(phys1) | 274 PTE_D | PTE_V | PTE_G | PTE_W | PTE_C_CACHE; 275 pte = pmap_pte(kernel_pmap, va1); 276 *pte = npte; 277 npte = TLBLO_PA_TO_PFN(phys2) | 278 PTE_D | PTE_V | PTE_G | PTE_W | PTE_C_CACHE; 279 pte = pmap_pte(kernel_pmap, va2); 280 *pte = npte; 281 sysm->valid1 = 1; 282 sysm->valid2 = 1; 283 return (va1); 284} 285 286static __inline void 287pmap_lmem_unmap(void) 288{ 289 struct local_sysmaps *sysm; 290 pt_entry_t *pte; 291 int cpu; 292 293 cpu = PCPU_GET(cpuid); 294 sysm = &sysmap_lmem[cpu]; 295 pte = pmap_pte(kernel_pmap, sysm->base); 296 *pte = PTE_G; 297 tlb_invalidate_address(kernel_pmap, sysm->base); 298 sysm->valid1 = 0; 299 if (sysm->valid2) { 300 pte = pmap_pte(kernel_pmap, sysm->base + PAGE_SIZE); 301 *pte = PTE_G; 302 tlb_invalidate_address(kernel_pmap, sysm->base + PAGE_SIZE); 303 sysm->valid2 = 0; 304 } 305 intr_restore(sysm->saved_intr); 306} 307#else /* __mips_n64 */ 308 309static __inline void 310pmap_alloc_lmem_map(void) 311{ 312} 313 314static __inline vm_offset_t 315pmap_lmem_map1(vm_paddr_t phys) 316{ 317 318 return (0); 319} 320 321static __inline vm_offset_t 322pmap_lmem_map2(vm_paddr_t phys1, vm_paddr_t phys2) 323{ 324 325 return (0); 326} 327 328static __inline vm_offset_t 329pmap_lmem_unmap(void) 330{ 331 332 return (0); 333} 334#endif /* !__mips_n64 */ 335 336/* 337 * Page table entry lookup routines. 338 */ 339static __inline pd_entry_t * 340pmap_segmap(pmap_t pmap, vm_offset_t va) 341{ 342 343 return (&pmap->pm_segtab[pmap_seg_index(va)]); 344} 345 346#ifdef __mips_n64 347static __inline pd_entry_t * 348pmap_pdpe_to_pde(pd_entry_t *pdpe, vm_offset_t va) 349{ 350 pd_entry_t *pde; 351 352 pde = (pd_entry_t *)*pdpe; 353 return (&pde[pmap_pde_index(va)]); 354} 355 356static __inline pd_entry_t * 357pmap_pde(pmap_t pmap, vm_offset_t va) 358{ 359 pd_entry_t *pdpe; 360 361 pdpe = pmap_segmap(pmap, va); 362 if (*pdpe == NULL) 363 return (NULL); 364 365 return (pmap_pdpe_to_pde(pdpe, va)); 366} 367#else 368static __inline pd_entry_t * 369pmap_pdpe_to_pde(pd_entry_t *pdpe, vm_offset_t va) 370{ 371 372 return (pdpe); 373} 374 375static __inline 376pd_entry_t *pmap_pde(pmap_t pmap, vm_offset_t va) 377{ 378 379 return (pmap_segmap(pmap, va)); 380} 381#endif 382 383static __inline pt_entry_t * 384pmap_pde_to_pte(pd_entry_t *pde, vm_offset_t va) 385{ 386 pt_entry_t *pte; 387 388 pte = (pt_entry_t *)*pde; 389 return (&pte[pmap_pte_index(va)]); 390} 391 392pt_entry_t * 393pmap_pte(pmap_t pmap, vm_offset_t va) 394{ 395 pd_entry_t *pde; 396 397 pde = pmap_pde(pmap, va); 398 if (pde == NULL || *pde == NULL) 399 return (NULL); 400 401 return (pmap_pde_to_pte(pde, va)); 402} 403 404vm_offset_t 405pmap_steal_memory(vm_size_t size) 406{ 407 vm_paddr_t bank_size, pa; 408 vm_offset_t va; 409 410 size = round_page(size); 411 bank_size = phys_avail[1] - phys_avail[0]; 412 while (size > bank_size) { 413 int i; 414 415 for (i = 0; phys_avail[i + 2]; i += 2) { 416 phys_avail[i] = phys_avail[i + 2]; 417 phys_avail[i + 1] = phys_avail[i + 3]; 418 } 419 phys_avail[i] = 0; 420 phys_avail[i + 1] = 0; 421 if (!phys_avail[0]) 422 panic("pmap_steal_memory: out of memory"); 423 bank_size = phys_avail[1] - phys_avail[0]; 424 } 425 426 pa = phys_avail[0]; 427 phys_avail[0] += size; 428 if (MIPS_DIRECT_MAPPABLE(pa) == 0) 429 panic("Out of memory below 512Meg?"); 430 va = MIPS_PHYS_TO_DIRECT(pa); 431 bzero((caddr_t)va, size); 432 return (va); 433} 434 435/* 436 * Bootstrap the system enough to run with virtual memory. This 437 * assumes that the phys_avail array has been initialized. 438 */ 439static void 440pmap_create_kernel_pagetable(void) 441{ 442 int i, j; 443 vm_offset_t ptaddr; 444 pt_entry_t *pte; 445#ifdef __mips_n64 446 pd_entry_t *pde; 447 vm_offset_t pdaddr; 448 int npt, npde; 449#endif 450 451 /* 452 * Allocate segment table for the kernel 453 */ 454 kernel_segmap = (pd_entry_t *)pmap_steal_memory(PAGE_SIZE); 455 456 /* 457 * Allocate second level page tables for the kernel 458 */ 459#ifdef __mips_n64 460 npde = howmany(NKPT, NPDEPG); 461 pdaddr = pmap_steal_memory(PAGE_SIZE * npde); 462#endif 463 nkpt = NKPT; 464 ptaddr = pmap_steal_memory(PAGE_SIZE * nkpt); 465 466 /* 467 * The R[4-7]?00 stores only one copy of the Global bit in the 468 * translation lookaside buffer for each 2 page entry. Thus invalid 469 * entrys must have the Global bit set so when Entry LO and Entry HI 470 * G bits are anded together they will produce a global bit to store 471 * in the tlb. 472 */ 473 for (i = 0, pte = (pt_entry_t *)ptaddr; i < (nkpt * NPTEPG); i++, pte++) 474 *pte = PTE_G; 475 476#ifdef __mips_n64 477 for (i = 0, npt = nkpt; npt > 0; i++) { 478 kernel_segmap[i] = (pd_entry_t)(pdaddr + i * PAGE_SIZE); 479 pde = (pd_entry_t *)kernel_segmap[i]; 480 481 for (j = 0; j < NPDEPG && npt > 0; j++, npt--) 482 pde[j] = (pd_entry_t)(ptaddr + (i * NPDEPG + j) * PAGE_SIZE); 483 } 484#else 485 for (i = 0, j = pmap_seg_index(VM_MIN_KERNEL_ADDRESS); i < nkpt; i++, j++) 486 kernel_segmap[j] = (pd_entry_t)(ptaddr + (i * PAGE_SIZE)); 487#endif 488 489 PMAP_LOCK_INIT(kernel_pmap); 490 kernel_pmap->pm_segtab = kernel_segmap; 491 CPU_FILL(&kernel_pmap->pm_active); 492 TAILQ_INIT(&kernel_pmap->pm_pvchunk); 493 kernel_pmap->pm_asid[0].asid = PMAP_ASID_RESERVED; 494 kernel_pmap->pm_asid[0].gen = 0; 495 kernel_vm_end += nkpt * NPTEPG * PAGE_SIZE; 496} 497 498void 499pmap_bootstrap(void) 500{ 501 int i; 502 int need_local_mappings = 0; 503 504 /* Sort. */ 505again: 506 for (i = 0; phys_avail[i + 1] != 0; i += 2) { 507 /* 508 * Keep the memory aligned on page boundary. 509 */ 510 phys_avail[i] = round_page(phys_avail[i]); 511 phys_avail[i + 1] = trunc_page(phys_avail[i + 1]); 512 513 if (i < 2) 514 continue; 515 if (phys_avail[i - 2] > phys_avail[i]) { 516 vm_paddr_t ptemp[2]; 517 518 ptemp[0] = phys_avail[i + 0]; 519 ptemp[1] = phys_avail[i + 1]; 520 521 phys_avail[i + 0] = phys_avail[i - 2]; 522 phys_avail[i + 1] = phys_avail[i - 1]; 523 524 phys_avail[i - 2] = ptemp[0]; 525 phys_avail[i - 1] = ptemp[1]; 526 goto again; 527 } 528 } 529 530 /* 531 * In 32 bit, we may have memory which cannot be mapped directly. 532 * This memory will need temporary mapping before it can be 533 * accessed. 534 */ 535 if (!MIPS_DIRECT_MAPPABLE(phys_avail[i - 1] - 1)) 536 need_local_mappings = 1; 537 538 /* 539 * Copy the phys_avail[] array before we start stealing memory from it. 540 */ 541 for (i = 0; phys_avail[i + 1] != 0; i += 2) { 542 physmem_desc[i] = phys_avail[i]; 543 physmem_desc[i + 1] = phys_avail[i + 1]; 544 } 545 546 Maxmem = atop(phys_avail[i - 1]); 547 548 if (bootverbose) { 549 printf("Physical memory chunk(s):\n"); 550 for (i = 0; phys_avail[i + 1] != 0; i += 2) { 551 vm_paddr_t size; 552 553 size = phys_avail[i + 1] - phys_avail[i]; 554 printf("%#08jx - %#08jx, %ju bytes (%ju pages)\n", 555 (uintmax_t) phys_avail[i], 556 (uintmax_t) phys_avail[i + 1] - 1, 557 (uintmax_t) size, (uintmax_t) size / PAGE_SIZE); 558 } 559 printf("Maxmem is 0x%0jx\n", ptoa((uintmax_t)Maxmem)); 560 } 561 /* 562 * Steal the message buffer from the beginning of memory. 563 */ 564 msgbufp = (struct msgbuf *)pmap_steal_memory(msgbufsize); 565 msgbufinit(msgbufp, msgbufsize); 566 567 /* 568 * Steal thread0 kstack. 569 */ 570 kstack0 = pmap_steal_memory(KSTACK_PAGES << PAGE_SHIFT); 571 572 virtual_avail = VM_MIN_KERNEL_ADDRESS; 573 virtual_end = VM_MAX_KERNEL_ADDRESS; 574 575#ifdef SMP 576 /* 577 * Steal some virtual address space to map the pcpu area. 578 */ 579 virtual_avail = roundup2(virtual_avail, PAGE_SIZE * 2); 580 pcpup = (struct pcpu *)virtual_avail; 581 virtual_avail += PAGE_SIZE * 2; 582 583 /* 584 * Initialize the wired TLB entry mapping the pcpu region for 585 * the BSP at 'pcpup'. Up until this point we were operating 586 * with the 'pcpup' for the BSP pointing to a virtual address 587 * in KSEG0 so there was no need for a TLB mapping. 588 */ 589 mips_pcpu_tlb_init(PCPU_ADDR(0)); 590 591 if (bootverbose) 592 printf("pcpu is available at virtual address %p.\n", pcpup); 593#endif 594 595 if (need_local_mappings) 596 pmap_alloc_lmem_map(); 597 pmap_create_kernel_pagetable(); 598 pmap_max_asid = VMNUM_PIDS; 599 mips_wr_entryhi(0); 600 mips_wr_pagemask(0); 601 602 /* 603 * Initialize the global pv list lock. 604 */ 605 rw_init(&pvh_global_lock, "pmap pv global"); 606} 607 608/* 609 * Initialize a vm_page's machine-dependent fields. 610 */ 611void 612pmap_page_init(vm_page_t m) 613{ 614 615 TAILQ_INIT(&m->md.pv_list); 616 m->md.pv_flags = 0; 617} 618 619/* 620 * Initialize the pmap module. 621 * Called by vm_init, to initialize any structures that the pmap 622 * system needs to map virtual memory. 623 */ 624void 625pmap_init(void) 626{ 627} 628 629/*************************************************** 630 * Low level helper routines..... 631 ***************************************************/ 632 633#ifdef SMP 634static __inline void 635pmap_call_on_active_cpus(pmap_t pmap, void (*fn)(void *), void *arg) 636{ 637 int cpuid, cpu, self; 638 cpuset_t active_cpus; 639 640 sched_pin(); 641 if (is_kernel_pmap(pmap)) { 642 smp_rendezvous(NULL, fn, NULL, arg); 643 goto out; 644 } 645 /* Force ASID update on inactive CPUs */ 646 CPU_FOREACH(cpu) { 647 if (!CPU_ISSET(cpu, &pmap->pm_active)) 648 pmap->pm_asid[cpu].gen = 0; 649 } 650 cpuid = PCPU_GET(cpuid); 651 /* 652 * XXX: barrier/locking for active? 653 * 654 * Take a snapshot of active here, any further changes are ignored. 655 * tlb update/invalidate should be harmless on inactive CPUs 656 */ 657 active_cpus = pmap->pm_active; 658 self = CPU_ISSET(cpuid, &active_cpus); 659 CPU_CLR(cpuid, &active_cpus); 660 /* Optimize for the case where this cpu is the only active one */ 661 if (CPU_EMPTY(&active_cpus)) { 662 if (self) 663 fn(arg); 664 } else { 665 if (self) 666 CPU_SET(cpuid, &active_cpus); 667 smp_rendezvous_cpus(active_cpus, NULL, fn, NULL, arg); 668 } 669out: 670 sched_unpin(); 671} 672#else /* !SMP */ 673static __inline void 674pmap_call_on_active_cpus(pmap_t pmap, void (*fn)(void *), void *arg) 675{ 676 int cpuid; 677 678 if (is_kernel_pmap(pmap)) { 679 fn(arg); 680 return; 681 } 682 cpuid = PCPU_GET(cpuid); 683 if (!CPU_ISSET(cpuid, &pmap->pm_active)) 684 pmap->pm_asid[cpuid].gen = 0; 685 else 686 fn(arg); 687} 688#endif /* SMP */ 689 690static void 691pmap_invalidate_all(pmap_t pmap) 692{ 693 694 pmap_call_on_active_cpus(pmap, 695 (void (*)(void *))tlb_invalidate_all_user, pmap); 696} 697 698struct pmap_invalidate_page_arg { 699 pmap_t pmap; 700 vm_offset_t va; 701}; 702 703static void 704pmap_invalidate_page_action(void *arg) 705{ 706 struct pmap_invalidate_page_arg *p = arg; 707 708 tlb_invalidate_address(p->pmap, p->va); 709} 710 711static void 712pmap_invalidate_page(pmap_t pmap, vm_offset_t va) 713{ 714 struct pmap_invalidate_page_arg arg; 715 716 arg.pmap = pmap; 717 arg.va = va; 718 pmap_call_on_active_cpus(pmap, pmap_invalidate_page_action, &arg); 719} 720 721struct pmap_update_page_arg { 722 pmap_t pmap; 723 vm_offset_t va; 724 pt_entry_t pte; 725}; 726 727static void 728pmap_update_page_action(void *arg) 729{ 730 struct pmap_update_page_arg *p = arg; 731 732 tlb_update(p->pmap, p->va, p->pte); 733} 734 735static void 736pmap_update_page(pmap_t pmap, vm_offset_t va, pt_entry_t pte) 737{ 738 struct pmap_update_page_arg arg; 739 740 arg.pmap = pmap; 741 arg.va = va; 742 arg.pte = pte; 743 pmap_call_on_active_cpus(pmap, pmap_update_page_action, &arg); 744} 745 746/* 747 * Routine: pmap_extract 748 * Function: 749 * Extract the physical page address associated 750 * with the given map/virtual_address pair. 751 */ 752vm_paddr_t 753pmap_extract(pmap_t pmap, vm_offset_t va) 754{ 755 pt_entry_t *pte; 756 vm_offset_t retval = 0; 757 758 PMAP_LOCK(pmap); 759 pte = pmap_pte(pmap, va); 760 if (pte) { 761 retval = TLBLO_PTE_TO_PA(*pte) | (va & PAGE_MASK); 762 } 763 PMAP_UNLOCK(pmap); 764 return (retval); 765} 766 767/* 768 * Routine: pmap_extract_and_hold 769 * Function: 770 * Atomically extract and hold the physical page 771 * with the given pmap and virtual address pair 772 * if that mapping permits the given protection. 773 */ 774vm_page_t 775pmap_extract_and_hold(pmap_t pmap, vm_offset_t va, vm_prot_t prot) 776{ 777 pt_entry_t *ptep; 778 pt_entry_t pte; 779 vm_page_t m; 780 vm_paddr_t pa; 781 782 m = NULL; 783 pa = 0; 784 PMAP_LOCK(pmap); 785retry: 786 ptep = pmap_pte(pmap, va); 787 if ((ptep != NULL) && ((pte = *ptep) != 0) && 788 pte_test(&pte, PTE_V) && 789 (pte_test(&pte, PTE_D) || (prot & VM_PROT_WRITE) == 0)) { 790 if (vm_page_pa_tryrelock(pmap, TLBLO_PTE_TO_PA(pte), &pa)) 791 goto retry; 792 793 m = PHYS_TO_VM_PAGE(TLBLO_PTE_TO_PA(pte)); 794 vm_page_hold(m); 795 } 796 PA_UNLOCK_COND(pa); 797 PMAP_UNLOCK(pmap); 798 return (m); 799} 800 801/*************************************************** 802 * Low level mapping routines..... 803 ***************************************************/ 804 805/* 806 * add a wired page to the kva 807 */ 808void 809pmap_kenter_attr(vm_offset_t va, vm_paddr_t pa, int attr) 810{ 811 pt_entry_t *pte; 812 pt_entry_t opte, npte; 813 814#ifdef PMAP_DEBUG 815 printf("pmap_kenter: va: %p -> pa: %p\n", (void *)va, (void *)pa); 816#endif 817 npte = TLBLO_PA_TO_PFN(pa) | PTE_D | PTE_V | PTE_G | PTE_W | attr; 818 819 pte = pmap_pte(kernel_pmap, va); 820 opte = *pte; 821 *pte = npte; 822 if (pte_test(&opte, PTE_V) && opte != npte) 823 pmap_update_page(kernel_pmap, va, npte); 824} 825 826void 827pmap_kenter(vm_offset_t va, vm_paddr_t pa) 828{ 829 830 KASSERT(is_cacheable_mem(pa), 831 ("pmap_kenter: memory at 0x%lx is not cacheable", (u_long)pa)); 832 833 pmap_kenter_attr(va, pa, PTE_C_CACHE); 834} 835 836/* 837 * remove a page from the kernel pagetables 838 */ 839 /* PMAP_INLINE */ void 840pmap_kremove(vm_offset_t va) 841{ 842 pt_entry_t *pte; 843 844 /* 845 * Write back all caches from the page being destroyed 846 */ 847 mips_dcache_wbinv_range_index(va, PAGE_SIZE); 848 849 pte = pmap_pte(kernel_pmap, va); 850 *pte = PTE_G; 851 pmap_invalidate_page(kernel_pmap, va); 852} 853 854/* 855 * Used to map a range of physical addresses into kernel 856 * virtual address space. 857 * 858 * The value passed in '*virt' is a suggested virtual address for 859 * the mapping. Architectures which can support a direct-mapped 860 * physical to virtual region can return the appropriate address 861 * within that region, leaving '*virt' unchanged. Other 862 * architectures should map the pages starting at '*virt' and 863 * update '*virt' with the first usable address after the mapped 864 * region. 865 * 866 * Use XKPHYS for 64 bit, and KSEG0 where possible for 32 bit. 867 */ 868vm_offset_t 869pmap_map(vm_offset_t *virt, vm_paddr_t start, vm_paddr_t end, int prot) 870{ 871 vm_offset_t va, sva; 872 873 if (MIPS_DIRECT_MAPPABLE(end - 1)) 874 return (MIPS_PHYS_TO_DIRECT(start)); 875 876 va = sva = *virt; 877 while (start < end) { 878 pmap_kenter(va, start); 879 va += PAGE_SIZE; 880 start += PAGE_SIZE; 881 } 882 *virt = va; 883 return (sva); 884} 885 886/* 887 * Add a list of wired pages to the kva 888 * this routine is only used for temporary 889 * kernel mappings that do not need to have 890 * page modification or references recorded. 891 * Note that old mappings are simply written 892 * over. The page *must* be wired. 893 */ 894void 895pmap_qenter(vm_offset_t va, vm_page_t *m, int count) 896{ 897 int i; 898 vm_offset_t origva = va; 899 900 for (i = 0; i < count; i++) { 901 pmap_flush_pvcache(m[i]); 902 pmap_kenter(va, VM_PAGE_TO_PHYS(m[i])); 903 va += PAGE_SIZE; 904 } 905 906 mips_dcache_wbinv_range_index(origva, PAGE_SIZE*count); 907} 908 909/* 910 * this routine jerks page mappings from the 911 * kernel -- it is meant only for temporary mappings. 912 */ 913void 914pmap_qremove(vm_offset_t va, int count) 915{ 916 /* 917 * No need to wb/inv caches here, 918 * pmap_kremove will do it for us 919 */ 920 921 while (count-- > 0) { 922 pmap_kremove(va); 923 va += PAGE_SIZE; 924 } 925} 926 927/*************************************************** 928 * Page table page management routines..... 929 ***************************************************/ 930 931/* 932 * Decrements a page table page's wire count, which is used to record the 933 * number of valid page table entries within the page. If the wire count 934 * drops to zero, then the page table page is unmapped. Returns TRUE if the 935 * page table page was unmapped and FALSE otherwise. 936 */ 937static PMAP_INLINE boolean_t 938pmap_unwire_ptp(pmap_t pmap, vm_offset_t va, vm_page_t m) 939{ 940 941 --m->wire_count; 942 if (m->wire_count == 0) { 943 _pmap_unwire_ptp(pmap, va, m); 944 return (TRUE); 945 } else 946 return (FALSE); 947} 948 949static void 950_pmap_unwire_ptp(pmap_t pmap, vm_offset_t va, vm_page_t m) 951{ 952 pd_entry_t *pde; 953 954 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 955 /* 956 * unmap the page table page 957 */ 958#ifdef __mips_n64 959 if (m->pindex < NUPDE) 960 pde = pmap_pde(pmap, va); 961 else 962 pde = pmap_segmap(pmap, va); 963#else 964 pde = pmap_pde(pmap, va); 965#endif 966 *pde = 0; 967 pmap->pm_stats.resident_count--; 968 969#ifdef __mips_n64 970 if (m->pindex < NUPDE) { 971 pd_entry_t *pdp; 972 vm_page_t pdpg; 973 974 /* 975 * Recursively decrement next level pagetable refcount 976 */ 977 pdp = (pd_entry_t *)*pmap_segmap(pmap, va); 978 pdpg = PHYS_TO_VM_PAGE(MIPS_DIRECT_TO_PHYS(pdp)); 979 pmap_unwire_ptp(pmap, va, pdpg); 980 } 981#endif 982 983 /* 984 * If the page is finally unwired, simply free it. 985 */ 986 vm_page_free_zero(m); 987 atomic_subtract_int(&cnt.v_wire_count, 1); 988} 989 990/* 991 * After removing a page table entry, this routine is used to 992 * conditionally free the page, and manage the hold/wire counts. 993 */ 994static int 995pmap_unuse_pt(pmap_t pmap, vm_offset_t va, pd_entry_t pde) 996{ 997 vm_page_t mpte; 998 999 if (va >= VM_MAXUSER_ADDRESS) 1000 return (0); 1001 KASSERT(pde != 0, ("pmap_unuse_pt: pde != 0")); 1002 mpte = PHYS_TO_VM_PAGE(MIPS_DIRECT_TO_PHYS(pde)); 1003 return (pmap_unwire_ptp(pmap, va, mpte)); 1004} 1005 1006void 1007pmap_pinit0(pmap_t pmap) 1008{ 1009 int i; 1010 1011 PMAP_LOCK_INIT(pmap); 1012 pmap->pm_segtab = kernel_segmap; 1013 CPU_ZERO(&pmap->pm_active); 1014 for (i = 0; i < MAXCPU; i++) { 1015 pmap->pm_asid[i].asid = PMAP_ASID_RESERVED; 1016 pmap->pm_asid[i].gen = 0; 1017 } 1018 PCPU_SET(curpmap, pmap); 1019 TAILQ_INIT(&pmap->pm_pvchunk); 1020 bzero(&pmap->pm_stats, sizeof pmap->pm_stats); 1021} 1022 1023void 1024pmap_grow_direct_page_cache() 1025{ 1026 1027#ifdef __mips_n64 1028 vm_pageout_grow_cache(3, 0, MIPS_XKPHYS_LARGEST_PHYS); 1029#else 1030 vm_pageout_grow_cache(3, 0, MIPS_KSEG0_LARGEST_PHYS); 1031#endif 1032} 1033 1034vm_page_t 1035pmap_alloc_direct_page(unsigned int index, int req) 1036{ 1037 vm_page_t m; 1038 1039 m = vm_page_alloc_freelist(VM_FREELIST_DIRECT, req | VM_ALLOC_WIRED | 1040 VM_ALLOC_ZERO); 1041 if (m == NULL) 1042 return (NULL); 1043 1044 if ((m->flags & PG_ZERO) == 0) 1045 pmap_zero_page(m); 1046 1047 m->pindex = index; 1048 return (m); 1049} 1050 1051/* 1052 * Initialize a preallocated and zeroed pmap structure, 1053 * such as one in a vmspace structure. 1054 */ 1055int 1056pmap_pinit(pmap_t pmap) 1057{ 1058 vm_offset_t ptdva; 1059 vm_page_t ptdpg; 1060 int i; 1061 1062 PMAP_LOCK_INIT(pmap); 1063 1064 /* 1065 * allocate the page directory page 1066 */ 1067 while ((ptdpg = pmap_alloc_direct_page(NUSERPGTBLS, VM_ALLOC_NORMAL)) == NULL) 1068 pmap_grow_direct_page_cache(); 1069 1070 ptdva = MIPS_PHYS_TO_DIRECT(VM_PAGE_TO_PHYS(ptdpg)); 1071 pmap->pm_segtab = (pd_entry_t *)ptdva; 1072 CPU_ZERO(&pmap->pm_active); 1073 for (i = 0; i < MAXCPU; i++) { 1074 pmap->pm_asid[i].asid = PMAP_ASID_RESERVED; 1075 pmap->pm_asid[i].gen = 0; 1076 } 1077 TAILQ_INIT(&pmap->pm_pvchunk); 1078 bzero(&pmap->pm_stats, sizeof pmap->pm_stats); 1079 1080 return (1); 1081} 1082 1083/* 1084 * this routine is called if the page table page is not 1085 * mapped correctly. 1086 */ 1087static vm_page_t 1088_pmap_allocpte(pmap_t pmap, unsigned ptepindex, int flags) 1089{ 1090 vm_offset_t pageva; 1091 vm_page_t m; 1092 1093 KASSERT((flags & (M_NOWAIT | M_WAITOK)) == M_NOWAIT || 1094 (flags & (M_NOWAIT | M_WAITOK)) == M_WAITOK, 1095 ("_pmap_allocpte: flags is neither M_NOWAIT nor M_WAITOK")); 1096 1097 /* 1098 * Find or fabricate a new pagetable page 1099 */ 1100 if ((m = pmap_alloc_direct_page(ptepindex, VM_ALLOC_NORMAL)) == NULL) { 1101 if (flags & M_WAITOK) { 1102 PMAP_UNLOCK(pmap); 1103 rw_wunlock(&pvh_global_lock); 1104 pmap_grow_direct_page_cache(); 1105 rw_wlock(&pvh_global_lock); 1106 PMAP_LOCK(pmap); 1107 } 1108 1109 /* 1110 * Indicate the need to retry. While waiting, the page 1111 * table page may have been allocated. 1112 */ 1113 return (NULL); 1114 } 1115 1116 /* 1117 * Map the pagetable page into the process address space, if it 1118 * isn't already there. 1119 */ 1120 pageva = MIPS_PHYS_TO_DIRECT(VM_PAGE_TO_PHYS(m)); 1121 1122#ifdef __mips_n64 1123 if (ptepindex >= NUPDE) { 1124 pmap->pm_segtab[ptepindex - NUPDE] = (pd_entry_t)pageva; 1125 } else { 1126 pd_entry_t *pdep, *pde; 1127 int segindex = ptepindex >> (SEGSHIFT - PDRSHIFT); 1128 int pdeindex = ptepindex & (NPDEPG - 1); 1129 vm_page_t pg; 1130 1131 pdep = &pmap->pm_segtab[segindex]; 1132 if (*pdep == NULL) { 1133 /* recurse for allocating page dir */ 1134 if (_pmap_allocpte(pmap, NUPDE + segindex, 1135 flags) == NULL) { 1136 /* alloc failed, release current */ 1137 --m->wire_count; 1138 atomic_subtract_int(&cnt.v_wire_count, 1); 1139 vm_page_free_zero(m); 1140 return (NULL); 1141 } 1142 } else { 1143 pg = PHYS_TO_VM_PAGE(MIPS_DIRECT_TO_PHYS(*pdep)); 1144 pg->wire_count++; 1145 } 1146 /* Next level entry */ 1147 pde = (pd_entry_t *)*pdep; 1148 pde[pdeindex] = (pd_entry_t)pageva; 1149 } 1150#else 1151 pmap->pm_segtab[ptepindex] = (pd_entry_t)pageva; 1152#endif 1153 pmap->pm_stats.resident_count++; 1154 return (m); 1155} 1156 1157static vm_page_t 1158pmap_allocpte(pmap_t pmap, vm_offset_t va, int flags) 1159{ 1160 unsigned ptepindex; 1161 pd_entry_t *pde; 1162 vm_page_t m; 1163 1164 KASSERT((flags & (M_NOWAIT | M_WAITOK)) == M_NOWAIT || 1165 (flags & (M_NOWAIT | M_WAITOK)) == M_WAITOK, 1166 ("pmap_allocpte: flags is neither M_NOWAIT nor M_WAITOK")); 1167 1168 /* 1169 * Calculate pagetable page index 1170 */ 1171 ptepindex = pmap_pde_pindex(va); 1172retry: 1173 /* 1174 * Get the page directory entry 1175 */ 1176 pde = pmap_pde(pmap, va); 1177 1178 /* 1179 * If the page table page is mapped, we just increment the hold 1180 * count, and activate it. 1181 */ 1182 if (pde != NULL && *pde != NULL) { 1183 m = PHYS_TO_VM_PAGE(MIPS_DIRECT_TO_PHYS(*pde)); 1184 m->wire_count++; 1185 } else { 1186 /* 1187 * Here if the pte page isn't mapped, or if it has been 1188 * deallocated. 1189 */ 1190 m = _pmap_allocpte(pmap, ptepindex, flags); 1191 if (m == NULL && (flags & M_WAITOK)) 1192 goto retry; 1193 } 1194 return (m); 1195} 1196 1197 1198/*************************************************** 1199 * Pmap allocation/deallocation routines. 1200 ***************************************************/ 1201 1202/* 1203 * Release any resources held by the given physical map. 1204 * Called when a pmap initialized by pmap_pinit is being released. 1205 * Should only be called if the map contains no valid mappings. 1206 */ 1207void 1208pmap_release(pmap_t pmap) 1209{ 1210 vm_offset_t ptdva; 1211 vm_page_t ptdpg; 1212 1213 KASSERT(pmap->pm_stats.resident_count == 0, 1214 ("pmap_release: pmap resident count %ld != 0", 1215 pmap->pm_stats.resident_count)); 1216 1217 ptdva = (vm_offset_t)pmap->pm_segtab; 1218 ptdpg = PHYS_TO_VM_PAGE(MIPS_DIRECT_TO_PHYS(ptdva)); 1219 1220 ptdpg->wire_count--; 1221 atomic_subtract_int(&cnt.v_wire_count, 1); 1222 vm_page_free_zero(ptdpg); 1223 PMAP_LOCK_DESTROY(pmap); 1224} 1225 1226/* 1227 * grow the number of kernel page table entries, if needed 1228 */ 1229void 1230pmap_growkernel(vm_offset_t addr) 1231{ 1232 vm_page_t nkpg; 1233 pd_entry_t *pde, *pdpe; 1234 pt_entry_t *pte; 1235 int i; 1236 1237 mtx_assert(&kernel_map->system_mtx, MA_OWNED); 1238 addr = roundup2(addr, NBSEG); 1239 if (addr - 1 >= kernel_map->max_offset) 1240 addr = kernel_map->max_offset; 1241 while (kernel_vm_end < addr) { 1242 pdpe = pmap_segmap(kernel_pmap, kernel_vm_end); 1243#ifdef __mips_n64 1244 if (*pdpe == 0) { 1245 /* new intermediate page table entry */ 1246 nkpg = pmap_alloc_direct_page(nkpt, VM_ALLOC_INTERRUPT); 1247 if (nkpg == NULL) 1248 panic("pmap_growkernel: no memory to grow kernel"); 1249 *pdpe = (pd_entry_t)MIPS_PHYS_TO_DIRECT(VM_PAGE_TO_PHYS(nkpg)); 1250 continue; /* try again */ 1251 } 1252#endif 1253 pde = pmap_pdpe_to_pde(pdpe, kernel_vm_end); 1254 if (*pde != 0) { 1255 kernel_vm_end = (kernel_vm_end + NBPDR) & ~PDRMASK; 1256 if (kernel_vm_end - 1 >= kernel_map->max_offset) { 1257 kernel_vm_end = kernel_map->max_offset; 1258 break; 1259 } 1260 continue; 1261 } 1262 1263 /* 1264 * This index is bogus, but out of the way 1265 */ 1266 nkpg = pmap_alloc_direct_page(nkpt, VM_ALLOC_INTERRUPT); 1267 if (!nkpg) 1268 panic("pmap_growkernel: no memory to grow kernel"); 1269 nkpt++; 1270 *pde = (pd_entry_t)MIPS_PHYS_TO_DIRECT(VM_PAGE_TO_PHYS(nkpg)); 1271 1272 /* 1273 * The R[4-7]?00 stores only one copy of the Global bit in 1274 * the translation lookaside buffer for each 2 page entry. 1275 * Thus invalid entrys must have the Global bit set so when 1276 * Entry LO and Entry HI G bits are anded together they will 1277 * produce a global bit to store in the tlb. 1278 */ 1279 pte = (pt_entry_t *)*pde; 1280 for (i = 0; i < NPTEPG; i++) 1281 pte[i] = PTE_G; 1282 1283 kernel_vm_end = (kernel_vm_end + NBPDR) & ~PDRMASK; 1284 if (kernel_vm_end - 1 >= kernel_map->max_offset) { 1285 kernel_vm_end = kernel_map->max_offset; 1286 break; 1287 } 1288 } 1289} 1290 1291/*************************************************** 1292 * page management routines. 1293 ***************************************************/ 1294 1295CTASSERT(sizeof(struct pv_chunk) == PAGE_SIZE); 1296#ifdef __mips_n64 1297CTASSERT(_NPCM == 3); 1298CTASSERT(_NPCPV == 168); 1299#else 1300CTASSERT(_NPCM == 11); 1301CTASSERT(_NPCPV == 336); 1302#endif 1303 1304static __inline struct pv_chunk * 1305pv_to_chunk(pv_entry_t pv) 1306{ 1307 1308 return ((struct pv_chunk *)((uintptr_t)pv & ~(uintptr_t)PAGE_MASK)); 1309} 1310 1311#define PV_PMAP(pv) (pv_to_chunk(pv)->pc_pmap) 1312 1313#ifdef __mips_n64 1314#define PC_FREE0_1 0xfffffffffffffffful 1315#define PC_FREE2 0x000000fffffffffful 1316#else 1317#define PC_FREE0_9 0xfffffffful /* Free values for index 0 through 9 */ 1318#define PC_FREE10 0x0000fffful /* Free values for index 10 */ 1319#endif 1320 1321static const u_long pc_freemask[_NPCM] = { 1322#ifdef __mips_n64 1323 PC_FREE0_1, PC_FREE0_1, PC_FREE2 1324#else 1325 PC_FREE0_9, PC_FREE0_9, PC_FREE0_9, 1326 PC_FREE0_9, PC_FREE0_9, PC_FREE0_9, 1327 PC_FREE0_9, PC_FREE0_9, PC_FREE0_9, 1328 PC_FREE0_9, PC_FREE10 1329#endif 1330}; 1331 1332static SYSCTL_NODE(_vm, OID_AUTO, pmap, CTLFLAG_RD, 0, "VM/pmap parameters"); 1333 1334SYSCTL_INT(_vm_pmap, OID_AUTO, pv_entry_count, CTLFLAG_RD, &pv_entry_count, 0, 1335 "Current number of pv entries"); 1336 1337#ifdef PV_STATS 1338static int pc_chunk_count, pc_chunk_allocs, pc_chunk_frees, pc_chunk_tryfail; 1339 1340SYSCTL_INT(_vm_pmap, OID_AUTO, pc_chunk_count, CTLFLAG_RD, &pc_chunk_count, 0, 1341 "Current number of pv entry chunks"); 1342SYSCTL_INT(_vm_pmap, OID_AUTO, pc_chunk_allocs, CTLFLAG_RD, &pc_chunk_allocs, 0, 1343 "Current number of pv entry chunks allocated"); 1344SYSCTL_INT(_vm_pmap, OID_AUTO, pc_chunk_frees, CTLFLAG_RD, &pc_chunk_frees, 0, 1345 "Current number of pv entry chunks frees"); 1346SYSCTL_INT(_vm_pmap, OID_AUTO, pc_chunk_tryfail, CTLFLAG_RD, &pc_chunk_tryfail, 0, 1347 "Number of times tried to get a chunk page but failed."); 1348 1349static long pv_entry_frees, pv_entry_allocs; 1350static int pv_entry_spare; 1351 1352SYSCTL_LONG(_vm_pmap, OID_AUTO, pv_entry_frees, CTLFLAG_RD, &pv_entry_frees, 0, 1353 "Current number of pv entry frees"); 1354SYSCTL_LONG(_vm_pmap, OID_AUTO, pv_entry_allocs, CTLFLAG_RD, &pv_entry_allocs, 0, 1355 "Current number of pv entry allocs"); 1356SYSCTL_INT(_vm_pmap, OID_AUTO, pv_entry_spare, CTLFLAG_RD, &pv_entry_spare, 0, 1357 "Current number of spare pv entries"); 1358#endif 1359 1360/* 1361 * We are in a serious low memory condition. Resort to 1362 * drastic measures to free some pages so we can allocate 1363 * another pv entry chunk. 1364 */ 1365static vm_page_t 1366pmap_pv_reclaim(pmap_t locked_pmap) 1367{ 1368 struct pch newtail; 1369 struct pv_chunk *pc; 1370 pd_entry_t *pde; 1371 pmap_t pmap; 1372 pt_entry_t *pte, oldpte; 1373 pv_entry_t pv; 1374 vm_offset_t va; 1375 vm_page_t m, m_pc; 1376 u_long inuse; 1377 int bit, field, freed, idx; 1378 1379 PMAP_LOCK_ASSERT(locked_pmap, MA_OWNED); 1380 pmap = NULL; 1381 m_pc = NULL; 1382 TAILQ_INIT(&newtail); 1383 while ((pc = TAILQ_FIRST(&pv_chunks)) != NULL) { 1384 TAILQ_REMOVE(&pv_chunks, pc, pc_lru); 1385 if (pmap != pc->pc_pmap) { 1386 if (pmap != NULL) { 1387 pmap_invalidate_all(pmap); 1388 if (pmap != locked_pmap) 1389 PMAP_UNLOCK(pmap); 1390 } 1391 pmap = pc->pc_pmap; 1392 /* Avoid deadlock and lock recursion. */ 1393 if (pmap > locked_pmap) 1394 PMAP_LOCK(pmap); 1395 else if (pmap != locked_pmap && !PMAP_TRYLOCK(pmap)) { 1396 pmap = NULL; 1397 TAILQ_INSERT_TAIL(&newtail, pc, pc_lru); 1398 continue; 1399 } 1400 } 1401 1402 /* 1403 * Destroy every non-wired, 4 KB page mapping in the chunk. 1404 */ 1405 freed = 0; 1406 for (field = 0; field < _NPCM; field++) { 1407 for (inuse = ~pc->pc_map[field] & pc_freemask[field]; 1408 inuse != 0; inuse &= ~(1UL << bit)) { 1409 bit = ffsl(inuse) - 1; 1410 idx = field * sizeof(inuse) * NBBY + bit; 1411 pv = &pc->pc_pventry[idx]; 1412 va = pv->pv_va; 1413 pde = pmap_pde(pmap, va); 1414 KASSERT(pde != NULL && *pde != 0, 1415 ("pmap_pv_reclaim: pde")); 1416 pte = pmap_pde_to_pte(pde, va); 1417 oldpte = *pte; 1418 KASSERT(!pte_test(&oldpte, PTE_W), 1419 ("wired pte for unwired page")); 1420 if (is_kernel_pmap(pmap)) 1421 *pte = PTE_G; 1422 else 1423 *pte = 0; 1424 pmap_invalidate_page(pmap, va); 1425 m = PHYS_TO_VM_PAGE(TLBLO_PTE_TO_PA(oldpte)); 1426 if (pte_test(&oldpte, PTE_D)) 1427 vm_page_dirty(m); 1428 if (m->md.pv_flags & PV_TABLE_REF) 1429 vm_page_aflag_set(m, PGA_REFERENCED); 1430 m->md.pv_flags &= ~PV_TABLE_REF; 1431 TAILQ_REMOVE(&m->md.pv_list, pv, pv_list); 1432 if (TAILQ_EMPTY(&m->md.pv_list)) 1433 vm_page_aflag_clear(m, PGA_WRITEABLE); 1434 pc->pc_map[field] |= 1UL << bit; 1435 pmap_unuse_pt(pmap, va, *pde); 1436 freed++; 1437 } 1438 } 1439 if (freed == 0) { 1440 TAILQ_INSERT_TAIL(&newtail, pc, pc_lru); 1441 continue; 1442 } 1443 /* Every freed mapping is for a 4 KB page. */ 1444 pmap->pm_stats.resident_count -= freed; 1445 PV_STAT(pv_entry_frees += freed); 1446 PV_STAT(pv_entry_spare += freed); 1447 pv_entry_count -= freed; 1448 TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list); 1449 for (field = 0; field < _NPCM; field++) 1450 if (pc->pc_map[field] != pc_freemask[field]) { 1451 TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc, 1452 pc_list); 1453 TAILQ_INSERT_TAIL(&newtail, pc, pc_lru); 1454 1455 /* 1456 * One freed pv entry in locked_pmap is 1457 * sufficient. 1458 */ 1459 if (pmap == locked_pmap) 1460 goto out; 1461 break; 1462 } 1463 if (field == _NPCM) { 1464 PV_STAT(pv_entry_spare -= _NPCPV); 1465 PV_STAT(pc_chunk_count--); 1466 PV_STAT(pc_chunk_frees++); 1467 /* Entire chunk is free; return it. */ 1468 m_pc = PHYS_TO_VM_PAGE(MIPS_DIRECT_TO_PHYS( 1469 (vm_offset_t)pc)); 1470 break; 1471 } 1472 } 1473out: 1474 TAILQ_CONCAT(&pv_chunks, &newtail, pc_lru); 1475 if (pmap != NULL) { 1476 pmap_invalidate_all(pmap); 1477 if (pmap != locked_pmap) 1478 PMAP_UNLOCK(pmap); 1479 } 1480 return (m_pc); 1481} 1482 1483/* 1484 * free the pv_entry back to the free list 1485 */ 1486static void 1487free_pv_entry(pmap_t pmap, pv_entry_t pv) 1488{ 1489 struct pv_chunk *pc; 1490 int bit, field, idx; 1491 1492 rw_assert(&pvh_global_lock, RA_WLOCKED); 1493 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 1494 PV_STAT(pv_entry_frees++); 1495 PV_STAT(pv_entry_spare++); 1496 pv_entry_count--; 1497 pc = pv_to_chunk(pv); 1498 idx = pv - &pc->pc_pventry[0]; 1499 field = idx / (sizeof(u_long) * NBBY); 1500 bit = idx % (sizeof(u_long) * NBBY); 1501 pc->pc_map[field] |= 1ul << bit; 1502 for (idx = 0; idx < _NPCM; idx++) 1503 if (pc->pc_map[idx] != pc_freemask[idx]) { 1504 /* 1505 * 98% of the time, pc is already at the head of the 1506 * list. If it isn't already, move it to the head. 1507 */ 1508 if (__predict_false(TAILQ_FIRST(&pmap->pm_pvchunk) != 1509 pc)) { 1510 TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list); 1511 TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc, 1512 pc_list); 1513 } 1514 return; 1515 } 1516 TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list); 1517 free_pv_chunk(pc); 1518} 1519 1520static void 1521free_pv_chunk(struct pv_chunk *pc) 1522{ 1523 vm_page_t m; 1524 1525 TAILQ_REMOVE(&pv_chunks, pc, pc_lru); 1526 PV_STAT(pv_entry_spare -= _NPCPV); 1527 PV_STAT(pc_chunk_count--); 1528 PV_STAT(pc_chunk_frees++); 1529 /* entire chunk is free, return it */ 1530 m = PHYS_TO_VM_PAGE(MIPS_DIRECT_TO_PHYS((vm_offset_t)pc)); 1531 vm_page_unwire(m, 0); 1532 vm_page_free(m); 1533} 1534 1535/* 1536 * get a new pv_entry, allocating a block from the system 1537 * when needed. 1538 */ 1539static pv_entry_t 1540get_pv_entry(pmap_t pmap, boolean_t try) 1541{ 1542 struct pv_chunk *pc; 1543 pv_entry_t pv; 1544 vm_page_t m; 1545 int bit, field, idx; 1546 1547 rw_assert(&pvh_global_lock, RA_WLOCKED); 1548 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 1549 PV_STAT(pv_entry_allocs++); 1550 pv_entry_count++; 1551retry: 1552 pc = TAILQ_FIRST(&pmap->pm_pvchunk); 1553 if (pc != NULL) { 1554 for (field = 0; field < _NPCM; field++) { 1555 if (pc->pc_map[field]) { 1556 bit = ffsl(pc->pc_map[field]) - 1; 1557 break; 1558 } 1559 } 1560 if (field < _NPCM) { 1561 idx = field * sizeof(pc->pc_map[field]) * NBBY + bit; 1562 pv = &pc->pc_pventry[idx]; 1563 pc->pc_map[field] &= ~(1ul << bit); 1564 /* If this was the last item, move it to tail */ 1565 for (field = 0; field < _NPCM; field++) 1566 if (pc->pc_map[field] != 0) { 1567 PV_STAT(pv_entry_spare--); 1568 return (pv); /* not full, return */ 1569 } 1570 TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list); 1571 TAILQ_INSERT_TAIL(&pmap->pm_pvchunk, pc, pc_list); 1572 PV_STAT(pv_entry_spare--); 1573 return (pv); 1574 } 1575 } 1576 /* No free items, allocate another chunk */ 1577 m = vm_page_alloc_freelist(VM_FREELIST_DIRECT, VM_ALLOC_NORMAL | 1578 VM_ALLOC_WIRED); 1579 if (m == NULL) { 1580 if (try) { 1581 pv_entry_count--; 1582 PV_STAT(pc_chunk_tryfail++); 1583 return (NULL); 1584 } 1585 m = pmap_pv_reclaim(pmap); 1586 if (m == NULL) 1587 goto retry; 1588 } 1589 PV_STAT(pc_chunk_count++); 1590 PV_STAT(pc_chunk_allocs++); 1591 pc = (struct pv_chunk *)MIPS_PHYS_TO_DIRECT(VM_PAGE_TO_PHYS(m)); 1592 pc->pc_pmap = pmap; 1593 pc->pc_map[0] = pc_freemask[0] & ~1ul; /* preallocated bit 0 */ 1594 for (field = 1; field < _NPCM; field++) 1595 pc->pc_map[field] = pc_freemask[field]; 1596 TAILQ_INSERT_TAIL(&pv_chunks, pc, pc_lru); 1597 pv = &pc->pc_pventry[0]; 1598 TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc, pc_list); 1599 PV_STAT(pv_entry_spare += _NPCPV - 1); 1600 return (pv); 1601} 1602 1603/* 1604 * If it is the first entry on the list, it is actually 1605 * in the header and we must copy the following entry up 1606 * to the header. Otherwise we must search the list for 1607 * the entry. In either case we free the now unused entry. 1608 */ 1609 1610static pv_entry_t 1611pmap_pvh_remove(struct md_page *pvh, pmap_t pmap, vm_offset_t va) 1612{ 1613 pv_entry_t pv; 1614 1615 rw_assert(&pvh_global_lock, RA_WLOCKED); 1616 TAILQ_FOREACH(pv, &pvh->pv_list, pv_list) { 1617 if (pmap == PV_PMAP(pv) && va == pv->pv_va) { 1618 TAILQ_REMOVE(&pvh->pv_list, pv, pv_list); 1619 break; 1620 } 1621 } 1622 return (pv); 1623} 1624 1625static void 1626pmap_pvh_free(struct md_page *pvh, pmap_t pmap, vm_offset_t va) 1627{ 1628 pv_entry_t pv; 1629 1630 pv = pmap_pvh_remove(pvh, pmap, va); 1631 KASSERT(pv != NULL, ("pmap_pvh_free: pv not found, pa %lx va %lx", 1632 (u_long)VM_PAGE_TO_PHYS(member2struct(vm_page, md, pvh)), 1633 (u_long)va)); 1634 free_pv_entry(pmap, pv); 1635} 1636 1637static void 1638pmap_remove_entry(pmap_t pmap, vm_page_t m, vm_offset_t va) 1639{ 1640 1641 rw_assert(&pvh_global_lock, RA_WLOCKED); 1642 pmap_pvh_free(&m->md, pmap, va); 1643 if (TAILQ_EMPTY(&m->md.pv_list)) 1644 vm_page_aflag_clear(m, PGA_WRITEABLE); 1645} 1646 1647/* 1648 * Conditionally create a pv entry. 1649 */ 1650static boolean_t 1651pmap_try_insert_pv_entry(pmap_t pmap, vm_page_t mpte, vm_offset_t va, 1652 vm_page_t m) 1653{ 1654 pv_entry_t pv; 1655 1656 rw_assert(&pvh_global_lock, RA_WLOCKED); 1657 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 1658 if ((pv = get_pv_entry(pmap, TRUE)) != NULL) { 1659 pv->pv_va = va; 1660 TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_list); 1661 return (TRUE); 1662 } else 1663 return (FALSE); 1664} 1665 1666/* 1667 * pmap_remove_pte: do the things to unmap a page in a process 1668 */ 1669static int 1670pmap_remove_pte(struct pmap *pmap, pt_entry_t *ptq, vm_offset_t va, 1671 pd_entry_t pde) 1672{ 1673 pt_entry_t oldpte; 1674 vm_page_t m; 1675 vm_paddr_t pa; 1676 1677 rw_assert(&pvh_global_lock, RA_WLOCKED); 1678 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 1679 1680 /* 1681 * Write back all cache lines from the page being unmapped. 1682 */ 1683 mips_dcache_wbinv_range_index(va, PAGE_SIZE); 1684 1685 oldpte = *ptq; 1686 if (is_kernel_pmap(pmap)) 1687 *ptq = PTE_G; 1688 else 1689 *ptq = 0; 1690 1691 if (pte_test(&oldpte, PTE_W)) 1692 pmap->pm_stats.wired_count -= 1; 1693 1694 pmap->pm_stats.resident_count -= 1; 1695 1696 if (pte_test(&oldpte, PTE_MANAGED)) { 1697 pa = TLBLO_PTE_TO_PA(oldpte); 1698 m = PHYS_TO_VM_PAGE(pa); 1699 if (pte_test(&oldpte, PTE_D)) { 1700 KASSERT(!pte_test(&oldpte, PTE_RO), 1701 ("%s: modified page not writable: va: %p, pte: %#jx", 1702 __func__, (void *)va, (uintmax_t)oldpte)); 1703 vm_page_dirty(m); 1704 } 1705 if (m->md.pv_flags & PV_TABLE_REF) 1706 vm_page_aflag_set(m, PGA_REFERENCED); 1707 m->md.pv_flags &= ~PV_TABLE_REF; 1708 1709 pmap_remove_entry(pmap, m, va); 1710 } 1711 return (pmap_unuse_pt(pmap, va, pde)); 1712} 1713 1714/* 1715 * Remove a single page from a process address space 1716 */ 1717static void 1718pmap_remove_page(struct pmap *pmap, vm_offset_t va) 1719{ 1720 pd_entry_t *pde; 1721 pt_entry_t *ptq; 1722 1723 rw_assert(&pvh_global_lock, RA_WLOCKED); 1724 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 1725 pde = pmap_pde(pmap, va); 1726 if (pde == NULL || *pde == 0) 1727 return; 1728 ptq = pmap_pde_to_pte(pde, va); 1729 1730 /* 1731 * If there is no pte for this address, just skip it! 1732 */ 1733 if (!pte_test(ptq, PTE_V)) 1734 return; 1735 1736 (void)pmap_remove_pte(pmap, ptq, va, *pde); 1737 pmap_invalidate_page(pmap, va); 1738} 1739 1740/* 1741 * Remove the given range of addresses from the specified map. 1742 * 1743 * It is assumed that the start and end are properly 1744 * rounded to the page size. 1745 */ 1746void 1747pmap_remove(struct pmap *pmap, vm_offset_t sva, vm_offset_t eva) 1748{ 1749 vm_offset_t va_next; 1750 pd_entry_t *pde, *pdpe; 1751 pt_entry_t *pte; 1752 1753 if (pmap->pm_stats.resident_count == 0) 1754 return; 1755 1756 rw_wlock(&pvh_global_lock); 1757 PMAP_LOCK(pmap); 1758 1759 /* 1760 * special handling of removing one page. a very common operation 1761 * and easy to short circuit some code. 1762 */ 1763 if ((sva + PAGE_SIZE) == eva) { 1764 pmap_remove_page(pmap, sva); 1765 goto out; 1766 } 1767 for (; sva < eva; sva = va_next) { 1768 pdpe = pmap_segmap(pmap, sva); 1769#ifdef __mips_n64 1770 if (*pdpe == 0) { 1771 va_next = (sva + NBSEG) & ~SEGMASK; 1772 if (va_next < sva) 1773 va_next = eva; 1774 continue; 1775 } 1776#endif 1777 va_next = (sva + NBPDR) & ~PDRMASK; 1778 if (va_next < sva) 1779 va_next = eva; 1780 1781 pde = pmap_pdpe_to_pde(pdpe, sva); 1782 if (*pde == 0) 1783 continue; 1784 if (va_next > eva) 1785 va_next = eva; 1786 for (pte = pmap_pde_to_pte(pde, sva); sva != va_next; pte++, 1787 sva += PAGE_SIZE) { 1788 if (!pte_test(pte, PTE_V)) 1789 continue; 1790 pmap_remove_pte(pmap, pte, sva, *pde); 1791 pmap_invalidate_page(pmap, sva); 1792 } 1793 } 1794out: 1795 rw_wunlock(&pvh_global_lock); 1796 PMAP_UNLOCK(pmap); 1797} 1798 1799/* 1800 * Routine: pmap_remove_all 1801 * Function: 1802 * Removes this physical page from 1803 * all physical maps in which it resides. 1804 * Reflects back modify bits to the pager. 1805 * 1806 * Notes: 1807 * Original versions of this routine were very 1808 * inefficient because they iteratively called 1809 * pmap_remove (slow...) 1810 */ 1811 1812void 1813pmap_remove_all(vm_page_t m) 1814{ 1815 pv_entry_t pv; 1816 pmap_t pmap; 1817 pd_entry_t *pde; 1818 pt_entry_t *pte, tpte; 1819 1820 KASSERT((m->oflags & VPO_UNMANAGED) == 0, 1821 ("pmap_remove_all: page %p is not managed", m)); 1822 rw_wlock(&pvh_global_lock); 1823 1824 if (m->md.pv_flags & PV_TABLE_REF) 1825 vm_page_aflag_set(m, PGA_REFERENCED); 1826 1827 while ((pv = TAILQ_FIRST(&m->md.pv_list)) != NULL) { 1828 pmap = PV_PMAP(pv); 1829 PMAP_LOCK(pmap); 1830 1831 /* 1832 * If it's last mapping writeback all caches from 1833 * the page being destroyed 1834 */ 1835 if (TAILQ_NEXT(pv, pv_list) == NULL) 1836 mips_dcache_wbinv_range_index(pv->pv_va, PAGE_SIZE); 1837 1838 pmap->pm_stats.resident_count--; 1839 1840 pde = pmap_pde(pmap, pv->pv_va); 1841 KASSERT(pde != NULL && *pde != 0, ("pmap_remove_all: pde")); 1842 pte = pmap_pde_to_pte(pde, pv->pv_va); 1843 1844 tpte = *pte; 1845 if (is_kernel_pmap(pmap)) 1846 *pte = PTE_G; 1847 else 1848 *pte = 0; 1849 1850 if (pte_test(&tpte, PTE_W)) 1851 pmap->pm_stats.wired_count--; 1852 1853 /* 1854 * Update the vm_page_t clean and reference bits. 1855 */ 1856 if (pte_test(&tpte, PTE_D)) { 1857 KASSERT(!pte_test(&tpte, PTE_RO), 1858 ("%s: modified page not writable: va: %p, pte: %#jx", 1859 __func__, (void *)pv->pv_va, (uintmax_t)tpte)); 1860 vm_page_dirty(m); 1861 } 1862 pmap_invalidate_page(pmap, pv->pv_va); 1863 1864 TAILQ_REMOVE(&m->md.pv_list, pv, pv_list); 1865 pmap_unuse_pt(pmap, pv->pv_va, *pde); 1866 free_pv_entry(pmap, pv); 1867 PMAP_UNLOCK(pmap); 1868 } 1869 1870 vm_page_aflag_clear(m, PGA_WRITEABLE); 1871 m->md.pv_flags &= ~PV_TABLE_REF; 1872 rw_wunlock(&pvh_global_lock); 1873} 1874 1875/* 1876 * Set the physical protection on the 1877 * specified range of this map as requested. 1878 */ 1879void 1880pmap_protect(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, vm_prot_t prot) 1881{ 1882 pt_entry_t *pte; 1883 pd_entry_t *pde, *pdpe; 1884 vm_offset_t va_next; 1885 1886 if ((prot & VM_PROT_READ) == VM_PROT_NONE) { 1887 pmap_remove(pmap, sva, eva); 1888 return; 1889 } 1890 if (prot & VM_PROT_WRITE) 1891 return; 1892 1893 rw_wlock(&pvh_global_lock); 1894 PMAP_LOCK(pmap); 1895 for (; sva < eva; sva = va_next) { 1896 pt_entry_t pbits; 1897 vm_page_t m; 1898 vm_paddr_t pa; 1899 1900 pdpe = pmap_segmap(pmap, sva); 1901#ifdef __mips_n64 1902 if (*pdpe == 0) { 1903 va_next = (sva + NBSEG) & ~SEGMASK; 1904 if (va_next < sva) 1905 va_next = eva; 1906 continue; 1907 } 1908#endif 1909 va_next = (sva + NBPDR) & ~PDRMASK; 1910 if (va_next < sva) 1911 va_next = eva; 1912 1913 pde = pmap_pdpe_to_pde(pdpe, sva); 1914 if (*pde == NULL) 1915 continue; 1916 if (va_next > eva) 1917 va_next = eva; 1918 1919 for (pte = pmap_pde_to_pte(pde, sva); sva != va_next; pte++, 1920 sva += PAGE_SIZE) { 1921 1922 /* Skip invalid PTEs */ 1923 if (!pte_test(pte, PTE_V)) 1924 continue; 1925 pbits = *pte; 1926 if (pte_test(&pbits, PTE_MANAGED | PTE_D)) { 1927 pa = TLBLO_PTE_TO_PA(pbits); 1928 m = PHYS_TO_VM_PAGE(pa); 1929 vm_page_dirty(m); 1930 } 1931 pte_clear(&pbits, PTE_D); 1932 pte_set(&pbits, PTE_RO); 1933 1934 if (pbits != *pte) { 1935 *pte = pbits; 1936 pmap_update_page(pmap, sva, pbits); 1937 } 1938 } 1939 } 1940 rw_wunlock(&pvh_global_lock); 1941 PMAP_UNLOCK(pmap); 1942} 1943 1944/* 1945 * Insert the given physical page (p) at 1946 * the specified virtual address (v) in the 1947 * target physical map with the protection requested. 1948 * 1949 * If specified, the page will be wired down, meaning 1950 * that the related pte can not be reclaimed. 1951 * 1952 * NB: This is the only routine which MAY NOT lazy-evaluate 1953 * or lose information. That is, this routine must actually 1954 * insert this page into the given map NOW. 1955 */ 1956void 1957pmap_enter(pmap_t pmap, vm_offset_t va, vm_prot_t access, vm_page_t m, 1958 vm_prot_t prot, boolean_t wired) 1959{ 1960 vm_paddr_t pa, opa; 1961 pt_entry_t *pte; 1962 pt_entry_t origpte, newpte; 1963 pv_entry_t pv; 1964 vm_page_t mpte, om; 1965 1966 va &= ~PAGE_MASK; 1967 KASSERT(va <= VM_MAX_KERNEL_ADDRESS, ("pmap_enter: toobig")); 1968 KASSERT((m->oflags & VPO_UNMANAGED) != 0 || va < kmi.clean_sva || 1969 va >= kmi.clean_eva, 1970 ("pmap_enter: managed mapping within the clean submap")); 1971 KASSERT((m->oflags & (VPO_UNMANAGED | VPO_BUSY)) != 0, 1972 ("pmap_enter: page %p is not busy", m)); 1973 pa = VM_PAGE_TO_PHYS(m); 1974 newpte = TLBLO_PA_TO_PFN(pa) | init_pte_prot(m, access, prot); 1975 if (wired) 1976 newpte |= PTE_W; 1977 if (is_kernel_pmap(pmap)) 1978 newpte |= PTE_G; 1979 if (is_cacheable_mem(pa)) 1980 newpte |= PTE_C_CACHE; 1981 else 1982 newpte |= PTE_C_UNCACHED; 1983 1984 mpte = NULL; 1985 1986 rw_wlock(&pvh_global_lock); 1987 PMAP_LOCK(pmap); 1988 1989 /* 1990 * In the case that a page table page is not resident, we are 1991 * creating it here. 1992 */ 1993 if (va < VM_MAXUSER_ADDRESS) { 1994 mpte = pmap_allocpte(pmap, va, M_WAITOK); 1995 } 1996 pte = pmap_pte(pmap, va); 1997 1998 /* 1999 * Page Directory table entry not valid, we need a new PT page 2000 */ 2001 if (pte == NULL) { 2002 panic("pmap_enter: invalid page directory, pdir=%p, va=%p", 2003 (void *)pmap->pm_segtab, (void *)va); 2004 } 2005 om = NULL; 2006 origpte = *pte; 2007 opa = TLBLO_PTE_TO_PA(origpte); 2008 2009 /* 2010 * Mapping has not changed, must be protection or wiring change. 2011 */ 2012 if (pte_test(&origpte, PTE_V) && opa == pa) { 2013 /* 2014 * Wiring change, just update stats. We don't worry about 2015 * wiring PT pages as they remain resident as long as there 2016 * are valid mappings in them. Hence, if a user page is 2017 * wired, the PT page will be also. 2018 */ 2019 if (wired && !pte_test(&origpte, PTE_W)) 2020 pmap->pm_stats.wired_count++; 2021 else if (!wired && pte_test(&origpte, PTE_W)) 2022 pmap->pm_stats.wired_count--; 2023 2024 KASSERT(!pte_test(&origpte, PTE_D | PTE_RO), 2025 ("%s: modified page not writable: va: %p, pte: %#jx", 2026 __func__, (void *)va, (uintmax_t)origpte)); 2027 2028 /* 2029 * Remove extra pte reference 2030 */ 2031 if (mpte) 2032 mpte->wire_count--; 2033 2034 if (pte_test(&origpte, PTE_MANAGED)) { 2035 m->md.pv_flags |= PV_TABLE_REF; 2036 om = m; 2037 newpte |= PTE_MANAGED; 2038 if (!pte_test(&newpte, PTE_RO)) 2039 vm_page_aflag_set(m, PGA_WRITEABLE); 2040 } 2041 goto validate; 2042 } 2043 2044 pv = NULL; 2045 2046 /* 2047 * Mapping has changed, invalidate old range and fall through to 2048 * handle validating new mapping. 2049 */ 2050 if (opa) { 2051 if (pte_test(&origpte, PTE_W)) 2052 pmap->pm_stats.wired_count--; 2053 2054 if (pte_test(&origpte, PTE_MANAGED)) { 2055 om = PHYS_TO_VM_PAGE(opa); 2056 pv = pmap_pvh_remove(&om->md, pmap, va); 2057 } 2058 if (mpte != NULL) { 2059 mpte->wire_count--; 2060 KASSERT(mpte->wire_count > 0, 2061 ("pmap_enter: missing reference to page table page," 2062 " va: %p", (void *)va)); 2063 } 2064 } else 2065 pmap->pm_stats.resident_count++; 2066 2067 /* 2068 * Enter on the PV list if part of our managed memory. 2069 */ 2070 if ((m->oflags & VPO_UNMANAGED) == 0) { 2071 m->md.pv_flags |= PV_TABLE_REF; 2072 if (pv == NULL) 2073 pv = get_pv_entry(pmap, FALSE); 2074 pv->pv_va = va; 2075 TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_list); 2076 newpte |= PTE_MANAGED; 2077 if (!pte_test(&newpte, PTE_RO)) 2078 vm_page_aflag_set(m, PGA_WRITEABLE); 2079 } else if (pv != NULL) 2080 free_pv_entry(pmap, pv); 2081 2082 /* 2083 * Increment counters 2084 */ 2085 if (wired) 2086 pmap->pm_stats.wired_count++; 2087 2088validate: 2089 2090#ifdef PMAP_DEBUG 2091 printf("pmap_enter: va: %p -> pa: %p\n", (void *)va, (void *)pa); 2092#endif 2093 2094 /* 2095 * if the mapping or permission bits are different, we need to 2096 * update the pte. 2097 */ 2098 if (origpte != newpte) { 2099 *pte = newpte; 2100 if (pte_test(&origpte, PTE_V)) { 2101 if (pte_test(&origpte, PTE_MANAGED) && opa != pa) { 2102 if (om->md.pv_flags & PV_TABLE_REF) 2103 vm_page_aflag_set(om, PGA_REFERENCED); 2104 om->md.pv_flags &= ~PV_TABLE_REF; 2105 } 2106 if (pte_test(&origpte, PTE_D)) { 2107 KASSERT(!pte_test(&origpte, PTE_RO), 2108 ("pmap_enter: modified page not writable:" 2109 " va: %p, pte: %#jx", (void *)va, (uintmax_t)origpte)); 2110 if (pte_test(&origpte, PTE_MANAGED)) 2111 vm_page_dirty(om); 2112 } 2113 if (pte_test(&origpte, PTE_MANAGED) && 2114 TAILQ_EMPTY(&om->md.pv_list)) 2115 vm_page_aflag_clear(om, PGA_WRITEABLE); 2116 pmap_update_page(pmap, va, newpte); 2117 } 2118 } 2119 2120 /* 2121 * Sync I & D caches for executable pages. Do this only if the 2122 * target pmap belongs to the current process. Otherwise, an 2123 * unresolvable TLB miss may occur. 2124 */ 2125 if (!is_kernel_pmap(pmap) && (pmap == &curproc->p_vmspace->vm_pmap) && 2126 (prot & VM_PROT_EXECUTE)) { 2127 mips_icache_sync_range(va, PAGE_SIZE); 2128 mips_dcache_wbinv_range(va, PAGE_SIZE); 2129 } 2130 rw_wunlock(&pvh_global_lock); 2131 PMAP_UNLOCK(pmap); 2132} 2133 2134/* 2135 * this code makes some *MAJOR* assumptions: 2136 * 1. Current pmap & pmap exists. 2137 * 2. Not wired. 2138 * 3. Read access. 2139 * 4. No page table pages. 2140 * but is *MUCH* faster than pmap_enter... 2141 */ 2142 2143void 2144pmap_enter_quick(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot) 2145{ 2146 2147 rw_wlock(&pvh_global_lock); 2148 PMAP_LOCK(pmap); 2149 (void)pmap_enter_quick_locked(pmap, va, m, prot, NULL); 2150 rw_wunlock(&pvh_global_lock); 2151 PMAP_UNLOCK(pmap); 2152} 2153 2154static vm_page_t 2155pmap_enter_quick_locked(pmap_t pmap, vm_offset_t va, vm_page_t m, 2156 vm_prot_t prot, vm_page_t mpte) 2157{ 2158 pt_entry_t *pte; 2159 vm_paddr_t pa; 2160 2161 KASSERT(va < kmi.clean_sva || va >= kmi.clean_eva || 2162 (m->oflags & VPO_UNMANAGED) != 0, 2163 ("pmap_enter_quick_locked: managed mapping within the clean submap")); 2164 rw_assert(&pvh_global_lock, RA_WLOCKED); 2165 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 2166 2167 /* 2168 * In the case that a page table page is not resident, we are 2169 * creating it here. 2170 */ 2171 if (va < VM_MAXUSER_ADDRESS) { 2172 pd_entry_t *pde; 2173 unsigned ptepindex; 2174 2175 /* 2176 * Calculate pagetable page index 2177 */ 2178 ptepindex = pmap_pde_pindex(va); 2179 if (mpte && (mpte->pindex == ptepindex)) { 2180 mpte->wire_count++; 2181 } else { 2182 /* 2183 * Get the page directory entry 2184 */ 2185 pde = pmap_pde(pmap, va); 2186 2187 /* 2188 * If the page table page is mapped, we just 2189 * increment the hold count, and activate it. 2190 */ 2191 if (pde && *pde != 0) { 2192 mpte = PHYS_TO_VM_PAGE( 2193 MIPS_DIRECT_TO_PHYS(*pde)); 2194 mpte->wire_count++; 2195 } else { 2196 mpte = _pmap_allocpte(pmap, ptepindex, 2197 M_NOWAIT); 2198 if (mpte == NULL) 2199 return (mpte); 2200 } 2201 } 2202 } else { 2203 mpte = NULL; 2204 } 2205 2206 pte = pmap_pte(pmap, va); 2207 if (pte_test(pte, PTE_V)) { 2208 if (mpte != NULL) { 2209 mpte->wire_count--; 2210 mpte = NULL; 2211 } 2212 return (mpte); 2213 } 2214 2215 /* 2216 * Enter on the PV list if part of our managed memory. 2217 */ 2218 if ((m->oflags & VPO_UNMANAGED) == 0 && 2219 !pmap_try_insert_pv_entry(pmap, mpte, va, m)) { 2220 if (mpte != NULL) { 2221 pmap_unwire_ptp(pmap, va, mpte); 2222 mpte = NULL; 2223 } 2224 return (mpte); 2225 } 2226 2227 /* 2228 * Increment counters 2229 */ 2230 pmap->pm_stats.resident_count++; 2231 2232 pa = VM_PAGE_TO_PHYS(m); 2233 2234 /* 2235 * Now validate mapping with RO protection 2236 */ 2237 *pte = PTE_RO | TLBLO_PA_TO_PFN(pa) | PTE_V; 2238 if ((m->oflags & VPO_UNMANAGED) == 0) 2239 *pte |= PTE_MANAGED; 2240 2241 if (is_cacheable_mem(pa)) 2242 *pte |= PTE_C_CACHE; 2243 else 2244 *pte |= PTE_C_UNCACHED; 2245 2246 if (is_kernel_pmap(pmap)) 2247 *pte |= PTE_G; 2248 else { 2249 /* 2250 * Sync I & D caches. Do this only if the target pmap 2251 * belongs to the current process. Otherwise, an 2252 * unresolvable TLB miss may occur. */ 2253 if (pmap == &curproc->p_vmspace->vm_pmap) { 2254 va &= ~PAGE_MASK; 2255 mips_icache_sync_range(va, PAGE_SIZE); 2256 mips_dcache_wbinv_range(va, PAGE_SIZE); 2257 } 2258 } 2259 return (mpte); 2260} 2261 2262/* 2263 * Make a temporary mapping for a physical address. This is only intended 2264 * to be used for panic dumps. 2265 * 2266 * Use XKPHYS for 64 bit, and KSEG0 where possible for 32 bit. 2267 */ 2268void * 2269pmap_kenter_temporary(vm_paddr_t pa, int i) 2270{ 2271 vm_offset_t va; 2272 2273 if (i != 0) 2274 printf("%s: ERROR!!! More than one page of virtual address mapping not supported\n", 2275 __func__); 2276 2277 if (MIPS_DIRECT_MAPPABLE(pa)) { 2278 va = MIPS_PHYS_TO_DIRECT(pa); 2279 } else { 2280#ifndef __mips_n64 /* XXX : to be converted to new style */ 2281 int cpu; 2282 register_t intr; 2283 struct local_sysmaps *sysm; 2284 pt_entry_t *pte, npte; 2285 2286 /* If this is used other than for dumps, we may need to leave 2287 * interrupts disasbled on return. If crash dumps don't work when 2288 * we get to this point, we might want to consider this (leaving things 2289 * disabled as a starting point ;-) 2290 */ 2291 intr = intr_disable(); 2292 cpu = PCPU_GET(cpuid); 2293 sysm = &sysmap_lmem[cpu]; 2294 /* Since this is for the debugger, no locks or any other fun */ 2295 npte = TLBLO_PA_TO_PFN(pa) | PTE_D | PTE_V | PTE_G | PTE_W | PTE_C_CACHE; 2296 pte = pmap_pte(kernel_pmap, sysm->base); 2297 *pte = npte; 2298 sysm->valid1 = 1; 2299 pmap_update_page(kernel_pmap, sysm->base, npte); 2300 va = sysm->base; 2301 intr_restore(intr); 2302#endif 2303 } 2304 return ((void *)va); 2305} 2306 2307void 2308pmap_kenter_temporary_free(vm_paddr_t pa) 2309{ 2310#ifndef __mips_n64 /* XXX : to be converted to new style */ 2311 int cpu; 2312 register_t intr; 2313 struct local_sysmaps *sysm; 2314#endif 2315 2316 if (MIPS_DIRECT_MAPPABLE(pa)) { 2317 /* nothing to do for this case */ 2318 return; 2319 } 2320#ifndef __mips_n64 /* XXX : to be converted to new style */ 2321 cpu = PCPU_GET(cpuid); 2322 sysm = &sysmap_lmem[cpu]; 2323 if (sysm->valid1) { 2324 pt_entry_t *pte; 2325 2326 intr = intr_disable(); 2327 pte = pmap_pte(kernel_pmap, sysm->base); 2328 *pte = PTE_G; 2329 pmap_invalidate_page(kernel_pmap, sysm->base); 2330 intr_restore(intr); 2331 sysm->valid1 = 0; 2332 } 2333#endif 2334} 2335 2336/* 2337 * Maps a sequence of resident pages belonging to the same object. 2338 * The sequence begins with the given page m_start. This page is 2339 * mapped at the given virtual address start. Each subsequent page is 2340 * mapped at a virtual address that is offset from start by the same 2341 * amount as the page is offset from m_start within the object. The 2342 * last page in the sequence is the page with the largest offset from 2343 * m_start that can be mapped at a virtual address less than the given 2344 * virtual address end. Not every virtual page between start and end 2345 * is mapped; only those for which a resident page exists with the 2346 * corresponding offset from m_start are mapped. 2347 */ 2348void 2349pmap_enter_object(pmap_t pmap, vm_offset_t start, vm_offset_t end, 2350 vm_page_t m_start, vm_prot_t prot) 2351{ 2352 vm_page_t m, mpte; 2353 vm_pindex_t diff, psize; 2354 2355 VM_OBJECT_LOCK_ASSERT(m_start->object, MA_OWNED); 2356 psize = atop(end - start); 2357 mpte = NULL; 2358 m = m_start; 2359 rw_wlock(&pvh_global_lock); 2360 PMAP_LOCK(pmap); 2361 while (m != NULL && (diff = m->pindex - m_start->pindex) < psize) { 2362 mpte = pmap_enter_quick_locked(pmap, start + ptoa(diff), m, 2363 prot, mpte); 2364 m = TAILQ_NEXT(m, listq); 2365 } 2366 rw_wunlock(&pvh_global_lock); 2367 PMAP_UNLOCK(pmap); 2368} 2369 2370/* 2371 * pmap_object_init_pt preloads the ptes for a given object 2372 * into the specified pmap. This eliminates the blast of soft 2373 * faults on process startup and immediately after an mmap. 2374 */ 2375void 2376pmap_object_init_pt(pmap_t pmap, vm_offset_t addr, 2377 vm_object_t object, vm_pindex_t pindex, vm_size_t size) 2378{ 2379 VM_OBJECT_LOCK_ASSERT(object, MA_OWNED); 2380 KASSERT(object->type == OBJT_DEVICE || object->type == OBJT_SG, 2381 ("pmap_object_init_pt: non-device object")); 2382} 2383 2384/* 2385 * Routine: pmap_change_wiring 2386 * Function: Change the wiring attribute for a map/virtual-address 2387 * pair. 2388 * In/out conditions: 2389 * The mapping must already exist in the pmap. 2390 */ 2391void 2392pmap_change_wiring(pmap_t pmap, vm_offset_t va, boolean_t wired) 2393{ 2394 pt_entry_t *pte; 2395 2396 PMAP_LOCK(pmap); 2397 pte = pmap_pte(pmap, va); 2398 2399 if (wired && !pte_test(pte, PTE_W)) 2400 pmap->pm_stats.wired_count++; 2401 else if (!wired && pte_test(pte, PTE_W)) 2402 pmap->pm_stats.wired_count--; 2403 2404 /* 2405 * Wiring is not a hardware characteristic so there is no need to 2406 * invalidate TLB. 2407 */ 2408 if (wired) 2409 pte_set(pte, PTE_W); 2410 else 2411 pte_clear(pte, PTE_W); 2412 PMAP_UNLOCK(pmap); 2413} 2414 2415/* 2416 * Copy the range specified by src_addr/len 2417 * from the source map to the range dst_addr/len 2418 * in the destination map. 2419 * 2420 * This routine is only advisory and need not do anything. 2421 */ 2422 2423void 2424pmap_copy(pmap_t dst_pmap, pmap_t src_pmap, vm_offset_t dst_addr, 2425 vm_size_t len, vm_offset_t src_addr) 2426{ 2427} 2428 2429/* 2430 * pmap_zero_page zeros the specified hardware page by mapping 2431 * the page into KVM and using bzero to clear its contents. 2432 * 2433 * Use XKPHYS for 64 bit, and KSEG0 where possible for 32 bit. 2434 */ 2435void 2436pmap_zero_page(vm_page_t m) 2437{ 2438 vm_offset_t va; 2439 vm_paddr_t phys = VM_PAGE_TO_PHYS(m); 2440 2441 if (MIPS_DIRECT_MAPPABLE(phys)) { 2442 va = MIPS_PHYS_TO_DIRECT(phys); 2443 bzero((caddr_t)va, PAGE_SIZE); 2444 mips_dcache_wbinv_range(va, PAGE_SIZE); 2445 } else { 2446 va = pmap_lmem_map1(phys); 2447 bzero((caddr_t)va, PAGE_SIZE); 2448 mips_dcache_wbinv_range(va, PAGE_SIZE); 2449 pmap_lmem_unmap(); 2450 } 2451} 2452 2453/* 2454 * pmap_zero_page_area zeros the specified hardware page by mapping 2455 * the page into KVM and using bzero to clear its contents. 2456 * 2457 * off and size may not cover an area beyond a single hardware page. 2458 */ 2459void 2460pmap_zero_page_area(vm_page_t m, int off, int size) 2461{ 2462 vm_offset_t va; 2463 vm_paddr_t phys = VM_PAGE_TO_PHYS(m); 2464 2465 if (MIPS_DIRECT_MAPPABLE(phys)) { 2466 va = MIPS_PHYS_TO_DIRECT(phys); 2467 bzero((char *)(caddr_t)va + off, size); 2468 mips_dcache_wbinv_range(va + off, size); 2469 } else { 2470 va = pmap_lmem_map1(phys); 2471 bzero((char *)va + off, size); 2472 mips_dcache_wbinv_range(va + off, size); 2473 pmap_lmem_unmap(); 2474 } 2475} 2476 2477void 2478pmap_zero_page_idle(vm_page_t m) 2479{ 2480 vm_offset_t va; 2481 vm_paddr_t phys = VM_PAGE_TO_PHYS(m); 2482 2483 if (MIPS_DIRECT_MAPPABLE(phys)) { 2484 va = MIPS_PHYS_TO_DIRECT(phys); 2485 bzero((caddr_t)va, PAGE_SIZE); 2486 mips_dcache_wbinv_range(va, PAGE_SIZE); 2487 } else { 2488 va = pmap_lmem_map1(phys); 2489 bzero((caddr_t)va, PAGE_SIZE); 2490 mips_dcache_wbinv_range(va, PAGE_SIZE); 2491 pmap_lmem_unmap(); 2492 } 2493} 2494 2495/* 2496 * pmap_copy_page copies the specified (machine independent) 2497 * page by mapping the page into virtual memory and using 2498 * bcopy to copy the page, one machine dependent page at a 2499 * time. 2500 * 2501 * Use XKPHYS for 64 bit, and KSEG0 where possible for 32 bit. 2502 */ 2503void 2504pmap_copy_page(vm_page_t src, vm_page_t dst) 2505{ 2506 vm_offset_t va_src, va_dst; 2507 vm_paddr_t phys_src = VM_PAGE_TO_PHYS(src); 2508 vm_paddr_t phys_dst = VM_PAGE_TO_PHYS(dst); 2509 2510 if (MIPS_DIRECT_MAPPABLE(phys_src) && MIPS_DIRECT_MAPPABLE(phys_dst)) { 2511 /* easy case, all can be accessed via KSEG0 */ 2512 /* 2513 * Flush all caches for VA that are mapped to this page 2514 * to make sure that data in SDRAM is up to date 2515 */ 2516 pmap_flush_pvcache(src); 2517 mips_dcache_wbinv_range_index( 2518 MIPS_PHYS_TO_DIRECT(phys_dst), PAGE_SIZE); 2519 va_src = MIPS_PHYS_TO_DIRECT(phys_src); 2520 va_dst = MIPS_PHYS_TO_DIRECT(phys_dst); 2521 bcopy((caddr_t)va_src, (caddr_t)va_dst, PAGE_SIZE); 2522 mips_dcache_wbinv_range(va_dst, PAGE_SIZE); 2523 } else { 2524 va_src = pmap_lmem_map2(phys_src, phys_dst); 2525 va_dst = va_src + PAGE_SIZE; 2526 bcopy((void *)va_src, (void *)va_dst, PAGE_SIZE); 2527 mips_dcache_wbinv_range(va_dst, PAGE_SIZE); 2528 pmap_lmem_unmap(); 2529 } 2530} 2531 2532/* 2533 * Returns true if the pmap's pv is one of the first 2534 * 16 pvs linked to from this page. This count may 2535 * be changed upwards or downwards in the future; it 2536 * is only necessary that true be returned for a small 2537 * subset of pmaps for proper page aging. 2538 */ 2539boolean_t 2540pmap_page_exists_quick(pmap_t pmap, vm_page_t m) 2541{ 2542 pv_entry_t pv; 2543 int loops = 0; 2544 boolean_t rv; 2545 2546 KASSERT((m->oflags & VPO_UNMANAGED) == 0, 2547 ("pmap_page_exists_quick: page %p is not managed", m)); 2548 rv = FALSE; 2549 rw_wlock(&pvh_global_lock); 2550 TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) { 2551 if (PV_PMAP(pv) == pmap) { 2552 rv = TRUE; 2553 break; 2554 } 2555 loops++; 2556 if (loops >= 16) 2557 break; 2558 } 2559 rw_wunlock(&pvh_global_lock); 2560 return (rv); 2561} 2562 2563/* 2564 * Remove all pages from specified address space 2565 * this aids process exit speeds. Also, this code 2566 * is special cased for current process only, but 2567 * can have the more generic (and slightly slower) 2568 * mode enabled. This is much faster than pmap_remove 2569 * in the case of running down an entire address space. 2570 */ 2571void 2572pmap_remove_pages(pmap_t pmap) 2573{ 2574 pd_entry_t *pde; 2575 pt_entry_t *pte, tpte; 2576 pv_entry_t pv; 2577 vm_page_t m; 2578 struct pv_chunk *pc, *npc; 2579 u_long inuse, bitmask; 2580 int allfree, bit, field, idx; 2581 2582 if (pmap != vmspace_pmap(curthread->td_proc->p_vmspace)) { 2583 printf("warning: pmap_remove_pages called with non-current pmap\n"); 2584 return; 2585 } 2586 rw_wlock(&pvh_global_lock); 2587 PMAP_LOCK(pmap); 2588 TAILQ_FOREACH_SAFE(pc, &pmap->pm_pvchunk, pc_list, npc) { 2589 allfree = 1; 2590 for (field = 0; field < _NPCM; field++) { 2591 inuse = ~pc->pc_map[field] & pc_freemask[field]; 2592 while (inuse != 0) { 2593 bit = ffsl(inuse) - 1; 2594 bitmask = 1UL << bit; 2595 idx = field * sizeof(inuse) * NBBY + bit; 2596 pv = &pc->pc_pventry[idx]; 2597 inuse &= ~bitmask; 2598 2599 pde = pmap_pde(pmap, pv->pv_va); 2600 KASSERT(pde != NULL && *pde != 0, 2601 ("pmap_remove_pages: pde")); 2602 pte = pmap_pde_to_pte(pde, pv->pv_va); 2603 if (!pte_test(pte, PTE_V)) 2604 panic("pmap_remove_pages: bad pte"); 2605 tpte = *pte; 2606 2607/* 2608 * We cannot remove wired pages from a process' mapping at this time 2609 */ 2610 if (pte_test(&tpte, PTE_W)) { 2611 allfree = 0; 2612 continue; 2613 } 2614 *pte = is_kernel_pmap(pmap) ? PTE_G : 0; 2615 2616 m = PHYS_TO_VM_PAGE(TLBLO_PTE_TO_PA(tpte)); 2617 KASSERT(m != NULL, 2618 ("pmap_remove_pages: bad tpte %#jx", 2619 (uintmax_t)tpte)); 2620 2621 /* 2622 * Update the vm_page_t clean and reference bits. 2623 */ 2624 if (pte_test(&tpte, PTE_D)) 2625 vm_page_dirty(m); 2626 2627 /* Mark free */ 2628 PV_STAT(pv_entry_frees++); 2629 PV_STAT(pv_entry_spare++); 2630 pv_entry_count--; 2631 pc->pc_map[field] |= bitmask; 2632 pmap->pm_stats.resident_count--; 2633 TAILQ_REMOVE(&m->md.pv_list, pv, pv_list); 2634 if (TAILQ_EMPTY(&m->md.pv_list)) 2635 vm_page_aflag_clear(m, PGA_WRITEABLE); 2636 pmap_unuse_pt(pmap, pv->pv_va, *pde); 2637 } 2638 } 2639 if (allfree) { 2640 TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list); 2641 free_pv_chunk(pc); 2642 } 2643 } 2644 pmap_invalidate_all(pmap); 2645 PMAP_UNLOCK(pmap); 2646 rw_wunlock(&pvh_global_lock); 2647} 2648 2649/* 2650 * pmap_testbit tests bits in pte's 2651 */ 2652static boolean_t 2653pmap_testbit(vm_page_t m, int bit) 2654{ 2655 pv_entry_t pv; 2656 pmap_t pmap; 2657 pt_entry_t *pte; 2658 boolean_t rv = FALSE; 2659 2660 if (m->oflags & VPO_UNMANAGED) 2661 return (rv); 2662 2663 rw_assert(&pvh_global_lock, RA_WLOCKED); 2664 TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) { 2665 pmap = PV_PMAP(pv); 2666 PMAP_LOCK(pmap); 2667 pte = pmap_pte(pmap, pv->pv_va); 2668 rv = pte_test(pte, bit); 2669 PMAP_UNLOCK(pmap); 2670 if (rv) 2671 break; 2672 } 2673 return (rv); 2674} 2675 2676/* 2677 * pmap_page_wired_mappings: 2678 * 2679 * Return the number of managed mappings to the given physical page 2680 * that are wired. 2681 */ 2682int 2683pmap_page_wired_mappings(vm_page_t m) 2684{ 2685 pv_entry_t pv; 2686 pmap_t pmap; 2687 pt_entry_t *pte; 2688 int count; 2689 2690 count = 0; 2691 if ((m->oflags & VPO_UNMANAGED) != 0) 2692 return (count); 2693 rw_wlock(&pvh_global_lock); 2694 TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) { 2695 pmap = PV_PMAP(pv); 2696 PMAP_LOCK(pmap); 2697 pte = pmap_pte(pmap, pv->pv_va); 2698 if (pte_test(pte, PTE_W)) 2699 count++; 2700 PMAP_UNLOCK(pmap); 2701 } 2702 rw_wunlock(&pvh_global_lock); 2703 return (count); 2704} 2705 2706/* 2707 * Clear the write and modified bits in each of the given page's mappings. 2708 */ 2709void 2710pmap_remove_write(vm_page_t m) 2711{ 2712 pmap_t pmap; 2713 pt_entry_t pbits, *pte; 2714 pv_entry_t pv; 2715 2716 KASSERT((m->oflags & VPO_UNMANAGED) == 0, 2717 ("pmap_remove_write: page %p is not managed", m)); 2718 2719 /* 2720 * If the page is not VPO_BUSY, then PGA_WRITEABLE cannot be set by 2721 * another thread while the object is locked. Thus, if PGA_WRITEABLE 2722 * is clear, no page table entries need updating. 2723 */ 2724 VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED); 2725 if ((m->oflags & VPO_BUSY) == 0 && 2726 (m->aflags & PGA_WRITEABLE) == 0) 2727 return; 2728 rw_wlock(&pvh_global_lock); 2729 TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) { 2730 pmap = PV_PMAP(pv); 2731 PMAP_LOCK(pmap); 2732 pte = pmap_pte(pmap, pv->pv_va); 2733 KASSERT(pte != NULL && pte_test(pte, PTE_V), 2734 ("page on pv_list has no pte")); 2735 pbits = *pte; 2736 if (pte_test(&pbits, PTE_D)) { 2737 pte_clear(&pbits, PTE_D); 2738 vm_page_dirty(m); 2739 } 2740 pte_set(&pbits, PTE_RO); 2741 if (pbits != *pte) { 2742 *pte = pbits; 2743 pmap_update_page(pmap, pv->pv_va, pbits); 2744 } 2745 PMAP_UNLOCK(pmap); 2746 } 2747 vm_page_aflag_clear(m, PGA_WRITEABLE); 2748 rw_wunlock(&pvh_global_lock); 2749} 2750 2751/* 2752 * pmap_ts_referenced: 2753 * 2754 * Return the count of reference bits for a page, clearing all of them. 2755 */ 2756int 2757pmap_ts_referenced(vm_page_t m) 2758{ 2759 2760 KASSERT((m->oflags & VPO_UNMANAGED) == 0, 2761 ("pmap_ts_referenced: page %p is not managed", m)); 2762 if (m->md.pv_flags & PV_TABLE_REF) { 2763 rw_wlock(&pvh_global_lock); 2764 m->md.pv_flags &= ~PV_TABLE_REF; 2765 rw_wunlock(&pvh_global_lock); 2766 return (1); 2767 } 2768 return (0); 2769} 2770 2771/* 2772 * pmap_is_modified: 2773 * 2774 * Return whether or not the specified physical page was modified 2775 * in any physical maps. 2776 */ 2777boolean_t 2778pmap_is_modified(vm_page_t m) 2779{ 2780 boolean_t rv; 2781 2782 KASSERT((m->oflags & VPO_UNMANAGED) == 0, 2783 ("pmap_is_modified: page %p is not managed", m)); 2784 2785 /* 2786 * If the page is not VPO_BUSY, then PGA_WRITEABLE cannot be 2787 * concurrently set while the object is locked. Thus, if PGA_WRITEABLE 2788 * is clear, no PTEs can have PTE_D set. 2789 */ 2790 VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED); 2791 if ((m->oflags & VPO_BUSY) == 0 && 2792 (m->aflags & PGA_WRITEABLE) == 0) 2793 return (FALSE); 2794 rw_wlock(&pvh_global_lock); 2795 rv = pmap_testbit(m, PTE_D); 2796 rw_wunlock(&pvh_global_lock); 2797 return (rv); 2798} 2799 2800/* N/C */ 2801 2802/* 2803 * pmap_is_prefaultable: 2804 * 2805 * Return whether or not the specified virtual address is elgible 2806 * for prefault. 2807 */ 2808boolean_t 2809pmap_is_prefaultable(pmap_t pmap, vm_offset_t addr) 2810{ 2811 pd_entry_t *pde; 2812 pt_entry_t *pte; 2813 boolean_t rv; 2814 2815 rv = FALSE; 2816 PMAP_LOCK(pmap); 2817 pde = pmap_pde(pmap, addr); 2818 if (pde != NULL && *pde != 0) { 2819 pte = pmap_pde_to_pte(pde, addr); 2820 rv = (*pte == 0); 2821 } 2822 PMAP_UNLOCK(pmap); 2823 return (rv); 2824} 2825 2826/* 2827 * Clear the modify bits on the specified physical page. 2828 */ 2829void 2830pmap_clear_modify(vm_page_t m) 2831{ 2832 pmap_t pmap; 2833 pt_entry_t *pte; 2834 pv_entry_t pv; 2835 2836 KASSERT((m->oflags & VPO_UNMANAGED) == 0, 2837 ("pmap_clear_modify: page %p is not managed", m)); 2838 VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED); 2839 KASSERT((m->oflags & VPO_BUSY) == 0, 2840 ("pmap_clear_modify: page %p is busy", m)); 2841 2842 /* 2843 * If the page is not PGA_WRITEABLE, then no PTEs can have PTE_D set. 2844 * If the object containing the page is locked and the page is not 2845 * VPO_BUSY, then PGA_WRITEABLE cannot be concurrently set. 2846 */ 2847 if ((m->aflags & PGA_WRITEABLE) == 0) 2848 return; 2849 rw_wlock(&pvh_global_lock); 2850 TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) { 2851 pmap = PV_PMAP(pv); 2852 PMAP_LOCK(pmap); 2853 pte = pmap_pte(pmap, pv->pv_va); 2854 if (pte_test(pte, PTE_D)) { 2855 pte_clear(pte, PTE_D); 2856 pmap_update_page(pmap, pv->pv_va, *pte); 2857 } 2858 PMAP_UNLOCK(pmap); 2859 } 2860 rw_wunlock(&pvh_global_lock); 2861} 2862 2863/* 2864 * pmap_is_referenced: 2865 * 2866 * Return whether or not the specified physical page was referenced 2867 * in any physical maps. 2868 */ 2869boolean_t 2870pmap_is_referenced(vm_page_t m) 2871{ 2872 2873 KASSERT((m->oflags & VPO_UNMANAGED) == 0, 2874 ("pmap_is_referenced: page %p is not managed", m)); 2875 return ((m->md.pv_flags & PV_TABLE_REF) != 0); 2876} 2877 2878/* 2879 * pmap_clear_reference: 2880 * 2881 * Clear the reference bit on the specified physical page. 2882 */ 2883void 2884pmap_clear_reference(vm_page_t m) 2885{ 2886 2887 KASSERT((m->oflags & VPO_UNMANAGED) == 0, 2888 ("pmap_clear_reference: page %p is not managed", m)); 2889 rw_wlock(&pvh_global_lock); 2890 if (m->md.pv_flags & PV_TABLE_REF) { 2891 m->md.pv_flags &= ~PV_TABLE_REF; 2892 } 2893 rw_wunlock(&pvh_global_lock); 2894} 2895 2896/* 2897 * Miscellaneous support routines follow 2898 */ 2899 2900/* 2901 * Map a set of physical memory pages into the kernel virtual 2902 * address space. Return a pointer to where it is mapped. This 2903 * routine is intended to be used for mapping device memory, 2904 * NOT real memory. 2905 */ 2906 2907/* 2908 * Map a set of physical memory pages into the kernel virtual 2909 * address space. Return a pointer to where it is mapped. This 2910 * routine is intended to be used for mapping device memory, 2911 * NOT real memory. 2912 * 2913 * Use XKPHYS uncached for 64 bit, and KSEG1 where possible for 32 bit. 2914 */ 2915void * 2916pmap_mapdev(vm_paddr_t pa, vm_size_t size) 2917{ 2918 vm_offset_t va, tmpva, offset; 2919 2920 /* 2921 * KSEG1 maps only first 512M of phys address space. For 2922 * pa > 0x20000000 we should make proper mapping * using pmap_kenter. 2923 */ 2924 if (MIPS_DIRECT_MAPPABLE(pa + size - 1)) 2925 return ((void *)MIPS_PHYS_TO_DIRECT_UNCACHED(pa)); 2926 else { 2927 offset = pa & PAGE_MASK; 2928 size = roundup(size + offset, PAGE_SIZE); 2929 2930 va = kmem_alloc_nofault(kernel_map, size); 2931 if (!va) 2932 panic("pmap_mapdev: Couldn't alloc kernel virtual memory"); 2933 pa = trunc_page(pa); 2934 for (tmpva = va; size > 0;) { 2935 pmap_kenter_attr(tmpva, pa, PTE_C_UNCACHED); 2936 size -= PAGE_SIZE; 2937 tmpva += PAGE_SIZE; 2938 pa += PAGE_SIZE; 2939 } 2940 } 2941 2942 return ((void *)(va + offset)); 2943} 2944 2945void 2946pmap_unmapdev(vm_offset_t va, vm_size_t size) 2947{ 2948#ifndef __mips_n64 2949 vm_offset_t base, offset, tmpva; 2950 2951 /* If the address is within KSEG1 then there is nothing to do */ 2952 if (va >= MIPS_KSEG1_START && va <= MIPS_KSEG1_END) 2953 return; 2954 2955 base = trunc_page(va); 2956 offset = va & PAGE_MASK; 2957 size = roundup(size + offset, PAGE_SIZE); 2958 for (tmpva = base; tmpva < base + size; tmpva += PAGE_SIZE) 2959 pmap_kremove(tmpva); 2960 kmem_free(kernel_map, base, size); 2961#endif 2962} 2963 2964/* 2965 * perform the pmap work for mincore 2966 */ 2967int 2968pmap_mincore(pmap_t pmap, vm_offset_t addr, vm_paddr_t *locked_pa) 2969{ 2970 pt_entry_t *ptep, pte; 2971 vm_paddr_t pa; 2972 vm_page_t m; 2973 int val; 2974 2975 PMAP_LOCK(pmap); 2976retry: 2977 ptep = pmap_pte(pmap, addr); 2978 pte = (ptep != NULL) ? *ptep : 0; 2979 if (!pte_test(&pte, PTE_V)) { 2980 val = 0; 2981 goto out; 2982 } 2983 val = MINCORE_INCORE; 2984 if (pte_test(&pte, PTE_D)) 2985 val |= MINCORE_MODIFIED | MINCORE_MODIFIED_OTHER; 2986 pa = TLBLO_PTE_TO_PA(pte); 2987 if (pte_test(&pte, PTE_MANAGED)) { 2988 /* 2989 * This may falsely report the given address as 2990 * MINCORE_REFERENCED. Unfortunately, due to the lack of 2991 * per-PTE reference information, it is impossible to 2992 * determine if the address is MINCORE_REFERENCED. 2993 */ 2994 m = PHYS_TO_VM_PAGE(pa); 2995 if ((m->aflags & PGA_REFERENCED) != 0) 2996 val |= MINCORE_REFERENCED | MINCORE_REFERENCED_OTHER; 2997 } 2998 if ((val & (MINCORE_MODIFIED_OTHER | MINCORE_REFERENCED_OTHER)) != 2999 (MINCORE_MODIFIED_OTHER | MINCORE_REFERENCED_OTHER) && 3000 pte_test(&pte, PTE_MANAGED)) { 3001 /* Ensure that "PHYS_TO_VM_PAGE(pa)->object" doesn't change. */ 3002 if (vm_page_pa_tryrelock(pmap, pa, locked_pa)) 3003 goto retry; 3004 } else 3005out: 3006 PA_UNLOCK_COND(*locked_pa); 3007 PMAP_UNLOCK(pmap); 3008 return (val); 3009} 3010 3011void 3012pmap_activate(struct thread *td) 3013{ 3014 pmap_t pmap, oldpmap; 3015 struct proc *p = td->td_proc; 3016 u_int cpuid; 3017 3018 critical_enter(); 3019 3020 pmap = vmspace_pmap(p->p_vmspace); 3021 oldpmap = PCPU_GET(curpmap); 3022 cpuid = PCPU_GET(cpuid); 3023 3024 if (oldpmap) 3025 CPU_CLR_ATOMIC(cpuid, &oldpmap->pm_active); 3026 CPU_SET_ATOMIC(cpuid, &pmap->pm_active); 3027 pmap_asid_alloc(pmap); 3028 if (td == curthread) { 3029 PCPU_SET(segbase, pmap->pm_segtab); 3030 mips_wr_entryhi(pmap->pm_asid[cpuid].asid); 3031 } 3032 3033 PCPU_SET(curpmap, pmap); 3034 critical_exit(); 3035} 3036 3037void 3038pmap_sync_icache(pmap_t pm, vm_offset_t va, vm_size_t sz) 3039{ 3040} 3041 3042/* 3043 * Increase the starting virtual address of the given mapping if a 3044 * different alignment might result in more superpage mappings. 3045 */ 3046void 3047pmap_align_superpage(vm_object_t object, vm_ooffset_t offset, 3048 vm_offset_t *addr, vm_size_t size) 3049{ 3050 vm_offset_t superpage_offset; 3051 3052 if (size < NBSEG) 3053 return; 3054 if (object != NULL && (object->flags & OBJ_COLORED) != 0) 3055 offset += ptoa(object->pg_color); 3056 superpage_offset = offset & SEGMASK; 3057 if (size - ((NBSEG - superpage_offset) & SEGMASK) < NBSEG || 3058 (*addr & SEGMASK) == superpage_offset) 3059 return; 3060 if ((*addr & SEGMASK) < superpage_offset) 3061 *addr = (*addr & ~SEGMASK) + superpage_offset; 3062 else 3063 *addr = ((*addr + SEGMASK) & ~SEGMASK) + superpage_offset; 3064} 3065 3066/* 3067 * Increase the starting virtual address of the given mapping so 3068 * that it is aligned to not be the second page in a TLB entry. 3069 * This routine assumes that the length is appropriately-sized so 3070 * that the allocation does not share a TLB entry at all if required. 3071 */ 3072void 3073pmap_align_tlb(vm_offset_t *addr) 3074{ 3075 if ((*addr & PAGE_SIZE) == 0) 3076 return; 3077 *addr += PAGE_SIZE; 3078 return; 3079} 3080 3081#ifdef DDB 3082DB_SHOW_COMMAND(ptable, ddb_pid_dump) 3083{ 3084 pmap_t pmap; 3085 struct thread *td = NULL; 3086 struct proc *p; 3087 int i, j, k; 3088 vm_paddr_t pa; 3089 vm_offset_t va; 3090 3091 if (have_addr) { 3092 td = db_lookup_thread(addr, TRUE); 3093 if (td == NULL) { 3094 db_printf("Invalid pid or tid"); 3095 return; 3096 } 3097 p = td->td_proc; 3098 if (p->p_vmspace == NULL) { 3099 db_printf("No vmspace for process"); 3100 return; 3101 } 3102 pmap = vmspace_pmap(p->p_vmspace); 3103 } else 3104 pmap = kernel_pmap; 3105 3106 db_printf("pmap:%p segtab:%p asid:%x generation:%x\n", 3107 pmap, pmap->pm_segtab, pmap->pm_asid[0].asid, 3108 pmap->pm_asid[0].gen); 3109 for (i = 0; i < NPDEPG; i++) { 3110 pd_entry_t *pdpe; 3111 pt_entry_t *pde; 3112 pt_entry_t pte; 3113 3114 pdpe = (pd_entry_t *)pmap->pm_segtab[i]; 3115 if (pdpe == NULL) 3116 continue; 3117 db_printf("[%4d] %p\n", i, pdpe); 3118#ifdef __mips_n64 3119 for (j = 0; j < NPDEPG; j++) { 3120 pde = (pt_entry_t *)pdpe[j]; 3121 if (pde == NULL) 3122 continue; 3123 db_printf("\t[%4d] %p\n", j, pde); 3124#else 3125 { 3126 j = 0; 3127 pde = (pt_entry_t *)pdpe; 3128#endif 3129 for (k = 0; k < NPTEPG; k++) { 3130 pte = pde[k]; 3131 if (pte == 0 || !pte_test(&pte, PTE_V)) 3132 continue; 3133 pa = TLBLO_PTE_TO_PA(pte); 3134 va = ((u_long)i << SEGSHIFT) | (j << PDRSHIFT) | (k << PAGE_SHIFT); 3135 db_printf("\t\t[%04d] va: %p pte: %8jx pa:%jx\n", 3136 k, (void *)va, (uintmax_t)pte, (uintmax_t)pa); 3137 } 3138 } 3139 } 3140} 3141#endif 3142 3143#if defined(DEBUG) 3144 3145static void pads(pmap_t pm); 3146void pmap_pvdump(vm_offset_t pa); 3147 3148/* print address space of pmap*/ 3149static void 3150pads(pmap_t pm) 3151{ 3152 unsigned va, i, j; 3153 pt_entry_t *ptep; 3154 3155 if (pm == kernel_pmap) 3156 return; 3157 for (i = 0; i < NPTEPG; i++) 3158 if (pm->pm_segtab[i]) 3159 for (j = 0; j < NPTEPG; j++) { 3160 va = (i << SEGSHIFT) + (j << PAGE_SHIFT); 3161 if (pm == kernel_pmap && va < KERNBASE) 3162 continue; 3163 if (pm != kernel_pmap && 3164 va >= VM_MAXUSER_ADDRESS) 3165 continue; 3166 ptep = pmap_pte(pm, va); 3167 if (pte_test(ptep, PTE_V)) 3168 printf("%x:%x ", va, *(int *)ptep); 3169 } 3170 3171} 3172 3173void 3174pmap_pvdump(vm_offset_t pa) 3175{ 3176 register pv_entry_t pv; 3177 vm_page_t m; 3178 3179 printf("pa %x", pa); 3180 m = PHYS_TO_VM_PAGE(pa); 3181 for (pv = TAILQ_FIRST(&m->md.pv_list); pv; 3182 pv = TAILQ_NEXT(pv, pv_list)) { 3183 printf(" -> pmap %p, va %x", (void *)pv->pv_pmap, pv->pv_va); 3184 pads(pv->pv_pmap); 3185 } 3186 printf(" "); 3187} 3188 3189/* N/C */ 3190#endif 3191 3192 3193/* 3194 * Allocate TLB address space tag (called ASID or TLBPID) and return it. 3195 * It takes almost as much or more time to search the TLB for a 3196 * specific ASID and flush those entries as it does to flush the entire TLB. 3197 * Therefore, when we allocate a new ASID, we just take the next number. When 3198 * we run out of numbers, we flush the TLB, increment the generation count 3199 * and start over. ASID zero is reserved for kernel use. 3200 */ 3201static void 3202pmap_asid_alloc(pmap) 3203 pmap_t pmap; 3204{ 3205 if (pmap->pm_asid[PCPU_GET(cpuid)].asid != PMAP_ASID_RESERVED && 3206 pmap->pm_asid[PCPU_GET(cpuid)].gen == PCPU_GET(asid_generation)); 3207 else { 3208 if (PCPU_GET(next_asid) == pmap_max_asid) { 3209 tlb_invalidate_all_user(NULL); 3210 PCPU_SET(asid_generation, 3211 (PCPU_GET(asid_generation) + 1) & ASIDGEN_MASK); 3212 if (PCPU_GET(asid_generation) == 0) { 3213 PCPU_SET(asid_generation, 1); 3214 } 3215 PCPU_SET(next_asid, 1); /* 0 means invalid */ 3216 } 3217 pmap->pm_asid[PCPU_GET(cpuid)].asid = PCPU_GET(next_asid); 3218 pmap->pm_asid[PCPU_GET(cpuid)].gen = PCPU_GET(asid_generation); 3219 PCPU_SET(next_asid, PCPU_GET(next_asid) + 1); 3220 } 3221} 3222 3223static pt_entry_t 3224init_pte_prot(vm_page_t m, vm_prot_t access, vm_prot_t prot) 3225{ 3226 pt_entry_t rw; 3227 3228 if (!(prot & VM_PROT_WRITE)) 3229 rw = PTE_V | PTE_RO; 3230 else if ((m->oflags & VPO_UNMANAGED) == 0) { 3231 if ((access & VM_PROT_WRITE) != 0) 3232 rw = PTE_V | PTE_D; 3233 else 3234 rw = PTE_V; 3235 } else 3236 /* Needn't emulate a modified bit for unmanaged pages. */ 3237 rw = PTE_V | PTE_D; 3238 return (rw); 3239} 3240 3241/* 3242 * pmap_emulate_modified : do dirty bit emulation 3243 * 3244 * On SMP, update just the local TLB, other CPUs will update their 3245 * TLBs from PTE lazily, if they get the exception. 3246 * Returns 0 in case of sucess, 1 if the page is read only and we 3247 * need to fault. 3248 */ 3249int 3250pmap_emulate_modified(pmap_t pmap, vm_offset_t va) 3251{ 3252 pt_entry_t *pte; 3253 3254 PMAP_LOCK(pmap); 3255 pte = pmap_pte(pmap, va); 3256 if (pte == NULL) 3257 panic("pmap_emulate_modified: can't find PTE"); 3258#ifdef SMP 3259 /* It is possible that some other CPU changed m-bit */ 3260 if (!pte_test(pte, PTE_V) || pte_test(pte, PTE_D)) { 3261 tlb_update(pmap, va, *pte); 3262 PMAP_UNLOCK(pmap); 3263 return (0); 3264 } 3265#else 3266 if (!pte_test(pte, PTE_V) || pte_test(pte, PTE_D)) 3267 panic("pmap_emulate_modified: invalid pte"); 3268#endif 3269 if (pte_test(pte, PTE_RO)) { 3270 PMAP_UNLOCK(pmap); 3271 return (1); 3272 } 3273 pte_set(pte, PTE_D); 3274 tlb_update(pmap, va, *pte); 3275 if (!pte_test(pte, PTE_MANAGED)) 3276 panic("pmap_emulate_modified: unmanaged page"); 3277 PMAP_UNLOCK(pmap); 3278 return (0); 3279} 3280 3281/* 3282 * Routine: pmap_kextract 3283 * Function: 3284 * Extract the physical page address associated 3285 * virtual address. 3286 */ 3287vm_paddr_t 3288pmap_kextract(vm_offset_t va) 3289{ 3290 int mapped; 3291 3292 /* 3293 * First, the direct-mapped regions. 3294 */ 3295#if defined(__mips_n64) 3296 if (va >= MIPS_XKPHYS_START && va < MIPS_XKPHYS_END) 3297 return (MIPS_XKPHYS_TO_PHYS(va)); 3298#endif 3299 if (va >= MIPS_KSEG0_START && va < MIPS_KSEG0_END) 3300 return (MIPS_KSEG0_TO_PHYS(va)); 3301 3302 if (va >= MIPS_KSEG1_START && va < MIPS_KSEG1_END) 3303 return (MIPS_KSEG1_TO_PHYS(va)); 3304 3305 /* 3306 * User virtual addresses. 3307 */ 3308 if (va < VM_MAXUSER_ADDRESS) { 3309 pt_entry_t *ptep; 3310 3311 if (curproc && curproc->p_vmspace) { 3312 ptep = pmap_pte(&curproc->p_vmspace->vm_pmap, va); 3313 if (ptep) { 3314 return (TLBLO_PTE_TO_PA(*ptep) | 3315 (va & PAGE_MASK)); 3316 } 3317 return (0); 3318 } 3319 } 3320 3321 /* 3322 * Should be kernel virtual here, otherwise fail 3323 */ 3324 mapped = (va >= MIPS_KSEG2_START || va < MIPS_KSEG2_END); 3325#if defined(__mips_n64) 3326 mapped = mapped || (va >= MIPS_XKSEG_START || va < MIPS_XKSEG_END); 3327#endif 3328 /* 3329 * Kernel virtual. 3330 */ 3331 3332 if (mapped) { 3333 pt_entry_t *ptep; 3334 3335 /* Is the kernel pmap initialized? */ 3336 if (!CPU_EMPTY(&kernel_pmap->pm_active)) { 3337 /* It's inside the virtual address range */ 3338 ptep = pmap_pte(kernel_pmap, va); 3339 if (ptep) { 3340 return (TLBLO_PTE_TO_PA(*ptep) | 3341 (va & PAGE_MASK)); 3342 } 3343 } 3344 return (0); 3345 } 3346 3347 panic("%s for unknown address space %p.", __func__, (void *)va); 3348} 3349 3350 3351void 3352pmap_flush_pvcache(vm_page_t m) 3353{ 3354 pv_entry_t pv; 3355 3356 if (m != NULL) { 3357 for (pv = TAILQ_FIRST(&m->md.pv_list); pv; 3358 pv = TAILQ_NEXT(pv, pv_list)) { 3359 mips_dcache_wbinv_range_index(pv->pv_va, PAGE_SIZE); 3360 } 3361 } 3362} 3363