pmap.c revision 16680
1145524Sdarrenr/* 2145524Sdarrenr * Copyright (c) 1991 Regents of the University of California. 3145524Sdarrenr * All rights reserved. 4145524Sdarrenr * Copyright (c) 1994 John S. Dyson 5145524Sdarrenr * All rights reserved. 6145524Sdarrenr * Copyright (c) 1994 David Greenman 7145524Sdarrenr * All rights reserved. 8145524Sdarrenr * 9145524Sdarrenr * This code is derived from software contributed to Berkeley by 10145524Sdarrenr * the Systems Programming Group of the University of Utah Computer 11145524Sdarrenr * Science Department and William Jolitz of UUNET Technologies Inc. 12 * 13 * Redistribution and use in source and binary forms, with or without 14 * modification, are permitted provided that the following conditions 15 * are met: 16 * 1. Redistributions of source code must retain the above copyright 17 * notice, this list of conditions and the following disclaimer. 18 * 2. Redistributions in binary form must reproduce the above copyright 19 * notice, this list of conditions and the following disclaimer in the 20 * documentation and/or other materials provided with the distribution. 21 * 3. All advertising materials mentioning features or use of this software 22 * must display the following acknowledgement: 23 * This product includes software developed by the University of 24 * California, Berkeley and its contributors. 25 * 4. Neither the name of the University nor the names of its contributors 26 * may be used to endorse or promote products derived from this software 27 * without specific prior written permission. 28 * 29 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 30 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 31 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 32 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 33 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 34 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 35 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 36 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 37 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 38 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 39 * SUCH DAMAGE. 40 * 41 * from: @(#)pmap.c 7.7 (Berkeley) 5/12/91 42 * $Id: pmap.c,v 1.106 1996/06/18 01:22:06 bde Exp $ 43 */ 44 45/* 46 * Manages physical address maps. 47 * 48 * In addition to hardware address maps, this 49 * module is called upon to provide software-use-only 50 * maps which may or may not be stored in the same 51 * form as hardware maps. These pseudo-maps are 52 * used to store intermediate results from copy 53 * operations to and from address spaces. 54 * 55 * Since the information managed by this module is 56 * also stored by the logical address mapping module, 57 * this module may throw away valid virtual-to-physical 58 * mappings at almost any time. However, invalidations 59 * of virtual-to-physical mappings must be done as 60 * requested. 61 * 62 * In order to cope with hardware architectures which 63 * make virtual-to-physical map invalidates expensive, 64 * this module may delay invalidate or reduced protection 65 * operations until such time as they are actually 66 * necessary. This module is given full information as 67 * to which processors are currently using which maps, 68 * and to when physical maps must be made correct. 69 */ 70 71#include <sys/param.h> 72#include <sys/systm.h> 73#include <sys/proc.h> 74#include <sys/malloc.h> 75#include <sys/msgbuf.h> 76#include <sys/queue.h> 77#include <sys/vmmeter.h> 78#include <sys/mman.h> 79 80#include <vm/vm.h> 81#include <vm/vm_param.h> 82#include <vm/vm_prot.h> 83#include <vm/lock.h> 84#include <vm/vm_kern.h> 85#include <vm/vm_page.h> 86#include <vm/vm_map.h> 87#include <vm/vm_object.h> 88#include <vm/vm_extern.h> 89#include <vm/vm_pageout.h> 90#include <vm/vm_pager.h> 91 92#include <machine/pcb.h> 93#include <machine/cputypes.h> 94#include <machine/md_var.h> 95 96#define PMAP_KEEP_PDIRS 97 98#if defined(DIAGNOSTIC) 99#define PMAP_DIAGNOSTIC 100#endif 101 102static void init_pv_entries __P((int)); 103 104/* 105 * Get PDEs and PTEs for user/kernel address space 106 */ 107#define pmap_pde(m, v) (&((m)->pm_pdir[(vm_offset_t)(v) >> PDRSHIFT])) 108#define pdir_pde(m, v) (m[(vm_offset_t)(v) >> PDRSHIFT]) 109 110#define pmap_pde_v(pte) ((*(int *)pte & PG_V) != 0) 111#define pmap_pte_w(pte) ((*(int *)pte & PG_W) != 0) 112#define pmap_pte_m(pte) ((*(int *)pte & PG_M) != 0) 113#define pmap_pte_u(pte) ((*(int *)pte & PG_A) != 0) 114#define pmap_pte_v(pte) ((*(int *)pte & PG_V) != 0) 115 116#define pmap_pte_set_w(pte, v) ((v)?(*(int *)pte |= PG_W):(*(int *)pte &= ~PG_W)) 117#define pmap_pte_set_prot(pte, v) ((*(int *)pte &= ~PG_PROT), (*(int *)pte |= (v))) 118 119/* 120 * Given a map and a machine independent protection code, 121 * convert to a vax protection code. 122 */ 123#define pte_prot(m, p) (protection_codes[p]) 124static int protection_codes[8]; 125 126static struct pmap kernel_pmap_store; 127pmap_t kernel_pmap; 128 129vm_offset_t avail_start; /* PA of first available physical page */ 130vm_offset_t avail_end; /* PA of last available physical page */ 131vm_offset_t virtual_avail; /* VA of first avail page (after kernel bss) */ 132vm_offset_t virtual_end; /* VA of last avail page (end of kernel AS) */ 133static boolean_t pmap_initialized = FALSE; /* Has pmap_init completed? */ 134static vm_offset_t vm_first_phys; 135 136static int nkpt; 137static vm_page_t nkpg; 138vm_offset_t kernel_vm_end; 139 140extern vm_offset_t clean_sva, clean_eva; 141extern int cpu_class; 142 143#define PV_FREELIST_MIN ((PAGE_SIZE / sizeof (struct pv_entry)) / 2) 144 145/* 146 * Data for the pv entry allocation mechanism 147 */ 148static int pv_freelistcnt; 149static pv_entry_t pv_freelist; 150static vm_offset_t pvva; 151static int npvvapg; 152 153/* 154 * All those kernel PT submaps that BSD is so fond of 155 */ 156pt_entry_t *CMAP1; 157static pt_entry_t *CMAP2, *ptmmap; 158static pv_entry_t *pv_table; 159caddr_t CADDR1, ptvmmap; 160static caddr_t CADDR2; 161static pt_entry_t *msgbufmap; 162struct msgbuf *msgbufp; 163 164pt_entry_t *PMAP1; 165unsigned *PADDR1; 166 167static void free_pv_entry __P((pv_entry_t pv)); 168static __inline unsigned * get_ptbase __P((pmap_t pmap)); 169static pv_entry_t get_pv_entry __P((void)); 170static void i386_protection_init __P((void)); 171static void pmap_alloc_pv_entry __P((void)); 172static void pmap_changebit __P((vm_offset_t pa, int bit, boolean_t setem)); 173 174static int pmap_is_managed __P((vm_offset_t pa)); 175static void pmap_remove_all __P((vm_offset_t pa)); 176static void pmap_enter_quick __P((pmap_t pmap, vm_offset_t va, 177 vm_offset_t pa)); 178static int pmap_remove_pte __P((struct pmap *pmap, unsigned *ptq, 179 vm_offset_t sva)); 180static void pmap_remove_page __P((struct pmap *pmap, vm_offset_t va)); 181static __inline int pmap_remove_entry __P((struct pmap *pmap, pv_entry_t *pv, 182 vm_offset_t va)); 183static boolean_t pmap_testbit __P((vm_offset_t pa, int bit)); 184static __inline void pmap_insert_entry __P((pmap_t pmap, vm_offset_t va, 185 vm_page_t mpte, vm_offset_t pa)); 186 187static __inline vm_page_t pmap_allocpte __P((pmap_t pmap, vm_offset_t va)); 188 189static __inline int pmap_release_free_page __P((pmap_t pmap, vm_page_t p)); 190static vm_page_t _pmap_allocpte __P((pmap_t pmap, int ptepindex)); 191 192#define VATRACK 4 193#define PDSTACKMAX 16 194static vm_offset_t pdstack[PDSTACKMAX]; 195static int pdstackptr; 196 197/* 198 * Bootstrap the system enough to run with virtual memory. 199 * 200 * On the i386 this is called after mapping has already been enabled 201 * and just syncs the pmap module with what has already been done. 202 * [We can't call it easily with mapping off since the kernel is not 203 * mapped with PA == VA, hence we would have to relocate every address 204 * from the linked base (virtual) address "KERNBASE" to the actual 205 * (physical) address starting relative to 0] 206 */ 207void 208pmap_bootstrap(firstaddr, loadaddr) 209 vm_offset_t firstaddr; 210 vm_offset_t loadaddr; 211{ 212 vm_offset_t va; 213 pt_entry_t *pte; 214 215 avail_start = firstaddr; 216 217 /* 218 * XXX The calculation of virtual_avail is wrong. It's NKPT*PAGE_SIZE too 219 * large. It should instead be correctly calculated in locore.s and 220 * not based on 'first' (which is a physical address, not a virtual 221 * address, for the start of unused physical memory). The kernel 222 * page tables are NOT double mapped and thus should not be included 223 * in this calculation. 224 */ 225 virtual_avail = (vm_offset_t) KERNBASE + firstaddr; 226 virtual_end = VM_MAX_KERNEL_ADDRESS; 227 228 /* 229 * Initialize protection array. 230 */ 231 i386_protection_init(); 232 233 /* 234 * The kernel's pmap is statically allocated so we don't have to use 235 * pmap_create, which is unlikely to work correctly at this part of 236 * the boot sequence (XXX and which no longer exists). 237 */ 238 kernel_pmap = &kernel_pmap_store; 239 240 kernel_pmap->pm_pdir = (pd_entry_t *) (KERNBASE + IdlePTD); 241 242 kernel_pmap->pm_count = 1; 243 nkpt = NKPT; 244 245 /* 246 * Reserve some special page table entries/VA space for temporary 247 * mapping of pages. 248 */ 249#define SYSMAP(c, p, v, n) \ 250 v = (c)va; va += ((n)*PAGE_SIZE); p = pte; pte += (n); 251 252 va = virtual_avail; 253 pte = (pt_entry_t *) pmap_pte(kernel_pmap, va); 254 255 /* 256 * CMAP1/CMAP2 are used for zeroing and copying pages. 257 */ 258 SYSMAP(caddr_t, CMAP1, CADDR1, 1) 259 SYSMAP(caddr_t, CMAP2, CADDR2, 1) 260 261 /* 262 * ptmmap is used for reading arbitrary physical pages via /dev/mem. 263 */ 264 SYSMAP(caddr_t, ptmmap, ptvmmap, 1) 265 266 /* 267 * msgbufmap is used to map the system message buffer. 268 */ 269 SYSMAP(struct msgbuf *, msgbufmap, msgbufp, 1) 270 271 /* 272 * ptemap is used for pmap_pte_quick 273 */ 274 SYSMAP(unsigned *, PMAP1, PADDR1, 1); 275 276 virtual_avail = va; 277 278 *(int *) CMAP1 = *(int *) CMAP2 = *(int *) PTD = 0; 279 pmap_update(); 280 281} 282 283/* 284 * Initialize the pmap module. 285 * Called by vm_init, to initialize any structures that the pmap 286 * system needs to map virtual memory. 287 * pmap_init has been enhanced to support in a fairly consistant 288 * way, discontiguous physical memory. 289 */ 290void 291pmap_init(phys_start, phys_end) 292 vm_offset_t phys_start, phys_end; 293{ 294 vm_offset_t addr; 295 vm_size_t npg, s; 296 int i; 297 298 /* 299 * calculate the number of pv_entries needed 300 */ 301 vm_first_phys = phys_avail[0]; 302 for (i = 0; phys_avail[i + 1]; i += 2); 303 npg = (phys_avail[(i - 2) + 1] - vm_first_phys) / PAGE_SIZE; 304 305 /* 306 * Allocate memory for random pmap data structures. Includes the 307 * pv_head_table. 308 */ 309 s = (vm_size_t) (sizeof(struct pv_entry *) * npg); 310 s = round_page(s); 311 addr = (vm_offset_t) kmem_alloc(kernel_map, s); 312 pv_table = (pv_entry_t *) addr; 313 314 /* 315 * init the pv free list 316 */ 317 init_pv_entries(npg); 318 /* 319 * Now it is safe to enable pv_table recording. 320 */ 321 pmap_initialized = TRUE; 322} 323 324/* 325 * Used to map a range of physical addresses into kernel 326 * virtual address space. 327 * 328 * For now, VM is already on, we only need to map the 329 * specified memory. 330 */ 331vm_offset_t 332pmap_map(virt, start, end, prot) 333 vm_offset_t virt; 334 vm_offset_t start; 335 vm_offset_t end; 336 int prot; 337{ 338 while (start < end) { 339 pmap_enter(kernel_pmap, virt, start, prot, FALSE); 340 virt += PAGE_SIZE; 341 start += PAGE_SIZE; 342 } 343 return (virt); 344} 345 346 347/*************************************************** 348 * Low level helper routines..... 349 ***************************************************/ 350 351#if defined(PMAP_DIAGNOSTIC) 352 353/* 354 * This code checks for non-writeable/modified pages. 355 * This should be an invalid condition. 356 */ 357static int 358pmap_nw_modified(pt_entry_t ptea) { 359 int pte; 360 361 pte = (int) ptea; 362 363 if ((pte & (PG_M|PG_RW)) == PG_M) 364 return 1; 365 else 366 return 0; 367} 368#endif 369 370 371/* 372 * this routine defines the region(s) of memory that should 373 * not be tested for the modified bit. 374 */ 375static __inline int 376pmap_track_modified( vm_offset_t va) { 377 if ((va < clean_sva) || (va >= clean_eva)) 378 return 1; 379 else 380 return 0; 381} 382 383/* 384 * The below are finer grained pmap_update routines. These eliminate 385 * the gratuitious tlb flushes on non-i386 architectures. 386 */ 387static __inline void 388pmap_update_1pg( vm_offset_t va) { 389#if defined(I386_CPU) 390 if (cpu_class == CPUCLASS_386) 391 pmap_update(); 392 else 393#endif 394 __asm __volatile(".byte 0xf,0x1,0x38": :"a" (va)); 395} 396 397static __inline void 398pmap_update_2pg( vm_offset_t va1, vm_offset_t va2) { 399#if defined(I386_CPU) 400 if (cpu_class == CPUCLASS_386) { 401 pmap_update(); 402 } else 403#endif 404 { 405 __asm __volatile(".byte 0xf,0x1,0x38": :"a" (va1)); 406 __asm __volatile(".byte 0xf,0x1,0x38": :"a" (va2)); 407 } 408} 409 410static __pure unsigned * 411get_ptbase(pmap) 412 pmap_t pmap; 413{ 414 unsigned frame = (unsigned) pmap->pm_pdir[PTDPTDI] & PG_FRAME; 415 416 /* are we current address space or kernel? */ 417 if (pmap == kernel_pmap || frame == (((unsigned) PTDpde) & PG_FRAME)) { 418 return (unsigned *) PTmap; 419 } 420 /* otherwise, we are alternate address space */ 421 if (frame != (((unsigned) APTDpde) & PG_FRAME)) { 422 APTDpde = (pd_entry_t) (frame | PG_RW | PG_V); 423 pmap_update(); 424 } 425 return (unsigned *) APTmap; 426} 427 428/* 429 * Routine: pmap_pte 430 * Function: 431 * Extract the page table entry associated 432 * with the given map/virtual_address pair. 433 */ 434 435__inline unsigned * __pure 436pmap_pte(pmap, va) 437 register pmap_t pmap; 438 vm_offset_t va; 439{ 440 if (pmap && *pmap_pde(pmap, va)) { 441 return get_ptbase(pmap) + i386_btop(va); 442 } 443 return (0); 444} 445 446/* 447 * Super fast pmap_pte routine best used when scanning 448 * the pv lists. This eliminates many coarse-grained 449 * pmap_update calls. 450 */ 451__inline unsigned * __pure 452pmap_pte_quick(pmap, va) 453 register pmap_t pmap; 454 vm_offset_t va; 455{ 456 unsigned pde; 457 if (pde = (unsigned) pmap->pm_pdir[va >> PDRSHIFT]) { 458 unsigned frame = (unsigned) pmap->pm_pdir[PTDPTDI] & PG_FRAME; 459 /* are we current address space or kernel? */ 460 if (pmap == kernel_pmap || frame == (((unsigned) PTDpde) & PG_FRAME)) { 461 return (unsigned *) PTmap + i386_btop(va); 462 } 463 * (int *) PMAP1 = (pde & PG_FRAME) | PG_V | PG_RW; 464 pmap_update_1pg((vm_offset_t) PADDR1); 465 return PADDR1 + ((unsigned) i386_btop(va) & (NPTEPG - 1)); 466 } 467 return (0); 468} 469 470 471/* 472 * Routine: pmap_extract 473 * Function: 474 * Extract the physical page address associated 475 * with the given map/virtual_address pair. 476 */ 477vm_offset_t __pure 478pmap_extract(pmap, va) 479 register pmap_t pmap; 480 vm_offset_t va; 481{ 482 if (pmap && *pmap_pde(pmap, va)) { 483 unsigned *pte; 484 pte = get_ptbase(pmap) + i386_btop(va); 485 return ((*pte & PG_FRAME) | (va & PAGE_MASK)); 486 } 487 return 0; 488 489} 490 491/* 492 * determine if a page is managed (memory vs. device) 493 */ 494static __inline __pure int 495pmap_is_managed(pa) 496 vm_offset_t pa; 497{ 498 int i; 499 500 if (!pmap_initialized) 501 return 0; 502 503 for (i = 0; phys_avail[i + 1]; i += 2) { 504 if (pa < phys_avail[i + 1] && pa >= phys_avail[i]) 505 return 1; 506 } 507 return 0; 508} 509 510/*************************************************** 511 * Low level mapping routines..... 512 ***************************************************/ 513 514/* 515 * Add a list of wired pages to the kva 516 * this routine is only used for temporary 517 * kernel mappings that do not need to have 518 * page modification or references recorded. 519 * Note that old mappings are simply written 520 * over. The page *must* be wired. 521 */ 522void 523pmap_qenter(va, m, count) 524 vm_offset_t va; 525 vm_page_t *m; 526 int count; 527{ 528 int i; 529 register unsigned *pte; 530 531 for (i = 0; i < count; i++) { 532 vm_offset_t tva = va + i * PAGE_SIZE; 533 unsigned npte = VM_PAGE_TO_PHYS(m[i]) | PG_RW | PG_V; 534 unsigned opte; 535 pte = (unsigned *)vtopte(tva); 536 opte = *pte; 537 *pte = npte; 538 if (opte) 539 pmap_update_1pg(tva); 540 } 541} 542/* 543 * this routine jerks page mappings from the 544 * kernel -- it is meant only for temporary mappings. 545 */ 546void 547pmap_qremove(va, count) 548 vm_offset_t va; 549 int count; 550{ 551 int i; 552 register unsigned *pte; 553 554 for (i = 0; i < count; i++) { 555 pte = (unsigned *)vtopte(va); 556 *pte = 0; 557 pmap_update_1pg(va); 558 va += PAGE_SIZE; 559 } 560} 561 562/* 563 * add a wired page to the kva 564 * note that in order for the mapping to take effect -- you 565 * should do a pmap_update after doing the pmap_kenter... 566 */ 567__inline void 568pmap_kenter(va, pa) 569 vm_offset_t va; 570 register vm_offset_t pa; 571{ 572 register unsigned *pte; 573 unsigned npte, opte; 574 575 npte = pa | PG_RW | PG_V; 576 pte = (unsigned *)vtopte(va); 577 opte = *pte; 578 *pte = npte; 579 if (opte) 580 pmap_update_1pg(va); 581} 582 583/* 584 * remove a page from the kernel pagetables 585 */ 586__inline void 587pmap_kremove(va) 588 vm_offset_t va; 589{ 590 register unsigned *pte; 591 592 pte = (unsigned *)vtopte(va); 593 *pte = 0; 594 pmap_update_1pg(va); 595} 596 597 598/*************************************************** 599 * Page table page management routines..... 600 ***************************************************/ 601 602/* 603 * This routine unholds page table pages, and if the hold count 604 * drops to zero, then it decrements the wire count. 605 */ 606static __inline int 607pmap_unwire_pte_hold(pmap_t pmap, vm_page_t m) { 608 vm_page_unhold(m); 609 if (m->hold_count == 0) { 610 vm_offset_t pteva; 611 /* 612 * unmap the page table page 613 */ 614 pmap->pm_pdir[m->pindex] = 0; 615 /* 616 * Do a pmap_update to make the invalidated mapping 617 * take effect immediately. 618 */ 619 pteva = UPT_MIN_ADDRESS + i386_ptob(m->pindex); 620 pmap_update_1pg(pteva); 621 /* 622 * If the page is finally unwired, simply free it. 623 */ 624 --m->wire_count; 625 if (m->wire_count == 0) { 626 vm_page_free_zero(m); 627 --cnt.v_wire_count; 628 } 629 return 1; 630 } 631 return 0; 632} 633 634/* 635 * After removing a page table entry, this routine is used to 636 * conditionally free the page, and manage the hold/wire counts. 637 */ 638int 639pmap_unuse_pt(pmap, va, mpte) 640 pmap_t pmap; 641 vm_offset_t va; 642 vm_page_t mpte; 643{ 644 if (va >= UPT_MIN_ADDRESS) 645 return 0; 646 647 if (mpte == NULL) { 648 vm_offset_t ptepa; 649 ptepa = ((vm_offset_t) *pmap_pde(pmap, va)); 650#if defined(PMAP_DIAGNOSTIC) 651 if (!ptepa) 652 panic("pmap_unuse_pt: pagetable page missing, va: 0x%x", va); 653#endif 654 if (!ptepa) 655 return 0; 656 mpte = PHYS_TO_VM_PAGE(ptepa); 657 } 658 659#if defined(PMAP_DIAGNOSTIC) 660 if (mpte->pindex != (va >> PDRSHIFT)) 661 panic("pmap_unuse_pt: pindex(0x%x) != va(0x%x)", 662 mpte->pindex, (va >> PDRSHIFT)); 663 664 if (mpte->hold_count == 0) { 665 panic("pmap_unuse_pt: hold count < 0, va: 0x%x", va); 666 } 667#endif 668 669 return pmap_unwire_pte_hold(pmap, mpte); 670} 671 672/* 673 * Initialize a preallocated and zeroed pmap structure, 674 * such as one in a vmspace structure. 675 */ 676void 677pmap_pinit(pmap) 678 register struct pmap *pmap; 679{ 680 vm_page_t ptdpg; 681 /* 682 * No need to allocate page table space yet but we do need a valid 683 * page directory table. 684 */ 685 686 if (pdstackptr > 0) { 687 --pdstackptr; 688 pmap->pm_pdir = (pd_entry_t *)pdstack[pdstackptr]; 689 } else { 690 pmap->pm_pdir = 691 (pd_entry_t *)kmem_alloc_pageable(kernel_map, PAGE_SIZE); 692 } 693 694 /* 695 * allocate object for the ptes 696 */ 697 pmap->pm_pteobj = vm_object_allocate( OBJT_DEFAULT, PTDPTDI + 1); 698 699 /* 700 * allocate the page directory page 701 */ 702retry: 703 ptdpg = vm_page_alloc( pmap->pm_pteobj, PTDPTDI, VM_ALLOC_ZERO); 704 if (ptdpg == NULL) { 705 VM_WAIT; 706 goto retry; 707 } 708 vm_page_wire(ptdpg); 709 ptdpg->flags &= ~(PG_MAPPED|PG_BUSY); /* not mapped normally */ 710 ptdpg->valid = VM_PAGE_BITS_ALL; 711 712 pmap_kenter((vm_offset_t) pmap->pm_pdir, VM_PAGE_TO_PHYS(ptdpg)); 713 if ((ptdpg->flags & PG_ZERO) == 0) 714 bzero(pmap->pm_pdir, PAGE_SIZE); 715 716 /* wire in kernel global address entries */ 717 bcopy(PTD + KPTDI, pmap->pm_pdir + KPTDI, nkpt * PTESIZE); 718 719 /* install self-referential address mapping entry */ 720 *(unsigned *) (pmap->pm_pdir + PTDPTDI) = 721 VM_PAGE_TO_PHYS(ptdpg) | PG_V | PG_RW; 722 723 pmap->pm_count = 1; 724} 725 726static int 727pmap_release_free_page(pmap, p) 728 struct pmap *pmap; 729 vm_page_t p; 730{ 731 int s; 732 unsigned *pde = (unsigned *) pmap->pm_pdir; 733 /* 734 * This code optimizes the case of freeing non-busy 735 * page-table pages. Those pages are zero now, and 736 * might as well be placed directly into the zero queue. 737 */ 738 s = splvm(); 739 if (p->flags & PG_BUSY) { 740 p->flags |= PG_WANTED; 741 tsleep(p, PVM, "pmaprl", 0); 742 splx(s); 743 return 0; 744 } 745 746 /* 747 * Remove the page table page from the processes address space. 748 */ 749 pde[p->pindex] = 0; 750 751 if (p->hold_count) { 752 int *kvap; 753 int i; 754#if defined(PMAP_DIAGNOSTIC) 755 panic("pmap_release: freeing held page table page"); 756#else 757 printf("pmap_release: freeing held page table page:\n"); 758#endif 759 kvap = (int *)vm_pager_map_page(p); 760 for(i=0;i<NPTEPG;i++) { 761 if (kvap[i]) { 762 printf("pte: 0x%x, index: %d\n", kvap[i],i); 763 } 764 } 765 vm_pager_unmap_page((vm_offset_t)kvap); 766 767 /* 768 * HACK ALERT!!! 769 * If this failure happens, we must clear the page, because 770 * there is likely a mapping still valid. This condition 771 * is an error, but at least this zero operation will mitigate 772 * some Sig-11's or crashes, because this page is thought 773 * to be zero. This is a robustness fix, and not meant to 774 * be a long term work-around. 775 */ 776 pmap_zero_page(VM_PAGE_TO_PHYS(p)); 777 } 778 /* 779 * Page directory pages need to have the kernel 780 * stuff cleared, so they can go into the zero queue also. 781 */ 782 if (p->pindex == PTDPTDI) { 783 bzero(pde + KPTDI, nkpt * PTESIZE); 784 pde[APTDPTDI] = 0; 785 pmap_kremove((vm_offset_t) pmap->pm_pdir); 786 } 787 788 vm_page_free_zero(p); 789 splx(s); 790 return 1; 791} 792 793/* 794 * this routine is called if the page table page is not 795 * mapped correctly. 796 */ 797static vm_page_t 798_pmap_allocpte(pmap, ptepindex) 799 pmap_t pmap; 800 int ptepindex; 801{ 802 vm_offset_t pteva, ptepa; 803 vm_page_t m; 804 805 /* 806 * Find or fabricate a new pagetable page 807 */ 808retry: 809 m = vm_page_lookup(pmap->pm_pteobj, ptepindex); 810 if (m == NULL) { 811 m = vm_page_alloc(pmap->pm_pteobj, ptepindex, VM_ALLOC_ZERO); 812 if (m == NULL) { 813 VM_WAIT; 814 goto retry; 815 } 816 if ((m->flags & PG_ZERO) == 0) 817 pmap_zero_page(VM_PAGE_TO_PHYS(m)); 818 m->flags &= ~(PG_ZERO|PG_BUSY); 819 m->valid = VM_PAGE_BITS_ALL; 820 } else { 821 if ((m->flags & PG_BUSY) || m->busy) { 822 m->flags |= PG_WANTED; 823 tsleep(m, PVM, "ptewai", 0); 824 goto retry; 825 } 826 } 827 828 /* 829 * mark the object writeable 830 */ 831 pmap->pm_pteobj->flags |= OBJ_WRITEABLE; 832 833 if (m->queue != PQ_NONE) { 834 int s = splvm(); 835 vm_page_unqueue(m); 836 splx(s); 837 } 838 839 if (m->hold_count == 0) { 840 if (m->wire_count == 0) 841 ++cnt.v_wire_count; 842 ++m->wire_count; 843 } 844 /* 845 * Increment the hold count for the page table page 846 * (denoting a new mapping.) 847 */ 848 ++m->hold_count; 849 850 /* 851 * Map the pagetable page into the process address space, if 852 * it isn't already there. 853 */ 854 855 pmap->pm_stats.resident_count++; 856 857 ptepa = VM_PAGE_TO_PHYS(m); 858 pmap->pm_pdir[ptepindex] = (pd_entry_t) (ptepa | PG_U | PG_RW | PG_V); 859 860 pteva = UPT_MIN_ADDRESS + i386_ptob(ptepindex); 861 pmap_update_1pg(pteva); 862 m->flags |= PG_MAPPED; 863 864 return m; 865} 866 867static __inline vm_page_t 868pmap_allocpte(pmap, va) 869 pmap_t pmap; 870 vm_offset_t va; 871{ 872 int ptepindex; 873 vm_offset_t ptepa; 874 vm_page_t m; 875 876 /* 877 * Calculate pagetable page index 878 */ 879 ptepindex = va >> PDRSHIFT; 880 881 /* 882 * Get the page directory entry 883 */ 884 ptepa = (vm_offset_t) pmap->pm_pdir[ptepindex]; 885 886 /* 887 * If the page table page is mapped, we just increment the 888 * hold count, and activate it. 889 */ 890 if (ptepa) { 891 m = PHYS_TO_VM_PAGE(ptepa); 892 ++m->hold_count; 893 return m; 894 } 895 /* 896 * Here if the pte page isn't mapped, or if it has been deallocated. 897 */ 898 return _pmap_allocpte(pmap, ptepindex); 899} 900 901 902/*************************************************** 903* Pmap allocation/deallocation routines. 904 ***************************************************/ 905 906/* 907 * Release any resources held by the given physical map. 908 * Called when a pmap initialized by pmap_pinit is being released. 909 * Should only be called if the map contains no valid mappings. 910 */ 911void 912pmap_release(pmap) 913 register struct pmap *pmap; 914{ 915 vm_page_t p,n,ptdpg; 916 vm_object_t object = pmap->pm_pteobj; 917 918 if (object->ref_count != 1) 919 panic("pmap_release: pteobj reference count != 1"); 920 921 ptdpg = NULL; 922retry: 923 for (p = TAILQ_FIRST(&object->memq); p != NULL; p = n) { 924 n = TAILQ_NEXT(p, listq); 925 if (p->pindex == PTDPTDI) { 926 ptdpg = p; 927 continue; 928 } 929 if (!pmap_release_free_page(pmap, p)) 930 goto retry; 931 } 932 if (ptdpg == NULL) 933 panic("pmap_release: missing page table directory page"); 934 935 if (!pmap_release_free_page(pmap, ptdpg)) 936 goto retry; 937 938 vm_object_deallocate(object); 939 if (pdstackptr < PDSTACKMAX) { 940 pdstack[pdstackptr] = (vm_offset_t) pmap->pm_pdir; 941 ++pdstackptr; 942 } else { 943 kmem_free(kernel_map, (vm_offset_t) pmap->pm_pdir, PAGE_SIZE); 944 } 945} 946 947/* 948 * grow the number of kernel page table entries, if needed 949 */ 950void 951pmap_growkernel(vm_offset_t addr) 952{ 953 struct proc *p; 954 struct pmap *pmap; 955 int s; 956 957 s = splhigh(); 958 if (kernel_vm_end == 0) { 959 kernel_vm_end = KERNBASE; 960 nkpt = 0; 961 while (pdir_pde(PTD, kernel_vm_end)) { 962 kernel_vm_end = (kernel_vm_end + PAGE_SIZE * NPTEPG) & ~(PAGE_SIZE * NPTEPG - 1); 963 ++nkpt; 964 } 965 } 966 addr = (addr + PAGE_SIZE * NPTEPG) & ~(PAGE_SIZE * NPTEPG - 1); 967 while (kernel_vm_end < addr) { 968 if (pdir_pde(PTD, kernel_vm_end)) { 969 kernel_vm_end = (kernel_vm_end + PAGE_SIZE * NPTEPG) & ~(PAGE_SIZE * NPTEPG - 1); 970 continue; 971 } 972 ++nkpt; 973 if (!nkpg) { 974 nkpg = vm_page_alloc(kernel_object, 0, VM_ALLOC_SYSTEM); 975 if (!nkpg) 976 panic("pmap_growkernel: no memory to grow kernel"); 977 vm_page_wire(nkpg); 978 vm_page_remove(nkpg); 979 pmap_zero_page(VM_PAGE_TO_PHYS(nkpg)); 980 } 981 pdir_pde(PTD, kernel_vm_end) = (pd_entry_t) (VM_PAGE_TO_PHYS(nkpg) | PG_V | PG_RW); 982 nkpg = NULL; 983 984 for (p = allproc.lh_first; p != 0; p = p->p_list.le_next) { 985 if (p->p_vmspace) { 986 pmap = &p->p_vmspace->vm_pmap; 987 *pmap_pde(pmap, kernel_vm_end) = pdir_pde(PTD, kernel_vm_end); 988 } 989 } 990 *pmap_pde(kernel_pmap, kernel_vm_end) = pdir_pde(PTD, kernel_vm_end); 991 kernel_vm_end = (kernel_vm_end + PAGE_SIZE * NPTEPG) & ~(PAGE_SIZE * NPTEPG - 1); 992 } 993 splx(s); 994} 995 996/* 997 * Retire the given physical map from service. 998 * Should only be called if the map contains 999 * no valid mappings. 1000 */ 1001void 1002pmap_destroy(pmap) 1003 register pmap_t pmap; 1004{ 1005 int count; 1006 1007 if (pmap == NULL) 1008 return; 1009 1010 count = --pmap->pm_count; 1011 if (count == 0) { 1012 pmap_release(pmap); 1013 free((caddr_t) pmap, M_VMPMAP); 1014 } 1015} 1016 1017/* 1018 * Add a reference to the specified pmap. 1019 */ 1020void 1021pmap_reference(pmap) 1022 pmap_t pmap; 1023{ 1024 if (pmap != NULL) { 1025 pmap->pm_count++; 1026 } 1027} 1028 1029/*************************************************** 1030* page management routines. 1031 ***************************************************/ 1032 1033/* 1034 * free the pv_entry back to the free list 1035 */ 1036static __inline void 1037free_pv_entry(pv) 1038 pv_entry_t pv; 1039{ 1040 ++pv_freelistcnt; 1041 pv->pv_next = pv_freelist; 1042 pv_freelist = pv; 1043} 1044 1045/* 1046 * get a new pv_entry, allocating a block from the system 1047 * when needed. 1048 * the memory allocation is performed bypassing the malloc code 1049 * because of the possibility of allocations at interrupt time. 1050 */ 1051static __inline pv_entry_t 1052get_pv_entry() 1053{ 1054 pv_entry_t tmp; 1055 1056 /* 1057 * get more pv_entry pages if needed 1058 */ 1059 if (pv_freelistcnt < PV_FREELIST_MIN || pv_freelist == 0) { 1060 pmap_alloc_pv_entry(); 1061 } 1062 /* 1063 * get a pv_entry off of the free list 1064 */ 1065 --pv_freelistcnt; 1066 tmp = pv_freelist; 1067 pv_freelist = tmp->pv_next; 1068 return tmp; 1069} 1070 1071/* 1072 * This *strange* allocation routine eliminates the possibility of a malloc 1073 * failure (*FATAL*) for a pv_entry_t data structure. 1074 * also -- this code is MUCH MUCH faster than the malloc equiv... 1075 * We really need to do the slab allocator thingie here. 1076 */ 1077static void 1078pmap_alloc_pv_entry() 1079{ 1080 /* 1081 * do we have any pre-allocated map-pages left? 1082 */ 1083 if (npvvapg) { 1084 vm_page_t m; 1085 1086 /* 1087 * allocate a physical page out of the vm system 1088 */ 1089 m = vm_page_alloc(kernel_object, 1090 OFF_TO_IDX(pvva - vm_map_min(kernel_map)), 1091 VM_ALLOC_INTERRUPT); 1092 if (m) { 1093 int newentries; 1094 int i; 1095 pv_entry_t entry; 1096 1097 newentries = (PAGE_SIZE / sizeof(struct pv_entry)); 1098 /* 1099 * wire the page 1100 */ 1101 vm_page_wire(m); 1102 m->flags &= ~PG_BUSY; 1103 /* 1104 * let the kernel see it 1105 */ 1106 pmap_kenter(pvva, VM_PAGE_TO_PHYS(m)); 1107 1108 entry = (pv_entry_t) pvva; 1109 /* 1110 * update the allocation pointers 1111 */ 1112 pvva += PAGE_SIZE; 1113 --npvvapg; 1114 1115 /* 1116 * free the entries into the free list 1117 */ 1118 for (i = 0; i < newentries; i++) { 1119 free_pv_entry(entry); 1120 entry++; 1121 } 1122 } 1123 } 1124 if (!pv_freelist) 1125 panic("get_pv_entry: cannot get a pv_entry_t"); 1126} 1127 1128/* 1129 * init the pv_entry allocation system 1130 */ 1131#define PVSPERPAGE 64 1132void 1133init_pv_entries(npg) 1134 int npg; 1135{ 1136 /* 1137 * allocate enough kvm space for PVSPERPAGE entries per page (lots) 1138 * kvm space is fairly cheap, be generous!!! (the system can panic if 1139 * this is too small.) 1140 */ 1141 npvvapg = ((npg * PVSPERPAGE) * sizeof(struct pv_entry) 1142 + PAGE_SIZE - 1) / PAGE_SIZE; 1143 pvva = kmem_alloc_pageable(kernel_map, npvvapg * PAGE_SIZE); 1144 /* 1145 * get the first batch of entries 1146 */ 1147 pmap_alloc_pv_entry(); 1148} 1149 1150/* 1151 * If it is the first entry on the list, it is actually 1152 * in the header and we must copy the following entry up 1153 * to the header. Otherwise we must search the list for 1154 * the entry. In either case we free the now unused entry. 1155 */ 1156static __inline int 1157pmap_remove_entry(pmap, ppv, va) 1158 struct pmap *pmap; 1159 pv_entry_t *ppv; 1160 vm_offset_t va; 1161{ 1162 pv_entry_t npv; 1163 int s; 1164 1165 s = splvm(); 1166 for (npv = *ppv; npv; (ppv = &npv->pv_next, npv = *ppv)) { 1167 if (pmap == npv->pv_pmap && va == npv->pv_va) { 1168 int rtval = pmap_unuse_pt(pmap, va, npv->pv_ptem); 1169 *ppv = npv->pv_next; 1170 free_pv_entry(npv); 1171 splx(s); 1172 return rtval; 1173 } 1174 } 1175 splx(s); 1176 return 0; 1177} 1178 1179/* 1180 * Create a pv entry for page at pa for 1181 * (pmap, va). 1182 */ 1183static __inline void 1184pmap_insert_entry(pmap, va, mpte, pa) 1185 pmap_t pmap; 1186 vm_offset_t va; 1187 vm_page_t mpte; 1188 vm_offset_t pa; 1189{ 1190 1191 int s; 1192 pv_entry_t *ppv, pv; 1193 1194 s = splvm(); 1195 pv = get_pv_entry(); 1196 pv->pv_va = va; 1197 pv->pv_pmap = pmap; 1198 pv->pv_ptem = mpte; 1199 1200 ppv = pa_to_pvh(pa); 1201 if (*ppv) 1202 pv->pv_next = *ppv; 1203 else 1204 pv->pv_next = NULL; 1205 *ppv = pv; 1206 splx(s); 1207} 1208 1209/* 1210 * pmap_remove_pte: do the things to unmap a page in a process 1211 */ 1212static int 1213pmap_remove_pte(pmap, ptq, va) 1214 struct pmap *pmap; 1215 unsigned *ptq; 1216 vm_offset_t va; 1217{ 1218 unsigned oldpte; 1219 pv_entry_t *ppv; 1220 1221 oldpte = *ptq; 1222 *ptq = 0; 1223 if (oldpte & PG_W) 1224 pmap->pm_stats.wired_count -= 1; 1225 pmap->pm_stats.resident_count -= 1; 1226 if (oldpte & PG_MANAGED) { 1227 if (oldpte & PG_M) { 1228#if defined(PMAP_DIAGNOSTIC) 1229 if (pmap_nw_modified((pt_entry_t) oldpte)) { 1230 printf("pmap_remove: modified page not writable: va: 0x%lx, pte: 0x%lx\n", va, (int) oldpte); 1231 } 1232#endif 1233 if (pmap_track_modified(va)) 1234 PHYS_TO_VM_PAGE(oldpte)->dirty = VM_PAGE_BITS_ALL; 1235 } 1236 ppv = pa_to_pvh(oldpte); 1237 return pmap_remove_entry(pmap, ppv, va); 1238 } else { 1239 return pmap_unuse_pt(pmap, va, NULL); 1240 } 1241 1242 return 0; 1243} 1244 1245/* 1246 * Remove a single page from a process address space 1247 */ 1248static void 1249pmap_remove_page(pmap, va) 1250 struct pmap *pmap; 1251 register vm_offset_t va; 1252{ 1253 register unsigned *ptq; 1254 1255 /* 1256 * if there is no pte for this address, just skip it!!! 1257 */ 1258 if (*pmap_pde(pmap, va) == 0) { 1259 return; 1260 } 1261 1262 /* 1263 * get a local va for mappings for this pmap. 1264 */ 1265 ptq = get_ptbase(pmap) + i386_btop(va); 1266 if (*ptq) { 1267 (void) pmap_remove_pte(pmap, ptq, va); 1268 pmap_update_1pg(va); 1269 } 1270 return; 1271} 1272 1273/* 1274 * Remove the given range of addresses from the specified map. 1275 * 1276 * It is assumed that the start and end are properly 1277 * rounded to the page size. 1278 */ 1279void 1280pmap_remove(pmap, sva, eva) 1281 struct pmap *pmap; 1282 register vm_offset_t sva; 1283 register vm_offset_t eva; 1284{ 1285 register unsigned *ptbase; 1286 vm_offset_t pdnxt; 1287 vm_offset_t ptpaddr; 1288 vm_offset_t sindex, eindex; 1289 vm_page_t mpte; 1290 int anyvalid; 1291 vm_offset_t vachanged[VATRACK]; 1292 1293 if (pmap == NULL) 1294 return; 1295 1296 /* 1297 * special handling of removing one page. a very 1298 * common operation and easy to short circuit some 1299 * code. 1300 */ 1301 if ((sva + PAGE_SIZE) == eva) { 1302 pmap_remove_page(pmap, sva); 1303 return; 1304 } 1305 1306 anyvalid = 0; 1307 1308 /* 1309 * Get a local virtual address for the mappings that are being 1310 * worked with. 1311 */ 1312 ptbase = get_ptbase(pmap); 1313 1314 sindex = i386_btop(sva); 1315 eindex = i386_btop(eva); 1316 1317 for (; sindex < eindex; sindex = pdnxt) { 1318 1319 /* 1320 * Calculate index for next page table. 1321 */ 1322 pdnxt = ((sindex + NPTEPG) & ~(NPTEPG - 1)); 1323 ptpaddr = (vm_offset_t) *pmap_pde(pmap, i386_ptob(sindex)); 1324 1325 /* 1326 * Weed out invalid mappings. Note: we assume that the page 1327 * directory table is always allocated, and in kernel virtual. 1328 */ 1329 if (ptpaddr == 0) 1330 continue; 1331 1332 if (sindex < i386_btop(UPT_MIN_ADDRESS)) { 1333 /* 1334 * get the vm_page_t for the page table page 1335 */ 1336 mpte = PHYS_TO_VM_PAGE(ptpaddr); 1337 1338 /* 1339 * if the pte isn't wired, just skip it. 1340 */ 1341 if (mpte->wire_count == 0) 1342 continue; 1343 } 1344 1345 /* 1346 * Limit our scan to either the end of the va represented 1347 * by the current page table page, or to the end of the 1348 * range being removed. 1349 */ 1350 if (pdnxt > eindex) { 1351 pdnxt = eindex; 1352 } 1353 1354 for ( ;sindex != pdnxt; sindex++) { 1355 vm_offset_t va; 1356 if (ptbase[sindex] == 0) { 1357 continue; 1358 } 1359 va = i386_ptob(sindex); 1360 1361 if (anyvalid < VATRACK) 1362 vachanged[anyvalid] = va; 1363 anyvalid++; 1364 if (pmap_remove_pte(pmap, 1365 ptbase + sindex, va)) 1366 break; 1367 } 1368 } 1369 1370 if (anyvalid) { 1371 if (anyvalid <= VATRACK) { 1372 int i; 1373 for(i=0;i<anyvalid;i++) 1374 pmap_update_1pg(vachanged[i]); 1375 } else { 1376 pmap_update(); 1377 } 1378 } 1379 1380} 1381 1382/* 1383 * Routine: pmap_remove_all 1384 * Function: 1385 * Removes this physical page from 1386 * all physical maps in which it resides. 1387 * Reflects back modify bits to the pager. 1388 * 1389 * Notes: 1390 * Original versions of this routine were very 1391 * inefficient because they iteratively called 1392 * pmap_remove (slow...) 1393 */ 1394static void 1395pmap_remove_all(pa) 1396 vm_offset_t pa; 1397{ 1398 register pv_entry_t pv, *ppv, npv; 1399 register unsigned *pte, tpte; 1400 vm_page_t m; 1401 int s; 1402 1403#if defined(PMAP_DIAGNOSTIC) 1404 /* 1405 * XXX this makes pmap_page_protect(NONE) illegal for non-managed 1406 * pages! 1407 */ 1408 if (!pmap_is_managed(pa)) { 1409 panic("pmap_page_protect: illegal for unmanaged page, va: 0x%lx", pa); 1410 } 1411#endif 1412 1413 s = splvm(); 1414 m = NULL; 1415 ppv = pa_to_pvh(pa); 1416 for (pv = *ppv; pv; pv=pv->pv_next) { 1417 pte = pmap_pte_quick(pv->pv_pmap, pv->pv_va); 1418 if (tpte = *pte) { 1419 pv->pv_pmap->pm_stats.resident_count--; 1420 *pte = 0; 1421 if (tpte & PG_W) 1422 pv->pv_pmap->pm_stats.wired_count--; 1423 /* 1424 * Update the vm_page_t clean and reference bits. 1425 */ 1426 if (tpte & PG_M) { 1427#if defined(PMAP_DIAGNOSTIC) 1428 if (pmap_nw_modified((pt_entry_t) tpte)) { 1429 printf("pmap_remove_all: modified page not writable: va: 0x%lx, pte: 0x%lx\n", pv->pv_va, tpte); 1430 } 1431#endif 1432 if (pmap_track_modified(pv->pv_va)) { 1433 if (m == NULL) 1434 m = PHYS_TO_VM_PAGE(pa); 1435 m->dirty = VM_PAGE_BITS_ALL; 1436 } 1437 } 1438 } 1439 } 1440 1441 for (pv = *ppv; pv; pv = npv) { 1442 npv = pv->pv_next; 1443 pmap_unuse_pt(pv->pv_pmap, pv->pv_va, pv->pv_ptem); 1444 free_pv_entry(pv); 1445 } 1446 *ppv = NULL; 1447 splx(s); 1448} 1449 1450/* 1451 * Set the physical protection on the 1452 * specified range of this map as requested. 1453 */ 1454void 1455pmap_protect(pmap, sva, eva, prot) 1456 register pmap_t pmap; 1457 vm_offset_t sva, eva; 1458 vm_prot_t prot; 1459{ 1460 register unsigned *ptbase; 1461 vm_offset_t pdnxt; 1462 vm_offset_t ptpaddr; 1463 vm_offset_t sindex, eindex; 1464 vm_page_t mpte; 1465 int anyvalid; 1466 1467 1468 if (pmap == NULL) 1469 return; 1470 1471 if ((prot & VM_PROT_READ) == VM_PROT_NONE) { 1472 pmap_remove(pmap, sva, eva); 1473 return; 1474 } 1475 if (prot & VM_PROT_WRITE) 1476 return; 1477 1478 anyvalid = 0; 1479 1480 ptbase = get_ptbase(pmap); 1481 1482 sindex = i386_btop(sva); 1483 eindex = i386_btop(eva); 1484 1485 for (; sindex < eindex; sindex = pdnxt) { 1486 1487 pdnxt = ((sindex + NPTEPG) & ~(NPTEPG - 1)); 1488 ptpaddr = (vm_offset_t) *pmap_pde(pmap, i386_ptob(sindex)); 1489 1490 /* 1491 * Weed out invalid mappings. Note: we assume that the page 1492 * directory table is always allocated, and in kernel virtual. 1493 */ 1494 if (ptpaddr == 0) 1495 continue; 1496 1497 /* 1498 * Skip page ranges, where the page table page isn't wired. 1499 * If the page table page is not wired, there are no page mappings 1500 * there. 1501 */ 1502 if (sindex < i386_btop(UPT_MIN_ADDRESS)) { 1503 mpte = PHYS_TO_VM_PAGE(ptpaddr); 1504 1505 if (mpte->wire_count == 0) 1506 continue; 1507 } 1508 1509 if (pdnxt > eindex) { 1510 pdnxt = eindex; 1511 } 1512 1513 for (; sindex != pdnxt; sindex++) { 1514 1515 unsigned pbits = ptbase[sindex]; 1516 1517 if (pbits & PG_RW) { 1518 if (pbits & PG_M) { 1519 vm_offset_t sva = i386_ptob(sindex); 1520 if (pmap_track_modified(sva)) { 1521 vm_page_t m = PHYS_TO_VM_PAGE(pbits); 1522 m->dirty = VM_PAGE_BITS_ALL; 1523 } 1524 } 1525 ptbase[sindex] = pbits & ~(PG_M|PG_RW); 1526 anyvalid = 1; 1527 } 1528 } 1529 } 1530 if (anyvalid) 1531 pmap_update(); 1532} 1533 1534/* 1535 * Insert the given physical page (p) at 1536 * the specified virtual address (v) in the 1537 * target physical map with the protection requested. 1538 * 1539 * If specified, the page will be wired down, meaning 1540 * that the related pte can not be reclaimed. 1541 * 1542 * NB: This is the only routine which MAY NOT lazy-evaluate 1543 * or lose information. That is, this routine must actually 1544 * insert this page into the given map NOW. 1545 */ 1546void 1547pmap_enter(pmap, va, pa, prot, wired) 1548 register pmap_t pmap; 1549 vm_offset_t va; 1550 register vm_offset_t pa; 1551 vm_prot_t prot; 1552 boolean_t wired; 1553{ 1554 register unsigned *pte; 1555 vm_offset_t opa; 1556 vm_offset_t origpte, newpte; 1557 vm_page_t mpte; 1558 1559 if (pmap == NULL) 1560 return; 1561 1562 va &= PG_FRAME; 1563#ifdef PMAP_DIAGNOSTIC 1564 if (va > VM_MAX_KERNEL_ADDRESS) 1565 panic("pmap_enter: toobig"); 1566 if ((va >= UPT_MIN_ADDRESS) && (va < UPT_MAX_ADDRESS)) 1567 panic("pmap_enter: invalid to pmap_enter page table pages (va: 0x%x)", va); 1568#endif 1569 1570 mpte = NULL; 1571 /* 1572 * In the case that a page table page is not 1573 * resident, we are creating it here. 1574 */ 1575 if (va < UPT_MIN_ADDRESS) 1576 mpte = pmap_allocpte(pmap, va); 1577 1578 pte = pmap_pte_quick(pmap, va); 1579 /* 1580 * Page Directory table entry not valid, we need a new PT page 1581 */ 1582 if (pte == NULL) { 1583 panic("pmap_enter: invalid page directory, pdir=%p, va=0x%lx\n", 1584 pmap->pm_pdir[PTDPTDI], va); 1585 } 1586 1587 origpte = *(vm_offset_t *)pte; 1588 pa &= PG_FRAME; 1589 opa = origpte & PG_FRAME; 1590 1591 /* 1592 * Mapping has not changed, must be protection or wiring change. 1593 */ 1594 if (opa == pa) { 1595 /* 1596 * Wiring change, just update stats. We don't worry about 1597 * wiring PT pages as they remain resident as long as there 1598 * are valid mappings in them. Hence, if a user page is wired, 1599 * the PT page will be also. 1600 */ 1601 if (wired && ((origpte & PG_W) == 0)) 1602 pmap->pm_stats.wired_count++; 1603 else if (!wired && (origpte & PG_W)) 1604 pmap->pm_stats.wired_count--; 1605 1606#if defined(PMAP_DIAGNOSTIC) 1607 if (pmap_nw_modified((pt_entry_t) origpte)) { 1608 printf("pmap_enter: modified page not writable: va: 0x%lx, pte: 0x%lx\n", va, origpte); 1609 } 1610#endif 1611 1612 /* 1613 * We might be turning off write access to the page, 1614 * so we go ahead and sense modify status. 1615 */ 1616 if (origpte & PG_MANAGED) { 1617 vm_page_t m; 1618 if (origpte & PG_M) { 1619 if (pmap_track_modified(va)) { 1620 m = PHYS_TO_VM_PAGE(pa); 1621 m->dirty = VM_PAGE_BITS_ALL; 1622 } 1623 } 1624 pa |= PG_MANAGED; 1625 } 1626 1627 if (mpte) 1628 --mpte->hold_count; 1629 1630 goto validate; 1631 } 1632 /* 1633 * Mapping has changed, invalidate old range and fall through to 1634 * handle validating new mapping. 1635 */ 1636 if (opa) { 1637 int err; 1638 err = pmap_remove_pte(pmap, pte, va); 1639 if (err) 1640 panic("pmap_enter: pte vanished, va: 0x%x", va); 1641 } 1642 1643 /* 1644 * Enter on the PV list if part of our managed memory Note that we 1645 * raise IPL while manipulating pv_table since pmap_enter can be 1646 * called at interrupt time. 1647 */ 1648 if (pmap_is_managed(pa)) { 1649 pmap_insert_entry(pmap, va, mpte, pa); 1650 pa |= PG_MANAGED; 1651 } 1652 1653 /* 1654 * Increment counters 1655 */ 1656 pmap->pm_stats.resident_count++; 1657 if (wired) 1658 pmap->pm_stats.wired_count++; 1659 1660validate: 1661 /* 1662 * Now validate mapping with desired protection/wiring. 1663 */ 1664 newpte = (vm_offset_t) (pa | pte_prot(pmap, prot) | PG_V); 1665 1666 if (wired) 1667 newpte |= PG_W; 1668 if (va < UPT_MIN_ADDRESS) 1669 newpte |= PG_U; 1670 1671 /* 1672 * if the mapping or permission bits are different, we need 1673 * to update the pte. 1674 */ 1675 if ((origpte & ~(PG_M|PG_A)) != newpte) { 1676 *pte = newpte; 1677 if (origpte) 1678 pmap_update_1pg(va); 1679 } 1680} 1681 1682/* 1683 * this code makes some *MAJOR* assumptions: 1684 * 1. Current pmap & pmap exists. 1685 * 2. Not wired. 1686 * 3. Read access. 1687 * 4. No page table pages. 1688 * 5. Tlbflush is deferred to calling procedure. 1689 * 6. Page IS managed. 1690 * but is *MUCH* faster than pmap_enter... 1691 */ 1692 1693static void 1694pmap_enter_quick(pmap, va, pa) 1695 register pmap_t pmap; 1696 vm_offset_t va; 1697 register vm_offset_t pa; 1698{ 1699 register unsigned *pte; 1700 vm_page_t mpte; 1701 1702 mpte = NULL; 1703 /* 1704 * In the case that a page table page is not 1705 * resident, we are creating it here. 1706 */ 1707 if (va < UPT_MIN_ADDRESS) 1708 mpte = pmap_allocpte(pmap, va); 1709 1710 /* 1711 * This call to vtopte makes the assumption that we are 1712 * entering the page into the current pmap. In order to support 1713 * quick entry into any pmap, one would likely use pmap_pte_quick. 1714 * But that isn't as quick as vtopte. 1715 */ 1716 pte = (unsigned *)vtopte(va); 1717 if (*pte) { 1718 if (mpte) 1719 pmap_unwire_pte_hold(pmap, mpte); 1720 return; 1721 } 1722 1723 /* 1724 * Enter on the PV list if part of our managed memory Note that we 1725 * raise IPL while manipulating pv_table since pmap_enter can be 1726 * called at interrupt time. 1727 */ 1728 pmap_insert_entry(pmap, va, mpte, pa); 1729 1730 /* 1731 * Increment counters 1732 */ 1733 pmap->pm_stats.resident_count++; 1734 1735 /* 1736 * Now validate mapping with RO protection 1737 */ 1738 *pte = pa | PG_V | PG_U | PG_MANAGED; 1739 1740 return; 1741} 1742 1743#define MAX_INIT_PT (96) 1744/* 1745 * pmap_object_init_pt preloads the ptes for a given object 1746 * into the specified pmap. This eliminates the blast of soft 1747 * faults on process startup and immediately after an mmap. 1748 */ 1749void 1750pmap_object_init_pt(pmap, addr, object, pindex, size, limit) 1751 pmap_t pmap; 1752 vm_offset_t addr; 1753 vm_object_t object; 1754 vm_pindex_t pindex; 1755 vm_size_t size; 1756 int limit; 1757{ 1758 vm_offset_t tmpidx; 1759 int psize; 1760 vm_page_t p; 1761 int objpgs; 1762 1763 psize = i386_btop(size); 1764 1765 if (!pmap || (object->type != OBJT_VNODE) || 1766 (limit && (psize > MAX_INIT_PT) && 1767 (object->resident_page_count > MAX_INIT_PT))) { 1768 return; 1769 } 1770 1771 if (psize + pindex > object->size) 1772 psize = object->size - pindex; 1773 1774 /* 1775 * if we are processing a major portion of the object, then scan the 1776 * entire thing. 1777 */ 1778 if (psize > (object->size >> 2)) { 1779 objpgs = psize; 1780 1781 for (p = TAILQ_FIRST(&object->memq); 1782 ((objpgs > 0) && (p != NULL)); 1783 p = TAILQ_NEXT(p, listq)) { 1784 1785 tmpidx = p->pindex; 1786 if (tmpidx < pindex) { 1787 continue; 1788 } 1789 tmpidx -= pindex; 1790 if (tmpidx >= psize) { 1791 continue; 1792 } 1793 if (((p->valid & VM_PAGE_BITS_ALL) == VM_PAGE_BITS_ALL) && 1794 (p->busy == 0) && 1795 (p->flags & (PG_BUSY | PG_FICTITIOUS)) == 0) { 1796 if (p->queue == PQ_CACHE) 1797 vm_page_deactivate(p); 1798 p->flags |= PG_BUSY; 1799 pmap_enter_quick(pmap, 1800 addr + i386_ptob(tmpidx), 1801 VM_PAGE_TO_PHYS(p)); 1802 p->flags |= PG_MAPPED; 1803 PAGE_WAKEUP(p); 1804 } 1805 objpgs -= 1; 1806 } 1807 } else { 1808 /* 1809 * else lookup the pages one-by-one. 1810 */ 1811 for (tmpidx = 0; tmpidx < psize; tmpidx += 1) { 1812 p = vm_page_lookup(object, tmpidx + pindex); 1813 if (p && 1814 ((p->valid & VM_PAGE_BITS_ALL) == VM_PAGE_BITS_ALL) && 1815 (p->busy == 0) && 1816 (p->flags & (PG_BUSY | PG_FICTITIOUS)) == 0) { 1817 if (p->queue == PQ_CACHE) 1818 vm_page_deactivate(p); 1819 p->flags |= PG_BUSY; 1820 pmap_enter_quick(pmap, 1821 addr + i386_ptob(tmpidx), 1822 VM_PAGE_TO_PHYS(p)); 1823 p->flags |= PG_MAPPED; 1824 PAGE_WAKEUP(p); 1825 } 1826 } 1827 } 1828 return; 1829} 1830 1831/* 1832 * pmap_prefault provides a quick way of clustering 1833 * pagefaults into a processes address space. It is a "cousin" 1834 * of pmap_object_init_pt, except it runs at page fault time instead 1835 * of mmap time. 1836 */ 1837#define PFBAK 2 1838#define PFFOR 2 1839#define PAGEORDER_SIZE (PFBAK+PFFOR) 1840 1841static int pmap_prefault_pageorder[] = { 1842 -PAGE_SIZE, PAGE_SIZE, -2 * PAGE_SIZE, 2 * PAGE_SIZE 1843}; 1844 1845void 1846pmap_prefault(pmap, addra, entry, object) 1847 pmap_t pmap; 1848 vm_offset_t addra; 1849 vm_map_entry_t entry; 1850 vm_object_t object; 1851{ 1852 int i; 1853 vm_offset_t starta; 1854 vm_offset_t addr; 1855 vm_pindex_t pindex; 1856 vm_page_t m; 1857 1858 if (entry->object.vm_object != object) 1859 return; 1860 1861 if (!curproc || (pmap != &curproc->p_vmspace->vm_pmap)) 1862 return; 1863 1864 starta = addra - PFBAK * PAGE_SIZE; 1865 if (starta < entry->start) { 1866 starta = entry->start; 1867 } else if (starta > addra) { 1868 starta = 0; 1869 } 1870 1871 for (i = 0; i < PAGEORDER_SIZE; i++) { 1872 vm_object_t lobject; 1873 unsigned *pte; 1874 1875 addr = addra + pmap_prefault_pageorder[i]; 1876 if (addr < starta || addr >= entry->end) 1877 continue; 1878 1879 if ((*pmap_pde(pmap, addr)) == NULL) 1880 continue; 1881 1882 pte = (unsigned *) vtopte(addr); 1883 if (*pte) 1884 continue; 1885 1886 pindex = ((addr - entry->start) + entry->offset) >> PAGE_SHIFT; 1887 lobject = object; 1888 for (m = vm_page_lookup(lobject, pindex); 1889 (!m && (lobject->type == OBJT_DEFAULT) && (lobject->backing_object)); 1890 lobject = lobject->backing_object) { 1891 if (lobject->backing_object_offset & PAGE_MASK) 1892 break; 1893 pindex += (lobject->backing_object_offset >> PAGE_SHIFT); 1894 m = vm_page_lookup(lobject->backing_object, pindex); 1895 } 1896 1897 /* 1898 * give-up when a page is not in memory 1899 */ 1900 if (m == NULL) 1901 break; 1902 1903 if (((m->valid & VM_PAGE_BITS_ALL) == VM_PAGE_BITS_ALL) && 1904 (m->busy == 0) && 1905 (m->flags & (PG_BUSY | PG_FICTITIOUS)) == 0) { 1906 1907 if (m->queue == PQ_CACHE) { 1908 vm_page_deactivate(m); 1909 } 1910 m->flags |= PG_BUSY; 1911 pmap_enter_quick(pmap, addr, VM_PAGE_TO_PHYS(m)); 1912 m->flags |= PG_MAPPED; 1913 PAGE_WAKEUP(m); 1914 } 1915 } 1916} 1917 1918/* 1919 * Routine: pmap_change_wiring 1920 * Function: Change the wiring attribute for a map/virtual-address 1921 * pair. 1922 * In/out conditions: 1923 * The mapping must already exist in the pmap. 1924 */ 1925void 1926pmap_change_wiring(pmap, va, wired) 1927 register pmap_t pmap; 1928 vm_offset_t va; 1929 boolean_t wired; 1930{ 1931 register unsigned *pte; 1932 1933 if (pmap == NULL) 1934 return; 1935 1936 pte = pmap_pte(pmap, va); 1937 1938 if (wired && !pmap_pte_w(pte)) 1939 pmap->pm_stats.wired_count++; 1940 else if (!wired && pmap_pte_w(pte)) 1941 pmap->pm_stats.wired_count--; 1942 1943 /* 1944 * Wiring is not a hardware characteristic so there is no need to 1945 * invalidate TLB. 1946 */ 1947 pmap_pte_set_w(pte, wired); 1948} 1949 1950 1951 1952/* 1953 * Copy the range specified by src_addr/len 1954 * from the source map to the range dst_addr/len 1955 * in the destination map. 1956 * 1957 * This routine is only advisory and need not do anything. 1958 */ 1959void 1960pmap_copy(dst_pmap, src_pmap, dst_addr, len, src_addr) 1961 pmap_t dst_pmap, src_pmap; 1962 vm_offset_t dst_addr; 1963 vm_size_t len; 1964 vm_offset_t src_addr; 1965{ 1966 vm_offset_t addr; 1967 vm_offset_t end_addr = src_addr + len; 1968 vm_offset_t pdnxt; 1969 unsigned src_frame, dst_frame; 1970 1971 if (dst_addr != src_addr) 1972 return; 1973 1974 src_frame = ((unsigned) src_pmap->pm_pdir[PTDPTDI]) & PG_FRAME; 1975 if (src_frame != (((unsigned) PTDpde) & PG_FRAME)) 1976 return; 1977 1978 dst_frame = ((unsigned) dst_pmap->pm_pdir[PTDPTDI]) & PG_FRAME; 1979 if (dst_frame != (((unsigned) APTDpde) & PG_FRAME)) { 1980 APTDpde = (pd_entry_t) (dst_frame | PG_RW | PG_V); 1981 pmap_update(); 1982 } 1983 1984 for(addr = src_addr; addr < end_addr; addr = pdnxt) { 1985 unsigned *src_pte, *dst_pte; 1986 vm_page_t dstmpte, srcmpte; 1987 vm_offset_t srcptepaddr; 1988 1989 if (addr >= UPT_MIN_ADDRESS) 1990 panic("pmap_copy: invalid to pmap_copy page tables\n"); 1991 pdnxt = ((addr + PAGE_SIZE*NPTEPG) & ~(PAGE_SIZE*NPTEPG - 1)); 1992 srcptepaddr = (vm_offset_t) src_pmap->pm_pdir[addr >> PDRSHIFT]; 1993 if (srcptepaddr == 0) { 1994 continue; 1995 } 1996 1997 srcmpte = PHYS_TO_VM_PAGE(srcptepaddr); 1998 if (srcmpte->hold_count == 0) 1999 continue; 2000 2001 if (pdnxt > end_addr) 2002 pdnxt = end_addr; 2003 2004 src_pte = (unsigned *) vtopte(addr); 2005 dst_pte = (unsigned *) avtopte(addr); 2006 while (addr < pdnxt) { 2007 unsigned ptetemp; 2008 ptetemp = *src_pte; 2009 /* 2010 * we only virtual copy managed pages 2011 */ 2012 if ((ptetemp & PG_MANAGED) != 0) { 2013 /* 2014 * We have to check after allocpte for the 2015 * pte still being around... allocpte can 2016 * block. 2017 */ 2018 dstmpte = pmap_allocpte(dst_pmap, addr); 2019 if ((*dst_pte == 0) && (ptetemp = *src_pte)) { 2020 /* 2021 * Simply clear the modified and accessed (referenced) 2022 * bits. 2023 */ 2024 *dst_pte = ptetemp & ~(PG_M|PG_A); 2025 dst_pmap->pm_stats.resident_count++; 2026 pmap_insert_entry(dst_pmap, addr, dstmpte, 2027 (ptetemp & PG_FRAME)); 2028 } else { 2029 pmap_unwire_pte_hold(dst_pmap, dstmpte); 2030 } 2031 if (dstmpte->hold_count >= srcmpte->hold_count) 2032 break; 2033 } 2034 addr += PAGE_SIZE; 2035 ++src_pte; 2036 ++dst_pte; 2037 } 2038 } 2039} 2040 2041/* 2042 * Routine: pmap_kernel 2043 * Function: 2044 * Returns the physical map handle for the kernel. 2045 */ 2046pmap_t 2047pmap_kernel() 2048{ 2049 return (kernel_pmap); 2050} 2051 2052/* 2053 * pmap_zero_page zeros the specified (machine independent) 2054 * page by mapping the page into virtual memory and using 2055 * bzero to clear its contents, one machine dependent page 2056 * at a time. 2057 */ 2058void 2059pmap_zero_page(phys) 2060 vm_offset_t phys; 2061{ 2062 if (*(int *) CMAP2) 2063 panic("pmap_zero_page: CMAP busy"); 2064 2065 *(int *) CMAP2 = PG_V | PG_RW | (phys & PG_FRAME); 2066 bzero(CADDR2, PAGE_SIZE); 2067 *(int *) CMAP2 = 0; 2068 pmap_update_1pg((vm_offset_t) CADDR2); 2069} 2070 2071/* 2072 * pmap_copy_page copies the specified (machine independent) 2073 * page by mapping the page into virtual memory and using 2074 * bcopy to copy the page, one machine dependent page at a 2075 * time. 2076 */ 2077void 2078pmap_copy_page(src, dst) 2079 vm_offset_t src; 2080 vm_offset_t dst; 2081{ 2082 if (*(int *) CMAP1 || *(int *) CMAP2) 2083 panic("pmap_copy_page: CMAP busy"); 2084 2085 *(int *) CMAP1 = PG_V | PG_RW | (src & PG_FRAME); 2086 *(int *) CMAP2 = PG_V | PG_RW | (dst & PG_FRAME); 2087 2088#if __GNUC__ > 1 2089 memcpy(CADDR2, CADDR1, PAGE_SIZE); 2090#else 2091 bcopy(CADDR1, CADDR2, PAGE_SIZE); 2092#endif 2093 *(int *) CMAP1 = 0; 2094 *(int *) CMAP2 = 0; 2095 pmap_update_2pg( (vm_offset_t) CADDR1, (vm_offset_t) CADDR2); 2096} 2097 2098 2099/* 2100 * Routine: pmap_pageable 2101 * Function: 2102 * Make the specified pages (by pmap, offset) 2103 * pageable (or not) as requested. 2104 * 2105 * A page which is not pageable may not take 2106 * a fault; therefore, its page table entry 2107 * must remain valid for the duration. 2108 * 2109 * This routine is merely advisory; pmap_enter 2110 * will specify that these pages are to be wired 2111 * down (or not) as appropriate. 2112 */ 2113void 2114pmap_pageable(pmap, sva, eva, pageable) 2115 pmap_t pmap; 2116 vm_offset_t sva, eva; 2117 boolean_t pageable; 2118{ 2119} 2120 2121/* 2122 * this routine returns true if a physical page resides 2123 * in the given pmap. 2124 */ 2125boolean_t 2126pmap_page_exists(pmap, pa) 2127 pmap_t pmap; 2128 vm_offset_t pa; 2129{ 2130 register pv_entry_t *ppv, pv; 2131 int s; 2132 2133 if (!pmap_is_managed(pa)) 2134 return FALSE; 2135 2136 s = splvm(); 2137 2138 ppv = pa_to_pvh(pa); 2139 /* 2140 * Not found, check current mappings returning immediately if found. 2141 */ 2142 for (pv = *ppv; pv; pv = pv->pv_next) { 2143 if (pv->pv_pmap == pmap) { 2144 splx(s); 2145 return TRUE; 2146 } 2147 } 2148 splx(s); 2149 return (FALSE); 2150} 2151 2152/* 2153 * pmap_testbit tests bits in pte's 2154 * note that the testbit/changebit routines are inline, 2155 * and a lot of things compile-time evaluate. 2156 */ 2157static __inline boolean_t 2158pmap_testbit(pa, bit) 2159 register vm_offset_t pa; 2160 int bit; 2161{ 2162 register pv_entry_t *ppv, pv; 2163 unsigned *pte; 2164 int s; 2165 2166 if (!pmap_is_managed(pa)) 2167 return FALSE; 2168 2169 ppv = pa_to_pvh(pa); 2170 if (*ppv == NULL) 2171 return FALSE; 2172 2173 s = splvm(); 2174 /* 2175 * Not found, check current mappings returning immediately if found. 2176 */ 2177 for (pv = *ppv ;pv; pv = pv->pv_next) { 2178 2179 /* 2180 * if the bit being tested is the modified bit, then 2181 * mark clean_map and ptes as never 2182 * modified. 2183 */ 2184 if (bit & (PG_A|PG_M)) { 2185 if (!pmap_track_modified(pv->pv_va)) 2186 continue; 2187 } 2188 2189 if (!pv->pv_pmap) { 2190#if defined(PMAP_DIAGNOSTIC) 2191 printf("Null pmap (tb) at va: 0x%lx\n", pv->pv_va); 2192#endif 2193 continue; 2194 } 2195 pte = pmap_pte_quick(pv->pv_pmap, pv->pv_va); 2196 if (pte == NULL) 2197 continue; 2198 if (*pte & bit) { 2199 splx(s); 2200 return TRUE; 2201 } 2202 } 2203 splx(s); 2204 return (FALSE); 2205} 2206 2207/* 2208 * this routine is used to modify bits in ptes 2209 */ 2210static __inline void 2211pmap_changebit(pa, bit, setem) 2212 vm_offset_t pa; 2213 int bit; 2214 boolean_t setem; 2215{ 2216 register pv_entry_t pv, *ppv; 2217 register unsigned *pte; 2218 vm_offset_t va; 2219 int changed; 2220 int s; 2221 2222 if (!pmap_is_managed(pa)) 2223 return; 2224 2225 s = splvm(); 2226 changed = 0; 2227 ppv = pa_to_pvh(pa); 2228 /* 2229 * Loop over all current mappings setting/clearing as appropos If 2230 * setting RO do we need to clear the VAC? 2231 */ 2232 for ( pv = *ppv; pv; pv = pv->pv_next) { 2233 va = pv->pv_va; 2234 2235 /* 2236 * don't write protect pager mappings 2237 */ 2238 if (!setem && (bit == PG_RW)) { 2239 if (va >= clean_sva && va < clean_eva) 2240 continue; 2241 } 2242 if (!pv->pv_pmap) { 2243#if defined(PMAP_DIAGNOSTIC) 2244 printf("Null pmap (cb) at va: 0x%lx\n", va); 2245#endif 2246 continue; 2247 } 2248 2249 pte = pmap_pte_quick(pv->pv_pmap, va); 2250 if (pte == NULL) 2251 continue; 2252 if (setem) { 2253 *(int *)pte |= bit; 2254 changed = 1; 2255 } else { 2256 vm_offset_t pbits = *(vm_offset_t *)pte; 2257 if (pbits & bit) 2258 changed = 1; 2259 if (bit == PG_RW) { 2260 if (pbits & PG_M) { 2261 vm_page_t m; 2262 vm_offset_t pa = pbits & PG_FRAME; 2263 m = PHYS_TO_VM_PAGE(pa); 2264 m->dirty = VM_PAGE_BITS_ALL; 2265 } 2266 *(int *)pte = pbits & ~(PG_M|PG_RW); 2267 } else { 2268 *(int *)pte = pbits & ~bit; 2269 } 2270 } 2271 } 2272 splx(s); 2273 if (changed) 2274 pmap_update(); 2275} 2276 2277/* 2278 * pmap_page_protect: 2279 * 2280 * Lower the permission for all mappings to a given page. 2281 */ 2282void 2283pmap_page_protect(phys, prot) 2284 vm_offset_t phys; 2285 vm_prot_t prot; 2286{ 2287 if ((prot & VM_PROT_WRITE) == 0) { 2288 if (prot & (VM_PROT_READ | VM_PROT_EXECUTE)) { 2289 pmap_changebit(phys, PG_RW, FALSE); 2290 } else { 2291 pmap_remove_all(phys); 2292 pmap_update(); 2293 } 2294 } 2295} 2296 2297vm_offset_t 2298pmap_phys_address(ppn) 2299 int ppn; 2300{ 2301 return (i386_ptob(ppn)); 2302} 2303 2304/* 2305 * pmap_is_referenced: 2306 * 2307 * Return whether or not the specified physical page was referenced 2308 * by any physical maps. 2309 */ 2310boolean_t 2311pmap_is_referenced(vm_offset_t pa) 2312{ 2313 register pv_entry_t *ppv, pv, lpv; 2314 unsigned *pte; 2315 int s; 2316 2317 if (!pmap_is_managed(pa)) 2318 return FALSE; 2319 2320 ppv = pa_to_pvh(pa); 2321 2322 s = splvm(); 2323 /* 2324 * Not found, check current mappings returning immediately if found. 2325 */ 2326 for (lpv = NULL, pv = *ppv ;pv; lpv = pv, pv = pv->pv_next) { 2327 /* 2328 * if the bit being tested is the modified bit, then 2329 * mark clean_map and ptes as never 2330 * modified. 2331 */ 2332 if (!pmap_track_modified(pv->pv_va)) 2333 continue; 2334 if (!pv->pv_pmap) { 2335 continue; 2336 } 2337 pte = pmap_pte_quick(pv->pv_pmap, pv->pv_va); 2338 if (pte == NULL) 2339 continue; 2340 if ((int) *pte & PG_A) { 2341 if (lpv) { 2342 lpv->pv_next = pv->pv_next; 2343 pv->pv_next = *ppv; 2344 *ppv = pv; 2345 } 2346 splx(s); 2347 return TRUE; 2348 } 2349 } 2350 splx(s); 2351 return (FALSE); 2352} 2353 2354/* 2355 * pmap_ts_referenced: 2356 * 2357 * Return the count of reference bits for a page, clearing all of them. 2358 * 2359 */ 2360int 2361pmap_ts_referenced(vm_offset_t pa) 2362{ 2363 register pv_entry_t *ppv, pv; 2364 unsigned *pte; 2365 int s; 2366 int rtval = 0; 2367 vm_offset_t vachanged[VATRACK]; 2368 2369 if (!pmap_is_managed(pa)) 2370 return FALSE; 2371 2372 s = splvm(); 2373 2374 ppv = pa_to_pvh(pa); 2375 2376 if (*ppv == NULL) { 2377 splx(s); 2378 return 0; 2379 } 2380 2381 /* 2382 * Not found, check current mappings returning immediately if found. 2383 */ 2384 for (pv = *ppv ;pv; pv = pv->pv_next) { 2385 /* 2386 * if the bit being tested is the modified bit, then 2387 * mark clean_map and ptes as never 2388 * modified. 2389 */ 2390 if (!pmap_track_modified(pv->pv_va)) 2391 continue; 2392 2393 if (!pv->pv_pmap) { 2394 continue; 2395 } 2396 pte = pmap_pte_quick(pv->pv_pmap, pv->pv_va); 2397 if (pte == NULL) 2398 continue; 2399 if (*pte & PG_A) { 2400 if (rtval < VATRACK) 2401 vachanged[rtval] = pv->pv_va; 2402 rtval++; 2403 *pte &= ~PG_A; 2404 } 2405 } 2406 splx(s); 2407 if (rtval) { 2408 if (rtval <= VATRACK) { 2409 int i; 2410 for(i=0;i<rtval;i++) 2411 pmap_update_1pg(vachanged[i]); 2412 } else { 2413 pmap_update(); 2414 } 2415 } 2416 return (rtval); 2417} 2418 2419/* 2420 * pmap_is_modified: 2421 * 2422 * Return whether or not the specified physical page was modified 2423 * in any physical maps. 2424 */ 2425boolean_t 2426pmap_is_modified(vm_offset_t pa) 2427{ 2428 return pmap_testbit((pa), PG_M); 2429} 2430 2431/* 2432 * Clear the modify bits on the specified physical page. 2433 */ 2434void 2435pmap_clear_modify(vm_offset_t pa) 2436{ 2437 pmap_changebit((pa), PG_M, FALSE); 2438} 2439 2440/* 2441 * pmap_clear_reference: 2442 * 2443 * Clear the reference bit on the specified physical page. 2444 */ 2445void 2446pmap_clear_reference(vm_offset_t pa) 2447{ 2448 pmap_changebit((pa), PG_A, FALSE); 2449} 2450 2451/* 2452 * Miscellaneous support routines follow 2453 */ 2454 2455static void 2456i386_protection_init() 2457{ 2458 register int *kp, prot; 2459 2460 kp = protection_codes; 2461 for (prot = 0; prot < 8; prot++) { 2462 switch (prot) { 2463 case VM_PROT_NONE | VM_PROT_NONE | VM_PROT_NONE: 2464 /* 2465 * Read access is also 0. There isn't any execute bit, 2466 * so just make it readable. 2467 */ 2468 case VM_PROT_READ | VM_PROT_NONE | VM_PROT_NONE: 2469 case VM_PROT_READ | VM_PROT_NONE | VM_PROT_EXECUTE: 2470 case VM_PROT_NONE | VM_PROT_NONE | VM_PROT_EXECUTE: 2471 *kp++ = 0; 2472 break; 2473 case VM_PROT_NONE | VM_PROT_WRITE | VM_PROT_NONE: 2474 case VM_PROT_NONE | VM_PROT_WRITE | VM_PROT_EXECUTE: 2475 case VM_PROT_READ | VM_PROT_WRITE | VM_PROT_NONE: 2476 case VM_PROT_READ | VM_PROT_WRITE | VM_PROT_EXECUTE: 2477 *kp++ = PG_RW; 2478 break; 2479 } 2480 } 2481} 2482 2483/* 2484 * Map a set of physical memory pages into the kernel virtual 2485 * address space. Return a pointer to where it is mapped. This 2486 * routine is intended to be used for mapping device memory, 2487 * NOT real memory. The non-cacheable bits are set on each 2488 * mapped page. 2489 */ 2490void * 2491pmap_mapdev(pa, size) 2492 vm_offset_t pa; 2493 vm_size_t size; 2494{ 2495 vm_offset_t va, tmpva; 2496 unsigned *pte; 2497 2498 size = roundup(size, PAGE_SIZE); 2499 2500 va = kmem_alloc_pageable(kernel_map, size); 2501 if (!va) 2502 panic("pmap_mapdev: Couldn't alloc kernel virtual memory"); 2503 2504 pa = pa & PG_FRAME; 2505 for (tmpva = va; size > 0;) { 2506 pte = (unsigned *)vtopte(tmpva); 2507 *pte = pa | PG_RW | PG_V | PG_N; 2508 size -= PAGE_SIZE; 2509 tmpva += PAGE_SIZE; 2510 pa += PAGE_SIZE; 2511 } 2512 pmap_update(); 2513 2514 return ((void *) va); 2515} 2516 2517/* 2518 * perform the pmap work for mincore 2519 */ 2520int 2521pmap_mincore(pmap, addr) 2522 pmap_t pmap; 2523 vm_offset_t addr; 2524{ 2525 2526 unsigned *ptep, pte; 2527 int val = 0; 2528 2529 ptep = pmap_pte(pmap, addr); 2530 if (ptep == 0) { 2531 return 0; 2532 } 2533 2534 if (pte = *ptep) { 2535 vm_offset_t pa; 2536 val = MINCORE_INCORE; 2537 pa = pte & PG_FRAME; 2538 2539 /* 2540 * Modified by us 2541 */ 2542 if (pte & PG_M) 2543 val |= MINCORE_MODIFIED|MINCORE_MODIFIED_OTHER; 2544 /* 2545 * Modified by someone 2546 */ 2547 else if (PHYS_TO_VM_PAGE(pa)->dirty || 2548 pmap_is_modified(pa)) 2549 val |= MINCORE_MODIFIED_OTHER; 2550 /* 2551 * Referenced by us 2552 */ 2553 if (pte & PG_U) 2554 val |= MINCORE_REFERENCED|MINCORE_REFERENCED_OTHER; 2555 2556 /* 2557 * Referenced by someone 2558 */ 2559 else if ((PHYS_TO_VM_PAGE(pa)->flags & PG_REFERENCED) || 2560 pmap_is_referenced(pa)) 2561 val |= MINCORE_REFERENCED_OTHER; 2562 } 2563 return val; 2564} 2565 2566#if defined(PMAP_DEBUG) 2567pmap_pid_dump(int pid) { 2568 pmap_t pmap; 2569 struct proc *p; 2570 int npte = 0; 2571 int index; 2572 for (p = allproc.lh_first; p != NULL; p = p->p_list.le_next) { 2573 if (p->p_pid != pid) 2574 continue; 2575 2576 if (p->p_vmspace) { 2577 int i,j; 2578 index = 0; 2579 pmap = &p->p_vmspace->vm_pmap; 2580 for(i=0;i<1024;i++) { 2581 pd_entry_t *pde; 2582 unsigned *pte; 2583 unsigned base = i << PDRSHIFT; 2584 2585 pde = &pmap->pm_pdir[i]; 2586 if (pde && pmap_pde_v(pde)) { 2587 for(j=0;j<1024;j++) { 2588 unsigned va = base + (j << PAGE_SHIFT); 2589 if (va >= (vm_offset_t) VM_MIN_KERNEL_ADDRESS) { 2590 if (index) { 2591 index = 0; 2592 printf("\n"); 2593 } 2594 return npte; 2595 } 2596 pte = pmap_pte( pmap, va); 2597 if (pte && pmap_pte_v(pte)) { 2598 vm_offset_t pa; 2599 vm_page_t m; 2600 pa = *(int *)pte; 2601 m = PHYS_TO_VM_PAGE((pa & PG_FRAME)); 2602 printf("va: 0x%x, pt: 0x%x, h: %d, w: %d, f: 0x%x", 2603 va, pa, m->hold_count, m->wire_count, m->flags); 2604 npte++; 2605 index++; 2606 if (index >= 2) { 2607 index = 0; 2608 printf("\n"); 2609 } else { 2610 printf(" "); 2611 } 2612 } 2613 } 2614 } 2615 } 2616 } 2617 } 2618 return npte; 2619} 2620#endif 2621 2622#if defined(DEBUG) 2623 2624static void pads __P((pmap_t pm)); 2625static void pmap_pvdump __P((vm_offset_t pa)); 2626 2627/* print address space of pmap*/ 2628static void 2629pads(pm) 2630 pmap_t pm; 2631{ 2632 unsigned va, i, j; 2633 unsigned *ptep; 2634 2635 if (pm == kernel_pmap) 2636 return; 2637 for (i = 0; i < 1024; i++) 2638 if (pm->pm_pdir[i]) 2639 for (j = 0; j < 1024; j++) { 2640 va = (i << PDRSHIFT) + (j << PAGE_SHIFT); 2641 if (pm == kernel_pmap && va < KERNBASE) 2642 continue; 2643 if (pm != kernel_pmap && va > UPT_MAX_ADDRESS) 2644 continue; 2645 ptep = pmap_pte(pm, va); 2646 if (pmap_pte_v(ptep)) 2647 printf("%x:%x ", va, *(int *) ptep); 2648 }; 2649 2650} 2651 2652static void 2653pmap_pvdump(pa) 2654 vm_offset_t pa; 2655{ 2656 register pv_entry_t pv; 2657 2658 printf("pa %x", pa); 2659 for (pv = pa_to_pvh(pa); pv; pv = pv->pv_next) { 2660#ifdef used_to_be 2661 printf(" -> pmap %x, va %x, flags %x", 2662 pv->pv_pmap, pv->pv_va, pv->pv_flags); 2663#endif 2664 printf(" -> pmap %x, va %x", 2665 pv->pv_pmap, pv->pv_va); 2666 pads(pv->pv_pmap); 2667 } 2668 printf(" "); 2669} 2670#endif 2671