pmap.c revision 12722
1/* 2 * Copyright (c) 1991 Regents of the University of California. 3 * All rights reserved. 4 * Copyright (c) 1994 John S. Dyson 5 * All rights reserved. 6 * Copyright (c) 1994 David Greenman 7 * All rights reserved. 8 * 9 * This code is derived from software contributed to Berkeley by 10 * the Systems Programming Group of the University of Utah Computer 11 * Science Department and William Jolitz of UUNET Technologies Inc. 12 * 13 * Redistribution and use in source and binary forms, with or without 14 * modification, are permitted provided that the following conditions 15 * are met: 16 * 1. Redistributions of source code must retain the above copyright 17 * notice, this list of conditions and the following disclaimer. 18 * 2. Redistributions in binary form must reproduce the above copyright 19 * notice, this list of conditions and the following disclaimer in the 20 * documentation and/or other materials provided with the distribution. 21 * 3. All advertising materials mentioning features or use of this software 22 * must display the following acknowledgement: 23 * This product includes software developed by the University of 24 * California, Berkeley and its contributors. 25 * 4. Neither the name of the University nor the names of its contributors 26 * may be used to endorse or promote products derived from this software 27 * without specific prior written permission. 28 * 29 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 30 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 31 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 32 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 33 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 34 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 35 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 36 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 37 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 38 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 39 * SUCH DAMAGE. 40 * 41 * from: @(#)pmap.c 7.7 (Berkeley) 5/12/91 42 * $Id: pmap.c,v 1.67 1995/12/07 12:45:36 davidg Exp $ 43 */ 44 45/* 46 * Derived from hp300 version by Mike Hibler, this version by William 47 * Jolitz uses a recursive map [a pde points to the page directory] to 48 * map the page tables using the pagetables themselves. This is done to 49 * reduce the impact on kernel virtual memory for lots of sparse address 50 * space, and to reduce the cost of memory to each process. 51 * 52 * Derived from: hp300/@(#)pmap.c 7.1 (Berkeley) 12/5/90 53 */ 54/* 55 * Major modifications by John S. Dyson primarily to support 56 * pageable page tables, eliminating pmap_attributes, 57 * discontiguous memory pages, and using more efficient string 58 * instructions. Jan 13, 1994. Further modifications on Mar 2, 1994, 59 * general clean-up and efficiency mods. 60 */ 61 62/* 63 * Manages physical address maps. 64 * 65 * In addition to hardware address maps, this 66 * module is called upon to provide software-use-only 67 * maps which may or may not be stored in the same 68 * form as hardware maps. These pseudo-maps are 69 * used to store intermediate results from copy 70 * operations to and from address spaces. 71 * 72 * Since the information managed by this module is 73 * also stored by the logical address mapping module, 74 * this module may throw away valid virtual-to-physical 75 * mappings at almost any time. However, invalidations 76 * of virtual-to-physical mappings must be done as 77 * requested. 78 * 79 * In order to cope with hardware architectures which 80 * make virtual-to-physical map invalidates expensive, 81 * this module may delay invalidate or reduced protection 82 * operations until such time as they are actually 83 * necessary. This module is given full information as 84 * to which processors are currently using which maps, 85 * and to when physical maps must be made correct. 86 */ 87 88#include <sys/param.h> 89#include <sys/systm.h> 90#include <sys/proc.h> 91#include <sys/malloc.h> 92#include <sys/msgbuf.h> 93#include <sys/queue.h> 94 95#include <vm/vm.h> 96#include <vm/vm_param.h> 97#include <vm/vm_prot.h> 98#include <vm/lock.h> 99#include <vm/vm_kern.h> 100#include <vm/vm_page.h> 101#include <vm/vm_map.h> 102#include <vm/vm_object.h> 103#include <vm/vm_extern.h> 104 105#include <machine/pcb.h> 106#include <machine/cputypes.h> 107#include <machine/md_var.h> 108 109#include <i386/isa/isa.h> 110 111static void init_pv_entries __P((int)); 112extern void pmap_object_init_pt __P((pmap_t pmap, vm_offset_t addr, 113 vm_object_t object, vm_offset_t offset, 114 vm_offset_t size)); 115static void pmap_remove_all __P((vm_offset_t pa)); 116static void pmap_remove_entry __P((struct pmap *pmap, pv_entry_t pv, 117 vm_offset_t va)); 118 119/* 120 * Get PDEs and PTEs for user/kernel address space 121 */ 122#define pmap_pde(m, v) (&((m)->pm_pdir[((vm_offset_t)(v) >> PD_SHIFT)&1023])) 123#define pdir_pde(m, v) (m[((vm_offset_t)(v) >> PD_SHIFT)&1023]) 124 125#define pmap_pte_pa(pte) (*(int *)(pte) & PG_FRAME) 126 127#define pmap_pde_v(pte) ((*(int *)pte & PG_V) != 0) 128#define pmap_pte_w(pte) ((*(int *)pte & PG_W) != 0) 129#define pmap_pte_m(pte) ((*(int *)pte & PG_M) != 0) 130#define pmap_pte_u(pte) ((*(int *)pte & PG_U) != 0) 131#define pmap_pte_v(pte) ((*(int *)pte & PG_V) != 0) 132 133#define pmap_pte_set_w(pte, v) ((v)?(*(int *)pte |= PG_W):(*(int *)pte &= ~PG_W)) 134#define pmap_pte_set_prot(pte, v) ((*(int *)pte &= ~PG_PROT), (*(int *)pte |= (v))) 135 136/* 137 * Given a map and a machine independent protection code, 138 * convert to a vax protection code. 139 */ 140#define pte_prot(m, p) (protection_codes[p]) 141static int protection_codes[8]; 142 143static struct pmap kernel_pmap_store; 144pmap_t kernel_pmap; 145 146vm_offset_t avail_start; /* PA of first available physical page */ 147vm_offset_t avail_end; /* PA of last available physical page */ 148vm_offset_t virtual_avail; /* VA of first avail page (after kernel bss) */ 149vm_offset_t virtual_end; /* VA of last avail page (end of kernel AS) */ 150static boolean_t pmap_initialized = FALSE; /* Has pmap_init completed? */ 151static vm_offset_t vm_first_phys; 152 153static int nkpt; 154 155extern vm_offset_t clean_sva, clean_eva; 156extern int cpu_class; 157 158/* 159 * All those kernel PT submaps that BSD is so fond of 160 */ 161pt_entry_t *CMAP1; 162static pt_entry_t *CMAP2, *ptmmap; 163static pv_entry_t pv_table; 164caddr_t CADDR1, ptvmmap; 165static caddr_t CADDR2; 166static pt_entry_t *msgbufmap; 167struct msgbuf *msgbufp; 168 169static void free_pv_entry __P((pv_entry_t pv)); 170static pt_entry_t * 171 get_pt_entry __P((pmap_t pmap)); 172static pv_entry_t 173 get_pv_entry __P((void)); 174static void i386_protection_init __P((void)); 175static void pmap_alloc_pv_entry __P((void)); 176static void pmap_changebit __P((vm_offset_t pa, int bit, boolean_t setem)); 177static void pmap_enter_quick __P((pmap_t pmap, vm_offset_t va, 178 vm_offset_t pa)); 179static int pmap_is_managed __P((vm_offset_t pa)); 180static boolean_t 181 pmap_testbit __P((vm_offset_t pa, int bit)); 182 183/* 184 * Routine: pmap_pte 185 * Function: 186 * Extract the page table entry associated 187 * with the given map/virtual_address pair. 188 * [ what about induced faults -wfj] 189 */ 190 191inline pt_entry_t * __pure 192pmap_pte(pmap, va) 193 register pmap_t pmap; 194 vm_offset_t va; 195{ 196 197 if (pmap && *pmap_pde(pmap, va)) { 198 vm_offset_t frame = (int) pmap->pm_pdir[PTDPTDI] & PG_FRAME; 199 200 /* are we current address space or kernel? */ 201 if ((pmap == kernel_pmap) || (frame == ((int) PTDpde & PG_FRAME))) 202 return ((pt_entry_t *) vtopte(va)); 203 /* otherwise, we are alternate address space */ 204 else { 205 if (frame != ((int) APTDpde & PG_FRAME)) { 206 APTDpde = pmap->pm_pdir[PTDPTDI]; 207 pmap_update(); 208 } 209 return ((pt_entry_t *) avtopte(va)); 210 } 211 } 212 return (0); 213} 214 215/* 216 * Routine: pmap_extract 217 * Function: 218 * Extract the physical page address associated 219 * with the given map/virtual_address pair. 220 */ 221 222vm_offset_t 223pmap_extract(pmap, va) 224 register pmap_t pmap; 225 vm_offset_t va; 226{ 227 vm_offset_t pa; 228 229 if (pmap && *pmap_pde(pmap, va)) { 230 vm_offset_t frame = (int) pmap->pm_pdir[PTDPTDI] & PG_FRAME; 231 232 /* are we current address space or kernel? */ 233 if ((pmap == kernel_pmap) 234 || (frame == ((int) PTDpde & PG_FRAME))) { 235 pa = *(int *) vtopte(va); 236 /* otherwise, we are alternate address space */ 237 } else { 238 if (frame != ((int) APTDpde & PG_FRAME)) { 239 APTDpde = pmap->pm_pdir[PTDPTDI]; 240 pmap_update(); 241 } 242 pa = *(int *) avtopte(va); 243 } 244 return ((pa & PG_FRAME) | (va & ~PG_FRAME)); 245 } 246 return 0; 247 248} 249 250/* 251 * determine if a page is managed (memory vs. device) 252 */ 253static inline int 254pmap_is_managed(pa) 255 vm_offset_t pa; 256{ 257 int i; 258 259 if (!pmap_initialized) 260 return 0; 261 262 for (i = 0; phys_avail[i + 1]; i += 2) { 263 if (pa >= phys_avail[i] && pa < phys_avail[i + 1]) 264 return 1; 265 } 266 return 0; 267} 268 269/* 270 * find the vm_page_t of a pte (only) given va of pte and pmap 271 */ 272static __inline vm_page_t 273pmap_pte_vm_page(pmap, pt) 274 pmap_t pmap; 275 vm_offset_t pt; 276{ 277 vm_page_t m; 278 279 pt = i386_trunc_page(pt); 280 pt = (pt - UPT_MIN_ADDRESS) / NBPG; 281 pt = ((vm_offset_t) pmap->pm_pdir[pt]) & PG_FRAME; 282 m = PHYS_TO_VM_PAGE(pt); 283 return m; 284} 285 286/* 287 * Wire a page table page 288 */ 289__inline void 290pmap_use_pt(pmap, va) 291 pmap_t pmap; 292 vm_offset_t va; 293{ 294 vm_offset_t pt; 295 296 if ((va >= UPT_MIN_ADDRESS) || !pmap_initialized) 297 return; 298 299 pt = (vm_offset_t) vtopte(va); 300 vm_page_hold(pmap_pte_vm_page(pmap, pt)); 301} 302 303/* 304 * Unwire a page table page 305 */ 306inline void 307pmap_unuse_pt(pmap, va) 308 pmap_t pmap; 309 vm_offset_t va; 310{ 311 vm_offset_t pt; 312 vm_page_t m; 313 314 if ((va >= UPT_MIN_ADDRESS) || !pmap_initialized) 315 return; 316 317 pt = (vm_offset_t) vtopte(va); 318 m = pmap_pte_vm_page(pmap, pt); 319 vm_page_unhold(m); 320 if (pmap != kernel_pmap && 321 (m->hold_count == 0) && 322 (m->wire_count == 0) && 323 (va < KPT_MIN_ADDRESS)) { 324 pmap_page_protect(VM_PAGE_TO_PHYS(m), VM_PROT_NONE); 325 vm_page_free(m); 326 } 327} 328 329/* [ macro again?, should I force kstack into user map here? -wfj ] */ 330void 331pmap_activate(pmap, pcbp) 332 register pmap_t pmap; 333 struct pcb *pcbp; 334{ 335 PMAP_ACTIVATE(pmap, pcbp); 336} 337 338/* 339 * Bootstrap the system enough to run with virtual memory. 340 * 341 * On the i386 this is called after mapping has already been enabled 342 * and just syncs the pmap module with what has already been done. 343 * [We can't call it easily with mapping off since the kernel is not 344 * mapped with PA == VA, hence we would have to relocate every address 345 * from the linked base (virtual) address "KERNBASE" to the actual 346 * (physical) address starting relative to 0] 347 */ 348void 349pmap_bootstrap(firstaddr, loadaddr) 350 vm_offset_t firstaddr; 351 vm_offset_t loadaddr; 352{ 353 vm_offset_t va; 354 pt_entry_t *pte; 355 356 avail_start = firstaddr; 357 358 /* 359 * XXX The calculation of virtual_avail is wrong. It's NKPT*NBPG too 360 * large. It should instead be correctly calculated in locore.s and 361 * not based on 'first' (which is a physical address, not a virtual 362 * address, for the start of unused physical memory). The kernel 363 * page tables are NOT double mapped and thus should not be included 364 * in this calculation. 365 */ 366 virtual_avail = (vm_offset_t) KERNBASE + firstaddr; 367 virtual_end = VM_MAX_KERNEL_ADDRESS; 368 369 /* 370 * Initialize protection array. 371 */ 372 i386_protection_init(); 373 374 /* 375 * The kernel's pmap is statically allocated so we don't have to use 376 * pmap_create, which is unlikely to work correctly at this part of 377 * the boot sequence. 378 */ 379 kernel_pmap = &kernel_pmap_store; 380 381 kernel_pmap->pm_pdir = (pd_entry_t *) (KERNBASE + IdlePTD); 382 383 kernel_pmap->pm_count = 1; 384 nkpt = NKPT; 385 386 /* 387 * Reserve some special page table entries/VA space for temporary 388 * mapping of pages. 389 */ 390#define SYSMAP(c, p, v, n) \ 391 v = (c)va; va += ((n)*NBPG); p = pte; pte += (n); 392 393 va = virtual_avail; 394 pte = pmap_pte(kernel_pmap, va); 395 396 /* 397 * CMAP1/CMAP2 are used for zeroing and copying pages. 398 */ 399 SYSMAP(caddr_t, CMAP1, CADDR1, 1) 400 SYSMAP(caddr_t, CMAP2, CADDR2, 1) 401 402 /* 403 * ptmmap is used for reading arbitrary physical pages via /dev/mem. 404 */ 405 SYSMAP(caddr_t, ptmmap, ptvmmap, 1) 406 407 /* 408 * msgbufmap is used to map the system message buffer. 409 */ 410 SYSMAP(struct msgbuf *, msgbufmap, msgbufp, 1) 411 412 virtual_avail = va; 413 414 *(int *) CMAP1 = *(int *) CMAP2 = *(int *) PTD = 0; 415 pmap_update(); 416} 417 418/* 419 * Initialize the pmap module. 420 * Called by vm_init, to initialize any structures that the pmap 421 * system needs to map virtual memory. 422 * pmap_init has been enhanced to support in a fairly consistant 423 * way, discontiguous physical memory. 424 */ 425void 426pmap_init(phys_start, phys_end) 427 vm_offset_t phys_start, phys_end; 428{ 429 vm_offset_t addr; 430 vm_size_t npg, s; 431 int i; 432 433 /* 434 * calculate the number of pv_entries needed 435 */ 436 vm_first_phys = phys_avail[0]; 437 for (i = 0; phys_avail[i + 1]; i += 2); 438 npg = (phys_avail[(i - 2) + 1] - vm_first_phys) / NBPG; 439 440 /* 441 * Allocate memory for random pmap data structures. Includes the 442 * pv_head_table. 443 */ 444 s = (vm_size_t) (sizeof(struct pv_entry) * npg); 445 s = i386_round_page(s); 446 addr = (vm_offset_t) kmem_alloc(kernel_map, s); 447 pv_table = (pv_entry_t) addr; 448 449 /* 450 * init the pv free list 451 */ 452 init_pv_entries(npg); 453 /* 454 * Now it is safe to enable pv_table recording. 455 */ 456 pmap_initialized = TRUE; 457} 458 459/* 460 * Used to map a range of physical addresses into kernel 461 * virtual address space. 462 * 463 * For now, VM is already on, we only need to map the 464 * specified memory. 465 */ 466vm_offset_t 467pmap_map(virt, start, end, prot) 468 vm_offset_t virt; 469 vm_offset_t start; 470 vm_offset_t end; 471 int prot; 472{ 473 while (start < end) { 474 pmap_enter(kernel_pmap, virt, start, prot, FALSE); 475 virt += PAGE_SIZE; 476 start += PAGE_SIZE; 477 } 478 return (virt); 479} 480 481/* 482 * Initialize a preallocated and zeroed pmap structure, 483 * such as one in a vmspace structure. 484 */ 485void 486pmap_pinit(pmap) 487 register struct pmap *pmap; 488{ 489 /* 490 * No need to allocate page table space yet but we do need a valid 491 * page directory table. 492 */ 493 pmap->pm_pdir = (pd_entry_t *) kmem_alloc(kernel_map, PAGE_SIZE); 494 495 /* wire in kernel global address entries */ 496 bcopy(PTD + KPTDI, pmap->pm_pdir + KPTDI, nkpt * PTESIZE); 497 498 /* install self-referential address mapping entry */ 499 *(int *) (pmap->pm_pdir + PTDPTDI) = 500 ((int) pmap_kextract((vm_offset_t) pmap->pm_pdir)) | PG_V | PG_KW; 501 502 pmap->pm_count = 1; 503} 504 505/* 506 * grow the number of kernel page table entries, if needed 507 */ 508 509static vm_page_t nkpg; 510vm_offset_t kernel_vm_end; 511 512void 513pmap_growkernel(vm_offset_t addr) 514{ 515 struct proc *p; 516 struct pmap *pmap; 517 int s; 518 519 s = splhigh(); 520 if (kernel_vm_end == 0) { 521 kernel_vm_end = KERNBASE; 522 nkpt = 0; 523 while (pdir_pde(PTD, kernel_vm_end)) { 524 kernel_vm_end = (kernel_vm_end + NBPG * NPTEPG) & ~(NBPG * NPTEPG - 1); 525 ++nkpt; 526 } 527 } 528 addr = (addr + NBPG * NPTEPG) & ~(NBPG * NPTEPG - 1); 529 while (kernel_vm_end < addr) { 530 if (pdir_pde(PTD, kernel_vm_end)) { 531 kernel_vm_end = (kernel_vm_end + NBPG * NPTEPG) & ~(NBPG * NPTEPG - 1); 532 continue; 533 } 534 ++nkpt; 535 if (!nkpg) { 536 nkpg = vm_page_alloc(kernel_object, 0, VM_ALLOC_SYSTEM); 537 if (!nkpg) 538 panic("pmap_growkernel: no memory to grow kernel"); 539 vm_page_wire(nkpg); 540 vm_page_remove(nkpg); 541 pmap_zero_page(VM_PAGE_TO_PHYS(nkpg)); 542 } 543 pdir_pde(PTD, kernel_vm_end) = (pd_entry_t) (VM_PAGE_TO_PHYS(nkpg) | PG_V | PG_KW); 544 nkpg = NULL; 545 546 for (p = (struct proc *) allproc; p != NULL; p = p->p_next) { 547 if (p->p_vmspace) { 548 pmap = &p->p_vmspace->vm_pmap; 549 *pmap_pde(pmap, kernel_vm_end) = pdir_pde(PTD, kernel_vm_end); 550 } 551 } 552 *pmap_pde(kernel_pmap, kernel_vm_end) = pdir_pde(PTD, kernel_vm_end); 553 kernel_vm_end = (kernel_vm_end + NBPG * NPTEPG) & ~(NBPG * NPTEPG - 1); 554 } 555 splx(s); 556} 557 558/* 559 * Retire the given physical map from service. 560 * Should only be called if the map contains 561 * no valid mappings. 562 */ 563void 564pmap_destroy(pmap) 565 register pmap_t pmap; 566{ 567 int count; 568 569 if (pmap == NULL) 570 return; 571 572 count = --pmap->pm_count; 573 if (count == 0) { 574 pmap_release(pmap); 575 free((caddr_t) pmap, M_VMPMAP); 576 } 577} 578 579/* 580 * Release any resources held by the given physical map. 581 * Called when a pmap initialized by pmap_pinit is being released. 582 * Should only be called if the map contains no valid mappings. 583 */ 584void 585pmap_release(pmap) 586 register struct pmap *pmap; 587{ 588 kmem_free(kernel_map, (vm_offset_t) pmap->pm_pdir, PAGE_SIZE); 589} 590 591/* 592 * Add a reference to the specified pmap. 593 */ 594void 595pmap_reference(pmap) 596 pmap_t pmap; 597{ 598 if (pmap != NULL) { 599 pmap->pm_count++; 600 } 601} 602 603#define PV_FREELIST_MIN ((NBPG / sizeof (struct pv_entry)) / 2) 604 605/* 606 * Data for the pv entry allocation mechanism 607 */ 608static int pv_freelistcnt; 609static pv_entry_t pv_freelist; 610static vm_offset_t pvva; 611static int npvvapg; 612 613/* 614 * free the pv_entry back to the free list 615 */ 616inline static void 617free_pv_entry(pv) 618 pv_entry_t pv; 619{ 620 if (!pv) 621 return; 622 ++pv_freelistcnt; 623 pv->pv_next = pv_freelist; 624 pv_freelist = pv; 625} 626 627/* 628 * get a new pv_entry, allocating a block from the system 629 * when needed. 630 * the memory allocation is performed bypassing the malloc code 631 * because of the possibility of allocations at interrupt time. 632 */ 633static inline pv_entry_t 634get_pv_entry() 635{ 636 pv_entry_t tmp; 637 638 /* 639 * get more pv_entry pages if needed 640 */ 641 if (pv_freelistcnt < PV_FREELIST_MIN || pv_freelist == 0) { 642 pmap_alloc_pv_entry(); 643 } 644 /* 645 * get a pv_entry off of the free list 646 */ 647 --pv_freelistcnt; 648 tmp = pv_freelist; 649 pv_freelist = tmp->pv_next; 650 return tmp; 651} 652 653/* 654 * this *strange* allocation routine *statistically* eliminates the 655 * *possibility* of a malloc failure (*FATAL*) for a pv_entry_t data structure. 656 * also -- this code is MUCH MUCH faster than the malloc equiv... 657 */ 658static void 659pmap_alloc_pv_entry() 660{ 661 /* 662 * do we have any pre-allocated map-pages left? 663 */ 664 if (npvvapg) { 665 vm_page_t m; 666 667 /* 668 * we do this to keep recursion away 669 */ 670 pv_freelistcnt += PV_FREELIST_MIN; 671 /* 672 * allocate a physical page out of the vm system 673 */ 674 m = vm_page_alloc(kernel_object, 675 pvva - vm_map_min(kernel_map), VM_ALLOC_INTERRUPT); 676 if (m) { 677 int newentries; 678 int i; 679 pv_entry_t entry; 680 681 newentries = (NBPG / sizeof(struct pv_entry)); 682 /* 683 * wire the page 684 */ 685 vm_page_wire(m); 686 m->flags &= ~PG_BUSY; 687 /* 688 * let the kernel see it 689 */ 690 pmap_kenter(pvva, VM_PAGE_TO_PHYS(m)); 691 692 entry = (pv_entry_t) pvva; 693 /* 694 * update the allocation pointers 695 */ 696 pvva += NBPG; 697 --npvvapg; 698 699 /* 700 * free the entries into the free list 701 */ 702 for (i = 0; i < newentries; i++) { 703 free_pv_entry(entry); 704 entry++; 705 } 706 } 707 pv_freelistcnt -= PV_FREELIST_MIN; 708 } 709 if (!pv_freelist) 710 panic("get_pv_entry: cannot get a pv_entry_t"); 711} 712 713 714 715/* 716 * init the pv_entry allocation system 717 */ 718#define PVSPERPAGE 64 719void 720init_pv_entries(npg) 721 int npg; 722{ 723 /* 724 * allocate enough kvm space for PVSPERPAGE entries per page (lots) 725 * kvm space is fairly cheap, be generous!!! (the system can panic if 726 * this is too small.) 727 */ 728 npvvapg = ((npg * PVSPERPAGE) * sizeof(struct pv_entry) + NBPG - 1) / NBPG; 729 pvva = kmem_alloc_pageable(kernel_map, npvvapg * NBPG); 730 /* 731 * get the first batch of entries 732 */ 733 free_pv_entry(get_pv_entry()); 734} 735 736static pt_entry_t * 737get_pt_entry(pmap) 738 pmap_t pmap; 739{ 740 vm_offset_t frame = (int) pmap->pm_pdir[PTDPTDI] & PG_FRAME; 741 742 /* are we current address space or kernel? */ 743 if (pmap == kernel_pmap || frame == ((int) PTDpde & PG_FRAME)) { 744 return PTmap; 745 } 746 /* otherwise, we are alternate address space */ 747 if (frame != ((int) APTDpde & PG_FRAME)) { 748 APTDpde = pmap->pm_pdir[PTDPTDI]; 749 pmap_update(); 750 } 751 return APTmap; 752} 753 754/* 755 * If it is the first entry on the list, it is actually 756 * in the header and we must copy the following entry up 757 * to the header. Otherwise we must search the list for 758 * the entry. In either case we free the now unused entry. 759 */ 760static void 761pmap_remove_entry(pmap, pv, va) 762 struct pmap *pmap; 763 pv_entry_t pv; 764 vm_offset_t va; 765{ 766 pv_entry_t npv; 767 int s; 768 769 s = splhigh(); 770 if (pmap == pv->pv_pmap && va == pv->pv_va) { 771 npv = pv->pv_next; 772 if (npv) { 773 *pv = *npv; 774 free_pv_entry(npv); 775 } else { 776 pv->pv_pmap = NULL; 777 } 778 } else { 779 for (npv = pv->pv_next; npv; npv = npv->pv_next) { 780 if (pmap == npv->pv_pmap && va == npv->pv_va) { 781 break; 782 } 783 pv = npv; 784 } 785 if (npv) { 786 pv->pv_next = npv->pv_next; 787 free_pv_entry(npv); 788 } 789 } 790 splx(s); 791} 792 793/* 794 * Remove the given range of addresses from the specified map. 795 * 796 * It is assumed that the start and end are properly 797 * rounded to the page size. 798 */ 799void 800pmap_remove(pmap, sva, eva) 801 struct pmap *pmap; 802 register vm_offset_t sva; 803 register vm_offset_t eva; 804{ 805 register pt_entry_t *ptp, *ptq; 806 vm_offset_t pa; 807 register pv_entry_t pv; 808 vm_offset_t va; 809 pt_entry_t oldpte; 810 811 if (pmap == NULL) 812 return; 813 814 ptp = get_pt_entry(pmap); 815 816 /* 817 * special handling of removing one page. a very 818 * common operation and easy to short circuit some 819 * code. 820 */ 821 if ((sva + NBPG) == eva) { 822 823 if (*pmap_pde(pmap, sva) == 0) 824 return; 825 826 ptq = ptp + i386_btop(sva); 827 828 if (!*ptq) 829 return; 830 /* 831 * Update statistics 832 */ 833 if (pmap_pte_w(ptq)) 834 pmap->pm_stats.wired_count--; 835 pmap->pm_stats.resident_count--; 836 837 pa = pmap_pte_pa(ptq); 838 oldpte = *ptq; 839 *ptq = 0; 840 841 if (pmap_is_managed(pa)) { 842 if ((int) oldpte & PG_M) { 843 if (sva < USRSTACK + (UPAGES * NBPG) || 844 (sva >= KERNBASE && (sva < clean_sva || sva >= clean_eva))) { 845 PHYS_TO_VM_PAGE(pa)->dirty |= VM_PAGE_BITS_ALL; 846 } 847 } 848 pv = pa_to_pvh(pa); 849 pmap_remove_entry(pmap, pv, sva); 850 } 851 pmap_unuse_pt(pmap, sva); 852 pmap_update(); 853 return; 854 } 855 sva = i386_btop(sva); 856 eva = i386_btop(eva); 857 858 while (sva < eva) { 859 /* 860 * Weed out invalid mappings. Note: we assume that the page 861 * directory table is always allocated, and in kernel virtual. 862 */ 863 864 if (*pmap_pde(pmap, i386_ptob(sva)) == 0) { 865 /* We can race ahead here, straight to next pde.. */ 866 sva = ((sva + NPTEPG) & ~(NPTEPG - 1)); 867 continue; 868 } 869 ptq = ptp + sva; 870 871 /* 872 * search for page table entries, use string operations that 873 * are much faster than explicitly scanning when page tables 874 * are not fully populated. 875 */ 876 if (*ptq == 0) { 877 vm_offset_t pdnxt = ((sva + NPTEPG) & ~(NPTEPG - 1)); 878 vm_offset_t nscan = pdnxt - sva; 879 int found = 0; 880 881 if ((nscan + sva) > eva) 882 nscan = eva - sva; 883 884 asm("xorl %%eax,%%eax;cld;repe;scasl;jz 1f;incl %%eax;1:;" : 885 "=D"(ptq), "=a"(found) : "c"(nscan), "0"(ptq) : "cx"); 886 887 if (!found) { 888 sva = pdnxt; 889 continue; 890 } 891 ptq -= 1; 892 893 sva = ptq - ptp; 894 } 895 /* 896 * Update statistics 897 */ 898 oldpte = *ptq; 899 if (((int) oldpte) & PG_W) 900 pmap->pm_stats.wired_count--; 901 pmap->pm_stats.resident_count--; 902 903 /* 904 * Invalidate the PTEs. XXX: should cluster them up and 905 * invalidate as many as possible at once. 906 */ 907 *ptq = 0; 908 909 va = i386_ptob(sva); 910 911 /* 912 * Remove from the PV table (raise IPL since we may be called 913 * at interrupt time). 914 */ 915 pa = ((int) oldpte) & PG_FRAME; 916 if (!pmap_is_managed(pa)) { 917 pmap_unuse_pt(pmap, va); 918 ++sva; 919 continue; 920 } 921 if ((int) oldpte & PG_M) { 922 if (sva < USRSTACK + (UPAGES * NBPG) || 923 (sva >= KERNBASE && (sva < clean_sva || sva >= clean_eva))) { 924 PHYS_TO_VM_PAGE(pa)->dirty |= VM_PAGE_BITS_ALL; 925 } 926 } 927 pv = pa_to_pvh(pa); 928 pmap_remove_entry(pmap, pv, va); 929 pmap_unuse_pt(pmap, va); 930 ++sva; 931 } 932 pmap_update(); 933} 934 935/* 936 * Routine: pmap_remove_all 937 * Function: 938 * Removes this physical page from 939 * all physical maps in which it resides. 940 * Reflects back modify bits to the pager. 941 * 942 * Notes: 943 * Original versions of this routine were very 944 * inefficient because they iteratively called 945 * pmap_remove (slow...) 946 */ 947static void 948pmap_remove_all(pa) 949 vm_offset_t pa; 950{ 951 register pv_entry_t pv, npv; 952 register pt_entry_t *pte, *ptp; 953 vm_offset_t va; 954 struct pmap *pmap; 955 vm_page_t m; 956 int s; 957 int anyvalid = 0; 958 959 /* 960 * Not one of ours 961 */ 962 /* 963 * XXX this makes pmap_page_protect(NONE) illegal for non-managed 964 * pages! 965 */ 966 if (!pmap_is_managed(pa)) 967 return; 968 969 pa = i386_trunc_page(pa); 970 pv = pa_to_pvh(pa); 971 m = PHYS_TO_VM_PAGE(pa); 972 973 s = splhigh(); 974 while (pv->pv_pmap != NULL) { 975 pmap = pv->pv_pmap; 976 ptp = get_pt_entry(pmap); 977 va = pv->pv_va; 978 pte = ptp + i386_btop(va); 979 if (pmap_pte_w(pte)) 980 pmap->pm_stats.wired_count--; 981 if (*pte) { 982 pmap->pm_stats.resident_count--; 983 anyvalid++; 984 985 /* 986 * Update the vm_page_t clean and reference bits. 987 */ 988 if ((int) *pte & PG_M) { 989 if (va < USRSTACK + (UPAGES * NBPG) || 990 (va >= KERNBASE && (va < clean_sva || va >= clean_eva))) { 991 PHYS_TO_VM_PAGE(pa)->dirty |= VM_PAGE_BITS_ALL; 992 } 993 } 994 *pte = 0; 995 pmap_unuse_pt(pmap, va); 996 } 997 npv = pv->pv_next; 998 if (npv) { 999 *pv = *npv; 1000 free_pv_entry(npv); 1001 } else { 1002 pv->pv_pmap = NULL; 1003 } 1004 } 1005 splx(s); 1006 if (anyvalid) 1007 pmap_update(); 1008} 1009 1010 1011/* 1012 * Set the physical protection on the 1013 * specified range of this map as requested. 1014 */ 1015void 1016pmap_protect(pmap, sva, eva, prot) 1017 register pmap_t pmap; 1018 vm_offset_t sva, eva; 1019 vm_prot_t prot; 1020{ 1021 register pt_entry_t *pte; 1022 register vm_offset_t va; 1023 int i386prot; 1024 register pt_entry_t *ptp; 1025 int evap = i386_btop(eva); 1026 int anyvalid = 0;; 1027 1028 if (pmap == NULL) 1029 return; 1030 1031 if ((prot & VM_PROT_READ) == VM_PROT_NONE) { 1032 pmap_remove(pmap, sva, eva); 1033 return; 1034 } 1035 if (prot & VM_PROT_WRITE) 1036 return; 1037 1038 ptp = get_pt_entry(pmap); 1039 1040 va = sva; 1041 while (va < eva) { 1042 int found = 0; 1043 int svap; 1044 vm_offset_t nscan; 1045 1046 /* 1047 * Page table page is not allocated. Skip it, we don't want to 1048 * force allocation of unnecessary PTE pages just to set the 1049 * protection. 1050 */ 1051 if (!*pmap_pde(pmap, va)) { 1052 /* XXX: avoid address wrap around */ 1053 nextpde: 1054 if (va >= i386_trunc_pdr((vm_offset_t) - 1)) 1055 break; 1056 va = i386_round_pdr(va + PAGE_SIZE); 1057 continue; 1058 } 1059 pte = ptp + i386_btop(va); 1060 1061 if (*pte == 0) { 1062 /* 1063 * scan for a non-empty pte 1064 */ 1065 svap = pte - ptp; 1066 nscan = ((svap + NPTEPG) & ~(NPTEPG - 1)) - svap; 1067 1068 if (nscan + svap > evap) 1069 nscan = evap - svap; 1070 1071 found = 0; 1072 if (nscan) 1073 asm("xorl %%eax,%%eax;cld;repe;scasl;jz 1f;incl %%eax;1:;" : 1074 "=D"(pte), "=a"(found) : "c"(nscan), "0"(pte) : "cx"); 1075 1076 if (!found) 1077 goto nextpde; 1078 1079 pte -= 1; 1080 svap = pte - ptp; 1081 1082 va = i386_ptob(svap); 1083 } 1084 anyvalid++; 1085 1086 i386prot = pte_prot(pmap, prot); 1087 if (va < UPT_MAX_ADDRESS) { 1088 i386prot |= PG_u; 1089 if (va >= UPT_MIN_ADDRESS) 1090 i386prot |= PG_RW; 1091 } 1092 pmap_pte_set_prot(pte, i386prot); 1093 va += PAGE_SIZE; 1094 } 1095 if (anyvalid) 1096 pmap_update(); 1097} 1098 1099/* 1100 * Insert the given physical page (p) at 1101 * the specified virtual address (v) in the 1102 * target physical map with the protection requested. 1103 * 1104 * If specified, the page will be wired down, meaning 1105 * that the related pte can not be reclaimed. 1106 * 1107 * NB: This is the only routine which MAY NOT lazy-evaluate 1108 * or lose information. That is, this routine must actually 1109 * insert this page into the given map NOW. 1110 */ 1111void 1112pmap_enter(pmap, va, pa, prot, wired) 1113 register pmap_t pmap; 1114 vm_offset_t va; 1115 register vm_offset_t pa; 1116 vm_prot_t prot; 1117 boolean_t wired; 1118{ 1119 register pt_entry_t *pte; 1120 register pt_entry_t npte; 1121 vm_offset_t opa; 1122 int ptevalid = 0; 1123 1124 if (pmap == NULL) 1125 return; 1126 1127 va = i386_trunc_page(va); 1128 pa = i386_trunc_page(pa); 1129 if (va > VM_MAX_KERNEL_ADDRESS) 1130 panic("pmap_enter: toobig"); 1131 1132 /* 1133 * Page Directory table entry not valid, we need a new PT page 1134 */ 1135 if (*pmap_pde(pmap, va) == 0) { 1136 printf("kernel page directory invalid pdir=%p, va=0x%lx\n", 1137 pmap->pm_pdir[PTDPTDI], va); 1138 panic("invalid kernel page directory"); 1139 } 1140 pte = pmap_pte(pmap, va); 1141 opa = pmap_pte_pa(pte); 1142 1143 /* 1144 * Mapping has not changed, must be protection or wiring change. 1145 */ 1146 if (opa == pa) { 1147 /* 1148 * Wiring change, just update stats. We don't worry about 1149 * wiring PT pages as they remain resident as long as there 1150 * are valid mappings in them. Hence, if a user page is wired, 1151 * the PT page will be also. 1152 */ 1153 if (wired && !pmap_pte_w(pte)) 1154 pmap->pm_stats.wired_count++; 1155 else if (!wired && pmap_pte_w(pte)) 1156 pmap->pm_stats.wired_count--; 1157 1158 goto validate; 1159 } 1160 /* 1161 * Mapping has changed, invalidate old range and fall through to 1162 * handle validating new mapping. 1163 */ 1164 if (opa) { 1165 pmap_remove(pmap, va, va + PAGE_SIZE); 1166 } 1167 /* 1168 * Enter on the PV list if part of our managed memory Note that we 1169 * raise IPL while manipulating pv_table since pmap_enter can be 1170 * called at interrupt time. 1171 */ 1172 if (pmap_is_managed(pa)) { 1173 register pv_entry_t pv, npv; 1174 int s; 1175 1176 pv = pa_to_pvh(pa); 1177 s = splhigh(); 1178 /* 1179 * No entries yet, use header as the first entry 1180 */ 1181 if (pv->pv_pmap == NULL) { 1182 pv->pv_va = va; 1183 pv->pv_pmap = pmap; 1184 pv->pv_next = NULL; 1185 } 1186 /* 1187 * There is at least one other VA mapping this page. Place 1188 * this entry after the header. 1189 */ 1190 else { 1191 npv = get_pv_entry(); 1192 npv->pv_va = va; 1193 npv->pv_pmap = pmap; 1194 npv->pv_next = pv->pv_next; 1195 pv->pv_next = npv; 1196 } 1197 splx(s); 1198 } 1199 1200 /* 1201 * Increment counters 1202 */ 1203 pmap->pm_stats.resident_count++; 1204 if (wired) 1205 pmap->pm_stats.wired_count++; 1206 1207validate: 1208 /* 1209 * Now validate mapping with desired protection/wiring. 1210 */ 1211 npte = (pt_entry_t) ((int) (pa | pte_prot(pmap, prot) | PG_V)); 1212 1213 /* 1214 * When forking (copy-on-write, etc): A process will turn off write 1215 * permissions for any of its writable pages. If the data (object) is 1216 * only referred to by one process, the processes map is modified 1217 * directly as opposed to using the object manipulation routine. When 1218 * using pmap_protect, the modified bits are not kept in the vm_page_t 1219 * data structure. Therefore, when using pmap_enter in vm_fault to 1220 * bring back writability of a page, there has been no memory of the 1221 * modified or referenced bits except at the pte level. this clause 1222 * supports the carryover of the modified and used (referenced) bits. 1223 */ 1224 if (pa == opa) 1225 (int) npte |= (int) *pte & (PG_M | PG_U); 1226 1227 if (wired) 1228 (int) npte |= PG_W; 1229 if (va < UPT_MIN_ADDRESS) 1230 (int) npte |= PG_u; 1231 else if (va < UPT_MAX_ADDRESS) 1232 (int) npte |= PG_u | PG_RW; 1233 1234 if (*pte != npte) { 1235 if (*pte) 1236 ptevalid++; 1237 *pte = npte; 1238 } 1239 if (ptevalid) { 1240 pmap_update(); 1241 } else { 1242 pmap_use_pt(pmap, va); 1243 } 1244} 1245 1246/* 1247 * Add a list of wired pages to the kva 1248 * this routine is only used for temporary 1249 * kernel mappings that do not need to have 1250 * page modification or references recorded. 1251 * Note that old mappings are simply written 1252 * over. The page *must* be wired. 1253 */ 1254void 1255pmap_qenter(va, m, count) 1256 vm_offset_t va; 1257 vm_page_t *m; 1258 int count; 1259{ 1260 int i; 1261 int anyvalid = 0; 1262 register pt_entry_t *pte; 1263 1264 for (i = 0; i < count; i++) { 1265 pte = vtopte(va + i * NBPG); 1266 if (*pte) 1267 anyvalid++; 1268 *pte = (pt_entry_t) ((int) (VM_PAGE_TO_PHYS(m[i]) | PG_RW | PG_V)); 1269 } 1270 if (anyvalid) 1271 pmap_update(); 1272} 1273/* 1274 * this routine jerks page mappings from the 1275 * kernel -- it is meant only for temporary mappings. 1276 */ 1277void 1278pmap_qremove(va, count) 1279 vm_offset_t va; 1280 int count; 1281{ 1282 int i; 1283 register pt_entry_t *pte; 1284 1285 for (i = 0; i < count; i++) { 1286 pte = vtopte(va + i * NBPG); 1287 *pte = 0; 1288 } 1289 pmap_update(); 1290} 1291 1292/* 1293 * add a wired page to the kva 1294 * note that in order for the mapping to take effect -- you 1295 * should do a pmap_update after doing the pmap_kenter... 1296 */ 1297void 1298pmap_kenter(va, pa) 1299 vm_offset_t va; 1300 register vm_offset_t pa; 1301{ 1302 register pt_entry_t *pte; 1303 int wasvalid = 0; 1304 1305 pte = vtopte(va); 1306 1307 if (*pte) 1308 wasvalid++; 1309 1310 *pte = (pt_entry_t) ((int) (pa | PG_RW | PG_V)); 1311 1312 if (wasvalid) 1313 pmap_update(); 1314} 1315 1316/* 1317 * remove a page from the kernel pagetables 1318 */ 1319void 1320pmap_kremove(va) 1321 vm_offset_t va; 1322{ 1323 register pt_entry_t *pte; 1324 1325 pte = vtopte(va); 1326 1327 *pte = (pt_entry_t) 0; 1328 pmap_update(); 1329} 1330 1331/* 1332 * this code makes some *MAJOR* assumptions: 1333 * 1. Current pmap & pmap exists. 1334 * 2. Not wired. 1335 * 3. Read access. 1336 * 4. No page table pages. 1337 * 5. Tlbflush is deferred to calling procedure. 1338 * 6. Page IS managed. 1339 * but is *MUCH* faster than pmap_enter... 1340 */ 1341 1342static inline void 1343pmap_enter_quick(pmap, va, pa) 1344 register pmap_t pmap; 1345 vm_offset_t va; 1346 register vm_offset_t pa; 1347{ 1348 register pt_entry_t *pte; 1349 register pv_entry_t pv, npv; 1350 int s; 1351 1352 /* 1353 * Enter on the PV list if part of our managed memory Note that we 1354 * raise IPL while manipulating pv_table since pmap_enter can be 1355 * called at interrupt time. 1356 */ 1357 1358 pte = vtopte(va); 1359 1360 /* a fault on the page table might occur here */ 1361 if (*pte) { 1362 pmap_remove(pmap, va, va + PAGE_SIZE); 1363 } 1364 pv = pa_to_pvh(pa); 1365 s = splhigh(); 1366 /* 1367 * No entries yet, use header as the first entry 1368 */ 1369 if (pv->pv_pmap == NULL) { 1370 pv->pv_pmap = pmap; 1371 pv->pv_va = va; 1372 pv->pv_next = NULL; 1373 } 1374 /* 1375 * There is at least one other VA mapping this page. Place this entry 1376 * after the header. 1377 */ 1378 else { 1379 npv = get_pv_entry(); 1380 npv->pv_va = va; 1381 npv->pv_pmap = pmap; 1382 npv->pv_next = pv->pv_next; 1383 pv->pv_next = npv; 1384 } 1385 splx(s); 1386 1387 /* 1388 * Increment counters 1389 */ 1390 pmap->pm_stats.resident_count++; 1391 1392 /* 1393 * Now validate mapping with desired protection/wiring. 1394 */ 1395 *pte = (pt_entry_t) ((int) (pa | PG_V | PG_u)); 1396 1397 pmap_use_pt(pmap, va); 1398 1399 return; 1400} 1401 1402#define MAX_INIT_PT (1024*2048) 1403/* 1404 * pmap_object_init_pt preloads the ptes for a given object 1405 * into the specified pmap. This eliminates the blast of soft 1406 * faults on process startup and immediately after an mmap. 1407 */ 1408void 1409pmap_object_init_pt(pmap, addr, object, offset, size) 1410 pmap_t pmap; 1411 vm_offset_t addr; 1412 vm_object_t object; 1413 vm_offset_t offset; 1414 vm_offset_t size; 1415{ 1416 vm_offset_t tmpoff; 1417 vm_page_t p; 1418 int objbytes; 1419 1420 if (!pmap || ((size > MAX_INIT_PT) && 1421 (object->resident_page_count > (MAX_INIT_PT / NBPG)))) { 1422 return; 1423 } 1424 1425 /* 1426 * if we are processing a major portion of the object, then scan the 1427 * entire thing. 1428 */ 1429 if (size > (object->size >> 2)) { 1430 objbytes = size; 1431 1432 for (p = object->memq.tqh_first; 1433 ((objbytes > 0) && (p != NULL)); 1434 p = p->listq.tqe_next) { 1435 1436 tmpoff = p->offset; 1437 if (tmpoff < offset) { 1438 continue; 1439 } 1440 tmpoff -= offset; 1441 if (tmpoff >= size) { 1442 continue; 1443 } 1444 if (((p->flags & (PG_ACTIVE | PG_INACTIVE)) != 0) && 1445 ((p->valid & VM_PAGE_BITS_ALL) == VM_PAGE_BITS_ALL) && 1446 (p->bmapped == 0) && 1447 (p->busy == 0) && 1448 (p->flags & (PG_BUSY | PG_FICTITIOUS | PG_CACHE)) == 0) { 1449 vm_page_hold(p); 1450 p->flags |= PG_MAPPED; 1451 pmap_enter_quick(pmap, addr + tmpoff, VM_PAGE_TO_PHYS(p)); 1452 vm_page_unhold(p); 1453 } 1454 objbytes -= NBPG; 1455 } 1456 } else { 1457 /* 1458 * else lookup the pages one-by-one. 1459 */ 1460 for (tmpoff = 0; tmpoff < size; tmpoff += NBPG) { 1461 p = vm_page_lookup(object, tmpoff + offset); 1462 if (p && ((p->flags & (PG_ACTIVE | PG_INACTIVE)) != 0) && 1463 (p->bmapped == 0) && (p->busy == 0) && 1464 ((p->valid & VM_PAGE_BITS_ALL) == VM_PAGE_BITS_ALL) && 1465 (p->flags & (PG_BUSY | PG_FICTITIOUS | PG_CACHE)) == 0) { 1466 vm_page_hold(p); 1467 p->flags |= PG_MAPPED; 1468 pmap_enter_quick(pmap, addr + tmpoff, VM_PAGE_TO_PHYS(p)); 1469 vm_page_unhold(p); 1470 } 1471 } 1472 } 1473} 1474 1475/* 1476 * Routine: pmap_change_wiring 1477 * Function: Change the wiring attribute for a map/virtual-address 1478 * pair. 1479 * In/out conditions: 1480 * The mapping must already exist in the pmap. 1481 */ 1482void 1483pmap_change_wiring(pmap, va, wired) 1484 register pmap_t pmap; 1485 vm_offset_t va; 1486 boolean_t wired; 1487{ 1488 register pt_entry_t *pte; 1489 1490 if (pmap == NULL) 1491 return; 1492 1493 pte = pmap_pte(pmap, va); 1494 1495 if (wired && !pmap_pte_w(pte)) 1496 pmap->pm_stats.wired_count++; 1497 else if (!wired && pmap_pte_w(pte)) 1498 pmap->pm_stats.wired_count--; 1499 1500 /* 1501 * Wiring is not a hardware characteristic so there is no need to 1502 * invalidate TLB. 1503 */ 1504 pmap_pte_set_w(pte, wired); 1505 /* 1506 * When unwiring, set the modified bit in the pte -- could have been 1507 * changed by the kernel 1508 */ 1509 if (!wired) 1510 (int) *pte |= PG_M; 1511} 1512 1513 1514 1515/* 1516 * Copy the range specified by src_addr/len 1517 * from the source map to the range dst_addr/len 1518 * in the destination map. 1519 * 1520 * This routine is only advisory and need not do anything. 1521 */ 1522void 1523pmap_copy(dst_pmap, src_pmap, dst_addr, len, src_addr) 1524 pmap_t dst_pmap, src_pmap; 1525 vm_offset_t dst_addr; 1526 vm_size_t len; 1527 vm_offset_t src_addr; 1528{ 1529} 1530 1531/* 1532 * Routine: pmap_kernel 1533 * Function: 1534 * Returns the physical map handle for the kernel. 1535 */ 1536pmap_t 1537pmap_kernel() 1538{ 1539 return (kernel_pmap); 1540} 1541 1542/* 1543 * pmap_zero_page zeros the specified (machine independent) 1544 * page by mapping the page into virtual memory and using 1545 * bzero to clear its contents, one machine dependent page 1546 * at a time. 1547 */ 1548void 1549pmap_zero_page(phys) 1550 vm_offset_t phys; 1551{ 1552 if (*(int *) CMAP2) 1553 panic("pmap_zero_page: CMAP busy"); 1554 1555 *(int *) CMAP2 = PG_V | PG_KW | i386_trunc_page(phys); 1556 bzero(CADDR2, NBPG); 1557 1558 *(int *) CMAP2 = 0; 1559 pmap_update(); 1560} 1561 1562/* 1563 * pmap_copy_page copies the specified (machine independent) 1564 * page by mapping the page into virtual memory and using 1565 * bcopy to copy the page, one machine dependent page at a 1566 * time. 1567 */ 1568void 1569pmap_copy_page(src, dst) 1570 vm_offset_t src; 1571 vm_offset_t dst; 1572{ 1573 if (*(int *) CMAP1 || *(int *) CMAP2) 1574 panic("pmap_copy_page: CMAP busy"); 1575 1576 *(int *) CMAP1 = PG_V | PG_KW | i386_trunc_page(src); 1577 *(int *) CMAP2 = PG_V | PG_KW | i386_trunc_page(dst); 1578 1579#if __GNUC__ > 1 1580 memcpy(CADDR2, CADDR1, NBPG); 1581#else 1582 bcopy(CADDR1, CADDR2, NBPG); 1583#endif 1584 *(int *) CMAP1 = 0; 1585 *(int *) CMAP2 = 0; 1586 pmap_update(); 1587} 1588 1589 1590/* 1591 * Routine: pmap_pageable 1592 * Function: 1593 * Make the specified pages (by pmap, offset) 1594 * pageable (or not) as requested. 1595 * 1596 * A page which is not pageable may not take 1597 * a fault; therefore, its page table entry 1598 * must remain valid for the duration. 1599 * 1600 * This routine is merely advisory; pmap_enter 1601 * will specify that these pages are to be wired 1602 * down (or not) as appropriate. 1603 */ 1604void 1605pmap_pageable(pmap, sva, eva, pageable) 1606 pmap_t pmap; 1607 vm_offset_t sva, eva; 1608 boolean_t pageable; 1609{ 1610} 1611 1612/* 1613 * this routine returns true if a physical page resides 1614 * in the given pmap. 1615 */ 1616boolean_t 1617pmap_page_exists(pmap, pa) 1618 pmap_t pmap; 1619 vm_offset_t pa; 1620{ 1621 register pv_entry_t pv; 1622 int s; 1623 1624 if (!pmap_is_managed(pa)) 1625 return FALSE; 1626 1627 pv = pa_to_pvh(pa); 1628 s = splhigh(); 1629 1630 /* 1631 * Not found, check current mappings returning immediately if found. 1632 */ 1633 if (pv->pv_pmap != NULL) { 1634 for (; pv; pv = pv->pv_next) { 1635 if (pv->pv_pmap == pmap) { 1636 splx(s); 1637 return TRUE; 1638 } 1639 } 1640 } 1641 splx(s); 1642 return (FALSE); 1643} 1644 1645/* 1646 * pmap_testbit tests bits in pte's 1647 * note that the testbit/changebit routines are inline, 1648 * and a lot of things compile-time evaluate. 1649 */ 1650static __inline boolean_t 1651pmap_testbit(pa, bit) 1652 register vm_offset_t pa; 1653 int bit; 1654{ 1655 register pv_entry_t pv; 1656 pt_entry_t *pte; 1657 int s; 1658 1659 if (!pmap_is_managed(pa)) 1660 return FALSE; 1661 1662 pv = pa_to_pvh(pa); 1663 s = splhigh(); 1664 1665 /* 1666 * Not found, check current mappings returning immediately if found. 1667 */ 1668 if (pv->pv_pmap != NULL) { 1669 for (; pv; pv = pv->pv_next) { 1670 /* 1671 * if the bit being tested is the modified bit, then 1672 * mark UPAGES as always modified, and ptes as never 1673 * modified. 1674 */ 1675 if (bit & PG_U) { 1676 if ((pv->pv_va >= clean_sva) && (pv->pv_va < clean_eva)) { 1677 continue; 1678 } 1679 } 1680 if (bit & PG_M) { 1681 if (pv->pv_va >= USRSTACK) { 1682 if (pv->pv_va >= clean_sva && pv->pv_va < clean_eva) { 1683 continue; 1684 } 1685 if (pv->pv_va < USRSTACK + (UPAGES * NBPG)) { 1686 splx(s); 1687 return TRUE; 1688 } else if (pv->pv_va < KERNBASE) { 1689 splx(s); 1690 return FALSE; 1691 } 1692 } 1693 } 1694 if (!pv->pv_pmap) { 1695 printf("Null pmap (tb) at va: 0x%lx\n", pv->pv_va); 1696 continue; 1697 } 1698 pte = pmap_pte(pv->pv_pmap, pv->pv_va); 1699 if ((int) *pte & bit) { 1700 splx(s); 1701 return TRUE; 1702 } 1703 } 1704 } 1705 splx(s); 1706 return (FALSE); 1707} 1708 1709/* 1710 * this routine is used to modify bits in ptes 1711 */ 1712static __inline void 1713pmap_changebit(pa, bit, setem) 1714 vm_offset_t pa; 1715 int bit; 1716 boolean_t setem; 1717{ 1718 register pv_entry_t pv; 1719 register pt_entry_t *pte, npte; 1720 vm_offset_t va; 1721 int s; 1722 1723 if (!pmap_is_managed(pa)) 1724 return; 1725 1726 pv = pa_to_pvh(pa); 1727 s = splhigh(); 1728 1729 /* 1730 * Loop over all current mappings setting/clearing as appropos If 1731 * setting RO do we need to clear the VAC? 1732 */ 1733 if (pv->pv_pmap != NULL) { 1734 for (; pv; pv = pv->pv_next) { 1735 va = pv->pv_va; 1736 1737 /* 1738 * don't write protect pager mappings 1739 */ 1740 if (!setem && (bit == PG_RW)) { 1741 if (va >= clean_sva && va < clean_eva) 1742 continue; 1743 } 1744 if (!pv->pv_pmap) { 1745 printf("Null pmap (cb) at va: 0x%lx\n", va); 1746 continue; 1747 } 1748 pte = pmap_pte(pv->pv_pmap, va); 1749 if (setem) 1750 (int) npte = (int) *pte | bit; 1751 else 1752 (int) npte = (int) *pte & ~bit; 1753 *pte = npte; 1754 } 1755 } 1756 splx(s); 1757 pmap_update(); 1758} 1759 1760/* 1761 * pmap_page_protect: 1762 * 1763 * Lower the permission for all mappings to a given page. 1764 */ 1765void 1766pmap_page_protect(phys, prot) 1767 vm_offset_t phys; 1768 vm_prot_t prot; 1769{ 1770 if ((prot & VM_PROT_WRITE) == 0) { 1771 if (prot & (VM_PROT_READ | VM_PROT_EXECUTE)) 1772 pmap_changebit(phys, PG_RW, FALSE); 1773 else 1774 pmap_remove_all(phys); 1775 } 1776} 1777 1778vm_offset_t 1779pmap_phys_address(ppn) 1780 int ppn; 1781{ 1782 return (i386_ptob(ppn)); 1783} 1784 1785/* 1786 * pmap_is_referenced: 1787 * 1788 * Return whether or not the specified physical page was referenced 1789 * by any physical maps. 1790 */ 1791boolean_t 1792pmap_is_referenced(vm_offset_t pa) 1793{ 1794 return pmap_testbit((pa), PG_U); 1795} 1796 1797/* 1798 * pmap_is_modified: 1799 * 1800 * Return whether or not the specified physical page was modified 1801 * in any physical maps. 1802 */ 1803boolean_t 1804pmap_is_modified(vm_offset_t pa) 1805{ 1806 return pmap_testbit((pa), PG_M); 1807} 1808 1809/* 1810 * Clear the modify bits on the specified physical page. 1811 */ 1812void 1813pmap_clear_modify(vm_offset_t pa) 1814{ 1815 pmap_changebit((pa), PG_M, FALSE); 1816} 1817 1818/* 1819 * pmap_clear_reference: 1820 * 1821 * Clear the reference bit on the specified physical page. 1822 */ 1823void 1824pmap_clear_reference(vm_offset_t pa) 1825{ 1826 pmap_changebit((pa), PG_U, FALSE); 1827} 1828 1829/* 1830 * Miscellaneous support routines follow 1831 */ 1832 1833static void 1834i386_protection_init() 1835{ 1836 register int *kp, prot; 1837 1838 kp = protection_codes; 1839 for (prot = 0; prot < 8; prot++) { 1840 switch (prot) { 1841 case VM_PROT_NONE | VM_PROT_NONE | VM_PROT_NONE: 1842 /* 1843 * Read access is also 0. There isn't any execute bit, 1844 * so just make it readable. 1845 */ 1846 case VM_PROT_READ | VM_PROT_NONE | VM_PROT_NONE: 1847 case VM_PROT_READ | VM_PROT_NONE | VM_PROT_EXECUTE: 1848 case VM_PROT_NONE | VM_PROT_NONE | VM_PROT_EXECUTE: 1849 *kp++ = 0; 1850 break; 1851 case VM_PROT_NONE | VM_PROT_WRITE | VM_PROT_NONE: 1852 case VM_PROT_NONE | VM_PROT_WRITE | VM_PROT_EXECUTE: 1853 case VM_PROT_READ | VM_PROT_WRITE | VM_PROT_NONE: 1854 case VM_PROT_READ | VM_PROT_WRITE | VM_PROT_EXECUTE: 1855 *kp++ = PG_RW; 1856 break; 1857 } 1858 } 1859} 1860 1861/* 1862 * Map a set of physical memory pages into the kernel virtual 1863 * address space. Return a pointer to where it is mapped. This 1864 * routine is intended to be used for mapping device memory, 1865 * NOT real memory. The non-cacheable bits are set on each 1866 * mapped page. 1867 */ 1868void * 1869pmap_mapdev(pa, size) 1870 vm_offset_t pa; 1871 vm_size_t size; 1872{ 1873 vm_offset_t va, tmpva; 1874 pt_entry_t *pte; 1875 1876 pa = trunc_page(pa); 1877 size = roundup(size, PAGE_SIZE); 1878 1879 va = kmem_alloc_pageable(kernel_map, size); 1880 if (!va) 1881 panic("pmap_mapdev: Couldn't alloc kernel virtual memory"); 1882 1883 for (tmpva = va; size > 0;) { 1884 pte = vtopte(tmpva); 1885 *pte = (pt_entry_t) ((int) (pa | PG_RW | PG_V | PG_N)); 1886 size -= PAGE_SIZE; 1887 tmpva += PAGE_SIZE; 1888 pa += PAGE_SIZE; 1889 } 1890 pmap_update(); 1891 1892 return ((void *) va); 1893} 1894 1895#ifdef DEBUG 1896/* print address space of pmap*/ 1897void 1898pads(pm) 1899 pmap_t pm; 1900{ 1901 unsigned va, i, j; 1902 pt_entry_t *ptep; 1903 1904 if (pm == kernel_pmap) 1905 return; 1906 for (i = 0; i < 1024; i++) 1907 if (pm->pm_pdir[i]) 1908 for (j = 0; j < 1024; j++) { 1909 va = (i << PD_SHIFT) + (j << PG_SHIFT); 1910 if (pm == kernel_pmap && va < KERNBASE) 1911 continue; 1912 if (pm != kernel_pmap && va > UPT_MAX_ADDRESS) 1913 continue; 1914 ptep = pmap_pte(pm, va); 1915 if (pmap_pte_v(ptep)) 1916 printf("%x:%x ", va, *(int *) ptep); 1917 }; 1918 1919} 1920 1921void 1922pmap_pvdump(pa) 1923 vm_offset_t pa; 1924{ 1925 register pv_entry_t pv; 1926 1927 printf("pa %x", pa); 1928 for (pv = pa_to_pvh(pa); pv; pv = pv->pv_next) { 1929#ifdef used_to_be 1930 printf(" -> pmap %x, va %x, flags %x", 1931 pv->pv_pmap, pv->pv_va, pv->pv_flags); 1932#endif 1933 printf(" -> pmap %x, va %x", 1934 pv->pv_pmap, pv->pv_va); 1935 pads(pv->pv_pmap); 1936 } 1937 printf(" "); 1938} 1939#endif 1940