pmap.c revision 12850
1/* 2 * Copyright (c) 1991 Regents of the University of California. 3 * All rights reserved. 4 * Copyright (c) 1994 John S. Dyson 5 * All rights reserved. 6 * Copyright (c) 1994 David Greenman 7 * All rights reserved. 8 * 9 * This code is derived from software contributed to Berkeley by 10 * the Systems Programming Group of the University of Utah Computer 11 * Science Department and William Jolitz of UUNET Technologies Inc. 12 * 13 * Redistribution and use in source and binary forms, with or without 14 * modification, are permitted provided that the following conditions 15 * are met: 16 * 1. Redistributions of source code must retain the above copyright 17 * notice, this list of conditions and the following disclaimer. 18 * 2. Redistributions in binary form must reproduce the above copyright 19 * notice, this list of conditions and the following disclaimer in the 20 * documentation and/or other materials provided with the distribution. 21 * 3. All advertising materials mentioning features or use of this software 22 * must display the following acknowledgement: 23 * This product includes software developed by the University of 24 * California, Berkeley and its contributors. 25 * 4. Neither the name of the University nor the names of its contributors 26 * may be used to endorse or promote products derived from this software 27 * without specific prior written permission. 28 * 29 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 30 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 31 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 32 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 33 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 34 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 35 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 36 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 37 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 38 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 39 * SUCH DAMAGE. 40 * 41 * from: @(#)pmap.c 7.7 (Berkeley) 5/12/91 42 * $Id: pmap.c,v 1.69 1995/12/11 04:54:58 dyson Exp $ 43 */ 44 45/* 46 * Derived from hp300 version by Mike Hibler, this version by William 47 * Jolitz uses a recursive map [a pde points to the page directory] to 48 * map the page tables using the pagetables themselves. This is done to 49 * reduce the impact on kernel virtual memory for lots of sparse address 50 * space, and to reduce the cost of memory to each process. 51 * 52 * Derived from: hp300/@(#)pmap.c 7.1 (Berkeley) 12/5/90 53 */ 54/* 55 * Major modifications by John S. Dyson primarily to support 56 * pageable page tables, eliminating pmap_attributes, 57 * discontiguous memory pages, and using more efficient string 58 * instructions. Jan 13, 1994. Further modifications on Mar 2, 1994, 59 * general clean-up and efficiency mods. 60 */ 61 62/* 63 * Manages physical address maps. 64 * 65 * In addition to hardware address maps, this 66 * module is called upon to provide software-use-only 67 * maps which may or may not be stored in the same 68 * form as hardware maps. These pseudo-maps are 69 * used to store intermediate results from copy 70 * operations to and from address spaces. 71 * 72 * Since the information managed by this module is 73 * also stored by the logical address mapping module, 74 * this module may throw away valid virtual-to-physical 75 * mappings at almost any time. However, invalidations 76 * of virtual-to-physical mappings must be done as 77 * requested. 78 * 79 * In order to cope with hardware architectures which 80 * make virtual-to-physical map invalidates expensive, 81 * this module may delay invalidate or reduced protection 82 * operations until such time as they are actually 83 * necessary. This module is given full information as 84 * to which processors are currently using which maps, 85 * and to when physical maps must be made correct. 86 */ 87 88#include <sys/param.h> 89#include <sys/systm.h> 90#include <sys/proc.h> 91#include <sys/malloc.h> 92#include <sys/msgbuf.h> 93#include <sys/queue.h> 94 95#include <vm/vm.h> 96#include <vm/vm_param.h> 97#include <vm/vm_prot.h> 98#include <vm/lock.h> 99#include <vm/vm_kern.h> 100#include <vm/vm_page.h> 101#include <vm/vm_map.h> 102#include <vm/vm_object.h> 103#include <vm/vm_extern.h> 104 105#include <machine/pcb.h> 106#include <machine/cputypes.h> 107#include <machine/md_var.h> 108 109#include <i386/isa/isa.h> 110 111/* 112 * Get PDEs and PTEs for user/kernel address space 113 */ 114#define pmap_pde(m, v) (&((m)->pm_pdir[((vm_offset_t)(v) >> PD_SHIFT)&1023])) 115#define pdir_pde(m, v) (m[((vm_offset_t)(v) >> PD_SHIFT)&1023]) 116 117#define pmap_pte_pa(pte) (*(int *)(pte) & PG_FRAME) 118 119#define pmap_pde_v(pte) ((*(int *)pte & PG_V) != 0) 120#define pmap_pte_w(pte) ((*(int *)pte & PG_W) != 0) 121#define pmap_pte_m(pte) ((*(int *)pte & PG_M) != 0) 122#define pmap_pte_u(pte) ((*(int *)pte & PG_U) != 0) 123#define pmap_pte_v(pte) ((*(int *)pte & PG_V) != 0) 124 125#define pmap_pte_set_w(pte, v) ((v)?(*(int *)pte |= PG_W):(*(int *)pte &= ~PG_W)) 126#define pmap_pte_set_prot(pte, v) ((*(int *)pte &= ~PG_PROT), (*(int *)pte |= (v))) 127 128/* 129 * Given a map and a machine independent protection code, 130 * convert to a vax protection code. 131 */ 132#define pte_prot(m, p) (protection_codes[p]) 133static int protection_codes[8]; 134 135static struct pmap kernel_pmap_store; 136pmap_t kernel_pmap; 137 138vm_offset_t avail_start; /* PA of first available physical page */ 139vm_offset_t avail_end; /* PA of last available physical page */ 140vm_offset_t virtual_avail; /* VA of first avail page (after kernel bss) */ 141vm_offset_t virtual_end; /* VA of last avail page (end of kernel AS) */ 142static boolean_t pmap_initialized = FALSE; /* Has pmap_init completed? */ 143static vm_offset_t vm_first_phys; 144 145static int nkpt; 146 147extern vm_offset_t clean_sva, clean_eva; 148extern int cpu_class; 149 150/* 151 * All those kernel PT submaps that BSD is so fond of 152 */ 153pt_entry_t *CMAP1; 154static pt_entry_t *CMAP2, *ptmmap; 155static pv_entry_t pv_table; 156caddr_t CADDR1, ptvmmap; 157static caddr_t CADDR2; 158static pt_entry_t *msgbufmap; 159struct msgbuf *msgbufp; 160 161static void free_pv_entry __P((pv_entry_t pv)); 162static pt_entry_t * 163 get_pt_entry __P((pmap_t pmap)); 164static pv_entry_t 165 get_pv_entry __P((void)); 166static void i386_protection_init __P((void)); 167static void init_pv_entries __P((int npg)); 168static void pmap_alloc_pv_entry __P((void)); 169static void pmap_changebit __P((vm_offset_t pa, int bit, boolean_t setem)); 170static void pmap_enter_quick __P((pmap_t pmap, vm_offset_t va, 171 vm_offset_t pa)); 172static int pmap_is_managed __P((vm_offset_t pa)); 173extern void pmap_object_init_pt __P((pmap_t pmap, vm_offset_t addr, 174 vm_object_t object, vm_offset_t offset, 175 vm_offset_t size)); 176static void pmap_remove_all __P((vm_offset_t pa)); 177static void pmap_remove_entry __P((struct pmap *pmap, pv_entry_t pv, 178 vm_offset_t va)); 179static vm_page_t 180 pmap_pte_vm_page __P((pmap_t pmap, vm_offset_t pt)); 181static boolean_t 182 pmap_testbit __P((vm_offset_t pa, int bit)); 183 184/* 185 * Routine: pmap_pte 186 * Function: 187 * Extract the page table entry associated 188 * with the given map/virtual_address pair. 189 * [ what about induced faults -wfj] 190 */ 191 192inline pt_entry_t * __pure 193pmap_pte(pmap, va) 194 register pmap_t pmap; 195 vm_offset_t va; 196{ 197 198 if (pmap && *pmap_pde(pmap, va)) { 199 vm_offset_t frame = (int) pmap->pm_pdir[PTDPTDI] & PG_FRAME; 200 201 /* are we current address space or kernel? */ 202 if ((pmap == kernel_pmap) || (frame == ((int) PTDpde & PG_FRAME))) 203 return ((pt_entry_t *) vtopte(va)); 204 /* otherwise, we are alternate address space */ 205 else { 206 if (frame != ((int) APTDpde & PG_FRAME)) { 207 APTDpde = pmap->pm_pdir[PTDPTDI]; 208 pmap_update(); 209 } 210 return ((pt_entry_t *) avtopte(va)); 211 } 212 } 213 return (0); 214} 215 216/* 217 * Routine: pmap_extract 218 * Function: 219 * Extract the physical page address associated 220 * with the given map/virtual_address pair. 221 */ 222 223vm_offset_t 224pmap_extract(pmap, va) 225 register pmap_t pmap; 226 vm_offset_t va; 227{ 228 vm_offset_t pa; 229 230 if (pmap && *pmap_pde(pmap, va)) { 231 vm_offset_t frame = (int) pmap->pm_pdir[PTDPTDI] & PG_FRAME; 232 233 /* are we current address space or kernel? */ 234 if ((pmap == kernel_pmap) 235 || (frame == ((int) PTDpde & PG_FRAME))) { 236 pa = *(int *) vtopte(va); 237 /* otherwise, we are alternate address space */ 238 } else { 239 if (frame != ((int) APTDpde & PG_FRAME)) { 240 APTDpde = pmap->pm_pdir[PTDPTDI]; 241 pmap_update(); 242 } 243 pa = *(int *) avtopte(va); 244 } 245 return ((pa & PG_FRAME) | (va & ~PG_FRAME)); 246 } 247 return 0; 248 249} 250 251/* 252 * determine if a page is managed (memory vs. device) 253 */ 254static inline int 255pmap_is_managed(pa) 256 vm_offset_t pa; 257{ 258 int i; 259 260 if (!pmap_initialized) 261 return 0; 262 263 for (i = 0; phys_avail[i + 1]; i += 2) { 264 if (pa >= phys_avail[i] && pa < phys_avail[i + 1]) 265 return 1; 266 } 267 return 0; 268} 269 270/* 271 * find the vm_page_t of a pte (only) given va of pte and pmap 272 */ 273static __inline vm_page_t 274pmap_pte_vm_page(pmap, pt) 275 pmap_t pmap; 276 vm_offset_t pt; 277{ 278 vm_page_t m; 279 280 pt = i386_trunc_page(pt); 281 pt = (pt - UPT_MIN_ADDRESS) / NBPG; 282 pt = ((vm_offset_t) pmap->pm_pdir[pt]) & PG_FRAME; 283 m = PHYS_TO_VM_PAGE(pt); 284 return m; 285} 286 287/* 288 * Wire a page table page 289 */ 290__inline void 291pmap_use_pt(pmap, va) 292 pmap_t pmap; 293 vm_offset_t va; 294{ 295 vm_offset_t pt; 296 297 if ((va >= UPT_MIN_ADDRESS) || !pmap_initialized) 298 return; 299 300 pt = (vm_offset_t) vtopte(va); 301 vm_page_hold(pmap_pte_vm_page(pmap, pt)); 302} 303 304/* 305 * Unwire a page table page 306 */ 307inline void 308pmap_unuse_pt(pmap, va) 309 pmap_t pmap; 310 vm_offset_t va; 311{ 312 vm_offset_t pt; 313 vm_page_t m; 314 315 if ((va >= UPT_MIN_ADDRESS) || !pmap_initialized) 316 return; 317 318 pt = (vm_offset_t) vtopte(va); 319 m = pmap_pte_vm_page(pmap, pt); 320 vm_page_unhold(m); 321 if (pmap != kernel_pmap && 322 (m->hold_count == 0) && 323 (m->wire_count == 0) && 324 (va < KPT_MIN_ADDRESS)) { 325 pmap_page_protect(VM_PAGE_TO_PHYS(m), VM_PROT_NONE); 326 vm_page_free(m); 327 } 328} 329 330/* [ macro again?, should I force kstack into user map here? -wfj ] */ 331void 332pmap_activate(pmap, pcbp) 333 register pmap_t pmap; 334 struct pcb *pcbp; 335{ 336 PMAP_ACTIVATE(pmap, pcbp); 337} 338 339/* 340 * Bootstrap the system enough to run with virtual memory. 341 * 342 * On the i386 this is called after mapping has already been enabled 343 * and just syncs the pmap module with what has already been done. 344 * [We can't call it easily with mapping off since the kernel is not 345 * mapped with PA == VA, hence we would have to relocate every address 346 * from the linked base (virtual) address "KERNBASE" to the actual 347 * (physical) address starting relative to 0] 348 */ 349void 350pmap_bootstrap(firstaddr, loadaddr) 351 vm_offset_t firstaddr; 352 vm_offset_t loadaddr; 353{ 354 vm_offset_t va; 355 pt_entry_t *pte; 356 357 avail_start = firstaddr; 358 359 /* 360 * XXX The calculation of virtual_avail is wrong. It's NKPT*NBPG too 361 * large. It should instead be correctly calculated in locore.s and 362 * not based on 'first' (which is a physical address, not a virtual 363 * address, for the start of unused physical memory). The kernel 364 * page tables are NOT double mapped and thus should not be included 365 * in this calculation. 366 */ 367 virtual_avail = (vm_offset_t) KERNBASE + firstaddr; 368 virtual_end = VM_MAX_KERNEL_ADDRESS; 369 370 /* 371 * Initialize protection array. 372 */ 373 i386_protection_init(); 374 375 /* 376 * The kernel's pmap is statically allocated so we don't have to use 377 * pmap_create, which is unlikely to work correctly at this part of 378 * the boot sequence. 379 */ 380 kernel_pmap = &kernel_pmap_store; 381 382 kernel_pmap->pm_pdir = (pd_entry_t *) (KERNBASE + IdlePTD); 383 384 kernel_pmap->pm_count = 1; 385 nkpt = NKPT; 386 387 /* 388 * Reserve some special page table entries/VA space for temporary 389 * mapping of pages. 390 */ 391#define SYSMAP(c, p, v, n) \ 392 v = (c)va; va += ((n)*NBPG); p = pte; pte += (n); 393 394 va = virtual_avail; 395 pte = pmap_pte(kernel_pmap, va); 396 397 /* 398 * CMAP1/CMAP2 are used for zeroing and copying pages. 399 */ 400 SYSMAP(caddr_t, CMAP1, CADDR1, 1) 401 SYSMAP(caddr_t, CMAP2, CADDR2, 1) 402 403 /* 404 * ptmmap is used for reading arbitrary physical pages via /dev/mem. 405 */ 406 SYSMAP(caddr_t, ptmmap, ptvmmap, 1) 407 408 /* 409 * msgbufmap is used to map the system message buffer. 410 */ 411 SYSMAP(struct msgbuf *, msgbufmap, msgbufp, 1) 412 413 virtual_avail = va; 414 415 *(int *) CMAP1 = *(int *) CMAP2 = *(int *) PTD = 0; 416 pmap_update(); 417} 418 419/* 420 * Initialize the pmap module. 421 * Called by vm_init, to initialize any structures that the pmap 422 * system needs to map virtual memory. 423 * pmap_init has been enhanced to support in a fairly consistant 424 * way, discontiguous physical memory. 425 */ 426void 427pmap_init(phys_start, phys_end) 428 vm_offset_t phys_start, phys_end; 429{ 430 vm_offset_t addr; 431 vm_size_t npg, s; 432 int i; 433 434 /* 435 * calculate the number of pv_entries needed 436 */ 437 vm_first_phys = phys_avail[0]; 438 for (i = 0; phys_avail[i + 1]; i += 2); 439 npg = (phys_avail[(i - 2) + 1] - vm_first_phys) / NBPG; 440 441 /* 442 * Allocate memory for random pmap data structures. Includes the 443 * pv_head_table. 444 */ 445 s = (vm_size_t) (sizeof(struct pv_entry) * npg); 446 s = i386_round_page(s); 447 addr = (vm_offset_t) kmem_alloc(kernel_map, s); 448 pv_table = (pv_entry_t) addr; 449 450 /* 451 * init the pv free list 452 */ 453 init_pv_entries(npg); 454 /* 455 * Now it is safe to enable pv_table recording. 456 */ 457 pmap_initialized = TRUE; 458} 459 460/* 461 * Used to map a range of physical addresses into kernel 462 * virtual address space. 463 * 464 * For now, VM is already on, we only need to map the 465 * specified memory. 466 */ 467vm_offset_t 468pmap_map(virt, start, end, prot) 469 vm_offset_t virt; 470 vm_offset_t start; 471 vm_offset_t end; 472 int prot; 473{ 474 while (start < end) { 475 pmap_enter(kernel_pmap, virt, start, prot, FALSE); 476 virt += PAGE_SIZE; 477 start += PAGE_SIZE; 478 } 479 return (virt); 480} 481 482/* 483 * Initialize a preallocated and zeroed pmap structure, 484 * such as one in a vmspace structure. 485 */ 486void 487pmap_pinit(pmap) 488 register struct pmap *pmap; 489{ 490 /* 491 * No need to allocate page table space yet but we do need a valid 492 * page directory table. 493 */ 494 pmap->pm_pdir = (pd_entry_t *) kmem_alloc(kernel_map, PAGE_SIZE); 495 496 /* wire in kernel global address entries */ 497 bcopy(PTD + KPTDI, pmap->pm_pdir + KPTDI, nkpt * PTESIZE); 498 499 /* install self-referential address mapping entry */ 500 *(int *) (pmap->pm_pdir + PTDPTDI) = 501 ((int) pmap_kextract((vm_offset_t) pmap->pm_pdir)) | PG_V | PG_KW; 502 503 pmap->pm_count = 1; 504} 505 506/* 507 * grow the number of kernel page table entries, if needed 508 */ 509 510static vm_page_t nkpg; 511vm_offset_t kernel_vm_end; 512 513void 514pmap_growkernel(vm_offset_t addr) 515{ 516 struct proc *p; 517 struct pmap *pmap; 518 int s; 519 520 s = splhigh(); 521 if (kernel_vm_end == 0) { 522 kernel_vm_end = KERNBASE; 523 nkpt = 0; 524 while (pdir_pde(PTD, kernel_vm_end)) { 525 kernel_vm_end = (kernel_vm_end + NBPG * NPTEPG) & ~(NBPG * NPTEPG - 1); 526 ++nkpt; 527 } 528 } 529 addr = (addr + NBPG * NPTEPG) & ~(NBPG * NPTEPG - 1); 530 while (kernel_vm_end < addr) { 531 if (pdir_pde(PTD, kernel_vm_end)) { 532 kernel_vm_end = (kernel_vm_end + NBPG * NPTEPG) & ~(NBPG * NPTEPG - 1); 533 continue; 534 } 535 ++nkpt; 536 if (!nkpg) { 537 nkpg = vm_page_alloc(kernel_object, 0, VM_ALLOC_SYSTEM); 538 if (!nkpg) 539 panic("pmap_growkernel: no memory to grow kernel"); 540 vm_page_wire(nkpg); 541 vm_page_remove(nkpg); 542 pmap_zero_page(VM_PAGE_TO_PHYS(nkpg)); 543 } 544 pdir_pde(PTD, kernel_vm_end) = (pd_entry_t) (VM_PAGE_TO_PHYS(nkpg) | PG_V | PG_KW); 545 nkpg = NULL; 546 547 for (p = (struct proc *) allproc; p != NULL; p = p->p_next) { 548 if (p->p_vmspace) { 549 pmap = &p->p_vmspace->vm_pmap; 550 *pmap_pde(pmap, kernel_vm_end) = pdir_pde(PTD, kernel_vm_end); 551 } 552 } 553 *pmap_pde(kernel_pmap, kernel_vm_end) = pdir_pde(PTD, kernel_vm_end); 554 kernel_vm_end = (kernel_vm_end + NBPG * NPTEPG) & ~(NBPG * NPTEPG - 1); 555 } 556 splx(s); 557} 558 559/* 560 * Retire the given physical map from service. 561 * Should only be called if the map contains 562 * no valid mappings. 563 */ 564void 565pmap_destroy(pmap) 566 register pmap_t pmap; 567{ 568 int count; 569 570 if (pmap == NULL) 571 return; 572 573 count = --pmap->pm_count; 574 if (count == 0) { 575 pmap_release(pmap); 576 free((caddr_t) pmap, M_VMPMAP); 577 } 578} 579 580/* 581 * Release any resources held by the given physical map. 582 * Called when a pmap initialized by pmap_pinit is being released. 583 * Should only be called if the map contains no valid mappings. 584 */ 585void 586pmap_release(pmap) 587 register struct pmap *pmap; 588{ 589 kmem_free(kernel_map, (vm_offset_t) pmap->pm_pdir, PAGE_SIZE); 590} 591 592/* 593 * Add a reference to the specified pmap. 594 */ 595void 596pmap_reference(pmap) 597 pmap_t pmap; 598{ 599 if (pmap != NULL) { 600 pmap->pm_count++; 601 } 602} 603 604#define PV_FREELIST_MIN ((NBPG / sizeof (struct pv_entry)) / 2) 605 606/* 607 * Data for the pv entry allocation mechanism 608 */ 609static int pv_freelistcnt; 610static pv_entry_t pv_freelist; 611static vm_offset_t pvva; 612static int npvvapg; 613 614/* 615 * free the pv_entry back to the free list 616 */ 617inline static void 618free_pv_entry(pv) 619 pv_entry_t pv; 620{ 621 if (!pv) 622 return; 623 ++pv_freelistcnt; 624 pv->pv_next = pv_freelist; 625 pv_freelist = pv; 626} 627 628/* 629 * get a new pv_entry, allocating a block from the system 630 * when needed. 631 * the memory allocation is performed bypassing the malloc code 632 * because of the possibility of allocations at interrupt time. 633 */ 634static inline pv_entry_t 635get_pv_entry() 636{ 637 pv_entry_t tmp; 638 639 /* 640 * get more pv_entry pages if needed 641 */ 642 if (pv_freelistcnt < PV_FREELIST_MIN || pv_freelist == 0) { 643 pmap_alloc_pv_entry(); 644 } 645 /* 646 * get a pv_entry off of the free list 647 */ 648 --pv_freelistcnt; 649 tmp = pv_freelist; 650 pv_freelist = tmp->pv_next; 651 return tmp; 652} 653 654/* 655 * this *strange* allocation routine *statistically* eliminates the 656 * *possibility* of a malloc failure (*FATAL*) for a pv_entry_t data structure. 657 * also -- this code is MUCH MUCH faster than the malloc equiv... 658 */ 659static void 660pmap_alloc_pv_entry() 661{ 662 /* 663 * do we have any pre-allocated map-pages left? 664 */ 665 if (npvvapg) { 666 vm_page_t m; 667 668 /* 669 * we do this to keep recursion away 670 */ 671 pv_freelistcnt += PV_FREELIST_MIN; 672 /* 673 * allocate a physical page out of the vm system 674 */ 675 m = vm_page_alloc(kernel_object, 676 OFF_TO_IDX(pvva - vm_map_min(kernel_map)), 677 VM_ALLOC_INTERRUPT); 678 if (m) { 679 int newentries; 680 int i; 681 pv_entry_t entry; 682 683 newentries = (NBPG / sizeof(struct pv_entry)); 684 /* 685 * wire the page 686 */ 687 vm_page_wire(m); 688 m->flags &= ~PG_BUSY; 689 /* 690 * let the kernel see it 691 */ 692 pmap_kenter(pvva, VM_PAGE_TO_PHYS(m)); 693 694 entry = (pv_entry_t) pvva; 695 /* 696 * update the allocation pointers 697 */ 698 pvva += NBPG; 699 --npvvapg; 700 701 /* 702 * free the entries into the free list 703 */ 704 for (i = 0; i < newentries; i++) { 705 free_pv_entry(entry); 706 entry++; 707 } 708 } 709 pv_freelistcnt -= PV_FREELIST_MIN; 710 } 711 if (!pv_freelist) 712 panic("get_pv_entry: cannot get a pv_entry_t"); 713} 714 715 716 717/* 718 * init the pv_entry allocation system 719 */ 720#define PVSPERPAGE 64 721void 722init_pv_entries(npg) 723 int npg; 724{ 725 /* 726 * allocate enough kvm space for PVSPERPAGE entries per page (lots) 727 * kvm space is fairly cheap, be generous!!! (the system can panic if 728 * this is too small.) 729 */ 730 npvvapg = ((npg * PVSPERPAGE) * sizeof(struct pv_entry) + NBPG - 1) / NBPG; 731 pvva = kmem_alloc_pageable(kernel_map, npvvapg * NBPG); 732 /* 733 * get the first batch of entries 734 */ 735 free_pv_entry(get_pv_entry()); 736} 737 738static pt_entry_t * 739get_pt_entry(pmap) 740 pmap_t pmap; 741{ 742 vm_offset_t frame = (int) pmap->pm_pdir[PTDPTDI] & PG_FRAME; 743 744 /* are we current address space or kernel? */ 745 if (pmap == kernel_pmap || frame == ((int) PTDpde & PG_FRAME)) { 746 return PTmap; 747 } 748 /* otherwise, we are alternate address space */ 749 if (frame != ((int) APTDpde & PG_FRAME)) { 750 APTDpde = pmap->pm_pdir[PTDPTDI]; 751 pmap_update(); 752 } 753 return APTmap; 754} 755 756/* 757 * If it is the first entry on the list, it is actually 758 * in the header and we must copy the following entry up 759 * to the header. Otherwise we must search the list for 760 * the entry. In either case we free the now unused entry. 761 */ 762static void 763pmap_remove_entry(pmap, pv, va) 764 struct pmap *pmap; 765 pv_entry_t pv; 766 vm_offset_t va; 767{ 768 pv_entry_t npv; 769 int s; 770 771 s = splhigh(); 772 if (pmap == pv->pv_pmap && va == pv->pv_va) { 773 npv = pv->pv_next; 774 if (npv) { 775 *pv = *npv; 776 free_pv_entry(npv); 777 } else { 778 pv->pv_pmap = NULL; 779 } 780 } else { 781 for (npv = pv->pv_next; npv; npv = npv->pv_next) { 782 if (pmap == npv->pv_pmap && va == npv->pv_va) { 783 break; 784 } 785 pv = npv; 786 } 787 if (npv) { 788 pv->pv_next = npv->pv_next; 789 free_pv_entry(npv); 790 } 791 } 792 splx(s); 793} 794 795/* 796 * Remove the given range of addresses from the specified map. 797 * 798 * It is assumed that the start and end are properly 799 * rounded to the page size. 800 */ 801void 802pmap_remove(pmap, sva, eva) 803 struct pmap *pmap; 804 register vm_offset_t sva; 805 register vm_offset_t eva; 806{ 807 register pt_entry_t *ptp, *ptq; 808 vm_offset_t pa; 809 register pv_entry_t pv; 810 vm_offset_t va; 811 pt_entry_t oldpte; 812 813 if (pmap == NULL) 814 return; 815 816 ptp = get_pt_entry(pmap); 817 818 /* 819 * special handling of removing one page. a very 820 * common operation and easy to short circuit some 821 * code. 822 */ 823 if ((sva + NBPG) == eva) { 824 825 if (*pmap_pde(pmap, sva) == 0) 826 return; 827 828 ptq = ptp + i386_btop(sva); 829 830 if (!*ptq) 831 return; 832 /* 833 * Update statistics 834 */ 835 if (pmap_pte_w(ptq)) 836 pmap->pm_stats.wired_count--; 837 pmap->pm_stats.resident_count--; 838 839 pa = pmap_pte_pa(ptq); 840 oldpte = *ptq; 841 *ptq = 0; 842 843 if (pmap_is_managed(pa)) { 844 if ((int) oldpte & PG_M) { 845 if (sva < USRSTACK + (UPAGES * NBPG) || 846 (sva >= KERNBASE && (sva < clean_sva || sva >= clean_eva))) { 847 PHYS_TO_VM_PAGE(pa)->dirty |= VM_PAGE_BITS_ALL; 848 } 849 } 850 pv = pa_to_pvh(pa); 851 pmap_remove_entry(pmap, pv, sva); 852 } 853 pmap_unuse_pt(pmap, sva); 854 pmap_update(); 855 return; 856 } 857 sva = i386_btop(sva); 858 eva = i386_btop(eva); 859 860 while (sva < eva) { 861 /* 862 * Weed out invalid mappings. Note: we assume that the page 863 * directory table is always allocated, and in kernel virtual. 864 */ 865 866 if (*pmap_pde(pmap, i386_ptob(sva)) == 0) { 867 /* We can race ahead here, straight to next pde.. */ 868 sva = ((sva + NPTEPG) & ~(NPTEPG - 1)); 869 continue; 870 } 871 ptq = ptp + sva; 872 873 /* 874 * search for page table entries, use string operations that 875 * are much faster than explicitly scanning when page tables 876 * are not fully populated. 877 */ 878 if (*ptq == 0) { 879 vm_offset_t pdnxt = ((sva + NPTEPG) & ~(NPTEPG - 1)); 880 vm_offset_t nscan = pdnxt - sva; 881 int found = 0; 882 883 if ((nscan + sva) > eva) 884 nscan = eva - sva; 885 886 asm("xorl %%eax,%%eax;cld;repe;scasl;jz 1f;incl %%eax;1:;" : 887 "=D"(ptq), "=a"(found) : "c"(nscan), "0"(ptq) : "cx"); 888 889 if (!found) { 890 sva = pdnxt; 891 continue; 892 } 893 ptq -= 1; 894 895 sva = ptq - ptp; 896 } 897 /* 898 * Update statistics 899 */ 900 oldpte = *ptq; 901 if (((int) oldpte) & PG_W) 902 pmap->pm_stats.wired_count--; 903 pmap->pm_stats.resident_count--; 904 905 /* 906 * Invalidate the PTEs. XXX: should cluster them up and 907 * invalidate as many as possible at once. 908 */ 909 *ptq = 0; 910 911 va = i386_ptob(sva); 912 913 /* 914 * Remove from the PV table (raise IPL since we may be called 915 * at interrupt time). 916 */ 917 pa = ((int) oldpte) & PG_FRAME; 918 if (!pmap_is_managed(pa)) { 919 pmap_unuse_pt(pmap, va); 920 ++sva; 921 continue; 922 } 923 if ((int) oldpte & PG_M) { 924 if (sva < USRSTACK + (UPAGES * NBPG) || 925 (sva >= KERNBASE && (sva < clean_sva || sva >= clean_eva))) { 926 PHYS_TO_VM_PAGE(pa)->dirty |= VM_PAGE_BITS_ALL; 927 } 928 } 929 pv = pa_to_pvh(pa); 930 pmap_remove_entry(pmap, pv, va); 931 pmap_unuse_pt(pmap, va); 932 ++sva; 933 } 934 pmap_update(); 935} 936 937/* 938 * Routine: pmap_remove_all 939 * Function: 940 * Removes this physical page from 941 * all physical maps in which it resides. 942 * Reflects back modify bits to the pager. 943 * 944 * Notes: 945 * Original versions of this routine were very 946 * inefficient because they iteratively called 947 * pmap_remove (slow...) 948 */ 949static void 950pmap_remove_all(pa) 951 vm_offset_t pa; 952{ 953 register pv_entry_t pv, npv; 954 register pt_entry_t *pte, *ptp; 955 vm_offset_t va; 956 struct pmap *pmap; 957 vm_page_t m; 958 int s; 959 int anyvalid = 0; 960 961 /* 962 * Not one of ours 963 */ 964 /* 965 * XXX this makes pmap_page_protect(NONE) illegal for non-managed 966 * pages! 967 */ 968 if (!pmap_is_managed(pa)) 969 return; 970 971 pa = i386_trunc_page(pa); 972 pv = pa_to_pvh(pa); 973 m = PHYS_TO_VM_PAGE(pa); 974 975 s = splhigh(); 976 while (pv->pv_pmap != NULL) { 977 pmap = pv->pv_pmap; 978 ptp = get_pt_entry(pmap); 979 va = pv->pv_va; 980 pte = ptp + i386_btop(va); 981 if (pmap_pte_w(pte)) 982 pmap->pm_stats.wired_count--; 983 if (*pte) { 984 pmap->pm_stats.resident_count--; 985 anyvalid++; 986 987 /* 988 * Update the vm_page_t clean and reference bits. 989 */ 990 if ((int) *pte & PG_M) { 991 if (va < USRSTACK + (UPAGES * NBPG) || 992 (va >= KERNBASE && (va < clean_sva || va >= clean_eva))) { 993 PHYS_TO_VM_PAGE(pa)->dirty |= VM_PAGE_BITS_ALL; 994 } 995 } 996 *pte = 0; 997 pmap_unuse_pt(pmap, va); 998 } 999 npv = pv->pv_next; 1000 if (npv) { 1001 *pv = *npv; 1002 free_pv_entry(npv); 1003 } else { 1004 pv->pv_pmap = NULL; 1005 } 1006 } 1007 splx(s); 1008 if (anyvalid) 1009 pmap_update(); 1010} 1011 1012 1013/* 1014 * Set the physical protection on the 1015 * specified range of this map as requested. 1016 */ 1017void 1018pmap_protect(pmap, sva, eva, prot) 1019 register pmap_t pmap; 1020 vm_offset_t sva, eva; 1021 vm_prot_t prot; 1022{ 1023 register pt_entry_t *pte; 1024 register vm_offset_t va; 1025 int i386prot; 1026 register pt_entry_t *ptp; 1027 int evap = i386_btop(eva); 1028 int anyvalid = 0;; 1029 1030 if (pmap == NULL) 1031 return; 1032 1033 if ((prot & VM_PROT_READ) == VM_PROT_NONE) { 1034 pmap_remove(pmap, sva, eva); 1035 return; 1036 } 1037 if (prot & VM_PROT_WRITE) 1038 return; 1039 1040 ptp = get_pt_entry(pmap); 1041 1042 va = sva; 1043 while (va < eva) { 1044 int found = 0; 1045 int svap; 1046 vm_offset_t nscan; 1047 1048 /* 1049 * Page table page is not allocated. Skip it, we don't want to 1050 * force allocation of unnecessary PTE pages just to set the 1051 * protection. 1052 */ 1053 if (!*pmap_pde(pmap, va)) { 1054 /* XXX: avoid address wrap around */ 1055 nextpde: 1056 if (va >= i386_trunc_pdr((vm_offset_t) - 1)) 1057 break; 1058 va = i386_round_pdr(va + PAGE_SIZE); 1059 continue; 1060 } 1061 pte = ptp + i386_btop(va); 1062 1063 if (*pte == 0) { 1064 /* 1065 * scan for a non-empty pte 1066 */ 1067 svap = pte - ptp; 1068 nscan = ((svap + NPTEPG) & ~(NPTEPG - 1)) - svap; 1069 1070 if (nscan + svap > evap) 1071 nscan = evap - svap; 1072 1073 found = 0; 1074 if (nscan) 1075 asm("xorl %%eax,%%eax;cld;repe;scasl;jz 1f;incl %%eax;1:;" : 1076 "=D"(pte), "=a"(found) : "c"(nscan), "0"(pte) : "cx"); 1077 1078 if (!found) 1079 goto nextpde; 1080 1081 pte -= 1; 1082 svap = pte - ptp; 1083 1084 va = i386_ptob(svap); 1085 } 1086 anyvalid++; 1087 1088 i386prot = pte_prot(pmap, prot); 1089 if (va < UPT_MAX_ADDRESS) { 1090 i386prot |= PG_u; 1091 if (va >= UPT_MIN_ADDRESS) 1092 i386prot |= PG_RW; 1093 } 1094 pmap_pte_set_prot(pte, i386prot); 1095 va += PAGE_SIZE; 1096 } 1097 if (anyvalid) 1098 pmap_update(); 1099} 1100 1101/* 1102 * Insert the given physical page (p) at 1103 * the specified virtual address (v) in the 1104 * target physical map with the protection requested. 1105 * 1106 * If specified, the page will be wired down, meaning 1107 * that the related pte can not be reclaimed. 1108 * 1109 * NB: This is the only routine which MAY NOT lazy-evaluate 1110 * or lose information. That is, this routine must actually 1111 * insert this page into the given map NOW. 1112 */ 1113void 1114pmap_enter(pmap, va, pa, prot, wired) 1115 register pmap_t pmap; 1116 vm_offset_t va; 1117 register vm_offset_t pa; 1118 vm_prot_t prot; 1119 boolean_t wired; 1120{ 1121 register pt_entry_t *pte; 1122 register pt_entry_t npte; 1123 vm_offset_t opa; 1124 int ptevalid = 0; 1125 1126 if (pmap == NULL) 1127 return; 1128 1129 va = i386_trunc_page(va); 1130 pa = i386_trunc_page(pa); 1131 if (va > VM_MAX_KERNEL_ADDRESS) 1132 panic("pmap_enter: toobig"); 1133 1134 /* 1135 * Page Directory table entry not valid, we need a new PT page 1136 */ 1137 if (*pmap_pde(pmap, va) == 0) { 1138 printf("kernel page directory invalid pdir=%p, va=0x%lx\n", 1139 pmap->pm_pdir[PTDPTDI], va); 1140 panic("invalid kernel page directory"); 1141 } 1142 pte = pmap_pte(pmap, va); 1143 opa = pmap_pte_pa(pte); 1144 1145 /* 1146 * Mapping has not changed, must be protection or wiring change. 1147 */ 1148 if (opa == pa) { 1149 /* 1150 * Wiring change, just update stats. We don't worry about 1151 * wiring PT pages as they remain resident as long as there 1152 * are valid mappings in them. Hence, if a user page is wired, 1153 * the PT page will be also. 1154 */ 1155 if (wired && !pmap_pte_w(pte)) 1156 pmap->pm_stats.wired_count++; 1157 else if (!wired && pmap_pte_w(pte)) 1158 pmap->pm_stats.wired_count--; 1159 1160 goto validate; 1161 } 1162 /* 1163 * Mapping has changed, invalidate old range and fall through to 1164 * handle validating new mapping. 1165 */ 1166 if (opa) { 1167 pmap_remove(pmap, va, va + PAGE_SIZE); 1168 } 1169 /* 1170 * Enter on the PV list if part of our managed memory Note that we 1171 * raise IPL while manipulating pv_table since pmap_enter can be 1172 * called at interrupt time. 1173 */ 1174 if (pmap_is_managed(pa)) { 1175 register pv_entry_t pv, npv; 1176 int s; 1177 1178 pv = pa_to_pvh(pa); 1179 s = splhigh(); 1180 /* 1181 * No entries yet, use header as the first entry 1182 */ 1183 if (pv->pv_pmap == NULL) { 1184 pv->pv_va = va; 1185 pv->pv_pmap = pmap; 1186 pv->pv_next = NULL; 1187 } 1188 /* 1189 * There is at least one other VA mapping this page. Place 1190 * this entry after the header. 1191 */ 1192 else { 1193 npv = get_pv_entry(); 1194 npv->pv_va = va; 1195 npv->pv_pmap = pmap; 1196 npv->pv_next = pv->pv_next; 1197 pv->pv_next = npv; 1198 } 1199 splx(s); 1200 } 1201 1202 /* 1203 * Increment counters 1204 */ 1205 pmap->pm_stats.resident_count++; 1206 if (wired) 1207 pmap->pm_stats.wired_count++; 1208 1209validate: 1210 /* 1211 * Now validate mapping with desired protection/wiring. 1212 */ 1213 npte = (pt_entry_t) ((int) (pa | pte_prot(pmap, prot) | PG_V)); 1214 1215 /* 1216 * When forking (copy-on-write, etc): A process will turn off write 1217 * permissions for any of its writable pages. If the data (object) is 1218 * only referred to by one process, the processes map is modified 1219 * directly as opposed to using the object manipulation routine. When 1220 * using pmap_protect, the modified bits are not kept in the vm_page_t 1221 * data structure. Therefore, when using pmap_enter in vm_fault to 1222 * bring back writability of a page, there has been no memory of the 1223 * modified or referenced bits except at the pte level. this clause 1224 * supports the carryover of the modified and used (referenced) bits. 1225 */ 1226 if (pa == opa) 1227 (int) npte |= (int) *pte & (PG_M | PG_U); 1228 1229 if (wired) 1230 (int) npte |= PG_W; 1231 if (va < UPT_MIN_ADDRESS) 1232 (int) npte |= PG_u; 1233 else if (va < UPT_MAX_ADDRESS) 1234 (int) npte |= PG_u | PG_RW; 1235 1236 if (*pte != npte) { 1237 if (*pte) 1238 ptevalid++; 1239 *pte = npte; 1240 } 1241 if (ptevalid) { 1242 pmap_update(); 1243 } else { 1244 pmap_use_pt(pmap, va); 1245 } 1246} 1247 1248/* 1249 * Add a list of wired pages to the kva 1250 * this routine is only used for temporary 1251 * kernel mappings that do not need to have 1252 * page modification or references recorded. 1253 * Note that old mappings are simply written 1254 * over. The page *must* be wired. 1255 */ 1256void 1257pmap_qenter(va, m, count) 1258 vm_offset_t va; 1259 vm_page_t *m; 1260 int count; 1261{ 1262 int i; 1263 int anyvalid = 0; 1264 register pt_entry_t *pte; 1265 1266 for (i = 0; i < count; i++) { 1267 pte = vtopte(va + i * NBPG); 1268 if (*pte) 1269 anyvalid++; 1270 *pte = (pt_entry_t) ((int) (VM_PAGE_TO_PHYS(m[i]) | PG_RW | PG_V)); 1271 } 1272 if (anyvalid) 1273 pmap_update(); 1274} 1275/* 1276 * this routine jerks page mappings from the 1277 * kernel -- it is meant only for temporary mappings. 1278 */ 1279void 1280pmap_qremove(va, count) 1281 vm_offset_t va; 1282 int count; 1283{ 1284 int i; 1285 register pt_entry_t *pte; 1286 1287 for (i = 0; i < count; i++) { 1288 pte = vtopte(va + i * NBPG); 1289 *pte = 0; 1290 } 1291 pmap_update(); 1292} 1293 1294/* 1295 * add a wired page to the kva 1296 * note that in order for the mapping to take effect -- you 1297 * should do a pmap_update after doing the pmap_kenter... 1298 */ 1299void 1300pmap_kenter(va, pa) 1301 vm_offset_t va; 1302 register vm_offset_t pa; 1303{ 1304 register pt_entry_t *pte; 1305 int wasvalid = 0; 1306 1307 pte = vtopte(va); 1308 1309 if (*pte) 1310 wasvalid++; 1311 1312 *pte = (pt_entry_t) ((int) (pa | PG_RW | PG_V)); 1313 1314 if (wasvalid) 1315 pmap_update(); 1316} 1317 1318/* 1319 * remove a page from the kernel pagetables 1320 */ 1321void 1322pmap_kremove(va) 1323 vm_offset_t va; 1324{ 1325 register pt_entry_t *pte; 1326 1327 pte = vtopte(va); 1328 1329 *pte = (pt_entry_t) 0; 1330 pmap_update(); 1331} 1332 1333/* 1334 * this code makes some *MAJOR* assumptions: 1335 * 1. Current pmap & pmap exists. 1336 * 2. Not wired. 1337 * 3. Read access. 1338 * 4. No page table pages. 1339 * 5. Tlbflush is deferred to calling procedure. 1340 * 6. Page IS managed. 1341 * but is *MUCH* faster than pmap_enter... 1342 */ 1343 1344static inline void 1345pmap_enter_quick(pmap, va, pa) 1346 register pmap_t pmap; 1347 vm_offset_t va; 1348 register vm_offset_t pa; 1349{ 1350 register pt_entry_t *pte; 1351 register pv_entry_t pv, npv; 1352 int s; 1353 1354 /* 1355 * Enter on the PV list if part of our managed memory Note that we 1356 * raise IPL while manipulating pv_table since pmap_enter can be 1357 * called at interrupt time. 1358 */ 1359 1360 pte = vtopte(va); 1361 1362 /* a fault on the page table might occur here */ 1363 if (*pte) { 1364 pmap_remove(pmap, va, va + PAGE_SIZE); 1365 } 1366 pv = pa_to_pvh(pa); 1367 s = splhigh(); 1368 /* 1369 * No entries yet, use header as the first entry 1370 */ 1371 if (pv->pv_pmap == NULL) { 1372 pv->pv_pmap = pmap; 1373 pv->pv_va = va; 1374 pv->pv_next = NULL; 1375 } 1376 /* 1377 * There is at least one other VA mapping this page. Place this entry 1378 * after the header. 1379 */ 1380 else { 1381 npv = get_pv_entry(); 1382 npv->pv_va = va; 1383 npv->pv_pmap = pmap; 1384 npv->pv_next = pv->pv_next; 1385 pv->pv_next = npv; 1386 } 1387 splx(s); 1388 1389 /* 1390 * Increment counters 1391 */ 1392 pmap->pm_stats.resident_count++; 1393 1394 /* 1395 * Now validate mapping with desired protection/wiring. 1396 */ 1397 *pte = (pt_entry_t) ((int) (pa | PG_V | PG_u)); 1398 1399 pmap_use_pt(pmap, va); 1400 1401 return; 1402} 1403 1404#define MAX_INIT_PT (512 * 4096) 1405/* 1406 * pmap_object_init_pt preloads the ptes for a given object 1407 * into the specified pmap. This eliminates the blast of soft 1408 * faults on process startup and immediately after an mmap. 1409 */ 1410void 1411pmap_object_init_pt(pmap, addr, object, pindex, size) 1412 pmap_t pmap; 1413 vm_offset_t addr; 1414 vm_object_t object; 1415 vm_pindex_t pindex; 1416 vm_size_t size; 1417{ 1418 vm_offset_t tmpidx; 1419 int psize; 1420 vm_page_t p; 1421 int objpgs; 1422 1423 if (!pmap || ((size > MAX_INIT_PT) && 1424 (object->resident_page_count > MAX_INIT_PT / PAGE_SIZE))) { 1425 return; 1426 } 1427 1428 psize = (size >> PAGE_SHIFT); 1429 /* 1430 * if we are processing a major portion of the object, then scan the 1431 * entire thing. 1432 */ 1433 if (psize > (object->size >> 2)) { 1434 objpgs = psize; 1435 1436 for (p = object->memq.tqh_first; 1437 ((objpgs > 0) && (p != NULL)); 1438 p = p->listq.tqe_next) { 1439 1440 tmpidx = p->pindex; 1441 if (tmpidx < pindex) { 1442 continue; 1443 } 1444 tmpidx -= pindex; 1445 if (tmpidx >= psize) { 1446 continue; 1447 } 1448 if (((p->flags & (PG_ACTIVE | PG_INACTIVE | PG_CACHE)) != 0) && 1449 ((p->valid & VM_PAGE_BITS_ALL) == VM_PAGE_BITS_ALL) && 1450 (p->bmapped == 0) && 1451 (p->busy == 0) && 1452 (p->flags & (PG_BUSY | PG_FICTITIOUS)) == 0) { 1453 if (p->flags & PG_CACHE) 1454 vm_page_deactivate(p); 1455 vm_page_hold(p); 1456 p->flags |= PG_MAPPED; 1457 pmap_enter_quick(pmap, 1458 addr + (tmpidx << PAGE_SHIFT), 1459 VM_PAGE_TO_PHYS(p)); 1460 vm_page_unhold(p); 1461 } 1462 objpgs -= 1; 1463 } 1464 } else { 1465 /* 1466 * else lookup the pages one-by-one. 1467 */ 1468 for (tmpidx = 0; tmpidx < psize; tmpidx += 1) { 1469 p = vm_page_lookup(object, tmpidx + pindex); 1470 if (p && 1471 ((p->flags & (PG_ACTIVE | PG_INACTIVE | PG_CACHE)) != 0) && 1472 (p->bmapped == 0) && 1473 (p->busy == 0) && 1474 ((p->valid & VM_PAGE_BITS_ALL) == VM_PAGE_BITS_ALL) && 1475 (p->flags & (PG_BUSY | PG_FICTITIOUS)) == 0) { 1476 if (p->flags & PG_CACHE) 1477 vm_page_deactivate(p); 1478 vm_page_hold(p); 1479 p->flags |= PG_MAPPED; 1480 pmap_enter_quick(pmap, 1481 addr + (tmpidx << PAGE_SHIFT), 1482 VM_PAGE_TO_PHYS(p)); 1483 vm_page_unhold(p); 1484 } 1485 } 1486 } 1487} 1488 1489/* 1490 * Routine: pmap_change_wiring 1491 * Function: Change the wiring attribute for a map/virtual-address 1492 * pair. 1493 * In/out conditions: 1494 * The mapping must already exist in the pmap. 1495 */ 1496void 1497pmap_change_wiring(pmap, va, wired) 1498 register pmap_t pmap; 1499 vm_offset_t va; 1500 boolean_t wired; 1501{ 1502 register pt_entry_t *pte; 1503 1504 if (pmap == NULL) 1505 return; 1506 1507 pte = pmap_pte(pmap, va); 1508 1509 if (wired && !pmap_pte_w(pte)) 1510 pmap->pm_stats.wired_count++; 1511 else if (!wired && pmap_pte_w(pte)) 1512 pmap->pm_stats.wired_count--; 1513 1514 /* 1515 * Wiring is not a hardware characteristic so there is no need to 1516 * invalidate TLB. 1517 */ 1518 pmap_pte_set_w(pte, wired); 1519 /* 1520 * When unwiring, set the modified bit in the pte -- could have been 1521 * changed by the kernel 1522 */ 1523 if (!wired) 1524 (int) *pte |= PG_M; 1525} 1526 1527 1528 1529/* 1530 * Copy the range specified by src_addr/len 1531 * from the source map to the range dst_addr/len 1532 * in the destination map. 1533 * 1534 * This routine is only advisory and need not do anything. 1535 */ 1536void 1537pmap_copy(dst_pmap, src_pmap, dst_addr, len, src_addr) 1538 pmap_t dst_pmap, src_pmap; 1539 vm_offset_t dst_addr; 1540 vm_size_t len; 1541 vm_offset_t src_addr; 1542{ 1543} 1544 1545/* 1546 * Routine: pmap_kernel 1547 * Function: 1548 * Returns the physical map handle for the kernel. 1549 */ 1550pmap_t 1551pmap_kernel() 1552{ 1553 return (kernel_pmap); 1554} 1555 1556/* 1557 * pmap_zero_page zeros the specified (machine independent) 1558 * page by mapping the page into virtual memory and using 1559 * bzero to clear its contents, one machine dependent page 1560 * at a time. 1561 */ 1562void 1563pmap_zero_page(phys) 1564 vm_offset_t phys; 1565{ 1566 if (*(int *) CMAP2) 1567 panic("pmap_zero_page: CMAP busy"); 1568 1569 *(int *) CMAP2 = PG_V | PG_KW | i386_trunc_page(phys); 1570 bzero(CADDR2, NBPG); 1571 1572 *(int *) CMAP2 = 0; 1573 pmap_update(); 1574} 1575 1576/* 1577 * pmap_copy_page copies the specified (machine independent) 1578 * page by mapping the page into virtual memory and using 1579 * bcopy to copy the page, one machine dependent page at a 1580 * time. 1581 */ 1582void 1583pmap_copy_page(src, dst) 1584 vm_offset_t src; 1585 vm_offset_t dst; 1586{ 1587 if (*(int *) CMAP1 || *(int *) CMAP2) 1588 panic("pmap_copy_page: CMAP busy"); 1589 1590 *(int *) CMAP1 = PG_V | PG_KW | i386_trunc_page(src); 1591 *(int *) CMAP2 = PG_V | PG_KW | i386_trunc_page(dst); 1592 1593#if __GNUC__ > 1 1594 memcpy(CADDR2, CADDR1, NBPG); 1595#else 1596 bcopy(CADDR1, CADDR2, NBPG); 1597#endif 1598 *(int *) CMAP1 = 0; 1599 *(int *) CMAP2 = 0; 1600 pmap_update(); 1601} 1602 1603 1604/* 1605 * Routine: pmap_pageable 1606 * Function: 1607 * Make the specified pages (by pmap, offset) 1608 * pageable (or not) as requested. 1609 * 1610 * A page which is not pageable may not take 1611 * a fault; therefore, its page table entry 1612 * must remain valid for the duration. 1613 * 1614 * This routine is merely advisory; pmap_enter 1615 * will specify that these pages are to be wired 1616 * down (or not) as appropriate. 1617 */ 1618void 1619pmap_pageable(pmap, sva, eva, pageable) 1620 pmap_t pmap; 1621 vm_offset_t sva, eva; 1622 boolean_t pageable; 1623{ 1624} 1625 1626/* 1627 * this routine returns true if a physical page resides 1628 * in the given pmap. 1629 */ 1630boolean_t 1631pmap_page_exists(pmap, pa) 1632 pmap_t pmap; 1633 vm_offset_t pa; 1634{ 1635 register pv_entry_t pv; 1636 int s; 1637 1638 if (!pmap_is_managed(pa)) 1639 return FALSE; 1640 1641 pv = pa_to_pvh(pa); 1642 s = splhigh(); 1643 1644 /* 1645 * Not found, check current mappings returning immediately if found. 1646 */ 1647 if (pv->pv_pmap != NULL) { 1648 for (; pv; pv = pv->pv_next) { 1649 if (pv->pv_pmap == pmap) { 1650 splx(s); 1651 return TRUE; 1652 } 1653 } 1654 } 1655 splx(s); 1656 return (FALSE); 1657} 1658 1659/* 1660 * pmap_testbit tests bits in pte's 1661 * note that the testbit/changebit routines are inline, 1662 * and a lot of things compile-time evaluate. 1663 */ 1664static __inline boolean_t 1665pmap_testbit(pa, bit) 1666 register vm_offset_t pa; 1667 int bit; 1668{ 1669 register pv_entry_t pv; 1670 pt_entry_t *pte; 1671 int s; 1672 1673 if (!pmap_is_managed(pa)) 1674 return FALSE; 1675 1676 pv = pa_to_pvh(pa); 1677 s = splhigh(); 1678 1679 /* 1680 * Not found, check current mappings returning immediately if found. 1681 */ 1682 if (pv->pv_pmap != NULL) { 1683 for (; pv; pv = pv->pv_next) { 1684 /* 1685 * if the bit being tested is the modified bit, then 1686 * mark UPAGES as always modified, and ptes as never 1687 * modified. 1688 */ 1689 if (bit & PG_U) { 1690 if ((pv->pv_va >= clean_sva) && (pv->pv_va < clean_eva)) { 1691 continue; 1692 } 1693 } 1694 if (bit & PG_M) { 1695 if (pv->pv_va >= USRSTACK) { 1696 if (pv->pv_va >= clean_sva && pv->pv_va < clean_eva) { 1697 continue; 1698 } 1699 if (pv->pv_va < USRSTACK + (UPAGES * NBPG)) { 1700 splx(s); 1701 return TRUE; 1702 } else if (pv->pv_va < KERNBASE) { 1703 splx(s); 1704 return FALSE; 1705 } 1706 } 1707 } 1708 if (!pv->pv_pmap) { 1709 printf("Null pmap (tb) at va: 0x%lx\n", pv->pv_va); 1710 continue; 1711 } 1712 pte = pmap_pte(pv->pv_pmap, pv->pv_va); 1713 if ((int) *pte & bit) { 1714 splx(s); 1715 return TRUE; 1716 } 1717 } 1718 } 1719 splx(s); 1720 return (FALSE); 1721} 1722 1723/* 1724 * this routine is used to modify bits in ptes 1725 */ 1726static __inline void 1727pmap_changebit(pa, bit, setem) 1728 vm_offset_t pa; 1729 int bit; 1730 boolean_t setem; 1731{ 1732 register pv_entry_t pv; 1733 register pt_entry_t *pte, npte; 1734 vm_offset_t va; 1735 int s; 1736 1737 if (!pmap_is_managed(pa)) 1738 return; 1739 1740 pv = pa_to_pvh(pa); 1741 s = splhigh(); 1742 1743 /* 1744 * Loop over all current mappings setting/clearing as appropos If 1745 * setting RO do we need to clear the VAC? 1746 */ 1747 if (pv->pv_pmap != NULL) { 1748 for (; pv; pv = pv->pv_next) { 1749 va = pv->pv_va; 1750 1751 /* 1752 * don't write protect pager mappings 1753 */ 1754 if (!setem && (bit == PG_RW)) { 1755 if (va >= clean_sva && va < clean_eva) 1756 continue; 1757 } 1758 if (!pv->pv_pmap) { 1759 printf("Null pmap (cb) at va: 0x%lx\n", va); 1760 continue; 1761 } 1762 pte = pmap_pte(pv->pv_pmap, va); 1763 if (setem) 1764 (int) npte = (int) *pte | bit; 1765 else 1766 (int) npte = (int) *pte & ~bit; 1767 *pte = npte; 1768 } 1769 } 1770 splx(s); 1771 pmap_update(); 1772} 1773 1774/* 1775 * pmap_page_protect: 1776 * 1777 * Lower the permission for all mappings to a given page. 1778 */ 1779void 1780pmap_page_protect(phys, prot) 1781 vm_offset_t phys; 1782 vm_prot_t prot; 1783{ 1784 if ((prot & VM_PROT_WRITE) == 0) { 1785 if (prot & (VM_PROT_READ | VM_PROT_EXECUTE)) 1786 pmap_changebit(phys, PG_RW, FALSE); 1787 else 1788 pmap_remove_all(phys); 1789 } 1790} 1791 1792vm_offset_t 1793pmap_phys_address(ppn) 1794 int ppn; 1795{ 1796 return (i386_ptob(ppn)); 1797} 1798 1799/* 1800 * pmap_is_referenced: 1801 * 1802 * Return whether or not the specified physical page was referenced 1803 * by any physical maps. 1804 */ 1805boolean_t 1806pmap_is_referenced(vm_offset_t pa) 1807{ 1808 return pmap_testbit((pa), PG_U); 1809} 1810 1811/* 1812 * pmap_is_modified: 1813 * 1814 * Return whether or not the specified physical page was modified 1815 * in any physical maps. 1816 */ 1817boolean_t 1818pmap_is_modified(vm_offset_t pa) 1819{ 1820 return pmap_testbit((pa), PG_M); 1821} 1822 1823/* 1824 * Clear the modify bits on the specified physical page. 1825 */ 1826void 1827pmap_clear_modify(vm_offset_t pa) 1828{ 1829 pmap_changebit((pa), PG_M, FALSE); 1830} 1831 1832/* 1833 * pmap_clear_reference: 1834 * 1835 * Clear the reference bit on the specified physical page. 1836 */ 1837void 1838pmap_clear_reference(vm_offset_t pa) 1839{ 1840 pmap_changebit((pa), PG_U, FALSE); 1841} 1842 1843/* 1844 * Miscellaneous support routines follow 1845 */ 1846 1847static void 1848i386_protection_init() 1849{ 1850 register int *kp, prot; 1851 1852 kp = protection_codes; 1853 for (prot = 0; prot < 8; prot++) { 1854 switch (prot) { 1855 case VM_PROT_NONE | VM_PROT_NONE | VM_PROT_NONE: 1856 /* 1857 * Read access is also 0. There isn't any execute bit, 1858 * so just make it readable. 1859 */ 1860 case VM_PROT_READ | VM_PROT_NONE | VM_PROT_NONE: 1861 case VM_PROT_READ | VM_PROT_NONE | VM_PROT_EXECUTE: 1862 case VM_PROT_NONE | VM_PROT_NONE | VM_PROT_EXECUTE: 1863 *kp++ = 0; 1864 break; 1865 case VM_PROT_NONE | VM_PROT_WRITE | VM_PROT_NONE: 1866 case VM_PROT_NONE | VM_PROT_WRITE | VM_PROT_EXECUTE: 1867 case VM_PROT_READ | VM_PROT_WRITE | VM_PROT_NONE: 1868 case VM_PROT_READ | VM_PROT_WRITE | VM_PROT_EXECUTE: 1869 *kp++ = PG_RW; 1870 break; 1871 } 1872 } 1873} 1874 1875/* 1876 * Map a set of physical memory pages into the kernel virtual 1877 * address space. Return a pointer to where it is mapped. This 1878 * routine is intended to be used for mapping device memory, 1879 * NOT real memory. The non-cacheable bits are set on each 1880 * mapped page. 1881 */ 1882void * 1883pmap_mapdev(pa, size) 1884 vm_offset_t pa; 1885 vm_size_t size; 1886{ 1887 vm_offset_t va, tmpva; 1888 pt_entry_t *pte; 1889 1890 pa = trunc_page(pa); 1891 size = roundup(size, PAGE_SIZE); 1892 1893 va = kmem_alloc_pageable(kernel_map, size); 1894 if (!va) 1895 panic("pmap_mapdev: Couldn't alloc kernel virtual memory"); 1896 1897 for (tmpva = va; size > 0;) { 1898 pte = vtopte(tmpva); 1899 *pte = (pt_entry_t) ((int) (pa | PG_RW | PG_V | PG_N)); 1900 size -= PAGE_SIZE; 1901 tmpva += PAGE_SIZE; 1902 pa += PAGE_SIZE; 1903 } 1904 pmap_update(); 1905 1906 return ((void *) va); 1907} 1908 1909#ifdef DEBUG 1910/* print address space of pmap*/ 1911void 1912pads(pm) 1913 pmap_t pm; 1914{ 1915 unsigned va, i, j; 1916 pt_entry_t *ptep; 1917 1918 if (pm == kernel_pmap) 1919 return; 1920 for (i = 0; i < 1024; i++) 1921 if (pm->pm_pdir[i]) 1922 for (j = 0; j < 1024; j++) { 1923 va = (i << PD_SHIFT) + (j << PG_SHIFT); 1924 if (pm == kernel_pmap && va < KERNBASE) 1925 continue; 1926 if (pm != kernel_pmap && va > UPT_MAX_ADDRESS) 1927 continue; 1928 ptep = pmap_pte(pm, va); 1929 if (pmap_pte_v(ptep)) 1930 printf("%x:%x ", va, *(int *) ptep); 1931 }; 1932 1933} 1934 1935void 1936pmap_pvdump(pa) 1937 vm_offset_t pa; 1938{ 1939 register pv_entry_t pv; 1940 1941 printf("pa %x", pa); 1942 for (pv = pa_to_pvh(pa); pv; pv = pv->pv_next) { 1943#ifdef used_to_be 1944 printf(" -> pmap %x, va %x, flags %x", 1945 pv->pv_pmap, pv->pv_va, pv->pv_flags); 1946#endif 1947 printf(" -> pmap %x, va %x", 1948 pv->pv_pmap, pv->pv_va); 1949 pads(pv->pv_pmap); 1950 } 1951 printf(" "); 1952} 1953#endif 1954