pmap.c revision 90361
1/* 2 * Copyright (c) 1991 Regents of the University of California. 3 * All rights reserved. 4 * Copyright (c) 1994 John S. Dyson 5 * All rights reserved. 6 * Copyright (c) 1994 David Greenman 7 * All rights reserved. 8 * Copyright (c) 1998,2000 Doug Rabson 9 * All rights reserved. 10 * 11 * This code is derived from software contributed to Berkeley by 12 * the Systems Programming Group of the University of Utah Computer 13 * Science Department and William Jolitz of UUNET Technologies Inc. 14 * 15 * Redistribution and use in source and binary forms, with or without 16 * modification, are permitted provided that the following conditions 17 * are met: 18 * 1. Redistributions of source code must retain the above copyright 19 * notice, this list of conditions and the following disclaimer. 20 * 2. Redistributions in binary form must reproduce the above copyright 21 * notice, this list of conditions and the following disclaimer in the 22 * documentation and/or other materials provided with the distribution. 23 * 3. All advertising materials mentioning features or use of this software 24 * must display the following acknowledgement: 25 * This product includes software developed by the University of 26 * California, Berkeley and its contributors. 27 * 4. Neither the name of the University nor the names of its contributors 28 * may be used to endorse or promote products derived from this software 29 * without specific prior written permission. 30 * 31 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 32 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 33 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 34 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 35 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 36 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 37 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 38 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 39 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 40 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 41 * SUCH DAMAGE. 42 * 43 * from: @(#)pmap.c 7.7 (Berkeley) 5/12/91 44 * from: i386 Id: pmap.c,v 1.193 1998/04/19 15:22:48 bde Exp 45 * with some ideas from NetBSD's alpha pmap 46 * $FreeBSD: head/sys/ia64/ia64/pmap.c 90361 2002-02-07 20:58:47Z julian $ 47 */ 48 49/* 50 * Manages physical address maps. 51 * 52 * In addition to hardware address maps, this 53 * module is called upon to provide software-use-only 54 * maps which may or may not be stored in the same 55 * form as hardware maps. These pseudo-maps are 56 * used to store intermediate results from copy 57 * operations to and from address spaces. 58 * 59 * Since the information managed by this module is 60 * also stored by the logical address mapping module, 61 * this module may throw away valid virtual-to-physical 62 * mappings at almost any time. However, invalidations 63 * of virtual-to-physical mappings must be done as 64 * requested. 65 * 66 * In order to cope with hardware architectures which 67 * make virtual-to-physical map invalidates expensive, 68 * this module may delay invalidate or reduced protection 69 * operations until such time as they are actually 70 * necessary. This module is given full information as 71 * to which processors are currently using which maps, 72 * and to when physical maps must be made correct. 73 */ 74 75/* 76 * Following the Linux model, region IDs are allocated in groups of 77 * eight so that a single region ID can be used for as many RRs as we 78 * want by encoding the RR number into the low bits of the ID. 79 * 80 * We reserve region ID 0 for the kernel and allocate the remaining 81 * IDs for user pmaps. 82 * 83 * Region 0..4 84 * User virtually mapped 85 * 86 * Region 5 87 * Kernel virtually mapped 88 * 89 * Region 6 90 * Kernel physically mapped uncacheable 91 * 92 * Region 7 93 * Kernel physically mapped cacheable 94 */ 95 96#include <sys/param.h> 97#include <sys/kernel.h> 98#include <sys/lock.h> 99#include <sys/malloc.h> 100#include <sys/mman.h> 101#include <sys/msgbuf.h> 102#include <sys/mutex.h> 103#include <sys/proc.h> 104#include <sys/sx.h> 105#include <sys/systm.h> 106#include <sys/vmmeter.h> 107#include <sys/sysctl.h> 108 109#include <vm/vm.h> 110#include <vm/vm_param.h> 111#include <vm/vm_kern.h> 112#include <vm/vm_page.h> 113#include <vm/vm_map.h> 114#include <vm/vm_object.h> 115#include <vm/vm_extern.h> 116#include <vm/vm_pageout.h> 117#include <vm/vm_pager.h> 118#include <vm/vm_zone.h> 119 120#include <sys/user.h> 121 122#include <machine/pal.h> 123#include <machine/md_var.h> 124 125MALLOC_DEFINE(M_PMAP, "PMAP", "PMAP Structures"); 126 127#ifndef PMAP_SHPGPERPROC 128#define PMAP_SHPGPERPROC 200 129#endif 130 131#if defined(DIAGNOSTIC) 132#define PMAP_DIAGNOSTIC 133#endif 134 135#define MINPV 2048 136 137#if 0 138#define PMAP_DIAGNOSTIC 139#define PMAP_DEBUG 140#endif 141 142#if !defined(PMAP_DIAGNOSTIC) 143#define PMAP_INLINE __inline 144#else 145#define PMAP_INLINE 146#endif 147 148/* 149 * Get PDEs and PTEs for user/kernel address space 150 */ 151#define pmap_pte_w(pte) ((pte)->pte_ig & PTE_IG_WIRED) 152#define pmap_pte_managed(pte) ((pte)->pte_ig & PTE_IG_MANAGED) 153#define pmap_pte_v(pte) ((pte)->pte_p) 154#define pmap_pte_pa(pte) (((pte)->pte_ppn) << 12) 155#define pmap_pte_prot(pte) (((pte)->pte_ar << 2) | (pte)->pte_pl) 156 157#define pmap_pte_set_w(pte, v) ((v)?((pte)->pte_ig |= PTE_IG_WIRED) \ 158 :((pte)->pte_ig &= ~PTE_IG_WIRED)) 159#define pmap_pte_set_prot(pte, v) do { \ 160 (pte)->pte_ar = v >> 2; \ 161 (pte)->pte_pl = v & 3; \ 162} while (0) 163 164/* 165 * Given a map and a machine independent protection code, 166 * convert to an ia64 protection code. 167 */ 168#define pte_prot(m, p) (protection_codes[m == pmap_kernel() ? 0 : 1][p]) 169#define pte_prot_pl(m, p) (pte_prot(m, p) & 3) 170#define pte_prot_ar(m, p) (pte_prot(m, p) >> 2) 171int protection_codes[2][8]; 172 173/* 174 * Return non-zero if this pmap is currently active 175 */ 176#define pmap_isactive(pmap) (pmap->pm_active) 177 178/* 179 * Statically allocated kernel pmap 180 */ 181static struct pmap kernel_pmap_store; 182pmap_t kernel_pmap; 183 184vm_offset_t avail_start; /* PA of first available physical page */ 185vm_offset_t avail_end; /* PA of last available physical page */ 186vm_offset_t virtual_avail; /* VA of first avail page (after kernel bss) */ 187vm_offset_t virtual_end; /* VA of last avail page (end of kernel AS) */ 188static boolean_t pmap_initialized = FALSE; /* Has pmap_init completed? */ 189 190vm_offset_t vhpt_base, vhpt_size; 191 192/* 193 * We use an object to own the kernel's 'page tables'. For simplicity, 194 * we use one page directory to index a set of pages containing 195 * ia64_lptes. This gives us up to 2Gb of kernel virtual space. 196 */ 197static vm_object_t kptobj; 198static int nkpt; 199static struct ia64_lpte **kptdir; 200#define KPTE_DIR_INDEX(va) \ 201 ((va >> (2*PAGE_SHIFT-5)) & ((1<<(PAGE_SHIFT-3))-1)) 202#define KPTE_PTE_INDEX(va) \ 203 ((va >> PAGE_SHIFT) & ((1<<(PAGE_SHIFT-5))-1)) 204#define NKPTEPG (PAGE_SIZE / sizeof(struct ia64_lpte)) 205 206vm_offset_t kernel_vm_end; 207 208/* 209 * Values for ptc.e. XXX values for SKI. 210 */ 211static u_int64_t pmap_ptc_e_base = 0x100000000; 212static u_int64_t pmap_ptc_e_count1 = 3; 213static u_int64_t pmap_ptc_e_count2 = 2; 214static u_int64_t pmap_ptc_e_stride1 = 0x2000; 215static u_int64_t pmap_ptc_e_stride2 = 0x100000000; 216 217/* 218 * Data for the RID allocator 219 */ 220static u_int64_t *pmap_ridbusy; 221static int pmap_ridmax, pmap_ridcount; 222 223/* 224 * Data for the pv entry allocation mechanism 225 */ 226static vm_zone_t pvzone; 227static struct vm_zone pvzone_store; 228static struct vm_object pvzone_obj; 229static int pv_entry_count = 0, pv_entry_max = 0, pv_entry_high_water = 0; 230static int pmap_pagedaemon_waken = 0; 231static struct pv_entry *pvinit; 232static struct pv_entry *pvbootentries; 233static int pvbootnext, pvbootmax; 234 235/* 236 * Data for allocating PTEs for user processes. 237 */ 238static vm_zone_t ptezone; 239static struct vm_zone ptezone_store; 240static struct vm_object ptezone_obj; 241static struct ia64_lpte *pteinit; 242 243/* 244 * VHPT instrumentation. 245 */ 246static int pmap_vhpt_inserts; 247static int pmap_vhpt_collisions; 248static int pmap_vhpt_resident; 249SYSCTL_DECL(_vm_stats); 250SYSCTL_NODE(_vm_stats, OID_AUTO, vhpt, CTLFLAG_RD, 0, ""); 251SYSCTL_INT(_vm_stats_vhpt, OID_AUTO, inserts, CTLFLAG_RD, 252 &pmap_vhpt_inserts, 0, ""); 253SYSCTL_INT(_vm_stats_vhpt, OID_AUTO, collisions, CTLFLAG_RD, 254 &pmap_vhpt_collisions, 0, ""); 255SYSCTL_INT(_vm_stats_vhpt, OID_AUTO, resident, CTLFLAG_RD, 256 &pmap_vhpt_resident, 0, ""); 257 258static PMAP_INLINE void free_pv_entry __P((pv_entry_t pv)); 259static pv_entry_t get_pv_entry __P((void)); 260static void ia64_protection_init __P((void)); 261 262static void pmap_invalidate_all __P((pmap_t pmap)); 263static void pmap_remove_all __P((vm_page_t m)); 264static void pmap_enter_quick __P((pmap_t pmap, vm_offset_t va, vm_page_t m)); 265 266vm_offset_t 267pmap_steal_memory(vm_size_t size) 268{ 269 vm_size_t bank_size; 270 vm_offset_t pa, va; 271 272 size = round_page(size); 273 274 bank_size = phys_avail[1] - phys_avail[0]; 275 while (size > bank_size) { 276 int i; 277 for (i = 0; phys_avail[i+2]; i+= 2) { 278 phys_avail[i] = phys_avail[i+2]; 279 phys_avail[i+1] = phys_avail[i+3]; 280 } 281 phys_avail[i] = 0; 282 phys_avail[i+1] = 0; 283 if (!phys_avail[0]) 284 panic("pmap_steal_memory: out of memory"); 285 bank_size = phys_avail[1] - phys_avail[0]; 286 } 287 288 pa = phys_avail[0]; 289 phys_avail[0] += size; 290 291 va = IA64_PHYS_TO_RR7(pa); 292 bzero((caddr_t) va, size); 293 return va; 294} 295 296/* 297 * Bootstrap the system enough to run with virtual memory. 298 */ 299void 300pmap_bootstrap() 301{ 302 int i, j, count, ridbits; 303 struct ia64_pal_result res; 304 305 /* 306 * Query the PAL Code to find the loop parameters for the 307 * ptc.e instruction. 308 */ 309 res = ia64_call_pal_static(PAL_PTCE_INFO, 0, 0, 0); 310 if (res.pal_status != 0) 311 panic("Can't configure ptc.e parameters"); 312 pmap_ptc_e_base = res.pal_result[0]; 313 pmap_ptc_e_count1 = res.pal_result[1] >> 32; 314 pmap_ptc_e_count2 = res.pal_result[1] & ((1L<<32) - 1); 315 pmap_ptc_e_stride1 = res.pal_result[2] >> 32; 316 pmap_ptc_e_stride2 = res.pal_result[2] & ((1L<<32) - 1); 317 if (bootverbose) 318 printf("ptc.e base=0x%lx, count1=%ld, count2=%ld, " 319 "stride1=0x%lx, stride2=0x%lx\n", 320 pmap_ptc_e_base, 321 pmap_ptc_e_count1, 322 pmap_ptc_e_count2, 323 pmap_ptc_e_stride1, 324 pmap_ptc_e_stride2); 325 326 /* 327 * Setup RIDs. RIDs 0..7 are reserved for the kernel. 328 */ 329 res = ia64_call_pal_static(PAL_VM_SUMMARY, 0, 0, 0); 330 if (res.pal_status != 0) { 331 if (bootverbose) 332 printf("Can't read VM Summary - assuming 18 Region ID bits\n"); 333 ridbits = 18; /* guaranteed minimum */ 334 } else { 335 ridbits = (res.pal_result[1] >> 8) & 0xff; 336 if (bootverbose) 337 printf("Processor supports %d Region ID bits\n", 338 ridbits); 339 } 340 pmap_ridmax = (1 << ridbits); 341 pmap_ridcount = 8; 342 pmap_ridbusy = (u_int64_t *) 343 pmap_steal_memory(pmap_ridmax / 8); 344 bzero(pmap_ridbusy, pmap_ridmax / 8); 345 pmap_ridbusy[0] |= 0xff; 346 347 /* 348 * Allocate some memory for initial kernel 'page tables'. 349 */ 350 kptdir = (struct ia64_lpte **) pmap_steal_memory(PAGE_SIZE); 351 for (i = 0; i < NKPT; i++) { 352 kptdir[i] = (struct ia64_lpte *) pmap_steal_memory(PAGE_SIZE); 353 } 354 nkpt = NKPT; 355 356 avail_start = phys_avail[0]; 357 for (i = 0; phys_avail[i+2]; i+= 2) ; 358 avail_end = phys_avail[i+1]; 359 count = i+2; 360 361 /* 362 * Figure out a useful size for the VHPT, based on the size of 363 * physical memory and try to locate a region which is large 364 * enough to contain the VHPT (which must be a power of two in 365 * size and aligned to a natural boundary). 366 */ 367 vhpt_size = 15; 368 while ((1<<vhpt_size) < ia64_btop(avail_end - avail_start) * 32) 369 vhpt_size++; 370 371 vhpt_base = 0; 372 while (!vhpt_base) { 373 vm_offset_t mask; 374 if (bootverbose) 375 printf("Trying VHPT size 0x%lx\n", (1L<<vhpt_size)); 376 mask = (1L << vhpt_size) - 1; 377 for (i = 0; i < count; i += 2) { 378 vm_offset_t base, limit; 379 base = (phys_avail[i] + mask) & ~mask; 380 limit = base + (1L << vhpt_size); 381 if (limit <= phys_avail[i+1]) 382 /* 383 * VHPT can fit in this region 384 */ 385 break; 386 } 387 if (!phys_avail[i]) { 388 /* 389 * Can't fit, try next smaller size. 390 */ 391 vhpt_size--; 392 } else { 393 vhpt_base = (phys_avail[i] + mask) & ~mask; 394 } 395 } 396 if (vhpt_size < 15) 397 panic("Can't find space for VHPT"); 398 399 if (bootverbose) 400 printf("Putting VHPT at %p\n", (void *) vhpt_base); 401 if (vhpt_base != phys_avail[i]) { 402 /* 403 * Split this region. 404 */ 405 if (bootverbose) 406 printf("Splitting [%p-%p]\n", 407 (void *) phys_avail[i], 408 (void *) phys_avail[i+1]); 409 for (j = count; j > i; j -= 2) { 410 phys_avail[j] = phys_avail[j-2]; 411 phys_avail[j+1] = phys_avail[j-2+1]; 412 } 413 phys_avail[count+2] = 0; 414 phys_avail[count+3] = 0; 415 phys_avail[i+1] = vhpt_base; 416 phys_avail[i+2] = vhpt_base + (1L << vhpt_size); 417 } else { 418 phys_avail[i] = vhpt_base + (1L << vhpt_size); 419 } 420 421 vhpt_base = IA64_PHYS_TO_RR7(vhpt_base); 422 bzero((void *) vhpt_base, (1L << vhpt_size)); 423 __asm __volatile("mov cr.pta=%0;; srlz.i;;" 424 :: "r" (vhpt_base + (1<<8) + (vhpt_size<<2) + 1)); 425 426 virtual_avail = IA64_RR_BASE(5); 427 virtual_end = IA64_RR_BASE(6)-1; 428 429 /* 430 * Initialize protection array. 431 */ 432 ia64_protection_init(); 433 434 /* 435 * The kernel's pmap is statically allocated so we don't have to use 436 * pmap_create, which is unlikely to work correctly at this part of 437 * the boot sequence (XXX and which no longer exists). 438 */ 439 kernel_pmap = &kernel_pmap_store; 440 for (i = 0; i < 5; i++) 441 kernel_pmap->pm_rid[i] = 0; 442 kernel_pmap->pm_count = 1; 443 kernel_pmap->pm_active = 1; 444 TAILQ_INIT(&kernel_pmap->pm_pvlist); 445 446 /* 447 * Region 5 is mapped via the vhpt. 448 */ 449 ia64_set_rr(IA64_RR_BASE(5), 450 (5 << 8) | (PAGE_SHIFT << 2) | 1); 451 452 /* 453 * Region 6 is direct mapped UC and region 7 is direct mapped 454 * WC. The details of this is controlled by the Alt {I,D}TLB 455 * handlers. Here we just make sure that they have the largest 456 * possible page size to minimise TLB usage. 457 */ 458 ia64_set_rr(IA64_RR_BASE(6), (6 << 8) | (28 << 2)); 459 ia64_set_rr(IA64_RR_BASE(7), (7 << 8) | (28 << 2)); 460 461 /* 462 * Set up proc0's PCB. 463 */ 464#if 0 465 thread0.td_pcb->pcb_hw.apcb_asn = 0; 466#endif 467 468 /* 469 * Reserve some memory for allocating pvs while bootstrapping 470 * the pv allocator. We need to have enough to cover mapping 471 * the kmem_alloc region used to allocate the initial_pvs in 472 * pmap_init. In general, the size of this region is 473 * appoximately (# physical pages) * (size of pv entry). 474 */ 475 pvbootmax = ((physmem * sizeof(struct pv_entry)) >> PAGE_SHIFT) + 128; 476 pvbootentries = (struct pv_entry *) 477 pmap_steal_memory(pvbootmax * sizeof(struct pv_entry)); 478 pvbootnext = 0; 479 480 /* 481 * Clear out any random TLB entries left over from booting. 482 */ 483 pmap_invalidate_all(kernel_pmap); 484} 485 486/* 487 * Initialize the pmap module. 488 * Called by vm_init, to initialize any structures that the pmap 489 * system needs to map virtual memory. 490 * pmap_init has been enhanced to support in a fairly consistant 491 * way, discontiguous physical memory. 492 */ 493void 494pmap_init(vm_offset_t phys_start, vm_offset_t phys_end) 495{ 496 int i; 497 int initial_pvs; 498 499 /* 500 * Allocate memory for random pmap data structures. Includes the 501 * pv_head_table. 502 */ 503 504 for(i = 0; i < vm_page_array_size; i++) { 505 vm_page_t m; 506 507 m = &vm_page_array[i]; 508 TAILQ_INIT(&m->md.pv_list); 509 m->md.pv_list_count = 0; 510 } 511 512 /* 513 * Init the pv free list and the PTE free list. 514 */ 515 initial_pvs = vm_page_array_size; 516 if (initial_pvs < MINPV) 517 initial_pvs = MINPV; 518 pvzone = &pvzone_store; 519 pvinit = (struct pv_entry *) kmem_alloc(kernel_map, 520 initial_pvs * sizeof (struct pv_entry)); 521 zbootinit(pvzone, "PV ENTRY", sizeof (struct pv_entry), pvinit, 522 vm_page_array_size); 523 524 ptezone = &ptezone_store; 525 pteinit = (struct ia64_lpte *) kmem_alloc(kernel_map, 526 initial_pvs * sizeof (struct ia64_lpte)); 527 zbootinit(ptezone, "PT ENTRY", sizeof (struct ia64_lpte), pteinit, 528 vm_page_array_size); 529 530 /* 531 * Create the object for the kernel's page tables. 532 */ 533 kptobj = vm_object_allocate(OBJT_DEFAULT, MAXKPT); 534 535 /* 536 * Now it is safe to enable pv_table recording. 537 */ 538 pmap_initialized = TRUE; 539} 540 541/* 542 * Initialize the address space (zone) for the pv_entries. Set a 543 * high water mark so that the system can recover from excessive 544 * numbers of pv entries. 545 */ 546void 547pmap_init2() 548{ 549 int shpgperproc = PMAP_SHPGPERPROC; 550 551 TUNABLE_INT_FETCH("vm.pmap.shpgperproc", &shpgperproc); 552 pv_entry_max = shpgperproc * maxproc + vm_page_array_size; 553 pv_entry_high_water = 9 * (pv_entry_max / 10); 554 zinitna(pvzone, &pvzone_obj, NULL, 0, pv_entry_max, ZONE_INTERRUPT, 1); 555 zinitna(ptezone, &ptezone_obj, NULL, 0, pv_entry_max, ZONE_INTERRUPT, 1); 556} 557 558 559/*************************************************** 560 * Manipulate TLBs for a pmap 561 ***************************************************/ 562 563static void 564pmap_invalidate_page(pmap_t pmap, vm_offset_t va) 565{ 566 KASSERT((pmap == kernel_pmap || pmap == PCPU_GET(current_pmap)), 567 ("invalidating TLB for non-current pmap")); 568 ia64_ptc_l(va, PAGE_SHIFT << 2); 569} 570 571static void 572pmap_invalidate_all(pmap_t pmap) 573{ 574 u_int64_t addr; 575 int i, j; 576 critical_t psr; 577 578 KASSERT((pmap == kernel_pmap || pmap == PCPU_GET(current_pmap)), 579 ("invalidating TLB for non-current pmap")); 580 581 psr = cpu_critical_enter(); 582 addr = pmap_ptc_e_base; 583 for (i = 0; i < pmap_ptc_e_count1; i++) { 584 for (j = 0; j < pmap_ptc_e_count2; j++) { 585 ia64_ptc_e(addr); 586 addr += pmap_ptc_e_stride2; 587 } 588 addr += pmap_ptc_e_stride1; 589 } 590 cpu_critical_exit(psr); 591} 592 593static u_int32_t 594pmap_allocate_rid(void) 595{ 596 int rid; 597 598 if (pmap_ridcount == pmap_ridmax) 599 panic("pmap_allocate_rid: All Region IDs used"); 600 601 do { 602 rid = arc4random() & (pmap_ridmax - 1); 603 } while (pmap_ridbusy[rid / 64] & (1L << (rid & 63))); 604 pmap_ridbusy[rid / 64] |= (1L << (rid & 63)); 605 pmap_ridcount++; 606 607 return rid; 608} 609 610static void 611pmap_free_rid(u_int32_t rid) 612{ 613 pmap_ridbusy[rid / 64] &= ~(1L << (rid & 63)); 614 pmap_ridcount--; 615} 616 617static void 618pmap_ensure_rid(pmap_t pmap, vm_offset_t va) 619{ 620 int rr; 621 622 rr = va >> 61; 623 if (pmap->pm_rid[rr]) 624 return; 625 pmap->pm_rid[rr] = pmap_allocate_rid(); 626 627 if (pmap == PCPU_GET(current_pmap)) 628 ia64_set_rr(IA64_RR_BASE(rr), 629 (pmap->pm_rid[rr] << 8)|(PAGE_SHIFT << 2)|1); 630} 631 632/*************************************************** 633 * Low level helper routines..... 634 ***************************************************/ 635 636/* 637 * Install a pte into the VHPT 638 */ 639static PMAP_INLINE void 640pmap_install_pte(struct ia64_lpte *vhpte, struct ia64_lpte *pte) 641{ 642 u_int64_t *vhp, *p; 643 644 /* invalidate the pte */ 645 atomic_set_64(&vhpte->pte_tag, 1L << 63); 646 ia64_mf(); /* make sure everyone sees */ 647 648 vhp = (u_int64_t *) vhpte; 649 p = (u_int64_t *) pte; 650 651 vhp[0] = p[0]; 652 vhp[1] = p[1]; 653 vhp[2] = p[2]; /* sets ti to one */ 654 655 ia64_mf(); 656} 657 658/* 659 * Compare essential parts of pte. 660 */ 661static PMAP_INLINE int 662pmap_equal_pte(struct ia64_lpte *pte1, struct ia64_lpte *pte2) 663{ 664 return *(u_int64_t *) pte1 == *(u_int64_t *) pte2; 665} 666 667/* 668 * this routine defines the region(s) of memory that should 669 * not be tested for the modified bit. 670 */ 671static PMAP_INLINE int 672pmap_track_modified(vm_offset_t va) 673{ 674 if ((va < kmi.clean_sva) || (va >= kmi.clean_eva)) 675 return 1; 676 else 677 return 0; 678} 679 680/* 681 * Create the U area for a new process. 682 * This routine directly affects the fork perf for a process. 683 */ 684void 685pmap_new_proc(struct proc *p) 686{ 687 struct user *up; 688 689 /* 690 * Use contigmalloc for user area so that we can use a region 691 * 7 address for it which makes it impossible to accidentally 692 * lose when recording a trapframe. 693 */ 694 up = contigmalloc(UAREA_PAGES * PAGE_SIZE, M_PMAP, 695 M_WAITOK, 696 0ul, 697 256*1024*1024 - 1, 698 PAGE_SIZE, 699 256*1024*1024); 700 701 p->p_md.md_uservirt = up; 702 p->p_uarea = (struct user *) 703 IA64_PHYS_TO_RR7(ia64_tpa((u_int64_t) up)); 704} 705 706/* 707 * Dispose the U area for a process that has exited. 708 * This routine directly impacts the exit perf of a process. 709 */ 710void 711pmap_dispose_proc(struct proc *p) 712{ 713 contigfree(p->p_md.md_uservirt, UAREA_PAGES * PAGE_SIZE, M_PMAP); 714 p->p_md.md_uservirt = 0; 715 p->p_uarea = 0; 716} 717 718/* 719 * Allow the U area for a process to be prejudicially paged out. 720 */ 721void 722pmap_swapout_proc(struct proc *p) 723{ 724} 725 726/* 727 * Bring the U area for a specified process back in. 728 */ 729void 730pmap_swapin_proc(struct proc *p) 731{ 732} 733 734/* 735 * Create the KSTACK for a new thread. 736 * This routine directly affects the fork perf for a process/thread. 737 */ 738void 739pmap_new_thread(struct thread *td) 740{ 741 vm_offset_t *ks; 742 743 /* 744 * Use contigmalloc for user area so that we can use a region 745 * 7 address for it which makes it impossible to accidentally 746 * lose when recording a trapframe. 747 */ 748 ks = contigmalloc(KSTACK_PAGES * PAGE_SIZE, M_PMAP, 749 M_WAITOK, 750 0ul, 751 256*1024*1024 - 1, 752 PAGE_SIZE, 753 256*1024*1024); 754 755 td->td_md.md_kstackvirt = ks; 756 td->td_kstack = IA64_PHYS_TO_RR7(ia64_tpa((u_int64_t)ks)); 757} 758 759/* 760 * Dispose the KSTACK for a thread that has exited. 761 * This routine directly impacts the exit perf of a process/thread. 762 */ 763void 764pmap_dispose_thread(struct thread *td) 765{ 766 contigfree(td->td_md.md_kstackvirt, KSTACK_PAGES * PAGE_SIZE, M_PMAP); 767 td->td_md.md_kstackvirt = 0; 768 td->td_kstack = 0; 769} 770 771/* 772 * Allow the KSTACK for a thread to be prejudicially paged out. 773 */ 774void 775pmap_swapout_thread(struct thread *td) 776{ 777} 778 779/* 780 * Bring the KSTACK for a specified thread back in. 781 */ 782void 783pmap_swapin_thread(struct thread *td) 784{ 785} 786 787/*************************************************** 788 * Page table page management routines..... 789 ***************************************************/ 790 791void 792pmap_pinit0(struct pmap *pmap) 793{ 794 int i; 795 796 /* 797 * kernel_pmap is the same as any other pmap. 798 */ 799 pmap_pinit(pmap); 800 pmap->pm_flags = 0; 801 for (i = 0; i < 5; i++) 802 pmap->pm_rid[i] = 0; 803 pmap->pm_count = 1; 804 pmap->pm_ptphint = NULL; 805 pmap->pm_active = 0; 806 TAILQ_INIT(&pmap->pm_pvlist); 807 bzero(&pmap->pm_stats, sizeof pmap->pm_stats); 808} 809 810/* 811 * Initialize a preallocated and zeroed pmap structure, 812 * such as one in a vmspace structure. 813 */ 814void 815pmap_pinit(struct pmap *pmap) 816{ 817 int i; 818 819 pmap->pm_flags = 0; 820 for (i = 0; i < 5; i++) 821 pmap->pm_rid[i] = 0; 822 pmap->pm_count = 1; 823 pmap->pm_ptphint = NULL; 824 pmap->pm_active = 0; 825 TAILQ_INIT(&pmap->pm_pvlist); 826 bzero(&pmap->pm_stats, sizeof pmap->pm_stats); 827} 828 829/* 830 * Wire in kernel global address entries. To avoid a race condition 831 * between pmap initialization and pmap_growkernel, this procedure 832 * should be called after the vmspace is attached to the process 833 * but before this pmap is activated. 834 */ 835void 836pmap_pinit2(struct pmap *pmap) 837{ 838} 839 840/*************************************************** 841* Pmap allocation/deallocation routines. 842 ***************************************************/ 843 844/* 845 * Release any resources held by the given physical map. 846 * Called when a pmap initialized by pmap_pinit is being released. 847 * Should only be called if the map contains no valid mappings. 848 */ 849void 850pmap_release(pmap_t pmap) 851{ 852 int i; 853 854#if defined(DIAGNOSTIC) 855 if (object->ref_count != 1) 856 panic("pmap_release: pteobj reference count != 1"); 857#endif 858 859 for (i = 0; i < 5; i++) 860 if (pmap->pm_rid[i]) 861 pmap_free_rid(pmap->pm_rid[i]); 862} 863 864/* 865 * grow the number of kernel page table entries, if needed 866 */ 867void 868pmap_growkernel(vm_offset_t addr) 869{ 870 struct ia64_lpte *ptepage; 871 vm_page_t nkpg; 872 873 if (kernel_vm_end == 0) { 874 kernel_vm_end = nkpt * PAGE_SIZE * NKPTEPG 875 + IA64_RR_BASE(5); 876 } 877 addr = (addr + PAGE_SIZE * NKPTEPG) & ~(PAGE_SIZE * NKPTEPG - 1); 878 while (kernel_vm_end < addr) { 879 if (kptdir[KPTE_DIR_INDEX(kernel_vm_end)]) { 880 kernel_vm_end = (kernel_vm_end + PAGE_SIZE * NKPTEPG) 881 & ~(PAGE_SIZE * NKPTEPG - 1); 882 continue; 883 } 884 885 /* 886 * We could handle more by increasing the size of kptdir. 887 */ 888 if (nkpt == MAXKPT) 889 panic("pmap_growkernel: out of kernel address space"); 890 891 /* 892 * This index is bogus, but out of the way 893 */ 894 nkpg = vm_page_alloc(kptobj, nkpt, VM_ALLOC_SYSTEM); 895 if (!nkpg) 896 panic("pmap_growkernel: no memory to grow kernel"); 897 898 nkpt++; 899 900 vm_page_wire(nkpg); 901 ptepage = (struct ia64_lpte *) 902 IA64_PHYS_TO_RR7(VM_PAGE_TO_PHYS(nkpg)); 903 bzero(ptepage, PAGE_SIZE); 904 kptdir[KPTE_DIR_INDEX(kernel_vm_end)] = ptepage; 905 906 kernel_vm_end = (kernel_vm_end + PAGE_SIZE * NKPTEPG) & ~(PAGE_SIZE * NKPTEPG - 1); 907 } 908} 909 910/* 911 * Retire the given physical map from service. 912 * Should only be called if the map contains 913 * no valid mappings. 914 */ 915void 916pmap_destroy(pmap_t pmap) 917{ 918 int count; 919 920 if (pmap == NULL) 921 return; 922 923 count = --pmap->pm_count; 924 if (count == 0) { 925 pmap_release(pmap); 926 panic("destroying a pmap is not yet implemented"); 927 } 928} 929 930/* 931 * Add a reference to the specified pmap. 932 */ 933void 934pmap_reference(pmap_t pmap) 935{ 936 if (pmap != NULL) { 937 pmap->pm_count++; 938 } 939} 940 941/*************************************************** 942* page management routines. 943 ***************************************************/ 944 945/* 946 * free the pv_entry back to the free list 947 */ 948static PMAP_INLINE void 949free_pv_entry(pv_entry_t pv) 950{ 951 pv_entry_count--; 952 zfree(pvzone, pv); 953} 954 955/* 956 * get a new pv_entry, allocating a block from the system 957 * when needed. 958 * the memory allocation is performed bypassing the malloc code 959 * because of the possibility of allocations at interrupt time. 960 */ 961static pv_entry_t 962get_pv_entry(void) 963{ 964 pv_entry_count++; 965 if (pv_entry_high_water && 966 (pv_entry_count > pv_entry_high_water) && 967 (pmap_pagedaemon_waken == 0)) { 968 pmap_pagedaemon_waken = 1; 969 wakeup (&vm_pages_needed); 970 } 971 return zalloc(pvzone); 972} 973 974/* 975 * Add an ia64_lpte to the VHPT. 976 */ 977static void 978pmap_enter_vhpt(struct ia64_lpte *pte, vm_offset_t va) 979{ 980 struct ia64_lpte *vhpte; 981 982 pmap_vhpt_inserts++; 983 pmap_vhpt_resident++; 984 985 vhpte = (struct ia64_lpte *) ia64_thash(va); 986 987 if (vhpte->pte_chain) 988 pmap_vhpt_collisions++; 989 990 pte->pte_chain = vhpte->pte_chain; 991 vhpte->pte_chain = ia64_tpa((vm_offset_t) pte); 992 993 if (!vhpte->pte_p && pte->pte_p) 994 pmap_install_pte(vhpte, pte); 995 else 996 ia64_mf(); 997} 998 999/* 1000 * Update VHPT after a pte has changed. 1001 */ 1002static void 1003pmap_update_vhpt(struct ia64_lpte *pte, vm_offset_t va) 1004{ 1005 struct ia64_lpte *vhpte; 1006 1007 vhpte = (struct ia64_lpte *) ia64_thash(va); 1008 1009 if ((!vhpte->pte_p || vhpte->pte_tag == pte->pte_tag) 1010 && pte->pte_p) 1011 pmap_install_pte(vhpte, pte); 1012} 1013 1014/* 1015 * Remove the ia64_lpte matching va from the VHPT. Return zero if it 1016 * worked or an appropriate error code otherwise. 1017 */ 1018static int 1019pmap_remove_vhpt(vm_offset_t va) 1020{ 1021 struct ia64_lpte *pte; 1022 struct ia64_lpte *lpte; 1023 struct ia64_lpte *vhpte; 1024 u_int64_t tag; 1025 int error = ENOENT; 1026 1027 vhpte = (struct ia64_lpte *) ia64_thash(va); 1028 1029 /* 1030 * If the VHPTE is invalid, there can't be a collision chain. 1031 */ 1032 if (!vhpte->pte_p) { 1033 KASSERT(!vhpte->pte_chain, ("bad vhpte")); 1034 printf("can't remove vhpt entry for 0x%lx\n", va); 1035 goto done; 1036 } 1037 1038 lpte = vhpte; 1039 pte = (struct ia64_lpte *) IA64_PHYS_TO_RR7(vhpte->pte_chain); 1040 tag = ia64_ttag(va); 1041 1042 while (pte->pte_tag != tag) { 1043 lpte = pte; 1044 if (pte->pte_chain) 1045 pte = (struct ia64_lpte *) IA64_PHYS_TO_RR7(pte->pte_chain); 1046 else { 1047 printf("can't remove vhpt entry for 0x%lx\n", va); 1048 goto done; 1049 } 1050 } 1051 1052 /* 1053 * Snip this pv_entry out of the collision chain. 1054 */ 1055 lpte->pte_chain = pte->pte_chain; 1056 1057 /* 1058 * If the VHPTE matches as well, change it to map the first 1059 * element from the chain if there is one. 1060 */ 1061 if (vhpte->pte_tag == tag) { 1062 if (vhpte->pte_chain) { 1063 pte = (struct ia64_lpte *) 1064 IA64_PHYS_TO_RR7(vhpte->pte_chain); 1065 pmap_install_pte(vhpte, pte); 1066 } else { 1067 vhpte->pte_p = 0; 1068 ia64_mf(); 1069 } 1070 } 1071 1072 pmap_vhpt_resident--; 1073 error = 0; 1074 done: 1075 return error; 1076} 1077 1078/* 1079 * Find the ia64_lpte for the given va, if any. 1080 */ 1081static struct ia64_lpte * 1082pmap_find_vhpt(vm_offset_t va) 1083{ 1084 struct ia64_lpte *pte; 1085 u_int64_t tag; 1086 1087 pte = (struct ia64_lpte *) ia64_thash(va); 1088 if (!pte->pte_chain) { 1089 pte = 0; 1090 goto done; 1091 } 1092 1093 tag = ia64_ttag(va); 1094 pte = (struct ia64_lpte *) IA64_PHYS_TO_RR7(pte->pte_chain); 1095 1096 while (pte->pte_tag != tag) { 1097 if (pte->pte_chain) { 1098 pte = (struct ia64_lpte *) IA64_PHYS_TO_RR7(pte->pte_chain); 1099 } else { 1100 pte = 0; 1101 break; 1102 } 1103 } 1104 1105 done: 1106 return pte; 1107} 1108 1109/* 1110 * Remove an entry from the list of managed mappings. 1111 */ 1112static int 1113pmap_remove_entry(pmap_t pmap, vm_page_t m, vm_offset_t va, pv_entry_t pv) 1114{ 1115 if (!pv) { 1116 if (m->md.pv_list_count < pmap->pm_stats.resident_count) { 1117 TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) { 1118 if (pmap == pv->pv_pmap && va == pv->pv_va) 1119 break; 1120 } 1121 } else { 1122 TAILQ_FOREACH(pv, &pmap->pm_pvlist, pv_plist) { 1123 if (va == pv->pv_va) 1124 break; 1125 } 1126 } 1127 } 1128 1129 if (pv) { 1130 TAILQ_REMOVE(&m->md.pv_list, pv, pv_list); 1131 m->md.pv_list_count--; 1132 if (TAILQ_FIRST(&m->md.pv_list) == NULL) 1133 vm_page_flag_clear(m, PG_MAPPED | PG_WRITEABLE); 1134 1135 TAILQ_REMOVE(&pmap->pm_pvlist, pv, pv_plist); 1136 free_pv_entry(pv); 1137 return 0; 1138 } else { 1139 return ENOENT; 1140 } 1141} 1142 1143/* 1144 * Create a pv entry for page at pa for 1145 * (pmap, va). 1146 */ 1147static void 1148pmap_insert_entry(pmap_t pmap, vm_offset_t va, vm_page_t m) 1149{ 1150 pv_entry_t pv; 1151 1152 pv = get_pv_entry(); 1153 pv->pv_pmap = pmap; 1154 pv->pv_va = va; 1155 1156 TAILQ_INSERT_TAIL(&pmap->pm_pvlist, pv, pv_plist); 1157 TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_list); 1158 m->md.pv_list_count++; 1159} 1160 1161/* 1162 * Routine: pmap_extract 1163 * Function: 1164 * Extract the physical page address associated 1165 * with the given map/virtual_address pair. 1166 */ 1167vm_offset_t 1168pmap_extract(pmap, va) 1169 register pmap_t pmap; 1170 vm_offset_t va; 1171{ 1172 pmap_t oldpmap; 1173 vm_offset_t pa; 1174 1175 oldpmap = pmap_install(pmap); 1176 pa = ia64_tpa(va); 1177 pmap_install(oldpmap); 1178 return pa; 1179} 1180 1181/*************************************************** 1182 * Low level mapping routines..... 1183 ***************************************************/ 1184 1185/* 1186 * Find the kernel lpte for mapping the given virtual address, which 1187 * must be in the part of region 5 which we can cover with our kernel 1188 * 'page tables'. 1189 */ 1190static struct ia64_lpte * 1191pmap_find_kpte(vm_offset_t va) 1192{ 1193 KASSERT((va >> 61) == 5, 1194 ("kernel mapping 0x%lx not in region 5", va)); 1195 KASSERT(IA64_RR_MASK(va) < (nkpt * PAGE_SIZE * NKPTEPG), 1196 ("kernel mapping 0x%lx out of range", va)); 1197 return &kptdir[KPTE_DIR_INDEX(va)][KPTE_PTE_INDEX(va)]; 1198} 1199 1200/* 1201 * Find a pte suitable for mapping a user-space address. If one exists 1202 * in the VHPT, that one will be returned, otherwise a new pte is 1203 * allocated. 1204 */ 1205static struct ia64_lpte * 1206pmap_find_pte(vm_offset_t va) 1207{ 1208 struct ia64_lpte *pte; 1209 1210 if (va >= VM_MAXUSER_ADDRESS) 1211 return pmap_find_kpte(va); 1212 1213 pte = pmap_find_vhpt(va); 1214 if (!pte) { 1215 pte = zalloc(ptezone); 1216 pte->pte_p = 0; 1217 } 1218 return pte; 1219} 1220 1221/* 1222 * Free a pte which is now unused. This simply returns it to the zone 1223 * allocator if it is a user mapping. For kernel mappings, clear the 1224 * valid bit to make it clear that the mapping is not currently used. 1225 */ 1226static void 1227pmap_free_pte(struct ia64_lpte *pte, vm_offset_t va) 1228{ 1229 if (va < VM_MAXUSER_ADDRESS) 1230 zfree(ptezone, pte); 1231 else 1232 pte->pte_p = 0; 1233} 1234 1235/* 1236 * Set a pte to contain a valid mapping and enter it in the VHPT. If 1237 * the pte was orginally valid, then its assumed to already be in the 1238 * VHPT. 1239 */ 1240static void 1241pmap_set_pte(struct ia64_lpte *pte, vm_offset_t va, vm_offset_t pa, 1242 int ig, int pl, int ar) 1243{ 1244 int wasvalid = pte->pte_p; 1245 1246 pte->pte_p = 1; 1247 pte->pte_ma = PTE_MA_WB; 1248 if (ig & PTE_IG_MANAGED) { 1249 pte->pte_a = 0; 1250 pte->pte_d = 0; 1251 } else { 1252 pte->pte_a = 1; 1253 pte->pte_d = 1; 1254 } 1255 pte->pte_pl = pl; 1256 pte->pte_ar = ar; 1257 pte->pte_ppn = pa >> 12; 1258 pte->pte_ed = 0; 1259 pte->pte_ig = ig; 1260 1261 pte->pte_ps = PAGE_SHIFT; 1262 pte->pte_key = 0; 1263 1264 pte->pte_tag = ia64_ttag(va); 1265 1266 if (wasvalid) { 1267 pmap_update_vhpt(pte, va); 1268 } else { 1269 pmap_enter_vhpt(pte, va); 1270 } 1271} 1272 1273/* 1274 * If a pte contains a valid mapping, clear it and update the VHPT. 1275 */ 1276static void 1277pmap_clear_pte(struct ia64_lpte *pte, vm_offset_t va) 1278{ 1279 if (pte->pte_p) { 1280 pmap_remove_vhpt(va); 1281 ia64_ptc_l(va, PAGE_SHIFT << 2); 1282 pte->pte_p = 0; 1283 } 1284} 1285 1286/* 1287 * Remove the (possibly managed) mapping represented by pte from the 1288 * given pmap. 1289 */ 1290static int 1291pmap_remove_pte(pmap_t pmap, struct ia64_lpte *pte, vm_offset_t va, 1292 pv_entry_t pv, int freepte) 1293{ 1294 int error; 1295 vm_page_t m; 1296 1297 KASSERT((pmap == kernel_pmap || pmap == PCPU_GET(current_pmap)), 1298 ("removing pte for non-current pmap")); 1299 1300 /* 1301 * First remove from the VHPT. 1302 */ 1303 error = pmap_remove_vhpt(va); 1304 if (error) 1305 return error; 1306 1307 /* 1308 * Make sure pmap_set_pte() knows it isn't in the VHPT. 1309 */ 1310 pte->pte_p = 0; 1311 1312 if (pte->pte_ig & PTE_IG_WIRED) 1313 pmap->pm_stats.wired_count -= 1; 1314 1315 pmap->pm_stats.resident_count -= 1; 1316 if (pte->pte_ig & PTE_IG_MANAGED) { 1317 m = PHYS_TO_VM_PAGE(pmap_pte_pa(pte)); 1318 if (pte->pte_d) 1319 if (pmap_track_modified(va)) 1320 vm_page_dirty(m); 1321 if (pte->pte_a) 1322 vm_page_flag_set(m, PG_REFERENCED); 1323 1324 if (freepte) 1325 pmap_free_pte(pte, va); 1326 return pmap_remove_entry(pmap, m, va, pv); 1327 } else { 1328 if (freepte) 1329 pmap_free_pte(pte, va); 1330 return 0; 1331 } 1332} 1333 1334/* 1335 * Add a list of wired pages to the kva 1336 * this routine is only used for temporary 1337 * kernel mappings that do not need to have 1338 * page modification or references recorded. 1339 * Note that old mappings are simply written 1340 * over. The page *must* be wired. 1341 */ 1342void 1343pmap_qenter(vm_offset_t va, vm_page_t *m, int count) 1344{ 1345 int i; 1346 struct ia64_lpte *pte; 1347 1348 for (i = 0; i < count; i++) { 1349 vm_offset_t tva = va + i * PAGE_SIZE; 1350 int wasvalid; 1351 pte = pmap_find_kpte(tva); 1352 wasvalid = pte->pte_p; 1353 pmap_set_pte(pte, tva, VM_PAGE_TO_PHYS(m[i]), 1354 0, PTE_PL_KERN, PTE_AR_RWX); 1355 if (wasvalid) 1356 ia64_ptc_l(tva, PAGE_SHIFT << 2); 1357 } 1358} 1359 1360/* 1361 * this routine jerks page mappings from the 1362 * kernel -- it is meant only for temporary mappings. 1363 */ 1364void 1365pmap_qremove(vm_offset_t va, int count) 1366{ 1367 int i; 1368 struct ia64_lpte *pte; 1369 1370 for (i = 0; i < count; i++) { 1371 pte = pmap_find_kpte(va); 1372 pmap_clear_pte(pte, va); 1373 va += PAGE_SIZE; 1374 } 1375} 1376 1377/* 1378 * Add a wired page to the kva. 1379 */ 1380void 1381pmap_kenter(vm_offset_t va, vm_offset_t pa) 1382{ 1383 struct ia64_lpte *pte; 1384 int wasvalid; 1385 1386 pte = pmap_find_kpte(va); 1387 wasvalid = pte->pte_p; 1388 pmap_set_pte(pte, va, pa, 0, PTE_PL_KERN, PTE_AR_RWX); 1389 if (wasvalid) 1390 ia64_ptc_l(va, PAGE_SHIFT << 2); 1391} 1392 1393/* 1394 * Remove a page from the kva 1395 */ 1396void 1397pmap_kremove(vm_offset_t va) 1398{ 1399 struct ia64_lpte *pte; 1400 1401 pte = pmap_find_kpte(va); 1402 pmap_clear_pte(pte, va); 1403} 1404 1405/* 1406 * Used to map a range of physical addresses into kernel 1407 * virtual address space. 1408 * 1409 * The value passed in '*virt' is a suggested virtual address for 1410 * the mapping. Architectures which can support a direct-mapped 1411 * physical to virtual region can return the appropriate address 1412 * within that region, leaving '*virt' unchanged. Other 1413 * architectures should map the pages starting at '*virt' and 1414 * update '*virt' with the first usable address after the mapped 1415 * region. 1416 */ 1417vm_offset_t 1418pmap_map(vm_offset_t *virt, vm_offset_t start, vm_offset_t end, int prot) 1419{ 1420 return IA64_PHYS_TO_RR7(start); 1421} 1422 1423/* 1424 * This routine is very drastic, but can save the system 1425 * in a pinch. 1426 */ 1427void 1428pmap_collect() 1429{ 1430 int i; 1431 vm_page_t m; 1432 static int warningdone = 0; 1433 1434 if (pmap_pagedaemon_waken == 0) 1435 return; 1436 1437 if (warningdone < 5) { 1438 printf("pmap_collect: collecting pv entries -- suggest increasing PMAP_SHPGPERPROC\n"); 1439 warningdone++; 1440 } 1441 1442 for(i = 0; i < vm_page_array_size; i++) { 1443 m = &vm_page_array[i]; 1444 if (m->wire_count || m->hold_count || m->busy || 1445 (m->flags & (PG_BUSY | PG_UNMANAGED))) 1446 continue; 1447 pmap_remove_all(m); 1448 } 1449 pmap_pagedaemon_waken = 0; 1450} 1451 1452/* 1453 * Remove a single page from a process address space 1454 */ 1455static void 1456pmap_remove_page(pmap_t pmap, vm_offset_t va) 1457{ 1458 struct ia64_lpte *pte; 1459 1460 KASSERT((pmap == kernel_pmap || pmap == PCPU_GET(current_pmap)), 1461 ("removing page for non-current pmap")); 1462 1463 pte = pmap_find_vhpt(va); 1464 if (pte) { 1465 pmap_remove_pte(pmap, pte, va, 0, 1); 1466 pmap_invalidate_page(pmap, va); 1467 } 1468 return; 1469} 1470 1471/* 1472 * Remove the given range of addresses from the specified map. 1473 * 1474 * It is assumed that the start and end are properly 1475 * rounded to the page size. 1476 */ 1477void 1478pmap_remove(pmap_t pmap, vm_offset_t sva, vm_offset_t eva) 1479{ 1480 pmap_t oldpmap; 1481 vm_offset_t va; 1482 pv_entry_t pv; 1483 struct ia64_lpte *pte; 1484 1485 if (pmap == NULL) 1486 return; 1487 1488 if (pmap->pm_stats.resident_count == 0) 1489 return; 1490 1491 oldpmap = pmap_install(pmap); 1492 1493 /* 1494 * special handling of removing one page. a very 1495 * common operation and easy to short circuit some 1496 * code. 1497 */ 1498 if (sva + PAGE_SIZE == eva) { 1499 pmap_remove_page(pmap, sva); 1500 pmap_install(oldpmap); 1501 return; 1502 } 1503 1504 if (pmap->pm_stats.resident_count < ((eva - sva) >> PAGE_SHIFT)) { 1505 TAILQ_FOREACH(pv, &pmap->pm_pvlist, pv_plist) { 1506 va = pv->pv_va; 1507 if (va >= sva && va < eva) { 1508 pte = pmap_find_vhpt(va); 1509 pmap_remove_pte(pmap, pte, va, pv, 1); 1510 pmap_invalidate_page(pmap, va); 1511 } 1512 } 1513 1514 } else { 1515 for (va = sva; va < eva; va = va += PAGE_SIZE) { 1516 pte = pmap_find_vhpt(va); 1517 if (pte) { 1518 pmap_remove_pte(pmap, pte, va, 0, 1); 1519 pmap_invalidate_page(pmap, va); 1520 } 1521 } 1522 } 1523 1524 pmap_install(oldpmap); 1525} 1526 1527/* 1528 * Routine: pmap_remove_all 1529 * Function: 1530 * Removes this physical page from 1531 * all physical maps in which it resides. 1532 * Reflects back modify bits to the pager. 1533 * 1534 * Notes: 1535 * Original versions of this routine were very 1536 * inefficient because they iteratively called 1537 * pmap_remove (slow...) 1538 */ 1539 1540static void 1541pmap_remove_all(vm_page_t m) 1542{ 1543 pmap_t oldpmap; 1544 pv_entry_t pv; 1545 int nmodify; 1546 int s; 1547 1548 nmodify = 0; 1549#if defined(PMAP_DIAGNOSTIC) 1550 /* 1551 * XXX this makes pmap_page_protect(NONE) illegal for non-managed 1552 * pages! 1553 */ 1554 if (!pmap_initialized || (m->flags & PG_FICTITIOUS)) { 1555 panic("pmap_page_protect: illegal for unmanaged page, va: 0x%lx", VM_PAGE_TO_PHYS(m)); 1556 } 1557#endif 1558 1559 s = splvm(); 1560 1561 while ((pv = TAILQ_FIRST(&m->md.pv_list)) != NULL) { 1562 struct ia64_lpte *pte; 1563 pmap_t pmap = pv->pv_pmap; 1564 vm_offset_t va = pv->pv_va; 1565 1566 oldpmap = pmap_install(pmap); 1567 pte = pmap_find_vhpt(va); 1568 if (pmap_pte_pa(pte) != VM_PAGE_TO_PHYS(m)) 1569 panic("pmap_remove_all: pv_table for %lx is inconsistent", VM_PAGE_TO_PHYS(m)); 1570 pmap_remove_pte(pmap, pte, va, pv, 1); 1571 pmap_invalidate_page(pmap, va); 1572 pmap_install(oldpmap); 1573 } 1574 1575 vm_page_flag_clear(m, PG_MAPPED | PG_WRITEABLE); 1576 1577 splx(s); 1578 return; 1579} 1580 1581/* 1582 * Set the physical protection on the 1583 * specified range of this map as requested. 1584 */ 1585void 1586pmap_protect(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, vm_prot_t prot) 1587{ 1588 pmap_t oldpmap; 1589 struct ia64_lpte *pte; 1590 int newprot; 1591 1592 if (pmap == NULL) 1593 return; 1594 1595 oldpmap = pmap_install(pmap); 1596 1597 if ((prot & VM_PROT_READ) == VM_PROT_NONE) { 1598 pmap_remove(pmap, sva, eva); 1599 pmap_install(oldpmap); 1600 return; 1601 } 1602 1603 if (prot & VM_PROT_WRITE) { 1604 pmap_install(oldpmap); 1605 return; 1606 } 1607 1608 newprot = pte_prot(pmap, prot); 1609 1610 if ((sva & PAGE_MASK) || (eva & PAGE_MASK)) 1611 panic("pmap_protect: unaligned addresses"); 1612 1613 while (sva < eva) { 1614 /* 1615 * If page is invalid, skip this page 1616 */ 1617 pte = pmap_find_vhpt(sva); 1618 if (!pte) { 1619 sva += PAGE_SIZE; 1620 continue; 1621 } 1622 1623 if (pmap_pte_prot(pte) != newprot) { 1624 if (pte->pte_ig & PTE_IG_MANAGED) { 1625 vm_offset_t pa = pmap_pte_pa(pte); 1626 vm_page_t m = PHYS_TO_VM_PAGE(pa); 1627 if (pte->pte_d) { 1628 if (pmap_track_modified(sva)) 1629 vm_page_dirty(m); 1630 pte->pte_d = 0; 1631 } 1632 if (pte->pte_a) { 1633 vm_page_flag_set(m, PG_REFERENCED); 1634 pte->pte_a = 0; 1635 } 1636 } 1637 pmap_pte_set_prot(pte, newprot); 1638 pmap_update_vhpt(pte, sva); 1639 pmap_invalidate_page(pmap, sva); 1640 } 1641 1642 sva += PAGE_SIZE; 1643 } 1644 pmap_install(oldpmap); 1645} 1646 1647/* 1648 * Insert the given physical page (p) at 1649 * the specified virtual address (v) in the 1650 * target physical map with the protection requested. 1651 * 1652 * If specified, the page will be wired down, meaning 1653 * that the related pte can not be reclaimed. 1654 * 1655 * NB: This is the only routine which MAY NOT lazy-evaluate 1656 * or lose information. That is, this routine must actually 1657 * insert this page into the given map NOW. 1658 */ 1659void 1660pmap_enter(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot, 1661 boolean_t wired) 1662{ 1663 pmap_t oldpmap; 1664 vm_offset_t pa; 1665 vm_offset_t opa; 1666 struct ia64_lpte origpte; 1667 struct ia64_lpte *pte; 1668 int managed; 1669 1670 if (pmap == NULL) 1671 return; 1672 1673 pmap_ensure_rid(pmap, va); 1674 1675 oldpmap = pmap_install(pmap); 1676 1677 va &= ~PAGE_MASK; 1678#ifdef PMAP_DIAGNOSTIC 1679 if (va > VM_MAX_KERNEL_ADDRESS) 1680 panic("pmap_enter: toobig"); 1681#endif 1682 1683 /* 1684 * Find (or create) a pte for the given mapping. 1685 */ 1686 pte = pmap_find_pte(va); 1687 origpte = *pte; 1688 1689 if (origpte.pte_p) 1690 opa = pmap_pte_pa(&origpte); 1691 else 1692 opa = 0; 1693 managed = 0; 1694 1695 pa = VM_PAGE_TO_PHYS(m) & ~PAGE_MASK; 1696 1697 /* 1698 * Mapping has not changed, must be protection or wiring change. 1699 */ 1700 if (origpte.pte_p && (opa == pa)) { 1701 /* 1702 * Wiring change, just update stats. We don't worry about 1703 * wiring PT pages as they remain resident as long as there 1704 * are valid mappings in them. Hence, if a user page is wired, 1705 * the PT page will be also. 1706 */ 1707 if (wired && ((origpte.pte_ig & PTE_IG_WIRED) == 0)) 1708 pmap->pm_stats.wired_count++; 1709 else if (!wired && (origpte.pte_ig & PTE_IG_WIRED)) 1710 pmap->pm_stats.wired_count--; 1711 1712 /* 1713 * We might be turning off write access to the page, 1714 * so we go ahead and sense modify status. 1715 */ 1716 if (origpte.pte_ig & PTE_IG_MANAGED) { 1717 if (origpte.pte_d && pmap_track_modified(va)) { 1718 vm_page_t om; 1719 om = PHYS_TO_VM_PAGE(opa); 1720 vm_page_dirty(om); 1721 } 1722 } 1723 1724 managed = origpte.pte_ig & PTE_IG_MANAGED; 1725 goto validate; 1726 } 1727 /* 1728 * Mapping has changed, invalidate old range and fall 1729 * through to handle validating new mapping. 1730 */ 1731 if (opa) { 1732 int error; 1733 error = pmap_remove_pte(pmap, pte, va, 0, 0); 1734 if (error) 1735 panic("pmap_enter: pte vanished, va: 0x%lx", va); 1736 } 1737 1738 /* 1739 * Enter on the PV list if part of our managed memory. 1740 */ 1741 if (pmap_initialized && (m->flags & PG_FICTITIOUS) == 0) { 1742 pmap_insert_entry(pmap, va, m); 1743 managed |= PTE_IG_MANAGED; 1744 } 1745 1746 /* 1747 * Increment counters 1748 */ 1749 pmap->pm_stats.resident_count++; 1750 if (wired) 1751 pmap->pm_stats.wired_count++; 1752 1753validate: 1754 1755 /* 1756 * Now validate mapping with desired protection/wiring. This 1757 * adds the pte to the VHPT if necessary. 1758 */ 1759 pmap_set_pte(pte, va, pa, managed | (wired ? PTE_IG_WIRED : 0), 1760 pte_prot_pl(pmap, prot), pte_prot_ar(pmap, prot)); 1761 1762 /* 1763 * if the mapping or permission bits are different, we need 1764 * to invalidate the page. 1765 */ 1766 if (!pmap_equal_pte(&origpte, pte)) 1767 pmap_invalidate_page(pmap, va); 1768 1769 pmap_install(oldpmap); 1770} 1771 1772/* 1773 * this code makes some *MAJOR* assumptions: 1774 * 1. Current pmap & pmap exists. 1775 * 2. Not wired. 1776 * 3. Read access. 1777 * 4. No page table pages. 1778 * 5. Tlbflush is deferred to calling procedure. 1779 * 6. Page IS managed. 1780 * but is *MUCH* faster than pmap_enter... 1781 */ 1782 1783static void 1784pmap_enter_quick(pmap_t pmap, vm_offset_t va, vm_page_t m) 1785{ 1786 struct ia64_lpte *pte; 1787 pmap_t oldpmap; 1788 1789 pmap_ensure_rid(pmap, va); 1790 1791 oldpmap = pmap_install(pmap); 1792 1793 pte = pmap_find_pte(va); 1794 if (pte->pte_p) 1795 return; 1796 1797 /* 1798 * Enter on the PV list since its part of our managed memory. 1799 */ 1800 pmap_insert_entry(pmap, va, m); 1801 1802 /* 1803 * Increment counters 1804 */ 1805 pmap->pm_stats.resident_count++; 1806 1807 /* 1808 * Initialise PTE with read-only protection and enter into VHPT. 1809 */ 1810 pmap_set_pte(pte, va, VM_PAGE_TO_PHYS(m), 1811 PTE_IG_MANAGED, 1812 PTE_PL_USER, PTE_AR_R); 1813 1814 pmap_install(oldpmap); 1815} 1816 1817/* 1818 * Make temporary mapping for a physical address. This is called 1819 * during dump. 1820 */ 1821void * 1822pmap_kenter_temporary(vm_offset_t pa, int i) 1823{ 1824 return (void *) IA64_PHYS_TO_RR7(pa - (i * PAGE_SIZE)); 1825} 1826 1827#define MAX_INIT_PT (96) 1828/* 1829 * pmap_object_init_pt preloads the ptes for a given object 1830 * into the specified pmap. This eliminates the blast of soft 1831 * faults on process startup and immediately after an mmap. 1832 */ 1833void 1834pmap_object_init_pt(pmap_t pmap, vm_offset_t addr, 1835 vm_object_t object, vm_pindex_t pindex, 1836 vm_size_t size, int limit) 1837{ 1838 pmap_t oldpmap; 1839 vm_offset_t tmpidx; 1840 int psize; 1841 vm_page_t p; 1842 int objpgs; 1843 1844 if (pmap == NULL || object == NULL) 1845 return; 1846 1847 oldpmap = pmap_install(pmap); 1848 1849 psize = ia64_btop(size); 1850 1851 if ((object->type != OBJT_VNODE) || 1852 ((limit & MAP_PREFAULT_PARTIAL) && (psize > MAX_INIT_PT) && 1853 (object->resident_page_count > MAX_INIT_PT))) { 1854 pmap_install(oldpmap); 1855 return; 1856 } 1857 1858 if (psize + pindex > object->size) { 1859 if (object->size < pindex) 1860 return; 1861 psize = object->size - pindex; 1862 } 1863 1864 /* 1865 * if we are processing a major portion of the object, then scan the 1866 * entire thing. 1867 */ 1868 if (psize > (object->resident_page_count >> 2)) { 1869 objpgs = psize; 1870 1871 for (p = TAILQ_FIRST(&object->memq); 1872 ((objpgs > 0) && (p != NULL)); 1873 p = TAILQ_NEXT(p, listq)) { 1874 1875 tmpidx = p->pindex; 1876 if (tmpidx < pindex) { 1877 continue; 1878 } 1879 tmpidx -= pindex; 1880 if (tmpidx >= psize) { 1881 continue; 1882 } 1883 /* 1884 * don't allow an madvise to blow away our really 1885 * free pages allocating pv entries. 1886 */ 1887 if ((limit & MAP_PREFAULT_MADVISE) && 1888 cnt.v_free_count < cnt.v_free_reserved) { 1889 break; 1890 } 1891 if (((p->valid & VM_PAGE_BITS_ALL) == VM_PAGE_BITS_ALL) && 1892 (p->busy == 0) && 1893 (p->flags & (PG_BUSY | PG_FICTITIOUS)) == 0) { 1894 if ((p->queue - p->pc) == PQ_CACHE) 1895 vm_page_deactivate(p); 1896 vm_page_busy(p); 1897 pmap_enter_quick(pmap, 1898 addr + ia64_ptob(tmpidx), p); 1899 vm_page_flag_set(p, PG_MAPPED); 1900 vm_page_wakeup(p); 1901 } 1902 objpgs -= 1; 1903 } 1904 } else { 1905 /* 1906 * else lookup the pages one-by-one. 1907 */ 1908 for (tmpidx = 0; tmpidx < psize; tmpidx += 1) { 1909 /* 1910 * don't allow an madvise to blow away our really 1911 * free pages allocating pv entries. 1912 */ 1913 if ((limit & MAP_PREFAULT_MADVISE) && 1914 cnt.v_free_count < cnt.v_free_reserved) { 1915 break; 1916 } 1917 p = vm_page_lookup(object, tmpidx + pindex); 1918 if (p && 1919 ((p->valid & VM_PAGE_BITS_ALL) == VM_PAGE_BITS_ALL) && 1920 (p->busy == 0) && 1921 (p->flags & (PG_BUSY | PG_FICTITIOUS)) == 0) { 1922 if ((p->queue - p->pc) == PQ_CACHE) 1923 vm_page_deactivate(p); 1924 vm_page_busy(p); 1925 pmap_enter_quick(pmap, 1926 addr + ia64_ptob(tmpidx), p); 1927 vm_page_flag_set(p, PG_MAPPED); 1928 vm_page_wakeup(p); 1929 } 1930 } 1931 } 1932 pmap_install(oldpmap); 1933 return; 1934} 1935 1936/* 1937 * pmap_prefault provides a quick way of clustering 1938 * pagefaults into a processes address space. It is a "cousin" 1939 * of pmap_object_init_pt, except it runs at page fault time instead 1940 * of mmap time. 1941 */ 1942#define PFBAK 4 1943#define PFFOR 4 1944#define PAGEORDER_SIZE (PFBAK+PFFOR) 1945 1946static int pmap_prefault_pageorder[] = { 1947 -PAGE_SIZE, PAGE_SIZE, 1948 -2 * PAGE_SIZE, 2 * PAGE_SIZE, 1949 -3 * PAGE_SIZE, 3 * PAGE_SIZE 1950 -4 * PAGE_SIZE, 4 * PAGE_SIZE 1951}; 1952 1953void 1954pmap_prefault(pmap, addra, entry) 1955 pmap_t pmap; 1956 vm_offset_t addra; 1957 vm_map_entry_t entry; 1958{ 1959 int i; 1960 vm_offset_t starta; 1961 vm_offset_t addr; 1962 vm_pindex_t pindex; 1963 vm_page_t m, mpte; 1964 vm_object_t object; 1965 1966 if (!curthread || (pmap != vmspace_pmap(curthread->td_proc->p_vmspace))) 1967 return; 1968 1969 object = entry->object.vm_object; 1970 1971 starta = addra - PFBAK * PAGE_SIZE; 1972 if (starta < entry->start) { 1973 starta = entry->start; 1974 } else if (starta > addra) { 1975 starta = 0; 1976 } 1977 1978 mpte = NULL; 1979 for (i = 0; i < PAGEORDER_SIZE; i++) { 1980 vm_object_t lobject; 1981 struct ia64_lpte *pte; 1982 1983 addr = addra + pmap_prefault_pageorder[i]; 1984 if (addr > addra + (PFFOR * PAGE_SIZE)) 1985 addr = 0; 1986 1987 if (addr < starta || addr >= entry->end) 1988 continue; 1989 1990 pte = pmap_find_vhpt(addr); 1991 if (pte && pte->pte_p) 1992 continue; 1993 1994 pindex = ((addr - entry->start) + entry->offset) >> PAGE_SHIFT; 1995 lobject = object; 1996 for (m = vm_page_lookup(lobject, pindex); 1997 (!m && (lobject->type == OBJT_DEFAULT) && (lobject->backing_object)); 1998 lobject = lobject->backing_object) { 1999 if (lobject->backing_object_offset & PAGE_MASK) 2000 break; 2001 pindex += (lobject->backing_object_offset >> PAGE_SHIFT); 2002 m = vm_page_lookup(lobject->backing_object, pindex); 2003 } 2004 2005 /* 2006 * give-up when a page is not in memory 2007 */ 2008 if (m == NULL) 2009 break; 2010 2011 if (((m->valid & VM_PAGE_BITS_ALL) == VM_PAGE_BITS_ALL) && 2012 (m->busy == 0) && 2013 (m->flags & (PG_BUSY | PG_FICTITIOUS)) == 0) { 2014 2015 if ((m->queue - m->pc) == PQ_CACHE) { 2016 vm_page_deactivate(m); 2017 } 2018 vm_page_busy(m); 2019 pmap_enter_quick(pmap, addr, m); 2020 vm_page_flag_set(m, PG_MAPPED); 2021 vm_page_wakeup(m); 2022 } 2023 } 2024} 2025 2026/* 2027 * Routine: pmap_change_wiring 2028 * Function: Change the wiring attribute for a map/virtual-address 2029 * pair. 2030 * In/out conditions: 2031 * The mapping must already exist in the pmap. 2032 */ 2033void 2034pmap_change_wiring(pmap, va, wired) 2035 register pmap_t pmap; 2036 vm_offset_t va; 2037 boolean_t wired; 2038{ 2039 pmap_t oldpmap; 2040 struct ia64_lpte *pte; 2041 2042 if (pmap == NULL) 2043 return; 2044 2045 oldpmap = pmap_install(pmap); 2046 2047 pte = pmap_find_vhpt(va); 2048 2049 if (wired && !pmap_pte_w(pte)) 2050 pmap->pm_stats.wired_count++; 2051 else if (!wired && pmap_pte_w(pte)) 2052 pmap->pm_stats.wired_count--; 2053 2054 /* 2055 * Wiring is not a hardware characteristic so there is no need to 2056 * invalidate TLB. 2057 */ 2058 pmap_pte_set_w(pte, wired); 2059 2060 pmap_install(oldpmap); 2061} 2062 2063 2064 2065/* 2066 * Copy the range specified by src_addr/len 2067 * from the source map to the range dst_addr/len 2068 * in the destination map. 2069 * 2070 * This routine is only advisory and need not do anything. 2071 */ 2072 2073void 2074pmap_copy(pmap_t dst_pmap, pmap_t src_pmap, vm_offset_t dst_addr, vm_size_t len, 2075 vm_offset_t src_addr) 2076{ 2077} 2078 2079/* 2080 * Routine: pmap_kernel 2081 * Function: 2082 * Returns the physical map handle for the kernel. 2083 */ 2084pmap_t 2085pmap_kernel() 2086{ 2087 return (kernel_pmap); 2088} 2089 2090/* 2091 * pmap_zero_page zeros the specified hardware page by 2092 * mapping it into virtual memory and using bzero to clear 2093 * its contents. 2094 */ 2095 2096void 2097pmap_zero_page(vm_offset_t pa) 2098{ 2099 vm_offset_t va = IA64_PHYS_TO_RR7(pa); 2100 bzero((caddr_t) va, PAGE_SIZE); 2101} 2102 2103 2104/* 2105 * pmap_zero_page_area zeros the specified hardware page by 2106 * mapping it into virtual memory and using bzero to clear 2107 * its contents. 2108 * 2109 * off and size must reside within a single page. 2110 */ 2111 2112void 2113pmap_zero_page_area(vm_offset_t pa, int off, int size) 2114{ 2115 vm_offset_t va = IA64_PHYS_TO_RR7(pa); 2116 bzero((char *)(caddr_t)va + off, size); 2117} 2118 2119/* 2120 * pmap_copy_page copies the specified (machine independent) 2121 * page by mapping the page into virtual memory and using 2122 * bcopy to copy the page, one machine dependent page at a 2123 * time. 2124 */ 2125void 2126pmap_copy_page(vm_offset_t src, vm_offset_t dst) 2127{ 2128 src = IA64_PHYS_TO_RR7(src); 2129 dst = IA64_PHYS_TO_RR7(dst); 2130 bcopy((caddr_t) src, (caddr_t) dst, PAGE_SIZE); 2131} 2132 2133 2134/* 2135 * Routine: pmap_pageable 2136 * Function: 2137 * Make the specified pages (by pmap, offset) 2138 * pageable (or not) as requested. 2139 * 2140 * A page which is not pageable may not take 2141 * a fault; therefore, its page table entry 2142 * must remain valid for the duration. 2143 * 2144 * This routine is merely advisory; pmap_enter 2145 * will specify that these pages are to be wired 2146 * down (or not) as appropriate. 2147 */ 2148void 2149pmap_pageable(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, 2150 boolean_t pageable) 2151{ 2152} 2153 2154/* 2155 * this routine returns true if a physical page resides 2156 * in the given pmap. 2157 */ 2158boolean_t 2159pmap_page_exists(pmap_t pmap, vm_page_t m) 2160{ 2161 register pv_entry_t pv; 2162 int s; 2163 2164 if (!pmap_initialized || (m->flags & PG_FICTITIOUS)) 2165 return FALSE; 2166 2167 s = splvm(); 2168 2169 /* 2170 * Not found, check current mappings returning immediately if found. 2171 */ 2172 TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) { 2173 if (pv->pv_pmap == pmap) { 2174 splx(s); 2175 return TRUE; 2176 } 2177 } 2178 splx(s); 2179 return (FALSE); 2180} 2181 2182#define PMAP_REMOVE_PAGES_CURPROC_ONLY 2183/* 2184 * Remove all pages from specified address space 2185 * this aids process exit speeds. Also, this code 2186 * is special cased for current process only, but 2187 * can have the more generic (and slightly slower) 2188 * mode enabled. This is much faster than pmap_remove 2189 * in the case of running down an entire address space. 2190 */ 2191void 2192pmap_remove_pages(pmap_t pmap, vm_offset_t sva, vm_offset_t eva) 2193{ 2194 pv_entry_t pv, npv; 2195 int s; 2196 2197#ifdef PMAP_REMOVE_PAGES_CURPROC_ONLY 2198 if (!curthread || (pmap != vmspace_pmap(curthread->td_proc->p_vmspace))) { 2199 printf("warning: pmap_remove_pages called with non-current pmap\n"); 2200 return; 2201 } 2202#endif 2203 2204 s = splvm(); 2205 for (pv = TAILQ_FIRST(&pmap->pm_pvlist); 2206 pv; 2207 pv = npv) { 2208 struct ia64_lpte *pte; 2209 2210 npv = TAILQ_NEXT(pv, pv_plist); 2211 2212 if (pv->pv_va >= eva || pv->pv_va < sva) { 2213 continue; 2214 } 2215 2216 pte = pmap_find_vhpt(pv->pv_va); 2217 if (!pte) 2218 panic("pmap_remove_pages: page on pm_pvlist has no pte\n"); 2219 2220 2221/* 2222 * We cannot remove wired pages from a process' mapping at this time 2223 */ 2224 if (pte->pte_ig & PTE_IG_WIRED) { 2225 continue; 2226 } 2227 2228 pmap_remove_pte(pmap, pte, pv->pv_va, pv, 1); 2229 } 2230 splx(s); 2231 2232 pmap_invalidate_all(pmap); 2233} 2234 2235/* 2236 * pmap_page_protect: 2237 * 2238 * Lower the permission for all mappings to a given page. 2239 */ 2240void 2241pmap_page_protect(vm_page_t m, vm_prot_t prot) 2242{ 2243 pv_entry_t pv; 2244 2245 if ((prot & VM_PROT_WRITE) != 0) 2246 return; 2247 if (prot & (VM_PROT_READ | VM_PROT_EXECUTE)) { 2248 TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) { 2249 int newprot = pte_prot(pv->pv_pmap, prot); 2250 pmap_t oldpmap = pmap_install(pv->pv_pmap); 2251 struct ia64_lpte *pte; 2252 pte = pmap_find_vhpt(pv->pv_va); 2253 pmap_pte_set_prot(pte, newprot); 2254 pmap_update_vhpt(pte, pv->pv_va); 2255 pmap_invalidate_page(pv->pv_pmap, pv->pv_va); 2256 pmap_install(oldpmap); 2257 } 2258 } else { 2259 pmap_remove_all(m); 2260 } 2261} 2262 2263vm_offset_t 2264pmap_phys_address(int ppn) 2265{ 2266 return (ia64_ptob(ppn)); 2267} 2268 2269/* 2270 * pmap_ts_referenced: 2271 * 2272 * Return the count of reference bits for a page, clearing all of them. 2273 * 2274 */ 2275int 2276pmap_ts_referenced(vm_page_t m) 2277{ 2278 pv_entry_t pv; 2279 int count = 0; 2280 2281 if (!pmap_initialized || (m->flags & PG_FICTITIOUS)) 2282 return 0; 2283 2284 TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) { 2285 pmap_t oldpmap = pmap_install(pv->pv_pmap); 2286 struct ia64_lpte *pte; 2287 pte = pmap_find_vhpt(pv->pv_va); 2288 if (pte->pte_a) { 2289 count++; 2290 pte->pte_a = 0; 2291 pmap_update_vhpt(pte, pv->pv_va); 2292 pmap_invalidate_page(pv->pv_pmap, pv->pv_va); 2293 } 2294 pmap_install(oldpmap); 2295 } 2296 2297 return count; 2298} 2299 2300#if 0 2301/* 2302 * pmap_is_referenced: 2303 * 2304 * Return whether or not the specified physical page was referenced 2305 * in any physical maps. 2306 */ 2307static boolean_t 2308pmap_is_referenced(vm_page_t m) 2309{ 2310 pv_entry_t pv; 2311 2312 if (!pmap_initialized || (m->flags & PG_FICTITIOUS)) 2313 return FALSE; 2314 2315 TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) { 2316 pmap_t oldpmap = pmap_install(pv->pv_pmap); 2317 struct ia64_lpte *pte = pmap_find_vhpt(pv->pv_va); 2318 pmap_install(oldpmap); 2319 if (pte->pte_a) 2320 return 1; 2321 } 2322 2323 return 0; 2324} 2325#endif 2326 2327/* 2328 * pmap_is_modified: 2329 * 2330 * Return whether or not the specified physical page was modified 2331 * in any physical maps. 2332 */ 2333boolean_t 2334pmap_is_modified(vm_page_t m) 2335{ 2336 pv_entry_t pv; 2337 2338 if (!pmap_initialized || (m->flags & PG_FICTITIOUS)) 2339 return FALSE; 2340 2341 TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) { 2342 pmap_t oldpmap = pmap_install(pv->pv_pmap); 2343 struct ia64_lpte *pte = pmap_find_vhpt(pv->pv_va); 2344 pmap_install(oldpmap); 2345 if (pte->pte_d) 2346 return 1; 2347 } 2348 2349 return 0; 2350} 2351 2352/* 2353 * Clear the modify bits on the specified physical page. 2354 */ 2355void 2356pmap_clear_modify(vm_page_t m) 2357{ 2358 pv_entry_t pv; 2359 2360 if (!pmap_initialized || (m->flags & PG_FICTITIOUS)) 2361 return; 2362 2363 TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) { 2364 pmap_t oldpmap = pmap_install(pv->pv_pmap); 2365 struct ia64_lpte *pte = pmap_find_vhpt(pv->pv_va); 2366 if (pte->pte_d) { 2367 pte->pte_d = 0; 2368 pmap_update_vhpt(pte, pv->pv_va); 2369 pmap_invalidate_page(pv->pv_pmap, pv->pv_va); 2370 } 2371 pmap_install(oldpmap); 2372 } 2373} 2374 2375/* 2376 * pmap_clear_reference: 2377 * 2378 * Clear the reference bit on the specified physical page. 2379 */ 2380void 2381pmap_clear_reference(vm_page_t m) 2382{ 2383 pv_entry_t pv; 2384 2385 if (!pmap_initialized || (m->flags & PG_FICTITIOUS)) 2386 return; 2387 2388 TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) { 2389 pmap_t oldpmap = pmap_install(pv->pv_pmap); 2390 struct ia64_lpte *pte = pmap_find_vhpt(pv->pv_va); 2391 if (pte->pte_a) { 2392 pte->pte_a = 0; 2393 pmap_update_vhpt(pte, pv->pv_va); 2394 pmap_invalidate_page(pv->pv_pmap, pv->pv_va); 2395 } 2396 pmap_install(oldpmap); 2397 } 2398} 2399 2400/* 2401 * Miscellaneous support routines follow 2402 */ 2403 2404static void 2405ia64_protection_init() 2406{ 2407 int prot, *kp, *up; 2408 2409 kp = protection_codes[0]; 2410 up = protection_codes[1]; 2411 2412 for (prot = 0; prot < 8; prot++) { 2413 switch (prot) { 2414 case VM_PROT_NONE | VM_PROT_NONE | VM_PROT_NONE: 2415 *kp++ = (PTE_AR_R << 2) | PTE_PL_KERN; 2416 *up++ = (PTE_AR_R << 2) | PTE_PL_KERN; 2417 break; 2418 2419 case VM_PROT_NONE | VM_PROT_NONE | VM_PROT_EXECUTE: 2420 *kp++ = (PTE_AR_X_RX << 2) | PTE_PL_KERN; 2421 *up++ = (PTE_AR_X_RX << 2) | PTE_PL_USER; 2422 break; 2423 2424 case VM_PROT_NONE | VM_PROT_WRITE | VM_PROT_NONE: 2425 *kp++ = (PTE_AR_RW << 2) | PTE_PL_KERN; 2426 *up++ = (PTE_AR_RW << 2) | PTE_PL_USER; 2427 break; 2428 2429 case VM_PROT_NONE | VM_PROT_WRITE | VM_PROT_EXECUTE: 2430 *kp++ = (PTE_AR_RWX << 2) | PTE_PL_KERN; 2431 *up++ = (PTE_AR_RWX << 2) | PTE_PL_USER; 2432 break; 2433 2434 case VM_PROT_READ | VM_PROT_NONE | VM_PROT_NONE: 2435 *kp++ = (PTE_AR_R << 2) | PTE_PL_KERN; 2436 *up++ = (PTE_AR_R << 2) | PTE_PL_USER; 2437 break; 2438 2439 case VM_PROT_READ | VM_PROT_NONE | VM_PROT_EXECUTE: 2440 *kp++ = (PTE_AR_RX << 2) | PTE_PL_KERN; 2441 *up++ = (PTE_AR_RX << 2) | PTE_PL_USER; 2442 break; 2443 2444 case VM_PROT_READ | VM_PROT_WRITE | VM_PROT_NONE: 2445 *kp++ = (PTE_AR_RW << 2) | PTE_PL_KERN; 2446 *up++ = (PTE_AR_RW << 2) | PTE_PL_USER; 2447 break; 2448 2449 case VM_PROT_READ | VM_PROT_WRITE | VM_PROT_EXECUTE: 2450 *kp++ = (PTE_AR_RWX << 2) | PTE_PL_KERN; 2451 *up++ = (PTE_AR_RWX << 2) | PTE_PL_USER; 2452 break; 2453 } 2454 } 2455} 2456 2457/* 2458 * Map a set of physical memory pages into the kernel virtual 2459 * address space. Return a pointer to where it is mapped. This 2460 * routine is intended to be used for mapping device memory, 2461 * NOT real memory. 2462 */ 2463void * 2464pmap_mapdev(vm_offset_t pa, vm_size_t size) 2465{ 2466 return (void*) IA64_PHYS_TO_RR6(pa); 2467} 2468 2469/* 2470 * 'Unmap' a range mapped by pmap_mapdev(). 2471 */ 2472void 2473pmap_unmapdev(vm_offset_t va, vm_size_t size) 2474{ 2475 return; 2476} 2477 2478/* 2479 * perform the pmap work for mincore 2480 */ 2481int 2482pmap_mincore(pmap_t pmap, vm_offset_t addr) 2483{ 2484 pmap_t oldpmap; 2485 struct ia64_lpte *pte; 2486 int val = 0; 2487 2488 oldpmap = pmap_install(pmap); 2489 pte = pmap_find_vhpt(addr); 2490 pmap_install(oldpmap); 2491 2492 if (!pte) 2493 return 0; 2494 2495 if (pmap_pte_v(pte)) { 2496 vm_page_t m; 2497 vm_offset_t pa; 2498 2499 val = MINCORE_INCORE; 2500 if ((pte->pte_ig & PTE_IG_MANAGED) == 0) 2501 return val; 2502 2503 pa = pmap_pte_pa(pte); 2504 2505 m = PHYS_TO_VM_PAGE(pa); 2506 2507 /* 2508 * Modified by us 2509 */ 2510 if (pte->pte_d) 2511 val |= MINCORE_MODIFIED|MINCORE_MODIFIED_OTHER; 2512 /* 2513 * Modified by someone 2514 */ 2515 else if (pmap_is_modified(m)) 2516 val |= MINCORE_MODIFIED_OTHER; 2517 /* 2518 * Referenced by us 2519 */ 2520 if (pte->pte_a) 2521 val |= MINCORE_REFERENCED|MINCORE_REFERENCED_OTHER; 2522 2523 /* 2524 * Referenced by someone 2525 */ 2526 else if (pmap_ts_referenced(m)) { 2527 val |= MINCORE_REFERENCED_OTHER; 2528 vm_page_flag_set(m, PG_REFERENCED); 2529 } 2530 } 2531 return val; 2532} 2533 2534void 2535pmap_activate(struct thread *td) 2536{ 2537 pmap_install(vmspace_pmap(td->td_proc->p_vmspace)); 2538} 2539 2540pmap_t 2541pmap_install(pmap_t pmap) 2542{ 2543 pmap_t oldpmap; 2544 int i; 2545 2546 oldpmap = PCPU_GET(current_pmap); 2547 2548 if (pmap == oldpmap || pmap == kernel_pmap) 2549 return pmap; 2550 2551 PCPU_SET(current_pmap, pmap); 2552 if (!pmap) { 2553 /* 2554 * RIDs 0..4 have no mappings to make sure we generate 2555 * page faults on accesses. 2556 */ 2557 ia64_set_rr(IA64_RR_BASE(0), (0 << 8)|(PAGE_SHIFT << 2)|1); 2558 ia64_set_rr(IA64_RR_BASE(1), (1 << 8)|(PAGE_SHIFT << 2)|1); 2559 ia64_set_rr(IA64_RR_BASE(2), (2 << 8)|(PAGE_SHIFT << 2)|1); 2560 ia64_set_rr(IA64_RR_BASE(3), (3 << 8)|(PAGE_SHIFT << 2)|1); 2561 ia64_set_rr(IA64_RR_BASE(4), (4 << 8)|(PAGE_SHIFT << 2)|1); 2562 return oldpmap; 2563 } 2564 2565 pmap->pm_active = 1; /* XXX use bitmap for SMP */ 2566 2567 for (i = 0; i < 5; i++) 2568 ia64_set_rr(IA64_RR_BASE(i), 2569 (pmap->pm_rid[i] << 8)|(PAGE_SHIFT << 2)|1); 2570 2571 return oldpmap; 2572} 2573 2574vm_offset_t 2575pmap_addr_hint(vm_object_t obj, vm_offset_t addr, vm_size_t size) 2576{ 2577 2578 return addr; 2579} 2580 2581#include "opt_ddb.h" 2582 2583#ifdef DDB 2584 2585#include <ddb/ddb.h> 2586 2587static const char* psnames[] = { 2588 "1B", "2B", "4B", "8B", 2589 "16B", "32B", "64B", "128B", 2590 "256B", "512B", "1K", "2K", 2591 "4K", "8K", "16K", "32K", 2592 "64K", "128K", "256K", "512K", 2593 "1M", "2M", "4M", "8M", 2594 "16M", "32M", "64M", "128M", 2595 "256M", "512M", "1G", "2G" 2596}; 2597 2598static void 2599print_trs(int type) 2600{ 2601 struct ia64_pal_result res; 2602 int i, maxtr; 2603 struct { 2604 struct ia64_pte pte; 2605 struct ia64_itir itir; 2606 struct ia64_ifa ifa; 2607 struct ia64_rr rr; 2608 } buf; 2609 static const char* manames[] = { 2610 "WB", "bad", "bad", "bad", 2611 "UC", "UCE", "WC", "NaT", 2612 2613 }; 2614 2615 res = ia64_call_pal_static(PAL_VM_SUMMARY, 0, 0, 0); 2616 if (res.pal_status != 0) { 2617 db_printf("Can't get VM summary\n"); 2618 return; 2619 } 2620 2621 if (type == 0) 2622 maxtr = (res.pal_result[0] >> 40) & 0xff; 2623 else 2624 maxtr = (res.pal_result[0] >> 32) & 0xff; 2625 2626 db_printf("V RID Virtual Page Physical Page PgSz ED AR PL D A MA P KEY\n"); 2627 for (i = 0; i <= maxtr; i++) { 2628 bzero(&buf, sizeof(buf)); 2629 res = ia64_call_pal_stacked_physical 2630 (PAL_VM_TR_READ, i, type, ia64_tpa((u_int64_t) &buf)); 2631 if (!(res.pal_result[0] & 1)) 2632 buf.pte.pte_ar = 0; 2633 if (!(res.pal_result[0] & 2)) 2634 buf.pte.pte_pl = 0; 2635 if (!(res.pal_result[0] & 4)) 2636 buf.pte.pte_d = 0; 2637 if (!(res.pal_result[0] & 8)) 2638 buf.pte.pte_ma = 0; 2639 db_printf( 2640 "%d %06x %013lx %013lx %4s %d %d %d %d %d %-3s %d %06x\n", 2641 buf.ifa.ifa_ig & 1, 2642 buf.rr.rr_rid, 2643 buf.ifa.ifa_vpn, 2644 buf.pte.pte_ppn, 2645 psnames[buf.itir.itir_ps], 2646 buf.pte.pte_ed, 2647 buf.pte.pte_ar, 2648 buf.pte.pte_pl, 2649 buf.pte.pte_d, 2650 buf.pte.pte_a, 2651 manames[buf.pte.pte_ma], 2652 buf.pte.pte_p, 2653 buf.itir.itir_key); 2654 } 2655} 2656 2657DB_COMMAND(itr, db_itr) 2658{ 2659 print_trs(0); 2660} 2661 2662DB_COMMAND(dtr, db_dtr) 2663{ 2664 print_trs(1); 2665} 2666 2667DB_COMMAND(rr, db_rr) 2668{ 2669 int i; 2670 u_int64_t t; 2671 struct ia64_rr rr; 2672 2673 printf("RR RID PgSz VE\n"); 2674 for (i = 0; i < 8; i++) { 2675 __asm __volatile ("mov %0=rr[%1]" 2676 : "=r"(t) 2677 : "r"(IA64_RR_BASE(i))); 2678 *(u_int64_t *) &rr = t; 2679 printf("%d %06x %4s %d\n", 2680 i, rr.rr_rid, psnames[rr.rr_ps], rr.rr_ve); 2681 } 2682} 2683 2684DB_COMMAND(thash, db_thash) 2685{ 2686 if (!have_addr) 2687 return; 2688 2689 db_printf("%p\n", (void *) ia64_thash(addr)); 2690} 2691 2692DB_COMMAND(ttag, db_ttag) 2693{ 2694 if (!have_addr) 2695 return; 2696 2697 db_printf("0x%lx\n", ia64_ttag(addr)); 2698} 2699 2700#endif 2701